2022-09-03T05:37:18.414 INFO:root:teuthology version: 1.1.1.dev418+gb58f2c18 2022-09-03T05:37:18.417 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --archive /home/teuthworker/archive/cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi/7008075 --owner scheduled_cbodley@teuthology --name cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi --description rgw/multisite/{clusters frontend/beast ignore-pg-availability notify omap_limits overrides realms/three-zone-plus-pubsub supported-random-distro$/{centos_8} tasks/test_multi} -- /home/teuthworker/archive/cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi/7008075/orig.config.yaml 2022-09-03T05:37:18.480 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2022-09-03T05:37:18.565 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi/7008075 branch: pr-47848 description: rgw/multisite/{clusters frontend/beast ignore-pg-availability notify omap_limits overrides realms/three-zone-plus-pubsub supported-random-distro$/{centos_8} tasks/test_multi} email: null first_in_suite: false job_id: '7008075' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 8.stream overrides: admin_socket: branch: pr-47848 ceph: conf: client: debug rgw: 20 debug rgw sync: 20 rgw crypt require ssl: false rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= rgw curl low speed time: 300 rgw data log num shards: 4 rgw md log max shards: 4 rgw sync data inject err probability: 0.1 rgw sync log trim interval: 0 rgw sync meta inject err probability: 0.1 rgw sync obj etag verify: true setgroup: ceph setuser: ceph client.0: rgw data notify interval msec: 0 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 rocksdb delete range threshold: 0 osd.0: osd_max_omap_entries_per_request: 10 osd.1: osd_max_omap_entries_per_request: 1000 osd.2: osd_max_omap_entries_per_request: 10000 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(PG_AVAILABILITY\) - \(PG_DEGRADED\) sha1: afd422385fab8119a539d289dd1aeaad4c056fe4 wait-for-scrub: false ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: flavor: default sha1: afd422385fab8119a539d289dd1aeaad4c056fe4 rgw: compression type: random frontend: beast rgw-multisite: realm: is default: true name: test-realm zonegroups: - enabled_features: - resharding endpoints: - c1.client.0 is_default: true is_master: true name: test-zonegroup zones: - endpoints: - c1.client.0 is_default: true is_master: true name: test-zone1 - endpoints: - c2.client.0 is_default: true name: test-zone2 - endpoints: - c1.client.1 name: test-zone3 - endpoints: - c2.client.1 is_pubsub: true name: test-zone4 rgw-multisite-tests: args: - tests.py - tests_ps.py selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-qa-rgw-s3select-fixes_related_to_new_csv_parser sha1: e20bde8244d2a27c6431acd07e9967aaa8f04f01 owner: scheduled_cbodley@teuthology priority: 75 repo: git://git.ceph.com/ceph-ci.git roles: - - c1.mon.a - c1.mgr.x - c1.osd.0 - c1.osd.1 - c1.osd.2 - c1.client.0 - c1.client.1 - - c2.mon.a - c2.mgr.x - c2.osd.0 - c2.osd.1 - c2.osd.2 - c2.client.0 - c2.client.1 sha1: afd422385fab8119a539d289dd1aeaad4c056fe4 sleep_before_teardown: 0 suite: rgw suite_branch: wip-qa-rgw-s3select-fixes_related_to_new_csv_parser suite_path: /home/teuthworker/src/github.com_cbodley_ceph_e20bde8244d2a27c6431acd07e9967aaa8f04f01/qa suite_relpath: qa suite_repo: https://github.com/cbodley/ceph.git suite_sha1: e20bde8244d2a27c6431acd07e9967aaa8f04f01 targets: smithi062.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSHRk+FEtEk0j2RQdYKdyMR/5SvhQ+BPYn0gwmkL5EAzXE5undqYEXihmsS0eF0BPoaxbe6Iw7OEnb3csdEOS1E5PFlWUcn1eJII8vdIiboGKEUVStcXyfSIi5xvHeLzDMmy0Oe2yj4J+mlHSpkntH9oDX8tjTwNf4wPzVI2hk9WWyB1ZDrdWZxtO5QjkecSTp10ZPZS6tbWJtxAFSEiZl5lMo/gi6fnRnD4uHf2xHMQa87pUknbtbobXHed/NFAKM+a/iBBNy7kdtQsFOjpbK02N+cHuYdAOrd4lTuZK3LPuTnBuw9CXESr03blNlAPK8nz/0mdmAysyIidk00OUflY0oN0aY9qgiyQ/f8jElda8reLmrqq1kFG9P7p1roLg+CBgSS2ytsV8nyLHhA0HJHuGdz9+7hOgfdpWsN2wBRez34hFLzoYhCzdd9HycuDp9dK5A3VaTzQ9owbXSfF/1MIFnqGpeUrtUl3//kuN5JBR03Tu31Ahq93aoEzcRFkE= smithi087.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA2MfLiBfBB1rQjxhGdqrEUq/1La6/wNGaBQ7PWfp8JYfQ54QThAn+gKdHG9f6dimygPtacq48uajMWnuqetlFwFvkL0kiK0e6dWMTVvD58jmrP8ymFkir0uO+4/i+CGGxFAEbuqtCah38yHkyefQ08/UBmZBsQqcN87vAJIBLiF3JjPTrF5nx8L4s09mXEcOMA7TkUxQhCOcIOMzmFF0NNx0dIT9+FGCmHCxnoI3xj4s+o94e2OFcdwXHBmBqGKoNHLpcLsVpIbScZyLkVmLamfLkrmIC7I5bKZO/OxIk7Yhv/QLRGf6+0wdnh0kQ6ZlRmhXbk9hqQ7rv+g6P+0GyAkIgpDw99T5eoGm8GiGB7ERNuwHiUS5zV+HZ0XcxC8AKa0uH58JrSJmM2PvQLnVhjLMjwPWNXtTymDoaSRHDf+cMK0XfQc1QOBvl26wZ/KIYXD+3H6ekFX62KG2d9hPZ2QQf2JWJZUDCzL3A9+W9ortahXotXOx4rzG+iyxI+DU= tasks: - install: null - ceph: cluster: c1 - ceph: cluster: c2 - rgw: c1.client.0: port: 8000 c1.client.1: port: 8001 c2.client.0: port: 8000 c2.client.1: port: 8001 - rgw-multisite: null - rgw-multisite-tests: config: reconfigure_delay: 60 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: b58f2c18636eb10faa77ed3614abd00cb85dfc2c timestamp: 2022-09-02_14:33:28 tube: smithi user: cbodley verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.1713 2022-09-03T05:37:18.566 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_cbodley_ceph_e20bde8244d2a27c6431acd07e9967aaa8f04f01/qa; will attempt to use it 2022-09-03T05:37:18.567 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_cbodley_ceph_e20bde8244d2a27c6431acd07e9967aaa8f04f01/qa/tasks 2022-09-03T05:37:18.568 INFO:teuthology.run_tasks:Running task internal.check_packages... 2022-09-03T05:37:18.570 INFO:teuthology.task.internal:Checking packages... 2022-09-03T05:37:18.605 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'afd422385fab8119a539d289dd1aeaad4c056fe4' 2022-09-03T05:37:18.605 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-09-03T05:37:18.606 INFO:teuthology.packaging:ref: None 2022-09-03T05:37:18.607 INFO:teuthology.packaging:tag: None 2022-09-03T05:37:18.607 INFO:teuthology.packaging:branch: pr-47848 2022-09-03T05:37:18.608 INFO:teuthology.packaging:sha1: afd422385fab8119a539d289dd1aeaad4c056fe4 2022-09-03T05:37:18.608 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pr-47848 2022-09-03T05:37:18.776 INFO:teuthology.task.internal:Found packages for ceph version 17.0.0-14689.gafd42238 2022-09-03T05:37:18.776 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2022-09-03T05:37:18.784 INFO:teuthology.task.internal:no buildpackages task found 2022-09-03T05:37:18.785 INFO:teuthology.run_tasks:Running task internal.save_config... 2022-09-03T05:37:18.821 INFO:teuthology.task.internal:Saving configuration 2022-09-03T05:37:18.855 INFO:teuthology.run_tasks:Running task internal.check_lock... 2022-09-03T05:37:18.862 INFO:teuthology.task.internal.check_lock:Checking locks... 2022-09-03T05:37:18.887 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi062.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi/7008075', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-09-03 05:32:08.422756', 'locked_by': 'scheduled_cbodley@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIRiV0iPXGWPnTpi8o0SV8+WgBgZAmDgBvBOGiV3wRySDavoTXXwU+U+g/rJhHGBLljY7shAad0N5sK+vvhi33o='} 2022-09-03T05:37:18.917 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi087.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi/7008075', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-09-03 05:32:08.420371', 'locked_by': 'scheduled_cbodley@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPiYaHfNKos/A8QItE4zeLE4xZOgsHiYSDpMEdeIqKcJLOm/aINo6B/dKxTWrkY3CmBdaUm8dMpsDvar04CFz7o='} 2022-09-03T05:37:18.920 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2022-09-03T05:37:18.930 INFO:teuthology.task.internal:roles: ubuntu@smithi062.front.sepia.ceph.com - ['c1.mon.a', 'c1.mgr.x', 'c1.osd.0', 'c1.osd.1', 'c1.osd.2', 'c1.client.0', 'c1.client.1'] 2022-09-03T05:37:18.930 INFO:teuthology.task.internal:roles: ubuntu@smithi087.front.sepia.ceph.com - ['c2.mon.a', 'c2.mgr.x', 'c2.osd.0', 'c2.osd.1', 'c2.osd.2', 'c2.client.0', 'c2.client.1'] 2022-09-03T05:37:18.931 INFO:teuthology.run_tasks:Running task console_log... 2022-09-03T05:37:19.062 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7eff5c2fc8c8>, signals=[15]) 2022-09-03T05:37:19.064 INFO:teuthology.run_tasks:Running task internal.connect... 2022-09-03T05:37:19.076 INFO:teuthology.task.internal:Opening connections... 2022-09-03T05:37:19.077 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi062.front.sepia.ceph.com 2022-09-03T05:37:19.079 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi062.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-09-03T05:37:19.230 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi087.front.sepia.ceph.com 2022-09-03T05:37:19.232 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-09-03T05:37:19.360 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2022-09-03T05:37:19.370 DEBUG:teuthology.orchestra.run.smithi062:> uname -m 2022-09-03T05:37:19.402 INFO:teuthology.orchestra.run.smithi062.stdout:x86_64 2022-09-03T05:37:19.403 DEBUG:teuthology.orchestra.run.smithi062:> cat /etc/os-release 2022-09-03T05:37:19.467 INFO:teuthology.orchestra.run.smithi062.stdout:NAME="CentOS Stream" 2022-09-03T05:37:19.468 INFO:teuthology.orchestra.run.smithi062.stdout:VERSION="8" 2022-09-03T05:37:19.468 INFO:teuthology.orchestra.run.smithi062.stdout:ID="centos" 2022-09-03T05:37:19.469 INFO:teuthology.orchestra.run.smithi062.stdout:ID_LIKE="rhel fedora" 2022-09-03T05:37:19.469 INFO:teuthology.orchestra.run.smithi062.stdout:VERSION_ID="8" 2022-09-03T05:37:19.470 INFO:teuthology.orchestra.run.smithi062.stdout:PLATFORM_ID="platform:el8" 2022-09-03T05:37:19.470 INFO:teuthology.orchestra.run.smithi062.stdout:PRETTY_NAME="CentOS Stream 8" 2022-09-03T05:37:19.470 INFO:teuthology.orchestra.run.smithi062.stdout:ANSI_COLOR="0;31" 2022-09-03T05:37:19.471 INFO:teuthology.orchestra.run.smithi062.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2022-09-03T05:37:19.471 INFO:teuthology.orchestra.run.smithi062.stdout:HOME_URL="https://centos.org/" 2022-09-03T05:37:19.472 INFO:teuthology.orchestra.run.smithi062.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2022-09-03T05:37:19.472 INFO:teuthology.orchestra.run.smithi062.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2022-09-03T05:37:19.473 INFO:teuthology.orchestra.run.smithi062.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2022-09-03T05:37:19.475 INFO:teuthology.lock.ops:Updating smithi062.front.sepia.ceph.com on lock server 2022-09-03T05:37:19.506 DEBUG:teuthology.orchestra.run.smithi087:> uname -m 2022-09-03T05:37:19.533 INFO:teuthology.orchestra.run.smithi087.stdout:x86_64 2022-09-03T05:37:19.534 DEBUG:teuthology.orchestra.run.smithi087:> cat /etc/os-release 2022-09-03T05:37:19.595 INFO:teuthology.orchestra.run.smithi087.stdout:NAME="CentOS Stream" 2022-09-03T05:37:19.596 INFO:teuthology.orchestra.run.smithi087.stdout:VERSION="8" 2022-09-03T05:37:19.596 INFO:teuthology.orchestra.run.smithi087.stdout:ID="centos" 2022-09-03T05:37:19.596 INFO:teuthology.orchestra.run.smithi087.stdout:ID_LIKE="rhel fedora" 2022-09-03T05:37:19.597 INFO:teuthology.orchestra.run.smithi087.stdout:VERSION_ID="8" 2022-09-03T05:37:19.597 INFO:teuthology.orchestra.run.smithi087.stdout:PLATFORM_ID="platform:el8" 2022-09-03T05:37:19.598 INFO:teuthology.orchestra.run.smithi087.stdout:PRETTY_NAME="CentOS Stream 8" 2022-09-03T05:37:19.598 INFO:teuthology.orchestra.run.smithi087.stdout:ANSI_COLOR="0;31" 2022-09-03T05:37:19.599 INFO:teuthology.orchestra.run.smithi087.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2022-09-03T05:37:19.599 INFO:teuthology.orchestra.run.smithi087.stdout:HOME_URL="https://centos.org/" 2022-09-03T05:37:19.600 INFO:teuthology.orchestra.run.smithi087.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2022-09-03T05:37:19.600 INFO:teuthology.orchestra.run.smithi087.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2022-09-03T05:37:19.601 INFO:teuthology.orchestra.run.smithi087.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2022-09-03T05:37:19.601 INFO:teuthology.lock.ops:Updating smithi087.front.sepia.ceph.com on lock server 2022-09-03T05:37:19.630 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2022-09-03T05:37:19.641 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2022-09-03T05:37:19.673 INFO:teuthology.task.internal:Checking for old test directory... 2022-09-03T05:37:19.673 DEBUG:teuthology.orchestra.run.smithi062:> test '!' -e /home/ubuntu/cephtest 2022-09-03T05:37:19.677 DEBUG:teuthology.orchestra.run.smithi087:> test '!' -e /home/ubuntu/cephtest 2022-09-03T05:37:19.699 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2022-09-03T05:37:19.710 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2022-09-03T05:37:19.711 DEBUG:teuthology.orchestra.run.smithi062:> test -z $(ls -A /var/lib/ceph) 2022-09-03T05:37:19.742 DEBUG:teuthology.orchestra.run.smithi087:> test -z $(ls -A /var/lib/ceph) 2022-09-03T05:37:19.773 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2022-09-03T05:37:19.842 INFO:teuthology.run_tasks:Running task kernel... 2022-09-03T05:37:19.858 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2022-09-03T05:37:19.859 INFO:teuthology.task.kernel:config {'c1.mon.a': {'kdb': True, 'sha1': 'distro'}, 'c2.mon.a': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2022-09-03T05:37:19.859 DEBUG:teuthology.orchestra.run.smithi062:> test -f /run/.containerenv -o -f /.dockerenv 2022-09-03T05:37:19.860 DEBUG:teuthology.orchestra.run.smithi087:> test -f /run/.containerenv -o -f /.dockerenv 2022-09-03T05:37:19.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T05:37:19.881 DEBUG:teuthology.orchestra.run.smithi087:> uname -r 2022-09-03T05:37:19.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T05:37:19.883 DEBUG:teuthology.orchestra.run.smithi062:> uname -r 2022-09-03T05:37:19.944 INFO:teuthology.orchestra.run.smithi087.stdout:4.18.0-408.el8.x86_64 2022-09-03T05:37:19.944 INFO:teuthology.task.kernel:Running kernel on smithi087: 4.18.0-408.el8.x86_64 2022-09-03T05:37:19.945 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum install -y kernel 2022-09-03T05:37:19.949 INFO:teuthology.orchestra.run.smithi062.stdout:4.18.0-408.el8.x86_64 2022-09-03T05:37:19.950 INFO:teuthology.task.kernel:Running kernel on smithi062: 4.18.0-408.el8.x86_64 2022-09-03T05:37:19.950 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum install -y kernel 2022-09-03T05:37:23.189 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-8 - AppStream 30 MB/s | 25 MB 00:00 2022-09-03T05:37:23.287 INFO:teuthology.orchestra.run.smithi062.stdout:CentOS-8 - AppStream 26 MB/s | 25 MB 00:00 2022-09-03T05:37:28.129 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-8 - BaseOS 43 MB/s | 25 MB 00:00 2022-09-03T05:37:28.549 INFO:teuthology.orchestra.run.smithi062.stdout:CentOS-8 - BaseOS 43 MB/s | 25 MB 00:00 2022-09-03T05:37:31.895 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-8 - Extras 148 kB/s | 18 kB 00:00 2022-09-03T05:37:32.299 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-8 - PowerTools 17 MB/s | 5.0 MB 00:00 2022-09-03T05:37:32.569 INFO:teuthology.orchestra.run.smithi062.stdout:CentOS-8 - Extras 181 kB/s | 18 kB 00:00 2022-09-03T05:37:32.946 INFO:teuthology.orchestra.run.smithi062.stdout:CentOS-8 - PowerTools 20 MB/s | 5.0 MB 00:00 2022-09-03T05:37:33.488 INFO:teuthology.orchestra.run.smithi087.stdout:Copr repo for python3-asyncssh owned by ceph 18 kB/s | 3.5 kB 00:00 2022-09-03T05:37:34.140 INFO:teuthology.orchestra.run.smithi062.stdout:Copr repo for python3-asyncssh owned by ceph 20 kB/s | 3.5 kB 00:00 2022-09-03T05:37:34.573 INFO:teuthology.orchestra.run.smithi087.stdout:Extra Packages for Enterprise Linux 13 MB/s | 13 MB 00:00 2022-09-03T05:37:35.271 INFO:teuthology.orchestra.run.smithi062.stdout:Extra Packages for Enterprise Linux 13 MB/s | 13 MB 00:01 2022-09-03T05:37:37.547 INFO:teuthology.orchestra.run.smithi087.stdout:lab-extras 602 kB/s | 24 kB 00:00 2022-09-03T05:37:38.513 INFO:teuthology.orchestra.run.smithi062.stdout:lab-extras 492 kB/s | 24 kB 00:00 2022-09-03T05:37:40.378 INFO:teuthology.orchestra.run.smithi087.stdout:Package kernel-4.18.0-408.el8.x86_64 is already installed. 2022-09-03T05:37:40.379 INFO:teuthology.orchestra.run.smithi087.stdout:Package kernel-4.18.0-305.3.1.el8.x86_64 is already installed. 2022-09-03T05:37:40.511 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:37:40.512 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:37:40.513 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:37:40.621 DEBUG:teuthology.orchestra.run.smithi087:> echo no | sudo yum reinstall kernel || true 2022-09-03T05:37:41.189 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:00:04 ago on Sat 03 Sep 2022 05:37:37 AM UTC. 2022-09-03T05:37:41.520 INFO:teuthology.orchestra.run.smithi062.stdout:Package kernel-4.18.0-408.el8.x86_64 is already installed. 2022-09-03T05:37:41.520 INFO:teuthology.orchestra.run.smithi062.stdout:Package kernel-4.18.0-305.3.1.el8.x86_64 is already installed. 2022-09-03T05:37:41.653 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:37:41.654 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:37:41.654 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:37:41.778 DEBUG:teuthology.orchestra.run.smithi062:> echo no | sudo yum reinstall kernel || true 2022-09-03T05:37:42.188 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:37:42.190 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:37:42.191 INFO:teuthology.orchestra.run.smithi087.stdout: Package Architecture Version Repository Size 2022-09-03T05:37:42.191 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:37:42.192 INFO:teuthology.orchestra.run.smithi087.stdout:Reinstalling: 2022-09-03T05:37:42.192 INFO:teuthology.orchestra.run.smithi087.stdout: kernel x86_64 4.18.0-408.el8 CentOS-BaseOS 8.5 M 2022-09-03T05:37:42.193 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:37:42.193 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:37:42.194 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:37:42.194 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:37:42.194 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 8.5 M 2022-09-03T05:37:42.194 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 0 2022-09-03T05:37:42.195 INFO:teuthology.orchestra.run.smithi087.stderr:Operation aborted. 2022-09-03T05:37:42.251 INFO:teuthology.orchestra.run.smithi087.stdout:Is this ok [y/N]: 2022-09-03T05:37:42.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum reinstall -y kernel || true 2022-09-03T05:37:42.414 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:00:04 ago on Sat 03 Sep 2022 05:37:38 AM UTC. 2022-09-03T05:37:42.813 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:00:05 ago on Sat 03 Sep 2022 05:37:37 AM UTC. 2022-09-03T05:37:43.514 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:37:43.515 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:37:43.516 INFO:teuthology.orchestra.run.smithi062.stdout: Package Architecture Version Repository Size 2022-09-03T05:37:43.516 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:37:43.516 INFO:teuthology.orchestra.run.smithi062.stdout:Reinstalling: 2022-09-03T05:37:43.516 INFO:teuthology.orchestra.run.smithi062.stdout: kernel x86_64 4.18.0-408.el8 CentOS-BaseOS 8.5 M 2022-09-03T05:37:43.517 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:37:43.517 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:37:43.517 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:37:43.517 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:37:43.518 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 8.5 M 2022-09-03T05:37:43.518 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 0 2022-09-03T05:37:43.529 INFO:teuthology.orchestra.run.smithi062.stderr:Operation aborted. 2022-09-03T05:37:43.578 INFO:teuthology.orchestra.run.smithi062.stdout:Is this ok [y/N]: 2022-09-03T05:37:43.578 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum reinstall -y kernel || true 2022-09-03T05:37:43.810 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:37:43.811 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:37:43.811 INFO:teuthology.orchestra.run.smithi087.stdout: Package Architecture Version Repository Size 2022-09-03T05:37:43.811 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:37:43.812 INFO:teuthology.orchestra.run.smithi087.stdout:Reinstalling: 2022-09-03T05:37:43.812 INFO:teuthology.orchestra.run.smithi087.stdout: kernel x86_64 4.18.0-408.el8 CentOS-BaseOS 8.5 M 2022-09-03T05:37:43.812 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:37:43.812 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:37:43.813 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:37:43.813 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:37:43.813 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 8.5 M 2022-09-03T05:37:43.813 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 0 2022-09-03T05:37:43.813 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:37:44.015 INFO:teuthology.orchestra.run.smithi087.stdout:kernel-4.18.0-408.el8.x86_64.rpm 42 MB/s | 8.5 MB 00:00 2022-09-03T05:37:44.016 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:37:44.016 INFO:teuthology.orchestra.run.smithi087.stdout:Total 42 MB/s | 8.5 MB 00:00 2022-09-03T05:37:44.114 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:37:44.171 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:00:06 ago on Sat 03 Sep 2022 05:37:38 AM UTC. 2022-09-03T05:37:44.481 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:37:44.481 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:37:44.654 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:37:44.662 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:37:44.957 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:37:45.138 INFO:teuthology.orchestra.run.smithi087.stdout: Reinstalling : kernel-4.18.0-408.el8.x86_64 1/2 2022-09-03T05:37:45.139 INFO:teuthology.orchestra.run.smithi087.stdout: Cleanup : kernel-4.18.0-408.el8.x86_64 2/2 2022-09-03T05:37:45.244 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:37:45.245 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:37:45.245 INFO:teuthology.orchestra.run.smithi062.stdout: Package Architecture Version Repository Size 2022-09-03T05:37:45.245 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:37:45.246 INFO:teuthology.orchestra.run.smithi062.stdout:Reinstalling: 2022-09-03T05:37:45.246 INFO:teuthology.orchestra.run.smithi062.stdout: kernel x86_64 4.18.0-408.el8 CentOS-BaseOS 8.5 M 2022-09-03T05:37:45.247 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:37:45.247 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:37:45.248 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:37:45.248 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:37:45.249 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 8.5 M 2022-09-03T05:37:45.250 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 0 2022-09-03T05:37:45.250 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:37:45.448 INFO:teuthology.orchestra.run.smithi062.stdout:kernel-4.18.0-408.el8.x86_64.rpm 42 MB/s | 8.5 MB 00:00 2022-09-03T05:37:45.449 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:37:45.449 INFO:teuthology.orchestra.run.smithi062.stdout:Total 42 MB/s | 8.5 MB 00:00 2022-09-03T05:37:45.537 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-4.18.0-408.el8.x86_64 2/2 2022-09-03T05:37:45.538 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-4.18.0-408.el8.x86_64 1/2 2022-09-03T05:37:45.574 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:37:45.965 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:37:45.966 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:37:46.051 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-4.18.0-408.el8.x86_64 2/2 2022-09-03T05:37:46.052 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:37:46.052 INFO:teuthology.orchestra.run.smithi087.stdout:Reinstalled: 2022-09-03T05:37:46.053 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-4.18.0-408.el8.x86_64 2022-09-03T05:37:46.054 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:37:46.055 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:37:46.147 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:37:46.152 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:37:46.197 DEBUG:teuthology.orchestra.run.smithi087:> rpm -q kernel --last | head -n 1 2022-09-03T05:37:46.323 INFO:teuthology.orchestra.run.smithi087.stdout:kernel-4.18.0-408.el8.x86_64 Sat 03 Sep 2022 05:37:44 AM UTC 2022-09-03T05:37:46.324 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-408.el8.x86_64 2022-09-03T05:37:46.325 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2022-09-03T05:37:46.325 INFO:teuthology.task.kernel:Enabling kdb on c2.mon.a... 2022-09-03T05:37:46.325 DEBUG:teuthology.orchestra.run.smithi087:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-09-03T05:37:46.372 INFO:teuthology.orchestra.run.smithi087.stdout:ttyS1 2022-09-03T05:37:46.393 DEBUG:teuthology.parallel:result is None 2022-09-03T05:37:46.461 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:37:46.687 INFO:teuthology.orchestra.run.smithi062.stdout: Reinstalling : kernel-4.18.0-408.el8.x86_64 1/2 2022-09-03T05:37:46.688 INFO:teuthology.orchestra.run.smithi062.stdout: Cleanup : kernel-4.18.0-408.el8.x86_64 2/2 2022-09-03T05:37:47.063 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: kernel-4.18.0-408.el8.x86_64 2/2 2022-09-03T05:37:47.063 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : kernel-4.18.0-408.el8.x86_64 1/2 2022-09-03T05:37:47.560 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : kernel-4.18.0-408.el8.x86_64 2/2 2022-09-03T05:37:47.561 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:37:47.561 INFO:teuthology.orchestra.run.smithi062.stdout:Reinstalled: 2022-09-03T05:37:47.561 INFO:teuthology.orchestra.run.smithi062.stdout: kernel-4.18.0-408.el8.x86_64 2022-09-03T05:37:47.561 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:37:47.562 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:37:47.646 DEBUG:teuthology.orchestra.run.smithi062:> rpm -q kernel --last | head -n 1 2022-09-03T05:37:47.792 INFO:teuthology.orchestra.run.smithi062.stdout:kernel-4.18.0-408.el8.x86_64 Sat 03 Sep 2022 05:37:46 AM UTC 2022-09-03T05:37:47.794 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-408.el8.x86_64 2022-09-03T05:37:47.794 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2022-09-03T05:37:47.795 INFO:teuthology.task.kernel:Enabling kdb on c1.mon.a... 2022-09-03T05:37:47.796 DEBUG:teuthology.orchestra.run.smithi062:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-09-03T05:37:47.853 INFO:teuthology.orchestra.run.smithi062.stdout:ttyS1 2022-09-03T05:37:47.867 DEBUG:teuthology.parallel:result is None 2022-09-03T05:37:47.868 INFO:teuthology.run_tasks:Running task internal.base... 2022-09-03T05:37:47.879 INFO:teuthology.task.internal:Creating test directory... 2022-09-03T05:37:47.880 DEBUG:teuthology.orchestra.run.smithi062:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-09-03T05:37:47.886 DEBUG:teuthology.orchestra.run.smithi087:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-09-03T05:37:47.910 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2022-09-03T05:37:47.927 INFO:teuthology.run_tasks:Running task internal.archive... 2022-09-03T05:37:47.953 INFO:teuthology.task.internal:Creating archive directory... 2022-09-03T05:37:47.953 DEBUG:teuthology.orchestra.run.smithi062:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-09-03T05:37:47.962 DEBUG:teuthology.orchestra.run.smithi087:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-09-03T05:37:48.010 INFO:teuthology.run_tasks:Running task internal.coredump... 2022-09-03T05:37:48.029 INFO:teuthology.task.internal:Enabling coredump saving... 2022-09-03T05:37:48.030 DEBUG:teuthology.orchestra.run.smithi062:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-09-03T05:37:48.034 DEBUG:teuthology.orchestra.run.smithi087:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-09-03T05:37:48.065 INFO:teuthology.orchestra.run.smithi062.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-09-03T05:37:48.075 INFO:teuthology.orchestra.run.smithi062.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-09-03T05:37:48.079 INFO:teuthology.orchestra.run.smithi087.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-09-03T05:37:48.090 INFO:teuthology.orchestra.run.smithi087.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-09-03T05:37:48.092 INFO:teuthology.run_tasks:Running task internal.sudo... 2022-09-03T05:37:48.101 INFO:teuthology.task.internal:Configuring sudo... 2022-09-03T05:37:48.101 DEBUG:teuthology.orchestra.run.smithi062:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-09-03T05:37:48.123 DEBUG:teuthology.orchestra.run.smithi087:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-09-03T05:37:48.162 INFO:teuthology.run_tasks:Running task internal.syslog... 2022-09-03T05:37:48.173 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2022-09-03T05:37:48.173 DEBUG:teuthology.orchestra.run.smithi062:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-09-03T05:37:48.191 DEBUG:teuthology.orchestra.run.smithi087:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-09-03T05:37:48.221 DEBUG:teuthology.orchestra.run.smithi062:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-09-03T05:37:48.314 DEBUG:teuthology.orchestra.run.smithi062:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2022-09-03T05:37:48.373 DEBUG:teuthology.orchestra.run.smithi062:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-09-03T05:37:48.427 DEBUG:teuthology.orchestra.run.smithi062:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2022-09-03T05:37:48.462 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:37:48.463 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-09-03T05:37:48.530 DEBUG:teuthology.orchestra.run.smithi087:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-09-03T05:37:48.577 DEBUG:teuthology.orchestra.run.smithi087:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2022-09-03T05:37:48.629 DEBUG:teuthology.orchestra.run.smithi087:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-09-03T05:37:48.715 DEBUG:teuthology.orchestra.run.smithi087:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2022-09-03T05:37:48.762 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:37:48.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-09-03T05:37:48.830 DEBUG:teuthology.orchestra.run.smithi062:> sudo service rsyslog restart 2022-09-03T05:37:48.833 DEBUG:teuthology.orchestra.run.smithi087:> sudo service rsyslog restart 2022-09-03T05:37:48.886 INFO:teuthology.orchestra.run.smithi062.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2022-09-03T05:37:48.931 INFO:teuthology.orchestra.run.smithi087.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2022-09-03T05:37:49.351 INFO:teuthology.run_tasks:Running task internal.timer... 2022-09-03T05:37:49.362 INFO:teuthology.task.internal:Starting timer... 2022-09-03T05:37:49.363 INFO:teuthology.run_tasks:Running task pcp... 2022-09-03T05:37:49.414 INFO:teuthology.run_tasks:Running task selinux... 2022-09-03T05:37:49.445 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2022-09-03T05:37:49.446 DEBUG:teuthology.orchestra.run.smithi062:> sudo service auditd rotate 2022-09-03T05:37:49.557 INFO:teuthology.orchestra.run.smithi062.stdout:Rotating logs: 2022-09-03T05:37:49.559 DEBUG:teuthology.orchestra.run.smithi087:> sudo service auditd rotate 2022-09-03T05:37:49.631 INFO:teuthology.orchestra.run.smithi087.stdout:Rotating logs: 2022-09-03T05:37:49.633 DEBUG:teuthology.task.selinux:Getting current SELinux state 2022-09-03T05:37:49.633 DEBUG:teuthology.orchestra.run.smithi062:> /usr/sbin/getenforce 2022-09-03T05:37:49.666 INFO:teuthology.orchestra.run.smithi062.stdout:Permissive 2022-09-03T05:37:49.667 DEBUG:teuthology.orchestra.run.smithi087:> /usr/sbin/getenforce 2022-09-03T05:37:49.704 INFO:teuthology.orchestra.run.smithi087.stdout:Permissive 2022-09-03T05:37:49.705 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi062.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi087.front.sepia.ceph.com': 'permissive'} 2022-09-03T05:37:49.706 DEBUG:teuthology.orchestra.run.smithi062:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|comm="sssd"\|comm="sss_cache"\|context=system_u:system_r:NetworkManager_dispatcher_t:s0\)' 2022-09-03T05:37:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T05:37:49.736 DEBUG:teuthology.orchestra.run.smithi087:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|comm="sssd"\|comm="sss_cache"\|context=system_u:system_r:NetworkManager_dispatcher_t:s0\)' 2022-09-03T05:37:49.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T05:37:49.777 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2022-09-03T05:37:49.778 DEBUG:teuthology.orchestra.run.smithi062:> sudo /usr/sbin/setenforce permissive 2022-09-03T05:37:49.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo /usr/sbin/setenforce permissive 2022-09-03T05:37:49.846 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2022-09-03T05:37:49.863 DEBUG:teuthology.repo_utils:Setting repo remote to git://git.ceph.com/git/ceph-cm-ansible.git 2022-09-03T05:37:49.871 INFO:teuthology.repo_utils:Fetching main from origin 2022-09-03T05:37:49.918 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_main to origin/main 2022-09-03T05:37:49.958 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2022-09-03T05:37:49.963 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi062.front.sepia.ceph.com,smithi087.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_main/cephlab.yml 2022-09-03T05:44:18.479 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi062.front.sepia.ceph.com'), Remote(name='ubuntu@smithi087.front.sepia.ceph.com')] 2022-09-03T05:44:18.482 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi062.front.sepia.ceph.com' 2022-09-03T05:44:18.485 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi062.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-09-03T05:44:18.572 DEBUG:teuthology.orchestra.run.smithi062:> true 2022-09-03T05:44:18.642 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2022-09-03T05:44:18.643 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-09-03T05:44:18.756 DEBUG:teuthology.orchestra.run.smithi087:> true 2022-09-03T05:44:18.832 INFO:teuthology.run_tasks:Running task clock... 2022-09-03T05:44:18.847 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2022-09-03T05:44:18.848 INFO:teuthology.orchestra.run:Running command with timeout 360 2022-09-03T05:44:18.851 DEBUG:teuthology.orchestra.run.smithi062:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2022-09-03T05:44:18.854 INFO:teuthology.orchestra.run:Running command with timeout 360 2022-09-03T05:44:18.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2022-09-03T05:44:18.887 INFO:teuthology.orchestra.run.smithi062.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2022-09-03T05:44:18.902 INFO:teuthology.orchestra.run.smithi062.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2022-09-03T05:44:18.906 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2022-09-03T05:44:18.922 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2022-09-03T05:44:18.974 INFO:teuthology.orchestra.run.smithi062.stderr:sudo: ntpd: command not found 2022-09-03T05:44:18.986 INFO:teuthology.orchestra.run.smithi062.stdout:506 Cannot talk to daemon 2022-09-03T05:44:18.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: ntpd: command not found 2022-09-03T05:44:19.000 INFO:teuthology.orchestra.run.smithi087.stdout:506 Cannot talk to daemon 2022-09-03T05:44:19.001 INFO:teuthology.orchestra.run.smithi062.stderr:Failed to start ntp.service: Unit ntp.service not found. 2022-09-03T05:44:19.013 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to start ntp.service: Unit ntp.service not found. 2022-09-03T05:44:19.016 INFO:teuthology.orchestra.run.smithi062.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2022-09-03T05:44:19.026 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2022-09-03T05:44:19.075 INFO:teuthology.orchestra.run.smithi062.stderr:bash: ntpq: command not found 2022-09-03T05:44:19.080 INFO:teuthology.orchestra.run.smithi062.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2022-09-03T05:44:19.080 INFO:teuthology.orchestra.run.smithi062.stdout:=============================================================================== 2022-09-03T05:44:19.081 INFO:teuthology.orchestra.run.smithi062.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-03T05:44:19.081 INFO:teuthology.orchestra.run.smithi062.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-03T05:44:19.082 INFO:teuthology.orchestra.run.smithi062.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-03T05:44:19.082 INFO:teuthology.orchestra.run.smithi062.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-03T05:44:19.091 INFO:teuthology.orchestra.run.smithi087.stderr:bash: ntpq: command not found 2022-09-03T05:44:19.095 INFO:teuthology.orchestra.run.smithi087.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2022-09-03T05:44:19.095 INFO:teuthology.orchestra.run.smithi087.stdout:=============================================================================== 2022-09-03T05:44:19.095 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-03T05:44:19.095 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-03T05:44:19.096 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-03T05:44:19.096 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2022-09-03T05:44:19.097 INFO:teuthology.run_tasks:Running task install... 2022-09-03T05:44:19.107 DEBUG:teuthology.task.install:project ceph 2022-09-03T05:44:19.108 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4'}} 2022-09-03T05:44:19.109 DEBUG:teuthology.task.install:config {'flavor': 'default', 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4'} 2022-09-03T05:44:19.109 INFO:teuthology.task.install:Using flavor: default 2022-09-03T05:44:19.116 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2022-09-03T05:44:19.117 INFO:teuthology.task.install:extra packages: [] 2022-09-03T05:44:19.118 DEBUG:teuthology.orchestra.run.smithi062:> sudo lsb_release -is 2022-09-03T05:44:19.157 INFO:teuthology.orchestra.run.smithi062.stdout:CentOSStream 2022-09-03T05:44:19.159 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-03T05:44:19.160 DEBUG:teuthology.orchestra.run.smithi087:> sudo lsb_release -is 2022-09-03T05:44:19.161 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4', 'tag': None, 'wait_for_package': False} 2022-09-03T05:44:19.163 DEBUG:teuthology.orchestra.run.smithi062:> sudo lsb_release -is 2022-09-03T05:44:19.199 INFO:teuthology.orchestra.run.smithi087.stdout:CentOSStream 2022-09-03T05:44:19.201 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-03T05:44:19.202 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4', 'tag': None, 'wait_for_package': False} 2022-09-03T05:44:19.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo lsb_release -is 2022-09-03T05:44:19.240 INFO:teuthology.orchestra.run.smithi062.stdout:CentOSStream 2022-09-03T05:44:19.241 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-03T05:44:19.242 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=afd422385fab8119a539d289dd1aeaad4c056fe4 2022-09-03T05:44:19.298 INFO:teuthology.orchestra.run.smithi087.stdout:CentOSStream 2022-09-03T05:44:19.298 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-03T05:44:19.299 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=afd422385fab8119a539d289dd1aeaad4c056fe4 2022-09-03T05:44:19.414 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/pr-47848/afd422385fab8119a539d289dd1aeaad4c056fe4/centos/8/flavors/default/ 2022-09-03T05:44:19.415 INFO:teuthology.task.install.rpm:Package version is 17.0.0-14689.gafd42238 2022-09-03T05:44:19.452 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/pr-47848/afd422385fab8119a539d289dd1aeaad4c056fe4/centos/8/flavors/default/ 2022-09-03T05:44:19.452 INFO:teuthology.task.install.rpm:Package version is 17.0.0-14689.gafd42238 2022-09-03T05:44:19.547 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/pr-47848/afd422385fab8119a539d289dd1aeaad4c056fe4/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/pr-47848/afd422385fab8119a539d289dd1aeaad4c056fe4/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/pr-47848/afd422385fab8119a539d289dd1aeaad4c056fe4/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2022-09-03T05:44:19.547 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:44:19.548 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd of=/etc/yum.repos.d/ceph.repo 2022-09-03T05:44:19.576 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/pr-47848/afd422385fab8119a539d289dd1aeaad4c056fe4/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/pr-47848/afd422385fab8119a539d289dd1aeaad4c056fe4/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/pr-47848/afd422385fab8119a539d289dd1aeaad4c056fe4/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2022-09-03T05:44:19.577 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:44:19.577 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/yum.repos.d/ceph.repo 2022-09-03T05:44:19.580 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2022-09-03T05:44:19.580 DEBUG:teuthology.orchestra.run.smithi062:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/afd422385fab8119a539d289dd1aeaad4c056fe4/;g' /etc/yum.repos.d/ceph.repo ; fi 2022-09-03T05:44:19.608 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2022-09-03T05:44:19.609 DEBUG:teuthology.orchestra.run.smithi087:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/afd422385fab8119a539d289dd1aeaad4c056fe4/;g' /etc/yum.repos.d/ceph.repo ; fi 2022-09-03T05:44:19.649 DEBUG:teuthology.orchestra.run.smithi062:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2022-09-03T05:44:19.679 DEBUG:teuthology.orchestra.run.smithi087:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2022-09-03T05:44:19.729 DEBUG:teuthology.orchestra.run.smithi062:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2022-09-03T05:44:19.758 DEBUG:teuthology.orchestra.run.smithi087:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2022-09-03T05:44:19.796 INFO:teuthology.orchestra.run.smithi062.stdout:check_obsoletes = 1 2022-09-03T05:44:19.798 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum clean all 2022-09-03T05:44:19.825 INFO:teuthology.orchestra.run.smithi087.stdout:check_obsoletes = 1 2022-09-03T05:44:19.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum clean all 2022-09-03T05:44:20.058 INFO:teuthology.orchestra.run.smithi062.stdout:63 files removed 2022-09-03T05:44:20.078 INFO:teuthology.orchestra.run.smithi087.stdout:63 files removed 2022-09-03T05:44:20.083 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-radosgw 2022-09-03T05:44:20.103 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-radosgw 2022-09-03T05:44:20.729 INFO:teuthology.orchestra.run.smithi062.stdout:ceph packages for x86_64 292 kB/s | 80 kB 00:00 2022-09-03T05:44:20.740 INFO:teuthology.orchestra.run.smithi087.stdout:ceph packages for x86_64 276 kB/s | 80 kB 00:00 2022-09-03T05:44:21.013 INFO:teuthology.orchestra.run.smithi062.stdout:ceph noarch packages 38 kB/s | 9.4 kB 00:00 2022-09-03T05:44:21.016 INFO:teuthology.orchestra.run.smithi087.stdout:ceph noarch packages 39 kB/s | 9.4 kB 00:00 2022-09-03T05:44:21.282 INFO:teuthology.orchestra.run.smithi062.stdout:ceph source packages 6.2 kB/s | 1.5 kB 00:00 2022-09-03T05:44:21.283 INFO:teuthology.orchestra.run.smithi087.stdout:ceph source packages 6.2 kB/s | 1.5 kB 00:00 2022-09-03T05:44:21.913 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-8 - AppStream 41 MB/s | 25 MB 00:00 2022-09-03T05:44:22.076 INFO:teuthology.orchestra.run.smithi062.stdout:CentOS-8 - AppStream 32 MB/s | 25 MB 00:00 2022-09-03T05:44:26.911 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-8 - BaseOS 42 MB/s | 25 MB 00:00 2022-09-03T05:44:27.377 INFO:teuthology.orchestra.run.smithi062.stdout:CentOS-8 - BaseOS 41 MB/s | 25 MB 00:00 2022-09-03T05:44:30.751 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-8 - Extras 156 kB/s | 18 kB 00:00 2022-09-03T05:44:31.137 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-8 - PowerTools 19 MB/s | 5.0 MB 00:00 2022-09-03T05:44:31.484 INFO:teuthology.orchestra.run.smithi062.stdout:CentOS-8 - Extras 162 kB/s | 18 kB 00:00 2022-09-03T05:44:31.854 INFO:teuthology.orchestra.run.smithi062.stdout:CentOS-8 - PowerTools 20 MB/s | 5.0 MB 00:00 2022-09-03T05:44:32.286 INFO:teuthology.orchestra.run.smithi087.stdout:Copr repo for python3-asyncssh owned by ceph 20 kB/s | 3.5 kB 00:00 2022-09-03T05:44:33.060 INFO:teuthology.orchestra.run.smithi062.stdout:Copr repo for python3-asyncssh owned by ceph 19 kB/s | 3.5 kB 00:00 2022-09-03T05:44:33.370 INFO:teuthology.orchestra.run.smithi087.stdout:Extra Packages for Enterprise Linux 13 MB/s | 13 MB 00:00 2022-09-03T05:44:34.169 INFO:teuthology.orchestra.run.smithi062.stdout:Extra Packages for Enterprise Linux 13 MB/s | 13 MB 00:00 2022-09-03T05:44:36.387 INFO:teuthology.orchestra.run.smithi087.stdout:lab-extras 603 kB/s | 24 kB 00:00 2022-09-03T05:44:37.435 INFO:teuthology.orchestra.run.smithi062.stdout:lab-extras 309 kB/s | 24 kB 00:00 2022-09-03T05:44:39.483 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:44:39.486 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:44:39.486 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:44:39.487 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:44:39.487 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:44:39.487 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-radosgw x86_64 2:17.0.0-14689.gafd42238.el8 ceph 17 M 2022-09-03T05:44:39.487 INFO:teuthology.orchestra.run.smithi087.stdout:Upgrading: 2022-09-03T05:44:39.487 INFO:teuthology.orchestra.run.smithi087.stdout: librados2 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 3.6 M 2022-09-03T05:44:39.488 INFO:teuthology.orchestra.run.smithi087.stdout: librbd1 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 3.6 M 2022-09-03T05:44:39.488 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2022-09-03T05:44:39.488 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-base x86_64 2:17.0.0-14689.gafd42238.el8 ceph 5.2 M 2022-09-03T05:44:39.488 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-common x86_64 2:17.0.0-14689.gafd42238.el8 ceph 24 M 2022-09-03T05:44:39.489 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-selinux x86_64 2:17.0.0-14689.gafd42238.el8 ceph 24 k 2022-09-03T05:44:39.489 INFO:teuthology.orchestra.run.smithi087.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2022-09-03T05:44:39.489 INFO:teuthology.orchestra.run.smithi087.stdout: libcephfs2 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 766 k 2022-09-03T05:44:39.489 INFO:teuthology.orchestra.run.smithi087.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2022-09-03T05:44:39.489 INFO:teuthology.orchestra.run.smithi087.stdout: librabbitmq x86_64 0.9.0-3.el8 CentOS-BaseOS 47 k 2022-09-03T05:44:39.490 INFO:teuthology.orchestra.run.smithi087.stdout: libradosstriper1 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 473 k 2022-09-03T05:44:39.490 INFO:teuthology.orchestra.run.smithi087.stdout: librdkafka x86_64 0.11.4-3.el8 CentOS-AppStream 354 k 2022-09-03T05:44:39.490 INFO:teuthology.orchestra.run.smithi087.stdout: librgw2 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 5.7 M 2022-09-03T05:44:39.490 INFO:teuthology.orchestra.run.smithi087.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2022-09-03T05:44:39.491 INFO:teuthology.orchestra.run.smithi087.stdout: lttng-ust x86_64 2.8.1-11.el8 CentOS-AppStream 259 k 2022-09-03T05:44:39.491 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ceph-argparse 2022-09-03T05:44:39.491 INFO:teuthology.orchestra.run.smithi087.stdout: x86_64 2:17.0.0-14689.gafd42238.el8 ceph 45 k 2022-09-03T05:44:39.491 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ceph-common x86_64 2:17.0.0-14689.gafd42238.el8 ceph 109 k 2022-09-03T05:44:39.491 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cephfs x86_64 2:17.0.0-14689.gafd42238.el8 ceph 190 k 2022-09-03T05:44:39.492 INFO:teuthology.orchestra.run.smithi087.stdout: python3-prettytable noarch 0.7.2-14.el8 CentOS-AppStream 44 k 2022-09-03T05:44:39.492 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rados x86_64 2:17.0.0-14689.gafd42238.el8 ceph 371 k 2022-09-03T05:44:39.492 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rbd x86_64 2:17.0.0-14689.gafd42238.el8 ceph 357 k 2022-09-03T05:44:39.492 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rgw x86_64 2:17.0.0-14689.gafd42238.el8 ceph 107 k 2022-09-03T05:44:39.493 INFO:teuthology.orchestra.run.smithi087.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2022-09-03T05:44:39.493 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:44:39.493 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:44:39.493 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:44:39.493 INFO:teuthology.orchestra.run.smithi087.stdout:Install 21 Packages 2022-09-03T05:44:39.494 INFO:teuthology.orchestra.run.smithi087.stdout:Upgrade 2 Packages 2022-09-03T05:44:39.494 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:44:39.495 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 65 M 2022-09-03T05:44:39.495 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:44:40.124 INFO:teuthology.orchestra.run.smithi087.stdout:(1/23): ceph-base-17.0.0-14689.gafd42238.el8.x8 8.2 MB/s | 5.2 MB 00:00 2022-09-03T05:44:40.214 INFO:teuthology.orchestra.run.smithi087.stdout:(2/23): ceph-selinux-17.0.0-14689.gafd42238.el8 265 kB/s | 24 kB 00:00 2022-09-03T05:44:40.331 INFO:teuthology.orchestra.run.smithi087.stdout:(3/23): libcephfs2-17.0.0-14689.gafd42238.el8.x 6.4 MB/s | 766 kB 00:00 2022-09-03T05:44:40.431 INFO:teuthology.orchestra.run.smithi087.stdout:(4/23): libradosstriper1-17.0.0-14689.gafd42238 4.6 MB/s | 473 kB 00:00 2022-09-03T05:44:40.760 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:44:40.763 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:44:40.763 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:44:40.764 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:44:40.764 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:44:40.765 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-radosgw x86_64 2:17.0.0-14689.gafd42238.el8 ceph 17 M 2022-09-03T05:44:40.765 INFO:teuthology.orchestra.run.smithi062.stdout:Upgrading: 2022-09-03T05:44:40.765 INFO:teuthology.orchestra.run.smithi062.stdout: librados2 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 3.6 M 2022-09-03T05:44:40.766 INFO:teuthology.orchestra.run.smithi062.stdout: librbd1 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 3.6 M 2022-09-03T05:44:40.766 INFO:teuthology.orchestra.run.smithi062.stdout:Installing dependencies: 2022-09-03T05:44:40.767 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-base x86_64 2:17.0.0-14689.gafd42238.el8 ceph 5.2 M 2022-09-03T05:44:40.767 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-common x86_64 2:17.0.0-14689.gafd42238.el8 ceph 24 M 2022-09-03T05:44:40.768 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-selinux x86_64 2:17.0.0-14689.gafd42238.el8 ceph 24 k 2022-09-03T05:44:40.768 INFO:teuthology.orchestra.run.smithi062.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2022-09-03T05:44:40.768 INFO:teuthology.orchestra.run.smithi062.stdout: libcephfs2 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 766 k 2022-09-03T05:44:40.769 INFO:teuthology.orchestra.run.smithi062.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2022-09-03T05:44:40.769 INFO:teuthology.orchestra.run.smithi062.stdout: librabbitmq x86_64 0.9.0-3.el8 CentOS-BaseOS 47 k 2022-09-03T05:44:40.769 INFO:teuthology.orchestra.run.smithi062.stdout: libradosstriper1 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 473 k 2022-09-03T05:44:40.769 INFO:teuthology.orchestra.run.smithi062.stdout: librdkafka x86_64 0.11.4-3.el8 CentOS-AppStream 354 k 2022-09-03T05:44:40.770 INFO:teuthology.orchestra.run.smithi062.stdout: librgw2 x86_64 2:17.0.0-14689.gafd42238.el8 ceph 5.7 M 2022-09-03T05:44:40.770 INFO:teuthology.orchestra.run.smithi062.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2022-09-03T05:44:40.770 INFO:teuthology.orchestra.run.smithi062.stdout: lttng-ust x86_64 2.8.1-11.el8 CentOS-AppStream 259 k 2022-09-03T05:44:40.770 INFO:teuthology.orchestra.run.smithi062.stdout: python3-ceph-argparse 2022-09-03T05:44:40.771 INFO:teuthology.orchestra.run.smithi062.stdout: x86_64 2:17.0.0-14689.gafd42238.el8 ceph 45 k 2022-09-03T05:44:40.771 INFO:teuthology.orchestra.run.smithi062.stdout: python3-ceph-common x86_64 2:17.0.0-14689.gafd42238.el8 ceph 109 k 2022-09-03T05:44:40.771 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cephfs x86_64 2:17.0.0-14689.gafd42238.el8 ceph 190 k 2022-09-03T05:44:40.771 INFO:teuthology.orchestra.run.smithi062.stdout: python3-prettytable noarch 0.7.2-14.el8 CentOS-AppStream 44 k 2022-09-03T05:44:40.772 INFO:teuthology.orchestra.run.smithi062.stdout: python3-rados x86_64 2:17.0.0-14689.gafd42238.el8 ceph 371 k 2022-09-03T05:44:40.772 INFO:teuthology.orchestra.run.smithi062.stdout: python3-rbd x86_64 2:17.0.0-14689.gafd42238.el8 ceph 357 k 2022-09-03T05:44:40.772 INFO:teuthology.orchestra.run.smithi062.stdout: python3-rgw x86_64 2:17.0.0-14689.gafd42238.el8 ceph 107 k 2022-09-03T05:44:40.773 INFO:teuthology.orchestra.run.smithi062.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2022-09-03T05:44:40.773 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:44:40.773 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:44:40.773 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:44:40.774 INFO:teuthology.orchestra.run.smithi062.stdout:Install 21 Packages 2022-09-03T05:44:40.774 INFO:teuthology.orchestra.run.smithi062.stdout:Upgrade 2 Packages 2022-09-03T05:44:40.774 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:44:40.775 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 65 M 2022-09-03T05:44:40.775 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:44:40.948 INFO:teuthology.orchestra.run.smithi087.stdout:(5/23): librgw2-17.0.0-14689.gafd42238.el8.x86_ 11 MB/s | 5.7 MB 00:00 2022-09-03T05:44:41.006 INFO:teuthology.orchestra.run.smithi087.stdout:(6/23): python3-ceph-argparse-17.0.0-14689.gafd 766 kB/s | 45 kB 00:00 2022-09-03T05:44:41.073 INFO:teuthology.orchestra.run.smithi087.stdout:(7/23): python3-ceph-common-17.0.0-14689.gafd42 1.6 MB/s | 109 kB 00:00 2022-09-03T05:44:41.148 INFO:teuthology.orchestra.run.smithi087.stdout:(8/23): python3-cephfs-17.0.0-14689.gafd42238.e 2.5 MB/s | 190 kB 00:00 2022-09-03T05:44:41.232 INFO:teuthology.orchestra.run.smithi087.stdout:(9/23): python3-rados-17.0.0-14689.gafd42238.el 4.4 MB/s | 371 kB 00:00 2022-09-03T05:44:41.316 INFO:teuthology.orchestra.run.smithi087.stdout:(10/23): python3-rbd-17.0.0-14689.gafd42238.el8 4.2 MB/s | 357 kB 00:00 2022-09-03T05:44:41.391 INFO:teuthology.orchestra.run.smithi087.stdout:(11/23): python3-rgw-17.0.0-14689.gafd42238.el8 1.4 MB/s | 107 kB 00:00 2022-09-03T05:44:41.450 INFO:teuthology.orchestra.run.smithi087.stdout:(12/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.9 MB/s | 354 kB 00:00 2022-09-03T05:44:41.507 INFO:teuthology.orchestra.run.smithi087.stdout:(13/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 4.5 MB/s | 259 kB 00:00 2022-09-03T05:44:41.536 INFO:teuthology.orchestra.run.smithi087.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 1.6 MB/s | 44 kB 00:00 2022-09-03T05:44:41.559 INFO:teuthology.orchestra.run.smithi087.stdout:(15/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 1.8 MB/s | 47 kB 00:00 2022-09-03T05:44:41.751 INFO:teuthology.orchestra.run.smithi087.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.6 MB/s | 306 kB 00:00 2022-09-03T05:44:41.905 INFO:teuthology.orchestra.run.smithi062.stdout:(1/23): ceph-base-17.0.0-14689.gafd42238.el8.x8 4.5 MB/s | 5.2 MB 00:01 2022-09-03T05:44:41.947 INFO:teuthology.orchestra.run.smithi062.stdout:(2/23): ceph-selinux-17.0.0-14689.gafd42238.el8 585 kB/s | 24 kB 00:00 2022-09-03T05:44:42.020 INFO:teuthology.orchestra.run.smithi087.stdout:(17/23): ceph-common-17.0.0-14689.gafd42238.el8 9.4 MB/s | 24 MB 00:02 2022-09-03T05:44:42.043 INFO:teuthology.orchestra.run.smithi087.stdout:(18/23): liboath-2.6.2-3.el8.x86_64.rpm 201 kB/s | 59 kB 00:00 2022-09-03T05:44:42.058 INFO:teuthology.orchestra.run.smithi062.stdout:(3/23): libcephfs2-17.0.0-14689.gafd42238.el8.x 6.7 MB/s | 766 kB 00:00 2022-09-03T05:44:42.127 INFO:teuthology.orchestra.run.smithi062.stdout:(4/23): libradosstriper1-17.0.0-14689.gafd42238 7.2 MB/s | 473 kB 00:00 2022-09-03T05:44:42.219 INFO:teuthology.orchestra.run.smithi087.stdout:(19/23): ceph-radosgw-17.0.0-14689.gafd42238.el 6.4 MB/s | 17 MB 00:02 2022-09-03T05:44:42.244 INFO:teuthology.orchestra.run.smithi087.stdout:(20/23): libunwind-1.3.1-3.el8.x86_64.rpm 333 kB/s | 75 kB 00:00 2022-09-03T05:44:42.377 INFO:teuthology.orchestra.run.smithi087.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 5.2 MB/s | 1.7 MB 00:00 2022-09-03T05:44:42.727 INFO:teuthology.orchestra.run.smithi087.stdout:(22/23): librados2-17.0.0-14689.gafd42238.el8.x 7.1 MB/s | 3.6 MB 00:00 2022-09-03T05:44:42.806 INFO:teuthology.orchestra.run.smithi062.stdout:(5/23): librgw2-17.0.0-14689.gafd42238.el8.x86_ 8.4 MB/s | 5.7 MB 00:00 2022-09-03T05:44:42.848 INFO:teuthology.orchestra.run.smithi062.stdout:(6/23): python3-ceph-argparse-17.0.0-14689.gafd 1.0 MB/s | 45 kB 00:00 2022-09-03T05:44:42.898 INFO:teuthology.orchestra.run.smithi062.stdout:(7/23): python3-ceph-common-17.0.0-14689.gafd42 2.1 MB/s | 109 kB 00:00 2022-09-03T05:44:42.902 INFO:teuthology.orchestra.run.smithi087.stdout:(23/23): librbd1-17.0.0-14689.gafd42238.el8.x86 5.4 MB/s | 3.6 MB 00:00 2022-09-03T05:44:42.906 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:44:42.906 INFO:teuthology.orchestra.run.smithi087.stdout:Total 19 MB/s | 65 MB 00:03 2022-09-03T05:44:42.949 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:44:42.977 INFO:teuthology.orchestra.run.smithi062.stdout:(8/23): python3-cephfs-17.0.0-14689.gafd42238.e 2.3 MB/s | 190 kB 00:00 2022-09-03T05:44:42.997 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:44:42.997 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:44:43.048 INFO:teuthology.orchestra.run.smithi062.stdout:(9/23): python3-rados-17.0.0-14689.gafd42238.el 5.1 MB/s | 371 kB 00:00 2022-09-03T05:44:43.116 INFO:teuthology.orchestra.run.smithi062.stdout:(10/23): python3-rbd-17.0.0-14689.gafd42238.el8 5.2 MB/s | 357 kB 00:00 2022-09-03T05:44:43.174 INFO:teuthology.orchestra.run.smithi062.stdout:(11/23): python3-rgw-17.0.0-14689.gafd42238.el8 1.8 MB/s | 107 kB 00:00 2022-09-03T05:44:43.233 INFO:teuthology.orchestra.run.smithi062.stdout:(12/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.9 MB/s | 354 kB 00:00 2022-09-03T05:44:43.258 INFO:teuthology.orchestra.run.smithi062.stdout:(13/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 10 MB/s | 259 kB 00:00 2022-09-03T05:44:43.308 INFO:teuthology.orchestra.run.smithi062.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 876 kB/s | 44 kB 00:00 2022-09-03T05:44:43.342 INFO:teuthology.orchestra.run.smithi062.stdout:(15/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 1.4 MB/s | 47 kB 00:00 2022-09-03T05:44:43.484 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:44:43.487 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:44:43.533 INFO:teuthology.orchestra.run.smithi062.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.6 MB/s | 306 kB 00:00 2022-09-03T05:44:43.611 INFO:teuthology.orchestra.run.smithi062.stdout:(17/23): liboath-2.6.2-3.el8.x86_64.rpm 784 kB/s | 59 kB 00:00 2022-09-03T05:44:43.659 INFO:teuthology.orchestra.run.smithi062.stdout:(18/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.5 MB/s | 75 kB 00:00 2022-09-03T05:44:43.868 INFO:teuthology.orchestra.run.smithi062.stdout:(19/23): thrift-0.13.0-2.el8.x86_64.rpm 8.3 MB/s | 1.7 MB 00:00 2022-09-03T05:44:44.177 INFO:teuthology.orchestra.run.smithi062.stdout:(20/23): ceph-radosgw-17.0.0-14689.gafd42238.el 5.1 MB/s | 17 MB 00:03 2022-09-03T05:44:44.335 INFO:teuthology.orchestra.run.smithi062.stdout:(21/23): librados2-17.0.0-14689.gafd42238.el8.x 7.7 MB/s | 3.6 MB 00:00 2022-09-03T05:44:44.671 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:44:44.719 INFO:teuthology.orchestra.run.smithi062.stdout:(22/23): ceph-common-17.0.0-14689.gafd42238.el8 6.0 MB/s | 24 MB 00:03 2022-09-03T05:44:44.777 INFO:teuthology.orchestra.run.smithi062.stdout:(23/23): librbd1-17.0.0-14689.gafd42238.el8.x86 6.0 MB/s | 3.6 MB 00:00 2022-09-03T05:44:44.780 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:44:44.780 INFO:teuthology.orchestra.run.smithi062.stdout:Total 16 MB/s | 65 MB 00:04 2022-09-03T05:44:44.820 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:44:44.821 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : thrift-0.13.0-2.el8.x86_64 1/25 2022-09-03T05:44:44.868 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:44:44.868 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:44:44.991 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/25 2022-09-03T05:44:45.156 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 3/25 2022-09-03T05:44:45.392 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:44:45.395 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:44:45.553 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 3/25 2022-09-03T05:44:45.748 INFO:teuthology.orchestra.run.smithi087.stdout: Upgrading : librados2-2:17.0.0-14689.gafd42238.el8.x86_64 4/25 2022-09-03T05:44:45.827 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librados2-2:17.0.0-14689.gafd42238.el8.x86_64 4/25 2022-09-03T05:44:46.070 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-rados-2:17.0.0-14689.gafd42238.el8.x86_64 5/25 2022-09-03T05:44:46.231 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 6/25 2022-09-03T05:44:46.608 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 6/25 2022-09-03T05:44:46.709 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:44:46.798 INFO:teuthology.orchestra.run.smithi087.stdout: Upgrading : librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 7/25 2022-09-03T05:44:46.847 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : thrift-0.13.0-2.el8.x86_64 1/25 2022-09-03T05:44:46.849 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 7/25 2022-09-03T05:44:47.098 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2022-09-03T05:44:47.163 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/25 2022-09-03T05:44:47.222 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 9/25 2022-09-03T05:44:47.277 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 9/25 2022-09-03T05:44:47.342 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 3/25 2022-09-03T05:44:47.416 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-ceph-argparse-2:17.0.0-14689.gafd42238.el8 10/25 2022-09-03T05:44:47.628 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cephfs-2:17.0.0-14689.gafd42238.el8.x86_64 11/25 2022-09-03T05:44:47.747 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 3/25 2022-09-03T05:44:47.822 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-rbd-2:17.0.0-14689.gafd42238.el8.x86_64 12/25 2022-09-03T05:44:47.940 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libradosstriper1-2:17.0.0-14689.gafd42238.el8.x86_ 13/25 2022-09-03T05:44:47.990 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libradosstriper1-2:17.0.0-14689.gafd42238.el8.x86_ 13/25 2022-09-03T05:44:48.075 INFO:teuthology.orchestra.run.smithi062.stdout: Upgrading : librados2-2:17.0.0-14689.gafd42238.el8.x86_64 4/25 2022-09-03T05:44:48.171 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: librados2-2:17.0.0-14689.gafd42238.el8.x86_64 4/25 2022-09-03T05:44:48.218 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 14/25 2022-09-03T05:44:48.410 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-rados-2:17.0.0-14689.gafd42238.el8.x86_64 5/25 2022-09-03T05:44:48.600 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 6/25 2022-09-03T05:44:48.907 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 15/25 2022-09-03T05:44:49.013 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 6/25 2022-09-03T05:44:49.106 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : librgw2-2:17.0.0-14689.gafd42238.el8.x86_64 16/25 2022-09-03T05:44:49.179 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librgw2-2:17.0.0-14689.gafd42238.el8.x86_64 16/25 2022-09-03T05:44:49.199 INFO:teuthology.orchestra.run.smithi062.stdout: Upgrading : librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 7/25 2022-09-03T05:44:49.255 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 7/25 2022-09-03T05:44:49.388 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-rgw-2:17.0.0-14689.gafd42238.el8.x86_64 17/25 2022-09-03T05:44:49.441 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2022-09-03T05:44:49.537 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2022-09-03T05:44:49.591 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 9/25 2022-09-03T05:44:49.662 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 9/25 2022-09-03T05:44:49.697 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-ceph-common-2:17.0.0-14689.gafd42238.el8.x 19/25 2022-09-03T05:44:49.853 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-ceph-argparse-2:17.0.0-14689.gafd42238.el8 10/25 2022-09-03T05:44:50.062 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-cephfs-2:17.0.0-14689.gafd42238.el8.x86_64 11/25 2022-09-03T05:44:50.278 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-rbd-2:17.0.0-14689.gafd42238.el8.x86_64 12/25 2022-09-03T05:44:50.441 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : libradosstriper1-2:17.0.0-14689.gafd42238.el8.x86_ 13/25 2022-09-03T05:44:50.508 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: libradosstriper1-2:17.0.0-14689.gafd42238.el8.x86_ 13/25 2022-09-03T05:44:50.753 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 14/25 2022-09-03T05:44:51.496 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 15/25 2022-09-03T05:44:51.674 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : librgw2-2:17.0.0-14689.gafd42238.el8.x86_64 16/25 2022-09-03T05:44:51.747 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: librgw2-2:17.0.0-14689.gafd42238.el8.x86_64 16/25 2022-09-03T05:44:51.892 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-rgw-2:17.0.0-14689.gafd42238.el8.x86_64 17/25 2022-09-03T05:44:52.062 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2022-09-03T05:44:52.232 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-ceph-common-2:17.0.0-14689.gafd42238.el8.x 19/25 2022-09-03T05:44:53.434 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 20/25 2022-09-03T05:44:53.647 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 20/25 2022-09-03T05:44:54.218 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 20/25 2022-09-03T05:44:54.474 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 21/25 2022-09-03T05:44:54.564 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 21/25 2022-09-03T05:44:54.746 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-selinux-2:17.0.0-14689.gafd42238.el8.x86_64 22/25 2022-09-03T05:44:56.202 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 20/25 2022-09-03T05:44:56.391 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 20/25 2022-09-03T05:44:57.010 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 20/25 2022-09-03T05:44:57.260 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 21/25 2022-09-03T05:44:57.371 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 21/25 2022-09-03T05:44:57.540 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-selinux-2:17.0.0-14689.gafd42238.el8.x86_64 22/25 2022-09-03T05:45:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-selinux-2:17.0.0-14689.gafd42238.el8.x86_64 22/25 2022-09-03T05:45:13.500 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /sys 2022-09-03T05:45:13.500 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /proc 2022-09-03T05:45:13.500 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /dev 2022-09-03T05:45:13.501 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /run 2022-09-03T05:45:13.502 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /mnt 2022-09-03T05:45:13.502 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /var/tmp 2022-09-03T05:45:13.502 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /home 2022-09-03T05:45:13.503 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /tmp 2022-09-03T05:45:13.503 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /dev 2022-09-03T05:45:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:15.222 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-radosgw-2:17.0.0-14689.gafd42238.el8.x86_64 23/25 2022-09-03T05:45:15.243 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-radosgw-2:17.0.0-14689.gafd42238.el8.x86_64 23/25 2022-09-03T05:45:15.244 INFO:teuthology.orchestra.run.smithi087.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2022-09-03T05:45:15.446 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2022-09-03T05:45:15.446 INFO:teuthology.orchestra.run.smithi087.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2022-09-03T05:45:16.056 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2022-09-03T05:45:16.056 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 1/25 2022-09-03T05:45:16.057 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 2/25 2022-09-03T05:45:16.057 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-radosgw-2:17.0.0-14689.gafd42238.el8.x86_64 3/25 2022-09-03T05:45:16.057 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-selinux-2:17.0.0-14689.gafd42238.el8.x86_64 4/25 2022-09-03T05:45:16.057 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 5/25 2022-09-03T05:45:16.057 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libradosstriper1-2:17.0.0-14689.gafd42238.el8.x86_ 6/25 2022-09-03T05:45:16.058 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librgw2-2:17.0.0-14689.gafd42238.el8.x86_64 7/25 2022-09-03T05:45:16.058 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-ceph-argparse-2:17.0.0-14689.gafd42238.el8 8/25 2022-09-03T05:45:16.058 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-ceph-common-2:17.0.0-14689.gafd42238.el8.x 9/25 2022-09-03T05:45:16.058 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cephfs-2:17.0.0-14689.gafd42238.el8.x86_64 10/25 2022-09-03T05:45:16.061 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-rados-2:17.0.0-14689.gafd42238.el8.x86_64 11/25 2022-09-03T05:45:16.062 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-rbd-2:17.0.0-14689.gafd42238.el8.x86_64 12/25 2022-09-03T05:45:16.062 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-rgw-2:17.0.0-14689.gafd42238.el8.x86_64 13/25 2022-09-03T05:45:16.062 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2022-09-03T05:45:16.063 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2022-09-03T05:45:16.063 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2022-09-03T05:45:16.064 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 17/25 2022-09-03T05:45:16.064 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2022-09-03T05:45:16.064 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 19/25 2022-09-03T05:45:16.065 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 20/25 2022-09-03T05:45:16.065 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 21/25 2022-09-03T05:45:16.066 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librados2-2:17.0.0-14689.gafd42238.el8.x86_64 22/25 2022-09-03T05:45:16.066 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2022-09-03T05:45:16.067 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 24/25 2022-09-03T05:45:16.900 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-selinux-2:17.0.0-14689.gafd42238.el8.x86_64 22/25 2022-09-03T05:45:16.900 INFO:teuthology.orchestra.run.smithi062.stdout:skipping the directory /sys 2022-09-03T05:45:16.900 INFO:teuthology.orchestra.run.smithi062.stdout:skipping the directory /proc 2022-09-03T05:45:16.901 INFO:teuthology.orchestra.run.smithi062.stdout:skipping the directory /dev 2022-09-03T05:45:16.901 INFO:teuthology.orchestra.run.smithi062.stdout:skipping the directory /run 2022-09-03T05:45:16.901 INFO:teuthology.orchestra.run.smithi062.stdout:skipping the directory /mnt 2022-09-03T05:45:16.901 INFO:teuthology.orchestra.run.smithi062.stdout:skipping the directory /var/tmp 2022-09-03T05:45:16.902 INFO:teuthology.orchestra.run.smithi062.stdout:skipping the directory /home 2022-09-03T05:45:16.902 INFO:teuthology.orchestra.run.smithi062.stdout:skipping the directory /tmp 2022-09-03T05:45:16.902 INFO:teuthology.orchestra.run.smithi062.stdout:skipping the directory /dev 2022-09-03T05:45:16.902 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:16.950 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2022-09-03T05:45:16.950 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:16.951 INFO:teuthology.orchestra.run.smithi087.stdout:Upgraded: 2022-09-03T05:45:16.951 INFO:teuthology.orchestra.run.smithi087.stdout: librados2-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.951 INFO:teuthology.orchestra.run.smithi087.stdout: librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.951 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:45:16.952 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.952 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.952 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-radosgw-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.952 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-selinux-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.953 INFO:teuthology.orchestra.run.smithi087.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2022-09-03T05:45:16.953 INFO:teuthology.orchestra.run.smithi087.stdout: libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.953 INFO:teuthology.orchestra.run.smithi087.stdout: liboath-2.6.2-3.el8.x86_64 2022-09-03T05:45:16.953 INFO:teuthology.orchestra.run.smithi087.stdout: librabbitmq-0.9.0-3.el8.x86_64 2022-09-03T05:45:16.954 INFO:teuthology.orchestra.run.smithi087.stdout: libradosstriper1-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.954 INFO:teuthology.orchestra.run.smithi087.stdout: librdkafka-0.11.4-3.el8.x86_64 2022-09-03T05:45:16.954 INFO:teuthology.orchestra.run.smithi087.stdout: librgw2-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.954 INFO:teuthology.orchestra.run.smithi087.stdout: libunwind-1.3.1-3.el8.x86_64 2022-09-03T05:45:16.955 INFO:teuthology.orchestra.run.smithi087.stdout: lttng-ust-2.8.1-11.el8.x86_64 2022-09-03T05:45:16.955 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ceph-argparse-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.955 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.955 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cephfs-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.956 INFO:teuthology.orchestra.run.smithi087.stdout: python3-prettytable-0.7.2-14.el8.noarch 2022-09-03T05:45:16.956 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rados-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.957 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rbd-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.957 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rgw-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:16.957 INFO:teuthology.orchestra.run.smithi087.stdout: thrift-0.13.0-2.el8.x86_64 2022-09-03T05:45:16.957 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:16.958 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:45:17.236 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-test 2022-09-03T05:45:17.813 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:00:41 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:45:18.741 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-radosgw-2:17.0.0-14689.gafd42238.el8.x86_64 23/25 2022-09-03T05:45:18.762 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-radosgw-2:17.0.0-14689.gafd42238.el8.x86_64 23/25 2022-09-03T05:45:18.762 INFO:teuthology.orchestra.run.smithi062.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2022-09-03T05:45:18.817 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:45:18.818 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:18.819 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:45:18.819 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:18.819 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:45:18.819 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-test x86_64 2:17.0.0-14689.gafd42238.el8 ceph 53 M 2022-09-03T05:45:18.820 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2022-09-03T05:45:18.820 INFO:teuthology.orchestra.run.smithi087.stdout: jq x86_64 1.6-3.el8 CentOS-AppStream 202 k 2022-09-03T05:45:18.820 INFO:teuthology.orchestra.run.smithi087.stdout: libcephsqlite x86_64 2:17.0.0-14689.gafd42238.el8 ceph 137 k 2022-09-03T05:45:18.820 INFO:teuthology.orchestra.run.smithi087.stdout: oniguruma x86_64 6.8.2-2.el8 CentOS-AppStream 187 k 2022-09-03T05:45:18.821 INFO:teuthology.orchestra.run.smithi087.stdout: socat x86_64 1.7.4.1-1.el8 CentOS-AppStream 323 k 2022-09-03T05:45:18.821 INFO:teuthology.orchestra.run.smithi087.stdout: xmlstarlet x86_64 1.6.1-20.el8 CentOS-PowerTools 71 k 2022-09-03T05:45:18.822 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:18.822 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:45:18.822 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:18.822 INFO:teuthology.orchestra.run.smithi087.stdout:Install 6 Packages 2022-09-03T05:45:18.823 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:18.823 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 54 M 2022-09-03T05:45:18.824 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 235 M 2022-09-03T05:45:18.824 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:45:18.919 INFO:teuthology.orchestra.run.smithi087.stdout:(1/6): jq-1.6-3.el8.x86_64.rpm 2.0 MB/s | 202 kB 00:00 2022-09-03T05:45:18.940 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2022-09-03T05:45:18.941 INFO:teuthology.orchestra.run.smithi062.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2022-09-03T05:45:18.953 INFO:teuthology.orchestra.run.smithi087.stdout:(2/6): oniguruma-6.8.2-2.el8.x86_64.rpm 5.7 MB/s | 187 kB 00:00 2022-09-03T05:45:18.986 INFO:teuthology.orchestra.run.smithi087.stdout:(3/6): socat-1.7.4.1-1.el8.x86_64.rpm 9.5 MB/s | 323 kB 00:00 2022-09-03T05:45:19.013 INFO:teuthology.orchestra.run.smithi087.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.8 MB/s | 71 kB 00:00 2022-09-03T05:45:19.072 INFO:teuthology.orchestra.run.smithi087.stdout:(5/6): libcephsqlite-17.0.0-14689.gafd42238.el8 553 kB/s | 137 kB 00:00 2022-09-03T05:45:19.547 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2022-09-03T05:45:19.547 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 1/25 2022-09-03T05:45:19.548 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 2/25 2022-09-03T05:45:19.548 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-radosgw-2:17.0.0-14689.gafd42238.el8.x86_64 3/25 2022-09-03T05:45:19.548 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-selinux-2:17.0.0-14689.gafd42238.el8.x86_64 4/25 2022-09-03T05:45:19.549 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 5/25 2022-09-03T05:45:19.549 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : libradosstriper1-2:17.0.0-14689.gafd42238.el8.x86_ 6/25 2022-09-03T05:45:19.549 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : librgw2-2:17.0.0-14689.gafd42238.el8.x86_64 7/25 2022-09-03T05:45:19.550 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-ceph-argparse-2:17.0.0-14689.gafd42238.el8 8/25 2022-09-03T05:45:19.550 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-ceph-common-2:17.0.0-14689.gafd42238.el8.x 9/25 2022-09-03T05:45:19.550 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-cephfs-2:17.0.0-14689.gafd42238.el8.x86_64 10/25 2022-09-03T05:45:19.550 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-rados-2:17.0.0-14689.gafd42238.el8.x86_64 11/25 2022-09-03T05:45:19.551 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-rbd-2:17.0.0-14689.gafd42238.el8.x86_64 12/25 2022-09-03T05:45:19.551 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-rgw-2:17.0.0-14689.gafd42238.el8.x86_64 13/25 2022-09-03T05:45:19.551 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2022-09-03T05:45:19.551 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2022-09-03T05:45:19.552 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2022-09-03T05:45:19.552 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 17/25 2022-09-03T05:45:19.553 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2022-09-03T05:45:19.553 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 19/25 2022-09-03T05:45:19.553 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 20/25 2022-09-03T05:45:19.553 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 21/25 2022-09-03T05:45:19.554 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : librados2-2:17.0.0-14689.gafd42238.el8.x86_64 22/25 2022-09-03T05:45:19.554 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2022-09-03T05:45:19.554 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 24/25 2022-09-03T05:45:20.380 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2022-09-03T05:45:20.380 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:20.380 INFO:teuthology.orchestra.run.smithi062.stdout:Upgraded: 2022-09-03T05:45:20.381 INFO:teuthology.orchestra.run.smithi062.stdout: librados2-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.381 INFO:teuthology.orchestra.run.smithi062.stdout: librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.381 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:45:20.381 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.382 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.382 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-radosgw-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.382 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-selinux-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.383 INFO:teuthology.orchestra.run.smithi062.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2022-09-03T05:45:20.383 INFO:teuthology.orchestra.run.smithi062.stdout: libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.384 INFO:teuthology.orchestra.run.smithi062.stdout: liboath-2.6.2-3.el8.x86_64 2022-09-03T05:45:20.384 INFO:teuthology.orchestra.run.smithi062.stdout: librabbitmq-0.9.0-3.el8.x86_64 2022-09-03T05:45:20.385 INFO:teuthology.orchestra.run.smithi062.stdout: libradosstriper1-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.386 INFO:teuthology.orchestra.run.smithi062.stdout: librdkafka-0.11.4-3.el8.x86_64 2022-09-03T05:45:20.386 INFO:teuthology.orchestra.run.smithi062.stdout: librgw2-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.386 INFO:teuthology.orchestra.run.smithi062.stdout: libunwind-1.3.1-3.el8.x86_64 2022-09-03T05:45:20.386 INFO:teuthology.orchestra.run.smithi062.stdout: lttng-ust-2.8.1-11.el8.x86_64 2022-09-03T05:45:20.387 INFO:teuthology.orchestra.run.smithi062.stdout: python3-ceph-argparse-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.387 INFO:teuthology.orchestra.run.smithi062.stdout: python3-ceph-common-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.387 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cephfs-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.387 INFO:teuthology.orchestra.run.smithi062.stdout: python3-prettytable-0.7.2-14.el8.noarch 2022-09-03T05:45:20.388 INFO:teuthology.orchestra.run.smithi062.stdout: python3-rados-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.388 INFO:teuthology.orchestra.run.smithi062.stdout: python3-rbd-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.388 INFO:teuthology.orchestra.run.smithi062.stdout: python3-rgw-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:20.388 INFO:teuthology.orchestra.run.smithi062.stdout: thrift-0.13.0-2.el8.x86_64 2022-09-03T05:45:20.389 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:20.389 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:45:20.649 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-test 2022-09-03T05:45:21.251 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:00:44 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:45:21.636 INFO:teuthology.orchestra.run.smithi087.stdout:(6/6): ceph-test-17.0.0-14689.gafd42238.el8.x86 19 MB/s | 53 MB 00:02 2022-09-03T05:45:21.640 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:45:21.640 INFO:teuthology.orchestra.run.smithi087.stdout:Total 19 MB/s | 54 MB 00:02 2022-09-03T05:45:21.682 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:45:21.702 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:45:21.702 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:45:22.011 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:45:22.014 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:45:22.336 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:45:22.336 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:45:22.336 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:45:22.337 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:45:22.337 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:45:22.338 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-test x86_64 2:17.0.0-14689.gafd42238.el8 ceph 53 M 2022-09-03T05:45:22.338 INFO:teuthology.orchestra.run.smithi062.stdout:Installing dependencies: 2022-09-03T05:45:22.338 INFO:teuthology.orchestra.run.smithi062.stdout: jq x86_64 1.6-3.el8 CentOS-AppStream 202 k 2022-09-03T05:45:22.339 INFO:teuthology.orchestra.run.smithi062.stdout: libcephsqlite x86_64 2:17.0.0-14689.gafd42238.el8 ceph 137 k 2022-09-03T05:45:22.339 INFO:teuthology.orchestra.run.smithi062.stdout: oniguruma x86_64 6.8.2-2.el8 CentOS-AppStream 187 k 2022-09-03T05:45:22.340 INFO:teuthology.orchestra.run.smithi062.stdout: socat x86_64 1.7.4.1-1.el8 CentOS-AppStream 323 k 2022-09-03T05:45:22.340 INFO:teuthology.orchestra.run.smithi062.stdout: xmlstarlet x86_64 1.6.1-20.el8 CentOS-PowerTools 71 k 2022-09-03T05:45:22.340 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:22.341 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:45:22.342 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:45:22.342 INFO:teuthology.orchestra.run.smithi062.stdout:Install 6 Packages 2022-09-03T05:45:22.342 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:22.343 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 54 M 2022-09-03T05:45:22.343 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 235 M 2022-09-03T05:45:22.344 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:45:22.406 INFO:teuthology.orchestra.run.smithi062.stdout:(1/6): jq-1.6-3.el8.x86_64.rpm 2.9 MB/s | 202 kB 00:00 2022-09-03T05:45:22.431 INFO:teuthology.orchestra.run.smithi062.stdout:(2/6): oniguruma-6.8.2-2.el8.x86_64.rpm 7.7 MB/s | 187 kB 00:00 2022-09-03T05:45:22.473 INFO:teuthology.orchestra.run.smithi062.stdout:(3/6): socat-1.7.4.1-1.el8.x86_64.rpm 7.5 MB/s | 323 kB 00:00 2022-09-03T05:45:22.499 INFO:teuthology.orchestra.run.smithi062.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.8 MB/s | 71 kB 00:00 2022-09-03T05:45:22.510 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:45:22.549 INFO:teuthology.orchestra.run.smithi062.stdout:(5/6): libcephsqlite-17.0.0-14689.gafd42238.el8 649 kB/s | 137 kB 00:00 2022-09-03T05:45:22.695 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2022-09-03T05:45:22.861 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2022-09-03T05:45:23.018 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2022-09-03T05:45:23.101 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2022-09-03T05:45:23.272 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : jq-1.6-3.el8.x86_64 4/6 2022-09-03T05:45:23.426 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libcephsqlite-2:17.0.0-14689.gafd42238.el8.x86_64 5/6 2022-09-03T05:45:24.932 INFO:teuthology.orchestra.run.smithi062.stdout:(6/6): ceph-test-17.0.0-14689.gafd42238.el8.x86 21 MB/s | 53 MB 00:02 2022-09-03T05:45:24.937 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:45:24.938 INFO:teuthology.orchestra.run.smithi062.stdout:Total 21 MB/s | 54 MB 00:02 2022-09-03T05:45:24.978 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:45:25.000 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:45:25.001 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:45:25.331 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:45:25.334 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:45:25.839 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:45:26.017 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2022-09-03T05:45:26.215 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2022-09-03T05:45:26.346 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2022-09-03T05:45:26.421 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2022-09-03T05:45:26.600 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : jq-1.6-3.el8.x86_64 4/6 2022-09-03T05:45:26.711 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : libcephsqlite-2:17.0.0-14689.gafd42238.el8.x86_64 5/6 2022-09-03T05:45:28.528 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libcephsqlite-2:17.0.0-14689.gafd42238.el8.x86_64 5/6 2022-09-03T05:45:28.756 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-test-2:17.0.0-14689.gafd42238.el8.x86_64 6/6 2022-09-03T05:45:28.991 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-test-2:17.0.0-14689.gafd42238.el8.x86_64 6/6 2022-09-03T05:45:28.995 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-test-2:17.0.0-14689.gafd42238.el8.x86_64 1/6 2022-09-03T05:45:28.996 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libcephsqlite-2:17.0.0-14689.gafd42238.el8.x86_64 2/6 2022-09-03T05:45:28.996 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : jq-1.6-3.el8.x86_64 3/6 2022-09-03T05:45:28.997 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2022-09-03T05:45:28.997 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2022-09-03T05:45:29.445 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2022-09-03T05:45:29.445 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:29.446 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:45:29.446 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-test-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:29.447 INFO:teuthology.orchestra.run.smithi087.stdout: jq-1.6-3.el8.x86_64 2022-09-03T05:45:29.447 INFO:teuthology.orchestra.run.smithi087.stdout: libcephsqlite-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:29.447 INFO:teuthology.orchestra.run.smithi087.stdout: oniguruma-6.8.2-2.el8.x86_64 2022-09-03T05:45:29.448 INFO:teuthology.orchestra.run.smithi087.stdout: socat-1.7.4.1-1.el8.x86_64 2022-09-03T05:45:29.449 INFO:teuthology.orchestra.run.smithi087.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2022-09-03T05:45:29.449 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:29.450 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:45:29.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph 2022-09-03T05:45:30.217 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:00:54 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:45:31.255 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:45:31.256 INFO:teuthology.orchestra.run.smithi087.stdout:======================================================================================== 2022-09-03T05:45:31.257 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:45:31.258 INFO:teuthology.orchestra.run.smithi087.stdout:======================================================================================== 2022-09-03T05:45:31.258 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:45:31.259 INFO:teuthology.orchestra.run.smithi087.stdout: ceph x86_64 2:17.0.0-14689.gafd42238.el8 ceph 6.4 k 2022-09-03T05:45:31.259 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2022-09-03T05:45:31.260 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mds x86_64 2:17.0.0-14689.gafd42238.el8 ceph 2.3 M 2022-09-03T05:45:31.260 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr x86_64 2:17.0.0-14689.gafd42238.el8 ceph 1.5 M 2022-09-03T05:45:31.264 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-modules-core noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 233 k 2022-09-03T05:45:31.264 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mon x86_64 2:17.0.0-14689.gafd42238.el8 ceph 3.9 M 2022-09-03T05:45:31.265 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-osd x86_64 2:17.0.0-14689.gafd42238.el8 ceph 17 M 2022-09-03T05:45:31.265 INFO:teuthology.orchestra.run.smithi087.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2022-09-03T05:45:31.265 INFO:teuthology.orchestra.run.smithi087.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2022-09-03T05:45:31.265 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cffi x86_64 1.11.5-5.el8 CentOS-BaseOS 237 k 2022-09-03T05:45:31.268 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2022-09-03T05:45:31.269 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2022-09-03T05:45:31.270 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cryptography x86_64 3.2.1-5.el8 CentOS-BaseOS 559 k 2022-09-03T05:45:31.270 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2022-09-03T05:45:31.271 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2022-09-03T05:45:31.271 INFO:teuthology.orchestra.run.smithi087.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2022-09-03T05:45:31.272 INFO:teuthology.orchestra.run.smithi087.stdout: python3-mako noarch 1.0.6-13.el8 CentOS-AppStream 157 k 2022-09-03T05:45:31.272 INFO:teuthology.orchestra.run.smithi087.stdout: python3-markupsafe x86_64 0.23-19.el8 CentOS-AppStream 39 k 2022-09-03T05:45:31.273 INFO:teuthology.orchestra.run.smithi087.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2022-09-03T05:45:31.273 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2022-09-03T05:45:31.276 INFO:teuthology.orchestra.run.smithi087.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2022-09-03T05:45:31.276 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 CentOS-AppStream 103 k 2022-09-03T05:45:31.277 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pycparser noarch 2.14-14.el8 CentOS-BaseOS 109 k 2022-09-03T05:45:31.277 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pytz noarch 2017.2-9.el8 CentOS-AppStream 54 k 2022-09-03T05:45:31.277 INFO:teuthology.orchestra.run.smithi087.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2022-09-03T05:45:31.277 INFO:teuthology.orchestra.run.smithi087.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2022-09-03T05:45:31.278 INFO:teuthology.orchestra.run.smithi087.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2022-09-03T05:45:31.278 INFO:teuthology.orchestra.run.smithi087.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2022-09-03T05:45:31.278 INFO:teuthology.orchestra.run.smithi087.stdout: python3-waitress noarch 1.2.1-2.el8.1 epel 241 k 2022-09-03T05:45:31.278 INFO:teuthology.orchestra.run.smithi087.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2022-09-03T05:45:31.279 INFO:teuthology.orchestra.run.smithi087.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2022-09-03T05:45:31.279 INFO:teuthology.orchestra.run.smithi087.stdout: python3-werkzeug noarch 0.12.2-4.el8 CentOS-AppStream 457 k 2022-09-03T05:45:31.279 INFO:teuthology.orchestra.run.smithi087.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2022-09-03T05:45:31.279 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:31.280 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:45:31.280 INFO:teuthology.orchestra.run.smithi087.stdout:======================================================================================== 2022-09-03T05:45:31.280 INFO:teuthology.orchestra.run.smithi087.stdout:Install 32 Packages 2022-09-03T05:45:31.280 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:31.281 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 28 M 2022-09-03T05:45:31.282 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 110 M 2022-09-03T05:45:31.282 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:45:31.397 INFO:teuthology.orchestra.run.smithi087.stdout:(1/32): ceph-17.0.0-14689.gafd42238.el8.x86_64. 49 kB/s | 6.4 kB 00:00 2022-09-03T05:45:31.653 INFO:teuthology.orchestra.run.smithi087.stdout:(2/32): ceph-mds-17.0.0-14689.gafd42238.el8.x86 5.8 MB/s | 2.3 MB 00:00 2022-09-03T05:45:31.695 INFO:teuthology.orchestra.run.smithi087.stdout:(3/32): ceph-mgr-17.0.0-14689.gafd42238.el8.x86 3.5 MB/s | 1.5 MB 00:00 2022-09-03T05:45:31.806 INFO:teuthology.orchestra.run.smithi087.stdout:(4/32): ceph-mgr-modules-core-17.0.0-14689.gafd 2.1 MB/s | 233 kB 00:00 2022-09-03T05:45:31.862 INFO:teuthology.orchestra.run.smithi087.stdout:(5/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.6 MB/s | 157 kB 00:00 2022-09-03T05:45:31.888 INFO:teuthology.orchestra.run.smithi087.stdout:(6/32): python3-markupsafe-0.23-19.el8.x86_64.r 1.5 MB/s | 39 kB 00:00 2022-09-03T05:45:31.914 INFO:teuthology.orchestra.run.smithi087.stdout:(7/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 4.1 MB/s | 103 kB 00:00 2022-09-03T05:45:31.938 INFO:teuthology.orchestra.run.smithi087.stdout:(8/32): python3-pytz-2017.2-9.el8.noarch.rpm 2.1 MB/s | 54 kB 00:00 2022-09-03T05:45:31.974 INFO:teuthology.orchestra.run.smithi087.stdout:(9/32): python3-werkzeug-0.12.2-4.el8.noarch.rp 13 MB/s | 457 kB 00:00 2022-09-03T05:45:31.997 INFO:teuthology.orchestra.run.smithi087.stdout:(10/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 9.4 MB/s | 237 kB 00:00 2022-09-03T05:45:32.062 INFO:teuthology.orchestra.run.smithi087.stdout:(11/32): python3-cryptography-3.2.1-5.el8.x86_6 8.5 MB/s | 559 kB 00:00 2022-09-03T05:45:32.098 INFO:teuthology.orchestra.run.smithi087.stdout:(12/32): python3-pycparser-2.14-14.el8.noarch.r 3.0 MB/s | 109 kB 00:00 2022-09-03T05:45:32.124 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: libcephsqlite-2:17.0.0-14689.gafd42238.el8.x86_64 5/6 2022-09-03T05:45:32.231 INFO:teuthology.orchestra.run.smithi087.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 327 kB/s | 44 kB 00:00 2022-09-03T05:45:32.298 INFO:teuthology.orchestra.run.smithi087.stdout:(14/32): ceph-mon-17.0.0-14689.gafd42238.el8.x8 4.3 MB/s | 3.9 MB 00:00 2022-09-03T05:45:32.333 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-test-2:17.0.0-14689.gafd42238.el8.x86_64 6/6 2022-09-03T05:45:32.351 INFO:teuthology.orchestra.run.smithi087.stdout:(15/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 1.5 MB/s | 185 kB 00:00 2022-09-03T05:45:32.540 INFO:teuthology.orchestra.run.smithi087.stdout:(16/32): ceph-osd-17.0.0-14689.gafd42238.el8.x8 19 MB/s | 17 MB 00:00 2022-09-03T05:45:32.565 INFO:teuthology.orchestra.run.smithi087.stdout:(17/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 646 kB/s | 173 kB 00:00 2022-09-03T05:45:32.591 INFO:teuthology.orchestra.run.smithi087.stdout:(18/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.6 MB/s | 384 kB 00:00 2022-09-03T05:45:32.596 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-test-2:17.0.0-14689.gafd42238.el8.x86_64 6/6 2022-09-03T05:45:32.597 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-test-2:17.0.0-14689.gafd42238.el8.x86_64 1/6 2022-09-03T05:45:32.597 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : libcephsqlite-2:17.0.0-14689.gafd42238.el8.x86_64 2/6 2022-09-03T05:45:32.598 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : jq-1.6-3.el8.x86_64 3/6 2022-09-03T05:45:32.598 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2022-09-03T05:45:32.598 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2022-09-03T05:45:32.608 INFO:teuthology.orchestra.run.smithi087.stdout:(19/32): python3-jaraco-functools-2.0-4.el8.noa 419 kB/s | 18 kB 00:00 2022-09-03T05:45:32.633 INFO:teuthology.orchestra.run.smithi087.stdout:(20/32): python3-jaraco-6.2-6.el8.noarch.rpm 120 kB/s | 11 kB 00:00 2022-09-03T05:45:32.650 INFO:teuthology.orchestra.run.smithi087.stdout:(21/32): python3-logutils-0.3.5-11.el8.noarch.r 829 kB/s | 49 kB 00:00 2022-09-03T05:45:32.667 INFO:teuthology.orchestra.run.smithi087.stdout:(22/32): python3-more-itertools-7.2.0-3.el8.noa 998 kB/s | 59 kB 00:00 2022-09-03T05:45:32.692 INFO:teuthology.orchestra.run.smithi087.stdout:(23/32): python3-portend-2.6-1.el8.noarch.rpm 373 kB/s | 16 kB 00:00 2022-09-03T05:45:32.711 INFO:teuthology.orchestra.run.smithi087.stdout:(24/32): python3-simplegeneric-0.8.1-17.el8.noa 443 kB/s | 19 kB 00:00 2022-09-03T05:45:32.735 INFO:teuthology.orchestra.run.smithi087.stdout:(25/32): python3-singledispatch-3.4.0.3-18.el8. 568 kB/s | 24 kB 00:00 2022-09-03T05:45:32.752 INFO:teuthology.orchestra.run.smithi087.stdout:(26/32): python3-tempora-1.14.1-5.el8.noarch.rp 695 kB/s | 29 kB 00:00 2022-09-03T05:45:32.777 INFO:teuthology.orchestra.run.smithi087.stdout:(27/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.9 MB/s | 283 kB 00:00 2022-09-03T05:45:32.794 INFO:teuthology.orchestra.run.smithi087.stdout:(28/32): python3-trustme-0.6.0-4.el8.noarch.rpm 468 kB/s | 27 kB 00:00 2022-09-03T05:45:32.832 INFO:teuthology.orchestra.run.smithi087.stdout:(29/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 4.9 MB/s | 251 kB 00:00 2022-09-03T05:45:32.854 INFO:teuthology.orchestra.run.smithi087.stdout:(30/32): python3-waitress-1.2.1-2.el8.1.noarch. 2.3 MB/s | 241 kB 00:00 2022-09-03T05:45:32.881 INFO:teuthology.orchestra.run.smithi087.stdout:(31/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.0 MB/s | 85 kB 00:00 2022-09-03T05:45:32.894 INFO:teuthology.orchestra.run.smithi087.stdout:(32/32): python3-zc-lockfile-2.0-2.el8.noarch.r 343 kB/s | 23 kB 00:00 2022-09-03T05:45:32.922 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:45:32.923 INFO:teuthology.orchestra.run.smithi087.stdout:Total 17 MB/s | 28 MB 00:01 2022-09-03T05:45:32.975 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:45:33.009 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:45:33.009 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:45:33.054 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2022-09-03T05:45:33.054 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:33.055 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:45:33.055 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-test-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:33.055 INFO:teuthology.orchestra.run.smithi062.stdout: jq-1.6-3.el8.x86_64 2022-09-03T05:45:33.056 INFO:teuthology.orchestra.run.smithi062.stdout: libcephsqlite-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:33.056 INFO:teuthology.orchestra.run.smithi062.stdout: oniguruma-6.8.2-2.el8.x86_64 2022-09-03T05:45:33.057 INFO:teuthology.orchestra.run.smithi062.stdout: socat-1.7.4.1-1.el8.x86_64 2022-09-03T05:45:33.057 INFO:teuthology.orchestra.run.smithi062.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2022-09-03T05:45:33.057 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:33.060 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:45:33.295 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph 2022-09-03T05:45:33.307 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:45:33.310 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:45:33.894 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:00:56 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:45:34.501 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:45:34.705 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2022-09-03T05:45:34.957 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2022-09-03T05:45:34.978 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:45:34.981 INFO:teuthology.orchestra.run.smithi062.stdout:======================================================================================== 2022-09-03T05:45:34.981 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:45:34.982 INFO:teuthology.orchestra.run.smithi062.stdout:======================================================================================== 2022-09-03T05:45:34.982 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:45:34.982 INFO:teuthology.orchestra.run.smithi062.stdout: ceph x86_64 2:17.0.0-14689.gafd42238.el8 ceph 6.4 k 2022-09-03T05:45:34.982 INFO:teuthology.orchestra.run.smithi062.stdout:Installing dependencies: 2022-09-03T05:45:34.983 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mds x86_64 2:17.0.0-14689.gafd42238.el8 ceph 2.3 M 2022-09-03T05:45:34.983 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr x86_64 2:17.0.0-14689.gafd42238.el8 ceph 1.5 M 2022-09-03T05:45:34.983 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-modules-core noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 233 k 2022-09-03T05:45:34.983 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mon x86_64 2:17.0.0-14689.gafd42238.el8 ceph 3.9 M 2022-09-03T05:45:34.984 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-osd x86_64 2:17.0.0-14689.gafd42238.el8 ceph 17 M 2022-09-03T05:45:34.984 INFO:teuthology.orchestra.run.smithi062.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2022-09-03T05:45:34.984 INFO:teuthology.orchestra.run.smithi062.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2022-09-03T05:45:34.984 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cffi x86_64 1.11.5-5.el8 CentOS-BaseOS 237 k 2022-09-03T05:45:34.985 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2022-09-03T05:45:34.985 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2022-09-03T05:45:34.985 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cryptography x86_64 3.2.1-5.el8 CentOS-BaseOS 559 k 2022-09-03T05:45:34.985 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2022-09-03T05:45:34.986 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2022-09-03T05:45:34.986 INFO:teuthology.orchestra.run.smithi062.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2022-09-03T05:45:34.986 INFO:teuthology.orchestra.run.smithi062.stdout: python3-mako noarch 1.0.6-13.el8 CentOS-AppStream 157 k 2022-09-03T05:45:34.986 INFO:teuthology.orchestra.run.smithi062.stdout: python3-markupsafe x86_64 0.23-19.el8 CentOS-AppStream 39 k 2022-09-03T05:45:34.987 INFO:teuthology.orchestra.run.smithi062.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2022-09-03T05:45:34.987 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2022-09-03T05:45:34.987 INFO:teuthology.orchestra.run.smithi062.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2022-09-03T05:45:34.987 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 CentOS-AppStream 103 k 2022-09-03T05:45:34.988 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pycparser noarch 2.14-14.el8 CentOS-BaseOS 109 k 2022-09-03T05:45:34.988 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pytz noarch 2017.2-9.el8 CentOS-AppStream 54 k 2022-09-03T05:45:34.988 INFO:teuthology.orchestra.run.smithi062.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2022-09-03T05:45:34.988 INFO:teuthology.orchestra.run.smithi062.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2022-09-03T05:45:34.988 INFO:teuthology.orchestra.run.smithi062.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2022-09-03T05:45:34.989 INFO:teuthology.orchestra.run.smithi062.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2022-09-03T05:45:34.989 INFO:teuthology.orchestra.run.smithi062.stdout: python3-waitress noarch 1.2.1-2.el8.1 epel 241 k 2022-09-03T05:45:34.989 INFO:teuthology.orchestra.run.smithi062.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2022-09-03T05:45:34.989 INFO:teuthology.orchestra.run.smithi062.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2022-09-03T05:45:34.990 INFO:teuthology.orchestra.run.smithi062.stdout: python3-werkzeug noarch 0.12.2-4.el8 CentOS-AppStream 457 k 2022-09-03T05:45:34.990 INFO:teuthology.orchestra.run.smithi062.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2022-09-03T05:45:34.990 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:34.990 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:45:34.991 INFO:teuthology.orchestra.run.smithi062.stdout:======================================================================================== 2022-09-03T05:45:34.991 INFO:teuthology.orchestra.run.smithi062.stdout:Install 32 Packages 2022-09-03T05:45:34.991 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:34.992 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 28 M 2022-09-03T05:45:34.992 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 110 M 2022-09-03T05:45:34.993 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:45:35.133 INFO:teuthology.orchestra.run.smithi062.stdout:(1/32): ceph-17.0.0-14689.gafd42238.el8.x86_64. 44 kB/s | 6.4 kB 00:00 2022-09-03T05:45:35.162 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2022-09-03T05:45:35.331 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-waitress-1.2.1-2.el8.1.noarch 4/32 2022-09-03T05:45:35.342 INFO:teuthology.orchestra.run.smithi062.stdout:(2/32): ceph-mgr-17.0.0-14689.gafd42238.el8.x86 4.3 MB/s | 1.5 MB 00:00 2022-09-03T05:45:35.383 INFO:teuthology.orchestra.run.smithi062.stdout:(3/32): ceph-mds-17.0.0-14689.gafd42238.el8.x86 5.7 MB/s | 2.3 MB 00:00 2022-09-03T05:45:35.451 INFO:teuthology.orchestra.run.smithi062.stdout:(4/32): ceph-mgr-modules-core-17.0.0-14689.gafd 3.4 MB/s | 233 kB 00:00 2022-09-03T05:45:35.495 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2022-09-03T05:45:35.509 INFO:teuthology.orchestra.run.smithi062.stdout:(5/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.6 MB/s | 157 kB 00:00 2022-09-03T05:45:35.537 INFO:teuthology.orchestra.run.smithi062.stdout:(6/32): python3-markupsafe-0.23-19.el8.x86_64.r 1.5 MB/s | 39 kB 00:00 2022-09-03T05:45:35.560 INFO:teuthology.orchestra.run.smithi062.stdout:(7/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 4.1 MB/s | 103 kB 00:00 2022-09-03T05:45:35.585 INFO:teuthology.orchestra.run.smithi062.stdout:(8/32): python3-pytz-2017.2-9.el8.noarch.rpm 2.1 MB/s | 54 kB 00:00 2022-09-03T05:45:35.644 INFO:teuthology.orchestra.run.smithi062.stdout:(9/32): python3-werkzeug-0.12.2-4.el8.noarch.rp 7.6 MB/s | 457 kB 00:00 2022-09-03T05:45:35.669 INFO:teuthology.orchestra.run.smithi062.stdout:(10/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 9.3 MB/s | 237 kB 00:00 2022-09-03T05:45:35.683 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2022-09-03T05:45:35.704 INFO:teuthology.orchestra.run.smithi062.stdout:(11/32): python3-cryptography-3.2.1-5.el8.x86_6 16 MB/s | 559 kB 00:00 2022-09-03T05:45:35.728 INFO:teuthology.orchestra.run.smithi062.stdout:(12/32): python3-pycparser-2.14-14.el8.noarch.r 4.3 MB/s | 109 kB 00:00 2022-09-03T05:45:35.819 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2022-09-03T05:45:35.871 INFO:teuthology.orchestra.run.smithi062.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 308 kB/s | 44 kB 00:00 2022-09-03T05:45:35.946 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2022-09-03T05:45:35.978 INFO:teuthology.orchestra.run.smithi062.stdout:(14/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 1.7 MB/s | 185 kB 00:00 2022-09-03T05:45:36.028 INFO:teuthology.orchestra.run.smithi062.stdout:(15/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 3.4 MB/s | 173 kB 00:00 2022-09-03T05:45:36.169 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2022-09-03T05:45:36.282 INFO:teuthology.orchestra.run.smithi062.stdout:(16/32): ceph-osd-17.0.0-14689.gafd42238.el8.x8 18 MB/s | 17 MB 00:00 2022-09-03T05:45:36.306 INFO:teuthology.orchestra.run.smithi062.stdout:(17/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.4 MB/s | 384 kB 00:00 2022-09-03T05:45:36.355 INFO:teuthology.orchestra.run.smithi062.stdout:(18/32): ceph-mon-17.0.0-14689.gafd42238.el8.x8 3.2 MB/s | 3.9 MB 00:01 2022-09-03T05:45:36.371 INFO:teuthology.orchestra.run.smithi062.stdout:(19/32): python3-jaraco-functools-2.0-4.el8.noa 262 kB/s | 18 kB 00:00 2022-09-03T05:45:36.397 INFO:teuthology.orchestra.run.smithi062.stdout:(20/32): python3-jaraco-6.2-6.el8.noarch.rpm 95 kB/s | 11 kB 00:00 2022-09-03T05:45:36.414 INFO:teuthology.orchestra.run.smithi062.stdout:(21/32): python3-more-itertools-7.2.0-3.el8.noa 1.4 MB/s | 59 kB 00:00 2022-09-03T05:45:36.415 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2022-09-03T05:45:36.455 INFO:teuthology.orchestra.run.smithi062.stdout:(22/32): python3-portend-2.6-1.el8.noarch.rpm 375 kB/s | 16 kB 00:00 2022-09-03T05:45:36.497 INFO:teuthology.orchestra.run.smithi062.stdout:(23/32): python3-logutils-0.3.5-11.el8.noarch.r 342 kB/s | 49 kB 00:00 2022-09-03T05:45:36.515 INFO:teuthology.orchestra.run.smithi062.stdout:(24/32): python3-simplegeneric-0.8.1-17.el8.noa 319 kB/s | 19 kB 00:00 2022-09-03T05:45:36.559 INFO:teuthology.orchestra.run.smithi062.stdout:(25/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.7 MB/s | 283 kB 00:00 2022-09-03T05:45:36.582 INFO:teuthology.orchestra.run.smithi062.stdout:(26/32): python3-singledispatch-3.4.0.3-18.el8. 285 kB/s | 24 kB 00:00 2022-09-03T05:45:36.598 INFO:teuthology.orchestra.run.smithi062.stdout:(27/32): python3-tempora-1.14.1-5.el8.noarch.rp 349 kB/s | 29 kB 00:00 2022-09-03T05:45:36.615 INFO:teuthology.orchestra.run.smithi062.stdout:(28/32): python3-trustme-0.6.0-4.el8.noarch.rpm 491 kB/s | 27 kB 00:00 2022-09-03T05:45:36.622 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2022-09-03T05:45:36.666 INFO:teuthology.orchestra.run.smithi062.stdout:(29/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.7 MB/s | 85 kB 00:00 2022-09-03T05:45:36.691 INFO:teuthology.orchestra.run.smithi062.stdout:(30/32): python3-waitress-1.2.1-2.el8.1.noarch. 2.5 MB/s | 241 kB 00:00 2022-09-03T05:45:36.716 INFO:teuthology.orchestra.run.smithi062.stdout:(31/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.1 MB/s | 251 kB 00:00 2022-09-03T05:45:36.742 INFO:teuthology.orchestra.run.smithi062.stdout:(32/32): python3-zc-lockfile-2.0-2.el8.noarch.r 305 kB/s | 23 kB 00:00 2022-09-03T05:45:36.745 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:45:36.745 INFO:teuthology.orchestra.run.smithi062.stdout:Total 16 MB/s | 28 MB 00:01 2022-09-03T05:45:36.817 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2022-09-03T05:45:36.825 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:45:36.864 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:45:36.864 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:45:37.177 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:45:37.180 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:45:37.256 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 13/32 2022-09-03T05:45:37.510 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 14/32 2022-09-03T05:45:37.673 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2022-09-03T05:45:37.889 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2022-09-03T05:45:38.058 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2022-09-03T05:45:38.374 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2022-09-03T05:45:38.398 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:45:38.571 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2022-09-03T05:45:38.585 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2022-09-03T05:45:38.726 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 20/32 2022-09-03T05:45:38.826 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2022-09-03T05:45:38.912 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2022-09-03T05:45:39.035 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2022-09-03T05:45:39.171 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2022-09-03T05:45:39.208 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-waitress-1.2.1-2.el8.1.noarch 4/32 2022-09-03T05:45:39.370 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2022-09-03T05:45:39.397 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2022-09-03T05:45:39.553 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2022-09-03T05:45:39.591 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2022-09-03T05:45:39.763 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2022-09-03T05:45:39.891 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2022-09-03T05:45:39.909 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 25/32 2022-09-03T05:45:40.116 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2022-09-03T05:45:40.238 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2022-09-03T05:45:40.344 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2022-09-03T05:45:40.542 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2022-09-03T05:45:40.621 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-modules-core-2:17.0.0-14689.gafd42238.el8 27/32 2022-09-03T05:45:40.747 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2022-09-03T05:45:40.787 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 28/32 2022-09-03T05:45:41.148 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 13/32 2022-09-03T05:45:41.422 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 14/32 2022-09-03T05:45:41.626 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2022-09-03T05:45:41.828 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2022-09-03T05:45:42.038 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2022-09-03T05:45:42.278 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 28/32 2022-09-03T05:45:42.366 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2022-09-03T05:45:42.485 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-osd-2:17.0.0-14689.gafd42238.el8.x86_64 29/32 2022-09-03T05:45:42.599 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2022-09-03T05:45:42.737 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 20/32 2022-09-03T05:45:42.868 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-osd-2:17.0.0-14689.gafd42238.el8.x86_64 29/32 2022-09-03T05:45:42.950 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2022-09-03T05:45:43.065 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mon-2:17.0.0-14689.gafd42238.el8.x86_64 30/32 2022-09-03T05:45:43.171 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2022-09-03T05:45:43.279 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mon-2:17.0.0-14689.gafd42238.el8.x86_64 30/32 2022-09-03T05:45:43.358 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2022-09-03T05:45:43.450 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mds-2:17.0.0-14689.gafd42238.el8.x86_64 31/32 2022-09-03T05:45:43.465 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mds-2:17.0.0-14689.gafd42238.el8.x86_64 31/32 2022-09-03T05:45:43.572 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-2:17.0.0-14689.gafd42238.el8.x86_64 32/32 2022-09-03T05:45:43.573 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2022-09-03T05:45:43.941 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 25/32 2022-09-03T05:45:43.942 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-2:17.0.0-14689.gafd42238.el8.x86_64 32/32 2022-09-03T05:45:43.945 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-2:17.0.0-14689.gafd42238.el8.x86_64 1/32 2022-09-03T05:45:43.945 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mds-2:17.0.0-14689.gafd42238.el8.x86_64 2/32 2022-09-03T05:45:43.946 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 3/32 2022-09-03T05:45:43.946 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mon-2:17.0.0-14689.gafd42238.el8.x86_64 4/32 2022-09-03T05:45:43.947 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-osd-2:17.0.0-14689.gafd42238.el8.x86_64 5/32 2022-09-03T05:45:43.947 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-modules-core-2:17.0.0-14689.gafd42238.el8 6/32 2022-09-03T05:45:43.948 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 7/32 2022-09-03T05:45:43.948 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2022-09-03T05:45:43.948 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2022-09-03T05:45:43.949 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 10/32 2022-09-03T05:45:43.949 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2022-09-03T05:45:43.950 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 12/32 2022-09-03T05:45:43.950 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 13/32 2022-09-03T05:45:43.951 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2022-09-03T05:45:43.951 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2022-09-03T05:45:43.951 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2022-09-03T05:45:43.952 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2022-09-03T05:45:43.952 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2022-09-03T05:45:43.953 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2022-09-03T05:45:43.953 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2022-09-03T05:45:43.954 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2022-09-03T05:45:43.954 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2022-09-03T05:45:43.954 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2022-09-03T05:45:43.955 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2022-09-03T05:45:43.955 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2022-09-03T05:45:43.956 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2022-09-03T05:45:43.956 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2022-09-03T05:45:43.956 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2022-09-03T05:45:43.958 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 29/32 2022-09-03T05:45:43.958 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2022-09-03T05:45:43.959 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2022-09-03T05:45:44.285 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2022-09-03T05:45:44.690 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-mgr-modules-core-2:17.0.0-14689.gafd42238.el8 27/32 2022-09-03T05:45:44.765 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2022-09-03T05:45:44.765 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:44.766 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:45:44.766 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:44.767 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mds-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:44.767 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:44.768 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-modules-core-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:45:44.768 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mon-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:44.769 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-osd-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:44.769 INFO:teuthology.orchestra.run.smithi087.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2022-09-03T05:45:44.770 INFO:teuthology.orchestra.run.smithi087.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2022-09-03T05:45:44.770 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cffi-1.11.5-5.el8.x86_64 2022-09-03T05:45:44.770 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cheroot-8.5.2-1.el8.noarch 2022-09-03T05:45:44.770 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2022-09-03T05:45:44.771 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2022-09-03T05:45:44.771 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-6.2-6.el8.noarch 2022-09-03T05:45:44.771 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2022-09-03T05:45:44.772 INFO:teuthology.orchestra.run.smithi087.stdout: python3-logutils-0.3.5-11.el8.noarch 2022-09-03T05:45:44.772 INFO:teuthology.orchestra.run.smithi087.stdout: python3-mako-1.0.6-13.el8.noarch 2022-09-03T05:45:44.772 INFO:teuthology.orchestra.run.smithi087.stdout: python3-markupsafe-0.23-19.el8.x86_64 2022-09-03T05:45:44.772 INFO:teuthology.orchestra.run.smithi087.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2022-09-03T05:45:44.773 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pecan-1.3.2-9.el8.noarch 2022-09-03T05:45:44.773 INFO:teuthology.orchestra.run.smithi087.stdout: python3-portend-2.6-1.el8.noarch 2022-09-03T05:45:44.773 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2022-09-03T05:45:44.773 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pycparser-2.14-14.el8.noarch 2022-09-03T05:45:44.774 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pytz-2017.2-9.el8.noarch 2022-09-03T05:45:44.774 INFO:teuthology.orchestra.run.smithi087.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2022-09-03T05:45:44.774 INFO:teuthology.orchestra.run.smithi087.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2022-09-03T05:45:44.774 INFO:teuthology.orchestra.run.smithi087.stdout: python3-tempora-1.14.1-5.el8.noarch 2022-09-03T05:45:44.775 INFO:teuthology.orchestra.run.smithi087.stdout: python3-trustme-0.6.0-4.el8.noarch 2022-09-03T05:45:44.775 INFO:teuthology.orchestra.run.smithi087.stdout: python3-waitress-1.2.1-2.el8.1.noarch 2022-09-03T05:45:44.775 INFO:teuthology.orchestra.run.smithi087.stdout: python3-webob-1.8.5-1.el8.1.noarch 2022-09-03T05:45:44.775 INFO:teuthology.orchestra.run.smithi087.stdout: python3-webtest-2.0.33-1.el8.noarch 2022-09-03T05:45:44.776 INFO:teuthology.orchestra.run.smithi087.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2022-09-03T05:45:44.776 INFO:teuthology.orchestra.run.smithi087.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2022-09-03T05:45:44.776 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:44.776 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:45:44.870 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 28/32 2022-09-03T05:45:44.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-base 2022-09-03T05:45:45.526 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:09 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:45:46.460 INFO:teuthology.orchestra.run.smithi087.stdout:Package ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:45:46.462 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 28/32 2022-09-03T05:45:46.533 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:45:46.534 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:45:46.535 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:45:46.606 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install cephadm 2022-09-03T05:45:46.678 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-osd-2:17.0.0-14689.gafd42238.el8.x86_64 29/32 2022-09-03T05:45:47.088 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-osd-2:17.0.0-14689.gafd42238.el8.x86_64 29/32 2022-09-03T05:45:47.189 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:11 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:45:47.250 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-mon-2:17.0.0-14689.gafd42238.el8.x86_64 30/32 2022-09-03T05:45:47.477 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-mon-2:17.0.0-14689.gafd42238.el8.x86_64 30/32 2022-09-03T05:45:47.650 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-mds-2:17.0.0-14689.gafd42238.el8.x86_64 31/32 2022-09-03T05:45:47.669 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-mds-2:17.0.0-14689.gafd42238.el8.x86_64 31/32 2022-09-03T05:45:47.783 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-2:17.0.0-14689.gafd42238.el8.x86_64 32/32 2022-09-03T05:45:48.196 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-2:17.0.0-14689.gafd42238.el8.x86_64 32/32 2022-09-03T05:45:48.196 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-2:17.0.0-14689.gafd42238.el8.x86_64 1/32 2022-09-03T05:45:48.196 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-mds-2:17.0.0-14689.gafd42238.el8.x86_64 2/32 2022-09-03T05:45:48.197 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 3/32 2022-09-03T05:45:48.197 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-mon-2:17.0.0-14689.gafd42238.el8.x86_64 4/32 2022-09-03T05:45:48.197 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-osd-2:17.0.0-14689.gafd42238.el8.x86_64 5/32 2022-09-03T05:45:48.198 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-mgr-modules-core-2:17.0.0-14689.gafd42238.el8 6/32 2022-09-03T05:45:48.198 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 7/32 2022-09-03T05:45:48.198 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2022-09-03T05:45:48.198 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2022-09-03T05:45:48.199 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 10/32 2022-09-03T05:45:48.200 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2022-09-03T05:45:48.200 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 12/32 2022-09-03T05:45:48.200 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 13/32 2022-09-03T05:45:48.201 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2022-09-03T05:45:48.201 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2022-09-03T05:45:48.201 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2022-09-03T05:45:48.201 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2022-09-03T05:45:48.202 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2022-09-03T05:45:48.202 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2022-09-03T05:45:48.202 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2022-09-03T05:45:48.203 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2022-09-03T05:45:48.203 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2022-09-03T05:45:48.203 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2022-09-03T05:45:48.204 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2022-09-03T05:45:48.204 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2022-09-03T05:45:48.207 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:45:48.208 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:48.208 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:45:48.208 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:48.209 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:45:48.209 INFO:teuthology.orchestra.run.smithi087.stdout: cephadm noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 81 k 2022-09-03T05:45:48.210 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:48.210 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:45:48.210 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:48.210 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2022-09-03T05:45:48.211 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:48.211 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 81 k 2022-09-03T05:45:48.211 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 367 k 2022-09-03T05:45:48.212 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:45:48.212 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2022-09-03T05:45:48.212 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2022-09-03T05:45:48.212 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2022-09-03T05:45:48.213 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 29/32 2022-09-03T05:45:48.213 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2022-09-03T05:45:48.214 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2022-09-03T05:45:48.972 INFO:teuthology.orchestra.run.smithi087.stdout:cephadm-17.0.0-14689.gafd42238.el8.noarch.rpm 105 kB/s | 81 kB 00:00 2022-09-03T05:45:48.973 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:45:48.973 INFO:teuthology.orchestra.run.smithi087.stdout:Total 105 kB/s | 81 kB 00:00 2022-09-03T05:45:48.974 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:45:48.975 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2022-09-03T05:45:48.976 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:48.977 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:45:48.977 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:48.978 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mds-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:48.978 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:48.979 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-modules-core-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:45:48.979 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mon-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:48.979 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-osd-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:48.980 INFO:teuthology.orchestra.run.smithi062.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2022-09-03T05:45:48.980 INFO:teuthology.orchestra.run.smithi062.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2022-09-03T05:45:48.980 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cffi-1.11.5-5.el8.x86_64 2022-09-03T05:45:48.981 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cheroot-8.5.2-1.el8.noarch 2022-09-03T05:45:48.981 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2022-09-03T05:45:48.982 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2022-09-03T05:45:48.982 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jaraco-6.2-6.el8.noarch 2022-09-03T05:45:48.982 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2022-09-03T05:45:48.983 INFO:teuthology.orchestra.run.smithi062.stdout: python3-logutils-0.3.5-11.el8.noarch 2022-09-03T05:45:48.984 INFO:teuthology.orchestra.run.smithi062.stdout: python3-mako-1.0.6-13.el8.noarch 2022-09-03T05:45:48.984 INFO:teuthology.orchestra.run.smithi062.stdout: python3-markupsafe-0.23-19.el8.x86_64 2022-09-03T05:45:48.985 INFO:teuthology.orchestra.run.smithi062.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2022-09-03T05:45:48.985 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pecan-1.3.2-9.el8.noarch 2022-09-03T05:45:48.985 INFO:teuthology.orchestra.run.smithi062.stdout: python3-portend-2.6-1.el8.noarch 2022-09-03T05:45:48.986 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2022-09-03T05:45:48.986 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pycparser-2.14-14.el8.noarch 2022-09-03T05:45:48.986 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pytz-2017.2-9.el8.noarch 2022-09-03T05:45:48.986 INFO:teuthology.orchestra.run.smithi062.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2022-09-03T05:45:48.987 INFO:teuthology.orchestra.run.smithi062.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2022-09-03T05:45:48.987 INFO:teuthology.orchestra.run.smithi062.stdout: python3-tempora-1.14.1-5.el8.noarch 2022-09-03T05:45:48.987 INFO:teuthology.orchestra.run.smithi062.stdout: python3-trustme-0.6.0-4.el8.noarch 2022-09-03T05:45:48.987 INFO:teuthology.orchestra.run.smithi062.stdout: python3-waitress-1.2.1-2.el8.1.noarch 2022-09-03T05:45:48.988 INFO:teuthology.orchestra.run.smithi062.stdout: python3-webob-1.8.5-1.el8.1.noarch 2022-09-03T05:45:48.988 INFO:teuthology.orchestra.run.smithi062.stdout: python3-webtest-2.0.33-1.el8.noarch 2022-09-03T05:45:48.989 INFO:teuthology.orchestra.run.smithi062.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2022-09-03T05:45:48.989 INFO:teuthology.orchestra.run.smithi062.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2022-09-03T05:45:48.989 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:48.990 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:45:48.992 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:45:48.992 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:45:48.993 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:45:48.993 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:45:49.191 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-base 2022-09-03T05:45:49.369 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:45:49.794 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:01:12 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:45:50.788 INFO:teuthology.orchestra.run.smithi062.stdout:Package ceph-base-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:45:50.870 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:45:50.871 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:45:50.871 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:45:50.875 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:45:50.950 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install cephadm 2022-09-03T05:45:51.129 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:45:51.316 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:45:51.548 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:01:14 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:45:51.770 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:45:51.771 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:51.771 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:45:51.771 INFO:teuthology.orchestra.run.smithi087.stdout: cephadm-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:45:51.771 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:51.772 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:45:52.093 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-immutable-object-cache 2022-09-03T05:45:52.622 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:45:52.623 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:45:52.624 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:45:52.624 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:45:52.624 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:45:52.624 INFO:teuthology.orchestra.run.smithi062.stdout: cephadm noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 81 k 2022-09-03T05:45:52.625 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:52.625 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:45:52.625 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:45:52.625 INFO:teuthology.orchestra.run.smithi062.stdout:Install 1 Package 2022-09-03T05:45:52.625 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:52.626 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 81 k 2022-09-03T05:45:52.626 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 367 k 2022-09-03T05:45:52.626 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:45:52.671 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:16 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:45:52.810 INFO:teuthology.orchestra.run.smithi062.stdout:cephadm-17.0.0-14689.gafd42238.el8.noarch.rpm 446 kB/s | 81 kB 00:00 2022-09-03T05:45:52.810 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:45:52.811 INFO:teuthology.orchestra.run.smithi062.stdout:Total 443 kB/s | 81 kB 00:00 2022-09-03T05:45:52.811 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:45:52.815 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:45:52.815 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:45:52.816 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:45:52.821 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:45:52.867 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:45:53.680 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:45:53.681 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:53.682 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repo Size 2022-09-03T05:45:53.682 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:53.682 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:45:53.682 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-immutable-object-cache x86_64 2:17.0.0-14689.gafd42238.el8 ceph 165 k 2022-09-03T05:45:53.683 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:53.683 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:45:53.683 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:53.683 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2022-09-03T05:45:53.683 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:53.684 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 165 k 2022-09-03T05:45:53.684 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 503 k 2022-09-03T05:45:53.684 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:45:53.886 INFO:teuthology.orchestra.run.smithi087.stdout:ceph-immutable-object-cache-17.0.0-14689.gafd42 815 kB/s | 165 kB 00:00 2022-09-03T05:45:53.886 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:45:53.887 INFO:teuthology.orchestra.run.smithi087.stdout:Total 809 kB/s | 165 kB 00:00 2022-09-03T05:45:53.887 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:45:53.897 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:45:53.897 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:45:53.943 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:45:53.946 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:45:54.083 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:45:54.230 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-immutable-object-cache-2:17.0.0-14689.gafd42238 1/1 2022-09-03T05:45:54.543 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:45:54.607 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.0.0-14689.gafd42238 1/1 2022-09-03T05:45:54.773 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:45:54.968 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:45:55.400 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:45:55.401 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:55.401 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:45:55.401 INFO:teuthology.orchestra.run.smithi062.stdout: cephadm-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:45:55.401 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:55.402 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:45:55.562 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-immutable-object-cache-2:17.0.0-14689.gafd42238 1/1 2022-09-03T05:45:55.563 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:55.563 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:45:55.563 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-immutable-object-cache-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:55.563 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:55.564 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:45:55.680 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-immutable-object-cache 2022-09-03T05:45:55.739 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr 2022-09-03T05:45:56.280 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:01:19 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:45:56.315 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:20 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:45:57.260 INFO:teuthology.orchestra.run.smithi087.stdout:Package ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:45:57.332 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:45:57.333 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:45:57.333 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:45:57.362 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:45:57.363 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:45:57.364 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repo Size 2022-09-03T05:45:57.364 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:45:57.365 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:45:57.365 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-immutable-object-cache x86_64 2:17.0.0-14689.gafd42238.el8 ceph 165 k 2022-09-03T05:45:57.365 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:57.367 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:45:57.367 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:45:57.368 INFO:teuthology.orchestra.run.smithi062.stdout:Install 1 Package 2022-09-03T05:45:57.369 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:57.369 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 165 k 2022-09-03T05:45:57.370 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 503 k 2022-09-03T05:45:57.370 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:45:57.415 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr-dashboard 2022-09-03T05:45:57.562 INFO:teuthology.orchestra.run.smithi062.stdout:ceph-immutable-object-cache-17.0.0-14689.gafd42 835 kB/s | 165 kB 00:00 2022-09-03T05:45:57.562 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:45:57.563 INFO:teuthology.orchestra.run.smithi062.stdout:Total 830 kB/s | 165 kB 00:00 2022-09-03T05:45:57.563 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:45:57.574 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:45:57.574 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:45:57.623 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:45:57.626 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:45:57.741 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:45:57.874 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-immutable-object-cache-2:17.0.0-14689.gafd42238 1/1 2022-09-03T05:45:57.984 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:21 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:45:58.254 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.0.0-14689.gafd42238 1/1 2022-09-03T05:45:58.616 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-immutable-object-cache-2:17.0.0-14689.gafd42238 1/1 2022-09-03T05:45:58.616 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:58.617 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:45:58.617 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-immutable-object-cache-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:45:58.617 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:45:58.617 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:45:58.831 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-mgr 2022-09-03T05:45:58.992 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:45:58.992 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:58.993 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:45:58.993 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:58.994 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:45:58.994 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-dashboard noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 1.6 M 2022-09-03T05:45:58.995 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2022-09-03T05:45:58.995 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-grafana-dashboards 2022-09-03T05:45:58.996 INFO:teuthology.orchestra.run.smithi087.stdout: noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 23 k 2022-09-03T05:45:58.997 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-prometheus-alerts noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 14 k 2022-09-03T05:45:58.997 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jwt noarch 1.6.1-2.el8 CentOS-BaseOS 43 k 2022-09-03T05:45:58.997 INFO:teuthology.orchestra.run.smithi087.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2022-09-03T05:45:58.998 INFO:teuthology.orchestra.run.smithi087.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2022-09-03T05:45:58.998 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:58.999 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:45:58.999 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:45:58.999 INFO:teuthology.orchestra.run.smithi087.stdout:Install 6 Packages 2022-09-03T05:45:59.000 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:45:59.000 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 1.9 M 2022-09-03T05:45:59.001 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 7.8 M 2022-09-03T05:45:59.002 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:45:59.146 INFO:teuthology.orchestra.run.smithi087.stdout:(1/6): ceph-grafana-dashboards-17.0.0-14689.gaf 150 kB/s | 23 kB 00:00 2022-09-03T05:45:59.171 INFO:teuthology.orchestra.run.smithi087.stdout:(2/6): ceph-prometheus-alerts-17.0.0-14689.gafd 81 kB/s | 14 kB 00:00 2022-09-03T05:45:59.221 INFO:teuthology.orchestra.run.smithi087.stdout:(3/6): python3-jwt-1.6.1-2.el8.noarch.rpm 567 kB/s | 43 kB 00:00 2022-09-03T05:45:59.263 INFO:teuthology.orchestra.run.smithi087.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 366 kB/s | 34 kB 00:00 2022-09-03T05:45:59.347 INFO:teuthology.orchestra.run.smithi087.stdout:(5/6): ceph-mgr-dashboard-17.0.0-14689.gafd4223 4.7 MB/s | 1.6 MB 00:00 2022-09-03T05:45:59.397 INFO:teuthology.orchestra.run.smithi087.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.1 MB/s | 196 kB 00:00 2022-09-03T05:45:59.399 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:45:59.399 INFO:teuthology.orchestra.run.smithi087.stdout:Total 4.8 MB/s | 1.9 MB 00:00 2022-09-03T05:45:59.409 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:45:59.417 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:45:59.418 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:45:59.441 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:01:22 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:45:59.498 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:45:59.501 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:45:59.741 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:45:59.961 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2022-09-03T05:46:00.155 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2022-09-03T05:46:00.374 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2022-09-03T05:46:00.448 INFO:teuthology.orchestra.run.smithi062.stdout:Package ceph-mgr-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-prometheus-alerts-2:17.0.0-14689.gafd42238.el8. 4/6 2022-09-03T05:46:00.529 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:00.529 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:46:00.529 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:00.605 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-mgr-dashboard 2022-09-03T05:46:00.832 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-grafana-dashboards-2:17.0.0-14689.gafd42238.el8 5/6 2022-09-03T05:46:01.209 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:01:24 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:02.065 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-dashboard-2:17.0.0-14689.gafd42238.el8.noar 6/6 2022-09-03T05:46:02.295 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:02.296 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:02.296 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:46:02.297 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:02.297 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:46:02.297 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-dashboard noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 1.6 M 2022-09-03T05:46:02.297 INFO:teuthology.orchestra.run.smithi062.stdout:Installing dependencies: 2022-09-03T05:46:02.298 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-grafana-dashboards 2022-09-03T05:46:02.298 INFO:teuthology.orchestra.run.smithi062.stdout: noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 23 k 2022-09-03T05:46:02.298 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-prometheus-alerts noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 14 k 2022-09-03T05:46:02.298 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jwt noarch 1.6.1-2.el8 CentOS-BaseOS 43 k 2022-09-03T05:46:02.299 INFO:teuthology.orchestra.run.smithi062.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2022-09-03T05:46:02.299 INFO:teuthology.orchestra.run.smithi062.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2022-09-03T05:46:02.299 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:02.299 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:46:02.299 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:02.300 INFO:teuthology.orchestra.run.smithi062.stdout:Install 6 Packages 2022-09-03T05:46:02.300 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:02.305 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 1.9 M 2022-09-03T05:46:02.305 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 7.8 M 2022-09-03T05:46:02.306 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:46:02.325 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.0.0-14689.gafd42238.el8.noar 6/6 2022-09-03T05:46:02.326 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-grafana-dashboards-2:17.0.0-14689.gafd42238.el8 1/6 2022-09-03T05:46:02.326 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-dashboard-2:17.0.0-14689.gafd42238.el8.noar 2/6 2022-09-03T05:46:02.327 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-prometheus-alerts-2:17.0.0-14689.gafd42238.el8. 3/6 2022-09-03T05:46:02.327 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2022-09-03T05:46:02.330 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2022-09-03T05:46:02.723 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2022-09-03T05:46:02.723 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:02.723 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:46:02.723 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-grafana-dashboards-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:02.724 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-dashboard-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:02.724 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-prometheus-alerts-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:02.724 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jwt-1.6.1-2.el8.noarch 2022-09-03T05:46:02.724 INFO:teuthology.orchestra.run.smithi087.stdout: python3-repoze-lru-0.7-6.el8.noarch 2022-09-03T05:46:02.725 INFO:teuthology.orchestra.run.smithi087.stdout: python3-routes-2.4.1-12.el8.noarch 2022-09-03T05:46:02.725 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:02.725 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:02.781 INFO:teuthology.orchestra.run.smithi062.stdout:(1/6): ceph-prometheus-alerts-17.0.0-14689.gafd 30 kB/s | 14 kB 00:00 2022-09-03T05:46:02.824 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr-diskprediction-local 2022-09-03T05:46:03.079 INFO:teuthology.orchestra.run.smithi062.stdout:(2/6): ceph-grafana-dashboards-17.0.0-14689.gaf 29 kB/s | 23 kB 00:00 2022-09-03T05:46:03.341 INFO:teuthology.orchestra.run.smithi062.stdout:(3/6): python3-jwt-1.6.1-2.el8.noarch.rpm 76 kB/s | 43 kB 00:00 2022-09-03T05:46:03.389 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:27 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:03.757 INFO:teuthology.orchestra.run.smithi062.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 50 kB/s | 34 kB 00:00 2022-09-03T05:46:04.102 INFO:teuthology.orchestra.run.smithi062.stdout:(5/6): ceph-mgr-dashboard-17.0.0-14689.gafd4223 936 kB/s | 1.6 MB 00:01 2022-09-03T05:46:04.185 INFO:teuthology.orchestra.run.smithi062.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 233 kB/s | 196 kB 00:00 2022-09-03T05:46:04.188 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:04.189 INFO:teuthology.orchestra.run.smithi062.stdout:Total 1.0 MB/s | 1.9 MB 00:01 2022-09-03T05:46:04.201 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:46:04.211 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:46:04.212 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:46:04.381 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:04.382 INFO:teuthology.orchestra.run.smithi087.stdout:===================================================================================================== 2022-09-03T05:46:04.383 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:46:04.383 INFO:teuthology.orchestra.run.smithi087.stdout:===================================================================================================== 2022-09-03T05:46:04.384 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:46:04.384 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-diskprediction-local noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 7.4 M 2022-09-03T05:46:04.385 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2022-09-03T05:46:04.385 INFO:teuthology.orchestra.run.smithi087.stdout: libgfortran x86_64 8.5.0-15.el8 CentOS-BaseOS 644 k 2022-09-03T05:46:04.386 INFO:teuthology.orchestra.run.smithi087.stdout: libquadmath x86_64 8.5.0-15.el8 CentOS-BaseOS 172 k 2022-09-03T05:46:04.387 INFO:teuthology.orchestra.run.smithi087.stdout: openblas x86_64 0.3.15-4.el8 CentOS-AppStream 4.7 M 2022-09-03T05:46:04.387 INFO:teuthology.orchestra.run.smithi087.stdout: openblas-threads x86_64 0.3.15-4.el8 CentOS-AppStream 4.9 M 2022-09-03T05:46:04.388 INFO:teuthology.orchestra.run.smithi087.stdout: python3-numpy x86_64 1:1.14.3-10.el8 CentOS-AppStream 3.7 M 2022-09-03T05:46:04.388 INFO:teuthology.orchestra.run.smithi087.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 CentOS-AppStream 225 k 2022-09-03T05:46:04.389 INFO:teuthology.orchestra.run.smithi087.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 CentOS-AppStream 14 M 2022-09-03T05:46:04.389 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:04.389 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:46:04.391 INFO:teuthology.orchestra.run.smithi087.stdout:===================================================================================================== 2022-09-03T05:46:04.391 INFO:teuthology.orchestra.run.smithi087.stdout:Install 8 Packages 2022-09-03T05:46:04.391 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:04.393 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 35 M 2022-09-03T05:46:04.393 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 210 M 2022-09-03T05:46:04.394 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:46:04.411 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:46:04.412 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:46:04.621 INFO:teuthology.orchestra.run.smithi087.stdout:(1/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 21 MB/s | 4.9 MB 00:00 2022-09-03T05:46:04.691 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:46:04.712 INFO:teuthology.orchestra.run.smithi087.stdout:(2/8): openblas-0.3.15-4.el8.x86_64.rpm 14 MB/s | 4.7 MB 00:00 2022-09-03T05:46:04.746 INFO:teuthology.orchestra.run.smithi087.stdout:(3/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 6.6 MB/s | 225 kB 00:00 2022-09-03T05:46:04.822 INFO:teuthology.orchestra.run.smithi087.stdout:(4/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 19 MB/s | 3.7 MB 00:00 2022-09-03T05:46:04.863 INFO:teuthology.orchestra.run.smithi087.stdout:(5/8): libgfortran-8.5.0-15.el8.x86_64.rpm 15 MB/s | 644 kB 00:00 2022-09-03T05:46:04.899 INFO:teuthology.orchestra.run.smithi087.stdout:(6/8): libquadmath-8.5.0-15.el8.x86_64.rpm 5.1 MB/s | 172 kB 00:00 2022-09-03T05:46:04.932 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2022-09-03T05:46:05.139 INFO:teuthology.orchestra.run.smithi087.stdout:(7/8): python3-scipy-1.0.0-21.module_el8.5.0+77 35 MB/s | 14 MB 00:00 2022-09-03T05:46:05.167 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2022-09-03T05:46:05.359 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2022-09-03T05:46:05.513 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-prometheus-alerts-2:17.0.0-14689.gafd42238.el8. 4/6 2022-09-03T05:46:05.830 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-grafana-dashboards-2:17.0.0-14689.gafd42238.el8 5/6 2022-09-03T05:46:06.047 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-mgr-dashboard-2:17.0.0-14689.gafd42238.el8.noar 6/6 2022-09-03T05:46:06.310 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.0.0-14689.gafd42238.el8.noar 6/6 2022-09-03T05:46:06.310 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-grafana-dashboards-2:17.0.0-14689.gafd42238.el8 1/6 2022-09-03T05:46:06.311 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-mgr-dashboard-2:17.0.0-14689.gafd42238.el8.noar 2/6 2022-09-03T05:46:06.311 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-prometheus-alerts-2:17.0.0-14689.gafd42238.el8. 3/6 2022-09-03T05:46:06.311 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2022-09-03T05:46:06.312 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2022-09-03T05:46:06.647 INFO:teuthology.orchestra.run.smithi087.stdout:(8/8): ceph-mgr-diskprediction-local-17.0.0-146 3.3 MB/s | 7.4 MB 00:02 2022-09-03T05:46:06.648 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:06.648 INFO:teuthology.orchestra.run.smithi087.stdout:Total 16 MB/s | 35 MB 00:02 2022-09-03T05:46:06.769 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2022-09-03T05:46:06.769 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:06.770 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:46:06.770 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-grafana-dashboards-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:06.770 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-dashboard-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:06.770 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-prometheus-alerts-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:06.771 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jwt-1.6.1-2.el8.noarch 2022-09-03T05:46:06.771 INFO:teuthology.orchestra.run.smithi062.stdout: python3-repoze-lru-0.7-6.el8.noarch 2022-09-03T05:46:06.771 INFO:teuthology.orchestra.run.smithi062.stdout: python3-routes-2.4.1-12.el8.noarch 2022-09-03T05:46:06.771 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:06.771 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:06.872 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:46:06.884 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-mgr-diskprediction-local 2022-09-03T05:46:06.889 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:46:06.890 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:46:07.255 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:46:07.257 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:46:07.478 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:01:30 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:07.959 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:46:08.112 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libquadmath-8.5.0-15.el8.x86_64 1/8 2022-09-03T05:46:08.261 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libquadmath-8.5.0-15.el8.x86_64 1/8 2022-09-03T05:46:08.378 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libgfortran-8.5.0-15.el8.x86_64 2/8 2022-09-03T05:46:08.556 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:08.562 INFO:teuthology.orchestra.run.smithi062.stdout:===================================================================================================== 2022-09-03T05:46:08.562 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:46:08.563 INFO:teuthology.orchestra.run.smithi062.stdout:===================================================================================================== 2022-09-03T05:46:08.563 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:46:08.563 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-diskprediction-local noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 7.4 M 2022-09-03T05:46:08.564 INFO:teuthology.orchestra.run.smithi062.stdout:Installing dependencies: 2022-09-03T05:46:08.564 INFO:teuthology.orchestra.run.smithi062.stdout: libgfortran x86_64 8.5.0-15.el8 CentOS-BaseOS 644 k 2022-09-03T05:46:08.565 INFO:teuthology.orchestra.run.smithi062.stdout: libquadmath x86_64 8.5.0-15.el8 CentOS-BaseOS 172 k 2022-09-03T05:46:08.565 INFO:teuthology.orchestra.run.smithi062.stdout: openblas x86_64 0.3.15-4.el8 CentOS-AppStream 4.7 M 2022-09-03T05:46:08.565 INFO:teuthology.orchestra.run.smithi062.stdout: openblas-threads x86_64 0.3.15-4.el8 CentOS-AppStream 4.9 M 2022-09-03T05:46:08.570 INFO:teuthology.orchestra.run.smithi062.stdout: python3-numpy x86_64 1:1.14.3-10.el8 CentOS-AppStream 3.7 M 2022-09-03T05:46:08.570 INFO:teuthology.orchestra.run.smithi062.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 CentOS-AppStream 225 k 2022-09-03T05:46:08.571 INFO:teuthology.orchestra.run.smithi062.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 CentOS-AppStream 14 M 2022-09-03T05:46:08.571 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:08.572 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:46:08.573 INFO:teuthology.orchestra.run.smithi062.stdout:===================================================================================================== 2022-09-03T05:46:08.573 INFO:teuthology.orchestra.run.smithi062.stdout:Install 8 Packages 2022-09-03T05:46:08.573 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:08.574 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 35 M 2022-09-03T05:46:08.574 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 210 M 2022-09-03T05:46:08.575 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:46:08.732 INFO:teuthology.orchestra.run.smithi062.stdout:(1/8): openblas-0.3.15-4.el8.x86_64.rpm 28 MB/s | 4.7 MB 00:00 2022-09-03T05:46:08.857 INFO:teuthology.orchestra.run.smithi062.stdout:(2/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 30 MB/s | 3.7 MB 00:00 2022-09-03T05:46:08.890 INFO:teuthology.orchestra.run.smithi062.stdout:(3/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 6.6 MB/s | 225 kB 00:00 2022-09-03T05:46:08.965 INFO:teuthology.orchestra.run.smithi062.stdout:(4/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 12 MB/s | 4.9 MB 00:00 2022-09-03T05:46:08.974 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libgfortran-8.5.0-15.el8.x86_64 2/8 2022-09-03T05:46:09.016 INFO:teuthology.orchestra.run.smithi062.stdout:(5/8): libgfortran-8.5.0-15.el8.x86_64.rpm 13 MB/s | 644 kB 00:00 2022-09-03T05:46:09.049 INFO:teuthology.orchestra.run.smithi062.stdout:(6/8): libquadmath-8.5.0-15.el8.x86_64.rpm 5.1 MB/s | 172 kB 00:00 2022-09-03T05:46:09.137 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2022-09-03T05:46:09.283 INFO:teuthology.orchestra.run.smithi062.stdout:(7/8): python3-scipy-1.0.0-21.module_el8.5.0+77 35 MB/s | 14 MB 00:00 2022-09-03T05:46:09.539 INFO:teuthology.orchestra.run.smithi062.stdout:(8/8): ceph-mgr-diskprediction-local-17.0.0-146 7.6 MB/s | 7.4 MB 00:00 2022-09-03T05:46:09.540 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:09.540 INFO:teuthology.orchestra.run.smithi062.stdout:Total 36 MB/s | 35 MB 00:00 2022-09-03T05:46:09.751 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2022-09-03T05:46:09.773 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:46:09.795 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:46:09.795 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:46:09.913 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2022-09-03T05:46:10.186 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:46:10.189 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:46:10.580 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2022-09-03T05:46:10.771 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:46:10.893 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : libquadmath-8.5.0-15.el8.x86_64 1/8 2022-09-03T05:46:10.918 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2022-09-03T05:46:11.019 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: libquadmath-8.5.0-15.el8.x86_64 1/8 2022-09-03T05:46:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2022-09-03T05:46:11.147 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : libgfortran-8.5.0-15.el8.x86_64 2/8 2022-09-03T05:46:11.777 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: libgfortran-8.5.0-15.el8.x86_64 2/8 2022-09-03T05:46:11.947 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2022-09-03T05:46:12.589 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2022-09-03T05:46:12.714 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2022-09-03T05:46:12.884 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2022-09-03T05:46:13.442 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2022-09-03T05:46:13.745 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2022-09-03T05:46:13.946 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2022-09-03T05:46:15.063 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2022-09-03T05:46:15.230 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-diskprediction-local-2:17.0.0-14689.gafd422 8/8 2022-09-03T05:46:15.508 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.0.0-14689.gafd422 8/8 2022-09-03T05:46:15.509 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.0.0-14689.gafd422 1/8 2022-09-03T05:46:15.509 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2022-09-03T05:46:15.509 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2022-09-03T05:46:15.509 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2022-09-03T05:46:15.510 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2022-09-03T05:46:15.510 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2022-09-03T05:46:15.510 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libgfortran-8.5.0-15.el8.x86_64 7/8 2022-09-03T05:46:15.819 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2022-09-03T05:46:15.930 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libquadmath-8.5.0-15.el8.x86_64 8/8 2022-09-03T05:46:15.930 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:15.931 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:46:15.931 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-diskprediction-local-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:15.932 INFO:teuthology.orchestra.run.smithi087.stdout: libgfortran-8.5.0-15.el8.x86_64 2022-09-03T05:46:15.932 INFO:teuthology.orchestra.run.smithi087.stdout: libquadmath-8.5.0-15.el8.x86_64 2022-09-03T05:46:15.933 INFO:teuthology.orchestra.run.smithi087.stdout: openblas-0.3.15-4.el8.x86_64 2022-09-03T05:46:15.933 INFO:teuthology.orchestra.run.smithi087.stdout: openblas-threads-0.3.15-4.el8.x86_64 2022-09-03T05:46:15.934 INFO:teuthology.orchestra.run.smithi087.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2022-09-03T05:46:15.934 INFO:teuthology.orchestra.run.smithi087.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2022-09-03T05:46:15.935 INFO:teuthology.orchestra.run.smithi087.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2022-09-03T05:46:15.935 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:15.936 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:16.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr-rook 2022-09-03T05:46:16.608 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:40 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:17.112 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2022-09-03T05:46:17.271 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-mgr-diskprediction-local-2:17.0.0-14689.gafd422 8/8 2022-09-03T05:46:17.553 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.0.0-14689.gafd422 8/8 2022-09-03T05:46:17.553 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.0.0-14689.gafd422 1/8 2022-09-03T05:46:17.554 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2022-09-03T05:46:17.555 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2022-09-03T05:46:17.555 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2022-09-03T05:46:17.555 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2022-09-03T05:46:17.555 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2022-09-03T05:46:17.556 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : libgfortran-8.5.0-15.el8.x86_64 7/8 2022-09-03T05:46:17.599 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:17.600 INFO:teuthology.orchestra.run.smithi087.stdout:========================================================================================= 2022-09-03T05:46:17.600 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:46:17.601 INFO:teuthology.orchestra.run.smithi087.stdout:========================================================================================= 2022-09-03T05:46:17.601 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:46:17.601 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-rook noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 47 k 2022-09-03T05:46:17.601 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2022-09-03T05:46:17.602 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2022-09-03T05:46:17.602 INFO:teuthology.orchestra.run.smithi087.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2022-09-03T05:46:17.602 INFO:teuthology.orchestra.run.smithi087.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2022-09-03T05:46:17.602 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jsonpatch noarch 1.21-2.el8 CentOS-AppStream 27 k 2022-09-03T05:46:17.603 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jsonpointer noarch 1.10-11.el8 CentOS-AppStream 20 k 2022-09-03T05:46:17.603 INFO:teuthology.orchestra.run.smithi087.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2022-09-03T05:46:17.603 INFO:teuthology.orchestra.run.smithi087.stdout: python3-oauthlib noarch 2.1.0-1.el8 CentOS-BaseOS 155 k 2022-09-03T05:46:17.603 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyasn1 noarch 0.3.7-6.el8 CentOS-AppStream 126 k 2022-09-03T05:46:17.603 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 CentOS-AppStream 110 k 2022-09-03T05:46:17.604 INFO:teuthology.orchestra.run.smithi087.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 CentOS-BaseOS 43 k 2022-09-03T05:46:17.604 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2022-09-03T05:46:17.605 INFO:teuthology.orchestra.run.smithi087.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2022-09-03T05:46:17.605 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:17.605 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:46:17.605 INFO:teuthology.orchestra.run.smithi087.stdout:========================================================================================= 2022-09-03T05:46:17.606 INFO:teuthology.orchestra.run.smithi087.stdout:Install 13 Packages 2022-09-03T05:46:17.606 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:17.607 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 1.9 M 2022-09-03T05:46:17.607 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 25 M 2022-09-03T05:46:17.607 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:46:17.694 INFO:teuthology.orchestra.run.smithi087.stdout:(1/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 296 kB/s | 27 kB 00:00 2022-09-03T05:46:17.719 INFO:teuthology.orchestra.run.smithi087.stdout:(2/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 5.1 MB/s | 126 kB 00:00 2022-09-03T05:46:17.738 INFO:teuthology.orchestra.run.smithi087.stdout:(3/13): python3-jsonpointer-1.10-11.el8.noarch. 151 kB/s | 20 kB 00:00 2022-09-03T05:46:17.761 INFO:teuthology.orchestra.run.smithi087.stdout:(4/13): python3-pyasn1-modules-0.3.7-6.el8.noar 2.6 MB/s | 110 kB 00:00 2022-09-03T05:46:17.778 INFO:teuthology.orchestra.run.smithi087.stdout:(5/13): python3-requests-oauthlib-1.0.0-1.el8.n 2.5 MB/s | 43 kB 00:00 2022-09-03T05:46:17.828 INFO:teuthology.orchestra.run.smithi087.stdout:(6/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.6 MB/s | 155 kB 00:00 2022-09-03T05:46:17.846 INFO:teuthology.orchestra.run.smithi087.stdout:(7/13): ceph-mgr-rook-17.0.0-14689.gafd42238.el 194 kB/s | 47 kB 00:00 2022-09-03T05:46:17.929 INFO:teuthology.orchestra.run.smithi087.stdout:(8/13): python3-cachetools-3.1.1-4.el8.noarch.r 219 kB/s | 33 kB 00:00 2022-09-03T05:46:17.946 INFO:teuthology.orchestra.run.smithi087.stdout:(9/13): python3-certifi-2018.10.15-7.el8.noarch 139 kB/s | 16 kB 00:00 2022-09-03T05:46:17.980 INFO:teuthology.orchestra.run.smithi087.stdout:(10/13): python3-google-auth-1.1.1-10.el8.noarc 606 kB/s | 81 kB 00:00 2022-09-03T05:46:17.988 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : libquadmath-8.5.0-15.el8.x86_64 8/8 2022-09-03T05:46:17.988 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:17.989 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:46:17.989 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-diskprediction-local-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:17.990 INFO:teuthology.orchestra.run.smithi062.stdout: libgfortran-8.5.0-15.el8.x86_64 2022-09-03T05:46:17.990 INFO:teuthology.orchestra.run.smithi062.stdout: libquadmath-8.5.0-15.el8.x86_64 2022-09-03T05:46:17.991 INFO:teuthology.orchestra.run.smithi062.stdout: openblas-0.3.15-4.el8.x86_64 2022-09-03T05:46:17.991 INFO:teuthology.orchestra.run.smithi062.stdout: openblas-threads-0.3.15-4.el8.x86_64 2022-09-03T05:46:17.992 INFO:teuthology.orchestra.run.smithi062.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2022-09-03T05:46:17.992 INFO:teuthology.orchestra.run.smithi062.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2022-09-03T05:46:17.993 INFO:teuthology.orchestra.run.smithi062.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2022-09-03T05:46:17.993 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:17.993 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:18.022 INFO:teuthology.orchestra.run.smithi087.stdout:(11/13): python3-rsa-4.8-1.el8.noarch.rpm 816 kB/s | 61 kB 00:00 2022-09-03T05:46:18.038 INFO:teuthology.orchestra.run.smithi087.stdout:(12/13): python3-websocket-client-0.56.0-5.el8. 1.0 MB/s | 61 kB 00:00 2022-09-03T05:46:18.117 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-mgr-rook 2022-09-03T05:46:18.172 INFO:teuthology.orchestra.run.smithi087.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 4.8 MB/s | 1.2 MB 00:00 2022-09-03T05:46:18.174 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:18.174 INFO:teuthology.orchestra.run.smithi087.stdout:Total 3.4 MB/s | 1.9 MB 00:00 2022-09-03T05:46:18.223 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:46:18.246 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:46:18.277 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:46:18.317 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:46:18.318 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:46:18.719 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:01:41 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:18.784 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:46:18.996 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2022-09-03T05:46:19.601 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2022-09-03T05:46:19.808 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:19.810 INFO:teuthology.orchestra.run.smithi062.stdout:========================================================================================= 2022-09-03T05:46:19.810 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:46:19.811 INFO:teuthology.orchestra.run.smithi062.stdout:========================================================================================= 2022-09-03T05:46:19.811 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:46:19.811 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-rook noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 47 k 2022-09-03T05:46:19.812 INFO:teuthology.orchestra.run.smithi062.stdout:Installing dependencies: 2022-09-03T05:46:19.812 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2022-09-03T05:46:19.813 INFO:teuthology.orchestra.run.smithi062.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2022-09-03T05:46:19.813 INFO:teuthology.orchestra.run.smithi062.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2022-09-03T05:46:19.814 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jsonpatch noarch 1.21-2.el8 CentOS-AppStream 27 k 2022-09-03T05:46:19.814 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jsonpointer noarch 1.10-11.el8 CentOS-AppStream 20 k 2022-09-03T05:46:19.814 INFO:teuthology.orchestra.run.smithi062.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2022-09-03T05:46:19.815 INFO:teuthology.orchestra.run.smithi062.stdout: python3-oauthlib noarch 2.1.0-1.el8 CentOS-BaseOS 155 k 2022-09-03T05:46:19.815 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pyasn1 noarch 0.3.7-6.el8 CentOS-AppStream 126 k 2022-09-03T05:46:19.816 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 CentOS-AppStream 110 k 2022-09-03T05:46:19.816 INFO:teuthology.orchestra.run.smithi062.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 CentOS-BaseOS 43 k 2022-09-03T05:46:19.816 INFO:teuthology.orchestra.run.smithi062.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2022-09-03T05:46:19.817 INFO:teuthology.orchestra.run.smithi062.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2022-09-03T05:46:19.817 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:19.818 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:46:19.818 INFO:teuthology.orchestra.run.smithi062.stdout:========================================================================================= 2022-09-03T05:46:19.819 INFO:teuthology.orchestra.run.smithi062.stdout:Install 13 Packages 2022-09-03T05:46:19.819 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:19.820 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 1.9 M 2022-09-03T05:46:19.821 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 25 M 2022-09-03T05:46:19.821 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:46:19.905 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-rsa-4.8-1.el8.noarch 3/13 2022-09-03T05:46:20.118 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2022-09-03T05:46:20.261 INFO:teuthology.orchestra.run.smithi062.stdout:(1/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 59 kB/s | 27 kB 00:00 2022-09-03T05:46:20.291 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2022-09-03T05:46:20.528 INFO:teuthology.orchestra.run.smithi062.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 28 kB/s | 20 kB 00:00 2022-09-03T05:46:20.542 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2022-09-03T05:46:20.678 INFO:teuthology.orchestra.run.smithi062.stdout:(3/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 303 kB/s | 126 kB 00:00 2022-09-03T05:46:20.696 INFO:teuthology.orchestra.run.smithi062.stdout:(4/13): ceph-mgr-rook-17.0.0-14689.gafd42238.el 53 kB/s | 47 kB 00:00 2022-09-03T05:46:20.722 INFO:teuthology.orchestra.run.smithi062.stdout:(5/13): python3-pyasn1-modules-0.3.7-6.el8.noar 573 kB/s | 110 kB 00:00 2022-09-03T05:46:20.793 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2022-09-03T05:46:20.796 INFO:teuthology.orchestra.run.smithi062.stdout:(6/13): python3-requests-oauthlib-1.0.0-1.el8.n 426 kB/s | 43 kB 00:00 2022-09-03T05:46:20.813 INFO:teuthology.orchestra.run.smithi062.stdout:(7/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.1 MB/s | 155 kB 00:00 2022-09-03T05:46:20.830 INFO:teuthology.orchestra.run.smithi062.stdout:(8/13): python3-cachetools-3.1.1-4.el8.noarch.r 316 kB/s | 33 kB 00:00 2022-09-03T05:46:20.897 INFO:teuthology.orchestra.run.smithi062.stdout:(9/13): python3-certifi-2018.10.15-7.el8.noarch 162 kB/s | 16 kB 00:00 2022-09-03T05:46:20.972 INFO:teuthology.orchestra.run.smithi062.stdout:(10/13): python3-google-auth-1.1.1-10.el8.noarc 512 kB/s | 81 kB 00:00 2022-09-03T05:46:20.998 INFO:teuthology.orchestra.run.smithi062.stdout:(11/13): python3-rsa-4.8-1.el8.noarch.rpm 613 kB/s | 61 kB 00:00 2022-09-03T05:46:21.023 INFO:teuthology.orchestra.run.smithi062.stdout:(12/13): python3-websocket-client-0.56.0-5.el8. 1.2 MB/s | 61 kB 00:00 2022-09-03T05:46:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2022-09-03T05:46:21.056 INFO:teuthology.orchestra.run.smithi062.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 5.2 MB/s | 1.2 MB 00:00 2022-09-03T05:46:21.059 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:21.060 INFO:teuthology.orchestra.run.smithi062.stdout:Total 1.5 MB/s | 1.9 MB 00:01 2022-09-03T05:46:21.121 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:46:21.138 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:46:21.139 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:46:21.231 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:46:21.234 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:46:21.766 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:46:22.017 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2022-09-03T05:46:22.061 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2022-09-03T05:46:22.248 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2022-09-03T05:46:22.308 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2022-09-03T05:46:22.428 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-rsa-4.8-1.el8.noarch 3/13 2022-09-03T05:46:22.441 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2022-09-03T05:46:22.627 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2022-09-03T05:46:22.644 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2022-09-03T05:46:22.785 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2022-09-03T05:46:22.793 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-rook-2:17.0.0-14689.gafd42238.el8.noarch 13/13 2022-09-03T05:46:23.010 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2022-09-03T05:46:23.049 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-rook-2:17.0.0-14689.gafd42238.el8.noarch 13/13 2022-09-03T05:46:23.050 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-rook-2:17.0.0-14689.gafd42238.el8.noarch 1/13 2022-09-03T05:46:23.050 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2022-09-03T05:46:23.051 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2022-09-03T05:46:23.051 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2022-09-03T05:46:23.052 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2022-09-03T05:46:23.052 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2022-09-03T05:46:23.052 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2022-09-03T05:46:23.053 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2022-09-03T05:46:23.053 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2022-09-03T05:46:23.053 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2022-09-03T05:46:23.054 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2022-09-03T05:46:23.054 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 12/13 2022-09-03T05:46:23.288 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2022-09-03T05:46:23.484 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2022-09-03T05:46:23.530 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2022-09-03T05:46:23.531 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:23.531 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:46:23.531 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-rook-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:23.531 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cachetools-3.1.1-4.el8.noarch 2022-09-03T05:46:23.532 INFO:teuthology.orchestra.run.smithi087.stdout: python3-certifi-2018.10.15-7.el8.noarch 2022-09-03T05:46:23.532 INFO:teuthology.orchestra.run.smithi087.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2022-09-03T05:46:23.532 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jsonpatch-1.21-2.el8.noarch 2022-09-03T05:46:23.532 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jsonpointer-1.10-11.el8.noarch 2022-09-03T05:46:23.533 INFO:teuthology.orchestra.run.smithi087.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2022-09-03T05:46:23.533 INFO:teuthology.orchestra.run.smithi087.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2022-09-03T05:46:23.533 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2022-09-03T05:46:23.533 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2022-09-03T05:46:23.533 INFO:teuthology.orchestra.run.smithi087.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2022-09-03T05:46:23.534 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rsa-4.8-1.el8.noarch 2022-09-03T05:46:23.534 INFO:teuthology.orchestra.run.smithi087.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2022-09-03T05:46:23.534 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:23.534 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:23.634 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr-cephadm 2022-09-03T05:46:24.203 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:48 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:24.576 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2022-09-03T05:46:24.819 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2022-09-03T05:46:24.953 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2022-09-03T05:46:25.155 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2022-09-03T05:46:25.217 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:25.218 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:25.218 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:46:25.219 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:25.219 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:46:25.219 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-cephadm noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 114 k 2022-09-03T05:46:25.220 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2022-09-03T05:46:25.220 INFO:teuthology.orchestra.run.smithi087.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2022-09-03T05:46:25.220 INFO:teuthology.orchestra.run.smithi087.stdout: python3-babel noarch 2.5.1-7.el8 CentOS-AppStream 4.8 M 2022-09-03T05:46:25.220 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jinja2 noarch 2.10.1-3.el8 CentOS-AppStream 538 k 2022-09-03T05:46:25.220 INFO:teuthology.orchestra.run.smithi087.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2022-09-03T05:46:25.221 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:25.221 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:46:25.221 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:25.221 INFO:teuthology.orchestra.run.smithi087.stdout:Install 5 Packages 2022-09-03T05:46:25.222 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:25.223 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 5.9 M 2022-09-03T05:46:25.223 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 26 M 2022-09-03T05:46:25.223 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:46:25.295 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-mgr-rook-2:17.0.0-14689.gafd42238.el8.noarch 13/13 2022-09-03T05:46:25.313 INFO:teuthology.orchestra.run.smithi087.stdout:(1/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 5.7 MB/s | 538 kB 00:00 2022-09-03T05:46:25.429 INFO:teuthology.orchestra.run.smithi087.stdout:(2/5): python3-babel-2.5.1-7.el8.noarch.rpm 23 MB/s | 4.8 MB 00:00 2022-09-03T05:46:25.538 INFO:teuthology.orchestra.run.smithi087.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 2.1 MB/s | 476 kB 00:00 2022-09-03T05:46:25.557 INFO:teuthology.orchestra.run.smithi087.stdout:(4/5): python3-natsort-7.1.1-2.el8.noarch.rpm 484 kB/s | 60 kB 00:00 2022-09-03T05:46:25.591 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-mgr-rook-2:17.0.0-14689.gafd42238.el8.noarch 13/13 2022-09-03T05:46:25.591 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-mgr-rook-2:17.0.0-14689.gafd42238.el8.noarch 1/13 2022-09-03T05:46:25.592 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2022-09-03T05:46:25.592 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2022-09-03T05:46:25.592 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2022-09-03T05:46:25.592 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2022-09-03T05:46:25.593 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2022-09-03T05:46:25.593 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2022-09-03T05:46:25.593 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2022-09-03T05:46:25.593 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2022-09-03T05:46:25.594 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2022-09-03T05:46:25.594 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2022-09-03T05:46:25.594 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 12/13 2022-09-03T05:46:25.772 INFO:teuthology.orchestra.run.smithi087.stdout:(5/5): ceph-mgr-cephadm-17.0.0-14689.gafd42238. 208 kB/s | 114 kB 00:00 2022-09-03T05:46:25.773 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:25.773 INFO:teuthology.orchestra.run.smithi087.stdout:Total 11 MB/s | 5.9 MB 00:00 2022-09-03T05:46:25.824 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:46:25.834 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:46:25.835 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:46:25.921 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:46:25.926 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:46:26.176 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:46:26.336 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2022-09-03T05:46:26.337 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:26.337 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:46:26.338 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-rook-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:26.338 INFO:teuthology.orchestra.run.smithi062.stdout: python3-cachetools-3.1.1-4.el8.noarch 2022-09-03T05:46:26.339 INFO:teuthology.orchestra.run.smithi062.stdout: python3-certifi-2018.10.15-7.el8.noarch 2022-09-03T05:46:26.339 INFO:teuthology.orchestra.run.smithi062.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2022-09-03T05:46:26.340 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jsonpatch-1.21-2.el8.noarch 2022-09-03T05:46:26.340 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jsonpointer-1.10-11.el8.noarch 2022-09-03T05:46:26.341 INFO:teuthology.orchestra.run.smithi062.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2022-09-03T05:46:26.341 INFO:teuthology.orchestra.run.smithi062.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2022-09-03T05:46:26.342 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2022-09-03T05:46:26.342 INFO:teuthology.orchestra.run.smithi062.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2022-09-03T05:46:26.342 INFO:teuthology.orchestra.run.smithi062.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2022-09-03T05:46:26.342 INFO:teuthology.orchestra.run.smithi062.stdout: python3-rsa-4.8-1.el8.noarch 2022-09-03T05:46:26.343 INFO:teuthology.orchestra.run.smithi062.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2022-09-03T05:46:26.344 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:26.345 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:26.455 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-mgr-cephadm 2022-09-03T05:46:26.456 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 1/5 2022-09-03T05:46:27.051 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:01:50 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:27.389 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2022-09-03T05:46:27.720 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 3/5 2022-09-03T05:46:27.962 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 4/5 2022-09-03T05:46:28.143 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-cephadm-2:17.0.0-14689.gafd42238.el8.noarch 5/5 2022-09-03T05:46:28.157 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:28.159 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:28.159 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:46:28.159 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:28.160 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:46:28.160 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-cephadm noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 114 k 2022-09-03T05:46:28.160 INFO:teuthology.orchestra.run.smithi062.stdout:Installing dependencies: 2022-09-03T05:46:28.160 INFO:teuthology.orchestra.run.smithi062.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2022-09-03T05:46:28.160 INFO:teuthology.orchestra.run.smithi062.stdout: python3-babel noarch 2.5.1-7.el8 CentOS-AppStream 4.8 M 2022-09-03T05:46:28.161 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jinja2 noarch 2.10.1-3.el8 CentOS-AppStream 538 k 2022-09-03T05:46:28.161 INFO:teuthology.orchestra.run.smithi062.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2022-09-03T05:46:28.161 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:28.161 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:46:28.162 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:28.162 INFO:teuthology.orchestra.run.smithi062.stdout:Install 5 Packages 2022-09-03T05:46:28.162 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:28.162 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 5.9 M 2022-09-03T05:46:28.162 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 26 M 2022-09-03T05:46:28.163 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:46:28.258 INFO:teuthology.orchestra.run.smithi062.stdout:(1/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 5.4 MB/s | 538 kB 00:00 2022-09-03T05:46:28.350 INFO:teuthology.orchestra.run.smithi062.stdout:(2/5): python3-babel-2.5.1-7.el8.noarch.rpm 25 MB/s | 4.8 MB 00:00 2022-09-03T05:46:28.370 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.0.0-14689.gafd42238.el8.noarch 5/5 2022-09-03T05:46:28.371 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/5 2022-09-03T05:46:28.371 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/5 2022-09-03T05:46:28.372 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/5 2022-09-03T05:46:28.372 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2022-09-03T05:46:28.483 INFO:teuthology.orchestra.run.smithi062.stdout:(3/5): python3-natsort-7.1.1-2.el8.noarch.rpm 454 kB/s | 60 kB 00:00 2022-09-03T05:46:28.567 INFO:teuthology.orchestra.run.smithi062.stdout:(4/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.5 MB/s | 476 kB 00:00 2022-09-03T05:46:28.761 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 5/5 2022-09-03T05:46:28.761 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:28.762 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:46:28.762 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-cephadm-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:28.763 INFO:teuthology.orchestra.run.smithi087.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2022-09-03T05:46:28.763 INFO:teuthology.orchestra.run.smithi087.stdout: python3-babel-2.5.1-7.el8.noarch 2022-09-03T05:46:28.764 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jinja2-2.10.1-3.el8.noarch 2022-09-03T05:46:28.764 INFO:teuthology.orchestra.run.smithi087.stdout: python3-natsort-7.1.1-2.el8.noarch 2022-09-03T05:46:28.764 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:28.765 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:28.768 INFO:teuthology.orchestra.run.smithi062.stdout:(5/5): ceph-mgr-cephadm-17.0.0-14689.gafd42238. 188 kB/s | 114 kB 00:00 2022-09-03T05:46:28.771 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:28.772 INFO:teuthology.orchestra.run.smithi062.stdout:Total 9.7 MB/s | 5.9 MB 00:00 2022-09-03T05:46:28.821 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:46:28.833 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:46:28.833 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:46:28.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-fuse 2022-09-03T05:46:28.924 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:46:28.927 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:46:29.199 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:46:29.451 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:53 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:29.533 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 1/5 2022-09-03T05:46:30.452 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:30.454 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:30.454 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:46:30.455 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:30.455 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:46:30.455 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-fuse x86_64 2:17.0.0-14689.gafd42238.el8 ceph 843 k 2022-09-03T05:46:30.456 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2022-09-03T05:46:30.456 INFO:teuthology.orchestra.run.smithi087.stdout: fuse x86_64 2.9.7-16.el8 CentOS-BaseOS 83 k 2022-09-03T05:46:30.456 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:30.456 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:46:30.457 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:30.457 INFO:teuthology.orchestra.run.smithi087.stdout:Install 2 Packages 2022-09-03T05:46:30.457 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:30.457 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 926 k 2022-09-03T05:46:30.458 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 2.9 M 2022-09-03T05:46:30.458 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:46:30.511 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2022-09-03T05:46:30.531 INFO:teuthology.orchestra.run.smithi087.stdout:(1/2): fuse-2.9.7-16.el8.x86_64.rpm 1.1 MB/s | 83 kB 00:00 2022-09-03T05:46:30.872 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 3/5 2022-09-03T05:46:31.132 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 4/5 2022-09-03T05:46:31.336 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-mgr-cephadm-2:17.0.0-14689.gafd42238.el8.noarch 5/5 2022-09-03T05:46:31.413 INFO:teuthology.orchestra.run.smithi087.stdout:(2/2): ceph-fuse-17.0.0-14689.gafd42238.el8.x86 880 kB/s | 843 kB 00:00 2022-09-03T05:46:31.414 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:31.414 INFO:teuthology.orchestra.run.smithi087.stdout:Total 965 kB/s | 926 kB 00:00 2022-09-03T05:46:31.424 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:46:31.444 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:46:31.444 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:46:31.518 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:46:31.520 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:46:31.602 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.0.0-14689.gafd42238.el8.noarch 5/5 2022-09-03T05:46:31.602 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-mgr-cephadm-2:17.0.0-14689.gafd42238.el8.noarch 1/5 2022-09-03T05:46:31.602 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/5 2022-09-03T05:46:31.603 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/5 2022-09-03T05:46:31.603 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2022-09-03T05:46:31.668 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:46:31.940 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : fuse-2.9.7-16.el8.x86_64 1/2 2022-09-03T05:46:32.027 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 5/5 2022-09-03T05:46:32.027 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:32.028 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:46:32.028 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-mgr-cephadm-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:32.028 INFO:teuthology.orchestra.run.smithi062.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2022-09-03T05:46:32.028 INFO:teuthology.orchestra.run.smithi062.stdout: python3-babel-2.5.1-7.el8.noarch 2022-09-03T05:46:32.029 INFO:teuthology.orchestra.run.smithi062.stdout: python3-jinja2-2.10.1-3.el8.noarch 2022-09-03T05:46:32.029 INFO:teuthology.orchestra.run.smithi062.stdout: python3-natsort-7.1.1-2.el8.noarch 2022-09-03T05:46:32.029 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:32.030 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:32.074 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 2/2 2022-09-03T05:46:32.128 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-fuse 2022-09-03T05:46:32.426 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 2/2 2022-09-03T05:46:32.427 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 1/2 2022-09-03T05:46:32.728 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:01:55 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:32.802 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : fuse-2.9.7-16.el8.x86_64 2/2 2022-09-03T05:46:32.802 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:32.803 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:46:32.803 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 fuse-2.9.7-16.el8.x86_64 2022-09-03T05:46:32.803 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:32.803 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:32.991 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-volume 2022-09-03T05:46:33.570 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:57 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:33.811 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:33.815 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:33.815 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:46:33.816 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:33.816 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:46:33.816 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-fuse x86_64 2:17.0.0-14689.gafd42238.el8 ceph 843 k 2022-09-03T05:46:33.817 INFO:teuthology.orchestra.run.smithi062.stdout:Installing dependencies: 2022-09-03T05:46:33.817 INFO:teuthology.orchestra.run.smithi062.stdout: fuse x86_64 2.9.7-16.el8 CentOS-BaseOS 83 k 2022-09-03T05:46:33.817 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:33.818 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:46:33.818 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:33.818 INFO:teuthology.orchestra.run.smithi062.stdout:Install 2 Packages 2022-09-03T05:46:33.819 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:33.819 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 926 k 2022-09-03T05:46:33.819 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 2.9 M 2022-09-03T05:46:33.819 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:46:33.883 INFO:teuthology.orchestra.run.smithi062.stdout:(1/2): fuse-2.9.7-16.el8.x86_64.rpm 1.2 MB/s | 83 kB 00:00 2022-09-03T05:46:34.583 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:34.584 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:34.584 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2022-09-03T05:46:34.584 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:34.585 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:46:34.585 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-volume noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 263 k 2022-09-03T05:46:34.585 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:34.585 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:46:34.585 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:34.586 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2022-09-03T05:46:34.586 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:34.586 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 263 k 2022-09-03T05:46:34.586 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 1.3 M 2022-09-03T05:46:34.587 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:46:34.766 INFO:teuthology.orchestra.run.smithi062.stdout:(2/2): ceph-fuse-17.0.0-14689.gafd42238.el8.x86 885 kB/s | 843 kB 00:00 2022-09-03T05:46:34.768 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:34.768 INFO:teuthology.orchestra.run.smithi062.stdout:Total 969 kB/s | 926 kB 00:00 2022-09-03T05:46:34.778 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:46:34.801 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:46:34.801 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:46:34.881 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:46:34.884 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:46:35.050 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:46:35.233 INFO:teuthology.orchestra.run.smithi087.stdout:ceph-volume-17.0.0-14689.gafd42238.el8.noarch.r 404 kB/s | 263 kB 00:00 2022-09-03T05:46:35.234 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:35.235 INFO:teuthology.orchestra.run.smithi087.stdout:Total 404 kB/s | 263 kB 00:00 2022-09-03T05:46:35.235 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:46:35.244 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:46:35.245 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:46:35.262 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:46:35.265 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:46:35.322 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : fuse-2.9.7-16.el8.x86_64 1/2 2022-09-03T05:46:35.453 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:46:35.460 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 2/2 2022-09-03T05:46:35.808 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-volume-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:46:35.854 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 2/2 2022-09-03T05:46:35.854 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 1/2 2022-09-03T05:46:36.167 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-volume-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:46:36.282 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : fuse-2.9.7-16.el8.x86_64 2/2 2022-09-03T05:46:36.283 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:36.284 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:46:36.284 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 fuse-2.9.7-16.el8.x86_64 2022-09-03T05:46:36.285 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:36.285 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:36.501 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install ceph-volume 2022-09-03T05:46:36.569 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-volume-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:46:36.569 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:36.570 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:46:36.570 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-volume-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:36.570 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:36.570 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:36.775 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install librados-devel 2022-09-03T05:46:37.119 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:00 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:37.354 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:01 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:38.197 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:38.197 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:38.198 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repository Size 2022-09-03T05:46:38.198 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:38.198 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:46:38.199 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-volume noarch 2:17.0.0-14689.gafd42238.el8 ceph-noarch 263 k 2022-09-03T05:46:38.199 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:38.200 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:46:38.200 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:38.201 INFO:teuthology.orchestra.run.smithi062.stdout:Install 1 Package 2022-09-03T05:46:38.201 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:38.201 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 263 k 2022-09-03T05:46:38.202 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 1.3 M 2022-09-03T05:46:38.203 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:46:38.360 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:38.360 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:38.361 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repo Size 2022-09-03T05:46:38.361 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:38.362 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:46:38.362 INFO:teuthology.orchestra.run.smithi087.stdout: librados-devel x86_64 2:17.0.0-14689.gafd42238.el8 ceph 139 k 2022-09-03T05:46:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:38.363 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:46:38.363 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:38.364 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2022-09-03T05:46:38.364 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:38.365 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 139 k 2022-09-03T05:46:38.365 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 511 k 2022-09-03T05:46:38.365 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:46:38.413 INFO:teuthology.orchestra.run.smithi062.stdout:ceph-volume-17.0.0-14689.gafd42238.el8.noarch.r 1.2 MB/s | 263 kB 00:00 2022-09-03T05:46:38.414 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:38.414 INFO:teuthology.orchestra.run.smithi062.stdout:Total 1.2 MB/s | 263 kB 00:00 2022-09-03T05:46:38.415 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:46:38.424 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:46:38.425 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:46:38.445 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:46:38.448 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:46:38.545 INFO:teuthology.orchestra.run.smithi087.stdout:librados-devel-17.0.0-14689.gafd42238.el8.x86_6 751 kB/s | 139 kB 00:00 2022-09-03T05:46:38.546 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:38.547 INFO:teuthology.orchestra.run.smithi087.stdout:Total 746 kB/s | 139 kB 00:00 2022-09-03T05:46:38.547 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:46:38.551 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:46:38.552 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:46:38.601 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:46:38.604 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:46:38.625 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:46:38.718 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:46:38.886 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : ceph-volume-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:46:38.906 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : librados-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:39.167 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librados-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:39.281 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: ceph-volume-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:46:39.549 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librados-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:39.550 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:39.550 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:46:39.550 INFO:teuthology.orchestra.run.smithi087.stdout: librados-devel-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:46:39.551 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:39.551 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:39.688 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : ceph-volume-2:17.0.0-14689.gafd42238.el8.noarch 1/1 2022-09-03T05:46:39.689 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:39.689 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:46:39.689 INFO:teuthology.orchestra.run.smithi062.stdout: ceph-volume-2:17.0.0-14689.gafd42238.el8.noarch 2022-09-03T05:46:39.690 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:39.690 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:39.721 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install libcephfs2 2022-09-03T05:46:39.895 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install librados-devel 2022-09-03T05:46:40.300 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:04 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:40.508 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:03 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:41.227 INFO:teuthology.orchestra.run.smithi087.stdout:Package libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:41.302 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:41.303 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:46:41.304 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:41.553 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install libcephfs-devel 2022-09-03T05:46:41.664 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:41.666 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:41.666 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repo Size 2022-09-03T05:46:41.667 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:41.667 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:46:41.668 INFO:teuthology.orchestra.run.smithi062.stdout: librados-devel x86_64 2:17.0.0-14689.gafd42238.el8 ceph 139 k 2022-09-03T05:46:41.668 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:41.669 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:46:41.669 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:41.670 INFO:teuthology.orchestra.run.smithi062.stdout:Install 1 Package 2022-09-03T05:46:41.670 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:41.670 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 139 k 2022-09-03T05:46:41.670 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 511 k 2022-09-03T05:46:41.671 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:46:41.970 INFO:teuthology.orchestra.run.smithi062.stdout:librados-devel-17.0.0-14689.gafd42238.el8.x86_6 456 kB/s | 139 kB 00:00 2022-09-03T05:46:41.971 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:41.972 INFO:teuthology.orchestra.run.smithi062.stdout:Total 454 kB/s | 139 kB 00:00 2022-09-03T05:46:41.972 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:46:41.978 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:46:41.978 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:46:42.033 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:46:42.036 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:46:42.122 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:06 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:42.156 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:46:42.366 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : librados-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:42.607 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: librados-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:43.056 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : librados-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:43.056 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:43.057 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:46:43.057 INFO:teuthology.orchestra.run.smithi062.stdout: librados-devel-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:46:43.058 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:43.058 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:43.121 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:43.122 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:43.122 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repo Size 2022-09-03T05:46:43.122 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:43.123 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:46:43.123 INFO:teuthology.orchestra.run.smithi087.stdout: libcephfs-devel x86_64 2:17.0.0-14689.gafd42238.el8 ceph 30 k 2022-09-03T05:46:43.123 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:43.124 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:46:43.124 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:43.124 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2022-09-03T05:46:43.124 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:43.125 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 30 k 2022-09-03T05:46:43.125 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 137 k 2022-09-03T05:46:43.125 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:46:43.282 INFO:teuthology.orchestra.run.smithi087.stdout:libcephfs-devel-17.0.0-14689.gafd42238.el8.x86_ 190 kB/s | 30 kB 00:00 2022-09-03T05:46:43.283 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:43.284 INFO:teuthology.orchestra.run.smithi087.stdout:Total 189 kB/s | 30 kB 00:00 2022-09-03T05:46:43.284 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:46:43.287 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:46:43.287 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:46:43.292 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install libcephfs2 2022-09-03T05:46:43.317 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:46:43.319 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:46:43.375 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:46:43.505 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libcephfs-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:43.716 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libcephfs-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:43.897 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:06 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:44.067 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libcephfs-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:44.068 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:44.068 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:46:44.068 INFO:teuthology.orchestra.run.smithi087.stdout: libcephfs-devel-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:46:44.068 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:44.069 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:44.166 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install librados2 2022-09-03T05:46:44.731 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:08 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:44.892 INFO:teuthology.orchestra.run.smithi062.stdout:Package libcephfs2-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:44.975 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:44.976 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:46:44.976 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:45.064 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install libcephfs-devel 2022-09-03T05:46:45.665 INFO:teuthology.orchestra.run.smithi087.stdout:Package librados2-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:45.667 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:08 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:45.739 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:45.740 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:46:45.741 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:45.821 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install librbd1 2022-09-03T05:46:46.393 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:10 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:46.766 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:46.767 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:46.768 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repo Size 2022-09-03T05:46:46.768 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:46.768 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:46:46.769 INFO:teuthology.orchestra.run.smithi062.stdout: libcephfs-devel x86_64 2:17.0.0-14689.gafd42238.el8 ceph 30 k 2022-09-03T05:46:46.769 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:46.769 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:46:46.769 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:46:46.769 INFO:teuthology.orchestra.run.smithi062.stdout:Install 1 Package 2022-09-03T05:46:46.770 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:46.770 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 30 k 2022-09-03T05:46:46.770 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 137 k 2022-09-03T05:46:46.770 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:46:47.220 INFO:teuthology.orchestra.run.smithi062.stdout:libcephfs-devel-17.0.0-14689.gafd42238.el8.x86_ 67 kB/s | 30 kB 00:00 2022-09-03T05:46:47.220 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:47.221 INFO:teuthology.orchestra.run.smithi062.stdout:Total 67 kB/s | 30 kB 00:00 2022-09-03T05:46:47.221 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:46:47.224 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:46:47.225 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:46:47.261 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:46:47.265 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:46:47.333 INFO:teuthology.orchestra.run.smithi087.stdout:Package librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:47.408 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:47.409 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:46:47.409 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:47.493 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install python3-rados 2022-09-03T05:46:47.510 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:46:48.065 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:12 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:48.799 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : libcephfs-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:48.998 INFO:teuthology.orchestra.run.smithi087.stdout:Package python3-rados-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:49.073 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:49.073 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:46:49.074 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:49.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install python3-rgw 2022-09-03T05:46:49.359 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: libcephfs-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:49.736 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:13 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:49.737 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : libcephfs-devel-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:49.737 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:49.738 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:46:49.738 INFO:teuthology.orchestra.run.smithi062.stdout: libcephfs-devel-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:46:49.738 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:46:49.738 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:49.837 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install librados2 2022-09-03T05:46:50.434 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:13 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:50.661 INFO:teuthology.orchestra.run.smithi087.stdout:Package python3-rgw-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:50.734 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:50.735 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:46:50.736 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:50.818 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install python3-cephfs 2022-09-03T05:46:51.386 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:15 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:51.435 INFO:teuthology.orchestra.run.smithi062.stdout:Package librados2-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:51.518 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:51.519 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:46:51.519 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:51.589 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install librbd1 2022-09-03T05:46:52.189 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:15 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:52.319 INFO:teuthology.orchestra.run.smithi087.stdout:Package python3-cephfs-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:52.391 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:52.392 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:46:52.392 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:52.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install python3-rbd 2022-09-03T05:46:53.190 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:17 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:53.191 INFO:teuthology.orchestra.run.smithi062.stdout:Package librbd1-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:53.276 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:53.277 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:46:53.277 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:53.359 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install python3-rados 2022-09-03T05:46:53.962 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:16 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:54.127 INFO:teuthology.orchestra.run.smithi087.stdout:Package python3-rbd-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:54.203 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:54.204 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2022-09-03T05:46:54.204 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:54.313 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install rbd-fuse 2022-09-03T05:46:54.878 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:18 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:54.968 INFO:teuthology.orchestra.run.smithi062.stdout:Package python3-rados-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:55.050 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:55.058 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:46:55.058 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:55.137 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install python3-rgw 2022-09-03T05:46:55.749 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:18 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:55.885 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:55.887 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:55.887 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repo Size 2022-09-03T05:46:55.887 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:55.887 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:46:55.888 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-fuse x86_64 2:17.0.0-14689.gafd42238.el8 ceph 94 k 2022-09-03T05:46:55.888 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:55.888 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:46:55.888 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:55.889 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2022-09-03T05:46:55.889 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:55.889 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 94 k 2022-09-03T05:46:55.889 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 280 k 2022-09-03T05:46:55.890 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:46:56.332 INFO:teuthology.orchestra.run.smithi087.stdout:rbd-fuse-17.0.0-14689.gafd42238.el8.x86_64.rpm 212 kB/s | 94 kB 00:00 2022-09-03T05:46:56.333 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:46:56.333 INFO:teuthology.orchestra.run.smithi087.stdout:Total 211 kB/s | 94 kB 00:00 2022-09-03T05:46:56.333 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:46:56.343 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:46:56.344 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:46:56.393 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:46:56.396 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:46:56.500 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:46:56.678 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : rbd-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:56.762 INFO:teuthology.orchestra.run.smithi062.stdout:Package python3-rgw-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:56.846 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:56.847 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:46:56.847 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:56.920 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: rbd-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:56.964 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install python3-cephfs 2022-09-03T05:46:57.281 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : rbd-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:46:57.281 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:57.282 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:46:57.282 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:46:57.282 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:57.283 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:46:57.467 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install rbd-mirror 2022-09-03T05:46:57.566 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:20 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:46:58.044 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:22 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:46:58.550 INFO:teuthology.orchestra.run.smithi062.stdout:Package python3-cephfs-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:46:58.634 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:46:58.635 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:46:58.636 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:46:58.717 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install python3-rbd 2022-09-03T05:46:59.050 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:46:59.051 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:59.051 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repo Size 2022-09-03T05:46:59.051 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:59.052 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:46:59.053 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-mirror x86_64 2:17.0.0-14689.gafd42238.el8 ceph 3.5 M 2022-09-03T05:46:59.054 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:59.054 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:46:59.055 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:46:59.055 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2022-09-03T05:46:59.055 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:46:59.056 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 3.5 M 2022-09-03T05:46:59.056 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 16 M 2022-09-03T05:46:59.056 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:46:59.319 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:22 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:47:00.236 INFO:teuthology.orchestra.run.smithi087.stdout:rbd-mirror-17.0.0-14689.gafd42238.el8.x86_64.rp 2.9 MB/s | 3.5 MB 00:01 2022-09-03T05:47:00.237 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:47:00.237 INFO:teuthology.orchestra.run.smithi087.stdout:Total 2.9 MB/s | 3.5 MB 00:01 2022-09-03T05:47:00.238 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:47:00.249 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:47:00.249 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:47:00.310 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:47:00.312 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:47:00.324 INFO:teuthology.orchestra.run.smithi062.stdout:Package python3-rbd-2:17.0.0-14689.gafd42238.el8.x86_64 is already installed. 2022-09-03T05:47:00.408 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:47:00.409 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do. 2022-09-03T05:47:00.410 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:47:00.496 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install rbd-fuse 2022-09-03T05:47:00.751 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:47:00.905 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : rbd-mirror-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:01.098 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:24 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:47:01.301 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: rbd-mirror-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:01.675 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : rbd-mirror-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:01.676 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:01.676 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:47:01.677 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-mirror-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:47:01.677 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:01.678 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:47:01.853 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install rbd-nbd 2022-09-03T05:47:02.179 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:47:02.180 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:47:02.180 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repo Size 2022-09-03T05:47:02.180 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:47:02.180 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:47:02.181 INFO:teuthology.orchestra.run.smithi062.stdout: rbd-fuse x86_64 2:17.0.0-14689.gafd42238.el8 ceph 94 k 2022-09-03T05:47:02.181 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:02.181 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:47:02.181 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:47:02.182 INFO:teuthology.orchestra.run.smithi062.stdout:Install 1 Package 2022-09-03T05:47:02.182 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:02.183 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 94 k 2022-09-03T05:47:02.183 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 280 k 2022-09-03T05:47:02.183 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:47:02.356 INFO:teuthology.orchestra.run.smithi062.stdout:rbd-fuse-17.0.0-14689.gafd42238.el8.x86_64.rpm 537 kB/s | 94 kB 00:00 2022-09-03T05:47:02.357 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:47:02.357 INFO:teuthology.orchestra.run.smithi062.stdout:Total 534 kB/s | 94 kB 00:00 2022-09-03T05:47:02.358 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:47:02.367 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:47:02.368 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:47:02.419 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:47:02.429 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:47:02.438 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:26 ago on Sat 03 Sep 2022 05:44:36 AM UTC. 2022-09-03T05:47:02.528 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:47:02.656 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : rbd-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:02.925 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: rbd-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:03.317 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : rbd-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:03.317 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:03.318 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:47:03.318 INFO:teuthology.orchestra.run.smithi062.stdout: rbd-fuse-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:47:03.318 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:03.318 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:47:03.449 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2022-09-03T05:47:03.450 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:47:03.451 INFO:teuthology.orchestra.run.smithi087.stdout: Package Architecture Version Repository Size 2022-09-03T05:47:03.451 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:47:03.451 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2022-09-03T05:47:03.451 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-nbd x86_64 2:17.0.0-14689.gafd42238.el8 ceph 175 k 2022-09-03T05:47:03.452 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:03.452 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2022-09-03T05:47:03.452 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2022-09-03T05:47:03.452 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2022-09-03T05:47:03.453 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:03.453 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 175 k 2022-09-03T05:47:03.453 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 556 k 2022-09-03T05:47:03.453 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2022-09-03T05:47:03.536 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install rbd-mirror 2022-09-03T05:47:03.673 INFO:teuthology.orchestra.run.smithi087.stdout:rbd-nbd-17.0.0-14689.gafd42238.el8.x86_64.rpm 797 kB/s | 175 kB 00:00 2022-09-03T05:47:03.673 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:47:03.674 INFO:teuthology.orchestra.run.smithi087.stdout:Total 793 kB/s | 175 kB 00:00 2022-09-03T05:47:03.674 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2022-09-03T05:47:03.683 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2022-09-03T05:47:03.683 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2022-09-03T05:47:03.730 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2022-09-03T05:47:03.733 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2022-09-03T05:47:03.876 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2022-09-03T05:47:04.019 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : rbd-nbd-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:04.138 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:27 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:47:04.298 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: rbd-nbd-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:04.706 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : rbd-nbd-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:04.706 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:04.707 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2022-09-03T05:47:04.707 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-nbd-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:47:04.707 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:04.708 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2022-09-03T05:47:04.940 DEBUG:teuthology.parallel:result is None 2022-09-03T05:47:05.217 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:47:05.218 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:47:05.219 INFO:teuthology.orchestra.run.smithi062.stdout: Package Arch Version Repo Size 2022-09-03T05:47:05.219 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:47:05.219 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:47:05.219 INFO:teuthology.orchestra.run.smithi062.stdout: rbd-mirror x86_64 2:17.0.0-14689.gafd42238.el8 ceph 3.5 M 2022-09-03T05:47:05.220 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:05.220 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:47:05.220 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:47:05.220 INFO:teuthology.orchestra.run.smithi062.stdout:Install 1 Package 2022-09-03T05:47:05.221 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:05.221 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 3.5 M 2022-09-03T05:47:05.224 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 16 M 2022-09-03T05:47:05.225 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:47:05.893 INFO:teuthology.orchestra.run.smithi062.stdout:rbd-mirror-17.0.0-14689.gafd42238.el8.x86_64.rp 5.1 MB/s | 3.5 MB 00:00 2022-09-03T05:47:05.895 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:47:05.895 INFO:teuthology.orchestra.run.smithi062.stdout:Total 5.1 MB/s | 3.5 MB 00:00 2022-09-03T05:47:05.896 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:47:05.908 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:47:05.909 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:47:05.973 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:47:05.976 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:47:06.449 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:47:06.623 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : rbd-mirror-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:07.013 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: rbd-mirror-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:07.410 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : rbd-mirror-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:07.411 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:07.411 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:47:07.411 INFO:teuthology.orchestra.run.smithi062.stdout: rbd-mirror-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:47:07.411 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:07.412 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:47:07.637 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum -y install rbd-nbd 2022-09-03T05:47:08.242 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:02:31 ago on Sat 03 Sep 2022 05:44:37 AM UTC. 2022-09-03T05:47:09.327 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved. 2022-09-03T05:47:09.328 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:47:09.328 INFO:teuthology.orchestra.run.smithi062.stdout: Package Architecture Version Repository Size 2022-09-03T05:47:09.328 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:47:09.329 INFO:teuthology.orchestra.run.smithi062.stdout:Installing: 2022-09-03T05:47:09.329 INFO:teuthology.orchestra.run.smithi062.stdout: rbd-nbd x86_64 2:17.0.0-14689.gafd42238.el8 ceph 175 k 2022-09-03T05:47:09.329 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:09.329 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction Summary 2022-09-03T05:47:09.330 INFO:teuthology.orchestra.run.smithi062.stdout:================================================================================ 2022-09-03T05:47:09.330 INFO:teuthology.orchestra.run.smithi062.stdout:Install 1 Package 2022-09-03T05:47:09.331 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:09.331 INFO:teuthology.orchestra.run.smithi062.stdout:Total download size: 175 k 2022-09-03T05:47:09.331 INFO:teuthology.orchestra.run.smithi062.stdout:Installed size: 556 k 2022-09-03T05:47:09.332 INFO:teuthology.orchestra.run.smithi062.stdout:Downloading Packages: 2022-09-03T05:47:09.538 INFO:teuthology.orchestra.run.smithi062.stdout:rbd-nbd-17.0.0-14689.gafd42238.el8.x86_64.rpm 823 kB/s | 175 kB 00:00 2022-09-03T05:47:09.539 INFO:teuthology.orchestra.run.smithi062.stdout:-------------------------------------------------------------------------------- 2022-09-03T05:47:09.540 INFO:teuthology.orchestra.run.smithi062.stdout:Total 819 kB/s | 175 kB 00:00 2022-09-03T05:47:09.540 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction check 2022-09-03T05:47:09.550 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction check succeeded. 2022-09-03T05:47:09.550 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction test 2022-09-03T05:47:09.602 INFO:teuthology.orchestra.run.smithi062.stdout:Transaction test succeeded. 2022-09-03T05:47:09.605 INFO:teuthology.orchestra.run.smithi062.stdout:Running transaction 2022-09-03T05:47:09.723 INFO:teuthology.orchestra.run.smithi062.stdout: Preparing : 1/1 2022-09-03T05:47:10.004 INFO:teuthology.orchestra.run.smithi062.stdout: Installing : rbd-nbd-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:10.270 INFO:teuthology.orchestra.run.smithi062.stdout: Running scriptlet: rbd-nbd-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:10.675 INFO:teuthology.orchestra.run.smithi062.stdout: Verifying : rbd-nbd-2:17.0.0-14689.gafd42238.el8.x86_64 1/1 2022-09-03T05:47:10.676 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:10.676 INFO:teuthology.orchestra.run.smithi062.stdout:Installed: 2022-09-03T05:47:10.677 INFO:teuthology.orchestra.run.smithi062.stdout: rbd-nbd-2:17.0.0-14689.gafd42238.el8.x86_64 2022-09-03T05:47:10.677 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:10.677 INFO:teuthology.orchestra.run.smithi062.stdout:Complete! 2022-09-03T05:47:10.861 DEBUG:teuthology.parallel:result is None 2022-09-03T05:47:10.862 DEBUG:teuthology.orchestra.run.smithi062:> sudo lsb_release -is 2022-09-03T05:47:10.902 INFO:teuthology.orchestra.run.smithi062.stdout:CentOSStream 2022-09-03T05:47:10.904 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-03T05:47:10.906 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=afd422385fab8119a539d289dd1aeaad4c056fe4 2022-09-03T05:47:11.093 DEBUG:teuthology.orchestra.run.smithi062:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2022-09-03T05:47:11.117 INFO:teuthology.orchestra.run.smithi062.stdout:17.0.0-14689.gafd42238.el8 2022-09-03T05:47:11.118 INFO:teuthology.packaging:The installed version of ceph is 17.0.0-14689.gafd42238.el8 2022-09-03T05:47:11.119 INFO:teuthology.task.install:The correct ceph version 17.0.0-14689.gafd42238 is installed. 2022-09-03T05:47:11.120 DEBUG:teuthology.orchestra.run.smithi087:> sudo lsb_release -is 2022-09-03T05:47:11.160 INFO:teuthology.orchestra.run.smithi087.stdout:CentOSStream 2022-09-03T05:47:11.162 DEBUG:teuthology.misc:System to be installed: CentOSStream 2022-09-03T05:47:11.163 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=afd422385fab8119a539d289dd1aeaad4c056fe4 2022-09-03T05:47:11.321 DEBUG:teuthology.orchestra.run.smithi087:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2022-09-03T05:47:11.345 INFO:teuthology.orchestra.run.smithi087.stdout:17.0.0-14689.gafd42238.el8 2022-09-03T05:47:11.350 INFO:teuthology.packaging:The installed version of ceph is 17.0.0-14689.gafd42238.el8 2022-09-03T05:47:11.351 INFO:teuthology.task.install:The correct ceph version 17.0.0-14689.gafd42238 is installed. 2022-09-03T05:47:11.353 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2022-09-03T05:47:11.353 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:11.354 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2022-09-03T05:47:11.383 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:11.383 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2022-09-03T05:47:11.414 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2022-09-03T05:47:11.415 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:11.416 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd of=/usr/bin/daemon-helper 2022-09-03T05:47:11.451 DEBUG:teuthology.orchestra.run.smithi062:> sudo chmod a=rx -- /usr/bin/daemon-helper 2022-09-03T05:47:11.519 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:11.520 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/usr/bin/daemon-helper 2022-09-03T05:47:11.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod a=rx -- /usr/bin/daemon-helper 2022-09-03T05:47:11.615 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2022-09-03T05:47:11.618 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:11.618 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd of=/usr/bin/adjust-ulimits 2022-09-03T05:47:11.648 DEBUG:teuthology.orchestra.run.smithi062:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2022-09-03T05:47:11.733 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:11.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/usr/bin/adjust-ulimits 2022-09-03T05:47:11.764 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2022-09-03T05:47:11.833 INFO:teuthology.run_tasks:Running task ceph... 2022-09-03T05:47:11.925 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2022-09-03T05:47:11.926 DEBUG:teuthology.orchestra.run.smithi062:> sudo chmod 777 /var/log/ceph 2022-09-03T05:47:11.931 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 777 /var/log/ceph 2022-09-03T05:47:11.958 INFO:tasks.ceph:Disabling ceph logrotate... 2022-09-03T05:47:11.958 DEBUG:teuthology.orchestra.run.smithi062:> sudo rm -f -- /etc/logrotate.d/ceph 2022-09-03T05:47:12.003 DEBUG:teuthology.orchestra.run.smithi087:> sudo rm -f -- /etc/logrotate.d/ceph 2022-09-03T05:47:12.031 INFO:tasks.ceph:Creating extra log directories... 2022-09-03T05:47:12.031 DEBUG:teuthology.orchestra.run.smithi062:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2022-09-03T05:47:12.075 DEBUG:teuthology.orchestra.run.smithi087:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2022-09-03T05:47:12.124 INFO:tasks.ceph:Creating ceph cluster c1... 2022-09-03T05:47:12.125 INFO:tasks.ceph:config {'conf': {'client': {'debug rgw': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync data inject err probability': 0.1, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)'], 'cpu_profile': set(), 'cluster': 'c1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2022-09-03T05:47:12.126 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi/7008075', 'branch': 'pr-47848', 'description': 'rgw/multisite/{clusters frontend/beast ignore-pg-availability notify omap_limits overrides realms/three-zone-plus-pubsub supported-random-distro$/{centos_8} tasks/test_multi}', 'email': None, 'first_in_suite': False, 'job_id': '7008075', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'name': 'cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi', 'no_nested_subset': False, 'nuke-on-error': True, 'os_type': 'centos', 'os_version': '8.stream', 'overrides': {'admin_socket': {'branch': 'pr-47848'}, 'ceph': {'conf': {'client': {'debug rgw': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync data inject err probability': 0.1, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)'], 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4', 'wait-for-scrub': False}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4'}}, 'rgw': {'compression type': 'random', 'frontend': 'beast'}, 'rgw-multisite': {'realm': {'is default': True, 'name': 'test-realm'}, 'zonegroups': [{'enabled_features': ['resharding'], 'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'test-zonegroup', 'zones': [{'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'test-zone1'}, {'endpoints': ['c2.client.0'], 'is_default': True, 'name': 'test-zone2'}, {'endpoints': ['c1.client.1'], 'name': 'test-zone3'}, {'endpoints': ['c2.client.1'], 'is_pubsub': True, 'name': 'test-zone4'}]}]}, 'rgw-multisite-tests': {'args': ['tests.py', 'tests_ps.py']}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'wip-qa-rgw-s3select-fixes_related_to_new_csv_parser', 'sha1': 'e20bde8244d2a27c6431acd07e9967aaa8f04f01'}}, 'owner': 'scheduled_cbodley@teuthology', 'priority': 75, 'repo': 'git://git.ceph.com/ceph-ci.git', 'roles': [['c1.mon.a', 'c1.mgr.x', 'c1.osd.0', 'c1.osd.1', 'c1.osd.2', 'c1.client.0', 'c1.client.1'], ['c2.mon.a', 'c2.mgr.x', 'c2.osd.0', 'c2.osd.1', 'c2.osd.2', 'c2.client.0', 'c2.client.1']], 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4', 'sleep_before_teardown': 0, 'suite': 'rgw', 'suite_branch': 'wip-qa-rgw-s3select-fixes_related_to_new_csv_parser', 'suite_path': '/home/teuthworker/src/github.com_cbodley_ceph_e20bde8244d2a27c6431acd07e9967aaa8f04f01/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/cbodley/ceph.git', 'suite_sha1': 'e20bde8244d2a27c6431acd07e9967aaa8f04f01', 'targets': {'smithi062.front.sepia.ceph.com': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSHRk+FEtEk0j2RQdYKdyMR/5SvhQ+BPYn0gwmkL5EAzXE5undqYEXihmsS0eF0BPoaxbe6Iw7OEnb3csdEOS1E5PFlWUcn1eJII8vdIiboGKEUVStcXyfSIi5xvHeLzDMmy0Oe2yj4J+mlHSpkntH9oDX8tjTwNf4wPzVI2hk9WWyB1ZDrdWZxtO5QjkecSTp10ZPZS6tbWJtxAFSEiZl5lMo/gi6fnRnD4uHf2xHMQa87pUknbtbobXHed/NFAKM+a/iBBNy7kdtQsFOjpbK02N+cHuYdAOrd4lTuZK3LPuTnBuw9CXESr03blNlAPK8nz/0mdmAysyIidk00OUflY0oN0aY9qgiyQ/f8jElda8reLmrqq1kFG9P7p1roLg+CBgSS2ytsV8nyLHhA0HJHuGdz9+7hOgfdpWsN2wBRez34hFLzoYhCzdd9HycuDp9dK5A3VaTzQ9owbXSfF/1MIFnqGpeUrtUl3//kuN5JBR03Tu31Ahq93aoEzcRFkE=', 'smithi087.front.sepia.ceph.com': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA2MfLiBfBB1rQjxhGdqrEUq/1La6/wNGaBQ7PWfp8JYfQ54QThAn+gKdHG9f6dimygPtacq48uajMWnuqetlFwFvkL0kiK0e6dWMTVvD58jmrP8ymFkir0uO+4/i+CGGxFAEbuqtCah38yHkyefQ08/UBmZBsQqcN87vAJIBLiF3JjPTrF5nx8L4s09mXEcOMA7TkUxQhCOcIOMzmFF0NNx0dIT9+FGCmHCxnoI3xj4s+o94e2OFcdwXHBmBqGKoNHLpcLsVpIbScZyLkVmLamfLkrmIC7I5bKZO/OxIk7Yhv/QLRGf6+0wdnh0kQ6ZlRmhXbk9hqQ7rv+g6P+0GyAkIgpDw99T5eoGm8GiGB7ERNuwHiUS5zV+HZ0XcxC8AKa0uH58JrSJmM2PvQLnVhjLMjwPWNXtTymDoaSRHDf+cMK0XfQc1QOBvl26wZ/KIYXD+3H6ekFX62KG2d9hPZ2QQf2JWJZUDCzL3A9+W9ortahXotXOx4rzG+iyxI+DU='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': None}, {'ceph': {'cluster': 'c1', 'conf': {'client': {'debug rgw': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync data inject err probability': 0.1, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)'], 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4', 'wait-for-scrub': False}}, {'ceph': {'cluster': 'c2'}}, {'rgw': {'c1.client.0': {'port': 8000}, 'c1.client.1': {'port': 8001}, 'c2.client.0': {'port': 8000}, 'c2.client.1': {'port': 8001}}}, {'rgw-multisite': None}, {'rgw-multisite-tests': {'config': {'reconfigure_delay': 60}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'b58f2c18636eb10faa77ed3614abd00cb85dfc2c', 'timestamp': '2022-09-02_14:33:28', 'tube': 'smithi', 'user': 'cbodley', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.1713'} 2022-09-03T05:47:12.127 DEBUG:teuthology.orchestra.run.smithi062:> install -d -m0755 -- /home/ubuntu/cephtest/c1.data 2022-09-03T05:47:12.145 DEBUG:teuthology.orchestra.run.smithi087:> install -d -m0755 -- /home/ubuntu/cephtest/c1.data 2022-09-03T05:47:12.175 DEBUG:teuthology.orchestra.run.smithi062:> sudo install -d -m0777 -- /var/run/ceph 2022-09-03T05:47:12.219 DEBUG:teuthology.orchestra.run.smithi087:> sudo install -d -m0777 -- /var/run/ceph 2022-09-03T05:47:12.252 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:12.253 DEBUG:teuthology.orchestra.run.smithi062:> dd if=/scratch_devs of=/dev/stdout 2022-09-03T05:47:12.320 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-09-03T05:47:12.321 DEBUG:teuthology.orchestra.run.smithi062:> stat /dev/vg_nvme/lv_1 && sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_1 2022-09-03T05:47:12.394 INFO:teuthology.orchestra.run.smithi062.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2022-09-03T05:47:12.395 INFO:teuthology.orchestra.run.smithi062.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-03T05:47:12.396 INFO:teuthology.orchestra.run.smithi062.stdout:Device: 6h/6d Inode: 369092 Links: 1 2022-09-03T05:47:12.396 INFO:teuthology.orchestra.run.smithi062.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-03T05:47:12.396 INFO:teuthology.orchestra.run.smithi062.stdout:Context: system_u:object_r:device_t:s0 2022-09-03T05:47:12.396 INFO:teuthology.orchestra.run.smithi062.stdout:Access: 2022-09-03 05:47:06.793645855 +0000 2022-09-03T05:47:12.397 INFO:teuthology.orchestra.run.smithi062.stdout:Modify: 2022-09-03 05:43:41.891409840 +0000 2022-09-03T05:47:12.398 INFO:teuthology.orchestra.run.smithi062.stdout:Change: 2022-09-03 05:43:41.891409840 +0000 2022-09-03T05:47:12.398 INFO:teuthology.orchestra.run.smithi062.stdout: Birth: - 2022-09-03T05:47:12.400 INFO:teuthology.orchestra.run.smithi062.stderr:1+0 records in 2022-09-03T05:47:12.400 INFO:teuthology.orchestra.run.smithi062.stderr:1+0 records out 2022-09-03T05:47:12.401 INFO:teuthology.orchestra.run.smithi062.stderr:512 bytes copied, 0.000220582 s, 2.3 MB/s 2022-09-03T05:47:12.401 DEBUG:teuthology.orchestra.run.smithi062:> stat /dev/vg_nvme/lv_2 && sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_2 2022-09-03T05:47:12.459 INFO:teuthology.orchestra.run.smithi062.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2022-09-03T05:47:12.459 INFO:teuthology.orchestra.run.smithi062.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-03T05:47:12.460 INFO:teuthology.orchestra.run.smithi062.stdout:Device: 6h/6d Inode: 361288 Links: 1 2022-09-03T05:47:12.460 INFO:teuthology.orchestra.run.smithi062.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-03T05:47:12.460 INFO:teuthology.orchestra.run.smithi062.stdout:Context: system_u:object_r:device_t:s0 2022-09-03T05:47:12.460 INFO:teuthology.orchestra.run.smithi062.stdout:Access: 2022-09-03 05:47:06.793645855 +0000 2022-09-03T05:47:12.461 INFO:teuthology.orchestra.run.smithi062.stdout:Modify: 2022-09-03 05:43:42.485400035 +0000 2022-09-03T05:47:12.462 INFO:teuthology.orchestra.run.smithi062.stdout:Change: 2022-09-03 05:43:42.485400035 +0000 2022-09-03T05:47:12.462 INFO:teuthology.orchestra.run.smithi062.stdout: Birth: - 2022-09-03T05:47:12.469 INFO:teuthology.orchestra.run.smithi062.stderr:1+0 records in 2022-09-03T05:47:12.469 INFO:teuthology.orchestra.run.smithi062.stderr:1+0 records out 2022-09-03T05:47:12.469 INFO:teuthology.orchestra.run.smithi062.stderr:512 bytes copied, 0.000195801 s, 2.6 MB/s 2022-09-03T05:47:12.471 DEBUG:teuthology.orchestra.run.smithi062:> stat /dev/vg_nvme/lv_3 && sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_3 2022-09-03T05:47:12.532 INFO:teuthology.orchestra.run.smithi062.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2022-09-03T05:47:12.532 INFO:teuthology.orchestra.run.smithi062.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-03T05:47:12.533 INFO:teuthology.orchestra.run.smithi062.stdout:Device: 6h/6d Inode: 367238 Links: 1 2022-09-03T05:47:12.533 INFO:teuthology.orchestra.run.smithi062.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-03T05:47:12.534 INFO:teuthology.orchestra.run.smithi062.stdout:Context: system_u:object_r:device_t:s0 2022-09-03T05:47:12.534 INFO:teuthology.orchestra.run.smithi062.stdout:Access: 2022-09-03 05:47:06.793645855 +0000 2022-09-03T05:47:12.535 INFO:teuthology.orchestra.run.smithi062.stdout:Modify: 2022-09-03 05:43:43.077390262 +0000 2022-09-03T05:47:12.535 INFO:teuthology.orchestra.run.smithi062.stdout:Change: 2022-09-03 05:43:43.077390262 +0000 2022-09-03T05:47:12.535 INFO:teuthology.orchestra.run.smithi062.stdout: Birth: - 2022-09-03T05:47:12.540 INFO:teuthology.orchestra.run.smithi062.stderr:1+0 records in 2022-09-03T05:47:12.541 INFO:teuthology.orchestra.run.smithi062.stderr:1+0 records out 2022-09-03T05:47:12.541 INFO:teuthology.orchestra.run.smithi062.stderr:512 bytes copied, 0.000212786 s, 2.4 MB/s 2022-09-03T05:47:12.544 DEBUG:teuthology.orchestra.run.smithi062:> stat /dev/vg_nvme/lv_4 && sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_4 2022-09-03T05:47:12.608 INFO:teuthology.orchestra.run.smithi062.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2022-09-03T05:47:12.608 INFO:teuthology.orchestra.run.smithi062.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-03T05:47:12.609 INFO:teuthology.orchestra.run.smithi062.stdout:Device: 6h/6d Inode: 370843 Links: 1 2022-09-03T05:47:12.609 INFO:teuthology.orchestra.run.smithi062.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-03T05:47:12.609 INFO:teuthology.orchestra.run.smithi062.stdout:Context: system_u:object_r:device_t:s0 2022-09-03T05:47:12.609 INFO:teuthology.orchestra.run.smithi062.stdout:Access: 2022-09-03 05:47:06.793645855 +0000 2022-09-03T05:47:12.609 INFO:teuthology.orchestra.run.smithi062.stdout:Modify: 2022-09-03 05:43:43.651380786 +0000 2022-09-03T05:47:12.610 INFO:teuthology.orchestra.run.smithi062.stdout:Change: 2022-09-03 05:43:43.651380786 +0000 2022-09-03T05:47:12.610 INFO:teuthology.orchestra.run.smithi062.stdout: Birth: - 2022-09-03T05:47:12.612 INFO:teuthology.orchestra.run.smithi062.stderr:1+0 records in 2022-09-03T05:47:12.613 INFO:teuthology.orchestra.run.smithi062.stderr:1+0 records out 2022-09-03T05:47:12.613 INFO:teuthology.orchestra.run.smithi062.stderr:512 bytes copied, 0.000190344 s, 2.7 MB/s 2022-09-03T05:47:12.616 INFO:tasks.ceph:osd dev map: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2022-09-03T05:47:12.617 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi062.front.sepia.ceph.com'): {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'}} 2022-09-03T05:47:12.617 INFO:tasks.ceph:Generating config... 2022-09-03T05:47:12.620 INFO:tasks.ceph:[client] debug rgw = 20 2022-09-03T05:47:12.631 INFO:tasks.ceph:[client] debug rgw sync = 20 2022-09-03T05:47:12.632 INFO:tasks.ceph:[client] rgw crypt require ssl = False 2022-09-03T05:47:12.632 INFO:tasks.ceph:[client] rgw crypt s3 kms backend = testing 2022-09-03T05:47:12.632 INFO:tasks.ceph:[client] rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2022-09-03T05:47:12.633 INFO:tasks.ceph:[client] rgw curl low speed time = 300 2022-09-03T05:47:12.633 INFO:tasks.ceph:[client] rgw data log num shards = 4 2022-09-03T05:47:12.634 INFO:tasks.ceph:[client] rgw md log max shards = 4 2022-09-03T05:47:12.636 INFO:tasks.ceph:[client] rgw sync data inject err probability = 0.1 2022-09-03T05:47:12.636 INFO:tasks.ceph:[client] rgw sync log trim interval = 0 2022-09-03T05:47:12.636 INFO:tasks.ceph:[client] rgw sync meta inject err probability = 0.1 2022-09-03T05:47:12.637 INFO:tasks.ceph:[client] rgw sync obj etag verify = True 2022-09-03T05:47:12.637 INFO:tasks.ceph:[client] setgroup = ceph 2022-09-03T05:47:12.637 INFO:tasks.ceph:[client] setuser = ceph 2022-09-03T05:47:12.638 INFO:tasks.ceph:[client.0] rgw data notify interval msec = 0 2022-09-03T05:47:12.638 INFO:tasks.ceph:[mgr] debug mgr = 20 2022-09-03T05:47:12.638 INFO:tasks.ceph:[mgr] debug ms = 1 2022-09-03T05:47:12.639 INFO:tasks.ceph:[mon] debug mon = 20 2022-09-03T05:47:12.639 INFO:tasks.ceph:[mon] debug ms = 1 2022-09-03T05:47:12.639 INFO:tasks.ceph:[mon] debug paxos = 20 2022-09-03T05:47:12.640 INFO:tasks.ceph:[osd] debug ms = 1 2022-09-03T05:47:12.640 INFO:tasks.ceph:[osd] debug osd = 20 2022-09-03T05:47:12.641 INFO:tasks.ceph:[osd] rocksdb delete range threshold = 0 2022-09-03T05:47:12.641 INFO:tasks.ceph:[osd.0] osd_max_omap_entries_per_request = 10 2022-09-03T05:47:12.641 INFO:tasks.ceph:[osd.1] osd_max_omap_entries_per_request = 1000 2022-09-03T05:47:12.642 INFO:tasks.ceph:[osd.2] osd_max_omap_entries_per_request = 10000 2022-09-03T05:47:12.642 INFO:tasks.ceph:Setting up c1.mon.a... 2022-09-03T05:47:12.643 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/c1.keyring 2022-09-03T05:47:12.733 INFO:teuthology.orchestra.run.smithi062.stdout:creating /etc/ceph/c1.keyring 2022-09-03T05:47:12.736 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/c1.keyring 2022-09-03T05:47:12.794 DEBUG:teuthology.orchestra.run.smithi062:> sudo chmod 0644 /etc/ceph/c1.keyring 2022-09-03T05:47:12.865 DEBUG:tasks.ceph:Ceph mon addresses: [('c1.mon.a', '172.21.15.62')] 2022-09-03T05:47:12.866 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.62'}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'debug ms': 1, 'debug osd': 20, 'rocksdb delete range threshold': 0}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rgw': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync data inject err probability': 0.1, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mon.a': {}, 'client.0': {'rgw data notify interval msec': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}} 2022-09-03T05:47:12.868 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:12.869 DEBUG:teuthology.orchestra.run.smithi062:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2022-09-03T05:47:12.923 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.62 --print /home/ubuntu/cephtest/c1.monmap 2022-09-03T05:47:13.006 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:47:13.006 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:47:13.007 INFO:teuthology.orchestra.run.smithi062.stdout:monmaptool: monmap file /home/ubuntu/cephtest/c1.monmap 2022-09-03T05:47:13.007 INFO:teuthology.orchestra.run.smithi062.stdout:monmaptool: generated fsid 6b295c6b-f22c-47e6-9e05-dafe77fe2ff7 2022-09-03T05:47:13.007 INFO:teuthology.orchestra.run.smithi062.stdout:setting min_mon_release = octopus 2022-09-03T05:47:13.008 INFO:teuthology.orchestra.run.smithi062.stdout:epoch 0 2022-09-03T05:47:13.008 INFO:teuthology.orchestra.run.smithi062.stdout:fsid 6b295c6b-f22c-47e6-9e05-dafe77fe2ff7 2022-09-03T05:47:13.008 INFO:teuthology.orchestra.run.smithi062.stdout:last_changed 2022-09-03T05:47:13.005836+0000 2022-09-03T05:47:13.008 INFO:teuthology.orchestra.run.smithi062.stdout:created 2022-09-03T05:47:13.005836+0000 2022-09-03T05:47:13.008 INFO:teuthology.orchestra.run.smithi062.stdout:min_mon_release 15 (octopus) 2022-09-03T05:47:13.009 INFO:teuthology.orchestra.run.smithi062.stdout:election_strategy: 1 2022-09-03T05:47:13.009 INFO:teuthology.orchestra.run.smithi062.stdout:0: [v2:172.21.15.62:3300/0,v1:172.21.15.62:6789/0] mon.a 2022-09-03T05:47:13.009 INFO:teuthology.orchestra.run.smithi062.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/c1.monmap (1 monitors) 2022-09-03T05:47:13.013 DEBUG:teuthology.orchestra.run.smithi062:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2022-09-03T05:47:13.098 INFO:tasks.ceph:Writing /etc/ceph/c1.conf for FSID 6b295c6b-f22c-47e6-9e05-dafe77fe2ff7... 2022-09-03T05:47:13.100 DEBUG:teuthology.orchestra.run.smithi062:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c1.conf && sudo chmod 0644 /etc/ceph/c1.conf > /dev/null 2022-09-03T05:47:13.136 DEBUG:teuthology.orchestra.run.smithi087:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c1.conf && sudo chmod 0644 /etc/ceph/c1.conf > /dev/null 2022-09-03T05:47:13.179 INFO:teuthology.orchestra.run.smithi062.stdout:[global] 2022-09-03T05:47:13.180 INFO:teuthology.orchestra.run.smithi062.stdout: chdir = "" 2022-09-03T05:47:13.180 INFO:teuthology.orchestra.run.smithi062.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2022-09-03T05:47:13.181 INFO:teuthology.orchestra.run.smithi062.stdout: auth supported = cephx 2022-09-03T05:47:13.181 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.182 INFO:teuthology.orchestra.run.smithi062.stdout: filestore xattr use omap = true 2022-09-03T05:47:13.192 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.193 INFO:teuthology.orchestra.run.smithi062.stdout: mon clock drift allowed = 1.000 2022-09-03T05:47:13.193 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.194 INFO:teuthology.orchestra.run.smithi062.stdout: osd crush chooseleaf type = 0 2022-09-03T05:47:13.194 INFO:teuthology.orchestra.run.smithi062.stdout: auth debug = true 2022-09-03T05:47:13.195 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.195 INFO:teuthology.orchestra.run.smithi062.stdout: ms die on old message = true 2022-09-03T05:47:13.196 INFO:teuthology.orchestra.run.smithi062.stdout: ms die on bug = true 2022-09-03T05:47:13.196 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.196 INFO:teuthology.orchestra.run.smithi062.stdout: mon max pg per osd = 10000 # >= luminous 2022-09-03T05:47:13.197 INFO:teuthology.orchestra.run.smithi062.stdout: mon pg warn max object skew = 0 2022-09-03T05:47:13.197 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.199 INFO:teuthology.orchestra.run.smithi062.stdout: # disable pg_autoscaler by default for new pools 2022-09-03T05:47:13.199 INFO:teuthology.orchestra.run.smithi062.stdout: osd_pool_default_pg_autoscale_mode = off 2022-09-03T05:47:13.199 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.200 INFO:teuthology.orchestra.run.smithi062.stdout: osd pool default size = 2 2022-09-03T05:47:13.200 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.201 INFO:teuthology.orchestra.run.smithi062.stdout: mon osd allow primary affinity = true 2022-09-03T05:47:13.201 INFO:teuthology.orchestra.run.smithi062.stdout: mon osd allow pg remap = true 2022-09-03T05:47:13.202 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on legacy crush tunables = false 2022-09-03T05:47:13.202 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on crush straw calc version zero = false 2022-09-03T05:47:13.203 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on no sortbitwise = false 2022-09-03T05:47:13.203 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on osd down out interval zero = false 2022-09-03T05:47:13.204 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on too few osds = false 2022-09-03T05:47:13.204 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2022-09-03T05:47:13.205 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_pool_no_redundancy = false 2022-09-03T05:47:13.205 INFO:teuthology.orchestra.run.smithi062.stdout: mon_allow_pool_size_one = true 2022-09-03T05:47:13.205 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.206 INFO:teuthology.orchestra.run.smithi062.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2022-09-03T05:47:13.206 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.207 INFO:teuthology.orchestra.run.smithi062.stdout: osd default data pool replay window = 5 2022-09-03T05:47:13.207 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.208 INFO:teuthology.orchestra.run.smithi062.stdout: mon allow pool delete = true 2022-09-03T05:47:13.208 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.209 INFO:teuthology.orchestra.run.smithi062.stdout: mon cluster log file level = debug 2022-09-03T05:47:13.210 INFO:teuthology.orchestra.run.smithi062.stdout: debug asserts on shutdown = true 2022-09-03T05:47:13.211 INFO:teuthology.orchestra.run.smithi062.stdout: mon health detail to clog = false 2022-09-03T05:47:13.211 INFO:teuthology.orchestra.run.smithi062.stdout: mon host = 172.21.15.62 2022-09-03T05:47:13.212 INFO:teuthology.orchestra.run.smithi062.stdout: fsid = 6b295c6b-f22c-47e6-9e05-dafe77fe2ff7 2022-09-03T05:47:13.212 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.213 INFO:teuthology.orchestra.run.smithi062.stdout:[osd] 2022-09-03T05:47:13.213 INFO:teuthology.orchestra.run.smithi062.stdout: osd journal size = 100 2022-09-03T05:47:13.213 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.214 INFO:teuthology.orchestra.run.smithi062.stdout: osd scrub load threshold = 5.0 2022-09-03T05:47:13.214 INFO:teuthology.orchestra.run.smithi062.stdout: osd scrub max interval = 600 2022-09-03T05:47:13.214 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.215 INFO:teuthology.orchestra.run.smithi062.stdout: osd recover clone overlap = true 2022-09-03T05:47:13.215 INFO:teuthology.orchestra.run.smithi062.stdout: osd recovery max chunk = 1048576 2022-09-03T05:47:13.215 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.215 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug shutdown = true 2022-09-03T05:47:13.216 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug op order = true 2022-09-03T05:47:13.216 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug verify stray on activate = true 2022-09-03T05:47:13.216 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.216 INFO:teuthology.orchestra.run.smithi062.stdout: osd open classes on start = true 2022-09-03T05:47:13.217 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug pg log writeout = true 2022-09-03T05:47:13.217 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.217 INFO:teuthology.orchestra.run.smithi062.stdout: osd deep scrub update digest min age = 30 2022-09-03T05:47:13.217 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.218 INFO:teuthology.orchestra.run.smithi062.stdout: osd map max advance = 10 2022-09-03T05:47:13.218 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.218 INFO:teuthology.orchestra.run.smithi062.stdout: journal zero on create = true 2022-09-03T05:47:13.218 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.218 INFO:teuthology.orchestra.run.smithi062.stdout: filestore ondisk finisher threads = 3 2022-09-03T05:47:13.219 INFO:teuthology.orchestra.run.smithi062.stdout: filestore apply finisher threads = 3 2022-09-03T05:47:13.219 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.220 INFO:teuthology.orchestra.run.smithi062.stdout: bdev debug aio = true 2022-09-03T05:47:13.220 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug misdirected ops = true 2022-09-03T05:47:13.220 INFO:teuthology.orchestra.run.smithi062.stdout: debug ms = 1 2022-09-03T05:47:13.220 INFO:teuthology.orchestra.run.smithi062.stdout: debug osd = 20 2022-09-03T05:47:13.221 INFO:teuthology.orchestra.run.smithi062.stdout: rocksdb delete range threshold = 0 2022-09-03T05:47:13.221 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.221 INFO:teuthology.orchestra.run.smithi062.stdout:[mgr] 2022-09-03T05:47:13.221 INFO:teuthology.orchestra.run.smithi062.stdout: debug ms = 1 2022-09-03T05:47:13.221 INFO:teuthology.orchestra.run.smithi062.stdout: debug mgr = 20 2022-09-03T05:47:13.222 INFO:teuthology.orchestra.run.smithi062.stdout: debug mon = 20 2022-09-03T05:47:13.222 INFO:teuthology.orchestra.run.smithi062.stdout: debug auth = 20 2022-09-03T05:47:13.222 INFO:teuthology.orchestra.run.smithi062.stdout: mon reweight min pgs per osd = 4 2022-09-03T05:47:13.222 INFO:teuthology.orchestra.run.smithi062.stdout: mon reweight min bytes per osd = 10 2022-09-03T05:47:13.223 INFO:teuthology.orchestra.run.smithi062.stdout: mgr/telemetry/nag = false 2022-09-03T05:47:13.223 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.223 INFO:teuthology.orchestra.run.smithi062.stdout:[mon] 2022-09-03T05:47:13.223 INFO:teuthology.orchestra.run.smithi062.stdout: debug ms = 1 2022-09-03T05:47:13.224 INFO:teuthology.orchestra.run.smithi062.stdout: debug mon = 20 2022-09-03T05:47:13.224 INFO:teuthology.orchestra.run.smithi062.stdout: debug paxos = 20 2022-09-03T05:47:13.224 INFO:teuthology.orchestra.run.smithi062.stdout: debug auth = 20 2022-09-03T05:47:13.224 INFO:teuthology.orchestra.run.smithi062.stdout: mon data avail warn = 5 2022-09-03T05:47:13.225 INFO:teuthology.orchestra.run.smithi062.stdout: mon mgr mkfs grace = 240 2022-09-03T05:47:13.225 INFO:teuthology.orchestra.run.smithi062.stdout: mon reweight min pgs per osd = 4 2022-09-03T05:47:13.225 INFO:teuthology.orchestra.run.smithi062.stdout: mon osd reporter subtree level = osd 2022-09-03T05:47:13.225 INFO:teuthology.orchestra.run.smithi062.stdout: mon osd prime pg temp = true 2022-09-03T05:47:13.226 INFO:teuthology.orchestra.run.smithi062.stdout: mon reweight min bytes per osd = 10 2022-09-03T05:47:13.226 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.226 INFO:teuthology.orchestra.run.smithi062.stdout: # rotate auth tickets quickly to exercise renewal paths 2022-09-03T05:47:13.227 INFO:teuthology.orchestra.run.smithi062.stdout: auth mon ticket ttl = 660 # 11m 2022-09-03T05:47:13.227 INFO:teuthology.orchestra.run.smithi062.stdout: auth service ticket ttl = 240 # 4m 2022-09-03T05:47:13.228 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.228 INFO:teuthology.orchestra.run.smithi062.stdout: # don't complain about insecure global_id in the test suite 2022-09-03T05:47:13.228 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_insecure_global_id_reclaim = false 2022-09-03T05:47:13.228 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2022-09-03T05:47:13.228 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.229 INFO:teuthology.orchestra.run.smithi062.stdout: # 1m isn't quite enough 2022-09-03T05:47:13.229 INFO:teuthology.orchestra.run.smithi062.stdout: mon_down_mkfs_grace = 2m 2022-09-03T05:47:13.229 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.231 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_filestore_osds = false 2022-09-03T05:47:13.232 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:13.232 INFO:teuthology.orchestra.run.smithi062.stdout:[client] 2022-09-03T05:47:13.232 INFO:teuthology.orchestra.run.smithi062.stdout: rgw cache enabled = true 2022-09-03T05:47:13.232 INFO:teuthology.orchestra.run.smithi062.stdout: rgw enable ops log = true 2022-09-03T05:47:13.233 INFO:teuthology.orchestra.run.smithi062.stdout: rgw enable usage log = true 2022-09-03T05:47:13.233 INFO:teuthology.orchestra.run.smithi062.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2022-09-03T05:47:13.233 INFO:teuthology.orchestra.run.smithi062.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2022-09-03T05:47:13.233 INFO:teuthology.orchestra.run.smithi062.stdout: debug rgw = 20 2022-09-03T05:47:13.234 INFO:teuthology.orchestra.run.smithi062.stdout: debug rgw sync = 20 2022-09-03T05:47:13.234 INFO:teuthology.orchestra.run.smithi062.stdout: rgw crypt require ssl = False 2022-09-03T05:47:13.234 INFO:teuthology.orchestra.run.smithi062.stdout: rgw crypt s3 kms backend = testing 2022-09-03T05:47:13.234 INFO:teuthology.orchestra.run.smithi062.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2022-09-03T05:47:13.234 INFO:teuthology.orchestra.run.smithi062.stdout: rgw curl low speed time = 300 2022-09-03T05:47:13.235 INFO:teuthology.orchestra.run.smithi062.stdout: rgw data log num shards = 4 2022-09-03T05:47:13.235 INFO:teuthology.orchestra.run.smithi062.stdout: rgw md log max shards = 4 2022-09-03T05:47:13.235 INFO:teuthology.orchestra.run.smithi062.stdout: rgw sync data inject err probability = 0.1 2022-09-03T05:47:13.235 INFO:teuthology.orchestra.run.smithi062.stdout: rgw sync log trim interval = 0 2022-09-03T05:47:13.236 INFO:teuthology.orchestra.run.smithi062.stdout: rgw sync meta inject err probability = 0.1 2022-09-03T05:47:13.236 INFO:teuthology.orchestra.run.smithi062.stdout: rgw sync obj etag verify = True 2022-09-03T05:47:13.237 INFO:teuthology.orchestra.run.smithi062.stdout: setgroup = ceph 2022-09-03T05:47:13.237 INFO:teuthology.orchestra.run.smithi062.stdout: setuser = ceph 2022-09-03T05:47:13.238 INFO:teuthology.orchestra.run.smithi062.stdout:[mon.a] 2022-09-03T05:47:13.238 INFO:teuthology.orchestra.run.smithi062.stdout:[client.0] 2022-09-03T05:47:13.238 INFO:teuthology.orchestra.run.smithi062.stdout: rgw data notify interval msec = 0 2022-09-03T05:47:13.238 INFO:teuthology.orchestra.run.smithi062.stdout:[osd.0] 2022-09-03T05:47:13.239 INFO:teuthology.orchestra.run.smithi062.stdout: osd_max_omap_entries_per_request = 10 2022-09-03T05:47:13.239 INFO:teuthology.orchestra.run.smithi062.stdout:[osd.1] 2022-09-03T05:47:13.239 INFO:teuthology.orchestra.run.smithi062.stdout: osd_max_omap_entries_per_request = 1000 2022-09-03T05:47:13.239 INFO:teuthology.orchestra.run.smithi062.stdout:[osd.2] 2022-09-03T05:47:13.240 INFO:teuthology.orchestra.run.smithi062.stdout: osd_max_omap_entries_per_request = 10000 2022-09-03T05:47:13.242 INFO:teuthology.orchestra.run.smithi087.stdout:[global] 2022-09-03T05:47:13.242 INFO:teuthology.orchestra.run.smithi087.stdout: chdir = "" 2022-09-03T05:47:13.243 INFO:teuthology.orchestra.run.smithi087.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2022-09-03T05:47:13.244 INFO:teuthology.orchestra.run.smithi087.stdout: auth supported = cephx 2022-09-03T05:47:13.245 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.246 INFO:teuthology.orchestra.run.smithi087.stdout: filestore xattr use omap = true 2022-09-03T05:47:13.246 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.246 INFO:teuthology.orchestra.run.smithi087.stdout: mon clock drift allowed = 1.000 2022-09-03T05:47:13.247 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.247 INFO:teuthology.orchestra.run.smithi087.stdout: osd crush chooseleaf type = 0 2022-09-03T05:47:13.247 INFO:teuthology.orchestra.run.smithi087.stdout: auth debug = true 2022-09-03T05:47:13.248 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.248 INFO:teuthology.orchestra.run.smithi087.stdout: ms die on old message = true 2022-09-03T05:47:13.248 INFO:teuthology.orchestra.run.smithi087.stdout: ms die on bug = true 2022-09-03T05:47:13.248 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.248 INFO:teuthology.orchestra.run.smithi087.stdout: mon max pg per osd = 10000 # >= luminous 2022-09-03T05:47:13.249 INFO:teuthology.orchestra.run.smithi087.stdout: mon pg warn max object skew = 0 2022-09-03T05:47:13.249 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.249 INFO:teuthology.orchestra.run.smithi087.stdout: # disable pg_autoscaler by default for new pools 2022-09-03T05:47:13.249 INFO:teuthology.orchestra.run.smithi087.stdout: osd_pool_default_pg_autoscale_mode = off 2022-09-03T05:47:13.250 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.250 INFO:teuthology.orchestra.run.smithi087.stdout: osd pool default size = 2 2022-09-03T05:47:13.251 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.251 INFO:teuthology.orchestra.run.smithi087.stdout: mon osd allow primary affinity = true 2022-09-03T05:47:13.251 INFO:teuthology.orchestra.run.smithi087.stdout: mon osd allow pg remap = true 2022-09-03T05:47:13.252 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on legacy crush tunables = false 2022-09-03T05:47:13.252 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on crush straw calc version zero = false 2022-09-03T05:47:13.253 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on no sortbitwise = false 2022-09-03T05:47:13.253 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on osd down out interval zero = false 2022-09-03T05:47:13.254 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on too few osds = false 2022-09-03T05:47:13.255 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2022-09-03T05:47:13.255 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_pool_no_redundancy = false 2022-09-03T05:47:13.255 INFO:teuthology.orchestra.run.smithi087.stdout: mon_allow_pool_size_one = true 2022-09-03T05:47:13.255 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.256 INFO:teuthology.orchestra.run.smithi087.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2022-09-03T05:47:13.256 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.256 INFO:teuthology.orchestra.run.smithi087.stdout: osd default data pool replay window = 5 2022-09-03T05:47:13.256 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.257 INFO:teuthology.orchestra.run.smithi087.stdout: mon allow pool delete = true 2022-09-03T05:47:13.257 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.258 INFO:teuthology.orchestra.run.smithi087.stdout: mon cluster log file level = debug 2022-09-03T05:47:13.258 INFO:teuthology.orchestra.run.smithi087.stdout: debug asserts on shutdown = true 2022-09-03T05:47:13.258 INFO:teuthology.orchestra.run.smithi087.stdout: mon health detail to clog = false 2022-09-03T05:47:13.258 INFO:teuthology.orchestra.run.smithi087.stdout: mon host = 172.21.15.62 2022-09-03T05:47:13.259 INFO:teuthology.orchestra.run.smithi087.stdout: fsid = 6b295c6b-f22c-47e6-9e05-dafe77fe2ff7 2022-09-03T05:47:13.259 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.259 INFO:teuthology.orchestra.run.smithi087.stdout:[osd] 2022-09-03T05:47:13.259 INFO:teuthology.orchestra.run.smithi087.stdout: osd journal size = 100 2022-09-03T05:47:13.260 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.260 INFO:teuthology.orchestra.run.smithi087.stdout: osd scrub load threshold = 5.0 2022-09-03T05:47:13.261 INFO:teuthology.orchestra.run.smithi087.stdout: osd scrub max interval = 600 2022-09-03T05:47:13.261 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.261 INFO:teuthology.orchestra.run.smithi087.stdout: osd recover clone overlap = true 2022-09-03T05:47:13.261 INFO:teuthology.orchestra.run.smithi087.stdout: osd recovery max chunk = 1048576 2022-09-03T05:47:13.262 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.262 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug shutdown = true 2022-09-03T05:47:13.262 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug op order = true 2022-09-03T05:47:13.263 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug verify stray on activate = true 2022-09-03T05:47:13.263 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.263 INFO:teuthology.orchestra.run.smithi087.stdout: osd open classes on start = true 2022-09-03T05:47:13.264 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug pg log writeout = true 2022-09-03T05:47:13.264 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.265 INFO:teuthology.orchestra.run.smithi087.stdout: osd deep scrub update digest min age = 30 2022-09-03T05:47:13.265 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.266 INFO:teuthology.orchestra.run.smithi087.stdout: osd map max advance = 10 2022-09-03T05:47:13.266 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.266 INFO:teuthology.orchestra.run.smithi087.stdout: journal zero on create = true 2022-09-03T05:47:13.267 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.267 INFO:teuthology.orchestra.run.smithi087.stdout: filestore ondisk finisher threads = 3 2022-09-03T05:47:13.268 INFO:teuthology.orchestra.run.smithi087.stdout: filestore apply finisher threads = 3 2022-09-03T05:47:13.268 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.269 INFO:teuthology.orchestra.run.smithi087.stdout: bdev debug aio = true 2022-09-03T05:47:13.269 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug misdirected ops = true 2022-09-03T05:47:13.270 INFO:teuthology.orchestra.run.smithi087.stdout: debug ms = 1 2022-09-03T05:47:13.270 INFO:teuthology.orchestra.run.smithi087.stdout: debug osd = 20 2022-09-03T05:47:13.271 INFO:teuthology.orchestra.run.smithi087.stdout: rocksdb delete range threshold = 0 2022-09-03T05:47:13.271 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.272 INFO:teuthology.orchestra.run.smithi087.stdout:[mgr] 2022-09-03T05:47:13.272 INFO:teuthology.orchestra.run.smithi087.stdout: debug ms = 1 2022-09-03T05:47:13.272 INFO:teuthology.orchestra.run.smithi087.stdout: debug mgr = 20 2022-09-03T05:47:13.272 INFO:teuthology.orchestra.run.smithi087.stdout: debug mon = 20 2022-09-03T05:47:13.273 INFO:teuthology.orchestra.run.smithi087.stdout: debug auth = 20 2022-09-03T05:47:13.273 INFO:teuthology.orchestra.run.smithi087.stdout: mon reweight min pgs per osd = 4 2022-09-03T05:47:13.273 INFO:teuthology.orchestra.run.smithi087.stdout: mon reweight min bytes per osd = 10 2022-09-03T05:47:13.274 INFO:teuthology.orchestra.run.smithi087.stdout: mgr/telemetry/nag = false 2022-09-03T05:47:13.274 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.274 INFO:teuthology.orchestra.run.smithi087.stdout:[mon] 2022-09-03T05:47:13.275 INFO:teuthology.orchestra.run.smithi087.stdout: debug ms = 1 2022-09-03T05:47:13.279 INFO:teuthology.orchestra.run.smithi087.stdout: debug mon = 20 2022-09-03T05:47:13.287 INFO:teuthology.orchestra.run.smithi087.stdout: debug paxos = 20 2022-09-03T05:47:13.287 INFO:teuthology.orchestra.run.smithi087.stdout: debug auth = 20 2022-09-03T05:47:13.287 INFO:teuthology.orchestra.run.smithi087.stdout: mon data avail warn = 5 2022-09-03T05:47:13.288 INFO:teuthology.orchestra.run.smithi087.stdout: mon mgr mkfs grace = 240 2022-09-03T05:47:13.288 INFO:teuthology.orchestra.run.smithi087.stdout: mon reweight min pgs per osd = 4 2022-09-03T05:47:13.288 INFO:teuthology.orchestra.run.smithi087.stdout: mon osd reporter subtree level = osd 2022-09-03T05:47:13.288 INFO:teuthology.orchestra.run.smithi087.stdout: mon osd prime pg temp = true 2022-09-03T05:47:13.289 INFO:teuthology.orchestra.run.smithi087.stdout: mon reweight min bytes per osd = 10 2022-09-03T05:47:13.289 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.289 INFO:teuthology.orchestra.run.smithi087.stdout: # rotate auth tickets quickly to exercise renewal paths 2022-09-03T05:47:13.289 INFO:teuthology.orchestra.run.smithi087.stdout: auth mon ticket ttl = 660 # 11m 2022-09-03T05:47:13.290 INFO:teuthology.orchestra.run.smithi087.stdout: auth service ticket ttl = 240 # 4m 2022-09-03T05:47:13.290 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.290 INFO:teuthology.orchestra.run.smithi087.stdout: # don't complain about insecure global_id in the test suite 2022-09-03T05:47:13.290 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_insecure_global_id_reclaim = false 2022-09-03T05:47:13.291 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2022-09-03T05:47:13.291 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.291 INFO:teuthology.orchestra.run.smithi087.stdout: # 1m isn't quite enough 2022-09-03T05:47:13.291 INFO:teuthology.orchestra.run.smithi087.stdout: mon_down_mkfs_grace = 2m 2022-09-03T05:47:13.292 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.292 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_filestore_osds = false 2022-09-03T05:47:13.292 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:13.292 INFO:teuthology.orchestra.run.smithi087.stdout:[client] 2022-09-03T05:47:13.293 INFO:teuthology.orchestra.run.smithi087.stdout: rgw cache enabled = true 2022-09-03T05:47:13.293 INFO:teuthology.orchestra.run.smithi087.stdout: rgw enable ops log = true 2022-09-03T05:47:13.293 INFO:teuthology.orchestra.run.smithi087.stdout: rgw enable usage log = true 2022-09-03T05:47:13.293 INFO:teuthology.orchestra.run.smithi087.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2022-09-03T05:47:13.294 INFO:teuthology.orchestra.run.smithi087.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2022-09-03T05:47:13.294 INFO:teuthology.orchestra.run.smithi087.stdout: debug rgw = 20 2022-09-03T05:47:13.294 INFO:teuthology.orchestra.run.smithi087.stdout: debug rgw sync = 20 2022-09-03T05:47:13.295 INFO:teuthology.orchestra.run.smithi087.stdout: rgw crypt require ssl = False 2022-09-03T05:47:13.295 INFO:teuthology.orchestra.run.smithi087.stdout: rgw crypt s3 kms backend = testing 2022-09-03T05:47:13.295 INFO:teuthology.orchestra.run.smithi087.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2022-09-03T05:47:13.296 INFO:teuthology.orchestra.run.smithi087.stdout: rgw curl low speed time = 300 2022-09-03T05:47:13.297 INFO:teuthology.orchestra.run.smithi087.stdout: rgw data log num shards = 4 2022-09-03T05:47:13.298 INFO:teuthology.orchestra.run.smithi087.stdout: rgw md log max shards = 4 2022-09-03T05:47:13.298 INFO:teuthology.orchestra.run.smithi087.stdout: rgw sync data inject err probability = 0.1 2022-09-03T05:47:13.299 INFO:teuthology.orchestra.run.smithi087.stdout: rgw sync log trim interval = 0 2022-09-03T05:47:13.299 INFO:teuthology.orchestra.run.smithi087.stdout: rgw sync meta inject err probability = 0.1 2022-09-03T05:47:13.299 INFO:teuthology.orchestra.run.smithi087.stdout: rgw sync obj etag verify = True 2022-09-03T05:47:13.300 INFO:teuthology.orchestra.run.smithi087.stdout: setgroup = ceph 2022-09-03T05:47:13.300 INFO:teuthology.orchestra.run.smithi087.stdout: setuser = ceph 2022-09-03T05:47:13.301 INFO:teuthology.orchestra.run.smithi087.stdout:[mon.a] 2022-09-03T05:47:13.301 INFO:teuthology.orchestra.run.smithi087.stdout:[client.0] 2022-09-03T05:47:13.301 INFO:teuthology.orchestra.run.smithi087.stdout: rgw data notify interval msec = 0 2022-09-03T05:47:13.301 INFO:teuthology.orchestra.run.smithi087.stdout:[osd.0] 2022-09-03T05:47:13.302 INFO:teuthology.orchestra.run.smithi087.stdout: osd_max_omap_entries_per_request = 10 2022-09-03T05:47:13.302 INFO:teuthology.orchestra.run.smithi087.stdout:[osd.1] 2022-09-03T05:47:13.302 INFO:teuthology.orchestra.run.smithi087.stdout: osd_max_omap_entries_per_request = 1000 2022-09-03T05:47:13.302 INFO:teuthology.orchestra.run.smithi087.stdout:[osd.2] 2022-09-03T05:47:13.303 INFO:teuthology.orchestra.run.smithi087.stdout: osd_max_omap_entries_per_request = 10000 2022-09-03T05:47:13.303 INFO:tasks.ceph:Creating admin key on c1.mon.a... 2022-09-03T05:47:13.304 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/c1.keyring 2022-09-03T05:47:13.365 INFO:tasks.ceph:Copying monmap to all nodes... 2022-09-03T05:47:13.366 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:13.367 DEBUG:teuthology.orchestra.run.smithi062:> dd if=/etc/ceph/c1.keyring of=/dev/stdout 2022-09-03T05:47:13.389 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:13.393 DEBUG:teuthology.orchestra.run.smithi062:> dd if=/home/ubuntu/cephtest/c1.monmap of=/dev/stdout 2022-09-03T05:47:13.455 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi062.front.sepia.ceph.com 2022-09-03T05:47:13.456 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:13.457 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd of=/etc/ceph/c1.keyring 2022-09-03T05:47:13.458 DEBUG:teuthology.orchestra.run.smithi062:> sudo chmod 0644 /etc/ceph/c1.keyring 2022-09-03T05:47:13.535 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:13.536 DEBUG:teuthology.orchestra.run.smithi062:> dd of=/home/ubuntu/cephtest/c1.monmap 2022-09-03T05:47:13.594 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi087.front.sepia.ceph.com 2022-09-03T05:47:13.594 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:13.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/ceph/c1.keyring 2022-09-03T05:47:13.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 0644 /etc/ceph/c1.keyring 2022-09-03T05:47:13.637 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:13.638 DEBUG:teuthology.orchestra.run.smithi087:> dd of=/home/ubuntu/cephtest/c1.monmap 2022-09-03T05:47:13.704 INFO:tasks.ceph:Setting up mon nodes... 2022-09-03T05:47:13.705 INFO:tasks.ceph:Setting up mgr nodes... 2022-09-03T05:47:13.706 DEBUG:teuthology.orchestra.run.smithi062:> sudo mkdir -p /var/lib/ceph/mgr/c1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/c1-x/keyring 2022-09-03T05:47:13.761 INFO:teuthology.orchestra.run.smithi062.stdout:creating /var/lib/ceph/mgr/c1-x/keyring 2022-09-03T05:47:13.766 INFO:tasks.ceph:Setting up mds nodes... 2022-09-03T05:47:13.770 INFO:tasks.ceph_client:Setting up client nodes... 2022-09-03T05:47:13.771 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/c1.client.0.keyring && sudo chmod 0644 /etc/ceph/c1.client.0.keyring 2022-09-03T05:47:13.816 INFO:teuthology.orchestra.run.smithi062.stdout:creating /etc/ceph/c1.client.0.keyring 2022-09-03T05:47:13.829 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/c1.client.1.keyring && sudo chmod 0644 /etc/ceph/c1.client.1.keyring 2022-09-03T05:47:13.876 INFO:teuthology.orchestra.run.smithi062.stdout:creating /etc/ceph/c1.client.1.keyring 2022-09-03T05:47:13.888 INFO:tasks.ceph:Running mkfs on osd nodes... 2022-09-03T05:47:13.889 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi062.front.sepia.ceph.com'): {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'}} 2022-09-03T05:47:13.890 DEBUG:teuthology.orchestra.run.smithi062:> sudo mkdir -p /var/lib/ceph/osd/c1-0 2022-09-03T05:47:13.962 INFO:tasks.ceph:roles_to_devs: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2022-09-03T05:47:13.963 INFO:tasks.ceph:role: c1.osd.0 2022-09-03T05:47:13.963 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi062.front.sepia.ceph.com 2022-09-03T05:47:13.964 DEBUG:teuthology.orchestra.run.smithi062:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2022-09-03T05:47:14.039 INFO:teuthology.orchestra.run.smithi062.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2022-09-03T05:47:14.039 INFO:teuthology.orchestra.run.smithi062.stdout: = sectsz=512 attr=2, projid32bit=1 2022-09-03T05:47:14.039 INFO:teuthology.orchestra.run.smithi062.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2022-09-03T05:47:14.039 INFO:teuthology.orchestra.run.smithi062.stdout: = reflink=1 bigtime=0 inobtcount=0 2022-09-03T05:47:14.040 INFO:teuthology.orchestra.run.smithi062.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2022-09-03T05:47:14.040 INFO:teuthology.orchestra.run.smithi062.stdout: = sunit=0 swidth=0 blks 2022-09-03T05:47:14.040 INFO:teuthology.orchestra.run.smithi062.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2022-09-03T05:47:14.040 INFO:teuthology.orchestra.run.smithi062.stdout:log =internal log bsize=4096 blocks=11443, version=2 2022-09-03T05:47:14.041 INFO:teuthology.orchestra.run.smithi062.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2022-09-03T05:47:14.041 INFO:teuthology.orchestra.run.smithi062.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2022-09-03T05:47:14.476 INFO:teuthology.orchestra.run.smithi062.stdout:Discarding blocks...Done. 2022-09-03T05:47:14.478 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi062.front.sepia.ceph.com -o noatime 2022-09-03T05:47:14.478 DEBUG:teuthology.orchestra.run.smithi062:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/c1-0 2022-09-03T05:47:14.654 DEBUG:teuthology.orchestra.run.smithi062:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-0 2022-09-03T05:47:14.690 DEBUG:teuthology.orchestra.run.smithi062:> sudo mkdir -p /var/lib/ceph/osd/c1-1 2022-09-03T05:47:14.760 INFO:tasks.ceph:roles_to_devs: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2022-09-03T05:47:14.761 INFO:tasks.ceph:role: c1.osd.1 2022-09-03T05:47:14.761 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi062.front.sepia.ceph.com 2022-09-03T05:47:14.764 DEBUG:teuthology.orchestra.run.smithi062:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2022-09-03T05:47:14.836 INFO:teuthology.orchestra.run.smithi062.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2022-09-03T05:47:14.837 INFO:teuthology.orchestra.run.smithi062.stdout: = sectsz=512 attr=2, projid32bit=1 2022-09-03T05:47:14.837 INFO:teuthology.orchestra.run.smithi062.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2022-09-03T05:47:14.837 INFO:teuthology.orchestra.run.smithi062.stdout: = reflink=1 bigtime=0 inobtcount=0 2022-09-03T05:47:14.837 INFO:teuthology.orchestra.run.smithi062.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2022-09-03T05:47:14.838 INFO:teuthology.orchestra.run.smithi062.stdout: = sunit=0 swidth=0 blks 2022-09-03T05:47:14.838 INFO:teuthology.orchestra.run.smithi062.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2022-09-03T05:47:14.838 INFO:teuthology.orchestra.run.smithi062.stdout:log =internal log bsize=4096 blocks=11443, version=2 2022-09-03T05:47:14.838 INFO:teuthology.orchestra.run.smithi062.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2022-09-03T05:47:14.839 INFO:teuthology.orchestra.run.smithi062.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2022-09-03T05:47:15.259 INFO:teuthology.orchestra.run.smithi062.stdout:Discarding blocks...Done. 2022-09-03T05:47:15.260 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi062.front.sepia.ceph.com -o noatime 2022-09-03T05:47:15.261 DEBUG:teuthology.orchestra.run.smithi062:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/c1-1 2022-09-03T05:47:15.373 DEBUG:teuthology.orchestra.run.smithi062:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-1 2022-09-03T05:47:15.407 DEBUG:teuthology.orchestra.run.smithi062:> sudo mkdir -p /var/lib/ceph/osd/c1-2 2022-09-03T05:47:15.478 INFO:tasks.ceph:roles_to_devs: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2022-09-03T05:47:15.479 INFO:tasks.ceph:role: c1.osd.2 2022-09-03T05:47:15.479 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi062.front.sepia.ceph.com 2022-09-03T05:47:15.480 DEBUG:teuthology.orchestra.run.smithi062:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2022-09-03T05:47:15.553 INFO:teuthology.orchestra.run.smithi062.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2022-09-03T05:47:15.554 INFO:teuthology.orchestra.run.smithi062.stdout: = sectsz=512 attr=2, projid32bit=1 2022-09-03T05:47:15.554 INFO:teuthology.orchestra.run.smithi062.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2022-09-03T05:47:15.554 INFO:teuthology.orchestra.run.smithi062.stdout: = reflink=1 bigtime=0 inobtcount=0 2022-09-03T05:47:15.554 INFO:teuthology.orchestra.run.smithi062.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2022-09-03T05:47:15.555 INFO:teuthology.orchestra.run.smithi062.stdout: = sunit=0 swidth=0 blks 2022-09-03T05:47:15.555 INFO:teuthology.orchestra.run.smithi062.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2022-09-03T05:47:15.555 INFO:teuthology.orchestra.run.smithi062.stdout:log =internal log bsize=4096 blocks=11443, version=2 2022-09-03T05:47:15.555 INFO:teuthology.orchestra.run.smithi062.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2022-09-03T05:47:15.555 INFO:teuthology.orchestra.run.smithi062.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2022-09-03T05:47:15.999 INFO:teuthology.orchestra.run.smithi062.stdout:Discarding blocks...Done. 2022-09-03T05:47:16.001 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi062.front.sepia.ceph.com -o noatime 2022-09-03T05:47:16.002 DEBUG:teuthology.orchestra.run.smithi062:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/c1-2 2022-09-03T05:47:16.155 DEBUG:teuthology.orchestra.run.smithi062:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-2 2022-09-03T05:47:16.193 DEBUG:teuthology.orchestra.run.smithi062:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/c1.monmap 2022-09-03T05:47:16.282 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:16.281+0000 7f4284044440 -1 auth: error reading file: /var/lib/ceph/osd/c1-0/keyring: can't open /var/lib/ceph/osd/c1-0/keyring: (2) No such file or directory 2022-09-03T05:47:16.284 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:16.283+0000 7f4284044440 -1 created new key in keyring /var/lib/ceph/osd/c1-0/keyring 2022-09-03T05:47:16.285 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:16.283+0000 7f4284044440 -1 bluestore(/var/lib/ceph/osd/c1-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-0/block: (2) No such file or directory 2022-09-03T05:47:16.285 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:16.283+0000 7f4284044440 -1 bluestore(/var/lib/ceph/osd/c1-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-0/block: (2) No such file or directory 2022-09-03T05:47:16.286 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:16.283+0000 7f4284044440 -1 bluestore(/var/lib/ceph/osd/c1-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-0/block: (2) No such file or directory 2022-09-03T05:47:16.289 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:16.288+0000 7f4284044440 -1 bluestore(/var/lib/ceph/osd/c1-0) _read_fsid unparsable uuid 2022-09-03T05:47:18.366 DEBUG:teuthology.orchestra.run.smithi062:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-0 2022-09-03T05:47:18.399 DEBUG:teuthology.orchestra.run.smithi062:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/c1.monmap 2022-09-03T05:47:18.489 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:18.487+0000 7f7e1c249440 -1 auth: error reading file: /var/lib/ceph/osd/c1-1/keyring: can't open /var/lib/ceph/osd/c1-1/keyring: (2) No such file or directory 2022-09-03T05:47:18.490 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:18.487+0000 7f7e1c249440 -1 created new key in keyring /var/lib/ceph/osd/c1-1/keyring 2022-09-03T05:47:18.490 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:18.487+0000 7f7e1c249440 -1 bluestore(/var/lib/ceph/osd/c1-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-1/block: (2) No such file or directory 2022-09-03T05:47:18.491 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:18.487+0000 7f7e1c249440 -1 bluestore(/var/lib/ceph/osd/c1-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-1/block: (2) No such file or directory 2022-09-03T05:47:18.491 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:18.487+0000 7f7e1c249440 -1 bluestore(/var/lib/ceph/osd/c1-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-1/block: (2) No such file or directory 2022-09-03T05:47:18.491 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:18.488+0000 7f7e1c249440 -1 bluestore(/var/lib/ceph/osd/c1-1) _read_fsid unparsable uuid 2022-09-03T05:47:20.561 DEBUG:teuthology.orchestra.run.smithi062:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-1 2022-09-03T05:47:20.592 DEBUG:teuthology.orchestra.run.smithi062:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/c1.monmap 2022-09-03T05:47:20.687 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:20.683+0000 7f7441ac9440 -1 auth: error reading file: /var/lib/ceph/osd/c1-2/keyring: can't open /var/lib/ceph/osd/c1-2/keyring: (2) No such file or directory 2022-09-03T05:47:20.688 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:20.683+0000 7f7441ac9440 -1 created new key in keyring /var/lib/ceph/osd/c1-2/keyring 2022-09-03T05:47:20.688 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:20.683+0000 7f7441ac9440 -1 bluestore(/var/lib/ceph/osd/c1-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-2/block: (2) No such file or directory 2022-09-03T05:47:20.688 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:20.683+0000 7f7441ac9440 -1 bluestore(/var/lib/ceph/osd/c1-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-2/block: (2) No such file or directory 2022-09-03T05:47:20.689 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:20.683+0000 7f7441ac9440 -1 bluestore(/var/lib/ceph/osd/c1-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-2/block: (2) No such file or directory 2022-09-03T05:47:20.689 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:47:20.684+0000 7f7441ac9440 -1 bluestore(/var/lib/ceph/osd/c1-2) _read_fsid unparsable uuid 2022-09-03T05:47:22.763 DEBUG:teuthology.orchestra.run.smithi062:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-2 2022-09-03T05:47:22.796 INFO:tasks.ceph:Reading keys from all nodes... 2022-09-03T05:47:22.797 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:22.797 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd if=/var/lib/ceph/mgr/c1-x/keyring of=/dev/stdout 2022-09-03T05:47:22.868 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:22.870 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd if=/var/lib/ceph/osd/c1-0/keyring of=/dev/stdout 2022-09-03T05:47:22.939 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:22.940 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd if=/var/lib/ceph/osd/c1-1/keyring of=/dev/stdout 2022-09-03T05:47:23.008 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:23.009 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd if=/var/lib/ceph/osd/c1-2/keyring of=/dev/stdout 2022-09-03T05:47:23.083 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:23.084 DEBUG:teuthology.orchestra.run.smithi062:> dd if=/etc/ceph/c1.client.0.keyring of=/dev/stdout 2022-09-03T05:47:23.144 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:23.145 DEBUG:teuthology.orchestra.run.smithi062:> dd if=/etc/ceph/c1.client.1.keyring of=/dev/stdout 2022-09-03T05:47:23.202 INFO:tasks.ceph:Adding keys to all mons... 2022-09-03T05:47:23.203 DEBUG:teuthology.orchestra.run.smithi062:> sudo tee -a /etc/ceph/c1.keyring 2022-09-03T05:47:23.270 INFO:teuthology.orchestra.run.smithi062.stdout:[mgr.x] 2022-09-03T05:47:23.272 INFO:teuthology.orchestra.run.smithi062.stdout: key = AQBh6hJj+OxiLRAAbXHyKzaxoifS/g6H0U5hAQ== 2022-09-03T05:47:23.272 INFO:teuthology.orchestra.run.smithi062.stdout:[osd.0] 2022-09-03T05:47:23.272 INFO:teuthology.orchestra.run.smithi062.stdout: key = AQBk6hJjZ1DMEBAA03krgAJa919+VXnuhu6Epw== 2022-09-03T05:47:23.273 INFO:teuthology.orchestra.run.smithi062.stdout:[osd.1] 2022-09-03T05:47:23.273 INFO:teuthology.orchestra.run.smithi062.stdout: key = AQBm6hJjEEsYHRAAlYFvOy58Pt6QGMmdsL1PWQ== 2022-09-03T05:47:23.273 INFO:teuthology.orchestra.run.smithi062.stdout:[osd.2] 2022-09-03T05:47:23.274 INFO:teuthology.orchestra.run.smithi062.stdout: key = AQBo6hJjZ+THKBAALynVHG8q/ZpMmkSbcCNXdA== 2022-09-03T05:47:23.274 INFO:teuthology.orchestra.run.smithi062.stdout:[client.0] 2022-09-03T05:47:23.275 INFO:teuthology.orchestra.run.smithi062.stdout: key = AQBh6hJjbjioMBAAkYFTE+59EbxFqUgdK7tP7Q== 2022-09-03T05:47:23.275 INFO:teuthology.orchestra.run.smithi062.stdout:[client.1] 2022-09-03T05:47:23.275 INFO:teuthology.orchestra.run.smithi062.stdout: key = AQBh6hJjT8kyNBAA9nSMhFYK4+iB8nBJSlaQLg== 2022-09-03T05:47:23.277 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2022-09-03T05:47:23.375 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2022-09-03T05:47:23.431 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2022-09-03T05:47:23.485 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2022-09-03T05:47:23.546 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2022-09-03T05:47:23.605 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2022-09-03T05:47:23.665 INFO:tasks.ceph:Running mkfs on mon nodes... 2022-09-03T05:47:23.666 DEBUG:teuthology.orchestra.run.smithi062:> sudo mkdir -p /var/lib/ceph/mon/c1-a 2022-09-03T05:47:23.696 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster c1 --mkfs -i a --monmap /home/ubuntu/cephtest/c1.monmap --keyring /etc/ceph/c1.keyring 2022-09-03T05:47:23.797 DEBUG:teuthology.orchestra.run.smithi062:> sudo chown -R ceph:ceph /var/lib/ceph/mon/c1-a 2022-09-03T05:47:23.827 DEBUG:teuthology.orchestra.run.smithi062:> rm -- /home/ubuntu/cephtest/c1.monmap 2022-09-03T05:47:23.886 INFO:tasks.ceph:Starting mon daemons in cluster c1... 2022-09-03T05:47:23.887 INFO:tasks.ceph.c1.mon.a:Restarting daemon 2022-09-03T05:47:23.887 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster c1 -i a 2022-09-03T05:47:23.930 INFO:tasks.ceph.c1.mon.a:Started 2022-09-03T05:47:23.930 INFO:tasks.ceph:Starting mgr daemons in cluster c1... 2022-09-03T05:47:23.931 INFO:tasks.ceph.c1.mgr.x:Restarting daemon 2022-09-03T05:47:23.931 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster c1 -i x 2022-09-03T05:47:23.934 INFO:tasks.ceph.c1.mgr.x:Started 2022-09-03T05:47:23.934 INFO:tasks.ceph:Setting crush tunables to default 2022-09-03T05:47:23.934 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph --cluster c1 osd crush tunables default 2022-09-03T05:47:24.147 INFO:teuthology.orchestra.run.smithi062.stderr:adjusted tunables profile to default 2022-09-03T05:47:24.157 INFO:tasks.ceph:Starting osd daemons in cluster c1... 2022-09-03T05:47:24.159 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:24.160 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd if=/var/lib/ceph/osd/c1-0/fsid of=/dev/stdout 2022-09-03T05:47:24.188 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:24.189 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd if=/var/lib/ceph/osd/c1-1/fsid of=/dev/stdout 2022-09-03T05:47:24.260 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:24.261 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd if=/var/lib/ceph/osd/c1-2/fsid of=/dev/stdout 2022-09-03T05:47:24.332 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph --cluster c1 osd new 262c8624-8f2c-4213-ba47-bab83424b50b 0 2022-09-03T05:47:24.590 INFO:teuthology.orchestra.run.smithi062.stdout:0 2022-09-03T05:47:24.603 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph --cluster c1 osd new 80ce2476-a467-4671-8981-a2d645ad11a7 1 2022-09-03T05:47:24.822 INFO:teuthology.orchestra.run.smithi062.stdout:1 2022-09-03T05:47:24.843 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph --cluster c1 osd new e808fa3c-036e-498c-b2a7-80030d8d9fe9 2 2022-09-03T05:47:25.063 INFO:teuthology.orchestra.run.smithi062.stdout:2 2022-09-03T05:47:25.076 INFO:tasks.ceph.c1.osd.0:Restarting daemon 2022-09-03T05:47:25.077 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 0 2022-09-03T05:47:25.081 INFO:tasks.ceph.c1.osd.0:Started 2022-09-03T05:47:25.081 INFO:tasks.ceph.c1.osd.1:Restarting daemon 2022-09-03T05:47:25.082 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 1 2022-09-03T05:47:25.085 INFO:tasks.ceph.c1.osd.1:Started 2022-09-03T05:47:25.085 INFO:tasks.ceph.c1.osd.2:Restarting daemon 2022-09-03T05:47:25.086 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 2 2022-09-03T05:47:25.089 INFO:tasks.ceph.c1.osd.2:Started 2022-09-03T05:47:25.090 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd dump --format=json 2022-09-03T05:47:25.161 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:25.156+0000 7ff8586c2080 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-03T05:47:25.234 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:25.233+0000 7ff8586c2080 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-03T05:47:25.323 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:25.323 INFO:teuthology.orchestra.run.smithi062.stdout:{"epoch":5,"fsid":"6b295c6b-f22c-47e6-9e05-dafe77fe2ff7","created":"2022-09-03T05:47:24.037394+0000","modified":"2022-09-03T05:47:25.061193+0000","last_up_change":"0.000000","last_in_change":"2022-09-03T05:47:25.061193+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"262c8624-8f2c-4213-ba47-bab83424b50b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"80ce2476-a467-4671-8981-a2d645ad11a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"e808fa3c-036e-498c-b2a7-80030d8d9fe9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:47:25.324 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:25.323+0000 7ff8586c2080 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-03T05:47:25.335 INFO:tasks.ceph.ceph_manager.c1:[] 2022-09-03T05:47:25.336 INFO:tasks.ceph:Waiting for OSDs to come up 2022-09-03T05:47:25.515 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:25.514+0000 7ff8586c2080 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-03T05:47:25.639 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c1 osd dump --format=json 2022-09-03T05:47:25.694 INFO:tasks.ceph.c1.osd.1.smithi062.stderr:2022-09-03T05:47:25.693+0000 7efcc9a0f440 -1 Falling back to public interface 2022-09-03T05:47:25.700 INFO:tasks.ceph.c1.osd.0.smithi062.stderr:2022-09-03T05:47:25.696+0000 7f453f2cf440 -1 Falling back to public interface 2022-09-03T05:47:25.709 INFO:tasks.ceph.c1.osd.2.smithi062.stderr:2022-09-03T05:47:25.707+0000 7f77af594440 -1 Falling back to public interface 2022-09-03T05:47:25.840 INFO:teuthology.misc.health.smithi062.stdout: 2022-09-03T05:47:25.841 INFO:teuthology.misc.health.smithi062.stdout:{"epoch":5,"fsid":"6b295c6b-f22c-47e6-9e05-dafe77fe2ff7","created":"2022-09-03T05:47:24.037394+0000","modified":"2022-09-03T05:47:25.061193+0000","last_up_change":"0.000000","last_in_change":"2022-09-03T05:47:25.061193+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"262c8624-8f2c-4213-ba47-bab83424b50b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"80ce2476-a467-4671-8981-a2d645ad11a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"e808fa3c-036e-498c-b2a7-80030d8d9fe9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:47:25.852 DEBUG:teuthology.misc:0 of 3 OSDs are up 2022-09-03T05:47:26.336 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:26.334+0000 7ff8586c2080 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-03T05:47:26.479 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:26.477+0000 7ff8586c2080 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-03T05:47:26.708 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:26.707+0000 7ff8586c2080 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-03T05:47:26.781 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:26.780+0000 7ff8586c2080 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-03T05:47:26.922 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:26.920+0000 7ff8586c2080 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-03T05:47:27.197 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:27.196+0000 7ff8586c2080 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-03T05:47:27.387 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:27.384+0000 7ff8586c2080 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2022-09-03T05:47:27.528 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:27.527+0000 7ff8586c2080 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-03T05:47:27.646 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:27.645+0000 7ff8586c2080 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-03T05:47:27.789 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:27.788+0000 7ff8586c2080 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-03T05:47:27.870 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:27.869+0000 7ff8586c2080 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-03T05:47:28.093 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:28.092+0000 7ff8586c2080 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-03T05:47:28.173 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:28.172+0000 7ff8586c2080 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2022-09-03T05:47:28.805 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:28.804+0000 7ff8586c2080 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-03T05:47:29.080 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:29.078+0000 7ff8586c2080 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-03T05:47:29.155 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:29.154+0000 7ff8586c2080 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-03T05:47:29.241 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:29.240+0000 7ff8586c2080 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-03T05:47:29.428 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:29.427+0000 7ff8586c2080 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-03T05:47:29.502 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:29.501+0000 7ff8586c2080 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-03T05:47:29.657 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:29.656+0000 7ff8586c2080 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-03T05:47:30.676 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T05:47:30.675+0000 7ff81249f700 -1 mgr.server handle_report got status from non-daemon mon.a 2022-09-03T05:47:32.156 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c1 osd dump --format=json 2022-09-03T05:47:32.476 INFO:teuthology.misc.health.smithi062.stdout: 2022-09-03T05:47:32.477 INFO:teuthology.misc.health.smithi062.stdout:{"epoch":5,"fsid":"6b295c6b-f22c-47e6-9e05-dafe77fe2ff7","created":"2022-09-03T05:47:24.037394+0000","modified":"2022-09-03T05:47:25.061193+0000","last_up_change":"0.000000","last_in_change":"2022-09-03T05:47:25.061193+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"262c8624-8f2c-4213-ba47-bab83424b50b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"80ce2476-a467-4671-8981-a2d645ad11a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"e808fa3c-036e-498c-b2a7-80030d8d9fe9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:47:32.495 DEBUG:teuthology.misc:0 of 3 OSDs are up 2022-09-03T05:47:33.414 INFO:tasks.ceph.c1.osd.0.smithi062.stderr:2022-09-03T05:47:33.412+0000 7f453f2cf440 -1 osd.0 0 log_to_monitors true 2022-09-03T05:47:33.426 INFO:tasks.ceph.c1.osd.1.smithi062.stderr:2022-09-03T05:47:33.424+0000 7efcc9a0f440 -1 osd.1 0 log_to_monitors true 2022-09-03T05:47:33.427 INFO:tasks.ceph.c1.osd.2.smithi062.stderr:2022-09-03T05:47:33.425+0000 7f77af594440 -1 osd.2 0 log_to_monitors true 2022-09-03T05:47:35.896 INFO:tasks.ceph.c1.osd.0.smithi062.stderr:2022-09-03T05:47:35.895+0000 7f453307c700 -1 osd.0 0 waiting for initial osdmap 2022-09-03T05:47:35.922 INFO:tasks.ceph.c1.osd.0.smithi062.stderr:2022-09-03T05:47:35.921+0000 7f452bc58700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-03T05:47:35.923 INFO:tasks.ceph.c1.osd.2.smithi062.stderr:2022-09-03T05:47:35.921+0000 7f77a3341700 -1 osd.2 0 waiting for initial osdmap 2022-09-03T05:47:35.950 INFO:tasks.ceph.c1.osd.2.smithi062.stderr:2022-09-03T05:47:35.949+0000 7f779bf1d700 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-03T05:47:36.353 INFO:tasks.ceph.c1.osd.1.smithi062.stderr:2022-09-03T05:47:36.352+0000 7efcbd7bc700 -1 osd.1 0 waiting for initial osdmap 2022-09-03T05:47:36.371 INFO:tasks.ceph.c1.osd.1.smithi062.stderr:2022-09-03T05:47:36.370+0000 7efcb6398700 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-03T05:47:38.799 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c1 osd dump --format=json 2022-09-03T05:47:39.152 INFO:teuthology.misc.health.smithi062.stdout: 2022-09-03T05:47:39.152 INFO:teuthology.misc.health.smithi062.stdout:{"epoch":10,"fsid":"6b295c6b-f22c-47e6-9e05-dafe77fe2ff7","created":"2022-09-03T05:47:24.037394+0000","modified":"2022-09-03T05:47:38.924282+0000","last_up_change":"2022-09-03T05:47:36.919507+0000","last_in_change":"2022-09-03T05:47:25.061193+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-03T05:47:37.730797+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"262c8624-8f2c-4213-ba47-bab83424b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6808","nonce":104154},{"type":"v1","addr":"172.21.15.62:6809","nonce":104154}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6810","nonce":104154},{"type":"v1","addr":"172.21.15.62:6811","nonce":104154}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6814","nonce":104154},{"type":"v1","addr":"172.21.15.62:6815","nonce":104154}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6812","nonce":104154},{"type":"v1","addr":"172.21.15.62:6813","nonce":104154}]},"public_addr":"172.21.15.62:6809/104154","cluster_addr":"172.21.15.62:6811/104154","heartbeat_back_addr":"172.21.15.62:6815/104154","heartbeat_front_addr":"172.21.15.62:6813/104154","state":["exists","up"]},{"osd":1,"uuid":"80ce2476-a467-4671-8981-a2d645ad11a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6800","nonce":104155},{"type":"v1","addr":"172.21.15.62:6801","nonce":104155}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6802","nonce":104155},{"type":"v1","addr":"172.21.15.62:6803","nonce":104155}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6806","nonce":104155},{"type":"v1","addr":"172.21.15.62:6807","nonce":104155}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6804","nonce":104155},{"type":"v1","addr":"172.21.15.62:6805","nonce":104155}]},"public_addr":"172.21.15.62:6801/104155","cluster_addr":"172.21.15.62:6803/104155","heartbeat_back_addr":"172.21.15.62:6807/104155","heartbeat_front_addr":"172.21.15.62:6805/104155","state":["exists","up"]},{"osd":2,"uuid":"e808fa3c-036e-498c-b2a7-80030d8d9fe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6816","nonce":104178},{"type":"v1","addr":"172.21.15.62:6817","nonce":104178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6818","nonce":104178},{"type":"v1","addr":"172.21.15.62:6819","nonce":104178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6822","nonce":104178},{"type":"v1","addr":"172.21.15.62:6823","nonce":104178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6820","nonce":104178},{"type":"v1","addr":"172.21.15.62:6821","nonce":104178}]},"public_addr":"172.21.15.62:6817/104178","cluster_addr":"172.21.15.62:6819/104178","heartbeat_back_addr":"172.21.15.62:6823/104178","heartbeat_front_addr":"172.21.15.62:6821/104178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:47:34.367063+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:47:34.426938+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:47:34.401804+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:47:39.164 DEBUG:teuthology.misc:3 of 3 OSDs are up 2022-09-03T05:47:39.165 INFO:tasks.ceph:Creating RBD pool 2022-09-03T05:47:39.165 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph --cluster c1 osd pool create rbd 8 2022-09-03T05:47:39.935 INFO:teuthology.orchestra.run.smithi062.stderr:pool 'rbd' created 2022-09-03T05:47:39.947 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph --cluster c1 osd pool application enable rbd rbd --yes-i-really-mean-it 2022-09-03T05:47:40.942 INFO:teuthology.orchestra.run.smithi062.stderr:enabled application 'rbd' on pool 'rbd' 2022-09-03T05:47:40.955 INFO:tasks.ceph:Starting mds daemons in cluster c1... 2022-09-03T05:47:40.956 INFO:tasks.ceph:Waiting until c1 daemons up and pgs clean... 2022-09-03T05:47:40.957 INFO:tasks.ceph.ceph_manager.c1:waiting for mgr available 2022-09-03T05:47:40.957 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 mgr dump --format=json 2022-09-03T05:47:40.958 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2022-09-03T05:47:41.340 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:41.352 INFO:teuthology.orchestra.run.smithi062.stdout:{"epoch":5,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6824","nonce":103746},{"type":"v1","addr":"172.21.15.62:6825","nonce":103746}]},"active_addr":"172.21.15.62:6825/103746","active_change":"2022-09-03T05:47:29.669525+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":0,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.62:0","nonce":3919839441}]},{"addrvec":[{"type":"v2","addr":"172.21.15.62:0","nonce":1715887725}]},{"addrvec":[{"type":"v2","addr":"172.21.15.62:0","nonce":1947265564}]},{"addrvec":[{"type":"v2","addr":"172.21.15.62:0","nonce":3868203001}]}]}} 2022-09-03T05:47:41.366 INFO:tasks.ceph.ceph_manager.c1:mgr available! 2022-09-03T05:47:41.367 INFO:tasks.ceph.ceph_manager.c1:waiting for all up 2022-09-03T05:47:41.368 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd dump --format=json 2022-09-03T05:47:41.691 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:41.691 INFO:teuthology.orchestra.run.smithi062.stdout:{"epoch":12,"fsid":"6b295c6b-f22c-47e6-9e05-dafe77fe2ff7","created":"2022-09-03T05:47:24.037394+0000","modified":"2022-09-03T05:47:40.930772+0000","last_up_change":"2022-09-03T05:47:36.919507+0000","last_in_change":"2022-09-03T05:47:25.061193+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-03T05:47:37.730797+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-03T05:47:39.525411+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"262c8624-8f2c-4213-ba47-bab83424b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6808","nonce":104154},{"type":"v1","addr":"172.21.15.62:6809","nonce":104154}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6810","nonce":104154},{"type":"v1","addr":"172.21.15.62:6811","nonce":104154}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6814","nonce":104154},{"type":"v1","addr":"172.21.15.62:6815","nonce":104154}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6812","nonce":104154},{"type":"v1","addr":"172.21.15.62:6813","nonce":104154}]},"public_addr":"172.21.15.62:6809/104154","cluster_addr":"172.21.15.62:6811/104154","heartbeat_back_addr":"172.21.15.62:6815/104154","heartbeat_front_addr":"172.21.15.62:6813/104154","state":["exists","up"]},{"osd":1,"uuid":"80ce2476-a467-4671-8981-a2d645ad11a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6800","nonce":104155},{"type":"v1","addr":"172.21.15.62:6801","nonce":104155}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6802","nonce":104155},{"type":"v1","addr":"172.21.15.62:6803","nonce":104155}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6806","nonce":104155},{"type":"v1","addr":"172.21.15.62:6807","nonce":104155}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6804","nonce":104155},{"type":"v1","addr":"172.21.15.62:6805","nonce":104155}]},"public_addr":"172.21.15.62:6801/104155","cluster_addr":"172.21.15.62:6803/104155","heartbeat_back_addr":"172.21.15.62:6807/104155","heartbeat_front_addr":"172.21.15.62:6805/104155","state":["exists","up"]},{"osd":2,"uuid":"e808fa3c-036e-498c-b2a7-80030d8d9fe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6816","nonce":104178},{"type":"v1","addr":"172.21.15.62:6817","nonce":104178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6818","nonce":104178},{"type":"v1","addr":"172.21.15.62:6819","nonce":104178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6822","nonce":104178},{"type":"v1","addr":"172.21.15.62:6823","nonce":104178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6820","nonce":104178},{"type":"v1","addr":"172.21.15.62:6821","nonce":104178}]},"public_addr":"172.21.15.62:6817/104178","cluster_addr":"172.21.15.62:6819/104178","heartbeat_back_addr":"172.21.15.62:6823/104178","heartbeat_front_addr":"172.21.15.62:6821/104178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:47:34.367063+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:47:34.426938+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:47:34.401804+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:47:41.707 INFO:tasks.ceph.ceph_manager.c1:all up! 2022-09-03T05:47:41.708 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd dump --format=json 2022-09-03T05:47:42.032 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:42.033 INFO:teuthology.orchestra.run.smithi062.stdout:{"epoch":13,"fsid":"6b295c6b-f22c-47e6-9e05-dafe77fe2ff7","created":"2022-09-03T05:47:24.037394+0000","modified":"2022-09-03T05:47:41.935418+0000","last_up_change":"2022-09-03T05:47:36.919507+0000","last_in_change":"2022-09-03T05:47:25.061193+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-03T05:47:37.730797+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-03T05:47:39.525411+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"262c8624-8f2c-4213-ba47-bab83424b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6808","nonce":104154},{"type":"v1","addr":"172.21.15.62:6809","nonce":104154}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6810","nonce":104154},{"type":"v1","addr":"172.21.15.62:6811","nonce":104154}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6814","nonce":104154},{"type":"v1","addr":"172.21.15.62:6815","nonce":104154}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6812","nonce":104154},{"type":"v1","addr":"172.21.15.62:6813","nonce":104154}]},"public_addr":"172.21.15.62:6809/104154","cluster_addr":"172.21.15.62:6811/104154","heartbeat_back_addr":"172.21.15.62:6815/104154","heartbeat_front_addr":"172.21.15.62:6813/104154","state":["exists","up"]},{"osd":1,"uuid":"80ce2476-a467-4671-8981-a2d645ad11a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6800","nonce":104155},{"type":"v1","addr":"172.21.15.62:6801","nonce":104155}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6802","nonce":104155},{"type":"v1","addr":"172.21.15.62:6803","nonce":104155}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6806","nonce":104155},{"type":"v1","addr":"172.21.15.62:6807","nonce":104155}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6804","nonce":104155},{"type":"v1","addr":"172.21.15.62:6805","nonce":104155}]},"public_addr":"172.21.15.62:6801/104155","cluster_addr":"172.21.15.62:6803/104155","heartbeat_back_addr":"172.21.15.62:6807/104155","heartbeat_front_addr":"172.21.15.62:6805/104155","state":["exists","up"]},{"osd":2,"uuid":"e808fa3c-036e-498c-b2a7-80030d8d9fe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6816","nonce":104178},{"type":"v1","addr":"172.21.15.62:6817","nonce":104178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6818","nonce":104178},{"type":"v1","addr":"172.21.15.62:6819","nonce":104178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6822","nonce":104178},{"type":"v1","addr":"172.21.15.62:6823","nonce":104178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.62:6820","nonce":104178},{"type":"v1","addr":"172.21.15.62:6821","nonce":104178}]},"public_addr":"172.21.15.62:6817/104178","cluster_addr":"172.21.15.62:6819/104178","heartbeat_back_addr":"172.21.15.62:6823/104178","heartbeat_front_addr":"172.21.15.62:6821/104178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:47:34.367063+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:47:34.426938+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:47:34.401804+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:47:42.043 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.0 flush_pg_stats 2022-09-03T05:47:42.044 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.1 flush_pg_stats 2022-09-03T05:47:42.045 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.2 flush_pg_stats 2022-09-03T05:47:42.215 INFO:teuthology.orchestra.run.smithi062.stdout:34359738373 2022-09-03T05:47:42.215 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.2 2022-09-03T05:47:42.217 INFO:teuthology.orchestra.run.smithi062.stdout:34359738372 2022-09-03T05:47:42.217 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.0 2022-09-03T05:47:42.219 INFO:teuthology.orchestra.run.smithi062.stdout:34359738373 2022-09-03T05:47:42.219 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.1 2022-09-03T05:47:42.549 INFO:teuthology.orchestra.run.smithi062.stdout:34359738371 2022-09-03T05:47:42.556 INFO:teuthology.orchestra.run.smithi062.stdout:34359738372 2022-09-03T05:47:42.560 INFO:teuthology.orchestra.run.smithi062.stdout:34359738372 2022-09-03T05:47:42.564 INFO:tasks.ceph.ceph_manager.c1:need seq 34359738372 got 34359738371 for osd.0 2022-09-03T05:47:42.573 INFO:tasks.ceph.ceph_manager.c1:need seq 34359738373 got 34359738372 for osd.1 2022-09-03T05:47:42.575 INFO:tasks.ceph.ceph_manager.c1:need seq 34359738373 got 34359738372 for osd.2 2022-09-03T05:47:43.566 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.0 2022-09-03T05:47:43.578 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.1 2022-09-03T05:47:43.580 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.2 2022-09-03T05:47:43.918 INFO:teuthology.orchestra.run.smithi062.stdout:34359738371 2022-09-03T05:47:43.933 INFO:tasks.ceph.ceph_manager.c1:need seq 34359738372 got 34359738371 for osd.0 2022-09-03T05:47:43.961 INFO:teuthology.orchestra.run.smithi062.stdout:34359738374 2022-09-03T05:47:43.968 INFO:teuthology.orchestra.run.smithi062.stdout:34359738374 2022-09-03T05:47:43.977 INFO:tasks.ceph.ceph_manager.c1:need seq 34359738373 got 34359738374 for osd.2 2022-09-03T05:47:43.978 DEBUG:teuthology.parallel:result is None 2022-09-03T05:47:43.983 INFO:tasks.ceph.ceph_manager.c1:need seq 34359738373 got 34359738374 for osd.1 2022-09-03T05:47:43.983 DEBUG:teuthology.parallel:result is None 2022-09-03T05:47:44.935 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.0 2022-09-03T05:47:45.270 INFO:teuthology.orchestra.run.smithi062.stdout:34359738373 2022-09-03T05:47:45.285 INFO:tasks.ceph.ceph_manager.c1:need seq 34359738372 got 34359738373 for osd.0 2022-09-03T05:47:45.357 DEBUG:teuthology.parallel:result is None 2022-09-03T05:47:45.357 INFO:tasks.ceph.ceph_manager.c1:waiting for clean 2022-09-03T05:47:45.358 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 pg dump --format=json 2022-09-03T05:47:45.678 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:45.679 INFO:teuthology.orchestra.run.smithi062.stderr:dumped all 2022-09-03T05:47:45.690 INFO:teuthology.orchestra.run.smithi062.stdout:{"pg_ready":true,"pg_map":{"version":14,"stamp":"2022-09-03T05:47:43.674908+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":7,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":314572800,"kb_used":24920,"kb_used_data":1472,"kb_used_omap":0,"kb_used_meta":23424,"kb_avail":314547880,"statfs":{"total":322122547200,"available":322097029120,"internally_reserved":0,"allocated":1507328,"data_stored":1267820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":23986176},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940122+0000","last_change":"2022-09-03T05:47:40.940520+0000","last_active":"2022-09-03T05:47:41.940122+0000","last_peered":"2022-09-03T05:47:41.940122+0000","last_clean":"2022-09-03T05:47:41.940122+0000","last_became_active":"2022-09-03T05:47:40.940190+0000","last_became_peered":"2022-09-03T05:47:40.940190+0000","last_unstale":"2022-09-03T05:47:41.940122+0000","last_undegraded":"2022-09-03T05:47:41.940122+0000","last_fullsized":"2022-09-03T05:47:41.940122+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T14:03:26.479293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940149+0000","last_change":"2022-09-03T05:47:40.940929+0000","last_active":"2022-09-03T05:47:41.940149+0000","last_peered":"2022-09-03T05:47:41.940149+0000","last_clean":"2022-09-03T05:47:41.940149+0000","last_became_active":"2022-09-03T05:47:40.940613+0000","last_became_peered":"2022-09-03T05:47:40.940613+0000","last_unstale":"2022-09-03T05:47:41.940149+0000","last_undegraded":"2022-09-03T05:47:41.940149+0000","last_fullsized":"2022-09-03T05:47:41.940149+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T08:58:49.321109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940101+0000","last_change":"2022-09-03T05:47:40.940805+0000","last_active":"2022-09-03T05:47:41.940101+0000","last_peered":"2022-09-03T05:47:41.940101+0000","last_clean":"2022-09-03T05:47:41.940101+0000","last_became_active":"2022-09-03T05:47:40.940225+0000","last_became_peered":"2022-09-03T05:47:40.940225+0000","last_unstale":"2022-09-03T05:47:41.940101+0000","last_undegraded":"2022-09-03T05:47:41.940101+0000","last_fullsized":"2022-09-03T05:47:41.940101+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T14:41:15.266244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":12,"state":"active+clean","last_fresh":"2022-09-03T05:47:40.940763+0000","last_change":"2022-09-03T05:47:40.939197+0000","last_active":"2022-09-03T05:47:40.940763+0000","last_peered":"2022-09-03T05:47:40.940763+0000","last_clean":"2022-09-03T05:47:40.940763+0000","last_became_active":"2022-09-03T05:47:40.938812+0000","last_became_peered":"2022-09-03T05:47:40.938812+0000","last_unstale":"2022-09-03T05:47:40.940763+0000","last_undegraded":"2022-09-03T05:47:40.940763+0000","last_fullsized":"2022-09-03T05:47:40.940763+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T13:05:55.239633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"10'97","reported_seq":139,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940035+0000","last_change":"2022-09-03T05:47:38.929460+0000","last_active":"2022-09-03T05:47:41.940035+0000","last_peered":"2022-09-03T05:47:41.940035+0000","last_clean":"2022-09-03T05:47:41.940035+0000","last_became_active":"2022-09-03T05:47:38.929217+0000","last_became_peered":"2022-09-03T05:47:38.929217+0000","last_unstale":"2022-09-03T05:47:41.940035+0000","last_undegraded":"2022-09-03T05:47:41.940035+0000","last_fullsized":"2022-09-03T05:47:41.940035+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:37.920902+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:37.920902+0000","last_clean_scrub_stamp":"2022-09-03T05:47:37.920902+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T10:54:37.140336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940100+0000","last_change":"2022-09-03T05:47:40.938807+0000","last_active":"2022-09-03T05:47:41.940100+0000","last_peered":"2022-09-03T05:47:41.940100+0000","last_clean":"2022-09-03T05:47:41.940100+0000","last_became_active":"2022-09-03T05:47:40.938536+0000","last_became_peered":"2022-09-03T05:47:40.938536+0000","last_unstale":"2022-09-03T05:47:41.940100+0000","last_undegraded":"2022-09-03T05:47:41.940100+0000","last_fullsized":"2022-09-03T05:47:41.940100+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T10:31:19.918806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":12,"state":"active+clean","last_fresh":"2022-09-03T05:47:40.941049+0000","last_change":"2022-09-03T05:47:40.939737+0000","last_active":"2022-09-03T05:47:40.941049+0000","last_peered":"2022-09-03T05:47:40.941049+0000","last_clean":"2022-09-03T05:47:40.941049+0000","last_became_active":"2022-09-03T05:47:40.938979+0000","last_became_peered":"2022-09-03T05:47:40.938979+0000","last_unstale":"2022-09-03T05:47:40.941049+0000","last_undegraded":"2022-09-03T05:47:40.941049+0000","last_fullsized":"2022-09-03T05:47:40.941049+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T09:31:38.743962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":12,"state":"active+clean","last_fresh":"2022-09-03T05:47:40.940078+0000","last_change":"2022-09-03T05:47:40.938491+0000","last_active":"2022-09-03T05:47:40.940078+0000","last_peered":"2022-09-03T05:47:40.940078+0000","last_clean":"2022-09-03T05:47:40.940078+0000","last_became_active":"2022-09-03T05:47:40.938154+0000","last_became_peered":"2022-09-03T05:47:40.938154+0000","last_unstale":"2022-09-03T05:47:40.940078+0000","last_undegraded":"2022-09-03T05:47:40.940078+0000","last_fullsized":"2022-09-03T05:47:40.940078+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T14:41:35.628053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940130+0000","last_change":"2022-09-03T05:47:40.940659+0000","last_active":"2022-09-03T05:47:41.940130+0000","last_peered":"2022-09-03T05:47:41.940130+0000","last_clean":"2022-09-03T05:47:41.940130+0000","last_became_active":"2022-09-03T05:47:40.940339+0000","last_became_peered":"2022-09-03T05:47:40.940339+0000","last_unstale":"2022-09-03T05:47:41.940130+0000","last_undegraded":"2022-09-03T05:47:41.940130+0000","last_fullsized":"2022-09-03T05:47:41.940130+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T14:10:21.316090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738374,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7660,"kb_used_data":100,"kb_used_omap":0,"kb_used_meta":7552,"kb_avail":104849940,"statfs":{"total":107374182400,"available":107366338560,"internally_reserved":0,"allocated":102400,"data_stored":27619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7733248},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8624,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":7936,"kb_avail":104848976,"statfs":{"total":107374182400,"available":107365351424,"internally_reserved":0,"allocated":696320,"data_stored":617987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8126464},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.318}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]}]},{"osd":1,"up_from":8,"seq":34359738374,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8636,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":7936,"kb_avail":104848964,"statfs":{"total":107374182400,"available":107365339136,"internally_reserved":0,"allocated":708608,"data_stored":622214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8126464},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46500000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-03T05:47:45.692 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 pg dump --format=json 2022-09-03T05:47:46.010 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:46.012 INFO:teuthology.orchestra.run.smithi062.stderr:dumped all 2022-09-03T05:47:46.023 INFO:teuthology.orchestra.run.smithi062.stdout:{"pg_ready":true,"pg_map":{"version":15,"stamp":"2022-09-03T05:47:45.676193+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":7,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":314572800,"kb_used":24920,"kb_used_data":1472,"kb_used_omap":0,"kb_used_meta":23424,"kb_avail":314547880,"statfs":{"total":322122547200,"available":322097029120,"internally_reserved":0,"allocated":1507328,"data_stored":1267820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":23986176},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.001284"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940122+0000","last_change":"2022-09-03T05:47:40.940520+0000","last_active":"2022-09-03T05:47:41.940122+0000","last_peered":"2022-09-03T05:47:41.940122+0000","last_clean":"2022-09-03T05:47:41.940122+0000","last_became_active":"2022-09-03T05:47:40.940190+0000","last_became_peered":"2022-09-03T05:47:40.940190+0000","last_unstale":"2022-09-03T05:47:41.940122+0000","last_undegraded":"2022-09-03T05:47:41.940122+0000","last_fullsized":"2022-09-03T05:47:41.940122+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T14:03:26.479293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940149+0000","last_change":"2022-09-03T05:47:40.940929+0000","last_active":"2022-09-03T05:47:41.940149+0000","last_peered":"2022-09-03T05:47:41.940149+0000","last_clean":"2022-09-03T05:47:41.940149+0000","last_became_active":"2022-09-03T05:47:40.940613+0000","last_became_peered":"2022-09-03T05:47:40.940613+0000","last_unstale":"2022-09-03T05:47:41.940149+0000","last_undegraded":"2022-09-03T05:47:41.940149+0000","last_fullsized":"2022-09-03T05:47:41.940149+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T08:58:49.321109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940101+0000","last_change":"2022-09-03T05:47:40.940805+0000","last_active":"2022-09-03T05:47:41.940101+0000","last_peered":"2022-09-03T05:47:41.940101+0000","last_clean":"2022-09-03T05:47:41.940101+0000","last_became_active":"2022-09-03T05:47:40.940225+0000","last_became_peered":"2022-09-03T05:47:40.940225+0000","last_unstale":"2022-09-03T05:47:41.940101+0000","last_undegraded":"2022-09-03T05:47:41.940101+0000","last_fullsized":"2022-09-03T05:47:41.940101+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T14:41:15.266244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":12,"state":"active+clean","last_fresh":"2022-09-03T05:47:40.940763+0000","last_change":"2022-09-03T05:47:40.939197+0000","last_active":"2022-09-03T05:47:40.940763+0000","last_peered":"2022-09-03T05:47:40.940763+0000","last_clean":"2022-09-03T05:47:40.940763+0000","last_became_active":"2022-09-03T05:47:40.938812+0000","last_became_peered":"2022-09-03T05:47:40.938812+0000","last_unstale":"2022-09-03T05:47:40.940763+0000","last_undegraded":"2022-09-03T05:47:40.940763+0000","last_fullsized":"2022-09-03T05:47:40.940763+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T13:05:55.239633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"10'97","reported_seq":139,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940035+0000","last_change":"2022-09-03T05:47:38.929460+0000","last_active":"2022-09-03T05:47:41.940035+0000","last_peered":"2022-09-03T05:47:41.940035+0000","last_clean":"2022-09-03T05:47:41.940035+0000","last_became_active":"2022-09-03T05:47:38.929217+0000","last_became_peered":"2022-09-03T05:47:38.929217+0000","last_unstale":"2022-09-03T05:47:41.940035+0000","last_undegraded":"2022-09-03T05:47:41.940035+0000","last_fullsized":"2022-09-03T05:47:41.940035+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:37.920902+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:37.920902+0000","last_clean_scrub_stamp":"2022-09-03T05:47:37.920902+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T10:54:37.140336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940100+0000","last_change":"2022-09-03T05:47:40.938807+0000","last_active":"2022-09-03T05:47:41.940100+0000","last_peered":"2022-09-03T05:47:41.940100+0000","last_clean":"2022-09-03T05:47:41.940100+0000","last_became_active":"2022-09-03T05:47:40.938536+0000","last_became_peered":"2022-09-03T05:47:40.938536+0000","last_unstale":"2022-09-03T05:47:41.940100+0000","last_undegraded":"2022-09-03T05:47:41.940100+0000","last_fullsized":"2022-09-03T05:47:41.940100+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T10:31:19.918806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":12,"state":"active+clean","last_fresh":"2022-09-03T05:47:40.941049+0000","last_change":"2022-09-03T05:47:40.939737+0000","last_active":"2022-09-03T05:47:40.941049+0000","last_peered":"2022-09-03T05:47:40.941049+0000","last_clean":"2022-09-03T05:47:40.941049+0000","last_became_active":"2022-09-03T05:47:40.938979+0000","last_became_peered":"2022-09-03T05:47:40.938979+0000","last_unstale":"2022-09-03T05:47:40.941049+0000","last_undegraded":"2022-09-03T05:47:40.941049+0000","last_fullsized":"2022-09-03T05:47:40.941049+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T09:31:38.743962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":12,"state":"active+clean","last_fresh":"2022-09-03T05:47:40.940078+0000","last_change":"2022-09-03T05:47:40.938491+0000","last_active":"2022-09-03T05:47:40.940078+0000","last_peered":"2022-09-03T05:47:40.940078+0000","last_clean":"2022-09-03T05:47:40.940078+0000","last_became_active":"2022-09-03T05:47:40.938154+0000","last_became_peered":"2022-09-03T05:47:40.938154+0000","last_unstale":"2022-09-03T05:47:40.940078+0000","last_undegraded":"2022-09-03T05:47:40.940078+0000","last_fullsized":"2022-09-03T05:47:40.940078+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T14:41:35.628053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":13,"state":"active+clean","last_fresh":"2022-09-03T05:47:41.940130+0000","last_change":"2022-09-03T05:47:40.940659+0000","last_active":"2022-09-03T05:47:41.940130+0000","last_peered":"2022-09-03T05:47:41.940130+0000","last_clean":"2022-09-03T05:47:41.940130+0000","last_became_active":"2022-09-03T05:47:40.940339+0000","last_became_peered":"2022-09-03T05:47:40.940339+0000","last_unstale":"2022-09-03T05:47:41.940130+0000","last_undegraded":"2022-09-03T05:47:41.940130+0000","last_fullsized":"2022-09-03T05:47:41.940130+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:47:39.927390+0000","last_clean_scrub_stamp":"2022-09-03T05:47:39.927390+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T14:10:21.316090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738374,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7660,"kb_used_data":100,"kb_used_omap":0,"kb_used_meta":7552,"kb_avail":104849940,"statfs":{"total":107374182400,"available":107366338560,"internally_reserved":0,"allocated":102400,"data_stored":27619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7733248},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8624,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":7936,"kb_avail":104848976,"statfs":{"total":107374182400,"available":107365351424,"internally_reserved":0,"allocated":696320,"data_stored":617987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8126464},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.318}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]}]},{"osd":1,"up_from":8,"seq":34359738374,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8636,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":7936,"kb_avail":104848964,"statfs":{"total":107374182400,"available":107365339136,"internally_reserved":0,"allocated":708608,"data_stored":622214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8126464},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46500000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-03T05:47:46.025 INFO:tasks.ceph.ceph_manager.c1:clean! 2022-09-03T05:47:46.026 INFO:tasks.ceph:Waiting until ceph cluster c1 is healthy... 2022-09-03T05:47:46.026 INFO:tasks.ceph.ceph_manager.c1:wait_until_healthy 2022-09-03T05:47:46.027 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 health --format=json 2022-09-03T05:47:46.391 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:46.392 INFO:teuthology.orchestra.run.smithi062.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2022-09-03T05:47:46.403 INFO:tasks.ceph.ceph_manager.c1:wait_until_healthy done 2022-09-03T05:47:46.404 INFO:teuthology.run_tasks:Running task ceph... 2022-09-03T05:47:46.415 INFO:tasks.ceph:Creating ceph cluster c2... 2022-09-03T05:47:46.416 INFO:tasks.ceph:config {'conf': {'client': {'debug rgw': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync data inject err probability': 0.1, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)'], 'cpu_profile': set(), 'cluster': 'c2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2022-09-03T05:47:46.416 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi/7008075', 'branch': 'pr-47848', 'description': 'rgw/multisite/{clusters frontend/beast ignore-pg-availability notify omap_limits overrides realms/three-zone-plus-pubsub supported-random-distro$/{centos_8} tasks/test_multi}', 'email': None, 'first_in_suite': False, 'job_id': '7008075', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'name': 'cbodley-2022-09-02_14:33:28-rgw-pr-47848-distro-default-smithi', 'no_nested_subset': False, 'nuke-on-error': True, 'os_type': 'centos', 'os_version': '8.stream', 'overrides': {'admin_socket': {'branch': 'pr-47848'}, 'ceph': {'conf': {'client': {'debug rgw': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync data inject err probability': 0.1, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)'], 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4', 'wait-for-scrub': False}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4'}}, 'rgw': {'compression type': 'random', 'frontend': 'beast'}, 'rgw-multisite': {'realm': {'is default': True, 'name': 'test-realm'}, 'zonegroups': [{'enabled_features': ['resharding'], 'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'test-zonegroup', 'zones': [{'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'test-zone1'}, {'endpoints': ['c2.client.0'], 'is_default': True, 'name': 'test-zone2'}, {'endpoints': ['c1.client.1'], 'name': 'test-zone3'}, {'endpoints': ['c2.client.1'], 'is_pubsub': True, 'name': 'test-zone4'}]}]}, 'rgw-multisite-tests': {'args': ['tests.py', 'tests_ps.py']}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'wip-qa-rgw-s3select-fixes_related_to_new_csv_parser', 'sha1': 'e20bde8244d2a27c6431acd07e9967aaa8f04f01'}}, 'owner': 'scheduled_cbodley@teuthology', 'priority': 75, 'repo': 'git://git.ceph.com/ceph-ci.git', 'roles': [['c1.mon.a', 'c1.mgr.x', 'c1.osd.0', 'c1.osd.1', 'c1.osd.2', 'c1.client.0', 'c1.client.1'], ['c2.mon.a', 'c2.mgr.x', 'c2.osd.0', 'c2.osd.1', 'c2.osd.2', 'c2.client.0', 'c2.client.1']], 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4', 'sleep_before_teardown': 0, 'suite': 'rgw', 'suite_branch': 'wip-qa-rgw-s3select-fixes_related_to_new_csv_parser', 'suite_path': '/home/teuthworker/src/github.com_cbodley_ceph_e20bde8244d2a27c6431acd07e9967aaa8f04f01/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/cbodley/ceph.git', 'suite_sha1': 'e20bde8244d2a27c6431acd07e9967aaa8f04f01', 'targets': {'smithi062.front.sepia.ceph.com': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSHRk+FEtEk0j2RQdYKdyMR/5SvhQ+BPYn0gwmkL5EAzXE5undqYEXihmsS0eF0BPoaxbe6Iw7OEnb3csdEOS1E5PFlWUcn1eJII8vdIiboGKEUVStcXyfSIi5xvHeLzDMmy0Oe2yj4J+mlHSpkntH9oDX8tjTwNf4wPzVI2hk9WWyB1ZDrdWZxtO5QjkecSTp10ZPZS6tbWJtxAFSEiZl5lMo/gi6fnRnD4uHf2xHMQa87pUknbtbobXHed/NFAKM+a/iBBNy7kdtQsFOjpbK02N+cHuYdAOrd4lTuZK3LPuTnBuw9CXESr03blNlAPK8nz/0mdmAysyIidk00OUflY0oN0aY9qgiyQ/f8jElda8reLmrqq1kFG9P7p1roLg+CBgSS2ytsV8nyLHhA0HJHuGdz9+7hOgfdpWsN2wBRez34hFLzoYhCzdd9HycuDp9dK5A3VaTzQ9owbXSfF/1MIFnqGpeUrtUl3//kuN5JBR03Tu31Ahq93aoEzcRFkE=', 'smithi087.front.sepia.ceph.com': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA2MfLiBfBB1rQjxhGdqrEUq/1La6/wNGaBQ7PWfp8JYfQ54QThAn+gKdHG9f6dimygPtacq48uajMWnuqetlFwFvkL0kiK0e6dWMTVvD58jmrP8ymFkir0uO+4/i+CGGxFAEbuqtCah38yHkyefQ08/UBmZBsQqcN87vAJIBLiF3JjPTrF5nx8L4s09mXEcOMA7TkUxQhCOcIOMzmFF0NNx0dIT9+FGCmHCxnoI3xj4s+o94e2OFcdwXHBmBqGKoNHLpcLsVpIbScZyLkVmLamfLkrmIC7I5bKZO/OxIk7Yhv/QLRGf6+0wdnh0kQ6ZlRmhXbk9hqQ7rv+g6P+0GyAkIgpDw99T5eoGm8GiGB7ERNuwHiUS5zV+HZ0XcxC8AKa0uH58JrSJmM2PvQLnVhjLMjwPWNXtTymDoaSRHDf+cMK0XfQc1QOBvl26wZ/KIYXD+3H6ekFX62KG2d9hPZ2QQf2JWJZUDCzL3A9+W9ortahXotXOx4rzG+iyxI+DU='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': None}, {'ceph': {'cluster': 'c1', 'conf': {'client': {'debug rgw': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync data inject err probability': 0.1, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)'], 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4', 'wait-for-scrub': False}}, {'ceph': {'cluster': 'c2', 'conf': {'client': {'debug rgw': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync data inject err probability': 0.1, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)'], 'sha1': 'afd422385fab8119a539d289dd1aeaad4c056fe4', 'wait-for-scrub': False}}, {'rgw': {'c1.client.0': {'port': 8000}, 'c1.client.1': {'port': 8001}, 'c2.client.0': {'port': 8000}, 'c2.client.1': {'port': 8001}}}, {'rgw-multisite': None}, {'rgw-multisite-tests': {'config': {'reconfigure_delay': 60}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'b58f2c18636eb10faa77ed3614abd00cb85dfc2c', 'timestamp': '2022-09-02_14:33:28', 'tube': 'smithi', 'user': 'cbodley', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.1713'} 2022-09-03T05:47:46.417 DEBUG:teuthology.orchestra.run.smithi062:> install -d -m0755 -- /home/ubuntu/cephtest/c2.data 2022-09-03T05:47:46.420 DEBUG:teuthology.orchestra.run.smithi087:> install -d -m0755 -- /home/ubuntu/cephtest/c2.data 2022-09-03T05:47:46.441 DEBUG:teuthology.orchestra.run.smithi062:> sudo install -d -m0777 -- /var/run/ceph 2022-09-03T05:47:46.481 DEBUG:teuthology.orchestra.run.smithi087:> sudo install -d -m0777 -- /var/run/ceph 2022-09-03T05:47:46.513 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:46.514 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/scratch_devs of=/dev/stdout 2022-09-03T05:47:46.575 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-09-03T05:47:46.576 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_1 && sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_1 2022-09-03T05:47:46.634 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2022-09-03T05:47:46.635 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-03T05:47:46.637 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 6h/6d Inode: 364286 Links: 1 2022-09-03T05:47:46.637 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-03T05:47:46.638 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2022-09-03T05:47:46.638 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2022-09-03 05:47:01.092634948 +0000 2022-09-03T05:47:46.639 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2022-09-03 05:43:33.446393439 +0000 2022-09-03T05:47:46.639 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2022-09-03 05:43:33.446393439 +0000 2022-09-03T05:47:46.640 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: - 2022-09-03T05:47:46.644 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2022-09-03T05:47:46.645 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2022-09-03T05:47:46.645 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000227975 s, 2.2 MB/s 2022-09-03T05:47:46.648 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_2 && sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_2 2022-09-03T05:47:46.712 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2022-09-03T05:47:46.713 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-03T05:47:46.713 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 6h/6d Inode: 370356 Links: 1 2022-09-03T05:47:46.714 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-03T05:47:46.714 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2022-09-03T05:47:46.714 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2022-09-03 05:47:01.092634948 +0000 2022-09-03T05:47:46.715 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2022-09-03 05:43:34.056379130 +0000 2022-09-03T05:47:46.715 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2022-09-03 05:43:34.056379130 +0000 2022-09-03T05:47:46.716 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: - 2022-09-03T05:47:46.721 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2022-09-03T05:47:46.722 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2022-09-03T05:47:46.722 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000215636 s, 2.4 MB/s 2022-09-03T05:47:46.724 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_3 && sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_3 2022-09-03T05:47:46.793 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2022-09-03T05:47:46.793 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-03T05:47:46.794 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 6h/6d Inode: 361285 Links: 1 2022-09-03T05:47:46.794 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-03T05:47:46.794 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2022-09-03T05:47:46.795 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2022-09-03 05:47:01.092634948 +0000 2022-09-03T05:47:46.795 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2022-09-03 05:43:34.637365502 +0000 2022-09-03T05:47:46.795 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2022-09-03 05:43:34.637365502 +0000 2022-09-03T05:47:46.796 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: - 2022-09-03T05:47:46.801 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2022-09-03T05:47:46.802 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2022-09-03T05:47:46.802 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000185676 s, 2.8 MB/s 2022-09-03T05:47:46.804 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_4 && sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_4 2022-09-03T05:47:46.862 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2022-09-03T05:47:46.863 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-09-03T05:47:46.863 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 6h/6d Inode: 369456 Links: 1 2022-09-03T05:47:46.864 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-09-03T05:47:46.864 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2022-09-03T05:47:46.864 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2022-09-03 05:47:01.092634948 +0000 2022-09-03T05:47:46.865 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2022-09-03 05:43:35.215351944 +0000 2022-09-03T05:47:46.865 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2022-09-03 05:43:35.215351944 +0000 2022-09-03T05:47:46.866 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: - 2022-09-03T05:47:46.872 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2022-09-03T05:47:46.872 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2022-09-03T05:47:46.873 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000219938 s, 2.3 MB/s 2022-09-03T05:47:46.875 INFO:tasks.ceph:osd dev map: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2022-09-03T05:47:46.875 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi087.front.sepia.ceph.com'): {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'}} 2022-09-03T05:47:46.876 INFO:tasks.ceph:Generating config... 2022-09-03T05:47:46.877 INFO:tasks.ceph:[client] debug rgw = 20 2022-09-03T05:47:46.877 INFO:tasks.ceph:[client] debug rgw sync = 20 2022-09-03T05:47:46.878 INFO:tasks.ceph:[client] rgw crypt require ssl = False 2022-09-03T05:47:46.878 INFO:tasks.ceph:[client] rgw crypt s3 kms backend = testing 2022-09-03T05:47:46.878 INFO:tasks.ceph:[client] rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2022-09-03T05:47:46.879 INFO:tasks.ceph:[client] rgw curl low speed time = 300 2022-09-03T05:47:46.879 INFO:tasks.ceph:[client] rgw data log num shards = 4 2022-09-03T05:47:46.879 INFO:tasks.ceph:[client] rgw md log max shards = 4 2022-09-03T05:47:46.879 INFO:tasks.ceph:[client] rgw sync data inject err probability = 0.1 2022-09-03T05:47:46.880 INFO:tasks.ceph:[client] rgw sync log trim interval = 0 2022-09-03T05:47:46.880 INFO:tasks.ceph:[client] rgw sync meta inject err probability = 0.1 2022-09-03T05:47:46.880 INFO:tasks.ceph:[client] rgw sync obj etag verify = True 2022-09-03T05:47:46.881 INFO:tasks.ceph:[client] setgroup = ceph 2022-09-03T05:47:46.881 INFO:tasks.ceph:[client] setuser = ceph 2022-09-03T05:47:46.881 INFO:tasks.ceph:[client.0] rgw data notify interval msec = 0 2022-09-03T05:47:46.882 INFO:tasks.ceph:[mgr] debug mgr = 20 2022-09-03T05:47:46.882 INFO:tasks.ceph:[mgr] debug ms = 1 2022-09-03T05:47:46.882 INFO:tasks.ceph:[mon] debug mon = 20 2022-09-03T05:47:46.883 INFO:tasks.ceph:[mon] debug ms = 1 2022-09-03T05:47:46.883 INFO:tasks.ceph:[mon] debug paxos = 20 2022-09-03T05:47:46.883 INFO:tasks.ceph:[osd] debug ms = 1 2022-09-03T05:47:46.884 INFO:tasks.ceph:[osd] debug osd = 20 2022-09-03T05:47:46.884 INFO:tasks.ceph:[osd] rocksdb delete range threshold = 0 2022-09-03T05:47:46.884 INFO:tasks.ceph:[osd.0] osd_max_omap_entries_per_request = 10 2022-09-03T05:47:46.884 INFO:tasks.ceph:[osd.1] osd_max_omap_entries_per_request = 1000 2022-09-03T05:47:46.885 INFO:tasks.ceph:[osd.2] osd_max_omap_entries_per_request = 10000 2022-09-03T05:47:46.886 INFO:tasks.ceph:Setting up c2.mon.a... 2022-09-03T05:47:46.886 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/c2.keyring 2022-09-03T05:47:46.971 INFO:teuthology.orchestra.run.smithi087.stdout:creating /etc/ceph/c2.keyring 2022-09-03T05:47:46.973 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/c2.keyring 2022-09-03T05:47:47.019 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 0644 /etc/ceph/c2.keyring 2022-09-03T05:47:47.088 DEBUG:tasks.ceph:Ceph mon addresses: [('c2.mon.a', '172.21.15.87')] 2022-09-03T05:47:47.089 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.87'}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'debug ms': 1, 'debug osd': 20, 'rocksdb delete range threshold': 0}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rgw': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync data inject err probability': 0.1, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mon.a': {}, 'client.0': {'rgw data notify interval msec': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}} 2022-09-03T05:47:47.093 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:47.093 DEBUG:teuthology.orchestra.run.smithi087:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2022-09-03T05:47:47.146 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.87 --print /home/ubuntu/cephtest/c2.monmap 2022-09-03T05:47:47.234 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:47:47.235 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:47:47.237 INFO:teuthology.orchestra.run.smithi087.stdout:monmaptool: monmap file /home/ubuntu/cephtest/c2.monmap 2022-09-03T05:47:47.242 INFO:teuthology.orchestra.run.smithi087.stdout:monmaptool: generated fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T05:47:47.242 INFO:teuthology.orchestra.run.smithi087.stdout:setting min_mon_release = octopus 2022-09-03T05:47:47.242 INFO:teuthology.orchestra.run.smithi087.stdout:epoch 0 2022-09-03T05:47:47.243 INFO:teuthology.orchestra.run.smithi087.stdout:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T05:47:47.243 INFO:teuthology.orchestra.run.smithi087.stdout:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T05:47:47.244 INFO:teuthology.orchestra.run.smithi087.stdout:created 2022-09-03T05:47:47.228157+0000 2022-09-03T05:47:47.244 INFO:teuthology.orchestra.run.smithi087.stdout:min_mon_release 15 (octopus) 2022-09-03T05:47:47.244 INFO:teuthology.orchestra.run.smithi087.stdout:election_strategy: 1 2022-09-03T05:47:47.245 INFO:teuthology.orchestra.run.smithi087.stdout:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T05:47:47.246 INFO:teuthology.orchestra.run.smithi087.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/c2.monmap (1 monitors) 2022-09-03T05:47:47.247 DEBUG:teuthology.orchestra.run.smithi087:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2022-09-03T05:47:47.297 INFO:tasks.ceph:Writing /etc/ceph/c2.conf for FSID 716c39f3-2c98-4096-aa80-186bfe26000f... 2022-09-03T05:47:47.299 DEBUG:teuthology.orchestra.run.smithi062:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c2.conf && sudo chmod 0644 /etc/ceph/c2.conf > /dev/null 2022-09-03T05:47:47.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c2.conf && sudo chmod 0644 /etc/ceph/c2.conf > /dev/null 2022-09-03T05:47:47.358 INFO:teuthology.orchestra.run.smithi062.stdout:[global] 2022-09-03T05:47:47.359 INFO:teuthology.orchestra.run.smithi062.stdout: chdir = "" 2022-09-03T05:47:47.359 INFO:teuthology.orchestra.run.smithi062.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2022-09-03T05:47:47.359 INFO:teuthology.orchestra.run.smithi062.stdout: auth supported = cephx 2022-09-03T05:47:47.359 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.360 INFO:teuthology.orchestra.run.smithi062.stdout: filestore xattr use omap = true 2022-09-03T05:47:47.360 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.360 INFO:teuthology.orchestra.run.smithi062.stdout: mon clock drift allowed = 1.000 2022-09-03T05:47:47.361 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.361 INFO:teuthology.orchestra.run.smithi062.stdout: osd crush chooseleaf type = 0 2022-09-03T05:47:47.361 INFO:teuthology.orchestra.run.smithi062.stdout: auth debug = true 2022-09-03T05:47:47.361 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.362 INFO:teuthology.orchestra.run.smithi062.stdout: ms die on old message = true 2022-09-03T05:47:47.362 INFO:teuthology.orchestra.run.smithi062.stdout: ms die on bug = true 2022-09-03T05:47:47.362 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.362 INFO:teuthology.orchestra.run.smithi062.stdout: mon max pg per osd = 10000 # >= luminous 2022-09-03T05:47:47.363 INFO:teuthology.orchestra.run.smithi062.stdout: mon pg warn max object skew = 0 2022-09-03T05:47:47.363 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.363 INFO:teuthology.orchestra.run.smithi062.stdout: # disable pg_autoscaler by default for new pools 2022-09-03T05:47:47.363 INFO:teuthology.orchestra.run.smithi062.stdout: osd_pool_default_pg_autoscale_mode = off 2022-09-03T05:47:47.364 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.364 INFO:teuthology.orchestra.run.smithi062.stdout: osd pool default size = 2 2022-09-03T05:47:47.364 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.364 INFO:teuthology.orchestra.run.smithi062.stdout: mon osd allow primary affinity = true 2022-09-03T05:47:47.365 INFO:teuthology.orchestra.run.smithi062.stdout: mon osd allow pg remap = true 2022-09-03T05:47:47.365 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on legacy crush tunables = false 2022-09-03T05:47:47.365 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on crush straw calc version zero = false 2022-09-03T05:47:47.365 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on no sortbitwise = false 2022-09-03T05:47:47.366 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on osd down out interval zero = false 2022-09-03T05:47:47.366 INFO:teuthology.orchestra.run.smithi062.stdout: mon warn on too few osds = false 2022-09-03T05:47:47.366 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2022-09-03T05:47:47.366 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_pool_no_redundancy = false 2022-09-03T05:47:47.367 INFO:teuthology.orchestra.run.smithi062.stdout: mon_allow_pool_size_one = true 2022-09-03T05:47:47.367 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.367 INFO:teuthology.orchestra.run.smithi062.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2022-09-03T05:47:47.368 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.368 INFO:teuthology.orchestra.run.smithi062.stdout: osd default data pool replay window = 5 2022-09-03T05:47:47.368 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.368 INFO:teuthology.orchestra.run.smithi062.stdout: mon allow pool delete = true 2022-09-03T05:47:47.368 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.369 INFO:teuthology.orchestra.run.smithi062.stdout: mon cluster log file level = debug 2022-09-03T05:47:47.369 INFO:teuthology.orchestra.run.smithi062.stdout: debug asserts on shutdown = true 2022-09-03T05:47:47.369 INFO:teuthology.orchestra.run.smithi062.stdout: mon health detail to clog = false 2022-09-03T05:47:47.369 INFO:teuthology.orchestra.run.smithi062.stdout: mon host = 172.21.15.87 2022-09-03T05:47:47.370 INFO:teuthology.orchestra.run.smithi062.stdout: fsid = 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T05:47:47.370 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.370 INFO:teuthology.orchestra.run.smithi062.stdout:[osd] 2022-09-03T05:47:47.371 INFO:teuthology.orchestra.run.smithi062.stdout: osd journal size = 100 2022-09-03T05:47:47.371 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.371 INFO:teuthology.orchestra.run.smithi062.stdout: osd scrub load threshold = 5.0 2022-09-03T05:47:47.371 INFO:teuthology.orchestra.run.smithi062.stdout: osd scrub max interval = 600 2022-09-03T05:47:47.372 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.372 INFO:teuthology.orchestra.run.smithi062.stdout: osd recover clone overlap = true 2022-09-03T05:47:47.372 INFO:teuthology.orchestra.run.smithi062.stdout: osd recovery max chunk = 1048576 2022-09-03T05:47:47.372 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.373 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug shutdown = true 2022-09-03T05:47:47.373 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug op order = true 2022-09-03T05:47:47.373 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug verify stray on activate = true 2022-09-03T05:47:47.373 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.373 INFO:teuthology.orchestra.run.smithi062.stdout: osd open classes on start = true 2022-09-03T05:47:47.374 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug pg log writeout = true 2022-09-03T05:47:47.374 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.374 INFO:teuthology.orchestra.run.smithi062.stdout: osd deep scrub update digest min age = 30 2022-09-03T05:47:47.374 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.375 INFO:teuthology.orchestra.run.smithi062.stdout: osd map max advance = 10 2022-09-03T05:47:47.375 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.375 INFO:teuthology.orchestra.run.smithi062.stdout: journal zero on create = true 2022-09-03T05:47:47.375 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.376 INFO:teuthology.orchestra.run.smithi062.stdout: filestore ondisk finisher threads = 3 2022-09-03T05:47:47.376 INFO:teuthology.orchestra.run.smithi062.stdout: filestore apply finisher threads = 3 2022-09-03T05:47:47.376 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.376 INFO:teuthology.orchestra.run.smithi062.stdout: bdev debug aio = true 2022-09-03T05:47:47.377 INFO:teuthology.orchestra.run.smithi062.stdout: osd debug misdirected ops = true 2022-09-03T05:47:47.377 INFO:teuthology.orchestra.run.smithi062.stdout: debug ms = 1 2022-09-03T05:47:47.377 INFO:teuthology.orchestra.run.smithi062.stdout: debug osd = 20 2022-09-03T05:47:47.378 INFO:teuthology.orchestra.run.smithi062.stdout: rocksdb delete range threshold = 0 2022-09-03T05:47:47.378 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.378 INFO:teuthology.orchestra.run.smithi062.stdout:[mgr] 2022-09-03T05:47:47.379 INFO:teuthology.orchestra.run.smithi062.stdout: debug ms = 1 2022-09-03T05:47:47.379 INFO:teuthology.orchestra.run.smithi062.stdout: debug mgr = 20 2022-09-03T05:47:47.379 INFO:teuthology.orchestra.run.smithi062.stdout: debug mon = 20 2022-09-03T05:47:47.380 INFO:teuthology.orchestra.run.smithi062.stdout: debug auth = 20 2022-09-03T05:47:47.380 INFO:teuthology.orchestra.run.smithi062.stdout: mon reweight min pgs per osd = 4 2022-09-03T05:47:47.380 INFO:teuthology.orchestra.run.smithi062.stdout: mon reweight min bytes per osd = 10 2022-09-03T05:47:47.381 INFO:teuthology.orchestra.run.smithi062.stdout: mgr/telemetry/nag = false 2022-09-03T05:47:47.381 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.382 INFO:teuthology.orchestra.run.smithi062.stdout:[mon] 2022-09-03T05:47:47.382 INFO:teuthology.orchestra.run.smithi062.stdout: debug ms = 1 2022-09-03T05:47:47.382 INFO:teuthology.orchestra.run.smithi062.stdout: debug mon = 20 2022-09-03T05:47:47.384 INFO:teuthology.orchestra.run.smithi062.stdout: debug paxos = 20 2022-09-03T05:47:47.385 INFO:teuthology.orchestra.run.smithi062.stdout: debug auth = 20 2022-09-03T05:47:47.385 INFO:teuthology.orchestra.run.smithi062.stdout: mon data avail warn = 5 2022-09-03T05:47:47.386 INFO:teuthology.orchestra.run.smithi062.stdout: mon mgr mkfs grace = 240 2022-09-03T05:47:47.386 INFO:teuthology.orchestra.run.smithi062.stdout: mon reweight min pgs per osd = 4 2022-09-03T05:47:47.387 INFO:teuthology.orchestra.run.smithi062.stdout: mon osd reporter subtree level = osd 2022-09-03T05:47:47.387 INFO:teuthology.orchestra.run.smithi062.stdout: mon osd prime pg temp = true 2022-09-03T05:47:47.388 INFO:teuthology.orchestra.run.smithi062.stdout: mon reweight min bytes per osd = 10 2022-09-03T05:47:47.388 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.388 INFO:teuthology.orchestra.run.smithi062.stdout: # rotate auth tickets quickly to exercise renewal paths 2022-09-03T05:47:47.389 INFO:teuthology.orchestra.run.smithi062.stdout: auth mon ticket ttl = 660 # 11m 2022-09-03T05:47:47.389 INFO:teuthology.orchestra.run.smithi062.stdout: auth service ticket ttl = 240 # 4m 2022-09-03T05:47:47.390 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.390 INFO:teuthology.orchestra.run.smithi062.stdout: # don't complain about insecure global_id in the test suite 2022-09-03T05:47:47.391 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_insecure_global_id_reclaim = false 2022-09-03T05:47:47.391 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2022-09-03T05:47:47.392 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.392 INFO:teuthology.orchestra.run.smithi062.stdout: # 1m isn't quite enough 2022-09-03T05:47:47.392 INFO:teuthology.orchestra.run.smithi062.stdout: mon_down_mkfs_grace = 2m 2022-09-03T05:47:47.393 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.393 INFO:teuthology.orchestra.run.smithi062.stdout: mon_warn_on_filestore_osds = false 2022-09-03T05:47:47.394 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:47:47.394 INFO:teuthology.orchestra.run.smithi062.stdout:[client] 2022-09-03T05:47:47.395 INFO:teuthology.orchestra.run.smithi062.stdout: rgw cache enabled = true 2022-09-03T05:47:47.395 INFO:teuthology.orchestra.run.smithi062.stdout: rgw enable ops log = true 2022-09-03T05:47:47.395 INFO:teuthology.orchestra.run.smithi062.stdout: rgw enable usage log = true 2022-09-03T05:47:47.396 INFO:teuthology.orchestra.run.smithi062.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2022-09-03T05:47:47.396 INFO:teuthology.orchestra.run.smithi062.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2022-09-03T05:47:47.397 INFO:teuthology.orchestra.run.smithi062.stdout: debug rgw = 20 2022-09-03T05:47:47.397 INFO:teuthology.orchestra.run.smithi062.stdout: debug rgw sync = 20 2022-09-03T05:47:47.397 INFO:teuthology.orchestra.run.smithi062.stdout: rgw crypt require ssl = False 2022-09-03T05:47:47.398 INFO:teuthology.orchestra.run.smithi062.stdout: rgw crypt s3 kms backend = testing 2022-09-03T05:47:47.398 INFO:teuthology.orchestra.run.smithi062.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2022-09-03T05:47:47.399 INFO:teuthology.orchestra.run.smithi062.stdout: rgw curl low speed time = 300 2022-09-03T05:47:47.399 INFO:teuthology.orchestra.run.smithi062.stdout: rgw data log num shards = 4 2022-09-03T05:47:47.400 INFO:teuthology.orchestra.run.smithi062.stdout: rgw md log max shards = 4 2022-09-03T05:47:47.400 INFO:teuthology.orchestra.run.smithi062.stdout: rgw sync data inject err probability = 0.1 2022-09-03T05:47:47.400 INFO:teuthology.orchestra.run.smithi062.stdout: rgw sync log trim interval = 0 2022-09-03T05:47:47.401 INFO:teuthology.orchestra.run.smithi062.stdout: rgw sync meta inject err probability = 0.1 2022-09-03T05:47:47.401 INFO:teuthology.orchestra.run.smithi062.stdout: rgw sync obj etag verify = True 2022-09-03T05:47:47.401 INFO:teuthology.orchestra.run.smithi062.stdout: setgroup = ceph 2022-09-03T05:47:47.402 INFO:teuthology.orchestra.run.smithi062.stdout: setuser = ceph 2022-09-03T05:47:47.402 INFO:teuthology.orchestra.run.smithi062.stdout:[mon.a] 2022-09-03T05:47:47.402 INFO:teuthology.orchestra.run.smithi062.stdout:[client.0] 2022-09-03T05:47:47.402 INFO:teuthology.orchestra.run.smithi062.stdout: rgw data notify interval msec = 0 2022-09-03T05:47:47.403 INFO:teuthology.orchestra.run.smithi062.stdout:[osd.0] 2022-09-03T05:47:47.403 INFO:teuthology.orchestra.run.smithi062.stdout: osd_max_omap_entries_per_request = 10 2022-09-03T05:47:47.403 INFO:teuthology.orchestra.run.smithi062.stdout:[osd.1] 2022-09-03T05:47:47.403 INFO:teuthology.orchestra.run.smithi062.stdout: osd_max_omap_entries_per_request = 1000 2022-09-03T05:47:47.404 INFO:teuthology.orchestra.run.smithi062.stdout:[osd.2] 2022-09-03T05:47:47.404 INFO:teuthology.orchestra.run.smithi062.stdout: osd_max_omap_entries_per_request = 10000 2022-09-03T05:47:47.406 INFO:teuthology.orchestra.run.smithi087.stdout:[global] 2022-09-03T05:47:47.406 INFO:teuthology.orchestra.run.smithi087.stdout: chdir = "" 2022-09-03T05:47:47.406 INFO:teuthology.orchestra.run.smithi087.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2022-09-03T05:47:47.407 INFO:teuthology.orchestra.run.smithi087.stdout: auth supported = cephx 2022-09-03T05:47:47.407 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.408 INFO:teuthology.orchestra.run.smithi087.stdout: filestore xattr use omap = true 2022-09-03T05:47:47.408 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.409 INFO:teuthology.orchestra.run.smithi087.stdout: mon clock drift allowed = 1.000 2022-09-03T05:47:47.409 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.409 INFO:teuthology.orchestra.run.smithi087.stdout: osd crush chooseleaf type = 0 2022-09-03T05:47:47.410 INFO:teuthology.orchestra.run.smithi087.stdout: auth debug = true 2022-09-03T05:47:47.410 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.410 INFO:teuthology.orchestra.run.smithi087.stdout: ms die on old message = true 2022-09-03T05:47:47.411 INFO:teuthology.orchestra.run.smithi087.stdout: ms die on bug = true 2022-09-03T05:47:47.411 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.411 INFO:teuthology.orchestra.run.smithi087.stdout: mon max pg per osd = 10000 # >= luminous 2022-09-03T05:47:47.412 INFO:teuthology.orchestra.run.smithi087.stdout: mon pg warn max object skew = 0 2022-09-03T05:47:47.412 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.412 INFO:teuthology.orchestra.run.smithi087.stdout: # disable pg_autoscaler by default for new pools 2022-09-03T05:47:47.412 INFO:teuthology.orchestra.run.smithi087.stdout: osd_pool_default_pg_autoscale_mode = off 2022-09-03T05:47:47.413 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.413 INFO:teuthology.orchestra.run.smithi087.stdout: osd pool default size = 2 2022-09-03T05:47:47.413 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.413 INFO:teuthology.orchestra.run.smithi087.stdout: mon osd allow primary affinity = true 2022-09-03T05:47:47.414 INFO:teuthology.orchestra.run.smithi087.stdout: mon osd allow pg remap = true 2022-09-03T05:47:47.414 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on legacy crush tunables = false 2022-09-03T05:47:47.414 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on crush straw calc version zero = false 2022-09-03T05:47:47.414 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on no sortbitwise = false 2022-09-03T05:47:47.415 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on osd down out interval zero = false 2022-09-03T05:47:47.415 INFO:teuthology.orchestra.run.smithi087.stdout: mon warn on too few osds = false 2022-09-03T05:47:47.415 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2022-09-03T05:47:47.415 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_pool_no_redundancy = false 2022-09-03T05:47:47.415 INFO:teuthology.orchestra.run.smithi087.stdout: mon_allow_pool_size_one = true 2022-09-03T05:47:47.416 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.416 INFO:teuthology.orchestra.run.smithi087.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2022-09-03T05:47:47.416 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.416 INFO:teuthology.orchestra.run.smithi087.stdout: osd default data pool replay window = 5 2022-09-03T05:47:47.417 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.417 INFO:teuthology.orchestra.run.smithi087.stdout: mon allow pool delete = true 2022-09-03T05:47:47.418 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.418 INFO:teuthology.orchestra.run.smithi087.stdout: mon cluster log file level = debug 2022-09-03T05:47:47.419 INFO:teuthology.orchestra.run.smithi087.stdout: debug asserts on shutdown = true 2022-09-03T05:47:47.419 INFO:teuthology.orchestra.run.smithi087.stdout: mon health detail to clog = false 2022-09-03T05:47:47.419 INFO:teuthology.orchestra.run.smithi087.stdout: mon host = 172.21.15.87 2022-09-03T05:47:47.419 INFO:teuthology.orchestra.run.smithi087.stdout: fsid = 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T05:47:47.420 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.420 INFO:teuthology.orchestra.run.smithi087.stdout:[osd] 2022-09-03T05:47:47.420 INFO:teuthology.orchestra.run.smithi087.stdout: osd journal size = 100 2022-09-03T05:47:47.420 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.420 INFO:teuthology.orchestra.run.smithi087.stdout: osd scrub load threshold = 5.0 2022-09-03T05:47:47.421 INFO:teuthology.orchestra.run.smithi087.stdout: osd scrub max interval = 600 2022-09-03T05:47:47.421 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.421 INFO:teuthology.orchestra.run.smithi087.stdout: osd recover clone overlap = true 2022-09-03T05:47:47.421 INFO:teuthology.orchestra.run.smithi087.stdout: osd recovery max chunk = 1048576 2022-09-03T05:47:47.422 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.422 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug shutdown = true 2022-09-03T05:47:47.422 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug op order = true 2022-09-03T05:47:47.423 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug verify stray on activate = true 2022-09-03T05:47:47.423 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.423 INFO:teuthology.orchestra.run.smithi087.stdout: osd open classes on start = true 2022-09-03T05:47:47.423 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug pg log writeout = true 2022-09-03T05:47:47.424 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.424 INFO:teuthology.orchestra.run.smithi087.stdout: osd deep scrub update digest min age = 30 2022-09-03T05:47:47.424 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.424 INFO:teuthology.orchestra.run.smithi087.stdout: osd map max advance = 10 2022-09-03T05:47:47.425 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.425 INFO:teuthology.orchestra.run.smithi087.stdout: journal zero on create = true 2022-09-03T05:47:47.425 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.425 INFO:teuthology.orchestra.run.smithi087.stdout: filestore ondisk finisher threads = 3 2022-09-03T05:47:47.425 INFO:teuthology.orchestra.run.smithi087.stdout: filestore apply finisher threads = 3 2022-09-03T05:47:47.426 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.426 INFO:teuthology.orchestra.run.smithi087.stdout: bdev debug aio = true 2022-09-03T05:47:47.426 INFO:teuthology.orchestra.run.smithi087.stdout: osd debug misdirected ops = true 2022-09-03T05:47:47.426 INFO:teuthology.orchestra.run.smithi087.stdout: debug ms = 1 2022-09-03T05:47:47.427 INFO:teuthology.orchestra.run.smithi087.stdout: debug osd = 20 2022-09-03T05:47:47.427 INFO:teuthology.orchestra.run.smithi087.stdout: rocksdb delete range threshold = 0 2022-09-03T05:47:47.427 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.427 INFO:teuthology.orchestra.run.smithi087.stdout:[mgr] 2022-09-03T05:47:47.428 INFO:teuthology.orchestra.run.smithi087.stdout: debug ms = 1 2022-09-03T05:47:47.428 INFO:teuthology.orchestra.run.smithi087.stdout: debug mgr = 20 2022-09-03T05:47:47.428 INFO:teuthology.orchestra.run.smithi087.stdout: debug mon = 20 2022-09-03T05:47:47.428 INFO:teuthology.orchestra.run.smithi087.stdout: debug auth = 20 2022-09-03T05:47:47.428 INFO:teuthology.orchestra.run.smithi087.stdout: mon reweight min pgs per osd = 4 2022-09-03T05:47:47.429 INFO:teuthology.orchestra.run.smithi087.stdout: mon reweight min bytes per osd = 10 2022-09-03T05:47:47.429 INFO:teuthology.orchestra.run.smithi087.stdout: mgr/telemetry/nag = false 2022-09-03T05:47:47.429 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.429 INFO:teuthology.orchestra.run.smithi087.stdout:[mon] 2022-09-03T05:47:47.430 INFO:teuthology.orchestra.run.smithi087.stdout: debug ms = 1 2022-09-03T05:47:47.430 INFO:teuthology.orchestra.run.smithi087.stdout: debug mon = 20 2022-09-03T05:47:47.430 INFO:teuthology.orchestra.run.smithi087.stdout: debug paxos = 20 2022-09-03T05:47:47.430 INFO:teuthology.orchestra.run.smithi087.stdout: debug auth = 20 2022-09-03T05:47:47.431 INFO:teuthology.orchestra.run.smithi087.stdout: mon data avail warn = 5 2022-09-03T05:47:47.431 INFO:teuthology.orchestra.run.smithi087.stdout: mon mgr mkfs grace = 240 2022-09-03T05:47:47.431 INFO:teuthology.orchestra.run.smithi087.stdout: mon reweight min pgs per osd = 4 2022-09-03T05:47:47.431 INFO:teuthology.orchestra.run.smithi087.stdout: mon osd reporter subtree level = osd 2022-09-03T05:47:47.432 INFO:teuthology.orchestra.run.smithi087.stdout: mon osd prime pg temp = true 2022-09-03T05:47:47.432 INFO:teuthology.orchestra.run.smithi087.stdout: mon reweight min bytes per osd = 10 2022-09-03T05:47:47.432 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.433 INFO:teuthology.orchestra.run.smithi087.stdout: # rotate auth tickets quickly to exercise renewal paths 2022-09-03T05:47:47.433 INFO:teuthology.orchestra.run.smithi087.stdout: auth mon ticket ttl = 660 # 11m 2022-09-03T05:47:47.433 INFO:teuthology.orchestra.run.smithi087.stdout: auth service ticket ttl = 240 # 4m 2022-09-03T05:47:47.433 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.433 INFO:teuthology.orchestra.run.smithi087.stdout: # don't complain about insecure global_id in the test suite 2022-09-03T05:47:47.434 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_insecure_global_id_reclaim = false 2022-09-03T05:47:47.434 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2022-09-03T05:47:47.434 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.434 INFO:teuthology.orchestra.run.smithi087.stdout: # 1m isn't quite enough 2022-09-03T05:47:47.435 INFO:teuthology.orchestra.run.smithi087.stdout: mon_down_mkfs_grace = 2m 2022-09-03T05:47:47.435 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.435 INFO:teuthology.orchestra.run.smithi087.stdout: mon_warn_on_filestore_osds = false 2022-09-03T05:47:47.435 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:47.436 INFO:teuthology.orchestra.run.smithi087.stdout:[client] 2022-09-03T05:47:47.436 INFO:teuthology.orchestra.run.smithi087.stdout: rgw cache enabled = true 2022-09-03T05:47:47.436 INFO:teuthology.orchestra.run.smithi087.stdout: rgw enable ops log = true 2022-09-03T05:47:47.436 INFO:teuthology.orchestra.run.smithi087.stdout: rgw enable usage log = true 2022-09-03T05:47:47.437 INFO:teuthology.orchestra.run.smithi087.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2022-09-03T05:47:47.437 INFO:teuthology.orchestra.run.smithi087.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2022-09-03T05:47:47.437 INFO:teuthology.orchestra.run.smithi087.stdout: debug rgw = 20 2022-09-03T05:47:47.437 INFO:teuthology.orchestra.run.smithi087.stdout: debug rgw sync = 20 2022-09-03T05:47:47.438 INFO:teuthology.orchestra.run.smithi087.stdout: rgw crypt require ssl = False 2022-09-03T05:47:47.438 INFO:teuthology.orchestra.run.smithi087.stdout: rgw crypt s3 kms backend = testing 2022-09-03T05:47:47.438 INFO:teuthology.orchestra.run.smithi087.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2022-09-03T05:47:47.438 INFO:teuthology.orchestra.run.smithi087.stdout: rgw curl low speed time = 300 2022-09-03T05:47:47.439 INFO:teuthology.orchestra.run.smithi087.stdout: rgw data log num shards = 4 2022-09-03T05:47:47.439 INFO:teuthology.orchestra.run.smithi087.stdout: rgw md log max shards = 4 2022-09-03T05:47:47.439 INFO:teuthology.orchestra.run.smithi087.stdout: rgw sync data inject err probability = 0.1 2022-09-03T05:47:47.439 INFO:teuthology.orchestra.run.smithi087.stdout: rgw sync log trim interval = 0 2022-09-03T05:47:47.440 INFO:teuthology.orchestra.run.smithi087.stdout: rgw sync meta inject err probability = 0.1 2022-09-03T05:47:47.440 INFO:teuthology.orchestra.run.smithi087.stdout: rgw sync obj etag verify = True 2022-09-03T05:47:47.440 INFO:teuthology.orchestra.run.smithi087.stdout: setgroup = ceph 2022-09-03T05:47:47.440 INFO:teuthology.orchestra.run.smithi087.stdout: setuser = ceph 2022-09-03T05:47:47.440 INFO:teuthology.orchestra.run.smithi087.stdout:[mon.a] 2022-09-03T05:47:47.441 INFO:teuthology.orchestra.run.smithi087.stdout:[client.0] 2022-09-03T05:47:47.441 INFO:teuthology.orchestra.run.smithi087.stdout: rgw data notify interval msec = 0 2022-09-03T05:47:47.441 INFO:teuthology.orchestra.run.smithi087.stdout:[osd.0] 2022-09-03T05:47:47.442 INFO:teuthology.orchestra.run.smithi087.stdout: osd_max_omap_entries_per_request = 10 2022-09-03T05:47:47.442 INFO:teuthology.orchestra.run.smithi087.stdout:[osd.1] 2022-09-03T05:47:47.442 INFO:teuthology.orchestra.run.smithi087.stdout: osd_max_omap_entries_per_request = 1000 2022-09-03T05:47:47.442 INFO:teuthology.orchestra.run.smithi087.stdout:[osd.2] 2022-09-03T05:47:47.442 INFO:teuthology.orchestra.run.smithi087.stdout: osd_max_omap_entries_per_request = 10000 2022-09-03T05:47:47.443 INFO:tasks.ceph:Creating admin key on c2.mon.a... 2022-09-03T05:47:47.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/c2.keyring 2022-09-03T05:47:47.503 INFO:tasks.ceph:Copying monmap to all nodes... 2022-09-03T05:47:47.503 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:47.504 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/etc/ceph/c2.keyring of=/dev/stdout 2022-09-03T05:47:47.522 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:47.523 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/home/ubuntu/cephtest/c2.monmap of=/dev/stdout 2022-09-03T05:47:47.581 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi062.front.sepia.ceph.com 2022-09-03T05:47:47.582 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:47.582 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd of=/etc/ceph/c2.keyring 2022-09-03T05:47:47.583 DEBUG:teuthology.orchestra.run.smithi062:> sudo chmod 0644 /etc/ceph/c2.keyring 2022-09-03T05:47:47.629 DEBUG:teuthology.orchestra.run.smithi062:> set -ex 2022-09-03T05:47:47.630 DEBUG:teuthology.orchestra.run.smithi062:> dd of=/home/ubuntu/cephtest/c2.monmap 2022-09-03T05:47:47.699 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi087.front.sepia.ceph.com 2022-09-03T05:47:47.700 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:47.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/ceph/c2.keyring 2022-09-03T05:47:47.702 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 0644 /etc/ceph/c2.keyring 2022-09-03T05:47:47.740 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:47.740 DEBUG:teuthology.orchestra.run.smithi087:> dd of=/home/ubuntu/cephtest/c2.monmap 2022-09-03T05:47:47.802 INFO:tasks.ceph:Setting up mon nodes... 2022-09-03T05:47:47.803 INFO:tasks.ceph:Setting up mgr nodes... 2022-09-03T05:47:47.804 DEBUG:teuthology.orchestra.run.smithi087:> sudo mkdir -p /var/lib/ceph/mgr/c2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/c2-x/keyring 2022-09-03T05:47:47.895 INFO:teuthology.orchestra.run.smithi087.stdout:creating /var/lib/ceph/mgr/c2-x/keyring 2022-09-03T05:47:47.897 INFO:tasks.ceph:Setting up mds nodes... 2022-09-03T05:47:47.898 INFO:tasks.ceph_client:Setting up client nodes... 2022-09-03T05:47:47.899 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/c2.client.0.keyring && sudo chmod 0644 /etc/ceph/c2.client.0.keyring 2022-09-03T05:47:47.943 INFO:teuthology.orchestra.run.smithi087.stdout:creating /etc/ceph/c2.client.0.keyring 2022-09-03T05:47:47.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/c2.client.1.keyring && sudo chmod 0644 /etc/ceph/c2.client.1.keyring 2022-09-03T05:47:48.038 INFO:teuthology.orchestra.run.smithi087.stdout:creating /etc/ceph/c2.client.1.keyring 2022-09-03T05:47:48.050 INFO:tasks.ceph:Running mkfs on osd nodes... 2022-09-03T05:47:48.051 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi062.front.sepia.ceph.com'): {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@smithi087.front.sepia.ceph.com'): {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'}} 2022-09-03T05:47:48.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo mkdir -p /var/lib/ceph/osd/c2-0 2022-09-03T05:47:48.118 INFO:tasks.ceph:roles_to_devs: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2022-09-03T05:47:48.119 INFO:tasks.ceph:role: c2.osd.0 2022-09-03T05:47:48.120 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi087.front.sepia.ceph.com 2022-09-03T05:47:48.121 DEBUG:teuthology.orchestra.run.smithi087:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2022-09-03T05:47:48.192 INFO:teuthology.orchestra.run.smithi087.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2022-09-03T05:47:48.193 INFO:teuthology.orchestra.run.smithi087.stdout: = sectsz=512 attr=2, projid32bit=1 2022-09-03T05:47:48.193 INFO:teuthology.orchestra.run.smithi087.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2022-09-03T05:47:48.193 INFO:teuthology.orchestra.run.smithi087.stdout: = reflink=1 bigtime=0 inobtcount=0 2022-09-03T05:47:48.194 INFO:teuthology.orchestra.run.smithi087.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2022-09-03T05:47:48.194 INFO:teuthology.orchestra.run.smithi087.stdout: = sunit=0 swidth=0 blks 2022-09-03T05:47:48.194 INFO:teuthology.orchestra.run.smithi087.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2022-09-03T05:47:48.195 INFO:teuthology.orchestra.run.smithi087.stdout:log =internal log bsize=4096 blocks=11443, version=2 2022-09-03T05:47:48.195 INFO:teuthology.orchestra.run.smithi087.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2022-09-03T05:47:48.195 INFO:teuthology.orchestra.run.smithi087.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2022-09-03T05:47:48.622 INFO:teuthology.orchestra.run.smithi087.stdout:Discarding blocks...Done. 2022-09-03T05:47:48.623 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi087.front.sepia.ceph.com -o noatime 2022-09-03T05:47:48.624 DEBUG:teuthology.orchestra.run.smithi087:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/c2-0 2022-09-03T05:47:48.797 DEBUG:teuthology.orchestra.run.smithi087:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-0 2022-09-03T05:47:48.832 DEBUG:teuthology.orchestra.run.smithi087:> sudo mkdir -p /var/lib/ceph/osd/c2-1 2022-09-03T05:47:48.899 INFO:tasks.ceph:roles_to_devs: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2022-09-03T05:47:48.900 INFO:tasks.ceph:role: c2.osd.1 2022-09-03T05:47:48.900 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi087.front.sepia.ceph.com 2022-09-03T05:47:48.900 DEBUG:teuthology.orchestra.run.smithi087:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2022-09-03T05:47:48.978 INFO:teuthology.orchestra.run.smithi087.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2022-09-03T05:47:48.979 INFO:teuthology.orchestra.run.smithi087.stdout: = sectsz=512 attr=2, projid32bit=1 2022-09-03T05:47:48.979 INFO:teuthology.orchestra.run.smithi087.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2022-09-03T05:47:48.980 INFO:teuthology.orchestra.run.smithi087.stdout: = reflink=1 bigtime=0 inobtcount=0 2022-09-03T05:47:48.980 INFO:teuthology.orchestra.run.smithi087.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2022-09-03T05:47:48.980 INFO:teuthology.orchestra.run.smithi087.stdout: = sunit=0 swidth=0 blks 2022-09-03T05:47:48.980 INFO:teuthology.orchestra.run.smithi087.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2022-09-03T05:47:48.981 INFO:teuthology.orchestra.run.smithi087.stdout:log =internal log bsize=4096 blocks=11443, version=2 2022-09-03T05:47:48.981 INFO:teuthology.orchestra.run.smithi087.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2022-09-03T05:47:48.981 INFO:teuthology.orchestra.run.smithi087.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2022-09-03T05:47:49.398 INFO:teuthology.orchestra.run.smithi087.stdout:Discarding blocks...Done. 2022-09-03T05:47:49.399 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi087.front.sepia.ceph.com -o noatime 2022-09-03T05:47:49.400 DEBUG:teuthology.orchestra.run.smithi087:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/c2-1 2022-09-03T05:47:49.500 DEBUG:teuthology.orchestra.run.smithi087:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-1 2022-09-03T05:47:49.532 DEBUG:teuthology.orchestra.run.smithi087:> sudo mkdir -p /var/lib/ceph/osd/c2-2 2022-09-03T05:47:49.600 INFO:tasks.ceph:roles_to_devs: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2022-09-03T05:47:49.600 INFO:tasks.ceph:role: c2.osd.2 2022-09-03T05:47:49.601 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi087.front.sepia.ceph.com 2022-09-03T05:47:49.601 DEBUG:teuthology.orchestra.run.smithi087:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2022-09-03T05:47:49.676 INFO:teuthology.orchestra.run.smithi087.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2022-09-03T05:47:49.676 INFO:teuthology.orchestra.run.smithi087.stdout: = sectsz=512 attr=2, projid32bit=1 2022-09-03T05:47:49.676 INFO:teuthology.orchestra.run.smithi087.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2022-09-03T05:47:49.677 INFO:teuthology.orchestra.run.smithi087.stdout: = reflink=1 bigtime=0 inobtcount=0 2022-09-03T05:47:49.677 INFO:teuthology.orchestra.run.smithi087.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2022-09-03T05:47:49.677 INFO:teuthology.orchestra.run.smithi087.stdout: = sunit=0 swidth=0 blks 2022-09-03T05:47:49.677 INFO:teuthology.orchestra.run.smithi087.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2022-09-03T05:47:49.677 INFO:teuthology.orchestra.run.smithi087.stdout:log =internal log bsize=4096 blocks=11443, version=2 2022-09-03T05:47:49.678 INFO:teuthology.orchestra.run.smithi087.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2022-09-03T05:47:49.678 INFO:teuthology.orchestra.run.smithi087.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2022-09-03T05:47:50.119 INFO:teuthology.orchestra.run.smithi087.stdout:Discarding blocks...Done. 2022-09-03T05:47:50.120 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi087.front.sepia.ceph.com -o noatime 2022-09-03T05:47:50.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/c2-2 2022-09-03T05:47:50.288 DEBUG:teuthology.orchestra.run.smithi087:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-2 2022-09-03T05:47:50.322 DEBUG:teuthology.orchestra.run.smithi087:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/c2.monmap 2022-09-03T05:47:50.408 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:50.407+0000 7f83bf7d1440 -1 auth: error reading file: /var/lib/ceph/osd/c2-0/keyring: can't open /var/lib/ceph/osd/c2-0/keyring: (2) No such file or directory 2022-09-03T05:47:50.411 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:50.409+0000 7f83bf7d1440 -1 created new key in keyring /var/lib/ceph/osd/c2-0/keyring 2022-09-03T05:47:50.412 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:50.410+0000 7f83bf7d1440 -1 bluestore(/var/lib/ceph/osd/c2-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-0/block: (2) No such file or directory 2022-09-03T05:47:50.412 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:50.410+0000 7f83bf7d1440 -1 bluestore(/var/lib/ceph/osd/c2-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-0/block: (2) No such file or directory 2022-09-03T05:47:50.412 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:50.410+0000 7f83bf7d1440 -1 bluestore(/var/lib/ceph/osd/c2-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-0/block: (2) No such file or directory 2022-09-03T05:47:50.420 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:50.417+0000 7f83bf7d1440 -1 bluestore(/var/lib/ceph/osd/c2-0) _read_fsid unparsable uuid 2022-09-03T05:47:52.485 DEBUG:teuthology.orchestra.run.smithi087:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-0 2022-09-03T05:47:52.514 DEBUG:teuthology.orchestra.run.smithi087:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/c2.monmap 2022-09-03T05:47:52.602 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:52.600+0000 7ff7dbcc1440 -1 auth: error reading file: /var/lib/ceph/osd/c2-1/keyring: can't open /var/lib/ceph/osd/c2-1/keyring: (2) No such file or directory 2022-09-03T05:47:52.603 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:52.600+0000 7ff7dbcc1440 -1 created new key in keyring /var/lib/ceph/osd/c2-1/keyring 2022-09-03T05:47:52.603 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:52.600+0000 7ff7dbcc1440 -1 bluestore(/var/lib/ceph/osd/c2-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-1/block: (2) No such file or directory 2022-09-03T05:47:52.603 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:52.600+0000 7ff7dbcc1440 -1 bluestore(/var/lib/ceph/osd/c2-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-1/block: (2) No such file or directory 2022-09-03T05:47:52.603 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:52.600+0000 7ff7dbcc1440 -1 bluestore(/var/lib/ceph/osd/c2-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-1/block: (2) No such file or directory 2022-09-03T05:47:52.604 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:52.601+0000 7ff7dbcc1440 -1 bluestore(/var/lib/ceph/osd/c2-1) _read_fsid unparsable uuid 2022-09-03T05:47:54.676 DEBUG:teuthology.orchestra.run.smithi087:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-1 2022-09-03T05:47:54.709 DEBUG:teuthology.orchestra.run.smithi087:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/c2.monmap 2022-09-03T05:47:54.799 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:54.797+0000 7f476c3c3440 -1 auth: error reading file: /var/lib/ceph/osd/c2-2/keyring: can't open /var/lib/ceph/osd/c2-2/keyring: (2) No such file or directory 2022-09-03T05:47:54.799 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:54.797+0000 7f476c3c3440 -1 created new key in keyring /var/lib/ceph/osd/c2-2/keyring 2022-09-03T05:47:54.800 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:54.797+0000 7f476c3c3440 -1 bluestore(/var/lib/ceph/osd/c2-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-2/block: (2) No such file or directory 2022-09-03T05:47:54.800 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:54.797+0000 7f476c3c3440 -1 bluestore(/var/lib/ceph/osd/c2-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-2/block: (2) No such file or directory 2022-09-03T05:47:54.800 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:54.797+0000 7f476c3c3440 -1 bluestore(/var/lib/ceph/osd/c2-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-2/block: (2) No such file or directory 2022-09-03T05:47:54.801 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:47:54.797+0000 7f476c3c3440 -1 bluestore(/var/lib/ceph/osd/c2-2) _read_fsid unparsable uuid 2022-09-03T05:47:56.865 DEBUG:teuthology.orchestra.run.smithi087:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-2 2022-09-03T05:47:56.895 INFO:tasks.ceph:Reading keys from all nodes... 2022-09-03T05:47:56.895 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:56.896 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/var/lib/ceph/mgr/c2-x/keyring of=/dev/stdout 2022-09-03T05:47:56.964 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:56.965 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/var/lib/ceph/osd/c2-0/keyring of=/dev/stdout 2022-09-03T05:47:57.033 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:57.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/var/lib/ceph/osd/c2-1/keyring of=/dev/stdout 2022-09-03T05:47:57.101 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:57.102 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/var/lib/ceph/osd/c2-2/keyring of=/dev/stdout 2022-09-03T05:47:57.171 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:57.172 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/etc/ceph/c2.client.0.keyring of=/dev/stdout 2022-09-03T05:47:57.229 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:57.230 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/etc/ceph/c2.client.1.keyring of=/dev/stdout 2022-09-03T05:47:57.287 INFO:tasks.ceph:Adding keys to all mons... 2022-09-03T05:47:57.288 DEBUG:teuthology.orchestra.run.smithi087:> sudo tee -a /etc/ceph/c2.keyring 2022-09-03T05:47:57.352 INFO:teuthology.orchestra.run.smithi087.stdout:[mgr.x] 2022-09-03T05:47:57.353 INFO:teuthology.orchestra.run.smithi087.stdout: key = AQCD6hJjWwtWNRAAqXSpNHUPJjltVX4tRU/lXQ== 2022-09-03T05:47:57.353 INFO:teuthology.orchestra.run.smithi087.stdout:[osd.0] 2022-09-03T05:47:57.353 INFO:teuthology.orchestra.run.smithi087.stdout: key = AQCG6hJjvaxYGBAApucz0LLo7qkVmaFoY0GZGg== 2022-09-03T05:47:57.353 INFO:teuthology.orchestra.run.smithi087.stdout:[osd.1] 2022-09-03T05:47:57.354 INFO:teuthology.orchestra.run.smithi087.stdout: key = AQCI6hJjsVPaIxAAj5hBo7evkDhiy5VCgnNWYQ== 2022-09-03T05:47:57.354 INFO:teuthology.orchestra.run.smithi087.stdout:[osd.2] 2022-09-03T05:47:57.354 INFO:teuthology.orchestra.run.smithi087.stdout: key = AQCK6hJj+nqTLxAAm6en9ZmXS/3R2bilpoJVCw== 2022-09-03T05:47:57.355 INFO:teuthology.orchestra.run.smithi087.stdout:[client.0] 2022-09-03T05:47:57.355 INFO:teuthology.orchestra.run.smithi087.stdout: key = AQCD6hJjeyE3OBAA+i1eC12R/cL/NyWiv3vx1w== 2022-09-03T05:47:57.355 INFO:teuthology.orchestra.run.smithi087.stdout:[client.1] 2022-09-03T05:47:57.355 INFO:teuthology.orchestra.run.smithi087.stdout: key = AQCE6hJjB95FAhAAMbf4Uw8pCME14Gj/s6x90Q== 2022-09-03T05:47:57.357 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2022-09-03T05:47:57.450 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2022-09-03T05:47:57.503 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2022-09-03T05:47:57.559 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2022-09-03T05:47:57.614 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2022-09-03T05:47:57.670 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2022-09-03T05:47:57.726 INFO:tasks.ceph:Running mkfs on mon nodes... 2022-09-03T05:47:57.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo mkdir -p /var/lib/ceph/mon/c2-a 2022-09-03T05:47:57.755 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster c2 --mkfs -i a --monmap /home/ubuntu/cephtest/c2.monmap --keyring /etc/ceph/c2.keyring 2022-09-03T05:47:57.852 DEBUG:teuthology.orchestra.run.smithi087:> sudo chown -R ceph:ceph /var/lib/ceph/mon/c2-a 2022-09-03T05:47:57.880 DEBUG:teuthology.orchestra.run.smithi087:> rm -- /home/ubuntu/cephtest/c2.monmap 2022-09-03T05:47:57.939 INFO:tasks.ceph:Starting mon daemons in cluster c2... 2022-09-03T05:47:57.939 INFO:tasks.ceph.c2.mon.a:Restarting daemon 2022-09-03T05:47:57.940 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster c2 -i a 2022-09-03T05:47:57.982 INFO:tasks.ceph.c2.mon.a:Started 2022-09-03T05:47:57.983 INFO:tasks.ceph:Starting mgr daemons in cluster c2... 2022-09-03T05:47:57.984 INFO:tasks.ceph.c2.mgr.x:Restarting daemon 2022-09-03T05:47:57.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster c2 -i x 2022-09-03T05:47:57.988 INFO:tasks.ceph.c2.mgr.x:Started 2022-09-03T05:47:57.989 INFO:tasks.ceph:Setting crush tunables to default 2022-09-03T05:47:57.989 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph --cluster c2 osd crush tunables default 2022-09-03T05:47:58.196 INFO:teuthology.orchestra.run.smithi087.stderr:adjusted tunables profile to default 2022-09-03T05:47:58.207 INFO:tasks.ceph:Starting osd daemons in cluster c2... 2022-09-03T05:47:58.208 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:58.208 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/var/lib/ceph/osd/c2-0/fsid of=/dev/stdout 2022-09-03T05:47:58.235 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:58.236 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/var/lib/ceph/osd/c2-1/fsid of=/dev/stdout 2022-09-03T05:47:58.304 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2022-09-03T05:47:58.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/var/lib/ceph/osd/c2-2/fsid of=/dev/stdout 2022-09-03T05:47:58.370 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph --cluster c2 osd new 9945844e-a7ff-4326-aced-8bda530e3414 0 2022-09-03T05:47:58.621 INFO:teuthology.orchestra.run.smithi087.stdout:0 2022-09-03T05:47:58.635 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph --cluster c2 osd new 5279d264-653f-4870-8c41-d77289d161ab 1 2022-09-03T05:47:58.844 INFO:teuthology.orchestra.run.smithi087.stdout:1 2022-09-03T05:47:58.857 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph --cluster c2 osd new db6a47ad-6658-4973-84ec-7a44740b1ef7 2 2022-09-03T05:47:59.066 INFO:teuthology.orchestra.run.smithi087.stdout:2 2022-09-03T05:47:59.080 INFO:tasks.ceph.c2.osd.0:Restarting daemon 2022-09-03T05:47:59.080 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 0 2022-09-03T05:47:59.084 INFO:tasks.ceph.c2.osd.0:Started 2022-09-03T05:47:59.085 INFO:tasks.ceph.c2.osd.1:Restarting daemon 2022-09-03T05:47:59.086 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 1 2022-09-03T05:47:59.089 INFO:tasks.ceph.c2.osd.1:Started 2022-09-03T05:47:59.090 INFO:tasks.ceph.c2.osd.2:Restarting daemon 2022-09-03T05:47:59.091 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 2 2022-09-03T05:47:59.093 INFO:tasks.ceph.c2.osd.2:Started 2022-09-03T05:47:59.094 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd dump --format=json 2022-09-03T05:47:59.198 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:47:59.197+0000 7f076be01080 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-09-03T05:47:59.271 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:47:59.269+0000 7f076be01080 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-09-03T05:47:59.301 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:47:59.302 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":5,"fsid":"716c39f3-2c98-4096-aa80-186bfe26000f","created":"2022-09-03T05:47:58.109666+0000","modified":"2022-09-03T05:47:59.064639+0000","last_up_change":"0.000000","last_in_change":"2022-09-03T05:47:59.064639+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"9945844e-a7ff-4326-aced-8bda530e3414","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"5279d264-653f-4870-8c41-d77289d161ab","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"db6a47ad-6658-4973-84ec-7a44740b1ef7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:47:59.314 INFO:tasks.ceph.ceph_manager.c2:[] 2022-09-03T05:47:59.315 INFO:tasks.ceph:Waiting for OSDs to come up 2022-09-03T05:47:59.363 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:47:59.362+0000 7f076be01080 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-09-03T05:47:59.546 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:47:59.544+0000 7f076be01080 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-09-03T05:47:59.618 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c2 osd dump --format=json 2022-09-03T05:47:59.692 INFO:tasks.ceph.c2.osd.0.smithi087.stderr:2022-09-03T05:47:59.690+0000 7f5b3cfeb440 -1 Falling back to public interface 2022-09-03T05:47:59.704 INFO:tasks.ceph.c2.osd.1.smithi087.stderr:2022-09-03T05:47:59.703+0000 7f54d6c86440 -1 Falling back to public interface 2022-09-03T05:47:59.718 INFO:tasks.ceph.c2.osd.2.smithi087.stderr:2022-09-03T05:47:59.716+0000 7f64b5371440 -1 Falling back to public interface 2022-09-03T05:47:59.815 INFO:teuthology.misc.health.smithi087.stdout: 2022-09-03T05:47:59.815 INFO:teuthology.misc.health.smithi087.stdout:{"epoch":5,"fsid":"716c39f3-2c98-4096-aa80-186bfe26000f","created":"2022-09-03T05:47:58.109666+0000","modified":"2022-09-03T05:47:59.064639+0000","last_up_change":"0.000000","last_in_change":"2022-09-03T05:47:59.064639+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"9945844e-a7ff-4326-aced-8bda530e3414","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"5279d264-653f-4870-8c41-d77289d161ab","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"db6a47ad-6658-4973-84ec-7a44740b1ef7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:47:59.827 DEBUG:teuthology.misc:0 of 3 OSDs are up 2022-09-03T05:48:00.332 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:00.331+0000 7f076be01080 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-09-03T05:48:00.467 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:00.466+0000 7f076be01080 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-09-03T05:48:00.687 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:00.685+0000 7f076be01080 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-09-03T05:48:00.755 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:00.754+0000 7f076be01080 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-09-03T05:48:00.891 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:00.889+0000 7f076be01080 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-09-03T05:48:01.151 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:01.149+0000 7f076be01080 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-09-03T05:48:01.330 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:01.328+0000 7f076be01080 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2022-09-03T05:48:01.467 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:01.465+0000 7f076be01080 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-09-03T05:48:01.583 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:01.577+0000 7f076be01080 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-09-03T05:48:01.717 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:01.716+0000 7f076be01080 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-09-03T05:48:01.795 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:01.793+0000 7f076be01080 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-09-03T05:48:02.010 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:02.009+0000 7f076be01080 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-09-03T05:48:02.086 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:02.084+0000 7f076be01080 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2022-09-03T05:48:02.466 INFO:tasks.ceph.c2.osd.1.smithi087.stderr:2022-09-03T05:48:02.464+0000 7f54d6c86440 -1 osd.1 0 log_to_monitors true 2022-09-03T05:48:02.477 INFO:tasks.ceph.c2.osd.0.smithi087.stderr:2022-09-03T05:48:02.476+0000 7f5b3cfeb440 -1 osd.0 0 log_to_monitors true 2022-09-03T05:48:02.702 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:02.699+0000 7f076be01080 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-09-03T05:48:02.952 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:02.949+0000 7f076be01080 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-09-03T05:48:03.023 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:03.022+0000 7f076be01080 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-09-03T05:48:03.104 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:03.102+0000 7f076be01080 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-09-03T05:48:03.281 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:03.279+0000 7f076be01080 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-09-03T05:48:03.350 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:03.349+0000 7f076be01080 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-09-03T05:48:03.501 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:03.496+0000 7f076be01080 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-09-03T05:48:04.388 INFO:tasks.ceph.c2.osd.1.smithi087.stderr:2022-09-03T05:48:04.385+0000 7f54caa33700 -1 osd.1 0 waiting for initial osdmap 2022-09-03T05:48:04.396 INFO:tasks.ceph.c2.osd.0.smithi087.stderr:2022-09-03T05:48:04.394+0000 7f5b30d98700 -1 osd.0 0 waiting for initial osdmap 2022-09-03T05:48:04.398 INFO:tasks.ceph.c2.osd.1.smithi087.stderr:2022-09-03T05:48:04.397+0000 7f54c360f700 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-03T05:48:04.405 INFO:tasks.ceph.c2.osd.0.smithi087.stderr:2022-09-03T05:48:04.404+0000 7f5b29974700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-03T05:48:04.536 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T05:48:04.534+0000 7f0725bde700 -1 mgr.server handle_report got status from non-daemon mon.a 2022-09-03T05:48:06.131 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c2 osd dump --format=json 2022-09-03T05:48:06.441 INFO:teuthology.misc.health.smithi087.stdout: 2022-09-03T05:48:06.441 INFO:teuthology.misc.health.smithi087.stdout:{"epoch":9,"fsid":"716c39f3-2c98-4096-aa80-186bfe26000f","created":"2022-09-03T05:47:58.109666+0000","modified":"2022-09-03T05:48:06.394912+0000","last_up_change":"2022-09-03T05:48:05.393596+0000","last_in_change":"2022-09-03T05:47:59.064639+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-03T05:48:05.555096+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"9","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"9945844e-a7ff-4326-aced-8bda530e3414","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":104182},{"type":"v1","addr":"172.21.15.87:6801","nonce":104182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":104182},{"type":"v1","addr":"172.21.15.87:6803","nonce":104182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":104182},{"type":"v1","addr":"172.21.15.87:6807","nonce":104182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":104182},{"type":"v1","addr":"172.21.15.87:6805","nonce":104182}]},"public_addr":"172.21.15.87:6801/104182","cluster_addr":"172.21.15.87:6803/104182","heartbeat_back_addr":"172.21.15.87:6807/104182","heartbeat_front_addr":"172.21.15.87:6805/104182","state":["exists","up"]},{"osd":1,"uuid":"5279d264-653f-4870-8c41-d77289d161ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":104198},{"type":"v1","addr":"172.21.15.87:6809","nonce":104198}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":104198},{"type":"v1","addr":"172.21.15.87:6811","nonce":104198}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":104198},{"type":"v1","addr":"172.21.15.87:6815","nonce":104198}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":104198},{"type":"v1","addr":"172.21.15.87:6813","nonce":104198}]},"public_addr":"172.21.15.87:6809/104198","cluster_addr":"172.21.15.87:6811/104198","heartbeat_back_addr":"172.21.15.87:6815/104198","heartbeat_front_addr":"172.21.15.87:6813/104198","state":["exists","up"]},{"osd":2,"uuid":"db6a47ad-6658-4973-84ec-7a44740b1ef7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:03.497813+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:03.457887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:48:06.454 DEBUG:teuthology.misc:2 of 3 OSDs are up 2022-09-03T05:48:07.492 INFO:tasks.ceph.c2.osd.2.smithi087.stderr:2022-09-03T05:48:07.491+0000 7f64b5371440 -1 osd.2 0 log_to_monitors true 2022-09-03T05:48:09.888 INFO:tasks.ceph.c2.osd.2.smithi087.stderr:2022-09-03T05:48:09.886+0000 7f64a911e700 -1 osd.2 0 waiting for initial osdmap 2022-09-03T05:48:09.897 INFO:tasks.ceph.c2.osd.2.smithi087.stderr:2022-09-03T05:48:09.897+0000 7f64a1cfa700 -1 osd.2 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-09-03T05:48:12.758 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c2 osd dump --format=json 2022-09-03T05:48:13.068 INFO:teuthology.misc.health.smithi087.stdout: 2022-09-03T05:48:13.068 INFO:teuthology.misc.health.smithi087.stdout:{"epoch":14,"fsid":"716c39f3-2c98-4096-aa80-186bfe26000f","created":"2022-09-03T05:47:58.109666+0000","modified":"2022-09-03T05:48:12.406866+0000","last_up_change":"2022-09-03T05:48:10.884846+0000","last_in_change":"2022-09-03T05:47:59.064639+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-03T05:48:05.555096+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9945844e-a7ff-4326-aced-8bda530e3414","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":104182},{"type":"v1","addr":"172.21.15.87:6801","nonce":104182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":104182},{"type":"v1","addr":"172.21.15.87:6803","nonce":104182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":104182},{"type":"v1","addr":"172.21.15.87:6807","nonce":104182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":104182},{"type":"v1","addr":"172.21.15.87:6805","nonce":104182}]},"public_addr":"172.21.15.87:6801/104182","cluster_addr":"172.21.15.87:6803/104182","heartbeat_back_addr":"172.21.15.87:6807/104182","heartbeat_front_addr":"172.21.15.87:6805/104182","state":["exists","up"]},{"osd":1,"uuid":"5279d264-653f-4870-8c41-d77289d161ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":104198},{"type":"v1","addr":"172.21.15.87:6809","nonce":104198}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":104198},{"type":"v1","addr":"172.21.15.87:6811","nonce":104198}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":104198},{"type":"v1","addr":"172.21.15.87:6815","nonce":104198}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":104198},{"type":"v1","addr":"172.21.15.87:6813","nonce":104198}]},"public_addr":"172.21.15.87:6809/104198","cluster_addr":"172.21.15.87:6811/104198","heartbeat_back_addr":"172.21.15.87:6815/104198","heartbeat_front_addr":"172.21.15.87:6813/104198","state":["exists","up"]},{"osd":2,"uuid":"db6a47ad-6658-4973-84ec-7a44740b1ef7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":104199},{"type":"v1","addr":"172.21.15.87:6817","nonce":104199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":104199},{"type":"v1","addr":"172.21.15.87:6819","nonce":104199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":104199},{"type":"v1","addr":"172.21.15.87:6823","nonce":104199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":104199},{"type":"v1","addr":"172.21.15.87:6821","nonce":104199}]},"public_addr":"172.21.15.87:6817/104199","cluster_addr":"172.21.15.87:6819/104199","heartbeat_back_addr":"172.21.15.87:6823/104199","heartbeat_front_addr":"172.21.15.87:6821/104199","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:03.497813+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:03.457887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:08.516923+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:48:13.081 DEBUG:teuthology.misc:3 of 3 OSDs are up 2022-09-03T05:48:13.085 INFO:tasks.ceph:Creating RBD pool 2022-09-03T05:48:13.085 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph --cluster c2 osd pool create rbd 8 2022-09-03T05:48:14.078 INFO:teuthology.orchestra.run.smithi087.stderr:pool 'rbd' created 2022-09-03T05:48:14.091 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph --cluster c2 osd pool application enable rbd rbd --yes-i-really-mean-it 2022-09-03T05:48:15.082 INFO:teuthology.orchestra.run.smithi087.stderr:enabled application 'rbd' on pool 'rbd' 2022-09-03T05:48:15.098 INFO:tasks.ceph:Starting mds daemons in cluster c2... 2022-09-03T05:48:15.099 INFO:tasks.ceph:Waiting until c2 daemons up and pgs clean... 2022-09-03T05:48:15.100 INFO:tasks.ceph.ceph_manager.c2:waiting for mgr available 2022-09-03T05:48:15.100 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 mgr dump --format=json 2022-09-03T05:48:15.101 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2022-09-03T05:48:15.455 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:48:15.470 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":5,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":103775},{"type":"v1","addr":"172.21.15.87:6825","nonce":103775}]},"active_addr":"172.21.15.87:6825/103775","active_change":"2022-09-03T05:48:03.509794+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":0,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.87:0","nonce":2997257487}]},{"addrvec":[{"type":"v2","addr":"172.21.15.87:0","nonce":4084047290}]},{"addrvec":[{"type":"v2","addr":"172.21.15.87:0","nonce":3962773843}]},{"addrvec":[{"type":"v2","addr":"172.21.15.87:0","nonce":3250959883}]}]}} 2022-09-03T05:48:15.473 INFO:tasks.ceph.ceph_manager.c2:mgr available! 2022-09-03T05:48:15.474 INFO:tasks.ceph.ceph_manager.c2:waiting for all up 2022-09-03T05:48:15.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd dump --format=json 2022-09-03T05:48:15.780 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:48:15.780 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":16,"fsid":"716c39f3-2c98-4096-aa80-186bfe26000f","created":"2022-09-03T05:47:58.109666+0000","modified":"2022-09-03T05:48:15.071690+0000","last_up_change":"2022-09-03T05:48:10.884846+0000","last_in_change":"2022-09-03T05:47:59.064639+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-03T05:48:05.555096+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-03T05:48:13.409781+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"9945844e-a7ff-4326-aced-8bda530e3414","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":104182},{"type":"v1","addr":"172.21.15.87:6801","nonce":104182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":104182},{"type":"v1","addr":"172.21.15.87:6803","nonce":104182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":104182},{"type":"v1","addr":"172.21.15.87:6807","nonce":104182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":104182},{"type":"v1","addr":"172.21.15.87:6805","nonce":104182}]},"public_addr":"172.21.15.87:6801/104182","cluster_addr":"172.21.15.87:6803/104182","heartbeat_back_addr":"172.21.15.87:6807/104182","heartbeat_front_addr":"172.21.15.87:6805/104182","state":["exists","up"]},{"osd":1,"uuid":"5279d264-653f-4870-8c41-d77289d161ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":104198},{"type":"v1","addr":"172.21.15.87:6809","nonce":104198}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":104198},{"type":"v1","addr":"172.21.15.87:6811","nonce":104198}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":104198},{"type":"v1","addr":"172.21.15.87:6815","nonce":104198}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":104198},{"type":"v1","addr":"172.21.15.87:6813","nonce":104198}]},"public_addr":"172.21.15.87:6809/104198","cluster_addr":"172.21.15.87:6811/104198","heartbeat_back_addr":"172.21.15.87:6815/104198","heartbeat_front_addr":"172.21.15.87:6813/104198","state":["exists","up"]},{"osd":2,"uuid":"db6a47ad-6658-4973-84ec-7a44740b1ef7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":104199},{"type":"v1","addr":"172.21.15.87:6817","nonce":104199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":104199},{"type":"v1","addr":"172.21.15.87:6819","nonce":104199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":104199},{"type":"v1","addr":"172.21.15.87:6823","nonce":104199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":104199},{"type":"v1","addr":"172.21.15.87:6821","nonce":104199}]},"public_addr":"172.21.15.87:6817/104199","cluster_addr":"172.21.15.87:6819/104199","heartbeat_back_addr":"172.21.15.87:6823/104199","heartbeat_front_addr":"172.21.15.87:6821/104199","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:03.497813+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:03.457887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:08.516923+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:48:15.797 INFO:tasks.ceph.ceph_manager.c2:all up! 2022-09-03T05:48:15.798 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd dump --format=json 2022-09-03T05:48:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:48:16.110 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":17,"fsid":"716c39f3-2c98-4096-aa80-186bfe26000f","created":"2022-09-03T05:47:58.109666+0000","modified":"2022-09-03T05:48:16.074605+0000","last_up_change":"2022-09-03T05:48:10.884846+0000","last_in_change":"2022-09-03T05:47:59.064639+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-03T05:48:05.555096+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-03T05:48:13.409781+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"9945844e-a7ff-4326-aced-8bda530e3414","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":104182},{"type":"v1","addr":"172.21.15.87:6801","nonce":104182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":104182},{"type":"v1","addr":"172.21.15.87:6803","nonce":104182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":104182},{"type":"v1","addr":"172.21.15.87:6807","nonce":104182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":104182},{"type":"v1","addr":"172.21.15.87:6805","nonce":104182}]},"public_addr":"172.21.15.87:6801/104182","cluster_addr":"172.21.15.87:6803/104182","heartbeat_back_addr":"172.21.15.87:6807/104182","heartbeat_front_addr":"172.21.15.87:6805/104182","state":["exists","up"]},{"osd":1,"uuid":"5279d264-653f-4870-8c41-d77289d161ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":104198},{"type":"v1","addr":"172.21.15.87:6809","nonce":104198}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":104198},{"type":"v1","addr":"172.21.15.87:6811","nonce":104198}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":104198},{"type":"v1","addr":"172.21.15.87:6815","nonce":104198}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":104198},{"type":"v1","addr":"172.21.15.87:6813","nonce":104198}]},"public_addr":"172.21.15.87:6809/104198","cluster_addr":"172.21.15.87:6811/104198","heartbeat_back_addr":"172.21.15.87:6815/104198","heartbeat_front_addr":"172.21.15.87:6813/104198","state":["exists","up"]},{"osd":2,"uuid":"db6a47ad-6658-4973-84ec-7a44740b1ef7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":104199},{"type":"v1","addr":"172.21.15.87:6817","nonce":104199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":104199},{"type":"v1","addr":"172.21.15.87:6819","nonce":104199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":104199},{"type":"v1","addr":"172.21.15.87:6823","nonce":104199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":104199},{"type":"v1","addr":"172.21.15.87:6821","nonce":104199}]},"public_addr":"172.21.15.87:6817/104199","cluster_addr":"172.21.15.87:6819/104199","heartbeat_back_addr":"172.21.15.87:6823/104199","heartbeat_front_addr":"172.21.15.87:6821/104199","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:03.497813+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:03.457887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2022-09-03T05:48:08.516923+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-03T05:48:16.125 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.0 flush_pg_stats 2022-09-03T05:48:16.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.1 flush_pg_stats 2022-09-03T05:48:16.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.2 flush_pg_stats 2022-09-03T05:48:16.296 INFO:teuthology.orchestra.run.smithi087.stdout:34359738373 2022-09-03T05:48:16.297 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.1 2022-09-03T05:48:16.299 INFO:teuthology.orchestra.run.smithi087.stdout:55834574852 2022-09-03T05:48:16.300 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.2 2022-09-03T05:48:16.305 INFO:teuthology.orchestra.run.smithi087.stdout:34359738373 2022-09-03T05:48:16.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.0 2022-09-03T05:48:16.650 INFO:teuthology.orchestra.run.smithi087.stdout:34359738371 2022-09-03T05:48:16.666 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738373 got 34359738371 for osd.1 2022-09-03T05:48:16.667 INFO:teuthology.orchestra.run.smithi087.stdout:55834574851 2022-09-03T05:48:16.681 INFO:tasks.ceph.ceph_manager.c2:need seq 55834574852 got 55834574851 for osd.2 2022-09-03T05:48:16.707 INFO:teuthology.orchestra.run.smithi087.stdout:34359738371 2022-09-03T05:48:16.726 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738373 got 34359738371 for osd.0 2022-09-03T05:48:17.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.1 2022-09-03T05:48:17.683 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.2 2022-09-03T05:48:17.729 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.0 2022-09-03T05:48:18.002 INFO:teuthology.orchestra.run.smithi087.stdout:34359738371 2022-09-03T05:48:18.023 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738373 got 34359738371 for osd.1 2022-09-03T05:48:18.053 INFO:teuthology.orchestra.run.smithi087.stdout:55834574851 2022-09-03T05:48:18.069 INFO:tasks.ceph.ceph_manager.c2:need seq 55834574852 got 55834574851 for osd.2 2022-09-03T05:48:18.081 INFO:teuthology.orchestra.run.smithi087.stdout:34359738371 2022-09-03T05:48:18.095 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738373 got 34359738371 for osd.0 2022-09-03T05:48:19.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.1 2022-09-03T05:48:19.070 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.2 2022-09-03T05:48:19.098 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.0 2022-09-03T05:48:19.367 INFO:teuthology.orchestra.run.smithi087.stdout:34359738373 2022-09-03T05:48:19.372 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738373 got 34359738373 for osd.1 2022-09-03T05:48:19.372 DEBUG:teuthology.parallel:result is None 2022-09-03T05:48:19.432 INFO:teuthology.orchestra.run.smithi087.stdout:55834574853 2022-09-03T05:48:19.446 INFO:tasks.ceph.ceph_manager.c2:need seq 55834574852 got 55834574853 for osd.2 2022-09-03T05:48:19.446 DEBUG:teuthology.parallel:result is None 2022-09-03T05:48:19.470 INFO:teuthology.orchestra.run.smithi087.stdout:34359738373 2022-09-03T05:48:19.479 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738373 got 34359738373 for osd.0 2022-09-03T05:48:19.479 DEBUG:teuthology.parallel:result is None 2022-09-03T05:48:19.480 INFO:tasks.ceph.ceph_manager.c2:waiting for clean 2022-09-03T05:48:19.481 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 pg dump --format=json 2022-09-03T05:48:19.821 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:48:19.823 INFO:teuthology.orchestra.run.smithi087.stderr:dumped all 2022-09-03T05:48:19.835 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2022-09-03T05:48:19.516506+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":5,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":314572800,"kb_used":25080,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":23488,"kb_avail":314547720,"statfs":{"total":322122547200,"available":322096865280,"internally_reserved":0,"allocated":1605632,"data_stored":1298057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":24051712},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001146"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535508+0000","last_change":"2022-09-03T05:48:15.080036+0000","last_active":"2022-09-03T05:48:15.535508+0000","last_peered":"2022-09-03T05:48:15.535508+0000","last_clean":"2022-09-03T05:48:15.535508+0000","last_became_active":"2022-09-03T05:48:15.079799+0000","last_became_peered":"2022-09-03T05:48:15.079799+0000","last_unstale":"2022-09-03T05:48:15.535508+0000","last_undegraded":"2022-09-03T05:48:15.535508+0000","last_fullsized":"2022-09-03T05:48:15.535508+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T12:50:29.616056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535498+0000","last_change":"2022-09-03T05:48:15.079857+0000","last_active":"2022-09-03T05:48:15.535498+0000","last_peered":"2022-09-03T05:48:15.535498+0000","last_clean":"2022-09-03T05:48:15.535498+0000","last_became_active":"2022-09-03T05:48:15.079594+0000","last_became_peered":"2022-09-03T05:48:15.079594+0000","last_unstale":"2022-09-03T05:48:15.535498+0000","last_undegraded":"2022-09-03T05:48:15.535498+0000","last_fullsized":"2022-09-03T05:48:15.535498+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T17:16:51.283150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535486+0000","last_change":"2022-09-03T05:48:15.079919+0000","last_active":"2022-09-03T05:48:15.535486+0000","last_peered":"2022-09-03T05:48:15.535486+0000","last_clean":"2022-09-03T05:48:15.535486+0000","last_became_active":"2022-09-03T05:48:15.079647+0000","last_became_peered":"2022-09-03T05:48:15.079647+0000","last_unstale":"2022-09-03T05:48:15.535486+0000","last_undegraded":"2022-09-03T05:48:15.535486+0000","last_fullsized":"2022-09-03T05:48:15.535486+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T17:27:10.076294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.081395+0000","last_change":"2022-09-03T05:48:15.080941+0000","last_active":"2022-09-03T05:48:15.081395+0000","last_peered":"2022-09-03T05:48:15.081395+0000","last_clean":"2022-09-03T05:48:15.081395+0000","last_became_active":"2022-09-03T05:48:15.080593+0000","last_became_peered":"2022-09-03T05:48:15.080593+0000","last_unstale":"2022-09-03T05:48:15.081395+0000","last_undegraded":"2022-09-03T05:48:15.081395+0000","last_fullsized":"2022-09-03T05:48:15.081395+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T07:04:00.170773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"10'97","reported_seq":143,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.077472+0000","last_change":"2022-09-03T05:48:07.476592+0000","last_active":"2022-09-03T05:48:15.077472+0000","last_peered":"2022-09-03T05:48:15.077472+0000","last_clean":"2022-09-03T05:48:15.077472+0000","last_became_active":"2022-09-03T05:48:07.475658+0000","last_became_peered":"2022-09-03T05:48:07.475658+0000","last_unstale":"2022-09-03T05:48:15.077472+0000","last_undegraded":"2022-09-03T05:48:15.077472+0000","last_fullsized":"2022-09-03T05:48:15.077472+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:06.394912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:06.394912+0000","last_clean_scrub_stamp":"2022-09-03T05:48:06.394912+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T17:08:39.266652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535462+0000","last_change":"2022-09-03T05:48:15.079263+0000","last_active":"2022-09-03T05:48:15.535462+0000","last_peered":"2022-09-03T05:48:15.535462+0000","last_clean":"2022-09-03T05:48:15.535462+0000","last_became_active":"2022-09-03T05:48:15.078984+0000","last_became_peered":"2022-09-03T05:48:15.078984+0000","last_unstale":"2022-09-03T05:48:15.535462+0000","last_undegraded":"2022-09-03T05:48:15.535462+0000","last_fullsized":"2022-09-03T05:48:15.535462+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T09:17:25.682635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.081389+0000","last_change":"2022-09-03T05:48:15.080848+0000","last_active":"2022-09-03T05:48:15.081389+0000","last_peered":"2022-09-03T05:48:15.081389+0000","last_clean":"2022-09-03T05:48:15.081389+0000","last_became_active":"2022-09-03T05:48:15.080591+0000","last_became_peered":"2022-09-03T05:48:15.080591+0000","last_unstale":"2022-09-03T05:48:15.081389+0000","last_undegraded":"2022-09-03T05:48:15.081389+0000","last_fullsized":"2022-09-03T05:48:15.081389+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T07:58:43.417661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.080783+0000","last_change":"2022-09-03T05:48:15.079011+0000","last_active":"2022-09-03T05:48:15.080783+0000","last_peered":"2022-09-03T05:48:15.080783+0000","last_clean":"2022-09-03T05:48:15.080783+0000","last_became_active":"2022-09-03T05:48:15.078778+0000","last_became_peered":"2022-09-03T05:48:15.078778+0000","last_unstale":"2022-09-03T05:48:15.080783+0000","last_undegraded":"2022-09-03T05:48:15.080783+0000","last_fullsized":"2022-09-03T05:48:15.080783+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T11:30:16.786597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535475+0000","last_change":"2022-09-03T05:48:15.079783+0000","last_active":"2022-09-03T05:48:15.535475+0000","last_peered":"2022-09-03T05:48:15.535475+0000","last_clean":"2022-09-03T05:48:15.535475+0000","last_became_active":"2022-09-03T05:48:15.079545+0000","last_became_peered":"2022-09-03T05:48:15.079545+0000","last_unstale":"2022-09-03T05:48:15.535475+0000","last_undegraded":"2022-09-03T05:48:15.535475+0000","last_fullsized":"2022-09-03T05:48:15.535475+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T09:45:13.525893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":13,"seq":55834574853,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7696,"kb_used_data":136,"kb_used_omap":0,"kb_used_meta":7552,"kb_avail":104849904,"statfs":{"total":107374182400,"available":107366301696,"internally_reserved":0,"allocated":139264,"data_stored":39107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7733248},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26700000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27800000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8660,"kb_used_data":716,"kb_used_omap":0,"kb_used_meta":7936,"kb_avail":104848940,"statfs":{"total":107374182400,"available":107365314560,"internally_reserved":0,"allocated":733184,"data_stored":629475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8126464},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.189}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.216}]}]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8724,"kb_used_data":716,"kb_used_omap":0,"kb_used_meta":8000,"kb_avail":104848876,"statfs":{"total":107374182400,"available":107365249024,"internally_reserved":0,"allocated":733184,"data_stored":629475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8192000},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29299999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-03T05:48:19.837 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 pg dump --format=json 2022-09-03T05:48:20.137 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:48:20.139 INFO:teuthology.orchestra.run.smithi087.stderr:dumped all 2022-09-03T05:48:20.151 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2022-09-03T05:48:19.516506+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":5,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":314572800,"kb_used":25080,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":23488,"kb_avail":314547720,"statfs":{"total":322122547200,"available":322096865280,"internally_reserved":0,"allocated":1605632,"data_stored":1298057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":24051712},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001146"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535508+0000","last_change":"2022-09-03T05:48:15.080036+0000","last_active":"2022-09-03T05:48:15.535508+0000","last_peered":"2022-09-03T05:48:15.535508+0000","last_clean":"2022-09-03T05:48:15.535508+0000","last_became_active":"2022-09-03T05:48:15.079799+0000","last_became_peered":"2022-09-03T05:48:15.079799+0000","last_unstale":"2022-09-03T05:48:15.535508+0000","last_undegraded":"2022-09-03T05:48:15.535508+0000","last_fullsized":"2022-09-03T05:48:15.535508+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T12:50:29.616056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535498+0000","last_change":"2022-09-03T05:48:15.079857+0000","last_active":"2022-09-03T05:48:15.535498+0000","last_peered":"2022-09-03T05:48:15.535498+0000","last_clean":"2022-09-03T05:48:15.535498+0000","last_became_active":"2022-09-03T05:48:15.079594+0000","last_became_peered":"2022-09-03T05:48:15.079594+0000","last_unstale":"2022-09-03T05:48:15.535498+0000","last_undegraded":"2022-09-03T05:48:15.535498+0000","last_fullsized":"2022-09-03T05:48:15.535498+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T17:16:51.283150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535486+0000","last_change":"2022-09-03T05:48:15.079919+0000","last_active":"2022-09-03T05:48:15.535486+0000","last_peered":"2022-09-03T05:48:15.535486+0000","last_clean":"2022-09-03T05:48:15.535486+0000","last_became_active":"2022-09-03T05:48:15.079647+0000","last_became_peered":"2022-09-03T05:48:15.079647+0000","last_unstale":"2022-09-03T05:48:15.535486+0000","last_undegraded":"2022-09-03T05:48:15.535486+0000","last_fullsized":"2022-09-03T05:48:15.535486+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T17:27:10.076294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.081395+0000","last_change":"2022-09-03T05:48:15.080941+0000","last_active":"2022-09-03T05:48:15.081395+0000","last_peered":"2022-09-03T05:48:15.081395+0000","last_clean":"2022-09-03T05:48:15.081395+0000","last_became_active":"2022-09-03T05:48:15.080593+0000","last_became_peered":"2022-09-03T05:48:15.080593+0000","last_unstale":"2022-09-03T05:48:15.081395+0000","last_undegraded":"2022-09-03T05:48:15.081395+0000","last_fullsized":"2022-09-03T05:48:15.081395+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T07:04:00.170773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"10'97","reported_seq":143,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.077472+0000","last_change":"2022-09-03T05:48:07.476592+0000","last_active":"2022-09-03T05:48:15.077472+0000","last_peered":"2022-09-03T05:48:15.077472+0000","last_clean":"2022-09-03T05:48:15.077472+0000","last_became_active":"2022-09-03T05:48:07.475658+0000","last_became_peered":"2022-09-03T05:48:07.475658+0000","last_unstale":"2022-09-03T05:48:15.077472+0000","last_undegraded":"2022-09-03T05:48:15.077472+0000","last_fullsized":"2022-09-03T05:48:15.077472+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:06.394912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:06.394912+0000","last_clean_scrub_stamp":"2022-09-03T05:48:06.394912+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T17:08:39.266652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535462+0000","last_change":"2022-09-03T05:48:15.079263+0000","last_active":"2022-09-03T05:48:15.535462+0000","last_peered":"2022-09-03T05:48:15.535462+0000","last_clean":"2022-09-03T05:48:15.535462+0000","last_became_active":"2022-09-03T05:48:15.078984+0000","last_became_peered":"2022-09-03T05:48:15.078984+0000","last_unstale":"2022-09-03T05:48:15.535462+0000","last_undegraded":"2022-09-03T05:48:15.535462+0000","last_fullsized":"2022-09-03T05:48:15.535462+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T09:17:25.682635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.081389+0000","last_change":"2022-09-03T05:48:15.080848+0000","last_active":"2022-09-03T05:48:15.081389+0000","last_peered":"2022-09-03T05:48:15.081389+0000","last_clean":"2022-09-03T05:48:15.081389+0000","last_became_active":"2022-09-03T05:48:15.080591+0000","last_became_peered":"2022-09-03T05:48:15.080591+0000","last_unstale":"2022-09-03T05:48:15.081389+0000","last_undegraded":"2022-09-03T05:48:15.081389+0000","last_fullsized":"2022-09-03T05:48:15.081389+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T07:58:43.417661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.080783+0000","last_change":"2022-09-03T05:48:15.079011+0000","last_active":"2022-09-03T05:48:15.080783+0000","last_peered":"2022-09-03T05:48:15.080783+0000","last_clean":"2022-09-03T05:48:15.080783+0000","last_became_active":"2022-09-03T05:48:15.078778+0000","last_became_peered":"2022-09-03T05:48:15.078778+0000","last_unstale":"2022-09-03T05:48:15.080783+0000","last_undegraded":"2022-09-03T05:48:15.080783+0000","last_fullsized":"2022-09-03T05:48:15.080783+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T11:30:16.786597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":16,"state":"active+clean","last_fresh":"2022-09-03T05:48:15.535475+0000","last_change":"2022-09-03T05:48:15.079783+0000","last_active":"2022-09-03T05:48:15.535475+0000","last_peered":"2022-09-03T05:48:15.535475+0000","last_clean":"2022-09-03T05:48:15.535475+0000","last_became_active":"2022-09-03T05:48:15.079545+0000","last_became_peered":"2022-09-03T05:48:15.079545+0000","last_unstale":"2022-09-03T05:48:15.535475+0000","last_undegraded":"2022-09-03T05:48:15.535475+0000","last_fullsized":"2022-09-03T05:48:15.535475+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-09-03T05:48:14.070947+0000","last_clean_scrub_stamp":"2022-09-03T05:48:14.070947+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-09-04T09:45:13.525893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":13,"seq":55834574853,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7696,"kb_used_data":136,"kb_used_omap":0,"kb_used_meta":7552,"kb_avail":104849904,"statfs":{"total":107374182400,"available":107366301696,"internally_reserved":0,"allocated":139264,"data_stored":39107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7733248},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26700000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27800000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8660,"kb_used_data":716,"kb_used_omap":0,"kb_used_meta":7936,"kb_avail":104848940,"statfs":{"total":107374182400,"available":107365314560,"internally_reserved":0,"allocated":733184,"data_stored":629475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8126464},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.189}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.216}]}]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8724,"kb_used_data":716,"kb_used_omap":0,"kb_used_meta":8000,"kb_avail":104848876,"statfs":{"total":107374182400,"available":107365249024,"internally_reserved":0,"allocated":733184,"data_stored":629475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8192000},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29299999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-09-03T05:48:20.152 INFO:tasks.ceph.ceph_manager.c2:clean! 2022-09-03T05:48:20.153 INFO:tasks.ceph:Waiting until ceph cluster c2 is healthy... 2022-09-03T05:48:20.153 INFO:tasks.ceph.ceph_manager.c2:wait_until_healthy 2022-09-03T05:48:20.154 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 health --format=json 2022-09-03T05:48:20.499 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T05:48:20.500 INFO:teuthology.orchestra.run.smithi087.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2022-09-03T05:48:20.513 INFO:tasks.ceph.ceph_manager.c2:wait_until_healthy done 2022-09-03T05:48:20.514 INFO:teuthology.run_tasks:Running task rgw... 2022-09-03T05:48:20.537 DEBUG:tasks.rgw:config is {'c1.client.0': {'port': 8000}, 'c1.client.1': {'port': 8001}, 'c2.client.0': {'port': 8000}, 'c2.client.1': {'port': 8001}} 2022-09-03T05:48:20.538 DEBUG:tasks.rgw:client list is dict_keys(['c1.client.0', 'c1.client.1', 'c2.client.0', 'c2.client.1']) 2022-09-03T05:48:20.538 INFO:tasks.rgw:Creating data pools 2022-09-03T05:48:20.539 DEBUG:tasks.rgw:Obtaining remote for client c1.client.0 2022-09-03T05:48:20.539 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c1 2022-09-03T05:48:21.056 INFO:teuthology.orchestra.run.smithi062.stderr:pool 'default.rgw.buckets.data' created 2022-09-03T05:48:21.070 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c1 2022-09-03T05:48:22.082 INFO:teuthology.orchestra.run.smithi062.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2022-09-03T05:48:22.097 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c1 2022-09-03T05:48:23.080 INFO:teuthology.orchestra.run.smithi062.stderr:pool 'default.rgw.buckets.index' created 2022-09-03T05:48:23.088 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c1 2022-09-03T05:48:24.093 INFO:teuthology.orchestra.run.smithi062.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2022-09-03T05:48:24.110 DEBUG:tasks.rgw:Obtaining remote for client c1.client.1 2022-09-03T05:48:24.111 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c1 2022-09-03T05:48:24.439 INFO:teuthology.orchestra.run.smithi062.stderr:pool 'default.rgw.buckets.data' already exists 2022-09-03T05:48:24.451 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c1 2022-09-03T05:48:25.070 INFO:teuthology.orchestra.run.smithi062.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2022-09-03T05:48:25.081 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c1 2022-09-03T05:48:25.412 INFO:teuthology.orchestra.run.smithi062.stderr:pool 'default.rgw.buckets.index' already exists 2022-09-03T05:48:25.418 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c1 2022-09-03T05:48:26.073 INFO:teuthology.orchestra.run.smithi062.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2022-09-03T05:48:26.088 DEBUG:tasks.rgw:Obtaining remote for client c2.client.0 2022-09-03T05:48:26.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c2 2022-09-03T05:48:27.110 INFO:teuthology.orchestra.run.smithi087.stderr:pool 'default.rgw.buckets.data' created 2022-09-03T05:48:27.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c2 2022-09-03T05:48:28.122 INFO:teuthology.orchestra.run.smithi087.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2022-09-03T05:48:28.136 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c2 2022-09-03T05:48:29.116 INFO:teuthology.orchestra.run.smithi087.stderr:pool 'default.rgw.buckets.index' created 2022-09-03T05:48:29.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c2 2022-09-03T05:48:30.135 INFO:teuthology.orchestra.run.smithi087.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2022-09-03T05:48:30.147 DEBUG:tasks.rgw:Obtaining remote for client c2.client.1 2022-09-03T05:48:30.149 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c2 2022-09-03T05:48:30.461 INFO:teuthology.orchestra.run.smithi087.stderr:pool 'default.rgw.buckets.data' already exists 2022-09-03T05:48:30.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c2 2022-09-03T05:48:31.113 INFO:teuthology.orchestra.run.smithi087.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2022-09-03T05:48:31.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c2 2022-09-03T05:48:31.432 INFO:teuthology.orchestra.run.smithi087.stderr:pool 'default.rgw.buckets.index' already exists 2022-09-03T05:48:31.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c2 2022-09-03T05:48:32.116 INFO:teuthology.orchestra.run.smithi087.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2022-09-03T05:48:32.128 DEBUG:tasks.rgw:Pools created 2022-09-03T05:48:32.129 INFO:tasks.rgw:Configuring compression type = random 2022-09-03T05:48:32.129 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'list'] 2022-09-03T05:48:32.130 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'list'] 2022-09-03T05:48:32.132 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user list 2022-09-03T05:48:32.216 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:32.216 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:32.218 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:32.217+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.113 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.112+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.114 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.112+0000 7fc6e2ac3600 20 realm 2022-09-03T05:48:34.115 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.112+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.116 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.114+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.116 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.114+0000 7fc6e2ac3600 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2022-09-03T05:48:34.117 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.114+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.118 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.114+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.118 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.114+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.119 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.117+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.120 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.117+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.120 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.118+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.121 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.118+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.123 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.119+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.124 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.119+0000 7fc6e2ac3600 10 Creating default zonegroup 2022-09-03T05:48:34.124 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.119+0000 7fc6e2ac3600 10 couldn't find old data placement pools config, setting up new ones for the zone 2022-09-03T05:48:34.131 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.129+0000 7fc6e2ac3600 10 WARNING: store->list_zones() returned r=-2 2022-09-03T05:48:34.131 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.129+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.132 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.129+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.133 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.131+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.133 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.132+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.133 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.132+0000 7fc6e2ac3600 10 could not read realm id: (2) No such file or directory 2022-09-03T05:48:34.133 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.132+0000 7fc6e2ac3600 10 WARNING: failed to set zone as default, r=-22 2022-09-03T05:48:34.134 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.132+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.134 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.132+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.135 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.134+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.137 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.135+0000 7fc6e2ac3600 20 rados_obj.operate() r=0 bl.length=873 2022-09-03T05:48:34.137 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.135+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.138 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.135+0000 7fc6e2ac3600 20 rados_obj.operate() r=0 bl.length=398 2022-09-03T05:48:34.138 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.135+0000 7fc6e2ac3600 10 Using default name default 2022-09-03T05:48:34.138 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.135+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.138 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:34.139 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.139 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 20 rados_obj.operate() r=0 bl.length=873 2022-09-03T05:48:34.139 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 20 zone default found 2022-09-03T05:48:34.139 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 4 Realm: () 2022-09-03T05:48:34.140 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 4 ZoneGroup: default (d80da1d5-c2d8-40ae-89aa-3d16e79a89c0) 2022-09-03T05:48:34.140 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 4 Zone: default (74ed39f8-feb8-49f4-94c6-d843d64ef963) 2022-09-03T05:48:34.140 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 10 cannot find current period zonegroup using local zonegroup configuration 2022-09-03T05:48:34.140 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 20 zonegroup default 2022-09-03T05:48:34.141 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:34.141 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:34.141 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:34.136+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:36.118 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:36.117+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:36.119 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:36.117+0000 7fc6e2ac3600 20 rados->read ofs=0 len=0 2022-09-03T05:48:36.124 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:36.123+0000 7fc6e2ac3600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:36.124 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:36.123+0000 7fc6e2ac3600 20 started sync module instance, tier type = 2022-09-03T05:48:36.125 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:36.123+0000 7fc6e2ac3600 20 started zone id=74ed39f8-feb8-49f4-94c6-d843d64ef963 (name=default) with tier type = 2022-09-03T05:48:38.155 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.153+0000 7fc6e2ac3600 20 add_watcher() i=0 2022-09-03T05:48:38.155 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.153+0000 7fc6e2ac3600 20 add_watcher() i=1 2022-09-03T05:48:38.156 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.153+0000 7fc6e2ac3600 20 add_watcher() i=2 2022-09-03T05:48:38.157 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.153+0000 7fc6e2ac3600 20 add_watcher() i=3 2022-09-03T05:48:38.157 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.153+0000 7fc6e2ac3600 20 add_watcher() i=4 2022-09-03T05:48:38.158 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.153+0000 7fc6e2ac3600 20 add_watcher() i=5 2022-09-03T05:48:38.159 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.153+0000 7fc6e2ac3600 20 add_watcher() i=6 2022-09-03T05:48:38.163 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.156+0000 7fc6e2ac3600 20 add_watcher() i=7 2022-09-03T05:48:38.163 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.156+0000 7fc6e2ac3600 2 all 8 watchers are set, enabling cache 2022-09-03T05:48:38.164 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.156+0000 7fc6e2ac3600 5 tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):410: oid=data_loggenerations_metadata not found 2022-09-03T05:48:38.164 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.156+0000 7fc6e2ac3600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.0 2022-09-03T05:48:38.165 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.156+0000 7fc6e2ac3600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:38.165 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.156+0000 7fc6e2ac3600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.0 does not exist 2022-09-03T05:48:38.166 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.156+0000 7fc6e2ac3600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.1 2022-09-03T05:48:38.166 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.156+0000 7fc6e2ac3600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:38.167 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.157+0000 7fc6e2ac3600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.1 does not exist 2022-09-03T05:48:38.167 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.157+0000 7fc6e2ac3600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.2 2022-09-03T05:48:38.168 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.157+0000 7fc6e2ac3600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:38.168 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.157+0000 7fc6e2ac3600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.2 does not exist 2022-09-03T05:48:38.169 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.157+0000 7fc6e2ac3600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.3 2022-09-03T05:48:38.169 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.157+0000 7fc6e2ac3600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:38.169 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.157+0000 7fc6e2ac3600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.3 does not exist 2022-09-03T05:48:38.170 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.157+0000 7fc6e2ac3600 20 static int rgw::cls::fifo::FIFO::create(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, std::optional >, bool, uint64_t, uint64_t):1203 entering 2022-09-03T05:48:38.170 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.158+0000 7fc6e2ac3600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:38.171 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.161+0000 7fc6cb244700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-09-03T05:48:38.171 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.161+0000 7fc6e2ac3600 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2022-09-03T05:48:38.172 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:38.161+0000 7fc6e2ac3600 20 check_secure_mon_conn(): mode 1 is insecure 2022-09-03T05:48:41.120 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.119+0000 7fc6e2ac3600 10 rgw_init_ioctx warning: failed to set recovery_priority on default.rgw.meta 2022-09-03T05:48:41.121 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.119+0000 7fc6e2ac3600 5 note: GC not initialized 2022-09-03T05:48:41.122 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.121+0000 7fc6c6a3b700 20 reqs_thread_entry: start 2022-09-03T05:48:41.122 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.121+0000 7fc6e2ac3600 20 init_complete bucket index max shards: 11 2022-09-03T05:48:41.123 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.121+0000 7fc6e2ac3600 10 rgw notify: Started notification manager with: 1 workers 2022-09-03T05:48:41.123 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.121+0000 7fc6c4a37700 20 rgw notify: INFO: next queues processing will happen at: Sat Sep 3 05:49:11 2022 2022-09-03T05:48:41.123 INFO:teuthology.orchestra.run.smithi062.stderr: 2022-09-03T05:48:41.123 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.121+0000 7fc6c4215700 20 reqs_thread_entry: start 2022-09-03T05:48:41.144 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T05:48:41.145 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.143+0000 7fc6e2ac3600 20 remove_watcher() i=0 2022-09-03T05:48:41.145 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.143+0000 7fc6e2ac3600 2 removed watcher, disabling cache 2022-09-03T05:48:41.147 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.146+0000 7fc6e2ac3600 20 remove_watcher() i=1 2022-09-03T05:48:41.150 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.149+0000 7fc6e2ac3600 20 remove_watcher() i=2 2022-09-03T05:48:41.152 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.151+0000 7fc6e2ac3600 20 remove_watcher() i=3 2022-09-03T05:48:41.155 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.154+0000 7fc6e2ac3600 20 remove_watcher() i=4 2022-09-03T05:48:41.158 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.155+0000 7fc6e2ac3600 20 remove_watcher() i=5 2022-09-03T05:48:41.159 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.157+0000 7fc6e2ac3600 20 remove_watcher() i=6 2022-09-03T05:48:41.160 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.159+0000 7fc6e2ac3600 20 remove_watcher() i=7 2022-09-03T05:48:41.161 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.160+0000 7fc6cb244700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-09-03T05:48:41.167 DEBUG:tasks.util.rgw: json result: [] 2022-09-03T05:48:41.168 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2022-09-03T05:48:41.169 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2022-09-03T05:48:41.170 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2022-09-03T05:48:41.226 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:41.227 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:41.232 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.231+0000 7f2f0a1e7600 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2022-09-03T05:48:41.233 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.231+0000 7f2f0a1e7600 20 check_secure_mon_conn(): mode 1 is insecure 2022-09-03T05:48:41.236 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.235+0000 7f2eef161700 20 reqs_thread_entry: start 2022-09-03T05:48:41.237 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.235+0000 7f2f0a1e7600 20 try_to_resolve_local_entities(): before: realm_id= realm_name= zonegroup_id= zonegroup_name= zone_id= zone_name=default 2022-09-03T05:48:41.238 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.235+0000 7f2f0a1e7600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.239 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.235+0000 7f2f0a1e7600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:41.239 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.235+0000 7f2f0a1e7600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.239 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.236+0000 7f2f0a1e7600 20 rados_obj.operate() r=0 bl.length=873 2022-09-03T05:48:41.247 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.244+0000 7f2f0a1e7600 20 RGWRados::pool_iterate: got zone_info.74ed39f8-feb8-49f4-94c6-d843d64ef963 2022-09-03T05:48:41.247 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.244+0000 7f2f0a1e7600 20 RGWRados::pool_iterate: got zonegroup_info.d80da1d5-c2d8-40ae-89aa-3d16e79a89c0 2022-09-03T05:48:41.248 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.244+0000 7f2f0a1e7600 20 RGWRados::pool_iterate: got zone_names.default 2022-09-03T05:48:41.248 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.244+0000 7f2f0a1e7600 20 RGWRados::pool_iterate: got zonegroups_names.default 2022-09-03T05:48:41.249 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.244+0000 7f2f0a1e7600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.249 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.244+0000 7f2f0a1e7600 20 rados_obj.operate() r=0 bl.length=873 2022-09-03T05:48:41.249 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.244+0000 7f2f0a1e7600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.250 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.244+0000 7f2f0a1e7600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:41.250 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.244+0000 7f2f0a1e7600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.251 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.245+0000 7f2f0a1e7600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:41.251 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.245+0000 7f2f0a1e7600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.252 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.245+0000 7f2f0a1e7600 20 rados_obj.operate() r=0 bl.length=398 2022-09-03T05:48:41.254 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:48:41.255 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "74ed39f8-feb8-49f4-94c6-d843d64ef963", 2022-09-03T05:48:41.255 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default", 2022-09-03T05:48:41.256 INFO:teuthology.orchestra.run.smithi062.stdout: "domain_root": "default.rgw.meta:root", 2022-09-03T05:48:41.256 INFO:teuthology.orchestra.run.smithi062.stdout: "control_pool": "default.rgw.control", 2022-09-03T05:48:41.257 INFO:teuthology.orchestra.run.smithi062.stdout: "gc_pool": "default.rgw.log:gc", 2022-09-03T05:48:41.257 INFO:teuthology.orchestra.run.smithi062.stdout: "lc_pool": "default.rgw.log:lc", 2022-09-03T05:48:41.258 INFO:teuthology.orchestra.run.smithi062.stdout: "log_pool": "default.rgw.log", 2022-09-03T05:48:41.258 INFO:teuthology.orchestra.run.smithi062.stdout: "intent_log_pool": "default.rgw.log:intent", 2022-09-03T05:48:41.259 INFO:teuthology.orchestra.run.smithi062.stdout: "usage_log_pool": "default.rgw.log:usage", 2022-09-03T05:48:41.259 INFO:teuthology.orchestra.run.smithi062.stdout: "roles_pool": "default.rgw.meta:roles", 2022-09-03T05:48:41.259 INFO:teuthology.orchestra.run.smithi062.stdout: "reshard_pool": "default.rgw.log:reshard", 2022-09-03T05:48:41.260 INFO:teuthology.orchestra.run.smithi062.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2022-09-03T05:48:41.260 INFO:teuthology.orchestra.run.smithi062.stdout: "user_email_pool": "default.rgw.meta:users.email", 2022-09-03T05:48:41.261 INFO:teuthology.orchestra.run.smithi062.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2022-09-03T05:48:41.261 INFO:teuthology.orchestra.run.smithi062.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2022-09-03T05:48:41.262 INFO:teuthology.orchestra.run.smithi062.stdout: "otp_pool": "default.rgw.otp", 2022-09-03T05:48:41.262 INFO:teuthology.orchestra.run.smithi062.stdout: "system_key": { 2022-09-03T05:48:41.262 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "", 2022-09-03T05:48:41.263 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "" 2022-09-03T05:48:41.263 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:48:41.264 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_pools": [ 2022-09-03T05:48:41.264 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:48:41.268 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "default-placement", 2022-09-03T05:48:41.268 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:48:41.268 INFO:teuthology.orchestra.run.smithi062.stdout: "index_pool": "default.rgw.buckets.index", 2022-09-03T05:48:41.269 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": { 2022-09-03T05:48:41.269 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD": { 2022-09-03T05:48:41.269 INFO:teuthology.orchestra.run.smithi062.stdout: "data_pool": "default.rgw.buckets.data", 2022-09-03T05:48:41.269 INFO:teuthology.orchestra.run.smithi062.stdout: "compression_type": "random" 2022-09-03T05:48:41.270 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:48:41.270 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:48:41.270 INFO:teuthology.orchestra.run.smithi062.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2022-09-03T05:48:41.270 INFO:teuthology.orchestra.run.smithi062.stdout: "index_type": 0 2022-09-03T05:48:41.271 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:48:41.271 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:48:41.271 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:48:41.272 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "", 2022-09-03T05:48:41.272 INFO:teuthology.orchestra.run.smithi062.stdout: "notif_pool": "default.rgw.log:notif" 2022-09-03T05:48:41.272 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:48:41.273 DEBUG:tasks.util.rgw: json result: {'id': '74ed39f8-feb8-49f4-94c6-d843d64ef963', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2022-09-03T05:48:41.273 INFO:tasks.util.rgw:rgwadmin: c1.client.1 : ['user', 'list'] 2022-09-03T05:48:41.274 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c1', 'user', 'list'] 2022-09-03T05:48:41.274 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c1 user list 2022-09-03T05:48:41.320 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:41.321 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:41.326 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.325+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.328 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.326+0000 7fab8ac91600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:41.328 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.326+0000 7fab8ac91600 20 realm 2022-09-03T05:48:41.328 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.326+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.329 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.326+0000 7fab8ac91600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:41.329 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.326+0000 7fab8ac91600 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2022-09-03T05:48:41.329 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.326+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.329 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.326+0000 7fab8ac91600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:41.330 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.326+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.330 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.327+0000 7fab8ac91600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:41.330 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.327+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.331 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.327+0000 7fab8ac91600 20 rados_obj.operate() r=0 bl.length=889 2022-09-03T05:48:41.331 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.327+0000 7fab8ac91600 20 searching for the correct realm 2022-09-03T05:48:41.337 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 RGWRados::pool_iterate: got zone_info.74ed39f8-feb8-49f4-94c6-d843d64ef963 2022-09-03T05:48:41.338 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 RGWRados::pool_iterate: got zonegroup_info.d80da1d5-c2d8-40ae-89aa-3d16e79a89c0 2022-09-03T05:48:41.338 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 RGWRados::pool_iterate: got zone_names.default 2022-09-03T05:48:41.338 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 RGWRados::pool_iterate: got zonegroups_names.default 2022-09-03T05:48:41.339 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.339 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:41.339 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.339 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:41.339 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.340 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 rados_obj.operate() r=0 bl.length=398 2022-09-03T05:48:41.340 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 zone default found 2022-09-03T05:48:41.340 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 4 Realm: () 2022-09-03T05:48:41.341 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 4 ZoneGroup: default (d80da1d5-c2d8-40ae-89aa-3d16e79a89c0) 2022-09-03T05:48:41.341 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 4 Zone: default (74ed39f8-feb8-49f4-94c6-d843d64ef963) 2022-09-03T05:48:41.342 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 10 cannot find current period zonegroup using local zonegroup configuration 2022-09-03T05:48:41.342 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 zonegroup default 2022-09-03T05:48:41.343 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.335+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.343 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.336+0000 7fab8ac91600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:41.343 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.336+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.344 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.336+0000 7fab8ac91600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:41.344 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.336+0000 7fab8ac91600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.345 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.336+0000 7fab8ac91600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:41.345 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.336+0000 7fab8ac91600 20 started sync module instance, tier type = 2022-09-03T05:48:41.345 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.336+0000 7fab8ac91600 20 started zone id=74ed39f8-feb8-49f4-94c6-d843d64ef963 (name=default) with tier type = 2022-09-03T05:48:41.362 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.361+0000 7fab8ac91600 20 add_watcher() i=0 2022-09-03T05:48:41.363 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.361+0000 7fab8ac91600 20 add_watcher() i=1 2022-09-03T05:48:41.363 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.361+0000 7fab8ac91600 20 add_watcher() i=2 2022-09-03T05:48:41.363 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.361+0000 7fab8ac91600 20 add_watcher() i=3 2022-09-03T05:48:41.363 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.361+0000 7fab8ac91600 20 add_watcher() i=4 2022-09-03T05:48:41.364 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.361+0000 7fab8ac91600 20 add_watcher() i=5 2022-09-03T05:48:41.364 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.361+0000 7fab8ac91600 20 add_watcher() i=6 2022-09-03T05:48:41.364 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.363+0000 7fab8ac91600 20 add_watcher() i=7 2022-09-03T05:48:41.365 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.363+0000 7fab8ac91600 2 all 8 watchers are set, enabling cache 2022-09-03T05:48:41.366 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.365+0000 7fab73412700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-09-03T05:48:41.366 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.365+0000 7fab8ac91600 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2022-09-03T05:48:41.366 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.365+0000 7fab8ac91600 20 check_secure_mon_conn(): mode 1 is insecure 2022-09-03T05:48:41.370 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.369+0000 7fab8ac91600 5 note: GC not initialized 2022-09-03T05:48:41.371 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.370+0000 7fab6ec09700 20 reqs_thread_entry: start 2022-09-03T05:48:41.371 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.370+0000 7fab8ac91600 20 init_complete bucket index max shards: 11 2022-09-03T05:48:41.372 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.370+0000 7fab8ac91600 10 rgw notify: Started notification manager with: 1 workers 2022-09-03T05:48:41.372 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.370+0000 7fab6cc05700 20 rgw notify: INFO: next queues processing will happen at: Sat Sep 3 05:49:11 2022 2022-09-03T05:48:41.372 INFO:teuthology.orchestra.run.smithi062.stderr: 2022-09-03T05:48:41.372 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.370+0000 7fab6c3e3700 20 reqs_thread_entry: start 2022-09-03T05:48:41.378 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T05:48:41.379 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.377+0000 7fab8ac91600 20 remove_watcher() i=0 2022-09-03T05:48:41.380 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.377+0000 7fab8ac91600 2 removed watcher, disabling cache 2022-09-03T05:48:41.380 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.378+0000 7fab8ac91600 20 remove_watcher() i=1 2022-09-03T05:48:41.381 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.379+0000 7fab8ac91600 20 remove_watcher() i=2 2022-09-03T05:48:41.381 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.380+0000 7fab8ac91600 20 remove_watcher() i=3 2022-09-03T05:48:41.382 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.381+0000 7fab8ac91600 20 remove_watcher() i=4 2022-09-03T05:48:41.383 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.382+0000 7fab8ac91600 20 remove_watcher() i=5 2022-09-03T05:48:41.384 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.382+0000 7fab8ac91600 20 remove_watcher() i=6 2022-09-03T05:48:41.385 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.384+0000 7fab8ac91600 20 remove_watcher() i=7 2022-09-03T05:48:41.386 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.385+0000 7fab73412700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-09-03T05:48:41.393 DEBUG:tasks.util.rgw: json result: [] 2022-09-03T05:48:41.393 INFO:tasks.util.rgw:rgwadmin: c1.client.1 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2022-09-03T05:48:41.394 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c1', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2022-09-03T05:48:41.394 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c1 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2022-09-03T05:48:41.445 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:41.446 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:41.451 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.450+0000 7fb590bfb600 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2022-09-03T05:48:41.451 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.450+0000 7fb590bfb600 20 check_secure_mon_conn(): mode 1 is insecure 2022-09-03T05:48:41.454 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.453+0000 7fb575b75700 20 reqs_thread_entry: start 2022-09-03T05:48:41.455 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.453+0000 7fb590bfb600 20 try_to_resolve_local_entities(): before: realm_id= realm_name= zonegroup_id= zonegroup_name= zone_id= zone_name=default 2022-09-03T05:48:41.455 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.453+0000 7fb590bfb600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.456 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.454+0000 7fb590bfb600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:41.456 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.454+0000 7fb590bfb600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.456 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.454+0000 7fb590bfb600 20 rados_obj.operate() r=0 bl.length=889 2022-09-03T05:48:41.463 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.461+0000 7fb590bfb600 20 RGWRados::pool_iterate: got zone_info.74ed39f8-feb8-49f4-94c6-d843d64ef963 2022-09-03T05:48:41.463 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.461+0000 7fb590bfb600 20 RGWRados::pool_iterate: got zonegroup_info.d80da1d5-c2d8-40ae-89aa-3d16e79a89c0 2022-09-03T05:48:41.464 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.461+0000 7fb590bfb600 20 RGWRados::pool_iterate: got zone_names.default 2022-09-03T05:48:41.464 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.461+0000 7fb590bfb600 20 RGWRados::pool_iterate: got zonegroups_names.default 2022-09-03T05:48:41.464 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.461+0000 7fb590bfb600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.464 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.461+0000 7fb590bfb600 20 rados_obj.operate() r=0 bl.length=889 2022-09-03T05:48:41.464 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.461+0000 7fb590bfb600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.465 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.462+0000 7fb590bfb600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:41.465 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.462+0000 7fb590bfb600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.465 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.462+0000 7fb590bfb600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:41.465 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.462+0000 7fb590bfb600 20 rados->read ofs=0 len=0 2022-09-03T05:48:41.466 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:41.462+0000 7fb590bfb600 20 rados_obj.operate() r=0 bl.length=398 2022-09-03T05:48:41.466 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:48:41.467 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "74ed39f8-feb8-49f4-94c6-d843d64ef963", 2022-09-03T05:48:41.467 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default", 2022-09-03T05:48:41.467 INFO:teuthology.orchestra.run.smithi062.stdout: "domain_root": "default.rgw.meta:root", 2022-09-03T05:48:41.467 INFO:teuthology.orchestra.run.smithi062.stdout: "control_pool": "default.rgw.control", 2022-09-03T05:48:41.468 INFO:teuthology.orchestra.run.smithi062.stdout: "gc_pool": "default.rgw.log:gc", 2022-09-03T05:48:41.468 INFO:teuthology.orchestra.run.smithi062.stdout: "lc_pool": "default.rgw.log:lc", 2022-09-03T05:48:41.468 INFO:teuthology.orchestra.run.smithi062.stdout: "log_pool": "default.rgw.log", 2022-09-03T05:48:41.468 INFO:teuthology.orchestra.run.smithi062.stdout: "intent_log_pool": "default.rgw.log:intent", 2022-09-03T05:48:41.469 INFO:teuthology.orchestra.run.smithi062.stdout: "usage_log_pool": "default.rgw.log:usage", 2022-09-03T05:48:41.469 INFO:teuthology.orchestra.run.smithi062.stdout: "roles_pool": "default.rgw.meta:roles", 2022-09-03T05:48:41.469 INFO:teuthology.orchestra.run.smithi062.stdout: "reshard_pool": "default.rgw.log:reshard", 2022-09-03T05:48:41.469 INFO:teuthology.orchestra.run.smithi062.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2022-09-03T05:48:41.469 INFO:teuthology.orchestra.run.smithi062.stdout: "user_email_pool": "default.rgw.meta:users.email", 2022-09-03T05:48:41.470 INFO:teuthology.orchestra.run.smithi062.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2022-09-03T05:48:41.470 INFO:teuthology.orchestra.run.smithi062.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2022-09-03T05:48:41.470 INFO:teuthology.orchestra.run.smithi062.stdout: "otp_pool": "default.rgw.otp", 2022-09-03T05:48:41.470 INFO:teuthology.orchestra.run.smithi062.stdout: "system_key": { 2022-09-03T05:48:41.471 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "", 2022-09-03T05:48:41.471 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "" 2022-09-03T05:48:41.471 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:48:41.471 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_pools": [ 2022-09-03T05:48:41.471 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:48:41.472 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "default-placement", 2022-09-03T05:48:41.472 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:48:41.472 INFO:teuthology.orchestra.run.smithi062.stdout: "index_pool": "default.rgw.buckets.index", 2022-09-03T05:48:41.472 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": { 2022-09-03T05:48:41.473 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD": { 2022-09-03T05:48:41.473 INFO:teuthology.orchestra.run.smithi062.stdout: "data_pool": "default.rgw.buckets.data", 2022-09-03T05:48:41.473 INFO:teuthology.orchestra.run.smithi062.stdout: "compression_type": "random" 2022-09-03T05:48:41.473 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:48:41.473 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:48:41.474 INFO:teuthology.orchestra.run.smithi062.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2022-09-03T05:48:41.474 INFO:teuthology.orchestra.run.smithi062.stdout: "index_type": 0 2022-09-03T05:48:41.474 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:48:41.474 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:48:41.475 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:48:41.475 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "", 2022-09-03T05:48:41.475 INFO:teuthology.orchestra.run.smithi062.stdout: "notif_pool": "default.rgw.log:notif" 2022-09-03T05:48:41.475 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:48:41.477 DEBUG:tasks.util.rgw: json result: {'id': '74ed39f8-feb8-49f4-94c6-d843d64ef963', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2022-09-03T05:48:41.477 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['user', 'list'] 2022-09-03T05:48:41.477 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'user', 'list'] 2022-09-03T05:48:41.478 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 user list 2022-09-03T05:48:41.557 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:41.558 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:41.563 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:41.562+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.144 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.140+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.144 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.140+0000 7feab6b9e600 20 realm 2022-09-03T05:48:43.145 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.140+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.145 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.141+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.145 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.141+0000 7feab6b9e600 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2022-09-03T05:48:43.146 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.141+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.146 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.141+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.146 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.141+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.147 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.141+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.147 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.141+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.148 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.142+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.148 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.142+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.149 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.142+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.149 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.142+0000 7feab6b9e600 10 Creating default zonegroup 2022-09-03T05:48:43.150 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.142+0000 7feab6b9e600 10 couldn't find old data placement pools config, setting up new ones for the zone 2022-09-03T05:48:43.154 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.153+0000 7feab6b9e600 10 WARNING: store->list_zones() returned r=-2 2022-09-03T05:48:43.155 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.153+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.155 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.153+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.157 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.155+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.158 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.155+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.158 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.155+0000 7feab6b9e600 10 could not read realm id: (2) No such file or directory 2022-09-03T05:48:43.159 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.155+0000 7feab6b9e600 10 WARNING: failed to set zone as default, r=-22 2022-09-03T05:48:43.159 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.155+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.159 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.156+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.161 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.158+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.161 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.158+0000 7feab6b9e600 20 rados_obj.operate() r=0 bl.length=873 2022-09-03T05:48:43.161 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.158+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.164 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.161+0000 7feab6b9e600 20 rados_obj.operate() r=0 bl.length=398 2022-09-03T05:48:43.165 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.161+0000 7feab6b9e600 10 Using default name default 2022-09-03T05:48:43.165 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.161+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.165 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:43.168 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.168 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 20 rados_obj.operate() r=0 bl.length=873 2022-09-03T05:48:43.168 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 20 zone default found 2022-09-03T05:48:43.168 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 4 Realm: () 2022-09-03T05:48:43.169 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 4 ZoneGroup: default (5578a644-ac93-4c31-949c-91f0cb2e0cc7) 2022-09-03T05:48:43.169 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 4 Zone: default (2dec4359-0111-406c-97cb-19a37639566b) 2022-09-03T05:48:43.170 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 10 cannot find current period zonegroup using local zonegroup configuration 2022-09-03T05:48:43.170 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 20 zonegroup default 2022-09-03T05:48:43.170 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.162+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:43.170 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.163+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:43.171 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:43.163+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:45.147 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:45.145+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:45.147 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:45.145+0000 7feab6b9e600 20 rados->read ofs=0 len=0 2022-09-03T05:48:45.148 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:45.146+0000 7feab6b9e600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:45.149 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:45.146+0000 7feab6b9e600 20 started sync module instance, tier type = 2022-09-03T05:48:45.149 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:45.146+0000 7feab6b9e600 20 started zone id=2dec4359-0111-406c-97cb-19a37639566b (name=default) with tier type = 2022-09-03T05:48:47.180 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.178+0000 7feab6b9e600 20 add_watcher() i=0 2022-09-03T05:48:47.181 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.178+0000 7feab6b9e600 20 add_watcher() i=1 2022-09-03T05:48:47.181 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.178+0000 7feab6b9e600 20 add_watcher() i=2 2022-09-03T05:48:47.181 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.178+0000 7feab6b9e600 20 add_watcher() i=3 2022-09-03T05:48:47.181 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.178+0000 7feab6b9e600 20 add_watcher() i=4 2022-09-03T05:48:47.182 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.178+0000 7feab6b9e600 20 add_watcher() i=5 2022-09-03T05:48:47.182 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.178+0000 7feab6b9e600 20 add_watcher() i=6 2022-09-03T05:48:47.184 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.179+0000 7feab6b9e600 20 add_watcher() i=7 2022-09-03T05:48:47.184 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.179+0000 7feab6b9e600 2 all 8 watchers are set, enabling cache 2022-09-03T05:48:47.184 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 5 tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):410: oid=data_loggenerations_metadata not found 2022-09-03T05:48:47.185 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.0 2022-09-03T05:48:47.185 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:47.185 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.0 does not exist 2022-09-03T05:48:47.185 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.1 2022-09-03T05:48:47.186 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:47.186 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.1 does not exist 2022-09-03T05:48:47.186 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.2 2022-09-03T05:48:47.186 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:47.187 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.2 does not exist 2022-09-03T05:48:47.187 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.3 2022-09-03T05:48:47.187 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:47.187 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.3 does not exist 2022-09-03T05:48:47.187 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.180+0000 7feab6b9e600 20 static int rgw::cls::fifo::FIFO::create(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, std::optional >, bool, uint64_t, uint64_t):1203 entering 2022-09-03T05:48:47.188 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.181+0000 7feab6b9e600 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2022-09-03T05:48:47.189 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.184+0000 7fea9f31f700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-09-03T05:48:47.189 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.184+0000 7feab6b9e600 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2022-09-03T05:48:47.189 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:47.184+0000 7feab6b9e600 20 check_secure_mon_conn(): mode 1 is insecure 2022-09-03T05:48:50.154 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.153+0000 7feab6b9e600 10 rgw_init_ioctx warning: failed to set recovery_priority on default.rgw.meta 2022-09-03T05:48:50.155 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.153+0000 7feab6b9e600 5 note: GC not initialized 2022-09-03T05:48:50.156 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.154+0000 7fea9ab16700 20 reqs_thread_entry: start 2022-09-03T05:48:50.156 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.154+0000 7feab6b9e600 20 init_complete bucket index max shards: 11 2022-09-03T05:48:50.157 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.154+0000 7feab6b9e600 10 rgw notify: Started notification manager with: 1 workers 2022-09-03T05:48:50.157 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.154+0000 7fea98b12700 20 rgw notify: INFO: next queues processing will happen at: Sat Sep 3 05:49:20 2022 2022-09-03T05:48:50.157 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T05:48:50.158 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.154+0000 7fea982f0700 20 reqs_thread_entry: start 2022-09-03T05:48:50.178 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T05:48:50.179 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.177+0000 7feab6b9e600 20 remove_watcher() i=0 2022-09-03T05:48:50.180 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.177+0000 7feab6b9e600 2 removed watcher, disabling cache 2022-09-03T05:48:50.180 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.178+0000 7feab6b9e600 20 remove_watcher() i=1 2022-09-03T05:48:50.181 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.179+0000 7feab6b9e600 20 remove_watcher() i=2 2022-09-03T05:48:50.183 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.181+0000 7feab6b9e600 20 remove_watcher() i=3 2022-09-03T05:48:50.185 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.184+0000 7feab6b9e600 20 remove_watcher() i=4 2022-09-03T05:48:50.186 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.185+0000 7feab6b9e600 20 remove_watcher() i=5 2022-09-03T05:48:50.189 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.188+0000 7feab6b9e600 20 remove_watcher() i=6 2022-09-03T05:48:50.190 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.189+0000 7feab6b9e600 20 remove_watcher() i=7 2022-09-03T05:48:50.191 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.189+0000 7fea9f31f700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-09-03T05:48:50.195 DEBUG:tasks.util.rgw: json result: [] 2022-09-03T05:48:50.196 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2022-09-03T05:48:50.196 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2022-09-03T05:48:50.197 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2022-09-03T05:48:50.251 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:50.251 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:50.256 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.255+0000 7fa3b9e33600 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2022-09-03T05:48:50.257 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.255+0000 7fa3b9e33600 20 check_secure_mon_conn(): mode 1 is insecure 2022-09-03T05:48:50.260 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.258+0000 7fa39edad700 20 reqs_thread_entry: start 2022-09-03T05:48:50.260 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.258+0000 7fa3b9e33600 20 try_to_resolve_local_entities(): before: realm_id= realm_name= zonegroup_id= zonegroup_name= zone_id= zone_name=default 2022-09-03T05:48:50.260 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.258+0000 7fa3b9e33600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.261 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.259+0000 7fa3b9e33600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:50.261 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.259+0000 7fa3b9e33600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.262 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.259+0000 7fa3b9e33600 20 rados_obj.operate() r=0 bl.length=873 2022-09-03T05:48:50.269 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 RGWRados::pool_iterate: got zone_info.2dec4359-0111-406c-97cb-19a37639566b 2022-09-03T05:48:50.270 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 RGWRados::pool_iterate: got zone_names.default 2022-09-03T05:48:50.270 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 RGWRados::pool_iterate: got zonegroups_names.default 2022-09-03T05:48:50.271 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 RGWRados::pool_iterate: got zonegroup_info.5578a644-ac93-4c31-949c-91f0cb2e0cc7 2022-09-03T05:48:50.271 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.272 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 rados_obj.operate() r=0 bl.length=873 2022-09-03T05:48:50.272 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.273 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:50.273 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.273 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:50.274 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.267+0000 7fa3b9e33600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.274 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.268+0000 7fa3b9e33600 20 rados_obj.operate() r=0 bl.length=398 2022-09-03T05:48:50.277 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:48:50.277 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2dec4359-0111-406c-97cb-19a37639566b", 2022-09-03T05:48:50.277 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default", 2022-09-03T05:48:50.278 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "default.rgw.meta:root", 2022-09-03T05:48:50.278 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "default.rgw.control", 2022-09-03T05:48:50.278 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "default.rgw.log:gc", 2022-09-03T05:48:50.279 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "default.rgw.log:lc", 2022-09-03T05:48:50.279 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "default.rgw.log", 2022-09-03T05:48:50.279 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "default.rgw.log:intent", 2022-09-03T05:48:50.280 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "default.rgw.log:usage", 2022-09-03T05:48:50.280 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "default.rgw.meta:roles", 2022-09-03T05:48:50.280 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "default.rgw.log:reshard", 2022-09-03T05:48:50.281 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2022-09-03T05:48:50.281 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "default.rgw.meta:users.email", 2022-09-03T05:48:50.281 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2022-09-03T05:48:50.282 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2022-09-03T05:48:50.282 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "default.rgw.otp", 2022-09-03T05:48:50.282 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T05:48:50.283 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "", 2022-09-03T05:48:50.283 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "" 2022-09-03T05:48:50.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:48:50.284 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T05:48:50.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:48:50.284 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T05:48:50.284 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:48:50.284 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "default.rgw.buckets.index", 2022-09-03T05:48:50.285 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T05:48:50.285 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T05:48:50.286 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "default.rgw.buckets.data", 2022-09-03T05:48:50.286 INFO:teuthology.orchestra.run.smithi087.stdout: "compression_type": "random" 2022-09-03T05:48:50.287 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:48:50.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:48:50.287 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2022-09-03T05:48:50.288 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T05:48:50.288 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:48:50.288 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:48:50.289 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:48:50.289 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "", 2022-09-03T05:48:50.290 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "default.rgw.log:notif" 2022-09-03T05:48:50.290 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:48:50.290 DEBUG:tasks.util.rgw: json result: {'id': '2dec4359-0111-406c-97cb-19a37639566b', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2022-09-03T05:48:50.291 INFO:tasks.util.rgw:rgwadmin: c2.client.1 : ['user', 'list'] 2022-09-03T05:48:50.291 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c2', 'user', 'list'] 2022-09-03T05:48:50.292 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c2 user list 2022-09-03T05:48:50.335 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:50.336 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:50.341 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.339+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.343 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.340+0000 7f5cbd097600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:50.344 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.340+0000 7f5cbd097600 20 realm 2022-09-03T05:48:50.344 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.340+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.344 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.341+0000 7f5cbd097600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:50.345 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.341+0000 7f5cbd097600 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2022-09-03T05:48:50.345 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.341+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.345 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.341+0000 7f5cbd097600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:50.345 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.341+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.346 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.341+0000 7f5cbd097600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:50.346 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.341+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.346 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.342+0000 7f5cbd097600 20 rados_obj.operate() r=0 bl.length=889 2022-09-03T05:48:50.347 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.342+0000 7f5cbd097600 20 searching for the correct realm 2022-09-03T05:48:50.351 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.348+0000 7f5cbd097600 20 RGWRados::pool_iterate: got zone_info.2dec4359-0111-406c-97cb-19a37639566b 2022-09-03T05:48:50.351 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.348+0000 7f5cbd097600 20 RGWRados::pool_iterate: got zone_names.default 2022-09-03T05:48:50.352 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.348+0000 7f5cbd097600 20 RGWRados::pool_iterate: got zonegroups_names.default 2022-09-03T05:48:50.352 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.348+0000 7f5cbd097600 20 RGWRados::pool_iterate: got zonegroup_info.5578a644-ac93-4c31-949c-91f0cb2e0cc7 2022-09-03T05:48:50.352 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.348+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.352 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.348+0000 7f5cbd097600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:50.353 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.348+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.353 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:50.353 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.353 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 rados_obj.operate() r=0 bl.length=398 2022-09-03T05:48:50.354 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 zone default found 2022-09-03T05:48:50.354 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 4 Realm: () 2022-09-03T05:48:50.354 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 4 ZoneGroup: default (5578a644-ac93-4c31-949c-91f0cb2e0cc7) 2022-09-03T05:48:50.354 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 4 Zone: default (2dec4359-0111-406c-97cb-19a37639566b) 2022-09-03T05:48:50.355 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 10 cannot find current period zonegroup using local zonegroup configuration 2022-09-03T05:48:50.355 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 zonegroup default 2022-09-03T05:48:50.355 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.355 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:50.356 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.356 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:50.356 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.356 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:50.356 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 started sync module instance, tier type = 2022-09-03T05:48:50.357 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.349+0000 7f5cbd097600 20 started zone id=2dec4359-0111-406c-97cb-19a37639566b (name=default) with tier type = 2022-09-03T05:48:50.372 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.371+0000 7f5cbd097600 20 add_watcher() i=0 2022-09-03T05:48:50.373 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.371+0000 7f5cbd097600 20 add_watcher() i=1 2022-09-03T05:48:50.373 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.371+0000 7f5cbd097600 20 add_watcher() i=2 2022-09-03T05:48:50.373 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.371+0000 7f5cbd097600 20 add_watcher() i=3 2022-09-03T05:48:50.374 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.371+0000 7f5cbd097600 20 add_watcher() i=4 2022-09-03T05:48:50.374 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.371+0000 7f5cbd097600 20 add_watcher() i=5 2022-09-03T05:48:50.374 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.371+0000 7f5cbd097600 20 add_watcher() i=6 2022-09-03T05:48:50.375 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.372+0000 7f5cbd097600 20 add_watcher() i=7 2022-09-03T05:48:50.375 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.372+0000 7f5cbd097600 2 all 8 watchers are set, enabling cache 2022-09-03T05:48:50.375 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.373+0000 7f5ca5818700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-09-03T05:48:50.376 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.373+0000 7f5cbd097600 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2022-09-03T05:48:50.376 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.373+0000 7f5cbd097600 20 check_secure_mon_conn(): mode 1 is insecure 2022-09-03T05:48:50.378 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.376+0000 7f5cbd097600 5 note: GC not initialized 2022-09-03T05:48:50.379 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.377+0000 7f5ca100f700 20 reqs_thread_entry: start 2022-09-03T05:48:50.379 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.377+0000 7f5cbd097600 20 init_complete bucket index max shards: 11 2022-09-03T05:48:50.379 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.377+0000 7f5cbd097600 10 rgw notify: Started notification manager with: 1 workers 2022-09-03T05:48:50.380 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.377+0000 7f5c9f00b700 20 rgw notify: INFO: next queues processing will happen at: Sat Sep 3 05:49:20 2022 2022-09-03T05:48:50.380 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T05:48:50.381 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.377+0000 7f5c9e7e9700 20 reqs_thread_entry: start 2022-09-03T05:48:50.386 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T05:48:50.386 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.384+0000 7f5cbd097600 20 remove_watcher() i=0 2022-09-03T05:48:50.387 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.384+0000 7f5cbd097600 2 removed watcher, disabling cache 2022-09-03T05:48:50.387 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.385+0000 7f5cbd097600 20 remove_watcher() i=1 2022-09-03T05:48:50.387 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.386+0000 7f5cbd097600 20 remove_watcher() i=2 2022-09-03T05:48:50.388 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.387+0000 7f5cbd097600 20 remove_watcher() i=3 2022-09-03T05:48:50.389 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.388+0000 7f5cbd097600 20 remove_watcher() i=4 2022-09-03T05:48:50.390 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.388+0000 7f5cbd097600 20 remove_watcher() i=5 2022-09-03T05:48:50.391 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.389+0000 7f5cbd097600 20 remove_watcher() i=6 2022-09-03T05:48:50.392 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.390+0000 7f5cbd097600 20 remove_watcher() i=7 2022-09-03T05:48:50.392 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.391+0000 7f5ca5818700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-09-03T05:48:50.396 DEBUG:tasks.util.rgw: json result: [] 2022-09-03T05:48:50.397 INFO:tasks.util.rgw:rgwadmin: c2.client.1 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2022-09-03T05:48:50.397 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c2', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2022-09-03T05:48:50.398 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c2 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2022-09-03T05:48:50.486 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:50.486 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:50.491 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.490+0000 7f6f19c2f600 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2022-09-03T05:48:50.492 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.490+0000 7f6f19c2f600 20 check_secure_mon_conn(): mode 1 is insecure 2022-09-03T05:48:50.495 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.493+0000 7f6efeba9700 20 reqs_thread_entry: start 2022-09-03T05:48:50.495 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.493+0000 7f6f19c2f600 20 try_to_resolve_local_entities(): before: realm_id= realm_name= zonegroup_id= zonegroup_name= zone_id= zone_name=default 2022-09-03T05:48:50.496 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.493+0000 7f6f19c2f600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.496 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.494+0000 7f6f19c2f600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:50.497 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.494+0000 7f6f19c2f600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.497 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.495+0000 7f6f19c2f600 20 rados_obj.operate() r=0 bl.length=889 2022-09-03T05:48:50.505 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.501+0000 7f6f19c2f600 20 RGWRados::pool_iterate: got zone_info.2dec4359-0111-406c-97cb-19a37639566b 2022-09-03T05:48:50.506 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.501+0000 7f6f19c2f600 20 RGWRados::pool_iterate: got zone_names.default 2022-09-03T05:48:50.506 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.501+0000 7f6f19c2f600 20 RGWRados::pool_iterate: got zonegroups_names.default 2022-09-03T05:48:50.507 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.501+0000 7f6f19c2f600 20 RGWRados::pool_iterate: got zonegroup_info.5578a644-ac93-4c31-949c-91f0cb2e0cc7 2022-09-03T05:48:50.507 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.501+0000 7f6f19c2f600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.507 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.502+0000 7f6f19c2f600 20 rados_obj.operate() r=0 bl.length=889 2022-09-03T05:48:50.508 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.502+0000 7f6f19c2f600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.508 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.502+0000 7f6f19c2f600 20 rados_obj.operate() r=-2 bl.length=0 2022-09-03T05:48:50.509 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.502+0000 7f6f19c2f600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.509 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.502+0000 7f6f19c2f600 20 rados_obj.operate() r=0 bl.length=46 2022-09-03T05:48:50.509 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.502+0000 7f6f19c2f600 20 rados->read ofs=0 len=0 2022-09-03T05:48:50.510 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:48:50.502+0000 7f6f19c2f600 20 rados_obj.operate() r=0 bl.length=398 2022-09-03T05:48:50.511 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:48:50.511 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2dec4359-0111-406c-97cb-19a37639566b", 2022-09-03T05:48:50.512 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default", 2022-09-03T05:48:50.512 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "default.rgw.meta:root", 2022-09-03T05:48:50.512 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "default.rgw.control", 2022-09-03T05:48:50.512 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "default.rgw.log:gc", 2022-09-03T05:48:50.513 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "default.rgw.log:lc", 2022-09-03T05:48:50.513 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "default.rgw.log", 2022-09-03T05:48:50.513 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "default.rgw.log:intent", 2022-09-03T05:48:50.513 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "default.rgw.log:usage", 2022-09-03T05:48:50.513 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "default.rgw.meta:roles", 2022-09-03T05:48:50.514 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "default.rgw.log:reshard", 2022-09-03T05:48:50.514 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2022-09-03T05:48:50.514 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "default.rgw.meta:users.email", 2022-09-03T05:48:50.514 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2022-09-03T05:48:50.515 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2022-09-03T05:48:50.515 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "default.rgw.otp", 2022-09-03T05:48:50.515 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T05:48:50.515 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "", 2022-09-03T05:48:50.516 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "" 2022-09-03T05:48:50.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:48:50.516 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T05:48:50.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:48:50.517 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T05:48:50.517 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:48:50.517 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "default.rgw.buckets.index", 2022-09-03T05:48:50.517 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T05:48:50.518 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T05:48:50.518 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "default.rgw.buckets.data", 2022-09-03T05:48:50.518 INFO:teuthology.orchestra.run.smithi087.stdout: "compression_type": "random" 2022-09-03T05:48:50.519 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:48:50.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:48:50.520 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2022-09-03T05:48:50.520 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T05:48:50.520 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:48:50.521 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:48:50.521 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:48:50.521 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "", 2022-09-03T05:48:50.522 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "default.rgw.log:notif" 2022-09-03T05:48:50.522 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:48:50.523 DEBUG:tasks.util.rgw: json result: {'id': '2dec4359-0111-406c-97cb-19a37639566b', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2022-09-03T05:48:50.523 INFO:tasks.rgw:Starting rgw... 2022-09-03T05:48:50.524 INFO:tasks.rgw:rgw c1.client.0 config is {'port': 8000} 2022-09-03T05:48:50.524 INFO:tasks.rgw:Using beast as radosgw frontend 2022-09-03T05:48:50.525 INFO:tasks.rgw.c1.client.0:Restarting daemon 2022-09-03T05:48:50.526 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.c1.client.0.stdout 2>&1 2022-09-03T05:48:50.529 INFO:tasks.rgw.c1.client.0:Started 2022-09-03T05:48:50.530 INFO:tasks.rgw:rgw c1.client.1 config is {'port': 8001} 2022-09-03T05:48:50.530 INFO:tasks.rgw:Using beast as radosgw frontend 2022-09-03T05:48:50.531 INFO:tasks.rgw.c1.client.1:Restarting daemon 2022-09-03T05:48:50.531 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground | sudo tee /var/log/ceph/rgw.c1.client.1.stdout 2>&1 2022-09-03T05:48:50.533 INFO:tasks.rgw.c1.client.1:Started 2022-09-03T05:48:50.534 INFO:tasks.rgw:rgw c2.client.0 config is {'port': 8000} 2022-09-03T05:48:50.534 INFO:tasks.rgw:Using beast as radosgw frontend 2022-09-03T05:48:50.535 INFO:tasks.rgw.c2.client.0:Restarting daemon 2022-09-03T05:48:50.535 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2022-09-03T05:48:50.537 INFO:tasks.rgw.c2.client.0:Started 2022-09-03T05:48:50.538 INFO:tasks.rgw:rgw c2.client.1 config is {'port': 8001} 2022-09-03T05:48:50.538 INFO:tasks.rgw:Using beast as radosgw frontend 2022-09-03T05:48:50.539 INFO:tasks.rgw.c2.client.1:Restarting daemon 2022-09-03T05:48:50.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground | sudo tee /var/log/ceph/rgw.c2.client.1.stdout 2>&1 2022-09-03T05:48:50.541 INFO:tasks.rgw.c2.client.1:Started 2022-09-03T05:48:50.542 INFO:tasks.rgw:Polling c1.client.0 until it starts accepting connections on http://smithi062.front.sepia.ceph.com:8000/ 2022-09-03T05:48:50.542 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8000/ 2022-09-03T05:48:50.665 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:48:50.665 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:48:50.668 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:48:50.669 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T05:48:51.670 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8000/ 2022-09-03T05:48:51.695 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:48:51.695 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:48:51.696 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 208k 0 --:--:-- --:--:-- --:--:-- 208k 2022-09-03T05:48:51.697 INFO:teuthology.orchestra.run.smithi062.stdout:anonymous 2022-09-03T05:48:51.697 INFO:tasks.rgw:Polling c1.client.1 until it starts accepting connections on http://smithi062.front.sepia.ceph.com:8001/ 2022-09-03T05:48:51.698 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001/ 2022-09-03T05:48:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:48:51.762 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:48:51.762 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:48:51.762 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T05:48:52.763 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001/ 2022-09-03T05:48:52.790 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:48:52.791 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:48:52.792 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:48:52.793 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T05:48:54.796 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001/ 2022-09-03T05:48:54.827 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:48:54.828 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:48:54.828 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:48:54.829 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T05:48:58.830 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001/ 2022-09-03T05:48:58.862 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:48:58.863 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:48:58.863 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2022-09-03T05:48:58.864 INFO:teuthology.orchestra.run.smithi062.stdout:anonymous 2022-09-03T05:48:58.865 INFO:tasks.rgw:Polling c2.client.0 until it starts accepting connections on http://smithi087.front.sepia.ceph.com:8000/ 2022-09-03T05:48:58.865 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000/ 2022-09-03T05:48:58.900 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:48:58.900 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:48:58.902 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2022-09-03T05:48:58.903 INFO:teuthology.orchestra.run.smithi087.stdout:anonymous 2022-09-03T05:48:58.903 INFO:tasks.rgw:Polling c2.client.1 until it starts accepting connections on http://smithi087.front.sepia.ceph.com:8001/ 2022-09-03T05:48:58.904 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8001/ 2022-09-03T05:48:58.969 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:48:58.970 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:48:58.970 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2022-09-03T05:48:58.973 INFO:teuthology.orchestra.run.smithi087.stdout:anonymous 2022-09-03T05:48:58.973 INFO:teuthology.run_tasks:Running task rgw-multisite... 2022-09-03T05:48:59.201 INFO:tasks.rgw_multisite:creating realm.. 2022-09-03T05:48:59.202 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['realm', 'create', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:48:59.202 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'realm', 'create', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:48:59.203 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 realm create --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:48:59.257 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:59.258 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:59.297 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:48:59.297 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:48:59.298 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-realm", 2022-09-03T05:48:59.298 INFO:teuthology.orchestra.run.smithi062.stdout: "current_period": "226f712d-7b05-4c77-9ead-974d95b142d0", 2022-09-03T05:48:59.299 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 1 2022-09-03T05:48:59.299 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:48:59.302 INFO:tasks.rgw_multisite:creating master zonegroup.. 2022-09-03T05:48:59.302 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'set', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:48:59.303 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'set', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:48:59.304 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup set --rgw-zonegroup test-zonegroup --rgw-realm test-realm --default --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:48:59.355 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:59.356 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:59.367 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:48:59.365+0000 7f9f8c4fb600 -1 old format 2022-09-03T05:48:59.375 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:48:59.375 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "test-zonegroup", 2022-09-03T05:48:59.375 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zonegroup", 2022-09-03T05:48:59.376 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "", 2022-09-03T05:48:59.376 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": true, 2022-09-03T05:48:59.376 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T05:48:59.376 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T05:48:59.377 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:48:59.377 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T05:48:59.377 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T05:48:59.377 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "", 2022-09-03T05:48:59.377 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [], 2022-09-03T05:48:59.378 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [], 2022-09-03T05:48:59.378 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "", 2022-09-03T05:48:59.378 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:48:59.379 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T05:48:59.379 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T05:48:59.379 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:48:59.379 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T05:48:59.380 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T05:48:59.380 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:48:59.380 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:48:59.382 INFO:tasks.rgw_multisite:creating master zone.. 2022-09-03T05:48:59.382 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--master', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--endpoints', 'http://smithi062.front.sepia.ceph.com:8000', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:48:59.382 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--master', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--endpoints', 'http://smithi062.front.sepia.ceph.com:8000', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:48:59.383 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --default --master --access-key ND2IR1XIGKP3UN6B --secret quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq --endpoints http://smithi062.front.sepia.ceph.com:8000 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:48:59.480 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:48:59.480 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:48:59.508 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:48:59.508 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:48:59.509 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T05:48:59.509 INFO:teuthology.orchestra.run.smithi062.stdout: "domain_root": "test-zone1.rgw.meta:root", 2022-09-03T05:48:59.509 INFO:teuthology.orchestra.run.smithi062.stdout: "control_pool": "test-zone1.rgw.control", 2022-09-03T05:48:59.509 INFO:teuthology.orchestra.run.smithi062.stdout: "gc_pool": "test-zone1.rgw.log:gc", 2022-09-03T05:48:59.510 INFO:teuthology.orchestra.run.smithi062.stdout: "lc_pool": "test-zone1.rgw.log:lc", 2022-09-03T05:48:59.510 INFO:teuthology.orchestra.run.smithi062.stdout: "log_pool": "test-zone1.rgw.log", 2022-09-03T05:48:59.510 INFO:teuthology.orchestra.run.smithi062.stdout: "intent_log_pool": "test-zone1.rgw.log:intent", 2022-09-03T05:48:59.510 INFO:teuthology.orchestra.run.smithi062.stdout: "usage_log_pool": "test-zone1.rgw.log:usage", 2022-09-03T05:48:59.511 INFO:teuthology.orchestra.run.smithi062.stdout: "roles_pool": "test-zone1.rgw.meta:roles", 2022-09-03T05:48:59.511 INFO:teuthology.orchestra.run.smithi062.stdout: "reshard_pool": "test-zone1.rgw.log:reshard", 2022-09-03T05:48:59.511 INFO:teuthology.orchestra.run.smithi062.stdout: "user_keys_pool": "test-zone1.rgw.meta:users.keys", 2022-09-03T05:48:59.512 INFO:teuthology.orchestra.run.smithi062.stdout: "user_email_pool": "test-zone1.rgw.meta:users.email", 2022-09-03T05:48:59.512 INFO:teuthology.orchestra.run.smithi062.stdout: "user_swift_pool": "test-zone1.rgw.meta:users.swift", 2022-09-03T05:48:59.512 INFO:teuthology.orchestra.run.smithi062.stdout: "user_uid_pool": "test-zone1.rgw.meta:users.uid", 2022-09-03T05:48:59.512 INFO:teuthology.orchestra.run.smithi062.stdout: "otp_pool": "test-zone1.rgw.otp", 2022-09-03T05:48:59.512 INFO:teuthology.orchestra.run.smithi062.stdout: "system_key": { 2022-09-03T05:48:59.513 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T05:48:59.513 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T05:48:59.514 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:48:59.514 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_pools": [ 2022-09-03T05:48:59.515 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:48:59.515 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "default-placement", 2022-09-03T05:48:59.515 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:48:59.516 INFO:teuthology.orchestra.run.smithi062.stdout: "index_pool": "test-zone1.rgw.buckets.index", 2022-09-03T05:48:59.516 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": { 2022-09-03T05:48:59.517 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD": { 2022-09-03T05:48:59.517 INFO:teuthology.orchestra.run.smithi062.stdout: "data_pool": "test-zone1.rgw.buckets.data" 2022-09-03T05:48:59.517 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:48:59.518 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:48:59.518 INFO:teuthology.orchestra.run.smithi062.stdout: "data_extra_pool": "test-zone1.rgw.buckets.non-ec", 2022-09-03T05:48:59.519 INFO:teuthology.orchestra.run.smithi062.stdout: "index_type": 0 2022-09-03T05:48:59.519 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:48:59.520 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:48:59.520 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:48:59.521 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:48:59.521 INFO:teuthology.orchestra.run.smithi062.stdout: "notif_pool": "test-zone1.rgw.log:notif" 2022-09-03T05:48:59.521 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:48:59.523 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool create test-zone1.rgw.buckets.data 64 64 --cluster c1 2022-09-03T05:49:00.155 INFO:teuthology.orchestra.run.smithi062.stderr:pool 'test-zone1.rgw.buckets.data' created 2022-09-03T05:49:00.174 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool application enable test-zone1.rgw.buckets.data rgw --cluster c1 2022-09-03T05:49:01.183 INFO:teuthology.orchestra.run.smithi062.stderr:enabled application 'rgw' on pool 'test-zone1.rgw.buckets.data' 2022-09-03T05:49:01.200 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'placement', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:01.200 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'placement', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:01.201 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone placement --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:01.258 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:01.258 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:01.282 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:49:01.283 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:01.284 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T05:49:01.284 INFO:teuthology.orchestra.run.smithi062.stdout: "domain_root": "test-zone1.rgw.meta:root", 2022-09-03T05:49:01.284 INFO:teuthology.orchestra.run.smithi062.stdout: "control_pool": "test-zone1.rgw.control", 2022-09-03T05:49:01.285 INFO:teuthology.orchestra.run.smithi062.stdout: "gc_pool": "test-zone1.rgw.log:gc", 2022-09-03T05:49:01.285 INFO:teuthology.orchestra.run.smithi062.stdout: "lc_pool": "test-zone1.rgw.log:lc", 2022-09-03T05:49:01.285 INFO:teuthology.orchestra.run.smithi062.stdout: "log_pool": "test-zone1.rgw.log", 2022-09-03T05:49:01.285 INFO:teuthology.orchestra.run.smithi062.stdout: "intent_log_pool": "test-zone1.rgw.log:intent", 2022-09-03T05:49:01.286 INFO:teuthology.orchestra.run.smithi062.stdout: "usage_log_pool": "test-zone1.rgw.log:usage", 2022-09-03T05:49:01.286 INFO:teuthology.orchestra.run.smithi062.stdout: "roles_pool": "test-zone1.rgw.meta:roles", 2022-09-03T05:49:01.286 INFO:teuthology.orchestra.run.smithi062.stdout: "reshard_pool": "test-zone1.rgw.log:reshard", 2022-09-03T05:49:01.286 INFO:teuthology.orchestra.run.smithi062.stdout: "user_keys_pool": "test-zone1.rgw.meta:users.keys", 2022-09-03T05:49:01.286 INFO:teuthology.orchestra.run.smithi062.stdout: "user_email_pool": "test-zone1.rgw.meta:users.email", 2022-09-03T05:49:01.287 INFO:teuthology.orchestra.run.smithi062.stdout: "user_swift_pool": "test-zone1.rgw.meta:users.swift", 2022-09-03T05:49:01.287 INFO:teuthology.orchestra.run.smithi062.stdout: "user_uid_pool": "test-zone1.rgw.meta:users.uid", 2022-09-03T05:49:01.287 INFO:teuthology.orchestra.run.smithi062.stdout: "otp_pool": "test-zone1.rgw.otp", 2022-09-03T05:49:01.287 INFO:teuthology.orchestra.run.smithi062.stdout: "system_key": { 2022-09-03T05:49:01.288 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T05:49:01.288 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T05:49:01.288 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:01.289 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_pools": [ 2022-09-03T05:49:01.289 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:01.289 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "default-placement", 2022-09-03T05:49:01.289 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:49:01.290 INFO:teuthology.orchestra.run.smithi062.stdout: "index_pool": "test-zone1.rgw.buckets.index", 2022-09-03T05:49:01.290 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": { 2022-09-03T05:49:01.290 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD": { 2022-09-03T05:49:01.290 INFO:teuthology.orchestra.run.smithi062.stdout: "data_pool": "test-zone1.rgw.buckets.data", 2022-09-03T05:49:01.291 INFO:teuthology.orchestra.run.smithi062.stdout: "compression_type": "random" 2022-09-03T05:49:01.291 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:01.291 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:01.291 INFO:teuthology.orchestra.run.smithi062.stdout: "data_extra_pool": "test-zone1.rgw.buckets.non-ec", 2022-09-03T05:49:01.291 INFO:teuthology.orchestra.run.smithi062.stdout: "index_type": 0 2022-09-03T05:49:01.292 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:01.292 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:01.292 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:01.292 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "", 2022-09-03T05:49:01.293 INFO:teuthology.orchestra.run.smithi062.stdout: "notif_pool": "test-zone1.rgw.log:notif" 2022-09-03T05:49:01.293 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:49:01.294 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:01.295 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:01.295 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:01.345 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:01.346 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:01.355 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:49:01.352+0000 7f08c06cf600 0 period (226f712d-7b05-4c77-9ead-974d95b142d0 does not have zone fca49d30-abba-4c2b-8462-66395ac063ba configured 2022-09-03T05:49:08.248 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:49:08.248 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:49:08.249 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 1, 2022-09-03T05:49:08.249 INFO:teuthology.orchestra.run.smithi062.stdout: "predecessor_uuid": "226f712d-7b05-4c77-9ead-974d95b142d0", 2022-09-03T05:49:08.249 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": [], 2022-09-03T05:49:08.249 INFO:teuthology.orchestra.run.smithi062.stdout: "period_map": { 2022-09-03T05:49:08.250 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:49:08.250 INFO:teuthology.orchestra.run.smithi062.stdout: "zonegroups": [ 2022-09-03T05:49:08.250 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:08.250 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "test-zonegroup", 2022-09-03T05:49:08.251 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zonegroup", 2022-09-03T05:49:08.251 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "", 2022-09-03T05:49:08.251 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": true, 2022-09-03T05:49:08.251 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T05:49:08.251 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T05:49:08.252 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:08.252 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T05:49:08.252 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T05:49:08.252 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:08.253 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [ 2022-09-03T05:49:08.253 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:08.253 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:08.253 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T05:49:08.253 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T05:49:08.254 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T05:49:08.255 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:08.255 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T05:49:08.255 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": false, 2022-09-03T05:49:08.255 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:49:08.256 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T05:49:08.256 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T05:49:08.256 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T05:49:08.256 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T05:49:08.256 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T05:49:08.257 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T05:49:08.257 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T05:49:08.257 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:08.257 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:08.258 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:08.258 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [ 2022-09-03T05:49:08.258 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:08.258 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default-placement", 2022-09-03T05:49:08.258 INFO:teuthology.orchestra.run.smithi062.stdout: "tags": [], 2022-09-03T05:49:08.259 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": [ 2022-09-03T05:49:08.259 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD" 2022-09-03T05:49:08.259 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:08.259 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:08.260 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:08.260 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "default-placement", 2022-09-03T05:49:08.260 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:08.260 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T05:49:08.260 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T05:49:08.261 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:08.261 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T05:49:08.261 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T05:49:08.262 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:08.262 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:08.262 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:08.262 INFO:teuthology.orchestra.run.smithi062.stdout: "short_zone_ids": [ 2022-09-03T05:49:08.263 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:08.263 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:08.263 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 449119331 2022-09-03T05:49:08.263 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:08.263 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:08.264 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:08.264 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T05:49:08.264 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:08.264 INFO:teuthology.orchestra.run.smithi062.stdout: "period_config": { 2022-09-03T05:49:08.265 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_quota": { 2022-09-03T05:49:08.265 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T05:49:08.265 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T05:49:08.265 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T05:49:08.265 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T05:49:08.266 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T05:49:08.266 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:08.266 INFO:teuthology.orchestra.run.smithi062.stdout: "user_quota": { 2022-09-03T05:49:08.266 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T05:49:08.267 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T05:49:08.267 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T05:49:08.267 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T05:49:08.267 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T05:49:08.267 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:08.268 INFO:teuthology.orchestra.run.smithi062.stdout: "user_ratelimit": { 2022-09-03T05:49:08.268 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T05:49:08.268 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T05:49:08.268 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T05:49:08.269 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T05:49:08.269 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T05:49:08.269 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:08.269 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_ratelimit": { 2022-09-03T05:49:08.269 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T05:49:08.270 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T05:49:08.270 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T05:49:08.270 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T05:49:08.271 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T05:49:08.271 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:08.271 INFO:teuthology.orchestra.run.smithi062.stdout: "anonymous_ratelimit": { 2022-09-03T05:49:08.272 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T05:49:08.272 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T05:49:08.272 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T05:49:08.272 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T05:49:08.273 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T05:49:08.273 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:08.273 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:08.273 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:08.274 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_name": "test-realm", 2022-09-03T05:49:08.274 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:49:08.274 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:49:08.276 DEBUG:tasks.rgw.c1.client.0:waiting for process to exit 2022-09-03T05:49:08.276 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T05:49:08.277 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T05:49:08.275+0000 7f7cd8def700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground (PID: 109721) UID: 0 2022-09-03T05:49:08.277 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T05:49:08.275+0000 7f7cdd05a600 -1 shutting down 2022-09-03T05:49:10.277 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.0 is failed for ~0s 2022-09-03T05:49:14.378 INFO:tasks.rgw.c1.client.0:Stopped 2022-09-03T05:49:14.378 INFO:tasks.rgw.c1.client.0:Restarting daemon 2022-09-03T05:49:14.379 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c1.client.0.stdout 2>&1 2022-09-03T05:49:14.382 INFO:tasks.rgw.c1.client.0:Started 2022-09-03T05:49:14.383 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8000 2022-09-03T05:49:14.410 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:49:14.411 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:14.412 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:14.412 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T05:49:15.413 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8000 2022-09-03T05:49:15.441 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:15.441 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:49:15.445 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T05:49:15.985 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.0 has been restored 2022-09-03T05:49:17.447 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8000 2022-09-03T05:49:17.479 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:49:17.480 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:17.481 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:17.481 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T05:49:21.484 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8000 2022-09-03T05:49:21.514 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:21.514 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:21.515 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2022-09-03T05:49:21.516 INFO:teuthology.orchestra.run.smithi062.stdout:anonymous 2022-09-03T05:49:21.517 INFO:tasks.rgw_multisite:creating admin user.. 2022-09-03T05:49:21.517 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'create', '--uid', 'realm-admin', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--display-name', 'Realm Admin', '--system', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:21.517 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'create', '--uid', 'realm-admin', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--display-name', 'Realm Admin', '--system', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:21.518 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user create --uid realm-admin --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --display-name 'Realm Admin' --system --access-key ND2IR1XIGKP3UN6B --secret quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:21.622 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:21.623 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:21.656 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:49:21.657 INFO:teuthology.orchestra.run.smithi062.stdout: "user_id": "realm-admin", 2022-09-03T05:49:21.657 INFO:teuthology.orchestra.run.smithi062.stdout: "display_name": "Realm Admin", 2022-09-03T05:49:21.657 INFO:teuthology.orchestra.run.smithi062.stdout: "email": "", 2022-09-03T05:49:21.657 INFO:teuthology.orchestra.run.smithi062.stdout: "suspended": 0, 2022-09-03T05:49:21.658 INFO:teuthology.orchestra.run.smithi062.stdout: "max_buckets": 1000, 2022-09-03T05:49:21.658 INFO:teuthology.orchestra.run.smithi062.stdout: "subusers": [], 2022-09-03T05:49:21.658 INFO:teuthology.orchestra.run.smithi062.stdout: "keys": [ 2022-09-03T05:49:21.658 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:21.659 INFO:teuthology.orchestra.run.smithi062.stdout: "user": "realm-admin", 2022-09-03T05:49:21.659 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T05:49:21.659 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T05:49:21.659 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:21.660 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:21.660 INFO:teuthology.orchestra.run.smithi062.stdout: "swift_keys": [], 2022-09-03T05:49:21.660 INFO:teuthology.orchestra.run.smithi062.stdout: "caps": [], 2022-09-03T05:49:21.661 INFO:teuthology.orchestra.run.smithi062.stdout: "op_mask": "read, write, delete", 2022-09-03T05:49:21.661 INFO:teuthology.orchestra.run.smithi062.stdout: "system": true, 2022-09-03T05:49:21.661 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "", 2022-09-03T05:49:21.662 INFO:teuthology.orchestra.run.smithi062.stdout: "default_storage_class": "", 2022-09-03T05:49:21.662 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_tags": [], 2022-09-03T05:49:21.663 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_quota": { 2022-09-03T05:49:21.663 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T05:49:21.664 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T05:49:21.666 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T05:49:21.666 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T05:49:21.667 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T05:49:21.667 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:21.668 INFO:teuthology.orchestra.run.smithi062.stdout: "user_quota": { 2022-09-03T05:49:21.668 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T05:49:21.669 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T05:49:21.669 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T05:49:21.670 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T05:49:21.670 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T05:49:21.671 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:21.671 INFO:teuthology.orchestra.run.smithi062.stdout: "temp_url_keys": [], 2022-09-03T05:49:21.671 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "rgw", 2022-09-03T05:49:21.672 INFO:teuthology.orchestra.run.smithi062.stdout: "mfa_ids": [] 2022-09-03T05:49:21.672 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:49:21.672 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:49:21.678 INFO:tasks.rgw_multisite:pulling realm configuration to c2 2022-09-03T05:49:21.679 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['realm', 'pull', '--rgw-realm', 'test-realm', '--url', 'http://smithi062.front.sepia.ceph.com:8000', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:21.680 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'realm', 'pull', '--rgw-realm', 'test-realm', '--url', 'http://smithi062.front.sepia.ceph.com:8000', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:21.680 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 realm pull --rgw-realm test-realm --url http://smithi062.front.sepia.ceph.com:8000 --access-key ND2IR1XIGKP3UN6B --secret quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:21.741 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:21.741 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:21.782 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:49:21.783 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:21.787 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-realm", 2022-09-03T05:49:21.787 INFO:teuthology.orchestra.run.smithi087.stdout: "current_period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:49:21.788 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 2 2022-09-03T05:49:21.788 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:49:21.791 INFO:tasks.rgw_multisite:creating zone.. 2022-09-03T05:49:21.791 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'create', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--endpoints', 'http://smithi087.front.sepia.ceph.com:8000', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:21.792 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'create', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--endpoints', 'http://smithi087.front.sepia.ceph.com:8000', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:21.796 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone create --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --default --access-key ND2IR1XIGKP3UN6B --secret quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq --endpoints http://smithi087.front.sepia.ceph.com:8000 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:21.852 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:21.853 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:21.876 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:49:21.874+0000 7f13c0fff600 0 failed reading obj info from .rgw.root:zone_info.fca49d30-abba-4c2b-8462-66395ac063ba: (2) No such file or directory 2022-09-03T05:49:21.876 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:49:21.874+0000 7f13c0fff600 0 WARNING: could not read zone params for zone id=fca49d30-abba-4c2b-8462-66395ac063ba name=test-zone1 2022-09-03T05:49:21.880 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:49:21.881 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T05:49:21.881 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T05:49:21.882 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "test-zone2.rgw.meta:root", 2022-09-03T05:49:21.882 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "test-zone2.rgw.control", 2022-09-03T05:49:21.883 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2022-09-03T05:49:21.883 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2022-09-03T05:49:21.883 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "test-zone2.rgw.log", 2022-09-03T05:49:21.884 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2022-09-03T05:49:21.884 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2022-09-03T05:49:21.884 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2022-09-03T05:49:21.885 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2022-09-03T05:49:21.885 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2022-09-03T05:49:21.886 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2022-09-03T05:49:21.886 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2022-09-03T05:49:21.886 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2022-09-03T05:49:21.887 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "test-zone2.rgw.otp", 2022-09-03T05:49:21.887 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T05:49:21.888 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T05:49:21.888 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T05:49:21.889 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:21.889 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T05:49:21.889 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:49:21.890 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T05:49:21.891 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:49:21.891 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2022-09-03T05:49:21.892 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T05:49:21.892 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T05:49:21.892 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "test-zone2.rgw.buckets.data" 2022-09-03T05:49:21.893 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:21.893 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:21.894 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2022-09-03T05:49:21.894 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T05:49:21.894 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:21.895 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:21.895 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:49:21.895 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:21.896 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2022-09-03T05:49:21.896 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:49:21.897 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool create test-zone2.rgw.buckets.data 64 64 --cluster c2 2022-09-03T05:49:22.284 INFO:teuthology.orchestra.run.smithi087.stderr:pool 'test-zone2.rgw.buckets.data' created 2022-09-03T05:49:22.295 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool application enable test-zone2.rgw.buckets.data rgw --cluster c2 2022-09-03T05:49:23.294 INFO:teuthology.orchestra.run.smithi087.stderr:enabled application 'rgw' on pool 'test-zone2.rgw.buckets.data' 2022-09-03T05:49:23.314 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'placement', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:23.315 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'placement', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:23.316 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone placement --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:23.370 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:23.371 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:23.392 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:49:23.392 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T05:49:23.393 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T05:49:23.393 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "test-zone2.rgw.meta:root", 2022-09-03T05:49:23.393 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "test-zone2.rgw.control", 2022-09-03T05:49:23.394 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2022-09-03T05:49:23.394 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2022-09-03T05:49:23.394 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "test-zone2.rgw.log", 2022-09-03T05:49:23.395 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2022-09-03T05:49:23.395 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2022-09-03T05:49:23.395 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2022-09-03T05:49:23.396 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2022-09-03T05:49:23.396 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2022-09-03T05:49:23.396 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2022-09-03T05:49:23.397 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2022-09-03T05:49:23.397 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2022-09-03T05:49:23.398 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "test-zone2.rgw.otp", 2022-09-03T05:49:23.398 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T05:49:23.398 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T05:49:23.399 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T05:49:23.399 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:23.400 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T05:49:23.400 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:49:23.400 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T05:49:23.401 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:49:23.401 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2022-09-03T05:49:23.402 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T05:49:23.402 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T05:49:23.402 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "test-zone2.rgw.buckets.data", 2022-09-03T05:49:23.403 INFO:teuthology.orchestra.run.smithi087.stdout: "compression_type": "random" 2022-09-03T05:49:23.403 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:23.403 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:23.403 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2022-09-03T05:49:23.404 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T05:49:23.404 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:23.404 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:23.404 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:49:23.405 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "", 2022-09-03T05:49:23.405 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2022-09-03T05:49:23.405 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:49:23.407 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:23.407 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:23.408 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:23.455 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:23.456 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:23.464 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:49:23.462+0000 7fc6dfea2600 0 period (b84ccf39-78f5-4998-9db6-42362bc8c98b does not have zone 9b3b3b7d-a26b-43ae-9603-22cc8f947364 configured 2022-09-03T05:49:30.345 INFO:teuthology.orchestra.run.smithi087.stderr:Sending period to new master zone fca49d30-abba-4c2b-8462-66395ac063ba 2022-09-03T05:49:30.388 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:49:30.388 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:49:30.389 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 2, 2022-09-03T05:49:30.389 INFO:teuthology.orchestra.run.smithi087.stdout: "predecessor_uuid": "226f712d-7b05-4c77-9ead-974d95b142d0", 2022-09-03T05:49:30.389 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": [], 2022-09-03T05:49:30.389 INFO:teuthology.orchestra.run.smithi087.stdout: "period_map": { 2022-09-03T05:49:30.390 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:49:30.390 INFO:teuthology.orchestra.run.smithi087.stdout: "zonegroups": [ 2022-09-03T05:49:30.390 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:49:30.391 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "test-zonegroup", 2022-09-03T05:49:30.391 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zonegroup", 2022-09-03T05:49:30.391 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "", 2022-09-03T05:49:30.391 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": true, 2022-09-03T05:49:30.392 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T05:49:30.392 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T05:49:30.392 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:49:30.392 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2022-09-03T05:49:30.393 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2022-09-03T05:49:30.393 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:30.393 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [ 2022-09-03T05:49:30.394 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:49:30.394 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T05:49:30.395 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T05:49:30.395 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T05:49:30.395 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T05:49:30.396 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:49:30.396 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T05:49:30.397 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T05:49:30.397 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:49:30.397 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T05:49:30.398 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T05:49:30.398 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T05:49:30.399 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T05:49:30.399 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T05:49:30.399 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T05:49:30.400 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T05:49:30.400 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:49:30.401 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:30.401 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:49:30.401 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:30.403 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone1", 2022-09-03T05:49:30.403 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T05:49:30.403 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T05:49:30.403 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:49:30.404 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T05:49:30.404 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T05:49:30.404 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:49:30.405 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T05:49:30.406 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T05:49:30.406 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T05:49:30.406 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T05:49:30.407 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T05:49:30.407 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T05:49:30.407 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T05:49:30.407 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:49:30.408 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:30.408 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:49:30.408 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [ 2022-09-03T05:49:30.408 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:49:30.409 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default-placement", 2022-09-03T05:49:30.409 INFO:teuthology.orchestra.run.smithi087.stdout: "tags": [], 2022-09-03T05:49:30.409 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": [ 2022-09-03T05:49:30.410 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD" 2022-09-03T05:49:30.410 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:49:30.410 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:30.410 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:49:30.411 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "default-placement", 2022-09-03T05:49:30.411 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:30.411 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2022-09-03T05:49:30.411 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2022-09-03T05:49:30.411 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:30.412 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled_features": [ 2022-09-03T05:49:30.412 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T05:49:30.412 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:49:30.412 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:30.413 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:49:30.413 INFO:teuthology.orchestra.run.smithi087.stdout: "short_zone_ids": [ 2022-09-03T05:49:30.413 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:49:30.413 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T05:49:30.414 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 3737622162 2022-09-03T05:49:30.414 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:30.414 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:49:30.414 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:30.415 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 449119331 2022-09-03T05:49:30.415 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:30.415 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:49:30.415 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:30.415 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T05:49:30.416 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:30.416 INFO:teuthology.orchestra.run.smithi087.stdout: "period_config": { 2022-09-03T05:49:30.416 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_quota": { 2022-09-03T05:49:30.416 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T05:49:30.417 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T05:49:30.417 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T05:49:30.417 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T05:49:30.417 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T05:49:30.417 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:30.418 INFO:teuthology.orchestra.run.smithi087.stdout: "user_quota": { 2022-09-03T05:49:30.418 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T05:49:30.418 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T05:49:30.418 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T05:49:30.419 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T05:49:30.419 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T05:49:30.419 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:30.419 INFO:teuthology.orchestra.run.smithi087.stdout: "user_ratelimit": { 2022-09-03T05:49:30.420 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T05:49:30.420 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T05:49:30.420 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T05:49:30.420 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T05:49:30.420 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T05:49:30.421 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:30.421 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_ratelimit": { 2022-09-03T05:49:30.421 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T05:49:30.421 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T05:49:30.422 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T05:49:30.422 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T05:49:30.422 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T05:49:30.422 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:30.424 INFO:teuthology.orchestra.run.smithi087.stdout: "anonymous_ratelimit": { 2022-09-03T05:49:30.424 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T05:49:30.424 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T05:49:30.424 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T05:49:30.425 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T05:49:30.425 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T05:49:30.425 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:30.425 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:30.426 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:30.426 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_name": "test-realm", 2022-09-03T05:49:30.426 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:49:30.426 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:49:30.427 DEBUG:tasks.rgw.c2.client.0:waiting for process to exit 2022-09-03T05:49:30.428 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T05:49:30.429 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T05:49:30.426+0000 7f3f14fca700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground (PID: 109687) UID: 0 2022-09-03T05:49:30.429 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T05:49:30.426+0000 7f3f19235600 -1 shutting down 2022-09-03T05:49:34.207 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~0s 2022-09-03T05:49:36.529 INFO:tasks.rgw.c2.client.0:Stopped 2022-09-03T05:49:36.530 INFO:tasks.rgw.c2.client.0:Restarting daemon 2022-09-03T05:49:36.531 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2022-09-03T05:49:36.535 INFO:tasks.rgw.c2.client.0:Started 2022-09-03T05:49:36.536 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T05:49:36.564 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:49:36.565 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:36.566 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:36.566 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T05:49:37.568 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T05:49:37.596 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:49:37.597 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:37.597 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:37.597 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T05:49:39.602 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T05:49:39.629 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:39.630 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:39.630 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T05:49:39.631 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:49:39.915 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 has been restored 2022-09-03T05:49:43.633 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T05:49:43.659 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:43.659 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:43.662 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2022-09-03T05:49:43.663 INFO:teuthology.orchestra.run.smithi087.stdout:anonymous 2022-09-03T05:49:43.663 INFO:tasks.rgw_multisite:creating zone.. 2022-09-03T05:49:43.664 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--endpoints', 'http://smithi062.front.sepia.ceph.com:8001', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:43.664 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--endpoints', 'http://smithi062.front.sepia.ceph.com:8001', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:43.665 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --access-key ND2IR1XIGKP3UN6B --secret quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq --endpoints http://smithi062.front.sepia.ceph.com:8001 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:43.728 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:43.728 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:43.759 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:49:43.757+0000 7faed1fc9600 0 failed reading obj info from .rgw.root:zone_info.9b3b3b7d-a26b-43ae-9603-22cc8f947364: (2) No such file or directory 2022-09-03T05:49:43.759 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:49:43.757+0000 7faed1fc9600 0 WARNING: could not read zone params for zone id=9b3b3b7d-a26b-43ae-9603-22cc8f947364 name=test-zone2 2022-09-03T05:49:43.764 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:49:43.764 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T05:49:43.765 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone3", 2022-09-03T05:49:43.765 INFO:teuthology.orchestra.run.smithi062.stdout: "domain_root": "test-zone3.rgw.meta:root", 2022-09-03T05:49:43.765 INFO:teuthology.orchestra.run.smithi062.stdout: "control_pool": "test-zone3.rgw.control", 2022-09-03T05:49:43.766 INFO:teuthology.orchestra.run.smithi062.stdout: "gc_pool": "test-zone3.rgw.log:gc", 2022-09-03T05:49:43.766 INFO:teuthology.orchestra.run.smithi062.stdout: "lc_pool": "test-zone3.rgw.log:lc", 2022-09-03T05:49:43.766 INFO:teuthology.orchestra.run.smithi062.stdout: "log_pool": "test-zone3.rgw.log", 2022-09-03T05:49:43.766 INFO:teuthology.orchestra.run.smithi062.stdout: "intent_log_pool": "test-zone3.rgw.log:intent", 2022-09-03T05:49:43.767 INFO:teuthology.orchestra.run.smithi062.stdout: "usage_log_pool": "test-zone3.rgw.log:usage", 2022-09-03T05:49:43.767 INFO:teuthology.orchestra.run.smithi062.stdout: "roles_pool": "test-zone3.rgw.meta:roles", 2022-09-03T05:49:43.767 INFO:teuthology.orchestra.run.smithi062.stdout: "reshard_pool": "test-zone3.rgw.log:reshard", 2022-09-03T05:49:43.767 INFO:teuthology.orchestra.run.smithi062.stdout: "user_keys_pool": "test-zone3.rgw.meta:users.keys", 2022-09-03T05:49:43.768 INFO:teuthology.orchestra.run.smithi062.stdout: "user_email_pool": "test-zone3.rgw.meta:users.email", 2022-09-03T05:49:43.768 INFO:teuthology.orchestra.run.smithi062.stdout: "user_swift_pool": "test-zone3.rgw.meta:users.swift", 2022-09-03T05:49:43.768 INFO:teuthology.orchestra.run.smithi062.stdout: "user_uid_pool": "test-zone3.rgw.meta:users.uid", 2022-09-03T05:49:43.768 INFO:teuthology.orchestra.run.smithi062.stdout: "otp_pool": "test-zone3.rgw.otp", 2022-09-03T05:49:43.769 INFO:teuthology.orchestra.run.smithi062.stdout: "system_key": { 2022-09-03T05:49:43.769 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T05:49:43.769 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T05:49:43.769 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:43.770 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_pools": [ 2022-09-03T05:49:43.770 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:43.770 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "default-placement", 2022-09-03T05:49:43.770 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:49:43.770 INFO:teuthology.orchestra.run.smithi062.stdout: "index_pool": "test-zone3.rgw.buckets.index", 2022-09-03T05:49:43.771 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": { 2022-09-03T05:49:43.771 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD": { 2022-09-03T05:49:43.771 INFO:teuthology.orchestra.run.smithi062.stdout: "data_pool": "test-zone3.rgw.buckets.data" 2022-09-03T05:49:43.771 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:43.772 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:43.772 INFO:teuthology.orchestra.run.smithi062.stdout: "data_extra_pool": "test-zone3.rgw.buckets.non-ec", 2022-09-03T05:49:43.772 INFO:teuthology.orchestra.run.smithi062.stdout: "index_type": 0 2022-09-03T05:49:43.772 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:43.772 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:43.773 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:43.773 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:43.773 INFO:teuthology.orchestra.run.smithi062.stdout: "notif_pool": "test-zone3.rgw.log:notif" 2022-09-03T05:49:43.773 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:49:43.774 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool create test-zone3.rgw.buckets.data 64 64 --cluster c1 2022-09-03T05:49:44.250 INFO:teuthology.orchestra.run.smithi062.stderr:pool 'test-zone3.rgw.buckets.data' created 2022-09-03T05:49:44.270 DEBUG:teuthology.orchestra.run.smithi062:> sudo ceph osd pool application enable test-zone3.rgw.buckets.data rgw --cluster c1 2022-09-03T05:49:45.281 INFO:teuthology.orchestra.run.smithi062.stderr:enabled application 'rgw' on pool 'test-zone3.rgw.buckets.data' 2022-09-03T05:49:45.298 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'placement', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:45.299 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'placement', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:45.300 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone placement --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:45.356 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:45.357 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:45.380 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:49:45.381 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T05:49:45.381 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone3", 2022-09-03T05:49:45.382 INFO:teuthology.orchestra.run.smithi062.stdout: "domain_root": "test-zone3.rgw.meta:root", 2022-09-03T05:49:45.382 INFO:teuthology.orchestra.run.smithi062.stdout: "control_pool": "test-zone3.rgw.control", 2022-09-03T05:49:45.382 INFO:teuthology.orchestra.run.smithi062.stdout: "gc_pool": "test-zone3.rgw.log:gc", 2022-09-03T05:49:45.383 INFO:teuthology.orchestra.run.smithi062.stdout: "lc_pool": "test-zone3.rgw.log:lc", 2022-09-03T05:49:45.383 INFO:teuthology.orchestra.run.smithi062.stdout: "log_pool": "test-zone3.rgw.log", 2022-09-03T05:49:45.383 INFO:teuthology.orchestra.run.smithi062.stdout: "intent_log_pool": "test-zone3.rgw.log:intent", 2022-09-03T05:49:45.383 INFO:teuthology.orchestra.run.smithi062.stdout: "usage_log_pool": "test-zone3.rgw.log:usage", 2022-09-03T05:49:45.384 INFO:teuthology.orchestra.run.smithi062.stdout: "roles_pool": "test-zone3.rgw.meta:roles", 2022-09-03T05:49:45.384 INFO:teuthology.orchestra.run.smithi062.stdout: "reshard_pool": "test-zone3.rgw.log:reshard", 2022-09-03T05:49:45.384 INFO:teuthology.orchestra.run.smithi062.stdout: "user_keys_pool": "test-zone3.rgw.meta:users.keys", 2022-09-03T05:49:45.384 INFO:teuthology.orchestra.run.smithi062.stdout: "user_email_pool": "test-zone3.rgw.meta:users.email", 2022-09-03T05:49:45.385 INFO:teuthology.orchestra.run.smithi062.stdout: "user_swift_pool": "test-zone3.rgw.meta:users.swift", 2022-09-03T05:49:45.385 INFO:teuthology.orchestra.run.smithi062.stdout: "user_uid_pool": "test-zone3.rgw.meta:users.uid", 2022-09-03T05:49:45.386 INFO:teuthology.orchestra.run.smithi062.stdout: "otp_pool": "test-zone3.rgw.otp", 2022-09-03T05:49:45.386 INFO:teuthology.orchestra.run.smithi062.stdout: "system_key": { 2022-09-03T05:49:45.386 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T05:49:45.387 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T05:49:45.387 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:45.387 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_pools": [ 2022-09-03T05:49:45.387 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:45.388 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "default-placement", 2022-09-03T05:49:45.388 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:49:45.388 INFO:teuthology.orchestra.run.smithi062.stdout: "index_pool": "test-zone3.rgw.buckets.index", 2022-09-03T05:49:45.388 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": { 2022-09-03T05:49:45.389 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD": { 2022-09-03T05:49:45.389 INFO:teuthology.orchestra.run.smithi062.stdout: "data_pool": "test-zone3.rgw.buckets.data", 2022-09-03T05:49:45.389 INFO:teuthology.orchestra.run.smithi062.stdout: "compression_type": "random" 2022-09-03T05:49:45.389 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:45.390 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:45.390 INFO:teuthology.orchestra.run.smithi062.stdout: "data_extra_pool": "test-zone3.rgw.buckets.non-ec", 2022-09-03T05:49:45.391 INFO:teuthology.orchestra.run.smithi062.stdout: "index_type": 0 2022-09-03T05:49:45.391 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:45.391 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:45.392 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:45.392 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "", 2022-09-03T05:49:45.393 INFO:teuthology.orchestra.run.smithi062.stdout: "notif_pool": "test-zone3.rgw.log:notif" 2022-09-03T05:49:45.393 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:49:45.394 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:45.395 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:45.396 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:45.447 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:45.447 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:45.456 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:49:45.455+0000 7ffbd43f7600 0 period (b84ccf39-78f5-4998-9db6-42362bc8c98b does not have zone 00aa9b6d-7a02-4715-9200-8a26b45eb25e configured 2022-09-03T05:49:52.325 INFO:teuthology.orchestra.run.smithi062.stderr:Sending period to new master zone fca49d30-abba-4c2b-8462-66395ac063ba 2022-09-03T05:49:52.356 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:49:52.356 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:49:52.357 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 3, 2022-09-03T05:49:52.357 INFO:teuthology.orchestra.run.smithi062.stdout: "predecessor_uuid": "226f712d-7b05-4c77-9ead-974d95b142d0", 2022-09-03T05:49:52.357 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": [], 2022-09-03T05:49:52.358 INFO:teuthology.orchestra.run.smithi062.stdout: "period_map": { 2022-09-03T05:49:52.358 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:49:52.358 INFO:teuthology.orchestra.run.smithi062.stdout: "zonegroups": [ 2022-09-03T05:49:52.359 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:52.359 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "test-zonegroup", 2022-09-03T05:49:52.359 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zonegroup", 2022-09-03T05:49:52.359 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "", 2022-09-03T05:49:52.360 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": true, 2022-09-03T05:49:52.360 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T05:49:52.360 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T05:49:52.361 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:52.361 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T05:49:52.361 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T05:49:52.361 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:52.361 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [ 2022-09-03T05:49:52.362 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:52.362 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T05:49:52.362 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone3", 2022-09-03T05:49:52.363 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T05:49:52.363 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T05:49:52.363 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:52.363 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T05:49:52.364 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T05:49:52.364 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:49:52.364 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T05:49:52.365 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T05:49:52.365 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T05:49:52.366 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T05:49:52.366 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T05:49:52.367 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T05:49:52.367 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T05:49:52.367 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:52.368 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.368 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:52.369 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T05:49:52.369 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone2", 2022-09-03T05:49:52.369 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T05:49:52.370 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T05:49:52.370 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:52.371 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T05:49:52.371 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T05:49:52.372 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:49:52.372 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T05:49:52.373 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T05:49:52.373 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T05:49:52.374 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T05:49:52.374 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T05:49:52.374 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T05:49:52.375 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T05:49:52.375 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:52.376 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.376 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:52.376 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:52.376 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T05:49:52.377 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T05:49:52.377 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T05:49:52.377 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:52.377 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T05:49:52.378 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T05:49:52.378 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:49:52.378 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T05:49:52.378 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T05:49:52.379 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T05:49:52.379 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T05:49:52.379 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T05:49:52.379 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T05:49:52.380 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T05:49:52.380 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:52.380 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:52.380 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:52.380 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [ 2022-09-03T05:49:52.381 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:52.381 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default-placement", 2022-09-03T05:49:52.381 INFO:teuthology.orchestra.run.smithi062.stdout: "tags": [], 2022-09-03T05:49:52.381 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": [ 2022-09-03T05:49:52.382 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD" 2022-09-03T05:49:52.382 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:52.382 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:52.382 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:52.383 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "default-placement", 2022-09-03T05:49:52.383 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:52.383 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T05:49:52.384 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T05:49:52.384 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.384 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T05:49:52.384 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T05:49:52.385 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:52.385 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:52.385 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:49:52.385 INFO:teuthology.orchestra.run.smithi062.stdout: "short_zone_ids": [ 2022-09-03T05:49:52.386 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:52.386 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T05:49:52.386 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 700722763 2022-09-03T05:49:52.386 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.387 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:52.387 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T05:49:52.387 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 3737622162 2022-09-03T05:49:52.387 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.387 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:49:52.388 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:52.388 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 449119331 2022-09-03T05:49:52.388 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:52.388 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:49:52.389 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.390 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T05:49:52.390 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:49:52.391 INFO:teuthology.orchestra.run.smithi062.stdout: "period_config": { 2022-09-03T05:49:52.391 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_quota": { 2022-09-03T05:49:52.391 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T05:49:52.391 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T05:49:52.392 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T05:49:52.392 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T05:49:52.392 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T05:49:52.393 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.393 INFO:teuthology.orchestra.run.smithi062.stdout: "user_quota": { 2022-09-03T05:49:52.393 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T05:49:52.393 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T05:49:52.394 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T05:49:52.394 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T05:49:52.394 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T05:49:52.394 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.394 INFO:teuthology.orchestra.run.smithi062.stdout: "user_ratelimit": { 2022-09-03T05:49:52.395 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T05:49:52.395 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T05:49:52.395 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T05:49:52.395 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T05:49:52.396 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T05:49:52.396 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.396 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_ratelimit": { 2022-09-03T05:49:52.396 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T05:49:52.397 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T05:49:52.397 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T05:49:52.397 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T05:49:52.397 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T05:49:52.397 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.398 INFO:teuthology.orchestra.run.smithi062.stdout: "anonymous_ratelimit": { 2022-09-03T05:49:52.398 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T05:49:52.398 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T05:49:52.398 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T05:49:52.399 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T05:49:52.399 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T05:49:52.399 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:49:52.400 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:49:52.400 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:52.400 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_name": "test-realm", 2022-09-03T05:49:52.400 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:49:52.401 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:49:52.401 DEBUG:tasks.rgw.c1.client.1:waiting for process to exit 2022-09-03T05:49:52.402 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T05:49:52.403 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T05:49:52.400+0000 7f3bb4e90700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground (PID: 109729) UID: 0 2022-09-03T05:49:52.403 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T05:49:52.400+0000 7f3bb90fb600 -1 shutting down 2022-09-03T05:49:55.837 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~0s 2022-09-03T05:49:58.504 INFO:tasks.rgw.c1.client.1:Stopped 2022-09-03T05:49:58.507 INFO:tasks.rgw.c1.client.1:Restarting daemon 2022-09-03T05:49:58.510 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c1.client.1.stdout 2>&1 2022-09-03T05:49:58.518 INFO:tasks.rgw.c1.client.1:Started 2022-09-03T05:49:58.519 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001 2022-09-03T05:49:58.552 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:49:58.553 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:58.554 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:58.554 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T05:49:59.556 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001 2022-09-03T05:49:59.582 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:49:59.583 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:49:59.588 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2022-09-03T05:49:59.589 INFO:teuthology.orchestra.run.smithi062.stdout:anonymous 2022-09-03T05:49:59.590 INFO:tasks.rgw_multisite:pulling realm configuration to c2 2022-09-03T05:49:59.590 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['realm', 'pull', '--rgw-realm', 'test-realm', '--url', 'http://smithi062.front.sepia.ceph.com:8000', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--url', 'http://smithi062.front.sepia.ceph.com:8000', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:59.590 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'realm', 'pull', '--rgw-realm', 'test-realm', '--url', 'http://smithi062.front.sepia.ceph.com:8000', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--url', 'http://smithi062.front.sepia.ceph.com:8000', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:59.591 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 realm pull --rgw-realm test-realm --url http://smithi062.front.sepia.ceph.com:8000 --access-key ND2IR1XIGKP3UN6B --secret quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq --url http://smithi062.front.sepia.ceph.com:8000 --access-key ND2IR1XIGKP3UN6B --secret quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:59.655 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:59.656 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:59.687 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T05:49:59.685+0000 7f4f68571700 -1 rgw realm watcher: RGWRealmWatcher::handle_error oid=realms.ec23be1e-4748-4838-8436-31585a76efae.control err=-107 2022-09-03T05:49:59.707 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:49:59.708 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:59.708 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-realm", 2022-09-03T05:49:59.708 INFO:teuthology.orchestra.run.smithi087.stdout: "current_period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:49:59.709 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 2 2022-09-03T05:49:59.709 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:49:59.709 INFO:tasks.rgw_multisite:creating zone.. 2022-09-03T05:49:59.710 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'create', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--endpoints', 'http://smithi087.front.sepia.ceph.com:8001', '--tier-type', 'pubsub', '--sync-from-all=0', '--sync-from', 'test-zone1', '--tier-config', 'start_with_full_sync=false,event_retention_days=7', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:59.711 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'create', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--endpoints', 'http://smithi087.front.sepia.ceph.com:8001', '--tier-type', 'pubsub', '--sync-from-all=0', '--sync-from', 'test-zone1', '--tier-config', 'start_with_full_sync=false,event_retention_days=7', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:49:59.711 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone create --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --access-key ND2IR1XIGKP3UN6B --secret quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq --endpoints http://smithi087.front.sepia.ceph.com:8001 --tier-type pubsub --sync-from-all=0 --sync-from test-zone1 --tier-config start_with_full_sync=false,event_retention_days=7 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T05:49:59.772 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:49:59.773 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:49:59.797 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:49:59.796+0000 7f298d60a600 0 failed reading obj info from .rgw.root:zone_info.00aa9b6d-7a02-4715-9200-8a26b45eb25e: (2) No such file or directory 2022-09-03T05:49:59.798 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:49:59.796+0000 7f298d60a600 0 WARNING: could not read zone params for zone id=00aa9b6d-7a02-4715-9200-8a26b45eb25e name=test-zone3 2022-09-03T05:49:59.798 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:49:59.797+0000 7f298d60a600 0 failed reading obj info from .rgw.root:zone_info.fca49d30-abba-4c2b-8462-66395ac063ba: (2) No such file or directory 2022-09-03T05:49:59.799 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:49:59.797+0000 7f298d60a600 0 WARNING: could not read zone params for zone id=fca49d30-abba-4c2b-8462-66395ac063ba name=test-zone1 2022-09-03T05:49:59.802 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:49:59.802 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T05:49:59.803 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone4", 2022-09-03T05:49:59.803 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "test-zone4.rgw.meta:root", 2022-09-03T05:49:59.803 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "test-zone4.rgw.control", 2022-09-03T05:49:59.803 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "test-zone4.rgw.log:gc", 2022-09-03T05:49:59.804 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "test-zone4.rgw.log:lc", 2022-09-03T05:49:59.804 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "test-zone4.rgw.log", 2022-09-03T05:49:59.804 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "test-zone4.rgw.log:intent", 2022-09-03T05:49:59.805 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "test-zone4.rgw.log:usage", 2022-09-03T05:49:59.805 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "test-zone4.rgw.meta:roles", 2022-09-03T05:49:59.805 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "test-zone4.rgw.log:reshard", 2022-09-03T05:49:59.806 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "test-zone4.rgw.meta:users.keys", 2022-09-03T05:49:59.806 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "test-zone4.rgw.meta:users.email", 2022-09-03T05:49:59.806 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "test-zone4.rgw.meta:users.swift", 2022-09-03T05:49:59.807 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "test-zone4.rgw.meta:users.uid", 2022-09-03T05:49:59.807 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "test-zone4.rgw.otp", 2022-09-03T05:49:59.807 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T05:49:59.808 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T05:49:59.808 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T05:49:59.808 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:59.809 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T05:49:59.809 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:49:59.809 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T05:49:59.809 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:49:59.810 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "test-zone4.rgw.buckets.index", 2022-09-03T05:49:59.810 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T05:49:59.810 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T05:49:59.811 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "test-zone4.rgw.buckets.data" 2022-09-03T05:49:59.811 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:59.811 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:59.811 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "test-zone4.rgw.buckets.non-ec", 2022-09-03T05:49:59.812 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T05:49:59.812 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:59.812 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:49:59.812 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:49:59.813 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_config": { 2022-09-03T05:49:59.813 INFO:teuthology.orchestra.run.smithi087.stdout: "event_retention_days": 7, 2022-09-03T05:49:59.813 INFO:teuthology.orchestra.run.smithi087.stdout: "start_with_full_sync": false 2022-09-03T05:49:59.813 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:49:59.813 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:49:59.814 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "test-zone4.rgw.log:notif" 2022-09-03T05:49:59.814 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:49:59.815 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool create test-zone4.rgw.buckets.data 64 64 --cluster c2 2022-09-03T05:50:01.115 INFO:teuthology.orchestra.run.smithi087.stderr:pool 'test-zone4.rgw.buckets.data' created 2022-09-03T05:50:01.135 DEBUG:teuthology.orchestra.run.smithi087:> sudo ceph osd pool application enable test-zone4.rgw.buckets.data rgw --cluster c2 2022-09-03T05:50:01.545 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 has been restored 2022-09-03T05:50:02.143 INFO:teuthology.orchestra.run.smithi087.stderr:enabled application 'rgw' on pool 'test-zone4.rgw.buckets.data' 2022-09-03T05:50:02.164 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'placement', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:50:02.164 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'placement', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:50:02.165 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone placement --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T05:50:02.224 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:02.224 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:02.246 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:02.247 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T05:50:02.247 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone4", 2022-09-03T05:50:02.247 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "test-zone4.rgw.meta:root", 2022-09-03T05:50:02.247 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "test-zone4.rgw.control", 2022-09-03T05:50:02.248 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "test-zone4.rgw.log:gc", 2022-09-03T05:50:02.248 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "test-zone4.rgw.log:lc", 2022-09-03T05:50:02.248 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "test-zone4.rgw.log", 2022-09-03T05:50:02.248 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "test-zone4.rgw.log:intent", 2022-09-03T05:50:02.249 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "test-zone4.rgw.log:usage", 2022-09-03T05:50:02.249 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "test-zone4.rgw.meta:roles", 2022-09-03T05:50:02.249 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "test-zone4.rgw.log:reshard", 2022-09-03T05:50:02.249 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "test-zone4.rgw.meta:users.keys", 2022-09-03T05:50:02.249 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "test-zone4.rgw.meta:users.email", 2022-09-03T05:50:02.250 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "test-zone4.rgw.meta:users.swift", 2022-09-03T05:50:02.250 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "test-zone4.rgw.meta:users.uid", 2022-09-03T05:50:02.250 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "test-zone4.rgw.otp", 2022-09-03T05:50:02.250 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T05:50:02.251 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T05:50:02.251 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T05:50:02.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:02.251 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T05:50:02.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:02.252 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T05:50:02.252 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:02.252 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "test-zone4.rgw.buckets.index", 2022-09-03T05:50:02.253 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T05:50:02.253 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T05:50:02.253 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "test-zone4.rgw.buckets.data", 2022-09-03T05:50:02.253 INFO:teuthology.orchestra.run.smithi087.stdout: "compression_type": "random" 2022-09-03T05:50:02.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:02.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:02.254 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "test-zone4.rgw.buckets.non-ec", 2022-09-03T05:50:02.254 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T05:50:02.254 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:02.255 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:02.255 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:02.255 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_config": { 2022-09-03T05:50:02.255 INFO:teuthology.orchestra.run.smithi087.stdout: "event_retention_days": 7, 2022-09-03T05:50:02.256 INFO:teuthology.orchestra.run.smithi087.stdout: "start_with_full_sync": false 2022-09-03T05:50:02.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:02.256 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "", 2022-09-03T05:50:02.256 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "test-zone4.rgw.log:notif" 2022-09-03T05:50:02.256 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:02.258 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:50:02.258 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:50:02.259 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T05:50:02.307 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:02.308 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:02.316 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:02.315+0000 7f58e34d0600 0 period (b84ccf39-78f5-4998-9db6-42362bc8c98b does not have zone 31b17c08-52e2-4c88-9aba-17b28ca757e4 configured 2022-09-03T05:50:09.174 INFO:teuthology.orchestra.run.smithi087.stderr:Sending period to new master zone fca49d30-abba-4c2b-8462-66395ac063ba 2022-09-03T05:50:09.204 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:09.204 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:09.204 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 4, 2022-09-03T05:50:09.205 INFO:teuthology.orchestra.run.smithi087.stdout: "predecessor_uuid": "226f712d-7b05-4c77-9ead-974d95b142d0", 2022-09-03T05:50:09.205 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": [], 2022-09-03T05:50:09.205 INFO:teuthology.orchestra.run.smithi087.stdout: "period_map": { 2022-09-03T05:50:09.205 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:09.206 INFO:teuthology.orchestra.run.smithi087.stdout: "zonegroups": [ 2022-09-03T05:50:09.206 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.206 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "test-zonegroup", 2022-09-03T05:50:09.206 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zonegroup", 2022-09-03T05:50:09.207 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "", 2022-09-03T05:50:09.207 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": true, 2022-09-03T05:50:09.207 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T05:50:09.208 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T05:50:09.208 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:09.208 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2022-09-03T05:50:09.208 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2022-09-03T05:50:09.209 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:50:09.209 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [ 2022-09-03T05:50:09.209 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.209 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T05:50:09.209 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone3", 2022-09-03T05:50:09.210 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T05:50:09.210 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T05:50:09.210 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:09.210 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T05:50:09.211 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T05:50:09.211 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:50:09.211 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T05:50:09.211 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T05:50:09.212 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T05:50:09.212 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T05:50:09.212 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T05:50:09.212 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T05:50:09.212 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T05:50:09.213 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:09.213 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.213 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.213 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T05:50:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone4", 2022-09-03T05:50:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T05:50:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T05:50:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T05:50:09.215 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T05:50:09.215 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:50:09.215 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T05:50:09.215 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "pubsub", 2022-09-03T05:50:09.216 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": false, 2022-09-03T05:50:09.216 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [ 2022-09-03T05:50:09.216 INFO:teuthology.orchestra.run.smithi087.stdout: "test-zone1" 2022-09-03T05:50:09.216 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:09.217 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T05:50:09.217 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T05:50:09.217 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T05:50:09.217 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:09.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.218 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T05:50:09.218 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T05:50:09.218 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T05:50:09.219 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T05:50:09.219 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:09.219 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T05:50:09.219 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T05:50:09.220 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:50:09.220 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T05:50:09.220 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T05:50:09.220 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T05:50:09.220 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T05:50:09.221 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T05:50:09.221 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T05:50:09.221 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T05:50:09.221 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:09.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.222 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.222 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:50:09.222 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone1", 2022-09-03T05:50:09.223 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T05:50:09.223 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T05:50:09.223 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:09.223 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T05:50:09.224 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T05:50:09.224 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T05:50:09.224 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T05:50:09.224 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T05:50:09.224 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T05:50:09.225 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T05:50:09.225 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T05:50:09.225 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T05:50:09.225 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T05:50:09.226 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:09.226 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:09.226 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:09.226 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [ 2022-09-03T05:50:09.227 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.227 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default-placement", 2022-09-03T05:50:09.227 INFO:teuthology.orchestra.run.smithi087.stdout: "tags": [], 2022-09-03T05:50:09.227 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": [ 2022-09-03T05:50:09.229 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD" 2022-09-03T05:50:09.229 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:09.229 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:09.229 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:09.230 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "default-placement", 2022-09-03T05:50:09.230 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:50:09.230 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2022-09-03T05:50:09.230 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2022-09-03T05:50:09.231 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.231 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled_features": [ 2022-09-03T05:50:09.231 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T05:50:09.231 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:09.232 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:09.232 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T05:50:09.232 INFO:teuthology.orchestra.run.smithi087.stdout: "short_zone_ids": [ 2022-09-03T05:50:09.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.233 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T05:50:09.233 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 700722763 2022-09-03T05:50:09.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.233 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T05:50:09.234 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 2398203312 2022-09-03T05:50:09.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.235 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T05:50:09.235 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 3737622162 2022-09-03T05:50:09.235 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.235 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:09.236 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:50:09.236 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 449119331 2022-09-03T05:50:09.236 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:09.236 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:09.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.237 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T05:50:09.237 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T05:50:09.237 INFO:teuthology.orchestra.run.smithi087.stdout: "period_config": { 2022-09-03T05:50:09.237 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_quota": { 2022-09-03T05:50:09.238 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T05:50:09.238 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T05:50:09.238 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T05:50:09.238 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T05:50:09.238 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T05:50:09.239 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.239 INFO:teuthology.orchestra.run.smithi087.stdout: "user_quota": { 2022-09-03T05:50:09.239 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T05:50:09.240 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T05:50:09.240 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T05:50:09.240 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T05:50:09.240 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T05:50:09.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.241 INFO:teuthology.orchestra.run.smithi087.stdout: "user_ratelimit": { 2022-09-03T05:50:09.241 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T05:50:09.241 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T05:50:09.242 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T05:50:09.242 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T05:50:09.242 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T05:50:09.242 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.242 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_ratelimit": { 2022-09-03T05:50:09.243 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T05:50:09.243 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T05:50:09.243 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T05:50:09.243 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T05:50:09.244 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T05:50:09.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.244 INFO:teuthology.orchestra.run.smithi087.stdout: "anonymous_ratelimit": { 2022-09-03T05:50:09.244 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T05:50:09.245 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T05:50:09.245 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T05:50:09.245 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T05:50:09.245 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T05:50:09.246 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:09.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:09.246 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T05:50:09.246 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_name": "test-realm", 2022-09-03T05:50:09.246 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:09.247 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:09.247 DEBUG:tasks.rgw.c2.client.1:waiting for process to exit 2022-09-03T05:50:09.248 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T05:50:09.249 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T05:50:09.247+0000 7fd0a12ce700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground (PID: 109692) UID: 0 2022-09-03T05:50:09.249 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T05:50:09.247+0000 7fd0a5539600 -1 shutting down 2022-09-03T05:50:14.054 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.1 is failed for ~0s 2022-09-03T05:50:15.351 INFO:tasks.rgw.c2.client.1:Stopped 2022-09-03T05:50:15.352 INFO:tasks.rgw.c2.client.1:Restarting daemon 2022-09-03T05:50:15.353 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.1.stdout 2>&1 2022-09-03T05:50:15.357 INFO:tasks.rgw.c2.client.1:Started 2022-09-03T05:50:15.358 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8001 2022-09-03T05:50:15.387 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:50:15.388 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:50:15.388 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:50:15.389 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T05:50:16.390 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8001 2022-09-03T05:50:16.418 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:50:16.419 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:50:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:50:16.421 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T05:50:18.423 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8001 2022-09-03T05:50:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T05:50:18.454 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:50:18.455 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:50:18.455 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T05:50:19.764 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.1 has been restored 2022-09-03T05:50:22.456 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8001 2022-09-03T05:50:22.485 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T05:50:22.485 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T05:50:22.486 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 134 100 134 0 0 130k 0 --:--:-- --:--:-- --:--:-- 130k 2022-09-03T05:50:22.488 INFO:teuthology.orchestra.run.smithi087.stdout:{"Code":"MethodNotAllowed","RequestId":"tx00000b34f8ef60b29fa39-006312eb1e-10f4-test-zone4","HostId":"10f4-test-zone4-test-zonegroup"} 2022-09-03T05:50:22.489 INFO:tasks.rgw_multisite:rgw multisite configuration completed 2022-09-03T05:50:22.490 INFO:teuthology.run_tasks:Running task rgw-multisite-tests... 2022-09-03T05:50:22.510 INFO:tasks.rgw_multisite_tests:importing tests from /home/teuthworker/src/github.com_cbodley_ceph_e20bde8244d2a27c6431acd07e9967aaa8f04f01/qa/../src/test/rgw/rgw_multi 2022-09-03T05:50:22.537 INFO:tasks.rgw_multisite_tests:creating test user.. 2022-09-03T05:50:22.538 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'create', '--uid', 'rgw-multisite-test-user', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--display-name', 'Multisite Test User', '--gen-access-key', '--gen-secret', '--caps', 'roles=*', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:50:22.539 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'create', '--uid', 'rgw-multisite-test-user', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--display-name', 'Multisite Test User', '--gen-access-key', '--gen-secret', '--caps', 'roles=*', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T05:50:22.539 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user create --uid rgw-multisite-test-user --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --display-name 'Multisite Test User' --gen-access-key --gen-secret --caps 'roles=*' --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T05:50:22.605 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:22.606 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:22.668 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:50:22.668 INFO:teuthology.orchestra.run.smithi062.stdout: "user_id": "rgw-multisite-test-user", 2022-09-03T05:50:22.668 INFO:teuthology.orchestra.run.smithi062.stdout: "display_name": "Multisite Test User", 2022-09-03T05:50:22.668 INFO:teuthology.orchestra.run.smithi062.stdout: "email": "", 2022-09-03T05:50:22.669 INFO:teuthology.orchestra.run.smithi062.stdout: "suspended": 0, 2022-09-03T05:50:22.669 INFO:teuthology.orchestra.run.smithi062.stdout: "max_buckets": 1000, 2022-09-03T05:50:22.669 INFO:teuthology.orchestra.run.smithi062.stdout: "subusers": [], 2022-09-03T05:50:22.669 INFO:teuthology.orchestra.run.smithi062.stdout: "keys": [ 2022-09-03T05:50:22.670 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:50:22.670 INFO:teuthology.orchestra.run.smithi062.stdout: "user": "rgw-multisite-test-user", 2022-09-03T05:50:22.670 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "1KKLJ23JMA7EOB7UYR06", 2022-09-03T05:50:22.670 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "U7znIZYraXT71sIgNFzmE3prCg547BlUYong81Hj" 2022-09-03T05:50:22.671 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:50:22.671 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:50:22.671 INFO:teuthology.orchestra.run.smithi062.stdout: "swift_keys": [], 2022-09-03T05:50:22.671 INFO:teuthology.orchestra.run.smithi062.stdout: "caps": [ 2022-09-03T05:50:22.672 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:50:22.672 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "roles", 2022-09-03T05:50:22.672 INFO:teuthology.orchestra.run.smithi062.stdout: "perm": "*" 2022-09-03T05:50:22.672 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:50:22.672 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T05:50:22.673 INFO:teuthology.orchestra.run.smithi062.stdout: "op_mask": "read, write, delete", 2022-09-03T05:50:22.673 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "", 2022-09-03T05:50:22.673 INFO:teuthology.orchestra.run.smithi062.stdout: "default_storage_class": "", 2022-09-03T05:50:22.673 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_tags": [], 2022-09-03T05:50:22.674 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_quota": { 2022-09-03T05:50:22.674 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T05:50:22.674 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T05:50:22.675 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T05:50:22.675 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T05:50:22.675 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T05:50:22.676 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:50:22.676 INFO:teuthology.orchestra.run.smithi062.stdout: "user_quota": { 2022-09-03T05:50:22.676 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T05:50:22.677 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T05:50:22.677 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T05:50:22.677 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T05:50:22.677 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T05:50:22.678 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:50:22.678 INFO:teuthology.orchestra.run.smithi062.stdout: "temp_url_keys": [], 2022-09-03T05:50:22.678 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "rgw", 2022-09-03T05:50:22.679 INFO:teuthology.orchestra.run.smithi062.stdout: "mfa_ids": [] 2022-09-03T05:50:22.679 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:50:22.679 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T05:50:22.682 INFO:rgw_multi.tests:meta checkpoint 2022-09-03T05:50:22.683 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:22.683 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:22.684 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:50:22.734 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:22.735 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:22.779 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:50:22.800 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:50:22.800 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:50:22.801 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184222.662031_59.1", 2022-09-03T05:50:22.801 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:50:22.662031Z" 2022-09-03T05:50:22.801 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:50:22.801 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:50:22.802 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:50:22.802 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:50:22.802 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:50:22.802 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:50:22.803 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:50:22.803 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:50:22.803 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:50:22.803 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:50:22.804 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:50:22.804 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:50:22.804 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:50:22.804 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:50:22.806 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:50:22.807 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:22.807 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:22.807 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:50:22.862 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:22.863 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:22.899 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:22.897+0000 7f783eb2d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:22.899 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:22.898+0000 7f783eb2d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:22.915 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:22.916 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:50:22.916 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:50:22.916 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:50:22.916 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:50:22.916 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:22.917 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:22.917 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:22.917 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:50:22.917 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:22.918 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:50:22.918 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:22.918 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:22.918 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:22.919 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:22.919 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:50:22.919 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:22.919 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:22.920 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:22.920 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:22.920 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:22.920 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:22.920 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:50:22.921 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:22.921 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:22.921 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:22.921 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:22.922 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:22.922 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:22.922 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:22.922 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:22.923 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:22.923 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:22.923 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:22.923 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:50:22.924 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:22.924 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:22.924 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:22.924 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:22.924 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:22.925 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:22.925 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:22.925 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:22.925 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:22.926 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:22.926 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:22.926 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:50:22.926 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:22.926 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:22.927 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:22.927 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:22.927 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:22.927 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:22.927 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:22.928 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:22.928 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:22.928 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:22.928 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:22.929 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:22.929 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:50:22.929 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:50:22.929 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:50:22.929 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:22.930 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:50:22Z" 2022-09-03T05:50:22.930 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:22.931 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:50:27.932 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:27.933 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:27.933 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:50:27.989 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:27.990 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:28.045 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:28.043+0000 7fd8e0369600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:28.046 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:28.044+0000 7fd8e0369600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:28.068 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:28.069 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:50:28.069 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:50:28.069 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:50:28.069 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:50:28.070 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:28.070 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:28.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:28.070 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:50:28.071 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:28.071 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:50:28.071 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:28.071 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:28.072 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:28.072 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:28.072 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:50:28.072 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:28.073 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:28.073 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:28.073 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:28.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:28.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:28.074 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:50:28.074 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:28.074 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:28.075 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:28.076 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:28.076 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:28.076 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:28.076 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:28.077 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:28.077 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:28.077 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:28.077 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:28.078 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:50:28.078 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:28.078 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:28.078 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:28.079 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:28.079 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:28.079 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:28.079 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:28.080 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:28.080 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:28.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:28.080 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:28.081 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:50:28.081 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:28.081 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:28.081 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:28.082 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:28.082 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:28.082 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:28.082 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:28.083 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:28.083 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:28.083 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:28.083 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:28.084 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:28.084 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:50:28.085 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:50:28.085 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:50:28.086 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:28.086 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:50:28Z" 2022-09-03T05:50:28.087 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:28.088 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:50:33.090 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:33.090 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:33.091 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:50:33.149 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:33.150 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:33.183 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:33.182+0000 7f2c7ecba600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:33.184 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:33.183+0000 7f2c7ecba600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:33.196 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:33.197 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:50:33.197 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:50:33.198 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:50:33.198 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:50:33.199 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:33.200 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:33.200 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:33.200 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:50:33.201 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:33.201 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:50:33.202 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:33.202 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:33.202 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:33.203 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:33.203 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:50:33.203 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:33.204 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:33.204 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:33.204 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:33.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:33.205 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:33.205 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:50:33.205 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:33.205 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:33.205 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:33.206 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:33.206 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:33.206 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:33.207 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:33.207 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:33.207 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:33.207 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:33.207 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:33.208 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:50:33.208 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:33.208 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:33.208 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:33.209 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:33.209 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:33.209 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:33.209 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:33.210 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:33.210 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:33.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:33.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:33.211 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:50:33.212 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:33.213 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:33.213 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:33.214 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:33.214 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:33.215 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:33.215 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:33.216 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:33.216 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:33.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:33.217 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:33.218 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:33.218 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:50:33.219 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:50:33.219 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:50:33.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:33.220 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:50:33Z" 2022-09-03T05:50:33.220 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:33.222 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:50:38.224 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:38.224 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:38.225 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:50:38.281 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:38.281 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:38.325 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:38.323+0000 7fe0d0e63600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:38.325 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:38.324+0000 7fe0d0e63600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:38.344 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:38.345 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:50:38.346 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:50:38.346 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:50:38.347 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:50:38.348 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:38.348 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:38.348 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:38.348 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:50:38.349 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:38.349 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:50:38.349 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:38.349 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:38.358 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:38.359 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:38.359 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:50:38.359 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:38.359 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:38.360 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:38.360 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:38.360 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:38.360 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:38.360 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:50:38.361 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:38.361 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:38.361 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:38.361 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:38.362 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:38.362 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:38.362 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:38.362 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:50:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:38.364 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:38.364 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:38.364 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:38.364 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:38.365 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:38.365 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:38.365 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:38.365 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:38.365 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:38.366 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:38.366 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:50:38.366 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:38.366 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:38.367 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:38.367 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:38.367 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:38.367 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:38.367 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:38.368 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:38.368 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:38.368 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:38.368 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:38.368 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:38.369 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:50:38.369 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:50:38.369 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:50:38.369 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:38.370 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:50:38Z" 2022-09-03T05:50:38.370 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:38.371 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:50:43.372 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:43.374 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:43.374 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:50:43.432 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:43.432 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:43.487 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:43.485+0000 7effbc320600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:43.487 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:43.486+0000 7effbc320600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:43.510 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:43.512 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:50:43.512 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:50:43.513 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:50:43.513 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:50:43.513 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:43.514 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:43.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:43.514 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:50:43.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:43.515 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:50:43.516 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:43.516 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:43.516 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:43.517 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:43.517 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:50:43.517 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:43.518 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:43.518 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:43.518 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:43.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:43.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:43.519 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:50:43.519 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:43.519 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:43.520 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:43.520 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:43.520 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:43.521 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:43.521 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:43.521 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:43.521 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:43.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:43.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:43.522 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:50:43.523 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:43.523 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:43.524 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:43.524 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:43.524 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:43.525 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:43.525 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:43.526 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:43.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:43.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:43.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:43.527 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:50:43.527 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:43.527 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:43.527 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:43.527 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:43.528 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:43.528 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:43.528 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:43.528 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:43.529 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:43.529 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:43.529 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:43.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:43.530 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:50:43.530 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:50:43.530 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:50:43.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:43.531 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:50:43Z" 2022-09-03T05:50:43.531 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:43.533 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:50:48.534 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:48.537 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:48.538 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:50:48.598 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:48.599 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:48.627 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:48.626+0000 7fe53f823600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:48.628 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:48.627+0000 7fe53f823600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:48.642 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:48.643 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:50:48.644 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:50:48.644 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:50:48.645 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:50:48.646 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:48.647 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:48.647 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:48.647 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:50:48.648 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:48.648 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:50:48.649 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:48.649 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:48.650 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:48.650 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:48.651 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:50:48.651 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:48.652 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:48.652 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:48.653 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:48.653 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:48.653 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:48.654 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:50:48.654 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:48.655 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:48.655 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:48.656 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:48.656 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:48.657 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:48.657 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:48.658 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:48.658 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:48.659 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:48.659 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:48.659 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:50:48.660 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:48.660 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:48.661 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:48.661 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:48.662 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:48.662 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:48.663 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:48.663 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:48.663 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:48.664 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:48.665 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:48.665 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:50:48.666 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:48.666 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:48.667 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:48.668 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:48.668 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:48.668 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:48.668 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:48.669 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:48.669 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:48.670 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:48.670 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:48.670 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:48.671 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:50:48.671 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:50:48.671 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:50:48.671 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:48.671 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:50:48Z" 2022-09-03T05:50:48.672 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:48.673 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:50:53.676 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:53.677 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:53.677 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:50:53.736 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:53.736 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:53.787 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:53.785+0000 7fb40d789600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:53.788 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:53.786+0000 7fb40d789600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:53.805 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:53.805 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:50:53.805 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:50:53.806 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:50:53.806 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:50:53.806 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:53.806 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:53.807 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:53.807 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:50:53.807 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:53.807 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:50:53.808 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:53.808 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:53.808 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:53.808 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:53.809 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:50:53.809 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:53.809 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:53.809 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:53.810 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:53.810 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:53.810 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:53.810 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:50:53.811 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:53.811 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:53.811 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:53.811 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:53.812 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:53.812 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:53.812 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:53.812 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:53.812 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:53.813 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:53.813 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:53.813 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:50:53.813 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:53.814 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:53.814 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:53.814 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:53.814 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:53.814 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:53.815 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:53.815 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:53.815 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:53.815 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:53.816 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:53.816 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:50:53.816 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:53.816 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:53.817 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:53.817 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:53.817 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:53.817 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:53.818 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:53.818 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:53.818 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:53.818 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:53.819 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:53.819 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:53.819 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:50:53.819 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:50:53.820 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:50:53.820 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:53.820 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:50:53Z" 2022-09-03T05:50:53.820 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:53.822 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:50:58.822 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:58.823 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:50:58.823 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:50:58.881 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:50:58.882 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:50:58.915 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:58.914+0000 7f251669d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:58.916 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:50:58.915+0000 7f251669d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:50:58.928 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:50:58.928 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:50:58.929 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:50:58.929 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:50:58.929 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:50:58.929 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:50:58.930 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:58.930 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:58.930 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:50:58.930 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:58.931 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:50:58.931 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:58.931 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:58.931 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:58.932 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:58.932 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:50:58.932 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:58.932 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:58.932 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:58.933 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:58.933 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:58.933 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:58.934 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:50:58.934 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:58.935 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:58.935 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:58.935 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:58.935 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:58.936 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:58.936 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:58.936 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:58.936 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:58.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:58.937 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:58.937 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:50:58.937 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:58.937 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:58.938 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:58.938 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:58.938 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:58.938 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:58.939 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:58.939 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:58.939 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:58.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:58.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:50:58.940 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:50:58.940 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:50:58.940 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:50:58.941 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:50:58.941 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:50:58.941 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:50:58.941 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:50:58.942 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:50:58.942 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:50:58.942 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:58.942 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:50:58.942 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:50:58.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:58.943 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:50:58.943 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:50:58.944 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:50:58.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:50:58.944 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:50:58Z" 2022-09-03T05:50:58.944 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:50:58.945 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:03.947 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:03.948 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:03.949 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:04.015 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:04.015 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:04.048 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:04.046+0000 7fc9cda92600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:04.050 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:04.047+0000 7fc9cda92600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:04.066 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:04.067 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:04.067 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:04.068 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:04.068 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:04.069 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:04.069 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:04.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:04.070 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:04.070 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:04.070 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:04.070 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:04.071 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:04.071 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:04.071 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:04.072 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:04.072 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:04.072 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:04.072 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:04.073 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:04.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:04.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:04.073 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:04.073 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:04.074 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:04.074 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:04.074 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:04.074 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:04.075 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:04.075 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:04.075 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:04.075 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:04.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:04.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:04.076 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:04.076 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:04.076 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:04.077 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:04.077 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:04.077 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:04.078 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:04.078 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:04.078 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:04.078 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:04.079 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:04.079 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:04.079 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:04.080 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:04.080 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:04.080 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:04.081 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:04.081 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:04.081 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:04.082 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:04.082 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:04.082 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:04.082 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:04.083 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:04.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:04.083 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:04.083 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:04.083 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:04.084 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:04.084 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:04Z" 2022-09-03T05:51:04.084 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:04.085 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:09.087 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:09.087 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:09.088 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:09.144 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:09.145 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:09.185 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:09.183+0000 7f58d26e6600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:09.186 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:09.184+0000 7f58d26e6600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:09.198 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:09.199 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:09.199 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:09.200 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:09.200 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:09.201 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:09.201 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:09.202 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:09.202 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:09.203 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:09.203 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:09.203 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:09.204 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:09.204 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:09.205 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:09.205 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:09.206 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:09.206 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:09.207 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:09.207 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:09.207 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:09.208 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:09.208 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:09.208 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:09.208 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:09.208 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:09.209 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:09.209 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:09.209 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:09.209 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:09.210 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:09.210 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:09.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:09.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:09.211 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:09.211 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:09.211 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:09.212 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:09.212 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:09.213 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:09.213 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:09.213 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:09.213 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:09.214 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:09.215 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:09.215 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:09.215 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:09.216 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:09.216 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:09.216 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:09.216 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:09.218 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:09.218 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:09.218 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:09.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:09.219 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:09.219 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:09.219 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:09.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:09.220 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:09Z" 2022-09-03T05:51:09.220 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:09.222 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:14.223 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:14.224 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:14.224 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:14.282 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:14.283 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:14.321 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:14.319+0000 7f1f1f9c7600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:14.321 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:14.320+0000 7f1f1f9c7600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:14.342 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:14.343 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:14.343 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:14.343 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:14.343 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:14.344 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:14.344 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:14.344 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:14.344 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:14.345 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:14.345 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:14.345 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:14.345 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:14.346 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:14.346 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:14.346 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:14.346 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:14.347 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:14.347 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:14.347 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:14.347 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:14.348 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:14.348 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:14.349 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:14.349 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:14.349 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:14.349 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:14.350 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:14.350 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:14.350 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:14.350 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:14.351 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:14.351 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:14.351 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:14.351 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:14.352 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:14.352 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:14.352 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:14.352 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:14.353 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:14.353 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:14.353 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:14.353 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:14.353 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:14.354 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:14.354 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:14.354 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:14.355 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:14.355 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:14.355 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:14.355 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:14.356 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:14.356 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:14.357 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:14.357 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:14.357 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:14.357 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:14.357 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:14.358 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:14.358 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:14.358 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:14.359 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:14.359 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:14.359 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:14Z" 2022-09-03T05:51:14.359 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:14.361 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:19.362 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:19.363 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:19.363 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:19.420 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:19.420 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:19.471 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:19.470+0000 7f07d27ab600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:19.472 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:19.471+0000 7f07d27ab600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:19.484 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:19.485 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:19.485 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:19.485 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:19.486 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:19.486 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:19.486 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:19.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:19.487 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:19.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:19.487 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:19.487 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:19.488 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:19.488 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:19.488 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:19.488 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:19.488 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:19.489 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:19.489 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:19.489 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:19.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:19.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:19.490 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:19.490 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:19.490 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:19.491 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:19.491 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:19.491 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:19.491 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:19.492 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:19.492 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:19.492 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:19.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:19.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:19.493 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:19.493 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:19.493 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:19.493 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:19.494 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:19.494 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:19.494 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:19.494 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:19.494 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:19.495 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:19.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:19.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:19.495 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:19.496 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:19.496 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:19.496 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:19.497 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:19.497 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:19.498 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:19.498 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:19.499 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:19.499 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:19.500 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:19.500 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:19.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:19.501 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:19.501 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:19.501 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:19.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:19.502 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:19Z" 2022-09-03T05:51:19.502 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:19.504 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:24.506 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:24.506 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:24.507 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:24.565 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:24.566 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:24.595 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:24.593+0000 7faa34f2e600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:24.596 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:24.594+0000 7faa34f2e600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:24.607 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:24.608 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:24.608 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:24.609 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:24.609 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:24.609 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:24.610 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:24.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:24.611 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:24.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:24.612 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:24.612 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:24.612 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:24.613 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:24.613 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:24.614 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:24.614 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:24.614 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:24.615 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:24.615 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:24.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:24.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:24.616 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:24.617 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:24.617 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:24.617 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:24.617 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:24.618 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:24.618 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:24.618 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:24.618 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:24.619 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:24.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:24.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:24.619 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:24.620 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:24.620 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:24.620 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:24.620 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:24.621 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:24.621 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:24.621 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:24.622 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:24.622 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:24.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:24.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:24.623 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:24.623 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:24.623 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:24.624 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:24.624 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:24.624 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:24.625 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:24.625 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:24.625 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:24.625 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:24.626 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:24.626 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:24.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:24.627 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:24.627 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:24.627 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:24.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:24.628 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:24Z" 2022-09-03T05:51:24.628 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:24.630 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:29.631 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:29.632 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:29.632 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:29.690 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:29.690 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:29.738 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:29.736+0000 7f5e45730600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:29.738 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:29.737+0000 7f5e45730600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:29.761 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:29.761 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:29.762 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:29.762 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:29.762 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:29.762 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:29.763 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:29.763 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:29.763 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:29.763 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:29.764 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:29.764 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:29.764 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:29.765 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:29.765 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:29.765 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:29.765 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:29.766 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:29.766 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:29.766 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:29.766 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:29.767 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:29.767 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:29.767 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:29.767 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:29.768 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:29.769 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:29.769 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:29.769 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:29.769 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:29.770 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:29.770 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:29.770 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:29.770 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:29.771 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:29.771 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:29.771 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:29.772 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:29.772 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:29.772 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:29.772 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:29.773 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:29.773 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:29.773 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:29.773 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:29.774 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:29.774 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:29.774 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:29.774 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:29.775 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:29.775 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:29.775 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:29.775 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:29.776 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:29.776 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:29.776 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:29.776 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:29.777 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:29.777 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:29.777 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:29.778 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:29.778 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:29.778 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:29.778 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:29Z" 2022-09-03T05:51:29.778 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:29.781 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:34.782 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:34.782 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:34.783 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:34.840 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:34.840 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:34.873 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:34.871+0000 7f5a0f842600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:34.874 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:34.872+0000 7f5a0f842600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:34.885 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:34.886 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:34.886 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:34.886 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:34.887 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:34.887 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:34.887 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:34.887 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:34.887 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:34.888 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:34.888 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:34.888 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:34.888 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:34.889 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:34.889 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:34.889 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:34.889 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:34.890 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:34.890 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:34.891 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:34.891 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:34.892 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:34.892 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:34.893 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:34.893 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:34.893 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:34.894 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:34.894 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:34.895 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:34.895 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:34.896 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:34.896 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:34.897 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:34.897 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:34.897 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:34.898 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:34.898 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:34.899 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:34.899 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:34.900 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:34.900 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:34.901 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:34.901 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:34.901 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:34.902 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:34.902 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:34.903 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:34.903 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:34.903 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:34.904 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:34.904 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:34.905 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:34.905 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:34.905 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:34.906 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:34.906 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:34.907 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:34.907 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:34.908 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:34.908 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:34.909 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:34.909 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:34.910 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:34.910 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:34Z" 2022-09-03T05:51:34.916 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:34.918 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:39.920 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:39.921 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:39.921 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:39.978 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:39.979 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:40.032 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:40.030+0000 7fdbfe7fa600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:40.033 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:40.032+0000 7fdbfe7fa600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:40.057 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:40.057 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:40.058 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:40.058 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:40.059 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:40.059 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:40.060 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:40.060 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:40.061 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:40.061 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:40.061 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:40.061 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:40.061 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:40.062 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:40.062 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:40.062 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:40.063 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:40.063 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:40.064 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:40.064 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:40.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:40.064 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:40.065 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:40.065 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:40.065 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:40.065 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:40.066 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:40.066 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:40.066 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:40.066 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:40.067 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:40.067 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:40.067 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:40.067 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:40.068 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:40.068 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:40.068 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:40.068 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:40.068 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:40.069 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:40.069 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:40.069 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:40.069 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:40.070 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:40.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:40.070 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:40.070 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:40.071 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:40.071 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:40.071 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:40.071 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:40.071 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:40.072 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:40.072 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:40.072 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:40.072 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:40.073 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:40.073 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:40.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:40.073 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:40.074 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:40.074 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:40.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:40.074 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:40Z" 2022-09-03T05:51:40.074 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:40.076 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:45.077 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:45.078 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:45.079 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:45.136 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:45.136 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:45.179 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:45.177+0000 7faba6496600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:45.180 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:45.178+0000 7faba6496600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:45.198 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:45.198 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:45.198 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:45.199 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:45.199 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:45.199 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:45.199 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:45.200 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:45.200 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:45.200 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:45.200 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:45.200 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:45.201 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:45.201 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:45.201 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:45.201 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:45.202 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:45.202 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:45.202 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:45.202 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:45.203 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:45.203 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:45.204 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:45.204 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:45.204 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:45.205 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:45.205 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:45.206 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:45.206 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:45.207 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:45.207 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:45.208 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:45.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:45.209 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:45.209 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:45.209 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:45.209 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:45.210 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:45.210 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:45.210 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:45.210 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:45.210 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:45.211 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:45.211 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:45.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:45.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:45.212 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:45.212 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:45.212 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:45.212 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:45.212 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:45.213 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:45.213 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:45.213 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:45.214 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:45.214 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:45.214 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:45.214 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:45.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:45.215 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:45.215 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:45.215 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:45.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:45.216 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:45Z" 2022-09-03T05:51:45.216 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:45.217 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:50.219 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:50.219 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:50.220 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:50.274 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:50.275 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:50.305 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:50.304+0000 7f00583b3600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:50.306 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:50.305+0000 7f00583b3600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:50.318 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:50.318 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:50.318 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:50.319 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:50.319 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:50.319 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:50.319 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:50.320 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:50.320 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:50.320 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:50.320 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:50.320 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:50.321 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:50.321 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:50.321 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:50.321 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:50.321 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:50.322 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:50.322 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:50.323 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:50.323 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:50.323 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:50.323 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:50.323 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:50.324 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:50.324 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:50.324 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:50.324 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:50.325 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:50.325 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:50.325 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:50.325 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:50.325 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:50.326 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:50.326 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:50.326 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:50.326 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:50.327 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:50.327 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:50.327 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:50.327 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:50.327 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:50.328 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:50.328 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:50.328 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:50.328 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:50.329 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:50.329 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:50.329 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:50.329 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:50.330 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:50.330 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:50.330 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:50.331 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:50.331 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:50.331 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:50.331 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:50.332 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:50.332 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:50.332 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:50.332 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:50.332 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:50.333 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:50.333 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:50Z" 2022-09-03T05:51:50.333 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:50.334 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:51:55.335 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:55.336 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:51:55.337 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:51:55.393 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:51:55.394 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:51:55.450 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:55.449+0000 7f5f474ab600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:55.451 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:51:55.450+0000 7f5f474ab600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:51:55.475 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:51:55.475 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:51:55.476 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:51:55.476 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:51:55.476 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:51:55.477 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:51:55.477 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:55.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:55.477 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:51:55.478 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:55.478 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:51:55.478 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:55.479 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:55.479 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:55.479 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:55.480 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:51:55.480 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:55.481 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:55.481 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:55.482 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:55.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:55.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:55.483 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:51:55.484 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:55.484 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:55.484 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:55.485 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:55.485 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:55.486 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:55.486 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:55.487 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:55.487 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:55.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:55.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:55.488 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:51:55.488 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:55.488 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:55.488 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:55.488 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:55.489 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:55.489 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:55.489 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:55.489 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:55.490 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:55.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:55.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:51:55.490 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:51:55.491 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:51:55.491 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:51:55.491 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:51:55.491 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:51:55.492 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:51:55.492 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:51:55.492 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:51:55.492 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:51:55.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:55.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:51:55.493 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:51:55.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:55.494 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:51:55.494 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:51:55.494 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:51:55.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:51:55.495 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:51:55Z" 2022-09-03T05:51:55.495 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:51:55.497 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:52:00.498 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:00.499 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:00.499 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:00.556 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:00.557 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:00.592 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:00.590+0000 7fcbf6f5e600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:00.593 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:00.591+0000 7fcbf6f5e600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:00.610 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:00.610 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:00.611 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:00.611 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:00.612 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:00.612 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:00.612 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:00.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:00.613 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:00.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:00.613 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:00.613 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:00.613 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:00.614 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:00.614 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:00.614 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:00.615 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:00.615 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:00.615 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:00.615 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:00.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:00.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:00.616 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:00.616 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:00.616 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:00.617 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:00.617 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:00.617 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:00.617 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:00.618 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:00.618 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:00.618 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:00.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:00.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:00.619 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:00.619 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:00.620 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:00.620 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:00.620 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:00.621 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:00.622 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:00.622 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:00.622 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:00.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:00.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:00.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:00.623 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:00.624 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:00.624 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:00.624 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:00.624 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:00.625 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:00.625 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:00.625 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:00.625 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:00.626 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:00.626 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:00.626 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:00.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:00.627 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:00.627 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:00.627 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:00.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:00.628 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:00Z" 2022-09-03T05:52:00.628 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:00.629 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:52:05.631 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:05.632 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:05.633 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:05.692 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:05.693 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:05.726 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:05.724+0000 7fda4dc4e600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:05.727 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:05.726+0000 7fda4dc4e600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:05.745 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:05.746 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:05.746 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:05.746 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:05.746 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:05.747 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:05.747 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:05.747 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:05.747 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:05.748 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:05.748 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:05.748 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:05.748 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:05.748 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:05.749 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:05.749 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:05.749 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:05.749 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:05.750 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:05.750 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:05.750 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:05.750 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:05.750 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:05.751 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:05.751 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:05.751 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:05.752 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:05.752 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:05.752 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:05.752 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:05.753 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:05.753 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:05.753 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:05.753 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:05.753 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:05.754 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:05.754 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:05.754 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:05.754 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:05.755 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:05.755 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:05.755 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:05.755 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:05.755 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:05.756 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:05.756 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:05.756 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:05.757 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:05.757 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:05.757 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:05.757 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:05.758 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:05.758 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:05.759 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:05.759 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:05.759 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:05.759 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:05.760 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:05.760 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:05.760 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:05.760 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:05.761 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:05.761 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:05.761 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:05Z" 2022-09-03T05:52:05.761 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:05.763 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:52:10.764 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:10.765 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:10.765 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:10.822 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:10.823 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:10.871 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:10.869+0000 7f125984d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:10.872 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:10.870+0000 7f125984d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:10.889 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:10.890 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:10.890 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:10.890 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:10.890 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:10.891 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:10.891 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:10.891 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:10.892 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:10.892 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:10.892 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:10.892 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:10.893 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:10.893 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:10.893 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:10.893 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:10.894 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:10.894 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:10.894 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:10.894 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:10.895 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:10.895 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:10.895 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:10.895 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:10.895 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:10.896 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:10.896 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:10.896 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:10.896 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:10.897 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:10.897 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:10.897 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:10.897 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:10.897 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:10.898 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:10.898 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:10.898 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:10.898 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:10.899 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:10.899 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:10.899 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:10.899 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:10.899 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:10.900 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:10.900 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:10.900 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:10.900 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:10.900 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:10.901 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:10.901 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:10.901 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:10.901 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:10.902 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:10.902 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:10.902 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:10.902 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:10.902 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:10.903 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:10.903 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:10.903 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:10.903 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:10.903 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:10.904 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:10.904 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:10Z" 2022-09-03T05:52:10.904 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:10.905 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184222.662031_59.1 target= 2022-09-03T05:52:15.907 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:15.908 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:15.909 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:15.967 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:15.967 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:15.999 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:15.997+0000 7f5629966600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.001 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:15.998+0000 7f5629966600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.012 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:16.013 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:16.013 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:16.014 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:16.014 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:16.015 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:16.015 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.016 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:16.017 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.017 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:16.017 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.017 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.018 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184222.662031_59.1", 2022-09-03T05:52:16.018 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.019 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:16.019 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.020 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:50:22.662031Z", 2022-09-03T05:52:16.020 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.020 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.021 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.021 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:16.021 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.022 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.022 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.022 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.022 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.022 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.023 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.023 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.023 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.024 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:16.024 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.025 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.025 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.025 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.026 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.026 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.026 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.027 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.027 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.028 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:16.028 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.028 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.029 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.029 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.029 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.029 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.030 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.030 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.030 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.030 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.031 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:16.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.031 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:16.031 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:16.031 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:16.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.032 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:16Z" 2022-09-03T05:52:16.032 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:16.033 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:52:16.034 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:52:16.035 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.035 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.036 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:16.098 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:16.099 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:16.142 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:16.140+0000 7fc067d41600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.143 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:16.141+0000 7fc067d41600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.160 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:52:16.160 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:52:16.161 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:52:16.161 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:52:16.161 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:52:16.161 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:16.162 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.162 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.162 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:52:16.163 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.163 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:52:16.163 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:16.163 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:16.164 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184222.662031_59.1", 2022-09-03T05:52:16.164 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:16.164 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:52:16.164 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:16.164 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:50:22.662031Z", 2022-09-03T05:52:16.165 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.165 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.165 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.165 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.166 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:52:16.166 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:16.166 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:16.166 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:16.169 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:16.170 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:16.170 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:16.170 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.170 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.171 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.171 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.171 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.171 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:52:16.171 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:16.172 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:16.172 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:16.172 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:16.172 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:16.172 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:16.173 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.173 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.173 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.174 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.174 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.174 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:52:16.175 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:16.175 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:16.175 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:16.175 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:16.176 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:16.176 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:16.176 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.176 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.176 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.177 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.177 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:52:16.177 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.177 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:52:16.177 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:52:16.178 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:52:16.178 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.178 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:52:16Z" 2022-09-03T05:52:16.178 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:52:16.180 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:52:16.180 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:52:16.180 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.181 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.181 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:16.233 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:16.233 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:16.261 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:16.259+0000 7f57500c6600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.262 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:16.260+0000 7f57500c6600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.280 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:16.281 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:16.281 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:16.281 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:16.282 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:16.282 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:16.282 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.283 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:16.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.283 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:16.283 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.284 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.284 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184222.662031_59.1", 2022-09-03T05:52:16.284 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.284 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:16.284 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.285 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:50:22.662031Z", 2022-09-03T05:52:16.285 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.285 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.286 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:16.286 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.286 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.287 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.287 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.287 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.287 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.287 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.288 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.288 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.289 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:16.289 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.289 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.289 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.289 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.290 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.290 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.290 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.290 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.291 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.291 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.291 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:16.292 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.292 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.292 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.292 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.292 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.293 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.293 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.293 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.293 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.294 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.294 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:16.294 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.294 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:16.295 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:16.295 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:16.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.295 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:16Z" 2022-09-03T05:52:16.295 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:16.297 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:52:16.297 INFO:rgw_multi.tests:meta checkpoint 2022-09-03T05:52:16.298 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.298 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.298 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:16.350 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:16.351 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:16.391 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:52:16.406 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:52:16.407 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.407 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184222.662031_59.1", 2022-09-03T05:52:16.407 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:50:22.662031Z" 2022-09-03T05:52:16.408 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.408 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.408 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:16.409 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:16.409 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.409 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.409 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:16.409 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:16.410 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.410 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.410 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:16.411 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:16.411 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.411 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:52:16.412 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:52:16.413 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.413 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.414 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:16.463 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:16.464 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:16.514 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:16.512+0000 7fa67a5b8600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.515 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:16.513+0000 7fa67a5b8600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.538 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:16.538 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:16.539 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:16.539 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:16.539 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:16.540 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:16.540 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.541 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:16.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.542 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:16.542 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.543 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.543 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184222.662031_59.1", 2022-09-03T05:52:16.544 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.544 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:16.544 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.545 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:50:22.662031Z", 2022-09-03T05:52:16.545 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.545 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.546 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:16.546 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.546 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.547 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.547 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.547 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.547 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.548 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.548 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.548 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.549 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:16.549 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.549 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.549 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.549 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.550 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.550 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.550 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.550 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.551 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.551 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:16.552 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.552 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.552 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.552 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.552 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.553 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.553 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.553 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.553 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.554 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.554 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:16.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.555 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:16.555 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:16.555 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:16.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.556 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:16Z" 2022-09-03T05:52:16.556 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:16.557 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:52:16.558 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:52:16.558 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.558 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.559 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:16.610 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:16.610 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:16.660 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:16.658+0000 7f9c92dd3600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.727 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:16.659+0000 7f9c92dd3600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.728 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:52:16.729 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:52:16.729 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:52:16.729 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:52:16.730 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:52:16.730 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:16.730 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.731 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.731 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:52:16.731 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.731 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:52:16.732 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:16.732 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:16.732 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184222.662031_59.1", 2022-09-03T05:52:16.732 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:16.732 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:52:16.733 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:16.733 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:50:22.662031Z", 2022-09-03T05:52:16.733 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.733 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.734 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.735 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.735 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:52:16.735 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:16.735 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:16.736 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:16.736 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:16.736 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:16.737 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:16.737 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.737 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.737 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.737 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.738 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.738 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:52:16.738 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:16.739 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:16.739 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:16.739 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:16.739 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:16.739 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:16.740 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.740 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.741 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.741 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.741 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:16.742 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:52:16.742 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:16.743 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:16.744 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:16.745 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:16.746 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:16.747 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:16.748 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.748 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:16.748 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.749 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:16.749 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:52:16.750 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.751 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:52:16.751 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:52:16.752 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:52:16.752 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:16.753 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:52:16Z" 2022-09-03T05:52:16.753 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:52:16.754 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:52:16.755 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:52:16.755 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.756 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:16.757 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:16.808 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:16.808 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:16.846 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:16.844+0000 7f3537a0d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.847 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:16.845+0000 7f3537a0d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:16.857 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:16.857 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:16.858 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:16.858 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:16.858 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:16.858 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:16.859 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.859 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.859 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:16.859 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.860 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:16.860 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.860 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.860 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184222.662031_59.1", 2022-09-03T05:52:16.861 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.861 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:16.861 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.861 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:50:22.662031Z", 2022-09-03T05:52:16.862 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.862 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.862 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.862 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.863 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:16.863 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.863 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.863 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.864 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.864 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.864 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.864 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.864 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.865 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.865 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.865 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.865 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:16.866 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.869 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.869 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.869 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.869 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.870 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.870 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.870 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.870 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.871 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.871 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:16.871 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:16.871 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:16.871 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:16.872 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:16.872 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:16.872 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:16.872 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:16.873 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:16.873 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:16.873 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.873 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:16.873 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:16.874 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.874 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:16.874 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:16.874 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:16.875 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:16.875 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:16Z" 2022-09-03T05:52:16.875 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:16.876 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:52:16.877 INFO:tasks.rgw_multisite_tests:running rgw multisite tests on '/home/teuthworker/src/github.com_cbodley_ceph_e20bde8244d2a27c6431acd07e9967aaa8f04f01/qa/../src/test/rgw/rgw_multi' with args=['tests.py', 'tests_ps.py'] 2022-09-03T05:52:16.933 DEBUG:boto:Using access key provided by client. 2022-09-03T05:52:16.936 DEBUG:boto:Using secret key provided by client. 2022-09-03T05:52:16.937 DEBUG:boto:Using access key provided by client. 2022-09-03T05:52:16.939 DEBUG:boto:Using secret key provided by client. 2022-09-03T05:52:16.940 DEBUG:boto:Using access key provided by client. 2022-09-03T05:52:16.940 DEBUG:boto:Using secret key provided by client. 2022-09-03T05:52:16.941 DEBUG:boto:Using access key provided by client. 2022-09-03T05:52:16.942 DEBUG:boto:Using secret key provided by client. 2022-09-03T05:52:16.943 DEBUG:boto:Using access key provided by client. 2022-09-03T05:52:16.943 DEBUG:boto:Using secret key provided by client. 2022-09-03T05:52:16.945 DEBUG:boto:Using access key provided by client. 2022-09-03T05:52:16.946 DEBUG:boto:Using secret key provided by client. 2022-09-03T05:52:16.947 DEBUG:boto:Using access key provided by client. 2022-09-03T05:52:16.948 DEBUG:boto:Using secret key provided by client. 2022-09-03T05:52:16.949 DEBUG:boto:Using access key provided by client. 2022-09-03T05:52:16.949 DEBUG:boto:Using secret key provided by client. 2022-09-03T05:52:16.950 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-1 2022-09-03T05:52:16.951 DEBUG:boto:path=/avysmw-1/ 2022-09-03T05:52:16.951 DEBUG:boto:auth_path=/avysmw-1/ 2022-09-03T05:52:16.952 DEBUG:boto:Method: PUT 2022-09-03T05:52:16.952 DEBUG:boto:Path: /avysmw-1/ 2022-09-03T05:52:16.953 DEBUG:boto:Data: 2022-09-03T05:52:16.953 DEBUG:boto:Headers: {} 2022-09-03T05:52:16.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:16.954 DEBUG:boto:Port: 8000 2022-09-03T05:52:16.955 DEBUG:boto:Params: {} 2022-09-03T05:52:16.955 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T05:52:16.956 DEBUG:boto:Token: None 2022-09-03T05:52:16.957 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:16 GMT /avysmw-1/ 2022-09-03T05:52:16.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pEmSfKLhDgfMn+8JYFKIGrDurVQ= 2022-09-03T05:52:16.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pEmSfKLhDgfMn+8JYFKIGrDurVQ=', 'Content-Length': '0'} 2022-09-03T05:52:19.229 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000fb0214889f1fde35-006312eb90-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:19.230 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-2 2022-09-03T05:52:19.231 DEBUG:boto:path=/avysmw-2/ 2022-09-03T05:52:19.231 DEBUG:boto:auth_path=/avysmw-2/ 2022-09-03T05:52:19.232 DEBUG:boto:Method: PUT 2022-09-03T05:52:19.232 DEBUG:boto:Path: /avysmw-2/ 2022-09-03T05:52:19.233 DEBUG:boto:Data: 2022-09-03T05:52:19.233 DEBUG:boto:Headers: {} 2022-09-03T05:52:19.234 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:19.234 DEBUG:boto:Port: 8000 2022-09-03T05:52:19.235 DEBUG:boto:Params: {} 2022-09-03T05:52:19.235 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T05:52:19.236 DEBUG:boto:Token: None 2022-09-03T05:52:19.237 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:19 GMT /avysmw-2/ 2022-09-03T05:52:19.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LzY23mZodP4dkpIzwSj87JuNnHM= 2022-09-03T05:52:19.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LzY23mZodP4dkpIzwSj87JuNnHM=', 'Content-Length': '0'} 2022-09-03T05:52:22.320 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000002098c08496ac1888-006312eb93-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:22.321 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-3 2022-09-03T05:52:22.322 DEBUG:boto:path=/avysmw-3/ 2022-09-03T05:52:22.323 DEBUG:boto:auth_path=/avysmw-3/ 2022-09-03T05:52:22.323 DEBUG:boto:Method: PUT 2022-09-03T05:52:22.324 DEBUG:boto:Path: /avysmw-3/ 2022-09-03T05:52:22.325 DEBUG:boto:Data: 2022-09-03T05:52:22.325 DEBUG:boto:Headers: {} 2022-09-03T05:52:22.326 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:22.326 DEBUG:boto:Port: 8001 2022-09-03T05:52:22.327 DEBUG:boto:Params: {} 2022-09-03T05:52:22.327 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T05:52:22.328 DEBUG:boto:Token: None 2022-09-03T05:52:22.328 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:22 GMT /avysmw-3/ 2022-09-03T05:52:22.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fp2v7SfUt6WdtOUCfcObrwy0kwA= 2022-09-03T05:52:22.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fp2v7SfUt6WdtOUCfcObrwy0kwA=', 'Content-Length': '0'} 2022-09-03T05:52:22.370 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000001e6090e93211c0ce-006312eb96-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:22.371 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:22.372 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:22.372 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:22.426 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:22.426 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:22.469 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:52:22.491 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:52:22.492 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:22.492 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184342.357275_73.1", 2022-09-03T05:52:22.492 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:22.357275Z" 2022-09-03T05:52:22.492 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:22.493 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:22.493 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:22.493 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:22.493 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:22.494 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:22.494 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:22.494 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:22.495 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:22.495 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:22.495 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:22.496 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:22.496 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:22.496 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:52:22.498 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184342.357275_73.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:22.499 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:52:22.500 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:22.501 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:22.502 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:22.561 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:22.562 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:22.612 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:22.610+0000 7f2572e0d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:22.612 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:22.611+0000 7f2572e0d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:22.636 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:22.637 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:22.638 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:22.638 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:22.639 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:22.639 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:22.640 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:22.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:22.641 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:22.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:22.642 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:22.642 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:22.642 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:22.643 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184339.219295_62.1", 2022-09-03T05:52:22.643 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:22.644 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:22.644 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:22.645 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.219295Z", 2022-09-03T05:52:22.645 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:22.645 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:22.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:22.646 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:22.646 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:22.646 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:22.647 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:22.647 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:22.647 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:22.648 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:22.648 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:22.648 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:22.648 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:22.649 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:22.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:22.649 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:22.650 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:22.650 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:22.650 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:22.650 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:22.651 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:22.651 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:22.651 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:22.651 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:22.652 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:22.652 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:22.652 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:22.653 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:22.653 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:22.653 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:22.654 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:22.654 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:22.654 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:22.654 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:22.655 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:22.655 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:22.655 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:22.656 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:22.656 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:22.656 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:22.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:22.657 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:22.657 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:22.657 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:22.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:22.658 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:22Z" 2022-09-03T05:52:22.658 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:22.659 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184339.219295_62.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:19.219295Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:22Z" } 2022-09-03T05:52:22.660 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184339.219295_62.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:19.219295Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:22.661 DEBUG:rgw_multi.tests:log_status={0: '1_1662184342.357275_73.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:22.661 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184339.219295_62.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:22.661 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184342.357275_73.1 target=1_1662184339.219295_62.1 2022-09-03T05:52:27.663 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:27.664 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:27.665 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:27.722 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:27.723 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:27.762 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:27.760+0000 7f7a8947c600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:27.763 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:27.762+0000 7f7a8947c600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:27.779 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:27.780 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:27.780 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:27.781 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:27.781 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:27.781 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:27.781 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:27.782 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:27.782 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:27.782 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:27.782 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:27.783 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:27.783 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:27.784 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184342.357275_73.1", 2022-09-03T05:52:27.784 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:27.784 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:27.785 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:27.785 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.357275Z", 2022-09-03T05:52:27.785 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:27.785 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:27.786 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:27.786 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:27.786 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:27.787 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:27.787 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:27.787 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:27.787 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:27.788 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:27.788 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:27.788 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:27.788 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:27.789 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:27.789 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:27.789 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:27.789 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:27.790 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:27.790 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:27.790 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:27.790 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:27.791 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:27.791 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:27.791 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:27.791 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:27.792 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:27.792 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:27.792 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:27.793 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:27.793 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:27.793 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:27.794 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:27.794 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:27.794 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:27.794 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:27.795 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:27.795 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:27.795 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:27.795 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:27.796 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:27.796 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:27.796 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:27.796 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:27.797 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:27.797 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:27.797 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:27Z" 2022-09-03T05:52:27.797 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:27.799 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184342.357275_73.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:22.357275Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:27Z" } 2022-09-03T05:52:27.799 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184342.357275_73.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:22.357275Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:27.800 DEBUG:rgw_multi.tests:log_status={0: '1_1662184342.357275_73.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:27.800 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184342.357275_73.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:27.801 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:52:27.801 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:52:27.802 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:27.802 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:27.803 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:27.862 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:27.862 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:27.911 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:27.909+0000 7f61c5aff600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:27.912 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:27.910+0000 7f61c5aff600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:27.927 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:52:27.928 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:52:27.928 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:52:27.929 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:52:27.929 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:52:27.929 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:27.929 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:27.930 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:27.930 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:52:27.930 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:27.931 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:52:27.931 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:27.931 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:27.932 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184342.357275_73.1", 2022-09-03T05:52:27.932 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:27.932 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:52:27.933 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:27.933 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:22.357275Z", 2022-09-03T05:52:27.933 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:27.933 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:27.933 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:27.934 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:27.934 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:52:27.934 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:27.934 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:27.935 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:27.935 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:27.935 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:27.935 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:27.936 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:27.936 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:27.936 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:27.936 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:27.937 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:27.937 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:52:27.937 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:27.937 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:27.938 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:27.938 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:27.938 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:27.938 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:27.939 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:27.939 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:27.939 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:27.939 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:27.940 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:27.940 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:52:27.940 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:27.941 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:27.941 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:27.941 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:27.942 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:27.942 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:27.942 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:27.942 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:27.943 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:27.943 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:27.943 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:52:27.943 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:27.944 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:52:27.944 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:52:27.944 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:52:27.944 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:27.945 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:52:27Z" 2022-09-03T05:52:27.945 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:52:27.946 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184342.357275_73.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:22.357275Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:27Z" } 2022-09-03T05:52:27.947 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184342.357275_73.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:22.357275Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:27.947 DEBUG:rgw_multi.tests:log_status={0: '1_1662184342.357275_73.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:27.948 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184342.357275_73.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:27.948 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:52:27.949 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:52:27.949 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:27.950 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:27.951 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:28.000 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:28.001 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:28.045 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:28.043+0000 7f056185b600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:28.045 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:28.044+0000 7f056185b600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:28.064 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:28.064 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:28.064 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:28.065 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:28.065 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:28.065 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:28.065 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.066 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:28.066 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.067 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:28.067 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.067 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.067 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184342.357275_73.1", 2022-09-03T05:52:28.068 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.068 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:28.069 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.069 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.357275Z", 2022-09-03T05:52:28.069 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.069 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.070 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.070 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:28.071 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.071 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.071 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:28.071 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.072 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:28.072 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.073 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.073 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.073 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.074 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.074 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:28.075 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.075 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.075 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:28.075 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.076 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:28.076 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.076 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.076 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.077 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.077 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.077 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.077 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:28.078 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.078 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.078 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:28.079 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.079 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:28.079 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.079 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.080 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.080 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.080 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.080 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:28.081 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.081 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:28.081 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:28.082 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:28.082 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.082 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:28Z" 2022-09-03T05:52:28.083 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:28.083 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184342.357275_73.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:22.357275Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:28Z" } 2022-09-03T05:52:28.084 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184342.357275_73.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:22.357275Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:28.084 DEBUG:rgw_multi.tests:log_status={0: '1_1662184342.357275_73.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:28.085 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184342.357275_73.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:28.085 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:52:28.086 DEBUG:boto:path=/avysmw-1/ 2022-09-03T05:52:28.087 DEBUG:boto:auth_path=/avysmw-1/ 2022-09-03T05:52:28.087 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.088 DEBUG:boto:Path: /avysmw-1/ 2022-09-03T05:52:28.088 DEBUG:boto:Data: 2022-09-03T05:52:28.089 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.089 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.090 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.091 DEBUG:boto:Params: {} 2022-09-03T05:52:28.091 DEBUG:boto:Token: None 2022-09-03T05:52:28.092 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-1/ 2022-09-03T05:52:28.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4F5JAwlsDBzlRn4I0XXB7ta3TRQ= 2022-09-03T05:52:28.093 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4F5JAwlsDBzlRn4I0XXB7ta3TRQ=', 'Content-Length': '0'} 2022-09-03T05:52:28.096 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009de9cbd8c8e7653f-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.097 DEBUG:boto:path=/avysmw-2/ 2022-09-03T05:52:28.097 DEBUG:boto:auth_path=/avysmw-2/ 2022-09-03T05:52:28.098 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.098 DEBUG:boto:Path: /avysmw-2/ 2022-09-03T05:52:28.099 DEBUG:boto:Data: 2022-09-03T05:52:28.100 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.101 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.101 DEBUG:boto:Params: {} 2022-09-03T05:52:28.102 DEBUG:boto:Token: None 2022-09-03T05:52:28.102 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-2/ 2022-09-03T05:52:28.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UFJheFx2iThSN8kgtkiVxK7Av58= 2022-09-03T05:52:28.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UFJheFx2iThSN8kgtkiVxK7Av58=', 'Content-Length': '0'} 2022-09-03T05:52:28.106 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000042761fba2862a488-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.107 DEBUG:boto:path=/avysmw-3/ 2022-09-03T05:52:28.108 DEBUG:boto:auth_path=/avysmw-3/ 2022-09-03T05:52:28.108 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.109 DEBUG:boto:Path: /avysmw-3/ 2022-09-03T05:52:28.109 DEBUG:boto:Data: 2022-09-03T05:52:28.110 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.111 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.111 DEBUG:boto:Params: {} 2022-09-03T05:52:28.112 DEBUG:boto:Token: None 2022-09-03T05:52:28.113 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-3/ 2022-09-03T05:52:28.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fPX3axYoKxL/ZiwArBCmqO0U1hc= 2022-09-03T05:52:28.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fPX3axYoKxL/ZiwArBCmqO0U1hc=', 'Content-Length': '0'} 2022-09-03T05:52:28.116 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005defa067465abb5e-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.117 DEBUG:boto:path=/avysmw-1/ 2022-09-03T05:52:28.118 DEBUG:boto:auth_path=/avysmw-1/ 2022-09-03T05:52:28.118 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.119 DEBUG:boto:Path: /avysmw-1/ 2022-09-03T05:52:28.119 DEBUG:boto:Data: 2022-09-03T05:52:28.120 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.120 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:28.121 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.122 DEBUG:boto:Params: {} 2022-09-03T05:52:28.122 DEBUG:boto:Token: None 2022-09-03T05:52:28.123 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-1/ 2022-09-03T05:52:28.123 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4F5JAwlsDBzlRn4I0XXB7ta3TRQ= 2022-09-03T05:52:28.124 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4F5JAwlsDBzlRn4I0XXB7ta3TRQ=', 'Content-Length': '0'} 2022-09-03T05:52:28.127 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000028c3ce6c7b3bddf9-006312eb9c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.127 DEBUG:boto:path=/avysmw-2/ 2022-09-03T05:52:28.128 DEBUG:boto:auth_path=/avysmw-2/ 2022-09-03T05:52:28.128 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.129 DEBUG:boto:Path: /avysmw-2/ 2022-09-03T05:52:28.130 DEBUG:boto:Data: 2022-09-03T05:52:28.130 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.131 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:28.131 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.132 DEBUG:boto:Params: {} 2022-09-03T05:52:28.132 DEBUG:boto:Token: None 2022-09-03T05:52:28.133 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-2/ 2022-09-03T05:52:28.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UFJheFx2iThSN8kgtkiVxK7Av58= 2022-09-03T05:52:28.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UFJheFx2iThSN8kgtkiVxK7Av58=', 'Content-Length': '0'} 2022-09-03T05:52:28.136 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000068bb5856245d3e87-006312eb9c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.137 DEBUG:boto:path=/avysmw-3/ 2022-09-03T05:52:28.138 DEBUG:boto:auth_path=/avysmw-3/ 2022-09-03T05:52:28.138 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.139 DEBUG:boto:Path: /avysmw-3/ 2022-09-03T05:52:28.139 DEBUG:boto:Data: 2022-09-03T05:52:28.140 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.140 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:28.141 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.141 DEBUG:boto:Params: {} 2022-09-03T05:52:28.142 DEBUG:boto:Token: None 2022-09-03T05:52:28.142 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-3/ 2022-09-03T05:52:28.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fPX3axYoKxL/ZiwArBCmqO0U1hc= 2022-09-03T05:52:28.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fPX3axYoKxL/ZiwArBCmqO0U1hc=', 'Content-Length': '0'} 2022-09-03T05:52:28.146 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000084c8a15a8c924767-006312eb9c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.146 DEBUG:boto:path=/avysmw-1/ 2022-09-03T05:52:28.147 DEBUG:boto:auth_path=/avysmw-1/ 2022-09-03T05:52:28.147 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.148 DEBUG:boto:Path: /avysmw-1/ 2022-09-03T05:52:28.148 DEBUG:boto:Data: 2022-09-03T05:52:28.149 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:28.150 DEBUG:boto:Port: 8001 2022-09-03T05:52:28.151 DEBUG:boto:Params: {} 2022-09-03T05:52:28.151 DEBUG:boto:Token: None 2022-09-03T05:52:28.152 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-1/ 2022-09-03T05:52:28.152 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4F5JAwlsDBzlRn4I0XXB7ta3TRQ= 2022-09-03T05:52:28.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4F5JAwlsDBzlRn4I0XXB7ta3TRQ=', 'Content-Length': '0'} 2022-09-03T05:52:28.156 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000268cc32b2c901644-006312eb9c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.156 DEBUG:boto:path=/avysmw-2/ 2022-09-03T05:52:28.157 DEBUG:boto:auth_path=/avysmw-2/ 2022-09-03T05:52:28.157 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.158 DEBUG:boto:Path: /avysmw-2/ 2022-09-03T05:52:28.158 DEBUG:boto:Data: 2022-09-03T05:52:28.159 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.160 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:28.160 DEBUG:boto:Port: 8001 2022-09-03T05:52:28.161 DEBUG:boto:Params: {} 2022-09-03T05:52:28.162 DEBUG:boto:Token: None 2022-09-03T05:52:28.163 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-2/ 2022-09-03T05:52:28.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UFJheFx2iThSN8kgtkiVxK7Av58= 2022-09-03T05:52:28.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UFJheFx2iThSN8kgtkiVxK7Av58=', 'Content-Length': '0'} 2022-09-03T05:52:28.166 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001cabe5283ff45644-006312eb9c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.167 DEBUG:boto:path=/avysmw-3/ 2022-09-03T05:52:28.167 DEBUG:boto:auth_path=/avysmw-3/ 2022-09-03T05:52:28.168 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.169 DEBUG:boto:Path: /avysmw-3/ 2022-09-03T05:52:28.169 DEBUG:boto:Data: 2022-09-03T05:52:28.170 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.170 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:28.171 DEBUG:boto:Port: 8001 2022-09-03T05:52:28.171 DEBUG:boto:Params: {} 2022-09-03T05:52:28.172 DEBUG:boto:Token: None 2022-09-03T05:52:28.172 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-3/ 2022-09-03T05:52:28.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fPX3axYoKxL/ZiwArBCmqO0U1hc= 2022-09-03T05:52:28.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fPX3axYoKxL/ZiwArBCmqO0U1hc=', 'Content-Length': '0'} 2022-09-03T05:52:28.176 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003562949ee08d0d04-006312eb9c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.177 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_create ... ok 2022-09-03T05:52:28.178 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-4 2022-09-03T05:52:28.179 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:28.179 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:28.180 DEBUG:boto:Method: PUT 2022-09-03T05:52:28.180 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:28.181 DEBUG:boto:Data: 2022-09-03T05:52:28.181 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.182 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.183 DEBUG:boto:Params: {} 2022-09-03T05:52:28.183 DEBUG:boto:Token: None 2022-09-03T05:52:28.184 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:28 GMT /avysmw-4/ 2022-09-03T05:52:28.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+AJAzgFKZuCI56z9T7admZOslS8= 2022-09-03T05:52:28.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+AJAzgFKZuCI56z9T7admZOslS8=', 'Content-Length': '0'} 2022-09-03T05:52:28.209 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000a69059ad03a02b0c-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.210 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-5 2022-09-03T05:52:28.211 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:28.211 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:28.212 DEBUG:boto:Method: PUT 2022-09-03T05:52:28.212 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:28.213 DEBUG:boto:Data: 2022-09-03T05:52:28.213 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.214 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:28.214 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.215 DEBUG:boto:Params: {} 2022-09-03T05:52:28.215 DEBUG:boto:Token: None 2022-09-03T05:52:28.216 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:28 GMT /avysmw-5/ 2022-09-03T05:52:28.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fe+p5uZTvmqoxEcbpk1UHAwQOuA= 2022-09-03T05:52:28.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fe+p5uZTvmqoxEcbpk1UHAwQOuA=', 'Content-Length': '0'} 2022-09-03T05:52:28.241 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000006987c7cc7c7f1b1a-006312eb9c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.242 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-6 2022-09-03T05:52:28.242 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:28.243 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:28.243 DEBUG:boto:Method: PUT 2022-09-03T05:52:28.244 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:28.244 DEBUG:boto:Data: 2022-09-03T05:52:28.245 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.245 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:28.246 DEBUG:boto:Port: 8001 2022-09-03T05:52:28.246 DEBUG:boto:Params: {} 2022-09-03T05:52:28.247 DEBUG:boto:Token: None 2022-09-03T05:52:28.247 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:28 GMT /avysmw-6/ 2022-09-03T05:52:28.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LJ8DxavoChG2Y9jdlANaMTNJjUk= 2022-09-03T05:52:28.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LJ8DxavoChG2Y9jdlANaMTNJjUk=', 'Content-Length': '0'} 2022-09-03T05:52:28.272 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000006ba8b6537413254f-006312eb9c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.273 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:28.274 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:28.275 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:28.337 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:28.338 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:28.387 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:52:28.410 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:52:28.410 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:28.410 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184348.261653_85.1", 2022-09-03T05:52:28.411 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:28.261653Z" 2022-09-03T05:52:28.411 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:28.411 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:28.412 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:28.412 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:28.412 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:28.412 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:28.413 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:28.413 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:28.413 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:28.413 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:28.414 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:28.414 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:28.414 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:28.414 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:52:28.416 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:28.416 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:52:28.417 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:28.417 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:28.418 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:28.467 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:28.468 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:28.508 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:28.506+0000 7f3293b9e600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:28.508 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:28.507+0000 7f3293b9e600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:28.532 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:28.532 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:28.533 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:28.533 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:28.533 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:28.533 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:28.534 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.534 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:28.535 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.535 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:28.535 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.535 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.536 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184348.261653_85.1", 2022-09-03T05:52:28.536 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.536 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:28.536 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.537 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.261653Z", 2022-09-03T05:52:28.537 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.537 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.538 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:28.538 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.539 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.539 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:28.539 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.539 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:28.540 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.540 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.540 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.540 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.541 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:28.542 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.542 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.542 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:28.542 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.543 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:28.543 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.543 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.544 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.544 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.545 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:28.545 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.545 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.545 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:28.546 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.546 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:28.546 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.546 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.547 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.547 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.547 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.547 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:28.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.548 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:28.548 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:28.549 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:28.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.549 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:28Z" 2022-09-03T05:52:28.550 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:28.551 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184348.261653_85.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:28.261653Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:28Z" } 2022-09-03T05:52:28.552 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184348.261653_85.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:28.261653Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:28.552 DEBUG:rgw_multi.tests:log_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:28.553 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:28.553 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:52:28.554 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:52:28.554 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:28.555 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:28.556 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:28.607 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:28.607 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:28.643 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:28.641+0000 7f5ebb321600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:28.644 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:28.642+0000 7f5ebb321600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:28.655 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:52:28.655 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:52:28.656 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:52:28.656 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:52:28.656 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:52:28.656 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:28.657 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:28.657 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:28.657 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:52:28.657 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:28.658 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:52:28.658 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:28.658 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:28.658 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184348.261653_85.1", 2022-09-03T05:52:28.660 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:28.660 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:52:28.661 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:28.661 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.261653Z", 2022-09-03T05:52:28.662 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:28.663 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:28.663 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:28.664 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:28.664 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:52:28.665 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:28.665 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:28.667 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:28.667 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:28.668 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:28.668 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:28.669 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.669 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:28.670 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:28.670 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:28.670 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:28.671 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:52:28.671 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:28.672 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:28.672 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:28.673 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:28.673 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:28.674 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:28.674 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.675 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:28.675 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:28.675 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:28.676 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:28.676 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:52:28.677 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:28.677 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:28.677 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:28.677 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:28.678 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:28.678 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:28.678 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.679 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:28.679 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:28.679 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:28.680 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:52:28.680 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:28.680 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:52:28.680 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:52:28.681 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:52:28.681 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:28.681 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:52:28Z" 2022-09-03T05:52:28.681 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:52:28.683 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184348.261653_85.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:28.261653Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:28Z" } 2022-09-03T05:52:28.683 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184348.261653_85.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:28.261653Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:28.684 DEBUG:rgw_multi.tests:log_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:28.685 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:28.685 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:52:28.685 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:52:28.686 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:28.687 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:28.687 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:28.736 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:28.737 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:28.787 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:28.785+0000 7fbcfca9d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:28.787 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:28.786+0000 7fbcfca9d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:28.807 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:28.807 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:28.807 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:28.808 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:28.808 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:28.808 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:28.808 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.809 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.809 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:28.809 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.809 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:28.810 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.810 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.810 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184348.261653_85.1", 2022-09-03T05:52:28.810 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.811 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:28.811 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.811 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.261653Z", 2022-09-03T05:52:28.811 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.812 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.812 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.812 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.812 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:28.813 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.813 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.813 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:28.813 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.814 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:28.814 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.814 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.814 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.815 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.815 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.815 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.815 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:28.816 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.816 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.816 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:28.816 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.817 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:28.817 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.817 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.817 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.818 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.818 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.818 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:28.818 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:28.819 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:28.819 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:28.819 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:28.819 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:28.820 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:28.820 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:28.820 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:28.820 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:28.821 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.821 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:28.821 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:28.821 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.822 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:28.822 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:28.822 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:28.822 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:28.823 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:28Z" 2022-09-03T05:52:28.823 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:28.824 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184348.261653_85.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:28.261653Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:28Z" } 2022-09-03T05:52:28.825 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184348.261653_85.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:28.261653Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:28.825 DEBUG:rgw_multi.tests:log_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:28.826 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:28.826 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:52:28.827 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:28.827 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:28.828 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.828 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:28.829 DEBUG:boto:Data: 2022-09-03T05:52:28.829 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.830 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.831 DEBUG:boto:Params: {} 2022-09-03T05:52:28.831 DEBUG:boto:Token: None 2022-09-03T05:52:28.832 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-4/ 2022-09-03T05:52:28.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jCyh6hUvCGpowvifRYkO9e+kHJ8= 2022-09-03T05:52:28.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jCyh6hUvCGpowvifRYkO9e+kHJ8=', 'Content-Length': '0'} 2022-09-03T05:52:28.836 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a0a7bd3bc654d7ad-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.836 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:28.837 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:28.837 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.838 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:28.838 DEBUG:boto:Data: 2022-09-03T05:52:28.839 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.839 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.840 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.840 DEBUG:boto:Params: {} 2022-09-03T05:52:28.841 DEBUG:boto:Token: None 2022-09-03T05:52:28.842 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-5/ 2022-09-03T05:52:28.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EsQHHArzbq46nsFkxp1mvz0Y20U= 2022-09-03T05:52:28.843 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EsQHHArzbq46nsFkxp1mvz0Y20U=', 'Content-Length': '0'} 2022-09-03T05:52:28.845 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003a4a2762cb97cab4-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.846 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:28.846 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:28.847 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.847 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:28.848 DEBUG:boto:Data: 2022-09-03T05:52:28.848 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.849 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.850 DEBUG:boto:Params: {} 2022-09-03T05:52:28.850 DEBUG:boto:Token: None 2022-09-03T05:52:28.851 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-6/ 2022-09-03T05:52:28.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:szKhONKK+mxhS5EJ9Bg/dBhQcSQ= 2022-09-03T05:52:28.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:szKhONKK+mxhS5EJ9Bg/dBhQcSQ=', 'Content-Length': '0'} 2022-09-03T05:52:28.854 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000099ad9cfac3863abb-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.855 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:28.856 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:28.856 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.857 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:28.857 DEBUG:boto:Data: 2022-09-03T05:52:28.858 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.858 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:28.859 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.859 DEBUG:boto:Params: {} 2022-09-03T05:52:28.860 DEBUG:boto:Token: None 2022-09-03T05:52:28.860 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-4/ 2022-09-03T05:52:28.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jCyh6hUvCGpowvifRYkO9e+kHJ8= 2022-09-03T05:52:28.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jCyh6hUvCGpowvifRYkO9e+kHJ8=', 'Content-Length': '0'} 2022-09-03T05:52:28.864 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002b7ea361e037103d-006312eb9c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.865 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:28.865 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:28.866 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.866 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:28.867 DEBUG:boto:Data: 2022-09-03T05:52:28.867 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.868 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:28.868 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.869 DEBUG:boto:Params: {} 2022-09-03T05:52:28.869 DEBUG:boto:Token: None 2022-09-03T05:52:28.870 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-5/ 2022-09-03T05:52:28.871 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EsQHHArzbq46nsFkxp1mvz0Y20U= 2022-09-03T05:52:28.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EsQHHArzbq46nsFkxp1mvz0Y20U=', 'Content-Length': '0'} 2022-09-03T05:52:28.873 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a2d510bb19293b44-006312eb9c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.874 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:28.874 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:28.875 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.876 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:28.876 DEBUG:boto:Data: 2022-09-03T05:52:28.877 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.877 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:28.878 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.878 DEBUG:boto:Params: {} 2022-09-03T05:52:28.879 DEBUG:boto:Token: None 2022-09-03T05:52:28.879 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-6/ 2022-09-03T05:52:28.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:szKhONKK+mxhS5EJ9Bg/dBhQcSQ= 2022-09-03T05:52:28.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:szKhONKK+mxhS5EJ9Bg/dBhQcSQ=', 'Content-Length': '0'} 2022-09-03T05:52:28.883 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003491bf26c6f2ef7e-006312eb9c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.884 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:28.884 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:28.885 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.885 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:28.885 DEBUG:boto:Data: 2022-09-03T05:52:28.886 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.887 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:28.887 DEBUG:boto:Port: 8001 2022-09-03T05:52:28.888 DEBUG:boto:Params: {} 2022-09-03T05:52:28.888 DEBUG:boto:Token: None 2022-09-03T05:52:28.889 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-4/ 2022-09-03T05:52:28.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jCyh6hUvCGpowvifRYkO9e+kHJ8= 2022-09-03T05:52:28.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jCyh6hUvCGpowvifRYkO9e+kHJ8=', 'Content-Length': '0'} 2022-09-03T05:52:28.893 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c89a89fd114bb863-006312eb9c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.894 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:28.895 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:28.895 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.896 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:28.896 DEBUG:boto:Data: 2022-09-03T05:52:28.897 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.897 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:28.898 DEBUG:boto:Port: 8001 2022-09-03T05:52:28.898 DEBUG:boto:Params: {} 2022-09-03T05:52:28.899 DEBUG:boto:Token: None 2022-09-03T05:52:28.899 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-5/ 2022-09-03T05:52:28.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EsQHHArzbq46nsFkxp1mvz0Y20U= 2022-09-03T05:52:28.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EsQHHArzbq46nsFkxp1mvz0Y20U=', 'Content-Length': '0'} 2022-09-03T05:52:28.903 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000078ed82267e4316e9-006312eb9c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.903 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:28.904 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:28.904 DEBUG:boto:Method: HEAD 2022-09-03T05:52:28.905 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:28.905 DEBUG:boto:Data: 2022-09-03T05:52:28.906 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:28.907 DEBUG:boto:Port: 8001 2022-09-03T05:52:28.907 DEBUG:boto:Params: {} 2022-09-03T05:52:28.908 DEBUG:boto:Token: None 2022-09-03T05:52:28.909 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:28 GMT /avysmw-6/ 2022-09-03T05:52:28.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:szKhONKK+mxhS5EJ9Bg/dBhQcSQ= 2022-09-03T05:52:28.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:szKhONKK+mxhS5EJ9Bg/dBhQcSQ=', 'Content-Length': '0'} 2022-09-03T05:52:28.912 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007310f0ddee28f871-006312eb9c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.913 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:28.914 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:28.914 DEBUG:boto:Method: PUT 2022-09-03T05:52:28.915 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:28.915 DEBUG:boto:Data: 2022-09-03T05:52:28.915 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.916 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.916 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.917 DEBUG:boto:Params: {} 2022-09-03T05:52:28.918 DEBUG:boto:Token: None 2022-09-03T05:52:28.918 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:28 GMT /avysmw-4/ 2022-09-03T05:52:28.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+AJAzgFKZuCI56z9T7admZOslS8= 2022-09-03T05:52:28.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+AJAzgFKZuCI56z9T7admZOslS8=', 'Content-Length': '0'} 2022-09-03T05:52:28.922 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000000e64fe54d75753fc-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.923 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:28.924 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:28.924 DEBUG:boto:Method: PUT 2022-09-03T05:52:28.925 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:28.925 DEBUG:boto:Data: 2022-09-03T05:52:28.926 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.926 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.927 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.927 DEBUG:boto:Params: {} 2022-09-03T05:52:28.928 DEBUG:boto:Token: None 2022-09-03T05:52:28.928 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:28 GMT /avysmw-5/ 2022-09-03T05:52:28.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fe+p5uZTvmqoxEcbpk1UHAwQOuA= 2022-09-03T05:52:28.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fe+p5uZTvmqoxEcbpk1UHAwQOuA=', 'Content-Length': '0'} 2022-09-03T05:52:28.932 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000fd6f05ef12251bd3-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.933 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:28.933 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:28.934 DEBUG:boto:Method: PUT 2022-09-03T05:52:28.934 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:28.935 DEBUG:boto:Data: 2022-09-03T05:52:28.935 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:28.936 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.937 DEBUG:boto:Params: {} 2022-09-03T05:52:28.937 DEBUG:boto:Token: None 2022-09-03T05:52:28.938 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:28 GMT /avysmw-6/ 2022-09-03T05:52:28.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LJ8DxavoChG2Y9jdlANaMTNJjUk= 2022-09-03T05:52:28.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LJ8DxavoChG2Y9jdlANaMTNJjUk=', 'Content-Length': '0'} 2022-09-03T05:52:28.942 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000094f14c7607e2e06d-006312eb9c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.942 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:28.943 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:28.944 DEBUG:boto:Method: PUT 2022-09-03T05:52:28.944 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:28.944 DEBUG:boto:Data: 2022-09-03T05:52:28.945 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.945 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:28.946 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.946 DEBUG:boto:Params: {} 2022-09-03T05:52:28.947 DEBUG:boto:Token: None 2022-09-03T05:52:28.948 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:28 GMT /avysmw-4/ 2022-09-03T05:52:28.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+AJAzgFKZuCI56z9T7admZOslS8= 2022-09-03T05:52:28.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+AJAzgFKZuCI56z9T7admZOslS8=', 'Content-Length': '0'} 2022-09-03T05:52:28.954 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000001c4571f8356579fb-006312eb9c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:28.955 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:28.956 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:28.956 DEBUG:boto:Method: PUT 2022-09-03T05:52:28.957 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:28.957 DEBUG:boto:Data: 2022-09-03T05:52:28.958 DEBUG:boto:Headers: {} 2022-09-03T05:52:28.958 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:28.959 DEBUG:boto:Port: 8000 2022-09-03T05:52:28.959 DEBUG:boto:Params: {} 2022-09-03T05:52:28.960 DEBUG:boto:Token: None 2022-09-03T05:52:28.961 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:28 GMT /avysmw-5/ 2022-09-03T05:52:28.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fe+p5uZTvmqoxEcbpk1UHAwQOuA= 2022-09-03T05:52:28.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fe+p5uZTvmqoxEcbpk1UHAwQOuA=', 'Content-Length': '0'} 2022-09-03T05:52:29.008 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ae9d6bba87a453b5-006312eb9c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.009 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:29.010 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:29.010 DEBUG:boto:Method: PUT 2022-09-03T05:52:29.011 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:29.011 DEBUG:boto:Data: 2022-09-03T05:52:29.012 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.012 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:29.013 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.013 DEBUG:boto:Params: {} 2022-09-03T05:52:29.014 DEBUG:boto:Token: None 2022-09-03T05:52:29.015 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:29 GMT /avysmw-6/ 2022-09-03T05:52:29.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XHcG61J3SkSrmXQ5NYlE2sIPe9s= 2022-09-03T05:52:29.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XHcG61J3SkSrmXQ5NYlE2sIPe9s=', 'Content-Length': '0'} 2022-09-03T05:52:29.061 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000001ffe5ae7e80b6187-006312eb9d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.062 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:29.063 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:29.063 DEBUG:boto:Method: PUT 2022-09-03T05:52:29.064 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:29.064 DEBUG:boto:Data: 2022-09-03T05:52:29.065 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.065 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.066 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.066 DEBUG:boto:Params: {} 2022-09-03T05:52:29.067 DEBUG:boto:Token: None 2022-09-03T05:52:29.068 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:29 GMT /avysmw-4/ 2022-09-03T05:52:29.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C6OPApWKIVmtSzGgnYNs23NXN/o= 2022-09-03T05:52:29.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C6OPApWKIVmtSzGgnYNs23NXN/o=', 'Content-Length': '0'} 2022-09-03T05:52:29.074 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000dcaf26a8eb6d1ab4-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.074 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:29.075 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:29.075 DEBUG:boto:Method: PUT 2022-09-03T05:52:29.076 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:29.076 DEBUG:boto:Data: 2022-09-03T05:52:29.077 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.078 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.078 DEBUG:boto:Params: {} 2022-09-03T05:52:29.079 DEBUG:boto:Token: None 2022-09-03T05:52:29.080 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:29 GMT /avysmw-5/ 2022-09-03T05:52:29.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B33KLqIgX0hG7+4m9U6ZHs7UXXY= 2022-09-03T05:52:29.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B33KLqIgX0hG7+4m9U6ZHs7UXXY=', 'Content-Length': '0'} 2022-09-03T05:52:29.127 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000028348e1303e57707-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.128 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:29.128 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:29.129 DEBUG:boto:Method: PUT 2022-09-03T05:52:29.129 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:29.130 DEBUG:boto:Data: 2022-09-03T05:52:29.131 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.131 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.132 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.132 DEBUG:boto:Params: {} 2022-09-03T05:52:29.133 DEBUG:boto:Token: None 2022-09-03T05:52:29.134 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:29 GMT /avysmw-6/ 2022-09-03T05:52:29.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XHcG61J3SkSrmXQ5NYlE2sIPe9s= 2022-09-03T05:52:29.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XHcG61J3SkSrmXQ5NYlE2sIPe9s=', 'Content-Length': '0'} 2022-09-03T05:52:29.182 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000cd7979f70e8a09a6-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.183 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:29.184 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:29.185 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.185 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:29.186 DEBUG:boto:Data: 2022-09-03T05:52:29.186 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:29.188 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.188 DEBUG:boto:Params: {} 2022-09-03T05:52:29.189 DEBUG:boto:Token: None 2022-09-03T05:52:29.189 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-4/ 2022-09-03T05:52:29.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw= 2022-09-03T05:52:29.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw=', 'Content-Length': '0'} 2022-09-03T05:52:29.193 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001671261563325b62-006312eb9d-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.193 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:29.194 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:29.195 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.196 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:29.197 DEBUG:boto:Data: 2022-09-03T05:52:29.197 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.198 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:29.198 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.199 DEBUG:boto:Params: {} 2022-09-03T05:52:29.199 DEBUG:boto:Token: None 2022-09-03T05:52:29.200 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-5/ 2022-09-03T05:52:29.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE= 2022-09-03T05:52:29.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE=', 'Content-Length': '0'} 2022-09-03T05:52:29.204 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f840eb177b14cc12-006312eb9d-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.205 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:29.205 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:29.206 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.207 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:29.207 DEBUG:boto:Data: 2022-09-03T05:52:29.208 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.208 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:29.209 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.209 DEBUG:boto:Params: {} 2022-09-03T05:52:29.210 DEBUG:boto:Token: None 2022-09-03T05:52:29.210 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-6/ 2022-09-03T05:52:29.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A= 2022-09-03T05:52:29.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A=', 'Content-Length': '0'} 2022-09-03T05:52:29.214 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000002c1677143d6fe23-006312eb9d-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.215 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:29.215 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:29.216 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.217 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:29.218 DEBUG:boto:Data: 2022-09-03T05:52:29.218 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.219 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:29.219 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.220 DEBUG:boto:Params: {} 2022-09-03T05:52:29.220 DEBUG:boto:Token: None 2022-09-03T05:52:29.221 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-4/ 2022-09-03T05:52:29.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw= 2022-09-03T05:52:29.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw=', 'Content-Length': '0'} 2022-09-03T05:52:29.224 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000099b8422b3921cc4b-006312eb9d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.225 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:29.226 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:29.226 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.227 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:29.227 DEBUG:boto:Data: 2022-09-03T05:52:29.228 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.228 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:29.229 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.229 DEBUG:boto:Params: {} 2022-09-03T05:52:29.230 DEBUG:boto:Token: None 2022-09-03T05:52:29.231 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-5/ 2022-09-03T05:52:29.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE= 2022-09-03T05:52:29.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE=', 'Content-Length': '0'} 2022-09-03T05:52:29.234 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000734e6b3575891c7b-006312eb9d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.235 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:29.235 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:29.236 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.236 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:29.237 DEBUG:boto:Data: 2022-09-03T05:52:29.237 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:29.238 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.239 DEBUG:boto:Params: {} 2022-09-03T05:52:29.240 DEBUG:boto:Token: None 2022-09-03T05:52:29.240 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-6/ 2022-09-03T05:52:29.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A= 2022-09-03T05:52:29.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A=', 'Content-Length': '0'} 2022-09-03T05:52:29.244 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004e7d94ae73639d0c-006312eb9d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.244 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:29.245 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:29.246 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.246 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:29.247 DEBUG:boto:Data: 2022-09-03T05:52:29.247 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.248 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.249 DEBUG:boto:Params: {} 2022-09-03T05:52:29.249 DEBUG:boto:Token: None 2022-09-03T05:52:29.250 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-4/ 2022-09-03T05:52:29.250 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw= 2022-09-03T05:52:29.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw=', 'Content-Length': '0'} 2022-09-03T05:52:29.253 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000087819ccafeb58a37-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.254 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:29.255 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:29.255 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.256 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:29.256 DEBUG:boto:Data: 2022-09-03T05:52:29.257 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.257 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.258 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.258 DEBUG:boto:Params: {} 2022-09-03T05:52:29.259 DEBUG:boto:Token: None 2022-09-03T05:52:29.260 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-5/ 2022-09-03T05:52:29.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE= 2022-09-03T05:52:29.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE=', 'Content-Length': '0'} 2022-09-03T05:52:29.263 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ee9d6b758781f75b-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.264 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:29.265 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:29.266 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.266 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:29.267 DEBUG:boto:Data: 2022-09-03T05:52:29.267 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.269 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.269 DEBUG:boto:Params: {} 2022-09-03T05:52:29.270 DEBUG:boto:Token: None 2022-09-03T05:52:29.270 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-6/ 2022-09-03T05:52:29.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A= 2022-09-03T05:52:29.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A=', 'Content-Length': '0'} 2022-09-03T05:52:29.274 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bd6926f68f5877e0-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.274 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.275 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.276 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:29.326 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:29.327 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:29.353 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:52:29.365 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:52:29.366 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:29.366 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184348.261653_85.1", 2022-09-03T05:52:29.367 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:28.261653Z" 2022-09-03T05:52:29.367 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:29.368 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:29.368 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:29.369 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:29.369 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:29.370 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:29.370 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:29.371 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:29.371 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:29.372 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:29.372 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:29.372 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:29.373 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:29.373 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:52:29.375 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:29.376 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:52:29.376 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.377 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.377 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:29.426 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:29.427 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:29.453 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:29.451+0000 7fe1db5bc600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:29.453 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:29.452+0000 7fe1db5bc600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:29.465 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:29.465 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:29.465 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:29.466 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:29.466 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:29.467 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:29.467 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.467 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:29.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:29.468 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:29.468 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:29.468 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:29.469 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184348.261653_85.1", 2022-09-03T05:52:29.469 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:29.469 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:29.469 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:29.470 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.261653Z", 2022-09-03T05:52:29.470 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.470 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.471 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:29.471 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:29.471 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:29.471 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:29.472 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:29.472 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:29.472 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:29.472 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:29.473 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:29.473 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.473 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.473 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.474 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:29.474 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:29.474 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:29.475 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:29.475 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:29.475 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:29.476 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:29.476 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:29.476 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:29.476 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.477 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:29.477 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:29.478 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:29.478 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:29.478 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:29.478 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:29.479 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:29.479 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:29.479 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:29.479 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.480 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.480 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.480 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:29.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.481 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:29.481 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:29.481 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:29.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.482 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:29Z" 2022-09-03T05:52:29.482 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:29.483 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184348.261653_85.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:28.261653Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:29Z" } 2022-09-03T05:52:29.484 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184348.261653_85.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:28.261653Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:29.485 DEBUG:rgw_multi.tests:log_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:29.485 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:29.486 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:52:29.486 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:52:29.487 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.487 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.488 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:29.540 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:29.540 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:29.579 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:29.577+0000 7f97c8321600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:29.579 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:29.578+0000 7f97c8321600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:29.601 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:52:29.602 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:52:29.602 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:52:29.602 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:52:29.603 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:52:29.603 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:29.603 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:29.603 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:29.604 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:52:29.604 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:29.604 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:52:29.604 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:29.605 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:29.605 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184348.261653_85.1", 2022-09-03T05:52:29.605 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:29.605 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:52:29.606 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:29.606 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.261653Z", 2022-09-03T05:52:29.606 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:29.606 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:29.607 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:29.607 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:29.607 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:52:29.607 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:29.608 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:29.608 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:29.608 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:29.608 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:29.609 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:29.609 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:29.609 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:29.609 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:29.609 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:29.610 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:29.610 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:52:29.610 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:29.610 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:29.611 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:29.611 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:29.611 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:29.611 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:29.612 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:29.612 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:29.612 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:29.612 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:29.613 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:29.613 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:52:29.613 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:29.613 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:29.613 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:29.614 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:29.614 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:29.614 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:29.614 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:29.615 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:29.615 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:29.615 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:29.615 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:52:29.616 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:29.616 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:52:29.616 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:52:29.616 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:52:29.617 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:29.617 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:52:29Z" 2022-09-03T05:52:29.617 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:52:29.618 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184348.261653_85.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:28.261653Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:29Z" } 2022-09-03T05:52:29.619 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184348.261653_85.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:28.261653Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:29.620 DEBUG:rgw_multi.tests:log_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:29.620 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:29.621 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:52:29.621 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:52:29.622 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.622 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.623 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:29.672 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:29.673 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:29.703 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:29.701+0000 7faf8dfaa600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:29.704 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:29.702+0000 7faf8dfaa600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:29.725 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:29.725 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:29.725 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:29.726 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:29.726 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:29.726 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:29.727 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.727 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.727 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:29.727 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:29.728 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:29.728 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:29.728 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:29.728 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184348.261653_85.1", 2022-09-03T05:52:29.729 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:29.729 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:29.729 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:29.729 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.261653Z", 2022-09-03T05:52:29.730 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.730 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.730 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.731 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:29.731 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:29.731 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:29.731 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:29.732 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:29.732 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:29.732 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:29.732 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:29.733 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:29.733 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.733 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.733 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.734 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:29.734 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:29.734 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:29.734 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:29.735 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:29.735 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:29.735 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:29.735 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:29.736 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:29.736 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.736 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.736 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.737 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:29.737 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:29.737 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:29.737 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:29.738 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:29.738 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:29.738 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:29.739 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:29.739 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:29.739 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:29.739 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.740 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:29.740 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:29.740 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.740 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:29.741 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:29.741 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:29.741 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:29.741 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:29Z" 2022-09-03T05:52:29.742 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:29.743 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184348.261653_85.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:28.261653Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:29Z" } 2022-09-03T05:52:29.744 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184348.261653_85.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:28.261653Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:29.744 DEBUG:rgw_multi.tests:log_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:29.745 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184348.261653_85.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:29.746 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:52:29.746 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:29.747 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:29.747 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.748 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:29.749 DEBUG:boto:Data: 2022-09-03T05:52:29.749 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.750 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:29.750 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.751 DEBUG:boto:Params: {} 2022-09-03T05:52:29.751 DEBUG:boto:Token: None 2022-09-03T05:52:29.753 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-4/ 2022-09-03T05:52:29.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw= 2022-09-03T05:52:29.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw=', 'Content-Length': '0'} 2022-09-03T05:52:29.757 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d7613b73838ab74c-006312eb9d-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.758 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:29.758 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:29.759 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.759 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:29.760 DEBUG:boto:Data: 2022-09-03T05:52:29.760 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.761 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:29.761 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.762 DEBUG:boto:Params: {} 2022-09-03T05:52:29.762 DEBUG:boto:Token: None 2022-09-03T05:52:29.763 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-5/ 2022-09-03T05:52:29.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE= 2022-09-03T05:52:29.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE=', 'Content-Length': '0'} 2022-09-03T05:52:29.767 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003ad1b0b4ce0a72f3-006312eb9d-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.767 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:29.768 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:29.768 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.769 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:29.769 DEBUG:boto:Data: 2022-09-03T05:52:29.770 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:29.771 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.771 DEBUG:boto:Params: {} 2022-09-03T05:52:29.772 DEBUG:boto:Token: None 2022-09-03T05:52:29.773 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-6/ 2022-09-03T05:52:29.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A= 2022-09-03T05:52:29.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A=', 'Content-Length': '0'} 2022-09-03T05:52:29.776 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a4c9fb467dfa6c19-006312eb9d-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.777 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:29.777 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:29.778 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.778 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:29.779 DEBUG:boto:Data: 2022-09-03T05:52:29.779 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.780 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:29.780 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.781 DEBUG:boto:Params: {} 2022-09-03T05:52:29.781 DEBUG:boto:Token: None 2022-09-03T05:52:29.782 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-4/ 2022-09-03T05:52:29.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw= 2022-09-03T05:52:29.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw=', 'Content-Length': '0'} 2022-09-03T05:52:29.786 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006391611d2cb997c1-006312eb9d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.786 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:29.787 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:29.787 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.788 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:29.789 DEBUG:boto:Data: 2022-09-03T05:52:29.789 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.790 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:29.790 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.791 DEBUG:boto:Params: {} 2022-09-03T05:52:29.791 DEBUG:boto:Token: None 2022-09-03T05:52:29.792 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-5/ 2022-09-03T05:52:29.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE= 2022-09-03T05:52:29.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE=', 'Content-Length': '0'} 2022-09-03T05:52:29.795 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000630a405cdfe79e11-006312eb9d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.796 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:29.797 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:29.797 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.798 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:29.798 DEBUG:boto:Data: 2022-09-03T05:52:29.799 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.799 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:29.800 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.800 DEBUG:boto:Params: {} 2022-09-03T05:52:29.801 DEBUG:boto:Token: None 2022-09-03T05:52:29.802 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-6/ 2022-09-03T05:52:29.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A= 2022-09-03T05:52:29.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A=', 'Content-Length': '0'} 2022-09-03T05:52:29.805 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f50dececf6479101-006312eb9d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.806 DEBUG:boto:path=/avysmw-4/ 2022-09-03T05:52:29.807 DEBUG:boto:auth_path=/avysmw-4/ 2022-09-03T05:52:29.807 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.808 DEBUG:boto:Path: /avysmw-4/ 2022-09-03T05:52:29.808 DEBUG:boto:Data: 2022-09-03T05:52:29.809 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.810 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.810 DEBUG:boto:Params: {} 2022-09-03T05:52:29.811 DEBUG:boto:Token: None 2022-09-03T05:52:29.812 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-4/ 2022-09-03T05:52:29.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw= 2022-09-03T05:52:29.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAV13FQYFwmckKUTqQohli6Ujhw=', 'Content-Length': '0'} 2022-09-03T05:52:29.815 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000934b67a4a9d785ad-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.816 DEBUG:boto:path=/avysmw-5/ 2022-09-03T05:52:29.817 DEBUG:boto:auth_path=/avysmw-5/ 2022-09-03T05:52:29.817 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.818 DEBUG:boto:Path: /avysmw-5/ 2022-09-03T05:52:29.818 DEBUG:boto:Data: 2022-09-03T05:52:29.819 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.820 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.821 DEBUG:boto:Params: {} 2022-09-03T05:52:29.821 DEBUG:boto:Token: None 2022-09-03T05:52:29.822 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-5/ 2022-09-03T05:52:29.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE= 2022-09-03T05:52:29.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PiZpyUox3SH2j2SDYcbRROUkWvE=', 'Content-Length': '0'} 2022-09-03T05:52:29.826 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005910b622cc09f112-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.827 DEBUG:boto:path=/avysmw-6/ 2022-09-03T05:52:29.827 DEBUG:boto:auth_path=/avysmw-6/ 2022-09-03T05:52:29.828 DEBUG:boto:Method: HEAD 2022-09-03T05:52:29.829 DEBUG:boto:Path: /avysmw-6/ 2022-09-03T05:52:29.829 DEBUG:boto:Data: 2022-09-03T05:52:29.830 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.831 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.831 DEBUG:boto:Params: {} 2022-09-03T05:52:29.832 DEBUG:boto:Token: None 2022-09-03T05:52:29.833 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:29 GMT /avysmw-6/ 2022-09-03T05:52:29.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A= 2022-09-03T05:52:29.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FhOi2nXH5ZZCQRLW39iPulE/I5A=', 'Content-Length': '0'} 2022-09-03T05:52:29.837 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000af4a7b7a48ba21e7-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.837 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_recreate ... ok 2022-09-03T05:52:29.838 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-7 2022-09-03T05:52:29.839 DEBUG:boto:path=/avysmw-7/ 2022-09-03T05:52:29.840 DEBUG:boto:auth_path=/avysmw-7/ 2022-09-03T05:52:29.840 DEBUG:boto:Method: PUT 2022-09-03T05:52:29.841 DEBUG:boto:Path: /avysmw-7/ 2022-09-03T05:52:29.841 DEBUG:boto:Data: 2022-09-03T05:52:29.842 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.843 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:29.843 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.844 DEBUG:boto:Params: {} 2022-09-03T05:52:29.844 DEBUG:boto:Token: None 2022-09-03T05:52:29.845 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:29 GMT /avysmw-7/ 2022-09-03T05:52:29.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+9bamFaxSp2iUY2vVN4FXX2Jec8= 2022-09-03T05:52:29.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+9bamFaxSp2iUY2vVN4FXX2Jec8=', 'Content-Length': '0'} 2022-09-03T05:52:29.871 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000d8f768cb5227dc5d-006312eb9d-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.871 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-8 2022-09-03T05:52:29.872 DEBUG:boto:path=/avysmw-8/ 2022-09-03T05:52:29.872 DEBUG:boto:auth_path=/avysmw-8/ 2022-09-03T05:52:29.873 DEBUG:boto:Method: PUT 2022-09-03T05:52:29.873 DEBUG:boto:Path: /avysmw-8/ 2022-09-03T05:52:29.874 DEBUG:boto:Data: 2022-09-03T05:52:29.875 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.875 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:29.876 DEBUG:boto:Port: 8000 2022-09-03T05:52:29.876 DEBUG:boto:Params: {} 2022-09-03T05:52:29.877 DEBUG:boto:Token: None 2022-09-03T05:52:29.878 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:29 GMT /avysmw-8/ 2022-09-03T05:52:29.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8cYcae1+6bNek6/pLtkxvWcLhAw= 2022-09-03T05:52:29.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8cYcae1+6bNek6/pLtkxvWcLhAw=', 'Content-Length': '0'} 2022-09-03T05:52:29.908 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ebf5f4361113fdc7-006312eb9d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.908 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-9 2022-09-03T05:52:29.909 DEBUG:boto:path=/avysmw-9/ 2022-09-03T05:52:29.909 DEBUG:boto:auth_path=/avysmw-9/ 2022-09-03T05:52:29.910 DEBUG:boto:Method: PUT 2022-09-03T05:52:29.911 DEBUG:boto:Path: /avysmw-9/ 2022-09-03T05:52:29.911 DEBUG:boto:Data: 2022-09-03T05:52:29.912 DEBUG:boto:Headers: {} 2022-09-03T05:52:29.912 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:29.913 DEBUG:boto:Port: 8001 2022-09-03T05:52:29.914 DEBUG:boto:Params: {} 2022-09-03T05:52:29.914 DEBUG:boto:Token: None 2022-09-03T05:52:29.915 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:29 GMT /avysmw-9/ 2022-09-03T05:52:29.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:420HtXlJiSKBLlrBMtcpVLZPffA= 2022-09-03T05:52:29.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:420HtXlJiSKBLlrBMtcpVLZPffA=', 'Content-Length': '0'} 2022-09-03T05:52:29.951 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000003d8a06cbeb611d63-006312eb9d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:29.952 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.952 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:29.953 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:30.004 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:30.004 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:30.030 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:52:30.046 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:52:30.046 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.047 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184349.938376_97.1", 2022-09-03T05:52:30.047 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:29.938376Z" 2022-09-03T05:52:30.047 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.048 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.048 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:30.048 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:30.048 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.049 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.049 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:30.049 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:30.049 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.050 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.050 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:30.050 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:30.051 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:30.051 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:52:30.051 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184349.938376_97.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:30.052 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:52:30.052 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.053 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.054 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:30.109 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:30.110 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:30.150 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:30.148+0000 7fa212896600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:30.151 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:30.149+0000 7fa212896600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:30.162 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:30.163 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:30.163 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:30.164 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:30.164 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:30.164 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:30.164 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.165 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:30.165 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:30.166 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:30.166 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:30.166 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:30.166 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184349.938376_97.1", 2022-09-03T05:52:30.167 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:30.167 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:30.167 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:30.168 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.938376Z", 2022-09-03T05:52:30.168 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.168 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.169 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.169 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:30.169 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:30.169 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:30.170 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:30.171 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:30.171 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:30.171 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:30.172 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:30.172 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:30.172 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.172 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.173 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:30.173 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:30.173 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:30.174 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:30.174 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:30.174 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:30.174 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:30.175 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:30.175 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:30.175 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.175 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.176 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.176 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:30.176 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:30.176 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:30.176 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:30.177 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:30.177 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:30.177 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:30.177 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:30.178 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:30.178 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.178 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.178 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.179 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:30.179 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.179 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:30.179 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:30.180 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:30.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.180 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:30Z" 2022-09-03T05:52:30.180 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:30.182 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184349.938376_97.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:29.938376Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:30Z" } 2022-09-03T05:52:30.182 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184349.938376_97.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:29.938376Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:30.183 DEBUG:rgw_multi.tests:log_status={0: '1_1662184349.938376_97.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:30.183 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184349.938376_97.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:30.184 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:52:30.184 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:52:30.185 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.185 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.186 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:30.237 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:30.238 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:30.284 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:30.282+0000 7fe6d617d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:30.284 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:30.282+0000 7fe6d617d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:30.301 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:52:30.302 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:52:30.302 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:52:30.303 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:52:30.303 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:52:30.304 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:30.304 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:30.305 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.305 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:52:30.306 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.306 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:52:30.307 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:30.307 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:30.308 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184349.938376_97.1", 2022-09-03T05:52:30.308 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:30.309 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:52:30.309 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:30.310 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.938376Z", 2022-09-03T05:52:30.310 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:30.311 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:30.312 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.312 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.313 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:52:30.313 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:30.314 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:30.314 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:30.315 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:30.315 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:30.316 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:30.316 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:30.317 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:30.317 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:30.318 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.318 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.319 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:52:30.319 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:30.319 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:30.320 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:30.320 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:30.321 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:30.321 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:30.321 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:30.322 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:30.322 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:30.322 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.322 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.323 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:52:30.323 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:30.323 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:30.323 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:30.324 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:30.324 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:30.324 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:30.324 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:30.325 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:30.325 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:30.325 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:30.325 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:52:30.326 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.326 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:52:30.326 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:52:30.326 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:52:30.327 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.327 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:52:30Z" 2022-09-03T05:52:30.327 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:52:30.329 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184349.938376_97.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:29.938376Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:30Z" } 2022-09-03T05:52:30.329 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184349.938376_97.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:29.938376Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:30.330 DEBUG:rgw_multi.tests:log_status={0: '1_1662184349.938376_97.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:30.330 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184349.938376_97.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:30.331 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:52:30.331 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:52:30.332 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.332 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.333 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:30.382 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:30.383 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:30.411 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:30.409+0000 7fb7ae9d8600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:30.412 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:30.410+0000 7fb7ae9d8600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:30.423 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:30.424 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:30.424 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:30.425 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:30.425 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:30.426 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:30.426 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.427 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.427 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:30.427 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:30.428 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:30.428 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:30.429 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:30.429 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184349.938376_97.1", 2022-09-03T05:52:30.430 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:30.430 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:30.430 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:30.431 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.938376Z", 2022-09-03T05:52:30.431 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.431 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.432 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:30.432 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:30.432 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:30.433 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:30.433 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:30.433 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:30.433 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:30.434 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:30.434 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:30.435 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.435 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.435 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.435 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:30.436 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:30.436 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:30.436 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:30.437 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:30.437 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:30.437 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:30.437 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:30.438 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:30.438 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.438 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.439 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:30.439 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:30.440 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:30.440 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:30.440 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:30.441 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:30.441 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:30.441 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:30.442 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:30.442 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:30.442 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.443 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:30.443 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:30.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.444 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:30.445 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:30.445 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:30.445 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:30.446 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:30Z" 2022-09-03T05:52:30.446 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:30.449 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184349.938376_97.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:29.938376Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:30Z" } 2022-09-03T05:52:30.450 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184349.938376_97.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:29.938376Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:30.450 DEBUG:rgw_multi.tests:log_status={0: '1_1662184349.938376_97.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:30.451 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184349.938376_97.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:30.452 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:52:30.452 DEBUG:boto:path=/avysmw-7/ 2022-09-03T05:52:30.453 DEBUG:boto:auth_path=/avysmw-7/ 2022-09-03T05:52:30.454 DEBUG:boto:Method: HEAD 2022-09-03T05:52:30.455 DEBUG:boto:Path: /avysmw-7/ 2022-09-03T05:52:30.456 DEBUG:boto:Data: 2022-09-03T05:52:30.457 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:30.459 DEBUG:boto:Port: 8000 2022-09-03T05:52:30.459 DEBUG:boto:Params: {} 2022-09-03T05:52:30.460 DEBUG:boto:Token: None 2022-09-03T05:52:30.462 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:30 GMT /avysmw-7/ 2022-09-03T05:52:30.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ioqudYaFwfT4dubvtAVIlEH4ps4= 2022-09-03T05:52:30.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ioqudYaFwfT4dubvtAVIlEH4ps4=', 'Content-Length': '0'} 2022-09-03T05:52:30.467 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ba7f4f678e71b6e7-006312eb9e-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.467 DEBUG:boto:path=/avysmw-8/ 2022-09-03T05:52:30.468 DEBUG:boto:auth_path=/avysmw-8/ 2022-09-03T05:52:30.469 DEBUG:boto:Method: HEAD 2022-09-03T05:52:30.470 DEBUG:boto:Path: /avysmw-8/ 2022-09-03T05:52:30.471 DEBUG:boto:Data: 2022-09-03T05:52:30.471 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:30.473 DEBUG:boto:Port: 8000 2022-09-03T05:52:30.474 DEBUG:boto:Params: {} 2022-09-03T05:52:30.474 DEBUG:boto:Token: None 2022-09-03T05:52:30.475 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:30 GMT /avysmw-8/ 2022-09-03T05:52:30.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:59m4LNZUjJNAMMLa6kuxPpSq1YE= 2022-09-03T05:52:30.476 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:59m4LNZUjJNAMMLa6kuxPpSq1YE=', 'Content-Length': '0'} 2022-09-03T05:52:30.479 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b0b12acabc1ec076-006312eb9e-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.480 DEBUG:boto:path=/avysmw-9/ 2022-09-03T05:52:30.480 DEBUG:boto:auth_path=/avysmw-9/ 2022-09-03T05:52:30.481 DEBUG:boto:Method: HEAD 2022-09-03T05:52:30.481 DEBUG:boto:Path: /avysmw-9/ 2022-09-03T05:52:30.482 DEBUG:boto:Data: 2022-09-03T05:52:30.482 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:30.483 DEBUG:boto:Port: 8000 2022-09-03T05:52:30.484 DEBUG:boto:Params: {} 2022-09-03T05:52:30.485 DEBUG:boto:Token: None 2022-09-03T05:52:30.485 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:30 GMT /avysmw-9/ 2022-09-03T05:52:30.486 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PIm20uYtnnyoLFup1hgww5mUDDM= 2022-09-03T05:52:30.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PIm20uYtnnyoLFup1hgww5mUDDM=', 'Content-Length': '0'} 2022-09-03T05:52:30.489 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b6e78c15dba08929-006312eb9e-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.490 DEBUG:boto:path=/avysmw-7/ 2022-09-03T05:52:30.490 DEBUG:boto:auth_path=/avysmw-7/ 2022-09-03T05:52:30.491 DEBUG:boto:Method: HEAD 2022-09-03T05:52:30.491 DEBUG:boto:Path: /avysmw-7/ 2022-09-03T05:52:30.492 DEBUG:boto:Data: 2022-09-03T05:52:30.492 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.493 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:30.493 DEBUG:boto:Port: 8000 2022-09-03T05:52:30.494 DEBUG:boto:Params: {} 2022-09-03T05:52:30.494 DEBUG:boto:Token: None 2022-09-03T05:52:30.495 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:30 GMT /avysmw-7/ 2022-09-03T05:52:30.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ioqudYaFwfT4dubvtAVIlEH4ps4= 2022-09-03T05:52:30.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ioqudYaFwfT4dubvtAVIlEH4ps4=', 'Content-Length': '0'} 2022-09-03T05:52:30.499 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000febcbf82b8933c4c-006312eb9e-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.499 DEBUG:boto:path=/avysmw-8/ 2022-09-03T05:52:30.500 DEBUG:boto:auth_path=/avysmw-8/ 2022-09-03T05:52:30.500 DEBUG:boto:Method: HEAD 2022-09-03T05:52:30.501 DEBUG:boto:Path: /avysmw-8/ 2022-09-03T05:52:30.501 DEBUG:boto:Data: 2022-09-03T05:52:30.502 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.503 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:30.503 DEBUG:boto:Port: 8000 2022-09-03T05:52:30.504 DEBUG:boto:Params: {} 2022-09-03T05:52:30.504 DEBUG:boto:Token: None 2022-09-03T05:52:30.505 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:30 GMT /avysmw-8/ 2022-09-03T05:52:30.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:59m4LNZUjJNAMMLa6kuxPpSq1YE= 2022-09-03T05:52:30.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:59m4LNZUjJNAMMLa6kuxPpSq1YE=', 'Content-Length': '0'} 2022-09-03T05:52:30.509 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000043975b3ffd736c6d-006312eb9e-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.510 DEBUG:boto:path=/avysmw-9/ 2022-09-03T05:52:30.511 DEBUG:boto:auth_path=/avysmw-9/ 2022-09-03T05:52:30.512 DEBUG:boto:Method: HEAD 2022-09-03T05:52:30.513 DEBUG:boto:Path: /avysmw-9/ 2022-09-03T05:52:30.513 DEBUG:boto:Data: 2022-09-03T05:52:30.515 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.516 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:30.516 DEBUG:boto:Port: 8000 2022-09-03T05:52:30.517 DEBUG:boto:Params: {} 2022-09-03T05:52:30.519 DEBUG:boto:Token: None 2022-09-03T05:52:30.519 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:30 GMT /avysmw-9/ 2022-09-03T05:52:30.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PIm20uYtnnyoLFup1hgww5mUDDM= 2022-09-03T05:52:30.521 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PIm20uYtnnyoLFup1hgww5mUDDM=', 'Content-Length': '0'} 2022-09-03T05:52:30.523 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007116ac634ac67b81-006312eb9e-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.523 DEBUG:boto:path=/avysmw-7/ 2022-09-03T05:52:30.524 DEBUG:boto:auth_path=/avysmw-7/ 2022-09-03T05:52:30.525 DEBUG:boto:Method: HEAD 2022-09-03T05:52:30.525 DEBUG:boto:Path: /avysmw-7/ 2022-09-03T05:52:30.526 DEBUG:boto:Data: 2022-09-03T05:52:30.526 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.527 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:30.527 DEBUG:boto:Port: 8001 2022-09-03T05:52:30.528 DEBUG:boto:Params: {} 2022-09-03T05:52:30.528 DEBUG:boto:Token: None 2022-09-03T05:52:30.529 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:30 GMT /avysmw-7/ 2022-09-03T05:52:30.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ioqudYaFwfT4dubvtAVIlEH4ps4= 2022-09-03T05:52:30.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ioqudYaFwfT4dubvtAVIlEH4ps4=', 'Content-Length': '0'} 2022-09-03T05:52:30.533 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000044e95d645640782f-006312eb9e-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.533 DEBUG:boto:path=/avysmw-8/ 2022-09-03T05:52:30.534 DEBUG:boto:auth_path=/avysmw-8/ 2022-09-03T05:52:30.534 DEBUG:boto:Method: HEAD 2022-09-03T05:52:30.535 DEBUG:boto:Path: /avysmw-8/ 2022-09-03T05:52:30.535 DEBUG:boto:Data: 2022-09-03T05:52:30.536 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.536 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:30.537 DEBUG:boto:Port: 8001 2022-09-03T05:52:30.538 DEBUG:boto:Params: {} 2022-09-03T05:52:30.538 DEBUG:boto:Token: None 2022-09-03T05:52:30.539 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:30 GMT /avysmw-8/ 2022-09-03T05:52:30.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:59m4LNZUjJNAMMLa6kuxPpSq1YE= 2022-09-03T05:52:30.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:59m4LNZUjJNAMMLa6kuxPpSq1YE=', 'Content-Length': '0'} 2022-09-03T05:52:30.543 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c7eb371f27362983-006312eb9e-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.543 DEBUG:boto:path=/avysmw-9/ 2022-09-03T05:52:30.544 DEBUG:boto:auth_path=/avysmw-9/ 2022-09-03T05:52:30.544 DEBUG:boto:Method: HEAD 2022-09-03T05:52:30.545 DEBUG:boto:Path: /avysmw-9/ 2022-09-03T05:52:30.545 DEBUG:boto:Data: 2022-09-03T05:52:30.546 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:30.547 DEBUG:boto:Port: 8001 2022-09-03T05:52:30.548 DEBUG:boto:Params: {} 2022-09-03T05:52:30.548 DEBUG:boto:Token: None 2022-09-03T05:52:30.549 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:30 GMT /avysmw-9/ 2022-09-03T05:52:30.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PIm20uYtnnyoLFup1hgww5mUDDM= 2022-09-03T05:52:30.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PIm20uYtnnyoLFup1hgww5mUDDM=', 'Content-Length': '0'} 2022-09-03T05:52:30.552 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b4f265c45827bf18-006312eb9e-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.553 DEBUG:boto:path=/avysmw-7/ 2022-09-03T05:52:30.554 DEBUG:boto:auth_path=/avysmw-7/ 2022-09-03T05:52:30.554 DEBUG:boto:Method: DELETE 2022-09-03T05:52:30.555 DEBUG:boto:Path: /avysmw-7/ 2022-09-03T05:52:30.555 DEBUG:boto:Data: 2022-09-03T05:52:30.556 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:30.557 DEBUG:boto:Port: 8000 2022-09-03T05:52:30.557 DEBUG:boto:Params: {} 2022-09-03T05:52:30.558 DEBUG:boto:Token: None 2022-09-03T05:52:30.558 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:52:30 GMT /avysmw-7/ 2022-09-03T05:52:30.559 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z8SKkgDe2Yl5x6aDAHyg8jLOOVE= 2022-09-03T05:52:30.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z8SKkgDe2Yl5x6aDAHyg8jLOOVE=', 'Content-Length': '0'} 2022-09-03T05:52:30.596 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000be64e40aa9de5caa-006312eb9e-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.596 DEBUG:boto:path=/avysmw-8/ 2022-09-03T05:52:30.597 DEBUG:boto:auth_path=/avysmw-8/ 2022-09-03T05:52:30.597 DEBUG:boto:Method: DELETE 2022-09-03T05:52:30.598 DEBUG:boto:Path: /avysmw-8/ 2022-09-03T05:52:30.599 DEBUG:boto:Data: 2022-09-03T05:52:30.599 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.600 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:30.600 DEBUG:boto:Port: 8000 2022-09-03T05:52:30.601 DEBUG:boto:Params: {} 2022-09-03T05:52:30.601 DEBUG:boto:Token: None 2022-09-03T05:52:30.602 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:52:30 GMT /avysmw-8/ 2022-09-03T05:52:30.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aCr/6dtVXAZzhxo82E0WJhkV1+0= 2022-09-03T05:52:30.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aCr/6dtVXAZzhxo82E0WJhkV1+0=', 'Content-Length': '0'} 2022-09-03T05:52:30.673 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000006d100c483cee1cec-006312eb9e-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.674 DEBUG:boto:path=/avysmw-9/ 2022-09-03T05:52:30.674 DEBUG:boto:auth_path=/avysmw-9/ 2022-09-03T05:52:30.675 DEBUG:boto:Method: DELETE 2022-09-03T05:52:30.676 DEBUG:boto:Path: /avysmw-9/ 2022-09-03T05:52:30.676 DEBUG:boto:Data: 2022-09-03T05:52:30.677 DEBUG:boto:Headers: {} 2022-09-03T05:52:30.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:30.678 DEBUG:boto:Port: 8001 2022-09-03T05:52:30.678 DEBUG:boto:Params: {} 2022-09-03T05:52:30.679 DEBUG:boto:Token: None 2022-09-03T05:52:30.679 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:52:30 GMT /avysmw-9/ 2022-09-03T05:52:30.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6zGN3NwCtTH58nyOIr3DoLL01rw= 2022-09-03T05:52:30.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6zGN3NwCtTH58nyOIr3DoLL01rw=', 'Content-Length': '0'} 2022-09-03T05:52:30.767 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000006a6e19091aa48d03-006312eb9e-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:52:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:30.768 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.769 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.771 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:30.821 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:30.821 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:30.869 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:52:30.884 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:52:30.885 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.886 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:52:30.886 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T05:52:30.887 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.887 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.888 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:30.888 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:30.889 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.889 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.889 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:30.889 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:30.890 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:30.890 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:30.890 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:30.890 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:30.891 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:30.891 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:52:30.892 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:30.893 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:52:30.893 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.894 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:30.894 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:30.950 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:30.950 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:31.000 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:30.998+0000 7fb458ba4600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:31.001 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:30.999+0000 7fb458ba4600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:31.024 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:31.025 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:31.025 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:31.025 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:31.026 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:31.026 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:31.026 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:31.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:31.027 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:31.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:31.027 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:31.027 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:31.028 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:31.028 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.570224_98.1", 2022-09-03T05:52:31.028 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:31.028 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:31.029 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:31.029 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.570224Z", 2022-09-03T05:52:31.029 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:31.029 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:31.030 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:31.030 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:31.030 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:31.031 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:31.031 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:31.031 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:31.031 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:31.032 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:31.032 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:31.032 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:31.033 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:31.033 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:31.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:31.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:31.034 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:31.034 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:31.034 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:31.035 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:31.035 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:31.035 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:31.035 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:31.036 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:31.036 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:31.036 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:31.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:31.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:31.037 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:31.037 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:31.037 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:31.038 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:31.038 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:31.038 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:31.038 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:31.039 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:31.039 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:31.039 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:31.039 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:31.040 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:31.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:31.040 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:31.040 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:31.041 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:31.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:31.041 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:31Z" 2022-09-03T05:52:31.041 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:31.043 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.570224_98.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.570224Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:31Z" } 2022-09-03T05:52:31.043 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.570224_98.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.570224Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:31.044 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:31.044 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.570224_98.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:31.045 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662184350.715807_109.1 target=1_1662184350.570224_98.1 2022-09-03T05:52:36.047 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:36.047 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:36.048 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:36.105 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:36.105 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:36.137 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:36.135+0000 7f3d5b15d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:36.138 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:36.136+0000 7f3d5b15d600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:36.151 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:36.151 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:36.152 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:36.152 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:36.152 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:36.152 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:36.153 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.153 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:36.153 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:36.154 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:36.154 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:36.154 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:36.154 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:52:36.155 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:36.155 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:36.155 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:36.155 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:52:36.156 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.156 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.156 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.156 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:36.156 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:36.157 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:36.157 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:36.157 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:36.157 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:36.158 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:36.158 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:36.158 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:36.158 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.159 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.159 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:36.159 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:36.160 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:36.160 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:36.160 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:36.160 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:36.161 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:36.161 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:36.161 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:36.161 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.161 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.162 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.162 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:36.162 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:36.162 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:36.163 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:36.163 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:36.163 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:36.163 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:36.164 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:36.164 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:36.164 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.164 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.165 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.165 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:36.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.165 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:36.165 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:36.166 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:36.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.166 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:36Z" 2022-09-03T05:52:36.166 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:36.168 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:36Z" } 2022-09-03T05:52:36.168 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:36.169 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:36.169 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:36.170 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:52:36.170 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:52:36.171 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:36.171 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:36.172 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:36.232 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:36.232 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:36.282 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:36.281+0000 7fc5cea83600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:36.283 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:36.282+0000 7fc5cea83600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:36.301 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:52:36.302 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:52:36.302 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:52:36.303 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:52:36.303 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:52:36.303 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:36.304 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:36.304 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:36.304 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:52:36.304 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:36.305 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:52:36.305 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:36.305 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:36.305 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:52:36.306 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:36.306 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:52:36.306 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:36.307 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:52:36.307 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:36.307 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:36.307 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:36.308 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:36.308 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:52:36.308 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:36.308 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:36.309 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:36.309 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:36.309 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:36.309 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:36.310 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:36.310 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:36.310 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:36.310 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:36.311 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:36.311 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:52:36.311 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:36.311 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:36.312 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:36.312 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:36.312 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:36.313 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:36.313 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:36.313 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:36.314 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:36.314 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:36.314 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:36.315 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:52:36.315 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:36.315 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:36.316 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:36.316 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:36.316 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:36.316 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:36.317 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:36.317 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:36.317 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:36.317 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:36.318 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:52:36.318 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:36.319 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:52:36.319 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:52:36.319 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:52:36.320 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:36.320 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:52:36Z" 2022-09-03T05:52:36.321 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:52:36.323 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:36Z" } 2022-09-03T05:52:36.324 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:36.325 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:36.325 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:36.326 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:52:36.327 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:52:36.328 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:36.329 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:36.330 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:36.382 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:36.382 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:36.417 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:36.416+0000 7f4df78ee600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:36.418 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:36.417+0000 7f4df78ee600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:36.439 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:36.440 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:36.440 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:36.440 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:36.441 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:36.441 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:36.441 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.441 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.442 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:36.442 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:36.443 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:36.443 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:36.443 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:36.444 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:52:36.444 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:36.444 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:36.444 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:36.445 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:52:36.445 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.445 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.446 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.446 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:36.446 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:36.446 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:36.447 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:36.447 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:36.447 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:36.447 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:36.448 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:36.448 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:36.448 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.448 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.449 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.449 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:36.449 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:36.449 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:36.450 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:36.450 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:36.450 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:36.450 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:36.451 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:36.451 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:36.451 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.451 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:36.452 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:36.452 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:36.453 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:36.453 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:36.453 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:36.453 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:36.454 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:36.454 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:36.454 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:36.455 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.455 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:36.455 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:36.455 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.456 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:36.456 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:36.456 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:36.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:36.457 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:36Z" 2022-09-03T05:52:36.457 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:36.458 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:36Z" } 2022-09-03T05:52:36.459 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:36.459 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:36.460 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '', 2: '', 3: ''} 2022-09-03T05:52:36.460 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:52:36.461 DEBUG:boto:path=/avysmw-7/ 2022-09-03T05:52:36.461 DEBUG:boto:auth_path=/avysmw-7/ 2022-09-03T05:52:36.462 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.463 DEBUG:boto:Path: /avysmw-7/ 2022-09-03T05:52:36.463 DEBUG:boto:Data: 2022-09-03T05:52:36.464 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.464 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.465 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.465 DEBUG:boto:Params: {} 2022-09-03T05:52:36.466 DEBUG:boto:Token: None 2022-09-03T05:52:36.466 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-7/ 2022-09-03T05:52:36.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NQibFZ+nuajPPelZRvNPLmo2UvY= 2022-09-03T05:52:36.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NQibFZ+nuajPPelZRvNPLmo2UvY=', 'Content-Length': '0'} 2022-09-03T05:52:36.469 DEBUG:boto:Response headers: [('Content-Length', '231'), ('x-amz-request-id', 'tx000004303f36f64b7766b-006312eba4-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.470 DEBUG:boto:path=/avysmw-8/ 2022-09-03T05:52:36.470 DEBUG:boto:auth_path=/avysmw-8/ 2022-09-03T05:52:36.471 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.472 DEBUG:boto:Path: /avysmw-8/ 2022-09-03T05:52:36.472 DEBUG:boto:Data: 2022-09-03T05:52:36.473 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.474 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.474 DEBUG:boto:Params: {} 2022-09-03T05:52:36.475 DEBUG:boto:Token: None 2022-09-03T05:52:36.476 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-8/ 2022-09-03T05:52:36.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Midx3bL1FudxUh3kTSxAecXuBV4= 2022-09-03T05:52:36.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Midx3bL1FudxUh3kTSxAecXuBV4=', 'Content-Length': '0'} 2022-09-03T05:52:36.478 DEBUG:boto:Response headers: [('Content-Length', '231'), ('x-amz-request-id', 'tx00000a2e285ce7ff60d75-006312eba4-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.479 DEBUG:boto:path=/avysmw-9/ 2022-09-03T05:52:36.479 DEBUG:boto:auth_path=/avysmw-9/ 2022-09-03T05:52:36.480 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.480 DEBUG:boto:Path: /avysmw-9/ 2022-09-03T05:52:36.481 DEBUG:boto:Data: 2022-09-03T05:52:36.482 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.482 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.483 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.483 DEBUG:boto:Params: {} 2022-09-03T05:52:36.484 DEBUG:boto:Token: None 2022-09-03T05:52:36.484 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-9/ 2022-09-03T05:52:36.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Jgf0JmB484kLQpJJ4HKctIeV98= 2022-09-03T05:52:36.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Jgf0JmB484kLQpJJ4HKctIeV98=', 'Content-Length': '0'} 2022-09-03T05:52:36.487 DEBUG:boto:Response headers: [('Content-Length', '231'), ('x-amz-request-id', 'tx00000d17a1ca189f6528a-006312eba4-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.487 DEBUG:boto:path=/avysmw-7/ 2022-09-03T05:52:36.488 DEBUG:boto:auth_path=/avysmw-7/ 2022-09-03T05:52:36.488 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.489 DEBUG:boto:Path: /avysmw-7/ 2022-09-03T05:52:36.489 DEBUG:boto:Data: 2022-09-03T05:52:36.490 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.491 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.491 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.492 DEBUG:boto:Params: {} 2022-09-03T05:52:36.492 DEBUG:boto:Token: None 2022-09-03T05:52:36.493 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-7/ 2022-09-03T05:52:36.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NQibFZ+nuajPPelZRvNPLmo2UvY= 2022-09-03T05:52:36.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NQibFZ+nuajPPelZRvNPLmo2UvY=', 'Content-Length': '0'} 2022-09-03T05:52:36.496 DEBUG:boto:Response headers: [('Content-Length', '231'), ('x-amz-request-id', 'tx00000439f5036921a5f36-006312eba4-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.496 DEBUG:boto:path=/avysmw-8/ 2022-09-03T05:52:36.497 DEBUG:boto:auth_path=/avysmw-8/ 2022-09-03T05:52:36.497 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.498 DEBUG:boto:Path: /avysmw-8/ 2022-09-03T05:52:36.498 DEBUG:boto:Data: 2022-09-03T05:52:36.499 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.499 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.500 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.501 DEBUG:boto:Params: {} 2022-09-03T05:52:36.501 DEBUG:boto:Token: None 2022-09-03T05:52:36.502 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-8/ 2022-09-03T05:52:36.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Midx3bL1FudxUh3kTSxAecXuBV4= 2022-09-03T05:52:36.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Midx3bL1FudxUh3kTSxAecXuBV4=', 'Content-Length': '0'} 2022-09-03T05:52:36.504 DEBUG:boto:Response headers: [('Content-Length', '231'), ('x-amz-request-id', 'tx00000ff0645246d3581c6-006312eba4-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.505 DEBUG:boto:path=/avysmw-9/ 2022-09-03T05:52:36.505 DEBUG:boto:auth_path=/avysmw-9/ 2022-09-03T05:52:36.506 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.506 DEBUG:boto:Path: /avysmw-9/ 2022-09-03T05:52:36.507 DEBUG:boto:Data: 2022-09-03T05:52:36.507 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.508 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.508 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.509 DEBUG:boto:Params: {} 2022-09-03T05:52:36.510 DEBUG:boto:Token: None 2022-09-03T05:52:36.510 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-9/ 2022-09-03T05:52:36.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Jgf0JmB484kLQpJJ4HKctIeV98= 2022-09-03T05:52:36.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Jgf0JmB484kLQpJJ4HKctIeV98=', 'Content-Length': '0'} 2022-09-03T05:52:36.513 DEBUG:boto:Response headers: [('Content-Length', '231'), ('x-amz-request-id', 'tx000008976c19301648609-006312eba4-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.513 DEBUG:boto:path=/avysmw-7/ 2022-09-03T05:52:36.514 DEBUG:boto:auth_path=/avysmw-7/ 2022-09-03T05:52:36.514 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.515 DEBUG:boto:Path: /avysmw-7/ 2022-09-03T05:52:36.515 DEBUG:boto:Data: 2022-09-03T05:52:36.516 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:36.517 DEBUG:boto:Port: 8001 2022-09-03T05:52:36.517 DEBUG:boto:Params: {} 2022-09-03T05:52:36.518 DEBUG:boto:Token: None 2022-09-03T05:52:36.518 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-7/ 2022-09-03T05:52:36.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NQibFZ+nuajPPelZRvNPLmo2UvY= 2022-09-03T05:52:36.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NQibFZ+nuajPPelZRvNPLmo2UvY=', 'Content-Length': '0'} 2022-09-03T05:52:36.521 DEBUG:boto:Response headers: [('Content-Length', '231'), ('x-amz-request-id', 'tx00000b23bedda300c1306-006312eba4-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.522 DEBUG:boto:path=/avysmw-8/ 2022-09-03T05:52:36.522 DEBUG:boto:auth_path=/avysmw-8/ 2022-09-03T05:52:36.523 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.523 DEBUG:boto:Path: /avysmw-8/ 2022-09-03T05:52:36.524 DEBUG:boto:Data: 2022-09-03T05:52:36.524 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:36.525 DEBUG:boto:Port: 8001 2022-09-03T05:52:36.526 DEBUG:boto:Params: {} 2022-09-03T05:52:36.526 DEBUG:boto:Token: None 2022-09-03T05:52:36.527 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-8/ 2022-09-03T05:52:36.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Midx3bL1FudxUh3kTSxAecXuBV4= 2022-09-03T05:52:36.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Midx3bL1FudxUh3kTSxAecXuBV4=', 'Content-Length': '0'} 2022-09-03T05:52:36.529 DEBUG:boto:Response headers: [('Content-Length', '231'), ('x-amz-request-id', 'tx000007626de650b0bb9a1-006312eba4-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.530 DEBUG:boto:path=/avysmw-9/ 2022-09-03T05:52:36.531 DEBUG:boto:auth_path=/avysmw-9/ 2022-09-03T05:52:36.531 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.532 DEBUG:boto:Path: /avysmw-9/ 2022-09-03T05:52:36.532 DEBUG:boto:Data: 2022-09-03T05:52:36.532 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.533 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:36.533 DEBUG:boto:Port: 8001 2022-09-03T05:52:36.534 DEBUG:boto:Params: {} 2022-09-03T05:52:36.534 DEBUG:boto:Token: None 2022-09-03T05:52:36.535 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-9/ 2022-09-03T05:52:36.536 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Jgf0JmB484kLQpJJ4HKctIeV98= 2022-09-03T05:52:36.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Jgf0JmB484kLQpJJ4HKctIeV98=', 'Content-Length': '0'} 2022-09-03T05:52:36.538 DEBUG:boto:Response headers: [('Content-Length', '231'), ('x-amz-request-id', 'tx0000082bbae65d480ab7a-006312eba4-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.539 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_remove ... ok 2022-09-03T05:52:36.540 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-10 2022-09-03T05:52:36.541 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:36.541 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:36.542 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.543 DEBUG:boto:Path: /avysmw-10/ 2022-09-03T05:52:36.543 DEBUG:boto:Data: 2022-09-03T05:52:36.544 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.546 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.547 DEBUG:boto:Params: {} 2022-09-03T05:52:36.548 DEBUG:boto:Token: None 2022-09-03T05:52:36.549 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:36 GMT /avysmw-10/ 2022-09-03T05:52:36.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j5HnYO1Iw45nZfnCZ9ySiUEYcNc= 2022-09-03T05:52:36.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j5HnYO1Iw45nZfnCZ9ySiUEYcNc=', 'Content-Length': '0'} 2022-09-03T05:52:36.580 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000006f04931f6e9d5fb9-006312eba4-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.580 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-11 2022-09-03T05:52:36.581 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:52:36.581 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:52:36.582 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.582 DEBUG:boto:Path: /avysmw-11/ 2022-09-03T05:52:36.583 DEBUG:boto:Data: 2022-09-03T05:52:36.583 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.584 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.584 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.585 DEBUG:boto:Params: {} 2022-09-03T05:52:36.585 DEBUG:boto:Token: None 2022-09-03T05:52:36.586 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:36 GMT /avysmw-11/ 2022-09-03T05:52:36.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ea0eRXtN+Zv1Or0sQLD4FoQV/CI= 2022-09-03T05:52:36.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ea0eRXtN+Zv1Or0sQLD4FoQV/CI=', 'Content-Length': '0'} 2022-09-03T05:52:36.624 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000cfb27104d96d9ffa-006312eba4-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.626 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-12 2022-09-03T05:52:36.626 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:52:36.627 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:52:36.627 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.628 DEBUG:boto:Path: /avysmw-12/ 2022-09-03T05:52:36.628 DEBUG:boto:Data: 2022-09-03T05:52:36.630 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:36.631 DEBUG:boto:Port: 8001 2022-09-03T05:52:36.632 DEBUG:boto:Params: {} 2022-09-03T05:52:36.633 DEBUG:boto:Token: None 2022-09-03T05:52:36.633 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:52:36 GMT /avysmw-12/ 2022-09-03T05:52:36.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jxP9CkhiHzbns1c9IqELgTHIf5w= 2022-09-03T05:52:36.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jxP9CkhiHzbns1c9IqELgTHIf5w=', 'Content-Length': '0'} 2022-09-03T05:52:36.662 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000007d2c02b1b41ba617-006312eba4-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.663 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:36.664 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:36.664 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.665 DEBUG:boto:Path: /avysmw-10/ 2022-09-03T05:52:36.665 DEBUG:boto:Data: 2022-09-03T05:52:36.666 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.667 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.667 DEBUG:boto:Params: {} 2022-09-03T05:52:36.668 DEBUG:boto:Token: None 2022-09-03T05:52:36.668 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-10/ 2022-09-03T05:52:36.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zHtknSd7SKa+QIXs1L7RH9LttiI= 2022-09-03T05:52:36.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zHtknSd7SKa+QIXs1L7RH9LttiI=', 'Content-Length': '0'} 2022-09-03T05:52:36.672 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ee34baf6d6ee36df-006312eba4-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.674 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:36.675 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:36.675 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.676 DEBUG:boto:Path: /avysmw-10/myobj 2022-09-03T05:52:36.676 DEBUG:boto:Data: 2022-09-03T05:52:36.677 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:36.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.678 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.679 DEBUG:boto:Params: {} 2022-09-03T05:52:36.679 DEBUG:boto:Token: None 2022-09-03T05:52:36.680 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:36 GMT /avysmw-10/myobj 2022-09-03T05:52:36.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FI0TiBv43cHhmYPjoUeB7qADGWQ= 2022-09-03T05:52:36.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FI0TiBv43cHhmYPjoUeB7qADGWQ='} 2022-09-03T05:52:36.708 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000681082b807aba42c-006312eba4-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.709 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:36.710 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:36.711 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.712 DEBUG:boto:Path: /avysmw-10/ 2022-09-03T05:52:36.713 DEBUG:boto:Data: 2022-09-03T05:52:36.713 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.714 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.715 DEBUG:boto:Params: {} 2022-09-03T05:52:36.715 DEBUG:boto:Token: None 2022-09-03T05:52:36.716 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-10/ 2022-09-03T05:52:36.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zHtknSd7SKa+QIXs1L7RH9LttiI= 2022-09-03T05:52:36.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zHtknSd7SKa+QIXs1L7RH9LttiI=', 'Content-Length': '0'} 2022-09-03T05:52:36.720 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000feda718df3445498-006312eba4-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.721 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:36.721 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:36.722 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.722 DEBUG:boto:Path: /avysmw-10/_myobj 2022-09-03T05:52:36.723 DEBUG:boto:Data: 2022-09-03T05:52:36.723 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:36.724 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.724 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.725 DEBUG:boto:Params: {} 2022-09-03T05:52:36.725 DEBUG:boto:Token: None 2022-09-03T05:52:36.726 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:36 GMT /avysmw-10/_myobj 2022-09-03T05:52:36.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VzJyWxKmGtcEra5upITMch/mOXs= 2022-09-03T05:52:36.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VzJyWxKmGtcEra5upITMch/mOXs='} 2022-09-03T05:52:36.767 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000a8f4456e56261649-006312eba4-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.768 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:36.769 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:36.769 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.770 DEBUG:boto:Path: /avysmw-10/ 2022-09-03T05:52:36.770 DEBUG:boto:Data: 2022-09-03T05:52:36.771 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.772 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.772 DEBUG:boto:Params: {} 2022-09-03T05:52:36.773 DEBUG:boto:Token: None 2022-09-03T05:52:36.773 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-10/ 2022-09-03T05:52:36.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zHtknSd7SKa+QIXs1L7RH9LttiI= 2022-09-03T05:52:36.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zHtknSd7SKa+QIXs1L7RH9LttiI=', 'Content-Length': '0'} 2022-09-03T05:52:36.777 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000dbfed00efc23437b-006312eba4-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.778 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:36.779 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:36.779 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.780 DEBUG:boto:Path: /avysmw-10/%3A 2022-09-03T05:52:36.780 DEBUG:boto:Data: 2022-09-03T05:52:36.781 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:36.781 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.782 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.782 DEBUG:boto:Params: {} 2022-09-03T05:52:36.783 DEBUG:boto:Token: None 2022-09-03T05:52:36.783 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:36 GMT /avysmw-10/%3A 2022-09-03T05:52:36.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:96OIwv/5h6xCADenHauMGdOzoys= 2022-09-03T05:52:36.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:96OIwv/5h6xCADenHauMGdOzoys='} 2022-09-03T05:52:36.800 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000026a32d7eab738cfe-006312eba4-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.801 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:36.802 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:36.802 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.803 DEBUG:boto:Path: /avysmw-10/ 2022-09-03T05:52:36.804 DEBUG:boto:Data: 2022-09-03T05:52:36.805 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.806 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.807 DEBUG:boto:Params: {} 2022-09-03T05:52:36.807 DEBUG:boto:Token: None 2022-09-03T05:52:36.808 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-10/ 2022-09-03T05:52:36.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zHtknSd7SKa+QIXs1L7RH9LttiI= 2022-09-03T05:52:36.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zHtknSd7SKa+QIXs1L7RH9LttiI=', 'Content-Length': '0'} 2022-09-03T05:52:36.812 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004da95b909cd0bf86-006312eba4-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.813 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:36.814 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:36.814 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.815 DEBUG:boto:Path: /avysmw-10/%26 2022-09-03T05:52:36.815 DEBUG:boto:Data: 2022-09-03T05:52:36.816 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:36.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:36.817 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.818 DEBUG:boto:Params: {} 2022-09-03T05:52:36.819 DEBUG:boto:Token: None 2022-09-03T05:52:36.820 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:36 GMT /avysmw-10/%26 2022-09-03T05:52:36.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rG0Wfdhor1eCkUFT6jSBYHSSY/8= 2022-09-03T05:52:36.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rG0Wfdhor1eCkUFT6jSBYHSSY/8='} 2022-09-03T05:52:36.866 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000b52caec3bc5b97af-006312eba4-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.867 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:52:36.868 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:52:36.868 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.869 DEBUG:boto:Path: /avysmw-11/ 2022-09-03T05:52:36.869 DEBUG:boto:Data: 2022-09-03T05:52:36.870 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.870 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.871 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.871 DEBUG:boto:Params: {} 2022-09-03T05:52:36.872 DEBUG:boto:Token: None 2022-09-03T05:52:36.872 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-11/ 2022-09-03T05:52:36.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t9uuGylIlLKERtdO5sXDi+tVjjg= 2022-09-03T05:52:36.874 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t9uuGylIlLKERtdO5sXDi+tVjjg=', 'Content-Length': '0'} 2022-09-03T05:52:36.876 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d329466fa8fcd3d0-006312eba4-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.877 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:52:36.877 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:52:36.878 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.879 DEBUG:boto:Path: /avysmw-11/myobj 2022-09-03T05:52:36.879 DEBUG:boto:Data: 2022-09-03T05:52:36.880 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:36.880 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.881 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.882 DEBUG:boto:Params: {} 2022-09-03T05:52:36.882 DEBUG:boto:Token: None 2022-09-03T05:52:36.883 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:36 GMT /avysmw-11/myobj 2022-09-03T05:52:36.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l96lEP7YwGGHJM2wNbdyObzJuIw= 2022-09-03T05:52:36.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l96lEP7YwGGHJM2wNbdyObzJuIw='} 2022-09-03T05:52:36.896 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000082ff49fe00aaaf55-006312eba4-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.897 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:52:36.898 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:52:36.898 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.899 DEBUG:boto:Path: /avysmw-11/ 2022-09-03T05:52:36.899 DEBUG:boto:Data: 2022-09-03T05:52:36.900 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.900 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.901 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.901 DEBUG:boto:Params: {} 2022-09-03T05:52:36.902 DEBUG:boto:Token: None 2022-09-03T05:52:36.903 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-11/ 2022-09-03T05:52:36.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t9uuGylIlLKERtdO5sXDi+tVjjg= 2022-09-03T05:52:36.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t9uuGylIlLKERtdO5sXDi+tVjjg=', 'Content-Length': '0'} 2022-09-03T05:52:36.906 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e65f7698714c1005-006312eba4-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.907 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:52:36.908 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:52:36.909 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.909 DEBUG:boto:Path: /avysmw-11/_myobj 2022-09-03T05:52:36.910 DEBUG:boto:Data: 2022-09-03T05:52:36.910 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:36.911 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.911 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.912 DEBUG:boto:Params: {} 2022-09-03T05:52:36.912 DEBUG:boto:Token: None 2022-09-03T05:52:36.913 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:36 GMT /avysmw-11/_myobj 2022-09-03T05:52:36.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Loo92Ex4UKqyUAIDN9oP0+dvhFs= 2022-09-03T05:52:36.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Loo92Ex4UKqyUAIDN9oP0+dvhFs='} 2022-09-03T05:52:36.958 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000001656b0665153c9b1-006312eba4-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.960 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:52:36.960 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:52:36.961 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.961 DEBUG:boto:Path: /avysmw-11/ 2022-09-03T05:52:36.962 DEBUG:boto:Data: 2022-09-03T05:52:36.962 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.963 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.963 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.964 DEBUG:boto:Params: {} 2022-09-03T05:52:36.964 DEBUG:boto:Token: None 2022-09-03T05:52:36.965 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-11/ 2022-09-03T05:52:36.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t9uuGylIlLKERtdO5sXDi+tVjjg= 2022-09-03T05:52:36.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t9uuGylIlLKERtdO5sXDi+tVjjg=', 'Content-Length': '0'} 2022-09-03T05:52:36.968 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003dcccc88a9ddfd4a-006312eba4-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.969 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:52:36.969 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:52:36.970 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.970 DEBUG:boto:Path: /avysmw-11/%3A 2022-09-03T05:52:36.971 DEBUG:boto:Data: 2022-09-03T05:52:36.972 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:36.972 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.973 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.973 DEBUG:boto:Params: {} 2022-09-03T05:52:36.974 DEBUG:boto:Token: None 2022-09-03T05:52:36.974 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:36 GMT /avysmw-11/%3A 2022-09-03T05:52:36.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQyjf9DlkKpmob1eF0+GtEAlPUU= 2022-09-03T05:52:36.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQyjf9DlkKpmob1eF0+GtEAlPUU='} 2022-09-03T05:52:36.986 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000ad0ed0831afe0d91-006312eba4-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.987 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:52:36.987 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:52:36.988 DEBUG:boto:Method: HEAD 2022-09-03T05:52:36.988 DEBUG:boto:Path: /avysmw-11/ 2022-09-03T05:52:36.989 DEBUG:boto:Data: 2022-09-03T05:52:36.989 DEBUG:boto:Headers: {} 2022-09-03T05:52:36.990 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:36.990 DEBUG:boto:Port: 8000 2022-09-03T05:52:36.991 DEBUG:boto:Params: {} 2022-09-03T05:52:36.992 DEBUG:boto:Token: None 2022-09-03T05:52:36.992 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:36 GMT /avysmw-11/ 2022-09-03T05:52:36.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t9uuGylIlLKERtdO5sXDi+tVjjg= 2022-09-03T05:52:36.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t9uuGylIlLKERtdO5sXDi+tVjjg=', 'Content-Length': '0'} 2022-09-03T05:52:36.996 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f62feff231a84242-006312eba4-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:36.997 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:52:36.997 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:52:36.998 DEBUG:boto:Method: PUT 2022-09-03T05:52:36.998 DEBUG:boto:Path: /avysmw-11/%26 2022-09-03T05:52:36.999 DEBUG:boto:Data: 2022-09-03T05:52:36.999 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:37.000 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:37.000 DEBUG:boto:Port: 8000 2022-09-03T05:52:37.001 DEBUG:boto:Params: {} 2022-09-03T05:52:37.001 DEBUG:boto:Token: None 2022-09-03T05:52:37.002 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:37 GMT /avysmw-11/%26 2022-09-03T05:52:37.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b9f0ouH8q9bGp33lNRKiXRX2CIc= 2022-09-03T05:52:37.003 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b9f0ouH8q9bGp33lNRKiXRX2CIc='} 2022-09-03T05:52:37.009 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000004b190bbf8641b77b-006312eba5-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:37.011 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:52:37.012 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:52:37.013 DEBUG:boto:Method: HEAD 2022-09-03T05:52:37.013 DEBUG:boto:Path: /avysmw-12/ 2022-09-03T05:52:37.014 DEBUG:boto:Data: 2022-09-03T05:52:37.015 DEBUG:boto:Headers: {} 2022-09-03T05:52:37.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:37.016 DEBUG:boto:Port: 8001 2022-09-03T05:52:37.016 DEBUG:boto:Params: {} 2022-09-03T05:52:37.017 DEBUG:boto:Token: None 2022-09-03T05:52:37.017 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:37 GMT /avysmw-12/ 2022-09-03T05:52:37.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:numXQXxiexZl8xC8FxpI4ys0zqE= 2022-09-03T05:52:37.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:numXQXxiexZl8xC8FxpI4ys0zqE=', 'Content-Length': '0'} 2022-09-03T05:52:37.021 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b571d43c3285164b-006312eba5-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:37.022 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:52:37.023 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:52:37.023 DEBUG:boto:Method: PUT 2022-09-03T05:52:37.024 DEBUG:boto:Path: /avysmw-12/myobj 2022-09-03T05:52:37.024 DEBUG:boto:Data: 2022-09-03T05:52:37.025 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:37.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:37.027 DEBUG:boto:Port: 8001 2022-09-03T05:52:37.028 DEBUG:boto:Params: {} 2022-09-03T05:52:37.029 DEBUG:boto:Token: None 2022-09-03T05:52:37.030 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:37 GMT /avysmw-12/myobj 2022-09-03T05:52:37.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vhjfM3fbcrPARD193CIDL9bav3o= 2022-09-03T05:52:37.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vhjfM3fbcrPARD193CIDL9bav3o='} 2022-09-03T05:52:37.046 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000009075d5ff0e93b86e-006312eba5-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:37.047 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:52:37.048 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:52:37.048 DEBUG:boto:Method: HEAD 2022-09-03T05:52:37.049 DEBUG:boto:Path: /avysmw-12/ 2022-09-03T05:52:37.049 DEBUG:boto:Data: 2022-09-03T05:52:37.050 DEBUG:boto:Headers: {} 2022-09-03T05:52:37.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:37.051 DEBUG:boto:Port: 8001 2022-09-03T05:52:37.052 DEBUG:boto:Params: {} 2022-09-03T05:52:37.052 DEBUG:boto:Token: None 2022-09-03T05:52:37.053 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:37 GMT /avysmw-12/ 2022-09-03T05:52:37.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:numXQXxiexZl8xC8FxpI4ys0zqE= 2022-09-03T05:52:37.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:numXQXxiexZl8xC8FxpI4ys0zqE=', 'Content-Length': '0'} 2022-09-03T05:52:37.057 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000048d6da3affc8975d-006312eba5-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:37.058 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:52:37.058 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:52:37.059 DEBUG:boto:Method: PUT 2022-09-03T05:52:37.060 DEBUG:boto:Path: /avysmw-12/_myobj 2022-09-03T05:52:37.060 DEBUG:boto:Data: 2022-09-03T05:52:37.061 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:37.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:37.062 DEBUG:boto:Port: 8001 2022-09-03T05:52:37.063 DEBUG:boto:Params: {} 2022-09-03T05:52:37.063 DEBUG:boto:Token: None 2022-09-03T05:52:37.064 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:37 GMT /avysmw-12/_myobj 2022-09-03T05:52:37.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zc74OoTMG0UKr5BRW2Onjc07L5Q= 2022-09-03T05:52:37.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zc74OoTMG0UKr5BRW2Onjc07L5Q='} 2022-09-03T05:52:37.084 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000097d8a727422bef37-006312eba5-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:37.085 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:52:37.086 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:52:37.086 DEBUG:boto:Method: HEAD 2022-09-03T05:52:37.087 DEBUG:boto:Path: /avysmw-12/ 2022-09-03T05:52:37.088 DEBUG:boto:Data: 2022-09-03T05:52:37.088 DEBUG:boto:Headers: {} 2022-09-03T05:52:37.089 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:37.090 DEBUG:boto:Port: 8001 2022-09-03T05:52:37.090 DEBUG:boto:Params: {} 2022-09-03T05:52:37.091 DEBUG:boto:Token: None 2022-09-03T05:52:37.091 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:37 GMT /avysmw-12/ 2022-09-03T05:52:37.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:numXQXxiexZl8xC8FxpI4ys0zqE= 2022-09-03T05:52:37.093 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:numXQXxiexZl8xC8FxpI4ys0zqE=', 'Content-Length': '0'} 2022-09-03T05:52:37.095 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c7842a23cdc7b5e6-006312eba5-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:37.096 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:52:37.097 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:52:37.097 DEBUG:boto:Method: PUT 2022-09-03T05:52:37.098 DEBUG:boto:Path: /avysmw-12/%3A 2022-09-03T05:52:37.099 DEBUG:boto:Data: 2022-09-03T05:52:37.099 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:37.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:37.100 DEBUG:boto:Port: 8001 2022-09-03T05:52:37.101 DEBUG:boto:Params: {} 2022-09-03T05:52:37.101 DEBUG:boto:Token: None 2022-09-03T05:52:37.102 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:37 GMT /avysmw-12/%3A 2022-09-03T05:52:37.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1BkoehPqh2LK2l0VU98g49yXQBM= 2022-09-03T05:52:37.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1BkoehPqh2LK2l0VU98g49yXQBM='} 2022-09-03T05:52:37.150 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000e06f7625e9b10faf-006312eba5-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:37.164 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:52:37.165 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:52:37.166 DEBUG:boto:Method: HEAD 2022-09-03T05:52:37.167 DEBUG:boto:Path: /avysmw-12/ 2022-09-03T05:52:37.168 DEBUG:boto:Data: 2022-09-03T05:52:37.168 DEBUG:boto:Headers: {} 2022-09-03T05:52:37.169 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:37.170 DEBUG:boto:Port: 8001 2022-09-03T05:52:37.171 DEBUG:boto:Params: {} 2022-09-03T05:52:37.172 DEBUG:boto:Token: None 2022-09-03T05:52:37.172 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:37 GMT /avysmw-12/ 2022-09-03T05:52:37.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:numXQXxiexZl8xC8FxpI4ys0zqE= 2022-09-03T05:52:37.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:numXQXxiexZl8xC8FxpI4ys0zqE=', 'Content-Length': '0'} 2022-09-03T05:52:37.176 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f416620782a5f92c-006312eba5-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:37.177 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:52:37.178 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:52:37.178 DEBUG:boto:Method: PUT 2022-09-03T05:52:37.179 DEBUG:boto:Path: /avysmw-12/%26 2022-09-03T05:52:37.179 DEBUG:boto:Data: 2022-09-03T05:52:37.180 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:52:37.180 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:37.181 DEBUG:boto:Port: 8001 2022-09-03T05:52:37.181 DEBUG:boto:Params: {} 2022-09-03T05:52:37.182 DEBUG:boto:Token: None 2022-09-03T05:52:37.183 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:52:37 GMT /avysmw-12/%26 2022-09-03T05:52:37.183 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5y0z4i2vViJfnpZFYJrBQlz21Gg= 2022-09-03T05:52:37.184 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:52:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5y0z4i2vViJfnpZFYJrBQlz21Gg='} 2022-09-03T05:52:37.192 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000003df574ac03a88091-006312eba5-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:37.193 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:37.194 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:37.195 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:37.247 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:37.247 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:37.299 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:52:37.310 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:52:37.311 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:37.311 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:52:37.311 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T05:52:37.312 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:37.312 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:37.312 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184356.648716_76.1", 2022-09-03T05:52:37.312 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:36.648716Z" 2022-09-03T05:52:37.313 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:37.313 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:37.313 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:37.313 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:37.314 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:37.314 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:37.314 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:37.314 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:52:37.315 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:37.315 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:52:37.316 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662184356.648716_76.1', 2: '', 3: ''} 2022-09-03T05:52:37.317 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:52:37.317 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:37.318 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:37.318 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:37.367 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:37.368 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:37.395 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:37.393+0000 7f8549810600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:37.395 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:37.394+0000 7f8549810600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:37.410 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:37.410 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:37.410 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:37.411 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:37.411 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:37.411 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:37.412 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:37.412 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:37.412 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:37.412 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:37.413 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:37.413 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:37.413 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:37.413 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:52:37.414 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:37.414 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:37.414 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:37.414 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:52:37.415 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:37.415 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:37.415 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:37.415 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:37.416 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:37.416 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:37.416 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:37.417 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184356.561415_64.1", 2022-09-03T05:52:37.417 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:37.417 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:37.417 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:37.418 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.561415Z", 2022-09-03T05:52:37.418 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:37.419 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:37.419 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:37.419 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:37.420 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:37.420 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:37.420 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:37.420 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:37.421 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:37.421 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:37.421 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:37.421 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:37.422 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:37.422 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:37.422 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:37.423 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:37.423 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:37.423 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:37.423 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:37.424 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:37.424 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:37.424 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:37.424 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:37.425 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:37.425 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:37.425 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:37.425 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:37.426 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:37.426 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:37.426 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:37.426 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:37.427 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:37.427 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:37.427 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:37Z" 2022-09-03T05:52:37.427 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:37.428 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184356.561415_64.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:52:36.561415Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:37Z" } 2022-09-03T05:52:37.429 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184356.561415_64.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:52:36.561415Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:37.429 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184356.648716_76.1', 2: '', 3: ''} 2022-09-03T05:52:37.430 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184356.561415_64.1', 2: '', 3: ''} 2022-09-03T05:52:37.430 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662184356.648716_76.1 target=1_1662184356.561415_64.1 2022-09-03T05:52:42.431 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:42.432 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:42.433 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:42.491 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:42.492 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:42.529 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:42.527+0000 7f8fa85eb600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:42.530 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:42.529+0000 7f8fa85eb600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:42.542 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:42.543 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:42.543 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:42.543 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:42.544 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:42.544 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:42.544 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.545 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:42.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:42.545 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:42.545 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:42.546 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:42.546 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:52:42.546 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:42.547 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:42.547 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:42.547 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:52:42.548 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.548 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:42.549 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:42.549 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:42.549 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:42.550 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184356.648716_76.1", 2022-09-03T05:52:42.550 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:42.550 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:42.550 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:42.551 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.648716Z", 2022-09-03T05:52:42.551 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.551 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:42.552 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:42.552 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:42.552 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:42.553 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:42.553 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:42.553 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:42.553 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:42.554 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:42.554 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.554 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:42.555 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:42.555 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:42.556 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:42.556 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:42.556 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:42.556 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:42.557 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:42.557 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:42.557 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.558 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.558 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.558 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:42.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.559 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:42.559 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:42.559 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:42.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.560 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:42Z" 2022-09-03T05:52:42.561 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:42.562 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184356.648716_76.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:52:36.648716Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:42Z" } 2022-09-03T05:52:42.563 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184356.648716_76.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:52:36.648716Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:42.564 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184356.648716_76.1', 2: '', 3: ''} 2022-09-03T05:52:42.564 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184356.648716_76.1', 2: '', 3: ''} 2022-09-03T05:52:42.565 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:52:42.565 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:52:42.566 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:42.566 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:42.567 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:42.626 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:42.627 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:42.666 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:42.665+0000 7fe8e3b2c600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:42.667 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:42.666+0000 7fe8e3b2c600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:42.689 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:52:42.689 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:52:42.690 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:52:42.690 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:52:42.691 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:52:42.691 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:42.691 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:42.692 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:42.692 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:52:42.693 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:42.693 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:52:42.693 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:42.694 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:42.694 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:52:42.694 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:42.694 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:52:42.695 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:42.695 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:52:42.695 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:42.695 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:42.696 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:42.696 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:42.696 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:52:42.696 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:42.697 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:42.697 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184356.648716_76.1", 2022-09-03T05:52:42.697 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:42.697 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:42.698 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:42.698 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.648716Z", 2022-09-03T05:52:42.698 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:42.698 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:42.699 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:42.699 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:42.699 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:52:42.699 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:42.700 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:42.700 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:42.700 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:42.700 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:42.701 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:42.701 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:42.701 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:42.702 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:42.702 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:42.702 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:52:42.702 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:52:42.703 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:52:42.703 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:52:42.703 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:52:42.704 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:52:42.704 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:52:42.704 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:52:42.705 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:52:42.705 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:52:42.705 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:42.705 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:52:42.706 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:52:42.706 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:42.706 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:52:42.707 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:52:42.707 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:52:42.707 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:52:42.707 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:52:42Z" 2022-09-03T05:52:42.708 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:52:42.709 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184356.648716_76.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:52:36.648716Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:42Z" } 2022-09-03T05:52:42.710 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184356.648716_76.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:52:36.648716Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:42.710 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184356.648716_76.1', 2: '', 3: ''} 2022-09-03T05:52:42.711 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184356.648716_76.1', 2: '', 3: ''} 2022-09-03T05:52:42.711 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:52:42.712 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:52:42.712 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:42.713 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:52:42.714 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:52:42.765 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:42.765 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:42.793 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:42.791+0000 7fa5f700a600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:42.794 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:42.792+0000 7fa5f700a600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:52:42.815 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:52:42.816 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:52:42.816 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:52:42.817 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:52:42.817 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:52:42.817 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:52:42.818 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.818 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.819 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:52:42.819 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:42.820 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:52:42.820 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:42.820 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:42.821 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:52:42.821 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:42.821 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:52:42.821 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:42.822 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:52:42.822 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.822 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.822 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.823 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:42.823 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:52:42.823 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:42.823 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:42.824 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184356.648716_76.1", 2022-09-03T05:52:42.824 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:42.824 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:42.824 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:42.825 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.648716Z", 2022-09-03T05:52:42.825 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.825 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.825 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.826 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:42.826 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:52:42.826 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:42.826 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:42.827 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:42.827 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:42.827 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:42.827 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:42.828 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:42.828 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.828 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.828 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.829 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:52:42.829 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:52:42.829 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:52:42.829 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:52:42.830 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:52:42.830 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:52:42.830 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:52:42.830 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:52:42.831 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:52:42.831 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:52:42.831 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.831 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:52:42.832 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:52:42.832 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.832 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:52:42.832 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:52:42.833 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:52:42.833 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:52:42.833 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:52:42Z" 2022-09-03T05:52:42.834 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:52:42.835 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184356.648716_76.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:52:36.648716Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:52:42Z" } 2022-09-03T05:52:42.836 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184356.648716_76.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:52:36.648716Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:52:42.836 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184356.648716_76.1', 2: '', 3: ''} 2022-09-03T05:52:42.837 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184356.648716_76.1', 2: '', 3: ''} 2022-09-03T05:52:42.837 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:52:42.838 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:52:42.838 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:52:42.839 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-10 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:52:42.895 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:42.896 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:42.954 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:42.952+0000 7fba41b5f600 1 waiting to reach incremental sync.. 2022-09-03T05:52:47.955 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:47.953+0000 7fba41b5f600 1 waiting to reach incremental sync.. 2022-09-03T05:52:52.957 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:52.955+0000 7fba41b5f600 1 bucket sync caught up with source: 2022-09-03T05:52:52.957 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:52:52.957 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:52:52.958 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:52.955+0000 7fba41b5f600 0 bucket checkpoint complete 2022-09-03T05:52:52.975 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-10 zones={test-zone2, test-zone1} 2022-09-03T05:52:52.976 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:52.976 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:52.977 DEBUG:boto:Method: HEAD 2022-09-03T05:52:52.977 DEBUG:boto:Path: /avysmw-10/ 2022-09-03T05:52:52.978 DEBUG:boto:Data: 2022-09-03T05:52:52.979 DEBUG:boto:Headers: {} 2022-09-03T05:52:52.979 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:52.980 DEBUG:boto:Port: 8000 2022-09-03T05:52:52.981 DEBUG:boto:Params: {} 2022-09-03T05:52:52.982 DEBUG:boto:Token: None 2022-09-03T05:52:52.983 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:52 GMT /avysmw-10/ 2022-09-03T05:52:52.984 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5jhkqGGZu6k2sXB05pA1HEkC9A= 2022-09-03T05:52:52.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5jhkqGGZu6k2sXB05pA1HEkC9A=', 'Content-Length': '0'} 2022-09-03T05:52:52.989 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000670da90ac774ee7b-006312ebb4-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:52.989 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:52.990 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:52.991 DEBUG:boto:Method: HEAD 2022-09-03T05:52:52.991 DEBUG:boto:Path: /avysmw-10/ 2022-09-03T05:52:52.992 DEBUG:boto:Data: 2022-09-03T05:52:52.992 DEBUG:boto:Headers: {} 2022-09-03T05:52:52.993 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:52.993 DEBUG:boto:Port: 8000 2022-09-03T05:52:52.994 DEBUG:boto:Params: {} 2022-09-03T05:52:52.995 DEBUG:boto:Token: None 2022-09-03T05:52:52.995 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:52 GMT /avysmw-10/ 2022-09-03T05:52:52.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5jhkqGGZu6k2sXB05pA1HEkC9A= 2022-09-03T05:52:52.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5jhkqGGZu6k2sXB05pA1HEkC9A=', 'Content-Length': '0'} 2022-09-03T05:52:52.999 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000010892d3b79ccd84f-006312ebb4-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.000 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:52:53.001 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.003 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.004 DEBUG:boto:path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.005 DEBUG:boto:auth_path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.006 DEBUG:boto:Method: GET 2022-09-03T05:52:53.007 DEBUG:boto:Path: /avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.008 DEBUG:boto:Data: 2022-09-03T05:52:53.009 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.009 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.010 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.011 DEBUG:boto:Params: {} 2022-09-03T05:52:53.012 DEBUG:boto:Token: None 2022-09-03T05:52:53.012 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/?versions 2022-09-03T05:52:53.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qpRgpE7e6RlxgXiUx5i9dF8KRhE= 2022-09-03T05:52:53.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qpRgpE7e6RlxgXiUx5i9dF8KRhE=', 'Content-Length': '0'} 2022-09-03T05:52:53.017 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b1d538b183b48c2-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.018 DEBUG:boto:b'avysmw-10999false&nulltrue2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.020 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:52:53.021 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:52:53.021 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:52:53.022 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:52:53.022 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:52:53.023 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.024 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.024 DEBUG:boto:path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.025 DEBUG:boto:auth_path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.025 DEBUG:boto:Method: GET 2022-09-03T05:52:53.026 DEBUG:boto:Path: /avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.027 DEBUG:boto:Data: 2022-09-03T05:52:53.027 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.028 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.028 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.029 DEBUG:boto:Params: {} 2022-09-03T05:52:53.029 DEBUG:boto:Token: None 2022-09-03T05:52:53.030 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/?versions 2022-09-03T05:52:53.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qpRgpE7e6RlxgXiUx5i9dF8KRhE= 2022-09-03T05:52:53.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qpRgpE7e6RlxgXiUx5i9dF8KRhE=', 'Content-Length': '0'} 2022-09-03T05:52:53.034 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff39f02c6caa0a4b-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.035 DEBUG:boto:b'avysmw-10999false&nulltrue2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.036 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:52:53.037 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:52:53.037 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:52:53.038 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:52:53.038 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.039 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.040 DEBUG:boto:path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.041 DEBUG:boto:auth_path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.042 DEBUG:boto:Method: GET 2022-09-03T05:52:53.042 DEBUG:boto:Path: /avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.043 DEBUG:boto:Data: 2022-09-03T05:52:53.044 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.045 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.046 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.047 DEBUG:boto:Params: {} 2022-09-03T05:52:53.048 DEBUG:boto:Token: None 2022-09-03T05:52:53.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/?versions 2022-09-03T05:52:53.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qpRgpE7e6RlxgXiUx5i9dF8KRhE= 2022-09-03T05:52:53.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qpRgpE7e6RlxgXiUx5i9dF8KRhE=', 'Content-Length': '0'} 2022-09-03T05:52:53.054 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011baf8cacd5c1f84-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.055 DEBUG:boto:b'avysmw-10999false&nulltrue2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.056 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.057 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.057 DEBUG:boto:path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.058 DEBUG:boto:auth_path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.059 DEBUG:boto:Method: GET 2022-09-03T05:52:53.059 DEBUG:boto:Path: /avysmw-10/?versions&max-keys=999 2022-09-03T05:52:53.060 DEBUG:boto:Data: 2022-09-03T05:52:53.060 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.061 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.062 DEBUG:boto:Params: {} 2022-09-03T05:52:53.062 DEBUG:boto:Token: None 2022-09-03T05:52:53.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/?versions 2022-09-03T05:52:53.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qpRgpE7e6RlxgXiUx5i9dF8KRhE= 2022-09-03T05:52:53.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qpRgpE7e6RlxgXiUx5i9dF8KRhE=', 'Content-Length': '0'} 2022-09-03T05:52:53.068 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a21007dc0caa8d04-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.069 DEBUG:boto:b'avysmw-10999false&nulltrue2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.071 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:52:53.071 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.072 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.073 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.074 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.074 DEBUG:boto:Method: GET 2022-09-03T05:52:53.075 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.076 DEBUG:boto:Data: 2022-09-03T05:52:53.076 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.077 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.077 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.078 DEBUG:boto:Params: {} 2022-09-03T05:52:53.078 DEBUG:boto:Token: None 2022-09-03T05:52:53.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1e5I82Qc59a57sttaLqu/UCDb0= 2022-09-03T05:52:53.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1e5I82Qc59a57sttaLqu/UCDb0=', 'Content-Length': '0'} 2022-09-03T05:52:53.083 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e933b1ab5cdfdf6a-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.126 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.127 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.128 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.129 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.130 DEBUG:boto:Method: GET 2022-09-03T05:52:53.131 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.132 DEBUG:boto:Data: 2022-09-03T05:52:53.133 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.135 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.136 DEBUG:boto:Params: {} 2022-09-03T05:52:53.137 DEBUG:boto:Token: None 2022-09-03T05:52:53.138 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1e5I82Qc59a57sttaLqu/UCDb0= 2022-09-03T05:52:53.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1e5I82Qc59a57sttaLqu/UCDb0=', 'Content-Length': '0'} 2022-09-03T05:52:53.143 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000055ed01569e33053c-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.145 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.146 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.147 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.147 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.148 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.149 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.150 DEBUG:boto:Data: 2022-09-03T05:52:53.151 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.152 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.153 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.154 DEBUG:boto:Params: {} 2022-09-03T05:52:53.155 DEBUG:boto:Token: None 2022-09-03T05:52:53.156 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VufC0prLK/pq4lTooXqtZ4bpuds= 2022-09-03T05:52:53.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VufC0prLK/pq4lTooXqtZ4bpuds=', 'Content-Length': '0'} 2022-09-03T05:52:53.161 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004354b1d64d7d3584-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.163 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.163 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.164 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.165 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.165 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.166 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.166 DEBUG:boto:Data: 2022-09-03T05:52:53.167 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.168 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.168 DEBUG:boto:Params: {} 2022-09-03T05:52:53.169 DEBUG:boto:Token: None 2022-09-03T05:52:53.170 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VufC0prLK/pq4lTooXqtZ4bpuds= 2022-09-03T05:52:53.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VufC0prLK/pq4lTooXqtZ4bpuds=', 'Content-Length': '0'} 2022-09-03T05:52:53.173 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000058a123d193e9f090-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.174 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:52:53.174 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.175 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.175 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.176 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.177 DEBUG:boto:Method: GET 2022-09-03T05:52:53.178 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.179 DEBUG:boto:Data: 2022-09-03T05:52:53.180 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.181 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.182 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.182 DEBUG:boto:Params: {} 2022-09-03T05:52:53.183 DEBUG:boto:Token: None 2022-09-03T05:52:53.184 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1e5I82Qc59a57sttaLqu/UCDb0= 2022-09-03T05:52:53.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1e5I82Qc59a57sttaLqu/UCDb0=', 'Content-Length': '0'} 2022-09-03T05:52:53.187 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c24b9de9dbad79c7-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.188 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.189 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.189 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.190 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:53.190 DEBUG:boto:Method: GET 2022-09-03T05:52:53.191 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.191 DEBUG:boto:Data: 2022-09-03T05:52:53.192 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.193 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.193 DEBUG:boto:Params: {} 2022-09-03T05:52:53.194 DEBUG:boto:Token: None 2022-09-03T05:52:53.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:53.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1e5I82Qc59a57sttaLqu/UCDb0= 2022-09-03T05:52:53.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1e5I82Qc59a57sttaLqu/UCDb0=', 'Content-Length': '0'} 2022-09-03T05:52:53.198 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002d767a57c4dad5b8-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.246 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.247 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.248 DEBUG:boto:path=/avysmw-10/?prefix=%26 2022-09-03T05:52:53.249 DEBUG:boto:auth_path=/avysmw-10/?prefix=%26 2022-09-03T05:52:53.250 DEBUG:boto:Method: GET 2022-09-03T05:52:53.250 DEBUG:boto:Path: /avysmw-10/?prefix=%26 2022-09-03T05:52:53.251 DEBUG:boto:Data: 2022-09-03T05:52:53.252 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.252 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.253 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.253 DEBUG:boto:Params: {} 2022-09-03T05:52:53.254 DEBUG:boto:Token: None 2022-09-03T05:52:53.255 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.259 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b8458e975c15887f-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.259 DEBUG:boto:b'avysmw-10&1000false&2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.260 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.261 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.261 DEBUG:boto:path=/avysmw-10/?prefix=%26 2022-09-03T05:52:53.262 DEBUG:boto:auth_path=/avysmw-10/?prefix=%26 2022-09-03T05:52:53.263 DEBUG:boto:Method: GET 2022-09-03T05:52:53.263 DEBUG:boto:Path: /avysmw-10/?prefix=%26 2022-09-03T05:52:53.264 DEBUG:boto:Data: 2022-09-03T05:52:53.264 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.265 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.266 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.266 DEBUG:boto:Params: {} 2022-09-03T05:52:53.267 DEBUG:boto:Token: None 2022-09-03T05:52:53.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.272 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004b478c535f7e9e73-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.272 DEBUG:boto:b'avysmw-10&1000false&2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.273 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.274 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.274 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.275 DEBUG:boto:Path: /avysmw-10/%26 2022-09-03T05:52:53.276 DEBUG:boto:Data: 2022-09-03T05:52:53.276 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.277 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.277 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.278 DEBUG:boto:Params: {} 2022-09-03T05:52:53.278 DEBUG:boto:Token: None 2022-09-03T05:52:53.279 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26 2022-09-03T05:52:53.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x3YW0ZbA3+CjQwkabMsjKYSUK8o= 2022-09-03T05:52:53.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x3YW0ZbA3+CjQwkabMsjKYSUK8o=', 'Content-Length': '0'} 2022-09-03T05:52:53.282 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000249f7efab4158cc5-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.283 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.283 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.284 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.285 DEBUG:boto:Path: /avysmw-10/%26 2022-09-03T05:52:53.286 DEBUG:boto:Data: 2022-09-03T05:52:53.286 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.287 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.287 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.288 DEBUG:boto:Params: {} 2022-09-03T05:52:53.288 DEBUG:boto:Token: None 2022-09-03T05:52:53.289 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26 2022-09-03T05:52:53.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x3YW0ZbA3+CjQwkabMsjKYSUK8o= 2022-09-03T05:52:53.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x3YW0ZbA3+CjQwkabMsjKYSUK8o=', 'Content-Length': '0'} 2022-09-03T05:52:53.292 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c6a443a7fa1de2ce-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.293 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:52:53.294 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.295 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.296 DEBUG:boto:Method: GET 2022-09-03T05:52:53.297 DEBUG:boto:Path: /avysmw-10/%26 2022-09-03T05:52:53.298 DEBUG:boto:Data: 2022-09-03T05:52:53.299 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.301 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.302 DEBUG:boto:Params: {} 2022-09-03T05:52:53.303 DEBUG:boto:Token: None 2022-09-03T05:52:53.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26 2022-09-03T05:52:53.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XlbnbMEhiIK27jdUG3bHRcXvueU= 2022-09-03T05:52:53.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XlbnbMEhiIK27jdUG3bHRcXvueU=', 'Content-Length': '0'} 2022-09-03T05:52:53.309 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004ec3f8e10d026b6d-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.351 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:53.352 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:53.353 DEBUG:boto:Method: GET 2022-09-03T05:52:53.354 DEBUG:boto:Path: /avysmw-10/%26 2022-09-03T05:52:53.355 DEBUG:boto:Data: 2022-09-03T05:52:53.356 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.357 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.357 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.358 DEBUG:boto:Params: {} 2022-09-03T05:52:53.359 DEBUG:boto:Token: None 2022-09-03T05:52:53.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%26 2022-09-03T05:52:53.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XlbnbMEhiIK27jdUG3bHRcXvueU= 2022-09-03T05:52:53.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XlbnbMEhiIK27jdUG3bHRcXvueU=', 'Content-Length': '0'} 2022-09-03T05:52:53.363 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dd446970f7bec11d-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.364 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.365 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.365 DEBUG:boto:path=/avysmw-10/?prefix=%26 2022-09-03T05:52:53.366 DEBUG:boto:auth_path=/avysmw-10/?prefix=%26 2022-09-03T05:52:53.367 DEBUG:boto:Method: GET 2022-09-03T05:52:53.367 DEBUG:boto:Path: /avysmw-10/?prefix=%26 2022-09-03T05:52:53.368 DEBUG:boto:Data: 2022-09-03T05:52:53.368 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.369 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.370 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.370 DEBUG:boto:Params: {} 2022-09-03T05:52:53.371 DEBUG:boto:Token: None 2022-09-03T05:52:53.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.376 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e61b69838aee88ad-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.377 DEBUG:boto:b'avysmw-10&1000false&2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.378 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.379 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.379 DEBUG:boto:path=/avysmw-10/?prefix=%26 2022-09-03T05:52:53.380 DEBUG:boto:auth_path=/avysmw-10/?prefix=%26 2022-09-03T05:52:53.381 DEBUG:boto:Method: GET 2022-09-03T05:52:53.381 DEBUG:boto:Path: /avysmw-10/?prefix=%26 2022-09-03T05:52:53.382 DEBUG:boto:Data: 2022-09-03T05:52:53.382 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.385 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.385 DEBUG:boto:Params: {} 2022-09-03T05:52:53.386 DEBUG:boto:Token: None 2022-09-03T05:52:53.387 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.392 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c466811139c1cbf-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.392 DEBUG:boto:b'avysmw-10&1000false&2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.393 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:52:53.394 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.394 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.395 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.396 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.396 DEBUG:boto:Method: GET 2022-09-03T05:52:53.397 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.397 DEBUG:boto:Data: 2022-09-03T05:52:53.398 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.398 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.399 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.399 DEBUG:boto:Params: {} 2022-09-03T05:52:53.400 DEBUG:boto:Token: None 2022-09-03T05:52:53.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FAWIl1PjH+5sJhkCIqSSpgxgrwA= 2022-09-03T05:52:53.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FAWIl1PjH+5sJhkCIqSSpgxgrwA=', 'Content-Length': '0'} 2022-09-03T05:52:53.405 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b4bbf5cded203e1f-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.406 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.406 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.407 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.407 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.408 DEBUG:boto:Method: GET 2022-09-03T05:52:53.409 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.410 DEBUG:boto:Data: 2022-09-03T05:52:53.411 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.413 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.413 DEBUG:boto:Params: {} 2022-09-03T05:52:53.415 DEBUG:boto:Token: None 2022-09-03T05:52:53.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FAWIl1PjH+5sJhkCIqSSpgxgrwA= 2022-09-03T05:52:53.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FAWIl1PjH+5sJhkCIqSSpgxgrwA=', 'Content-Length': '0'} 2022-09-03T05:52:53.419 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007e96a541ae1e552b-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.466 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.467 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.468 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.468 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.469 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.469 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.470 DEBUG:boto:Data: 2022-09-03T05:52:53.470 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.471 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.471 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.472 DEBUG:boto:Params: {} 2022-09-03T05:52:53.472 DEBUG:boto:Token: None 2022-09-03T05:52:53.473 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GrzyZzYaoNJkg9UEVoktOeF2rX4= 2022-09-03T05:52:53.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GrzyZzYaoNJkg9UEVoktOeF2rX4=', 'Content-Length': '0'} 2022-09-03T05:52:53.477 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e3a460abeaff6fb5-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.478 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.478 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.479 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.479 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.480 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.480 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.481 DEBUG:boto:Data: 2022-09-03T05:52:53.481 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.482 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.482 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.483 DEBUG:boto:Params: {} 2022-09-03T05:52:53.484 DEBUG:boto:Token: None 2022-09-03T05:52:53.484 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GrzyZzYaoNJkg9UEVoktOeF2rX4= 2022-09-03T05:52:53.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GrzyZzYaoNJkg9UEVoktOeF2rX4=', 'Content-Length': '0'} 2022-09-03T05:52:53.488 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000884acf27ada36f42-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.489 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:52:53.489 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.490 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.490 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.491 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.492 DEBUG:boto:Method: GET 2022-09-03T05:52:53.492 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.493 DEBUG:boto:Data: 2022-09-03T05:52:53.493 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.494 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.494 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.495 DEBUG:boto:Params: {} 2022-09-03T05:52:53.495 DEBUG:boto:Token: None 2022-09-03T05:52:53.496 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FAWIl1PjH+5sJhkCIqSSpgxgrwA= 2022-09-03T05:52:53.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FAWIl1PjH+5sJhkCIqSSpgxgrwA=', 'Content-Length': '0'} 2022-09-03T05:52:53.500 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001e1419e37b0f46c3-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.501 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.502 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.502 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.503 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.503 DEBUG:boto:Method: GET 2022-09-03T05:52:53.504 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.504 DEBUG:boto:Data: 2022-09-03T05:52:53.505 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.506 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.506 DEBUG:boto:Params: {} 2022-09-03T05:52:53.507 DEBUG:boto:Token: None 2022-09-03T05:52:53.508 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:53.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FAWIl1PjH+5sJhkCIqSSpgxgrwA= 2022-09-03T05:52:53.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FAWIl1PjH+5sJhkCIqSSpgxgrwA=', 'Content-Length': '0'} 2022-09-03T05:52:53.512 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000042d72c3dedd78d6e-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.513 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.513 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.514 DEBUG:boto:path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:53.514 DEBUG:boto:auth_path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:53.515 DEBUG:boto:Method: GET 2022-09-03T05:52:53.516 DEBUG:boto:Path: /avysmw-10/?prefix=%3A 2022-09-03T05:52:53.517 DEBUG:boto:Data: 2022-09-03T05:52:53.518 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.519 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.520 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.521 DEBUG:boto:Params: {} 2022-09-03T05:52:53.522 DEBUG:boto:Token: None 2022-09-03T05:52:53.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.525 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.528 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c77bae4af3f41fc9-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.529 DEBUG:boto:b'avysmw-10:1000false:2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.536 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.537 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.538 DEBUG:boto:path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:53.539 DEBUG:boto:auth_path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:53.539 DEBUG:boto:Method: GET 2022-09-03T05:52:53.540 DEBUG:boto:Path: /avysmw-10/?prefix=%3A 2022-09-03T05:52:53.541 DEBUG:boto:Data: 2022-09-03T05:52:53.541 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.542 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.542 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.543 DEBUG:boto:Params: {} 2022-09-03T05:52:53.543 DEBUG:boto:Token: None 2022-09-03T05:52:53.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.549 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9777adb006b03db-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.549 DEBUG:boto:b'avysmw-10:1000false:2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.550 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.551 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.551 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.552 DEBUG:boto:Path: /avysmw-10/%3A 2022-09-03T05:52:53.552 DEBUG:boto:Data: 2022-09-03T05:52:53.553 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.553 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.554 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.554 DEBUG:boto:Params: {} 2022-09-03T05:52:53.555 DEBUG:boto:Token: None 2022-09-03T05:52:53.556 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A 2022-09-03T05:52:53.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rAJU4kymgwJ7Aui5ZDidEDrwTSc= 2022-09-03T05:52:53.557 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rAJU4kymgwJ7Aui5ZDidEDrwTSc=', 'Content-Length': '0'} 2022-09-03T05:52:53.559 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004e249d6f5326e77b-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.560 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.561 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.561 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.562 DEBUG:boto:Path: /avysmw-10/%3A 2022-09-03T05:52:53.562 DEBUG:boto:Data: 2022-09-03T05:52:53.563 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.563 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.564 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.565 DEBUG:boto:Params: {} 2022-09-03T05:52:53.565 DEBUG:boto:Token: None 2022-09-03T05:52:53.566 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A 2022-09-03T05:52:53.566 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rAJU4kymgwJ7Aui5ZDidEDrwTSc= 2022-09-03T05:52:53.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rAJU4kymgwJ7Aui5ZDidEDrwTSc=', 'Content-Length': '0'} 2022-09-03T05:52:53.569 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ee31f761ec0c6eca-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.570 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:52:53.571 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.571 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.572 DEBUG:boto:Method: GET 2022-09-03T05:52:53.572 DEBUG:boto:Path: /avysmw-10/%3A 2022-09-03T05:52:53.573 DEBUG:boto:Data: 2022-09-03T05:52:53.573 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.574 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.575 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.575 DEBUG:boto:Params: {} 2022-09-03T05:52:53.576 DEBUG:boto:Token: None 2022-09-03T05:52:53.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A 2022-09-03T05:52:53.577 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bPZ+FTDa9n0z6sUu94yiDuvEHcw= 2022-09-03T05:52:53.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bPZ+FTDa9n0z6sUu94yiDuvEHcw=', 'Content-Length': '0'} 2022-09-03T05:52:53.580 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000125bed71d9144d6f-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.581 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:53.582 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:53.583 DEBUG:boto:Method: GET 2022-09-03T05:52:53.583 DEBUG:boto:Path: /avysmw-10/%3A 2022-09-03T05:52:53.584 DEBUG:boto:Data: 2022-09-03T05:52:53.585 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.586 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.587 DEBUG:boto:Params: {} 2022-09-03T05:52:53.587 DEBUG:boto:Token: None 2022-09-03T05:52:53.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/%3A 2022-09-03T05:52:53.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bPZ+FTDa9n0z6sUu94yiDuvEHcw= 2022-09-03T05:52:53.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bPZ+FTDa9n0z6sUu94yiDuvEHcw=', 'Content-Length': '0'} 2022-09-03T05:52:53.591 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006904763d23cd1898-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.592 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.593 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.593 DEBUG:boto:path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:53.594 DEBUG:boto:auth_path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:53.595 DEBUG:boto:Method: GET 2022-09-03T05:52:53.595 DEBUG:boto:Path: /avysmw-10/?prefix=%3A 2022-09-03T05:52:53.596 DEBUG:boto:Data: 2022-09-03T05:52:53.596 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.597 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.597 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.598 DEBUG:boto:Params: {} 2022-09-03T05:52:53.599 DEBUG:boto:Token: None 2022-09-03T05:52:53.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.604 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006ce0120b22029019-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.605 DEBUG:boto:b'avysmw-10:1000false:2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.606 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.607 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.608 DEBUG:boto:path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:53.609 DEBUG:boto:auth_path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:53.610 DEBUG:boto:Method: GET 2022-09-03T05:52:53.611 DEBUG:boto:Path: /avysmw-10/?prefix=%3A 2022-09-03T05:52:53.612 DEBUG:boto:Data: 2022-09-03T05:52:53.613 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.614 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.615 DEBUG:boto:Params: {} 2022-09-03T05:52:53.615 DEBUG:boto:Token: None 2022-09-03T05:52:53.616 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.620 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000371e6e38813381aa-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.621 DEBUG:boto:b'avysmw-10:1000false:2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.622 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:52:53.622 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.623 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.623 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.624 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.625 DEBUG:boto:Method: GET 2022-09-03T05:52:53.625 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.626 DEBUG:boto:Data: 2022-09-03T05:52:53.626 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.627 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.628 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.629 DEBUG:boto:Params: {} 2022-09-03T05:52:53.630 DEBUG:boto:Token: None 2022-09-03T05:52:53.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MGltJKC+nIZdh3kYtuk73D8Ffgk= 2022-09-03T05:52:53.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MGltJKC+nIZdh3kYtuk73D8Ffgk=', 'Content-Length': '0'} 2022-09-03T05:52:53.635 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000039915709838dc397-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.636 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.637 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.638 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.639 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.640 DEBUG:boto:Method: GET 2022-09-03T05:52:53.640 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.642 DEBUG:boto:Data: 2022-09-03T05:52:53.643 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.644 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.645 DEBUG:boto:Params: {} 2022-09-03T05:52:53.646 DEBUG:boto:Token: None 2022-09-03T05:52:53.647 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MGltJKC+nIZdh3kYtuk73D8Ffgk= 2022-09-03T05:52:53.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MGltJKC+nIZdh3kYtuk73D8Ffgk=', 'Content-Length': '0'} 2022-09-03T05:52:53.651 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ab3b69fc666b9551-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.652 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.653 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.653 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.654 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.654 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.655 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.655 DEBUG:boto:Data: 2022-09-03T05:52:53.656 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.657 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.657 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.658 DEBUG:boto:Params: {} 2022-09-03T05:52:53.659 DEBUG:boto:Token: None 2022-09-03T05:52:53.660 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9/18S7LRwYvTRBeGClsDmN8YGpQ= 2022-09-03T05:52:53.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9/18S7LRwYvTRBeGClsDmN8YGpQ=', 'Content-Length': '0'} 2022-09-03T05:52:53.665 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003e660965c04eacc9-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.666 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.666 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.667 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.667 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.668 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.668 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.669 DEBUG:boto:Data: 2022-09-03T05:52:53.669 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.670 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.671 DEBUG:boto:Params: {} 2022-09-03T05:52:53.672 DEBUG:boto:Token: None 2022-09-03T05:52:53.672 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9/18S7LRwYvTRBeGClsDmN8YGpQ= 2022-09-03T05:52:53.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9/18S7LRwYvTRBeGClsDmN8YGpQ=', 'Content-Length': '0'} 2022-09-03T05:52:53.675 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ffc139354e24c270-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.676 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:52:53.676 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.677 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.677 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.678 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.679 DEBUG:boto:Method: GET 2022-09-03T05:52:53.679 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.680 DEBUG:boto:Data: 2022-09-03T05:52:53.680 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.681 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.681 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.682 DEBUG:boto:Params: {} 2022-09-03T05:52:53.682 DEBUG:boto:Token: None 2022-09-03T05:52:53.683 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MGltJKC+nIZdh3kYtuk73D8Ffgk= 2022-09-03T05:52:53.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MGltJKC+nIZdh3kYtuk73D8Ffgk=', 'Content-Length': '0'} 2022-09-03T05:52:53.686 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d16e029c55263e0c-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.687 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.688 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.688 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.689 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.689 DEBUG:boto:Method: GET 2022-09-03T05:52:53.690 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.690 DEBUG:boto:Data: 2022-09-03T05:52:53.691 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.692 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.692 DEBUG:boto:Params: {} 2022-09-03T05:52:53.693 DEBUG:boto:Token: None 2022-09-03T05:52:53.694 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:53.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MGltJKC+nIZdh3kYtuk73D8Ffgk= 2022-09-03T05:52:53.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MGltJKC+nIZdh3kYtuk73D8Ffgk=', 'Content-Length': '0'} 2022-09-03T05:52:53.697 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000650fd411d019f176-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.698 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.699 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.699 DEBUG:boto:path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.700 DEBUG:boto:auth_path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.700 DEBUG:boto:Method: GET 2022-09-03T05:52:53.701 DEBUG:boto:Path: /avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.701 DEBUG:boto:Data: 2022-09-03T05:52:53.702 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.702 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.703 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.703 DEBUG:boto:Params: {} 2022-09-03T05:52:53.704 DEBUG:boto:Token: None 2022-09-03T05:52:53.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.705 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.709 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dd4a67d18aa5aa88-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.750 DEBUG:boto:b'avysmw-10_myobj1000false_myobj2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.752 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.752 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.753 DEBUG:boto:path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.754 DEBUG:boto:auth_path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.754 DEBUG:boto:Method: GET 2022-09-03T05:52:53.755 DEBUG:boto:Path: /avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.755 DEBUG:boto:Data: 2022-09-03T05:52:53.756 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.757 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.757 DEBUG:boto:Params: {} 2022-09-03T05:52:53.758 DEBUG:boto:Token: None 2022-09-03T05:52:53.759 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.763 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c63fedcd052e0019-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.764 DEBUG:boto:b'avysmw-10_myobj1000false_myobj2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.765 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.765 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.766 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.766 DEBUG:boto:Path: /avysmw-10/_myobj 2022-09-03T05:52:53.767 DEBUG:boto:Data: 2022-09-03T05:52:53.767 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.768 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.768 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.769 DEBUG:boto:Params: {} 2022-09-03T05:52:53.769 DEBUG:boto:Token: None 2022-09-03T05:52:53.770 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj 2022-09-03T05:52:53.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nMVhfzsSpHaUcGMZG27kxjbnkCg= 2022-09-03T05:52:53.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nMVhfzsSpHaUcGMZG27kxjbnkCg=', 'Content-Length': '0'} 2022-09-03T05:52:53.775 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000056b75beae50b3279-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.776 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.777 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.777 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.778 DEBUG:boto:Path: /avysmw-10/_myobj 2022-09-03T05:52:53.778 DEBUG:boto:Data: 2022-09-03T05:52:53.779 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.780 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.781 DEBUG:boto:Params: {} 2022-09-03T05:52:53.781 DEBUG:boto:Token: None 2022-09-03T05:52:53.782 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj 2022-09-03T05:52:53.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nMVhfzsSpHaUcGMZG27kxjbnkCg= 2022-09-03T05:52:53.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nMVhfzsSpHaUcGMZG27kxjbnkCg=', 'Content-Length': '0'} 2022-09-03T05:52:53.785 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a1e648bda3ccb288-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.786 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:52:53.787 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.787 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.788 DEBUG:boto:Method: GET 2022-09-03T05:52:53.788 DEBUG:boto:Path: /avysmw-10/_myobj 2022-09-03T05:52:53.789 DEBUG:boto:Data: 2022-09-03T05:52:53.790 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.790 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.791 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.791 DEBUG:boto:Params: {} 2022-09-03T05:52:53.792 DEBUG:boto:Token: None 2022-09-03T05:52:53.792 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj 2022-09-03T05:52:53.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZPlh3Qjzoy68DYiI0Ac5PqmRZgU= 2022-09-03T05:52:53.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZPlh3Qjzoy68DYiI0Ac5PqmRZgU=', 'Content-Length': '0'} 2022-09-03T05:52:53.796 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000693390c44bfdd8ce-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.798 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:53.799 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:53.800 DEBUG:boto:Method: GET 2022-09-03T05:52:53.801 DEBUG:boto:Path: /avysmw-10/_myobj 2022-09-03T05:52:53.802 DEBUG:boto:Data: 2022-09-03T05:52:53.803 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.804 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.805 DEBUG:boto:Params: {} 2022-09-03T05:52:53.805 DEBUG:boto:Token: None 2022-09-03T05:52:53.806 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/_myobj 2022-09-03T05:52:53.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZPlh3Qjzoy68DYiI0Ac5PqmRZgU= 2022-09-03T05:52:53.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZPlh3Qjzoy68DYiI0Ac5PqmRZgU=', 'Content-Length': '0'} 2022-09-03T05:52:53.809 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006c8fcba576c845f0-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.850 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.851 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.852 DEBUG:boto:path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.853 DEBUG:boto:auth_path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.854 DEBUG:boto:Method: GET 2022-09-03T05:52:53.855 DEBUG:boto:Path: /avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.856 DEBUG:boto:Data: 2022-09-03T05:52:53.856 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.857 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.857 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.858 DEBUG:boto:Params: {} 2022-09-03T05:52:53.859 DEBUG:boto:Token: None 2022-09-03T05:52:53.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.863 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000116279ba6ac1f2e4-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.864 DEBUG:boto:b'avysmw-10_myobj1000false_myobj2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.865 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.866 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.866 DEBUG:boto:path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.867 DEBUG:boto:auth_path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.867 DEBUG:boto:Method: GET 2022-09-03T05:52:53.868 DEBUG:boto:Path: /avysmw-10/?prefix=_myobj 2022-09-03T05:52:53.868 DEBUG:boto:Data: 2022-09-03T05:52:53.869 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.870 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.870 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.871 DEBUG:boto:Params: {} 2022-09-03T05:52:53.871 DEBUG:boto:Token: None 2022-09-03T05:52:53.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:53.876 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023db914238cd6b27-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.878 DEBUG:boto:b'avysmw-10_myobj1000false_myobj2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:53.879 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:52:53.880 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:53.881 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:53.882 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.882 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.883 DEBUG:boto:Method: GET 2022-09-03T05:52:53.884 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.885 DEBUG:boto:Data: 2022-09-03T05:52:53.886 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.886 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.887 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.887 DEBUG:boto:Params: {} 2022-09-03T05:52:53.888 DEBUG:boto:Token: None 2022-09-03T05:52:53.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nH/DH3ZbJjCLB09jHIo2IrV+hJM= 2022-09-03T05:52:53.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nH/DH3ZbJjCLB09jHIo2IrV+hJM=', 'Content-Length': '0'} 2022-09-03T05:52:53.892 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000000cf8ba2f072c6f3-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.934 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:53.935 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:53.935 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.936 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.937 DEBUG:boto:Method: GET 2022-09-03T05:52:53.937 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.938 DEBUG:boto:Data: 2022-09-03T05:52:53.939 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.940 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.940 DEBUG:boto:Params: {} 2022-09-03T05:52:53.941 DEBUG:boto:Token: None 2022-09-03T05:52:53.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nH/DH3ZbJjCLB09jHIo2IrV+hJM= 2022-09-03T05:52:53.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nH/DH3ZbJjCLB09jHIo2IrV+hJM=', 'Content-Length': '0'} 2022-09-03T05:52:53.945 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e70d619b22d446f0-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.946 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:53.947 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:53.947 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.948 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.948 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.949 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.949 DEBUG:boto:Data: 2022-09-03T05:52:53.950 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.950 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.951 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.951 DEBUG:boto:Params: {} 2022-09-03T05:52:53.952 DEBUG:boto:Token: None 2022-09-03T05:52:53.953 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yd7cftjxR86kdpjyfAxFMd6JOhI= 2022-09-03T05:52:53.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yd7cftjxR86kdpjyfAxFMd6JOhI=', 'Content-Length': '0'} 2022-09-03T05:52:53.956 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000024a10635fcba167a-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.957 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:53.958 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:53.958 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.959 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.959 DEBUG:boto:Method: HEAD 2022-09-03T05:52:53.960 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.961 DEBUG:boto:Data: 2022-09-03T05:52:53.961 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.962 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.962 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.963 DEBUG:boto:Params: {} 2022-09-03T05:52:53.964 DEBUG:boto:Token: None 2022-09-03T05:52:53.964 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yd7cftjxR86kdpjyfAxFMd6JOhI= 2022-09-03T05:52:53.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yd7cftjxR86kdpjyfAxFMd6JOhI=', 'Content-Length': '0'} 2022-09-03T05:52:53.968 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c734acb2099c5e46-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.969 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:52:53.969 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:53.970 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:53.970 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.971 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.971 DEBUG:boto:Method: GET 2022-09-03T05:52:53.972 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.972 DEBUG:boto:Data: 2022-09-03T05:52:53.973 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.973 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.974 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.974 DEBUG:boto:Params: {} 2022-09-03T05:52:53.975 DEBUG:boto:Token: None 2022-09-03T05:52:53.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nH/DH3ZbJjCLB09jHIo2IrV+hJM= 2022-09-03T05:52:53.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nH/DH3ZbJjCLB09jHIo2IrV+hJM=', 'Content-Length': '0'} 2022-09-03T05:52:53.979 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ad284fbeae78475e-006312ebb5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.980 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:53.980 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:53.981 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.982 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.982 DEBUG:boto:Method: GET 2022-09-03T05:52:53.983 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.983 DEBUG:boto:Data: 2022-09-03T05:52:53.984 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:53.985 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.985 DEBUG:boto:Params: {} 2022-09-03T05:52:53.986 DEBUG:boto:Token: None 2022-09-03T05:52:53.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:53.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nH/DH3ZbJjCLB09jHIo2IrV+hJM= 2022-09-03T05:52:53.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nH/DH3ZbJjCLB09jHIo2IrV+hJM=', 'Content-Length': '0'} 2022-09-03T05:52:53.990 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003a482ab046b56799-006312ebb5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:53.991 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:53.991 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:53.992 DEBUG:boto:path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:53.992 DEBUG:boto:auth_path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:53.993 DEBUG:boto:Method: GET 2022-09-03T05:52:53.993 DEBUG:boto:Path: /avysmw-10/?prefix=myobj 2022-09-03T05:52:53.994 DEBUG:boto:Data: 2022-09-03T05:52:53.994 DEBUG:boto:Headers: {} 2022-09-03T05:52:53.995 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:53.995 DEBUG:boto:Port: 8000 2022-09-03T05:52:53.996 DEBUG:boto:Params: {} 2022-09-03T05:52:53.996 DEBUG:boto:Token: None 2022-09-03T05:52:53.997 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:53 GMT /avysmw-10/ 2022-09-03T05:52:53.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc= 2022-09-03T05:52:53.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/IlgfZGShfz3MMjx7ieG+Tr2xvc=', 'Content-Length': '0'} 2022-09-03T05:52:54.003 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006fa0d6fd92839885-006312ebb6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.004 DEBUG:boto:b'avysmw-10myobj1000falsemyobj2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.005 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.006 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.007 DEBUG:boto:path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:54.008 DEBUG:boto:auth_path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:54.008 DEBUG:boto:Method: GET 2022-09-03T05:52:54.009 DEBUG:boto:Path: /avysmw-10/?prefix=myobj 2022-09-03T05:52:54.010 DEBUG:boto:Data: 2022-09-03T05:52:54.010 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.012 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.012 DEBUG:boto:Params: {} 2022-09-03T05:52:54.013 DEBUG:boto:Token: None 2022-09-03T05:52:54.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.018 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004709bd1204922caa-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.018 DEBUG:boto:b'avysmw-10myobj1000falsemyobj2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.020 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:54.020 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:54.021 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.021 DEBUG:boto:Path: /avysmw-10/myobj 2022-09-03T05:52:54.022 DEBUG:boto:Data: 2022-09-03T05:52:54.022 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.023 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:54.023 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.024 DEBUG:boto:Params: {} 2022-09-03T05:52:54.024 DEBUG:boto:Token: None 2022-09-03T05:52:54.025 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/myobj 2022-09-03T05:52:54.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A1022y9Xni0z0CS+poS4MRP99m8= 2022-09-03T05:52:54.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A1022y9Xni0z0CS+poS4MRP99m8=', 'Content-Length': '0'} 2022-09-03T05:52:54.029 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000023e683b65608dcba-006312ebb6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.029 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:54.030 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:54.031 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.031 DEBUG:boto:Path: /avysmw-10/myobj 2022-09-03T05:52:54.032 DEBUG:boto:Data: 2022-09-03T05:52:54.032 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.033 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.033 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.034 DEBUG:boto:Params: {} 2022-09-03T05:52:54.035 DEBUG:boto:Token: None 2022-09-03T05:52:54.036 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/myobj 2022-09-03T05:52:54.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A1022y9Xni0z0CS+poS4MRP99m8= 2022-09-03T05:52:54.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A1022y9Xni0z0CS+poS4MRP99m8=', 'Content-Length': '0'} 2022-09-03T05:52:54.041 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000718115e75d878816-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.042 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:52:54.043 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:54.044 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:54.045 DEBUG:boto:Method: GET 2022-09-03T05:52:54.045 DEBUG:boto:Path: /avysmw-10/myobj 2022-09-03T05:52:54.046 DEBUG:boto:Data: 2022-09-03T05:52:54.046 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.047 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:54.047 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.048 DEBUG:boto:Params: {} 2022-09-03T05:52:54.048 DEBUG:boto:Token: None 2022-09-03T05:52:54.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/myobj 2022-09-03T05:52:54.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BxCV4dP0F1jSWUDXz8G2Xztljj8= 2022-09-03T05:52:54.050 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BxCV4dP0F1jSWUDXz8G2Xztljj8=', 'Content-Length': '0'} 2022-09-03T05:52:54.052 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001397049b71033d31-006312ebb6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.053 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:54.054 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:54.054 DEBUG:boto:Method: GET 2022-09-03T05:52:54.055 DEBUG:boto:Path: /avysmw-10/myobj 2022-09-03T05:52:54.056 DEBUG:boto:Data: 2022-09-03T05:52:54.056 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.057 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.058 DEBUG:boto:Params: {} 2022-09-03T05:52:54.058 DEBUG:boto:Token: None 2022-09-03T05:52:54.059 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/myobj 2022-09-03T05:52:54.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BxCV4dP0F1jSWUDXz8G2Xztljj8= 2022-09-03T05:52:54.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BxCV4dP0F1jSWUDXz8G2Xztljj8=', 'Content-Length': '0'} 2022-09-03T05:52:54.062 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b263717ff3d6132b-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.106 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.107 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.108 DEBUG:boto:path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:54.108 DEBUG:boto:auth_path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:54.109 DEBUG:boto:Method: GET 2022-09-03T05:52:54.109 DEBUG:boto:Path: /avysmw-10/?prefix=myobj 2022-09-03T05:52:54.110 DEBUG:boto:Data: 2022-09-03T05:52:54.110 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.111 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:52:54.111 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.112 DEBUG:boto:Params: {} 2022-09-03T05:52:54.113 DEBUG:boto:Token: None 2022-09-03T05:52:54.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.117 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000220968aca74a75d-006312ebb6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.118 DEBUG:boto:b'avysmw-10myobj1000falsemyobj2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.119 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.120 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.120 DEBUG:boto:path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:54.121 DEBUG:boto:auth_path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:54.121 DEBUG:boto:Method: GET 2022-09-03T05:52:54.122 DEBUG:boto:Path: /avysmw-10/?prefix=myobj 2022-09-03T05:52:54.122 DEBUG:boto:Data: 2022-09-03T05:52:54.123 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.124 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.124 DEBUG:boto:Params: {} 2022-09-03T05:52:54.125 DEBUG:boto:Token: None 2022-09-03T05:52:54.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.130 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000668ea3c6a30b9c51-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.131 DEBUG:boto:b'avysmw-10myobj1000falsemyobj2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.131 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-10 zones={test-zone2, test-zone1} 2022-09-03T05:52:54.132 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:52:54.132 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:52:54.133 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-10 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:52:54.198 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:54.199 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:54.247 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:54.244+0000 7f9080ddf600 1 bucket sync caught up with source: 2022-09-03T05:52:54.247 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:52:54.247 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:52:54.248 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:52:54.244+0000 7f9080ddf600 0 bucket checkpoint complete 2022-09-03T05:52:54.262 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-10 zones={test-zone3, test-zone1} 2022-09-03T05:52:54.263 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.263 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.264 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.264 DEBUG:boto:Path: /avysmw-10/ 2022-09-03T05:52:54.265 DEBUG:boto:Data: 2022-09-03T05:52:54.265 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.266 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.266 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.267 DEBUG:boto:Params: {} 2022-09-03T05:52:54.267 DEBUG:boto:Token: None 2022-09-03T05:52:54.268 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pnN0Ytk4hFc+m2W4iGoqm0ANNcQ= 2022-09-03T05:52:54.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pnN0Ytk4hFc+m2W4iGoqm0ANNcQ=', 'Content-Length': '0'} 2022-09-03T05:52:54.272 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005e87e0c6db4e3615-006312ebb6-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.273 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.273 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.274 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.274 DEBUG:boto:Path: /avysmw-10/ 2022-09-03T05:52:54.275 DEBUG:boto:Data: 2022-09-03T05:52:54.275 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.276 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.276 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.277 DEBUG:boto:Params: {} 2022-09-03T05:52:54.278 DEBUG:boto:Token: None 2022-09-03T05:52:54.278 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pnN0Ytk4hFc+m2W4iGoqm0ANNcQ= 2022-09-03T05:52:54.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pnN0Ytk4hFc+m2W4iGoqm0ANNcQ=', 'Content-Length': '0'} 2022-09-03T05:52:54.282 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b1e89c9e756524fe-006312ebb6-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.283 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:52:54.283 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.284 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.284 DEBUG:boto:path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.285 DEBUG:boto:auth_path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.285 DEBUG:boto:Method: GET 2022-09-03T05:52:54.286 DEBUG:boto:Path: /avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.287 DEBUG:boto:Data: 2022-09-03T05:52:54.287 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.288 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.289 DEBUG:boto:Params: {} 2022-09-03T05:52:54.290 DEBUG:boto:Token: None 2022-09-03T05:52:54.290 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/?versions 2022-09-03T05:52:54.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nMUNR4K9GwPrCZmgxxQpNXRIOgI= 2022-09-03T05:52:54.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nMUNR4K9GwPrCZmgxxQpNXRIOgI=', 'Content-Length': '0'} 2022-09-03T05:52:54.294 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005b3d4352392a1188-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.295 DEBUG:boto:b'avysmw-10999false&nulltrue2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.296 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:52:54.296 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:52:54.297 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:52:54.297 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:52:54.298 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:52:54.298 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.299 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.300 DEBUG:boto:path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.300 DEBUG:boto:auth_path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.301 DEBUG:boto:Method: GET 2022-09-03T05:52:54.302 DEBUG:boto:Path: /avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.303 DEBUG:boto:Data: 2022-09-03T05:52:54.304 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.304 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.305 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.306 DEBUG:boto:Params: {} 2022-09-03T05:52:54.307 DEBUG:boto:Token: None 2022-09-03T05:52:54.308 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/?versions 2022-09-03T05:52:54.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nMUNR4K9GwPrCZmgxxQpNXRIOgI= 2022-09-03T05:52:54.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nMUNR4K9GwPrCZmgxxQpNXRIOgI=', 'Content-Length': '0'} 2022-09-03T05:52:54.312 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f4afcdc2b120107d-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.312 DEBUG:boto:b'avysmw-10999false&nulltrue2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.313 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:52:54.314 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:52:54.314 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:52:54.315 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:52:54.315 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.316 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.316 DEBUG:boto:path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.317 DEBUG:boto:auth_path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.317 DEBUG:boto:Method: GET 2022-09-03T05:52:54.318 DEBUG:boto:Path: /avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.318 DEBUG:boto:Data: 2022-09-03T05:52:54.319 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.320 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.320 DEBUG:boto:Params: {} 2022-09-03T05:52:54.321 DEBUG:boto:Token: None 2022-09-03T05:52:54.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/?versions 2022-09-03T05:52:54.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nMUNR4K9GwPrCZmgxxQpNXRIOgI= 2022-09-03T05:52:54.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nMUNR4K9GwPrCZmgxxQpNXRIOgI=', 'Content-Length': '0'} 2022-09-03T05:52:54.325 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000082ed44e542d96b08-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.326 DEBUG:boto:b'avysmw-10999false&nulltrue2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.330 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.332 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.333 DEBUG:boto:path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.334 DEBUG:boto:auth_path=/avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.335 DEBUG:boto:Method: GET 2022-09-03T05:52:54.336 DEBUG:boto:Path: /avysmw-10/?versions&max-keys=999 2022-09-03T05:52:54.337 DEBUG:boto:Data: 2022-09-03T05:52:54.337 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.339 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.340 DEBUG:boto:Params: {} 2022-09-03T05:52:54.340 DEBUG:boto:Token: None 2022-09-03T05:52:54.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/?versions 2022-09-03T05:52:54.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nMUNR4K9GwPrCZmgxxQpNXRIOgI= 2022-09-03T05:52:54.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nMUNR4K9GwPrCZmgxxQpNXRIOgI=', 'Content-Length': '0'} 2022-09-03T05:52:54.345 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009898611a15bec794-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.346 DEBUG:boto:b'avysmw-10999false&nulltrue2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.347 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:52:54.347 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.348 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.349 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.349 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.350 DEBUG:boto:Method: GET 2022-09-03T05:52:54.350 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.351 DEBUG:boto:Data: 2022-09-03T05:52:54.351 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.352 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.353 DEBUG:boto:Params: {} 2022-09-03T05:52:54.354 DEBUG:boto:Token: None 2022-09-03T05:52:54.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cFtpmrNlQSyhS6qHMiWXFk5nwl4= 2022-09-03T05:52:54.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cFtpmrNlQSyhS6qHMiWXFk5nwl4=', 'Content-Length': '0'} 2022-09-03T05:52:54.358 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001a6180a88505c6ea-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.359 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.359 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.360 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.361 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.362 DEBUG:boto:Method: GET 2022-09-03T05:52:54.362 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.363 DEBUG:boto:Data: 2022-09-03T05:52:54.363 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.364 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.365 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.366 DEBUG:boto:Params: {} 2022-09-03T05:52:54.366 DEBUG:boto:Token: None 2022-09-03T05:52:54.367 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.368 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cFtpmrNlQSyhS6qHMiWXFk5nwl4= 2022-09-03T05:52:54.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cFtpmrNlQSyhS6qHMiWXFk5nwl4=', 'Content-Length': '0'} 2022-09-03T05:52:54.370 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004b94a274e81a28f1-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.371 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.371 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.372 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.372 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.373 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.373 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.374 DEBUG:boto:Data: 2022-09-03T05:52:54.374 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.375 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.376 DEBUG:boto:Params: {} 2022-09-03T05:52:54.376 DEBUG:boto:Token: None 2022-09-03T05:52:54.377 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OqURm65lHzNs/TBjmcgvLIK6jJ0= 2022-09-03T05:52:54.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OqURm65lHzNs/TBjmcgvLIK6jJ0=', 'Content-Length': '0'} 2022-09-03T05:52:54.380 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001043853d40b34e9c-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.381 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.382 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.383 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.384 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.384 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.385 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.385 DEBUG:boto:Data: 2022-09-03T05:52:54.386 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.387 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.387 DEBUG:boto:Params: {} 2022-09-03T05:52:54.388 DEBUG:boto:Token: None 2022-09-03T05:52:54.388 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OqURm65lHzNs/TBjmcgvLIK6jJ0= 2022-09-03T05:52:54.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OqURm65lHzNs/TBjmcgvLIK6jJ0=', 'Content-Length': '0'} 2022-09-03T05:52:54.391 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000752f890a32dfbcfb-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.392 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:52:54.393 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.393 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.394 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.394 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.395 DEBUG:boto:Method: GET 2022-09-03T05:52:54.396 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.397 DEBUG:boto:Data: 2022-09-03T05:52:54.398 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.399 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.400 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.401 DEBUG:boto:Params: {} 2022-09-03T05:52:54.401 DEBUG:boto:Token: None 2022-09-03T05:52:54.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cFtpmrNlQSyhS6qHMiWXFk5nwl4= 2022-09-03T05:52:54.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cFtpmrNlQSyhS6qHMiWXFk5nwl4=', 'Content-Length': '0'} 2022-09-03T05:52:54.406 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002f4b7aa34b93d192-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.407 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.408 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.408 DEBUG:boto:path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.409 DEBUG:boto:auth_path=/avysmw-10/%26?versionId=null 2022-09-03T05:52:54.409 DEBUG:boto:Method: GET 2022-09-03T05:52:54.410 DEBUG:boto:Path: /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.410 DEBUG:boto:Data: 2022-09-03T05:52:54.411 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.412 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.413 DEBUG:boto:Params: {} 2022-09-03T05:52:54.413 DEBUG:boto:Token: None 2022-09-03T05:52:54.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26?versionId=null 2022-09-03T05:52:54.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cFtpmrNlQSyhS6qHMiWXFk5nwl4= 2022-09-03T05:52:54.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cFtpmrNlQSyhS6qHMiWXFk5nwl4=', 'Content-Length': '0'} 2022-09-03T05:52:54.417 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000047868e7987ceab95-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.458 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.459 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.459 DEBUG:boto:path=/avysmw-10/?prefix=%26 2022-09-03T05:52:54.460 DEBUG:boto:auth_path=/avysmw-10/?prefix=%26 2022-09-03T05:52:54.461 DEBUG:boto:Method: GET 2022-09-03T05:52:54.461 DEBUG:boto:Path: /avysmw-10/?prefix=%26 2022-09-03T05:52:54.462 DEBUG:boto:Data: 2022-09-03T05:52:54.462 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.463 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.464 DEBUG:boto:Params: {} 2022-09-03T05:52:54.465 DEBUG:boto:Token: None 2022-09-03T05:52:54.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.469 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076e284dac7783a7e-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.470 DEBUG:boto:b'avysmw-10&1000false&2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.471 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.472 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.472 DEBUG:boto:path=/avysmw-10/?prefix=%26 2022-09-03T05:52:54.473 DEBUG:boto:auth_path=/avysmw-10/?prefix=%26 2022-09-03T05:52:54.473 DEBUG:boto:Method: GET 2022-09-03T05:52:54.474 DEBUG:boto:Path: /avysmw-10/?prefix=%26 2022-09-03T05:52:54.475 DEBUG:boto:Data: 2022-09-03T05:52:54.475 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.476 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.477 DEBUG:boto:Params: {} 2022-09-03T05:52:54.477 DEBUG:boto:Token: None 2022-09-03T05:52:54.478 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.482 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008975244e2d72c115-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.483 DEBUG:boto:b'avysmw-10&1000false&2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.484 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.484 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.485 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.486 DEBUG:boto:Path: /avysmw-10/%26 2022-09-03T05:52:54.486 DEBUG:boto:Data: 2022-09-03T05:52:54.487 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.488 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.488 DEBUG:boto:Params: {} 2022-09-03T05:52:54.489 DEBUG:boto:Token: None 2022-09-03T05:52:54.489 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26 2022-09-03T05:52:54.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C+MDcmEmi+JxRL8iFqRuDt1KMGk= 2022-09-03T05:52:54.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C+MDcmEmi+JxRL8iFqRuDt1KMGk=', 'Content-Length': '0'} 2022-09-03T05:52:54.494 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000055c94afb4d8fbdca-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.494 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.495 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.496 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.497 DEBUG:boto:Path: /avysmw-10/%26 2022-09-03T05:52:54.498 DEBUG:boto:Data: 2022-09-03T05:52:54.498 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.500 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.501 DEBUG:boto:Params: {} 2022-09-03T05:52:54.502 DEBUG:boto:Token: None 2022-09-03T05:52:54.503 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26 2022-09-03T05:52:54.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C+MDcmEmi+JxRL8iFqRuDt1KMGk= 2022-09-03T05:52:54.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C+MDcmEmi+JxRL8iFqRuDt1KMGk=', 'Content-Length': '0'} 2022-09-03T05:52:54.506 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000073cfcf025101dd42-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.507 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:52:54.507 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.508 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.508 DEBUG:boto:Method: GET 2022-09-03T05:52:54.509 DEBUG:boto:Path: /avysmw-10/%26 2022-09-03T05:52:54.509 DEBUG:boto:Data: 2022-09-03T05:52:54.510 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.510 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.511 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.512 DEBUG:boto:Params: {} 2022-09-03T05:52:54.512 DEBUG:boto:Token: None 2022-09-03T05:52:54.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26 2022-09-03T05:52:54.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dbARu/5ZECPpYOcwlK0MCMgczVU= 2022-09-03T05:52:54.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dbARu/5ZECPpYOcwlK0MCMgczVU=', 'Content-Length': '0'} 2022-09-03T05:52:54.516 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a75ea9d4d2478dca-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.517 DEBUG:boto:path=/avysmw-10/%26 2022-09-03T05:52:54.518 DEBUG:boto:auth_path=/avysmw-10/%26 2022-09-03T05:52:54.519 DEBUG:boto:Method: GET 2022-09-03T05:52:54.519 DEBUG:boto:Path: /avysmw-10/%26 2022-09-03T05:52:54.520 DEBUG:boto:Data: 2022-09-03T05:52:54.520 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.521 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.521 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.522 DEBUG:boto:Params: {} 2022-09-03T05:52:54.523 DEBUG:boto:Token: None 2022-09-03T05:52:54.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%26 2022-09-03T05:52:54.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dbARu/5ZECPpYOcwlK0MCMgczVU= 2022-09-03T05:52:54.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dbARu/5ZECPpYOcwlK0MCMgczVU=', 'Content-Length': '0'} 2022-09-03T05:52:54.526 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000137678f34023e680-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.570 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.571 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.572 DEBUG:boto:path=/avysmw-10/?prefix=%26 2022-09-03T05:52:54.572 DEBUG:boto:auth_path=/avysmw-10/?prefix=%26 2022-09-03T05:52:54.573 DEBUG:boto:Method: GET 2022-09-03T05:52:54.573 DEBUG:boto:Path: /avysmw-10/?prefix=%26 2022-09-03T05:52:54.574 DEBUG:boto:Data: 2022-09-03T05:52:54.575 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.577 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.578 DEBUG:boto:Params: {} 2022-09-03T05:52:54.578 DEBUG:boto:Token: None 2022-09-03T05:52:54.579 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.581 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.583 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003164a2e9c6758b9a-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.584 DEBUG:boto:b'avysmw-10&1000false&2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.585 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.586 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.586 DEBUG:boto:path=/avysmw-10/?prefix=%26 2022-09-03T05:52:54.587 DEBUG:boto:auth_path=/avysmw-10/?prefix=%26 2022-09-03T05:52:54.588 DEBUG:boto:Method: GET 2022-09-03T05:52:54.589 DEBUG:boto:Path: /avysmw-10/?prefix=%26 2022-09-03T05:52:54.590 DEBUG:boto:Data: 2022-09-03T05:52:54.590 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.592 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.593 DEBUG:boto:Params: {} 2022-09-03T05:52:54.594 DEBUG:boto:Token: None 2022-09-03T05:52:54.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.600 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067cf6afb25b51047-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.601 DEBUG:boto:b'avysmw-10&1000false&2022-09-03T05:52:36.824Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.602 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:52:54.603 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.603 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.604 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.605 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.605 DEBUG:boto:Method: GET 2022-09-03T05:52:54.606 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.606 DEBUG:boto:Data: 2022-09-03T05:52:54.607 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.608 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.609 DEBUG:boto:Params: {} 2022-09-03T05:52:54.610 DEBUG:boto:Token: None 2022-09-03T05:52:54.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ggHoOFrkV94kXih2UT2NkjK4jvA= 2022-09-03T05:52:54.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ggHoOFrkV94kXih2UT2NkjK4jvA=', 'Content-Length': '0'} 2022-09-03T05:52:54.615 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004811e8c77f4cb0b0-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.658 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.659 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.660 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.660 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.661 DEBUG:boto:Method: GET 2022-09-03T05:52:54.661 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.662 DEBUG:boto:Data: 2022-09-03T05:52:54.662 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.663 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.664 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.664 DEBUG:boto:Params: {} 2022-09-03T05:52:54.665 DEBUG:boto:Token: None 2022-09-03T05:52:54.666 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ggHoOFrkV94kXih2UT2NkjK4jvA= 2022-09-03T05:52:54.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ggHoOFrkV94kXih2UT2NkjK4jvA=', 'Content-Length': '0'} 2022-09-03T05:52:54.670 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000edb642a17acbad26-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.671 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.671 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.672 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.672 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.673 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.674 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.674 DEBUG:boto:Data: 2022-09-03T05:52:54.675 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.675 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.676 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.676 DEBUG:boto:Params: {} 2022-09-03T05:52:54.677 DEBUG:boto:Token: None 2022-09-03T05:52:54.678 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Bfrt9G/ZpUicgATEsdfyaagfh8= 2022-09-03T05:52:54.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Bfrt9G/ZpUicgATEsdfyaagfh8=', 'Content-Length': '0'} 2022-09-03T05:52:54.681 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000083e0c0aea19350c1-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.682 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.682 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.683 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.683 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.684 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.685 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.685 DEBUG:boto:Data: 2022-09-03T05:52:54.686 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.687 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.687 DEBUG:boto:Params: {} 2022-09-03T05:52:54.688 DEBUG:boto:Token: None 2022-09-03T05:52:54.689 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Bfrt9G/ZpUicgATEsdfyaagfh8= 2022-09-03T05:52:54.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Bfrt9G/ZpUicgATEsdfyaagfh8=', 'Content-Length': '0'} 2022-09-03T05:52:54.694 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001909b7b4b5d114a7-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.696 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:52:54.696 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.697 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.698 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.699 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.700 DEBUG:boto:Method: GET 2022-09-03T05:52:54.701 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.702 DEBUG:boto:Data: 2022-09-03T05:52:54.702 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.704 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.704 DEBUG:boto:Params: {} 2022-09-03T05:52:54.705 DEBUG:boto:Token: None 2022-09-03T05:52:54.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ggHoOFrkV94kXih2UT2NkjK4jvA= 2022-09-03T05:52:54.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ggHoOFrkV94kXih2UT2NkjK4jvA=', 'Content-Length': '0'} 2022-09-03T05:52:54.709 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000006110226f504dc25-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.710 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.711 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.712 DEBUG:boto:path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.712 DEBUG:boto:auth_path=/avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.713 DEBUG:boto:Method: GET 2022-09-03T05:52:54.713 DEBUG:boto:Path: /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.714 DEBUG:boto:Data: 2022-09-03T05:52:54.715 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.715 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.716 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.717 DEBUG:boto:Params: {} 2022-09-03T05:52:54.717 DEBUG:boto:Token: None 2022-09-03T05:52:54.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A?versionId=null 2022-09-03T05:52:54.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ggHoOFrkV94kXih2UT2NkjK4jvA= 2022-09-03T05:52:54.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ggHoOFrkV94kXih2UT2NkjK4jvA=', 'Content-Length': '0'} 2022-09-03T05:52:54.722 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fb0ae45f2406b5bf-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.766 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.767 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.767 DEBUG:boto:path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:54.768 DEBUG:boto:auth_path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:54.768 DEBUG:boto:Method: GET 2022-09-03T05:52:54.769 DEBUG:boto:Path: /avysmw-10/?prefix=%3A 2022-09-03T05:52:54.769 DEBUG:boto:Data: 2022-09-03T05:52:54.770 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.771 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.772 DEBUG:boto:Params: {} 2022-09-03T05:52:54.772 DEBUG:boto:Token: None 2022-09-03T05:52:54.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.777 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000066a1e28081b4ec29-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.778 DEBUG:boto:b'avysmw-10:1000false:2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.779 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.780 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.780 DEBUG:boto:path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:54.781 DEBUG:boto:auth_path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:54.781 DEBUG:boto:Method: GET 2022-09-03T05:52:54.782 DEBUG:boto:Path: /avysmw-10/?prefix=%3A 2022-09-03T05:52:54.782 DEBUG:boto:Data: 2022-09-03T05:52:54.783 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.784 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.784 DEBUG:boto:Params: {} 2022-09-03T05:52:54.785 DEBUG:boto:Token: None 2022-09-03T05:52:54.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.790 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e5b0e805ec0609e-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.791 DEBUG:boto:b'avysmw-10:1000false:2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.792 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.792 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.793 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.794 DEBUG:boto:Path: /avysmw-10/%3A 2022-09-03T05:52:54.794 DEBUG:boto:Data: 2022-09-03T05:52:54.795 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.796 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.796 DEBUG:boto:Params: {} 2022-09-03T05:52:54.797 DEBUG:boto:Token: None 2022-09-03T05:52:54.797 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A 2022-09-03T05:52:54.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rUJ5seAwCLQIzzUIoO2I9Va9sk4= 2022-09-03T05:52:54.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rUJ5seAwCLQIzzUIoO2I9Va9sk4=', 'Content-Length': '0'} 2022-09-03T05:52:54.801 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006125c44d685fd125-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.802 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.802 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.803 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.803 DEBUG:boto:Path: /avysmw-10/%3A 2022-09-03T05:52:54.804 DEBUG:boto:Data: 2022-09-03T05:52:54.804 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.805 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.806 DEBUG:boto:Params: {} 2022-09-03T05:52:54.806 DEBUG:boto:Token: None 2022-09-03T05:52:54.807 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A 2022-09-03T05:52:54.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rUJ5seAwCLQIzzUIoO2I9Va9sk4= 2022-09-03T05:52:54.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rUJ5seAwCLQIzzUIoO2I9Va9sk4=', 'Content-Length': '0'} 2022-09-03T05:52:54.810 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b5dd3c2847bee4f1-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.811 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:52:54.811 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.812 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.813 DEBUG:boto:Method: GET 2022-09-03T05:52:54.813 DEBUG:boto:Path: /avysmw-10/%3A 2022-09-03T05:52:54.814 DEBUG:boto:Data: 2022-09-03T05:52:54.814 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.816 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.816 DEBUG:boto:Params: {} 2022-09-03T05:52:54.817 DEBUG:boto:Token: None 2022-09-03T05:52:54.818 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A 2022-09-03T05:52:54.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q0iNNtKtum9bpAoYhi40iIVar+c= 2022-09-03T05:52:54.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q0iNNtKtum9bpAoYhi40iIVar+c=', 'Content-Length': '0'} 2022-09-03T05:52:54.822 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003b5504d5a80da804-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.822 DEBUG:boto:path=/avysmw-10/%3A 2022-09-03T05:52:54.823 DEBUG:boto:auth_path=/avysmw-10/%3A 2022-09-03T05:52:54.824 DEBUG:boto:Method: GET 2022-09-03T05:52:54.824 DEBUG:boto:Path: /avysmw-10/%3A 2022-09-03T05:52:54.825 DEBUG:boto:Data: 2022-09-03T05:52:54.825 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.826 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.827 DEBUG:boto:Params: {} 2022-09-03T05:52:54.828 DEBUG:boto:Token: None 2022-09-03T05:52:54.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/%3A 2022-09-03T05:52:54.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q0iNNtKtum9bpAoYhi40iIVar+c= 2022-09-03T05:52:54.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q0iNNtKtum9bpAoYhi40iIVar+c=', 'Content-Length': '0'} 2022-09-03T05:52:54.832 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d9adc51c581bff82-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.874 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.875 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.876 DEBUG:boto:path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:54.876 DEBUG:boto:auth_path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:54.877 DEBUG:boto:Method: GET 2022-09-03T05:52:54.878 DEBUG:boto:Path: /avysmw-10/?prefix=%3A 2022-09-03T05:52:54.878 DEBUG:boto:Data: 2022-09-03T05:52:54.879 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.880 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.881 DEBUG:boto:Params: {} 2022-09-03T05:52:54.882 DEBUG:boto:Token: None 2022-09-03T05:52:54.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000057d6c066488971d3-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.890 DEBUG:boto:b'avysmw-10:1000false:2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.892 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:54.893 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:54.898 DEBUG:boto:path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:54.899 DEBUG:boto:auth_path=/avysmw-10/?prefix=%3A 2022-09-03T05:52:54.899 DEBUG:boto:Method: GET 2022-09-03T05:52:54.900 DEBUG:boto:Path: /avysmw-10/?prefix=%3A 2022-09-03T05:52:54.900 DEBUG:boto:Data: 2022-09-03T05:52:54.901 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.902 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.902 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.903 DEBUG:boto:Params: {} 2022-09-03T05:52:54.904 DEBUG:boto:Token: None 2022-09-03T05:52:54.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/ 2022-09-03T05:52:54.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA= 2022-09-03T05:52:54.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRPatH5VqMbqbK2GtKRNxs6XHNA=', 'Content-Length': '0'} 2022-09-03T05:52:54.909 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003d19d27d602dd2bf-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.909 DEBUG:boto:b'avysmw-10:1000false:2022-09-03T05:52:36.787Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:54.910 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:52:54.911 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:54.912 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:54.913 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.914 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.915 DEBUG:boto:Method: GET 2022-09-03T05:52:54.916 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.917 DEBUG:boto:Data: 2022-09-03T05:52:54.919 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.921 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.922 DEBUG:boto:Params: {} 2022-09-03T05:52:54.923 DEBUG:boto:Token: None 2022-09-03T05:52:54.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P1otOqx9bOS/qGhXgJYgznMIPhg= 2022-09-03T05:52:54.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P1otOqx9bOS/qGhXgJYgznMIPhg=', 'Content-Length': '0'} 2022-09-03T05:52:54.931 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008a25dcc64eed7c7a-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.932 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:54.933 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:54.933 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.934 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.935 DEBUG:boto:Method: GET 2022-09-03T05:52:54.936 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.937 DEBUG:boto:Data: 2022-09-03T05:52:54.938 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.940 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.941 DEBUG:boto:Params: {} 2022-09-03T05:52:54.942 DEBUG:boto:Token: None 2022-09-03T05:52:54.944 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P1otOqx9bOS/qGhXgJYgznMIPhg= 2022-09-03T05:52:54.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P1otOqx9bOS/qGhXgJYgznMIPhg=', 'Content-Length': '0'} 2022-09-03T05:52:54.949 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007283d5951068b9c6-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.950 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:54.951 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:54.952 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.952 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.953 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.954 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.954 DEBUG:boto:Data: 2022-09-03T05:52:54.955 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.956 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.957 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.957 DEBUG:boto:Params: {} 2022-09-03T05:52:54.958 DEBUG:boto:Token: None 2022-09-03T05:52:54.959 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xf4LLzFMgXN1mfgVtfySs3I8xF4= 2022-09-03T05:52:54.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xf4LLzFMgXN1mfgVtfySs3I8xF4=', 'Content-Length': '0'} 2022-09-03T05:52:54.963 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000e44efc6e6a074cf-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.964 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:54.965 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:54.966 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.967 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.967 DEBUG:boto:Method: HEAD 2022-09-03T05:52:54.968 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.969 DEBUG:boto:Data: 2022-09-03T05:52:54.969 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.970 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:54.970 DEBUG:boto:Port: 8000 2022-09-03T05:52:54.971 DEBUG:boto:Params: {} 2022-09-03T05:52:54.972 DEBUG:boto:Token: None 2022-09-03T05:52:54.973 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.973 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xf4LLzFMgXN1mfgVtfySs3I8xF4= 2022-09-03T05:52:54.974 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xf4LLzFMgXN1mfgVtfySs3I8xF4=', 'Content-Length': '0'} 2022-09-03T05:52:54.976 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004e78eb470f5aae9c-006312ebb6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:54.977 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:52:54.978 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:54.978 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:54.979 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.980 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.980 DEBUG:boto:Method: GET 2022-09-03T05:52:54.981 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.981 DEBUG:boto:Data: 2022-09-03T05:52:54.982 DEBUG:boto:Headers: {} 2022-09-03T05:52:54.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:54.983 DEBUG:boto:Port: 8001 2022-09-03T05:52:54.984 DEBUG:boto:Params: {} 2022-09-03T05:52:54.985 DEBUG:boto:Token: None 2022-09-03T05:52:54.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:54 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:54.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P1otOqx9bOS/qGhXgJYgznMIPhg= 2022-09-03T05:52:54.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P1otOqx9bOS/qGhXgJYgznMIPhg=', 'Content-Length': '0'} 2022-09-03T05:52:54.990 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e88bf91cd8f35867-006312ebb6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.030 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:55.031 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:55.032 DEBUG:boto:path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:55.033 DEBUG:boto:auth_path=/avysmw-10/_myobj?versionId=null 2022-09-03T05:52:55.034 DEBUG:boto:Method: GET 2022-09-03T05:52:55.034 DEBUG:boto:Path: /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:55.035 DEBUG:boto:Data: 2022-09-03T05:52:55.035 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.037 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.038 DEBUG:boto:Params: {} 2022-09-03T05:52:55.039 DEBUG:boto:Token: None 2022-09-03T05:52:55.040 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/_myobj?versionId=null 2022-09-03T05:52:55.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BoMx9PldzVW+JCfiucIzrgj7x9U= 2022-09-03T05:52:55.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BoMx9PldzVW+JCfiucIzrgj7x9U=', 'Content-Length': '0'} 2022-09-03T05:52:55.045 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002a9e955df846f62f-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.047 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:55.049 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:55.050 DEBUG:boto:path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.051 DEBUG:boto:auth_path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.052 DEBUG:boto:Method: GET 2022-09-03T05:52:55.053 DEBUG:boto:Path: /avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.054 DEBUG:boto:Data: 2022-09-03T05:52:55.055 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.055 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.056 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.057 DEBUG:boto:Params: {} 2022-09-03T05:52:55.057 DEBUG:boto:Token: None 2022-09-03T05:52:55.058 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/ 2022-09-03T05:52:55.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4= 2022-09-03T05:52:55.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4=', 'Content-Length': '0'} 2022-09-03T05:52:55.063 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a5efa40018341f65-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.063 DEBUG:boto:b'avysmw-10_myobj1000false_myobj2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:55.064 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:55.065 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:55.067 DEBUG:boto:path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.067 DEBUG:boto:auth_path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.068 DEBUG:boto:Method: GET 2022-09-03T05:52:55.069 DEBUG:boto:Path: /avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.069 DEBUG:boto:Data: 2022-09-03T05:52:55.070 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.071 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.071 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.072 DEBUG:boto:Params: {} 2022-09-03T05:52:55.073 DEBUG:boto:Token: None 2022-09-03T05:52:55.073 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/ 2022-09-03T05:52:55.074 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4= 2022-09-03T05:52:55.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4=', 'Content-Length': '0'} 2022-09-03T05:52:55.078 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e57cea3041f5ff67-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.079 DEBUG:boto:b'avysmw-10_myobj1000false_myobj2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:55.080 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:55.080 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:55.081 DEBUG:boto:Method: HEAD 2022-09-03T05:52:55.082 DEBUG:boto:Path: /avysmw-10/_myobj 2022-09-03T05:52:55.083 DEBUG:boto:Data: 2022-09-03T05:52:55.084 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.086 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.087 DEBUG:boto:Params: {} 2022-09-03T05:52:55.088 DEBUG:boto:Token: None 2022-09-03T05:52:55.089 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/_myobj 2022-09-03T05:52:55.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqjjP61Zc+wBowFdU0ZWBuXYsEw= 2022-09-03T05:52:55.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqjjP61Zc+wBowFdU0ZWBuXYsEw=', 'Content-Length': '0'} 2022-09-03T05:52:55.093 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bfc5768e7ac20296-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.094 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:55.094 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:55.095 DEBUG:boto:Method: HEAD 2022-09-03T05:52:55.095 DEBUG:boto:Path: /avysmw-10/_myobj 2022-09-03T05:52:55.096 DEBUG:boto:Data: 2022-09-03T05:52:55.097 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.098 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.099 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.100 DEBUG:boto:Params: {} 2022-09-03T05:52:55.101 DEBUG:boto:Token: None 2022-09-03T05:52:55.102 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/_myobj 2022-09-03T05:52:55.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqjjP61Zc+wBowFdU0ZWBuXYsEw= 2022-09-03T05:52:55.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqjjP61Zc+wBowFdU0ZWBuXYsEw=', 'Content-Length': '0'} 2022-09-03T05:52:55.106 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cb59a66228119f66-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.107 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:52:55.108 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:55.109 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:55.110 DEBUG:boto:Method: GET 2022-09-03T05:52:55.111 DEBUG:boto:Path: /avysmw-10/_myobj 2022-09-03T05:52:55.112 DEBUG:boto:Data: 2022-09-03T05:52:55.113 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.115 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.116 DEBUG:boto:Params: {} 2022-09-03T05:52:55.117 DEBUG:boto:Token: None 2022-09-03T05:52:55.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/_myobj 2022-09-03T05:52:55.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U8pjQVIb7T129lh1FSosr77mKq4= 2022-09-03T05:52:55.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U8pjQVIb7T129lh1FSosr77mKq4=', 'Content-Length': '0'} 2022-09-03T05:52:55.122 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007c24cae73678e500-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.123 DEBUG:boto:path=/avysmw-10/_myobj 2022-09-03T05:52:55.124 DEBUG:boto:auth_path=/avysmw-10/_myobj 2022-09-03T05:52:55.125 DEBUG:boto:Method: GET 2022-09-03T05:52:55.126 DEBUG:boto:Path: /avysmw-10/_myobj 2022-09-03T05:52:55.127 DEBUG:boto:Data: 2022-09-03T05:52:55.128 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.129 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.129 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.130 DEBUG:boto:Params: {} 2022-09-03T05:52:55.132 DEBUG:boto:Token: None 2022-09-03T05:52:55.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/_myobj 2022-09-03T05:52:55.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U8pjQVIb7T129lh1FSosr77mKq4= 2022-09-03T05:52:55.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U8pjQVIb7T129lh1FSosr77mKq4=', 'Content-Length': '0'} 2022-09-03T05:52:55.137 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d7cba6c21736e7ea-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.178 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:55.179 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:55.181 DEBUG:boto:path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.182 DEBUG:boto:auth_path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.183 DEBUG:boto:Method: GET 2022-09-03T05:52:55.184 DEBUG:boto:Path: /avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.184 DEBUG:boto:Data: 2022-09-03T05:52:55.185 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.186 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.187 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.188 DEBUG:boto:Params: {} 2022-09-03T05:52:55.189 DEBUG:boto:Token: None 2022-09-03T05:52:55.190 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/ 2022-09-03T05:52:55.191 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4= 2022-09-03T05:52:55.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4=', 'Content-Length': '0'} 2022-09-03T05:52:55.194 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000042107e7a897e8945-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.195 DEBUG:boto:b'avysmw-10_myobj1000false_myobj2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:55.196 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:55.196 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:55.197 DEBUG:boto:path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.197 DEBUG:boto:auth_path=/avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.198 DEBUG:boto:Method: GET 2022-09-03T05:52:55.199 DEBUG:boto:Path: /avysmw-10/?prefix=_myobj 2022-09-03T05:52:55.200 DEBUG:boto:Data: 2022-09-03T05:52:55.200 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.201 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.203 DEBUG:boto:Params: {} 2022-09-03T05:52:55.204 DEBUG:boto:Token: None 2022-09-03T05:52:55.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/ 2022-09-03T05:52:55.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4= 2022-09-03T05:52:55.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4=', 'Content-Length': '0'} 2022-09-03T05:52:55.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000013777c1e450dd1ef-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.210 DEBUG:boto:b'avysmw-10_myobj1000false_myobj2022-09-03T05:52:36.730Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:55.211 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:52:55.212 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.213 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.214 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.215 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.216 DEBUG:boto:Method: GET 2022-09-03T05:52:55.216 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.217 DEBUG:boto:Data: 2022-09-03T05:52:55.218 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.219 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.219 DEBUG:boto:Params: {} 2022-09-03T05:52:55.220 DEBUG:boto:Token: None 2022-09-03T05:52:55.221 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VV+ksWJvxeRS1D3wjPd6iJvcIvQ= 2022-09-03T05:52:55.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VV+ksWJvxeRS1D3wjPd6iJvcIvQ=', 'Content-Length': '0'} 2022-09-03T05:52:55.224 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000e10048f42f6ce68-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.270 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.271 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.271 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.272 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.273 DEBUG:boto:Method: GET 2022-09-03T05:52:55.273 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.274 DEBUG:boto:Data: 2022-09-03T05:52:55.274 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.275 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.275 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.276 DEBUG:boto:Params: {} 2022-09-03T05:52:55.276 DEBUG:boto:Token: None 2022-09-03T05:52:55.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VV+ksWJvxeRS1D3wjPd6iJvcIvQ= 2022-09-03T05:52:55.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VV+ksWJvxeRS1D3wjPd6iJvcIvQ=', 'Content-Length': '0'} 2022-09-03T05:52:55.281 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fd0eb0b12aadbd84-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.281 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.282 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.282 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.283 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.283 DEBUG:boto:Method: HEAD 2022-09-03T05:52:55.284 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.284 DEBUG:boto:Data: 2022-09-03T05:52:55.285 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.286 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.286 DEBUG:boto:Params: {} 2022-09-03T05:52:55.287 DEBUG:boto:Token: None 2022-09-03T05:52:55.288 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qVO5OLbgMfI1Hu+RtpY1IX3giQs= 2022-09-03T05:52:55.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qVO5OLbgMfI1Hu+RtpY1IX3giQs=', 'Content-Length': '0'} 2022-09-03T05:52:55.291 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e5c94aaf6d6b2311-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.291 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.292 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.293 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.293 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.293 DEBUG:boto:Method: HEAD 2022-09-03T05:52:55.294 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.295 DEBUG:boto:Data: 2022-09-03T05:52:55.295 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.296 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.296 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.296 DEBUG:boto:Params: {} 2022-09-03T05:52:55.297 DEBUG:boto:Token: None 2022-09-03T05:52:55.298 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qVO5OLbgMfI1Hu+RtpY1IX3giQs= 2022-09-03T05:52:55.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qVO5OLbgMfI1Hu+RtpY1IX3giQs=', 'Content-Length': '0'} 2022-09-03T05:52:55.301 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009877cf20e1a22a79-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.301 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:52:55.302 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.303 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.303 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.304 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.304 DEBUG:boto:Method: GET 2022-09-03T05:52:55.305 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.305 DEBUG:boto:Data: 2022-09-03T05:52:55.306 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.306 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.307 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.307 DEBUG:boto:Params: {} 2022-09-03T05:52:55.308 DEBUG:boto:Token: None 2022-09-03T05:52:55.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VV+ksWJvxeRS1D3wjPd6iJvcIvQ= 2022-09-03T05:52:55.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VV+ksWJvxeRS1D3wjPd6iJvcIvQ=', 'Content-Length': '0'} 2022-09-03T05:52:55.312 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000df3a3d8154528165-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.313 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.313 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.314 DEBUG:boto:path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.314 DEBUG:boto:auth_path=/avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.315 DEBUG:boto:Method: GET 2022-09-03T05:52:55.316 DEBUG:boto:Path: /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.316 DEBUG:boto:Data: 2022-09-03T05:52:55.317 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.317 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.318 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.318 DEBUG:boto:Params: {} 2022-09-03T05:52:55.319 DEBUG:boto:Token: None 2022-09-03T05:52:55.319 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj?versionId=null 2022-09-03T05:52:55.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VV+ksWJvxeRS1D3wjPd6iJvcIvQ= 2022-09-03T05:52:55.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VV+ksWJvxeRS1D3wjPd6iJvcIvQ=', 'Content-Length': '0'} 2022-09-03T05:52:55.323 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e4a01bb504795df7-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.323 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:55.324 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:55.324 DEBUG:boto:path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:55.325 DEBUG:boto:auth_path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:55.325 DEBUG:boto:Method: GET 2022-09-03T05:52:55.326 DEBUG:boto:Path: /avysmw-10/?prefix=myobj 2022-09-03T05:52:55.327 DEBUG:boto:Data: 2022-09-03T05:52:55.327 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.328 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.328 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.329 DEBUG:boto:Params: {} 2022-09-03T05:52:55.329 DEBUG:boto:Token: None 2022-09-03T05:52:55.330 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/ 2022-09-03T05:52:55.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4= 2022-09-03T05:52:55.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4=', 'Content-Length': '0'} 2022-09-03T05:52:55.334 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001287678da521ac0e-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.335 DEBUG:boto:b'avysmw-10myobj1000falsemyobj2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:55.336 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:55.336 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:55.337 DEBUG:boto:path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:55.337 DEBUG:boto:auth_path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:55.338 DEBUG:boto:Method: GET 2022-09-03T05:52:55.338 DEBUG:boto:Path: /avysmw-10/?prefix=myobj 2022-09-03T05:52:55.339 DEBUG:boto:Data: 2022-09-03T05:52:55.339 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.340 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.341 DEBUG:boto:Params: {} 2022-09-03T05:52:55.342 DEBUG:boto:Token: None 2022-09-03T05:52:55.342 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/ 2022-09-03T05:52:55.343 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4= 2022-09-03T05:52:55.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4=', 'Content-Length': '0'} 2022-09-03T05:52:55.346 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008650d849320e356a-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.347 DEBUG:boto:b'avysmw-10myobj1000falsemyobj2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:55.348 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.348 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.349 DEBUG:boto:Method: HEAD 2022-09-03T05:52:55.349 DEBUG:boto:Path: /avysmw-10/myobj 2022-09-03T05:52:55.350 DEBUG:boto:Data: 2022-09-03T05:52:55.350 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.351 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.352 DEBUG:boto:Params: {} 2022-09-03T05:52:55.352 DEBUG:boto:Token: None 2022-09-03T05:52:55.353 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj 2022-09-03T05:52:55.353 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uJNpxBJspI2O7G6Io6N0pRkWaFs= 2022-09-03T05:52:55.354 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uJNpxBJspI2O7G6Io6N0pRkWaFs=', 'Content-Length': '0'} 2022-09-03T05:52:55.356 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000747e8e4fef69d2dc-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.357 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.357 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.358 DEBUG:boto:Method: HEAD 2022-09-03T05:52:55.358 DEBUG:boto:Path: /avysmw-10/myobj 2022-09-03T05:52:55.359 DEBUG:boto:Data: 2022-09-03T05:52:55.359 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.360 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.361 DEBUG:boto:Params: {} 2022-09-03T05:52:55.362 DEBUG:boto:Token: None 2022-09-03T05:52:55.362 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj 2022-09-03T05:52:55.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uJNpxBJspI2O7G6Io6N0pRkWaFs= 2022-09-03T05:52:55.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uJNpxBJspI2O7G6Io6N0pRkWaFs=', 'Content-Length': '0'} 2022-09-03T05:52:55.365 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000311c4097a738e7f6-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.366 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:52:55.367 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.367 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.368 DEBUG:boto:Method: GET 2022-09-03T05:52:55.368 DEBUG:boto:Path: /avysmw-10/myobj 2022-09-03T05:52:55.369 DEBUG:boto:Data: 2022-09-03T05:52:55.369 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.371 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.371 DEBUG:boto:Params: {} 2022-09-03T05:52:55.372 DEBUG:boto:Token: None 2022-09-03T05:52:55.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj 2022-09-03T05:52:55.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kikt/Fdm0m8YNSIOAVmC7JSQbgI= 2022-09-03T05:52:55.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kikt/Fdm0m8YNSIOAVmC7JSQbgI=', 'Content-Length': '0'} 2022-09-03T05:52:55.376 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000586ef97fd9c2004e-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.422 DEBUG:boto:path=/avysmw-10/myobj 2022-09-03T05:52:55.423 DEBUG:boto:auth_path=/avysmw-10/myobj 2022-09-03T05:52:55.423 DEBUG:boto:Method: GET 2022-09-03T05:52:55.424 DEBUG:boto:Path: /avysmw-10/myobj 2022-09-03T05:52:55.424 DEBUG:boto:Data: 2022-09-03T05:52:55.425 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.425 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.426 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.426 DEBUG:boto:Params: {} 2022-09-03T05:52:55.427 DEBUG:boto:Token: None 2022-09-03T05:52:55.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/myobj 2022-09-03T05:52:55.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kikt/Fdm0m8YNSIOAVmC7JSQbgI= 2022-09-03T05:52:55.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kikt/Fdm0m8YNSIOAVmC7JSQbgI=', 'Content-Length': '0'} 2022-09-03T05:52:55.431 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007d8a781ea5805efa-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.432 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:55.432 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:55.433 DEBUG:boto:path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:55.434 DEBUG:boto:auth_path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:55.434 DEBUG:boto:Method: GET 2022-09-03T05:52:55.435 DEBUG:boto:Path: /avysmw-10/?prefix=myobj 2022-09-03T05:52:55.435 DEBUG:boto:Data: 2022-09-03T05:52:55.436 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.436 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:52:55.437 DEBUG:boto:Port: 8001 2022-09-03T05:52:55.437 DEBUG:boto:Params: {} 2022-09-03T05:52:55.438 DEBUG:boto:Token: None 2022-09-03T05:52:55.439 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/ 2022-09-03T05:52:55.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4= 2022-09-03T05:52:55.440 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4=', 'Content-Length': '0'} 2022-09-03T05:52:55.443 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8c9e6d37b7e627e-006312ebb7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.443 DEBUG:boto:b'avysmw-10myobj1000falsemyobj2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:55.444 DEBUG:boto:path=/avysmw-10/ 2022-09-03T05:52:55.445 DEBUG:boto:auth_path=/avysmw-10/ 2022-09-03T05:52:55.445 DEBUG:boto:path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:55.446 DEBUG:boto:auth_path=/avysmw-10/?prefix=myobj 2022-09-03T05:52:55.446 DEBUG:boto:Method: GET 2022-09-03T05:52:55.447 DEBUG:boto:Path: /avysmw-10/?prefix=myobj 2022-09-03T05:52:55.447 DEBUG:boto:Data: 2022-09-03T05:52:55.448 DEBUG:boto:Headers: {} 2022-09-03T05:52:55.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:52:55.449 DEBUG:boto:Port: 8000 2022-09-03T05:52:55.450 DEBUG:boto:Params: {} 2022-09-03T05:52:55.450 DEBUG:boto:Token: None 2022-09-03T05:52:55.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:52:55 GMT /avysmw-10/ 2022-09-03T05:52:55.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4= 2022-09-03T05:52:55.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:52:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tdwg33ac4e9GRo6+0xBBdX37CM4=', 'Content-Length': '0'} 2022-09-03T05:52:55.455 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002ab4eedf6e8a1a35-006312ebb7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:52:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:52:55.456 DEBUG:boto:b'avysmw-10myobj1000falsemyobj2022-09-03T05:52:36.685Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:52:55.457 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-10 zones={test-zone3, test-zone1} 2022-09-03T05:52:55.457 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:52:55.458 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:52:55.459 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-10 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:52:55.523 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:52:55.523 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:52:55.559 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:52:55.558+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:52:55.560 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:52:55.560 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:00.561 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:00.559+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:00.561 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:53:00.562 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:05.562 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:05.560+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:05.563 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:53:05.563 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:10.564 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:10.562+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:10.564 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:53:10.565 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:15.565 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:15.563+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:15.566 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:53:15.566 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:20.566 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:20.564+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:20.567 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:53:20.567 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:25.568 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:25.566+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:25.568 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:53:25.569 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:30.569 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:30.567+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:30.570 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:53:30.570 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:35.571 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:35.569+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:35.571 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:53:35.571 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:40.572 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:40.570+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:40.573 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2022-09-03T05:53:40.573 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:45.574 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:45.571+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:45.574 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, , , ] 2022-09-03T05:53:45.575 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:50.576 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:50.573+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:50.576 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, , , ] 2022-09-03T05:53:50.577 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:53:55.576 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:53:55.574+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:53:55.577 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, , , ] 2022-09-03T05:53:55.577 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:54:00.578 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:00.576+0000 7fa217b03600 1 waiting for incremental sync to catch up: 2022-09-03T05:54:00.578 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, , , ] 2022-09-03T05:54:00.579 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:54:05.579 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:05.577+0000 7fa217b03600 1 bucket sync caught up with source: 2022-09-03T05:54:05.580 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:54:05.580 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.7.5, , 00000000001.5.5, , , , 00000000001.3.5, 00000000001.5.5, , ] 2022-09-03T05:54:05.580 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:05.577+0000 7fa217b03600 0 bucket checkpoint complete 2022-09-03T05:54:05.601 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-11', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:05.602 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-11', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:05.603 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-11 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:05.671 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:05.672 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:05.729 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:05.727+0000 7f15a0b61600 1 bucket sync caught up with source: 2022-09-03T05:54:05.730 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.16.5, , 00000000001.20.5, , , , 00000000001.22.5, 00000000001.14.5, , ] 2022-09-03T05:54:05.730 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.16.5, , 00000000001.20.5, , , , 00000000001.22.5, 00000000001.14.5, , ] 2022-09-03T05:54:05.730 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:05.727+0000 7f15a0b61600 0 bucket checkpoint complete 2022-09-03T05:54:05.753 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-11 zones={test-zone1, test-zone2} 2022-09-03T05:54:05.754 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:05.755 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:05.756 DEBUG:boto:Method: HEAD 2022-09-03T05:54:05.756 DEBUG:boto:Path: /avysmw-11/ 2022-09-03T05:54:05.757 DEBUG:boto:Data: 2022-09-03T05:54:05.757 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.759 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.759 DEBUG:boto:Params: {} 2022-09-03T05:54:05.760 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T05:54:05.761 DEBUG:boto:Token: None 2022-09-03T05:54:05.762 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/ 2022-09-03T05:54:05.763 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AkzWW266jcye7G5SKCDTSoOjtus= 2022-09-03T05:54:05.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AkzWW266jcye7G5SKCDTSoOjtus=', 'Content-Length': '0'} 2022-09-03T05:54:05.767 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000002c05b013a2f6b3-006312ebfd-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.768 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:05.769 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:05.769 DEBUG:boto:Method: HEAD 2022-09-03T05:54:05.770 DEBUG:boto:Path: /avysmw-11/ 2022-09-03T05:54:05.770 DEBUG:boto:Data: 2022-09-03T05:54:05.771 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.771 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:05.773 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.773 DEBUG:boto:Params: {} 2022-09-03T05:54:05.774 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T05:54:05.775 DEBUG:boto:Token: None 2022-09-03T05:54:05.776 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/ 2022-09-03T05:54:05.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AkzWW266jcye7G5SKCDTSoOjtus= 2022-09-03T05:54:05.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AkzWW266jcye7G5SKCDTSoOjtus=', 'Content-Length': '0'} 2022-09-03T05:54:05.781 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005d1232dab74a2a5e-006312ebfd-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.782 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:05.783 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:05.783 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:05.784 DEBUG:boto:path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.784 DEBUG:boto:auth_path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.785 DEBUG:boto:Method: GET 2022-09-03T05:54:05.785 DEBUG:boto:Path: /avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.786 DEBUG:boto:Data: 2022-09-03T05:54:05.787 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.788 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.789 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.790 DEBUG:boto:Params: {} 2022-09-03T05:54:05.792 DEBUG:boto:Token: None 2022-09-03T05:54:05.792 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/?versions 2022-09-03T05:54:05.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQW/fD8msqMEvy4ve3FvZr3a6Ec= 2022-09-03T05:54:05.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQW/fD8msqMEvy4ve3FvZr3a6Ec=', 'Content-Length': '0'} 2022-09-03T05:54:05.796 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c99f58e2d2a5caf-006312ebfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.797 DEBUG:boto:b'avysmw-11999false&nulltrue2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:05.798 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:54:05.799 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:54:05.799 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:54:05.800 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:05.800 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:05.801 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:05.801 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:05.802 DEBUG:boto:path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.802 DEBUG:boto:auth_path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.803 DEBUG:boto:Method: GET 2022-09-03T05:54:05.804 DEBUG:boto:Path: /avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.804 DEBUG:boto:Data: 2022-09-03T05:54:05.805 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.806 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:05.807 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.808 DEBUG:boto:Params: {} 2022-09-03T05:54:05.809 DEBUG:boto:Token: None 2022-09-03T05:54:05.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/?versions 2022-09-03T05:54:05.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQW/fD8msqMEvy4ve3FvZr3a6Ec= 2022-09-03T05:54:05.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQW/fD8msqMEvy4ve3FvZr3a6Ec=', 'Content-Length': '0'} 2022-09-03T05:54:05.814 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9abc580d1bb8a0d-006312ebfd-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.815 DEBUG:boto:b'avysmw-11999false&nulltrue2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:05.816 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:54:05.817 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:54:05.817 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:54:05.817 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:05.818 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:05.819 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:05.819 DEBUG:boto:path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.820 DEBUG:boto:auth_path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.820 DEBUG:boto:Method: GET 2022-09-03T05:54:05.821 DEBUG:boto:Path: /avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.821 DEBUG:boto:Data: 2022-09-03T05:54:05.822 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.823 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.823 DEBUG:boto:Params: {} 2022-09-03T05:54:05.824 DEBUG:boto:Token: None 2022-09-03T05:54:05.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/?versions 2022-09-03T05:54:05.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQW/fD8msqMEvy4ve3FvZr3a6Ec= 2022-09-03T05:54:05.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQW/fD8msqMEvy4ve3FvZr3a6Ec=', 'Content-Length': '0'} 2022-09-03T05:54:05.828 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fce345d46671c275-006312ebfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.829 DEBUG:boto:b'avysmw-11999false&nulltrue2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:05.830 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:05.831 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:05.831 DEBUG:boto:path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.832 DEBUG:boto:auth_path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.832 DEBUG:boto:Method: GET 2022-09-03T05:54:05.833 DEBUG:boto:Path: /avysmw-11/?versions&max-keys=999 2022-09-03T05:54:05.833 DEBUG:boto:Data: 2022-09-03T05:54:05.834 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.834 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:05.835 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.835 DEBUG:boto:Params: {} 2022-09-03T05:54:05.836 DEBUG:boto:Token: None 2022-09-03T05:54:05.836 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/?versions 2022-09-03T05:54:05.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQW/fD8msqMEvy4ve3FvZr3a6Ec= 2022-09-03T05:54:05.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQW/fD8msqMEvy4ve3FvZr3a6Ec=', 'Content-Length': '0'} 2022-09-03T05:54:05.840 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bd1bf35a779e76d3-006312ebfd-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.841 DEBUG:boto:b'avysmw-11999false&nulltrue2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:05.842 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:05.843 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.843 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.844 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.844 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.845 DEBUG:boto:Method: GET 2022-09-03T05:54:05.845 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.846 DEBUG:boto:Data: 2022-09-03T05:54:05.846 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.847 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.848 DEBUG:boto:Params: {} 2022-09-03T05:54:05.849 DEBUG:boto:Token: None 2022-09-03T05:54:05.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VZ3OKpGv0Far8ofAwwZbhJFXCew= 2022-09-03T05:54:05.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VZ3OKpGv0Far8ofAwwZbhJFXCew=', 'Content-Length': '0'} 2022-09-03T05:54:05.852 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000282e53b1523870ac-006312ebfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.853 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.854 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.855 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.855 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.856 DEBUG:boto:Method: GET 2022-09-03T05:54:05.856 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.857 DEBUG:boto:Data: 2022-09-03T05:54:05.857 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.858 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:05.859 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.859 DEBUG:boto:Params: {} 2022-09-03T05:54:05.860 DEBUG:boto:Token: None 2022-09-03T05:54:05.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VZ3OKpGv0Far8ofAwwZbhJFXCew= 2022-09-03T05:54:05.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VZ3OKpGv0Far8ofAwwZbhJFXCew=', 'Content-Length': '0'} 2022-09-03T05:54:05.865 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007309957f2846f1f3-006312ebfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.866 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.866 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.867 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.867 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.868 DEBUG:boto:Method: HEAD 2022-09-03T05:54:05.868 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.869 DEBUG:boto:Data: 2022-09-03T05:54:05.869 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.870 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.871 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.872 DEBUG:boto:Params: {} 2022-09-03T05:54:05.873 DEBUG:boto:Token: None 2022-09-03T05:54:05.873 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BKuhvdlj9Yla1uevOWZBAXt7bO0= 2022-09-03T05:54:05.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BKuhvdlj9Yla1uevOWZBAXt7bO0=', 'Content-Length': '0'} 2022-09-03T05:54:05.877 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000185a87e55db71a1-006312ebfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.877 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.878 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.879 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.879 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.880 DEBUG:boto:Method: HEAD 2022-09-03T05:54:05.880 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.881 DEBUG:boto:Data: 2022-09-03T05:54:05.881 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.882 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:05.882 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.883 DEBUG:boto:Params: {} 2022-09-03T05:54:05.884 DEBUG:boto:Token: None 2022-09-03T05:54:05.884 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BKuhvdlj9Yla1uevOWZBAXt7bO0= 2022-09-03T05:54:05.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BKuhvdlj9Yla1uevOWZBAXt7bO0=', 'Content-Length': '0'} 2022-09-03T05:54:05.888 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d3545b61c128f996-006312ebfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.889 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:05.889 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.890 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.891 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.891 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.892 DEBUG:boto:Method: GET 2022-09-03T05:54:05.892 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.893 DEBUG:boto:Data: 2022-09-03T05:54:05.894 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.895 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.895 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.896 DEBUG:boto:Params: {} 2022-09-03T05:54:05.896 DEBUG:boto:Token: None 2022-09-03T05:54:05.897 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VZ3OKpGv0Far8ofAwwZbhJFXCew= 2022-09-03T05:54:05.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VZ3OKpGv0Far8ofAwwZbhJFXCew=', 'Content-Length': '0'} 2022-09-03T05:54:05.901 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000a9fefd869f76aa1-006312ebfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.902 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.903 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.904 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.905 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:05.906 DEBUG:boto:Method: GET 2022-09-03T05:54:05.907 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.907 DEBUG:boto:Data: 2022-09-03T05:54:05.908 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.909 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:05.909 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.910 DEBUG:boto:Params: {} 2022-09-03T05:54:05.911 DEBUG:boto:Token: None 2022-09-03T05:54:05.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:05.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VZ3OKpGv0Far8ofAwwZbhJFXCew= 2022-09-03T05:54:05.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VZ3OKpGv0Far8ofAwwZbhJFXCew=', 'Content-Length': '0'} 2022-09-03T05:54:05.915 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cd75b9d622239d18-006312ebfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.916 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:05.917 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:05.918 DEBUG:boto:path=/avysmw-11/?prefix=%26 2022-09-03T05:54:05.919 DEBUG:boto:auth_path=/avysmw-11/?prefix=%26 2022-09-03T05:54:05.920 DEBUG:boto:Method: GET 2022-09-03T05:54:05.921 DEBUG:boto:Path: /avysmw-11/?prefix=%26 2022-09-03T05:54:05.922 DEBUG:boto:Data: 2022-09-03T05:54:05.923 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.925 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.926 DEBUG:boto:Params: {} 2022-09-03T05:54:05.927 DEBUG:boto:Token: None 2022-09-03T05:54:05.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/ 2022-09-03T05:54:05.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bE6A7hmPIm6pNI2JolE7UfKkSRw= 2022-09-03T05:54:05.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bE6A7hmPIm6pNI2JolE7UfKkSRw=', 'Content-Length': '0'} 2022-09-03T05:54:05.933 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f64b5bd341c99468-006312ebfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.934 DEBUG:boto:b'avysmw-11&1000false&2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:05.935 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:05.936 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:05.937 DEBUG:boto:path=/avysmw-11/?prefix=%26 2022-09-03T05:54:05.937 DEBUG:boto:auth_path=/avysmw-11/?prefix=%26 2022-09-03T05:54:05.938 DEBUG:boto:Method: GET 2022-09-03T05:54:05.938 DEBUG:boto:Path: /avysmw-11/?prefix=%26 2022-09-03T05:54:05.939 DEBUG:boto:Data: 2022-09-03T05:54:05.939 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.940 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:05.940 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.941 DEBUG:boto:Params: {} 2022-09-03T05:54:05.941 DEBUG:boto:Token: None 2022-09-03T05:54:05.942 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/ 2022-09-03T05:54:05.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bE6A7hmPIm6pNI2JolE7UfKkSRw= 2022-09-03T05:54:05.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bE6A7hmPIm6pNI2JolE7UfKkSRw=', 'Content-Length': '0'} 2022-09-03T05:54:05.946 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dbbf669a76d82a0d-006312ebfd-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.947 DEBUG:boto:b'avysmw-11&1000false&2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:05.948 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.948 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.949 DEBUG:boto:Method: HEAD 2022-09-03T05:54:05.949 DEBUG:boto:Path: /avysmw-11/%26 2022-09-03T05:54:05.950 DEBUG:boto:Data: 2022-09-03T05:54:05.950 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.951 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.952 DEBUG:boto:Params: {} 2022-09-03T05:54:05.952 DEBUG:boto:Token: None 2022-09-03T05:54:05.953 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26 2022-09-03T05:54:05.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DOOrfW2kh+CwzO9zY+SkPXnyPvE= 2022-09-03T05:54:05.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DOOrfW2kh+CwzO9zY+SkPXnyPvE=', 'Content-Length': '0'} 2022-09-03T05:54:05.956 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dc07bd47901c00ff-006312ebfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.957 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.958 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.959 DEBUG:boto:Method: HEAD 2022-09-03T05:54:05.960 DEBUG:boto:Path: /avysmw-11/%26 2022-09-03T05:54:05.961 DEBUG:boto:Data: 2022-09-03T05:54:05.962 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.963 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:05.964 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.964 DEBUG:boto:Params: {} 2022-09-03T05:54:05.965 DEBUG:boto:Token: None 2022-09-03T05:54:05.965 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26 2022-09-03T05:54:05.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DOOrfW2kh+CwzO9zY+SkPXnyPvE= 2022-09-03T05:54:05.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DOOrfW2kh+CwzO9zY+SkPXnyPvE=', 'Content-Length': '0'} 2022-09-03T05:54:05.969 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000065c5b0371452c588-006312ebfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.970 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:05.970 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.971 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.971 DEBUG:boto:Method: GET 2022-09-03T05:54:05.972 DEBUG:boto:Path: /avysmw-11/%26 2022-09-03T05:54:05.972 DEBUG:boto:Data: 2022-09-03T05:54:05.973 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.974 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.975 DEBUG:boto:Params: {} 2022-09-03T05:54:05.975 DEBUG:boto:Token: None 2022-09-03T05:54:05.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26 2022-09-03T05:54:05.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CMf80Z9ozDRKunYGr4mU3KHu7V8= 2022-09-03T05:54:05.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CMf80Z9ozDRKunYGr4mU3KHu7V8=', 'Content-Length': '0'} 2022-09-03T05:54:05.980 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000057c95c4dea71f930-006312ebfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.981 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:05.981 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:05.982 DEBUG:boto:Method: GET 2022-09-03T05:54:05.982 DEBUG:boto:Path: /avysmw-11/%26 2022-09-03T05:54:05.983 DEBUG:boto:Data: 2022-09-03T05:54:05.983 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.984 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:05.984 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.985 DEBUG:boto:Params: {} 2022-09-03T05:54:05.985 DEBUG:boto:Token: None 2022-09-03T05:54:05.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/%26 2022-09-03T05:54:05.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CMf80Z9ozDRKunYGr4mU3KHu7V8= 2022-09-03T05:54:05.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CMf80Z9ozDRKunYGr4mU3KHu7V8=', 'Content-Length': '0'} 2022-09-03T05:54:05.989 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000049e0c692e097f1de-006312ebfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:05.990 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:05.991 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:05.992 DEBUG:boto:path=/avysmw-11/?prefix=%26 2022-09-03T05:54:05.992 DEBUG:boto:auth_path=/avysmw-11/?prefix=%26 2022-09-03T05:54:05.993 DEBUG:boto:Method: GET 2022-09-03T05:54:05.993 DEBUG:boto:Path: /avysmw-11/?prefix=%26 2022-09-03T05:54:05.994 DEBUG:boto:Data: 2022-09-03T05:54:05.995 DEBUG:boto:Headers: {} 2022-09-03T05:54:05.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:05.996 DEBUG:boto:Port: 8000 2022-09-03T05:54:05.996 DEBUG:boto:Params: {} 2022-09-03T05:54:05.997 DEBUG:boto:Token: None 2022-09-03T05:54:05.997 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:05 GMT /avysmw-11/ 2022-09-03T05:54:05.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bE6A7hmPIm6pNI2JolE7UfKkSRw= 2022-09-03T05:54:05.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bE6A7hmPIm6pNI2JolE7UfKkSRw=', 'Content-Length': '0'} 2022-09-03T05:54:06.001 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f9a29140d95611fc-006312ebfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.002 DEBUG:boto:b'avysmw-11&1000false&2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.003 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.004 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.004 DEBUG:boto:path=/avysmw-11/?prefix=%26 2022-09-03T05:54:06.005 DEBUG:boto:auth_path=/avysmw-11/?prefix=%26 2022-09-03T05:54:06.005 DEBUG:boto:Method: GET 2022-09-03T05:54:06.006 DEBUG:boto:Path: /avysmw-11/?prefix=%26 2022-09-03T05:54:06.006 DEBUG:boto:Data: 2022-09-03T05:54:06.007 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.007 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.008 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.008 DEBUG:boto:Params: {} 2022-09-03T05:54:06.009 DEBUG:boto:Token: None 2022-09-03T05:54:06.010 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.011 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.014 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044871e41738b9052-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.015 DEBUG:boto:b'avysmw-11&1000false&2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.016 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:06.017 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.017 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.018 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.018 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.019 DEBUG:boto:Method: GET 2022-09-03T05:54:06.020 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.020 DEBUG:boto:Data: 2022-09-03T05:54:06.021 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.022 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.022 DEBUG:boto:Params: {} 2022-09-03T05:54:06.023 DEBUG:boto:Token: None 2022-09-03T05:54:06.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wwRB93PvLiffc22Ez+/aVrTpias= 2022-09-03T05:54:06.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wwRB93PvLiffc22Ez+/aVrTpias=', 'Content-Length': '0'} 2022-09-03T05:54:06.027 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000024f9958c67ac87b0-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.028 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.029 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.029 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.031 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.031 DEBUG:boto:Method: GET 2022-09-03T05:54:06.032 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.032 DEBUG:boto:Data: 2022-09-03T05:54:06.033 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.033 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.034 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.034 DEBUG:boto:Params: {} 2022-09-03T05:54:06.035 DEBUG:boto:Token: None 2022-09-03T05:54:06.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wwRB93PvLiffc22Ez+/aVrTpias= 2022-09-03T05:54:06.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wwRB93PvLiffc22Ez+/aVrTpias=', 'Content-Length': '0'} 2022-09-03T05:54:06.040 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000416dfaf531373387-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.086 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.087 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.088 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.088 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.089 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.089 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.090 DEBUG:boto:Data: 2022-09-03T05:54:06.090 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.091 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.091 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.092 DEBUG:boto:Params: {} 2022-09-03T05:54:06.092 DEBUG:boto:Token: None 2022-09-03T05:54:06.093 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GkpVbwe5j4pUgCUt4puadO9J7lM= 2022-09-03T05:54:06.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GkpVbwe5j4pUgCUt4puadO9J7lM=', 'Content-Length': '0'} 2022-09-03T05:54:06.096 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004e4d8025cafbd3e6-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.097 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.097 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.098 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.098 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.099 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.099 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.100 DEBUG:boto:Data: 2022-09-03T05:54:06.100 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.101 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.101 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.102 DEBUG:boto:Params: {} 2022-09-03T05:54:06.102 DEBUG:boto:Token: None 2022-09-03T05:54:06.103 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GkpVbwe5j4pUgCUt4puadO9J7lM= 2022-09-03T05:54:06.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GkpVbwe5j4pUgCUt4puadO9J7lM=', 'Content-Length': '0'} 2022-09-03T05:54:06.106 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000884d8c5a953f3903-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.107 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:06.108 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.108 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.109 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.109 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.110 DEBUG:boto:Method: GET 2022-09-03T05:54:06.110 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.111 DEBUG:boto:Data: 2022-09-03T05:54:06.111 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.112 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.112 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.113 DEBUG:boto:Params: {} 2022-09-03T05:54:06.113 DEBUG:boto:Token: None 2022-09-03T05:54:06.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wwRB93PvLiffc22Ez+/aVrTpias= 2022-09-03T05:54:06.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wwRB93PvLiffc22Ez+/aVrTpias=', 'Content-Length': '0'} 2022-09-03T05:54:06.117 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006a33dba93c66af07-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.118 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.119 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.119 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.120 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.120 DEBUG:boto:Method: GET 2022-09-03T05:54:06.121 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.121 DEBUG:boto:Data: 2022-09-03T05:54:06.122 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.123 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.123 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.124 DEBUG:boto:Params: {} 2022-09-03T05:54:06.124 DEBUG:boto:Token: None 2022-09-03T05:54:06.125 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:06.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wwRB93PvLiffc22Ez+/aVrTpias= 2022-09-03T05:54:06.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wwRB93PvLiffc22Ez+/aVrTpias=', 'Content-Length': '0'} 2022-09-03T05:54:06.130 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000043a0567a189ab93a-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.131 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.132 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.133 DEBUG:boto:path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:06.133 DEBUG:boto:auth_path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:06.134 DEBUG:boto:Method: GET 2022-09-03T05:54:06.135 DEBUG:boto:Path: /avysmw-11/?prefix=%3A 2022-09-03T05:54:06.135 DEBUG:boto:Data: 2022-09-03T05:54:06.136 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.136 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.137 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.138 DEBUG:boto:Params: {} 2022-09-03T05:54:06.146 DEBUG:boto:Token: None 2022-09-03T05:54:06.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.152 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069734f18fcd05ca2-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.152 DEBUG:boto:b'avysmw-11:1000false:2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.153 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.154 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.154 DEBUG:boto:path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:06.155 DEBUG:boto:auth_path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:06.155 DEBUG:boto:Method: GET 2022-09-03T05:54:06.156 DEBUG:boto:Path: /avysmw-11/?prefix=%3A 2022-09-03T05:54:06.157 DEBUG:boto:Data: 2022-09-03T05:54:06.157 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.158 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.159 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.159 DEBUG:boto:Params: {} 2022-09-03T05:54:06.160 DEBUG:boto:Token: None 2022-09-03T05:54:06.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.165 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b315ff76861840a3-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.210 DEBUG:boto:b'avysmw-11:1000false:2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.211 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.212 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.212 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.213 DEBUG:boto:Path: /avysmw-11/%3A 2022-09-03T05:54:06.214 DEBUG:boto:Data: 2022-09-03T05:54:06.214 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.215 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.216 DEBUG:boto:Params: {} 2022-09-03T05:54:06.216 DEBUG:boto:Token: None 2022-09-03T05:54:06.217 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A 2022-09-03T05:54:06.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gDwSX87FfYifPovlpqNCIzuaaYU= 2022-09-03T05:54:06.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gDwSX87FfYifPovlpqNCIzuaaYU=', 'Content-Length': '0'} 2022-09-03T05:54:06.221 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000009e11dfe8d6c32d9-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.222 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.222 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.223 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.223 DEBUG:boto:Path: /avysmw-11/%3A 2022-09-03T05:54:06.224 DEBUG:boto:Data: 2022-09-03T05:54:06.224 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.225 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.225 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.226 DEBUG:boto:Params: {} 2022-09-03T05:54:06.226 DEBUG:boto:Token: None 2022-09-03T05:54:06.227 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A 2022-09-03T05:54:06.227 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gDwSX87FfYifPovlpqNCIzuaaYU= 2022-09-03T05:54:06.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gDwSX87FfYifPovlpqNCIzuaaYU=', 'Content-Length': '0'} 2022-09-03T05:54:06.230 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dc322eccd477e09c-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.231 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:06.232 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.232 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.233 DEBUG:boto:Method: GET 2022-09-03T05:54:06.233 DEBUG:boto:Path: /avysmw-11/%3A 2022-09-03T05:54:06.234 DEBUG:boto:Data: 2022-09-03T05:54:06.234 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.236 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.236 DEBUG:boto:Params: {} 2022-09-03T05:54:06.237 DEBUG:boto:Token: None 2022-09-03T05:54:06.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A 2022-09-03T05:54:06.238 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UntkC7FWT1eQpWSXt3W+hL30ww8= 2022-09-03T05:54:06.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UntkC7FWT1eQpWSXt3W+hL30ww8=', 'Content-Length': '0'} 2022-09-03T05:54:06.241 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000048b582b5e65c3bc9-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.242 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:06.242 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:06.243 DEBUG:boto:Method: GET 2022-09-03T05:54:06.244 DEBUG:boto:Path: /avysmw-11/%3A 2022-09-03T05:54:06.244 DEBUG:boto:Data: 2022-09-03T05:54:06.245 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.245 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.246 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.246 DEBUG:boto:Params: {} 2022-09-03T05:54:06.247 DEBUG:boto:Token: None 2022-09-03T05:54:06.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/%3A 2022-09-03T05:54:06.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UntkC7FWT1eQpWSXt3W+hL30ww8= 2022-09-03T05:54:06.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UntkC7FWT1eQpWSXt3W+hL30ww8=', 'Content-Length': '0'} 2022-09-03T05:54:06.251 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006cd915d10cebf94e-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.252 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.253 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.253 DEBUG:boto:path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:06.254 DEBUG:boto:auth_path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:06.254 DEBUG:boto:Method: GET 2022-09-03T05:54:06.255 DEBUG:boto:Path: /avysmw-11/?prefix=%3A 2022-09-03T05:54:06.255 DEBUG:boto:Data: 2022-09-03T05:54:06.256 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.256 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.257 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.258 DEBUG:boto:Params: {} 2022-09-03T05:54:06.259 DEBUG:boto:Token: None 2022-09-03T05:54:06.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.264 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000f8973fdd416d302-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.265 DEBUG:boto:b'avysmw-11:1000false:2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.266 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.267 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.267 DEBUG:boto:path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:06.268 DEBUG:boto:auth_path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:06.268 DEBUG:boto:Method: GET 2022-09-03T05:54:06.269 DEBUG:boto:Path: /avysmw-11/?prefix=%3A 2022-09-03T05:54:06.269 DEBUG:boto:Data: 2022-09-03T05:54:06.270 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.270 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.271 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.272 DEBUG:boto:Params: {} 2022-09-03T05:54:06.273 DEBUG:boto:Token: None 2022-09-03T05:54:06.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.279 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c6cc12cfe4a8edf-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.279 DEBUG:boto:b'avysmw-11:1000false:2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.280 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:06.281 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.281 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.282 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.283 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.283 DEBUG:boto:Method: GET 2022-09-03T05:54:06.284 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.284 DEBUG:boto:Data: 2022-09-03T05:54:06.285 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.286 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.287 DEBUG:boto:Params: {} 2022-09-03T05:54:06.287 DEBUG:boto:Token: None 2022-09-03T05:54:06.288 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wsk5DpJ6W5eMxpdnNa200a2wFK4= 2022-09-03T05:54:06.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wsk5DpJ6W5eMxpdnNa200a2wFK4=', 'Content-Length': '0'} 2022-09-03T05:54:06.291 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cd6d0ded2fa9b0b5-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.292 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.293 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.293 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.294 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.295 DEBUG:boto:Method: GET 2022-09-03T05:54:06.295 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.296 DEBUG:boto:Data: 2022-09-03T05:54:06.296 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.297 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.297 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.298 DEBUG:boto:Params: {} 2022-09-03T05:54:06.298 DEBUG:boto:Token: None 2022-09-03T05:54:06.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wsk5DpJ6W5eMxpdnNa200a2wFK4= 2022-09-03T05:54:06.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wsk5DpJ6W5eMxpdnNa200a2wFK4=', 'Content-Length': '0'} 2022-09-03T05:54:06.302 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cef6c9589c61e1dc-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.346 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.347 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.347 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.348 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.349 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.349 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.350 DEBUG:boto:Data: 2022-09-03T05:54:06.350 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.351 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.352 DEBUG:boto:Params: {} 2022-09-03T05:54:06.352 DEBUG:boto:Token: None 2022-09-03T05:54:06.353 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.353 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zRP08fIyPuzB7oMSHaEAafCXW6M= 2022-09-03T05:54:06.354 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zRP08fIyPuzB7oMSHaEAafCXW6M=', 'Content-Length': '0'} 2022-09-03T05:54:06.356 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a940cc66955643ed-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.357 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.358 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.359 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.360 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.361 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.361 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.362 DEBUG:boto:Data: 2022-09-03T05:54:06.363 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.364 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.365 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.366 DEBUG:boto:Params: {} 2022-09-03T05:54:06.367 DEBUG:boto:Token: None 2022-09-03T05:54:06.368 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zRP08fIyPuzB7oMSHaEAafCXW6M= 2022-09-03T05:54:06.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zRP08fIyPuzB7oMSHaEAafCXW6M=', 'Content-Length': '0'} 2022-09-03T05:54:06.373 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000688222164d10cfcc-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.373 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:06.374 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.374 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.375 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.375 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.376 DEBUG:boto:Method: GET 2022-09-03T05:54:06.376 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.377 DEBUG:boto:Data: 2022-09-03T05:54:06.378 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.378 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.379 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.379 DEBUG:boto:Params: {} 2022-09-03T05:54:06.380 DEBUG:boto:Token: None 2022-09-03T05:54:06.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wsk5DpJ6W5eMxpdnNa200a2wFK4= 2022-09-03T05:54:06.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wsk5DpJ6W5eMxpdnNa200a2wFK4=', 'Content-Length': '0'} 2022-09-03T05:54:06.384 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a5aff8c1712ba393-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.385 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.385 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.386 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.386 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.387 DEBUG:boto:Method: GET 2022-09-03T05:54:06.388 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.388 DEBUG:boto:Data: 2022-09-03T05:54:06.389 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.389 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.390 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.390 DEBUG:boto:Params: {} 2022-09-03T05:54:06.391 DEBUG:boto:Token: None 2022-09-03T05:54:06.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:06.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wsk5DpJ6W5eMxpdnNa200a2wFK4= 2022-09-03T05:54:06.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wsk5DpJ6W5eMxpdnNa200a2wFK4=', 'Content-Length': '0'} 2022-09-03T05:54:06.395 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d7a5d1b8b043b82a-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.396 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.397 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.397 DEBUG:boto:path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.398 DEBUG:boto:auth_path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.398 DEBUG:boto:Method: GET 2022-09-03T05:54:06.399 DEBUG:boto:Path: /avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.399 DEBUG:boto:Data: 2022-09-03T05:54:06.400 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.401 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.401 DEBUG:boto:Params: {} 2022-09-03T05:54:06.402 DEBUG:boto:Token: None 2022-09-03T05:54:06.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.406 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a72bfa9ed3f845fa-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.407 DEBUG:boto:b'avysmw-11_myobj1000false_myobj2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.408 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.408 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.409 DEBUG:boto:path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.409 DEBUG:boto:auth_path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.410 DEBUG:boto:Method: GET 2022-09-03T05:54:06.411 DEBUG:boto:Path: /avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.412 DEBUG:boto:Data: 2022-09-03T05:54:06.412 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.413 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.414 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.415 DEBUG:boto:Params: {} 2022-09-03T05:54:06.415 DEBUG:boto:Token: None 2022-09-03T05:54:06.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.420 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000df396b33e8780e7d-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.421 DEBUG:boto:b'avysmw-11_myobj1000false_myobj2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.421 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.422 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.423 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.423 DEBUG:boto:Path: /avysmw-11/_myobj 2022-09-03T05:54:06.424 DEBUG:boto:Data: 2022-09-03T05:54:06.424 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.425 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.425 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.425 DEBUG:boto:Params: {} 2022-09-03T05:54:06.426 DEBUG:boto:Token: None 2022-09-03T05:54:06.427 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj 2022-09-03T05:54:06.427 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8mkcsVOk9YfwCIhMhjtom4hu+Vg= 2022-09-03T05:54:06.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8mkcsVOk9YfwCIhMhjtom4hu+Vg=', 'Content-Length': '0'} 2022-09-03T05:54:06.430 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004ce511957237eb2f-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.431 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.432 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.432 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.433 DEBUG:boto:Path: /avysmw-11/_myobj 2022-09-03T05:54:06.433 DEBUG:boto:Data: 2022-09-03T05:54:06.434 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.435 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.435 DEBUG:boto:Params: {} 2022-09-03T05:54:06.436 DEBUG:boto:Token: None 2022-09-03T05:54:06.436 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj 2022-09-03T05:54:06.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8mkcsVOk9YfwCIhMhjtom4hu+Vg= 2022-09-03T05:54:06.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8mkcsVOk9YfwCIhMhjtom4hu+Vg=', 'Content-Length': '0'} 2022-09-03T05:54:06.440 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b92a6ff53d91c896-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.441 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:06.441 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.442 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.442 DEBUG:boto:Method: GET 2022-09-03T05:54:06.443 DEBUG:boto:Path: /avysmw-11/_myobj 2022-09-03T05:54:06.444 DEBUG:boto:Data: 2022-09-03T05:54:06.444 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.445 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.446 DEBUG:boto:Params: {} 2022-09-03T05:54:06.446 DEBUG:boto:Token: None 2022-09-03T05:54:06.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj 2022-09-03T05:54:06.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tXflpFlLj4w50rnMYyKiub00LUI= 2022-09-03T05:54:06.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tXflpFlLj4w50rnMYyKiub00LUI=', 'Content-Length': '0'} 2022-09-03T05:54:06.451 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fa3e038411bd62f9-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.494 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:06.495 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:06.496 DEBUG:boto:Method: GET 2022-09-03T05:54:06.497 DEBUG:boto:Path: /avysmw-11/_myobj 2022-09-03T05:54:06.497 DEBUG:boto:Data: 2022-09-03T05:54:06.498 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.498 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.499 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.500 DEBUG:boto:Params: {} 2022-09-03T05:54:06.502 DEBUG:boto:Token: None 2022-09-03T05:54:06.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/_myobj 2022-09-03T05:54:06.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tXflpFlLj4w50rnMYyKiub00LUI= 2022-09-03T05:54:06.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tXflpFlLj4w50rnMYyKiub00LUI=', 'Content-Length': '0'} 2022-09-03T05:54:06.507 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000daf2578b4cdd21e5-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.508 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.509 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.510 DEBUG:boto:path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.511 DEBUG:boto:auth_path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.512 DEBUG:boto:Method: GET 2022-09-03T05:54:06.513 DEBUG:boto:Path: /avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.514 DEBUG:boto:Data: 2022-09-03T05:54:06.515 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.517 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.518 DEBUG:boto:Params: {} 2022-09-03T05:54:06.519 DEBUG:boto:Token: None 2022-09-03T05:54:06.520 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.521 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.524 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dcba6b57b2d48256-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.524 DEBUG:boto:b'avysmw-11_myobj1000false_myobj2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.525 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.526 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.526 DEBUG:boto:path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.527 DEBUG:boto:auth_path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.528 DEBUG:boto:Method: GET 2022-09-03T05:54:06.528 DEBUG:boto:Path: /avysmw-11/?prefix=_myobj 2022-09-03T05:54:06.529 DEBUG:boto:Data: 2022-09-03T05:54:06.529 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.530 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.530 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.531 DEBUG:boto:Params: {} 2022-09-03T05:54:06.532 DEBUG:boto:Token: None 2022-09-03T05:54:06.533 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.537 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fe2ac8215ccad507-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.538 DEBUG:boto:b'avysmw-11_myobj1000false_myobj2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.539 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:06.539 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.540 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.540 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.541 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.542 DEBUG:boto:Method: GET 2022-09-03T05:54:06.543 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.544 DEBUG:boto:Data: 2022-09-03T05:54:06.545 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.546 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.547 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.548 DEBUG:boto:Params: {} 2022-09-03T05:54:06.549 DEBUG:boto:Token: None 2022-09-03T05:54:06.550 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FvB0pNmVVEtbAfxN153/RFHP2Y= 2022-09-03T05:54:06.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FvB0pNmVVEtbAfxN153/RFHP2Y=', 'Content-Length': '0'} 2022-09-03T05:54:06.554 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e0397aa83a6524b1-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.555 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.556 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.556 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.557 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.557 DEBUG:boto:Method: GET 2022-09-03T05:54:06.558 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.558 DEBUG:boto:Data: 2022-09-03T05:54:06.559 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.559 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.560 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.560 DEBUG:boto:Params: {} 2022-09-03T05:54:06.561 DEBUG:boto:Token: None 2022-09-03T05:54:06.562 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FvB0pNmVVEtbAfxN153/RFHP2Y= 2022-09-03T05:54:06.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FvB0pNmVVEtbAfxN153/RFHP2Y=', 'Content-Length': '0'} 2022-09-03T05:54:06.565 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009d37f58b94d60b47-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.566 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.567 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.567 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.568 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.568 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.569 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.569 DEBUG:boto:Data: 2022-09-03T05:54:06.570 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.571 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.571 DEBUG:boto:Params: {} 2022-09-03T05:54:06.572 DEBUG:boto:Token: None 2022-09-03T05:54:06.572 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vGKtkXSaCZBkA7jyj0zcgqFXBGQ= 2022-09-03T05:54:06.574 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vGKtkXSaCZBkA7jyj0zcgqFXBGQ=', 'Content-Length': '0'} 2022-09-03T05:54:06.576 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000913d58eae03e5573-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.576 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.577 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.577 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.578 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.578 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.579 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.579 DEBUG:boto:Data: 2022-09-03T05:54:06.580 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.580 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.581 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.581 DEBUG:boto:Params: {} 2022-09-03T05:54:06.582 DEBUG:boto:Token: None 2022-09-03T05:54:06.582 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vGKtkXSaCZBkA7jyj0zcgqFXBGQ= 2022-09-03T05:54:06.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vGKtkXSaCZBkA7jyj0zcgqFXBGQ=', 'Content-Length': '0'} 2022-09-03T05:54:06.586 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a2df84bb758b04f4-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.587 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:06.587 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.588 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.589 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.589 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.590 DEBUG:boto:Method: GET 2022-09-03T05:54:06.590 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.591 DEBUG:boto:Data: 2022-09-03T05:54:06.591 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.592 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.593 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.593 DEBUG:boto:Params: {} 2022-09-03T05:54:06.594 DEBUG:boto:Token: None 2022-09-03T05:54:06.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FvB0pNmVVEtbAfxN153/RFHP2Y= 2022-09-03T05:54:06.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FvB0pNmVVEtbAfxN153/RFHP2Y=', 'Content-Length': '0'} 2022-09-03T05:54:06.599 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f1063e5b5dbaa250-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.599 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.600 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.601 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.601 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.602 DEBUG:boto:Method: GET 2022-09-03T05:54:06.603 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.603 DEBUG:boto:Data: 2022-09-03T05:54:06.604 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.605 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.605 DEBUG:boto:Params: {} 2022-09-03T05:54:06.606 DEBUG:boto:Token: None 2022-09-03T05:54:06.606 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:06.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FvB0pNmVVEtbAfxN153/RFHP2Y= 2022-09-03T05:54:06.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FvB0pNmVVEtbAfxN153/RFHP2Y=', 'Content-Length': '0'} 2022-09-03T05:54:06.610 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000aa039ba67db3ed35-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.611 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.611 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.612 DEBUG:boto:path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:06.612 DEBUG:boto:auth_path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:06.613 DEBUG:boto:Method: GET 2022-09-03T05:54:06.614 DEBUG:boto:Path: /avysmw-11/?prefix=myobj 2022-09-03T05:54:06.614 DEBUG:boto:Data: 2022-09-03T05:54:06.615 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.616 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.616 DEBUG:boto:Params: {} 2022-09-03T05:54:06.617 DEBUG:boto:Token: None 2022-09-03T05:54:06.618 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.622 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031f8ddddcb7972ba-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.666 DEBUG:boto:b'avysmw-11myobj1000falsemyobj2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.667 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.668 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.668 DEBUG:boto:path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:06.669 DEBUG:boto:auth_path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:06.669 DEBUG:boto:Method: GET 2022-09-03T05:54:06.670 DEBUG:boto:Path: /avysmw-11/?prefix=myobj 2022-09-03T05:54:06.670 DEBUG:boto:Data: 2022-09-03T05:54:06.671 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.671 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.672 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.673 DEBUG:boto:Params: {} 2022-09-03T05:54:06.673 DEBUG:boto:Token: None 2022-09-03T05:54:06.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.678 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e8f2304b44bc17c1-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.679 DEBUG:boto:b'avysmw-11myobj1000falsemyobj2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.680 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.681 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.681 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.682 DEBUG:boto:Path: /avysmw-11/myobj 2022-09-03T05:54:06.682 DEBUG:boto:Data: 2022-09-03T05:54:06.683 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.684 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.684 DEBUG:boto:Params: {} 2022-09-03T05:54:06.685 DEBUG:boto:Token: None 2022-09-03T05:54:06.685 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj 2022-09-03T05:54:06.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HhY8rUNOGKIsRIvUVRa2fE6RdSw= 2022-09-03T05:54:06.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HhY8rUNOGKIsRIvUVRa2fE6RdSw=', 'Content-Length': '0'} 2022-09-03T05:54:06.689 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b2dba6231984b3c4-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.690 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.690 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.691 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.692 DEBUG:boto:Path: /avysmw-11/myobj 2022-09-03T05:54:06.692 DEBUG:boto:Data: 2022-09-03T05:54:06.693 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.693 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.694 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.694 DEBUG:boto:Params: {} 2022-09-03T05:54:06.695 DEBUG:boto:Token: None 2022-09-03T05:54:06.695 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj 2022-09-03T05:54:06.696 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HhY8rUNOGKIsRIvUVRa2fE6RdSw= 2022-09-03T05:54:06.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HhY8rUNOGKIsRIvUVRa2fE6RdSw=', 'Content-Length': '0'} 2022-09-03T05:54:06.699 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000099e2bdeb02bc9051-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.700 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:06.700 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.701 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.701 DEBUG:boto:Method: GET 2022-09-03T05:54:06.702 DEBUG:boto:Path: /avysmw-11/myobj 2022-09-03T05:54:06.703 DEBUG:boto:Data: 2022-09-03T05:54:06.703 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.705 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.705 DEBUG:boto:Params: {} 2022-09-03T05:54:06.706 DEBUG:boto:Token: None 2022-09-03T05:54:06.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj 2022-09-03T05:54:06.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vZpUoEIUhahUDd9GarjbY/UHaEw= 2022-09-03T05:54:06.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vZpUoEIUhahUDd9GarjbY/UHaEw=', 'Content-Length': '0'} 2022-09-03T05:54:06.710 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002e5a55dc200d186f-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.711 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:06.711 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:06.712 DEBUG:boto:Method: GET 2022-09-03T05:54:06.713 DEBUG:boto:Path: /avysmw-11/myobj 2022-09-03T05:54:06.713 DEBUG:boto:Data: 2022-09-03T05:54:06.714 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.714 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.715 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.715 DEBUG:boto:Params: {} 2022-09-03T05:54:06.716 DEBUG:boto:Token: None 2022-09-03T05:54:06.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/myobj 2022-09-03T05:54:06.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vZpUoEIUhahUDd9GarjbY/UHaEw= 2022-09-03T05:54:06.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vZpUoEIUhahUDd9GarjbY/UHaEw=', 'Content-Length': '0'} 2022-09-03T05:54:06.720 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000a2033336782736c-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.766 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.767 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.767 DEBUG:boto:path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:06.768 DEBUG:boto:auth_path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:06.768 DEBUG:boto:Method: GET 2022-09-03T05:54:06.769 DEBUG:boto:Path: /avysmw-11/?prefix=myobj 2022-09-03T05:54:06.769 DEBUG:boto:Data: 2022-09-03T05:54:06.770 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:06.771 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.772 DEBUG:boto:Params: {} 2022-09-03T05:54:06.773 DEBUG:boto:Token: None 2022-09-03T05:54:06.774 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.779 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000476cc2594ce50895-006312ebfe-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.780 DEBUG:boto:b'avysmw-11myobj1000falsemyobj2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.782 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.782 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.783 DEBUG:boto:path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:06.783 DEBUG:boto:auth_path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:06.784 DEBUG:boto:Method: GET 2022-09-03T05:54:06.785 DEBUG:boto:Path: /avysmw-11/?prefix=myobj 2022-09-03T05:54:06.785 DEBUG:boto:Data: 2022-09-03T05:54:06.786 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.786 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.787 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.788 DEBUG:boto:Params: {} 2022-09-03T05:54:06.788 DEBUG:boto:Token: None 2022-09-03T05:54:06.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY= 2022-09-03T05:54:06.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ve7N7StpT5ST/4mvg5j4pBBwahY=', 'Content-Length': '0'} 2022-09-03T05:54:06.793 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027e11710b5dec878-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.794 DEBUG:boto:b'avysmw-11myobj1000falsemyobj2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.795 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-11 zones={test-zone1, test-zone2} 2022-09-03T05:54:06.796 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-11', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:06.797 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-11', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:06.798 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-11 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:06.865 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:06.865 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:06.913 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:06.911+0000 7fbcf953d600 1 bucket sync caught up with source: 2022-09-03T05:54:06.914 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.16.5, , 00000000001.20.5, , , , 00000000001.22.5, 00000000001.14.5, , ] 2022-09-03T05:54:06.914 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.16.5, , 00000000001.20.5, , , , 00000000001.22.5, 00000000001.14.5, , ] 2022-09-03T05:54:06.914 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:06.911+0000 7fbcf953d600 0 bucket checkpoint complete 2022-09-03T05:54:06.927 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-11 zones={test-zone3, test-zone2} 2022-09-03T05:54:06.927 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.928 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.929 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.929 DEBUG:boto:Path: /avysmw-11/ 2022-09-03T05:54:06.930 DEBUG:boto:Data: 2022-09-03T05:54:06.931 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:06.932 DEBUG:boto:Port: 8001 2022-09-03T05:54:06.932 DEBUG:boto:Params: {} 2022-09-03T05:54:06.933 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T05:54:06.934 DEBUG:boto:Token: None 2022-09-03T05:54:06.934 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zGr6kWnEXjea/M/Rd8Iy0f7pnoc= 2022-09-03T05:54:06.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zGr6kWnEXjea/M/Rd8Iy0f7pnoc=', 'Content-Length': '0'} 2022-09-03T05:54:06.939 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d8596a78cb582062-006312ebfe-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.940 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.941 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.941 DEBUG:boto:Method: HEAD 2022-09-03T05:54:06.942 DEBUG:boto:Path: /avysmw-11/ 2022-09-03T05:54:06.942 DEBUG:boto:Data: 2022-09-03T05:54:06.943 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.943 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.944 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.944 DEBUG:boto:Params: {} 2022-09-03T05:54:06.945 DEBUG:boto:Token: None 2022-09-03T05:54:06.945 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/ 2022-09-03T05:54:06.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zGr6kWnEXjea/M/Rd8Iy0f7pnoc= 2022-09-03T05:54:06.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zGr6kWnEXjea/M/Rd8Iy0f7pnoc=', 'Content-Length': '0'} 2022-09-03T05:54:06.949 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000507f2523d4c38192-006312ebfe-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.950 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:06.950 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.951 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.951 DEBUG:boto:path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.952 DEBUG:boto:auth_path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.952 DEBUG:boto:Method: GET 2022-09-03T05:54:06.953 DEBUG:boto:Path: /avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.953 DEBUG:boto:Data: 2022-09-03T05:54:06.954 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:06.955 DEBUG:boto:Port: 8001 2022-09-03T05:54:06.955 DEBUG:boto:Params: {} 2022-09-03T05:54:06.956 DEBUG:boto:Token: None 2022-09-03T05:54:06.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/?versions 2022-09-03T05:54:06.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Fgvt+2gznGg0V0vOA9SH+svePM= 2022-09-03T05:54:06.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Fgvt+2gznGg0V0vOA9SH+svePM=', 'Content-Length': '0'} 2022-09-03T05:54:06.960 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002fc74b6e4b360dbf-006312ebfe-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.961 DEBUG:boto:b'avysmw-11999false&nulltrue2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.962 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:54:06.962 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:54:06.963 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:54:06.963 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:06.964 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:06.964 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.965 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.965 DEBUG:boto:path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.966 DEBUG:boto:auth_path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.966 DEBUG:boto:Method: GET 2022-09-03T05:54:06.967 DEBUG:boto:Path: /avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.967 DEBUG:boto:Data: 2022-09-03T05:54:06.968 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.968 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.969 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.970 DEBUG:boto:Params: {} 2022-09-03T05:54:06.970 DEBUG:boto:Token: None 2022-09-03T05:54:06.971 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/?versions 2022-09-03T05:54:06.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Fgvt+2gznGg0V0vOA9SH+svePM= 2022-09-03T05:54:06.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Fgvt+2gznGg0V0vOA9SH+svePM=', 'Content-Length': '0'} 2022-09-03T05:54:06.975 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b211c11a8f16bcf-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.975 DEBUG:boto:b'avysmw-11999false&nulltrue2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.976 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:54:06.977 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:54:06.977 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:54:06.978 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:06.979 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.979 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.980 DEBUG:boto:path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.980 DEBUG:boto:auth_path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.981 DEBUG:boto:Method: GET 2022-09-03T05:54:06.981 DEBUG:boto:Path: /avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.982 DEBUG:boto:Data: 2022-09-03T05:54:06.982 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:06.983 DEBUG:boto:Port: 8001 2022-09-03T05:54:06.984 DEBUG:boto:Params: {} 2022-09-03T05:54:06.985 DEBUG:boto:Token: None 2022-09-03T05:54:06.985 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/?versions 2022-09-03T05:54:06.986 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Fgvt+2gznGg0V0vOA9SH+svePM= 2022-09-03T05:54:06.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Fgvt+2gznGg0V0vOA9SH+svePM=', 'Content-Length': '0'} 2022-09-03T05:54:06.989 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000723e791f7e0bdf08-006312ebfe-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:06.990 DEBUG:boto:b'avysmw-11999false&nulltrue2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:06.991 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:06.991 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:06.992 DEBUG:boto:path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.992 DEBUG:boto:auth_path=/avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.993 DEBUG:boto:Method: GET 2022-09-03T05:54:06.993 DEBUG:boto:Path: /avysmw-11/?versions&max-keys=999 2022-09-03T05:54:06.994 DEBUG:boto:Data: 2022-09-03T05:54:06.994 DEBUG:boto:Headers: {} 2022-09-03T05:54:06.995 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:06.995 DEBUG:boto:Port: 8000 2022-09-03T05:54:06.996 DEBUG:boto:Params: {} 2022-09-03T05:54:06.997 DEBUG:boto:Token: None 2022-09-03T05:54:06.997 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:06 GMT /avysmw-11/?versions 2022-09-03T05:54:06.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Fgvt+2gznGg0V0vOA9SH+svePM= 2022-09-03T05:54:06.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Fgvt+2gznGg0V0vOA9SH+svePM=', 'Content-Length': '0'} 2022-09-03T05:54:07.001 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007574fa2ccfd718bb-006312ebfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.002 DEBUG:boto:b'avysmw-11999false&nulltrue2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.003 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:07.004 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.004 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.005 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.005 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.006 DEBUG:boto:Method: GET 2022-09-03T05:54:07.007 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.007 DEBUG:boto:Data: 2022-09-03T05:54:07.008 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.009 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.009 DEBUG:boto:Params: {} 2022-09-03T05:54:07.010 DEBUG:boto:Token: None 2022-09-03T05:54:07.010 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vAmiWdLlta5oy4Q/9GCXABxIN4M= 2022-09-03T05:54:07.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vAmiWdLlta5oy4Q/9GCXABxIN4M=', 'Content-Length': '0'} 2022-09-03T05:54:07.014 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ff65babf61fbd1f6-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.015 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.015 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.016 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.016 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.017 DEBUG:boto:Method: GET 2022-09-03T05:54:07.017 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.018 DEBUG:boto:Data: 2022-09-03T05:54:07.018 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.019 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.019 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.020 DEBUG:boto:Params: {} 2022-09-03T05:54:07.021 DEBUG:boto:Token: None 2022-09-03T05:54:07.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vAmiWdLlta5oy4Q/9GCXABxIN4M= 2022-09-03T05:54:07.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vAmiWdLlta5oy4Q/9GCXABxIN4M=', 'Content-Length': '0'} 2022-09-03T05:54:07.025 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ebc6be3cdacff89d-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.026 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.026 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.027 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.027 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.028 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.028 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.029 DEBUG:boto:Data: 2022-09-03T05:54:07.029 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.030 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.030 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.031 DEBUG:boto:Params: {} 2022-09-03T05:54:07.031 DEBUG:boto:Token: None 2022-09-03T05:54:07.032 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S876WufZLK+U6H/8994zV3ckchM= 2022-09-03T05:54:07.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S876WufZLK+U6H/8994zV3ckchM=', 'Content-Length': '0'} 2022-09-03T05:54:07.035 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c16b45cacdbc22eb-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.036 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.036 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.037 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.037 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.038 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.038 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.039 DEBUG:boto:Data: 2022-09-03T05:54:07.039 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.040 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.040 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.041 DEBUG:boto:Params: {} 2022-09-03T05:54:07.041 DEBUG:boto:Token: None 2022-09-03T05:54:07.042 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S876WufZLK+U6H/8994zV3ckchM= 2022-09-03T05:54:07.043 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S876WufZLK+U6H/8994zV3ckchM=', 'Content-Length': '0'} 2022-09-03T05:54:07.046 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000e54d6560e344735-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.046 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:07.047 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.047 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.048 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.048 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.049 DEBUG:boto:Method: GET 2022-09-03T05:54:07.050 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.050 DEBUG:boto:Data: 2022-09-03T05:54:07.051 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.052 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.052 DEBUG:boto:Params: {} 2022-09-03T05:54:07.053 DEBUG:boto:Token: None 2022-09-03T05:54:07.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vAmiWdLlta5oy4Q/9GCXABxIN4M= 2022-09-03T05:54:07.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vAmiWdLlta5oy4Q/9GCXABxIN4M=', 'Content-Length': '0'} 2022-09-03T05:54:07.059 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000000f2c33a00135f63-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.060 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.061 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.062 DEBUG:boto:path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.063 DEBUG:boto:auth_path=/avysmw-11/%26?versionId=null 2022-09-03T05:54:07.064 DEBUG:boto:Method: GET 2022-09-03T05:54:07.065 DEBUG:boto:Path: /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.066 DEBUG:boto:Data: 2022-09-03T05:54:07.067 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.068 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.069 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.069 DEBUG:boto:Params: {} 2022-09-03T05:54:07.070 DEBUG:boto:Token: None 2022-09-03T05:54:07.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26?versionId=null 2022-09-03T05:54:07.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vAmiWdLlta5oy4Q/9GCXABxIN4M= 2022-09-03T05:54:07.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vAmiWdLlta5oy4Q/9GCXABxIN4M=', 'Content-Length': '0'} 2022-09-03T05:54:07.075 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a9c68bacace060b2-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.076 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.076 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.077 DEBUG:boto:path=/avysmw-11/?prefix=%26 2022-09-03T05:54:07.077 DEBUG:boto:auth_path=/avysmw-11/?prefix=%26 2022-09-03T05:54:07.078 DEBUG:boto:Method: GET 2022-09-03T05:54:07.079 DEBUG:boto:Path: /avysmw-11/?prefix=%26 2022-09-03T05:54:07.080 DEBUG:boto:Data: 2022-09-03T05:54:07.081 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.083 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.083 DEBUG:boto:Params: {} 2022-09-03T05:54:07.085 DEBUG:boto:Token: None 2022-09-03T05:54:07.086 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.087 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.091 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004da66ddd1c552f20-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.092 DEBUG:boto:b'avysmw-11&1000false&2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.093 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.093 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.094 DEBUG:boto:path=/avysmw-11/?prefix=%26 2022-09-03T05:54:07.095 DEBUG:boto:auth_path=/avysmw-11/?prefix=%26 2022-09-03T05:54:07.095 DEBUG:boto:Method: GET 2022-09-03T05:54:07.096 DEBUG:boto:Path: /avysmw-11/?prefix=%26 2022-09-03T05:54:07.096 DEBUG:boto:Data: 2022-09-03T05:54:07.097 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.097 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.098 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.099 DEBUG:boto:Params: {} 2022-09-03T05:54:07.099 DEBUG:boto:Token: None 2022-09-03T05:54:07.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.105 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a5fcc0c951b94c8c-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.150 DEBUG:boto:b'avysmw-11&1000false&2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.151 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.152 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.153 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.153 DEBUG:boto:Path: /avysmw-11/%26 2022-09-03T05:54:07.154 DEBUG:boto:Data: 2022-09-03T05:54:07.155 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.156 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.157 DEBUG:boto:Params: {} 2022-09-03T05:54:07.159 DEBUG:boto:Token: None 2022-09-03T05:54:07.160 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26 2022-09-03T05:54:07.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KN4tuyJEiS8n5fMpi7TT/CwwrwI= 2022-09-03T05:54:07.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KN4tuyJEiS8n5fMpi7TT/CwwrwI=', 'Content-Length': '0'} 2022-09-03T05:54:07.165 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009eacf720e077c0d7-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.166 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.167 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.167 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.168 DEBUG:boto:Path: /avysmw-11/%26 2022-09-03T05:54:07.168 DEBUG:boto:Data: 2022-09-03T05:54:07.169 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.170 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.170 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.171 DEBUG:boto:Params: {} 2022-09-03T05:54:07.171 DEBUG:boto:Token: None 2022-09-03T05:54:07.172 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26 2022-09-03T05:54:07.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KN4tuyJEiS8n5fMpi7TT/CwwrwI= 2022-09-03T05:54:07.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KN4tuyJEiS8n5fMpi7TT/CwwrwI=', 'Content-Length': '0'} 2022-09-03T05:54:07.176 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004bac8386272ceaf3-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.176 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:07.177 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.177 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.178 DEBUG:boto:Method: GET 2022-09-03T05:54:07.179 DEBUG:boto:Path: /avysmw-11/%26 2022-09-03T05:54:07.179 DEBUG:boto:Data: 2022-09-03T05:54:07.180 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.180 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.181 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.182 DEBUG:boto:Params: {} 2022-09-03T05:54:07.182 DEBUG:boto:Token: None 2022-09-03T05:54:07.183 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26 2022-09-03T05:54:07.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FDB/v0A4V780UYCKX9hZ2niRlbo= 2022-09-03T05:54:07.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FDB/v0A4V780UYCKX9hZ2niRlbo=', 'Content-Length': '0'} 2022-09-03T05:54:07.187 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dbb192b75f6bc161-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.188 DEBUG:boto:path=/avysmw-11/%26 2022-09-03T05:54:07.188 DEBUG:boto:auth_path=/avysmw-11/%26 2022-09-03T05:54:07.189 DEBUG:boto:Method: GET 2022-09-03T05:54:07.189 DEBUG:boto:Path: /avysmw-11/%26 2022-09-03T05:54:07.190 DEBUG:boto:Data: 2022-09-03T05:54:07.191 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.191 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.192 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.192 DEBUG:boto:Params: {} 2022-09-03T05:54:07.193 DEBUG:boto:Token: None 2022-09-03T05:54:07.194 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%26 2022-09-03T05:54:07.194 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FDB/v0A4V780UYCKX9hZ2niRlbo= 2022-09-03T05:54:07.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FDB/v0A4V780UYCKX9hZ2niRlbo=', 'Content-Length': '0'} 2022-09-03T05:54:07.197 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a18687d588683716-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.198 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.199 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.199 DEBUG:boto:path=/avysmw-11/?prefix=%26 2022-09-03T05:54:07.200 DEBUG:boto:auth_path=/avysmw-11/?prefix=%26 2022-09-03T05:54:07.200 DEBUG:boto:Method: GET 2022-09-03T05:54:07.201 DEBUG:boto:Path: /avysmw-11/?prefix=%26 2022-09-03T05:54:07.202 DEBUG:boto:Data: 2022-09-03T05:54:07.202 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.203 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.204 DEBUG:boto:Params: {} 2022-09-03T05:54:07.204 DEBUG:boto:Token: None 2022-09-03T05:54:07.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1f042b7419aa7c9-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.250 DEBUG:boto:b'avysmw-11&1000false&2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.251 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.252 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.252 DEBUG:boto:path=/avysmw-11/?prefix=%26 2022-09-03T05:54:07.253 DEBUG:boto:auth_path=/avysmw-11/?prefix=%26 2022-09-03T05:54:07.254 DEBUG:boto:Method: GET 2022-09-03T05:54:07.255 DEBUG:boto:Path: /avysmw-11/?prefix=%26 2022-09-03T05:54:07.255 DEBUG:boto:Data: 2022-09-03T05:54:07.256 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.257 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.258 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.258 DEBUG:boto:Params: {} 2022-09-03T05:54:07.259 DEBUG:boto:Token: None 2022-09-03T05:54:07.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.264 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090c4dbac1552e3c1-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.265 DEBUG:boto:b'avysmw-11&1000false&2022-09-03T05:52:37.005Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.266 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:07.266 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.267 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.268 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.268 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.269 DEBUG:boto:Method: GET 2022-09-03T05:54:07.269 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.270 DEBUG:boto:Data: 2022-09-03T05:54:07.270 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.271 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.272 DEBUG:boto:Params: {} 2022-09-03T05:54:07.273 DEBUG:boto:Token: None 2022-09-03T05:54:07.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYtdGhTPxqgrfqSeX27NL59d8zk= 2022-09-03T05:54:07.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYtdGhTPxqgrfqSeX27NL59d8zk=', 'Content-Length': '0'} 2022-09-03T05:54:07.277 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c1569371b07d0a1c-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.278 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.278 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.279 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.279 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.280 DEBUG:boto:Method: GET 2022-09-03T05:54:07.281 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.281 DEBUG:boto:Data: 2022-09-03T05:54:07.282 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.282 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.283 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.283 DEBUG:boto:Params: {} 2022-09-03T05:54:07.284 DEBUG:boto:Token: None 2022-09-03T05:54:07.284 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.285 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYtdGhTPxqgrfqSeX27NL59d8zk= 2022-09-03T05:54:07.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYtdGhTPxqgrfqSeX27NL59d8zk=', 'Content-Length': '0'} 2022-09-03T05:54:07.288 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000024d1dc610870001a-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.288 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.289 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.289 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.290 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.290 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.291 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.291 DEBUG:boto:Data: 2022-09-03T05:54:07.292 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.293 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.293 DEBUG:boto:Params: {} 2022-09-03T05:54:07.294 DEBUG:boto:Token: None 2022-09-03T05:54:07.294 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t2hnUDiXhba1VyV+TC0TwMHAH9A= 2022-09-03T05:54:07.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t2hnUDiXhba1VyV+TC0TwMHAH9A=', 'Content-Length': '0'} 2022-09-03T05:54:07.298 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000001d02dd98da3d0e1-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.298 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.299 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.299 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.300 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.300 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.301 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.301 DEBUG:boto:Data: 2022-09-03T05:54:07.302 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.302 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.303 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.303 DEBUG:boto:Params: {} 2022-09-03T05:54:07.304 DEBUG:boto:Token: None 2022-09-03T05:54:07.304 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t2hnUDiXhba1VyV+TC0TwMHAH9A= 2022-09-03T05:54:07.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t2hnUDiXhba1VyV+TC0TwMHAH9A=', 'Content-Length': '0'} 2022-09-03T05:54:07.307 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b8e79573f46e5a62-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.308 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:07.309 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.309 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.310 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.311 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.311 DEBUG:boto:Method: GET 2022-09-03T05:54:07.312 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.312 DEBUG:boto:Data: 2022-09-03T05:54:07.313 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.314 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.315 DEBUG:boto:Params: {} 2022-09-03T05:54:07.315 DEBUG:boto:Token: None 2022-09-03T05:54:07.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYtdGhTPxqgrfqSeX27NL59d8zk= 2022-09-03T05:54:07.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYtdGhTPxqgrfqSeX27NL59d8zk=', 'Content-Length': '0'} 2022-09-03T05:54:07.320 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000114a9822ab69f4db-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.321 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.321 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.322 DEBUG:boto:path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.322 DEBUG:boto:auth_path=/avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.323 DEBUG:boto:Method: GET 2022-09-03T05:54:07.324 DEBUG:boto:Path: /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.324 DEBUG:boto:Data: 2022-09-03T05:54:07.325 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.325 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.326 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.327 DEBUG:boto:Params: {} 2022-09-03T05:54:07.327 DEBUG:boto:Token: None 2022-09-03T05:54:07.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A?versionId=null 2022-09-03T05:54:07.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYtdGhTPxqgrfqSeX27NL59d8zk= 2022-09-03T05:54:07.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYtdGhTPxqgrfqSeX27NL59d8zk=', 'Content-Length': '0'} 2022-09-03T05:54:07.332 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a5bb5a5dc768aeb6-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.332 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.333 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.333 DEBUG:boto:path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:07.334 DEBUG:boto:auth_path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:07.334 DEBUG:boto:Method: GET 2022-09-03T05:54:07.335 DEBUG:boto:Path: /avysmw-11/?prefix=%3A 2022-09-03T05:54:07.336 DEBUG:boto:Data: 2022-09-03T05:54:07.337 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.338 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.339 DEBUG:boto:Params: {} 2022-09-03T05:54:07.339 DEBUG:boto:Token: None 2022-09-03T05:54:07.340 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.344 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000439a4be07f6f1c55-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.345 DEBUG:boto:b'avysmw-11:1000false:2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.346 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.346 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.347 DEBUG:boto:path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:07.347 DEBUG:boto:auth_path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:07.348 DEBUG:boto:Method: GET 2022-09-03T05:54:07.349 DEBUG:boto:Path: /avysmw-11/?prefix=%3A 2022-09-03T05:54:07.349 DEBUG:boto:Data: 2022-09-03T05:54:07.350 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.351 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.351 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.352 DEBUG:boto:Params: {} 2022-09-03T05:54:07.353 DEBUG:boto:Token: None 2022-09-03T05:54:07.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.358 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000efae61c00471497c-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.359 DEBUG:boto:b'avysmw-11:1000false:2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.360 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.360 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.361 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.361 DEBUG:boto:Path: /avysmw-11/%3A 2022-09-03T05:54:07.362 DEBUG:boto:Data: 2022-09-03T05:54:07.362 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.363 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.364 DEBUG:boto:Params: {} 2022-09-03T05:54:07.365 DEBUG:boto:Token: None 2022-09-03T05:54:07.365 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A 2022-09-03T05:54:07.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TTw6vv7c/C2zcdZImypyGoRUdZE= 2022-09-03T05:54:07.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TTw6vv7c/C2zcdZImypyGoRUdZE=', 'Content-Length': '0'} 2022-09-03T05:54:07.369 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000099a18ccd93c24507-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.369 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.370 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.370 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.371 DEBUG:boto:Path: /avysmw-11/%3A 2022-09-03T05:54:07.371 DEBUG:boto:Data: 2022-09-03T05:54:07.372 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.373 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.373 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.374 DEBUG:boto:Params: {} 2022-09-03T05:54:07.374 DEBUG:boto:Token: None 2022-09-03T05:54:07.375 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A 2022-09-03T05:54:07.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TTw6vv7c/C2zcdZImypyGoRUdZE= 2022-09-03T05:54:07.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TTw6vv7c/C2zcdZImypyGoRUdZE=', 'Content-Length': '0'} 2022-09-03T05:54:07.378 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b55bf3cd7a59691e-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.379 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:07.380 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.380 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.381 DEBUG:boto:Method: GET 2022-09-03T05:54:07.381 DEBUG:boto:Path: /avysmw-11/%3A 2022-09-03T05:54:07.382 DEBUG:boto:Data: 2022-09-03T05:54:07.383 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.384 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.385 DEBUG:boto:Params: {} 2022-09-03T05:54:07.386 DEBUG:boto:Token: None 2022-09-03T05:54:07.387 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A 2022-09-03T05:54:07.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SRyAlQGSLR+rrLLL+t3GXRtoxyg= 2022-09-03T05:54:07.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SRyAlQGSLR+rrLLL+t3GXRtoxyg=', 'Content-Length': '0'} 2022-09-03T05:54:07.391 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ca3e0663b3362332-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.392 DEBUG:boto:path=/avysmw-11/%3A 2022-09-03T05:54:07.392 DEBUG:boto:auth_path=/avysmw-11/%3A 2022-09-03T05:54:07.393 DEBUG:boto:Method: GET 2022-09-03T05:54:07.394 DEBUG:boto:Path: /avysmw-11/%3A 2022-09-03T05:54:07.394 DEBUG:boto:Data: 2022-09-03T05:54:07.395 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.396 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.397 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.397 DEBUG:boto:Params: {} 2022-09-03T05:54:07.398 DEBUG:boto:Token: None 2022-09-03T05:54:07.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/%3A 2022-09-03T05:54:07.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SRyAlQGSLR+rrLLL+t3GXRtoxyg= 2022-09-03T05:54:07.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SRyAlQGSLR+rrLLL+t3GXRtoxyg=', 'Content-Length': '0'} 2022-09-03T05:54:07.403 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000074a404d1a5e4662-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.446 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.448 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.448 DEBUG:boto:path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:07.450 DEBUG:boto:auth_path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:07.451 DEBUG:boto:Method: GET 2022-09-03T05:54:07.452 DEBUG:boto:Path: /avysmw-11/?prefix=%3A 2022-09-03T05:54:07.454 DEBUG:boto:Data: 2022-09-03T05:54:07.455 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.457 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.458 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.460 DEBUG:boto:Params: {} 2022-09-03T05:54:07.461 DEBUG:boto:Token: None 2022-09-03T05:54:07.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.471 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000072ba438e7a6ec7b-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.472 DEBUG:boto:b'avysmw-11:1000false:2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.474 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.475 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.476 DEBUG:boto:path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:07.476 DEBUG:boto:auth_path=/avysmw-11/?prefix=%3A 2022-09-03T05:54:07.477 DEBUG:boto:Method: GET 2022-09-03T05:54:07.478 DEBUG:boto:Path: /avysmw-11/?prefix=%3A 2022-09-03T05:54:07.479 DEBUG:boto:Data: 2022-09-03T05:54:07.479 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.480 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.481 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.482 DEBUG:boto:Params: {} 2022-09-03T05:54:07.482 DEBUG:boto:Token: None 2022-09-03T05:54:07.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.487 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c4106cca946e6ba2-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.488 DEBUG:boto:b'avysmw-11:1000false:2022-09-03T05:52:36.978Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.488 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:07.489 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.490 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.490 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.491 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.491 DEBUG:boto:Method: GET 2022-09-03T05:54:07.492 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.492 DEBUG:boto:Data: 2022-09-03T05:54:07.493 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.494 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.494 DEBUG:boto:Params: {} 2022-09-03T05:54:07.495 DEBUG:boto:Token: None 2022-09-03T05:54:07.496 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w9Ns97nNSoFvUwoVvzNmagjurMk= 2022-09-03T05:54:07.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w9Ns97nNSoFvUwoVvzNmagjurMk=', 'Content-Length': '0'} 2022-09-03T05:54:07.499 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006a090c42c7e547ba-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.500 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.500 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.501 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.501 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.502 DEBUG:boto:Method: GET 2022-09-03T05:54:07.503 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.503 DEBUG:boto:Data: 2022-09-03T05:54:07.504 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.504 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.505 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.506 DEBUG:boto:Params: {} 2022-09-03T05:54:07.506 DEBUG:boto:Token: None 2022-09-03T05:54:07.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w9Ns97nNSoFvUwoVvzNmagjurMk= 2022-09-03T05:54:07.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w9Ns97nNSoFvUwoVvzNmagjurMk=', 'Content-Length': '0'} 2022-09-03T05:54:07.511 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000461860a6bc82721f-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.512 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.512 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.513 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.513 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.514 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.515 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.515 DEBUG:boto:Data: 2022-09-03T05:54:07.516 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.517 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.517 DEBUG:boto:Params: {} 2022-09-03T05:54:07.518 DEBUG:boto:Token: None 2022-09-03T05:54:07.518 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X+c58ez+qtn7nSbPZpo2TQ9mn9Y= 2022-09-03T05:54:07.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X+c58ez+qtn7nSbPZpo2TQ9mn9Y=', 'Content-Length': '0'} 2022-09-03T05:54:07.521 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000088227643fc7d52d6-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.522 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.523 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.523 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.524 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.524 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.525 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.525 DEBUG:boto:Data: 2022-09-03T05:54:07.526 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.526 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.527 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.527 DEBUG:boto:Params: {} 2022-09-03T05:54:07.528 DEBUG:boto:Token: None 2022-09-03T05:54:07.528 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X+c58ez+qtn7nSbPZpo2TQ9mn9Y= 2022-09-03T05:54:07.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X+c58ez+qtn7nSbPZpo2TQ9mn9Y=', 'Content-Length': '0'} 2022-09-03T05:54:07.532 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004f3b6e345094108b-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.532 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:07.533 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.533 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.534 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.534 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.535 DEBUG:boto:Method: GET 2022-09-03T05:54:07.536 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.536 DEBUG:boto:Data: 2022-09-03T05:54:07.537 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.538 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.538 DEBUG:boto:Params: {} 2022-09-03T05:54:07.539 DEBUG:boto:Token: None 2022-09-03T05:54:07.539 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w9Ns97nNSoFvUwoVvzNmagjurMk= 2022-09-03T05:54:07.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w9Ns97nNSoFvUwoVvzNmagjurMk=', 'Content-Length': '0'} 2022-09-03T05:54:07.543 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001930f502558775b6-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.590 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.591 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.591 DEBUG:boto:path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.592 DEBUG:boto:auth_path=/avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.592 DEBUG:boto:Method: GET 2022-09-03T05:54:07.593 DEBUG:boto:Path: /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.593 DEBUG:boto:Data: 2022-09-03T05:54:07.594 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.594 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.595 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.596 DEBUG:boto:Params: {} 2022-09-03T05:54:07.596 DEBUG:boto:Token: None 2022-09-03T05:54:07.597 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj?versionId=null 2022-09-03T05:54:07.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w9Ns97nNSoFvUwoVvzNmagjurMk= 2022-09-03T05:54:07.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w9Ns97nNSoFvUwoVvzNmagjurMk=', 'Content-Length': '0'} 2022-09-03T05:54:07.600 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000042e892525a1b8ced-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.601 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.601 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.602 DEBUG:boto:path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.603 DEBUG:boto:auth_path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.603 DEBUG:boto:Method: GET 2022-09-03T05:54:07.604 DEBUG:boto:Path: /avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.604 DEBUG:boto:Data: 2022-09-03T05:54:07.605 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.605 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.606 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.606 DEBUG:boto:Params: {} 2022-09-03T05:54:07.607 DEBUG:boto:Token: None 2022-09-03T05:54:07.607 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.611 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000977af776362f92ad-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.612 DEBUG:boto:b'avysmw-11_myobj1000false_myobj2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.613 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.613 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.614 DEBUG:boto:path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.614 DEBUG:boto:auth_path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.615 DEBUG:boto:Method: GET 2022-09-03T05:54:07.615 DEBUG:boto:Path: /avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.616 DEBUG:boto:Data: 2022-09-03T05:54:07.616 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.617 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.617 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.618 DEBUG:boto:Params: {} 2022-09-03T05:54:07.619 DEBUG:boto:Token: None 2022-09-03T05:54:07.619 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.623 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d4df246838256a8c-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.624 DEBUG:boto:b'avysmw-11_myobj1000false_myobj2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.624 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.625 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.625 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.626 DEBUG:boto:Path: /avysmw-11/_myobj 2022-09-03T05:54:07.626 DEBUG:boto:Data: 2022-09-03T05:54:07.627 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.628 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.629 DEBUG:boto:Params: {} 2022-09-03T05:54:07.629 DEBUG:boto:Token: None 2022-09-03T05:54:07.630 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj 2022-09-03T05:54:07.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9EAMVBjxBY5ROviMgf7ZrwRBkyk= 2022-09-03T05:54:07.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9EAMVBjxBY5ROviMgf7ZrwRBkyk=', 'Content-Length': '0'} 2022-09-03T05:54:07.633 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e9bbb0f49bd2060f-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.634 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.635 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.635 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.636 DEBUG:boto:Path: /avysmw-11/_myobj 2022-09-03T05:54:07.636 DEBUG:boto:Data: 2022-09-03T05:54:07.637 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.637 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.638 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.638 DEBUG:boto:Params: {} 2022-09-03T05:54:07.639 DEBUG:boto:Token: None 2022-09-03T05:54:07.639 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj 2022-09-03T05:54:07.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9EAMVBjxBY5ROviMgf7ZrwRBkyk= 2022-09-03T05:54:07.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9EAMVBjxBY5ROviMgf7ZrwRBkyk=', 'Content-Length': '0'} 2022-09-03T05:54:07.642 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000017b063b41c0247fa-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.643 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:07.644 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.644 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.645 DEBUG:boto:Method: GET 2022-09-03T05:54:07.645 DEBUG:boto:Path: /avysmw-11/_myobj 2022-09-03T05:54:07.646 DEBUG:boto:Data: 2022-09-03T05:54:07.646 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.647 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.648 DEBUG:boto:Params: {} 2022-09-03T05:54:07.648 DEBUG:boto:Token: None 2022-09-03T05:54:07.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj 2022-09-03T05:54:07.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QAvirFtzrNgXJlcDCIDl5WrffLc= 2022-09-03T05:54:07.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QAvirFtzrNgXJlcDCIDl5WrffLc=', 'Content-Length': '0'} 2022-09-03T05:54:07.652 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004a2ebe61dcf08104-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.653 DEBUG:boto:path=/avysmw-11/_myobj 2022-09-03T05:54:07.654 DEBUG:boto:auth_path=/avysmw-11/_myobj 2022-09-03T05:54:07.654 DEBUG:boto:Method: GET 2022-09-03T05:54:07.655 DEBUG:boto:Path: /avysmw-11/_myobj 2022-09-03T05:54:07.656 DEBUG:boto:Data: 2022-09-03T05:54:07.657 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.657 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.658 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.659 DEBUG:boto:Params: {} 2022-09-03T05:54:07.660 DEBUG:boto:Token: None 2022-09-03T05:54:07.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/_myobj 2022-09-03T05:54:07.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QAvirFtzrNgXJlcDCIDl5WrffLc= 2022-09-03T05:54:07.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QAvirFtzrNgXJlcDCIDl5WrffLc=', 'Content-Length': '0'} 2022-09-03T05:54:07.664 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bd314c866a6bc1a1-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.665 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.666 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.667 DEBUG:boto:path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.668 DEBUG:boto:auth_path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.669 DEBUG:boto:Method: GET 2022-09-03T05:54:07.670 DEBUG:boto:Path: /avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.670 DEBUG:boto:Data: 2022-09-03T05:54:07.671 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.672 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.673 DEBUG:boto:Params: {} 2022-09-03T05:54:07.674 DEBUG:boto:Token: None 2022-09-03T05:54:07.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.680 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000081c4a6be242dbd92-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.681 DEBUG:boto:b'avysmw-11_myobj1000false_myobj2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.682 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.683 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.684 DEBUG:boto:path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.685 DEBUG:boto:auth_path=/avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.686 DEBUG:boto:Method: GET 2022-09-03T05:54:07.687 DEBUG:boto:Path: /avysmw-11/?prefix=_myobj 2022-09-03T05:54:07.688 DEBUG:boto:Data: 2022-09-03T05:54:07.689 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.689 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.690 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.690 DEBUG:boto:Params: {} 2022-09-03T05:54:07.691 DEBUG:boto:Token: None 2022-09-03T05:54:07.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.695 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075bb0ae93ea86500-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.696 DEBUG:boto:b'avysmw-11_myobj1000false_myobj2022-09-03T05:52:36.916Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.697 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:07.698 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.698 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.699 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.699 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.700 DEBUG:boto:Method: GET 2022-09-03T05:54:07.701 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.701 DEBUG:boto:Data: 2022-09-03T05:54:07.702 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.702 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.703 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.703 DEBUG:boto:Params: {} 2022-09-03T05:54:07.704 DEBUG:boto:Token: None 2022-09-03T05:54:07.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0mfCom0inTz4lW5e8bTgGpuzJtc= 2022-09-03T05:54:07.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0mfCom0inTz4lW5e8bTgGpuzJtc=', 'Content-Length': '0'} 2022-09-03T05:54:07.710 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005dc47c28a65645a1-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.711 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.712 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.713 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.713 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.714 DEBUG:boto:Method: GET 2022-09-03T05:54:07.715 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.715 DEBUG:boto:Data: 2022-09-03T05:54:07.716 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.717 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.717 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.718 DEBUG:boto:Params: {} 2022-09-03T05:54:07.720 DEBUG:boto:Token: None 2022-09-03T05:54:07.721 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.722 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0mfCom0inTz4lW5e8bTgGpuzJtc= 2022-09-03T05:54:07.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0mfCom0inTz4lW5e8bTgGpuzJtc=', 'Content-Length': '0'} 2022-09-03T05:54:07.726 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c48102bfd1b623e7-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.766 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.767 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.767 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.768 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.768 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.769 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.769 DEBUG:boto:Data: 2022-09-03T05:54:07.770 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.771 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.771 DEBUG:boto:Params: {} 2022-09-03T05:54:07.772 DEBUG:boto:Token: None 2022-09-03T05:54:07.773 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:72qui+dYxUt1z0W30sbv0C4H8Zo= 2022-09-03T05:54:07.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:72qui+dYxUt1z0W30sbv0C4H8Zo=', 'Content-Length': '0'} 2022-09-03T05:54:07.778 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c03a1ac64c8264e5-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.779 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.780 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.780 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.781 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.781 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.782 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.783 DEBUG:boto:Data: 2022-09-03T05:54:07.783 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.784 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.784 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.785 DEBUG:boto:Params: {} 2022-09-03T05:54:07.785 DEBUG:boto:Token: None 2022-09-03T05:54:07.786 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:72qui+dYxUt1z0W30sbv0C4H8Zo= 2022-09-03T05:54:07.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:72qui+dYxUt1z0W30sbv0C4H8Zo=', 'Content-Length': '0'} 2022-09-03T05:54:07.789 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b6fd9e9ead7dffda-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.790 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:07.791 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.791 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.792 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.792 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.793 DEBUG:boto:Method: GET 2022-09-03T05:54:07.794 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.794 DEBUG:boto:Data: 2022-09-03T05:54:07.795 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.796 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.797 DEBUG:boto:Params: {} 2022-09-03T05:54:07.797 DEBUG:boto:Token: None 2022-09-03T05:54:07.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0mfCom0inTz4lW5e8bTgGpuzJtc= 2022-09-03T05:54:07.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0mfCom0inTz4lW5e8bTgGpuzJtc=', 'Content-Length': '0'} 2022-09-03T05:54:07.802 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005a555479c6e63c85-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.802 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.803 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.804 DEBUG:boto:path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.805 DEBUG:boto:auth_path=/avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.805 DEBUG:boto:Method: GET 2022-09-03T05:54:07.806 DEBUG:boto:Path: /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.806 DEBUG:boto:Data: 2022-09-03T05:54:07.807 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.807 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.808 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.809 DEBUG:boto:Params: {} 2022-09-03T05:54:07.809 DEBUG:boto:Token: None 2022-09-03T05:54:07.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj?versionId=null 2022-09-03T05:54:07.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0mfCom0inTz4lW5e8bTgGpuzJtc= 2022-09-03T05:54:07.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0mfCom0inTz4lW5e8bTgGpuzJtc=', 'Content-Length': '0'} 2022-09-03T05:54:07.813 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002276a52c2cd1e95d-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.815 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.815 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.815 DEBUG:boto:path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:07.816 DEBUG:boto:auth_path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:07.816 DEBUG:boto:Method: GET 2022-09-03T05:54:07.817 DEBUG:boto:Path: /avysmw-11/?prefix=myobj 2022-09-03T05:54:07.818 DEBUG:boto:Data: 2022-09-03T05:54:07.818 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.819 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.820 DEBUG:boto:Params: {} 2022-09-03T05:54:07.820 DEBUG:boto:Token: None 2022-09-03T05:54:07.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.825 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000741dea4f5f8e28d4-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.825 DEBUG:boto:b'avysmw-11myobj1000falsemyobj2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.826 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.827 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.827 DEBUG:boto:path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:07.828 DEBUG:boto:auth_path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:07.828 DEBUG:boto:Method: GET 2022-09-03T05:54:07.829 DEBUG:boto:Path: /avysmw-11/?prefix=myobj 2022-09-03T05:54:07.829 DEBUG:boto:Data: 2022-09-03T05:54:07.830 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.831 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.831 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.832 DEBUG:boto:Params: {} 2022-09-03T05:54:07.832 DEBUG:boto:Token: None 2022-09-03T05:54:07.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.836 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e2d6a5de68f046fb-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.837 DEBUG:boto:b'avysmw-11myobj1000falsemyobj2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.838 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.838 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.839 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.839 DEBUG:boto:Path: /avysmw-11/myobj 2022-09-03T05:54:07.840 DEBUG:boto:Data: 2022-09-03T05:54:07.840 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.841 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.841 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.842 DEBUG:boto:Params: {} 2022-09-03T05:54:07.842 DEBUG:boto:Token: None 2022-09-03T05:54:07.843 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj 2022-09-03T05:54:07.843 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LHVetbGrZeDLCfRywd2rHDfiXLU= 2022-09-03T05:54:07.844 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LHVetbGrZeDLCfRywd2rHDfiXLU=', 'Content-Length': '0'} 2022-09-03T05:54:07.846 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bbb50e29e89863de-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.847 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.847 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.848 DEBUG:boto:Method: HEAD 2022-09-03T05:54:07.848 DEBUG:boto:Path: /avysmw-11/myobj 2022-09-03T05:54:07.849 DEBUG:boto:Data: 2022-09-03T05:54:07.849 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.850 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.850 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.851 DEBUG:boto:Params: {} 2022-09-03T05:54:07.851 DEBUG:boto:Token: None 2022-09-03T05:54:07.852 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj 2022-09-03T05:54:07.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LHVetbGrZeDLCfRywd2rHDfiXLU= 2022-09-03T05:54:07.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LHVetbGrZeDLCfRywd2rHDfiXLU=', 'Content-Length': '0'} 2022-09-03T05:54:07.855 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002755c90944ac633e-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.856 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:07.857 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.857 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.858 DEBUG:boto:Method: GET 2022-09-03T05:54:07.859 DEBUG:boto:Path: /avysmw-11/myobj 2022-09-03T05:54:07.859 DEBUG:boto:Data: 2022-09-03T05:54:07.860 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.861 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.861 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.862 DEBUG:boto:Params: {} 2022-09-03T05:54:07.862 DEBUG:boto:Token: None 2022-09-03T05:54:07.863 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj 2022-09-03T05:54:07.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JN/+kRyuQ+2KlQ1G7aUSgG4jSYA= 2022-09-03T05:54:07.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JN/+kRyuQ+2KlQ1G7aUSgG4jSYA=', 'Content-Length': '0'} 2022-09-03T05:54:07.866 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e8a85b686774a67d-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.910 DEBUG:boto:path=/avysmw-11/myobj 2022-09-03T05:54:07.911 DEBUG:boto:auth_path=/avysmw-11/myobj 2022-09-03T05:54:07.911 DEBUG:boto:Method: GET 2022-09-03T05:54:07.912 DEBUG:boto:Path: /avysmw-11/myobj 2022-09-03T05:54:07.912 DEBUG:boto:Data: 2022-09-03T05:54:07.913 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.913 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.914 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.915 DEBUG:boto:Params: {} 2022-09-03T05:54:07.915 DEBUG:boto:Token: None 2022-09-03T05:54:07.916 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/myobj 2022-09-03T05:54:07.917 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JN/+kRyuQ+2KlQ1G7aUSgG4jSYA= 2022-09-03T05:54:07.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JN/+kRyuQ+2KlQ1G7aUSgG4jSYA=', 'Content-Length': '0'} 2022-09-03T05:54:07.920 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:36 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003cfb3c50307d629e-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.920 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.921 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.921 DEBUG:boto:path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:07.922 DEBUG:boto:auth_path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:07.923 DEBUG:boto:Method: GET 2022-09-03T05:54:07.923 DEBUG:boto:Path: /avysmw-11/?prefix=myobj 2022-09-03T05:54:07.924 DEBUG:boto:Data: 2022-09-03T05:54:07.924 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.925 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:07.925 DEBUG:boto:Port: 8001 2022-09-03T05:54:07.926 DEBUG:boto:Params: {} 2022-09-03T05:54:07.926 DEBUG:boto:Token: None 2022-09-03T05:54:07.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.931 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d834b267f68a3852-006312ebff-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.932 DEBUG:boto:b'avysmw-11myobj1000falsemyobj2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.933 DEBUG:boto:path=/avysmw-11/ 2022-09-03T05:54:07.933 DEBUG:boto:auth_path=/avysmw-11/ 2022-09-03T05:54:07.934 DEBUG:boto:path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:07.934 DEBUG:boto:auth_path=/avysmw-11/?prefix=myobj 2022-09-03T05:54:07.935 DEBUG:boto:Method: GET 2022-09-03T05:54:07.935 DEBUG:boto:Path: /avysmw-11/?prefix=myobj 2022-09-03T05:54:07.936 DEBUG:boto:Data: 2022-09-03T05:54:07.936 DEBUG:boto:Headers: {} 2022-09-03T05:54:07.937 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:07.937 DEBUG:boto:Port: 8000 2022-09-03T05:54:07.938 DEBUG:boto:Params: {} 2022-09-03T05:54:07.939 DEBUG:boto:Token: None 2022-09-03T05:54:07.939 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:07 GMT /avysmw-11/ 2022-09-03T05:54:07.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI= 2022-09-03T05:54:07.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIJm4/5KkA2SN4m9ZK6/Mx9GBcI=', 'Content-Length': '0'} 2022-09-03T05:54:07.943 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005118ebeb31eeccb5-006312ebff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:07.944 DEBUG:boto:b'avysmw-11myobj1000falsemyobj2022-09-03T05:52:36.887Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:07.944 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-11 zones={test-zone3, test-zone2} 2022-09-03T05:54:07.945 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-12', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:07.946 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-12', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:07.946 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-12 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:08.012 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:08.013 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:08.053 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:08.050+0000 7fc9dfdd6600 1 bucket sync caught up with source: 2022-09-03T05:54:08.053 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.33.5, , 00000000001.21.5, , , , 00000000001.17.5, 00000000001.19.5, , ] 2022-09-03T05:54:08.053 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.33.5, , 00000000001.21.5, , , , 00000000001.17.5, 00000000001.19.5, , ] 2022-09-03T05:54:08.054 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:08.050+0000 7fc9dfdd6600 0 bucket checkpoint complete 2022-09-03T05:54:08.069 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-12 zones={test-zone1, test-zone3} 2022-09-03T05:54:08.071 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.071 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.072 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.073 DEBUG:boto:Path: /avysmw-12/ 2022-09-03T05:54:08.074 DEBUG:boto:Data: 2022-09-03T05:54:08.075 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.076 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.076 DEBUG:boto:Params: {} 2022-09-03T05:54:08.077 DEBUG:boto:Token: None 2022-09-03T05:54:08.078 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:luOD5Q+UcV5Bm5ZNL8lUy8XuWZI= 2022-09-03T05:54:08.079 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:luOD5Q+UcV5Bm5ZNL8lUy8XuWZI=', 'Content-Length': '0'} 2022-09-03T05:54:08.081 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003072c42ebb9135ee-006312ec00-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.082 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.082 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.083 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.083 DEBUG:boto:Path: /avysmw-12/ 2022-09-03T05:54:08.084 DEBUG:boto:Data: 2022-09-03T05:54:08.084 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.085 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.086 DEBUG:boto:Params: {} 2022-09-03T05:54:08.087 DEBUG:boto:Token: None 2022-09-03T05:54:08.087 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:luOD5Q+UcV5Bm5ZNL8lUy8XuWZI= 2022-09-03T05:54:08.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:luOD5Q+UcV5Bm5ZNL8lUy8XuWZI=', 'Content-Length': '0'} 2022-09-03T05:54:08.091 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000057f578498983c290-006312ec00-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.091 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:08.092 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.092 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.093 DEBUG:boto:path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.093 DEBUG:boto:auth_path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.094 DEBUG:boto:Method: GET 2022-09-03T05:54:08.094 DEBUG:boto:Path: /avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.095 DEBUG:boto:Data: 2022-09-03T05:54:08.096 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.096 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.097 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.097 DEBUG:boto:Params: {} 2022-09-03T05:54:08.098 DEBUG:boto:Token: None 2022-09-03T05:54:08.099 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/?versions 2022-09-03T05:54:08.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MI43sk+cDJj8z0Nbfwn4L66OK5E= 2022-09-03T05:54:08.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MI43sk+cDJj8z0Nbfwn4L66OK5E=', 'Content-Length': '0'} 2022-09-03T05:54:08.103 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f74e9991b551ea03-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.103 DEBUG:boto:b'avysmw-12999false&nulltrue2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.104 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:54:08.105 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:54:08.105 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:54:08.106 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:08.106 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:08.107 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.108 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.108 DEBUG:boto:path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.109 DEBUG:boto:auth_path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.109 DEBUG:boto:Method: GET 2022-09-03T05:54:08.110 DEBUG:boto:Path: /avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.111 DEBUG:boto:Data: 2022-09-03T05:54:08.111 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.112 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.112 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.113 DEBUG:boto:Params: {} 2022-09-03T05:54:08.113 DEBUG:boto:Token: None 2022-09-03T05:54:08.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/?versions 2022-09-03T05:54:08.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MI43sk+cDJj8z0Nbfwn4L66OK5E= 2022-09-03T05:54:08.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MI43sk+cDJj8z0Nbfwn4L66OK5E=', 'Content-Length': '0'} 2022-09-03T05:54:08.118 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000030e2e5973d1399bc-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.118 DEBUG:boto:b'avysmw-12999false&nulltrue2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.119 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:54:08.120 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:54:08.121 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:54:08.122 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:08.123 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.124 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.125 DEBUG:boto:path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.126 DEBUG:boto:auth_path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.127 DEBUG:boto:Method: GET 2022-09-03T05:54:08.127 DEBUG:boto:Path: /avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.128 DEBUG:boto:Data: 2022-09-03T05:54:08.128 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.129 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.129 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.130 DEBUG:boto:Params: {} 2022-09-03T05:54:08.131 DEBUG:boto:Token: None 2022-09-03T05:54:08.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/?versions 2022-09-03T05:54:08.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MI43sk+cDJj8z0Nbfwn4L66OK5E= 2022-09-03T05:54:08.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MI43sk+cDJj8z0Nbfwn4L66OK5E=', 'Content-Length': '0'} 2022-09-03T05:54:08.135 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009747527f39d77e12-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.135 DEBUG:boto:b'avysmw-12999false&nulltrue2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.136 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.137 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.138 DEBUG:boto:path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.138 DEBUG:boto:auth_path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.139 DEBUG:boto:Method: GET 2022-09-03T05:54:08.139 DEBUG:boto:Path: /avysmw-12/?versions&max-keys=999 2022-09-03T05:54:08.140 DEBUG:boto:Data: 2022-09-03T05:54:08.140 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.141 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.141 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.142 DEBUG:boto:Params: {} 2022-09-03T05:54:08.142 DEBUG:boto:Token: None 2022-09-03T05:54:08.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/?versions 2022-09-03T05:54:08.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MI43sk+cDJj8z0Nbfwn4L66OK5E= 2022-09-03T05:54:08.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MI43sk+cDJj8z0Nbfwn4L66OK5E=', 'Content-Length': '0'} 2022-09-03T05:54:08.147 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000163645d7316dcf3a-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.148 DEBUG:boto:b'avysmw-12999false&nulltrue2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.149 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:08.149 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.150 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.150 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.151 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.152 DEBUG:boto:Method: GET 2022-09-03T05:54:08.152 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.153 DEBUG:boto:Data: 2022-09-03T05:54:08.153 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.154 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.154 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.155 DEBUG:boto:Params: {} 2022-09-03T05:54:08.155 DEBUG:boto:Token: None 2022-09-03T05:54:08.156 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GMEJNzRkkRRIdst/UBzrrrzbKdY= 2022-09-03T05:54:08.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GMEJNzRkkRRIdst/UBzrrrzbKdY=', 'Content-Length': '0'} 2022-09-03T05:54:08.169 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d72a4f81d8edeec5-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.210 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.211 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.212 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.214 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.215 DEBUG:boto:Method: GET 2022-09-03T05:54:08.216 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.216 DEBUG:boto:Data: 2022-09-03T05:54:08.217 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.218 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.219 DEBUG:boto:Params: {} 2022-09-03T05:54:08.219 DEBUG:boto:Token: None 2022-09-03T05:54:08.220 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GMEJNzRkkRRIdst/UBzrrrzbKdY= 2022-09-03T05:54:08.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GMEJNzRkkRRIdst/UBzrrrzbKdY=', 'Content-Length': '0'} 2022-09-03T05:54:08.224 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005a7643262a8d8004-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.225 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.225 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.226 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.226 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.227 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.227 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.228 DEBUG:boto:Data: 2022-09-03T05:54:08.228 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.229 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.229 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.230 DEBUG:boto:Params: {} 2022-09-03T05:54:08.231 DEBUG:boto:Token: None 2022-09-03T05:54:08.231 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IGJJyazXoI6OOci6QKHr8O9OyYk= 2022-09-03T05:54:08.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IGJJyazXoI6OOci6QKHr8O9OyYk=', 'Content-Length': '0'} 2022-09-03T05:54:08.234 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004e53678052140395-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.235 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.236 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.236 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.236 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.237 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.238 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.238 DEBUG:boto:Data: 2022-09-03T05:54:08.239 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.240 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.240 DEBUG:boto:Params: {} 2022-09-03T05:54:08.241 DEBUG:boto:Token: None 2022-09-03T05:54:08.241 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IGJJyazXoI6OOci6QKHr8O9OyYk= 2022-09-03T05:54:08.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IGJJyazXoI6OOci6QKHr8O9OyYk=', 'Content-Length': '0'} 2022-09-03T05:54:08.244 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001c7d351bce5f0cf7-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.245 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:08.246 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.246 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.247 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.247 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.248 DEBUG:boto:Method: GET 2022-09-03T05:54:08.248 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.249 DEBUG:boto:Data: 2022-09-03T05:54:08.249 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.251 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.251 DEBUG:boto:Params: {} 2022-09-03T05:54:08.252 DEBUG:boto:Token: None 2022-09-03T05:54:08.252 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GMEJNzRkkRRIdst/UBzrrrzbKdY= 2022-09-03T05:54:08.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GMEJNzRkkRRIdst/UBzrrrzbKdY=', 'Content-Length': '0'} 2022-09-03T05:54:08.255 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f5227c028170edcd-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.256 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.257 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.257 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.258 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:08.258 DEBUG:boto:Method: GET 2022-09-03T05:54:08.259 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.259 DEBUG:boto:Data: 2022-09-03T05:54:08.260 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.261 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.261 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.261 DEBUG:boto:Params: {} 2022-09-03T05:54:08.262 DEBUG:boto:Token: None 2022-09-03T05:54:08.263 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:08.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GMEJNzRkkRRIdst/UBzrrrzbKdY= 2022-09-03T05:54:08.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GMEJNzRkkRRIdst/UBzrrrzbKdY=', 'Content-Length': '0'} 2022-09-03T05:54:08.266 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a1de73429f4b0bf2-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.267 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.267 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.268 DEBUG:boto:path=/avysmw-12/?prefix=%26 2022-09-03T05:54:08.269 DEBUG:boto:auth_path=/avysmw-12/?prefix=%26 2022-09-03T05:54:08.269 DEBUG:boto:Method: GET 2022-09-03T05:54:08.270 DEBUG:boto:Path: /avysmw-12/?prefix=%26 2022-09-03T05:54:08.270 DEBUG:boto:Data: 2022-09-03T05:54:08.271 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.272 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.272 DEBUG:boto:Params: {} 2022-09-03T05:54:08.273 DEBUG:boto:Token: None 2022-09-03T05:54:08.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.277 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004ab615f045dd7a9a-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.278 DEBUG:boto:b'avysmw-12&1000false&2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.279 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.279 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.280 DEBUG:boto:path=/avysmw-12/?prefix=%26 2022-09-03T05:54:08.280 DEBUG:boto:auth_path=/avysmw-12/?prefix=%26 2022-09-03T05:54:08.281 DEBUG:boto:Method: GET 2022-09-03T05:54:08.281 DEBUG:boto:Path: /avysmw-12/?prefix=%26 2022-09-03T05:54:08.282 DEBUG:boto:Data: 2022-09-03T05:54:08.282 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.283 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.284 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.284 DEBUG:boto:Params: {} 2022-09-03T05:54:08.285 DEBUG:boto:Token: None 2022-09-03T05:54:08.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.289 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c13ff50596e5d38-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.334 DEBUG:boto:b'avysmw-12&1000false&2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.335 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.336 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.337 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.337 DEBUG:boto:Path: /avysmw-12/%26 2022-09-03T05:54:08.338 DEBUG:boto:Data: 2022-09-03T05:54:08.338 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.339 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.340 DEBUG:boto:Params: {} 2022-09-03T05:54:08.340 DEBUG:boto:Token: None 2022-09-03T05:54:08.341 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26 2022-09-03T05:54:08.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o2piZBS05w62rVuwtqebzPoKT08= 2022-09-03T05:54:08.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o2piZBS05w62rVuwtqebzPoKT08=', 'Content-Length': '0'} 2022-09-03T05:54:08.344 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a653072110a8aa55-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.345 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.345 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.346 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.346 DEBUG:boto:Path: /avysmw-12/%26 2022-09-03T05:54:08.347 DEBUG:boto:Data: 2022-09-03T05:54:08.348 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.348 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.349 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.349 DEBUG:boto:Params: {} 2022-09-03T05:54:08.350 DEBUG:boto:Token: None 2022-09-03T05:54:08.351 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26 2022-09-03T05:54:08.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o2piZBS05w62rVuwtqebzPoKT08= 2022-09-03T05:54:08.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o2piZBS05w62rVuwtqebzPoKT08=', 'Content-Length': '0'} 2022-09-03T05:54:08.354 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000097cc5927f2b3a4cf-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.355 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:08.355 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.356 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.356 DEBUG:boto:Method: GET 2022-09-03T05:54:08.357 DEBUG:boto:Path: /avysmw-12/%26 2022-09-03T05:54:08.358 DEBUG:boto:Data: 2022-09-03T05:54:08.358 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.359 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.360 DEBUG:boto:Params: {} 2022-09-03T05:54:08.361 DEBUG:boto:Token: None 2022-09-03T05:54:08.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26 2022-09-03T05:54:08.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oxjy/uPg8J/p1VwzuWk+cgP/O1M= 2022-09-03T05:54:08.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oxjy/uPg8J/p1VwzuWk+cgP/O1M=', 'Content-Length': '0'} 2022-09-03T05:54:08.365 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000059ca3b98f0f5e7ec-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.366 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:08.366 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:08.367 DEBUG:boto:Method: GET 2022-09-03T05:54:08.367 DEBUG:boto:Path: /avysmw-12/%26 2022-09-03T05:54:08.368 DEBUG:boto:Data: 2022-09-03T05:54:08.369 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.370 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.371 DEBUG:boto:Params: {} 2022-09-03T05:54:08.372 DEBUG:boto:Token: None 2022-09-03T05:54:08.373 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%26 2022-09-03T05:54:08.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oxjy/uPg8J/p1VwzuWk+cgP/O1M= 2022-09-03T05:54:08.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oxjy/uPg8J/p1VwzuWk+cgP/O1M=', 'Content-Length': '0'} 2022-09-03T05:54:08.376 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000068d8e2674a8dda8f-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.377 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.377 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.378 DEBUG:boto:path=/avysmw-12/?prefix=%26 2022-09-03T05:54:08.379 DEBUG:boto:auth_path=/avysmw-12/?prefix=%26 2022-09-03T05:54:08.379 DEBUG:boto:Method: GET 2022-09-03T05:54:08.380 DEBUG:boto:Path: /avysmw-12/?prefix=%26 2022-09-03T05:54:08.380 DEBUG:boto:Data: 2022-09-03T05:54:08.381 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.381 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.382 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.383 DEBUG:boto:Params: {} 2022-09-03T05:54:08.383 DEBUG:boto:Token: None 2022-09-03T05:54:08.384 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.388 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085a0bc9860faa08b-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.388 DEBUG:boto:b'avysmw-12&1000false&2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.389 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.390 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.390 DEBUG:boto:path=/avysmw-12/?prefix=%26 2022-09-03T05:54:08.391 DEBUG:boto:auth_path=/avysmw-12/?prefix=%26 2022-09-03T05:54:08.391 DEBUG:boto:Method: GET 2022-09-03T05:54:08.392 DEBUG:boto:Path: /avysmw-12/?prefix=%26 2022-09-03T05:54:08.392 DEBUG:boto:Data: 2022-09-03T05:54:08.393 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.394 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.394 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.395 DEBUG:boto:Params: {} 2022-09-03T05:54:08.396 DEBUG:boto:Token: None 2022-09-03T05:54:08.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.402 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b8d2d3ff4f89b79-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.403 DEBUG:boto:b'avysmw-12&1000false&2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.404 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:08.405 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.406 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.407 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.408 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.408 DEBUG:boto:Method: GET 2022-09-03T05:54:08.409 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.409 DEBUG:boto:Data: 2022-09-03T05:54:08.410 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.410 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.411 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.411 DEBUG:boto:Params: {} 2022-09-03T05:54:08.412 DEBUG:boto:Token: None 2022-09-03T05:54:08.413 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.414 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EF84ciOMZ8cFoNL2FN8CcgW4SdA= 2022-09-03T05:54:08.414 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EF84ciOMZ8cFoNL2FN8CcgW4SdA=', 'Content-Length': '0'} 2022-09-03T05:54:08.416 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000348064c8e0b0b640-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.417 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.417 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.418 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.419 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.419 DEBUG:boto:Method: GET 2022-09-03T05:54:08.420 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.420 DEBUG:boto:Data: 2022-09-03T05:54:08.421 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.422 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.422 DEBUG:boto:Params: {} 2022-09-03T05:54:08.423 DEBUG:boto:Token: None 2022-09-03T05:54:08.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EF84ciOMZ8cFoNL2FN8CcgW4SdA= 2022-09-03T05:54:08.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EF84ciOMZ8cFoNL2FN8CcgW4SdA=', 'Content-Length': '0'} 2022-09-03T05:54:08.427 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000daf111f525e34288-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.428 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.428 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.429 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.429 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.430 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.430 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.431 DEBUG:boto:Data: 2022-09-03T05:54:08.431 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.433 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.433 DEBUG:boto:Params: {} 2022-09-03T05:54:08.434 DEBUG:boto:Token: None 2022-09-03T05:54:08.434 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XPBp82xDb/fWkV2+C0gS3bqe/7g= 2022-09-03T05:54:08.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XPBp82xDb/fWkV2+C0gS3bqe/7g=', 'Content-Length': '0'} 2022-09-03T05:54:08.438 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a06ab0ca77512a1f-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.438 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.439 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.440 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.441 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.442 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.443 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.444 DEBUG:boto:Data: 2022-09-03T05:54:08.445 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.446 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.447 DEBUG:boto:Params: {} 2022-09-03T05:54:08.448 DEBUG:boto:Token: None 2022-09-03T05:54:08.448 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XPBp82xDb/fWkV2+C0gS3bqe/7g= 2022-09-03T05:54:08.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XPBp82xDb/fWkV2+C0gS3bqe/7g=', 'Content-Length': '0'} 2022-09-03T05:54:08.452 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006e28dd80c12a81e1-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.452 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:08.453 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.453 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.454 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.455 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.455 DEBUG:boto:Method: GET 2022-09-03T05:54:08.456 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.456 DEBUG:boto:Data: 2022-09-03T05:54:08.457 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.457 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.458 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.458 DEBUG:boto:Params: {} 2022-09-03T05:54:08.459 DEBUG:boto:Token: None 2022-09-03T05:54:08.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.460 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EF84ciOMZ8cFoNL2FN8CcgW4SdA= 2022-09-03T05:54:08.461 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EF84ciOMZ8cFoNL2FN8CcgW4SdA=', 'Content-Length': '0'} 2022-09-03T05:54:08.463 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000071371bc9ed2822c3-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.465 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.466 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.467 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.468 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.469 DEBUG:boto:Method: GET 2022-09-03T05:54:08.470 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.471 DEBUG:boto:Data: 2022-09-03T05:54:08.471 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.473 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.474 DEBUG:boto:Params: {} 2022-09-03T05:54:08.476 DEBUG:boto:Token: None 2022-09-03T05:54:08.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:08.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EF84ciOMZ8cFoNL2FN8CcgW4SdA= 2022-09-03T05:54:08.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EF84ciOMZ8cFoNL2FN8CcgW4SdA=', 'Content-Length': '0'} 2022-09-03T05:54:08.482 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000038d234557d8b9ecb-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.522 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.523 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.524 DEBUG:boto:path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:08.525 DEBUG:boto:auth_path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:08.526 DEBUG:boto:Method: GET 2022-09-03T05:54:08.527 DEBUG:boto:Path: /avysmw-12/?prefix=%3A 2022-09-03T05:54:08.528 DEBUG:boto:Data: 2022-09-03T05:54:08.529 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.530 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.530 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.531 DEBUG:boto:Params: {} 2022-09-03T05:54:08.531 DEBUG:boto:Token: None 2022-09-03T05:54:08.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.536 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008bfa3050fd0dc589-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.537 DEBUG:boto:b'avysmw-12:1000false:2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.538 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.538 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.539 DEBUG:boto:path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:08.539 DEBUG:boto:auth_path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:08.540 DEBUG:boto:Method: GET 2022-09-03T05:54:08.540 DEBUG:boto:Path: /avysmw-12/?prefix=%3A 2022-09-03T05:54:08.541 DEBUG:boto:Data: 2022-09-03T05:54:08.542 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.542 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.543 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.543 DEBUG:boto:Params: {} 2022-09-03T05:54:08.544 DEBUG:boto:Token: None 2022-09-03T05:54:08.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.546 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.548 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c4642e5308ad36ed-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.549 DEBUG:boto:b'avysmw-12:1000false:2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.550 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.550 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.551 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.551 DEBUG:boto:Path: /avysmw-12/%3A 2022-09-03T05:54:08.552 DEBUG:boto:Data: 2022-09-03T05:54:08.552 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.553 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.553 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.554 DEBUG:boto:Params: {} 2022-09-03T05:54:08.554 DEBUG:boto:Token: None 2022-09-03T05:54:08.555 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A 2022-09-03T05:54:08.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZbxFxDlKoWc1SiP2qtB3XMuwn2s= 2022-09-03T05:54:08.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZbxFxDlKoWc1SiP2qtB3XMuwn2s=', 'Content-Length': '0'} 2022-09-03T05:54:08.558 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002e46f613306f8d41-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.559 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.560 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.561 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.562 DEBUG:boto:Path: /avysmw-12/%3A 2022-09-03T05:54:08.563 DEBUG:boto:Data: 2022-09-03T05:54:08.564 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.565 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.566 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.567 DEBUG:boto:Params: {} 2022-09-03T05:54:08.568 DEBUG:boto:Token: None 2022-09-03T05:54:08.569 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A 2022-09-03T05:54:08.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZbxFxDlKoWc1SiP2qtB3XMuwn2s= 2022-09-03T05:54:08.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZbxFxDlKoWc1SiP2qtB3XMuwn2s=', 'Content-Length': '0'} 2022-09-03T05:54:08.574 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c2d02608d3a4b045-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.575 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:08.575 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.576 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.576 DEBUG:boto:Method: GET 2022-09-03T05:54:08.577 DEBUG:boto:Path: /avysmw-12/%3A 2022-09-03T05:54:08.578 DEBUG:boto:Data: 2022-09-03T05:54:08.578 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.579 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.579 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.580 DEBUG:boto:Params: {} 2022-09-03T05:54:08.580 DEBUG:boto:Token: None 2022-09-03T05:54:08.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A 2022-09-03T05:54:08.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kjkwMQuNP07Diochu5tu7nVA52o= 2022-09-03T05:54:08.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kjkwMQuNP07Diochu5tu7nVA52o=', 'Content-Length': '0'} 2022-09-03T05:54:08.585 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d499443ae23b549d-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.626 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:08.627 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:08.628 DEBUG:boto:Method: GET 2022-09-03T05:54:08.628 DEBUG:boto:Path: /avysmw-12/%3A 2022-09-03T05:54:08.629 DEBUG:boto:Data: 2022-09-03T05:54:08.630 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.631 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.631 DEBUG:boto:Params: {} 2022-09-03T05:54:08.632 DEBUG:boto:Token: None 2022-09-03T05:54:08.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/%3A 2022-09-03T05:54:08.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kjkwMQuNP07Diochu5tu7nVA52o= 2022-09-03T05:54:08.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kjkwMQuNP07Diochu5tu7nVA52o=', 'Content-Length': '0'} 2022-09-03T05:54:08.640 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fae370a368660d3e-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.641 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.642 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.642 DEBUG:boto:path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:08.643 DEBUG:boto:auth_path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:08.644 DEBUG:boto:Method: GET 2022-09-03T05:54:08.644 DEBUG:boto:Path: /avysmw-12/?prefix=%3A 2022-09-03T05:54:08.645 DEBUG:boto:Data: 2022-09-03T05:54:08.646 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.647 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.647 DEBUG:boto:Params: {} 2022-09-03T05:54:08.648 DEBUG:boto:Token: None 2022-09-03T05:54:08.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.651 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.654 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075c88d70ac5b76b9-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.655 DEBUG:boto:b'avysmw-12:1000false:2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.657 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.658 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.658 DEBUG:boto:path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:08.659 DEBUG:boto:auth_path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:08.659 DEBUG:boto:Method: GET 2022-09-03T05:54:08.660 DEBUG:boto:Path: /avysmw-12/?prefix=%3A 2022-09-03T05:54:08.661 DEBUG:boto:Data: 2022-09-03T05:54:08.661 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.662 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.663 DEBUG:boto:Params: {} 2022-09-03T05:54:08.664 DEBUG:boto:Token: None 2022-09-03T05:54:08.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.668 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000994ce46b291f725d-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.669 DEBUG:boto:b'avysmw-12:1000false:2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.670 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:08.671 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.672 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.674 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.675 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.675 DEBUG:boto:Method: GET 2022-09-03T05:54:08.676 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.676 DEBUG:boto:Data: 2022-09-03T05:54:08.677 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.679 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.679 DEBUG:boto:Params: {} 2022-09-03T05:54:08.680 DEBUG:boto:Token: None 2022-09-03T05:54:08.681 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.682 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0CrdbVvG/3QGxhM8VRimhbxd7Bg= 2022-09-03T05:54:08.683 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0CrdbVvG/3QGxhM8VRimhbxd7Bg=', 'Content-Length': '0'} 2022-09-03T05:54:08.686 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005823459606cf405c-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.687 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.688 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.689 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.690 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.691 DEBUG:boto:Method: GET 2022-09-03T05:54:08.691 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.692 DEBUG:boto:Data: 2022-09-03T05:54:08.692 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.693 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.694 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.695 DEBUG:boto:Params: {} 2022-09-03T05:54:08.696 DEBUG:boto:Token: None 2022-09-03T05:54:08.697 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0CrdbVvG/3QGxhM8VRimhbxd7Bg= 2022-09-03T05:54:08.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0CrdbVvG/3QGxhM8VRimhbxd7Bg=', 'Content-Length': '0'} 2022-09-03T05:54:08.701 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ca4463fef6f3c4ac-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.702 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.703 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.704 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.705 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.706 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.707 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.708 DEBUG:boto:Data: 2022-09-03T05:54:08.709 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.710 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.710 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.711 DEBUG:boto:Params: {} 2022-09-03T05:54:08.711 DEBUG:boto:Token: None 2022-09-03T05:54:08.712 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2CsLmnSZcNnEDb+isTmEj/ZEA8= 2022-09-03T05:54:08.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2CsLmnSZcNnEDb+isTmEj/ZEA8=', 'Content-Length': '0'} 2022-09-03T05:54:08.715 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007f8db65e634adde5-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.716 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.717 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.717 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.717 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.718 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.719 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.719 DEBUG:boto:Data: 2022-09-03T05:54:08.720 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.720 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.721 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.721 DEBUG:boto:Params: {} 2022-09-03T05:54:08.722 DEBUG:boto:Token: None 2022-09-03T05:54:08.722 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2CsLmnSZcNnEDb+isTmEj/ZEA8= 2022-09-03T05:54:08.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2CsLmnSZcNnEDb+isTmEj/ZEA8=', 'Content-Length': '0'} 2022-09-03T05:54:08.725 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000709ea6653d0383f4-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.726 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:08.727 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.727 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.728 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.729 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.730 DEBUG:boto:Method: GET 2022-09-03T05:54:08.730 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.731 DEBUG:boto:Data: 2022-09-03T05:54:08.732 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.734 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.735 DEBUG:boto:Params: {} 2022-09-03T05:54:08.736 DEBUG:boto:Token: None 2022-09-03T05:54:08.737 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.738 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0CrdbVvG/3QGxhM8VRimhbxd7Bg= 2022-09-03T05:54:08.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0CrdbVvG/3QGxhM8VRimhbxd7Bg=', 'Content-Length': '0'} 2022-09-03T05:54:08.741 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a706efa1d4a3a1b2-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.782 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.783 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.783 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.784 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.784 DEBUG:boto:Method: GET 2022-09-03T05:54:08.785 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.785 DEBUG:boto:Data: 2022-09-03T05:54:08.786 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.787 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.788 DEBUG:boto:Params: {} 2022-09-03T05:54:08.788 DEBUG:boto:Token: None 2022-09-03T05:54:08.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:08.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0CrdbVvG/3QGxhM8VRimhbxd7Bg= 2022-09-03T05:54:08.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0CrdbVvG/3QGxhM8VRimhbxd7Bg=', 'Content-Length': '0'} 2022-09-03T05:54:08.793 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009d0a1f2892e1697a-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.794 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.794 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.795 DEBUG:boto:path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.796 DEBUG:boto:auth_path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.797 DEBUG:boto:Method: GET 2022-09-03T05:54:08.797 DEBUG:boto:Path: /avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.798 DEBUG:boto:Data: 2022-09-03T05:54:08.798 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.799 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.799 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.800 DEBUG:boto:Params: {} 2022-09-03T05:54:08.800 DEBUG:boto:Token: None 2022-09-03T05:54:08.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.805 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002ecf8923bb2b8212-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.805 DEBUG:boto:b'avysmw-12_myobj1000false_myobj2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.806 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.807 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.807 DEBUG:boto:path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.808 DEBUG:boto:auth_path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.808 DEBUG:boto:Method: GET 2022-09-03T05:54:08.809 DEBUG:boto:Path: /avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.809 DEBUG:boto:Data: 2022-09-03T05:54:08.810 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.811 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.811 DEBUG:boto:Params: {} 2022-09-03T05:54:08.812 DEBUG:boto:Token: None 2022-09-03T05:54:08.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.816 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008ffad1c677db961b-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.817 DEBUG:boto:b'avysmw-12_myobj1000false_myobj2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.818 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.819 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.820 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.821 DEBUG:boto:Path: /avysmw-12/_myobj 2022-09-03T05:54:08.822 DEBUG:boto:Data: 2022-09-03T05:54:08.823 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.824 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.825 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.826 DEBUG:boto:Params: {} 2022-09-03T05:54:08.827 DEBUG:boto:Token: None 2022-09-03T05:54:08.828 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj 2022-09-03T05:54:08.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5cGzJ2l7NPgP+AQxarLS+ORk9Yo= 2022-09-03T05:54:08.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5cGzJ2l7NPgP+AQxarLS+ORk9Yo=', 'Content-Length': '0'} 2022-09-03T05:54:08.831 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001a0e04dd348b1ce7-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.832 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.832 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.833 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.833 DEBUG:boto:Path: /avysmw-12/_myobj 2022-09-03T05:54:08.834 DEBUG:boto:Data: 2022-09-03T05:54:08.834 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.835 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.836 DEBUG:boto:Params: {} 2022-09-03T05:54:08.836 DEBUG:boto:Token: None 2022-09-03T05:54:08.837 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj 2022-09-03T05:54:08.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5cGzJ2l7NPgP+AQxarLS+ORk9Yo= 2022-09-03T05:54:08.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5cGzJ2l7NPgP+AQxarLS+ORk9Yo=', 'Content-Length': '0'} 2022-09-03T05:54:08.840 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000389ea6bb8d2ee212-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.841 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:08.842 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.842 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.843 DEBUG:boto:Method: GET 2022-09-03T05:54:08.843 DEBUG:boto:Path: /avysmw-12/_myobj 2022-09-03T05:54:08.844 DEBUG:boto:Data: 2022-09-03T05:54:08.844 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.845 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.845 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.846 DEBUG:boto:Params: {} 2022-09-03T05:54:08.847 DEBUG:boto:Token: None 2022-09-03T05:54:08.847 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj 2022-09-03T05:54:08.848 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1YBpxxK0Zm38UnznLKyBQNAj/6s= 2022-09-03T05:54:08.848 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1YBpxxK0Zm38UnznLKyBQNAj/6s=', 'Content-Length': '0'} 2022-09-03T05:54:08.850 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009f53229618bd71f9-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.851 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:08.852 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:08.853 DEBUG:boto:Method: GET 2022-09-03T05:54:08.854 DEBUG:boto:Path: /avysmw-12/_myobj 2022-09-03T05:54:08.855 DEBUG:boto:Data: 2022-09-03T05:54:08.856 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.856 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.857 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.857 DEBUG:boto:Params: {} 2022-09-03T05:54:08.858 DEBUG:boto:Token: None 2022-09-03T05:54:08.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/_myobj 2022-09-03T05:54:08.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1YBpxxK0Zm38UnznLKyBQNAj/6s= 2022-09-03T05:54:08.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1YBpxxK0Zm38UnznLKyBQNAj/6s=', 'Content-Length': '0'} 2022-09-03T05:54:08.862 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000361035c1ec328ad4-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.863 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.863 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.864 DEBUG:boto:path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.864 DEBUG:boto:auth_path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.865 DEBUG:boto:Method: GET 2022-09-03T05:54:08.865 DEBUG:boto:Path: /avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.866 DEBUG:boto:Data: 2022-09-03T05:54:08.866 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.867 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.867 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.868 DEBUG:boto:Params: {} 2022-09-03T05:54:08.869 DEBUG:boto:Token: None 2022-09-03T05:54:08.869 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.873 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fae686a67d8df31f-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.874 DEBUG:boto:b'avysmw-12_myobj1000false_myobj2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.874 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:08.875 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:08.875 DEBUG:boto:path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.876 DEBUG:boto:auth_path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.876 DEBUG:boto:Method: GET 2022-09-03T05:54:08.877 DEBUG:boto:Path: /avysmw-12/?prefix=_myobj 2022-09-03T05:54:08.877 DEBUG:boto:Data: 2022-09-03T05:54:08.878 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.879 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.879 DEBUG:boto:Params: {} 2022-09-03T05:54:08.880 DEBUG:boto:Token: None 2022-09-03T05:54:08.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/ 2022-09-03T05:54:08.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo= 2022-09-03T05:54:08.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWA/BVQOsLvw30UJjRXjgxN3sYo=', 'Content-Length': '0'} 2022-09-03T05:54:08.884 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009cd32158df1a9b21-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.930 DEBUG:boto:b'avysmw-12_myobj1000false_myobj2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:08.931 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:08.932 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:08.932 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:08.933 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.933 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.934 DEBUG:boto:Method: GET 2022-09-03T05:54:08.934 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.935 DEBUG:boto:Data: 2022-09-03T05:54:08.935 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.936 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.937 DEBUG:boto:Params: {} 2022-09-03T05:54:08.937 DEBUG:boto:Token: None 2022-09-03T05:54:08.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KJH/GQ5PgevjgFP2CId9Q6VDt80= 2022-09-03T05:54:08.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KJH/GQ5PgevjgFP2CId9Q6VDt80=', 'Content-Length': '0'} 2022-09-03T05:54:08.942 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dbf686103be0a462-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.943 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:08.943 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:08.944 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.944 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.945 DEBUG:boto:Method: GET 2022-09-03T05:54:08.945 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.946 DEBUG:boto:Data: 2022-09-03T05:54:08.947 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.948 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.948 DEBUG:boto:Params: {} 2022-09-03T05:54:08.949 DEBUG:boto:Token: None 2022-09-03T05:54:08.950 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KJH/GQ5PgevjgFP2CId9Q6VDt80= 2022-09-03T05:54:08.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KJH/GQ5PgevjgFP2CId9Q6VDt80=', 'Content-Length': '0'} 2022-09-03T05:54:08.954 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a0b0424aa4e1fb0c-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.955 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:08.955 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:08.956 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.956 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.957 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.957 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.958 DEBUG:boto:Data: 2022-09-03T05:54:08.958 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.960 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.960 DEBUG:boto:Params: {} 2022-09-03T05:54:08.961 DEBUG:boto:Token: None 2022-09-03T05:54:08.962 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5IZHfgWXIBxco30S/ljMUOybEE0= 2022-09-03T05:54:08.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5IZHfgWXIBxco30S/ljMUOybEE0=', 'Content-Length': '0'} 2022-09-03T05:54:08.965 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005a6e273b81ec2d0d-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.965 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:08.966 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:08.966 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.967 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.967 DEBUG:boto:Method: HEAD 2022-09-03T05:54:08.968 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.968 DEBUG:boto:Data: 2022-09-03T05:54:08.969 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.969 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:08.970 DEBUG:boto:Port: 8001 2022-09-03T05:54:08.970 DEBUG:boto:Params: {} 2022-09-03T05:54:08.971 DEBUG:boto:Token: None 2022-09-03T05:54:08.971 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5IZHfgWXIBxco30S/ljMUOybEE0= 2022-09-03T05:54:08.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5IZHfgWXIBxco30S/ljMUOybEE0=', 'Content-Length': '0'} 2022-09-03T05:54:08.975 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003134fc11564e8754-006312ec00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:08.975 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:08.976 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:08.976 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:08.977 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.978 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.978 DEBUG:boto:Method: GET 2022-09-03T05:54:08.979 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.979 DEBUG:boto:Data: 2022-09-03T05:54:08.980 DEBUG:boto:Headers: {} 2022-09-03T05:54:08.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:08.981 DEBUG:boto:Port: 8000 2022-09-03T05:54:08.981 DEBUG:boto:Params: {} 2022-09-03T05:54:08.982 DEBUG:boto:Token: None 2022-09-03T05:54:08.983 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:08 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:08.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KJH/GQ5PgevjgFP2CId9Q6VDt80= 2022-09-03T05:54:08.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KJH/GQ5PgevjgFP2CId9Q6VDt80=', 'Content-Length': '0'} 2022-09-03T05:54:08.986 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ff5becc0dabd0adb-006312ec00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.030 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:09.031 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:09.031 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:09.032 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:09.033 DEBUG:boto:Method: GET 2022-09-03T05:54:09.033 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:09.034 DEBUG:boto:Data: 2022-09-03T05:54:09.035 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.036 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.037 DEBUG:boto:Params: {} 2022-09-03T05:54:09.038 DEBUG:boto:Token: None 2022-09-03T05:54:09.039 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:09.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AlGhNew7tfw2t8gJf4rdCiR7mlI= 2022-09-03T05:54:09.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AlGhNew7tfw2t8gJf4rdCiR7mlI=', 'Content-Length': '0'} 2022-09-03T05:54:09.042 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000037027511a89ec8ed-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.043 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.044 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.044 DEBUG:boto:path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:09.045 DEBUG:boto:auth_path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:09.046 DEBUG:boto:Method: GET 2022-09-03T05:54:09.047 DEBUG:boto:Path: /avysmw-12/?prefix=myobj 2022-09-03T05:54:09.047 DEBUG:boto:Data: 2022-09-03T05:54:09.048 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.049 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:09.049 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.050 DEBUG:boto:Params: {} 2022-09-03T05:54:09.051 DEBUG:boto:Token: None 2022-09-03T05:54:09.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.056 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e92c513b146f743-006312ec01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.057 DEBUG:boto:b'avysmw-12myobj1000falsemyobj2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.059 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.059 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.060 DEBUG:boto:path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:09.061 DEBUG:boto:auth_path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:09.062 DEBUG:boto:Method: GET 2022-09-03T05:54:09.062 DEBUG:boto:Path: /avysmw-12/?prefix=myobj 2022-09-03T05:54:09.063 DEBUG:boto:Data: 2022-09-03T05:54:09.064 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.065 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.066 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.066 DEBUG:boto:Params: {} 2022-09-03T05:54:09.067 DEBUG:boto:Token: None 2022-09-03T05:54:09.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.072 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c763c901b2c86a6-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.114 DEBUG:boto:b'avysmw-12myobj1000falsemyobj2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.116 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:09.117 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:09.118 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.119 DEBUG:boto:Path: /avysmw-12/myobj 2022-09-03T05:54:09.120 DEBUG:boto:Data: 2022-09-03T05:54:09.121 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:09.124 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.124 DEBUG:boto:Params: {} 2022-09-03T05:54:09.125 DEBUG:boto:Token: None 2022-09-03T05:54:09.127 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/myobj 2022-09-03T05:54:09.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ipQ6i03IRogbipVvWj2/HEURj7c= 2022-09-03T05:54:09.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ipQ6i03IRogbipVvWj2/HEURj7c=', 'Content-Length': '0'} 2022-09-03T05:54:09.130 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000d735ed8ece2c52c-006312ec01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.131 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:09.132 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:09.132 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.133 DEBUG:boto:Path: /avysmw-12/myobj 2022-09-03T05:54:09.133 DEBUG:boto:Data: 2022-09-03T05:54:09.134 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.135 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.135 DEBUG:boto:Params: {} 2022-09-03T05:54:09.136 DEBUG:boto:Token: None 2022-09-03T05:54:09.137 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/myobj 2022-09-03T05:54:09.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ipQ6i03IRogbipVvWj2/HEURj7c= 2022-09-03T05:54:09.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ipQ6i03IRogbipVvWj2/HEURj7c=', 'Content-Length': '0'} 2022-09-03T05:54:09.140 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009a4d9eaad2a3c83f-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.141 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:09.142 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:09.143 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:09.144 DEBUG:boto:Method: GET 2022-09-03T05:54:09.144 DEBUG:boto:Path: /avysmw-12/myobj 2022-09-03T05:54:09.145 DEBUG:boto:Data: 2022-09-03T05:54:09.145 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.146 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:09.146 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.147 DEBUG:boto:Params: {} 2022-09-03T05:54:09.148 DEBUG:boto:Token: None 2022-09-03T05:54:09.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/myobj 2022-09-03T05:54:09.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LeW/8EtKUO2RGnxcOQVTVjO1KxY= 2022-09-03T05:54:09.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LeW/8EtKUO2RGnxcOQVTVjO1KxY=', 'Content-Length': '0'} 2022-09-03T05:54:09.153 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ff1ff4b61979f815-006312ec01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.154 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:09.154 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:09.155 DEBUG:boto:Method: GET 2022-09-03T05:54:09.156 DEBUG:boto:Path: /avysmw-12/myobj 2022-09-03T05:54:09.157 DEBUG:boto:Data: 2022-09-03T05:54:09.158 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.159 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.160 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.161 DEBUG:boto:Params: {} 2022-09-03T05:54:09.162 DEBUG:boto:Token: None 2022-09-03T05:54:09.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/myobj 2022-09-03T05:54:09.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LeW/8EtKUO2RGnxcOQVTVjO1KxY= 2022-09-03T05:54:09.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LeW/8EtKUO2RGnxcOQVTVjO1KxY=', 'Content-Length': '0'} 2022-09-03T05:54:09.166 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dda2c3ae5b324fc5-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.167 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.167 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.168 DEBUG:boto:path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:09.169 DEBUG:boto:auth_path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:09.169 DEBUG:boto:Method: GET 2022-09-03T05:54:09.170 DEBUG:boto:Path: /avysmw-12/?prefix=myobj 2022-09-03T05:54:09.170 DEBUG:boto:Data: 2022-09-03T05:54:09.171 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:09.172 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.173 DEBUG:boto:Params: {} 2022-09-03T05:54:09.173 DEBUG:boto:Token: None 2022-09-03T05:54:09.174 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.178 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ea7f06c0c93c86e-006312ec01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.178 DEBUG:boto:b'avysmw-12myobj1000falsemyobj2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.179 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.180 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.180 DEBUG:boto:path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:09.181 DEBUG:boto:auth_path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:09.181 DEBUG:boto:Method: GET 2022-09-03T05:54:09.182 DEBUG:boto:Path: /avysmw-12/?prefix=myobj 2022-09-03T05:54:09.182 DEBUG:boto:Data: 2022-09-03T05:54:09.183 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.183 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.184 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.184 DEBUG:boto:Params: {} 2022-09-03T05:54:09.185 DEBUG:boto:Token: None 2022-09-03T05:54:09.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.186 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.190 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003a7f67a11adc587c-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.190 DEBUG:boto:b'avysmw-12myobj1000falsemyobj2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.191 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-12 zones={test-zone1, test-zone3} 2022-09-03T05:54:09.192 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-12', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:09.192 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-12', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:09.193 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-12 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:09.256 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:09.257 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:09.313 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:09.311+0000 7fd0cf9e0600 1 bucket sync caught up with source: 2022-09-03T05:54:09.313 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.33.5, , 00000000001.21.5, , , , 00000000001.17.5, 00000000001.19.5, , ] 2022-09-03T05:54:09.314 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.33.5, , 00000000001.21.5, , , , 00000000001.17.5, 00000000001.19.5, , ] 2022-09-03T05:54:09.314 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:09.311+0000 7fd0cf9e0600 0 bucket checkpoint complete 2022-09-03T05:54:09.337 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-12 zones={test-zone2, test-zone3} 2022-09-03T05:54:09.337 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.338 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.339 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.339 DEBUG:boto:Path: /avysmw-12/ 2022-09-03T05:54:09.340 DEBUG:boto:Data: 2022-09-03T05:54:09.340 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.341 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.341 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.342 DEBUG:boto:Params: {} 2022-09-03T05:54:09.343 DEBUG:boto:Token: None 2022-09-03T05:54:09.343 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iIQasAkvKiLdSf5/7RNIz2fcsF0= 2022-09-03T05:54:09.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iIQasAkvKiLdSf5/7RNIz2fcsF0=', 'Content-Length': '0'} 2022-09-03T05:54:09.348 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ac2209f02cef6d20-006312ec01-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.348 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.349 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.349 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.350 DEBUG:boto:Path: /avysmw-12/ 2022-09-03T05:54:09.350 DEBUG:boto:Data: 2022-09-03T05:54:09.351 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.352 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.353 DEBUG:boto:Params: {} 2022-09-03T05:54:09.353 DEBUG:boto:Token: None 2022-09-03T05:54:09.354 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iIQasAkvKiLdSf5/7RNIz2fcsF0= 2022-09-03T05:54:09.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iIQasAkvKiLdSf5/7RNIz2fcsF0=', 'Content-Length': '0'} 2022-09-03T05:54:09.358 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000029e0b0c8956a61b5-006312ec01-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.359 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:09.359 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.360 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.360 DEBUG:boto:path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.361 DEBUG:boto:auth_path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.361 DEBUG:boto:Method: GET 2022-09-03T05:54:09.362 DEBUG:boto:Path: /avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.363 DEBUG:boto:Data: 2022-09-03T05:54:09.363 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.364 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.365 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.366 DEBUG:boto:Params: {} 2022-09-03T05:54:09.367 DEBUG:boto:Token: None 2022-09-03T05:54:09.368 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/?versions 2022-09-03T05:54:09.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pz4IIs9j93WZgfSeYxE/L8Jx1RI= 2022-09-03T05:54:09.371 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pz4IIs9j93WZgfSeYxE/L8Jx1RI=', 'Content-Length': '0'} 2022-09-03T05:54:09.374 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000005954570ffaf2d0a-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.374 DEBUG:boto:b'avysmw-12999false&nulltrue2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.375 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:54:09.376 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:54:09.376 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:54:09.377 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:09.377 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:09.378 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.378 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.379 DEBUG:boto:path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.379 DEBUG:boto:auth_path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.380 DEBUG:boto:Method: GET 2022-09-03T05:54:09.380 DEBUG:boto:Path: /avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.381 DEBUG:boto:Data: 2022-09-03T05:54:09.381 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.382 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.382 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.383 DEBUG:boto:Params: {} 2022-09-03T05:54:09.385 DEBUG:boto:Token: None 2022-09-03T05:54:09.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/?versions 2022-09-03T05:54:09.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pz4IIs9j93WZgfSeYxE/L8Jx1RI= 2022-09-03T05:54:09.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pz4IIs9j93WZgfSeYxE/L8Jx1RI=', 'Content-Length': '0'} 2022-09-03T05:54:09.391 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000374584630cd7a99b-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.391 DEBUG:boto:b'avysmw-12999false&nulltrue2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.392 DEBUG:tasks.rgw_multi.zone_rados:o=& 2022-09-03T05:54:09.393 DEBUG:tasks.rgw_multi.zone_rados:o=: 2022-09-03T05:54:09.393 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2022-09-03T05:54:09.394 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:09.394 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.395 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.395 DEBUG:boto:path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.396 DEBUG:boto:auth_path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.396 DEBUG:boto:Method: GET 2022-09-03T05:54:09.397 DEBUG:boto:Path: /avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.397 DEBUG:boto:Data: 2022-09-03T05:54:09.398 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.399 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.399 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.400 DEBUG:boto:Params: {} 2022-09-03T05:54:09.400 DEBUG:boto:Token: None 2022-09-03T05:54:09.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/?versions 2022-09-03T05:54:09.401 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pz4IIs9j93WZgfSeYxE/L8Jx1RI= 2022-09-03T05:54:09.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pz4IIs9j93WZgfSeYxE/L8Jx1RI=', 'Content-Length': '0'} 2022-09-03T05:54:09.405 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006d02ee2882b94944-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.405 DEBUG:boto:b'avysmw-12999false&nulltrue2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.407 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.407 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.408 DEBUG:boto:path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.408 DEBUG:boto:auth_path=/avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.409 DEBUG:boto:Method: GET 2022-09-03T05:54:09.409 DEBUG:boto:Path: /avysmw-12/?versions&max-keys=999 2022-09-03T05:54:09.410 DEBUG:boto:Data: 2022-09-03T05:54:09.410 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.411 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.411 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.412 DEBUG:boto:Params: {} 2022-09-03T05:54:09.412 DEBUG:boto:Token: None 2022-09-03T05:54:09.413 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/?versions 2022-09-03T05:54:09.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pz4IIs9j93WZgfSeYxE/L8Jx1RI= 2022-09-03T05:54:09.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pz4IIs9j93WZgfSeYxE/L8Jx1RI=', 'Content-Length': '0'} 2022-09-03T05:54:09.418 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031a0261b4bcbcca6-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.419 DEBUG:boto:b'avysmw-12999false&nulltrue2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal:nulltrue2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal_myobjnulltrue2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalmyobjnulltrue2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.419 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:09.420 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.421 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.421 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.422 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.422 DEBUG:boto:Method: GET 2022-09-03T05:54:09.423 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.423 DEBUG:boto:Data: 2022-09-03T05:54:09.424 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.424 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.425 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.425 DEBUG:boto:Params: {} 2022-09-03T05:54:09.427 DEBUG:boto:Token: None 2022-09-03T05:54:09.427 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N91PtCbf4fth5U1JgzhK2W0KMSU= 2022-09-03T05:54:09.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N91PtCbf4fth5U1JgzhK2W0KMSU=', 'Content-Length': '0'} 2022-09-03T05:54:09.430 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000037f905e31bff9953-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.431 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.432 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.432 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.433 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.433 DEBUG:boto:Method: GET 2022-09-03T05:54:09.434 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.434 DEBUG:boto:Data: 2022-09-03T05:54:09.435 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.435 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.436 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.436 DEBUG:boto:Params: {} 2022-09-03T05:54:09.437 DEBUG:boto:Token: None 2022-09-03T05:54:09.438 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N91PtCbf4fth5U1JgzhK2W0KMSU= 2022-09-03T05:54:09.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N91PtCbf4fth5U1JgzhK2W0KMSU=', 'Content-Length': '0'} 2022-09-03T05:54:09.441 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001a1cbeacc1e20171-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.482 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.483 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.484 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.484 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.485 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.486 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.486 DEBUG:boto:Data: 2022-09-03T05:54:09.487 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.488 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.489 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.490 DEBUG:boto:Params: {} 2022-09-03T05:54:09.491 DEBUG:boto:Token: None 2022-09-03T05:54:09.492 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z+s9fzwQAoYbkd7iO5lBgY3guBk= 2022-09-03T05:54:09.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z+s9fzwQAoYbkd7iO5lBgY3guBk=', 'Content-Length': '0'} 2022-09-03T05:54:09.495 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f0f746133059c3cb-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.496 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.497 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.497 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.498 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.498 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.499 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.499 DEBUG:boto:Data: 2022-09-03T05:54:09.500 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.501 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.501 DEBUG:boto:Params: {} 2022-09-03T05:54:09.502 DEBUG:boto:Token: None 2022-09-03T05:54:09.503 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z+s9fzwQAoYbkd7iO5lBgY3guBk= 2022-09-03T05:54:09.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z+s9fzwQAoYbkd7iO5lBgY3guBk=', 'Content-Length': '0'} 2022-09-03T05:54:09.506 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ec8ee8db027b2516-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.507 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:09.507 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.508 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.508 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.509 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.509 DEBUG:boto:Method: GET 2022-09-03T05:54:09.510 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.510 DEBUG:boto:Data: 2022-09-03T05:54:09.511 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.511 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.512 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.513 DEBUG:boto:Params: {} 2022-09-03T05:54:09.513 DEBUG:boto:Token: None 2022-09-03T05:54:09.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N91PtCbf4fth5U1JgzhK2W0KMSU= 2022-09-03T05:54:09.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N91PtCbf4fth5U1JgzhK2W0KMSU=', 'Content-Length': '0'} 2022-09-03T05:54:09.518 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000842d7702790d0158-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.519 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.520 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.521 DEBUG:boto:path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.522 DEBUG:boto:auth_path=/avysmw-12/%26?versionId=null 2022-09-03T05:54:09.523 DEBUG:boto:Method: GET 2022-09-03T05:54:09.523 DEBUG:boto:Path: /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.524 DEBUG:boto:Data: 2022-09-03T05:54:09.524 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.525 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.526 DEBUG:boto:Params: {} 2022-09-03T05:54:09.527 DEBUG:boto:Token: None 2022-09-03T05:54:09.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26?versionId=null 2022-09-03T05:54:09.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N91PtCbf4fth5U1JgzhK2W0KMSU= 2022-09-03T05:54:09.529 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N91PtCbf4fth5U1JgzhK2W0KMSU=', 'Content-Length': '0'} 2022-09-03T05:54:09.531 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000007e3729a2b643841-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.532 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.532 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.533 DEBUG:boto:path=/avysmw-12/?prefix=%26 2022-09-03T05:54:09.533 DEBUG:boto:auth_path=/avysmw-12/?prefix=%26 2022-09-03T05:54:09.534 DEBUG:boto:Method: GET 2022-09-03T05:54:09.534 DEBUG:boto:Path: /avysmw-12/?prefix=%26 2022-09-03T05:54:09.535 DEBUG:boto:Data: 2022-09-03T05:54:09.536 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.536 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.537 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.537 DEBUG:boto:Params: {} 2022-09-03T05:54:09.538 DEBUG:boto:Token: None 2022-09-03T05:54:09.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.542 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a2b71b163bccca73-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.543 DEBUG:boto:b'avysmw-12&1000false&2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.544 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.544 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.545 DEBUG:boto:path=/avysmw-12/?prefix=%26 2022-09-03T05:54:09.545 DEBUG:boto:auth_path=/avysmw-12/?prefix=%26 2022-09-03T05:54:09.546 DEBUG:boto:Method: GET 2022-09-03T05:54:09.547 DEBUG:boto:Path: /avysmw-12/?prefix=%26 2022-09-03T05:54:09.547 DEBUG:boto:Data: 2022-09-03T05:54:09.548 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.548 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.549 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.549 DEBUG:boto:Params: {} 2022-09-03T05:54:09.550 DEBUG:boto:Token: None 2022-09-03T05:54:09.551 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.554 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000053831b073abc02ed-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.555 DEBUG:boto:b'avysmw-12&1000false&2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.556 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.557 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.557 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.558 DEBUG:boto:Path: /avysmw-12/%26 2022-09-03T05:54:09.558 DEBUG:boto:Data: 2022-09-03T05:54:09.559 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.559 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.560 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.560 DEBUG:boto:Params: {} 2022-09-03T05:54:09.561 DEBUG:boto:Token: None 2022-09-03T05:54:09.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26 2022-09-03T05:54:09.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O9afGQW4m5oFQY4RwWcZZg7yd/U= 2022-09-03T05:54:09.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O9afGQW4m5oFQY4RwWcZZg7yd/U=', 'Content-Length': '0'} 2022-09-03T05:54:09.564 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000850a9f9164bba6eb-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.565 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.566 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.566 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.567 DEBUG:boto:Path: /avysmw-12/%26 2022-09-03T05:54:09.567 DEBUG:boto:Data: 2022-09-03T05:54:09.568 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.569 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.569 DEBUG:boto:Params: {} 2022-09-03T05:54:09.570 DEBUG:boto:Token: None 2022-09-03T05:54:09.570 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26 2022-09-03T05:54:09.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O9afGQW4m5oFQY4RwWcZZg7yd/U= 2022-09-03T05:54:09.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O9afGQW4m5oFQY4RwWcZZg7yd/U=', 'Content-Length': '0'} 2022-09-03T05:54:09.574 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007b077b686451ec25-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.574 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2022-09-03T05:54:09.575 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.575 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.576 DEBUG:boto:Method: GET 2022-09-03T05:54:09.577 DEBUG:boto:Path: /avysmw-12/%26 2022-09-03T05:54:09.577 DEBUG:boto:Data: 2022-09-03T05:54:09.578 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.578 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.579 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.579 DEBUG:boto:Params: {} 2022-09-03T05:54:09.580 DEBUG:boto:Token: None 2022-09-03T05:54:09.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26 2022-09-03T05:54:09.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FH6OL7P0mvMKmFygMhOar/JMKus= 2022-09-03T05:54:09.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FH6OL7P0mvMKmFygMhOar/JMKus=', 'Content-Length': '0'} 2022-09-03T05:54:09.583 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005c73f0c28db88526-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.626 DEBUG:boto:path=/avysmw-12/%26 2022-09-03T05:54:09.627 DEBUG:boto:auth_path=/avysmw-12/%26 2022-09-03T05:54:09.627 DEBUG:boto:Method: GET 2022-09-03T05:54:09.628 DEBUG:boto:Path: /avysmw-12/%26 2022-09-03T05:54:09.629 DEBUG:boto:Data: 2022-09-03T05:54:09.629 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.630 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.631 DEBUG:boto:Params: {} 2022-09-03T05:54:09.631 DEBUG:boto:Token: None 2022-09-03T05:54:09.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%26 2022-09-03T05:54:09.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FH6OL7P0mvMKmFygMhOar/JMKus= 2022-09-03T05:54:09.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FH6OL7P0mvMKmFygMhOar/JMKus=', 'Content-Length': '0'} 2022-09-03T05:54:09.635 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bc4414a3c240b1b5-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.636 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.637 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.637 DEBUG:boto:path=/avysmw-12/?prefix=%26 2022-09-03T05:54:09.638 DEBUG:boto:auth_path=/avysmw-12/?prefix=%26 2022-09-03T05:54:09.638 DEBUG:boto:Method: GET 2022-09-03T05:54:09.639 DEBUG:boto:Path: /avysmw-12/?prefix=%26 2022-09-03T05:54:09.639 DEBUG:boto:Data: 2022-09-03T05:54:09.640 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.641 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.641 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.642 DEBUG:boto:Params: {} 2022-09-03T05:54:09.642 DEBUG:boto:Token: None 2022-09-03T05:54:09.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.647 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb93b70f4f63ad2b-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.648 DEBUG:boto:b'avysmw-12&1000false&2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.649 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.650 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.651 DEBUG:boto:path=/avysmw-12/?prefix=%26 2022-09-03T05:54:09.652 DEBUG:boto:auth_path=/avysmw-12/?prefix=%26 2022-09-03T05:54:09.653 DEBUG:boto:Method: GET 2022-09-03T05:54:09.654 DEBUG:boto:Path: /avysmw-12/?prefix=%26 2022-09-03T05:54:09.655 DEBUG:boto:Data: 2022-09-03T05:54:09.656 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.658 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.659 DEBUG:boto:Params: {} 2022-09-03T05:54:09.660 DEBUG:boto:Token: None 2022-09-03T05:54:09.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.665 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b3123e2d991a0685-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.666 DEBUG:boto:b'avysmw-12&1000false&2022-09-03T05:52:37.185Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.668 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:09.669 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.669 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.670 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.671 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.672 DEBUG:boto:Method: GET 2022-09-03T05:54:09.673 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.673 DEBUG:boto:Data: 2022-09-03T05:54:09.674 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.675 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.676 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.676 DEBUG:boto:Params: {} 2022-09-03T05:54:09.677 DEBUG:boto:Token: None 2022-09-03T05:54:09.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hjQdgTDtrghuwe66PsVl8RlpcuU= 2022-09-03T05:54:09.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hjQdgTDtrghuwe66PsVl8RlpcuU=', 'Content-Length': '0'} 2022-09-03T05:54:09.682 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000606b09b224e88e98-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.682 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.683 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.684 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.685 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.686 DEBUG:boto:Method: GET 2022-09-03T05:54:09.687 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.688 DEBUG:boto:Data: 2022-09-03T05:54:09.689 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.690 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.691 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.692 DEBUG:boto:Params: {} 2022-09-03T05:54:09.693 DEBUG:boto:Token: None 2022-09-03T05:54:09.694 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hjQdgTDtrghuwe66PsVl8RlpcuU= 2022-09-03T05:54:09.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hjQdgTDtrghuwe66PsVl8RlpcuU=', 'Content-Length': '0'} 2022-09-03T05:54:09.700 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000acd77587376ecc80-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.701 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.702 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.703 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.703 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.704 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.704 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.705 DEBUG:boto:Data: 2022-09-03T05:54:09.705 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.706 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.706 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.707 DEBUG:boto:Params: {} 2022-09-03T05:54:09.707 DEBUG:boto:Token: None 2022-09-03T05:54:09.708 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tQvSZxch/J9HLJ96Hq2mfEFiOKY= 2022-09-03T05:54:09.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tQvSZxch/J9HLJ96Hq2mfEFiOKY=', 'Content-Length': '0'} 2022-09-03T05:54:09.711 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008909b8b596569d12-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.712 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.712 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.713 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.713 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.714 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.714 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.715 DEBUG:boto:Data: 2022-09-03T05:54:09.716 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.717 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.717 DEBUG:boto:Params: {} 2022-09-03T05:54:09.718 DEBUG:boto:Token: None 2022-09-03T05:54:09.718 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tQvSZxch/J9HLJ96Hq2mfEFiOKY= 2022-09-03T05:54:09.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tQvSZxch/J9HLJ96Hq2mfEFiOKY=', 'Content-Length': '0'} 2022-09-03T05:54:09.721 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b82eb825a5bf7719-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.722 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:09.723 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.723 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.724 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.724 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.725 DEBUG:boto:Method: GET 2022-09-03T05:54:09.725 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.726 DEBUG:boto:Data: 2022-09-03T05:54:09.727 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.727 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.728 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.728 DEBUG:boto:Params: {} 2022-09-03T05:54:09.729 DEBUG:boto:Token: None 2022-09-03T05:54:09.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hjQdgTDtrghuwe66PsVl8RlpcuU= 2022-09-03T05:54:09.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hjQdgTDtrghuwe66PsVl8RlpcuU=', 'Content-Length': '0'} 2022-09-03T05:54:09.733 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bf80eceed4859bb5-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.734 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.734 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.735 DEBUG:boto:path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.735 DEBUG:boto:auth_path=/avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.736 DEBUG:boto:Method: GET 2022-09-03T05:54:09.737 DEBUG:boto:Path: /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.737 DEBUG:boto:Data: 2022-09-03T05:54:09.738 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.739 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.739 DEBUG:boto:Params: {} 2022-09-03T05:54:09.740 DEBUG:boto:Token: None 2022-09-03T05:54:09.740 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A?versionId=null 2022-09-03T05:54:09.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hjQdgTDtrghuwe66PsVl8RlpcuU= 2022-09-03T05:54:09.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hjQdgTDtrghuwe66PsVl8RlpcuU=', 'Content-Length': '0'} 2022-09-03T05:54:09.744 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001139d6cd93a39338-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.745 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.745 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.746 DEBUG:boto:path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:09.746 DEBUG:boto:auth_path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:09.747 DEBUG:boto:Method: GET 2022-09-03T05:54:09.747 DEBUG:boto:Path: /avysmw-12/?prefix=%3A 2022-09-03T05:54:09.748 DEBUG:boto:Data: 2022-09-03T05:54:09.748 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.749 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.749 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.750 DEBUG:boto:Params: {} 2022-09-03T05:54:09.750 DEBUG:boto:Token: None 2022-09-03T05:54:09.751 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.755 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003fb3775c3ff59ae0-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.756 DEBUG:boto:b'avysmw-12:1000false:2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.756 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.757 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.758 DEBUG:boto:path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:09.758 DEBUG:boto:auth_path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:09.759 DEBUG:boto:Method: GET 2022-09-03T05:54:09.759 DEBUG:boto:Path: /avysmw-12/?prefix=%3A 2022-09-03T05:54:09.760 DEBUG:boto:Data: 2022-09-03T05:54:09.760 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.761 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.762 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.762 DEBUG:boto:Params: {} 2022-09-03T05:54:09.763 DEBUG:boto:Token: None 2022-09-03T05:54:09.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.767 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d454d90ecb9cd59f-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.768 DEBUG:boto:b'avysmw-12:1000false:2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.769 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.769 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.770 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.771 DEBUG:boto:Path: /avysmw-12/%3A 2022-09-03T05:54:09.771 DEBUG:boto:Data: 2022-09-03T05:54:09.772 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.772 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.773 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.773 DEBUG:boto:Params: {} 2022-09-03T05:54:09.774 DEBUG:boto:Token: None 2022-09-03T05:54:09.774 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A 2022-09-03T05:54:09.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0IyhP/dVMWa4JSYWv6j4DHJ1l7s= 2022-09-03T05:54:09.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0IyhP/dVMWa4JSYWv6j4DHJ1l7s=', 'Content-Length': '0'} 2022-09-03T05:54:09.778 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d5ff2688ec7caa98-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.779 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.779 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.780 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.781 DEBUG:boto:Path: /avysmw-12/%3A 2022-09-03T05:54:09.781 DEBUG:boto:Data: 2022-09-03T05:54:09.782 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.782 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.783 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.783 DEBUG:boto:Params: {} 2022-09-03T05:54:09.784 DEBUG:boto:Token: None 2022-09-03T05:54:09.785 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A 2022-09-03T05:54:09.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0IyhP/dVMWa4JSYWv6j4DHJ1l7s= 2022-09-03T05:54:09.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0IyhP/dVMWa4JSYWv6j4DHJ1l7s=', 'Content-Length': '0'} 2022-09-03T05:54:09.788 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001f8346d34931159c-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.788 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2022-09-03T05:54:09.789 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.790 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.790 DEBUG:boto:Method: GET 2022-09-03T05:54:09.791 DEBUG:boto:Path: /avysmw-12/%3A 2022-09-03T05:54:09.791 DEBUG:boto:Data: 2022-09-03T05:54:09.792 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.792 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.793 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.793 DEBUG:boto:Params: {} 2022-09-03T05:54:09.794 DEBUG:boto:Token: None 2022-09-03T05:54:09.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A 2022-09-03T05:54:09.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1e44yDGGvOcuJAKiDRcwFzQ3aNg= 2022-09-03T05:54:09.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1e44yDGGvOcuJAKiDRcwFzQ3aNg=', 'Content-Length': '0'} 2022-09-03T05:54:09.798 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008a2f0929034afa22-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.842 DEBUG:boto:path=/avysmw-12/%3A 2022-09-03T05:54:09.843 DEBUG:boto:auth_path=/avysmw-12/%3A 2022-09-03T05:54:09.843 DEBUG:boto:Method: GET 2022-09-03T05:54:09.844 DEBUG:boto:Path: /avysmw-12/%3A 2022-09-03T05:54:09.845 DEBUG:boto:Data: 2022-09-03T05:54:09.845 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.847 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.847 DEBUG:boto:Params: {} 2022-09-03T05:54:09.848 DEBUG:boto:Token: None 2022-09-03T05:54:09.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/%3A 2022-09-03T05:54:09.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1e44yDGGvOcuJAKiDRcwFzQ3aNg= 2022-09-03T05:54:09.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1e44yDGGvOcuJAKiDRcwFzQ3aNg=', 'Content-Length': '0'} 2022-09-03T05:54:09.853 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000059b9e0f20dd83e95-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.854 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.854 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.855 DEBUG:boto:path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:09.855 DEBUG:boto:auth_path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:09.856 DEBUG:boto:Method: GET 2022-09-03T05:54:09.856 DEBUG:boto:Path: /avysmw-12/?prefix=%3A 2022-09-03T05:54:09.857 DEBUG:boto:Data: 2022-09-03T05:54:09.858 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.858 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.859 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.859 DEBUG:boto:Params: {} 2022-09-03T05:54:09.860 DEBUG:boto:Token: None 2022-09-03T05:54:09.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.866 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a031264974cf1d9a-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.866 DEBUG:boto:b'avysmw-12:1000false:2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.867 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.868 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.868 DEBUG:boto:path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:09.869 DEBUG:boto:auth_path=/avysmw-12/?prefix=%3A 2022-09-03T05:54:09.869 DEBUG:boto:Method: GET 2022-09-03T05:54:09.870 DEBUG:boto:Path: /avysmw-12/?prefix=%3A 2022-09-03T05:54:09.871 DEBUG:boto:Data: 2022-09-03T05:54:09.872 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.873 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.874 DEBUG:boto:Params: {} 2022-09-03T05:54:09.875 DEBUG:boto:Token: None 2022-09-03T05:54:09.876 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.877 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.878 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.882 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000347de9239ab5137-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.883 DEBUG:boto:b'avysmw-12:1000false:2022-09-03T05:52:37.107Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.885 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:09.886 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:09.887 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:09.888 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.889 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.890 DEBUG:boto:Method: GET 2022-09-03T05:54:09.890 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.891 DEBUG:boto:Data: 2022-09-03T05:54:09.892 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.892 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.893 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.894 DEBUG:boto:Params: {} 2022-09-03T05:54:09.895 DEBUG:boto:Token: None 2022-09-03T05:54:09.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4JuNTPcMg84GH41SyI7uEs8Ie7M= 2022-09-03T05:54:09.898 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4JuNTPcMg84GH41SyI7uEs8Ie7M=', 'Content-Length': '0'} 2022-09-03T05:54:09.900 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003e728f5bac7a47f2-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.902 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:09.903 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:09.904 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.905 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.906 DEBUG:boto:Method: GET 2022-09-03T05:54:09.907 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.909 DEBUG:boto:Data: 2022-09-03T05:54:09.910 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.911 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.911 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.912 DEBUG:boto:Params: {} 2022-09-03T05:54:09.913 DEBUG:boto:Token: None 2022-09-03T05:54:09.913 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4JuNTPcMg84GH41SyI7uEs8Ie7M= 2022-09-03T05:54:09.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4JuNTPcMg84GH41SyI7uEs8Ie7M=', 'Content-Length': '0'} 2022-09-03T05:54:09.919 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000017eda4ef861895ec-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.920 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:09.922 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:09.923 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.924 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.925 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.925 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.927 DEBUG:boto:Data: 2022-09-03T05:54:09.928 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.929 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.930 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.931 DEBUG:boto:Params: {} 2022-09-03T05:54:09.932 DEBUG:boto:Token: None 2022-09-03T05:54:09.933 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n+gxqr2/iQXKRsoY62jIjYYtuYM= 2022-09-03T05:54:09.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n+gxqr2/iQXKRsoY62jIjYYtuYM=', 'Content-Length': '0'} 2022-09-03T05:54:09.937 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000093a09443d84c0aa0-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.939 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:09.940 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:09.941 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.942 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.942 DEBUG:boto:Method: HEAD 2022-09-03T05:54:09.943 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.943 DEBUG:boto:Data: 2022-09-03T05:54:09.944 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.945 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.945 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.946 DEBUG:boto:Params: {} 2022-09-03T05:54:09.947 DEBUG:boto:Token: None 2022-09-03T05:54:09.947 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n+gxqr2/iQXKRsoY62jIjYYtuYM= 2022-09-03T05:54:09.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n+gxqr2/iQXKRsoY62jIjYYtuYM=', 'Content-Length': '0'} 2022-09-03T05:54:09.951 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008a70c72c3a877043-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.952 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:09.953 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:09.953 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:09.954 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.955 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.955 DEBUG:boto:Method: GET 2022-09-03T05:54:09.956 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.957 DEBUG:boto:Data: 2022-09-03T05:54:09.957 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.958 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.959 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.960 DEBUG:boto:Params: {} 2022-09-03T05:54:09.960 DEBUG:boto:Token: None 2022-09-03T05:54:09.961 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4JuNTPcMg84GH41SyI7uEs8Ie7M= 2022-09-03T05:54:09.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4JuNTPcMg84GH41SyI7uEs8Ie7M=', 'Content-Length': '0'} 2022-09-03T05:54:09.965 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003105d2602656a554-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.966 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:09.967 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:09.967 DEBUG:boto:path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.968 DEBUG:boto:auth_path=/avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.969 DEBUG:boto:Method: GET 2022-09-03T05:54:09.970 DEBUG:boto:Path: /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.970 DEBUG:boto:Data: 2022-09-03T05:54:09.971 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.972 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:09.972 DEBUG:boto:Port: 8001 2022-09-03T05:54:09.973 DEBUG:boto:Params: {} 2022-09-03T05:54:09.974 DEBUG:boto:Token: None 2022-09-03T05:54:09.975 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/_myobj?versionId=null 2022-09-03T05:54:09.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4JuNTPcMg84GH41SyI7uEs8Ie7M= 2022-09-03T05:54:09.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4JuNTPcMg84GH41SyI7uEs8Ie7M=', 'Content-Length': '0'} 2022-09-03T05:54:09.979 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000084f752367e91fb2f-006312ec01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.981 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.982 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:09.983 DEBUG:boto:path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:09.984 DEBUG:boto:auth_path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:09.985 DEBUG:boto:Method: GET 2022-09-03T05:54:09.987 DEBUG:boto:Path: /avysmw-12/?prefix=_myobj 2022-09-03T05:54:09.988 DEBUG:boto:Data: 2022-09-03T05:54:09.989 DEBUG:boto:Headers: {} 2022-09-03T05:54:09.989 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:09.990 DEBUG:boto:Port: 8000 2022-09-03T05:54:09.991 DEBUG:boto:Params: {} 2022-09-03T05:54:09.991 DEBUG:boto:Token: None 2022-09-03T05:54:09.992 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:09 GMT /avysmw-12/ 2022-09-03T05:54:09.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE= 2022-09-03T05:54:09.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oqCWyDeofnp6Os70X7GcW4mikSE=', 'Content-Length': '0'} 2022-09-03T05:54:09.996 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000032d41769394a786e-006312ec01-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:09.997 DEBUG:boto:b'avysmw-12_myobj1000false_myobj2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:09.998 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:09.999 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:10.000 DEBUG:boto:path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:10.000 DEBUG:boto:auth_path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:10.001 DEBUG:boto:Method: GET 2022-09-03T05:54:10.001 DEBUG:boto:Path: /avysmw-12/?prefix=_myobj 2022-09-03T05:54:10.002 DEBUG:boto:Data: 2022-09-03T05:54:10.003 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.004 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.005 DEBUG:boto:Params: {} 2022-09-03T05:54:10.006 DEBUG:boto:Token: None 2022-09-03T05:54:10.006 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/ 2022-09-03T05:54:10.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8= 2022-09-03T05:54:10.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8=', 'Content-Length': '0'} 2022-09-03T05:54:10.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000051a5dc1679927c59-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.011 DEBUG:boto:b'avysmw-12_myobj1000false_myobj2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:10.012 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:10.013 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:10.014 DEBUG:boto:Method: HEAD 2022-09-03T05:54:10.014 DEBUG:boto:Path: /avysmw-12/_myobj 2022-09-03T05:54:10.015 DEBUG:boto:Data: 2022-09-03T05:54:10.015 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.016 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.016 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.017 DEBUG:boto:Params: {} 2022-09-03T05:54:10.017 DEBUG:boto:Token: None 2022-09-03T05:54:10.018 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/_myobj 2022-09-03T05:54:10.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:89okFvMhhaCBpOyhujwqZJS/2oQ= 2022-09-03T05:54:10.019 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:89okFvMhhaCBpOyhujwqZJS/2oQ=', 'Content-Length': '0'} 2022-09-03T05:54:10.021 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000cd800913ebaabbc-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.022 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:10.022 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:10.023 DEBUG:boto:Method: HEAD 2022-09-03T05:54:10.023 DEBUG:boto:Path: /avysmw-12/_myobj 2022-09-03T05:54:10.024 DEBUG:boto:Data: 2022-09-03T05:54:10.025 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.025 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.026 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.026 DEBUG:boto:Params: {} 2022-09-03T05:54:10.027 DEBUG:boto:Token: None 2022-09-03T05:54:10.028 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/_myobj 2022-09-03T05:54:10.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:89okFvMhhaCBpOyhujwqZJS/2oQ= 2022-09-03T05:54:10.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:89okFvMhhaCBpOyhujwqZJS/2oQ=', 'Content-Length': '0'} 2022-09-03T05:54:10.031 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000be5a6eb74cd106d-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.032 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2022-09-03T05:54:10.032 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:10.033 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:10.033 DEBUG:boto:Method: GET 2022-09-03T05:54:10.034 DEBUG:boto:Path: /avysmw-12/_myobj 2022-09-03T05:54:10.034 DEBUG:boto:Data: 2022-09-03T05:54:10.035 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.036 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.036 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.037 DEBUG:boto:Params: {} 2022-09-03T05:54:10.037 DEBUG:boto:Token: None 2022-09-03T05:54:10.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/_myobj 2022-09-03T05:54:10.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3eRo1arQ7ZB0cflo79kLLWxoCbg= 2022-09-03T05:54:10.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3eRo1arQ7ZB0cflo79kLLWxoCbg=', 'Content-Length': '0'} 2022-09-03T05:54:10.041 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000094b9a9937f73e33b-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.042 DEBUG:boto:path=/avysmw-12/_myobj 2022-09-03T05:54:10.043 DEBUG:boto:auth_path=/avysmw-12/_myobj 2022-09-03T05:54:10.043 DEBUG:boto:Method: GET 2022-09-03T05:54:10.044 DEBUG:boto:Path: /avysmw-12/_myobj 2022-09-03T05:54:10.044 DEBUG:boto:Data: 2022-09-03T05:54:10.045 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.046 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.047 DEBUG:boto:Params: {} 2022-09-03T05:54:10.047 DEBUG:boto:Token: None 2022-09-03T05:54:10.048 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/_myobj 2022-09-03T05:54:10.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3eRo1arQ7ZB0cflo79kLLWxoCbg= 2022-09-03T05:54:10.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3eRo1arQ7ZB0cflo79kLLWxoCbg=', 'Content-Length': '0'} 2022-09-03T05:54:10.051 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c7ef681ccd7e0b4e-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.094 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:10.095 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:10.096 DEBUG:boto:path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:10.096 DEBUG:boto:auth_path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:10.097 DEBUG:boto:Method: GET 2022-09-03T05:54:10.097 DEBUG:boto:Path: /avysmw-12/?prefix=_myobj 2022-09-03T05:54:10.098 DEBUG:boto:Data: 2022-09-03T05:54:10.099 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.099 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.100 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.100 DEBUG:boto:Params: {} 2022-09-03T05:54:10.101 DEBUG:boto:Token: None 2022-09-03T05:54:10.101 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/ 2022-09-03T05:54:10.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8= 2022-09-03T05:54:10.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8=', 'Content-Length': '0'} 2022-09-03T05:54:10.105 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006369416298e3fc2f-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.106 DEBUG:boto:b'avysmw-12_myobj1000false_myobj2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:10.107 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:10.107 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:10.108 DEBUG:boto:path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:10.108 DEBUG:boto:auth_path=/avysmw-12/?prefix=_myobj 2022-09-03T05:54:10.109 DEBUG:boto:Method: GET 2022-09-03T05:54:10.110 DEBUG:boto:Path: /avysmw-12/?prefix=_myobj 2022-09-03T05:54:10.110 DEBUG:boto:Data: 2022-09-03T05:54:10.111 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.112 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.112 DEBUG:boto:Params: {} 2022-09-03T05:54:10.113 DEBUG:boto:Token: None 2022-09-03T05:54:10.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/ 2022-09-03T05:54:10.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8= 2022-09-03T05:54:10.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8=', 'Content-Length': '0'} 2022-09-03T05:54:10.117 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dba5bdcbce0bb2c2-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.118 DEBUG:boto:b'avysmw-12_myobj1000false_myobj2022-09-03T05:52:37.067Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:10.119 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:10.120 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.120 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.121 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.121 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.122 DEBUG:boto:Method: GET 2022-09-03T05:54:10.122 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.123 DEBUG:boto:Data: 2022-09-03T05:54:10.124 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.124 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.125 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.125 DEBUG:boto:Params: {} 2022-09-03T05:54:10.126 DEBUG:boto:Token: None 2022-09-03T05:54:10.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V7JFxHxZ9JCxmrOTRGov3rnMhaw= 2022-09-03T05:54:10.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V7JFxHxZ9JCxmrOTRGov3rnMhaw=', 'Content-Length': '0'} 2022-09-03T05:54:10.130 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000babcb9f90dbaf9b5-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.131 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.131 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.132 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.132 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.133 DEBUG:boto:Method: GET 2022-09-03T05:54:10.133 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.134 DEBUG:boto:Data: 2022-09-03T05:54:10.134 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.136 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.136 DEBUG:boto:Params: {} 2022-09-03T05:54:10.137 DEBUG:boto:Token: None 2022-09-03T05:54:10.137 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V7JFxHxZ9JCxmrOTRGov3rnMhaw= 2022-09-03T05:54:10.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V7JFxHxZ9JCxmrOTRGov3rnMhaw=', 'Content-Length': '0'} 2022-09-03T05:54:10.141 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ad854849dc878291-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.142 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.142 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.143 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.144 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.144 DEBUG:boto:Method: HEAD 2022-09-03T05:54:10.145 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.145 DEBUG:boto:Data: 2022-09-03T05:54:10.146 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.146 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.147 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.147 DEBUG:boto:Params: {} 2022-09-03T05:54:10.148 DEBUG:boto:Token: None 2022-09-03T05:54:10.148 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ASv6IswLfHDUO0T7Dqqbc9I8WCM= 2022-09-03T05:54:10.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ASv6IswLfHDUO0T7Dqqbc9I8WCM=', 'Content-Length': '0'} 2022-09-03T05:54:10.151 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000491baf594cbc8c4f-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.152 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.153 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.153 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.154 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.154 DEBUG:boto:Method: HEAD 2022-09-03T05:54:10.155 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.155 DEBUG:boto:Data: 2022-09-03T05:54:10.156 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.157 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.157 DEBUG:boto:Params: {} 2022-09-03T05:54:10.158 DEBUG:boto:Token: None 2022-09-03T05:54:10.158 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ASv6IswLfHDUO0T7Dqqbc9I8WCM= 2022-09-03T05:54:10.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ASv6IswLfHDUO0T7Dqqbc9I8WCM=', 'Content-Length': '0'} 2022-09-03T05:54:10.162 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006da1daac03b0f26a-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.163 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:10.163 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.164 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.164 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.165 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.165 DEBUG:boto:Method: GET 2022-09-03T05:54:10.166 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.166 DEBUG:boto:Data: 2022-09-03T05:54:10.167 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.167 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.168 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.169 DEBUG:boto:Params: {} 2022-09-03T05:54:10.169 DEBUG:boto:Token: None 2022-09-03T05:54:10.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V7JFxHxZ9JCxmrOTRGov3rnMhaw= 2022-09-03T05:54:10.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V7JFxHxZ9JCxmrOTRGov3rnMhaw=', 'Content-Length': '0'} 2022-09-03T05:54:10.173 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cf8694d45b132848-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.180 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.180 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.181 DEBUG:boto:path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.181 DEBUG:boto:auth_path=/avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.182 DEBUG:boto:Method: GET 2022-09-03T05:54:10.183 DEBUG:boto:Path: /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.183 DEBUG:boto:Data: 2022-09-03T05:54:10.184 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.185 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.185 DEBUG:boto:Params: {} 2022-09-03T05:54:10.186 DEBUG:boto:Token: None 2022-09-03T05:54:10.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj?versionId=null 2022-09-03T05:54:10.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V7JFxHxZ9JCxmrOTRGov3rnMhaw= 2022-09-03T05:54:10.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V7JFxHxZ9JCxmrOTRGov3rnMhaw=', 'Content-Length': '0'} 2022-09-03T05:54:10.190 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004ba96df6123fff50-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.191 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:10.191 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:10.192 DEBUG:boto:path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:10.192 DEBUG:boto:auth_path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:10.193 DEBUG:boto:Method: GET 2022-09-03T05:54:10.193 DEBUG:boto:Path: /avysmw-12/?prefix=myobj 2022-09-03T05:54:10.194 DEBUG:boto:Data: 2022-09-03T05:54:10.195 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.195 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.196 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.196 DEBUG:boto:Params: {} 2022-09-03T05:54:10.197 DEBUG:boto:Token: None 2022-09-03T05:54:10.197 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/ 2022-09-03T05:54:10.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8= 2022-09-03T05:54:10.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8=', 'Content-Length': '0'} 2022-09-03T05:54:10.201 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f589761cd130c960-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.202 DEBUG:boto:b'avysmw-12myobj1000falsemyobj2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:10.203 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:10.204 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:10.204 DEBUG:boto:path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:10.205 DEBUG:boto:auth_path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:10.205 DEBUG:boto:Method: GET 2022-09-03T05:54:10.206 DEBUG:boto:Path: /avysmw-12/?prefix=myobj 2022-09-03T05:54:10.206 DEBUG:boto:Data: 2022-09-03T05:54:10.207 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.207 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.208 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.208 DEBUG:boto:Params: {} 2022-09-03T05:54:10.209 DEBUG:boto:Token: None 2022-09-03T05:54:10.210 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/ 2022-09-03T05:54:10.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8= 2022-09-03T05:54:10.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8=', 'Content-Length': '0'} 2022-09-03T05:54:10.214 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091c1912e2f40e258-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.215 DEBUG:boto:b'avysmw-12myobj1000falsemyobj2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:10.216 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.216 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.217 DEBUG:boto:Method: HEAD 2022-09-03T05:54:10.217 DEBUG:boto:Path: /avysmw-12/myobj 2022-09-03T05:54:10.218 DEBUG:boto:Data: 2022-09-03T05:54:10.218 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.219 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.219 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.220 DEBUG:boto:Params: {} 2022-09-03T05:54:10.220 DEBUG:boto:Token: None 2022-09-03T05:54:10.221 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj 2022-09-03T05:54:10.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:esYf5+UoLNmPybQOYC/74Eg4vRk= 2022-09-03T05:54:10.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:esYf5+UoLNmPybQOYC/74Eg4vRk=', 'Content-Length': '0'} 2022-09-03T05:54:10.224 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000044e1f1d68811f8c1-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.225 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.226 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.226 DEBUG:boto:Method: HEAD 2022-09-03T05:54:10.227 DEBUG:boto:Path: /avysmw-12/myobj 2022-09-03T05:54:10.227 DEBUG:boto:Data: 2022-09-03T05:54:10.228 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.229 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.229 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.230 DEBUG:boto:Params: {} 2022-09-03T05:54:10.230 DEBUG:boto:Token: None 2022-09-03T05:54:10.231 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj 2022-09-03T05:54:10.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:esYf5+UoLNmPybQOYC/74Eg4vRk= 2022-09-03T05:54:10.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:esYf5+UoLNmPybQOYC/74Eg4vRk=', 'Content-Length': '0'} 2022-09-03T05:54:10.234 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b36df0aa9ff19605-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.235 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:10.235 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.236 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.236 DEBUG:boto:Method: GET 2022-09-03T05:54:10.237 DEBUG:boto:Path: /avysmw-12/myobj 2022-09-03T05:54:10.237 DEBUG:boto:Data: 2022-09-03T05:54:10.238 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.239 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.239 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.240 DEBUG:boto:Params: {} 2022-09-03T05:54:10.240 DEBUG:boto:Token: None 2022-09-03T05:54:10.241 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj 2022-09-03T05:54:10.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IdGXBw/ek+xoxJ6J0VjNmzp8+Zg= 2022-09-03T05:54:10.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IdGXBw/ek+xoxJ6J0VjNmzp8+Zg=', 'Content-Length': '0'} 2022-09-03T05:54:10.244 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003cef3c3e88fc275d-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.245 DEBUG:boto:path=/avysmw-12/myobj 2022-09-03T05:54:10.246 DEBUG:boto:auth_path=/avysmw-12/myobj 2022-09-03T05:54:10.246 DEBUG:boto:Method: GET 2022-09-03T05:54:10.247 DEBUG:boto:Path: /avysmw-12/myobj 2022-09-03T05:54:10.247 DEBUG:boto:Data: 2022-09-03T05:54:10.248 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.249 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.249 DEBUG:boto:Params: {} 2022-09-03T05:54:10.250 DEBUG:boto:Token: None 2022-09-03T05:54:10.251 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/myobj 2022-09-03T05:54:10.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IdGXBw/ek+xoxJ6J0VjNmzp8+Zg= 2022-09-03T05:54:10.252 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IdGXBw/ek+xoxJ6J0VjNmzp8+Zg=', 'Content-Length': '0'} 2022-09-03T05:54:10.254 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:52:37 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000691181025416d02-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.298 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:10.299 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:10.300 DEBUG:boto:path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:10.300 DEBUG:boto:auth_path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:10.301 DEBUG:boto:Method: GET 2022-09-03T05:54:10.302 DEBUG:boto:Path: /avysmw-12/?prefix=myobj 2022-09-03T05:54:10.302 DEBUG:boto:Data: 2022-09-03T05:54:10.303 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.303 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.304 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.304 DEBUG:boto:Params: {} 2022-09-03T05:54:10.305 DEBUG:boto:Token: None 2022-09-03T05:54:10.307 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/ 2022-09-03T05:54:10.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8= 2022-09-03T05:54:10.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8=', 'Content-Length': '0'} 2022-09-03T05:54:10.312 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000350e462824077947-006312ec02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.313 DEBUG:boto:b'avysmw-12myobj1000falsemyobj2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:10.315 DEBUG:boto:path=/avysmw-12/ 2022-09-03T05:54:10.316 DEBUG:boto:auth_path=/avysmw-12/ 2022-09-03T05:54:10.316 DEBUG:boto:path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:10.317 DEBUG:boto:auth_path=/avysmw-12/?prefix=myobj 2022-09-03T05:54:10.318 DEBUG:boto:Method: GET 2022-09-03T05:54:10.318 DEBUG:boto:Path: /avysmw-12/?prefix=myobj 2022-09-03T05:54:10.319 DEBUG:boto:Data: 2022-09-03T05:54:10.320 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.321 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.321 DEBUG:boto:Params: {} 2022-09-03T05:54:10.322 DEBUG:boto:Token: None 2022-09-03T05:54:10.323 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:10 GMT /avysmw-12/ 2022-09-03T05:54:10.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8= 2022-09-03T05:54:10.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z1kZBLiU5c2UxHpksiXzGQAf4E8=', 'Content-Length': '0'} 2022-09-03T05:54:10.328 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cc2ffaad9328a095-006312ec02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.329 DEBUG:boto:b'avysmw-12myobj1000falsemyobj2022-09-03T05:52:37.034Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:10.330 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-12 zones={test-zone2, test-zone3} 2022-09-03T05:54:10.332 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_object_sync ... ok 2022-09-03T05:54:10.333 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-13 2022-09-03T05:54:10.334 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:10.334 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:10.335 DEBUG:boto:Method: PUT 2022-09-03T05:54:10.336 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:54:10.337 DEBUG:boto:Data: 2022-09-03T05:54:10.338 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:10.340 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.341 DEBUG:boto:Params: {} 2022-09-03T05:54:10.342 DEBUG:boto:Token: None 2022-09-03T05:54:10.343 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:54:10 GMT /avysmw-13/ 2022-09-03T05:54:10.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A+4xUIGjwoSaYEh1sQ8fa2xoD+Y= 2022-09-03T05:54:10.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A+4xUIGjwoSaYEh1sQ8fa2xoD+Y=', 'Content-Length': '0'} 2022-09-03T05:54:10.377 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000004aca442b9f5b4a62-006312ec02-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.378 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-14 2022-09-03T05:54:10.379 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:10.380 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:10.380 DEBUG:boto:Method: PUT 2022-09-03T05:54:10.381 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:54:10.382 DEBUG:boto:Data: 2022-09-03T05:54:10.383 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.383 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.384 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.385 DEBUG:boto:Params: {} 2022-09-03T05:54:10.386 DEBUG:boto:Token: None 2022-09-03T05:54:10.387 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:54:10 GMT /avysmw-14/ 2022-09-03T05:54:10.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b7p9+MSrThpZkv4eeofsmvZ2Qog= 2022-09-03T05:54:10.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b7p9+MSrThpZkv4eeofsmvZ2Qog=', 'Content-Length': '0'} 2022-09-03T05:54:10.427 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000007dcacab3a25a7d4e-006312ec02-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.428 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-15 2022-09-03T05:54:10.429 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:10.430 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:10.431 DEBUG:boto:Method: PUT 2022-09-03T05:54:10.431 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:54:10.432 DEBUG:boto:Data: 2022-09-03T05:54:10.432 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.433 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.433 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.434 DEBUG:boto:Params: {} 2022-09-03T05:54:10.434 DEBUG:boto:Token: None 2022-09-03T05:54:10.435 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:54:10 GMT /avysmw-15/ 2022-09-03T05:54:10.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FQ3isuThJ91hwpv9kMDZCdaIg0M= 2022-09-03T05:54:10.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FQ3isuThJ91hwpv9kMDZCdaIg0M=', 'Content-Length': '0'} 2022-09-03T05:54:10.467 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000040f7206e3d08f307-006312ec02-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.468 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:10.468 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:10.469 DEBUG:boto:Method: HEAD 2022-09-03T05:54:10.469 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:54:10.470 DEBUG:boto:Data: 2022-09-03T05:54:10.470 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.471 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:10.471 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.472 DEBUG:boto:Params: {} 2022-09-03T05:54:10.473 DEBUG:boto:Token: None 2022-09-03T05:54:10.473 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:10 GMT /avysmw-13/ 2022-09-03T05:54:10.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VpIq3S9ufNCk/nbz9cfz/FvucbQ= 2022-09-03T05:54:10.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VpIq3S9ufNCk/nbz9cfz/FvucbQ=', 'Content-Length': '0'} 2022-09-03T05:54:10.477 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b0579c884e6fdb07-006312ec02-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.478 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:10.479 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:10.479 DEBUG:boto:Method: PUT 2022-09-03T05:54:10.480 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:10.480 DEBUG:boto:Data: 2022-09-03T05:54:10.481 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:54:10.482 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:10.482 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.483 DEBUG:boto:Params: {} 2022-09-03T05:54:10.484 DEBUG:boto:Token: None 2022-09-03T05:54:10.485 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:54:10 GMT /avysmw-13/myobj 2022-09-03T05:54:10.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tg+8uRCwVkJTvkxGcsFs7tcoeTE= 2022-09-03T05:54:10.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tg+8uRCwVkJTvkxGcsFs7tcoeTE='} 2022-09-03T05:54:10.494 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000595ea0bc419c4e41-006312ec02-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.495 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:10.496 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:10.496 DEBUG:boto:Method: HEAD 2022-09-03T05:54:10.497 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:54:10.498 DEBUG:boto:Data: 2022-09-03T05:54:10.498 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.499 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.500 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.500 DEBUG:boto:Params: {} 2022-09-03T05:54:10.501 DEBUG:boto:Token: None 2022-09-03T05:54:10.501 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:10 GMT /avysmw-14/ 2022-09-03T05:54:10.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IYrBFSYj29OTNOwFRTOy8XJRIAs= 2022-09-03T05:54:10.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IYrBFSYj29OTNOwFRTOy8XJRIAs=', 'Content-Length': '0'} 2022-09-03T05:54:10.505 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000a8c5e30e0c699a2-006312ec02-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.506 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:10.507 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:10.508 DEBUG:boto:Method: PUT 2022-09-03T05:54:10.508 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:54:10.509 DEBUG:boto:Data: 2022-09-03T05:54:10.510 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:54:10.510 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:10.511 DEBUG:boto:Port: 8000 2022-09-03T05:54:10.511 DEBUG:boto:Params: {} 2022-09-03T05:54:10.512 DEBUG:boto:Token: None 2022-09-03T05:54:10.513 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:54:10 GMT /avysmw-14/myobj 2022-09-03T05:54:10.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uWqQ8HkGT38nMFD/7Ok4D/yPZPI= 2022-09-03T05:54:10.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uWqQ8HkGT38nMFD/7Ok4D/yPZPI='} 2022-09-03T05:54:10.526 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000feab2dbf35e80f37-006312ec02-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.527 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:10.527 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:10.528 DEBUG:boto:Method: HEAD 2022-09-03T05:54:10.529 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:54:10.530 DEBUG:boto:Data: 2022-09-03T05:54:10.530 DEBUG:boto:Headers: {} 2022-09-03T05:54:10.531 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.531 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.532 DEBUG:boto:Params: {} 2022-09-03T05:54:10.532 DEBUG:boto:Token: None 2022-09-03T05:54:10.533 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:10 GMT /avysmw-15/ 2022-09-03T05:54:10.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u0UYYBGLToAjJGTHDWVT9UR2uqE= 2022-09-03T05:54:10.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u0UYYBGLToAjJGTHDWVT9UR2uqE=', 'Content-Length': '0'} 2022-09-03T05:54:10.540 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000677250e4e672438c-006312ec02-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.541 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:10.542 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:10.542 DEBUG:boto:Method: PUT 2022-09-03T05:54:10.543 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:54:10.543 DEBUG:boto:Data: 2022-09-03T05:54:10.544 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T05:54:10.544 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:10.545 DEBUG:boto:Port: 8001 2022-09-03T05:54:10.545 DEBUG:boto:Params: {} 2022-09-03T05:54:10.546 DEBUG:boto:Token: None 2022-09-03T05:54:10.546 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 05:54:10 GMT /avysmw-15/myobj 2022-09-03T05:54:10.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LEqNtOLHBn3AN8V5jmnmnM0J4Lg= 2022-09-03T05:54:10.547 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:54:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LEqNtOLHBn3AN8V5jmnmnM0J4Lg='} 2022-09-03T05:54:10.561 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000ef1a0791ef69ad98-006312ec02-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:10.562 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:10.563 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:10.564 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:54:10.626 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:10.627 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:10.685 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:54:10.712 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:54:10.713 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:54:10.713 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:54:10.713 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T05:54:10.714 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:54:10.714 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:54:10.714 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184450.454511_110.1", 2022-09-03T05:54:10.714 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:54:10.454511Z" 2022-09-03T05:54:10.715 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:54:10.715 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:54:10.715 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:54:10.716 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:54:10.716 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:54:10.716 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:54:10.716 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:54:10.717 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:54:10.717 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:54:10.717 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:54:10.718 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662184450.454511_110.1', 2: '', 3: ''} 2022-09-03T05:54:10.719 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:54:10.719 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:10.720 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:10.721 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:54:10.770 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:10.770 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:10.812 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:10.810+0000 7fa8f61a1600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:54:10.813 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:10.811+0000 7fa8f61a1600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:54:10.826 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:54:10.826 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:54:10.827 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:54:10.827 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:54:10.828 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:54:10.828 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:54:10.829 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:10.829 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:10.830 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:54:10.830 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:10.831 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:54:10.831 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:10.832 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:10.832 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:54:10.833 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:10.833 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:54:10.834 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:10.834 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:54:10.835 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:10.835 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:10.836 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:10.836 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:10.836 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:54:10.836 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:10.837 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:10.837 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184450.454511_110.1", 2022-09-03T05:54:10.838 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:10.838 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:54:10.838 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:10.839 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.454511Z", 2022-09-03T05:54:10.839 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:10.839 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:10.839 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:10.840 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:10.840 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:54:10.840 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:10.840 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:10.841 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:54:10.841 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:10.841 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:54:10.842 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:10.842 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:54:10.842 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:10.843 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:10.843 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:10.843 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:10.844 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:54:10.844 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:10.844 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:10.845 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:54:10.845 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:10.845 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:54:10.846 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:10.846 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:54:10.846 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:10.847 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:10.847 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:10.847 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:54:10.848 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:10.848 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:54:10.848 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:54:10.848 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:54:10.849 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:10.849 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:54:10Z" 2022-09-03T05:54:10.849 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:54:10.851 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184450.454511_110.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:54:10.454511Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:54:10Z" } 2022-09-03T05:54:10.852 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184450.454511_110.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:54:10.454511Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:54:10.852 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184450.454511_110.1', 2: '', 3: ''} 2022-09-03T05:54:10.853 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184450.454511_110.1', 2: '', 3: ''} 2022-09-03T05:54:10.853 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:54:10.854 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:54:10.854 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:10.855 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:10.856 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:54:10.907 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:10.908 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:10.934 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:10.931+0000 7fb792b54600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:54:10.935 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:10.933+0000 7fb792b54600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:54:10.946 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:54:10.947 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:54:10.947 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:54:10.948 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:54:10.948 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:54:10.948 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:54:10.949 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:54:10.949 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:54:10.950 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:54:10.950 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:54:10.951 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:54:10.951 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:54:10.951 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:54:10.951 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:54:10.952 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:54:10.952 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:54:10.952 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:54:10.952 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:54:10.953 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:54:10.953 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:54:10.953 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:54:10.953 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:54:10.953 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:54:10.954 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:54:10.954 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:54:10.955 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184450.454511_110.1", 2022-09-03T05:54:10.955 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:54:10.956 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:54:10.956 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:54:10.957 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.454511Z", 2022-09-03T05:54:10.957 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:54:10.958 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:54:10.958 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:54:10.959 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:54:10.959 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:54:10.959 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:54:10.960 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:54:10.960 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:54:10.961 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:54:10.961 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:54:10.961 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:54:10.962 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:54:10.962 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:54:10.962 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:54:10.963 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:54:10.963 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:54:10.963 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:54:10.963 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:54:10.963 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:54:10.964 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:54:10.964 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:54:10.964 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:54:10.965 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:54:10.965 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:54:10.965 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:54:10.965 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:54:10.966 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:54:10.966 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:54:10.966 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:54:10.966 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:54:10.967 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:54:10.967 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:54:10.967 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:54:10.967 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:54:10Z" 2022-09-03T05:54:10.968 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:54:10.969 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184450.454511_110.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:54:10.454511Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:54:10Z" } 2022-09-03T05:54:10.970 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184450.454511_110.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:54:10.454511Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:54:10.970 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184450.454511_110.1', 2: '', 3: ''} 2022-09-03T05:54:10.971 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184450.454511_110.1', 2: '', 3: ''} 2022-09-03T05:54:10.971 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:54:10.972 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:54:10.972 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:10.973 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:10.973 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:54:11.022 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:11.023 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:11.049 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:11.047+0000 7fe6530b7600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:54:11.050 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:11.048+0000 7fe6530b7600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:54:11.061 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:54:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:54:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:54:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:54:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:54:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:54:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:54:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:54:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:54:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:54:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:54:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:54:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184450.450607_108.1", 2022-09-03T05:54:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:54:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.450607Z", 2022-09-03T05:54:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:54:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:54:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:54:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:54:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:54:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:54:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:54:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:54:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:54:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:54:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:54:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:54:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:54:11Z" 2022-09-03T05:54:11.079 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:54:11.080 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184450.450607_108.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:54:10.450607Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:54:11Z" } 2022-09-03T05:54:11.081 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184450.450607_108.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:54:10.450607Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:54:11.081 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184450.454511_110.1', 2: '', 3: ''} 2022-09-03T05:54:11.082 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184450.450607_108.1', 2: '', 3: ''} 2022-09-03T05:54:11.082 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662184450.454511_110.1 target=1_1662184450.450607_108.1 2022-09-03T05:54:16.083 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:16.085 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:54:16.086 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:54:16.144 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:16.145 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:16.184 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:16.182+0000 7fe23fdc4600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:54:16.185 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:16.183+0000 7fe23fdc4600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:54:16.200 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:54:16.201 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:54:16.201 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:54:16.201 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:54:16.202 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:54:16.202 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:54:16.202 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:16.203 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:16.203 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:54:16.203 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:16.203 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:54:16.204 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:16.204 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:16.204 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:54:16.205 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:16.205 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:54:16.205 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:16.206 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:54:16.206 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:16.208 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:16.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:16.209 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:16.209 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:54:16.210 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:16.210 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:16.211 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184450.454511_110.1", 2022-09-03T05:54:16.211 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:16.212 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:54:16.212 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:16.212 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.454511Z", 2022-09-03T05:54:16.213 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:16.213 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:16.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:16.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:16.214 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:54:16.214 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:16.215 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:16.215 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:54:16.215 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:16.215 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:54:16.216 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:16.216 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:54:16.216 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:16.216 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:16.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:16.217 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:54:16.217 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:54:16.217 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:54:16.218 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:54:16.218 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:54:16.218 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:54:16.218 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:54:16.219 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:54:16.219 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:54:16.219 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:54:16.219 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:16.220 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:54:16.220 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:54:16.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:16.220 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:54:16.221 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:54:16.221 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:54:16.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:54:16.221 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:54:16Z" 2022-09-03T05:54:16.222 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:54:16.233 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184450.454511_110.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:54:10.454511Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:54:16Z" } 2022-09-03T05:54:16.234 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184450.454511_110.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:54:10.454511Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:54:16.235 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184450.454511_110.1', 2: '', 3: ''} 2022-09-03T05:54:16.236 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184450.454511_110.1', 2: '', 3: ''} 2022-09-03T05:54:16.236 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:54:16.237 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:16.238 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:16.239 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:16.300 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:16.301 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:16.347 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:16.345+0000 7fdc7df43600 1 waiting to reach incremental sync.. 2022-09-03T05:54:21.348 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:21.346+0000 7fdc7df43600 1 waiting to reach incremental sync.. 2022-09-03T05:54:26.349 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:26.346+0000 7fdc7df43600 1 waiting to reach incremental sync.. 2022-09-03T05:54:31.351 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:31.350+0000 7fdc7df43600 1 bucket sync caught up with source: 2022-09-03T05:54:31.352 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:31.352 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:31.353 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:31.350+0000 7fdc7df43600 0 bucket checkpoint complete 2022-09-03T05:54:31.376 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-13 zones={test-zone2, test-zone1} 2022-09-03T05:54:31.377 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.377 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.378 DEBUG:boto:Method: HEAD 2022-09-03T05:54:31.378 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:54:31.379 DEBUG:boto:Data: 2022-09-03T05:54:31.379 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.380 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.380 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.381 DEBUG:boto:Params: {} 2022-09-03T05:54:31.382 DEBUG:boto:Token: None 2022-09-03T05:54:31.383 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/ 2022-09-03T05:54:31.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VcUWYzCAoGAQjbI1qj+Df5KBmys= 2022-09-03T05:54:31.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VcUWYzCAoGAQjbI1qj+Df5KBmys=', 'Content-Length': '0'} 2022-09-03T05:54:31.386 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000333d8b764e28ab9c-006312ec17-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.387 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.388 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.388 DEBUG:boto:Method: HEAD 2022-09-03T05:54:31.389 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:54:31.389 DEBUG:boto:Data: 2022-09-03T05:54:31.390 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.391 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.391 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.392 DEBUG:boto:Params: {} 2022-09-03T05:54:31.392 DEBUG:boto:Token: None 2022-09-03T05:54:31.393 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/ 2022-09-03T05:54:31.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VcUWYzCAoGAQjbI1qj+Df5KBmys= 2022-09-03T05:54:31.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VcUWYzCAoGAQjbI1qj+Df5KBmys=', 'Content-Length': '0'} 2022-09-03T05:54:31.397 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d10ef64e16608379-006312ec17-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.397 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:31.398 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.398 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.399 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.399 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.400 DEBUG:boto:Method: GET 2022-09-03T05:54:31.401 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.401 DEBUG:boto:Data: 2022-09-03T05:54:31.402 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.402 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.403 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.403 DEBUG:boto:Params: {} 2022-09-03T05:54:31.404 DEBUG:boto:Token: None 2022-09-03T05:54:31.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/?versions 2022-09-03T05:54:31.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw= 2022-09-03T05:54:31.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw=', 'Content-Length': '0'} 2022-09-03T05:54:31.408 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1bd7e2a585a01e2-006312ec17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.450 DEBUG:boto:b'avysmw-13999falsemyobjnulltrue2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.452 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:31.454 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:31.455 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.456 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.457 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.458 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.459 DEBUG:boto:Method: GET 2022-09-03T05:54:31.460 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.461 DEBUG:boto:Data: 2022-09-03T05:54:31.462 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.464 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.465 DEBUG:boto:Params: {} 2022-09-03T05:54:31.466 DEBUG:boto:Token: None 2022-09-03T05:54:31.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/?versions 2022-09-03T05:54:31.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw= 2022-09-03T05:54:31.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw=', 'Content-Length': '0'} 2022-09-03T05:54:31.473 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006bde6d8d005c88a5-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.474 DEBUG:boto:b'avysmw-13999falsemyobjnulltrue2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.476 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:31.477 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.478 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.479 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.480 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.481 DEBUG:boto:Method: GET 2022-09-03T05:54:31.482 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.483 DEBUG:boto:Data: 2022-09-03T05:54:31.484 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.485 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.486 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.487 DEBUG:boto:Params: {} 2022-09-03T05:54:31.488 DEBUG:boto:Token: None 2022-09-03T05:54:31.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/?versions 2022-09-03T05:54:31.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw= 2022-09-03T05:54:31.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw=', 'Content-Length': '0'} 2022-09-03T05:54:31.494 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000425003759ce76b14-006312ec17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.496 DEBUG:boto:b'avysmw-13999falsemyobjnulltrue2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.497 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.498 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.499 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.500 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.501 DEBUG:boto:Method: GET 2022-09-03T05:54:31.502 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.503 DEBUG:boto:Data: 2022-09-03T05:54:31.504 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.506 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.507 DEBUG:boto:Params: {} 2022-09-03T05:54:31.508 DEBUG:boto:Token: None 2022-09-03T05:54:31.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/?versions 2022-09-03T05:54:31.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw= 2022-09-03T05:54:31.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw=', 'Content-Length': '0'} 2022-09-03T05:54:31.514 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bea3093288d8927a-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.516 DEBUG:boto:b'avysmw-13999falsemyobjnulltrue2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.517 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:31.518 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.519 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.521 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.522 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.523 DEBUG:boto:Method: GET 2022-09-03T05:54:31.524 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.525 DEBUG:boto:Data: 2022-09-03T05:54:31.526 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.527 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.528 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.529 DEBUG:boto:Params: {} 2022-09-03T05:54:31.531 DEBUG:boto:Token: None 2022-09-03T05:54:31.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1dqjBUAzJX8hT1oEqofLRZNONe0= 2022-09-03T05:54:31.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1dqjBUAzJX8hT1oEqofLRZNONe0=', 'Content-Length': '0'} 2022-09-03T05:54:31.538 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008f8391c1cfa0c6d4-006312ec17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.540 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.541 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.542 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.543 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.544 DEBUG:boto:Method: GET 2022-09-03T05:54:31.545 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.545 DEBUG:boto:Data: 2022-09-03T05:54:31.547 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.549 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.549 DEBUG:boto:Params: {} 2022-09-03T05:54:31.551 DEBUG:boto:Token: None 2022-09-03T05:54:31.552 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.553 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1dqjBUAzJX8hT1oEqofLRZNONe0= 2022-09-03T05:54:31.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1dqjBUAzJX8hT1oEqofLRZNONe0=', 'Content-Length': '0'} 2022-09-03T05:54:31.557 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000314c5111cd8c5b11-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.558 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.559 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.560 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.561 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.562 DEBUG:boto:Method: HEAD 2022-09-03T05:54:31.562 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.563 DEBUG:boto:Data: 2022-09-03T05:54:31.564 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.564 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.565 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.566 DEBUG:boto:Params: {} 2022-09-03T05:54:31.566 DEBUG:boto:Token: None 2022-09-03T05:54:31.567 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ONOE3WuI7VLmm/jPVRCRTowrhYk= 2022-09-03T05:54:31.568 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ONOE3WuI7VLmm/jPVRCRTowrhYk=', 'Content-Length': '0'} 2022-09-03T05:54:31.571 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001de68ef01fcac357-006312ec17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.572 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.572 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.573 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.573 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.574 DEBUG:boto:Method: HEAD 2022-09-03T05:54:31.574 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.575 DEBUG:boto:Data: 2022-09-03T05:54:31.575 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.576 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.577 DEBUG:boto:Params: {} 2022-09-03T05:54:31.577 DEBUG:boto:Token: None 2022-09-03T05:54:31.578 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ONOE3WuI7VLmm/jPVRCRTowrhYk= 2022-09-03T05:54:31.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ONOE3WuI7VLmm/jPVRCRTowrhYk=', 'Content-Length': '0'} 2022-09-03T05:54:31.581 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e4bf77c819a7b495-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.582 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:31.582 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.583 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.583 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.584 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.584 DEBUG:boto:Method: GET 2022-09-03T05:54:31.585 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.585 DEBUG:boto:Data: 2022-09-03T05:54:31.586 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.586 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.587 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.587 DEBUG:boto:Params: {} 2022-09-03T05:54:31.588 DEBUG:boto:Token: None 2022-09-03T05:54:31.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1dqjBUAzJX8hT1oEqofLRZNONe0= 2022-09-03T05:54:31.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1dqjBUAzJX8hT1oEqofLRZNONe0=', 'Content-Length': '0'} 2022-09-03T05:54:31.592 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000001f45b451b56f499-006312ec17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.593 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.593 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.594 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.594 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.595 DEBUG:boto:Method: GET 2022-09-03T05:54:31.595 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.596 DEBUG:boto:Data: 2022-09-03T05:54:31.596 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.597 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.598 DEBUG:boto:Params: {} 2022-09-03T05:54:31.599 DEBUG:boto:Token: None 2022-09-03T05:54:31.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:31.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1dqjBUAzJX8hT1oEqofLRZNONe0= 2022-09-03T05:54:31.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1dqjBUAzJX8hT1oEqofLRZNONe0=', 'Content-Length': '0'} 2022-09-03T05:54:31.602 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004b2d2904570c7c82-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.603 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.604 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.604 DEBUG:boto:path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:31.605 DEBUG:boto:auth_path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:31.605 DEBUG:boto:Method: GET 2022-09-03T05:54:31.606 DEBUG:boto:Path: /avysmw-13/?prefix=myobj 2022-09-03T05:54:31.606 DEBUG:boto:Data: 2022-09-03T05:54:31.607 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.607 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.608 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.608 DEBUG:boto:Params: {} 2022-09-03T05:54:31.609 DEBUG:boto:Token: None 2022-09-03T05:54:31.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/ 2022-09-03T05:54:31.610 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5UC8JxYnG3mVpSAM7XImZS7AYI= 2022-09-03T05:54:31.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5UC8JxYnG3mVpSAM7XImZS7AYI=', 'Content-Length': '0'} 2022-09-03T05:54:31.613 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007917c87c2c796df1-006312ec17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.614 DEBUG:boto:b'avysmw-13myobj1000falsemyobj2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.615 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.616 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.616 DEBUG:boto:path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:31.617 DEBUG:boto:auth_path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:31.617 DEBUG:boto:Method: GET 2022-09-03T05:54:31.618 DEBUG:boto:Path: /avysmw-13/?prefix=myobj 2022-09-03T05:54:31.618 DEBUG:boto:Data: 2022-09-03T05:54:31.619 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.619 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.620 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.620 DEBUG:boto:Params: {} 2022-09-03T05:54:31.621 DEBUG:boto:Token: None 2022-09-03T05:54:31.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/ 2022-09-03T05:54:31.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5UC8JxYnG3mVpSAM7XImZS7AYI= 2022-09-03T05:54:31.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5UC8JxYnG3mVpSAM7XImZS7AYI=', 'Content-Length': '0'} 2022-09-03T05:54:31.625 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b12e7b9800b144de-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.670 DEBUG:boto:b'avysmw-13myobj1000falsemyobj2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.672 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.673 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.674 DEBUG:boto:Method: HEAD 2022-09-03T05:54:31.675 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:31.676 DEBUG:boto:Data: 2022-09-03T05:54:31.677 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.678 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.678 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.679 DEBUG:boto:Params: {} 2022-09-03T05:54:31.680 DEBUG:boto:Token: None 2022-09-03T05:54:31.681 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj 2022-09-03T05:54:31.682 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wyqM4BDGQRVMIOyLIgf1hMILPzc= 2022-09-03T05:54:31.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wyqM4BDGQRVMIOyLIgf1hMILPzc=', 'Content-Length': '0'} 2022-09-03T05:54:31.684 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007bfbfd43c972c913-006312ec17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.685 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.685 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.686 DEBUG:boto:Method: HEAD 2022-09-03T05:54:31.687 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:31.687 DEBUG:boto:Data: 2022-09-03T05:54:31.688 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.689 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.689 DEBUG:boto:Params: {} 2022-09-03T05:54:31.690 DEBUG:boto:Token: None 2022-09-03T05:54:31.690 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj 2022-09-03T05:54:31.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wyqM4BDGQRVMIOyLIgf1hMILPzc= 2022-09-03T05:54:31.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wyqM4BDGQRVMIOyLIgf1hMILPzc=', 'Content-Length': '0'} 2022-09-03T05:54:31.694 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000049937c8d3d5f79d2-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.695 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:31.695 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.696 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.696 DEBUG:boto:Method: GET 2022-09-03T05:54:31.697 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:31.697 DEBUG:boto:Data: 2022-09-03T05:54:31.698 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.699 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.699 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.700 DEBUG:boto:Params: {} 2022-09-03T05:54:31.700 DEBUG:boto:Token: None 2022-09-03T05:54:31.701 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj 2022-09-03T05:54:31.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zYpzZJ6UpU8scmsPyyMfWcCKPYM= 2022-09-03T05:54:31.702 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zYpzZJ6UpU8scmsPyyMfWcCKPYM=', 'Content-Length': '0'} 2022-09-03T05:54:31.704 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000026241b76007dfc87-006312ec17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.705 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.706 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.706 DEBUG:boto:Method: GET 2022-09-03T05:54:31.707 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:31.708 DEBUG:boto:Data: 2022-09-03T05:54:31.708 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.709 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.709 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.710 DEBUG:boto:Params: {} 2022-09-03T05:54:31.710 DEBUG:boto:Token: None 2022-09-03T05:54:31.711 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/myobj 2022-09-03T05:54:31.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zYpzZJ6UpU8scmsPyyMfWcCKPYM= 2022-09-03T05:54:31.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zYpzZJ6UpU8scmsPyyMfWcCKPYM=', 'Content-Length': '0'} 2022-09-03T05:54:31.714 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c52082e69e6524d1-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.715 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.716 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.716 DEBUG:boto:path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:31.717 DEBUG:boto:auth_path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:31.717 DEBUG:boto:Method: GET 2022-09-03T05:54:31.718 DEBUG:boto:Path: /avysmw-13/?prefix=myobj 2022-09-03T05:54:31.719 DEBUG:boto:Data: 2022-09-03T05:54:31.719 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.720 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:31.720 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.721 DEBUG:boto:Params: {} 2022-09-03T05:54:31.721 DEBUG:boto:Token: None 2022-09-03T05:54:31.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/ 2022-09-03T05:54:31.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5UC8JxYnG3mVpSAM7XImZS7AYI= 2022-09-03T05:54:31.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5UC8JxYnG3mVpSAM7XImZS7AYI=', 'Content-Length': '0'} 2022-09-03T05:54:31.726 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000896ad55e64136150-006312ec17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.727 DEBUG:boto:b'avysmw-13myobj1000falsemyobj2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.728 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.728 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.729 DEBUG:boto:path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:31.729 DEBUG:boto:auth_path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:31.730 DEBUG:boto:Method: GET 2022-09-03T05:54:31.730 DEBUG:boto:Path: /avysmw-13/?prefix=myobj 2022-09-03T05:54:31.731 DEBUG:boto:Data: 2022-09-03T05:54:31.731 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.732 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.733 DEBUG:boto:Params: {} 2022-09-03T05:54:31.734 DEBUG:boto:Token: None 2022-09-03T05:54:31.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/ 2022-09-03T05:54:31.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5UC8JxYnG3mVpSAM7XImZS7AYI= 2022-09-03T05:54:31.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5UC8JxYnG3mVpSAM7XImZS7AYI=', 'Content-Length': '0'} 2022-09-03T05:54:31.740 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005b67b67afe5d14e8-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.782 DEBUG:boto:b'avysmw-13myobj1000falsemyobj2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.784 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-13 zones={test-zone2, test-zone1} 2022-09-03T05:54:31.785 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:31.786 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:31.787 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:31.854 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:31.854 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:31.911 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:31.909+0000 7f82be074600 1 bucket sync caught up with source: 2022-09-03T05:54:31.911 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:31.911 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:31.912 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:31.909+0000 7f82be074600 0 bucket checkpoint complete 2022-09-03T05:54:31.924 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-13 zones={test-zone3, test-zone1} 2022-09-03T05:54:31.924 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.925 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.925 DEBUG:boto:Method: HEAD 2022-09-03T05:54:31.926 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:54:31.926 DEBUG:boto:Data: 2022-09-03T05:54:31.927 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:31.928 DEBUG:boto:Port: 8001 2022-09-03T05:54:31.928 DEBUG:boto:Params: {} 2022-09-03T05:54:31.929 DEBUG:boto:Token: None 2022-09-03T05:54:31.930 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/ 2022-09-03T05:54:31.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VcUWYzCAoGAQjbI1qj+Df5KBmys= 2022-09-03T05:54:31.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VcUWYzCAoGAQjbI1qj+Df5KBmys=', 'Content-Length': '0'} 2022-09-03T05:54:31.934 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ebe5ad92b839e7f3-006312ec17-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.934 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.935 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.935 DEBUG:boto:Method: HEAD 2022-09-03T05:54:31.936 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:54:31.936 DEBUG:boto:Data: 2022-09-03T05:54:31.937 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.937 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.938 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.938 DEBUG:boto:Params: {} 2022-09-03T05:54:31.939 DEBUG:boto:Token: None 2022-09-03T05:54:31.939 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/ 2022-09-03T05:54:31.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VcUWYzCAoGAQjbI1qj+Df5KBmys= 2022-09-03T05:54:31.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VcUWYzCAoGAQjbI1qj+Df5KBmys=', 'Content-Length': '0'} 2022-09-03T05:54:31.943 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000675a60611f73314b-006312ec17-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.944 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:31.945 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.946 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.946 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.947 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.948 DEBUG:boto:Method: GET 2022-09-03T05:54:31.948 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.949 DEBUG:boto:Data: 2022-09-03T05:54:31.949 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.950 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:31.950 DEBUG:boto:Port: 8001 2022-09-03T05:54:31.951 DEBUG:boto:Params: {} 2022-09-03T05:54:31.951 DEBUG:boto:Token: None 2022-09-03T05:54:31.952 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/?versions 2022-09-03T05:54:31.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw= 2022-09-03T05:54:31.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw=', 'Content-Length': '0'} 2022-09-03T05:54:31.956 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007520535e66033533-006312ec17-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.956 DEBUG:boto:b'avysmw-13999falsemyobjnulltrue2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.957 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:31.958 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:31.959 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.959 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.960 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.960 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.961 DEBUG:boto:Method: GET 2022-09-03T05:54:31.961 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.962 DEBUG:boto:Data: 2022-09-03T05:54:31.962 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.963 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.964 DEBUG:boto:Params: {} 2022-09-03T05:54:31.964 DEBUG:boto:Token: None 2022-09-03T05:54:31.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/?versions 2022-09-03T05:54:31.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw= 2022-09-03T05:54:31.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw=', 'Content-Length': '0'} 2022-09-03T05:54:31.969 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002eb9aa8503ee0cab-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.969 DEBUG:boto:b'avysmw-13999falsemyobjnulltrue2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.970 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:31.971 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.972 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.973 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.974 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.975 DEBUG:boto:Method: GET 2022-09-03T05:54:31.976 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.977 DEBUG:boto:Data: 2022-09-03T05:54:31.977 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:31.979 DEBUG:boto:Port: 8001 2022-09-03T05:54:31.980 DEBUG:boto:Params: {} 2022-09-03T05:54:31.980 DEBUG:boto:Token: None 2022-09-03T05:54:31.981 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/?versions 2022-09-03T05:54:31.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw= 2022-09-03T05:54:31.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw=', 'Content-Length': '0'} 2022-09-03T05:54:31.984 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000be1a626473fcaebc-006312ec17-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.985 DEBUG:boto:b'avysmw-13999falsemyobjnulltrue2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.986 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:31.987 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:31.987 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.988 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.988 DEBUG:boto:Method: GET 2022-09-03T05:54:31.989 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:54:31.989 DEBUG:boto:Data: 2022-09-03T05:54:31.990 DEBUG:boto:Headers: {} 2022-09-03T05:54:31.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:31.991 DEBUG:boto:Port: 8000 2022-09-03T05:54:31.991 DEBUG:boto:Params: {} 2022-09-03T05:54:31.992 DEBUG:boto:Token: None 2022-09-03T05:54:31.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:31 GMT /avysmw-13/?versions 2022-09-03T05:54:31.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw= 2022-09-03T05:54:31.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:essN2Y4wp3wq22mv0wkC7NoBgdw=', 'Content-Length': '0'} 2022-09-03T05:54:31.996 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000080952ad6000c334b-006312ec17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:31.997 DEBUG:boto:b'avysmw-13999falsemyobjnulltrue2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:31.998 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:31.998 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:31.999 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:31.999 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.000 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.000 DEBUG:boto:Method: GET 2022-09-03T05:54:32.001 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.001 DEBUG:boto:Data: 2022-09-03T05:54:32.002 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.002 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:32.003 DEBUG:boto:Port: 8001 2022-09-03T05:54:32.003 DEBUG:boto:Params: {} 2022-09-03T05:54:32.004 DEBUG:boto:Token: None 2022-09-03T05:54:32.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eAiDv9rX8GG/+SmU7BOj7VDd9E8= 2022-09-03T05:54:32.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eAiDv9rX8GG/+SmU7BOj7VDd9E8=', 'Content-Length': '0'} 2022-09-03T05:54:32.008 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006a1d2bd219dcd5ca-006312ec18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.009 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:32.010 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:32.010 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.011 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.011 DEBUG:boto:Method: GET 2022-09-03T05:54:32.012 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.012 DEBUG:boto:Data: 2022-09-03T05:54:32.013 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.014 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.015 DEBUG:boto:Params: {} 2022-09-03T05:54:32.015 DEBUG:boto:Token: None 2022-09-03T05:54:32.016 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.016 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eAiDv9rX8GG/+SmU7BOj7VDd9E8= 2022-09-03T05:54:32.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eAiDv9rX8GG/+SmU7BOj7VDd9E8=', 'Content-Length': '0'} 2022-09-03T05:54:32.019 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003970ae15c3a85293-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.020 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:32.020 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:32.021 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.021 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.022 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.022 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.023 DEBUG:boto:Data: 2022-09-03T05:54:32.023 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.024 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:32.024 DEBUG:boto:Port: 8001 2022-09-03T05:54:32.025 DEBUG:boto:Params: {} 2022-09-03T05:54:32.025 DEBUG:boto:Token: None 2022-09-03T05:54:32.026 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VvCflF1MBzahhFuLUBvsFK1Ca6I= 2022-09-03T05:54:32.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VvCflF1MBzahhFuLUBvsFK1Ca6I=', 'Content-Length': '0'} 2022-09-03T05:54:32.029 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003d825da0fd9935ed-006312ec18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.030 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:32.031 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:32.031 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.032 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.032 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.033 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.033 DEBUG:boto:Data: 2022-09-03T05:54:32.034 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.034 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.035 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.035 DEBUG:boto:Params: {} 2022-09-03T05:54:32.036 DEBUG:boto:Token: None 2022-09-03T05:54:32.037 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VvCflF1MBzahhFuLUBvsFK1Ca6I= 2022-09-03T05:54:32.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VvCflF1MBzahhFuLUBvsFK1Ca6I=', 'Content-Length': '0'} 2022-09-03T05:54:32.040 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ad34f656ab60375b-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.041 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:32.042 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:32.042 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:32.043 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.043 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.044 DEBUG:boto:Method: GET 2022-09-03T05:54:32.044 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.045 DEBUG:boto:Data: 2022-09-03T05:54:32.045 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.046 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:32.046 DEBUG:boto:Port: 8001 2022-09-03T05:54:32.047 DEBUG:boto:Params: {} 2022-09-03T05:54:32.047 DEBUG:boto:Token: None 2022-09-03T05:54:32.048 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eAiDv9rX8GG/+SmU7BOj7VDd9E8= 2022-09-03T05:54:32.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eAiDv9rX8GG/+SmU7BOj7VDd9E8=', 'Content-Length': '0'} 2022-09-03T05:54:32.051 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000002cee8cf30f17d5d-006312ec18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.052 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:32.053 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:32.053 DEBUG:boto:path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.054 DEBUG:boto:auth_path=/avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.055 DEBUG:boto:Method: GET 2022-09-03T05:54:32.055 DEBUG:boto:Path: /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.056 DEBUG:boto:Data: 2022-09-03T05:54:32.056 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.057 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.058 DEBUG:boto:Params: {} 2022-09-03T05:54:32.058 DEBUG:boto:Token: None 2022-09-03T05:54:32.059 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj?versionId=null 2022-09-03T05:54:32.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eAiDv9rX8GG/+SmU7BOj7VDd9E8= 2022-09-03T05:54:32.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eAiDv9rX8GG/+SmU7BOj7VDd9E8=', 'Content-Length': '0'} 2022-09-03T05:54:32.062 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bad530c799993ea5-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.063 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:32.064 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:32.064 DEBUG:boto:path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:32.065 DEBUG:boto:auth_path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:32.065 DEBUG:boto:Method: GET 2022-09-03T05:54:32.066 DEBUG:boto:Path: /avysmw-13/?prefix=myobj 2022-09-03T05:54:32.066 DEBUG:boto:Data: 2022-09-03T05:54:32.067 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:32.068 DEBUG:boto:Port: 8001 2022-09-03T05:54:32.068 DEBUG:boto:Params: {} 2022-09-03T05:54:32.069 DEBUG:boto:Token: None 2022-09-03T05:54:32.070 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/ 2022-09-03T05:54:32.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZXyCyiWXpDZsbSdbRJXRCyiZQsg= 2022-09-03T05:54:32.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZXyCyiWXpDZsbSdbRJXRCyiZQsg=', 'Content-Length': '0'} 2022-09-03T05:54:32.073 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007532db4166e7a5e4-006312ec18-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.074 DEBUG:boto:b'avysmw-13myobj1000falsemyobj2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.075 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:32.075 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:32.076 DEBUG:boto:path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:32.076 DEBUG:boto:auth_path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:32.077 DEBUG:boto:Method: GET 2022-09-03T05:54:32.078 DEBUG:boto:Path: /avysmw-13/?prefix=myobj 2022-09-03T05:54:32.078 DEBUG:boto:Data: 2022-09-03T05:54:32.079 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.080 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.080 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.081 DEBUG:boto:Params: {} 2022-09-03T05:54:32.081 DEBUG:boto:Token: None 2022-09-03T05:54:32.082 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/ 2022-09-03T05:54:32.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZXyCyiWXpDZsbSdbRJXRCyiZQsg= 2022-09-03T05:54:32.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZXyCyiWXpDZsbSdbRJXRCyiZQsg=', 'Content-Length': '0'} 2022-09-03T05:54:32.086 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f10d02c75632ad8f-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.086 DEBUG:boto:b'avysmw-13myobj1000falsemyobj2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.087 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:32.088 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:32.088 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.089 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:32.089 DEBUG:boto:Data: 2022-09-03T05:54:32.090 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.090 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:32.091 DEBUG:boto:Port: 8001 2022-09-03T05:54:32.091 DEBUG:boto:Params: {} 2022-09-03T05:54:32.092 DEBUG:boto:Token: None 2022-09-03T05:54:32.092 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj 2022-09-03T05:54:32.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W0doO2prZa6lJia3Td5r6S85GSk= 2022-09-03T05:54:32.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W0doO2prZa6lJia3Td5r6S85GSk=', 'Content-Length': '0'} 2022-09-03T05:54:32.095 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008abd3deaf16580c8-006312ec18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.096 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:32.097 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:32.097 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.098 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:32.098 DEBUG:boto:Data: 2022-09-03T05:54:32.099 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.100 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.100 DEBUG:boto:Params: {} 2022-09-03T05:54:32.101 DEBUG:boto:Token: None 2022-09-03T05:54:32.101 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj 2022-09-03T05:54:32.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W0doO2prZa6lJia3Td5r6S85GSk= 2022-09-03T05:54:32.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W0doO2prZa6lJia3Td5r6S85GSk=', 'Content-Length': '0'} 2022-09-03T05:54:32.104 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000589b8b10b5214766-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.105 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:32.106 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:32.106 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:32.107 DEBUG:boto:Method: GET 2022-09-03T05:54:32.107 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:32.108 DEBUG:boto:Data: 2022-09-03T05:54:32.108 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:32.109 DEBUG:boto:Port: 8001 2022-09-03T05:54:32.110 DEBUG:boto:Params: {} 2022-09-03T05:54:32.111 DEBUG:boto:Token: None 2022-09-03T05:54:32.111 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj 2022-09-03T05:54:32.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gdHL39pNkS0IO21SAxqOxUQgUtA= 2022-09-03T05:54:32.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gdHL39pNkS0IO21SAxqOxUQgUtA=', 'Content-Length': '0'} 2022-09-03T05:54:32.114 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f851957c07177ab0-006312ec18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.115 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:32.116 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:32.116 DEBUG:boto:Method: GET 2022-09-03T05:54:32.117 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:32.117 DEBUG:boto:Data: 2022-09-03T05:54:32.118 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.118 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.119 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.119 DEBUG:boto:Params: {} 2022-09-03T05:54:32.120 DEBUG:boto:Token: None 2022-09-03T05:54:32.121 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/myobj 2022-09-03T05:54:32.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gdHL39pNkS0IO21SAxqOxUQgUtA= 2022-09-03T05:54:32.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gdHL39pNkS0IO21SAxqOxUQgUtA=', 'Content-Length': '0'} 2022-09-03T05:54:32.124 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fe1ef227cae04e5c-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.125 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:32.125 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:32.126 DEBUG:boto:path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:32.126 DEBUG:boto:auth_path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:32.127 DEBUG:boto:Method: GET 2022-09-03T05:54:32.127 DEBUG:boto:Path: /avysmw-13/?prefix=myobj 2022-09-03T05:54:32.128 DEBUG:boto:Data: 2022-09-03T05:54:32.128 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.129 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:32.129 DEBUG:boto:Port: 8001 2022-09-03T05:54:32.130 DEBUG:boto:Params: {} 2022-09-03T05:54:32.131 DEBUG:boto:Token: None 2022-09-03T05:54:32.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/ 2022-09-03T05:54:32.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZXyCyiWXpDZsbSdbRJXRCyiZQsg= 2022-09-03T05:54:32.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZXyCyiWXpDZsbSdbRJXRCyiZQsg=', 'Content-Length': '0'} 2022-09-03T05:54:32.135 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000059fec2a0613dc6cd-006312ec18-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.178 DEBUG:boto:b'avysmw-13myobj1000falsemyobj2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.179 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:32.180 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:32.181 DEBUG:boto:path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:32.182 DEBUG:boto:auth_path=/avysmw-13/?prefix=myobj 2022-09-03T05:54:32.183 DEBUG:boto:Method: GET 2022-09-03T05:54:32.184 DEBUG:boto:Path: /avysmw-13/?prefix=myobj 2022-09-03T05:54:32.185 DEBUG:boto:Data: 2022-09-03T05:54:32.186 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.188 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.189 DEBUG:boto:Params: {} 2022-09-03T05:54:32.190 DEBUG:boto:Token: None 2022-09-03T05:54:32.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-13/ 2022-09-03T05:54:32.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZXyCyiWXpDZsbSdbRJXRCyiZQsg= 2022-09-03T05:54:32.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZXyCyiWXpDZsbSdbRJXRCyiZQsg=', 'Content-Length': '0'} 2022-09-03T05:54:32.196 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000941fef8b28d1c2a8-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.197 DEBUG:boto:b'avysmw-13myobj1000falsemyobj2022-09-03T05:54:10.488Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.198 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-13 zones={test-zone3, test-zone1} 2022-09-03T05:54:32.199 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:32.199 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:32.200 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:32.264 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:32.264 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:32.315 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:32.313+0000 7efda2edc600 1 bucket sync caught up with source: 2022-09-03T05:54:32.315 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:32.315 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:32.316 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:32.313+0000 7efda2edc600 0 bucket checkpoint complete 2022-09-03T05:54:32.336 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:32.336 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:32.337 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-14 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:32.398 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:32.398 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:32.433 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:32.432+0000 7f142ac7d600 1 bucket sync caught up with source: 2022-09-03T05:54:32.434 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.20.5, , , , , , , , , ] 2022-09-03T05:54:32.434 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.20.5, , , , , , , , , ] 2022-09-03T05:54:32.435 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:32.432+0000 7f142ac7d600 0 bucket checkpoint complete 2022-09-03T05:54:32.452 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-14 zones={test-zone1, test-zone2} 2022-09-03T05:54:32.452 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.453 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.454 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.454 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:54:32.455 DEBUG:boto:Data: 2022-09-03T05:54:32.455 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.456 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.457 DEBUG:boto:Params: {} 2022-09-03T05:54:32.458 DEBUG:boto:Token: None 2022-09-03T05:54:32.458 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/ 2022-09-03T05:54:32.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DSpW2v9RiL5apq1aAKf7hEhW1+4= 2022-09-03T05:54:32.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DSpW2v9RiL5apq1aAKf7hEhW1+4=', 'Content-Length': '0'} 2022-09-03T05:54:32.462 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cec43df1b06fe0e2-006312ec18-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.463 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.464 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.464 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.465 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:54:32.465 DEBUG:boto:Data: 2022-09-03T05:54:32.466 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.467 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.467 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.468 DEBUG:boto:Params: {} 2022-09-03T05:54:32.468 DEBUG:boto:Token: None 2022-09-03T05:54:32.469 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/ 2022-09-03T05:54:32.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DSpW2v9RiL5apq1aAKf7hEhW1+4= 2022-09-03T05:54:32.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DSpW2v9RiL5apq1aAKf7hEhW1+4=', 'Content-Length': '0'} 2022-09-03T05:54:32.473 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000018c95481b0714ea7-006312ec18-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.473 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:32.474 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.474 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.475 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.475 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.476 DEBUG:boto:Method: GET 2022-09-03T05:54:32.477 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.477 DEBUG:boto:Data: 2022-09-03T05:54:32.478 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.479 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.479 DEBUG:boto:Params: {} 2022-09-03T05:54:32.480 DEBUG:boto:Token: None 2022-09-03T05:54:32.481 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/?versions 2022-09-03T05:54:32.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uEHhzaTZ3CvjfpzAiDG/gv7sa5g= 2022-09-03T05:54:32.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uEHhzaTZ3CvjfpzAiDG/gv7sa5g=', 'Content-Length': '0'} 2022-09-03T05:54:32.485 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000914a71ca97c1e49b-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.526 DEBUG:boto:b'avysmw-14999falsemyobjnulltrue2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.528 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:32.528 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:32.529 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.530 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.531 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.532 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.533 DEBUG:boto:Method: GET 2022-09-03T05:54:32.533 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.534 DEBUG:boto:Data: 2022-09-03T05:54:32.534 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.535 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.535 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.536 DEBUG:boto:Params: {} 2022-09-03T05:54:32.537 DEBUG:boto:Token: None 2022-09-03T05:54:32.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/?versions 2022-09-03T05:54:32.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uEHhzaTZ3CvjfpzAiDG/gv7sa5g= 2022-09-03T05:54:32.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uEHhzaTZ3CvjfpzAiDG/gv7sa5g=', 'Content-Length': '0'} 2022-09-03T05:54:32.542 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cc451949e7ed8f68-006312ec18-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.542 DEBUG:boto:b'avysmw-14999falsemyobjnulltrue2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.543 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:32.544 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.544 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.545 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.545 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.546 DEBUG:boto:Method: GET 2022-09-03T05:54:32.547 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.547 DEBUG:boto:Data: 2022-09-03T05:54:32.548 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.549 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.549 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.550 DEBUG:boto:Params: {} 2022-09-03T05:54:32.551 DEBUG:boto:Token: None 2022-09-03T05:54:32.551 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/?versions 2022-09-03T05:54:32.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uEHhzaTZ3CvjfpzAiDG/gv7sa5g= 2022-09-03T05:54:32.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uEHhzaTZ3CvjfpzAiDG/gv7sa5g=', 'Content-Length': '0'} 2022-09-03T05:54:32.555 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001d24510281f411c3-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.556 DEBUG:boto:b'avysmw-14999falsemyobjnulltrue2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.557 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.558 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.558 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.559 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.560 DEBUG:boto:Method: GET 2022-09-03T05:54:32.560 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:54:32.561 DEBUG:boto:Data: 2022-09-03T05:54:32.561 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.562 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.562 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.563 DEBUG:boto:Params: {} 2022-09-03T05:54:32.564 DEBUG:boto:Token: None 2022-09-03T05:54:32.564 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/?versions 2022-09-03T05:54:32.565 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uEHhzaTZ3CvjfpzAiDG/gv7sa5g= 2022-09-03T05:54:32.565 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uEHhzaTZ3CvjfpzAiDG/gv7sa5g=', 'Content-Length': '0'} 2022-09-03T05:54:32.568 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b30db6ee4695b14b-006312ec18-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.569 DEBUG:boto:b'avysmw-14999falsemyobjnulltrue2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.570 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:32.570 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.571 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.571 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.572 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.573 DEBUG:boto:Method: GET 2022-09-03T05:54:32.573 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.574 DEBUG:boto:Data: 2022-09-03T05:54:32.575 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.577 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.578 DEBUG:boto:Params: {} 2022-09-03T05:54:32.580 DEBUG:boto:Token: None 2022-09-03T05:54:32.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q04GaR/gcB4rihld7GXOhNRG9yI= 2022-09-03T05:54:32.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q04GaR/gcB4rihld7GXOhNRG9yI=', 'Content-Length': '0'} 2022-09-03T05:54:32.585 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000433aacbc755baf8b-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.587 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.588 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.589 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.590 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.591 DEBUG:boto:Method: GET 2022-09-03T05:54:32.592 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.593 DEBUG:boto:Data: 2022-09-03T05:54:32.594 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.595 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.596 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.597 DEBUG:boto:Params: {} 2022-09-03T05:54:32.598 DEBUG:boto:Token: None 2022-09-03T05:54:32.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q04GaR/gcB4rihld7GXOhNRG9yI= 2022-09-03T05:54:32.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q04GaR/gcB4rihld7GXOhNRG9yI=', 'Content-Length': '0'} 2022-09-03T05:54:32.603 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cb367ea9209478c8-006312ec18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.605 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.605 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.606 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.606 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.607 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.607 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.608 DEBUG:boto:Data: 2022-09-03T05:54:32.609 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.609 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.610 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.610 DEBUG:boto:Params: {} 2022-09-03T05:54:32.611 DEBUG:boto:Token: None 2022-09-03T05:54:32.612 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jg8d0sHT6jTVAiR4wSZdLPnofY0= 2022-09-03T05:54:32.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jg8d0sHT6jTVAiR4wSZdLPnofY0=', 'Content-Length': '0'} 2022-09-03T05:54:32.615 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000533033c4b66e21df-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.615 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.616 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.616 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.617 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.617 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.618 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.618 DEBUG:boto:Data: 2022-09-03T05:54:32.619 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.619 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.620 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.620 DEBUG:boto:Params: {} 2022-09-03T05:54:32.621 DEBUG:boto:Token: None 2022-09-03T05:54:32.622 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jg8d0sHT6jTVAiR4wSZdLPnofY0= 2022-09-03T05:54:32.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jg8d0sHT6jTVAiR4wSZdLPnofY0=', 'Content-Length': '0'} 2022-09-03T05:54:32.625 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fe299a69c20eb77b-006312ec18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.626 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:32.626 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.627 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.627 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.628 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.628 DEBUG:boto:Method: GET 2022-09-03T05:54:32.629 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.629 DEBUG:boto:Data: 2022-09-03T05:54:32.630 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.631 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.632 DEBUG:boto:Params: {} 2022-09-03T05:54:32.632 DEBUG:boto:Token: None 2022-09-03T05:54:32.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q04GaR/gcB4rihld7GXOhNRG9yI= 2022-09-03T05:54:32.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q04GaR/gcB4rihld7GXOhNRG9yI=', 'Content-Length': '0'} 2022-09-03T05:54:32.636 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000012f6e7d3540092bd-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.637 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.637 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.638 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.639 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.639 DEBUG:boto:Method: GET 2022-09-03T05:54:32.640 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.640 DEBUG:boto:Data: 2022-09-03T05:54:32.641 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.641 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.642 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.642 DEBUG:boto:Params: {} 2022-09-03T05:54:32.643 DEBUG:boto:Token: None 2022-09-03T05:54:32.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:32.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q04GaR/gcB4rihld7GXOhNRG9yI= 2022-09-03T05:54:32.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q04GaR/gcB4rihld7GXOhNRG9yI=', 'Content-Length': '0'} 2022-09-03T05:54:32.647 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009d20602c7c94ff13-006312ec18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.647 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.648 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.649 DEBUG:boto:path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:32.649 DEBUG:boto:auth_path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:32.650 DEBUG:boto:Method: GET 2022-09-03T05:54:32.650 DEBUG:boto:Path: /avysmw-14/?prefix=myobj 2022-09-03T05:54:32.651 DEBUG:boto:Data: 2022-09-03T05:54:32.651 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.652 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.653 DEBUG:boto:Params: {} 2022-09-03T05:54:32.653 DEBUG:boto:Token: None 2022-09-03T05:54:32.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/ 2022-09-03T05:54:32.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ta6GAM19afxVtg5Ctjnhv7QsJHM= 2022-09-03T05:54:32.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ta6GAM19afxVtg5Ctjnhv7QsJHM=', 'Content-Length': '0'} 2022-09-03T05:54:32.659 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005de3f77bcb8b64bf-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.660 DEBUG:boto:b'avysmw-14myobj1000falsemyobj2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.660 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.661 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.662 DEBUG:boto:path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:32.662 DEBUG:boto:auth_path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:32.663 DEBUG:boto:Method: GET 2022-09-03T05:54:32.664 DEBUG:boto:Path: /avysmw-14/?prefix=myobj 2022-09-03T05:54:32.665 DEBUG:boto:Data: 2022-09-03T05:54:32.666 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.666 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.667 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.668 DEBUG:boto:Params: {} 2022-09-03T05:54:32.669 DEBUG:boto:Token: None 2022-09-03T05:54:32.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/ 2022-09-03T05:54:32.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ta6GAM19afxVtg5Ctjnhv7QsJHM= 2022-09-03T05:54:32.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ta6GAM19afxVtg5Ctjnhv7QsJHM=', 'Content-Length': '0'} 2022-09-03T05:54:32.678 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bba48daaf2124217-006312ec18-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.718 DEBUG:boto:b'avysmw-14myobj1000falsemyobj2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.719 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.720 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.720 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.721 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:54:32.721 DEBUG:boto:Data: 2022-09-03T05:54:32.722 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.723 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.723 DEBUG:boto:Params: {} 2022-09-03T05:54:32.724 DEBUG:boto:Token: None 2022-09-03T05:54:32.724 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj 2022-09-03T05:54:32.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p+1FSdC0bsDTrkotLGFPvNRXIlQ= 2022-09-03T05:54:32.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p+1FSdC0bsDTrkotLGFPvNRXIlQ=', 'Content-Length': '0'} 2022-09-03T05:54:32.727 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f9b48c1591e350fb-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.728 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.729 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.729 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.730 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:54:32.730 DEBUG:boto:Data: 2022-09-03T05:54:32.731 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.731 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.732 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.733 DEBUG:boto:Params: {} 2022-09-03T05:54:32.734 DEBUG:boto:Token: None 2022-09-03T05:54:32.735 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj 2022-09-03T05:54:32.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p+1FSdC0bsDTrkotLGFPvNRXIlQ= 2022-09-03T05:54:32.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p+1FSdC0bsDTrkotLGFPvNRXIlQ=', 'Content-Length': '0'} 2022-09-03T05:54:32.739 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000344c766c3eb479ce-006312ec18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.740 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:32.740 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.741 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.741 DEBUG:boto:Method: GET 2022-09-03T05:54:32.742 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:54:32.743 DEBUG:boto:Data: 2022-09-03T05:54:32.743 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.746 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.747 DEBUG:boto:Params: {} 2022-09-03T05:54:32.748 DEBUG:boto:Token: None 2022-09-03T05:54:32.749 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj 2022-09-03T05:54:32.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XYKA+UmuywF4yEMr9EFvPUybX9k= 2022-09-03T05:54:32.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XYKA+UmuywF4yEMr9EFvPUybX9k=', 'Content-Length': '0'} 2022-09-03T05:54:32.752 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000004efca5921dc4f70-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.753 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:32.754 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:32.755 DEBUG:boto:Method: GET 2022-09-03T05:54:32.755 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:54:32.756 DEBUG:boto:Data: 2022-09-03T05:54:32.756 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.757 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.757 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.758 DEBUG:boto:Params: {} 2022-09-03T05:54:32.759 DEBUG:boto:Token: None 2022-09-03T05:54:32.759 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/myobj 2022-09-03T05:54:32.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XYKA+UmuywF4yEMr9EFvPUybX9k= 2022-09-03T05:54:32.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XYKA+UmuywF4yEMr9EFvPUybX9k=', 'Content-Length': '0'} 2022-09-03T05:54:32.762 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000aebb97ce80eb4d9c-006312ec18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.763 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.764 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.765 DEBUG:boto:path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:32.765 DEBUG:boto:auth_path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:32.766 DEBUG:boto:Method: GET 2022-09-03T05:54:32.766 DEBUG:boto:Path: /avysmw-14/?prefix=myobj 2022-09-03T05:54:32.767 DEBUG:boto:Data: 2022-09-03T05:54:32.767 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:32.768 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.769 DEBUG:boto:Params: {} 2022-09-03T05:54:32.770 DEBUG:boto:Token: None 2022-09-03T05:54:32.770 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/ 2022-09-03T05:54:32.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ta6GAM19afxVtg5Ctjnhv7QsJHM= 2022-09-03T05:54:32.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ta6GAM19afxVtg5Ctjnhv7QsJHM=', 'Content-Length': '0'} 2022-09-03T05:54:32.774 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045ad60a6dc9471b1-006312ec18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.775 DEBUG:boto:b'avysmw-14myobj1000falsemyobj2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.776 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.776 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.777 DEBUG:boto:path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:32.777 DEBUG:boto:auth_path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:32.778 DEBUG:boto:Method: GET 2022-09-03T05:54:32.778 DEBUG:boto:Path: /avysmw-14/?prefix=myobj 2022-09-03T05:54:32.779 DEBUG:boto:Data: 2022-09-03T05:54:32.779 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.780 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.780 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.781 DEBUG:boto:Params: {} 2022-09-03T05:54:32.781 DEBUG:boto:Token: None 2022-09-03T05:54:32.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/ 2022-09-03T05:54:32.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ta6GAM19afxVtg5Ctjnhv7QsJHM= 2022-09-03T05:54:32.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ta6GAM19afxVtg5Ctjnhv7QsJHM=', 'Content-Length': '0'} 2022-09-03T05:54:32.787 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006598836169524d7d-006312ec18-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.834 DEBUG:boto:b'avysmw-14myobj1000falsemyobj2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:32.835 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-14 zones={test-zone1, test-zone2} 2022-09-03T05:54:32.836 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:32.836 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:32.837 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-14 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:32.896 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:32.897 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:32.952 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:32.950+0000 7f6039864600 1 bucket sync caught up with source: 2022-09-03T05:54:32.953 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.20.5, , , , , , , , , ] 2022-09-03T05:54:32.953 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.20.5, , , , , , , , , ] 2022-09-03T05:54:32.953 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:32.950+0000 7f6039864600 0 bucket checkpoint complete 2022-09-03T05:54:32.977 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-14 zones={test-zone3, test-zone2} 2022-09-03T05:54:32.978 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.979 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.980 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.980 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:54:32.981 DEBUG:boto:Data: 2022-09-03T05:54:32.982 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.982 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:32.983 DEBUG:boto:Port: 8001 2022-09-03T05:54:32.983 DEBUG:boto:Params: {} 2022-09-03T05:54:32.984 DEBUG:boto:Token: None 2022-09-03T05:54:32.985 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/ 2022-09-03T05:54:32.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DSpW2v9RiL5apq1aAKf7hEhW1+4= 2022-09-03T05:54:32.986 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DSpW2v9RiL5apq1aAKf7hEhW1+4=', 'Content-Length': '0'} 2022-09-03T05:54:32.989 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009658bec05f460ae7-006312ec18-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:32.990 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:32.990 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:32.991 DEBUG:boto:Method: HEAD 2022-09-03T05:54:32.991 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:54:32.992 DEBUG:boto:Data: 2022-09-03T05:54:32.992 DEBUG:boto:Headers: {} 2022-09-03T05:54:32.993 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:32.993 DEBUG:boto:Port: 8000 2022-09-03T05:54:32.994 DEBUG:boto:Params: {} 2022-09-03T05:54:32.994 DEBUG:boto:Token: None 2022-09-03T05:54:32.995 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:32 GMT /avysmw-14/ 2022-09-03T05:54:32.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DSpW2v9RiL5apq1aAKf7hEhW1+4= 2022-09-03T05:54:32.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DSpW2v9RiL5apq1aAKf7hEhW1+4=', 'Content-Length': '0'} 2022-09-03T05:54:32.999 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b1287cda6d9aa991-006312ec18-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.000 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:33.000 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:33.001 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:33.001 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.002 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.003 DEBUG:boto:Method: GET 2022-09-03T05:54:33.003 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.004 DEBUG:boto:Data: 2022-09-03T05:54:33.004 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.005 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.005 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.006 DEBUG:boto:Params: {} 2022-09-03T05:54:33.006 DEBUG:boto:Token: None 2022-09-03T05:54:33.007 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/?versions 2022-09-03T05:54:33.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2lggv9jBVqLZARxFW/9clu/x96Q= 2022-09-03T05:54:33.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2lggv9jBVqLZARxFW/9clu/x96Q=', 'Content-Length': '0'} 2022-09-03T05:54:33.011 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a85756135a6989a-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.012 DEBUG:boto:b'avysmw-14999falsemyobjnulltrue2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.013 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:33.013 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:33.014 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:33.014 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:33.015 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.016 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.017 DEBUG:boto:Method: GET 2022-09-03T05:54:33.018 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.019 DEBUG:boto:Data: 2022-09-03T05:54:33.019 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.020 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.021 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.022 DEBUG:boto:Params: {} 2022-09-03T05:54:33.023 DEBUG:boto:Token: None 2022-09-03T05:54:33.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/?versions 2022-09-03T05:54:33.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2lggv9jBVqLZARxFW/9clu/x96Q= 2022-09-03T05:54:33.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2lggv9jBVqLZARxFW/9clu/x96Q=', 'Content-Length': '0'} 2022-09-03T05:54:33.028 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006ed4227c16f925c4-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.029 DEBUG:boto:b'avysmw-14999falsemyobjnulltrue2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.030 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:33.030 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:33.031 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:33.031 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.032 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.033 DEBUG:boto:Method: GET 2022-09-03T05:54:33.033 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.034 DEBUG:boto:Data: 2022-09-03T05:54:33.035 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.036 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.036 DEBUG:boto:Params: {} 2022-09-03T05:54:33.037 DEBUG:boto:Token: None 2022-09-03T05:54:33.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/?versions 2022-09-03T05:54:33.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2lggv9jBVqLZARxFW/9clu/x96Q= 2022-09-03T05:54:33.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2lggv9jBVqLZARxFW/9clu/x96Q=', 'Content-Length': '0'} 2022-09-03T05:54:33.042 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000086293e2162b92db1-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.042 DEBUG:boto:b'avysmw-14999falsemyobjnulltrue2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.043 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:33.044 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:33.044 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.045 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.045 DEBUG:boto:Method: GET 2022-09-03T05:54:33.046 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:54:33.047 DEBUG:boto:Data: 2022-09-03T05:54:33.047 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.048 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.048 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.049 DEBUG:boto:Params: {} 2022-09-03T05:54:33.049 DEBUG:boto:Token: None 2022-09-03T05:54:33.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/?versions 2022-09-03T05:54:33.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2lggv9jBVqLZARxFW/9clu/x96Q= 2022-09-03T05:54:33.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2lggv9jBVqLZARxFW/9clu/x96Q=', 'Content-Length': '0'} 2022-09-03T05:54:33.054 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004bf0e05d3671c65f-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.055 DEBUG:boto:b'avysmw-14999falsemyobjnulltrue2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.056 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:33.056 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.057 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.057 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.058 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.058 DEBUG:boto:Method: GET 2022-09-03T05:54:33.059 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.059 DEBUG:boto:Data: 2022-09-03T05:54:33.060 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.062 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.063 DEBUG:boto:Params: {} 2022-09-03T05:54:33.064 DEBUG:boto:Token: None 2022-09-03T05:54:33.065 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.066 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y52slZJxXQURN0IUXOkM8ETHK+k= 2022-09-03T05:54:33.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y52slZJxXQURN0IUXOkM8ETHK+k=', 'Content-Length': '0'} 2022-09-03T05:54:33.069 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ca931664d2fd46d3-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.070 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.071 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.071 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.072 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.072 DEBUG:boto:Method: GET 2022-09-03T05:54:33.073 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.073 DEBUG:boto:Data: 2022-09-03T05:54:33.074 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.075 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.075 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.076 DEBUG:boto:Params: {} 2022-09-03T05:54:33.076 DEBUG:boto:Token: None 2022-09-03T05:54:33.077 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y52slZJxXQURN0IUXOkM8ETHK+k= 2022-09-03T05:54:33.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y52slZJxXQURN0IUXOkM8ETHK+k=', 'Content-Length': '0'} 2022-09-03T05:54:33.081 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f9852bacc5dbf916-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.082 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.082 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.083 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.084 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.084 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.085 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.085 DEBUG:boto:Data: 2022-09-03T05:54:33.086 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.086 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.087 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.087 DEBUG:boto:Params: {} 2022-09-03T05:54:33.088 DEBUG:boto:Token: None 2022-09-03T05:54:33.089 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uGfJChKFg1GG86GXGnQec5xQ5+0= 2022-09-03T05:54:33.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uGfJChKFg1GG86GXGnQec5xQ5+0=', 'Content-Length': '0'} 2022-09-03T05:54:33.092 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000b0a8a5895d7940e-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.093 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.094 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.094 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.095 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.095 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.096 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.097 DEBUG:boto:Data: 2022-09-03T05:54:33.097 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.098 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.098 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.099 DEBUG:boto:Params: {} 2022-09-03T05:54:33.099 DEBUG:boto:Token: None 2022-09-03T05:54:33.100 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uGfJChKFg1GG86GXGnQec5xQ5+0= 2022-09-03T05:54:33.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uGfJChKFg1GG86GXGnQec5xQ5+0=', 'Content-Length': '0'} 2022-09-03T05:54:33.103 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000056b91204c5b923ff-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.104 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:33.104 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.105 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.106 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.106 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.107 DEBUG:boto:Method: GET 2022-09-03T05:54:33.107 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.108 DEBUG:boto:Data: 2022-09-03T05:54:33.108 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.110 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.110 DEBUG:boto:Params: {} 2022-09-03T05:54:33.111 DEBUG:boto:Token: None 2022-09-03T05:54:33.112 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y52slZJxXQURN0IUXOkM8ETHK+k= 2022-09-03T05:54:33.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y52slZJxXQURN0IUXOkM8ETHK+k=', 'Content-Length': '0'} 2022-09-03T05:54:33.116 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008f6942cbd19e119d-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.118 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.119 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.119 DEBUG:boto:path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.120 DEBUG:boto:auth_path=/avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.121 DEBUG:boto:Method: GET 2022-09-03T05:54:33.122 DEBUG:boto:Path: /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.123 DEBUG:boto:Data: 2022-09-03T05:54:33.124 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.125 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.126 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.127 DEBUG:boto:Params: {} 2022-09-03T05:54:33.128 DEBUG:boto:Token: None 2022-09-03T05:54:33.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj?versionId=null 2022-09-03T05:54:33.129 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y52slZJxXQURN0IUXOkM8ETHK+k= 2022-09-03T05:54:33.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y52slZJxXQURN0IUXOkM8ETHK+k=', 'Content-Length': '0'} 2022-09-03T05:54:33.132 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fde919db2da0a288-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.174 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:33.175 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:33.176 DEBUG:boto:path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:33.176 DEBUG:boto:auth_path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:33.177 DEBUG:boto:Method: GET 2022-09-03T05:54:33.178 DEBUG:boto:Path: /avysmw-14/?prefix=myobj 2022-09-03T05:54:33.179 DEBUG:boto:Data: 2022-09-03T05:54:33.180 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.180 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.181 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.182 DEBUG:boto:Params: {} 2022-09-03T05:54:33.182 DEBUG:boto:Token: None 2022-09-03T05:54:33.183 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/ 2022-09-03T05:54:33.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lTrlTgnJH31z8hfxBMHRyixW8Bc= 2022-09-03T05:54:33.184 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lTrlTgnJH31z8hfxBMHRyixW8Bc=', 'Content-Length': '0'} 2022-09-03T05:54:33.187 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb0aa051211f2b2b-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.188 DEBUG:boto:b'avysmw-14myobj1000falsemyobj2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.189 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:33.190 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:33.190 DEBUG:boto:path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:33.191 DEBUG:boto:auth_path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:33.191 DEBUG:boto:Method: GET 2022-09-03T05:54:33.192 DEBUG:boto:Path: /avysmw-14/?prefix=myobj 2022-09-03T05:54:33.192 DEBUG:boto:Data: 2022-09-03T05:54:33.193 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.194 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.194 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.195 DEBUG:boto:Params: {} 2022-09-03T05:54:33.196 DEBUG:boto:Token: None 2022-09-03T05:54:33.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/ 2022-09-03T05:54:33.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lTrlTgnJH31z8hfxBMHRyixW8Bc= 2022-09-03T05:54:33.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lTrlTgnJH31z8hfxBMHRyixW8Bc=', 'Content-Length': '0'} 2022-09-03T05:54:33.201 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027ba93aff5826cec-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.202 DEBUG:boto:b'avysmw-14myobj1000falsemyobj2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.203 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.203 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.204 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.204 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:54:33.205 DEBUG:boto:Data: 2022-09-03T05:54:33.205 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.206 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.206 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.207 DEBUG:boto:Params: {} 2022-09-03T05:54:33.207 DEBUG:boto:Token: None 2022-09-03T05:54:33.208 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj 2022-09-03T05:54:33.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUzbEuZBiVjeJOT7kdC3Uacwnfo= 2022-09-03T05:54:33.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUzbEuZBiVjeJOT7kdC3Uacwnfo=', 'Content-Length': '0'} 2022-09-03T05:54:33.211 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000093e8fb0ff9d425e2-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.212 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.212 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.213 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.213 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:54:33.214 DEBUG:boto:Data: 2022-09-03T05:54:33.214 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.215 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.215 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.216 DEBUG:boto:Params: {} 2022-09-03T05:54:33.216 DEBUG:boto:Token: None 2022-09-03T05:54:33.217 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj 2022-09-03T05:54:33.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUzbEuZBiVjeJOT7kdC3Uacwnfo= 2022-09-03T05:54:33.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUzbEuZBiVjeJOT7kdC3Uacwnfo=', 'Content-Length': '0'} 2022-09-03T05:54:33.220 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a4abab8af26a6d47-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.221 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:33.222 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.222 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.223 DEBUG:boto:Method: GET 2022-09-03T05:54:33.224 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:54:33.224 DEBUG:boto:Data: 2022-09-03T05:54:33.225 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.226 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.226 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.227 DEBUG:boto:Params: {} 2022-09-03T05:54:33.228 DEBUG:boto:Token: None 2022-09-03T05:54:33.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj 2022-09-03T05:54:33.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ms8EDoQ2e29qCiGN4/n2UaavSBU= 2022-09-03T05:54:33.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ms8EDoQ2e29qCiGN4/n2UaavSBU=', 'Content-Length': '0'} 2022-09-03T05:54:33.232 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a88f822ea91c2707-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.232 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:54:33.233 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:54:33.233 DEBUG:boto:Method: GET 2022-09-03T05:54:33.234 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:54:33.235 DEBUG:boto:Data: 2022-09-03T05:54:33.235 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.236 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.236 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.237 DEBUG:boto:Params: {} 2022-09-03T05:54:33.237 DEBUG:boto:Token: None 2022-09-03T05:54:33.238 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/myobj 2022-09-03T05:54:33.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ms8EDoQ2e29qCiGN4/n2UaavSBU= 2022-09-03T05:54:33.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ms8EDoQ2e29qCiGN4/n2UaavSBU=', 'Content-Length': '0'} 2022-09-03T05:54:33.241 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b4efbf437c2f9acb-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.242 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:33.243 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:33.243 DEBUG:boto:path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:33.244 DEBUG:boto:auth_path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:33.245 DEBUG:boto:Method: GET 2022-09-03T05:54:33.245 DEBUG:boto:Path: /avysmw-14/?prefix=myobj 2022-09-03T05:54:33.246 DEBUG:boto:Data: 2022-09-03T05:54:33.246 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.247 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.247 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.248 DEBUG:boto:Params: {} 2022-09-03T05:54:33.248 DEBUG:boto:Token: None 2022-09-03T05:54:33.249 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/ 2022-09-03T05:54:33.250 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lTrlTgnJH31z8hfxBMHRyixW8Bc= 2022-09-03T05:54:33.250 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lTrlTgnJH31z8hfxBMHRyixW8Bc=', 'Content-Length': '0'} 2022-09-03T05:54:33.253 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a6ae1a2545bbd38-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.254 DEBUG:boto:b'avysmw-14myobj1000falsemyobj2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.254 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:54:33.255 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:54:33.256 DEBUG:boto:path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:33.256 DEBUG:boto:auth_path=/avysmw-14/?prefix=myobj 2022-09-03T05:54:33.257 DEBUG:boto:Method: GET 2022-09-03T05:54:33.257 DEBUG:boto:Path: /avysmw-14/?prefix=myobj 2022-09-03T05:54:33.258 DEBUG:boto:Data: 2022-09-03T05:54:33.258 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.259 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.259 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.260 DEBUG:boto:Params: {} 2022-09-03T05:54:33.260 DEBUG:boto:Token: None 2022-09-03T05:54:33.261 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-14/ 2022-09-03T05:54:33.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lTrlTgnJH31z8hfxBMHRyixW8Bc= 2022-09-03T05:54:33.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lTrlTgnJH31z8hfxBMHRyixW8Bc=', 'Content-Length': '0'} 2022-09-03T05:54:33.265 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001bb49d1eee9ca607-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.266 DEBUG:boto:b'avysmw-14myobj1000falsemyobj2022-09-03T05:54:10.516Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.267 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-14 zones={test-zone3, test-zone2} 2022-09-03T05:54:33.267 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:33.268 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:33.269 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-15 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:33.328 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:33.329 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:33.384 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:33.382+0000 7f812c2aa600 1 bucket sync caught up with source: 2022-09-03T05:54:33.385 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.22.5, , , , , , , , , ] 2022-09-03T05:54:33.385 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.22.5, , , , , , , , , ] 2022-09-03T05:54:33.386 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:54:33.382+0000 7f812c2aa600 0 bucket checkpoint complete 2022-09-03T05:54:33.402 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-15 zones={test-zone1, test-zone3} 2022-09-03T05:54:33.403 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.403 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.404 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.404 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:54:33.405 DEBUG:boto:Data: 2022-09-03T05:54:33.405 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.406 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.407 DEBUG:boto:Params: {} 2022-09-03T05:54:33.407 DEBUG:boto:Token: None 2022-09-03T05:54:33.408 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/ 2022-09-03T05:54:33.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qCk1wme83vx+HRzjyPQwOhLOR+0= 2022-09-03T05:54:33.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qCk1wme83vx+HRzjyPQwOhLOR+0=', 'Content-Length': '0'} 2022-09-03T05:54:33.414 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c7de3c717f2eeacf-006312ec19-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.415 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.415 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.416 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.416 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:54:33.417 DEBUG:boto:Data: 2022-09-03T05:54:33.418 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.420 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.420 DEBUG:boto:Params: {} 2022-09-03T05:54:33.421 DEBUG:boto:Token: None 2022-09-03T05:54:33.421 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/ 2022-09-03T05:54:33.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qCk1wme83vx+HRzjyPQwOhLOR+0= 2022-09-03T05:54:33.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qCk1wme83vx+HRzjyPQwOhLOR+0=', 'Content-Length': '0'} 2022-09-03T05:54:33.425 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000955af2c6b6886744-006312ec19-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.426 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:33.427 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.428 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.429 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.429 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.430 DEBUG:boto:Method: GET 2022-09-03T05:54:33.431 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.432 DEBUG:boto:Data: 2022-09-03T05:54:33.433 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.434 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.435 DEBUG:boto:Params: {} 2022-09-03T05:54:33.436 DEBUG:boto:Token: None 2022-09-03T05:54:33.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/?versions 2022-09-03T05:54:33.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE= 2022-09-03T05:54:33.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE=', 'Content-Length': '0'} 2022-09-03T05:54:33.441 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db9ada9db8a57d6b-006312ec19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.441 DEBUG:boto:b'avysmw-15999falsemyobjnulltrue2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.442 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:33.443 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:33.443 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.444 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.444 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.445 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.445 DEBUG:boto:Method: GET 2022-09-03T05:54:33.446 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.447 DEBUG:boto:Data: 2022-09-03T05:54:33.447 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.448 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.449 DEBUG:boto:Params: {} 2022-09-03T05:54:33.449 DEBUG:boto:Token: None 2022-09-03T05:54:33.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/?versions 2022-09-03T05:54:33.450 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE= 2022-09-03T05:54:33.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE=', 'Content-Length': '0'} 2022-09-03T05:54:33.453 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005c6d18730b0d4fa4-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.455 DEBUG:boto:b'avysmw-15999falsemyobjnulltrue2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.455 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:33.456 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.457 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.457 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.458 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.458 DEBUG:boto:Method: GET 2022-09-03T05:54:33.459 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.460 DEBUG:boto:Data: 2022-09-03T05:54:33.460 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.462 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.462 DEBUG:boto:Params: {} 2022-09-03T05:54:33.463 DEBUG:boto:Token: None 2022-09-03T05:54:33.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/?versions 2022-09-03T05:54:33.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE= 2022-09-03T05:54:33.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE=', 'Content-Length': '0'} 2022-09-03T05:54:33.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ccbe981536ee1792-006312ec19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.469 DEBUG:boto:b'avysmw-15999falsemyobjnulltrue2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.470 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.471 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.471 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.472 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.472 DEBUG:boto:Method: GET 2022-09-03T05:54:33.473 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.473 DEBUG:boto:Data: 2022-09-03T05:54:33.474 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.475 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.476 DEBUG:boto:Params: {} 2022-09-03T05:54:33.477 DEBUG:boto:Token: None 2022-09-03T05:54:33.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/?versions 2022-09-03T05:54:33.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE= 2022-09-03T05:54:33.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE=', 'Content-Length': '0'} 2022-09-03T05:54:33.481 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007061616b5101e717-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.482 DEBUG:boto:b'avysmw-15999falsemyobjnulltrue2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.482 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:33.483 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.483 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.484 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.485 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.485 DEBUG:boto:Method: GET 2022-09-03T05:54:33.486 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.486 DEBUG:boto:Data: 2022-09-03T05:54:33.487 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.488 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.488 DEBUG:boto:Params: {} 2022-09-03T05:54:33.489 DEBUG:boto:Token: None 2022-09-03T05:54:33.490 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU= 2022-09-03T05:54:33.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU=', 'Content-Length': '0'} 2022-09-03T05:54:33.493 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000043ee1cb07b83e8cb-006312ec19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.494 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.494 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.495 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.496 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.496 DEBUG:boto:Method: GET 2022-09-03T05:54:33.497 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.497 DEBUG:boto:Data: 2022-09-03T05:54:33.498 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.500 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.501 DEBUG:boto:Params: {} 2022-09-03T05:54:33.502 DEBUG:boto:Token: None 2022-09-03T05:54:33.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU= 2022-09-03T05:54:33.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU=', 'Content-Length': '0'} 2022-09-03T05:54:33.507 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000458f96ba95a44f5b-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.508 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.509 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.509 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.510 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.510 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.511 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.511 DEBUG:boto:Data: 2022-09-03T05:54:33.512 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.513 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.513 DEBUG:boto:Params: {} 2022-09-03T05:54:33.514 DEBUG:boto:Token: None 2022-09-03T05:54:33.514 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xK+18TNjFd3/dPzVrYjrbUVlOxg= 2022-09-03T05:54:33.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xK+18TNjFd3/dPzVrYjrbUVlOxg=', 'Content-Length': '0'} 2022-09-03T05:54:33.518 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005388f7cbbc666140-006312ec19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.519 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.520 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.521 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.521 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.522 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.522 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.523 DEBUG:boto:Data: 2022-09-03T05:54:33.523 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.524 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.525 DEBUG:boto:Params: {} 2022-09-03T05:54:33.526 DEBUG:boto:Token: None 2022-09-03T05:54:33.526 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xK+18TNjFd3/dPzVrYjrbUVlOxg= 2022-09-03T05:54:33.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xK+18TNjFd3/dPzVrYjrbUVlOxg=', 'Content-Length': '0'} 2022-09-03T05:54:33.529 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ef34bbdca163eabc-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.531 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:33.532 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.533 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.534 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.535 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.536 DEBUG:boto:Method: GET 2022-09-03T05:54:33.536 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.537 DEBUG:boto:Data: 2022-09-03T05:54:33.537 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.539 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.539 DEBUG:boto:Params: {} 2022-09-03T05:54:33.540 DEBUG:boto:Token: None 2022-09-03T05:54:33.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU= 2022-09-03T05:54:33.542 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU=', 'Content-Length': '0'} 2022-09-03T05:54:33.544 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f2024de1a288b20e-006312ec19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.586 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.587 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.588 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.589 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.590 DEBUG:boto:Method: GET 2022-09-03T05:54:33.592 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.593 DEBUG:boto:Data: 2022-09-03T05:54:33.594 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.596 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.597 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.597 DEBUG:boto:Params: {} 2022-09-03T05:54:33.598 DEBUG:boto:Token: None 2022-09-03T05:54:33.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU= 2022-09-03T05:54:33.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU=', 'Content-Length': '0'} 2022-09-03T05:54:33.602 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005f4491c2206a120a-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.603 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.604 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.604 DEBUG:boto:path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:33.605 DEBUG:boto:auth_path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:33.606 DEBUG:boto:Method: GET 2022-09-03T05:54:33.606 DEBUG:boto:Path: /avysmw-15/?prefix=myobj 2022-09-03T05:54:33.607 DEBUG:boto:Data: 2022-09-03T05:54:33.607 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.608 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.609 DEBUG:boto:Params: {} 2022-09-03T05:54:33.609 DEBUG:boto:Token: None 2022-09-03T05:54:33.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/ 2022-09-03T05:54:33.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MYUEv+3NJuP/G8ETkZJFIjdfuB8= 2022-09-03T05:54:33.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MYUEv+3NJuP/G8ETkZJFIjdfuB8=', 'Content-Length': '0'} 2022-09-03T05:54:33.614 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006bae05392cfb642b-006312ec19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.614 DEBUG:boto:b'avysmw-15myobj1000falsemyobj2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.615 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.616 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.616 DEBUG:boto:path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:33.617 DEBUG:boto:auth_path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:33.618 DEBUG:boto:Method: GET 2022-09-03T05:54:33.618 DEBUG:boto:Path: /avysmw-15/?prefix=myobj 2022-09-03T05:54:33.619 DEBUG:boto:Data: 2022-09-03T05:54:33.619 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.620 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.620 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.621 DEBUG:boto:Params: {} 2022-09-03T05:54:33.621 DEBUG:boto:Token: None 2022-09-03T05:54:33.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/ 2022-09-03T05:54:33.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MYUEv+3NJuP/G8ETkZJFIjdfuB8= 2022-09-03T05:54:33.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MYUEv+3NJuP/G8ETkZJFIjdfuB8=', 'Content-Length': '0'} 2022-09-03T05:54:33.626 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079fd8aba01490d23-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.666 DEBUG:boto:b'avysmw-15myobj1000falsemyobj2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.667 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.668 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.668 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.669 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:54:33.669 DEBUG:boto:Data: 2022-09-03T05:54:33.670 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.671 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.671 DEBUG:boto:Params: {} 2022-09-03T05:54:33.672 DEBUG:boto:Token: None 2022-09-03T05:54:33.673 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj 2022-09-03T05:54:33.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0hEP20kVqkA3r75LMMAYOAvJbto= 2022-09-03T05:54:33.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0hEP20kVqkA3r75LMMAYOAvJbto=', 'Content-Length': '0'} 2022-09-03T05:54:33.686 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006c9507cca6139e3b-006312ec19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.686 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.687 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.688 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.688 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:54:33.689 DEBUG:boto:Data: 2022-09-03T05:54:33.689 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.690 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.690 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.691 DEBUG:boto:Params: {} 2022-09-03T05:54:33.691 DEBUG:boto:Token: None 2022-09-03T05:54:33.692 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj 2022-09-03T05:54:33.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0hEP20kVqkA3r75LMMAYOAvJbto= 2022-09-03T05:54:33.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0hEP20kVqkA3r75LMMAYOAvJbto=', 'Content-Length': '0'} 2022-09-03T05:54:33.695 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004c4181da10636bd4-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.696 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:33.697 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.697 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.698 DEBUG:boto:Method: GET 2022-09-03T05:54:33.698 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:54:33.699 DEBUG:boto:Data: 2022-09-03T05:54:33.699 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.700 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.701 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.701 DEBUG:boto:Params: {} 2022-09-03T05:54:33.702 DEBUG:boto:Token: None 2022-09-03T05:54:33.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj 2022-09-03T05:54:33.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FK8reNynG/yR6JUhAilWlx1TCOY= 2022-09-03T05:54:33.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FK8reNynG/yR6JUhAilWlx1TCOY=', 'Content-Length': '0'} 2022-09-03T05:54:33.706 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000027f2a675b752eabc-006312ec19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.707 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.707 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.708 DEBUG:boto:Method: GET 2022-09-03T05:54:33.708 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:54:33.709 DEBUG:boto:Data: 2022-09-03T05:54:33.709 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.710 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.710 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.711 DEBUG:boto:Params: {} 2022-09-03T05:54:33.711 DEBUG:boto:Token: None 2022-09-03T05:54:33.712 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj 2022-09-03T05:54:33.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FK8reNynG/yR6JUhAilWlx1TCOY= 2022-09-03T05:54:33.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FK8reNynG/yR6JUhAilWlx1TCOY=', 'Content-Length': '0'} 2022-09-03T05:54:33.715 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000950fde4f355b55ec-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.716 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.717 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.717 DEBUG:boto:path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:33.718 DEBUG:boto:auth_path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:33.719 DEBUG:boto:Method: GET 2022-09-03T05:54:33.719 DEBUG:boto:Path: /avysmw-15/?prefix=myobj 2022-09-03T05:54:33.720 DEBUG:boto:Data: 2022-09-03T05:54:33.720 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.721 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:33.721 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.722 DEBUG:boto:Params: {} 2022-09-03T05:54:33.723 DEBUG:boto:Token: None 2022-09-03T05:54:33.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/ 2022-09-03T05:54:33.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MYUEv+3NJuP/G8ETkZJFIjdfuB8= 2022-09-03T05:54:33.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MYUEv+3NJuP/G8ETkZJFIjdfuB8=', 'Content-Length': '0'} 2022-09-03T05:54:33.727 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d67ac6c65338755e-006312ec19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.728 DEBUG:boto:b'avysmw-15myobj1000falsemyobj2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.729 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.729 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.730 DEBUG:boto:path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:33.730 DEBUG:boto:auth_path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:33.731 DEBUG:boto:Method: GET 2022-09-03T05:54:33.731 DEBUG:boto:Path: /avysmw-15/?prefix=myobj 2022-09-03T05:54:33.732 DEBUG:boto:Data: 2022-09-03T05:54:33.733 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.734 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.735 DEBUG:boto:Params: {} 2022-09-03T05:54:33.736 DEBUG:boto:Token: None 2022-09-03T05:54:33.737 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/ 2022-09-03T05:54:33.738 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MYUEv+3NJuP/G8ETkZJFIjdfuB8= 2022-09-03T05:54:33.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MYUEv+3NJuP/G8ETkZJFIjdfuB8=', 'Content-Length': '0'} 2022-09-03T05:54:33.742 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d6c284ed9e4361ed-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.743 DEBUG:boto:b'avysmw-15myobj1000falsemyobj2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.744 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-15 zones={test-zone1, test-zone3} 2022-09-03T05:54:33.744 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:33.745 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:33.746 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-15 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:33.810 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:33.810 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:33.849 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:33.847+0000 7faf5d35f600 1 bucket sync caught up with source: 2022-09-03T05:54:33.850 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.22.5, , , , , , , , , ] 2022-09-03T05:54:33.850 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.22.5, , , , , , , , , ] 2022-09-03T05:54:33.851 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:33.847+0000 7faf5d35f600 0 bucket checkpoint complete 2022-09-03T05:54:33.865 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-15 zones={test-zone2, test-zone3} 2022-09-03T05:54:33.865 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.866 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.867 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.868 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:54:33.868 DEBUG:boto:Data: 2022-09-03T05:54:33.869 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.870 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.870 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.871 DEBUG:boto:Params: {} 2022-09-03T05:54:33.871 DEBUG:boto:Token: None 2022-09-03T05:54:33.872 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/ 2022-09-03T05:54:33.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qCk1wme83vx+HRzjyPQwOhLOR+0= 2022-09-03T05:54:33.874 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qCk1wme83vx+HRzjyPQwOhLOR+0=', 'Content-Length': '0'} 2022-09-03T05:54:33.877 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000050dc465497aaeb76-006312ec19-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.878 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.879 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.879 DEBUG:boto:Method: HEAD 2022-09-03T05:54:33.880 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:54:33.880 DEBUG:boto:Data: 2022-09-03T05:54:33.881 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.883 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.884 DEBUG:boto:Params: {} 2022-09-03T05:54:33.885 DEBUG:boto:Token: None 2022-09-03T05:54:33.886 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/ 2022-09-03T05:54:33.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qCk1wme83vx+HRzjyPQwOhLOR+0= 2022-09-03T05:54:33.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qCk1wme83vx+HRzjyPQwOhLOR+0=', 'Content-Length': '0'} 2022-09-03T05:54:33.889 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007bd53f80eb25258d-006312ec19-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.890 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:54:33.891 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.891 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.892 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.892 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.893 DEBUG:boto:Method: GET 2022-09-03T05:54:33.893 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.894 DEBUG:boto:Data: 2022-09-03T05:54:33.895 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.895 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.896 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.896 DEBUG:boto:Params: {} 2022-09-03T05:54:33.897 DEBUG:boto:Token: None 2022-09-03T05:54:33.897 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/?versions 2022-09-03T05:54:33.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE= 2022-09-03T05:54:33.900 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE=', 'Content-Length': '0'} 2022-09-03T05:54:33.903 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000360bedacba12f5be-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.903 DEBUG:boto:b'avysmw-15999falsemyobjnulltrue2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.904 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:33.905 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:54:33.905 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.906 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.907 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.907 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.908 DEBUG:boto:Method: GET 2022-09-03T05:54:33.908 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.909 DEBUG:boto:Data: 2022-09-03T05:54:33.909 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.910 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.911 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.911 DEBUG:boto:Params: {} 2022-09-03T05:54:33.912 DEBUG:boto:Token: None 2022-09-03T05:54:33.913 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/?versions 2022-09-03T05:54:33.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE= 2022-09-03T05:54:33.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE=', 'Content-Length': '0'} 2022-09-03T05:54:33.916 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044ba5926b97dc5b8-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.917 DEBUG:boto:b'avysmw-15999falsemyobjnulltrue2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.918 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2022-09-03T05:54:33.919 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.919 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.920 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.921 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.921 DEBUG:boto:Method: GET 2022-09-03T05:54:33.922 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.923 DEBUG:boto:Data: 2022-09-03T05:54:33.923 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.924 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.924 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.925 DEBUG:boto:Params: {} 2022-09-03T05:54:33.926 DEBUG:boto:Token: None 2022-09-03T05:54:33.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/?versions 2022-09-03T05:54:33.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE= 2022-09-03T05:54:33.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE=', 'Content-Length': '0'} 2022-09-03T05:54:33.931 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000524d04fd33d2c67-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.932 DEBUG:boto:b'avysmw-15999falsemyobjnulltrue2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.933 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:33.934 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:33.935 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.935 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.936 DEBUG:boto:Method: GET 2022-09-03T05:54:33.936 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:54:33.937 DEBUG:boto:Data: 2022-09-03T05:54:33.937 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.938 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:33.939 DEBUG:boto:Port: 8001 2022-09-03T05:54:33.939 DEBUG:boto:Params: {} 2022-09-03T05:54:33.940 DEBUG:boto:Token: None 2022-09-03T05:54:33.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/?versions 2022-09-03T05:54:33.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE= 2022-09-03T05:54:33.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPCTkpaNYBEchTkouTc4DgZ/SbE=', 'Content-Length': '0'} 2022-09-03T05:54:33.946 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c731af54811848bc-006312ec19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:33.947 DEBUG:boto:b'avysmw-15999falsemyobjnulltrue2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:33.948 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:33.949 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:33.949 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:33.950 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.951 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.951 DEBUG:boto:Method: GET 2022-09-03T05:54:33.952 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.952 DEBUG:boto:Data: 2022-09-03T05:54:33.953 DEBUG:boto:Headers: {} 2022-09-03T05:54:33.953 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:33.954 DEBUG:boto:Port: 8000 2022-09-03T05:54:33.955 DEBUG:boto:Params: {} 2022-09-03T05:54:33.955 DEBUG:boto:Token: None 2022-09-03T05:54:33.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:33 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:33.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU= 2022-09-03T05:54:33.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YLQ+2KXNnbP1AATbQfkZ8FRJGtU=', 'Content-Length': '0'} 2022-09-03T05:54:33.959 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000543c124416b770a7-006312ec19-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.002 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:34.004 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:34.005 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.006 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.007 DEBUG:boto:Method: GET 2022-09-03T05:54:34.008 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.009 DEBUG:boto:Data: 2022-09-03T05:54:34.010 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:34.012 DEBUG:boto:Port: 8001 2022-09-03T05:54:34.013 DEBUG:boto:Params: {} 2022-09-03T05:54:34.014 DEBUG:boto:Token: None 2022-09-03T05:54:34.015 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.016 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cb+xvAk9QxMbAgBLK8T4DmUH0A0= 2022-09-03T05:54:34.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cb+xvAk9QxMbAgBLK8T4DmUH0A0=', 'Content-Length': '0'} 2022-09-03T05:54:34.019 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001caa53b9f214c0d1-006312ec1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.020 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:34.020 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:34.021 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.021 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.022 DEBUG:boto:Method: HEAD 2022-09-03T05:54:34.023 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.024 DEBUG:boto:Data: 2022-09-03T05:54:34.024 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.025 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:34.025 DEBUG:boto:Port: 8000 2022-09-03T05:54:34.027 DEBUG:boto:Params: {} 2022-09-03T05:54:34.027 DEBUG:boto:Token: None 2022-09-03T05:54:34.028 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ML+kkevz5+k++dw7rO6EbB0CAjA= 2022-09-03T05:54:34.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ML+kkevz5+k++dw7rO6EbB0CAjA=', 'Content-Length': '0'} 2022-09-03T05:54:34.031 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009f1d67220a48e63a-006312ec1a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.032 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:34.033 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:34.033 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.034 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.034 DEBUG:boto:Method: HEAD 2022-09-03T05:54:34.035 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.035 DEBUG:boto:Data: 2022-09-03T05:54:34.036 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:34.037 DEBUG:boto:Port: 8001 2022-09-03T05:54:34.038 DEBUG:boto:Params: {} 2022-09-03T05:54:34.038 DEBUG:boto:Token: None 2022-09-03T05:54:34.039 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ML+kkevz5+k++dw7rO6EbB0CAjA= 2022-09-03T05:54:34.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ML+kkevz5+k++dw7rO6EbB0CAjA=', 'Content-Length': '0'} 2022-09-03T05:54:34.042 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000096dcab0f8754c744-006312ec1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.043 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:34.044 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:34.044 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:34.045 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.045 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.046 DEBUG:boto:Method: GET 2022-09-03T05:54:34.047 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.047 DEBUG:boto:Data: 2022-09-03T05:54:34.048 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.048 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:34.049 DEBUG:boto:Port: 8000 2022-09-03T05:54:34.049 DEBUG:boto:Params: {} 2022-09-03T05:54:34.050 DEBUG:boto:Token: None 2022-09-03T05:54:34.051 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cb+xvAk9QxMbAgBLK8T4DmUH0A0= 2022-09-03T05:54:34.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cb+xvAk9QxMbAgBLK8T4DmUH0A0=', 'Content-Length': '0'} 2022-09-03T05:54:34.054 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004f8aa7c471ef647c-006312ec1a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.055 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:34.055 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:34.056 DEBUG:boto:path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.056 DEBUG:boto:auth_path=/avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.057 DEBUG:boto:Method: GET 2022-09-03T05:54:34.057 DEBUG:boto:Path: /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.058 DEBUG:boto:Data: 2022-09-03T05:54:34.058 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:34.060 DEBUG:boto:Port: 8001 2022-09-03T05:54:34.060 DEBUG:boto:Params: {} 2022-09-03T05:54:34.061 DEBUG:boto:Token: None 2022-09-03T05:54:34.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/myobj?versionId=null 2022-09-03T05:54:34.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cb+xvAk9QxMbAgBLK8T4DmUH0A0= 2022-09-03T05:54:34.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cb+xvAk9QxMbAgBLK8T4DmUH0A0=', 'Content-Length': '0'} 2022-09-03T05:54:34.065 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002e0afc17ee2dd976-006312ec1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.110 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:34.111 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:34.111 DEBUG:boto:path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:34.112 DEBUG:boto:auth_path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:34.113 DEBUG:boto:Method: GET 2022-09-03T05:54:34.113 DEBUG:boto:Path: /avysmw-15/?prefix=myobj 2022-09-03T05:54:34.114 DEBUG:boto:Data: 2022-09-03T05:54:34.115 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.115 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:34.116 DEBUG:boto:Port: 8000 2022-09-03T05:54:34.116 DEBUG:boto:Params: {} 2022-09-03T05:54:34.117 DEBUG:boto:Token: None 2022-09-03T05:54:34.117 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/ 2022-09-03T05:54:34.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24gST+UdYe8/SezaQWSJg5el0U0= 2022-09-03T05:54:34.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24gST+UdYe8/SezaQWSJg5el0U0=', 'Content-Length': '0'} 2022-09-03T05:54:34.121 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000be0789f149c7446b-006312ec1a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.122 DEBUG:boto:b'avysmw-15myobj1000falsemyobj2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:34.123 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:34.124 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:34.124 DEBUG:boto:path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:34.125 DEBUG:boto:auth_path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:34.125 DEBUG:boto:Method: GET 2022-09-03T05:54:34.126 DEBUG:boto:Path: /avysmw-15/?prefix=myobj 2022-09-03T05:54:34.126 DEBUG:boto:Data: 2022-09-03T05:54:34.127 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:34.128 DEBUG:boto:Port: 8001 2022-09-03T05:54:34.129 DEBUG:boto:Params: {} 2022-09-03T05:54:34.130 DEBUG:boto:Token: None 2022-09-03T05:54:34.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/ 2022-09-03T05:54:34.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24gST+UdYe8/SezaQWSJg5el0U0= 2022-09-03T05:54:34.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24gST+UdYe8/SezaQWSJg5el0U0=', 'Content-Length': '0'} 2022-09-03T05:54:34.135 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b1975a23c8e33dc2-006312ec1a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.135 DEBUG:boto:b'avysmw-15myobj1000falsemyobj2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:34.137 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:34.137 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:34.138 DEBUG:boto:Method: HEAD 2022-09-03T05:54:34.138 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:54:34.139 DEBUG:boto:Data: 2022-09-03T05:54:34.139 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.140 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:34.140 DEBUG:boto:Port: 8000 2022-09-03T05:54:34.141 DEBUG:boto:Params: {} 2022-09-03T05:54:34.142 DEBUG:boto:Token: None 2022-09-03T05:54:34.142 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/myobj 2022-09-03T05:54:34.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F/MUWq7aNrUTzaQBfbwu54noZRg= 2022-09-03T05:54:34.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F/MUWq7aNrUTzaQBfbwu54noZRg=', 'Content-Length': '0'} 2022-09-03T05:54:34.146 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000004883ef4079954d4-006312ec1a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.146 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:34.147 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:34.147 DEBUG:boto:Method: HEAD 2022-09-03T05:54:34.148 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:54:34.148 DEBUG:boto:Data: 2022-09-03T05:54:34.149 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.149 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:34.150 DEBUG:boto:Port: 8001 2022-09-03T05:54:34.151 DEBUG:boto:Params: {} 2022-09-03T05:54:34.151 DEBUG:boto:Token: None 2022-09-03T05:54:34.152 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/myobj 2022-09-03T05:54:34.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F/MUWq7aNrUTzaQBfbwu54noZRg= 2022-09-03T05:54:34.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F/MUWq7aNrUTzaQBfbwu54noZRg=', 'Content-Length': '0'} 2022-09-03T05:54:34.155 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d06b69f0b646174a-006312ec1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.157 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2022-09-03T05:54:34.157 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:34.158 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:34.158 DEBUG:boto:Method: GET 2022-09-03T05:54:34.159 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:54:34.159 DEBUG:boto:Data: 2022-09-03T05:54:34.160 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.160 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:34.161 DEBUG:boto:Port: 8000 2022-09-03T05:54:34.162 DEBUG:boto:Params: {} 2022-09-03T05:54:34.162 DEBUG:boto:Token: None 2022-09-03T05:54:34.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/myobj 2022-09-03T05:54:34.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GeZIH/ZwM9+Cnga1guiEI5IxUYk= 2022-09-03T05:54:34.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GeZIH/ZwM9+Cnga1guiEI5IxUYk=', 'Content-Length': '0'} 2022-09-03T05:54:34.167 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000084086e2e7ad3f7ea-006312ec1a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.168 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:54:34.168 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:54:34.169 DEBUG:boto:Method: GET 2022-09-03T05:54:34.170 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:54:34.170 DEBUG:boto:Data: 2022-09-03T05:54:34.171 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:34.172 DEBUG:boto:Port: 8001 2022-09-03T05:54:34.173 DEBUG:boto:Params: {} 2022-09-03T05:54:34.173 DEBUG:boto:Token: None 2022-09-03T05:54:34.174 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/myobj 2022-09-03T05:54:34.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GeZIH/ZwM9+Cnga1guiEI5IxUYk= 2022-09-03T05:54:34.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GeZIH/ZwM9+Cnga1guiEI5IxUYk=', 'Content-Length': '0'} 2022-09-03T05:54:34.177 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f388c5c362c000f3-006312ec1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.179 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:34.180 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:34.181 DEBUG:boto:path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:34.182 DEBUG:boto:auth_path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:34.182 DEBUG:boto:Method: GET 2022-09-03T05:54:34.183 DEBUG:boto:Path: /avysmw-15/?prefix=myobj 2022-09-03T05:54:34.184 DEBUG:boto:Data: 2022-09-03T05:54:34.185 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.186 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:54:34.186 DEBUG:boto:Port: 8000 2022-09-03T05:54:34.187 DEBUG:boto:Params: {} 2022-09-03T05:54:34.188 DEBUG:boto:Token: None 2022-09-03T05:54:34.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/ 2022-09-03T05:54:34.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24gST+UdYe8/SezaQWSJg5el0U0= 2022-09-03T05:54:34.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24gST+UdYe8/SezaQWSJg5el0U0=', 'Content-Length': '0'} 2022-09-03T05:54:34.192 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b07c3c9596ecc5ee-006312ec1a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.193 DEBUG:boto:b'avysmw-15myobj1000falsemyobj2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:34.195 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:54:34.196 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:54:34.197 DEBUG:boto:path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:34.198 DEBUG:boto:auth_path=/avysmw-15/?prefix=myobj 2022-09-03T05:54:34.199 DEBUG:boto:Method: GET 2022-09-03T05:54:34.199 DEBUG:boto:Path: /avysmw-15/?prefix=myobj 2022-09-03T05:54:34.200 DEBUG:boto:Data: 2022-09-03T05:54:34.200 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:54:34.202 DEBUG:boto:Port: 8001 2022-09-03T05:54:34.202 DEBUG:boto:Params: {} 2022-09-03T05:54:34.203 DEBUG:boto:Token: None 2022-09-03T05:54:34.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:54:34 GMT /avysmw-15/ 2022-09-03T05:54:34.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24gST+UdYe8/SezaQWSJg5el0U0= 2022-09-03T05:54:34.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24gST+UdYe8/SezaQWSJg5el0U0=', 'Content-Length': '0'} 2022-09-03T05:54:34.207 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007bfd75508a4de231-006312ec1a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.208 DEBUG:boto:b'avysmw-15myobj1000falsemyobj2022-09-03T05:54:10.549Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:54:34.209 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-15 zones={test-zone2, test-zone3} 2022-09-03T05:54:34.210 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:54:34.210 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:54:34.211 DEBUG:boto:Method: HEAD 2022-09-03T05:54:34.211 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:54:34.212 DEBUG:boto:Data: 2022-09-03T05:54:34.212 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.213 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:34.214 DEBUG:boto:Port: 8000 2022-09-03T05:54:34.214 DEBUG:boto:Params: {} 2022-09-03T05:54:34.215 DEBUG:boto:Token: None 2022-09-03T05:54:34.215 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:34 GMT /avysmw-13/ 2022-09-03T05:54:34.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:twpDIHmRR3JBiTUL8q7uqhm1s08= 2022-09-03T05:54:34.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:twpDIHmRR3JBiTUL8q7uqhm1s08=', 'Content-Length': '0'} 2022-09-03T05:54:34.219 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000078dfa28e60d5aceb-006312ec1a-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.220 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:34.221 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:34.221 DEBUG:boto:Method: HEAD 2022-09-03T05:54:34.222 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:34.222 DEBUG:boto:Data: 2022-09-03T05:54:34.223 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.223 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:34.224 DEBUG:boto:Port: 8000 2022-09-03T05:54:34.225 DEBUG:boto:Params: {} 2022-09-03T05:54:34.225 DEBUG:boto:Token: None 2022-09-03T05:54:34.226 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:54:34 GMT /avysmw-13/myobj 2022-09-03T05:54:34.227 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rROv/ycVl83oaT6wk80f2ImuEu8= 2022-09-03T05:54:34.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rROv/ycVl83oaT6wk80f2ImuEu8=', 'Content-Length': '0'} 2022-09-03T05:54:34.230 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a8c09909d5fed387-006312ec1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.232 DEBUG:boto:path=/avysmw-13/myobj 2022-09-03T05:54:34.233 DEBUG:boto:auth_path=/avysmw-13/myobj 2022-09-03T05:54:34.234 DEBUG:boto:Method: DELETE 2022-09-03T05:54:34.235 DEBUG:boto:Path: /avysmw-13/myobj 2022-09-03T05:54:34.235 DEBUG:boto:Data: 2022-09-03T05:54:34.236 DEBUG:boto:Headers: {} 2022-09-03T05:54:34.236 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:54:34.237 DEBUG:boto:Port: 8000 2022-09-03T05:54:34.237 DEBUG:boto:Params: {} 2022-09-03T05:54:34.239 DEBUG:boto:Token: None 2022-09-03T05:54:34.240 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:54:34 GMT /avysmw-13/myobj 2022-09-03T05:54:34.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kg26gnTsHMBDazCbS/x0Y9QRHLQ= 2022-09-03T05:54:34.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:54:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kg26gnTsHMBDazCbS/x0Y9QRHLQ=', 'Content-Length': '0'} 2022-09-03T05:54:34.246 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000101c56a9a88a1bab-006312ec1a-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:54:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:54:34.247 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:34.248 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:54:34.249 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:54:34.306 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:54:34.307 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:54:34.354 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:34.352+0000 7f95c7000600 1 waiting for incremental sync to catch up: 2022-09-03T05:54:34.354 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:34.355 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:54:39.355 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:39.353+0000 7f95c7000600 1 waiting for incremental sync to catch up: 2022-09-03T05:54:39.356 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:39.356 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:54:44.357 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:44.355+0000 7f95c7000600 1 waiting for incremental sync to catch up: 2022-09-03T05:54:44.357 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:44.358 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:54:49.358 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:49.356+0000 7f95c7000600 1 waiting for incremental sync to catch up: 2022-09-03T05:54:49.359 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:49.360 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:54:54.360 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:54.358+0000 7f95c7000600 1 waiting for incremental sync to catch up: 2022-09-03T05:54:54.360 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:54.361 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:54:59.361 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:54:59.359+0000 7f95c7000600 1 waiting for incremental sync to catch up: 2022-09-03T05:54:59.362 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:54:59.362 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:55:04.362 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:04.360+0000 7f95c7000600 1 waiting for incremental sync to catch up: 2022-09-03T05:55:04.363 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:55:04.363 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:55:09.364 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:09.362+0000 7f95c7000600 1 waiting for incremental sync to catch up: 2022-09-03T05:55:09.365 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.8.5, , , , , , , , , ] 2022-09-03T05:55:09.366 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:55:14.365 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:14.363+0000 7f95c7000600 1 bucket sync caught up with source: 2022-09-03T05:55:14.366 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:55:14.366 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:55:14.366 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:14.363+0000 7f95c7000600 0 bucket checkpoint complete 2022-09-03T05:55:14.385 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-13 zones={test-zone2, test-zone1} 2022-09-03T05:55:14.387 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.388 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.389 DEBUG:boto:Method: HEAD 2022-09-03T05:55:14.390 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:55:14.390 DEBUG:boto:Data: 2022-09-03T05:55:14.391 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.392 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:14.392 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.393 DEBUG:boto:Params: {} 2022-09-03T05:55:14.394 DEBUG:boto:Token: None 2022-09-03T05:55:14.395 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/ 2022-09-03T05:55:14.396 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:orjW73leTtwFrKTUYrtxce4K8H4= 2022-09-03T05:55:14.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:orjW73leTtwFrKTUYrtxce4K8H4=', 'Content-Length': '0'} 2022-09-03T05:55:14.402 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001902f3d9bda36652-006312ec42-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.403 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.404 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.405 DEBUG:boto:Method: HEAD 2022-09-03T05:55:14.406 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:55:14.407 DEBUG:boto:Data: 2022-09-03T05:55:14.407 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.408 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:14.409 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.410 DEBUG:boto:Params: {} 2022-09-03T05:55:14.411 DEBUG:boto:Token: None 2022-09-03T05:55:14.412 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/ 2022-09-03T05:55:14.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:orjW73leTtwFrKTUYrtxce4K8H4= 2022-09-03T05:55:14.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:orjW73leTtwFrKTUYrtxce4K8H4=', 'Content-Length': '0'} 2022-09-03T05:55:14.418 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c9fe77911ac12298-006312ec42-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.418 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:55:14.419 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.419 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.420 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.421 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.421 DEBUG:boto:Method: GET 2022-09-03T05:55:14.422 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.422 DEBUG:boto:Data: 2022-09-03T05:55:14.423 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.423 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:14.424 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.424 DEBUG:boto:Params: {} 2022-09-03T05:55:14.425 DEBUG:boto:Token: None 2022-09-03T05:55:14.426 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/?versions 2022-09-03T05:55:14.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY= 2022-09-03T05:55:14.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY=', 'Content-Length': '0'} 2022-09-03T05:55:14.429 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009686eace0d29af7d-006312ec42-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.430 DEBUG:boto:b'avysmw-13999false' 2022-09-03T05:55:14.431 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:55:14.432 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.432 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.433 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.433 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.434 DEBUG:boto:Method: GET 2022-09-03T05:55:14.435 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.435 DEBUG:boto:Data: 2022-09-03T05:55:14.436 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.436 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:14.437 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.437 DEBUG:boto:Params: {} 2022-09-03T05:55:14.438 DEBUG:boto:Token: None 2022-09-03T05:55:14.439 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/?versions 2022-09-03T05:55:14.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY= 2022-09-03T05:55:14.440 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY=', 'Content-Length': '0'} 2022-09-03T05:55:14.443 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1f20d574ccf6c1a-006312ec42-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.443 DEBUG:boto:b'avysmw-13999false' 2022-09-03T05:55:14.444 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.445 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.445 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.446 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.446 DEBUG:boto:Method: GET 2022-09-03T05:55:14.447 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.448 DEBUG:boto:Data: 2022-09-03T05:55:14.448 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.449 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:14.449 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.450 DEBUG:boto:Params: {} 2022-09-03T05:55:14.450 DEBUG:boto:Token: None 2022-09-03T05:55:14.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/?versions 2022-09-03T05:55:14.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY= 2022-09-03T05:55:14.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY=', 'Content-Length': '0'} 2022-09-03T05:55:14.455 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b040888978889639-006312ec42-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.456 DEBUG:boto:b'avysmw-13999false' 2022-09-03T05:55:14.458 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.459 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.460 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.460 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.461 DEBUG:boto:Method: GET 2022-09-03T05:55:14.462 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.462 DEBUG:boto:Data: 2022-09-03T05:55:14.463 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:14.464 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.464 DEBUG:boto:Params: {} 2022-09-03T05:55:14.465 DEBUG:boto:Token: None 2022-09-03T05:55:14.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/?versions 2022-09-03T05:55:14.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY= 2022-09-03T05:55:14.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY=', 'Content-Length': '0'} 2022-09-03T05:55:14.469 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f6df1f9d09b049be-006312ec42-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.470 DEBUG:boto:b'avysmw-13999false' 2022-09-03T05:55:14.471 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-13 zones={test-zone2, test-zone1} 2022-09-03T05:55:14.471 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:14.472 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:14.473 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:55:14.539 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:14.540 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:14.604 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:14.602+0000 7f96946bf600 1 bucket sync caught up with source: 2022-09-03T05:55:14.605 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:55:14.605 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:55:14.606 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:14.602+0000 7f96946bf600 0 bucket checkpoint complete 2022-09-03T05:55:14.632 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-13 zones={test-zone3, test-zone1} 2022-09-03T05:55:14.632 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.633 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.634 DEBUG:boto:Method: HEAD 2022-09-03T05:55:14.634 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:55:14.635 DEBUG:boto:Data: 2022-09-03T05:55:14.635 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:14.636 DEBUG:boto:Port: 8001 2022-09-03T05:55:14.637 DEBUG:boto:Params: {} 2022-09-03T05:55:14.638 DEBUG:boto:Token: None 2022-09-03T05:55:14.639 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/ 2022-09-03T05:55:14.639 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:orjW73leTtwFrKTUYrtxce4K8H4= 2022-09-03T05:55:14.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:orjW73leTtwFrKTUYrtxce4K8H4=', 'Content-Length': '0'} 2022-09-03T05:55:14.642 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f9427ae0121bf68f-006312ec42-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.644 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.645 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.646 DEBUG:boto:Method: HEAD 2022-09-03T05:55:14.647 DEBUG:boto:Path: /avysmw-13/ 2022-09-03T05:55:14.647 DEBUG:boto:Data: 2022-09-03T05:55:14.648 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.648 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:14.649 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.649 DEBUG:boto:Params: {} 2022-09-03T05:55:14.650 DEBUG:boto:Token: None 2022-09-03T05:55:14.651 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/ 2022-09-03T05:55:14.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:orjW73leTtwFrKTUYrtxce4K8H4= 2022-09-03T05:55:14.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:orjW73leTtwFrKTUYrtxce4K8H4=', 'Content-Length': '0'} 2022-09-03T05:55:14.655 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001785dad8cbaea781-006312ec42-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.656 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:55:14.656 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.657 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.657 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.658 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.659 DEBUG:boto:Method: GET 2022-09-03T05:55:14.659 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.660 DEBUG:boto:Data: 2022-09-03T05:55:14.660 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:14.662 DEBUG:boto:Port: 8001 2022-09-03T05:55:14.662 DEBUG:boto:Params: {} 2022-09-03T05:55:14.663 DEBUG:boto:Token: None 2022-09-03T05:55:14.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/?versions 2022-09-03T05:55:14.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY= 2022-09-03T05:55:14.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY=', 'Content-Length': '0'} 2022-09-03T05:55:14.668 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e0d7beaf6506996-006312ec42-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.710 DEBUG:boto:b'avysmw-13999false' 2022-09-03T05:55:14.711 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:55:14.712 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.712 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.713 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.714 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.714 DEBUG:boto:Method: GET 2022-09-03T05:55:14.715 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.716 DEBUG:boto:Data: 2022-09-03T05:55:14.716 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:14.718 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.718 DEBUG:boto:Params: {} 2022-09-03T05:55:14.720 DEBUG:boto:Token: None 2022-09-03T05:55:14.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/?versions 2022-09-03T05:55:14.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY= 2022-09-03T05:55:14.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY=', 'Content-Length': '0'} 2022-09-03T05:55:14.724 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000047bd673ba89499e1-006312ec42-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.726 DEBUG:boto:b'avysmw-13999false' 2022-09-03T05:55:14.731 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.732 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.733 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.734 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.736 DEBUG:boto:Method: GET 2022-09-03T05:55:14.737 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.737 DEBUG:boto:Data: 2022-09-03T05:55:14.738 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:14.739 DEBUG:boto:Port: 8001 2022-09-03T05:55:14.740 DEBUG:boto:Params: {} 2022-09-03T05:55:14.741 DEBUG:boto:Token: None 2022-09-03T05:55:14.742 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/?versions 2022-09-03T05:55:14.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY= 2022-09-03T05:55:14.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY=', 'Content-Length': '0'} 2022-09-03T05:55:14.746 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000066bd29297ebfd41f-006312ec42-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.747 DEBUG:boto:b'avysmw-13999false' 2022-09-03T05:55:14.748 DEBUG:boto:path=/avysmw-13/ 2022-09-03T05:55:14.749 DEBUG:boto:auth_path=/avysmw-13/ 2022-09-03T05:55:14.750 DEBUG:boto:path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.750 DEBUG:boto:auth_path=/avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.751 DEBUG:boto:Method: GET 2022-09-03T05:55:14.751 DEBUG:boto:Path: /avysmw-13/?versions&max-keys=999 2022-09-03T05:55:14.752 DEBUG:boto:Data: 2022-09-03T05:55:14.752 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.753 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:14.754 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.755 DEBUG:boto:Params: {} 2022-09-03T05:55:14.756 DEBUG:boto:Token: None 2022-09-03T05:55:14.758 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:14 GMT /avysmw-13/?versions 2022-09-03T05:55:14.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY= 2022-09-03T05:55:14.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KoLlm1CS4dVxDOmmOusnJMqCdkY=', 'Content-Length': '0'} 2022-09-03T05:55:14.762 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e9cf982250df21ca-006312ec42-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.763 DEBUG:boto:b'avysmw-13999false' 2022-09-03T05:55:14.764 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-13 zones={test-zone3, test-zone1} 2022-09-03T05:55:14.764 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:14.765 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:14.766 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:55:14.830 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:14.830 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:14.872 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:14.870+0000 7fa268a0d600 1 bucket sync caught up with source: 2022-09-03T05:55:14.873 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:55:14.873 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.10.5, , , , , , , , , ] 2022-09-03T05:55:14.873 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:14.870+0000 7fa268a0d600 0 bucket checkpoint complete 2022-09-03T05:55:14.889 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:14.890 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:14.891 DEBUG:boto:Method: HEAD 2022-09-03T05:55:14.892 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:55:14.893 DEBUG:boto:Data: 2022-09-03T05:55:14.894 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.895 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:14.897 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.898 DEBUG:boto:Params: {} 2022-09-03T05:55:14.899 DEBUG:boto:Token: None 2022-09-03T05:55:14.900 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:14 GMT /avysmw-14/ 2022-09-03T05:55:14.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mG8l/OlCIVK65T/U4d7hHwRxK58= 2022-09-03T05:55:14.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mG8l/OlCIVK65T/U4d7hHwRxK58=', 'Content-Length': '0'} 2022-09-03T05:55:14.906 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000972596c3ed06473c-006312ec42-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.908 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:55:14.909 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:55:14.910 DEBUG:boto:Method: HEAD 2022-09-03T05:55:14.911 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:55:14.911 DEBUG:boto:Data: 2022-09-03T05:55:14.912 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.913 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:14.914 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.914 DEBUG:boto:Params: {} 2022-09-03T05:55:14.915 DEBUG:boto:Token: None 2022-09-03T05:55:14.915 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:14 GMT /avysmw-14/myobj 2022-09-03T05:55:14.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9TXgb0OU0JNVZsUc83Rt1YPWO8c= 2022-09-03T05:55:14.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9TXgb0OU0JNVZsUc83Rt1YPWO8c=', 'Content-Length': '0'} 2022-09-03T05:55:14.919 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000063cc2d465431553-006312ec42-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.920 DEBUG:boto:path=/avysmw-14/myobj 2022-09-03T05:55:14.921 DEBUG:boto:auth_path=/avysmw-14/myobj 2022-09-03T05:55:14.921 DEBUG:boto:Method: DELETE 2022-09-03T05:55:14.922 DEBUG:boto:Path: /avysmw-14/myobj 2022-09-03T05:55:14.922 DEBUG:boto:Data: 2022-09-03T05:55:14.923 DEBUG:boto:Headers: {} 2022-09-03T05:55:14.923 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:14.924 DEBUG:boto:Port: 8000 2022-09-03T05:55:14.924 DEBUG:boto:Params: {} 2022-09-03T05:55:14.925 DEBUG:boto:Token: None 2022-09-03T05:55:14.925 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:55:14 GMT /avysmw-14/myobj 2022-09-03T05:55:14.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZlxX2KdjmOzPKur3BLnxUcCIMaM= 2022-09-03T05:55:14.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZlxX2KdjmOzPKur3BLnxUcCIMaM=', 'Content-Length': '0'} 2022-09-03T05:55:14.933 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000009ac73d0531d3de65-006312ec42-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:14.934 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:14.935 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:14.936 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-14 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:55:14.995 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:14.996 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:15.058 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:15.056+0000 7f85223a3600 1 waiting for incremental sync to catch up: 2022-09-03T05:55:15.059 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.20.5, , , , , , , , , ] 2022-09-03T05:55:15.059 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.22.5, , , , , , , , , ] 2022-09-03T05:55:20.059 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:20.057+0000 7f85223a3600 1 waiting for incremental sync to catch up: 2022-09-03T05:55:20.060 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.20.5, , , , , , , , , ] 2022-09-03T05:55:20.060 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.22.5, , , , , , , , , ] 2022-09-03T05:55:25.061 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:25.059+0000 7f85223a3600 1 waiting for incremental sync to catch up: 2022-09-03T05:55:25.061 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.20.5, , , , , , , , , ] 2022-09-03T05:55:25.062 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.22.5, , , , , , , , , ] 2022-09-03T05:55:30.062 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:30.061+0000 7f85223a3600 1 waiting for incremental sync to catch up: 2022-09-03T05:55:30.063 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.20.5, , , , , , , , , ] 2022-09-03T05:55:30.063 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.22.5, , , , , , , , , ] 2022-09-03T05:55:35.066 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:35.062+0000 7f85223a3600 1 bucket sync caught up with source: 2022-09-03T05:55:35.067 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.22.5, , , , , , , , , ] 2022-09-03T05:55:35.067 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.22.5, , , , , , , , , ] 2022-09-03T05:55:35.068 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:35.062+0000 7f85223a3600 0 bucket checkpoint complete 2022-09-03T05:55:35.085 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-14 zones={test-zone1, test-zone2} 2022-09-03T05:55:35.086 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.087 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.088 DEBUG:boto:Method: HEAD 2022-09-03T05:55:35.089 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:55:35.090 DEBUG:boto:Data: 2022-09-03T05:55:35.091 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:35.092 DEBUG:boto:Port: 8000 2022-09-03T05:55:35.093 DEBUG:boto:Params: {} 2022-09-03T05:55:35.094 DEBUG:boto:Token: None 2022-09-03T05:55:35.096 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/ 2022-09-03T05:55:35.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WognXapJkPL6SWW5dnIMHKxsY98= 2022-09-03T05:55:35.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WognXapJkPL6SWW5dnIMHKxsY98=', 'Content-Length': '0'} 2022-09-03T05:55:35.102 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000376ae335372aca1b-006312ec57-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.103 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.104 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.105 DEBUG:boto:Method: HEAD 2022-09-03T05:55:35.106 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:55:35.107 DEBUG:boto:Data: 2022-09-03T05:55:35.110 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.111 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:35.112 DEBUG:boto:Port: 8000 2022-09-03T05:55:35.113 DEBUG:boto:Params: {} 2022-09-03T05:55:35.114 DEBUG:boto:Token: None 2022-09-03T05:55:35.115 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/ 2022-09-03T05:55:35.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WognXapJkPL6SWW5dnIMHKxsY98= 2022-09-03T05:55:35.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WognXapJkPL6SWW5dnIMHKxsY98=', 'Content-Length': '0'} 2022-09-03T05:55:35.121 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009f7b1aee18604392-006312ec57-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.122 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:55:35.123 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.123 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.124 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.124 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.125 DEBUG:boto:Method: GET 2022-09-03T05:55:35.125 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.126 DEBUG:boto:Data: 2022-09-03T05:55:35.127 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.127 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:35.128 DEBUG:boto:Port: 8000 2022-09-03T05:55:35.128 DEBUG:boto:Params: {} 2022-09-03T05:55:35.129 DEBUG:boto:Token: None 2022-09-03T05:55:35.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/?versions 2022-09-03T05:55:35.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA= 2022-09-03T05:55:35.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA=', 'Content-Length': '0'} 2022-09-03T05:55:35.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a7571cadf2f7b1f-006312ec57-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.135 DEBUG:boto:b'avysmw-14999false' 2022-09-03T05:55:35.137 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:55:35.138 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.139 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.140 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.141 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.141 DEBUG:boto:Method: GET 2022-09-03T05:55:35.142 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.143 DEBUG:boto:Data: 2022-09-03T05:55:35.144 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.145 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:35.146 DEBUG:boto:Port: 8000 2022-09-03T05:55:35.147 DEBUG:boto:Params: {} 2022-09-03T05:55:35.147 DEBUG:boto:Token: None 2022-09-03T05:55:35.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/?versions 2022-09-03T05:55:35.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA= 2022-09-03T05:55:35.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA=', 'Content-Length': '0'} 2022-09-03T05:55:35.152 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000051d5d2f592c3f600-006312ec57-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.154 DEBUG:boto:b'avysmw-14999false' 2022-09-03T05:55:35.155 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.156 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.157 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.158 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.159 DEBUG:boto:Method: GET 2022-09-03T05:55:35.160 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.161 DEBUG:boto:Data: 2022-09-03T05:55:35.162 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:35.164 DEBUG:boto:Port: 8000 2022-09-03T05:55:35.165 DEBUG:boto:Params: {} 2022-09-03T05:55:35.166 DEBUG:boto:Token: None 2022-09-03T05:55:35.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/?versions 2022-09-03T05:55:35.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA= 2022-09-03T05:55:35.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA=', 'Content-Length': '0'} 2022-09-03T05:55:35.173 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e0bbc6d90d1d6cd-006312ec57-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.174 DEBUG:boto:b'avysmw-14999false' 2022-09-03T05:55:35.175 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.176 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.177 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.178 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.179 DEBUG:boto:Method: GET 2022-09-03T05:55:35.180 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.181 DEBUG:boto:Data: 2022-09-03T05:55:35.182 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.183 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:35.183 DEBUG:boto:Port: 8000 2022-09-03T05:55:35.184 DEBUG:boto:Params: {} 2022-09-03T05:55:35.185 DEBUG:boto:Token: None 2022-09-03T05:55:35.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/?versions 2022-09-03T05:55:35.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA= 2022-09-03T05:55:35.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA=', 'Content-Length': '0'} 2022-09-03T05:55:35.193 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e2637cc259adbb9-006312ec57-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.194 DEBUG:boto:b'avysmw-14999false' 2022-09-03T05:55:35.194 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-14 zones={test-zone1, test-zone2} 2022-09-03T05:55:35.195 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:35.196 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:35.197 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-14 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:55:35.262 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:35.263 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:35.315 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:35.313+0000 7fa38052e600 1 bucket sync caught up with source: 2022-09-03T05:55:35.315 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.22.5, , , , , , , , , ] 2022-09-03T05:55:35.316 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.22.5, , , , , , , , , ] 2022-09-03T05:55:35.316 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:35.313+0000 7fa38052e600 0 bucket checkpoint complete 2022-09-03T05:55:35.338 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-14 zones={test-zone3, test-zone2} 2022-09-03T05:55:35.339 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.340 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.341 DEBUG:boto:Method: HEAD 2022-09-03T05:55:35.341 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:55:35.342 DEBUG:boto:Data: 2022-09-03T05:55:35.342 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.343 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:35.343 DEBUG:boto:Port: 8001 2022-09-03T05:55:35.345 DEBUG:boto:Params: {} 2022-09-03T05:55:35.346 DEBUG:boto:Token: None 2022-09-03T05:55:35.347 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/ 2022-09-03T05:55:35.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WognXapJkPL6SWW5dnIMHKxsY98= 2022-09-03T05:55:35.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WognXapJkPL6SWW5dnIMHKxsY98=', 'Content-Length': '0'} 2022-09-03T05:55:35.356 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000076a88f970e35889a-006312ec57-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.357 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.358 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.359 DEBUG:boto:Method: HEAD 2022-09-03T05:55:35.360 DEBUG:boto:Path: /avysmw-14/ 2022-09-03T05:55:35.360 DEBUG:boto:Data: 2022-09-03T05:55:35.361 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.361 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:35.362 DEBUG:boto:Port: 8000 2022-09-03T05:55:35.364 DEBUG:boto:Params: {} 2022-09-03T05:55:35.365 DEBUG:boto:Token: None 2022-09-03T05:55:35.366 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/ 2022-09-03T05:55:35.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WognXapJkPL6SWW5dnIMHKxsY98= 2022-09-03T05:55:35.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WognXapJkPL6SWW5dnIMHKxsY98=', 'Content-Length': '0'} 2022-09-03T05:55:35.370 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000018fb8b64f10b5578-006312ec57-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.371 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:55:35.372 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.373 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.373 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.374 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.374 DEBUG:boto:Method: GET 2022-09-03T05:55:35.375 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.376 DEBUG:boto:Data: 2022-09-03T05:55:35.376 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:35.377 DEBUG:boto:Port: 8001 2022-09-03T05:55:35.378 DEBUG:boto:Params: {} 2022-09-03T05:55:35.378 DEBUG:boto:Token: None 2022-09-03T05:55:35.379 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/?versions 2022-09-03T05:55:35.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA= 2022-09-03T05:55:35.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA=', 'Content-Length': '0'} 2022-09-03T05:55:35.383 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000043a7ac2cdb93d65a-006312ec57-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.384 DEBUG:boto:b'avysmw-14999false' 2022-09-03T05:55:35.385 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:55:35.385 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.386 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.386 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.387 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.387 DEBUG:boto:Method: GET 2022-09-03T05:55:35.388 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.389 DEBUG:boto:Data: 2022-09-03T05:55:35.389 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.390 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:35.390 DEBUG:boto:Port: 8000 2022-09-03T05:55:35.391 DEBUG:boto:Params: {} 2022-09-03T05:55:35.391 DEBUG:boto:Token: None 2022-09-03T05:55:35.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/?versions 2022-09-03T05:55:35.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA= 2022-09-03T05:55:35.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA=', 'Content-Length': '0'} 2022-09-03T05:55:35.396 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000016b31d0e37a52aa0-006312ec57-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.397 DEBUG:boto:b'avysmw-14999false' 2022-09-03T05:55:35.398 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.398 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.399 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.399 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.400 DEBUG:boto:Method: GET 2022-09-03T05:55:35.400 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.401 DEBUG:boto:Data: 2022-09-03T05:55:35.401 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:35.402 DEBUG:boto:Port: 8001 2022-09-03T05:55:35.403 DEBUG:boto:Params: {} 2022-09-03T05:55:35.404 DEBUG:boto:Token: None 2022-09-03T05:55:35.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/?versions 2022-09-03T05:55:35.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA= 2022-09-03T05:55:35.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA=', 'Content-Length': '0'} 2022-09-03T05:55:35.408 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000080c63d7f2ac81357-006312ec57-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.450 DEBUG:boto:b'avysmw-14999false' 2022-09-03T05:55:35.451 DEBUG:boto:path=/avysmw-14/ 2022-09-03T05:55:35.452 DEBUG:boto:auth_path=/avysmw-14/ 2022-09-03T05:55:35.452 DEBUG:boto:path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.453 DEBUG:boto:auth_path=/avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.454 DEBUG:boto:Method: GET 2022-09-03T05:55:35.454 DEBUG:boto:Path: /avysmw-14/?versions&max-keys=999 2022-09-03T05:55:35.455 DEBUG:boto:Data: 2022-09-03T05:55:35.455 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.456 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:35.456 DEBUG:boto:Port: 8000 2022-09-03T05:55:35.457 DEBUG:boto:Params: {} 2022-09-03T05:55:35.457 DEBUG:boto:Token: None 2022-09-03T05:55:35.458 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:35 GMT /avysmw-14/?versions 2022-09-03T05:55:35.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA= 2022-09-03T05:55:35.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azHkMG8ELjj/2SfQAv1FOoG46mA=', 'Content-Length': '0'} 2022-09-03T05:55:35.463 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b949c6204fdc574-006312ec57-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.463 DEBUG:boto:b'avysmw-14999false' 2022-09-03T05:55:35.464 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-14 zones={test-zone3, test-zone2} 2022-09-03T05:55:35.465 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:35.465 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:35.466 DEBUG:boto:Method: HEAD 2022-09-03T05:55:35.466 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:55:35.467 DEBUG:boto:Data: 2022-09-03T05:55:35.467 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.468 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:35.469 DEBUG:boto:Port: 8001 2022-09-03T05:55:35.469 DEBUG:boto:Params: {} 2022-09-03T05:55:35.470 DEBUG:boto:Token: None 2022-09-03T05:55:35.471 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:35 GMT /avysmw-15/ 2022-09-03T05:55:35.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:enF6VI5Qi2HqIgbbLznOznbenAs= 2022-09-03T05:55:35.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:enF6VI5Qi2HqIgbbLznOznbenAs=', 'Content-Length': '0'} 2022-09-03T05:55:35.475 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004baa9c24ad392442-006312ec57-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.476 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:55:35.476 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:55:35.477 DEBUG:boto:Method: HEAD 2022-09-03T05:55:35.477 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:55:35.478 DEBUG:boto:Data: 2022-09-03T05:55:35.478 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:35.479 DEBUG:boto:Port: 8001 2022-09-03T05:55:35.480 DEBUG:boto:Params: {} 2022-09-03T05:55:35.480 DEBUG:boto:Token: None 2022-09-03T05:55:35.481 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:35 GMT /avysmw-15/myobj 2022-09-03T05:55:35.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qhx0y5OJygzbuP/VGPcKrBCXqUk= 2022-09-03T05:55:35.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qhx0y5OJygzbuP/VGPcKrBCXqUk=', 'Content-Length': '0'} 2022-09-03T05:55:35.484 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:54:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009f32f8c01a08d7fc-006312ec57-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.485 DEBUG:boto:path=/avysmw-15/myobj 2022-09-03T05:55:35.486 DEBUG:boto:auth_path=/avysmw-15/myobj 2022-09-03T05:55:35.487 DEBUG:boto:Method: DELETE 2022-09-03T05:55:35.487 DEBUG:boto:Path: /avysmw-15/myobj 2022-09-03T05:55:35.488 DEBUG:boto:Data: 2022-09-03T05:55:35.488 DEBUG:boto:Headers: {} 2022-09-03T05:55:35.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:35.489 DEBUG:boto:Port: 8001 2022-09-03T05:55:35.490 DEBUG:boto:Params: {} 2022-09-03T05:55:35.490 DEBUG:boto:Token: None 2022-09-03T05:55:35.491 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:55:35 GMT /avysmw-15/myobj 2022-09-03T05:55:35.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wla30Ep2i+O25+Qe5EfdA7zS5O0= 2022-09-03T05:55:35.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wla30Ep2i+O25+Qe5EfdA7zS5O0=', 'Content-Length': '0'} 2022-09-03T05:55:35.506 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000079193f5decbb2fa1-006312ec57-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:55:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:35.507 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:35.508 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:35.509 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-15 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:55:35.569 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:35.570 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:35.613 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:35.611+0000 7f3e1fd32600 1 waiting for incremental sync to catch up: 2022-09-03T05:55:35.613 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.22.5, , , , , , , , , ] 2022-09-03T05:55:35.614 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.24.5, , , , , , , , , ] 2022-09-03T05:55:40.615 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:40.613+0000 7f3e1fd32600 1 waiting for incremental sync to catch up: 2022-09-03T05:55:40.615 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.22.5, , , , , , , , , ] 2022-09-03T05:55:40.616 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.24.5, , , , , , , , , ] 2022-09-03T05:55:45.616 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:45.615+0000 7f3e1fd32600 1 waiting for incremental sync to catch up: 2022-09-03T05:55:45.617 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.22.5, , , , , , , , , ] 2022-09-03T05:55:45.618 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.24.5, , , , , , , , , ] 2022-09-03T05:55:50.618 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:50.616+0000 7f3e1fd32600 1 bucket sync caught up with source: 2022-09-03T05:55:50.618 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.24.5, , , , , , , , , ] 2022-09-03T05:55:50.618 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.24.5, , , , , , , , , ] 2022-09-03T05:55:50.619 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:50.616+0000 7f3e1fd32600 0 bucket checkpoint complete 2022-09-03T05:55:50.640 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-15 zones={test-zone1, test-zone3} 2022-09-03T05:55:50.641 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:50.642 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:50.643 DEBUG:boto:Method: HEAD 2022-09-03T05:55:50.643 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:55:50.644 DEBUG:boto:Data: 2022-09-03T05:55:50.644 DEBUG:boto:Headers: {} 2022-09-03T05:55:50.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:50.645 DEBUG:boto:Port: 8000 2022-09-03T05:55:50.645 DEBUG:boto:Params: {} 2022-09-03T05:55:50.646 DEBUG:boto:Token: None 2022-09-03T05:55:50.647 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:50 GMT /avysmw-15/ 2022-09-03T05:55:50.647 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s84BqwcikxsnFpT9zxEeNhhZVNc= 2022-09-03T05:55:50.648 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s84BqwcikxsnFpT9zxEeNhhZVNc=', 'Content-Length': '0'} 2022-09-03T05:55:50.651 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000026b2d57db2f199c4-006312ec66-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:50.652 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:50.652 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:50.653 DEBUG:boto:Method: HEAD 2022-09-03T05:55:50.653 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:55:50.654 DEBUG:boto:Data: 2022-09-03T05:55:50.654 DEBUG:boto:Headers: {} 2022-09-03T05:55:50.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:50.655 DEBUG:boto:Port: 8001 2022-09-03T05:55:50.656 DEBUG:boto:Params: {} 2022-09-03T05:55:50.656 DEBUG:boto:Token: None 2022-09-03T05:55:50.657 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:50 GMT /avysmw-15/ 2022-09-03T05:55:50.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s84BqwcikxsnFpT9zxEeNhhZVNc= 2022-09-03T05:55:50.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s84BqwcikxsnFpT9zxEeNhhZVNc=', 'Content-Length': '0'} 2022-09-03T05:55:50.661 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cfd494295b47762a-006312ec66-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:50.661 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:55:50.662 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:50.663 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:50.663 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.664 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.664 DEBUG:boto:Method: GET 2022-09-03T05:55:50.665 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.665 DEBUG:boto:Data: 2022-09-03T05:55:50.666 DEBUG:boto:Headers: {} 2022-09-03T05:55:50.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:50.667 DEBUG:boto:Port: 8000 2022-09-03T05:55:50.667 DEBUG:boto:Params: {} 2022-09-03T05:55:50.668 DEBUG:boto:Token: None 2022-09-03T05:55:50.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:50 GMT /avysmw-15/?versions 2022-09-03T05:55:50.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q= 2022-09-03T05:55:50.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q=', 'Content-Length': '0'} 2022-09-03T05:55:50.672 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006b440fa555afec9c-006312ec66-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:50.718 DEBUG:boto:b'avysmw-15999false' 2022-09-03T05:55:50.719 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:55:50.719 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:50.720 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:50.720 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.721 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.721 DEBUG:boto:Method: GET 2022-09-03T05:55:50.722 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.723 DEBUG:boto:Data: 2022-09-03T05:55:50.723 DEBUG:boto:Headers: {} 2022-09-03T05:55:50.724 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:50.724 DEBUG:boto:Port: 8001 2022-09-03T05:55:50.725 DEBUG:boto:Params: {} 2022-09-03T05:55:50.725 DEBUG:boto:Token: None 2022-09-03T05:55:50.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:50 GMT /avysmw-15/?versions 2022-09-03T05:55:50.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q= 2022-09-03T05:55:50.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q=', 'Content-Length': '0'} 2022-09-03T05:55:50.730 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002dd47e407c083273-006312ec66-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:50.731 DEBUG:boto:b'avysmw-15999false' 2022-09-03T05:55:50.731 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:50.732 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:50.732 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.734 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.735 DEBUG:boto:Method: GET 2022-09-03T05:55:50.736 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.737 DEBUG:boto:Data: 2022-09-03T05:55:50.738 DEBUG:boto:Headers: {} 2022-09-03T05:55:50.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:50.739 DEBUG:boto:Port: 8000 2022-09-03T05:55:50.739 DEBUG:boto:Params: {} 2022-09-03T05:55:50.740 DEBUG:boto:Token: None 2022-09-03T05:55:50.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:50 GMT /avysmw-15/?versions 2022-09-03T05:55:50.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q= 2022-09-03T05:55:50.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q=', 'Content-Length': '0'} 2022-09-03T05:55:50.745 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b3423b55b584edec-006312ec66-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:50.746 DEBUG:boto:b'avysmw-15999false' 2022-09-03T05:55:50.747 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:50.747 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:50.748 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.748 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.749 DEBUG:boto:Method: GET 2022-09-03T05:55:50.749 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.750 DEBUG:boto:Data: 2022-09-03T05:55:50.750 DEBUG:boto:Headers: {} 2022-09-03T05:55:50.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:50.751 DEBUG:boto:Port: 8001 2022-09-03T05:55:50.752 DEBUG:boto:Params: {} 2022-09-03T05:55:50.752 DEBUG:boto:Token: None 2022-09-03T05:55:50.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:50 GMT /avysmw-15/?versions 2022-09-03T05:55:50.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q= 2022-09-03T05:55:50.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q=', 'Content-Length': '0'} 2022-09-03T05:55:50.757 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017cdd6e8b679b46b-006312ec66-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:50.758 DEBUG:boto:b'avysmw-15999false' 2022-09-03T05:55:50.758 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-15 zones={test-zone1, test-zone3} 2022-09-03T05:55:50.759 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:50.760 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:50.760 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-15 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:55:50.824 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:50.825 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:50.881 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:50.879+0000 7efd66a3b600 1 bucket sync caught up with source: 2022-09-03T05:55:50.882 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.24.5, , , , , , , , , ] 2022-09-03T05:55:50.882 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.24.5, , , , , , , , , ] 2022-09-03T05:55:50.883 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:50.879+0000 7efd66a3b600 0 bucket checkpoint complete 2022-09-03T05:55:50.900 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-15 zones={test-zone2, test-zone3} 2022-09-03T05:55:50.901 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:50.902 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:50.904 DEBUG:boto:Method: HEAD 2022-09-03T05:55:50.905 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:55:50.906 DEBUG:boto:Data: 2022-09-03T05:55:50.907 DEBUG:boto:Headers: {} 2022-09-03T05:55:50.908 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:50.909 DEBUG:boto:Port: 8000 2022-09-03T05:55:50.910 DEBUG:boto:Params: {} 2022-09-03T05:55:50.911 DEBUG:boto:Token: None 2022-09-03T05:55:50.913 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:50 GMT /avysmw-15/ 2022-09-03T05:55:50.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s84BqwcikxsnFpT9zxEeNhhZVNc= 2022-09-03T05:55:50.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s84BqwcikxsnFpT9zxEeNhhZVNc=', 'Content-Length': '0'} 2022-09-03T05:55:50.918 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d35788d762573e10-006312ec66-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:50.920 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:50.921 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:50.922 DEBUG:boto:Method: HEAD 2022-09-03T05:55:50.923 DEBUG:boto:Path: /avysmw-15/ 2022-09-03T05:55:50.924 DEBUG:boto:Data: 2022-09-03T05:55:50.925 DEBUG:boto:Headers: {} 2022-09-03T05:55:50.926 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:50.927 DEBUG:boto:Port: 8001 2022-09-03T05:55:50.928 DEBUG:boto:Params: {} 2022-09-03T05:55:50.929 DEBUG:boto:Token: None 2022-09-03T05:55:50.930 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:50 GMT /avysmw-15/ 2022-09-03T05:55:50.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s84BqwcikxsnFpT9zxEeNhhZVNc= 2022-09-03T05:55:50.932 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s84BqwcikxsnFpT9zxEeNhhZVNc=', 'Content-Length': '0'} 2022-09-03T05:55:50.941 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a44278157eac6c64-006312ec66-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:50.942 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:55:50.942 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:50.943 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:50.944 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.945 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.946 DEBUG:boto:Method: GET 2022-09-03T05:55:50.947 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:55:50.947 DEBUG:boto:Data: 2022-09-03T05:55:50.948 DEBUG:boto:Headers: {} 2022-09-03T05:55:50.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:50.950 DEBUG:boto:Port: 8000 2022-09-03T05:55:50.951 DEBUG:boto:Params: {} 2022-09-03T05:55:50.952 DEBUG:boto:Token: None 2022-09-03T05:55:50.953 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:50 GMT /avysmw-15/?versions 2022-09-03T05:55:50.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q= 2022-09-03T05:55:50.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K2ilffaaC7fK77Y692Jjg9RJ90Q=', 'Content-Length': '0'} 2022-09-03T05:55:50.958 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a56138ab0e1c9941-006312ec66-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:51.011 DEBUG:boto:b'avysmw-15999false' 2022-09-03T05:55:51.013 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:55:51.014 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:51.014 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:51.015 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:51.016 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:51.016 DEBUG:boto:Method: GET 2022-09-03T05:55:51.017 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:55:51.017 DEBUG:boto:Data: 2022-09-03T05:55:51.018 DEBUG:boto:Headers: {} 2022-09-03T05:55:51.019 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:51.019 DEBUG:boto:Port: 8001 2022-09-03T05:55:51.020 DEBUG:boto:Params: {} 2022-09-03T05:55:51.021 DEBUG:boto:Token: None 2022-09-03T05:55:51.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:51 GMT /avysmw-15/?versions 2022-09-03T05:55:51.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1HG9P/rPyXGuGv7LcuC/K736eU= 2022-09-03T05:55:51.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1HG9P/rPyXGuGv7LcuC/K736eU=', 'Content-Length': '0'} 2022-09-03T05:55:51.026 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b9d83d15dfc6b62d-006312ec67-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:51.027 DEBUG:boto:b'avysmw-15999false' 2022-09-03T05:55:51.028 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:51.029 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:51.031 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:51.031 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:51.032 DEBUG:boto:Method: GET 2022-09-03T05:55:51.033 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:55:51.034 DEBUG:boto:Data: 2022-09-03T05:55:51.035 DEBUG:boto:Headers: {} 2022-09-03T05:55:51.036 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:51.037 DEBUG:boto:Port: 8000 2022-09-03T05:55:51.038 DEBUG:boto:Params: {} 2022-09-03T05:55:51.039 DEBUG:boto:Token: None 2022-09-03T05:55:51.039 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:51 GMT /avysmw-15/?versions 2022-09-03T05:55:51.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1HG9P/rPyXGuGv7LcuC/K736eU= 2022-09-03T05:55:51.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1HG9P/rPyXGuGv7LcuC/K736eU=', 'Content-Length': '0'} 2022-09-03T05:55:51.043 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000059153fee9fdd2a0f-006312ec67-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:51.043 DEBUG:boto:b'avysmw-15999false' 2022-09-03T05:55:51.044 DEBUG:boto:path=/avysmw-15/ 2022-09-03T05:55:51.045 DEBUG:boto:auth_path=/avysmw-15/ 2022-09-03T05:55:51.046 DEBUG:boto:path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:51.047 DEBUG:boto:auth_path=/avysmw-15/?versions&max-keys=999 2022-09-03T05:55:51.048 DEBUG:boto:Method: GET 2022-09-03T05:55:51.049 DEBUG:boto:Path: /avysmw-15/?versions&max-keys=999 2022-09-03T05:55:51.050 DEBUG:boto:Data: 2022-09-03T05:55:51.050 DEBUG:boto:Headers: {} 2022-09-03T05:55:51.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:51.052 DEBUG:boto:Port: 8001 2022-09-03T05:55:51.053 DEBUG:boto:Params: {} 2022-09-03T05:55:51.055 DEBUG:boto:Token: None 2022-09-03T05:55:51.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:55:51 GMT /avysmw-15/?versions 2022-09-03T05:55:51.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1HG9P/rPyXGuGv7LcuC/K736eU= 2022-09-03T05:55:51.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1HG9P/rPyXGuGv7LcuC/K736eU=', 'Content-Length': '0'} 2022-09-03T05:55:51.061 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f9bef400e82a339-006312ec67-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:55:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:51.102 DEBUG:boto:b'avysmw-15999false' 2022-09-03T05:55:51.103 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-15 zones={test-zone2, test-zone3} 2022-09-03T05:55:51.104 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_object_delete ... ok 2022-09-03T05:55:51.106 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-16 2022-09-03T05:55:51.106 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:55:51.107 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:55:51.107 DEBUG:boto:Method: PUT 2022-09-03T05:55:51.108 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:55:51.108 DEBUG:boto:Data: 2022-09-03T05:55:51.109 DEBUG:boto:Headers: {} 2022-09-03T05:55:51.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:51.110 DEBUG:boto:Port: 8000 2022-09-03T05:55:51.110 DEBUG:boto:Params: {} 2022-09-03T05:55:51.111 DEBUG:boto:Token: None 2022-09-03T05:55:51.112 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:55:51 GMT /avysmw-16/ 2022-09-03T05:55:51.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzjU6vqYVnTeO/m1KIVJwBrmESE= 2022-09-03T05:55:51.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzjU6vqYVnTeO/m1KIVJwBrmESE=', 'Content-Length': '0'} 2022-09-03T05:55:51.143 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000f98e79e00b90acdd-006312ec67-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:51.145 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-17 2022-09-03T05:55:51.147 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:55:51.148 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:55:51.149 DEBUG:boto:Method: PUT 2022-09-03T05:55:51.150 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:55:51.150 DEBUG:boto:Data: 2022-09-03T05:55:51.151 DEBUG:boto:Headers: {} 2022-09-03T05:55:51.151 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:51.152 DEBUG:boto:Port: 8000 2022-09-03T05:55:51.152 DEBUG:boto:Params: {} 2022-09-03T05:55:51.153 DEBUG:boto:Token: None 2022-09-03T05:55:51.154 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:55:51 GMT /avysmw-17/ 2022-09-03T05:55:51.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uEYDvYg4t1nf4GEV4Z0ZzEmY3EU= 2022-09-03T05:55:51.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uEYDvYg4t1nf4GEV4Z0ZzEmY3EU=', 'Content-Length': '0'} 2022-09-03T05:55:52.193 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c575f410f0009af2-006312ec67-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:52.197 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-18 2022-09-03T05:55:52.198 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:55:52.199 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:55:52.200 DEBUG:boto:Method: PUT 2022-09-03T05:55:52.200 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:55:52.201 DEBUG:boto:Data: 2022-09-03T05:55:52.202 DEBUG:boto:Headers: {} 2022-09-03T05:55:52.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:52.204 DEBUG:boto:Port: 8001 2022-09-03T05:55:52.205 DEBUG:boto:Params: {} 2022-09-03T05:55:52.206 DEBUG:boto:Token: None 2022-09-03T05:55:52.207 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:55:52 GMT /avysmw-18/ 2022-09-03T05:55:52.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/QBHRNsWMBvDlGSDPbGOerEEFw= 2022-09-03T05:55:52.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/QBHRNsWMBvDlGSDPbGOerEEFw=', 'Content-Length': '0'} 2022-09-03T05:55:52.237 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000049e1df7770a6116c-006312ec68-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:52.239 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:55:52.239 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:55:52.240 DEBUG:boto:path=/avysmw-16/?versioning 2022-09-03T05:55:52.241 DEBUG:boto:auth_path=/avysmw-16/?versioning 2022-09-03T05:55:52.241 DEBUG:boto:Method: PUT 2022-09-03T05:55:52.242 DEBUG:boto:Path: /avysmw-16/?versioning 2022-09-03T05:55:52.243 DEBUG:boto:Data: Enabled Disabled 2022-09-03T05:55:52.243 DEBUG:boto:Headers: {} 2022-09-03T05:55:52.244 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:52.245 DEBUG:boto:Port: 8000 2022-09-03T05:55:52.245 DEBUG:boto:Params: {} 2022-09-03T05:55:52.246 DEBUG:boto:Token: None 2022-09-03T05:55:52.247 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:55:52 GMT /avysmw-16/?versioning 2022-09-03T05:55:52.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R6rmTMtXMKn68ovD+nBHK2GrYAA= 2022-09-03T05:55:52.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R6rmTMtXMKn68ovD+nBHK2GrYAA=', 'Content-Length': '228'} 2022-09-03T05:55:52.255 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000295613b27ef6b6cb-006312ec68-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:52.256 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:55:52.256 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:55:52.257 DEBUG:boto:path=/avysmw-17/?versioning 2022-09-03T05:55:52.257 DEBUG:boto:auth_path=/avysmw-17/?versioning 2022-09-03T05:55:52.258 DEBUG:boto:Method: PUT 2022-09-03T05:55:52.258 DEBUG:boto:Path: /avysmw-17/?versioning 2022-09-03T05:55:52.259 DEBUG:boto:Data: Enabled Disabled 2022-09-03T05:55:52.259 DEBUG:boto:Headers: {} 2022-09-03T05:55:52.260 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:55:52.260 DEBUG:boto:Port: 8000 2022-09-03T05:55:52.261 DEBUG:boto:Params: {} 2022-09-03T05:55:52.261 DEBUG:boto:Token: None 2022-09-03T05:55:52.262 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:55:52 GMT /avysmw-17/?versioning 2022-09-03T05:55:52.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tacwKkK0CN4ildMZEu9TLtNx0d4= 2022-09-03T05:55:52.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tacwKkK0CN4ildMZEu9TLtNx0d4=', 'Content-Length': '228'} 2022-09-03T05:55:52.277 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000659ca258865be78d-006312ec68-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:52.278 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:55:52.279 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:55:52.280 DEBUG:boto:path=/avysmw-18/?versioning 2022-09-03T05:55:52.281 DEBUG:boto:auth_path=/avysmw-18/?versioning 2022-09-03T05:55:52.282 DEBUG:boto:Method: PUT 2022-09-03T05:55:52.283 DEBUG:boto:Path: /avysmw-18/?versioning 2022-09-03T05:55:52.283 DEBUG:boto:Data: Enabled Disabled 2022-09-03T05:55:52.284 DEBUG:boto:Headers: {} 2022-09-03T05:55:52.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:55:52.285 DEBUG:boto:Port: 8001 2022-09-03T05:55:52.285 DEBUG:boto:Params: {} 2022-09-03T05:55:52.286 DEBUG:boto:Token: None 2022-09-03T05:55:52.286 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:55:52 GMT /avysmw-18/?versioning 2022-09-03T05:55:52.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yOSOg6dQlgwLEwQ/jp0Y01dFXVg= 2022-09-03T05:55:52.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yOSOg6dQlgwLEwQ/jp0Y01dFXVg=', 'Content-Length': '228'} 2022-09-03T05:55:52.301 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000058052f6a18c45385-006312ec68-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:52.302 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:52.303 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:52.304 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:55:52.358 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:52.359 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:52.416 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T05:55:52.447 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T05:55:52.448 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:55:52.448 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:55:52.449 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T05:55:52.449 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:55:52.450 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:55:52.450 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184552.296862_136.1", 2022-09-03T05:55:52.451 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:55:52.296862Z" 2022-09-03T05:55:52.451 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:55:52.451 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:55:52.452 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:55:52.452 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:55:52.452 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:55:52.453 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:55:52.453 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:55:52.453 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T05:55:52.453 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:55:52.454 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T05:55:52.455 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662184552.296862_136.1', 2: '', 3: ''} 2022-09-03T05:55:52.456 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T05:55:52.456 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:52.457 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:52.457 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:55:52.509 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:52.509 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:52.558 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:52.556+0000 7f5dce655600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:55:52.559 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:52.557+0000 7f5dce655600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:55:52.570 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:55:52.571 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:55:52.572 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:55:52.572 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:55:52.573 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:55:52.573 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:55:52.574 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:52.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:52.575 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:55:52.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:52.576 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:55:52.576 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:52.576 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:52.577 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:55:52.577 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:52.578 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:55:52.578 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:52.579 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:55:52.579 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:52.579 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:52.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:52.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:52.580 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:55:52.580 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:52.581 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:52.581 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184552.173517_125.1", 2022-09-03T05:55:52.581 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:52.581 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:55:52.582 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:52.582 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.173517Z", 2022-09-03T05:55:52.582 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:52.583 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:52.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:52.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:52.583 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:55:52.584 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:52.584 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:52.584 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:55:52.584 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:52.585 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:55:52.585 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:52.585 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:55:52.586 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:52.586 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:52.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:52.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:52.587 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:55:52.587 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:52.588 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:52.588 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:55:52.588 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:52.589 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:55:52.589 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:52.589 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:55:52.590 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:52.590 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:52.590 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:52.591 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:55:52.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:52.591 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:55:52.591 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:55:52.592 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:55:52.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:52.592 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:55:52Z" 2022-09-03T05:55:52.592 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:55:52.594 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184552.173517_125.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:55:52.173517Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:55:52Z" } 2022-09-03T05:55:52.594 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184552.173517_125.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:55:52.173517Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:55:52.595 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184552.296862_136.1', 2: '', 3: ''} 2022-09-03T05:55:52.595 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184552.173517_125.1', 2: '', 3: ''} 2022-09-03T05:55:52.596 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662184552.296862_136.1 target=1_1662184552.173517_125.1 2022-09-03T05:55:57.597 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:57.598 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:57.600 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:55:57.658 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:57.659 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:57.699 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:57.697+0000 7f4f9b9ca600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:55:57.700 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:57.698+0000 7f4f9b9ca600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:55:57.718 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:55:57.718 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:55:57.719 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:55:57.719 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:55:57.720 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:55:57.720 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:55:57.721 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:57.721 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:57.721 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:55:57.722 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:57.722 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:55:57.723 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:57.723 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:57.724 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:55:57.724 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:57.725 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:55:57.725 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:57.726 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:55:57.726 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:57.727 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:57.727 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:57.728 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:57.728 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:55:57.729 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:57.729 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:57.729 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184552.296862_136.1", 2022-09-03T05:55:57.730 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:57.730 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:55:57.731 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:57.731 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.296862Z", 2022-09-03T05:55:57.732 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:57.732 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:57.732 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:57.733 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:57.733 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:55:57.733 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:57.733 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:57.734 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:55:57.734 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:57.734 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:55:57.735 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:57.735 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:55:57.735 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:57.735 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:57.736 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:57.736 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:57.736 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:55:57.736 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:57.737 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:57.737 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:55:57.737 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:57.737 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:55:57.738 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:57.738 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:55:57.738 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:57.738 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:57.739 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:57.739 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:55:57.739 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:57.740 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:55:57.740 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:55:57.740 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:55:57.741 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:57.741 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:55:57Z" 2022-09-03T05:55:57.741 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:55:57.743 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184552.296862_136.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:55:52.296862Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:55:57Z" } 2022-09-03T05:55:57.743 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184552.296862_136.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:55:52.296862Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:55:57.744 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184552.296862_136.1', 2: '', 3: ''} 2022-09-03T05:55:57.744 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184552.296862_136.1', 2: '', 3: ''} 2022-09-03T05:55:57.745 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T05:55:57.745 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T05:55:57.746 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:57.746 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:57.747 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:55:57.806 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:57.807 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:57.837 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:57.834+0000 7fb423e75600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:55:57.838 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:55:57.835+0000 7fb423e75600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:55:57.850 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T05:55:57.851 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T05:55:57.851 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T05:55:57.852 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T05:55:57.852 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T05:55:57.852 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:55:57.853 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:55:57.853 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:55:57.853 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T05:55:57.854 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:55:57.854 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T05:55:57.854 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:55:57.854 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:55:57.855 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:55:57.855 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:55:57.855 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T05:55:57.856 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:55:57.856 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:55:57.856 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:55:57.857 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:55:57.857 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:55:57.857 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:55:57.857 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T05:55:57.858 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:55:57.858 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:55:57.858 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184552.296862_136.1", 2022-09-03T05:55:57.859 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:55:57.859 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:55:57.859 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:55:57.859 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.296862Z", 2022-09-03T05:55:57.860 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:55:57.860 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:55:57.860 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:55:57.860 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:55:57.861 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T05:55:57.861 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:55:57.861 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:55:57.861 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:55:57.862 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:55:57.862 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:55:57.862 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:55:57.862 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:55:57.863 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:55:57.863 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:55:57.863 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:55:57.863 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T05:55:57.864 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T05:55:57.864 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T05:55:57.864 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T05:55:57.864 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T05:55:57.865 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T05:55:57.865 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T05:55:57.865 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T05:55:57.866 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T05:55:57.866 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T05:55:57.866 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:55:57.866 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T05:55:57.867 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T05:55:57.867 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:55:57.867 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T05:55:57.867 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T05:55:57.868 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T05:55:57.868 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T05:55:57.868 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T05:55:57Z" 2022-09-03T05:55:57.868 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T05:55:57.870 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184552.296862_136.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:55:52.296862Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:55:57Z" } 2022-09-03T05:55:57.870 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184552.296862_136.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:55:52.296862Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:55:57.871 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184552.296862_136.1', 2: '', 3: ''} 2022-09-03T05:55:57.871 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184552.296862_136.1', 2: '', 3: ''} 2022-09-03T05:55:57.872 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T05:55:57.872 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T05:55:57.873 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:57.873 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T05:55:57.874 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T05:55:57.926 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:57.926 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:57.968 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:57.966+0000 7efd5ade3600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:55:57.969 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:57.967+0000 7efd5ade3600 20 RGW-SYNC:meta: read sync status 2022-09-03T05:55:57.980 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T05:55:57.980 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T05:55:57.981 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T05:55:57.981 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T05:55:57.982 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T05:55:57.982 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T05:55:57.983 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:57.984 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:57.984 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T05:55:57.985 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:57.985 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T05:55:57.986 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:57.986 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:57.987 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T05:55:57.987 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:57.988 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T05:55:57.988 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:57.989 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T05:55:57.989 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:57.990 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:57.990 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:57.991 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:57.991 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T05:55:57.992 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:57.992 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:57.993 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184552.296862_136.1", 2022-09-03T05:55:57.993 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:57.994 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:55:57.994 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:57.995 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.296862Z", 2022-09-03T05:55:57.995 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:57.996 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:57.996 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:57.997 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:57.997 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T05:55:57.998 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:57.998 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:57.999 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:55:57.999 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:58.000 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:55:58.000 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:58.001 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:55:58.001 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:58.002 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:58.002 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:58.003 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T05:55:58.003 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T05:55:58.004 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T05:55:58.004 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T05:55:58.005 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T05:55:58.005 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T05:55:58.006 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T05:55:58.006 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T05:55:58.007 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T05:55:58.007 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T05:55:58.008 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:58.008 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T05:55:58.009 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T05:55:58.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:58.010 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T05:55:58.010 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T05:55:58.011 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T05:55:58.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T05:55:58.012 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T05:55:57Z" 2022-09-03T05:55:58.012 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T05:55:58.014 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184552.296862_136.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:55:52.296862Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T05:55:57Z" } 2022-09-03T05:55:58.016 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184552.296862_136.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:55:52.296862Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T05:55:58.016 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184552.296862_136.1', 2: '', 3: ''} 2022-09-03T05:55:58.017 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184552.296862_136.1', 2: '', 3: ''} 2022-09-03T05:55:58.017 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T05:55:58.018 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:55:58.018 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:55:58.019 DEBUG:boto:Method: HEAD 2022-09-03T05:55:58.019 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:55:58.020 DEBUG:boto:Data: 2022-09-03T05:55:58.020 DEBUG:boto:Headers: {} 2022-09-03T05:55:58.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:58.021 DEBUG:boto:Port: 8000 2022-09-03T05:55:58.022 DEBUG:boto:Params: {} 2022-09-03T05:55:58.022 DEBUG:boto:Token: None 2022-09-03T05:55:58.023 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:55:58 GMT /avysmw-16/ 2022-09-03T05:55:58.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L/WJAAo6tUQ+l0Hfjacsl+s2Mbw= 2022-09-03T05:55:58.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:55:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L/WJAAo6tUQ+l0Hfjacsl+s2Mbw=', 'Content-Length': '0'} 2022-09-03T05:55:58.027 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000098377549e10f2d11-006312ec6e-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:58.028 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:55:58.028 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:55:58.029 DEBUG:boto:Method: PUT 2022-09-03T05:55:58.029 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T05:55:58.030 DEBUG:boto:Data: 2022-09-03T05:55:58.030 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T05:55:58.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:55:58.031 DEBUG:boto:Port: 8000 2022-09-03T05:55:58.032 DEBUG:boto:Params: {} 2022-09-03T05:55:58.032 DEBUG:boto:Token: None 2022-09-03T05:55:58.033 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 05:55:58 GMT /avysmw-16/dummy 2022-09-03T05:55:58.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g3qzUTplgTtovXUGiROSUHB9U3M= 2022-09-03T05:55:58.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:55:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g3qzUTplgTtovXUGiROSUHB9U3M='} 2022-09-03T05:55:58.048 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-amz-request-id', 'tx000003ea7fa0c9183b253-006312ec6e-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:55:58.049 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:58.049 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:55:58.050 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:55:58.106 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:55:58.106 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:55:58.141 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:55:58.139+0000 7f4cd539d600 1 waiting to reach incremental sync.. 2022-09-03T05:56:03.144 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:56:03.140+0000 7f4cd539d600 1 waiting to reach incremental sync.. 2022-09-03T05:56:08.143 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:56:08.141+0000 7f4cd539d600 1 waiting to reach incremental sync.. 2022-09-03T05:56:13.145 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:56:13.142+0000 7f4cd539d600 1 bucket sync caught up with source: 2022-09-03T05:56:13.145 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T05:56:13.146 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T05:56:13.146 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:56:13.142+0000 7f4cd539d600 0 bucket checkpoint complete 2022-09-03T05:56:13.161 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:13.162 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:13.163 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:56:13.238 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:56:13.239 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:56:13.290 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:13.288+0000 7fe26d4cb600 1 bucket sync caught up with source: 2022-09-03T05:56:13.291 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T05:56:13.291 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T05:56:13.292 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:13.288+0000 7fe26d4cb600 0 bucket checkpoint complete 2022-09-03T05:56:13.303 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:13.304 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:13.305 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:56:13.369 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:56:13.370 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:56:13.406 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:56:13.403+0000 7fe813ea0600 1 bucket sync caught up with source: 2022-09-03T05:56:13.407 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T05:56:13.407 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T05:56:13.408 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:56:13.403+0000 7fe813ea0600 0 bucket checkpoint complete 2022-09-03T05:56:13.429 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:56:13.430 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:56:13.431 DEBUG:boto:Method: HEAD 2022-09-03T05:56:13.432 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:56:13.433 DEBUG:boto:Data: 2022-09-03T05:56:13.434 DEBUG:boto:Headers: {} 2022-09-03T05:56:13.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:56:13.435 DEBUG:boto:Port: 8000 2022-09-03T05:56:13.436 DEBUG:boto:Params: {} 2022-09-03T05:56:13.437 DEBUG:boto:Token: None 2022-09-03T05:56:13.439 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:56:13 GMT /avysmw-17/ 2022-09-03T05:56:13.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r1qR2ut1f8EdQt7PrbAO6YlwdfA= 2022-09-03T05:56:13.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:56:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r1qR2ut1f8EdQt7PrbAO6YlwdfA=', 'Content-Length': '0'} 2022-09-03T05:56:13.444 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d337d82e21c77070-006312ec7d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:56:13.445 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:56:13.446 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:56:13.447 DEBUG:boto:Method: PUT 2022-09-03T05:56:13.448 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T05:56:13.449 DEBUG:boto:Data: 2022-09-03T05:56:13.449 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T05:56:13.450 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:56:13.450 DEBUG:boto:Port: 8000 2022-09-03T05:56:13.451 DEBUG:boto:Params: {} 2022-09-03T05:56:13.451 DEBUG:boto:Token: None 2022-09-03T05:56:13.452 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 05:56:13 GMT /avysmw-17/dummy 2022-09-03T05:56:13.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5it8LY8x+jOuAOptxBdivK7PiT0= 2022-09-03T05:56:13.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:56:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5it8LY8x+jOuAOptxBdivK7PiT0='} 2022-09-03T05:56:13.473 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-amz-request-id', 'tx00000c85aed9371de8523-006312ec7d-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:56:13.474 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:13.475 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:13.476 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:56:13.535 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:56:13.536 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:56:13.569 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:13.567+0000 7fe655740600 1 waiting to reach incremental sync.. 2022-09-03T05:56:18.569 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:18.568+0000 7fe655740600 1 waiting to reach incremental sync.. 2022-09-03T05:56:23.570 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:23.568+0000 7fe655740600 1 waiting to reach incremental sync.. 2022-09-03T05:56:28.571 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:28.569+0000 7fe655740600 1 waiting to reach incremental sync.. 2022-09-03T05:56:33.573 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:33.571+0000 7fe655740600 1 bucket sync caught up with source: 2022-09-03T05:56:33.573 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.17.10, , , , , ] 2022-09-03T05:56:33.574 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000002.17.10, , , , , ] 2022-09-03T05:56:33.574 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:33.571+0000 7fe655740600 0 bucket checkpoint complete 2022-09-03T05:56:33.591 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:33.592 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:33.593 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:56:33.656 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:56:33.657 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:56:33.698 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:33.693+0000 7fa719c93600 1 bucket sync caught up with source: 2022-09-03T05:56:33.698 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.17.10, , , , , ] 2022-09-03T05:56:33.698 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000002.17.10, , , , , ] 2022-09-03T05:56:33.699 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:33.693+0000 7fa719c93600 0 bucket checkpoint complete 2022-09-03T05:56:33.709 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:56:33.711 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:56:33.712 DEBUG:boto:Method: HEAD 2022-09-03T05:56:33.713 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:56:33.714 DEBUG:boto:Data: 2022-09-03T05:56:33.715 DEBUG:boto:Headers: {} 2022-09-03T05:56:33.715 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:56:33.716 DEBUG:boto:Port: 8001 2022-09-03T05:56:33.717 DEBUG:boto:Params: {} 2022-09-03T05:56:33.718 DEBUG:boto:Token: None 2022-09-03T05:56:33.719 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:56:33 GMT /avysmw-18/ 2022-09-03T05:56:33.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uWe42Fi7jSeP7IH9z9ANn+Of2YA= 2022-09-03T05:56:33.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:56:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uWe42Fi7jSeP7IH9z9ANn+Of2YA=', 'Content-Length': '0'} 2022-09-03T05:56:33.724 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002e39bb3ee7820fec-006312ec91-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:56:33.725 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:56:33.726 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:56:33.727 DEBUG:boto:Method: PUT 2022-09-03T05:56:33.728 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T05:56:33.728 DEBUG:boto:Data: 2022-09-03T05:56:33.729 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T05:56:33.730 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:56:33.731 DEBUG:boto:Port: 8001 2022-09-03T05:56:33.732 DEBUG:boto:Params: {} 2022-09-03T05:56:33.733 DEBUG:boto:Token: None 2022-09-03T05:56:33.734 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 05:56:33 GMT /avysmw-18/dummy 2022-09-03T05:56:33.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZZT0mEFyQnrl3H1+9GTcM4++0M= 2022-09-03T05:56:33.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:56:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZZT0mEFyQnrl3H1+9GTcM4++0M='} 2022-09-03T05:56:33.747 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-amz-request-id', 'tx00000a1dce425d29bd68e-006312ec91-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:56:33.749 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:33.750 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:33.751 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:56:33.810 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:56:33.810 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:56:33.852 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:33.850+0000 7f54d8565600 1 waiting to reach incremental sync.. 2022-09-03T05:56:38.852 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:38.851+0000 7f54d8565600 1 waiting to reach incremental sync.. 2022-09-03T05:56:43.853 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:43.851+0000 7f54d8565600 1 waiting to reach incremental sync.. 2022-09-03T05:56:48.853 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:48.852+0000 7f54d8565600 1 waiting to reach incremental sync.. 2022-09-03T05:56:53.856 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:53.853+0000 7f54d8565600 1 bucket sync caught up with source: 2022-09-03T05:56:53.857 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.45.10, , , , , ] 2022-09-03T05:56:53.857 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000002.45.10, , , , , ] 2022-09-03T05:56:53.858 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:56:53.853+0000 7f54d8565600 0 bucket checkpoint complete 2022-09-03T05:56:53.876 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:53.877 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:56:53.878 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:56:53.941 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:56:53.942 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:56:53.993 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:56:53.991+0000 7f9d2361f600 1 waiting for incremental sync to catch up: 2022-09-03T05:56:53.993 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000002.45.10, , , , , ] 2022-09-03T05:56:53.994 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000004.49.9, , , , , ] 2022-09-03T05:56:58.994 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:56:58.992+0000 7f9d2361f600 1 waiting for incremental sync to catch up: 2022-09-03T05:56:58.995 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000002.45.10, , , , , ] 2022-09-03T05:56:58.995 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000004.49.9, , , , , ] 2022-09-03T05:57:03.996 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:57:03.993+0000 7f9d2361f600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:03.996 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000002.45.10, , , , , ] 2022-09-03T05:57:03.997 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000004.49.9, , , , , ] 2022-09-03T05:57:08.997 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:57:08.995+0000 7f9d2361f600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:08.998 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000002.45.10, , , , , ] 2022-09-03T05:57:08.998 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000004.49.9, , , , , ] 2022-09-03T05:57:13.998 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:57:13.996+0000 7f9d2361f600 1 bucket sync caught up with source: 2022-09-03T05:57:13.999 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000004.49.9, , , , , ] 2022-09-03T05:57:13.999 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000004.49.9, , , , , ] 2022-09-03T05:57:14.000 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:57:13.996+0000 7f9d2361f600 0 bucket checkpoint complete 2022-09-03T05:57:14.016 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:57:14.017 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:57:14.018 DEBUG:boto:Method: HEAD 2022-09-03T05:57:14.019 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:57:14.020 DEBUG:boto:Data: 2022-09-03T05:57:14.020 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.021 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.022 DEBUG:boto:Params: {} 2022-09-03T05:57:14.022 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T05:57:14.023 DEBUG:boto:Token: None 2022-09-03T05:57:14.024 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/ 2022-09-03T05:57:14.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1iVPSMeSSZLZSpcbhPYRW0TZw0A= 2022-09-03T05:57:14.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1iVPSMeSSZLZSpcbhPYRW0TZw0A=', 'Content-Length': '0'} 2022-09-03T05:57:14.030 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000479204e568b82ad9-006312ecba-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.031 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.031 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.032 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.032 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T05:57:14.033 DEBUG:boto:Data: 2022-09-03T05:57:14.033 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.034 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.034 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.035 DEBUG:boto:Params: {} 2022-09-03T05:57:14.035 DEBUG:boto:Token: None 2022-09-03T05:57:14.036 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone1 2022-09-03T05:57:14.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hHlASQ8os1RlfIReilfl/QYOvsc= 2022-09-03T05:57:14.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hHlASQ8os1RlfIReilfl/QYOvsc='} 2022-09-03T05:57:14.050 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-amz-request-id', 'tx00000c46807e1821e05a2-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.051 DEBUG:rgw_multi.tests:version1 id=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:57:14.052 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.053 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.053 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.054 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T05:57:14.054 DEBUG:boto:Data: 2022-09-03T05:57:14.055 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.055 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.056 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.056 DEBUG:boto:Params: {} 2022-09-03T05:57:14.057 DEBUG:boto:Token: None 2022-09-03T05:57:14.057 DEBUG:boto:StringToSign: PUT Lg6VKF8IoH3qF+fuERshyA== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone1 2022-09-03T05:57:14.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:abhxJl2eiHwpAkDupHznmc2W2zM= 2022-09-03T05:57:14.059 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:abhxJl2eiHwpAkDupHznmc2W2zM='} 2022-09-03T05:57:14.071 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"2e0e95285f08a07dea17e7ee111b21c8"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '9oRgQiRPjgriCIUKT4nonjIzjRdL6DA'), ('x-amz-request-id', 'tx0000023070e603c65eef9-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.072 DEBUG:rgw_multi.tests:version2 id=9oRgQiRPjgriCIUKT4nonjIzjRdL6DA 2022-09-03T05:57:14.072 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.073 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.073 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=9oRgQiRPjgriCIUKT4nonjIzjRdL6DA 2022-09-03T05:57:14.074 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=9oRgQiRPjgriCIUKT4nonjIzjRdL6DA 2022-09-03T05:57:14.074 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.075 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=9oRgQiRPjgriCIUKT4nonjIzjRdL6DA 2022-09-03T05:57:14.075 DEBUG:boto:Data: 2022-09-03T05:57:14.076 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.077 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.077 DEBUG:boto:Params: {} 2022-09-03T05:57:14.078 DEBUG:boto:Token: None 2022-09-03T05:57:14.079 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone1?versionId=9oRgQiRPjgriCIUKT4nonjIzjRdL6DA 2022-09-03T05:57:14.079 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGSHmVZzVbr7t2YIf+e71ZOw0LY= 2022-09-03T05:57:14.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGSHmVZzVbr7t2YIf+e71ZOw0LY=', 'Content-Length': '0'} 2022-09-03T05:57:14.094 DEBUG:boto:Response headers: [('x-amz-version-id', '9oRgQiRPjgriCIUKT4nonjIzjRdL6DA'), ('x-amz-request-id', 'tx000002864401979c84099-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.096 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.097 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.098 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.099 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T05:57:14.099 DEBUG:boto:Data: 2022-09-03T05:57:14.100 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.101 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.101 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.102 DEBUG:boto:Params: {} 2022-09-03T05:57:14.102 DEBUG:boto:Token: None 2022-09-03T05:57:14.103 DEBUG:boto:StringToSign: PUT HiNA5/HU7CwjRIhUZJ8d7Q== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone1 2022-09-03T05:57:14.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iFH1DL6/TVw3p02i5e7exAWiaXk= 2022-09-03T05:57:14.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iFH1DL6/TVw3p02i5e7exAWiaXk='} 2022-09-03T05:57:14.146 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"1e2340e7f1d4ec2c23448854649f1ded"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'SY66tqRVJ0HzBn-W9OkWSZ8eF30ySXz'), ('x-amz-request-id', 'tx0000054cd5c15197cbaa6-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.147 DEBUG:rgw_multi.tests:version3 id=SY66tqRVJ0HzBn-W9OkWSZ8eF30ySXz 2022-09-03T05:57:14.147 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.148 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:57:14.148 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=SY66tqRVJ0HzBn-W9OkWSZ8eF30ySXz 2022-09-03T05:57:14.149 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=SY66tqRVJ0HzBn-W9OkWSZ8eF30ySXz 2022-09-03T05:57:14.150 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.151 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=SY66tqRVJ0HzBn-W9OkWSZ8eF30ySXz 2022-09-03T05:57:14.151 DEBUG:boto:Data: 2022-09-03T05:57:14.152 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.152 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.153 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.154 DEBUG:boto:Params: {} 2022-09-03T05:57:14.155 DEBUG:boto:Token: None 2022-09-03T05:57:14.156 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone1?versionId=SY66tqRVJ0HzBn-W9OkWSZ8eF30ySXz 2022-09-03T05:57:14.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bvbQjwIPqpTKqJH6uVa0jIHm1Bg= 2022-09-03T05:57:14.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bvbQjwIPqpTKqJH6uVa0jIHm1Bg=', 'Content-Length': '0'} 2022-09-03T05:57:14.171 DEBUG:boto:Response headers: [('x-amz-version-id', 'SY66tqRVJ0HzBn-W9OkWSZ8eF30ySXz'), ('x-amz-request-id', 'tx00000d177d570dedfed03-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.172 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:57:14.173 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:57:14.173 DEBUG:boto:Method: HEAD 2022-09-03T05:57:14.174 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:57:14.174 DEBUG:boto:Data: 2022-09-03T05:57:14.175 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.175 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.176 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.176 DEBUG:boto:Params: {} 2022-09-03T05:57:14.177 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T05:57:14.178 DEBUG:boto:Token: None 2022-09-03T05:57:14.178 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/ 2022-09-03T05:57:14.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1iVPSMeSSZLZSpcbhPYRW0TZw0A= 2022-09-03T05:57:14.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1iVPSMeSSZLZSpcbhPYRW0TZw0A=', 'Content-Length': '0'} 2022-09-03T05:57:14.185 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f6f1287ac97073fd-006312ecba-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.186 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.187 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.188 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.188 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T05:57:14.189 DEBUG:boto:Data: 2022-09-03T05:57:14.189 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.190 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.191 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.191 DEBUG:boto:Params: {} 2022-09-03T05:57:14.192 DEBUG:boto:Token: None 2022-09-03T05:57:14.193 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone2 2022-09-03T05:57:14.194 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dAuk7P7e1X9AjytfKtMPveiBMPc= 2022-09-03T05:57:14.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dAuk7P7e1X9AjytfKtMPveiBMPc='} 2022-09-03T05:57:14.209 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-amz-request-id', 'tx0000017cb035843659b23-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.210 DEBUG:rgw_multi.tests:version1 id=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:57:14.211 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.212 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.213 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.213 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T05:57:14.214 DEBUG:boto:Data: 2022-09-03T05:57:14.215 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.216 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.217 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.218 DEBUG:boto:Params: {} 2022-09-03T05:57:14.219 DEBUG:boto:Token: None 2022-09-03T05:57:14.220 DEBUG:boto:StringToSign: PUT Lg6VKF8IoH3qF+fuERshyA== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone2 2022-09-03T05:57:14.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O1UXe0wzkINmdVdvOcw73QKhQ4k= 2022-09-03T05:57:14.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O1UXe0wzkINmdVdvOcw73QKhQ4k='} 2022-09-03T05:57:14.232 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"2e0e95285f08a07dea17e7ee111b21c8"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '34j8hGPfd-QHx1f.NqM3PhD-r1WTdhW'), ('x-amz-request-id', 'tx00000689d34bc2f6f25e4-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.233 DEBUG:rgw_multi.tests:version2 id=34j8hGPfd-QHx1f.NqM3PhD-r1WTdhW 2022-09-03T05:57:14.235 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.235 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.236 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=34j8hGPfd-QHx1f.NqM3PhD-r1WTdhW 2022-09-03T05:57:14.236 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=34j8hGPfd-QHx1f.NqM3PhD-r1WTdhW 2022-09-03T05:57:14.237 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.237 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=34j8hGPfd-QHx1f.NqM3PhD-r1WTdhW 2022-09-03T05:57:14.238 DEBUG:boto:Data: 2022-09-03T05:57:14.238 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.239 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.240 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.240 DEBUG:boto:Params: {} 2022-09-03T05:57:14.241 DEBUG:boto:Token: None 2022-09-03T05:57:14.241 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone2?versionId=34j8hGPfd-QHx1f.NqM3PhD-r1WTdhW 2022-09-03T05:57:14.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y6PbyFxLJ2nQeJ9TrRJAbEhvP+Q= 2022-09-03T05:57:14.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y6PbyFxLJ2nQeJ9TrRJAbEhvP+Q=', 'Content-Length': '0'} 2022-09-03T05:57:14.259 DEBUG:boto:Response headers: [('x-amz-version-id', '34j8hGPfd-QHx1f.NqM3PhD-r1WTdhW'), ('x-amz-request-id', 'tx0000002577a2c70ce18c4-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.260 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.261 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.262 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.263 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T05:57:14.264 DEBUG:boto:Data: 2022-09-03T05:57:14.265 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.266 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.267 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.268 DEBUG:boto:Params: {} 2022-09-03T05:57:14.269 DEBUG:boto:Token: None 2022-09-03T05:57:14.270 DEBUG:boto:StringToSign: PUT HiNA5/HU7CwjRIhUZJ8d7Q== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone2 2022-09-03T05:57:14.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XpkhPjcFVTHt8vV5hNpNGIl4TE0= 2022-09-03T05:57:14.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XpkhPjcFVTHt8vV5hNpNGIl4TE0='} 2022-09-03T05:57:14.318 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"1e2340e7f1d4ec2c23448854649f1ded"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'gTsyhys0.T1GsRI3ekMiaYVw4jsy2st'), ('x-amz-request-id', 'tx00000deb92af6a2864558-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.319 DEBUG:rgw_multi.tests:version3 id=gTsyhys0.T1GsRI3ekMiaYVw4jsy2st 2022-09-03T05:57:14.320 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.320 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:57:14.321 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=gTsyhys0.T1GsRI3ekMiaYVw4jsy2st 2022-09-03T05:57:14.322 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=gTsyhys0.T1GsRI3ekMiaYVw4jsy2st 2022-09-03T05:57:14.323 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.324 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=gTsyhys0.T1GsRI3ekMiaYVw4jsy2st 2022-09-03T05:57:14.325 DEBUG:boto:Data: 2022-09-03T05:57:14.326 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.327 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.328 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.328 DEBUG:boto:Params: {} 2022-09-03T05:57:14.329 DEBUG:boto:Token: None 2022-09-03T05:57:14.330 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone2?versionId=gTsyhys0.T1GsRI3ekMiaYVw4jsy2st 2022-09-03T05:57:14.332 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hZ6nuJKZqnHzn0/dfVwyhEW4Lw4= 2022-09-03T05:57:14.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hZ6nuJKZqnHzn0/dfVwyhEW4Lw4=', 'Content-Length': '0'} 2022-09-03T05:57:14.352 DEBUG:boto:Response headers: [('x-amz-version-id', 'gTsyhys0.T1GsRI3ekMiaYVw4jsy2st'), ('x-amz-request-id', 'tx00000454222d65cf2e17e-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.353 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:57:14.354 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:57:14.354 DEBUG:boto:Method: HEAD 2022-09-03T05:57:14.355 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:57:14.355 DEBUG:boto:Data: 2022-09-03T05:57:14.356 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.357 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.357 DEBUG:boto:Params: {} 2022-09-03T05:57:14.358 DEBUG:boto:Token: None 2022-09-03T05:57:14.359 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/ 2022-09-03T05:57:14.359 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1iVPSMeSSZLZSpcbhPYRW0TZw0A= 2022-09-03T05:57:14.360 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1iVPSMeSSZLZSpcbhPYRW0TZw0A=', 'Content-Length': '0'} 2022-09-03T05:57:14.363 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009cba3b591a3660ad-006312ecba-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.364 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.365 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.366 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.367 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T05:57:14.368 DEBUG:boto:Data: 2022-09-03T05:57:14.368 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.369 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.370 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.371 DEBUG:boto:Params: {} 2022-09-03T05:57:14.372 DEBUG:boto:Token: None 2022-09-03T05:57:14.373 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone3 2022-09-03T05:57:14.374 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jP6UABofdrUSR3LqpOvvwXVkcLg= 2022-09-03T05:57:14.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jP6UABofdrUSR3LqpOvvwXVkcLg='} 2022-09-03T05:57:14.387 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-amz-request-id', 'tx000005c64631193b83e92-006312ecba-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.388 DEBUG:rgw_multi.tests:version1 id=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:57:14.389 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.389 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.390 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.390 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T05:57:14.391 DEBUG:boto:Data: 2022-09-03T05:57:14.391 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.392 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.393 DEBUG:boto:Params: {} 2022-09-03T05:57:14.393 DEBUG:boto:Token: None 2022-09-03T05:57:14.394 DEBUG:boto:StringToSign: PUT Lg6VKF8IoH3qF+fuERshyA== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone3 2022-09-03T05:57:14.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AGiIgauqU9A1nXkZkoJI5DNd2/U= 2022-09-03T05:57:14.395 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AGiIgauqU9A1nXkZkoJI5DNd2/U='} 2022-09-03T05:57:14.416 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"2e0e95285f08a07dea17e7ee111b21c8"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'WfJOxC5LRhDnOwALXkpoKEKUfiWKu9f'), ('x-amz-request-id', 'tx000009882a60470bab2c7-006312ecba-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.417 DEBUG:rgw_multi.tests:version2 id=WfJOxC5LRhDnOwALXkpoKEKUfiWKu9f 2022-09-03T05:57:14.417 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.418 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.418 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=WfJOxC5LRhDnOwALXkpoKEKUfiWKu9f 2022-09-03T05:57:14.419 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=WfJOxC5LRhDnOwALXkpoKEKUfiWKu9f 2022-09-03T05:57:14.419 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.420 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=WfJOxC5LRhDnOwALXkpoKEKUfiWKu9f 2022-09-03T05:57:14.420 DEBUG:boto:Data: 2022-09-03T05:57:14.421 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.421 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.422 DEBUG:boto:Params: {} 2022-09-03T05:57:14.423 DEBUG:boto:Token: None 2022-09-03T05:57:14.423 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone3?versionId=WfJOxC5LRhDnOwALXkpoKEKUfiWKu9f 2022-09-03T05:57:14.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dAhtEowUSdjoyf0co6AUCtScUKU= 2022-09-03T05:57:14.424 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dAhtEowUSdjoyf0co6AUCtScUKU=', 'Content-Length': '0'} 2022-09-03T05:57:14.445 DEBUG:boto:Response headers: [('x-amz-version-id', 'WfJOxC5LRhDnOwALXkpoKEKUfiWKu9f'), ('x-amz-request-id', 'tx00000bce2bac11f1dcf5f-006312ecba-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.446 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.446 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.447 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.447 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T05:57:14.448 DEBUG:boto:Data: 2022-09-03T05:57:14.449 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.449 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.450 DEBUG:boto:Params: {} 2022-09-03T05:57:14.451 DEBUG:boto:Token: None 2022-09-03T05:57:14.451 DEBUG:boto:StringToSign: PUT HiNA5/HU7CwjRIhUZJ8d7Q== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone3 2022-09-03T05:57:14.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WBk39IHQikUIJNrV5PdaisfuA4g= 2022-09-03T05:57:14.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WBk39IHQikUIJNrV5PdaisfuA4g='} 2022-09-03T05:57:14.503 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"1e2340e7f1d4ec2c23448854649f1ded"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'OhfUPNapfP51-FnRny6JNTIZ-STnDSH'), ('x-amz-request-id', 'tx00000604c125e43651430-006312ecba-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.505 DEBUG:rgw_multi.tests:version3 id=OhfUPNapfP51-FnRny6JNTIZ-STnDSH 2022-09-03T05:57:14.506 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.507 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:57:14.508 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=OhfUPNapfP51-FnRny6JNTIZ-STnDSH 2022-09-03T05:57:14.509 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=OhfUPNapfP51-FnRny6JNTIZ-STnDSH 2022-09-03T05:57:14.510 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.510 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=OhfUPNapfP51-FnRny6JNTIZ-STnDSH 2022-09-03T05:57:14.511 DEBUG:boto:Data: 2022-09-03T05:57:14.512 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.514 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.515 DEBUG:boto:Params: {} 2022-09-03T05:57:14.516 DEBUG:boto:Token: None 2022-09-03T05:57:14.517 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-16/obj-test-zone3?versionId=OhfUPNapfP51-FnRny6JNTIZ-STnDSH 2022-09-03T05:57:14.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6jDpImvnWwu/myN8x3utbyBdJ8Y= 2022-09-03T05:57:14.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6jDpImvnWwu/myN8x3utbyBdJ8Y=', 'Content-Length': '0'} 2022-09-03T05:57:14.539 DEBUG:boto:Response headers: [('x-amz-version-id', 'OhfUPNapfP51-FnRny6JNTIZ-STnDSH'), ('x-amz-request-id', 'tx00000b4d2cdd7a4478d75-006312ecba-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.540 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:57:14.540 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:57:14.541 DEBUG:boto:Method: HEAD 2022-09-03T05:57:14.542 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:57:14.543 DEBUG:boto:Data: 2022-09-03T05:57:14.544 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.546 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.547 DEBUG:boto:Params: {} 2022-09-03T05:57:14.548 DEBUG:boto:Token: None 2022-09-03T05:57:14.548 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/ 2022-09-03T05:57:14.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5A9A9xltFXKJN09EqU5ErSepH8E= 2022-09-03T05:57:14.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5A9A9xltFXKJN09EqU5ErSepH8E=', 'Content-Length': '0'} 2022-09-03T05:57:14.552 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000745d119fc2edb7da-006312ecba-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.553 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.554 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.554 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.555 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T05:57:14.555 DEBUG:boto:Data: 2022-09-03T05:57:14.556 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.557 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.557 DEBUG:boto:Params: {} 2022-09-03T05:57:14.558 DEBUG:boto:Token: None 2022-09-03T05:57:14.558 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone1 2022-09-03T05:57:14.559 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbtCsZ5BtkQ1g7vA5050mPy4wT0= 2022-09-03T05:57:14.560 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbtCsZ5BtkQ1g7vA5050mPy4wT0='} 2022-09-03T05:57:14.578 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-amz-request-id', 'tx00000b87c7eb208d47fe0-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.579 DEBUG:rgw_multi.tests:version1 id=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:57:14.579 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.580 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.580 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.581 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T05:57:14.581 DEBUG:boto:Data: 2022-09-03T05:57:14.582 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.583 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.583 DEBUG:boto:Params: {} 2022-09-03T05:57:14.584 DEBUG:boto:Token: None 2022-09-03T05:57:14.585 DEBUG:boto:StringToSign: PUT Lg6VKF8IoH3qF+fuERshyA== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone1 2022-09-03T05:57:14.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+phRXtLqAGy4UPBNeSDGaHOUlHo= 2022-09-03T05:57:14.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+phRXtLqAGy4UPBNeSDGaHOUlHo='} 2022-09-03T05:57:14.599 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"2e0e95285f08a07dea17e7ee111b21c8"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'sh87lwfyEVlVYxtzYyxHvEAWWYaFsNO'), ('x-amz-request-id', 'tx000000ecb52c8d22319ed-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.600 DEBUG:rgw_multi.tests:version2 id=sh87lwfyEVlVYxtzYyxHvEAWWYaFsNO 2022-09-03T05:57:14.601 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.601 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.602 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sh87lwfyEVlVYxtzYyxHvEAWWYaFsNO 2022-09-03T05:57:14.602 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sh87lwfyEVlVYxtzYyxHvEAWWYaFsNO 2022-09-03T05:57:14.603 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.603 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sh87lwfyEVlVYxtzYyxHvEAWWYaFsNO 2022-09-03T05:57:14.604 DEBUG:boto:Data: 2022-09-03T05:57:14.604 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.605 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.606 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.606 DEBUG:boto:Params: {} 2022-09-03T05:57:14.607 DEBUG:boto:Token: None 2022-09-03T05:57:14.608 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone1?versionId=sh87lwfyEVlVYxtzYyxHvEAWWYaFsNO 2022-09-03T05:57:14.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G1SO8z6vu3lfe+CrZRvYH9DBLDk= 2022-09-03T05:57:14.610 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G1SO8z6vu3lfe+CrZRvYH9DBLDk=', 'Content-Length': '0'} 2022-09-03T05:57:14.630 DEBUG:boto:Response headers: [('x-amz-version-id', 'sh87lwfyEVlVYxtzYyxHvEAWWYaFsNO'), ('x-amz-request-id', 'tx000008f8542c22db0f0d5-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.631 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.631 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.632 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.632 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T05:57:14.633 DEBUG:boto:Data: 2022-09-03T05:57:14.633 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.634 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.635 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.635 DEBUG:boto:Params: {} 2022-09-03T05:57:14.636 DEBUG:boto:Token: None 2022-09-03T05:57:14.637 DEBUG:boto:StringToSign: PUT HiNA5/HU7CwjRIhUZJ8d7Q== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone1 2022-09-03T05:57:14.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CQvYiDxWNGWdPvr62J6riAYOtSk= 2022-09-03T05:57:14.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CQvYiDxWNGWdPvr62J6riAYOtSk='} 2022-09-03T05:57:14.686 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"1e2340e7f1d4ec2c23448854649f1ded"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '9qzYQkxdH4KExWxifGWWa.oZcXUTvPp'), ('x-amz-request-id', 'tx00000fc7d70422cb84fd9-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.687 DEBUG:rgw_multi.tests:version3 id=9qzYQkxdH4KExWxifGWWa.oZcXUTvPp 2022-09-03T05:57:14.688 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.688 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:57:14.689 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=9qzYQkxdH4KExWxifGWWa.oZcXUTvPp 2022-09-03T05:57:14.689 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=9qzYQkxdH4KExWxifGWWa.oZcXUTvPp 2022-09-03T05:57:14.690 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.690 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=9qzYQkxdH4KExWxifGWWa.oZcXUTvPp 2022-09-03T05:57:14.691 DEBUG:boto:Data: 2022-09-03T05:57:14.691 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.692 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:14.692 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.693 DEBUG:boto:Params: {} 2022-09-03T05:57:14.694 DEBUG:boto:Token: None 2022-09-03T05:57:14.694 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone1?versionId=9qzYQkxdH4KExWxifGWWa.oZcXUTvPp 2022-09-03T05:57:14.695 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q68JxEI1WXU10JhHVrvjLlKiDZM= 2022-09-03T05:57:14.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q68JxEI1WXU10JhHVrvjLlKiDZM=', 'Content-Length': '0'} 2022-09-03T05:57:14.706 DEBUG:boto:Response headers: [('x-amz-version-id', '9qzYQkxdH4KExWxifGWWa.oZcXUTvPp'), ('x-amz-request-id', 'tx000005dd4caef835c9e92-006312ecba-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.706 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:57:14.707 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:57:14.707 DEBUG:boto:Method: HEAD 2022-09-03T05:57:14.708 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:57:14.709 DEBUG:boto:Data: 2022-09-03T05:57:14.709 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.711 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.712 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.713 DEBUG:boto:Params: {} 2022-09-03T05:57:14.714 DEBUG:boto:Token: None 2022-09-03T05:57:14.715 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/ 2022-09-03T05:57:14.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5A9A9xltFXKJN09EqU5ErSepH8E= 2022-09-03T05:57:14.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5A9A9xltFXKJN09EqU5ErSepH8E=', 'Content-Length': '0'} 2022-09-03T05:57:14.720 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000020d70f798b164a9a-006312ecba-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.721 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.722 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.723 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.724 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T05:57:14.725 DEBUG:boto:Data: 2022-09-03T05:57:14.726 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.726 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.727 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.727 DEBUG:boto:Params: {} 2022-09-03T05:57:14.728 DEBUG:boto:Token: None 2022-09-03T05:57:14.728 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone2 2022-09-03T05:57:14.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gHGXTpuhGasIIEWwGKmtanhV7zI= 2022-09-03T05:57:14.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gHGXTpuhGasIIEWwGKmtanhV7zI='} 2022-09-03T05:57:14.749 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-amz-request-id', 'tx00000e2b86cd8e905fac5-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.750 DEBUG:rgw_multi.tests:version1 id=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:57:14.750 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.751 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.751 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.752 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T05:57:14.752 DEBUG:boto:Data: 2022-09-03T05:57:14.753 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.753 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.755 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.756 DEBUG:boto:Params: {} 2022-09-03T05:57:14.757 DEBUG:boto:Token: None 2022-09-03T05:57:14.757 DEBUG:boto:StringToSign: PUT Lg6VKF8IoH3qF+fuERshyA== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone2 2022-09-03T05:57:14.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PaTPn4KZxNxIDLu1r776x8wlkLo= 2022-09-03T05:57:14.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PaTPn4KZxNxIDLu1r776x8wlkLo='} 2022-09-03T05:57:14.771 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"2e0e95285f08a07dea17e7ee111b21c8"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'BAjCcfuvwF2vpRnb-1.e5BQKMhQmEd7'), ('x-amz-request-id', 'tx00000ee1d7ae36a6bb6a4-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.772 DEBUG:rgw_multi.tests:version2 id=BAjCcfuvwF2vpRnb-1.e5BQKMhQmEd7 2022-09-03T05:57:14.773 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.773 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.774 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=BAjCcfuvwF2vpRnb-1.e5BQKMhQmEd7 2022-09-03T05:57:14.774 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=BAjCcfuvwF2vpRnb-1.e5BQKMhQmEd7 2022-09-03T05:57:14.775 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.775 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=BAjCcfuvwF2vpRnb-1.e5BQKMhQmEd7 2022-09-03T05:57:14.776 DEBUG:boto:Data: 2022-09-03T05:57:14.776 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.777 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.777 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.778 DEBUG:boto:Params: {} 2022-09-03T05:57:14.778 DEBUG:boto:Token: None 2022-09-03T05:57:14.779 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone2?versionId=BAjCcfuvwF2vpRnb-1.e5BQKMhQmEd7 2022-09-03T05:57:14.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nwPc4OQxPSh1+hdn+C3Cjc1jOVo= 2022-09-03T05:57:14.780 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nwPc4OQxPSh1+hdn+C3Cjc1jOVo=', 'Content-Length': '0'} 2022-09-03T05:57:14.792 DEBUG:boto:Response headers: [('x-amz-version-id', 'BAjCcfuvwF2vpRnb-1.e5BQKMhQmEd7'), ('x-amz-request-id', 'tx00000fb39a95a2058177d-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.794 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.795 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.795 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.796 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T05:57:14.797 DEBUG:boto:Data: 2022-09-03T05:57:14.798 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.799 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.800 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.801 DEBUG:boto:Params: {} 2022-09-03T05:57:14.802 DEBUG:boto:Token: None 2022-09-03T05:57:14.803 DEBUG:boto:StringToSign: PUT HiNA5/HU7CwjRIhUZJ8d7Q== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone2 2022-09-03T05:57:14.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iAwCgiLgTCgLQ5bUHOqQRHgbH/0= 2022-09-03T05:57:14.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iAwCgiLgTCgLQ5bUHOqQRHgbH/0='} 2022-09-03T05:57:14.855 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"1e2340e7f1d4ec2c23448854649f1ded"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'yoKUwfO-WkSJk8GpTYYztHSUVX1vQLf'), ('x-amz-request-id', 'tx000006b6fca34c2cb071e-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.856 DEBUG:rgw_multi.tests:version3 id=yoKUwfO-WkSJk8GpTYYztHSUVX1vQLf 2022-09-03T05:57:14.857 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.858 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:57:14.859 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=yoKUwfO-WkSJk8GpTYYztHSUVX1vQLf 2022-09-03T05:57:14.860 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=yoKUwfO-WkSJk8GpTYYztHSUVX1vQLf 2022-09-03T05:57:14.861 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.861 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=yoKUwfO-WkSJk8GpTYYztHSUVX1vQLf 2022-09-03T05:57:14.862 DEBUG:boto:Data: 2022-09-03T05:57:14.863 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.863 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:14.864 DEBUG:boto:Port: 8000 2022-09-03T05:57:14.864 DEBUG:boto:Params: {} 2022-09-03T05:57:14.865 DEBUG:boto:Token: None 2022-09-03T05:57:14.866 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone2?versionId=yoKUwfO-WkSJk8GpTYYztHSUVX1vQLf 2022-09-03T05:57:14.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xmgziC9WvsN9vZow+Cp3HWEupqk= 2022-09-03T05:57:14.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xmgziC9WvsN9vZow+Cp3HWEupqk=', 'Content-Length': '0'} 2022-09-03T05:57:14.879 DEBUG:boto:Response headers: [('x-amz-version-id', 'yoKUwfO-WkSJk8GpTYYztHSUVX1vQLf'), ('x-amz-request-id', 'tx000001299eab9239ad934-006312ecba-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.880 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:57:14.881 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:57:14.882 DEBUG:boto:Method: HEAD 2022-09-03T05:57:14.882 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:57:14.883 DEBUG:boto:Data: 2022-09-03T05:57:14.883 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.884 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.885 DEBUG:boto:Params: {} 2022-09-03T05:57:14.885 DEBUG:boto:Token: None 2022-09-03T05:57:14.886 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/ 2022-09-03T05:57:14.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5A9A9xltFXKJN09EqU5ErSepH8E= 2022-09-03T05:57:14.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5A9A9xltFXKJN09EqU5ErSepH8E=', 'Content-Length': '0'} 2022-09-03T05:57:14.890 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e7bef9adb6fbb47f-006312ecba-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.891 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:14.892 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:14.893 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.894 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T05:57:14.895 DEBUG:boto:Data: 2022-09-03T05:57:14.895 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.897 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.898 DEBUG:boto:Params: {} 2022-09-03T05:57:14.899 DEBUG:boto:Token: None 2022-09-03T05:57:14.900 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone3 2022-09-03T05:57:14.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U2zafA+kM4dNrfDkmytCspnBnuI= 2022-09-03T05:57:14.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U2zafA+kM4dNrfDkmytCspnBnuI='} 2022-09-03T05:57:14.912 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-amz-request-id', 'tx00000b2068fb65172b20d-006312ecba-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.913 DEBUG:rgw_multi.tests:version1 id=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:57:14.913 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:14.914 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:14.914 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.915 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T05:57:14.915 DEBUG:boto:Data: 2022-09-03T05:57:14.916 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.917 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.917 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.918 DEBUG:boto:Params: {} 2022-09-03T05:57:14.918 DEBUG:boto:Token: None 2022-09-03T05:57:14.919 DEBUG:boto:StringToSign: PUT Lg6VKF8IoH3qF+fuERshyA== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone3 2022-09-03T05:57:14.920 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pWAArjAyEBsX2f4D8UKXh8wG2XQ= 2022-09-03T05:57:14.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pWAArjAyEBsX2f4D8UKXh8wG2XQ='} 2022-09-03T05:57:14.930 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"2e0e95285f08a07dea17e7ee111b21c8"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'lE0sUnHOzfPND0GY9otAZ5OHGArkjxH'), ('x-amz-request-id', 'tx0000071330a0656e3f5bc-006312ecba-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.931 DEBUG:rgw_multi.tests:version2 id=lE0sUnHOzfPND0GY9otAZ5OHGArkjxH 2022-09-03T05:57:14.932 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:14.932 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:14.933 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=lE0sUnHOzfPND0GY9otAZ5OHGArkjxH 2022-09-03T05:57:14.933 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=lE0sUnHOzfPND0GY9otAZ5OHGArkjxH 2022-09-03T05:57:14.934 DEBUG:boto:Method: DELETE 2022-09-03T05:57:14.935 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=lE0sUnHOzfPND0GY9otAZ5OHGArkjxH 2022-09-03T05:57:14.936 DEBUG:boto:Data: 2022-09-03T05:57:14.936 DEBUG:boto:Headers: {} 2022-09-03T05:57:14.937 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.937 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.938 DEBUG:boto:Params: {} 2022-09-03T05:57:14.940 DEBUG:boto:Token: None 2022-09-03T05:57:14.941 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone3?versionId=lE0sUnHOzfPND0GY9otAZ5OHGArkjxH 2022-09-03T05:57:14.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yakBYmmlVxiIxgeyJOFurRB8UwA= 2022-09-03T05:57:14.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yakBYmmlVxiIxgeyJOFurRB8UwA=', 'Content-Length': '0'} 2022-09-03T05:57:14.955 DEBUG:boto:Response headers: [('x-amz-version-id', 'lE0sUnHOzfPND0GY9otAZ5OHGArkjxH'), ('x-amz-request-id', 'tx0000064e1b31e27e7d064-006312ecba-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:14.957 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:14.957 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:14.958 DEBUG:boto:Method: PUT 2022-09-03T05:57:14.958 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T05:57:14.959 DEBUG:boto:Data: 2022-09-03T05:57:14.959 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:14.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:14.960 DEBUG:boto:Port: 8001 2022-09-03T05:57:14.962 DEBUG:boto:Params: {} 2022-09-03T05:57:14.962 DEBUG:boto:Token: None 2022-09-03T05:57:14.963 DEBUG:boto:StringToSign: PUT HiNA5/HU7CwjRIhUZJ8d7Q== application/octet-stream Sat, 03 Sep 2022 05:57:14 GMT /avysmw-17/obj-test-zone3 2022-09-03T05:57:14.964 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QA5q346Z25f7WgK4p9/RnDyhHrs= 2022-09-03T05:57:14.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QA5q346Z25f7WgK4p9/RnDyhHrs='} 2022-09-03T05:57:15.010 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"1e2340e7f1d4ec2c23448854649f1ded"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'j7tV8tIQa4Zo6tqwLHCd.j9DvYXuUSs'), ('x-amz-request-id', 'tx0000074eef21a12f709b3-006312ecba-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.012 DEBUG:rgw_multi.tests:version3 id=j7tV8tIQa4Zo6tqwLHCd.j9DvYXuUSs 2022-09-03T05:57:15.012 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:15.013 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:57:15.013 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=j7tV8tIQa4Zo6tqwLHCd.j9DvYXuUSs 2022-09-03T05:57:15.014 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=j7tV8tIQa4Zo6tqwLHCd.j9DvYXuUSs 2022-09-03T05:57:15.015 DEBUG:boto:Method: DELETE 2022-09-03T05:57:15.015 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=j7tV8tIQa4Zo6tqwLHCd.j9DvYXuUSs 2022-09-03T05:57:15.016 DEBUG:boto:Data: 2022-09-03T05:57:15.016 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:15.017 DEBUG:boto:Port: 8001 2022-09-03T05:57:15.018 DEBUG:boto:Params: {} 2022-09-03T05:57:15.018 DEBUG:boto:Token: None 2022-09-03T05:57:15.019 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:15 GMT /avysmw-17/obj-test-zone3?versionId=j7tV8tIQa4Zo6tqwLHCd.j9DvYXuUSs 2022-09-03T05:57:15.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KXvCY+1AWSit9uxCKZeeoVB/6Fs= 2022-09-03T05:57:15.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KXvCY+1AWSit9uxCKZeeoVB/6Fs=', 'Content-Length': '0'} 2022-09-03T05:57:15.040 DEBUG:boto:Response headers: [('x-amz-version-id', 'j7tV8tIQa4Zo6tqwLHCd.j9DvYXuUSs'), ('x-amz-request-id', 'tx00000bdacb85db64834eb-006312ecbb-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.041 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:57:15.042 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:57:15.042 DEBUG:boto:Method: HEAD 2022-09-03T05:57:15.043 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:57:15.043 DEBUG:boto:Data: 2022-09-03T05:57:15.044 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:15.045 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.045 DEBUG:boto:Params: {} 2022-09-03T05:57:15.046 DEBUG:boto:Token: None 2022-09-03T05:57:15.047 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/ 2022-09-03T05:57:15.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7vSatp9MWEhyrQi9YuqVv/EBKoo= 2022-09-03T05:57:15.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7vSatp9MWEhyrQi9YuqVv/EBKoo=', 'Content-Length': '0'} 2022-09-03T05:57:15.050 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e382ae2e22eb9ecc-006312ecbb-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.051 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.051 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.052 DEBUG:boto:Method: PUT 2022-09-03T05:57:15.052 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T05:57:15.053 DEBUG:boto:Data: 2022-09-03T05:57:15.053 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:15.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:15.054 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.055 DEBUG:boto:Params: {} 2022-09-03T05:57:15.056 DEBUG:boto:Token: None 2022-09-03T05:57:15.056 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone1 2022-09-03T05:57:15.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8tzgRON/idgUVQddYxgNRfjXvTI= 2022-09-03T05:57:15.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8tzgRON/idgUVQddYxgNRfjXvTI='} 2022-09-03T05:57:15.077 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-amz-request-id', 'tx000006355073db94a3e68-006312ecbb-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.078 DEBUG:rgw_multi.tests:version1 id=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:57:15.078 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.079 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.080 DEBUG:boto:Method: PUT 2022-09-03T05:57:15.080 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T05:57:15.080 DEBUG:boto:Data: 2022-09-03T05:57:15.081 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:15.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:15.083 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.084 DEBUG:boto:Params: {} 2022-09-03T05:57:15.085 DEBUG:boto:Token: None 2022-09-03T05:57:15.086 DEBUG:boto:StringToSign: PUT Lg6VKF8IoH3qF+fuERshyA== application/octet-stream Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone1 2022-09-03T05:57:15.087 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5/Dy7xt0pXhDQbr7VKdog+b7rOE= 2022-09-03T05:57:15.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5/Dy7xt0pXhDQbr7VKdog+b7rOE='} 2022-09-03T05:57:15.131 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"2e0e95285f08a07dea17e7ee111b21c8"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 't1JFMRv4r6MQasRBEnMjJ4H8D0TcQoG'), ('x-amz-request-id', 'tx000009f219bf232869a5e-006312ecbb-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.132 DEBUG:rgw_multi.tests:version2 id=t1JFMRv4r6MQasRBEnMjJ4H8D0TcQoG 2022-09-03T05:57:15.133 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.134 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.135 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=t1JFMRv4r6MQasRBEnMjJ4H8D0TcQoG 2022-09-03T05:57:15.136 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=t1JFMRv4r6MQasRBEnMjJ4H8D0TcQoG 2022-09-03T05:57:15.137 DEBUG:boto:Method: DELETE 2022-09-03T05:57:15.137 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=t1JFMRv4r6MQasRBEnMjJ4H8D0TcQoG 2022-09-03T05:57:15.139 DEBUG:boto:Data: 2022-09-03T05:57:15.140 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.141 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:15.142 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.142 DEBUG:boto:Params: {} 2022-09-03T05:57:15.143 DEBUG:boto:Token: None 2022-09-03T05:57:15.143 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone1?versionId=t1JFMRv4r6MQasRBEnMjJ4H8D0TcQoG 2022-09-03T05:57:15.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DWdhZcK3gZyKb9b8KYGRwCxOVwg= 2022-09-03T05:57:15.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DWdhZcK3gZyKb9b8KYGRwCxOVwg=', 'Content-Length': '0'} 2022-09-03T05:57:15.164 DEBUG:boto:Response headers: [('x-amz-version-id', 't1JFMRv4r6MQasRBEnMjJ4H8D0TcQoG'), ('x-amz-request-id', 'tx000000d949c50f8f2f1fe-006312ecbb-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.166 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.166 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.167 DEBUG:boto:Method: PUT 2022-09-03T05:57:15.167 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T05:57:15.168 DEBUG:boto:Data: 2022-09-03T05:57:15.168 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:15.169 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:15.170 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.171 DEBUG:boto:Params: {} 2022-09-03T05:57:15.172 DEBUG:boto:Token: None 2022-09-03T05:57:15.173 DEBUG:boto:StringToSign: PUT HiNA5/HU7CwjRIhUZJ8d7Q== application/octet-stream Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone1 2022-09-03T05:57:15.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4tmEnBazfFO6JPPNTh83xsnkwO4= 2022-09-03T05:57:15.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4tmEnBazfFO6JPPNTh83xsnkwO4='} 2022-09-03T05:57:15.191 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"1e2340e7f1d4ec2c23448854649f1ded"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'Ej4WiGax..OmNfIMj1ygjMqd6RRemND'), ('x-amz-request-id', 'tx00000b01fda0c35cbed50-006312ecbb-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.192 DEBUG:rgw_multi.tests:version3 id=Ej4WiGax..OmNfIMj1ygjMqd6RRemND 2022-09-03T05:57:15.193 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.193 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:57:15.194 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=Ej4WiGax..OmNfIMj1ygjMqd6RRemND 2022-09-03T05:57:15.195 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=Ej4WiGax..OmNfIMj1ygjMqd6RRemND 2022-09-03T05:57:15.196 DEBUG:boto:Method: DELETE 2022-09-03T05:57:15.196 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=Ej4WiGax..OmNfIMj1ygjMqd6RRemND 2022-09-03T05:57:15.197 DEBUG:boto:Data: 2022-09-03T05:57:15.197 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.198 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:57:15.199 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.199 DEBUG:boto:Params: {} 2022-09-03T05:57:15.200 DEBUG:boto:Token: None 2022-09-03T05:57:15.201 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone1?versionId=Ej4WiGax..OmNfIMj1ygjMqd6RRemND 2022-09-03T05:57:15.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYzrBLqSko3Pp11SHJbLW0FZFAg= 2022-09-03T05:57:15.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYzrBLqSko3Pp11SHJbLW0FZFAg=', 'Content-Length': '0'} 2022-09-03T05:57:15.219 DEBUG:boto:Response headers: [('x-amz-version-id', 'Ej4WiGax..OmNfIMj1ygjMqd6RRemND'), ('x-amz-request-id', 'tx000005dc8ed3e6d7cf65c-006312ecbb-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.220 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:57:15.221 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:57:15.221 DEBUG:boto:Method: HEAD 2022-09-03T05:57:15.222 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:57:15.222 DEBUG:boto:Data: 2022-09-03T05:57:15.223 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.223 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:15.224 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.224 DEBUG:boto:Params: {} 2022-09-03T05:57:15.225 DEBUG:boto:Token: None 2022-09-03T05:57:15.225 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/ 2022-09-03T05:57:15.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7vSatp9MWEhyrQi9YuqVv/EBKoo= 2022-09-03T05:57:15.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7vSatp9MWEhyrQi9YuqVv/EBKoo=', 'Content-Length': '0'} 2022-09-03T05:57:15.229 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004a126411d2873421-006312ecbb-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.230 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.230 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.231 DEBUG:boto:Method: PUT 2022-09-03T05:57:15.232 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T05:57:15.232 DEBUG:boto:Data: 2022-09-03T05:57:15.233 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:15.233 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:15.234 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.234 DEBUG:boto:Params: {} 2022-09-03T05:57:15.235 DEBUG:boto:Token: None 2022-09-03T05:57:15.235 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone2 2022-09-03T05:57:15.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/tqIHyKN09An8yO3V6FDorafBnQ= 2022-09-03T05:57:15.236 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/tqIHyKN09An8yO3V6FDorafBnQ='} 2022-09-03T05:57:15.249 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-amz-request-id', 'tx0000003b7268bcbaacc48-006312ecbb-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.251 DEBUG:rgw_multi.tests:version1 id=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:57:15.251 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.252 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.253 DEBUG:boto:Method: PUT 2022-09-03T05:57:15.253 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T05:57:15.254 DEBUG:boto:Data: 2022-09-03T05:57:15.255 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:15.256 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:15.256 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.257 DEBUG:boto:Params: {} 2022-09-03T05:57:15.258 DEBUG:boto:Token: None 2022-09-03T05:57:15.258 DEBUG:boto:StringToSign: PUT Lg6VKF8IoH3qF+fuERshyA== application/octet-stream Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone2 2022-09-03T05:57:15.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2VgGuRy9rXcOuH1uG8EceAkjcbY= 2022-09-03T05:57:15.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2VgGuRy9rXcOuH1uG8EceAkjcbY='} 2022-09-03T05:57:15.270 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"2e0e95285f08a07dea17e7ee111b21c8"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'rsSekueOgPkJ-NtexJXyY3IpkYmcKln'), ('x-amz-request-id', 'tx00000088ca64571bdeb02-006312ecbb-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.272 DEBUG:rgw_multi.tests:version2 id=rsSekueOgPkJ-NtexJXyY3IpkYmcKln 2022-09-03T05:57:15.272 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.273 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.273 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=rsSekueOgPkJ-NtexJXyY3IpkYmcKln 2022-09-03T05:57:15.275 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=rsSekueOgPkJ-NtexJXyY3IpkYmcKln 2022-09-03T05:57:15.276 DEBUG:boto:Method: DELETE 2022-09-03T05:57:15.277 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=rsSekueOgPkJ-NtexJXyY3IpkYmcKln 2022-09-03T05:57:15.278 DEBUG:boto:Data: 2022-09-03T05:57:15.279 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.279 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:15.280 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.280 DEBUG:boto:Params: {} 2022-09-03T05:57:15.281 DEBUG:boto:Token: None 2022-09-03T05:57:15.282 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone2?versionId=rsSekueOgPkJ-NtexJXyY3IpkYmcKln 2022-09-03T05:57:15.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j9TrefzwuKcOpCRpG07yBwKAe2I= 2022-09-03T05:57:15.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j9TrefzwuKcOpCRpG07yBwKAe2I=', 'Content-Length': '0'} 2022-09-03T05:57:15.305 DEBUG:boto:Response headers: [('x-amz-version-id', 'rsSekueOgPkJ-NtexJXyY3IpkYmcKln'), ('x-amz-request-id', 'tx00000c0c1a9cdd1e2941b-006312ecbb-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.306 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.307 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.307 DEBUG:boto:Method: PUT 2022-09-03T05:57:15.308 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T05:57:15.309 DEBUG:boto:Data: 2022-09-03T05:57:15.310 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:15.311 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:15.312 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.313 DEBUG:boto:Params: {} 2022-09-03T05:57:15.314 DEBUG:boto:Token: None 2022-09-03T05:57:15.315 DEBUG:boto:StringToSign: PUT HiNA5/HU7CwjRIhUZJ8d7Q== application/octet-stream Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone2 2022-09-03T05:57:15.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xwr3galAorMKDaRLBCOos38VPuQ= 2022-09-03T05:57:15.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xwr3galAorMKDaRLBCOos38VPuQ='} 2022-09-03T05:57:15.366 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"1e2340e7f1d4ec2c23448854649f1ded"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'ugr.GhEw7y.N8eph7USG9n90--1vYgb'), ('x-amz-request-id', 'tx000003eb0c95f1626d7b5-006312ecbb-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.368 DEBUG:rgw_multi.tests:version3 id=ugr.GhEw7y.N8eph7USG9n90--1vYgb 2022-09-03T05:57:15.369 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.370 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:57:15.371 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=ugr.GhEw7y.N8eph7USG9n90--1vYgb 2022-09-03T05:57:15.371 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=ugr.GhEw7y.N8eph7USG9n90--1vYgb 2022-09-03T05:57:15.372 DEBUG:boto:Method: DELETE 2022-09-03T05:57:15.372 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=ugr.GhEw7y.N8eph7USG9n90--1vYgb 2022-09-03T05:57:15.373 DEBUG:boto:Data: 2022-09-03T05:57:15.373 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.374 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:57:15.374 DEBUG:boto:Port: 8000 2022-09-03T05:57:15.375 DEBUG:boto:Params: {} 2022-09-03T05:57:15.375 DEBUG:boto:Token: None 2022-09-03T05:57:15.376 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone2?versionId=ugr.GhEw7y.N8eph7USG9n90--1vYgb 2022-09-03T05:57:15.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QKp/542/sOPs4/I4Tx2wCw78RRI= 2022-09-03T05:57:15.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QKp/542/sOPs4/I4Tx2wCw78RRI=', 'Content-Length': '0'} 2022-09-03T05:57:15.394 DEBUG:boto:Response headers: [('x-amz-version-id', 'ugr.GhEw7y.N8eph7USG9n90--1vYgb'), ('x-amz-request-id', 'tx0000046fcd795e76962c5-006312ecbb-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.395 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:57:15.396 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:57:15.396 DEBUG:boto:Method: HEAD 2022-09-03T05:57:15.397 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:57:15.397 DEBUG:boto:Data: 2022-09-03T05:57:15.398 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.398 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:15.399 DEBUG:boto:Port: 8001 2022-09-03T05:57:15.399 DEBUG:boto:Params: {} 2022-09-03T05:57:15.400 DEBUG:boto:Token: None 2022-09-03T05:57:15.400 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/ 2022-09-03T05:57:15.401 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7vSatp9MWEhyrQi9YuqVv/EBKoo= 2022-09-03T05:57:15.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7vSatp9MWEhyrQi9YuqVv/EBKoo=', 'Content-Length': '0'} 2022-09-03T05:57:15.404 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b80cf0d252fc1e53-006312ecbb-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.405 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.406 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.406 DEBUG:boto:Method: PUT 2022-09-03T05:57:15.407 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T05:57:15.407 DEBUG:boto:Data: 2022-09-03T05:57:15.408 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:15.408 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:15.409 DEBUG:boto:Port: 8001 2022-09-03T05:57:15.409 DEBUG:boto:Params: {} 2022-09-03T05:57:15.410 DEBUG:boto:Token: None 2022-09-03T05:57:15.410 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone3 2022-09-03T05:57:15.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qXkUbxZTVvFDe/+XWh1mjT0VfNg= 2022-09-03T05:57:15.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qXkUbxZTVvFDe/+XWh1mjT0VfNg='} 2022-09-03T05:57:15.426 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-amz-request-id', 'tx00000cb81c4c63b379477-006312ecbb-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.428 DEBUG:rgw_multi.tests:version1 id=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:57:15.428 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.429 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.430 DEBUG:boto:Method: PUT 2022-09-03T05:57:15.431 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T05:57:15.431 DEBUG:boto:Data: 2022-09-03T05:57:15.432 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:15.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:15.433 DEBUG:boto:Port: 8001 2022-09-03T05:57:15.433 DEBUG:boto:Params: {} 2022-09-03T05:57:15.434 DEBUG:boto:Token: None 2022-09-03T05:57:15.434 DEBUG:boto:StringToSign: PUT Lg6VKF8IoH3qF+fuERshyA== application/octet-stream Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone3 2022-09-03T05:57:15.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KoWMZm4BdX/+Fy8x8F1wkQOnqXo= 2022-09-03T05:57:15.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'Lg6VKF8IoH3qF+fuERshyA==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KoWMZm4BdX/+Fy8x8F1wkQOnqXo='} 2022-09-03T05:57:15.455 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"2e0e95285f08a07dea17e7ee111b21c8"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'xrj8HUlHOjyn7raWE-o9-EQU.iOFG1.'), ('x-amz-request-id', 'tx000005388131fc7204387-006312ecbb-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.455 DEBUG:rgw_multi.tests:version2 id=xrj8HUlHOjyn7raWE-o9-EQU.iOFG1. 2022-09-03T05:57:15.456 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.457 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.457 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=xrj8HUlHOjyn7raWE-o9-EQU.iOFG1. 2022-09-03T05:57:15.457 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=xrj8HUlHOjyn7raWE-o9-EQU.iOFG1. 2022-09-03T05:57:15.458 DEBUG:boto:Method: DELETE 2022-09-03T05:57:15.458 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=xrj8HUlHOjyn7raWE-o9-EQU.iOFG1. 2022-09-03T05:57:15.459 DEBUG:boto:Data: 2022-09-03T05:57:15.459 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:15.460 DEBUG:boto:Port: 8001 2022-09-03T05:57:15.461 DEBUG:boto:Params: {} 2022-09-03T05:57:15.461 DEBUG:boto:Token: None 2022-09-03T05:57:15.462 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone3?versionId=xrj8HUlHOjyn7raWE-o9-EQU.iOFG1. 2022-09-03T05:57:15.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S+03KJ0NBLts9W40r4Qki6mQneU= 2022-09-03T05:57:15.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S+03KJ0NBLts9W40r4Qki6mQneU=', 'Content-Length': '0'} 2022-09-03T05:57:15.474 DEBUG:boto:Response headers: [('x-amz-version-id', 'xrj8HUlHOjyn7raWE-o9-EQU.iOFG1.'), ('x-amz-request-id', 'tx00000fe8336c236f732ec-006312ecbb-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.475 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.476 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.476 DEBUG:boto:Method: PUT 2022-09-03T05:57:15.477 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T05:57:15.478 DEBUG:boto:Data: 2022-09-03T05:57:15.479 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T05:57:15.480 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:15.480 DEBUG:boto:Port: 8001 2022-09-03T05:57:15.481 DEBUG:boto:Params: {} 2022-09-03T05:57:15.482 DEBUG:boto:Token: None 2022-09-03T05:57:15.483 DEBUG:boto:StringToSign: PUT HiNA5/HU7CwjRIhUZJ8d7Q== application/octet-stream Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone3 2022-09-03T05:57:15.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Exh2pykMwBFKAIVP3z2eJ0DwaP0= 2022-09-03T05:57:15.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'HiNA5/HU7CwjRIhUZJ8d7Q==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Exh2pykMwBFKAIVP3z2eJ0DwaP0='} 2022-09-03T05:57:15.530 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"1e2340e7f1d4ec2c23448854649f1ded"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'uk2ZXPLPnchTD7iiYFAuolOLvtYz-ux'), ('x-amz-request-id', 'tx000000af20bbb05f861ee-006312ecbb-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.531 DEBUG:rgw_multi.tests:version3 id=uk2ZXPLPnchTD7iiYFAuolOLvtYz-ux 2022-09-03T05:57:15.532 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.533 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:57:15.533 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=uk2ZXPLPnchTD7iiYFAuolOLvtYz-ux 2022-09-03T05:57:15.534 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=uk2ZXPLPnchTD7iiYFAuolOLvtYz-ux 2022-09-03T05:57:15.534 DEBUG:boto:Method: DELETE 2022-09-03T05:57:15.535 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=uk2ZXPLPnchTD7iiYFAuolOLvtYz-ux 2022-09-03T05:57:15.535 DEBUG:boto:Data: 2022-09-03T05:57:15.536 DEBUG:boto:Headers: {} 2022-09-03T05:57:15.536 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:57:15.537 DEBUG:boto:Port: 8001 2022-09-03T05:57:15.537 DEBUG:boto:Params: {} 2022-09-03T05:57:15.538 DEBUG:boto:Token: None 2022-09-03T05:57:15.539 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 05:57:15 GMT /avysmw-18/obj-test-zone3?versionId=uk2ZXPLPnchTD7iiYFAuolOLvtYz-ux 2022-09-03T05:57:15.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hAjccyW2Xpu+CZkW9KLGNb1hIrQ= 2022-09-03T05:57:15.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:57:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hAjccyW2Xpu+CZkW9KLGNb1hIrQ=', 'Content-Length': '0'} 2022-09-03T05:57:15.550 DEBUG:boto:Response headers: [('x-amz-version-id', 'uk2ZXPLPnchTD7iiYFAuolOLvtYz-ux'), ('x-amz-request-id', 'tx00000d3432a1ceffa6c92-006312ecbb-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:57:15.550 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-16 2022-09-03T05:57:15.551 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:57:15.552 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:57:15.552 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:57:15.616 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:57:15.616 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:57:15.664 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:57:15.662+0000 7f31edd59600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:15.665 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000004.17.9, , , , , ] 2022-09-03T05:57:15.665 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , , , 00000000005.19.9, , , , , ] 2022-09-03T05:57:20.666 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:57:20.664+0000 7f31edd59600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:20.666 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000004.17.9, , , , , ] 2022-09-03T05:57:20.667 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , , , 00000000005.19.9, , , , , ] 2022-09-03T05:57:25.667 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:57:25.665+0000 7f31edd59600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:25.668 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000004.17.9, , , , , ] 2022-09-03T05:57:25.668 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , , , 00000000005.19.9, , , , , ] 2022-09-03T05:57:30.669 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:57:30.666+0000 7f31edd59600 1 bucket sync caught up with source: 2022-09-03T05:57:30.669 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:30.670 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , , , 00000000005.19.9, , , , , ] 2022-09-03T05:57:30.670 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:57:30.666+0000 7f31edd59600 0 bucket checkpoint complete 2022-09-03T05:57:30.682 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-16 2022-09-03T05:57:30.683 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:57:30.683 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:57:30.684 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:57:30.751 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:57:30.751 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:57:30.793 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:57:30.791+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:30.793 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:30.794 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:35.794 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:57:35.793+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:35.795 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:35.795 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:40.796 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:57:40.794+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:40.797 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:40.797 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:45.797 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:57:45.796+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:45.798 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:45.798 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:50.799 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:57:50.797+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:50.800 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:50.800 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:55.800 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:57:55.798+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:57:55.801 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:57:55.801 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:00.802 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:58:00.799+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:00.803 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:00.803 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:05.803 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:58:05.801+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:05.804 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:05.804 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:10.805 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:58:10.802+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:10.805 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:10.805 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:15.806 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:58:15.804+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:15.807 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:15.807 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:20.808 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:58:20.806+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:20.808 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:20.809 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:25.811 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:58:25.807+0000 7fe9619a7600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:25.812 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000008.26.11, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:25.812 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:30.811 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:58:30.809+0000 7fe9619a7600 1 bucket sync caught up with source: 2022-09-03T05:58:30.811 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:30.811 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.30.5, , 00000000002.5.10, , 00000000008.24.10, , , , , ] 2022-09-03T05:58:30.812 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:58:30.809+0000 7fe9619a7600 0 bucket checkpoint complete 2022-09-03T05:58:30.828 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-16 2022-09-03T05:58:30.829 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:58:30.830 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:58:30.830 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:58:30.894 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:58:30.895 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:58:30.951 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:58:30.949+0000 7fae9b9c0600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:30.951 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000002.5.10, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:30.952 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:35.953 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:58:35.950+0000 7fae9b9c0600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:35.954 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000002.5.10, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:35.954 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:40.954 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:58:40.952+0000 7fae9b9c0600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:40.955 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000002.5.10, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:40.955 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:45.955 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:58:45.953+0000 7fae9b9c0600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:45.956 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000002.5.10, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:45.956 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:50.957 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:58:50.954+0000 7fae9b9c0600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:50.958 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000002.5.10, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:50.958 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:55.958 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:58:55.956+0000 7fae9b9c0600 1 waiting for incremental sync to catch up: 2022-09-03T05:58:55.959 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000002.5.10, , 00000000009.27.9, , , , , ] 2022-09-03T05:58:55.959 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:59:00.959 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:00.957+0000 7fae9b9c0600 1 bucket sync caught up with source: 2022-09-03T05:59:00.960 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:59:00.960 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:59:00.961 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:00.957+0000 7fae9b9c0600 0 bucket checkpoint complete 2022-09-03T05:59:00.981 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-16 2022-09-03T05:59:00.982 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:00.983 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:00.983 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:01.050 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:01.050 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:01.090 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:01.087+0000 7f97a38e1600 1 bucket sync caught up with source: 2022-09-03T05:59:01.090 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.34.9, , 00000000010.29.5, , 00000000007.40.9, , , , , ] 2022-09-03T05:59:01.091 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.34.9, , 00000000010.29.5, , 00000000007.40.9, , , , , ] 2022-09-03T05:59:01.091 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:01.087+0000 7f97a38e1600 0 bucket checkpoint complete 2022-09-03T05:59:01.103 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-16 2022-09-03T05:59:01.104 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:01.105 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:01.105 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:01.165 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:01.166 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:01.202 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:01.199+0000 7f4c1495c600 1 bucket sync caught up with source: 2022-09-03T05:59:01.203 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.34.9, , 00000000010.29.5, , 00000000007.40.9, , , , , ] 2022-09-03T05:59:01.203 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.34.9, , 00000000010.29.5, , 00000000007.40.9, , , , , ] 2022-09-03T05:59:01.203 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:01.199+0000 7f4c1495c600 0 bucket checkpoint complete 2022-09-03T05:59:01.224 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-16 2022-09-03T05:59:01.225 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-16 2022-09-03T05:59:01.225 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:01.226 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:01.227 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:01.285 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:01.285 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:01.327 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:01.324+0000 7fade39bc600 1 bucket sync caught up with source: 2022-09-03T05:59:01.327 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.45.9, , 00000000003.23.9, , 00000000014.60.5, , , , , ] 2022-09-03T05:59:01.328 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.45.9, , 00000000003.23.9, , 00000000014.60.5, , , , , ] 2022-09-03T05:59:01.328 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:01.324+0000 7fade39bc600 0 bucket checkpoint complete 2022-09-03T05:59:01.343 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-16 2022-09-03T05:59:01.344 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:01.345 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:01.345 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:01.408 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:01.408 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:01.453 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:01.451+0000 7f3fe49b5600 1 bucket sync caught up with source: 2022-09-03T05:59:01.454 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000003.45.9, , 00000000003.23.9, , 00000000014.60.5, , , , , ] 2022-09-03T05:59:01.454 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000003.45.9, , 00000000003.23.9, , 00000000014.60.5, , , , , ] 2022-09-03T05:59:01.455 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:01.451+0000 7f3fe49b5600 0 bucket checkpoint complete 2022-09-03T05:59:01.469 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-16 2022-09-03T05:59:01.470 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-16 zones={test-zone2, test-zone1} 2022-09-03T05:59:01.471 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.471 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.472 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.473 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:59:01.473 DEBUG:boto:Data: 2022-09-03T05:59:01.474 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.474 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.475 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.475 DEBUG:boto:Params: {} 2022-09-03T05:59:01.476 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T05:59:01.477 DEBUG:boto:Token: None 2022-09-03T05:59:01.478 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/ 2022-09-03T05:59:01.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KW/8aORfyrej9w4killuseItVis= 2022-09-03T05:59:01.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KW/8aORfyrej9w4killuseItVis=', 'Content-Length': '0'} 2022-09-03T05:59:01.484 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000050ababd82a908dc7-006312ed25-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.485 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.486 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.487 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.487 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:59:01.488 DEBUG:boto:Data: 2022-09-03T05:59:01.489 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.490 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.491 DEBUG:boto:Params: {} 2022-09-03T05:59:01.492 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T05:59:01.493 DEBUG:boto:Token: None 2022-09-03T05:59:01.493 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/ 2022-09-03T05:59:01.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KW/8aORfyrej9w4killuseItVis= 2022-09-03T05:59:01.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KW/8aORfyrej9w4killuseItVis=', 'Content-Length': '0'} 2022-09-03T05:59:01.499 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005a091041a9f1dc07-006312ed25-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.500 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:59:01.501 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.501 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.502 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.503 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.503 DEBUG:boto:Method: GET 2022-09-03T05:59:01.504 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.504 DEBUG:boto:Data: 2022-09-03T05:59:01.505 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.506 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.506 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.507 DEBUG:boto:Params: {} 2022-09-03T05:59:01.508 DEBUG:boto:Token: None 2022-09-03T05:59:01.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/?versions 2022-09-03T05:59:01.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bS0fpcj9A/9lGPf6DbIau1IsoKI= 2022-09-03T05:59:01.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bS0fpcj9A/9lGPf6DbIau1IsoKI=', 'Content-Length': '0'} 2022-09-03T05:59:01.513 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6d4911f1fb798af-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.514 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.516 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:01.516 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:01.517 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:01.518 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:01.519 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:59:01.520 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.520 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.521 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.521 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.522 DEBUG:boto:Method: GET 2022-09-03T05:59:01.522 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.523 DEBUG:boto:Data: 2022-09-03T05:59:01.523 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.524 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.525 DEBUG:boto:Params: {} 2022-09-03T05:59:01.526 DEBUG:boto:Token: None 2022-09-03T05:59:01.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/?versions 2022-09-03T05:59:01.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bS0fpcj9A/9lGPf6DbIau1IsoKI= 2022-09-03T05:59:01.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bS0fpcj9A/9lGPf6DbIau1IsoKI=', 'Content-Length': '0'} 2022-09-03T05:59:01.531 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e70551cd80698d01-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.531 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.532 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:01.533 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:01.533 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:01.534 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:01.534 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.535 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.536 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.536 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.537 DEBUG:boto:Method: GET 2022-09-03T05:59:01.537 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.538 DEBUG:boto:Data: 2022-09-03T05:59:01.538 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.539 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.539 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.540 DEBUG:boto:Params: {} 2022-09-03T05:59:01.541 DEBUG:boto:Token: None 2022-09-03T05:59:01.541 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/?versions 2022-09-03T05:59:01.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bS0fpcj9A/9lGPf6DbIau1IsoKI= 2022-09-03T05:59:01.542 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bS0fpcj9A/9lGPf6DbIau1IsoKI=', 'Content-Length': '0'} 2022-09-03T05:59:01.545 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000096d5fc2b19e810a-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.546 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.547 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.547 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.548 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.549 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.549 DEBUG:boto:Method: GET 2022-09-03T05:59:01.550 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:01.550 DEBUG:boto:Data: 2022-09-03T05:59:01.551 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.552 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.552 DEBUG:boto:Params: {} 2022-09-03T05:59:01.553 DEBUG:boto:Token: None 2022-09-03T05:59:01.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/?versions 2022-09-03T05:59:01.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bS0fpcj9A/9lGPf6DbIau1IsoKI= 2022-09-03T05:59:01.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bS0fpcj9A/9lGPf6DbIau1IsoKI=', 'Content-Length': '0'} 2022-09-03T05:59:01.557 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000056ef5fc01f324d44-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.558 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.559 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:01.560 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.560 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.561 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.561 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.562 DEBUG:boto:Method: GET 2022-09-03T05:59:01.563 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.564 DEBUG:boto:Data: 2022-09-03T05:59:01.564 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.565 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.565 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.566 DEBUG:boto:Params: {} 2022-09-03T05:59:01.567 DEBUG:boto:Token: None 2022-09-03T05:59:01.568 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y= 2022-09-03T05:59:01.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y=', 'Content-Length': '0'} 2022-09-03T05:59:01.571 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000062128f6c58c3660e-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.572 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.573 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.574 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.574 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.576 DEBUG:boto:Method: GET 2022-09-03T05:59:01.577 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.578 DEBUG:boto:Data: 2022-09-03T05:59:01.579 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.580 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.581 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.582 DEBUG:boto:Params: {} 2022-09-03T05:59:01.583 DEBUG:boto:Token: None 2022-09-03T05:59:01.584 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y= 2022-09-03T05:59:01.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y=', 'Content-Length': '0'} 2022-09-03T05:59:01.588 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000cd0d17fdf9134c3d-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.589 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.589 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.590 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.591 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.591 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.592 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.592 DEBUG:boto:Data: 2022-09-03T05:59:01.593 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.593 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.594 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.594 DEBUG:boto:Params: {} 2022-09-03T05:59:01.595 DEBUG:boto:Token: None 2022-09-03T05:59:01.596 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XWwGdZdmAiEhaukjU1E6YT05ZbU= 2022-09-03T05:59:01.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XWwGdZdmAiEhaukjU1E6YT05ZbU=', 'Content-Length': '0'} 2022-09-03T05:59:01.599 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000060ee9ce61f44768-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.600 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.600 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.601 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.602 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.602 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.603 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.604 DEBUG:boto:Data: 2022-09-03T05:59:01.605 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.606 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.607 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.607 DEBUG:boto:Params: {} 2022-09-03T05:59:01.608 DEBUG:boto:Token: None 2022-09-03T05:59:01.609 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.610 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XWwGdZdmAiEhaukjU1E6YT05ZbU= 2022-09-03T05:59:01.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XWwGdZdmAiEhaukjU1E6YT05ZbU=', 'Content-Length': '0'} 2022-09-03T05:59:01.613 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000df0143071badabe7-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.613 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:01.614 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.614 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.615 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.615 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.616 DEBUG:boto:Method: GET 2022-09-03T05:59:01.616 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.617 DEBUG:boto:Data: 2022-09-03T05:59:01.618 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.618 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.619 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.619 DEBUG:boto:Params: {} 2022-09-03T05:59:01.620 DEBUG:boto:Token: None 2022-09-03T05:59:01.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y= 2022-09-03T05:59:01.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y=', 'Content-Length': '0'} 2022-09-03T05:59:01.624 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003dea3104bcc87d40-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.624 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.625 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.626 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.626 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.627 DEBUG:boto:Method: GET 2022-09-03T05:59:01.627 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.628 DEBUG:boto:Data: 2022-09-03T05:59:01.628 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.629 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.630 DEBUG:boto:Params: {} 2022-09-03T05:59:01.631 DEBUG:boto:Token: None 2022-09-03T05:59:01.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y= 2022-09-03T05:59:01.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y=', 'Content-Length': '0'} 2022-09-03T05:59:01.635 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000669866a4249548e2-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.635 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.636 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.637 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:01.637 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:01.638 DEBUG:boto:Method: GET 2022-09-03T05:59:01.638 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:01.639 DEBUG:boto:Data: 2022-09-03T05:59:01.639 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.640 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.641 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.641 DEBUG:boto:Params: {} 2022-09-03T05:59:01.642 DEBUG:boto:Token: None 2022-09-03T05:59:01.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/ 2022-09-03T05:59:01.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE= 2022-09-03T05:59:01.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE=', 'Content-Length': '0'} 2022-09-03T05:59:01.647 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e6d15506570864a0-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.648 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.649 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.649 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.650 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:01.650 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:01.651 DEBUG:boto:Method: GET 2022-09-03T05:59:01.651 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:01.652 DEBUG:boto:Data: 2022-09-03T05:59:01.653 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.653 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.654 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.654 DEBUG:boto:Params: {} 2022-09-03T05:59:01.655 DEBUG:boto:Token: None 2022-09-03T05:59:01.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/ 2022-09-03T05:59:01.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE= 2022-09-03T05:59:01.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE=', 'Content-Length': '0'} 2022-09-03T05:59:01.659 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098783636167fb58b-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.702 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.703 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.704 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.704 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.705 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T05:59:01.705 DEBUG:boto:Data: 2022-09-03T05:59:01.706 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.706 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.707 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.708 DEBUG:boto:Params: {} 2022-09-03T05:59:01.710 DEBUG:boto:Token: None 2022-09-03T05:59:01.711 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy 2022-09-03T05:59:01.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Vg9Qu81nkuKZ2jnw0CLcZqPoXY= 2022-09-03T05:59:01.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Vg9Qu81nkuKZ2jnw0CLcZqPoXY=', 'Content-Length': '0'} 2022-09-03T05:59:01.716 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000395baa577caac1ec-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.717 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.718 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.719 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.719 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T05:59:01.720 DEBUG:boto:Data: 2022-09-03T05:59:01.720 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.721 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.721 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.722 DEBUG:boto:Params: {} 2022-09-03T05:59:01.722 DEBUG:boto:Token: None 2022-09-03T05:59:01.723 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy 2022-09-03T05:59:01.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Vg9Qu81nkuKZ2jnw0CLcZqPoXY= 2022-09-03T05:59:01.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Vg9Qu81nkuKZ2jnw0CLcZqPoXY=', 'Content-Length': '0'} 2022-09-03T05:59:01.727 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000811d534f541dd1d0-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.727 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:01.728 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.729 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.729 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.730 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.731 DEBUG:boto:Method: GET 2022-09-03T05:59:01.731 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.732 DEBUG:boto:Data: 2022-09-03T05:59:01.733 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.733 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.734 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.734 DEBUG:boto:Params: {} 2022-09-03T05:59:01.735 DEBUG:boto:Token: None 2022-09-03T05:59:01.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y= 2022-09-03T05:59:01.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y=', 'Content-Length': '0'} 2022-09-03T05:59:01.739 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000494aa79fe35ca606-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.739 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:01.740 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:01.741 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.742 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.743 DEBUG:boto:Method: GET 2022-09-03T05:59:01.744 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.745 DEBUG:boto:Data: 2022-09-03T05:59:01.746 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.748 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.749 DEBUG:boto:Params: {} 2022-09-03T05:59:01.750 DEBUG:boto:Token: None 2022-09-03T05:59:01.751 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:01.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y= 2022-09-03T05:59:01.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bkbbAIxeT4wuagpfuhJ1NpnYQ6Y=', 'Content-Length': '0'} 2022-09-03T05:59:01.755 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006bf63827b3ea28e6-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.756 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.757 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.757 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:01.758 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:01.759 DEBUG:boto:Method: GET 2022-09-03T05:59:01.759 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:01.760 DEBUG:boto:Data: 2022-09-03T05:59:01.760 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.761 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.761 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.762 DEBUG:boto:Params: {} 2022-09-03T05:59:01.762 DEBUG:boto:Token: None 2022-09-03T05:59:01.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/ 2022-09-03T05:59:01.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE= 2022-09-03T05:59:01.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE=', 'Content-Length': '0'} 2022-09-03T05:59:01.767 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069948f010a208dd4-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.767 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.768 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.769 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.770 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:01.770 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:01.771 DEBUG:boto:Method: GET 2022-09-03T05:59:01.771 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:01.772 DEBUG:boto:Data: 2022-09-03T05:59:01.773 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.773 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.774 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.774 DEBUG:boto:Params: {} 2022-09-03T05:59:01.775 DEBUG:boto:Token: None 2022-09-03T05:59:01.775 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/ 2022-09-03T05:59:01.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE= 2022-09-03T05:59:01.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE=', 'Content-Length': '0'} 2022-09-03T05:59:01.779 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e9e52c6ce1fe608f-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.780 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.781 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:01.782 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.782 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.783 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.783 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.784 DEBUG:boto:Method: GET 2022-09-03T05:59:01.785 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.785 DEBUG:boto:Data: 2022-09-03T05:59:01.786 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.786 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.787 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.787 DEBUG:boto:Params: {} 2022-09-03T05:59:01.788 DEBUG:boto:Token: None 2022-09-03T05:59:01.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA= 2022-09-03T05:59:01.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA=', 'Content-Length': '0'} 2022-09-03T05:59:01.792 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bcea85ed9e564fc4-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.793 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.794 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.795 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.796 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.797 DEBUG:boto:Method: GET 2022-09-03T05:59:01.797 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.798 DEBUG:boto:Data: 2022-09-03T05:59:01.798 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.799 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.799 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.800 DEBUG:boto:Params: {} 2022-09-03T05:59:01.801 DEBUG:boto:Token: None 2022-09-03T05:59:01.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA= 2022-09-03T05:59:01.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA=', 'Content-Length': '0'} 2022-09-03T05:59:01.805 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000982f963e57a5d136-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.846 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.847 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.848 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.849 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.850 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.851 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.851 DEBUG:boto:Data: 2022-09-03T05:59:01.852 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.853 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.854 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.855 DEBUG:boto:Params: {} 2022-09-03T05:59:01.855 DEBUG:boto:Token: None 2022-09-03T05:59:01.856 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pwmBZrZT87Df/uFGps9jqyiSuUg= 2022-09-03T05:59:01.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pwmBZrZT87Df/uFGps9jqyiSuUg=', 'Content-Length': '0'} 2022-09-03T05:59:01.859 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000248a1a50bb63758b-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.860 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.861 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.862 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.862 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.863 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.864 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.864 DEBUG:boto:Data: 2022-09-03T05:59:01.865 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.866 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.866 DEBUG:boto:Params: {} 2022-09-03T05:59:01.867 DEBUG:boto:Token: None 2022-09-03T05:59:01.868 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pwmBZrZT87Df/uFGps9jqyiSuUg= 2022-09-03T05:59:01.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pwmBZrZT87Df/uFGps9jqyiSuUg=', 'Content-Length': '0'} 2022-09-03T05:59:01.871 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ab677104fabef773-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.872 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:01.873 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.873 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.874 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.874 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.875 DEBUG:boto:Method: GET 2022-09-03T05:59:01.875 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.876 DEBUG:boto:Data: 2022-09-03T05:59:01.876 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.877 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.877 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.878 DEBUG:boto:Params: {} 2022-09-03T05:59:01.879 DEBUG:boto:Token: None 2022-09-03T05:59:01.879 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA= 2022-09-03T05:59:01.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA=', 'Content-Length': '0'} 2022-09-03T05:59:01.882 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b9bff6d5eccfa794-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.883 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.884 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.885 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.885 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.886 DEBUG:boto:Method: GET 2022-09-03T05:59:01.886 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.887 DEBUG:boto:Data: 2022-09-03T05:59:01.887 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.888 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.888 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.889 DEBUG:boto:Params: {} 2022-09-03T05:59:01.889 DEBUG:boto:Token: None 2022-09-03T05:59:01.890 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA= 2022-09-03T05:59:01.891 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA=', 'Content-Length': '0'} 2022-09-03T05:59:01.893 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e5062f6ede12fe7e-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.894 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.895 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.895 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:01.896 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:01.896 DEBUG:boto:Method: GET 2022-09-03T05:59:01.897 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:01.897 DEBUG:boto:Data: 2022-09-03T05:59:01.898 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.898 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.899 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.899 DEBUG:boto:Params: {} 2022-09-03T05:59:01.900 DEBUG:boto:Token: None 2022-09-03T05:59:01.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/ 2022-09-03T05:59:01.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE= 2022-09-03T05:59:01.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE=', 'Content-Length': '0'} 2022-09-03T05:59:01.905 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f6a5e49c2ffe3bc-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.946 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.947 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:01.948 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:01.948 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:01.949 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:01.949 DEBUG:boto:Method: GET 2022-09-03T05:59:01.950 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:01.951 DEBUG:boto:Data: 2022-09-03T05:59:01.951 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.952 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.952 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.953 DEBUG:boto:Params: {} 2022-09-03T05:59:01.953 DEBUG:boto:Token: None 2022-09-03T05:59:01.954 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/ 2022-09-03T05:59:01.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE= 2022-09-03T05:59:01.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yo+CPgXakrfHeROkRRAy1KZmteE=', 'Content-Length': '0'} 2022-09-03T05:59:01.958 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000953d535a8563cc48-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.959 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:01.959 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.960 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.960 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.961 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T05:59:01.961 DEBUG:boto:Data: 2022-09-03T05:59:01.962 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.963 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.963 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.964 DEBUG:boto:Params: {} 2022-09-03T05:59:01.964 DEBUG:boto:Token: None 2022-09-03T05:59:01.965 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1 2022-09-03T05:59:01.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3/3qn59vZOTp5qroJFPH0pkygpU= 2022-09-03T05:59:01.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3/3qn59vZOTp5qroJFPH0pkygpU=', 'Content-Length': '0'} 2022-09-03T05:59:01.968 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000def685011955ac20-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.969 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.970 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.970 DEBUG:boto:Method: HEAD 2022-09-03T05:59:01.971 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T05:59:01.971 DEBUG:boto:Data: 2022-09-03T05:59:01.972 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.972 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.973 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.973 DEBUG:boto:Params: {} 2022-09-03T05:59:01.974 DEBUG:boto:Token: None 2022-09-03T05:59:01.974 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1 2022-09-03T05:59:01.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3/3qn59vZOTp5qroJFPH0pkygpU= 2022-09-03T05:59:01.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3/3qn59vZOTp5qroJFPH0pkygpU=', 'Content-Length': '0'} 2022-09-03T05:59:01.978 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000050dc53e71ea0cf4f-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.979 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:01.979 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.980 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.980 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.981 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.981 DEBUG:boto:Method: GET 2022-09-03T05:59:01.982 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.983 DEBUG:boto:Data: 2022-09-03T05:59:01.983 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.984 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:01.984 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.985 DEBUG:boto:Params: {} 2022-09-03T05:59:01.985 DEBUG:boto:Token: None 2022-09-03T05:59:01.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA= 2022-09-03T05:59:01.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA=', 'Content-Length': '0'} 2022-09-03T05:59:01.989 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d038552c03e188c9-006312ed25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:01.990 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.991 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:01.991 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.992 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.992 DEBUG:boto:Method: GET 2022-09-03T05:59:01.993 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.993 DEBUG:boto:Data: 2022-09-03T05:59:01.994 DEBUG:boto:Headers: {} 2022-09-03T05:59:01.994 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:01.995 DEBUG:boto:Port: 8000 2022-09-03T05:59:01.995 DEBUG:boto:Params: {} 2022-09-03T05:59:01.996 DEBUG:boto:Token: None 2022-09-03T05:59:01.997 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:01 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:01.997 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA= 2022-09-03T05:59:01.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FQe+KKHrwwXPmRhZoDfoH4sa/qA=', 'Content-Length': '0'} 2022-09-03T05:59:02.000 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a0d100d0efd79fe9-006312ed25-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.001 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.001 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.002 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:02.002 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:02.003 DEBUG:boto:Method: GET 2022-09-03T05:59:02.003 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:02.004 DEBUG:boto:Data: 2022-09-03T05:59:02.004 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.005 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.005 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.006 DEBUG:boto:Params: {} 2022-09-03T05:59:02.007 DEBUG:boto:Token: None 2022-09-03T05:59:02.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.011 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6e7bfda762d0d76-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.012 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.013 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.014 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.014 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:02.015 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:02.015 DEBUG:boto:Method: GET 2022-09-03T05:59:02.016 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:02.016 DEBUG:boto:Data: 2022-09-03T05:59:02.017 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.018 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.018 DEBUG:boto:Params: {} 2022-09-03T05:59:02.019 DEBUG:boto:Token: None 2022-09-03T05:59:02.019 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.023 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009acf3f2ecd3fbaec-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.024 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.025 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:02.025 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.026 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.027 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.027 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.028 DEBUG:boto:Method: GET 2022-09-03T05:59:02.029 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.029 DEBUG:boto:Data: 2022-09-03T05:59:02.030 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.030 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.031 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.031 DEBUG:boto:Params: {} 2022-09-03T05:59:02.032 DEBUG:boto:Token: None 2022-09-03T05:59:02.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258= 2022-09-03T05:59:02.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258=', 'Content-Length': '0'} 2022-09-03T05:59:02.036 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ba2e8aaca26d692c-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.037 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.038 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.039 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.039 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.040 DEBUG:boto:Method: GET 2022-09-03T05:59:02.040 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.041 DEBUG:boto:Data: 2022-09-03T05:59:02.041 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.042 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.043 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.043 DEBUG:boto:Params: {} 2022-09-03T05:59:02.045 DEBUG:boto:Token: None 2022-09-03T05:59:02.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258= 2022-09-03T05:59:02.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258=', 'Content-Length': '0'} 2022-09-03T05:59:02.050 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004f9bfafad48655cf-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.098 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.099 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.099 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.100 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.101 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.101 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.102 DEBUG:boto:Data: 2022-09-03T05:59:02.102 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.103 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.104 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.104 DEBUG:boto:Params: {} 2022-09-03T05:59:02.105 DEBUG:boto:Token: None 2022-09-03T05:59:02.105 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.106 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SwLtWALIigSSwZdRl/8gbJoSYT8= 2022-09-03T05:59:02.107 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SwLtWALIigSSwZdRl/8gbJoSYT8=', 'Content-Length': '0'} 2022-09-03T05:59:02.109 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000bf85ed80e54405d-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.109 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.110 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.110 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.111 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.111 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.112 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.112 DEBUG:boto:Data: 2022-09-03T05:59:02.113 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.114 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.115 DEBUG:boto:Params: {} 2022-09-03T05:59:02.115 DEBUG:boto:Token: None 2022-09-03T05:59:02.116 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SwLtWALIigSSwZdRl/8gbJoSYT8= 2022-09-03T05:59:02.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SwLtWALIigSSwZdRl/8gbJoSYT8=', 'Content-Length': '0'} 2022-09-03T05:59:02.119 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004f4d0d305a06f975-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.120 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:02.121 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.121 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.122 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.122 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.123 DEBUG:boto:Method: GET 2022-09-03T05:59:02.124 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.124 DEBUG:boto:Data: 2022-09-03T05:59:02.125 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.125 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.126 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.126 DEBUG:boto:Params: {} 2022-09-03T05:59:02.127 DEBUG:boto:Token: None 2022-09-03T05:59:02.128 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258= 2022-09-03T05:59:02.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258=', 'Content-Length': '0'} 2022-09-03T05:59:02.131 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008373df2b4ab3d5ce-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.132 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.132 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.133 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.133 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.134 DEBUG:boto:Method: GET 2022-09-03T05:59:02.135 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.135 DEBUG:boto:Data: 2022-09-03T05:59:02.136 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.136 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.137 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.137 DEBUG:boto:Params: {} 2022-09-03T05:59:02.138 DEBUG:boto:Token: None 2022-09-03T05:59:02.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258= 2022-09-03T05:59:02.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258=', 'Content-Length': '0'} 2022-09-03T05:59:02.142 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e2ada522e1b61e1e-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.143 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.143 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.144 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.145 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.145 DEBUG:boto:Method: GET 2022-09-03T05:59:02.146 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.146 DEBUG:boto:Data: 2022-09-03T05:59:02.147 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.147 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.148 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.148 DEBUG:boto:Params: {} 2022-09-03T05:59:02.149 DEBUG:boto:Token: None 2022-09-03T05:59:02.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.151 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.154 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000944e8e07116506b0-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.155 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.155 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.156 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.157 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.157 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.158 DEBUG:boto:Method: GET 2022-09-03T05:59:02.158 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.159 DEBUG:boto:Data: 2022-09-03T05:59:02.160 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.160 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.161 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.161 DEBUG:boto:Params: {} 2022-09-03T05:59:02.162 DEBUG:boto:Token: None 2022-09-03T05:59:02.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.166 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002a585d17753276c-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.167 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.168 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.169 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.169 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.170 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T05:59:02.170 DEBUG:boto:Data: 2022-09-03T05:59:02.171 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.171 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.172 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.172 DEBUG:boto:Params: {} 2022-09-03T05:59:02.173 DEBUG:boto:Token: None 2022-09-03T05:59:02.174 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2 2022-09-03T05:59:02.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dFs6OgBzD025MhR+C0xr4Yh/gfE= 2022-09-03T05:59:02.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dFs6OgBzD025MhR+C0xr4Yh/gfE=', 'Content-Length': '0'} 2022-09-03T05:59:02.177 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000024e79c49afed72e5-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.178 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.178 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.179 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.179 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T05:59:02.180 DEBUG:boto:Data: 2022-09-03T05:59:02.180 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.181 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.181 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.182 DEBUG:boto:Params: {} 2022-09-03T05:59:02.182 DEBUG:boto:Token: None 2022-09-03T05:59:02.183 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2 2022-09-03T05:59:02.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dFs6OgBzD025MhR+C0xr4Yh/gfE= 2022-09-03T05:59:02.184 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dFs6OgBzD025MhR+C0xr4Yh/gfE=', 'Content-Length': '0'} 2022-09-03T05:59:02.187 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d52c0a9c282c503d-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.188 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:02.188 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.189 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.189 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.190 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.190 DEBUG:boto:Method: GET 2022-09-03T05:59:02.191 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.191 DEBUG:boto:Data: 2022-09-03T05:59:02.192 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.193 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.193 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.194 DEBUG:boto:Params: {} 2022-09-03T05:59:02.194 DEBUG:boto:Token: None 2022-09-03T05:59:02.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258= 2022-09-03T05:59:02.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258=', 'Content-Length': '0'} 2022-09-03T05:59:02.198 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000df442e086d4865b-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.199 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.199 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:02.200 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.201 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.201 DEBUG:boto:Method: GET 2022-09-03T05:59:02.202 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.202 DEBUG:boto:Data: 2022-09-03T05:59:02.203 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.204 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.204 DEBUG:boto:Params: {} 2022-09-03T05:59:02.205 DEBUG:boto:Token: None 2022-09-03T05:59:02.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:02.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258= 2022-09-03T05:59:02.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:buHeE31HmT6rcdkFOFYuWAeW258=', 'Content-Length': '0'} 2022-09-03T05:59:02.209 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000725d0bea2bffed30-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.250 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.251 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.252 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.252 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.253 DEBUG:boto:Method: GET 2022-09-03T05:59:02.253 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.254 DEBUG:boto:Data: 2022-09-03T05:59:02.255 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.255 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.256 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.256 DEBUG:boto:Params: {} 2022-09-03T05:59:02.257 DEBUG:boto:Token: None 2022-09-03T05:59:02.258 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.262 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c7ca05bc215839c3-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.263 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.265 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.266 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.267 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.268 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.269 DEBUG:boto:Method: GET 2022-09-03T05:59:02.270 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:02.271 DEBUG:boto:Data: 2022-09-03T05:59:02.272 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.274 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.275 DEBUG:boto:Params: {} 2022-09-03T05:59:02.276 DEBUG:boto:Token: None 2022-09-03T05:59:02.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.281 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000310dde9c46ed245a-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.281 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.282 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:02.283 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.284 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.284 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.285 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.285 DEBUG:boto:Method: GET 2022-09-03T05:59:02.286 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.287 DEBUG:boto:Data: 2022-09-03T05:59:02.288 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.288 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.289 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.289 DEBUG:boto:Params: {} 2022-09-03T05:59:02.290 DEBUG:boto:Token: None 2022-09-03T05:59:02.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0= 2022-09-03T05:59:02.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0=', 'Content-Length': '0'} 2022-09-03T05:59:02.294 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001a511f7d9e90f111-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.338 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.339 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.340 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.341 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.342 DEBUG:boto:Method: GET 2022-09-03T05:59:02.343 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.344 DEBUG:boto:Data: 2022-09-03T05:59:02.344 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.345 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.346 DEBUG:boto:Params: {} 2022-09-03T05:59:02.346 DEBUG:boto:Token: None 2022-09-03T05:59:02.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0= 2022-09-03T05:59:02.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0=', 'Content-Length': '0'} 2022-09-03T05:59:02.350 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000092b8b8b96feeb429-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.351 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.352 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.352 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.353 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.353 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.354 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.355 DEBUG:boto:Data: 2022-09-03T05:59:02.356 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.357 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.358 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.358 DEBUG:boto:Params: {} 2022-09-03T05:59:02.360 DEBUG:boto:Token: None 2022-09-03T05:59:02.361 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B6932CE9dkX5JqnpWD4tHkgr20U= 2022-09-03T05:59:02.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B6932CE9dkX5JqnpWD4tHkgr20U=', 'Content-Length': '0'} 2022-09-03T05:59:02.364 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c9d0ed8dbff772e6-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.365 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.366 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.367 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.367 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.368 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.368 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.369 DEBUG:boto:Data: 2022-09-03T05:59:02.369 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.370 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.371 DEBUG:boto:Params: {} 2022-09-03T05:59:02.371 DEBUG:boto:Token: None 2022-09-03T05:59:02.372 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B6932CE9dkX5JqnpWD4tHkgr20U= 2022-09-03T05:59:02.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B6932CE9dkX5JqnpWD4tHkgr20U=', 'Content-Length': '0'} 2022-09-03T05:59:02.375 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e65e6befe88ac1cf-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.376 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:02.377 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.377 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.378 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.378 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.379 DEBUG:boto:Method: GET 2022-09-03T05:59:02.379 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.380 DEBUG:boto:Data: 2022-09-03T05:59:02.381 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.381 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.382 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.382 DEBUG:boto:Params: {} 2022-09-03T05:59:02.383 DEBUG:boto:Token: None 2022-09-03T05:59:02.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0= 2022-09-03T05:59:02.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0=', 'Content-Length': '0'} 2022-09-03T05:59:02.386 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000444d06fc28d8457f-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.387 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.388 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.388 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.389 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.390 DEBUG:boto:Method: GET 2022-09-03T05:59:02.390 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.391 DEBUG:boto:Data: 2022-09-03T05:59:02.391 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.392 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.393 DEBUG:boto:Params: {} 2022-09-03T05:59:02.394 DEBUG:boto:Token: None 2022-09-03T05:59:02.394 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0= 2022-09-03T05:59:02.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0=', 'Content-Length': '0'} 2022-09-03T05:59:02.398 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000824a36802359922f-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.399 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.399 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.400 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.400 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.401 DEBUG:boto:Method: GET 2022-09-03T05:59:02.401 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.402 DEBUG:boto:Data: 2022-09-03T05:59:02.403 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.403 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.404 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.404 DEBUG:boto:Params: {} 2022-09-03T05:59:02.405 DEBUG:boto:Token: None 2022-09-03T05:59:02.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.409 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7505cee1ccb7004-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.410 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.411 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.411 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.412 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.412 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.413 DEBUG:boto:Method: GET 2022-09-03T05:59:02.413 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.414 DEBUG:boto:Data: 2022-09-03T05:59:02.415 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.415 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.416 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.416 DEBUG:boto:Params: {} 2022-09-03T05:59:02.417 DEBUG:boto:Token: None 2022-09-03T05:59:02.417 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.421 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e44fd3a95866662c-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.466 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.467 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.468 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.468 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.469 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T05:59:02.469 DEBUG:boto:Data: 2022-09-03T05:59:02.470 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.471 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.471 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.472 DEBUG:boto:Params: {} 2022-09-03T05:59:02.473 DEBUG:boto:Token: None 2022-09-03T05:59:02.473 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3 2022-09-03T05:59:02.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+oSQ/+vPpSzqdjNnrIRvIlsaZG4= 2022-09-03T05:59:02.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+oSQ/+vPpSzqdjNnrIRvIlsaZG4=', 'Content-Length': '0'} 2022-09-03T05:59:02.477 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004c1e112ae37b4c1a-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.478 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.479 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.479 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.480 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T05:59:02.480 DEBUG:boto:Data: 2022-09-03T05:59:02.481 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.482 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.482 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.483 DEBUG:boto:Params: {} 2022-09-03T05:59:02.483 DEBUG:boto:Token: None 2022-09-03T05:59:02.484 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3 2022-09-03T05:59:02.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+oSQ/+vPpSzqdjNnrIRvIlsaZG4= 2022-09-03T05:59:02.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+oSQ/+vPpSzqdjNnrIRvIlsaZG4=', 'Content-Length': '0'} 2022-09-03T05:59:02.488 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000033c9cf3c3d629568-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.489 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:02.490 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.491 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.492 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.493 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.494 DEBUG:boto:Method: GET 2022-09-03T05:59:02.495 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.496 DEBUG:boto:Data: 2022-09-03T05:59:02.496 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.497 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.497 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.498 DEBUG:boto:Params: {} 2022-09-03T05:59:02.498 DEBUG:boto:Token: None 2022-09-03T05:59:02.499 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0= 2022-09-03T05:59:02.500 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0=', 'Content-Length': '0'} 2022-09-03T05:59:02.503 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004e2b1376236a58e2-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.504 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.505 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:02.506 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.507 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.508 DEBUG:boto:Method: GET 2022-09-03T05:59:02.509 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.510 DEBUG:boto:Data: 2022-09-03T05:59:02.511 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.513 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.514 DEBUG:boto:Params: {} 2022-09-03T05:59:02.515 DEBUG:boto:Token: None 2022-09-03T05:59:02.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:02.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0= 2022-09-03T05:59:02.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z8zcwMOzXq87xJu7vMCFHFPvrF0=', 'Content-Length': '0'} 2022-09-03T05:59:02.523 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c0433d8bfbf18285-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.524 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.524 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.525 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.525 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.526 DEBUG:boto:Method: GET 2022-09-03T05:59:02.527 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.527 DEBUG:boto:Data: 2022-09-03T05:59:02.528 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.529 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:02.530 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.531 DEBUG:boto:Params: {} 2022-09-03T05:59:02.532 DEBUG:boto:Token: None 2022-09-03T05:59:02.533 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.538 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ba539e30a634937-006312ed26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.582 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.583 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.584 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.584 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.585 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.586 DEBUG:boto:Method: GET 2022-09-03T05:59:02.586 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:02.587 DEBUG:boto:Data: 2022-09-03T05:59:02.587 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.588 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.589 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.589 DEBUG:boto:Params: {} 2022-09-03T05:59:02.590 DEBUG:boto:Token: None 2022-09-03T05:59:02.590 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000926c35b7b2c6cf46-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.596 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.596 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-16 zones={test-zone2, test-zone1} 2022-09-03T05:59:02.597 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-16 zones={test-zone3, test-zone1} 2022-09-03T05:59:02.597 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.598 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.599 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.600 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:59:02.600 DEBUG:boto:Data: 2022-09-03T05:59:02.601 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.602 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.602 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.603 DEBUG:boto:Params: {} 2022-09-03T05:59:02.604 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T05:59:02.604 DEBUG:boto:Token: None 2022-09-03T05:59:02.605 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hi+SzFR/3hSXKs9xKufH8AYjfi0= 2022-09-03T05:59:02.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hi+SzFR/3hSXKs9xKufH8AYjfi0=', 'Content-Length': '0'} 2022-09-03T05:59:02.610 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f6eb0b8e26df121a-006312ed26-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.610 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.611 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.612 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.612 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:59:02.613 DEBUG:boto:Data: 2022-09-03T05:59:02.613 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.614 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.615 DEBUG:boto:Params: {} 2022-09-03T05:59:02.615 DEBUG:boto:Token: None 2022-09-03T05:59:02.616 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hi+SzFR/3hSXKs9xKufH8AYjfi0= 2022-09-03T05:59:02.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hi+SzFR/3hSXKs9xKufH8AYjfi0=', 'Content-Length': '0'} 2022-09-03T05:59:02.620 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000079487f9fa84588e6-006312ed26-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.621 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:59:02.621 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.622 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.623 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.623 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.624 DEBUG:boto:Method: GET 2022-09-03T05:59:02.625 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.625 DEBUG:boto:Data: 2022-09-03T05:59:02.626 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.627 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.628 DEBUG:boto:Params: {} 2022-09-03T05:59:02.629 DEBUG:boto:Token: None 2022-09-03T05:59:02.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/?versions 2022-09-03T05:59:02.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4tXDQGAN2A6LINN5sGN9bYMk6dI= 2022-09-03T05:59:02.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4tXDQGAN2A6LINN5sGN9bYMk6dI=', 'Content-Length': '0'} 2022-09-03T05:59:02.635 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c32521dc69229c7-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.636 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.638 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:02.639 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:02.639 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:02.640 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:02.641 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:59:02.641 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.642 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.642 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.643 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.644 DEBUG:boto:Method: GET 2022-09-03T05:59:02.644 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.645 DEBUG:boto:Data: 2022-09-03T05:59:02.645 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.646 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.647 DEBUG:boto:Params: {} 2022-09-03T05:59:02.647 DEBUG:boto:Token: None 2022-09-03T05:59:02.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/?versions 2022-09-03T05:59:02.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4tXDQGAN2A6LINN5sGN9bYMk6dI= 2022-09-03T05:59:02.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4tXDQGAN2A6LINN5sGN9bYMk6dI=', 'Content-Length': '0'} 2022-09-03T05:59:02.652 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031a3e251b62fd645-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.653 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.654 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:02.655 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:02.655 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:02.656 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:02.656 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.657 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.657 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.658 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.658 DEBUG:boto:Method: GET 2022-09-03T05:59:02.659 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.659 DEBUG:boto:Data: 2022-09-03T05:59:02.660 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.661 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.661 DEBUG:boto:Params: {} 2022-09-03T05:59:02.662 DEBUG:boto:Token: None 2022-09-03T05:59:02.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/?versions 2022-09-03T05:59:02.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4tXDQGAN2A6LINN5sGN9bYMk6dI= 2022-09-03T05:59:02.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4tXDQGAN2A6LINN5sGN9bYMk6dI=', 'Content-Length': '0'} 2022-09-03T05:59:02.667 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c04906a4f384a9d-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.667 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.668 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.669 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.669 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.670 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.670 DEBUG:boto:Method: GET 2022-09-03T05:59:02.671 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:02.671 DEBUG:boto:Data: 2022-09-03T05:59:02.672 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.673 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.673 DEBUG:boto:Params: {} 2022-09-03T05:59:02.674 DEBUG:boto:Token: None 2022-09-03T05:59:02.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/?versions 2022-09-03T05:59:02.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4tXDQGAN2A6LINN5sGN9bYMk6dI= 2022-09-03T05:59:02.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4tXDQGAN2A6LINN5sGN9bYMk6dI=', 'Content-Length': '0'} 2022-09-03T05:59:02.679 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd25172e279954bd-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.680 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.681 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:02.682 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.682 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.683 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.683 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.684 DEBUG:boto:Method: GET 2022-09-03T05:59:02.684 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.685 DEBUG:boto:Data: 2022-09-03T05:59:02.685 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.686 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.687 DEBUG:boto:Params: {} 2022-09-03T05:59:02.688 DEBUG:boto:Token: None 2022-09-03T05:59:02.689 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0= 2022-09-03T05:59:02.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0=', 'Content-Length': '0'} 2022-09-03T05:59:02.692 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d323a14443d6145d-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.693 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.693 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.694 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.694 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.695 DEBUG:boto:Method: GET 2022-09-03T05:59:02.696 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.696 DEBUG:boto:Data: 2022-09-03T05:59:02.697 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.697 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.698 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.698 DEBUG:boto:Params: {} 2022-09-03T05:59:02.699 DEBUG:boto:Token: None 2022-09-03T05:59:02.700 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0= 2022-09-03T05:59:02.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0=', 'Content-Length': '0'} 2022-09-03T05:59:02.703 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000034ead7fb786aa65-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.704 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.705 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.705 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.706 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.706 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.707 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.707 DEBUG:boto:Data: 2022-09-03T05:59:02.708 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.708 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.709 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.709 DEBUG:boto:Params: {} 2022-09-03T05:59:02.710 DEBUG:boto:Token: None 2022-09-03T05:59:02.710 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zBd2+WFBh87iAwqhV2US8zhl4iw= 2022-09-03T05:59:02.711 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zBd2+WFBh87iAwqhV2US8zhl4iw=', 'Content-Length': '0'} 2022-09-03T05:59:02.713 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000173a49ad0a8d2c7b-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.714 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.715 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.715 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.716 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.716 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.717 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.717 DEBUG:boto:Data: 2022-09-03T05:59:02.718 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.719 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.719 DEBUG:boto:Params: {} 2022-09-03T05:59:02.720 DEBUG:boto:Token: None 2022-09-03T05:59:02.721 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zBd2+WFBh87iAwqhV2US8zhl4iw= 2022-09-03T05:59:02.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zBd2+WFBh87iAwqhV2US8zhl4iw=', 'Content-Length': '0'} 2022-09-03T05:59:02.724 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000000626924784012720-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.725 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:02.725 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.726 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.726 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.727 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.727 DEBUG:boto:Method: GET 2022-09-03T05:59:02.728 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.728 DEBUG:boto:Data: 2022-09-03T05:59:02.729 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.730 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.730 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.731 DEBUG:boto:Params: {} 2022-09-03T05:59:02.731 DEBUG:boto:Token: None 2022-09-03T05:59:02.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0= 2022-09-03T05:59:02.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0=', 'Content-Length': '0'} 2022-09-03T05:59:02.735 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000cadc6cb1ef59e206-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.736 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.737 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.737 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.738 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.738 DEBUG:boto:Method: GET 2022-09-03T05:59:02.739 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.740 DEBUG:boto:Data: 2022-09-03T05:59:02.740 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.741 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.741 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.742 DEBUG:boto:Params: {} 2022-09-03T05:59:02.743 DEBUG:boto:Token: None 2022-09-03T05:59:02.743 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0= 2022-09-03T05:59:02.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0=', 'Content-Length': '0'} 2022-09-03T05:59:02.747 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b7bdeb0b4c2f9d7c-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.748 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.748 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.749 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:02.749 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:02.750 DEBUG:boto:Method: GET 2022-09-03T05:59:02.750 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:02.751 DEBUG:boto:Data: 2022-09-03T05:59:02.751 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.752 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.753 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.753 DEBUG:boto:Params: {} 2022-09-03T05:59:02.754 DEBUG:boto:Token: None 2022-09-03T05:59:02.754 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.755 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.758 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069eb7c079e8b658e-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.759 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.760 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.761 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.761 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:02.762 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:02.763 DEBUG:boto:Method: GET 2022-09-03T05:59:02.763 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:02.764 DEBUG:boto:Data: 2022-09-03T05:59:02.764 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.765 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.766 DEBUG:boto:Params: {} 2022-09-03T05:59:02.767 DEBUG:boto:Token: None 2022-09-03T05:59:02.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.773 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008357ac74d499c970-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.818 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.819 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.820 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.821 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.822 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T05:59:02.822 DEBUG:boto:Data: 2022-09-03T05:59:02.823 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.823 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.824 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.825 DEBUG:boto:Params: {} 2022-09-03T05:59:02.825 DEBUG:boto:Token: None 2022-09-03T05:59:02.826 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy 2022-09-03T05:59:02.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6/KYCd+4BFfNTY/+9kEFl4oMBUo= 2022-09-03T05:59:02.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6/KYCd+4BFfNTY/+9kEFl4oMBUo=', 'Content-Length': '0'} 2022-09-03T05:59:02.830 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000085888843f734842b-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.831 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.831 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.832 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.833 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T05:59:02.833 DEBUG:boto:Data: 2022-09-03T05:59:02.834 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.834 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.835 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.835 DEBUG:boto:Params: {} 2022-09-03T05:59:02.836 DEBUG:boto:Token: None 2022-09-03T05:59:02.837 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy 2022-09-03T05:59:02.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6/KYCd+4BFfNTY/+9kEFl4oMBUo= 2022-09-03T05:59:02.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6/KYCd+4BFfNTY/+9kEFl4oMBUo=', 'Content-Length': '0'} 2022-09-03T05:59:02.840 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000390b910ce8925291-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.841 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:02.842 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.842 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.843 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.844 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.844 DEBUG:boto:Method: GET 2022-09-03T05:59:02.845 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.845 DEBUG:boto:Data: 2022-09-03T05:59:02.846 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.847 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.847 DEBUG:boto:Params: {} 2022-09-03T05:59:02.848 DEBUG:boto:Token: None 2022-09-03T05:59:02.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0= 2022-09-03T05:59:02.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0=', 'Content-Length': '0'} 2022-09-03T05:59:02.852 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006f74779a50e05cba-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.853 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:02.853 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:02.854 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.855 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.856 DEBUG:boto:Method: GET 2022-09-03T05:59:02.857 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.859 DEBUG:boto:Data: 2022-09-03T05:59:02.859 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.860 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.861 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.862 DEBUG:boto:Params: {} 2022-09-03T05:59:02.863 DEBUG:boto:Token: None 2022-09-03T05:59:02.863 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:02.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0= 2022-09-03T05:59:02.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/5nfc8/38vBKO1DBW0Mx4K3PM0=', 'Content-Length': '0'} 2022-09-03T05:59:02.867 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006dbdaef1495bdf1e-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.868 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.869 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.870 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:02.870 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:02.871 DEBUG:boto:Method: GET 2022-09-03T05:59:02.872 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:02.873 DEBUG:boto:Data: 2022-09-03T05:59:02.873 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.874 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.875 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.875 DEBUG:boto:Params: {} 2022-09-03T05:59:02.876 DEBUG:boto:Token: None 2022-09-03T05:59:02.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.877 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.878 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.880 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007170d0e785849605-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.922 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.923 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:02.924 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:02.924 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:02.925 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:02.925 DEBUG:boto:Method: GET 2022-09-03T05:59:02.926 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:02.927 DEBUG:boto:Data: 2022-09-03T05:59:02.927 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.928 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.928 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.929 DEBUG:boto:Params: {} 2022-09-03T05:59:02.929 DEBUG:boto:Token: None 2022-09-03T05:59:02.930 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/ 2022-09-03T05:59:02.931 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98= 2022-09-03T05:59:02.932 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5uWWi8sU6nqMYD1KSTMMQFGY98=', 'Content-Length': '0'} 2022-09-03T05:59:02.935 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067b88f91ecd2e8d6-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.936 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:02.937 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:02.938 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.938 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.939 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.939 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.940 DEBUG:boto:Method: GET 2022-09-03T05:59:02.941 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.941 DEBUG:boto:Data: 2022-09-03T05:59:02.942 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.942 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.943 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.943 DEBUG:boto:Params: {} 2022-09-03T05:59:02.944 DEBUG:boto:Token: None 2022-09-03T05:59:02.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1GQhCwOF8gCO2GovAaQDiYULNw= 2022-09-03T05:59:02.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1GQhCwOF8gCO2GovAaQDiYULNw=', 'Content-Length': '0'} 2022-09-03T05:59:02.948 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001b8a9cd664537767-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.949 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.950 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.951 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.951 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.952 DEBUG:boto:Method: GET 2022-09-03T05:59:02.952 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.953 DEBUG:boto:Data: 2022-09-03T05:59:02.953 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.954 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.955 DEBUG:boto:Params: {} 2022-09-03T05:59:02.956 DEBUG:boto:Token: None 2022-09-03T05:59:02.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1GQhCwOF8gCO2GovAaQDiYULNw= 2022-09-03T05:59:02.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1GQhCwOF8gCO2GovAaQDiYULNw=', 'Content-Length': '0'} 2022-09-03T05:59:02.960 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005f7bf58c087d48a7-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.961 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.961 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.962 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.963 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.963 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.964 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.964 DEBUG:boto:Data: 2022-09-03T05:59:02.965 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.966 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.967 DEBUG:boto:Params: {} 2022-09-03T05:59:02.967 DEBUG:boto:Token: None 2022-09-03T05:59:02.968 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t/GfbKCyxbSigFk4AWNyd3/ZI04= 2022-09-03T05:59:02.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t/GfbKCyxbSigFk4AWNyd3/ZI04=', 'Content-Length': '0'} 2022-09-03T05:59:02.972 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003218650176bdad9d-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.974 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.975 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.976 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.976 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.977 DEBUG:boto:Method: HEAD 2022-09-03T05:59:02.978 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.979 DEBUG:boto:Data: 2022-09-03T05:59:02.979 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:02.980 DEBUG:boto:Port: 8000 2022-09-03T05:59:02.981 DEBUG:boto:Params: {} 2022-09-03T05:59:02.982 DEBUG:boto:Token: None 2022-09-03T05:59:02.982 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t/GfbKCyxbSigFk4AWNyd3/ZI04= 2022-09-03T05:59:02.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t/GfbKCyxbSigFk4AWNyd3/ZI04=', 'Content-Length': '0'} 2022-09-03T05:59:02.986 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007e8e6fd0f8660f78-006312ed26-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.986 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:02.987 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.987 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.988 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.988 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.989 DEBUG:boto:Method: GET 2022-09-03T05:59:02.990 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.990 DEBUG:boto:Data: 2022-09-03T05:59:02.991 DEBUG:boto:Headers: {} 2022-09-03T05:59:02.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:02.992 DEBUG:boto:Port: 8001 2022-09-03T05:59:02.992 DEBUG:boto:Params: {} 2022-09-03T05:59:02.993 DEBUG:boto:Token: None 2022-09-03T05:59:02.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:02 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1GQhCwOF8gCO2GovAaQDiYULNw= 2022-09-03T05:59:02.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1GQhCwOF8gCO2GovAaQDiYULNw=', 'Content-Length': '0'} 2022-09-03T05:59:02.997 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003a32e4325d7c8d17-006312ed26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:02.998 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.998 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:02.999 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:02.999 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.000 DEBUG:boto:Method: GET 2022-09-03T05:59:03.000 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.001 DEBUG:boto:Data: 2022-09-03T05:59:03.002 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.002 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.003 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.003 DEBUG:boto:Params: {} 2022-09-03T05:59:03.004 DEBUG:boto:Token: None 2022-09-03T05:59:03.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI= 2022-09-03T05:59:03.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI=', 'Content-Length': '0'} 2022-09-03T05:59:03.008 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bf968f736cf06398-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.009 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.010 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.010 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.011 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.012 DEBUG:boto:Method: GET 2022-09-03T05:59:03.012 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.013 DEBUG:boto:Data: 2022-09-03T05:59:03.013 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.014 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.015 DEBUG:boto:Params: {} 2022-09-03T05:59:03.015 DEBUG:boto:Token: None 2022-09-03T05:59:03.016 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.020 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000de611bd07951305c-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.022 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.023 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.023 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.024 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.025 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.026 DEBUG:boto:Method: GET 2022-09-03T05:59:03.027 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.027 DEBUG:boto:Data: 2022-09-03T05:59:03.028 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.029 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.030 DEBUG:boto:Params: {} 2022-09-03T05:59:03.031 DEBUG:boto:Token: None 2022-09-03T05:59:03.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.037 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000194fc31721e3c6e8-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.038 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.039 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.039 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.040 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.040 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T05:59:03.041 DEBUG:boto:Data: 2022-09-03T05:59:03.041 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.042 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.042 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.043 DEBUG:boto:Params: {} 2022-09-03T05:59:03.043 DEBUG:boto:Token: None 2022-09-03T05:59:03.044 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone1 2022-09-03T05:59:03.044 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ph3UXZpecg2WKCo/ua7S22lqgys= 2022-09-03T05:59:03.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ph3UXZpecg2WKCo/ua7S22lqgys=', 'Content-Length': '0'} 2022-09-03T05:59:03.048 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000540789eaa5c23a12-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.048 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.049 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.049 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.050 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T05:59:03.050 DEBUG:boto:Data: 2022-09-03T05:59:03.051 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.052 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.052 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.052 DEBUG:boto:Params: {} 2022-09-03T05:59:03.053 DEBUG:boto:Token: None 2022-09-03T05:59:03.054 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone1 2022-09-03T05:59:03.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ph3UXZpecg2WKCo/ua7S22lqgys= 2022-09-03T05:59:03.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ph3UXZpecg2WKCo/ua7S22lqgys=', 'Content-Length': '0'} 2022-09-03T05:59:03.058 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000db78bb1138a3792b-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.059 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:03.059 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.060 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.061 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.061 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.062 DEBUG:boto:Method: GET 2022-09-03T05:59:03.062 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.063 DEBUG:boto:Data: 2022-09-03T05:59:03.064 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.064 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.065 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.066 DEBUG:boto:Params: {} 2022-09-03T05:59:03.067 DEBUG:boto:Token: None 2022-09-03T05:59:03.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI= 2022-09-03T05:59:03.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI=', 'Content-Length': '0'} 2022-09-03T05:59:03.074 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005a193a33f414d20b-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.075 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.075 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.076 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.076 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.077 DEBUG:boto:Method: GET 2022-09-03T05:59:03.078 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.079 DEBUG:boto:Data: 2022-09-03T05:59:03.079 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.080 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.080 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.081 DEBUG:boto:Params: {} 2022-09-03T05:59:03.082 DEBUG:boto:Token: None 2022-09-03T05:59:03.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI= 2022-09-03T05:59:03.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI=', 'Content-Length': '0'} 2022-09-03T05:59:03.086 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c7b4f6a9d48b66d4-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.130 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.131 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.132 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.132 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.133 DEBUG:boto:Method: GET 2022-09-03T05:59:03.133 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.134 DEBUG:boto:Data: 2022-09-03T05:59:03.135 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.136 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.136 DEBUG:boto:Params: {} 2022-09-03T05:59:03.137 DEBUG:boto:Token: None 2022-09-03T05:59:03.138 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.142 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000032397570ebb57c7a-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.143 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.144 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.144 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.145 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.145 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.146 DEBUG:boto:Method: GET 2022-09-03T05:59:03.146 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:03.147 DEBUG:boto:Data: 2022-09-03T05:59:03.148 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.149 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.150 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.150 DEBUG:boto:Params: {} 2022-09-03T05:59:03.152 DEBUG:boto:Token: None 2022-09-03T05:59:03.153 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.159 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008ff1483193b291ff-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.160 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.161 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:03.161 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.162 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.162 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.163 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.163 DEBUG:boto:Method: GET 2022-09-03T05:59:03.164 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.164 DEBUG:boto:Data: 2022-09-03T05:59:03.165 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.166 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.167 DEBUG:boto:Params: {} 2022-09-03T05:59:03.167 DEBUG:boto:Token: None 2022-09-03T05:59:03.168 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.169 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c= 2022-09-03T05:59:03.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c=', 'Content-Length': '0'} 2022-09-03T05:59:03.171 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c1f94777a1a4267f-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.172 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.173 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.173 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.174 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.175 DEBUG:boto:Method: GET 2022-09-03T05:59:03.175 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.176 DEBUG:boto:Data: 2022-09-03T05:59:03.176 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.177 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.177 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.178 DEBUG:boto:Params: {} 2022-09-03T05:59:03.178 DEBUG:boto:Token: None 2022-09-03T05:59:03.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c= 2022-09-03T05:59:03.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c=', 'Content-Length': '0'} 2022-09-03T05:59:03.182 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003d8139bf67e8723d-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.183 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.184 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.184 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.185 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.185 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.186 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.186 DEBUG:boto:Data: 2022-09-03T05:59:03.187 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.188 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.188 DEBUG:boto:Params: {} 2022-09-03T05:59:03.189 DEBUG:boto:Token: None 2022-09-03T05:59:03.189 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1VwMzQAQZIKG7Wyr8BWOH7M6hnM= 2022-09-03T05:59:03.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1VwMzQAQZIKG7Wyr8BWOH7M6hnM=', 'Content-Length': '0'} 2022-09-03T05:59:03.193 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000193e87aacb8013f9-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.193 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.194 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.194 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.195 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.195 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.196 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.196 DEBUG:boto:Data: 2022-09-03T05:59:03.197 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.198 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.198 DEBUG:boto:Params: {} 2022-09-03T05:59:03.199 DEBUG:boto:Token: None 2022-09-03T05:59:03.200 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1VwMzQAQZIKG7Wyr8BWOH7M6hnM= 2022-09-03T05:59:03.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1VwMzQAQZIKG7Wyr8BWOH7M6hnM=', 'Content-Length': '0'} 2022-09-03T05:59:03.203 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a3822a5fdf51d302-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.203 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:03.204 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.205 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.205 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.206 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.207 DEBUG:boto:Method: GET 2022-09-03T05:59:03.207 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.208 DEBUG:boto:Data: 2022-09-03T05:59:03.208 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.209 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.209 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.210 DEBUG:boto:Params: {} 2022-09-03T05:59:03.210 DEBUG:boto:Token: None 2022-09-03T05:59:03.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c= 2022-09-03T05:59:03.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c=', 'Content-Length': '0'} 2022-09-03T05:59:03.215 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000afae93e09e2561c9-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.216 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.217 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.218 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.219 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.220 DEBUG:boto:Method: GET 2022-09-03T05:59:03.221 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.222 DEBUG:boto:Data: 2022-09-03T05:59:03.223 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.224 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.225 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.226 DEBUG:boto:Params: {} 2022-09-03T05:59:03.227 DEBUG:boto:Token: None 2022-09-03T05:59:03.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c= 2022-09-03T05:59:03.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c=', 'Content-Length': '0'} 2022-09-03T05:59:03.231 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000087726e75359c234b-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.274 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.275 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.275 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.276 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.276 DEBUG:boto:Method: GET 2022-09-03T05:59:03.277 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.277 DEBUG:boto:Data: 2022-09-03T05:59:03.278 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.279 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.279 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.280 DEBUG:boto:Params: {} 2022-09-03T05:59:03.280 DEBUG:boto:Token: None 2022-09-03T05:59:03.281 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.285 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000af1221eae29bcfb2-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.286 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.287 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.287 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.288 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.288 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.289 DEBUG:boto:Method: GET 2022-09-03T05:59:03.290 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.291 DEBUG:boto:Data: 2022-09-03T05:59:03.292 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.294 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.295 DEBUG:boto:Params: {} 2022-09-03T05:59:03.297 DEBUG:boto:Token: None 2022-09-03T05:59:03.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.302 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000078a0329c58367ab6-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.303 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.304 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.305 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.306 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.307 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T05:59:03.308 DEBUG:boto:Data: 2022-09-03T05:59:03.309 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.310 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.311 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.312 DEBUG:boto:Params: {} 2022-09-03T05:59:03.313 DEBUG:boto:Token: None 2022-09-03T05:59:03.314 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2 2022-09-03T05:59:03.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KZ4lUiC44aU8xaCGA7uKmiHLcug= 2022-09-03T05:59:03.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KZ4lUiC44aU8xaCGA7uKmiHLcug=', 'Content-Length': '0'} 2022-09-03T05:59:03.319 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000168cc5288d950ffc-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.320 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.321 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.322 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.322 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T05:59:03.323 DEBUG:boto:Data: 2022-09-03T05:59:03.324 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.325 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.326 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.327 DEBUG:boto:Params: {} 2022-09-03T05:59:03.328 DEBUG:boto:Token: None 2022-09-03T05:59:03.328 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2 2022-09-03T05:59:03.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KZ4lUiC44aU8xaCGA7uKmiHLcug= 2022-09-03T05:59:03.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KZ4lUiC44aU8xaCGA7uKmiHLcug=', 'Content-Length': '0'} 2022-09-03T05:59:03.332 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e78eaecd72b56f8e-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.332 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:03.333 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.334 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.334 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.335 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.335 DEBUG:boto:Method: GET 2022-09-03T05:59:03.336 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.336 DEBUG:boto:Data: 2022-09-03T05:59:03.337 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.338 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.338 DEBUG:boto:Params: {} 2022-09-03T05:59:03.339 DEBUG:boto:Token: None 2022-09-03T05:59:03.340 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c= 2022-09-03T05:59:03.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c=', 'Content-Length': '0'} 2022-09-03T05:59:03.343 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fffcbfa655257257-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.344 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.345 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:03.345 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.346 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.346 DEBUG:boto:Method: GET 2022-09-03T05:59:03.347 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.348 DEBUG:boto:Data: 2022-09-03T05:59:03.348 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.349 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.349 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.350 DEBUG:boto:Params: {} 2022-09-03T05:59:03.351 DEBUG:boto:Token: None 2022-09-03T05:59:03.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:03.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c= 2022-09-03T05:59:03.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yTT5y5TzErUC2bvCXQLCLZNuT+c=', 'Content-Length': '0'} 2022-09-03T05:59:03.355 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d711adaf37007053-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.355 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.356 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.357 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.357 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.358 DEBUG:boto:Method: GET 2022-09-03T05:59:03.358 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.359 DEBUG:boto:Data: 2022-09-03T05:59:03.360 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.361 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.361 DEBUG:boto:Params: {} 2022-09-03T05:59:03.362 DEBUG:boto:Token: None 2022-09-03T05:59:03.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.367 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000052bf3b16c685351f-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.367 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.368 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.369 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.369 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.370 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.370 DEBUG:boto:Method: GET 2022-09-03T05:59:03.371 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:03.371 DEBUG:boto:Data: 2022-09-03T05:59:03.372 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.372 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.373 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.373 DEBUG:boto:Params: {} 2022-09-03T05:59:03.374 DEBUG:boto:Token: None 2022-09-03T05:59:03.375 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.379 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007053fb5cf1e8fb58-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.428 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.429 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:03.430 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.431 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.432 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.433 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.434 DEBUG:boto:Method: GET 2022-09-03T05:59:03.435 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.436 DEBUG:boto:Data: 2022-09-03T05:59:03.437 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.439 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.440 DEBUG:boto:Params: {} 2022-09-03T05:59:03.442 DEBUG:boto:Token: None 2022-09-03T05:59:03.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ= 2022-09-03T05:59:03.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ=', 'Content-Length': '0'} 2022-09-03T05:59:03.447 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f3add8d8e63b85d0-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.449 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.449 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.450 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.450 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.451 DEBUG:boto:Method: GET 2022-09-03T05:59:03.452 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.452 DEBUG:boto:Data: 2022-09-03T05:59:03.453 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.454 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.454 DEBUG:boto:Params: {} 2022-09-03T05:59:03.455 DEBUG:boto:Token: None 2022-09-03T05:59:03.457 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ= 2022-09-03T05:59:03.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ=', 'Content-Length': '0'} 2022-09-03T05:59:03.460 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000157679395447cec9-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.461 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.462 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.464 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.465 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.465 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.466 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.466 DEBUG:boto:Data: 2022-09-03T05:59:03.467 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.468 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.468 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.468 DEBUG:boto:Params: {} 2022-09-03T05:59:03.469 DEBUG:boto:Token: None 2022-09-03T05:59:03.470 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:td2eRSLT7ZQtVVInb24EQiUH08E= 2022-09-03T05:59:03.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:td2eRSLT7ZQtVVInb24EQiUH08E=', 'Content-Length': '0'} 2022-09-03T05:59:03.473 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005c92e4ff6da655e8-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.475 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.475 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.476 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.477 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.477 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.478 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.479 DEBUG:boto:Data: 2022-09-03T05:59:03.480 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.481 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.481 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.482 DEBUG:boto:Params: {} 2022-09-03T05:59:03.483 DEBUG:boto:Token: None 2022-09-03T05:59:03.484 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:td2eRSLT7ZQtVVInb24EQiUH08E= 2022-09-03T05:59:03.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:td2eRSLT7ZQtVVInb24EQiUH08E=', 'Content-Length': '0'} 2022-09-03T05:59:03.488 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000025e37340d7b9fc4d-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.489 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:03.489 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.490 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.491 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.491 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.492 DEBUG:boto:Method: GET 2022-09-03T05:59:03.492 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.493 DEBUG:boto:Data: 2022-09-03T05:59:03.494 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.495 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.496 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.496 DEBUG:boto:Params: {} 2022-09-03T05:59:03.497 DEBUG:boto:Token: None 2022-09-03T05:59:03.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ= 2022-09-03T05:59:03.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ=', 'Content-Length': '0'} 2022-09-03T05:59:03.501 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009927d88ba956a51c-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.502 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.503 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.504 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.505 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.505 DEBUG:boto:Method: GET 2022-09-03T05:59:03.506 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.507 DEBUG:boto:Data: 2022-09-03T05:59:03.507 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.508 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.509 DEBUG:boto:Params: {} 2022-09-03T05:59:03.509 DEBUG:boto:Token: None 2022-09-03T05:59:03.510 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ= 2022-09-03T05:59:03.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ=', 'Content-Length': '0'} 2022-09-03T05:59:03.514 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000003ee5fcc1ded5aa6-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.515 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.516 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.516 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.517 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.517 DEBUG:boto:Method: GET 2022-09-03T05:59:03.518 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.518 DEBUG:boto:Data: 2022-09-03T05:59:03.519 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.519 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.520 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.520 DEBUG:boto:Params: {} 2022-09-03T05:59:03.521 DEBUG:boto:Token: None 2022-09-03T05:59:03.522 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.526 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001ac117db0b065365-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.527 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.528 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.528 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.529 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.529 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.530 DEBUG:boto:Method: GET 2022-09-03T05:59:03.531 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.531 DEBUG:boto:Data: 2022-09-03T05:59:03.532 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.532 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.533 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.534 DEBUG:boto:Params: {} 2022-09-03T05:59:03.534 DEBUG:boto:Token: None 2022-09-03T05:59:03.535 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.536 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.539 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a1e9ff458b508ad-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.540 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.541 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.541 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.542 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.542 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T05:59:03.543 DEBUG:boto:Data: 2022-09-03T05:59:03.543 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.544 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.544 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.545 DEBUG:boto:Params: {} 2022-09-03T05:59:03.545 DEBUG:boto:Token: None 2022-09-03T05:59:03.546 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3 2022-09-03T05:59:03.546 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nX9CCUboY/AnjDNKkxDqYUveRuY= 2022-09-03T05:59:03.547 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nX9CCUboY/AnjDNKkxDqYUveRuY=', 'Content-Length': '0'} 2022-09-03T05:59:03.549 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000367a0749310172ec-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.550 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.551 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.551 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.552 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T05:59:03.552 DEBUG:boto:Data: 2022-09-03T05:59:03.553 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.553 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.554 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.554 DEBUG:boto:Params: {} 2022-09-03T05:59:03.555 DEBUG:boto:Token: None 2022-09-03T05:59:03.556 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3 2022-09-03T05:59:03.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nX9CCUboY/AnjDNKkxDqYUveRuY= 2022-09-03T05:59:03.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nX9CCUboY/AnjDNKkxDqYUveRuY=', 'Content-Length': '0'} 2022-09-03T05:59:03.560 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d60ca637d8ba203c-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.561 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:03.562 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.562 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.563 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.564 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.564 DEBUG:boto:Method: GET 2022-09-03T05:59:03.565 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.565 DEBUG:boto:Data: 2022-09-03T05:59:03.566 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.566 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.567 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.568 DEBUG:boto:Params: {} 2022-09-03T05:59:03.568 DEBUG:boto:Token: None 2022-09-03T05:59:03.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ= 2022-09-03T05:59:03.570 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ=', 'Content-Length': '0'} 2022-09-03T05:59:03.572 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000613313e83bf52501-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.614 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.615 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:03.615 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.616 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.616 DEBUG:boto:Method: GET 2022-09-03T05:59:03.617 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.617 DEBUG:boto:Data: 2022-09-03T05:59:03.618 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.618 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.619 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.620 DEBUG:boto:Params: {} 2022-09-03T05:59:03.620 DEBUG:boto:Token: None 2022-09-03T05:59:03.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:03.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ= 2022-09-03T05:59:03.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2knGrwNnOC2+tJyZoj3RG1EjLqQ=', 'Content-Length': '0'} 2022-09-03T05:59:03.625 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000440a1594d3f686cf-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.626 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.626 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.627 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.627 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.628 DEBUG:boto:Method: GET 2022-09-03T05:59:03.628 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.629 DEBUG:boto:Data: 2022-09-03T05:59:03.629 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.631 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.631 DEBUG:boto:Params: {} 2022-09-03T05:59:03.632 DEBUG:boto:Token: None 2022-09-03T05:59:03.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.636 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000016457097f8b0ac6e-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.637 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.638 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.639 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.639 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.640 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.640 DEBUG:boto:Method: GET 2022-09-03T05:59:03.641 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:03.641 DEBUG:boto:Data: 2022-09-03T05:59:03.642 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:03.643 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.644 DEBUG:boto:Params: {} 2022-09-03T05:59:03.644 DEBUG:boto:Token: None 2022-09-03T05:59:03.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.649 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003c7527c2ed3708ad-006312ed27-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.650 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.650 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-16 zones={test-zone3, test-zone1} 2022-09-03T05:59:03.651 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-16 zones={test-zone3, test-zone2} 2022-09-03T05:59:03.652 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.652 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.653 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.653 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:59:03.654 DEBUG:boto:Data: 2022-09-03T05:59:03.654 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.655 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.656 DEBUG:boto:Params: {} 2022-09-03T05:59:03.656 DEBUG:boto:Token: None 2022-09-03T05:59:03.657 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D5yPexsMoDNs6SvNyFEkulXFBeg= 2022-09-03T05:59:03.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D5yPexsMoDNs6SvNyFEkulXFBeg=', 'Content-Length': '0'} 2022-09-03T05:59:03.661 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bdaed6ddcc8085ed-006312ed27-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.662 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.662 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.663 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.663 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:59:03.664 DEBUG:boto:Data: 2022-09-03T05:59:03.664 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.665 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.666 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.666 DEBUG:boto:Params: {} 2022-09-03T05:59:03.667 DEBUG:boto:Token: None 2022-09-03T05:59:03.667 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.668 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D5yPexsMoDNs6SvNyFEkulXFBeg= 2022-09-03T05:59:03.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D5yPexsMoDNs6SvNyFEkulXFBeg=', 'Content-Length': '0'} 2022-09-03T05:59:03.671 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002d521e419f38f537-006312ed27-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.672 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:59:03.672 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.673 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.673 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.675 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.676 DEBUG:boto:Method: GET 2022-09-03T05:59:03.676 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.677 DEBUG:boto:Data: 2022-09-03T05:59:03.677 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.678 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.679 DEBUG:boto:Params: {} 2022-09-03T05:59:03.680 DEBUG:boto:Token: None 2022-09-03T05:59:03.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/?versions 2022-09-03T05:59:03.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TchPfpSnqd6B8URdsSbiV2cZ/nM= 2022-09-03T05:59:03.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TchPfpSnqd6B8URdsSbiV2cZ/nM=', 'Content-Length': '0'} 2022-09-03T05:59:03.684 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e25e52a633929050-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.685 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.686 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:03.686 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:03.687 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:03.687 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:03.688 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:59:03.688 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.689 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.689 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.690 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.690 DEBUG:boto:Method: GET 2022-09-03T05:59:03.691 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.692 DEBUG:boto:Data: 2022-09-03T05:59:03.692 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.693 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.693 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.694 DEBUG:boto:Params: {} 2022-09-03T05:59:03.694 DEBUG:boto:Token: None 2022-09-03T05:59:03.695 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/?versions 2022-09-03T05:59:03.695 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TchPfpSnqd6B8URdsSbiV2cZ/nM= 2022-09-03T05:59:03.696 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TchPfpSnqd6B8URdsSbiV2cZ/nM=', 'Content-Length': '0'} 2022-09-03T05:59:03.699 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000196d6732d56f4979-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.699 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.700 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:03.701 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:03.701 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:03.702 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:03.702 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.703 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.703 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.704 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.704 DEBUG:boto:Method: GET 2022-09-03T05:59:03.705 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.705 DEBUG:boto:Data: 2022-09-03T05:59:03.706 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.707 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.708 DEBUG:boto:Params: {} 2022-09-03T05:59:03.708 DEBUG:boto:Token: None 2022-09-03T05:59:03.709 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/?versions 2022-09-03T05:59:03.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TchPfpSnqd6B8URdsSbiV2cZ/nM= 2022-09-03T05:59:03.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TchPfpSnqd6B8URdsSbiV2cZ/nM=', 'Content-Length': '0'} 2022-09-03T05:59:03.713 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000469605a9ac44ae7e-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.713 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.714 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.715 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.716 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.716 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.717 DEBUG:boto:Method: GET 2022-09-03T05:59:03.717 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T05:59:03.718 DEBUG:boto:Data: 2022-09-03T05:59:03.718 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.719 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.719 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.720 DEBUG:boto:Params: {} 2022-09-03T05:59:03.721 DEBUG:boto:Token: None 2022-09-03T05:59:03.721 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/?versions 2022-09-03T05:59:03.722 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TchPfpSnqd6B8URdsSbiV2cZ/nM= 2022-09-03T05:59:03.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TchPfpSnqd6B8URdsSbiV2cZ/nM=', 'Content-Length': '0'} 2022-09-03T05:59:03.726 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d3d88a030b5689a-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.726 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.727 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:03.728 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.728 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.729 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.729 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.730 DEBUG:boto:Method: GET 2022-09-03T05:59:03.731 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.731 DEBUG:boto:Data: 2022-09-03T05:59:03.732 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.733 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.734 DEBUG:boto:Params: {} 2022-09-03T05:59:03.734 DEBUG:boto:Token: None 2022-09-03T05:59:03.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c= 2022-09-03T05:59:03.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c=', 'Content-Length': '0'} 2022-09-03T05:59:03.738 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000028d08d1d047f4e8b-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.739 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.740 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.740 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.741 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.741 DEBUG:boto:Method: GET 2022-09-03T05:59:03.742 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.742 DEBUG:boto:Data: 2022-09-03T05:59:03.743 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.744 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.744 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.745 DEBUG:boto:Params: {} 2022-09-03T05:59:03.745 DEBUG:boto:Token: None 2022-09-03T05:59:03.746 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c= 2022-09-03T05:59:03.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c=', 'Content-Length': '0'} 2022-09-03T05:59:03.749 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001c52d380922c84fe-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.750 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.751 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.751 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.752 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.752 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.753 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.753 DEBUG:boto:Data: 2022-09-03T05:59:03.754 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.754 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.755 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.755 DEBUG:boto:Params: {} 2022-09-03T05:59:03.756 DEBUG:boto:Token: None 2022-09-03T05:59:03.757 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fpA8bDy1kosrm6DsPZX7/e+2HA8= 2022-09-03T05:59:03.758 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fpA8bDy1kosrm6DsPZX7/e+2HA8=', 'Content-Length': '0'} 2022-09-03T05:59:03.760 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000bcca50c699d3fd5d-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.761 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.762 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.762 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.763 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.763 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.764 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.764 DEBUG:boto:Data: 2022-09-03T05:59:03.765 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.765 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.766 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.766 DEBUG:boto:Params: {} 2022-09-03T05:59:03.767 DEBUG:boto:Token: None 2022-09-03T05:59:03.767 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fpA8bDy1kosrm6DsPZX7/e+2HA8= 2022-09-03T05:59:03.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fpA8bDy1kosrm6DsPZX7/e+2HA8=', 'Content-Length': '0'} 2022-09-03T05:59:03.770 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000cb31aa8f26a5db3f-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.771 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:03.772 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.773 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.773 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.774 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.774 DEBUG:boto:Method: GET 2022-09-03T05:59:03.775 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.775 DEBUG:boto:Data: 2022-09-03T05:59:03.776 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.777 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.778 DEBUG:boto:Params: {} 2022-09-03T05:59:03.778 DEBUG:boto:Token: None 2022-09-03T05:59:03.779 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c= 2022-09-03T05:59:03.780 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c=', 'Content-Length': '0'} 2022-09-03T05:59:03.782 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003677c3f407e53bb2-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.783 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.784 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.784 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.785 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.785 DEBUG:boto:Method: GET 2022-09-03T05:59:03.786 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.787 DEBUG:boto:Data: 2022-09-03T05:59:03.787 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.788 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.788 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.789 DEBUG:boto:Params: {} 2022-09-03T05:59:03.789 DEBUG:boto:Token: None 2022-09-03T05:59:03.790 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.791 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c= 2022-09-03T05:59:03.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c=', 'Content-Length': '0'} 2022-09-03T05:59:03.793 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ad0d9d1ffd5989f2-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.794 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.795 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.795 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:03.796 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:03.796 DEBUG:boto:Method: GET 2022-09-03T05:59:03.797 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:03.797 DEBUG:boto:Data: 2022-09-03T05:59:03.798 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.799 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.799 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.800 DEBUG:boto:Params: {} 2022-09-03T05:59:03.800 DEBUG:boto:Token: None 2022-09-03T05:59:03.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.805 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007e4c8672aaed8009-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.846 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.848 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.848 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.849 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:03.849 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:03.850 DEBUG:boto:Method: GET 2022-09-03T05:59:03.850 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:03.851 DEBUG:boto:Data: 2022-09-03T05:59:03.852 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.852 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.853 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.853 DEBUG:boto:Params: {} 2022-09-03T05:59:03.854 DEBUG:boto:Token: None 2022-09-03T05:59:03.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.855 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.859 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000682a422b3c14d484-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.861 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.862 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.862 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.863 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.864 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T05:59:03.864 DEBUG:boto:Data: 2022-09-03T05:59:03.864 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.866 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.866 DEBUG:boto:Params: {} 2022-09-03T05:59:03.867 DEBUG:boto:Token: None 2022-09-03T05:59:03.868 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy 2022-09-03T05:59:03.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WFG/q57AiKIztuspjkO6+zi+Ywk= 2022-09-03T05:59:03.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WFG/q57AiKIztuspjkO6+zi+Ywk=', 'Content-Length': '0'} 2022-09-03T05:59:03.872 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000000dddd5d8868de9f4-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.874 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.876 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.876 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.877 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T05:59:03.878 DEBUG:boto:Data: 2022-09-03T05:59:03.879 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.880 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.881 DEBUG:boto:Params: {} 2022-09-03T05:59:03.882 DEBUG:boto:Token: None 2022-09-03T05:59:03.883 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy 2022-09-03T05:59:03.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WFG/q57AiKIztuspjkO6+zi+Ywk= 2022-09-03T05:59:03.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WFG/q57AiKIztuspjkO6+zi+Ywk=', 'Content-Length': '0'} 2022-09-03T05:59:03.887 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b9f017279b42f57a-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.888 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:03.889 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.890 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.890 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.891 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.892 DEBUG:boto:Method: GET 2022-09-03T05:59:03.893 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.894 DEBUG:boto:Data: 2022-09-03T05:59:03.895 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.897 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.898 DEBUG:boto:Params: {} 2022-09-03T05:59:03.900 DEBUG:boto:Token: None 2022-09-03T05:59:03.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c= 2022-09-03T05:59:03.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c=', 'Content-Length': '0'} 2022-09-03T05:59:03.905 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000067435f5a5346502d-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.906 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T05:59:03.907 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T05:59:03.908 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.908 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.909 DEBUG:boto:Method: GET 2022-09-03T05:59:03.909 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.910 DEBUG:boto:Data: 2022-09-03T05:59:03.911 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.911 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.912 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.912 DEBUG:boto:Params: {} 2022-09-03T05:59:03.913 DEBUG:boto:Token: None 2022-09-03T05:59:03.914 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T05:59:03.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c= 2022-09-03T05:59:03.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJT7fQIZ9v9KckeSCJo8FRZw85c=', 'Content-Length': '0'} 2022-09-03T05:59:03.918 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006e27a75da21cf2c2-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.919 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.920 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.921 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:03.922 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:03.923 DEBUG:boto:Method: GET 2022-09-03T05:59:03.924 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:03.925 DEBUG:boto:Data: 2022-09-03T05:59:03.926 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.927 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.928 DEBUG:boto:Params: {} 2022-09-03T05:59:03.929 DEBUG:boto:Token: None 2022-09-03T05:59:03.930 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.931 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.935 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000889fb73338ec656c-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.936 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.938 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:03.939 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:03.939 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:03.940 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T05:59:03.940 DEBUG:boto:Method: GET 2022-09-03T05:59:03.941 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T05:59:03.941 DEBUG:boto:Data: 2022-09-03T05:59:03.942 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.943 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.943 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.944 DEBUG:boto:Params: {} 2022-09-03T05:59:03.944 DEBUG:boto:Token: None 2022-09-03T05:59:03.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/ 2022-09-03T05:59:03.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s= 2022-09-03T05:59:03.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IwZlZOsfmERLSrF78ZHECc+/U2s=', 'Content-Length': '0'} 2022-09-03T05:59:03.949 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f11be6d2793239cc-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.950 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:03.950 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:03.951 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.952 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.952 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.953 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.953 DEBUG:boto:Method: GET 2022-09-03T05:59:03.954 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.954 DEBUG:boto:Data: 2022-09-03T05:59:03.955 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.956 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.956 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.957 DEBUG:boto:Params: {} 2022-09-03T05:59:03.957 DEBUG:boto:Token: None 2022-09-03T05:59:03.958 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.959 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI= 2022-09-03T05:59:03.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI=', 'Content-Length': '0'} 2022-09-03T05:59:03.961 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cb42d12b725d1a0e-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.962 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.963 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.963 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.964 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.965 DEBUG:boto:Method: GET 2022-09-03T05:59:03.965 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.966 DEBUG:boto:Data: 2022-09-03T05:59:03.966 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.967 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.967 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.968 DEBUG:boto:Params: {} 2022-09-03T05:59:03.968 DEBUG:boto:Token: None 2022-09-03T05:59:03.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI= 2022-09-03T05:59:03.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oftSlrYZkkXHkvBzOLgHoq4WdJI=', 'Content-Length': '0'} 2022-09-03T05:59:03.973 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b0550eeaff3b52fd-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.973 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.974 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.975 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.975 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.976 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.976 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.977 DEBUG:boto:Data: 2022-09-03T05:59:03.978 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:03.979 DEBUG:boto:Port: 8001 2022-09-03T05:59:03.979 DEBUG:boto:Params: {} 2022-09-03T05:59:03.980 DEBUG:boto:Token: None 2022-09-03T05:59:03.980 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AxWO9Bbqmpyy1YCFx5Vb8QYOm8s= 2022-09-03T05:59:03.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AxWO9Bbqmpyy1YCFx5Vb8QYOm8s=', 'Content-Length': '0'} 2022-09-03T05:59:03.984 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000090c29ebd6f9f8b-006312ed27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.984 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.985 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.986 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.986 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.987 DEBUG:boto:Method: HEAD 2022-09-03T05:59:03.987 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.988 DEBUG:boto:Data: 2022-09-03T05:59:03.988 DEBUG:boto:Headers: {} 2022-09-03T05:59:03.989 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:03.989 DEBUG:boto:Port: 8000 2022-09-03T05:59:03.990 DEBUG:boto:Params: {} 2022-09-03T05:59:03.990 DEBUG:boto:Token: None 2022-09-03T05:59:03.991 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:03 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AxWO9Bbqmpyy1YCFx5Vb8QYOm8s= 2022-09-03T05:59:03.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AxWO9Bbqmpyy1YCFx5Vb8QYOm8s=', 'Content-Length': '0'} 2022-09-03T05:59:03.994 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a4fdd6a09636e252-006312ed27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:03.995 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:03.996 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.996 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:03.997 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.997 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.998 DEBUG:boto:Method: GET 2022-09-03T05:59:03.998 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:03.999 DEBUG:boto:Data: 2022-09-03T05:59:03.999 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.000 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.001 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.001 DEBUG:boto:Params: {} 2022-09-03T05:59:04.002 DEBUG:boto:Token: None 2022-09-03T05:59:04.002 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f/Fg+Zia9GqkqPqlHdYISmjwdtY= 2022-09-03T05:59:04.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f/Fg+Zia9GqkqPqlHdYISmjwdtY=', 'Content-Length': '0'} 2022-09-03T05:59:04.006 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003c3ebc3a0474d7db-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.007 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.007 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.008 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.008 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.009 DEBUG:boto:Method: GET 2022-09-03T05:59:04.010 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.010 DEBUG:boto:Data: 2022-09-03T05:59:04.011 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.011 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.012 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.012 DEBUG:boto:Params: {} 2022-09-03T05:59:04.013 DEBUG:boto:Token: None 2022-09-03T05:59:04.014 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f/Fg+Zia9GqkqPqlHdYISmjwdtY= 2022-09-03T05:59:04.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f/Fg+Zia9GqkqPqlHdYISmjwdtY=', 'Content-Length': '0'} 2022-09-03T05:59:04.017 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002a4ff2b8a4514187-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.058 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.060 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.061 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.070 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.071 DEBUG:boto:Method: GET 2022-09-03T05:59:04.072 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.074 DEBUG:boto:Data: 2022-09-03T05:59:04.075 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.078 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.079 DEBUG:boto:Params: {} 2022-09-03T05:59:04.079 DEBUG:boto:Token: None 2022-09-03T05:59:04.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.086 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000082994a6c316718b8-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.087 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.088 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.089 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.090 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.092 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.092 DEBUG:boto:Method: GET 2022-09-03T05:59:04.093 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.094 DEBUG:boto:Data: 2022-09-03T05:59:04.094 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.096 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.096 DEBUG:boto:Params: {} 2022-09-03T05:59:04.097 DEBUG:boto:Token: None 2022-09-03T05:59:04.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.102 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079395797e267de86-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.103 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.105 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.105 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.106 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.107 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T05:59:04.107 DEBUG:boto:Data: 2022-09-03T05:59:04.108 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.109 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.110 DEBUG:boto:Params: {} 2022-09-03T05:59:04.110 DEBUG:boto:Token: None 2022-09-03T05:59:04.111 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone1 2022-09-03T05:59:04.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuv+l3smXXoJBLVIybWkODC3x0k= 2022-09-03T05:59:04.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuv+l3smXXoJBLVIybWkODC3x0k=', 'Content-Length': '0'} 2022-09-03T05:59:04.116 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f2a6a5b2eafa49c6-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.117 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.117 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.118 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.119 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T05:59:04.119 DEBUG:boto:Data: 2022-09-03T05:59:04.120 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.120 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.121 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.122 DEBUG:boto:Params: {} 2022-09-03T05:59:04.122 DEBUG:boto:Token: None 2022-09-03T05:59:04.123 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone1 2022-09-03T05:59:04.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuv+l3smXXoJBLVIybWkODC3x0k= 2022-09-03T05:59:04.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuv+l3smXXoJBLVIybWkODC3x0k=', 'Content-Length': '0'} 2022-09-03T05:59:04.130 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006f1fc86362bc917d-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.131 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:04.131 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.132 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.132 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.133 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.135 DEBUG:boto:Method: GET 2022-09-03T05:59:04.136 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.137 DEBUG:boto:Data: 2022-09-03T05:59:04.137 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.138 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.139 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.140 DEBUG:boto:Params: {} 2022-09-03T05:59:04.141 DEBUG:boto:Token: None 2022-09-03T05:59:04.142 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f/Fg+Zia9GqkqPqlHdYISmjwdtY= 2022-09-03T05:59:04.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f/Fg+Zia9GqkqPqlHdYISmjwdtY=', 'Content-Length': '0'} 2022-09-03T05:59:04.148 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d9b7648ec77bae25-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.190 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.191 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:04.191 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.192 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.193 DEBUG:boto:Method: GET 2022-09-03T05:59:04.193 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.194 DEBUG:boto:Data: 2022-09-03T05:59:04.194 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.195 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.195 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.196 DEBUG:boto:Params: {} 2022-09-03T05:59:04.196 DEBUG:boto:Token: None 2022-09-03T05:59:04.197 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T05:59:04.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f/Fg+Zia9GqkqPqlHdYISmjwdtY= 2022-09-03T05:59:04.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f/Fg+Zia9GqkqPqlHdYISmjwdtY=', 'Content-Length': '0'} 2022-09-03T05:59:04.201 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e625ae48cf1738a9-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.202 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.202 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.203 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.203 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.204 DEBUG:boto:Method: GET 2022-09-03T05:59:04.204 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.205 DEBUG:boto:Data: 2022-09-03T05:59:04.205 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.206 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.207 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.207 DEBUG:boto:Params: {} 2022-09-03T05:59:04.208 DEBUG:boto:Token: None 2022-09-03T05:59:04.208 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.213 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000192f937d203a3643-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.214 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.215 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.216 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.216 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.217 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.217 DEBUG:boto:Method: GET 2022-09-03T05:59:04.218 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T05:59:04.218 DEBUG:boto:Data: 2022-09-03T05:59:04.219 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.219 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.220 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.220 DEBUG:boto:Params: {} 2022-09-03T05:59:04.221 DEBUG:boto:Token: None 2022-09-03T05:59:04.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.226 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000980b0c1f9a68071a-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.227 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.229 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:04.230 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.230 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.231 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.232 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.233 DEBUG:boto:Method: GET 2022-09-03T05:59:04.233 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.234 DEBUG:boto:Data: 2022-09-03T05:59:04.235 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.236 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.237 DEBUG:boto:Params: {} 2022-09-03T05:59:04.238 DEBUG:boto:Token: None 2022-09-03T05:59:04.238 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo= 2022-09-03T05:59:04.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo=', 'Content-Length': '0'} 2022-09-03T05:59:04.243 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000f72121d563ea685-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.245 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.246 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.247 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.248 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.249 DEBUG:boto:Method: GET 2022-09-03T05:59:04.250 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.251 DEBUG:boto:Data: 2022-09-03T05:59:04.252 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.253 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.254 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.255 DEBUG:boto:Params: {} 2022-09-03T05:59:04.256 DEBUG:boto:Token: None 2022-09-03T05:59:04.258 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo= 2022-09-03T05:59:04.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo=', 'Content-Length': '0'} 2022-09-03T05:59:04.263 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000070aa505bb85bc8c1-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.264 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.265 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.266 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.266 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.267 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.268 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.268 DEBUG:boto:Data: 2022-09-03T05:59:04.269 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.270 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.270 DEBUG:boto:Params: {} 2022-09-03T05:59:04.271 DEBUG:boto:Token: None 2022-09-03T05:59:04.272 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c0ZtcFm69OB1w47MjaggGRhYo+4= 2022-09-03T05:59:04.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c0ZtcFm69OB1w47MjaggGRhYo+4=', 'Content-Length': '0'} 2022-09-03T05:59:04.276 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000023ec5b20d4f4a109-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.277 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.277 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.278 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.278 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.279 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.279 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.280 DEBUG:boto:Data: 2022-09-03T05:59:04.280 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.281 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.281 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.282 DEBUG:boto:Params: {} 2022-09-03T05:59:04.283 DEBUG:boto:Token: None 2022-09-03T05:59:04.283 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.284 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c0ZtcFm69OB1w47MjaggGRhYo+4= 2022-09-03T05:59:04.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c0ZtcFm69OB1w47MjaggGRhYo+4=', 'Content-Length': '0'} 2022-09-03T05:59:04.287 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cec5a706699dfff6-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.288 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:04.288 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.289 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.289 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.290 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.290 DEBUG:boto:Method: GET 2022-09-03T05:59:04.291 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.292 DEBUG:boto:Data: 2022-09-03T05:59:04.292 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.293 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.294 DEBUG:boto:Params: {} 2022-09-03T05:59:04.294 DEBUG:boto:Token: None 2022-09-03T05:59:04.295 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo= 2022-09-03T05:59:04.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo=', 'Content-Length': '0'} 2022-09-03T05:59:04.299 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006a1d515d2d1dd0f8-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.300 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.300 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.301 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.301 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.302 DEBUG:boto:Method: GET 2022-09-03T05:59:04.303 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.303 DEBUG:boto:Data: 2022-09-03T05:59:04.304 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.305 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.305 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.306 DEBUG:boto:Params: {} 2022-09-03T05:59:04.307 DEBUG:boto:Token: None 2022-09-03T05:59:04.307 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo= 2022-09-03T05:59:04.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo=', 'Content-Length': '0'} 2022-09-03T05:59:04.311 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000063586fadf8b80071-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.355 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.356 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.357 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.358 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.359 DEBUG:boto:Method: GET 2022-09-03T05:59:04.361 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.362 DEBUG:boto:Data: 2022-09-03T05:59:04.363 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.364 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.364 DEBUG:boto:Params: {} 2022-09-03T05:59:04.365 DEBUG:boto:Token: None 2022-09-03T05:59:04.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.370 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000732219f03b234f5c-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.371 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.372 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.373 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.374 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.374 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.375 DEBUG:boto:Method: GET 2022-09-03T05:59:04.375 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.376 DEBUG:boto:Data: 2022-09-03T05:59:04.376 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.377 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.378 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.378 DEBUG:boto:Params: {} 2022-09-03T05:59:04.379 DEBUG:boto:Token: None 2022-09-03T05:59:04.380 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.385 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b3d008646e70a02e-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.386 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.387 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.388 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.389 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.389 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T05:59:04.390 DEBUG:boto:Data: 2022-09-03T05:59:04.391 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.393 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.394 DEBUG:boto:Params: {} 2022-09-03T05:59:04.395 DEBUG:boto:Token: None 2022-09-03T05:59:04.396 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2 2022-09-03T05:59:04.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Z5qn44xkUKDeFcURRLNoCCVat0= 2022-09-03T05:59:04.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Z5qn44xkUKDeFcURRLNoCCVat0=', 'Content-Length': '0'} 2022-09-03T05:59:04.402 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007c66b2cc032fdad0-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.403 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.403 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.404 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.405 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T05:59:04.405 DEBUG:boto:Data: 2022-09-03T05:59:04.406 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.406 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.407 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.407 DEBUG:boto:Params: {} 2022-09-03T05:59:04.408 DEBUG:boto:Token: None 2022-09-03T05:59:04.408 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2 2022-09-03T05:59:04.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Z5qn44xkUKDeFcURRLNoCCVat0= 2022-09-03T05:59:04.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Z5qn44xkUKDeFcURRLNoCCVat0=', 'Content-Length': '0'} 2022-09-03T05:59:04.413 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005260f16beb916850-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.414 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:04.414 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.415 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.415 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.416 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.417 DEBUG:boto:Method: GET 2022-09-03T05:59:04.417 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.418 DEBUG:boto:Data: 2022-09-03T05:59:04.418 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.419 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.420 DEBUG:boto:Params: {} 2022-09-03T05:59:04.420 DEBUG:boto:Token: None 2022-09-03T05:59:04.421 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo= 2022-09-03T05:59:04.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo=', 'Content-Length': '0'} 2022-09-03T05:59:04.425 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d39ff91588b76a46-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.426 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.427 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:04.427 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.428 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.428 DEBUG:boto:Method: GET 2022-09-03T05:59:04.429 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.430 DEBUG:boto:Data: 2022-09-03T05:59:04.430 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.431 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.431 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.432 DEBUG:boto:Params: {} 2022-09-03T05:59:04.433 DEBUG:boto:Token: None 2022-09-03T05:59:04.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T05:59:04.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo= 2022-09-03T05:59:04.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8WZqR59ZSbOpxbJBWsZQGHvaoo=', 'Content-Length': '0'} 2022-09-03T05:59:04.437 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009a79e6a98847d37d-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.438 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.439 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.439 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.440 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.440 DEBUG:boto:Method: GET 2022-09-03T05:59:04.441 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.441 DEBUG:boto:Data: 2022-09-03T05:59:04.442 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.443 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.444 DEBUG:boto:Params: {} 2022-09-03T05:59:04.444 DEBUG:boto:Token: None 2022-09-03T05:59:04.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.450 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a435ac76eeda6f0-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.451 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.453 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.453 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.454 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.455 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.456 DEBUG:boto:Method: GET 2022-09-03T05:59:04.457 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T05:59:04.458 DEBUG:boto:Data: 2022-09-03T05:59:04.459 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.460 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.461 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.462 DEBUG:boto:Params: {} 2022-09-03T05:59:04.463 DEBUG:boto:Token: None 2022-09-03T05:59:04.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.470 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005431b59ac64e2cc3-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.472 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.473 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:04.474 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.475 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.476 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.477 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.478 DEBUG:boto:Method: GET 2022-09-03T05:59:04.479 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.480 DEBUG:boto:Data: 2022-09-03T05:59:04.482 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.484 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.485 DEBUG:boto:Params: {} 2022-09-03T05:59:04.486 DEBUG:boto:Token: None 2022-09-03T05:59:04.486 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.487 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk= 2022-09-03T05:59:04.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk=', 'Content-Length': '0'} 2022-09-03T05:59:04.490 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002027c8d92747be01-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.491 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.492 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.493 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.494 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.495 DEBUG:boto:Method: GET 2022-09-03T05:59:04.496 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.497 DEBUG:boto:Data: 2022-09-03T05:59:04.498 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.499 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.500 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.501 DEBUG:boto:Params: {} 2022-09-03T05:59:04.502 DEBUG:boto:Token: None 2022-09-03T05:59:04.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk= 2022-09-03T05:59:04.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk=', 'Content-Length': '0'} 2022-09-03T05:59:04.508 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000982ef7b58657e29b-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.509 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.511 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.511 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.512 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.513 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.514 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.515 DEBUG:boto:Data: 2022-09-03T05:59:04.516 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.517 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.518 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.519 DEBUG:boto:Params: {} 2022-09-03T05:59:04.520 DEBUG:boto:Token: None 2022-09-03T05:59:04.521 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yT6AqwL0VaKxm1z+0GAmM4a4wjA= 2022-09-03T05:59:04.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yT6AqwL0VaKxm1z+0GAmM4a4wjA=', 'Content-Length': '0'} 2022-09-03T05:59:04.525 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008c4e125201f39d29-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.526 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.527 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.527 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.528 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.528 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.529 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.529 DEBUG:boto:Data: 2022-09-03T05:59:04.530 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.530 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.531 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.531 DEBUG:boto:Params: {} 2022-09-03T05:59:04.532 DEBUG:boto:Token: None 2022-09-03T05:59:04.532 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yT6AqwL0VaKxm1z+0GAmM4a4wjA= 2022-09-03T05:59:04.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yT6AqwL0VaKxm1z+0GAmM4a4wjA=', 'Content-Length': '0'} 2022-09-03T05:59:04.536 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000070967a618c210069-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.536 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:04.537 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.537 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.538 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.539 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.539 DEBUG:boto:Method: GET 2022-09-03T05:59:04.540 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.540 DEBUG:boto:Data: 2022-09-03T05:59:04.541 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.542 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.542 DEBUG:boto:Params: {} 2022-09-03T05:59:04.543 DEBUG:boto:Token: None 2022-09-03T05:59:04.543 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk= 2022-09-03T05:59:04.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk=', 'Content-Length': '0'} 2022-09-03T05:59:04.547 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000048c3b8212fcd6dc5-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.594 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.595 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.595 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.596 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.597 DEBUG:boto:Method: GET 2022-09-03T05:59:04.597 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.598 DEBUG:boto:Data: 2022-09-03T05:59:04.599 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.599 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.600 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.601 DEBUG:boto:Params: {} 2022-09-03T05:59:04.602 DEBUG:boto:Token: None 2022-09-03T05:59:04.602 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk= 2022-09-03T05:59:04.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk=', 'Content-Length': '0'} 2022-09-03T05:59:04.606 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000350bd02be0f16a6d-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.607 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.608 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.608 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.609 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.609 DEBUG:boto:Method: GET 2022-09-03T05:59:04.610 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.610 DEBUG:boto:Data: 2022-09-03T05:59:04.611 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.612 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.613 DEBUG:boto:Params: {} 2022-09-03T05:59:04.613 DEBUG:boto:Token: None 2022-09-03T05:59:04.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.618 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001f5fcd2f17b10010-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.619 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.620 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.620 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.621 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.621 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.622 DEBUG:boto:Method: GET 2022-09-03T05:59:04.623 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.623 DEBUG:boto:Data: 2022-09-03T05:59:04.624 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.624 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.625 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.626 DEBUG:boto:Params: {} 2022-09-03T05:59:04.626 DEBUG:boto:Token: None 2022-09-03T05:59:04.627 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.632 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000639c6429d277dee9-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.674 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.676 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.677 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.678 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.679 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T05:59:04.680 DEBUG:boto:Data: 2022-09-03T05:59:04.680 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.681 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.682 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.683 DEBUG:boto:Params: {} 2022-09-03T05:59:04.684 DEBUG:boto:Token: None 2022-09-03T05:59:04.685 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3 2022-09-03T05:59:04.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZPCymxCgzJynN0uedMLyAWe8mys= 2022-09-03T05:59:04.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZPCymxCgzJynN0uedMLyAWe8mys=', 'Content-Length': '0'} 2022-09-03T05:59:04.690 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000057af8622aa620fa8-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.691 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.692 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.693 DEBUG:boto:Method: HEAD 2022-09-03T05:59:04.693 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T05:59:04.694 DEBUG:boto:Data: 2022-09-03T05:59:04.694 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.695 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.696 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.697 DEBUG:boto:Params: {} 2022-09-03T05:59:04.697 DEBUG:boto:Token: None 2022-09-03T05:59:04.698 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3 2022-09-03T05:59:04.699 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZPCymxCgzJynN0uedMLyAWe8mys= 2022-09-03T05:59:04.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZPCymxCgzJynN0uedMLyAWe8mys=', 'Content-Length': '0'} 2022-09-03T05:59:04.702 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001ead341c7fbc06da-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.703 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:04.703 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.704 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.704 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.705 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.706 DEBUG:boto:Method: GET 2022-09-03T05:59:04.706 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.707 DEBUG:boto:Data: 2022-09-03T05:59:04.707 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.708 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.709 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.709 DEBUG:boto:Params: {} 2022-09-03T05:59:04.710 DEBUG:boto:Token: None 2022-09-03T05:59:04.711 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk= 2022-09-03T05:59:04.712 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk=', 'Content-Length': '0'} 2022-09-03T05:59:04.714 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000472ad50a7d674421-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.715 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.716 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:04.716 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.717 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.717 DEBUG:boto:Method: GET 2022-09-03T05:59:04.718 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.719 DEBUG:boto:Data: 2022-09-03T05:59:04.720 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.720 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.721 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.721 DEBUG:boto:Params: {} 2022-09-03T05:59:04.722 DEBUG:boto:Token: None 2022-09-03T05:59:04.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T05:59:04.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk= 2022-09-03T05:59:04.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kfR1ot2X8/3TYllXhi7M/gH8Htk=', 'Content-Length': '0'} 2022-09-03T05:59:04.726 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000053163a846460d89-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.727 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.728 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.729 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.729 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.730 DEBUG:boto:Method: GET 2022-09-03T05:59:04.730 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.731 DEBUG:boto:Data: 2022-09-03T05:59:04.732 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:04.733 DEBUG:boto:Port: 8001 2022-09-03T05:59:04.734 DEBUG:boto:Params: {} 2022-09-03T05:59:04.735 DEBUG:boto:Token: None 2022-09-03T05:59:04.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.739 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000634038248d4c9619-006312ed28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.740 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.741 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:04.741 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:04.742 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.743 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.743 DEBUG:boto:Method: GET 2022-09-03T05:59:04.744 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T05:59:04.744 DEBUG:boto:Data: 2022-09-03T05:59:04.745 DEBUG:boto:Headers: {} 2022-09-03T05:59:04.745 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:04.746 DEBUG:boto:Port: 8000 2022-09-03T05:59:04.747 DEBUG:boto:Params: {} 2022-09-03T05:59:04.747 DEBUG:boto:Token: None 2022-09-03T05:59:04.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:04 GMT /avysmw-16/ 2022-09-03T05:59:04.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE= 2022-09-03T05:59:04.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9eccmZox79U1vZph1FvSPnZPkEE=', 'Content-Length': '0'} 2022-09-03T05:59:04.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bde340e82dea8cac-006312ed28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:04.753 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:04.754 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-16 zones={test-zone3, test-zone2} 2022-09-03T05:59:04.755 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-17 2022-09-03T05:59:04.756 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:04.756 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:04.757 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:04.821 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:04.821 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:04.862 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:04.860+0000 7fd60f713600 1 bucket sync caught up with source: 2022-09-03T05:59:04.863 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.22.5, , 00000000003.15.9, , 00000000007.19.9, , , , , ] 2022-09-03T05:59:04.863 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.22.5, , 00000000003.15.9, , 00000000007.19.9, , , , , ] 2022-09-03T05:59:04.863 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:04.860+0000 7fd60f713600 0 bucket checkpoint complete 2022-09-03T05:59:04.886 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-17 2022-09-03T05:59:04.887 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:04.888 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:04.888 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:04.955 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:04.955 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:06.001 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:05.998+0000 7f09a90c9600 1 bucket sync caught up with source: 2022-09-03T05:59:06.001 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000010.22.5, , 00000000003.15.9, , 00000000007.19.9, , , , , ] 2022-09-03T05:59:06.001 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.22.5, , 00000000003.15.9, , 00000000007.19.9, , , , , ] 2022-09-03T05:59:06.002 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:05.998+0000 7f09a90c9600 0 bucket checkpoint complete 2022-09-03T05:59:06.017 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-17 2022-09-03T05:59:06.018 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:06.018 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:06.019 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:06.083 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:06.084 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:06.131 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:06.129+0000 7f847f4bc600 1 bucket sync caught up with source: 2022-09-03T05:59:06.132 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.22.5, , 00000000003.15.9, , 00000000007.19.9, , , , , ] 2022-09-03T05:59:06.132 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.22.5, , 00000000003.15.9, , 00000000007.19.9, , , , , ] 2022-09-03T05:59:06.133 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:06.129+0000 7f847f4bc600 0 bucket checkpoint complete 2022-09-03T05:59:06.154 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-17 2022-09-03T05:59:06.155 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:06.156 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:06.157 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:06.224 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:06.224 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:06.277 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:06.275+0000 7f573fbb7600 1 bucket sync caught up with source: 2022-09-03T05:59:06.278 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.16.9, , 00000000010.46.5, , 00000000008.29.9, , , , , ] 2022-09-03T05:59:06.278 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.16.9, , 00000000010.46.5, , 00000000008.29.9, , , , , ] 2022-09-03T05:59:06.278 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:06.275+0000 7f573fbb7600 0 bucket checkpoint complete 2022-09-03T05:59:06.298 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-17 2022-09-03T05:59:06.299 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:06.300 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:06.300 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:06.358 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:06.359 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:06.403 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:06.400+0000 7f7a91329600 1 bucket sync caught up with source: 2022-09-03T05:59:06.404 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.16.9, , 00000000010.46.5, , 00000000008.29.9, , , , , ] 2022-09-03T05:59:06.404 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.16.9, , 00000000010.46.5, , 00000000008.29.9, , , , , ] 2022-09-03T05:59:06.404 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:06.400+0000 7f7a91329600 0 bucket checkpoint complete 2022-09-03T05:59:06.416 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-17 2022-09-03T05:59:06.418 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-17 2022-09-03T05:59:06.418 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:06.419 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:06.420 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:06.478 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:06.479 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:06.513 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:06.510+0000 7f2741dfc600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:06.514 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000008.28.11, , , , , ] 2022-09-03T05:59:06.514 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:11.515 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:11.513+0000 7f2741dfc600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:11.515 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000008.28.11, , , , , ] 2022-09-03T05:59:11.516 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:16.516 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:16.514+0000 7f2741dfc600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:16.517 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000008.28.11, , , , , ] 2022-09-03T05:59:16.517 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:21.518 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:21.516+0000 7f2741dfc600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:21.518 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000008.28.11, , , , , ] 2022-09-03T05:59:21.519 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:26.519 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:26.517+0000 7f2741dfc600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:26.520 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000008.28.11, , , , , ] 2022-09-03T05:59:26.520 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:31.521 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:31.519+0000 7f2741dfc600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:31.522 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000008.28.11, , , , , ] 2022-09-03T05:59:31.522 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:36.522 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:36.520+0000 7f2741dfc600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:36.522 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000008.28.11, , , , , ] 2022-09-03T05:59:36.523 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:41.524 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:41.522+0000 7f2741dfc600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:41.524 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000008.28.11, , , , , ] 2022-09-03T05:59:41.525 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:46.525 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:46.523+0000 7f2741dfc600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:46.526 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000008.28.11, , , , , ] 2022-09-03T05:59:46.526 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:51.527 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:51.524+0000 7f2741dfc600 1 bucket sync caught up with source: 2022-09-03T05:59:51.528 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:51.528 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:51.529 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:51.524+0000 7f2741dfc600 0 bucket checkpoint complete 2022-09-03T05:59:51.549 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-17 2022-09-03T05:59:51.550 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:51.551 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:51.552 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:51.616 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:51.616 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:51.655 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:51.653+0000 7f1dd2976600 1 bucket sync caught up with source: 2022-09-03T05:59:51.655 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:51.656 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000003.15.9, , 00000000003.35.9, , 00000000014.40.5, , , , , ] 2022-09-03T05:59:51.656 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:51.653+0000 7f1dd2976600 0 bucket checkpoint complete 2022-09-03T05:59:51.668 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-17 2022-09-03T05:59:51.670 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-17 zones={test-zone2, test-zone1} 2022-09-03T05:59:51.671 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.672 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.673 DEBUG:boto:Method: HEAD 2022-09-03T05:59:51.674 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:59:51.675 DEBUG:boto:Data: 2022-09-03T05:59:51.676 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.676 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.677 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.677 DEBUG:boto:Params: {} 2022-09-03T05:59:51.678 DEBUG:boto:Token: None 2022-09-03T05:59:51.679 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/ 2022-09-03T05:59:51.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUyjHNhbN0WdyAxLAFIRRtifjdU= 2022-09-03T05:59:51.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUyjHNhbN0WdyAxLAFIRRtifjdU=', 'Content-Length': '0'} 2022-09-03T05:59:51.684 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000f85eae303e86030-006312ed57-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.684 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.685 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.685 DEBUG:boto:Method: HEAD 2022-09-03T05:59:51.686 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:59:51.687 DEBUG:boto:Data: 2022-09-03T05:59:51.687 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.688 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.689 DEBUG:boto:Params: {} 2022-09-03T05:59:51.689 DEBUG:boto:Token: None 2022-09-03T05:59:51.690 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/ 2022-09-03T05:59:51.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUyjHNhbN0WdyAxLAFIRRtifjdU= 2022-09-03T05:59:51.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUyjHNhbN0WdyAxLAFIRRtifjdU=', 'Content-Length': '0'} 2022-09-03T05:59:51.694 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ead9f21269e333e1-006312ed57-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.694 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:59:51.695 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.696 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.696 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.697 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.697 DEBUG:boto:Method: GET 2022-09-03T05:59:51.698 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.699 DEBUG:boto:Data: 2022-09-03T05:59:51.699 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.700 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.700 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.701 DEBUG:boto:Params: {} 2022-09-03T05:59:51.702 DEBUG:boto:Token: None 2022-09-03T05:59:51.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/?versions 2022-09-03T05:59:51.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QdLmlXfLPosyVM08i0Ux1bW+64s= 2022-09-03T05:59:51.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QdLmlXfLPosyVM08i0Ux1bW+64s=', 'Content-Length': '0'} 2022-09-03T05:59:51.706 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009d8455433fe49d20-006312ed57-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.710 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:51.711 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:51.712 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:51.712 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:51.713 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:51.714 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:59:51.714 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.715 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.716 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.717 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.718 DEBUG:boto:Method: GET 2022-09-03T05:59:51.718 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.719 DEBUG:boto:Data: 2022-09-03T05:59:51.720 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.721 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.722 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.723 DEBUG:boto:Params: {} 2022-09-03T05:59:51.724 DEBUG:boto:Token: None 2022-09-03T05:59:51.725 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/?versions 2022-09-03T05:59:51.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QdLmlXfLPosyVM08i0Ux1bW+64s= 2022-09-03T05:59:51.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QdLmlXfLPosyVM08i0Ux1bW+64s=', 'Content-Length': '0'} 2022-09-03T05:59:51.730 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003a9ccae22d254c64-006312ed57-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.730 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:51.731 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:51.732 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:51.732 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:51.733 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:51.733 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.734 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.735 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.735 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.736 DEBUG:boto:Method: GET 2022-09-03T05:59:51.737 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.737 DEBUG:boto:Data: 2022-09-03T05:59:51.738 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.738 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.739 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.739 DEBUG:boto:Params: {} 2022-09-03T05:59:51.740 DEBUG:boto:Token: None 2022-09-03T05:59:51.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/?versions 2022-09-03T05:59:51.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QdLmlXfLPosyVM08i0Ux1bW+64s= 2022-09-03T05:59:51.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QdLmlXfLPosyVM08i0Ux1bW+64s=', 'Content-Length': '0'} 2022-09-03T05:59:51.744 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fecb5fdc38db62ba-006312ed57-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.745 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:51.746 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.747 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.747 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.748 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.748 DEBUG:boto:Method: GET 2022-09-03T05:59:51.749 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:51.750 DEBUG:boto:Data: 2022-09-03T05:59:51.750 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.752 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.753 DEBUG:boto:Params: {} 2022-09-03T05:59:51.754 DEBUG:boto:Token: None 2022-09-03T05:59:51.755 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/?versions 2022-09-03T05:59:51.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QdLmlXfLPosyVM08i0Ux1bW+64s= 2022-09-03T05:59:51.758 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QdLmlXfLPosyVM08i0Ux1bW+64s=', 'Content-Length': '0'} 2022-09-03T05:59:51.760 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000008c0088acbe1a228-006312ed57-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.761 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:51.762 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:51.763 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.763 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.764 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.765 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.766 DEBUG:boto:Method: GET 2022-09-03T05:59:51.767 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.768 DEBUG:boto:Data: 2022-09-03T05:59:51.769 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.770 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.771 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.772 DEBUG:boto:Params: {} 2022-09-03T05:59:51.773 DEBUG:boto:Token: None 2022-09-03T05:59:51.775 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU= 2022-09-03T05:59:51.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU=', 'Content-Length': '0'} 2022-09-03T05:59:51.778 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ded13dc37b1a5a0d-006312ed57-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.779 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.780 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.780 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.781 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.782 DEBUG:boto:Method: GET 2022-09-03T05:59:51.782 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.783 DEBUG:boto:Data: 2022-09-03T05:59:51.783 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.784 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.784 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.785 DEBUG:boto:Params: {} 2022-09-03T05:59:51.785 DEBUG:boto:Token: None 2022-09-03T05:59:51.786 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU= 2022-09-03T05:59:51.788 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU=', 'Content-Length': '0'} 2022-09-03T05:59:51.790 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000008a40b2a7122444c-006312ed57-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.791 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.791 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.792 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.792 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.793 DEBUG:boto:Method: HEAD 2022-09-03T05:59:51.793 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.794 DEBUG:boto:Data: 2022-09-03T05:59:51.794 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.795 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.796 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.796 DEBUG:boto:Params: {} 2022-09-03T05:59:51.797 DEBUG:boto:Token: None 2022-09-03T05:59:51.798 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jr3lp1CGy+FDV0bghlWnU1bKNpw= 2022-09-03T05:59:51.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jr3lp1CGy+FDV0bghlWnU1bKNpw=', 'Content-Length': '0'} 2022-09-03T05:59:51.801 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a8602884e08a7f58-006312ed57-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.802 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.802 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.803 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.803 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.804 DEBUG:boto:Method: HEAD 2022-09-03T05:59:51.805 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.805 DEBUG:boto:Data: 2022-09-03T05:59:51.806 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.806 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.807 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.807 DEBUG:boto:Params: {} 2022-09-03T05:59:51.808 DEBUG:boto:Token: None 2022-09-03T05:59:51.809 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jr3lp1CGy+FDV0bghlWnU1bKNpw= 2022-09-03T05:59:51.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jr3lp1CGy+FDV0bghlWnU1bKNpw=', 'Content-Length': '0'} 2022-09-03T05:59:51.812 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a818deaed126638d-006312ed57-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.813 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:51.814 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.814 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.815 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.816 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.816 DEBUG:boto:Method: GET 2022-09-03T05:59:51.817 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.818 DEBUG:boto:Data: 2022-09-03T05:59:51.818 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.819 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.819 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.820 DEBUG:boto:Params: {} 2022-09-03T05:59:51.820 DEBUG:boto:Token: None 2022-09-03T05:59:51.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU= 2022-09-03T05:59:51.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU=', 'Content-Length': '0'} 2022-09-03T05:59:51.825 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000017ed679466a1489e-006312ed57-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.825 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.826 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.827 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.827 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.828 DEBUG:boto:Method: GET 2022-09-03T05:59:51.828 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.829 DEBUG:boto:Data: 2022-09-03T05:59:51.829 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.830 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.831 DEBUG:boto:Params: {} 2022-09-03T05:59:51.832 DEBUG:boto:Token: None 2022-09-03T05:59:51.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU= 2022-09-03T05:59:51.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU=', 'Content-Length': '0'} 2022-09-03T05:59:51.836 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d6a9ddb67991e6ee-006312ed57-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.838 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.839 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.839 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:51.840 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:51.840 DEBUG:boto:Method: GET 2022-09-03T05:59:51.841 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:51.841 DEBUG:boto:Data: 2022-09-03T05:59:51.842 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.843 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.844 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.844 DEBUG:boto:Params: {} 2022-09-03T05:59:51.845 DEBUG:boto:Token: None 2022-09-03T05:59:51.846 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/ 2022-09-03T05:59:51.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BWewjtVJWUEimsqcWjgq2ZpsXuI= 2022-09-03T05:59:51.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BWewjtVJWUEimsqcWjgq2ZpsXuI=', 'Content-Length': '0'} 2022-09-03T05:59:51.849 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000083a41a1e06cf393e-006312ed57-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.894 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:51.895 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.896 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.897 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:51.897 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:51.898 DEBUG:boto:Method: GET 2022-09-03T05:59:51.898 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:51.899 DEBUG:boto:Data: 2022-09-03T05:59:51.899 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.900 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.901 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.901 DEBUG:boto:Params: {} 2022-09-03T05:59:51.902 DEBUG:boto:Token: None 2022-09-03T05:59:51.903 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/ 2022-09-03T05:59:51.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BWewjtVJWUEimsqcWjgq2ZpsXuI= 2022-09-03T05:59:51.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BWewjtVJWUEimsqcWjgq2ZpsXuI=', 'Content-Length': '0'} 2022-09-03T05:59:51.907 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000079fad4e69d78da9-006312ed57-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.907 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:51.908 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.909 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.909 DEBUG:boto:Method: HEAD 2022-09-03T05:59:51.910 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T05:59:51.910 DEBUG:boto:Data: 2022-09-03T05:59:51.911 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.912 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.912 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.913 DEBUG:boto:Params: {} 2022-09-03T05:59:51.913 DEBUG:boto:Token: None 2022-09-03T05:59:51.914 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy 2022-09-03T05:59:51.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+d9vS94Szsudyq29shSQx6DMmjs= 2022-09-03T05:59:51.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+d9vS94Szsudyq29shSQx6DMmjs=', 'Content-Length': '0'} 2022-09-03T05:59:51.918 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f52844e278e79e93-006312ed57-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.919 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.919 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.920 DEBUG:boto:Method: HEAD 2022-09-03T05:59:51.920 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T05:59:51.921 DEBUG:boto:Data: 2022-09-03T05:59:51.921 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.923 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.923 DEBUG:boto:Params: {} 2022-09-03T05:59:51.924 DEBUG:boto:Token: None 2022-09-03T05:59:51.925 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy 2022-09-03T05:59:51.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+d9vS94Szsudyq29shSQx6DMmjs= 2022-09-03T05:59:51.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+d9vS94Szsudyq29shSQx6DMmjs=', 'Content-Length': '0'} 2022-09-03T05:59:51.931 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d017713125b63119-006312ed57-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.932 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:51.933 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.934 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.935 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.935 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.936 DEBUG:boto:Method: GET 2022-09-03T05:59:51.936 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.937 DEBUG:boto:Data: 2022-09-03T05:59:51.938 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.938 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.939 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.939 DEBUG:boto:Params: {} 2022-09-03T05:59:51.940 DEBUG:boto:Token: None 2022-09-03T05:59:51.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU= 2022-09-03T05:59:51.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU=', 'Content-Length': '0'} 2022-09-03T05:59:51.944 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000005dc447868b5bbfe-006312ed57-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.945 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:51.946 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:51.947 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.947 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.948 DEBUG:boto:Method: GET 2022-09-03T05:59:51.949 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.950 DEBUG:boto:Data: 2022-09-03T05:59:51.951 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.952 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.953 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.954 DEBUG:boto:Params: {} 2022-09-03T05:59:51.955 DEBUG:boto:Token: None 2022-09-03T05:59:51.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:51.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU= 2022-09-03T05:59:51.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYu9pQBjcEG47F9bBnbrtQ4X6LU=', 'Content-Length': '0'} 2022-09-03T05:59:51.959 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ed5013e893598043-006312ed57-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.960 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.961 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.962 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:51.962 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:51.963 DEBUG:boto:Method: GET 2022-09-03T05:59:51.963 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:51.964 DEBUG:boto:Data: 2022-09-03T05:59:51.964 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:51.965 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.966 DEBUG:boto:Params: {} 2022-09-03T05:59:51.967 DEBUG:boto:Token: None 2022-09-03T05:59:51.967 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/ 2022-09-03T05:59:51.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BWewjtVJWUEimsqcWjgq2ZpsXuI= 2022-09-03T05:59:51.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BWewjtVJWUEimsqcWjgq2ZpsXuI=', 'Content-Length': '0'} 2022-09-03T05:59:51.971 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b9ef3531277d7af-006312ed57-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:51.972 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:51.974 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:51.974 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:51.975 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:51.975 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:51.976 DEBUG:boto:Method: GET 2022-09-03T05:59:51.977 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:51.978 DEBUG:boto:Data: 2022-09-03T05:59:51.979 DEBUG:boto:Headers: {} 2022-09-03T05:59:51.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:51.980 DEBUG:boto:Port: 8000 2022-09-03T05:59:51.980 DEBUG:boto:Params: {} 2022-09-03T05:59:51.981 DEBUG:boto:Token: None 2022-09-03T05:59:51.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:51 GMT /avysmw-17/ 2022-09-03T05:59:51.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BWewjtVJWUEimsqcWjgq2ZpsXuI= 2022-09-03T05:59:51.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BWewjtVJWUEimsqcWjgq2ZpsXuI=', 'Content-Length': '0'} 2022-09-03T05:59:51.986 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1470d6e7a6ce501-006312ed57-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.034 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.035 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:52.036 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.037 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.038 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.038 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.039 DEBUG:boto:Method: GET 2022-09-03T05:59:52.040 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.041 DEBUG:boto:Data: 2022-09-03T05:59:52.042 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.042 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.043 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.044 DEBUG:boto:Params: {} 2022-09-03T05:59:52.045 DEBUG:boto:Token: None 2022-09-03T05:59:52.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg= 2022-09-03T05:59:52.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg=', 'Content-Length': '0'} 2022-09-03T05:59:52.051 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001e0d39eaf4765f67-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.052 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.052 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.053 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.053 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.054 DEBUG:boto:Method: GET 2022-09-03T05:59:52.054 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.055 DEBUG:boto:Data: 2022-09-03T05:59:52.055 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.057 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.057 DEBUG:boto:Params: {} 2022-09-03T05:59:52.058 DEBUG:boto:Token: None 2022-09-03T05:59:52.058 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg= 2022-09-03T05:59:52.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg=', 'Content-Length': '0'} 2022-09-03T05:59:52.063 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008bf7097d5f4651e3-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.064 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.065 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.065 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.066 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.066 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.067 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.067 DEBUG:boto:Data: 2022-09-03T05:59:52.068 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.068 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.069 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.069 DEBUG:boto:Params: {} 2022-09-03T05:59:52.070 DEBUG:boto:Token: None 2022-09-03T05:59:52.071 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6r7zHG4hAwNCWrn8yKIkIcax3d0= 2022-09-03T05:59:52.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6r7zHG4hAwNCWrn8yKIkIcax3d0=', 'Content-Length': '0'} 2022-09-03T05:59:52.074 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004779b0b8ee9d1063-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.075 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.076 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.076 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.077 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.077 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.078 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.078 DEBUG:boto:Data: 2022-09-03T05:59:52.079 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.080 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.081 DEBUG:boto:Params: {} 2022-09-03T05:59:52.081 DEBUG:boto:Token: None 2022-09-03T05:59:52.082 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6r7zHG4hAwNCWrn8yKIkIcax3d0= 2022-09-03T05:59:52.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6r7zHG4hAwNCWrn8yKIkIcax3d0=', 'Content-Length': '0'} 2022-09-03T05:59:52.088 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000428f9f753c1e41b0-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.089 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:52.090 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.090 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.091 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.091 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.092 DEBUG:boto:Method: GET 2022-09-03T05:59:52.092 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.093 DEBUG:boto:Data: 2022-09-03T05:59:52.094 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.094 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.095 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.096 DEBUG:boto:Params: {} 2022-09-03T05:59:52.097 DEBUG:boto:Token: None 2022-09-03T05:59:52.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg= 2022-09-03T05:59:52.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg=', 'Content-Length': '0'} 2022-09-03T05:59:52.103 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e73bc5ceed83da28-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.146 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.147 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.148 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.149 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.150 DEBUG:boto:Method: GET 2022-09-03T05:59:52.151 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.152 DEBUG:boto:Data: 2022-09-03T05:59:52.153 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.154 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.155 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.156 DEBUG:boto:Params: {} 2022-09-03T05:59:52.157 DEBUG:boto:Token: None 2022-09-03T05:59:52.158 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg= 2022-09-03T05:59:52.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg=', 'Content-Length': '0'} 2022-09-03T05:59:52.164 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008fed7f9e22f71cd8-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.165 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.166 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.167 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.168 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.168 DEBUG:boto:Method: GET 2022-09-03T05:59:52.169 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.169 DEBUG:boto:Data: 2022-09-03T05:59:52.170 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.170 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.171 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.172 DEBUG:boto:Params: {} 2022-09-03T05:59:52.172 DEBUG:boto:Token: None 2022-09-03T05:59:52.173 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.177 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1476646e7fcfa9e-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.178 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.178 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.179 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.180 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.180 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.181 DEBUG:boto:Method: GET 2022-09-03T05:59:52.181 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.182 DEBUG:boto:Data: 2022-09-03T05:59:52.183 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.185 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.186 DEBUG:boto:Params: {} 2022-09-03T05:59:52.187 DEBUG:boto:Token: None 2022-09-03T05:59:52.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.192 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000007c189854751190c-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.193 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.194 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.194 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.195 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.195 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T05:59:52.196 DEBUG:boto:Data: 2022-09-03T05:59:52.196 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.197 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.198 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.198 DEBUG:boto:Params: {} 2022-09-03T05:59:52.199 DEBUG:boto:Token: None 2022-09-03T05:59:52.199 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1 2022-09-03T05:59:52.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IN/hEdIRcMibStZ/B35DYdkdJNI= 2022-09-03T05:59:52.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IN/hEdIRcMibStZ/B35DYdkdJNI=', 'Content-Length': '0'} 2022-09-03T05:59:52.202 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b86efa431a276abc-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.203 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.204 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.204 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.205 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T05:59:52.205 DEBUG:boto:Data: 2022-09-03T05:59:52.206 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.206 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.207 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.207 DEBUG:boto:Params: {} 2022-09-03T05:59:52.208 DEBUG:boto:Token: None 2022-09-03T05:59:52.209 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1 2022-09-03T05:59:52.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IN/hEdIRcMibStZ/B35DYdkdJNI= 2022-09-03T05:59:52.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IN/hEdIRcMibStZ/B35DYdkdJNI=', 'Content-Length': '0'} 2022-09-03T05:59:52.214 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003a37387b15de0390-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:52.216 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.217 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.218 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.219 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.220 DEBUG:boto:Method: GET 2022-09-03T05:59:52.221 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.221 DEBUG:boto:Data: 2022-09-03T05:59:52.222 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.222 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.223 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.223 DEBUG:boto:Params: {} 2022-09-03T05:59:52.224 DEBUG:boto:Token: None 2022-09-03T05:59:52.224 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg= 2022-09-03T05:59:52.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg=', 'Content-Length': '0'} 2022-09-03T05:59:52.228 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008d3257b41370d267-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.229 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.229 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:52.230 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.230 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.231 DEBUG:boto:Method: GET 2022-09-03T05:59:52.232 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.232 DEBUG:boto:Data: 2022-09-03T05:59:52.233 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.233 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.234 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.234 DEBUG:boto:Params: {} 2022-09-03T05:59:52.235 DEBUG:boto:Token: None 2022-09-03T05:59:52.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:52.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg= 2022-09-03T05:59:52.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUvf5aeGSO0d1J6013NeVJWp7Fg=', 'Content-Length': '0'} 2022-09-03T05:59:52.239 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000093bbb6d3c2d3b85c-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.282 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.283 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.284 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.284 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.285 DEBUG:boto:Method: GET 2022-09-03T05:59:52.285 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.286 DEBUG:boto:Data: 2022-09-03T05:59:52.287 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.287 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.288 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.289 DEBUG:boto:Params: {} 2022-09-03T05:59:52.290 DEBUG:boto:Token: None 2022-09-03T05:59:52.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.296 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000681dccd95871f4c3-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.297 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.299 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.299 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.300 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.301 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.301 DEBUG:boto:Method: GET 2022-09-03T05:59:52.302 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:52.302 DEBUG:boto:Data: 2022-09-03T05:59:52.303 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.304 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.304 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.305 DEBUG:boto:Params: {} 2022-09-03T05:59:52.306 DEBUG:boto:Token: None 2022-09-03T05:59:52.307 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.313 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000081e4d5f9a5a6d766-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.314 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.315 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:52.315 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.316 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.316 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.317 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.317 DEBUG:boto:Method: GET 2022-09-03T05:59:52.318 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.319 DEBUG:boto:Data: 2022-09-03T05:59:52.319 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.320 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.321 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.321 DEBUG:boto:Params: {} 2022-09-03T05:59:52.322 DEBUG:boto:Token: None 2022-09-03T05:59:52.323 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0= 2022-09-03T05:59:52.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0=', 'Content-Length': '0'} 2022-09-03T05:59:52.326 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000012b4dc887521edcb-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.327 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.328 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.328 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.329 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.329 DEBUG:boto:Method: GET 2022-09-03T05:59:52.330 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.331 DEBUG:boto:Data: 2022-09-03T05:59:52.331 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.332 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.333 DEBUG:boto:Params: {} 2022-09-03T05:59:52.334 DEBUG:boto:Token: None 2022-09-03T05:59:52.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0= 2022-09-03T05:59:52.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0=', 'Content-Length': '0'} 2022-09-03T05:59:52.338 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000029c479991d051f5d-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.340 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.341 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.342 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.343 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.344 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.345 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.346 DEBUG:boto:Data: 2022-09-03T05:59:52.347 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.348 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.348 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.349 DEBUG:boto:Params: {} 2022-09-03T05:59:52.350 DEBUG:boto:Token: None 2022-09-03T05:59:52.351 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nh/ZQZ4NxkHHyT/8/rzu5LTgLSw= 2022-09-03T05:59:52.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nh/ZQZ4NxkHHyT/8/rzu5LTgLSw=', 'Content-Length': '0'} 2022-09-03T05:59:52.355 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f7018706ae8d6c86-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.356 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.356 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.357 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.357 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.358 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.359 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.360 DEBUG:boto:Data: 2022-09-03T05:59:52.361 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.361 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.362 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.363 DEBUG:boto:Params: {} 2022-09-03T05:59:52.364 DEBUG:boto:Token: None 2022-09-03T05:59:52.365 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nh/ZQZ4NxkHHyT/8/rzu5LTgLSw= 2022-09-03T05:59:52.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nh/ZQZ4NxkHHyT/8/rzu5LTgLSw=', 'Content-Length': '0'} 2022-09-03T05:59:52.370 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fa59aeda426a2d97-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.371 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:52.372 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.373 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.374 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.375 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.376 DEBUG:boto:Method: GET 2022-09-03T05:59:52.377 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.377 DEBUG:boto:Data: 2022-09-03T05:59:52.378 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.379 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.379 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.380 DEBUG:boto:Params: {} 2022-09-03T05:59:52.380 DEBUG:boto:Token: None 2022-09-03T05:59:52.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0= 2022-09-03T05:59:52.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0=', 'Content-Length': '0'} 2022-09-03T05:59:52.384 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000045b1d94da79cc6fc-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.426 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.427 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.428 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.428 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.429 DEBUG:boto:Method: GET 2022-09-03T05:59:52.429 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.430 DEBUG:boto:Data: 2022-09-03T05:59:52.431 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.431 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.432 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.432 DEBUG:boto:Params: {} 2022-09-03T05:59:52.433 DEBUG:boto:Token: None 2022-09-03T05:59:52.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0= 2022-09-03T05:59:52.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0=', 'Content-Length': '0'} 2022-09-03T05:59:52.437 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000034318a7c70307652-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.438 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.439 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.439 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.440 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.441 DEBUG:boto:Method: GET 2022-09-03T05:59:52.441 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.442 DEBUG:boto:Data: 2022-09-03T05:59:52.442 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.443 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.443 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.444 DEBUG:boto:Params: {} 2022-09-03T05:59:52.445 DEBUG:boto:Token: None 2022-09-03T05:59:52.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.449 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bc5c96e953ffea65-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.450 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.451 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.451 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.452 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.452 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.453 DEBUG:boto:Method: GET 2022-09-03T05:59:52.453 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.454 DEBUG:boto:Data: 2022-09-03T05:59:52.455 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.455 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.456 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.456 DEBUG:boto:Params: {} 2022-09-03T05:59:52.457 DEBUG:boto:Token: None 2022-09-03T05:59:52.458 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.463 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000178525ff6eef1421-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.464 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.465 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.465 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.466 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.466 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T05:59:52.467 DEBUG:boto:Data: 2022-09-03T05:59:52.467 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.468 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.468 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.469 DEBUG:boto:Params: {} 2022-09-03T05:59:52.469 DEBUG:boto:Token: None 2022-09-03T05:59:52.470 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2 2022-09-03T05:59:52.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dmt7MuJU0phdbrjRmOq/nUnGooA= 2022-09-03T05:59:52.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dmt7MuJU0phdbrjRmOq/nUnGooA=', 'Content-Length': '0'} 2022-09-03T05:59:52.474 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c4e62d9b33e2da42-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.475 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.475 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.476 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.476 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T05:59:52.477 DEBUG:boto:Data: 2022-09-03T05:59:52.477 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.478 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.479 DEBUG:boto:Params: {} 2022-09-03T05:59:52.480 DEBUG:boto:Token: None 2022-09-03T05:59:52.480 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2 2022-09-03T05:59:52.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dmt7MuJU0phdbrjRmOq/nUnGooA= 2022-09-03T05:59:52.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dmt7MuJU0phdbrjRmOq/nUnGooA=', 'Content-Length': '0'} 2022-09-03T05:59:52.484 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000079cfe13f965c600b-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.484 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:52.485 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.486 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.486 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.487 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.487 DEBUG:boto:Method: GET 2022-09-03T05:59:52.488 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.488 DEBUG:boto:Data: 2022-09-03T05:59:52.489 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.489 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.490 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.491 DEBUG:boto:Params: {} 2022-09-03T05:59:52.491 DEBUG:boto:Token: None 2022-09-03T05:59:52.492 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0= 2022-09-03T05:59:52.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0=', 'Content-Length': '0'} 2022-09-03T05:59:52.495 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d753eaee2f7d61cd-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.496 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.496 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:52.497 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.498 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.498 DEBUG:boto:Method: GET 2022-09-03T05:59:52.499 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.499 DEBUG:boto:Data: 2022-09-03T05:59:52.500 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.501 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.501 DEBUG:boto:Params: {} 2022-09-03T05:59:52.502 DEBUG:boto:Token: None 2022-09-03T05:59:52.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:52.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0= 2022-09-03T05:59:52.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l6rtdZ9r2xrlGmGpRPKeFtUi2I0=', 'Content-Length': '0'} 2022-09-03T05:59:52.506 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000df214ab008db5de2-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.550 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.551 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.551 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.552 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.552 DEBUG:boto:Method: GET 2022-09-03T05:59:52.553 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.554 DEBUG:boto:Data: 2022-09-03T05:59:52.554 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.555 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.555 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.556 DEBUG:boto:Params: {} 2022-09-03T05:59:52.556 DEBUG:boto:Token: None 2022-09-03T05:59:52.557 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.557 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.561 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088aa1e668ebd94da-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.561 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.562 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.563 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.564 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.564 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.565 DEBUG:boto:Method: GET 2022-09-03T05:59:52.565 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:52.566 DEBUG:boto:Data: 2022-09-03T05:59:52.567 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.568 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.568 DEBUG:boto:Params: {} 2022-09-03T05:59:52.569 DEBUG:boto:Token: None 2022-09-03T05:59:52.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.574 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a7b4c9fbfda1100e-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.574 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.575 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:52.576 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.576 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.577 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.578 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.578 DEBUG:boto:Method: GET 2022-09-03T05:59:52.579 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.580 DEBUG:boto:Data: 2022-09-03T05:59:52.580 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.581 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.582 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.582 DEBUG:boto:Params: {} 2022-09-03T05:59:52.583 DEBUG:boto:Token: None 2022-09-03T05:59:52.584 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM= 2022-09-03T05:59:52.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM=', 'Content-Length': '0'} 2022-09-03T05:59:52.587 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c732ae2c03ca3988-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.588 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.589 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.589 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.590 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.590 DEBUG:boto:Method: GET 2022-09-03T05:59:52.591 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.591 DEBUG:boto:Data: 2022-09-03T05:59:52.592 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.592 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.593 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.594 DEBUG:boto:Params: {} 2022-09-03T05:59:52.594 DEBUG:boto:Token: None 2022-09-03T05:59:52.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM= 2022-09-03T05:59:52.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM=', 'Content-Length': '0'} 2022-09-03T05:59:52.598 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e4020da575975e5a-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.599 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.600 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.600 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.601 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.601 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.602 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.602 DEBUG:boto:Data: 2022-09-03T05:59:52.603 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.603 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.604 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.605 DEBUG:boto:Params: {} 2022-09-03T05:59:52.605 DEBUG:boto:Token: None 2022-09-03T05:59:52.606 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZjG6OGIvwkzvLQIKfF/YCFq9HSE= 2022-09-03T05:59:52.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZjG6OGIvwkzvLQIKfF/YCFq9HSE=', 'Content-Length': '0'} 2022-09-03T05:59:52.610 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000480bbbae546f9aca-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.610 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.611 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.611 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.612 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.612 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.613 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.613 DEBUG:boto:Data: 2022-09-03T05:59:52.614 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.615 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.616 DEBUG:boto:Params: {} 2022-09-03T05:59:52.617 DEBUG:boto:Token: None 2022-09-03T05:59:52.618 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZjG6OGIvwkzvLQIKfF/YCFq9HSE= 2022-09-03T05:59:52.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZjG6OGIvwkzvLQIKfF/YCFq9HSE=', 'Content-Length': '0'} 2022-09-03T05:59:52.623 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000de4779a2d3eb069b-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.624 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:52.625 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.626 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.627 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.628 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.629 DEBUG:boto:Method: GET 2022-09-03T05:59:52.630 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.631 DEBUG:boto:Data: 2022-09-03T05:59:52.631 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.633 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.633 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.634 DEBUG:boto:Params: {} 2022-09-03T05:59:52.635 DEBUG:boto:Token: None 2022-09-03T05:59:52.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM= 2022-09-03T05:59:52.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM=', 'Content-Length': '0'} 2022-09-03T05:59:52.640 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005bf2af3943d27808-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.641 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.642 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.642 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.643 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.644 DEBUG:boto:Method: GET 2022-09-03T05:59:52.645 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.646 DEBUG:boto:Data: 2022-09-03T05:59:52.647 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.648 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.649 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.650 DEBUG:boto:Params: {} 2022-09-03T05:59:52.651 DEBUG:boto:Token: None 2022-09-03T05:59:52.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM= 2022-09-03T05:59:52.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM=', 'Content-Length': '0'} 2022-09-03T05:59:52.656 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009e4bce828c9928c8-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.698 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.699 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.699 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.700 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.700 DEBUG:boto:Method: GET 2022-09-03T05:59:52.701 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.702 DEBUG:boto:Data: 2022-09-03T05:59:52.702 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.703 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.703 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.704 DEBUG:boto:Params: {} 2022-09-03T05:59:52.704 DEBUG:boto:Token: None 2022-09-03T05:59:52.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.710 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000601aac6dedf38af8-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.711 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.711 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.712 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.713 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.713 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.714 DEBUG:boto:Method: GET 2022-09-03T05:59:52.714 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.715 DEBUG:boto:Data: 2022-09-03T05:59:52.715 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.716 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.717 DEBUG:boto:Params: {} 2022-09-03T05:59:52.717 DEBUG:boto:Token: None 2022-09-03T05:59:52.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.722 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000179d97f2784f5d78-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.723 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.724 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.724 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.725 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.725 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T05:59:52.726 DEBUG:boto:Data: 2022-09-03T05:59:52.726 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.727 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.727 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.728 DEBUG:boto:Params: {} 2022-09-03T05:59:52.728 DEBUG:boto:Token: None 2022-09-03T05:59:52.729 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3 2022-09-03T05:59:52.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bXvOdJpLL3XtwEVSaM42/iruOco= 2022-09-03T05:59:52.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bXvOdJpLL3XtwEVSaM42/iruOco=', 'Content-Length': '0'} 2022-09-03T05:59:52.733 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005db5b762135fd0b1-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.734 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.735 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.735 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.736 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T05:59:52.736 DEBUG:boto:Data: 2022-09-03T05:59:52.737 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.739 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.740 DEBUG:boto:Params: {} 2022-09-03T05:59:52.741 DEBUG:boto:Token: None 2022-09-03T05:59:52.742 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3 2022-09-03T05:59:52.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bXvOdJpLL3XtwEVSaM42/iruOco= 2022-09-03T05:59:52.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bXvOdJpLL3XtwEVSaM42/iruOco=', 'Content-Length': '0'} 2022-09-03T05:59:52.747 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000094b120489ebad284-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.748 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:52.748 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.749 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.749 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.750 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.751 DEBUG:boto:Method: GET 2022-09-03T05:59:52.751 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.752 DEBUG:boto:Data: 2022-09-03T05:59:52.752 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.753 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.754 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.754 DEBUG:boto:Params: {} 2022-09-03T05:59:52.755 DEBUG:boto:Token: None 2022-09-03T05:59:52.756 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM= 2022-09-03T05:59:52.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM=', 'Content-Length': '0'} 2022-09-03T05:59:52.760 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004fe0b35a92bc8030-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.761 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.761 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:52.763 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.763 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.764 DEBUG:boto:Method: GET 2022-09-03T05:59:52.765 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.766 DEBUG:boto:Data: 2022-09-03T05:59:52.766 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.767 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.767 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.768 DEBUG:boto:Params: {} 2022-09-03T05:59:52.769 DEBUG:boto:Token: None 2022-09-03T05:59:52.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:52.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM= 2022-09-03T05:59:52.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kMgZVswo0mLnImKmgSqxjvLvbiM=', 'Content-Length': '0'} 2022-09-03T05:59:52.773 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000360841e9581e44b7-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.818 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.819 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.819 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.820 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.821 DEBUG:boto:Method: GET 2022-09-03T05:59:52.821 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.822 DEBUG:boto:Data: 2022-09-03T05:59:52.822 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.823 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:52.823 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.824 DEBUG:boto:Params: {} 2022-09-03T05:59:52.824 DEBUG:boto:Token: None 2022-09-03T05:59:52.825 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.828 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001e8d068e7f1f31ad-006312ed58-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.829 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.830 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.831 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.831 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.832 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.832 DEBUG:boto:Method: GET 2022-09-03T05:59:52.833 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:52.833 DEBUG:boto:Data: 2022-09-03T05:59:52.834 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.834 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.835 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.835 DEBUG:boto:Params: {} 2022-09-03T05:59:52.836 DEBUG:boto:Token: None 2022-09-03T05:59:52.836 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs= 2022-09-03T05:59:52.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BitDs+zoyVtUvI+gltH62R/ZWs=', 'Content-Length': '0'} 2022-09-03T05:59:52.841 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000062325a9426e997bf-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.842 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.843 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-17 zones={test-zone2, test-zone1} 2022-09-03T05:59:52.844 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-17 zones={test-zone3, test-zone1} 2022-09-03T05:59:52.845 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.846 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.847 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.848 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:59:52.848 DEBUG:boto:Data: 2022-09-03T05:59:52.849 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:52.850 DEBUG:boto:Port: 8001 2022-09-03T05:59:52.851 DEBUG:boto:Params: {} 2022-09-03T05:59:52.851 DEBUG:boto:Token: None 2022-09-03T05:59:52.852 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WSC1qZTDfzDTKM6ML3U2H9RCiCQ= 2022-09-03T05:59:52.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WSC1qZTDfzDTKM6ML3U2H9RCiCQ=', 'Content-Length': '0'} 2022-09-03T05:59:52.857 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000091a7413a30bbe9b5-006312ed58-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.858 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.859 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.860 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.861 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:59:52.862 DEBUG:boto:Data: 2022-09-03T05:59:52.863 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.863 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.864 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.865 DEBUG:boto:Params: {} 2022-09-03T05:59:52.865 DEBUG:boto:Token: None 2022-09-03T05:59:52.866 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/ 2022-09-03T05:59:52.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WSC1qZTDfzDTKM6ML3U2H9RCiCQ= 2022-09-03T05:59:52.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WSC1qZTDfzDTKM6ML3U2H9RCiCQ=', 'Content-Length': '0'} 2022-09-03T05:59:52.870 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000210314849e6460f5-006312ed58-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.870 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:59:52.871 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.872 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.872 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.873 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.873 DEBUG:boto:Method: GET 2022-09-03T05:59:52.874 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.874 DEBUG:boto:Data: 2022-09-03T05:59:52.875 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:52.876 DEBUG:boto:Port: 8001 2022-09-03T05:59:52.876 DEBUG:boto:Params: {} 2022-09-03T05:59:52.877 DEBUG:boto:Token: None 2022-09-03T05:59:52.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/?versions 2022-09-03T05:59:52.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PV7jFlXzLpuZzL4ZXCIeTcU8mOI= 2022-09-03T05:59:52.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PV7jFlXzLpuZzL4ZXCIeTcU8mOI=', 'Content-Length': '0'} 2022-09-03T05:59:52.882 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b66c5aa02db51c34-006312ed58-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.883 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.884 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:52.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:52.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:52.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:52.886 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:59:52.886 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.887 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.888 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.888 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.889 DEBUG:boto:Method: GET 2022-09-03T05:59:52.889 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.890 DEBUG:boto:Data: 2022-09-03T05:59:52.890 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.891 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.891 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.892 DEBUG:boto:Params: {} 2022-09-03T05:59:52.892 DEBUG:boto:Token: None 2022-09-03T05:59:52.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/?versions 2022-09-03T05:59:52.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PV7jFlXzLpuZzL4ZXCIeTcU8mOI= 2022-09-03T05:59:52.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PV7jFlXzLpuZzL4ZXCIeTcU8mOI=', 'Content-Length': '0'} 2022-09-03T05:59:52.897 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d88aa6440b2c4a42-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.897 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.898 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:52.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:52.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:52.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:52.901 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.901 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.902 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.902 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.903 DEBUG:boto:Method: GET 2022-09-03T05:59:52.903 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.904 DEBUG:boto:Data: 2022-09-03T05:59:52.904 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:52.905 DEBUG:boto:Port: 8001 2022-09-03T05:59:52.906 DEBUG:boto:Params: {} 2022-09-03T05:59:52.906 DEBUG:boto:Token: None 2022-09-03T05:59:52.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/?versions 2022-09-03T05:59:52.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PV7jFlXzLpuZzL4ZXCIeTcU8mOI= 2022-09-03T05:59:52.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PV7jFlXzLpuZzL4ZXCIeTcU8mOI=', 'Content-Length': '0'} 2022-09-03T05:59:52.911 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d78ae6d8624af1a9-006312ed58-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.911 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.912 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:52.913 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:52.914 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.914 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.915 DEBUG:boto:Method: GET 2022-09-03T05:59:52.915 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:52.916 DEBUG:boto:Data: 2022-09-03T05:59:52.917 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.917 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.918 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.918 DEBUG:boto:Params: {} 2022-09-03T05:59:52.919 DEBUG:boto:Token: None 2022-09-03T05:59:52.920 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/?versions 2022-09-03T05:59:52.920 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PV7jFlXzLpuZzL4ZXCIeTcU8mOI= 2022-09-03T05:59:52.921 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PV7jFlXzLpuZzL4ZXCIeTcU8mOI=', 'Content-Length': '0'} 2022-09-03T05:59:52.924 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b017c61c91429b8d-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.925 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:52.926 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:52.927 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:52.927 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:52.928 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.929 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.930 DEBUG:boto:Method: GET 2022-09-03T05:59:52.931 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.932 DEBUG:boto:Data: 2022-09-03T05:59:52.933 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:52.935 DEBUG:boto:Port: 8001 2022-09-03T05:59:52.936 DEBUG:boto:Params: {} 2022-09-03T05:59:52.937 DEBUG:boto:Token: None 2022-09-03T05:59:52.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:unLAH+fj4j5dQPThsA/stjE9Sts= 2022-09-03T05:59:52.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:unLAH+fj4j5dQPThsA/stjE9Sts=', 'Content-Length': '0'} 2022-09-03T05:59:52.943 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000db232dd658a374c5-006312ed58-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.945 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:52.945 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:52.946 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.947 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.948 DEBUG:boto:Method: GET 2022-09-03T05:59:52.949 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.950 DEBUG:boto:Data: 2022-09-03T05:59:52.951 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.952 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.953 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.954 DEBUG:boto:Params: {} 2022-09-03T05:59:52.955 DEBUG:boto:Token: None 2022-09-03T05:59:52.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:unLAH+fj4j5dQPThsA/stjE9Sts= 2022-09-03T05:59:52.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:unLAH+fj4j5dQPThsA/stjE9Sts=', 'Content-Length': '0'} 2022-09-03T05:59:52.959 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000105a6e57809afe37-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.959 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:52.960 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:52.961 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.961 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.962 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.963 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.964 DEBUG:boto:Data: 2022-09-03T05:59:52.965 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:52.966 DEBUG:boto:Port: 8001 2022-09-03T05:59:52.967 DEBUG:boto:Params: {} 2022-09-03T05:59:52.968 DEBUG:boto:Token: None 2022-09-03T05:59:52.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y6plo58gHRr2Cc+cDn2wRK3ajL8= 2022-09-03T05:59:52.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y6plo58gHRr2Cc+cDn2wRK3ajL8=', 'Content-Length': '0'} 2022-09-03T05:59:52.972 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000da55e1d50f66b703-006312ed58-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.973 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:52.974 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:52.975 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.976 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.977 DEBUG:boto:Method: HEAD 2022-09-03T05:59:52.978 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.978 DEBUG:boto:Data: 2022-09-03T05:59:52.979 DEBUG:boto:Headers: {} 2022-09-03T05:59:52.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:52.981 DEBUG:boto:Port: 8000 2022-09-03T05:59:52.982 DEBUG:boto:Params: {} 2022-09-03T05:59:52.983 DEBUG:boto:Token: None 2022-09-03T05:59:52.984 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:52 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:52.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y6plo58gHRr2Cc+cDn2wRK3ajL8= 2022-09-03T05:59:52.986 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y6plo58gHRr2Cc+cDn2wRK3ajL8=', 'Content-Length': '0'} 2022-09-03T05:59:52.988 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009077c9088e81d5af-006312ed58-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:52.989 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:52.989 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:53.003 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:53.004 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.005 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.005 DEBUG:boto:Method: GET 2022-09-03T05:59:53.006 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.007 DEBUG:boto:Data: 2022-09-03T05:59:53.007 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.008 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.009 DEBUG:boto:Params: {} 2022-09-03T05:59:53.010 DEBUG:boto:Token: None 2022-09-03T05:59:53.010 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DVFXr4th1HvBGKVkpV+Q3VIkijs= 2022-09-03T05:59:53.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DVFXr4th1HvBGKVkpV+Q3VIkijs=', 'Content-Length': '0'} 2022-09-03T05:59:53.014 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000307e730e396e2115-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.015 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:53.016 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:53.017 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.018 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.019 DEBUG:boto:Method: GET 2022-09-03T05:59:53.020 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.020 DEBUG:boto:Data: 2022-09-03T05:59:53.021 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.022 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.022 DEBUG:boto:Params: {} 2022-09-03T05:59:53.023 DEBUG:boto:Token: None 2022-09-03T05:59:53.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DVFXr4th1HvBGKVkpV+Q3VIkijs= 2022-09-03T05:59:53.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DVFXr4th1HvBGKVkpV+Q3VIkijs=', 'Content-Length': '0'} 2022-09-03T05:59:53.027 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000df99a5fdc25e6b44-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.028 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.029 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.029 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:53.030 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:53.030 DEBUG:boto:Method: GET 2022-09-03T05:59:53.031 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:53.031 DEBUG:boto:Data: 2022-09-03T05:59:53.032 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.033 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.033 DEBUG:boto:Params: {} 2022-09-03T05:59:53.034 DEBUG:boto:Token: None 2022-09-03T05:59:53.035 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.038 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000082b7747429a15727-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.086 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.088 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.089 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.090 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:53.091 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:53.091 DEBUG:boto:Method: GET 2022-09-03T05:59:53.092 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:53.093 DEBUG:boto:Data: 2022-09-03T05:59:53.094 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.096 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.097 DEBUG:boto:Params: {} 2022-09-03T05:59:53.097 DEBUG:boto:Token: None 2022-09-03T05:59:53.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.103 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091df16bbd3212783-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.103 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.104 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:53.105 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:53.106 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.106 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T05:59:53.107 DEBUG:boto:Data: 2022-09-03T05:59:53.107 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.108 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.108 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.109 DEBUG:boto:Params: {} 2022-09-03T05:59:53.109 DEBUG:boto:Token: None 2022-09-03T05:59:53.110 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/dummy 2022-09-03T05:59:53.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzdKwi8ZS6HoeJ15bXMtQwz21Ao= 2022-09-03T05:59:53.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzdKwi8ZS6HoeJ15bXMtQwz21Ao=', 'Content-Length': '0'} 2022-09-03T05:59:53.116 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000069ccd0f064212bc9-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.117 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:53.118 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:53.118 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.119 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T05:59:53.120 DEBUG:boto:Data: 2022-09-03T05:59:53.120 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.121 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.121 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.122 DEBUG:boto:Params: {} 2022-09-03T05:59:53.123 DEBUG:boto:Token: None 2022-09-03T05:59:53.123 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/dummy 2022-09-03T05:59:53.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzdKwi8ZS6HoeJ15bXMtQwz21Ao= 2022-09-03T05:59:53.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzdKwi8ZS6HoeJ15bXMtQwz21Ao=', 'Content-Length': '0'} 2022-09-03T05:59:53.128 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000bd01836c68b351ec-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.129 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:53.130 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:53.130 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:53.131 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.131 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.132 DEBUG:boto:Method: GET 2022-09-03T05:59:53.132 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.133 DEBUG:boto:Data: 2022-09-03T05:59:53.133 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.134 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.135 DEBUG:boto:Params: {} 2022-09-03T05:59:53.136 DEBUG:boto:Token: None 2022-09-03T05:59:53.136 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DVFXr4th1HvBGKVkpV+Q3VIkijs= 2022-09-03T05:59:53.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DVFXr4th1HvBGKVkpV+Q3VIkijs=', 'Content-Length': '0'} 2022-09-03T05:59:53.140 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000af115bd97d079a22-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.141 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:53.141 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:53.142 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.142 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.143 DEBUG:boto:Method: GET 2022-09-03T05:59:53.143 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.144 DEBUG:boto:Data: 2022-09-03T05:59:53.144 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.145 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.146 DEBUG:boto:Params: {} 2022-09-03T05:59:53.147 DEBUG:boto:Token: None 2022-09-03T05:59:53.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:53.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DVFXr4th1HvBGKVkpV+Q3VIkijs= 2022-09-03T05:59:53.148 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DVFXr4th1HvBGKVkpV+Q3VIkijs=', 'Content-Length': '0'} 2022-09-03T05:59:53.151 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000084032dffc9775d4e-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.151 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.152 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.153 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:53.153 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:53.154 DEBUG:boto:Method: GET 2022-09-03T05:59:53.154 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:53.155 DEBUG:boto:Data: 2022-09-03T05:59:53.155 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.156 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.157 DEBUG:boto:Params: {} 2022-09-03T05:59:53.157 DEBUG:boto:Token: None 2022-09-03T05:59:53.158 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.162 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6c11c9fa57aeee6-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.163 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.164 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.164 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.165 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:53.165 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:53.166 DEBUG:boto:Method: GET 2022-09-03T05:59:53.167 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:53.168 DEBUG:boto:Data: 2022-09-03T05:59:53.168 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.169 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.169 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.170 DEBUG:boto:Params: {} 2022-09-03T05:59:53.170 DEBUG:boto:Token: None 2022-09-03T05:59:53.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.179 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000070cda74fd8120913-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.227 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.228 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:53.229 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.230 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.230 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.231 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.232 DEBUG:boto:Method: GET 2022-09-03T05:59:53.232 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.233 DEBUG:boto:Data: 2022-09-03T05:59:53.233 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.234 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.235 DEBUG:boto:Params: {} 2022-09-03T05:59:53.236 DEBUG:boto:Token: None 2022-09-03T05:59:53.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0= 2022-09-03T05:59:53.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0=', 'Content-Length': '0'} 2022-09-03T05:59:53.242 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009e4c0673a949a772-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.243 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.244 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.245 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.245 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.246 DEBUG:boto:Method: GET 2022-09-03T05:59:53.247 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.248 DEBUG:boto:Data: 2022-09-03T05:59:53.249 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.251 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.252 DEBUG:boto:Params: {} 2022-09-03T05:59:53.253 DEBUG:boto:Token: None 2022-09-03T05:59:53.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0= 2022-09-03T05:59:53.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0=', 'Content-Length': '0'} 2022-09-03T05:59:53.259 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000005e4ea7858277661-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.260 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.261 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.261 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.262 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.263 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.263 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.264 DEBUG:boto:Data: 2022-09-03T05:59:53.265 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.265 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.266 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.266 DEBUG:boto:Params: {} 2022-09-03T05:59:53.267 DEBUG:boto:Token: None 2022-09-03T05:59:53.268 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L9XBYDPAdAtlXvG3ev0RCLkTgsk= 2022-09-03T05:59:53.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L9XBYDPAdAtlXvG3ev0RCLkTgsk=', 'Content-Length': '0'} 2022-09-03T05:59:53.272 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000503908f3601289c5-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.272 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.273 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.273 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.274 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.275 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.275 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.276 DEBUG:boto:Data: 2022-09-03T05:59:53.276 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.277 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.278 DEBUG:boto:Params: {} 2022-09-03T05:59:53.279 DEBUG:boto:Token: None 2022-09-03T05:59:53.279 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L9XBYDPAdAtlXvG3ev0RCLkTgsk= 2022-09-03T05:59:53.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L9XBYDPAdAtlXvG3ev0RCLkTgsk=', 'Content-Length': '0'} 2022-09-03T05:59:53.283 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000027318ee1b0742f5-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.284 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:53.285 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.286 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.287 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.288 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.289 DEBUG:boto:Method: GET 2022-09-03T05:59:53.290 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.291 DEBUG:boto:Data: 2022-09-03T05:59:53.292 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.293 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.294 DEBUG:boto:Params: {} 2022-09-03T05:59:53.294 DEBUG:boto:Token: None 2022-09-03T05:59:53.295 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0= 2022-09-03T05:59:53.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0=', 'Content-Length': '0'} 2022-09-03T05:59:53.300 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a4a54c4f2dd731e8-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.301 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.302 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.303 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.303 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.304 DEBUG:boto:Method: GET 2022-09-03T05:59:53.305 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.306 DEBUG:boto:Data: 2022-09-03T05:59:53.307 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.308 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.309 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.310 DEBUG:boto:Params: {} 2022-09-03T05:59:53.311 DEBUG:boto:Token: None 2022-09-03T05:59:53.312 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.313 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0= 2022-09-03T05:59:53.314 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0=', 'Content-Length': '0'} 2022-09-03T05:59:53.317 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004c23084fef62abd1-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.317 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.318 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.319 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.319 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.320 DEBUG:boto:Method: GET 2022-09-03T05:59:53.321 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.321 DEBUG:boto:Data: 2022-09-03T05:59:53.322 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.322 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.323 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.324 DEBUG:boto:Params: {} 2022-09-03T05:59:53.324 DEBUG:boto:Token: None 2022-09-03T05:59:53.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.329 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008c9629317a7e988e-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.330 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.331 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.331 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.332 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.333 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.334 DEBUG:boto:Method: GET 2022-09-03T05:59:53.335 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.336 DEBUG:boto:Data: 2022-09-03T05:59:53.337 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.339 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.340 DEBUG:boto:Params: {} 2022-09-03T05:59:53.341 DEBUG:boto:Token: None 2022-09-03T05:59:53.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.346 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002dcdc3ea88bc5390-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.346 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.348 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.349 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.350 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.351 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T05:59:53.351 DEBUG:boto:Data: 2022-09-03T05:59:53.352 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.352 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.353 DEBUG:boto:Params: {} 2022-09-03T05:59:53.354 DEBUG:boto:Token: None 2022-09-03T05:59:53.354 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1 2022-09-03T05:59:53.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:shCTTetsbR1QFnWW8Y7HSF2E2ho= 2022-09-03T05:59:53.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:shCTTetsbR1QFnWW8Y7HSF2E2ho=', 'Content-Length': '0'} 2022-09-03T05:59:53.358 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000fd962465220cdc9-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.359 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.360 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.361 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.361 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T05:59:53.362 DEBUG:boto:Data: 2022-09-03T05:59:53.362 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.363 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.364 DEBUG:boto:Params: {} 2022-09-03T05:59:53.364 DEBUG:boto:Token: None 2022-09-03T05:59:53.365 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1 2022-09-03T05:59:53.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:shCTTetsbR1QFnWW8Y7HSF2E2ho= 2022-09-03T05:59:53.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:shCTTetsbR1QFnWW8Y7HSF2E2ho=', 'Content-Length': '0'} 2022-09-03T05:59:53.370 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000080b4497cf5ba430b-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.371 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:53.372 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.373 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.374 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.374 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.375 DEBUG:boto:Method: GET 2022-09-03T05:59:53.376 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.376 DEBUG:boto:Data: 2022-09-03T05:59:53.377 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.378 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.378 DEBUG:boto:Params: {} 2022-09-03T05:59:53.379 DEBUG:boto:Token: None 2022-09-03T05:59:53.380 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0= 2022-09-03T05:59:53.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0=', 'Content-Length': '0'} 2022-09-03T05:59:53.384 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ac831a9f74880ebf-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.430 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.431 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:53.432 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.432 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.433 DEBUG:boto:Method: GET 2022-09-03T05:59:53.433 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.434 DEBUG:boto:Data: 2022-09-03T05:59:53.435 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.436 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.436 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.437 DEBUG:boto:Params: {} 2022-09-03T05:59:53.437 DEBUG:boto:Token: None 2022-09-03T05:59:53.438 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:53.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0= 2022-09-03T05:59:53.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xPmvJrBYTlnt++KfdsBL6B+lVq0=', 'Content-Length': '0'} 2022-09-03T05:59:53.442 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a4ce3158a6dd4351-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.443 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.443 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.444 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.445 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.445 DEBUG:boto:Method: GET 2022-09-03T05:59:53.446 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.447 DEBUG:boto:Data: 2022-09-03T05:59:53.447 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.448 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.449 DEBUG:boto:Params: {} 2022-09-03T05:59:53.449 DEBUG:boto:Token: None 2022-09-03T05:59:53.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.455 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f5e0c94f1a016c2d-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.455 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.456 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.457 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.457 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.458 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.459 DEBUG:boto:Method: GET 2022-09-03T05:59:53.459 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:53.460 DEBUG:boto:Data: 2022-09-03T05:59:53.460 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.461 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.462 DEBUG:boto:Params: {} 2022-09-03T05:59:53.463 DEBUG:boto:Token: None 2022-09-03T05:59:53.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.470 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ebf0a77f594a6d81-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.471 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.473 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:53.473 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.474 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.475 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.476 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.476 DEBUG:boto:Method: GET 2022-09-03T05:59:53.477 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.478 DEBUG:boto:Data: 2022-09-03T05:59:53.478 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.479 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.480 DEBUG:boto:Params: {} 2022-09-03T05:59:53.480 DEBUG:boto:Token: None 2022-09-03T05:59:53.481 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE= 2022-09-03T05:59:53.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE=', 'Content-Length': '0'} 2022-09-03T05:59:53.485 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000012aefaf070fdefd5-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.486 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.486 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.487 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.488 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.488 DEBUG:boto:Method: GET 2022-09-03T05:59:53.489 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.489 DEBUG:boto:Data: 2022-09-03T05:59:53.490 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.491 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.491 DEBUG:boto:Params: {} 2022-09-03T05:59:53.492 DEBUG:boto:Token: None 2022-09-03T05:59:53.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE= 2022-09-03T05:59:53.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE=', 'Content-Length': '0'} 2022-09-03T05:59:53.496 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f77ff866871e1122-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.497 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.498 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.498 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.499 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.499 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.500 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.500 DEBUG:boto:Data: 2022-09-03T05:59:53.501 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.501 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.502 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.502 DEBUG:boto:Params: {} 2022-09-03T05:59:53.503 DEBUG:boto:Token: None 2022-09-03T05:59:53.503 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hht2RYRiJC/uY/YCKVkeq30Ijok= 2022-09-03T05:59:53.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hht2RYRiJC/uY/YCKVkeq30Ijok=', 'Content-Length': '0'} 2022-09-03T05:59:53.507 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000aaae36897c46dc82-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.507 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.508 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.509 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.509 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.509 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.510 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.511 DEBUG:boto:Data: 2022-09-03T05:59:53.511 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.512 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.513 DEBUG:boto:Params: {} 2022-09-03T05:59:53.513 DEBUG:boto:Token: None 2022-09-03T05:59:53.514 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hht2RYRiJC/uY/YCKVkeq30Ijok= 2022-09-03T05:59:53.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hht2RYRiJC/uY/YCKVkeq30Ijok=', 'Content-Length': '0'} 2022-09-03T05:59:53.517 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000025b84307b190e0a7-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.518 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:53.518 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.519 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.519 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.520 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.520 DEBUG:boto:Method: GET 2022-09-03T05:59:53.521 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.521 DEBUG:boto:Data: 2022-09-03T05:59:53.522 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.523 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.523 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.524 DEBUG:boto:Params: {} 2022-09-03T05:59:53.524 DEBUG:boto:Token: None 2022-09-03T05:59:53.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.525 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE= 2022-09-03T05:59:53.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE=', 'Content-Length': '0'} 2022-09-03T05:59:53.528 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000068f9ab0e9ab5919c-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.570 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.571 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.571 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.572 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.572 DEBUG:boto:Method: GET 2022-09-03T05:59:53.573 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.573 DEBUG:boto:Data: 2022-09-03T05:59:53.574 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.574 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.575 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.575 DEBUG:boto:Params: {} 2022-09-03T05:59:53.576 DEBUG:boto:Token: None 2022-09-03T05:59:53.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.577 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE= 2022-09-03T05:59:53.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE=', 'Content-Length': '0'} 2022-09-03T05:59:53.580 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009576c82a7cc600a3-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.581 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.582 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.582 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.583 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.583 DEBUG:boto:Method: GET 2022-09-03T05:59:53.584 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.584 DEBUG:boto:Data: 2022-09-03T05:59:53.585 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.585 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.586 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.586 DEBUG:boto:Params: {} 2022-09-03T05:59:53.587 DEBUG:boto:Token: None 2022-09-03T05:59:53.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.591 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000cd478666f078ff0-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.592 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.593 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.593 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.594 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.595 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.595 DEBUG:boto:Method: GET 2022-09-03T05:59:53.596 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.596 DEBUG:boto:Data: 2022-09-03T05:59:53.597 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.598 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.598 DEBUG:boto:Params: {} 2022-09-03T05:59:53.599 DEBUG:boto:Token: None 2022-09-03T05:59:53.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.603 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011922c94e11175be-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.604 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.605 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.605 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.606 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.606 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T05:59:53.607 DEBUG:boto:Data: 2022-09-03T05:59:53.607 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.608 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.609 DEBUG:boto:Params: {} 2022-09-03T05:59:53.609 DEBUG:boto:Token: None 2022-09-03T05:59:53.610 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2 2022-09-03T05:59:53.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rZrE1FstazRYl+upANmjguqgPsQ= 2022-09-03T05:59:53.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rZrE1FstazRYl+upANmjguqgPsQ=', 'Content-Length': '0'} 2022-09-03T05:59:53.614 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000023505b915416526e-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.615 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.615 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.616 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.616 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T05:59:53.617 DEBUG:boto:Data: 2022-09-03T05:59:53.617 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.618 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.618 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.619 DEBUG:boto:Params: {} 2022-09-03T05:59:53.619 DEBUG:boto:Token: None 2022-09-03T05:59:53.620 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2 2022-09-03T05:59:53.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rZrE1FstazRYl+upANmjguqgPsQ= 2022-09-03T05:59:53.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rZrE1FstazRYl+upANmjguqgPsQ=', 'Content-Length': '0'} 2022-09-03T05:59:53.623 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ae209d4185e8b93b-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.624 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:53.624 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.625 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.626 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.626 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.627 DEBUG:boto:Method: GET 2022-09-03T05:59:53.627 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.628 DEBUG:boto:Data: 2022-09-03T05:59:53.628 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.629 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.630 DEBUG:boto:Params: {} 2022-09-03T05:59:53.630 DEBUG:boto:Token: None 2022-09-03T05:59:53.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE= 2022-09-03T05:59:53.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE=', 'Content-Length': '0'} 2022-09-03T05:59:53.634 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e11bb117bda3f054-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.679 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.680 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:53.681 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.682 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.683 DEBUG:boto:Method: GET 2022-09-03T05:59:53.684 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.685 DEBUG:boto:Data: 2022-09-03T05:59:53.686 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.688 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.689 DEBUG:boto:Params: {} 2022-09-03T05:59:53.690 DEBUG:boto:Token: None 2022-09-03T05:59:53.691 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:53.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE= 2022-09-03T05:59:53.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7DfpsBmxj8EY8cHsSSI0zAfwWTE=', 'Content-Length': '0'} 2022-09-03T05:59:53.696 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fe4457dc78bb122f-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.738 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.739 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.740 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.740 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.741 DEBUG:boto:Method: GET 2022-09-03T05:59:53.741 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.742 DEBUG:boto:Data: 2022-09-03T05:59:53.743 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.743 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.744 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.744 DEBUG:boto:Params: {} 2022-09-03T05:59:53.745 DEBUG:boto:Token: None 2022-09-03T05:59:53.746 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.751 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007cd69719e55bc3cf-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.753 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.754 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.755 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.755 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.756 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.757 DEBUG:boto:Method: GET 2022-09-03T05:59:53.757 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:53.758 DEBUG:boto:Data: 2022-09-03T05:59:53.758 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.759 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.760 DEBUG:boto:Params: {} 2022-09-03T05:59:53.760 DEBUG:boto:Token: None 2022-09-03T05:59:53.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.767 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000043b032553ab15798-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.768 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.768 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:53.769 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.770 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.770 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.771 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.772 DEBUG:boto:Method: GET 2022-09-03T05:59:53.773 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.773 DEBUG:boto:Data: 2022-09-03T05:59:53.774 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.775 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.775 DEBUG:boto:Params: {} 2022-09-03T05:59:53.776 DEBUG:boto:Token: None 2022-09-03T05:59:53.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg= 2022-09-03T05:59:53.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg=', 'Content-Length': '0'} 2022-09-03T05:59:53.780 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000d0f9a10425907ab-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.781 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.781 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.782 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.782 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.783 DEBUG:boto:Method: GET 2022-09-03T05:59:53.783 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.784 DEBUG:boto:Data: 2022-09-03T05:59:53.784 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.785 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.785 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.786 DEBUG:boto:Params: {} 2022-09-03T05:59:53.787 DEBUG:boto:Token: None 2022-09-03T05:59:53.787 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg= 2022-09-03T05:59:53.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg=', 'Content-Length': '0'} 2022-09-03T05:59:53.791 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000042c80c312c11e442-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.792 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.792 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.793 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.793 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.794 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.794 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.795 DEBUG:boto:Data: 2022-09-03T05:59:53.795 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.796 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.796 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.797 DEBUG:boto:Params: {} 2022-09-03T05:59:53.798 DEBUG:boto:Token: None 2022-09-03T05:59:53.798 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KQrf2zo3mlJJyuSGerHRdKIbNGY= 2022-09-03T05:59:53.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KQrf2zo3mlJJyuSGerHRdKIbNGY=', 'Content-Length': '0'} 2022-09-03T05:59:53.801 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008ed6d9333b423fcc-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.802 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.803 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.803 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.804 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.804 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.805 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.805 DEBUG:boto:Data: 2022-09-03T05:59:53.806 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.807 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.807 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.808 DEBUG:boto:Params: {} 2022-09-03T05:59:53.809 DEBUG:boto:Token: None 2022-09-03T05:59:53.809 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KQrf2zo3mlJJyuSGerHRdKIbNGY= 2022-09-03T05:59:53.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KQrf2zo3mlJJyuSGerHRdKIbNGY=', 'Content-Length': '0'} 2022-09-03T05:59:53.813 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e0b1f6cd84c03884-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.814 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:53.814 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.815 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.816 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.816 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.817 DEBUG:boto:Method: GET 2022-09-03T05:59:53.818 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.818 DEBUG:boto:Data: 2022-09-03T05:59:53.819 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.820 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.820 DEBUG:boto:Params: {} 2022-09-03T05:59:53.821 DEBUG:boto:Token: None 2022-09-03T05:59:53.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg= 2022-09-03T05:59:53.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg=', 'Content-Length': '0'} 2022-09-03T05:59:53.825 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000159ac48ff152d96f-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.826 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.826 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.827 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.827 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.828 DEBUG:boto:Method: GET 2022-09-03T05:59:53.828 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.829 DEBUG:boto:Data: 2022-09-03T05:59:53.830 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.831 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.831 DEBUG:boto:Params: {} 2022-09-03T05:59:53.832 DEBUG:boto:Token: None 2022-09-03T05:59:53.832 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg= 2022-09-03T05:59:53.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg=', 'Content-Length': '0'} 2022-09-03T05:59:53.836 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000368883449ea2ba88-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.878 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.879 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.880 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:53.881 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:53.881 DEBUG:boto:Method: GET 2022-09-03T05:59:53.882 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:53.882 DEBUG:boto:Data: 2022-09-03T05:59:53.883 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.884 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.884 DEBUG:boto:Params: {} 2022-09-03T05:59:53.885 DEBUG:boto:Token: None 2022-09-03T05:59:53.886 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.890 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ec618b00be57343e-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.890 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.891 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.892 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.892 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:53.893 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:53.893 DEBUG:boto:Method: GET 2022-09-03T05:59:53.894 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:53.895 DEBUG:boto:Data: 2022-09-03T05:59:53.895 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.896 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.897 DEBUG:boto:Params: {} 2022-09-03T05:59:53.898 DEBUG:boto:Token: None 2022-09-03T05:59:53.899 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.904 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007f6325050e9c4f00-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.905 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:53.906 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.907 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.908 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.909 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T05:59:53.909 DEBUG:boto:Data: 2022-09-03T05:59:53.910 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.911 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.912 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.913 DEBUG:boto:Params: {} 2022-09-03T05:59:53.913 DEBUG:boto:Token: None 2022-09-03T05:59:53.914 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3 2022-09-03T05:59:53.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n0n5g16tuXgdhbfwuQXMAXpOydo= 2022-09-03T05:59:53.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n0n5g16tuXgdhbfwuQXMAXpOydo=', 'Content-Length': '0'} 2022-09-03T05:59:53.918 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009678e185da5b289f-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.919 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.920 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.920 DEBUG:boto:Method: HEAD 2022-09-03T05:59:53.921 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T05:59:53.921 DEBUG:boto:Data: 2022-09-03T05:59:53.922 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.923 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.923 DEBUG:boto:Params: {} 2022-09-03T05:59:53.924 DEBUG:boto:Token: None 2022-09-03T05:59:53.925 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3 2022-09-03T05:59:53.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n0n5g16tuXgdhbfwuQXMAXpOydo= 2022-09-03T05:59:53.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n0n5g16tuXgdhbfwuQXMAXpOydo=', 'Content-Length': '0'} 2022-09-03T05:59:53.928 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b5e3d604aa633435-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.929 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:53.930 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.930 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.931 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.932 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.932 DEBUG:boto:Method: GET 2022-09-03T05:59:53.933 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.933 DEBUG:boto:Data: 2022-09-03T05:59:53.934 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.935 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.935 DEBUG:boto:Params: {} 2022-09-03T05:59:53.936 DEBUG:boto:Token: None 2022-09-03T05:59:53.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg= 2022-09-03T05:59:53.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg=', 'Content-Length': '0'} 2022-09-03T05:59:53.942 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c1065b1e0c7a0727-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.943 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.944 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:53.945 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.946 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.946 DEBUG:boto:Method: GET 2022-09-03T05:59:53.947 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.947 DEBUG:boto:Data: 2022-09-03T05:59:53.948 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:53.949 DEBUG:boto:Port: 8000 2022-09-03T05:59:53.950 DEBUG:boto:Params: {} 2022-09-03T05:59:53.951 DEBUG:boto:Token: None 2022-09-03T05:59:53.952 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:53.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg= 2022-09-03T05:59:53.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/eKDEsG96H6rHwUWVpcHffytRg=', 'Content-Length': '0'} 2022-09-03T05:59:53.955 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000001265c8ef673ef84-006312ed59-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:53.956 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:53.956 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:53.957 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:53.957 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:53.958 DEBUG:boto:Method: GET 2022-09-03T05:59:53.959 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:53.959 DEBUG:boto:Data: 2022-09-03T05:59:53.960 DEBUG:boto:Headers: {} 2022-09-03T05:59:53.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:53.961 DEBUG:boto:Port: 8001 2022-09-03T05:59:53.961 DEBUG:boto:Params: {} 2022-09-03T05:59:53.962 DEBUG:boto:Token: None 2022-09-03T05:59:53.963 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:53 GMT /avysmw-17/ 2022-09-03T05:59:53.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs= 2022-09-03T05:59:53.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rrzMUJ2SBIst2MWQ0oxXqkbXLqs=', 'Content-Length': '0'} 2022-09-03T05:59:53.967 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a73610c20036d6c8-006312ed59-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.014 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.015 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.016 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.016 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.017 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.018 DEBUG:boto:Method: GET 2022-09-03T05:59:54.018 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.019 DEBUG:boto:Data: 2022-09-03T05:59:54.019 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:54.020 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.021 DEBUG:boto:Params: {} 2022-09-03T05:59:54.022 DEBUG:boto:Token: None 2022-09-03T05:59:54.022 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.026 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001bebc780393ecaef-006312ed5a-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.027 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.029 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-17 zones={test-zone3, test-zone1} 2022-09-03T05:59:54.029 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-17 zones={test-zone3, test-zone2} 2022-09-03T05:59:54.030 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.031 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.032 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.033 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:59:54.034 DEBUG:boto:Data: 2022-09-03T05:59:54.035 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.037 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.038 DEBUG:boto:Params: {} 2022-09-03T05:59:54.039 DEBUG:boto:Token: None 2022-09-03T05:59:54.040 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3jxckHtzFCF/lUeTUI1nquZuGqg= 2022-09-03T05:59:54.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3jxckHtzFCF/lUeTUI1nquZuGqg=', 'Content-Length': '0'} 2022-09-03T05:59:54.046 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b3532901e408a5b5-006312ed5a-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.047 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.048 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.049 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.050 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:59:54.051 DEBUG:boto:Data: 2022-09-03T05:59:54.052 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.052 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.053 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.054 DEBUG:boto:Params: {} 2022-09-03T05:59:54.055 DEBUG:boto:Token: None 2022-09-03T05:59:54.056 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3jxckHtzFCF/lUeTUI1nquZuGqg= 2022-09-03T05:59:54.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3jxckHtzFCF/lUeTUI1nquZuGqg=', 'Content-Length': '0'} 2022-09-03T05:59:54.061 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004d3c540e3274cbdd-006312ed5a-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.062 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:59:54.062 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.063 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.063 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.064 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.064 DEBUG:boto:Method: GET 2022-09-03T05:59:54.065 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.066 DEBUG:boto:Data: 2022-09-03T05:59:54.067 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.068 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.071 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.073 DEBUG:boto:Params: {} 2022-09-03T05:59:54.074 DEBUG:boto:Token: None 2022-09-03T05:59:54.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/?versions 2022-09-03T05:59:54.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xKhsf8tHDf9tq17D6WKPS0S6RKM= 2022-09-03T05:59:54.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xKhsf8tHDf9tq17D6WKPS0S6RKM=', 'Content-Length': '0'} 2022-09-03T05:59:54.081 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e54e3dc0523ad6d7-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.081 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.082 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:54.083 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:54.084 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:54.084 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:54.085 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:59:54.085 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.086 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.087 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.087 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.088 DEBUG:boto:Method: GET 2022-09-03T05:59:54.088 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.089 DEBUG:boto:Data: 2022-09-03T05:59:54.089 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.090 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.090 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.091 DEBUG:boto:Params: {} 2022-09-03T05:59:54.092 DEBUG:boto:Token: None 2022-09-03T05:59:54.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/?versions 2022-09-03T05:59:54.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xKhsf8tHDf9tq17D6WKPS0S6RKM= 2022-09-03T05:59:54.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xKhsf8tHDf9tq17D6WKPS0S6RKM=', 'Content-Length': '0'} 2022-09-03T05:59:54.098 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f7ac493e414bf7c0-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.099 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.101 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:54.102 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:54.103 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:54.104 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:54.105 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.106 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.106 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.107 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.107 DEBUG:boto:Method: GET 2022-09-03T05:59:54.108 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.109 DEBUG:boto:Data: 2022-09-03T05:59:54.109 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.110 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.111 DEBUG:boto:Params: {} 2022-09-03T05:59:54.112 DEBUG:boto:Token: None 2022-09-03T05:59:54.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/?versions 2022-09-03T05:59:54.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xKhsf8tHDf9tq17D6WKPS0S6RKM= 2022-09-03T05:59:54.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xKhsf8tHDf9tq17D6WKPS0S6RKM=', 'Content-Length': '0'} 2022-09-03T05:59:54.118 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085ebb18fd20016e3-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.119 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.121 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.121 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.122 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.122 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.123 DEBUG:boto:Method: GET 2022-09-03T05:59:54.123 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T05:59:54.124 DEBUG:boto:Data: 2022-09-03T05:59:54.125 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.125 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.126 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.126 DEBUG:boto:Params: {} 2022-09-03T05:59:54.127 DEBUG:boto:Token: None 2022-09-03T05:59:54.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/?versions 2022-09-03T05:59:54.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xKhsf8tHDf9tq17D6WKPS0S6RKM= 2022-09-03T05:59:54.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xKhsf8tHDf9tq17D6WKPS0S6RKM=', 'Content-Length': '0'} 2022-09-03T05:59:54.131 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4c06a3615fb0bca-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.132 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.134 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:54.135 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.137 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.137 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.138 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.152 DEBUG:boto:Method: GET 2022-09-03T05:59:54.153 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.154 DEBUG:boto:Data: 2022-09-03T05:59:54.155 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.158 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.159 DEBUG:boto:Params: {} 2022-09-03T05:59:54.160 DEBUG:boto:Token: None 2022-09-03T05:59:54.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM= 2022-09-03T05:59:54.163 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM=', 'Content-Length': '0'} 2022-09-03T05:59:54.166 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000090a7fe6747c0e057-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.168 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.169 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.170 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.171 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.172 DEBUG:boto:Method: GET 2022-09-03T05:59:54.173 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.174 DEBUG:boto:Data: 2022-09-03T05:59:54.175 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.175 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.176 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.177 DEBUG:boto:Params: {} 2022-09-03T05:59:54.177 DEBUG:boto:Token: None 2022-09-03T05:59:54.178 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM= 2022-09-03T05:59:54.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM=', 'Content-Length': '0'} 2022-09-03T05:59:54.182 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000029b6994fff3cef35-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.182 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.183 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.183 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.184 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.184 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.185 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.185 DEBUG:boto:Data: 2022-09-03T05:59:54.186 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.186 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.187 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.187 DEBUG:boto:Params: {} 2022-09-03T05:59:54.188 DEBUG:boto:Token: None 2022-09-03T05:59:54.188 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bUjiI6gFYIIlv8vskJNM1la++4o= 2022-09-03T05:59:54.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bUjiI6gFYIIlv8vskJNM1la++4o=', 'Content-Length': '0'} 2022-09-03T05:59:54.192 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000008067e03245c3fbf-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.193 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.194 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.195 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.196 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.197 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.198 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.199 DEBUG:boto:Data: 2022-09-03T05:59:54.200 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.201 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.202 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.203 DEBUG:boto:Params: {} 2022-09-03T05:59:54.204 DEBUG:boto:Token: None 2022-09-03T05:59:54.205 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bUjiI6gFYIIlv8vskJNM1la++4o= 2022-09-03T05:59:54.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bUjiI6gFYIIlv8vskJNM1la++4o=', 'Content-Length': '0'} 2022-09-03T05:59:54.210 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000092e6622cdad7ca3b-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.211 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:54.212 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.213 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.213 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.214 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.215 DEBUG:boto:Method: GET 2022-09-03T05:59:54.215 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.216 DEBUG:boto:Data: 2022-09-03T05:59:54.216 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.217 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.218 DEBUG:boto:Params: {} 2022-09-03T05:59:54.219 DEBUG:boto:Token: None 2022-09-03T05:59:54.219 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM= 2022-09-03T05:59:54.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM=', 'Content-Length': '0'} 2022-09-03T05:59:54.224 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c55df0269f0e7990-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.225 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.226 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.226 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.227 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.227 DEBUG:boto:Method: GET 2022-09-03T05:59:54.228 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.228 DEBUG:boto:Data: 2022-09-03T05:59:54.229 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.229 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.230 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.230 DEBUG:boto:Params: {} 2022-09-03T05:59:54.231 DEBUG:boto:Token: None 2022-09-03T05:59:54.232 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM= 2022-09-03T05:59:54.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM=', 'Content-Length': '0'} 2022-09-03T05:59:54.235 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003b35c584219ea5d5-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.236 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.236 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.237 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:54.237 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:54.238 DEBUG:boto:Method: GET 2022-09-03T05:59:54.238 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:54.239 DEBUG:boto:Data: 2022-09-03T05:59:54.239 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.240 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.241 DEBUG:boto:Params: {} 2022-09-03T05:59:54.241 DEBUG:boto:Token: None 2022-09-03T05:59:54.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.246 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000046ae5206225e16d4-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.247 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.247 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.248 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.248 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:54.249 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:54.250 DEBUG:boto:Method: GET 2022-09-03T05:59:54.250 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:54.251 DEBUG:boto:Data: 2022-09-03T05:59:54.251 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.252 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.252 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.253 DEBUG:boto:Params: {} 2022-09-03T05:59:54.253 DEBUG:boto:Token: None 2022-09-03T05:59:54.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.258 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ae5e9fc5531effa3-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.258 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.259 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.260 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.260 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.261 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T05:59:54.261 DEBUG:boto:Data: 2022-09-03T05:59:54.262 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.263 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.263 DEBUG:boto:Params: {} 2022-09-03T05:59:54.264 DEBUG:boto:Token: None 2022-09-03T05:59:54.264 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy 2022-09-03T05:59:54.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yRrT/2LIvvG4QYeMY7pnz7FkM7s= 2022-09-03T05:59:54.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yRrT/2LIvvG4QYeMY7pnz7FkM7s=', 'Content-Length': '0'} 2022-09-03T05:59:54.268 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000073854cbb1acd9dca-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.269 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.269 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.270 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.270 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T05:59:54.271 DEBUG:boto:Data: 2022-09-03T05:59:54.271 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.272 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.272 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.273 DEBUG:boto:Params: {} 2022-09-03T05:59:54.274 DEBUG:boto:Token: None 2022-09-03T05:59:54.274 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy 2022-09-03T05:59:54.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yRrT/2LIvvG4QYeMY7pnz7FkM7s= 2022-09-03T05:59:54.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yRrT/2LIvvG4QYeMY7pnz7FkM7s=', 'Content-Length': '0'} 2022-09-03T05:59:54.278 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007c29b0d4398ba287-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.278 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:54.279 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.280 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.280 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.281 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.281 DEBUG:boto:Method: GET 2022-09-03T05:59:54.282 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.282 DEBUG:boto:Data: 2022-09-03T05:59:54.283 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.283 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.284 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.284 DEBUG:boto:Params: {} 2022-09-03T05:59:54.285 DEBUG:boto:Token: None 2022-09-03T05:59:54.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM= 2022-09-03T05:59:54.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM=', 'Content-Length': '0'} 2022-09-03T05:59:54.289 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000058e2908ba176ce3d-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.290 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T05:59:54.290 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T05:59:54.291 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.291 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.292 DEBUG:boto:Method: GET 2022-09-03T05:59:54.292 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.293 DEBUG:boto:Data: 2022-09-03T05:59:54.293 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.294 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.294 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.295 DEBUG:boto:Params: {} 2022-09-03T05:59:54.296 DEBUG:boto:Token: None 2022-09-03T05:59:54.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T05:59:54.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM= 2022-09-03T05:59:54.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0mMj5OaMinOxIHw1u1vh8ExzzM=', 'Content-Length': '0'} 2022-09-03T05:59:54.300 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000041b217d59fae6e32-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.301 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.302 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.302 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:54.303 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:54.303 DEBUG:boto:Method: GET 2022-09-03T05:59:54.304 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:54.304 DEBUG:boto:Data: 2022-09-03T05:59:54.305 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.306 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.307 DEBUG:boto:Params: {} 2022-09-03T05:59:54.307 DEBUG:boto:Token: None 2022-09-03T05:59:54.308 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.312 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000070a246e79932b131-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.354 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.355 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.356 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.356 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:54.357 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T05:59:54.357 DEBUG:boto:Method: GET 2022-09-03T05:59:54.358 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T05:59:54.358 DEBUG:boto:Data: 2022-09-03T05:59:54.359 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.359 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.360 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.360 DEBUG:boto:Params: {} 2022-09-03T05:59:54.361 DEBUG:boto:Token: None 2022-09-03T05:59:54.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.366 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005d561ebf0ca24831-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.367 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.368 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:54.368 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.369 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.369 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.370 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.371 DEBUG:boto:Method: GET 2022-09-03T05:59:54.371 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.372 DEBUG:boto:Data: 2022-09-03T05:59:54.372 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.374 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.374 DEBUG:boto:Params: {} 2022-09-03T05:59:54.375 DEBUG:boto:Token: None 2022-09-03T05:59:54.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw= 2022-09-03T05:59:54.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw=', 'Content-Length': '0'} 2022-09-03T05:59:54.380 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fc0700e6ef804c6c-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.380 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.381 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.381 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.382 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.383 DEBUG:boto:Method: GET 2022-09-03T05:59:54.383 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.384 DEBUG:boto:Data: 2022-09-03T05:59:54.384 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.385 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.385 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.386 DEBUG:boto:Params: {} 2022-09-03T05:59:54.387 DEBUG:boto:Token: None 2022-09-03T05:59:54.388 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw= 2022-09-03T05:59:54.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw=', 'Content-Length': '0'} 2022-09-03T05:59:54.391 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000046a4d71645b7bd6a-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.392 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.392 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.393 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.393 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.394 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.394 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.395 DEBUG:boto:Data: 2022-09-03T05:59:54.395 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.396 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.397 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.397 DEBUG:boto:Params: {} 2022-09-03T05:59:54.398 DEBUG:boto:Token: None 2022-09-03T05:59:54.398 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:awle5IOVFXcPXBI0YT5e4hXo9cc= 2022-09-03T05:59:54.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:awle5IOVFXcPXBI0YT5e4hXo9cc=', 'Content-Length': '0'} 2022-09-03T05:59:54.402 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000022f4c910dda6376b-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.403 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.403 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.404 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.404 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.405 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.405 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.406 DEBUG:boto:Data: 2022-09-03T05:59:54.406 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.407 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.407 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.408 DEBUG:boto:Params: {} 2022-09-03T05:59:54.409 DEBUG:boto:Token: None 2022-09-03T05:59:54.409 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:awle5IOVFXcPXBI0YT5e4hXo9cc= 2022-09-03T05:59:54.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:awle5IOVFXcPXBI0YT5e4hXo9cc=', 'Content-Length': '0'} 2022-09-03T05:59:54.412 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d9923f20ba979233-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.413 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:54.414 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.415 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.416 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.417 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.418 DEBUG:boto:Method: GET 2022-09-03T05:59:54.419 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.419 DEBUG:boto:Data: 2022-09-03T05:59:54.420 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.422 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.422 DEBUG:boto:Params: {} 2022-09-03T05:59:54.423 DEBUG:boto:Token: None 2022-09-03T05:59:54.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw= 2022-09-03T05:59:54.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw=', 'Content-Length': '0'} 2022-09-03T05:59:54.427 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b063c33509bd4e70-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.428 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.429 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.430 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.431 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.432 DEBUG:boto:Method: GET 2022-09-03T05:59:54.432 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.433 DEBUG:boto:Data: 2022-09-03T05:59:54.433 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.434 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.435 DEBUG:boto:Params: {} 2022-09-03T05:59:54.436 DEBUG:boto:Token: None 2022-09-03T05:59:54.436 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw= 2022-09-03T05:59:54.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw=', 'Content-Length': '0'} 2022-09-03T05:59:54.440 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cf22bb82a601602d-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.441 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.442 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.442 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.443 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.443 DEBUG:boto:Method: GET 2022-09-03T05:59:54.444 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.445 DEBUG:boto:Data: 2022-09-03T05:59:54.445 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.446 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.446 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.447 DEBUG:boto:Params: {} 2022-09-03T05:59:54.447 DEBUG:boto:Token: None 2022-09-03T05:59:54.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.452 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf2a506d09666b7e-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.453 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.454 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.454 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.455 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.455 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.456 DEBUG:boto:Method: GET 2022-09-03T05:59:54.456 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.457 DEBUG:boto:Data: 2022-09-03T05:59:54.457 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.458 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.459 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.459 DEBUG:boto:Params: {} 2022-09-03T05:59:54.460 DEBUG:boto:Token: None 2022-09-03T05:59:54.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.464 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c9f6912b0d925714-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.465 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.466 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.466 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.467 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.468 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T05:59:54.468 DEBUG:boto:Data: 2022-09-03T05:59:54.469 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.470 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.470 DEBUG:boto:Params: {} 2022-09-03T05:59:54.471 DEBUG:boto:Token: None 2022-09-03T05:59:54.472 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1 2022-09-03T05:59:54.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u/wPhMELQuL8ihPjrm66+mqDR/I= 2022-09-03T05:59:54.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u/wPhMELQuL8ihPjrm66+mqDR/I=', 'Content-Length': '0'} 2022-09-03T05:59:54.476 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002093e42b18829a24-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.476 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.477 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.477 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.478 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T05:59:54.479 DEBUG:boto:Data: 2022-09-03T05:59:54.479 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.480 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.480 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.481 DEBUG:boto:Params: {} 2022-09-03T05:59:54.481 DEBUG:boto:Token: None 2022-09-03T05:59:54.482 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1 2022-09-03T05:59:54.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u/wPhMELQuL8ihPjrm66+mqDR/I= 2022-09-03T05:59:54.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u/wPhMELQuL8ihPjrm66+mqDR/I=', 'Content-Length': '0'} 2022-09-03T05:59:54.485 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000897ca6b28758e64-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.486 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:54.487 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.487 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.488 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.488 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.489 DEBUG:boto:Method: GET 2022-09-03T05:59:54.490 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.490 DEBUG:boto:Data: 2022-09-03T05:59:54.491 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.492 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.492 DEBUG:boto:Params: {} 2022-09-03T05:59:54.493 DEBUG:boto:Token: None 2022-09-03T05:59:54.494 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw= 2022-09-03T05:59:54.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw=', 'Content-Length': '0'} 2022-09-03T05:59:54.497 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000b3e69352fe0d580-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.498 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.499 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:54.499 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.500 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.500 DEBUG:boto:Method: GET 2022-09-03T05:59:54.501 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.501 DEBUG:boto:Data: 2022-09-03T05:59:54.502 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.503 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.503 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.504 DEBUG:boto:Params: {} 2022-09-03T05:59:54.504 DEBUG:boto:Token: None 2022-09-03T05:59:54.505 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T05:59:54.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw= 2022-09-03T05:59:54.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6vAJd2fwkqWUoq65sWUwfpKUvKw=', 'Content-Length': '0'} 2022-09-03T05:59:54.508 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e90ce8abf8cf06e7-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.509 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.510 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.510 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.511 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.511 DEBUG:boto:Method: GET 2022-09-03T05:59:54.512 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.512 DEBUG:boto:Data: 2022-09-03T05:59:54.513 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.514 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.515 DEBUG:boto:Params: {} 2022-09-03T05:59:54.515 DEBUG:boto:Token: None 2022-09-03T05:59:54.516 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.519 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f8f87b409ee9b54e-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.520 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.521 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.521 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.522 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.522 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.523 DEBUG:boto:Method: GET 2022-09-03T05:59:54.524 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T05:59:54.524 DEBUG:boto:Data: 2022-09-03T05:59:54.525 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.525 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.526 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.526 DEBUG:boto:Params: {} 2022-09-03T05:59:54.527 DEBUG:boto:Token: None 2022-09-03T05:59:54.528 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.529 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.531 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000078a94bb182ad79d1-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.532 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.533 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:54.533 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.534 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.535 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.535 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.536 DEBUG:boto:Method: GET 2022-09-03T05:59:54.536 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.537 DEBUG:boto:Data: 2022-09-03T05:59:54.537 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.538 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.539 DEBUG:boto:Params: {} 2022-09-03T05:59:54.540 DEBUG:boto:Token: None 2022-09-03T05:59:54.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE= 2022-09-03T05:59:54.542 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE=', 'Content-Length': '0'} 2022-09-03T05:59:54.544 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008c7791db4a3595ed-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.590 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.590 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.591 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.592 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.592 DEBUG:boto:Method: GET 2022-09-03T05:59:54.593 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.593 DEBUG:boto:Data: 2022-09-03T05:59:54.594 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.594 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.595 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.595 DEBUG:boto:Params: {} 2022-09-03T05:59:54.596 DEBUG:boto:Token: None 2022-09-03T05:59:54.597 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE= 2022-09-03T05:59:54.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE=', 'Content-Length': '0'} 2022-09-03T05:59:54.600 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002e18c8af01dc422a-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.601 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.601 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.602 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.602 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.603 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.603 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.604 DEBUG:boto:Data: 2022-09-03T05:59:54.604 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.605 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.606 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.606 DEBUG:boto:Params: {} 2022-09-03T05:59:54.607 DEBUG:boto:Token: None 2022-09-03T05:59:54.607 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JpwqxkiZoCDSOx+KouAJQHa6Gtk= 2022-09-03T05:59:54.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JpwqxkiZoCDSOx+KouAJQHa6Gtk=', 'Content-Length': '0'} 2022-09-03T05:59:54.611 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009551573049236a39-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.612 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.612 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.613 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.613 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.614 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.614 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.615 DEBUG:boto:Data: 2022-09-03T05:59:54.615 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.616 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.617 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.618 DEBUG:boto:Params: {} 2022-09-03T05:59:54.619 DEBUG:boto:Token: None 2022-09-03T05:59:54.620 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JpwqxkiZoCDSOx+KouAJQHa6Gtk= 2022-09-03T05:59:54.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JpwqxkiZoCDSOx+KouAJQHa6Gtk=', 'Content-Length': '0'} 2022-09-03T05:59:54.625 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007b3e45087d55149d-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.626 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:54.627 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.628 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.629 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.630 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.631 DEBUG:boto:Method: GET 2022-09-03T05:59:54.632 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.632 DEBUG:boto:Data: 2022-09-03T05:59:54.633 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.634 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.635 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.636 DEBUG:boto:Params: {} 2022-09-03T05:59:54.637 DEBUG:boto:Token: None 2022-09-03T05:59:54.638 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.639 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE= 2022-09-03T05:59:54.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE=', 'Content-Length': '0'} 2022-09-03T05:59:54.644 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ff485d8f52cf51f8-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.645 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.646 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.647 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.648 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.649 DEBUG:boto:Method: GET 2022-09-03T05:59:54.649 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.650 DEBUG:boto:Data: 2022-09-03T05:59:54.650 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.651 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.652 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.652 DEBUG:boto:Params: {} 2022-09-03T05:59:54.653 DEBUG:boto:Token: None 2022-09-03T05:59:54.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.654 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE= 2022-09-03T05:59:54.655 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE=', 'Content-Length': '0'} 2022-09-03T05:59:54.657 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000026b02b6d5525ff7c-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.658 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.658 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.659 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.659 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.660 DEBUG:boto:Method: GET 2022-09-03T05:59:54.660 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.661 DEBUG:boto:Data: 2022-09-03T05:59:54.661 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.663 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.663 DEBUG:boto:Params: {} 2022-09-03T05:59:54.664 DEBUG:boto:Token: None 2022-09-03T05:59:54.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.668 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f4742036162376d-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.669 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.670 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.670 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.671 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.671 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.672 DEBUG:boto:Method: GET 2022-09-03T05:59:54.672 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.673 DEBUG:boto:Data: 2022-09-03T05:59:54.673 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.674 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.674 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.675 DEBUG:boto:Params: {} 2022-09-03T05:59:54.676 DEBUG:boto:Token: None 2022-09-03T05:59:54.676 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.677 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.680 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000edbc90dfb6a80518-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.681 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.682 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.682 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.683 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.683 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T05:59:54.684 DEBUG:boto:Data: 2022-09-03T05:59:54.684 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.685 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.685 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.686 DEBUG:boto:Params: {} 2022-09-03T05:59:54.686 DEBUG:boto:Token: None 2022-09-03T05:59:54.687 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2 2022-09-03T05:59:54.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oTX4Lu0JjKxztcT2XSQfKS/0Ygs= 2022-09-03T05:59:54.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oTX4Lu0JjKxztcT2XSQfKS/0Ygs=', 'Content-Length': '0'} 2022-09-03T05:59:54.690 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ce3adcbcbbfba803-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.691 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.692 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.692 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.693 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T05:59:54.693 DEBUG:boto:Data: 2022-09-03T05:59:54.694 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.694 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.695 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.695 DEBUG:boto:Params: {} 2022-09-03T05:59:54.696 DEBUG:boto:Token: None 2022-09-03T05:59:54.696 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2 2022-09-03T05:59:54.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oTX4Lu0JjKxztcT2XSQfKS/0Ygs= 2022-09-03T05:59:54.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oTX4Lu0JjKxztcT2XSQfKS/0Ygs=', 'Content-Length': '0'} 2022-09-03T05:59:54.700 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d784fccc59d1262b-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.701 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:54.701 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.702 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.702 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.703 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.703 DEBUG:boto:Method: GET 2022-09-03T05:59:54.704 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.704 DEBUG:boto:Data: 2022-09-03T05:59:54.705 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.706 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.706 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.707 DEBUG:boto:Params: {} 2022-09-03T05:59:54.707 DEBUG:boto:Token: None 2022-09-03T05:59:54.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE= 2022-09-03T05:59:54.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE=', 'Content-Length': '0'} 2022-09-03T05:59:54.711 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a7d3d29a6802a97f-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.758 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.759 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:54.759 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.760 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.760 DEBUG:boto:Method: GET 2022-09-03T05:59:54.761 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.761 DEBUG:boto:Data: 2022-09-03T05:59:54.762 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.762 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.763 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.764 DEBUG:boto:Params: {} 2022-09-03T05:59:54.764 DEBUG:boto:Token: None 2022-09-03T05:59:54.765 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T05:59:54.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE= 2022-09-03T05:59:54.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yNQvRuK7nyaeA22gWskgDDL3qdE=', 'Content-Length': '0'} 2022-09-03T05:59:54.768 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006de7235e0beb0c25-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.769 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.770 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.770 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.771 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.771 DEBUG:boto:Method: GET 2022-09-03T05:59:54.772 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.772 DEBUG:boto:Data: 2022-09-03T05:59:54.773 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.773 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.774 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.774 DEBUG:boto:Params: {} 2022-09-03T05:59:54.775 DEBUG:boto:Token: None 2022-09-03T05:59:54.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.779 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000e85be2022f3a5cb-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.780 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.781 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.781 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.782 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.783 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.783 DEBUG:boto:Method: GET 2022-09-03T05:59:54.784 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T05:59:54.784 DEBUG:boto:Data: 2022-09-03T05:59:54.785 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.785 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.786 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.786 DEBUG:boto:Params: {} 2022-09-03T05:59:54.787 DEBUG:boto:Token: None 2022-09-03T05:59:54.787 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.792 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001ee9f07025a97361-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.792 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.793 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:54.794 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.794 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.795 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.795 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.796 DEBUG:boto:Method: GET 2022-09-03T05:59:54.796 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.797 DEBUG:boto:Data: 2022-09-03T05:59:54.797 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.798 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.799 DEBUG:boto:Params: {} 2022-09-03T05:59:54.800 DEBUG:boto:Token: None 2022-09-03T05:59:54.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w= 2022-09-03T05:59:54.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w=', 'Content-Length': '0'} 2022-09-03T05:59:54.804 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f4cf6b5b67e8a832-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.805 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.805 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.806 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.806 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.807 DEBUG:boto:Method: GET 2022-09-03T05:59:54.808 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.808 DEBUG:boto:Data: 2022-09-03T05:59:54.809 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.809 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.810 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.810 DEBUG:boto:Params: {} 2022-09-03T05:59:54.811 DEBUG:boto:Token: None 2022-09-03T05:59:54.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w= 2022-09-03T05:59:54.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w=', 'Content-Length': '0'} 2022-09-03T05:59:54.815 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c14109b0a4c5753c-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.816 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.817 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.817 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.818 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.818 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.819 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.819 DEBUG:boto:Data: 2022-09-03T05:59:54.820 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.821 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.821 DEBUG:boto:Params: {} 2022-09-03T05:59:54.822 DEBUG:boto:Token: None 2022-09-03T05:59:54.823 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cDa8hbeFqF6uLSX9uC3yaMOCiC0= 2022-09-03T05:59:54.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cDa8hbeFqF6uLSX9uC3yaMOCiC0=', 'Content-Length': '0'} 2022-09-03T05:59:54.827 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000251b18ae9777174c-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.828 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.828 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.829 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.829 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.830 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.830 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.831 DEBUG:boto:Data: 2022-09-03T05:59:54.832 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.833 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.833 DEBUG:boto:Params: {} 2022-09-03T05:59:54.834 DEBUG:boto:Token: None 2022-09-03T05:59:54.835 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cDa8hbeFqF6uLSX9uC3yaMOCiC0= 2022-09-03T05:59:54.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cDa8hbeFqF6uLSX9uC3yaMOCiC0=', 'Content-Length': '0'} 2022-09-03T05:59:54.838 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c35c9f725e60c5d4-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.839 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:54.839 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.840 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.840 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.841 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.842 DEBUG:boto:Method: GET 2022-09-03T05:59:54.843 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.843 DEBUG:boto:Data: 2022-09-03T05:59:54.844 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.844 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.845 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.845 DEBUG:boto:Params: {} 2022-09-03T05:59:54.846 DEBUG:boto:Token: None 2022-09-03T05:59:54.846 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.847 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w= 2022-09-03T05:59:54.848 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w=', 'Content-Length': '0'} 2022-09-03T05:59:54.850 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005c2a4a3bc8378e7a-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.894 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.895 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.896 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.897 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.898 DEBUG:boto:Method: GET 2022-09-03T05:59:54.899 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.900 DEBUG:boto:Data: 2022-09-03T05:59:54.901 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.901 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.902 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.903 DEBUG:boto:Params: {} 2022-09-03T05:59:54.903 DEBUG:boto:Token: None 2022-09-03T05:59:54.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w= 2022-09-03T05:59:54.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w=', 'Content-Length': '0'} 2022-09-03T05:59:54.907 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000003a0ab1643fdfbba-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.908 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.909 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.909 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.910 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.910 DEBUG:boto:Method: GET 2022-09-03T05:59:54.911 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.911 DEBUG:boto:Data: 2022-09-03T05:59:54.912 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.912 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.913 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.914 DEBUG:boto:Params: {} 2022-09-03T05:59:54.914 DEBUG:boto:Token: None 2022-09-03T05:59:54.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.919 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fafc7ecf41e4aa10-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.919 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.920 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.921 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.921 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.922 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.922 DEBUG:boto:Method: GET 2022-09-03T05:59:54.923 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.924 DEBUG:boto:Data: 2022-09-03T05:59:54.924 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.925 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.926 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.926 DEBUG:boto:Params: {} 2022-09-03T05:59:54.927 DEBUG:boto:Token: None 2022-09-03T05:59:54.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.933 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eeb5ab7a774707e6-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.934 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.936 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.936 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.937 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.937 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T05:59:54.938 DEBUG:boto:Data: 2022-09-03T05:59:54.939 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.940 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.940 DEBUG:boto:Params: {} 2022-09-03T05:59:54.941 DEBUG:boto:Token: None 2022-09-03T05:59:54.941 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3 2022-09-03T05:59:54.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z/n5T/L96NP5FOJDbjALsIO7rjw= 2022-09-03T05:59:54.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z/n5T/L96NP5FOJDbjALsIO7rjw=', 'Content-Length': '0'} 2022-09-03T05:59:54.945 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ed6ed5e9d9845952-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.946 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.946 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.947 DEBUG:boto:Method: HEAD 2022-09-03T05:59:54.947 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T05:59:54.948 DEBUG:boto:Data: 2022-09-03T05:59:54.948 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.949 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.950 DEBUG:boto:Params: {} 2022-09-03T05:59:54.951 DEBUG:boto:Token: None 2022-09-03T05:59:54.952 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3 2022-09-03T05:59:54.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z/n5T/L96NP5FOJDbjALsIO7rjw= 2022-09-03T05:59:54.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z/n5T/L96NP5FOJDbjALsIO7rjw=', 'Content-Length': '0'} 2022-09-03T05:59:54.957 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003705a7434c3a9fbf-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.958 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:54.959 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.959 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.960 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.960 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.961 DEBUG:boto:Method: GET 2022-09-03T05:59:54.961 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.962 DEBUG:boto:Data: 2022-09-03T05:59:54.962 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.963 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.964 DEBUG:boto:Params: {} 2022-09-03T05:59:54.965 DEBUG:boto:Token: None 2022-09-03T05:59:54.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w= 2022-09-03T05:59:54.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w=', 'Content-Length': '0'} 2022-09-03T05:59:54.969 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004121a1690952acc1-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.970 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.970 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:54.971 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.971 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.972 DEBUG:boto:Method: GET 2022-09-03T05:59:54.973 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.973 DEBUG:boto:Data: 2022-09-03T05:59:54.974 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.974 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.975 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.975 DEBUG:boto:Params: {} 2022-09-03T05:59:54.976 DEBUG:boto:Token: None 2022-09-03T05:59:54.977 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T05:59:54.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w= 2022-09-03T05:59:54.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a9nOdmtNmcrSx3H1UC+8ZFk0q4w=', 'Content-Length': '0'} 2022-09-03T05:59:54.980 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:14 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b5dd524bd31975b4-006312ed5a-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.981 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.982 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.982 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.983 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.983 DEBUG:boto:Method: GET 2022-09-03T05:59:54.984 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.984 DEBUG:boto:Data: 2022-09-03T05:59:54.985 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.986 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:54.986 DEBUG:boto:Port: 8001 2022-09-03T05:59:54.987 DEBUG:boto:Params: {} 2022-09-03T05:59:54.988 DEBUG:boto:Token: None 2022-09-03T05:59:54.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:54 GMT /avysmw-17/ 2022-09-03T05:59:54.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY= 2022-09-03T05:59:54.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVZ61/C/PYqjlUS10fzLF4IWUzY=', 'Content-Length': '0'} 2022-09-03T05:59:54.992 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d774d2220f4f9151-006312ed5a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:54.993 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:54.994 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:54.994 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:54.995 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.995 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.996 DEBUG:boto:Method: GET 2022-09-03T05:59:54.996 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T05:59:54.997 DEBUG:boto:Data: 2022-09-03T05:59:54.997 DEBUG:boto:Headers: {} 2022-09-03T05:59:54.998 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:54.998 DEBUG:boto:Port: 8000 2022-09-03T05:59:54.999 DEBUG:boto:Params: {} 2022-09-03T05:59:55.000 DEBUG:boto:Token: None 2022-09-03T05:59:55.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:55 GMT /avysmw-17/ 2022-09-03T05:59:55.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GohfzKAKyStCeB2TVjo8OdUeh/8= 2022-09-03T05:59:55.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GohfzKAKyStCeB2TVjo8OdUeh/8=', 'Content-Length': '0'} 2022-09-03T05:59:55.004 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001747414030fd37ac-006312ed5b-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.004 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:55.005 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-17 zones={test-zone3, test-zone2} 2022-09-03T05:59:55.006 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-18 2022-09-03T05:59:55.006 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.007 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.008 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:55.071 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:55.072 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:55.121 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:55.118+0000 7feaebb28600 1 bucket sync caught up with source: 2022-09-03T05:59:55.121 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.26.5, , 00000000003.16.9, , 00000000007.25.9, , , , , ] 2022-09-03T05:59:55.121 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.26.5, , 00000000003.16.9, , 00000000007.25.9, , , , , ] 2022-09-03T05:59:55.122 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:55.118+0000 7feaebb28600 0 bucket checkpoint complete 2022-09-03T05:59:55.138 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-18 2022-09-03T05:59:55.139 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.140 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.140 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:55.206 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:55.207 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:55.247 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:55.244+0000 7f5e759ba600 1 bucket sync caught up with source: 2022-09-03T05:59:55.247 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000010.26.5, , 00000000003.16.9, , 00000000007.25.9, , , , , ] 2022-09-03T05:59:55.247 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000010.26.5, , 00000000003.16.9, , 00000000007.25.9, , , , , ] 2022-09-03T05:59:55.248 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:55.244+0000 7f5e759ba600 0 bucket checkpoint complete 2022-09-03T05:59:55.260 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-18 2022-09-03T05:59:55.261 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.263 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.263 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:55.321 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:55.321 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:55.359 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:55.357+0000 7f72e741c600 1 bucket sync caught up with source: 2022-09-03T05:59:55.359 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.26.5, , 00000000003.16.9, , 00000000007.25.9, , , , , ] 2022-09-03T05:59:55.360 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000010.26.5, , 00000000003.16.9, , 00000000007.25.9, , , , , ] 2022-09-03T05:59:55.360 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:55.357+0000 7f72e741c600 0 bucket checkpoint complete 2022-09-03T05:59:55.371 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-18 2022-09-03T05:59:55.372 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.373 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.374 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:55.435 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:55.436 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:55.479 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:55.476+0000 7faf18cd4600 1 bucket sync caught up with source: 2022-09-03T05:59:55.480 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.28.9, , 00000000010.52.5, , 00000000007.44.9, , , , , ] 2022-09-03T05:59:55.480 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.28.9, , 00000000010.52.5, , 00000000007.44.9, , , , , ] 2022-09-03T05:59:55.480 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:55.476+0000 7faf18cd4600 0 bucket checkpoint complete 2022-09-03T05:59:55.494 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-18 2022-09-03T05:59:55.495 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.495 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.497 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:55.554 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:55.555 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:55.595 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:55.592+0000 7f32d95c5600 1 bucket sync caught up with source: 2022-09-03T05:59:55.596 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.28.9, , 00000000010.52.5, , 00000000007.44.9, , , , , ] 2022-09-03T05:59:55.596 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.28.9, , 00000000010.52.5, , 00000000007.44.9, , , , , ] 2022-09-03T05:59:55.597 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:55.592+0000 7f32d95c5600 0 bucket checkpoint complete 2022-09-03T05:59:55.618 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-18 2022-09-03T05:59:55.619 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-18 2022-09-03T05:59:55.620 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.621 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.622 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:55.681 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:55.681 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:55.725 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:55.723+0000 7f2a72da6600 1 bucket sync caught up with source: 2022-09-03T05:59:55.726 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.35.9, , 00000000003.41.9, , 00000000014.69.5, , , , , ] 2022-09-03T05:59:55.726 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.35.9, , 00000000003.41.9, , 00000000014.69.5, , , , , ] 2022-09-03T05:59:55.726 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T05:59:55.723+0000 7f2a72da6600 0 bucket checkpoint complete 2022-09-03T05:59:55.738 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-18 2022-09-03T05:59:55.739 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.739 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:55.740 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:55.797 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:55.797 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:55.836 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:55.834+0000 7f0443f1b600 1 bucket sync caught up with source: 2022-09-03T05:59:55.837 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000003.35.9, , 00000000003.41.9, , 00000000014.69.5, , , , , ] 2022-09-03T05:59:55.838 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000003.35.9, , 00000000003.41.9, , 00000000014.69.5, , , , , ] 2022-09-03T05:59:55.838 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:55.834+0000 7f0443f1b600 0 bucket checkpoint complete 2022-09-03T05:59:55.857 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-18 2022-09-03T05:59:55.857 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-18 zones={test-zone2, test-zone1} 2022-09-03T05:59:55.858 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:55.858 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:55.859 DEBUG:boto:Method: HEAD 2022-09-03T05:59:55.859 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:59:55.860 DEBUG:boto:Data: 2022-09-03T05:59:55.860 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.861 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:55.862 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.862 DEBUG:boto:Params: {} 2022-09-03T05:59:55.863 DEBUG:boto:Token: None 2022-09-03T05:59:55.863 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/ 2022-09-03T05:59:55.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VpYs48ugKTiVBNjAYA/jSxpJWkU= 2022-09-03T05:59:55.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VpYs48ugKTiVBNjAYA/jSxpJWkU=', 'Content-Length': '0'} 2022-09-03T05:59:55.867 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000852d8621cace3e54-006312ed5b-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.868 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:55.868 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:55.869 DEBUG:boto:Method: HEAD 2022-09-03T05:59:55.870 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:59:55.870 DEBUG:boto:Data: 2022-09-03T05:59:55.871 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:55.872 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.873 DEBUG:boto:Params: {} 2022-09-03T05:59:55.873 DEBUG:boto:Token: None 2022-09-03T05:59:55.874 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/ 2022-09-03T05:59:55.875 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VpYs48ugKTiVBNjAYA/jSxpJWkU= 2022-09-03T05:59:55.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VpYs48ugKTiVBNjAYA/jSxpJWkU=', 'Content-Length': '0'} 2022-09-03T05:59:55.878 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000078723305c1209586-006312ed5b-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.879 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:59:55.879 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:55.880 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:55.881 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.881 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.882 DEBUG:boto:Method: GET 2022-09-03T05:59:55.883 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.883 DEBUG:boto:Data: 2022-09-03T05:59:55.884 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.884 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:55.885 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.885 DEBUG:boto:Params: {} 2022-09-03T05:59:55.886 DEBUG:boto:Token: None 2022-09-03T05:59:55.887 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/?versions 2022-09-03T05:59:55.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FioEeVWyRzHWMAZyzGyPg6eyO0= 2022-09-03T05:59:55.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FioEeVWyRzHWMAZyzGyPg6eyO0=', 'Content-Length': '0'} 2022-09-03T05:59:55.891 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000961d199ec8882a7b-006312ed5b-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.891 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:55.893 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:55.894 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:55.894 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:55.895 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:55.896 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:59:55.897 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:55.898 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:55.899 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.900 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.901 DEBUG:boto:Method: GET 2022-09-03T05:59:55.902 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.903 DEBUG:boto:Data: 2022-09-03T05:59:55.904 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:55.906 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.906 DEBUG:boto:Params: {} 2022-09-03T05:59:55.907 DEBUG:boto:Token: None 2022-09-03T05:59:55.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/?versions 2022-09-03T05:59:55.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FioEeVWyRzHWMAZyzGyPg6eyO0= 2022-09-03T05:59:55.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FioEeVWyRzHWMAZyzGyPg6eyO0=', 'Content-Length': '0'} 2022-09-03T05:59:55.911 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000f5f90ff58d2f9c6-006312ed5b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.912 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:55.913 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:55.913 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:55.914 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:55.914 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:55.915 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:55.915 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:55.916 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.917 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.917 DEBUG:boto:Method: GET 2022-09-03T05:59:55.918 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.918 DEBUG:boto:Data: 2022-09-03T05:59:55.919 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.919 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:55.920 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.921 DEBUG:boto:Params: {} 2022-09-03T05:59:55.922 DEBUG:boto:Token: None 2022-09-03T05:59:55.923 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/?versions 2022-09-03T05:59:55.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FioEeVWyRzHWMAZyzGyPg6eyO0= 2022-09-03T05:59:55.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FioEeVWyRzHWMAZyzGyPg6eyO0=', 'Content-Length': '0'} 2022-09-03T05:59:55.927 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001ba68071b1dbd4f7-006312ed5b-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.928 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:55.929 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:55.929 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:55.930 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.930 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.931 DEBUG:boto:Method: GET 2022-09-03T05:59:55.931 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:55.932 DEBUG:boto:Data: 2022-09-03T05:59:55.933 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:55.934 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.934 DEBUG:boto:Params: {} 2022-09-03T05:59:55.935 DEBUG:boto:Token: None 2022-09-03T05:59:55.936 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/?versions 2022-09-03T05:59:55.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FioEeVWyRzHWMAZyzGyPg6eyO0= 2022-09-03T05:59:55.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FioEeVWyRzHWMAZyzGyPg6eyO0=', 'Content-Length': '0'} 2022-09-03T05:59:55.939 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000358c739159bd96c8-006312ed5b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.940 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:55.942 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:55.943 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:55.944 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:55.944 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.945 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.946 DEBUG:boto:Method: GET 2022-09-03T05:59:55.947 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.949 DEBUG:boto:Data: 2022-09-03T05:59:55.949 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.950 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:55.951 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.952 DEBUG:boto:Params: {} 2022-09-03T05:59:55.953 DEBUG:boto:Token: None 2022-09-03T05:59:55.954 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:72OSxz58+kmDOilaKlmzZTCrRSg= 2022-09-03T05:59:55.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:72OSxz58+kmDOilaKlmzZTCrRSg=', 'Content-Length': '0'} 2022-09-03T05:59:55.957 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009acc0ef4aaa61c25-006312ed5b-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.958 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:55.959 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:55.959 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.960 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.960 DEBUG:boto:Method: GET 2022-09-03T05:59:55.961 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.962 DEBUG:boto:Data: 2022-09-03T05:59:55.962 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:55.963 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.964 DEBUG:boto:Params: {} 2022-09-03T05:59:55.964 DEBUG:boto:Token: None 2022-09-03T05:59:55.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:72OSxz58+kmDOilaKlmzZTCrRSg= 2022-09-03T05:59:55.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:72OSxz58+kmDOilaKlmzZTCrRSg=', 'Content-Length': '0'} 2022-09-03T05:59:55.968 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000066c5ef087e8a239e-006312ed5b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.969 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:55.970 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:55.970 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.971 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.971 DEBUG:boto:Method: HEAD 2022-09-03T05:59:55.972 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.972 DEBUG:boto:Data: 2022-09-03T05:59:55.973 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.973 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:55.974 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.975 DEBUG:boto:Params: {} 2022-09-03T05:59:55.976 DEBUG:boto:Token: None 2022-09-03T05:59:55.976 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bymT95Fg9HU0tDgQVpoV8/WQhRc= 2022-09-03T05:59:55.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bymT95Fg9HU0tDgQVpoV8/WQhRc=', 'Content-Length': '0'} 2022-09-03T05:59:55.979 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000945ac8d55705b1ee-006312ed5b-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.980 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:55.981 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:55.981 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.982 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.983 DEBUG:boto:Method: HEAD 2022-09-03T05:59:55.983 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.984 DEBUG:boto:Data: 2022-09-03T05:59:55.984 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:55.986 DEBUG:boto:Port: 8000 2022-09-03T05:59:55.987 DEBUG:boto:Params: {} 2022-09-03T05:59:55.988 DEBUG:boto:Token: None 2022-09-03T05:59:55.989 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:55 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bymT95Fg9HU0tDgQVpoV8/WQhRc= 2022-09-03T05:59:55.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bymT95Fg9HU0tDgQVpoV8/WQhRc=', 'Content-Length': '0'} 2022-09-03T05:59:55.993 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e33db00140e91e83-006312ed5b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:55.994 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:55.994 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:55.995 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:55.995 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.996 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.996 DEBUG:boto:Method: GET 2022-09-03T05:59:55.997 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:55.997 DEBUG:boto:Data: 2022-09-03T05:59:55.998 DEBUG:boto:Headers: {} 2022-09-03T05:59:55.999 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.000 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.001 DEBUG:boto:Params: {} 2022-09-03T05:59:56.002 DEBUG:boto:Token: None 2022-09-03T05:59:56.003 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE= 2022-09-03T05:59:56.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE=', 'Content-Length': '0'} 2022-09-03T05:59:56.007 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000081b2144b1dde8f41-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.008 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:56.009 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:56.009 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.010 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.010 DEBUG:boto:Method: GET 2022-09-03T05:59:56.011 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.012 DEBUG:boto:Data: 2022-09-03T05:59:56.012 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.013 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.014 DEBUG:boto:Params: {} 2022-09-03T05:59:56.015 DEBUG:boto:Token: None 2022-09-03T05:59:56.015 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.016 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE= 2022-09-03T05:59:56.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE=', 'Content-Length': '0'} 2022-09-03T05:59:56.019 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000bc71fb464c8a845d-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.020 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.020 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.021 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:56.021 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:56.022 DEBUG:boto:Method: GET 2022-09-03T05:59:56.023 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:56.023 DEBUG:boto:Data: 2022-09-03T05:59:56.024 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.024 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.025 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.025 DEBUG:boto:Params: {} 2022-09-03T05:59:56.026 DEBUG:boto:Token: None 2022-09-03T05:59:56.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.030 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017bd94b75fe73d8b-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.074 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.075 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.076 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.076 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:56.077 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:56.077 DEBUG:boto:Method: GET 2022-09-03T05:59:56.078 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:56.078 DEBUG:boto:Data: 2022-09-03T05:59:56.079 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.080 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.080 DEBUG:boto:Params: {} 2022-09-03T05:59:56.081 DEBUG:boto:Token: None 2022-09-03T05:59:56.082 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.085 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000010bea7fae7125261-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.086 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.087 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:56.088 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:56.088 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.089 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T05:59:56.090 DEBUG:boto:Data: 2022-09-03T05:59:56.090 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.091 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.091 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.092 DEBUG:boto:Params: {} 2022-09-03T05:59:56.093 DEBUG:boto:Token: None 2022-09-03T05:59:56.093 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy 2022-09-03T05:59:56.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjK6KRM+wFE7FqEh+ATuZx2dG2U= 2022-09-03T05:59:56.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjK6KRM+wFE7FqEh+ATuZx2dG2U=', 'Content-Length': '0'} 2022-09-03T05:59:56.097 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005f72e51ce27579e2-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.098 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:56.098 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:56.099 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.099 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T05:59:56.100 DEBUG:boto:Data: 2022-09-03T05:59:56.101 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.101 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.102 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.102 DEBUG:boto:Params: {} 2022-09-03T05:59:56.103 DEBUG:boto:Token: None 2022-09-03T05:59:56.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy 2022-09-03T05:59:56.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjK6KRM+wFE7FqEh+ATuZx2dG2U= 2022-09-03T05:59:56.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjK6KRM+wFE7FqEh+ATuZx2dG2U=', 'Content-Length': '0'} 2022-09-03T05:59:56.107 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000186a0120cd38913b-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.108 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:56.108 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:56.109 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:56.110 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.110 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.111 DEBUG:boto:Method: GET 2022-09-03T05:59:56.111 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.112 DEBUG:boto:Data: 2022-09-03T05:59:56.113 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.113 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.114 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.115 DEBUG:boto:Params: {} 2022-09-03T05:59:56.115 DEBUG:boto:Token: None 2022-09-03T05:59:56.116 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE= 2022-09-03T05:59:56.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE=', 'Content-Length': '0'} 2022-09-03T05:59:56.119 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000594f437c607399b1-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.121 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:56.122 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:56.123 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.124 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.124 DEBUG:boto:Method: GET 2022-09-03T05:59:56.125 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.126 DEBUG:boto:Data: 2022-09-03T05:59:56.127 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.128 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.129 DEBUG:boto:Params: {} 2022-09-03T05:59:56.129 DEBUG:boto:Token: None 2022-09-03T05:59:56.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE= 2022-09-03T05:59:56.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE=', 'Content-Length': '0'} 2022-09-03T05:59:56.134 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009c9b26f429cacc5b-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.135 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.136 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.137 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:56.138 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:56.139 DEBUG:boto:Method: GET 2022-09-03T05:59:56.140 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:56.141 DEBUG:boto:Data: 2022-09-03T05:59:56.142 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.143 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.144 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.144 DEBUG:boto:Params: {} 2022-09-03T05:59:56.145 DEBUG:boto:Token: None 2022-09-03T05:59:56.145 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.149 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0eebb722ef03737-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.150 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.151 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.152 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.152 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:56.153 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:56.153 DEBUG:boto:Method: GET 2022-09-03T05:59:56.154 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:56.154 DEBUG:boto:Data: 2022-09-03T05:59:56.155 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.156 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.157 DEBUG:boto:Params: {} 2022-09-03T05:59:56.158 DEBUG:boto:Token: None 2022-09-03T05:59:56.159 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.163 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fad8877fcff17e70-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.163 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.164 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:56.165 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.165 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.166 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.166 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.167 DEBUG:boto:Method: GET 2022-09-03T05:59:56.168 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.168 DEBUG:boto:Data: 2022-09-03T05:59:56.169 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.169 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.170 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.171 DEBUG:boto:Params: {} 2022-09-03T05:59:56.171 DEBUG:boto:Token: None 2022-09-03T05:59:56.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE= 2022-09-03T05:59:56.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE=', 'Content-Length': '0'} 2022-09-03T05:59:56.176 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000929c667469a59d1a-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.176 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.177 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.178 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.178 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.179 DEBUG:boto:Method: GET 2022-09-03T05:59:56.180 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.181 DEBUG:boto:Data: 2022-09-03T05:59:56.182 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.184 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.185 DEBUG:boto:Params: {} 2022-09-03T05:59:56.187 DEBUG:boto:Token: None 2022-09-03T05:59:56.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE= 2022-09-03T05:59:56.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE=', 'Content-Length': '0'} 2022-09-03T05:59:56.193 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005186eebb9d2c4eeb-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.195 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.196 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.196 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.197 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.198 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.199 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.200 DEBUG:boto:Data: 2022-09-03T05:59:56.201 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.201 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.202 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.202 DEBUG:boto:Params: {} 2022-09-03T05:59:56.203 DEBUG:boto:Token: None 2022-09-03T05:59:56.204 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gwfqp1PGoPvTXwq0Oaf0J103RK4= 2022-09-03T05:59:56.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gwfqp1PGoPvTXwq0Oaf0J103RK4=', 'Content-Length': '0'} 2022-09-03T05:59:56.206 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e7eb9506175ee5c2-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.207 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.208 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.208 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.209 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.209 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.210 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.210 DEBUG:boto:Data: 2022-09-03T05:59:56.211 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.211 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.212 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.212 DEBUG:boto:Params: {} 2022-09-03T05:59:56.213 DEBUG:boto:Token: None 2022-09-03T05:59:56.213 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gwfqp1PGoPvTXwq0Oaf0J103RK4= 2022-09-03T05:59:56.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gwfqp1PGoPvTXwq0Oaf0J103RK4=', 'Content-Length': '0'} 2022-09-03T05:59:56.217 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007374c6ff2197c161-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.217 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:56.218 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.218 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.219 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.220 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.220 DEBUG:boto:Method: GET 2022-09-03T05:59:56.221 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.221 DEBUG:boto:Data: 2022-09-03T05:59:56.222 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.222 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.223 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.223 DEBUG:boto:Params: {} 2022-09-03T05:59:56.224 DEBUG:boto:Token: None 2022-09-03T05:59:56.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE= 2022-09-03T05:59:56.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE=', 'Content-Length': '0'} 2022-09-03T05:59:56.228 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000a248d2ea2cfe6fe-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.229 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.229 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.230 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.230 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.231 DEBUG:boto:Method: GET 2022-09-03T05:59:56.232 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.232 DEBUG:boto:Data: 2022-09-03T05:59:56.233 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.233 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.234 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.234 DEBUG:boto:Params: {} 2022-09-03T05:59:56.235 DEBUG:boto:Token: None 2022-09-03T05:59:56.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE= 2022-09-03T05:59:56.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE=', 'Content-Length': '0'} 2022-09-03T05:59:56.239 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c72c67a9cbe51048-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.240 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.240 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.241 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.241 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.242 DEBUG:boto:Method: GET 2022-09-03T05:59:56.242 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.243 DEBUG:boto:Data: 2022-09-03T05:59:56.244 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.244 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.245 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.245 DEBUG:boto:Params: {} 2022-09-03T05:59:56.246 DEBUG:boto:Token: None 2022-09-03T05:59:56.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.250 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000024bc74f88e55a24f-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.251 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.252 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.253 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.253 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.254 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.254 DEBUG:boto:Method: GET 2022-09-03T05:59:56.255 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.255 DEBUG:boto:Data: 2022-09-03T05:59:56.256 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.256 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.257 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.258 DEBUG:boto:Params: {} 2022-09-03T05:59:56.258 DEBUG:boto:Token: None 2022-09-03T05:59:56.259 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.263 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c7a3d01288d73bc-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.263 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.264 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.265 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.265 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.266 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T05:59:56.266 DEBUG:boto:Data: 2022-09-03T05:59:56.267 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.267 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.268 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.269 DEBUG:boto:Params: {} 2022-09-03T05:59:56.269 DEBUG:boto:Token: None 2022-09-03T05:59:56.270 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1 2022-09-03T05:59:56.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V5S0AScwQkI34aM88ZraRU/RVf8= 2022-09-03T05:59:56.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V5S0AScwQkI34aM88ZraRU/RVf8=', 'Content-Length': '0'} 2022-09-03T05:59:56.274 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000863e419fa995ebd8-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.275 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.275 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.276 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.276 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T05:59:56.277 DEBUG:boto:Data: 2022-09-03T05:59:56.277 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.278 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.279 DEBUG:boto:Params: {} 2022-09-03T05:59:56.280 DEBUG:boto:Token: None 2022-09-03T05:59:56.281 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1 2022-09-03T05:59:56.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V5S0AScwQkI34aM88ZraRU/RVf8= 2022-09-03T05:59:56.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V5S0AScwQkI34aM88ZraRU/RVf8=', 'Content-Length': '0'} 2022-09-03T05:59:56.286 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003304dfeebe00bfb7-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.287 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:56.288 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.289 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.290 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.290 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.291 DEBUG:boto:Method: GET 2022-09-03T05:59:56.292 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.292 DEBUG:boto:Data: 2022-09-03T05:59:56.293 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.294 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.295 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.296 DEBUG:boto:Params: {} 2022-09-03T05:59:56.296 DEBUG:boto:Token: None 2022-09-03T05:59:56.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE= 2022-09-03T05:59:56.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE=', 'Content-Length': '0'} 2022-09-03T05:59:56.300 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002e2b87a61effdaa3-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.342 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.343 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:56.344 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.344 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.345 DEBUG:boto:Method: GET 2022-09-03T05:59:56.346 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.346 DEBUG:boto:Data: 2022-09-03T05:59:56.347 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.347 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.348 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.348 DEBUG:boto:Params: {} 2022-09-03T05:59:56.349 DEBUG:boto:Token: None 2022-09-03T05:59:56.350 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:56.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE= 2022-09-03T05:59:56.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G9sR6Qc2EItUZxEqs0SLAuy+hSE=', 'Content-Length': '0'} 2022-09-03T05:59:56.353 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d2c4db1aa9094dd7-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.354 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.355 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.355 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.356 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.356 DEBUG:boto:Method: GET 2022-09-03T05:59:56.357 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.358 DEBUG:boto:Data: 2022-09-03T05:59:56.358 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.359 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.359 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.360 DEBUG:boto:Params: {} 2022-09-03T05:59:56.361 DEBUG:boto:Token: None 2022-09-03T05:59:56.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.365 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a58fc4c06d41e3b-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.366 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.366 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.367 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.368 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.368 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.369 DEBUG:boto:Method: GET 2022-09-03T05:59:56.370 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:56.370 DEBUG:boto:Data: 2022-09-03T05:59:56.371 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.372 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.373 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.374 DEBUG:boto:Params: {} 2022-09-03T05:59:56.374 DEBUG:boto:Token: None 2022-09-03T05:59:56.375 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.379 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4a0d16ae1a8be56-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.380 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.381 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:56.382 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.383 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.384 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.384 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.385 DEBUG:boto:Method: GET 2022-09-03T05:59:56.387 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.388 DEBUG:boto:Data: 2022-09-03T05:59:56.389 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.390 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.391 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.391 DEBUG:boto:Params: {} 2022-09-03T05:59:56.393 DEBUG:boto:Token: None 2022-09-03T05:59:56.394 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E= 2022-09-03T05:59:56.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E=', 'Content-Length': '0'} 2022-09-03T05:59:56.398 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b1ec1468bcb2650b-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.399 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.400 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.401 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.402 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.403 DEBUG:boto:Method: GET 2022-09-03T05:59:56.404 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.405 DEBUG:boto:Data: 2022-09-03T05:59:56.406 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.407 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.407 DEBUG:boto:Params: {} 2022-09-03T05:59:56.408 DEBUG:boto:Token: None 2022-09-03T05:59:56.409 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E= 2022-09-03T05:59:56.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E=', 'Content-Length': '0'} 2022-09-03T05:59:56.412 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000030697970f7a2a29e-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.454 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.455 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.456 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.456 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.457 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.457 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.458 DEBUG:boto:Data: 2022-09-03T05:59:56.458 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.459 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.459 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.460 DEBUG:boto:Params: {} 2022-09-03T05:59:56.460 DEBUG:boto:Token: None 2022-09-03T05:59:56.461 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pdWVmdC0xLnfLwj3mRReOu8MwZ8= 2022-09-03T05:59:56.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pdWVmdC0xLnfLwj3mRReOu8MwZ8=', 'Content-Length': '0'} 2022-09-03T05:59:56.464 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a3363004683ea1c1-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.465 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.466 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.466 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.467 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.467 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.468 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.468 DEBUG:boto:Data: 2022-09-03T05:59:56.469 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.469 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.470 DEBUG:boto:Params: {} 2022-09-03T05:59:56.471 DEBUG:boto:Token: None 2022-09-03T05:59:56.471 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pdWVmdC0xLnfLwj3mRReOu8MwZ8= 2022-09-03T05:59:56.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pdWVmdC0xLnfLwj3mRReOu8MwZ8=', 'Content-Length': '0'} 2022-09-03T05:59:56.475 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ce1d9b2ebafb0ad9-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.475 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:56.476 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.477 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.478 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.479 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.480 DEBUG:boto:Method: GET 2022-09-03T05:59:56.480 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.481 DEBUG:boto:Data: 2022-09-03T05:59:56.481 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.482 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.482 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.483 DEBUG:boto:Params: {} 2022-09-03T05:59:56.484 DEBUG:boto:Token: None 2022-09-03T05:59:56.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E= 2022-09-03T05:59:56.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E=', 'Content-Length': '0'} 2022-09-03T05:59:56.487 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000654dc45713f4f64d-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.488 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.489 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.489 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.490 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.491 DEBUG:boto:Method: GET 2022-09-03T05:59:56.492 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.492 DEBUG:boto:Data: 2022-09-03T05:59:56.493 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.494 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.494 DEBUG:boto:Params: {} 2022-09-03T05:59:56.495 DEBUG:boto:Token: None 2022-09-03T05:59:56.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E= 2022-09-03T05:59:56.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E=', 'Content-Length': '0'} 2022-09-03T05:59:56.500 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008bdc4365a42ac6d7-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.501 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.502 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.502 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.503 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.504 DEBUG:boto:Method: GET 2022-09-03T05:59:56.504 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.505 DEBUG:boto:Data: 2022-09-03T05:59:56.506 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.507 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.508 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.509 DEBUG:boto:Params: {} 2022-09-03T05:59:56.509 DEBUG:boto:Token: None 2022-09-03T05:59:56.510 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.515 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000192b5ce77713ace2-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.516 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.517 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.517 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.518 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.518 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.519 DEBUG:boto:Method: GET 2022-09-03T05:59:56.519 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.520 DEBUG:boto:Data: 2022-09-03T05:59:56.520 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.521 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.521 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.522 DEBUG:boto:Params: {} 2022-09-03T05:59:56.523 DEBUG:boto:Token: None 2022-09-03T05:59:56.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.527 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002fb5352f849d3b3-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.527 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.528 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.529 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.529 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.530 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T05:59:56.531 DEBUG:boto:Data: 2022-09-03T05:59:56.531 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.531 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.532 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.532 DEBUG:boto:Params: {} 2022-09-03T05:59:56.533 DEBUG:boto:Token: None 2022-09-03T05:59:56.534 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2 2022-09-03T05:59:56.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:maKpmbV1g94L02GDa6OKnPbJrkU= 2022-09-03T05:59:56.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:maKpmbV1g94L02GDa6OKnPbJrkU=', 'Content-Length': '0'} 2022-09-03T05:59:56.537 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c89380dcf597b51c-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.538 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.538 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.539 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.539 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T05:59:56.540 DEBUG:boto:Data: 2022-09-03T05:59:56.540 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.541 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.542 DEBUG:boto:Params: {} 2022-09-03T05:59:56.542 DEBUG:boto:Token: None 2022-09-03T05:59:56.543 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2 2022-09-03T05:59:56.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:maKpmbV1g94L02GDa6OKnPbJrkU= 2022-09-03T05:59:56.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:maKpmbV1g94L02GDa6OKnPbJrkU=', 'Content-Length': '0'} 2022-09-03T05:59:56.546 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008170864cff428992-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.547 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:56.548 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.548 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.549 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.549 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.550 DEBUG:boto:Method: GET 2022-09-03T05:59:56.551 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.551 DEBUG:boto:Data: 2022-09-03T05:59:56.552 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.552 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.553 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.553 DEBUG:boto:Params: {} 2022-09-03T05:59:56.554 DEBUG:boto:Token: None 2022-09-03T05:59:56.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E= 2022-09-03T05:59:56.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E=', 'Content-Length': '0'} 2022-09-03T05:59:56.558 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ce8444afe1654f32-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.559 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.559 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:56.560 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.560 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.561 DEBUG:boto:Method: GET 2022-09-03T05:59:56.561 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.562 DEBUG:boto:Data: 2022-09-03T05:59:56.563 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.565 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.565 DEBUG:boto:Params: {} 2022-09-03T05:59:56.566 DEBUG:boto:Token: None 2022-09-03T05:59:56.566 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:56.567 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E= 2022-09-03T05:59:56.568 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aMJyBOXEY7vEVxLAAmsL4MqVR1E=', 'Content-Length': '0'} 2022-09-03T05:59:56.570 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000093ec0c2e0c797291-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.618 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.620 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.622 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.622 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.624 DEBUG:boto:Method: GET 2022-09-03T05:59:56.625 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.626 DEBUG:boto:Data: 2022-09-03T05:59:56.627 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.628 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.630 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.631 DEBUG:boto:Params: {} 2022-09-03T05:59:56.631 DEBUG:boto:Token: None 2022-09-03T05:59:56.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.636 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005721624044eec81d-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.637 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.638 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.639 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.639 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.640 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.640 DEBUG:boto:Method: GET 2022-09-03T05:59:56.641 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:56.642 DEBUG:boto:Data: 2022-09-03T05:59:56.642 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.643 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.644 DEBUG:boto:Params: {} 2022-09-03T05:59:56.644 DEBUG:boto:Token: None 2022-09-03T05:59:56.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.649 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bab1e68800670800-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.649 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.650 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:56.651 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.651 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.652 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.653 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.653 DEBUG:boto:Method: GET 2022-09-03T05:59:56.654 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.654 DEBUG:boto:Data: 2022-09-03T05:59:56.655 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.655 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.656 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.656 DEBUG:boto:Params: {} 2022-09-03T05:59:56.657 DEBUG:boto:Token: None 2022-09-03T05:59:56.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ= 2022-09-03T05:59:56.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ=', 'Content-Length': '0'} 2022-09-03T05:59:56.661 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003e00783aa9a9b4f3-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.662 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.662 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.663 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.664 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.664 DEBUG:boto:Method: GET 2022-09-03T05:59:56.665 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.665 DEBUG:boto:Data: 2022-09-03T05:59:56.666 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.667 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.668 DEBUG:boto:Params: {} 2022-09-03T05:59:56.668 DEBUG:boto:Token: None 2022-09-03T05:59:56.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ= 2022-09-03T05:59:56.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ=', 'Content-Length': '0'} 2022-09-03T05:59:56.672 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000081e56a08f19550bc-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.673 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.674 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.674 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.675 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.676 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.676 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.677 DEBUG:boto:Data: 2022-09-03T05:59:56.677 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.678 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.678 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.679 DEBUG:boto:Params: {} 2022-09-03T05:59:56.679 DEBUG:boto:Token: None 2022-09-03T05:59:56.680 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2zZkBcuYTwvKvqb+eV7bSRvHEwI= 2022-09-03T05:59:56.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2zZkBcuYTwvKvqb+eV7bSRvHEwI=', 'Content-Length': '0'} 2022-09-03T05:59:56.683 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009b51c10c3b6d9df9-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.684 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.684 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.685 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.685 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.686 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.686 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.687 DEBUG:boto:Data: 2022-09-03T05:59:56.688 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.689 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.689 DEBUG:boto:Params: {} 2022-09-03T05:59:56.690 DEBUG:boto:Token: None 2022-09-03T05:59:56.690 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2zZkBcuYTwvKvqb+eV7bSRvHEwI= 2022-09-03T05:59:56.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2zZkBcuYTwvKvqb+eV7bSRvHEwI=', 'Content-Length': '0'} 2022-09-03T05:59:56.694 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000025f4342cef4ac63c-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.694 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:56.695 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.696 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.696 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.697 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.697 DEBUG:boto:Method: GET 2022-09-03T05:59:56.698 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.699 DEBUG:boto:Data: 2022-09-03T05:59:56.699 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.700 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.700 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.701 DEBUG:boto:Params: {} 2022-09-03T05:59:56.701 DEBUG:boto:Token: None 2022-09-03T05:59:56.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ= 2022-09-03T05:59:56.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ=', 'Content-Length': '0'} 2022-09-03T05:59:56.706 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b13596442564fe9d-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.707 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.707 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.708 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.708 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.709 DEBUG:boto:Method: GET 2022-09-03T05:59:56.709 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.710 DEBUG:boto:Data: 2022-09-03T05:59:56.711 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.711 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.712 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.712 DEBUG:boto:Params: {} 2022-09-03T05:59:56.713 DEBUG:boto:Token: None 2022-09-03T05:59:56.713 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.714 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ= 2022-09-03T05:59:56.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ=', 'Content-Length': '0'} 2022-09-03T05:59:56.717 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bcd06eb8bceb7b57-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.718 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.718 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.719 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.719 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.720 DEBUG:boto:Method: GET 2022-09-03T05:59:56.720 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.721 DEBUG:boto:Data: 2022-09-03T05:59:56.721 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.722 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.722 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.723 DEBUG:boto:Params: {} 2022-09-03T05:59:56.724 DEBUG:boto:Token: None 2022-09-03T05:59:56.724 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.728 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5718a81ccb5fb01-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.729 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.730 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.730 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.731 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.731 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.732 DEBUG:boto:Method: GET 2022-09-03T05:59:56.733 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.733 DEBUG:boto:Data: 2022-09-03T05:59:56.734 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.735 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.735 DEBUG:boto:Params: {} 2022-09-03T05:59:56.736 DEBUG:boto:Token: None 2022-09-03T05:59:56.737 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.738 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.741 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001699450a76a2ad5d-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.741 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.742 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.743 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.743 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.744 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T05:59:56.744 DEBUG:boto:Data: 2022-09-03T05:59:56.745 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.745 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.746 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.746 DEBUG:boto:Params: {} 2022-09-03T05:59:56.747 DEBUG:boto:Token: None 2022-09-03T05:59:56.747 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3 2022-09-03T05:59:56.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vqipLGVI0YA3z10fe/uLfMmxNGk= 2022-09-03T05:59:56.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vqipLGVI0YA3z10fe/uLfMmxNGk=', 'Content-Length': '0'} 2022-09-03T05:59:56.751 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003b257a9bc694050f-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.752 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.752 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.753 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.753 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T05:59:56.754 DEBUG:boto:Data: 2022-09-03T05:59:56.754 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.755 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.756 DEBUG:boto:Params: {} 2022-09-03T05:59:56.756 DEBUG:boto:Token: None 2022-09-03T05:59:56.757 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3 2022-09-03T05:59:56.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vqipLGVI0YA3z10fe/uLfMmxNGk= 2022-09-03T05:59:56.758 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vqipLGVI0YA3z10fe/uLfMmxNGk=', 'Content-Length': '0'} 2022-09-03T05:59:56.760 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c21d1d52826f08f2-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.761 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:56.762 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.762 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.763 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.763 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.764 DEBUG:boto:Method: GET 2022-09-03T05:59:56.765 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.765 DEBUG:boto:Data: 2022-09-03T05:59:56.766 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.766 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.767 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.767 DEBUG:boto:Params: {} 2022-09-03T05:59:56.768 DEBUG:boto:Token: None 2022-09-03T05:59:56.768 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ= 2022-09-03T05:59:56.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ=', 'Content-Length': '0'} 2022-09-03T05:59:56.772 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d2bdba1381132cca-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.821 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.822 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:56.822 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.823 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.823 DEBUG:boto:Method: GET 2022-09-03T05:59:56.824 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.824 DEBUG:boto:Data: 2022-09-03T05:59:56.825 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.825 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.826 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.827 DEBUG:boto:Params: {} 2022-09-03T05:59:56.827 DEBUG:boto:Token: None 2022-09-03T05:59:56.828 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:56.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ= 2022-09-03T05:59:56.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/s6Hyr92dl/LRNqYeeiyodM/ZeQ=', 'Content-Length': '0'} 2022-09-03T05:59:56.832 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009c60fdba55e2335b-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.833 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.833 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.834 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.834 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.835 DEBUG:boto:Method: GET 2022-09-03T05:59:56.836 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.836 DEBUG:boto:Data: 2022-09-03T05:59:56.837 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.838 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:56.838 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.839 DEBUG:boto:Params: {} 2022-09-03T05:59:56.840 DEBUG:boto:Token: None 2022-09-03T05:59:56.840 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.844 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed9cec7efc651f93-006312ed5c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.845 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.846 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.846 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.847 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.847 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.848 DEBUG:boto:Method: GET 2022-09-03T05:59:56.849 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:56.849 DEBUG:boto:Data: 2022-09-03T05:59:56.850 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.851 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.852 DEBUG:boto:Params: {} 2022-09-03T05:59:56.853 DEBUG:boto:Token: None 2022-09-03T05:59:56.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs= 2022-09-03T05:59:56.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7KgmfIsA6GOTOD5jt1Wc/FCrCs=', 'Content-Length': '0'} 2022-09-03T05:59:56.858 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ade38e82b5b6b8c6-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.859 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.860 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-18 zones={test-zone2, test-zone1} 2022-09-03T05:59:56.860 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-18 zones={test-zone3, test-zone1} 2022-09-03T05:59:56.861 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.862 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.862 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.863 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:59:56.863 DEBUG:boto:Data: 2022-09-03T05:59:56.864 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:56.865 DEBUG:boto:Port: 8001 2022-09-03T05:59:56.865 DEBUG:boto:Params: {} 2022-09-03T05:59:56.866 DEBUG:boto:Token: None 2022-09-03T05:59:56.867 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:glgyKppp4ZXj7ModAugIfiWtt5U= 2022-09-03T05:59:56.868 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:glgyKppp4ZXj7ModAugIfiWtt5U=', 'Content-Length': '0'} 2022-09-03T05:59:56.871 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000008d8ea498e97a078-006312ed5c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.872 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.872 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.873 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.873 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:59:56.874 DEBUG:boto:Data: 2022-09-03T05:59:56.874 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.875 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.876 DEBUG:boto:Params: {} 2022-09-03T05:59:56.876 DEBUG:boto:Token: None 2022-09-03T05:59:56.877 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/ 2022-09-03T05:59:56.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:glgyKppp4ZXj7ModAugIfiWtt5U= 2022-09-03T05:59:56.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:glgyKppp4ZXj7ModAugIfiWtt5U=', 'Content-Length': '0'} 2022-09-03T05:59:56.882 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000051b0613d5ebfeeaa-006312ed5c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.883 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:59:56.883 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.884 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.884 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.885 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.886 DEBUG:boto:Method: GET 2022-09-03T05:59:56.886 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.887 DEBUG:boto:Data: 2022-09-03T05:59:56.888 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.888 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:56.889 DEBUG:boto:Port: 8001 2022-09-03T05:59:56.890 DEBUG:boto:Params: {} 2022-09-03T05:59:56.891 DEBUG:boto:Token: None 2022-09-03T05:59:56.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/?versions 2022-09-03T05:59:56.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xeJPpFLVrXsIJvXg5YvKdhhZ3S4= 2022-09-03T05:59:56.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xeJPpFLVrXsIJvXg5YvKdhhZ3S4=', 'Content-Length': '0'} 2022-09-03T05:59:56.897 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007079acde46a38f6e-006312ed5c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.897 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.899 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:56.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:56.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:56.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:56.901 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:59:56.901 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.902 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.902 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.903 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.904 DEBUG:boto:Method: GET 2022-09-03T05:59:56.904 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.905 DEBUG:boto:Data: 2022-09-03T05:59:56.906 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.907 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.908 DEBUG:boto:Params: {} 2022-09-03T05:59:56.908 DEBUG:boto:Token: None 2022-09-03T05:59:56.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/?versions 2022-09-03T05:59:56.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xeJPpFLVrXsIJvXg5YvKdhhZ3S4= 2022-09-03T05:59:56.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xeJPpFLVrXsIJvXg5YvKdhhZ3S4=', 'Content-Length': '0'} 2022-09-03T05:59:56.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb44d9807d3701a2-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.915 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.916 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:56.917 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:56.918 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:56.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:56.919 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.920 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.920 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.921 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.921 DEBUG:boto:Method: GET 2022-09-03T05:59:56.922 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.923 DEBUG:boto:Data: 2022-09-03T05:59:56.923 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:56.924 DEBUG:boto:Port: 8001 2022-09-03T05:59:56.925 DEBUG:boto:Params: {} 2022-09-03T05:59:56.925 DEBUG:boto:Token: None 2022-09-03T05:59:56.926 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/?versions 2022-09-03T05:59:56.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xeJPpFLVrXsIJvXg5YvKdhhZ3S4= 2022-09-03T05:59:56.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xeJPpFLVrXsIJvXg5YvKdhhZ3S4=', 'Content-Length': '0'} 2022-09-03T05:59:56.936 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009bc55f3d86cc6b99-006312ed5c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.937 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.939 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:56.939 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:56.940 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.941 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.941 DEBUG:boto:Method: GET 2022-09-03T05:59:56.942 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:56.942 DEBUG:boto:Data: 2022-09-03T05:59:56.943 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.943 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.944 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.944 DEBUG:boto:Params: {} 2022-09-03T05:59:56.945 DEBUG:boto:Token: None 2022-09-03T05:59:56.946 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/?versions 2022-09-03T05:59:56.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xeJPpFLVrXsIJvXg5YvKdhhZ3S4= 2022-09-03T05:59:56.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xeJPpFLVrXsIJvXg5YvKdhhZ3S4=', 'Content-Length': '0'} 2022-09-03T05:59:56.950 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067c2499469d1f519-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.951 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:56.952 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:56.952 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:56.953 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:56.954 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.954 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.955 DEBUG:boto:Method: GET 2022-09-03T05:59:56.956 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.956 DEBUG:boto:Data: 2022-09-03T05:59:56.957 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.958 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:56.958 DEBUG:boto:Port: 8001 2022-09-03T05:59:56.959 DEBUG:boto:Params: {} 2022-09-03T05:59:56.960 DEBUG:boto:Token: None 2022-09-03T05:59:56.961 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE= 2022-09-03T05:59:56.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE=', 'Content-Length': '0'} 2022-09-03T05:59:56.965 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000489e09515c74757e-006312ed5c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.965 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:56.966 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:56.967 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.968 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.968 DEBUG:boto:Method: GET 2022-09-03T05:59:56.969 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.969 DEBUG:boto:Data: 2022-09-03T05:59:56.970 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.971 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.971 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.972 DEBUG:boto:Params: {} 2022-09-03T05:59:56.973 DEBUG:boto:Token: None 2022-09-03T05:59:56.973 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.974 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE= 2022-09-03T05:59:56.975 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CTM5dl3e6PYVewL1JfxjDD/giqE=', 'Content-Length': '0'} 2022-09-03T05:59:56.977 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000004ce3860ea75f3f8-006312ed5c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.978 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:56.978 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:56.979 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.980 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.980 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.981 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.982 DEBUG:boto:Data: 2022-09-03T05:59:56.982 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:56.984 DEBUG:boto:Port: 8001 2022-09-03T05:59:56.984 DEBUG:boto:Params: {} 2022-09-03T05:59:56.985 DEBUG:boto:Token: None 2022-09-03T05:59:56.986 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L5bqTXyHAK8QNfBYHhgnwKTsSBU= 2022-09-03T05:59:56.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L5bqTXyHAK8QNfBYHhgnwKTsSBU=', 'Content-Length': '0'} 2022-09-03T05:59:56.990 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000742b49d2bb4c0f65-006312ed5c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:56.991 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:56.991 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:56.992 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.992 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.993 DEBUG:boto:Method: HEAD 2022-09-03T05:59:56.994 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:56.994 DEBUG:boto:Data: 2022-09-03T05:59:56.995 DEBUG:boto:Headers: {} 2022-09-03T05:59:56.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:56.996 DEBUG:boto:Port: 8000 2022-09-03T05:59:56.997 DEBUG:boto:Params: {} 2022-09-03T05:59:56.998 DEBUG:boto:Token: None 2022-09-03T05:59:56.999 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:56 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L5bqTXyHAK8QNfBYHhgnwKTsSBU= 2022-09-03T05:59:57.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L5bqTXyHAK8QNfBYHhgnwKTsSBU=', 'Content-Length': '0'} 2022-09-03T05:59:57.004 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d3e8fd837b16fc5d-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.005 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:57.006 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:57.006 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:57.007 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.007 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.008 DEBUG:boto:Method: GET 2022-09-03T05:59:57.009 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.009 DEBUG:boto:Data: 2022-09-03T05:59:57.010 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.010 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.011 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.012 DEBUG:boto:Params: {} 2022-09-03T05:59:57.012 DEBUG:boto:Token: None 2022-09-03T05:59:57.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw= 2022-09-03T05:59:57.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw=', 'Content-Length': '0'} 2022-09-03T05:59:57.017 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c8d8fb94977914ee-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.018 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:57.018 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:57.019 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.020 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.020 DEBUG:boto:Method: GET 2022-09-03T05:59:57.021 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.021 DEBUG:boto:Data: 2022-09-03T05:59:57.022 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.023 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.023 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.024 DEBUG:boto:Params: {} 2022-09-03T05:59:57.024 DEBUG:boto:Token: None 2022-09-03T05:59:57.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw= 2022-09-03T05:59:57.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw=', 'Content-Length': '0'} 2022-09-03T05:59:57.029 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c0fb80b47b92a677-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.030 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.030 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.031 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:57.032 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:57.032 DEBUG:boto:Method: GET 2022-09-03T05:59:57.033 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:57.034 DEBUG:boto:Data: 2022-09-03T05:59:57.034 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.035 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.036 DEBUG:boto:Params: {} 2022-09-03T05:59:57.037 DEBUG:boto:Token: None 2022-09-03T05:59:57.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.042 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000259e89c80055d5b2-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.086 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.088 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.088 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.089 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:57.090 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:57.091 DEBUG:boto:Method: GET 2022-09-03T05:59:57.091 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:57.092 DEBUG:boto:Data: 2022-09-03T05:59:57.092 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.093 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.094 DEBUG:boto:Params: {} 2022-09-03T05:59:57.095 DEBUG:boto:Token: None 2022-09-03T05:59:57.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.101 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb07611c76d7087b-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.101 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.103 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:57.103 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:57.104 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.105 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T05:59:57.105 DEBUG:boto:Data: 2022-09-03T05:59:57.106 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.107 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.108 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.109 DEBUG:boto:Params: {} 2022-09-03T05:59:57.110 DEBUG:boto:Token: None 2022-09-03T05:59:57.111 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/dummy 2022-09-03T05:59:57.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sQcohdGhTEhbuI/txePEk41G3Qc= 2022-09-03T05:59:57.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sQcohdGhTEhbuI/txePEk41G3Qc=', 'Content-Length': '0'} 2022-09-03T05:59:57.115 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006d2f7ee2ddaa8edb-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.116 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:57.117 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:57.118 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.119 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T05:59:57.120 DEBUG:boto:Data: 2022-09-03T05:59:57.121 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.123 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.124 DEBUG:boto:Params: {} 2022-09-03T05:59:57.125 DEBUG:boto:Token: None 2022-09-03T05:59:57.126 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/dummy 2022-09-03T05:59:57.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sQcohdGhTEhbuI/txePEk41G3Qc= 2022-09-03T05:59:57.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sQcohdGhTEhbuI/txePEk41G3Qc=', 'Content-Length': '0'} 2022-09-03T05:59:57.131 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e3de7265b2dabdb5-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.132 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:57.132 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:57.133 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:57.133 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.134 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.134 DEBUG:boto:Method: GET 2022-09-03T05:59:57.135 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.135 DEBUG:boto:Data: 2022-09-03T05:59:57.136 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.137 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.138 DEBUG:boto:Params: {} 2022-09-03T05:59:57.139 DEBUG:boto:Token: None 2022-09-03T05:59:57.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw= 2022-09-03T05:59:57.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw=', 'Content-Length': '0'} 2022-09-03T05:59:57.143 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000cee6d223ec448f1f-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.144 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:57.144 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:57.145 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.145 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.146 DEBUG:boto:Method: GET 2022-09-03T05:59:57.146 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.147 DEBUG:boto:Data: 2022-09-03T05:59:57.147 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.148 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.148 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.149 DEBUG:boto:Params: {} 2022-09-03T05:59:57.149 DEBUG:boto:Token: None 2022-09-03T05:59:57.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.151 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw= 2022-09-03T05:59:57.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw=', 'Content-Length': '0'} 2022-09-03T05:59:57.154 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000002268316cc3697573-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.155 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.155 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.156 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:57.156 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:57.157 DEBUG:boto:Method: GET 2022-09-03T05:59:57.158 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:57.158 DEBUG:boto:Data: 2022-09-03T05:59:57.159 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.159 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.160 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.161 DEBUG:boto:Params: {} 2022-09-03T05:59:57.161 DEBUG:boto:Token: None 2022-09-03T05:59:57.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.163 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.166 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e773dcb331814c2-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.206 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.209 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.210 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.210 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:57.211 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:57.212 DEBUG:boto:Method: GET 2022-09-03T05:59:57.212 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:57.213 DEBUG:boto:Data: 2022-09-03T05:59:57.214 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.214 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.215 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.215 DEBUG:boto:Params: {} 2022-09-03T05:59:57.216 DEBUG:boto:Token: None 2022-09-03T05:59:57.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.221 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000263992330275063f-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.222 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.223 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:57.224 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.224 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.225 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.226 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.227 DEBUG:boto:Method: GET 2022-09-03T05:59:57.228 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.228 DEBUG:boto:Data: 2022-09-03T05:59:57.230 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.231 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.232 DEBUG:boto:Params: {} 2022-09-03T05:59:57.233 DEBUG:boto:Token: None 2022-09-03T05:59:57.234 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo= 2022-09-03T05:59:57.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo=', 'Content-Length': '0'} 2022-09-03T05:59:57.237 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007bfd3b00ec6658b6-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.238 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.238 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.239 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.239 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.240 DEBUG:boto:Method: GET 2022-09-03T05:59:57.240 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.241 DEBUG:boto:Data: 2022-09-03T05:59:57.241 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.242 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.243 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.243 DEBUG:boto:Params: {} 2022-09-03T05:59:57.244 DEBUG:boto:Token: None 2022-09-03T05:59:57.244 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo= 2022-09-03T05:59:57.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo=', 'Content-Length': '0'} 2022-09-03T05:59:57.248 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000712c6e027324d08d-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.248 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.249 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.249 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.250 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.250 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.251 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.251 DEBUG:boto:Data: 2022-09-03T05:59:57.252 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.252 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.253 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.253 DEBUG:boto:Params: {} 2022-09-03T05:59:57.254 DEBUG:boto:Token: None 2022-09-03T05:59:57.254 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qxrLqbo2lLF66JVWNQ9DpEBxI1E= 2022-09-03T05:59:57.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qxrLqbo2lLF66JVWNQ9DpEBxI1E=', 'Content-Length': '0'} 2022-09-03T05:59:57.258 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c4d536b400a3466b-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.259 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.259 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.260 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.260 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.261 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.261 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.262 DEBUG:boto:Data: 2022-09-03T05:59:57.262 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.263 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.264 DEBUG:boto:Params: {} 2022-09-03T05:59:57.264 DEBUG:boto:Token: None 2022-09-03T05:59:57.265 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qxrLqbo2lLF66JVWNQ9DpEBxI1E= 2022-09-03T05:59:57.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qxrLqbo2lLF66JVWNQ9DpEBxI1E=', 'Content-Length': '0'} 2022-09-03T05:59:57.268 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005efc2195fa12df62-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.269 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:57.270 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.271 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.272 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.273 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.274 DEBUG:boto:Method: GET 2022-09-03T05:59:57.275 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.276 DEBUG:boto:Data: 2022-09-03T05:59:57.277 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.279 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.280 DEBUG:boto:Params: {} 2022-09-03T05:59:57.281 DEBUG:boto:Token: None 2022-09-03T05:59:57.283 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.284 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo= 2022-09-03T05:59:57.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo=', 'Content-Length': '0'} 2022-09-03T05:59:57.288 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e17fcbcb143d831e-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.290 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.291 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.292 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.292 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.293 DEBUG:boto:Method: GET 2022-09-03T05:59:57.293 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.294 DEBUG:boto:Data: 2022-09-03T05:59:57.294 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.295 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.296 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.297 DEBUG:boto:Params: {} 2022-09-03T05:59:57.298 DEBUG:boto:Token: None 2022-09-03T05:59:57.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo= 2022-09-03T05:59:57.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo=', 'Content-Length': '0'} 2022-09-03T05:59:57.303 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000612ad29c00286101-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.304 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.304 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.305 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.305 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.306 DEBUG:boto:Method: GET 2022-09-03T05:59:57.306 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.307 DEBUG:boto:Data: 2022-09-03T05:59:57.307 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.308 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.308 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.309 DEBUG:boto:Params: {} 2022-09-03T05:59:57.309 DEBUG:boto:Token: None 2022-09-03T05:59:57.310 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.314 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000014bd9be18593ca9b-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.315 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.316 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.316 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.317 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.317 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.318 DEBUG:boto:Method: GET 2022-09-03T05:59:57.318 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.319 DEBUG:boto:Data: 2022-09-03T05:59:57.319 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.320 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.321 DEBUG:boto:Params: {} 2022-09-03T05:59:57.322 DEBUG:boto:Token: None 2022-09-03T05:59:57.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.326 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006736aec2513d7c30-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.327 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.328 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.328 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.329 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.329 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T05:59:57.330 DEBUG:boto:Data: 2022-09-03T05:59:57.330 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.331 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.331 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.332 DEBUG:boto:Params: {} 2022-09-03T05:59:57.332 DEBUG:boto:Token: None 2022-09-03T05:59:57.333 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1 2022-09-03T05:59:57.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:386EwGZPIALGqYWKxdhT7wZD/x4= 2022-09-03T05:59:57.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:386EwGZPIALGqYWKxdhT7wZD/x4=', 'Content-Length': '0'} 2022-09-03T05:59:57.336 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000013239b5440c77935-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.337 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.338 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.338 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.339 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T05:59:57.339 DEBUG:boto:Data: 2022-09-03T05:59:57.340 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.341 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.341 DEBUG:boto:Params: {} 2022-09-03T05:59:57.342 DEBUG:boto:Token: None 2022-09-03T05:59:57.342 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1 2022-09-03T05:59:57.343 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:386EwGZPIALGqYWKxdhT7wZD/x4= 2022-09-03T05:59:57.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:386EwGZPIALGqYWKxdhT7wZD/x4=', 'Content-Length': '0'} 2022-09-03T05:59:57.346 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004202c7d30110064c-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.346 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:57.347 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.347 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.348 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.348 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.349 DEBUG:boto:Method: GET 2022-09-03T05:59:57.349 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.350 DEBUG:boto:Data: 2022-09-03T05:59:57.351 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.352 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.352 DEBUG:boto:Params: {} 2022-09-03T05:59:57.353 DEBUG:boto:Token: None 2022-09-03T05:59:57.353 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo= 2022-09-03T05:59:57.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo=', 'Content-Length': '0'} 2022-09-03T05:59:57.357 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009e464a24384489f1-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.357 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.358 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:57.359 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.359 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.360 DEBUG:boto:Method: GET 2022-09-03T05:59:57.360 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.361 DEBUG:boto:Data: 2022-09-03T05:59:57.361 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.362 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.363 DEBUG:boto:Params: {} 2022-09-03T05:59:57.363 DEBUG:boto:Token: None 2022-09-03T05:59:57.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:57.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo= 2022-09-03T05:59:57.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNpUl8BIjX9zICdlkm/2E4oJCMo=', 'Content-Length': '0'} 2022-09-03T05:59:57.367 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008a722ec4f2f2ad7f-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.368 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.368 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.369 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.369 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.370 DEBUG:boto:Method: GET 2022-09-03T05:59:57.371 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.371 DEBUG:boto:Data: 2022-09-03T05:59:57.372 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.372 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.373 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.373 DEBUG:boto:Params: {} 2022-09-03T05:59:57.374 DEBUG:boto:Token: None 2022-09-03T05:59:57.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.378 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000050c5fea4aa8be77-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.379 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.379 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.380 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.381 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.381 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.382 DEBUG:boto:Method: GET 2022-09-03T05:59:57.382 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:57.383 DEBUG:boto:Data: 2022-09-03T05:59:57.383 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.384 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.385 DEBUG:boto:Params: {} 2022-09-03T05:59:57.385 DEBUG:boto:Token: None 2022-09-03T05:59:57.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.390 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bee126d063f4bac0-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.390 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.391 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:57.392 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.392 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.393 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.393 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.394 DEBUG:boto:Method: GET 2022-09-03T05:59:57.394 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.395 DEBUG:boto:Data: 2022-09-03T05:59:57.395 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.396 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.397 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.397 DEBUG:boto:Params: {} 2022-09-03T05:59:57.398 DEBUG:boto:Token: None 2022-09-03T05:59:57.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo= 2022-09-03T05:59:57.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo=', 'Content-Length': '0'} 2022-09-03T05:59:57.402 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b39a7ad5778587a1-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.446 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.447 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.448 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.448 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.449 DEBUG:boto:Method: GET 2022-09-03T05:59:57.449 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.450 DEBUG:boto:Data: 2022-09-03T05:59:57.450 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.452 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.452 DEBUG:boto:Params: {} 2022-09-03T05:59:57.453 DEBUG:boto:Token: None 2022-09-03T05:59:57.454 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo= 2022-09-03T05:59:57.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo=', 'Content-Length': '0'} 2022-09-03T05:59:57.457 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e181a10984e422b1-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.458 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.459 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.460 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.460 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.461 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.461 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.462 DEBUG:boto:Data: 2022-09-03T05:59:57.462 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.463 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.464 DEBUG:boto:Params: {} 2022-09-03T05:59:57.464 DEBUG:boto:Token: None 2022-09-03T05:59:57.465 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tFrgwsQbDyhCQAtAnekPn6QayVA= 2022-09-03T05:59:57.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tFrgwsQbDyhCQAtAnekPn6QayVA=', 'Content-Length': '0'} 2022-09-03T05:59:57.468 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000040f61e02b8813d73-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.469 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.470 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.470 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.471 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.471 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.472 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.472 DEBUG:boto:Data: 2022-09-03T05:59:57.473 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.474 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.474 DEBUG:boto:Params: {} 2022-09-03T05:59:57.475 DEBUG:boto:Token: None 2022-09-03T05:59:57.476 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tFrgwsQbDyhCQAtAnekPn6QayVA= 2022-09-03T05:59:57.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tFrgwsQbDyhCQAtAnekPn6QayVA=', 'Content-Length': '0'} 2022-09-03T05:59:57.479 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e01b7fa4da47cb0f-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.480 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:57.481 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.481 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.482 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.482 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.483 DEBUG:boto:Method: GET 2022-09-03T05:59:57.483 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.484 DEBUG:boto:Data: 2022-09-03T05:59:57.484 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.485 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.486 DEBUG:boto:Params: {} 2022-09-03T05:59:57.487 DEBUG:boto:Token: None 2022-09-03T05:59:57.487 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo= 2022-09-03T05:59:57.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo=', 'Content-Length': '0'} 2022-09-03T05:59:57.490 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009bd469203d347047-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.491 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.492 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.492 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.493 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.493 DEBUG:boto:Method: GET 2022-09-03T05:59:57.494 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.494 DEBUG:boto:Data: 2022-09-03T05:59:57.495 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.496 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.497 DEBUG:boto:Params: {} 2022-09-03T05:59:57.497 DEBUG:boto:Token: None 2022-09-03T05:59:57.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo= 2022-09-03T05:59:57.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo=', 'Content-Length': '0'} 2022-09-03T05:59:57.502 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000041fa31d519d3b0d7-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.542 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.543 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.543 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.544 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.545 DEBUG:boto:Method: GET 2022-09-03T05:59:57.546 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.546 DEBUG:boto:Data: 2022-09-03T05:59:57.547 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.548 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.548 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.549 DEBUG:boto:Params: {} 2022-09-03T05:59:57.549 DEBUG:boto:Token: None 2022-09-03T05:59:57.550 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.555 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4d45eb351dbaf00-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.556 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.557 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.558 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.559 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.560 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.561 DEBUG:boto:Method: GET 2022-09-03T05:59:57.562 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.562 DEBUG:boto:Data: 2022-09-03T05:59:57.563 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.564 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.565 DEBUG:boto:Params: {} 2022-09-03T05:59:57.565 DEBUG:boto:Token: None 2022-09-03T05:59:57.566 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.567 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.568 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.570 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d461b731e500f4ca-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.571 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.572 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.573 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.574 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.574 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T05:59:57.575 DEBUG:boto:Data: 2022-09-03T05:59:57.575 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.576 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.577 DEBUG:boto:Params: {} 2022-09-03T05:59:57.577 DEBUG:boto:Token: None 2022-09-03T05:59:57.578 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2 2022-09-03T05:59:57.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H9VpZvhS+W2YCXN2DiLLieYmh90= 2022-09-03T05:59:57.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H9VpZvhS+W2YCXN2DiLLieYmh90=', 'Content-Length': '0'} 2022-09-03T05:59:57.582 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d50879a8eab810db-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.583 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.583 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.584 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.584 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T05:59:57.585 DEBUG:boto:Data: 2022-09-03T05:59:57.585 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.586 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.587 DEBUG:boto:Params: {} 2022-09-03T05:59:57.588 DEBUG:boto:Token: None 2022-09-03T05:59:57.588 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2 2022-09-03T05:59:57.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H9VpZvhS+W2YCXN2DiLLieYmh90= 2022-09-03T05:59:57.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H9VpZvhS+W2YCXN2DiLLieYmh90=', 'Content-Length': '0'} 2022-09-03T05:59:57.592 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d19eed92eb864cff-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.593 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:57.594 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.594 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.595 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.595 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.596 DEBUG:boto:Method: GET 2022-09-03T05:59:57.597 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.597 DEBUG:boto:Data: 2022-09-03T05:59:57.598 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.600 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.601 DEBUG:boto:Params: {} 2022-09-03T05:59:57.602 DEBUG:boto:Token: None 2022-09-03T05:59:57.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo= 2022-09-03T05:59:57.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo=', 'Content-Length': '0'} 2022-09-03T05:59:57.607 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000075f52e380f87c56c-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.608 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.608 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:57.609 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.609 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.610 DEBUG:boto:Method: GET 2022-09-03T05:59:57.611 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.611 DEBUG:boto:Data: 2022-09-03T05:59:57.612 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.614 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.614 DEBUG:boto:Params: {} 2022-09-03T05:59:57.615 DEBUG:boto:Token: None 2022-09-03T05:59:57.616 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:57.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo= 2022-09-03T05:59:57.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1CjTwrUgUHOKI9JowcaL83/Tlfo=', 'Content-Length': '0'} 2022-09-03T05:59:57.620 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000d54141ae097f020-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.620 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.621 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.622 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.622 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.623 DEBUG:boto:Method: GET 2022-09-03T05:59:57.623 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.624 DEBUG:boto:Data: 2022-09-03T05:59:57.625 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.626 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.626 DEBUG:boto:Params: {} 2022-09-03T05:59:57.627 DEBUG:boto:Token: None 2022-09-03T05:59:57.628 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.632 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000073e9aebe15f97c0f-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.679 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.684 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.685 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.686 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.686 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.687 DEBUG:boto:Method: GET 2022-09-03T05:59:57.688 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:57.688 DEBUG:boto:Data: 2022-09-03T05:59:57.689 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.690 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.691 DEBUG:boto:Params: {} 2022-09-03T05:59:57.691 DEBUG:boto:Token: None 2022-09-03T05:59:57.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.693 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.696 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001238ea55e45e1333-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.697 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.698 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:57.698 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.699 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.700 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.700 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.701 DEBUG:boto:Method: GET 2022-09-03T05:59:57.701 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.702 DEBUG:boto:Data: 2022-09-03T05:59:57.702 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.703 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.704 DEBUG:boto:Params: {} 2022-09-03T05:59:57.705 DEBUG:boto:Token: None 2022-09-03T05:59:57.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw= 2022-09-03T05:59:57.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw=', 'Content-Length': '0'} 2022-09-03T05:59:57.709 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000098f11e7c184eac43-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.710 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.710 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.711 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.711 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.712 DEBUG:boto:Method: GET 2022-09-03T05:59:57.713 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.713 DEBUG:boto:Data: 2022-09-03T05:59:57.714 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.715 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.715 DEBUG:boto:Params: {} 2022-09-03T05:59:57.716 DEBUG:boto:Token: None 2022-09-03T05:59:57.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw= 2022-09-03T05:59:57.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw=', 'Content-Length': '0'} 2022-09-03T05:59:57.720 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000023ab8e71197ba78c-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.721 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.722 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.722 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.723 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.723 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.724 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.724 DEBUG:boto:Data: 2022-09-03T05:59:57.725 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.725 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.726 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.726 DEBUG:boto:Params: {} 2022-09-03T05:59:57.727 DEBUG:boto:Token: None 2022-09-03T05:59:57.727 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QfsF9ncSSFeLFF6miamHrCHMWIY= 2022-09-03T05:59:57.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QfsF9ncSSFeLFF6miamHrCHMWIY=', 'Content-Length': '0'} 2022-09-03T05:59:57.731 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001692f58fba24046e-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.731 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.732 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.732 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.733 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.733 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.734 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.734 DEBUG:boto:Data: 2022-09-03T05:59:57.735 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.735 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.736 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.736 DEBUG:boto:Params: {} 2022-09-03T05:59:57.737 DEBUG:boto:Token: None 2022-09-03T05:59:57.738 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.738 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QfsF9ncSSFeLFF6miamHrCHMWIY= 2022-09-03T05:59:57.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QfsF9ncSSFeLFF6miamHrCHMWIY=', 'Content-Length': '0'} 2022-09-03T05:59:57.741 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000046ddb48bef7b23c1-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.742 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:57.742 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.743 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.743 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.744 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.744 DEBUG:boto:Method: GET 2022-09-03T05:59:57.745 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.746 DEBUG:boto:Data: 2022-09-03T05:59:57.746 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.747 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.748 DEBUG:boto:Params: {} 2022-09-03T05:59:57.748 DEBUG:boto:Token: None 2022-09-03T05:59:57.749 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw= 2022-09-03T05:59:57.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw=', 'Content-Length': '0'} 2022-09-03T05:59:57.752 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000db4b19bad1e3927b-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.753 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.754 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.754 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.755 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.755 DEBUG:boto:Method: GET 2022-09-03T05:59:57.756 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.756 DEBUG:boto:Data: 2022-09-03T05:59:57.757 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.758 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.759 DEBUG:boto:Params: {} 2022-09-03T05:59:57.759 DEBUG:boto:Token: None 2022-09-03T05:59:57.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw= 2022-09-03T05:59:57.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw=', 'Content-Length': '0'} 2022-09-03T05:59:57.763 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e9b1d4680d1c5fe8-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.764 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.765 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.765 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.766 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.766 DEBUG:boto:Method: GET 2022-09-03T05:59:57.767 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.767 DEBUG:boto:Data: 2022-09-03T05:59:57.768 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.769 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.769 DEBUG:boto:Params: {} 2022-09-03T05:59:57.770 DEBUG:boto:Token: None 2022-09-03T05:59:57.771 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.775 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d3a43d4238f73af-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.775 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.777 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.777 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.778 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.778 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.779 DEBUG:boto:Method: GET 2022-09-03T05:59:57.779 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.780 DEBUG:boto:Data: 2022-09-03T05:59:57.780 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.781 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.781 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.782 DEBUG:boto:Params: {} 2022-09-03T05:59:57.783 DEBUG:boto:Token: None 2022-09-03T05:59:57.783 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.787 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003440b66c380ef9c2-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.788 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.789 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.790 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.790 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.791 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T05:59:57.792 DEBUG:boto:Data: 2022-09-03T05:59:57.792 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.793 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.794 DEBUG:boto:Params: {} 2022-09-03T05:59:57.794 DEBUG:boto:Token: None 2022-09-03T05:59:57.795 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3 2022-09-03T05:59:57.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EFbsFeakwGDDROa5/FxS3iwBzak= 2022-09-03T05:59:57.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EFbsFeakwGDDROa5/FxS3iwBzak=', 'Content-Length': '0'} 2022-09-03T05:59:57.801 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007768ebdd35084941-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.802 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.803 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.803 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.804 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T05:59:57.804 DEBUG:boto:Data: 2022-09-03T05:59:57.805 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.806 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.806 DEBUG:boto:Params: {} 2022-09-03T05:59:57.807 DEBUG:boto:Token: None 2022-09-03T05:59:57.808 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3 2022-09-03T05:59:57.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EFbsFeakwGDDROa5/FxS3iwBzak= 2022-09-03T05:59:57.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EFbsFeakwGDDROa5/FxS3iwBzak=', 'Content-Length': '0'} 2022-09-03T05:59:57.812 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d6fdd01106da44ed-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.813 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:57.814 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.815 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.816 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.816 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.817 DEBUG:boto:Method: GET 2022-09-03T05:59:57.817 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.818 DEBUG:boto:Data: 2022-09-03T05:59:57.818 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.820 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.821 DEBUG:boto:Params: {} 2022-09-03T05:59:57.822 DEBUG:boto:Token: None 2022-09-03T05:59:57.823 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw= 2022-09-03T05:59:57.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw=', 'Content-Length': '0'} 2022-09-03T05:59:57.828 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009da60ef28ba9187e-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.870 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.871 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:57.871 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.872 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.872 DEBUG:boto:Method: GET 2022-09-03T05:59:57.873 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.873 DEBUG:boto:Data: 2022-09-03T05:59:57.874 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.875 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.876 DEBUG:boto:Params: {} 2022-09-03T05:59:57.876 DEBUG:boto:Token: None 2022-09-03T05:59:57.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:57.877 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw= 2022-09-03T05:59:57.878 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYDz7kDiLHlYykHPgW+JxkTK9mw=', 'Content-Length': '0'} 2022-09-03T05:59:57.880 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000044f52db3e5435db6-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.881 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.882 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.882 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.883 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.883 DEBUG:boto:Method: GET 2022-09-03T05:59:57.884 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.884 DEBUG:boto:Data: 2022-09-03T05:59:57.885 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.886 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.887 DEBUG:boto:Params: {} 2022-09-03T05:59:57.888 DEBUG:boto:Token: None 2022-09-03T05:59:57.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.893 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ba945d21daa4af7-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.894 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.895 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.896 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.897 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.897 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.898 DEBUG:boto:Method: GET 2022-09-03T05:59:57.898 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:57.899 DEBUG:boto:Data: 2022-09-03T05:59:57.899 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.900 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:57.900 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.901 DEBUG:boto:Params: {} 2022-09-03T05:59:57.901 DEBUG:boto:Token: None 2022-09-03T05:59:57.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek= 2022-09-03T05:59:57.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:55FuzYNHWonBLf1NVKNrzptnQek=', 'Content-Length': '0'} 2022-09-03T05:59:57.906 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000012f287ee95aaab86-006312ed5d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.908 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.909 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-18 zones={test-zone3, test-zone1} 2022-09-03T05:59:57.910 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-18 zones={test-zone3, test-zone2} 2022-09-03T05:59:57.911 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.912 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.913 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.914 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:59:57.914 DEBUG:boto:Data: 2022-09-03T05:59:57.915 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.916 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.916 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.917 DEBUG:boto:Params: {} 2022-09-03T05:59:57.918 DEBUG:boto:Token: None 2022-09-03T05:59:57.918 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ymy5/1uXnNQnpXkmWe/Dqb2Vhrk= 2022-09-03T05:59:57.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ymy5/1uXnNQnpXkmWe/Dqb2Vhrk=', 'Content-Length': '0'} 2022-09-03T05:59:57.922 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004e9c61aa3f2f905e-006312ed5d-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.923 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.923 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.924 DEBUG:boto:Method: HEAD 2022-09-03T05:59:57.924 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:59:57.925 DEBUG:boto:Data: 2022-09-03T05:59:57.925 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.926 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:57.926 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.927 DEBUG:boto:Params: {} 2022-09-03T05:59:57.927 DEBUG:boto:Token: None 2022-09-03T05:59:57.928 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/ 2022-09-03T05:59:57.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ymy5/1uXnNQnpXkmWe/Dqb2Vhrk= 2022-09-03T05:59:57.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ymy5/1uXnNQnpXkmWe/Dqb2Vhrk=', 'Content-Length': '0'} 2022-09-03T05:59:57.932 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a1f4b2fe08566abc-006312ed5d-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.933 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T05:59:57.933 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.934 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.935 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.935 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.936 DEBUG:boto:Method: GET 2022-09-03T05:59:57.936 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.937 DEBUG:boto:Data: 2022-09-03T05:59:57.937 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.938 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.938 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.939 DEBUG:boto:Params: {} 2022-09-03T05:59:57.939 DEBUG:boto:Token: None 2022-09-03T05:59:57.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/?versions 2022-09-03T05:59:57.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xhPJlULzaCLep1SndNhVHOsbGtU= 2022-09-03T05:59:57.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xhPJlULzaCLep1SndNhVHOsbGtU=', 'Content-Length': '0'} 2022-09-03T05:59:57.944 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e11ee41d61ac806e-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.945 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.946 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:57.947 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:57.947 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:57.947 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:57.948 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T05:59:57.949 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.949 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.950 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.950 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.951 DEBUG:boto:Method: GET 2022-09-03T05:59:57.951 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.952 DEBUG:boto:Data: 2022-09-03T05:59:57.952 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.953 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:57.953 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.954 DEBUG:boto:Params: {} 2022-09-03T05:59:57.955 DEBUG:boto:Token: None 2022-09-03T05:59:57.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/?versions 2022-09-03T05:59:57.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xhPJlULzaCLep1SndNhVHOsbGtU= 2022-09-03T05:59:57.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xhPJlULzaCLep1SndNhVHOsbGtU=', 'Content-Length': '0'} 2022-09-03T05:59:57.959 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a41220c9b4074c3a-006312ed5d-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.960 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.961 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T05:59:57.961 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T05:59:57.962 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T05:59:57.962 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T05:59:57.963 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.963 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.964 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.964 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.965 DEBUG:boto:Method: GET 2022-09-03T05:59:57.965 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.966 DEBUG:boto:Data: 2022-09-03T05:59:57.966 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.967 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.968 DEBUG:boto:Params: {} 2022-09-03T05:59:57.968 DEBUG:boto:Token: None 2022-09-03T05:59:57.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/?versions 2022-09-03T05:59:57.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xhPJlULzaCLep1SndNhVHOsbGtU= 2022-09-03T05:59:57.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xhPJlULzaCLep1SndNhVHOsbGtU=', 'Content-Length': '0'} 2022-09-03T05:59:57.973 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007581965647657947-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.974 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.975 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:57.976 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:57.976 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.977 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.977 DEBUG:boto:Method: GET 2022-09-03T05:59:57.978 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T05:59:57.978 DEBUG:boto:Data: 2022-09-03T05:59:57.979 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.979 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:57.980 DEBUG:boto:Port: 8000 2022-09-03T05:59:57.980 DEBUG:boto:Params: {} 2022-09-03T05:59:57.981 DEBUG:boto:Token: None 2022-09-03T05:59:57.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/?versions 2022-09-03T05:59:57.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xhPJlULzaCLep1SndNhVHOsbGtU= 2022-09-03T05:59:57.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xhPJlULzaCLep1SndNhVHOsbGtU=', 'Content-Length': '0'} 2022-09-03T05:59:57.986 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a56b5b0fc0c8204-006312ed5d-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.986 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:57.987 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:57.988 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:57.988 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:57.989 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.989 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.990 DEBUG:boto:Method: GET 2022-09-03T05:59:57.991 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.991 DEBUG:boto:Data: 2022-09-03T05:59:57.992 DEBUG:boto:Headers: {} 2022-09-03T05:59:57.992 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:57.993 DEBUG:boto:Port: 8001 2022-09-03T05:59:57.993 DEBUG:boto:Params: {} 2022-09-03T05:59:57.994 DEBUG:boto:Token: None 2022-09-03T05:59:57.995 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:57 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:57.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw= 2022-09-03T05:59:57.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sYLo5zEUl3vdogzOMN3kHJ52iBw=', 'Content-Length': '0'} 2022-09-03T05:59:57.998 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008d53d404bfbbf34e-006312ed5d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:57.999 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:57.999 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:58.000 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.000 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.001 DEBUG:boto:Method: GET 2022-09-03T05:59:58.002 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.002 DEBUG:boto:Data: 2022-09-03T05:59:58.003 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.003 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.004 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.004 DEBUG:boto:Params: {} 2022-09-03T05:59:58.005 DEBUG:boto:Token: None 2022-09-03T05:59:58.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks= 2022-09-03T05:59:58.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks=', 'Content-Length': '0'} 2022-09-03T05:59:58.009 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000051a3754d3e47eb3d-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.010 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:58.011 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:58.011 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.012 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.012 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.013 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.013 DEBUG:boto:Data: 2022-09-03T05:59:58.014 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.015 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.015 DEBUG:boto:Params: {} 2022-09-03T05:59:58.016 DEBUG:boto:Token: None 2022-09-03T05:59:58.016 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:plz276UMu4HWk9ll2MxHnygTgtc= 2022-09-03T05:59:58.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:plz276UMu4HWk9ll2MxHnygTgtc=', 'Content-Length': '0'} 2022-09-03T05:59:58.020 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e700fce2c7b4c00e-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.020 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:58.021 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:58.022 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.022 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.023 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.023 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.024 DEBUG:boto:Data: 2022-09-03T05:59:58.024 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.025 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.025 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.026 DEBUG:boto:Params: {} 2022-09-03T05:59:58.026 DEBUG:boto:Token: None 2022-09-03T05:59:58.027 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:plz276UMu4HWk9ll2MxHnygTgtc= 2022-09-03T05:59:58.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:plz276UMu4HWk9ll2MxHnygTgtc=', 'Content-Length': '0'} 2022-09-03T05:59:58.030 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007858aa8828e6e8a5-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.031 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:58.031 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:58.032 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:58.032 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.033 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.033 DEBUG:boto:Method: GET 2022-09-03T05:59:58.034 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.035 DEBUG:boto:Data: 2022-09-03T05:59:58.035 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.036 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.037 DEBUG:boto:Params: {} 2022-09-03T05:59:58.037 DEBUG:boto:Token: None 2022-09-03T05:59:58.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks= 2022-09-03T05:59:58.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks=', 'Content-Length': '0'} 2022-09-03T05:59:58.041 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e11eafa4c6c3ac19-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.042 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:58.043 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:58.043 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.044 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.044 DEBUG:boto:Method: GET 2022-09-03T05:59:58.045 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.045 DEBUG:boto:Data: 2022-09-03T05:59:58.046 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.047 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.047 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.048 DEBUG:boto:Params: {} 2022-09-03T05:59:58.048 DEBUG:boto:Token: None 2022-09-03T05:59:58.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks= 2022-09-03T05:59:58.050 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks=', 'Content-Length': '0'} 2022-09-03T05:59:58.052 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005e16a330c80ea645-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.053 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.054 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.054 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:58.055 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:58.055 DEBUG:boto:Method: GET 2022-09-03T05:59:58.056 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:58.056 DEBUG:boto:Data: 2022-09-03T05:59:58.057 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.058 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.058 DEBUG:boto:Params: {} 2022-09-03T05:59:58.059 DEBUG:boto:Token: None 2022-09-03T05:59:58.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.064 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b4bb41e710dd0858-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.106 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.108 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.109 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.110 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:58.110 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:58.111 DEBUG:boto:Method: GET 2022-09-03T05:59:58.112 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:58.112 DEBUG:boto:Data: 2022-09-03T05:59:58.113 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.113 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.114 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.114 DEBUG:boto:Params: {} 2022-09-03T05:59:58.115 DEBUG:boto:Token: None 2022-09-03T05:59:58.116 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.119 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d8c58c93c02e1d5-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.120 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.121 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:58.122 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:58.122 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.123 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T05:59:58.123 DEBUG:boto:Data: 2022-09-03T05:59:58.124 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.124 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.125 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.125 DEBUG:boto:Params: {} 2022-09-03T05:59:58.126 DEBUG:boto:Token: None 2022-09-03T05:59:58.126 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/dummy 2022-09-03T05:59:58.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jV5i/TTWf3fajElKrNJXzmseJIQ= 2022-09-03T05:59:58.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jV5i/TTWf3fajElKrNJXzmseJIQ=', 'Content-Length': '0'} 2022-09-03T05:59:58.130 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000352f6823a1fefe9c-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.131 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:58.132 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:58.132 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.133 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T05:59:58.133 DEBUG:boto:Data: 2022-09-03T05:59:58.134 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.134 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.135 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.135 DEBUG:boto:Params: {} 2022-09-03T05:59:58.136 DEBUG:boto:Token: None 2022-09-03T05:59:58.136 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/dummy 2022-09-03T05:59:58.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jV5i/TTWf3fajElKrNJXzmseJIQ= 2022-09-03T05:59:58.137 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jV5i/TTWf3fajElKrNJXzmseJIQ=', 'Content-Length': '0'} 2022-09-03T05:59:58.140 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000037d343f1bed73683-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.141 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T05:59:58.141 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:58.142 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:58.142 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.143 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.143 DEBUG:boto:Method: GET 2022-09-03T05:59:58.144 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.144 DEBUG:boto:Data: 2022-09-03T05:59:58.145 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.146 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.147 DEBUG:boto:Params: {} 2022-09-03T05:59:58.147 DEBUG:boto:Token: None 2022-09-03T05:59:58.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks= 2022-09-03T05:59:58.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks=', 'Content-Length': '0'} 2022-09-03T05:59:58.151 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007f433e02b22d40a6-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.152 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T05:59:58.152 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T05:59:58.153 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.153 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.154 DEBUG:boto:Method: GET 2022-09-03T05:59:58.155 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.155 DEBUG:boto:Data: 2022-09-03T05:59:58.156 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.156 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.157 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.157 DEBUG:boto:Params: {} 2022-09-03T05:59:58.158 DEBUG:boto:Token: None 2022-09-03T05:59:58.158 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T05:59:58.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks= 2022-09-03T05:59:58.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GZ1l8cSPjumcCc9pVPrL85Lcrks=', 'Content-Length': '0'} 2022-09-03T05:59:58.162 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c08f8ff8a02f1d56-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.163 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.163 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.164 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:58.164 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:58.165 DEBUG:boto:Method: GET 2022-09-03T05:59:58.165 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:58.166 DEBUG:boto:Data: 2022-09-03T05:59:58.166 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.167 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.168 DEBUG:boto:Params: {} 2022-09-03T05:59:58.168 DEBUG:boto:Token: None 2022-09-03T05:59:58.169 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.170 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.173 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8a6d05c7cdafef4-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.173 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.174 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.175 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.175 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:58.176 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T05:59:58.176 DEBUG:boto:Method: GET 2022-09-03T05:59:58.177 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T05:59:58.177 DEBUG:boto:Data: 2022-09-03T05:59:58.178 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.178 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.179 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.180 DEBUG:boto:Params: {} 2022-09-03T05:59:58.180 DEBUG:boto:Token: None 2022-09-03T05:59:58.181 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.181 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.185 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bd13692b16707835-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.185 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.186 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:58.187 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.187 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.188 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.188 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.189 DEBUG:boto:Method: GET 2022-09-03T05:59:58.190 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.190 DEBUG:boto:Data: 2022-09-03T05:59:58.191 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.191 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.192 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.193 DEBUG:boto:Params: {} 2022-09-03T05:59:58.194 DEBUG:boto:Token: None 2022-09-03T05:59:58.194 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI= 2022-09-03T05:59:58.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI=', 'Content-Length': '0'} 2022-09-03T05:59:58.198 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002b191be34e87e81e-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.199 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.199 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.200 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.201 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.202 DEBUG:boto:Method: GET 2022-09-03T05:59:58.203 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.204 DEBUG:boto:Data: 2022-09-03T05:59:58.204 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.205 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.205 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.206 DEBUG:boto:Params: {} 2022-09-03T05:59:58.207 DEBUG:boto:Token: None 2022-09-03T05:59:58.207 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI= 2022-09-03T05:59:58.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI=', 'Content-Length': '0'} 2022-09-03T05:59:58.211 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000072d236d4c2d49697-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.212 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.212 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.213 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.213 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.214 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.215 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.216 DEBUG:boto:Data: 2022-09-03T05:59:58.217 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.218 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.218 DEBUG:boto:Params: {} 2022-09-03T05:59:58.219 DEBUG:boto:Token: None 2022-09-03T05:59:58.219 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CdBndsboQ/yvLYFffXcEgNZ/qCA= 2022-09-03T05:59:58.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CdBndsboQ/yvLYFffXcEgNZ/qCA=', 'Content-Length': '0'} 2022-09-03T05:59:58.224 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003083adbfe7fa2e8f-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.225 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.225 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.226 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.226 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.227 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.227 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.228 DEBUG:boto:Data: 2022-09-03T05:59:58.228 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.229 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.229 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.229 DEBUG:boto:Params: {} 2022-09-03T05:59:58.230 DEBUG:boto:Token: None 2022-09-03T05:59:58.231 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CdBndsboQ/yvLYFffXcEgNZ/qCA= 2022-09-03T05:59:58.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CdBndsboQ/yvLYFffXcEgNZ/qCA=', 'Content-Length': '0'} 2022-09-03T05:59:58.236 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000001d97668d073f740-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.237 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:58.238 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.239 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.240 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.240 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.241 DEBUG:boto:Method: GET 2022-09-03T05:59:58.242 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.243 DEBUG:boto:Data: 2022-09-03T05:59:58.243 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.244 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.244 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.245 DEBUG:boto:Params: {} 2022-09-03T05:59:58.246 DEBUG:boto:Token: None 2022-09-03T05:59:58.246 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI= 2022-09-03T05:59:58.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI=', 'Content-Length': '0'} 2022-09-03T05:59:58.250 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d166a9e0109e2207-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.298 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.299 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.299 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.300 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.300 DEBUG:boto:Method: GET 2022-09-03T05:59:58.301 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.301 DEBUG:boto:Data: 2022-09-03T05:59:58.302 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.303 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.303 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.304 DEBUG:boto:Params: {} 2022-09-03T05:59:58.304 DEBUG:boto:Token: None 2022-09-03T05:59:58.305 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.306 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI= 2022-09-03T05:59:58.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI=', 'Content-Length': '0'} 2022-09-03T05:59:58.309 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000013f3bb917d7002ac-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.309 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.310 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.311 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.311 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.312 DEBUG:boto:Method: GET 2022-09-03T05:59:58.312 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.313 DEBUG:boto:Data: 2022-09-03T05:59:58.313 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.314 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.315 DEBUG:boto:Params: {} 2022-09-03T05:59:58.315 DEBUG:boto:Token: None 2022-09-03T05:59:58.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.320 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000052cd57f4035ea40f-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.320 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.321 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.322 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.322 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.323 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.323 DEBUG:boto:Method: GET 2022-09-03T05:59:58.324 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.324 DEBUG:boto:Data: 2022-09-03T05:59:58.325 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.326 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.326 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.327 DEBUG:boto:Params: {} 2022-09-03T05:59:58.327 DEBUG:boto:Token: None 2022-09-03T05:59:58.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.332 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e152139a21c531c5-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.332 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.333 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.334 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.334 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.335 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T05:59:58.336 DEBUG:boto:Data: 2022-09-03T05:59:58.336 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.337 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.338 DEBUG:boto:Params: {} 2022-09-03T05:59:58.338 DEBUG:boto:Token: None 2022-09-03T05:59:58.339 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1 2022-09-03T05:59:58.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DkpOFaaGM1xu3KpCTCPNPMQZqnU= 2022-09-03T05:59:58.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DkpOFaaGM1xu3KpCTCPNPMQZqnU=', 'Content-Length': '0'} 2022-09-03T05:59:58.343 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006424c070411a9f56-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.343 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.344 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.344 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.345 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T05:59:58.346 DEBUG:boto:Data: 2022-09-03T05:59:58.346 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.347 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.347 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.348 DEBUG:boto:Params: {} 2022-09-03T05:59:58.348 DEBUG:boto:Token: None 2022-09-03T05:59:58.349 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1 2022-09-03T05:59:58.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DkpOFaaGM1xu3KpCTCPNPMQZqnU= 2022-09-03T05:59:58.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DkpOFaaGM1xu3KpCTCPNPMQZqnU=', 'Content-Length': '0'} 2022-09-03T05:59:58.353 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f286042d2ee406e8-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.353 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T05:59:58.354 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.354 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.355 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.356 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.356 DEBUG:boto:Method: GET 2022-09-03T05:59:58.357 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.357 DEBUG:boto:Data: 2022-09-03T05:59:58.358 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.359 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.359 DEBUG:boto:Params: {} 2022-09-03T05:59:58.360 DEBUG:boto:Token: None 2022-09-03T05:59:58.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI= 2022-09-03T05:59:58.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI=', 'Content-Length': '0'} 2022-09-03T05:59:58.364 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fb21772db7acd107-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.364 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.365 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:58.365 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.366 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.367 DEBUG:boto:Method: GET 2022-09-03T05:59:58.367 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.368 DEBUG:boto:Data: 2022-09-03T05:59:58.368 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.369 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.369 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.370 DEBUG:boto:Params: {} 2022-09-03T05:59:58.370 DEBUG:boto:Token: None 2022-09-03T05:59:58.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T05:59:58.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI= 2022-09-03T05:59:58.372 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZ6K1WRuhvDg7kphd7xJ6tyqjHI=', 'Content-Length': '0'} 2022-09-03T05:59:58.374 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dcaf48ad7cdf3d81-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.375 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.376 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.376 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.377 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.377 DEBUG:boto:Method: GET 2022-09-03T05:59:58.378 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.378 DEBUG:boto:Data: 2022-09-03T05:59:58.379 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.380 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.381 DEBUG:boto:Params: {} 2022-09-03T05:59:58.381 DEBUG:boto:Token: None 2022-09-03T05:59:58.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.385 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000b60fe886c49ba47-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.386 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.387 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.388 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.388 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.389 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.389 DEBUG:boto:Method: GET 2022-09-03T05:59:58.390 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T05:59:58.390 DEBUG:boto:Data: 2022-09-03T05:59:58.391 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.391 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.392 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.392 DEBUG:boto:Params: {} 2022-09-03T05:59:58.393 DEBUG:boto:Token: None 2022-09-03T05:59:58.393 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.394 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.395 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.398 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000924b98b5f901a836-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.442 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.443 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:58.444 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.444 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.445 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.445 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.446 DEBUG:boto:Method: GET 2022-09-03T05:59:58.446 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.447 DEBUG:boto:Data: 2022-09-03T05:59:58.448 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.449 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.450 DEBUG:boto:Params: {} 2022-09-03T05:59:58.450 DEBUG:boto:Token: None 2022-09-03T05:59:58.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg= 2022-09-03T05:59:58.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg=', 'Content-Length': '0'} 2022-09-03T05:59:58.455 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000469dada69971b4a8-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.455 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.456 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.457 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.457 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.458 DEBUG:boto:Method: GET 2022-09-03T05:59:58.458 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.459 DEBUG:boto:Data: 2022-09-03T05:59:58.459 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.460 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.460 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.461 DEBUG:boto:Params: {} 2022-09-03T05:59:58.462 DEBUG:boto:Token: None 2022-09-03T05:59:58.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg= 2022-09-03T05:59:58.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg=', 'Content-Length': '0'} 2022-09-03T05:59:58.466 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e699da2ae3208e74-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.467 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.467 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.468 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.468 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.469 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.469 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.470 DEBUG:boto:Data: 2022-09-03T05:59:58.470 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.471 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.471 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.472 DEBUG:boto:Params: {} 2022-09-03T05:59:58.473 DEBUG:boto:Token: None 2022-09-03T05:59:58.473 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NyQ/e8ZzNQZEqBnjZSrC6k2K240= 2022-09-03T05:59:58.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NyQ/e8ZzNQZEqBnjZSrC6k2K240=', 'Content-Length': '0'} 2022-09-03T05:59:58.477 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007b5f138250793203-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.477 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.478 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.478 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.479 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.480 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.480 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.481 DEBUG:boto:Data: 2022-09-03T05:59:58.481 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.482 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.482 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.483 DEBUG:boto:Params: {} 2022-09-03T05:59:58.483 DEBUG:boto:Token: None 2022-09-03T05:59:58.484 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NyQ/e8ZzNQZEqBnjZSrC6k2K240= 2022-09-03T05:59:58.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NyQ/e8ZzNQZEqBnjZSrC6k2K240=', 'Content-Length': '0'} 2022-09-03T05:59:58.487 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b67ce23e27e229f1-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.488 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:58.489 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.490 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.490 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.491 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.491 DEBUG:boto:Method: GET 2022-09-03T05:59:58.492 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.492 DEBUG:boto:Data: 2022-09-03T05:59:58.493 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.494 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.494 DEBUG:boto:Params: {} 2022-09-03T05:59:58.495 DEBUG:boto:Token: None 2022-09-03T05:59:58.496 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg= 2022-09-03T05:59:58.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg=', 'Content-Length': '0'} 2022-09-03T05:59:58.499 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008d7bc3d2bafdaf03-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.546 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.547 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.547 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.548 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.548 DEBUG:boto:Method: GET 2022-09-03T05:59:58.549 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.550 DEBUG:boto:Data: 2022-09-03T05:59:58.550 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.551 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.551 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.552 DEBUG:boto:Params: {} 2022-09-03T05:59:58.552 DEBUG:boto:Token: None 2022-09-03T05:59:58.553 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg= 2022-09-03T05:59:58.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg=', 'Content-Length': '0'} 2022-09-03T05:59:58.557 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000053e2d074503d3bd1-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.557 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.558 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.559 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.559 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.560 DEBUG:boto:Method: GET 2022-09-03T05:59:58.560 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.561 DEBUG:boto:Data: 2022-09-03T05:59:58.561 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.562 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.562 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.563 DEBUG:boto:Params: {} 2022-09-03T05:59:58.564 DEBUG:boto:Token: None 2022-09-03T05:59:58.564 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.565 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.565 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.569 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000505017cbb3555c4c-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.569 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.570 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.571 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.571 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.572 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.573 DEBUG:boto:Method: GET 2022-09-03T05:59:58.573 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.574 DEBUG:boto:Data: 2022-09-03T05:59:58.575 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.575 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.576 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.576 DEBUG:boto:Params: {} 2022-09-03T05:59:58.577 DEBUG:boto:Token: None 2022-09-03T05:59:58.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.581 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1531af308d38a71-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.582 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.583 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.583 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.584 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.584 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T05:59:58.585 DEBUG:boto:Data: 2022-09-03T05:59:58.585 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.586 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.587 DEBUG:boto:Params: {} 2022-09-03T05:59:58.587 DEBUG:boto:Token: None 2022-09-03T05:59:58.588 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2 2022-09-03T05:59:58.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5SK5P7uY901FqDF4znPdO8FVuC4= 2022-09-03T05:59:58.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5SK5P7uY901FqDF4znPdO8FVuC4=', 'Content-Length': '0'} 2022-09-03T05:59:58.592 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f04341495ddd4454-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.593 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.593 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.594 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.594 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T05:59:58.595 DEBUG:boto:Data: 2022-09-03T05:59:58.595 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.596 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.596 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.597 DEBUG:boto:Params: {} 2022-09-03T05:59:58.597 DEBUG:boto:Token: None 2022-09-03T05:59:58.598 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2 2022-09-03T05:59:58.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5SK5P7uY901FqDF4znPdO8FVuC4= 2022-09-03T05:59:58.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5SK5P7uY901FqDF4znPdO8FVuC4=', 'Content-Length': '0'} 2022-09-03T05:59:58.602 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000aa4b002c9ac78e3a-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.603 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T05:59:58.603 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.604 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.604 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.605 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.605 DEBUG:boto:Method: GET 2022-09-03T05:59:58.606 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.607 DEBUG:boto:Data: 2022-09-03T05:59:58.607 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.608 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.609 DEBUG:boto:Params: {} 2022-09-03T05:59:58.609 DEBUG:boto:Token: None 2022-09-03T05:59:58.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg= 2022-09-03T05:59:58.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg=', 'Content-Length': '0'} 2022-09-03T05:59:58.614 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c9725e05412f1b90-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.615 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.615 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:58.616 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.616 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.617 DEBUG:boto:Method: GET 2022-09-03T05:59:58.617 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.618 DEBUG:boto:Data: 2022-09-03T05:59:58.618 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.619 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.620 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.620 DEBUG:boto:Params: {} 2022-09-03T05:59:58.621 DEBUG:boto:Token: None 2022-09-03T05:59:58.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T05:59:58.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg= 2022-09-03T05:59:58.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FcR6dmK9ahQHmaS5VMJNGtGNQmg=', 'Content-Length': '0'} 2022-09-03T05:59:58.625 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000edfb7a53fa4e1e86-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.626 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.626 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.627 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.627 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.628 DEBUG:boto:Method: GET 2022-09-03T05:59:58.628 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.629 DEBUG:boto:Data: 2022-09-03T05:59:58.629 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.630 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.631 DEBUG:boto:Params: {} 2022-09-03T05:59:58.632 DEBUG:boto:Token: None 2022-09-03T05:59:58.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.636 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009bbd4603df8cb472-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.637 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.638 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.638 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.639 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.639 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.640 DEBUG:boto:Method: GET 2022-09-03T05:59:58.640 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T05:59:58.641 DEBUG:boto:Data: 2022-09-03T05:59:58.642 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.642 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.643 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.643 DEBUG:boto:Params: {} 2022-09-03T05:59:58.644 DEBUG:boto:Token: None 2022-09-03T05:59:58.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.648 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002760c5cce731bff2-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.649 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.650 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:58.651 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.651 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.652 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.652 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.653 DEBUG:boto:Method: GET 2022-09-03T05:59:58.653 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.654 DEBUG:boto:Data: 2022-09-03T05:59:58.655 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.656 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.656 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.657 DEBUG:boto:Params: {} 2022-09-03T05:59:58.657 DEBUG:boto:Token: None 2022-09-03T05:59:58.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk= 2022-09-03T05:59:58.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk=', 'Content-Length': '0'} 2022-09-03T05:59:58.661 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006311ba2ab5653da5-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.662 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.663 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.663 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.664 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.664 DEBUG:boto:Method: GET 2022-09-03T05:59:58.665 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.666 DEBUG:boto:Data: 2022-09-03T05:59:58.666 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.667 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.667 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.668 DEBUG:boto:Params: {} 2022-09-03T05:59:58.668 DEBUG:boto:Token: None 2022-09-03T05:59:58.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk= 2022-09-03T05:59:58.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk=', 'Content-Length': '0'} 2022-09-03T05:59:58.673 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000abb51f7cd17ace73-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.673 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.674 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.675 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.675 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.676 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.676 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.677 DEBUG:boto:Data: 2022-09-03T05:59:58.677 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.678 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.679 DEBUG:boto:Params: {} 2022-09-03T05:59:58.679 DEBUG:boto:Token: None 2022-09-03T05:59:58.680 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPEhjwRrMZ3PqqhM4BoZT1ncyCo= 2022-09-03T05:59:58.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPEhjwRrMZ3PqqhM4BoZT1ncyCo=', 'Content-Length': '0'} 2022-09-03T05:59:58.683 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000413a865acdac6f29-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.684 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.684 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.685 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.685 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.686 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.686 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.687 DEBUG:boto:Data: 2022-09-03T05:59:58.687 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.688 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.688 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.689 DEBUG:boto:Params: {} 2022-09-03T05:59:58.689 DEBUG:boto:Token: None 2022-09-03T05:59:58.690 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPEhjwRrMZ3PqqhM4BoZT1ncyCo= 2022-09-03T05:59:58.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPEhjwRrMZ3PqqhM4BoZT1ncyCo=', 'Content-Length': '0'} 2022-09-03T05:59:58.694 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002bcea59ee87ced31-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.695 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:58.695 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.696 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.696 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.697 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.697 DEBUG:boto:Method: GET 2022-09-03T05:59:58.698 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.699 DEBUG:boto:Data: 2022-09-03T05:59:58.699 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.700 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.700 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.701 DEBUG:boto:Params: {} 2022-09-03T05:59:58.702 DEBUG:boto:Token: None 2022-09-03T05:59:58.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk= 2022-09-03T05:59:58.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk=', 'Content-Length': '0'} 2022-09-03T05:59:58.707 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e9669f72fb45b1c9-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.708 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.709 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.710 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.711 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.713 DEBUG:boto:Method: GET 2022-09-03T05:59:58.714 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.714 DEBUG:boto:Data: 2022-09-03T05:59:58.715 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.715 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.716 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.717 DEBUG:boto:Params: {} 2022-09-03T05:59:58.718 DEBUG:boto:Token: None 2022-09-03T05:59:58.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk= 2022-09-03T05:59:58.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk=', 'Content-Length': '0'} 2022-09-03T05:59:58.723 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000df19843b0c0e2108-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.724 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.725 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.725 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.726 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.726 DEBUG:boto:Method: GET 2022-09-03T05:59:58.727 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.728 DEBUG:boto:Data: 2022-09-03T05:59:58.729 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.730 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.731 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.731 DEBUG:boto:Params: {} 2022-09-03T05:59:58.732 DEBUG:boto:Token: None 2022-09-03T05:59:58.733 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.738 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e768d8bf0b3c23b6-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.739 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.741 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.742 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.742 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.743 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.744 DEBUG:boto:Method: GET 2022-09-03T05:59:58.744 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.745 DEBUG:boto:Data: 2022-09-03T05:59:58.746 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.747 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.748 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.748 DEBUG:boto:Params: {} 2022-09-03T05:59:58.749 DEBUG:boto:Token: None 2022-09-03T05:59:58.750 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.754 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000008c579b2965bcfbe-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.802 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.804 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.805 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.806 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.807 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T05:59:58.808 DEBUG:boto:Data: 2022-09-03T05:59:58.809 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.811 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.812 DEBUG:boto:Params: {} 2022-09-03T05:59:58.813 DEBUG:boto:Token: None 2022-09-03T05:59:58.814 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3 2022-09-03T05:59:58.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nXNP9NfOat//Fbjvk4ID/N1vGV4= 2022-09-03T05:59:58.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nXNP9NfOat//Fbjvk4ID/N1vGV4=', 'Content-Length': '0'} 2022-09-03T05:59:58.819 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f65e83d314cae2d6-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.821 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.822 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.822 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.823 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T05:59:58.824 DEBUG:boto:Data: 2022-09-03T05:59:58.825 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.826 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.827 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.828 DEBUG:boto:Params: {} 2022-09-03T05:59:58.829 DEBUG:boto:Token: None 2022-09-03T05:59:58.830 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3 2022-09-03T05:59:58.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nXNP9NfOat//Fbjvk4ID/N1vGV4= 2022-09-03T05:59:58.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nXNP9NfOat//Fbjvk4ID/N1vGV4=', 'Content-Length': '0'} 2022-09-03T05:59:58.835 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000852f132e166e6973-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.837 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T05:59:58.838 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.838 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.839 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.840 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.841 DEBUG:boto:Method: GET 2022-09-03T05:59:58.842 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.842 DEBUG:boto:Data: 2022-09-03T05:59:58.843 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.844 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.845 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.845 DEBUG:boto:Params: {} 2022-09-03T05:59:58.846 DEBUG:boto:Token: None 2022-09-03T05:59:58.847 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.848 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk= 2022-09-03T05:59:58.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk=', 'Content-Length': '0'} 2022-09-03T05:59:58.851 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000023359c951ca0297b-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.894 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.895 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:58.895 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.896 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.897 DEBUG:boto:Method: GET 2022-09-03T05:59:58.897 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.898 DEBUG:boto:Data: 2022-09-03T05:59:58.898 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.899 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.900 DEBUG:boto:Params: {} 2022-09-03T05:59:58.900 DEBUG:boto:Token: None 2022-09-03T05:59:58.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T05:59:58.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk= 2022-09-03T05:59:58.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:np4w0cCn8peh8H5APY+8svLEtqk=', 'Content-Length': '0'} 2022-09-03T05:59:58.904 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:57:15 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001eff49a21879d1eb-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.905 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.906 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.907 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.907 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.908 DEBUG:boto:Method: GET 2022-09-03T05:59:58.908 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.909 DEBUG:boto:Data: 2022-09-03T05:59:58.909 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.910 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:58.910 DEBUG:boto:Port: 8001 2022-09-03T05:59:58.911 DEBUG:boto:Params: {} 2022-09-03T05:59:58.911 DEBUG:boto:Token: None 2022-09-03T05:59:58.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.916 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020da5bfb3c30ac8c-006312ed5e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.917 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.917 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:58.918 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:58.919 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.919 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.920 DEBUG:boto:Method: GET 2022-09-03T05:59:58.920 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T05:59:58.921 DEBUG:boto:Data: 2022-09-03T05:59:58.921 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.922 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.922 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.923 DEBUG:boto:Params: {} 2022-09-03T05:59:58.923 DEBUG:boto:Token: None 2022-09-03T05:59:58.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-18/ 2022-09-03T05:59:58.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s= 2022-09-03T05:59:58.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hCaT8ZG6h1JSYu94dUF1p9xSV2s=', 'Content-Length': '0'} 2022-09-03T05:59:58.928 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a94b0c81571a5b20-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.928 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.929 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-18 zones={test-zone3, test-zone2} 2022-09-03T05:59:58.930 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:58.930 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:58.931 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.931 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:59:58.932 DEBUG:boto:Data: 2022-09-03T05:59:58.932 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:58.933 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.934 DEBUG:boto:Params: {} 2022-09-03T05:59:58.934 DEBUG:boto:Token: None 2022-09-03T05:59:58.935 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-16/ 2022-09-03T05:59:58.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sUu3d2FKKS1FBY9Iwnxhy79q0ao= 2022-09-03T05:59:58.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sUu3d2FKKS1FBY9Iwnxhy79q0ao=', 'Content-Length': '0'} 2022-09-03T05:59:58.939 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ca80154b13eee8e1-006312ed5e-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.940 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:58.940 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:58.941 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999&prefix=obj-test-zone1 2022-09-03T05:59:58.942 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999&prefix=obj-test-zone1 2022-09-03T05:59:58.942 DEBUG:boto:Method: GET 2022-09-03T05:59:58.943 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999&prefix=obj-test-zone1 2022-09-03T05:59:58.943 DEBUG:boto:Data: 2022-09-03T05:59:58.944 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:58.945 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.945 DEBUG:boto:Params: {} 2022-09-03T05:59:58.946 DEBUG:boto:Token: None 2022-09-03T05:59:58.947 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-16/?versions 2022-09-03T05:59:58.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:28bQzag0mjoRnfyLMBvHRwWZiXg= 2022-09-03T05:59:58.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:28bQzag0mjoRnfyLMBvHRwWZiXg=', 'Content-Length': '0'} 2022-09-03T05:59:58.951 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023d7475385bb00b2-006312ed5e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.952 DEBUG:boto:b'avysmw-16obj-test-zone1999falseobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.954 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:58.955 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T05:59:58.955 DEBUG:boto:path=/avysmw-16/obj-test-zone1?acl 2022-09-03T05:59:58.956 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?acl 2022-09-03T05:59:58.957 DEBUG:boto:Method: PUT 2022-09-03T05:59:58.958 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?acl 2022-09-03T05:59:58.958 DEBUG:boto:Data: 2022-09-03T05:59:58.959 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T05:59:58.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:58.960 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.960 DEBUG:boto:Params: {} 2022-09-03T05:59:58.961 DEBUG:boto:Token: None 2022-09-03T05:59:58.962 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:59:58 GMT x-amz-acl:public-read /avysmw-16/obj-test-zone1?acl 2022-09-03T05:59:58.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z+E+2uEk4ayAib+2cXLf7V1QDcE= 2022-09-03T05:59:58.963 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z+E+2uEk4ayAib+2cXLf7V1QDcE=', 'Content-Length': '0'} 2022-09-03T05:59:58.971 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000a2ba921582b1eed2-006312ed5e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.972 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:58.973 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:58.973 DEBUG:boto:Method: HEAD 2022-09-03T05:59:58.974 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:59:58.974 DEBUG:boto:Data: 2022-09-03T05:59:58.975 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.975 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.976 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.976 DEBUG:boto:Params: {} 2022-09-03T05:59:58.977 DEBUG:boto:Token: None 2022-09-03T05:59:58.979 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:58 GMT /avysmw-16/ 2022-09-03T05:59:58.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sUu3d2FKKS1FBY9Iwnxhy79q0ao= 2022-09-03T05:59:58.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sUu3d2FKKS1FBY9Iwnxhy79q0ao=', 'Content-Length': '0'} 2022-09-03T05:59:58.984 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b2c6554b2afccd3e-006312ed5e-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.984 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:58.985 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:58.985 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999&prefix=obj-test-zone2 2022-09-03T05:59:58.986 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999&prefix=obj-test-zone2 2022-09-03T05:59:58.987 DEBUG:boto:Method: GET 2022-09-03T05:59:58.987 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999&prefix=obj-test-zone2 2022-09-03T05:59:58.988 DEBUG:boto:Data: 2022-09-03T05:59:58.988 DEBUG:boto:Headers: {} 2022-09-03T05:59:58.989 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:58.989 DEBUG:boto:Port: 8000 2022-09-03T05:59:58.990 DEBUG:boto:Params: {} 2022-09-03T05:59:58.991 DEBUG:boto:Token: None 2022-09-03T05:59:58.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:58 GMT /avysmw-16/?versions 2022-09-03T05:59:58.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:28bQzag0mjoRnfyLMBvHRwWZiXg= 2022-09-03T05:59:58.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:28bQzag0mjoRnfyLMBvHRwWZiXg=', 'Content-Length': '0'} 2022-09-03T05:59:58.995 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff87b60bcf669bb6-006312ed5e-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:58.996 DEBUG:boto:b'avysmw-16obj-test-zone2999falseobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:58.998 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:58.999 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T05:59:59.000 DEBUG:boto:path=/avysmw-16/obj-test-zone2?acl 2022-09-03T05:59:59.001 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?acl 2022-09-03T05:59:59.001 DEBUG:boto:Method: PUT 2022-09-03T05:59:59.002 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?acl 2022-09-03T05:59:59.003 DEBUG:boto:Data: 2022-09-03T05:59:59.004 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T05:59:59.004 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:59.005 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.005 DEBUG:boto:Params: {} 2022-09-03T05:59:59.006 DEBUG:boto:Token: None 2022-09-03T05:59:59.007 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:59:59 GMT x-amz-acl:public-read /avysmw-16/obj-test-zone2?acl 2022-09-03T05:59:59.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzByfW7Te+wv8FzmzdkYXuYOlS8= 2022-09-03T05:59:59.008 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzByfW7Te+wv8FzmzdkYXuYOlS8=', 'Content-Length': '0'} 2022-09-03T05:59:59.015 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000003b28e30c8601c717-006312ed5f-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.016 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:59.017 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:59.018 DEBUG:boto:Method: HEAD 2022-09-03T05:59:59.019 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T05:59:59.019 DEBUG:boto:Data: 2022-09-03T05:59:59.020 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:59.020 DEBUG:boto:Port: 8001 2022-09-03T05:59:59.021 DEBUG:boto:Params: {} 2022-09-03T05:59:59.022 DEBUG:boto:Token: None 2022-09-03T05:59:59.022 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:59 GMT /avysmw-16/ 2022-09-03T05:59:59.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e9BI66llXG/RPvHc+gK0YFMCfHI= 2022-09-03T05:59:59.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e9BI66llXG/RPvHc+gK0YFMCfHI=', 'Content-Length': '0'} 2022-09-03T05:59:59.027 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d3c7beaaee520826-006312ed5f-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.028 DEBUG:boto:path=/avysmw-16/ 2022-09-03T05:59:59.029 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T05:59:59.030 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999&prefix=obj-test-zone3 2022-09-03T05:59:59.031 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999&prefix=obj-test-zone3 2022-09-03T05:59:59.031 DEBUG:boto:Method: GET 2022-09-03T05:59:59.032 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999&prefix=obj-test-zone3 2022-09-03T05:59:59.033 DEBUG:boto:Data: 2022-09-03T05:59:59.034 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:59.035 DEBUG:boto:Port: 8001 2022-09-03T05:59:59.036 DEBUG:boto:Params: {} 2022-09-03T05:59:59.036 DEBUG:boto:Token: None 2022-09-03T05:59:59.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:59 GMT /avysmw-16/?versions 2022-09-03T05:59:59.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EfMd+jM66RoqwxCTaMxG3aEXyXc= 2022-09-03T05:59:59.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EfMd+jM66RoqwxCTaMxG3aEXyXc=', 'Content-Length': '0'} 2022-09-03T05:59:59.041 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009c60e233c7c46afa-006312ed5f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.041 DEBUG:boto:b'avysmw-16obj-test-zone3999falseobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:59.042 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:59.043 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T05:59:59.043 DEBUG:boto:path=/avysmw-16/obj-test-zone3?acl 2022-09-03T05:59:59.044 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?acl 2022-09-03T05:59:59.044 DEBUG:boto:Method: PUT 2022-09-03T05:59:59.045 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?acl 2022-09-03T05:59:59.045 DEBUG:boto:Data: 2022-09-03T05:59:59.046 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T05:59:59.046 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:59.047 DEBUG:boto:Port: 8001 2022-09-03T05:59:59.047 DEBUG:boto:Params: {} 2022-09-03T05:59:59.048 DEBUG:boto:Token: None 2022-09-03T05:59:59.048 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:59:59 GMT x-amz-acl:public-read /avysmw-16/obj-test-zone3?acl 2022-09-03T05:59:59.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BzPVYZAlV3wQKzLVa0bDAdhV1Ww= 2022-09-03T05:59:59.050 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BzPVYZAlV3wQKzLVa0bDAdhV1Ww=', 'Content-Length': '0'} 2022-09-03T05:59:59.058 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000134289209d679a4c-006312ed5f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.059 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:59.060 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:59.061 DEBUG:boto:Method: HEAD 2022-09-03T05:59:59.061 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:59:59.063 DEBUG:boto:Data: 2022-09-03T05:59:59.064 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.065 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:59.066 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.067 DEBUG:boto:Params: {} 2022-09-03T05:59:59.069 DEBUG:boto:Token: None 2022-09-03T05:59:59.070 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:59 GMT /avysmw-17/ 2022-09-03T05:59:59.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Cecd7OeXgGldGMVu4dhsgXcrW4= 2022-09-03T05:59:59.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Cecd7OeXgGldGMVu4dhsgXcrW4=', 'Content-Length': '0'} 2022-09-03T05:59:59.077 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000396551816fff921a-006312ed5f-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.077 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:59.078 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:59.079 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999&prefix=obj-test-zone1 2022-09-03T05:59:59.079 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999&prefix=obj-test-zone1 2022-09-03T05:59:59.080 DEBUG:boto:Method: GET 2022-09-03T05:59:59.080 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999&prefix=obj-test-zone1 2022-09-03T05:59:59.081 DEBUG:boto:Data: 2022-09-03T05:59:59.081 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:59.082 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.083 DEBUG:boto:Params: {} 2022-09-03T05:59:59.083 DEBUG:boto:Token: None 2022-09-03T05:59:59.084 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:59 GMT /avysmw-17/?versions 2022-09-03T05:59:59.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JDrfbtXD5FHgmC2l5ror+/w7JQI= 2022-09-03T05:59:59.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JDrfbtXD5FHgmC2l5ror+/w7JQI=', 'Content-Length': '0'} 2022-09-03T05:59:59.088 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dd2b5b349e37efac-006312ed5f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.088 DEBUG:boto:b'avysmw-17obj-test-zone1999falseobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:59.089 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:59.090 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T05:59:59.090 DEBUG:boto:path=/avysmw-17/obj-test-zone1?acl 2022-09-03T05:59:59.091 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?acl 2022-09-03T05:59:59.091 DEBUG:boto:Method: PUT 2022-09-03T05:59:59.092 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?acl 2022-09-03T05:59:59.092 DEBUG:boto:Data: 2022-09-03T05:59:59.093 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T05:59:59.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:59.094 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.095 DEBUG:boto:Params: {} 2022-09-03T05:59:59.096 DEBUG:boto:Token: None 2022-09-03T05:59:59.096 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:59:59 GMT x-amz-acl:public-read /avysmw-17/obj-test-zone1?acl 2022-09-03T05:59:59.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:myE3hZqRCElULp80YdOtuC82FSI= 2022-09-03T05:59:59.097 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:myE3hZqRCElULp80YdOtuC82FSI=', 'Content-Length': '0'} 2022-09-03T05:59:59.110 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000150c6b05705b1754-006312ed5f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.111 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:59.111 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:59.112 DEBUG:boto:Method: HEAD 2022-09-03T05:59:59.112 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:59:59.113 DEBUG:boto:Data: 2022-09-03T05:59:59.113 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.114 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:59.114 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.115 DEBUG:boto:Params: {} 2022-09-03T05:59:59.115 DEBUG:boto:Token: None 2022-09-03T05:59:59.116 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:59 GMT /avysmw-17/ 2022-09-03T05:59:59.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Cecd7OeXgGldGMVu4dhsgXcrW4= 2022-09-03T05:59:59.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Cecd7OeXgGldGMVu4dhsgXcrW4=', 'Content-Length': '0'} 2022-09-03T05:59:59.120 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d0e76b61cedb9719-006312ed5f-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.120 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:59.121 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:59.121 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999&prefix=obj-test-zone2 2022-09-03T05:59:59.122 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999&prefix=obj-test-zone2 2022-09-03T05:59:59.122 DEBUG:boto:Method: GET 2022-09-03T05:59:59.123 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999&prefix=obj-test-zone2 2022-09-03T05:59:59.123 DEBUG:boto:Data: 2022-09-03T05:59:59.124 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.124 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:59.125 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.125 DEBUG:boto:Params: {} 2022-09-03T05:59:59.126 DEBUG:boto:Token: None 2022-09-03T05:59:59.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:59 GMT /avysmw-17/?versions 2022-09-03T05:59:59.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JDrfbtXD5FHgmC2l5ror+/w7JQI= 2022-09-03T05:59:59.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JDrfbtXD5FHgmC2l5ror+/w7JQI=', 'Content-Length': '0'} 2022-09-03T05:59:59.133 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000026eca36358038ed6-006312ed5f-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.134 DEBUG:boto:b'avysmw-17obj-test-zone2999falseobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:59.134 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:59.135 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T05:59:59.136 DEBUG:boto:path=/avysmw-17/obj-test-zone2?acl 2022-09-03T05:59:59.137 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?acl 2022-09-03T05:59:59.137 DEBUG:boto:Method: PUT 2022-09-03T05:59:59.138 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?acl 2022-09-03T05:59:59.139 DEBUG:boto:Data: 2022-09-03T05:59:59.139 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T05:59:59.140 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:59.140 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.141 DEBUG:boto:Params: {} 2022-09-03T05:59:59.141 DEBUG:boto:Token: None 2022-09-03T05:59:59.143 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:59:59 GMT x-amz-acl:public-read /avysmw-17/obj-test-zone2?acl 2022-09-03T05:59:59.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P7Ukp6qb7LOho9QmHWwX7gHhPjQ= 2022-09-03T05:59:59.144 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P7Ukp6qb7LOho9QmHWwX7gHhPjQ=', 'Content-Length': '0'} 2022-09-03T05:59:59.150 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000cdef9a91a35e8648-006312ed5f-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.151 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:59.152 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:59.152 DEBUG:boto:Method: HEAD 2022-09-03T05:59:59.153 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T05:59:59.153 DEBUG:boto:Data: 2022-09-03T05:59:59.154 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.154 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:59.155 DEBUG:boto:Port: 8001 2022-09-03T05:59:59.155 DEBUG:boto:Params: {} 2022-09-03T05:59:59.156 DEBUG:boto:Token: None 2022-09-03T05:59:59.156 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:59 GMT /avysmw-17/ 2022-09-03T05:59:59.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Cecd7OeXgGldGMVu4dhsgXcrW4= 2022-09-03T05:59:59.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Cecd7OeXgGldGMVu4dhsgXcrW4=', 'Content-Length': '0'} 2022-09-03T05:59:59.160 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cb9530d1311e70b9-006312ed5f-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.161 DEBUG:boto:path=/avysmw-17/ 2022-09-03T05:59:59.161 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T05:59:59.162 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999&prefix=obj-test-zone3 2022-09-03T05:59:59.162 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999&prefix=obj-test-zone3 2022-09-03T05:59:59.163 DEBUG:boto:Method: GET 2022-09-03T05:59:59.163 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999&prefix=obj-test-zone3 2022-09-03T05:59:59.164 DEBUG:boto:Data: 2022-09-03T05:59:59.165 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:59.166 DEBUG:boto:Port: 8001 2022-09-03T05:59:59.166 DEBUG:boto:Params: {} 2022-09-03T05:59:59.167 DEBUG:boto:Token: None 2022-09-03T05:59:59.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:59 GMT /avysmw-17/?versions 2022-09-03T05:59:59.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JDrfbtXD5FHgmC2l5ror+/w7JQI= 2022-09-03T05:59:59.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JDrfbtXD5FHgmC2l5ror+/w7JQI=', 'Content-Length': '0'} 2022-09-03T05:59:59.171 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d55e3a92e53992d7-006312ed5f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.172 DEBUG:boto:b'avysmw-17obj-test-zone3999falseobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:59.172 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:59.173 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T05:59:59.173 DEBUG:boto:path=/avysmw-17/obj-test-zone3?acl 2022-09-03T05:59:59.174 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?acl 2022-09-03T05:59:59.175 DEBUG:boto:Method: PUT 2022-09-03T05:59:59.175 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?acl 2022-09-03T05:59:59.175 DEBUG:boto:Data: 2022-09-03T05:59:59.176 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T05:59:59.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:59.177 DEBUG:boto:Port: 8001 2022-09-03T05:59:59.177 DEBUG:boto:Params: {} 2022-09-03T05:59:59.178 DEBUG:boto:Token: None 2022-09-03T05:59:59.179 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:59:59 GMT x-amz-acl:public-read /avysmw-17/obj-test-zone3?acl 2022-09-03T05:59:59.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vrkKmJJG4Ks3+RnnPljc95u76bs= 2022-09-03T05:59:59.180 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vrkKmJJG4Ks3+RnnPljc95u76bs=', 'Content-Length': '0'} 2022-09-03T05:59:59.189 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000065cb2f3ee5ea5438-006312ed5f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.190 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:59.191 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:59.191 DEBUG:boto:Method: HEAD 2022-09-03T05:59:59.192 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:59:59.192 DEBUG:boto:Data: 2022-09-03T05:59:59.193 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.193 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:59.194 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.194 DEBUG:boto:Params: {} 2022-09-03T05:59:59.195 DEBUG:boto:Token: None 2022-09-03T05:59:59.195 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:59 GMT /avysmw-18/ 2022-09-03T05:59:59.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjcILSMTzwQ64vlAuBpOUnmBgvI= 2022-09-03T05:59:59.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjcILSMTzwQ64vlAuBpOUnmBgvI=', 'Content-Length': '0'} 2022-09-03T05:59:59.200 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003e02861f375da682-006312ed5f-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.200 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:59.201 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:59.201 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999&prefix=obj-test-zone1 2022-09-03T05:59:59.202 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999&prefix=obj-test-zone1 2022-09-03T05:59:59.202 DEBUG:boto:Method: GET 2022-09-03T05:59:59.203 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999&prefix=obj-test-zone1 2022-09-03T05:59:59.203 DEBUG:boto:Data: 2022-09-03T05:59:59.204 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.204 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:59.205 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.205 DEBUG:boto:Params: {} 2022-09-03T05:59:59.206 DEBUG:boto:Token: None 2022-09-03T05:59:59.207 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:59 GMT /avysmw-18/?versions 2022-09-03T05:59:59.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Bfg9iuF4iyn7rMtA7SmbKachHQ= 2022-09-03T05:59:59.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Bfg9iuF4iyn7rMtA7SmbKachHQ=', 'Content-Length': '0'} 2022-09-03T05:59:59.210 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aca7b6b4bafbebf6-006312ed5f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.254 DEBUG:boto:b'avysmw-18obj-test-zone1999falseobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:59.255 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:59.256 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T05:59:59.256 DEBUG:boto:path=/avysmw-18/obj-test-zone1?acl 2022-09-03T05:59:59.256 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?acl 2022-09-03T05:59:59.257 DEBUG:boto:Method: PUT 2022-09-03T05:59:59.257 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?acl 2022-09-03T05:59:59.258 DEBUG:boto:Data: 2022-09-03T05:59:59.258 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T05:59:59.259 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T05:59:59.259 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.260 DEBUG:boto:Params: {} 2022-09-03T05:59:59.261 DEBUG:boto:Token: None 2022-09-03T05:59:59.261 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:59:59 GMT x-amz-acl:public-read /avysmw-18/obj-test-zone1?acl 2022-09-03T05:59:59.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sxvIMiDlG7O1neOGtCF4JXoD39Y= 2022-09-03T05:59:59.263 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sxvIMiDlG7O1neOGtCF4JXoD39Y=', 'Content-Length': '0'} 2022-09-03T05:59:59.275 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000f7c7cb2edcbd98e1-006312ed5f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.276 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:59.277 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:59.277 DEBUG:boto:Method: HEAD 2022-09-03T05:59:59.278 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:59:59.278 DEBUG:boto:Data: 2022-09-03T05:59:59.279 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.279 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:59.280 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.280 DEBUG:boto:Params: {} 2022-09-03T05:59:59.281 DEBUG:boto:Token: None 2022-09-03T05:59:59.281 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:59 GMT /avysmw-18/ 2022-09-03T05:59:59.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjcILSMTzwQ64vlAuBpOUnmBgvI= 2022-09-03T05:59:59.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjcILSMTzwQ64vlAuBpOUnmBgvI=', 'Content-Length': '0'} 2022-09-03T05:59:59.285 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ac6e8caae9f2e87d-006312ed5f-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.286 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:59.287 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:59.287 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999&prefix=obj-test-zone2 2022-09-03T05:59:59.288 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999&prefix=obj-test-zone2 2022-09-03T05:59:59.288 DEBUG:boto:Method: GET 2022-09-03T05:59:59.289 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999&prefix=obj-test-zone2 2022-09-03T05:59:59.289 DEBUG:boto:Data: 2022-09-03T05:59:59.290 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.290 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:59.291 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.291 DEBUG:boto:Params: {} 2022-09-03T05:59:59.292 DEBUG:boto:Token: None 2022-09-03T05:59:59.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:59 GMT /avysmw-18/?versions 2022-09-03T05:59:59.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Bfg9iuF4iyn7rMtA7SmbKachHQ= 2022-09-03T05:59:59.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Bfg9iuF4iyn7rMtA7SmbKachHQ=', 'Content-Length': '0'} 2022-09-03T05:59:59.297 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004135a54c4d18f573-006312ed5f-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.298 DEBUG:boto:b'avysmw-18obj-test-zone2999falseobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:59.299 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:59.299 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T05:59:59.300 DEBUG:boto:path=/avysmw-18/obj-test-zone2?acl 2022-09-03T05:59:59.300 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?acl 2022-09-03T05:59:59.301 DEBUG:boto:Method: PUT 2022-09-03T05:59:59.301 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?acl 2022-09-03T05:59:59.301 DEBUG:boto:Data: 2022-09-03T05:59:59.302 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T05:59:59.302 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T05:59:59.303 DEBUG:boto:Port: 8000 2022-09-03T05:59:59.304 DEBUG:boto:Params: {} 2022-09-03T05:59:59.304 DEBUG:boto:Token: None 2022-09-03T05:59:59.305 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:59:59 GMT x-amz-acl:public-read /avysmw-18/obj-test-zone2?acl 2022-09-03T05:59:59.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WsDf7Gf+Dr/x3NGn4ULqQAt5EV0= 2022-09-03T05:59:59.306 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WsDf7Gf+Dr/x3NGn4ULqQAt5EV0=', 'Content-Length': '0'} 2022-09-03T05:59:59.314 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000084c040ea40ccd559-006312ed5f-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.315 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:59.315 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:59.316 DEBUG:boto:Method: HEAD 2022-09-03T05:59:59.316 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T05:59:59.317 DEBUG:boto:Data: 2022-09-03T05:59:59.317 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.318 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:59.318 DEBUG:boto:Port: 8001 2022-09-03T05:59:59.319 DEBUG:boto:Params: {} 2022-09-03T05:59:59.319 DEBUG:boto:Token: None 2022-09-03T05:59:59.320 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 05:59:59 GMT /avysmw-18/ 2022-09-03T05:59:59.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjcILSMTzwQ64vlAuBpOUnmBgvI= 2022-09-03T05:59:59.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjcILSMTzwQ64vlAuBpOUnmBgvI=', 'Content-Length': '0'} 2022-09-03T05:59:59.324 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000068b9702f2a042e3f-006312ed5f-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.325 DEBUG:boto:path=/avysmw-18/ 2022-09-03T05:59:59.325 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T05:59:59.326 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999&prefix=obj-test-zone3 2022-09-03T05:59:59.327 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999&prefix=obj-test-zone3 2022-09-03T05:59:59.327 DEBUG:boto:Method: GET 2022-09-03T05:59:59.328 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999&prefix=obj-test-zone3 2022-09-03T05:59:59.328 DEBUG:boto:Data: 2022-09-03T05:59:59.329 DEBUG:boto:Headers: {} 2022-09-03T05:59:59.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:59.331 DEBUG:boto:Port: 8001 2022-09-03T05:59:59.332 DEBUG:boto:Params: {} 2022-09-03T05:59:59.333 DEBUG:boto:Token: None 2022-09-03T05:59:59.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 05:59:59 GMT /avysmw-18/?versions 2022-09-03T05:59:59.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Bfg9iuF4iyn7rMtA7SmbKachHQ= 2022-09-03T05:59:59.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Bfg9iuF4iyn7rMtA7SmbKachHQ=', 'Content-Length': '0'} 2022-09-03T05:59:59.338 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c3f73a622860041-006312ed5f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.339 DEBUG:boto:b'avysmw-18obj-test-zone3999falseobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T05:59:59.340 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:59.340 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T05:59:59.341 DEBUG:boto:path=/avysmw-18/obj-test-zone3?acl 2022-09-03T05:59:59.341 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?acl 2022-09-03T05:59:59.342 DEBUG:boto:Method: PUT 2022-09-03T05:59:59.342 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?acl 2022-09-03T05:59:59.343 DEBUG:boto:Data: 2022-09-03T05:59:59.343 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T05:59:59.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T05:59:59.344 DEBUG:boto:Port: 8001 2022-09-03T05:59:59.345 DEBUG:boto:Params: {} 2022-09-03T05:59:59.345 DEBUG:boto:Token: None 2022-09-03T05:59:59.346 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 05:59:59 GMT x-amz-acl:public-read /avysmw-18/obj-test-zone3?acl 2022-09-03T05:59:59.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xmNBocAnXuSBOSa0cpOlk2ZTLR4= 2022-09-03T05:59:59.347 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 05:59:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xmNBocAnXuSBOSa0cpOlk2ZTLR4=', 'Content-Length': '0'} 2022-09-03T05:59:59.354 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c508953858da24c7-006312ed5f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T05:59:59.355 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-16 2022-09-03T05:59:59.355 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:59.356 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T05:59:59.357 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T05:59:59.421 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T05:59:59.421 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T05:59:59.465 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T05:59:59.463+0000 7f1ad7b2e600 1 waiting for incremental sync to catch up: 2022-09-03T05:59:59.465 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T05:59:59.465 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:04.466 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:04.464+0000 7f1ad7b2e600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:04.467 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:04.467 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:09.468 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:09.466+0000 7f1ad7b2e600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:09.469 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:09.469 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:14.470 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:14.467+0000 7f1ad7b2e600 1 bucket sync caught up with source: 2022-09-03T06:00:14.470 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000011.33.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:14.470 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:14.471 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:14.467+0000 7f1ad7b2e600 0 bucket checkpoint complete 2022-09-03T06:00:14.487 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-16 2022-09-03T06:00:14.488 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:00:14.489 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:00:14.490 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:00:14.556 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:00:14.556 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:00:14.603 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:00:14.601+0000 7f291602b600 1 bucket sync caught up with source: 2022-09-03T06:00:14.604 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:14.604 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:14.605 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:00:14.601+0000 7f291602b600 0 bucket checkpoint complete 2022-09-03T06:00:14.617 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-16 2022-09-03T06:00:14.619 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:00:14.621 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:00:14.622 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:00:14.685 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:00:14.685 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:00:14.727 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:14.725+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:14.728 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:14.728 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:19.729 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:19.727+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:19.729 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000003.7.9, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:19.730 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:24.730 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:24.728+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:24.731 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:24.732 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:29.732 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:29.729+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:29.733 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:29.734 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:34.733 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:34.731+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:34.734 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:34.735 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:39.735 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:39.732+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:39.736 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:39.736 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:44.737 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:44.734+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:44.738 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:44.738 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:49.738 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:49.735+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:49.738 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:49.739 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:54.739 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:54.736+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:54.740 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:54.740 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:59.740 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:00:59.738+0000 7f66a1dfe600 1 waiting for incremental sync to catch up: 2022-09-03T06:00:59.741 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000010.30.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:00:59.741 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:01:04.742 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:01:04.739+0000 7f66a1dfe600 1 bucket sync caught up with source: 2022-09-03T06:01:04.742 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:01:04.743 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.33.5, , 00000000004.10.5, , 00000000009.27.9, , , , , ] 2022-09-03T06:01:04.743 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:01:04.739+0000 7f66a1dfe600 0 bucket checkpoint complete 2022-09-03T06:01:04.758 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-16 2022-09-03T06:01:04.759 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:01:04.760 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:01:04.761 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:01:04.829 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:01:04.829 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:01:04.872 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:04.869+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:04.872 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:04.872 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:09.873 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:09.871+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:09.874 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:09.875 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:14.875 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:14.873+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:14.876 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:14.876 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:19.876 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:19.874+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:19.877 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:19.878 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:24.878 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:24.876+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:24.879 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:24.880 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:29.879 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:29.877+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:29.880 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:29.881 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:34.881 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:34.879+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:34.881 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:34.882 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:39.882 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:39.880+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:39.883 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:39.884 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:44.884 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:44.882+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:44.885 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:44.885 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:49.885 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:49.882+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:49.886 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:49.886 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:54.887 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:54.883+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:54.887 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:54.887 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:01:59.888 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:01:59.885+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:01:59.889 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:01:59.889 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:04.890 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:04.887+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:04.890 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:02:04.890 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:09.891 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:09.889+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:09.892 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:02:09.892 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:14.893 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:14.891+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:14.894 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:02:14.895 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:19.900 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:19.892+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:19.901 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:02:19.901 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:24.896 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:24.893+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:24.896 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:02:24.897 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:29.897 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:29.895+0000 7f215a28b600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:29.898 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000007.40.9, , , , , ] 2022-09-03T06:02:29.898 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:34.899 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:34.897+0000 7f215a28b600 1 bucket sync caught up with source: 2022-09-03T06:02:34.899 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:34.899 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:34.900 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:34.897+0000 7f215a28b600 0 bucket checkpoint complete 2022-09-03T06:02:34.922 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-16 2022-09-03T06:02:34.923 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:34.924 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:34.925 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:34.990 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:34.990 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:35.036 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:35.033+0000 7f7f5bd29600 1 bucket sync caught up with source: 2022-09-03T06:02:35.036 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:35.037 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.37.5, , 00000000011.32.5, , 00000000008.43.5, , , , , ] 2022-09-03T06:02:35.037 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:35.033+0000 7f7f5bd29600 0 bucket checkpoint complete 2022-09-03T06:02:35.050 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-16 2022-09-03T06:02:35.053 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-16 2022-09-03T06:02:35.054 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:35.055 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:35.056 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:35.115 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:35.116 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:35.159 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:35.151+0000 7f8d61785600 1 bucket sync caught up with source: 2022-09-03T06:02:35.160 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000005.50.5, , 00000000004.26.5, , 00000000015.63.5, , , , , ] 2022-09-03T06:02:35.160 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000005.50.5, , 00000000004.26.5, , 00000000015.63.5, , , , , ] 2022-09-03T06:02:35.160 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:35.151+0000 7f8d61785600 0 bucket checkpoint complete 2022-09-03T06:02:35.165 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-16 2022-09-03T06:02:35.166 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:35.166 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:35.167 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-16 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:35.234 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:35.234 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:35.273 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:35.268+0000 7f9f6a1df600 1 bucket sync caught up with source: 2022-09-03T06:02:35.273 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000005.50.5, , 00000000004.26.5, , 00000000015.63.5, , , , , ] 2022-09-03T06:02:35.274 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000005.50.5, , 00000000004.26.5, , 00000000015.63.5, , , , , ] 2022-09-03T06:02:35.274 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:35.268+0000 7f9f6a1df600 0 bucket checkpoint complete 2022-09-03T06:02:35.288 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-16 2022-09-03T06:02:35.289 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-16 zones={test-zone2, test-zone1} 2022-09-03T06:02:35.290 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.291 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.300 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.301 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T06:02:35.302 DEBUG:boto:Data: 2022-09-03T06:02:35.302 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.303 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.303 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.304 DEBUG:boto:Params: {} 2022-09-03T06:02:35.305 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:02:35.306 DEBUG:boto:Token: None 2022-09-03T06:02:35.307 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:btZQ8gTcFV6/hznPwT5wkuKdx1M= 2022-09-03T06:02:35.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:btZQ8gTcFV6/hznPwT5wkuKdx1M=', 'Content-Length': '0'} 2022-09-03T06:02:35.315 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000023fb818999fd7673-006312edfb-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.316 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.317 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.317 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.318 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T06:02:35.318 DEBUG:boto:Data: 2022-09-03T06:02:35.319 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.320 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.321 DEBUG:boto:Params: {} 2022-09-03T06:02:35.323 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:02:35.324 DEBUG:boto:Token: None 2022-09-03T06:02:35.325 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:btZQ8gTcFV6/hznPwT5wkuKdx1M= 2022-09-03T06:02:35.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:btZQ8gTcFV6/hznPwT5wkuKdx1M=', 'Content-Length': '0'} 2022-09-03T06:02:35.332 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b82c5031e9db8e8e-006312edfb-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.333 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:02:35.334 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.335 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.336 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.337 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.338 DEBUG:boto:Method: GET 2022-09-03T06:02:35.339 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.340 DEBUG:boto:Data: 2022-09-03T06:02:35.341 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.342 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.343 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.344 DEBUG:boto:Params: {} 2022-09-03T06:02:35.345 DEBUG:boto:Token: None 2022-09-03T06:02:35.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/?versions 2022-09-03T06:02:35.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iKiXYNO4mzd1jtrxBXow7YUCmPs= 2022-09-03T06:02:35.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iKiXYNO4mzd1jtrxBXow7YUCmPs=', 'Content-Length': '0'} 2022-09-03T06:02:35.350 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000018f6a31aecc2520f-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.352 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.354 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:35.355 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:35.356 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:35.356 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:35.357 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:02:35.359 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.360 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.361 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.362 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.363 DEBUG:boto:Method: GET 2022-09-03T06:02:35.364 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.365 DEBUG:boto:Data: 2022-09-03T06:02:35.366 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.368 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.370 DEBUG:boto:Params: {} 2022-09-03T06:02:35.371 DEBUG:boto:Token: None 2022-09-03T06:02:35.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/?versions 2022-09-03T06:02:35.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iKiXYNO4mzd1jtrxBXow7YUCmPs= 2022-09-03T06:02:35.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iKiXYNO4mzd1jtrxBXow7YUCmPs=', 'Content-Length': '0'} 2022-09-03T06:02:35.377 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb688c5c0c0f6a32-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.378 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.379 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:35.380 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:35.381 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:35.381 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:35.382 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.383 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.383 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.384 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.384 DEBUG:boto:Method: GET 2022-09-03T06:02:35.385 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.385 DEBUG:boto:Data: 2022-09-03T06:02:35.386 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.387 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.388 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.389 DEBUG:boto:Params: {} 2022-09-03T06:02:35.390 DEBUG:boto:Token: None 2022-09-03T06:02:35.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/?versions 2022-09-03T06:02:35.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iKiXYNO4mzd1jtrxBXow7YUCmPs= 2022-09-03T06:02:35.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iKiXYNO4mzd1jtrxBXow7YUCmPs=', 'Content-Length': '0'} 2022-09-03T06:02:35.396 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000668e23ce616a74fb-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.398 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.399 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.400 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.401 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.402 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.403 DEBUG:boto:Method: GET 2022-09-03T06:02:35.404 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:35.405 DEBUG:boto:Data: 2022-09-03T06:02:35.406 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.408 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.409 DEBUG:boto:Params: {} 2022-09-03T06:02:35.411 DEBUG:boto:Token: None 2022-09-03T06:02:35.412 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/?versions 2022-09-03T06:02:35.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iKiXYNO4mzd1jtrxBXow7YUCmPs= 2022-09-03T06:02:35.414 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iKiXYNO4mzd1jtrxBXow7YUCmPs=', 'Content-Length': '0'} 2022-09-03T06:02:35.417 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d69725cb600b7bed-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.419 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.420 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:35.421 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.423 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.424 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.425 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.426 DEBUG:boto:Method: GET 2022-09-03T06:02:35.427 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.428 DEBUG:boto:Data: 2022-09-03T06:02:35.429 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.430 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.431 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.432 DEBUG:boto:Params: {} 2022-09-03T06:02:35.433 DEBUG:boto:Token: None 2022-09-03T06:02:35.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.436 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U= 2022-09-03T06:02:35.437 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U=', 'Content-Length': '0'} 2022-09-03T06:02:35.439 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009afdfb6745130cf1-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.441 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.442 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.443 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.444 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.445 DEBUG:boto:Method: GET 2022-09-03T06:02:35.446 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.447 DEBUG:boto:Data: 2022-09-03T06:02:35.448 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.450 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.451 DEBUG:boto:Params: {} 2022-09-03T06:02:35.452 DEBUG:boto:Token: None 2022-09-03T06:02:35.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U= 2022-09-03T06:02:35.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U=', 'Content-Length': '0'} 2022-09-03T06:02:35.458 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d2d4a902c3f8cc9d-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.460 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.461 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.462 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.463 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.463 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.464 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.465 DEBUG:boto:Data: 2022-09-03T06:02:35.466 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.467 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.468 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.469 DEBUG:boto:Params: {} 2022-09-03T06:02:35.470 DEBUG:boto:Token: None 2022-09-03T06:02:35.471 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiFRrtvsbgVHPttnvBstBE1UNTc= 2022-09-03T06:02:35.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiFRrtvsbgVHPttnvBstBE1UNTc=', 'Content-Length': '0'} 2022-09-03T06:02:35.474 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000df46addbb5694b1b-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.475 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.476 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.476 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.477 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.477 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.478 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.479 DEBUG:boto:Data: 2022-09-03T06:02:35.479 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.480 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.480 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.481 DEBUG:boto:Params: {} 2022-09-03T06:02:35.482 DEBUG:boto:Token: None 2022-09-03T06:02:35.482 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiFRrtvsbgVHPttnvBstBE1UNTc= 2022-09-03T06:02:35.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiFRrtvsbgVHPttnvBstBE1UNTc=', 'Content-Length': '0'} 2022-09-03T06:02:35.485 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000050193add906bb548-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.486 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:35.487 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.487 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.488 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.488 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.489 DEBUG:boto:Method: GET 2022-09-03T06:02:35.489 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.490 DEBUG:boto:Data: 2022-09-03T06:02:35.490 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.491 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.491 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.492 DEBUG:boto:Params: {} 2022-09-03T06:02:35.493 DEBUG:boto:Token: None 2022-09-03T06:02:35.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U= 2022-09-03T06:02:35.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U=', 'Content-Length': '0'} 2022-09-03T06:02:35.496 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b879df373076b6f5-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.497 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.498 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.498 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.499 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.500 DEBUG:boto:Method: GET 2022-09-03T06:02:35.501 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.502 DEBUG:boto:Data: 2022-09-03T06:02:35.502 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.503 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.504 DEBUG:boto:Params: {} 2022-09-03T06:02:35.504 DEBUG:boto:Token: None 2022-09-03T06:02:35.505 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U= 2022-09-03T06:02:35.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U=', 'Content-Length': '0'} 2022-09-03T06:02:35.508 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008fce6e60342966cc-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.509 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.510 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.511 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:35.511 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:35.512 DEBUG:boto:Method: GET 2022-09-03T06:02:35.512 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:35.513 DEBUG:boto:Data: 2022-09-03T06:02:35.513 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.514 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.515 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.516 DEBUG:boto:Params: {} 2022-09-03T06:02:35.516 DEBUG:boto:Token: None 2022-09-03T06:02:35.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo= 2022-09-03T06:02:35.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo=', 'Content-Length': '0'} 2022-09-03T06:02:35.522 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cadab2988c9f0a25-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.523 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.524 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.525 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.526 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:35.527 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:35.527 DEBUG:boto:Method: GET 2022-09-03T06:02:35.528 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:35.529 DEBUG:boto:Data: 2022-09-03T06:02:35.529 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.530 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.531 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.532 DEBUG:boto:Params: {} 2022-09-03T06:02:35.533 DEBUG:boto:Token: None 2022-09-03T06:02:35.534 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo= 2022-09-03T06:02:35.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo=', 'Content-Length': '0'} 2022-09-03T06:02:35.539 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e11adf51520eb972-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.541 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.542 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.543 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.544 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.544 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T06:02:35.545 DEBUG:boto:Data: 2022-09-03T06:02:35.545 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.546 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.546 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.547 DEBUG:boto:Params: {} 2022-09-03T06:02:35.548 DEBUG:boto:Token: None 2022-09-03T06:02:35.548 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy 2022-09-03T06:02:35.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LdBR45mkKaXhzKwYeZ4F6gefC/s= 2022-09-03T06:02:35.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LdBR45mkKaXhzKwYeZ4F6gefC/s=', 'Content-Length': '0'} 2022-09-03T06:02:35.552 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000026a0a97ee12d64fb-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.553 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.555 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.556 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.556 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T06:02:35.557 DEBUG:boto:Data: 2022-09-03T06:02:35.557 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.558 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.558 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.559 DEBUG:boto:Params: {} 2022-09-03T06:02:35.559 DEBUG:boto:Token: None 2022-09-03T06:02:35.560 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy 2022-09-03T06:02:35.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LdBR45mkKaXhzKwYeZ4F6gefC/s= 2022-09-03T06:02:35.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LdBR45mkKaXhzKwYeZ4F6gefC/s=', 'Content-Length': '0'} 2022-09-03T06:02:35.564 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c4f5fc14cf1cb5f1-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.565 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:35.566 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.566 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.567 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.568 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.569 DEBUG:boto:Method: GET 2022-09-03T06:02:35.570 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.570 DEBUG:boto:Data: 2022-09-03T06:02:35.571 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.572 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.573 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.573 DEBUG:boto:Params: {} 2022-09-03T06:02:35.574 DEBUG:boto:Token: None 2022-09-03T06:02:35.575 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U= 2022-09-03T06:02:35.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U=', 'Content-Length': '0'} 2022-09-03T06:02:35.578 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b9709ffebc70881d-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.579 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:35.580 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:35.580 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.581 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.582 DEBUG:boto:Method: GET 2022-09-03T06:02:35.583 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.583 DEBUG:boto:Data: 2022-09-03T06:02:35.584 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.585 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.586 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.587 DEBUG:boto:Params: {} 2022-09-03T06:02:35.587 DEBUG:boto:Token: None 2022-09-03T06:02:35.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:35.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U= 2022-09-03T06:02:35.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GERSBSIdTxZS/h0f7JY2vefiW1U=', 'Content-Length': '0'} 2022-09-03T06:02:35.592 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000184fdcf76528e0a1-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.593 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.594 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.595 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:35.596 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:35.597 DEBUG:boto:Method: GET 2022-09-03T06:02:35.598 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:35.599 DEBUG:boto:Data: 2022-09-03T06:02:35.600 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.601 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.602 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.603 DEBUG:boto:Params: {} 2022-09-03T06:02:35.604 DEBUG:boto:Token: None 2022-09-03T06:02:35.605 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo= 2022-09-03T06:02:35.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo=', 'Content-Length': '0'} 2022-09-03T06:02:35.611 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000dda2927241f2a21-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.612 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.614 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.615 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.615 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:35.616 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:35.616 DEBUG:boto:Method: GET 2022-09-03T06:02:35.617 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:35.618 DEBUG:boto:Data: 2022-09-03T06:02:35.618 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.619 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.619 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.620 DEBUG:boto:Params: {} 2022-09-03T06:02:35.621 DEBUG:boto:Token: None 2022-09-03T06:02:35.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo= 2022-09-03T06:02:35.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo=', 'Content-Length': '0'} 2022-09-03T06:02:35.625 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000024f4b50de762e79a-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.626 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.627 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:35.627 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.628 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.628 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.629 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.629 DEBUG:boto:Method: GET 2022-09-03T06:02:35.630 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.630 DEBUG:boto:Data: 2022-09-03T06:02:35.631 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.631 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.632 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.632 DEBUG:boto:Params: {} 2022-09-03T06:02:35.633 DEBUG:boto:Token: None 2022-09-03T06:02:35.634 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc= 2022-09-03T06:02:35.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc=', 'Content-Length': '0'} 2022-09-03T06:02:35.637 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005640823d714c7d58-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.678 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.679 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.679 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.680 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.680 DEBUG:boto:Method: GET 2022-09-03T06:02:35.681 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.682 DEBUG:boto:Data: 2022-09-03T06:02:35.682 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.683 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.684 DEBUG:boto:Params: {} 2022-09-03T06:02:35.684 DEBUG:boto:Token: None 2022-09-03T06:02:35.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc= 2022-09-03T06:02:35.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc=', 'Content-Length': '0'} 2022-09-03T06:02:35.689 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b14b1708ac80c500-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.689 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.690 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.691 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.691 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.692 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.692 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.693 DEBUG:boto:Data: 2022-09-03T06:02:35.693 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.694 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.694 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.695 DEBUG:boto:Params: {} 2022-09-03T06:02:35.695 DEBUG:boto:Token: None 2022-09-03T06:02:35.696 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.696 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:goUjRoc8Uhk40OOs46QX5ZoMmwc= 2022-09-03T06:02:35.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:goUjRoc8Uhk40OOs46QX5ZoMmwc=', 'Content-Length': '0'} 2022-09-03T06:02:35.699 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e132acca07150985-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.700 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.700 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.701 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.701 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.702 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.703 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.703 DEBUG:boto:Data: 2022-09-03T06:02:35.704 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.704 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.705 DEBUG:boto:Params: {} 2022-09-03T06:02:35.706 DEBUG:boto:Token: None 2022-09-03T06:02:35.706 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:goUjRoc8Uhk40OOs46QX5ZoMmwc= 2022-09-03T06:02:35.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:goUjRoc8Uhk40OOs46QX5ZoMmwc=', 'Content-Length': '0'} 2022-09-03T06:02:35.710 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006026b8a01bace91e-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.711 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:35.712 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.713 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.714 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.715 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.716 DEBUG:boto:Method: GET 2022-09-03T06:02:35.717 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.718 DEBUG:boto:Data: 2022-09-03T06:02:35.719 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.720 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.721 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.722 DEBUG:boto:Params: {} 2022-09-03T06:02:35.723 DEBUG:boto:Token: None 2022-09-03T06:02:35.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc= 2022-09-03T06:02:35.725 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc=', 'Content-Length': '0'} 2022-09-03T06:02:35.727 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000023c630704213f85d-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.729 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.732 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.732 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.733 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.733 DEBUG:boto:Method: GET 2022-09-03T06:02:35.734 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.734 DEBUG:boto:Data: 2022-09-03T06:02:35.735 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.739 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.740 DEBUG:boto:Params: {} 2022-09-03T06:02:35.741 DEBUG:boto:Token: None 2022-09-03T06:02:35.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc= 2022-09-03T06:02:35.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc=', 'Content-Length': '0'} 2022-09-03T06:02:35.746 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002a0fe47d4a84b192-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.748 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.749 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.750 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.751 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.753 DEBUG:boto:Method: GET 2022-09-03T06:02:35.754 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.755 DEBUG:boto:Data: 2022-09-03T06:02:35.756 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.757 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.758 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.759 DEBUG:boto:Params: {} 2022-09-03T06:02:35.760 DEBUG:boto:Token: None 2022-09-03T06:02:35.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo= 2022-09-03T06:02:35.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo=', 'Content-Length': '0'} 2022-09-03T06:02:35.766 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c0c16738afbd1609-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.768 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.770 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.770 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.771 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.771 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.772 DEBUG:boto:Method: GET 2022-09-03T06:02:35.772 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.773 DEBUG:boto:Data: 2022-09-03T06:02:35.773 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.775 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.775 DEBUG:boto:Params: {} 2022-09-03T06:02:35.776 DEBUG:boto:Token: None 2022-09-03T06:02:35.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo= 2022-09-03T06:02:35.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo=', 'Content-Length': '0'} 2022-09-03T06:02:35.781 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000097ffb7dd4b03e5fb-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.826 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.827 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.828 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.829 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.830 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T06:02:35.830 DEBUG:boto:Data: 2022-09-03T06:02:35.831 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.832 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.833 DEBUG:boto:Params: {} 2022-09-03T06:02:35.834 DEBUG:boto:Token: None 2022-09-03T06:02:35.834 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1 2022-09-03T06:02:35.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MhHKVLZ1ypGsRHe3Ls5MW72H3U= 2022-09-03T06:02:35.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MhHKVLZ1ypGsRHe3Ls5MW72H3U=', 'Content-Length': '0'} 2022-09-03T06:02:35.838 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000407aa44765851a41-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.842 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.843 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.844 DEBUG:boto:Method: HEAD 2022-09-03T06:02:35.845 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T06:02:35.846 DEBUG:boto:Data: 2022-09-03T06:02:35.847 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.848 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.849 DEBUG:boto:Params: {} 2022-09-03T06:02:35.851 DEBUG:boto:Token: None 2022-09-03T06:02:35.853 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1 2022-09-03T06:02:35.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MhHKVLZ1ypGsRHe3Ls5MW72H3U= 2022-09-03T06:02:35.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MhHKVLZ1ypGsRHe3Ls5MW72H3U=', 'Content-Length': '0'} 2022-09-03T06:02:35.858 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000155aa04c8c39d32a-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.859 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:35.860 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.861 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.861 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.862 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.863 DEBUG:boto:Method: GET 2022-09-03T06:02:35.864 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.864 DEBUG:boto:Data: 2022-09-03T06:02:35.865 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.865 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.866 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.866 DEBUG:boto:Params: {} 2022-09-03T06:02:35.867 DEBUG:boto:Token: None 2022-09-03T06:02:35.868 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc= 2022-09-03T06:02:35.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc=', 'Content-Length': '0'} 2022-09-03T06:02:35.871 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f40b7b626b78de6f-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.915 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.916 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:35.917 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.918 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.919 DEBUG:boto:Method: GET 2022-09-03T06:02:35.920 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.921 DEBUG:boto:Data: 2022-09-03T06:02:35.922 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.923 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.924 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.925 DEBUG:boto:Params: {} 2022-09-03T06:02:35.926 DEBUG:boto:Token: None 2022-09-03T06:02:35.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:35.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc= 2022-09-03T06:02:35.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScvkknQqtsU3djErbnLen5RLYhc=', 'Content-Length': '0'} 2022-09-03T06:02:35.933 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fe8992f479fb6f66-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.934 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.935 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.936 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.937 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.938 DEBUG:boto:Method: GET 2022-09-03T06:02:35.938 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.939 DEBUG:boto:Data: 2022-09-03T06:02:35.940 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.941 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.942 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.943 DEBUG:boto:Params: {} 2022-09-03T06:02:35.945 DEBUG:boto:Token: None 2022-09-03T06:02:35.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo= 2022-09-03T06:02:35.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo=', 'Content-Length': '0'} 2022-09-03T06:02:35.950 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1bb383f0dea38e3-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.951 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.953 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:35.954 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:35.958 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.959 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.959 DEBUG:boto:Method: GET 2022-09-03T06:02:35.960 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:35.961 DEBUG:boto:Data: 2022-09-03T06:02:35.962 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.964 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.965 DEBUG:boto:Params: {} 2022-09-03T06:02:35.966 DEBUG:boto:Token: None 2022-09-03T06:02:35.967 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/ 2022-09-03T06:02:35.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo= 2022-09-03T06:02:35.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q6QMXSEauuanMRHL24W/TFLBjNo=', 'Content-Length': '0'} 2022-09-03T06:02:35.973 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000012298a1d81af594e-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.973 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:35.974 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:35.975 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:35.976 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:35.977 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:35.978 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:35.978 DEBUG:boto:Method: GET 2022-09-03T06:02:35.979 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:35.979 DEBUG:boto:Data: 2022-09-03T06:02:35.980 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.981 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:35.981 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.982 DEBUG:boto:Params: {} 2022-09-03T06:02:35.983 DEBUG:boto:Token: None 2022-09-03T06:02:35.983 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:35.984 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8mOLehwPQUI0pb2ke3QzjpDs2KQ= 2022-09-03T06:02:35.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8mOLehwPQUI0pb2ke3QzjpDs2KQ=', 'Content-Length': '0'} 2022-09-03T06:02:35.987 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b7a9d1687d41cb49-006312edfb-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.988 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:35.989 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:35.989 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:35.990 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:35.990 DEBUG:boto:Method: GET 2022-09-03T06:02:35.991 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:35.991 DEBUG:boto:Data: 2022-09-03T06:02:35.992 DEBUG:boto:Headers: {} 2022-09-03T06:02:35.992 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:35.993 DEBUG:boto:Port: 8000 2022-09-03T06:02:35.993 DEBUG:boto:Params: {} 2022-09-03T06:02:35.994 DEBUG:boto:Token: None 2022-09-03T06:02:35.995 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:35 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:35.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8mOLehwPQUI0pb2ke3QzjpDs2KQ= 2022-09-03T06:02:35.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8mOLehwPQUI0pb2ke3QzjpDs2KQ=', 'Content-Length': '0'} 2022-09-03T06:02:35.998 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ee57d9f059eed7a2-006312edfb-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:35.999 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.000 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.000 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.001 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.001 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.002 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.002 DEBUG:boto:Data: 2022-09-03T06:02:36.003 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.004 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.004 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.005 DEBUG:boto:Params: {} 2022-09-03T06:02:36.006 DEBUG:boto:Token: None 2022-09-03T06:02:36.006 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o1IfddPUzaGdz3/QjHOXDwJ0eEk= 2022-09-03T06:02:36.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o1IfddPUzaGdz3/QjHOXDwJ0eEk=', 'Content-Length': '0'} 2022-09-03T06:02:36.011 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001d83ca5878a42701-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.012 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.013 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.013 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.014 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.015 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.016 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.017 DEBUG:boto:Data: 2022-09-03T06:02:36.017 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.018 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.019 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.020 DEBUG:boto:Params: {} 2022-09-03T06:02:36.020 DEBUG:boto:Token: None 2022-09-03T06:02:36.021 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.021 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o1IfddPUzaGdz3/QjHOXDwJ0eEk= 2022-09-03T06:02:36.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o1IfddPUzaGdz3/QjHOXDwJ0eEk=', 'Content-Length': '0'} 2022-09-03T06:02:36.024 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ec0cd7664df339be-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.025 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:36.025 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.026 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.026 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.027 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.027 DEBUG:boto:Method: GET 2022-09-03T06:02:36.028 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.028 DEBUG:boto:Data: 2022-09-03T06:02:36.029 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.029 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.030 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.031 DEBUG:boto:Params: {} 2022-09-03T06:02:36.031 DEBUG:boto:Token: None 2022-09-03T06:02:36.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+fWWJt7zhVt+YL1ZEhYETF4E6w= 2022-09-03T06:02:36.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+fWWJt7zhVt+YL1ZEhYETF4E6w=', 'Content-Length': '0'} 2022-09-03T06:02:36.035 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000047fc650a76aae4f8-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.036 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.036 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.037 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.037 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.038 DEBUG:boto:Method: GET 2022-09-03T06:02:36.039 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.039 DEBUG:boto:Data: 2022-09-03T06:02:36.040 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.042 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.043 DEBUG:boto:Params: {} 2022-09-03T06:02:36.044 DEBUG:boto:Token: None 2022-09-03T06:02:36.044 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+fWWJt7zhVt+YL1ZEhYETF4E6w= 2022-09-03T06:02:36.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+fWWJt7zhVt+YL1ZEhYETF4E6w=', 'Content-Length': '0'} 2022-09-03T06:02:36.048 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c15ba534ca0af57f-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.049 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.049 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.050 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.051 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.051 DEBUG:boto:Method: GET 2022-09-03T06:02:36.052 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.053 DEBUG:boto:Data: 2022-09-03T06:02:36.053 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.054 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.054 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.055 DEBUG:boto:Params: {} 2022-09-03T06:02:36.055 DEBUG:boto:Token: None 2022-09-03T06:02:36.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.057 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.059 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000084362d0170b31a17-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.060 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.061 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.061 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.062 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.063 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.063 DEBUG:boto:Method: GET 2022-09-03T06:02:36.064 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.064 DEBUG:boto:Data: 2022-09-03T06:02:36.065 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.065 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.066 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.066 DEBUG:boto:Params: {} 2022-09-03T06:02:36.067 DEBUG:boto:Token: None 2022-09-03T06:02:36.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.071 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cde4a047821e4ed9-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.072 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.073 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.074 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.074 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.075 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T06:02:36.075 DEBUG:boto:Data: 2022-09-03T06:02:36.076 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.076 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.077 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.077 DEBUG:boto:Params: {} 2022-09-03T06:02:36.078 DEBUG:boto:Token: None 2022-09-03T06:02:36.079 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone2 2022-09-03T06:02:36.079 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UTzpRPWNVo7FoJnF0CdixMlkR80= 2022-09-03T06:02:36.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UTzpRPWNVo7FoJnF0CdixMlkR80=', 'Content-Length': '0'} 2022-09-03T06:02:36.083 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000896a8bbdbe50b5c5-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.084 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.084 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.085 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.085 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T06:02:36.086 DEBUG:boto:Data: 2022-09-03T06:02:36.086 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.087 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.087 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.088 DEBUG:boto:Params: {} 2022-09-03T06:02:36.088 DEBUG:boto:Token: None 2022-09-03T06:02:36.089 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone2 2022-09-03T06:02:36.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UTzpRPWNVo7FoJnF0CdixMlkR80= 2022-09-03T06:02:36.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UTzpRPWNVo7FoJnF0CdixMlkR80=', 'Content-Length': '0'} 2022-09-03T06:02:36.094 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b87b5f80f58b68b5-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.095 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:36.095 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.096 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.096 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.097 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.097 DEBUG:boto:Method: GET 2022-09-03T06:02:36.098 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.099 DEBUG:boto:Data: 2022-09-03T06:02:36.099 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.100 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.100 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.101 DEBUG:boto:Params: {} 2022-09-03T06:02:36.101 DEBUG:boto:Token: None 2022-09-03T06:02:36.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+fWWJt7zhVt+YL1ZEhYETF4E6w= 2022-09-03T06:02:36.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+fWWJt7zhVt+YL1ZEhYETF4E6w=', 'Content-Length': '0'} 2022-09-03T06:02:36.105 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b05ec45179dd3628-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.106 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.106 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:36.107 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.107 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.108 DEBUG:boto:Method: GET 2022-09-03T06:02:36.108 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.109 DEBUG:boto:Data: 2022-09-03T06:02:36.109 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.110 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.111 DEBUG:boto:Params: {} 2022-09-03T06:02:36.112 DEBUG:boto:Token: None 2022-09-03T06:02:36.112 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:36.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+fWWJt7zhVt+YL1ZEhYETF4E6w= 2022-09-03T06:02:36.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+fWWJt7zhVt+YL1ZEhYETF4E6w=', 'Content-Length': '0'} 2022-09-03T06:02:36.116 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008778c9dd78390907-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.158 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.159 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.160 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.162 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.163 DEBUG:boto:Method: GET 2022-09-03T06:02:36.164 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.165 DEBUG:boto:Data: 2022-09-03T06:02:36.165 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.166 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.167 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.168 DEBUG:boto:Params: {} 2022-09-03T06:02:36.169 DEBUG:boto:Token: None 2022-09-03T06:02:36.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.173 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000165bd298f34f83cf-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.174 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.175 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.176 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.177 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.178 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.179 DEBUG:boto:Method: GET 2022-09-03T06:02:36.180 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:36.180 DEBUG:boto:Data: 2022-09-03T06:02:36.181 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.183 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.184 DEBUG:boto:Params: {} 2022-09-03T06:02:36.185 DEBUG:boto:Token: None 2022-09-03T06:02:36.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.192 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000178a45dc0fb81f63-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.193 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.194 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:36.196 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.196 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.197 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.198 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.198 DEBUG:boto:Method: GET 2022-09-03T06:02:36.199 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.199 DEBUG:boto:Data: 2022-09-03T06:02:36.200 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.201 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.201 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.202 DEBUG:boto:Params: {} 2022-09-03T06:02:36.203 DEBUG:boto:Token: None 2022-09-03T06:02:36.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A= 2022-09-03T06:02:36.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A=', 'Content-Length': '0'} 2022-09-03T06:02:36.210 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000956984e3af837ab5-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.211 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.212 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.213 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.214 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.215 DEBUG:boto:Method: GET 2022-09-03T06:02:36.216 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.216 DEBUG:boto:Data: 2022-09-03T06:02:36.217 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.218 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.219 DEBUG:boto:Params: {} 2022-09-03T06:02:36.219 DEBUG:boto:Token: None 2022-09-03T06:02:36.220 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A= 2022-09-03T06:02:36.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A=', 'Content-Length': '0'} 2022-09-03T06:02:36.224 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000079e1535488c8192f-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.225 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.225 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.226 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.227 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.228 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.228 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.229 DEBUG:boto:Data: 2022-09-03T06:02:36.229 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.230 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.231 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.231 DEBUG:boto:Params: {} 2022-09-03T06:02:36.232 DEBUG:boto:Token: None 2022-09-03T06:02:36.233 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L7ESqR0mobBqZe4hZQFav0ZzBLg= 2022-09-03T06:02:36.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L7ESqR0mobBqZe4hZQFav0ZzBLg=', 'Content-Length': '0'} 2022-09-03T06:02:36.236 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be03420317bbaf9b-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.237 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.238 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.238 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.239 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.239 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.240 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.240 DEBUG:boto:Data: 2022-09-03T06:02:36.241 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.241 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.242 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.242 DEBUG:boto:Params: {} 2022-09-03T06:02:36.243 DEBUG:boto:Token: None 2022-09-03T06:02:36.243 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L7ESqR0mobBqZe4hZQFav0ZzBLg= 2022-09-03T06:02:36.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L7ESqR0mobBqZe4hZQFav0ZzBLg=', 'Content-Length': '0'} 2022-09-03T06:02:36.246 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000992a27025827f6df-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.247 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:36.248 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.248 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.249 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.249 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.250 DEBUG:boto:Method: GET 2022-09-03T06:02:36.250 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.251 DEBUG:boto:Data: 2022-09-03T06:02:36.251 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.252 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.252 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.253 DEBUG:boto:Params: {} 2022-09-03T06:02:36.254 DEBUG:boto:Token: None 2022-09-03T06:02:36.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A= 2022-09-03T06:02:36.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A=', 'Content-Length': '0'} 2022-09-03T06:02:36.258 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a796affe0da09283-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.258 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.259 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.260 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.260 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.261 DEBUG:boto:Method: GET 2022-09-03T06:02:36.261 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.262 DEBUG:boto:Data: 2022-09-03T06:02:36.262 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.263 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.264 DEBUG:boto:Params: {} 2022-09-03T06:02:36.264 DEBUG:boto:Token: None 2022-09-03T06:02:36.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A= 2022-09-03T06:02:36.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A=', 'Content-Length': '0'} 2022-09-03T06:02:36.269 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000005c41294802fa3fd-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.270 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.272 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.272 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.273 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.274 DEBUG:boto:Method: GET 2022-09-03T06:02:36.275 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.276 DEBUG:boto:Data: 2022-09-03T06:02:36.277 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.277 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.278 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.279 DEBUG:boto:Params: {} 2022-09-03T06:02:36.280 DEBUG:boto:Token: None 2022-09-03T06:02:36.280 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.285 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d26a15221b7dea25-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.285 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.287 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.288 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.288 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.289 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.289 DEBUG:boto:Method: GET 2022-09-03T06:02:36.290 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.291 DEBUG:boto:Data: 2022-09-03T06:02:36.291 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.293 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.293 DEBUG:boto:Params: {} 2022-09-03T06:02:36.294 DEBUG:boto:Token: None 2022-09-03T06:02:36.295 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006e34bdce5861427a-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.302 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.303 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.304 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.305 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.305 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T06:02:36.306 DEBUG:boto:Data: 2022-09-03T06:02:36.306 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.307 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.307 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.308 DEBUG:boto:Params: {} 2022-09-03T06:02:36.308 DEBUG:boto:Token: None 2022-09-03T06:02:36.309 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3 2022-09-03T06:02:36.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3x3ZTH7ZkDyX43DUqgI1Bg5+Ic0= 2022-09-03T06:02:36.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3x3ZTH7ZkDyX43DUqgI1Bg5+Ic0=', 'Content-Length': '0'} 2022-09-03T06:02:36.312 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cc73583d1f821ad5-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.313 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.314 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.315 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.315 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T06:02:36.316 DEBUG:boto:Data: 2022-09-03T06:02:36.316 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.317 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.317 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.318 DEBUG:boto:Params: {} 2022-09-03T06:02:36.319 DEBUG:boto:Token: None 2022-09-03T06:02:36.320 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3 2022-09-03T06:02:36.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3x3ZTH7ZkDyX43DUqgI1Bg5+Ic0= 2022-09-03T06:02:36.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3x3ZTH7ZkDyX43DUqgI1Bg5+Ic0=', 'Content-Length': '0'} 2022-09-03T06:02:36.326 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000006d76d4537302d9c-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.327 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:36.328 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.329 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.330 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.331 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.332 DEBUG:boto:Method: GET 2022-09-03T06:02:36.333 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.334 DEBUG:boto:Data: 2022-09-03T06:02:36.335 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.335 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.336 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.337 DEBUG:boto:Params: {} 2022-09-03T06:02:36.338 DEBUG:boto:Token: None 2022-09-03T06:02:36.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A= 2022-09-03T06:02:36.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A=', 'Content-Length': '0'} 2022-09-03T06:02:36.344 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a98f970e7dd0d0ce-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.345 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.345 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:36.346 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.346 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.347 DEBUG:boto:Method: GET 2022-09-03T06:02:36.347 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.348 DEBUG:boto:Data: 2022-09-03T06:02:36.350 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.352 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.353 DEBUG:boto:Params: {} 2022-09-03T06:02:36.354 DEBUG:boto:Token: None 2022-09-03T06:02:36.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:36.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A= 2022-09-03T06:02:36.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j0gv0aYVOEZRDgCyeEIc+OTJq8A=', 'Content-Length': '0'} 2022-09-03T06:02:36.359 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000006e2c74b09894564-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.360 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.360 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.361 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.362 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.362 DEBUG:boto:Method: GET 2022-09-03T06:02:36.363 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.364 DEBUG:boto:Data: 2022-09-03T06:02:36.364 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.365 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:36.366 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.367 DEBUG:boto:Params: {} 2022-09-03T06:02:36.368 DEBUG:boto:Token: None 2022-09-03T06:02:36.370 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.371 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.375 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a859dbbea581f61-006312edfc-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.377 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.378 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.380 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.381 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.382 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.383 DEBUG:boto:Method: GET 2022-09-03T06:02:36.396 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:36.396 DEBUG:boto:Data: 2022-09-03T06:02:36.397 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.398 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.398 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.399 DEBUG:boto:Params: {} 2022-09-03T06:02:36.400 DEBUG:boto:Token: None 2022-09-03T06:02:36.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.413 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000253f7b43c06dd583-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.462 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.519 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-16 zones={test-zone2, test-zone1} 2022-09-03T06:02:36.521 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-16 zones={test-zone3, test-zone1} 2022-09-03T06:02:36.521 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.522 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.523 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.524 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T06:02:36.525 DEBUG:boto:Data: 2022-09-03T06:02:36.526 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.526 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.527 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.527 DEBUG:boto:Params: {} 2022-09-03T06:02:36.528 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:02:36.529 DEBUG:boto:Token: None 2022-09-03T06:02:36.529 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fpFQyuhJrtPTAEFsMNVKzo1kR/o= 2022-09-03T06:02:36.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fpFQyuhJrtPTAEFsMNVKzo1kR/o=', 'Content-Length': '0'} 2022-09-03T06:02:36.540 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000008f8fcac81459a37-006312edfc-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.542 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.543 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.544 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.545 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T06:02:36.546 DEBUG:boto:Data: 2022-09-03T06:02:36.547 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.548 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.549 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.550 DEBUG:boto:Params: {} 2022-09-03T06:02:36.551 DEBUG:boto:Token: None 2022-09-03T06:02:36.551 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fpFQyuhJrtPTAEFsMNVKzo1kR/o= 2022-09-03T06:02:36.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fpFQyuhJrtPTAEFsMNVKzo1kR/o=', 'Content-Length': '0'} 2022-09-03T06:02:36.559 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ef27954be2dad36a-006312edfc-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.560 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:02:36.561 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.562 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.563 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.564 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.566 DEBUG:boto:Method: GET 2022-09-03T06:02:36.567 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.568 DEBUG:boto:Data: 2022-09-03T06:02:36.569 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.575 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.576 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.578 DEBUG:boto:Params: {} 2022-09-03T06:02:36.580 DEBUG:boto:Token: None 2022-09-03T06:02:36.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/?versions 2022-09-03T06:02:36.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0ixYP4D52zhNlq+uPT5WXj3DXX8= 2022-09-03T06:02:36.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0ixYP4D52zhNlq+uPT5WXj3DXX8=', 'Content-Length': '0'} 2022-09-03T06:02:36.588 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000013949ab5b4203058-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.589 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.590 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:36.590 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:36.591 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:36.592 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:36.592 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:02:36.593 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.593 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.594 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.594 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.595 DEBUG:boto:Method: GET 2022-09-03T06:02:36.596 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.596 DEBUG:boto:Data: 2022-09-03T06:02:36.597 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.598 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.598 DEBUG:boto:Params: {} 2022-09-03T06:02:36.599 DEBUG:boto:Token: None 2022-09-03T06:02:36.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/?versions 2022-09-03T06:02:36.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0ixYP4D52zhNlq+uPT5WXj3DXX8= 2022-09-03T06:02:36.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0ixYP4D52zhNlq+uPT5WXj3DXX8=', 'Content-Length': '0'} 2022-09-03T06:02:36.604 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000049ff7bf02046f5df-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.605 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.606 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:36.607 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:36.608 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:36.608 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:36.609 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.609 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.610 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.611 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.612 DEBUG:boto:Method: GET 2022-09-03T06:02:36.613 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.617 DEBUG:boto:Data: 2022-09-03T06:02:36.619 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.619 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.620 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.620 DEBUG:boto:Params: {} 2022-09-03T06:02:36.621 DEBUG:boto:Token: None 2022-09-03T06:02:36.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/?versions 2022-09-03T06:02:36.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0ixYP4D52zhNlq+uPT5WXj3DXX8= 2022-09-03T06:02:36.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0ixYP4D52zhNlq+uPT5WXj3DXX8=', 'Content-Length': '0'} 2022-09-03T06:02:36.629 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a7250bc8c5bfba17-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.630 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.631 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.631 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.632 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.632 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.633 DEBUG:boto:Method: GET 2022-09-03T06:02:36.637 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:36.638 DEBUG:boto:Data: 2022-09-03T06:02:36.638 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.640 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.640 DEBUG:boto:Params: {} 2022-09-03T06:02:36.641 DEBUG:boto:Token: None 2022-09-03T06:02:36.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/?versions 2022-09-03T06:02:36.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0ixYP4D52zhNlq+uPT5WXj3DXX8= 2022-09-03T06:02:36.643 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0ixYP4D52zhNlq+uPT5WXj3DXX8=', 'Content-Length': '0'} 2022-09-03T06:02:36.646 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022c80e90ed48ce3d-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.647 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.648 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:36.649 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.649 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.650 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.650 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.651 DEBUG:boto:Method: GET 2022-09-03T06:02:36.651 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.652 DEBUG:boto:Data: 2022-09-03T06:02:36.652 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.653 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.654 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.654 DEBUG:boto:Params: {} 2022-09-03T06:02:36.655 DEBUG:boto:Token: None 2022-09-03T06:02:36.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw= 2022-09-03T06:02:36.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw=', 'Content-Length': '0'} 2022-09-03T06:02:36.659 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006c974bffcfa1afa1-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.660 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.661 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.661 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.662 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.663 DEBUG:boto:Method: GET 2022-09-03T06:02:36.663 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.664 DEBUG:boto:Data: 2022-09-03T06:02:36.665 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.666 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.667 DEBUG:boto:Params: {} 2022-09-03T06:02:36.669 DEBUG:boto:Token: None 2022-09-03T06:02:36.670 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw= 2022-09-03T06:02:36.672 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw=', 'Content-Length': '0'} 2022-09-03T06:02:36.674 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004103bbd834351db8-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.676 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.677 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.678 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.679 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.680 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.680 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.681 DEBUG:boto:Data: 2022-09-03T06:02:36.682 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.684 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.685 DEBUG:boto:Params: {} 2022-09-03T06:02:36.686 DEBUG:boto:Token: None 2022-09-03T06:02:36.687 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a6nXTFH/GF4/r+ISpyo+0S6glkY= 2022-09-03T06:02:36.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a6nXTFH/GF4/r+ISpyo+0S6glkY=', 'Content-Length': '0'} 2022-09-03T06:02:36.690 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004602436b65288c55-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.691 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.691 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.692 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.692 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.693 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.693 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.694 DEBUG:boto:Data: 2022-09-03T06:02:36.695 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.696 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.696 DEBUG:boto:Params: {} 2022-09-03T06:02:36.697 DEBUG:boto:Token: None 2022-09-03T06:02:36.697 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a6nXTFH/GF4/r+ISpyo+0S6glkY= 2022-09-03T06:02:36.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a6nXTFH/GF4/r+ISpyo+0S6glkY=', 'Content-Length': '0'} 2022-09-03T06:02:36.701 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000059fcd0aee1df427c-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.702 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:36.702 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.703 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.703 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.704 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.704 DEBUG:boto:Method: GET 2022-09-03T06:02:36.705 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.706 DEBUG:boto:Data: 2022-09-03T06:02:36.706 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.708 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.709 DEBUG:boto:Params: {} 2022-09-03T06:02:36.709 DEBUG:boto:Token: None 2022-09-03T06:02:36.710 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw= 2022-09-03T06:02:36.711 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw=', 'Content-Length': '0'} 2022-09-03T06:02:36.714 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d43b7b66495a8950-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.715 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.715 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.716 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.717 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.717 DEBUG:boto:Method: GET 2022-09-03T06:02:36.718 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.718 DEBUG:boto:Data: 2022-09-03T06:02:36.719 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.719 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.720 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.721 DEBUG:boto:Params: {} 2022-09-03T06:02:36.722 DEBUG:boto:Token: None 2022-09-03T06:02:36.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw= 2022-09-03T06:02:36.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw=', 'Content-Length': '0'} 2022-09-03T06:02:36.726 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000000633063366ec2bbf-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.727 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.728 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.729 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:36.729 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:36.730 DEBUG:boto:Method: GET 2022-09-03T06:02:36.730 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:36.731 DEBUG:boto:Data: 2022-09-03T06:02:36.732 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.733 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.733 DEBUG:boto:Params: {} 2022-09-03T06:02:36.734 DEBUG:boto:Token: None 2022-09-03T06:02:36.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.739 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b05c67b443efcc1-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.740 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.741 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.741 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.742 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:36.743 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:36.743 DEBUG:boto:Method: GET 2022-09-03T06:02:36.744 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:36.745 DEBUG:boto:Data: 2022-09-03T06:02:36.745 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.746 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.747 DEBUG:boto:Params: {} 2022-09-03T06:02:36.747 DEBUG:boto:Token: None 2022-09-03T06:02:36.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dcc0629ac21544ba-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.753 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.754 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.755 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.755 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.756 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T06:02:36.756 DEBUG:boto:Data: 2022-09-03T06:02:36.757 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.758 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.758 DEBUG:boto:Params: {} 2022-09-03T06:02:36.759 DEBUG:boto:Token: None 2022-09-03T06:02:36.760 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy 2022-09-03T06:02:36.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XMSe5YbuQqzOb/PX8z63aRv1Izk= 2022-09-03T06:02:36.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XMSe5YbuQqzOb/PX8z63aRv1Izk=', 'Content-Length': '0'} 2022-09-03T06:02:36.764 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008d7b24216917f2d9-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.765 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.766 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.767 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.768 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T06:02:36.769 DEBUG:boto:Data: 2022-09-03T06:02:36.769 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.771 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.772 DEBUG:boto:Params: {} 2022-09-03T06:02:36.773 DEBUG:boto:Token: None 2022-09-03T06:02:36.774 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy 2022-09-03T06:02:36.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XMSe5YbuQqzOb/PX8z63aRv1Izk= 2022-09-03T06:02:36.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XMSe5YbuQqzOb/PX8z63aRv1Izk=', 'Content-Length': '0'} 2022-09-03T06:02:36.778 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003c15228daecb67f9-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.779 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:36.780 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.781 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.782 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.783 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.784 DEBUG:boto:Method: GET 2022-09-03T06:02:36.784 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.785 DEBUG:boto:Data: 2022-09-03T06:02:36.785 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.786 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.787 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.787 DEBUG:boto:Params: {} 2022-09-03T06:02:36.788 DEBUG:boto:Token: None 2022-09-03T06:02:36.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw= 2022-09-03T06:02:36.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw=', 'Content-Length': '0'} 2022-09-03T06:02:36.792 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f3a5dbaf4d601b49-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.793 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:36.794 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:36.795 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.795 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.796 DEBUG:boto:Method: GET 2022-09-03T06:02:36.796 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.797 DEBUG:boto:Data: 2022-09-03T06:02:36.798 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.799 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.799 DEBUG:boto:Params: {} 2022-09-03T06:02:36.800 DEBUG:boto:Token: None 2022-09-03T06:02:36.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:36.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw= 2022-09-03T06:02:36.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q49bcInhfKHyEi9uneqb5auA3pw=', 'Content-Length': '0'} 2022-09-03T06:02:36.806 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000002077d126b2d731-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.807 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.808 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.809 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:36.809 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:36.810 DEBUG:boto:Method: GET 2022-09-03T06:02:36.811 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:36.811 DEBUG:boto:Data: 2022-09-03T06:02:36.812 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.812 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.813 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.813 DEBUG:boto:Params: {} 2022-09-03T06:02:36.814 DEBUG:boto:Token: None 2022-09-03T06:02:36.814 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.818 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce55a70ef9425b78-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.819 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.820 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.821 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.821 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:36.822 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:36.823 DEBUG:boto:Method: GET 2022-09-03T06:02:36.823 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:36.824 DEBUG:boto:Data: 2022-09-03T06:02:36.824 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.825 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.825 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.826 DEBUG:boto:Params: {} 2022-09-03T06:02:36.827 DEBUG:boto:Token: None 2022-09-03T06:02:36.827 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:36.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:36.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:36.831 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2f446fd3af30287-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.874 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:36.875 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:36.876 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.876 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.877 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.878 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.878 DEBUG:boto:Method: GET 2022-09-03T06:02:36.879 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.879 DEBUG:boto:Data: 2022-09-03T06:02:36.880 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.881 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.881 DEBUG:boto:Params: {} 2022-09-03T06:02:36.882 DEBUG:boto:Token: None 2022-09-03T06:02:36.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L6ZFaNEZAr2dJRs/5tmZXjXsl2w= 2022-09-03T06:02:36.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L6ZFaNEZAr2dJRs/5tmZXjXsl2w=', 'Content-Length': '0'} 2022-09-03T06:02:36.886 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a2cea3f84da601df-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.887 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.887 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.888 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.888 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.889 DEBUG:boto:Method: GET 2022-09-03T06:02:36.890 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.890 DEBUG:boto:Data: 2022-09-03T06:02:36.891 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.891 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.892 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.892 DEBUG:boto:Params: {} 2022-09-03T06:02:36.893 DEBUG:boto:Token: None 2022-09-03T06:02:36.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L6ZFaNEZAr2dJRs/5tmZXjXsl2w= 2022-09-03T06:02:36.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L6ZFaNEZAr2dJRs/5tmZXjXsl2w=', 'Content-Length': '0'} 2022-09-03T06:02:36.897 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000043116467c96947ce-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.898 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.898 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.899 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.899 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.900 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.900 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.901 DEBUG:boto:Data: 2022-09-03T06:02:36.902 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.903 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.903 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.904 DEBUG:boto:Params: {} 2022-09-03T06:02:36.904 DEBUG:boto:Token: None 2022-09-03T06:02:36.905 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T3wNb4faJcwE+7EeTa5qFKtFPrw= 2022-09-03T06:02:36.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T3wNb4faJcwE+7EeTa5qFKtFPrw=', 'Content-Length': '0'} 2022-09-03T06:02:36.909 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000642bc8eddc7b4c24-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.910 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.911 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.911 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.912 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.912 DEBUG:boto:Method: HEAD 2022-09-03T06:02:36.913 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.913 DEBUG:boto:Data: 2022-09-03T06:02:36.914 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.914 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.915 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.915 DEBUG:boto:Params: {} 2022-09-03T06:02:36.916 DEBUG:boto:Token: None 2022-09-03T06:02:36.916 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.917 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T3wNb4faJcwE+7EeTa5qFKtFPrw= 2022-09-03T06:02:36.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T3wNb4faJcwE+7EeTa5qFKtFPrw=', 'Content-Length': '0'} 2022-09-03T06:02:36.920 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008f1df7123419ff8b-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.921 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:36.921 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.922 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.922 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.923 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.924 DEBUG:boto:Method: GET 2022-09-03T06:02:36.924 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.925 DEBUG:boto:Data: 2022-09-03T06:02:36.925 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.926 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.926 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.927 DEBUG:boto:Params: {} 2022-09-03T06:02:36.928 DEBUG:boto:Token: None 2022-09-03T06:02:36.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L6ZFaNEZAr2dJRs/5tmZXjXsl2w= 2022-09-03T06:02:36.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L6ZFaNEZAr2dJRs/5tmZXjXsl2w=', 'Content-Length': '0'} 2022-09-03T06:02:36.932 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007a7859c07cb93902-006312edfc-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.978 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.979 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:36.980 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.980 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.981 DEBUG:boto:Method: GET 2022-09-03T06:02:36.982 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.983 DEBUG:boto:Data: 2022-09-03T06:02:36.985 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.986 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:36.986 DEBUG:boto:Port: 8000 2022-09-03T06:02:36.987 DEBUG:boto:Params: {} 2022-09-03T06:02:36.988 DEBUG:boto:Token: None 2022-09-03T06:02:36.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:36.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L6ZFaNEZAr2dJRs/5tmZXjXsl2w= 2022-09-03T06:02:36.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L6ZFaNEZAr2dJRs/5tmZXjXsl2w=', 'Content-Length': '0'} 2022-09-03T06:02:36.992 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000abf23700262ba38f-006312edfc-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:36.993 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:36.993 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:36.994 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:36.994 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:36.995 DEBUG:boto:Method: GET 2022-09-03T06:02:36.995 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:36.996 DEBUG:boto:Data: 2022-09-03T06:02:36.996 DEBUG:boto:Headers: {} 2022-09-03T06:02:36.997 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:36.998 DEBUG:boto:Port: 8001 2022-09-03T06:02:36.998 DEBUG:boto:Params: {} 2022-09-03T06:02:36.999 DEBUG:boto:Token: None 2022-09-03T06:02:36.999 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:36 GMT /avysmw-16/ 2022-09-03T06:02:37.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q= 2022-09-03T06:02:37.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+B24sdAuyDrxBzRLb3ZJ+9uf5/Q=', 'Content-Length': '0'} 2022-09-03T06:02:37.003 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f9f868ee5f9cd13-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.004 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.005 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.006 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.006 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:37.007 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:37.007 DEBUG:boto:Method: GET 2022-09-03T06:02:37.008 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:37.008 DEBUG:boto:Data: 2022-09-03T06:02:37.009 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.010 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.010 DEBUG:boto:Params: {} 2022-09-03T06:02:37.011 DEBUG:boto:Token: None 2022-09-03T06:02:37.012 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.016 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d403b1c42d0c189d-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.016 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.017 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:37.018 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:37.018 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.019 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T06:02:37.019 DEBUG:boto:Data: 2022-09-03T06:02:37.020 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.021 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.022 DEBUG:boto:Params: {} 2022-09-03T06:02:37.022 DEBUG:boto:Token: None 2022-09-03T06:02:37.023 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone1 2022-09-03T06:02:37.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AkxtEPQN5PXL/Cn/4NAJpFZYhF8= 2022-09-03T06:02:37.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AkxtEPQN5PXL/Cn/4NAJpFZYhF8=', 'Content-Length': '0'} 2022-09-03T06:02:37.026 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000b22b241f93f9a26-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.027 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:37.028 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:37.028 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.029 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T06:02:37.029 DEBUG:boto:Data: 2022-09-03T06:02:37.030 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.031 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.032 DEBUG:boto:Params: {} 2022-09-03T06:02:37.033 DEBUG:boto:Token: None 2022-09-03T06:02:37.034 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone1 2022-09-03T06:02:37.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AkxtEPQN5PXL/Cn/4NAJpFZYhF8= 2022-09-03T06:02:37.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AkxtEPQN5PXL/Cn/4NAJpFZYhF8=', 'Content-Length': '0'} 2022-09-03T06:02:37.039 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004e351a3947c657f0-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.040 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:37.044 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:37.044 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:37.045 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:37.046 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:37.047 DEBUG:boto:Method: GET 2022-09-03T06:02:37.048 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:37.049 DEBUG:boto:Data: 2022-09-03T06:02:37.050 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.052 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.053 DEBUG:boto:Params: {} 2022-09-03T06:02:37.054 DEBUG:boto:Token: None 2022-09-03T06:02:37.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:37.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CKKM0yZ1XUgtbcagNVTDEkkKGmo= 2022-09-03T06:02:37.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CKKM0yZ1XUgtbcagNVTDEkkKGmo=', 'Content-Length': '0'} 2022-09-03T06:02:37.061 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000085ea81b7b62cba72-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.106 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:37.107 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:37.108 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:37.110 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:37.111 DEBUG:boto:Method: GET 2022-09-03T06:02:37.112 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:37.112 DEBUG:boto:Data: 2022-09-03T06:02:37.113 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.115 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.116 DEBUG:boto:Params: {} 2022-09-03T06:02:37.118 DEBUG:boto:Token: None 2022-09-03T06:02:37.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:37.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CKKM0yZ1XUgtbcagNVTDEkkKGmo= 2022-09-03T06:02:37.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CKKM0yZ1XUgtbcagNVTDEkkKGmo=', 'Content-Length': '0'} 2022-09-03T06:02:37.124 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000033ebb3101ec3ab2d-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.125 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.126 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.127 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:37.128 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:37.129 DEBUG:boto:Method: GET 2022-09-03T06:02:37.130 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:37.130 DEBUG:boto:Data: 2022-09-03T06:02:37.131 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.132 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.133 DEBUG:boto:Params: {} 2022-09-03T06:02:37.134 DEBUG:boto:Token: None 2022-09-03T06:02:37.135 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.139 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000030f26e29b473c9ff-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.140 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.141 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.141 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.142 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:37.142 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:37.143 DEBUG:boto:Method: GET 2022-09-03T06:02:37.143 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:37.144 DEBUG:boto:Data: 2022-09-03T06:02:37.145 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.146 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.146 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.147 DEBUG:boto:Params: {} 2022-09-03T06:02:37.148 DEBUG:boto:Token: None 2022-09-03T06:02:37.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.152 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b75eb08f75bb4142-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.153 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.155 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:37.156 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.156 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.157 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.158 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.159 DEBUG:boto:Method: GET 2022-09-03T06:02:37.160 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.161 DEBUG:boto:Data: 2022-09-03T06:02:37.161 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.163 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.163 DEBUG:boto:Params: {} 2022-09-03T06:02:37.164 DEBUG:boto:Token: None 2022-09-03T06:02:37.164 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0= 2022-09-03T06:02:37.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0=', 'Content-Length': '0'} 2022-09-03T06:02:37.169 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ee41cef08beea98b-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.170 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.171 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.172 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.173 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.174 DEBUG:boto:Method: GET 2022-09-03T06:02:37.175 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.176 DEBUG:boto:Data: 2022-09-03T06:02:37.177 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.178 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.179 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.180 DEBUG:boto:Params: {} 2022-09-03T06:02:37.181 DEBUG:boto:Token: None 2022-09-03T06:02:37.182 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.183 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0= 2022-09-03T06:02:37.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0=', 'Content-Length': '0'} 2022-09-03T06:02:37.188 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002be49a85ab27ab68-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.190 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.191 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.192 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.193 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.194 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.195 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.196 DEBUG:boto:Data: 2022-09-03T06:02:37.197 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.198 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.199 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.199 DEBUG:boto:Params: {} 2022-09-03T06:02:37.201 DEBUG:boto:Token: None 2022-09-03T06:02:37.202 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.203 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yYdfMZCKPqJy9/G3wfWxEWYEXOE= 2022-09-03T06:02:37.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yYdfMZCKPqJy9/G3wfWxEWYEXOE=', 'Content-Length': '0'} 2022-09-03T06:02:37.207 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001853ab3d58d052e1-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.208 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.209 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.210 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.211 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.212 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.213 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.213 DEBUG:boto:Data: 2022-09-03T06:02:37.214 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.216 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.217 DEBUG:boto:Params: {} 2022-09-03T06:02:37.218 DEBUG:boto:Token: None 2022-09-03T06:02:37.219 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yYdfMZCKPqJy9/G3wfWxEWYEXOE= 2022-09-03T06:02:37.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yYdfMZCKPqJy9/G3wfWxEWYEXOE=', 'Content-Length': '0'} 2022-09-03T06:02:37.223 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c95b9fca97442f10-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.224 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:37.225 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.225 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.226 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.227 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.227 DEBUG:boto:Method: GET 2022-09-03T06:02:37.228 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.228 DEBUG:boto:Data: 2022-09-03T06:02:37.229 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.230 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.230 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.231 DEBUG:boto:Params: {} 2022-09-03T06:02:37.231 DEBUG:boto:Token: None 2022-09-03T06:02:37.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0= 2022-09-03T06:02:37.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0=', 'Content-Length': '0'} 2022-09-03T06:02:37.237 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f2283e3297799c9f-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.278 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.279 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.280 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.281 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.282 DEBUG:boto:Method: GET 2022-09-03T06:02:37.283 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.284 DEBUG:boto:Data: 2022-09-03T06:02:37.284 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.286 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.287 DEBUG:boto:Params: {} 2022-09-03T06:02:37.288 DEBUG:boto:Token: None 2022-09-03T06:02:37.288 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0= 2022-09-03T06:02:37.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0=', 'Content-Length': '0'} 2022-09-03T06:02:37.292 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002654fe8292cf03d6-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.293 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.293 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.294 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.294 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.295 DEBUG:boto:Method: GET 2022-09-03T06:02:37.296 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.296 DEBUG:boto:Data: 2022-09-03T06:02:37.297 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.297 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.298 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.298 DEBUG:boto:Params: {} 2022-09-03T06:02:37.299 DEBUG:boto:Token: None 2022-09-03T06:02:37.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.303 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ca7164638b294f67-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.304 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.305 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.305 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.306 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.306 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.307 DEBUG:boto:Method: GET 2022-09-03T06:02:37.307 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.308 DEBUG:boto:Data: 2022-09-03T06:02:37.308 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.309 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.310 DEBUG:boto:Params: {} 2022-09-03T06:02:37.310 DEBUG:boto:Token: None 2022-09-03T06:02:37.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.315 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002529d1fd248bd38c-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.316 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.317 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.317 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.318 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.318 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T06:02:37.319 DEBUG:boto:Data: 2022-09-03T06:02:37.319 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.320 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.320 DEBUG:boto:Params: {} 2022-09-03T06:02:37.321 DEBUG:boto:Token: None 2022-09-03T06:02:37.322 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2 2022-09-03T06:02:37.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u6fStpw4bikT0jp/bGvrJbASlNE= 2022-09-03T06:02:37.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u6fStpw4bikT0jp/bGvrJbASlNE=', 'Content-Length': '0'} 2022-09-03T06:02:37.325 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003b160ce7dcfb32f7-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.326 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.327 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.327 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.328 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T06:02:37.328 DEBUG:boto:Data: 2022-09-03T06:02:37.329 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.330 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.330 DEBUG:boto:Params: {} 2022-09-03T06:02:37.331 DEBUG:boto:Token: None 2022-09-03T06:02:37.331 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2 2022-09-03T06:02:37.332 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u6fStpw4bikT0jp/bGvrJbASlNE= 2022-09-03T06:02:37.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u6fStpw4bikT0jp/bGvrJbASlNE=', 'Content-Length': '0'} 2022-09-03T06:02:37.335 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f12e79727f987c1f-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.336 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:37.336 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.337 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.337 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.338 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.339 DEBUG:boto:Method: GET 2022-09-03T06:02:37.339 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.340 DEBUG:boto:Data: 2022-09-03T06:02:37.340 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.341 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.341 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.342 DEBUG:boto:Params: {} 2022-09-03T06:02:37.343 DEBUG:boto:Token: None 2022-09-03T06:02:37.343 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0= 2022-09-03T06:02:37.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0=', 'Content-Length': '0'} 2022-09-03T06:02:37.348 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008c28bcb45c5944fc-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.348 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.349 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:37.349 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.350 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.351 DEBUG:boto:Method: GET 2022-09-03T06:02:37.351 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.352 DEBUG:boto:Data: 2022-09-03T06:02:37.352 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.353 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.353 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.354 DEBUG:boto:Params: {} 2022-09-03T06:02:37.355 DEBUG:boto:Token: None 2022-09-03T06:02:37.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:37.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0= 2022-09-03T06:02:37.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:luTGlr6MsGkpIQ8JRE7KX9wJLR0=', 'Content-Length': '0'} 2022-09-03T06:02:37.359 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b17250934a498857-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.402 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.403 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.404 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.404 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.405 DEBUG:boto:Method: GET 2022-09-03T06:02:37.405 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.406 DEBUG:boto:Data: 2022-09-03T06:02:37.407 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.409 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.409 DEBUG:boto:Params: {} 2022-09-03T06:02:37.410 DEBUG:boto:Token: None 2022-09-03T06:02:37.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.416 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006bd04cc3c17ff1d7-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.418 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.419 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.420 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.421 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.422 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.423 DEBUG:boto:Method: GET 2022-09-03T06:02:37.423 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:37.424 DEBUG:boto:Data: 2022-09-03T06:02:37.424 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.425 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.425 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.426 DEBUG:boto:Params: {} 2022-09-03T06:02:37.427 DEBUG:boto:Token: None 2022-09-03T06:02:37.427 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.431 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e8e7f158838fce1e-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.432 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.433 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:37.434 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.435 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.435 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.436 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.436 DEBUG:boto:Method: GET 2022-09-03T06:02:37.437 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.437 DEBUG:boto:Data: 2022-09-03T06:02:37.438 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.439 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.440 DEBUG:boto:Params: {} 2022-09-03T06:02:37.441 DEBUG:boto:Token: None 2022-09-03T06:02:37.442 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ= 2022-09-03T06:02:37.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ=', 'Content-Length': '0'} 2022-09-03T06:02:37.447 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009834c42d7fa66551-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.448 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.449 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.449 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.450 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.451 DEBUG:boto:Method: GET 2022-09-03T06:02:37.451 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.452 DEBUG:boto:Data: 2022-09-03T06:02:37.452 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.453 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.454 DEBUG:boto:Params: {} 2022-09-03T06:02:37.455 DEBUG:boto:Token: None 2022-09-03T06:02:37.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.456 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ= 2022-09-03T06:02:37.457 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ=', 'Content-Length': '0'} 2022-09-03T06:02:37.460 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000aa66f5487149bc6f-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.461 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.461 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.462 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.462 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.463 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.464 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.464 DEBUG:boto:Data: 2022-09-03T06:02:37.465 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.465 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.466 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.466 DEBUG:boto:Params: {} 2022-09-03T06:02:37.467 DEBUG:boto:Token: None 2022-09-03T06:02:37.468 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ts9u9OeTSjfkOb7rqfHXRxBxDE= 2022-09-03T06:02:37.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ts9u9OeTSjfkOb7rqfHXRxBxDE=', 'Content-Length': '0'} 2022-09-03T06:02:37.473 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000212ed55d0eb8e937-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.474 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.475 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.475 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.476 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.476 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.477 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.478 DEBUG:boto:Data: 2022-09-03T06:02:37.479 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.480 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.480 DEBUG:boto:Params: {} 2022-09-03T06:02:37.481 DEBUG:boto:Token: None 2022-09-03T06:02:37.482 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ts9u9OeTSjfkOb7rqfHXRxBxDE= 2022-09-03T06:02:37.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ts9u9OeTSjfkOb7rqfHXRxBxDE=', 'Content-Length': '0'} 2022-09-03T06:02:37.487 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008d5d72dc2ecc2423-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.488 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:37.489 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.490 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.491 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.491 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.492 DEBUG:boto:Method: GET 2022-09-03T06:02:37.493 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.494 DEBUG:boto:Data: 2022-09-03T06:02:37.494 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.495 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.495 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.496 DEBUG:boto:Params: {} 2022-09-03T06:02:37.497 DEBUG:boto:Token: None 2022-09-03T06:02:37.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ= 2022-09-03T06:02:37.500 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ=', 'Content-Length': '0'} 2022-09-03T06:02:37.502 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ee921743b8260adb-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.504 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.505 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.506 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.506 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.507 DEBUG:boto:Method: GET 2022-09-03T06:02:37.508 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.509 DEBUG:boto:Data: 2022-09-03T06:02:37.510 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.511 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.512 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.513 DEBUG:boto:Params: {} 2022-09-03T06:02:37.514 DEBUG:boto:Token: None 2022-09-03T06:02:37.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ= 2022-09-03T06:02:37.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ=', 'Content-Length': '0'} 2022-09-03T06:02:37.520 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ab4e1fbe7ed66578-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.562 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.563 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.563 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.564 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.564 DEBUG:boto:Method: GET 2022-09-03T06:02:37.565 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.566 DEBUG:boto:Data: 2022-09-03T06:02:37.566 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.567 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.568 DEBUG:boto:Params: {} 2022-09-03T06:02:37.569 DEBUG:boto:Token: None 2022-09-03T06:02:37.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.574 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000462e6d9e0b84d84-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.575 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.575 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.576 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.577 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.577 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.578 DEBUG:boto:Method: GET 2022-09-03T06:02:37.578 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.579 DEBUG:boto:Data: 2022-09-03T06:02:37.580 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.580 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.581 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.581 DEBUG:boto:Params: {} 2022-09-03T06:02:37.582 DEBUG:boto:Token: None 2022-09-03T06:02:37.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.586 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ca70dc2fc54f545e-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.587 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.588 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.588 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.589 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.589 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T06:02:37.590 DEBUG:boto:Data: 2022-09-03T06:02:37.590 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.591 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.592 DEBUG:boto:Params: {} 2022-09-03T06:02:37.592 DEBUG:boto:Token: None 2022-09-03T06:02:37.593 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3 2022-09-03T06:02:37.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QvXbd8pUyAAAlv5iP5CjBHlmUcY= 2022-09-03T06:02:37.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QvXbd8pUyAAAlv5iP5CjBHlmUcY=', 'Content-Length': '0'} 2022-09-03T06:02:37.597 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006fee15db30031d5b-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.597 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.598 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.598 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.599 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T06:02:37.599 DEBUG:boto:Data: 2022-09-03T06:02:37.600 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.601 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.601 DEBUG:boto:Params: {} 2022-09-03T06:02:37.602 DEBUG:boto:Token: None 2022-09-03T06:02:37.602 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3 2022-09-03T06:02:37.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QvXbd8pUyAAAlv5iP5CjBHlmUcY= 2022-09-03T06:02:37.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QvXbd8pUyAAAlv5iP5CjBHlmUcY=', 'Content-Length': '0'} 2022-09-03T06:02:37.606 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c51f1623e353eb71-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.607 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:37.607 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.608 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.608 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.609 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.609 DEBUG:boto:Method: GET 2022-09-03T06:02:37.610 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.610 DEBUG:boto:Data: 2022-09-03T06:02:37.611 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.612 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.613 DEBUG:boto:Params: {} 2022-09-03T06:02:37.613 DEBUG:boto:Token: None 2022-09-03T06:02:37.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ= 2022-09-03T06:02:37.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ=', 'Content-Length': '0'} 2022-09-03T06:02:37.617 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c34fb1c7bd05d2fe-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.658 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.659 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:37.660 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.660 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.661 DEBUG:boto:Method: GET 2022-09-03T06:02:37.661 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.662 DEBUG:boto:Data: 2022-09-03T06:02:37.662 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.663 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.663 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.664 DEBUG:boto:Params: {} 2022-09-03T06:02:37.665 DEBUG:boto:Token: None 2022-09-03T06:02:37.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:37.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ= 2022-09-03T06:02:37.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qh0rZqqFWHFwetbvmT61kgypaeQ=', 'Content-Length': '0'} 2022-09-03T06:02:37.669 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000e4b6c72ba4c40e8-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.670 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.671 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.672 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.672 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.673 DEBUG:boto:Method: GET 2022-09-03T06:02:37.674 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.675 DEBUG:boto:Data: 2022-09-03T06:02:37.675 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.676 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.677 DEBUG:boto:Params: {} 2022-09-03T06:02:37.678 DEBUG:boto:Token: None 2022-09-03T06:02:37.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.682 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da6aa10b333ee05a-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.683 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.684 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.684 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.685 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.686 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.686 DEBUG:boto:Method: GET 2022-09-03T06:02:37.687 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:37.687 DEBUG:boto:Data: 2022-09-03T06:02:37.688 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:37.689 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.690 DEBUG:boto:Params: {} 2022-09-03T06:02:37.690 DEBUG:boto:Token: None 2022-09-03T06:02:37.691 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.695 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000013b512bb5aefbfc-006312edfd-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.696 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.696 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-16 zones={test-zone3, test-zone1} 2022-09-03T06:02:37.697 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-16 zones={test-zone3, test-zone2} 2022-09-03T06:02:37.698 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.698 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.699 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.699 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T06:02:37.700 DEBUG:boto:Data: 2022-09-03T06:02:37.700 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.701 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.702 DEBUG:boto:Params: {} 2022-09-03T06:02:37.702 DEBUG:boto:Token: None 2022-09-03T06:02:37.703 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mApuQQ+xByAfwvM4pYTMsE8rbc4= 2022-09-03T06:02:37.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mApuQQ+xByAfwvM4pYTMsE8rbc4=', 'Content-Length': '0'} 2022-09-03T06:02:37.709 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001db4031f33a265b9-006312edfd-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.710 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.711 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.712 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.713 DEBUG:boto:Path: /avysmw-16/ 2022-09-03T06:02:37.714 DEBUG:boto:Data: 2022-09-03T06:02:37.715 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.715 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:37.717 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.718 DEBUG:boto:Params: {} 2022-09-03T06:02:37.718 DEBUG:boto:Token: None 2022-09-03T06:02:37.719 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mApuQQ+xByAfwvM4pYTMsE8rbc4= 2022-09-03T06:02:37.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mApuQQ+xByAfwvM4pYTMsE8rbc4=', 'Content-Length': '0'} 2022-09-03T06:02:37.723 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000fdc72727449e5e5d-006312edfd-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.724 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:02:37.724 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.725 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.725 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.726 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.727 DEBUG:boto:Method: GET 2022-09-03T06:02:37.727 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.728 DEBUG:boto:Data: 2022-09-03T06:02:37.728 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.729 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.730 DEBUG:boto:Params: {} 2022-09-03T06:02:37.730 DEBUG:boto:Token: None 2022-09-03T06:02:37.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/?versions 2022-09-03T06:02:37.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYsIUSMAALuYm0TvCNtkvCJh2/8= 2022-09-03T06:02:37.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYsIUSMAALuYm0TvCNtkvCJh2/8=', 'Content-Length': '0'} 2022-09-03T06:02:37.736 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000068dc32227d4fd946-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.736 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.737 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:37.739 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:37.739 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:37.740 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:37.740 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:02:37.741 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.741 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.742 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.742 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.743 DEBUG:boto:Method: GET 2022-09-03T06:02:37.743 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.744 DEBUG:boto:Data: 2022-09-03T06:02:37.744 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.745 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:37.745 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.746 DEBUG:boto:Params: {} 2022-09-03T06:02:37.747 DEBUG:boto:Token: None 2022-09-03T06:02:37.747 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/?versions 2022-09-03T06:02:37.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYsIUSMAALuYm0TvCNtkvCJh2/8= 2022-09-03T06:02:37.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYsIUSMAALuYm0TvCNtkvCJh2/8=', 'Content-Length': '0'} 2022-09-03T06:02:37.751 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb7a6665b3cb4d83-006312edfd-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.752 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.753 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:37.754 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:37.754 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:37.755 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:37.755 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.756 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.756 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.757 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.757 DEBUG:boto:Method: GET 2022-09-03T06:02:37.758 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.758 DEBUG:boto:Data: 2022-09-03T06:02:37.759 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.760 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.760 DEBUG:boto:Params: {} 2022-09-03T06:02:37.761 DEBUG:boto:Token: None 2022-09-03T06:02:37.762 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/?versions 2022-09-03T06:02:37.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYsIUSMAALuYm0TvCNtkvCJh2/8= 2022-09-03T06:02:37.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYsIUSMAALuYm0TvCNtkvCJh2/8=', 'Content-Length': '0'} 2022-09-03T06:02:37.766 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e15f0387c44914f9-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.767 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.768 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.768 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.769 DEBUG:boto:path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.769 DEBUG:boto:auth_path=/avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.770 DEBUG:boto:Method: GET 2022-09-03T06:02:37.771 DEBUG:boto:Path: /avysmw-16/?versions&max-keys=999 2022-09-03T06:02:37.771 DEBUG:boto:Data: 2022-09-03T06:02:37.772 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.772 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:37.773 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.773 DEBUG:boto:Params: {} 2022-09-03T06:02:37.774 DEBUG:boto:Token: None 2022-09-03T06:02:37.775 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/?versions 2022-09-03T06:02:37.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYsIUSMAALuYm0TvCNtkvCJh2/8= 2022-09-03T06:02:37.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYsIUSMAALuYm0TvCNtkvCJh2/8=', 'Content-Length': '0'} 2022-09-03T06:02:37.779 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006672d961e2716c02-006312edfd-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.780 DEBUG:boto:b'avysmw-16999falsedummygNIZcq0z9Z.90E4hCKtPuJhniNFx5vBtrue2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1aYbEy84We0aneWDx62itIemti5vbHoutrue2022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2V2-lM2mlha96yWVnVanQER25LMSeNhGtrue2022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone35AkL4GBpYlOpN9xDzKN0s-PgBaOse-Ptrue2022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.781 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:37.782 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.782 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.783 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.783 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.784 DEBUG:boto:Method: GET 2022-09-03T06:02:37.785 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.785 DEBUG:boto:Data: 2022-09-03T06:02:37.786 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.786 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.787 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.787 DEBUG:boto:Params: {} 2022-09-03T06:02:37.788 DEBUG:boto:Token: None 2022-09-03T06:02:37.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI= 2022-09-03T06:02:37.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI=', 'Content-Length': '0'} 2022-09-03T06:02:37.792 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000283575677a8892f5-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.793 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.793 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.794 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.794 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.795 DEBUG:boto:Method: GET 2022-09-03T06:02:37.795 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.796 DEBUG:boto:Data: 2022-09-03T06:02:37.796 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.797 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:37.798 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.798 DEBUG:boto:Params: {} 2022-09-03T06:02:37.799 DEBUG:boto:Token: None 2022-09-03T06:02:37.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI= 2022-09-03T06:02:37.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI=', 'Content-Length': '0'} 2022-09-03T06:02:37.803 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000000c29f9e4ff02b952-006312edfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.804 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.804 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.805 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.805 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.806 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.806 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.807 DEBUG:boto:Data: 2022-09-03T06:02:37.807 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.808 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.809 DEBUG:boto:Params: {} 2022-09-03T06:02:37.809 DEBUG:boto:Token: None 2022-09-03T06:02:37.810 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MiDsyT/l9gJydj3Maj3218dhnLU= 2022-09-03T06:02:37.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MiDsyT/l9gJydj3Maj3218dhnLU=', 'Content-Length': '0'} 2022-09-03T06:02:37.813 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000bc46a4e812d17305-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.814 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.815 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.815 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.816 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.816 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.817 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.817 DEBUG:boto:Data: 2022-09-03T06:02:37.818 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.818 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:37.819 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.819 DEBUG:boto:Params: {} 2022-09-03T06:02:37.820 DEBUG:boto:Token: None 2022-09-03T06:02:37.820 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MiDsyT/l9gJydj3Maj3218dhnLU= 2022-09-03T06:02:37.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MiDsyT/l9gJydj3Maj3218dhnLU=', 'Content-Length': '0'} 2022-09-03T06:02:37.824 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a4ab56dd46fce8fa-006312edfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.825 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:37.825 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.826 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.826 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.827 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.827 DEBUG:boto:Method: GET 2022-09-03T06:02:37.828 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.828 DEBUG:boto:Data: 2022-09-03T06:02:37.829 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.829 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.830 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.831 DEBUG:boto:Params: {} 2022-09-03T06:02:37.831 DEBUG:boto:Token: None 2022-09-03T06:02:37.832 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI= 2022-09-03T06:02:37.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI=', 'Content-Length': '0'} 2022-09-03T06:02:37.836 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004113d5c1f61f17c7-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.836 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.837 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.838 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.838 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.839 DEBUG:boto:Method: GET 2022-09-03T06:02:37.839 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.840 DEBUG:boto:Data: 2022-09-03T06:02:37.841 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.842 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:37.843 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.844 DEBUG:boto:Params: {} 2022-09-03T06:02:37.845 DEBUG:boto:Token: None 2022-09-03T06:02:37.846 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.847 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI= 2022-09-03T06:02:37.848 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI=', 'Content-Length': '0'} 2022-09-03T06:02:37.852 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000000f81809fc0411e28-006312edfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.853 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.854 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.855 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:37.855 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:37.856 DEBUG:boto:Method: GET 2022-09-03T06:02:37.857 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:37.857 DEBUG:boto:Data: 2022-09-03T06:02:37.858 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.858 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.859 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.860 DEBUG:boto:Params: {} 2022-09-03T06:02:37.861 DEBUG:boto:Token: None 2022-09-03T06:02:37.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.867 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000017b3762faf5eec9-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.911 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.912 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:37.913 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:37.914 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:37.915 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:37.917 DEBUG:boto:Method: GET 2022-09-03T06:02:37.918 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:37.919 DEBUG:boto:Data: 2022-09-03T06:02:37.919 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.920 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:37.921 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.922 DEBUG:boto:Params: {} 2022-09-03T06:02:37.923 DEBUG:boto:Token: None 2022-09-03T06:02:37.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/ 2022-09-03T06:02:37.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU= 2022-09-03T06:02:37.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I74qqfTixxc6wJxzbt1WypuNqNU=', 'Content-Length': '0'} 2022-09-03T06:02:37.932 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d6d716b42ace5a89-006312edfd-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.933 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:37.935 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.935 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.936 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.938 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T06:02:37.939 DEBUG:boto:Data: 2022-09-03T06:02:37.940 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.940 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.941 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.942 DEBUG:boto:Params: {} 2022-09-03T06:02:37.943 DEBUG:boto:Token: None 2022-09-03T06:02:37.944 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy 2022-09-03T06:02:37.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NwuLt3lsD+v44ZtdQPf0KJoc2iQ= 2022-09-03T06:02:37.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NwuLt3lsD+v44ZtdQPf0KJoc2iQ=', 'Content-Length': '0'} 2022-09-03T06:02:37.951 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007a13b76ce09191f8-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.953 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.954 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.955 DEBUG:boto:Method: HEAD 2022-09-03T06:02:37.955 DEBUG:boto:Path: /avysmw-16/dummy 2022-09-03T06:02:37.956 DEBUG:boto:Data: 2022-09-03T06:02:37.957 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.958 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:37.959 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.959 DEBUG:boto:Params: {} 2022-09-03T06:02:37.960 DEBUG:boto:Token: None 2022-09-03T06:02:37.961 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy 2022-09-03T06:02:37.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NwuLt3lsD+v44ZtdQPf0KJoc2iQ= 2022-09-03T06:02:37.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NwuLt3lsD+v44ZtdQPf0KJoc2iQ=', 'Content-Length': '0'} 2022-09-03T06:02:37.965 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000eac33da144fcf67a-006312edfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.965 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:37.966 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.967 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.970 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.971 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.972 DEBUG:boto:Method: GET 2022-09-03T06:02:37.973 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.974 DEBUG:boto:Data: 2022-09-03T06:02:37.975 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.976 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:37.977 DEBUG:boto:Port: 8001 2022-09-03T06:02:37.978 DEBUG:boto:Params: {} 2022-09-03T06:02:37.979 DEBUG:boto:Token: None 2022-09-03T06:02:37.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI= 2022-09-03T06:02:37.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI=', 'Content-Length': '0'} 2022-09-03T06:02:37.987 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006461072a2ab83f58-006312edfd-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:37.988 DEBUG:boto:path=/avysmw-16/dummy 2022-09-03T06:02:37.989 DEBUG:boto:auth_path=/avysmw-16/dummy 2022-09-03T06:02:37.990 DEBUG:boto:path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.990 DEBUG:boto:auth_path=/avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.991 DEBUG:boto:Method: GET 2022-09-03T06:02:37.991 DEBUG:boto:Path: /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.992 DEBUG:boto:Data: 2022-09-03T06:02:37.992 DEBUG:boto:Headers: {} 2022-09-03T06:02:37.993 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:37.993 DEBUG:boto:Port: 8000 2022-09-03T06:02:37.994 DEBUG:boto:Params: {} 2022-09-03T06:02:37.995 DEBUG:boto:Token: None 2022-09-03T06:02:37.995 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:37 GMT /avysmw-16/dummy?versionId=gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB 2022-09-03T06:02:37.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI= 2022-09-03T06:02:37.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kxz6LgVW4PVoozYut6AMdMK6MEI=', 'Content-Length': '0'} 2022-09-03T06:02:37.999 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:55:58 GMT'), ('x-amz-version-id', 'gNIZcq0z9Z.90E4hCKtPuJhniNFx5vB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f0b82bbf53a36ce8-006312edfd-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.000 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.000 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.001 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:38.001 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:38.002 DEBUG:boto:Method: GET 2022-09-03T06:02:38.003 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:38.003 DEBUG:boto:Data: 2022-09-03T06:02:38.004 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.004 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.005 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.005 DEBUG:boto:Params: {} 2022-09-03T06:02:38.006 DEBUG:boto:Token: None 2022-09-03T06:02:38.007 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a5480c95a56c097-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.011 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.012 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.013 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.013 DEBUG:boto:path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:38.014 DEBUG:boto:auth_path=/avysmw-16/?prefix=dummy 2022-09-03T06:02:38.014 DEBUG:boto:Method: GET 2022-09-03T06:02:38.015 DEBUG:boto:Path: /avysmw-16/?prefix=dummy 2022-09-03T06:02:38.015 DEBUG:boto:Data: 2022-09-03T06:02:38.016 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.017 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.017 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.018 DEBUG:boto:Params: {} 2022-09-03T06:02:38.019 DEBUG:boto:Token: None 2022-09-03T06:02:38.019 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.023 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001243c79776ba8cad-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.024 DEBUG:boto:b'avysmw-16dummy1000falsedummy2022-09-03T05:55:58.035Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.025 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:38.025 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.026 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.026 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.027 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.027 DEBUG:boto:Method: GET 2022-09-03T06:02:38.028 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.029 DEBUG:boto:Data: 2022-09-03T06:02:38.029 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.030 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.030 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.031 DEBUG:boto:Params: {} 2022-09-03T06:02:38.032 DEBUG:boto:Token: None 2022-09-03T06:02:38.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI= 2022-09-03T06:02:38.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI=', 'Content-Length': '0'} 2022-09-03T06:02:38.036 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000001ca0513e4feb59e-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.083 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.084 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.085 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.085 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.086 DEBUG:boto:Method: GET 2022-09-03T06:02:38.086 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.087 DEBUG:boto:Data: 2022-09-03T06:02:38.088 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.089 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.090 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.091 DEBUG:boto:Params: {} 2022-09-03T06:02:38.092 DEBUG:boto:Token: None 2022-09-03T06:02:38.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI= 2022-09-03T06:02:38.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI=', 'Content-Length': '0'} 2022-09-03T06:02:38.098 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d6c9ca0a58bb3357-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.099 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.100 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.100 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.101 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.101 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.102 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.103 DEBUG:boto:Data: 2022-09-03T06:02:38.104 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.104 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.105 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.105 DEBUG:boto:Params: {} 2022-09-03T06:02:38.106 DEBUG:boto:Token: None 2022-09-03T06:02:38.107 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Re1+0VfRTvLTR0CEVSkLpP7Uryc= 2022-09-03T06:02:38.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Re1+0VfRTvLTR0CEVSkLpP7Uryc=', 'Content-Length': '0'} 2022-09-03T06:02:38.111 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000065c00c072facbfa2-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.112 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.112 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.113 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.113 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.114 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.115 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.115 DEBUG:boto:Data: 2022-09-03T06:02:38.116 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.117 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.117 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.118 DEBUG:boto:Params: {} 2022-09-03T06:02:38.118 DEBUG:boto:Token: None 2022-09-03T06:02:38.119 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Re1+0VfRTvLTR0CEVSkLpP7Uryc= 2022-09-03T06:02:38.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Re1+0VfRTvLTR0CEVSkLpP7Uryc=', 'Content-Length': '0'} 2022-09-03T06:02:38.123 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000996ccdce06c65076-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.123 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:38.124 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.125 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.125 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.126 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.127 DEBUG:boto:Method: GET 2022-09-03T06:02:38.127 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.128 DEBUG:boto:Data: 2022-09-03T06:02:38.128 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.129 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.130 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.130 DEBUG:boto:Params: {} 2022-09-03T06:02:38.131 DEBUG:boto:Token: None 2022-09-03T06:02:38.132 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI= 2022-09-03T06:02:38.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI=', 'Content-Length': '0'} 2022-09-03T06:02:38.137 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f595522d663ad7b1-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.139 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.139 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.140 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.140 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.141 DEBUG:boto:Method: GET 2022-09-03T06:02:38.142 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.143 DEBUG:boto:Data: 2022-09-03T06:02:38.144 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.144 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.145 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.147 DEBUG:boto:Params: {} 2022-09-03T06:02:38.149 DEBUG:boto:Token: None 2022-09-03T06:02:38.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.152 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI= 2022-09-03T06:02:38.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI=', 'Content-Length': '0'} 2022-09-03T06:02:38.156 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006a5d4357f86d34c8-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.202 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.203 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.204 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.205 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.206 DEBUG:boto:Method: GET 2022-09-03T06:02:38.207 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.208 DEBUG:boto:Data: 2022-09-03T06:02:38.209 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.211 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.212 DEBUG:boto:Params: {} 2022-09-03T06:02:38.213 DEBUG:boto:Token: None 2022-09-03T06:02:38.215 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.220 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002977a32e35166320-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.221 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.222 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.222 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.223 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.223 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.224 DEBUG:boto:Method: GET 2022-09-03T06:02:38.225 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.226 DEBUG:boto:Data: 2022-09-03T06:02:38.226 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.227 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.228 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.228 DEBUG:boto:Params: {} 2022-09-03T06:02:38.229 DEBUG:boto:Token: None 2022-09-03T06:02:38.230 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.234 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063cae9212df4f524-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.235 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.236 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.237 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.237 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.238 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T06:02:38.239 DEBUG:boto:Data: 2022-09-03T06:02:38.239 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.240 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.241 DEBUG:boto:Params: {} 2022-09-03T06:02:38.242 DEBUG:boto:Token: None 2022-09-03T06:02:38.243 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1 2022-09-03T06:02:38.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HE6r+3NRmNQPVyH9kBY2RG+Weks= 2022-09-03T06:02:38.246 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HE6r+3NRmNQPVyH9kBY2RG+Weks=', 'Content-Length': '0'} 2022-09-03T06:02:38.249 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003e726a33196bd0bd-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.250 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.251 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.252 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.253 DEBUG:boto:Path: /avysmw-16/obj-test-zone1 2022-09-03T06:02:38.254 DEBUG:boto:Data: 2022-09-03T06:02:38.255 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.256 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.257 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.258 DEBUG:boto:Params: {} 2022-09-03T06:02:38.259 DEBUG:boto:Token: None 2022-09-03T06:02:38.260 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1 2022-09-03T06:02:38.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HE6r+3NRmNQPVyH9kBY2RG+Weks= 2022-09-03T06:02:38.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HE6r+3NRmNQPVyH9kBY2RG+Weks=', 'Content-Length': '0'} 2022-09-03T06:02:38.266 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000230ccf4bafc83592-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.267 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:38.267 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.268 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.269 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.270 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.271 DEBUG:boto:Method: GET 2022-09-03T06:02:38.272 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.273 DEBUG:boto:Data: 2022-09-03T06:02:38.274 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.275 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.276 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.277 DEBUG:boto:Params: {} 2022-09-03T06:02:38.278 DEBUG:boto:Token: None 2022-09-03T06:02:38.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI= 2022-09-03T06:02:38.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI=', 'Content-Length': '0'} 2022-09-03T06:02:38.284 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000d310ec8eabf7048-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.285 DEBUG:boto:path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.286 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1 2022-09-03T06:02:38.287 DEBUG:boto:path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.287 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.288 DEBUG:boto:Method: GET 2022-09-03T06:02:38.288 DEBUG:boto:Path: /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.289 DEBUG:boto:Data: 2022-09-03T06:02:38.290 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.292 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.293 DEBUG:boto:Params: {} 2022-09-03T06:02:38.294 DEBUG:boto:Token: None 2022-09-03T06:02:38.295 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone1?versionId=aYbEy84We0aneWDx62itIemti5vbHou 2022-09-03T06:02:38.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI= 2022-09-03T06:02:38.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N23JU+9nJq9VjdYTYtmbSdJIqGI=', 'Content-Length': '0'} 2022-09-03T06:02:38.300 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:58 GMT'), ('x-amz-version-id', 'aYbEy84We0aneWDx62itIemti5vbHou'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000423afb16d38a6288-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.302 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.303 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.304 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.304 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.305 DEBUG:boto:Method: GET 2022-09-03T06:02:38.305 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.306 DEBUG:boto:Data: 2022-09-03T06:02:38.306 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.307 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.307 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.308 DEBUG:boto:Params: {} 2022-09-03T06:02:38.309 DEBUG:boto:Token: None 2022-09-03T06:02:38.310 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.315 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008eec3243ae017027-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.316 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.317 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.318 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.319 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.319 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.320 DEBUG:boto:Method: GET 2022-09-03T06:02:38.321 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone1 2022-09-03T06:02:38.321 DEBUG:boto:Data: 2022-09-03T06:02:38.322 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.322 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.323 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.323 DEBUG:boto:Params: {} 2022-09-03T06:02:38.324 DEBUG:boto:Token: None 2022-09-03T06:02:38.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.336 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1401538c765b49a-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.337 DEBUG:boto:b'avysmw-16obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.038Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.338 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:38.339 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.340 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.341 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.341 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.342 DEBUG:boto:Method: GET 2022-09-03T06:02:38.343 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.343 DEBUG:boto:Data: 2022-09-03T06:02:38.344 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.345 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.346 DEBUG:boto:Params: {} 2022-09-03T06:02:38.347 DEBUG:boto:Token: None 2022-09-03T06:02:38.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY= 2022-09-03T06:02:38.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY=', 'Content-Length': '0'} 2022-09-03T06:02:38.351 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f39c5c67ee7fba14-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.405 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.406 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.406 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.407 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.407 DEBUG:boto:Method: GET 2022-09-03T06:02:38.408 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.409 DEBUG:boto:Data: 2022-09-03T06:02:38.409 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.411 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.412 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.413 DEBUG:boto:Params: {} 2022-09-03T06:02:38.414 DEBUG:boto:Token: None 2022-09-03T06:02:38.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY= 2022-09-03T06:02:38.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY=', 'Content-Length': '0'} 2022-09-03T06:02:38.418 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d6599aa1d00b052f-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.419 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.420 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.421 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.421 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.422 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.423 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.423 DEBUG:boto:Data: 2022-09-03T06:02:38.424 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.424 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.425 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.425 DEBUG:boto:Params: {} 2022-09-03T06:02:38.426 DEBUG:boto:Token: None 2022-09-03T06:02:38.427 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ec48RZiPxNyulG8BkwglJJpjLjA= 2022-09-03T06:02:38.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ec48RZiPxNyulG8BkwglJJpjLjA=', 'Content-Length': '0'} 2022-09-03T06:02:38.440 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000897047692f01b7de-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.441 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.442 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.443 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.444 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.444 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.445 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.446 DEBUG:boto:Data: 2022-09-03T06:02:38.447 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.448 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.449 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.449 DEBUG:boto:Params: {} 2022-09-03T06:02:38.450 DEBUG:boto:Token: None 2022-09-03T06:02:38.453 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ec48RZiPxNyulG8BkwglJJpjLjA= 2022-09-03T06:02:38.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ec48RZiPxNyulG8BkwglJJpjLjA=', 'Content-Length': '0'} 2022-09-03T06:02:38.457 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004cf988a3f10ec0f5-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.458 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:38.458 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.460 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.460 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.461 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.461 DEBUG:boto:Method: GET 2022-09-03T06:02:38.462 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.462 DEBUG:boto:Data: 2022-09-03T06:02:38.463 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.464 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.464 DEBUG:boto:Params: {} 2022-09-03T06:02:38.465 DEBUG:boto:Token: None 2022-09-03T06:02:38.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY= 2022-09-03T06:02:38.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY=', 'Content-Length': '0'} 2022-09-03T06:02:38.469 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e159cc1e176c13f5-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.470 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.471 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.471 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.472 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.472 DEBUG:boto:Method: GET 2022-09-03T06:02:38.473 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.473 DEBUG:boto:Data: 2022-09-03T06:02:38.474 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.474 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.475 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.476 DEBUG:boto:Params: {} 2022-09-03T06:02:38.476 DEBUG:boto:Token: None 2022-09-03T06:02:38.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY= 2022-09-03T06:02:38.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY=', 'Content-Length': '0'} 2022-09-03T06:02:38.481 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006bda927b0624ff6d-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.483 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.484 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.485 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.486 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.487 DEBUG:boto:Method: GET 2022-09-03T06:02:38.488 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.490 DEBUG:boto:Data: 2022-09-03T06:02:38.491 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.492 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.492 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.493 DEBUG:boto:Params: {} 2022-09-03T06:02:38.494 DEBUG:boto:Token: None 2022-09-03T06:02:38.495 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.499 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005d8695d267936ce0-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.500 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.501 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.502 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.502 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.503 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.504 DEBUG:boto:Method: GET 2022-09-03T06:02:38.505 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.506 DEBUG:boto:Data: 2022-09-03T06:02:38.507 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.508 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.509 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.509 DEBUG:boto:Params: {} 2022-09-03T06:02:38.510 DEBUG:boto:Token: None 2022-09-03T06:02:38.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.512 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.513 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.516 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000aa5c278666e2ce7-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.516 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.517 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.518 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.518 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.519 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T06:02:38.520 DEBUG:boto:Data: 2022-09-03T06:02:38.520 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.521 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.521 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.522 DEBUG:boto:Params: {} 2022-09-03T06:02:38.523 DEBUG:boto:Token: None 2022-09-03T06:02:38.523 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2 2022-09-03T06:02:38.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VQZH5GcaG8GMKVhtEAdVqBckzFA= 2022-09-03T06:02:38.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VQZH5GcaG8GMKVhtEAdVqBckzFA=', 'Content-Length': '0'} 2022-09-03T06:02:38.527 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000078d80531791da63-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.528 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.529 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.530 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.531 DEBUG:boto:Path: /avysmw-16/obj-test-zone2 2022-09-03T06:02:38.532 DEBUG:boto:Data: 2022-09-03T06:02:38.533 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.533 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.535 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.535 DEBUG:boto:Params: {} 2022-09-03T06:02:38.536 DEBUG:boto:Token: None 2022-09-03T06:02:38.536 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2 2022-09-03T06:02:38.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VQZH5GcaG8GMKVhtEAdVqBckzFA= 2022-09-03T06:02:38.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VQZH5GcaG8GMKVhtEAdVqBckzFA=', 'Content-Length': '0'} 2022-09-03T06:02:38.542 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000d8ab469646a7289-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.543 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:38.544 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.544 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.545 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.546 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.547 DEBUG:boto:Method: GET 2022-09-03T06:02:38.548 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.550 DEBUG:boto:Data: 2022-09-03T06:02:38.551 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.552 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.553 DEBUG:boto:Params: {} 2022-09-03T06:02:38.554 DEBUG:boto:Token: None 2022-09-03T06:02:38.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY= 2022-09-03T06:02:38.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY=', 'Content-Length': '0'} 2022-09-03T06:02:38.559 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e95ab545defef9bb-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.603 DEBUG:boto:path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.605 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2 2022-09-03T06:02:38.606 DEBUG:boto:path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.607 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.608 DEBUG:boto:Method: GET 2022-09-03T06:02:38.609 DEBUG:boto:Path: /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.611 DEBUG:boto:Data: 2022-09-03T06:02:38.611 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.612 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.613 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.614 DEBUG:boto:Params: {} 2022-09-03T06:02:38.615 DEBUG:boto:Token: None 2022-09-03T06:02:38.616 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone2?versionId=V2-lM2mlha96yWVnVanQER25LMSeNhG 2022-09-03T06:02:38.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY= 2022-09-03T06:02:38.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+b4J2AR0YhXoCX8e9XPuA6wy3HY=', 'Content-Length': '0'} 2022-09-03T06:02:38.621 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'V2-lM2mlha96yWVnVanQER25LMSeNhG'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000094cfe6a8e7238c09-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.622 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.623 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.624 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.624 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.625 DEBUG:boto:Method: GET 2022-09-03T06:02:38.626 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.627 DEBUG:boto:Data: 2022-09-03T06:02:38.628 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.630 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.631 DEBUG:boto:Params: {} 2022-09-03T06:02:38.632 DEBUG:boto:Token: None 2022-09-03T06:02:38.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.638 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d4db08409c8ccef7-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.639 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.641 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.642 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.643 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.644 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.645 DEBUG:boto:Method: GET 2022-09-03T06:02:38.646 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone2 2022-09-03T06:02:38.647 DEBUG:boto:Data: 2022-09-03T06:02:38.648 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.648 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.649 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.650 DEBUG:boto:Params: {} 2022-09-03T06:02:38.651 DEBUG:boto:Token: None 2022-09-03T06:02:38.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.656 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a8c441b7b65e9fd7-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.658 DEBUG:boto:b'avysmw-16obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.197Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.658 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:38.659 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.660 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.660 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.661 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.661 DEBUG:boto:Method: GET 2022-09-03T06:02:38.662 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.662 DEBUG:boto:Data: 2022-09-03T06:02:38.663 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.664 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.664 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.665 DEBUG:boto:Params: {} 2022-09-03T06:02:38.665 DEBUG:boto:Token: None 2022-09-03T06:02:38.666 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k= 2022-09-03T06:02:38.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k=', 'Content-Length': '0'} 2022-09-03T06:02:38.670 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006084276c8661961b-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.671 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.672 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.673 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.676 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.677 DEBUG:boto:Method: GET 2022-09-03T06:02:38.679 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.680 DEBUG:boto:Data: 2022-09-03T06:02:38.681 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.682 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.683 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.684 DEBUG:boto:Params: {} 2022-09-03T06:02:38.685 DEBUG:boto:Token: None 2022-09-03T06:02:38.686 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k= 2022-09-03T06:02:38.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k=', 'Content-Length': '0'} 2022-09-03T06:02:38.691 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005d11707753990f1a-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.692 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.693 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.694 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.695 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.696 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.697 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.698 DEBUG:boto:Data: 2022-09-03T06:02:38.699 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.700 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.701 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.701 DEBUG:boto:Params: {} 2022-09-03T06:02:38.702 DEBUG:boto:Token: None 2022-09-03T06:02:38.703 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FfDGH6dYxLIpqyzSxxNGP87s+oc= 2022-09-03T06:02:38.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FfDGH6dYxLIpqyzSxxNGP87s+oc=', 'Content-Length': '0'} 2022-09-03T06:02:38.709 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003f85deedddc6ff40-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.710 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.711 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.711 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.712 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.712 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.713 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.714 DEBUG:boto:Data: 2022-09-03T06:02:38.714 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.715 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.716 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.717 DEBUG:boto:Params: {} 2022-09-03T06:02:38.718 DEBUG:boto:Token: None 2022-09-03T06:02:38.719 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FfDGH6dYxLIpqyzSxxNGP87s+oc= 2022-09-03T06:02:38.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FfDGH6dYxLIpqyzSxxNGP87s+oc=', 'Content-Length': '0'} 2022-09-03T06:02:38.724 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000263a32d6d4d72002-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.725 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:38.726 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.727 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.728 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.729 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.729 DEBUG:boto:Method: GET 2022-09-03T06:02:38.730 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.731 DEBUG:boto:Data: 2022-09-03T06:02:38.731 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.732 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.733 DEBUG:boto:Params: {} 2022-09-03T06:02:38.734 DEBUG:boto:Token: None 2022-09-03T06:02:38.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k= 2022-09-03T06:02:38.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k=', 'Content-Length': '0'} 2022-09-03T06:02:38.739 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000013c2d12a67598bb4-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.782 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.783 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.783 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.784 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.784 DEBUG:boto:Method: GET 2022-09-03T06:02:38.785 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.786 DEBUG:boto:Data: 2022-09-03T06:02:38.787 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.787 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.788 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.788 DEBUG:boto:Params: {} 2022-09-03T06:02:38.789 DEBUG:boto:Token: None 2022-09-03T06:02:38.790 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k= 2022-09-03T06:02:38.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k=', 'Content-Length': '0'} 2022-09-03T06:02:38.793 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008ce57f6eaa8ebd81-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.794 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.795 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.795 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.796 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.796 DEBUG:boto:Method: GET 2022-09-03T06:02:38.797 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.798 DEBUG:boto:Data: 2022-09-03T06:02:38.798 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.799 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.799 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.800 DEBUG:boto:Params: {} 2022-09-03T06:02:38.800 DEBUG:boto:Token: None 2022-09-03T06:02:38.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.805 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bda3c2d91e0e6f15-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.805 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.807 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.807 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.808 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.808 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.809 DEBUG:boto:Method: GET 2022-09-03T06:02:38.809 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.810 DEBUG:boto:Data: 2022-09-03T06:02:38.810 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.811 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.811 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.812 DEBUG:boto:Params: {} 2022-09-03T06:02:38.812 DEBUG:boto:Token: None 2022-09-03T06:02:38.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.817 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d598fdc40818123d-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.818 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.820 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.820 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.821 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.822 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T06:02:38.822 DEBUG:boto:Data: 2022-09-03T06:02:38.823 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.824 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.825 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.826 DEBUG:boto:Params: {} 2022-09-03T06:02:38.827 DEBUG:boto:Token: None 2022-09-03T06:02:38.827 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3 2022-09-03T06:02:38.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ffbCujMOIxNribFfRKjWLe/azig= 2022-09-03T06:02:38.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ffbCujMOIxNribFfRKjWLe/azig=', 'Content-Length': '0'} 2022-09-03T06:02:38.832 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000047512118a1646a54-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.833 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.833 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.834 DEBUG:boto:Method: HEAD 2022-09-03T06:02:38.834 DEBUG:boto:Path: /avysmw-16/obj-test-zone3 2022-09-03T06:02:38.835 DEBUG:boto:Data: 2022-09-03T06:02:38.835 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.836 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.836 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.837 DEBUG:boto:Params: {} 2022-09-03T06:02:38.838 DEBUG:boto:Token: None 2022-09-03T06:02:38.839 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3 2022-09-03T06:02:38.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ffbCujMOIxNribFfRKjWLe/azig= 2022-09-03T06:02:38.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ffbCujMOIxNribFfRKjWLe/azig=', 'Content-Length': '0'} 2022-09-03T06:02:38.842 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000057476bf1c6e36a13-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.843 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:38.843 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.844 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.844 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.845 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.845 DEBUG:boto:Method: GET 2022-09-03T06:02:38.846 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.846 DEBUG:boto:Data: 2022-09-03T06:02:38.847 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.848 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.849 DEBUG:boto:Params: {} 2022-09-03T06:02:38.849 DEBUG:boto:Token: None 2022-09-03T06:02:38.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k= 2022-09-03T06:02:38.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k=', 'Content-Length': '0'} 2022-09-03T06:02:38.853 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000051fb54accefc5bb2-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.854 DEBUG:boto:path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.854 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3 2022-09-03T06:02:38.855 DEBUG:boto:path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.855 DEBUG:boto:auth_path=/avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.856 DEBUG:boto:Method: GET 2022-09-03T06:02:38.856 DEBUG:boto:Path: /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.857 DEBUG:boto:Data: 2022-09-03T06:02:38.858 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.858 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.859 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.859 DEBUG:boto:Params: {} 2022-09-03T06:02:38.860 DEBUG:boto:Token: None 2022-09-03T06:02:38.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/obj-test-zone3?versionId=5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P 2022-09-03T06:02:38.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k= 2022-09-03T06:02:38.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOvst1ipwK2Uc25V0x0ueJK0r6k=', 'Content-Length': '0'} 2022-09-03T06:02:38.864 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', '5AkL4GBpYlOpN9xDzKN0s-PgBaOse-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c8c562e58aad6544-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.865 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.866 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.866 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.867 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.867 DEBUG:boto:Method: GET 2022-09-03T06:02:38.868 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.868 DEBUG:boto:Data: 2022-09-03T06:02:38.869 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.869 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:38.870 DEBUG:boto:Port: 8001 2022-09-03T06:02:38.870 DEBUG:boto:Params: {} 2022-09-03T06:02:38.871 DEBUG:boto:Token: None 2022-09-03T06:02:38.871 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.875 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004679eeaf4463da78-006312edfe-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.876 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.877 DEBUG:boto:path=/avysmw-16/ 2022-09-03T06:02:38.877 DEBUG:boto:auth_path=/avysmw-16/ 2022-09-03T06:02:38.878 DEBUG:boto:path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.878 DEBUG:boto:auth_path=/avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.879 DEBUG:boto:Method: GET 2022-09-03T06:02:38.879 DEBUG:boto:Path: /avysmw-16/?prefix=obj-test-zone3 2022-09-03T06:02:38.880 DEBUG:boto:Data: 2022-09-03T06:02:38.880 DEBUG:boto:Headers: {} 2022-09-03T06:02:38.881 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:38.882 DEBUG:boto:Port: 8000 2022-09-03T06:02:38.883 DEBUG:boto:Params: {} 2022-09-03T06:02:38.883 DEBUG:boto:Token: None 2022-09-03T06:02:38.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:38 GMT /avysmw-16/ 2022-09-03T06:02:38.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8= 2022-09-03T06:02:38.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ig1o7ycFw02yOdgX4+T105r8bC8=', 'Content-Length': '0'} 2022-09-03T06:02:38.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000003e9fb5cdd4343e6-006312edfe-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:38.890 DEBUG:boto:b'avysmw-16obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.377Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:38.892 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-16 zones={test-zone3, test-zone2} 2022-09-03T06:02:38.892 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-17 2022-09-03T06:02:38.893 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:38.894 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:38.895 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:38.959 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:38.960 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:39.010 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:39.008+0000 7fb4e5128600 1 bucket sync caught up with source: 2022-09-03T06:02:39.011 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000011.25.5, , 00000000004.18.5, , 00000000008.22.5, , , , , ] 2022-09-03T06:02:39.011 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.25.5, , 00000000004.18.5, , 00000000008.22.5, , , , , ] 2022-09-03T06:02:39.011 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:39.008+0000 7fb4e5128600 0 bucket checkpoint complete 2022-09-03T06:02:39.028 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-17 2022-09-03T06:02:39.029 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.029 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.030 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:39.095 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:39.096 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:39.154 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:39.151+0000 7fe8023c6600 1 bucket sync caught up with source: 2022-09-03T06:02:39.154 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000011.25.5, , 00000000004.18.5, , 00000000008.22.5, , , , , ] 2022-09-03T06:02:39.154 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000011.25.5, , 00000000004.18.5, , 00000000008.22.5, , , , , ] 2022-09-03T06:02:39.155 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:39.151+0000 7fe8023c6600 0 bucket checkpoint complete 2022-09-03T06:02:39.174 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-17 2022-09-03T06:02:39.175 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.176 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.177 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:39.234 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:39.235 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:39.279 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:39.277+0000 7f4e93a8b600 1 bucket sync caught up with source: 2022-09-03T06:02:39.280 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000011.25.5, , 00000000004.18.5, , 00000000008.22.5, , , , , ] 2022-09-03T06:02:39.280 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.25.5, , 00000000004.18.5, , 00000000008.22.5, , , , , ] 2022-09-03T06:02:39.280 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:39.277+0000 7f4e93a8b600 0 bucket checkpoint complete 2022-09-03T06:02:39.291 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-17 2022-09-03T06:02:39.292 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.293 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.294 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:39.353 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:39.354 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:39.405 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:39.403+0000 7fc8fa8ed600 1 bucket sync caught up with source: 2022-09-03T06:02:39.405 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.19.5, , 00000000011.49.5, , 00000000009.32.5, , , , , ] 2022-09-03T06:02:39.406 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.19.5, , 00000000011.49.5, , 00000000009.32.5, , , , , ] 2022-09-03T06:02:39.406 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:39.403+0000 7fc8fa8ed600 0 bucket checkpoint complete 2022-09-03T06:02:39.421 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-17 2022-09-03T06:02:39.422 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.422 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.423 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:39.483 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:39.484 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:39.520 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:39.518+0000 7feefd681600 1 bucket sync caught up with source: 2022-09-03T06:02:39.520 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.19.5, , 00000000011.49.5, , 00000000009.32.5, , , , , ] 2022-09-03T06:02:39.521 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.19.5, , 00000000011.49.5, , 00000000009.32.5, , , , , ] 2022-09-03T06:02:39.521 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:39.518+0000 7feefd681600 0 bucket checkpoint complete 2022-09-03T06:02:39.533 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-17 2022-09-03T06:02:39.534 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-17 2022-09-03T06:02:39.535 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.535 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.536 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:39.594 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:39.595 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:39.644 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:39.642+0000 7f5aab05c600 1 bucket sync caught up with source: 2022-09-03T06:02:39.644 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.18.5, , 00000000004.38.5, , 00000000015.43.5, , , , , ] 2022-09-03T06:02:39.644 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.18.5, , 00000000004.38.5, , 00000000015.43.5, , , , , ] 2022-09-03T06:02:39.645 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:39.642+0000 7f5aab05c600 0 bucket checkpoint complete 2022-09-03T06:02:39.661 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-17 2022-09-03T06:02:39.662 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.663 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:39.663 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-17 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:39.720 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:39.721 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:39.755 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:39.752+0000 7fefd2de3600 1 bucket sync caught up with source: 2022-09-03T06:02:39.755 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000004.18.5, , 00000000004.38.5, , 00000000015.43.5, , , , , ] 2022-09-03T06:02:39.756 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000004.18.5, , 00000000004.38.5, , 00000000015.43.5, , , , , ] 2022-09-03T06:02:39.756 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:39.752+0000 7fefd2de3600 0 bucket checkpoint complete 2022-09-03T06:02:39.767 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-17 2022-09-03T06:02:39.768 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-17 zones={test-zone2, test-zone1} 2022-09-03T06:02:39.769 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:39.770 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:39.771 DEBUG:boto:Method: HEAD 2022-09-03T06:02:39.772 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T06:02:39.772 DEBUG:boto:Data: 2022-09-03T06:02:39.774 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.774 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:39.775 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.775 DEBUG:boto:Params: {} 2022-09-03T06:02:39.776 DEBUG:boto:Token: None 2022-09-03T06:02:39.777 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/ 2022-09-03T06:02:39.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0n9kgx/ggrtse12nzl+zGk9A2vw= 2022-09-03T06:02:39.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0n9kgx/ggrtse12nzl+zGk9A2vw=', 'Content-Length': '0'} 2022-09-03T06:02:39.781 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002a2b7286ec2ade93-006312edff-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.782 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:39.783 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:39.784 DEBUG:boto:Method: HEAD 2022-09-03T06:02:39.785 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T06:02:39.786 DEBUG:boto:Data: 2022-09-03T06:02:39.787 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.788 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:39.789 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.789 DEBUG:boto:Params: {} 2022-09-03T06:02:39.790 DEBUG:boto:Token: None 2022-09-03T06:02:39.790 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/ 2022-09-03T06:02:39.791 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0n9kgx/ggrtse12nzl+zGk9A2vw= 2022-09-03T06:02:39.792 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0n9kgx/ggrtse12nzl+zGk9A2vw=', 'Content-Length': '0'} 2022-09-03T06:02:39.794 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007237588f5652c561-006312edff-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.795 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:02:39.796 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:39.796 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:39.797 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.797 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.798 DEBUG:boto:Method: GET 2022-09-03T06:02:39.798 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.799 DEBUG:boto:Data: 2022-09-03T06:02:39.799 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.800 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:39.801 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.801 DEBUG:boto:Params: {} 2022-09-03T06:02:39.802 DEBUG:boto:Token: None 2022-09-03T06:02:39.802 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/?versions 2022-09-03T06:02:39.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7NtPMMaWWJUT1CTlC6K7888GAQA= 2022-09-03T06:02:39.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7NtPMMaWWJUT1CTlC6K7888GAQA=', 'Content-Length': '0'} 2022-09-03T06:02:39.806 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000252ddcc5b73b8b05-006312edff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.807 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:39.809 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:39.809 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:39.810 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:39.810 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:39.812 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:02:39.812 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:39.813 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:39.813 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.814 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.815 DEBUG:boto:Method: GET 2022-09-03T06:02:39.815 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.816 DEBUG:boto:Data: 2022-09-03T06:02:39.816 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:39.818 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.819 DEBUG:boto:Params: {} 2022-09-03T06:02:39.820 DEBUG:boto:Token: None 2022-09-03T06:02:39.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/?versions 2022-09-03T06:02:39.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7NtPMMaWWJUT1CTlC6K7888GAQA= 2022-09-03T06:02:39.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7NtPMMaWWJUT1CTlC6K7888GAQA=', 'Content-Length': '0'} 2022-09-03T06:02:39.825 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000003307775b923ed62-006312edff-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.825 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:39.826 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:39.827 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:39.827 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:39.828 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:39.828 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:39.829 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:39.830 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.830 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.831 DEBUG:boto:Method: GET 2022-09-03T06:02:39.831 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.832 DEBUG:boto:Data: 2022-09-03T06:02:39.832 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.833 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:39.833 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.834 DEBUG:boto:Params: {} 2022-09-03T06:02:39.834 DEBUG:boto:Token: None 2022-09-03T06:02:39.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/?versions 2022-09-03T06:02:39.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7NtPMMaWWJUT1CTlC6K7888GAQA= 2022-09-03T06:02:39.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7NtPMMaWWJUT1CTlC6K7888GAQA=', 'Content-Length': '0'} 2022-09-03T06:02:39.840 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007371bb19919fe087-006312edff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.840 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:39.841 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:39.842 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:39.843 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.843 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.844 DEBUG:boto:Method: GET 2022-09-03T06:02:39.844 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:39.845 DEBUG:boto:Data: 2022-09-03T06:02:39.845 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:39.846 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.847 DEBUG:boto:Params: {} 2022-09-03T06:02:39.848 DEBUG:boto:Token: None 2022-09-03T06:02:39.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/?versions 2022-09-03T06:02:39.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7NtPMMaWWJUT1CTlC6K7888GAQA= 2022-09-03T06:02:39.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7NtPMMaWWJUT1CTlC6K7888GAQA=', 'Content-Length': '0'} 2022-09-03T06:02:39.852 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000129ec12d1ebe32c1-006312edff-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.853 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:39.853 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:39.854 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:39.855 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:39.855 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.856 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.856 DEBUG:boto:Method: GET 2022-09-03T06:02:39.857 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.857 DEBUG:boto:Data: 2022-09-03T06:02:39.858 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.858 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:39.859 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.860 DEBUG:boto:Params: {} 2022-09-03T06:02:39.860 DEBUG:boto:Token: None 2022-09-03T06:02:39.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1T2ee716Nc/LCyb6qxjRrbM75yA= 2022-09-03T06:02:39.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1T2ee716Nc/LCyb6qxjRrbM75yA=', 'Content-Length': '0'} 2022-09-03T06:02:39.865 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000405547f93c7c818f-006312edff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.865 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:39.866 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:39.867 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.867 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.868 DEBUG:boto:Method: GET 2022-09-03T06:02:39.868 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.869 DEBUG:boto:Data: 2022-09-03T06:02:39.869 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.870 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:39.871 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.871 DEBUG:boto:Params: {} 2022-09-03T06:02:39.872 DEBUG:boto:Token: None 2022-09-03T06:02:39.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1T2ee716Nc/LCyb6qxjRrbM75yA= 2022-09-03T06:02:39.874 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1T2ee716Nc/LCyb6qxjRrbM75yA=', 'Content-Length': '0'} 2022-09-03T06:02:39.876 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003be1d9eee1cf0cce-006312edff-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.877 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:39.877 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:39.878 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.878 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.879 DEBUG:boto:Method: HEAD 2022-09-03T06:02:39.879 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.880 DEBUG:boto:Data: 2022-09-03T06:02:39.880 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.881 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:39.881 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.882 DEBUG:boto:Params: {} 2022-09-03T06:02:39.883 DEBUG:boto:Token: None 2022-09-03T06:02:39.883 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B8Gxf2aB1koD0EeXxxkN+8XAtNU= 2022-09-03T06:02:39.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B8Gxf2aB1koD0EeXxxkN+8XAtNU=', 'Content-Length': '0'} 2022-09-03T06:02:39.886 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a698eaf7e8738bc3-006312edff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.887 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:39.888 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:39.888 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.889 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.889 DEBUG:boto:Method: HEAD 2022-09-03T06:02:39.890 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.890 DEBUG:boto:Data: 2022-09-03T06:02:39.891 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.891 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:39.892 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.892 DEBUG:boto:Params: {} 2022-09-03T06:02:39.893 DEBUG:boto:Token: None 2022-09-03T06:02:39.894 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B8Gxf2aB1koD0EeXxxkN+8XAtNU= 2022-09-03T06:02:39.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B8Gxf2aB1koD0EeXxxkN+8XAtNU=', 'Content-Length': '0'} 2022-09-03T06:02:39.897 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004190001da3b06adf-006312edff-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.898 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:39.898 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:39.899 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:39.899 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.900 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.901 DEBUG:boto:Method: GET 2022-09-03T06:02:39.901 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.902 DEBUG:boto:Data: 2022-09-03T06:02:39.903 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.904 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:39.904 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.905 DEBUG:boto:Params: {} 2022-09-03T06:02:39.905 DEBUG:boto:Token: None 2022-09-03T06:02:39.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1T2ee716Nc/LCyb6qxjRrbM75yA= 2022-09-03T06:02:39.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1T2ee716Nc/LCyb6qxjRrbM75yA=', 'Content-Length': '0'} 2022-09-03T06:02:39.910 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000419675d2f3ca1037-006312edff-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.911 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:39.911 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:39.912 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.912 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.913 DEBUG:boto:Method: GET 2022-09-03T06:02:39.914 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.914 DEBUG:boto:Data: 2022-09-03T06:02:39.915 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.916 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:39.916 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.917 DEBUG:boto:Params: {} 2022-09-03T06:02:39.917 DEBUG:boto:Token: None 2022-09-03T06:02:39.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:39.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1T2ee716Nc/LCyb6qxjRrbM75yA= 2022-09-03T06:02:39.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1T2ee716Nc/LCyb6qxjRrbM75yA=', 'Content-Length': '0'} 2022-09-03T06:02:39.921 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000389f2f3ad32f956e-006312edff-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.922 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:39.923 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:39.923 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:39.924 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:39.925 DEBUG:boto:Method: GET 2022-09-03T06:02:39.925 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:39.926 DEBUG:boto:Data: 2022-09-03T06:02:39.926 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.927 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:39.927 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.928 DEBUG:boto:Params: {} 2022-09-03T06:02:39.928 DEBUG:boto:Token: None 2022-09-03T06:02:39.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/ 2022-09-03T06:02:39.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CRWkhccUifp7TOZ7kt83Qop5dKY= 2022-09-03T06:02:39.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CRWkhccUifp7TOZ7kt83Qop5dKY=', 'Content-Length': '0'} 2022-09-03T06:02:39.933 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058d30db0ad04e9df-006312edff-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.978 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:39.979 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:39.980 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:39.980 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:39.981 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:39.982 DEBUG:boto:Method: GET 2022-09-03T06:02:39.983 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:39.984 DEBUG:boto:Data: 2022-09-03T06:02:39.984 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:39.985 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.986 DEBUG:boto:Params: {} 2022-09-03T06:02:39.987 DEBUG:boto:Token: None 2022-09-03T06:02:39.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/ 2022-09-03T06:02:39.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CRWkhccUifp7TOZ7kt83Qop5dKY= 2022-09-03T06:02:39.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CRWkhccUifp7TOZ7kt83Qop5dKY=', 'Content-Length': '0'} 2022-09-03T06:02:39.991 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a7a0a833aa9c853-006312edff-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:39.992 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:39.993 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:39.993 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:39.994 DEBUG:boto:Method: HEAD 2022-09-03T06:02:39.994 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T06:02:39.995 DEBUG:boto:Data: 2022-09-03T06:02:39.995 DEBUG:boto:Headers: {} 2022-09-03T06:02:39.996 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:39.996 DEBUG:boto:Port: 8000 2022-09-03T06:02:39.997 DEBUG:boto:Params: {} 2022-09-03T06:02:39.998 DEBUG:boto:Token: None 2022-09-03T06:02:39.999 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:39 GMT /avysmw-17/dummy 2022-09-03T06:02:40.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H9PL7gXphRjhro9HI0c8BNdN3Cs= 2022-09-03T06:02:40.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H9PL7gXphRjhro9HI0c8BNdN3Cs=', 'Content-Length': '0'} 2022-09-03T06:02:40.005 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000032d172805e239211-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.006 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:40.006 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:40.007 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.007 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T06:02:40.008 DEBUG:boto:Data: 2022-09-03T06:02:40.008 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.010 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.011 DEBUG:boto:Params: {} 2022-09-03T06:02:40.011 DEBUG:boto:Token: None 2022-09-03T06:02:40.012 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/dummy 2022-09-03T06:02:40.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6fCNKj5Xk71fhB3arGI1tJbV1Mk= 2022-09-03T06:02:40.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6fCNKj5Xk71fhB3arGI1tJbV1Mk=', 'Content-Length': '0'} 2022-09-03T06:02:40.016 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000013089984e70bc49e-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.016 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:40.017 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:40.017 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:40.018 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.019 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.019 DEBUG:boto:Method: GET 2022-09-03T06:02:40.020 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.020 DEBUG:boto:Data: 2022-09-03T06:02:40.021 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.022 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.022 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.023 DEBUG:boto:Params: {} 2022-09-03T06:02:40.024 DEBUG:boto:Token: None 2022-09-03T06:02:40.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1S7rBj7ICMO8Ru1U6h3SJ6Bc1bA= 2022-09-03T06:02:40.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1S7rBj7ICMO8Ru1U6h3SJ6Bc1bA=', 'Content-Length': '0'} 2022-09-03T06:02:40.028 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008b80913f9d7fd57d-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.029 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:40.030 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:40.031 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.032 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.033 DEBUG:boto:Method: GET 2022-09-03T06:02:40.034 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.035 DEBUG:boto:Data: 2022-09-03T06:02:40.036 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.037 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.038 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.039 DEBUG:boto:Params: {} 2022-09-03T06:02:40.040 DEBUG:boto:Token: None 2022-09-03T06:02:40.041 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.042 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1S7rBj7ICMO8Ru1U6h3SJ6Bc1bA= 2022-09-03T06:02:40.043 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1S7rBj7ICMO8Ru1U6h3SJ6Bc1bA=', 'Content-Length': '0'} 2022-09-03T06:02:40.045 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f2ec85494096ef0f-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.046 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.047 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.048 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:40.049 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:40.050 DEBUG:boto:Method: GET 2022-09-03T06:02:40.051 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:40.052 DEBUG:boto:Data: 2022-09-03T06:02:40.053 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.054 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.056 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.057 DEBUG:boto:Params: {} 2022-09-03T06:02:40.058 DEBUG:boto:Token: None 2022-09-03T06:02:40.059 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.064 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f81fd0df01184e1b-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.064 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.065 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.066 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.066 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:40.067 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:40.067 DEBUG:boto:Method: GET 2022-09-03T06:02:40.068 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:40.069 DEBUG:boto:Data: 2022-09-03T06:02:40.069 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.070 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.070 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.071 DEBUG:boto:Params: {} 2022-09-03T06:02:40.071 DEBUG:boto:Token: None 2022-09-03T06:02:40.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.076 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036c80a5c33210fba-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.077 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.077 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:40.078 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.079 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.080 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.081 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.083 DEBUG:boto:Method: GET 2022-09-03T06:02:40.084 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.085 DEBUG:boto:Data: 2022-09-03T06:02:40.086 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.087 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.088 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.089 DEBUG:boto:Params: {} 2022-09-03T06:02:40.091 DEBUG:boto:Token: None 2022-09-03T06:02:40.092 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE= 2022-09-03T06:02:40.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE=', 'Content-Length': '0'} 2022-09-03T06:02:40.098 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000badb6c37cbd0bd0b-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.099 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.100 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.101 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.102 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.103 DEBUG:boto:Method: GET 2022-09-03T06:02:40.104 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.105 DEBUG:boto:Data: 2022-09-03T06:02:40.106 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.107 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.108 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.109 DEBUG:boto:Params: {} 2022-09-03T06:02:40.110 DEBUG:boto:Token: None 2022-09-03T06:02:40.111 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE= 2022-09-03T06:02:40.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE=', 'Content-Length': '0'} 2022-09-03T06:02:40.115 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002921d055b162a6b3-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.116 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.117 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.118 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.118 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.119 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.120 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.120 DEBUG:boto:Data: 2022-09-03T06:02:40.121 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.121 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.122 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.122 DEBUG:boto:Params: {} 2022-09-03T06:02:40.123 DEBUG:boto:Token: None 2022-09-03T06:02:40.123 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p7MtMEKRXEQu46QhxiPV5Zxmgho= 2022-09-03T06:02:40.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p7MtMEKRXEQu46QhxiPV5Zxmgho=', 'Content-Length': '0'} 2022-09-03T06:02:40.127 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dd4e16da6e47de26-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.129 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.130 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.132 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.133 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.134 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.135 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.136 DEBUG:boto:Data: 2022-09-03T06:02:40.136 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.138 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.139 DEBUG:boto:Params: {} 2022-09-03T06:02:40.139 DEBUG:boto:Token: None 2022-09-03T06:02:40.140 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p7MtMEKRXEQu46QhxiPV5Zxmgho= 2022-09-03T06:02:40.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p7MtMEKRXEQu46QhxiPV5Zxmgho=', 'Content-Length': '0'} 2022-09-03T06:02:40.144 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007143529db0be3672-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.147 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:40.148 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.149 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.150 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.151 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.151 DEBUG:boto:Method: GET 2022-09-03T06:02:40.152 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.153 DEBUG:boto:Data: 2022-09-03T06:02:40.153 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.154 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.156 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.157 DEBUG:boto:Params: {} 2022-09-03T06:02:40.158 DEBUG:boto:Token: None 2022-09-03T06:02:40.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE= 2022-09-03T06:02:40.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE=', 'Content-Length': '0'} 2022-09-03T06:02:40.164 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c2c2d81980eadc22-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.210 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.211 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.212 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.213 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.214 DEBUG:boto:Method: GET 2022-09-03T06:02:40.215 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.216 DEBUG:boto:Data: 2022-09-03T06:02:40.217 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.219 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.220 DEBUG:boto:Params: {} 2022-09-03T06:02:40.221 DEBUG:boto:Token: None 2022-09-03T06:02:40.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE= 2022-09-03T06:02:40.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE=', 'Content-Length': '0'} 2022-09-03T06:02:40.227 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002e997ddd83c76d9f-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.227 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.228 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.229 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.229 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.230 DEBUG:boto:Method: GET 2022-09-03T06:02:40.231 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.231 DEBUG:boto:Data: 2022-09-03T06:02:40.232 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.233 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.233 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.234 DEBUG:boto:Params: {} 2022-09-03T06:02:40.235 DEBUG:boto:Token: None 2022-09-03T06:02:40.235 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.240 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000014bf76a9f911a212-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.240 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.241 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.242 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.243 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.244 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.245 DEBUG:boto:Method: GET 2022-09-03T06:02:40.246 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.246 DEBUG:boto:Data: 2022-09-03T06:02:40.247 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.249 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.250 DEBUG:boto:Params: {} 2022-09-03T06:02:40.251 DEBUG:boto:Token: None 2022-09-03T06:02:40.251 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.252 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.255 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f99366ba9230f38b-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.298 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.299 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.300 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.301 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.301 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T06:02:40.302 DEBUG:boto:Data: 2022-09-03T06:02:40.303 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.304 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.305 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.306 DEBUG:boto:Params: {} 2022-09-03T06:02:40.307 DEBUG:boto:Token: None 2022-09-03T06:02:40.307 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1 2022-09-03T06:02:40.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7XAOyR14z4Iebf+4QQQngkGIJsc= 2022-09-03T06:02:40.308 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7XAOyR14z4Iebf+4QQQngkGIJsc=', 'Content-Length': '0'} 2022-09-03T06:02:40.311 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004c91ccc03decc4a2-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.312 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.312 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.313 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.313 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T06:02:40.314 DEBUG:boto:Data: 2022-09-03T06:02:40.314 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.315 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.315 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.316 DEBUG:boto:Params: {} 2022-09-03T06:02:40.316 DEBUG:boto:Token: None 2022-09-03T06:02:40.317 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1 2022-09-03T06:02:40.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7XAOyR14z4Iebf+4QQQngkGIJsc= 2022-09-03T06:02:40.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7XAOyR14z4Iebf+4QQQngkGIJsc=', 'Content-Length': '0'} 2022-09-03T06:02:40.320 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000090073e5a5702b4f0-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.321 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:40.321 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.322 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.322 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.323 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.323 DEBUG:boto:Method: GET 2022-09-03T06:02:40.324 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.324 DEBUG:boto:Data: 2022-09-03T06:02:40.325 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.326 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.326 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.327 DEBUG:boto:Params: {} 2022-09-03T06:02:40.327 DEBUG:boto:Token: None 2022-09-03T06:02:40.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE= 2022-09-03T06:02:40.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE=', 'Content-Length': '0'} 2022-09-03T06:02:40.331 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fce575d205a25a3e-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.332 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.333 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:40.333 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.334 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.334 DEBUG:boto:Method: GET 2022-09-03T06:02:40.335 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.336 DEBUG:boto:Data: 2022-09-03T06:02:40.336 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.337 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.338 DEBUG:boto:Params: {} 2022-09-03T06:02:40.338 DEBUG:boto:Token: None 2022-09-03T06:02:40.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:40.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE= 2022-09-03T06:02:40.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ji5Z0ol6gEHIw6l6wJCiayJ9ovE=', 'Content-Length': '0'} 2022-09-03T06:02:40.342 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f2594bdfbfa75979-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.343 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.344 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.344 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.345 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.345 DEBUG:boto:Method: GET 2022-09-03T06:02:40.346 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.347 DEBUG:boto:Data: 2022-09-03T06:02:40.347 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.348 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.348 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.349 DEBUG:boto:Params: {} 2022-09-03T06:02:40.350 DEBUG:boto:Token: None 2022-09-03T06:02:40.350 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.354 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000b28ee432bc17576-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.354 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.355 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.356 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.356 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.357 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.357 DEBUG:boto:Method: GET 2022-09-03T06:02:40.358 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:40.359 DEBUG:boto:Data: 2022-09-03T06:02:40.359 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.360 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.361 DEBUG:boto:Params: {} 2022-09-03T06:02:40.362 DEBUG:boto:Token: None 2022-09-03T06:02:40.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.366 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007f237eb5ea1b8d43-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.367 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.368 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:40.368 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.369 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.369 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.370 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.370 DEBUG:boto:Method: GET 2022-09-03T06:02:40.371 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.371 DEBUG:boto:Data: 2022-09-03T06:02:40.372 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.373 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.373 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.374 DEBUG:boto:Params: {} 2022-09-03T06:02:40.375 DEBUG:boto:Token: None 2022-09-03T06:02:40.375 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc= 2022-09-03T06:02:40.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc=', 'Content-Length': '0'} 2022-09-03T06:02:40.379 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000453d519de47e7da4-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.422 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.423 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.423 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.424 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.424 DEBUG:boto:Method: GET 2022-09-03T06:02:40.425 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.426 DEBUG:boto:Data: 2022-09-03T06:02:40.427 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.428 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.428 DEBUG:boto:Params: {} 2022-09-03T06:02:40.429 DEBUG:boto:Token: None 2022-09-03T06:02:40.431 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc= 2022-09-03T06:02:40.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc=', 'Content-Length': '0'} 2022-09-03T06:02:40.436 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004cea859dbe2b8d84-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.439 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.439 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.440 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.440 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.441 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.441 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.450 DEBUG:boto:Data: 2022-09-03T06:02:40.451 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.451 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.452 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.453 DEBUG:boto:Params: {} 2022-09-03T06:02:40.453 DEBUG:boto:Token: None 2022-09-03T06:02:40.454 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7ED86FlCmGArjvYh5+fYcactiY4= 2022-09-03T06:02:40.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7ED86FlCmGArjvYh5+fYcactiY4=', 'Content-Length': '0'} 2022-09-03T06:02:40.457 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000534ee8f7f6079bbc-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.458 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.459 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.459 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.460 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.460 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.461 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.461 DEBUG:boto:Data: 2022-09-03T06:02:40.462 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.463 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.463 DEBUG:boto:Params: {} 2022-09-03T06:02:40.464 DEBUG:boto:Token: None 2022-09-03T06:02:40.465 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7ED86FlCmGArjvYh5+fYcactiY4= 2022-09-03T06:02:40.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7ED86FlCmGArjvYh5+fYcactiY4=', 'Content-Length': '0'} 2022-09-03T06:02:40.468 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fbe0e85ee5ef9cf5-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.468 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:40.469 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.470 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.470 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.471 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.471 DEBUG:boto:Method: GET 2022-09-03T06:02:40.472 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.473 DEBUG:boto:Data: 2022-09-03T06:02:40.473 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.474 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.474 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.475 DEBUG:boto:Params: {} 2022-09-03T06:02:40.475 DEBUG:boto:Token: None 2022-09-03T06:02:40.476 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc= 2022-09-03T06:02:40.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc=', 'Content-Length': '0'} 2022-09-03T06:02:40.479 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dd1a4091d8a4d37c-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.481 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.481 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.482 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.482 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.483 DEBUG:boto:Method: GET 2022-09-03T06:02:40.484 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.484 DEBUG:boto:Data: 2022-09-03T06:02:40.485 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.486 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.486 DEBUG:boto:Params: {} 2022-09-03T06:02:40.487 DEBUG:boto:Token: None 2022-09-03T06:02:40.488 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc= 2022-09-03T06:02:40.489 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc=', 'Content-Length': '0'} 2022-09-03T06:02:40.491 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000073e5054fb1f00a6-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.492 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.492 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.493 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.493 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.494 DEBUG:boto:Method: GET 2022-09-03T06:02:40.495 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.496 DEBUG:boto:Data: 2022-09-03T06:02:40.496 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.497 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.497 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.498 DEBUG:boto:Params: {} 2022-09-03T06:02:40.498 DEBUG:boto:Token: None 2022-09-03T06:02:40.499 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.500 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.503 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000034d035004f4f4d1d-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.504 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.505 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.505 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.506 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.506 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.507 DEBUG:boto:Method: GET 2022-09-03T06:02:40.508 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.509 DEBUG:boto:Data: 2022-09-03T06:02:40.510 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.511 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.511 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.512 DEBUG:boto:Params: {} 2022-09-03T06:02:40.513 DEBUG:boto:Token: None 2022-09-03T06:02:40.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.518 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000004a5ce1a3067fa1-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.519 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.520 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.520 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.521 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.522 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T06:02:40.523 DEBUG:boto:Data: 2022-09-03T06:02:40.524 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.525 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.526 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.527 DEBUG:boto:Params: {} 2022-09-03T06:02:40.527 DEBUG:boto:Token: None 2022-09-03T06:02:40.528 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2 2022-09-03T06:02:40.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e6Gaql7jQwEJlQkHF/47L2cn7mw= 2022-09-03T06:02:40.529 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e6Gaql7jQwEJlQkHF/47L2cn7mw=', 'Content-Length': '0'} 2022-09-03T06:02:40.532 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a485d7540be2f03a-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.532 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.533 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.534 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.535 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T06:02:40.535 DEBUG:boto:Data: 2022-09-03T06:02:40.536 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.538 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.539 DEBUG:boto:Params: {} 2022-09-03T06:02:40.540 DEBUG:boto:Token: None 2022-09-03T06:02:40.541 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2 2022-09-03T06:02:40.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e6Gaql7jQwEJlQkHF/47L2cn7mw= 2022-09-03T06:02:40.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e6Gaql7jQwEJlQkHF/47L2cn7mw=', 'Content-Length': '0'} 2022-09-03T06:02:40.546 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005952530e724dcfe8-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.547 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:40.548 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.548 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.549 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.550 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.551 DEBUG:boto:Method: GET 2022-09-03T06:02:40.552 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.553 DEBUG:boto:Data: 2022-09-03T06:02:40.554 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.555 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.556 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.557 DEBUG:boto:Params: {} 2022-09-03T06:02:40.558 DEBUG:boto:Token: None 2022-09-03T06:02:40.559 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc= 2022-09-03T06:02:40.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc=', 'Content-Length': '0'} 2022-09-03T06:02:40.563 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003ca97103bdd078ad-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.564 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.565 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:40.565 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.566 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.566 DEBUG:boto:Method: GET 2022-09-03T06:02:40.567 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.567 DEBUG:boto:Data: 2022-09-03T06:02:40.568 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.569 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.570 DEBUG:boto:Params: {} 2022-09-03T06:02:40.571 DEBUG:boto:Token: None 2022-09-03T06:02:40.571 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:40.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc= 2022-09-03T06:02:40.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qeT1t+ADDjMnGJGFu6wEnsGSzqc=', 'Content-Length': '0'} 2022-09-03T06:02:40.574 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b0b441fc51e20aee-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.622 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.623 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.624 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.624 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.625 DEBUG:boto:Method: GET 2022-09-03T06:02:40.626 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.626 DEBUG:boto:Data: 2022-09-03T06:02:40.627 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.628 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.628 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.629 DEBUG:boto:Params: {} 2022-09-03T06:02:40.630 DEBUG:boto:Token: None 2022-09-03T06:02:40.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.634 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c1d33bb31eb5c31-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.635 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.636 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.637 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.637 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.638 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.638 DEBUG:boto:Method: GET 2022-09-03T06:02:40.639 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:40.639 DEBUG:boto:Data: 2022-09-03T06:02:40.640 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.641 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.641 DEBUG:boto:Params: {} 2022-09-03T06:02:40.642 DEBUG:boto:Token: None 2022-09-03T06:02:40.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.647 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9f5c1b0432abe94-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.648 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.648 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:40.649 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.650 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.650 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.651 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.651 DEBUG:boto:Method: GET 2022-09-03T06:02:40.652 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.652 DEBUG:boto:Data: 2022-09-03T06:02:40.653 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.653 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.654 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.654 DEBUG:boto:Params: {} 2022-09-03T06:02:40.655 DEBUG:boto:Token: None 2022-09-03T06:02:40.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8= 2022-09-03T06:02:40.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8=', 'Content-Length': '0'} 2022-09-03T06:02:40.659 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001dda941ffafb9185-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.706 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.707 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.708 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.709 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.710 DEBUG:boto:Method: GET 2022-09-03T06:02:40.712 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.712 DEBUG:boto:Data: 2022-09-03T06:02:40.713 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.715 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.716 DEBUG:boto:Params: {} 2022-09-03T06:02:40.718 DEBUG:boto:Token: None 2022-09-03T06:02:40.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8= 2022-09-03T06:02:40.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8=', 'Content-Length': '0'} 2022-09-03T06:02:40.724 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004e92d32ad881629f-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.726 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.727 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.728 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.728 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.729 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.730 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.731 DEBUG:boto:Data: 2022-09-03T06:02:40.732 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.733 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.734 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.735 DEBUG:boto:Params: {} 2022-09-03T06:02:40.736 DEBUG:boto:Token: None 2022-09-03T06:02:40.737 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.738 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WsIZAJUMlu94l68NpWm92OMhW40= 2022-09-03T06:02:40.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WsIZAJUMlu94l68NpWm92OMhW40=', 'Content-Length': '0'} 2022-09-03T06:02:40.742 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c8e7afa1a1dbc7bf-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.743 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.743 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.744 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.744 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.745 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.745 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.746 DEBUG:boto:Data: 2022-09-03T06:02:40.746 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.747 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.748 DEBUG:boto:Params: {} 2022-09-03T06:02:40.748 DEBUG:boto:Token: None 2022-09-03T06:02:40.749 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WsIZAJUMlu94l68NpWm92OMhW40= 2022-09-03T06:02:40.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WsIZAJUMlu94l68NpWm92OMhW40=', 'Content-Length': '0'} 2022-09-03T06:02:40.752 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c543571f7b0fa86f-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.753 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:40.753 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.754 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.754 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.755 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.756 DEBUG:boto:Method: GET 2022-09-03T06:02:40.757 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.758 DEBUG:boto:Data: 2022-09-03T06:02:40.759 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.760 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.761 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.761 DEBUG:boto:Params: {} 2022-09-03T06:02:40.762 DEBUG:boto:Token: None 2022-09-03T06:02:40.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8= 2022-09-03T06:02:40.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8=', 'Content-Length': '0'} 2022-09-03T06:02:40.767 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000423549ae681ca0cd-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.768 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.768 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.769 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.769 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.770 DEBUG:boto:Method: GET 2022-09-03T06:02:40.771 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.771 DEBUG:boto:Data: 2022-09-03T06:02:40.772 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.772 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.773 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.773 DEBUG:boto:Params: {} 2022-09-03T06:02:40.774 DEBUG:boto:Token: None 2022-09-03T06:02:40.775 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8= 2022-09-03T06:02:40.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8=', 'Content-Length': '0'} 2022-09-03T06:02:40.778 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000530423831d61328-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.779 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.779 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.780 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.780 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.781 DEBUG:boto:Method: GET 2022-09-03T06:02:40.781 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.782 DEBUG:boto:Data: 2022-09-03T06:02:40.782 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.783 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.783 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.784 DEBUG:boto:Params: {} 2022-09-03T06:02:40.785 DEBUG:boto:Token: None 2022-09-03T06:02:40.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.789 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000012f0b888a8038608-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.789 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.790 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.791 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.791 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.792 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.792 DEBUG:boto:Method: GET 2022-09-03T06:02:40.793 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.794 DEBUG:boto:Data: 2022-09-03T06:02:40.794 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.795 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.796 DEBUG:boto:Params: {} 2022-09-03T06:02:40.796 DEBUG:boto:Token: None 2022-09-03T06:02:40.797 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.798 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.801 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000db7edf895f1430d-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.801 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.802 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.803 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.803 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.804 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T06:02:40.804 DEBUG:boto:Data: 2022-09-03T06:02:40.805 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.805 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.806 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.806 DEBUG:boto:Params: {} 2022-09-03T06:02:40.807 DEBUG:boto:Token: None 2022-09-03T06:02:40.807 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3 2022-09-03T06:02:40.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mpErwI/AWOuRSpEgWYvdRydRm4Q= 2022-09-03T06:02:40.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mpErwI/AWOuRSpEgWYvdRydRm4Q=', 'Content-Length': '0'} 2022-09-03T06:02:40.811 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000064c485f6315d4ab6-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.811 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.812 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.812 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.813 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T06:02:40.813 DEBUG:boto:Data: 2022-09-03T06:02:40.814 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.814 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.815 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.816 DEBUG:boto:Params: {} 2022-09-03T06:02:40.816 DEBUG:boto:Token: None 2022-09-03T06:02:40.817 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3 2022-09-03T06:02:40.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mpErwI/AWOuRSpEgWYvdRydRm4Q= 2022-09-03T06:02:40.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mpErwI/AWOuRSpEgWYvdRydRm4Q=', 'Content-Length': '0'} 2022-09-03T06:02:40.821 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000064d5ded396da534f-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.822 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:40.823 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.824 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.825 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.826 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.827 DEBUG:boto:Method: GET 2022-09-03T06:02:40.828 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.829 DEBUG:boto:Data: 2022-09-03T06:02:40.830 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.831 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.832 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.833 DEBUG:boto:Params: {} 2022-09-03T06:02:40.834 DEBUG:boto:Token: None 2022-09-03T06:02:40.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8= 2022-09-03T06:02:40.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8=', 'Content-Length': '0'} 2022-09-03T06:02:40.840 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000740c6aa5e62b8543-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.842 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.842 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:40.843 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.844 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.845 DEBUG:boto:Method: GET 2022-09-03T06:02:40.846 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.847 DEBUG:boto:Data: 2022-09-03T06:02:40.848 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.849 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.850 DEBUG:boto:Params: {} 2022-09-03T06:02:40.851 DEBUG:boto:Token: None 2022-09-03T06:02:40.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:40.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8= 2022-09-03T06:02:40.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8an+b0T5CvFKDBwWDR3KBNobV8=', 'Content-Length': '0'} 2022-09-03T06:02:40.857 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c147b3e28fca3db7-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.860 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.860 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.861 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.862 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.862 DEBUG:boto:Method: GET 2022-09-03T06:02:40.863 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.863 DEBUG:boto:Data: 2022-09-03T06:02:40.864 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.865 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:40.866 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.867 DEBUG:boto:Params: {} 2022-09-03T06:02:40.867 DEBUG:boto:Token: None 2022-09-03T06:02:40.868 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.872 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb9372c8cc5f28c2-006312ee00-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.873 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.874 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.875 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.875 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.876 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.876 DEBUG:boto:Method: GET 2022-09-03T06:02:40.877 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:40.878 DEBUG:boto:Data: 2022-09-03T06:02:40.878 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.879 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.880 DEBUG:boto:Params: {} 2022-09-03T06:02:40.881 DEBUG:boto:Token: None 2022-09-03T06:02:40.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.882 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc= 2022-09-03T06:02:40.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0PippjzIqbbB+Cbss+GTWCJSSHc=', 'Content-Length': '0'} 2022-09-03T06:02:40.886 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007cf4ebcff379628a-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.887 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.887 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-17 zones={test-zone2, test-zone1} 2022-09-03T06:02:40.889 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-17 zones={test-zone3, test-zone1} 2022-09-03T06:02:40.889 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.890 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.891 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.892 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T06:02:40.893 DEBUG:boto:Data: 2022-09-03T06:02:40.894 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.895 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:40.896 DEBUG:boto:Port: 8001 2022-09-03T06:02:40.896 DEBUG:boto:Params: {} 2022-09-03T06:02:40.897 DEBUG:boto:Token: None 2022-09-03T06:02:40.898 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.899 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l/0+iJkOB8Kc67YD2YVVBPpd9Tw= 2022-09-03T06:02:40.900 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l/0+iJkOB8Kc67YD2YVVBPpd9Tw=', 'Content-Length': '0'} 2022-09-03T06:02:40.903 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000037a15f4da86b32e8-006312ee00-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.903 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.904 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.905 DEBUG:boto:Method: HEAD 2022-09-03T06:02:40.905 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T06:02:40.906 DEBUG:boto:Data: 2022-09-03T06:02:40.907 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.908 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.909 DEBUG:boto:Params: {} 2022-09-03T06:02:40.909 DEBUG:boto:Token: None 2022-09-03T06:02:40.910 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/ 2022-09-03T06:02:40.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l/0+iJkOB8Kc67YD2YVVBPpd9Tw= 2022-09-03T06:02:40.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l/0+iJkOB8Kc67YD2YVVBPpd9Tw=', 'Content-Length': '0'} 2022-09-03T06:02:40.914 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009a22dc3764c67830-006312ee00-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.915 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:02:40.915 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.916 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.916 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.917 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.918 DEBUG:boto:Method: GET 2022-09-03T06:02:40.918 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.919 DEBUG:boto:Data: 2022-09-03T06:02:40.919 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:40.920 DEBUG:boto:Port: 8001 2022-09-03T06:02:40.921 DEBUG:boto:Params: {} 2022-09-03T06:02:40.921 DEBUG:boto:Token: None 2022-09-03T06:02:40.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/?versions 2022-09-03T06:02:40.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+F7CB3EPX3NGXu4fZXFXHdAq77I= 2022-09-03T06:02:40.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+F7CB3EPX3NGXu4fZXFXHdAq77I=', 'Content-Length': '0'} 2022-09-03T06:02:40.927 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc62739d55a2d90b-006312ee00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.928 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.929 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:40.930 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:40.930 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:40.931 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:40.931 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:02:40.932 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.932 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.933 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.934 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.934 DEBUG:boto:Method: GET 2022-09-03T06:02:40.935 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.935 DEBUG:boto:Data: 2022-09-03T06:02:40.936 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.937 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.937 DEBUG:boto:Params: {} 2022-09-03T06:02:40.938 DEBUG:boto:Token: None 2022-09-03T06:02:40.939 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/?versions 2022-09-03T06:02:40.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+F7CB3EPX3NGXu4fZXFXHdAq77I= 2022-09-03T06:02:40.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+F7CB3EPX3NGXu4fZXFXHdAq77I=', 'Content-Length': '0'} 2022-09-03T06:02:40.943 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9466862d5c844f9-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.943 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.944 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:40.945 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:40.945 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:40.946 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:40.946 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.947 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.947 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.948 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.949 DEBUG:boto:Method: GET 2022-09-03T06:02:40.949 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.950 DEBUG:boto:Data: 2022-09-03T06:02:40.950 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:40.951 DEBUG:boto:Port: 8001 2022-09-03T06:02:40.952 DEBUG:boto:Params: {} 2022-09-03T06:02:40.952 DEBUG:boto:Token: None 2022-09-03T06:02:40.953 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/?versions 2022-09-03T06:02:40.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+F7CB3EPX3NGXu4fZXFXHdAq77I= 2022-09-03T06:02:40.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+F7CB3EPX3NGXu4fZXFXHdAq77I=', 'Content-Length': '0'} 2022-09-03T06:02:40.957 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007398ca6f19c30fe6-006312ee00-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.958 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.959 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:40.959 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:40.960 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.961 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.962 DEBUG:boto:Method: GET 2022-09-03T06:02:40.962 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:40.963 DEBUG:boto:Data: 2022-09-03T06:02:40.964 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.964 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.965 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.965 DEBUG:boto:Params: {} 2022-09-03T06:02:40.966 DEBUG:boto:Token: None 2022-09-03T06:02:40.967 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/?versions 2022-09-03T06:02:40.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+F7CB3EPX3NGXu4fZXFXHdAq77I= 2022-09-03T06:02:40.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+F7CB3EPX3NGXu4fZXFXHdAq77I=', 'Content-Length': '0'} 2022-09-03T06:02:40.970 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b82d2cb3daf1d223-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.971 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:40.972 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:40.973 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:40.974 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:40.975 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.975 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.976 DEBUG:boto:Method: GET 2022-09-03T06:02:40.977 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.978 DEBUG:boto:Data: 2022-09-03T06:02:40.979 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:40.980 DEBUG:boto:Port: 8001 2022-09-03T06:02:40.981 DEBUG:boto:Params: {} 2022-09-03T06:02:40.982 DEBUG:boto:Token: None 2022-09-03T06:02:40.983 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1S7rBj7ICMO8Ru1U6h3SJ6Bc1bA= 2022-09-03T06:02:40.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1S7rBj7ICMO8Ru1U6h3SJ6Bc1bA=', 'Content-Length': '0'} 2022-09-03T06:02:40.986 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000540853178cee6815-006312ee00-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:40.987 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:40.988 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:40.989 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.990 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.991 DEBUG:boto:Method: GET 2022-09-03T06:02:40.991 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.992 DEBUG:boto:Data: 2022-09-03T06:02:40.993 DEBUG:boto:Headers: {} 2022-09-03T06:02:40.994 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:40.994 DEBUG:boto:Port: 8000 2022-09-03T06:02:40.995 DEBUG:boto:Params: {} 2022-09-03T06:02:40.996 DEBUG:boto:Token: None 2022-09-03T06:02:40.997 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:40 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:40.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1S7rBj7ICMO8Ru1U6h3SJ6Bc1bA= 2022-09-03T06:02:40.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1S7rBj7ICMO8Ru1U6h3SJ6Bc1bA=', 'Content-Length': '0'} 2022-09-03T06:02:41.001 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000079ac9d0f0045d862-006312ee00-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.002 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.003 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.003 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.004 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.005 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.005 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.006 DEBUG:boto:Data: 2022-09-03T06:02:41.007 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.009 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.010 DEBUG:boto:Params: {} 2022-09-03T06:02:41.010 DEBUG:boto:Token: None 2022-09-03T06:02:41.011 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9dD5rcXjBuvufDZzpFg76IAssG8= 2022-09-03T06:02:41.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9dD5rcXjBuvufDZzpFg76IAssG8=', 'Content-Length': '0'} 2022-09-03T06:02:41.015 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000624b77bf08190747-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.016 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.018 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.018 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.019 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.020 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.020 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.022 DEBUG:boto:Data: 2022-09-03T06:02:41.023 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.023 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.024 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.025 DEBUG:boto:Params: {} 2022-09-03T06:02:41.026 DEBUG:boto:Token: None 2022-09-03T06:02:41.027 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9dD5rcXjBuvufDZzpFg76IAssG8= 2022-09-03T06:02:41.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9dD5rcXjBuvufDZzpFg76IAssG8=', 'Content-Length': '0'} 2022-09-03T06:02:41.031 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005d0496cbbc7f60fa-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.032 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:41.033 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.033 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.034 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.035 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.035 DEBUG:boto:Method: GET 2022-09-03T06:02:41.036 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.036 DEBUG:boto:Data: 2022-09-03T06:02:41.037 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.038 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.039 DEBUG:boto:Params: {} 2022-09-03T06:02:41.039 DEBUG:boto:Token: None 2022-09-03T06:02:41.040 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw= 2022-09-03T06:02:41.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw=', 'Content-Length': '0'} 2022-09-03T06:02:41.044 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000002d91d07ec9d7f3c0-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.046 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.047 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.047 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.048 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.049 DEBUG:boto:Method: GET 2022-09-03T06:02:41.050 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.050 DEBUG:boto:Data: 2022-09-03T06:02:41.051 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.052 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.052 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.053 DEBUG:boto:Params: {} 2022-09-03T06:02:41.054 DEBUG:boto:Token: None 2022-09-03T06:02:41.055 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw= 2022-09-03T06:02:41.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw=', 'Content-Length': '0'} 2022-09-03T06:02:41.058 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000bbce45ee576c1045-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.060 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.060 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.061 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:41.062 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:41.063 DEBUG:boto:Method: GET 2022-09-03T06:02:41.063 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:41.064 DEBUG:boto:Data: 2022-09-03T06:02:41.065 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.065 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.066 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.067 DEBUG:boto:Params: {} 2022-09-03T06:02:41.068 DEBUG:boto:Token: None 2022-09-03T06:02:41.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.073 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000064c97d053d396211-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.114 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.115 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.116 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.117 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:41.118 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:41.118 DEBUG:boto:Method: GET 2022-09-03T06:02:41.119 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:41.120 DEBUG:boto:Data: 2022-09-03T06:02:41.120 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.121 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.122 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.123 DEBUG:boto:Params: {} 2022-09-03T06:02:41.123 DEBUG:boto:Token: None 2022-09-03T06:02:41.124 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.128 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000052037a827cb38bf1-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.129 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.130 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.131 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.131 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.132 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T06:02:41.133 DEBUG:boto:Data: 2022-09-03T06:02:41.133 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.135 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.136 DEBUG:boto:Params: {} 2022-09-03T06:02:41.136 DEBUG:boto:Token: None 2022-09-03T06:02:41.137 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy 2022-09-03T06:02:41.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmJuOt3jHxFncUgse6dkBHzjpFw= 2022-09-03T06:02:41.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmJuOt3jHxFncUgse6dkBHzjpFw=', 'Content-Length': '0'} 2022-09-03T06:02:41.143 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000015da84f2a332dda9-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.144 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.145 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.145 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.146 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T06:02:41.146 DEBUG:boto:Data: 2022-09-03T06:02:41.147 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.147 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.148 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.149 DEBUG:boto:Params: {} 2022-09-03T06:02:41.149 DEBUG:boto:Token: None 2022-09-03T06:02:41.150 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy 2022-09-03T06:02:41.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmJuOt3jHxFncUgse6dkBHzjpFw= 2022-09-03T06:02:41.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmJuOt3jHxFncUgse6dkBHzjpFw=', 'Content-Length': '0'} 2022-09-03T06:02:41.154 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000000558864a584ef685-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.155 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:41.155 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.156 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.157 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.157 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.158 DEBUG:boto:Method: GET 2022-09-03T06:02:41.158 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.159 DEBUG:boto:Data: 2022-09-03T06:02:41.160 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.160 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.161 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.161 DEBUG:boto:Params: {} 2022-09-03T06:02:41.162 DEBUG:boto:Token: None 2022-09-03T06:02:41.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw= 2022-09-03T06:02:41.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw=', 'Content-Length': '0'} 2022-09-03T06:02:41.166 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b752c2735770191e-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.166 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.167 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.168 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.168 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.169 DEBUG:boto:Method: GET 2022-09-03T06:02:41.169 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.170 DEBUG:boto:Data: 2022-09-03T06:02:41.170 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.172 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.172 DEBUG:boto:Params: {} 2022-09-03T06:02:41.173 DEBUG:boto:Token: None 2022-09-03T06:02:41.173 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw= 2022-09-03T06:02:41.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw=', 'Content-Length': '0'} 2022-09-03T06:02:41.177 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000033c22affd2222663-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.177 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.178 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.179 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:41.179 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:41.180 DEBUG:boto:Method: GET 2022-09-03T06:02:41.180 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:41.181 DEBUG:boto:Data: 2022-09-03T06:02:41.181 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.182 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.183 DEBUG:boto:Params: {} 2022-09-03T06:02:41.184 DEBUG:boto:Token: None 2022-09-03T06:02:41.184 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.188 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001d22f40acefc8bb7-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.189 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.190 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.190 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.191 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:41.191 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:41.192 DEBUG:boto:Method: GET 2022-09-03T06:02:41.193 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:41.193 DEBUG:boto:Data: 2022-09-03T06:02:41.194 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.195 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.195 DEBUG:boto:Params: {} 2022-09-03T06:02:41.196 DEBUG:boto:Token: None 2022-09-03T06:02:41.197 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.201 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005880567c415c16dd-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.201 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.202 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:41.203 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.203 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.204 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.205 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.205 DEBUG:boto:Method: GET 2022-09-03T06:02:41.206 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.206 DEBUG:boto:Data: 2022-09-03T06:02:41.207 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.207 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.208 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.209 DEBUG:boto:Params: {} 2022-09-03T06:02:41.209 DEBUG:boto:Token: None 2022-09-03T06:02:41.210 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo= 2022-09-03T06:02:41.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo=', 'Content-Length': '0'} 2022-09-03T06:02:41.213 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002bd40ca42dc443a8-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.214 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.214 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.215 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.216 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.216 DEBUG:boto:Method: GET 2022-09-03T06:02:41.217 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.217 DEBUG:boto:Data: 2022-09-03T06:02:41.218 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.219 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.219 DEBUG:boto:Params: {} 2022-09-03T06:02:41.220 DEBUG:boto:Token: None 2022-09-03T06:02:41.221 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo= 2022-09-03T06:02:41.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo=', 'Content-Length': '0'} 2022-09-03T06:02:41.224 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000458cc2e8a26e9ea4-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.225 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.226 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.226 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.227 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.227 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.228 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.228 DEBUG:boto:Data: 2022-09-03T06:02:41.229 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.230 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.230 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.231 DEBUG:boto:Params: {} 2022-09-03T06:02:41.232 DEBUG:boto:Token: None 2022-09-03T06:02:41.232 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xIWDqmcz7PtmOgmy7LTRmpaRHLk= 2022-09-03T06:02:41.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xIWDqmcz7PtmOgmy7LTRmpaRHLk=', 'Content-Length': '0'} 2022-09-03T06:02:41.236 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bb50f6f713cad75f-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.236 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.237 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.237 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.238 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.239 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.239 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.240 DEBUG:boto:Data: 2022-09-03T06:02:41.240 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.241 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.241 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.242 DEBUG:boto:Params: {} 2022-09-03T06:02:41.242 DEBUG:boto:Token: None 2022-09-03T06:02:41.243 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xIWDqmcz7PtmOgmy7LTRmpaRHLk= 2022-09-03T06:02:41.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xIWDqmcz7PtmOgmy7LTRmpaRHLk=', 'Content-Length': '0'} 2022-09-03T06:02:41.247 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d661aed874b8c078-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.248 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:41.248 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.249 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.249 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.250 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.250 DEBUG:boto:Method: GET 2022-09-03T06:02:41.251 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.252 DEBUG:boto:Data: 2022-09-03T06:02:41.252 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.253 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.254 DEBUG:boto:Params: {} 2022-09-03T06:02:41.255 DEBUG:boto:Token: None 2022-09-03T06:02:41.255 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo= 2022-09-03T06:02:41.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo=', 'Content-Length': '0'} 2022-09-03T06:02:41.259 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f7aef28b1eb30d5f-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.260 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.260 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.261 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.262 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.263 DEBUG:boto:Method: GET 2022-09-03T06:02:41.263 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.264 DEBUG:boto:Data: 2022-09-03T06:02:41.264 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.265 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.265 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.266 DEBUG:boto:Params: {} 2022-09-03T06:02:41.266 DEBUG:boto:Token: None 2022-09-03T06:02:41.267 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo= 2022-09-03T06:02:41.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo=', 'Content-Length': '0'} 2022-09-03T06:02:41.270 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000133a48c26214e518-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.271 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.271 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.272 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.273 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.273 DEBUG:boto:Method: GET 2022-09-03T06:02:41.274 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.274 DEBUG:boto:Data: 2022-09-03T06:02:41.275 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.275 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.276 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.276 DEBUG:boto:Params: {} 2022-09-03T06:02:41.277 DEBUG:boto:Token: None 2022-09-03T06:02:41.278 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.281 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb7d84beed38e9b7-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.282 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.284 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.284 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.285 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.285 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.286 DEBUG:boto:Method: GET 2022-09-03T06:02:41.286 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.287 DEBUG:boto:Data: 2022-09-03T06:02:41.287 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.288 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.289 DEBUG:boto:Params: {} 2022-09-03T06:02:41.289 DEBUG:boto:Token: None 2022-09-03T06:02:41.290 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.294 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069fdb15e751a55c7-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.334 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.335 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.336 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.336 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.337 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T06:02:41.337 DEBUG:boto:Data: 2022-09-03T06:02:41.338 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.339 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.340 DEBUG:boto:Params: {} 2022-09-03T06:02:41.341 DEBUG:boto:Token: None 2022-09-03T06:02:41.341 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1 2022-09-03T06:02:41.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FeNUwsSGU/yuFhvW3ES0PKCTwVU= 2022-09-03T06:02:41.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FeNUwsSGU/yuFhvW3ES0PKCTwVU=', 'Content-Length': '0'} 2022-09-03T06:02:41.345 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009d1423c0362043cf-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.346 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.347 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.348 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.348 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T06:02:41.349 DEBUG:boto:Data: 2022-09-03T06:02:41.349 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.350 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.351 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.351 DEBUG:boto:Params: {} 2022-09-03T06:02:41.352 DEBUG:boto:Token: None 2022-09-03T06:02:41.353 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1 2022-09-03T06:02:41.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FeNUwsSGU/yuFhvW3ES0PKCTwVU= 2022-09-03T06:02:41.354 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FeNUwsSGU/yuFhvW3ES0PKCTwVU=', 'Content-Length': '0'} 2022-09-03T06:02:41.357 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000022bdf0ebbd30cd81-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.357 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:41.358 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.358 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.359 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.360 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.360 DEBUG:boto:Method: GET 2022-09-03T06:02:41.361 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.361 DEBUG:boto:Data: 2022-09-03T06:02:41.362 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.363 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.364 DEBUG:boto:Params: {} 2022-09-03T06:02:41.364 DEBUG:boto:Token: None 2022-09-03T06:02:41.365 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo= 2022-09-03T06:02:41.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo=', 'Content-Length': '0'} 2022-09-03T06:02:41.368 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000004af1186f0936c7a-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.369 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.370 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:41.370 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.371 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.371 DEBUG:boto:Method: GET 2022-09-03T06:02:41.372 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.373 DEBUG:boto:Data: 2022-09-03T06:02:41.373 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.374 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.374 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.375 DEBUG:boto:Params: {} 2022-09-03T06:02:41.375 DEBUG:boto:Token: None 2022-09-03T06:02:41.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:41.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo= 2022-09-03T06:02:41.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2luB5yDEBuEEfi0XZLMieqk7lo=', 'Content-Length': '0'} 2022-09-03T06:02:41.379 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000050fc89f9239ba6a4-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.381 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.383 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.383 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.384 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.385 DEBUG:boto:Method: GET 2022-09-03T06:02:41.385 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.386 DEBUG:boto:Data: 2022-09-03T06:02:41.387 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.388 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.389 DEBUG:boto:Params: {} 2022-09-03T06:02:41.392 DEBUG:boto:Token: None 2022-09-03T06:02:41.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.397 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a84497be3d7cd0b5-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.398 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.399 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.399 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.400 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.401 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.401 DEBUG:boto:Method: GET 2022-09-03T06:02:41.402 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:41.402 DEBUG:boto:Data: 2022-09-03T06:02:41.403 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.403 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.404 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.405 DEBUG:boto:Params: {} 2022-09-03T06:02:41.405 DEBUG:boto:Token: None 2022-09-03T06:02:41.406 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.410 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000010cba3898f8cf51f-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.410 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.411 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:41.412 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.412 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.413 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.413 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.414 DEBUG:boto:Method: GET 2022-09-03T06:02:41.415 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.415 DEBUG:boto:Data: 2022-09-03T06:02:41.416 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.417 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.417 DEBUG:boto:Params: {} 2022-09-03T06:02:41.418 DEBUG:boto:Token: None 2022-09-03T06:02:41.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ= 2022-09-03T06:02:41.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ=', 'Content-Length': '0'} 2022-09-03T06:02:41.422 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005a2931dda46e6740-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.423 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.423 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.424 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.424 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.425 DEBUG:boto:Method: GET 2022-09-03T06:02:41.426 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.426 DEBUG:boto:Data: 2022-09-03T06:02:41.427 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.428 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.428 DEBUG:boto:Params: {} 2022-09-03T06:02:41.429 DEBUG:boto:Token: None 2022-09-03T06:02:41.429 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ= 2022-09-03T06:02:41.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ=', 'Content-Length': '0'} 2022-09-03T06:02:41.433 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f378fb1ec97d3fd6-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.434 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.435 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.435 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.436 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.436 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.437 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.437 DEBUG:boto:Data: 2022-09-03T06:02:41.438 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.439 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.439 DEBUG:boto:Params: {} 2022-09-03T06:02:41.440 DEBUG:boto:Token: None 2022-09-03T06:02:41.441 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P7GIU5I3Dlyzdy/EyN3Sa9kWtCA= 2022-09-03T06:02:41.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P7GIU5I3Dlyzdy/EyN3Sa9kWtCA=', 'Content-Length': '0'} 2022-09-03T06:02:41.445 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c614475ba263ce9d-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.447 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.448 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.448 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.449 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.450 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.451 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.452 DEBUG:boto:Data: 2022-09-03T06:02:41.453 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.454 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.455 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.455 DEBUG:boto:Params: {} 2022-09-03T06:02:41.456 DEBUG:boto:Token: None 2022-09-03T06:02:41.457 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P7GIU5I3Dlyzdy/EyN3Sa9kWtCA= 2022-09-03T06:02:41.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P7GIU5I3Dlyzdy/EyN3Sa9kWtCA=', 'Content-Length': '0'} 2022-09-03T06:02:41.460 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008528091e7a3f9271-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.461 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:41.462 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.462 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.463 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.463 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.464 DEBUG:boto:Method: GET 2022-09-03T06:02:41.464 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.465 DEBUG:boto:Data: 2022-09-03T06:02:41.466 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.468 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.468 DEBUG:boto:Params: {} 2022-09-03T06:02:41.470 DEBUG:boto:Token: None 2022-09-03T06:02:41.471 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ= 2022-09-03T06:02:41.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ=', 'Content-Length': '0'} 2022-09-03T06:02:41.476 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000701ea3a01697e77f-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.478 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.479 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.479 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.480 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.480 DEBUG:boto:Method: GET 2022-09-03T06:02:41.481 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.482 DEBUG:boto:Data: 2022-09-03T06:02:41.482 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.483 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.484 DEBUG:boto:Params: {} 2022-09-03T06:02:41.484 DEBUG:boto:Token: None 2022-09-03T06:02:41.485 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.486 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ= 2022-09-03T06:02:41.487 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ=', 'Content-Length': '0'} 2022-09-03T06:02:41.489 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009768b6cfd9d3aef9-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.489 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.490 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.491 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.491 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.492 DEBUG:boto:Method: GET 2022-09-03T06:02:41.492 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.493 DEBUG:boto:Data: 2022-09-03T06:02:41.493 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.494 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.494 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.495 DEBUG:boto:Params: {} 2022-09-03T06:02:41.495 DEBUG:boto:Token: None 2022-09-03T06:02:41.496 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.500 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000068910f18f4a9dbba-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.542 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.544 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.546 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.547 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.548 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.549 DEBUG:boto:Method: GET 2022-09-03T06:02:41.550 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.551 DEBUG:boto:Data: 2022-09-03T06:02:41.552 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.553 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.556 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.556 DEBUG:boto:Params: {} 2022-09-03T06:02:41.557 DEBUG:boto:Token: None 2022-09-03T06:02:41.558 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.562 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b88a48881b218e8-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.562 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.563 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.564 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.564 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.565 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T06:02:41.565 DEBUG:boto:Data: 2022-09-03T06:02:41.567 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.568 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.568 DEBUG:boto:Params: {} 2022-09-03T06:02:41.569 DEBUG:boto:Token: None 2022-09-03T06:02:41.569 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2 2022-09-03T06:02:41.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xLqOm/OIvryX12So6Ru4E8xOeSw= 2022-09-03T06:02:41.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xLqOm/OIvryX12So6Ru4E8xOeSw=', 'Content-Length': '0'} 2022-09-03T06:02:41.573 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000130805381afd7f5b-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.574 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.574 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.575 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.576 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T06:02:41.576 DEBUG:boto:Data: 2022-09-03T06:02:41.578 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.579 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.579 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.580 DEBUG:boto:Params: {} 2022-09-03T06:02:41.581 DEBUG:boto:Token: None 2022-09-03T06:02:41.581 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2 2022-09-03T06:02:41.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xLqOm/OIvryX12So6Ru4E8xOeSw= 2022-09-03T06:02:41.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xLqOm/OIvryX12So6Ru4E8xOeSw=', 'Content-Length': '0'} 2022-09-03T06:02:41.585 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b7a378a8bc022568-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.585 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:41.586 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.587 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.587 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.590 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.591 DEBUG:boto:Method: GET 2022-09-03T06:02:41.592 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.593 DEBUG:boto:Data: 2022-09-03T06:02:41.593 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.594 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.595 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.595 DEBUG:boto:Params: {} 2022-09-03T06:02:41.596 DEBUG:boto:Token: None 2022-09-03T06:02:41.596 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ= 2022-09-03T06:02:41.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ=', 'Content-Length': '0'} 2022-09-03T06:02:41.600 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000023f10b0ebc5645da-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.601 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.602 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:41.602 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.603 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.603 DEBUG:boto:Method: GET 2022-09-03T06:02:41.604 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.605 DEBUG:boto:Data: 2022-09-03T06:02:41.605 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.606 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.606 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.607 DEBUG:boto:Params: {} 2022-09-03T06:02:41.607 DEBUG:boto:Token: None 2022-09-03T06:02:41.608 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:41.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ= 2022-09-03T06:02:41.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z3FtVtiT9RPBxxahmoWpQQp3aDQ=', 'Content-Length': '0'} 2022-09-03T06:02:41.611 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a51aeea881ebecc0-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.612 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.613 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.613 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.614 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.615 DEBUG:boto:Method: GET 2022-09-03T06:02:41.615 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.617 DEBUG:boto:Data: 2022-09-03T06:02:41.617 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.618 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.618 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.619 DEBUG:boto:Params: {} 2022-09-03T06:02:41.620 DEBUG:boto:Token: None 2022-09-03T06:02:41.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.624 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b3003b6adc97e80-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.625 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.626 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.627 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.627 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.628 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.628 DEBUG:boto:Method: GET 2022-09-03T06:02:41.629 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:41.629 DEBUG:boto:Data: 2022-09-03T06:02:41.630 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.631 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.632 DEBUG:boto:Params: {} 2022-09-03T06:02:41.632 DEBUG:boto:Token: None 2022-09-03T06:02:41.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.637 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b9a88f39ec06e064-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.637 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.638 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:41.639 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.639 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.640 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.640 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.641 DEBUG:boto:Method: GET 2022-09-03T06:02:41.642 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.643 DEBUG:boto:Data: 2022-09-03T06:02:41.643 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.644 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.645 DEBUG:boto:Params: {} 2022-09-03T06:02:41.646 DEBUG:boto:Token: None 2022-09-03T06:02:41.647 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU= 2022-09-03T06:02:41.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU=', 'Content-Length': '0'} 2022-09-03T06:02:41.652 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a1301ea9b79c4c05-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.653 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.654 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.655 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.655 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.656 DEBUG:boto:Method: GET 2022-09-03T06:02:41.657 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.658 DEBUG:boto:Data: 2022-09-03T06:02:41.659 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.661 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.662 DEBUG:boto:Params: {} 2022-09-03T06:02:41.663 DEBUG:boto:Token: None 2022-09-03T06:02:41.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU= 2022-09-03T06:02:41.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU=', 'Content-Length': '0'} 2022-09-03T06:02:41.668 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c5118cbcf21ac043-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.669 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.670 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.671 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.672 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.672 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.673 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.673 DEBUG:boto:Data: 2022-09-03T06:02:41.674 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.675 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.675 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.676 DEBUG:boto:Params: {} 2022-09-03T06:02:41.676 DEBUG:boto:Token: None 2022-09-03T06:02:41.677 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.677 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ef1epAvkkpqnKxtuXlIapihU9ao= 2022-09-03T06:02:41.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ef1epAvkkpqnKxtuXlIapihU9ao=', 'Content-Length': '0'} 2022-09-03T06:02:41.680 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003f8d210b16b5344f-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.681 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.682 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.683 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.684 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.685 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.686 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.687 DEBUG:boto:Data: 2022-09-03T06:02:41.688 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.690 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.691 DEBUG:boto:Params: {} 2022-09-03T06:02:41.692 DEBUG:boto:Token: None 2022-09-03T06:02:41.693 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ef1epAvkkpqnKxtuXlIapihU9ao= 2022-09-03T06:02:41.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ef1epAvkkpqnKxtuXlIapihU9ao=', 'Content-Length': '0'} 2022-09-03T06:02:41.696 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003108e42839532127-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.697 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:41.698 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.698 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.699 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.700 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.700 DEBUG:boto:Method: GET 2022-09-03T06:02:41.701 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.701 DEBUG:boto:Data: 2022-09-03T06:02:41.702 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.704 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.705 DEBUG:boto:Params: {} 2022-09-03T06:02:41.707 DEBUG:boto:Token: None 2022-09-03T06:02:41.709 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.710 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU= 2022-09-03T06:02:41.711 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU=', 'Content-Length': '0'} 2022-09-03T06:02:41.714 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b74f9980b8922b82-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.715 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.716 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.717 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.718 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.719 DEBUG:boto:Method: GET 2022-09-03T06:02:41.720 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.721 DEBUG:boto:Data: 2022-09-03T06:02:41.722 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.723 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.724 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.725 DEBUG:boto:Params: {} 2022-09-03T06:02:41.726 DEBUG:boto:Token: None 2022-09-03T06:02:41.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU= 2022-09-03T06:02:41.728 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU=', 'Content-Length': '0'} 2022-09-03T06:02:41.731 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bad40fe205700a27-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.732 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.732 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.733 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.734 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.734 DEBUG:boto:Method: GET 2022-09-03T06:02:41.735 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.736 DEBUG:boto:Data: 2022-09-03T06:02:41.737 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.739 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.740 DEBUG:boto:Params: {} 2022-09-03T06:02:41.741 DEBUG:boto:Token: None 2022-09-03T06:02:41.742 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.747 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ec70c2c58c472240-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.748 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.749 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.750 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.750 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.751 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.751 DEBUG:boto:Method: GET 2022-09-03T06:02:41.752 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.752 DEBUG:boto:Data: 2022-09-03T06:02:41.753 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.753 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.754 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.755 DEBUG:boto:Params: {} 2022-09-03T06:02:41.755 DEBUG:boto:Token: None 2022-09-03T06:02:41.756 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.760 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000034dcc362bfccdd23-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.761 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.763 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.764 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.765 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.766 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T06:02:41.767 DEBUG:boto:Data: 2022-09-03T06:02:41.767 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.769 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.770 DEBUG:boto:Params: {} 2022-09-03T06:02:41.771 DEBUG:boto:Token: None 2022-09-03T06:02:41.772 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3 2022-09-03T06:02:41.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+XrVCZyK40f5UlmntZfSAft19k= 2022-09-03T06:02:41.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+XrVCZyK40f5UlmntZfSAft19k=', 'Content-Length': '0'} 2022-09-03T06:02:41.778 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000026605caa6746b372-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.779 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.780 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.780 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.781 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T06:02:41.781 DEBUG:boto:Data: 2022-09-03T06:02:41.782 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.782 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.783 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.783 DEBUG:boto:Params: {} 2022-09-03T06:02:41.784 DEBUG:boto:Token: None 2022-09-03T06:02:41.785 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3 2022-09-03T06:02:41.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+XrVCZyK40f5UlmntZfSAft19k= 2022-09-03T06:02:41.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+XrVCZyK40f5UlmntZfSAft19k=', 'Content-Length': '0'} 2022-09-03T06:02:41.789 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000005f70b5d243a27a6-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.790 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:41.790 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.791 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.792 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.792 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.793 DEBUG:boto:Method: GET 2022-09-03T06:02:41.794 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.794 DEBUG:boto:Data: 2022-09-03T06:02:41.795 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.796 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.796 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.797 DEBUG:boto:Params: {} 2022-09-03T06:02:41.798 DEBUG:boto:Token: None 2022-09-03T06:02:41.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU= 2022-09-03T06:02:41.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU=', 'Content-Length': '0'} 2022-09-03T06:02:41.801 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004e5c69f670f11bba-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.802 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.803 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:41.803 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.804 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.805 DEBUG:boto:Method: GET 2022-09-03T06:02:41.806 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.806 DEBUG:boto:Data: 2022-09-03T06:02:41.807 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.809 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.809 DEBUG:boto:Params: {} 2022-09-03T06:02:41.810 DEBUG:boto:Token: None 2022-09-03T06:02:41.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:41.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU= 2022-09-03T06:02:41.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n8X4iTtALYPM7z+QrOna++f6cQU=', 'Content-Length': '0'} 2022-09-03T06:02:41.815 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007ebb75e38f858a25-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.817 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.818 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.819 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.819 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.820 DEBUG:boto:Method: GET 2022-09-03T06:02:41.821 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.822 DEBUG:boto:Data: 2022-09-03T06:02:41.822 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.823 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.824 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.824 DEBUG:boto:Params: {} 2022-09-03T06:02:41.825 DEBUG:boto:Token: None 2022-09-03T06:02:41.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.829 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f5256fea051ae53b-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.830 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.831 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.832 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.832 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.833 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.833 DEBUG:boto:Method: GET 2022-09-03T06:02:41.834 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:41.834 DEBUG:boto:Data: 2022-09-03T06:02:41.835 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:02:41.836 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.836 DEBUG:boto:Params: {} 2022-09-03T06:02:41.837 DEBUG:boto:Token: None 2022-09-03T06:02:41.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA= 2022-09-03T06:02:41.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GKmQXrif6I7F7Ezo45nM/9PbDlA=', 'Content-Length': '0'} 2022-09-03T06:02:41.841 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa0fb267b0d7440b-006312ee01-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.842 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.843 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-17 zones={test-zone3, test-zone1} 2022-09-03T06:02:41.843 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-17 zones={test-zone3, test-zone2} 2022-09-03T06:02:41.844 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.844 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.845 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.845 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T06:02:41.846 DEBUG:boto:Data: 2022-09-03T06:02:41.847 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.848 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.848 DEBUG:boto:Params: {} 2022-09-03T06:02:41.849 DEBUG:boto:Token: None 2022-09-03T06:02:41.849 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VjNp53m5oJC7IYRvayw48Phoq6U= 2022-09-03T06:02:41.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VjNp53m5oJC7IYRvayw48Phoq6U=', 'Content-Length': '0'} 2022-09-03T06:02:41.856 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000066f1053917595f4f-006312ee01-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.857 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.858 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.858 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.859 DEBUG:boto:Path: /avysmw-17/ 2022-09-03T06:02:41.859 DEBUG:boto:Data: 2022-09-03T06:02:41.860 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.860 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:41.861 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.861 DEBUG:boto:Params: {} 2022-09-03T06:02:41.862 DEBUG:boto:Token: None 2022-09-03T06:02:41.863 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/ 2022-09-03T06:02:41.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VjNp53m5oJC7IYRvayw48Phoq6U= 2022-09-03T06:02:41.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VjNp53m5oJC7IYRvayw48Phoq6U=', 'Content-Length': '0'} 2022-09-03T06:02:41.867 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000606b9fb5f1d7b61b-006312ee01-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.868 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:02:41.868 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.869 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.869 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.870 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.871 DEBUG:boto:Method: GET 2022-09-03T06:02:41.874 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.874 DEBUG:boto:Data: 2022-09-03T06:02:41.875 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.876 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.877 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.877 DEBUG:boto:Params: {} 2022-09-03T06:02:41.878 DEBUG:boto:Token: None 2022-09-03T06:02:41.879 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/?versions 2022-09-03T06:02:41.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mwz70mPlH+clHm13DJdAgGJSrqY= 2022-09-03T06:02:41.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mwz70mPlH+clHm13DJdAgGJSrqY=', 'Content-Length': '0'} 2022-09-03T06:02:41.883 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000430e7c8ccbed317d-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.883 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.884 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:41.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:41.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:41.886 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:41.887 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:02:41.888 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.888 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.889 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.889 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.890 DEBUG:boto:Method: GET 2022-09-03T06:02:41.890 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.891 DEBUG:boto:Data: 2022-09-03T06:02:41.892 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.893 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:41.893 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.894 DEBUG:boto:Params: {} 2022-09-03T06:02:41.895 DEBUG:boto:Token: None 2022-09-03T06:02:41.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/?versions 2022-09-03T06:02:41.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mwz70mPlH+clHm13DJdAgGJSrqY= 2022-09-03T06:02:41.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mwz70mPlH+clHm13DJdAgGJSrqY=', 'Content-Length': '0'} 2022-09-03T06:02:41.900 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000453bb1130edf361a-006312ee01-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.901 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.902 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:02:41.902 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:02:41.903 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:02:41.903 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:02:41.904 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.904 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.905 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.905 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.906 DEBUG:boto:Method: GET 2022-09-03T06:02:41.907 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.907 DEBUG:boto:Data: 2022-09-03T06:02:41.908 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.909 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.909 DEBUG:boto:Params: {} 2022-09-03T06:02:41.910 DEBUG:boto:Token: None 2022-09-03T06:02:41.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/?versions 2022-09-03T06:02:41.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mwz70mPlH+clHm13DJdAgGJSrqY= 2022-09-03T06:02:41.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mwz70mPlH+clHm13DJdAgGJSrqY=', 'Content-Length': '0'} 2022-09-03T06:02:41.915 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000389be25221e060b6-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.916 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.917 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:41.917 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:41.918 DEBUG:boto:path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.919 DEBUG:boto:auth_path=/avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.919 DEBUG:boto:Method: GET 2022-09-03T06:02:41.920 DEBUG:boto:Path: /avysmw-17/?versions&max-keys=999 2022-09-03T06:02:41.921 DEBUG:boto:Data: 2022-09-03T06:02:41.924 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.924 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:41.925 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.925 DEBUG:boto:Params: {} 2022-09-03T06:02:41.926 DEBUG:boto:Token: None 2022-09-03T06:02:41.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/?versions 2022-09-03T06:02:41.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mwz70mPlH+clHm13DJdAgGJSrqY= 2022-09-03T06:02:41.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mwz70mPlH+clHm13DJdAgGJSrqY=', 'Content-Length': '0'} 2022-09-03T06:02:41.931 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e79ec856346daa93-006312ee01-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.931 DEBUG:boto:b'avysmw-17999falsedummy12sb8StD5CsUkI7KT.ebtUf4sajm0w5true2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1sV69xi0pkVZOSv7CwV0724w2lJ-62-Ptrue2022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2Jf80yK-pTIgDpP74ft.tHsQQnD9zDlttrue2022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiNtrue2022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:41.933 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:41.933 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.934 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.935 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.935 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.936 DEBUG:boto:Method: GET 2022-09-03T06:02:41.937 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.937 DEBUG:boto:Data: 2022-09-03T06:02:41.938 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.938 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.939 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.939 DEBUG:boto:Params: {} 2022-09-03T06:02:41.940 DEBUG:boto:Token: None 2022-09-03T06:02:41.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw= 2022-09-03T06:02:41.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw=', 'Content-Length': '0'} 2022-09-03T06:02:41.944 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000dc2a5fd5534d6689-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.946 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.947 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.948 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.948 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.949 DEBUG:boto:Method: GET 2022-09-03T06:02:41.950 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.951 DEBUG:boto:Data: 2022-09-03T06:02:41.952 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.953 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:41.954 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.955 DEBUG:boto:Params: {} 2022-09-03T06:02:41.956 DEBUG:boto:Token: None 2022-09-03T06:02:41.957 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw= 2022-09-03T06:02:41.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMf2tLq1+BIoa1PX4W3NtAtL6hw=', 'Content-Length': '0'} 2022-09-03T06:02:41.963 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008386752352d11791-006312ee01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.964 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.965 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.965 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.966 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.967 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.968 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.969 DEBUG:boto:Data: 2022-09-03T06:02:41.969 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.970 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:41.970 DEBUG:boto:Port: 8001 2022-09-03T06:02:41.971 DEBUG:boto:Params: {} 2022-09-03T06:02:41.973 DEBUG:boto:Token: None 2022-09-03T06:02:41.974 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9dD5rcXjBuvufDZzpFg76IAssG8= 2022-09-03T06:02:41.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9dD5rcXjBuvufDZzpFg76IAssG8=', 'Content-Length': '0'} 2022-09-03T06:02:41.980 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000098fdc9c1979cbee7-006312ee01-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:41.982 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:41.983 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:41.984 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.984 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.985 DEBUG:boto:Method: HEAD 2022-09-03T06:02:41.987 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.988 DEBUG:boto:Data: 2022-09-03T06:02:41.989 DEBUG:boto:Headers: {} 2022-09-03T06:02:41.990 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:41.991 DEBUG:boto:Port: 8000 2022-09-03T06:02:41.993 DEBUG:boto:Params: {} 2022-09-03T06:02:41.995 DEBUG:boto:Token: None 2022-09-03T06:02:41.996 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:41 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:41.997 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9dD5rcXjBuvufDZzpFg76IAssG8= 2022-09-03T06:02:41.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9dD5rcXjBuvufDZzpFg76IAssG8=', 'Content-Length': '0'} 2022-09-03T06:02:42.000 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001d0209a06d84494c-006312ee01-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.002 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:42.003 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:42.004 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:42.004 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.005 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.006 DEBUG:boto:Method: GET 2022-09-03T06:02:42.006 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.007 DEBUG:boto:Data: 2022-09-03T06:02:42.007 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.009 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.009 DEBUG:boto:Params: {} 2022-09-03T06:02:42.010 DEBUG:boto:Token: None 2022-09-03T06:02:42.010 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z5PsPSu+6mMibVVgprX/oWwWnII= 2022-09-03T06:02:42.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z5PsPSu+6mMibVVgprX/oWwWnII=', 'Content-Length': '0'} 2022-09-03T06:02:42.015 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b1bcebae8d392f71-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.016 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:42.016 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:42.017 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.017 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.018 DEBUG:boto:Method: GET 2022-09-03T06:02:42.019 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.019 DEBUG:boto:Data: 2022-09-03T06:02:42.020 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.021 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.021 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.022 DEBUG:boto:Params: {} 2022-09-03T06:02:42.023 DEBUG:boto:Token: None 2022-09-03T06:02:42.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z5PsPSu+6mMibVVgprX/oWwWnII= 2022-09-03T06:02:42.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z5PsPSu+6mMibVVgprX/oWwWnII=', 'Content-Length': '0'} 2022-09-03T06:02:42.035 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004ba69cf4207b8eb9-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.036 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.036 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.037 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:42.037 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:42.039 DEBUG:boto:Method: GET 2022-09-03T06:02:42.039 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:42.040 DEBUG:boto:Data: 2022-09-03T06:02:42.040 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.041 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.042 DEBUG:boto:Params: {} 2022-09-03T06:02:42.042 DEBUG:boto:Token: None 2022-09-03T06:02:42.043 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.044 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.047 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000084217f27d5c24afd-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.095 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.096 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.096 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.097 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:42.098 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:42.098 DEBUG:boto:Method: GET 2022-09-03T06:02:42.099 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:42.100 DEBUG:boto:Data: 2022-09-03T06:02:42.102 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.103 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.104 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.106 DEBUG:boto:Params: {} 2022-09-03T06:02:42.106 DEBUG:boto:Token: None 2022-09-03T06:02:42.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.111 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098d9a1e60cc3e6cf-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.112 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.113 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:42.113 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:42.115 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.115 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T06:02:42.116 DEBUG:boto:Data: 2022-09-03T06:02:42.116 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.117 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.118 DEBUG:boto:Params: {} 2022-09-03T06:02:42.118 DEBUG:boto:Token: None 2022-09-03T06:02:42.119 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/dummy 2022-09-03T06:02:42.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pm9Hsxo2HXM0fV6ClWBJmQUK+xU= 2022-09-03T06:02:42.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pm9Hsxo2HXM0fV6ClWBJmQUK+xU=', 'Content-Length': '0'} 2022-09-03T06:02:42.123 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003b73fccc89e478e4-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.123 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:42.124 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:42.125 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.125 DEBUG:boto:Path: /avysmw-17/dummy 2022-09-03T06:02:42.126 DEBUG:boto:Data: 2022-09-03T06:02:42.126 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.127 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.127 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.128 DEBUG:boto:Params: {} 2022-09-03T06:02:42.128 DEBUG:boto:Token: None 2022-09-03T06:02:42.129 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/dummy 2022-09-03T06:02:42.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pm9Hsxo2HXM0fV6ClWBJmQUK+xU= 2022-09-03T06:02:42.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pm9Hsxo2HXM0fV6ClWBJmQUK+xU=', 'Content-Length': '0'} 2022-09-03T06:02:42.133 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001875cdc9da5c3521-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.134 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:02:42.135 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:42.136 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:42.136 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.137 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.137 DEBUG:boto:Method: GET 2022-09-03T06:02:42.138 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.139 DEBUG:boto:Data: 2022-09-03T06:02:42.139 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.140 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.141 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.141 DEBUG:boto:Params: {} 2022-09-03T06:02:42.142 DEBUG:boto:Token: None 2022-09-03T06:02:42.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z5PsPSu+6mMibVVgprX/oWwWnII= 2022-09-03T06:02:42.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z5PsPSu+6mMibVVgprX/oWwWnII=', 'Content-Length': '0'} 2022-09-03T06:02:42.147 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000000f4f6fe71339df20-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.148 DEBUG:boto:path=/avysmw-17/dummy 2022-09-03T06:02:42.149 DEBUG:boto:auth_path=/avysmw-17/dummy 2022-09-03T06:02:42.149 DEBUG:boto:path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.150 DEBUG:boto:auth_path=/avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.151 DEBUG:boto:Method: GET 2022-09-03T06:02:42.151 DEBUG:boto:Path: /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.152 DEBUG:boto:Data: 2022-09-03T06:02:42.152 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.153 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.153 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.154 DEBUG:boto:Params: {} 2022-09-03T06:02:42.155 DEBUG:boto:Token: None 2022-09-03T06:02:42.156 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/dummy?versionId=12sb8StD5CsUkI7KT.ebtUf4sajm0w5 2022-09-03T06:02:42.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z5PsPSu+6mMibVVgprX/oWwWnII= 2022-09-03T06:02:42.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z5PsPSu+6mMibVVgprX/oWwWnII=', 'Content-Length': '0'} 2022-09-03T06:02:42.160 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:13 GMT'), ('x-amz-version-id', '12sb8StD5CsUkI7KT.ebtUf4sajm0w5'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a1e024a37d8cf4fe-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.161 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.161 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.162 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:42.162 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:42.163 DEBUG:boto:Method: GET 2022-09-03T06:02:42.163 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:42.164 DEBUG:boto:Data: 2022-09-03T06:02:42.165 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.166 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.167 DEBUG:boto:Params: {} 2022-09-03T06:02:42.169 DEBUG:boto:Token: None 2022-09-03T06:02:42.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.175 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000061b08f25164afc24-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.176 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.177 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.179 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.179 DEBUG:boto:path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:42.180 DEBUG:boto:auth_path=/avysmw-17/?prefix=dummy 2022-09-03T06:02:42.180 DEBUG:boto:Method: GET 2022-09-03T06:02:42.181 DEBUG:boto:Path: /avysmw-17/?prefix=dummy 2022-09-03T06:02:42.182 DEBUG:boto:Data: 2022-09-03T06:02:42.183 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.184 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.185 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.186 DEBUG:boto:Params: {} 2022-09-03T06:02:42.186 DEBUG:boto:Token: None 2022-09-03T06:02:42.187 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.191 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a4379277e808818-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.191 DEBUG:boto:b'avysmw-17dummy1000falsedummy2022-09-03T05:56:13.455Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.192 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:42.193 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.193 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.194 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.194 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.195 DEBUG:boto:Method: GET 2022-09-03T06:02:42.196 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.196 DEBUG:boto:Data: 2022-09-03T06:02:42.197 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.198 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.198 DEBUG:boto:Params: {} 2022-09-03T06:02:42.199 DEBUG:boto:Token: None 2022-09-03T06:02:42.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w= 2022-09-03T06:02:42.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w=', 'Content-Length': '0'} 2022-09-03T06:02:42.203 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000053e714de807b4079-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.246 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.247 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.248 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.249 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.249 DEBUG:boto:Method: GET 2022-09-03T06:02:42.250 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.251 DEBUG:boto:Data: 2022-09-03T06:02:42.252 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.252 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.259 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.260 DEBUG:boto:Params: {} 2022-09-03T06:02:42.261 DEBUG:boto:Token: None 2022-09-03T06:02:42.269 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w= 2022-09-03T06:02:42.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w=', 'Content-Length': '0'} 2022-09-03T06:02:42.275 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000eeaf9b97db6588e7-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.276 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.277 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.278 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.280 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.281 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.282 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.283 DEBUG:boto:Data: 2022-09-03T06:02:42.284 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.286 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.287 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.288 DEBUG:boto:Params: {} 2022-09-03T06:02:42.289 DEBUG:boto:Token: None 2022-09-03T06:02:42.290 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2vPKeV/joRsulnMatlxVv/znDkM= 2022-09-03T06:02:42.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2vPKeV/joRsulnMatlxVv/znDkM=', 'Content-Length': '0'} 2022-09-03T06:02:42.296 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f076203a74a70932-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.297 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.297 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.299 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.299 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.300 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.301 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.302 DEBUG:boto:Data: 2022-09-03T06:02:42.303 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.304 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.305 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.306 DEBUG:boto:Params: {} 2022-09-03T06:02:42.307 DEBUG:boto:Token: None 2022-09-03T06:02:42.309 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2vPKeV/joRsulnMatlxVv/znDkM= 2022-09-03T06:02:42.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2vPKeV/joRsulnMatlxVv/znDkM=', 'Content-Length': '0'} 2022-09-03T06:02:42.315 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f7074eca2572358f-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.320 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:42.321 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.322 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.322 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.323 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.323 DEBUG:boto:Method: GET 2022-09-03T06:02:42.324 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.325 DEBUG:boto:Data: 2022-09-03T06:02:42.329 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.331 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.331 DEBUG:boto:Params: {} 2022-09-03T06:02:42.332 DEBUG:boto:Token: None 2022-09-03T06:02:42.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w= 2022-09-03T06:02:42.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w=', 'Content-Length': '0'} 2022-09-03T06:02:42.337 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007dfc85b9236e5d98-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.340 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.341 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.342 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.343 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.344 DEBUG:boto:Method: GET 2022-09-03T06:02:42.345 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.346 DEBUG:boto:Data: 2022-09-03T06:02:42.347 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.348 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.349 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.350 DEBUG:boto:Params: {} 2022-09-03T06:02:42.351 DEBUG:boto:Token: None 2022-09-03T06:02:42.352 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.353 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w= 2022-09-03T06:02:42.354 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w=', 'Content-Length': '0'} 2022-09-03T06:02:42.357 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000334ae094a491a055-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.358 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.359 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.360 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.361 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.362 DEBUG:boto:Method: GET 2022-09-03T06:02:42.363 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.364 DEBUG:boto:Data: 2022-09-03T06:02:42.365 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.366 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.367 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.368 DEBUG:boto:Params: {} 2022-09-03T06:02:42.369 DEBUG:boto:Token: None 2022-09-03T06:02:42.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.377 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d266993f5218feb-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.378 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.380 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.381 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.381 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.382 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.383 DEBUG:boto:Method: GET 2022-09-03T06:02:42.384 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.385 DEBUG:boto:Data: 2022-09-03T06:02:42.386 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.387 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.388 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.389 DEBUG:boto:Params: {} 2022-09-03T06:02:42.391 DEBUG:boto:Token: None 2022-09-03T06:02:42.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.401 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d020748ec50a444f-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.402 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.404 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.405 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.406 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.407 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T06:02:42.408 DEBUG:boto:Data: 2022-09-03T06:02:42.409 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.410 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.411 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.412 DEBUG:boto:Params: {} 2022-09-03T06:02:42.413 DEBUG:boto:Token: None 2022-09-03T06:02:42.414 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1 2022-09-03T06:02:42.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oWaCibcuV0O3eOhwv3RyjeK/A94= 2022-09-03T06:02:42.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oWaCibcuV0O3eOhwv3RyjeK/A94=', 'Content-Length': '0'} 2022-09-03T06:02:42.420 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008728f9be89a702bf-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.422 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.423 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.424 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.425 DEBUG:boto:Path: /avysmw-17/obj-test-zone1 2022-09-03T06:02:42.426 DEBUG:boto:Data: 2022-09-03T06:02:42.427 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.428 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.428 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.429 DEBUG:boto:Params: {} 2022-09-03T06:02:42.434 DEBUG:boto:Token: None 2022-09-03T06:02:42.435 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1 2022-09-03T06:02:42.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oWaCibcuV0O3eOhwv3RyjeK/A94= 2022-09-03T06:02:42.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oWaCibcuV0O3eOhwv3RyjeK/A94=', 'Content-Length': '0'} 2022-09-03T06:02:42.439 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000c0eb2db01c61252-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.440 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:02:42.441 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.441 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.442 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.442 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.443 DEBUG:boto:Method: GET 2022-09-03T06:02:42.444 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.444 DEBUG:boto:Data: 2022-09-03T06:02:42.445 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.446 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.446 DEBUG:boto:Params: {} 2022-09-03T06:02:42.447 DEBUG:boto:Token: None 2022-09-03T06:02:42.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w= 2022-09-03T06:02:42.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w=', 'Content-Length': '0'} 2022-09-03T06:02:42.452 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000521ff98499ee48a2-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.453 DEBUG:boto:path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.454 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1 2022-09-03T06:02:42.454 DEBUG:boto:path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.455 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.455 DEBUG:boto:Method: GET 2022-09-03T06:02:42.456 DEBUG:boto:Path: /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.457 DEBUG:boto:Data: 2022-09-03T06:02:42.458 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.458 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.459 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.459 DEBUG:boto:Params: {} 2022-09-03T06:02:42.460 DEBUG:boto:Token: None 2022-09-03T06:02:42.461 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone1?versionId=sV69xi0pkVZOSv7CwV0724w2lJ-62-P 2022-09-03T06:02:42.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w= 2022-09-03T06:02:42.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L8BvqD0s6JMa6iNtfeZuYz/zF4w=', 'Content-Length': '0'} 2022-09-03T06:02:42.470 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'sV69xi0pkVZOSv7CwV0724w2lJ-62-P'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000231e04b403161f59-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.515 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.516 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.517 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.518 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.519 DEBUG:boto:Method: GET 2022-09-03T06:02:42.520 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.521 DEBUG:boto:Data: 2022-09-03T06:02:42.522 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.523 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.524 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.525 DEBUG:boto:Params: {} 2022-09-03T06:02:42.525 DEBUG:boto:Token: None 2022-09-03T06:02:42.526 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.530 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007751addfc997a562-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.531 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.531 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.532 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.533 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.533 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.534 DEBUG:boto:Method: GET 2022-09-03T06:02:42.534 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone1 2022-09-03T06:02:42.535 DEBUG:boto:Data: 2022-09-03T06:02:42.535 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.536 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.538 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.539 DEBUG:boto:Params: {} 2022-09-03T06:02:42.539 DEBUG:boto:Token: None 2022-09-03T06:02:42.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.544 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091058179bcdb3b90-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.546 DEBUG:boto:b'avysmw-17obj-test-zone11000falseobj-test-zone12022-09-03T05:57:14.561Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.547 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:42.547 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.548 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.548 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.549 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.550 DEBUG:boto:Method: GET 2022-09-03T06:02:42.551 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.551 DEBUG:boto:Data: 2022-09-03T06:02:42.552 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.552 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.553 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.553 DEBUG:boto:Params: {} 2022-09-03T06:02:42.554 DEBUG:boto:Token: None 2022-09-03T06:02:42.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA= 2022-09-03T06:02:42.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA=', 'Content-Length': '0'} 2022-09-03T06:02:42.559 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000652efb63ca3a454a-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.559 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.560 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.561 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.561 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.562 DEBUG:boto:Method: GET 2022-09-03T06:02:42.563 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.564 DEBUG:boto:Data: 2022-09-03T06:02:42.564 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.565 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.566 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.566 DEBUG:boto:Params: {} 2022-09-03T06:02:42.567 DEBUG:boto:Token: None 2022-09-03T06:02:42.568 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA= 2022-09-03T06:02:42.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA=', 'Content-Length': '0'} 2022-09-03T06:02:42.573 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e4a79cc55dafdc9e-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.574 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.575 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.576 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.577 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.578 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.579 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.580 DEBUG:boto:Data: 2022-09-03T06:02:42.581 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.583 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.583 DEBUG:boto:Params: {} 2022-09-03T06:02:42.584 DEBUG:boto:Token: None 2022-09-03T06:02:42.584 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RD4ptD5HK6J8Q7gBnbwxRjpEAlc= 2022-09-03T06:02:42.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RD4ptD5HK6J8Q7gBnbwxRjpEAlc=', 'Content-Length': '0'} 2022-09-03T06:02:42.588 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008668f7d4112b74b1-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.588 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.589 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.589 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.590 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.590 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.591 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.592 DEBUG:boto:Data: 2022-09-03T06:02:42.592 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.593 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.593 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.594 DEBUG:boto:Params: {} 2022-09-03T06:02:42.594 DEBUG:boto:Token: None 2022-09-03T06:02:42.595 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.595 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RD4ptD5HK6J8Q7gBnbwxRjpEAlc= 2022-09-03T06:02:42.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RD4ptD5HK6J8Q7gBnbwxRjpEAlc=', 'Content-Length': '0'} 2022-09-03T06:02:42.598 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000649ee336e43440b4-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.599 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:42.599 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.600 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.601 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.601 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.602 DEBUG:boto:Method: GET 2022-09-03T06:02:42.602 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.603 DEBUG:boto:Data: 2022-09-03T06:02:42.604 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.605 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.605 DEBUG:boto:Params: {} 2022-09-03T06:02:42.606 DEBUG:boto:Token: None 2022-09-03T06:02:42.607 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA= 2022-09-03T06:02:42.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA=', 'Content-Length': '0'} 2022-09-03T06:02:42.610 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be5ea141f3e23389-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.654 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.655 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.655 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.656 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.657 DEBUG:boto:Method: GET 2022-09-03T06:02:42.657 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.658 DEBUG:boto:Data: 2022-09-03T06:02:42.659 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.659 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.660 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.660 DEBUG:boto:Params: {} 2022-09-03T06:02:42.661 DEBUG:boto:Token: None 2022-09-03T06:02:42.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.662 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA= 2022-09-03T06:02:42.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA=', 'Content-Length': '0'} 2022-09-03T06:02:42.665 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000019afee8b73ecb2d2-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.666 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.667 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.667 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.668 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.668 DEBUG:boto:Method: GET 2022-09-03T06:02:42.669 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.669 DEBUG:boto:Data: 2022-09-03T06:02:42.670 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.671 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.671 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.672 DEBUG:boto:Params: {} 2022-09-03T06:02:42.672 DEBUG:boto:Token: None 2022-09-03T06:02:42.673 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.677 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004302d493d19a3877-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.678 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.679 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.679 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.680 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.680 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.681 DEBUG:boto:Method: GET 2022-09-03T06:02:42.681 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.682 DEBUG:boto:Data: 2022-09-03T06:02:42.683 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.683 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.684 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.684 DEBUG:boto:Params: {} 2022-09-03T06:02:42.685 DEBUG:boto:Token: None 2022-09-03T06:02:42.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.689 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000242b379c91f3d122-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.690 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.691 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.691 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.692 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.692 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T06:02:42.693 DEBUG:boto:Data: 2022-09-03T06:02:42.693 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.694 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.694 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.695 DEBUG:boto:Params: {} 2022-09-03T06:02:42.696 DEBUG:boto:Token: None 2022-09-03T06:02:42.696 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2 2022-09-03T06:02:42.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dWTZNHNr+mpJY2NaFEPyCxhQJ80= 2022-09-03T06:02:42.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dWTZNHNr+mpJY2NaFEPyCxhQJ80=', 'Content-Length': '0'} 2022-09-03T06:02:42.700 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003ec3450310c38130-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.701 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.702 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.702 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.703 DEBUG:boto:Path: /avysmw-17/obj-test-zone2 2022-09-03T06:02:42.704 DEBUG:boto:Data: 2022-09-03T06:02:42.704 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.705 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.705 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.706 DEBUG:boto:Params: {} 2022-09-03T06:02:42.706 DEBUG:boto:Token: None 2022-09-03T06:02:42.707 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2 2022-09-03T06:02:42.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dWTZNHNr+mpJY2NaFEPyCxhQJ80= 2022-09-03T06:02:42.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dWTZNHNr+mpJY2NaFEPyCxhQJ80=', 'Content-Length': '0'} 2022-09-03T06:02:42.711 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008b10cfe44d401e17-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.712 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:02:42.713 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.713 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.714 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.714 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.715 DEBUG:boto:Method: GET 2022-09-03T06:02:42.715 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.716 DEBUG:boto:Data: 2022-09-03T06:02:42.716 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.717 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.718 DEBUG:boto:Params: {} 2022-09-03T06:02:42.719 DEBUG:boto:Token: None 2022-09-03T06:02:42.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA= 2022-09-03T06:02:42.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA=', 'Content-Length': '0'} 2022-09-03T06:02:42.723 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000034a81aeb3e5f03e2-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.724 DEBUG:boto:path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.724 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2 2022-09-03T06:02:42.725 DEBUG:boto:path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.725 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.726 DEBUG:boto:Method: GET 2022-09-03T06:02:42.727 DEBUG:boto:Path: /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.727 DEBUG:boto:Data: 2022-09-03T06:02:42.728 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.728 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.729 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.729 DEBUG:boto:Params: {} 2022-09-03T06:02:42.730 DEBUG:boto:Token: None 2022-09-03T06:02:42.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone2?versionId=Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt 2022-09-03T06:02:42.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA= 2022-09-03T06:02:42.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K4wkspW+CQlIYUKkJ500Csm8HdA=', 'Content-Length': '0'} 2022-09-03T06:02:42.735 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'Jf80yK-pTIgDpP74ft.tHsQQnD9zDlt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003de6307daf5b4136-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.778 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.779 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.779 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.780 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.780 DEBUG:boto:Method: GET 2022-09-03T06:02:42.781 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.781 DEBUG:boto:Data: 2022-09-03T06:02:42.782 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.783 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.784 DEBUG:boto:Params: {} 2022-09-03T06:02:42.784 DEBUG:boto:Token: None 2022-09-03T06:02:42.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.791 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003fc5bbe31e4e7ef9-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.794 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.795 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.795 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.796 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.796 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.797 DEBUG:boto:Method: GET 2022-09-03T06:02:42.797 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone2 2022-09-03T06:02:42.798 DEBUG:boto:Data: 2022-09-03T06:02:42.799 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.799 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.800 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.800 DEBUG:boto:Params: {} 2022-09-03T06:02:42.801 DEBUG:boto:Token: None 2022-09-03T06:02:42.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.806 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000733302a9671d3fe7-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.807 DEBUG:boto:b'avysmw-17obj-test-zone21000falseobj-test-zone22022-09-03T05:57:14.731Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.808 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:42.809 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.810 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.811 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.812 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.813 DEBUG:boto:Method: GET 2022-09-03T06:02:42.814 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.815 DEBUG:boto:Data: 2022-09-03T06:02:42.816 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.818 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.819 DEBUG:boto:Params: {} 2022-09-03T06:02:42.820 DEBUG:boto:Token: None 2022-09-03T06:02:42.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0= 2022-09-03T06:02:42.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0=', 'Content-Length': '0'} 2022-09-03T06:02:42.826 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b7fe497ccf1eff0d-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.827 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.828 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.829 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.830 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.831 DEBUG:boto:Method: GET 2022-09-03T06:02:42.832 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.833 DEBUG:boto:Data: 2022-09-03T06:02:42.834 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.835 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.836 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.837 DEBUG:boto:Params: {} 2022-09-03T06:02:42.838 DEBUG:boto:Token: None 2022-09-03T06:02:42.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0= 2022-09-03T06:02:42.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0=', 'Content-Length': '0'} 2022-09-03T06:02:42.842 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000017071991ca237379-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.843 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.844 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.845 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.845 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.846 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.847 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.849 DEBUG:boto:Data: 2022-09-03T06:02:42.851 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.852 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.853 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.853 DEBUG:boto:Params: {} 2022-09-03T06:02:42.855 DEBUG:boto:Token: None 2022-09-03T06:02:42.856 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5oRkz/eenGhkY5psTwSHzw04Hp0= 2022-09-03T06:02:42.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5oRkz/eenGhkY5psTwSHzw04Hp0=', 'Content-Length': '0'} 2022-09-03T06:02:42.859 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ace22168b078e03a-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.860 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.861 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.861 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.862 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.862 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.863 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.863 DEBUG:boto:Data: 2022-09-03T06:02:42.864 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.864 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.865 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.865 DEBUG:boto:Params: {} 2022-09-03T06:02:42.866 DEBUG:boto:Token: None 2022-09-03T06:02:42.866 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5oRkz/eenGhkY5psTwSHzw04Hp0= 2022-09-03T06:02:42.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5oRkz/eenGhkY5psTwSHzw04Hp0=', 'Content-Length': '0'} 2022-09-03T06:02:42.870 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ae8789ac4ea3d193-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.870 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:42.871 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.872 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.872 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.873 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.873 DEBUG:boto:Method: GET 2022-09-03T06:02:42.874 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.874 DEBUG:boto:Data: 2022-09-03T06:02:42.875 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.876 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.876 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.877 DEBUG:boto:Params: {} 2022-09-03T06:02:42.877 DEBUG:boto:Token: None 2022-09-03T06:02:42.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0= 2022-09-03T06:02:42.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0=', 'Content-Length': '0'} 2022-09-03T06:02:42.881 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c727ed7c0014769d-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.882 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.883 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.883 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.885 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.886 DEBUG:boto:Method: GET 2022-09-03T06:02:42.887 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.888 DEBUG:boto:Data: 2022-09-03T06:02:42.889 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.891 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.891 DEBUG:boto:Params: {} 2022-09-03T06:02:42.893 DEBUG:boto:Token: None 2022-09-03T06:02:42.894 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0= 2022-09-03T06:02:42.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0=', 'Content-Length': '0'} 2022-09-03T06:02:42.901 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000019255f09646fb0b0-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.904 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.905 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.906 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:42.907 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:42.908 DEBUG:boto:Method: GET 2022-09-03T06:02:42.909 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:42.911 DEBUG:boto:Data: 2022-09-03T06:02:42.912 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.913 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.913 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.914 DEBUG:boto:Params: {} 2022-09-03T06:02:42.915 DEBUG:boto:Token: None 2022-09-03T06:02:42.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.919 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000055ac215a58d45222-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.920 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.921 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:42.921 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:42.922 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:42.923 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:42.923 DEBUG:boto:Method: GET 2022-09-03T06:02:42.924 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:42.925 DEBUG:boto:Data: 2022-09-03T06:02:42.925 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.926 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.926 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.927 DEBUG:boto:Params: {} 2022-09-03T06:02:42.927 DEBUG:boto:Token: None 2022-09-03T06:02:42.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/ 2022-09-03T06:02:42.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo= 2022-09-03T06:02:42.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HIttqUO2ktPUQMB61QgYGwlsDoo=', 'Content-Length': '0'} 2022-09-03T06:02:42.932 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a09c18624db37be-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.933 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:42.934 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.935 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.935 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.936 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T06:02:42.937 DEBUG:boto:Data: 2022-09-03T06:02:42.937 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.938 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.939 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.940 DEBUG:boto:Params: {} 2022-09-03T06:02:42.941 DEBUG:boto:Token: None 2022-09-03T06:02:42.942 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone3 2022-09-03T06:02:42.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YRNHt1OqY3WQX16gk9sOs5gG01Q= 2022-09-03T06:02:42.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YRNHt1OqY3WQX16gk9sOs5gG01Q=', 'Content-Length': '0'} 2022-09-03T06:02:42.949 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d5813f7ddadd3fe9-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.952 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.953 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.953 DEBUG:boto:Method: HEAD 2022-09-03T06:02:42.954 DEBUG:boto:Path: /avysmw-17/obj-test-zone3 2022-09-03T06:02:42.954 DEBUG:boto:Data: 2022-09-03T06:02:42.961 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.962 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.962 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.963 DEBUG:boto:Params: {} 2022-09-03T06:02:42.963 DEBUG:boto:Token: None 2022-09-03T06:02:42.964 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone3 2022-09-03T06:02:42.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YRNHt1OqY3WQX16gk9sOs5gG01Q= 2022-09-03T06:02:42.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YRNHt1OqY3WQX16gk9sOs5gG01Q=', 'Content-Length': '0'} 2022-09-03T06:02:42.971 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a643052bdb0778cd-006312ee02-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.972 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:02:42.972 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.974 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.975 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.975 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.976 DEBUG:boto:Method: GET 2022-09-03T06:02:42.977 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.978 DEBUG:boto:Data: 2022-09-03T06:02:42.979 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:42.980 DEBUG:boto:Port: 8001 2022-09-03T06:02:42.981 DEBUG:boto:Params: {} 2022-09-03T06:02:42.982 DEBUG:boto:Token: None 2022-09-03T06:02:42.983 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:42 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.984 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0= 2022-09-03T06:02:42.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:euivwJzzedzgJBlpRl4YpmeGHH0=', 'Content-Length': '0'} 2022-09-03T06:02:42.988 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c3e37d4688e0a34c-006312ee02-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:42.989 DEBUG:boto:path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.991 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3 2022-09-03T06:02:42.992 DEBUG:boto:path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.992 DEBUG:boto:auth_path=/avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.993 DEBUG:boto:Method: GET 2022-09-03T06:02:42.993 DEBUG:boto:Path: /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:42.994 DEBUG:boto:Data: 2022-09-03T06:02:42.995 DEBUG:boto:Headers: {} 2022-09-03T06:02:42.995 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:42.996 DEBUG:boto:Port: 8000 2022-09-03T06:02:42.997 DEBUG:boto:Params: {} 2022-09-03T06:02:42.999 DEBUG:boto:Token: None 2022-09-03T06:02:43.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:43 GMT /avysmw-17/obj-test-zone3?versionId=H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN 2022-09-03T06:02:43.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n2RfzttRr5LOMXm8bDRGw2MqEY4= 2022-09-03T06:02:43.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n2RfzttRr5LOMXm8bDRGw2MqEY4=', 'Content-Length': '0'} 2022-09-03T06:02:43.006 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'H.ihqj6VVazO5x3ZSzCPfR8S7HaZLiN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000d9503462dc013d0-006312ee03-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:02:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:43.046 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:43.048 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:43.049 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:43.050 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:43.051 DEBUG:boto:Method: GET 2022-09-03T06:02:43.052 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:43.053 DEBUG:boto:Data: 2022-09-03T06:02:43.054 DEBUG:boto:Headers: {} 2022-09-03T06:02:43.055 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:02:43.057 DEBUG:boto:Port: 8001 2022-09-03T06:02:43.058 DEBUG:boto:Params: {} 2022-09-03T06:02:43.059 DEBUG:boto:Token: None 2022-09-03T06:02:43.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:43 GMT /avysmw-17/ 2022-09-03T06:02:43.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y/GfyXM12nB92SzlBxG+PiWqPSc= 2022-09-03T06:02:43.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y/GfyXM12nB92SzlBxG+PiWqPSc=', 'Content-Length': '0'} 2022-09-03T06:02:43.067 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000038137c142e21183e-006312ee03-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:43.068 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:43.070 DEBUG:boto:path=/avysmw-17/ 2022-09-03T06:02:43.071 DEBUG:boto:auth_path=/avysmw-17/ 2022-09-03T06:02:43.072 DEBUG:boto:path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:43.073 DEBUG:boto:auth_path=/avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:43.074 DEBUG:boto:Method: GET 2022-09-03T06:02:43.075 DEBUG:boto:Path: /avysmw-17/?prefix=obj-test-zone3 2022-09-03T06:02:43.076 DEBUG:boto:Data: 2022-09-03T06:02:43.077 DEBUG:boto:Headers: {} 2022-09-03T06:02:43.078 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:02:43.079 DEBUG:boto:Port: 8000 2022-09-03T06:02:43.079 DEBUG:boto:Params: {} 2022-09-03T06:02:43.080 DEBUG:boto:Token: None 2022-09-03T06:02:43.081 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:02:43 GMT /avysmw-17/ 2022-09-03T06:02:43.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y/GfyXM12nB92SzlBxG+PiWqPSc= 2022-09-03T06:02:43.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:02:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y/GfyXM12nB92SzlBxG+PiWqPSc=', 'Content-Length': '0'} 2022-09-03T06:02:43.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cc399e3684a10139-006312ee03-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:02:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:02:43.085 DEBUG:boto:b'avysmw-17obj-test-zone31000falseobj-test-zone32022-09-03T05:57:14.903Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:02:43.086 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-17 zones={test-zone3, test-zone2} 2022-09-03T06:02:43.086 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-18 2022-09-03T06:02:43.087 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.088 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.089 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:43.152 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:43.153 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:43.204 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:43.201+0000 7f901c677600 1 bucket sync caught up with source: 2022-09-03T06:02:43.204 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000011.29.5, , 00000000004.19.5, , 00000000008.28.5, , , , , ] 2022-09-03T06:02:43.204 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.29.5, , 00000000004.19.5, , 00000000008.28.5, , , , , ] 2022-09-03T06:02:43.205 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:43.201+0000 7f901c677600 0 bucket checkpoint complete 2022-09-03T06:02:43.224 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-18 2022-09-03T06:02:43.225 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.226 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.227 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:43.296 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:43.297 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:43.341 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:43.339+0000 7f09dd280600 1 bucket sync caught up with source: 2022-09-03T06:02:43.342 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000011.29.5, , 00000000004.19.5, , 00000000008.28.5, , , , , ] 2022-09-03T06:02:43.342 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000011.29.5, , 00000000004.19.5, , 00000000008.28.5, , , , , ] 2022-09-03T06:02:43.342 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:43.339+0000 7f09dd280600 0 bucket checkpoint complete 2022-09-03T06:02:43.361 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-18 2022-09-03T06:02:43.362 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.363 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.363 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:43.422 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:43.423 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:43.466 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:43.463+0000 7f238264a600 1 bucket sync caught up with source: 2022-09-03T06:02:43.467 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000011.29.5, , 00000000004.19.5, , 00000000008.28.5, , , , , ] 2022-09-03T06:02:43.467 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000011.29.5, , 00000000004.19.5, , 00000000008.28.5, , , , , ] 2022-09-03T06:02:43.469 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:02:43.463+0000 7f238264a600 0 bucket checkpoint complete 2022-09-03T06:02:43.477 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-18 2022-09-03T06:02:43.478 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.479 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.479 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:43.538 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:43.539 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:43.587 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:43.585+0000 7fbf59567600 1 bucket sync caught up with source: 2022-09-03T06:02:43.588 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.31.5, , 00000000011.55.5, , 00000000008.47.5, , , , , ] 2022-09-03T06:02:43.588 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.31.5, , 00000000011.55.5, , 00000000008.47.5, , , , , ] 2022-09-03T06:02:43.588 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:43.585+0000 7fbf59567600 0 bucket checkpoint complete 2022-09-03T06:02:43.606 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-18 2022-09-03T06:02:43.608 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.609 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.610 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:43.671 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:43.672 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:43.726 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:43.724+0000 7fc6baa0a600 1 bucket sync caught up with source: 2022-09-03T06:02:43.726 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.31.5, , 00000000011.55.5, , 00000000008.47.5, , , , , ] 2022-09-03T06:02:43.727 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.31.5, , 00000000011.55.5, , 00000000008.47.5, , , , , ] 2022-09-03T06:02:43.727 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:43.724+0000 7fc6baa0a600 0 bucket checkpoint complete 2022-09-03T06:02:43.742 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-18 2022-09-03T06:02:43.743 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-18 2022-09-03T06:02:43.744 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.744 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:02:43.745 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:02:43.803 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:02:43.803 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:02:43.844 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:43.842+0000 7fbfb3105600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:43.844 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.38.5, , 00000000003.41.9, , 00000000015.72.5, , , , , ] 2022-09-03T06:02:43.845 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:02:48.846 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:48.843+0000 7fbfb3105600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:48.847 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.38.5, , 00000000003.41.9, , 00000000015.72.5, , , , , ] 2022-09-03T06:02:48.847 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:02:53.847 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:53.844+0000 7fbfb3105600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:53.848 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.38.5, , 00000000003.41.9, , 00000000015.72.5, , , , , ] 2022-09-03T06:02:53.848 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:02:58.849 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:02:58.846+0000 7fbfb3105600 1 waiting for incremental sync to catch up: 2022-09-03T06:02:58.849 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.38.5, , 00000000003.41.9, , 00000000015.72.5, , , , , ] 2022-09-03T06:02:58.850 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:03:03.850 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:03.847+0000 7fbfb3105600 1 waiting for incremental sync to catch up: 2022-09-03T06:03:03.851 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.38.5, , 00000000003.41.9, , 00000000015.72.5, , , , , ] 2022-09-03T06:03:03.851 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:03:08.852 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:08.850+0000 7fbfb3105600 1 waiting for incremental sync to catch up: 2022-09-03T06:03:08.852 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.38.5, , 00000000003.41.9, , 00000000015.72.5, , , , , ] 2022-09-03T06:03:08.853 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:03:13.853 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:13.851+0000 7fbfb3105600 1 bucket sync caught up with source: 2022-09-03T06:03:13.854 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:03:13.854 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:03:13.854 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:13.851+0000 7fbfb3105600 0 bucket checkpoint complete 2022-09-03T06:03:13.871 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-18 2022-09-03T06:03:13.872 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:13.873 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:13.875 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:03:13.939 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:13.939 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:13.983 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:13.981+0000 7f20ac7fc600 1 bucket sync caught up with source: 2022-09-03T06:03:13.984 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:03:13.984 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000004.38.5, , 00000000004.44.5, , 00000000015.72.5, , , , , ] 2022-09-03T06:03:13.984 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:13.981+0000 7f20ac7fc600 0 bucket checkpoint complete 2022-09-03T06:03:13.996 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-18 2022-09-03T06:03:13.998 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-18 zones={test-zone2, test-zone1} 2022-09-03T06:03:13.999 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.000 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.001 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.002 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T06:03:14.002 DEBUG:boto:Data: 2022-09-03T06:03:14.003 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.004 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.004 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.005 DEBUG:boto:Params: {} 2022-09-03T06:03:14.006 DEBUG:boto:Token: None 2022-09-03T06:03:14.007 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m8bWCh0gIZpvbJ/d82BZMXe3Zw8= 2022-09-03T06:03:14.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m8bWCh0gIZpvbJ/d82BZMXe3Zw8=', 'Content-Length': '0'} 2022-09-03T06:03:14.011 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009cc48bb96452bc77-006312ee22-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.012 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.012 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.013 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.014 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T06:03:14.014 DEBUG:boto:Data: 2022-09-03T06:03:14.015 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.016 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.016 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.017 DEBUG:boto:Params: {} 2022-09-03T06:03:14.018 DEBUG:boto:Token: None 2022-09-03T06:03:14.019 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m8bWCh0gIZpvbJ/d82BZMXe3Zw8= 2022-09-03T06:03:14.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m8bWCh0gIZpvbJ/d82BZMXe3Zw8=', 'Content-Length': '0'} 2022-09-03T06:03:14.024 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d260c1b1d3197bbc-006312ee22-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.025 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:03:14.026 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.026 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.027 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.027 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.028 DEBUG:boto:Method: GET 2022-09-03T06:03:14.028 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.029 DEBUG:boto:Data: 2022-09-03T06:03:14.029 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.030 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.030 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.031 DEBUG:boto:Params: {} 2022-09-03T06:03:14.031 DEBUG:boto:Token: None 2022-09-03T06:03:14.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/?versions 2022-09-03T06:03:14.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K43wp0aEdDoYDKGWYIssz+ep/1I= 2022-09-03T06:03:14.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K43wp0aEdDoYDKGWYIssz+ep/1I=', 'Content-Length': '0'} 2022-09-03T06:03:14.036 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000001a747bf1411bd8e-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.037 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.038 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:03:14.039 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:03:14.039 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:03:14.040 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:03:14.040 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:03:14.041 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.041 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.042 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.042 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.043 DEBUG:boto:Method: GET 2022-09-03T06:03:14.044 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.044 DEBUG:boto:Data: 2022-09-03T06:03:14.045 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.046 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.046 DEBUG:boto:Params: {} 2022-09-03T06:03:14.047 DEBUG:boto:Token: None 2022-09-03T06:03:14.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/?versions 2022-09-03T06:03:14.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K43wp0aEdDoYDKGWYIssz+ep/1I= 2022-09-03T06:03:14.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K43wp0aEdDoYDKGWYIssz+ep/1I=', 'Content-Length': '0'} 2022-09-03T06:03:14.051 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b66e67c13b04cac0-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.052 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.053 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:03:14.054 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:03:14.055 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:03:14.055 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:03:14.056 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.057 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.057 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.058 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.058 DEBUG:boto:Method: GET 2022-09-03T06:03:14.059 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.059 DEBUG:boto:Data: 2022-09-03T06:03:14.060 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.060 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.061 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.062 DEBUG:boto:Params: {} 2022-09-03T06:03:14.062 DEBUG:boto:Token: None 2022-09-03T06:03:14.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/?versions 2022-09-03T06:03:14.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K43wp0aEdDoYDKGWYIssz+ep/1I= 2022-09-03T06:03:14.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K43wp0aEdDoYDKGWYIssz+ep/1I=', 'Content-Length': '0'} 2022-09-03T06:03:14.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000046e40c91b05b65de-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.070 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.072 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.073 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.074 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.075 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.075 DEBUG:boto:Method: GET 2022-09-03T06:03:14.076 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:14.077 DEBUG:boto:Data: 2022-09-03T06:03:14.078 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.080 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.081 DEBUG:boto:Params: {} 2022-09-03T06:03:14.082 DEBUG:boto:Token: None 2022-09-03T06:03:14.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/?versions 2022-09-03T06:03:14.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K43wp0aEdDoYDKGWYIssz+ep/1I= 2022-09-03T06:03:14.086 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K43wp0aEdDoYDKGWYIssz+ep/1I=', 'Content-Length': '0'} 2022-09-03T06:03:14.089 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000362930ecbbf2265c-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.090 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.091 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:03:14.091 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.092 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.092 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.093 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.093 DEBUG:boto:Method: GET 2022-09-03T06:03:14.094 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.094 DEBUG:boto:Data: 2022-09-03T06:03:14.095 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.096 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.097 DEBUG:boto:Params: {} 2022-09-03T06:03:14.097 DEBUG:boto:Token: None 2022-09-03T06:03:14.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc= 2022-09-03T06:03:14.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc=', 'Content-Length': '0'} 2022-09-03T06:03:14.101 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009ff273364d0ca0dc-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.103 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.103 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.104 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.104 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.105 DEBUG:boto:Method: GET 2022-09-03T06:03:14.105 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.106 DEBUG:boto:Data: 2022-09-03T06:03:14.107 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.107 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.108 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.108 DEBUG:boto:Params: {} 2022-09-03T06:03:14.109 DEBUG:boto:Token: None 2022-09-03T06:03:14.109 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.110 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc= 2022-09-03T06:03:14.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc=', 'Content-Length': '0'} 2022-09-03T06:03:14.113 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000013b531936abf2500-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.114 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.115 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.116 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.117 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.118 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.119 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.120 DEBUG:boto:Data: 2022-09-03T06:03:14.121 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.122 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.122 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.123 DEBUG:boto:Params: {} 2022-09-03T06:03:14.124 DEBUG:boto:Token: None 2022-09-03T06:03:14.124 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9FUp+pNGRgkgAWha02hr4UxeYn8= 2022-09-03T06:03:14.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9FUp+pNGRgkgAWha02hr4UxeYn8=', 'Content-Length': '0'} 2022-09-03T06:03:14.128 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c769029f60b57bfd-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.128 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.129 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.130 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.131 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.131 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.132 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.133 DEBUG:boto:Data: 2022-09-03T06:03:14.133 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.135 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.136 DEBUG:boto:Params: {} 2022-09-03T06:03:14.137 DEBUG:boto:Token: None 2022-09-03T06:03:14.138 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9FUp+pNGRgkgAWha02hr4UxeYn8= 2022-09-03T06:03:14.141 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9FUp+pNGRgkgAWha02hr4UxeYn8=', 'Content-Length': '0'} 2022-09-03T06:03:14.143 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004bf8e16b514c8019-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.144 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:03:14.144 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.145 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.145 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.146 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.147 DEBUG:boto:Method: GET 2022-09-03T06:03:14.147 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.148 DEBUG:boto:Data: 2022-09-03T06:03:14.148 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.149 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.150 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.151 DEBUG:boto:Params: {} 2022-09-03T06:03:14.151 DEBUG:boto:Token: None 2022-09-03T06:03:14.152 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc= 2022-09-03T06:03:14.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc=', 'Content-Length': '0'} 2022-09-03T06:03:14.156 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007358ab32b73e544d-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.157 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.157 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.158 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.159 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.159 DEBUG:boto:Method: GET 2022-09-03T06:03:14.160 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.160 DEBUG:boto:Data: 2022-09-03T06:03:14.161 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.162 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.163 DEBUG:boto:Params: {} 2022-09-03T06:03:14.164 DEBUG:boto:Token: None 2022-09-03T06:03:14.164 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc= 2022-09-03T06:03:14.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc=', 'Content-Length': '0'} 2022-09-03T06:03:14.168 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000934cc826dea032e8-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.169 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.169 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.170 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:14.171 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:14.171 DEBUG:boto:Method: GET 2022-09-03T06:03:14.172 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:14.172 DEBUG:boto:Data: 2022-09-03T06:03:14.173 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.174 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.174 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.175 DEBUG:boto:Params: {} 2022-09-03T06:03:14.175 DEBUG:boto:Token: None 2022-09-03T06:03:14.176 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.180 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003c837b8cc63b9f20-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.181 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.182 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.182 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.183 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:14.184 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:14.185 DEBUG:boto:Method: GET 2022-09-03T06:03:14.185 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:14.186 DEBUG:boto:Data: 2022-09-03T06:03:14.187 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.188 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.189 DEBUG:boto:Params: {} 2022-09-03T06:03:14.190 DEBUG:boto:Token: None 2022-09-03T06:03:14.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.196 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000374a5ec089fca127-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.238 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.239 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.240 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.240 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.241 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T06:03:14.241 DEBUG:boto:Data: 2022-09-03T06:03:14.242 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.243 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.244 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.244 DEBUG:boto:Params: {} 2022-09-03T06:03:14.245 DEBUG:boto:Token: None 2022-09-03T06:03:14.246 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy 2022-09-03T06:03:14.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gsAl98sGqxMbG+Uskvj2aRSgX4E= 2022-09-03T06:03:14.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gsAl98sGqxMbG+Uskvj2aRSgX4E=', 'Content-Length': '0'} 2022-09-03T06:03:14.251 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007e0849ffac452d85-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.251 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.252 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.253 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.253 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T06:03:14.254 DEBUG:boto:Data: 2022-09-03T06:03:14.254 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.255 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.255 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.256 DEBUG:boto:Params: {} 2022-09-03T06:03:14.257 DEBUG:boto:Token: None 2022-09-03T06:03:14.257 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy 2022-09-03T06:03:14.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gsAl98sGqxMbG+Uskvj2aRSgX4E= 2022-09-03T06:03:14.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gsAl98sGqxMbG+Uskvj2aRSgX4E=', 'Content-Length': '0'} 2022-09-03T06:03:14.262 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000072dfd80dc0455c1f-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.263 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:03:14.263 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.264 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.264 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.265 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.266 DEBUG:boto:Method: GET 2022-09-03T06:03:14.266 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.267 DEBUG:boto:Data: 2022-09-03T06:03:14.268 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.268 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.269 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.270 DEBUG:boto:Params: {} 2022-09-03T06:03:14.271 DEBUG:boto:Token: None 2022-09-03T06:03:14.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc= 2022-09-03T06:03:14.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc=', 'Content-Length': '0'} 2022-09-03T06:03:14.275 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f411fa8f406da878-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.276 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:14.276 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:14.277 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.278 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.278 DEBUG:boto:Method: GET 2022-09-03T06:03:14.279 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.280 DEBUG:boto:Data: 2022-09-03T06:03:14.281 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.283 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.284 DEBUG:boto:Params: {} 2022-09-03T06:03:14.285 DEBUG:boto:Token: None 2022-09-03T06:03:14.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:14.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc= 2022-09-03T06:03:14.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iDuSyYFsX4dkAswl83tvCkC8kBc=', 'Content-Length': '0'} 2022-09-03T06:03:14.292 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c8feab6f6b5afb63-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.293 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.294 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.295 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:14.295 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:14.296 DEBUG:boto:Method: GET 2022-09-03T06:03:14.297 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:14.297 DEBUG:boto:Data: 2022-09-03T06:03:14.298 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.298 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.299 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.299 DEBUG:boto:Params: {} 2022-09-03T06:03:14.300 DEBUG:boto:Token: None 2022-09-03T06:03:14.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.301 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.304 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001646ae1f9b88cc1a-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.305 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.306 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.307 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.307 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:14.308 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:14.308 DEBUG:boto:Method: GET 2022-09-03T06:03:14.309 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:14.309 DEBUG:boto:Data: 2022-09-03T06:03:14.310 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.310 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.311 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.311 DEBUG:boto:Params: {} 2022-09-03T06:03:14.312 DEBUG:boto:Token: None 2022-09-03T06:03:14.313 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.317 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006b2e7e2946012908-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.318 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.319 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:03:14.320 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.320 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.321 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.321 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.322 DEBUG:boto:Method: GET 2022-09-03T06:03:14.322 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.323 DEBUG:boto:Data: 2022-09-03T06:03:14.323 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.324 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.324 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.325 DEBUG:boto:Params: {} 2022-09-03T06:03:14.325 DEBUG:boto:Token: None 2022-09-03T06:03:14.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso= 2022-09-03T06:03:14.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso=', 'Content-Length': '0'} 2022-09-03T06:03:14.330 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001bd46a6fbdde7292-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.331 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.331 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.332 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.333 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.333 DEBUG:boto:Method: GET 2022-09-03T06:03:14.334 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.334 DEBUG:boto:Data: 2022-09-03T06:03:14.335 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.336 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.337 DEBUG:boto:Params: {} 2022-09-03T06:03:14.337 DEBUG:boto:Token: None 2022-09-03T06:03:14.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso= 2022-09-03T06:03:14.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso=', 'Content-Length': '0'} 2022-09-03T06:03:14.347 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e624d98561f9ada5-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.348 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.348 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.349 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.349 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.350 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.351 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.351 DEBUG:boto:Data: 2022-09-03T06:03:14.352 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.352 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.353 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.353 DEBUG:boto:Params: {} 2022-09-03T06:03:14.354 DEBUG:boto:Token: None 2022-09-03T06:03:14.354 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2VEYlx0+HrDKY4rf0CXcjTyUk88= 2022-09-03T06:03:14.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2VEYlx0+HrDKY4rf0CXcjTyUk88=', 'Content-Length': '0'} 2022-09-03T06:03:14.357 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000f617921d329bb41-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.358 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.359 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.359 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.360 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.360 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.361 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.361 DEBUG:boto:Data: 2022-09-03T06:03:14.362 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.363 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.364 DEBUG:boto:Params: {} 2022-09-03T06:03:14.364 DEBUG:boto:Token: None 2022-09-03T06:03:14.365 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2VEYlx0+HrDKY4rf0CXcjTyUk88= 2022-09-03T06:03:14.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2VEYlx0+HrDKY4rf0CXcjTyUk88=', 'Content-Length': '0'} 2022-09-03T06:03:14.373 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004443c3a1e8b37dcc-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.374 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:03:14.375 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.375 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.376 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.377 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.377 DEBUG:boto:Method: GET 2022-09-03T06:03:14.378 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.379 DEBUG:boto:Data: 2022-09-03T06:03:14.380 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.380 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.381 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.381 DEBUG:boto:Params: {} 2022-09-03T06:03:14.382 DEBUG:boto:Token: None 2022-09-03T06:03:14.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso= 2022-09-03T06:03:14.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso=', 'Content-Length': '0'} 2022-09-03T06:03:14.389 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000351aa5807c655269-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.390 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.391 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.391 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.392 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.392 DEBUG:boto:Method: GET 2022-09-03T06:03:14.393 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.393 DEBUG:boto:Data: 2022-09-03T06:03:14.394 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.394 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.395 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.396 DEBUG:boto:Params: {} 2022-09-03T06:03:14.396 DEBUG:boto:Token: None 2022-09-03T06:03:14.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso= 2022-09-03T06:03:14.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso=', 'Content-Length': '0'} 2022-09-03T06:03:14.401 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d6ea2362efcd508d-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.446 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.447 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.448 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.448 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.449 DEBUG:boto:Method: GET 2022-09-03T06:03:14.449 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.450 DEBUG:boto:Data: 2022-09-03T06:03:14.450 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.451 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.451 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.452 DEBUG:boto:Params: {} 2022-09-03T06:03:14.452 DEBUG:boto:Token: None 2022-09-03T06:03:14.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.457 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e23ae3dc07a32bc8-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.457 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.458 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.459 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.459 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.460 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.460 DEBUG:boto:Method: GET 2022-09-03T06:03:14.461 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.461 DEBUG:boto:Data: 2022-09-03T06:03:14.462 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.463 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.463 DEBUG:boto:Params: {} 2022-09-03T06:03:14.464 DEBUG:boto:Token: None 2022-09-03T06:03:14.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.469 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f2dd5a75f94a535c-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.470 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.470 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.471 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.472 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.472 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T06:03:14.473 DEBUG:boto:Data: 2022-09-03T06:03:14.473 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.474 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.474 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.475 DEBUG:boto:Params: {} 2022-09-03T06:03:14.475 DEBUG:boto:Token: None 2022-09-03T06:03:14.476 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1 2022-09-03T06:03:14.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:otdAyKA/XC5K/L+k6xuogWXWvyE= 2022-09-03T06:03:14.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:otdAyKA/XC5K/L+k6xuogWXWvyE=', 'Content-Length': '0'} 2022-09-03T06:03:14.479 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dd3745f73acb5371-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.480 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.481 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.481 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.482 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T06:03:14.482 DEBUG:boto:Data: 2022-09-03T06:03:14.483 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.484 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.484 DEBUG:boto:Params: {} 2022-09-03T06:03:14.485 DEBUG:boto:Token: None 2022-09-03T06:03:14.485 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1 2022-09-03T06:03:14.486 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:otdAyKA/XC5K/L+k6xuogWXWvyE= 2022-09-03T06:03:14.487 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:otdAyKA/XC5K/L+k6xuogWXWvyE=', 'Content-Length': '0'} 2022-09-03T06:03:14.489 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000024b3a40a88e1c25-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.490 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:03:14.491 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.491 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.492 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.492 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.493 DEBUG:boto:Method: GET 2022-09-03T06:03:14.493 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.494 DEBUG:boto:Data: 2022-09-03T06:03:14.494 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.495 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.495 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.496 DEBUG:boto:Params: {} 2022-09-03T06:03:14.497 DEBUG:boto:Token: None 2022-09-03T06:03:14.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso= 2022-09-03T06:03:14.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso=', 'Content-Length': '0'} 2022-09-03T06:03:14.500 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d06e756ec609255f-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.501 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.502 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:14.502 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.503 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.503 DEBUG:boto:Method: GET 2022-09-03T06:03:14.504 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.504 DEBUG:boto:Data: 2022-09-03T06:03:14.505 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.506 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.507 DEBUG:boto:Params: {} 2022-09-03T06:03:14.507 DEBUG:boto:Token: None 2022-09-03T06:03:14.508 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:14.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso= 2022-09-03T06:03:14.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OPmtPvCbD0mHBwpMJrBv/476Fso=', 'Content-Length': '0'} 2022-09-03T06:03:14.511 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000037e412f7dda32050-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.512 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.513 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.513 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.514 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.514 DEBUG:boto:Method: GET 2022-09-03T06:03:14.515 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.515 DEBUG:boto:Data: 2022-09-03T06:03:14.516 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.516 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.517 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.517 DEBUG:boto:Params: {} 2022-09-03T06:03:14.518 DEBUG:boto:Token: None 2022-09-03T06:03:14.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.522 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076fdd4d4494a0539-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.523 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.524 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.525 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.526 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.527 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.528 DEBUG:boto:Method: GET 2022-09-03T06:03:14.528 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:14.529 DEBUG:boto:Data: 2022-09-03T06:03:14.530 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.530 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.531 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.531 DEBUG:boto:Params: {} 2022-09-03T06:03:14.532 DEBUG:boto:Token: None 2022-09-03T06:03:14.533 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.537 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009764495a1488a8d8-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.538 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.538 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:03:14.539 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.540 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.540 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.541 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.541 DEBUG:boto:Method: GET 2022-09-03T06:03:14.542 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.543 DEBUG:boto:Data: 2022-09-03T06:03:14.543 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.544 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.544 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.545 DEBUG:boto:Params: {} 2022-09-03T06:03:14.546 DEBUG:boto:Token: None 2022-09-03T06:03:14.546 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y= 2022-09-03T06:03:14.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y=', 'Content-Length': '0'} 2022-09-03T06:03:14.550 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000eef5d02d5c993ce6-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.590 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.591 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.591 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.592 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.593 DEBUG:boto:Method: GET 2022-09-03T06:03:14.593 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.594 DEBUG:boto:Data: 2022-09-03T06:03:14.594 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.595 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.595 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.596 DEBUG:boto:Params: {} 2022-09-03T06:03:14.596 DEBUG:boto:Token: None 2022-09-03T06:03:14.597 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.598 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y= 2022-09-03T06:03:14.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y=', 'Content-Length': '0'} 2022-09-03T06:03:14.601 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003698f3dd8d0191d3-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.602 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.602 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.603 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.603 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.604 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.604 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.605 DEBUG:boto:Data: 2022-09-03T06:03:14.605 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.606 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.606 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.607 DEBUG:boto:Params: {} 2022-09-03T06:03:14.607 DEBUG:boto:Token: None 2022-09-03T06:03:14.608 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:267lQZHq1EQbW3+Rh6bF6eEODmo= 2022-09-03T06:03:14.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:267lQZHq1EQbW3+Rh6bF6eEODmo=', 'Content-Length': '0'} 2022-09-03T06:03:14.611 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000021881e95d8fffd67-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.612 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.613 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.613 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.614 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.614 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.615 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.615 DEBUG:boto:Data: 2022-09-03T06:03:14.616 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.617 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.617 DEBUG:boto:Params: {} 2022-09-03T06:03:14.618 DEBUG:boto:Token: None 2022-09-03T06:03:14.619 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:267lQZHq1EQbW3+Rh6bF6eEODmo= 2022-09-03T06:03:14.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:267lQZHq1EQbW3+Rh6bF6eEODmo=', 'Content-Length': '0'} 2022-09-03T06:03:14.622 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004eee09fe9fed769f-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.623 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:03:14.623 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.624 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.625 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.625 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.626 DEBUG:boto:Method: GET 2022-09-03T06:03:14.626 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.627 DEBUG:boto:Data: 2022-09-03T06:03:14.627 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.628 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.628 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.629 DEBUG:boto:Params: {} 2022-09-03T06:03:14.630 DEBUG:boto:Token: None 2022-09-03T06:03:14.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y= 2022-09-03T06:03:14.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y=', 'Content-Length': '0'} 2022-09-03T06:03:14.634 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000b4e2feec72133af-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.635 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.636 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.637 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.637 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.638 DEBUG:boto:Method: GET 2022-09-03T06:03:14.639 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.639 DEBUG:boto:Data: 2022-09-03T06:03:14.640 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.641 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.642 DEBUG:boto:Params: {} 2022-09-03T06:03:14.643 DEBUG:boto:Token: None 2022-09-03T06:03:14.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y= 2022-09-03T06:03:14.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y=', 'Content-Length': '0'} 2022-09-03T06:03:14.647 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009282f3f1dcb1bbc7-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.648 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.649 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.650 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.650 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.651 DEBUG:boto:Method: GET 2022-09-03T06:03:14.651 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.652 DEBUG:boto:Data: 2022-09-03T06:03:14.652 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.653 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.653 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.654 DEBUG:boto:Params: {} 2022-09-03T06:03:14.655 DEBUG:boto:Token: None 2022-09-03T06:03:14.655 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.659 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000060f347a383fd0b73-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.660 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.661 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.662 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.662 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.663 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.664 DEBUG:boto:Method: GET 2022-09-03T06:03:14.664 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.665 DEBUG:boto:Data: 2022-09-03T06:03:14.665 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.666 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.667 DEBUG:boto:Params: {} 2022-09-03T06:03:14.667 DEBUG:boto:Token: None 2022-09-03T06:03:14.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.672 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009628a5263444d632-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.673 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.674 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.674 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.675 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.675 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T06:03:14.676 DEBUG:boto:Data: 2022-09-03T06:03:14.677 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.677 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.678 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.678 DEBUG:boto:Params: {} 2022-09-03T06:03:14.679 DEBUG:boto:Token: None 2022-09-03T06:03:14.679 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2 2022-09-03T06:03:14.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:szqyUlDmrZ8OPeAOtqmcFCPHuvI= 2022-09-03T06:03:14.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:szqyUlDmrZ8OPeAOtqmcFCPHuvI=', 'Content-Length': '0'} 2022-09-03T06:03:14.683 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000011f4198899a08f05-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.684 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.685 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.685 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.686 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T06:03:14.687 DEBUG:boto:Data: 2022-09-03T06:03:14.687 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.689 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.689 DEBUG:boto:Params: {} 2022-09-03T06:03:14.690 DEBUG:boto:Token: None 2022-09-03T06:03:14.691 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2 2022-09-03T06:03:14.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:szqyUlDmrZ8OPeAOtqmcFCPHuvI= 2022-09-03T06:03:14.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:szqyUlDmrZ8OPeAOtqmcFCPHuvI=', 'Content-Length': '0'} 2022-09-03T06:03:14.695 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000106d6e813f2e886f-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.696 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:03:14.697 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.697 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.698 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.699 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.699 DEBUG:boto:Method: GET 2022-09-03T06:03:14.700 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.701 DEBUG:boto:Data: 2022-09-03T06:03:14.701 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.702 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.703 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.703 DEBUG:boto:Params: {} 2022-09-03T06:03:14.704 DEBUG:boto:Token: None 2022-09-03T06:03:14.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.705 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y= 2022-09-03T06:03:14.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y=', 'Content-Length': '0'} 2022-09-03T06:03:14.708 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000deeac6d1939e63d9-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.750 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.751 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:14.752 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.752 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.753 DEBUG:boto:Method: GET 2022-09-03T06:03:14.753 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.754 DEBUG:boto:Data: 2022-09-03T06:03:14.754 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.755 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.756 DEBUG:boto:Params: {} 2022-09-03T06:03:14.757 DEBUG:boto:Token: None 2022-09-03T06:03:14.757 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:14.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y= 2022-09-03T06:03:14.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsztgiEMOVJUfkY7Tj2TqFwuv6Y=', 'Content-Length': '0'} 2022-09-03T06:03:14.761 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001622ab9523d3306c-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.762 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.762 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.763 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.763 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.764 DEBUG:boto:Method: GET 2022-09-03T06:03:14.764 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.765 DEBUG:boto:Data: 2022-09-03T06:03:14.766 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.766 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.767 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.767 DEBUG:boto:Params: {} 2022-09-03T06:03:14.768 DEBUG:boto:Token: None 2022-09-03T06:03:14.768 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.772 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058bf900e2e187152-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.773 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.774 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.775 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.775 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.776 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.777 DEBUG:boto:Method: GET 2022-09-03T06:03:14.777 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:14.778 DEBUG:boto:Data: 2022-09-03T06:03:14.778 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.779 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.780 DEBUG:boto:Params: {} 2022-09-03T06:03:14.781 DEBUG:boto:Token: None 2022-09-03T06:03:14.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.785 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bcd1effd8280e6a5-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.787 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.788 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:03:14.789 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.790 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.791 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.792 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.793 DEBUG:boto:Method: GET 2022-09-03T06:03:14.794 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.795 DEBUG:boto:Data: 2022-09-03T06:03:14.796 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.797 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.798 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.798 DEBUG:boto:Params: {} 2022-09-03T06:03:14.799 DEBUG:boto:Token: None 2022-09-03T06:03:14.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik= 2022-09-03T06:03:14.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik=', 'Content-Length': '0'} 2022-09-03T06:03:14.803 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000071b60e9e081cd08d-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.804 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.805 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.805 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.806 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.806 DEBUG:boto:Method: GET 2022-09-03T06:03:14.807 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.808 DEBUG:boto:Data: 2022-09-03T06:03:14.808 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.809 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.810 DEBUG:boto:Params: {} 2022-09-03T06:03:14.811 DEBUG:boto:Token: None 2022-09-03T06:03:14.812 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik= 2022-09-03T06:03:14.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik=', 'Content-Length': '0'} 2022-09-03T06:03:14.816 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000585aaca3f08f34a1-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.817 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.818 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.818 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.819 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.820 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.820 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.821 DEBUG:boto:Data: 2022-09-03T06:03:14.821 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.822 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.822 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.823 DEBUG:boto:Params: {} 2022-09-03T06:03:14.823 DEBUG:boto:Token: None 2022-09-03T06:03:14.824 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NCpwWQyZFgdcmYJX0cGVREMTZIY= 2022-09-03T06:03:14.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NCpwWQyZFgdcmYJX0cGVREMTZIY=', 'Content-Length': '0'} 2022-09-03T06:03:14.829 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000082d0784ffdd608d9-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.830 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.831 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.832 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.833 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.834 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.835 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.835 DEBUG:boto:Data: 2022-09-03T06:03:14.836 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.836 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.837 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.838 DEBUG:boto:Params: {} 2022-09-03T06:03:14.838 DEBUG:boto:Token: None 2022-09-03T06:03:14.839 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NCpwWQyZFgdcmYJX0cGVREMTZIY= 2022-09-03T06:03:14.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NCpwWQyZFgdcmYJX0cGVREMTZIY=', 'Content-Length': '0'} 2022-09-03T06:03:14.842 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000719eb88d10537e3-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.843 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:03:14.844 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.844 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.845 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.845 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.846 DEBUG:boto:Method: GET 2022-09-03T06:03:14.846 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.847 DEBUG:boto:Data: 2022-09-03T06:03:14.847 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.848 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.848 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.849 DEBUG:boto:Params: {} 2022-09-03T06:03:14.850 DEBUG:boto:Token: None 2022-09-03T06:03:14.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik= 2022-09-03T06:03:14.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik=', 'Content-Length': '0'} 2022-09-03T06:03:14.854 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000097fcbf4744f3489c-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.855 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.855 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.856 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.856 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.857 DEBUG:boto:Method: GET 2022-09-03T06:03:14.858 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.858 DEBUG:boto:Data: 2022-09-03T06:03:14.859 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.859 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.860 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.860 DEBUG:boto:Params: {} 2022-09-03T06:03:14.861 DEBUG:boto:Token: None 2022-09-03T06:03:14.862 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik= 2022-09-03T06:03:14.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik=', 'Content-Length': '0'} 2022-09-03T06:03:14.866 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000073cc1b136adc8d28-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.910 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.911 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.911 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.912 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.913 DEBUG:boto:Method: GET 2022-09-03T06:03:14.913 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.914 DEBUG:boto:Data: 2022-09-03T06:03:14.914 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.915 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.916 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.916 DEBUG:boto:Params: {} 2022-09-03T06:03:14.917 DEBUG:boto:Token: None 2022-09-03T06:03:14.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.921 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004000a1b8d2f17113-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.922 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.923 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.924 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.924 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.925 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.925 DEBUG:boto:Method: GET 2022-09-03T06:03:14.926 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.926 DEBUG:boto:Data: 2022-09-03T06:03:14.927 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.928 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.929 DEBUG:boto:Params: {} 2022-09-03T06:03:14.930 DEBUG:boto:Token: None 2022-09-03T06:03:14.931 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.933 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.936 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ecb6f7550d77c283-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.937 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.938 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.938 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.939 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.939 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T06:03:14.940 DEBUG:boto:Data: 2022-09-03T06:03:14.940 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.941 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.941 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.942 DEBUG:boto:Params: {} 2022-09-03T06:03:14.943 DEBUG:boto:Token: None 2022-09-03T06:03:14.943 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3 2022-09-03T06:03:14.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I2Ea/OL/MsFHcVtQkvyLJ+/6qoI= 2022-09-03T06:03:14.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I2Ea/OL/MsFHcVtQkvyLJ+/6qoI=', 'Content-Length': '0'} 2022-09-03T06:03:14.947 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000043a87d30c6c54d83-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.947 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.948 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.948 DEBUG:boto:Method: HEAD 2022-09-03T06:03:14.949 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T06:03:14.949 DEBUG:boto:Data: 2022-09-03T06:03:14.950 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.951 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.952 DEBUG:boto:Params: {} 2022-09-03T06:03:14.953 DEBUG:boto:Token: None 2022-09-03T06:03:14.953 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3 2022-09-03T06:03:14.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I2Ea/OL/MsFHcVtQkvyLJ+/6qoI= 2022-09-03T06:03:14.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I2Ea/OL/MsFHcVtQkvyLJ+/6qoI=', 'Content-Length': '0'} 2022-09-03T06:03:14.957 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000072486c9c1884f68f-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.958 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:03:14.958 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.959 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.959 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.960 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.961 DEBUG:boto:Method: GET 2022-09-03T06:03:14.961 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.962 DEBUG:boto:Data: 2022-09-03T06:03:14.962 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.963 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.963 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.964 DEBUG:boto:Params: {} 2022-09-03T06:03:14.964 DEBUG:boto:Token: None 2022-09-03T06:03:14.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik= 2022-09-03T06:03:14.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik=', 'Content-Length': '0'} 2022-09-03T06:03:14.968 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dfe683102db6ca6c-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.969 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.970 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:14.970 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.971 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.971 DEBUG:boto:Method: GET 2022-09-03T06:03:14.972 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.972 DEBUG:boto:Data: 2022-09-03T06:03:14.973 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.973 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.974 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.974 DEBUG:boto:Params: {} 2022-09-03T06:03:14.975 DEBUG:boto:Token: None 2022-09-03T06:03:14.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:14.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik= 2022-09-03T06:03:14.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YChIaGr3WazRTnMrC8kIuaGCaik=', 'Content-Length': '0'} 2022-09-03T06:03:14.979 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fed61a29b94ae5ac-006312ee22-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.980 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.981 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.981 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.982 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.982 DEBUG:boto:Method: GET 2022-09-03T06:03:14.983 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.983 DEBUG:boto:Data: 2022-09-03T06:03:14.984 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.984 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:14.985 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.985 DEBUG:boto:Params: {} 2022-09-03T06:03:14.986 DEBUG:boto:Token: None 2022-09-03T06:03:14.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:14 GMT /avysmw-18/ 2022-09-03T06:03:14.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ= 2022-09-03T06:03:14.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dp8olnKSfw4ILEoSmEvuSwPdXVQ=', 'Content-Length': '0'} 2022-09-03T06:03:14.990 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001ffb285ac4409dc2-006312ee22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:14.991 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:14.992 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:14.993 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:14.993 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.994 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.995 DEBUG:boto:Method: GET 2022-09-03T06:03:14.996 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:14.997 DEBUG:boto:Data: 2022-09-03T06:03:14.997 DEBUG:boto:Headers: {} 2022-09-03T06:03:14.998 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:14.998 DEBUG:boto:Port: 8000 2022-09-03T06:03:14.999 DEBUG:boto:Params: {} 2022-09-03T06:03:14.999 DEBUG:boto:Token: None 2022-09-03T06:03:15.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.004 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1c330070402deac-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.005 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.006 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-18 zones={test-zone2, test-zone1} 2022-09-03T06:03:15.007 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-18 zones={test-zone3, test-zone1} 2022-09-03T06:03:15.007 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.008 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.008 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.009 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T06:03:15.009 DEBUG:boto:Data: 2022-09-03T06:03:15.010 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.012 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.012 DEBUG:boto:Params: {} 2022-09-03T06:03:15.013 DEBUG:boto:Token: None 2022-09-03T06:03:15.014 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x0Op1EA5Z+b8TzUAr0yiB/sPRYs= 2022-09-03T06:03:15.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x0Op1EA5Z+b8TzUAr0yiB/sPRYs=', 'Content-Length': '0'} 2022-09-03T06:03:15.019 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000796fc98bce74ab46-006312ee23-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.020 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.020 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.021 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.021 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T06:03:15.022 DEBUG:boto:Data: 2022-09-03T06:03:15.023 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.024 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.025 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.025 DEBUG:boto:Params: {} 2022-09-03T06:03:15.026 DEBUG:boto:Token: None 2022-09-03T06:03:15.027 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x0Op1EA5Z+b8TzUAr0yiB/sPRYs= 2022-09-03T06:03:15.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x0Op1EA5Z+b8TzUAr0yiB/sPRYs=', 'Content-Length': '0'} 2022-09-03T06:03:15.032 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000086330f136b7bab55-006312ee23-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.033 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:03:15.033 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.034 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.035 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.035 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.036 DEBUG:boto:Method: GET 2022-09-03T06:03:15.036 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.037 DEBUG:boto:Data: 2022-09-03T06:03:15.037 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.039 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.040 DEBUG:boto:Params: {} 2022-09-03T06:03:15.040 DEBUG:boto:Token: None 2022-09-03T06:03:15.041 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/?versions 2022-09-03T06:03:15.042 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BXpSZS2HN26UL7CTW38R7aw+ajc= 2022-09-03T06:03:15.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BXpSZS2HN26UL7CTW38R7aw+ajc=', 'Content-Length': '0'} 2022-09-03T06:03:15.045 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8a3a2c3db9ac75c-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.046 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.047 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:03:15.048 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:03:15.048 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:03:15.049 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:03:15.049 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:03:15.050 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.051 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.052 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.053 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.054 DEBUG:boto:Method: GET 2022-09-03T06:03:15.055 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.056 DEBUG:boto:Data: 2022-09-03T06:03:15.057 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.058 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.058 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.059 DEBUG:boto:Params: {} 2022-09-03T06:03:15.059 DEBUG:boto:Token: None 2022-09-03T06:03:15.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/?versions 2022-09-03T06:03:15.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BXpSZS2HN26UL7CTW38R7aw+ajc= 2022-09-03T06:03:15.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BXpSZS2HN26UL7CTW38R7aw+ajc=', 'Content-Length': '0'} 2022-09-03T06:03:15.065 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007cb1dca70f554711-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.066 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.067 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:03:15.067 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:03:15.068 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:03:15.068 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:03:15.069 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.069 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.070 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.070 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.071 DEBUG:boto:Method: GET 2022-09-03T06:03:15.071 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.072 DEBUG:boto:Data: 2022-09-03T06:03:15.072 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.074 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.074 DEBUG:boto:Params: {} 2022-09-03T06:03:15.075 DEBUG:boto:Token: None 2022-09-03T06:03:15.076 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/?versions 2022-09-03T06:03:15.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BXpSZS2HN26UL7CTW38R7aw+ajc= 2022-09-03T06:03:15.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BXpSZS2HN26UL7CTW38R7aw+ajc=', 'Content-Length': '0'} 2022-09-03T06:03:15.080 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091985d180db091d2-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.081 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.082 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.083 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.083 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.084 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.084 DEBUG:boto:Method: GET 2022-09-03T06:03:15.085 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:15.085 DEBUG:boto:Data: 2022-09-03T06:03:15.086 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.087 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.087 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.088 DEBUG:boto:Params: {} 2022-09-03T06:03:15.089 DEBUG:boto:Token: None 2022-09-03T06:03:15.090 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/?versions 2022-09-03T06:03:15.091 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BXpSZS2HN26UL7CTW38R7aw+ajc= 2022-09-03T06:03:15.092 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BXpSZS2HN26UL7CTW38R7aw+ajc=', 'Content-Length': '0'} 2022-09-03T06:03:15.095 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004ba9b00fb426272f-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.096 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.097 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:03:15.099 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.100 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.101 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.102 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.103 DEBUG:boto:Method: GET 2022-09-03T06:03:15.104 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.105 DEBUG:boto:Data: 2022-09-03T06:03:15.106 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.107 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.108 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.109 DEBUG:boto:Params: {} 2022-09-03T06:03:15.109 DEBUG:boto:Token: None 2022-09-03T06:03:15.111 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM= 2022-09-03T06:03:15.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM=', 'Content-Length': '0'} 2022-09-03T06:03:15.114 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008fa83e651e921717-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.115 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.116 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.117 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.117 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.118 DEBUG:boto:Method: GET 2022-09-03T06:03:15.118 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.119 DEBUG:boto:Data: 2022-09-03T06:03:15.119 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.120 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.120 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.121 DEBUG:boto:Params: {} 2022-09-03T06:03:15.122 DEBUG:boto:Token: None 2022-09-03T06:03:15.122 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.123 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM= 2022-09-03T06:03:15.124 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM=', 'Content-Length': '0'} 2022-09-03T06:03:15.126 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000be054a09a26d54fe-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.127 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.128 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.128 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.129 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.129 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.130 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.130 DEBUG:boto:Data: 2022-09-03T06:03:15.131 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.131 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.132 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.133 DEBUG:boto:Params: {} 2022-09-03T06:03:15.133 DEBUG:boto:Token: None 2022-09-03T06:03:15.134 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YiboQcNxQQx8bbgHwqMSo9GGRME= 2022-09-03T06:03:15.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YiboQcNxQQx8bbgHwqMSo9GGRME=', 'Content-Length': '0'} 2022-09-03T06:03:15.137 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000079c353c626569005-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.138 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.139 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.139 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.140 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.140 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.141 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.141 DEBUG:boto:Data: 2022-09-03T06:03:15.142 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.143 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.144 DEBUG:boto:Params: {} 2022-09-03T06:03:15.144 DEBUG:boto:Token: None 2022-09-03T06:03:15.145 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YiboQcNxQQx8bbgHwqMSo9GGRME= 2022-09-03T06:03:15.146 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YiboQcNxQQx8bbgHwqMSo9GGRME=', 'Content-Length': '0'} 2022-09-03T06:03:15.148 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f5bf4f876f0321e0-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.149 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:03:15.150 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.150 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.151 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.152 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.153 DEBUG:boto:Method: GET 2022-09-03T06:03:15.154 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.155 DEBUG:boto:Data: 2022-09-03T06:03:15.156 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.157 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.158 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.159 DEBUG:boto:Params: {} 2022-09-03T06:03:15.160 DEBUG:boto:Token: None 2022-09-03T06:03:15.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM= 2022-09-03T06:03:15.163 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM=', 'Content-Length': '0'} 2022-09-03T06:03:15.166 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f96f2c9d7dfb5004-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.167 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.168 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.169 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.170 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.170 DEBUG:boto:Method: GET 2022-09-03T06:03:15.171 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.171 DEBUG:boto:Data: 2022-09-03T06:03:15.172 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.173 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.173 DEBUG:boto:Params: {} 2022-09-03T06:03:15.174 DEBUG:boto:Token: None 2022-09-03T06:03:15.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM= 2022-09-03T06:03:15.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM=', 'Content-Length': '0'} 2022-09-03T06:03:15.178 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b585ff188a480df7-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.179 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.180 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.181 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:15.181 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:15.182 DEBUG:boto:Method: GET 2022-09-03T06:03:15.182 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:15.183 DEBUG:boto:Data: 2022-09-03T06:03:15.183 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.184 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.185 DEBUG:boto:Params: {} 2022-09-03T06:03:15.185 DEBUG:boto:Token: None 2022-09-03T06:03:15.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.190 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000030a9da6c7bdf5e13-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.234 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.235 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.236 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.236 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:15.237 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:15.237 DEBUG:boto:Method: GET 2022-09-03T06:03:15.238 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:15.238 DEBUG:boto:Data: 2022-09-03T06:03:15.239 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.240 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.240 DEBUG:boto:Params: {} 2022-09-03T06:03:15.241 DEBUG:boto:Token: None 2022-09-03T06:03:15.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.246 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bfc650d76ab476b1-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.246 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.247 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.248 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.249 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.249 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T06:03:15.250 DEBUG:boto:Data: 2022-09-03T06:03:15.250 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.251 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.251 DEBUG:boto:Params: {} 2022-09-03T06:03:15.252 DEBUG:boto:Token: None 2022-09-03T06:03:15.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy 2022-09-03T06:03:15.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5lNsuF/R1jquInjcyWGaoQLnFkg= 2022-09-03T06:03:15.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5lNsuF/R1jquInjcyWGaoQLnFkg=', 'Content-Length': '0'} 2022-09-03T06:03:15.256 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f39371605ba6f360-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.257 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.258 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.258 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.259 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T06:03:15.259 DEBUG:boto:Data: 2022-09-03T06:03:15.260 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.261 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.261 DEBUG:boto:Params: {} 2022-09-03T06:03:15.262 DEBUG:boto:Token: None 2022-09-03T06:03:15.262 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy 2022-09-03T06:03:15.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5lNsuF/R1jquInjcyWGaoQLnFkg= 2022-09-03T06:03:15.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5lNsuF/R1jquInjcyWGaoQLnFkg=', 'Content-Length': '0'} 2022-09-03T06:03:15.266 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000fbe0351189fb200d-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.267 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:03:15.267 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.268 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.269 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.269 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.270 DEBUG:boto:Method: GET 2022-09-03T06:03:15.271 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.271 DEBUG:boto:Data: 2022-09-03T06:03:15.272 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.272 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.273 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.274 DEBUG:boto:Params: {} 2022-09-03T06:03:15.275 DEBUG:boto:Token: None 2022-09-03T06:03:15.276 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM= 2022-09-03T06:03:15.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM=', 'Content-Length': '0'} 2022-09-03T06:03:15.279 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000fe10625253107a16-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.280 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:15.281 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:15.282 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.283 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.284 DEBUG:boto:Method: GET 2022-09-03T06:03:15.285 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.286 DEBUG:boto:Data: 2022-09-03T06:03:15.287 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.289 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.290 DEBUG:boto:Params: {} 2022-09-03T06:03:15.291 DEBUG:boto:Token: None 2022-09-03T06:03:15.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:15.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM= 2022-09-03T06:03:15.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MDRFctHJLSCohmAtplgCOhE1nbM=', 'Content-Length': '0'} 2022-09-03T06:03:15.295 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d3ae4e3fe270c81e-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.296 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.297 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.297 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:15.298 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:15.299 DEBUG:boto:Method: GET 2022-09-03T06:03:15.299 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:15.300 DEBUG:boto:Data: 2022-09-03T06:03:15.300 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.301 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.301 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.302 DEBUG:boto:Params: {} 2022-09-03T06:03:15.302 DEBUG:boto:Token: None 2022-09-03T06:03:15.303 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.307 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000615d6fac7528f7d4-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.308 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.309 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.309 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.310 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:15.310 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:15.311 DEBUG:boto:Method: GET 2022-09-03T06:03:15.312 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:15.312 DEBUG:boto:Data: 2022-09-03T06:03:15.313 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.314 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.315 DEBUG:boto:Params: {} 2022-09-03T06:03:15.315 DEBUG:boto:Token: None 2022-09-03T06:03:15.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.320 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f8c2bef910087046-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.320 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.321 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:03:15.322 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.322 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.323 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.324 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.325 DEBUG:boto:Method: GET 2022-09-03T06:03:15.325 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.326 DEBUG:boto:Data: 2022-09-03T06:03:15.327 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.328 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.329 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.329 DEBUG:boto:Params: {} 2022-09-03T06:03:15.330 DEBUG:boto:Token: None 2022-09-03T06:03:15.331 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc= 2022-09-03T06:03:15.332 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc=', 'Content-Length': '0'} 2022-09-03T06:03:15.334 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bcc8bd9c4b10b392-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.335 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.336 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.336 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.337 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.337 DEBUG:boto:Method: GET 2022-09-03T06:03:15.338 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.338 DEBUG:boto:Data: 2022-09-03T06:03:15.339 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.340 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.340 DEBUG:boto:Params: {} 2022-09-03T06:03:15.341 DEBUG:boto:Token: None 2022-09-03T06:03:15.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc= 2022-09-03T06:03:15.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc=', 'Content-Length': '0'} 2022-09-03T06:03:15.345 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000e5f441fe76661a4-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.346 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.346 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.347 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.347 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.348 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.348 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.349 DEBUG:boto:Data: 2022-09-03T06:03:15.349 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.350 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.350 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.351 DEBUG:boto:Params: {} 2022-09-03T06:03:15.351 DEBUG:boto:Token: None 2022-09-03T06:03:15.352 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScfUku8t1d9P0ClDBvOX7OtVZ4Y= 2022-09-03T06:03:15.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScfUku8t1d9P0ClDBvOX7OtVZ4Y=', 'Content-Length': '0'} 2022-09-03T06:03:15.355 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000962dc4fa5509857f-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.356 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.356 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.357 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.357 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.358 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.358 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.359 DEBUG:boto:Data: 2022-09-03T06:03:15.359 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.361 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.361 DEBUG:boto:Params: {} 2022-09-03T06:03:15.362 DEBUG:boto:Token: None 2022-09-03T06:03:15.362 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScfUku8t1d9P0ClDBvOX7OtVZ4Y= 2022-09-03T06:03:15.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScfUku8t1d9P0ClDBvOX7OtVZ4Y=', 'Content-Length': '0'} 2022-09-03T06:03:15.365 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c48ac7155400a528-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.366 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:03:15.367 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.367 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.368 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.368 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.369 DEBUG:boto:Method: GET 2022-09-03T06:03:15.370 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.371 DEBUG:boto:Data: 2022-09-03T06:03:15.371 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.372 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.372 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.373 DEBUG:boto:Params: {} 2022-09-03T06:03:15.374 DEBUG:boto:Token: None 2022-09-03T06:03:15.375 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc= 2022-09-03T06:03:15.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc=', 'Content-Length': '0'} 2022-09-03T06:03:15.379 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ed7dd9ca8cd759c6-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.422 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.423 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.424 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.425 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.427 DEBUG:boto:Method: GET 2022-09-03T06:03:15.427 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.428 DEBUG:boto:Data: 2022-09-03T06:03:15.429 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.430 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.430 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.431 DEBUG:boto:Params: {} 2022-09-03T06:03:15.431 DEBUG:boto:Token: None 2022-09-03T06:03:15.432 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc= 2022-09-03T06:03:15.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc=', 'Content-Length': '0'} 2022-09-03T06:03:15.436 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c676d4dfad4b432a-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.437 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.437 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.438 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.439 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.440 DEBUG:boto:Method: GET 2022-09-03T06:03:15.441 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.441 DEBUG:boto:Data: 2022-09-03T06:03:15.442 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.444 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.445 DEBUG:boto:Params: {} 2022-09-03T06:03:15.446 DEBUG:boto:Token: None 2022-09-03T06:03:15.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.452 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd1cc0af80ece41c-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.452 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.453 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.454 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.454 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.455 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.455 DEBUG:boto:Method: GET 2022-09-03T06:03:15.456 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.456 DEBUG:boto:Data: 2022-09-03T06:03:15.457 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.457 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.458 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.458 DEBUG:boto:Params: {} 2022-09-03T06:03:15.459 DEBUG:boto:Token: None 2022-09-03T06:03:15.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.460 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.461 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.464 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002befba21df8be431-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.464 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.465 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.466 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.466 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.467 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T06:03:15.467 DEBUG:boto:Data: 2022-09-03T06:03:15.468 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.468 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.469 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.469 DEBUG:boto:Params: {} 2022-09-03T06:03:15.470 DEBUG:boto:Token: None 2022-09-03T06:03:15.470 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1 2022-09-03T06:03:15.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vvPycICE6mxuYRt2T8weL+bPdOw= 2022-09-03T06:03:15.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vvPycICE6mxuYRt2T8weL+bPdOw=', 'Content-Length': '0'} 2022-09-03T06:03:15.474 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000a7a1ac803006235-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.475 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.475 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.476 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.476 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T06:03:15.477 DEBUG:boto:Data: 2022-09-03T06:03:15.477 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.478 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.479 DEBUG:boto:Params: {} 2022-09-03T06:03:15.479 DEBUG:boto:Token: None 2022-09-03T06:03:15.480 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1 2022-09-03T06:03:15.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vvPycICE6mxuYRt2T8weL+bPdOw= 2022-09-03T06:03:15.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vvPycICE6mxuYRt2T8weL+bPdOw=', 'Content-Length': '0'} 2022-09-03T06:03:15.484 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e6a8540f4fafb810-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.484 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:03:15.485 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.485 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.486 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.486 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.487 DEBUG:boto:Method: GET 2022-09-03T06:03:15.487 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.488 DEBUG:boto:Data: 2022-09-03T06:03:15.489 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.490 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.490 DEBUG:boto:Params: {} 2022-09-03T06:03:15.491 DEBUG:boto:Token: None 2022-09-03T06:03:15.491 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc= 2022-09-03T06:03:15.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc=', 'Content-Length': '0'} 2022-09-03T06:03:15.495 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000003b3f18ea6a00423-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.496 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.496 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:15.497 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.498 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.498 DEBUG:boto:Method: GET 2022-09-03T06:03:15.499 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.499 DEBUG:boto:Data: 2022-09-03T06:03:15.500 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.501 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.501 DEBUG:boto:Params: {} 2022-09-03T06:03:15.502 DEBUG:boto:Token: None 2022-09-03T06:03:15.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:15.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc= 2022-09-03T06:03:15.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tNrOx9WCiPX93BfU4dVUShfIvvc=', 'Content-Length': '0'} 2022-09-03T06:03:15.506 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000042eefddba4ff477-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.507 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.507 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.508 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.508 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.509 DEBUG:boto:Method: GET 2022-09-03T06:03:15.509 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.510 DEBUG:boto:Data: 2022-09-03T06:03:15.511 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.511 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.512 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.512 DEBUG:boto:Params: {} 2022-09-03T06:03:15.513 DEBUG:boto:Token: None 2022-09-03T06:03:15.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.518 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f0536ade27d9c3c-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.562 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.563 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.564 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.564 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.565 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.565 DEBUG:boto:Method: GET 2022-09-03T06:03:15.566 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:15.567 DEBUG:boto:Data: 2022-09-03T06:03:15.567 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.569 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.569 DEBUG:boto:Params: {} 2022-09-03T06:03:15.570 DEBUG:boto:Token: None 2022-09-03T06:03:15.571 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.575 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3e25829b28d6dc9-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.576 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.577 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:03:15.577 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.578 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.578 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.579 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.580 DEBUG:boto:Method: GET 2022-09-03T06:03:15.581 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.582 DEBUG:boto:Data: 2022-09-03T06:03:15.583 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.584 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.585 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.585 DEBUG:boto:Params: {} 2022-09-03T06:03:15.587 DEBUG:boto:Token: None 2022-09-03T06:03:15.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4= 2022-09-03T06:03:15.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4=', 'Content-Length': '0'} 2022-09-03T06:03:15.592 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009cbcc55a6790c7f9-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.594 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.595 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.596 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.596 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.597 DEBUG:boto:Method: GET 2022-09-03T06:03:15.598 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.598 DEBUG:boto:Data: 2022-09-03T06:03:15.599 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.601 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.602 DEBUG:boto:Params: {} 2022-09-03T06:03:15.603 DEBUG:boto:Token: None 2022-09-03T06:03:15.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4= 2022-09-03T06:03:15.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4=', 'Content-Length': '0'} 2022-09-03T06:03:15.609 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000238db60a4b1e64a7-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.654 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.655 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.656 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.657 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.657 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.658 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.659 DEBUG:boto:Data: 2022-09-03T06:03:15.660 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.661 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.662 DEBUG:boto:Params: {} 2022-09-03T06:03:15.662 DEBUG:boto:Token: None 2022-09-03T06:03:15.663 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kpyrcNtR9zjt2upuFmqnRWyv4Yc= 2022-09-03T06:03:15.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kpyrcNtR9zjt2upuFmqnRWyv4Yc=', 'Content-Length': '0'} 2022-09-03T06:03:15.666 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000074a6119032ffc52a-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.667 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.667 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.668 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.669 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.669 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.670 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.670 DEBUG:boto:Data: 2022-09-03T06:03:15.671 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.672 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.673 DEBUG:boto:Params: {} 2022-09-03T06:03:15.673 DEBUG:boto:Token: None 2022-09-03T06:03:15.674 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.674 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kpyrcNtR9zjt2upuFmqnRWyv4Yc= 2022-09-03T06:03:15.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kpyrcNtR9zjt2upuFmqnRWyv4Yc=', 'Content-Length': '0'} 2022-09-03T06:03:15.677 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b319b23beba4fe3e-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.678 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:03:15.678 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.679 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.679 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.680 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.681 DEBUG:boto:Method: GET 2022-09-03T06:03:15.682 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.682 DEBUG:boto:Data: 2022-09-03T06:03:15.683 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.684 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.684 DEBUG:boto:Params: {} 2022-09-03T06:03:15.685 DEBUG:boto:Token: None 2022-09-03T06:03:15.686 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4= 2022-09-03T06:03:15.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4=', 'Content-Length': '0'} 2022-09-03T06:03:15.689 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000026573461350371a9-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.690 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.691 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.692 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.692 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.693 DEBUG:boto:Method: GET 2022-09-03T06:03:15.694 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.694 DEBUG:boto:Data: 2022-09-03T06:03:15.695 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.696 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.696 DEBUG:boto:Params: {} 2022-09-03T06:03:15.697 DEBUG:boto:Token: None 2022-09-03T06:03:15.697 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4= 2022-09-03T06:03:15.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4=', 'Content-Length': '0'} 2022-09-03T06:03:15.701 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002a0a6da3fd1f4bd3-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.702 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.703 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.704 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.704 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.705 DEBUG:boto:Method: GET 2022-09-03T06:03:15.705 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.706 DEBUG:boto:Data: 2022-09-03T06:03:15.707 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.708 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.709 DEBUG:boto:Params: {} 2022-09-03T06:03:15.709 DEBUG:boto:Token: None 2022-09-03T06:03:15.710 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.712 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.714 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007590a35a7f68487e-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.715 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.716 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.716 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.717 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.718 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.719 DEBUG:boto:Method: GET 2022-09-03T06:03:15.720 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.721 DEBUG:boto:Data: 2022-09-03T06:03:15.722 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.723 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.724 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.725 DEBUG:boto:Params: {} 2022-09-03T06:03:15.726 DEBUG:boto:Token: None 2022-09-03T06:03:15.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.732 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b36c3ebe418d0f8-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.734 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.735 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.736 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.737 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.737 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T06:03:15.738 DEBUG:boto:Data: 2022-09-03T06:03:15.739 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.740 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.741 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.742 DEBUG:boto:Params: {} 2022-09-03T06:03:15.743 DEBUG:boto:Token: None 2022-09-03T06:03:15.743 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2 2022-09-03T06:03:15.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tmz+3tCvndhe0BBein1Q5PxG4D8= 2022-09-03T06:03:15.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tmz+3tCvndhe0BBein1Q5PxG4D8=', 'Content-Length': '0'} 2022-09-03T06:03:15.748 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000088dfc9c214f0f004-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.750 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.750 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.751 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.751 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T06:03:15.752 DEBUG:boto:Data: 2022-09-03T06:03:15.752 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.753 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.753 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.754 DEBUG:boto:Params: {} 2022-09-03T06:03:15.754 DEBUG:boto:Token: None 2022-09-03T06:03:15.755 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2 2022-09-03T06:03:15.755 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tmz+3tCvndhe0BBein1Q5PxG4D8= 2022-09-03T06:03:15.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tmz+3tCvndhe0BBein1Q5PxG4D8=', 'Content-Length': '0'} 2022-09-03T06:03:15.759 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cbde7caa481e68be-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.760 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:03:15.761 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.762 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.763 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.764 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.765 DEBUG:boto:Method: GET 2022-09-03T06:03:15.766 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.767 DEBUG:boto:Data: 2022-09-03T06:03:15.768 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.769 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.770 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.770 DEBUG:boto:Params: {} 2022-09-03T06:03:15.771 DEBUG:boto:Token: None 2022-09-03T06:03:15.772 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4= 2022-09-03T06:03:15.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4=', 'Content-Length': '0'} 2022-09-03T06:03:15.775 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005ae1f207e42b7221-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.822 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.823 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:15.823 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.824 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.824 DEBUG:boto:Method: GET 2022-09-03T06:03:15.825 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.825 DEBUG:boto:Data: 2022-09-03T06:03:15.826 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.827 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.827 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.828 DEBUG:boto:Params: {} 2022-09-03T06:03:15.828 DEBUG:boto:Token: None 2022-09-03T06:03:15.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:15.830 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4= 2022-09-03T06:03:15.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rRKNc2LUQOIeYu5i42GYghYurw4=', 'Content-Length': '0'} 2022-09-03T06:03:15.833 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000556cecd3eeae4f80-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.834 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.834 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.835 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.835 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.836 DEBUG:boto:Method: GET 2022-09-03T06:03:15.836 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.837 DEBUG:boto:Data: 2022-09-03T06:03:15.838 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.838 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.839 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.839 DEBUG:boto:Params: {} 2022-09-03T06:03:15.840 DEBUG:boto:Token: None 2022-09-03T06:03:15.840 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.844 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad7c228c2b925f9b-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.845 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.846 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.846 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.847 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.847 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.848 DEBUG:boto:Method: GET 2022-09-03T06:03:15.848 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:15.849 DEBUG:boto:Data: 2022-09-03T06:03:15.849 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.851 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.851 DEBUG:boto:Params: {} 2022-09-03T06:03:15.852 DEBUG:boto:Token: None 2022-09-03T06:03:15.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.856 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000029986c42857600af-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.857 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.858 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:03:15.858 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.859 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.860 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.860 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.861 DEBUG:boto:Method: GET 2022-09-03T06:03:15.861 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.862 DEBUG:boto:Data: 2022-09-03T06:03:15.862 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.863 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.863 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.864 DEBUG:boto:Params: {} 2022-09-03T06:03:15.865 DEBUG:boto:Token: None 2022-09-03T06:03:15.865 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac= 2022-09-03T06:03:15.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac=', 'Content-Length': '0'} 2022-09-03T06:03:15.869 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000080b2e1ee3d546859-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.870 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.870 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.871 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.872 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.872 DEBUG:boto:Method: GET 2022-09-03T06:03:15.873 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.874 DEBUG:boto:Data: 2022-09-03T06:03:15.874 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.876 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.876 DEBUG:boto:Params: {} 2022-09-03T06:03:15.877 DEBUG:boto:Token: None 2022-09-03T06:03:15.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac= 2022-09-03T06:03:15.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac=', 'Content-Length': '0'} 2022-09-03T06:03:15.881 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007c5331b8827f9099-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.883 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.883 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.884 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.885 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.886 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.886 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.887 DEBUG:boto:Data: 2022-09-03T06:03:15.888 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.889 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.890 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.890 DEBUG:boto:Params: {} 2022-09-03T06:03:15.891 DEBUG:boto:Token: None 2022-09-03T06:03:15.892 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.892 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GHWP6JicMmUIRIOpUV8hyzyc4b0= 2022-09-03T06:03:15.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GHWP6JicMmUIRIOpUV8hyzyc4b0=', 'Content-Length': '0'} 2022-09-03T06:03:15.895 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e64363830b6a8b31-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.896 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.897 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.897 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.898 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.899 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.899 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.900 DEBUG:boto:Data: 2022-09-03T06:03:15.900 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.901 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.902 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.902 DEBUG:boto:Params: {} 2022-09-03T06:03:15.903 DEBUG:boto:Token: None 2022-09-03T06:03:15.904 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GHWP6JicMmUIRIOpUV8hyzyc4b0= 2022-09-03T06:03:15.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GHWP6JicMmUIRIOpUV8hyzyc4b0=', 'Content-Length': '0'} 2022-09-03T06:03:15.907 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fdc24e82c85b914b-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.908 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:03:15.909 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.910 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.910 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.911 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.912 DEBUG:boto:Method: GET 2022-09-03T06:03:15.912 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.913 DEBUG:boto:Data: 2022-09-03T06:03:15.913 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.914 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.915 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.915 DEBUG:boto:Params: {} 2022-09-03T06:03:15.916 DEBUG:boto:Token: None 2022-09-03T06:03:15.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.917 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac= 2022-09-03T06:03:15.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac=', 'Content-Length': '0'} 2022-09-03T06:03:15.920 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006a104294fac7331b-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.921 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.922 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.922 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.923 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.923 DEBUG:boto:Method: GET 2022-09-03T06:03:15.924 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.924 DEBUG:boto:Data: 2022-09-03T06:03:15.925 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.925 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.926 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.927 DEBUG:boto:Params: {} 2022-09-03T06:03:15.927 DEBUG:boto:Token: None 2022-09-03T06:03:15.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac= 2022-09-03T06:03:15.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac=', 'Content-Length': '0'} 2022-09-03T06:03:15.932 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001ea7db8c41debd35-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.934 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.935 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.936 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:15.937 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:15.938 DEBUG:boto:Method: GET 2022-09-03T06:03:15.939 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:15.940 DEBUG:boto:Data: 2022-09-03T06:03:15.941 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.942 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.943 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.944 DEBUG:boto:Params: {} 2022-09-03T06:03:15.945 DEBUG:boto:Token: None 2022-09-03T06:03:15.946 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.951 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b51056b9331cc509-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.952 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.953 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:15.954 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:15.954 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:15.955 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:15.955 DEBUG:boto:Method: GET 2022-09-03T06:03:15.956 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:15.956 DEBUG:boto:Data: 2022-09-03T06:03:15.957 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.957 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.958 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.958 DEBUG:boto:Params: {} 2022-09-03T06:03:15.959 DEBUG:boto:Token: None 2022-09-03T06:03:15.960 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/ 2022-09-03T06:03:15.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg= 2022-09-03T06:03:15.961 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVGbaOyGClp6TckNlGAGeq6RhCg=', 'Content-Length': '0'} 2022-09-03T06:03:15.963 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a778e5f3c6a0cbdb-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.964 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:15.965 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.966 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.966 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.967 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T06:03:15.967 DEBUG:boto:Data: 2022-09-03T06:03:15.968 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.968 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.969 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.969 DEBUG:boto:Params: {} 2022-09-03T06:03:15.970 DEBUG:boto:Token: None 2022-09-03T06:03:15.971 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone3 2022-09-03T06:03:15.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S2Ey1VG0TS0mF6kzOs9DuPmleYE= 2022-09-03T06:03:15.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S2Ey1VG0TS0mF6kzOs9DuPmleYE=', 'Content-Length': '0'} 2022-09-03T06:03:15.974 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b201c7eb5dd4105a-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.975 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.975 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.976 DEBUG:boto:Method: HEAD 2022-09-03T06:03:15.977 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T06:03:15.978 DEBUG:boto:Data: 2022-09-03T06:03:15.979 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:15.980 DEBUG:boto:Port: 8000 2022-09-03T06:03:15.981 DEBUG:boto:Params: {} 2022-09-03T06:03:15.981 DEBUG:boto:Token: None 2022-09-03T06:03:15.982 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone3 2022-09-03T06:03:15.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S2Ey1VG0TS0mF6kzOs9DuPmleYE= 2022-09-03T06:03:15.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S2Ey1VG0TS0mF6kzOs9DuPmleYE=', 'Content-Length': '0'} 2022-09-03T06:03:15.987 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d24ef168b05da9ea-006312ee23-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:15.988 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:03:15.989 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.990 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:15.991 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.992 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.993 DEBUG:boto:Method: GET 2022-09-03T06:03:15.994 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.994 DEBUG:boto:Data: 2022-09-03T06:03:15.995 DEBUG:boto:Headers: {} 2022-09-03T06:03:15.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:15.996 DEBUG:boto:Port: 8001 2022-09-03T06:03:15.997 DEBUG:boto:Params: {} 2022-09-03T06:03:15.997 DEBUG:boto:Token: None 2022-09-03T06:03:15.998 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:15 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:15.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac= 2022-09-03T06:03:15.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+FcQiH/D0RTZi+pYjY2TBZ4ktac=', 'Content-Length': '0'} 2022-09-03T06:03:16.001 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fbd77fe012beacdb-006312ee23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.046 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.048 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.049 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.050 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.052 DEBUG:boto:Method: GET 2022-09-03T06:03:16.053 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.053 DEBUG:boto:Data: 2022-09-03T06:03:16.054 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.055 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:16.055 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.056 DEBUG:boto:Params: {} 2022-09-03T06:03:16.056 DEBUG:boto:Token: None 2022-09-03T06:03:16.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cF9AHlQ+MBVJlt7MIJ5skmBKRRo= 2022-09-03T06:03:16.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cF9AHlQ+MBVJlt7MIJ5skmBKRRo=', 'Content-Length': '0'} 2022-09-03T06:03:16.061 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000041a749c1951aa05f-006312ee24-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.062 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.062 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.063 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:16.063 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:16.064 DEBUG:boto:Method: GET 2022-09-03T06:03:16.064 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:16.065 DEBUG:boto:Data: 2022-09-03T06:03:16.065 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.066 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.066 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.067 DEBUG:boto:Params: {} 2022-09-03T06:03:16.068 DEBUG:boto:Token: None 2022-09-03T06:03:16.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.072 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d645483a64003caf-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.073 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.074 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.074 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.075 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:16.075 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:16.076 DEBUG:boto:Method: GET 2022-09-03T06:03:16.076 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:16.077 DEBUG:boto:Data: 2022-09-03T06:03:16.077 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.078 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:16.078 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.079 DEBUG:boto:Params: {} 2022-09-03T06:03:16.079 DEBUG:boto:Token: None 2022-09-03T06:03:16.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c37b0f987a922cd-006312ee24-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.085 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.086 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-18 zones={test-zone3, test-zone1} 2022-09-03T06:03:16.087 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-18 zones={test-zone3, test-zone2} 2022-09-03T06:03:16.087 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.088 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.088 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.089 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T06:03:16.089 DEBUG:boto:Data: 2022-09-03T06:03:16.090 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.090 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.091 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.091 DEBUG:boto:Params: {} 2022-09-03T06:03:16.092 DEBUG:boto:Token: None 2022-09-03T06:03:16.092 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ePfqhJQu+os3u0XwBqGsN6hixm4= 2022-09-03T06:03:16.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ePfqhJQu+os3u0XwBqGsN6hixm4=', 'Content-Length': '0'} 2022-09-03T06:03:16.097 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009366ed914754834c-006312ee24-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.098 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.098 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.099 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.100 DEBUG:boto:Path: /avysmw-18/ 2022-09-03T06:03:16.101 DEBUG:boto:Data: 2022-09-03T06:03:16.101 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.102 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.102 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.103 DEBUG:boto:Params: {} 2022-09-03T06:03:16.103 DEBUG:boto:Token: None 2022-09-03T06:03:16.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ePfqhJQu+os3u0XwBqGsN6hixm4= 2022-09-03T06:03:16.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ePfqhJQu+os3u0XwBqGsN6hixm4=', 'Content-Length': '0'} 2022-09-03T06:03:16.108 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c075eb1cf968e1a3-006312ee24-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.109 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:03:16.110 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.111 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.111 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.112 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.112 DEBUG:boto:Method: GET 2022-09-03T06:03:16.113 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.113 DEBUG:boto:Data: 2022-09-03T06:03:16.114 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.115 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.116 DEBUG:boto:Params: {} 2022-09-03T06:03:16.116 DEBUG:boto:Token: None 2022-09-03T06:03:16.117 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/?versions 2022-09-03T06:03:16.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2XiXgeMjGxNI2Jrd48TvFNtGd1Q= 2022-09-03T06:03:16.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2XiXgeMjGxNI2Jrd48TvFNtGd1Q=', 'Content-Length': '0'} 2022-09-03T06:03:16.122 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c3a8e1edfa7171fb-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.122 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.123 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:03:16.124 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:03:16.124 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:03:16.125 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:03:16.125 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:03:16.126 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.127 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.127 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.128 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.128 DEBUG:boto:Method: GET 2022-09-03T06:03:16.129 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.130 DEBUG:boto:Data: 2022-09-03T06:03:16.130 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.131 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.131 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.132 DEBUG:boto:Params: {} 2022-09-03T06:03:16.132 DEBUG:boto:Token: None 2022-09-03T06:03:16.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/?versions 2022-09-03T06:03:16.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2XiXgeMjGxNI2Jrd48TvFNtGd1Q= 2022-09-03T06:03:16.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2XiXgeMjGxNI2Jrd48TvFNtGd1Q=', 'Content-Length': '0'} 2022-09-03T06:03:16.137 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000021ccc16d8c2c5d1-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.138 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.139 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:03:16.140 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2022-09-03T06:03:16.140 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2022-09-03T06:03:16.141 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2022-09-03T06:03:16.141 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.142 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.142 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.143 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.143 DEBUG:boto:Method: GET 2022-09-03T06:03:16.144 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.144 DEBUG:boto:Data: 2022-09-03T06:03:16.145 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.146 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.146 DEBUG:boto:Params: {} 2022-09-03T06:03:16.147 DEBUG:boto:Token: None 2022-09-03T06:03:16.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/?versions 2022-09-03T06:03:16.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2XiXgeMjGxNI2Jrd48TvFNtGd1Q= 2022-09-03T06:03:16.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2XiXgeMjGxNI2Jrd48TvFNtGd1Q=', 'Content-Length': '0'} 2022-09-03T06:03:16.151 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000097905320aa44518c-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.152 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.153 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.154 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.154 DEBUG:boto:path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.155 DEBUG:boto:auth_path=/avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.155 DEBUG:boto:Method: GET 2022-09-03T06:03:16.156 DEBUG:boto:Path: /avysmw-18/?versions&max-keys=999 2022-09-03T06:03:16.156 DEBUG:boto:Data: 2022-09-03T06:03:16.157 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.157 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.158 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.158 DEBUG:boto:Params: {} 2022-09-03T06:03:16.159 DEBUG:boto:Token: None 2022-09-03T06:03:16.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/?versions 2022-09-03T06:03:16.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2XiXgeMjGxNI2Jrd48TvFNtGd1Q= 2022-09-03T06:03:16.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2XiXgeMjGxNI2Jrd48TvFNtGd1Q=', 'Content-Length': '0'} 2022-09-03T06:03:16.163 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c05c7b48edd51241-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.164 DEBUG:boto:b'avysmw-18999falsedummyrMCJTRbcZDXqkRiOSNNL3qB-91UFM6.true2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone1nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uutrue2022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone2BawwORIX6QOqyuiFo4WB5lBEtfSnFE2true2022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj-test-zone3CfuuomqFo.i4eJFtRFZjZAHoEOAdulvtrue2022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.165 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:03:16.166 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.166 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.167 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.167 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.168 DEBUG:boto:Method: GET 2022-09-03T06:03:16.169 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.170 DEBUG:boto:Data: 2022-09-03T06:03:16.171 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.172 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.173 DEBUG:boto:Params: {} 2022-09-03T06:03:16.175 DEBUG:boto:Token: None 2022-09-03T06:03:16.176 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8= 2022-09-03T06:03:16.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8=', 'Content-Length': '0'} 2022-09-03T06:03:16.181 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000995b649598e5fc8f-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.181 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.182 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.183 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.183 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.184 DEBUG:boto:Method: GET 2022-09-03T06:03:16.184 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.185 DEBUG:boto:Data: 2022-09-03T06:03:16.185 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.186 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.186 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.187 DEBUG:boto:Params: {} 2022-09-03T06:03:16.188 DEBUG:boto:Token: None 2022-09-03T06:03:16.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8= 2022-09-03T06:03:16.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8=', 'Content-Length': '0'} 2022-09-03T06:03:16.192 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000052a34c6bf350b694-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.192 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.193 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.193 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.194 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.195 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.195 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.196 DEBUG:boto:Data: 2022-09-03T06:03:16.196 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.197 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.198 DEBUG:boto:Params: {} 2022-09-03T06:03:16.198 DEBUG:boto:Token: None 2022-09-03T06:03:16.199 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1+s7IpKypOgHyUmkcoTqS22LOmo= 2022-09-03T06:03:16.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1+s7IpKypOgHyUmkcoTqS22LOmo=', 'Content-Length': '0'} 2022-09-03T06:03:16.202 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a4f7d301809f7956-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.203 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.204 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.205 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.206 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.207 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.207 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.208 DEBUG:boto:Data: 2022-09-03T06:03:16.209 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.210 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.210 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.211 DEBUG:boto:Params: {} 2022-09-03T06:03:16.212 DEBUG:boto:Token: None 2022-09-03T06:03:16.212 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1+s7IpKypOgHyUmkcoTqS22LOmo= 2022-09-03T06:03:16.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1+s7IpKypOgHyUmkcoTqS22LOmo=', 'Content-Length': '0'} 2022-09-03T06:03:16.216 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004c23c3194f2d84eb-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:03:16.217 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.217 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.218 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.219 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.219 DEBUG:boto:Method: GET 2022-09-03T06:03:16.220 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.221 DEBUG:boto:Data: 2022-09-03T06:03:16.221 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.222 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.222 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.223 DEBUG:boto:Params: {} 2022-09-03T06:03:16.224 DEBUG:boto:Token: None 2022-09-03T06:03:16.224 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8= 2022-09-03T06:03:16.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8=', 'Content-Length': '0'} 2022-09-03T06:03:16.228 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000123491bd29c62bf4-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.229 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.229 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.230 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.231 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.231 DEBUG:boto:Method: GET 2022-09-03T06:03:16.232 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.233 DEBUG:boto:Data: 2022-09-03T06:03:16.233 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.234 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.234 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.235 DEBUG:boto:Params: {} 2022-09-03T06:03:16.236 DEBUG:boto:Token: None 2022-09-03T06:03:16.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8= 2022-09-03T06:03:16.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8=', 'Content-Length': '0'} 2022-09-03T06:03:16.240 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f035d45a7b3d7a30-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.242 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.242 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.243 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:16.244 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:16.245 DEBUG:boto:Method: GET 2022-09-03T06:03:16.246 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:16.246 DEBUG:boto:Data: 2022-09-03T06:03:16.247 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.248 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.249 DEBUG:boto:Params: {} 2022-09-03T06:03:16.249 DEBUG:boto:Token: None 2022-09-03T06:03:16.250 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.250 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.254 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b027753d9c691e27-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.302 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.304 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.305 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.306 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:16.307 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:16.308 DEBUG:boto:Method: GET 2022-09-03T06:03:16.309 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:16.310 DEBUG:boto:Data: 2022-09-03T06:03:16.311 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.312 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.312 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.313 DEBUG:boto:Params: {} 2022-09-03T06:03:16.318 DEBUG:boto:Token: None 2022-09-03T06:03:16.319 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.325 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000df5198b5cebebc2a-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.326 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.328 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.329 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.330 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.331 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T06:03:16.331 DEBUG:boto:Data: 2022-09-03T06:03:16.332 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.333 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.334 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.335 DEBUG:boto:Params: {} 2022-09-03T06:03:16.336 DEBUG:boto:Token: None 2022-09-03T06:03:16.336 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy 2022-09-03T06:03:16.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZA7AT7G0Nr8D2cejRhUrQf2iG84= 2022-09-03T06:03:16.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZA7AT7G0Nr8D2cejRhUrQf2iG84=', 'Content-Length': '0'} 2022-09-03T06:03:16.340 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c1e9b46724e2322a-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.341 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.342 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.343 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.344 DEBUG:boto:Path: /avysmw-18/dummy 2022-09-03T06:03:16.345 DEBUG:boto:Data: 2022-09-03T06:03:16.345 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.346 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.347 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.348 DEBUG:boto:Params: {} 2022-09-03T06:03:16.349 DEBUG:boto:Token: None 2022-09-03T06:03:16.349 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy 2022-09-03T06:03:16.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZA7AT7G0Nr8D2cejRhUrQf2iG84= 2022-09-03T06:03:16.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZA7AT7G0Nr8D2cejRhUrQf2iG84=', 'Content-Length': '0'} 2022-09-03T06:03:16.353 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c20b7c9cf819b3e9-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.355 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:03:16.356 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.357 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.358 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.359 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.359 DEBUG:boto:Method: GET 2022-09-03T06:03:16.360 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.360 DEBUG:boto:Data: 2022-09-03T06:03:16.361 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.362 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.363 DEBUG:boto:Params: {} 2022-09-03T06:03:16.363 DEBUG:boto:Token: None 2022-09-03T06:03:16.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8= 2022-09-03T06:03:16.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8=', 'Content-Length': '0'} 2022-09-03T06:03:16.367 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000aae95a3ad0a2c0e8-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.368 DEBUG:boto:path=/avysmw-18/dummy 2022-09-03T06:03:16.369 DEBUG:boto:auth_path=/avysmw-18/dummy 2022-09-03T06:03:16.369 DEBUG:boto:path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.370 DEBUG:boto:auth_path=/avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.370 DEBUG:boto:Method: GET 2022-09-03T06:03:16.371 DEBUG:boto:Path: /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.371 DEBUG:boto:Data: 2022-09-03T06:03:16.372 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.373 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.373 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.374 DEBUG:boto:Params: {} 2022-09-03T06:03:16.375 DEBUG:boto:Token: None 2022-09-03T06:03:16.375 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/dummy?versionId=rMCJTRbcZDXqkRiOSNNL3qB-91UFM6. 2022-09-03T06:03:16.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8= 2022-09-03T06:03:16.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rndl4CyZ6y5MxQEM7czmDGGz9r8=', 'Content-Length': '0'} 2022-09-03T06:03:16.383 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:56:33 GMT'), ('x-amz-version-id', 'rMCJTRbcZDXqkRiOSNNL3qB-91UFM6.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000073b8e38a0fe322d5-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.384 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.384 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.385 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:16.385 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:16.387 DEBUG:boto:Method: GET 2022-09-03T06:03:16.387 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:16.388 DEBUG:boto:Data: 2022-09-03T06:03:16.388 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.389 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.389 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.390 DEBUG:boto:Params: {} 2022-09-03T06:03:16.391 DEBUG:boto:Token: None 2022-09-03T06:03:16.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.395 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000006cc80851ce17e11-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.396 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.397 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.398 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.398 DEBUG:boto:path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:16.399 DEBUG:boto:auth_path=/avysmw-18/?prefix=dummy 2022-09-03T06:03:16.400 DEBUG:boto:Method: GET 2022-09-03T06:03:16.400 DEBUG:boto:Path: /avysmw-18/?prefix=dummy 2022-09-03T06:03:16.401 DEBUG:boto:Data: 2022-09-03T06:03:16.401 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.402 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.402 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.403 DEBUG:boto:Params: {} 2022-09-03T06:03:16.404 DEBUG:boto:Token: None 2022-09-03T06:03:16.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.410 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac7b2189ddc40e00-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.411 DEBUG:boto:b'avysmw-18dummy1000falsedummy2022-09-03T05:56:33.737Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.412 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:03:16.413 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.413 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.414 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.414 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.415 DEBUG:boto:Method: GET 2022-09-03T06:03:16.415 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.416 DEBUG:boto:Data: 2022-09-03T06:03:16.417 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.417 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.418 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.419 DEBUG:boto:Params: {} 2022-09-03T06:03:16.419 DEBUG:boto:Token: None 2022-09-03T06:03:16.420 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo= 2022-09-03T06:03:16.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo=', 'Content-Length': '0'} 2022-09-03T06:03:16.425 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000038c1efb4a5b9a6d6-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.426 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.427 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.428 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.429 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.430 DEBUG:boto:Method: GET 2022-09-03T06:03:16.430 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.431 DEBUG:boto:Data: 2022-09-03T06:03:16.432 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.432 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.433 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.433 DEBUG:boto:Params: {} 2022-09-03T06:03:16.434 DEBUG:boto:Token: None 2022-09-03T06:03:16.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo= 2022-09-03T06:03:16.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo=', 'Content-Length': '0'} 2022-09-03T06:03:16.438 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000127d1961dc26d0f1-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.439 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.440 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.441 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.441 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.442 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.443 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.443 DEBUG:boto:Data: 2022-09-03T06:03:16.444 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.444 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.445 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.445 DEBUG:boto:Params: {} 2022-09-03T06:03:16.446 DEBUG:boto:Token: None 2022-09-03T06:03:16.447 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:df0hxMSZIw97P53i54FNmXei+Yk= 2022-09-03T06:03:16.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:df0hxMSZIw97P53i54FNmXei+Yk=', 'Content-Length': '0'} 2022-09-03T06:03:16.450 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007e8271cad1f18fb0-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.451 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.451 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.452 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.452 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.453 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.453 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.454 DEBUG:boto:Data: 2022-09-03T06:03:16.454 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.455 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.456 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.457 DEBUG:boto:Params: {} 2022-09-03T06:03:16.457 DEBUG:boto:Token: None 2022-09-03T06:03:16.458 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:df0hxMSZIw97P53i54FNmXei+Yk= 2022-09-03T06:03:16.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:df0hxMSZIw97P53i54FNmXei+Yk=', 'Content-Length': '0'} 2022-09-03T06:03:16.462 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000acec7a0dfed3309c-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.463 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:03:16.463 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.464 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.465 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.466 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.466 DEBUG:boto:Method: GET 2022-09-03T06:03:16.467 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.468 DEBUG:boto:Data: 2022-09-03T06:03:16.469 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.470 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.471 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.472 DEBUG:boto:Params: {} 2022-09-03T06:03:16.472 DEBUG:boto:Token: None 2022-09-03T06:03:16.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo= 2022-09-03T06:03:16.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo=', 'Content-Length': '0'} 2022-09-03T06:03:16.477 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c21101ee4ceab058-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.478 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.479 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.479 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.480 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.481 DEBUG:boto:Method: GET 2022-09-03T06:03:16.482 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.482 DEBUG:boto:Data: 2022-09-03T06:03:16.483 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.484 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.484 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.485 DEBUG:boto:Params: {} 2022-09-03T06:03:16.486 DEBUG:boto:Token: None 2022-09-03T06:03:16.487 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo= 2022-09-03T06:03:16.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo=', 'Content-Length': '0'} 2022-09-03T06:03:16.491 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c047683f9efff692-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.534 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.535 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.536 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.536 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.537 DEBUG:boto:Method: GET 2022-09-03T06:03:16.537 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.538 DEBUG:boto:Data: 2022-09-03T06:03:16.539 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.540 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.541 DEBUG:boto:Params: {} 2022-09-03T06:03:16.542 DEBUG:boto:Token: None 2022-09-03T06:03:16.543 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.547 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000094bd8e8ffec149bc-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.548 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.549 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.550 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.550 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.551 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.551 DEBUG:boto:Method: GET 2022-09-03T06:03:16.552 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.553 DEBUG:boto:Data: 2022-09-03T06:03:16.553 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.554 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.555 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.555 DEBUG:boto:Params: {} 2022-09-03T06:03:16.556 DEBUG:boto:Token: None 2022-09-03T06:03:16.557 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.557 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.561 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008aeba9623fce765d-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.562 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.563 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.564 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.564 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.565 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T06:03:16.565 DEBUG:boto:Data: 2022-09-03T06:03:16.566 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.567 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.568 DEBUG:boto:Params: {} 2022-09-03T06:03:16.569 DEBUG:boto:Token: None 2022-09-03T06:03:16.569 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1 2022-09-03T06:03:16.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jLHNb06uHkr+htgmQqWvRirlzh4= 2022-09-03T06:03:16.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jLHNb06uHkr+htgmQqWvRirlzh4=', 'Content-Length': '0'} 2022-09-03T06:03:16.574 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004d90f3469864bc09-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.575 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.575 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.576 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.577 DEBUG:boto:Path: /avysmw-18/obj-test-zone1 2022-09-03T06:03:16.577 DEBUG:boto:Data: 2022-09-03T06:03:16.578 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.579 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.580 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.580 DEBUG:boto:Params: {} 2022-09-03T06:03:16.581 DEBUG:boto:Token: None 2022-09-03T06:03:16.582 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1 2022-09-03T06:03:16.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jLHNb06uHkr+htgmQqWvRirlzh4= 2022-09-03T06:03:16.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jLHNb06uHkr+htgmQqWvRirlzh4=', 'Content-Length': '0'} 2022-09-03T06:03:16.586 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008581e7ab111bccea-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.587 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2022-09-03T06:03:16.588 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.588 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.589 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.589 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.590 DEBUG:boto:Method: GET 2022-09-03T06:03:16.590 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.591 DEBUG:boto:Data: 2022-09-03T06:03:16.592 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.592 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.593 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.593 DEBUG:boto:Params: {} 2022-09-03T06:03:16.594 DEBUG:boto:Token: None 2022-09-03T06:03:16.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.595 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo= 2022-09-03T06:03:16.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo=', 'Content-Length': '0'} 2022-09-03T06:03:16.598 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000007a09f15e5d687d5-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.642 DEBUG:boto:path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.644 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1 2022-09-03T06:03:16.645 DEBUG:boto:path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.645 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.646 DEBUG:boto:Method: GET 2022-09-03T06:03:16.647 DEBUG:boto:Path: /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.647 DEBUG:boto:Data: 2022-09-03T06:03:16.648 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.649 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.649 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.650 DEBUG:boto:Params: {} 2022-09-03T06:03:16.651 DEBUG:boto:Token: None 2022-09-03T06:03:16.651 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone1?versionId=nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu 2022-09-03T06:03:16.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo= 2022-09-03T06:03:16.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eYeLwXnVSUpUpepITLJnJn+iIIo=', 'Content-Length': '0'} 2022-09-03T06:03:16.655 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'nfPu6J2a9l9rmQRY2NqnTzWX7N1e2Uu'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007c52f545e37f8000-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.656 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.657 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.657 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.658 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.658 DEBUG:boto:Method: GET 2022-09-03T06:03:16.659 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.660 DEBUG:boto:Data: 2022-09-03T06:03:16.660 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.662 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.662 DEBUG:boto:Params: {} 2022-09-03T06:03:16.663 DEBUG:boto:Token: None 2022-09-03T06:03:16.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.669 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000818871c443aa7f10-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.669 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.670 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.671 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.671 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.672 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.673 DEBUG:boto:Method: GET 2022-09-03T06:03:16.673 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone1 2022-09-03T06:03:16.674 DEBUG:boto:Data: 2022-09-03T06:03:16.674 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.675 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.675 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.676 DEBUG:boto:Params: {} 2022-09-03T06:03:16.677 DEBUG:boto:Token: None 2022-09-03T06:03:16.677 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.682 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b91f76ac1cd55e71-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.682 DEBUG:boto:b'avysmw-18obj-test-zone11000falseobj-test-zone12022-09-03T05:57:15.059Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.684 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:03:16.685 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.685 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.686 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.687 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.687 DEBUG:boto:Method: GET 2022-09-03T06:03:16.688 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.688 DEBUG:boto:Data: 2022-09-03T06:03:16.689 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.690 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.690 DEBUG:boto:Params: {} 2022-09-03T06:03:16.691 DEBUG:boto:Token: None 2022-09-03T06:03:16.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0= 2022-09-03T06:03:16.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0=', 'Content-Length': '0'} 2022-09-03T06:03:16.695 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003e77a376473ba720-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.696 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.697 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.697 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.698 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.698 DEBUG:boto:Method: GET 2022-09-03T06:03:16.699 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.699 DEBUG:boto:Data: 2022-09-03T06:03:16.700 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.700 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.701 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.701 DEBUG:boto:Params: {} 2022-09-03T06:03:16.702 DEBUG:boto:Token: None 2022-09-03T06:03:16.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0= 2022-09-03T06:03:16.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0=', 'Content-Length': '0'} 2022-09-03T06:03:16.706 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000072c893a9d18c6b1f-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.707 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.708 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.709 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.709 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.710 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.710 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.711 DEBUG:boto:Data: 2022-09-03T06:03:16.711 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.712 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.713 DEBUG:boto:Params: {} 2022-09-03T06:03:16.713 DEBUG:boto:Token: None 2022-09-03T06:03:16.714 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kx3AU5nCkXRircsPZJs6LxS62pI= 2022-09-03T06:03:16.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kx3AU5nCkXRircsPZJs6LxS62pI=', 'Content-Length': '0'} 2022-09-03T06:03:16.717 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009dfccca587ce45ef-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.718 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.719 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.719 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.720 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.720 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.721 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.721 DEBUG:boto:Data: 2022-09-03T06:03:16.722 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.722 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.723 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.723 DEBUG:boto:Params: {} 2022-09-03T06:03:16.724 DEBUG:boto:Token: None 2022-09-03T06:03:16.725 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kx3AU5nCkXRircsPZJs6LxS62pI= 2022-09-03T06:03:16.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kx3AU5nCkXRircsPZJs6LxS62pI=', 'Content-Length': '0'} 2022-09-03T06:03:16.728 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000692707bcde3fa7bc-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.728 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:03:16.729 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.730 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.730 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.731 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.731 DEBUG:boto:Method: GET 2022-09-03T06:03:16.732 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.732 DEBUG:boto:Data: 2022-09-03T06:03:16.733 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.734 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.744 DEBUG:boto:Params: {} 2022-09-03T06:03:16.745 DEBUG:boto:Token: None 2022-09-03T06:03:16.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0= 2022-09-03T06:03:16.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0=', 'Content-Length': '0'} 2022-09-03T06:03:16.749 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000013c6a0fb5bcb9122-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.750 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.750 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.751 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.751 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.752 DEBUG:boto:Method: GET 2022-09-03T06:03:16.752 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.753 DEBUG:boto:Data: 2022-09-03T06:03:16.753 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.754 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.754 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.755 DEBUG:boto:Params: {} 2022-09-03T06:03:16.756 DEBUG:boto:Token: None 2022-09-03T06:03:16.756 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0= 2022-09-03T06:03:16.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0=', 'Content-Length': '0'} 2022-09-03T06:03:16.761 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007112f4f4e2d5ea0c-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.762 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.762 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.763 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.763 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.764 DEBUG:boto:Method: GET 2022-09-03T06:03:16.764 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.765 DEBUG:boto:Data: 2022-09-03T06:03:16.766 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.767 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.767 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.768 DEBUG:boto:Params: {} 2022-09-03T06:03:16.769 DEBUG:boto:Token: None 2022-09-03T06:03:16.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.774 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000773de75ba54ab726-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.775 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.776 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.776 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.777 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.777 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.778 DEBUG:boto:Method: GET 2022-09-03T06:03:16.778 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.779 DEBUG:boto:Data: 2022-09-03T06:03:16.779 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.780 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.781 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.781 DEBUG:boto:Params: {} 2022-09-03T06:03:16.782 DEBUG:boto:Token: None 2022-09-03T06:03:16.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.786 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000257a373d5f98d85e-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.787 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.788 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.788 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.789 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.790 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T06:03:16.790 DEBUG:boto:Data: 2022-09-03T06:03:16.791 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.791 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.791 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.792 DEBUG:boto:Params: {} 2022-09-03T06:03:16.793 DEBUG:boto:Token: None 2022-09-03T06:03:16.793 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2 2022-09-03T06:03:16.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:psEYBkhOkWBwDxeGUHN4AoeXNg0= 2022-09-03T06:03:16.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:psEYBkhOkWBwDxeGUHN4AoeXNg0=', 'Content-Length': '0'} 2022-09-03T06:03:16.797 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000122d37c9e49ed2af-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.799 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.799 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.800 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.801 DEBUG:boto:Path: /avysmw-18/obj-test-zone2 2022-09-03T06:03:16.801 DEBUG:boto:Data: 2022-09-03T06:03:16.802 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.802 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.803 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.803 DEBUG:boto:Params: {} 2022-09-03T06:03:16.804 DEBUG:boto:Token: None 2022-09-03T06:03:16.804 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2 2022-09-03T06:03:16.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:psEYBkhOkWBwDxeGUHN4AoeXNg0= 2022-09-03T06:03:16.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:psEYBkhOkWBwDxeGUHN4AoeXNg0=', 'Content-Length': '0'} 2022-09-03T06:03:16.808 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006bd833624ecbb730-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.809 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2022-09-03T06:03:16.809 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.810 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.811 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.811 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.812 DEBUG:boto:Method: GET 2022-09-03T06:03:16.812 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.813 DEBUG:boto:Data: 2022-09-03T06:03:16.813 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.814 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.815 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.815 DEBUG:boto:Params: {} 2022-09-03T06:03:16.816 DEBUG:boto:Token: None 2022-09-03T06:03:16.817 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0= 2022-09-03T06:03:16.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0=', 'Content-Length': '0'} 2022-09-03T06:03:16.820 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002d696923914640a3-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.821 DEBUG:boto:path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.822 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2 2022-09-03T06:03:16.822 DEBUG:boto:path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.823 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.823 DEBUG:boto:Method: GET 2022-09-03T06:03:16.824 DEBUG:boto:Path: /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.824 DEBUG:boto:Data: 2022-09-03T06:03:16.825 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.825 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.826 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.827 DEBUG:boto:Params: {} 2022-09-03T06:03:16.827 DEBUG:boto:Token: None 2022-09-03T06:03:16.828 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone2?versionId=BawwORIX6QOqyuiFo4WB5lBEtfSnFE2 2022-09-03T06:03:16.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0= 2022-09-03T06:03:16.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kk+LancCqeyJyhwmhFcDOEegNK0=', 'Content-Length': '0'} 2022-09-03T06:03:16.832 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'BawwORIX6QOqyuiFo4WB5lBEtfSnFE2'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007eb3d897a7d42e25-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.874 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.875 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.876 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.877 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.878 DEBUG:boto:Method: GET 2022-09-03T06:03:16.879 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.880 DEBUG:boto:Data: 2022-09-03T06:03:16.881 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.882 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.882 DEBUG:boto:Params: {} 2022-09-03T06:03:16.883 DEBUG:boto:Token: None 2022-09-03T06:03:16.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000124c7c546dbb8258-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.890 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.891 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:16.892 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:16.892 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.893 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.893 DEBUG:boto:Method: GET 2022-09-03T06:03:16.894 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone2 2022-09-03T06:03:16.894 DEBUG:boto:Data: 2022-09-03T06:03:16.895 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.895 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.896 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.897 DEBUG:boto:Params: {} 2022-09-03T06:03:16.897 DEBUG:boto:Token: None 2022-09-03T06:03:16.898 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/ 2022-09-03T06:03:16.899 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc= 2022-09-03T06:03:16.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6MtnE5x+7JSVVF8melv13CzWxLc=', 'Content-Length': '0'} 2022-09-03T06:03:16.902 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000049f28f8a7f92affb-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.902 DEBUG:boto:b'avysmw-18obj-test-zone21000falseobj-test-zone22022-09-03T05:57:15.238Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:16.903 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:03:16.904 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.904 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.905 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.905 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.906 DEBUG:boto:Method: GET 2022-09-03T06:03:16.907 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.907 DEBUG:boto:Data: 2022-09-03T06:03:16.908 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.908 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.909 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.909 DEBUG:boto:Params: {} 2022-09-03T06:03:16.910 DEBUG:boto:Token: None 2022-09-03T06:03:16.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cF9AHlQ+MBVJlt7MIJ5skmBKRRo= 2022-09-03T06:03:16.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cF9AHlQ+MBVJlt7MIJ5skmBKRRo=', 'Content-Length': '0'} 2022-09-03T06:03:16.914 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007b48c5195fe6cc92-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.915 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.915 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.916 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.917 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.917 DEBUG:boto:Method: GET 2022-09-03T06:03:16.918 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.918 DEBUG:boto:Data: 2022-09-03T06:03:16.919 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.919 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.920 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.920 DEBUG:boto:Params: {} 2022-09-03T06:03:16.921 DEBUG:boto:Token: None 2022-09-03T06:03:16.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cF9AHlQ+MBVJlt7MIJ5skmBKRRo= 2022-09-03T06:03:16.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cF9AHlQ+MBVJlt7MIJ5skmBKRRo=', 'Content-Length': '0'} 2022-09-03T06:03:16.925 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c08f4839fb63a31f-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.926 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.927 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.927 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.928 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.928 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.929 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.929 DEBUG:boto:Data: 2022-09-03T06:03:16.930 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.931 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.931 DEBUG:boto:Params: {} 2022-09-03T06:03:16.932 DEBUG:boto:Token: None 2022-09-03T06:03:16.932 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.933 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5N1zkkAGl468MakezeeNnZ4334I= 2022-09-03T06:03:16.934 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5N1zkkAGl468MakezeeNnZ4334I=', 'Content-Length': '0'} 2022-09-03T06:03:16.936 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000267c9205debbb0dc-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.937 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.937 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.938 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.938 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.939 DEBUG:boto:Method: HEAD 2022-09-03T06:03:16.939 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.940 DEBUG:boto:Data: 2022-09-03T06:03:16.940 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.941 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:16.941 DEBUG:boto:Port: 8000 2022-09-03T06:03:16.942 DEBUG:boto:Params: {} 2022-09-03T06:03:16.943 DEBUG:boto:Token: None 2022-09-03T06:03:16.943 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5N1zkkAGl468MakezeeNnZ4334I= 2022-09-03T06:03:16.945 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5N1zkkAGl468MakezeeNnZ4334I=', 'Content-Length': '0'} 2022-09-03T06:03:16.947 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c05c726c6949ad04-006312ee24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:16.948 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:03:16.948 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.949 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:16.949 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.950 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.951 DEBUG:boto:Method: GET 2022-09-03T06:03:16.951 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.952 DEBUG:boto:Data: 2022-09-03T06:03:16.952 DEBUG:boto:Headers: {} 2022-09-03T06:03:16.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:16.953 DEBUG:boto:Port: 8001 2022-09-03T06:03:16.954 DEBUG:boto:Params: {} 2022-09-03T06:03:16.955 DEBUG:boto:Token: None 2022-09-03T06:03:16.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:16 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:16.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cF9AHlQ+MBVJlt7MIJ5skmBKRRo= 2022-09-03T06:03:16.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cF9AHlQ+MBVJlt7MIJ5skmBKRRo=', 'Content-Length': '0'} 2022-09-03T06:03:16.959 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d61eb072992d04b7-006312ee24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.002 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.003 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.003 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.004 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.004 DEBUG:boto:Method: GET 2022-09-03T06:03:17.005 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.006 DEBUG:boto:Data: 2022-09-03T06:03:17.006 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.007 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:17.007 DEBUG:boto:Port: 8000 2022-09-03T06:03:17.008 DEBUG:boto:Params: {} 2022-09-03T06:03:17.008 DEBUG:boto:Token: None 2022-09-03T06:03:17.009 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:17 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yuxknOusQWtLPEs1sjforivnLXA= 2022-09-03T06:03:17.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yuxknOusQWtLPEs1sjforivnLXA=', 'Content-Length': '0'} 2022-09-03T06:03:17.012 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000045466ae10f0d55b1-006312ee25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.013 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:17.014 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:17.014 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.015 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.015 DEBUG:boto:Method: GET 2022-09-03T06:03:17.016 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.016 DEBUG:boto:Data: 2022-09-03T06:03:17.017 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:17.018 DEBUG:boto:Port: 8001 2022-09-03T06:03:17.019 DEBUG:boto:Params: {} 2022-09-03T06:03:17.019 DEBUG:boto:Token: None 2022-09-03T06:03:17.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:17 GMT /avysmw-18/ 2022-09-03T06:03:17.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNn6T1rNQA+7kQqea73jAQ89yg8= 2022-09-03T06:03:17.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNn6T1rNQA+7kQqea73jAQ89yg8=', 'Content-Length': '0'} 2022-09-03T06:03:17.023 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000012bc03369ffe261e-006312ee25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.024 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:17.025 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:17.026 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:17.026 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.027 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.027 DEBUG:boto:Method: GET 2022-09-03T06:03:17.028 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.028 DEBUG:boto:Data: 2022-09-03T06:03:17.029 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.029 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:17.030 DEBUG:boto:Port: 8000 2022-09-03T06:03:17.030 DEBUG:boto:Params: {} 2022-09-03T06:03:17.031 DEBUG:boto:Token: None 2022-09-03T06:03:17.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:17 GMT /avysmw-18/ 2022-09-03T06:03:17.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNn6T1rNQA+7kQqea73jAQ89yg8= 2022-09-03T06:03:17.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNn6T1rNQA+7kQqea73jAQ89yg8=', 'Content-Length': '0'} 2022-09-03T06:03:17.036 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d05adedaba72b58-006312ee25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.036 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:17.037 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.038 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.038 DEBUG:boto:Method: HEAD 2022-09-03T06:03:17.039 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T06:03:17.039 DEBUG:boto:Data: 2022-09-03T06:03:17.040 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.040 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:17.041 DEBUG:boto:Port: 8001 2022-09-03T06:03:17.041 DEBUG:boto:Params: {} 2022-09-03T06:03:17.042 DEBUG:boto:Token: None 2022-09-03T06:03:17.042 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:17 GMT /avysmw-18/obj-test-zone3 2022-09-03T06:03:17.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lc3nfj7PgIwdM9BFlFVC8CJAye4= 2022-09-03T06:03:17.043 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lc3nfj7PgIwdM9BFlFVC8CJAye4=', 'Content-Length': '0'} 2022-09-03T06:03:17.046 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000fabfe280ac64149-006312ee25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.047 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.047 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.048 DEBUG:boto:Method: HEAD 2022-09-03T06:03:17.048 DEBUG:boto:Path: /avysmw-18/obj-test-zone3 2022-09-03T06:03:17.049 DEBUG:boto:Data: 2022-09-03T06:03:17.049 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.050 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:17.050 DEBUG:boto:Port: 8000 2022-09-03T06:03:17.051 DEBUG:boto:Params: {} 2022-09-03T06:03:17.051 DEBUG:boto:Token: None 2022-09-03T06:03:17.052 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:17 GMT /avysmw-18/obj-test-zone3 2022-09-03T06:03:17.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lc3nfj7PgIwdM9BFlFVC8CJAye4= 2022-09-03T06:03:17.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lc3nfj7PgIwdM9BFlFVC8CJAye4=', 'Content-Length': '0'} 2022-09-03T06:03:17.055 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000087c6664cc5b50e6c-006312ee25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.056 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2022-09-03T06:03:17.057 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.057 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.058 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.058 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.059 DEBUG:boto:Method: GET 2022-09-03T06:03:17.059 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.060 DEBUG:boto:Data: 2022-09-03T06:03:17.060 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:17.061 DEBUG:boto:Port: 8001 2022-09-03T06:03:17.062 DEBUG:boto:Params: {} 2022-09-03T06:03:17.063 DEBUG:boto:Token: None 2022-09-03T06:03:17.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:17 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yuxknOusQWtLPEs1sjforivnLXA= 2022-09-03T06:03:17.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yuxknOusQWtLPEs1sjforivnLXA=', 'Content-Length': '0'} 2022-09-03T06:03:17.068 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e23eeb7218af1f28-006312ee25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.069 DEBUG:boto:path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.069 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3 2022-09-03T06:03:17.070 DEBUG:boto:path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.070 DEBUG:boto:auth_path=/avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.071 DEBUG:boto:Method: GET 2022-09-03T06:03:17.071 DEBUG:boto:Path: /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.072 DEBUG:boto:Data: 2022-09-03T06:03:17.072 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.073 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:17.074 DEBUG:boto:Port: 8000 2022-09-03T06:03:17.074 DEBUG:boto:Params: {} 2022-09-03T06:03:17.075 DEBUG:boto:Token: None 2022-09-03T06:03:17.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:17 GMT /avysmw-18/obj-test-zone3?versionId=CfuuomqFo.i4eJFtRFZjZAHoEOAdulv 2022-09-03T06:03:17.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yuxknOusQWtLPEs1sjforivnLXA= 2022-09-03T06:03:17.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yuxknOusQWtLPEs1sjforivnLXA=', 'Content-Length': '0'} 2022-09-03T06:03:17.079 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 05:59:59 GMT'), ('x-amz-version-id', 'CfuuomqFo.i4eJFtRFZjZAHoEOAdulv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e50ad934c24c63bf-006312ee25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.080 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:17.081 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:17.081 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.082 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.083 DEBUG:boto:Method: GET 2022-09-03T06:03:17.084 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.084 DEBUG:boto:Data: 2022-09-03T06:03:17.085 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:03:17.086 DEBUG:boto:Port: 8001 2022-09-03T06:03:17.087 DEBUG:boto:Params: {} 2022-09-03T06:03:17.087 DEBUG:boto:Token: None 2022-09-03T06:03:17.088 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:17 GMT /avysmw-18/ 2022-09-03T06:03:17.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNn6T1rNQA+7kQqea73jAQ89yg8= 2022-09-03T06:03:17.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNn6T1rNQA+7kQqea73jAQ89yg8=', 'Content-Length': '0'} 2022-09-03T06:03:17.092 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1fbb37d9f9ed4b6-006312ee25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.093 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:17.094 DEBUG:boto:path=/avysmw-18/ 2022-09-03T06:03:17.095 DEBUG:boto:auth_path=/avysmw-18/ 2022-09-03T06:03:17.095 DEBUG:boto:path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.096 DEBUG:boto:auth_path=/avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.096 DEBUG:boto:Method: GET 2022-09-03T06:03:17.097 DEBUG:boto:Path: /avysmw-18/?prefix=obj-test-zone3 2022-09-03T06:03:17.097 DEBUG:boto:Data: 2022-09-03T06:03:17.098 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.098 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:03:17.099 DEBUG:boto:Port: 8000 2022-09-03T06:03:17.099 DEBUG:boto:Params: {} 2022-09-03T06:03:17.100 DEBUG:boto:Token: None 2022-09-03T06:03:17.101 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:03:17 GMT /avysmw-18/ 2022-09-03T06:03:17.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNn6T1rNQA+7kQqea73jAQ89yg8= 2022-09-03T06:03:17.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNn6T1rNQA+7kQqea73jAQ89yg8=', 'Content-Length': '0'} 2022-09-03T06:03:17.104 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000736a4e5542d223d8-006312ee25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.147 DEBUG:boto:b'avysmw-18obj-test-zone31000falseobj-test-zone32022-09-03T05:57:15.413Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:03:17.149 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-18 zones={test-zone3, test-zone2} 2022-09-03T06:03:17.150 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_versioned_object_incremental_sync ... ok 2022-09-03T06:03:17.156 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:03:17.157 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:03:17.157 DEBUG:boto:Method: PUT 2022-09-03T06:03:17.158 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:03:17.158 DEBUG:boto:Data: 2022-09-03T06:03:17.159 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.159 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:17.160 DEBUG:boto:Port: 8000 2022-09-03T06:03:17.160 DEBUG:boto:Params: {} 2022-09-03T06:03:17.161 DEBUG:boto:Token: None 2022-09-03T06:03:17.161 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:03:17 GMT /avysmw-19/ 2022-09-03T06:03:17.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vp9mOPELf7vgtK7uzOhF2WrN+tM= 2022-09-03T06:03:17.163 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vp9mOPELf7vgtK7uzOhF2WrN+tM=', 'Content-Length': '0'} 2022-09-03T06:03:17.182 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000b01b720e734f2585-006312ee25-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.183 DEBUG:rgw_multi.tests:created bucket=avysmw-19 2022-09-03T06:03:17.184 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:03:17.184 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:03:17.185 DEBUG:boto:path=/avysmw-19/?versioning 2022-09-03T06:03:17.185 DEBUG:boto:auth_path=/avysmw-19/?versioning 2022-09-03T06:03:17.186 DEBUG:boto:Method: PUT 2022-09-03T06:03:17.187 DEBUG:boto:Path: /avysmw-19/?versioning 2022-09-03T06:03:17.187 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:03:17.188 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.189 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:17.190 DEBUG:boto:Port: 8000 2022-09-03T06:03:17.191 DEBUG:boto:Params: {} 2022-09-03T06:03:17.191 DEBUG:boto:Token: None 2022-09-03T06:03:17.192 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:03:17 GMT /avysmw-19/?versioning 2022-09-03T06:03:17.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e5vJVxYVweo8IA+Q1mDhjR38iMA= 2022-09-03T06:03:17.194 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e5vJVxYVweo8IA+Q1mDhjR38iMA=', 'Content-Length': '228'} 2022-09-03T06:03:17.203 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c8f1d3e35557c738-006312ee25-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.204 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:03:17.205 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:03:17.205 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:03:17.265 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:17.265 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:17.315 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T06:03:17.329 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:03:17.330 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:03:17.330 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:03:17.330 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T06:03:17.331 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:03:17.331 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:03:17.331 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184997.201443_148.1", 2022-09-03T06:03:17.331 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:03:17.201443Z" 2022-09-03T06:03:17.332 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:03:17.332 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:03:17.332 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:03:17.333 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:03:17.333 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:03:17.333 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:03:17.334 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:03:17.334 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:03:17.334 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:03:17.335 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:03:17.336 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662184997.201443_148.1', 2: '', 3: ''} 2022-09-03T06:03:17.337 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:03:17.337 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:03:17.338 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:03:17.339 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:03:17.394 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:17.395 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:17.458 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:17.456+0000 7f37c2a4e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:03:17.460 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:17.458+0000 7f37c2a4e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:03:17.477 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:03:17.477 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:03:17.477 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:03:17.478 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:03:17.478 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:03:17.478 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:03:17.478 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.479 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:03:17.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:03:17.480 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:03:17.480 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:03:17.480 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:03:17.481 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:03:17.481 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:03:17.481 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:03:17.482 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:03:17.482 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:03:17.482 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.482 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:03:17.483 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:03:17.483 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:03:17.484 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:03:17.484 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184997.201443_148.1", 2022-09-03T06:03:17.485 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:03:17.485 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:03:17.485 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:03:17.485 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.201443Z", 2022-09-03T06:03:17.486 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.486 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:03:17.487 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:03:17.487 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:03:17.487 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:03:17.488 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:03:17.488 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:03:17.488 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:03:17.488 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:03:17.489 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:03:17.489 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.489 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:03:17.491 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:03:17.491 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:03:17.491 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:03:17.492 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:03:17.492 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:03:17.492 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:03:17.492 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:03:17.493 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:03:17.494 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.494 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.494 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.494 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:03:17.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.495 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:03:17.496 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:03:17.496 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:03:17.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.496 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:03:17Z" 2022-09-03T06:03:17.497 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:03:17.498 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184997.201443_148.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:03:17.201443Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:03:17Z" } 2022-09-03T06:03:17.499 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184997.201443_148.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:03:17.201443Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:03:17.500 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184997.201443_148.1', 2: '', 3: ''} 2022-09-03T06:03:17.501 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184997.201443_148.1', 2: '', 3: ''} 2022-09-03T06:03:17.501 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:03:17.502 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:03:17.502 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:03:17.503 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:03:17.504 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:03:17.555 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:17.555 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:17.584 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:17.582+0000 7f5a2e259600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:03:17.584 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:17.583+0000 7f5a2e259600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:03:17.596 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:03:17.597 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:03:17.597 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:03:17.598 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:03:17.598 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:03:17.599 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:03:17.599 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:03:17.599 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:03:17.600 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:03:17.600 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:03:17.600 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:03:17.601 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:03:17.601 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:03:17.602 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:03:17.602 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:03:17.602 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:03:17.602 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:03:17.603 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:03:17.603 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:03:17.603 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:03:17.603 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:03:17.604 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:03:17.604 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:03:17.605 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:03:17.605 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:03:17.606 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184997.201443_148.1", 2022-09-03T06:03:17.606 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:03:17.607 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:03:17.607 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:03:17.608 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.201443Z", 2022-09-03T06:03:17.608 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:03:17.609 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:03:17.609 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:03:17.610 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:03:17.610 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:03:17.611 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:03:17.611 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:03:17.611 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:03:17.611 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:03:17.612 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:03:17.612 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:03:17.612 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:03:17.613 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:03:17.613 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:03:17.613 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:03:17.613 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:03:17.614 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:03:17.614 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:03:17.614 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:03:17.615 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:03:17.615 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:03:17.615 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:03:17.615 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:03:17.616 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:03:17.616 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:03:17.616 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:03:17.616 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:03:17.617 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:03:17.617 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:03:17.617 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:03:17.617 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:03:17.618 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:03:17.618 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:03:17.618 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:03:17Z" 2022-09-03T06:03:17.618 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:03:17.620 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184997.201443_148.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:03:17.201443Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:03:17Z" } 2022-09-03T06:03:17.621 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184997.201443_148.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:03:17.201443Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:03:17.621 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184997.201443_148.1', 2: '', 3: ''} 2022-09-03T06:03:17.622 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184997.201443_148.1', 2: '', 3: ''} 2022-09-03T06:03:17.622 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:03:17.623 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:03:17.624 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:03:17.624 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:03:17.625 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:03:17.674 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:17.674 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:17.699 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:17.696+0000 7f7ceb873600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:03:17.700 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:17.697+0000 7f7ceb873600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:03:17.711 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:03:17.711 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:03:17.712 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:03:17.712 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:03:17.712 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:03:17.713 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:03:17.713 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.713 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.714 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:03:17.714 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:03:17.715 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:03:17.715 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:03:17.716 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:03:17.716 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:03:17.717 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:03:17.717 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:03:17.717 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:03:17.717 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:03:17.718 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.718 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.718 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.718 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:03:17.719 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:03:17.719 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:03:17.719 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:03:17.720 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184997.201443_148.1", 2022-09-03T06:03:17.720 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:03:17.720 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:03:17.721 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:03:17.721 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.201443Z", 2022-09-03T06:03:17.721 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.722 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.722 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.722 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:03:17.722 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:03:17.723 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:03:17.723 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:03:17.723 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:03:17.723 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:03:17.724 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:03:17.724 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:03:17.724 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:03:17.725 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.725 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.726 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.726 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:03:17.727 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:03:17.727 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:03:17.727 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:03:17.728 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:03:17.728 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:03:17.728 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:03:17.728 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:03:17.729 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:03:17.729 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:03:17.729 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.729 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:03:17.730 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:03:17.730 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.730 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:03:17.731 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:03:17.731 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:03:17.731 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:03:17.731 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:03:17Z" 2022-09-03T06:03:17.732 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:03:17.733 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184997.201443_148.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:03:17.201443Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:03:17Z" } 2022-09-03T06:03:17.734 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184997.201443_148.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:03:17.201443Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:03:17.734 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662184997.201443_148.1', 2: '', 3: ''} 2022-09-03T06:03:17.735 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184997.201443_148.1', 2: '', 3: ''} 2022-09-03T06:03:17.736 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:03:17.736 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:03:17.737 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:03:17.738 DEBUG:boto:Method: HEAD 2022-09-03T06:03:17.739 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:03:17.740 DEBUG:boto:Data: 2022-09-03T06:03:17.741 DEBUG:boto:Headers: {} 2022-09-03T06:03:17.742 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:17.743 DEBUG:boto:Port: 8000 2022-09-03T06:03:17.744 DEBUG:boto:Params: {} 2022-09-03T06:03:17.745 DEBUG:boto:Token: None 2022-09-03T06:03:17.746 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:03:17 GMT /avysmw-19/ 2022-09-03T06:03:17.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qlDw0NChSDB4/ZI1dKF3oxXy+pg= 2022-09-03T06:03:17.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qlDw0NChSDB4/ZI1dKF3oxXy+pg=', 'Content-Length': '0'} 2022-09-03T06:03:17.751 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d0ed9ebbfced7d23-006312ee25-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.752 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:03:17.752 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:03:17.753 DEBUG:boto:Method: PUT 2022-09-03T06:03:17.753 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:03:17.754 DEBUG:boto:Data: 2022-09-03T06:03:17.754 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:03:17.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:03:17.756 DEBUG:boto:Port: 8000 2022-09-03T06:03:17.756 DEBUG:boto:Params: {} 2022-09-03T06:03:17.757 DEBUG:boto:Token: None 2022-09-03T06:03:17.758 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:03:17 GMT /avysmw-19/dummy 2022-09-03T06:03:17.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EmZk60Y7dloprl6wKAQTMTJ2V0U= 2022-09-03T06:03:17.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:03:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EmZk60Y7dloprl6wKAQTMTJ2V0U='} 2022-09-03T06:03:17.782 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-amz-request-id', 'tx00000bb5ffa3546ca815a-006312ee25-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:03:17.783 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-19 2022-09-03T06:03:17.783 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:17.785 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:17.785 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:03:17.842 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:17.842 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:17.876 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:17.873+0000 7f47f9edf600 1 waiting to reach incremental sync.. 2022-09-03T06:03:22.876 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:22.874+0000 7f47f9edf600 1 waiting to reach incremental sync.. 2022-09-03T06:03:27.877 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:27.875+0000 7f47f9edf600 1 waiting to reach incremental sync.. 2022-09-03T06:03:32.879 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:32.877+0000 7f47f9edf600 1 bucket sync caught up with source: 2022-09-03T06:03:32.880 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T06:03:32.880 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T06:03:32.880 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:32.877+0000 7f47f9edf600 0 bucket checkpoint complete 2022-09-03T06:03:32.905 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-19 2022-09-03T06:03:32.906 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:32.906 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:32.907 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:03:32.974 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:32.975 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:33.013 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:33.011+0000 7fef7ccd3600 1 bucket sync caught up with source: 2022-09-03T06:03:33.013 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T06:03:33.014 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T06:03:33.014 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:33.011+0000 7fef7ccd3600 0 bucket checkpoint complete 2022-09-03T06:03:33.030 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-19 2022-09-03T06:03:33.030 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:33.031 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:33.032 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:03:33.092 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:33.093 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:33.126 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:33.124+0000 7f658ed0f600 1 bucket sync caught up with source: 2022-09-03T06:03:33.127 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T06:03:33.127 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000002.13.10, , , , , ] 2022-09-03T06:03:33.127 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:03:33.124+0000 7f658ed0f600 0 bucket checkpoint complete 2022-09-03T06:03:33.138 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-19 2022-09-03T06:03:33.138 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:33.139 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:33.140 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:03:33.201 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:33.201 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:33.253 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:33.251+0000 7fa9afd52600 1 waiting to reach incremental sync.. 2022-09-03T06:03:38.253 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:38.251+0000 7fa9afd52600 1 waiting to reach incremental sync.. 2022-09-03T06:03:43.254 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:43.251+0000 7fa9afd52600 1 waiting to reach incremental sync.. 2022-09-03T06:03:48.254 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:48.251+0000 7fa9afd52600 1 waiting to reach incremental sync.. 2022-09-03T06:03:53.256 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:53.253+0000 7fa9afd52600 1 bucket sync caught up with source: 2022-09-03T06:03:53.257 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.31.10, , , , , ] 2022-09-03T06:03:53.257 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000002.31.10, , , , , ] 2022-09-03T06:03:53.257 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:53.253+0000 7fa9afd52600 0 bucket checkpoint complete 2022-09-03T06:03:53.275 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-19 2022-09-03T06:03:53.276 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:53.277 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:03:53.277 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:03:53.342 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:03:53.343 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:03:53.397 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:53.394+0000 7f66d6328600 1 waiting for incremental sync to catch up: 2022-09-03T06:03:53.398 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.31.10, , , , , ] 2022-09-03T06:03:53.398 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000003.33.9, , , , , ] 2022-09-03T06:03:58.399 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:03:58.396+0000 7f66d6328600 1 waiting for incremental sync to catch up: 2022-09-03T06:03:58.400 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.31.10, , , , , ] 2022-09-03T06:03:58.400 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000003.33.9, , , , , ] 2022-09-03T06:04:03.401 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:03.398+0000 7f66d6328600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:03.401 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.31.10, , , , , ] 2022-09-03T06:04:03.402 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000003.33.9, , , , , ] 2022-09-03T06:04:08.402 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:08.400+0000 7f66d6328600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:08.402 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000002.31.10, , , , , ] 2022-09-03T06:04:08.402 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000003.33.9, , , , , ] 2022-09-03T06:04:13.404 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:13.401+0000 7f66d6328600 1 bucket sync caught up with source: 2022-09-03T06:04:13.404 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , , , ] 2022-09-03T06:04:13.405 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000003.33.9, , , , , ] 2022-09-03T06:04:13.405 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:13.401+0000 7f66d6328600 0 bucket checkpoint complete 2022-09-03T06:04:13.421 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-19 2022-09-03T06:04:13.422 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-19 2022-09-03T06:04:13.423 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:13.424 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:13.426 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:04:13.489 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:04:13.490 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:04:13.539 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:13.537+0000 7f13595a9600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:13.540 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000003.39.9, , , , , ] 2022-09-03T06:04:13.540 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.41.9, , , , , ] 2022-09-03T06:04:18.541 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:18.539+0000 7f13595a9600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:18.541 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000003.39.9, , , , , ] 2022-09-03T06:04:18.542 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.41.9, , , , , ] 2022-09-03T06:04:23.542 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:23.540+0000 7f13595a9600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:23.543 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000003.39.9, , , , , ] 2022-09-03T06:04:23.543 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.41.9, , , , , ] 2022-09-03T06:04:28.544 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:28.542+0000 7f13595a9600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:28.544 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000003.39.9, , , , , ] 2022-09-03T06:04:28.544 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.41.9, , , , , ] 2022-09-03T06:04:33.545 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:33.543+0000 7f13595a9600 1 bucket sync caught up with source: 2022-09-03T06:04:33.546 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.41.9, , , , , ] 2022-09-03T06:04:33.546 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.41.9, , , , , ] 2022-09-03T06:04:33.547 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:33.543+0000 7f13595a9600 0 bucket checkpoint complete 2022-09-03T06:04:33.563 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-19 2022-09-03T06:04:33.564 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:33.565 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:33.566 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:04:33.632 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:04:33.632 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:04:33.671 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:04:33.669+0000 7f122835e600 1 bucket sync caught up with source: 2022-09-03T06:04:33.672 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000004.41.9, , , , , ] 2022-09-03T06:04:33.672 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000004.41.9, , , , , ] 2022-09-03T06:04:33.672 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:04:33.669+0000 7f122835e600 0 bucket checkpoint complete 2022-09-03T06:04:33.692 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-19 2022-09-03T06:04:33.693 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-19 zones={test-zone2, test-zone1} 2022-09-03T06:04:33.694 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.695 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.695 DEBUG:boto:Method: HEAD 2022-09-03T06:04:33.696 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:33.696 DEBUG:boto:Data: 2022-09-03T06:04:33.697 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.697 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.698 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.698 DEBUG:boto:Params: {} 2022-09-03T06:04:33.699 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:04:33.700 DEBUG:boto:Token: None 2022-09-03T06:04:33.701 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/ 2022-09-03T06:04:33.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TTAd3uL2QZAo3ZQWcLnbqIADsNY= 2022-09-03T06:04:33.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TTAd3uL2QZAo3ZQWcLnbqIADsNY=', 'Content-Length': '0'} 2022-09-03T06:04:33.707 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c5865d405a920f98-006312ee71-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.708 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.709 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.710 DEBUG:boto:Method: HEAD 2022-09-03T06:04:33.712 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:33.713 DEBUG:boto:Data: 2022-09-03T06:04:33.714 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.715 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.716 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.717 DEBUG:boto:Params: {} 2022-09-03T06:04:33.719 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:04:33.720 DEBUG:boto:Token: None 2022-09-03T06:04:33.721 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/ 2022-09-03T06:04:33.722 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TTAd3uL2QZAo3ZQWcLnbqIADsNY= 2022-09-03T06:04:33.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TTAd3uL2QZAo3ZQWcLnbqIADsNY=', 'Content-Length': '0'} 2022-09-03T06:04:33.728 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008cbc83e325fda484-006312ee71-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.729 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:04:33.730 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.731 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.732 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.733 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.735 DEBUG:boto:Method: GET 2022-09-03T06:04:33.736 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.737 DEBUG:boto:Data: 2022-09-03T06:04:33.737 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.738 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.739 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.740 DEBUG:boto:Params: {} 2022-09-03T06:04:33.741 DEBUG:boto:Token: None 2022-09-03T06:04:33.742 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/?versions 2022-09-03T06:04:33.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:34cX6ts+0jxxJHdQ1cDHMUTgRFE= 2022-09-03T06:04:33.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:34cX6ts+0jxxJHdQ1cDHMUTgRFE=', 'Content-Length': '0'} 2022-09-03T06:04:33.747 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d64996c0996d816f-006312ee71-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.748 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:33.749 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:04:33.750 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:04:33.750 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.751 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.751 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.752 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.752 DEBUG:boto:Method: GET 2022-09-03T06:04:33.753 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.754 DEBUG:boto:Data: 2022-09-03T06:04:33.755 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.757 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.758 DEBUG:boto:Params: {} 2022-09-03T06:04:33.758 DEBUG:boto:Token: None 2022-09-03T06:04:33.759 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/?versions 2022-09-03T06:04:33.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:34cX6ts+0jxxJHdQ1cDHMUTgRFE= 2022-09-03T06:04:33.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:34cX6ts+0jxxJHdQ1cDHMUTgRFE=', 'Content-Length': '0'} 2022-09-03T06:04:33.763 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000048b78902f02cfef-006312ee71-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.764 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:33.764 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:04:33.765 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.765 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.766 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.767 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.767 DEBUG:boto:Method: GET 2022-09-03T06:04:33.768 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.768 DEBUG:boto:Data: 2022-09-03T06:04:33.769 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.769 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.770 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.770 DEBUG:boto:Params: {} 2022-09-03T06:04:33.771 DEBUG:boto:Token: None 2022-09-03T06:04:33.771 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/?versions 2022-09-03T06:04:33.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:34cX6ts+0jxxJHdQ1cDHMUTgRFE= 2022-09-03T06:04:33.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:34cX6ts+0jxxJHdQ1cDHMUTgRFE=', 'Content-Length': '0'} 2022-09-03T06:04:33.775 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000846449aa96b1de20-006312ee71-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.776 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:33.776 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.777 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.777 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.778 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.779 DEBUG:boto:Method: GET 2022-09-03T06:04:33.779 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:33.780 DEBUG:boto:Data: 2022-09-03T06:04:33.780 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.781 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.781 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.782 DEBUG:boto:Params: {} 2022-09-03T06:04:33.782 DEBUG:boto:Token: None 2022-09-03T06:04:33.783 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/?versions 2022-09-03T06:04:33.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:34cX6ts+0jxxJHdQ1cDHMUTgRFE= 2022-09-03T06:04:33.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:34cX6ts+0jxxJHdQ1cDHMUTgRFE=', 'Content-Length': '0'} 2022-09-03T06:04:33.786 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c625b4569231c03d-006312ee71-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.787 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:33.788 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:04:33.788 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.789 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.790 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.790 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.791 DEBUG:boto:Method: GET 2022-09-03T06:04:33.791 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.792 DEBUG:boto:Data: 2022-09-03T06:04:33.792 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.793 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.793 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.794 DEBUG:boto:Params: {} 2022-09-03T06:04:33.795 DEBUG:boto:Token: None 2022-09-03T06:04:33.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0= 2022-09-03T06:04:33.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0=', 'Content-Length': '0'} 2022-09-03T06:04:33.799 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004772f5537ac5f485-006312ee71-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.799 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.800 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.800 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.801 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.801 DEBUG:boto:Method: GET 2022-09-03T06:04:33.802 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.803 DEBUG:boto:Data: 2022-09-03T06:04:33.803 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.804 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.805 DEBUG:boto:Params: {} 2022-09-03T06:04:33.805 DEBUG:boto:Token: None 2022-09-03T06:04:33.806 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0= 2022-09-03T06:04:33.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0=', 'Content-Length': '0'} 2022-09-03T06:04:33.810 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007ed08c4c60157def-006312ee71-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.811 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.812 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.812 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.813 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.814 DEBUG:boto:Method: HEAD 2022-09-03T06:04:33.814 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.815 DEBUG:boto:Data: 2022-09-03T06:04:33.815 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.816 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.816 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.817 DEBUG:boto:Params: {} 2022-09-03T06:04:33.818 DEBUG:boto:Token: None 2022-09-03T06:04:33.818 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XXgUVGCo5Q0NddC6Y5T9xmnPYQ0= 2022-09-03T06:04:33.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XXgUVGCo5Q0NddC6Y5T9xmnPYQ0=', 'Content-Length': '0'} 2022-09-03T06:04:33.821 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001f440180d66dd402-006312ee71-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.822 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.823 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.823 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.824 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.824 DEBUG:boto:Method: HEAD 2022-09-03T06:04:33.825 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.825 DEBUG:boto:Data: 2022-09-03T06:04:33.826 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.827 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.827 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.828 DEBUG:boto:Params: {} 2022-09-03T06:04:33.828 DEBUG:boto:Token: None 2022-09-03T06:04:33.829 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.830 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XXgUVGCo5Q0NddC6Y5T9xmnPYQ0= 2022-09-03T06:04:33.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XXgUVGCo5Q0NddC6Y5T9xmnPYQ0=', 'Content-Length': '0'} 2022-09-03T06:04:33.833 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c21112c7d58edbb7-006312ee71-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.833 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:04:33.834 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.835 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.835 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.836 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.836 DEBUG:boto:Method: GET 2022-09-03T06:04:33.837 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.837 DEBUG:boto:Data: 2022-09-03T06:04:33.838 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.838 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.839 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.839 DEBUG:boto:Params: {} 2022-09-03T06:04:33.840 DEBUG:boto:Token: None 2022-09-03T06:04:33.841 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0= 2022-09-03T06:04:33.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0=', 'Content-Length': '0'} 2022-09-03T06:04:33.844 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000066ab81ea71103ed7-006312ee71-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.845 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.845 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.846 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.846 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.847 DEBUG:boto:Method: GET 2022-09-03T06:04:33.848 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.848 DEBUG:boto:Data: 2022-09-03T06:04:33.849 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.850 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.850 DEBUG:boto:Params: {} 2022-09-03T06:04:33.851 DEBUG:boto:Token: None 2022-09-03T06:04:33.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0= 2022-09-03T06:04:33.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0=', 'Content-Length': '0'} 2022-09-03T06:04:33.855 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f36722c1a29cdc3c-006312ee71-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.856 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.856 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.857 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:33.857 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:33.858 DEBUG:boto:Method: GET 2022-09-03T06:04:33.858 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:33.859 DEBUG:boto:Data: 2022-09-03T06:04:33.859 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.860 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.860 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.861 DEBUG:boto:Params: {} 2022-09-03T06:04:33.861 DEBUG:boto:Token: None 2022-09-03T06:04:33.862 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/ 2022-09-03T06:04:33.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oaqE20qfDR9jyv+ICY+hdoX9Qgo= 2022-09-03T06:04:33.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oaqE20qfDR9jyv+ICY+hdoX9Qgo=', 'Content-Length': '0'} 2022-09-03T06:04:33.866 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000619ce7d101fca1c8-006312ee71-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.910 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:33.911 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.912 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.912 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:33.913 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:33.913 DEBUG:boto:Method: GET 2022-09-03T06:04:33.914 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:33.914 DEBUG:boto:Data: 2022-09-03T06:04:33.915 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.916 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.916 DEBUG:boto:Params: {} 2022-09-03T06:04:33.917 DEBUG:boto:Token: None 2022-09-03T06:04:33.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/ 2022-09-03T06:04:33.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oaqE20qfDR9jyv+ICY+hdoX9Qgo= 2022-09-03T06:04:33.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oaqE20qfDR9jyv+ICY+hdoX9Qgo=', 'Content-Length': '0'} 2022-09-03T06:04:33.922 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000065b4fe8f049672e7-006312ee71-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.922 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:33.923 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.924 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.924 DEBUG:boto:Method: HEAD 2022-09-03T06:04:33.925 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:04:33.925 DEBUG:boto:Data: 2022-09-03T06:04:33.926 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.926 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.927 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.927 DEBUG:boto:Params: {} 2022-09-03T06:04:33.928 DEBUG:boto:Token: None 2022-09-03T06:04:33.928 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy 2022-09-03T06:04:33.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QBYXwslsH5ik9tc39Nz7YPgMxpc= 2022-09-03T06:04:33.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QBYXwslsH5ik9tc39Nz7YPgMxpc=', 'Content-Length': '0'} 2022-09-03T06:04:33.932 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b85602ba4c4503ae-006312ee71-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.932 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.933 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.933 DEBUG:boto:Method: HEAD 2022-09-03T06:04:33.934 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:04:33.935 DEBUG:boto:Data: 2022-09-03T06:04:33.935 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.936 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.937 DEBUG:boto:Params: {} 2022-09-03T06:04:33.937 DEBUG:boto:Token: None 2022-09-03T06:04:33.938 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy 2022-09-03T06:04:33.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QBYXwslsH5ik9tc39Nz7YPgMxpc= 2022-09-03T06:04:33.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QBYXwslsH5ik9tc39Nz7YPgMxpc=', 'Content-Length': '0'} 2022-09-03T06:04:33.941 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000960de3ad7b87fe1c-006312ee71-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.942 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:04:33.943 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.943 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.944 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.944 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.945 DEBUG:boto:Method: GET 2022-09-03T06:04:33.945 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.946 DEBUG:boto:Data: 2022-09-03T06:04:33.947 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.947 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.948 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.948 DEBUG:boto:Params: {} 2022-09-03T06:04:33.949 DEBUG:boto:Token: None 2022-09-03T06:04:33.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0= 2022-09-03T06:04:33.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0=', 'Content-Length': '0'} 2022-09-03T06:04:33.953 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005ebbf60c113eece2-006312ee71-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.953 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:33.954 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:33.954 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.955 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.956 DEBUG:boto:Method: GET 2022-09-03T06:04:33.956 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.957 DEBUG:boto:Data: 2022-09-03T06:04:33.958 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.960 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.960 DEBUG:boto:Params: {} 2022-09-03T06:04:33.961 DEBUG:boto:Token: None 2022-09-03T06:04:33.962 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:33.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0= 2022-09-03T06:04:33.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXGlzZ6PEmDVjbaFc0qoB7C93c0=', 'Content-Length': '0'} 2022-09-03T06:04:33.967 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000028caf5fba68b7a56-006312ee71-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.968 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.969 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.970 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:33.970 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:33.971 DEBUG:boto:Method: GET 2022-09-03T06:04:33.972 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:33.972 DEBUG:boto:Data: 2022-09-03T06:04:33.973 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.974 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:33.974 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.975 DEBUG:boto:Params: {} 2022-09-03T06:04:33.976 DEBUG:boto:Token: None 2022-09-03T06:04:33.977 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/ 2022-09-03T06:04:33.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oaqE20qfDR9jyv+ICY+hdoX9Qgo= 2022-09-03T06:04:33.979 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oaqE20qfDR9jyv+ICY+hdoX9Qgo=', 'Content-Length': '0'} 2022-09-03T06:04:33.982 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fcbdc345a9cb199e-006312ee71-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:33.983 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:33.983 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:33.984 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:33.984 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:33.985 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:33.986 DEBUG:boto:Method: GET 2022-09-03T06:04:33.986 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:33.987 DEBUG:boto:Data: 2022-09-03T06:04:33.988 DEBUG:boto:Headers: {} 2022-09-03T06:04:33.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:33.989 DEBUG:boto:Port: 8000 2022-09-03T06:04:33.989 DEBUG:boto:Params: {} 2022-09-03T06:04:33.990 DEBUG:boto:Token: None 2022-09-03T06:04:33.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:33 GMT /avysmw-19/ 2022-09-03T06:04:33.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oaqE20qfDR9jyv+ICY+hdoX9Qgo= 2022-09-03T06:04:33.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oaqE20qfDR9jyv+ICY+hdoX9Qgo=', 'Content-Length': '0'} 2022-09-03T06:04:33.995 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000086ca8d511cdfa237-006312ee71-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.042 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.043 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-19 zones={test-zone2, test-zone1} 2022-09-03T06:04:34.044 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-19 zones={test-zone3, test-zone1} 2022-09-03T06:04:34.044 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.045 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.045 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.046 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.046 DEBUG:boto:Data: 2022-09-03T06:04:34.047 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.048 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.050 DEBUG:boto:Params: {} 2022-09-03T06:04:34.050 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:04:34.051 DEBUG:boto:Token: None 2022-09-03T06:04:34.052 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.057 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003406ad1af4839377-006312ee72-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.058 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.059 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.059 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.060 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.060 DEBUG:boto:Data: 2022-09-03T06:04:34.061 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.062 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.063 DEBUG:boto:Params: {} 2022-09-03T06:04:34.063 DEBUG:boto:Token: None 2022-09-03T06:04:34.064 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.068 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002c7760bc71031ce5-006312ee72-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.069 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:04:34.069 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.070 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.070 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.071 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.071 DEBUG:boto:Method: GET 2022-09-03T06:04:34.072 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.072 DEBUG:boto:Data: 2022-09-03T06:04:34.073 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.074 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.074 DEBUG:boto:Params: {} 2022-09-03T06:04:34.075 DEBUG:boto:Token: None 2022-09-03T06:04:34.076 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/?versions 2022-09-03T06:04:34.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU= 2022-09-03T06:04:34.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU=', 'Content-Length': '0'} 2022-09-03T06:04:34.080 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000081fb7fde4523e10a-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.080 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.081 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:04:34.082 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:04:34.082 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.083 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.083 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.084 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.084 DEBUG:boto:Method: GET 2022-09-03T06:04:34.085 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.085 DEBUG:boto:Data: 2022-09-03T06:04:34.086 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.086 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.087 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.087 DEBUG:boto:Params: {} 2022-09-03T06:04:34.088 DEBUG:boto:Token: None 2022-09-03T06:04:34.088 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/?versions 2022-09-03T06:04:34.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU= 2022-09-03T06:04:34.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU=', 'Content-Length': '0'} 2022-09-03T06:04:34.092 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fe0a150d97b416c3-006312ee72-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.093 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.094 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:04:34.095 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.095 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.096 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.096 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.097 DEBUG:boto:Method: GET 2022-09-03T06:04:34.097 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.098 DEBUG:boto:Data: 2022-09-03T06:04:34.098 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.099 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.100 DEBUG:boto:Params: {} 2022-09-03T06:04:34.100 DEBUG:boto:Token: None 2022-09-03T06:04:34.101 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/?versions 2022-09-03T06:04:34.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU= 2022-09-03T06:04:34.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU=', 'Content-Length': '0'} 2022-09-03T06:04:34.105 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000450b0ca99cadb2e9-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.105 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.106 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.107 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.107 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.108 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.108 DEBUG:boto:Method: GET 2022-09-03T06:04:34.109 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.109 DEBUG:boto:Data: 2022-09-03T06:04:34.110 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.111 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.111 DEBUG:boto:Params: {} 2022-09-03T06:04:34.112 DEBUG:boto:Token: None 2022-09-03T06:04:34.112 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/?versions 2022-09-03T06:04:34.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU= 2022-09-03T06:04:34.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU=', 'Content-Length': '0'} 2022-09-03T06:04:34.116 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067a2ecb3d439f8ee-006312ee72-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.117 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.118 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:04:34.118 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.119 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.119 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.120 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.120 DEBUG:boto:Method: GET 2022-09-03T06:04:34.121 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.121 DEBUG:boto:Data: 2022-09-03T06:04:34.122 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.123 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.123 DEBUG:boto:Params: {} 2022-09-03T06:04:34.124 DEBUG:boto:Token: None 2022-09-03T06:04:34.125 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.128 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000072e77c54ab8aec07-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.129 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.129 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.130 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.130 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.131 DEBUG:boto:Method: GET 2022-09-03T06:04:34.131 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.132 DEBUG:boto:Data: 2022-09-03T06:04:34.132 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.133 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.133 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.134 DEBUG:boto:Params: {} 2022-09-03T06:04:34.135 DEBUG:boto:Token: None 2022-09-03T06:04:34.135 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.136 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.139 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000252411c5b185dcfb-006312ee72-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.139 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.140 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.140 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.141 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.141 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.142 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.142 DEBUG:boto:Data: 2022-09-03T06:04:34.143 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.144 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.144 DEBUG:boto:Params: {} 2022-09-03T06:04:34.145 DEBUG:boto:Token: None 2022-09-03T06:04:34.145 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44+L6+9kypvXu3XdnCdPe9t0xws= 2022-09-03T06:04:34.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44+L6+9kypvXu3XdnCdPe9t0xws=', 'Content-Length': '0'} 2022-09-03T06:04:34.149 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a27cd250d7b6b0a0-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.149 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.150 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.150 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.151 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.151 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.152 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.152 DEBUG:boto:Data: 2022-09-03T06:04:34.153 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.153 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.154 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.154 DEBUG:boto:Params: {} 2022-09-03T06:04:34.155 DEBUG:boto:Token: None 2022-09-03T06:04:34.155 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44+L6+9kypvXu3XdnCdPe9t0xws= 2022-09-03T06:04:34.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44+L6+9kypvXu3XdnCdPe9t0xws=', 'Content-Length': '0'} 2022-09-03T06:04:34.159 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000063fa4d517becbcda-006312ee72-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.159 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:04:34.160 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.160 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.161 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.161 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.162 DEBUG:boto:Method: GET 2022-09-03T06:04:34.162 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.163 DEBUG:boto:Data: 2022-09-03T06:04:34.164 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.164 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.165 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.165 DEBUG:boto:Params: {} 2022-09-03T06:04:34.166 DEBUG:boto:Token: None 2022-09-03T06:04:34.166 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.170 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000daea3a582393981f-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.171 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.171 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.172 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.172 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.173 DEBUG:boto:Method: GET 2022-09-03T06:04:34.173 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.174 DEBUG:boto:Data: 2022-09-03T06:04:34.174 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.175 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.176 DEBUG:boto:Params: {} 2022-09-03T06:04:34.176 DEBUG:boto:Token: None 2022-09-03T06:04:34.177 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.180 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000066676f4e87dd0979-006312ee72-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.181 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.182 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.182 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.183 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.184 DEBUG:boto:Method: GET 2022-09-03T06:04:34.185 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:34.185 DEBUG:boto:Data: 2022-09-03T06:04:34.186 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.186 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.187 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.187 DEBUG:boto:Params: {} 2022-09-03T06:04:34.188 DEBUG:boto:Token: None 2022-09-03T06:04:34.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0= 2022-09-03T06:04:34.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0=', 'Content-Length': '0'} 2022-09-03T06:04:34.193 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000611e5e369cfecb1d-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.194 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.195 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.196 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.196 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.197 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.198 DEBUG:boto:Method: GET 2022-09-03T06:04:34.199 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:34.200 DEBUG:boto:Data: 2022-09-03T06:04:34.201 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.202 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.203 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.204 DEBUG:boto:Params: {} 2022-09-03T06:04:34.205 DEBUG:boto:Token: None 2022-09-03T06:04:34.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0= 2022-09-03T06:04:34.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0=', 'Content-Length': '0'} 2022-09-03T06:04:34.212 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003291d6f7bf29c2db-006312ee72-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.257 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.258 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.259 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.260 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.260 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:04:34.261 DEBUG:boto:Data: 2022-09-03T06:04:34.261 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.262 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.263 DEBUG:boto:Params: {} 2022-09-03T06:04:34.263 DEBUG:boto:Token: None 2022-09-03T06:04:34.264 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy 2022-09-03T06:04:34.264 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W7mtfinH7Pm4D3vToA/NvOoaTS4= 2022-09-03T06:04:34.265 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W7mtfinH7Pm4D3vToA/NvOoaTS4=', 'Content-Length': '0'} 2022-09-03T06:04:34.269 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000048cf9988c7948c3c-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.270 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.271 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.272 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.272 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:04:34.273 DEBUG:boto:Data: 2022-09-03T06:04:34.273 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.275 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.275 DEBUG:boto:Params: {} 2022-09-03T06:04:34.276 DEBUG:boto:Token: None 2022-09-03T06:04:34.276 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy 2022-09-03T06:04:34.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W7mtfinH7Pm4D3vToA/NvOoaTS4= 2022-09-03T06:04:34.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W7mtfinH7Pm4D3vToA/NvOoaTS4=', 'Content-Length': '0'} 2022-09-03T06:04:34.281 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000023ae5fa18b57f114-006312ee72-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.282 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:04:34.283 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.284 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.284 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.285 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.285 DEBUG:boto:Method: GET 2022-09-03T06:04:34.286 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.287 DEBUG:boto:Data: 2022-09-03T06:04:34.288 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.289 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.289 DEBUG:boto:Params: {} 2022-09-03T06:04:34.290 DEBUG:boto:Token: None 2022-09-03T06:04:34.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.295 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ff5531d26722f663-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.296 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.297 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.298 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.299 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.300 DEBUG:boto:Method: GET 2022-09-03T06:04:34.301 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.302 DEBUG:boto:Data: 2022-09-03T06:04:34.303 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.303 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.304 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.305 DEBUG:boto:Params: {} 2022-09-03T06:04:34.306 DEBUG:boto:Token: None 2022-09-03T06:04:34.306 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.308 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.310 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000240f6da5d1920a9f-006312ee72-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.312 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.313 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.314 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.315 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.316 DEBUG:boto:Method: GET 2022-09-03T06:04:34.317 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:34.318 DEBUG:boto:Data: 2022-09-03T06:04:34.318 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.319 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.320 DEBUG:boto:Params: {} 2022-09-03T06:04:34.321 DEBUG:boto:Token: None 2022-09-03T06:04:34.321 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0= 2022-09-03T06:04:34.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0=', 'Content-Length': '0'} 2022-09-03T06:04:34.326 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ea49fc44157d7a1c-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.327 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.329 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.330 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.331 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.331 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.333 DEBUG:boto:Method: GET 2022-09-03T06:04:34.333 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:34.334 DEBUG:boto:Data: 2022-09-03T06:04:34.335 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.337 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.338 DEBUG:boto:Params: {} 2022-09-03T06:04:34.339 DEBUG:boto:Token: None 2022-09-03T06:04:34.340 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0= 2022-09-03T06:04:34.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0=', 'Content-Length': '0'} 2022-09-03T06:04:34.345 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000065372fd97f71d16f-006312ee72-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.347 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.348 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-19 zones={test-zone3, test-zone1} 2022-09-03T06:04:34.348 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-19 zones={test-zone3, test-zone2} 2022-09-03T06:04:34.349 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.349 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.350 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.350 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.351 DEBUG:boto:Data: 2022-09-03T06:04:34.351 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.352 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.353 DEBUG:boto:Params: {} 2022-09-03T06:04:34.354 DEBUG:boto:Token: None 2022-09-03T06:04:34.354 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.358 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003d3ef23d6a284065-006312ee72-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.359 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.359 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.360 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.360 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.361 DEBUG:boto:Data: 2022-09-03T06:04:34.361 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.362 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.362 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.363 DEBUG:boto:Params: {} 2022-09-03T06:04:34.363 DEBUG:boto:Token: None 2022-09-03T06:04:34.364 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.368 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009df591415252ef00-006312ee72-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.369 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:04:34.369 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.370 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.371 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.371 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.372 DEBUG:boto:Method: GET 2022-09-03T06:04:34.372 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.373 DEBUG:boto:Data: 2022-09-03T06:04:34.373 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.374 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.374 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.375 DEBUG:boto:Params: {} 2022-09-03T06:04:34.376 DEBUG:boto:Token: None 2022-09-03T06:04:34.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/?versions 2022-09-03T06:04:34.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU= 2022-09-03T06:04:34.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU=', 'Content-Length': '0'} 2022-09-03T06:04:34.380 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa7d4686e3503417-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.422 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.423 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:04:34.423 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:04:34.424 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.425 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.425 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.425 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.426 DEBUG:boto:Method: GET 2022-09-03T06:04:34.427 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.427 DEBUG:boto:Data: 2022-09-03T06:04:34.428 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.428 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.429 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.430 DEBUG:boto:Params: {} 2022-09-03T06:04:34.430 DEBUG:boto:Token: None 2022-09-03T06:04:34.431 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/?versions 2022-09-03T06:04:34.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU= 2022-09-03T06:04:34.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU=', 'Content-Length': '0'} 2022-09-03T06:04:34.435 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6f2134d21dcb3b0-006312ee72-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.436 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.436 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:04:34.437 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.437 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.438 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.439 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.439 DEBUG:boto:Method: GET 2022-09-03T06:04:34.440 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.440 DEBUG:boto:Data: 2022-09-03T06:04:34.441 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.442 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.442 DEBUG:boto:Params: {} 2022-09-03T06:04:34.443 DEBUG:boto:Token: None 2022-09-03T06:04:34.444 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/?versions 2022-09-03T06:04:34.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU= 2022-09-03T06:04:34.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU=', 'Content-Length': '0'} 2022-09-03T06:04:34.448 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000009c33ab6b97aa7ea-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.448 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.449 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.450 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.450 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.451 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.452 DEBUG:boto:Method: GET 2022-09-03T06:04:34.452 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:04:34.453 DEBUG:boto:Data: 2022-09-03T06:04:34.453 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.454 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.455 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.455 DEBUG:boto:Params: {} 2022-09-03T06:04:34.456 DEBUG:boto:Token: None 2022-09-03T06:04:34.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/?versions 2022-09-03T06:04:34.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU= 2022-09-03T06:04:34.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66W22Y7+MEjKBg2udvzEq+PL9zU=', 'Content-Length': '0'} 2022-09-03T06:04:34.461 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000493beba654e961b2-006312ee72-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.463 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.463 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:04:34.464 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.464 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.465 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.466 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.467 DEBUG:boto:Method: GET 2022-09-03T06:04:34.468 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.468 DEBUG:boto:Data: 2022-09-03T06:04:34.469 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.470 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.470 DEBUG:boto:Params: {} 2022-09-03T06:04:34.471 DEBUG:boto:Token: None 2022-09-03T06:04:34.472 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.475 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000094dd7a80ddb7d087-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.476 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.477 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.478 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.478 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.479 DEBUG:boto:Method: GET 2022-09-03T06:04:34.480 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.480 DEBUG:boto:Data: 2022-09-03T06:04:34.481 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.481 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.482 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.483 DEBUG:boto:Params: {} 2022-09-03T06:04:34.484 DEBUG:boto:Token: None 2022-09-03T06:04:34.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.488 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000029e035b6564c56db-006312ee72-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.489 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.489 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.490 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.491 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.491 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.492 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.492 DEBUG:boto:Data: 2022-09-03T06:04:34.493 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.494 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.494 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.495 DEBUG:boto:Params: {} 2022-09-03T06:04:34.495 DEBUG:boto:Token: None 2022-09-03T06:04:34.496 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44+L6+9kypvXu3XdnCdPe9t0xws= 2022-09-03T06:04:34.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44+L6+9kypvXu3XdnCdPe9t0xws=', 'Content-Length': '0'} 2022-09-03T06:04:34.500 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f5a0681695a56816-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.501 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.501 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.502 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.502 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.503 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.504 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.504 DEBUG:boto:Data: 2022-09-03T06:04:34.505 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.505 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.506 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.507 DEBUG:boto:Params: {} 2022-09-03T06:04:34.508 DEBUG:boto:Token: None 2022-09-03T06:04:34.509 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44+L6+9kypvXu3XdnCdPe9t0xws= 2022-09-03T06:04:34.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44+L6+9kypvXu3XdnCdPe9t0xws=', 'Content-Length': '0'} 2022-09-03T06:04:34.512 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ecaeda3ee43d181f-006312ee72-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.513 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:04:34.514 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.514 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.515 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.515 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.516 DEBUG:boto:Method: GET 2022-09-03T06:04:34.517 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.518 DEBUG:boto:Data: 2022-09-03T06:04:34.519 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.520 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.521 DEBUG:boto:Params: {} 2022-09-03T06:04:34.522 DEBUG:boto:Token: None 2022-09-03T06:04:34.522 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.526 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008a45589cf4664dc1-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.527 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.527 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.528 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.528 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.529 DEBUG:boto:Method: GET 2022-09-03T06:04:34.529 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.530 DEBUG:boto:Data: 2022-09-03T06:04:34.530 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.531 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.531 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.532 DEBUG:boto:Params: {} 2022-09-03T06:04:34.533 DEBUG:boto:Token: None 2022-09-03T06:04:34.533 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.537 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b814638724d5e5ad-006312ee72-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.537 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.538 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.539 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.539 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.540 DEBUG:boto:Method: GET 2022-09-03T06:04:34.540 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:34.541 DEBUG:boto:Data: 2022-09-03T06:04:34.541 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.542 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.542 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.543 DEBUG:boto:Params: {} 2022-09-03T06:04:34.544 DEBUG:boto:Token: None 2022-09-03T06:04:34.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0= 2022-09-03T06:04:34.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0=', 'Content-Length': '0'} 2022-09-03T06:04:34.548 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058bbfab9d7a50961-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.549 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.549 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.550 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.551 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.551 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.552 DEBUG:boto:Method: GET 2022-09-03T06:04:34.552 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:34.553 DEBUG:boto:Data: 2022-09-03T06:04:34.553 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.554 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.554 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.555 DEBUG:boto:Params: {} 2022-09-03T06:04:34.555 DEBUG:boto:Token: None 2022-09-03T06:04:34.556 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0= 2022-09-03T06:04:34.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0=', 'Content-Length': '0'} 2022-09-03T06:04:34.565 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e1458423b0bfba3-006312ee72-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.566 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.567 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.567 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.568 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.568 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:04:34.569 DEBUG:boto:Data: 2022-09-03T06:04:34.569 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.570 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.571 DEBUG:boto:Params: {} 2022-09-03T06:04:34.571 DEBUG:boto:Token: None 2022-09-03T06:04:34.572 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy 2022-09-03T06:04:34.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W7mtfinH7Pm4D3vToA/NvOoaTS4= 2022-09-03T06:04:34.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W7mtfinH7Pm4D3vToA/NvOoaTS4=', 'Content-Length': '0'} 2022-09-03T06:04:34.575 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006df11eddaa7ffd8a-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.576 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.577 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.577 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.578 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:04:34.578 DEBUG:boto:Data: 2022-09-03T06:04:34.579 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.579 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.580 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.580 DEBUG:boto:Params: {} 2022-09-03T06:04:34.581 DEBUG:boto:Token: None 2022-09-03T06:04:34.581 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy 2022-09-03T06:04:34.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W7mtfinH7Pm4D3vToA/NvOoaTS4= 2022-09-03T06:04:34.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W7mtfinH7Pm4D3vToA/NvOoaTS4=', 'Content-Length': '0'} 2022-09-03T06:04:34.585 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000dd12b3417925c8af-006312ee72-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.586 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:04:34.586 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.587 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.587 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.588 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.588 DEBUG:boto:Method: GET 2022-09-03T06:04:34.589 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.589 DEBUG:boto:Data: 2022-09-03T06:04:34.590 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.590 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.591 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.591 DEBUG:boto:Params: {} 2022-09-03T06:04:34.592 DEBUG:boto:Token: None 2022-09-03T06:04:34.593 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.596 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000048d51e171c01c8f3-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.597 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:04:34.597 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:04:34.598 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.598 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.599 DEBUG:boto:Method: GET 2022-09-03T06:04:34.600 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.600 DEBUG:boto:Data: 2022-09-03T06:04:34.601 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.601 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.602 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.602 DEBUG:boto:Params: {} 2022-09-03T06:04:34.603 DEBUG:boto:Token: None 2022-09-03T06:04:34.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:04:34.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc= 2022-09-03T06:04:34.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYnUK3Q8+xLGNsEYvgf30ZH6rMc=', 'Content-Length': '0'} 2022-09-03T06:04:34.607 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e0768779899a0dff-006312ee72-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.607 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.608 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.608 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.609 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.609 DEBUG:boto:Method: GET 2022-09-03T06:04:34.610 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:34.610 DEBUG:boto:Data: 2022-09-03T06:04:34.611 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.612 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.612 DEBUG:boto:Params: {} 2022-09-03T06:04:34.613 DEBUG:boto:Token: None 2022-09-03T06:04:34.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0= 2022-09-03T06:04:34.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0=', 'Content-Length': '0'} 2022-09-03T06:04:34.617 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003260f233abf7145b-006312ee72-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.618 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.619 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.619 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.620 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.620 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:04:34.621 DEBUG:boto:Method: GET 2022-09-03T06:04:34.622 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:04:34.622 DEBUG:boto:Data: 2022-09-03T06:04:34.623 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.623 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.624 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.624 DEBUG:boto:Params: {} 2022-09-03T06:04:34.625 DEBUG:boto:Token: None 2022-09-03T06:04:34.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0= 2022-09-03T06:04:34.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oBKfHvh0R6YnfJnso+HhFP5CHa0=', 'Content-Length': '0'} 2022-09-03T06:04:34.629 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000092e272a7d6237ec5-006312ee72-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.670 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:04:34.671 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-19 zones={test-zone3, test-zone2} 2022-09-03T06:04:34.671 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.672 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.673 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.673 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.674 DEBUG:boto:Data: 2022-09-03T06:04:34.675 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.677 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.677 DEBUG:boto:Params: {} 2022-09-03T06:04:34.678 DEBUG:boto:Token: None 2022-09-03T06:04:34.679 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.685 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000574efbc3a7ab72cf-006312ee72-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.685 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:34.686 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:34.687 DEBUG:boto:Method: PUT 2022-09-03T06:04:34.687 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:34.688 DEBUG:boto:Data: 2022-09-03T06:04:34.688 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:34.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.689 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.690 DEBUG:boto:Params: {} 2022-09-03T06:04:34.690 DEBUG:boto:Token: None 2022-09-03T06:04:34.691 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/obj 2022-09-03T06:04:34.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ= 2022-09-03T06:04:34.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ='} 2022-09-03T06:04:34.717 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-amz-request-id', 'tx000002bcbf3a0069ffc78-006312ee72-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.718 DEBUG:rgw_multi.tests:zone=test-zone1 version=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:04:34.718 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.719 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.719 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.720 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.720 DEBUG:boto:Data: 2022-09-03T06:04:34.721 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.721 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.722 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.723 DEBUG:boto:Params: {} 2022-09-03T06:04:34.723 DEBUG:boto:Token: None 2022-09-03T06:04:34.724 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.725 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.728 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cad3a29aed65b257-006312ee72-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.728 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:34.729 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:34.729 DEBUG:boto:Method: PUT 2022-09-03T06:04:34.730 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:34.730 DEBUG:boto:Data: 2022-09-03T06:04:34.731 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:34.731 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.732 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.733 DEBUG:boto:Params: {} 2022-09-03T06:04:34.733 DEBUG:boto:Token: None 2022-09-03T06:04:34.734 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/obj 2022-09-03T06:04:34.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ= 2022-09-03T06:04:34.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ='} 2022-09-03T06:04:34.754 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-amz-request-id', 'tx00000a33b9a24af1df0ac-006312ee72-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.755 DEBUG:rgw_multi.tests:zone=test-zone2 version=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:04:34.755 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.756 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.756 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.757 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.758 DEBUG:boto:Data: 2022-09-03T06:04:34.758 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.759 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.760 DEBUG:boto:Params: {} 2022-09-03T06:04:34.760 DEBUG:boto:Token: None 2022-09-03T06:04:34.761 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.765 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000077cac0a561322375-006312ee72-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.766 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:34.766 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:34.767 DEBUG:boto:Method: PUT 2022-09-03T06:04:34.767 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:34.768 DEBUG:boto:Data: 2022-09-03T06:04:34.768 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:34.769 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.769 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.770 DEBUG:boto:Params: {} 2022-09-03T06:04:34.771 DEBUG:boto:Token: None 2022-09-03T06:04:34.771 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/obj 2022-09-03T06:04:34.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ= 2022-09-03T06:04:34.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ='} 2022-09-03T06:04:34.798 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-amz-request-id', 'tx000007402675943966b9c-006312ee72-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.799 DEBUG:rgw_multi.tests:zone=test-zone3 version=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:04:34.799 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.800 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.801 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.801 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.802 DEBUG:boto:Data: 2022-09-03T06:04:34.802 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.803 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.803 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.804 DEBUG:boto:Params: {} 2022-09-03T06:04:34.804 DEBUG:boto:Token: None 2022-09-03T06:04:34.806 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.810 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '8'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000375227a58809e261-006312ee72-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.810 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:34.811 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:34.811 DEBUG:boto:Method: PUT 2022-09-03T06:04:34.812 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:34.812 DEBUG:boto:Data: 2022-09-03T06:04:34.813 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:34.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.814 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.814 DEBUG:boto:Params: {} 2022-09-03T06:04:34.815 DEBUG:boto:Token: None 2022-09-03T06:04:34.815 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/obj 2022-09-03T06:04:34.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ= 2022-09-03T06:04:34.817 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ='} 2022-09-03T06:04:34.834 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-amz-request-id', 'tx00000469834c34b00585f-006312ee72-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.834 DEBUG:rgw_multi.tests:zone=test-zone1 version=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:04:34.835 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.835 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.836 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.836 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.837 DEBUG:boto:Data: 2022-09-03T06:04:34.837 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.838 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.838 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.839 DEBUG:boto:Params: {} 2022-09-03T06:04:34.840 DEBUG:boto:Token: None 2022-09-03T06:04:34.840 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.844 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '8'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a329b9f10438c737-006312ee72-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.845 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:34.845 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:34.846 DEBUG:boto:Method: PUT 2022-09-03T06:04:34.846 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:34.847 DEBUG:boto:Data: 2022-09-03T06:04:34.847 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:34.848 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.848 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.849 DEBUG:boto:Params: {} 2022-09-03T06:04:34.849 DEBUG:boto:Token: None 2022-09-03T06:04:34.850 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/obj 2022-09-03T06:04:34.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ= 2022-09-03T06:04:34.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ='} 2022-09-03T06:04:34.868 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-amz-request-id', 'tx000009ed1a3812f46f662-006312ee72-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.869 DEBUG:rgw_multi.tests:zone=test-zone2 version=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:04:34.870 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.870 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.871 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.871 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.872 DEBUG:boto:Data: 2022-09-03T06:04:34.872 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.873 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.873 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.874 DEBUG:boto:Params: {} 2022-09-03T06:04:34.875 DEBUG:boto:Token: None 2022-09-03T06:04:34.875 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.876 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.879 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '8'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a1862be0593f019e-006312ee72-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.880 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:34.880 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:34.881 DEBUG:boto:Method: PUT 2022-09-03T06:04:34.881 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:34.882 DEBUG:boto:Data: 2022-09-03T06:04:34.882 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:34.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:34.883 DEBUG:boto:Port: 8001 2022-09-03T06:04:34.884 DEBUG:boto:Params: {} 2022-09-03T06:04:34.884 DEBUG:boto:Token: None 2022-09-03T06:04:34.885 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/obj 2022-09-03T06:04:34.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ= 2022-09-03T06:04:34.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ='} 2022-09-03T06:04:34.934 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-amz-request-id', 'tx00000fe3a46b04f08976d-006312ee72-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.935 DEBUG:rgw_multi.tests:zone=test-zone3 version=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:04:34.935 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.936 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.936 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.937 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.937 DEBUG:boto:Data: 2022-09-03T06:04:34.938 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.938 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.939 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.940 DEBUG:boto:Params: {} 2022-09-03T06:04:34.940 DEBUG:boto:Token: None 2022-09-03T06:04:34.941 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.944 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '16'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003ac1eccb383b54e4-006312ee72-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.945 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:34.946 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:34.946 DEBUG:boto:Method: PUT 2022-09-03T06:04:34.947 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:34.947 DEBUG:boto:Data: 2022-09-03T06:04:34.948 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:34.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:34.949 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.949 DEBUG:boto:Params: {} 2022-09-03T06:04:34.950 DEBUG:boto:Token: None 2022-09-03T06:04:34.950 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/obj 2022-09-03T06:04:34.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ= 2022-09-03T06:04:34.952 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ='} 2022-09-03T06:04:34.964 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-amz-request-id', 'tx00000c3052555819ae3be-006312ee72-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.965 DEBUG:rgw_multi.tests:zone=test-zone1 version=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:04:34.965 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:34.966 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:34.966 DEBUG:boto:Method: HEAD 2022-09-03T06:04:34.967 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:34.967 DEBUG:boto:Data: 2022-09-03T06:04:34.968 DEBUG:boto:Headers: {} 2022-09-03T06:04:34.968 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.968 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.969 DEBUG:boto:Params: {} 2022-09-03T06:04:34.970 DEBUG:boto:Token: None 2022-09-03T06:04:34.970 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/ 2022-09-03T06:04:34.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA= 2022-09-03T06:04:34.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W76m1+4wLjO7fdauJaDD00FdZYA=', 'Content-Length': '0'} 2022-09-03T06:04:34.974 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '16'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000063350b2e520f734c-006312ee72-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.975 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:34.975 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:34.976 DEBUG:boto:Method: PUT 2022-09-03T06:04:34.976 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:34.977 DEBUG:boto:Data: 2022-09-03T06:04:34.977 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:34.978 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:34.978 DEBUG:boto:Port: 8000 2022-09-03T06:04:34.979 DEBUG:boto:Params: {} 2022-09-03T06:04:34.979 DEBUG:boto:Token: None 2022-09-03T06:04:34.980 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:34 GMT /avysmw-19/obj 2022-09-03T06:04:34.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ= 2022-09-03T06:04:34.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qv4F7/XvRjyb7RtnAqkczgmqVJQ='} 2022-09-03T06:04:34.998 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-amz-request-id', 'tx00000c1035b150297698a-006312ee72-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:34.999 DEBUG:rgw_multi.tests:zone=test-zone2 version=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:04:34.999 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.000 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.000 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.001 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.001 DEBUG:boto:Data: 2022-09-03T06:04:35.002 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.003 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.003 DEBUG:boto:Params: {} 2022-09-03T06:04:35.004 DEBUG:boto:Token: None 2022-09-03T06:04:35.004 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.008 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '16'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006dda36a16b0a281b-006312ee73-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.009 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.009 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.010 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.011 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.011 DEBUG:boto:Data: 2022-09-03T06:04:35.012 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.012 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.013 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.013 DEBUG:boto:Params: {} 2022-09-03T06:04:35.014 DEBUG:boto:Token: None 2022-09-03T06:04:35.014 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.033 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-amz-request-id', 'tx00000d476da67bc21c47c-006312ee73-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.033 DEBUG:rgw_multi.tests:zone=test-zone3 version=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:04:35.034 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.034 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.035 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.035 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.036 DEBUG:boto:Data: 2022-09-03T06:04:35.036 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.037 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.038 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.038 DEBUG:boto:Params: {} 2022-09-03T06:04:35.039 DEBUG:boto:Token: None 2022-09-03T06:04:35.039 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.043 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000035dfa1cc8271252b-006312ee73-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.044 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.044 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.045 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.045 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.046 DEBUG:boto:Data: 2022-09-03T06:04:35.046 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.047 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.048 DEBUG:boto:Params: {} 2022-09-03T06:04:35.048 DEBUG:boto:Token: None 2022-09-03T06:04:35.049 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.050 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.062 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-amz-request-id', 'tx000008968c3485438a34c-006312ee73-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.063 DEBUG:rgw_multi.tests:zone=test-zone1 version=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:04:35.063 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.064 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.065 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.065 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.065 DEBUG:boto:Data: 2022-09-03T06:04:35.066 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.067 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.067 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.068 DEBUG:boto:Params: {} 2022-09-03T06:04:35.068 DEBUG:boto:Token: None 2022-09-03T06:04:35.069 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.073 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000014fb7f30c2c92e56-006312ee73-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.074 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.075 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.076 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.076 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.077 DEBUG:boto:Data: 2022-09-03T06:04:35.078 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.078 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.079 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.079 DEBUG:boto:Params: {} 2022-09-03T06:04:35.080 DEBUG:boto:Token: None 2022-09-03T06:04:35.081 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.099 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-amz-request-id', 'tx00000802f357a3605324a-006312ee73-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.100 DEBUG:rgw_multi.tests:zone=test-zone2 version=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:04:35.102 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.102 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.103 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.104 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.105 DEBUG:boto:Data: 2022-09-03T06:04:35.106 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.107 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.107 DEBUG:boto:Params: {} 2022-09-03T06:04:35.108 DEBUG:boto:Token: None 2022-09-03T06:04:35.108 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.112 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000046fe1af48d403797-006312ee73-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.113 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.114 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.114 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.115 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.115 DEBUG:boto:Data: 2022-09-03T06:04:35.115 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.116 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.117 DEBUG:boto:Params: {} 2022-09-03T06:04:35.117 DEBUG:boto:Token: None 2022-09-03T06:04:35.118 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.130 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-amz-request-id', 'tx000007682f9b8c7a79c48-006312ee73-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.132 DEBUG:rgw_multi.tests:zone=test-zone3 version=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:04:35.133 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.134 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.135 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.135 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.136 DEBUG:boto:Data: 2022-09-03T06:04:35.137 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.138 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.139 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.140 DEBUG:boto:Params: {} 2022-09-03T06:04:35.140 DEBUG:boto:Token: None 2022-09-03T06:04:35.141 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.145 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '32'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000aea64b0014756e55-006312ee73-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.145 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.146 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.146 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.147 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.147 DEBUG:boto:Data: 2022-09-03T06:04:35.148 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.148 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.149 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.149 DEBUG:boto:Params: {} 2022-09-03T06:04:35.150 DEBUG:boto:Token: None 2022-09-03T06:04:35.151 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.151 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.152 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.170 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-amz-request-id', 'tx000005bde85f26cdf346d-006312ee73-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.171 DEBUG:rgw_multi.tests:zone=test-zone1 version=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:04:35.172 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.173 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.173 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.174 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.174 DEBUG:boto:Data: 2022-09-03T06:04:35.175 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.175 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.176 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.176 DEBUG:boto:Params: {} 2022-09-03T06:04:35.177 DEBUG:boto:Token: None 2022-09-03T06:04:35.178 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.182 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '32'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009756b2d69c95dc47-006312ee73-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.182 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.183 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.183 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.184 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.184 DEBUG:boto:Data: 2022-09-03T06:04:35.185 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.185 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.186 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.187 DEBUG:boto:Params: {} 2022-09-03T06:04:35.187 DEBUG:boto:Token: None 2022-09-03T06:04:35.188 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.204 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-amz-request-id', 'tx00000681ed61360778f2e-006312ee73-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.205 DEBUG:rgw_multi.tests:zone=test-zone2 version=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:04:35.206 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.206 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.207 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.208 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.208 DEBUG:boto:Data: 2022-09-03T06:04:35.209 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.209 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.210 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.210 DEBUG:boto:Params: {} 2022-09-03T06:04:35.211 DEBUG:boto:Token: None 2022-09-03T06:04:35.211 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.215 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '32'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e552bb601fd0071c-006312ee73-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.216 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.216 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.217 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.217 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.218 DEBUG:boto:Data: 2022-09-03T06:04:35.218 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.219 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.219 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.220 DEBUG:boto:Params: {} 2022-09-03T06:04:35.220 DEBUG:boto:Token: None 2022-09-03T06:04:35.221 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.250 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-amz-request-id', 'tx000000794a9a443c83ed4-006312ee73-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.251 DEBUG:rgw_multi.tests:zone=test-zone3 version=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:04:35.251 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.252 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.252 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.253 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.253 DEBUG:boto:Data: 2022-09-03T06:04:35.254 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.254 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.255 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.255 DEBUG:boto:Params: {} 2022-09-03T06:04:35.256 DEBUG:boto:Token: None 2022-09-03T06:04:35.256 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.260 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '40'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ee11635036dc6b74-006312ee73-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.261 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.262 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.262 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.263 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.263 DEBUG:boto:Data: 2022-09-03T06:04:35.264 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.265 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.265 DEBUG:boto:Params: {} 2022-09-03T06:04:35.266 DEBUG:boto:Token: None 2022-09-03T06:04:35.266 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.286 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-amz-request-id', 'tx000008ba0f526b9fbeb64-006312ee73-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.286 DEBUG:rgw_multi.tests:zone=test-zone1 version=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:04:35.287 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.287 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.288 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.289 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.289 DEBUG:boto:Data: 2022-09-03T06:04:35.289 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.290 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.290 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.291 DEBUG:boto:Params: {} 2022-09-03T06:04:35.292 DEBUG:boto:Token: None 2022-09-03T06:04:35.292 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.296 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '40'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002655db2603f6759a-006312ee73-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.297 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.297 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.298 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.298 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.299 DEBUG:boto:Data: 2022-09-03T06:04:35.299 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.300 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.301 DEBUG:boto:Params: {} 2022-09-03T06:04:35.301 DEBUG:boto:Token: None 2022-09-03T06:04:35.302 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.316 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-amz-request-id', 'tx00000c056305433a84a0e-006312ee73-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.317 DEBUG:rgw_multi.tests:zone=test-zone2 version=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:04:35.317 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.318 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.318 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.319 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.319 DEBUG:boto:Data: 2022-09-03T06:04:35.320 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.321 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.321 DEBUG:boto:Params: {} 2022-09-03T06:04:35.322 DEBUG:boto:Token: None 2022-09-03T06:04:35.322 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.326 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '40'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009eea5d44dd7bb37e-006312ee73-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.327 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.328 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.328 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.329 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.329 DEBUG:boto:Data: 2022-09-03T06:04:35.330 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.331 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.331 DEBUG:boto:Params: {} 2022-09-03T06:04:35.332 DEBUG:boto:Token: None 2022-09-03T06:04:35.332 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.348 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-amz-request-id', 'tx000005e953614b1cfc8af-006312ee73-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.349 DEBUG:rgw_multi.tests:zone=test-zone3 version=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:04:35.350 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.350 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.351 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.351 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.352 DEBUG:boto:Data: 2022-09-03T06:04:35.352 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.353 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.353 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.353 DEBUG:boto:Params: {} 2022-09-03T06:04:35.354 DEBUG:boto:Token: None 2022-09-03T06:04:35.355 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.360 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '48'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006c3c1d3c8c582040-006312ee73-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.362 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.363 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.363 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.364 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.365 DEBUG:boto:Data: 2022-09-03T06:04:35.365 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.366 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.367 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.367 DEBUG:boto:Params: {} 2022-09-03T06:04:35.368 DEBUG:boto:Token: None 2022-09-03T06:04:35.368 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.381 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-amz-request-id', 'tx0000060cc9568ab99553d-006312ee73-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.381 DEBUG:rgw_multi.tests:zone=test-zone1 version=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:04:35.382 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.382 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.383 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.383 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.384 DEBUG:boto:Data: 2022-09-03T06:04:35.384 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.385 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.385 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.386 DEBUG:boto:Params: {} 2022-09-03T06:04:35.387 DEBUG:boto:Token: None 2022-09-03T06:04:35.388 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.402 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '48'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e54213d295422fa4-006312ee73-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.403 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.404 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.405 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.406 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.406 DEBUG:boto:Data: 2022-09-03T06:04:35.407 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.408 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.409 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.410 DEBUG:boto:Params: {} 2022-09-03T06:04:35.411 DEBUG:boto:Token: None 2022-09-03T06:04:35.412 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.414 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.426 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-amz-request-id', 'tx00000d6aa77dec8a13024-006312ee73-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.427 DEBUG:rgw_multi.tests:zone=test-zone2 version=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:04:35.428 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.429 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.430 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.431 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.432 DEBUG:boto:Data: 2022-09-03T06:04:35.433 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.435 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.436 DEBUG:boto:Params: {} 2022-09-03T06:04:35.437 DEBUG:boto:Token: None 2022-09-03T06:04:35.438 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.442 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '48'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a865e4055125325d-006312ee73-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.442 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.443 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.443 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.444 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.444 DEBUG:boto:Data: 2022-09-03T06:04:35.445 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.446 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.446 DEBUG:boto:Params: {} 2022-09-03T06:04:35.447 DEBUG:boto:Token: None 2022-09-03T06:04:35.448 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.463 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-amz-request-id', 'tx0000070af5397ca76adc4-006312ee73-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.464 DEBUG:rgw_multi.tests:zone=test-zone3 version=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:04:35.464 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.465 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.465 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.466 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.466 DEBUG:boto:Data: 2022-09-03T06:04:35.467 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.468 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.469 DEBUG:boto:Params: {} 2022-09-03T06:04:35.470 DEBUG:boto:Token: None 2022-09-03T06:04:35.470 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.474 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '56'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004632a04b32aa0547-006312ee73-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.475 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.475 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.476 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.476 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.477 DEBUG:boto:Data: 2022-09-03T06:04:35.478 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.479 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.479 DEBUG:boto:Params: {} 2022-09-03T06:04:35.480 DEBUG:boto:Token: None 2022-09-03T06:04:35.481 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.497 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-amz-request-id', 'tx0000044fd126ef1976e72-006312ee73-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.498 DEBUG:rgw_multi.tests:zone=test-zone1 version=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:04:35.499 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.499 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.500 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.500 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.501 DEBUG:boto:Data: 2022-09-03T06:04:35.501 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.502 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.502 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.503 DEBUG:boto:Params: {} 2022-09-03T06:04:35.503 DEBUG:boto:Token: None 2022-09-03T06:04:35.504 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.508 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '56'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000096160176251e3399-006312ee73-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.509 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.509 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.510 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.510 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.511 DEBUG:boto:Data: 2022-09-03T06:04:35.511 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.512 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.512 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.513 DEBUG:boto:Params: {} 2022-09-03T06:04:35.513 DEBUG:boto:Token: None 2022-09-03T06:04:35.514 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.527 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-amz-request-id', 'tx000007149694ed01d9a48-006312ee73-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.528 DEBUG:rgw_multi.tests:zone=test-zone2 version=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:04:35.528 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.529 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.529 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.530 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.530 DEBUG:boto:Data: 2022-09-03T06:04:35.531 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.531 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.532 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.532 DEBUG:boto:Params: {} 2022-09-03T06:04:35.533 DEBUG:boto:Token: None 2022-09-03T06:04:35.533 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.537 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '56'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ebd8dc70a5810e12-006312ee73-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.538 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.538 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.539 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.539 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.540 DEBUG:boto:Data: 2022-09-03T06:04:35.540 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.542 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.542 DEBUG:boto:Params: {} 2022-09-03T06:04:35.543 DEBUG:boto:Token: None 2022-09-03T06:04:35.543 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.554 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-amz-request-id', 'tx000003ec1b68d4469f112-006312ee73-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.555 DEBUG:rgw_multi.tests:zone=test-zone3 version=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:04:35.555 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.556 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.556 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.557 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.557 DEBUG:boto:Data: 2022-09-03T06:04:35.558 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.558 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.559 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.559 DEBUG:boto:Params: {} 2022-09-03T06:04:35.560 DEBUG:boto:Token: None 2022-09-03T06:04:35.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.564 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '9'), ('X-RGW-Bytes-Used', '64'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a03bd7297068de65-006312ee73-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.565 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.565 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.566 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.566 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.567 DEBUG:boto:Data: 2022-09-03T06:04:35.567 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.568 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.569 DEBUG:boto:Params: {} 2022-09-03T06:04:35.570 DEBUG:boto:Token: None 2022-09-03T06:04:35.570 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.583 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-amz-request-id', 'tx00000a5833746fd3a067c-006312ee73-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.584 DEBUG:rgw_multi.tests:zone=test-zone1 version=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:04:35.584 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.585 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.586 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.586 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.587 DEBUG:boto:Data: 2022-09-03T06:04:35.587 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.588 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.589 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.590 DEBUG:boto:Params: {} 2022-09-03T06:04:35.591 DEBUG:boto:Token: None 2022-09-03T06:04:35.591 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.596 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '9'), ('X-RGW-Bytes-Used', '64'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000876398089998a25f-006312ee73-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.597 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.598 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.599 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.600 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.601 DEBUG:boto:Data: 2022-09-03T06:04:35.602 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.602 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.603 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.603 DEBUG:boto:Params: {} 2022-09-03T06:04:35.604 DEBUG:boto:Token: None 2022-09-03T06:04:35.605 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.618 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-amz-request-id', 'tx0000091cae7f4f95bcce8-006312ee73-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.619 DEBUG:rgw_multi.tests:zone=test-zone2 version=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:04:35.620 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.620 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.621 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.621 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.622 DEBUG:boto:Data: 2022-09-03T06:04:35.622 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.623 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.624 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.624 DEBUG:boto:Params: {} 2022-09-03T06:04:35.625 DEBUG:boto:Token: None 2022-09-03T06:04:35.625 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.629 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '9'), ('X-RGW-Bytes-Used', '64'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000046343c3685ea99dc-006312ee73-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.630 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.631 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.631 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.632 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.632 DEBUG:boto:Data: 2022-09-03T06:04:35.633 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.633 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.634 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.634 DEBUG:boto:Params: {} 2022-09-03T06:04:35.635 DEBUG:boto:Token: None 2022-09-03T06:04:35.635 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.636 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.637 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.647 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-amz-request-id', 'tx00000bb43d24cd63afe88-006312ee73-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.648 DEBUG:rgw_multi.tests:zone=test-zone3 version=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:04:35.648 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.649 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.649 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.650 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.650 DEBUG:boto:Data: 2022-09-03T06:04:35.651 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.651 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.652 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.652 DEBUG:boto:Params: {} 2022-09-03T06:04:35.653 DEBUG:boto:Token: None 2022-09-03T06:04:35.653 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.654 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.655 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.657 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '72'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e803e6100fde70bf-006312ee73-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.658 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.659 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.659 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.660 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.660 DEBUG:boto:Data: 2022-09-03T06:04:35.661 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:04:35.662 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.662 DEBUG:boto:Params: {} 2022-09-03T06:04:35.663 DEBUG:boto:Token: None 2022-09-03T06:04:35.664 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.683 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-amz-request-id', 'tx00000e9f84678add7ca75-006312ee73-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.684 DEBUG:rgw_multi.tests:zone=test-zone1 version=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:04:35.684 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.685 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.685 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.686 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.686 DEBUG:boto:Data: 2022-09-03T06:04:35.687 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.687 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.688 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.688 DEBUG:boto:Params: {} 2022-09-03T06:04:35.689 DEBUG:boto:Token: None 2022-09-03T06:04:35.690 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.694 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '72'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002647b7bcc3c13c8c-006312ee73-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.695 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.695 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.696 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.696 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.697 DEBUG:boto:Data: 2022-09-03T06:04:35.697 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.698 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:04:35.699 DEBUG:boto:Port: 8000 2022-09-03T06:04:35.699 DEBUG:boto:Params: {} 2022-09-03T06:04:35.700 DEBUG:boto:Token: None 2022-09-03T06:04:35.701 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.714 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-amz-request-id', 'tx00000db7d21d0fa7d2b0a-006312ee73-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.715 DEBUG:rgw_multi.tests:zone=test-zone2 version=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:04:35.715 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:04:35.716 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:04:35.716 DEBUG:boto:Method: HEAD 2022-09-03T06:04:35.717 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:04:35.717 DEBUG:boto:Data: 2022-09-03T06:04:35.718 DEBUG:boto:Headers: {} 2022-09-03T06:04:35.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.719 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.720 DEBUG:boto:Params: {} 2022-09-03T06:04:35.720 DEBUG:boto:Token: None 2022-09-03T06:04:35.721 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/ 2022-09-03T06:04:35.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc= 2022-09-03T06:04:35.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1bX9gOa9sSpQSx7CVW5VEYSZvRc=', 'Content-Length': '0'} 2022-09-03T06:04:35.724 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '72'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000002825b6d6e7778ba-006312ee73-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.725 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:04:35.726 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:04:35.726 DEBUG:boto:Method: PUT 2022-09-03T06:04:35.727 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:04:35.727 DEBUG:boto:Data: 2022-09-03T06:04:35.728 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue'} 2022-09-03T06:04:35.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:04:35.729 DEBUG:boto:Port: 8001 2022-09-03T06:04:35.729 DEBUG:boto:Params: {} 2022-09-03T06:04:35.730 DEBUG:boto:Token: None 2022-09-03T06:04:35.730 DEBUG:boto:StringToSign: PUT lmY06/L8E1cH1nU2kr9LHg== application/octet-stream Sat, 03 Sep 2022 06:04:35 GMT /avysmw-19/obj 2022-09-03T06:04:35.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM= 2022-09-03T06:04:35.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'lmY06/L8E1cH1nU2kr9LHg==', 'Content-Length': '8', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:04:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JETpqm2Ly3paGlkNvQUarO12LyM='} 2022-09-03T06:04:35.748 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-amz-request-id', 'tx00000945415046711c47d-006312ee73-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:04:35.749 DEBUG:rgw_multi.tests:zone=test-zone3 version=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:04:35.750 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-19 2022-09-03T06:04:35.750 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:35.751 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:35.751 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:04:35.815 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:04:35.816 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:04:35.866 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:04:35.863+0000 7f1b8b987600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:35.866 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000005.19.9, , , , , ] 2022-09-03T06:04:35.867 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000005.19.9, , , 00000000020.48.11, , ] 2022-09-03T06:04:40.868 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:04:40.865+0000 7f1b8b987600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:40.868 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000005.19.9, , , , , ] 2022-09-03T06:04:40.868 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000005.19.9, , , 00000000020.48.11, , ] 2022-09-03T06:04:45.869 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:04:45.867+0000 7f1b8b987600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:45.870 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000005.19.9, , , , , ] 2022-09-03T06:04:45.870 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000005.19.9, , , 00000000020.48.11, , ] 2022-09-03T06:04:50.870 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:04:50.868+0000 7f1b8b987600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:50.871 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000005.19.9, , , 00000000017.43.5, , ] 2022-09-03T06:04:50.872 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000005.19.9, , , 00000000020.48.11, , ] 2022-09-03T06:04:55.872 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:04:55.869+0000 7f1b8b987600 1 bucket sync caught up with source: 2022-09-03T06:04:55.873 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000005.19.9, , , 00000000028.72.5, , ] 2022-09-03T06:04:55.873 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000005.19.9, , , 00000000020.48.11, , ] 2022-09-03T06:04:55.873 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:04:55.869+0000 7f1b8b987600 0 bucket checkpoint complete 2022-09-03T06:04:55.894 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-19 2022-09-03T06:04:55.895 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:55.896 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:55.896 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:04:55.963 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:04:55.963 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:04:56.002 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:56.000+0000 7fdcf7090600 1 bucket sync caught up with source: 2022-09-03T06:04:56.003 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000005.19.9, , , 00000000028.72.5, , ] 2022-09-03T06:04:56.003 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000005.19.9, , , 00000000028.72.5, , ] 2022-09-03T06:04:56.003 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:04:56.000+0000 7fdcf7090600 0 bucket checkpoint complete 2022-09-03T06:04:56.016 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-19 2022-09-03T06:04:56.016 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:56.017 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:04:56.018 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:04:56.082 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:04:56.083 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:04:56.141 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:04:56.139+0000 7f840c75a600 1 waiting for incremental sync to catch up: 2022-09-03T06:04:56.142 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000005.19.9, , , 00000000020.48.11, , ] 2022-09-03T06:04:56.142 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000005.19.9, , , 00000000028.72.5, , ] 2022-09-03T06:05:01.143 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:05:01.140+0000 7f840c75a600 1 bucket sync caught up with source: 2022-09-03T06:05:01.144 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000005.19.9, , , 00000000028.72.5, , ] 2022-09-03T06:05:01.144 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000005.19.9, , , 00000000028.72.5, , ] 2022-09-03T06:05:01.145 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:05:01.140+0000 7f840c75a600 0 bucket checkpoint complete 2022-09-03T06:05:01.166 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-19 2022-09-03T06:05:01.167 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:05:01.168 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:05:01.169 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:05:01.235 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:05:01.236 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:05:01.299 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:01.297+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:01.300 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000001.23.5, , ] 2022-09-03T06:05:01.300 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:06.301 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:06.298+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:06.301 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000001.23.5, , ] 2022-09-03T06:05:06.301 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:11.302 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:11.300+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:11.302 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000001.23.5, , ] 2022-09-03T06:05:11.303 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:16.304 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:16.301+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:16.304 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000001.23.5, , ] 2022-09-03T06:05:16.304 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:21.305 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:21.303+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:21.306 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000001.23.5, , ] 2022-09-03T06:05:21.306 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:26.306 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:26.304+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:26.307 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000001.23.5, , ] 2022-09-03T06:05:26.307 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:31.308 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:31.306+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:31.309 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000009.39.5, , ] 2022-09-03T06:05:31.309 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:36.309 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:36.307+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:36.310 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000009.39.5, , ] 2022-09-03T06:05:36.310 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:41.311 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:41.308+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:41.311 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000009.39.5, , ] 2022-09-03T06:05:41.311 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:46.312 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:46.309+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:46.313 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000009.39.5, , ] 2022-09-03T06:05:46.313 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:51.314 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:51.311+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:51.315 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:05:51.315 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:05:56.315 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:05:56.312+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:05:56.316 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:05:56.316 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:01.317 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:01.315+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:01.318 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:01.318 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:06.318 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:06.316+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:06.319 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:06.319 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:11.320 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:11.318+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:11.320 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:11.321 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:16.322 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:16.320+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:16.322 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:16.322 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:21.323 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:21.321+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:21.323 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:21.324 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:26.325 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:26.322+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:26.326 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:26.326 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:31.326 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:31.324+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:31.327 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:31.327 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:36.328 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:36.326+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:36.328 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:36.328 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:41.329 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:41.326+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:41.330 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:41.331 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:46.331 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:46.328+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:46.331 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:46.332 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:51.332 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:51.329+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:51.333 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:51.333 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:06:56.334 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:06:56.331+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:06:56.334 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:06:56.334 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:07:01.335 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:01.333+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:07:01.335 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:07:01.336 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:07:06.336 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:06.335+0000 7faaec17f600 1 waiting for incremental sync to catch up: 2022-09-03T06:07:06.337 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000021.63.5, , ] 2022-09-03T06:07:06.337 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:07:11.338 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:11.336+0000 7faaec17f600 1 bucket sync caught up with source: 2022-09-03T06:07:11.338 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000044.126.9, , ] 2022-09-03T06:07:11.338 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000042.122.8, , ] 2022-09-03T06:07:11.339 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:11.336+0000 7faaec17f600 0 bucket checkpoint complete 2022-09-03T06:07:11.360 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-19 2022-09-03T06:07:11.361 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:11.362 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:11.363 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:07:11.429 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:11.429 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:11.478 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:11.476+0000 7f56191ff600 1 bucket sync caught up with source: 2022-09-03T06:07:11.478 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.35.9, , , 00000000044.126.9, , ] 2022-09-03T06:07:11.479 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.35.9, , , 00000000044.126.9, , ] 2022-09-03T06:07:11.479 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:11.476+0000 7f56191ff600 0 bucket checkpoint complete 2022-09-03T06:07:11.493 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-19 2022-09-03T06:07:11.494 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-19 2022-09-03T06:07:11.495 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:11.495 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:11.496 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:07:11.554 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:11.554 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:11.602 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:11.600+0000 7faef20ec600 1 bucket sync caught up with source: 2022-09-03T06:07:11.602 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000004.41.9, , , 00000000045.128.9, , ] 2022-09-03T06:07:11.602 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000004.41.9, , , 00000000045.128.9, , ] 2022-09-03T06:07:11.603 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:11.600+0000 7faef20ec600 0 bucket checkpoint complete 2022-09-03T06:07:11.616 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-19 2022-09-03T06:07:11.617 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:11.618 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:11.619 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-19 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:07:11.683 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:11.684 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:11.723 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:11.721+0000 7f86670d6600 1 bucket sync caught up with source: 2022-09-03T06:07:11.724 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000004.41.9, , , 00000000045.128.9, , ] 2022-09-03T06:07:11.724 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000004.41.9, , , 00000000045.128.9, , ] 2022-09-03T06:07:11.724 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:11.721+0000 7f86670d6600 0 bucket checkpoint complete 2022-09-03T06:07:11.737 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-19 2022-09-03T06:07:11.738 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-19 zones={test-zone2, test-zone1} 2022-09-03T06:07:11.739 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:11.739 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:11.740 DEBUG:boto:Method: HEAD 2022-09-03T06:07:11.740 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:07:11.741 DEBUG:boto:Data: 2022-09-03T06:07:11.741 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.742 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:11.742 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.743 DEBUG:boto:Params: {} 2022-09-03T06:07:11.744 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:07:11.745 DEBUG:boto:Token: None 2022-09-03T06:07:11.746 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/ 2022-09-03T06:07:11.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K3p/CehrGSJLf2uQAsShqDy+cQQ= 2022-09-03T06:07:11.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K3p/CehrGSJLf2uQAsShqDy+cQQ=', 'Content-Length': '0'} 2022-09-03T06:07:11.752 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '31'), ('X-RGW-Bytes-Used', '240'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b5a9b4655a60dd5e-006312ef0f-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.753 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:11.753 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:11.754 DEBUG:boto:Method: HEAD 2022-09-03T06:07:11.754 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:07:11.755 DEBUG:boto:Data: 2022-09-03T06:07:11.755 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:11.756 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.757 DEBUG:boto:Params: {} 2022-09-03T06:07:11.757 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:07:11.758 DEBUG:boto:Token: None 2022-09-03T06:07:11.758 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/ 2022-09-03T06:07:11.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K3p/CehrGSJLf2uQAsShqDy+cQQ= 2022-09-03T06:07:11.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K3p/CehrGSJLf2uQAsShqDy+cQQ=', 'Content-Length': '0'} 2022-09-03T06:07:11.763 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '31'), ('X-RGW-Bytes-Used', '240'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000868e2899a3e2285-006312ef0f-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.764 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:07:11.765 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:11.765 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:11.766 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.767 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.767 DEBUG:boto:Method: GET 2022-09-03T06:07:11.768 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.769 DEBUG:boto:Data: 2022-09-03T06:07:11.769 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.770 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:11.770 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.771 DEBUG:boto:Params: {} 2022-09-03T06:07:11.771 DEBUG:boto:Token: None 2022-09-03T06:07:11.772 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/?versions 2022-09-03T06:07:11.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOAGgE5XitdaIxg8OmarS5S8wtU= 2022-09-03T06:07:11.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOAGgE5XitdaIxg8OmarS5S8wtU=', 'Content-Length': '0'} 2022-09-03T06:07:11.777 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005d59047909069c52-006312ef0f-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.778 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:11.781 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:07:11.782 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.783 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.784 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.785 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.786 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.786 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.787 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.788 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.789 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.790 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.791 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.791 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.792 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.793 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.794 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.794 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.795 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.795 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.796 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.796 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.796 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.797 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.797 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.798 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.798 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.799 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.800 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.800 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.801 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.801 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.802 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:07:11.803 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:11.804 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:11.805 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.806 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.806 DEBUG:boto:Method: GET 2022-09-03T06:07:11.807 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.807 DEBUG:boto:Data: 2022-09-03T06:07:11.808 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:11.809 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.809 DEBUG:boto:Params: {} 2022-09-03T06:07:11.810 DEBUG:boto:Token: None 2022-09-03T06:07:11.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/?versions 2022-09-03T06:07:11.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOAGgE5XitdaIxg8OmarS5S8wtU= 2022-09-03T06:07:11.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOAGgE5XitdaIxg8OmarS5S8wtU=', 'Content-Length': '0'} 2022-09-03T06:07:11.815 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c94512088636372-006312ef0f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.816 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:11.818 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:07:11.819 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.819 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.820 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.820 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.821 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.821 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.821 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.822 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.822 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.823 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.823 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.824 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.824 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.825 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.825 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.826 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.826 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.827 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.827 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.828 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.828 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.828 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.829 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.829 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.830 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.830 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.831 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.831 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.832 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.832 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:11.833 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:11.833 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:11.834 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.834 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.835 DEBUG:boto:Method: GET 2022-09-03T06:07:11.835 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.836 DEBUG:boto:Data: 2022-09-03T06:07:11.836 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.837 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:11.837 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.838 DEBUG:boto:Params: {} 2022-09-03T06:07:11.839 DEBUG:boto:Token: None 2022-09-03T06:07:11.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/?versions 2022-09-03T06:07:11.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOAGgE5XitdaIxg8OmarS5S8wtU= 2022-09-03T06:07:11.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOAGgE5XitdaIxg8OmarS5S8wtU=', 'Content-Length': '0'} 2022-09-03T06:07:11.843 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085e49c9ab03aa90a-006312ef0f-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.844 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:11.846 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:11.847 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:11.847 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.848 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.848 DEBUG:boto:Method: GET 2022-09-03T06:07:11.849 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:11.849 DEBUG:boto:Data: 2022-09-03T06:07:11.850 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:11.851 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.852 DEBUG:boto:Params: {} 2022-09-03T06:07:11.852 DEBUG:boto:Token: None 2022-09-03T06:07:11.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/?versions 2022-09-03T06:07:11.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOAGgE5XitdaIxg8OmarS5S8wtU= 2022-09-03T06:07:11.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOAGgE5XitdaIxg8OmarS5S8wtU=', 'Content-Length': '0'} 2022-09-03T06:07:11.857 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a171c5c3e318e7cc-006312ef0f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.858 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:11.860 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:07:11.860 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:11.861 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:11.862 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.862 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.863 DEBUG:boto:Method: GET 2022-09-03T06:07:11.863 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.864 DEBUG:boto:Data: 2022-09-03T06:07:11.864 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.865 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:11.866 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.866 DEBUG:boto:Params: {} 2022-09-03T06:07:11.867 DEBUG:boto:Token: None 2022-09-03T06:07:11.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9JXNwW7Bs3JX8qK19mngvvphIuU= 2022-09-03T06:07:11.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9JXNwW7Bs3JX8qK19mngvvphIuU=', 'Content-Length': '0'} 2022-09-03T06:07:11.871 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d506722de04df64b-006312ef0f-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.872 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:11.872 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:11.873 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.873 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.874 DEBUG:boto:Method: GET 2022-09-03T06:07:11.874 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.875 DEBUG:boto:Data: 2022-09-03T06:07:11.875 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.876 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:11.876 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.877 DEBUG:boto:Params: {} 2022-09-03T06:07:11.878 DEBUG:boto:Token: None 2022-09-03T06:07:11.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9JXNwW7Bs3JX8qK19mngvvphIuU= 2022-09-03T06:07:11.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9JXNwW7Bs3JX8qK19mngvvphIuU=', 'Content-Length': '0'} 2022-09-03T06:07:11.881 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a162ddd3be1a0c5d-006312ef0f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.882 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:11.883 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:11.883 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.884 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.884 DEBUG:boto:Method: HEAD 2022-09-03T06:07:11.885 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.885 DEBUG:boto:Data: 2022-09-03T06:07:11.885 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.886 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:11.887 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.887 DEBUG:boto:Params: {} 2022-09-03T06:07:11.888 DEBUG:boto:Token: None 2022-09-03T06:07:11.888 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Foa1EovD+saQxpTYDrAT6vBJlA= 2022-09-03T06:07:11.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Foa1EovD+saQxpTYDrAT6vBJlA=', 'Content-Length': '0'} 2022-09-03T06:07:11.891 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003ff278ad8fd8b130-006312ef0f-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.892 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:11.893 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:11.894 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.895 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.895 DEBUG:boto:Method: HEAD 2022-09-03T06:07:11.896 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.897 DEBUG:boto:Data: 2022-09-03T06:07:11.898 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.899 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:11.900 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.901 DEBUG:boto:Params: {} 2022-09-03T06:07:11.902 DEBUG:boto:Token: None 2022-09-03T06:07:11.903 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Foa1EovD+saQxpTYDrAT6vBJlA= 2022-09-03T06:07:11.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Foa1EovD+saQxpTYDrAT6vBJlA=', 'Content-Length': '0'} 2022-09-03T06:07:11.907 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000934fb5bbdf0c1efd-006312ef0f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.908 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:07:11.909 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:11.909 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:11.910 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.910 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.911 DEBUG:boto:Method: GET 2022-09-03T06:07:11.911 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.912 DEBUG:boto:Data: 2022-09-03T06:07:11.912 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.913 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:11.914 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.914 DEBUG:boto:Params: {} 2022-09-03T06:07:11.915 DEBUG:boto:Token: None 2022-09-03T06:07:11.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9JXNwW7Bs3JX8qK19mngvvphIuU= 2022-09-03T06:07:11.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9JXNwW7Bs3JX8qK19mngvvphIuU=', 'Content-Length': '0'} 2022-09-03T06:07:11.919 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000002ee7463fe7cdc1e8-006312ef0f-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.920 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:11.920 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:11.921 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.922 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.922 DEBUG:boto:Method: GET 2022-09-03T06:07:11.923 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.924 DEBUG:boto:Data: 2022-09-03T06:07:11.924 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.925 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:11.926 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.926 DEBUG:boto:Params: {} 2022-09-03T06:07:11.927 DEBUG:boto:Token: None 2022-09-03T06:07:11.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:11.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9JXNwW7Bs3JX8qK19mngvvphIuU= 2022-09-03T06:07:11.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9JXNwW7Bs3JX8qK19mngvvphIuU=', 'Content-Length': '0'} 2022-09-03T06:07:11.931 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f2eceecff6b54370-006312ef0f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.933 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:11.934 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:11.935 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:11.936 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:11.937 DEBUG:boto:Method: GET 2022-09-03T06:07:11.938 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:11.939 DEBUG:boto:Data: 2022-09-03T06:07:11.939 DEBUG:boto:Headers: {} 2022-09-03T06:07:11.940 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:11.941 DEBUG:boto:Port: 8000 2022-09-03T06:07:11.942 DEBUG:boto:Params: {} 2022-09-03T06:07:11.943 DEBUG:boto:Token: None 2022-09-03T06:07:11.943 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:11 GMT /avysmw-19/ 2022-09-03T06:07:11.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:d/Va24VqxjUsWbAFr8YIgTdosww= 2022-09-03T06:07:11.945 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:d/Va24VqxjUsWbAFr8YIgTdosww=', 'Content-Length': '0'} 2022-09-03T06:07:11.947 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b9ac3d5ecf3de4b-006312ef0f-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:11.994 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:11.995 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:11.996 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:11.996 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:11.997 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:11.997 DEBUG:boto:Method: GET 2022-09-03T06:07:11.998 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:11.999 DEBUG:boto:Data: 2022-09-03T06:07:11.999 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.000 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.000 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.001 DEBUG:boto:Params: {} 2022-09-03T06:07:12.001 DEBUG:boto:Token: None 2022-09-03T06:07:12.002 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002ed2d0ebe9d9db8b-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.011 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.013 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:12.013 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:12.014 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.015 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:07:12.016 DEBUG:boto:Data: 2022-09-03T06:07:12.016 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.017 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.017 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.018 DEBUG:boto:Params: {} 2022-09-03T06:07:12.018 DEBUG:boto:Token: None 2022-09-03T06:07:12.019 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/dummy 2022-09-03T06:07:12.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t5hXmKSKG3fpIp7srysXbgnP9zI= 2022-09-03T06:07:12.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t5hXmKSKG3fpIp7srysXbgnP9zI=', 'Content-Length': '0'} 2022-09-03T06:07:12.022 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009d781644781dea5d-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.023 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:12.024 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:12.024 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.025 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:07:12.025 DEBUG:boto:Data: 2022-09-03T06:07:12.026 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.027 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.027 DEBUG:boto:Params: {} 2022-09-03T06:07:12.028 DEBUG:boto:Token: None 2022-09-03T06:07:12.028 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/dummy 2022-09-03T06:07:12.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t5hXmKSKG3fpIp7srysXbgnP9zI= 2022-09-03T06:07:12.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t5hXmKSKG3fpIp7srysXbgnP9zI=', 'Content-Length': '0'} 2022-09-03T06:07:12.032 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000bb1975521c67ee74-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.033 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:07:12.034 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:12.035 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:12.036 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:12.037 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:12.038 DEBUG:boto:Method: GET 2022-09-03T06:07:12.038 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:12.039 DEBUG:boto:Data: 2022-09-03T06:07:12.039 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.040 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.040 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.041 DEBUG:boto:Params: {} 2022-09-03T06:07:12.042 DEBUG:boto:Token: None 2022-09-03T06:07:12.043 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:12.044 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9tOCgKToYh03k7kpD5TsnxImhug= 2022-09-03T06:07:12.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9tOCgKToYh03k7kpD5TsnxImhug=', 'Content-Length': '0'} 2022-09-03T06:07:12.046 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000081a82a878861f234-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.047 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:12.048 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:12.049 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:12.049 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:12.050 DEBUG:boto:Method: GET 2022-09-03T06:07:12.051 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:12.051 DEBUG:boto:Data: 2022-09-03T06:07:12.052 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.052 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.053 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.053 DEBUG:boto:Params: {} 2022-09-03T06:07:12.054 DEBUG:boto:Token: None 2022-09-03T06:07:12.055 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:12.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9tOCgKToYh03k7kpD5TsnxImhug= 2022-09-03T06:07:12.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9tOCgKToYh03k7kpD5TsnxImhug=', 'Content-Length': '0'} 2022-09-03T06:07:12.058 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000045f91203ae384de1-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.059 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.059 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.060 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:12.060 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:12.061 DEBUG:boto:Method: GET 2022-09-03T06:07:12.061 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:12.062 DEBUG:boto:Data: 2022-09-03T06:07:12.062 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.063 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.063 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.064 DEBUG:boto:Params: {} 2022-09-03T06:07:12.065 DEBUG:boto:Token: None 2022-09-03T06:07:12.065 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.066 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e1990e208888bf8-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.110 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.111 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.112 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.112 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:12.113 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:12.113 DEBUG:boto:Method: GET 2022-09-03T06:07:12.114 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:12.114 DEBUG:boto:Data: 2022-09-03T06:07:12.115 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.116 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.117 DEBUG:boto:Params: {} 2022-09-03T06:07:12.118 DEBUG:boto:Token: None 2022-09-03T06:07:12.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.123 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e70f1d55866eb8f-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.124 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.125 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.126 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.127 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.128 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.128 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.129 DEBUG:boto:Method: GET 2022-09-03T06:07:12.130 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.130 DEBUG:boto:Data: 2022-09-03T06:07:12.131 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.131 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.132 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.132 DEBUG:boto:Params: {} 2022-09-03T06:07:12.133 DEBUG:boto:Token: None 2022-09-03T06:07:12.134 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.137 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001c805e8f6d7f22dd-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.138 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.139 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.139 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.140 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.141 DEBUG:boto:Method: GET 2022-09-03T06:07:12.141 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.142 DEBUG:boto:Data: 2022-09-03T06:07:12.143 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.144 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.144 DEBUG:boto:Params: {} 2022-09-03T06:07:12.145 DEBUG:boto:Token: None 2022-09-03T06:07:12.146 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.147 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.149 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000258f01c88940c747-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.150 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.151 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.151 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.152 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.152 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.153 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.153 DEBUG:boto:Data: 2022-09-03T06:07:12.154 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.154 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.155 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.156 DEBUG:boto:Params: {} 2022-09-03T06:07:12.156 DEBUG:boto:Token: None 2022-09-03T06:07:12.157 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W9WP2ucc9L1mqAQkalduPHKs/IE= 2022-09-03T06:07:12.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W9WP2ucc9L1mqAQkalduPHKs/IE=', 'Content-Length': '0'} 2022-09-03T06:07:12.160 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000033879c11ca110b5b-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.161 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.161 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.162 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.162 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.163 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.164 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.164 DEBUG:boto:Data: 2022-09-03T06:07:12.165 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.166 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.166 DEBUG:boto:Params: {} 2022-09-03T06:07:12.167 DEBUG:boto:Token: None 2022-09-03T06:07:12.167 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W9WP2ucc9L1mqAQkalduPHKs/IE= 2022-09-03T06:07:12.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W9WP2ucc9L1mqAQkalduPHKs/IE=', 'Content-Length': '0'} 2022-09-03T06:07:12.170 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004a37f1019480c1a8-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.171 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.172 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.172 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.173 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.173 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.174 DEBUG:boto:Method: GET 2022-09-03T06:07:12.175 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.175 DEBUG:boto:Data: 2022-09-03T06:07:12.176 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.176 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.177 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.177 DEBUG:boto:Params: {} 2022-09-03T06:07:12.178 DEBUG:boto:Token: None 2022-09-03T06:07:12.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.182 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000090b98b0947ca6d3d-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.183 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.184 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.185 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.186 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.186 DEBUG:boto:Method: GET 2022-09-03T06:07:12.187 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.187 DEBUG:boto:Data: 2022-09-03T06:07:12.188 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.189 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.189 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.190 DEBUG:boto:Params: {} 2022-09-03T06:07:12.191 DEBUG:boto:Token: None 2022-09-03T06:07:12.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.196 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004f5fd9e17e34b7b2-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.242 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.243 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.243 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.244 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.244 DEBUG:boto:Method: GET 2022-09-03T06:07:12.245 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.245 DEBUG:boto:Data: 2022-09-03T06:07:12.246 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.246 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.247 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.247 DEBUG:boto:Params: {} 2022-09-03T06:07:12.248 DEBUG:boto:Token: None 2022-09-03T06:07:12.249 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.250 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.253 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008f2faae8483ebf1f-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.253 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.254 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.255 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.255 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.256 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.256 DEBUG:boto:Method: GET 2022-09-03T06:07:12.257 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.258 DEBUG:boto:Data: 2022-09-03T06:07:12.258 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.259 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.259 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.260 DEBUG:boto:Params: {} 2022-09-03T06:07:12.260 DEBUG:boto:Token: None 2022-09-03T06:07:12.261 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.265 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b9623e3e5c42c4e9-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.265 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.266 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.267 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.268 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.268 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:12.269 DEBUG:boto:Data: 2022-09-03T06:07:12.269 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.270 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.270 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.271 DEBUG:boto:Params: {} 2022-09-03T06:07:12.272 DEBUG:boto:Token: None 2022-09-03T06:07:12.272 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj 2022-09-03T06:07:12.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U= 2022-09-03T06:07:12.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U=', 'Content-Length': '0'} 2022-09-03T06:07:12.277 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ba7125704ae03073-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.278 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.280 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.281 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.282 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:12.283 DEBUG:boto:Data: 2022-09-03T06:07:12.284 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.285 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.286 DEBUG:boto:Params: {} 2022-09-03T06:07:12.287 DEBUG:boto:Token: None 2022-09-03T06:07:12.288 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj 2022-09-03T06:07:12.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U= 2022-09-03T06:07:12.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U=', 'Content-Length': '0'} 2022-09-03T06:07:12.294 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ed33f9e8a9ebfc26-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.295 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.296 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.298 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.298 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.299 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.300 DEBUG:boto:Method: GET 2022-09-03T06:07:12.301 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.302 DEBUG:boto:Data: 2022-09-03T06:07:12.302 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.303 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.303 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.304 DEBUG:boto:Params: {} 2022-09-03T06:07:12.304 DEBUG:boto:Token: None 2022-09-03T06:07:12.305 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.306 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.308 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000765abb0a23f1f088-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.309 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.310 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.310 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.311 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.311 DEBUG:boto:Method: GET 2022-09-03T06:07:12.312 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.312 DEBUG:boto:Data: 2022-09-03T06:07:12.313 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.314 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.315 DEBUG:boto:Params: {} 2022-09-03T06:07:12.315 DEBUG:boto:Token: None 2022-09-03T06:07:12.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.319 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f58ba93cf1fcf0a2-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.320 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.321 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.321 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.322 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.322 DEBUG:boto:Method: GET 2022-09-03T06:07:12.323 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.324 DEBUG:boto:Data: 2022-09-03T06:07:12.324 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.325 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.325 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.326 DEBUG:boto:Params: {} 2022-09-03T06:07:12.327 DEBUG:boto:Token: None 2022-09-03T06:07:12.327 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.331 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e6ae3a3dd641070-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.332 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.334 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.335 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.336 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.337 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.338 DEBUG:boto:Method: GET 2022-09-03T06:07:12.339 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.339 DEBUG:boto:Data: 2022-09-03T06:07:12.340 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.341 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.342 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.343 DEBUG:boto:Params: {} 2022-09-03T06:07:12.344 DEBUG:boto:Token: None 2022-09-03T06:07:12.345 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.350 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020bba2e1fa971e8d-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.351 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.351 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.352 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.353 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.353 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.354 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.355 DEBUG:boto:Method: GET 2022-09-03T06:07:12.356 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.357 DEBUG:boto:Data: 2022-09-03T06:07:12.358 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.359 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.360 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.361 DEBUG:boto:Params: {} 2022-09-03T06:07:12.363 DEBUG:boto:Token: None 2022-09-03T06:07:12.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aF37vFUuO7FDm9tnSc0Eq/OMU7E= 2022-09-03T06:07:12.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aF37vFUuO7FDm9tnSc0Eq/OMU7E=', 'Content-Length': '0'} 2022-09-03T06:07:12.368 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007632905a0a7d897f-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.370 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.371 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.372 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.372 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.373 DEBUG:boto:Method: GET 2022-09-03T06:07:12.374 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.375 DEBUG:boto:Data: 2022-09-03T06:07:12.376 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.378 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.379 DEBUG:boto:Params: {} 2022-09-03T06:07:12.380 DEBUG:boto:Token: None 2022-09-03T06:07:12.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aF37vFUuO7FDm9tnSc0Eq/OMU7E= 2022-09-03T06:07:12.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aF37vFUuO7FDm9tnSc0Eq/OMU7E=', 'Content-Length': '0'} 2022-09-03T06:07:12.386 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003c51289ef73fdc20-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.426 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.427 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.427 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.428 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.428 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.429 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.429 DEBUG:boto:Data: 2022-09-03T06:07:12.430 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.430 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.432 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.432 DEBUG:boto:Params: {} 2022-09-03T06:07:12.433 DEBUG:boto:Token: None 2022-09-03T06:07:12.433 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pPZObXodW7Mr2mxV31HrX1umoWM= 2022-09-03T06:07:12.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pPZObXodW7Mr2mxV31HrX1umoWM=', 'Content-Length': '0'} 2022-09-03T06:07:12.437 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e819609393c52558-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.438 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.439 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.439 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.440 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.440 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.441 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.441 DEBUG:boto:Data: 2022-09-03T06:07:12.442 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.443 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.443 DEBUG:boto:Params: {} 2022-09-03T06:07:12.444 DEBUG:boto:Token: None 2022-09-03T06:07:12.444 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pPZObXodW7Mr2mxV31HrX1umoWM= 2022-09-03T06:07:12.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pPZObXodW7Mr2mxV31HrX1umoWM=', 'Content-Length': '0'} 2022-09-03T06:07:12.448 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003f05d87f357a974c-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.448 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.449 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.450 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.450 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.451 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.451 DEBUG:boto:Method: GET 2022-09-03T06:07:12.452 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.452 DEBUG:boto:Data: 2022-09-03T06:07:12.453 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.454 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.454 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.455 DEBUG:boto:Params: {} 2022-09-03T06:07:12.456 DEBUG:boto:Token: None 2022-09-03T06:07:12.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aF37vFUuO7FDm9tnSc0Eq/OMU7E= 2022-09-03T06:07:12.457 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aF37vFUuO7FDm9tnSc0Eq/OMU7E=', 'Content-Length': '0'} 2022-09-03T06:07:12.459 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000067e2f033a8857c30-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.460 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.461 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.461 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.462 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.463 DEBUG:boto:Method: GET 2022-09-03T06:07:12.464 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.465 DEBUG:boto:Data: 2022-09-03T06:07:12.466 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.468 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.469 DEBUG:boto:Params: {} 2022-09-03T06:07:12.470 DEBUG:boto:Token: None 2022-09-03T06:07:12.471 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:12.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aF37vFUuO7FDm9tnSc0Eq/OMU7E= 2022-09-03T06:07:12.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aF37vFUuO7FDm9tnSc0Eq/OMU7E=', 'Content-Length': '0'} 2022-09-03T06:07:12.476 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000012130488f4d1beca-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.477 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.478 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.479 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.479 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.482 DEBUG:boto:Method: GET 2022-09-03T06:07:12.482 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.483 DEBUG:boto:Data: 2022-09-03T06:07:12.483 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.484 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.485 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.485 DEBUG:boto:Params: {} 2022-09-03T06:07:12.486 DEBUG:boto:Token: None 2022-09-03T06:07:12.487 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.492 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c3b8f9e63b766fd-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.493 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.495 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.496 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.497 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.498 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.499 DEBUG:boto:Method: GET 2022-09-03T06:07:12.500 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.501 DEBUG:boto:Data: 2022-09-03T06:07:12.502 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.504 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.505 DEBUG:boto:Params: {} 2022-09-03T06:07:12.506 DEBUG:boto:Token: None 2022-09-03T06:07:12.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.511 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dba7e9ee6b421c3d-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.512 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.512 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.513 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.514 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.514 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:12.515 DEBUG:boto:Data: 2022-09-03T06:07:12.515 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.516 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.516 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.517 DEBUG:boto:Params: {} 2022-09-03T06:07:12.517 DEBUG:boto:Token: None 2022-09-03T06:07:12.518 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj 2022-09-03T06:07:12.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U= 2022-09-03T06:07:12.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U=', 'Content-Length': '0'} 2022-09-03T06:07:12.523 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002b19f907bff1822c-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.525 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.526 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.527 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.527 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:12.528 DEBUG:boto:Data: 2022-09-03T06:07:12.529 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.530 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.531 DEBUG:boto:Params: {} 2022-09-03T06:07:12.531 DEBUG:boto:Token: None 2022-09-03T06:07:12.532 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj 2022-09-03T06:07:12.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U= 2022-09-03T06:07:12.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U=', 'Content-Length': '0'} 2022-09-03T06:07:12.537 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000df62a1f98b6935e4-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.538 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.538 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.539 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.540 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.540 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.541 DEBUG:boto:Method: GET 2022-09-03T06:07:12.541 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.542 DEBUG:boto:Data: 2022-09-03T06:07:12.543 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.543 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.544 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.544 DEBUG:boto:Params: {} 2022-09-03T06:07:12.545 DEBUG:boto:Token: None 2022-09-03T06:07:12.546 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.546 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.547 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.549 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c9067273e871d038-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.594 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.595 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.596 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.597 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.598 DEBUG:boto:Method: GET 2022-09-03T06:07:12.598 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.599 DEBUG:boto:Data: 2022-09-03T06:07:12.600 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.601 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.601 DEBUG:boto:Params: {} 2022-09-03T06:07:12.602 DEBUG:boto:Token: None 2022-09-03T06:07:12.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.606 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008f9e78d91ba85895-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.607 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.608 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.608 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.609 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.610 DEBUG:boto:Method: GET 2022-09-03T06:07:12.611 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.612 DEBUG:boto:Data: 2022-09-03T06:07:12.612 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.613 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.614 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.615 DEBUG:boto:Params: {} 2022-09-03T06:07:12.616 DEBUG:boto:Token: None 2022-09-03T06:07:12.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.622 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed213a7e8c8d696f-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.623 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.624 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.624 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.625 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.626 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.626 DEBUG:boto:Method: GET 2022-09-03T06:07:12.627 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.627 DEBUG:boto:Data: 2022-09-03T06:07:12.628 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.628 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.629 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.629 DEBUG:boto:Params: {} 2022-09-03T06:07:12.630 DEBUG:boto:Token: None 2022-09-03T06:07:12.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.634 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000007a332d1b82e2e8a-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.635 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.636 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.636 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.637 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.638 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.638 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.639 DEBUG:boto:Method: GET 2022-09-03T06:07:12.639 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.640 DEBUG:boto:Data: 2022-09-03T06:07:12.640 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.641 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.642 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.642 DEBUG:boto:Params: {} 2022-09-03T06:07:12.643 DEBUG:boto:Token: None 2022-09-03T06:07:12.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aBk3+yfgseHCK/10CMi8KwqBdRY= 2022-09-03T06:07:12.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aBk3+yfgseHCK/10CMi8KwqBdRY=', 'Content-Length': '0'} 2022-09-03T06:07:12.647 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000035f20b9773e69aec-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.647 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.648 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.649 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.649 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.650 DEBUG:boto:Method: GET 2022-09-03T06:07:12.650 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.651 DEBUG:boto:Data: 2022-09-03T06:07:12.651 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.652 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.653 DEBUG:boto:Params: {} 2022-09-03T06:07:12.653 DEBUG:boto:Token: None 2022-09-03T06:07:12.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aBk3+yfgseHCK/10CMi8KwqBdRY= 2022-09-03T06:07:12.655 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aBk3+yfgseHCK/10CMi8KwqBdRY=', 'Content-Length': '0'} 2022-09-03T06:07:12.657 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000009efa15dfed80b03-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.698 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.699 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.699 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.700 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.700 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.701 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.701 DEBUG:boto:Data: 2022-09-03T06:07:12.702 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.702 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.703 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.703 DEBUG:boto:Params: {} 2022-09-03T06:07:12.704 DEBUG:boto:Token: None 2022-09-03T06:07:12.705 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.705 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gcqeJira6dPWf2CSQkXplOHzzj8= 2022-09-03T06:07:12.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gcqeJira6dPWf2CSQkXplOHzzj8=', 'Content-Length': '0'} 2022-09-03T06:07:12.708 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000056ace544874db2cd-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.709 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.709 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.710 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.710 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.711 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.711 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.712 DEBUG:boto:Data: 2022-09-03T06:07:12.712 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.713 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.714 DEBUG:boto:Params: {} 2022-09-03T06:07:12.714 DEBUG:boto:Token: None 2022-09-03T06:07:12.715 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gcqeJira6dPWf2CSQkXplOHzzj8= 2022-09-03T06:07:12.716 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gcqeJira6dPWf2CSQkXplOHzzj8=', 'Content-Length': '0'} 2022-09-03T06:07:12.718 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b51491845e6ba4bb-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.719 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.719 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.720 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.720 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.721 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.721 DEBUG:boto:Method: GET 2022-09-03T06:07:12.722 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.722 DEBUG:boto:Data: 2022-09-03T06:07:12.723 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.723 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.724 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.724 DEBUG:boto:Params: {} 2022-09-03T06:07:12.725 DEBUG:boto:Token: None 2022-09-03T06:07:12.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aBk3+yfgseHCK/10CMi8KwqBdRY= 2022-09-03T06:07:12.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aBk3+yfgseHCK/10CMi8KwqBdRY=', 'Content-Length': '0'} 2022-09-03T06:07:12.729 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c765c408b6be60f9-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.730 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.731 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.731 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.732 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.732 DEBUG:boto:Method: GET 2022-09-03T06:07:12.733 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.733 DEBUG:boto:Data: 2022-09-03T06:07:12.734 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.735 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.735 DEBUG:boto:Params: {} 2022-09-03T06:07:12.736 DEBUG:boto:Token: None 2022-09-03T06:07:12.737 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:12.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aBk3+yfgseHCK/10CMi8KwqBdRY= 2022-09-03T06:07:12.738 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aBk3+yfgseHCK/10CMi8KwqBdRY=', 'Content-Length': '0'} 2022-09-03T06:07:12.740 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000080a782a9e84ad100-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.741 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.742 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.742 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.743 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.743 DEBUG:boto:Method: GET 2022-09-03T06:07:12.744 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.744 DEBUG:boto:Data: 2022-09-03T06:07:12.745 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.745 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.746 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.747 DEBUG:boto:Params: {} 2022-09-03T06:07:12.747 DEBUG:boto:Token: None 2022-09-03T06:07:12.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa3d37f789588fda-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.753 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.754 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.754 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.755 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.755 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.756 DEBUG:boto:Method: GET 2022-09-03T06:07:12.756 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.757 DEBUG:boto:Data: 2022-09-03T06:07:12.757 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.759 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.759 DEBUG:boto:Params: {} 2022-09-03T06:07:12.760 DEBUG:boto:Token: None 2022-09-03T06:07:12.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.764 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002bb28d1f7fd499b9-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.765 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.766 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.766 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.767 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.767 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:12.768 DEBUG:boto:Data: 2022-09-03T06:07:12.768 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.769 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.769 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.770 DEBUG:boto:Params: {} 2022-09-03T06:07:12.770 DEBUG:boto:Token: None 2022-09-03T06:07:12.771 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj 2022-09-03T06:07:12.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U= 2022-09-03T06:07:12.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U=', 'Content-Length': '0'} 2022-09-03T06:07:12.776 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000171b523d28fb97a1-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.776 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.777 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.777 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.778 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:12.779 DEBUG:boto:Data: 2022-09-03T06:07:12.779 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.780 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.780 DEBUG:boto:Params: {} 2022-09-03T06:07:12.781 DEBUG:boto:Token: None 2022-09-03T06:07:12.782 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj 2022-09-03T06:07:12.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U= 2022-09-03T06:07:12.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fV84qRTWggG1YvvqbkqnZ/Yd28U=', 'Content-Length': '0'} 2022-09-03T06:07:12.786 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000096c87888aba358cf-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.787 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.787 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.788 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.788 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.789 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.789 DEBUG:boto:Method: GET 2022-09-03T06:07:12.790 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.790 DEBUG:boto:Data: 2022-09-03T06:07:12.791 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.791 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.792 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.792 DEBUG:boto:Params: {} 2022-09-03T06:07:12.793 DEBUG:boto:Token: None 2022-09-03T06:07:12.793 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.797 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fc547fc92dd4e1b5-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.847 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.848 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.849 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.850 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.851 DEBUG:boto:Method: GET 2022-09-03T06:07:12.852 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.853 DEBUG:boto:Data: 2022-09-03T06:07:12.854 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.855 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.856 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.857 DEBUG:boto:Params: {} 2022-09-03T06:07:12.858 DEBUG:boto:Token: None 2022-09-03T06:07:12.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:12.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo= 2022-09-03T06:07:12.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c6aJd5v7TZIDf5RJVUTgz3UAqNo=', 'Content-Length': '0'} 2022-09-03T06:07:12.863 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009ee9d3d89701b646-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.864 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.865 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.865 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.866 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.866 DEBUG:boto:Method: GET 2022-09-03T06:07:12.867 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.867 DEBUG:boto:Data: 2022-09-03T06:07:12.868 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.868 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.869 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.869 DEBUG:boto:Params: {} 2022-09-03T06:07:12.870 DEBUG:boto:Token: None 2022-09-03T06:07:12.871 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.871 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.872 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.875 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a85dd53caf4638c-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.876 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.876 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:12.877 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:12.878 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.878 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:12.879 DEBUG:boto:Method: GET 2022-09-03T06:07:12.879 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:12.880 DEBUG:boto:Data: 2022-09-03T06:07:12.880 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.881 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.882 DEBUG:boto:Params: {} 2022-09-03T06:07:12.882 DEBUG:boto:Token: None 2022-09-03T06:07:12.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/ 2022-09-03T06:07:12.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM= 2022-09-03T06:07:12.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bc3k2f3769Vw3OC6pEr0AktLNkM=', 'Content-Length': '0'} 2022-09-03T06:07:12.887 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000050cf00964924887d-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.888 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:12.889 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.889 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.890 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.890 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.891 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.892 DEBUG:boto:Method: GET 2022-09-03T06:07:12.892 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.893 DEBUG:boto:Data: 2022-09-03T06:07:12.893 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.894 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.894 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.895 DEBUG:boto:Params: {} 2022-09-03T06:07:12.895 DEBUG:boto:Token: None 2022-09-03T06:07:12.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R2o4mAcL/2QsoO8wTTfMMvS1X70= 2022-09-03T06:07:12.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R2o4mAcL/2QsoO8wTTfMMvS1X70=', 'Content-Length': '0'} 2022-09-03T06:07:12.900 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a57140bec6e0e974-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.901 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.902 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.903 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.905 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.905 DEBUG:boto:Method: GET 2022-09-03T06:07:12.906 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.906 DEBUG:boto:Data: 2022-09-03T06:07:12.907 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.908 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.908 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.909 DEBUG:boto:Params: {} 2022-09-03T06:07:12.909 DEBUG:boto:Token: None 2022-09-03T06:07:12.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R2o4mAcL/2QsoO8wTTfMMvS1X70= 2022-09-03T06:07:12.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R2o4mAcL/2QsoO8wTTfMMvS1X70=', 'Content-Length': '0'} 2022-09-03T06:07:12.914 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000650f454886dfd05c-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.954 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.955 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.956 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.956 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.957 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.958 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.959 DEBUG:boto:Data: 2022-09-03T06:07:12.960 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.960 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.961 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.962 DEBUG:boto:Params: {} 2022-09-03T06:07:12.962 DEBUG:boto:Token: None 2022-09-03T06:07:12.963 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.964 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XTOJG2MpbEhkV+wUVQ0HZSX4ico= 2022-09-03T06:07:12.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XTOJG2MpbEhkV+wUVQ0HZSX4ico=', 'Content-Length': '0'} 2022-09-03T06:07:12.968 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006a048e727a377996-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.969 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.969 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.970 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.970 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.971 DEBUG:boto:Method: HEAD 2022-09-03T06:07:12.971 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.972 DEBUG:boto:Data: 2022-09-03T06:07:12.972 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.973 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:12.973 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.974 DEBUG:boto:Params: {} 2022-09-03T06:07:12.974 DEBUG:boto:Token: None 2022-09-03T06:07:12.975 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XTOJG2MpbEhkV+wUVQ0HZSX4ico= 2022-09-03T06:07:12.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XTOJG2MpbEhkV+wUVQ0HZSX4ico=', 'Content-Length': '0'} 2022-09-03T06:07:12.979 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ca565d772ece496e-006312ef10-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:12.980 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:12.980 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:12.981 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:12.982 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.982 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.983 DEBUG:boto:Method: GET 2022-09-03T06:07:12.983 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.984 DEBUG:boto:Data: 2022-09-03T06:07:12.984 DEBUG:boto:Headers: {} 2022-09-03T06:07:12.985 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:12.986 DEBUG:boto:Port: 8000 2022-09-03T06:07:12.986 DEBUG:boto:Params: {} 2022-09-03T06:07:12.987 DEBUG:boto:Token: None 2022-09-03T06:07:12.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:12 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:12.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R2o4mAcL/2QsoO8wTTfMMvS1X70= 2022-09-03T06:07:12.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R2o4mAcL/2QsoO8wTTfMMvS1X70=', 'Content-Length': '0'} 2022-09-03T06:07:12.992 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003b6f2f3a134f56ac-006312ef10-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.034 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.035 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.036 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:13.036 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:13.037 DEBUG:boto:Method: GET 2022-09-03T06:07:13.037 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:13.038 DEBUG:boto:Data: 2022-09-03T06:07:13.038 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.039 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.039 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.040 DEBUG:boto:Params: {} 2022-09-03T06:07:13.040 DEBUG:boto:Token: None 2022-09-03T06:07:13.041 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:13.042 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qr73WKvd7mV5KHabRiM3KPRwCE4= 2022-09-03T06:07:13.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qr73WKvd7mV5KHabRiM3KPRwCE4=', 'Content-Length': '0'} 2022-09-03T06:07:13.051 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009c447ccf37acae50-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.052 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.053 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.053 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.054 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.055 DEBUG:boto:Method: GET 2022-09-03T06:07:13.055 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.056 DEBUG:boto:Data: 2022-09-03T06:07:13.057 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.057 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.058 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.059 DEBUG:boto:Params: {} 2022-09-03T06:07:13.061 DEBUG:boto:Token: None 2022-09-03T06:07:13.062 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb1c28e11f1254dd-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.072 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.074 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.075 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.076 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.077 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.077 DEBUG:boto:Method: GET 2022-09-03T06:07:13.078 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.079 DEBUG:boto:Data: 2022-09-03T06:07:13.079 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.080 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.081 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.082 DEBUG:boto:Params: {} 2022-09-03T06:07:13.083 DEBUG:boto:Token: None 2022-09-03T06:07:13.084 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.086 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.089 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000035735b7afb291cad-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.090 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.091 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.092 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.093 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.094 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:13.095 DEBUG:boto:Data: 2022-09-03T06:07:13.095 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.096 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.097 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.099 DEBUG:boto:Params: {} 2022-09-03T06:07:13.101 DEBUG:boto:Token: None 2022-09-03T06:07:13.101 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj 2022-09-03T06:07:13.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM= 2022-09-03T06:07:13.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM=', 'Content-Length': '0'} 2022-09-03T06:07:13.107 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000707edae46eafef2f-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.108 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.108 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.109 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.109 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:13.110 DEBUG:boto:Data: 2022-09-03T06:07:13.111 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.112 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.112 DEBUG:boto:Params: {} 2022-09-03T06:07:13.113 DEBUG:boto:Token: None 2022-09-03T06:07:13.114 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj 2022-09-03T06:07:13.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM= 2022-09-03T06:07:13.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM=', 'Content-Length': '0'} 2022-09-03T06:07:13.119 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f24047c16b508d0e-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.120 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.121 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.121 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.122 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.122 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.123 DEBUG:boto:Method: GET 2022-09-03T06:07:13.123 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.124 DEBUG:boto:Data: 2022-09-03T06:07:13.124 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.125 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.125 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.126 DEBUG:boto:Params: {} 2022-09-03T06:07:13.127 DEBUG:boto:Token: None 2022-09-03T06:07:13.128 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.129 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs= 2022-09-03T06:07:13.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs=', 'Content-Length': '0'} 2022-09-03T06:07:13.133 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000510bdc6eff3bded0-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.135 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.136 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.137 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.138 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.139 DEBUG:boto:Method: GET 2022-09-03T06:07:13.140 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.141 DEBUG:boto:Data: 2022-09-03T06:07:13.142 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.144 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.145 DEBUG:boto:Params: {} 2022-09-03T06:07:13.146 DEBUG:boto:Token: None 2022-09-03T06:07:13.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs= 2022-09-03T06:07:13.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs=', 'Content-Length': '0'} 2022-09-03T06:07:13.153 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009e6e4d11bd54eb41-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.198 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.199 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.201 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.202 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.203 DEBUG:boto:Method: GET 2022-09-03T06:07:13.204 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.205 DEBUG:boto:Data: 2022-09-03T06:07:13.206 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.207 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.210 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.211 DEBUG:boto:Params: {} 2022-09-03T06:07:13.211 DEBUG:boto:Token: None 2022-09-03T06:07:13.212 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.216 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000089e50d669213ac1d-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.217 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.218 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.219 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.219 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.220 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.220 DEBUG:boto:Method: GET 2022-09-03T06:07:13.221 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.221 DEBUG:boto:Data: 2022-09-03T06:07:13.222 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.223 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.223 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.224 DEBUG:boto:Params: {} 2022-09-03T06:07:13.224 DEBUG:boto:Token: None 2022-09-03T06:07:13.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.229 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac90936c5d3de10f-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.231 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.232 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.233 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.234 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.234 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.235 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.236 DEBUG:boto:Method: GET 2022-09-03T06:07:13.236 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.237 DEBUG:boto:Data: 2022-09-03T06:07:13.237 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.238 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.239 DEBUG:boto:Params: {} 2022-09-03T06:07:13.240 DEBUG:boto:Token: None 2022-09-03T06:07:13.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kuNZXG3RaIPDuD9LZE+ipMiYeMY= 2022-09-03T06:07:13.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kuNZXG3RaIPDuD9LZE+ipMiYeMY=', 'Content-Length': '0'} 2022-09-03T06:07:13.244 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000f368f0750f67fec-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.245 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.246 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.246 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.247 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.247 DEBUG:boto:Method: GET 2022-09-03T06:07:13.248 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.249 DEBUG:boto:Data: 2022-09-03T06:07:13.250 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.252 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.253 DEBUG:boto:Params: {} 2022-09-03T06:07:13.254 DEBUG:boto:Token: None 2022-09-03T06:07:13.255 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kuNZXG3RaIPDuD9LZE+ipMiYeMY= 2022-09-03T06:07:13.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kuNZXG3RaIPDuD9LZE+ipMiYeMY=', 'Content-Length': '0'} 2022-09-03T06:07:13.260 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000038f4c526b7a4affe-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.261 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.262 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.262 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.263 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.263 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.264 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.265 DEBUG:boto:Data: 2022-09-03T06:07:13.265 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.266 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.266 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.267 DEBUG:boto:Params: {} 2022-09-03T06:07:13.267 DEBUG:boto:Token: None 2022-09-03T06:07:13.268 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x6mRFC2b1WGM2tcLZRAt/UWw69M= 2022-09-03T06:07:13.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x6mRFC2b1WGM2tcLZRAt/UWw69M=', 'Content-Length': '0'} 2022-09-03T06:07:13.272 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e109e17b1b40d785-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.273 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.273 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.274 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.274 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.275 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.276 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.276 DEBUG:boto:Data: 2022-09-03T06:07:13.277 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.278 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.278 DEBUG:boto:Params: {} 2022-09-03T06:07:13.279 DEBUG:boto:Token: None 2022-09-03T06:07:13.279 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x6mRFC2b1WGM2tcLZRAt/UWw69M= 2022-09-03T06:07:13.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x6mRFC2b1WGM2tcLZRAt/UWw69M=', 'Content-Length': '0'} 2022-09-03T06:07:13.283 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002011071afb8f4158-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.283 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.284 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.285 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.285 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.286 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.286 DEBUG:boto:Method: GET 2022-09-03T06:07:13.287 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.287 DEBUG:boto:Data: 2022-09-03T06:07:13.288 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.289 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.289 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.290 DEBUG:boto:Params: {} 2022-09-03T06:07:13.290 DEBUG:boto:Token: None 2022-09-03T06:07:13.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kuNZXG3RaIPDuD9LZE+ipMiYeMY= 2022-09-03T06:07:13.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kuNZXG3RaIPDuD9LZE+ipMiYeMY=', 'Content-Length': '0'} 2022-09-03T06:07:13.295 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005aa0875ed7ae2204-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.296 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.297 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.298 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.298 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.299 DEBUG:boto:Method: GET 2022-09-03T06:07:13.300 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.301 DEBUG:boto:Data: 2022-09-03T06:07:13.301 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.302 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.302 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.303 DEBUG:boto:Params: {} 2022-09-03T06:07:13.303 DEBUG:boto:Token: None 2022-09-03T06:07:13.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:13.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kuNZXG3RaIPDuD9LZE+ipMiYeMY= 2022-09-03T06:07:13.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kuNZXG3RaIPDuD9LZE+ipMiYeMY=', 'Content-Length': '0'} 2022-09-03T06:07:13.308 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002a5b18e9dc15a062-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.309 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.309 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.310 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.310 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.311 DEBUG:boto:Method: GET 2022-09-03T06:07:13.311 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.312 DEBUG:boto:Data: 2022-09-03T06:07:13.313 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.313 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.314 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.314 DEBUG:boto:Params: {} 2022-09-03T06:07:13.315 DEBUG:boto:Token: None 2022-09-03T06:07:13.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.321 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000004abddc0cc27e195-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.322 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.323 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.323 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.324 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.324 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.325 DEBUG:boto:Method: GET 2022-09-03T06:07:13.325 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.326 DEBUG:boto:Data: 2022-09-03T06:07:13.327 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.327 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.328 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.328 DEBUG:boto:Params: {} 2022-09-03T06:07:13.329 DEBUG:boto:Token: None 2022-09-03T06:07:13.330 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.334 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005f019371c4e95d00-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.335 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.336 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.337 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.338 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.338 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:13.339 DEBUG:boto:Data: 2022-09-03T06:07:13.340 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.341 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.341 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.342 DEBUG:boto:Params: {} 2022-09-03T06:07:13.343 DEBUG:boto:Token: None 2022-09-03T06:07:13.344 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj 2022-09-03T06:07:13.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM= 2022-09-03T06:07:13.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM=', 'Content-Length': '0'} 2022-09-03T06:07:13.350 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000097436b12b94a1331-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.351 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.352 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.353 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.354 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:13.354 DEBUG:boto:Data: 2022-09-03T06:07:13.355 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.357 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.358 DEBUG:boto:Params: {} 2022-09-03T06:07:13.358 DEBUG:boto:Token: None 2022-09-03T06:07:13.360 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj 2022-09-03T06:07:13.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM= 2022-09-03T06:07:13.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM=', 'Content-Length': '0'} 2022-09-03T06:07:13.365 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a75fa2f8520cbc8e-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.366 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.367 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.367 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.368 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.368 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.369 DEBUG:boto:Method: GET 2022-09-03T06:07:13.369 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.370 DEBUG:boto:Data: 2022-09-03T06:07:13.371 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.371 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.372 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.372 DEBUG:boto:Params: {} 2022-09-03T06:07:13.373 DEBUG:boto:Token: None 2022-09-03T06:07:13.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs= 2022-09-03T06:07:13.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs=', 'Content-Length': '0'} 2022-09-03T06:07:13.378 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002c08644a05bf8bda-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.422 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.423 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.423 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.424 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.424 DEBUG:boto:Method: GET 2022-09-03T06:07:13.425 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.426 DEBUG:boto:Data: 2022-09-03T06:07:13.426 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.427 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.428 DEBUG:boto:Params: {} 2022-09-03T06:07:13.428 DEBUG:boto:Token: None 2022-09-03T06:07:13.429 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs= 2022-09-03T06:07:13.430 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs=', 'Content-Length': '0'} 2022-09-03T06:07:13.432 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be2018e2e2062ad3-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.433 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.434 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.435 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.435 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.436 DEBUG:boto:Method: GET 2022-09-03T06:07:13.436 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.437 DEBUG:boto:Data: 2022-09-03T06:07:13.437 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.438 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.439 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.440 DEBUG:boto:Params: {} 2022-09-03T06:07:13.441 DEBUG:boto:Token: None 2022-09-03T06:07:13.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.445 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006d42c3d6bf8517d1-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.446 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.447 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.447 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.448 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.448 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.449 DEBUG:boto:Method: GET 2022-09-03T06:07:13.449 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.450 DEBUG:boto:Data: 2022-09-03T06:07:13.451 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.452 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.452 DEBUG:boto:Params: {} 2022-09-03T06:07:13.453 DEBUG:boto:Token: None 2022-09-03T06:07:13.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.457 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000068b043c375a254a3-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.458 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.459 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.460 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.460 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.461 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.461 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.462 DEBUG:boto:Method: GET 2022-09-03T06:07:13.463 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.463 DEBUG:boto:Data: 2022-09-03T06:07:13.464 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.464 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.465 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.465 DEBUG:boto:Params: {} 2022-09-03T06:07:13.466 DEBUG:boto:Token: None 2022-09-03T06:07:13.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyaAjMRrKaWKA5dHcBmbpCGkBg0= 2022-09-03T06:07:13.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyaAjMRrKaWKA5dHcBmbpCGkBg0=', 'Content-Length': '0'} 2022-09-03T06:07:13.470 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000741fc64e53ab3d8c-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.471 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.472 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.472 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.473 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.473 DEBUG:boto:Method: GET 2022-09-03T06:07:13.474 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.475 DEBUG:boto:Data: 2022-09-03T06:07:13.475 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.476 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.477 DEBUG:boto:Params: {} 2022-09-03T06:07:13.477 DEBUG:boto:Token: None 2022-09-03T06:07:13.478 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyaAjMRrKaWKA5dHcBmbpCGkBg0= 2022-09-03T06:07:13.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyaAjMRrKaWKA5dHcBmbpCGkBg0=', 'Content-Length': '0'} 2022-09-03T06:07:13.481 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a5fe32420f350488-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.482 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.483 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.483 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.484 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.484 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.485 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.485 DEBUG:boto:Data: 2022-09-03T06:07:13.486 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.486 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.487 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.487 DEBUG:boto:Params: {} 2022-09-03T06:07:13.488 DEBUG:boto:Token: None 2022-09-03T06:07:13.488 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sH39rMLqR1w6WoBN0rFtaUR65A8= 2022-09-03T06:07:13.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sH39rMLqR1w6WoBN0rFtaUR65A8=', 'Content-Length': '0'} 2022-09-03T06:07:13.492 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006e46f1422df83c6f-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.493 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.493 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.494 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.494 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.495 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.495 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.496 DEBUG:boto:Data: 2022-09-03T06:07:13.497 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.497 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.498 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.498 DEBUG:boto:Params: {} 2022-09-03T06:07:13.499 DEBUG:boto:Token: None 2022-09-03T06:07:13.499 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sH39rMLqR1w6WoBN0rFtaUR65A8= 2022-09-03T06:07:13.501 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sH39rMLqR1w6WoBN0rFtaUR65A8=', 'Content-Length': '0'} 2022-09-03T06:07:13.503 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b54ee3e2c76dcfc8-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.503 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.504 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.504 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.505 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.505 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.506 DEBUG:boto:Method: GET 2022-09-03T06:07:13.507 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.507 DEBUG:boto:Data: 2022-09-03T06:07:13.508 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.508 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.509 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.509 DEBUG:boto:Params: {} 2022-09-03T06:07:13.510 DEBUG:boto:Token: None 2022-09-03T06:07:13.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyaAjMRrKaWKA5dHcBmbpCGkBg0= 2022-09-03T06:07:13.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyaAjMRrKaWKA5dHcBmbpCGkBg0=', 'Content-Length': '0'} 2022-09-03T06:07:13.514 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000924222921ed115c4-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.515 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.515 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.516 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.517 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.517 DEBUG:boto:Method: GET 2022-09-03T06:07:13.518 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.518 DEBUG:boto:Data: 2022-09-03T06:07:13.519 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.519 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.520 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.520 DEBUG:boto:Params: {} 2022-09-03T06:07:13.521 DEBUG:boto:Token: None 2022-09-03T06:07:13.521 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:13.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyaAjMRrKaWKA5dHcBmbpCGkBg0= 2022-09-03T06:07:13.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyaAjMRrKaWKA5dHcBmbpCGkBg0=', 'Content-Length': '0'} 2022-09-03T06:07:13.525 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f0ca502ba1853832-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.526 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.526 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.527 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.528 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.528 DEBUG:boto:Method: GET 2022-09-03T06:07:13.529 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.529 DEBUG:boto:Data: 2022-09-03T06:07:13.530 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.530 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.531 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.531 DEBUG:boto:Params: {} 2022-09-03T06:07:13.532 DEBUG:boto:Token: None 2022-09-03T06:07:13.533 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.537 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a23a07e796d47302-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.582 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.583 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.584 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.584 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.586 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.586 DEBUG:boto:Method: GET 2022-09-03T06:07:13.587 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.587 DEBUG:boto:Data: 2022-09-03T06:07:13.588 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.588 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.589 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.589 DEBUG:boto:Params: {} 2022-09-03T06:07:13.590 DEBUG:boto:Token: None 2022-09-03T06:07:13.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005583e0f7b43f4e00-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.596 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.597 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.598 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.598 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.599 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:13.599 DEBUG:boto:Data: 2022-09-03T06:07:13.600 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.601 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.601 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.602 DEBUG:boto:Params: {} 2022-09-03T06:07:13.602 DEBUG:boto:Token: None 2022-09-03T06:07:13.603 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj 2022-09-03T06:07:13.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM= 2022-09-03T06:07:13.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM=', 'Content-Length': '0'} 2022-09-03T06:07:13.609 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000033e81653c48428d2-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.610 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.611 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.612 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.613 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:13.614 DEBUG:boto:Data: 2022-09-03T06:07:13.615 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.617 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.617 DEBUG:boto:Params: {} 2022-09-03T06:07:13.618 DEBUG:boto:Token: None 2022-09-03T06:07:13.618 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj 2022-09-03T06:07:13.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM= 2022-09-03T06:07:13.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM=', 'Content-Length': '0'} 2022-09-03T06:07:13.623 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000076c87512e9dc5293-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.624 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.624 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.625 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.625 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.626 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.627 DEBUG:boto:Method: GET 2022-09-03T06:07:13.627 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.628 DEBUG:boto:Data: 2022-09-03T06:07:13.628 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.629 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.629 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.630 DEBUG:boto:Params: {} 2022-09-03T06:07:13.631 DEBUG:boto:Token: None 2022-09-03T06:07:13.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs= 2022-09-03T06:07:13.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs=', 'Content-Length': '0'} 2022-09-03T06:07:13.635 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ad9155d48af14c26-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.636 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.637 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.637 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.638 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.638 DEBUG:boto:Method: GET 2022-09-03T06:07:13.639 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.640 DEBUG:boto:Data: 2022-09-03T06:07:13.640 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.641 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.642 DEBUG:boto:Params: {} 2022-09-03T06:07:13.643 DEBUG:boto:Token: None 2022-09-03T06:07:13.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs= 2022-09-03T06:07:13.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs=', 'Content-Length': '0'} 2022-09-03T06:07:13.647 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000091b72e185b8c3d97-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.690 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.691 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.691 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.692 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.692 DEBUG:boto:Method: GET 2022-09-03T06:07:13.693 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.693 DEBUG:boto:Data: 2022-09-03T06:07:13.694 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.695 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.695 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.696 DEBUG:boto:Params: {} 2022-09-03T06:07:13.697 DEBUG:boto:Token: None 2022-09-03T06:07:13.697 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.702 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cbf29e7b1b3e223d-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.703 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.704 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.705 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.706 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.707 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.708 DEBUG:boto:Method: GET 2022-09-03T06:07:13.709 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.710 DEBUG:boto:Data: 2022-09-03T06:07:13.711 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.714 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.715 DEBUG:boto:Params: {} 2022-09-03T06:07:13.718 DEBUG:boto:Token: None 2022-09-03T06:07:13.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.724 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004ad79daf7805b698-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.725 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.725 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.726 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.727 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.731 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.731 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.732 DEBUG:boto:Method: GET 2022-09-03T06:07:13.732 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.733 DEBUG:boto:Data: 2022-09-03T06:07:13.734 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.734 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.735 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.735 DEBUG:boto:Params: {} 2022-09-03T06:07:13.736 DEBUG:boto:Token: None 2022-09-03T06:07:13.737 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MPbLNN9LMgdFH9+uwq7D7vPaPik= 2022-09-03T06:07:13.738 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MPbLNN9LMgdFH9+uwq7D7vPaPik=', 'Content-Length': '0'} 2022-09-03T06:07:13.740 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001be882268bcfb0e7-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.782 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.783 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.784 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.785 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.786 DEBUG:boto:Method: GET 2022-09-03T06:07:13.787 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.788 DEBUG:boto:Data: 2022-09-03T06:07:13.789 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.790 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.791 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.792 DEBUG:boto:Params: {} 2022-09-03T06:07:13.793 DEBUG:boto:Token: None 2022-09-03T06:07:13.794 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MPbLNN9LMgdFH9+uwq7D7vPaPik= 2022-09-03T06:07:13.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MPbLNN9LMgdFH9+uwq7D7vPaPik=', 'Content-Length': '0'} 2022-09-03T06:07:13.799 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000966dbeffa4785a32-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.801 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.802 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.803 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.804 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.805 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.806 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.807 DEBUG:boto:Data: 2022-09-03T06:07:13.808 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.809 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.810 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.811 DEBUG:boto:Params: {} 2022-09-03T06:07:13.812 DEBUG:boto:Token: None 2022-09-03T06:07:13.813 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Zq1kCN9ugcUWRnEIN08UlWvcvw= 2022-09-03T06:07:13.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Zq1kCN9ugcUWRnEIN08UlWvcvw=', 'Content-Length': '0'} 2022-09-03T06:07:13.818 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000041273293e437628f-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.819 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.820 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.821 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.822 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.823 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.823 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.824 DEBUG:boto:Data: 2022-09-03T06:07:13.825 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.827 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.828 DEBUG:boto:Params: {} 2022-09-03T06:07:13.829 DEBUG:boto:Token: None 2022-09-03T06:07:13.830 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Zq1kCN9ugcUWRnEIN08UlWvcvw= 2022-09-03T06:07:13.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Zq1kCN9ugcUWRnEIN08UlWvcvw=', 'Content-Length': '0'} 2022-09-03T06:07:13.835 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ae95f73834ddde74-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.835 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.836 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.837 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.837 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.838 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.838 DEBUG:boto:Method: GET 2022-09-03T06:07:13.839 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.839 DEBUG:boto:Data: 2022-09-03T06:07:13.840 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.841 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.841 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.842 DEBUG:boto:Params: {} 2022-09-03T06:07:13.843 DEBUG:boto:Token: None 2022-09-03T06:07:13.843 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.844 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MPbLNN9LMgdFH9+uwq7D7vPaPik= 2022-09-03T06:07:13.845 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MPbLNN9LMgdFH9+uwq7D7vPaPik=', 'Content-Length': '0'} 2022-09-03T06:07:13.847 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000860bd9d24fe5c1bd-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.848 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.848 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.849 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.849 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.850 DEBUG:boto:Method: GET 2022-09-03T06:07:13.851 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.851 DEBUG:boto:Data: 2022-09-03T06:07:13.852 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.852 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.853 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.853 DEBUG:boto:Params: {} 2022-09-03T06:07:13.854 DEBUG:boto:Token: None 2022-09-03T06:07:13.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:13.855 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MPbLNN9LMgdFH9+uwq7D7vPaPik= 2022-09-03T06:07:13.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MPbLNN9LMgdFH9+uwq7D7vPaPik=', 'Content-Length': '0'} 2022-09-03T06:07:13.858 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001d2e9c2dc9650ee1-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.902 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.903 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.903 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.904 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.905 DEBUG:boto:Method: GET 2022-09-03T06:07:13.906 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.906 DEBUG:boto:Data: 2022-09-03T06:07:13.907 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.907 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.908 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.908 DEBUG:boto:Params: {} 2022-09-03T06:07:13.909 DEBUG:boto:Token: None 2022-09-03T06:07:13.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.915 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fa84d16c6192e1f7-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.916 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.917 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:13.917 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:13.918 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.919 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:13.919 DEBUG:boto:Method: GET 2022-09-03T06:07:13.920 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:13.920 DEBUG:boto:Data: 2022-09-03T06:07:13.921 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.922 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.922 DEBUG:boto:Params: {} 2022-09-03T06:07:13.923 DEBUG:boto:Token: None 2022-09-03T06:07:13.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/ 2022-09-03T06:07:13.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss= 2022-09-03T06:07:13.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tckvDV/M79ip6GTH85LymWZgkss=', 'Content-Length': '0'} 2022-09-03T06:07:13.928 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f91b499bf891ecbf-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.929 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:13.930 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.930 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.931 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.931 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:13.932 DEBUG:boto:Data: 2022-09-03T06:07:13.933 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.933 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.934 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.934 DEBUG:boto:Params: {} 2022-09-03T06:07:13.935 DEBUG:boto:Token: None 2022-09-03T06:07:13.935 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj 2022-09-03T06:07:13.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM= 2022-09-03T06:07:13.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM=', 'Content-Length': '0'} 2022-09-03T06:07:13.941 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000bfc4eef9353adac-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.942 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.943 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.944 DEBUG:boto:Method: HEAD 2022-09-03T06:07:13.944 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:13.945 DEBUG:boto:Data: 2022-09-03T06:07:13.945 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.946 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:13.946 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.947 DEBUG:boto:Params: {} 2022-09-03T06:07:13.948 DEBUG:boto:Token: None 2022-09-03T06:07:13.948 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj 2022-09-03T06:07:13.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM= 2022-09-03T06:07:13.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ix50wjsQZbXC4Zo2mT0LxyUaM=', 'Content-Length': '0'} 2022-09-03T06:07:13.959 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004a7f618fb7bff4fd-006312ef11-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:13.960 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:13.961 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:13.962 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:13.963 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.964 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.965 DEBUG:boto:Method: GET 2022-09-03T06:07:13.966 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.968 DEBUG:boto:Data: 2022-09-03T06:07:13.969 DEBUG:boto:Headers: {} 2022-09-03T06:07:13.970 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:13.971 DEBUG:boto:Port: 8000 2022-09-03T06:07:13.972 DEBUG:boto:Params: {} 2022-09-03T06:07:13.973 DEBUG:boto:Token: None 2022-09-03T06:07:13.975 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:13 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:13.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs= 2022-09-03T06:07:13.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nhHhniVeJDEcyhE4ij8VHAeHUqs=', 'Content-Length': '0'} 2022-09-03T06:07:13.981 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f1f7441cf95851b6-006312ef11-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.022 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.023 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.023 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.024 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.026 DEBUG:boto:Method: GET 2022-09-03T06:07:14.027 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.027 DEBUG:boto:Data: 2022-09-03T06:07:14.028 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.030 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.031 DEBUG:boto:Params: {} 2022-09-03T06:07:14.033 DEBUG:boto:Token: None 2022-09-03T06:07:14.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg= 2022-09-03T06:07:14.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg=', 'Content-Length': '0'} 2022-09-03T06:07:14.040 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002cfbca33d941c078-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.042 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.044 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.045 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.046 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.047 DEBUG:boto:Method: GET 2022-09-03T06:07:14.048 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.049 DEBUG:boto:Data: 2022-09-03T06:07:14.050 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.051 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.052 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.053 DEBUG:boto:Params: {} 2022-09-03T06:07:14.054 DEBUG:boto:Token: None 2022-09-03T06:07:14.055 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.057 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.061 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e561bdbd5280f8c-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.062 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.063 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.064 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.064 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.065 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.065 DEBUG:boto:Method: GET 2022-09-03T06:07:14.066 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.066 DEBUG:boto:Data: 2022-09-03T06:07:14.067 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.068 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.068 DEBUG:boto:Params: {} 2022-09-03T06:07:14.069 DEBUG:boto:Token: None 2022-09-03T06:07:14.069 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.074 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f5f0226b790ac3fa-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.075 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.076 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.077 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.078 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.079 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.086 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.087 DEBUG:boto:Method: GET 2022-09-03T06:07:14.088 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.089 DEBUG:boto:Data: 2022-09-03T06:07:14.090 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.091 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.092 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.094 DEBUG:boto:Params: {} 2022-09-03T06:07:14.095 DEBUG:boto:Token: None 2022-09-03T06:07:14.096 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iqcKVhzx2hinzRtXlhZkIxp5dEg= 2022-09-03T06:07:14.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iqcKVhzx2hinzRtXlhZkIxp5dEg=', 'Content-Length': '0'} 2022-09-03T06:07:14.101 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007f5deb5860961bf8-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.103 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.104 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.105 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.106 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.107 DEBUG:boto:Method: GET 2022-09-03T06:07:14.108 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.109 DEBUG:boto:Data: 2022-09-03T06:07:14.110 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.112 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.113 DEBUG:boto:Params: {} 2022-09-03T06:07:14.114 DEBUG:boto:Token: None 2022-09-03T06:07:14.116 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iqcKVhzx2hinzRtXlhZkIxp5dEg= 2022-09-03T06:07:14.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iqcKVhzx2hinzRtXlhZkIxp5dEg=', 'Content-Length': '0'} 2022-09-03T06:07:14.121 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000015198bc02be63e70-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.123 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.124 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.125 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.126 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.127 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.128 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.128 DEBUG:boto:Data: 2022-09-03T06:07:14.129 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.130 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.131 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.132 DEBUG:boto:Params: {} 2022-09-03T06:07:14.133 DEBUG:boto:Token: None 2022-09-03T06:07:14.134 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8nat/b2BPTUXWeTybkruEkUJxUM= 2022-09-03T06:07:14.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8nat/b2BPTUXWeTybkruEkUJxUM=', 'Content-Length': '0'} 2022-09-03T06:07:14.141 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004c337bad5f2cdd69-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.143 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.144 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.145 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.146 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.147 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.148 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.149 DEBUG:boto:Data: 2022-09-03T06:07:14.150 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.151 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.152 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.153 DEBUG:boto:Params: {} 2022-09-03T06:07:14.154 DEBUG:boto:Token: None 2022-09-03T06:07:14.155 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8nat/b2BPTUXWeTybkruEkUJxUM= 2022-09-03T06:07:14.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8nat/b2BPTUXWeTybkruEkUJxUM=', 'Content-Length': '0'} 2022-09-03T06:07:14.160 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000085f701999f3a57d3-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.161 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.162 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.163 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.164 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.165 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.166 DEBUG:boto:Method: GET 2022-09-03T06:07:14.167 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.168 DEBUG:boto:Data: 2022-09-03T06:07:14.169 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.170 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.171 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.172 DEBUG:boto:Params: {} 2022-09-03T06:07:14.173 DEBUG:boto:Token: None 2022-09-03T06:07:14.174 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iqcKVhzx2hinzRtXlhZkIxp5dEg= 2022-09-03T06:07:14.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iqcKVhzx2hinzRtXlhZkIxp5dEg=', 'Content-Length': '0'} 2022-09-03T06:07:14.180 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fefd574458e71efd-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.182 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.183 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.184 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.185 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.186 DEBUG:boto:Method: GET 2022-09-03T06:07:14.187 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.188 DEBUG:boto:Data: 2022-09-03T06:07:14.188 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.189 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.190 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.191 DEBUG:boto:Params: {} 2022-09-03T06:07:14.191 DEBUG:boto:Token: None 2022-09-03T06:07:14.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:14.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iqcKVhzx2hinzRtXlhZkIxp5dEg= 2022-09-03T06:07:14.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iqcKVhzx2hinzRtXlhZkIxp5dEg=', 'Content-Length': '0'} 2022-09-03T06:07:14.196 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008bf741687bf3bcd1-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.197 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.197 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.198 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.198 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.199 DEBUG:boto:Method: GET 2022-09-03T06:07:14.199 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.200 DEBUG:boto:Data: 2022-09-03T06:07:14.201 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.201 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.202 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.202 DEBUG:boto:Params: {} 2022-09-03T06:07:14.203 DEBUG:boto:Token: None 2022-09-03T06:07:14.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.208 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6300cbd206b1d49-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.209 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.211 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.212 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.212 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.213 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.213 DEBUG:boto:Method: GET 2022-09-03T06:07:14.214 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.214 DEBUG:boto:Data: 2022-09-03T06:07:14.215 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.216 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.216 DEBUG:boto:Params: {} 2022-09-03T06:07:14.217 DEBUG:boto:Token: None 2022-09-03T06:07:14.218 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.222 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001aef56af428b3a82-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.223 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.225 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.226 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.226 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.227 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:14.227 DEBUG:boto:Data: 2022-09-03T06:07:14.228 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.229 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.230 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.230 DEBUG:boto:Params: {} 2022-09-03T06:07:14.231 DEBUG:boto:Token: None 2022-09-03T06:07:14.232 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj 2022-09-03T06:07:14.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA= 2022-09-03T06:07:14.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA=', 'Content-Length': '0'} 2022-09-03T06:07:14.238 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003a48702aea572c82-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.239 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.239 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.240 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.240 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:14.241 DEBUG:boto:Data: 2022-09-03T06:07:14.241 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.242 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.242 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.243 DEBUG:boto:Params: {} 2022-09-03T06:07:14.243 DEBUG:boto:Token: None 2022-09-03T06:07:14.244 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj 2022-09-03T06:07:14.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA= 2022-09-03T06:07:14.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA=', 'Content-Length': '0'} 2022-09-03T06:07:14.248 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000385b16cde700bea6-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.249 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.250 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.250 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.251 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.252 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.252 DEBUG:boto:Method: GET 2022-09-03T06:07:14.253 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.253 DEBUG:boto:Data: 2022-09-03T06:07:14.254 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.254 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.255 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.255 DEBUG:boto:Params: {} 2022-09-03T06:07:14.256 DEBUG:boto:Token: None 2022-09-03T06:07:14.257 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg= 2022-09-03T06:07:14.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg=', 'Content-Length': '0'} 2022-09-03T06:07:14.260 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000016fa7b08515d432d-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.261 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.262 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.262 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.263 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.264 DEBUG:boto:Method: GET 2022-09-03T06:07:14.264 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.265 DEBUG:boto:Data: 2022-09-03T06:07:14.265 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.266 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.266 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.267 DEBUG:boto:Params: {} 2022-09-03T06:07:14.268 DEBUG:boto:Token: None 2022-09-03T06:07:14.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg= 2022-09-03T06:07:14.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg=', 'Content-Length': '0'} 2022-09-03T06:07:14.277 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008263d96ec922b6ba-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.318 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.319 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.320 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.320 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.321 DEBUG:boto:Method: GET 2022-09-03T06:07:14.321 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.322 DEBUG:boto:Data: 2022-09-03T06:07:14.322 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.323 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.323 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.324 DEBUG:boto:Params: {} 2022-09-03T06:07:14.325 DEBUG:boto:Token: None 2022-09-03T06:07:14.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.329 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003738e1a191e7991f-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.330 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.331 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.332 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.332 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.333 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.333 DEBUG:boto:Method: GET 2022-09-03T06:07:14.334 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.334 DEBUG:boto:Data: 2022-09-03T06:07:14.335 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.336 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.337 DEBUG:boto:Params: {} 2022-09-03T06:07:14.337 DEBUG:boto:Token: None 2022-09-03T06:07:14.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.339 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.342 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b08acc1e0e951cf2-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.343 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.344 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.345 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.349 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.353 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.353 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.354 DEBUG:boto:Method: GET 2022-09-03T06:07:14.355 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.355 DEBUG:boto:Data: 2022-09-03T06:07:14.356 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.356 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.357 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.357 DEBUG:boto:Params: {} 2022-09-03T06:07:14.358 DEBUG:boto:Token: None 2022-09-03T06:07:14.358 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.359 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qx5P8F29fS/SFLxHAiT+Nb3xz64= 2022-09-03T06:07:14.360 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qx5P8F29fS/SFLxHAiT+Nb3xz64=', 'Content-Length': '0'} 2022-09-03T06:07:14.362 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e728c3d1ae008e4d-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.363 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.364 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.364 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.365 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.365 DEBUG:boto:Method: GET 2022-09-03T06:07:14.366 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.367 DEBUG:boto:Data: 2022-09-03T06:07:14.367 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.368 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.368 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.369 DEBUG:boto:Params: {} 2022-09-03T06:07:14.369 DEBUG:boto:Token: None 2022-09-03T06:07:14.370 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.371 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qx5P8F29fS/SFLxHAiT+Nb3xz64= 2022-09-03T06:07:14.372 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qx5P8F29fS/SFLxHAiT+Nb3xz64=', 'Content-Length': '0'} 2022-09-03T06:07:14.375 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003b5a308d4eeb3194-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.376 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.377 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.381 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.382 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.382 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.383 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.384 DEBUG:boto:Data: 2022-09-03T06:07:14.384 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.385 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.385 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.386 DEBUG:boto:Params: {} 2022-09-03T06:07:14.386 DEBUG:boto:Token: None 2022-09-03T06:07:14.387 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CZOcmwGM5F5NDxy15dTai1/4AYw= 2022-09-03T06:07:14.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CZOcmwGM5F5NDxy15dTai1/4AYw=', 'Content-Length': '0'} 2022-09-03T06:07:14.390 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000988446f2c68d433a-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.391 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.392 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.392 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.393 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.393 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.394 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.394 DEBUG:boto:Data: 2022-09-03T06:07:14.395 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.396 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.396 DEBUG:boto:Params: {} 2022-09-03T06:07:14.397 DEBUG:boto:Token: None 2022-09-03T06:07:14.397 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CZOcmwGM5F5NDxy15dTai1/4AYw= 2022-09-03T06:07:14.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CZOcmwGM5F5NDxy15dTai1/4AYw=', 'Content-Length': '0'} 2022-09-03T06:07:14.401 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c3491e58ddad67e1-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.401 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.402 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.403 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.403 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.404 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.404 DEBUG:boto:Method: GET 2022-09-03T06:07:14.405 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.405 DEBUG:boto:Data: 2022-09-03T06:07:14.406 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.406 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.407 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.407 DEBUG:boto:Params: {} 2022-09-03T06:07:14.408 DEBUG:boto:Token: None 2022-09-03T06:07:14.409 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qx5P8F29fS/SFLxHAiT+Nb3xz64= 2022-09-03T06:07:14.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qx5P8F29fS/SFLxHAiT+Nb3xz64=', 'Content-Length': '0'} 2022-09-03T06:07:14.412 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009027f7e19f686b17-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.454 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.455 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.455 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.456 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.456 DEBUG:boto:Method: GET 2022-09-03T06:07:14.457 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.458 DEBUG:boto:Data: 2022-09-03T06:07:14.459 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.461 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.462 DEBUG:boto:Params: {} 2022-09-03T06:07:14.463 DEBUG:boto:Token: None 2022-09-03T06:07:14.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:14.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qx5P8F29fS/SFLxHAiT+Nb3xz64= 2022-09-03T06:07:14.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qx5P8F29fS/SFLxHAiT+Nb3xz64=', 'Content-Length': '0'} 2022-09-03T06:07:14.469 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000044ecbd3f11473e81-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.471 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.473 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.473 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.474 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.474 DEBUG:boto:Method: GET 2022-09-03T06:07:14.475 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.475 DEBUG:boto:Data: 2022-09-03T06:07:14.476 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.476 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.477 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.478 DEBUG:boto:Params: {} 2022-09-03T06:07:14.478 DEBUG:boto:Token: None 2022-09-03T06:07:14.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.483 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b3568c9d210165c-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.484 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.485 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.485 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.486 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.487 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.487 DEBUG:boto:Method: GET 2022-09-03T06:07:14.488 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.488 DEBUG:boto:Data: 2022-09-03T06:07:14.489 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.490 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.491 DEBUG:boto:Params: {} 2022-09-03T06:07:14.492 DEBUG:boto:Token: None 2022-09-03T06:07:14.492 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.496 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000030bddc8f3ebfe546-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.497 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.498 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.498 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.499 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.499 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:14.500 DEBUG:boto:Data: 2022-09-03T06:07:14.500 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.501 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.501 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.502 DEBUG:boto:Params: {} 2022-09-03T06:07:14.503 DEBUG:boto:Token: None 2022-09-03T06:07:14.503 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj 2022-09-03T06:07:14.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA= 2022-09-03T06:07:14.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA=', 'Content-Length': '0'} 2022-09-03T06:07:14.508 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000013360ab6351ad19c-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.510 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.511 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.512 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.513 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:14.513 DEBUG:boto:Data: 2022-09-03T06:07:14.514 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.517 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.518 DEBUG:boto:Params: {} 2022-09-03T06:07:14.519 DEBUG:boto:Token: None 2022-09-03T06:07:14.520 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj 2022-09-03T06:07:14.521 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA= 2022-09-03T06:07:14.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA=', 'Content-Length': '0'} 2022-09-03T06:07:14.527 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000099580a1fe0a3b615-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.528 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.528 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.529 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.530 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.531 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.531 DEBUG:boto:Method: GET 2022-09-03T06:07:14.532 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.532 DEBUG:boto:Data: 2022-09-03T06:07:14.533 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.534 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.535 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.536 DEBUG:boto:Params: {} 2022-09-03T06:07:14.538 DEBUG:boto:Token: None 2022-09-03T06:07:14.539 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg= 2022-09-03T06:07:14.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg=', 'Content-Length': '0'} 2022-09-03T06:07:14.544 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b722ca6abda60cf8-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.586 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.587 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.588 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.588 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.589 DEBUG:boto:Method: GET 2022-09-03T06:07:14.590 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.590 DEBUG:boto:Data: 2022-09-03T06:07:14.591 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.592 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.593 DEBUG:boto:Params: {} 2022-09-03T06:07:14.593 DEBUG:boto:Token: None 2022-09-03T06:07:14.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg= 2022-09-03T06:07:14.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg=', 'Content-Length': '0'} 2022-09-03T06:07:14.600 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000058da174948f6b97b-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.601 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.602 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.603 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.604 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.605 DEBUG:boto:Method: GET 2022-09-03T06:07:14.606 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.607 DEBUG:boto:Data: 2022-09-03T06:07:14.608 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.609 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.609 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.610 DEBUG:boto:Params: {} 2022-09-03T06:07:14.611 DEBUG:boto:Token: None 2022-09-03T06:07:14.611 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.616 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036b0af9638ed8066-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.617 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.618 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.618 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.619 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.620 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.620 DEBUG:boto:Method: GET 2022-09-03T06:07:14.621 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.621 DEBUG:boto:Data: 2022-09-03T06:07:14.622 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.623 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.623 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.624 DEBUG:boto:Params: {} 2022-09-03T06:07:14.625 DEBUG:boto:Token: None 2022-09-03T06:07:14.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.636 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000054d6d18373503821-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.637 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.638 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.639 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.642 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.642 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.643 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.643 DEBUG:boto:Method: GET 2022-09-03T06:07:14.644 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.644 DEBUG:boto:Data: 2022-09-03T06:07:14.645 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.646 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.646 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.647 DEBUG:boto:Params: {} 2022-09-03T06:07:14.648 DEBUG:boto:Token: None 2022-09-03T06:07:14.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zLUTwMUDprGDhUm5CkOZPo3Pwe8= 2022-09-03T06:07:14.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zLUTwMUDprGDhUm5CkOZPo3Pwe8=', 'Content-Length': '0'} 2022-09-03T06:07:14.652 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bf4ce05a2309fb13-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.654 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.655 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.656 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.656 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.657 DEBUG:boto:Method: GET 2022-09-03T06:07:14.658 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.659 DEBUG:boto:Data: 2022-09-03T06:07:14.659 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.660 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.661 DEBUG:boto:Params: {} 2022-09-03T06:07:14.662 DEBUG:boto:Token: None 2022-09-03T06:07:14.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zLUTwMUDprGDhUm5CkOZPo3Pwe8= 2022-09-03T06:07:14.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zLUTwMUDprGDhUm5CkOZPo3Pwe8=', 'Content-Length': '0'} 2022-09-03T06:07:14.669 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005e9b0fb9b1e02167-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.670 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.671 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.672 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.673 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.674 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.674 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.675 DEBUG:boto:Data: 2022-09-03T06:07:14.675 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.676 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.676 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.676 DEBUG:boto:Params: {} 2022-09-03T06:07:14.677 DEBUG:boto:Token: None 2022-09-03T06:07:14.678 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0agwyDNH4U/SBikDIy4MEXTS6BM= 2022-09-03T06:07:14.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0agwyDNH4U/SBikDIy4MEXTS6BM=', 'Content-Length': '0'} 2022-09-03T06:07:14.681 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000366b69a1321ae84d-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.683 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.683 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.684 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.685 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.685 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.686 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.687 DEBUG:boto:Data: 2022-09-03T06:07:14.688 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.690 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.691 DEBUG:boto:Params: {} 2022-09-03T06:07:14.692 DEBUG:boto:Token: None 2022-09-03T06:07:14.693 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0agwyDNH4U/SBikDIy4MEXTS6BM= 2022-09-03T06:07:14.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0agwyDNH4U/SBikDIy4MEXTS6BM=', 'Content-Length': '0'} 2022-09-03T06:07:14.698 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002501343ed015bed1-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.700 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.700 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.701 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.702 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.703 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.704 DEBUG:boto:Method: GET 2022-09-03T06:07:14.705 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.706 DEBUG:boto:Data: 2022-09-03T06:07:14.707 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.708 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.709 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.710 DEBUG:boto:Params: {} 2022-09-03T06:07:14.711 DEBUG:boto:Token: None 2022-09-03T06:07:14.712 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zLUTwMUDprGDhUm5CkOZPo3Pwe8= 2022-09-03T06:07:14.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zLUTwMUDprGDhUm5CkOZPo3Pwe8=', 'Content-Length': '0'} 2022-09-03T06:07:14.717 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000089408affb3cb44ae-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.719 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.719 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.720 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.720 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.721 DEBUG:boto:Method: GET 2022-09-03T06:07:14.721 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.722 DEBUG:boto:Data: 2022-09-03T06:07:14.722 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.723 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.723 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.724 DEBUG:boto:Params: {} 2022-09-03T06:07:14.724 DEBUG:boto:Token: None 2022-09-03T06:07:14.725 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:14.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zLUTwMUDprGDhUm5CkOZPo3Pwe8= 2022-09-03T06:07:14.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zLUTwMUDprGDhUm5CkOZPo3Pwe8=', 'Content-Length': '0'} 2022-09-03T06:07:14.728 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b6d63d4b7b4e230e-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.729 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.730 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.730 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.731 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.731 DEBUG:boto:Method: GET 2022-09-03T06:07:14.732 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.732 DEBUG:boto:Data: 2022-09-03T06:07:14.733 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.733 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.734 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.734 DEBUG:boto:Params: {} 2022-09-03T06:07:14.735 DEBUG:boto:Token: None 2022-09-03T06:07:14.736 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.740 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac5d9c61ba17d6ac-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.741 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.742 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.742 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.743 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.743 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.744 DEBUG:boto:Method: GET 2022-09-03T06:07:14.744 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.745 DEBUG:boto:Data: 2022-09-03T06:07:14.745 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.746 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.747 DEBUG:boto:Params: {} 2022-09-03T06:07:14.747 DEBUG:boto:Token: None 2022-09-03T06:07:14.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000035fd75914e6e2b3c-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.753 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.754 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.755 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.755 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.756 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:14.756 DEBUG:boto:Data: 2022-09-03T06:07:14.757 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.757 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.758 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.758 DEBUG:boto:Params: {} 2022-09-03T06:07:14.759 DEBUG:boto:Token: None 2022-09-03T06:07:14.759 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj 2022-09-03T06:07:14.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA= 2022-09-03T06:07:14.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA=', 'Content-Length': '0'} 2022-09-03T06:07:14.764 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005558648366fdb6a6-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.765 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.765 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.766 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.766 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:14.767 DEBUG:boto:Data: 2022-09-03T06:07:14.767 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.768 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.769 DEBUG:boto:Params: {} 2022-09-03T06:07:14.769 DEBUG:boto:Token: None 2022-09-03T06:07:14.770 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj 2022-09-03T06:07:14.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA= 2022-09-03T06:07:14.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qzsgZ+/e8CNNHtCOXPL5Et5QhaA=', 'Content-Length': '0'} 2022-09-03T06:07:14.778 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003b5a89a47c9bb73a-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.779 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.780 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.781 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.782 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.783 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.783 DEBUG:boto:Method: GET 2022-09-03T06:07:14.784 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.785 DEBUG:boto:Data: 2022-09-03T06:07:14.785 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.786 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.786 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.787 DEBUG:boto:Params: {} 2022-09-03T06:07:14.789 DEBUG:boto:Token: None 2022-09-03T06:07:14.790 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.791 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg= 2022-09-03T06:07:14.792 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg=', 'Content-Length': '0'} 2022-09-03T06:07:14.794 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c829150161a334bc-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.795 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.797 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.798 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.799 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.800 DEBUG:boto:Method: GET 2022-09-03T06:07:14.801 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.801 DEBUG:boto:Data: 2022-09-03T06:07:14.802 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.803 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.803 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.804 DEBUG:boto:Params: {} 2022-09-03T06:07:14.805 DEBUG:boto:Token: None 2022-09-03T06:07:14.805 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:14.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg= 2022-09-03T06:07:14.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3hUGwlsX9uN66TpHrTug7mJ+Fg=', 'Content-Length': '0'} 2022-09-03T06:07:14.809 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a7624e33b071ae44-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.810 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.811 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.811 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.812 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.813 DEBUG:boto:Method: GET 2022-09-03T06:07:14.813 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.814 DEBUG:boto:Data: 2022-09-03T06:07:14.814 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.815 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.816 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.816 DEBUG:boto:Params: {} 2022-09-03T06:07:14.817 DEBUG:boto:Token: None 2022-09-03T06:07:14.817 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.822 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000327bfcb40a27df1c-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.870 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.872 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:14.872 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:14.873 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.875 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:14.876 DEBUG:boto:Method: GET 2022-09-03T06:07:14.876 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:14.877 DEBUG:boto:Data: 2022-09-03T06:07:14.877 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.879 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.879 DEBUG:boto:Params: {} 2022-09-03T06:07:14.880 DEBUG:boto:Token: None 2022-09-03T06:07:14.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/ 2022-09-03T06:07:14.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE= 2022-09-03T06:07:14.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0L06TIP8ls2UqQ+rBArpPIErYtE=', 'Content-Length': '0'} 2022-09-03T06:07:14.885 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e0cfdd79da6eea5a-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.886 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:14.887 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.887 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.888 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.888 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.889 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.889 DEBUG:boto:Method: GET 2022-09-03T06:07:14.890 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.890 DEBUG:boto:Data: 2022-09-03T06:07:14.891 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.891 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.892 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.893 DEBUG:boto:Params: {} 2022-09-03T06:07:14.894 DEBUG:boto:Token: None 2022-09-03T06:07:14.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BWNPHEQbaPKA/xm9UIz15XGFD1Q= 2022-09-03T06:07:14.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BWNPHEQbaPKA/xm9UIz15XGFD1Q=', 'Content-Length': '0'} 2022-09-03T06:07:14.899 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000956c300f8de037ca-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.900 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.900 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.901 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.901 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.902 DEBUG:boto:Method: GET 2022-09-03T06:07:14.902 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.903 DEBUG:boto:Data: 2022-09-03T06:07:14.903 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.904 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.905 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.905 DEBUG:boto:Params: {} 2022-09-03T06:07:14.906 DEBUG:boto:Token: None 2022-09-03T06:07:14.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BWNPHEQbaPKA/xm9UIz15XGFD1Q= 2022-09-03T06:07:14.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BWNPHEQbaPKA/xm9UIz15XGFD1Q=', 'Content-Length': '0'} 2022-09-03T06:07:14.910 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000996217f988a2037e-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.911 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.912 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.913 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.914 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.915 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.916 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.917 DEBUG:boto:Data: 2022-09-03T06:07:14.918 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.919 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.920 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.921 DEBUG:boto:Params: {} 2022-09-03T06:07:14.922 DEBUG:boto:Token: None 2022-09-03T06:07:14.923 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:osvExwF51umMkStwByGO4jUoRjY= 2022-09-03T06:07:14.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:osvExwF51umMkStwByGO4jUoRjY=', 'Content-Length': '0'} 2022-09-03T06:07:14.928 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000080f721cbbdbdba7-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.929 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.930 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.931 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.932 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.933 DEBUG:boto:Method: HEAD 2022-09-03T06:07:14.934 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.935 DEBUG:boto:Data: 2022-09-03T06:07:14.936 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.937 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.938 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.939 DEBUG:boto:Params: {} 2022-09-03T06:07:14.939 DEBUG:boto:Token: None 2022-09-03T06:07:14.940 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:osvExwF51umMkStwByGO4jUoRjY= 2022-09-03T06:07:14.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:osvExwF51umMkStwByGO4jUoRjY=', 'Content-Length': '0'} 2022-09-03T06:07:14.943 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000052a899a3ab5c6f24-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.944 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:14.945 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.946 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.946 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.947 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.947 DEBUG:boto:Method: GET 2022-09-03T06:07:14.948 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.948 DEBUG:boto:Data: 2022-09-03T06:07:14.949 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:14.950 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.950 DEBUG:boto:Params: {} 2022-09-03T06:07:14.951 DEBUG:boto:Token: None 2022-09-03T06:07:14.952 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BWNPHEQbaPKA/xm9UIz15XGFD1Q= 2022-09-03T06:07:14.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BWNPHEQbaPKA/xm9UIz15XGFD1Q=', 'Content-Length': '0'} 2022-09-03T06:07:14.956 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b5088cbc347feadf-006312ef12-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:14.957 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:14.958 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:14.959 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.960 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.961 DEBUG:boto:Method: GET 2022-09-03T06:07:14.962 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.963 DEBUG:boto:Data: 2022-09-03T06:07:14.964 DEBUG:boto:Headers: {} 2022-09-03T06:07:14.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:14.965 DEBUG:boto:Port: 8000 2022-09-03T06:07:14.966 DEBUG:boto:Params: {} 2022-09-03T06:07:14.967 DEBUG:boto:Token: None 2022-09-03T06:07:14.967 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:14 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:14.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BWNPHEQbaPKA/xm9UIz15XGFD1Q= 2022-09-03T06:07:14.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BWNPHEQbaPKA/xm9UIz15XGFD1Q=', 'Content-Length': '0'} 2022-09-03T06:07:14.971 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000ce6b9d14bc7f84b-006312ef12-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.018 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.019 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.021 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.022 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.023 DEBUG:boto:Method: GET 2022-09-03T06:07:15.024 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.025 DEBUG:boto:Data: 2022-09-03T06:07:15.026 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.027 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.028 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.029 DEBUG:boto:Params: {} 2022-09-03T06:07:15.030 DEBUG:boto:Token: None 2022-09-03T06:07:15.031 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.035 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c966e07e5ca0ff24-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.036 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.037 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.038 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.039 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.039 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.040 DEBUG:boto:Method: GET 2022-09-03T06:07:15.041 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.042 DEBUG:boto:Data: 2022-09-03T06:07:15.043 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.043 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.044 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.045 DEBUG:boto:Params: {} 2022-09-03T06:07:15.046 DEBUG:boto:Token: None 2022-09-03T06:07:15.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.050 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005468f7fc1d3f0005-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.051 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.052 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.053 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.054 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.055 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:15.055 DEBUG:boto:Data: 2022-09-03T06:07:15.056 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.057 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.057 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.058 DEBUG:boto:Params: {} 2022-09-03T06:07:15.058 DEBUG:boto:Token: None 2022-09-03T06:07:15.059 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj 2022-09-03T06:07:15.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ= 2022-09-03T06:07:15.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ=', 'Content-Length': '0'} 2022-09-03T06:07:15.065 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000eb6f32b4751ac94e-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.066 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.067 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.068 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.069 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:15.070 DEBUG:boto:Data: 2022-09-03T06:07:15.071 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.072 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.073 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.074 DEBUG:boto:Params: {} 2022-09-03T06:07:15.075 DEBUG:boto:Token: None 2022-09-03T06:07:15.076 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj 2022-09-03T06:07:15.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ= 2022-09-03T06:07:15.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ=', 'Content-Length': '0'} 2022-09-03T06:07:15.082 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ca0eadd66b366340-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.083 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.084 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.085 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.086 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.087 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.087 DEBUG:boto:Method: GET 2022-09-03T06:07:15.088 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.088 DEBUG:boto:Data: 2022-09-03T06:07:15.089 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.090 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.090 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.091 DEBUG:boto:Params: {} 2022-09-03T06:07:15.092 DEBUG:boto:Token: None 2022-09-03T06:07:15.092 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw= 2022-09-03T06:07:15.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw=', 'Content-Length': '0'} 2022-09-03T06:07:15.096 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003febbd6f1c159598-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.138 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.139 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.139 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.140 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.141 DEBUG:boto:Method: GET 2022-09-03T06:07:15.141 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.142 DEBUG:boto:Data: 2022-09-03T06:07:15.142 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.143 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.144 DEBUG:boto:Params: {} 2022-09-03T06:07:15.145 DEBUG:boto:Token: None 2022-09-03T06:07:15.145 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw= 2022-09-03T06:07:15.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw=', 'Content-Length': '0'} 2022-09-03T06:07:15.150 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000013b4a101209a6326-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.151 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.151 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.152 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.152 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.153 DEBUG:boto:Method: GET 2022-09-03T06:07:15.153 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.154 DEBUG:boto:Data: 2022-09-03T06:07:15.155 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.155 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.156 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.156 DEBUG:boto:Params: {} 2022-09-03T06:07:15.157 DEBUG:boto:Token: None 2022-09-03T06:07:15.158 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.162 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ce4ee3e40179b53-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.163 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.164 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.165 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.165 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.166 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.166 DEBUG:boto:Method: GET 2022-09-03T06:07:15.167 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.168 DEBUG:boto:Data: 2022-09-03T06:07:15.168 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.169 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.169 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.170 DEBUG:boto:Params: {} 2022-09-03T06:07:15.171 DEBUG:boto:Token: None 2022-09-03T06:07:15.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.175 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003cf4ee5974c57431-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.176 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.177 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.178 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.178 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.179 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.179 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.180 DEBUG:boto:Method: GET 2022-09-03T06:07:15.180 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.181 DEBUG:boto:Data: 2022-09-03T06:07:15.181 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.182 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.183 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.183 DEBUG:boto:Params: {} 2022-09-03T06:07:15.184 DEBUG:boto:Token: None 2022-09-03T06:07:15.185 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wec7lOK4jgYbtpP0u/54HgDElxE= 2022-09-03T06:07:15.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wec7lOK4jgYbtpP0u/54HgDElxE=', 'Content-Length': '0'} 2022-09-03T06:07:15.188 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e3a46044eaca9cef-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.189 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.190 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.190 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.191 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.191 DEBUG:boto:Method: GET 2022-09-03T06:07:15.192 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.192 DEBUG:boto:Data: 2022-09-03T06:07:15.193 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.193 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.194 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.195 DEBUG:boto:Params: {} 2022-09-03T06:07:15.195 DEBUG:boto:Token: None 2022-09-03T06:07:15.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wec7lOK4jgYbtpP0u/54HgDElxE= 2022-09-03T06:07:15.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wec7lOK4jgYbtpP0u/54HgDElxE=', 'Content-Length': '0'} 2022-09-03T06:07:15.199 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bd61190fede73e11-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.246 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.247 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.247 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.248 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.248 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.249 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.249 DEBUG:boto:Data: 2022-09-03T06:07:15.250 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.251 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.251 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.252 DEBUG:boto:Params: {} 2022-09-03T06:07:15.253 DEBUG:boto:Token: None 2022-09-03T06:07:15.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mmk7PmCJBJHRGXIknrmtAp8ol0w= 2022-09-03T06:07:15.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mmk7PmCJBJHRGXIknrmtAp8ol0w=', 'Content-Length': '0'} 2022-09-03T06:07:15.257 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001520f9eafb6bd640-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.257 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.258 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.259 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.259 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.260 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.260 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.261 DEBUG:boto:Data: 2022-09-03T06:07:15.261 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.262 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.263 DEBUG:boto:Params: {} 2022-09-03T06:07:15.263 DEBUG:boto:Token: None 2022-09-03T06:07:15.264 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mmk7PmCJBJHRGXIknrmtAp8ol0w= 2022-09-03T06:07:15.265 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mmk7PmCJBJHRGXIknrmtAp8ol0w=', 'Content-Length': '0'} 2022-09-03T06:07:15.267 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ced081667a803fc1-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.268 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.269 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.269 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.270 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.270 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.271 DEBUG:boto:Method: GET 2022-09-03T06:07:15.271 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.272 DEBUG:boto:Data: 2022-09-03T06:07:15.272 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.274 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.274 DEBUG:boto:Params: {} 2022-09-03T06:07:15.275 DEBUG:boto:Token: None 2022-09-03T06:07:15.276 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wec7lOK4jgYbtpP0u/54HgDElxE= 2022-09-03T06:07:15.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wec7lOK4jgYbtpP0u/54HgDElxE=', 'Content-Length': '0'} 2022-09-03T06:07:15.279 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000044806efb05ebd26a-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.280 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.281 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.281 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.282 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.282 DEBUG:boto:Method: GET 2022-09-03T06:07:15.283 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.283 DEBUG:boto:Data: 2022-09-03T06:07:15.284 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.285 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.286 DEBUG:boto:Params: {} 2022-09-03T06:07:15.286 DEBUG:boto:Token: None 2022-09-03T06:07:15.287 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:15.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wec7lOK4jgYbtpP0u/54HgDElxE= 2022-09-03T06:07:15.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wec7lOK4jgYbtpP0u/54HgDElxE=', 'Content-Length': '0'} 2022-09-03T06:07:15.290 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fb11656e79f2b11b-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.291 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.292 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.292 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.293 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.293 DEBUG:boto:Method: GET 2022-09-03T06:07:15.294 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.294 DEBUG:boto:Data: 2022-09-03T06:07:15.295 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.295 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.296 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.296 DEBUG:boto:Params: {} 2022-09-03T06:07:15.297 DEBUG:boto:Token: None 2022-09-03T06:07:15.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.302 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c9b6fe77f3205615-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.303 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.304 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.304 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.305 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.305 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.306 DEBUG:boto:Method: GET 2022-09-03T06:07:15.306 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.307 DEBUG:boto:Data: 2022-09-03T06:07:15.307 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.308 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.309 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.309 DEBUG:boto:Params: {} 2022-09-03T06:07:15.310 DEBUG:boto:Token: None 2022-09-03T06:07:15.310 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.315 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd1bc6f500da44b2-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.315 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.316 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.317 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.317 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.318 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:15.318 DEBUG:boto:Data: 2022-09-03T06:07:15.319 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.319 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.320 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.320 DEBUG:boto:Params: {} 2022-09-03T06:07:15.321 DEBUG:boto:Token: None 2022-09-03T06:07:15.321 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj 2022-09-03T06:07:15.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ= 2022-09-03T06:07:15.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ=', 'Content-Length': '0'} 2022-09-03T06:07:15.326 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c74285ada3a5c987-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.327 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.327 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.328 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.328 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:15.329 DEBUG:boto:Data: 2022-09-03T06:07:15.330 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.331 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.331 DEBUG:boto:Params: {} 2022-09-03T06:07:15.332 DEBUG:boto:Token: None 2022-09-03T06:07:15.332 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj 2022-09-03T06:07:15.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ= 2022-09-03T06:07:15.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ=', 'Content-Length': '0'} 2022-09-03T06:07:15.337 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000011e918b375055268-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.338 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.338 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.339 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.339 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.340 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.340 DEBUG:boto:Method: GET 2022-09-03T06:07:15.341 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.342 DEBUG:boto:Data: 2022-09-03T06:07:15.342 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.343 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.343 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.344 DEBUG:boto:Params: {} 2022-09-03T06:07:15.344 DEBUG:boto:Token: None 2022-09-03T06:07:15.345 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.346 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw= 2022-09-03T06:07:15.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw=', 'Content-Length': '0'} 2022-09-03T06:07:15.353 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002a16a9b7c366a402-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.399 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.400 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.400 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.401 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.402 DEBUG:boto:Method: GET 2022-09-03T06:07:15.403 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.403 DEBUG:boto:Data: 2022-09-03T06:07:15.404 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.405 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.406 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.406 DEBUG:boto:Params: {} 2022-09-03T06:07:15.407 DEBUG:boto:Token: None 2022-09-03T06:07:15.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw= 2022-09-03T06:07:15.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw=', 'Content-Length': '0'} 2022-09-03T06:07:15.413 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003b2538d924da7b56-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.414 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.415 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.416 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.417 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.418 DEBUG:boto:Method: GET 2022-09-03T06:07:15.419 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.420 DEBUG:boto:Data: 2022-09-03T06:07:15.421 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.421 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.422 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.422 DEBUG:boto:Params: {} 2022-09-03T06:07:15.424 DEBUG:boto:Token: None 2022-09-03T06:07:15.425 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.431 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a99555b57009621a-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.438 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.439 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.440 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.440 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.441 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.441 DEBUG:boto:Method: GET 2022-09-03T06:07:15.442 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.442 DEBUG:boto:Data: 2022-09-03T06:07:15.443 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.444 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.444 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.445 DEBUG:boto:Params: {} 2022-09-03T06:07:15.445 DEBUG:boto:Token: None 2022-09-03T06:07:15.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.450 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000012bd9dab727eaaf2-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.451 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.452 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.452 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.453 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.454 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.455 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.456 DEBUG:boto:Method: GET 2022-09-03T06:07:15.457 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.458 DEBUG:boto:Data: 2022-09-03T06:07:15.459 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.460 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.460 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.461 DEBUG:boto:Params: {} 2022-09-03T06:07:15.462 DEBUG:boto:Token: None 2022-09-03T06:07:15.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aBfeCpNnwdYla5ogRSgvjTSy+tU= 2022-09-03T06:07:15.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aBfeCpNnwdYla5ogRSgvjTSy+tU=', 'Content-Length': '0'} 2022-09-03T06:07:15.468 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003f1f3f523c704964-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.470 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.471 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.472 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.473 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.474 DEBUG:boto:Method: GET 2022-09-03T06:07:15.475 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.476 DEBUG:boto:Data: 2022-09-03T06:07:15.477 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.479 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.480 DEBUG:boto:Params: {} 2022-09-03T06:07:15.481 DEBUG:boto:Token: None 2022-09-03T06:07:15.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aBfeCpNnwdYla5ogRSgvjTSy+tU= 2022-09-03T06:07:15.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aBfeCpNnwdYla5ogRSgvjTSy+tU=', 'Content-Length': '0'} 2022-09-03T06:07:15.487 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d750c61712ba74fe-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.488 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.489 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.489 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.490 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.490 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.491 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.491 DEBUG:boto:Data: 2022-09-03T06:07:15.492 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.492 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.493 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.494 DEBUG:boto:Params: {} 2022-09-03T06:07:15.494 DEBUG:boto:Token: None 2022-09-03T06:07:15.495 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RrmcjmVynL6oCK9iZV9eU5K0NwA= 2022-09-03T06:07:15.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RrmcjmVynL6oCK9iZV9eU5K0NwA=', 'Content-Length': '0'} 2022-09-03T06:07:15.498 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000037cba111d44981bb-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.499 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.499 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.500 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.500 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.501 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.501 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.502 DEBUG:boto:Data: 2022-09-03T06:07:15.503 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.504 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.504 DEBUG:boto:Params: {} 2022-09-03T06:07:15.505 DEBUG:boto:Token: None 2022-09-03T06:07:15.506 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RrmcjmVynL6oCK9iZV9eU5K0NwA= 2022-09-03T06:07:15.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RrmcjmVynL6oCK9iZV9eU5K0NwA=', 'Content-Length': '0'} 2022-09-03T06:07:15.517 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000024316eed420dffed-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.518 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.518 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.519 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.519 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.520 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.521 DEBUG:boto:Method: GET 2022-09-03T06:07:15.521 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.522 DEBUG:boto:Data: 2022-09-03T06:07:15.523 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.524 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.525 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.526 DEBUG:boto:Params: {} 2022-09-03T06:07:15.528 DEBUG:boto:Token: None 2022-09-03T06:07:15.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aBfeCpNnwdYla5ogRSgvjTSy+tU= 2022-09-03T06:07:15.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aBfeCpNnwdYla5ogRSgvjTSy+tU=', 'Content-Length': '0'} 2022-09-03T06:07:15.534 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000346fc097d7dc2d05-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.579 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.579 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.580 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.580 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.581 DEBUG:boto:Method: GET 2022-09-03T06:07:15.581 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.582 DEBUG:boto:Data: 2022-09-03T06:07:15.583 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.584 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.584 DEBUG:boto:Params: {} 2022-09-03T06:07:15.585 DEBUG:boto:Token: None 2022-09-03T06:07:15.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:15.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aBfeCpNnwdYla5ogRSgvjTSy+tU= 2022-09-03T06:07:15.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aBfeCpNnwdYla5ogRSgvjTSy+tU=', 'Content-Length': '0'} 2022-09-03T06:07:15.589 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000914b6b2b17a65c01-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.590 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.590 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.591 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.591 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.592 DEBUG:boto:Method: GET 2022-09-03T06:07:15.592 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.593 DEBUG:boto:Data: 2022-09-03T06:07:15.594 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.594 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.595 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.595 DEBUG:boto:Params: {} 2022-09-03T06:07:15.596 DEBUG:boto:Token: None 2022-09-03T06:07:15.597 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.601 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000039be8f48b46884a1-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.602 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.604 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.605 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.606 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.607 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.608 DEBUG:boto:Method: GET 2022-09-03T06:07:15.609 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.610 DEBUG:boto:Data: 2022-09-03T06:07:15.611 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.613 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.614 DEBUG:boto:Params: {} 2022-09-03T06:07:15.615 DEBUG:boto:Token: None 2022-09-03T06:07:15.616 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.622 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f6d54b847f5ab30-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.623 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.625 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.626 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.626 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.627 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:15.628 DEBUG:boto:Data: 2022-09-03T06:07:15.628 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.629 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.630 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.630 DEBUG:boto:Params: {} 2022-09-03T06:07:15.631 DEBUG:boto:Token: None 2022-09-03T06:07:15.632 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj 2022-09-03T06:07:15.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ= 2022-09-03T06:07:15.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ=', 'Content-Length': '0'} 2022-09-03T06:07:15.638 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e9a9ca1eed860bc6-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.639 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.640 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.641 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.641 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:15.642 DEBUG:boto:Data: 2022-09-03T06:07:15.642 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.644 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.644 DEBUG:boto:Params: {} 2022-09-03T06:07:15.645 DEBUG:boto:Token: None 2022-09-03T06:07:15.646 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj 2022-09-03T06:07:15.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ= 2022-09-03T06:07:15.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ=', 'Content-Length': '0'} 2022-09-03T06:07:15.651 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000283060545f490ab-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.653 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.654 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.655 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.657 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.658 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.659 DEBUG:boto:Method: GET 2022-09-03T06:07:15.660 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.661 DEBUG:boto:Data: 2022-09-03T06:07:15.661 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.662 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.663 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.664 DEBUG:boto:Params: {} 2022-09-03T06:07:15.664 DEBUG:boto:Token: None 2022-09-03T06:07:15.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw= 2022-09-03T06:07:15.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw=', 'Content-Length': '0'} 2022-09-03T06:07:15.669 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d479ce61bfd538c3-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.670 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.671 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.672 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.672 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.673 DEBUG:boto:Method: GET 2022-09-03T06:07:15.674 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.675 DEBUG:boto:Data: 2022-09-03T06:07:15.676 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.677 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.678 DEBUG:boto:Params: {} 2022-09-03T06:07:15.680 DEBUG:boto:Token: None 2022-09-03T06:07:15.681 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.682 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw= 2022-09-03T06:07:15.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw=', 'Content-Length': '0'} 2022-09-03T06:07:15.685 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f76c987c176f57c5-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.726 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.727 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.728 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.728 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.729 DEBUG:boto:Method: GET 2022-09-03T06:07:15.730 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.730 DEBUG:boto:Data: 2022-09-03T06:07:15.731 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.731 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.732 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.733 DEBUG:boto:Params: {} 2022-09-03T06:07:15.733 DEBUG:boto:Token: None 2022-09-03T06:07:15.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.738 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001fc541c01faae562-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.739 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.740 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.741 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.741 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.742 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.742 DEBUG:boto:Method: GET 2022-09-03T06:07:15.743 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.743 DEBUG:boto:Data: 2022-09-03T06:07:15.744 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.744 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.745 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.746 DEBUG:boto:Params: {} 2022-09-03T06:07:15.747 DEBUG:boto:Token: None 2022-09-03T06:07:15.747 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.751 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fdc5d13895427beb-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.752 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.754 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.754 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.755 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.756 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.756 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.757 DEBUG:boto:Method: GET 2022-09-03T06:07:15.758 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.758 DEBUG:boto:Data: 2022-09-03T06:07:15.759 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.759 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.760 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.760 DEBUG:boto:Params: {} 2022-09-03T06:07:15.761 DEBUG:boto:Token: None 2022-09-03T06:07:15.762 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+LxL+ICTGwe+T49PO4o8lfvqJsE= 2022-09-03T06:07:15.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+LxL+ICTGwe+T49PO4o8lfvqJsE=', 'Content-Length': '0'} 2022-09-03T06:07:15.765 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006992149344c2e005-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.766 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.767 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.767 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.768 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.768 DEBUG:boto:Method: GET 2022-09-03T06:07:15.769 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.769 DEBUG:boto:Data: 2022-09-03T06:07:15.770 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.771 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.771 DEBUG:boto:Params: {} 2022-09-03T06:07:15.772 DEBUG:boto:Token: None 2022-09-03T06:07:15.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+LxL+ICTGwe+T49PO4o8lfvqJsE= 2022-09-03T06:07:15.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+LxL+ICTGwe+T49PO4o8lfvqJsE=', 'Content-Length': '0'} 2022-09-03T06:07:15.776 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003933391fbf0aa138-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.777 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.778 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.778 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.779 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.779 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.780 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.780 DEBUG:boto:Data: 2022-09-03T06:07:15.781 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.782 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.782 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.783 DEBUG:boto:Params: {} 2022-09-03T06:07:15.784 DEBUG:boto:Token: None 2022-09-03T06:07:15.784 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Zv6X18vwmpXA8+ikIjF6WXleto= 2022-09-03T06:07:15.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Zv6X18vwmpXA8+ikIjF6WXleto=', 'Content-Length': '0'} 2022-09-03T06:07:15.788 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000076fc7b241800c6b2-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.789 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.789 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.790 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.790 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.791 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.792 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.792 DEBUG:boto:Data: 2022-09-03T06:07:15.793 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.794 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.795 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.795 DEBUG:boto:Params: {} 2022-09-03T06:07:15.796 DEBUG:boto:Token: None 2022-09-03T06:07:15.797 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Zv6X18vwmpXA8+ikIjF6WXleto= 2022-09-03T06:07:15.798 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Zv6X18vwmpXA8+ikIjF6WXleto=', 'Content-Length': '0'} 2022-09-03T06:07:15.801 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000074074cedc67bc60e-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.801 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.802 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.803 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.803 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.804 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.805 DEBUG:boto:Method: GET 2022-09-03T06:07:15.805 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.806 DEBUG:boto:Data: 2022-09-03T06:07:15.806 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.807 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.807 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.808 DEBUG:boto:Params: {} 2022-09-03T06:07:15.809 DEBUG:boto:Token: None 2022-09-03T06:07:15.809 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+LxL+ICTGwe+T49PO4o8lfvqJsE= 2022-09-03T06:07:15.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+LxL+ICTGwe+T49PO4o8lfvqJsE=', 'Content-Length': '0'} 2022-09-03T06:07:15.813 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f84a200833d56cc6-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.814 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.814 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.815 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.815 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.816 DEBUG:boto:Method: GET 2022-09-03T06:07:15.817 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.817 DEBUG:boto:Data: 2022-09-03T06:07:15.819 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.821 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.822 DEBUG:boto:Params: {} 2022-09-03T06:07:15.822 DEBUG:boto:Token: None 2022-09-03T06:07:15.823 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:15.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+LxL+ICTGwe+T49PO4o8lfvqJsE= 2022-09-03T06:07:15.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+LxL+ICTGwe+T49PO4o8lfvqJsE=', 'Content-Length': '0'} 2022-09-03T06:07:15.826 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fbdd23b1f7343bef-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.827 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.828 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.829 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.829 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.830 DEBUG:boto:Method: GET 2022-09-03T06:07:15.830 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.831 DEBUG:boto:Data: 2022-09-03T06:07:15.831 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.832 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.833 DEBUG:boto:Params: {} 2022-09-03T06:07:15.835 DEBUG:boto:Token: None 2022-09-03T06:07:15.836 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.842 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000856680bb83837d13-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.843 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.844 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:15.844 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:15.845 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.846 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:15.848 DEBUG:boto:Method: GET 2022-09-03T06:07:15.849 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:15.850 DEBUG:boto:Data: 2022-09-03T06:07:15.851 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.852 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.853 DEBUG:boto:Params: {} 2022-09-03T06:07:15.854 DEBUG:boto:Token: None 2022-09-03T06:07:15.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/ 2022-09-03T06:07:15.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4= 2022-09-03T06:07:15.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PD9p74O4LTKXb17xA30zYNIbVI4=', 'Content-Length': '0'} 2022-09-03T06:07:15.861 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006b7834f213d8da6e-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.904 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:15.905 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.906 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.906 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.907 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:15.907 DEBUG:boto:Data: 2022-09-03T06:07:15.908 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.908 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.909 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.909 DEBUG:boto:Params: {} 2022-09-03T06:07:15.910 DEBUG:boto:Token: None 2022-09-03T06:07:15.911 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj 2022-09-03T06:07:15.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ= 2022-09-03T06:07:15.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ=', 'Content-Length': '0'} 2022-09-03T06:07:15.916 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008b6f1ab8a8cc7d65-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.916 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.917 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.917 DEBUG:boto:Method: HEAD 2022-09-03T06:07:15.918 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:15.919 DEBUG:boto:Data: 2022-09-03T06:07:15.919 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.920 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.921 DEBUG:boto:Params: {} 2022-09-03T06:07:15.921 DEBUG:boto:Token: None 2022-09-03T06:07:15.922 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj 2022-09-03T06:07:15.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ= 2022-09-03T06:07:15.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aHt6UcvLuVJgTallPgNuhMD1sQQ=', 'Content-Length': '0'} 2022-09-03T06:07:15.927 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000afa19ed9f9043a5b-006312ef13-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.928 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:15.929 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.929 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.930 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.930 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.931 DEBUG:boto:Method: GET 2022-09-03T06:07:15.932 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.932 DEBUG:boto:Data: 2022-09-03T06:07:15.933 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.933 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:15.934 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.934 DEBUG:boto:Params: {} 2022-09-03T06:07:15.935 DEBUG:boto:Token: None 2022-09-03T06:07:15.935 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw= 2022-09-03T06:07:15.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw=', 'Content-Length': '0'} 2022-09-03T06:07:15.939 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000849191664f7d8d01-006312ef13-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:15.982 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:15.984 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:15.985 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.986 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.987 DEBUG:boto:Method: GET 2022-09-03T06:07:15.988 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.989 DEBUG:boto:Data: 2022-09-03T06:07:15.990 DEBUG:boto:Headers: {} 2022-09-03T06:07:15.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:15.991 DEBUG:boto:Port: 8000 2022-09-03T06:07:15.991 DEBUG:boto:Params: {} 2022-09-03T06:07:15.992 DEBUG:boto:Token: None 2022-09-03T06:07:15.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:15 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:15.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw= 2022-09-03T06:07:15.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTlV8raGlsoFxAHFTjes9KORTCw=', 'Content-Length': '0'} 2022-09-03T06:07:16.006 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d6f3d6a4e955e7ab-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.008 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.009 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.009 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.011 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.012 DEBUG:boto:Method: GET 2022-09-03T06:07:16.013 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.014 DEBUG:boto:Data: 2022-09-03T06:07:16.015 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.017 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.017 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.018 DEBUG:boto:Params: {} 2022-09-03T06:07:16.019 DEBUG:boto:Token: None 2022-09-03T06:07:16.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.021 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.024 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001eac15e36cf21f5f-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.025 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.027 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.028 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.029 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.029 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.030 DEBUG:boto:Method: GET 2022-09-03T06:07:16.030 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.031 DEBUG:boto:Data: 2022-09-03T06:07:16.034 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.035 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.036 DEBUG:boto:Params: {} 2022-09-03T06:07:16.037 DEBUG:boto:Token: None 2022-09-03T06:07:16.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.041 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000add271c96f11c03b-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.042 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.043 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.043 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.044 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.046 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.047 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.047 DEBUG:boto:Method: GET 2022-09-03T06:07:16.048 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.048 DEBUG:boto:Data: 2022-09-03T06:07:16.049 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.050 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.050 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.051 DEBUG:boto:Params: {} 2022-09-03T06:07:16.052 DEBUG:boto:Token: None 2022-09-03T06:07:16.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JWxb2nh78vZ2BqI64mK1TIko5+Y= 2022-09-03T06:07:16.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JWxb2nh78vZ2BqI64mK1TIko5+Y=', 'Content-Length': '0'} 2022-09-03T06:07:16.057 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000062c637057b7565a6-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.058 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.059 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.059 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.060 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.060 DEBUG:boto:Method: GET 2022-09-03T06:07:16.061 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.061 DEBUG:boto:Data: 2022-09-03T06:07:16.062 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.063 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.064 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.065 DEBUG:boto:Params: {} 2022-09-03T06:07:16.067 DEBUG:boto:Token: None 2022-09-03T06:07:16.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JWxb2nh78vZ2BqI64mK1TIko5+Y= 2022-09-03T06:07:16.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JWxb2nh78vZ2BqI64mK1TIko5+Y=', 'Content-Length': '0'} 2022-09-03T06:07:16.073 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004ec498435ebfe567-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.075 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.076 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.076 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.077 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.078 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.078 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.079 DEBUG:boto:Data: 2022-09-03T06:07:16.080 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.080 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.081 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.082 DEBUG:boto:Params: {} 2022-09-03T06:07:16.083 DEBUG:boto:Token: None 2022-09-03T06:07:16.084 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PsV2ZnYblhkwCae21gmKTMuPkrE= 2022-09-03T06:07:16.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PsV2ZnYblhkwCae21gmKTMuPkrE=', 'Content-Length': '0'} 2022-09-03T06:07:16.088 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c1376bc8b6301822-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.089 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.091 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.091 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.092 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.093 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.094 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.095 DEBUG:boto:Data: 2022-09-03T06:07:16.096 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.101 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.102 DEBUG:boto:Params: {} 2022-09-03T06:07:16.103 DEBUG:boto:Token: None 2022-09-03T06:07:16.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PsV2ZnYblhkwCae21gmKTMuPkrE= 2022-09-03T06:07:16.106 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PsV2ZnYblhkwCae21gmKTMuPkrE=', 'Content-Length': '0'} 2022-09-03T06:07:16.108 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006f48e2bdf71a7f50-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.109 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.110 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.110 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.111 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.111 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.112 DEBUG:boto:Method: GET 2022-09-03T06:07:16.113 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.113 DEBUG:boto:Data: 2022-09-03T06:07:16.114 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.115 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.116 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.116 DEBUG:boto:Params: {} 2022-09-03T06:07:16.117 DEBUG:boto:Token: None 2022-09-03T06:07:16.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JWxb2nh78vZ2BqI64mK1TIko5+Y= 2022-09-03T06:07:16.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JWxb2nh78vZ2BqI64mK1TIko5+Y=', 'Content-Length': '0'} 2022-09-03T06:07:16.122 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000b84af0b39ed87a4-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.123 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.124 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.124 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.125 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.126 DEBUG:boto:Method: GET 2022-09-03T06:07:16.126 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.127 DEBUG:boto:Data: 2022-09-03T06:07:16.128 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.129 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.129 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.130 DEBUG:boto:Params: {} 2022-09-03T06:07:16.131 DEBUG:boto:Token: None 2022-09-03T06:07:16.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:16.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JWxb2nh78vZ2BqI64mK1TIko5+Y= 2022-09-03T06:07:16.133 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JWxb2nh78vZ2BqI64mK1TIko5+Y=', 'Content-Length': '0'} 2022-09-03T06:07:16.135 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003c55af2b4968a642-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.136 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.137 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.138 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.138 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.139 DEBUG:boto:Method: GET 2022-09-03T06:07:16.140 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.141 DEBUG:boto:Data: 2022-09-03T06:07:16.142 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.143 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.143 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.144 DEBUG:boto:Params: {} 2022-09-03T06:07:16.145 DEBUG:boto:Token: None 2022-09-03T06:07:16.146 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.147 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.150 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000057e115be1531e3a8-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.151 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.152 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.153 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.153 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.154 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.155 DEBUG:boto:Method: GET 2022-09-03T06:07:16.155 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.156 DEBUG:boto:Data: 2022-09-03T06:07:16.156 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.157 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.158 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.158 DEBUG:boto:Params: {} 2022-09-03T06:07:16.159 DEBUG:boto:Token: None 2022-09-03T06:07:16.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.164 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d533c3671df9a92-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.165 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.166 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.167 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.168 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.168 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:16.169 DEBUG:boto:Data: 2022-09-03T06:07:16.169 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.170 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.171 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.171 DEBUG:boto:Params: {} 2022-09-03T06:07:16.172 DEBUG:boto:Token: None 2022-09-03T06:07:16.173 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj 2022-09-03T06:07:16.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM= 2022-09-03T06:07:16.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM=', 'Content-Length': '0'} 2022-09-03T06:07:16.178 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002d031d9a9b0d4fa0-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.179 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.180 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.181 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.182 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:16.183 DEBUG:boto:Data: 2022-09-03T06:07:16.184 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.186 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.187 DEBUG:boto:Params: {} 2022-09-03T06:07:16.188 DEBUG:boto:Token: None 2022-09-03T06:07:16.189 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj 2022-09-03T06:07:16.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM= 2022-09-03T06:07:16.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM=', 'Content-Length': '0'} 2022-09-03T06:07:16.194 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000398fff4bd3446e4f-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.195 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.196 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.197 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.197 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.198 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.198 DEBUG:boto:Method: GET 2022-09-03T06:07:16.199 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.199 DEBUG:boto:Data: 2022-09-03T06:07:16.200 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.200 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.201 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.201 DEBUG:boto:Params: {} 2022-09-03T06:07:16.202 DEBUG:boto:Token: None 2022-09-03T06:07:16.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc= 2022-09-03T06:07:16.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc=', 'Content-Length': '0'} 2022-09-03T06:07:16.206 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d0e74135393a5f16-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.207 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.208 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.208 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.209 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.209 DEBUG:boto:Method: GET 2022-09-03T06:07:16.210 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.210 DEBUG:boto:Data: 2022-09-03T06:07:16.211 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.211 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.212 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.212 DEBUG:boto:Params: {} 2022-09-03T06:07:16.213 DEBUG:boto:Token: None 2022-09-03T06:07:16.214 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc= 2022-09-03T06:07:16.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc=', 'Content-Length': '0'} 2022-09-03T06:07:16.217 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c6f3a08ead98b140-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.262 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.263 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.264 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.265 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.265 DEBUG:boto:Method: GET 2022-09-03T06:07:16.266 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.266 DEBUG:boto:Data: 2022-09-03T06:07:16.267 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.267 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.268 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.269 DEBUG:boto:Params: {} 2022-09-03T06:07:16.270 DEBUG:boto:Token: None 2022-09-03T06:07:16.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.276 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000562fcc805f783da5-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.277 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.278 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.279 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.279 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.280 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.281 DEBUG:boto:Method: GET 2022-09-03T06:07:16.281 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.282 DEBUG:boto:Data: 2022-09-03T06:07:16.282 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.283 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.284 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.284 DEBUG:boto:Params: {} 2022-09-03T06:07:16.285 DEBUG:boto:Token: None 2022-09-03T06:07:16.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.289 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008cabf5c02878a555-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.290 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.291 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.292 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.292 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.293 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.293 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.294 DEBUG:boto:Method: GET 2022-09-03T06:07:16.294 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.295 DEBUG:boto:Data: 2022-09-03T06:07:16.295 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.296 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.296 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.297 DEBUG:boto:Params: {} 2022-09-03T06:07:16.298 DEBUG:boto:Token: None 2022-09-03T06:07:16.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uhPPc2lrIDSsfRbTAyRx1wv5knk= 2022-09-03T06:07:16.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uhPPc2lrIDSsfRbTAyRx1wv5knk=', 'Content-Length': '0'} 2022-09-03T06:07:16.302 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a40fe7aa1027aa9d-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.308 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.309 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.309 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.310 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.311 DEBUG:boto:Method: GET 2022-09-03T06:07:16.312 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.312 DEBUG:boto:Data: 2022-09-03T06:07:16.313 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.314 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.315 DEBUG:boto:Params: {} 2022-09-03T06:07:16.315 DEBUG:boto:Token: None 2022-09-03T06:07:16.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uhPPc2lrIDSsfRbTAyRx1wv5knk= 2022-09-03T06:07:16.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uhPPc2lrIDSsfRbTAyRx1wv5knk=', 'Content-Length': '0'} 2022-09-03T06:07:16.320 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e68ce4133d73651c-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.321 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.322 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.322 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.323 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.323 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.324 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.324 DEBUG:boto:Data: 2022-09-03T06:07:16.325 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.325 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.326 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.326 DEBUG:boto:Params: {} 2022-09-03T06:07:16.327 DEBUG:boto:Token: None 2022-09-03T06:07:16.328 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aCHSUleDvE8B7g9cxXXKXLt97rI= 2022-09-03T06:07:16.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aCHSUleDvE8B7g9cxXXKXLt97rI=', 'Content-Length': '0'} 2022-09-03T06:07:16.331 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008541553973816359-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.332 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.333 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.334 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.335 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.336 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.337 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.337 DEBUG:boto:Data: 2022-09-03T06:07:16.338 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.339 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.340 DEBUG:boto:Params: {} 2022-09-03T06:07:16.340 DEBUG:boto:Token: None 2022-09-03T06:07:16.341 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aCHSUleDvE8B7g9cxXXKXLt97rI= 2022-09-03T06:07:16.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aCHSUleDvE8B7g9cxXXKXLt97rI=', 'Content-Length': '0'} 2022-09-03T06:07:16.345 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e3da021e6c1db285-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.346 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.346 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.347 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.347 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.348 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.349 DEBUG:boto:Method: GET 2022-09-03T06:07:16.349 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.350 DEBUG:boto:Data: 2022-09-03T06:07:16.351 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.351 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.352 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.352 DEBUG:boto:Params: {} 2022-09-03T06:07:16.353 DEBUG:boto:Token: None 2022-09-03T06:07:16.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uhPPc2lrIDSsfRbTAyRx1wv5knk= 2022-09-03T06:07:16.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uhPPc2lrIDSsfRbTAyRx1wv5knk=', 'Content-Length': '0'} 2022-09-03T06:07:16.357 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c2313851d20f53bc-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.398 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.399 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.400 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.401 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.401 DEBUG:boto:Method: GET 2022-09-03T06:07:16.403 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.403 DEBUG:boto:Data: 2022-09-03T06:07:16.404 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.405 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.405 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.406 DEBUG:boto:Params: {} 2022-09-03T06:07:16.407 DEBUG:boto:Token: None 2022-09-03T06:07:16.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:16.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uhPPc2lrIDSsfRbTAyRx1wv5knk= 2022-09-03T06:07:16.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uhPPc2lrIDSsfRbTAyRx1wv5knk=', 'Content-Length': '0'} 2022-09-03T06:07:16.412 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000081be6d58f30235b4-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.413 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.414 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.414 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.415 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.415 DEBUG:boto:Method: GET 2022-09-03T06:07:16.416 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.417 DEBUG:boto:Data: 2022-09-03T06:07:16.417 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.418 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.419 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.420 DEBUG:boto:Params: {} 2022-09-03T06:07:16.421 DEBUG:boto:Token: None 2022-09-03T06:07:16.421 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.426 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e71e617d5aaee455-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.427 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.428 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.429 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.429 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.430 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.431 DEBUG:boto:Method: GET 2022-09-03T06:07:16.431 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.432 DEBUG:boto:Data: 2022-09-03T06:07:16.433 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.433 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.434 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.435 DEBUG:boto:Params: {} 2022-09-03T06:07:16.435 DEBUG:boto:Token: None 2022-09-03T06:07:16.436 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.440 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c67b176adc93a05c-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.441 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.442 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.443 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.443 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.444 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:16.445 DEBUG:boto:Data: 2022-09-03T06:07:16.445 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.446 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.447 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.447 DEBUG:boto:Params: {} 2022-09-03T06:07:16.448 DEBUG:boto:Token: None 2022-09-03T06:07:16.448 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj 2022-09-03T06:07:16.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM= 2022-09-03T06:07:16.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM=', 'Content-Length': '0'} 2022-09-03T06:07:16.453 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e7d490ffc9a86849-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.454 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.455 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.456 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.456 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:16.457 DEBUG:boto:Data: 2022-09-03T06:07:16.457 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.459 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.459 DEBUG:boto:Params: {} 2022-09-03T06:07:16.460 DEBUG:boto:Token: None 2022-09-03T06:07:16.460 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj 2022-09-03T06:07:16.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM= 2022-09-03T06:07:16.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM=', 'Content-Length': '0'} 2022-09-03T06:07:16.465 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000293fa3b1fd2a150f-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.466 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.466 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.467 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.467 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.468 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.468 DEBUG:boto:Method: GET 2022-09-03T06:07:16.469 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.470 DEBUG:boto:Data: 2022-09-03T06:07:16.470 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.471 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.471 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.472 DEBUG:boto:Params: {} 2022-09-03T06:07:16.472 DEBUG:boto:Token: None 2022-09-03T06:07:16.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc= 2022-09-03T06:07:16.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc=', 'Content-Length': '0'} 2022-09-03T06:07:16.477 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f9978b04b1f59099-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.479 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.480 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.481 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.482 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.483 DEBUG:boto:Method: GET 2022-09-03T06:07:16.483 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.484 DEBUG:boto:Data: 2022-09-03T06:07:16.485 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.486 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.487 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.487 DEBUG:boto:Params: {} 2022-09-03T06:07:16.488 DEBUG:boto:Token: None 2022-09-03T06:07:16.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc= 2022-09-03T06:07:16.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc=', 'Content-Length': '0'} 2022-09-03T06:07:16.492 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000497b7e24ef40b7aa-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.493 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.494 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.494 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.495 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.495 DEBUG:boto:Method: GET 2022-09-03T06:07:16.496 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.497 DEBUG:boto:Data: 2022-09-03T06:07:16.498 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.499 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.500 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.501 DEBUG:boto:Params: {} 2022-09-03T06:07:16.502 DEBUG:boto:Token: None 2022-09-03T06:07:16.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.507 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e2b295b3ea3ddb0d-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.508 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.508 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.509 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.510 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.510 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.511 DEBUG:boto:Method: GET 2022-09-03T06:07:16.511 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.512 DEBUG:boto:Data: 2022-09-03T06:07:16.512 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.513 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.514 DEBUG:boto:Params: {} 2022-09-03T06:07:16.514 DEBUG:boto:Token: None 2022-09-03T06:07:16.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.519 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b322cfbc986b4dc-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.520 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.521 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.522 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.522 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.523 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.524 DEBUG:boto:Method: GET 2022-09-03T06:07:16.525 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.525 DEBUG:boto:Data: 2022-09-03T06:07:16.526 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.526 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.527 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.528 DEBUG:boto:Params: {} 2022-09-03T06:07:16.528 DEBUG:boto:Token: None 2022-09-03T06:07:16.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lzVcCIIdooIYpg11FGDVRFgn20Q= 2022-09-03T06:07:16.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lzVcCIIdooIYpg11FGDVRFgn20Q=', 'Content-Length': '0'} 2022-09-03T06:07:16.532 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000000708d10b8b87a5e-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.574 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.575 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.575 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.576 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.577 DEBUG:boto:Method: GET 2022-09-03T06:07:16.577 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.578 DEBUG:boto:Data: 2022-09-03T06:07:16.578 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.579 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.579 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.580 DEBUG:boto:Params: {} 2022-09-03T06:07:16.581 DEBUG:boto:Token: None 2022-09-03T06:07:16.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lzVcCIIdooIYpg11FGDVRFgn20Q= 2022-09-03T06:07:16.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lzVcCIIdooIYpg11FGDVRFgn20Q=', 'Content-Length': '0'} 2022-09-03T06:07:16.585 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000afb6d398d92336a7-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.586 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.586 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.587 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.587 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.588 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.588 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.589 DEBUG:boto:Data: 2022-09-03T06:07:16.589 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.590 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.590 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.591 DEBUG:boto:Params: {} 2022-09-03T06:07:16.591 DEBUG:boto:Token: None 2022-09-03T06:07:16.592 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v2A3Xt+UP5cjZUzK9umNOi1XSAI= 2022-09-03T06:07:16.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v2A3Xt+UP5cjZUzK9umNOi1XSAI=', 'Content-Length': '0'} 2022-09-03T06:07:16.603 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004d32047d68c2da8e-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.604 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.605 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.606 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.606 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.607 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.607 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.608 DEBUG:boto:Data: 2022-09-03T06:07:16.608 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.609 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.610 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.611 DEBUG:boto:Params: {} 2022-09-03T06:07:16.612 DEBUG:boto:Token: None 2022-09-03T06:07:16.613 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v2A3Xt+UP5cjZUzK9umNOi1XSAI= 2022-09-03T06:07:16.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v2A3Xt+UP5cjZUzK9umNOi1XSAI=', 'Content-Length': '0'} 2022-09-03T06:07:16.618 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c39b08f9e06b9dbf-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.619 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.620 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.620 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.621 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.622 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.622 DEBUG:boto:Method: GET 2022-09-03T06:07:16.623 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.624 DEBUG:boto:Data: 2022-09-03T06:07:16.625 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.627 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.628 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.629 DEBUG:boto:Params: {} 2022-09-03T06:07:16.630 DEBUG:boto:Token: None 2022-09-03T06:07:16.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lzVcCIIdooIYpg11FGDVRFgn20Q= 2022-09-03T06:07:16.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lzVcCIIdooIYpg11FGDVRFgn20Q=', 'Content-Length': '0'} 2022-09-03T06:07:16.634 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006521dfdc915a3b37-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.635 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.635 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.636 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.636 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.637 DEBUG:boto:Method: GET 2022-09-03T06:07:16.638 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.638 DEBUG:boto:Data: 2022-09-03T06:07:16.639 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.640 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.641 DEBUG:boto:Params: {} 2022-09-03T06:07:16.642 DEBUG:boto:Token: None 2022-09-03T06:07:16.642 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:16.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lzVcCIIdooIYpg11FGDVRFgn20Q= 2022-09-03T06:07:16.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lzVcCIIdooIYpg11FGDVRFgn20Q=', 'Content-Length': '0'} 2022-09-03T06:07:16.648 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009eec097eb9192a44-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.649 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.651 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.651 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.652 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.653 DEBUG:boto:Method: GET 2022-09-03T06:07:16.654 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.655 DEBUG:boto:Data: 2022-09-03T06:07:16.655 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.656 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.656 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.657 DEBUG:boto:Params: {} 2022-09-03T06:07:16.658 DEBUG:boto:Token: None 2022-09-03T06:07:16.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.663 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c326e2754f25c14-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.709 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.711 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.711 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.712 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.713 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.714 DEBUG:boto:Method: GET 2022-09-03T06:07:16.715 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.716 DEBUG:boto:Data: 2022-09-03T06:07:16.717 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.719 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.720 DEBUG:boto:Params: {} 2022-09-03T06:07:16.721 DEBUG:boto:Token: None 2022-09-03T06:07:16.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.722 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.726 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003536cb8f6c01ffdc-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.728 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.729 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.730 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.731 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.732 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:16.733 DEBUG:boto:Data: 2022-09-03T06:07:16.733 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.735 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.735 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.736 DEBUG:boto:Params: {} 2022-09-03T06:07:16.736 DEBUG:boto:Token: None 2022-09-03T06:07:16.737 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj 2022-09-03T06:07:16.738 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM= 2022-09-03T06:07:16.738 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM=', 'Content-Length': '0'} 2022-09-03T06:07:16.742 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003351e9b1404419bf-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.743 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.743 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.744 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.744 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:16.745 DEBUG:boto:Data: 2022-09-03T06:07:16.745 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.746 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.747 DEBUG:boto:Params: {} 2022-09-03T06:07:16.747 DEBUG:boto:Token: None 2022-09-03T06:07:16.748 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj 2022-09-03T06:07:16.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM= 2022-09-03T06:07:16.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UwRcmfiS+6QKyy7FoNjo4Rw7agM=', 'Content-Length': '0'} 2022-09-03T06:07:16.753 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004661721ab58fd9f2-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.754 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.755 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.756 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.757 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.759 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.760 DEBUG:boto:Method: GET 2022-09-03T06:07:16.760 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.761 DEBUG:boto:Data: 2022-09-03T06:07:16.762 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.763 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.764 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.765 DEBUG:boto:Params: {} 2022-09-03T06:07:16.766 DEBUG:boto:Token: None 2022-09-03T06:07:16.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc= 2022-09-03T06:07:16.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc=', 'Content-Length': '0'} 2022-09-03T06:07:16.770 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000721969a4fc69b5f9-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.771 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.772 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.772 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.773 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.774 DEBUG:boto:Method: GET 2022-09-03T06:07:16.774 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.775 DEBUG:boto:Data: 2022-09-03T06:07:16.775 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.776 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.777 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.777 DEBUG:boto:Params: {} 2022-09-03T06:07:16.778 DEBUG:boto:Token: None 2022-09-03T06:07:16.778 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:16.779 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc= 2022-09-03T06:07:16.780 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YsLD36ajEzUykf45qx4RYne9FEc=', 'Content-Length': '0'} 2022-09-03T06:07:16.783 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006a712339a9bc508b-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.785 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.786 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.787 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.788 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.789 DEBUG:boto:Method: GET 2022-09-03T06:07:16.790 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.791 DEBUG:boto:Data: 2022-09-03T06:07:16.792 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.793 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.794 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.795 DEBUG:boto:Params: {} 2022-09-03T06:07:16.796 DEBUG:boto:Token: None 2022-09-03T06:07:16.797 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.803 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003629f8bf226c5f10-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.804 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.805 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.806 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.807 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.808 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.809 DEBUG:boto:Method: GET 2022-09-03T06:07:16.810 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.811 DEBUG:boto:Data: 2022-09-03T06:07:16.812 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.813 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.814 DEBUG:boto:Params: {} 2022-09-03T06:07:16.814 DEBUG:boto:Token: None 2022-09-03T06:07:16.815 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.818 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045977f0b23cea828-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.863 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:16.867 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.867 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.868 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.869 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.870 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.870 DEBUG:boto:Method: GET 2022-09-03T06:07:16.872 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.872 DEBUG:boto:Data: 2022-09-03T06:07:16.873 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.874 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.874 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.875 DEBUG:boto:Params: {} 2022-09-03T06:07:16.876 DEBUG:boto:Token: None 2022-09-03T06:07:16.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:twosQ1Mcqkurw6FA8w24cTYrrcA= 2022-09-03T06:07:16.878 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:twosQ1Mcqkurw6FA8w24cTYrrcA=', 'Content-Length': '0'} 2022-09-03T06:07:16.881 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fcc12e54ea90b8f7-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.883 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.884 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.885 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.886 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.887 DEBUG:boto:Method: GET 2022-09-03T06:07:16.888 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.889 DEBUG:boto:Data: 2022-09-03T06:07:16.889 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.890 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.891 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.892 DEBUG:boto:Params: {} 2022-09-03T06:07:16.893 DEBUG:boto:Token: None 2022-09-03T06:07:16.894 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:twosQ1Mcqkurw6FA8w24cTYrrcA= 2022-09-03T06:07:16.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:twosQ1Mcqkurw6FA8w24cTYrrcA=', 'Content-Length': '0'} 2022-09-03T06:07:16.900 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003581c49d5f202835-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.902 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.902 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.903 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.903 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.904 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.905 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.905 DEBUG:boto:Data: 2022-09-03T06:07:16.906 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.907 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.907 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.908 DEBUG:boto:Params: {} 2022-09-03T06:07:16.910 DEBUG:boto:Token: None 2022-09-03T06:07:16.911 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lFbE4m+kzWHdHkirIEaJSiizjCs= 2022-09-03T06:07:16.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lFbE4m+kzWHdHkirIEaJSiizjCs=', 'Content-Length': '0'} 2022-09-03T06:07:16.916 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dd836ad247aa0c0d-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.917 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.918 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.918 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.919 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.919 DEBUG:boto:Method: HEAD 2022-09-03T06:07:16.920 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.921 DEBUG:boto:Data: 2022-09-03T06:07:16.921 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.923 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.924 DEBUG:boto:Params: {} 2022-09-03T06:07:16.924 DEBUG:boto:Token: None 2022-09-03T06:07:16.925 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lFbE4m+kzWHdHkirIEaJSiizjCs= 2022-09-03T06:07:16.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lFbE4m+kzWHdHkirIEaJSiizjCs=', 'Content-Length': '0'} 2022-09-03T06:07:16.929 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006bf41d501f922019-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.930 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:16.931 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.932 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.934 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.935 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.937 DEBUG:boto:Method: GET 2022-09-03T06:07:16.938 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.939 DEBUG:boto:Data: 2022-09-03T06:07:16.940 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.942 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.943 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.944 DEBUG:boto:Params: {} 2022-09-03T06:07:16.945 DEBUG:boto:Token: None 2022-09-03T06:07:16.946 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:twosQ1Mcqkurw6FA8w24cTYrrcA= 2022-09-03T06:07:16.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:twosQ1Mcqkurw6FA8w24cTYrrcA=', 'Content-Length': '0'} 2022-09-03T06:07:16.955 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ac968374e8c99d0a-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.958 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:16.959 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:16.961 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.962 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.963 DEBUG:boto:Method: GET 2022-09-03T06:07:16.964 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.964 DEBUG:boto:Data: 2022-09-03T06:07:16.965 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:16.967 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.967 DEBUG:boto:Params: {} 2022-09-03T06:07:16.968 DEBUG:boto:Token: None 2022-09-03T06:07:16.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:16.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:twosQ1Mcqkurw6FA8w24cTYrrcA= 2022-09-03T06:07:16.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:twosQ1Mcqkurw6FA8w24cTYrrcA=', 'Content-Length': '0'} 2022-09-03T06:07:16.974 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a54de6932584963d-006312ef14-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:16.975 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:16.976 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:16.976 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.977 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:16.978 DEBUG:boto:Method: GET 2022-09-03T06:07:16.979 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:16.980 DEBUG:boto:Data: 2022-09-03T06:07:16.980 DEBUG:boto:Headers: {} 2022-09-03T06:07:16.981 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:16.982 DEBUG:boto:Port: 8000 2022-09-03T06:07:16.982 DEBUG:boto:Params: {} 2022-09-03T06:07:16.986 DEBUG:boto:Token: None 2022-09-03T06:07:16.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:16 GMT /avysmw-19/ 2022-09-03T06:07:16.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc= 2022-09-03T06:07:16.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB7fiXuMQ8k1C//9/pvXncyRPnc=', 'Content-Length': '0'} 2022-09-03T06:07:16.993 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b8f9fb79e3f4ea31-006312ef14-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.035 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.037 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.039 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.041 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.041 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.042 DEBUG:boto:Method: GET 2022-09-03T06:07:17.042 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.043 DEBUG:boto:Data: 2022-09-03T06:07:17.044 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.045 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.046 DEBUG:boto:Params: {} 2022-09-03T06:07:17.046 DEBUG:boto:Token: None 2022-09-03T06:07:17.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.051 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000871d70731f0fb060-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.052 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.053 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.053 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.054 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.054 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:17.055 DEBUG:boto:Data: 2022-09-03T06:07:17.056 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.056 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.057 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.057 DEBUG:boto:Params: {} 2022-09-03T06:07:17.058 DEBUG:boto:Token: None 2022-09-03T06:07:17.059 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj 2022-09-03T06:07:17.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M= 2022-09-03T06:07:17.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M=', 'Content-Length': '0'} 2022-09-03T06:07:17.063 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000251f74c453d8b607-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.064 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.065 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.065 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.066 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:17.066 DEBUG:boto:Data: 2022-09-03T06:07:17.067 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.068 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.068 DEBUG:boto:Params: {} 2022-09-03T06:07:17.069 DEBUG:boto:Token: None 2022-09-03T06:07:17.069 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj 2022-09-03T06:07:17.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M= 2022-09-03T06:07:17.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M=', 'Content-Length': '0'} 2022-09-03T06:07:17.082 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b5ec2f17f261acbb-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.084 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:17.085 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.085 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.086 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.087 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.088 DEBUG:boto:Method: GET 2022-09-03T06:07:17.089 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.090 DEBUG:boto:Data: 2022-09-03T06:07:17.091 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.092 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.092 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.093 DEBUG:boto:Params: {} 2022-09-03T06:07:17.094 DEBUG:boto:Token: None 2022-09-03T06:07:17.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM= 2022-09-03T06:07:17.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM=', 'Content-Length': '0'} 2022-09-03T06:07:17.100 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bb88320566fa13a3-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.101 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.101 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.102 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.102 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.103 DEBUG:boto:Method: GET 2022-09-03T06:07:17.104 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.104 DEBUG:boto:Data: 2022-09-03T06:07:17.105 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.106 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.106 DEBUG:boto:Params: {} 2022-09-03T06:07:17.107 DEBUG:boto:Token: None 2022-09-03T06:07:17.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM= 2022-09-03T06:07:17.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM=', 'Content-Length': '0'} 2022-09-03T06:07:17.111 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000058d91152c29ddcb9-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.154 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.155 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.156 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.157 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.158 DEBUG:boto:Method: GET 2022-09-03T06:07:17.159 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.160 DEBUG:boto:Data: 2022-09-03T06:07:17.161 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.162 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.162 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.163 DEBUG:boto:Params: {} 2022-09-03T06:07:17.164 DEBUG:boto:Token: None 2022-09-03T06:07:17.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.170 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d085987ea3aa3671-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.172 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.173 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.173 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.174 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.175 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.176 DEBUG:boto:Method: GET 2022-09-03T06:07:17.177 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.177 DEBUG:boto:Data: 2022-09-03T06:07:17.178 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.178 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.179 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.180 DEBUG:boto:Params: {} 2022-09-03T06:07:17.180 DEBUG:boto:Token: None 2022-09-03T06:07:17.181 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.183 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.191 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085e4c9064702829d-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.192 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.193 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:17.193 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.194 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.194 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.195 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.195 DEBUG:boto:Method: GET 2022-09-03T06:07:17.196 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.197 DEBUG:boto:Data: 2022-09-03T06:07:17.197 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.198 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.199 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.199 DEBUG:boto:Params: {} 2022-09-03T06:07:17.200 DEBUG:boto:Token: None 2022-09-03T06:07:17.200 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PZaqHxmwbyrLXqJcDD3YLsFp5Zo= 2022-09-03T06:07:17.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PZaqHxmwbyrLXqJcDD3YLsFp5Zo=', 'Content-Length': '0'} 2022-09-03T06:07:17.204 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000008b5fa7423c40c1d-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.246 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.247 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.248 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.249 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.249 DEBUG:boto:Method: GET 2022-09-03T06:07:17.250 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.251 DEBUG:boto:Data: 2022-09-03T06:07:17.252 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.253 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.254 DEBUG:boto:Params: {} 2022-09-03T06:07:17.255 DEBUG:boto:Token: None 2022-09-03T06:07:17.256 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PZaqHxmwbyrLXqJcDD3YLsFp5Zo= 2022-09-03T06:07:17.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PZaqHxmwbyrLXqJcDD3YLsFp5Zo=', 'Content-Length': '0'} 2022-09-03T06:07:17.260 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f19135a82272447b-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.261 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.262 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.263 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.263 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.264 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.264 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.266 DEBUG:boto:Data: 2022-09-03T06:07:17.266 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.267 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.268 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.269 DEBUG:boto:Params: {} 2022-09-03T06:07:17.269 DEBUG:boto:Token: None 2022-09-03T06:07:17.270 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i7ybDECmln54NN+Ux9JMb2XnF+o= 2022-09-03T06:07:17.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i7ybDECmln54NN+Ux9JMb2XnF+o=', 'Content-Length': '0'} 2022-09-03T06:07:17.279 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000058320e8b9beaf7e2-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.280 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.281 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.282 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.284 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.285 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.286 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.287 DEBUG:boto:Data: 2022-09-03T06:07:17.288 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.290 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.291 DEBUG:boto:Params: {} 2022-09-03T06:07:17.292 DEBUG:boto:Token: None 2022-09-03T06:07:17.293 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i7ybDECmln54NN+Ux9JMb2XnF+o= 2022-09-03T06:07:17.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i7ybDECmln54NN+Ux9JMb2XnF+o=', 'Content-Length': '0'} 2022-09-03T06:07:17.298 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ddecc9cf51baf89b-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.300 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:17.300 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.301 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.301 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.302 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.303 DEBUG:boto:Method: GET 2022-09-03T06:07:17.303 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.304 DEBUG:boto:Data: 2022-09-03T06:07:17.304 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.305 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.306 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.306 DEBUG:boto:Params: {} 2022-09-03T06:07:17.307 DEBUG:boto:Token: None 2022-09-03T06:07:17.308 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PZaqHxmwbyrLXqJcDD3YLsFp5Zo= 2022-09-03T06:07:17.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PZaqHxmwbyrLXqJcDD3YLsFp5Zo=', 'Content-Length': '0'} 2022-09-03T06:07:17.314 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006dfdace79ad02a7b-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.315 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.317 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.318 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.319 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.320 DEBUG:boto:Method: GET 2022-09-03T06:07:17.321 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.322 DEBUG:boto:Data: 2022-09-03T06:07:17.323 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.324 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.326 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.327 DEBUG:boto:Params: {} 2022-09-03T06:07:17.328 DEBUG:boto:Token: None 2022-09-03T06:07:17.329 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:17.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PZaqHxmwbyrLXqJcDD3YLsFp5Zo= 2022-09-03T06:07:17.332 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PZaqHxmwbyrLXqJcDD3YLsFp5Zo=', 'Content-Length': '0'} 2022-09-03T06:07:17.335 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000deb82b3c9f602b93-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.337 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.338 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.339 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.340 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.341 DEBUG:boto:Method: GET 2022-09-03T06:07:17.342 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.342 DEBUG:boto:Data: 2022-09-03T06:07:17.343 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.344 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.344 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.345 DEBUG:boto:Params: {} 2022-09-03T06:07:17.345 DEBUG:boto:Token: None 2022-09-03T06:07:17.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.351 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f26094bc7c0b3ebd-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.351 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.352 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.353 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.353 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.354 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.355 DEBUG:boto:Method: GET 2022-09-03T06:07:17.355 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.356 DEBUG:boto:Data: 2022-09-03T06:07:17.357 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.360 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.361 DEBUG:boto:Params: {} 2022-09-03T06:07:17.362 DEBUG:boto:Token: None 2022-09-03T06:07:17.363 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.368 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a42dccf2e5200b30-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.413 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.414 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.415 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.416 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.416 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:17.417 DEBUG:boto:Data: 2022-09-03T06:07:17.417 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.418 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.418 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.419 DEBUG:boto:Params: {} 2022-09-03T06:07:17.419 DEBUG:boto:Token: None 2022-09-03T06:07:17.420 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj 2022-09-03T06:07:17.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M= 2022-09-03T06:07:17.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M=', 'Content-Length': '0'} 2022-09-03T06:07:17.425 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005538e9abc207edbb-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.426 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.426 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.427 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.428 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:17.428 DEBUG:boto:Data: 2022-09-03T06:07:17.429 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.430 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.430 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.431 DEBUG:boto:Params: {} 2022-09-03T06:07:17.432 DEBUG:boto:Token: None 2022-09-03T06:07:17.432 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj 2022-09-03T06:07:17.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M= 2022-09-03T06:07:17.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M=', 'Content-Length': '0'} 2022-09-03T06:07:17.438 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000073e72465c872769d-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.439 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:17.440 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.441 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.441 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.442 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.442 DEBUG:boto:Method: GET 2022-09-03T06:07:17.443 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.444 DEBUG:boto:Data: 2022-09-03T06:07:17.445 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.446 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.447 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.448 DEBUG:boto:Params: {} 2022-09-03T06:07:17.449 DEBUG:boto:Token: None 2022-09-03T06:07:17.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM= 2022-09-03T06:07:17.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM=', 'Content-Length': '0'} 2022-09-03T06:07:17.455 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d461433ea989d5e6-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.458 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.459 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.460 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.461 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.462 DEBUG:boto:Method: GET 2022-09-03T06:07:17.463 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.464 DEBUG:boto:Data: 2022-09-03T06:07:17.465 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.467 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.468 DEBUG:boto:Params: {} 2022-09-03T06:07:17.469 DEBUG:boto:Token: None 2022-09-03T06:07:17.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM= 2022-09-03T06:07:17.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM=', 'Content-Length': '0'} 2022-09-03T06:07:17.475 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004ffda5c8eb291fbf-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.475 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.476 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.477 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.477 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.478 DEBUG:boto:Method: GET 2022-09-03T06:07:17.479 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.480 DEBUG:boto:Data: 2022-09-03T06:07:17.481 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.482 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.483 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.484 DEBUG:boto:Params: {} 2022-09-03T06:07:17.485 DEBUG:boto:Token: None 2022-09-03T06:07:17.486 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.487 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.487 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.491 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000009ed26f1b97b2223-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.534 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.535 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.536 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.536 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.537 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.537 DEBUG:boto:Method: GET 2022-09-03T06:07:17.538 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.539 DEBUG:boto:Data: 2022-09-03T06:07:17.539 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.541 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.541 DEBUG:boto:Params: {} 2022-09-03T06:07:17.542 DEBUG:boto:Token: None 2022-09-03T06:07:17.543 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.550 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020c54f92f0c19cc1-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.551 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.552 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:17.552 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.553 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.553 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.554 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.555 DEBUG:boto:Method: GET 2022-09-03T06:07:17.555 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.556 DEBUG:boto:Data: 2022-09-03T06:07:17.556 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.557 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.557 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.558 DEBUG:boto:Params: {} 2022-09-03T06:07:17.559 DEBUG:boto:Token: None 2022-09-03T06:07:17.559 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nC/9XPmZyXO7nK7c/6k9QCkMLRg= 2022-09-03T06:07:17.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nC/9XPmZyXO7nK7c/6k9QCkMLRg=', 'Content-Length': '0'} 2022-09-03T06:07:17.563 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006eee36a6f6d4fb4d-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.564 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.564 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.565 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.565 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.566 DEBUG:boto:Method: GET 2022-09-03T06:07:17.566 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.567 DEBUG:boto:Data: 2022-09-03T06:07:17.567 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.568 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.569 DEBUG:boto:Params: {} 2022-09-03T06:07:17.570 DEBUG:boto:Token: None 2022-09-03T06:07:17.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nC/9XPmZyXO7nK7c/6k9QCkMLRg= 2022-09-03T06:07:17.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nC/9XPmZyXO7nK7c/6k9QCkMLRg=', 'Content-Length': '0'} 2022-09-03T06:07:17.574 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c2d4663381b53db8-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.622 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.623 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.623 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.624 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.624 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.625 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.625 DEBUG:boto:Data: 2022-09-03T06:07:17.626 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.627 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.628 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.629 DEBUG:boto:Params: {} 2022-09-03T06:07:17.630 DEBUG:boto:Token: None 2022-09-03T06:07:17.630 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f2QhGfLMcXORDmpatk85nH88IkU= 2022-09-03T06:07:17.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f2QhGfLMcXORDmpatk85nH88IkU=', 'Content-Length': '0'} 2022-09-03T06:07:17.634 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e405e0c36bba7d64-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.635 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.636 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.636 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.637 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.637 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.638 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.638 DEBUG:boto:Data: 2022-09-03T06:07:17.639 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.640 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.641 DEBUG:boto:Params: {} 2022-09-03T06:07:17.641 DEBUG:boto:Token: None 2022-09-03T06:07:17.642 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f2QhGfLMcXORDmpatk85nH88IkU= 2022-09-03T06:07:17.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f2QhGfLMcXORDmpatk85nH88IkU=', 'Content-Length': '0'} 2022-09-03T06:07:17.646 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005092ae5fd9a2e0fe-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.647 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:17.647 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.648 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.649 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.649 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.650 DEBUG:boto:Method: GET 2022-09-03T06:07:17.651 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.651 DEBUG:boto:Data: 2022-09-03T06:07:17.652 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.653 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.654 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.654 DEBUG:boto:Params: {} 2022-09-03T06:07:17.655 DEBUG:boto:Token: None 2022-09-03T06:07:17.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nC/9XPmZyXO7nK7c/6k9QCkMLRg= 2022-09-03T06:07:17.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nC/9XPmZyXO7nK7c/6k9QCkMLRg=', 'Content-Length': '0'} 2022-09-03T06:07:17.659 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005ff14063afb131ea-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.660 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.661 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.662 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.662 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.663 DEBUG:boto:Method: GET 2022-09-03T06:07:17.663 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.664 DEBUG:boto:Data: 2022-09-03T06:07:17.664 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.665 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.666 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.666 DEBUG:boto:Params: {} 2022-09-03T06:07:17.667 DEBUG:boto:Token: None 2022-09-03T06:07:17.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:17.668 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nC/9XPmZyXO7nK7c/6k9QCkMLRg= 2022-09-03T06:07:17.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nC/9XPmZyXO7nK7c/6k9QCkMLRg=', 'Content-Length': '0'} 2022-09-03T06:07:17.671 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005ee247dcb83044ce-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.672 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.673 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.674 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.675 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.676 DEBUG:boto:Method: GET 2022-09-03T06:07:17.677 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.678 DEBUG:boto:Data: 2022-09-03T06:07:17.679 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.680 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.681 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.682 DEBUG:boto:Params: {} 2022-09-03T06:07:17.684 DEBUG:boto:Token: None 2022-09-03T06:07:17.684 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.690 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000542ec5f844d4c047-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.691 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.692 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.693 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.693 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.694 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.695 DEBUG:boto:Method: GET 2022-09-03T06:07:17.695 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.696 DEBUG:boto:Data: 2022-09-03T06:07:17.696 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.697 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.697 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.698 DEBUG:boto:Params: {} 2022-09-03T06:07:17.698 DEBUG:boto:Token: None 2022-09-03T06:07:17.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.702 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.705 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037cb6f093aa0b1c6-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.706 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.707 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.708 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.708 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.709 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:17.710 DEBUG:boto:Data: 2022-09-03T06:07:17.710 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.711 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.711 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.712 DEBUG:boto:Params: {} 2022-09-03T06:07:17.712 DEBUG:boto:Token: None 2022-09-03T06:07:17.713 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj 2022-09-03T06:07:17.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M= 2022-09-03T06:07:17.714 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M=', 'Content-Length': '0'} 2022-09-03T06:07:17.718 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c4f70874d8bf69b2-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.719 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.719 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.720 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.720 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:17.721 DEBUG:boto:Data: 2022-09-03T06:07:17.721 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.722 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.723 DEBUG:boto:Params: {} 2022-09-03T06:07:17.723 DEBUG:boto:Token: None 2022-09-03T06:07:17.724 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj 2022-09-03T06:07:17.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M= 2022-09-03T06:07:17.725 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M=', 'Content-Length': '0'} 2022-09-03T06:07:17.729 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000eb2e0592082b1592-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.729 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:17.730 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.731 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.731 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.732 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.732 DEBUG:boto:Method: GET 2022-09-03T06:07:17.733 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.733 DEBUG:boto:Data: 2022-09-03T06:07:17.734 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.734 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.735 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.735 DEBUG:boto:Params: {} 2022-09-03T06:07:17.736 DEBUG:boto:Token: None 2022-09-03T06:07:17.737 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM= 2022-09-03T06:07:17.738 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM=', 'Content-Length': '0'} 2022-09-03T06:07:17.740 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b57154f1f13d2b41-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.741 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.742 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.742 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.743 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.743 DEBUG:boto:Method: GET 2022-09-03T06:07:17.744 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.744 DEBUG:boto:Data: 2022-09-03T06:07:17.745 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.746 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.746 DEBUG:boto:Params: {} 2022-09-03T06:07:17.747 DEBUG:boto:Token: None 2022-09-03T06:07:17.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:17.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM= 2022-09-03T06:07:17.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKaKi/2TzVbwUbx2LWY39un5cwM=', 'Content-Length': '0'} 2022-09-03T06:07:17.751 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d2a92aa3f5bba7fa-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.752 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.753 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.753 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.754 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.754 DEBUG:boto:Method: GET 2022-09-03T06:07:17.755 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.755 DEBUG:boto:Data: 2022-09-03T06:07:17.756 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.756 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.757 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.757 DEBUG:boto:Params: {} 2022-09-03T06:07:17.758 DEBUG:boto:Token: None 2022-09-03T06:07:17.759 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.763 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c6c0a3c4079b2db-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.764 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.764 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.765 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.765 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.766 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.766 DEBUG:boto:Method: GET 2022-09-03T06:07:17.767 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.768 DEBUG:boto:Data: 2022-09-03T06:07:17.768 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.769 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.769 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.770 DEBUG:boto:Params: {} 2022-09-03T06:07:17.770 DEBUG:boto:Token: None 2022-09-03T06:07:17.771 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.775 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7fe6a71200605b4-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.775 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.776 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:17.777 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.777 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.778 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.778 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.779 DEBUG:boto:Method: GET 2022-09-03T06:07:17.779 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.780 DEBUG:boto:Data: 2022-09-03T06:07:17.781 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.781 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.782 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.782 DEBUG:boto:Params: {} 2022-09-03T06:07:17.783 DEBUG:boto:Token: None 2022-09-03T06:07:17.783 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GB/MkA9cOrLrUrRtat6O4YVNHFo= 2022-09-03T06:07:17.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GB/MkA9cOrLrUrRtat6O4YVNHFo=', 'Content-Length': '0'} 2022-09-03T06:07:17.787 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000d173ae72bdb8ad9-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.834 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.835 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.836 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.836 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.837 DEBUG:boto:Method: GET 2022-09-03T06:07:17.838 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.839 DEBUG:boto:Data: 2022-09-03T06:07:17.840 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.841 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.843 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.844 DEBUG:boto:Params: {} 2022-09-03T06:07:17.845 DEBUG:boto:Token: None 2022-09-03T06:07:17.846 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.848 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GB/MkA9cOrLrUrRtat6O4YVNHFo= 2022-09-03T06:07:17.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GB/MkA9cOrLrUrRtat6O4YVNHFo=', 'Content-Length': '0'} 2022-09-03T06:07:17.851 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000032814e1d445a6421-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.852 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.853 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.853 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.854 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.855 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.855 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.856 DEBUG:boto:Data: 2022-09-03T06:07:17.856 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.857 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.858 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.858 DEBUG:boto:Params: {} 2022-09-03T06:07:17.859 DEBUG:boto:Token: None 2022-09-03T06:07:17.859 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XgcTrnmxAZ6cmc7HwBM4yhVc/Fg= 2022-09-03T06:07:17.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XgcTrnmxAZ6cmc7HwBM4yhVc/Fg=', 'Content-Length': '0'} 2022-09-03T06:07:17.864 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000794b718f2aed06a4-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.865 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.865 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.866 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.867 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.867 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.868 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.868 DEBUG:boto:Data: 2022-09-03T06:07:17.869 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.870 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.870 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.871 DEBUG:boto:Params: {} 2022-09-03T06:07:17.871 DEBUG:boto:Token: None 2022-09-03T06:07:17.872 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XgcTrnmxAZ6cmc7HwBM4yhVc/Fg= 2022-09-03T06:07:17.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XgcTrnmxAZ6cmc7HwBM4yhVc/Fg=', 'Content-Length': '0'} 2022-09-03T06:07:17.876 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008026be895eb57d97-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.877 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:17.877 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.878 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.878 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.879 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.879 DEBUG:boto:Method: GET 2022-09-03T06:07:17.880 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.881 DEBUG:boto:Data: 2022-09-03T06:07:17.881 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.882 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.882 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.883 DEBUG:boto:Params: {} 2022-09-03T06:07:17.883 DEBUG:boto:Token: None 2022-09-03T06:07:17.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GB/MkA9cOrLrUrRtat6O4YVNHFo= 2022-09-03T06:07:17.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GB/MkA9cOrLrUrRtat6O4YVNHFo=', 'Content-Length': '0'} 2022-09-03T06:07:17.889 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000079d6b11e14b351e3-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.891 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.892 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.893 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.894 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.895 DEBUG:boto:Method: GET 2022-09-03T06:07:17.896 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.898 DEBUG:boto:Data: 2022-09-03T06:07:17.899 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.900 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.901 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.902 DEBUG:boto:Params: {} 2022-09-03T06:07:17.903 DEBUG:boto:Token: None 2022-09-03T06:07:17.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:17.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GB/MkA9cOrLrUrRtat6O4YVNHFo= 2022-09-03T06:07:17.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GB/MkA9cOrLrUrRtat6O4YVNHFo=', 'Content-Length': '0'} 2022-09-03T06:07:17.908 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000092a9eda4d9bc82ad-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.954 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.955 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.956 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.957 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.958 DEBUG:boto:Method: GET 2022-09-03T06:07:17.959 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.961 DEBUG:boto:Data: 2022-09-03T06:07:17.962 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.963 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.963 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.964 DEBUG:boto:Params: {} 2022-09-03T06:07:17.965 DEBUG:boto:Token: None 2022-09-03T06:07:17.966 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.971 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e169d0a24e6d995-006312ef15-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.972 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.973 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:17.974 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:17.975 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.975 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:17.976 DEBUG:boto:Method: GET 2022-09-03T06:07:17.977 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:17.977 DEBUG:boto:Data: 2022-09-03T06:07:17.978 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:17.979 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.980 DEBUG:boto:Params: {} 2022-09-03T06:07:17.981 DEBUG:boto:Token: None 2022-09-03T06:07:17.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/ 2022-09-03T06:07:17.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs= 2022-09-03T06:07:17.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RbPYrK2sFvVp4KcZX8QySyFoLAs=', 'Content-Length': '0'} 2022-09-03T06:07:17.992 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007f19cc37f0f77acd-006312ef15-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:17.992 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:17.994 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:17.995 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:17.995 DEBUG:boto:Method: HEAD 2022-09-03T06:07:17.996 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:17.996 DEBUG:boto:Data: 2022-09-03T06:07:17.997 DEBUG:boto:Headers: {} 2022-09-03T06:07:17.997 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:17.998 DEBUG:boto:Port: 8000 2022-09-03T06:07:17.998 DEBUG:boto:Params: {} 2022-09-03T06:07:17.999 DEBUG:boto:Token: None 2022-09-03T06:07:17.999 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:17 GMT /avysmw-19/obj 2022-09-03T06:07:18.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M= 2022-09-03T06:07:18.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nmMq/uSl7PYQdmHh/qO1QxH4c5M=', 'Content-Length': '0'} 2022-09-03T06:07:18.004 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ca596efcc1a8aa88-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.005 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.006 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.007 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.007 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:18.008 DEBUG:boto:Data: 2022-09-03T06:07:18.008 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.009 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.010 DEBUG:boto:Params: {} 2022-09-03T06:07:18.011 DEBUG:boto:Token: None 2022-09-03T06:07:18.012 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj 2022-09-03T06:07:18.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w= 2022-09-03T06:07:18.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w=', 'Content-Length': '0'} 2022-09-03T06:07:18.018 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b79531c6699bc060-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.019 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.020 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.021 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.022 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.023 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.024 DEBUG:boto:Method: GET 2022-09-03T06:07:18.025 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.026 DEBUG:boto:Data: 2022-09-03T06:07:18.026 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.027 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.028 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.029 DEBUG:boto:Params: {} 2022-09-03T06:07:18.030 DEBUG:boto:Token: None 2022-09-03T06:07:18.031 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw= 2022-09-03T06:07:18.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw=', 'Content-Length': '0'} 2022-09-03T06:07:18.036 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000e0e26bacde9331f-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.081 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.082 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.083 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.084 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.085 DEBUG:boto:Method: GET 2022-09-03T06:07:18.086 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.087 DEBUG:boto:Data: 2022-09-03T06:07:18.088 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.089 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.092 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.094 DEBUG:boto:Params: {} 2022-09-03T06:07:18.096 DEBUG:boto:Token: None 2022-09-03T06:07:18.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw= 2022-09-03T06:07:18.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw=', 'Content-Length': '0'} 2022-09-03T06:07:18.107 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005203d619abac4a97-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.108 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.109 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.110 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.111 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.112 DEBUG:boto:Method: GET 2022-09-03T06:07:18.113 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.114 DEBUG:boto:Data: 2022-09-03T06:07:18.115 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.116 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.117 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.118 DEBUG:boto:Params: {} 2022-09-03T06:07:18.119 DEBUG:boto:Token: None 2022-09-03T06:07:18.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.126 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000123a9be942bcf62-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.128 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.129 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.130 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.131 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.132 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.133 DEBUG:boto:Method: GET 2022-09-03T06:07:18.134 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.135 DEBUG:boto:Data: 2022-09-03T06:07:18.136 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.138 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.139 DEBUG:boto:Params: {} 2022-09-03T06:07:18.140 DEBUG:boto:Token: None 2022-09-03T06:07:18.141 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.150 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a6c292cfc5857f2-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.152 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.154 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.155 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.156 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.156 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.157 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.158 DEBUG:boto:Method: GET 2022-09-03T06:07:18.159 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.159 DEBUG:boto:Data: 2022-09-03T06:07:18.160 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.161 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.162 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.163 DEBUG:boto:Params: {} 2022-09-03T06:07:18.164 DEBUG:boto:Token: None 2022-09-03T06:07:18.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2DPVq0t+63SeFbS9MpV7gUZr+iM= 2022-09-03T06:07:18.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2DPVq0t+63SeFbS9MpV7gUZr+iM=', 'Content-Length': '0'} 2022-09-03T06:07:18.170 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ed9f7e8b4ddfbcb3-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.172 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.172 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.173 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.174 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.174 DEBUG:boto:Method: GET 2022-09-03T06:07:18.175 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.175 DEBUG:boto:Data: 2022-09-03T06:07:18.176 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.177 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.178 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.179 DEBUG:boto:Params: {} 2022-09-03T06:07:18.180 DEBUG:boto:Token: None 2022-09-03T06:07:18.181 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2DPVq0t+63SeFbS9MpV7gUZr+iM= 2022-09-03T06:07:18.183 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2DPVq0t+63SeFbS9MpV7gUZr+iM=', 'Content-Length': '0'} 2022-09-03T06:07:18.186 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008a7dbb5fdf3ca359-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.226 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.227 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.227 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.228 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.228 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.229 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.229 DEBUG:boto:Data: 2022-09-03T06:07:18.230 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.230 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.231 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.231 DEBUG:boto:Params: {} 2022-09-03T06:07:18.232 DEBUG:boto:Token: None 2022-09-03T06:07:18.233 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06://kBguisgBVtkhjd4kVFjQu/cbI= 2022-09-03T06:07:18.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06://kBguisgBVtkhjd4kVFjQu/cbI=', 'Content-Length': '0'} 2022-09-03T06:07:18.236 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e0c6aa4322913910-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.236 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.237 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.237 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.238 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.238 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.239 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.239 DEBUG:boto:Data: 2022-09-03T06:07:18.240 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.241 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.241 DEBUG:boto:Params: {} 2022-09-03T06:07:18.242 DEBUG:boto:Token: None 2022-09-03T06:07:18.242 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06://kBguisgBVtkhjd4kVFjQu/cbI= 2022-09-03T06:07:18.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06://kBguisgBVtkhjd4kVFjQu/cbI=', 'Content-Length': '0'} 2022-09-03T06:07:18.246 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000707ede5870754ab7-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.246 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.247 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.247 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.248 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.248 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.249 DEBUG:boto:Method: GET 2022-09-03T06:07:18.250 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.250 DEBUG:boto:Data: 2022-09-03T06:07:18.251 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.251 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.252 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.252 DEBUG:boto:Params: {} 2022-09-03T06:07:18.253 DEBUG:boto:Token: None 2022-09-03T06:07:18.253 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2DPVq0t+63SeFbS9MpV7gUZr+iM= 2022-09-03T06:07:18.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2DPVq0t+63SeFbS9MpV7gUZr+iM=', 'Content-Length': '0'} 2022-09-03T06:07:18.257 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ca363378b004ea3c-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.258 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.258 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.259 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.259 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.260 DEBUG:boto:Method: GET 2022-09-03T06:07:18.260 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.261 DEBUG:boto:Data: 2022-09-03T06:07:18.261 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.263 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.263 DEBUG:boto:Params: {} 2022-09-03T06:07:18.264 DEBUG:boto:Token: None 2022-09-03T06:07:18.264 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:18.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2DPVq0t+63SeFbS9MpV7gUZr+iM= 2022-09-03T06:07:18.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2DPVq0t+63SeFbS9MpV7gUZr+iM=', 'Content-Length': '0'} 2022-09-03T06:07:18.268 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000020c6a1109dd6a52f-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.269 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.269 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.270 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.270 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.271 DEBUG:boto:Method: GET 2022-09-03T06:07:18.271 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.272 DEBUG:boto:Data: 2022-09-03T06:07:18.272 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.273 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.274 DEBUG:boto:Params: {} 2022-09-03T06:07:18.274 DEBUG:boto:Token: None 2022-09-03T06:07:18.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.279 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009cbaff537eb7f4f1-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.280 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.281 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.281 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.282 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.282 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.283 DEBUG:boto:Method: GET 2022-09-03T06:07:18.283 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.284 DEBUG:boto:Data: 2022-09-03T06:07:18.284 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.285 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.286 DEBUG:boto:Params: {} 2022-09-03T06:07:18.286 DEBUG:boto:Token: None 2022-09-03T06:07:18.287 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.291 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da24c4b5df4d6edb-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.292 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.293 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.294 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.294 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.295 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:18.296 DEBUG:boto:Data: 2022-09-03T06:07:18.296 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.297 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.297 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.298 DEBUG:boto:Params: {} 2022-09-03T06:07:18.299 DEBUG:boto:Token: None 2022-09-03T06:07:18.299 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj 2022-09-03T06:07:18.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w= 2022-09-03T06:07:18.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w=', 'Content-Length': '0'} 2022-09-03T06:07:18.304 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000424e6470480fa2fb-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.305 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.306 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.306 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.307 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:18.308 DEBUG:boto:Data: 2022-09-03T06:07:18.308 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.309 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.310 DEBUG:boto:Params: {} 2022-09-03T06:07:18.310 DEBUG:boto:Token: None 2022-09-03T06:07:18.311 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj 2022-09-03T06:07:18.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w= 2022-09-03T06:07:18.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w=', 'Content-Length': '0'} 2022-09-03T06:07:18.316 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000e821198c09a701c-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.317 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.318 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.318 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.319 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.319 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.320 DEBUG:boto:Method: GET 2022-09-03T06:07:18.320 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.322 DEBUG:boto:Data: 2022-09-03T06:07:18.322 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.323 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.324 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.324 DEBUG:boto:Params: {} 2022-09-03T06:07:18.325 DEBUG:boto:Token: None 2022-09-03T06:07:18.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw= 2022-09-03T06:07:18.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw=', 'Content-Length': '0'} 2022-09-03T06:07:18.329 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a27dbc8b4fd6550a-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.330 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.331 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.331 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.332 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.332 DEBUG:boto:Method: GET 2022-09-03T06:07:18.333 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.333 DEBUG:boto:Data: 2022-09-03T06:07:18.334 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.335 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.336 DEBUG:boto:Params: {} 2022-09-03T06:07:18.336 DEBUG:boto:Token: None 2022-09-03T06:07:18.337 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.338 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw= 2022-09-03T06:07:18.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw=', 'Content-Length': '0'} 2022-09-03T06:07:18.340 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006b4be91e5c3db49e-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.382 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.383 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.384 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.384 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.385 DEBUG:boto:Method: GET 2022-09-03T06:07:18.385 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.386 DEBUG:boto:Data: 2022-09-03T06:07:18.386 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.387 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.388 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.388 DEBUG:boto:Params: {} 2022-09-03T06:07:18.389 DEBUG:boto:Token: None 2022-09-03T06:07:18.390 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.394 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b0c0c7ebbe2602b-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.395 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.396 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.397 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.398 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.399 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.400 DEBUG:boto:Method: GET 2022-09-03T06:07:18.401 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.401 DEBUG:boto:Data: 2022-09-03T06:07:18.402 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.403 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.404 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.405 DEBUG:boto:Params: {} 2022-09-03T06:07:18.405 DEBUG:boto:Token: None 2022-09-03T06:07:18.406 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.407 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.410 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000185ed6fa936cae8e-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.411 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.412 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.413 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.413 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.414 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.414 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.415 DEBUG:boto:Method: GET 2022-09-03T06:07:18.415 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.416 DEBUG:boto:Data: 2022-09-03T06:07:18.416 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.417 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.417 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.418 DEBUG:boto:Params: {} 2022-09-03T06:07:18.418 DEBUG:boto:Token: None 2022-09-03T06:07:18.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Wen3t0eA4PMrBIQXTN016z2AZ8= 2022-09-03T06:07:18.421 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Wen3t0eA4PMrBIQXTN016z2AZ8=', 'Content-Length': '0'} 2022-09-03T06:07:18.423 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000113fff333c1772bb-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.470 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.471 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.472 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.472 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.473 DEBUG:boto:Method: GET 2022-09-03T06:07:18.473 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.474 DEBUG:boto:Data: 2022-09-03T06:07:18.474 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.475 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.476 DEBUG:boto:Params: {} 2022-09-03T06:07:18.476 DEBUG:boto:Token: None 2022-09-03T06:07:18.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Wen3t0eA4PMrBIQXTN016z2AZ8= 2022-09-03T06:07:18.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Wen3t0eA4PMrBIQXTN016z2AZ8=', 'Content-Length': '0'} 2022-09-03T06:07:18.480 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b2271f7084686bf2-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.481 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.482 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.482 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.483 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.483 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.484 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.484 DEBUG:boto:Data: 2022-09-03T06:07:18.485 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.485 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.485 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.486 DEBUG:boto:Params: {} 2022-09-03T06:07:18.487 DEBUG:boto:Token: None 2022-09-03T06:07:18.487 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i3jphsnBuB2xqzDi8S2g9d4/vdU= 2022-09-03T06:07:18.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i3jphsnBuB2xqzDi8S2g9d4/vdU=', 'Content-Length': '0'} 2022-09-03T06:07:18.490 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005c019604683b0fc5-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.491 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.492 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.492 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.493 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.493 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.493 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.494 DEBUG:boto:Data: 2022-09-03T06:07:18.495 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.495 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.495 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.496 DEBUG:boto:Params: {} 2022-09-03T06:07:18.497 DEBUG:boto:Token: None 2022-09-03T06:07:18.497 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i3jphsnBuB2xqzDi8S2g9d4/vdU= 2022-09-03T06:07:18.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i3jphsnBuB2xqzDi8S2g9d4/vdU=', 'Content-Length': '0'} 2022-09-03T06:07:18.500 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d7a7f04b46858605-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.501 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.501 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.502 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.502 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.503 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.503 DEBUG:boto:Method: GET 2022-09-03T06:07:18.504 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.505 DEBUG:boto:Data: 2022-09-03T06:07:18.505 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.506 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.506 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.507 DEBUG:boto:Params: {} 2022-09-03T06:07:18.507 DEBUG:boto:Token: None 2022-09-03T06:07:18.508 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Wen3t0eA4PMrBIQXTN016z2AZ8= 2022-09-03T06:07:18.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Wen3t0eA4PMrBIQXTN016z2AZ8=', 'Content-Length': '0'} 2022-09-03T06:07:18.511 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006b1f7d74144e55a7-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.512 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.512 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.513 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.513 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.514 DEBUG:boto:Method: GET 2022-09-03T06:07:18.514 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.515 DEBUG:boto:Data: 2022-09-03T06:07:18.515 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.517 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.517 DEBUG:boto:Params: {} 2022-09-03T06:07:18.518 DEBUG:boto:Token: None 2022-09-03T06:07:18.518 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:18.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Wen3t0eA4PMrBIQXTN016z2AZ8= 2022-09-03T06:07:18.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Wen3t0eA4PMrBIQXTN016z2AZ8=', 'Content-Length': '0'} 2022-09-03T06:07:18.522 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002a3014444c455639-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.522 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.523 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.524 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.524 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.525 DEBUG:boto:Method: GET 2022-09-03T06:07:18.525 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.526 DEBUG:boto:Data: 2022-09-03T06:07:18.526 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.527 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.527 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.528 DEBUG:boto:Params: {} 2022-09-03T06:07:18.528 DEBUG:boto:Token: None 2022-09-03T06:07:18.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.533 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063f958177e292c15-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.533 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.534 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.535 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.535 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.536 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.536 DEBUG:boto:Method: GET 2022-09-03T06:07:18.537 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.537 DEBUG:boto:Data: 2022-09-03T06:07:18.538 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.539 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.539 DEBUG:boto:Params: {} 2022-09-03T06:07:18.540 DEBUG:boto:Token: None 2022-09-03T06:07:18.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.544 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ef036085d1624009-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.545 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.546 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.546 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.547 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.547 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:18.548 DEBUG:boto:Data: 2022-09-03T06:07:18.548 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.549 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.549 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.550 DEBUG:boto:Params: {} 2022-09-03T06:07:18.550 DEBUG:boto:Token: None 2022-09-03T06:07:18.551 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj 2022-09-03T06:07:18.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w= 2022-09-03T06:07:18.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w=', 'Content-Length': '0'} 2022-09-03T06:07:18.555 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000095a14c834ac9cd89-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.556 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.557 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.557 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.558 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:18.558 DEBUG:boto:Data: 2022-09-03T06:07:18.559 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.560 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.560 DEBUG:boto:Params: {} 2022-09-03T06:07:18.561 DEBUG:boto:Token: None 2022-09-03T06:07:18.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj 2022-09-03T06:07:18.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w= 2022-09-03T06:07:18.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w=', 'Content-Length': '0'} 2022-09-03T06:07:18.566 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fa01303de00b7e7f-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.567 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.567 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.568 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.568 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.569 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.569 DEBUG:boto:Method: GET 2022-09-03T06:07:18.570 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.570 DEBUG:boto:Data: 2022-09-03T06:07:18.571 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.571 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.572 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.572 DEBUG:boto:Params: {} 2022-09-03T06:07:18.573 DEBUG:boto:Token: None 2022-09-03T06:07:18.574 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw= 2022-09-03T06:07:18.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw=', 'Content-Length': '0'} 2022-09-03T06:07:18.577 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008dac8081b9a1c6cb-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.618 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.619 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.619 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.620 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.620 DEBUG:boto:Method: GET 2022-09-03T06:07:18.621 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.622 DEBUG:boto:Data: 2022-09-03T06:07:18.622 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.623 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.623 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.624 DEBUG:boto:Params: {} 2022-09-03T06:07:18.625 DEBUG:boto:Token: None 2022-09-03T06:07:18.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw= 2022-09-03T06:07:18.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw=', 'Content-Length': '0'} 2022-09-03T06:07:18.629 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d13c50ced8ccaff2-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.630 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.630 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.631 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.631 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.632 DEBUG:boto:Method: GET 2022-09-03T06:07:18.632 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.633 DEBUG:boto:Data: 2022-09-03T06:07:18.633 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.634 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.635 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.635 DEBUG:boto:Params: {} 2022-09-03T06:07:18.636 DEBUG:boto:Token: None 2022-09-03T06:07:18.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.641 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a302290e39a4bc70-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.641 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.642 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.643 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.643 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.644 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.644 DEBUG:boto:Method: GET 2022-09-03T06:07:18.645 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.645 DEBUG:boto:Data: 2022-09-03T06:07:18.646 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.647 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.648 DEBUG:boto:Params: {} 2022-09-03T06:07:18.648 DEBUG:boto:Token: None 2022-09-03T06:07:18.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.653 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000043c6faa8e9c9fa52-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.654 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.655 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.655 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.656 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.657 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.657 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.658 DEBUG:boto:Method: GET 2022-09-03T06:07:18.658 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.659 DEBUG:boto:Data: 2022-09-03T06:07:18.659 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.660 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.660 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.661 DEBUG:boto:Params: {} 2022-09-03T06:07:18.661 DEBUG:boto:Token: None 2022-09-03T06:07:18.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qHynDdqtF5tmfugdh82L/EWKnF0= 2022-09-03T06:07:18.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qHynDdqtF5tmfugdh82L/EWKnF0=', 'Content-Length': '0'} 2022-09-03T06:07:18.665 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cb1c0a420f5e1de7-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.666 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.667 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.667 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.668 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.668 DEBUG:boto:Method: GET 2022-09-03T06:07:18.669 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.669 DEBUG:boto:Data: 2022-09-03T06:07:18.670 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.671 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.671 DEBUG:boto:Params: {} 2022-09-03T06:07:18.672 DEBUG:boto:Token: None 2022-09-03T06:07:18.673 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qHynDdqtF5tmfugdh82L/EWKnF0= 2022-09-03T06:07:18.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qHynDdqtF5tmfugdh82L/EWKnF0=', 'Content-Length': '0'} 2022-09-03T06:07:18.676 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ada7e9db1f37699e-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.677 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.677 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.678 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.678 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.679 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.679 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.680 DEBUG:boto:Data: 2022-09-03T06:07:18.680 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.681 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.681 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.682 DEBUG:boto:Params: {} 2022-09-03T06:07:18.683 DEBUG:boto:Token: None 2022-09-03T06:07:18.683 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+K7xDCRdkmY8ANo6v3YKv0+GUro= 2022-09-03T06:07:18.685 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+K7xDCRdkmY8ANo6v3YKv0+GUro=', 'Content-Length': '0'} 2022-09-03T06:07:18.687 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000080eb188291822d00-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.688 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.689 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.691 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.692 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.693 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.694 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.695 DEBUG:boto:Data: 2022-09-03T06:07:18.695 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.696 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.697 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.698 DEBUG:boto:Params: {} 2022-09-03T06:07:18.699 DEBUG:boto:Token: None 2022-09-03T06:07:18.700 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.701 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+K7xDCRdkmY8ANo6v3YKv0+GUro= 2022-09-03T06:07:18.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+K7xDCRdkmY8ANo6v3YKv0+GUro=', 'Content-Length': '0'} 2022-09-03T06:07:18.705 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ded2e548544dd581-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.706 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.706 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.707 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.707 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.708 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.708 DEBUG:boto:Method: GET 2022-09-03T06:07:18.709 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.709 DEBUG:boto:Data: 2022-09-03T06:07:18.710 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.710 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.711 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.712 DEBUG:boto:Params: {} 2022-09-03T06:07:18.712 DEBUG:boto:Token: None 2022-09-03T06:07:18.713 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.714 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qHynDdqtF5tmfugdh82L/EWKnF0= 2022-09-03T06:07:18.725 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qHynDdqtF5tmfugdh82L/EWKnF0=', 'Content-Length': '0'} 2022-09-03T06:07:18.733 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007c75e2b0cd15e212-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.734 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.734 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.735 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.735 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.736 DEBUG:boto:Method: GET 2022-09-03T06:07:18.737 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.737 DEBUG:boto:Data: 2022-09-03T06:07:18.738 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.739 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.739 DEBUG:boto:Params: {} 2022-09-03T06:07:18.740 DEBUG:boto:Token: None 2022-09-03T06:07:18.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:18.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qHynDdqtF5tmfugdh82L/EWKnF0= 2022-09-03T06:07:18.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qHynDdqtF5tmfugdh82L/EWKnF0=', 'Content-Length': '0'} 2022-09-03T06:07:18.744 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008dacd6d865e940f2-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.745 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.745 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.746 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.747 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.747 DEBUG:boto:Method: GET 2022-09-03T06:07:18.748 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.748 DEBUG:boto:Data: 2022-09-03T06:07:18.749 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.749 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.750 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.750 DEBUG:boto:Params: {} 2022-09-03T06:07:18.751 DEBUG:boto:Token: None 2022-09-03T06:07:18.752 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.756 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf3dcf7a890d0df9-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.798 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.803 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.804 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.805 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.805 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.807 DEBUG:boto:Method: GET 2022-09-03T06:07:18.808 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.809 DEBUG:boto:Data: 2022-09-03T06:07:18.809 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.811 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.811 DEBUG:boto:Params: {} 2022-09-03T06:07:18.812 DEBUG:boto:Token: None 2022-09-03T06:07:18.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.817 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028ce8b887d1bdc1c-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.818 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.819 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.819 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.820 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.821 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:18.821 DEBUG:boto:Data: 2022-09-03T06:07:18.822 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.822 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.823 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.823 DEBUG:boto:Params: {} 2022-09-03T06:07:18.824 DEBUG:boto:Token: None 2022-09-03T06:07:18.824 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj 2022-09-03T06:07:18.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w= 2022-09-03T06:07:18.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w=', 'Content-Length': '0'} 2022-09-03T06:07:18.829 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000eed399f650f76bdd-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.830 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.831 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.831 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.832 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:18.833 DEBUG:boto:Data: 2022-09-03T06:07:18.833 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.834 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.835 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.836 DEBUG:boto:Params: {} 2022-09-03T06:07:18.836 DEBUG:boto:Token: None 2022-09-03T06:07:18.838 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj 2022-09-03T06:07:18.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w= 2022-09-03T06:07:18.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f1sbZv+VJwR0rMvI9ZzMYgcCI7w=', 'Content-Length': '0'} 2022-09-03T06:07:18.843 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002c18a99fddee98c7-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.843 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.844 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.845 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.845 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.846 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.846 DEBUG:boto:Method: GET 2022-09-03T06:07:18.847 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.847 DEBUG:boto:Data: 2022-09-03T06:07:18.848 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.849 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.849 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.850 DEBUG:boto:Params: {} 2022-09-03T06:07:18.850 DEBUG:boto:Token: None 2022-09-03T06:07:18.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw= 2022-09-03T06:07:18.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw=', 'Content-Length': '0'} 2022-09-03T06:07:18.855 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000055f498f582fb363a-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.856 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.856 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.857 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.857 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.858 DEBUG:boto:Method: GET 2022-09-03T06:07:18.858 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.859 DEBUG:boto:Data: 2022-09-03T06:07:18.859 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.860 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.860 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.861 DEBUG:boto:Params: {} 2022-09-03T06:07:18.861 DEBUG:boto:Token: None 2022-09-03T06:07:18.862 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:18.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw= 2022-09-03T06:07:18.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xqocZiWutPSHHohHIhSgkAeeOFw=', 'Content-Length': '0'} 2022-09-03T06:07:18.865 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008ce5127572abd287-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.866 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.867 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.867 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.868 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.869 DEBUG:boto:Method: GET 2022-09-03T06:07:18.869 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.870 DEBUG:boto:Data: 2022-09-03T06:07:18.870 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.871 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.871 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.872 DEBUG:boto:Params: {} 2022-09-03T06:07:18.872 DEBUG:boto:Token: None 2022-09-03T06:07:18.873 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.874 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.877 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000700c17fd93c762f3-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.878 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.879 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:18.879 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:18.880 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.880 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:18.881 DEBUG:boto:Method: GET 2022-09-03T06:07:18.881 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:18.882 DEBUG:boto:Data: 2022-09-03T06:07:18.882 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.883 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.884 DEBUG:boto:Params: {} 2022-09-03T06:07:18.885 DEBUG:boto:Token: None 2022-09-03T06:07:18.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/ 2022-09-03T06:07:18.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM= 2022-09-03T06:07:18.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5t4WuTmo9I/L58wAWhN6U3qqvNM=', 'Content-Length': '0'} 2022-09-03T06:07:18.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007f893ffe996dc7c6-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.890 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:18.891 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.892 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.892 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.893 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.894 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.894 DEBUG:boto:Method: GET 2022-09-03T06:07:18.895 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.895 DEBUG:boto:Data: 2022-09-03T06:07:18.896 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.897 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.897 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.898 DEBUG:boto:Params: {} 2022-09-03T06:07:18.898 DEBUG:boto:Token: None 2022-09-03T06:07:18.899 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tYqDZNo+CTMQONlEYUALDEjgrA0= 2022-09-03T06:07:18.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tYqDZNo+CTMQONlEYUALDEjgrA0=', 'Content-Length': '0'} 2022-09-03T06:07:18.903 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000eee517c649673a0f-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.904 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.904 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.905 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.905 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.906 DEBUG:boto:Method: GET 2022-09-03T06:07:18.906 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.907 DEBUG:boto:Data: 2022-09-03T06:07:18.907 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.908 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.908 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.909 DEBUG:boto:Params: {} 2022-09-03T06:07:18.909 DEBUG:boto:Token: None 2022-09-03T06:07:18.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tYqDZNo+CTMQONlEYUALDEjgrA0= 2022-09-03T06:07:18.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tYqDZNo+CTMQONlEYUALDEjgrA0=', 'Content-Length': '0'} 2022-09-03T06:07:18.914 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000060f703b86e4fe6e3-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.914 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.915 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.915 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.916 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.916 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.917 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.917 DEBUG:boto:Data: 2022-09-03T06:07:18.918 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.919 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.919 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.920 DEBUG:boto:Params: {} 2022-09-03T06:07:18.920 DEBUG:boto:Token: None 2022-09-03T06:07:18.921 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.921 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:61gDchr01RGscol025J49rvJw40= 2022-09-03T06:07:18.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:61gDchr01RGscol025J49rvJw40=', 'Content-Length': '0'} 2022-09-03T06:07:18.924 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000a25cb5a45fc2212-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.925 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.925 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.926 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.927 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.927 DEBUG:boto:Method: HEAD 2022-09-03T06:07:18.928 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.929 DEBUG:boto:Data: 2022-09-03T06:07:18.929 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:18.931 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.931 DEBUG:boto:Params: {} 2022-09-03T06:07:18.932 DEBUG:boto:Token: None 2022-09-03T06:07:18.933 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:61gDchr01RGscol025J49rvJw40= 2022-09-03T06:07:18.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:61gDchr01RGscol025J49rvJw40=', 'Content-Length': '0'} 2022-09-03T06:07:18.938 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f2fd1bd042698726-006312ef16-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.939 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:18.940 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.941 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.942 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.944 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.945 DEBUG:boto:Method: GET 2022-09-03T06:07:18.945 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.946 DEBUG:boto:Data: 2022-09-03T06:07:18.946 DEBUG:boto:Headers: {} 2022-09-03T06:07:18.947 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:18.947 DEBUG:boto:Port: 8000 2022-09-03T06:07:18.948 DEBUG:boto:Params: {} 2022-09-03T06:07:18.948 DEBUG:boto:Token: None 2022-09-03T06:07:18.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:18 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tYqDZNo+CTMQONlEYUALDEjgrA0= 2022-09-03T06:07:18.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tYqDZNo+CTMQONlEYUALDEjgrA0=', 'Content-Length': '0'} 2022-09-03T06:07:18.953 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007f28e908b114c247-006312ef16-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:18.994 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:18.995 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:18.996 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.997 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:18.998 DEBUG:boto:Method: GET 2022-09-03T06:07:18.999 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:19.000 DEBUG:boto:Data: 2022-09-03T06:07:19.001 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.002 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.003 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.004 DEBUG:boto:Params: {} 2022-09-03T06:07:19.004 DEBUG:boto:Token: None 2022-09-03T06:07:19.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:19.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eCfzRb8lZz/VNjO3jAxiHlnT2tk= 2022-09-03T06:07:19.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eCfzRb8lZz/VNjO3jAxiHlnT2tk=', 'Content-Length': '0'} 2022-09-03T06:07:19.009 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000365d46d67236f890-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.010 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.010 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.011 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.011 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.012 DEBUG:boto:Method: GET 2022-09-03T06:07:19.013 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.013 DEBUG:boto:Data: 2022-09-03T06:07:19.014 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.014 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.015 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.015 DEBUG:boto:Params: {} 2022-09-03T06:07:19.016 DEBUG:boto:Token: None 2022-09-03T06:07:19.016 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.021 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000006f24a147df23033-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.022 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.022 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.023 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.024 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.024 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.025 DEBUG:boto:Method: GET 2022-09-03T06:07:19.026 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.027 DEBUG:boto:Data: 2022-09-03T06:07:19.028 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.030 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.031 DEBUG:boto:Params: {} 2022-09-03T06:07:19.032 DEBUG:boto:Token: None 2022-09-03T06:07:19.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.041 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f919c9c4b3b7f398-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.043 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.044 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.044 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.045 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.045 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:19.046 DEBUG:boto:Data: 2022-09-03T06:07:19.046 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.047 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.047 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.048 DEBUG:boto:Params: {} 2022-09-03T06:07:19.049 DEBUG:boto:Token: None 2022-09-03T06:07:19.049 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj 2022-09-03T06:07:19.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ= 2022-09-03T06:07:19.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.054 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002c9f6a2ef96b6fa7-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.055 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.056 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.056 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.057 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:19.058 DEBUG:boto:Data: 2022-09-03T06:07:19.058 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.059 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.060 DEBUG:boto:Params: {} 2022-09-03T06:07:19.060 DEBUG:boto:Token: None 2022-09-03T06:07:19.061 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj 2022-09-03T06:07:19.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ= 2022-09-03T06:07:19.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.066 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c65e39df0edd336c-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.066 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.067 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.067 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.068 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.069 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.069 DEBUG:boto:Method: GET 2022-09-03T06:07:19.070 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.071 DEBUG:boto:Data: 2022-09-03T06:07:19.071 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.072 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.073 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.073 DEBUG:boto:Params: {} 2022-09-03T06:07:19.074 DEBUG:boto:Token: None 2022-09-03T06:07:19.074 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ= 2022-09-03T06:07:19.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.078 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000095103794e323b483-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.081 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.081 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.082 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.082 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.083 DEBUG:boto:Method: GET 2022-09-03T06:07:19.084 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.084 DEBUG:boto:Data: 2022-09-03T06:07:19.085 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.086 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.086 DEBUG:boto:Params: {} 2022-09-03T06:07:19.087 DEBUG:boto:Token: None 2022-09-03T06:07:19.088 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ= 2022-09-03T06:07:19.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.091 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000020cbcbda443a1ff3-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.138 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.139 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.139 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.140 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.140 DEBUG:boto:Method: GET 2022-09-03T06:07:19.141 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.142 DEBUG:boto:Data: 2022-09-03T06:07:19.143 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.144 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.145 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.146 DEBUG:boto:Params: {} 2022-09-03T06:07:19.147 DEBUG:boto:Token: None 2022-09-03T06:07:19.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.152 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002602c41167220a32-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.152 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.153 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.154 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.154 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.155 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.155 DEBUG:boto:Method: GET 2022-09-03T06:07:19.156 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.157 DEBUG:boto:Data: 2022-09-03T06:07:19.157 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.158 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.159 DEBUG:boto:Params: {} 2022-09-03T06:07:19.160 DEBUG:boto:Token: None 2022-09-03T06:07:19.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.165 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009891896593630fe0-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.166 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.166 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.167 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.167 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.168 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.169 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.169 DEBUG:boto:Method: GET 2022-09-03T06:07:19.170 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.171 DEBUG:boto:Data: 2022-09-03T06:07:19.171 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.172 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.172 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.173 DEBUG:boto:Params: {} 2022-09-03T06:07:19.174 DEBUG:boto:Token: None 2022-09-03T06:07:19.174 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uxs97v4w3h+Ikd1ray9uoGHtv9Y= 2022-09-03T06:07:19.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uxs97v4w3h+Ikd1ray9uoGHtv9Y=', 'Content-Length': '0'} 2022-09-03T06:07:19.178 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001742e4df7474e09d-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.222 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.223 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.224 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.224 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.225 DEBUG:boto:Method: GET 2022-09-03T06:07:19.225 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.226 DEBUG:boto:Data: 2022-09-03T06:07:19.227 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.227 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.228 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.229 DEBUG:boto:Params: {} 2022-09-03T06:07:19.230 DEBUG:boto:Token: None 2022-09-03T06:07:19.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uxs97v4w3h+Ikd1ray9uoGHtv9Y= 2022-09-03T06:07:19.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uxs97v4w3h+Ikd1ray9uoGHtv9Y=', 'Content-Length': '0'} 2022-09-03T06:07:19.234 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000488a1beba4382dd-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.235 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.236 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.236 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.237 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.237 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.238 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.238 DEBUG:boto:Data: 2022-09-03T06:07:19.239 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.239 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.240 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.240 DEBUG:boto:Params: {} 2022-09-03T06:07:19.241 DEBUG:boto:Token: None 2022-09-03T06:07:19.242 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:73ILLWknv20S5Wj6fBKKXhSRse4= 2022-09-03T06:07:19.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:73ILLWknv20S5Wj6fBKKXhSRse4=', 'Content-Length': '0'} 2022-09-03T06:07:19.246 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000023f1538f6184fc7f-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.246 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.247 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.248 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.248 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.249 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.249 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.250 DEBUG:boto:Data: 2022-09-03T06:07:19.250 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.251 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.252 DEBUG:boto:Params: {} 2022-09-03T06:07:19.252 DEBUG:boto:Token: None 2022-09-03T06:07:19.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:73ILLWknv20S5Wj6fBKKXhSRse4= 2022-09-03T06:07:19.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:73ILLWknv20S5Wj6fBKKXhSRse4=', 'Content-Length': '0'} 2022-09-03T06:07:19.257 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000041e0d09315b8cda9-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.258 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.258 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.259 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.259 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.260 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.260 DEBUG:boto:Method: GET 2022-09-03T06:07:19.261 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.262 DEBUG:boto:Data: 2022-09-03T06:07:19.262 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.263 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.263 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.264 DEBUG:boto:Params: {} 2022-09-03T06:07:19.264 DEBUG:boto:Token: None 2022-09-03T06:07:19.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uxs97v4w3h+Ikd1ray9uoGHtv9Y= 2022-09-03T06:07:19.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uxs97v4w3h+Ikd1ray9uoGHtv9Y=', 'Content-Length': '0'} 2022-09-03T06:07:19.268 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005a5261e4c3e9de70-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.269 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.270 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.270 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.271 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.271 DEBUG:boto:Method: GET 2022-09-03T06:07:19.272 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.272 DEBUG:boto:Data: 2022-09-03T06:07:19.273 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.274 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.274 DEBUG:boto:Params: {} 2022-09-03T06:07:19.275 DEBUG:boto:Token: None 2022-09-03T06:07:19.276 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:19.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uxs97v4w3h+Ikd1ray9uoGHtv9Y= 2022-09-03T06:07:19.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uxs97v4w3h+Ikd1ray9uoGHtv9Y=', 'Content-Length': '0'} 2022-09-03T06:07:19.279 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000044b20c7f77de286a-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.280 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.280 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.281 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.281 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.282 DEBUG:boto:Method: GET 2022-09-03T06:07:19.283 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.283 DEBUG:boto:Data: 2022-09-03T06:07:19.284 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.284 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.285 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.285 DEBUG:boto:Params: {} 2022-09-03T06:07:19.286 DEBUG:boto:Token: None 2022-09-03T06:07:19.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.290 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c3d2fd9d12c7ffbc-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.291 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.292 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.293 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.293 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.294 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.294 DEBUG:boto:Method: GET 2022-09-03T06:07:19.295 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.296 DEBUG:boto:Data: 2022-09-03T06:07:19.296 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.297 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.297 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.298 DEBUG:boto:Params: {} 2022-09-03T06:07:19.299 DEBUG:boto:Token: None 2022-09-03T06:07:19.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.303 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063b8a7d9d1467fe6-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.304 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.305 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.306 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.306 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.307 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:19.307 DEBUG:boto:Data: 2022-09-03T06:07:19.308 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.308 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.309 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.309 DEBUG:boto:Params: {} 2022-09-03T06:07:19.310 DEBUG:boto:Token: None 2022-09-03T06:07:19.311 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj 2022-09-03T06:07:19.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ= 2022-09-03T06:07:19.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.325 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000df877c54f1cce557-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.327 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.328 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.329 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.329 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:19.330 DEBUG:boto:Data: 2022-09-03T06:07:19.331 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.333 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.334 DEBUG:boto:Params: {} 2022-09-03T06:07:19.335 DEBUG:boto:Token: None 2022-09-03T06:07:19.335 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj 2022-09-03T06:07:19.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ= 2022-09-03T06:07:19.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.341 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007a1250382859df4a-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.342 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.343 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.344 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.345 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.348 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.349 DEBUG:boto:Method: GET 2022-09-03T06:07:19.350 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.351 DEBUG:boto:Data: 2022-09-03T06:07:19.352 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.353 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.354 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.355 DEBUG:boto:Params: {} 2022-09-03T06:07:19.356 DEBUG:boto:Token: None 2022-09-03T06:07:19.357 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ= 2022-09-03T06:07:19.359 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.363 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000095cf4f47e3812805-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.411 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.412 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.412 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.413 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.413 DEBUG:boto:Method: GET 2022-09-03T06:07:19.414 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.414 DEBUG:boto:Data: 2022-09-03T06:07:19.415 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.417 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.417 DEBUG:boto:Params: {} 2022-09-03T06:07:19.419 DEBUG:boto:Token: None 2022-09-03T06:07:19.420 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ= 2022-09-03T06:07:19.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.427 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000096b0a56bf3b87b54-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.430 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.431 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.432 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.433 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.434 DEBUG:boto:Method: GET 2022-09-03T06:07:19.435 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.436 DEBUG:boto:Data: 2022-09-03T06:07:19.437 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.438 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.440 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.441 DEBUG:boto:Params: {} 2022-09-03T06:07:19.442 DEBUG:boto:Token: None 2022-09-03T06:07:19.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.448 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008f2eb9594c5cbec8-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.449 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.451 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.452 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.453 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.454 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.456 DEBUG:boto:Method: GET 2022-09-03T06:07:19.456 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.457 DEBUG:boto:Data: 2022-09-03T06:07:19.458 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.459 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.460 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.461 DEBUG:boto:Params: {} 2022-09-03T06:07:19.462 DEBUG:boto:Token: None 2022-09-03T06:07:19.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a4288674309d802-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.470 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.471 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.472 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.473 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.474 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.475 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.476 DEBUG:boto:Method: GET 2022-09-03T06:07:19.477 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.477 DEBUG:boto:Data: 2022-09-03T06:07:19.478 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.479 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.480 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.481 DEBUG:boto:Params: {} 2022-09-03T06:07:19.482 DEBUG:boto:Token: None 2022-09-03T06:07:19.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FG3qDmEETuklz1LRU+zlZt5uwr0= 2022-09-03T06:07:19.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FG3qDmEETuklz1LRU+zlZt5uwr0=', 'Content-Length': '0'} 2022-09-03T06:07:19.487 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007f20536bca0a2bdc-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.489 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.490 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.491 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.491 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.492 DEBUG:boto:Method: GET 2022-09-03T06:07:19.493 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.494 DEBUG:boto:Data: 2022-09-03T06:07:19.495 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.497 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.497 DEBUG:boto:Params: {} 2022-09-03T06:07:19.499 DEBUG:boto:Token: None 2022-09-03T06:07:19.499 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FG3qDmEETuklz1LRU+zlZt5uwr0= 2022-09-03T06:07:19.501 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FG3qDmEETuklz1LRU+zlZt5uwr0=', 'Content-Length': '0'} 2022-09-03T06:07:19.504 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bf179f4c535b02d0-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.506 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.507 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.508 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.509 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.510 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.511 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.512 DEBUG:boto:Data: 2022-09-03T06:07:19.513 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.514 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.515 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.516 DEBUG:boto:Params: {} 2022-09-03T06:07:19.518 DEBUG:boto:Token: None 2022-09-03T06:07:19.519 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3ds9pP8eJ10goTHP6YFopHR9l0= 2022-09-03T06:07:19.521 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3ds9pP8eJ10goTHP6YFopHR9l0=', 'Content-Length': '0'} 2022-09-03T06:07:19.523 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001906f6644c1901df-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.524 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.525 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.525 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.526 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.526 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.527 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.527 DEBUG:boto:Data: 2022-09-03T06:07:19.528 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.528 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.529 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.529 DEBUG:boto:Params: {} 2022-09-03T06:07:19.530 DEBUG:boto:Token: None 2022-09-03T06:07:19.531 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3ds9pP8eJ10goTHP6YFopHR9l0= 2022-09-03T06:07:19.532 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3ds9pP8eJ10goTHP6YFopHR9l0=', 'Content-Length': '0'} 2022-09-03T06:07:19.535 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000283b3253ebde34d8-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.536 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.536 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.537 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.537 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.538 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.538 DEBUG:boto:Method: GET 2022-09-03T06:07:19.539 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.539 DEBUG:boto:Data: 2022-09-03T06:07:19.540 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.540 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.541 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.541 DEBUG:boto:Params: {} 2022-09-03T06:07:19.542 DEBUG:boto:Token: None 2022-09-03T06:07:19.543 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FG3qDmEETuklz1LRU+zlZt5uwr0= 2022-09-03T06:07:19.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FG3qDmEETuklz1LRU+zlZt5uwr0=', 'Content-Length': '0'} 2022-09-03T06:07:19.546 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006196e3846c0bfc7f-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.547 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.548 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.548 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.549 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.549 DEBUG:boto:Method: GET 2022-09-03T06:07:19.550 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.551 DEBUG:boto:Data: 2022-09-03T06:07:19.551 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.552 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.552 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.553 DEBUG:boto:Params: {} 2022-09-03T06:07:19.554 DEBUG:boto:Token: None 2022-09-03T06:07:19.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:19.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FG3qDmEETuklz1LRU+zlZt5uwr0= 2022-09-03T06:07:19.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FG3qDmEETuklz1LRU+zlZt5uwr0=', 'Content-Length': '0'} 2022-09-03T06:07:19.558 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000076b372c9a39e0cbe-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.559 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.559 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.560 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.560 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.561 DEBUG:boto:Method: GET 2022-09-03T06:07:19.561 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.562 DEBUG:boto:Data: 2022-09-03T06:07:19.562 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.563 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.563 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.564 DEBUG:boto:Params: {} 2022-09-03T06:07:19.565 DEBUG:boto:Token: None 2022-09-03T06:07:19.565 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.566 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.566 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.570 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000107192805ecc6d29-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.571 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.572 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.573 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.574 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.575 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.576 DEBUG:boto:Method: GET 2022-09-03T06:07:19.577 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.578 DEBUG:boto:Data: 2022-09-03T06:07:19.579 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.580 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.581 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.582 DEBUG:boto:Params: {} 2022-09-03T06:07:19.583 DEBUG:boto:Token: None 2022-09-03T06:07:19.584 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.590 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cca45a4fc1bc2be0-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.591 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.595 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.596 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.596 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.597 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:19.597 DEBUG:boto:Data: 2022-09-03T06:07:19.598 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.598 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.599 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.599 DEBUG:boto:Params: {} 2022-09-03T06:07:19.600 DEBUG:boto:Token: None 2022-09-03T06:07:19.600 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj 2022-09-03T06:07:19.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ= 2022-09-03T06:07:19.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.605 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002c4b108fb72ce8b5-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.606 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.607 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.607 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.608 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:19.608 DEBUG:boto:Data: 2022-09-03T06:07:19.609 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.609 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.610 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.610 DEBUG:boto:Params: {} 2022-09-03T06:07:19.611 DEBUG:boto:Token: None 2022-09-03T06:07:19.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj 2022-09-03T06:07:19.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ= 2022-09-03T06:07:19.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.616 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b334d5e0d6babb0c-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.617 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.618 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.618 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.619 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.619 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.620 DEBUG:boto:Method: GET 2022-09-03T06:07:19.620 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.621 DEBUG:boto:Data: 2022-09-03T06:07:19.621 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.622 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.622 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.623 DEBUG:boto:Params: {} 2022-09-03T06:07:19.623 DEBUG:boto:Token: None 2022-09-03T06:07:19.624 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ= 2022-09-03T06:07:19.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.629 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004a4d8287243422d5-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.630 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.630 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.632 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.633 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.633 DEBUG:boto:Method: GET 2022-09-03T06:07:19.634 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.634 DEBUG:boto:Data: 2022-09-03T06:07:19.635 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.637 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.638 DEBUG:boto:Params: {} 2022-09-03T06:07:19.639 DEBUG:boto:Token: None 2022-09-03T06:07:19.640 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ= 2022-09-03T06:07:19.643 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.645 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be23e39c53dbb0f6-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.686 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.687 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.687 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.688 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.689 DEBUG:boto:Method: GET 2022-09-03T06:07:19.690 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.691 DEBUG:boto:Data: 2022-09-03T06:07:19.692 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.693 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.695 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.696 DEBUG:boto:Params: {} 2022-09-03T06:07:19.697 DEBUG:boto:Token: None 2022-09-03T06:07:19.698 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.699 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.700 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.703 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005bf94a2827038e25-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.704 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.705 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.706 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.707 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.708 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.708 DEBUG:boto:Method: GET 2022-09-03T06:07:19.709 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.709 DEBUG:boto:Data: 2022-09-03T06:07:19.710 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.711 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.711 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.712 DEBUG:boto:Params: {} 2022-09-03T06:07:19.712 DEBUG:boto:Token: None 2022-09-03T06:07:19.713 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.714 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.714 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.717 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a92b9f6ab9324eff-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.718 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.719 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.720 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.720 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.721 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.722 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.723 DEBUG:boto:Method: GET 2022-09-03T06:07:19.724 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.725 DEBUG:boto:Data: 2022-09-03T06:07:19.725 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.726 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.727 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.728 DEBUG:boto:Params: {} 2022-09-03T06:07:19.730 DEBUG:boto:Token: None 2022-09-03T06:07:19.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p44JTJqOAxjxSYV5p9dC1bf57/k= 2022-09-03T06:07:19.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p44JTJqOAxjxSYV5p9dC1bf57/k=', 'Content-Length': '0'} 2022-09-03T06:07:19.734 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a102b7fceef216d4-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.782 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.783 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.784 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.785 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.786 DEBUG:boto:Method: GET 2022-09-03T06:07:19.787 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.788 DEBUG:boto:Data: 2022-09-03T06:07:19.789 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.790 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.790 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.791 DEBUG:boto:Params: {} 2022-09-03T06:07:19.792 DEBUG:boto:Token: None 2022-09-03T06:07:19.793 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p44JTJqOAxjxSYV5p9dC1bf57/k= 2022-09-03T06:07:19.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p44JTJqOAxjxSYV5p9dC1bf57/k=', 'Content-Length': '0'} 2022-09-03T06:07:19.799 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000089cf42f7e5309c96-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.799 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.800 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.801 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.801 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.802 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.803 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.803 DEBUG:boto:Data: 2022-09-03T06:07:19.804 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.804 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.805 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.805 DEBUG:boto:Params: {} 2022-09-03T06:07:19.807 DEBUG:boto:Token: None 2022-09-03T06:07:19.807 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XJC7UfyaGpQMjXRe+XFAV0BpRCg= 2022-09-03T06:07:19.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XJC7UfyaGpQMjXRe+XFAV0BpRCg=', 'Content-Length': '0'} 2022-09-03T06:07:19.811 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000385ef93164f919ed-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.813 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.813 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.814 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.815 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.816 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.817 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.817 DEBUG:boto:Data: 2022-09-03T06:07:19.818 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.819 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.820 DEBUG:boto:Params: {} 2022-09-03T06:07:19.820 DEBUG:boto:Token: None 2022-09-03T06:07:19.821 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XJC7UfyaGpQMjXRe+XFAV0BpRCg= 2022-09-03T06:07:19.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XJC7UfyaGpQMjXRe+XFAV0BpRCg=', 'Content-Length': '0'} 2022-09-03T06:07:19.825 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000eb582b27259531cf-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.826 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.826 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.827 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.828 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.829 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.829 DEBUG:boto:Method: GET 2022-09-03T06:07:19.830 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.831 DEBUG:boto:Data: 2022-09-03T06:07:19.831 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.833 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.834 DEBUG:boto:Params: {} 2022-09-03T06:07:19.835 DEBUG:boto:Token: None 2022-09-03T06:07:19.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p44JTJqOAxjxSYV5p9dC1bf57/k= 2022-09-03T06:07:19.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p44JTJqOAxjxSYV5p9dC1bf57/k=', 'Content-Length': '0'} 2022-09-03T06:07:19.839 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c6201441afa6bb25-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.840 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.842 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.843 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.843 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.844 DEBUG:boto:Method: GET 2022-09-03T06:07:19.844 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.845 DEBUG:boto:Data: 2022-09-03T06:07:19.845 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.846 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.847 DEBUG:boto:Params: {} 2022-09-03T06:07:19.848 DEBUG:boto:Token: None 2022-09-03T06:07:19.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:19.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p44JTJqOAxjxSYV5p9dC1bf57/k= 2022-09-03T06:07:19.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p44JTJqOAxjxSYV5p9dC1bf57/k=', 'Content-Length': '0'} 2022-09-03T06:07:19.853 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d19dd7c3dd0d1228-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.854 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.855 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.856 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.857 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.858 DEBUG:boto:Method: GET 2022-09-03T06:07:19.859 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.860 DEBUG:boto:Data: 2022-09-03T06:07:19.861 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.862 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.863 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.864 DEBUG:boto:Params: {} 2022-09-03T06:07:19.865 DEBUG:boto:Token: None 2022-09-03T06:07:19.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.872 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000067defd8522324be-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.873 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.875 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.876 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.877 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.878 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.879 DEBUG:boto:Method: GET 2022-09-03T06:07:19.880 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.881 DEBUG:boto:Data: 2022-09-03T06:07:19.882 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.885 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.886 DEBUG:boto:Params: {} 2022-09-03T06:07:19.887 DEBUG:boto:Token: None 2022-09-03T06:07:19.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/ 2022-09-03T06:07:19.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY= 2022-09-03T06:07:19.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwALhEyGqc1RkB5gEEJumQqUaiY=', 'Content-Length': '0'} 2022-09-03T06:07:19.893 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000623e9579ec960ea2-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.894 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:19.895 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.896 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.897 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.898 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:19.898 DEBUG:boto:Data: 2022-09-03T06:07:19.899 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.900 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.900 DEBUG:boto:Params: {} 2022-09-03T06:07:19.901 DEBUG:boto:Token: None 2022-09-03T06:07:19.902 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj 2022-09-03T06:07:19.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ= 2022-09-03T06:07:19.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.907 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d3dd5ee9859255cd-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.908 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.908 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.909 DEBUG:boto:Method: HEAD 2022-09-03T06:07:19.909 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:19.910 DEBUG:boto:Data: 2022-09-03T06:07:19.910 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.911 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.911 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.912 DEBUG:boto:Params: {} 2022-09-03T06:07:19.913 DEBUG:boto:Token: None 2022-09-03T06:07:19.914 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj 2022-09-03T06:07:19.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ= 2022-09-03T06:07:19.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ViF/xMeQWJQMa4ntjgXKNGan8QQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.918 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000294b695206b2155b-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.920 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:19.921 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.922 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.923 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.924 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.925 DEBUG:boto:Method: GET 2022-09-03T06:07:19.926 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.927 DEBUG:boto:Data: 2022-09-03T06:07:19.928 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.929 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:19.930 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.931 DEBUG:boto:Params: {} 2022-09-03T06:07:19.932 DEBUG:boto:Token: None 2022-09-03T06:07:19.934 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ= 2022-09-03T06:07:19.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.938 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dbbcd21aa41bfeb0-006312ef17-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.982 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:19.983 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:19.984 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.984 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.985 DEBUG:boto:Method: GET 2022-09-03T06:07:19.985 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.986 DEBUG:boto:Data: 2022-09-03T06:07:19.986 DEBUG:boto:Headers: {} 2022-09-03T06:07:19.987 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:19.987 DEBUG:boto:Port: 8000 2022-09-03T06:07:19.988 DEBUG:boto:Params: {} 2022-09-03T06:07:19.989 DEBUG:boto:Token: None 2022-09-03T06:07:19.989 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:19 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:19.990 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ= 2022-09-03T06:07:19.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FXFkjS0AvsMT6JzxaZrt0KjWUWQ=', 'Content-Length': '0'} 2022-09-03T06:07:19.995 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002df0acc7effcfc3b-006312ef17-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:19.996 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:19.996 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:19.997 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.997 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:19.998 DEBUG:boto:Method: GET 2022-09-03T06:07:19.999 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:19.999 DEBUG:boto:Data: 2022-09-03T06:07:20.000 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.000 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.001 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.002 DEBUG:boto:Params: {} 2022-09-03T06:07:20.002 DEBUG:boto:Token: None 2022-09-03T06:07:20.003 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.008 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005332c3de06e8941b-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.010 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.011 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.012 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.013 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.014 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.015 DEBUG:boto:Method: GET 2022-09-03T06:07:20.016 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:20.016 DEBUG:boto:Data: 2022-09-03T06:07:20.017 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.018 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.018 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.019 DEBUG:boto:Params: {} 2022-09-03T06:07:20.019 DEBUG:boto:Token: None 2022-09-03T06:07:20.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.021 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.024 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f7a1d4b864a5b6f9-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.066 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.067 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:20.068 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.069 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.069 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.070 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.070 DEBUG:boto:Method: GET 2022-09-03T06:07:20.071 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.071 DEBUG:boto:Data: 2022-09-03T06:07:20.072 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.073 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.073 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.074 DEBUG:boto:Params: {} 2022-09-03T06:07:20.074 DEBUG:boto:Token: None 2022-09-03T06:07:20.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WBMHnh9Ny88Kbw7SyqOxD/Jnbh0= 2022-09-03T06:07:20.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WBMHnh9Ny88Kbw7SyqOxD/Jnbh0=', 'Content-Length': '0'} 2022-09-03T06:07:20.079 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a6943e9982e55ced-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.080 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.080 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.081 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.081 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.082 DEBUG:boto:Method: GET 2022-09-03T06:07:20.083 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.083 DEBUG:boto:Data: 2022-09-03T06:07:20.084 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.085 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.086 DEBUG:boto:Params: {} 2022-09-03T06:07:20.086 DEBUG:boto:Token: None 2022-09-03T06:07:20.087 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WBMHnh9Ny88Kbw7SyqOxD/Jnbh0= 2022-09-03T06:07:20.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WBMHnh9Ny88Kbw7SyqOxD/Jnbh0=', 'Content-Length': '0'} 2022-09-03T06:07:20.092 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f084e12d85a7287f-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.093 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.094 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.095 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.095 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.096 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.096 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.097 DEBUG:boto:Data: 2022-09-03T06:07:20.097 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.098 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.098 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.099 DEBUG:boto:Params: {} 2022-09-03T06:07:20.100 DEBUG:boto:Token: None 2022-09-03T06:07:20.100 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zh/TTtnmK/FMpY68pjmMcxM4PyM= 2022-09-03T06:07:20.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zh/TTtnmK/FMpY68pjmMcxM4PyM=', 'Content-Length': '0'} 2022-09-03T06:07:20.104 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000722877cbb50180a9-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.105 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.105 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.106 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.106 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.107 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.107 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.108 DEBUG:boto:Data: 2022-09-03T06:07:20.108 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.110 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.110 DEBUG:boto:Params: {} 2022-09-03T06:07:20.111 DEBUG:boto:Token: None 2022-09-03T06:07:20.111 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zh/TTtnmK/FMpY68pjmMcxM4PyM= 2022-09-03T06:07:20.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zh/TTtnmK/FMpY68pjmMcxM4PyM=', 'Content-Length': '0'} 2022-09-03T06:07:20.115 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000368baced3dde4066-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.115 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:20.116 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.116 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.117 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.118 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.118 DEBUG:boto:Method: GET 2022-09-03T06:07:20.119 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.120 DEBUG:boto:Data: 2022-09-03T06:07:20.120 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.121 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.121 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.122 DEBUG:boto:Params: {} 2022-09-03T06:07:20.122 DEBUG:boto:Token: None 2022-09-03T06:07:20.123 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.123 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WBMHnh9Ny88Kbw7SyqOxD/Jnbh0= 2022-09-03T06:07:20.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WBMHnh9Ny88Kbw7SyqOxD/Jnbh0=', 'Content-Length': '0'} 2022-09-03T06:07:20.127 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000072a5f2290ee4b8b1-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.128 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.128 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.129 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.129 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.130 DEBUG:boto:Method: GET 2022-09-03T06:07:20.130 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.131 DEBUG:boto:Data: 2022-09-03T06:07:20.131 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.132 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.133 DEBUG:boto:Params: {} 2022-09-03T06:07:20.133 DEBUG:boto:Token: None 2022-09-03T06:07:20.134 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:20.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WBMHnh9Ny88Kbw7SyqOxD/Jnbh0= 2022-09-03T06:07:20.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WBMHnh9Ny88Kbw7SyqOxD/Jnbh0=', 'Content-Length': '0'} 2022-09-03T06:07:20.138 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f19417581cc4915e-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.139 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.139 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.140 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.140 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.141 DEBUG:boto:Method: GET 2022-09-03T06:07:20.141 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:20.142 DEBUG:boto:Data: 2022-09-03T06:07:20.142 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.143 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.144 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.144 DEBUG:boto:Params: {} 2022-09-03T06:07:20.145 DEBUG:boto:Token: None 2022-09-03T06:07:20.145 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.149 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb6a43b2ad5f4a3c-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.194 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.195 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.196 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.197 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.198 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.199 DEBUG:boto:Method: GET 2022-09-03T06:07:20.200 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:20.200 DEBUG:boto:Data: 2022-09-03T06:07:20.201 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.202 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.202 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.203 DEBUG:boto:Params: {} 2022-09-03T06:07:20.204 DEBUG:boto:Token: None 2022-09-03T06:07:20.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.210 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f3c260b0edae5255-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.211 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.212 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.213 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.214 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.215 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:20.216 DEBUG:boto:Data: 2022-09-03T06:07:20.217 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.218 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.219 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.220 DEBUG:boto:Params: {} 2022-09-03T06:07:20.221 DEBUG:boto:Token: None 2022-09-03T06:07:20.222 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj 2022-09-03T06:07:20.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5fkafzFT4Zqsd6wvpthNoPgta/8= 2022-09-03T06:07:20.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5fkafzFT4Zqsd6wvpthNoPgta/8=', 'Content-Length': '0'} 2022-09-03T06:07:20.229 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006da9b4f60aa9e9d6-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.231 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.232 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.233 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.234 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:20.235 DEBUG:boto:Data: 2022-09-03T06:07:20.236 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.237 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.238 DEBUG:boto:Params: {} 2022-09-03T06:07:20.238 DEBUG:boto:Token: None 2022-09-03T06:07:20.239 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj 2022-09-03T06:07:20.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5fkafzFT4Zqsd6wvpthNoPgta/8= 2022-09-03T06:07:20.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5fkafzFT4Zqsd6wvpthNoPgta/8=', 'Content-Length': '0'} 2022-09-03T06:07:20.244 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ed9fb28468872dad-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.245 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:20.245 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.246 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.246 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.247 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.247 DEBUG:boto:Method: GET 2022-09-03T06:07:20.248 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.248 DEBUG:boto:Data: 2022-09-03T06:07:20.249 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.250 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.250 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.251 DEBUG:boto:Params: {} 2022-09-03T06:07:20.251 DEBUG:boto:Token: None 2022-09-03T06:07:20.252 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ= 2022-09-03T06:07:20.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ=', 'Content-Length': '0'} 2022-09-03T06:07:20.255 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000df99c299510a42b7-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.257 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.258 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.259 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.259 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.260 DEBUG:boto:Method: GET 2022-09-03T06:07:20.261 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.262 DEBUG:boto:Data: 2022-09-03T06:07:20.263 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.265 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.266 DEBUG:boto:Params: {} 2022-09-03T06:07:20.267 DEBUG:boto:Token: None 2022-09-03T06:07:20.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ= 2022-09-03T06:07:20.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ=', 'Content-Length': '0'} 2022-09-03T06:07:20.273 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c493b1ae66a0c06b-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.315 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.316 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.317 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.318 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.319 DEBUG:boto:Method: GET 2022-09-03T06:07:20.320 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:20.321 DEBUG:boto:Data: 2022-09-03T06:07:20.321 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.322 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.323 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.324 DEBUG:boto:Params: {} 2022-09-03T06:07:20.325 DEBUG:boto:Token: None 2022-09-03T06:07:20.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.331 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000074e6546ea7cab397-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.332 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.334 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.334 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.335 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.335 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.336 DEBUG:boto:Method: GET 2022-09-03T06:07:20.337 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:20.338 DEBUG:boto:Data: 2022-09-03T06:07:20.338 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.339 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.340 DEBUG:boto:Params: {} 2022-09-03T06:07:20.340 DEBUG:boto:Token: None 2022-09-03T06:07:20.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.346 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1564471e7f9e8a3-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.346 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.347 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:20.348 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.348 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.349 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.349 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.350 DEBUG:boto:Method: GET 2022-09-03T06:07:20.350 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.351 DEBUG:boto:Data: 2022-09-03T06:07:20.351 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.352 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.353 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.353 DEBUG:boto:Params: {} 2022-09-03T06:07:20.354 DEBUG:boto:Token: None 2022-09-03T06:07:20.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lNmAWbG1USFaoHc/6kSki2A+etE= 2022-09-03T06:07:20.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lNmAWbG1USFaoHc/6kSki2A+etE=', 'Content-Length': '0'} 2022-09-03T06:07:20.359 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b14d353842900ee3-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.361 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.362 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.362 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.363 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.363 DEBUG:boto:Method: GET 2022-09-03T06:07:20.364 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.365 DEBUG:boto:Data: 2022-09-03T06:07:20.365 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.366 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.366 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.367 DEBUG:boto:Params: {} 2022-09-03T06:07:20.367 DEBUG:boto:Token: None 2022-09-03T06:07:20.368 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lNmAWbG1USFaoHc/6kSki2A+etE= 2022-09-03T06:07:20.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lNmAWbG1USFaoHc/6kSki2A+etE=', 'Content-Length': '0'} 2022-09-03T06:07:20.372 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c06552f3f6bfeab8-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.373 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.373 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.374 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.374 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.375 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.375 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.376 DEBUG:boto:Data: 2022-09-03T06:07:20.376 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.377 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.377 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.378 DEBUG:boto:Params: {} 2022-09-03T06:07:20.379 DEBUG:boto:Token: None 2022-09-03T06:07:20.379 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HN58WFwDRuSrh0N9J4t+/5jK0KM= 2022-09-03T06:07:20.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HN58WFwDRuSrh0N9J4t+/5jK0KM=', 'Content-Length': '0'} 2022-09-03T06:07:20.383 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c6629f83d025c27d-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.384 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.384 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.385 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.385 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.386 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.386 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.387 DEBUG:boto:Data: 2022-09-03T06:07:20.387 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.389 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.389 DEBUG:boto:Params: {} 2022-09-03T06:07:20.390 DEBUG:boto:Token: None 2022-09-03T06:07:20.390 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HN58WFwDRuSrh0N9J4t+/5jK0KM= 2022-09-03T06:07:20.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HN58WFwDRuSrh0N9J4t+/5jK0KM=', 'Content-Length': '0'} 2022-09-03T06:07:20.394 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002779593f01bfb4ff-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.394 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:20.395 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.395 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.396 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.397 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.397 DEBUG:boto:Method: GET 2022-09-03T06:07:20.398 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.398 DEBUG:boto:Data: 2022-09-03T06:07:20.399 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.399 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.400 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.400 DEBUG:boto:Params: {} 2022-09-03T06:07:20.401 DEBUG:boto:Token: None 2022-09-03T06:07:20.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lNmAWbG1USFaoHc/6kSki2A+etE= 2022-09-03T06:07:20.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lNmAWbG1USFaoHc/6kSki2A+etE=', 'Content-Length': '0'} 2022-09-03T06:07:20.405 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000574bbaa4fc67c2d5-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.446 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.447 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.447 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.448 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.448 DEBUG:boto:Method: GET 2022-09-03T06:07:20.449 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.450 DEBUG:boto:Data: 2022-09-03T06:07:20.450 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.451 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.452 DEBUG:boto:Params: {} 2022-09-03T06:07:20.453 DEBUG:boto:Token: None 2022-09-03T06:07:20.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:20.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lNmAWbG1USFaoHc/6kSki2A+etE= 2022-09-03T06:07:20.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lNmAWbG1USFaoHc/6kSki2A+etE=', 'Content-Length': '0'} 2022-09-03T06:07:20.457 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e9ebc89145cdeefd-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.458 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.458 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.459 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.459 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.460 DEBUG:boto:Method: GET 2022-09-03T06:07:20.461 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:20.461 DEBUG:boto:Data: 2022-09-03T06:07:20.462 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.463 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.463 DEBUG:boto:Params: {} 2022-09-03T06:07:20.464 DEBUG:boto:Token: None 2022-09-03T06:07:20.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000aaaa515e7ce46f3-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.469 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.470 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.470 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.471 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.471 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.472 DEBUG:boto:Method: GET 2022-09-03T06:07:20.472 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:20.473 DEBUG:boto:Data: 2022-09-03T06:07:20.473 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.474 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.474 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.475 DEBUG:boto:Params: {} 2022-09-03T06:07:20.476 DEBUG:boto:Token: None 2022-09-03T06:07:20.476 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.480 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d58bd9264a8ee429-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.481 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.482 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.482 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.483 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.483 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:20.484 DEBUG:boto:Data: 2022-09-03T06:07:20.484 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.485 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.485 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.486 DEBUG:boto:Params: {} 2022-09-03T06:07:20.486 DEBUG:boto:Token: None 2022-09-03T06:07:20.487 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj 2022-09-03T06:07:20.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5fkafzFT4Zqsd6wvpthNoPgta/8= 2022-09-03T06:07:20.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5fkafzFT4Zqsd6wvpthNoPgta/8=', 'Content-Length': '0'} 2022-09-03T06:07:20.492 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000092725e205b3c38a7-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.492 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.493 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.493 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.494 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:20.495 DEBUG:boto:Data: 2022-09-03T06:07:20.495 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.496 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.497 DEBUG:boto:Params: {} 2022-09-03T06:07:20.497 DEBUG:boto:Token: None 2022-09-03T06:07:20.498 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj 2022-09-03T06:07:20.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5fkafzFT4Zqsd6wvpthNoPgta/8= 2022-09-03T06:07:20.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5fkafzFT4Zqsd6wvpthNoPgta/8=', 'Content-Length': '0'} 2022-09-03T06:07:20.502 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009af3391fc08c4dbb-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.503 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:20.504 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.505 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.506 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.507 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.508 DEBUG:boto:Method: GET 2022-09-03T06:07:20.509 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.510 DEBUG:boto:Data: 2022-09-03T06:07:20.511 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.512 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.513 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.514 DEBUG:boto:Params: {} 2022-09-03T06:07:20.515 DEBUG:boto:Token: None 2022-09-03T06:07:20.516 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ= 2022-09-03T06:07:20.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ=', 'Content-Length': '0'} 2022-09-03T06:07:20.521 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005b52641b0f0c2512-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.522 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.523 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.524 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.525 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.526 DEBUG:boto:Method: GET 2022-09-03T06:07:20.527 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.528 DEBUG:boto:Data: 2022-09-03T06:07:20.529 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.530 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.531 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.532 DEBUG:boto:Params: {} 2022-09-03T06:07:20.533 DEBUG:boto:Token: None 2022-09-03T06:07:20.534 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ= 2022-09-03T06:07:20.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ=', 'Content-Length': '0'} 2022-09-03T06:07:20.539 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c1c4f27a4b05dc82-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.540 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.541 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.542 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.543 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.543 DEBUG:boto:Method: GET 2022-09-03T06:07:20.544 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:20.545 DEBUG:boto:Data: 2022-09-03T06:07:20.546 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.546 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:20.547 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.547 DEBUG:boto:Params: {} 2022-09-03T06:07:20.548 DEBUG:boto:Token: None 2022-09-03T06:07:20.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.554 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c83fc488841ce6b-006312ef18-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.555 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.556 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.556 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.557 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.558 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:20.558 DEBUG:boto:Method: GET 2022-09-03T06:07:20.559 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:20.559 DEBUG:boto:Data: 2022-09-03T06:07:20.560 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.560 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.561 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.561 DEBUG:boto:Params: {} 2022-09-03T06:07:20.562 DEBUG:boto:Token: None 2022-09-03T06:07:20.563 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.565 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.569 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c2a293f9f4bacac0-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.610 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.611 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-19 zones={test-zone2, test-zone1} 2022-09-03T06:07:20.612 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-19 zones={test-zone3, test-zone1} 2022-09-03T06:07:20.612 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.614 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.616 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.617 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:07:20.618 DEBUG:boto:Data: 2022-09-03T06:07:20.619 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.620 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.621 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.621 DEBUG:boto:Params: {} 2022-09-03T06:07:20.622 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:07:20.623 DEBUG:boto:Token: None 2022-09-03T06:07:20.623 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Fa0YFiMloFBMJ5DV3pfuW9G8kg= 2022-09-03T06:07:20.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Fa0YFiMloFBMJ5DV3pfuW9G8kg=', 'Content-Length': '0'} 2022-09-03T06:07:20.630 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '31'), ('X-RGW-Bytes-Used', '240'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000053421b0ce4f4d4c1-006312ef18-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.631 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.631 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.632 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.632 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:07:20.633 DEBUG:boto:Data: 2022-09-03T06:07:20.633 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.634 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.634 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.635 DEBUG:boto:Params: {} 2022-09-03T06:07:20.635 DEBUG:boto:Token: None 2022-09-03T06:07:20.636 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Fa0YFiMloFBMJ5DV3pfuW9G8kg= 2022-09-03T06:07:20.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Fa0YFiMloFBMJ5DV3pfuW9G8kg=', 'Content-Length': '0'} 2022-09-03T06:07:20.642 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '31'), ('X-RGW-Bytes-Used', '240'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008cecac405891283d-006312ef18-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.643 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:07:20.643 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.644 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.644 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.645 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.645 DEBUG:boto:Method: GET 2022-09-03T06:07:20.646 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.647 DEBUG:boto:Data: 2022-09-03T06:07:20.647 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.648 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.649 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.649 DEBUG:boto:Params: {} 2022-09-03T06:07:20.650 DEBUG:boto:Token: None 2022-09-03T06:07:20.651 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/?versions 2022-09-03T06:07:20.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3FOwU7rwpZolS062Cb9cF91At9E= 2022-09-03T06:07:20.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3FOwU7rwpZolS062Cb9cF91At9E=', 'Content-Length': '0'} 2022-09-03T06:07:20.657 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004df47c17242cfa34-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.658 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.663 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:07:20.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.665 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.666 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.666 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.667 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.667 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.668 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.668 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.669 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.669 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.671 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.671 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.672 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.672 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.672 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.673 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.673 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.674 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.674 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.675 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.675 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.676 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.677 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.677 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.678 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.678 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.679 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.679 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:07:20.680 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.680 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.681 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.681 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.682 DEBUG:boto:Method: GET 2022-09-03T06:07:20.682 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.683 DEBUG:boto:Data: 2022-09-03T06:07:20.683 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.684 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.684 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.685 DEBUG:boto:Params: {} 2022-09-03T06:07:20.685 DEBUG:boto:Token: None 2022-09-03T06:07:20.686 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/?versions 2022-09-03T06:07:20.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3FOwU7rwpZolS062Cb9cF91At9E= 2022-09-03T06:07:20.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3FOwU7rwpZolS062Cb9cF91At9E=', 'Content-Length': '0'} 2022-09-03T06:07:20.691 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a955a0c14405f5ab-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.692 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.694 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:07:20.695 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.695 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.696 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.696 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.696 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.697 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.697 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.698 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.698 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.699 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.699 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.700 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.700 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.701 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.701 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.702 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.702 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.703 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.703 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.704 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.704 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.705 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.705 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.706 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.706 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.707 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.707 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.708 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.708 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.709 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:20.709 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.710 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.710 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.711 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.711 DEBUG:boto:Method: GET 2022-09-03T06:07:20.712 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.712 DEBUG:boto:Data: 2022-09-03T06:07:20.713 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.714 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.715 DEBUG:boto:Params: {} 2022-09-03T06:07:20.715 DEBUG:boto:Token: None 2022-09-03T06:07:20.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/?versions 2022-09-03T06:07:20.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3FOwU7rwpZolS062Cb9cF91At9E= 2022-09-03T06:07:20.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3FOwU7rwpZolS062Cb9cF91At9E=', 'Content-Length': '0'} 2022-09-03T06:07:20.721 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000330a50690821b100-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.722 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.724 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.724 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.725 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.725 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.726 DEBUG:boto:Method: GET 2022-09-03T06:07:20.726 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:20.727 DEBUG:boto:Data: 2022-09-03T06:07:20.728 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.729 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.729 DEBUG:boto:Params: {} 2022-09-03T06:07:20.730 DEBUG:boto:Token: None 2022-09-03T06:07:20.740 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/?versions 2022-09-03T06:07:20.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3FOwU7rwpZolS062Cb9cF91At9E= 2022-09-03T06:07:20.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3FOwU7rwpZolS062Cb9cF91At9E=', 'Content-Length': '0'} 2022-09-03T06:07:20.745 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f40d41a1547f567-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.746 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.748 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:07:20.748 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.749 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.750 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.750 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.751 DEBUG:boto:Method: GET 2022-09-03T06:07:20.751 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.752 DEBUG:boto:Data: 2022-09-03T06:07:20.752 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.753 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.753 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.754 DEBUG:boto:Params: {} 2022-09-03T06:07:20.755 DEBUG:boto:Token: None 2022-09-03T06:07:20.755 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0= 2022-09-03T06:07:20.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0=', 'Content-Length': '0'} 2022-09-03T06:07:20.759 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000079bac465c27e7af-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.760 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.760 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.761 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.761 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.762 DEBUG:boto:Method: GET 2022-09-03T06:07:20.762 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.763 DEBUG:boto:Data: 2022-09-03T06:07:20.763 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.764 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.765 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.765 DEBUG:boto:Params: {} 2022-09-03T06:07:20.766 DEBUG:boto:Token: None 2022-09-03T06:07:20.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0= 2022-09-03T06:07:20.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0=', 'Content-Length': '0'} 2022-09-03T06:07:20.770 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009dc223c4726e51a9-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.771 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.771 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.772 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.773 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.774 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.774 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.775 DEBUG:boto:Data: 2022-09-03T06:07:20.775 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.776 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.776 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.777 DEBUG:boto:Params: {} 2022-09-03T06:07:20.778 DEBUG:boto:Token: None 2022-09-03T06:07:20.778 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.779 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aLkvPX+F9HunNdf1TQMjz2AHBW4= 2022-09-03T06:07:20.780 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aLkvPX+F9HunNdf1TQMjz2AHBW4=', 'Content-Length': '0'} 2022-09-03T06:07:20.782 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000bccc99b645cfa11a-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.783 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.783 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.784 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.785 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.785 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.786 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.786 DEBUG:boto:Data: 2022-09-03T06:07:20.787 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.788 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.789 DEBUG:boto:Params: {} 2022-09-03T06:07:20.790 DEBUG:boto:Token: None 2022-09-03T06:07:20.791 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aLkvPX+F9HunNdf1TQMjz2AHBW4= 2022-09-03T06:07:20.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aLkvPX+F9HunNdf1TQMjz2AHBW4=', 'Content-Length': '0'} 2022-09-03T06:07:20.796 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f9b51e1cdcc75da7-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.797 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:07:20.798 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.799 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.800 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.801 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.802 DEBUG:boto:Method: GET 2022-09-03T06:07:20.802 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.803 DEBUG:boto:Data: 2022-09-03T06:07:20.804 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.806 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.807 DEBUG:boto:Params: {} 2022-09-03T06:07:20.808 DEBUG:boto:Token: None 2022-09-03T06:07:20.809 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0= 2022-09-03T06:07:20.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0=', 'Content-Length': '0'} 2022-09-03T06:07:20.813 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000091cab4b45f12fac3-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.814 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.814 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.815 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.815 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.816 DEBUG:boto:Method: GET 2022-09-03T06:07:20.817 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.817 DEBUG:boto:Data: 2022-09-03T06:07:20.818 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.818 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.819 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.819 DEBUG:boto:Params: {} 2022-09-03T06:07:20.820 DEBUG:boto:Token: None 2022-09-03T06:07:20.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0= 2022-09-03T06:07:20.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0=', 'Content-Length': '0'} 2022-09-03T06:07:20.824 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001bf3d251d91b2006-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.825 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.826 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.827 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:20.827 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:20.828 DEBUG:boto:Method: GET 2022-09-03T06:07:20.829 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:20.829 DEBUG:boto:Data: 2022-09-03T06:07:20.830 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.831 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.832 DEBUG:boto:Params: {} 2022-09-03T06:07:20.834 DEBUG:boto:Token: None 2022-09-03T06:07:20.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.840 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a77a57bb0444c608-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.841 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.843 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.843 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.844 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:20.844 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:20.845 DEBUG:boto:Method: GET 2022-09-03T06:07:20.846 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:20.847 DEBUG:boto:Data: 2022-09-03T06:07:20.848 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.850 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.851 DEBUG:boto:Params: {} 2022-09-03T06:07:20.852 DEBUG:boto:Token: None 2022-09-03T06:07:20.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.858 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000038bbf69795858bc8-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.859 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.860 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.860 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.861 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.861 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:07:20.862 DEBUG:boto:Data: 2022-09-03T06:07:20.862 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.863 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.863 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.864 DEBUG:boto:Params: {} 2022-09-03T06:07:20.864 DEBUG:boto:Token: None 2022-09-03T06:07:20.865 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy 2022-09-03T06:07:20.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K3TQzVLksdrfazaBAM+lMQ010Sw= 2022-09-03T06:07:20.866 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K3TQzVLksdrfazaBAM+lMQ010Sw=', 'Content-Length': '0'} 2022-09-03T06:07:20.868 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003db2b21b46cc5108-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.869 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.870 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.871 DEBUG:boto:Method: HEAD 2022-09-03T06:07:20.871 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:07:20.872 DEBUG:boto:Data: 2022-09-03T06:07:20.872 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.873 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.874 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.875 DEBUG:boto:Params: {} 2022-09-03T06:07:20.876 DEBUG:boto:Token: None 2022-09-03T06:07:20.877 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy 2022-09-03T06:07:20.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K3TQzVLksdrfazaBAM+lMQ010Sw= 2022-09-03T06:07:20.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K3TQzVLksdrfazaBAM+lMQ010Sw=', 'Content-Length': '0'} 2022-09-03T06:07:20.882 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001b3aabc564dc152b-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.883 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:07:20.885 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.885 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.886 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.887 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.887 DEBUG:boto:Method: GET 2022-09-03T06:07:20.888 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.888 DEBUG:boto:Data: 2022-09-03T06:07:20.889 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.889 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.890 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.891 DEBUG:boto:Params: {} 2022-09-03T06:07:20.892 DEBUG:boto:Token: None 2022-09-03T06:07:20.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0= 2022-09-03T06:07:20.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0=', 'Content-Length': '0'} 2022-09-03T06:07:20.901 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001e05b5d519b330a9-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.902 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:20.903 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:20.903 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.904 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.905 DEBUG:boto:Method: GET 2022-09-03T06:07:20.905 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.906 DEBUG:boto:Data: 2022-09-03T06:07:20.906 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.907 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.908 DEBUG:boto:Params: {} 2022-09-03T06:07:20.909 DEBUG:boto:Token: None 2022-09-03T06:07:20.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:20.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0= 2022-09-03T06:07:20.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/yoaa+D7QtkZURmLCHgqHMmaYT0=', 'Content-Length': '0'} 2022-09-03T06:07:20.913 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000019581ed06b3df56a-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.914 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.914 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.915 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:20.915 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:20.916 DEBUG:boto:Method: GET 2022-09-03T06:07:20.916 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:20.917 DEBUG:boto:Data: 2022-09-03T06:07:20.917 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.918 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.918 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.919 DEBUG:boto:Params: {} 2022-09-03T06:07:20.920 DEBUG:boto:Token: None 2022-09-03T06:07:20.920 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.921 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.921 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.924 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009c0170ead535507e-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.925 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.926 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:20.926 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:20.927 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:20.928 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:20.928 DEBUG:boto:Method: GET 2022-09-03T06:07:20.929 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:20.929 DEBUG:boto:Data: 2022-09-03T06:07:20.930 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:20.931 DEBUG:boto:Port: 8000 2022-09-03T06:07:20.931 DEBUG:boto:Params: {} 2022-09-03T06:07:20.932 DEBUG:boto:Token: None 2022-09-03T06:07:20.933 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/ 2022-09-03T06:07:20.933 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8= 2022-09-03T06:07:20.934 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUUAvFuZiOW7o9ls7/NOrZrn7E8=', 'Content-Length': '0'} 2022-09-03T06:07:20.937 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001b2a674e0d1f240f-006312ef18-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.978 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:20.979 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:20.980 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.980 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.981 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.982 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.982 DEBUG:boto:Method: GET 2022-09-03T06:07:20.984 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.985 DEBUG:boto:Data: 2022-09-03T06:07:20.986 DEBUG:boto:Headers: {} 2022-09-03T06:07:20.987 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:20.988 DEBUG:boto:Port: 8001 2022-09-03T06:07:20.989 DEBUG:boto:Params: {} 2022-09-03T06:07:20.990 DEBUG:boto:Token: None 2022-09-03T06:07:20.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:20 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ= 2022-09-03T06:07:20.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y0kVK3KZ7b3bSuqi9mw3Tu06sGQ=', 'Content-Length': '0'} 2022-09-03T06:07:20.995 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002c790a7c4b5b2cc1-006312ef18-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:20.996 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:20.997 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:20.997 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.998 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.998 DEBUG:boto:Method: GET 2022-09-03T06:07:20.999 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:20.999 DEBUG:boto:Data: 2022-09-03T06:07:21.000 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.001 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.002 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.003 DEBUG:boto:Params: {} 2022-09-03T06:07:21.004 DEBUG:boto:Token: None 2022-09-03T06:07:21.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.009 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000489d1848e0d7b12e-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.010 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.011 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.012 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.012 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.013 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.013 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.014 DEBUG:boto:Data: 2022-09-03T06:07:21.014 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.015 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.016 DEBUG:boto:Params: {} 2022-09-03T06:07:21.016 DEBUG:boto:Token: None 2022-09-03T06:07:21.017 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q9njlbLXK2NVTbPhfe1TARgiAWc= 2022-09-03T06:07:21.019 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q9njlbLXK2NVTbPhfe1TARgiAWc=', 'Content-Length': '0'} 2022-09-03T06:07:21.021 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006118a0552b2b8368-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.022 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.023 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.023 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.024 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.024 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.025 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.026 DEBUG:boto:Data: 2022-09-03T06:07:21.026 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.027 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.028 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.028 DEBUG:boto:Params: {} 2022-09-03T06:07:21.029 DEBUG:boto:Token: None 2022-09-03T06:07:21.029 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.030 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q9njlbLXK2NVTbPhfe1TARgiAWc= 2022-09-03T06:07:21.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q9njlbLXK2NVTbPhfe1TARgiAWc=', 'Content-Length': '0'} 2022-09-03T06:07:21.033 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009ba9aa46512fb717-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.034 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.035 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.035 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.036 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.037 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.037 DEBUG:boto:Method: GET 2022-09-03T06:07:21.038 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.039 DEBUG:boto:Data: 2022-09-03T06:07:21.039 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.040 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.040 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.041 DEBUG:boto:Params: {} 2022-09-03T06:07:21.041 DEBUG:boto:Token: None 2022-09-03T06:07:21.042 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.046 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000094f69a0ec02099ad-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.048 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.049 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.050 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.051 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.052 DEBUG:boto:Method: GET 2022-09-03T06:07:21.053 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.054 DEBUG:boto:Data: 2022-09-03T06:07:21.055 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.057 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.058 DEBUG:boto:Params: {} 2022-09-03T06:07:21.059 DEBUG:boto:Token: None 2022-09-03T06:07:21.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.065 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b4b6515a08575891-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.067 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.068 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.069 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.069 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.070 DEBUG:boto:Method: GET 2022-09-03T06:07:21.071 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.071 DEBUG:boto:Data: 2022-09-03T06:07:21.072 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.072 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.073 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.073 DEBUG:boto:Params: {} 2022-09-03T06:07:21.074 DEBUG:boto:Token: None 2022-09-03T06:07:21.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.079 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008cff9be98eadea2a-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.122 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.123 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.124 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.125 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.126 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.127 DEBUG:boto:Method: GET 2022-09-03T06:07:21.129 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.130 DEBUG:boto:Data: 2022-09-03T06:07:21.131 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.132 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.133 DEBUG:boto:Params: {} 2022-09-03T06:07:21.134 DEBUG:boto:Token: None 2022-09-03T06:07:21.134 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.138 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3ace5944977cddb-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.139 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.140 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.141 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.141 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.142 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:21.142 DEBUG:boto:Data: 2022-09-03T06:07:21.143 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.144 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.145 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.146 DEBUG:boto:Params: {} 2022-09-03T06:07:21.147 DEBUG:boto:Token: None 2022-09-03T06:07:21.148 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj 2022-09-03T06:07:21.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng= 2022-09-03T06:07:21.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng=', 'Content-Length': '0'} 2022-09-03T06:07:21.155 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000087c6a8ca67cec0d9-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.156 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.157 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.158 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.159 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:21.160 DEBUG:boto:Data: 2022-09-03T06:07:21.160 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.161 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.162 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.162 DEBUG:boto:Params: {} 2022-09-03T06:07:21.163 DEBUG:boto:Token: None 2022-09-03T06:07:21.164 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj 2022-09-03T06:07:21.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng= 2022-09-03T06:07:21.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng=', 'Content-Length': '0'} 2022-09-03T06:07:21.168 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009a1f367205e47c95-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.169 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.170 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.170 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.171 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.172 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.172 DEBUG:boto:Method: GET 2022-09-03T06:07:21.173 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.173 DEBUG:boto:Data: 2022-09-03T06:07:21.174 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.174 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.175 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.175 DEBUG:boto:Params: {} 2022-09-03T06:07:21.176 DEBUG:boto:Token: None 2022-09-03T06:07:21.176 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.180 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000056ea2486f06103c5-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.181 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.181 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.182 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.182 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.183 DEBUG:boto:Method: GET 2022-09-03T06:07:21.183 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.184 DEBUG:boto:Data: 2022-09-03T06:07:21.184 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.185 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.186 DEBUG:boto:Params: {} 2022-09-03T06:07:21.187 DEBUG:boto:Token: None 2022-09-03T06:07:21.187 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.190 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003db72d0e1302344b-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.234 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.235 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.236 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.237 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.237 DEBUG:boto:Method: GET 2022-09-03T06:07:21.238 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.238 DEBUG:boto:Data: 2022-09-03T06:07:21.239 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.240 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.241 DEBUG:boto:Params: {} 2022-09-03T06:07:21.241 DEBUG:boto:Token: None 2022-09-03T06:07:21.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.246 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007388aba9555b1a3f-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.247 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.248 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.249 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.249 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.250 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.251 DEBUG:boto:Method: GET 2022-09-03T06:07:21.251 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.252 DEBUG:boto:Data: 2022-09-03T06:07:21.252 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.253 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.254 DEBUG:boto:Params: {} 2022-09-03T06:07:21.254 DEBUG:boto:Token: None 2022-09-03T06:07:21.255 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.260 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e544b97a70d652b-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.261 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.261 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.262 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.263 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.263 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.264 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.264 DEBUG:boto:Method: GET 2022-09-03T06:07:21.265 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.265 DEBUG:boto:Data: 2022-09-03T06:07:21.266 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.267 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.267 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.268 DEBUG:boto:Params: {} 2022-09-03T06:07:21.268 DEBUG:boto:Token: None 2022-09-03T06:07:21.269 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.270 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xfJ2BbzjKdSJCQHXIp2W/40jCME= 2022-09-03T06:07:21.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xfJ2BbzjKdSJCQHXIp2W/40jCME=', 'Content-Length': '0'} 2022-09-03T06:07:21.273 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000033eb66def7b09203-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.318 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.319 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.320 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.320 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.321 DEBUG:boto:Method: GET 2022-09-03T06:07:21.321 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.322 DEBUG:boto:Data: 2022-09-03T06:07:21.322 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.323 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.323 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.324 DEBUG:boto:Params: {} 2022-09-03T06:07:21.325 DEBUG:boto:Token: None 2022-09-03T06:07:21.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xfJ2BbzjKdSJCQHXIp2W/40jCME= 2022-09-03T06:07:21.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xfJ2BbzjKdSJCQHXIp2W/40jCME=', 'Content-Length': '0'} 2022-09-03T06:07:21.329 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007ffd110efee366dc-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.330 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.331 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.331 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.332 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.332 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.333 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.333 DEBUG:boto:Data: 2022-09-03T06:07:21.334 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.335 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.335 DEBUG:boto:Params: {} 2022-09-03T06:07:21.336 DEBUG:boto:Token: None 2022-09-03T06:07:21.336 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2iChIvbWpJZURt+CRxGM2xThS9w= 2022-09-03T06:07:21.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2iChIvbWpJZURt+CRxGM2xThS9w=', 'Content-Length': '0'} 2022-09-03T06:07:21.340 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cd908ded646cf27f-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.340 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.341 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.341 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.342 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.342 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.343 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.343 DEBUG:boto:Data: 2022-09-03T06:07:21.344 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.345 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.345 DEBUG:boto:Params: {} 2022-09-03T06:07:21.346 DEBUG:boto:Token: None 2022-09-03T06:07:21.347 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2iChIvbWpJZURt+CRxGM2xThS9w= 2022-09-03T06:07:21.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2iChIvbWpJZURt+CRxGM2xThS9w=', 'Content-Length': '0'} 2022-09-03T06:07:21.350 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007ac9364974f313ef-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.351 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.351 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.352 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.352 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.353 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.353 DEBUG:boto:Method: GET 2022-09-03T06:07:21.354 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.354 DEBUG:boto:Data: 2022-09-03T06:07:21.355 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.356 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.357 DEBUG:boto:Params: {} 2022-09-03T06:07:21.357 DEBUG:boto:Token: None 2022-09-03T06:07:21.358 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.359 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xfJ2BbzjKdSJCQHXIp2W/40jCME= 2022-09-03T06:07:21.359 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xfJ2BbzjKdSJCQHXIp2W/40jCME=', 'Content-Length': '0'} 2022-09-03T06:07:21.362 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009992caff8251d56c-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.362 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.363 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.364 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.364 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.365 DEBUG:boto:Method: GET 2022-09-03T06:07:21.365 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.366 DEBUG:boto:Data: 2022-09-03T06:07:21.366 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.367 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.368 DEBUG:boto:Params: {} 2022-09-03T06:07:21.368 DEBUG:boto:Token: None 2022-09-03T06:07:21.369 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:21.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xfJ2BbzjKdSJCQHXIp2W/40jCME= 2022-09-03T06:07:21.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xfJ2BbzjKdSJCQHXIp2W/40jCME=', 'Content-Length': '0'} 2022-09-03T06:07:21.373 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003d98bc02765c62b3-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.373 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.374 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.374 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.375 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.376 DEBUG:boto:Method: GET 2022-09-03T06:07:21.376 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.377 DEBUG:boto:Data: 2022-09-03T06:07:21.377 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.378 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.378 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.379 DEBUG:boto:Params: {} 2022-09-03T06:07:21.379 DEBUG:boto:Token: None 2022-09-03T06:07:21.380 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.384 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dcdd94f73b4b2915-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.385 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.386 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.387 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.387 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.388 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.388 DEBUG:boto:Method: GET 2022-09-03T06:07:21.389 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.389 DEBUG:boto:Data: 2022-09-03T06:07:21.390 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.391 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.391 DEBUG:boto:Params: {} 2022-09-03T06:07:21.392 DEBUG:boto:Token: None 2022-09-03T06:07:21.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.397 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011ade2331703cf86-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.397 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.398 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.399 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.399 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.400 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:21.400 DEBUG:boto:Data: 2022-09-03T06:07:21.401 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.402 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.402 DEBUG:boto:Params: {} 2022-09-03T06:07:21.403 DEBUG:boto:Token: None 2022-09-03T06:07:21.403 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj 2022-09-03T06:07:21.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng= 2022-09-03T06:07:21.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng=', 'Content-Length': '0'} 2022-09-03T06:07:21.408 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002f1536a2f717e7b5-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.409 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.410 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.410 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.411 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:21.411 DEBUG:boto:Data: 2022-09-03T06:07:21.412 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.413 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.413 DEBUG:boto:Params: {} 2022-09-03T06:07:21.414 DEBUG:boto:Token: None 2022-09-03T06:07:21.414 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj 2022-09-03T06:07:21.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng= 2022-09-03T06:07:21.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng=', 'Content-Length': '0'} 2022-09-03T06:07:21.419 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001c04a11f77ca0ffe-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.419 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.420 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.421 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.421 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.422 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.422 DEBUG:boto:Method: GET 2022-09-03T06:07:21.423 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.423 DEBUG:boto:Data: 2022-09-03T06:07:21.424 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.424 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.425 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.425 DEBUG:boto:Params: {} 2022-09-03T06:07:21.426 DEBUG:boto:Token: None 2022-09-03T06:07:21.427 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.427 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.430 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001659628db3457f9f-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.432 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.432 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.433 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.433 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.434 DEBUG:boto:Method: GET 2022-09-03T06:07:21.435 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.436 DEBUG:boto:Data: 2022-09-03T06:07:21.436 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.437 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.437 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.438 DEBUG:boto:Params: {} 2022-09-03T06:07:21.439 DEBUG:boto:Token: None 2022-09-03T06:07:21.439 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.443 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ed9d9380520b2a83-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.444 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.444 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.445 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.445 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.446 DEBUG:boto:Method: GET 2022-09-03T06:07:21.447 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.447 DEBUG:boto:Data: 2022-09-03T06:07:21.448 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.449 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.450 DEBUG:boto:Params: {} 2022-09-03T06:07:21.450 DEBUG:boto:Token: None 2022-09-03T06:07:21.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.455 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001caabb89a75232f2-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.502 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.503 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.504 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.504 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.505 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.505 DEBUG:boto:Method: GET 2022-09-03T06:07:21.506 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.506 DEBUG:boto:Data: 2022-09-03T06:07:21.507 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.508 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.509 DEBUG:boto:Params: {} 2022-09-03T06:07:21.509 DEBUG:boto:Token: None 2022-09-03T06:07:21.510 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.514 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8c36912a7448753-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.516 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.517 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.518 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.519 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.520 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.521 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.522 DEBUG:boto:Method: GET 2022-09-03T06:07:21.523 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.523 DEBUG:boto:Data: 2022-09-03T06:07:21.524 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.525 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.526 DEBUG:boto:Params: {} 2022-09-03T06:07:21.527 DEBUG:boto:Token: None 2022-09-03T06:07:21.528 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9OanJkg+oI04FdmqgF8amSjH70= 2022-09-03T06:07:21.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9OanJkg+oI04FdmqgF8amSjH70=', 'Content-Length': '0'} 2022-09-03T06:07:21.533 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000065e8d48d9a138a2d-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.534 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.535 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.536 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.537 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.538 DEBUG:boto:Method: GET 2022-09-03T06:07:21.539 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.539 DEBUG:boto:Data: 2022-09-03T06:07:21.540 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.541 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.541 DEBUG:boto:Params: {} 2022-09-03T06:07:21.542 DEBUG:boto:Token: None 2022-09-03T06:07:21.543 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9OanJkg+oI04FdmqgF8amSjH70= 2022-09-03T06:07:21.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9OanJkg+oI04FdmqgF8amSjH70=', 'Content-Length': '0'} 2022-09-03T06:07:21.546 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000a394a87c5b32dcd-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.590 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.591 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.591 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.592 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.593 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.594 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.594 DEBUG:boto:Data: 2022-09-03T06:07:21.595 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.595 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.596 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.596 DEBUG:boto:Params: {} 2022-09-03T06:07:21.597 DEBUG:boto:Token: None 2022-09-03T06:07:21.597 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.598 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AqjvPQ+7BkYtRCtHyr4BQxUoJEU= 2022-09-03T06:07:21.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AqjvPQ+7BkYtRCtHyr4BQxUoJEU=', 'Content-Length': '0'} 2022-09-03T06:07:21.601 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dc41410d8f2a6a76-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.602 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.602 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.603 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.603 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.604 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.604 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.605 DEBUG:boto:Data: 2022-09-03T06:07:21.605 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.606 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.606 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.607 DEBUG:boto:Params: {} 2022-09-03T06:07:21.607 DEBUG:boto:Token: None 2022-09-03T06:07:21.608 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AqjvPQ+7BkYtRCtHyr4BQxUoJEU= 2022-09-03T06:07:21.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AqjvPQ+7BkYtRCtHyr4BQxUoJEU=', 'Content-Length': '0'} 2022-09-03T06:07:21.611 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002da1d0f27eb32fb3-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.612 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.613 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.613 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.614 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.614 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.615 DEBUG:boto:Method: GET 2022-09-03T06:07:21.615 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.616 DEBUG:boto:Data: 2022-09-03T06:07:21.616 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.617 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.617 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.618 DEBUG:boto:Params: {} 2022-09-03T06:07:21.619 DEBUG:boto:Token: None 2022-09-03T06:07:21.619 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9OanJkg+oI04FdmqgF8amSjH70= 2022-09-03T06:07:21.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9OanJkg+oI04FdmqgF8amSjH70=', 'Content-Length': '0'} 2022-09-03T06:07:21.623 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000036fc25106bdae3f7-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.624 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.624 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.625 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.625 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.626 DEBUG:boto:Method: GET 2022-09-03T06:07:21.626 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.627 DEBUG:boto:Data: 2022-09-03T06:07:21.628 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.628 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.629 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.629 DEBUG:boto:Params: {} 2022-09-03T06:07:21.630 DEBUG:boto:Token: None 2022-09-03T06:07:21.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:21.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9OanJkg+oI04FdmqgF8amSjH70= 2022-09-03T06:07:21.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9OanJkg+oI04FdmqgF8amSjH70=', 'Content-Length': '0'} 2022-09-03T06:07:21.634 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001a73a76d541bcaef-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.635 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.635 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.636 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.636 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.637 DEBUG:boto:Method: GET 2022-09-03T06:07:21.638 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.638 DEBUG:boto:Data: 2022-09-03T06:07:21.639 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.640 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.640 DEBUG:boto:Params: {} 2022-09-03T06:07:21.641 DEBUG:boto:Token: None 2022-09-03T06:07:21.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.643 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.646 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a68f36356e2a200-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.646 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.647 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.648 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.648 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.649 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.649 DEBUG:boto:Method: GET 2022-09-03T06:07:21.650 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.651 DEBUG:boto:Data: 2022-09-03T06:07:21.651 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.652 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.653 DEBUG:boto:Params: {} 2022-09-03T06:07:21.653 DEBUG:boto:Token: None 2022-09-03T06:07:21.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.655 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.658 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000072ed4f59283538da-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.659 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.660 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.661 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.661 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.662 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:21.662 DEBUG:boto:Data: 2022-09-03T06:07:21.663 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.663 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.664 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.665 DEBUG:boto:Params: {} 2022-09-03T06:07:21.666 DEBUG:boto:Token: None 2022-09-03T06:07:21.666 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj 2022-09-03T06:07:21.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng= 2022-09-03T06:07:21.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng=', 'Content-Length': '0'} 2022-09-03T06:07:21.671 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000189af86969ea25e2-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.672 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.673 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.674 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.674 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:21.675 DEBUG:boto:Data: 2022-09-03T06:07:21.675 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.676 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.677 DEBUG:boto:Params: {} 2022-09-03T06:07:21.677 DEBUG:boto:Token: None 2022-09-03T06:07:21.678 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj 2022-09-03T06:07:21.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng= 2022-09-03T06:07:21.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng=', 'Content-Length': '0'} 2022-09-03T06:07:21.683 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000def4c9dfb614f9d-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.684 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.684 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.685 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.685 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.686 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.687 DEBUG:boto:Method: GET 2022-09-03T06:07:21.687 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.688 DEBUG:boto:Data: 2022-09-03T06:07:21.688 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.689 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.690 DEBUG:boto:Params: {} 2022-09-03T06:07:21.691 DEBUG:boto:Token: None 2022-09-03T06:07:21.691 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.695 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000047b9cead5079952c-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.742 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.743 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.743 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.744 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.744 DEBUG:boto:Method: GET 2022-09-03T06:07:21.745 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.745 DEBUG:boto:Data: 2022-09-03T06:07:21.746 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.747 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.748 DEBUG:boto:Params: {} 2022-09-03T06:07:21.748 DEBUG:boto:Token: None 2022-09-03T06:07:21.749 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.753 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003fbd390d40e52b14-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.754 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.754 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.755 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.755 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.756 DEBUG:boto:Method: GET 2022-09-03T06:07:21.756 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.757 DEBUG:boto:Data: 2022-09-03T06:07:21.757 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.758 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.759 DEBUG:boto:Params: {} 2022-09-03T06:07:21.760 DEBUG:boto:Token: None 2022-09-03T06:07:21.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.764 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e22964b28e822d37-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.765 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.766 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.767 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.767 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.768 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.768 DEBUG:boto:Method: GET 2022-09-03T06:07:21.769 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.769 DEBUG:boto:Data: 2022-09-03T06:07:21.770 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.771 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.771 DEBUG:boto:Params: {} 2022-09-03T06:07:21.772 DEBUG:boto:Token: None 2022-09-03T06:07:21.772 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.777 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093bbb0103f5bb959-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.778 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.779 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.780 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.780 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.781 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.781 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.782 DEBUG:boto:Method: GET 2022-09-03T06:07:21.782 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.783 DEBUG:boto:Data: 2022-09-03T06:07:21.784 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.785 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.786 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.787 DEBUG:boto:Params: {} 2022-09-03T06:07:21.788 DEBUG:boto:Token: None 2022-09-03T06:07:21.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azxwnJzjvSCYqCQkRiY7dGT1YEg= 2022-09-03T06:07:21.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azxwnJzjvSCYqCQkRiY7dGT1YEg=', 'Content-Length': '0'} 2022-09-03T06:07:21.793 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006b90238fd9deae33-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.794 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.794 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.795 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.795 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.796 DEBUG:boto:Method: GET 2022-09-03T06:07:21.796 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.797 DEBUG:boto:Data: 2022-09-03T06:07:21.797 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.798 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.799 DEBUG:boto:Params: {} 2022-09-03T06:07:21.800 DEBUG:boto:Token: None 2022-09-03T06:07:21.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azxwnJzjvSCYqCQkRiY7dGT1YEg= 2022-09-03T06:07:21.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azxwnJzjvSCYqCQkRiY7dGT1YEg=', 'Content-Length': '0'} 2022-09-03T06:07:21.805 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008c85d01118293398-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.806 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.807 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.808 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.808 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.809 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.810 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.811 DEBUG:boto:Data: 2022-09-03T06:07:21.812 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.814 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.815 DEBUG:boto:Params: {} 2022-09-03T06:07:21.815 DEBUG:boto:Token: None 2022-09-03T06:07:21.816 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZdpVy6+xAnDVWgVPr7M5r93LhM= 2022-09-03T06:07:21.817 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZdpVy6+xAnDVWgVPr7M5r93LhM=', 'Content-Length': '0'} 2022-09-03T06:07:21.819 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fd655baf9d3b66ee-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.820 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.821 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.822 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.822 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.823 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.823 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.824 DEBUG:boto:Data: 2022-09-03T06:07:21.824 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.825 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.825 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.826 DEBUG:boto:Params: {} 2022-09-03T06:07:21.827 DEBUG:boto:Token: None 2022-09-03T06:07:21.827 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZdpVy6+xAnDVWgVPr7M5r93LhM= 2022-09-03T06:07:21.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZdpVy6+xAnDVWgVPr7M5r93LhM=', 'Content-Length': '0'} 2022-09-03T06:07:21.832 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d4a727c681af7d7e-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.834 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.835 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.836 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.836 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.837 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.837 DEBUG:boto:Method: GET 2022-09-03T06:07:21.838 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.839 DEBUG:boto:Data: 2022-09-03T06:07:21.839 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.840 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.840 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.841 DEBUG:boto:Params: {} 2022-09-03T06:07:21.841 DEBUG:boto:Token: None 2022-09-03T06:07:21.842 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.843 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azxwnJzjvSCYqCQkRiY7dGT1YEg= 2022-09-03T06:07:21.843 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azxwnJzjvSCYqCQkRiY7dGT1YEg=', 'Content-Length': '0'} 2022-09-03T06:07:21.845 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000891ee80bf3af8fa9-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.846 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.847 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.847 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.848 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.849 DEBUG:boto:Method: GET 2022-09-03T06:07:21.849 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.850 DEBUG:boto:Data: 2022-09-03T06:07:21.850 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.851 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.852 DEBUG:boto:Params: {} 2022-09-03T06:07:21.852 DEBUG:boto:Token: None 2022-09-03T06:07:21.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:21.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azxwnJzjvSCYqCQkRiY7dGT1YEg= 2022-09-03T06:07:21.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azxwnJzjvSCYqCQkRiY7dGT1YEg=', 'Content-Length': '0'} 2022-09-03T06:07:21.862 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008113dc067956fbac-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.863 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.864 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.864 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.865 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.866 DEBUG:boto:Method: GET 2022-09-03T06:07:21.866 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.867 DEBUG:boto:Data: 2022-09-03T06:07:21.867 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.868 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.868 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.869 DEBUG:boto:Params: {} 2022-09-03T06:07:21.870 DEBUG:boto:Token: None 2022-09-03T06:07:21.870 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.871 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.874 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a5098bb8174441a9-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.875 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.876 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.877 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.878 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.878 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.879 DEBUG:boto:Method: GET 2022-09-03T06:07:21.879 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:21.880 DEBUG:boto:Data: 2022-09-03T06:07:21.881 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.882 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.883 DEBUG:boto:Params: {} 2022-09-03T06:07:21.883 DEBUG:boto:Token: None 2022-09-03T06:07:21.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/ 2022-09-03T06:07:21.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s= 2022-09-03T06:07:21.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lVdKI/k9pFGAt8OIEO8EkPK2x9s=', 'Content-Length': '0'} 2022-09-03T06:07:21.888 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000426cca385eee66ea-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.889 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:21.891 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.892 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.893 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.893 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:21.894 DEBUG:boto:Data: 2022-09-03T06:07:21.895 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.896 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.897 DEBUG:boto:Params: {} 2022-09-03T06:07:21.900 DEBUG:boto:Token: None 2022-09-03T06:07:21.902 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj 2022-09-03T06:07:21.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng= 2022-09-03T06:07:21.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng=', 'Content-Length': '0'} 2022-09-03T06:07:21.908 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000421d44be16a46355-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.910 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.911 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.912 DEBUG:boto:Method: HEAD 2022-09-03T06:07:21.912 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:21.913 DEBUG:boto:Data: 2022-09-03T06:07:21.913 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.914 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.914 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.915 DEBUG:boto:Params: {} 2022-09-03T06:07:21.915 DEBUG:boto:Token: None 2022-09-03T06:07:21.916 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj 2022-09-03T06:07:21.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng= 2022-09-03T06:07:21.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8e/h79Hfy4RyhzHipcLO+njjRng=', 'Content-Length': '0'} 2022-09-03T06:07:21.922 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000078533ad6df2ccad0-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.923 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:21.924 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.925 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.926 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.927 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.929 DEBUG:boto:Method: GET 2022-09-03T06:07:21.930 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.931 DEBUG:boto:Data: 2022-09-03T06:07:21.932 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:21.934 DEBUG:boto:Port: 8001 2022-09-03T06:07:21.935 DEBUG:boto:Params: {} 2022-09-03T06:07:21.936 DEBUG:boto:Token: None 2022-09-03T06:07:21.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.944 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000af61df1fc77aa056-006312ef19-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.986 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:21.987 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:21.987 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.988 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.988 DEBUG:boto:Method: GET 2022-09-03T06:07:21.989 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.989 DEBUG:boto:Data: 2022-09-03T06:07:21.990 DEBUG:boto:Headers: {} 2022-09-03T06:07:21.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:21.991 DEBUG:boto:Port: 8000 2022-09-03T06:07:21.992 DEBUG:boto:Params: {} 2022-09-03T06:07:21.992 DEBUG:boto:Token: None 2022-09-03T06:07:21.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:21 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:21.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk= 2022-09-03T06:07:21.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FIq0lVVG/o602YpflZHbAipkbNk=', 'Content-Length': '0'} 2022-09-03T06:07:21.997 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fc0e8473c9fd7a36-006312ef19-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:21.998 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:21.998 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:21.999 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:21.999 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.000 DEBUG:boto:Method: GET 2022-09-03T06:07:22.001 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.001 DEBUG:boto:Data: 2022-09-03T06:07:22.002 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.002 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.003 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.003 DEBUG:boto:Params: {} 2022-09-03T06:07:22.004 DEBUG:boto:Token: None 2022-09-03T06:07:22.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.009 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000138d49e673a8a5d5-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.010 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.011 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.011 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.012 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.012 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.013 DEBUG:boto:Method: GET 2022-09-03T06:07:22.013 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.014 DEBUG:boto:Data: 2022-09-03T06:07:22.015 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.016 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.016 DEBUG:boto:Params: {} 2022-09-03T06:07:22.017 DEBUG:boto:Token: None 2022-09-03T06:07:22.018 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.019 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.023 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000185492d64f96901c-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.024 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.025 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.026 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.027 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.027 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.028 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.029 DEBUG:boto:Method: GET 2022-09-03T06:07:22.030 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.032 DEBUG:boto:Data: 2022-09-03T06:07:22.033 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.034 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.035 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.035 DEBUG:boto:Params: {} 2022-09-03T06:07:22.036 DEBUG:boto:Token: None 2022-09-03T06:07:22.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5TPrdPRg9RogL71XNoQfd+k60Gs= 2022-09-03T06:07:22.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5TPrdPRg9RogL71XNoQfd+k60Gs=', 'Content-Length': '0'} 2022-09-03T06:07:22.041 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dcb15e178e6eedcd-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.042 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.043 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.044 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.044 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.045 DEBUG:boto:Method: GET 2022-09-03T06:07:22.045 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.046 DEBUG:boto:Data: 2022-09-03T06:07:22.047 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.048 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.048 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.049 DEBUG:boto:Params: {} 2022-09-03T06:07:22.050 DEBUG:boto:Token: None 2022-09-03T06:07:22.051 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5TPrdPRg9RogL71XNoQfd+k60Gs= 2022-09-03T06:07:22.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5TPrdPRg9RogL71XNoQfd+k60Gs=', 'Content-Length': '0'} 2022-09-03T06:07:22.059 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000eb811bbea4884444-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.060 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.061 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.062 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.063 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.063 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.064 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.064 DEBUG:boto:Data: 2022-09-03T06:07:22.065 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.066 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.066 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.067 DEBUG:boto:Params: {} 2022-09-03T06:07:22.067 DEBUG:boto:Token: None 2022-09-03T06:07:22.068 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wItre5OcFWnWb4GLXk0gv0cO9Rg= 2022-09-03T06:07:22.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wItre5OcFWnWb4GLXk0gv0cO9Rg=', 'Content-Length': '0'} 2022-09-03T06:07:22.072 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006d8fe494cd920ab2-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.073 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.074 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.075 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.075 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.076 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.076 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.077 DEBUG:boto:Data: 2022-09-03T06:07:22.077 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.078 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.079 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.080 DEBUG:boto:Params: {} 2022-09-03T06:07:22.080 DEBUG:boto:Token: None 2022-09-03T06:07:22.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wItre5OcFWnWb4GLXk0gv0cO9Rg= 2022-09-03T06:07:22.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wItre5OcFWnWb4GLXk0gv0cO9Rg=', 'Content-Length': '0'} 2022-09-03T06:07:22.086 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000053d2afe50aeae71-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.087 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.087 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.088 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.088 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.089 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.089 DEBUG:boto:Method: GET 2022-09-03T06:07:22.090 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.090 DEBUG:boto:Data: 2022-09-03T06:07:22.091 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.093 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.094 DEBUG:boto:Params: {} 2022-09-03T06:07:22.094 DEBUG:boto:Token: None 2022-09-03T06:07:22.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5TPrdPRg9RogL71XNoQfd+k60Gs= 2022-09-03T06:07:22.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5TPrdPRg9RogL71XNoQfd+k60Gs=', 'Content-Length': '0'} 2022-09-03T06:07:22.099 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fafb3aee1bcae805-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.100 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.100 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.101 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.101 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.102 DEBUG:boto:Method: GET 2022-09-03T06:07:22.102 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.103 DEBUG:boto:Data: 2022-09-03T06:07:22.103 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.104 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.105 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.105 DEBUG:boto:Params: {} 2022-09-03T06:07:22.106 DEBUG:boto:Token: None 2022-09-03T06:07:22.106 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:22.107 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5TPrdPRg9RogL71XNoQfd+k60Gs= 2022-09-03T06:07:22.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5TPrdPRg9RogL71XNoQfd+k60Gs=', 'Content-Length': '0'} 2022-09-03T06:07:22.110 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d5cf948a87073b56-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.158 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.159 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.160 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.161 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.162 DEBUG:boto:Method: GET 2022-09-03T06:07:22.163 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.164 DEBUG:boto:Data: 2022-09-03T06:07:22.165 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.166 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.167 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.168 DEBUG:boto:Params: {} 2022-09-03T06:07:22.169 DEBUG:boto:Token: None 2022-09-03T06:07:22.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.175 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e8c33b913014e4b-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.176 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.177 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.177 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.178 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.178 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.179 DEBUG:boto:Method: GET 2022-09-03T06:07:22.181 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.182 DEBUG:boto:Data: 2022-09-03T06:07:22.183 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.183 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.184 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.184 DEBUG:boto:Params: {} 2022-09-03T06:07:22.185 DEBUG:boto:Token: None 2022-09-03T06:07:22.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.186 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.190 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008e473eb195af514a-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.190 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.191 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.192 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.192 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.193 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:22.193 DEBUG:boto:Data: 2022-09-03T06:07:22.194 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.195 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.195 DEBUG:boto:Params: {} 2022-09-03T06:07:22.196 DEBUG:boto:Token: None 2022-09-03T06:07:22.197 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj 2022-09-03T06:07:22.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY= 2022-09-03T06:07:22.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY=', 'Content-Length': '0'} 2022-09-03T06:07:22.203 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a474f862e4ece9d9-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.203 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.204 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.205 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.205 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:22.206 DEBUG:boto:Data: 2022-09-03T06:07:22.206 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.207 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.207 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.208 DEBUG:boto:Params: {} 2022-09-03T06:07:22.208 DEBUG:boto:Token: None 2022-09-03T06:07:22.209 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj 2022-09-03T06:07:22.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY= 2022-09-03T06:07:22.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY=', 'Content-Length': '0'} 2022-09-03T06:07:22.214 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000eebac2d013b08722-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.214 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.215 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.216 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.216 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.217 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.217 DEBUG:boto:Method: GET 2022-09-03T06:07:22.218 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.218 DEBUG:boto:Data: 2022-09-03T06:07:22.219 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.219 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.220 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.220 DEBUG:boto:Params: {} 2022-09-03T06:07:22.221 DEBUG:boto:Token: None 2022-09-03T06:07:22.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc= 2022-09-03T06:07:22.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc=', 'Content-Length': '0'} 2022-09-03T06:07:22.226 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001ed03a8c7627335e-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.266 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.267 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.267 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.268 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.268 DEBUG:boto:Method: GET 2022-09-03T06:07:22.269 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.269 DEBUG:boto:Data: 2022-09-03T06:07:22.270 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.271 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.272 DEBUG:boto:Params: {} 2022-09-03T06:07:22.273 DEBUG:boto:Token: None 2022-09-03T06:07:22.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc= 2022-09-03T06:07:22.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc=', 'Content-Length': '0'} 2022-09-03T06:07:22.277 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000022b2148727fbaae8-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.278 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.279 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.280 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.281 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.282 DEBUG:boto:Method: GET 2022-09-03T06:07:22.282 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.283 DEBUG:boto:Data: 2022-09-03T06:07:22.283 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.284 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.285 DEBUG:boto:Params: {} 2022-09-03T06:07:22.285 DEBUG:boto:Token: None 2022-09-03T06:07:22.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.290 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e5e905fab36ea7df-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.291 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.292 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.292 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.293 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.293 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.294 DEBUG:boto:Method: GET 2022-09-03T06:07:22.295 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.295 DEBUG:boto:Data: 2022-09-03T06:07:22.296 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.296 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.297 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.297 DEBUG:boto:Params: {} 2022-09-03T06:07:22.298 DEBUG:boto:Token: None 2022-09-03T06:07:22.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.303 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000be548aea4788f7b4-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.304 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.304 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.305 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.306 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.306 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.307 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.307 DEBUG:boto:Method: GET 2022-09-03T06:07:22.308 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.309 DEBUG:boto:Data: 2022-09-03T06:07:22.309 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.310 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.310 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.311 DEBUG:boto:Params: {} 2022-09-03T06:07:22.311 DEBUG:boto:Token: None 2022-09-03T06:07:22.312 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.313 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiGhVY7xWEVX4FercuXBgGgT8e4= 2022-09-03T06:07:22.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiGhVY7xWEVX4FercuXBgGgT8e4=', 'Content-Length': '0'} 2022-09-03T06:07:22.316 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000015a6f9b2b40d6d7-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.317 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.317 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.318 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.318 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.319 DEBUG:boto:Method: GET 2022-09-03T06:07:22.319 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.320 DEBUG:boto:Data: 2022-09-03T06:07:22.320 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.321 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.322 DEBUG:boto:Params: {} 2022-09-03T06:07:22.323 DEBUG:boto:Token: None 2022-09-03T06:07:22.323 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiGhVY7xWEVX4FercuXBgGgT8e4= 2022-09-03T06:07:22.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiGhVY7xWEVX4FercuXBgGgT8e4=', 'Content-Length': '0'} 2022-09-03T06:07:22.327 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003b5f60f9783274d1-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.328 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.329 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.329 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.330 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.330 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.331 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.331 DEBUG:boto:Data: 2022-09-03T06:07:22.332 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.333 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.333 DEBUG:boto:Params: {} 2022-09-03T06:07:22.334 DEBUG:boto:Token: None 2022-09-03T06:07:22.335 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Eig7QdlokTiE9S6/XmafFPR4qnY= 2022-09-03T06:07:22.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Eig7QdlokTiE9S6/XmafFPR4qnY=', 'Content-Length': '0'} 2022-09-03T06:07:22.338 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000076470c600be07550-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.339 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.339 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.340 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.341 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.341 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.343 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.344 DEBUG:boto:Data: 2022-09-03T06:07:22.344 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.345 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.346 DEBUG:boto:Params: {} 2022-09-03T06:07:22.347 DEBUG:boto:Token: None 2022-09-03T06:07:22.347 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Eig7QdlokTiE9S6/XmafFPR4qnY= 2022-09-03T06:07:22.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Eig7QdlokTiE9S6/XmafFPR4qnY=', 'Content-Length': '0'} 2022-09-03T06:07:22.351 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004f68f610aa42ea51-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.351 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.353 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.354 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.355 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.355 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.356 DEBUG:boto:Method: GET 2022-09-03T06:07:22.357 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.358 DEBUG:boto:Data: 2022-09-03T06:07:22.359 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.360 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.362 DEBUG:boto:Params: {} 2022-09-03T06:07:22.362 DEBUG:boto:Token: None 2022-09-03T06:07:22.363 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiGhVY7xWEVX4FercuXBgGgT8e4= 2022-09-03T06:07:22.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiGhVY7xWEVX4FercuXBgGgT8e4=', 'Content-Length': '0'} 2022-09-03T06:07:22.368 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000078e18680590666e1-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.369 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.370 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.371 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.372 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.374 DEBUG:boto:Method: GET 2022-09-03T06:07:22.375 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.376 DEBUG:boto:Data: 2022-09-03T06:07:22.377 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.378 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.380 DEBUG:boto:Params: {} 2022-09-03T06:07:22.380 DEBUG:boto:Token: None 2022-09-03T06:07:22.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:22.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiGhVY7xWEVX4FercuXBgGgT8e4= 2022-09-03T06:07:22.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiGhVY7xWEVX4FercuXBgGgT8e4=', 'Content-Length': '0'} 2022-09-03T06:07:22.385 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000097f47676054e5cf-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.430 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.431 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.431 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.432 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.432 DEBUG:boto:Method: GET 2022-09-03T06:07:22.434 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.434 DEBUG:boto:Data: 2022-09-03T06:07:22.435 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.435 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.436 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.436 DEBUG:boto:Params: {} 2022-09-03T06:07:22.437 DEBUG:boto:Token: None 2022-09-03T06:07:22.438 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.443 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000099d91ba111e7c8e3-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.445 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.446 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.447 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.448 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.449 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.450 DEBUG:boto:Method: GET 2022-09-03T06:07:22.451 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.452 DEBUG:boto:Data: 2022-09-03T06:07:22.453 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.454 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.455 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.455 DEBUG:boto:Params: {} 2022-09-03T06:07:22.456 DEBUG:boto:Token: None 2022-09-03T06:07:22.457 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.462 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c3ed5779167e460-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.462 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.463 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.464 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.465 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.465 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:22.466 DEBUG:boto:Data: 2022-09-03T06:07:22.466 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.467 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.468 DEBUG:boto:Params: {} 2022-09-03T06:07:22.469 DEBUG:boto:Token: None 2022-09-03T06:07:22.469 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj 2022-09-03T06:07:22.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY= 2022-09-03T06:07:22.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY=', 'Content-Length': '0'} 2022-09-03T06:07:22.475 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a6003ee31c430e14-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.476 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.477 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.478 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.479 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:22.480 DEBUG:boto:Data: 2022-09-03T06:07:22.480 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.481 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.481 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.482 DEBUG:boto:Params: {} 2022-09-03T06:07:22.482 DEBUG:boto:Token: None 2022-09-03T06:07:22.483 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj 2022-09-03T06:07:22.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY= 2022-09-03T06:07:22.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY=', 'Content-Length': '0'} 2022-09-03T06:07:22.487 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009433e763666b30d2-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.488 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.489 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.489 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.490 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.490 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.491 DEBUG:boto:Method: GET 2022-09-03T06:07:22.492 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.492 DEBUG:boto:Data: 2022-09-03T06:07:22.493 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.494 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.494 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.495 DEBUG:boto:Params: {} 2022-09-03T06:07:22.495 DEBUG:boto:Token: None 2022-09-03T06:07:22.496 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc= 2022-09-03T06:07:22.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc=', 'Content-Length': '0'} 2022-09-03T06:07:22.500 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e52666d8e3be4143-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.546 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.547 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.548 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.548 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.549 DEBUG:boto:Method: GET 2022-09-03T06:07:22.549 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.550 DEBUG:boto:Data: 2022-09-03T06:07:22.550 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.551 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.552 DEBUG:boto:Params: {} 2022-09-03T06:07:22.552 DEBUG:boto:Token: None 2022-09-03T06:07:22.553 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc= 2022-09-03T06:07:22.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc=', 'Content-Length': '0'} 2022-09-03T06:07:22.557 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003385546c1ebe635c-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.558 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.558 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.559 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.559 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.560 DEBUG:boto:Method: GET 2022-09-03T06:07:22.560 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.561 DEBUG:boto:Data: 2022-09-03T06:07:22.561 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.562 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.563 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.563 DEBUG:boto:Params: {} 2022-09-03T06:07:22.564 DEBUG:boto:Token: None 2022-09-03T06:07:22.565 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.565 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.566 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.570 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fde4383bacc2cbe8-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.571 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.572 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.573 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.573 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.573 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.574 DEBUG:boto:Method: GET 2022-09-03T06:07:22.575 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.575 DEBUG:boto:Data: 2022-09-03T06:07:22.576 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.577 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.577 DEBUG:boto:Params: {} 2022-09-03T06:07:22.578 DEBUG:boto:Token: None 2022-09-03T06:07:22.578 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.582 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000059f3f2105563cd22-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.583 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.584 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.584 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.585 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.585 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.586 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.587 DEBUG:boto:Method: GET 2022-09-03T06:07:22.587 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.588 DEBUG:boto:Data: 2022-09-03T06:07:22.588 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.589 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.589 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.590 DEBUG:boto:Params: {} 2022-09-03T06:07:22.590 DEBUG:boto:Token: None 2022-09-03T06:07:22.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0U7O6LHaSl/DP/6CYOLrZEVbcto= 2022-09-03T06:07:22.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0U7O6LHaSl/DP/6CYOLrZEVbcto=', 'Content-Length': '0'} 2022-09-03T06:07:22.594 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000141c5bd453309f34-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.595 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.595 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.596 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.597 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.598 DEBUG:boto:Method: GET 2022-09-03T06:07:22.599 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.600 DEBUG:boto:Data: 2022-09-03T06:07:22.601 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.602 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.603 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.603 DEBUG:boto:Params: {} 2022-09-03T06:07:22.604 DEBUG:boto:Token: None 2022-09-03T06:07:22.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0U7O6LHaSl/DP/6CYOLrZEVbcto= 2022-09-03T06:07:22.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0U7O6LHaSl/DP/6CYOLrZEVbcto=', 'Content-Length': '0'} 2022-09-03T06:07:22.608 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003e1946c762874f29-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.609 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.609 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.610 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.610 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.611 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.611 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.612 DEBUG:boto:Data: 2022-09-03T06:07:22.612 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.613 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.614 DEBUG:boto:Params: {} 2022-09-03T06:07:22.615 DEBUG:boto:Token: None 2022-09-03T06:07:22.615 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aoo8mug/meAgK0f9LIE0t+bP2ok= 2022-09-03T06:07:22.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aoo8mug/meAgK0f9LIE0t+bP2ok=', 'Content-Length': '0'} 2022-09-03T06:07:22.619 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000b09ce16354b2154-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.620 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.621 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.622 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.623 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.624 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.625 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.626 DEBUG:boto:Data: 2022-09-03T06:07:22.626 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.628 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.629 DEBUG:boto:Params: {} 2022-09-03T06:07:22.629 DEBUG:boto:Token: None 2022-09-03T06:07:22.630 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aoo8mug/meAgK0f9LIE0t+bP2ok= 2022-09-03T06:07:22.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aoo8mug/meAgK0f9LIE0t+bP2ok=', 'Content-Length': '0'} 2022-09-03T06:07:22.633 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003e6f4a5d7f3a1add-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.634 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.634 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.635 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.636 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.637 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.638 DEBUG:boto:Method: GET 2022-09-03T06:07:22.639 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.640 DEBUG:boto:Data: 2022-09-03T06:07:22.641 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.642 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.643 DEBUG:boto:Params: {} 2022-09-03T06:07:22.643 DEBUG:boto:Token: None 2022-09-03T06:07:22.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0U7O6LHaSl/DP/6CYOLrZEVbcto= 2022-09-03T06:07:22.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0U7O6LHaSl/DP/6CYOLrZEVbcto=', 'Content-Length': '0'} 2022-09-03T06:07:22.647 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be11ed569da4f04d-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.694 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.695 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.695 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.696 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.697 DEBUG:boto:Method: GET 2022-09-03T06:07:22.697 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.698 DEBUG:boto:Data: 2022-09-03T06:07:22.699 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.701 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.701 DEBUG:boto:Params: {} 2022-09-03T06:07:22.702 DEBUG:boto:Token: None 2022-09-03T06:07:22.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:22.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0U7O6LHaSl/DP/6CYOLrZEVbcto= 2022-09-03T06:07:22.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0U7O6LHaSl/DP/6CYOLrZEVbcto=', 'Content-Length': '0'} 2022-09-03T06:07:22.707 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002778704c8a484765-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.708 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.709 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.709 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.710 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.710 DEBUG:boto:Method: GET 2022-09-03T06:07:22.711 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.711 DEBUG:boto:Data: 2022-09-03T06:07:22.712 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.713 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.714 DEBUG:boto:Params: {} 2022-09-03T06:07:22.714 DEBUG:boto:Token: None 2022-09-03T06:07:22.715 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.716 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.720 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f66d2d6e99d27c58-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.721 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.722 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.722 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.723 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.723 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.724 DEBUG:boto:Method: GET 2022-09-03T06:07:22.724 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.725 DEBUG:boto:Data: 2022-09-03T06:07:22.725 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.726 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.726 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.727 DEBUG:boto:Params: {} 2022-09-03T06:07:22.727 DEBUG:boto:Token: None 2022-09-03T06:07:22.728 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.732 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000249696aaa2a1759c-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.774 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.776 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.776 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.777 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.777 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:22.778 DEBUG:boto:Data: 2022-09-03T06:07:22.779 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.781 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.783 DEBUG:boto:Params: {} 2022-09-03T06:07:22.789 DEBUG:boto:Token: None 2022-09-03T06:07:22.790 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj 2022-09-03T06:07:22.791 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY= 2022-09-03T06:07:22.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY=', 'Content-Length': '0'} 2022-09-03T06:07:22.795 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000057176470a52a3b98-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.796 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.796 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.797 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.799 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:22.800 DEBUG:boto:Data: 2022-09-03T06:07:22.802 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.802 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.803 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.805 DEBUG:boto:Params: {} 2022-09-03T06:07:22.806 DEBUG:boto:Token: None 2022-09-03T06:07:22.807 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj 2022-09-03T06:07:22.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY= 2022-09-03T06:07:22.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zQcJFNq0i1Q7h2Te0k6CWzTm+XY=', 'Content-Length': '0'} 2022-09-03T06:07:22.815 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d1493e3c7a085a26-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.818 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.819 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.823 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.824 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.825 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.826 DEBUG:boto:Method: GET 2022-09-03T06:07:22.828 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.829 DEBUG:boto:Data: 2022-09-03T06:07:22.829 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.832 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.833 DEBUG:boto:Params: {} 2022-09-03T06:07:22.835 DEBUG:boto:Token: None 2022-09-03T06:07:22.836 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc= 2022-09-03T06:07:22.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc=', 'Content-Length': '0'} 2022-09-03T06:07:22.842 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009994d8fb9011d446-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.843 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.844 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.845 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.845 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.846 DEBUG:boto:Method: GET 2022-09-03T06:07:22.847 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.847 DEBUG:boto:Data: 2022-09-03T06:07:22.848 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.849 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.849 DEBUG:boto:Params: {} 2022-09-03T06:07:22.850 DEBUG:boto:Token: None 2022-09-03T06:07:22.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:22.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc= 2022-09-03T06:07:22.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf1dkpoHgzBZajEc5bjsbhVZEnc=', 'Content-Length': '0'} 2022-09-03T06:07:22.854 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003616e5cf5a385742-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.855 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.856 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.857 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.857 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.858 DEBUG:boto:Method: GET 2022-09-03T06:07:22.858 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.859 DEBUG:boto:Data: 2022-09-03T06:07:22.860 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.861 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.861 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.862 DEBUG:boto:Params: {} 2022-09-03T06:07:22.863 DEBUG:boto:Token: None 2022-09-03T06:07:22.863 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.869 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000be82d6ac05aee252-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.870 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.872 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:22.873 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:22.873 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.874 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:22.874 DEBUG:boto:Method: GET 2022-09-03T06:07:22.875 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:22.876 DEBUG:boto:Data: 2022-09-03T06:07:22.877 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.877 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.878 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.878 DEBUG:boto:Params: {} 2022-09-03T06:07:22.879 DEBUG:boto:Token: None 2022-09-03T06:07:22.879 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/ 2022-09-03T06:07:22.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E= 2022-09-03T06:07:22.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJBC3DdcgX3lN60Gn1sFRGgiA8E=', 'Content-Length': '0'} 2022-09-03T06:07:22.884 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b535a87ed6be78c6-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.926 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:22.927 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.928 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.929 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.929 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.930 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.931 DEBUG:boto:Method: GET 2022-09-03T06:07:22.932 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.933 DEBUG:boto:Data: 2022-09-03T06:07:22.934 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.935 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.935 DEBUG:boto:Params: {} 2022-09-03T06:07:22.936 DEBUG:boto:Token: None 2022-09-03T06:07:22.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YTnJ8BYcVox7Esyx0d2ywBs2jc= 2022-09-03T06:07:22.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YTnJ8BYcVox7Esyx0d2ywBs2jc=', 'Content-Length': '0'} 2022-09-03T06:07:22.945 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008f2df631e8471edc-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.946 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.947 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.948 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.949 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.949 DEBUG:boto:Method: GET 2022-09-03T06:07:22.950 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.951 DEBUG:boto:Data: 2022-09-03T06:07:22.951 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.952 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.952 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.953 DEBUG:boto:Params: {} 2022-09-03T06:07:22.954 DEBUG:boto:Token: None 2022-09-03T06:07:22.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YTnJ8BYcVox7Esyx0d2ywBs2jc= 2022-09-03T06:07:22.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YTnJ8BYcVox7Esyx0d2ywBs2jc=', 'Content-Length': '0'} 2022-09-03T06:07:22.961 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000849b0c4bac1e83e2-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.961 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.962 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.962 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.963 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.963 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.964 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.965 DEBUG:boto:Data: 2022-09-03T06:07:22.965 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.966 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.967 DEBUG:boto:Params: {} 2022-09-03T06:07:22.967 DEBUG:boto:Token: None 2022-09-03T06:07:22.968 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:esQopPyN9juOZhpwJcenJZNN2jg= 2022-09-03T06:07:22.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:esQopPyN9juOZhpwJcenJZNN2jg=', 'Content-Length': '0'} 2022-09-03T06:07:22.975 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000d8b71c745880810-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.976 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.977 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.977 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.978 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.979 DEBUG:boto:Method: HEAD 2022-09-03T06:07:22.979 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.980 DEBUG:boto:Data: 2022-09-03T06:07:22.980 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.981 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:22.981 DEBUG:boto:Port: 8000 2022-09-03T06:07:22.981 DEBUG:boto:Params: {} 2022-09-03T06:07:22.982 DEBUG:boto:Token: None 2022-09-03T06:07:22.983 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:esQopPyN9juOZhpwJcenJZNN2jg= 2022-09-03T06:07:22.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:esQopPyN9juOZhpwJcenJZNN2jg=', 'Content-Length': '0'} 2022-09-03T06:07:22.987 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002110b31c9a7c00f3-006312ef1a-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.988 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:22.988 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:22.989 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:22.989 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.990 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.990 DEBUG:boto:Method: GET 2022-09-03T06:07:22.991 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.992 DEBUG:boto:Data: 2022-09-03T06:07:22.992 DEBUG:boto:Headers: {} 2022-09-03T06:07:22.993 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:22.993 DEBUG:boto:Port: 8001 2022-09-03T06:07:22.994 DEBUG:boto:Params: {} 2022-09-03T06:07:22.994 DEBUG:boto:Token: None 2022-09-03T06:07:22.995 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:22 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:22.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YTnJ8BYcVox7Esyx0d2ywBs2jc= 2022-09-03T06:07:22.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YTnJ8BYcVox7Esyx0d2ywBs2jc=', 'Content-Length': '0'} 2022-09-03T06:07:22.998 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000098399ce8fbf4fe09-006312ef1a-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:22.999 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.000 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.000 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:23.001 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:23.001 DEBUG:boto:Method: GET 2022-09-03T06:07:23.002 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:23.002 DEBUG:boto:Data: 2022-09-03T06:07:23.004 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.005 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.006 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.006 DEBUG:boto:Params: {} 2022-09-03T06:07:23.007 DEBUG:boto:Token: None 2022-09-03T06:07:23.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:23.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uARSOr+uOT5gBGuR5c5UWKLk3f8= 2022-09-03T06:07:23.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uARSOr+uOT5gBGuR5c5UWKLk3f8=', 'Content-Length': '0'} 2022-09-03T06:07:23.011 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000017d542f384e84efb-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.012 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.013 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.013 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.014 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.015 DEBUG:boto:Method: GET 2022-09-03T06:07:23.016 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.016 DEBUG:boto:Data: 2022-09-03T06:07:23.017 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.018 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.018 DEBUG:boto:Params: {} 2022-09-03T06:07:23.019 DEBUG:boto:Token: None 2022-09-03T06:07:23.019 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.024 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e6e1a5b43508c0fd-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.025 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.026 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.026 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.027 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.027 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.028 DEBUG:boto:Method: GET 2022-09-03T06:07:23.028 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.029 DEBUG:boto:Data: 2022-09-03T06:07:23.029 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.030 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.031 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.031 DEBUG:boto:Params: {} 2022-09-03T06:07:23.032 DEBUG:boto:Token: None 2022-09-03T06:07:23.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.037 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c0a7c49b8a392de-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.038 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.039 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.040 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.040 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.041 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:23.041 DEBUG:boto:Data: 2022-09-03T06:07:23.042 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.043 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.043 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.044 DEBUG:boto:Params: {} 2022-09-03T06:07:23.044 DEBUG:boto:Token: None 2022-09-03T06:07:23.045 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj 2022-09-03T06:07:23.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g= 2022-09-03T06:07:23.046 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g=', 'Content-Length': '0'} 2022-09-03T06:07:23.050 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ba53ae04ae3fa8c2-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.051 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.051 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.052 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.053 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:23.053 DEBUG:boto:Data: 2022-09-03T06:07:23.054 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.055 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.055 DEBUG:boto:Params: {} 2022-09-03T06:07:23.056 DEBUG:boto:Token: None 2022-09-03T06:07:23.056 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj 2022-09-03T06:07:23.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g= 2022-09-03T06:07:23.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g=', 'Content-Length': '0'} 2022-09-03T06:07:23.066 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002b01fa06f1848a6f-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.067 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:23.068 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.068 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.070 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.070 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.071 DEBUG:boto:Method: GET 2022-09-03T06:07:23.071 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.072 DEBUG:boto:Data: 2022-09-03T06:07:23.072 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.073 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.074 DEBUG:boto:Params: {} 2022-09-03T06:07:23.075 DEBUG:boto:Token: None 2022-09-03T06:07:23.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po= 2022-09-03T06:07:23.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po=', 'Content-Length': '0'} 2022-09-03T06:07:23.079 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000032ede28d73b29cf0-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.132 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.133 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.134 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.261 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.262 DEBUG:boto:Method: GET 2022-09-03T06:07:23.262 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.263 DEBUG:boto:Data: 2022-09-03T06:07:23.264 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.265 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.265 DEBUG:boto:Params: {} 2022-09-03T06:07:23.267 DEBUG:boto:Token: None 2022-09-03T06:07:23.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po= 2022-09-03T06:07:23.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po=', 'Content-Length': '0'} 2022-09-03T06:07:23.272 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d4691e83310386e1-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.274 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.275 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.276 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.277 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.279 DEBUG:boto:Method: GET 2022-09-03T06:07:23.283 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.286 DEBUG:boto:Data: 2022-09-03T06:07:23.287 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.289 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.289 DEBUG:boto:Params: {} 2022-09-03T06:07:23.290 DEBUG:boto:Token: None 2022-09-03T06:07:23.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.297 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a833817afa496e9-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.298 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.301 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.302 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.303 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.304 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.305 DEBUG:boto:Method: GET 2022-09-03T06:07:23.306 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.307 DEBUG:boto:Data: 2022-09-03T06:07:23.308 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.310 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.311 DEBUG:boto:Params: {} 2022-09-03T06:07:23.312 DEBUG:boto:Token: None 2022-09-03T06:07:23.313 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.320 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000afee00d4a53bbc92-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.322 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.324 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:23.325 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.326 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.327 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.328 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.329 DEBUG:boto:Method: GET 2022-09-03T06:07:23.331 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.332 DEBUG:boto:Data: 2022-09-03T06:07:23.333 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.335 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.336 DEBUG:boto:Params: {} 2022-09-03T06:07:23.337 DEBUG:boto:Token: None 2022-09-03T06:07:23.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j+THS2k0ROf3w0G0OWAXt2H9Fpw= 2022-09-03T06:07:23.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j+THS2k0ROf3w0G0OWAXt2H9Fpw=', 'Content-Length': '0'} 2022-09-03T06:07:23.346 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008ea298f86f49bf2e-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.347 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.347 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.348 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.348 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.349 DEBUG:boto:Method: GET 2022-09-03T06:07:23.349 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.350 DEBUG:boto:Data: 2022-09-03T06:07:23.351 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.352 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.352 DEBUG:boto:Params: {} 2022-09-03T06:07:23.353 DEBUG:boto:Token: None 2022-09-03T06:07:23.353 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j+THS2k0ROf3w0G0OWAXt2H9Fpw= 2022-09-03T06:07:23.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j+THS2k0ROf3w0G0OWAXt2H9Fpw=', 'Content-Length': '0'} 2022-09-03T06:07:23.358 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009f39c16714ac4d49-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.359 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.359 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.360 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.360 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.361 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.361 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.362 DEBUG:boto:Data: 2022-09-03T06:07:23.363 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.364 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.364 DEBUG:boto:Params: {} 2022-09-03T06:07:23.365 DEBUG:boto:Token: None 2022-09-03T06:07:23.365 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k/0yzpej6dmFpdsWnLZ8UcR9VFI= 2022-09-03T06:07:23.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k/0yzpej6dmFpdsWnLZ8UcR9VFI=', 'Content-Length': '0'} 2022-09-03T06:07:23.372 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007045804484bbdcb4-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.373 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.374 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.374 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.375 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.376 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.376 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.377 DEBUG:boto:Data: 2022-09-03T06:07:23.377 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.378 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.378 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.379 DEBUG:boto:Params: {} 2022-09-03T06:07:23.379 DEBUG:boto:Token: None 2022-09-03T06:07:23.380 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k/0yzpej6dmFpdsWnLZ8UcR9VFI= 2022-09-03T06:07:23.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k/0yzpej6dmFpdsWnLZ8UcR9VFI=', 'Content-Length': '0'} 2022-09-03T06:07:23.384 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009d8e7f35f64bd4c3-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.384 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:23.385 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.386 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.386 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.387 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.387 DEBUG:boto:Method: GET 2022-09-03T06:07:23.388 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.388 DEBUG:boto:Data: 2022-09-03T06:07:23.389 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.390 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.391 DEBUG:boto:Params: {} 2022-09-03T06:07:23.392 DEBUG:boto:Token: None 2022-09-03T06:07:23.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j+THS2k0ROf3w0G0OWAXt2H9Fpw= 2022-09-03T06:07:23.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j+THS2k0ROf3w0G0OWAXt2H9Fpw=', 'Content-Length': '0'} 2022-09-03T06:07:23.396 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007b6e6eac30900a37-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.397 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.397 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.398 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.399 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.399 DEBUG:boto:Method: GET 2022-09-03T06:07:23.400 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.400 DEBUG:boto:Data: 2022-09-03T06:07:23.401 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.402 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.402 DEBUG:boto:Params: {} 2022-09-03T06:07:23.403 DEBUG:boto:Token: None 2022-09-03T06:07:23.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:23.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j+THS2k0ROf3w0G0OWAXt2H9Fpw= 2022-09-03T06:07:23.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j+THS2k0ROf3w0G0OWAXt2H9Fpw=', 'Content-Length': '0'} 2022-09-03T06:07:23.407 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005566a6fb37205565-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.450 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.451 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.452 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.452 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.453 DEBUG:boto:Method: GET 2022-09-03T06:07:23.454 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.454 DEBUG:boto:Data: 2022-09-03T06:07:23.455 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.455 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.457 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.460 DEBUG:boto:Params: {} 2022-09-03T06:07:23.465 DEBUG:boto:Token: None 2022-09-03T06:07:23.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.470 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000756f81ec01b912eb-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.471 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.472 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.473 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.473 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.474 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.474 DEBUG:boto:Method: GET 2022-09-03T06:07:23.475 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.475 DEBUG:boto:Data: 2022-09-03T06:07:23.476 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.477 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.477 DEBUG:boto:Params: {} 2022-09-03T06:07:23.478 DEBUG:boto:Token: None 2022-09-03T06:07:23.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.483 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001957b1de335ea176-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.484 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.485 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.485 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.486 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.486 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:23.487 DEBUG:boto:Data: 2022-09-03T06:07:23.487 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.488 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.489 DEBUG:boto:Params: {} 2022-09-03T06:07:23.489 DEBUG:boto:Token: None 2022-09-03T06:07:23.490 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj 2022-09-03T06:07:23.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g= 2022-09-03T06:07:23.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g=', 'Content-Length': '0'} 2022-09-03T06:07:23.495 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000095a0c739866e6824-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.496 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.497 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.497 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.498 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:23.498 DEBUG:boto:Data: 2022-09-03T06:07:23.499 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.501 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.502 DEBUG:boto:Params: {} 2022-09-03T06:07:23.503 DEBUG:boto:Token: None 2022-09-03T06:07:23.503 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj 2022-09-03T06:07:23.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g= 2022-09-03T06:07:23.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g=', 'Content-Length': '0'} 2022-09-03T06:07:23.508 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d73039e7e195e247-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.509 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:23.510 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.510 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.511 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.511 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.512 DEBUG:boto:Method: GET 2022-09-03T06:07:23.512 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.513 DEBUG:boto:Data: 2022-09-03T06:07:23.514 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.514 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.515 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.515 DEBUG:boto:Params: {} 2022-09-03T06:07:23.516 DEBUG:boto:Token: None 2022-09-03T06:07:23.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po= 2022-09-03T06:07:23.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po=', 'Content-Length': '0'} 2022-09-03T06:07:23.521 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000036e0c18a60028d1f-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.562 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.564 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.566 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.566 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.567 DEBUG:boto:Method: GET 2022-09-03T06:07:23.568 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.568 DEBUG:boto:Data: 2022-09-03T06:07:23.569 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.569 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.570 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.571 DEBUG:boto:Params: {} 2022-09-03T06:07:23.572 DEBUG:boto:Token: None 2022-09-03T06:07:23.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po= 2022-09-03T06:07:23.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po=', 'Content-Length': '0'} 2022-09-03T06:07:23.579 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009f34c82f07b49b14-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.580 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.582 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.583 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.584 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.585 DEBUG:boto:Method: GET 2022-09-03T06:07:23.586 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.587 DEBUG:boto:Data: 2022-09-03T06:07:23.587 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.588 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.589 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.590 DEBUG:boto:Params: {} 2022-09-03T06:07:23.591 DEBUG:boto:Token: None 2022-09-03T06:07:23.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.596 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000954167b69cfcc2e4-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.597 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.599 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.600 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.600 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.601 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.601 DEBUG:boto:Method: GET 2022-09-03T06:07:23.602 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.602 DEBUG:boto:Data: 2022-09-03T06:07:23.603 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.605 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.606 DEBUG:boto:Params: {} 2022-09-03T06:07:23.607 DEBUG:boto:Token: None 2022-09-03T06:07:23.607 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.612 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028a9094a409e3a64-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.654 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.655 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:23.656 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.656 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.657 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.658 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.659 DEBUG:boto:Method: GET 2022-09-03T06:07:23.660 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.661 DEBUG:boto:Data: 2022-09-03T06:07:23.661 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.663 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.663 DEBUG:boto:Params: {} 2022-09-03T06:07:23.664 DEBUG:boto:Token: None 2022-09-03T06:07:23.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BH9bfmr0hUrICM1IbN7EgI1EVao= 2022-09-03T06:07:23.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BH9bfmr0hUrICM1IbN7EgI1EVao=', 'Content-Length': '0'} 2022-09-03T06:07:23.669 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009bffe4bc8a06ff8b-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.671 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.672 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.672 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.673 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.674 DEBUG:boto:Method: GET 2022-09-03T06:07:23.676 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.677 DEBUG:boto:Data: 2022-09-03T06:07:23.678 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.679 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.680 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.681 DEBUG:boto:Params: {} 2022-09-03T06:07:23.682 DEBUG:boto:Token: None 2022-09-03T06:07:23.683 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BH9bfmr0hUrICM1IbN7EgI1EVao= 2022-09-03T06:07:23.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BH9bfmr0hUrICM1IbN7EgI1EVao=', 'Content-Length': '0'} 2022-09-03T06:07:23.689 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d8ab9383ee189ee4-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.690 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.691 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.692 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.693 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.694 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.694 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.696 DEBUG:boto:Data: 2022-09-03T06:07:23.696 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.697 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.697 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.698 DEBUG:boto:Params: {} 2022-09-03T06:07:23.699 DEBUG:boto:Token: None 2022-09-03T06:07:23.700 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8YhKzJBBW6dpAThmYCAcBE2xAMw= 2022-09-03T06:07:23.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8YhKzJBBW6dpAThmYCAcBE2xAMw=', 'Content-Length': '0'} 2022-09-03T06:07:23.703 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000590e380d04cc3cb3-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.704 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.704 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.705 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.705 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.707 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.708 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.709 DEBUG:boto:Data: 2022-09-03T06:07:23.710 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.714 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.714 DEBUG:boto:Params: {} 2022-09-03T06:07:23.715 DEBUG:boto:Token: None 2022-09-03T06:07:23.716 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8YhKzJBBW6dpAThmYCAcBE2xAMw= 2022-09-03T06:07:23.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8YhKzJBBW6dpAThmYCAcBE2xAMw=', 'Content-Length': '0'} 2022-09-03T06:07:23.729 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bf03f9c13ff7078e-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.731 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:23.732 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.733 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.734 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.735 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.736 DEBUG:boto:Method: GET 2022-09-03T06:07:23.737 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.738 DEBUG:boto:Data: 2022-09-03T06:07:23.739 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.740 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.741 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.742 DEBUG:boto:Params: {} 2022-09-03T06:07:23.744 DEBUG:boto:Token: None 2022-09-03T06:07:23.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BH9bfmr0hUrICM1IbN7EgI1EVao= 2022-09-03T06:07:23.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BH9bfmr0hUrICM1IbN7EgI1EVao=', 'Content-Length': '0'} 2022-09-03T06:07:23.750 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000066143c5c5cbe378a-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.751 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.752 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.753 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.754 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.755 DEBUG:boto:Method: GET 2022-09-03T06:07:23.756 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.757 DEBUG:boto:Data: 2022-09-03T06:07:23.758 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.760 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.761 DEBUG:boto:Params: {} 2022-09-03T06:07:23.763 DEBUG:boto:Token: None 2022-09-03T06:07:23.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:23.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BH9bfmr0hUrICM1IbN7EgI1EVao= 2022-09-03T06:07:23.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BH9bfmr0hUrICM1IbN7EgI1EVao=', 'Content-Length': '0'} 2022-09-03T06:07:23.770 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a2ee53370d7be736-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.772 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.773 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.774 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.775 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.776 DEBUG:boto:Method: GET 2022-09-03T06:07:23.777 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.778 DEBUG:boto:Data: 2022-09-03T06:07:23.779 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.781 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.782 DEBUG:boto:Params: {} 2022-09-03T06:07:23.785 DEBUG:boto:Token: None 2022-09-03T06:07:23.786 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.793 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f8704b9fc72c8d38-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.795 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.798 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.799 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.799 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.800 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.800 DEBUG:boto:Method: GET 2022-09-03T06:07:23.801 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.802 DEBUG:boto:Data: 2022-09-03T06:07:23.803 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.804 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.805 DEBUG:boto:Params: {} 2022-09-03T06:07:23.806 DEBUG:boto:Token: None 2022-09-03T06:07:23.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.811 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bdfe83c1a9012c27-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.812 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.814 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.815 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.816 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.817 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:23.818 DEBUG:boto:Data: 2022-09-03T06:07:23.818 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.820 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.821 DEBUG:boto:Params: {} 2022-09-03T06:07:23.823 DEBUG:boto:Token: None 2022-09-03T06:07:23.824 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj 2022-09-03T06:07:23.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g= 2022-09-03T06:07:23.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g=', 'Content-Length': '0'} 2022-09-03T06:07:23.829 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000227f26cf899e0d29-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.830 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.831 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.832 DEBUG:boto:Method: HEAD 2022-09-03T06:07:23.832 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:23.833 DEBUG:boto:Data: 2022-09-03T06:07:23.834 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.835 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.836 DEBUG:boto:Params: {} 2022-09-03T06:07:23.837 DEBUG:boto:Token: None 2022-09-03T06:07:23.838 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj 2022-09-03T06:07:23.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g= 2022-09-03T06:07:23.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ruM4oFOufQLUgo8VmyYUAjUnL4g=', 'Content-Length': '0'} 2022-09-03T06:07:23.843 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000077f826fb29afba80-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.845 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:23.845 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.846 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.847 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.847 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.848 DEBUG:boto:Method: GET 2022-09-03T06:07:23.849 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.849 DEBUG:boto:Data: 2022-09-03T06:07:23.850 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.852 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.853 DEBUG:boto:Params: {} 2022-09-03T06:07:23.854 DEBUG:boto:Token: None 2022-09-03T06:07:23.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po= 2022-09-03T06:07:23.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po=', 'Content-Length': '0'} 2022-09-03T06:07:23.862 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f42de4a61148d695-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.864 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.865 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.866 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.867 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.868 DEBUG:boto:Method: GET 2022-09-03T06:07:23.869 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.870 DEBUG:boto:Data: 2022-09-03T06:07:23.871 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.873 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.874 DEBUG:boto:Params: {} 2022-09-03T06:07:23.875 DEBUG:boto:Token: None 2022-09-03T06:07:23.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:23.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po= 2022-09-03T06:07:23.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4yZHMcecXEaaLNlyD7o0k43i9Po=', 'Content-Length': '0'} 2022-09-03T06:07:23.881 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000622e5e961bcfc64f-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.922 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.923 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.923 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.924 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.924 DEBUG:boto:Method: GET 2022-09-03T06:07:23.925 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.926 DEBUG:boto:Data: 2022-09-03T06:07:23.927 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.928 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.928 DEBUG:boto:Params: {} 2022-09-03T06:07:23.929 DEBUG:boto:Token: None 2022-09-03T06:07:23.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.934 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f40712b8a8a39f46-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.935 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.936 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:23.936 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:23.937 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.937 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:23.938 DEBUG:boto:Method: GET 2022-09-03T06:07:23.939 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:23.939 DEBUG:boto:Data: 2022-09-03T06:07:23.940 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.941 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:23.942 DEBUG:boto:Port: 8000 2022-09-03T06:07:23.942 DEBUG:boto:Params: {} 2022-09-03T06:07:23.943 DEBUG:boto:Token: None 2022-09-03T06:07:23.944 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/ 2022-09-03T06:07:23.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA= 2022-09-03T06:07:23.945 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HxFmnwjm1tGLNInm58VRVyES5bA=', 'Content-Length': '0'} 2022-09-03T06:07:23.948 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a9f01792fb806fd-006312ef1b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:23.948 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:23.950 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:23.950 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:23.951 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:23.952 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:23.952 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:23.953 DEBUG:boto:Method: GET 2022-09-03T06:07:23.954 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:23.955 DEBUG:boto:Data: 2022-09-03T06:07:23.956 DEBUG:boto:Headers: {} 2022-09-03T06:07:23.956 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:23.957 DEBUG:boto:Port: 8001 2022-09-03T06:07:23.958 DEBUG:boto:Params: {} 2022-09-03T06:07:23.959 DEBUG:boto:Token: None 2022-09-03T06:07:23.960 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:23 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:23.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xVItIU6hUUOQX9Jd2MRrVMnCbkI= 2022-09-03T06:07:23.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xVItIU6hUUOQX9Jd2MRrVMnCbkI=', 'Content-Length': '0'} 2022-09-03T06:07:23.965 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000062affddd95c967ff-006312ef1b-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.010 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.011 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.011 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.012 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.013 DEBUG:boto:Method: GET 2022-09-03T06:07:24.013 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.014 DEBUG:boto:Data: 2022-09-03T06:07:24.015 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.018 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.019 DEBUG:boto:Params: {} 2022-09-03T06:07:24.020 DEBUG:boto:Token: None 2022-09-03T06:07:24.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NXf26U4O/4fEoTSoyP40YP9xMSE= 2022-09-03T06:07:24.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NXf26U4O/4fEoTSoyP40YP9xMSE=', 'Content-Length': '0'} 2022-09-03T06:07:24.027 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a6b6032bd43ee543-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.029 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.030 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.031 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.031 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.032 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.033 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.034 DEBUG:boto:Data: 2022-09-03T06:07:24.034 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.036 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.036 DEBUG:boto:Params: {} 2022-09-03T06:07:24.037 DEBUG:boto:Token: None 2022-09-03T06:07:24.037 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BkgNbD/v68ZqbG04WQXWWDozZtg= 2022-09-03T06:07:24.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BkgNbD/v68ZqbG04WQXWWDozZtg=', 'Content-Length': '0'} 2022-09-03T06:07:24.041 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000033df43fd3e551848-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.041 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.042 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.042 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.043 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.043 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.044 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.045 DEBUG:boto:Data: 2022-09-03T06:07:24.045 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.046 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.046 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.047 DEBUG:boto:Params: {} 2022-09-03T06:07:24.047 DEBUG:boto:Token: None 2022-09-03T06:07:24.048 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BkgNbD/v68ZqbG04WQXWWDozZtg= 2022-09-03T06:07:24.050 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BkgNbD/v68ZqbG04WQXWWDozZtg=', 'Content-Length': '0'} 2022-09-03T06:07:24.052 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000043c8b5b74d027805-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.053 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.053 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.054 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.055 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.055 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.056 DEBUG:boto:Method: GET 2022-09-03T06:07:24.057 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.057 DEBUG:boto:Data: 2022-09-03T06:07:24.058 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.060 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.061 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.062 DEBUG:boto:Params: {} 2022-09-03T06:07:24.063 DEBUG:boto:Token: None 2022-09-03T06:07:24.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NXf26U4O/4fEoTSoyP40YP9xMSE= 2022-09-03T06:07:24.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NXf26U4O/4fEoTSoyP40YP9xMSE=', 'Content-Length': '0'} 2022-09-03T06:07:24.069 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004237e593bea421a7-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.070 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.071 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.071 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.072 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.072 DEBUG:boto:Method: GET 2022-09-03T06:07:24.073 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.073 DEBUG:boto:Data: 2022-09-03T06:07:24.074 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.075 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.076 DEBUG:boto:Params: {} 2022-09-03T06:07:24.077 DEBUG:boto:Token: None 2022-09-03T06:07:24.077 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:24.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NXf26U4O/4fEoTSoyP40YP9xMSE= 2022-09-03T06:07:24.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NXf26U4O/4fEoTSoyP40YP9xMSE=', 'Content-Length': '0'} 2022-09-03T06:07:24.081 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f63e83afab78ef98-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.081 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.083 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.083 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.084 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.084 DEBUG:boto:Method: GET 2022-09-03T06:07:24.085 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.086 DEBUG:boto:Data: 2022-09-03T06:07:24.086 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.087 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.087 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.088 DEBUG:boto:Params: {} 2022-09-03T06:07:24.088 DEBUG:boto:Token: None 2022-09-03T06:07:24.089 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.094 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e325fc7819301dd1-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.095 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.096 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.097 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.098 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.099 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.100 DEBUG:boto:Method: GET 2022-09-03T06:07:24.101 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.102 DEBUG:boto:Data: 2022-09-03T06:07:24.103 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.104 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.105 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.106 DEBUG:boto:Params: {} 2022-09-03T06:07:24.107 DEBUG:boto:Token: None 2022-09-03T06:07:24.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.113 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e5fd8a2d392761c-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.114 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.115 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.116 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.117 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.117 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:24.118 DEBUG:boto:Data: 2022-09-03T06:07:24.119 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.120 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.120 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.121 DEBUG:boto:Params: {} 2022-09-03T06:07:24.122 DEBUG:boto:Token: None 2022-09-03T06:07:24.123 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj 2022-09-03T06:07:24.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA= 2022-09-03T06:07:24.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA=', 'Content-Length': '0'} 2022-09-03T06:07:24.129 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001e5aa0cc267b8166-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.130 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.131 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.132 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.133 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:24.134 DEBUG:boto:Data: 2022-09-03T06:07:24.135 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.136 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.136 DEBUG:boto:Params: {} 2022-09-03T06:07:24.137 DEBUG:boto:Token: None 2022-09-03T06:07:24.137 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj 2022-09-03T06:07:24.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA= 2022-09-03T06:07:24.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA=', 'Content-Length': '0'} 2022-09-03T06:07:24.144 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000090b1dbecd6f5f7d-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.145 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.146 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.146 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.147 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.147 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.148 DEBUG:boto:Method: GET 2022-09-03T06:07:24.148 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.149 DEBUG:boto:Data: 2022-09-03T06:07:24.150 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.151 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.152 DEBUG:boto:Params: {} 2022-09-03T06:07:24.153 DEBUG:boto:Token: None 2022-09-03T06:07:24.153 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg= 2022-09-03T06:07:24.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg=', 'Content-Length': '0'} 2022-09-03T06:07:24.158 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000095d689829863ec75-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.159 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.159 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.160 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.160 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.161 DEBUG:boto:Method: GET 2022-09-03T06:07:24.161 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.162 DEBUG:boto:Data: 2022-09-03T06:07:24.163 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.164 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.164 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.165 DEBUG:boto:Params: {} 2022-09-03T06:07:24.165 DEBUG:boto:Token: None 2022-09-03T06:07:24.166 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg= 2022-09-03T06:07:24.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg=', 'Content-Length': '0'} 2022-09-03T06:07:24.170 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000818f9bbaf035c3d8-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.219 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.220 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.221 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.222 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.222 DEBUG:boto:Method: GET 2022-09-03T06:07:24.223 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.223 DEBUG:boto:Data: 2022-09-03T06:07:24.224 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.224 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.225 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.225 DEBUG:boto:Params: {} 2022-09-03T06:07:24.226 DEBUG:boto:Token: None 2022-09-03T06:07:24.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.235 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000049c0669fc6229ad9-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.236 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.237 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.238 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.239 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.239 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.240 DEBUG:boto:Method: GET 2022-09-03T06:07:24.241 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.242 DEBUG:boto:Data: 2022-09-03T06:07:24.243 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.244 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.245 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.246 DEBUG:boto:Params: {} 2022-09-03T06:07:24.248 DEBUG:boto:Token: None 2022-09-03T06:07:24.250 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.255 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e558d474c073067-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.256 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.258 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.259 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.260 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.262 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.263 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.264 DEBUG:boto:Method: GET 2022-09-03T06:07:24.265 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.266 DEBUG:boto:Data: 2022-09-03T06:07:24.267 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.270 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.271 DEBUG:boto:Params: {} 2022-09-03T06:07:24.272 DEBUG:boto:Token: None 2022-09-03T06:07:24.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pKiG2PVuc0ARWSso+UDYyLfDaa0= 2022-09-03T06:07:24.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pKiG2PVuc0ARWSso+UDYyLfDaa0=', 'Content-Length': '0'} 2022-09-03T06:07:24.285 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c16e4d12db5615be-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.286 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.287 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.289 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.293 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.295 DEBUG:boto:Method: GET 2022-09-03T06:07:24.297 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.298 DEBUG:boto:Data: 2022-09-03T06:07:24.299 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.300 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.301 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.302 DEBUG:boto:Params: {} 2022-09-03T06:07:24.303 DEBUG:boto:Token: None 2022-09-03T06:07:24.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pKiG2PVuc0ARWSso+UDYyLfDaa0= 2022-09-03T06:07:24.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pKiG2PVuc0ARWSso+UDYyLfDaa0=', 'Content-Length': '0'} 2022-09-03T06:07:24.308 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b60f563e7993b180-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.309 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.309 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.310 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.310 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.311 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.311 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.312 DEBUG:boto:Data: 2022-09-03T06:07:24.312 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.313 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.314 DEBUG:boto:Params: {} 2022-09-03T06:07:24.314 DEBUG:boto:Token: None 2022-09-03T06:07:24.315 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RobYEfN32iBw9MzouCaaM1ZIpbA= 2022-09-03T06:07:24.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RobYEfN32iBw9MzouCaaM1ZIpbA=', 'Content-Length': '0'} 2022-09-03T06:07:24.318 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a2fd250c27248dd7-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.319 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.320 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.320 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.321 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.321 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.322 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.322 DEBUG:boto:Data: 2022-09-03T06:07:24.323 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.323 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.324 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.324 DEBUG:boto:Params: {} 2022-09-03T06:07:24.325 DEBUG:boto:Token: None 2022-09-03T06:07:24.325 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RobYEfN32iBw9MzouCaaM1ZIpbA= 2022-09-03T06:07:24.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RobYEfN32iBw9MzouCaaM1ZIpbA=', 'Content-Length': '0'} 2022-09-03T06:07:24.328 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004367925a7bd5e965-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.329 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.330 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.330 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.331 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.331 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.332 DEBUG:boto:Method: GET 2022-09-03T06:07:24.332 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.333 DEBUG:boto:Data: 2022-09-03T06:07:24.333 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.334 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.335 DEBUG:boto:Params: {} 2022-09-03T06:07:24.336 DEBUG:boto:Token: None 2022-09-03T06:07:24.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pKiG2PVuc0ARWSso+UDYyLfDaa0= 2022-09-03T06:07:24.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pKiG2PVuc0ARWSso+UDYyLfDaa0=', 'Content-Length': '0'} 2022-09-03T06:07:24.340 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000043d6084c221e9625-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.383 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.384 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.385 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.385 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.386 DEBUG:boto:Method: GET 2022-09-03T06:07:24.386 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.387 DEBUG:boto:Data: 2022-09-03T06:07:24.388 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.389 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.389 DEBUG:boto:Params: {} 2022-09-03T06:07:24.390 DEBUG:boto:Token: None 2022-09-03T06:07:24.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:24.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pKiG2PVuc0ARWSso+UDYyLfDaa0= 2022-09-03T06:07:24.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pKiG2PVuc0ARWSso+UDYyLfDaa0=', 'Content-Length': '0'} 2022-09-03T06:07:24.395 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ee11f246002da091-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.396 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.396 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.397 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.397 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.398 DEBUG:boto:Method: GET 2022-09-03T06:07:24.398 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.399 DEBUG:boto:Data: 2022-09-03T06:07:24.400 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.401 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.401 DEBUG:boto:Params: {} 2022-09-03T06:07:24.402 DEBUG:boto:Token: None 2022-09-03T06:07:24.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.407 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009032774481a0e6c0-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.408 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.409 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.409 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.410 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.410 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.411 DEBUG:boto:Method: GET 2022-09-03T06:07:24.411 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.412 DEBUG:boto:Data: 2022-09-03T06:07:24.413 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.413 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.414 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.415 DEBUG:boto:Params: {} 2022-09-03T06:07:24.415 DEBUG:boto:Token: None 2022-09-03T06:07:24.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.420 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000004061137c490a651-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.421 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.421 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.422 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.422 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.423 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:24.423 DEBUG:boto:Data: 2022-09-03T06:07:24.424 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.425 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.425 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.426 DEBUG:boto:Params: {} 2022-09-03T06:07:24.426 DEBUG:boto:Token: None 2022-09-03T06:07:24.427 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj 2022-09-03T06:07:24.427 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA= 2022-09-03T06:07:24.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA=', 'Content-Length': '0'} 2022-09-03T06:07:24.432 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000958bbdd1bc8b01e-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.433 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.433 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.434 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.435 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:24.435 DEBUG:boto:Data: 2022-09-03T06:07:24.436 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.436 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.437 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.437 DEBUG:boto:Params: {} 2022-09-03T06:07:24.438 DEBUG:boto:Token: None 2022-09-03T06:07:24.438 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj 2022-09-03T06:07:24.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA= 2022-09-03T06:07:24.440 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA=', 'Content-Length': '0'} 2022-09-03T06:07:24.443 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000043bbc2f5290b05e4-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.444 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.444 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.445 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.445 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.446 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.446 DEBUG:boto:Method: GET 2022-09-03T06:07:24.447 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.447 DEBUG:boto:Data: 2022-09-03T06:07:24.448 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.449 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.449 DEBUG:boto:Params: {} 2022-09-03T06:07:24.450 DEBUG:boto:Token: None 2022-09-03T06:07:24.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg= 2022-09-03T06:07:24.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg=', 'Content-Length': '0'} 2022-09-03T06:07:24.454 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000880059037da8dc8e-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.495 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.495 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.496 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.496 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.497 DEBUG:boto:Method: GET 2022-09-03T06:07:24.498 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.498 DEBUG:boto:Data: 2022-09-03T06:07:24.499 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.500 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.501 DEBUG:boto:Params: {} 2022-09-03T06:07:24.501 DEBUG:boto:Token: None 2022-09-03T06:07:24.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg= 2022-09-03T06:07:24.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg=', 'Content-Length': '0'} 2022-09-03T06:07:24.506 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cfff0f16a47645d7-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.507 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.507 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.508 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.508 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.509 DEBUG:boto:Method: GET 2022-09-03T06:07:24.510 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.510 DEBUG:boto:Data: 2022-09-03T06:07:24.511 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.511 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.512 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.512 DEBUG:boto:Params: {} 2022-09-03T06:07:24.513 DEBUG:boto:Token: None 2022-09-03T06:07:24.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.518 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000083b31efb3aa3b82f-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.518 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.520 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.520 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.521 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.521 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.522 DEBUG:boto:Method: GET 2022-09-03T06:07:24.522 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.523 DEBUG:boto:Data: 2022-09-03T06:07:24.523 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.525 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.525 DEBUG:boto:Params: {} 2022-09-03T06:07:24.526 DEBUG:boto:Token: None 2022-09-03T06:07:24.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.532 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c8b26065cae72c30-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.580 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.582 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.583 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.584 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.585 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.586 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.587 DEBUG:boto:Method: GET 2022-09-03T06:07:24.588 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.589 DEBUG:boto:Data: 2022-09-03T06:07:24.590 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.592 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.593 DEBUG:boto:Params: {} 2022-09-03T06:07:24.594 DEBUG:boto:Token: None 2022-09-03T06:07:24.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ynJ6BvN/tXq/RRXFNHJHVudjj1U= 2022-09-03T06:07:24.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ynJ6BvN/tXq/RRXFNHJHVudjj1U=', 'Content-Length': '0'} 2022-09-03T06:07:24.600 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000034e17f572d594f55-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.602 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.602 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.603 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.603 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.604 DEBUG:boto:Method: GET 2022-09-03T06:07:24.605 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.605 DEBUG:boto:Data: 2022-09-03T06:07:24.606 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.607 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.607 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.608 DEBUG:boto:Params: {} 2022-09-03T06:07:24.608 DEBUG:boto:Token: None 2022-09-03T06:07:24.609 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.610 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ynJ6BvN/tXq/RRXFNHJHVudjj1U= 2022-09-03T06:07:24.610 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ynJ6BvN/tXq/RRXFNHJHVudjj1U=', 'Content-Length': '0'} 2022-09-03T06:07:24.612 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bbd8ab072c29148a-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.614 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.614 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.615 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.615 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.616 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.617 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.617 DEBUG:boto:Data: 2022-09-03T06:07:24.618 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.619 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.619 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.620 DEBUG:boto:Params: {} 2022-09-03T06:07:24.621 DEBUG:boto:Token: None 2022-09-03T06:07:24.622 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fv0Hrd0rcFwlJJSKoGiTSIxDeGI= 2022-09-03T06:07:24.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fv0Hrd0rcFwlJJSKoGiTSIxDeGI=', 'Content-Length': '0'} 2022-09-03T06:07:24.626 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dc228d48d04ca464-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.627 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.628 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.628 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.629 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.629 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.630 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.630 DEBUG:boto:Data: 2022-09-03T06:07:24.631 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.632 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.632 DEBUG:boto:Params: {} 2022-09-03T06:07:24.633 DEBUG:boto:Token: None 2022-09-03T06:07:24.633 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fv0Hrd0rcFwlJJSKoGiTSIxDeGI= 2022-09-03T06:07:24.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fv0Hrd0rcFwlJJSKoGiTSIxDeGI=', 'Content-Length': '0'} 2022-09-03T06:07:24.637 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d73b47030c7c7d20-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.637 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.638 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.638 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.639 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.640 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.640 DEBUG:boto:Method: GET 2022-09-03T06:07:24.641 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.641 DEBUG:boto:Data: 2022-09-03T06:07:24.642 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.643 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.643 DEBUG:boto:Params: {} 2022-09-03T06:07:24.644 DEBUG:boto:Token: None 2022-09-03T06:07:24.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ynJ6BvN/tXq/RRXFNHJHVudjj1U= 2022-09-03T06:07:24.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ynJ6BvN/tXq/RRXFNHJHVudjj1U=', 'Content-Length': '0'} 2022-09-03T06:07:24.648 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008249444a2721ec92-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.650 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.650 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.651 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.651 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.652 DEBUG:boto:Method: GET 2022-09-03T06:07:24.652 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.653 DEBUG:boto:Data: 2022-09-03T06:07:24.653 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.654 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.654 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.655 DEBUG:boto:Params: {} 2022-09-03T06:07:24.655 DEBUG:boto:Token: None 2022-09-03T06:07:24.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:24.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ynJ6BvN/tXq/RRXFNHJHVudjj1U= 2022-09-03T06:07:24.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ynJ6BvN/tXq/RRXFNHJHVudjj1U=', 'Content-Length': '0'} 2022-09-03T06:07:24.660 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000037224a757c046250-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.661 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.663 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.664 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.665 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.665 DEBUG:boto:Method: GET 2022-09-03T06:07:24.666 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.667 DEBUG:boto:Data: 2022-09-03T06:07:24.668 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.669 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.670 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.671 DEBUG:boto:Params: {} 2022-09-03T06:07:24.671 DEBUG:boto:Token: None 2022-09-03T06:07:24.672 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.678 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c222829446ced266-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.679 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.680 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.680 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.681 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.681 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.682 DEBUG:boto:Method: GET 2022-09-03T06:07:24.683 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.684 DEBUG:boto:Data: 2022-09-03T06:07:24.684 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.685 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.685 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.686 DEBUG:boto:Params: {} 2022-09-03T06:07:24.686 DEBUG:boto:Token: None 2022-09-03T06:07:24.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.698 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb6e62048a14666d-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.699 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.700 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.701 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.702 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.702 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:24.703 DEBUG:boto:Data: 2022-09-03T06:07:24.704 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.705 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.706 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.706 DEBUG:boto:Params: {} 2022-09-03T06:07:24.708 DEBUG:boto:Token: None 2022-09-03T06:07:24.709 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj 2022-09-03T06:07:24.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA= 2022-09-03T06:07:24.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA=', 'Content-Length': '0'} 2022-09-03T06:07:24.714 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a3b17bd523ae7faa-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.715 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.715 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.716 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.716 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:24.717 DEBUG:boto:Data: 2022-09-03T06:07:24.717 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.718 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.719 DEBUG:boto:Params: {} 2022-09-03T06:07:24.719 DEBUG:boto:Token: None 2022-09-03T06:07:24.720 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj 2022-09-03T06:07:24.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA= 2022-09-03T06:07:24.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA=', 'Content-Length': '0'} 2022-09-03T06:07:24.725 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004a9c9d1b35ab8dee-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.726 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.726 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.727 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.728 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.728 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.729 DEBUG:boto:Method: GET 2022-09-03T06:07:24.729 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.730 DEBUG:boto:Data: 2022-09-03T06:07:24.731 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.732 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.732 DEBUG:boto:Params: {} 2022-09-03T06:07:24.733 DEBUG:boto:Token: None 2022-09-03T06:07:24.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg= 2022-09-03T06:07:24.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg=', 'Content-Length': '0'} 2022-09-03T06:07:24.739 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bc06ab72c3ac364c-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.782 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.783 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.784 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.784 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.785 DEBUG:boto:Method: GET 2022-09-03T06:07:24.785 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.786 DEBUG:boto:Data: 2022-09-03T06:07:24.786 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.788 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.788 DEBUG:boto:Params: {} 2022-09-03T06:07:24.789 DEBUG:boto:Token: None 2022-09-03T06:07:24.790 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:24.791 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg= 2022-09-03T06:07:24.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10cIbFsTsPOW6+aQyRlyBPKpzOg=', 'Content-Length': '0'} 2022-09-03T06:07:24.793 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000467f2960f2efe7f2-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.794 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.795 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.795 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.796 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.798 DEBUG:boto:Method: GET 2022-09-03T06:07:24.799 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.800 DEBUG:boto:Data: 2022-09-03T06:07:24.801 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.802 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.803 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.803 DEBUG:boto:Params: {} 2022-09-03T06:07:24.804 DEBUG:boto:Token: None 2022-09-03T06:07:24.804 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.810 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000913f9d7589f904fe-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.811 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.812 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.813 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.815 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.816 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.817 DEBUG:boto:Method: GET 2022-09-03T06:07:24.818 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.818 DEBUG:boto:Data: 2022-09-03T06:07:24.819 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.820 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.821 DEBUG:boto:Params: {} 2022-09-03T06:07:24.821 DEBUG:boto:Token: None 2022-09-03T06:07:24.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.826 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001cbb2ddf1695003a-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.827 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.828 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.828 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.829 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.830 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.830 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.831 DEBUG:boto:Method: GET 2022-09-03T06:07:24.831 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.832 DEBUG:boto:Data: 2022-09-03T06:07:24.833 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.833 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.834 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.835 DEBUG:boto:Params: {} 2022-09-03T06:07:24.835 DEBUG:boto:Token: None 2022-09-03T06:07:24.836 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oPnYac0u36/IV3JEKVjvjJlfW5Y= 2022-09-03T06:07:24.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oPnYac0u36/IV3JEKVjvjJlfW5Y=', 'Content-Length': '0'} 2022-09-03T06:07:24.840 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000d222e8949b1ddb8-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.841 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.842 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.842 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.843 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.843 DEBUG:boto:Method: GET 2022-09-03T06:07:24.844 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.844 DEBUG:boto:Data: 2022-09-03T06:07:24.845 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.845 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.846 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.847 DEBUG:boto:Params: {} 2022-09-03T06:07:24.847 DEBUG:boto:Token: None 2022-09-03T06:07:24.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oPnYac0u36/IV3JEKVjvjJlfW5Y= 2022-09-03T06:07:24.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oPnYac0u36/IV3JEKVjvjJlfW5Y=', 'Content-Length': '0'} 2022-09-03T06:07:24.851 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000002d52c73675df6cb-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.898 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.899 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.900 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.900 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.901 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.902 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.902 DEBUG:boto:Data: 2022-09-03T06:07:24.903 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.903 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.904 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.905 DEBUG:boto:Params: {} 2022-09-03T06:07:24.905 DEBUG:boto:Token: None 2022-09-03T06:07:24.906 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vmEyFT5DHbz3rlARb1LtoORhfMI= 2022-09-03T06:07:24.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vmEyFT5DHbz3rlARb1LtoORhfMI=', 'Content-Length': '0'} 2022-09-03T06:07:24.910 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b17d393ce27f85e1-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.913 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.913 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.914 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.915 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.915 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.916 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.916 DEBUG:boto:Data: 2022-09-03T06:07:24.917 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.917 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.918 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.918 DEBUG:boto:Params: {} 2022-09-03T06:07:24.919 DEBUG:boto:Token: None 2022-09-03T06:07:24.920 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.920 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vmEyFT5DHbz3rlARb1LtoORhfMI= 2022-09-03T06:07:24.921 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vmEyFT5DHbz3rlARb1LtoORhfMI=', 'Content-Length': '0'} 2022-09-03T06:07:24.923 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c7958e6c391fcc6d-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.924 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:24.925 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.925 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.926 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.926 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.927 DEBUG:boto:Method: GET 2022-09-03T06:07:24.927 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.928 DEBUG:boto:Data: 2022-09-03T06:07:24.929 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.929 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.930 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.930 DEBUG:boto:Params: {} 2022-09-03T06:07:24.931 DEBUG:boto:Token: None 2022-09-03T06:07:24.932 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oPnYac0u36/IV3JEKVjvjJlfW5Y= 2022-09-03T06:07:24.933 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oPnYac0u36/IV3JEKVjvjJlfW5Y=', 'Content-Length': '0'} 2022-09-03T06:07:24.935 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000898fe630e665f5cf-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.936 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.936 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.937 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.937 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.938 DEBUG:boto:Method: GET 2022-09-03T06:07:24.939 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.939 DEBUG:boto:Data: 2022-09-03T06:07:24.940 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.940 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.941 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.941 DEBUG:boto:Params: {} 2022-09-03T06:07:24.942 DEBUG:boto:Token: None 2022-09-03T06:07:24.943 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:24.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oPnYac0u36/IV3JEKVjvjJlfW5Y= 2022-09-03T06:07:24.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oPnYac0u36/IV3JEKVjvjJlfW5Y=', 'Content-Length': '0'} 2022-09-03T06:07:24.947 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ed658abf955ddf7f-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.948 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.948 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.949 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.949 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.950 DEBUG:boto:Method: GET 2022-09-03T06:07:24.951 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.951 DEBUG:boto:Data: 2022-09-03T06:07:24.952 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.952 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.953 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.953 DEBUG:boto:Params: {} 2022-09-03T06:07:24.954 DEBUG:boto:Token: None 2022-09-03T06:07:24.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.959 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eadc44341a961612-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.960 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.961 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:24.962 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:24.962 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.963 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:24.963 DEBUG:boto:Method: GET 2022-09-03T06:07:24.964 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:24.965 DEBUG:boto:Data: 2022-09-03T06:07:24.966 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.967 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.967 DEBUG:boto:Params: {} 2022-09-03T06:07:24.968 DEBUG:boto:Token: None 2022-09-03T06:07:24.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/ 2022-09-03T06:07:24.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08= 2022-09-03T06:07:24.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+MxjI8KzmlQyhTLZ9zNRM9LxW08=', 'Content-Length': '0'} 2022-09-03T06:07:24.974 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000039e01af7e8957259-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.975 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:24.976 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.976 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.977 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.978 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:24.979 DEBUG:boto:Data: 2022-09-03T06:07:24.979 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:24.981 DEBUG:boto:Port: 8001 2022-09-03T06:07:24.981 DEBUG:boto:Params: {} 2022-09-03T06:07:24.982 DEBUG:boto:Token: None 2022-09-03T06:07:24.983 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj 2022-09-03T06:07:24.984 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA= 2022-09-03T06:07:24.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA=', 'Content-Length': '0'} 2022-09-03T06:07:24.988 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c101af53a737bf19-006312ef1c-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:24.989 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:24.989 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:24.990 DEBUG:boto:Method: HEAD 2022-09-03T06:07:24.991 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:24.991 DEBUG:boto:Data: 2022-09-03T06:07:24.992 DEBUG:boto:Headers: {} 2022-09-03T06:07:24.992 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:24.993 DEBUG:boto:Port: 8000 2022-09-03T06:07:24.993 DEBUG:boto:Params: {} 2022-09-03T06:07:24.994 DEBUG:boto:Token: None 2022-09-03T06:07:24.995 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:24 GMT /avysmw-19/obj 2022-09-03T06:07:24.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA= 2022-09-03T06:07:24.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onhQQrgh5WNQlJ7EDzlqq79kQZA=', 'Content-Length': '0'} 2022-09-03T06:07:25.000 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fd1ae6caede46d6f-006312ef1c-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.001 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.002 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.003 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.005 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.006 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.006 DEBUG:boto:Method: GET 2022-09-03T06:07:25.007 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.008 DEBUG:boto:Data: 2022-09-03T06:07:25.009 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.010 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.011 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.012 DEBUG:boto:Params: {} 2022-09-03T06:07:25.013 DEBUG:boto:Token: None 2022-09-03T06:07:25.014 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.016 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM= 2022-09-03T06:07:25.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM=', 'Content-Length': '0'} 2022-09-03T06:07:25.020 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003f521dd0d2ff55a9-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.064 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.065 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.066 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.066 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.067 DEBUG:boto:Method: GET 2022-09-03T06:07:25.068 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.068 DEBUG:boto:Data: 2022-09-03T06:07:25.069 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.069 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.070 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.071 DEBUG:boto:Params: {} 2022-09-03T06:07:25.071 DEBUG:boto:Token: None 2022-09-03T06:07:25.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM= 2022-09-03T06:07:25.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM=', 'Content-Length': '0'} 2022-09-03T06:07:25.075 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000074bc081d76833102-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.076 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.077 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.077 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.078 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.078 DEBUG:boto:Method: GET 2022-09-03T06:07:25.079 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.079 DEBUG:boto:Data: 2022-09-03T06:07:25.080 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.080 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.081 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.081 DEBUG:boto:Params: {} 2022-09-03T06:07:25.082 DEBUG:boto:Token: None 2022-09-03T06:07:25.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.087 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b8553cefe6d537a4-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.088 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.088 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.089 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.090 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.090 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.091 DEBUG:boto:Method: GET 2022-09-03T06:07:25.091 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.092 DEBUG:boto:Data: 2022-09-03T06:07:25.092 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.094 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.094 DEBUG:boto:Params: {} 2022-09-03T06:07:25.095 DEBUG:boto:Token: None 2022-09-03T06:07:25.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.100 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d95e7123a9358fdc-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.101 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.102 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.103 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.104 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.105 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.107 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.107 DEBUG:boto:Method: GET 2022-09-03T06:07:25.109 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.110 DEBUG:boto:Data: 2022-09-03T06:07:25.110 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.112 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.113 DEBUG:boto:Params: {} 2022-09-03T06:07:25.114 DEBUG:boto:Token: None 2022-09-03T06:07:25.115 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8AZpDGQglvOVX5Wj9BOMERkWOqk= 2022-09-03T06:07:25.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8AZpDGQglvOVX5Wj9BOMERkWOqk=', 'Content-Length': '0'} 2022-09-03T06:07:25.119 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000db7e01116cfbb530-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.120 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.121 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.122 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.124 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.124 DEBUG:boto:Method: GET 2022-09-03T06:07:25.125 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.126 DEBUG:boto:Data: 2022-09-03T06:07:25.127 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.129 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.130 DEBUG:boto:Params: {} 2022-09-03T06:07:25.132 DEBUG:boto:Token: None 2022-09-03T06:07:25.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8AZpDGQglvOVX5Wj9BOMERkWOqk= 2022-09-03T06:07:25.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8AZpDGQglvOVX5Wj9BOMERkWOqk=', 'Content-Length': '0'} 2022-09-03T06:07:25.137 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b9e7eeb26c3322fa-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.138 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.139 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.140 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.141 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.142 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.142 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.143 DEBUG:boto:Data: 2022-09-03T06:07:25.144 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.146 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.147 DEBUG:boto:Params: {} 2022-09-03T06:07:25.148 DEBUG:boto:Token: None 2022-09-03T06:07:25.149 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SUdDawIvfpII8ie34W4DvO68qfM= 2022-09-03T06:07:25.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SUdDawIvfpII8ie34W4DvO68qfM=', 'Content-Length': '0'} 2022-09-03T06:07:25.153 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007d5c0889d4fadeb8-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.154 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.155 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.155 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.156 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.156 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.157 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.157 DEBUG:boto:Data: 2022-09-03T06:07:25.158 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.159 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.160 DEBUG:boto:Params: {} 2022-09-03T06:07:25.160 DEBUG:boto:Token: None 2022-09-03T06:07:25.161 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SUdDawIvfpII8ie34W4DvO68qfM= 2022-09-03T06:07:25.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SUdDawIvfpII8ie34W4DvO68qfM=', 'Content-Length': '0'} 2022-09-03T06:07:25.164 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000b4038a81d8ea5c0-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.165 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.166 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.167 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.168 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.169 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.170 DEBUG:boto:Method: GET 2022-09-03T06:07:25.171 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.172 DEBUG:boto:Data: 2022-09-03T06:07:25.173 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.174 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.175 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.175 DEBUG:boto:Params: {} 2022-09-03T06:07:25.176 DEBUG:boto:Token: None 2022-09-03T06:07:25.177 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8AZpDGQglvOVX5Wj9BOMERkWOqk= 2022-09-03T06:07:25.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8AZpDGQglvOVX5Wj9BOMERkWOqk=', 'Content-Length': '0'} 2022-09-03T06:07:25.183 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000035124e1fb549461-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.226 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.227 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.228 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.229 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.230 DEBUG:boto:Method: GET 2022-09-03T06:07:25.231 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.232 DEBUG:boto:Data: 2022-09-03T06:07:25.233 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.235 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.236 DEBUG:boto:Params: {} 2022-09-03T06:07:25.237 DEBUG:boto:Token: None 2022-09-03T06:07:25.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:25.238 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8AZpDGQglvOVX5Wj9BOMERkWOqk= 2022-09-03T06:07:25.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8AZpDGQglvOVX5Wj9BOMERkWOqk=', 'Content-Length': '0'} 2022-09-03T06:07:25.244 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000846befa182d53c8e-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.245 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.246 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.247 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.248 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.248 DEBUG:boto:Method: GET 2022-09-03T06:07:25.249 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.250 DEBUG:boto:Data: 2022-09-03T06:07:25.251 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.252 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.252 DEBUG:boto:Params: {} 2022-09-03T06:07:25.253 DEBUG:boto:Token: None 2022-09-03T06:07:25.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.261 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff0605f04ccba0be-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.262 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.263 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.263 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.264 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.264 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.265 DEBUG:boto:Method: GET 2022-09-03T06:07:25.266 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.266 DEBUG:boto:Data: 2022-09-03T06:07:25.267 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.267 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.268 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.268 DEBUG:boto:Params: {} 2022-09-03T06:07:25.269 DEBUG:boto:Token: None 2022-09-03T06:07:25.270 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.270 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.271 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.274 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c31d04103f7c3f2-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.274 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.275 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.276 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.276 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.277 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:25.277 DEBUG:boto:Data: 2022-09-03T06:07:25.278 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.279 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.279 DEBUG:boto:Params: {} 2022-09-03T06:07:25.280 DEBUG:boto:Token: None 2022-09-03T06:07:25.281 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj 2022-09-03T06:07:25.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY= 2022-09-03T06:07:25.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY=', 'Content-Length': '0'} 2022-09-03T06:07:25.285 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f6e98ed1639c4c76-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.286 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.287 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.287 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.288 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:25.288 DEBUG:boto:Data: 2022-09-03T06:07:25.289 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.290 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.290 DEBUG:boto:Params: {} 2022-09-03T06:07:25.291 DEBUG:boto:Token: None 2022-09-03T06:07:25.291 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj 2022-09-03T06:07:25.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY= 2022-09-03T06:07:25.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY=', 'Content-Length': '0'} 2022-09-03T06:07:25.296 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000037a95f4842b860d9-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.297 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.298 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.298 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.299 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.299 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.300 DEBUG:boto:Method: GET 2022-09-03T06:07:25.300 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.301 DEBUG:boto:Data: 2022-09-03T06:07:25.301 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.302 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.302 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.303 DEBUG:boto:Params: {} 2022-09-03T06:07:25.304 DEBUG:boto:Token: None 2022-09-03T06:07:25.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM= 2022-09-03T06:07:25.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM=', 'Content-Length': '0'} 2022-09-03T06:07:25.308 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000075a5aed1ca5bbf9f-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.309 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.309 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.310 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.310 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.311 DEBUG:boto:Method: GET 2022-09-03T06:07:25.311 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.312 DEBUG:boto:Data: 2022-09-03T06:07:25.313 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.314 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.314 DEBUG:boto:Params: {} 2022-09-03T06:07:25.315 DEBUG:boto:Token: None 2022-09-03T06:07:25.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM= 2022-09-03T06:07:25.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM=', 'Content-Length': '0'} 2022-09-03T06:07:25.319 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000047d73372e38a0887-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.320 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.321 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.321 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.322 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.323 DEBUG:boto:Method: GET 2022-09-03T06:07:25.323 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.324 DEBUG:boto:Data: 2022-09-03T06:07:25.324 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.325 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.325 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.326 DEBUG:boto:Params: {} 2022-09-03T06:07:25.326 DEBUG:boto:Token: None 2022-09-03T06:07:25.327 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.331 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5413da30769a559-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.332 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.333 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.333 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.334 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.334 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.335 DEBUG:boto:Method: GET 2022-09-03T06:07:25.336 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.336 DEBUG:boto:Data: 2022-09-03T06:07:25.337 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.338 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.338 DEBUG:boto:Params: {} 2022-09-03T06:07:25.339 DEBUG:boto:Token: None 2022-09-03T06:07:25.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.344 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007922f3e366e9aec2-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.344 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.345 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.346 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.346 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.347 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.347 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.348 DEBUG:boto:Method: GET 2022-09-03T06:07:25.349 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.349 DEBUG:boto:Data: 2022-09-03T06:07:25.350 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.350 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.351 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.351 DEBUG:boto:Params: {} 2022-09-03T06:07:25.352 DEBUG:boto:Token: None 2022-09-03T06:07:25.352 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.353 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a+9c9ZhnyB5us/52jg00TEk+wlU= 2022-09-03T06:07:25.354 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a+9c9ZhnyB5us/52jg00TEk+wlU=', 'Content-Length': '0'} 2022-09-03T06:07:25.356 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cf032a2ff58426b4-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.357 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.358 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.358 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.359 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.359 DEBUG:boto:Method: GET 2022-09-03T06:07:25.360 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.360 DEBUG:boto:Data: 2022-09-03T06:07:25.361 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.362 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.363 DEBUG:boto:Params: {} 2022-09-03T06:07:25.363 DEBUG:boto:Token: None 2022-09-03T06:07:25.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a+9c9ZhnyB5us/52jg00TEk+wlU= 2022-09-03T06:07:25.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a+9c9ZhnyB5us/52jg00TEk+wlU=', 'Content-Length': '0'} 2022-09-03T06:07:25.368 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b836bb10418e4e6a-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.410 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.411 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.412 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.412 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.413 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.413 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.414 DEBUG:boto:Data: 2022-09-03T06:07:25.415 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.415 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.416 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.416 DEBUG:boto:Params: {} 2022-09-03T06:07:25.417 DEBUG:boto:Token: None 2022-09-03T06:07:25.418 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nUXEUltvFEN+R/j2kJqoXJvCgJk= 2022-09-03T06:07:25.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nUXEUltvFEN+R/j2kJqoXJvCgJk=', 'Content-Length': '0'} 2022-09-03T06:07:25.421 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000016aeb77576628161-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.423 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.424 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.424 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.425 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.427 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.428 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.429 DEBUG:boto:Data: 2022-09-03T06:07:25.430 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.431 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.432 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.433 DEBUG:boto:Params: {} 2022-09-03T06:07:25.434 DEBUG:boto:Token: None 2022-09-03T06:07:25.435 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nUXEUltvFEN+R/j2kJqoXJvCgJk= 2022-09-03T06:07:25.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nUXEUltvFEN+R/j2kJqoXJvCgJk=', 'Content-Length': '0'} 2022-09-03T06:07:25.440 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bb7ac27c03047956-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.441 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.442 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.442 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.443 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.443 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.444 DEBUG:boto:Method: GET 2022-09-03T06:07:25.444 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.445 DEBUG:boto:Data: 2022-09-03T06:07:25.445 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.446 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.447 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.447 DEBUG:boto:Params: {} 2022-09-03T06:07:25.448 DEBUG:boto:Token: None 2022-09-03T06:07:25.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a+9c9ZhnyB5us/52jg00TEk+wlU= 2022-09-03T06:07:25.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a+9c9ZhnyB5us/52jg00TEk+wlU=', 'Content-Length': '0'} 2022-09-03T06:07:25.452 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000072b711f1d71bc3dd-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.454 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.455 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.456 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.456 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.457 DEBUG:boto:Method: GET 2022-09-03T06:07:25.458 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.459 DEBUG:boto:Data: 2022-09-03T06:07:25.460 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.461 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.462 DEBUG:boto:Params: {} 2022-09-03T06:07:25.462 DEBUG:boto:Token: None 2022-09-03T06:07:25.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:25.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a+9c9ZhnyB5us/52jg00TEk+wlU= 2022-09-03T06:07:25.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a+9c9ZhnyB5us/52jg00TEk+wlU=', 'Content-Length': '0'} 2022-09-03T06:07:25.467 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e936530b36366387-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.469 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.470 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.471 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.471 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.472 DEBUG:boto:Method: GET 2022-09-03T06:07:25.473 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.474 DEBUG:boto:Data: 2022-09-03T06:07:25.475 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.477 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.478 DEBUG:boto:Params: {} 2022-09-03T06:07:25.479 DEBUG:boto:Token: None 2022-09-03T06:07:25.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.485 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0862c358f6ad9cf-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.486 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.487 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.488 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.489 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.489 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.490 DEBUG:boto:Method: GET 2022-09-03T06:07:25.491 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.491 DEBUG:boto:Data: 2022-09-03T06:07:25.492 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.494 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.495 DEBUG:boto:Params: {} 2022-09-03T06:07:25.496 DEBUG:boto:Token: None 2022-09-03T06:07:25.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.502 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000670c923c3878ab49-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.503 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.504 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.505 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.505 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.506 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:25.507 DEBUG:boto:Data: 2022-09-03T06:07:25.507 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.508 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.509 DEBUG:boto:Params: {} 2022-09-03T06:07:25.510 DEBUG:boto:Token: None 2022-09-03T06:07:25.510 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj 2022-09-03T06:07:25.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY= 2022-09-03T06:07:25.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY=', 'Content-Length': '0'} 2022-09-03T06:07:25.515 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fc57fd9df83afea5-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.516 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.516 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.517 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.518 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:25.518 DEBUG:boto:Data: 2022-09-03T06:07:25.519 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.519 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.520 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.520 DEBUG:boto:Params: {} 2022-09-03T06:07:25.521 DEBUG:boto:Token: None 2022-09-03T06:07:25.522 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj 2022-09-03T06:07:25.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY= 2022-09-03T06:07:25.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY=', 'Content-Length': '0'} 2022-09-03T06:07:25.527 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009db8e7ccb6135595-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.528 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.529 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.530 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.531 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.532 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.533 DEBUG:boto:Method: GET 2022-09-03T06:07:25.534 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.535 DEBUG:boto:Data: 2022-09-03T06:07:25.536 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.536 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.537 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.537 DEBUG:boto:Params: {} 2022-09-03T06:07:25.538 DEBUG:boto:Token: None 2022-09-03T06:07:25.539 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM= 2022-09-03T06:07:25.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM=', 'Content-Length': '0'} 2022-09-03T06:07:25.542 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000725801e296693080-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.586 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.587 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.587 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.588 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.589 DEBUG:boto:Method: GET 2022-09-03T06:07:25.590 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.590 DEBUG:boto:Data: 2022-09-03T06:07:25.591 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.592 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.592 DEBUG:boto:Params: {} 2022-09-03T06:07:25.593 DEBUG:boto:Token: None 2022-09-03T06:07:25.594 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM= 2022-09-03T06:07:25.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM=', 'Content-Length': '0'} 2022-09-03T06:07:25.597 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004aedb8efce5b572e-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.598 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.599 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.599 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.600 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.600 DEBUG:boto:Method: GET 2022-09-03T06:07:25.601 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.601 DEBUG:boto:Data: 2022-09-03T06:07:25.602 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.602 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.603 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.603 DEBUG:boto:Params: {} 2022-09-03T06:07:25.604 DEBUG:boto:Token: None 2022-09-03T06:07:25.605 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.609 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1899e6fbddee644-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.610 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.611 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.611 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.612 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.613 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.613 DEBUG:boto:Method: GET 2022-09-03T06:07:25.614 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.614 DEBUG:boto:Data: 2022-09-03T06:07:25.615 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.616 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.616 DEBUG:boto:Params: {} 2022-09-03T06:07:25.617 DEBUG:boto:Token: None 2022-09-03T06:07:25.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.621 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006db041586664e5d6-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.622 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.623 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.624 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.624 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.625 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.625 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.626 DEBUG:boto:Method: GET 2022-09-03T06:07:25.627 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.627 DEBUG:boto:Data: 2022-09-03T06:07:25.628 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.628 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.629 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.629 DEBUG:boto:Params: {} 2022-09-03T06:07:25.630 DEBUG:boto:Token: None 2022-09-03T06:07:25.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EgVMUbeUT3yTxkh0syLezx3mzW0= 2022-09-03T06:07:25.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EgVMUbeUT3yTxkh0syLezx3mzW0=', 'Content-Length': '0'} 2022-09-03T06:07:25.634 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e318f01ee818bf64-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.635 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.636 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.636 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.637 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.637 DEBUG:boto:Method: GET 2022-09-03T06:07:25.638 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.638 DEBUG:boto:Data: 2022-09-03T06:07:25.639 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.640 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.640 DEBUG:boto:Params: {} 2022-09-03T06:07:25.641 DEBUG:boto:Token: None 2022-09-03T06:07:25.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EgVMUbeUT3yTxkh0syLezx3mzW0= 2022-09-03T06:07:25.643 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EgVMUbeUT3yTxkh0syLezx3mzW0=', 'Content-Length': '0'} 2022-09-03T06:07:25.646 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003c9eeea6da1132c0-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.647 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.648 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.649 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.650 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.651 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.652 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.653 DEBUG:boto:Data: 2022-09-03T06:07:25.654 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.655 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.656 DEBUG:boto:Params: {} 2022-09-03T06:07:25.656 DEBUG:boto:Token: None 2022-09-03T06:07:25.657 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2bB6dOK7zHoakpC/qnKRUJaaNG8= 2022-09-03T06:07:25.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2bB6dOK7zHoakpC/qnKRUJaaNG8=', 'Content-Length': '0'} 2022-09-03T06:07:25.661 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000008f45a6fa0e97d90-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.662 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.663 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.664 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.665 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.666 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.667 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.667 DEBUG:boto:Data: 2022-09-03T06:07:25.668 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.669 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.670 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.670 DEBUG:boto:Params: {} 2022-09-03T06:07:25.671 DEBUG:boto:Token: None 2022-09-03T06:07:25.672 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2bB6dOK7zHoakpC/qnKRUJaaNG8= 2022-09-03T06:07:25.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2bB6dOK7zHoakpC/qnKRUJaaNG8=', 'Content-Length': '0'} 2022-09-03T06:07:25.675 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c22eea7c23909383-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.676 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.676 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.677 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.678 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.679 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.680 DEBUG:boto:Method: GET 2022-09-03T06:07:25.680 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.681 DEBUG:boto:Data: 2022-09-03T06:07:25.682 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.683 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.684 DEBUG:boto:Params: {} 2022-09-03T06:07:25.684 DEBUG:boto:Token: None 2022-09-03T06:07:25.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EgVMUbeUT3yTxkh0syLezx3mzW0= 2022-09-03T06:07:25.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EgVMUbeUT3yTxkh0syLezx3mzW0=', 'Content-Length': '0'} 2022-09-03T06:07:25.689 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008438664e14a9cfb6-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.689 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.690 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.691 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.691 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.692 DEBUG:boto:Method: GET 2022-09-03T06:07:25.692 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.693 DEBUG:boto:Data: 2022-09-03T06:07:25.693 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.694 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.694 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.695 DEBUG:boto:Params: {} 2022-09-03T06:07:25.696 DEBUG:boto:Token: None 2022-09-03T06:07:25.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:25.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EgVMUbeUT3yTxkh0syLezx3mzW0= 2022-09-03T06:07:25.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EgVMUbeUT3yTxkh0syLezx3mzW0=', 'Content-Length': '0'} 2022-09-03T06:07:25.700 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b1d0ed7e4fef25a6-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.701 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.701 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.702 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.702 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.703 DEBUG:boto:Method: GET 2022-09-03T06:07:25.703 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.704 DEBUG:boto:Data: 2022-09-03T06:07:25.704 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.705 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.705 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.706 DEBUG:boto:Params: {} 2022-09-03T06:07:25.706 DEBUG:boto:Token: None 2022-09-03T06:07:25.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.711 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c16126821cccfe6c-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.712 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.713 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.713 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.714 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.714 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.715 DEBUG:boto:Method: GET 2022-09-03T06:07:25.715 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.716 DEBUG:boto:Data: 2022-09-03T06:07:25.716 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.717 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.718 DEBUG:boto:Params: {} 2022-09-03T06:07:25.719 DEBUG:boto:Token: None 2022-09-03T06:07:25.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.723 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002fb13a04f9ec08a9-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.724 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.725 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.725 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.726 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.726 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:25.727 DEBUG:boto:Data: 2022-09-03T06:07:25.727 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.728 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.729 DEBUG:boto:Params: {} 2022-09-03T06:07:25.729 DEBUG:boto:Token: None 2022-09-03T06:07:25.730 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj 2022-09-03T06:07:25.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY= 2022-09-03T06:07:25.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY=', 'Content-Length': '0'} 2022-09-03T06:07:25.735 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000015a4aa7184d4c4f-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.735 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.736 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.737 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.737 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:25.737 DEBUG:boto:Data: 2022-09-03T06:07:25.738 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.739 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.740 DEBUG:boto:Params: {} 2022-09-03T06:07:25.740 DEBUG:boto:Token: None 2022-09-03T06:07:25.741 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj 2022-09-03T06:07:25.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY= 2022-09-03T06:07:25.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY=', 'Content-Length': '0'} 2022-09-03T06:07:25.746 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ee8a18571e6354f6-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.747 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.747 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.748 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.748 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.749 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.749 DEBUG:boto:Method: GET 2022-09-03T06:07:25.750 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.750 DEBUG:boto:Data: 2022-09-03T06:07:25.751 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.752 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.752 DEBUG:boto:Params: {} 2022-09-03T06:07:25.753 DEBUG:boto:Token: None 2022-09-03T06:07:25.754 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM= 2022-09-03T06:07:25.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM=', 'Content-Length': '0'} 2022-09-03T06:07:25.757 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000083cd7d54408a8904-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.798 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.799 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.800 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.800 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.801 DEBUG:boto:Method: GET 2022-09-03T06:07:25.801 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.802 DEBUG:boto:Data: 2022-09-03T06:07:25.803 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.803 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.804 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.804 DEBUG:boto:Params: {} 2022-09-03T06:07:25.805 DEBUG:boto:Token: None 2022-09-03T06:07:25.805 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM= 2022-09-03T06:07:25.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qp3iGmYZb/geSNngdz6KMje+fvM=', 'Content-Length': '0'} 2022-09-03T06:07:25.809 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004b8bdb03e781af3e-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.810 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.811 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.811 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.812 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.812 DEBUG:boto:Method: GET 2022-09-03T06:07:25.813 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.813 DEBUG:boto:Data: 2022-09-03T06:07:25.815 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.816 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.816 DEBUG:boto:Params: {} 2022-09-03T06:07:25.817 DEBUG:boto:Token: None 2022-09-03T06:07:25.818 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.826 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bcb7b1d10d1a282d-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.827 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.828 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.829 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.829 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.830 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.830 DEBUG:boto:Method: GET 2022-09-03T06:07:25.831 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.831 DEBUG:boto:Data: 2022-09-03T06:07:25.832 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.833 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.833 DEBUG:boto:Params: {} 2022-09-03T06:07:25.834 DEBUG:boto:Token: None 2022-09-03T06:07:25.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.841 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000408c454f7004bea3-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.842 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.843 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.844 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.844 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.845 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.845 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.846 DEBUG:boto:Method: GET 2022-09-03T06:07:25.847 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.847 DEBUG:boto:Data: 2022-09-03T06:07:25.848 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.849 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.849 DEBUG:boto:Params: {} 2022-09-03T06:07:25.851 DEBUG:boto:Token: None 2022-09-03T06:07:25.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9UwkSVBiD/uSukL3VbAcP+mzlpQ= 2022-09-03T06:07:25.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9UwkSVBiD/uSukL3VbAcP+mzlpQ=', 'Content-Length': '0'} 2022-09-03T06:07:25.856 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ac78cc68ab874dce-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.857 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.858 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.859 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.859 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.860 DEBUG:boto:Method: GET 2022-09-03T06:07:25.861 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.862 DEBUG:boto:Data: 2022-09-03T06:07:25.863 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.865 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.866 DEBUG:boto:Params: {} 2022-09-03T06:07:25.867 DEBUG:boto:Token: None 2022-09-03T06:07:25.868 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9UwkSVBiD/uSukL3VbAcP+mzlpQ= 2022-09-03T06:07:25.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9UwkSVBiD/uSukL3VbAcP+mzlpQ=', 'Content-Length': '0'} 2022-09-03T06:07:25.873 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004462ca92d238d296-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.875 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.876 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.877 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.877 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.878 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.878 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.879 DEBUG:boto:Data: 2022-09-03T06:07:25.879 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.880 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.881 DEBUG:boto:Params: {} 2022-09-03T06:07:25.881 DEBUG:boto:Token: None 2022-09-03T06:07:25.882 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rno19yCwJsBcicciwh7cofw8A4A= 2022-09-03T06:07:25.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rno19yCwJsBcicciwh7cofw8A4A=', 'Content-Length': '0'} 2022-09-03T06:07:25.885 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000f897f3ed9dc56cc-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.886 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.887 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.887 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.888 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.888 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.889 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.889 DEBUG:boto:Data: 2022-09-03T06:07:25.890 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.891 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.891 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.892 DEBUG:boto:Params: {} 2022-09-03T06:07:25.893 DEBUG:boto:Token: None 2022-09-03T06:07:25.894 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rno19yCwJsBcicciwh7cofw8A4A= 2022-09-03T06:07:25.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rno19yCwJsBcicciwh7cofw8A4A=', 'Content-Length': '0'} 2022-09-03T06:07:25.897 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004b795df7ec2c65b3-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.898 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.899 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.900 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.900 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.901 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.902 DEBUG:boto:Method: GET 2022-09-03T06:07:25.902 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.903 DEBUG:boto:Data: 2022-09-03T06:07:25.903 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.904 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.904 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.905 DEBUG:boto:Params: {} 2022-09-03T06:07:25.906 DEBUG:boto:Token: None 2022-09-03T06:07:25.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9UwkSVBiD/uSukL3VbAcP+mzlpQ= 2022-09-03T06:07:25.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9UwkSVBiD/uSukL3VbAcP+mzlpQ=', 'Content-Length': '0'} 2022-09-03T06:07:25.911 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003a62da9bf671c447-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.912 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.913 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.913 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.914 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.914 DEBUG:boto:Method: GET 2022-09-03T06:07:25.915 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.915 DEBUG:boto:Data: 2022-09-03T06:07:25.916 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.917 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.917 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.918 DEBUG:boto:Params: {} 2022-09-03T06:07:25.919 DEBUG:boto:Token: None 2022-09-03T06:07:25.920 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:25.921 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9UwkSVBiD/uSukL3VbAcP+mzlpQ= 2022-09-03T06:07:25.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9UwkSVBiD/uSukL3VbAcP+mzlpQ=', 'Content-Length': '0'} 2022-09-03T06:07:25.924 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000095836f867a127548-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.926 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.927 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.928 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.929 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.931 DEBUG:boto:Method: GET 2022-09-03T06:07:25.931 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.932 DEBUG:boto:Data: 2022-09-03T06:07:25.933 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.935 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.935 DEBUG:boto:Params: {} 2022-09-03T06:07:25.936 DEBUG:boto:Token: None 2022-09-03T06:07:25.936 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.941 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007626865bdde5ac07-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.942 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.944 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:25.945 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:25.946 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.947 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:25.948 DEBUG:boto:Method: GET 2022-09-03T06:07:25.949 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:25.950 DEBUG:boto:Data: 2022-09-03T06:07:25.951 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.952 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.953 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.954 DEBUG:boto:Params: {} 2022-09-03T06:07:25.955 DEBUG:boto:Token: None 2022-09-03T06:07:25.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/ 2022-09-03T06:07:25.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4= 2022-09-03T06:07:25.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqQp7hwTrFtcemGyydwudK4gNz4=', 'Content-Length': '0'} 2022-09-03T06:07:25.962 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db3061551dce5489-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.964 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:25.965 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.966 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.968 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.969 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:25.970 DEBUG:boto:Data: 2022-09-03T06:07:25.970 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.971 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:25.972 DEBUG:boto:Port: 8001 2022-09-03T06:07:25.972 DEBUG:boto:Params: {} 2022-09-03T06:07:25.973 DEBUG:boto:Token: None 2022-09-03T06:07:25.973 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj 2022-09-03T06:07:25.974 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY= 2022-09-03T06:07:25.974 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY=', 'Content-Length': '0'} 2022-09-03T06:07:25.978 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fa0be8185f397333-006312ef1d-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.979 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.980 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.981 DEBUG:boto:Method: HEAD 2022-09-03T06:07:25.982 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:25.983 DEBUG:boto:Data: 2022-09-03T06:07:25.983 DEBUG:boto:Headers: {} 2022-09-03T06:07:25.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:25.985 DEBUG:boto:Port: 8000 2022-09-03T06:07:25.985 DEBUG:boto:Params: {} 2022-09-03T06:07:25.986 DEBUG:boto:Token: None 2022-09-03T06:07:25.987 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:25 GMT /avysmw-19/obj 2022-09-03T06:07:25.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY= 2022-09-03T06:07:25.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vfRJl7Qj7LMhWk5YE7xTk/TydY=', 'Content-Length': '0'} 2022-09-03T06:07:25.992 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000209282866324f170-006312ef1d-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:25.993 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:25.994 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:25.995 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:25.996 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.996 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.997 DEBUG:boto:Method: GET 2022-09-03T06:07:25.998 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:25.999 DEBUG:boto:Data: 2022-09-03T06:07:26.000 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.001 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.002 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.003 DEBUG:boto:Params: {} 2022-09-03T06:07:26.004 DEBUG:boto:Token: None 2022-09-03T06:07:26.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.008 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000085d78c917d3f6b63-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.050 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.051 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.052 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.053 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.053 DEBUG:boto:Method: GET 2022-09-03T06:07:26.054 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.054 DEBUG:boto:Data: 2022-09-03T06:07:26.055 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.057 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.057 DEBUG:boto:Params: {} 2022-09-03T06:07:26.058 DEBUG:boto:Token: None 2022-09-03T06:07:26.059 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.063 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b8bf5598501e9246-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.065 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.065 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.066 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.066 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.067 DEBUG:boto:Method: GET 2022-09-03T06:07:26.067 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.068 DEBUG:boto:Data: 2022-09-03T06:07:26.068 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.069 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.069 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.070 DEBUG:boto:Params: {} 2022-09-03T06:07:26.071 DEBUG:boto:Token: None 2022-09-03T06:07:26.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.077 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009903c7622a217278-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.078 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.079 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.080 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.081 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.081 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.082 DEBUG:boto:Method: GET 2022-09-03T06:07:26.082 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.083 DEBUG:boto:Data: 2022-09-03T06:07:26.084 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.085 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.086 DEBUG:boto:Params: {} 2022-09-03T06:07:26.087 DEBUG:boto:Token: None 2022-09-03T06:07:26.088 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.092 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000008e816355673d51a-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.093 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.094 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.095 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.095 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.096 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.096 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.097 DEBUG:boto:Method: GET 2022-09-03T06:07:26.098 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.098 DEBUG:boto:Data: 2022-09-03T06:07:26.099 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.101 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.102 DEBUG:boto:Params: {} 2022-09-03T06:07:26.103 DEBUG:boto:Token: None 2022-09-03T06:07:26.104 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hk+xMg31SjNyuUZjEjYuCRRTqm4= 2022-09-03T06:07:26.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hk+xMg31SjNyuUZjEjYuCRRTqm4=', 'Content-Length': '0'} 2022-09-03T06:07:26.107 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000045a4d3e5fa154084-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.108 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.109 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.110 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.110 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.111 DEBUG:boto:Method: GET 2022-09-03T06:07:26.112 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.112 DEBUG:boto:Data: 2022-09-03T06:07:26.113 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.113 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.114 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.115 DEBUG:boto:Params: {} 2022-09-03T06:07:26.115 DEBUG:boto:Token: None 2022-09-03T06:07:26.116 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hk+xMg31SjNyuUZjEjYuCRRTqm4= 2022-09-03T06:07:26.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hk+xMg31SjNyuUZjEjYuCRRTqm4=', 'Content-Length': '0'} 2022-09-03T06:07:26.120 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dc2c18d0b57591be-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.121 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.122 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.123 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.123 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.124 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.125 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.126 DEBUG:boto:Data: 2022-09-03T06:07:26.126 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.127 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.128 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.129 DEBUG:boto:Params: {} 2022-09-03T06:07:26.129 DEBUG:boto:Token: None 2022-09-03T06:07:26.130 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BD716ALdVuP7SVOI/q8MQL/zwwg= 2022-09-03T06:07:26.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BD716ALdVuP7SVOI/q8MQL/zwwg=', 'Content-Length': '0'} 2022-09-03T06:07:26.134 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000005288f4e3f0e9d7c-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.135 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.136 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.136 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.137 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.138 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.138 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.139 DEBUG:boto:Data: 2022-09-03T06:07:26.140 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.140 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.141 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.142 DEBUG:boto:Params: {} 2022-09-03T06:07:26.143 DEBUG:boto:Token: None 2022-09-03T06:07:26.143 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BD716ALdVuP7SVOI/q8MQL/zwwg= 2022-09-03T06:07:26.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BD716ALdVuP7SVOI/q8MQL/zwwg=', 'Content-Length': '0'} 2022-09-03T06:07:26.147 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dc7911642ee8e730-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.147 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.148 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.148 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.149 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.149 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.150 DEBUG:boto:Method: GET 2022-09-03T06:07:26.151 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.151 DEBUG:boto:Data: 2022-09-03T06:07:26.152 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.152 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.153 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.153 DEBUG:boto:Params: {} 2022-09-03T06:07:26.154 DEBUG:boto:Token: None 2022-09-03T06:07:26.154 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hk+xMg31SjNyuUZjEjYuCRRTqm4= 2022-09-03T06:07:26.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hk+xMg31SjNyuUZjEjYuCRRTqm4=', 'Content-Length': '0'} 2022-09-03T06:07:26.158 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000aa46c4196ed077b8-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.159 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.159 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.160 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.161 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.161 DEBUG:boto:Method: GET 2022-09-03T06:07:26.162 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.162 DEBUG:boto:Data: 2022-09-03T06:07:26.163 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.164 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.164 DEBUG:boto:Params: {} 2022-09-03T06:07:26.165 DEBUG:boto:Token: None 2022-09-03T06:07:26.166 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:26.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hk+xMg31SjNyuUZjEjYuCRRTqm4= 2022-09-03T06:07:26.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hk+xMg31SjNyuUZjEjYuCRRTqm4=', 'Content-Length': '0'} 2022-09-03T06:07:26.169 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000b69caf26f6f0ef4-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.170 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.171 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.171 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.172 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.172 DEBUG:boto:Method: GET 2022-09-03T06:07:26.173 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.173 DEBUG:boto:Data: 2022-09-03T06:07:26.174 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.174 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.175 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.175 DEBUG:boto:Params: {} 2022-09-03T06:07:26.176 DEBUG:boto:Token: None 2022-09-03T06:07:26.177 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.181 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9037a7b02b1c7a4-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.181 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.182 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.183 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.183 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.184 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.184 DEBUG:boto:Method: GET 2022-09-03T06:07:26.185 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.185 DEBUG:boto:Data: 2022-09-03T06:07:26.186 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.187 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.188 DEBUG:boto:Params: {} 2022-09-03T06:07:26.188 DEBUG:boto:Token: None 2022-09-03T06:07:26.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.193 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007550d49446273101-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.194 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.194 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.195 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.195 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.196 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:26.197 DEBUG:boto:Data: 2022-09-03T06:07:26.197 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.198 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.199 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.199 DEBUG:boto:Params: {} 2022-09-03T06:07:26.200 DEBUG:boto:Token: None 2022-09-03T06:07:26.200 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj 2022-09-03T06:07:26.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI= 2022-09-03T06:07:26.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI=', 'Content-Length': '0'} 2022-09-03T06:07:26.205 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000431c5d4ce9f2690b-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.206 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.206 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.207 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.207 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:26.208 DEBUG:boto:Data: 2022-09-03T06:07:26.208 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.209 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.209 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.210 DEBUG:boto:Params: {} 2022-09-03T06:07:26.210 DEBUG:boto:Token: None 2022-09-03T06:07:26.211 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj 2022-09-03T06:07:26.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI= 2022-09-03T06:07:26.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI=', 'Content-Length': '0'} 2022-09-03T06:07:26.215 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c883b99dde12dfd6-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.217 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.217 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.218 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.218 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.219 DEBUG:boto:Method: GET 2022-09-03T06:07:26.219 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.220 DEBUG:boto:Data: 2022-09-03T06:07:26.220 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.221 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.222 DEBUG:boto:Params: {} 2022-09-03T06:07:26.223 DEBUG:boto:Token: None 2022-09-03T06:07:26.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.227 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000df23eba2e1f9fb7d-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.227 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.228 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.229 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.229 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.230 DEBUG:boto:Method: GET 2022-09-03T06:07:26.230 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.231 DEBUG:boto:Data: 2022-09-03T06:07:26.231 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.232 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.232 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.233 DEBUG:boto:Params: {} 2022-09-03T06:07:26.233 DEBUG:boto:Token: None 2022-09-03T06:07:26.234 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.235 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.237 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a196b9e356b3978d-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.282 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.283 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.284 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.284 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.285 DEBUG:boto:Method: GET 2022-09-03T06:07:26.285 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.286 DEBUG:boto:Data: 2022-09-03T06:07:26.286 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.287 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.287 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.288 DEBUG:boto:Params: {} 2022-09-03T06:07:26.289 DEBUG:boto:Token: None 2022-09-03T06:07:26.289 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.290 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.293 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000024526d78addc4c5-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.294 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.295 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.295 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.296 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.296 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.297 DEBUG:boto:Method: GET 2022-09-03T06:07:26.297 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.298 DEBUG:boto:Data: 2022-09-03T06:07:26.298 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.299 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.300 DEBUG:boto:Params: {} 2022-09-03T06:07:26.301 DEBUG:boto:Token: None 2022-09-03T06:07:26.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.305 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b83994dd37d52aaf-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.306 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.307 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.307 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.308 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.308 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.309 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.309 DEBUG:boto:Method: GET 2022-09-03T06:07:26.310 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.310 DEBUG:boto:Data: 2022-09-03T06:07:26.311 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.312 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.313 DEBUG:boto:Params: {} 2022-09-03T06:07:26.313 DEBUG:boto:Token: None 2022-09-03T06:07:26.314 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KcLkOkY33s0k0pHxvJWGFeYBeRQ= 2022-09-03T06:07:26.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KcLkOkY33s0k0pHxvJWGFeYBeRQ=', 'Content-Length': '0'} 2022-09-03T06:07:26.318 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f4c0449022ad4ecf-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.319 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.319 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.320 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.320 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.321 DEBUG:boto:Method: GET 2022-09-03T06:07:26.321 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.322 DEBUG:boto:Data: 2022-09-03T06:07:26.322 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.323 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.323 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.324 DEBUG:boto:Params: {} 2022-09-03T06:07:26.324 DEBUG:boto:Token: None 2022-09-03T06:07:26.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KcLkOkY33s0k0pHxvJWGFeYBeRQ= 2022-09-03T06:07:26.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KcLkOkY33s0k0pHxvJWGFeYBeRQ=', 'Content-Length': '0'} 2022-09-03T06:07:26.328 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b593a5708a592b75-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.329 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.330 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.330 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.331 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.331 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.332 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.332 DEBUG:boto:Data: 2022-09-03T06:07:26.333 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.334 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.335 DEBUG:boto:Params: {} 2022-09-03T06:07:26.335 DEBUG:boto:Token: None 2022-09-03T06:07:26.336 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WOEQq4od8QhIAa4LlVO7qF4nJCQ= 2022-09-03T06:07:26.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WOEQq4od8QhIAa4LlVO7qF4nJCQ=', 'Content-Length': '0'} 2022-09-03T06:07:26.339 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000014cafbcee1880dd0-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.340 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.341 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.342 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.342 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.343 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.343 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.344 DEBUG:boto:Data: 2022-09-03T06:07:26.344 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.345 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.346 DEBUG:boto:Params: {} 2022-09-03T06:07:26.346 DEBUG:boto:Token: None 2022-09-03T06:07:26.347 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WOEQq4od8QhIAa4LlVO7qF4nJCQ= 2022-09-03T06:07:26.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WOEQq4od8QhIAa4LlVO7qF4nJCQ=', 'Content-Length': '0'} 2022-09-03T06:07:26.351 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005c674db992a40b25-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.352 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.353 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.354 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.355 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.356 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.357 DEBUG:boto:Method: GET 2022-09-03T06:07:26.358 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.359 DEBUG:boto:Data: 2022-09-03T06:07:26.360 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.361 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.362 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.362 DEBUG:boto:Params: {} 2022-09-03T06:07:26.363 DEBUG:boto:Token: None 2022-09-03T06:07:26.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KcLkOkY33s0k0pHxvJWGFeYBeRQ= 2022-09-03T06:07:26.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KcLkOkY33s0k0pHxvJWGFeYBeRQ=', 'Content-Length': '0'} 2022-09-03T06:07:26.368 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005a9ddfcffb2b85ab-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.414 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.415 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.416 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.417 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.418 DEBUG:boto:Method: GET 2022-09-03T06:07:26.419 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.420 DEBUG:boto:Data: 2022-09-03T06:07:26.420 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.422 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.423 DEBUG:boto:Params: {} 2022-09-03T06:07:26.423 DEBUG:boto:Token: None 2022-09-03T06:07:26.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:26.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KcLkOkY33s0k0pHxvJWGFeYBeRQ= 2022-09-03T06:07:26.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KcLkOkY33s0k0pHxvJWGFeYBeRQ=', 'Content-Length': '0'} 2022-09-03T06:07:26.428 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000860d31d38051f6ae-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.429 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.430 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.432 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.433 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.434 DEBUG:boto:Method: GET 2022-09-03T06:07:26.435 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.435 DEBUG:boto:Data: 2022-09-03T06:07:26.436 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.437 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.438 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.439 DEBUG:boto:Params: {} 2022-09-03T06:07:26.440 DEBUG:boto:Token: None 2022-09-03T06:07:26.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.446 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000506c3f7fa3c9f271-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.447 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.448 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.448 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.449 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.449 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.450 DEBUG:boto:Method: GET 2022-09-03T06:07:26.451 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.451 DEBUG:boto:Data: 2022-09-03T06:07:26.452 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.452 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.453 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.453 DEBUG:boto:Params: {} 2022-09-03T06:07:26.454 DEBUG:boto:Token: None 2022-09-03T06:07:26.455 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.459 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000422c0bac2ab5ae30-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.459 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.460 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.461 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.461 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.462 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:26.462 DEBUG:boto:Data: 2022-09-03T06:07:26.463 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.464 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.465 DEBUG:boto:Params: {} 2022-09-03T06:07:26.466 DEBUG:boto:Token: None 2022-09-03T06:07:26.467 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj 2022-09-03T06:07:26.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI= 2022-09-03T06:07:26.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI=', 'Content-Length': '0'} 2022-09-03T06:07:26.473 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000063f00c2db9a07c67-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.474 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.475 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.476 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.477 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:26.478 DEBUG:boto:Data: 2022-09-03T06:07:26.479 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.480 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.481 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.482 DEBUG:boto:Params: {} 2022-09-03T06:07:26.483 DEBUG:boto:Token: None 2022-09-03T06:07:26.484 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj 2022-09-03T06:07:26.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI= 2022-09-03T06:07:26.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI=', 'Content-Length': '0'} 2022-09-03T06:07:26.490 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c1a8350f8dc99c1c-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.491 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.491 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.492 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.493 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.493 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.494 DEBUG:boto:Method: GET 2022-09-03T06:07:26.494 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.495 DEBUG:boto:Data: 2022-09-03T06:07:26.495 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.496 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.497 DEBUG:boto:Params: {} 2022-09-03T06:07:26.497 DEBUG:boto:Token: None 2022-09-03T06:07:26.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.501 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000052ca92d297b183dd-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.502 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.503 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.503 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.504 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.504 DEBUG:boto:Method: GET 2022-09-03T06:07:26.505 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.505 DEBUG:boto:Data: 2022-09-03T06:07:26.506 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.506 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.507 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.507 DEBUG:boto:Params: {} 2022-09-03T06:07:26.508 DEBUG:boto:Token: None 2022-09-03T06:07:26.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.512 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e8d365b0fa28b91e-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.558 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.559 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.560 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.561 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.562 DEBUG:boto:Method: GET 2022-09-03T06:07:26.563 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.564 DEBUG:boto:Data: 2022-09-03T06:07:26.564 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.565 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.565 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.566 DEBUG:boto:Params: {} 2022-09-03T06:07:26.566 DEBUG:boto:Token: None 2022-09-03T06:07:26.567 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.568 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.571 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c5e916b87c045653-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.572 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.573 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.573 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.574 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.574 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.575 DEBUG:boto:Method: GET 2022-09-03T06:07:26.575 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.576 DEBUG:boto:Data: 2022-09-03T06:07:26.576 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.577 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.578 DEBUG:boto:Params: {} 2022-09-03T06:07:26.579 DEBUG:boto:Token: None 2022-09-03T06:07:26.579 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.583 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006fbfe78900f5c378-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.584 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.585 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.585 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.586 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.587 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.587 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.588 DEBUG:boto:Method: GET 2022-09-03T06:07:26.589 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.590 DEBUG:boto:Data: 2022-09-03T06:07:26.591 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.592 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.593 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.594 DEBUG:boto:Params: {} 2022-09-03T06:07:26.595 DEBUG:boto:Token: None 2022-09-03T06:07:26.596 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pFtG2O5Mq+E2uqB6rjy7OZfHA98= 2022-09-03T06:07:26.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pFtG2O5Mq+E2uqB6rjy7OZfHA98=', 'Content-Length': '0'} 2022-09-03T06:07:26.601 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000009d7027f5e6e4319-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.642 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.643 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.643 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.644 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.644 DEBUG:boto:Method: GET 2022-09-03T06:07:26.645 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.645 DEBUG:boto:Data: 2022-09-03T06:07:26.646 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.647 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.648 DEBUG:boto:Params: {} 2022-09-03T06:07:26.648 DEBUG:boto:Token: None 2022-09-03T06:07:26.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pFtG2O5Mq+E2uqB6rjy7OZfHA98= 2022-09-03T06:07:26.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pFtG2O5Mq+E2uqB6rjy7OZfHA98=', 'Content-Length': '0'} 2022-09-03T06:07:26.652 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007944b9d338f6b645-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.653 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.654 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.654 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.655 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.655 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.656 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.656 DEBUG:boto:Data: 2022-09-03T06:07:26.657 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.658 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.658 DEBUG:boto:Params: {} 2022-09-03T06:07:26.659 DEBUG:boto:Token: None 2022-09-03T06:07:26.660 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e+/21Is8tjuhuR/aOqMQbJ6p0qg= 2022-09-03T06:07:26.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e+/21Is8tjuhuR/aOqMQbJ6p0qg=', 'Content-Length': '0'} 2022-09-03T06:07:26.663 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e424e421d36ad255-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.664 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.664 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.665 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.665 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.666 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.666 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.667 DEBUG:boto:Data: 2022-09-03T06:07:26.667 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.668 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.668 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.669 DEBUG:boto:Params: {} 2022-09-03T06:07:26.669 DEBUG:boto:Token: None 2022-09-03T06:07:26.670 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e+/21Is8tjuhuR/aOqMQbJ6p0qg= 2022-09-03T06:07:26.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e+/21Is8tjuhuR/aOqMQbJ6p0qg=', 'Content-Length': '0'} 2022-09-03T06:07:26.673 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000035fc3931ac2dd920-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.674 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.675 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.675 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.676 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.676 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.677 DEBUG:boto:Method: GET 2022-09-03T06:07:26.677 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.678 DEBUG:boto:Data: 2022-09-03T06:07:26.678 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.679 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.680 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.680 DEBUG:boto:Params: {} 2022-09-03T06:07:26.681 DEBUG:boto:Token: None 2022-09-03T06:07:26.681 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.682 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pFtG2O5Mq+E2uqB6rjy7OZfHA98= 2022-09-03T06:07:26.683 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pFtG2O5Mq+E2uqB6rjy7OZfHA98=', 'Content-Length': '0'} 2022-09-03T06:07:26.685 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000869b78e977d2c59c-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.686 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.686 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.687 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.687 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.688 DEBUG:boto:Method: GET 2022-09-03T06:07:26.688 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.689 DEBUG:boto:Data: 2022-09-03T06:07:26.690 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.690 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.691 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.691 DEBUG:boto:Params: {} 2022-09-03T06:07:26.692 DEBUG:boto:Token: None 2022-09-03T06:07:26.693 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:26.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pFtG2O5Mq+E2uqB6rjy7OZfHA98= 2022-09-03T06:07:26.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pFtG2O5Mq+E2uqB6rjy7OZfHA98=', 'Content-Length': '0'} 2022-09-03T06:07:26.698 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000aa9601b89240a4ff-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.699 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.699 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.700 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.700 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.701 DEBUG:boto:Method: GET 2022-09-03T06:07:26.701 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.702 DEBUG:boto:Data: 2022-09-03T06:07:26.702 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.703 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.704 DEBUG:boto:Params: {} 2022-09-03T06:07:26.705 DEBUG:boto:Token: None 2022-09-03T06:07:26.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.710 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e739f132fd30910e-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.711 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.712 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.712 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.713 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.713 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.714 DEBUG:boto:Method: GET 2022-09-03T06:07:26.714 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.715 DEBUG:boto:Data: 2022-09-03T06:07:26.715 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.717 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.717 DEBUG:boto:Params: {} 2022-09-03T06:07:26.718 DEBUG:boto:Token: None 2022-09-03T06:07:26.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.722 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000532e714bcf67ffe3-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.723 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.724 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.724 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.725 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.726 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:26.726 DEBUG:boto:Data: 2022-09-03T06:07:26.727 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.727 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.728 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.728 DEBUG:boto:Params: {} 2022-09-03T06:07:26.729 DEBUG:boto:Token: None 2022-09-03T06:07:26.729 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj 2022-09-03T06:07:26.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI= 2022-09-03T06:07:26.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI=', 'Content-Length': '0'} 2022-09-03T06:07:26.734 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009e9708df21355d52-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.735 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.735 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.736 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.737 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:26.737 DEBUG:boto:Data: 2022-09-03T06:07:26.738 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.739 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.739 DEBUG:boto:Params: {} 2022-09-03T06:07:26.740 DEBUG:boto:Token: None 2022-09-03T06:07:26.740 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj 2022-09-03T06:07:26.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI= 2022-09-03T06:07:26.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI=', 'Content-Length': '0'} 2022-09-03T06:07:26.745 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006fecebab97169d1f-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.745 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.746 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.747 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.747 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.748 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.748 DEBUG:boto:Method: GET 2022-09-03T06:07:26.749 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.749 DEBUG:boto:Data: 2022-09-03T06:07:26.750 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.750 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.751 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.752 DEBUG:boto:Params: {} 2022-09-03T06:07:26.752 DEBUG:boto:Token: None 2022-09-03T06:07:26.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.756 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000190024bb4cd6ebfe-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.798 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.800 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.801 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.803 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.804 DEBUG:boto:Method: GET 2022-09-03T06:07:26.804 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.805 DEBUG:boto:Data: 2022-09-03T06:07:26.806 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.806 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.807 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.807 DEBUG:boto:Params: {} 2022-09-03T06:07:26.808 DEBUG:boto:Token: None 2022-09-03T06:07:26.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.813 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000863c7fcdc2e56f22-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.814 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.815 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.815 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.816 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.817 DEBUG:boto:Method: GET 2022-09-03T06:07:26.817 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.818 DEBUG:boto:Data: 2022-09-03T06:07:26.818 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.819 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.820 DEBUG:boto:Params: {} 2022-09-03T06:07:26.820 DEBUG:boto:Token: None 2022-09-03T06:07:26.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.825 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008490d6656f963d5c-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.826 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.827 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.828 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.828 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.829 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.829 DEBUG:boto:Method: GET 2022-09-03T06:07:26.830 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.830 DEBUG:boto:Data: 2022-09-03T06:07:26.831 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.832 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.832 DEBUG:boto:Params: {} 2022-09-03T06:07:26.833 DEBUG:boto:Token: None 2022-09-03T06:07:26.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.837 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000006f704426a958b84-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.838 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.839 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.840 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.840 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.841 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.841 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.842 DEBUG:boto:Method: GET 2022-09-03T06:07:26.842 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.843 DEBUG:boto:Data: 2022-09-03T06:07:26.843 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.844 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.844 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.845 DEBUG:boto:Params: {} 2022-09-03T06:07:26.846 DEBUG:boto:Token: None 2022-09-03T06:07:26.846 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.847 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gYhqzxQbA1W34gvMi6LoszeB71c= 2022-09-03T06:07:26.848 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gYhqzxQbA1W34gvMi6LoszeB71c=', 'Content-Length': '0'} 2022-09-03T06:07:26.850 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000299d640591ae6812-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.850 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.851 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.852 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.852 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.853 DEBUG:boto:Method: GET 2022-09-03T06:07:26.853 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.854 DEBUG:boto:Data: 2022-09-03T06:07:26.854 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.855 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.855 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.856 DEBUG:boto:Params: {} 2022-09-03T06:07:26.856 DEBUG:boto:Token: None 2022-09-03T06:07:26.857 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gYhqzxQbA1W34gvMi6LoszeB71c= 2022-09-03T06:07:26.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gYhqzxQbA1W34gvMi6LoszeB71c=', 'Content-Length': '0'} 2022-09-03T06:07:26.860 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c3da012eaba27c52-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.861 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.862 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.862 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.863 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.863 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.864 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.864 DEBUG:boto:Data: 2022-09-03T06:07:26.865 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.866 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.866 DEBUG:boto:Params: {} 2022-09-03T06:07:26.867 DEBUG:boto:Token: None 2022-09-03T06:07:26.867 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KBTCGfIFDek5++fkj2xrOsTn79s= 2022-09-03T06:07:26.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KBTCGfIFDek5++fkj2xrOsTn79s=', 'Content-Length': '0'} 2022-09-03T06:07:26.871 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000005e862dc9fc3b0f2-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.871 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.872 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.872 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.873 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.873 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.874 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.874 DEBUG:boto:Data: 2022-09-03T06:07:26.875 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.876 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.876 DEBUG:boto:Params: {} 2022-09-03T06:07:26.877 DEBUG:boto:Token: None 2022-09-03T06:07:26.877 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KBTCGfIFDek5++fkj2xrOsTn79s= 2022-09-03T06:07:26.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KBTCGfIFDek5++fkj2xrOsTn79s=', 'Content-Length': '0'} 2022-09-03T06:07:26.881 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c59beec0ea4321f9-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.882 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.882 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.883 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.884 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.884 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.885 DEBUG:boto:Method: GET 2022-09-03T06:07:26.885 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.886 DEBUG:boto:Data: 2022-09-03T06:07:26.886 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.887 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.887 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.888 DEBUG:boto:Params: {} 2022-09-03T06:07:26.888 DEBUG:boto:Token: None 2022-09-03T06:07:26.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gYhqzxQbA1W34gvMi6LoszeB71c= 2022-09-03T06:07:26.891 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gYhqzxQbA1W34gvMi6LoszeB71c=', 'Content-Length': '0'} 2022-09-03T06:07:26.893 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006e6fa7688ef2973a-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.894 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.894 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.895 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.896 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.896 DEBUG:boto:Method: GET 2022-09-03T06:07:26.897 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.897 DEBUG:boto:Data: 2022-09-03T06:07:26.898 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.899 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.899 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.900 DEBUG:boto:Params: {} 2022-09-03T06:07:26.900 DEBUG:boto:Token: None 2022-09-03T06:07:26.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:26.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gYhqzxQbA1W34gvMi6LoszeB71c= 2022-09-03T06:07:26.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gYhqzxQbA1W34gvMi6LoszeB71c=', 'Content-Length': '0'} 2022-09-03T06:07:26.905 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002836914a8c1fcb94-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.906 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.906 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.907 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.907 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.908 DEBUG:boto:Method: GET 2022-09-03T06:07:26.908 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.909 DEBUG:boto:Data: 2022-09-03T06:07:26.909 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.910 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.910 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.911 DEBUG:boto:Params: {} 2022-09-03T06:07:26.912 DEBUG:boto:Token: None 2022-09-03T06:07:26.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.916 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006e3502b4fbb7209b-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.917 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.918 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:26.919 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:26.919 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.920 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:26.920 DEBUG:boto:Method: GET 2022-09-03T06:07:26.921 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:26.921 DEBUG:boto:Data: 2022-09-03T06:07:26.922 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.923 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.923 DEBUG:boto:Params: {} 2022-09-03T06:07:26.924 DEBUG:boto:Token: None 2022-09-03T06:07:26.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/ 2022-09-03T06:07:26.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw= 2022-09-03T06:07:26.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZyghdII6/C6Ils33GWC5IeFVArw=', 'Content-Length': '0'} 2022-09-03T06:07:26.930 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001dae8f2ee7bad0ae-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.931 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:26.932 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.932 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.933 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.933 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:26.934 DEBUG:boto:Data: 2022-09-03T06:07:26.934 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.935 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.935 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.936 DEBUG:boto:Params: {} 2022-09-03T06:07:26.936 DEBUG:boto:Token: None 2022-09-03T06:07:26.937 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj 2022-09-03T06:07:26.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI= 2022-09-03T06:07:26.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI=', 'Content-Length': '0'} 2022-09-03T06:07:26.941 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c4afab57bf78a807-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.942 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.943 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.943 DEBUG:boto:Method: HEAD 2022-09-03T06:07:26.944 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:26.944 DEBUG:boto:Data: 2022-09-03T06:07:26.945 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.945 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.946 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.946 DEBUG:boto:Params: {} 2022-09-03T06:07:26.947 DEBUG:boto:Token: None 2022-09-03T06:07:26.948 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj 2022-09-03T06:07:26.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI= 2022-09-03T06:07:26.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/0WTPVks+kF8yrl3/P9FzRbIPI=', 'Content-Length': '0'} 2022-09-03T06:07:26.952 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002b369f1671332fa0-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.954 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:26.955 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.956 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.957 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.958 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.959 DEBUG:boto:Method: GET 2022-09-03T06:07:26.960 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.962 DEBUG:boto:Data: 2022-09-03T06:07:26.962 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:26.963 DEBUG:boto:Port: 8001 2022-09-03T06:07:26.964 DEBUG:boto:Params: {} 2022-09-03T06:07:26.965 DEBUG:boto:Token: None 2022-09-03T06:07:26.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.969 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005f59a3ca6ebb606f-006312ef1e-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:26.970 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:26.971 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:26.971 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.972 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.972 DEBUG:boto:Method: GET 2022-09-03T06:07:26.973 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.973 DEBUG:boto:Data: 2022-09-03T06:07:26.974 DEBUG:boto:Headers: {} 2022-09-03T06:07:26.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:26.975 DEBUG:boto:Port: 8000 2022-09-03T06:07:26.976 DEBUG:boto:Params: {} 2022-09-03T06:07:26.976 DEBUG:boto:Token: None 2022-09-03T06:07:26.977 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:26 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:26.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE= 2022-09-03T06:07:26.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZCUlC36xFiQcew8kZfoWImGKkE=', 'Content-Length': '0'} 2022-09-03T06:07:26.980 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002f824dcb697a535a-006312ef1e-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.026 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.027 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.028 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.029 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.030 DEBUG:boto:Method: GET 2022-09-03T06:07:27.031 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.032 DEBUG:boto:Data: 2022-09-03T06:07:27.033 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.034 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.035 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.035 DEBUG:boto:Params: {} 2022-09-03T06:07:27.036 DEBUG:boto:Token: None 2022-09-03T06:07:27.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.041 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1e252505b8e127c-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.042 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.043 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.044 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.045 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.046 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.047 DEBUG:boto:Method: GET 2022-09-03T06:07:27.048 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.049 DEBUG:boto:Data: 2022-09-03T06:07:27.050 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.052 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.052 DEBUG:boto:Params: {} 2022-09-03T06:07:27.054 DEBUG:boto:Token: None 2022-09-03T06:07:27.054 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.059 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a9c2d625c5e9d2a8-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.059 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.060 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.061 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.061 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.062 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.063 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.064 DEBUG:boto:Method: GET 2022-09-03T06:07:27.064 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.065 DEBUG:boto:Data: 2022-09-03T06:07:27.065 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.066 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.067 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.068 DEBUG:boto:Params: {} 2022-09-03T06:07:27.069 DEBUG:boto:Token: None 2022-09-03T06:07:27.070 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qZyEcNxIUnJf3sSNBIrpUQjHhgY= 2022-09-03T06:07:27.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qZyEcNxIUnJf3sSNBIrpUQjHhgY=', 'Content-Length': '0'} 2022-09-03T06:07:27.075 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000074bce4aae79f5a6a-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.076 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.077 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.078 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.079 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.079 DEBUG:boto:Method: GET 2022-09-03T06:07:27.080 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.081 DEBUG:boto:Data: 2022-09-03T06:07:27.081 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.082 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.083 DEBUG:boto:Params: {} 2022-09-03T06:07:27.083 DEBUG:boto:Token: None 2022-09-03T06:07:27.084 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qZyEcNxIUnJf3sSNBIrpUQjHhgY= 2022-09-03T06:07:27.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qZyEcNxIUnJf3sSNBIrpUQjHhgY=', 'Content-Length': '0'} 2022-09-03T06:07:27.087 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000172768c15bffa5d5-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.088 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.089 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.089 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.090 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.090 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.091 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.091 DEBUG:boto:Data: 2022-09-03T06:07:27.092 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.093 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.093 DEBUG:boto:Params: {} 2022-09-03T06:07:27.094 DEBUG:boto:Token: None 2022-09-03T06:07:27.095 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k9140WobFJX52/vjfWhMUoqfoto= 2022-09-03T06:07:27.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k9140WobFJX52/vjfWhMUoqfoto=', 'Content-Length': '0'} 2022-09-03T06:07:27.098 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000af8ae0854f0bf857-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.099 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.100 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.100 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.101 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.101 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.102 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.102 DEBUG:boto:Data: 2022-09-03T06:07:27.103 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.104 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.105 DEBUG:boto:Params: {} 2022-09-03T06:07:27.106 DEBUG:boto:Token: None 2022-09-03T06:07:27.107 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.107 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k9140WobFJX52/vjfWhMUoqfoto= 2022-09-03T06:07:27.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k9140WobFJX52/vjfWhMUoqfoto=', 'Content-Length': '0'} 2022-09-03T06:07:27.110 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000028b6ba9ec651ba5f-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.111 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.111 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.112 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.113 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.113 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.114 DEBUG:boto:Method: GET 2022-09-03T06:07:27.114 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.115 DEBUG:boto:Data: 2022-09-03T06:07:27.116 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.117 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.117 DEBUG:boto:Params: {} 2022-09-03T06:07:27.118 DEBUG:boto:Token: None 2022-09-03T06:07:27.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qZyEcNxIUnJf3sSNBIrpUQjHhgY= 2022-09-03T06:07:27.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qZyEcNxIUnJf3sSNBIrpUQjHhgY=', 'Content-Length': '0'} 2022-09-03T06:07:27.122 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f25a8975e286e8d6-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.166 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.167 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.167 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.168 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.168 DEBUG:boto:Method: GET 2022-09-03T06:07:27.169 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.170 DEBUG:boto:Data: 2022-09-03T06:07:27.170 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.172 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.173 DEBUG:boto:Params: {} 2022-09-03T06:07:27.174 DEBUG:boto:Token: None 2022-09-03T06:07:27.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:27.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qZyEcNxIUnJf3sSNBIrpUQjHhgY= 2022-09-03T06:07:27.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qZyEcNxIUnJf3sSNBIrpUQjHhgY=', 'Content-Length': '0'} 2022-09-03T06:07:27.180 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d6934d9ae74e2c64-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.180 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.181 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.182 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.182 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.183 DEBUG:boto:Method: GET 2022-09-03T06:07:27.183 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.184 DEBUG:boto:Data: 2022-09-03T06:07:27.184 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.185 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.186 DEBUG:boto:Params: {} 2022-09-03T06:07:27.187 DEBUG:boto:Token: None 2022-09-03T06:07:27.187 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.191 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007481c2dd943b6c46-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.192 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.193 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.194 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.194 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.195 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.195 DEBUG:boto:Method: GET 2022-09-03T06:07:27.196 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.196 DEBUG:boto:Data: 2022-09-03T06:07:27.197 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.198 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.198 DEBUG:boto:Params: {} 2022-09-03T06:07:27.199 DEBUG:boto:Token: None 2022-09-03T06:07:27.200 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.204 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007aa5dd87545025b7-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.205 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.206 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.206 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.207 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.207 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:27.208 DEBUG:boto:Data: 2022-09-03T06:07:27.208 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.209 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.209 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.210 DEBUG:boto:Params: {} 2022-09-03T06:07:27.211 DEBUG:boto:Token: None 2022-09-03T06:07:27.211 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj 2022-09-03T06:07:27.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o= 2022-09-03T06:07:27.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o=', 'Content-Length': '0'} 2022-09-03T06:07:27.217 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000560aec9ff63fa51a-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.218 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.218 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.219 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.219 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:27.220 DEBUG:boto:Data: 2022-09-03T06:07:27.220 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.221 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.222 DEBUG:boto:Params: {} 2022-09-03T06:07:27.223 DEBUG:boto:Token: None 2022-09-03T06:07:27.224 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj 2022-09-03T06:07:27.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o= 2022-09-03T06:07:27.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o=', 'Content-Length': '0'} 2022-09-03T06:07:27.229 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000070d5c39680e248fc-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.229 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.230 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.231 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.231 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.232 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.232 DEBUG:boto:Method: GET 2022-09-03T06:07:27.233 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.233 DEBUG:boto:Data: 2022-09-03T06:07:27.234 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.235 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.235 DEBUG:boto:Params: {} 2022-09-03T06:07:27.236 DEBUG:boto:Token: None 2022-09-03T06:07:27.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ= 2022-09-03T06:07:27.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ=', 'Content-Length': '0'} 2022-09-03T06:07:27.240 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000078a25857f224556b-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.282 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.284 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.285 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.285 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.286 DEBUG:boto:Method: GET 2022-09-03T06:07:27.288 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.289 DEBUG:boto:Data: 2022-09-03T06:07:27.290 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.291 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.292 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.294 DEBUG:boto:Params: {} 2022-09-03T06:07:27.295 DEBUG:boto:Token: None 2022-09-03T06:07:27.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ= 2022-09-03T06:07:27.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ=', 'Content-Length': '0'} 2022-09-03T06:07:27.301 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b72d0681fd5b30a8-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.303 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.304 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.305 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.306 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.307 DEBUG:boto:Method: GET 2022-09-03T06:07:27.308 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.310 DEBUG:boto:Data: 2022-09-03T06:07:27.311 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.313 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.314 DEBUG:boto:Params: {} 2022-09-03T06:07:27.315 DEBUG:boto:Token: None 2022-09-03T06:07:27.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.323 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000482acf9f449fb194-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.324 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.326 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.326 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.327 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.327 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.328 DEBUG:boto:Method: GET 2022-09-03T06:07:27.328 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.329 DEBUG:boto:Data: 2022-09-03T06:07:27.330 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.331 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.331 DEBUG:boto:Params: {} 2022-09-03T06:07:27.332 DEBUG:boto:Token: None 2022-09-03T06:07:27.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.337 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f598db0e364b8bbf-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.338 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.339 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.339 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.340 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.340 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.341 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.342 DEBUG:boto:Method: GET 2022-09-03T06:07:27.342 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.343 DEBUG:boto:Data: 2022-09-03T06:07:27.343 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.344 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.345 DEBUG:boto:Params: {} 2022-09-03T06:07:27.345 DEBUG:boto:Token: None 2022-09-03T06:07:27.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lqpJIT51TBBXTBSapZZMZHy4cTU= 2022-09-03T06:07:27.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lqpJIT51TBBXTBSapZZMZHy4cTU=', 'Content-Length': '0'} 2022-09-03T06:07:27.350 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004d8ca9a26578869d-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.351 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.351 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.352 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.352 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.353 DEBUG:boto:Method: GET 2022-09-03T06:07:27.353 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.354 DEBUG:boto:Data: 2022-09-03T06:07:27.354 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.355 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.356 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.356 DEBUG:boto:Params: {} 2022-09-03T06:07:27.357 DEBUG:boto:Token: None 2022-09-03T06:07:27.358 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lqpJIT51TBBXTBSapZZMZHy4cTU= 2022-09-03T06:07:27.359 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lqpJIT51TBBXTBSapZZMZHy4cTU=', 'Content-Length': '0'} 2022-09-03T06:07:27.361 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006c96e7071bd914a6-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.362 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.363 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.363 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.364 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.364 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.365 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.366 DEBUG:boto:Data: 2022-09-03T06:07:27.366 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.367 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.368 DEBUG:boto:Params: {} 2022-09-03T06:07:27.368 DEBUG:boto:Token: None 2022-09-03T06:07:27.369 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DjU2fQjkyMMXlzf/t3GXm4hHt8c= 2022-09-03T06:07:27.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DjU2fQjkyMMXlzf/t3GXm4hHt8c=', 'Content-Length': '0'} 2022-09-03T06:07:27.372 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000034f85a3665f4b0e2-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.373 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.373 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.374 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.374 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.375 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.376 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.376 DEBUG:boto:Data: 2022-09-03T06:07:27.377 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.378 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.378 DEBUG:boto:Params: {} 2022-09-03T06:07:27.379 DEBUG:boto:Token: None 2022-09-03T06:07:27.379 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DjU2fQjkyMMXlzf/t3GXm4hHt8c= 2022-09-03T06:07:27.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DjU2fQjkyMMXlzf/t3GXm4hHt8c=', 'Content-Length': '0'} 2022-09-03T06:07:27.383 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002a26fe91a38ab917-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.383 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.384 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.385 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.385 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.386 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.386 DEBUG:boto:Method: GET 2022-09-03T06:07:27.387 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.387 DEBUG:boto:Data: 2022-09-03T06:07:27.388 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.389 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.390 DEBUG:boto:Params: {} 2022-09-03T06:07:27.390 DEBUG:boto:Token: None 2022-09-03T06:07:27.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lqpJIT51TBBXTBSapZZMZHy4cTU= 2022-09-03T06:07:27.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lqpJIT51TBBXTBSapZZMZHy4cTU=', 'Content-Length': '0'} 2022-09-03T06:07:27.394 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000043a5f9d17cd405e-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.395 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.396 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.396 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.397 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.398 DEBUG:boto:Method: GET 2022-09-03T06:07:27.398 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.399 DEBUG:boto:Data: 2022-09-03T06:07:27.399 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.400 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.401 DEBUG:boto:Params: {} 2022-09-03T06:07:27.402 DEBUG:boto:Token: None 2022-09-03T06:07:27.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:27.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lqpJIT51TBBXTBSapZZMZHy4cTU= 2022-09-03T06:07:27.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lqpJIT51TBBXTBSapZZMZHy4cTU=', 'Content-Length': '0'} 2022-09-03T06:07:27.406 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000035145acec25ae694-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.450 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.451 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.452 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.452 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.453 DEBUG:boto:Method: GET 2022-09-03T06:07:27.453 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.454 DEBUG:boto:Data: 2022-09-03T06:07:27.455 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.455 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.456 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.457 DEBUG:boto:Params: {} 2022-09-03T06:07:27.458 DEBUG:boto:Token: None 2022-09-03T06:07:27.459 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.461 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.464 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028e7e74efe66f717-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.465 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.466 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.467 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.468 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.468 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.469 DEBUG:boto:Method: GET 2022-09-03T06:07:27.469 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.470 DEBUG:boto:Data: 2022-09-03T06:07:27.470 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.471 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.472 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.473 DEBUG:boto:Params: {} 2022-09-03T06:07:27.474 DEBUG:boto:Token: None 2022-09-03T06:07:27.475 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.480 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f2e26c8612ffbc01-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.481 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.482 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.482 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.483 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.483 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:27.484 DEBUG:boto:Data: 2022-09-03T06:07:27.485 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.486 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.486 DEBUG:boto:Params: {} 2022-09-03T06:07:27.487 DEBUG:boto:Token: None 2022-09-03T06:07:27.487 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj 2022-09-03T06:07:27.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o= 2022-09-03T06:07:27.489 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o=', 'Content-Length': '0'} 2022-09-03T06:07:27.493 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be73fadde6e8eeb7-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.494 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.495 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.496 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.497 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:27.498 DEBUG:boto:Data: 2022-09-03T06:07:27.498 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.500 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.501 DEBUG:boto:Params: {} 2022-09-03T06:07:27.502 DEBUG:boto:Token: None 2022-09-03T06:07:27.503 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj 2022-09-03T06:07:27.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o= 2022-09-03T06:07:27.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o=', 'Content-Length': '0'} 2022-09-03T06:07:27.508 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006c4a5385563b2c69-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.509 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.509 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.510 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.510 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.511 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.511 DEBUG:boto:Method: GET 2022-09-03T06:07:27.512 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.512 DEBUG:boto:Data: 2022-09-03T06:07:27.513 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.514 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.515 DEBUG:boto:Params: {} 2022-09-03T06:07:27.516 DEBUG:boto:Token: None 2022-09-03T06:07:27.516 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ= 2022-09-03T06:07:27.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ=', 'Content-Length': '0'} 2022-09-03T06:07:27.520 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000583ea1530ea97c1a-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.562 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.563 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.563 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.564 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.565 DEBUG:boto:Method: GET 2022-09-03T06:07:27.565 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.566 DEBUG:boto:Data: 2022-09-03T06:07:27.566 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.567 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.568 DEBUG:boto:Params: {} 2022-09-03T06:07:27.568 DEBUG:boto:Token: None 2022-09-03T06:07:27.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ= 2022-09-03T06:07:27.570 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ=', 'Content-Length': '0'} 2022-09-03T06:07:27.573 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000976a5c6d320d2702-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.573 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.574 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.574 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.575 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.576 DEBUG:boto:Method: GET 2022-09-03T06:07:27.576 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.577 DEBUG:boto:Data: 2022-09-03T06:07:27.577 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.578 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.578 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.579 DEBUG:boto:Params: {} 2022-09-03T06:07:27.579 DEBUG:boto:Token: None 2022-09-03T06:07:27.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.581 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.584 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aebdf7303869c44d-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.585 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.586 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.587 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.587 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.588 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.588 DEBUG:boto:Method: GET 2022-09-03T06:07:27.589 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.589 DEBUG:boto:Data: 2022-09-03T06:07:27.590 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.590 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.591 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.591 DEBUG:boto:Params: {} 2022-09-03T06:07:27.592 DEBUG:boto:Token: None 2022-09-03T06:07:27.592 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.596 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000041636689c43549e9-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.597 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.598 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.598 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.599 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.599 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.600 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.600 DEBUG:boto:Method: GET 2022-09-03T06:07:27.601 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.602 DEBUG:boto:Data: 2022-09-03T06:07:27.602 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.603 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.604 DEBUG:boto:Params: {} 2022-09-03T06:07:27.605 DEBUG:boto:Token: None 2022-09-03T06:07:27.605 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fljo4pmnt2ckOKrDXnFFx4uDEBo= 2022-09-03T06:07:27.607 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fljo4pmnt2ckOKrDXnFFx4uDEBo=', 'Content-Length': '0'} 2022-09-03T06:07:27.609 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000074f51f58bde75af0-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.610 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.610 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.611 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.611 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.612 DEBUG:boto:Method: GET 2022-09-03T06:07:27.613 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.613 DEBUG:boto:Data: 2022-09-03T06:07:27.614 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.615 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.615 DEBUG:boto:Params: {} 2022-09-03T06:07:27.616 DEBUG:boto:Token: None 2022-09-03T06:07:27.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fljo4pmnt2ckOKrDXnFFx4uDEBo= 2022-09-03T06:07:27.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fljo4pmnt2ckOKrDXnFFx4uDEBo=', 'Content-Length': '0'} 2022-09-03T06:07:27.620 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002c2bb65ea2f5c36b-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.621 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.621 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.622 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.622 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.623 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.623 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.624 DEBUG:boto:Data: 2022-09-03T06:07:27.624 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.625 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.626 DEBUG:boto:Params: {} 2022-09-03T06:07:27.626 DEBUG:boto:Token: None 2022-09-03T06:07:27.627 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eJmLrGSrBqqVvfDKS5UtrD6nxgk= 2022-09-03T06:07:27.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eJmLrGSrBqqVvfDKS5UtrD6nxgk=', 'Content-Length': '0'} 2022-09-03T06:07:27.630 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b4d23b6d5930fcd1-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.631 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.632 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.632 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.633 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.633 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.634 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.634 DEBUG:boto:Data: 2022-09-03T06:07:27.635 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.636 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.636 DEBUG:boto:Params: {} 2022-09-03T06:07:27.637 DEBUG:boto:Token: None 2022-09-03T06:07:27.637 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eJmLrGSrBqqVvfDKS5UtrD6nxgk= 2022-09-03T06:07:27.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eJmLrGSrBqqVvfDKS5UtrD6nxgk=', 'Content-Length': '0'} 2022-09-03T06:07:27.641 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bfa8e311d306aafc-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.642 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.642 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.643 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.643 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.644 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.644 DEBUG:boto:Method: GET 2022-09-03T06:07:27.645 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.645 DEBUG:boto:Data: 2022-09-03T06:07:27.646 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.647 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.648 DEBUG:boto:Params: {} 2022-09-03T06:07:27.648 DEBUG:boto:Token: None 2022-09-03T06:07:27.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fljo4pmnt2ckOKrDXnFFx4uDEBo= 2022-09-03T06:07:27.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fljo4pmnt2ckOKrDXnFFx4uDEBo=', 'Content-Length': '0'} 2022-09-03T06:07:27.652 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000089acc84eb6746af3-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.653 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.654 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.655 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.655 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.656 DEBUG:boto:Method: GET 2022-09-03T06:07:27.656 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.657 DEBUG:boto:Data: 2022-09-03T06:07:27.658 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.660 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.661 DEBUG:boto:Params: {} 2022-09-03T06:07:27.661 DEBUG:boto:Token: None 2022-09-03T06:07:27.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:27.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fljo4pmnt2ckOKrDXnFFx4uDEBo= 2022-09-03T06:07:27.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fljo4pmnt2ckOKrDXnFFx4uDEBo=', 'Content-Length': '0'} 2022-09-03T06:07:27.666 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ec0bd9274b8f7e87-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.667 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.668 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.668 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.669 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.671 DEBUG:boto:Method: GET 2022-09-03T06:07:27.671 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.672 DEBUG:boto:Data: 2022-09-03T06:07:27.673 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.674 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.675 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.676 DEBUG:boto:Params: {} 2022-09-03T06:07:27.677 DEBUG:boto:Token: None 2022-09-03T06:07:27.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.684 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000264134266c89712a-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.685 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.687 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.688 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.689 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.690 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.691 DEBUG:boto:Method: GET 2022-09-03T06:07:27.692 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.692 DEBUG:boto:Data: 2022-09-03T06:07:27.693 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.694 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.694 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.695 DEBUG:boto:Params: {} 2022-09-03T06:07:27.696 DEBUG:boto:Token: None 2022-09-03T06:07:27.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.701 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ed78c4c371c2216-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.702 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.703 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.703 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.704 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.704 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:27.705 DEBUG:boto:Data: 2022-09-03T06:07:27.705 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.706 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.706 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.707 DEBUG:boto:Params: {} 2022-09-03T06:07:27.708 DEBUG:boto:Token: None 2022-09-03T06:07:27.708 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj 2022-09-03T06:07:27.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o= 2022-09-03T06:07:27.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o=', 'Content-Length': '0'} 2022-09-03T06:07:27.713 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b712e85b18608447-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.714 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.715 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.716 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.716 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:27.717 DEBUG:boto:Data: 2022-09-03T06:07:27.717 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.718 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.719 DEBUG:boto:Params: {} 2022-09-03T06:07:27.720 DEBUG:boto:Token: None 2022-09-03T06:07:27.721 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj 2022-09-03T06:07:27.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o= 2022-09-03T06:07:27.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o=', 'Content-Length': '0'} 2022-09-03T06:07:27.725 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007f09e16f47390348-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.726 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.727 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.727 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.728 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.729 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.729 DEBUG:boto:Method: GET 2022-09-03T06:07:27.730 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.730 DEBUG:boto:Data: 2022-09-03T06:07:27.731 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.732 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.732 DEBUG:boto:Params: {} 2022-09-03T06:07:27.733 DEBUG:boto:Token: None 2022-09-03T06:07:27.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ= 2022-09-03T06:07:27.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ=', 'Content-Length': '0'} 2022-09-03T06:07:27.737 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f3302c08f279212f-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.738 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.739 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.739 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.740 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.740 DEBUG:boto:Method: GET 2022-09-03T06:07:27.741 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.741 DEBUG:boto:Data: 2022-09-03T06:07:27.742 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.743 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.743 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.744 DEBUG:boto:Params: {} 2022-09-03T06:07:27.744 DEBUG:boto:Token: None 2022-09-03T06:07:27.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ= 2022-09-03T06:07:27.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ=', 'Content-Length': '0'} 2022-09-03T06:07:27.748 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b1242c8a374240c7-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.749 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.750 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.750 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.751 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.752 DEBUG:boto:Method: GET 2022-09-03T06:07:27.752 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.753 DEBUG:boto:Data: 2022-09-03T06:07:27.753 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.754 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.755 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.755 DEBUG:boto:Params: {} 2022-09-03T06:07:27.756 DEBUG:boto:Token: None 2022-09-03T06:07:27.756 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.758 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.760 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ba0b4b7c8ee75382-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.761 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.762 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.762 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.763 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.764 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.764 DEBUG:boto:Method: GET 2022-09-03T06:07:27.765 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.765 DEBUG:boto:Data: 2022-09-03T06:07:27.766 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.766 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.767 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.768 DEBUG:boto:Params: {} 2022-09-03T06:07:27.768 DEBUG:boto:Token: None 2022-09-03T06:07:27.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.773 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e58d84cac5dfbfa1-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.773 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.774 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.775 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.776 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.776 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.777 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.777 DEBUG:boto:Method: GET 2022-09-03T06:07:27.778 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.779 DEBUG:boto:Data: 2022-09-03T06:07:27.779 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.781 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.782 DEBUG:boto:Params: {} 2022-09-03T06:07:27.784 DEBUG:boto:Token: None 2022-09-03T06:07:27.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bgG/yWbMfjqpuf/0+pK77NtNdos= 2022-09-03T06:07:27.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bgG/yWbMfjqpuf/0+pK77NtNdos=', 'Content-Length': '0'} 2022-09-03T06:07:27.788 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be2feb8e7f66d2f6-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.789 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.790 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.791 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.791 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.792 DEBUG:boto:Method: GET 2022-09-03T06:07:27.792 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.793 DEBUG:boto:Data: 2022-09-03T06:07:27.794 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.794 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.795 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.795 DEBUG:boto:Params: {} 2022-09-03T06:07:27.796 DEBUG:boto:Token: None 2022-09-03T06:07:27.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bgG/yWbMfjqpuf/0+pK77NtNdos= 2022-09-03T06:07:27.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bgG/yWbMfjqpuf/0+pK77NtNdos=', 'Content-Length': '0'} 2022-09-03T06:07:27.803 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a7051bec64c6d05f-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.804 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.805 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.806 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.807 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.807 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.808 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.809 DEBUG:boto:Data: 2022-09-03T06:07:27.809 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.810 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.811 DEBUG:boto:Params: {} 2022-09-03T06:07:27.811 DEBUG:boto:Token: None 2022-09-03T06:07:27.812 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xw0gWNLvVHApmZZgOwVZPrq3SNU= 2022-09-03T06:07:27.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xw0gWNLvVHApmZZgOwVZPrq3SNU=', 'Content-Length': '0'} 2022-09-03T06:07:27.815 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be29d77fe1619b3f-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.816 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.817 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.817 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.818 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.818 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.819 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.819 DEBUG:boto:Data: 2022-09-03T06:07:27.820 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.821 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.821 DEBUG:boto:Params: {} 2022-09-03T06:07:27.822 DEBUG:boto:Token: None 2022-09-03T06:07:27.822 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xw0gWNLvVHApmZZgOwVZPrq3SNU= 2022-09-03T06:07:27.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xw0gWNLvVHApmZZgOwVZPrq3SNU=', 'Content-Length': '0'} 2022-09-03T06:07:27.826 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000259e29784ef10e91-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.827 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.827 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.828 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.828 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.829 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.829 DEBUG:boto:Method: GET 2022-09-03T06:07:27.830 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.831 DEBUG:boto:Data: 2022-09-03T06:07:27.831 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.832 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.833 DEBUG:boto:Params: {} 2022-09-03T06:07:27.834 DEBUG:boto:Token: None 2022-09-03T06:07:27.834 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bgG/yWbMfjqpuf/0+pK77NtNdos= 2022-09-03T06:07:27.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bgG/yWbMfjqpuf/0+pK77NtNdos=', 'Content-Length': '0'} 2022-09-03T06:07:27.838 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000001d359a32df695a3-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.839 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.839 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.840 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.841 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.842 DEBUG:boto:Method: GET 2022-09-03T06:07:27.843 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.844 DEBUG:boto:Data: 2022-09-03T06:07:27.845 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.847 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.848 DEBUG:boto:Params: {} 2022-09-03T06:07:27.849 DEBUG:boto:Token: None 2022-09-03T06:07:27.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:27.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bgG/yWbMfjqpuf/0+pK77NtNdos= 2022-09-03T06:07:27.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bgG/yWbMfjqpuf/0+pK77NtNdos=', 'Content-Length': '0'} 2022-09-03T06:07:27.855 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006369657bab673669-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.898 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.900 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.901 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.902 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.902 DEBUG:boto:Method: GET 2022-09-03T06:07:27.903 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.904 DEBUG:boto:Data: 2022-09-03T06:07:27.904 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.905 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.906 DEBUG:boto:Params: {} 2022-09-03T06:07:27.906 DEBUG:boto:Token: None 2022-09-03T06:07:27.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.912 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fa820abd7519e65e-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.913 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.914 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:27.914 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:27.915 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.916 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:27.917 DEBUG:boto:Method: GET 2022-09-03T06:07:27.918 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:27.918 DEBUG:boto:Data: 2022-09-03T06:07:27.919 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.921 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.922 DEBUG:boto:Params: {} 2022-09-03T06:07:27.923 DEBUG:boto:Token: None 2022-09-03T06:07:27.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/ 2022-09-03T06:07:27.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE= 2022-09-03T06:07:27.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xoi3TmMeztD7z1Hla1aVqixvUYE=', 'Content-Length': '0'} 2022-09-03T06:07:27.929 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000612dbf8e8040f69d-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.930 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:27.931 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.932 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.933 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.934 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:27.935 DEBUG:boto:Data: 2022-09-03T06:07:27.936 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.937 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.937 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.938 DEBUG:boto:Params: {} 2022-09-03T06:07:27.939 DEBUG:boto:Token: None 2022-09-03T06:07:27.939 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj 2022-09-03T06:07:27.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o= 2022-09-03T06:07:27.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o=', 'Content-Length': '0'} 2022-09-03T06:07:27.944 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f218df90b59b5bbb-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.945 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.945 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.946 DEBUG:boto:Method: HEAD 2022-09-03T06:07:27.946 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:27.947 DEBUG:boto:Data: 2022-09-03T06:07:27.947 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:27.948 DEBUG:boto:Port: 8000 2022-09-03T06:07:27.949 DEBUG:boto:Params: {} 2022-09-03T06:07:27.950 DEBUG:boto:Token: None 2022-09-03T06:07:27.950 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj 2022-09-03T06:07:27.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o= 2022-09-03T06:07:27.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dtMJKLEFs0xneqnSIxx4bVUDh2o=', 'Content-Length': '0'} 2022-09-03T06:07:27.955 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006af2b2c89ba5fd66-006312ef1f-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:27.956 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:27.957 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:27.957 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:27.958 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.959 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.959 DEBUG:boto:Method: GET 2022-09-03T06:07:27.960 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.960 DEBUG:boto:Data: 2022-09-03T06:07:27.961 DEBUG:boto:Headers: {} 2022-09-03T06:07:27.961 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:27.962 DEBUG:boto:Port: 8001 2022-09-03T06:07:27.963 DEBUG:boto:Params: {} 2022-09-03T06:07:27.964 DEBUG:boto:Token: None 2022-09-03T06:07:27.964 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:27 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:27.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ= 2022-09-03T06:07:27.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:stW6dskSadKS5WKKL0tcBTpBJtQ=', 'Content-Length': '0'} 2022-09-03T06:07:27.969 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cd440331234e6233-006312ef1f-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.012 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.013 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.013 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.014 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.015 DEBUG:boto:Method: GET 2022-09-03T06:07:28.016 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.016 DEBUG:boto:Data: 2022-09-03T06:07:28.017 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.018 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.019 DEBUG:boto:Params: {} 2022-09-03T06:07:28.020 DEBUG:boto:Token: None 2022-09-03T06:07:28.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.021 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4= 2022-09-03T06:07:28.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4=', 'Content-Length': '0'} 2022-09-03T06:07:28.024 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e2c610dae7e46967-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.025 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.026 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.026 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.027 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.028 DEBUG:boto:Method: GET 2022-09-03T06:07:28.029 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.030 DEBUG:boto:Data: 2022-09-03T06:07:28.031 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.033 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.034 DEBUG:boto:Params: {} 2022-09-03T06:07:28.036 DEBUG:boto:Token: None 2022-09-03T06:07:28.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.043 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004dbef0afae5cb792-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.044 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.045 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.046 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.048 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.049 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.049 DEBUG:boto:Method: GET 2022-09-03T06:07:28.051 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.052 DEBUG:boto:Data: 2022-09-03T06:07:28.053 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.055 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.056 DEBUG:boto:Params: {} 2022-09-03T06:07:28.057 DEBUG:boto:Token: None 2022-09-03T06:07:28.058 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.064 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d301471ba5c5095f-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.065 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.065 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.066 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.067 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.067 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.068 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.068 DEBUG:boto:Method: GET 2022-09-03T06:07:28.069 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.069 DEBUG:boto:Data: 2022-09-03T06:07:28.070 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.070 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.071 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.071 DEBUG:boto:Params: {} 2022-09-03T06:07:28.072 DEBUG:boto:Token: None 2022-09-03T06:07:28.073 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Znu5Of/n8i2e3gdYF25HAiLNLrQ= 2022-09-03T06:07:28.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Znu5Of/n8i2e3gdYF25HAiLNLrQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.076 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bcb0a7fcf109d4de-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.077 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.078 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.078 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.079 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.079 DEBUG:boto:Method: GET 2022-09-03T06:07:28.080 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.081 DEBUG:boto:Data: 2022-09-03T06:07:28.081 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.082 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.083 DEBUG:boto:Params: {} 2022-09-03T06:07:28.084 DEBUG:boto:Token: None 2022-09-03T06:07:28.084 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Znu5Of/n8i2e3gdYF25HAiLNLrQ= 2022-09-03T06:07:28.086 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Znu5Of/n8i2e3gdYF25HAiLNLrQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.088 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000092e229ec30d6446c-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.089 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.089 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.090 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.090 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.091 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.091 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.092 DEBUG:boto:Data: 2022-09-03T06:07:28.092 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.093 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.094 DEBUG:boto:Params: {} 2022-09-03T06:07:28.094 DEBUG:boto:Token: None 2022-09-03T06:07:28.095 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:85Rphl8fc8CO3EGz0/BvFmHgM44= 2022-09-03T06:07:28.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:85Rphl8fc8CO3EGz0/BvFmHgM44=', 'Content-Length': '0'} 2022-09-03T06:07:28.098 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007c33808c41acd5c1-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.099 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.100 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.100 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.101 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.101 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.102 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.102 DEBUG:boto:Data: 2022-09-03T06:07:28.103 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.104 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.104 DEBUG:boto:Params: {} 2022-09-03T06:07:28.105 DEBUG:boto:Token: None 2022-09-03T06:07:28.105 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.106 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:85Rphl8fc8CO3EGz0/BvFmHgM44= 2022-09-03T06:07:28.107 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:85Rphl8fc8CO3EGz0/BvFmHgM44=', 'Content-Length': '0'} 2022-09-03T06:07:28.109 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000038f57788131fe308-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.109 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.110 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.111 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.111 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.112 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.112 DEBUG:boto:Method: GET 2022-09-03T06:07:28.113 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.114 DEBUG:boto:Data: 2022-09-03T06:07:28.114 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.115 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.116 DEBUG:boto:Params: {} 2022-09-03T06:07:28.117 DEBUG:boto:Token: None 2022-09-03T06:07:28.117 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Znu5Of/n8i2e3gdYF25HAiLNLrQ= 2022-09-03T06:07:28.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Znu5Of/n8i2e3gdYF25HAiLNLrQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.121 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e01bd1cc846a02a0-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.162 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.163 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.165 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.166 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.167 DEBUG:boto:Method: GET 2022-09-03T06:07:28.168 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.169 DEBUG:boto:Data: 2022-09-03T06:07:28.170 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.172 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.173 DEBUG:boto:Params: {} 2022-09-03T06:07:28.174 DEBUG:boto:Token: None 2022-09-03T06:07:28.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:28.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Znu5Of/n8i2e3gdYF25HAiLNLrQ= 2022-09-03T06:07:28.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Znu5Of/n8i2e3gdYF25HAiLNLrQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.179 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ed5b1547383b411f-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.180 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.181 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.182 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.183 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.184 DEBUG:boto:Method: GET 2022-09-03T06:07:28.185 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.185 DEBUG:boto:Data: 2022-09-03T06:07:28.187 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.188 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.189 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.189 DEBUG:boto:Params: {} 2022-09-03T06:07:28.190 DEBUG:boto:Token: None 2022-09-03T06:07:28.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.197 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000001bdc9c89f1e30e4-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.198 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.199 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.200 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.200 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.201 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.201 DEBUG:boto:Method: GET 2022-09-03T06:07:28.202 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.202 DEBUG:boto:Data: 2022-09-03T06:07:28.203 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.204 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.205 DEBUG:boto:Params: {} 2022-09-03T06:07:28.205 DEBUG:boto:Token: None 2022-09-03T06:07:28.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.211 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000000c30bd78770f539-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.255 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.256 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.257 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.258 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.258 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:28.259 DEBUG:boto:Data: 2022-09-03T06:07:28.259 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.261 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.261 DEBUG:boto:Params: {} 2022-09-03T06:07:28.262 DEBUG:boto:Token: None 2022-09-03T06:07:28.262 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj 2022-09-03T06:07:28.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ= 2022-09-03T06:07:28.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.268 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c62cb7772ad24929-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.268 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.269 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.270 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.270 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:28.271 DEBUG:boto:Data: 2022-09-03T06:07:28.271 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.272 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.272 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.273 DEBUG:boto:Params: {} 2022-09-03T06:07:28.274 DEBUG:boto:Token: None 2022-09-03T06:07:28.274 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj 2022-09-03T06:07:28.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ= 2022-09-03T06:07:28.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.280 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007a8ef94496e422e7-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.281 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.282 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.283 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.285 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.286 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.287 DEBUG:boto:Method: GET 2022-09-03T06:07:28.289 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.290 DEBUG:boto:Data: 2022-09-03T06:07:28.290 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.291 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.292 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.292 DEBUG:boto:Params: {} 2022-09-03T06:07:28.293 DEBUG:boto:Token: None 2022-09-03T06:07:28.294 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4= 2022-09-03T06:07:28.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4=', 'Content-Length': '0'} 2022-09-03T06:07:28.300 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000964c7cf6161066d1-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.302 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.303 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.303 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.304 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.304 DEBUG:boto:Method: GET 2022-09-03T06:07:28.307 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.307 DEBUG:boto:Data: 2022-09-03T06:07:28.308 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.310 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.310 DEBUG:boto:Params: {} 2022-09-03T06:07:28.311 DEBUG:boto:Token: None 2022-09-03T06:07:28.312 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4= 2022-09-03T06:07:28.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4=', 'Content-Length': '0'} 2022-09-03T06:07:28.315 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fe200b125af181a6-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.316 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.317 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.318 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.318 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.319 DEBUG:boto:Method: GET 2022-09-03T06:07:28.320 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.321 DEBUG:boto:Data: 2022-09-03T06:07:28.321 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.322 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.322 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.323 DEBUG:boto:Params: {} 2022-09-03T06:07:28.324 DEBUG:boto:Token: None 2022-09-03T06:07:28.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.329 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e69c482ea233e4e-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.330 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.331 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.332 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.333 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.333 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.334 DEBUG:boto:Method: GET 2022-09-03T06:07:28.335 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.336 DEBUG:boto:Data: 2022-09-03T06:07:28.336 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.337 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.338 DEBUG:boto:Params: {} 2022-09-03T06:07:28.339 DEBUG:boto:Token: None 2022-09-03T06:07:28.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.345 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c2d1696cf97e98d4-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.346 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.347 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.347 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.348 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.348 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.349 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.350 DEBUG:boto:Method: GET 2022-09-03T06:07:28.351 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.351 DEBUG:boto:Data: 2022-09-03T06:07:28.352 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.353 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.353 DEBUG:boto:Params: {} 2022-09-03T06:07:28.354 DEBUG:boto:Token: None 2022-09-03T06:07:28.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FCHc13A0v3we+5nKStglq9atks= 2022-09-03T06:07:28.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FCHc13A0v3we+5nKStglq9atks=', 'Content-Length': '0'} 2022-09-03T06:07:28.359 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002da70b832d2e0efd-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.360 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.360 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.361 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.361 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.362 DEBUG:boto:Method: GET 2022-09-03T06:07:28.362 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.363 DEBUG:boto:Data: 2022-09-03T06:07:28.363 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.364 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.364 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.365 DEBUG:boto:Params: {} 2022-09-03T06:07:28.366 DEBUG:boto:Token: None 2022-09-03T06:07:28.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FCHc13A0v3we+5nKStglq9atks= 2022-09-03T06:07:28.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FCHc13A0v3we+5nKStglq9atks=', 'Content-Length': '0'} 2022-09-03T06:07:28.370 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000004cf18c95d8c4bae-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.370 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.371 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.371 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.372 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.372 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.373 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.374 DEBUG:boto:Data: 2022-09-03T06:07:28.374 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.375 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.376 DEBUG:boto:Params: {} 2022-09-03T06:07:28.376 DEBUG:boto:Token: None 2022-09-03T06:07:28.377 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dwRWjm1Ya6mDqkrjVQKMr5i1GIg= 2022-09-03T06:07:28.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dwRWjm1Ya6mDqkrjVQKMr5i1GIg=', 'Content-Length': '0'} 2022-09-03T06:07:28.380 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c5e2818f1bc8fbcd-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.381 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.381 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.382 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.382 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.383 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.383 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.384 DEBUG:boto:Data: 2022-09-03T06:07:28.384 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.385 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.385 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.386 DEBUG:boto:Params: {} 2022-09-03T06:07:28.387 DEBUG:boto:Token: None 2022-09-03T06:07:28.387 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dwRWjm1Ya6mDqkrjVQKMr5i1GIg= 2022-09-03T06:07:28.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dwRWjm1Ya6mDqkrjVQKMr5i1GIg=', 'Content-Length': '0'} 2022-09-03T06:07:28.391 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003ab182744d1212ad-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.392 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.392 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.393 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.393 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.394 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.395 DEBUG:boto:Method: GET 2022-09-03T06:07:28.395 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.396 DEBUG:boto:Data: 2022-09-03T06:07:28.396 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.397 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.398 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.398 DEBUG:boto:Params: {} 2022-09-03T06:07:28.399 DEBUG:boto:Token: None 2022-09-03T06:07:28.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FCHc13A0v3we+5nKStglq9atks= 2022-09-03T06:07:28.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FCHc13A0v3we+5nKStglq9atks=', 'Content-Length': '0'} 2022-09-03T06:07:28.403 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c4987f875d5a4c2a-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.446 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.447 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.448 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.449 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.451 DEBUG:boto:Method: GET 2022-09-03T06:07:28.452 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.452 DEBUG:boto:Data: 2022-09-03T06:07:28.453 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.454 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.455 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.455 DEBUG:boto:Params: {} 2022-09-03T06:07:28.456 DEBUG:boto:Token: None 2022-09-03T06:07:28.457 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:28.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FCHc13A0v3we+5nKStglq9atks= 2022-09-03T06:07:28.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FCHc13A0v3we+5nKStglq9atks=', 'Content-Length': '0'} 2022-09-03T06:07:28.460 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000095e5f48f00a043a2-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.461 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.462 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.463 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.463 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.464 DEBUG:boto:Method: GET 2022-09-03T06:07:28.464 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.465 DEBUG:boto:Data: 2022-09-03T06:07:28.466 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.468 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.469 DEBUG:boto:Params: {} 2022-09-03T06:07:28.470 DEBUG:boto:Token: None 2022-09-03T06:07:28.471 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.477 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004473ae1ffb8c020f-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.477 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.478 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.479 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.479 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.480 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.480 DEBUG:boto:Method: GET 2022-09-03T06:07:28.481 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.482 DEBUG:boto:Data: 2022-09-03T06:07:28.482 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.483 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.484 DEBUG:boto:Params: {} 2022-09-03T06:07:28.485 DEBUG:boto:Token: None 2022-09-03T06:07:28.485 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.486 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.487 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.489 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c66a9b27e852cf1-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.490 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.491 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.492 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.493 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.493 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:28.494 DEBUG:boto:Data: 2022-09-03T06:07:28.494 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.495 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.496 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.496 DEBUG:boto:Params: {} 2022-09-03T06:07:28.497 DEBUG:boto:Token: None 2022-09-03T06:07:28.498 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj 2022-09-03T06:07:28.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ= 2022-09-03T06:07:28.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.503 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fb120dfd50649772-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.503 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.504 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.504 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.505 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:28.505 DEBUG:boto:Data: 2022-09-03T06:07:28.506 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.507 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.507 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.508 DEBUG:boto:Params: {} 2022-09-03T06:07:28.508 DEBUG:boto:Token: None 2022-09-03T06:07:28.509 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj 2022-09-03T06:07:28.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ= 2022-09-03T06:07:28.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.514 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001df6b227e67a57f2-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.515 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.516 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.516 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.517 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.517 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.518 DEBUG:boto:Method: GET 2022-09-03T06:07:28.518 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.519 DEBUG:boto:Data: 2022-09-03T06:07:28.519 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.521 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.521 DEBUG:boto:Params: {} 2022-09-03T06:07:28.522 DEBUG:boto:Token: None 2022-09-03T06:07:28.522 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4= 2022-09-03T06:07:28.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4=', 'Content-Length': '0'} 2022-09-03T06:07:28.526 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000090e3707f44522384-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.527 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.527 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.528 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.528 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.529 DEBUG:boto:Method: GET 2022-09-03T06:07:28.530 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.530 DEBUG:boto:Data: 2022-09-03T06:07:28.531 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.531 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.532 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.533 DEBUG:boto:Params: {} 2022-09-03T06:07:28.533 DEBUG:boto:Token: None 2022-09-03T06:07:28.534 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4= 2022-09-03T06:07:28.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4=', 'Content-Length': '0'} 2022-09-03T06:07:28.538 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d3a64cb04753e61b-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.582 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.583 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.584 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.584 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.585 DEBUG:boto:Method: GET 2022-09-03T06:07:28.585 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.586 DEBUG:boto:Data: 2022-09-03T06:07:28.587 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.588 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.588 DEBUG:boto:Params: {} 2022-09-03T06:07:28.589 DEBUG:boto:Token: None 2022-09-03T06:07:28.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.594 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003c344a06a8ffbc13-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.595 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.595 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.596 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.597 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.597 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.598 DEBUG:boto:Method: GET 2022-09-03T06:07:28.599 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.600 DEBUG:boto:Data: 2022-09-03T06:07:28.600 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.601 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.601 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.602 DEBUG:boto:Params: {} 2022-09-03T06:07:28.602 DEBUG:boto:Token: None 2022-09-03T06:07:28.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.607 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000933a574169861ffe-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.608 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.608 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.609 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.609 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.610 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.611 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.612 DEBUG:boto:Method: GET 2022-09-03T06:07:28.612 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.613 DEBUG:boto:Data: 2022-09-03T06:07:28.614 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.616 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.617 DEBUG:boto:Params: {} 2022-09-03T06:07:28.618 DEBUG:boto:Token: None 2022-09-03T06:07:28.619 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yuRi+iy7U/q5MEscA0LSWzA5S+k= 2022-09-03T06:07:28.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yuRi+iy7U/q5MEscA0LSWzA5S+k=', 'Content-Length': '0'} 2022-09-03T06:07:28.622 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000568134b31f523411-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.666 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.667 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.667 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.668 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.668 DEBUG:boto:Method: GET 2022-09-03T06:07:28.669 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.670 DEBUG:boto:Data: 2022-09-03T06:07:28.670 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.671 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.671 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.672 DEBUG:boto:Params: {} 2022-09-03T06:07:28.672 DEBUG:boto:Token: None 2022-09-03T06:07:28.673 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.674 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yuRi+iy7U/q5MEscA0LSWzA5S+k= 2022-09-03T06:07:28.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yuRi+iy7U/q5MEscA0LSWzA5S+k=', 'Content-Length': '0'} 2022-09-03T06:07:28.677 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000018c3e613e75d18d5-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.678 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.678 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.679 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.679 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.680 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.680 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.681 DEBUG:boto:Data: 2022-09-03T06:07:28.682 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.682 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.683 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.683 DEBUG:boto:Params: {} 2022-09-03T06:07:28.684 DEBUG:boto:Token: None 2022-09-03T06:07:28.684 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kKBdacgH07yyU4dYyqasVXb/WnI= 2022-09-03T06:07:28.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kKBdacgH07yyU4dYyqasVXb/WnI=', 'Content-Length': '0'} 2022-09-03T06:07:28.688 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000091bbecd0ca83c871-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.688 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.689 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.690 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.690 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.691 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.692 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.693 DEBUG:boto:Data: 2022-09-03T06:07:28.694 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.696 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.697 DEBUG:boto:Params: {} 2022-09-03T06:07:28.697 DEBUG:boto:Token: None 2022-09-03T06:07:28.698 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.699 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kKBdacgH07yyU4dYyqasVXb/WnI= 2022-09-03T06:07:28.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kKBdacgH07yyU4dYyqasVXb/WnI=', 'Content-Length': '0'} 2022-09-03T06:07:28.701 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008363aa48faf4e66a-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.702 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.702 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.703 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.704 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.704 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.705 DEBUG:boto:Method: GET 2022-09-03T06:07:28.705 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.706 DEBUG:boto:Data: 2022-09-03T06:07:28.706 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.707 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.708 DEBUG:boto:Params: {} 2022-09-03T06:07:28.709 DEBUG:boto:Token: None 2022-09-03T06:07:28.709 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.710 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yuRi+iy7U/q5MEscA0LSWzA5S+k= 2022-09-03T06:07:28.711 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yuRi+iy7U/q5MEscA0LSWzA5S+k=', 'Content-Length': '0'} 2022-09-03T06:07:28.713 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dc3b15eae54f0951-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.714 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.715 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.716 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.717 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.718 DEBUG:boto:Method: GET 2022-09-03T06:07:28.719 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.720 DEBUG:boto:Data: 2022-09-03T06:07:28.721 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.723 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.723 DEBUG:boto:Params: {} 2022-09-03T06:07:28.724 DEBUG:boto:Token: None 2022-09-03T06:07:28.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:28.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yuRi+iy7U/q5MEscA0LSWzA5S+k= 2022-09-03T06:07:28.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yuRi+iy7U/q5MEscA0LSWzA5S+k=', 'Content-Length': '0'} 2022-09-03T06:07:28.729 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f5fba591e07b60a7-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.730 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.731 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.731 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.732 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.732 DEBUG:boto:Method: GET 2022-09-03T06:07:28.733 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.733 DEBUG:boto:Data: 2022-09-03T06:07:28.734 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.735 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.736 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.736 DEBUG:boto:Params: {} 2022-09-03T06:07:28.738 DEBUG:boto:Token: None 2022-09-03T06:07:28.738 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.743 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007dd815807378c1f8-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.744 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.745 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.746 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.747 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.748 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.749 DEBUG:boto:Method: GET 2022-09-03T06:07:28.749 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.750 DEBUG:boto:Data: 2022-09-03T06:07:28.750 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.751 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.752 DEBUG:boto:Params: {} 2022-09-03T06:07:28.753 DEBUG:boto:Token: None 2022-09-03T06:07:28.754 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.758 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000efde63da23653331-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.758 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.759 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.760 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.760 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.761 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:28.761 DEBUG:boto:Data: 2022-09-03T06:07:28.762 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.762 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.763 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.763 DEBUG:boto:Params: {} 2022-09-03T06:07:28.764 DEBUG:boto:Token: None 2022-09-03T06:07:28.764 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj 2022-09-03T06:07:28.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ= 2022-09-03T06:07:28.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.769 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007adc5e35c7eaa3e0-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.770 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.770 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.771 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.771 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:28.772 DEBUG:boto:Data: 2022-09-03T06:07:28.772 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.773 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.773 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.774 DEBUG:boto:Params: {} 2022-09-03T06:07:28.774 DEBUG:boto:Token: None 2022-09-03T06:07:28.775 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj 2022-09-03T06:07:28.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ= 2022-09-03T06:07:28.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.779 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be178c4f7f9e0da7-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.780 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.781 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.781 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.782 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.782 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.783 DEBUG:boto:Method: GET 2022-09-03T06:07:28.783 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.784 DEBUG:boto:Data: 2022-09-03T06:07:28.784 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.785 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.785 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.786 DEBUG:boto:Params: {} 2022-09-03T06:07:28.787 DEBUG:boto:Token: None 2022-09-03T06:07:28.787 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4= 2022-09-03T06:07:28.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4=', 'Content-Length': '0'} 2022-09-03T06:07:28.792 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d3da61037cc83f4e-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.793 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.794 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.795 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.795 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.796 DEBUG:boto:Method: GET 2022-09-03T06:07:28.797 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.797 DEBUG:boto:Data: 2022-09-03T06:07:28.805 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.806 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.806 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.807 DEBUG:boto:Params: {} 2022-09-03T06:07:28.807 DEBUG:boto:Token: None 2022-09-03T06:07:28.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:28.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4= 2022-09-03T06:07:28.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmD+SipjGRCJjiHm8TBC1sSATE4=', 'Content-Length': '0'} 2022-09-03T06:07:28.811 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000048146f194ac338c1-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.812 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.813 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.813 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.814 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.814 DEBUG:boto:Method: GET 2022-09-03T06:07:28.815 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.816 DEBUG:boto:Data: 2022-09-03T06:07:28.816 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.817 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.818 DEBUG:boto:Params: {} 2022-09-03T06:07:28.819 DEBUG:boto:Token: None 2022-09-03T06:07:28.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.823 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000084e312039bdd535c-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.824 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.825 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.826 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.826 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.827 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.827 DEBUG:boto:Method: GET 2022-09-03T06:07:28.828 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.828 DEBUG:boto:Data: 2022-09-03T06:07:28.829 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.829 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.830 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.830 DEBUG:boto:Params: {} 2022-09-03T06:07:28.831 DEBUG:boto:Token: None 2022-09-03T06:07:28.831 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.832 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.835 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000150a3ef1532c1d8a-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.836 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.837 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.837 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.838 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.838 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.839 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.840 DEBUG:boto:Method: GET 2022-09-03T06:07:28.840 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.841 DEBUG:boto:Data: 2022-09-03T06:07:28.841 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.843 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.843 DEBUG:boto:Params: {} 2022-09-03T06:07:28.844 DEBUG:boto:Token: None 2022-09-03T06:07:28.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V+3S/6fW2XmjuOJUxTUGTc9qNzY= 2022-09-03T06:07:28.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V+3S/6fW2XmjuOJUxTUGTc9qNzY=', 'Content-Length': '0'} 2022-09-03T06:07:28.849 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002315bfebc57fa3db-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.894 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.895 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.896 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.896 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.897 DEBUG:boto:Method: GET 2022-09-03T06:07:28.898 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.898 DEBUG:boto:Data: 2022-09-03T06:07:28.899 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.900 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.900 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.901 DEBUG:boto:Params: {} 2022-09-03T06:07:28.901 DEBUG:boto:Token: None 2022-09-03T06:07:28.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V+3S/6fW2XmjuOJUxTUGTc9qNzY= 2022-09-03T06:07:28.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V+3S/6fW2XmjuOJUxTUGTc9qNzY=', 'Content-Length': '0'} 2022-09-03T06:07:28.906 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009a65f99c5037a6a7-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.906 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.908 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.909 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.909 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.910 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.910 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.911 DEBUG:boto:Data: 2022-09-03T06:07:28.911 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.912 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.912 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.913 DEBUG:boto:Params: {} 2022-09-03T06:07:28.914 DEBUG:boto:Token: None 2022-09-03T06:07:28.914 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cAt1RHFgvbs6GvEH+Q69HqHhNGk= 2022-09-03T06:07:28.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cAt1RHFgvbs6GvEH+Q69HqHhNGk=', 'Content-Length': '0'} 2022-09-03T06:07:28.917 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a66386da86c98285-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.918 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.919 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.919 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.920 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.920 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.921 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.922 DEBUG:boto:Data: 2022-09-03T06:07:28.922 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.923 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.923 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.924 DEBUG:boto:Params: {} 2022-09-03T06:07:28.924 DEBUG:boto:Token: None 2022-09-03T06:07:28.925 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cAt1RHFgvbs6GvEH+Q69HqHhNGk= 2022-09-03T06:07:28.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cAt1RHFgvbs6GvEH+Q69HqHhNGk=', 'Content-Length': '0'} 2022-09-03T06:07:28.928 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000068e8210c9c04a6f6-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.929 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:28.930 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.930 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.931 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.931 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.932 DEBUG:boto:Method: GET 2022-09-03T06:07:28.932 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.933 DEBUG:boto:Data: 2022-09-03T06:07:28.933 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.935 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.935 DEBUG:boto:Params: {} 2022-09-03T06:07:28.936 DEBUG:boto:Token: None 2022-09-03T06:07:28.936 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V+3S/6fW2XmjuOJUxTUGTc9qNzY= 2022-09-03T06:07:28.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V+3S/6fW2XmjuOJUxTUGTc9qNzY=', 'Content-Length': '0'} 2022-09-03T06:07:28.940 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000171530e084235bb8-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.941 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.941 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.942 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.943 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.943 DEBUG:boto:Method: GET 2022-09-03T06:07:28.944 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.944 DEBUG:boto:Data: 2022-09-03T06:07:28.945 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.945 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.946 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.947 DEBUG:boto:Params: {} 2022-09-03T06:07:28.947 DEBUG:boto:Token: None 2022-09-03T06:07:28.948 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:28.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V+3S/6fW2XmjuOJUxTUGTc9qNzY= 2022-09-03T06:07:28.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V+3S/6fW2XmjuOJUxTUGTc9qNzY=', 'Content-Length': '0'} 2022-09-03T06:07:28.951 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000079bf2e3285e8f738-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.952 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.953 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.953 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.954 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.954 DEBUG:boto:Method: GET 2022-09-03T06:07:28.955 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.955 DEBUG:boto:Data: 2022-09-03T06:07:28.956 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.956 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.957 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.957 DEBUG:boto:Params: {} 2022-09-03T06:07:28.958 DEBUG:boto:Token: None 2022-09-03T06:07:28.959 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.959 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.963 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001409c40a4a2460bd-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.963 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.964 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:28.965 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:28.965 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.966 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:28.966 DEBUG:boto:Method: GET 2022-09-03T06:07:28.967 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:28.968 DEBUG:boto:Data: 2022-09-03T06:07:28.968 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.969 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.969 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.970 DEBUG:boto:Params: {} 2022-09-03T06:07:28.970 DEBUG:boto:Token: None 2022-09-03T06:07:28.971 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/ 2022-09-03T06:07:28.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk= 2022-09-03T06:07:28.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h+rv1l9+2LBINq0jNCBRkh6PXFk=', 'Content-Length': '0'} 2022-09-03T06:07:28.975 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b5086e1224c5ce15-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.976 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:28.976 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.977 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.978 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.978 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:28.979 DEBUG:boto:Data: 2022-09-03T06:07:28.979 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:28.980 DEBUG:boto:Port: 8001 2022-09-03T06:07:28.980 DEBUG:boto:Params: {} 2022-09-03T06:07:28.981 DEBUG:boto:Token: None 2022-09-03T06:07:28.982 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj 2022-09-03T06:07:28.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ= 2022-09-03T06:07:28.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.986 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007d476e6a44283308-006312ef20-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.987 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:28.988 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:28.989 DEBUG:boto:Method: HEAD 2022-09-03T06:07:28.989 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:28.990 DEBUG:boto:Data: 2022-09-03T06:07:28.990 DEBUG:boto:Headers: {} 2022-09-03T06:07:28.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:28.991 DEBUG:boto:Port: 8000 2022-09-03T06:07:28.992 DEBUG:boto:Params: {} 2022-09-03T06:07:28.992 DEBUG:boto:Token: None 2022-09-03T06:07:28.994 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:28 GMT /avysmw-19/obj 2022-09-03T06:07:28.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ= 2022-09-03T06:07:28.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kShryQi7PrFPGMXvJiZoR9kmuvQ=', 'Content-Length': '0'} 2022-09-03T06:07:28.999 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007ab435602f699580-006312ef20-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:28.999 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:29.000 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.000 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.001 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.001 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.002 DEBUG:boto:Method: GET 2022-09-03T06:07:29.002 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.003 DEBUG:boto:Data: 2022-09-03T06:07:29.004 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.004 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.005 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.006 DEBUG:boto:Params: {} 2022-09-03T06:07:29.007 DEBUG:boto:Token: None 2022-09-03T06:07:29.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90= 2022-09-03T06:07:29.011 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90=', 'Content-Length': '0'} 2022-09-03T06:07:29.013 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000056d052ef32a9e760-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.015 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.016 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.016 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.017 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.017 DEBUG:boto:Method: GET 2022-09-03T06:07:29.018 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.019 DEBUG:boto:Data: 2022-09-03T06:07:29.019 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:29.021 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.021 DEBUG:boto:Params: {} 2022-09-03T06:07:29.022 DEBUG:boto:Token: None 2022-09-03T06:07:29.022 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90= 2022-09-03T06:07:29.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90=', 'Content-Length': '0'} 2022-09-03T06:07:29.026 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fa203af84b820808-006312ef21-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.070 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.072 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.072 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.073 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.075 DEBUG:boto:Method: GET 2022-09-03T06:07:29.076 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:29.076 DEBUG:boto:Data: 2022-09-03T06:07:29.077 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.078 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.078 DEBUG:boto:Params: {} 2022-09-03T06:07:29.079 DEBUG:boto:Token: None 2022-09-03T06:07:29.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000abcd30bf2944919f-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.085 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.085 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.086 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.087 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.087 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.088 DEBUG:boto:Method: GET 2022-09-03T06:07:29.089 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:29.089 DEBUG:boto:Data: 2022-09-03T06:07:29.090 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.091 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:29.091 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.092 DEBUG:boto:Params: {} 2022-09-03T06:07:29.092 DEBUG:boto:Token: None 2022-09-03T06:07:29.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.097 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087736f7eb96e4c8b-006312ef21-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.098 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.099 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-19 zones={test-zone3, test-zone1} 2022-09-03T06:07:29.099 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-19 zones={test-zone3, test-zone2} 2022-09-03T06:07:29.100 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.100 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.101 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.101 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:07:29.102 DEBUG:boto:Data: 2022-09-03T06:07:29.102 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.103 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.104 DEBUG:boto:Params: {} 2022-09-03T06:07:29.104 DEBUG:boto:Token: None 2022-09-03T06:07:29.105 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.106 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tVN0nz4Tq2BK+jkvyz7WyUlK3ag= 2022-09-03T06:07:29.106 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tVN0nz4Tq2BK+jkvyz7WyUlK3ag=', 'Content-Length': '0'} 2022-09-03T06:07:29.109 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '31'), ('X-RGW-Bytes-Used', '240'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003cb72fc54a66a12a-006312ef21-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.110 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.110 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.111 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.111 DEBUG:boto:Path: /avysmw-19/ 2022-09-03T06:07:29.112 DEBUG:boto:Data: 2022-09-03T06:07:29.112 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.113 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.113 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.114 DEBUG:boto:Params: {} 2022-09-03T06:07:29.114 DEBUG:boto:Token: None 2022-09-03T06:07:29.115 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tVN0nz4Tq2BK+jkvyz7WyUlK3ag= 2022-09-03T06:07:29.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tVN0nz4Tq2BK+jkvyz7WyUlK3ag=', 'Content-Length': '0'} 2022-09-03T06:07:29.119 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '31'), ('X-RGW-Bytes-Used', '240'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005ee36eb9cf6897b1-006312ef21-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.120 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:07:29.120 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.121 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.122 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.122 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.123 DEBUG:boto:Method: GET 2022-09-03T06:07:29.123 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.124 DEBUG:boto:Data: 2022-09-03T06:07:29.124 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.125 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.125 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.126 DEBUG:boto:Params: {} 2022-09-03T06:07:29.126 DEBUG:boto:Token: None 2022-09-03T06:07:29.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/?versions 2022-09-03T06:07:29.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vJzdjsV4lrSMglV5EtcSZ3AkgAY= 2022-09-03T06:07:29.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vJzdjsV4lrSMglV5EtcSZ3AkgAY=', 'Content-Length': '0'} 2022-09-03T06:07:29.131 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000099cd9084a6623ec0-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.132 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.134 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:07:29.135 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.135 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.136 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.136 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.137 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.137 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.138 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.138 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.139 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.139 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.140 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.140 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.141 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.141 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.142 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.142 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.143 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.143 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.143 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.144 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.145 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.145 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.145 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.147 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.147 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.148 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.148 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.149 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.149 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.150 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.150 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:07:29.151 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.151 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.152 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.152 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.153 DEBUG:boto:Method: GET 2022-09-03T06:07:29.153 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.154 DEBUG:boto:Data: 2022-09-03T06:07:29.154 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.155 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.155 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.156 DEBUG:boto:Params: {} 2022-09-03T06:07:29.156 DEBUG:boto:Token: None 2022-09-03T06:07:29.157 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/?versions 2022-09-03T06:07:29.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vJzdjsV4lrSMglV5EtcSZ3AkgAY= 2022-09-03T06:07:29.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vJzdjsV4lrSMglV5EtcSZ3AkgAY=', 'Content-Length': '0'} 2022-09-03T06:07:29.162 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b032fed1072d0ec-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.163 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.167 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2022-09-03T06:07:29.168 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.169 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.170 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.171 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.172 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.173 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.174 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.175 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.175 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.176 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.177 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.178 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.178 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.179 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.180 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.181 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.181 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.181 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.182 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.183 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.183 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.184 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.184 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.185 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.185 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.186 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.186 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.187 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.187 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.188 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:07:29.189 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.189 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.190 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.190 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.191 DEBUG:boto:Method: GET 2022-09-03T06:07:29.191 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.192 DEBUG:boto:Data: 2022-09-03T06:07:29.192 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.193 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.193 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.194 DEBUG:boto:Params: {} 2022-09-03T06:07:29.194 DEBUG:boto:Token: None 2022-09-03T06:07:29.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/?versions 2022-09-03T06:07:29.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vJzdjsV4lrSMglV5EtcSZ3AkgAY= 2022-09-03T06:07:29.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vJzdjsV4lrSMglV5EtcSZ3AkgAY=', 'Content-Length': '0'} 2022-09-03T06:07:29.199 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d54af06900cf5d3-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.200 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.202 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.203 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.203 DEBUG:boto:path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.204 DEBUG:boto:auth_path=/avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.205 DEBUG:boto:Method: GET 2022-09-03T06:07:29.205 DEBUG:boto:Path: /avysmw-19/?versions&max-keys=999 2022-09-03T06:07:29.206 DEBUG:boto:Data: 2022-09-03T06:07:29.206 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.207 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.208 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.208 DEBUG:boto:Params: {} 2022-09-03T06:07:29.209 DEBUG:boto:Token: None 2022-09-03T06:07:29.210 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/?versions 2022-09-03T06:07:29.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vJzdjsV4lrSMglV5EtcSZ3AkgAY= 2022-09-03T06:07:29.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vJzdjsV4lrSMglV5EtcSZ3AkgAY=', 'Content-Length': '0'} 2022-09-03T06:07:29.215 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d865c21f9fc3ebb5-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.216 DEBUG:boto:b'avysmw-19999falsedummy-rH-SzMWsuuY0qYsBMDn8HPU34kD-4Jtrue2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjPh8NnAz2uekZeWU.bk2t0FNiid5Y89Atrue2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjQeoNIad-aHQE2-jp0omJBV0bohf2870false2022-09-03T06:04:35.733Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuRYy6DU4k2K-KtcN1XeLobkGdim1hx0false2022-09-03T06:04:35.666Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNlhZ8pB7Lc2820arMcd4cS9THFycWYTfalse2022-09-03T06:04:35.572Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjaE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzRfalse2022-09-03T06:04:35.607Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpuNwkcR0FtLcsf9mkvpxQzlZdAwVMwsfalse2022-09-03T06:04:35.638Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjGyrupACcf-E.8-DtDlOuMHOhjQWdAhvfalse2022-09-03T06:04:35.516Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjb9WPVWL2.JbwzkP2Bts3H8uy-GpGktnfalse2022-09-03T06:04:35.545Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjhp5M42QarVag6fifkB-JtVdZXOY7pXLfalse2022-09-03T06:04:35.484Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1Efalse2022-09-03T06:04:35.415Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjlMSLdAViSTA3kW5kBxGTWxZ.Cdpottjfalse2022-09-03T06:04:35.371Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjmWvr3TGkMMvnzREAzfBmctHihqNqlsgfalse2022-09-03T06:04:35.451Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjB712AEDTSngkrH4lM9kJJU1oPTSqH0Ufalse2022-09-03T06:04:35.304Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjR78AB7kULQsVVpdt6q20Ckg2x2N8Q2Vfalse2022-09-03T06:04:35.269Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnmzhy11EG9ZQx.u878SRuCGqgFPvjpgfalse2022-09-03T06:04:35.334Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj6uV51ig1eoLqPWbieLYdwHpl6xaQYjgfalse2022-09-03T06:04:35.223Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobja11VL7IeGSUXvmP2Q0EwVOSErHiHlcNfalse2022-09-03T06:04:35.190Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjoMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkwfalse2022-09-03T06:04:35.153Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34cbRa0IccRiFCht3CqirmXYwe7jpuPfalse2022-09-03T06:04:35.120Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjdMMNFGcpGTBxFBR4ulYemiNEgXkVEfvfalse2022-09-03T06:04:35.083Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjuwwXAQ.l7uosanR0We1fU5cnpeCTvF3false2022-09-03T06:04:35.052Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85G0hkf-Eia8CZrFNN.-6SaWe1ilZNCfalse2022-09-03T06:04:34.982Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjNcMAfdGd3hoA3JP.E1mP7q4GjNbxtnBfalse2022-09-03T06:04:34.953Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjj45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZofalse2022-09-03T06:04:35.017Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj--U94MsxK6J7xIZega8EK5RG5RoC35.false2022-09-03T06:04:34.887Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1tnGMKYbKFsgWolupnJo3QNfru.ueukfalse2022-09-03T06:04:34.852Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjpiA5MXRR6-9PpmdSMeLZfVgAqVmT5Infalse2022-09-03T06:04:34.818Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7Mfalse2022-09-03T06:04:34.694Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjINkkVt.t6i1066J16-a17HsiLEaCqWqfalse2022-09-03T06:04:34.774Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0false2022-09-03T06:04:34.736Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.219 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:07:29.219 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.220 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.221 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.221 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.222 DEBUG:boto:Method: GET 2022-09-03T06:07:29.222 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.223 DEBUG:boto:Data: 2022-09-03T06:07:29.224 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.224 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.225 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.225 DEBUG:boto:Params: {} 2022-09-03T06:07:29.226 DEBUG:boto:Token: None 2022-09-03T06:07:29.227 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ= 2022-09-03T06:07:29.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ=', 'Content-Length': '0'} 2022-09-03T06:07:29.230 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000710c6c61890ef4e2-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.231 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.232 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.233 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.233 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.234 DEBUG:boto:Method: GET 2022-09-03T06:07:29.235 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.236 DEBUG:boto:Data: 2022-09-03T06:07:29.237 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.239 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.240 DEBUG:boto:Params: {} 2022-09-03T06:07:29.241 DEBUG:boto:Token: None 2022-09-03T06:07:29.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ= 2022-09-03T06:07:29.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ=', 'Content-Length': '0'} 2022-09-03T06:07:29.245 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000cea8e9feac944388-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.246 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.247 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.247 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.248 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.248 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.249 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.249 DEBUG:boto:Data: 2022-09-03T06:07:29.250 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.251 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.252 DEBUG:boto:Params: {} 2022-09-03T06:07:29.252 DEBUG:boto:Token: None 2022-09-03T06:07:29.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ofxrnkn23QsMgRh0nQrNxVVG7y8= 2022-09-03T06:07:29.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ofxrnkn23QsMgRh0nQrNxVVG7y8=', 'Content-Length': '0'} 2022-09-03T06:07:29.256 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000be2108afdd016316-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.257 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.259 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.259 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.260 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.261 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.262 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.263 DEBUG:boto:Data: 2022-09-03T06:07:29.264 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.265 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.266 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.267 DEBUG:boto:Params: {} 2022-09-03T06:07:29.268 DEBUG:boto:Token: None 2022-09-03T06:07:29.269 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ofxrnkn23QsMgRh0nQrNxVVG7y8= 2022-09-03T06:07:29.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ofxrnkn23QsMgRh0nQrNxVVG7y8=', 'Content-Length': '0'} 2022-09-03T06:07:29.272 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000014303d25053bf20d-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.273 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:07:29.273 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.274 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.275 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.275 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.276 DEBUG:boto:Method: GET 2022-09-03T06:07:29.276 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.277 DEBUG:boto:Data: 2022-09-03T06:07:29.278 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.279 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.280 DEBUG:boto:Params: {} 2022-09-03T06:07:29.280 DEBUG:boto:Token: None 2022-09-03T06:07:29.281 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ= 2022-09-03T06:07:29.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ=', 'Content-Length': '0'} 2022-09-03T06:07:29.284 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e3d1d6893b73da6b-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.285 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.286 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.286 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.287 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.287 DEBUG:boto:Method: GET 2022-09-03T06:07:29.288 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.289 DEBUG:boto:Data: 2022-09-03T06:07:29.289 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.290 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.290 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.291 DEBUG:boto:Params: {} 2022-09-03T06:07:29.292 DEBUG:boto:Token: None 2022-09-03T06:07:29.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ= 2022-09-03T06:07:29.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ=', 'Content-Length': '0'} 2022-09-03T06:07:29.296 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006384dbe440eeea3a-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.297 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.297 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.298 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:29.298 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:29.299 DEBUG:boto:Method: GET 2022-09-03T06:07:29.300 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:29.300 DEBUG:boto:Data: 2022-09-03T06:07:29.301 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.301 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.302 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.303 DEBUG:boto:Params: {} 2022-09-03T06:07:29.303 DEBUG:boto:Token: None 2022-09-03T06:07:29.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.308 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000885690e9b52fb762-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.350 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.351 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.352 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.353 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:29.354 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:29.355 DEBUG:boto:Method: GET 2022-09-03T06:07:29.356 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:29.356 DEBUG:boto:Data: 2022-09-03T06:07:29.357 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.358 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.359 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.360 DEBUG:boto:Params: {} 2022-09-03T06:07:29.361 DEBUG:boto:Token: None 2022-09-03T06:07:29.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.367 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c9fb2a21692cb1da-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.368 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.368 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.369 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.369 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.370 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:07:29.371 DEBUG:boto:Data: 2022-09-03T06:07:29.371 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.371 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.372 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.372 DEBUG:boto:Params: {} 2022-09-03T06:07:29.373 DEBUG:boto:Token: None 2022-09-03T06:07:29.374 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy 2022-09-03T06:07:29.374 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SELbcW0qGXMBoB10PxQO66cNtsU= 2022-09-03T06:07:29.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SELbcW0qGXMBoB10PxQO66cNtsU=', 'Content-Length': '0'} 2022-09-03T06:07:29.378 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000aae859914d01d368-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.379 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.379 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.380 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.381 DEBUG:boto:Path: /avysmw-19/dummy 2022-09-03T06:07:29.381 DEBUG:boto:Data: 2022-09-03T06:07:29.382 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.382 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.383 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.383 DEBUG:boto:Params: {} 2022-09-03T06:07:29.384 DEBUG:boto:Token: None 2022-09-03T06:07:29.385 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy 2022-09-03T06:07:29.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SELbcW0qGXMBoB10PxQO66cNtsU= 2022-09-03T06:07:29.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SELbcW0qGXMBoB10PxQO66cNtsU=', 'Content-Length': '0'} 2022-09-03T06:07:29.388 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ac56a8532931def4-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.390 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2022-09-03T06:07:29.391 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.392 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.393 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.394 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.395 DEBUG:boto:Method: GET 2022-09-03T06:07:29.395 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.396 DEBUG:boto:Data: 2022-09-03T06:07:29.397 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.398 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.399 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.400 DEBUG:boto:Params: {} 2022-09-03T06:07:29.401 DEBUG:boto:Token: None 2022-09-03T06:07:29.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ= 2022-09-03T06:07:29.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ=', 'Content-Length': '0'} 2022-09-03T06:07:29.407 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004afc55884cb6a4d2-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.408 DEBUG:boto:path=/avysmw-19/dummy 2022-09-03T06:07:29.409 DEBUG:boto:auth_path=/avysmw-19/dummy 2022-09-03T06:07:29.410 DEBUG:boto:path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.410 DEBUG:boto:auth_path=/avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.411 DEBUG:boto:Method: GET 2022-09-03T06:07:29.411 DEBUG:boto:Path: /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.412 DEBUG:boto:Data: 2022-09-03T06:07:29.412 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.413 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.413 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.414 DEBUG:boto:Params: {} 2022-09-03T06:07:29.415 DEBUG:boto:Token: None 2022-09-03T06:07:29.415 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/dummy?versionId=-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J 2022-09-03T06:07:29.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ= 2022-09-03T06:07:29.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GNf2mPwE+G6VX9+T60Jgrrb8WGQ=', 'Content-Length': '0'} 2022-09-03T06:07:29.418 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:03:17 GMT'), ('x-amz-version-id', '-rH-SzMWsuuY0qYsBMDn8HPU34kD-4J'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e56e8bf7e397d983-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.419 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.420 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.420 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:29.421 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:29.421 DEBUG:boto:Method: GET 2022-09-03T06:07:29.422 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:29.422 DEBUG:boto:Data: 2022-09-03T06:07:29.423 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.424 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.424 DEBUG:boto:Params: {} 2022-09-03T06:07:29.425 DEBUG:boto:Token: None 2022-09-03T06:07:29.426 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.430 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000410e2e150faf5a65-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.430 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.431 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.432 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.432 DEBUG:boto:path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:29.433 DEBUG:boto:auth_path=/avysmw-19/?prefix=dummy 2022-09-03T06:07:29.434 DEBUG:boto:Method: GET 2022-09-03T06:07:29.434 DEBUG:boto:Path: /avysmw-19/?prefix=dummy 2022-09-03T06:07:29.435 DEBUG:boto:Data: 2022-09-03T06:07:29.436 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.436 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.437 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.437 DEBUG:boto:Params: {} 2022-09-03T06:07:29.438 DEBUG:boto:Token: None 2022-09-03T06:07:29.439 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.440 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.443 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000540a54cd1462c70e-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.443 DEBUG:boto:b'avysmw-19dummy1000falsedummy2022-09-03T06:03:17.760Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.444 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:29.445 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.445 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.446 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.446 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.447 DEBUG:boto:Method: GET 2022-09-03T06:07:29.447 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.448 DEBUG:boto:Data: 2022-09-03T06:07:29.448 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.449 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.450 DEBUG:boto:Params: {} 2022-09-03T06:07:29.451 DEBUG:boto:Token: None 2022-09-03T06:07:29.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90= 2022-09-03T06:07:29.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90=', 'Content-Length': '0'} 2022-09-03T06:07:29.455 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001fce263e2f296bd7-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.456 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.456 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.457 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.458 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.459 DEBUG:boto:Method: GET 2022-09-03T06:07:29.460 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.461 DEBUG:boto:Data: 2022-09-03T06:07:29.462 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.463 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.464 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.465 DEBUG:boto:Params: {} 2022-09-03T06:07:29.466 DEBUG:boto:Token: None 2022-09-03T06:07:29.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90= 2022-09-03T06:07:29.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90=', 'Content-Length': '0'} 2022-09-03T06:07:29.472 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000627dabae472f5790-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.518 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.519 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.520 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.521 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.522 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.523 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.524 DEBUG:boto:Data: 2022-09-03T06:07:29.525 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.526 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.527 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.527 DEBUG:boto:Params: {} 2022-09-03T06:07:29.528 DEBUG:boto:Token: None 2022-09-03T06:07:29.528 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gVAhXBIo4iVSsjYt0JKEe4SEYeA= 2022-09-03T06:07:29.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gVAhXBIo4iVSsjYt0JKEe4SEYeA=', 'Content-Length': '0'} 2022-09-03T06:07:29.532 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c693aadc7721956a-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.533 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.534 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.535 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.538 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.539 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.540 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.540 DEBUG:boto:Data: 2022-09-03T06:07:29.541 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.541 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.542 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.543 DEBUG:boto:Params: {} 2022-09-03T06:07:29.544 DEBUG:boto:Token: None 2022-09-03T06:07:29.544 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gVAhXBIo4iVSsjYt0JKEe4SEYeA= 2022-09-03T06:07:29.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gVAhXBIo4iVSsjYt0JKEe4SEYeA=', 'Content-Length': '0'} 2022-09-03T06:07:29.548 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d89e02d8f9254c68-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.549 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:29.550 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.551 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.552 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.553 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.554 DEBUG:boto:Method: GET 2022-09-03T06:07:29.554 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.555 DEBUG:boto:Data: 2022-09-03T06:07:29.555 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.557 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.557 DEBUG:boto:Params: {} 2022-09-03T06:07:29.558 DEBUG:boto:Token: None 2022-09-03T06:07:29.559 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.559 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90= 2022-09-03T06:07:29.560 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90=', 'Content-Length': '0'} 2022-09-03T06:07:29.562 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007c1e49fce09fa1b0-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.563 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.563 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.564 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.565 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.565 DEBUG:boto:Method: GET 2022-09-03T06:07:29.566 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.566 DEBUG:boto:Data: 2022-09-03T06:07:29.567 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.567 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.568 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.568 DEBUG:boto:Params: {} 2022-09-03T06:07:29.569 DEBUG:boto:Token: None 2022-09-03T06:07:29.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90= 2022-09-03T06:07:29.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90=', 'Content-Length': '0'} 2022-09-03T06:07:29.573 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000de00017eacc7dfea-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.574 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.575 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.576 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.576 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.577 DEBUG:boto:Method: GET 2022-09-03T06:07:29.577 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:29.578 DEBUG:boto:Data: 2022-09-03T06:07:29.578 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.579 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.579 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.580 DEBUG:boto:Params: {} 2022-09-03T06:07:29.580 DEBUG:boto:Token: None 2022-09-03T06:07:29.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.587 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c87de8d9e2903b28-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.634 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.636 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.637 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.638 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.639 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.640 DEBUG:boto:Method: GET 2022-09-03T06:07:29.641 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:29.642 DEBUG:boto:Data: 2022-09-03T06:07:29.644 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.644 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.645 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.646 DEBUG:boto:Params: {} 2022-09-03T06:07:29.646 DEBUG:boto:Token: None 2022-09-03T06:07:29.647 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.648 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.651 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000376b4623e9e76e8c-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.652 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.653 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.654 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.655 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.655 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:29.656 DEBUG:boto:Data: 2022-09-03T06:07:29.657 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.658 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.658 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.659 DEBUG:boto:Params: {} 2022-09-03T06:07:29.661 DEBUG:boto:Token: None 2022-09-03T06:07:29.662 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj 2022-09-03T06:07:29.662 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CVcbTvOeO3ruUxz36OD3jkrDz3M= 2022-09-03T06:07:29.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CVcbTvOeO3ruUxz36OD3jkrDz3M=', 'Content-Length': '0'} 2022-09-03T06:07:29.668 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000067905a4f4df23a4e-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.669 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.671 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.672 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.673 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:29.674 DEBUG:boto:Data: 2022-09-03T06:07:29.675 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.676 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.677 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.677 DEBUG:boto:Params: {} 2022-09-03T06:07:29.678 DEBUG:boto:Token: None 2022-09-03T06:07:29.679 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj 2022-09-03T06:07:29.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CVcbTvOeO3ruUxz36OD3jkrDz3M= 2022-09-03T06:07:29.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CVcbTvOeO3ruUxz36OD3jkrDz3M=', 'Content-Length': '0'} 2022-09-03T06:07:29.688 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009e05625c6ee631de-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.689 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:29.690 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.691 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.693 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.694 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.695 DEBUG:boto:Method: GET 2022-09-03T06:07:29.696 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.697 DEBUG:boto:Data: 2022-09-03T06:07:29.698 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.700 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.701 DEBUG:boto:Params: {} 2022-09-03T06:07:29.701 DEBUG:boto:Token: None 2022-09-03T06:07:29.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90= 2022-09-03T06:07:29.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90=', 'Content-Length': '0'} 2022-09-03T06:07:29.706 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e8967fc2bee5366e-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.707 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.709 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.709 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.710 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.711 DEBUG:boto:Method: GET 2022-09-03T06:07:29.711 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.712 DEBUG:boto:Data: 2022-09-03T06:07:29.712 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.713 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.713 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.714 DEBUG:boto:Params: {} 2022-09-03T06:07:29.715 DEBUG:boto:Token: None 2022-09-03T06:07:29.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:29.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90= 2022-09-03T06:07:29.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvGUQp2NtcI/DhFwRKEsBedgp90=', 'Content-Length': '0'} 2022-09-03T06:07:29.719 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000009fc95fc23c39abc-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.766 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.767 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.768 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.768 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.769 DEBUG:boto:Method: GET 2022-09-03T06:07:29.769 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:29.770 DEBUG:boto:Data: 2022-09-03T06:07:29.770 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.772 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.773 DEBUG:boto:Params: {} 2022-09-03T06:07:29.773 DEBUG:boto:Token: None 2022-09-03T06:07:29.774 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.788 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bdc5ca0a9192d807-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.789 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.790 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.790 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.791 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.792 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.792 DEBUG:boto:Method: GET 2022-09-03T06:07:29.793 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:29.793 DEBUG:boto:Data: 2022-09-03T06:07:29.794 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.794 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.795 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.795 DEBUG:boto:Params: {} 2022-09-03T06:07:29.796 DEBUG:boto:Token: None 2022-09-03T06:07:29.797 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.798 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.801 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020adbcb7d9cb185b-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.801 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.802 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:29.803 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.804 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.804 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.805 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.806 DEBUG:boto:Method: GET 2022-09-03T06:07:29.807 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.807 DEBUG:boto:Data: 2022-09-03T06:07:29.808 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.809 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.809 DEBUG:boto:Params: {} 2022-09-03T06:07:29.810 DEBUG:boto:Token: None 2022-09-03T06:07:29.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aLg2OoZAGCkGDmM1Iwm2ap0FQNI= 2022-09-03T06:07:29.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aLg2OoZAGCkGDmM1Iwm2ap0FQNI=', 'Content-Length': '0'} 2022-09-03T06:07:29.814 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d60962fcac61802b-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.815 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.816 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.816 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.817 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.818 DEBUG:boto:Method: GET 2022-09-03T06:07:29.818 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.819 DEBUG:boto:Data: 2022-09-03T06:07:29.819 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.820 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.821 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.821 DEBUG:boto:Params: {} 2022-09-03T06:07:29.822 DEBUG:boto:Token: None 2022-09-03T06:07:29.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aLg2OoZAGCkGDmM1Iwm2ap0FQNI= 2022-09-03T06:07:29.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aLg2OoZAGCkGDmM1Iwm2ap0FQNI=', 'Content-Length': '0'} 2022-09-03T06:07:29.826 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008d5fad4ed359e264-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.827 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.828 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.828 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.829 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.829 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.830 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.830 DEBUG:boto:Data: 2022-09-03T06:07:29.831 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.832 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.832 DEBUG:boto:Params: {} 2022-09-03T06:07:29.833 DEBUG:boto:Token: None 2022-09-03T06:07:29.833 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GdmESH6VAZ+/VAvia7CM1WTgrhE= 2022-09-03T06:07:29.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GdmESH6VAZ+/VAvia7CM1WTgrhE=', 'Content-Length': '0'} 2022-09-03T06:07:29.837 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dc63f7365c69cbce-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.838 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.839 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.840 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.841 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.842 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.843 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.843 DEBUG:boto:Data: 2022-09-03T06:07:29.844 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.845 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.846 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.847 DEBUG:boto:Params: {} 2022-09-03T06:07:29.847 DEBUG:boto:Token: None 2022-09-03T06:07:29.848 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GdmESH6VAZ+/VAvia7CM1WTgrhE= 2022-09-03T06:07:29.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GdmESH6VAZ+/VAvia7CM1WTgrhE=', 'Content-Length': '0'} 2022-09-03T06:07:29.851 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000edccb7eeff503e24-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.852 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:29.853 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.853 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.854 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.854 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.855 DEBUG:boto:Method: GET 2022-09-03T06:07:29.855 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.856 DEBUG:boto:Data: 2022-09-03T06:07:29.856 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.858 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.858 DEBUG:boto:Params: {} 2022-09-03T06:07:29.859 DEBUG:boto:Token: None 2022-09-03T06:07:29.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aLg2OoZAGCkGDmM1Iwm2ap0FQNI= 2022-09-03T06:07:29.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aLg2OoZAGCkGDmM1Iwm2ap0FQNI=', 'Content-Length': '0'} 2022-09-03T06:07:29.863 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000406e812bac0be1c2-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.906 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.907 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.907 DEBUG:boto:path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.908 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.909 DEBUG:boto:Method: GET 2022-09-03T06:07:29.909 DEBUG:boto:Path: /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.910 DEBUG:boto:Data: 2022-09-03T06:07:29.910 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.911 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.912 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.912 DEBUG:boto:Params: {} 2022-09-03T06:07:29.913 DEBUG:boto:Token: None 2022-09-03T06:07:29.914 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj?versionId=QeoNIad-aHQE2-jp0omJBV0bohf2870 2022-09-03T06:07:29.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aLg2OoZAGCkGDmM1Iwm2ap0FQNI= 2022-09-03T06:07:29.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aLg2OoZAGCkGDmM1Iwm2ap0FQNI=', 'Content-Length': '0'} 2022-09-03T06:07:29.919 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'QeoNIad-aHQE2-jp0omJBV0bohf2870'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000055147b8313211abb-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.919 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.920 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.921 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.921 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.922 DEBUG:boto:Method: GET 2022-09-03T06:07:29.922 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:29.923 DEBUG:boto:Data: 2022-09-03T06:07:29.923 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.925 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.925 DEBUG:boto:Params: {} 2022-09-03T06:07:29.926 DEBUG:boto:Token: None 2022-09-03T06:07:29.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.931 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001dc4f24c62eb8fcc-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.932 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.933 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:29.933 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:29.934 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.934 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:29.935 DEBUG:boto:Method: GET 2022-09-03T06:07:29.935 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:29.936 DEBUG:boto:Data: 2022-09-03T06:07:29.936 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.937 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:29.938 DEBUG:boto:Port: 8000 2022-09-03T06:07:29.938 DEBUG:boto:Params: {} 2022-09-03T06:07:29.939 DEBUG:boto:Token: None 2022-09-03T06:07:29.939 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/ 2022-09-03T06:07:29.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo= 2022-09-03T06:07:29.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Ky/dI6v/YR/M9V/lrFcBQjxKvo=', 'Content-Length': '0'} 2022-09-03T06:07:29.944 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000013453873fde602f2-006312ef21-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:29.986 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:29.988 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:29.989 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:29.990 DEBUG:boto:Method: HEAD 2022-09-03T06:07:29.991 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:29.992 DEBUG:boto:Data: 2022-09-03T06:07:29.993 DEBUG:boto:Headers: {} 2022-09-03T06:07:29.994 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:29.995 DEBUG:boto:Port: 8001 2022-09-03T06:07:29.995 DEBUG:boto:Params: {} 2022-09-03T06:07:29.996 DEBUG:boto:Token: None 2022-09-03T06:07:29.997 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:29 GMT /avysmw-19/obj 2022-09-03T06:07:29.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CVcbTvOeO3ruUxz36OD3jkrDz3M= 2022-09-03T06:07:29.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CVcbTvOeO3ruUxz36OD3jkrDz3M=', 'Content-Length': '0'} 2022-09-03T06:07:30.002 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000032ad2067bda14d57-006312ef21-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.003 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.003 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.004 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.005 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:30.005 DEBUG:boto:Data: 2022-09-03T06:07:30.006 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.006 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.007 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.008 DEBUG:boto:Params: {} 2022-09-03T06:07:30.009 DEBUG:boto:Token: None 2022-09-03T06:07:30.010 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj 2022-09-03T06:07:30.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ= 2022-09-03T06:07:30.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ=', 'Content-Length': '0'} 2022-09-03T06:07:30.016 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000109db42b80c8a9c4-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.017 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.017 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.018 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.018 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.019 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.020 DEBUG:boto:Method: GET 2022-09-03T06:07:30.020 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.021 DEBUG:boto:Data: 2022-09-03T06:07:30.021 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.022 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.023 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.023 DEBUG:boto:Params: {} 2022-09-03T06:07:30.024 DEBUG:boto:Token: None 2022-09-03T06:07:30.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU= 2022-09-03T06:07:30.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU=', 'Content-Length': '0'} 2022-09-03T06:07:30.028 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000052d95ad4ac505a80-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.029 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.029 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.030 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.030 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.031 DEBUG:boto:Method: GET 2022-09-03T06:07:30.032 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.032 DEBUG:boto:Data: 2022-09-03T06:07:30.033 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.033 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.034 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.034 DEBUG:boto:Params: {} 2022-09-03T06:07:30.035 DEBUG:boto:Token: None 2022-09-03T06:07:30.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU= 2022-09-03T06:07:30.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU=', 'Content-Length': '0'} 2022-09-03T06:07:30.039 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000097863eb0fee51921-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.040 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.041 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.041 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.042 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.042 DEBUG:boto:Method: GET 2022-09-03T06:07:30.043 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.043 DEBUG:boto:Data: 2022-09-03T06:07:30.044 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.045 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.046 DEBUG:boto:Params: {} 2022-09-03T06:07:30.046 DEBUG:boto:Token: None 2022-09-03T06:07:30.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.051 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009379016b0cb68cbd-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.052 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.053 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.053 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.054 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.055 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.055 DEBUG:boto:Method: GET 2022-09-03T06:07:30.056 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.056 DEBUG:boto:Data: 2022-09-03T06:07:30.057 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.058 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.058 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.059 DEBUG:boto:Params: {} 2022-09-03T06:07:30.059 DEBUG:boto:Token: None 2022-09-03T06:07:30.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.064 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a1754719af6325f-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.065 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.067 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.068 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.069 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.070 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.071 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.072 DEBUG:boto:Method: GET 2022-09-03T06:07:30.073 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.074 DEBUG:boto:Data: 2022-09-03T06:07:30.074 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.076 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.077 DEBUG:boto:Params: {} 2022-09-03T06:07:30.078 DEBUG:boto:Token: None 2022-09-03T06:07:30.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.079 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YUaxG3sib1IFODpiLYVc6rUYJqY= 2022-09-03T06:07:30.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YUaxG3sib1IFODpiLYVc6rUYJqY=', 'Content-Length': '0'} 2022-09-03T06:07:30.083 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d1d88c08f8447d56-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.126 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.127 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.128 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.129 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.130 DEBUG:boto:Method: GET 2022-09-03T06:07:30.130 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.131 DEBUG:boto:Data: 2022-09-03T06:07:30.132 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.132 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.133 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.133 DEBUG:boto:Params: {} 2022-09-03T06:07:30.134 DEBUG:boto:Token: None 2022-09-03T06:07:30.135 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.136 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YUaxG3sib1IFODpiLYVc6rUYJqY= 2022-09-03T06:07:30.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YUaxG3sib1IFODpiLYVc6rUYJqY=', 'Content-Length': '0'} 2022-09-03T06:07:30.139 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cc45ff90de0674ef-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.140 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.140 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.141 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.141 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.142 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.143 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.144 DEBUG:boto:Data: 2022-09-03T06:07:30.144 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.145 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.146 DEBUG:boto:Params: {} 2022-09-03T06:07:30.147 DEBUG:boto:Token: None 2022-09-03T06:07:30.148 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VPcJJPEBMZ6GK6oE37jWfC/JeTo= 2022-09-03T06:07:30.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VPcJJPEBMZ6GK6oE37jWfC/JeTo=', 'Content-Length': '0'} 2022-09-03T06:07:30.152 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008e5a10143c453f56-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.153 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.153 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.154 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.155 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.155 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.156 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.157 DEBUG:boto:Data: 2022-09-03T06:07:30.158 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.159 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.159 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.160 DEBUG:boto:Params: {} 2022-09-03T06:07:30.161 DEBUG:boto:Token: None 2022-09-03T06:07:30.162 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VPcJJPEBMZ6GK6oE37jWfC/JeTo= 2022-09-03T06:07:30.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VPcJJPEBMZ6GK6oE37jWfC/JeTo=', 'Content-Length': '0'} 2022-09-03T06:07:30.166 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004aa61845126fa61c-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.167 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.168 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.169 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.169 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.170 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.170 DEBUG:boto:Method: GET 2022-09-03T06:07:30.171 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.171 DEBUG:boto:Data: 2022-09-03T06:07:30.172 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.173 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.173 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.174 DEBUG:boto:Params: {} 2022-09-03T06:07:30.175 DEBUG:boto:Token: None 2022-09-03T06:07:30.176 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YUaxG3sib1IFODpiLYVc6rUYJqY= 2022-09-03T06:07:30.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YUaxG3sib1IFODpiLYVc6rUYJqY=', 'Content-Length': '0'} 2022-09-03T06:07:30.180 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000048f4d1e05e2bddd3-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.181 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.181 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.182 DEBUG:boto:path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.182 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.183 DEBUG:boto:Method: GET 2022-09-03T06:07:30.184 DEBUG:boto:Path: /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.184 DEBUG:boto:Data: 2022-09-03T06:07:30.185 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.185 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.186 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.187 DEBUG:boto:Params: {} 2022-09-03T06:07:30.188 DEBUG:boto:Token: None 2022-09-03T06:07:30.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=uRYy6DU4k2K-KtcN1XeLobkGdim1hx0 2022-09-03T06:07:30.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YUaxG3sib1IFODpiLYVc6rUYJqY= 2022-09-03T06:07:30.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YUaxG3sib1IFODpiLYVc6rUYJqY=', 'Content-Length': '0'} 2022-09-03T06:07:30.192 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uRYy6DU4k2K-KtcN1XeLobkGdim1hx0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000086b392f3d41d6a98-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.238 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.239 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.240 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.240 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.241 DEBUG:boto:Method: GET 2022-09-03T06:07:30.241 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.242 DEBUG:boto:Data: 2022-09-03T06:07:30.242 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.243 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.244 DEBUG:boto:Params: {} 2022-09-03T06:07:30.245 DEBUG:boto:Token: None 2022-09-03T06:07:30.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.250 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c345bc56218bcba5-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.251 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.252 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.252 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.253 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.253 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.254 DEBUG:boto:Method: GET 2022-09-03T06:07:30.254 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.255 DEBUG:boto:Data: 2022-09-03T06:07:30.255 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.256 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.256 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.257 DEBUG:boto:Params: {} 2022-09-03T06:07:30.258 DEBUG:boto:Token: None 2022-09-03T06:07:30.259 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.263 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e354b6990ff0ce8b-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.263 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.264 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.265 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.265 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.266 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:30.266 DEBUG:boto:Data: 2022-09-03T06:07:30.267 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.267 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.268 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.268 DEBUG:boto:Params: {} 2022-09-03T06:07:30.269 DEBUG:boto:Token: None 2022-09-03T06:07:30.269 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj 2022-09-03T06:07:30.270 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ= 2022-09-03T06:07:30.271 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ=', 'Content-Length': '0'} 2022-09-03T06:07:30.275 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a20112f4e1054b89-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.276 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.277 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.277 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.278 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:30.278 DEBUG:boto:Data: 2022-09-03T06:07:30.279 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.279 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.280 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.280 DEBUG:boto:Params: {} 2022-09-03T06:07:30.281 DEBUG:boto:Token: None 2022-09-03T06:07:30.281 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj 2022-09-03T06:07:30.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ= 2022-09-03T06:07:30.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ=', 'Content-Length': '0'} 2022-09-03T06:07:30.286 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002dc0b37bd58d0812-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.287 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.287 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.288 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.288 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.289 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.289 DEBUG:boto:Method: GET 2022-09-03T06:07:30.290 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.290 DEBUG:boto:Data: 2022-09-03T06:07:30.291 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.291 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.292 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.293 DEBUG:boto:Params: {} 2022-09-03T06:07:30.293 DEBUG:boto:Token: None 2022-09-03T06:07:30.294 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU= 2022-09-03T06:07:30.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU=', 'Content-Length': '0'} 2022-09-03T06:07:30.298 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f5cb2ebc05a98499-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.338 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.339 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.339 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.340 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.340 DEBUG:boto:Method: GET 2022-09-03T06:07:30.341 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.341 DEBUG:boto:Data: 2022-09-03T06:07:30.342 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.343 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.343 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.344 DEBUG:boto:Params: {} 2022-09-03T06:07:30.344 DEBUG:boto:Token: None 2022-09-03T06:07:30.345 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.346 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU= 2022-09-03T06:07:30.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU=', 'Content-Length': '0'} 2022-09-03T06:07:30.349 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e5f90e766c688fc8-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.349 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.350 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.350 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.351 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.351 DEBUG:boto:Method: GET 2022-09-03T06:07:30.352 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.352 DEBUG:boto:Data: 2022-09-03T06:07:30.353 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.353 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.354 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.354 DEBUG:boto:Params: {} 2022-09-03T06:07:30.355 DEBUG:boto:Token: None 2022-09-03T06:07:30.356 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.360 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cbc34a2e556c9e70-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.360 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.361 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.362 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.362 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.363 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.364 DEBUG:boto:Method: GET 2022-09-03T06:07:30.364 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.365 DEBUG:boto:Data: 2022-09-03T06:07:30.365 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.366 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.366 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.367 DEBUG:boto:Params: {} 2022-09-03T06:07:30.367 DEBUG:boto:Token: None 2022-09-03T06:07:30.368 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.372 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cfd5305fab643fc6-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.373 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.374 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.375 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.375 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.376 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.376 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.377 DEBUG:boto:Method: GET 2022-09-03T06:07:30.377 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.378 DEBUG:boto:Data: 2022-09-03T06:07:30.378 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.379 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.380 DEBUG:boto:Params: {} 2022-09-03T06:07:30.380 DEBUG:boto:Token: None 2022-09-03T06:07:30.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8SD957F7cztgHmt4cwmo2bWPSWk= 2022-09-03T06:07:30.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8SD957F7cztgHmt4cwmo2bWPSWk=', 'Content-Length': '0'} 2022-09-03T06:07:30.384 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001a27f50581ef0091-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.385 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.386 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.386 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.387 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.387 DEBUG:boto:Method: GET 2022-09-03T06:07:30.388 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.388 DEBUG:boto:Data: 2022-09-03T06:07:30.389 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.389 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.390 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.390 DEBUG:boto:Params: {} 2022-09-03T06:07:30.391 DEBUG:boto:Token: None 2022-09-03T06:07:30.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8SD957F7cztgHmt4cwmo2bWPSWk= 2022-09-03T06:07:30.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8SD957F7cztgHmt4cwmo2bWPSWk=', 'Content-Length': '0'} 2022-09-03T06:07:30.395 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e8538d56e97e2097-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.396 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.396 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.397 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.397 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.398 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.398 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.399 DEBUG:boto:Data: 2022-09-03T06:07:30.399 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.401 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.401 DEBUG:boto:Params: {} 2022-09-03T06:07:30.402 DEBUG:boto:Token: None 2022-09-03T06:07:30.402 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cd9WWfMnp4jIJD4DGID25XAV4sk= 2022-09-03T06:07:30.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cd9WWfMnp4jIJD4DGID25XAV4sk=', 'Content-Length': '0'} 2022-09-03T06:07:30.406 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f00c08d18c01896b-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.406 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.407 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.408 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.409 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.409 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.410 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.410 DEBUG:boto:Data: 2022-09-03T06:07:30.411 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.411 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.412 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.412 DEBUG:boto:Params: {} 2022-09-03T06:07:30.413 DEBUG:boto:Token: None 2022-09-03T06:07:30.414 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.414 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cd9WWfMnp4jIJD4DGID25XAV4sk= 2022-09-03T06:07:30.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cd9WWfMnp4jIJD4DGID25XAV4sk=', 'Content-Length': '0'} 2022-09-03T06:07:30.417 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001f6b0134d68cda32-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.418 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.418 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.419 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.419 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.420 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.420 DEBUG:boto:Method: GET 2022-09-03T06:07:30.421 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.421 DEBUG:boto:Data: 2022-09-03T06:07:30.422 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.423 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.424 DEBUG:boto:Params: {} 2022-09-03T06:07:30.424 DEBUG:boto:Token: None 2022-09-03T06:07:30.425 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8SD957F7cztgHmt4cwmo2bWPSWk= 2022-09-03T06:07:30.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8SD957F7cztgHmt4cwmo2bWPSWk=', 'Content-Length': '0'} 2022-09-03T06:07:30.428 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d11dc9f3b0f761df-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.429 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.430 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.430 DEBUG:boto:path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.431 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.431 DEBUG:boto:Method: GET 2022-09-03T06:07:30.432 DEBUG:boto:Path: /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.432 DEBUG:boto:Data: 2022-09-03T06:07:30.433 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.433 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.434 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.434 DEBUG:boto:Params: {} 2022-09-03T06:07:30.435 DEBUG:boto:Token: None 2022-09-03T06:07:30.435 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=NlhZ8pB7Lc2820arMcd4cS9THFycWYT 2022-09-03T06:07:30.436 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8SD957F7cztgHmt4cwmo2bWPSWk= 2022-09-03T06:07:30.437 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8SD957F7cztgHmt4cwmo2bWPSWk=', 'Content-Length': '0'} 2022-09-03T06:07:30.439 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'NlhZ8pB7Lc2820arMcd4cS9THFycWYT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000065bd438942465133-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.439 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.440 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.441 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.441 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.442 DEBUG:boto:Method: GET 2022-09-03T06:07:30.442 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.443 DEBUG:boto:Data: 2022-09-03T06:07:30.443 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.444 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.444 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.445 DEBUG:boto:Params: {} 2022-09-03T06:07:30.445 DEBUG:boto:Token: None 2022-09-03T06:07:30.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.451 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb4adb3f51d860b1-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.452 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.453 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.454 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.455 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.456 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.457 DEBUG:boto:Method: GET 2022-09-03T06:07:30.458 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.458 DEBUG:boto:Data: 2022-09-03T06:07:30.459 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.461 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.461 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.462 DEBUG:boto:Params: {} 2022-09-03T06:07:30.464 DEBUG:boto:Token: None 2022-09-03T06:07:30.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a853c2b184f6cfe-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.510 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.512 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.512 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.513 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.513 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:30.514 DEBUG:boto:Data: 2022-09-03T06:07:30.515 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.516 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.517 DEBUG:boto:Params: {} 2022-09-03T06:07:30.518 DEBUG:boto:Token: None 2022-09-03T06:07:30.519 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj 2022-09-03T06:07:30.521 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ= 2022-09-03T06:07:30.522 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ=', 'Content-Length': '0'} 2022-09-03T06:07:30.526 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000060a152effc143316-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.527 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.529 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.530 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.530 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:30.531 DEBUG:boto:Data: 2022-09-03T06:07:30.531 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.532 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.532 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.533 DEBUG:boto:Params: {} 2022-09-03T06:07:30.534 DEBUG:boto:Token: None 2022-09-03T06:07:30.534 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj 2022-09-03T06:07:30.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ= 2022-09-03T06:07:30.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ=', 'Content-Length': '0'} 2022-09-03T06:07:30.540 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006781cfbb58d46f77-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.541 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.542 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.543 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.544 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.544 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.545 DEBUG:boto:Method: GET 2022-09-03T06:07:30.545 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.546 DEBUG:boto:Data: 2022-09-03T06:07:30.546 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.547 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.548 DEBUG:boto:Params: {} 2022-09-03T06:07:30.548 DEBUG:boto:Token: None 2022-09-03T06:07:30.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU= 2022-09-03T06:07:30.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU=', 'Content-Length': '0'} 2022-09-03T06:07:30.553 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000355430b0ff416029-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.594 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.595 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.595 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.596 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.596 DEBUG:boto:Method: GET 2022-09-03T06:07:30.597 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.598 DEBUG:boto:Data: 2022-09-03T06:07:30.598 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.599 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.599 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.600 DEBUG:boto:Params: {} 2022-09-03T06:07:30.600 DEBUG:boto:Token: None 2022-09-03T06:07:30.601 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU= 2022-09-03T06:07:30.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU=', 'Content-Length': '0'} 2022-09-03T06:07:30.605 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007ca1439a8c37623d-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.606 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.606 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.607 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.607 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.608 DEBUG:boto:Method: GET 2022-09-03T06:07:30.608 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.609 DEBUG:boto:Data: 2022-09-03T06:07:30.609 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.610 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.611 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.611 DEBUG:boto:Params: {} 2022-09-03T06:07:30.612 DEBUG:boto:Token: None 2022-09-03T06:07:30.612 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.616 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000af620b0a5e00519f-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.617 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.618 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.619 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.620 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.620 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.622 DEBUG:boto:Method: GET 2022-09-03T06:07:30.623 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.623 DEBUG:boto:Data: 2022-09-03T06:07:30.624 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.625 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.627 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.627 DEBUG:boto:Params: {} 2022-09-03T06:07:30.628 DEBUG:boto:Token: None 2022-09-03T06:07:30.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.634 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e3979742dc3c7c4e-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.635 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.636 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.637 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.638 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.639 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.639 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.640 DEBUG:boto:Method: GET 2022-09-03T06:07:30.641 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.641 DEBUG:boto:Data: 2022-09-03T06:07:30.642 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.643 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.644 DEBUG:boto:Params: {} 2022-09-03T06:07:30.644 DEBUG:boto:Token: None 2022-09-03T06:07:30.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ckZoWdF42n6SXyid0ZscJi5YlKE= 2022-09-03T06:07:30.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ckZoWdF42n6SXyid0ZscJi5YlKE=', 'Content-Length': '0'} 2022-09-03T06:07:30.649 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001f1e58fb864f0e6a-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.650 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.650 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.651 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.651 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.652 DEBUG:boto:Method: GET 2022-09-03T06:07:30.653 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.653 DEBUG:boto:Data: 2022-09-03T06:07:30.654 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.654 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.655 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.655 DEBUG:boto:Params: {} 2022-09-03T06:07:30.656 DEBUG:boto:Token: None 2022-09-03T06:07:30.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ckZoWdF42n6SXyid0ZscJi5YlKE= 2022-09-03T06:07:30.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ckZoWdF42n6SXyid0ZscJi5YlKE=', 'Content-Length': '0'} 2022-09-03T06:07:30.660 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000689c12d69ac2e90c-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.661 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.661 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.662 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.662 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.663 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.663 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.664 DEBUG:boto:Data: 2022-09-03T06:07:30.664 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.665 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.665 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.666 DEBUG:boto:Params: {} 2022-09-03T06:07:30.666 DEBUG:boto:Token: None 2022-09-03T06:07:30.667 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.668 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3va+k/uCwK8McfN4yIB469Faq0Y= 2022-09-03T06:07:30.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3va+k/uCwK8McfN4yIB469Faq0Y=', 'Content-Length': '0'} 2022-09-03T06:07:30.670 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a814c9a43933ef00-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.671 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.671 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.672 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.672 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.673 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.673 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.674 DEBUG:boto:Data: 2022-09-03T06:07:30.675 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.675 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.676 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.676 DEBUG:boto:Params: {} 2022-09-03T06:07:30.677 DEBUG:boto:Token: None 2022-09-03T06:07:30.677 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3va+k/uCwK8McfN4yIB469Faq0Y= 2022-09-03T06:07:30.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3va+k/uCwK8McfN4yIB469Faq0Y=', 'Content-Length': '0'} 2022-09-03T06:07:30.680 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000027622a2c1df56069-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.681 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.682 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.682 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.683 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.683 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.684 DEBUG:boto:Method: GET 2022-09-03T06:07:30.684 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.685 DEBUG:boto:Data: 2022-09-03T06:07:30.685 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.686 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.687 DEBUG:boto:Params: {} 2022-09-03T06:07:30.688 DEBUG:boto:Token: None 2022-09-03T06:07:30.688 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ckZoWdF42n6SXyid0ZscJi5YlKE= 2022-09-03T06:07:30.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ckZoWdF42n6SXyid0ZscJi5YlKE=', 'Content-Length': '0'} 2022-09-03T06:07:30.692 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000029c698d1339bcd8f-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.694 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.695 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.696 DEBUG:boto:path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.696 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.697 DEBUG:boto:Method: GET 2022-09-03T06:07:30.698 DEBUG:boto:Path: /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.698 DEBUG:boto:Data: 2022-09-03T06:07:30.699 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.699 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.700 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.701 DEBUG:boto:Params: {} 2022-09-03T06:07:30.702 DEBUG:boto:Token: None 2022-09-03T06:07:30.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR 2022-09-03T06:07:30.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ckZoWdF42n6SXyid0ZscJi5YlKE= 2022-09-03T06:07:30.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ckZoWdF42n6SXyid0ZscJi5YlKE=', 'Content-Length': '0'} 2022-09-03T06:07:30.706 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'aE4Ze6MiEFUP08DuOAtGXC8Xt3FXzzR'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000064f5222b9f59d2df-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.707 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.708 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.708 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.709 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.710 DEBUG:boto:Method: GET 2022-09-03T06:07:30.710 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.711 DEBUG:boto:Data: 2022-09-03T06:07:30.711 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.712 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.713 DEBUG:boto:Params: {} 2022-09-03T06:07:30.714 DEBUG:boto:Token: None 2022-09-03T06:07:30.714 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.719 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b3184653312a092-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.719 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.720 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.721 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.722 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.722 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.723 DEBUG:boto:Method: GET 2022-09-03T06:07:30.723 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.724 DEBUG:boto:Data: 2022-09-03T06:07:30.725 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.725 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.726 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.727 DEBUG:boto:Params: {} 2022-09-03T06:07:30.728 DEBUG:boto:Token: None 2022-09-03T06:07:30.729 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.734 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e0cb1c386a2a88e6-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.782 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.784 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.785 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.786 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.787 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:30.788 DEBUG:boto:Data: 2022-09-03T06:07:30.789 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.789 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.790 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.791 DEBUG:boto:Params: {} 2022-09-03T06:07:30.791 DEBUG:boto:Token: None 2022-09-03T06:07:30.792 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj 2022-09-03T06:07:30.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ= 2022-09-03T06:07:30.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ=', 'Content-Length': '0'} 2022-09-03T06:07:30.798 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000015e9d5c4fdfd3a5-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.799 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.800 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.800 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.802 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:30.802 DEBUG:boto:Data: 2022-09-03T06:07:30.803 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.804 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.804 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.805 DEBUG:boto:Params: {} 2022-09-03T06:07:30.805 DEBUG:boto:Token: None 2022-09-03T06:07:30.806 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj 2022-09-03T06:07:30.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ= 2022-09-03T06:07:30.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9jd6sh7gCFe0LOPVpCg6grxvqEQ=', 'Content-Length': '0'} 2022-09-03T06:07:30.820 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006ce540544a314c38-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.821 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.822 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.822 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.823 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.823 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.824 DEBUG:boto:Method: GET 2022-09-03T06:07:30.825 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.825 DEBUG:boto:Data: 2022-09-03T06:07:30.826 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.827 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.827 DEBUG:boto:Params: {} 2022-09-03T06:07:30.828 DEBUG:boto:Token: None 2022-09-03T06:07:30.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU= 2022-09-03T06:07:30.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU=', 'Content-Length': '0'} 2022-09-03T06:07:30.832 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000050a45d576b481b1b-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.833 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.834 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.834 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.835 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.835 DEBUG:boto:Method: GET 2022-09-03T06:07:30.836 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.837 DEBUG:boto:Data: 2022-09-03T06:07:30.837 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.838 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.838 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.839 DEBUG:boto:Params: {} 2022-09-03T06:07:30.839 DEBUG:boto:Token: None 2022-09-03T06:07:30.840 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:30.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU= 2022-09-03T06:07:30.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:24eJiqYUYaQqZd2hHZ/TCTMNwqU=', 'Content-Length': '0'} 2022-09-03T06:07:30.844 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fd89a54cb9640575-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.845 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.845 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.846 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.846 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.847 DEBUG:boto:Method: GET 2022-09-03T06:07:30.847 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.848 DEBUG:boto:Data: 2022-09-03T06:07:30.848 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.849 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.850 DEBUG:boto:Params: {} 2022-09-03T06:07:30.850 DEBUG:boto:Token: None 2022-09-03T06:07:30.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.855 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d993a62172b499d9-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.857 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.858 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.858 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.859 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.860 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.861 DEBUG:boto:Method: GET 2022-09-03T06:07:30.861 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.862 DEBUG:boto:Data: 2022-09-03T06:07:30.862 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.863 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.864 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.864 DEBUG:boto:Params: {} 2022-09-03T06:07:30.865 DEBUG:boto:Token: None 2022-09-03T06:07:30.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:30.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:30.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:30.870 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009748432d35461a5f-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.871 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:30.873 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.874 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.875 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.876 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.877 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.878 DEBUG:boto:Method: GET 2022-09-03T06:07:30.881 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.882 DEBUG:boto:Data: 2022-09-03T06:07:30.884 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.885 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.887 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.888 DEBUG:boto:Params: {} 2022-09-03T06:07:30.889 DEBUG:boto:Token: None 2022-09-03T06:07:30.890 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RzwdrV/bWjx+owL0cEW31FAVoQs= 2022-09-03T06:07:30.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RzwdrV/bWjx+owL0cEW31FAVoQs=', 'Content-Length': '0'} 2022-09-03T06:07:30.896 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f95cf966a8839eb1-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.938 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.939 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.939 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.940 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.940 DEBUG:boto:Method: GET 2022-09-03T06:07:30.941 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.942 DEBUG:boto:Data: 2022-09-03T06:07:30.942 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.943 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.943 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.944 DEBUG:boto:Params: {} 2022-09-03T06:07:30.944 DEBUG:boto:Token: None 2022-09-03T06:07:30.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RzwdrV/bWjx+owL0cEW31FAVoQs= 2022-09-03T06:07:30.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RzwdrV/bWjx+owL0cEW31FAVoQs=', 'Content-Length': '0'} 2022-09-03T06:07:30.949 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000051d4261db6d5778e-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.950 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.950 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.951 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.951 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.952 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.952 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.953 DEBUG:boto:Data: 2022-09-03T06:07:30.953 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.954 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.955 DEBUG:boto:Params: {} 2022-09-03T06:07:30.955 DEBUG:boto:Token: None 2022-09-03T06:07:30.956 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bv2OTgzLhLD9B3VApsBqgv+A6n8= 2022-09-03T06:07:30.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bv2OTgzLhLD9B3VApsBqgv+A6n8=', 'Content-Length': '0'} 2022-09-03T06:07:30.959 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000064e3730f8da1e0b5-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.960 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.961 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.961 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.962 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.962 DEBUG:boto:Method: HEAD 2022-09-03T06:07:30.963 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.963 DEBUG:boto:Data: 2022-09-03T06:07:30.964 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.964 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.965 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.965 DEBUG:boto:Params: {} 2022-09-03T06:07:30.966 DEBUG:boto:Token: None 2022-09-03T06:07:30.966 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bv2OTgzLhLD9B3VApsBqgv+A6n8= 2022-09-03T06:07:30.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bv2OTgzLhLD9B3VApsBqgv+A6n8=', 'Content-Length': '0'} 2022-09-03T06:07:30.969 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000094f4572097393539-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.970 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:30.971 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.971 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.972 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.972 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.973 DEBUG:boto:Method: GET 2022-09-03T06:07:30.973 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.974 DEBUG:boto:Data: 2022-09-03T06:07:30.975 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.975 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.976 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.976 DEBUG:boto:Params: {} 2022-09-03T06:07:30.977 DEBUG:boto:Token: None 2022-09-03T06:07:30.977 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RzwdrV/bWjx+owL0cEW31FAVoQs= 2022-09-03T06:07:30.979 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RzwdrV/bWjx+owL0cEW31FAVoQs=', 'Content-Length': '0'} 2022-09-03T06:07:30.981 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000578ef4ef76aabc23-006312ef22-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.982 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:30.982 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:30.983 DEBUG:boto:path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.983 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.984 DEBUG:boto:Method: GET 2022-09-03T06:07:30.985 DEBUG:boto:Path: /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.985 DEBUG:boto:Data: 2022-09-03T06:07:30.986 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.986 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:30.987 DEBUG:boto:Port: 8000 2022-09-03T06:07:30.987 DEBUG:boto:Params: {} 2022-09-03T06:07:30.988 DEBUG:boto:Token: None 2022-09-03T06:07:30.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/obj?versionId=puNwkcR0FtLcsf9mkvpxQzlZdAwVMws 2022-09-03T06:07:30.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RzwdrV/bWjx+owL0cEW31FAVoQs= 2022-09-03T06:07:30.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RzwdrV/bWjx+owL0cEW31FAVoQs=', 'Content-Length': '0'} 2022-09-03T06:07:30.992 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'puNwkcR0FtLcsf9mkvpxQzlZdAwVMws'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002965ee2fa059d5fd-006312ef22-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:30.993 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:30.993 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:30.994 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.994 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:30.995 DEBUG:boto:Method: GET 2022-09-03T06:07:30.995 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:30.996 DEBUG:boto:Data: 2022-09-03T06:07:30.996 DEBUG:boto:Headers: {} 2022-09-03T06:07:30.997 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:30.997 DEBUG:boto:Port: 8001 2022-09-03T06:07:30.998 DEBUG:boto:Params: {} 2022-09-03T06:07:30.999 DEBUG:boto:Token: None 2022-09-03T06:07:30.999 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:30 GMT /avysmw-19/ 2022-09-03T06:07:31.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo= 2022-09-03T06:07:31.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CVlt3cHvTfeClkaKPC592nULMo=', 'Content-Length': '0'} 2022-09-03T06:07:31.003 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005aa6a4ff7a9bd3ef-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.004 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.005 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.006 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.006 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.007 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.007 DEBUG:boto:Method: GET 2022-09-03T06:07:31.008 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.008 DEBUG:boto:Data: 2022-09-03T06:07:31.009 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.010 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.011 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.011 DEBUG:boto:Params: {} 2022-09-03T06:07:31.012 DEBUG:boto:Token: None 2022-09-03T06:07:31.012 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.017 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e4dd0ad8c475a8f8-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.017 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.018 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.019 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.019 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.020 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:31.020 DEBUG:boto:Data: 2022-09-03T06:07:31.021 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.022 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.022 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.023 DEBUG:boto:Params: {} 2022-09-03T06:07:31.024 DEBUG:boto:Token: None 2022-09-03T06:07:31.025 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj 2022-09-03T06:07:31.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE= 2022-09-03T06:07:31.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE=', 'Content-Length': '0'} 2022-09-03T06:07:31.031 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e964af07fcaf71d9-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.032 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.033 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.034 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.034 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:31.035 DEBUG:boto:Data: 2022-09-03T06:07:31.035 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.036 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.036 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.037 DEBUG:boto:Params: {} 2022-09-03T06:07:31.038 DEBUG:boto:Token: None 2022-09-03T06:07:31.039 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj 2022-09-03T06:07:31.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE= 2022-09-03T06:07:31.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE=', 'Content-Length': '0'} 2022-09-03T06:07:31.044 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006ffaae2e25999e66-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.045 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.045 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.046 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.046 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.047 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.048 DEBUG:boto:Method: GET 2022-09-03T06:07:31.048 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.049 DEBUG:boto:Data: 2022-09-03T06:07:31.049 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.050 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.051 DEBUG:boto:Params: {} 2022-09-03T06:07:31.052 DEBUG:boto:Token: None 2022-09-03T06:07:31.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I= 2022-09-03T06:07:31.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I=', 'Content-Length': '0'} 2022-09-03T06:07:31.056 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a7942e2a44778f38-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.102 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.104 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.105 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.105 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.106 DEBUG:boto:Method: GET 2022-09-03T06:07:31.106 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.107 DEBUG:boto:Data: 2022-09-03T06:07:31.107 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.108 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.108 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.109 DEBUG:boto:Params: {} 2022-09-03T06:07:31.109 DEBUG:boto:Token: None 2022-09-03T06:07:31.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I= 2022-09-03T06:07:31.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I=', 'Content-Length': '0'} 2022-09-03T06:07:31.113 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f297b97543c68770-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.114 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.115 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.115 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.116 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.116 DEBUG:boto:Method: GET 2022-09-03T06:07:31.117 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.117 DEBUG:boto:Data: 2022-09-03T06:07:31.118 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.118 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.119 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.120 DEBUG:boto:Params: {} 2022-09-03T06:07:31.120 DEBUG:boto:Token: None 2022-09-03T06:07:31.121 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.126 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f908031915ea81cd-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.126 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.127 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.128 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.128 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.129 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.130 DEBUG:boto:Method: GET 2022-09-03T06:07:31.130 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.131 DEBUG:boto:Data: 2022-09-03T06:07:31.131 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.132 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.133 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.133 DEBUG:boto:Params: {} 2022-09-03T06:07:31.134 DEBUG:boto:Token: None 2022-09-03T06:07:31.135 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.139 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000012df66d6c6341428-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.140 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.141 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.141 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.142 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.142 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.143 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.143 DEBUG:boto:Method: GET 2022-09-03T06:07:31.144 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.145 DEBUG:boto:Data: 2022-09-03T06:07:31.145 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.146 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.146 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.147 DEBUG:boto:Params: {} 2022-09-03T06:07:31.147 DEBUG:boto:Token: None 2022-09-03T06:07:31.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EtVW7ZMSwpaTc0h8miOvfVQxxiM= 2022-09-03T06:07:31.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EtVW7ZMSwpaTc0h8miOvfVQxxiM=', 'Content-Length': '0'} 2022-09-03T06:07:31.151 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000062d1b5f9e0bef439-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.152 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.153 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.153 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.154 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.155 DEBUG:boto:Method: GET 2022-09-03T06:07:31.155 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.156 DEBUG:boto:Data: 2022-09-03T06:07:31.156 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.157 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.157 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.158 DEBUG:boto:Params: {} 2022-09-03T06:07:31.158 DEBUG:boto:Token: None 2022-09-03T06:07:31.159 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EtVW7ZMSwpaTc0h8miOvfVQxxiM= 2022-09-03T06:07:31.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EtVW7ZMSwpaTc0h8miOvfVQxxiM=', 'Content-Length': '0'} 2022-09-03T06:07:31.162 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ab42aad61eb3a0b7-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.206 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.207 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.208 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.208 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.209 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.209 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.210 DEBUG:boto:Data: 2022-09-03T06:07:31.210 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.211 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.211 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.212 DEBUG:boto:Params: {} 2022-09-03T06:07:31.212 DEBUG:boto:Token: None 2022-09-03T06:07:31.213 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jct4QzGXo0Kqw2tijbscPgO4lNc= 2022-09-03T06:07:31.214 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jct4QzGXo0Kqw2tijbscPgO4lNc=', 'Content-Length': '0'} 2022-09-03T06:07:31.217 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fcdab3830117c25d-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.217 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.218 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.219 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.220 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.220 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.221 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.221 DEBUG:boto:Data: 2022-09-03T06:07:31.222 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.222 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.223 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.223 DEBUG:boto:Params: {} 2022-09-03T06:07:31.224 DEBUG:boto:Token: None 2022-09-03T06:07:31.225 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jct4QzGXo0Kqw2tijbscPgO4lNc= 2022-09-03T06:07:31.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jct4QzGXo0Kqw2tijbscPgO4lNc=', 'Content-Length': '0'} 2022-09-03T06:07:31.228 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000061725940aebb7a57-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.229 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.230 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.230 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.231 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.231 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.232 DEBUG:boto:Method: GET 2022-09-03T06:07:31.232 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.233 DEBUG:boto:Data: 2022-09-03T06:07:31.234 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.235 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.235 DEBUG:boto:Params: {} 2022-09-03T06:07:31.236 DEBUG:boto:Token: None 2022-09-03T06:07:31.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EtVW7ZMSwpaTc0h8miOvfVQxxiM= 2022-09-03T06:07:31.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EtVW7ZMSwpaTc0h8miOvfVQxxiM=', 'Content-Length': '0'} 2022-09-03T06:07:31.241 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004f9fd1c7cc8d5415-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.241 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.242 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.243 DEBUG:boto:path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.243 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.244 DEBUG:boto:Method: GET 2022-09-03T06:07:31.244 DEBUG:boto:Path: /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.245 DEBUG:boto:Data: 2022-09-03T06:07:31.245 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.246 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.247 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.247 DEBUG:boto:Params: {} 2022-09-03T06:07:31.248 DEBUG:boto:Token: None 2022-09-03T06:07:31.248 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=GyrupACcf-E.8-DtDlOuMHOhjQWdAhv 2022-09-03T06:07:31.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EtVW7ZMSwpaTc0h8miOvfVQxxiM= 2022-09-03T06:07:31.250 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EtVW7ZMSwpaTc0h8miOvfVQxxiM=', 'Content-Length': '0'} 2022-09-03T06:07:31.252 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'GyrupACcf-E.8-DtDlOuMHOhjQWdAhv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f66d489da7c3a694-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.253 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.253 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.254 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.254 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.255 DEBUG:boto:Method: GET 2022-09-03T06:07:31.256 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.256 DEBUG:boto:Data: 2022-09-03T06:07:31.257 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.257 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.258 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.258 DEBUG:boto:Params: {} 2022-09-03T06:07:31.259 DEBUG:boto:Token: None 2022-09-03T06:07:31.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.265 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007bdb116e67405cba-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.265 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.266 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.267 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.267 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.268 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.269 DEBUG:boto:Method: GET 2022-09-03T06:07:31.269 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.270 DEBUG:boto:Data: 2022-09-03T06:07:31.271 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.271 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.272 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.272 DEBUG:boto:Params: {} 2022-09-03T06:07:31.273 DEBUG:boto:Token: None 2022-09-03T06:07:31.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.278 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f239958afc2f25c6-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.279 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.279 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.280 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.281 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.282 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:31.283 DEBUG:boto:Data: 2022-09-03T06:07:31.283 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.284 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.285 DEBUG:boto:Params: {} 2022-09-03T06:07:31.285 DEBUG:boto:Token: None 2022-09-03T06:07:31.286 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj 2022-09-03T06:07:31.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE= 2022-09-03T06:07:31.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE=', 'Content-Length': '0'} 2022-09-03T06:07:31.291 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005c94345e19a4e460-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.292 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.292 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.293 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.293 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:31.294 DEBUG:boto:Data: 2022-09-03T06:07:31.294 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.295 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.295 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.296 DEBUG:boto:Params: {} 2022-09-03T06:07:31.296 DEBUG:boto:Token: None 2022-09-03T06:07:31.297 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj 2022-09-03T06:07:31.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE= 2022-09-03T06:07:31.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE=', 'Content-Length': '0'} 2022-09-03T06:07:31.302 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c8ffbcf877a327fe-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.302 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.303 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.304 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.304 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.305 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.305 DEBUG:boto:Method: GET 2022-09-03T06:07:31.307 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.308 DEBUG:boto:Data: 2022-09-03T06:07:31.308 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.309 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.310 DEBUG:boto:Params: {} 2022-09-03T06:07:31.310 DEBUG:boto:Token: None 2022-09-03T06:07:31.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I= 2022-09-03T06:07:31.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I=', 'Content-Length': '0'} 2022-09-03T06:07:31.315 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003f6de6d512e4be93-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.315 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.316 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.316 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.317 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.318 DEBUG:boto:Method: GET 2022-09-03T06:07:31.318 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.319 DEBUG:boto:Data: 2022-09-03T06:07:31.319 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.320 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.320 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.321 DEBUG:boto:Params: {} 2022-09-03T06:07:31.322 DEBUG:boto:Token: None 2022-09-03T06:07:31.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I= 2022-09-03T06:07:31.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I=', 'Content-Length': '0'} 2022-09-03T06:07:31.326 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000a921661cfaea92c-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.327 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.327 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.328 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.328 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.329 DEBUG:boto:Method: GET 2022-09-03T06:07:31.329 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.330 DEBUG:boto:Data: 2022-09-03T06:07:31.330 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.331 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.331 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.332 DEBUG:boto:Params: {} 2022-09-03T06:07:31.333 DEBUG:boto:Token: None 2022-09-03T06:07:31.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.337 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d0fdeba1d8fd69b0-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.378 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.380 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.381 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.382 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.383 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.385 DEBUG:boto:Method: GET 2022-09-03T06:07:31.386 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.387 DEBUG:boto:Data: 2022-09-03T06:07:31.388 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.389 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.390 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.391 DEBUG:boto:Params: {} 2022-09-03T06:07:31.392 DEBUG:boto:Token: None 2022-09-03T06:07:31.393 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.400 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dea5a06c9cfb9b60-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.401 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.404 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.405 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.406 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.407 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.408 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.410 DEBUG:boto:Method: GET 2022-09-03T06:07:31.411 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.412 DEBUG:boto:Data: 2022-09-03T06:07:31.413 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.414 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.415 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.416 DEBUG:boto:Params: {} 2022-09-03T06:07:31.417 DEBUG:boto:Token: None 2022-09-03T06:07:31.418 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yv9+BnoY6/dvgk/zNOkE3I8mLPo= 2022-09-03T06:07:31.421 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yv9+BnoY6/dvgk/zNOkE3I8mLPo=', 'Content-Length': '0'} 2022-09-03T06:07:31.424 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003754be695befc30c-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.425 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.426 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.427 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.428 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.430 DEBUG:boto:Method: GET 2022-09-03T06:07:31.431 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.432 DEBUG:boto:Data: 2022-09-03T06:07:31.433 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.435 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.436 DEBUG:boto:Params: {} 2022-09-03T06:07:31.437 DEBUG:boto:Token: None 2022-09-03T06:07:31.439 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yv9+BnoY6/dvgk/zNOkE3I8mLPo= 2022-09-03T06:07:31.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yv9+BnoY6/dvgk/zNOkE3I8mLPo=', 'Content-Length': '0'} 2022-09-03T06:07:31.443 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000512650fe9086fee6-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.444 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.445 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.446 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.446 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.447 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.447 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.448 DEBUG:boto:Data: 2022-09-03T06:07:31.448 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.449 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.450 DEBUG:boto:Params: {} 2022-09-03T06:07:31.451 DEBUG:boto:Token: None 2022-09-03T06:07:31.451 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sz1DAJ+XzQ45RuL80bxtzA7awJ8= 2022-09-03T06:07:31.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sz1DAJ+XzQ45RuL80bxtzA7awJ8=', 'Content-Length': '0'} 2022-09-03T06:07:31.455 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d221276eb4c65977-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.456 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.456 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.457 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.457 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.458 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.458 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.459 DEBUG:boto:Data: 2022-09-03T06:07:31.460 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.460 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.461 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.461 DEBUG:boto:Params: {} 2022-09-03T06:07:31.462 DEBUG:boto:Token: None 2022-09-03T06:07:31.463 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sz1DAJ+XzQ45RuL80bxtzA7awJ8= 2022-09-03T06:07:31.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sz1DAJ+XzQ45RuL80bxtzA7awJ8=', 'Content-Length': '0'} 2022-09-03T06:07:31.466 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004ac0f9a4db88ebf8-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.467 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.468 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.469 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.469 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.470 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.471 DEBUG:boto:Method: GET 2022-09-03T06:07:31.471 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.472 DEBUG:boto:Data: 2022-09-03T06:07:31.472 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.473 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.474 DEBUG:boto:Params: {} 2022-09-03T06:07:31.475 DEBUG:boto:Token: None 2022-09-03T06:07:31.475 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yv9+BnoY6/dvgk/zNOkE3I8mLPo= 2022-09-03T06:07:31.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yv9+BnoY6/dvgk/zNOkE3I8mLPo=', 'Content-Length': '0'} 2022-09-03T06:07:31.479 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004b50434957a36398-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.480 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.481 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.481 DEBUG:boto:path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.482 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.483 DEBUG:boto:Method: GET 2022-09-03T06:07:31.483 DEBUG:boto:Path: /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.484 DEBUG:boto:Data: 2022-09-03T06:07:31.484 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.485 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.486 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.487 DEBUG:boto:Params: {} 2022-09-03T06:07:31.487 DEBUG:boto:Token: None 2022-09-03T06:07:31.488 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn 2022-09-03T06:07:31.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yv9+BnoY6/dvgk/zNOkE3I8mLPo= 2022-09-03T06:07:31.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yv9+BnoY6/dvgk/zNOkE3I8mLPo=', 'Content-Length': '0'} 2022-09-03T06:07:31.492 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'b9WPVWL2.JbwzkP2Bts3H8uy-GpGktn'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008d7f0fdcebdb0990-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.534 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.535 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.536 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.536 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.537 DEBUG:boto:Method: GET 2022-09-03T06:07:31.539 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.540 DEBUG:boto:Data: 2022-09-03T06:07:31.541 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.542 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.542 DEBUG:boto:Params: {} 2022-09-03T06:07:31.543 DEBUG:boto:Token: None 2022-09-03T06:07:31.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.548 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000630ad8022dad2469-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.549 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.550 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.550 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.551 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.551 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.552 DEBUG:boto:Method: GET 2022-09-03T06:07:31.552 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.553 DEBUG:boto:Data: 2022-09-03T06:07:31.553 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.554 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.554 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.555 DEBUG:boto:Params: {} 2022-09-03T06:07:31.556 DEBUG:boto:Token: None 2022-09-03T06:07:31.557 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.561 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f98eda213cee044d-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.562 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.563 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.564 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.564 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.565 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:31.565 DEBUG:boto:Data: 2022-09-03T06:07:31.566 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.566 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.567 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.567 DEBUG:boto:Params: {} 2022-09-03T06:07:31.568 DEBUG:boto:Token: None 2022-09-03T06:07:31.569 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj 2022-09-03T06:07:31.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE= 2022-09-03T06:07:31.570 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE=', 'Content-Length': '0'} 2022-09-03T06:07:31.573 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ecd377a3ab5d04d1-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.574 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.575 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.575 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.576 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:31.576 DEBUG:boto:Data: 2022-09-03T06:07:31.577 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.577 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.578 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.579 DEBUG:boto:Params: {} 2022-09-03T06:07:31.579 DEBUG:boto:Token: None 2022-09-03T06:07:31.580 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj 2022-09-03T06:07:31.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE= 2022-09-03T06:07:31.581 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE=', 'Content-Length': '0'} 2022-09-03T06:07:31.585 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001a22451a9744e59d-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.585 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.586 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.587 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.587 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.588 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.588 DEBUG:boto:Method: GET 2022-09-03T06:07:31.589 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.589 DEBUG:boto:Data: 2022-09-03T06:07:31.590 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.590 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.591 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.592 DEBUG:boto:Params: {} 2022-09-03T06:07:31.592 DEBUG:boto:Token: None 2022-09-03T06:07:31.593 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I= 2022-09-03T06:07:31.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I=', 'Content-Length': '0'} 2022-09-03T06:07:31.597 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000011250744676c7a89-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.597 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.598 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.599 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.599 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.600 DEBUG:boto:Method: GET 2022-09-03T06:07:31.600 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.601 DEBUG:boto:Data: 2022-09-03T06:07:31.601 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.602 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.602 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.603 DEBUG:boto:Params: {} 2022-09-03T06:07:31.604 DEBUG:boto:Token: None 2022-09-03T06:07:31.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I= 2022-09-03T06:07:31.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I=', 'Content-Length': '0'} 2022-09-03T06:07:31.608 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000066dcc4c4b483200d-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.608 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.609 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.609 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.610 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.611 DEBUG:boto:Method: GET 2022-09-03T06:07:31.611 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.612 DEBUG:boto:Data: 2022-09-03T06:07:31.612 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.614 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.614 DEBUG:boto:Params: {} 2022-09-03T06:07:31.615 DEBUG:boto:Token: None 2022-09-03T06:07:31.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.619 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ace8455f56a6aaa7-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.620 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.621 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.621 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.622 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.623 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.623 DEBUG:boto:Method: GET 2022-09-03T06:07:31.624 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.624 DEBUG:boto:Data: 2022-09-03T06:07:31.625 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.625 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.626 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.627 DEBUG:boto:Params: {} 2022-09-03T06:07:31.629 DEBUG:boto:Token: None 2022-09-03T06:07:31.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.636 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000026645ceef491d4c3-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.637 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.639 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.640 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.641 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.642 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.643 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.644 DEBUG:boto:Method: GET 2022-09-03T06:07:31.645 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.645 DEBUG:boto:Data: 2022-09-03T06:07:31.646 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.647 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.648 DEBUG:boto:Params: {} 2022-09-03T06:07:31.648 DEBUG:boto:Token: None 2022-09-03T06:07:31.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BepHcIZY35wIlvGowpFkQabPEfI= 2022-09-03T06:07:31.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BepHcIZY35wIlvGowpFkQabPEfI=', 'Content-Length': '0'} 2022-09-03T06:07:31.652 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bc4a30df64774d6e-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.698 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.699 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.699 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.700 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.701 DEBUG:boto:Method: GET 2022-09-03T06:07:31.701 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.702 DEBUG:boto:Data: 2022-09-03T06:07:31.702 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.703 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.703 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.704 DEBUG:boto:Params: {} 2022-09-03T06:07:31.705 DEBUG:boto:Token: None 2022-09-03T06:07:31.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BepHcIZY35wIlvGowpFkQabPEfI= 2022-09-03T06:07:31.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BepHcIZY35wIlvGowpFkQabPEfI=', 'Content-Length': '0'} 2022-09-03T06:07:31.709 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e4869784ee7497fd-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.710 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.710 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.711 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.711 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.712 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.712 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.713 DEBUG:boto:Data: 2022-09-03T06:07:31.713 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.714 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.715 DEBUG:boto:Params: {} 2022-09-03T06:07:31.715 DEBUG:boto:Token: None 2022-09-03T06:07:31.716 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ogQ/KIuLMfPtZ29VSYm0HCg+eDU= 2022-09-03T06:07:31.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ogQ/KIuLMfPtZ29VSYm0HCg+eDU=', 'Content-Length': '0'} 2022-09-03T06:07:31.719 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008bae1612f5a0754f-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.720 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.720 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.721 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.722 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.722 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.723 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.723 DEBUG:boto:Data: 2022-09-03T06:07:31.724 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.724 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.725 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.725 DEBUG:boto:Params: {} 2022-09-03T06:07:31.726 DEBUG:boto:Token: None 2022-09-03T06:07:31.726 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ogQ/KIuLMfPtZ29VSYm0HCg+eDU= 2022-09-03T06:07:31.728 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ogQ/KIuLMfPtZ29VSYm0HCg+eDU=', 'Content-Length': '0'} 2022-09-03T06:07:31.730 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000026d90f004d6c0124-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.731 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.731 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.732 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.732 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.733 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.733 DEBUG:boto:Method: GET 2022-09-03T06:07:31.734 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.734 DEBUG:boto:Data: 2022-09-03T06:07:31.735 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.735 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.736 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.736 DEBUG:boto:Params: {} 2022-09-03T06:07:31.737 DEBUG:boto:Token: None 2022-09-03T06:07:31.738 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.738 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BepHcIZY35wIlvGowpFkQabPEfI= 2022-09-03T06:07:31.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BepHcIZY35wIlvGowpFkQabPEfI=', 'Content-Length': '0'} 2022-09-03T06:07:31.742 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004590d01eb65d1244-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.743 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.743 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.744 DEBUG:boto:path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.745 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.745 DEBUG:boto:Method: GET 2022-09-03T06:07:31.746 DEBUG:boto:Path: /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.746 DEBUG:boto:Data: 2022-09-03T06:07:31.747 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.747 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.748 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.748 DEBUG:boto:Params: {} 2022-09-03T06:07:31.749 DEBUG:boto:Token: None 2022-09-03T06:07:31.750 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=hp5M42QarVag6fifkB-JtVdZXOY7pXL 2022-09-03T06:07:31.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BepHcIZY35wIlvGowpFkQabPEfI= 2022-09-03T06:07:31.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BepHcIZY35wIlvGowpFkQabPEfI=', 'Content-Length': '0'} 2022-09-03T06:07:31.753 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'hp5M42QarVag6fifkB-JtVdZXOY7pXL'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000087d0d936a188041b-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.754 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.755 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.755 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.756 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.756 DEBUG:boto:Method: GET 2022-09-03T06:07:31.757 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.757 DEBUG:boto:Data: 2022-09-03T06:07:31.758 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.759 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.759 DEBUG:boto:Params: {} 2022-09-03T06:07:31.760 DEBUG:boto:Token: None 2022-09-03T06:07:31.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.766 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a135848efc5030e2-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.767 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.768 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.768 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.769 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.769 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.770 DEBUG:boto:Method: GET 2022-09-03T06:07:31.771 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.772 DEBUG:boto:Data: 2022-09-03T06:07:31.772 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.773 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.774 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.774 DEBUG:boto:Params: {} 2022-09-03T06:07:31.775 DEBUG:boto:Token: None 2022-09-03T06:07:31.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.781 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000084948966d5417ea9-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.781 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.782 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.783 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.784 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.784 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:31.785 DEBUG:boto:Data: 2022-09-03T06:07:31.786 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.788 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.788 DEBUG:boto:Params: {} 2022-09-03T06:07:31.790 DEBUG:boto:Token: None 2022-09-03T06:07:31.791 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj 2022-09-03T06:07:31.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE= 2022-09-03T06:07:31.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE=', 'Content-Length': '0'} 2022-09-03T06:07:31.797 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000715082f6f44bfcdf-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.798 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.799 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.800 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.800 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:31.801 DEBUG:boto:Data: 2022-09-03T06:07:31.801 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.802 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.802 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.803 DEBUG:boto:Params: {} 2022-09-03T06:07:31.803 DEBUG:boto:Token: None 2022-09-03T06:07:31.804 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj 2022-09-03T06:07:31.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE= 2022-09-03T06:07:31.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4ojAaDWfpZQ5r6BEl0JQvBUZjPE=', 'Content-Length': '0'} 2022-09-03T06:07:31.810 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000036ac1730dd9d828a-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.811 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.812 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.813 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.813 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.814 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.815 DEBUG:boto:Method: GET 2022-09-03T06:07:31.815 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.816 DEBUG:boto:Data: 2022-09-03T06:07:31.816 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.817 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.818 DEBUG:boto:Params: {} 2022-09-03T06:07:31.819 DEBUG:boto:Token: None 2022-09-03T06:07:31.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I= 2022-09-03T06:07:31.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I=', 'Content-Length': '0'} 2022-09-03T06:07:31.823 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e29425404de63f0b-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.823 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.824 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.825 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.825 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.826 DEBUG:boto:Method: GET 2022-09-03T06:07:31.826 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.827 DEBUG:boto:Data: 2022-09-03T06:07:31.827 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.828 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.828 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.829 DEBUG:boto:Params: {} 2022-09-03T06:07:31.830 DEBUG:boto:Token: None 2022-09-03T06:07:31.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:31.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I= 2022-09-03T06:07:31.831 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09eYmaS9VSKPEs+iOhSXEmeor6I=', 'Content-Length': '0'} 2022-09-03T06:07:31.834 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000364dc6317ee759ce-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.834 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.835 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.835 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.836 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.836 DEBUG:boto:Method: GET 2022-09-03T06:07:31.837 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.838 DEBUG:boto:Data: 2022-09-03T06:07:31.838 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.839 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.839 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.840 DEBUG:boto:Params: {} 2022-09-03T06:07:31.840 DEBUG:boto:Token: None 2022-09-03T06:07:31.841 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.845 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011715ef60e5da8a6-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.846 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.847 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.848 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.848 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.849 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.849 DEBUG:boto:Method: GET 2022-09-03T06:07:31.850 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.850 DEBUG:boto:Data: 2022-09-03T06:07:31.851 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.851 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.852 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.852 DEBUG:boto:Params: {} 2022-09-03T06:07:31.853 DEBUG:boto:Token: None 2022-09-03T06:07:31.854 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.858 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d1b2aeeb0d2c388-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.858 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.859 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.860 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.860 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.861 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.861 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.862 DEBUG:boto:Method: GET 2022-09-03T06:07:31.863 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.863 DEBUG:boto:Data: 2022-09-03T06:07:31.864 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.865 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.865 DEBUG:boto:Params: {} 2022-09-03T06:07:31.866 DEBUG:boto:Token: None 2022-09-03T06:07:31.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C/PVhqLI2EZuy6Ht7iRnQfrJqzw= 2022-09-03T06:07:31.868 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C/PVhqLI2EZuy6Ht7iRnQfrJqzw=', 'Content-Length': '0'} 2022-09-03T06:07:31.870 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000593f56f4b5cfe3f6-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.871 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.872 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.872 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.873 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.873 DEBUG:boto:Method: GET 2022-09-03T06:07:31.874 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.875 DEBUG:boto:Data: 2022-09-03T06:07:31.875 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.876 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.876 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.877 DEBUG:boto:Params: {} 2022-09-03T06:07:31.877 DEBUG:boto:Token: None 2022-09-03T06:07:31.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C/PVhqLI2EZuy6Ht7iRnQfrJqzw= 2022-09-03T06:07:31.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C/PVhqLI2EZuy6Ht7iRnQfrJqzw=', 'Content-Length': '0'} 2022-09-03T06:07:31.882 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000a136d4b5a8b864f-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.883 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.883 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.884 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.884 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.885 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.885 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.886 DEBUG:boto:Data: 2022-09-03T06:07:31.886 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.887 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.887 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.888 DEBUG:boto:Params: {} 2022-09-03T06:07:31.888 DEBUG:boto:Token: None 2022-09-03T06:07:31.889 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BK4S2rnFBU9dnNSIYMMq9jozAXs= 2022-09-03T06:07:31.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BK4S2rnFBU9dnNSIYMMq9jozAXs=', 'Content-Length': '0'} 2022-09-03T06:07:31.892 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004481d4b0bd5f1962-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.893 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.894 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.894 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.895 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.895 DEBUG:boto:Method: HEAD 2022-09-03T06:07:31.896 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.896 DEBUG:boto:Data: 2022-09-03T06:07:31.897 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.897 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.898 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.898 DEBUG:boto:Params: {} 2022-09-03T06:07:31.899 DEBUG:boto:Token: None 2022-09-03T06:07:31.900 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BK4S2rnFBU9dnNSIYMMq9jozAXs= 2022-09-03T06:07:31.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BK4S2rnFBU9dnNSIYMMq9jozAXs=', 'Content-Length': '0'} 2022-09-03T06:07:31.903 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c0b7414e7b6720b5-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.903 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:31.904 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.905 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.905 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.906 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.906 DEBUG:boto:Method: GET 2022-09-03T06:07:31.907 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.908 DEBUG:boto:Data: 2022-09-03T06:07:31.908 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.909 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.910 DEBUG:boto:Params: {} 2022-09-03T06:07:31.910 DEBUG:boto:Token: None 2022-09-03T06:07:31.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C/PVhqLI2EZuy6Ht7iRnQfrJqzw= 2022-09-03T06:07:31.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C/PVhqLI2EZuy6Ht7iRnQfrJqzw=', 'Content-Length': '0'} 2022-09-03T06:07:31.914 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005e3e3fb19897e6b8-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.958 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:31.959 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:31.959 DEBUG:boto:path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.960 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.961 DEBUG:boto:Method: GET 2022-09-03T06:07:31.961 DEBUG:boto:Path: /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.962 DEBUG:boto:Data: 2022-09-03T06:07:31.962 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.963 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.963 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.964 DEBUG:boto:Params: {} 2022-09-03T06:07:31.964 DEBUG:boto:Token: None 2022-09-03T06:07:31.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/obj?versionId=1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E 2022-09-03T06:07:31.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C/PVhqLI2EZuy6Ht7iRnQfrJqzw= 2022-09-03T06:07:31.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C/PVhqLI2EZuy6Ht7iRnQfrJqzw=', 'Content-Length': '0'} 2022-09-03T06:07:31.969 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '1AEIq4XeiNxTLp8XGd-E5bBJ-Q3oD1E'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000287304168c575eca-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.970 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.971 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.972 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.972 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.973 DEBUG:boto:Method: GET 2022-09-03T06:07:31.973 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.974 DEBUG:boto:Data: 2022-09-03T06:07:31.974 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.976 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:31.977 DEBUG:boto:Port: 8001 2022-09-03T06:07:31.978 DEBUG:boto:Params: {} 2022-09-03T06:07:31.979 DEBUG:boto:Token: None 2022-09-03T06:07:31.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.984 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c74e9e46d7cde92-006312ef23-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.985 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.986 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:31.987 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:31.987 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.988 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:31.989 DEBUG:boto:Method: GET 2022-09-03T06:07:31.989 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:31.990 DEBUG:boto:Data: 2022-09-03T06:07:31.990 DEBUG:boto:Headers: {} 2022-09-03T06:07:31.991 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:31.991 DEBUG:boto:Port: 8000 2022-09-03T06:07:31.992 DEBUG:boto:Params: {} 2022-09-03T06:07:31.993 DEBUG:boto:Token: None 2022-09-03T06:07:31.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:31 GMT /avysmw-19/ 2022-09-03T06:07:31.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8= 2022-09-03T06:07:31.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZtcudKzacVjeEoAvEyij12FdEO8=', 'Content-Length': '0'} 2022-09-03T06:07:31.997 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000615b64d5e906125-006312ef23-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:31.998 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:31.999 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.000 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.000 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.001 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:32.001 DEBUG:boto:Data: 2022-09-03T06:07:32.002 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.002 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.003 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.003 DEBUG:boto:Params: {} 2022-09-03T06:07:32.004 DEBUG:boto:Token: None 2022-09-03T06:07:32.005 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj 2022-09-03T06:07:32.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M= 2022-09-03T06:07:32.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M=', 'Content-Length': '0'} 2022-09-03T06:07:32.009 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002dfb22434d212479-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.010 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.011 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.011 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.012 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:32.013 DEBUG:boto:Data: 2022-09-03T06:07:32.014 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.015 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.016 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.017 DEBUG:boto:Params: {} 2022-09-03T06:07:32.018 DEBUG:boto:Token: None 2022-09-03T06:07:32.019 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj 2022-09-03T06:07:32.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M= 2022-09-03T06:07:32.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M=', 'Content-Length': '0'} 2022-09-03T06:07:32.025 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d1faa24a738270e8-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.025 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.026 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.027 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.027 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.028 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.029 DEBUG:boto:Method: GET 2022-09-03T06:07:32.031 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.032 DEBUG:boto:Data: 2022-09-03T06:07:32.033 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.034 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.035 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.036 DEBUG:boto:Params: {} 2022-09-03T06:07:32.037 DEBUG:boto:Token: None 2022-09-03T06:07:32.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34= 2022-09-03T06:07:32.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34=', 'Content-Length': '0'} 2022-09-03T06:07:32.043 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ff9ebdd01f08681b-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.044 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.045 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.046 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.047 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.048 DEBUG:boto:Method: GET 2022-09-03T06:07:32.049 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.051 DEBUG:boto:Data: 2022-09-03T06:07:32.053 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.053 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.054 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.055 DEBUG:boto:Params: {} 2022-09-03T06:07:32.056 DEBUG:boto:Token: None 2022-09-03T06:07:32.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34= 2022-09-03T06:07:32.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34=', 'Content-Length': '0'} 2022-09-03T06:07:32.060 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005af181135eddae04-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.061 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.062 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.063 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.063 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.064 DEBUG:boto:Method: GET 2022-09-03T06:07:32.064 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.065 DEBUG:boto:Data: 2022-09-03T06:07:32.066 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.068 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.069 DEBUG:boto:Params: {} 2022-09-03T06:07:32.070 DEBUG:boto:Token: None 2022-09-03T06:07:32.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.081 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009c00f98617fa5ff1-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.081 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.083 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.084 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.084 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.085 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.085 DEBUG:boto:Method: GET 2022-09-03T06:07:32.086 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.086 DEBUG:boto:Data: 2022-09-03T06:07:32.087 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.087 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.088 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.088 DEBUG:boto:Params: {} 2022-09-03T06:07:32.089 DEBUG:boto:Token: None 2022-09-03T06:07:32.090 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.091 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.094 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e5963f503fd7fac1-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.095 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.096 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.096 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.097 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.097 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.098 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.098 DEBUG:boto:Method: GET 2022-09-03T06:07:32.099 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.100 DEBUG:boto:Data: 2022-09-03T06:07:32.100 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.101 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.101 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.102 DEBUG:boto:Params: {} 2022-09-03T06:07:32.102 DEBUG:boto:Token: None 2022-09-03T06:07:32.103 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iIQ2PcYqDyP0vXzVzttpRFv38XA= 2022-09-03T06:07:32.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iIQ2PcYqDyP0vXzVzttpRFv38XA=', 'Content-Length': '0'} 2022-09-03T06:07:32.107 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b6ee1bff392a9f2f-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.154 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.155 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.155 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.156 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.157 DEBUG:boto:Method: GET 2022-09-03T06:07:32.158 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.158 DEBUG:boto:Data: 2022-09-03T06:07:32.159 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.159 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.160 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.160 DEBUG:boto:Params: {} 2022-09-03T06:07:32.161 DEBUG:boto:Token: None 2022-09-03T06:07:32.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iIQ2PcYqDyP0vXzVzttpRFv38XA= 2022-09-03T06:07:32.163 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iIQ2PcYqDyP0vXzVzttpRFv38XA=', 'Content-Length': '0'} 2022-09-03T06:07:32.166 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000011da1cd745cef0c1-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.167 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.167 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.168 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.169 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.169 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.170 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.170 DEBUG:boto:Data: 2022-09-03T06:07:32.171 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.172 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.172 DEBUG:boto:Params: {} 2022-09-03T06:07:32.173 DEBUG:boto:Token: None 2022-09-03T06:07:32.173 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJc9A7l72TUQa0b9bzVuxmdgJOs= 2022-09-03T06:07:32.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJc9A7l72TUQa0b9bzVuxmdgJOs=', 'Content-Length': '0'} 2022-09-03T06:07:32.177 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000018d3183b326ddc0-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.177 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.178 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.179 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.179 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.180 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.181 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.181 DEBUG:boto:Data: 2022-09-03T06:07:32.182 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.182 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.183 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.183 DEBUG:boto:Params: {} 2022-09-03T06:07:32.184 DEBUG:boto:Token: None 2022-09-03T06:07:32.184 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJc9A7l72TUQa0b9bzVuxmdgJOs= 2022-09-03T06:07:32.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJc9A7l72TUQa0b9bzVuxmdgJOs=', 'Content-Length': '0'} 2022-09-03T06:07:32.188 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001cadf150ccc4937b-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.189 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.189 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.190 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.190 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.191 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.192 DEBUG:boto:Method: GET 2022-09-03T06:07:32.192 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.193 DEBUG:boto:Data: 2022-09-03T06:07:32.193 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.194 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.195 DEBUG:boto:Params: {} 2022-09-03T06:07:32.195 DEBUG:boto:Token: None 2022-09-03T06:07:32.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iIQ2PcYqDyP0vXzVzttpRFv38XA= 2022-09-03T06:07:32.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iIQ2PcYqDyP0vXzVzttpRFv38XA=', 'Content-Length': '0'} 2022-09-03T06:07:32.199 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d6f4bddc5fea2437-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.200 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.201 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.201 DEBUG:boto:path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.202 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.202 DEBUG:boto:Method: GET 2022-09-03T06:07:32.203 DEBUG:boto:Path: /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.203 DEBUG:boto:Data: 2022-09-03T06:07:32.204 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.204 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.205 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.205 DEBUG:boto:Params: {} 2022-09-03T06:07:32.206 DEBUG:boto:Token: None 2022-09-03T06:07:32.207 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj 2022-09-03T06:07:32.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iIQ2PcYqDyP0vXzVzttpRFv38XA= 2022-09-03T06:07:32.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iIQ2PcYqDyP0vXzVzttpRFv38XA=', 'Content-Length': '0'} 2022-09-03T06:07:32.210 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'lMSLdAViSTA3kW5kBxGTWxZ.Cdpottj'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000167a60e79ef6d21f-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.211 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.211 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.212 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.212 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.213 DEBUG:boto:Method: GET 2022-09-03T06:07:32.213 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.214 DEBUG:boto:Data: 2022-09-03T06:07:32.215 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.216 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.216 DEBUG:boto:Params: {} 2022-09-03T06:07:32.217 DEBUG:boto:Token: None 2022-09-03T06:07:32.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.222 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a10e2698cbfc828-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.222 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.223 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.224 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.224 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.225 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.225 DEBUG:boto:Method: GET 2022-09-03T06:07:32.226 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.227 DEBUG:boto:Data: 2022-09-03T06:07:32.227 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.228 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.228 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.229 DEBUG:boto:Params: {} 2022-09-03T06:07:32.229 DEBUG:boto:Token: None 2022-09-03T06:07:32.230 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.234 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007303589b7550edaa-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.274 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.275 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.276 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.276 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.277 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:32.277 DEBUG:boto:Data: 2022-09-03T06:07:32.278 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.279 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.279 DEBUG:boto:Params: {} 2022-09-03T06:07:32.280 DEBUG:boto:Token: None 2022-09-03T06:07:32.281 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj 2022-09-03T06:07:32.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M= 2022-09-03T06:07:32.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M=', 'Content-Length': '0'} 2022-09-03T06:07:32.286 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d0b804faffc14822-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.287 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.288 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.288 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.289 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:32.289 DEBUG:boto:Data: 2022-09-03T06:07:32.290 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.291 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.292 DEBUG:boto:Params: {} 2022-09-03T06:07:32.292 DEBUG:boto:Token: None 2022-09-03T06:07:32.293 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj 2022-09-03T06:07:32.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M= 2022-09-03T06:07:32.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M=', 'Content-Length': '0'} 2022-09-03T06:07:32.297 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000034359b3505ebb52b-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.298 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.299 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.299 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.300 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.301 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.302 DEBUG:boto:Method: GET 2022-09-03T06:07:32.303 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.304 DEBUG:boto:Data: 2022-09-03T06:07:32.305 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.306 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.307 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.308 DEBUG:boto:Params: {} 2022-09-03T06:07:32.308 DEBUG:boto:Token: None 2022-09-03T06:07:32.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34= 2022-09-03T06:07:32.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34=', 'Content-Length': '0'} 2022-09-03T06:07:32.313 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000109f95abf7b89a6b-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.315 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.316 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.317 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.317 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.318 DEBUG:boto:Method: GET 2022-09-03T06:07:32.319 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.320 DEBUG:boto:Data: 2022-09-03T06:07:32.321 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.322 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.323 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.324 DEBUG:boto:Params: {} 2022-09-03T06:07:32.325 DEBUG:boto:Token: None 2022-09-03T06:07:32.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34= 2022-09-03T06:07:32.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34=', 'Content-Length': '0'} 2022-09-03T06:07:32.329 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000d0655f301d08040-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.330 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.331 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.332 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.332 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.333 DEBUG:boto:Method: GET 2022-09-03T06:07:32.333 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.334 DEBUG:boto:Data: 2022-09-03T06:07:32.335 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.336 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.337 DEBUG:boto:Params: {} 2022-09-03T06:07:32.338 DEBUG:boto:Token: None 2022-09-03T06:07:32.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.345 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b1849f402ddfc71-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.347 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.348 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.349 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.350 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.351 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.352 DEBUG:boto:Method: GET 2022-09-03T06:07:32.353 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.354 DEBUG:boto:Data: 2022-09-03T06:07:32.355 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.356 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.357 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.357 DEBUG:boto:Params: {} 2022-09-03T06:07:32.358 DEBUG:boto:Token: None 2022-09-03T06:07:32.359 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.365 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008afd13762e1f5c37-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.366 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.367 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.368 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.369 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.370 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.371 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.372 DEBUG:boto:Method: GET 2022-09-03T06:07:32.373 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.374 DEBUG:boto:Data: 2022-09-03T06:07:32.375 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.376 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.377 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.378 DEBUG:boto:Params: {} 2022-09-03T06:07:32.379 DEBUG:boto:Token: None 2022-09-03T06:07:32.379 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hQVdUTUa0gyTE434/qGsdGSmDzA= 2022-09-03T06:07:32.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hQVdUTUa0gyTE434/qGsdGSmDzA=', 'Content-Length': '0'} 2022-09-03T06:07:32.383 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000082bed0b5eff7869d-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.384 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.384 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.385 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.386 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.386 DEBUG:boto:Method: GET 2022-09-03T06:07:32.387 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.388 DEBUG:boto:Data: 2022-09-03T06:07:32.389 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.390 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.391 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.392 DEBUG:boto:Params: {} 2022-09-03T06:07:32.393 DEBUG:boto:Token: None 2022-09-03T06:07:32.394 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hQVdUTUa0gyTE434/qGsdGSmDzA= 2022-09-03T06:07:32.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hQVdUTUa0gyTE434/qGsdGSmDzA=', 'Content-Length': '0'} 2022-09-03T06:07:32.399 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000017bbb27768951709-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.400 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.401 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.401 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.402 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.402 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.403 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.403 DEBUG:boto:Data: 2022-09-03T06:07:32.404 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.404 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.405 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.406 DEBUG:boto:Params: {} 2022-09-03T06:07:32.407 DEBUG:boto:Token: None 2022-09-03T06:07:32.407 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EHdU4N3uYHQH9tmJ8zRq2xu+vcc= 2022-09-03T06:07:32.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EHdU4N3uYHQH9tmJ8zRq2xu+vcc=', 'Content-Length': '0'} 2022-09-03T06:07:32.411 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d4eb0f1ed4b2a6d9-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.412 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.412 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.413 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.413 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.414 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.414 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.415 DEBUG:boto:Data: 2022-09-03T06:07:32.415 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.416 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.416 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.417 DEBUG:boto:Params: {} 2022-09-03T06:07:32.417 DEBUG:boto:Token: None 2022-09-03T06:07:32.418 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EHdU4N3uYHQH9tmJ8zRq2xu+vcc= 2022-09-03T06:07:32.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EHdU4N3uYHQH9tmJ8zRq2xu+vcc=', 'Content-Length': '0'} 2022-09-03T06:07:32.422 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000338c795c48b62269-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.423 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.423 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.424 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.425 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.425 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.426 DEBUG:boto:Method: GET 2022-09-03T06:07:32.426 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.427 DEBUG:boto:Data: 2022-09-03T06:07:32.427 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.428 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.429 DEBUG:boto:Params: {} 2022-09-03T06:07:32.430 DEBUG:boto:Token: None 2022-09-03T06:07:32.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hQVdUTUa0gyTE434/qGsdGSmDzA= 2022-09-03T06:07:32.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hQVdUTUa0gyTE434/qGsdGSmDzA=', 'Content-Length': '0'} 2022-09-03T06:07:32.434 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000013cebdd484e6f2e4-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.478 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.479 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.480 DEBUG:boto:path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.481 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.482 DEBUG:boto:Method: GET 2022-09-03T06:07:32.483 DEBUG:boto:Path: /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.484 DEBUG:boto:Data: 2022-09-03T06:07:32.485 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.486 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.487 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.487 DEBUG:boto:Params: {} 2022-09-03T06:07:32.488 DEBUG:boto:Token: None 2022-09-03T06:07:32.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=mWvr3TGkMMvnzREAzfBmctHihqNqlsg 2022-09-03T06:07:32.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hQVdUTUa0gyTE434/qGsdGSmDzA= 2022-09-03T06:07:32.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hQVdUTUa0gyTE434/qGsdGSmDzA=', 'Content-Length': '0'} 2022-09-03T06:07:32.493 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'mWvr3TGkMMvnzREAzfBmctHihqNqlsg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a46c3f1ceb988994-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.494 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.494 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.495 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.495 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.496 DEBUG:boto:Method: GET 2022-09-03T06:07:32.496 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.497 DEBUG:boto:Data: 2022-09-03T06:07:32.498 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.498 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.499 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.499 DEBUG:boto:Params: {} 2022-09-03T06:07:32.500 DEBUG:boto:Token: None 2022-09-03T06:07:32.500 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.501 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.504 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f323679a6643cdb1-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.505 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.506 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.506 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.507 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.508 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.508 DEBUG:boto:Method: GET 2022-09-03T06:07:32.509 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.509 DEBUG:boto:Data: 2022-09-03T06:07:32.510 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.510 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.511 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.511 DEBUG:boto:Params: {} 2022-09-03T06:07:32.512 DEBUG:boto:Token: None 2022-09-03T06:07:32.512 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.518 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008dcb5a788c162d51-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.519 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.519 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.520 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.521 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.521 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:32.522 DEBUG:boto:Data: 2022-09-03T06:07:32.522 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.523 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.523 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.524 DEBUG:boto:Params: {} 2022-09-03T06:07:32.524 DEBUG:boto:Token: None 2022-09-03T06:07:32.525 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj 2022-09-03T06:07:32.526 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M= 2022-09-03T06:07:32.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M=', 'Content-Length': '0'} 2022-09-03T06:07:32.530 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009f2d71cd44617cb8-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.531 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.531 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.532 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.532 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:32.533 DEBUG:boto:Data: 2022-09-03T06:07:32.533 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.534 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.534 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.535 DEBUG:boto:Params: {} 2022-09-03T06:07:32.536 DEBUG:boto:Token: None 2022-09-03T06:07:32.536 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj 2022-09-03T06:07:32.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M= 2022-09-03T06:07:32.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M=', 'Content-Length': '0'} 2022-09-03T06:07:32.541 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e4b1913578a616ae-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.542 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.542 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.543 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.543 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.544 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.544 DEBUG:boto:Method: GET 2022-09-03T06:07:32.545 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.546 DEBUG:boto:Data: 2022-09-03T06:07:32.546 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.547 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.548 DEBUG:boto:Params: {} 2022-09-03T06:07:32.549 DEBUG:boto:Token: None 2022-09-03T06:07:32.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34= 2022-09-03T06:07:32.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34=', 'Content-Length': '0'} 2022-09-03T06:07:32.553 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a505021943881cae-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.553 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.554 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.555 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.555 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.556 DEBUG:boto:Method: GET 2022-09-03T06:07:32.556 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.557 DEBUG:boto:Data: 2022-09-03T06:07:32.557 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.558 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.558 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.559 DEBUG:boto:Params: {} 2022-09-03T06:07:32.559 DEBUG:boto:Token: None 2022-09-03T06:07:32.560 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34= 2022-09-03T06:07:32.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34=', 'Content-Length': '0'} 2022-09-03T06:07:32.564 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b232c1987bbfc38e-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.606 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.607 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.608 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.608 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.609 DEBUG:boto:Method: GET 2022-09-03T06:07:32.609 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.610 DEBUG:boto:Data: 2022-09-03T06:07:32.610 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.611 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.612 DEBUG:boto:Params: {} 2022-09-03T06:07:32.613 DEBUG:boto:Token: None 2022-09-03T06:07:32.613 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.618 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e90797fc3f20d30-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.620 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.621 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.623 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.624 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.625 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.626 DEBUG:boto:Method: GET 2022-09-03T06:07:32.627 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.628 DEBUG:boto:Data: 2022-09-03T06:07:32.629 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.630 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.630 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.631 DEBUG:boto:Params: {} 2022-09-03T06:07:32.632 DEBUG:boto:Token: None 2022-09-03T06:07:32.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.637 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000140695f6a8a4d2f1-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.638 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.638 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.639 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.640 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.641 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.642 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.643 DEBUG:boto:Method: GET 2022-09-03T06:07:32.644 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.645 DEBUG:boto:Data: 2022-09-03T06:07:32.647 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.648 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.648 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.649 DEBUG:boto:Params: {} 2022-09-03T06:07:32.650 DEBUG:boto:Token: None 2022-09-03T06:07:32.650 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iVh4ObycbzUGE/iExQtOyS1usvc= 2022-09-03T06:07:32.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iVh4ObycbzUGE/iExQtOyS1usvc=', 'Content-Length': '0'} 2022-09-03T06:07:32.654 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000065ca8d7492f131d4-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.702 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.703 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.704 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.704 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.705 DEBUG:boto:Method: GET 2022-09-03T06:07:32.705 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.706 DEBUG:boto:Data: 2022-09-03T06:07:32.707 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.707 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.708 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.708 DEBUG:boto:Params: {} 2022-09-03T06:07:32.709 DEBUG:boto:Token: None 2022-09-03T06:07:32.709 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.710 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iVh4ObycbzUGE/iExQtOyS1usvc= 2022-09-03T06:07:32.711 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iVh4ObycbzUGE/iExQtOyS1usvc=', 'Content-Length': '0'} 2022-09-03T06:07:32.713 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000dd5d5f88c4e42c27-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.714 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.715 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.715 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.716 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.716 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.717 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.717 DEBUG:boto:Data: 2022-09-03T06:07:32.718 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.719 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.719 DEBUG:boto:Params: {} 2022-09-03T06:07:32.720 DEBUG:boto:Token: None 2022-09-03T06:07:32.720 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4PTOJ5v5K50pIieYEgnMYfT4uPI= 2022-09-03T06:07:32.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4PTOJ5v5K50pIieYEgnMYfT4uPI=', 'Content-Length': '0'} 2022-09-03T06:07:32.724 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005ef660d706c7accc-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.724 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.725 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.725 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.726 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.727 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.727 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.728 DEBUG:boto:Data: 2022-09-03T06:07:32.728 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.729 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.729 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.730 DEBUG:boto:Params: {} 2022-09-03T06:07:32.730 DEBUG:boto:Token: None 2022-09-03T06:07:32.731 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4PTOJ5v5K50pIieYEgnMYfT4uPI= 2022-09-03T06:07:32.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4PTOJ5v5K50pIieYEgnMYfT4uPI=', 'Content-Length': '0'} 2022-09-03T06:07:32.734 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c769971dacdd240f-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.735 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.735 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.736 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.736 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.737 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.738 DEBUG:boto:Method: GET 2022-09-03T06:07:32.738 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.739 DEBUG:boto:Data: 2022-09-03T06:07:32.739 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.740 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.740 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.741 DEBUG:boto:Params: {} 2022-09-03T06:07:32.741 DEBUG:boto:Token: None 2022-09-03T06:07:32.742 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iVh4ObycbzUGE/iExQtOyS1usvc= 2022-09-03T06:07:32.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iVh4ObycbzUGE/iExQtOyS1usvc=', 'Content-Length': '0'} 2022-09-03T06:07:32.745 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003d35a5daa561c463-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.746 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.747 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.747 DEBUG:boto:path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.748 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.748 DEBUG:boto:Method: GET 2022-09-03T06:07:32.749 DEBUG:boto:Path: /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.749 DEBUG:boto:Data: 2022-09-03T06:07:32.750 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.750 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.751 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.751 DEBUG:boto:Params: {} 2022-09-03T06:07:32.752 DEBUG:boto:Token: None 2022-09-03T06:07:32.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=B712AEDTSngkrH4lM9kJJU1oPTSqH0U 2022-09-03T06:07:32.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iVh4ObycbzUGE/iExQtOyS1usvc= 2022-09-03T06:07:32.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iVh4ObycbzUGE/iExQtOyS1usvc=', 'Content-Length': '0'} 2022-09-03T06:07:32.756 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'B712AEDTSngkrH4lM9kJJU1oPTSqH0U'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000873c195758592e67-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.757 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.758 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.759 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.759 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.760 DEBUG:boto:Method: GET 2022-09-03T06:07:32.760 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.761 DEBUG:boto:Data: 2022-09-03T06:07:32.761 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.762 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.762 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.763 DEBUG:boto:Params: {} 2022-09-03T06:07:32.764 DEBUG:boto:Token: None 2022-09-03T06:07:32.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.769 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000384b495acc4d8258-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.769 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.770 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.771 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.772 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.773 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.774 DEBUG:boto:Method: GET 2022-09-03T06:07:32.774 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.775 DEBUG:boto:Data: 2022-09-03T06:07:32.776 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.777 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.778 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.778 DEBUG:boto:Params: {} 2022-09-03T06:07:32.779 DEBUG:boto:Token: None 2022-09-03T06:07:32.780 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.785 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7468c6b1442211c-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.786 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.787 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.787 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.788 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.788 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:32.789 DEBUG:boto:Data: 2022-09-03T06:07:32.789 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.790 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.790 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.791 DEBUG:boto:Params: {} 2022-09-03T06:07:32.792 DEBUG:boto:Token: None 2022-09-03T06:07:32.792 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj 2022-09-03T06:07:32.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M= 2022-09-03T06:07:32.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M=', 'Content-Length': '0'} 2022-09-03T06:07:32.797 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000caf8d6cb01db585d-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.798 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.799 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.799 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.800 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:32.800 DEBUG:boto:Data: 2022-09-03T06:07:32.801 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.801 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.802 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.802 DEBUG:boto:Params: {} 2022-09-03T06:07:32.803 DEBUG:boto:Token: None 2022-09-03T06:07:32.803 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj 2022-09-03T06:07:32.804 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M= 2022-09-03T06:07:32.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I7PXhSguhs8Z1qKODIA7bxi7X2M=', 'Content-Length': '0'} 2022-09-03T06:07:32.808 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006a1af260f5d3681e-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.809 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.810 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.810 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.811 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.812 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.813 DEBUG:boto:Method: GET 2022-09-03T06:07:32.814 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.815 DEBUG:boto:Data: 2022-09-03T06:07:32.816 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.818 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.819 DEBUG:boto:Params: {} 2022-09-03T06:07:32.820 DEBUG:boto:Token: None 2022-09-03T06:07:32.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34= 2022-09-03T06:07:32.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34=', 'Content-Length': '0'} 2022-09-03T06:07:32.826 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007c994287f15a9b5c-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.827 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.828 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.829 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.830 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.840 DEBUG:boto:Method: GET 2022-09-03T06:07:32.841 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.841 DEBUG:boto:Data: 2022-09-03T06:07:32.842 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.843 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.843 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.844 DEBUG:boto:Params: {} 2022-09-03T06:07:32.844 DEBUG:boto:Token: None 2022-09-03T06:07:32.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:32.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34= 2022-09-03T06:07:32.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XRp3DLfTBqZorw9NzuMUf129+34=', 'Content-Length': '0'} 2022-09-03T06:07:32.848 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002d4bc172dbe47600-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.849 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.850 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.851 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.851 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.852 DEBUG:boto:Method: GET 2022-09-03T06:07:32.852 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.853 DEBUG:boto:Data: 2022-09-03T06:07:32.853 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.854 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.855 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.855 DEBUG:boto:Params: {} 2022-09-03T06:07:32.856 DEBUG:boto:Token: None 2022-09-03T06:07:32.857 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.862 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000de4d5739d6015947-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.862 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.863 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:32.864 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:32.864 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.865 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:32.865 DEBUG:boto:Method: GET 2022-09-03T06:07:32.866 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:32.866 DEBUG:boto:Data: 2022-09-03T06:07:32.867 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.867 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.868 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.868 DEBUG:boto:Params: {} 2022-09-03T06:07:32.869 DEBUG:boto:Token: None 2022-09-03T06:07:32.870 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/ 2022-09-03T06:07:32.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk= 2022-09-03T06:07:32.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieZu+QW2ifj+k9Ea6jzMImPfFUk=', 'Content-Length': '0'} 2022-09-03T06:07:32.874 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001b613b06ab5002c2-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.875 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:32.875 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.876 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.877 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.877 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.878 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.878 DEBUG:boto:Method: GET 2022-09-03T06:07:32.879 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.879 DEBUG:boto:Data: 2022-09-03T06:07:32.880 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.881 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.881 DEBUG:boto:Params: {} 2022-09-03T06:07:32.882 DEBUG:boto:Token: None 2022-09-03T06:07:32.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ai8ZEASe5W/fuMFnvZyDf6D0sdM= 2022-09-03T06:07:32.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ai8ZEASe5W/fuMFnvZyDf6D0sdM=', 'Content-Length': '0'} 2022-09-03T06:07:32.886 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a7cff0656a222729-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.887 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.888 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.888 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.889 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.889 DEBUG:boto:Method: GET 2022-09-03T06:07:32.890 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.891 DEBUG:boto:Data: 2022-09-03T06:07:32.892 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.893 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.893 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.894 DEBUG:boto:Params: {} 2022-09-03T06:07:32.895 DEBUG:boto:Token: None 2022-09-03T06:07:32.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ai8ZEASe5W/fuMFnvZyDf6D0sdM= 2022-09-03T06:07:32.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ai8ZEASe5W/fuMFnvZyDf6D0sdM=', 'Content-Length': '0'} 2022-09-03T06:07:32.902 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000024b9111d51417258-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.942 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.943 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.943 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.944 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.944 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.945 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.945 DEBUG:boto:Data: 2022-09-03T06:07:32.946 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.949 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.949 DEBUG:boto:Params: {} 2022-09-03T06:07:32.950 DEBUG:boto:Token: None 2022-09-03T06:07:32.952 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8alA1RHplQGJe6yNzOKoXrjT5xI= 2022-09-03T06:07:32.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8alA1RHplQGJe6yNzOKoXrjT5xI=', 'Content-Length': '0'} 2022-09-03T06:07:32.956 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002835c35c22833922-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.958 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.959 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.960 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.960 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.961 DEBUG:boto:Method: HEAD 2022-09-03T06:07:32.962 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.963 DEBUG:boto:Data: 2022-09-03T06:07:32.964 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.965 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.966 DEBUG:boto:Params: {} 2022-09-03T06:07:32.967 DEBUG:boto:Token: None 2022-09-03T06:07:32.968 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8alA1RHplQGJe6yNzOKoXrjT5xI= 2022-09-03T06:07:32.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8alA1RHplQGJe6yNzOKoXrjT5xI=', 'Content-Length': '0'} 2022-09-03T06:07:32.973 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003cec7b7e9f8e7180-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.974 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:32.975 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.976 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.976 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.977 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.977 DEBUG:boto:Method: GET 2022-09-03T06:07:32.978 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.979 DEBUG:boto:Data: 2022-09-03T06:07:32.979 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:32.981 DEBUG:boto:Port: 8001 2022-09-03T06:07:32.982 DEBUG:boto:Params: {} 2022-09-03T06:07:32.983 DEBUG:boto:Token: None 2022-09-03T06:07:32.983 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.984 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ai8ZEASe5W/fuMFnvZyDf6D0sdM= 2022-09-03T06:07:32.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ai8ZEASe5W/fuMFnvZyDf6D0sdM=', 'Content-Length': '0'} 2022-09-03T06:07:32.987 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f951fbbc0587ca37-006312ef24-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:32.988 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:32.989 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:32.989 DEBUG:boto:path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.990 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.990 DEBUG:boto:Method: GET 2022-09-03T06:07:32.991 DEBUG:boto:Path: /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.992 DEBUG:boto:Data: 2022-09-03T06:07:32.992 DEBUG:boto:Headers: {} 2022-09-03T06:07:32.993 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:32.993 DEBUG:boto:Port: 8000 2022-09-03T06:07:32.994 DEBUG:boto:Params: {} 2022-09-03T06:07:32.995 DEBUG:boto:Token: None 2022-09-03T06:07:32.995 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:32 GMT /avysmw-19/obj?versionId=R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V 2022-09-03T06:07:32.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ai8ZEASe5W/fuMFnvZyDf6D0sdM= 2022-09-03T06:07:32.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ai8ZEASe5W/fuMFnvZyDf6D0sdM=', 'Content-Length': '0'} 2022-09-03T06:07:32.999 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'R78AB7kULQsVVpdt6q20Ckg2x2N8Q2V'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000051c4f724ed68ba2a-006312ef24-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.000 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.001 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.001 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.002 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.002 DEBUG:boto:Method: GET 2022-09-03T06:07:33.003 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.003 DEBUG:boto:Data: 2022-09-03T06:07:33.004 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.004 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.005 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.006 DEBUG:boto:Params: {} 2022-09-03T06:07:33.006 DEBUG:boto:Token: None 2022-09-03T06:07:33.007 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.012 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002fd75f9a0a2e24e2-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.058 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.060 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.061 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.062 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.063 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.064 DEBUG:boto:Method: GET 2022-09-03T06:07:33.065 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.066 DEBUG:boto:Data: 2022-09-03T06:07:33.067 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.068 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.069 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.070 DEBUG:boto:Params: {} 2022-09-03T06:07:33.071 DEBUG:boto:Token: None 2022-09-03T06:07:33.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.077 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e6a740a44295892-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.078 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.079 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.079 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.080 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.080 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:33.081 DEBUG:boto:Data: 2022-09-03T06:07:33.081 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.082 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.083 DEBUG:boto:Params: {} 2022-09-03T06:07:33.084 DEBUG:boto:Token: None 2022-09-03T06:07:33.084 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj 2022-09-03T06:07:33.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo= 2022-09-03T06:07:33.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo=', 'Content-Length': '0'} 2022-09-03T06:07:33.089 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ee321418fac660b0-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.090 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.090 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.091 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.091 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:33.092 DEBUG:boto:Data: 2022-09-03T06:07:33.092 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.093 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.093 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.094 DEBUG:boto:Params: {} 2022-09-03T06:07:33.095 DEBUG:boto:Token: None 2022-09-03T06:07:33.095 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj 2022-09-03T06:07:33.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo= 2022-09-03T06:07:33.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo=', 'Content-Length': '0'} 2022-09-03T06:07:33.100 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000076f46eb8f7526ebc-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.101 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.101 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.102 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.102 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.103 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.103 DEBUG:boto:Method: GET 2022-09-03T06:07:33.104 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.104 DEBUG:boto:Data: 2022-09-03T06:07:33.105 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.106 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.107 DEBUG:boto:Params: {} 2022-09-03T06:07:33.107 DEBUG:boto:Token: None 2022-09-03T06:07:33.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE= 2022-09-03T06:07:33.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE=', 'Content-Length': '0'} 2022-09-03T06:07:33.112 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009cff1be56d06ed6b-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.113 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.113 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.114 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.114 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.115 DEBUG:boto:Method: GET 2022-09-03T06:07:33.115 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.116 DEBUG:boto:Data: 2022-09-03T06:07:33.116 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.117 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.117 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.118 DEBUG:boto:Params: {} 2022-09-03T06:07:33.118 DEBUG:boto:Token: None 2022-09-03T06:07:33.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE= 2022-09-03T06:07:33.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE=', 'Content-Length': '0'} 2022-09-03T06:07:33.122 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000007a15cddc72a7a00-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.123 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.124 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.124 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.125 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.125 DEBUG:boto:Method: GET 2022-09-03T06:07:33.126 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.127 DEBUG:boto:Data: 2022-09-03T06:07:33.128 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.129 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.130 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.131 DEBUG:boto:Params: {} 2022-09-03T06:07:33.132 DEBUG:boto:Token: None 2022-09-03T06:07:33.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.139 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000f138f0ddddbd529-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.140 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.141 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.141 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.142 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.142 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.143 DEBUG:boto:Method: GET 2022-09-03T06:07:33.144 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.144 DEBUG:boto:Data: 2022-09-03T06:07:33.145 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.145 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.146 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.146 DEBUG:boto:Params: {} 2022-09-03T06:07:33.147 DEBUG:boto:Token: None 2022-09-03T06:07:33.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.157 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e57805dcbfbf86a-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.198 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.201 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.202 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.203 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.204 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.205 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.205 DEBUG:boto:Method: GET 2022-09-03T06:07:33.206 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.208 DEBUG:boto:Data: 2022-09-03T06:07:33.209 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.211 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.212 DEBUG:boto:Params: {} 2022-09-03T06:07:33.213 DEBUG:boto:Token: None 2022-09-03T06:07:33.213 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BL+Siva4fCaACpWtLjdGVdDXwEE= 2022-09-03T06:07:33.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BL+Siva4fCaACpWtLjdGVdDXwEE=', 'Content-Length': '0'} 2022-09-03T06:07:33.217 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a718788ed2c67d46-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.218 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.219 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.220 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.221 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.222 DEBUG:boto:Method: GET 2022-09-03T06:07:33.222 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.223 DEBUG:boto:Data: 2022-09-03T06:07:33.223 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.224 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.225 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.225 DEBUG:boto:Params: {} 2022-09-03T06:07:33.226 DEBUG:boto:Token: None 2022-09-03T06:07:33.227 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BL+Siva4fCaACpWtLjdGVdDXwEE= 2022-09-03T06:07:33.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BL+Siva4fCaACpWtLjdGVdDXwEE=', 'Content-Length': '0'} 2022-09-03T06:07:33.232 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002a783711caee8e6a-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.233 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.234 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.235 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.236 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.237 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.238 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.239 DEBUG:boto:Data: 2022-09-03T06:07:33.240 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.241 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.241 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.242 DEBUG:boto:Params: {} 2022-09-03T06:07:33.243 DEBUG:boto:Token: None 2022-09-03T06:07:33.243 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NoEdLJRRqKPblhQY0kZ/yPA72pw= 2022-09-03T06:07:33.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NoEdLJRRqKPblhQY0kZ/yPA72pw=', 'Content-Length': '0'} 2022-09-03T06:07:33.247 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e36f6a69c1ac7a6b-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.249 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.250 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.251 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.252 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.252 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.253 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.254 DEBUG:boto:Data: 2022-09-03T06:07:33.255 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.256 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.257 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.258 DEBUG:boto:Params: {} 2022-09-03T06:07:33.259 DEBUG:boto:Token: None 2022-09-03T06:07:33.260 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NoEdLJRRqKPblhQY0kZ/yPA72pw= 2022-09-03T06:07:33.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NoEdLJRRqKPblhQY0kZ/yPA72pw=', 'Content-Length': '0'} 2022-09-03T06:07:33.263 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b0550d5d7e6f7796-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.264 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.265 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.265 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.266 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.266 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.267 DEBUG:boto:Method: GET 2022-09-03T06:07:33.268 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.268 DEBUG:boto:Data: 2022-09-03T06:07:33.269 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.270 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.271 DEBUG:boto:Params: {} 2022-09-03T06:07:33.271 DEBUG:boto:Token: None 2022-09-03T06:07:33.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BL+Siva4fCaACpWtLjdGVdDXwEE= 2022-09-03T06:07:33.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BL+Siva4fCaACpWtLjdGVdDXwEE=', 'Content-Length': '0'} 2022-09-03T06:07:33.276 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c1e3c8af1b8ce6f4-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.277 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.278 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.279 DEBUG:boto:path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.280 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.281 DEBUG:boto:Method: GET 2022-09-03T06:07:33.282 DEBUG:boto:Path: /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.283 DEBUG:boto:Data: 2022-09-03T06:07:33.283 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.284 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.284 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.285 DEBUG:boto:Params: {} 2022-09-03T06:07:33.285 DEBUG:boto:Token: None 2022-09-03T06:07:33.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=nmzhy11EG9ZQx.u878SRuCGqgFPvjpg 2022-09-03T06:07:33.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BL+Siva4fCaACpWtLjdGVdDXwEE= 2022-09-03T06:07:33.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BL+Siva4fCaACpWtLjdGVdDXwEE=', 'Content-Length': '0'} 2022-09-03T06:07:33.289 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'nmzhy11EG9ZQx.u878SRuCGqgFPvjpg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000912fdba0753afe5c-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.290 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.291 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.291 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.292 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.292 DEBUG:boto:Method: GET 2022-09-03T06:07:33.293 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.293 DEBUG:boto:Data: 2022-09-03T06:07:33.294 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.295 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.295 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.296 DEBUG:boto:Params: {} 2022-09-03T06:07:33.296 DEBUG:boto:Token: None 2022-09-03T06:07:33.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000033e29fcbf749d76-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.301 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.302 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.303 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.303 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.304 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.304 DEBUG:boto:Method: GET 2022-09-03T06:07:33.305 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.305 DEBUG:boto:Data: 2022-09-03T06:07:33.306 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.306 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.307 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.307 DEBUG:boto:Params: {} 2022-09-03T06:07:33.308 DEBUG:boto:Token: None 2022-09-03T06:07:33.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.313 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d086ffe2dbfa8700-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.313 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.314 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.315 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.315 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.316 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:33.317 DEBUG:boto:Data: 2022-09-03T06:07:33.317 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.318 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.318 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.319 DEBUG:boto:Params: {} 2022-09-03T06:07:33.319 DEBUG:boto:Token: None 2022-09-03T06:07:33.320 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj 2022-09-03T06:07:33.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo= 2022-09-03T06:07:33.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo=', 'Content-Length': '0'} 2022-09-03T06:07:33.324 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000074abe76eb1207301-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.325 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.326 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.326 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.327 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:33.327 DEBUG:boto:Data: 2022-09-03T06:07:33.328 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.328 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.329 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.329 DEBUG:boto:Params: {} 2022-09-03T06:07:33.330 DEBUG:boto:Token: None 2022-09-03T06:07:33.330 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj 2022-09-03T06:07:33.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo= 2022-09-03T06:07:33.332 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo=', 'Content-Length': '0'} 2022-09-03T06:07:33.336 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000046166476b7e2fc3d-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.337 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.337 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.338 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.338 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.339 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.339 DEBUG:boto:Method: GET 2022-09-03T06:07:33.340 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.341 DEBUG:boto:Data: 2022-09-03T06:07:33.341 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.342 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.342 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.343 DEBUG:boto:Params: {} 2022-09-03T06:07:33.343 DEBUG:boto:Token: None 2022-09-03T06:07:33.344 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE= 2022-09-03T06:07:33.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE=', 'Content-Length': '0'} 2022-09-03T06:07:33.347 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000115468cb34067f1a-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.348 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.349 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.349 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.350 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.351 DEBUG:boto:Method: GET 2022-09-03T06:07:33.351 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.352 DEBUG:boto:Data: 2022-09-03T06:07:33.352 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.353 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.353 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.354 DEBUG:boto:Params: {} 2022-09-03T06:07:33.355 DEBUG:boto:Token: None 2022-09-03T06:07:33.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE= 2022-09-03T06:07:33.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE=', 'Content-Length': '0'} 2022-09-03T06:07:33.359 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002ac3f5c2a36840e2-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.360 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.360 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.361 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.361 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.362 DEBUG:boto:Method: GET 2022-09-03T06:07:33.362 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.363 DEBUG:boto:Data: 2022-09-03T06:07:33.364 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.364 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.365 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.365 DEBUG:boto:Params: {} 2022-09-03T06:07:33.366 DEBUG:boto:Token: None 2022-09-03T06:07:33.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.371 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003a468cc2f26d3870-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.373 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.373 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.374 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.375 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.375 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.376 DEBUG:boto:Method: GET 2022-09-03T06:07:33.376 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.377 DEBUG:boto:Data: 2022-09-03T06:07:33.377 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.378 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.378 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.379 DEBUG:boto:Params: {} 2022-09-03T06:07:33.380 DEBUG:boto:Token: None 2022-09-03T06:07:33.380 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.384 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b08733b836b1bda-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.426 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.427 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.428 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.428 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.429 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.429 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.430 DEBUG:boto:Method: GET 2022-09-03T06:07:33.431 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.431 DEBUG:boto:Data: 2022-09-03T06:07:33.432 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.433 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.433 DEBUG:boto:Params: {} 2022-09-03T06:07:33.434 DEBUG:boto:Token: None 2022-09-03T06:07:33.435 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.436 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5UdO6IWTsh/lfXbYlP+kku8UNSc= 2022-09-03T06:07:33.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5UdO6IWTsh/lfXbYlP+kku8UNSc=', 'Content-Length': '0'} 2022-09-03T06:07:33.439 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001f156edf73eecca3-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.442 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.443 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.444 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.444 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.445 DEBUG:boto:Method: GET 2022-09-03T06:07:33.446 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.447 DEBUG:boto:Data: 2022-09-03T06:07:33.447 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.448 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.448 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.449 DEBUG:boto:Params: {} 2022-09-03T06:07:33.450 DEBUG:boto:Token: None 2022-09-03T06:07:33.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5UdO6IWTsh/lfXbYlP+kku8UNSc= 2022-09-03T06:07:33.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5UdO6IWTsh/lfXbYlP+kku8UNSc=', 'Content-Length': '0'} 2022-09-03T06:07:33.454 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000090f98dc89bb2f5b4-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.456 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.457 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.458 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.460 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.461 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.462 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.463 DEBUG:boto:Data: 2022-09-03T06:07:33.463 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.464 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.465 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.466 DEBUG:boto:Params: {} 2022-09-03T06:07:33.467 DEBUG:boto:Token: None 2022-09-03T06:07:33.468 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PhScrMVf3TmENWlk2M6SBYp4D2E= 2022-09-03T06:07:33.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PhScrMVf3TmENWlk2M6SBYp4D2E=', 'Content-Length': '0'} 2022-09-03T06:07:33.471 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000547b80561e60781c-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.472 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.473 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.473 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.474 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.474 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.475 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.475 DEBUG:boto:Data: 2022-09-03T06:07:33.476 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.476 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.477 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.477 DEBUG:boto:Params: {} 2022-09-03T06:07:33.478 DEBUG:boto:Token: None 2022-09-03T06:07:33.478 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PhScrMVf3TmENWlk2M6SBYp4D2E= 2022-09-03T06:07:33.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PhScrMVf3TmENWlk2M6SBYp4D2E=', 'Content-Length': '0'} 2022-09-03T06:07:33.482 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000007cc7984634a3773-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.482 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.483 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.484 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.484 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.485 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.486 DEBUG:boto:Method: GET 2022-09-03T06:07:33.486 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.487 DEBUG:boto:Data: 2022-09-03T06:07:33.487 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.489 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.489 DEBUG:boto:Params: {} 2022-09-03T06:07:33.490 DEBUG:boto:Token: None 2022-09-03T06:07:33.490 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5UdO6IWTsh/lfXbYlP+kku8UNSc= 2022-09-03T06:07:33.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5UdO6IWTsh/lfXbYlP+kku8UNSc=', 'Content-Length': '0'} 2022-09-03T06:07:33.494 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001f3a888b7acb3c81-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.495 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.497 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.497 DEBUG:boto:path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.498 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.499 DEBUG:boto:Method: GET 2022-09-03T06:07:33.500 DEBUG:boto:Path: /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.501 DEBUG:boto:Data: 2022-09-03T06:07:33.502 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.503 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.504 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.505 DEBUG:boto:Params: {} 2022-09-03T06:07:33.506 DEBUG:boto:Token: None 2022-09-03T06:07:33.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=6uV51ig1eoLqPWbieLYdwHpl6xaQYjg 2022-09-03T06:07:33.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5UdO6IWTsh/lfXbYlP+kku8UNSc= 2022-09-03T06:07:33.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5UdO6IWTsh/lfXbYlP+kku8UNSc=', 'Content-Length': '0'} 2022-09-03T06:07:33.510 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '6uV51ig1eoLqPWbieLYdwHpl6xaQYjg'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000000793128a7303157-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.511 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.512 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.512 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.513 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.513 DEBUG:boto:Method: GET 2022-09-03T06:07:33.514 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.514 DEBUG:boto:Data: 2022-09-03T06:07:33.515 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.517 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.517 DEBUG:boto:Params: {} 2022-09-03T06:07:33.518 DEBUG:boto:Token: None 2022-09-03T06:07:33.518 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.524 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b09d96eeb1cc6eca-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.525 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.525 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.526 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.527 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.527 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.528 DEBUG:boto:Method: GET 2022-09-03T06:07:33.528 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.529 DEBUG:boto:Data: 2022-09-03T06:07:33.530 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.531 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.532 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.533 DEBUG:boto:Params: {} 2022-09-03T06:07:33.534 DEBUG:boto:Token: None 2022-09-03T06:07:33.535 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.536 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.541 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005d863f66bb416f4d-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.586 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.587 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.588 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.589 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.589 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:33.590 DEBUG:boto:Data: 2022-09-03T06:07:33.590 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.591 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.592 DEBUG:boto:Params: {} 2022-09-03T06:07:33.592 DEBUG:boto:Token: None 2022-09-03T06:07:33.593 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj 2022-09-03T06:07:33.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo= 2022-09-03T06:07:33.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo=', 'Content-Length': '0'} 2022-09-03T06:07:33.598 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002438475144c3b8dc-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.599 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.600 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.600 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.601 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:33.601 DEBUG:boto:Data: 2022-09-03T06:07:33.602 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.602 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.603 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.603 DEBUG:boto:Params: {} 2022-09-03T06:07:33.604 DEBUG:boto:Token: None 2022-09-03T06:07:33.604 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj 2022-09-03T06:07:33.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo= 2022-09-03T06:07:33.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo=', 'Content-Length': '0'} 2022-09-03T06:07:33.609 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009038aff632701e95-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.610 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.610 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.611 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.611 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.612 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.612 DEBUG:boto:Method: GET 2022-09-03T06:07:33.613 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.613 DEBUG:boto:Data: 2022-09-03T06:07:33.614 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.615 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.615 DEBUG:boto:Params: {} 2022-09-03T06:07:33.616 DEBUG:boto:Token: None 2022-09-03T06:07:33.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE= 2022-09-03T06:07:33.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE=', 'Content-Length': '0'} 2022-09-03T06:07:33.620 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000073e7d708240eff71-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.666 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.667 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.668 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.669 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.669 DEBUG:boto:Method: GET 2022-09-03T06:07:33.670 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.670 DEBUG:boto:Data: 2022-09-03T06:07:33.671 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.671 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.672 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.672 DEBUG:boto:Params: {} 2022-09-03T06:07:33.673 DEBUG:boto:Token: None 2022-09-03T06:07:33.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.674 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE= 2022-09-03T06:07:33.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE=', 'Content-Length': '0'} 2022-09-03T06:07:33.677 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000069c82c080266870f-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.678 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.679 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.679 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.680 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.681 DEBUG:boto:Method: GET 2022-09-03T06:07:33.682 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.683 DEBUG:boto:Data: 2022-09-03T06:07:33.683 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.684 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.685 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.686 DEBUG:boto:Params: {} 2022-09-03T06:07:33.686 DEBUG:boto:Token: None 2022-09-03T06:07:33.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.692 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bf1335f3902c45b7-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.692 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.693 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.694 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.695 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.695 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.696 DEBUG:boto:Method: GET 2022-09-03T06:07:33.696 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.697 DEBUG:boto:Data: 2022-09-03T06:07:33.698 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.698 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.699 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.699 DEBUG:boto:Params: {} 2022-09-03T06:07:33.700 DEBUG:boto:Token: None 2022-09-03T06:07:33.700 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.701 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.702 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.704 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3c48481e69f471a-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.705 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.706 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.707 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.707 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.708 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.708 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.709 DEBUG:boto:Method: GET 2022-09-03T06:07:33.709 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.710 DEBUG:boto:Data: 2022-09-03T06:07:33.710 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.711 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.711 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.712 DEBUG:boto:Params: {} 2022-09-03T06:07:33.712 DEBUG:boto:Token: None 2022-09-03T06:07:33.713 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:31sqY55S6XhlkzuHkxiOJ6fdo+I= 2022-09-03T06:07:33.714 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:31sqY55S6XhlkzuHkxiOJ6fdo+I=', 'Content-Length': '0'} 2022-09-03T06:07:33.716 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008fc0d6133b4bf97c-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.717 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.718 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.718 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.719 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.719 DEBUG:boto:Method: GET 2022-09-03T06:07:33.720 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.720 DEBUG:boto:Data: 2022-09-03T06:07:33.721 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.721 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.722 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.722 DEBUG:boto:Params: {} 2022-09-03T06:07:33.723 DEBUG:boto:Token: None 2022-09-03T06:07:33.724 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:31sqY55S6XhlkzuHkxiOJ6fdo+I= 2022-09-03T06:07:33.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:31sqY55S6XhlkzuHkxiOJ6fdo+I=', 'Content-Length': '0'} 2022-09-03T06:07:33.728 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000000d7bcc623f625bf-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.729 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.729 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.730 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.730 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.731 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.731 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.732 DEBUG:boto:Data: 2022-09-03T06:07:33.732 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.733 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.734 DEBUG:boto:Params: {} 2022-09-03T06:07:33.734 DEBUG:boto:Token: None 2022-09-03T06:07:33.735 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4R3iwhpKLF12xunWFOn1bLyPum4= 2022-09-03T06:07:33.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4R3iwhpKLF12xunWFOn1bLyPum4=', 'Content-Length': '0'} 2022-09-03T06:07:33.738 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000042fe9aab26bf81be-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.739 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.740 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.740 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.740 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.741 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.741 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.742 DEBUG:boto:Data: 2022-09-03T06:07:33.743 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.743 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.744 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.744 DEBUG:boto:Params: {} 2022-09-03T06:07:33.745 DEBUG:boto:Token: None 2022-09-03T06:07:33.745 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4R3iwhpKLF12xunWFOn1bLyPum4= 2022-09-03T06:07:33.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4R3iwhpKLF12xunWFOn1bLyPum4=', 'Content-Length': '0'} 2022-09-03T06:07:33.748 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000541b71bedb24fb14-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.749 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.750 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.750 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.751 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.752 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.752 DEBUG:boto:Method: GET 2022-09-03T06:07:33.753 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.753 DEBUG:boto:Data: 2022-09-03T06:07:33.754 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.754 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.755 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.756 DEBUG:boto:Params: {} 2022-09-03T06:07:33.756 DEBUG:boto:Token: None 2022-09-03T06:07:33.757 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:31sqY55S6XhlkzuHkxiOJ6fdo+I= 2022-09-03T06:07:33.758 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:31sqY55S6XhlkzuHkxiOJ6fdo+I=', 'Content-Length': '0'} 2022-09-03T06:07:33.760 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000059b3e859fde89a9f-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.806 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.807 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.808 DEBUG:boto:path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.808 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.809 DEBUG:boto:Method: GET 2022-09-03T06:07:33.810 DEBUG:boto:Path: /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.811 DEBUG:boto:Data: 2022-09-03T06:07:33.812 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.813 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.813 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.814 DEBUG:boto:Params: {} 2022-09-03T06:07:33.815 DEBUG:boto:Token: None 2022-09-03T06:07:33.816 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN 2022-09-03T06:07:33.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:31sqY55S6XhlkzuHkxiOJ6fdo+I= 2022-09-03T06:07:33.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:31sqY55S6XhlkzuHkxiOJ6fdo+I=', 'Content-Length': '0'} 2022-09-03T06:07:33.821 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'a11VL7IeGSUXvmP2Q0EwVOSErHiHlcN'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bdca431d5cd24ee4-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.822 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.823 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.824 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.825 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.826 DEBUG:boto:Method: GET 2022-09-03T06:07:33.827 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.828 DEBUG:boto:Data: 2022-09-03T06:07:33.829 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.829 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.830 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.831 DEBUG:boto:Params: {} 2022-09-03T06:07:33.831 DEBUG:boto:Token: None 2022-09-03T06:07:33.832 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.837 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db52e77fd270dc75-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.838 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.839 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.840 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.840 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.841 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.842 DEBUG:boto:Method: GET 2022-09-03T06:07:33.843 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.844 DEBUG:boto:Data: 2022-09-03T06:07:33.845 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.846 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.847 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.848 DEBUG:boto:Params: {} 2022-09-03T06:07:33.848 DEBUG:boto:Token: None 2022-09-03T06:07:33.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.854 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b40a45c44cae7e2a-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.854 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.855 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.856 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.856 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.857 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:33.857 DEBUG:boto:Data: 2022-09-03T06:07:33.858 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.859 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.859 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.860 DEBUG:boto:Params: {} 2022-09-03T06:07:33.860 DEBUG:boto:Token: None 2022-09-03T06:07:33.861 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj 2022-09-03T06:07:33.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo= 2022-09-03T06:07:33.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo=', 'Content-Length': '0'} 2022-09-03T06:07:33.867 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a48cb06a8ef43442-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.868 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.869 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.869 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.870 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:33.871 DEBUG:boto:Data: 2022-09-03T06:07:33.871 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.872 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.872 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.873 DEBUG:boto:Params: {} 2022-09-03T06:07:33.873 DEBUG:boto:Token: None 2022-09-03T06:07:33.874 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj 2022-09-03T06:07:33.875 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo= 2022-09-03T06:07:33.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kCE6IKU3f4hp3injuzJQrK94DFo=', 'Content-Length': '0'} 2022-09-03T06:07:33.879 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000105a2417da87095c-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.880 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.881 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.881 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.882 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.882 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.883 DEBUG:boto:Method: GET 2022-09-03T06:07:33.884 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.885 DEBUG:boto:Data: 2022-09-03T06:07:33.885 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.886 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.887 DEBUG:boto:Params: {} 2022-09-03T06:07:33.888 DEBUG:boto:Token: None 2022-09-03T06:07:33.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE= 2022-09-03T06:07:33.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE=', 'Content-Length': '0'} 2022-09-03T06:07:33.892 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004c2c6c8840624c0a-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.893 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.894 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.895 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.895 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.896 DEBUG:boto:Method: GET 2022-09-03T06:07:33.897 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.898 DEBUG:boto:Data: 2022-09-03T06:07:33.899 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.900 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.900 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.901 DEBUG:boto:Params: {} 2022-09-03T06:07:33.901 DEBUG:boto:Token: None 2022-09-03T06:07:33.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:33.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE= 2022-09-03T06:07:33.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HTDX8mRWs2xv3pfPBGKF5AZg4JE=', 'Content-Length': '0'} 2022-09-03T06:07:33.906 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000105e1f91e55d28a4-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.906 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.907 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.907 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.908 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.908 DEBUG:boto:Method: GET 2022-09-03T06:07:33.909 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.909 DEBUG:boto:Data: 2022-09-03T06:07:33.910 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.911 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.911 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.912 DEBUG:boto:Params: {} 2022-09-03T06:07:33.912 DEBUG:boto:Token: None 2022-09-03T06:07:33.913 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.917 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000cc26ff12f7cfd66-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.918 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.918 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:33.919 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:33.920 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.920 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:33.921 DEBUG:boto:Method: GET 2022-09-03T06:07:33.921 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:33.922 DEBUG:boto:Data: 2022-09-03T06:07:33.922 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.923 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.923 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.924 DEBUG:boto:Params: {} 2022-09-03T06:07:33.924 DEBUG:boto:Token: None 2022-09-03T06:07:33.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/ 2022-09-03T06:07:33.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM= 2022-09-03T06:07:33.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hoy65zJHrTRP2GOtBDt2stcAOgM=', 'Content-Length': '0'} 2022-09-03T06:07:33.929 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003c3a0ef41e703140-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.930 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:33.931 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.931 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.932 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.933 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.933 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.934 DEBUG:boto:Method: GET 2022-09-03T06:07:33.934 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.935 DEBUG:boto:Data: 2022-09-03T06:07:33.936 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.937 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.938 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.939 DEBUG:boto:Params: {} 2022-09-03T06:07:33.940 DEBUG:boto:Token: None 2022-09-03T06:07:33.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zabCRcgEwMrkrBi2y/JuKP73x4U= 2022-09-03T06:07:33.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zabCRcgEwMrkrBi2y/JuKP73x4U=', 'Content-Length': '0'} 2022-09-03T06:07:33.946 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008465d2245ff97dc0-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.947 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.948 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.948 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.949 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.949 DEBUG:boto:Method: GET 2022-09-03T06:07:33.950 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.950 DEBUG:boto:Data: 2022-09-03T06:07:33.951 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.951 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.952 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.952 DEBUG:boto:Params: {} 2022-09-03T06:07:33.953 DEBUG:boto:Token: None 2022-09-03T06:07:33.954 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zabCRcgEwMrkrBi2y/JuKP73x4U= 2022-09-03T06:07:33.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zabCRcgEwMrkrBi2y/JuKP73x4U=', 'Content-Length': '0'} 2022-09-03T06:07:33.958 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cbad9d9437d5aa30-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.959 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.960 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.960 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.961 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.961 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.962 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.962 DEBUG:boto:Data: 2022-09-03T06:07:33.963 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.964 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.964 DEBUG:boto:Params: {} 2022-09-03T06:07:33.965 DEBUG:boto:Token: None 2022-09-03T06:07:33.966 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eZ05gZCF8mSxoaRe69XV9QB+b3Y= 2022-09-03T06:07:33.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eZ05gZCF8mSxoaRe69XV9QB+b3Y=', 'Content-Length': '0'} 2022-09-03T06:07:33.969 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007390c9aeff10985e-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.970 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.970 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.971 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.972 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.972 DEBUG:boto:Method: HEAD 2022-09-03T06:07:33.973 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.973 DEBUG:boto:Data: 2022-09-03T06:07:33.974 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.974 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:33.975 DEBUG:boto:Port: 8000 2022-09-03T06:07:33.975 DEBUG:boto:Params: {} 2022-09-03T06:07:33.976 DEBUG:boto:Token: None 2022-09-03T06:07:33.976 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eZ05gZCF8mSxoaRe69XV9QB+b3Y= 2022-09-03T06:07:33.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eZ05gZCF8mSxoaRe69XV9QB+b3Y=', 'Content-Length': '0'} 2022-09-03T06:07:33.980 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bab54279396c3053-006312ef25-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:33.980 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:33.981 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:33.982 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:33.982 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.983 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.983 DEBUG:boto:Method: GET 2022-09-03T06:07:33.984 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.984 DEBUG:boto:Data: 2022-09-03T06:07:33.985 DEBUG:boto:Headers: {} 2022-09-03T06:07:33.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:33.986 DEBUG:boto:Port: 8001 2022-09-03T06:07:33.987 DEBUG:boto:Params: {} 2022-09-03T06:07:33.987 DEBUG:boto:Token: None 2022-09-03T06:07:33.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:33 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:33.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zabCRcgEwMrkrBi2y/JuKP73x4U= 2022-09-03T06:07:33.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zabCRcgEwMrkrBi2y/JuKP73x4U=', 'Content-Length': '0'} 2022-09-03T06:07:33.992 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000021f16a84d94b3de9-006312ef25-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.034 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.035 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.035 DEBUG:boto:path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:34.036 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:34.037 DEBUG:boto:Method: GET 2022-09-03T06:07:34.037 DEBUG:boto:Path: /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:34.038 DEBUG:boto:Data: 2022-09-03T06:07:34.038 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.039 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.040 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.040 DEBUG:boto:Params: {} 2022-09-03T06:07:34.041 DEBUG:boto:Token: None 2022-09-03T06:07:34.042 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw 2022-09-03T06:07:34.042 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V9Hg2awupapO0zfbCWCDV/r/0FM= 2022-09-03T06:07:34.043 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V9Hg2awupapO0zfbCWCDV/r/0FM=', 'Content-Length': '0'} 2022-09-03T06:07:34.045 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'oMSCsFJe0Z8nu5tWkj2H4ZD2yg75tkw'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b5ab0ab209ecf4fc-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.046 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.046 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.047 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.047 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.048 DEBUG:boto:Method: GET 2022-09-03T06:07:34.048 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.049 DEBUG:boto:Data: 2022-09-03T06:07:34.049 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.051 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.051 DEBUG:boto:Params: {} 2022-09-03T06:07:34.052 DEBUG:boto:Token: None 2022-09-03T06:07:34.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.057 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006ddbb759b057854c-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.057 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.058 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.059 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.059 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.060 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.061 DEBUG:boto:Method: GET 2022-09-03T06:07:34.061 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.062 DEBUG:boto:Data: 2022-09-03T06:07:34.062 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.063 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.064 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.065 DEBUG:boto:Params: {} 2022-09-03T06:07:34.066 DEBUG:boto:Token: None 2022-09-03T06:07:34.067 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.073 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005828dd42650a5dbf-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.074 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.075 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.075 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.076 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.076 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:34.077 DEBUG:boto:Data: 2022-09-03T06:07:34.077 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.078 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.078 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.079 DEBUG:boto:Params: {} 2022-09-03T06:07:34.080 DEBUG:boto:Token: None 2022-09-03T06:07:34.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj 2022-09-03T06:07:34.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY= 2022-09-03T06:07:34.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY=', 'Content-Length': '0'} 2022-09-03T06:07:34.085 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003ae6c5371da12842-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.086 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.087 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.087 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.088 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:34.088 DEBUG:boto:Data: 2022-09-03T06:07:34.089 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.089 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.090 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.090 DEBUG:boto:Params: {} 2022-09-03T06:07:34.091 DEBUG:boto:Token: None 2022-09-03T06:07:34.091 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj 2022-09-03T06:07:34.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY= 2022-09-03T06:07:34.092 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY=', 'Content-Length': '0'} 2022-09-03T06:07:34.096 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e46126a13add49ee-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.097 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.097 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.098 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.098 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.099 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.099 DEBUG:boto:Method: GET 2022-09-03T06:07:34.100 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.100 DEBUG:boto:Data: 2022-09-03T06:07:34.101 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.101 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.102 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.102 DEBUG:boto:Params: {} 2022-09-03T06:07:34.103 DEBUG:boto:Token: None 2022-09-03T06:07:34.104 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8= 2022-09-03T06:07:34.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8=', 'Content-Length': '0'} 2022-09-03T06:07:34.107 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ec0f02f16adc3294-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.108 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.108 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.109 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.109 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.110 DEBUG:boto:Method: GET 2022-09-03T06:07:34.110 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.111 DEBUG:boto:Data: 2022-09-03T06:07:34.111 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.112 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.112 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.113 DEBUG:boto:Params: {} 2022-09-03T06:07:34.113 DEBUG:boto:Token: None 2022-09-03T06:07:34.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8= 2022-09-03T06:07:34.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8=', 'Content-Length': '0'} 2022-09-03T06:07:34.117 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000097a9e44546edf3b3-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.118 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.119 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.119 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.120 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.120 DEBUG:boto:Method: GET 2022-09-03T06:07:34.121 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.121 DEBUG:boto:Data: 2022-09-03T06:07:34.122 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.123 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.123 DEBUG:boto:Params: {} 2022-09-03T06:07:34.124 DEBUG:boto:Token: None 2022-09-03T06:07:34.125 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.129 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001026e0fa905d8a20-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.129 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.130 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.131 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.131 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.132 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.132 DEBUG:boto:Method: GET 2022-09-03T06:07:34.133 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.133 DEBUG:boto:Data: 2022-09-03T06:07:34.134 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.134 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.135 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.135 DEBUG:boto:Params: {} 2022-09-03T06:07:34.136 DEBUG:boto:Token: None 2022-09-03T06:07:34.137 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.141 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000056432828ccf7f5ee-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.141 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.143 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.143 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.144 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.145 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.146 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.147 DEBUG:boto:Method: GET 2022-09-03T06:07:34.148 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.149 DEBUG:boto:Data: 2022-09-03T06:07:34.150 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.151 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.152 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.152 DEBUG:boto:Params: {} 2022-09-03T06:07:34.153 DEBUG:boto:Token: None 2022-09-03T06:07:34.154 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4vGkKn+oWGbwIcsKJKt80vKuyPQ= 2022-09-03T06:07:34.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4vGkKn+oWGbwIcsKJKt80vKuyPQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.158 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000455c8bba6843ecb-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.159 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.159 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.160 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.161 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.161 DEBUG:boto:Method: GET 2022-09-03T06:07:34.162 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.162 DEBUG:boto:Data: 2022-09-03T06:07:34.163 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.164 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.164 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.165 DEBUG:boto:Params: {} 2022-09-03T06:07:34.165 DEBUG:boto:Token: None 2022-09-03T06:07:34.166 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4vGkKn+oWGbwIcsKJKt80vKuyPQ= 2022-09-03T06:07:34.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4vGkKn+oWGbwIcsKJKt80vKuyPQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.170 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000019dce130811545b2-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.171 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.172 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.172 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.173 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.173 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.174 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.174 DEBUG:boto:Data: 2022-09-03T06:07:34.175 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.176 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.176 DEBUG:boto:Params: {} 2022-09-03T06:07:34.177 DEBUG:boto:Token: None 2022-09-03T06:07:34.178 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PFisT2zRmA4mms47XFkHY0J6jcY= 2022-09-03T06:07:34.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PFisT2zRmA4mms47XFkHY0J6jcY=', 'Content-Length': '0'} 2022-09-03T06:07:34.181 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000005c84606a7b9d8f5-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.182 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.182 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.183 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.183 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.184 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.184 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.185 DEBUG:boto:Data: 2022-09-03T06:07:34.185 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.186 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.186 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.187 DEBUG:boto:Params: {} 2022-09-03T06:07:34.187 DEBUG:boto:Token: None 2022-09-03T06:07:34.188 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PFisT2zRmA4mms47XFkHY0J6jcY= 2022-09-03T06:07:34.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PFisT2zRmA4mms47XFkHY0J6jcY=', 'Content-Length': '0'} 2022-09-03T06:07:34.191 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fc53fef2ebc1e033-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.192 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.193 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.193 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.194 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.194 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.195 DEBUG:boto:Method: GET 2022-09-03T06:07:34.195 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.196 DEBUG:boto:Data: 2022-09-03T06:07:34.196 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.197 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.198 DEBUG:boto:Params: {} 2022-09-03T06:07:34.198 DEBUG:boto:Token: None 2022-09-03T06:07:34.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4vGkKn+oWGbwIcsKJKt80vKuyPQ= 2022-09-03T06:07:34.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4vGkKn+oWGbwIcsKJKt80vKuyPQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.203 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000056fa504abb72f67e-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.204 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.204 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.205 DEBUG:boto:path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.205 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.206 DEBUG:boto:Method: GET 2022-09-03T06:07:34.207 DEBUG:boto:Path: /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.207 DEBUG:boto:Data: 2022-09-03T06:07:34.208 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.208 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.209 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.209 DEBUG:boto:Params: {} 2022-09-03T06:07:34.210 DEBUG:boto:Token: None 2022-09-03T06:07:34.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=34cbRa0IccRiFCht3CqirmXYwe7jpuP 2022-09-03T06:07:34.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4vGkKn+oWGbwIcsKJKt80vKuyPQ= 2022-09-03T06:07:34.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4vGkKn+oWGbwIcsKJKt80vKuyPQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.214 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', '34cbRa0IccRiFCht3CqirmXYwe7jpuP'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007bbce75aef003878-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.215 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.216 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.216 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.217 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.217 DEBUG:boto:Method: GET 2022-09-03T06:07:34.218 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.219 DEBUG:boto:Data: 2022-09-03T06:07:34.219 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.220 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.221 DEBUG:boto:Params: {} 2022-09-03T06:07:34.221 DEBUG:boto:Token: None 2022-09-03T06:07:34.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.226 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000095086756e151c88f-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.270 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.272 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.273 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.274 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.274 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.275 DEBUG:boto:Method: GET 2022-09-03T06:07:34.275 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.276 DEBUG:boto:Data: 2022-09-03T06:07:34.276 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.277 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.277 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.278 DEBUG:boto:Params: {} 2022-09-03T06:07:34.278 DEBUG:boto:Token: None 2022-09-03T06:07:34.280 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.284 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000086bc533ed10b614f-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.284 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.285 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.287 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.288 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.288 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:34.289 DEBUG:boto:Data: 2022-09-03T06:07:34.290 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.290 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.291 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.291 DEBUG:boto:Params: {} 2022-09-03T06:07:34.292 DEBUG:boto:Token: None 2022-09-03T06:07:34.293 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj 2022-09-03T06:07:34.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY= 2022-09-03T06:07:34.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY=', 'Content-Length': '0'} 2022-09-03T06:07:34.299 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f7ac141ea60d7e25-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.301 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.302 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.303 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.303 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:34.304 DEBUG:boto:Data: 2022-09-03T06:07:34.305 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.306 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.306 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.307 DEBUG:boto:Params: {} 2022-09-03T06:07:34.307 DEBUG:boto:Token: None 2022-09-03T06:07:34.308 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj 2022-09-03T06:07:34.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY= 2022-09-03T06:07:34.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY=', 'Content-Length': '0'} 2022-09-03T06:07:34.312 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001f10730a65a0020c-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.313 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.314 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.314 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.315 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.315 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.316 DEBUG:boto:Method: GET 2022-09-03T06:07:34.317 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.318 DEBUG:boto:Data: 2022-09-03T06:07:34.319 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.320 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.320 DEBUG:boto:Params: {} 2022-09-03T06:07:34.321 DEBUG:boto:Token: None 2022-09-03T06:07:34.321 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8= 2022-09-03T06:07:34.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8=', 'Content-Length': '0'} 2022-09-03T06:07:34.325 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002ce6a1347f292807-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.325 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.326 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.326 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.327 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.327 DEBUG:boto:Method: GET 2022-09-03T06:07:34.328 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.329 DEBUG:boto:Data: 2022-09-03T06:07:34.329 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.330 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.330 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.331 DEBUG:boto:Params: {} 2022-09-03T06:07:34.331 DEBUG:boto:Token: None 2022-09-03T06:07:34.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.332 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8= 2022-09-03T06:07:34.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8=', 'Content-Length': '0'} 2022-09-03T06:07:34.335 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000262ef27b0fa7b304-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.336 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.337 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.337 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.338 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.339 DEBUG:boto:Method: GET 2022-09-03T06:07:34.339 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.340 DEBUG:boto:Data: 2022-09-03T06:07:34.340 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.341 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.341 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.342 DEBUG:boto:Params: {} 2022-09-03T06:07:34.342 DEBUG:boto:Token: None 2022-09-03T06:07:34.343 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.347 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a39ee9799d6ca6e-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.348 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.349 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.349 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.350 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.350 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.351 DEBUG:boto:Method: GET 2022-09-03T06:07:34.351 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.352 DEBUG:boto:Data: 2022-09-03T06:07:34.353 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.353 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.354 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.354 DEBUG:boto:Params: {} 2022-09-03T06:07:34.355 DEBUG:boto:Token: None 2022-09-03T06:07:34.356 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.357 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.362 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fe7325ff79be917d-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.363 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.364 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.365 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.366 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.366 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.367 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.367 DEBUG:boto:Method: GET 2022-09-03T06:07:34.368 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.368 DEBUG:boto:Data: 2022-09-03T06:07:34.370 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.371 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.371 DEBUG:boto:Params: {} 2022-09-03T06:07:34.372 DEBUG:boto:Token: None 2022-09-03T06:07:34.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dMnBa/ri54yPO6RhxrwsOEl+rQ= 2022-09-03T06:07:34.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dMnBa/ri54yPO6RhxrwsOEl+rQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.379 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000df36ad4d007ff4b4-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.422 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.423 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.424 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.425 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.426 DEBUG:boto:Method: GET 2022-09-03T06:07:34.427 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.428 DEBUG:boto:Data: 2022-09-03T06:07:34.429 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.430 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.431 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.432 DEBUG:boto:Params: {} 2022-09-03T06:07:34.433 DEBUG:boto:Token: None 2022-09-03T06:07:34.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dMnBa/ri54yPO6RhxrwsOEl+rQ= 2022-09-03T06:07:34.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dMnBa/ri54yPO6RhxrwsOEl+rQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.437 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009d2df97ff5f90bc4-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.438 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.438 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.439 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.439 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.440 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.440 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.441 DEBUG:boto:Data: 2022-09-03T06:07:34.441 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.443 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.443 DEBUG:boto:Params: {} 2022-09-03T06:07:34.444 DEBUG:boto:Token: None 2022-09-03T06:07:34.445 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dNZez/82O44VAMczECHMCrrvVd0= 2022-09-03T06:07:34.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dNZez/82O44VAMczECHMCrrvVd0=', 'Content-Length': '0'} 2022-09-03T06:07:34.448 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e8683da185af35b5-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.449 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.449 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.450 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.450 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.451 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.451 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.452 DEBUG:boto:Data: 2022-09-03T06:07:34.452 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.453 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.453 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.454 DEBUG:boto:Params: {} 2022-09-03T06:07:34.455 DEBUG:boto:Token: None 2022-09-03T06:07:34.455 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.456 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dNZez/82O44VAMczECHMCrrvVd0= 2022-09-03T06:07:34.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dNZez/82O44VAMczECHMCrrvVd0=', 'Content-Length': '0'} 2022-09-03T06:07:34.458 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000df3516dcb00f2853-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.459 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.460 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.461 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.461 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.462 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.463 DEBUG:boto:Method: GET 2022-09-03T06:07:34.463 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.464 DEBUG:boto:Data: 2022-09-03T06:07:34.466 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.468 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.469 DEBUG:boto:Params: {} 2022-09-03T06:07:34.470 DEBUG:boto:Token: None 2022-09-03T06:07:34.471 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dMnBa/ri54yPO6RhxrwsOEl+rQ= 2022-09-03T06:07:34.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dMnBa/ri54yPO6RhxrwsOEl+rQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.476 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001be620a353397807-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.477 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.477 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.478 DEBUG:boto:path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.478 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.479 DEBUG:boto:Method: GET 2022-09-03T06:07:34.480 DEBUG:boto:Path: /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.480 DEBUG:boto:Data: 2022-09-03T06:07:34.481 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.481 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.482 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.482 DEBUG:boto:Params: {} 2022-09-03T06:07:34.483 DEBUG:boto:Token: None 2022-09-03T06:07:34.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv 2022-09-03T06:07:34.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dMnBa/ri54yPO6RhxrwsOEl+rQ= 2022-09-03T06:07:34.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dMnBa/ri54yPO6RhxrwsOEl+rQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.487 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'dMMNFGcpGTBxFBR4ulYemiNEgXkVEfv'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009d7193b3216bf96d-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.530 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.531 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.531 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.532 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.533 DEBUG:boto:Method: GET 2022-09-03T06:07:34.533 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.534 DEBUG:boto:Data: 2022-09-03T06:07:34.534 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.535 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.536 DEBUG:boto:Params: {} 2022-09-03T06:07:34.537 DEBUG:boto:Token: None 2022-09-03T06:07:34.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.542 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002cd687223d2a271d-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.542 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.543 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.544 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.544 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.545 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.546 DEBUG:boto:Method: GET 2022-09-03T06:07:34.546 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.547 DEBUG:boto:Data: 2022-09-03T06:07:34.547 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.548 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.548 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.549 DEBUG:boto:Params: {} 2022-09-03T06:07:34.550 DEBUG:boto:Token: None 2022-09-03T06:07:34.550 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.555 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000033c5e9e5bde706e3-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.555 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.556 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.557 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.557 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.558 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:34.558 DEBUG:boto:Data: 2022-09-03T06:07:34.559 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.560 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.561 DEBUG:boto:Params: {} 2022-09-03T06:07:34.561 DEBUG:boto:Token: None 2022-09-03T06:07:34.562 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj 2022-09-03T06:07:34.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY= 2022-09-03T06:07:34.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY=', 'Content-Length': '0'} 2022-09-03T06:07:34.567 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000068cdb3b78e6e8d49-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.567 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.568 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.568 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.569 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:34.569 DEBUG:boto:Data: 2022-09-03T06:07:34.570 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.571 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.571 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.572 DEBUG:boto:Params: {} 2022-09-03T06:07:34.572 DEBUG:boto:Token: None 2022-09-03T06:07:34.573 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj 2022-09-03T06:07:34.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY= 2022-09-03T06:07:34.574 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY=', 'Content-Length': '0'} 2022-09-03T06:07:34.577 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e5a2ad5f1fc17824-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.578 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.579 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.579 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.580 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.580 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.581 DEBUG:boto:Method: GET 2022-09-03T06:07:34.582 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.582 DEBUG:boto:Data: 2022-09-03T06:07:34.583 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.584 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.584 DEBUG:boto:Params: {} 2022-09-03T06:07:34.585 DEBUG:boto:Token: None 2022-09-03T06:07:34.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8= 2022-09-03T06:07:34.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8=', 'Content-Length': '0'} 2022-09-03T06:07:34.589 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000063ffd663d2aa9213-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.590 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.590 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.591 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.592 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.593 DEBUG:boto:Method: GET 2022-09-03T06:07:34.594 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.594 DEBUG:boto:Data: 2022-09-03T06:07:34.595 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.595 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.596 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.597 DEBUG:boto:Params: {} 2022-09-03T06:07:34.597 DEBUG:boto:Token: None 2022-09-03T06:07:34.598 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8= 2022-09-03T06:07:34.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8=', 'Content-Length': '0'} 2022-09-03T06:07:34.602 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000b3283a415f7468a-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.603 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.604 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.605 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.606 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.607 DEBUG:boto:Method: GET 2022-09-03T06:07:34.608 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.609 DEBUG:boto:Data: 2022-09-03T06:07:34.610 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.610 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.611 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.612 DEBUG:boto:Params: {} 2022-09-03T06:07:34.612 DEBUG:boto:Token: None 2022-09-03T06:07:34.613 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.618 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2ee96c2102042ed-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.618 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.619 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.620 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.620 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.621 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.622 DEBUG:boto:Method: GET 2022-09-03T06:07:34.623 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.624 DEBUG:boto:Data: 2022-09-03T06:07:34.625 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.625 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.626 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.627 DEBUG:boto:Params: {} 2022-09-03T06:07:34.628 DEBUG:boto:Token: None 2022-09-03T06:07:34.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.634 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007fed7fe1831968cb-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.634 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.635 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.636 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.636 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.637 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.637 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.638 DEBUG:boto:Method: GET 2022-09-03T06:07:34.638 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.639 DEBUG:boto:Data: 2022-09-03T06:07:34.639 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.640 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.641 DEBUG:boto:Params: {} 2022-09-03T06:07:34.642 DEBUG:boto:Token: None 2022-09-03T06:07:34.642 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4z0vBpAghX09gzEEyMeM5VBGIRI= 2022-09-03T06:07:34.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4z0vBpAghX09gzEEyMeM5VBGIRI=', 'Content-Length': '0'} 2022-09-03T06:07:34.646 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000011770f54f9434122-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.648 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.649 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.649 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.650 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.651 DEBUG:boto:Method: GET 2022-09-03T06:07:34.652 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.653 DEBUG:boto:Data: 2022-09-03T06:07:34.653 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.654 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.655 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.655 DEBUG:boto:Params: {} 2022-09-03T06:07:34.656 DEBUG:boto:Token: None 2022-09-03T06:07:34.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4z0vBpAghX09gzEEyMeM5VBGIRI= 2022-09-03T06:07:34.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4z0vBpAghX09gzEEyMeM5VBGIRI=', 'Content-Length': '0'} 2022-09-03T06:07:34.660 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000659ede1e2b973f26-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.661 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.661 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.662 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.662 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.663 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.663 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.664 DEBUG:boto:Data: 2022-09-03T06:07:34.664 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.665 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.665 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.666 DEBUG:boto:Params: {} 2022-09-03T06:07:34.666 DEBUG:boto:Token: None 2022-09-03T06:07:34.667 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.668 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gqya5vy/7EX3Ck7YaVP3H8s+3kg= 2022-09-03T06:07:34.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gqya5vy/7EX3Ck7YaVP3H8s+3kg=', 'Content-Length': '0'} 2022-09-03T06:07:34.671 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000657ac4d1a835c964-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.672 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.673 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.674 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.675 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.675 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.676 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.677 DEBUG:boto:Data: 2022-09-03T06:07:34.677 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.678 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.679 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.679 DEBUG:boto:Params: {} 2022-09-03T06:07:34.680 DEBUG:boto:Token: None 2022-09-03T06:07:34.680 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gqya5vy/7EX3Ck7YaVP3H8s+3kg= 2022-09-03T06:07:34.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gqya5vy/7EX3Ck7YaVP3H8s+3kg=', 'Content-Length': '0'} 2022-09-03T06:07:34.684 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007eb76262f85ce3d6-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.685 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.685 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.686 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.686 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.687 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.687 DEBUG:boto:Method: GET 2022-09-03T06:07:34.688 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.688 DEBUG:boto:Data: 2022-09-03T06:07:34.689 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.690 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.690 DEBUG:boto:Params: {} 2022-09-03T06:07:34.691 DEBUG:boto:Token: None 2022-09-03T06:07:34.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4z0vBpAghX09gzEEyMeM5VBGIRI= 2022-09-03T06:07:34.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4z0vBpAghX09gzEEyMeM5VBGIRI=', 'Content-Length': '0'} 2022-09-03T06:07:34.695 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005320d1dbd2e19975-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.738 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.739 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.739 DEBUG:boto:path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.740 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.740 DEBUG:boto:Method: GET 2022-09-03T06:07:34.741 DEBUG:boto:Path: /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.742 DEBUG:boto:Data: 2022-09-03T06:07:34.742 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.743 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.743 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.744 DEBUG:boto:Params: {} 2022-09-03T06:07:34.744 DEBUG:boto:Token: None 2022-09-03T06:07:34.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=uwwXAQ.l7uosanR0We1fU5cnpeCTvF3 2022-09-03T06:07:34.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4z0vBpAghX09gzEEyMeM5VBGIRI= 2022-09-03T06:07:34.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4z0vBpAghX09gzEEyMeM5VBGIRI=', 'Content-Length': '0'} 2022-09-03T06:07:34.749 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'uwwXAQ.l7uosanR0We1fU5cnpeCTvF3'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000074d83fa9ef18253c-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.750 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.751 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.752 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.752 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.753 DEBUG:boto:Method: GET 2022-09-03T06:07:34.754 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.755 DEBUG:boto:Data: 2022-09-03T06:07:34.756 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.758 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.758 DEBUG:boto:Params: {} 2022-09-03T06:07:34.759 DEBUG:boto:Token: None 2022-09-03T06:07:34.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.765 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e012b29256cc0c2d-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.766 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.767 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.767 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.768 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.768 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.769 DEBUG:boto:Method: GET 2022-09-03T06:07:34.769 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.770 DEBUG:boto:Data: 2022-09-03T06:07:34.770 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.771 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.771 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.772 DEBUG:boto:Params: {} 2022-09-03T06:07:34.773 DEBUG:boto:Token: None 2022-09-03T06:07:34.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.780 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5d12b9757c88c1a-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.781 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.782 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.783 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.783 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.784 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:34.794 DEBUG:boto:Data: 2022-09-03T06:07:34.794 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.795 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.796 DEBUG:boto:Params: {} 2022-09-03T06:07:34.796 DEBUG:boto:Token: None 2022-09-03T06:07:34.797 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj 2022-09-03T06:07:34.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY= 2022-09-03T06:07:34.798 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY=', 'Content-Length': '0'} 2022-09-03T06:07:34.802 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000047d3d99aab98466e-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.803 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.803 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.804 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.804 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:34.805 DEBUG:boto:Data: 2022-09-03T06:07:34.805 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.806 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.806 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.807 DEBUG:boto:Params: {} 2022-09-03T06:07:34.808 DEBUG:boto:Token: None 2022-09-03T06:07:34.808 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj 2022-09-03T06:07:34.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY= 2022-09-03T06:07:34.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k++G0oZIVQLK8TT5xkcXxYyC6cY=', 'Content-Length': '0'} 2022-09-03T06:07:34.813 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007601d2da0b6e424d-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.814 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.815 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.815 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.816 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.816 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.817 DEBUG:boto:Method: GET 2022-09-03T06:07:34.817 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.818 DEBUG:boto:Data: 2022-09-03T06:07:34.819 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.820 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.820 DEBUG:boto:Params: {} 2022-09-03T06:07:34.821 DEBUG:boto:Token: None 2022-09-03T06:07:34.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8= 2022-09-03T06:07:34.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8=', 'Content-Length': '0'} 2022-09-03T06:07:34.825 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000df3c7b63faa34c60-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.827 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.828 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.828 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.829 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.830 DEBUG:boto:Method: GET 2022-09-03T06:07:34.831 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.831 DEBUG:boto:Data: 2022-09-03T06:07:34.832 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.833 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.833 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.834 DEBUG:boto:Params: {} 2022-09-03T06:07:34.835 DEBUG:boto:Token: None 2022-09-03T06:07:34.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:34.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8= 2022-09-03T06:07:34.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gB+y8n+4ThZR07JL0o3HWjgTly8=', 'Content-Length': '0'} 2022-09-03T06:07:34.839 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000456302d239119478-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.882 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.883 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.883 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.884 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.884 DEBUG:boto:Method: GET 2022-09-03T06:07:34.885 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.885 DEBUG:boto:Data: 2022-09-03T06:07:34.886 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.887 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.887 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.888 DEBUG:boto:Params: {} 2022-09-03T06:07:34.888 DEBUG:boto:Token: None 2022-09-03T06:07:34.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.893 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044dce82d72abb2e2-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.894 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.895 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:34.896 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:34.896 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.897 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:34.897 DEBUG:boto:Method: GET 2022-09-03T06:07:34.898 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:34.898 DEBUG:boto:Data: 2022-09-03T06:07:34.899 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.900 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.900 DEBUG:boto:Params: {} 2022-09-03T06:07:34.901 DEBUG:boto:Token: None 2022-09-03T06:07:34.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/ 2022-09-03T06:07:34.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4= 2022-09-03T06:07:34.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jviw17oLQQFcWe9n6uaYsn3HMF4=', 'Content-Length': '0'} 2022-09-03T06:07:34.906 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009c93ae8555012f3d-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.907 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:34.907 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.908 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.909 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.909 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.910 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.910 DEBUG:boto:Method: GET 2022-09-03T06:07:34.911 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.911 DEBUG:boto:Data: 2022-09-03T06:07:34.912 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.913 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.913 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.914 DEBUG:boto:Params: {} 2022-09-03T06:07:34.914 DEBUG:boto:Token: None 2022-09-03T06:07:34.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h5Ohb0ns859aWeVK+ai0STe4Hdk= 2022-09-03T06:07:34.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h5Ohb0ns859aWeVK+ai0STe4Hdk=', 'Content-Length': '0'} 2022-09-03T06:07:34.919 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006ce9bc8397098078-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.919 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.920 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.921 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.921 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.922 DEBUG:boto:Method: GET 2022-09-03T06:07:34.922 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.923 DEBUG:boto:Data: 2022-09-03T06:07:34.924 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.924 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.925 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.925 DEBUG:boto:Params: {} 2022-09-03T06:07:34.926 DEBUG:boto:Token: None 2022-09-03T06:07:34.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h5Ohb0ns859aWeVK+ai0STe4Hdk= 2022-09-03T06:07:34.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h5Ohb0ns859aWeVK+ai0STe4Hdk=', 'Content-Length': '0'} 2022-09-03T06:07:34.932 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d9c773b483e03f8c-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.933 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.934 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.935 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.936 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.937 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.937 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.938 DEBUG:boto:Data: 2022-09-03T06:07:34.938 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.939 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.940 DEBUG:boto:Params: {} 2022-09-03T06:07:34.941 DEBUG:boto:Token: None 2022-09-03T06:07:34.941 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DEuSvpP8QHOVUc8GdawwVj77+vQ= 2022-09-03T06:07:34.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DEuSvpP8QHOVUc8GdawwVj77+vQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.945 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d0d3a4f9aa9cc34f-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.945 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.946 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.946 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.947 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.948 DEBUG:boto:Method: HEAD 2022-09-03T06:07:34.948 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.949 DEBUG:boto:Data: 2022-09-03T06:07:34.950 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.951 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.952 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.952 DEBUG:boto:Params: {} 2022-09-03T06:07:34.953 DEBUG:boto:Token: None 2022-09-03T06:07:34.955 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DEuSvpP8QHOVUc8GdawwVj77+vQ= 2022-09-03T06:07:34.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DEuSvpP8QHOVUc8GdawwVj77+vQ=', 'Content-Length': '0'} 2022-09-03T06:07:34.959 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004786ebfe37c48162-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.959 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:34.960 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.960 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.961 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.961 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.962 DEBUG:boto:Method: GET 2022-09-03T06:07:34.963 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.964 DEBUG:boto:Data: 2022-09-03T06:07:34.964 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:34.965 DEBUG:boto:Port: 8001 2022-09-03T06:07:34.967 DEBUG:boto:Params: {} 2022-09-03T06:07:34.967 DEBUG:boto:Token: None 2022-09-03T06:07:34.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h5Ohb0ns859aWeVK+ai0STe4Hdk= 2022-09-03T06:07:34.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h5Ohb0ns859aWeVK+ai0STe4Hdk=', 'Content-Length': '0'} 2022-09-03T06:07:34.972 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001656d09161d3171f-006312ef26-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:34.973 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:34.973 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:34.974 DEBUG:boto:path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.974 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.975 DEBUG:boto:Method: GET 2022-09-03T06:07:34.976 DEBUG:boto:Path: /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.976 DEBUG:boto:Data: 2022-09-03T06:07:34.977 DEBUG:boto:Headers: {} 2022-09-03T06:07:34.978 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:34.978 DEBUG:boto:Port: 8000 2022-09-03T06:07:34.979 DEBUG:boto:Params: {} 2022-09-03T06:07:34.980 DEBUG:boto:Token: None 2022-09-03T06:07:34.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:34 GMT /avysmw-19/obj?versionId=85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC 2022-09-03T06:07:34.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h5Ohb0ns859aWeVK+ai0STe4Hdk= 2022-09-03T06:07:34.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h5Ohb0ns859aWeVK+ai0STe4Hdk=', 'Content-Length': '0'} 2022-09-03T06:07:34.984 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '85G0hkf-Eia8CZrFNN.-6SaWe1ilZNC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000180d966ee7e8a368-006312ef26-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.026 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.027 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.027 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.028 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.028 DEBUG:boto:Method: GET 2022-09-03T06:07:35.029 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.030 DEBUG:boto:Data: 2022-09-03T06:07:35.030 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.032 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.034 DEBUG:boto:Params: {} 2022-09-03T06:07:35.035 DEBUG:boto:Token: None 2022-09-03T06:07:35.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.040 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d863b2b59150f11-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.041 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.043 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.043 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.044 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.044 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.045 DEBUG:boto:Method: GET 2022-09-03T06:07:35.046 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.046 DEBUG:boto:Data: 2022-09-03T06:07:35.047 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.048 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.048 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.049 DEBUG:boto:Params: {} 2022-09-03T06:07:35.049 DEBUG:boto:Token: None 2022-09-03T06:07:35.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.055 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002360a62e2efa3ae1-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.055 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.056 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.057 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.057 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.058 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:35.058 DEBUG:boto:Data: 2022-09-03T06:07:35.059 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.060 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.060 DEBUG:boto:Params: {} 2022-09-03T06:07:35.061 DEBUG:boto:Token: None 2022-09-03T06:07:35.061 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj 2022-09-03T06:07:35.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY= 2022-09-03T06:07:35.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY=', 'Content-Length': '0'} 2022-09-03T06:07:35.066 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003e8e776bfc4b8cae-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.067 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.067 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.068 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.068 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:35.069 DEBUG:boto:Data: 2022-09-03T06:07:35.069 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.070 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.070 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.071 DEBUG:boto:Params: {} 2022-09-03T06:07:35.072 DEBUG:boto:Token: None 2022-09-03T06:07:35.072 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj 2022-09-03T06:07:35.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY= 2022-09-03T06:07:35.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY=', 'Content-Length': '0'} 2022-09-03T06:07:35.077 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fa6715eb2207ab8f-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.078 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.079 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.079 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.080 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.080 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.081 DEBUG:boto:Method: GET 2022-09-03T06:07:35.081 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.082 DEBUG:boto:Data: 2022-09-03T06:07:35.082 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.083 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.084 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.084 DEBUG:boto:Params: {} 2022-09-03T06:07:35.085 DEBUG:boto:Token: None 2022-09-03T06:07:35.085 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA= 2022-09-03T06:07:35.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA=', 'Content-Length': '0'} 2022-09-03T06:07:35.089 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002876d509815e1e0c-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.134 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.136 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.136 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.137 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.137 DEBUG:boto:Method: GET 2022-09-03T06:07:35.138 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.138 DEBUG:boto:Data: 2022-09-03T06:07:35.139 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.140 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.140 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.141 DEBUG:boto:Params: {} 2022-09-03T06:07:35.141 DEBUG:boto:Token: None 2022-09-03T06:07:35.142 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA= 2022-09-03T06:07:35.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA=', 'Content-Length': '0'} 2022-09-03T06:07:35.145 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000023f3d3bb29d48d3a-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.146 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.147 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.147 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.148 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.148 DEBUG:boto:Method: GET 2022-09-03T06:07:35.149 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.149 DEBUG:boto:Data: 2022-09-03T06:07:35.150 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.151 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.151 DEBUG:boto:Params: {} 2022-09-03T06:07:35.152 DEBUG:boto:Token: None 2022-09-03T06:07:35.153 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.157 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007ec335ad88e75891-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.157 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.158 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.159 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.159 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.160 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.160 DEBUG:boto:Method: GET 2022-09-03T06:07:35.161 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.161 DEBUG:boto:Data: 2022-09-03T06:07:35.162 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.163 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.163 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.164 DEBUG:boto:Params: {} 2022-09-03T06:07:35.164 DEBUG:boto:Token: None 2022-09-03T06:07:35.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.169 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c6d180caad809fa-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.170 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.170 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.171 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.172 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.172 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.173 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.173 DEBUG:boto:Method: GET 2022-09-03T06:07:35.174 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.174 DEBUG:boto:Data: 2022-09-03T06:07:35.175 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.176 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.176 DEBUG:boto:Params: {} 2022-09-03T06:07:35.177 DEBUG:boto:Token: None 2022-09-03T06:07:35.178 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CQsRKZjM9lK6AnegoJRxMECCIs0= 2022-09-03T06:07:35.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CQsRKZjM9lK6AnegoJRxMECCIs0=', 'Content-Length': '0'} 2022-09-03T06:07:35.181 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003f15b36657141322-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.182 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.183 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.183 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.184 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.184 DEBUG:boto:Method: GET 2022-09-03T06:07:35.185 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.185 DEBUG:boto:Data: 2022-09-03T06:07:35.186 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.187 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.187 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.188 DEBUG:boto:Params: {} 2022-09-03T06:07:35.188 DEBUG:boto:Token: None 2022-09-03T06:07:35.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CQsRKZjM9lK6AnegoJRxMECCIs0= 2022-09-03T06:07:35.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CQsRKZjM9lK6AnegoJRxMECCIs0=', 'Content-Length': '0'} 2022-09-03T06:07:35.192 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009e00352fc7495d7c-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.193 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.194 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.194 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.195 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.195 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.196 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.196 DEBUG:boto:Data: 2022-09-03T06:07:35.197 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.198 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.198 DEBUG:boto:Params: {} 2022-09-03T06:07:35.199 DEBUG:boto:Token: None 2022-09-03T06:07:35.199 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ivoZpkwai1XqcNPplF5Nkn9nW4A= 2022-09-03T06:07:35.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ivoZpkwai1XqcNPplF5Nkn9nW4A=', 'Content-Length': '0'} 2022-09-03T06:07:35.202 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000493e4f100e7c702-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.203 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.204 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.204 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.205 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.205 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.206 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.206 DEBUG:boto:Data: 2022-09-03T06:07:35.207 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.207 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.208 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.208 DEBUG:boto:Params: {} 2022-09-03T06:07:35.209 DEBUG:boto:Token: None 2022-09-03T06:07:35.209 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ivoZpkwai1XqcNPplF5Nkn9nW4A= 2022-09-03T06:07:35.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ivoZpkwai1XqcNPplF5Nkn9nW4A=', 'Content-Length': '0'} 2022-09-03T06:07:35.213 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c69109319da458f9-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.213 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.214 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.214 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.215 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.216 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.216 DEBUG:boto:Method: GET 2022-09-03T06:07:35.217 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.217 DEBUG:boto:Data: 2022-09-03T06:07:35.218 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.219 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.219 DEBUG:boto:Params: {} 2022-09-03T06:07:35.220 DEBUG:boto:Token: None 2022-09-03T06:07:35.220 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CQsRKZjM9lK6AnegoJRxMECCIs0= 2022-09-03T06:07:35.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CQsRKZjM9lK6AnegoJRxMECCIs0=', 'Content-Length': '0'} 2022-09-03T06:07:35.224 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000400884edde7d3d6d-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.225 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.225 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.226 DEBUG:boto:path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.227 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.227 DEBUG:boto:Method: GET 2022-09-03T06:07:35.228 DEBUG:boto:Path: /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.228 DEBUG:boto:Data: 2022-09-03T06:07:35.229 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.229 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.230 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.230 DEBUG:boto:Params: {} 2022-09-03T06:07:35.231 DEBUG:boto:Token: None 2022-09-03T06:07:35.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB 2022-09-03T06:07:35.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CQsRKZjM9lK6AnegoJRxMECCIs0= 2022-09-03T06:07:35.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CQsRKZjM9lK6AnegoJRxMECCIs0=', 'Content-Length': '0'} 2022-09-03T06:07:35.235 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'NcMAfdGd3hoA3JP.E1mP7q4GjNbxtnB'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003582c6a36373cff2-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.235 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.236 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.237 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.237 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.238 DEBUG:boto:Method: GET 2022-09-03T06:07:35.238 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.239 DEBUG:boto:Data: 2022-09-03T06:07:35.239 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.240 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.241 DEBUG:boto:Params: {} 2022-09-03T06:07:35.241 DEBUG:boto:Token: None 2022-09-03T06:07:35.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.246 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cdf514dff28a0c7f-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.247 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.248 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.248 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.249 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.249 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.250 DEBUG:boto:Method: GET 2022-09-03T06:07:35.250 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.251 DEBUG:boto:Data: 2022-09-03T06:07:35.251 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.252 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.253 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.253 DEBUG:boto:Params: {} 2022-09-03T06:07:35.254 DEBUG:boto:Token: None 2022-09-03T06:07:35.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.258 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ae94ec4d2bfa7585-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.306 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.307 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.308 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.308 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.309 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:35.309 DEBUG:boto:Data: 2022-09-03T06:07:35.310 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.310 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.311 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.311 DEBUG:boto:Params: {} 2022-09-03T06:07:35.312 DEBUG:boto:Token: None 2022-09-03T06:07:35.313 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj 2022-09-03T06:07:35.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY= 2022-09-03T06:07:35.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY=', 'Content-Length': '0'} 2022-09-03T06:07:35.318 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000db48cfdda570b21-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.319 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.320 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.320 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.321 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:35.321 DEBUG:boto:Data: 2022-09-03T06:07:35.322 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.322 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.323 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.324 DEBUG:boto:Params: {} 2022-09-03T06:07:35.324 DEBUG:boto:Token: None 2022-09-03T06:07:35.325 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj 2022-09-03T06:07:35.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY= 2022-09-03T06:07:35.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY=', 'Content-Length': '0'} 2022-09-03T06:07:35.330 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000077a02751086b935c-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.331 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.331 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.332 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.332 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.333 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.333 DEBUG:boto:Method: GET 2022-09-03T06:07:35.334 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.334 DEBUG:boto:Data: 2022-09-03T06:07:35.335 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.336 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.337 DEBUG:boto:Params: {} 2022-09-03T06:07:35.337 DEBUG:boto:Token: None 2022-09-03T06:07:35.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA= 2022-09-03T06:07:35.339 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA=', 'Content-Length': '0'} 2022-09-03T06:07:35.342 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000b7ce55feb7a92270-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.390 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.391 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.391 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.392 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.393 DEBUG:boto:Method: GET 2022-09-03T06:07:35.393 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.394 DEBUG:boto:Data: 2022-09-03T06:07:35.394 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.395 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.395 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.396 DEBUG:boto:Params: {} 2022-09-03T06:07:35.396 DEBUG:boto:Token: None 2022-09-03T06:07:35.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA= 2022-09-03T06:07:35.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA=', 'Content-Length': '0'} 2022-09-03T06:07:35.401 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000001627849ca04a4d0-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.402 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.402 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.403 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.404 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.404 DEBUG:boto:Method: GET 2022-09-03T06:07:35.405 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.405 DEBUG:boto:Data: 2022-09-03T06:07:35.406 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.407 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.407 DEBUG:boto:Params: {} 2022-09-03T06:07:35.408 DEBUG:boto:Token: None 2022-09-03T06:07:35.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.413 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000059db949056804609-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.414 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.414 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.415 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.416 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.416 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.417 DEBUG:boto:Method: GET 2022-09-03T06:07:35.417 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.418 DEBUG:boto:Data: 2022-09-03T06:07:35.418 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.419 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.419 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.420 DEBUG:boto:Params: {} 2022-09-03T06:07:35.420 DEBUG:boto:Token: None 2022-09-03T06:07:35.421 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.425 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff7fcc7bc098680e-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.426 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.427 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.427 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.428 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.428 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.429 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.429 DEBUG:boto:Method: GET 2022-09-03T06:07:35.430 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.431 DEBUG:boto:Data: 2022-09-03T06:07:35.431 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.432 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.433 DEBUG:boto:Params: {} 2022-09-03T06:07:35.433 DEBUG:boto:Token: None 2022-09-03T06:07:35.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RQ9U8hlh8PqMmvrpRIJI4JkVApg= 2022-09-03T06:07:35.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RQ9U8hlh8PqMmvrpRIJI4JkVApg=', 'Content-Length': '0'} 2022-09-03T06:07:35.438 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000bf8c3a47205e40d6-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.439 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.439 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.440 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.440 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.441 DEBUG:boto:Method: GET 2022-09-03T06:07:35.442 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.442 DEBUG:boto:Data: 2022-09-03T06:07:35.443 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.443 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.444 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.444 DEBUG:boto:Params: {} 2022-09-03T06:07:35.445 DEBUG:boto:Token: None 2022-09-03T06:07:35.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RQ9U8hlh8PqMmvrpRIJI4JkVApg= 2022-09-03T06:07:35.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RQ9U8hlh8PqMmvrpRIJI4JkVApg=', 'Content-Length': '0'} 2022-09-03T06:07:35.449 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000074fe3464664c52cc-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.450 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.451 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.451 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.452 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.452 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.453 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.453 DEBUG:boto:Data: 2022-09-03T06:07:35.454 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.454 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.455 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.455 DEBUG:boto:Params: {} 2022-09-03T06:07:35.456 DEBUG:boto:Token: None 2022-09-03T06:07:35.456 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FlZAYSPv38Sg72hkE1DkMJ5+ej4= 2022-09-03T06:07:35.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FlZAYSPv38Sg72hkE1DkMJ5+ej4=', 'Content-Length': '0'} 2022-09-03T06:07:35.460 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000862f246a69455b77-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.461 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.462 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.462 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.463 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.463 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.464 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.464 DEBUG:boto:Data: 2022-09-03T06:07:35.465 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.466 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.466 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.467 DEBUG:boto:Params: {} 2022-09-03T06:07:35.467 DEBUG:boto:Token: None 2022-09-03T06:07:35.468 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FlZAYSPv38Sg72hkE1DkMJ5+ej4= 2022-09-03T06:07:35.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FlZAYSPv38Sg72hkE1DkMJ5+ej4=', 'Content-Length': '0'} 2022-09-03T06:07:35.472 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a4e31da8e1d68286-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.472 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.473 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.473 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.474 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.475 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.475 DEBUG:boto:Method: GET 2022-09-03T06:07:35.476 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.476 DEBUG:boto:Data: 2022-09-03T06:07:35.477 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.477 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.478 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.478 DEBUG:boto:Params: {} 2022-09-03T06:07:35.479 DEBUG:boto:Token: None 2022-09-03T06:07:35.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RQ9U8hlh8PqMmvrpRIJI4JkVApg= 2022-09-03T06:07:35.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RQ9U8hlh8PqMmvrpRIJI4JkVApg=', 'Content-Length': '0'} 2022-09-03T06:07:35.483 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f7366e35d0254d52-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.484 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.485 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.485 DEBUG:boto:path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.486 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.486 DEBUG:boto:Method: GET 2022-09-03T06:07:35.487 DEBUG:boto:Path: /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.488 DEBUG:boto:Data: 2022-09-03T06:07:35.488 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.489 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.489 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.490 DEBUG:boto:Params: {} 2022-09-03T06:07:35.490 DEBUG:boto:Token: None 2022-09-03T06:07:35.491 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo 2022-09-03T06:07:35.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RQ9U8hlh8PqMmvrpRIJI4JkVApg= 2022-09-03T06:07:35.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RQ9U8hlh8PqMmvrpRIJI4JkVApg=', 'Content-Length': '0'} 2022-09-03T06:07:35.495 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'j45Ej.s5H7i.d8hnuJ-hZzdMK-w2sZo'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000061e7f331fe808f1f-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.496 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.497 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.498 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.498 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.499 DEBUG:boto:Method: GET 2022-09-03T06:07:35.500 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.500 DEBUG:boto:Data: 2022-09-03T06:07:35.501 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.501 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.502 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.502 DEBUG:boto:Params: {} 2022-09-03T06:07:35.503 DEBUG:boto:Token: None 2022-09-03T06:07:35.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.508 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d1eef561f6a7ff7-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.508 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.509 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.510 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.510 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.511 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.511 DEBUG:boto:Method: GET 2022-09-03T06:07:35.512 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.512 DEBUG:boto:Data: 2022-09-03T06:07:35.513 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.513 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.514 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.514 DEBUG:boto:Params: {} 2022-09-03T06:07:35.515 DEBUG:boto:Token: None 2022-09-03T06:07:35.516 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.520 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a637612a62d4e37-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.520 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.521 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.522 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.522 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.523 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:35.523 DEBUG:boto:Data: 2022-09-03T06:07:35.524 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.525 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.525 DEBUG:boto:Params: {} 2022-09-03T06:07:35.526 DEBUG:boto:Token: None 2022-09-03T06:07:35.526 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj 2022-09-03T06:07:35.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY= 2022-09-03T06:07:35.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY=', 'Content-Length': '0'} 2022-09-03T06:07:35.531 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000207b8e781d9f3b4d-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.532 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.532 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.533 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.533 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:35.534 DEBUG:boto:Data: 2022-09-03T06:07:35.534 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.535 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.535 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.536 DEBUG:boto:Params: {} 2022-09-03T06:07:35.536 DEBUG:boto:Token: None 2022-09-03T06:07:35.537 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj 2022-09-03T06:07:35.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY= 2022-09-03T06:07:35.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY=', 'Content-Length': '0'} 2022-09-03T06:07:35.542 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006c903d2930ff6073-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.542 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.543 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.544 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.544 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.545 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.545 DEBUG:boto:Method: GET 2022-09-03T06:07:35.546 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.546 DEBUG:boto:Data: 2022-09-03T06:07:35.547 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.548 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.548 DEBUG:boto:Params: {} 2022-09-03T06:07:35.549 DEBUG:boto:Token: None 2022-09-03T06:07:35.550 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA= 2022-09-03T06:07:35.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA=', 'Content-Length': '0'} 2022-09-03T06:07:35.553 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d1c8876ec1485ac2-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.594 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.595 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.595 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.596 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.596 DEBUG:boto:Method: GET 2022-09-03T06:07:35.597 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.597 DEBUG:boto:Data: 2022-09-03T06:07:35.598 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.599 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.599 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.600 DEBUG:boto:Params: {} 2022-09-03T06:07:35.601 DEBUG:boto:Token: None 2022-09-03T06:07:35.602 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA= 2022-09-03T06:07:35.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA=', 'Content-Length': '0'} 2022-09-03T06:07:35.605 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000a21bc2e790c580e3-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.606 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.607 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.607 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.608 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.609 DEBUG:boto:Method: GET 2022-09-03T06:07:35.610 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.610 DEBUG:boto:Data: 2022-09-03T06:07:35.611 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.612 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.613 DEBUG:boto:Params: {} 2022-09-03T06:07:35.614 DEBUG:boto:Token: None 2022-09-03T06:07:35.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.618 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c33a447cbfe6da3a-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.619 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.621 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.622 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.623 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.623 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.624 DEBUG:boto:Method: GET 2022-09-03T06:07:35.625 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.626 DEBUG:boto:Data: 2022-09-03T06:07:35.627 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.628 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.628 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.629 DEBUG:boto:Params: {} 2022-09-03T06:07:35.629 DEBUG:boto:Token: None 2022-09-03T06:07:35.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.635 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ddac0ea01f3bcfc3-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.635 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.636 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.637 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.637 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.638 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.639 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.639 DEBUG:boto:Method: GET 2022-09-03T06:07:35.640 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.640 DEBUG:boto:Data: 2022-09-03T06:07:35.641 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.642 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.643 DEBUG:boto:Params: {} 2022-09-03T06:07:35.644 DEBUG:boto:Token: None 2022-09-03T06:07:35.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xYyXTQp/Hg3GFU/01vUoNV0/+eo= 2022-09-03T06:07:35.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xYyXTQp/Hg3GFU/01vUoNV0/+eo=', 'Content-Length': '0'} 2022-09-03T06:07:35.649 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000007268d5d737bcd464-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.650 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.651 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.651 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.652 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.653 DEBUG:boto:Method: GET 2022-09-03T06:07:35.654 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.655 DEBUG:boto:Data: 2022-09-03T06:07:35.656 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.657 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.658 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.660 DEBUG:boto:Params: {} 2022-09-03T06:07:35.661 DEBUG:boto:Token: None 2022-09-03T06:07:35.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xYyXTQp/Hg3GFU/01vUoNV0/+eo= 2022-09-03T06:07:35.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xYyXTQp/Hg3GFU/01vUoNV0/+eo=', 'Content-Length': '0'} 2022-09-03T06:07:35.668 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000684ddf02ffa2b319-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.669 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.670 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.670 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.671 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.672 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.672 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.673 DEBUG:boto:Data: 2022-09-03T06:07:35.674 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.675 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.676 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.677 DEBUG:boto:Params: {} 2022-09-03T06:07:35.679 DEBUG:boto:Token: None 2022-09-03T06:07:35.680 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+W7kG66PcaylI62z8BcdbcvBWx4= 2022-09-03T06:07:35.683 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+W7kG66PcaylI62z8BcdbcvBWx4=', 'Content-Length': '0'} 2022-09-03T06:07:35.685 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000153166d1e7e58617-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.687 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.688 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.689 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.689 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.690 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.690 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.691 DEBUG:boto:Data: 2022-09-03T06:07:35.692 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.692 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.693 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.694 DEBUG:boto:Params: {} 2022-09-03T06:07:35.695 DEBUG:boto:Token: None 2022-09-03T06:07:35.696 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+W7kG66PcaylI62z8BcdbcvBWx4= 2022-09-03T06:07:35.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+W7kG66PcaylI62z8BcdbcvBWx4=', 'Content-Length': '0'} 2022-09-03T06:07:35.702 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002d39d27c296cfd87-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.703 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.704 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.705 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.706 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.707 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.708 DEBUG:boto:Method: GET 2022-09-03T06:07:35.709 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.710 DEBUG:boto:Data: 2022-09-03T06:07:35.711 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.712 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.713 DEBUG:boto:Params: {} 2022-09-03T06:07:35.714 DEBUG:boto:Token: None 2022-09-03T06:07:35.715 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xYyXTQp/Hg3GFU/01vUoNV0/+eo= 2022-09-03T06:07:35.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xYyXTQp/Hg3GFU/01vUoNV0/+eo=', 'Content-Length': '0'} 2022-09-03T06:07:35.719 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000fa35e399e70facf-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.720 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.721 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.722 DEBUG:boto:path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.723 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.723 DEBUG:boto:Method: GET 2022-09-03T06:07:35.724 DEBUG:boto:Path: /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.725 DEBUG:boto:Data: 2022-09-03T06:07:35.726 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.726 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.727 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.728 DEBUG:boto:Params: {} 2022-09-03T06:07:35.729 DEBUG:boto:Token: None 2022-09-03T06:07:35.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=--U94MsxK6J7xIZega8EK5RG5RoC35. 2022-09-03T06:07:35.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xYyXTQp/Hg3GFU/01vUoNV0/+eo= 2022-09-03T06:07:35.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xYyXTQp/Hg3GFU/01vUoNV0/+eo=', 'Content-Length': '0'} 2022-09-03T06:07:35.734 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '--U94MsxK6J7xIZega8EK5RG5RoC35.'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c3b64d685c51f14c-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.735 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.736 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.736 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.737 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.737 DEBUG:boto:Method: GET 2022-09-03T06:07:35.738 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.739 DEBUG:boto:Data: 2022-09-03T06:07:35.740 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.740 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.741 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.742 DEBUG:boto:Params: {} 2022-09-03T06:07:35.743 DEBUG:boto:Token: None 2022-09-03T06:07:35.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.749 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000322599afbc84339c-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.750 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.751 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.752 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.752 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.753 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.754 DEBUG:boto:Method: GET 2022-09-03T06:07:35.755 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.755 DEBUG:boto:Data: 2022-09-03T06:07:35.756 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.757 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.757 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.758 DEBUG:boto:Params: {} 2022-09-03T06:07:35.759 DEBUG:boto:Token: None 2022-09-03T06:07:35.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.764 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a820f9c77975ea4-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.765 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.766 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.767 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.767 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.768 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:35.769 DEBUG:boto:Data: 2022-09-03T06:07:35.770 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.771 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.772 DEBUG:boto:Params: {} 2022-09-03T06:07:35.773 DEBUG:boto:Token: None 2022-09-03T06:07:35.774 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj 2022-09-03T06:07:35.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY= 2022-09-03T06:07:35.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY=', 'Content-Length': '0'} 2022-09-03T06:07:35.780 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f0e09c9e91162a87-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.781 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.782 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.783 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.784 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:35.785 DEBUG:boto:Data: 2022-09-03T06:07:35.786 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.787 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.788 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.789 DEBUG:boto:Params: {} 2022-09-03T06:07:35.790 DEBUG:boto:Token: None 2022-09-03T06:07:35.791 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj 2022-09-03T06:07:35.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY= 2022-09-03T06:07:35.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jzSLy92+B/JzURa9g1L2aiH7cWY=', 'Content-Length': '0'} 2022-09-03T06:07:35.798 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000000de23c7cc8598160-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.799 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.800 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.801 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.802 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.802 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.803 DEBUG:boto:Method: GET 2022-09-03T06:07:35.804 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.804 DEBUG:boto:Data: 2022-09-03T06:07:35.805 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.806 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.806 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.807 DEBUG:boto:Params: {} 2022-09-03T06:07:35.808 DEBUG:boto:Token: None 2022-09-03T06:07:35.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA= 2022-09-03T06:07:35.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA=', 'Content-Length': '0'} 2022-09-03T06:07:35.812 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000026bd794aa7d6da11-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.858 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.859 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.860 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.861 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.862 DEBUG:boto:Method: GET 2022-09-03T06:07:35.863 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.863 DEBUG:boto:Data: 2022-09-03T06:07:35.864 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.864 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.865 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.865 DEBUG:boto:Params: {} 2022-09-03T06:07:35.866 DEBUG:boto:Token: None 2022-09-03T06:07:35.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:35.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA= 2022-09-03T06:07:35.868 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0c0u3bSSDmzk7Uo3uSMLLr7UGvA=', 'Content-Length': '0'} 2022-09-03T06:07:35.870 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c3871cf928e52406-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.871 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.871 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.872 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.872 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.873 DEBUG:boto:Method: GET 2022-09-03T06:07:35.874 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.874 DEBUG:boto:Data: 2022-09-03T06:07:35.875 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.876 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.876 DEBUG:boto:Params: {} 2022-09-03T06:07:35.877 DEBUG:boto:Token: None 2022-09-03T06:07:35.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.883 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000013e2feafe7f204ec-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.884 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.885 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:35.886 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:35.886 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.887 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:35.887 DEBUG:boto:Method: GET 2022-09-03T06:07:35.888 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:35.888 DEBUG:boto:Data: 2022-09-03T06:07:35.889 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.890 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.891 DEBUG:boto:Params: {} 2022-09-03T06:07:35.891 DEBUG:boto:Token: None 2022-09-03T06:07:35.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/ 2022-09-03T06:07:35.892 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg= 2022-09-03T06:07:35.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+xZ+6JSJRJMyqNaewDk/gRg91Zg=', 'Content-Length': '0'} 2022-09-03T06:07:35.896 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000771ec6fbaeba3b46-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.942 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:35.944 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:35.945 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.946 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.947 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.948 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.948 DEBUG:boto:Method: GET 2022-09-03T06:07:35.949 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.950 DEBUG:boto:Data: 2022-09-03T06:07:35.951 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.952 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.953 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.954 DEBUG:boto:Params: {} 2022-09-03T06:07:35.955 DEBUG:boto:Token: None 2022-09-03T06:07:35.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WHPuyLa/ROf3iEcmKBU5045+eaQ= 2022-09-03T06:07:35.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WHPuyLa/ROf3iEcmKBU5045+eaQ=', 'Content-Length': '0'} 2022-09-03T06:07:35.961 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009e15d4265de17453-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.963 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.964 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.964 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.965 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.966 DEBUG:boto:Method: GET 2022-09-03T06:07:35.967 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.968 DEBUG:boto:Data: 2022-09-03T06:07:35.968 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.969 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.969 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.970 DEBUG:boto:Params: {} 2022-09-03T06:07:35.970 DEBUG:boto:Token: None 2022-09-03T06:07:35.971 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WHPuyLa/ROf3iEcmKBU5045+eaQ= 2022-09-03T06:07:35.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WHPuyLa/ROf3iEcmKBU5045+eaQ=', 'Content-Length': '0'} 2022-09-03T06:07:35.976 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000002c3bc62adfaffa19-006312ef27-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.977 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.977 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.978 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.978 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.979 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.979 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.981 DEBUG:boto:Data: 2022-09-03T06:07:35.982 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:35.984 DEBUG:boto:Port: 8001 2022-09-03T06:07:35.985 DEBUG:boto:Params: {} 2022-09-03T06:07:35.986 DEBUG:boto:Token: None 2022-09-03T06:07:35.987 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHZEK28muyQUjlC3pZe/ZlAYTEY= 2022-09-03T06:07:35.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHZEK28muyQUjlC3pZe/ZlAYTEY=', 'Content-Length': '0'} 2022-09-03T06:07:35.992 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000642eeb11f7dbd835-006312ef27-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:35.993 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:35.993 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:35.994 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.994 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.995 DEBUG:boto:Method: HEAD 2022-09-03T06:07:35.995 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:35.996 DEBUG:boto:Data: 2022-09-03T06:07:35.996 DEBUG:boto:Headers: {} 2022-09-03T06:07:35.997 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:35.997 DEBUG:boto:Port: 8000 2022-09-03T06:07:35.998 DEBUG:boto:Params: {} 2022-09-03T06:07:35.998 DEBUG:boto:Token: None 2022-09-03T06:07:35.999 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:35 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:36.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHZEK28muyQUjlC3pZe/ZlAYTEY= 2022-09-03T06:07:36.000 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHZEK28muyQUjlC3pZe/ZlAYTEY=', 'Content-Length': '0'} 2022-09-03T06:07:36.002 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000cb9d2e404a1d7fd7-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.003 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.004 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.004 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.005 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:36.005 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:36.006 DEBUG:boto:Method: GET 2022-09-03T06:07:36.007 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:36.007 DEBUG:boto:Data: 2022-09-03T06:07:36.008 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.009 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.010 DEBUG:boto:Params: {} 2022-09-03T06:07:36.010 DEBUG:boto:Token: None 2022-09-03T06:07:36.011 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:36.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQ3mADnp5Tl0fRqw7m3wDCAhCA0= 2022-09-03T06:07:36.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQ3mADnp5Tl0fRqw7m3wDCAhCA0=', 'Content-Length': '0'} 2022-09-03T06:07:36.015 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000d68f051bb4f83aaa-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.015 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.016 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.017 DEBUG:boto:path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:36.017 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:36.018 DEBUG:boto:Method: GET 2022-09-03T06:07:36.018 DEBUG:boto:Path: /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:36.019 DEBUG:boto:Data: 2022-09-03T06:07:36.019 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.020 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.020 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.021 DEBUG:boto:Params: {} 2022-09-03T06:07:36.022 DEBUG:boto:Token: None 2022-09-03T06:07:36.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=1tnGMKYbKFsgWolupnJo3QNfru.ueuk 2022-09-03T06:07:36.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQ3mADnp5Tl0fRqw7m3wDCAhCA0= 2022-09-03T06:07:36.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQ3mADnp5Tl0fRqw7m3wDCAhCA0=', 'Content-Length': '0'} 2022-09-03T06:07:36.029 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '1tnGMKYbKFsgWolupnJo3QNfru.ueuk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000872340949a3efe95-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.074 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.075 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.075 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.076 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.077 DEBUG:boto:Method: GET 2022-09-03T06:07:36.077 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.078 DEBUG:boto:Data: 2022-09-03T06:07:36.078 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.079 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.080 DEBUG:boto:Params: {} 2022-09-03T06:07:36.081 DEBUG:boto:Token: None 2022-09-03T06:07:36.081 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.086 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2fa7f98ebb6f200-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.087 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.088 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.088 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.089 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.089 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.090 DEBUG:boto:Method: GET 2022-09-03T06:07:36.090 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.091 DEBUG:boto:Data: 2022-09-03T06:07:36.091 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.092 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.092 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.093 DEBUG:boto:Params: {} 2022-09-03T06:07:36.093 DEBUG:boto:Token: None 2022-09-03T06:07:36.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.098 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003362a64ed59f37d7-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.099 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.099 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.100 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.100 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.101 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:36.101 DEBUG:boto:Data: 2022-09-03T06:07:36.102 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.102 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.103 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.103 DEBUG:boto:Params: {} 2022-09-03T06:07:36.104 DEBUG:boto:Token: None 2022-09-03T06:07:36.105 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj 2022-09-03T06:07:36.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4= 2022-09-03T06:07:36.106 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4=', 'Content-Length': '0'} 2022-09-03T06:07:36.109 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be33ee0b8dd5c3c2-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.110 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.111 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.112 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.112 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:36.113 DEBUG:boto:Data: 2022-09-03T06:07:36.113 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.114 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.114 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.115 DEBUG:boto:Params: {} 2022-09-03T06:07:36.115 DEBUG:boto:Token: None 2022-09-03T06:07:36.116 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj 2022-09-03T06:07:36.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4= 2022-09-03T06:07:36.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4=', 'Content-Length': '0'} 2022-09-03T06:07:36.121 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004c5537a32c1bd302-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.122 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.122 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.123 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.124 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.124 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.125 DEBUG:boto:Method: GET 2022-09-03T06:07:36.125 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.126 DEBUG:boto:Data: 2022-09-03T06:07:36.127 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.127 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.128 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.128 DEBUG:boto:Params: {} 2022-09-03T06:07:36.129 DEBUG:boto:Token: None 2022-09-03T06:07:36.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA= 2022-09-03T06:07:36.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA=', 'Content-Length': '0'} 2022-09-03T06:07:36.133 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000080135214689844b1-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.134 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.135 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.136 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.136 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.137 DEBUG:boto:Method: GET 2022-09-03T06:07:36.137 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.138 DEBUG:boto:Data: 2022-09-03T06:07:36.139 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.140 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.140 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.141 DEBUG:boto:Params: {} 2022-09-03T06:07:36.141 DEBUG:boto:Token: None 2022-09-03T06:07:36.142 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA= 2022-09-03T06:07:36.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA=', 'Content-Length': '0'} 2022-09-03T06:07:36.146 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003c1a38acca95a9f4-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.191 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.192 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.192 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.193 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.193 DEBUG:boto:Method: GET 2022-09-03T06:07:36.194 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.194 DEBUG:boto:Data: 2022-09-03T06:07:36.195 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.195 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.196 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.196 DEBUG:boto:Params: {} 2022-09-03T06:07:36.197 DEBUG:boto:Token: None 2022-09-03T06:07:36.197 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.202 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009f81d4878b4b7678-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.202 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.203 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.204 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.204 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.205 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.205 DEBUG:boto:Method: GET 2022-09-03T06:07:36.206 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.206 DEBUG:boto:Data: 2022-09-03T06:07:36.207 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.207 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.208 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.208 DEBUG:boto:Params: {} 2022-09-03T06:07:36.209 DEBUG:boto:Token: None 2022-09-03T06:07:36.209 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.213 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003428c309ef9737b0-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.214 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.215 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.215 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.216 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.216 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.217 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.218 DEBUG:boto:Method: GET 2022-09-03T06:07:36.218 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.219 DEBUG:boto:Data: 2022-09-03T06:07:36.219 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.220 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.221 DEBUG:boto:Params: {} 2022-09-03T06:07:36.221 DEBUG:boto:Token: None 2022-09-03T06:07:36.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oq0aB4PRyzH8teb9higzKB3ar64= 2022-09-03T06:07:36.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oq0aB4PRyzH8teb9higzKB3ar64=', 'Content-Length': '0'} 2022-09-03T06:07:36.226 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000748eb10a35d94c56-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.227 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.227 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.228 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.229 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.229 DEBUG:boto:Method: GET 2022-09-03T06:07:36.230 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.231 DEBUG:boto:Data: 2022-09-03T06:07:36.231 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.232 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.233 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.233 DEBUG:boto:Params: {} 2022-09-03T06:07:36.234 DEBUG:boto:Token: None 2022-09-03T06:07:36.235 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oq0aB4PRyzH8teb9higzKB3ar64= 2022-09-03T06:07:36.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oq0aB4PRyzH8teb9higzKB3ar64=', 'Content-Length': '0'} 2022-09-03T06:07:36.239 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000ef7b0dc931a39ee1-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.241 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.242 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.242 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.243 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.243 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.244 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.245 DEBUG:boto:Data: 2022-09-03T06:07:36.246 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.247 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.248 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.248 DEBUG:boto:Params: {} 2022-09-03T06:07:36.249 DEBUG:boto:Token: None 2022-09-03T06:07:36.250 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vCQWzuxAxK36UOGqfx19jNnmtn8= 2022-09-03T06:07:36.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vCQWzuxAxK36UOGqfx19jNnmtn8=', 'Content-Length': '0'} 2022-09-03T06:07:36.255 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000005a31dc71963b7f4-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.256 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.256 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.257 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.257 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.258 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.258 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.259 DEBUG:boto:Data: 2022-09-03T06:07:36.259 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.260 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.260 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.261 DEBUG:boto:Params: {} 2022-09-03T06:07:36.262 DEBUG:boto:Token: None 2022-09-03T06:07:36.262 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vCQWzuxAxK36UOGqfx19jNnmtn8= 2022-09-03T06:07:36.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vCQWzuxAxK36UOGqfx19jNnmtn8=', 'Content-Length': '0'} 2022-09-03T06:07:36.266 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f34d45de5f1ad984-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.267 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.268 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.269 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.270 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.271 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.272 DEBUG:boto:Method: GET 2022-09-03T06:07:36.273 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.273 DEBUG:boto:Data: 2022-09-03T06:07:36.274 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.275 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.276 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.277 DEBUG:boto:Params: {} 2022-09-03T06:07:36.278 DEBUG:boto:Token: None 2022-09-03T06:07:36.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oq0aB4PRyzH8teb9higzKB3ar64= 2022-09-03T06:07:36.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oq0aB4PRyzH8teb9higzKB3ar64=', 'Content-Length': '0'} 2022-09-03T06:07:36.283 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000091ad787060c8c2fd-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.284 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.284 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.285 DEBUG:boto:path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.285 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.286 DEBUG:boto:Method: GET 2022-09-03T06:07:36.287 DEBUG:boto:Path: /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.288 DEBUG:boto:Data: 2022-09-03T06:07:36.289 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.289 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.290 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.291 DEBUG:boto:Params: {} 2022-09-03T06:07:36.292 DEBUG:boto:Token: None 2022-09-03T06:07:36.293 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=piA5MXRR6-9PpmdSMeLZfVgAqVmT5In 2022-09-03T06:07:36.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oq0aB4PRyzH8teb9higzKB3ar64= 2022-09-03T06:07:36.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oq0aB4PRyzH8teb9higzKB3ar64=', 'Content-Length': '0'} 2022-09-03T06:07:36.297 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'piA5MXRR6-9PpmdSMeLZfVgAqVmT5In'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000023965a0c398ad344-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.298 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.299 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.299 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.300 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.301 DEBUG:boto:Method: GET 2022-09-03T06:07:36.301 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.302 DEBUG:boto:Data: 2022-09-03T06:07:36.302 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.303 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.303 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.304 DEBUG:boto:Params: {} 2022-09-03T06:07:36.305 DEBUG:boto:Token: None 2022-09-03T06:07:36.306 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.307 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.310 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000238a4c3a8afca029-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.354 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.355 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.356 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.357 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.357 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.358 DEBUG:boto:Method: GET 2022-09-03T06:07:36.358 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.359 DEBUG:boto:Data: 2022-09-03T06:07:36.360 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.360 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.361 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.361 DEBUG:boto:Params: {} 2022-09-03T06:07:36.362 DEBUG:boto:Token: None 2022-09-03T06:07:36.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.367 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d4cd93cb36114618-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.368 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.369 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.370 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.370 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.371 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:36.372 DEBUG:boto:Data: 2022-09-03T06:07:36.372 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.373 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.374 DEBUG:boto:Params: {} 2022-09-03T06:07:36.374 DEBUG:boto:Token: None 2022-09-03T06:07:36.375 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj 2022-09-03T06:07:36.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4= 2022-09-03T06:07:36.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4=', 'Content-Length': '0'} 2022-09-03T06:07:36.381 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c3e8befc3b058248-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.383 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.384 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.385 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.385 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:36.387 DEBUG:boto:Data: 2022-09-03T06:07:36.387 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.388 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.389 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.389 DEBUG:boto:Params: {} 2022-09-03T06:07:36.390 DEBUG:boto:Token: None 2022-09-03T06:07:36.391 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj 2022-09-03T06:07:36.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4= 2022-09-03T06:07:36.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4=', 'Content-Length': '0'} 2022-09-03T06:07:36.397 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000008af565083da4a663-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.398 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.398 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.399 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.400 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.400 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.401 DEBUG:boto:Method: GET 2022-09-03T06:07:36.401 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.402 DEBUG:boto:Data: 2022-09-03T06:07:36.402 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.403 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.403 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.404 DEBUG:boto:Params: {} 2022-09-03T06:07:36.405 DEBUG:boto:Token: None 2022-09-03T06:07:36.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA= 2022-09-03T06:07:36.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA=', 'Content-Length': '0'} 2022-09-03T06:07:36.409 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000001707d10fa83df7e6-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.410 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.410 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.411 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.411 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.412 DEBUG:boto:Method: GET 2022-09-03T06:07:36.413 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.413 DEBUG:boto:Data: 2022-09-03T06:07:36.414 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.414 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.415 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.415 DEBUG:boto:Params: {} 2022-09-03T06:07:36.416 DEBUG:boto:Token: None 2022-09-03T06:07:36.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA= 2022-09-03T06:07:36.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA=', 'Content-Length': '0'} 2022-09-03T06:07:36.420 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004d04685a04ad8d9c-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.462 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.463 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.463 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.464 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.465 DEBUG:boto:Method: GET 2022-09-03T06:07:36.466 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.466 DEBUG:boto:Data: 2022-09-03T06:07:36.467 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.468 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.469 DEBUG:boto:Params: {} 2022-09-03T06:07:36.469 DEBUG:boto:Token: None 2022-09-03T06:07:36.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.475 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a9435a3664c8573-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.475 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.476 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.477 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.478 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.478 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.479 DEBUG:boto:Method: GET 2022-09-03T06:07:36.479 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.480 DEBUG:boto:Data: 2022-09-03T06:07:36.480 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.481 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.482 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.482 DEBUG:boto:Params: {} 2022-09-03T06:07:36.483 DEBUG:boto:Token: None 2022-09-03T06:07:36.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.488 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000586e414c7b4a66ea-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.489 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.490 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.490 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.491 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.491 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.492 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.492 DEBUG:boto:Method: GET 2022-09-03T06:07:36.493 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.494 DEBUG:boto:Data: 2022-09-03T06:07:36.495 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.497 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.498 DEBUG:boto:Params: {} 2022-09-03T06:07:36.499 DEBUG:boto:Token: None 2022-09-03T06:07:36.501 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jujl+rQEbyFM8HO/t8bNctfbVmI= 2022-09-03T06:07:36.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jujl+rQEbyFM8HO/t8bNctfbVmI=', 'Content-Length': '0'} 2022-09-03T06:07:36.505 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000065ca40f06bb387ac-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.506 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.507 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.507 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.508 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.508 DEBUG:boto:Method: GET 2022-09-03T06:07:36.509 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.509 DEBUG:boto:Data: 2022-09-03T06:07:36.510 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.511 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.511 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.512 DEBUG:boto:Params: {} 2022-09-03T06:07:36.512 DEBUG:boto:Token: None 2022-09-03T06:07:36.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jujl+rQEbyFM8HO/t8bNctfbVmI= 2022-09-03T06:07:36.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jujl+rQEbyFM8HO/t8bNctfbVmI=', 'Content-Length': '0'} 2022-09-03T06:07:36.516 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000762be5259ef42ecb-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.517 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.518 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.518 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.519 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.519 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.520 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.520 DEBUG:boto:Data: 2022-09-03T06:07:36.521 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.521 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.522 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.522 DEBUG:boto:Params: {} 2022-09-03T06:07:36.523 DEBUG:boto:Token: None 2022-09-03T06:07:36.523 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9iQiqq7e8+RjsiyUeOsee2YJj7U= 2022-09-03T06:07:36.525 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9iQiqq7e8+RjsiyUeOsee2YJj7U=', 'Content-Length': '0'} 2022-09-03T06:07:36.527 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000e08803468101be53-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.528 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.529 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.529 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.530 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.531 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.532 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.533 DEBUG:boto:Data: 2022-09-03T06:07:36.533 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.534 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.535 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.536 DEBUG:boto:Params: {} 2022-09-03T06:07:36.537 DEBUG:boto:Token: None 2022-09-03T06:07:36.538 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9iQiqq7e8+RjsiyUeOsee2YJj7U= 2022-09-03T06:07:36.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9iQiqq7e8+RjsiyUeOsee2YJj7U=', 'Content-Length': '0'} 2022-09-03T06:07:36.541 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009f59789a58965b27-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.542 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.543 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.543 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.544 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.545 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.545 DEBUG:boto:Method: GET 2022-09-03T06:07:36.546 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.546 DEBUG:boto:Data: 2022-09-03T06:07:36.547 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.548 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.548 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.549 DEBUG:boto:Params: {} 2022-09-03T06:07:36.549 DEBUG:boto:Token: None 2022-09-03T06:07:36.550 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jujl+rQEbyFM8HO/t8bNctfbVmI= 2022-09-03T06:07:36.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jujl+rQEbyFM8HO/t8bNctfbVmI=', 'Content-Length': '0'} 2022-09-03T06:07:36.553 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000557769954f8d63fa-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.554 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.555 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.555 DEBUG:boto:path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.556 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.557 DEBUG:boto:Method: GET 2022-09-03T06:07:36.557 DEBUG:boto:Path: /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.558 DEBUG:boto:Data: 2022-09-03T06:07:36.559 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.559 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.560 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.560 DEBUG:boto:Params: {} 2022-09-03T06:07:36.561 DEBUG:boto:Token: None 2022-09-03T06:07:36.562 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M 2022-09-03T06:07:36.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jujl+rQEbyFM8HO/t8bNctfbVmI= 2022-09-03T06:07:36.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jujl+rQEbyFM8HO/t8bNctfbVmI=', 'Content-Length': '0'} 2022-09-03T06:07:36.566 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', '2Wre2KfzBqAocyTEgRBwQE5Jo-6ek7M'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000095960fd3077bf0f8-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.567 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.568 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.569 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.570 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.571 DEBUG:boto:Method: GET 2022-09-03T06:07:36.572 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.573 DEBUG:boto:Data: 2022-09-03T06:07:36.574 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.575 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.575 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.576 DEBUG:boto:Params: {} 2022-09-03T06:07:36.576 DEBUG:boto:Token: None 2022-09-03T06:07:36.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.581 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007f89e25e6cb395e1-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.582 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.583 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.583 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.584 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.584 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.585 DEBUG:boto:Method: GET 2022-09-03T06:07:36.585 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.586 DEBUG:boto:Data: 2022-09-03T06:07:36.587 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.587 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.588 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.588 DEBUG:boto:Params: {} 2022-09-03T06:07:36.589 DEBUG:boto:Token: None 2022-09-03T06:07:36.590 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b5b07f52562ecce7-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.596 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.597 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.598 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.599 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.599 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:36.600 DEBUG:boto:Data: 2022-09-03T06:07:36.600 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.601 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.601 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.602 DEBUG:boto:Params: {} 2022-09-03T06:07:36.602 DEBUG:boto:Token: None 2022-09-03T06:07:36.603 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj 2022-09-03T06:07:36.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4= 2022-09-03T06:07:36.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4=', 'Content-Length': '0'} 2022-09-03T06:07:36.608 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004b769b6801d07810-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.609 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.610 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.610 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.611 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:36.611 DEBUG:boto:Data: 2022-09-03T06:07:36.612 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.612 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.613 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.614 DEBUG:boto:Params: {} 2022-09-03T06:07:36.614 DEBUG:boto:Token: None 2022-09-03T06:07:36.615 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj 2022-09-03T06:07:36.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4= 2022-09-03T06:07:36.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4=', 'Content-Length': '0'} 2022-09-03T06:07:36.620 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000860217004e61faff-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.620 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.621 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.622 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.622 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.623 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.623 DEBUG:boto:Method: GET 2022-09-03T06:07:36.624 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.625 DEBUG:boto:Data: 2022-09-03T06:07:36.625 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.626 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.627 DEBUG:boto:Params: {} 2022-09-03T06:07:36.628 DEBUG:boto:Token: None 2022-09-03T06:07:36.628 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA= 2022-09-03T06:07:36.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA=', 'Content-Length': '0'} 2022-09-03T06:07:36.632 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000c6297f9d044f9622-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.632 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.633 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.634 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.634 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.635 DEBUG:boto:Method: GET 2022-09-03T06:07:36.635 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.636 DEBUG:boto:Data: 2022-09-03T06:07:36.636 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.637 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.637 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.638 DEBUG:boto:Params: {} 2022-09-03T06:07:36.638 DEBUG:boto:Token: None 2022-09-03T06:07:36.639 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA= 2022-09-03T06:07:36.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA=', 'Content-Length': '0'} 2022-09-03T06:07:36.650 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000004434a1bf25fccac4-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.690 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.691 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.691 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.692 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.693 DEBUG:boto:Method: GET 2022-09-03T06:07:36.693 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.694 DEBUG:boto:Data: 2022-09-03T06:07:36.694 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.695 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.696 DEBUG:boto:Params: {} 2022-09-03T06:07:36.697 DEBUG:boto:Token: None 2022-09-03T06:07:36.697 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.703 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000826370aa93682212-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.704 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.705 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.706 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.707 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.708 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.709 DEBUG:boto:Method: GET 2022-09-03T06:07:36.710 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.711 DEBUG:boto:Data: 2022-09-03T06:07:36.711 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.712 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.713 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.714 DEBUG:boto:Params: {} 2022-09-03T06:07:36.714 DEBUG:boto:Token: None 2022-09-03T06:07:36.715 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.721 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000000eb8d6ecd76b008-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.722 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.723 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.724 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.726 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.727 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.727 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.728 DEBUG:boto:Method: GET 2022-09-03T06:07:36.729 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.730 DEBUG:boto:Data: 2022-09-03T06:07:36.731 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.733 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.733 DEBUG:boto:Params: {} 2022-09-03T06:07:36.734 DEBUG:boto:Token: None 2022-09-03T06:07:36.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:itu6F8LvScB78+kZPzO2uTzIRSg= 2022-09-03T06:07:36.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:itu6F8LvScB78+kZPzO2uTzIRSg=', 'Content-Length': '0'} 2022-09-03T06:07:36.739 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000872daa5bf162a8cb-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.782 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.783 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.784 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.784 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.785 DEBUG:boto:Method: GET 2022-09-03T06:07:36.785 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.786 DEBUG:boto:Data: 2022-09-03T06:07:36.787 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.787 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.788 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.788 DEBUG:boto:Params: {} 2022-09-03T06:07:36.789 DEBUG:boto:Token: None 2022-09-03T06:07:36.790 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:itu6F8LvScB78+kZPzO2uTzIRSg= 2022-09-03T06:07:36.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:itu6F8LvScB78+kZPzO2uTzIRSg=', 'Content-Length': '0'} 2022-09-03T06:07:36.793 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000863549140fceb0fc-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.794 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.795 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.795 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.796 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.796 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.797 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.797 DEBUG:boto:Data: 2022-09-03T06:07:36.798 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.799 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.799 DEBUG:boto:Params: {} 2022-09-03T06:07:36.800 DEBUG:boto:Token: None 2022-09-03T06:07:36.801 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cHK6L1+SamtNGucgkTWvlSBuNWI= 2022-09-03T06:07:36.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cHK6L1+SamtNGucgkTWvlSBuNWI=', 'Content-Length': '0'} 2022-09-03T06:07:36.804 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000797c9c62620d60d4-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.805 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.805 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.806 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.807 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.807 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.808 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.808 DEBUG:boto:Data: 2022-09-03T06:07:36.809 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.809 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.810 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.810 DEBUG:boto:Params: {} 2022-09-03T06:07:36.811 DEBUG:boto:Token: None 2022-09-03T06:07:36.811 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cHK6L1+SamtNGucgkTWvlSBuNWI= 2022-09-03T06:07:36.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cHK6L1+SamtNGucgkTWvlSBuNWI=', 'Content-Length': '0'} 2022-09-03T06:07:36.815 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000006e96a7868f47fa0e-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.815 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.816 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.816 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.817 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.817 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.818 DEBUG:boto:Method: GET 2022-09-03T06:07:36.818 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.819 DEBUG:boto:Data: 2022-09-03T06:07:36.819 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.820 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.821 DEBUG:boto:Params: {} 2022-09-03T06:07:36.822 DEBUG:boto:Token: None 2022-09-03T06:07:36.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:itu6F8LvScB78+kZPzO2uTzIRSg= 2022-09-03T06:07:36.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:itu6F8LvScB78+kZPzO2uTzIRSg=', 'Content-Length': '0'} 2022-09-03T06:07:36.826 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000083ea0e5ae702c882-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.827 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.827 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.828 DEBUG:boto:path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.828 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.829 DEBUG:boto:Method: GET 2022-09-03T06:07:36.829 DEBUG:boto:Path: /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.830 DEBUG:boto:Data: 2022-09-03T06:07:36.830 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.831 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.831 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.832 DEBUG:boto:Params: {} 2022-09-03T06:07:36.832 DEBUG:boto:Token: None 2022-09-03T06:07:36.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=INkkVt.t6i1066J16-a17HsiLEaCqWq 2022-09-03T06:07:36.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:itu6F8LvScB78+kZPzO2uTzIRSg= 2022-09-03T06:07:36.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:itu6F8LvScB78+kZPzO2uTzIRSg=', 'Content-Length': '0'} 2022-09-03T06:07:36.836 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'INkkVt.t6i1066J16-a17HsiLEaCqWq'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000aae748cc411e2b75-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.837 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.838 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.838 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.839 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.839 DEBUG:boto:Method: GET 2022-09-03T06:07:36.840 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.840 DEBUG:boto:Data: 2022-09-03T06:07:36.841 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.841 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.842 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.842 DEBUG:boto:Params: {} 2022-09-03T06:07:36.843 DEBUG:boto:Token: None 2022-09-03T06:07:36.844 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.844 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.845 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.848 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003cf254359f24bb6c-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.849 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.850 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.850 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.851 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.851 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.852 DEBUG:boto:Method: GET 2022-09-03T06:07:36.852 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.853 DEBUG:boto:Data: 2022-09-03T06:07:36.853 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.854 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.854 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.855 DEBUG:boto:Params: {} 2022-09-03T06:07:36.856 DEBUG:boto:Token: None 2022-09-03T06:07:36.856 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.869 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9fc0643fa120bbd-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.869 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.870 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.871 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.871 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.872 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:36.873 DEBUG:boto:Data: 2022-09-03T06:07:36.873 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.874 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.874 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.875 DEBUG:boto:Params: {} 2022-09-03T06:07:36.876 DEBUG:boto:Token: None 2022-09-03T06:07:36.876 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj 2022-09-03T06:07:36.877 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4= 2022-09-03T06:07:36.877 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4=', 'Content-Length': '0'} 2022-09-03T06:07:36.881 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000475692da6c2d926e-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.882 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.882 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.883 DEBUG:boto:Method: HEAD 2022-09-03T06:07:36.884 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:36.884 DEBUG:boto:Data: 2022-09-03T06:07:36.885 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.885 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.886 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.886 DEBUG:boto:Params: {} 2022-09-03T06:07:36.887 DEBUG:boto:Token: None 2022-09-03T06:07:36.887 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj 2022-09-03T06:07:36.888 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4= 2022-09-03T06:07:36.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W1Q46GSecKgOyk6VpmdRBEn7M+4=', 'Content-Length': '0'} 2022-09-03T06:07:36.892 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000080f2e3b910bfce49-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.894 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.895 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.896 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.897 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.898 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.899 DEBUG:boto:Method: GET 2022-09-03T06:07:36.900 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.901 DEBUG:boto:Data: 2022-09-03T06:07:36.901 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.902 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.902 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.903 DEBUG:boto:Params: {} 2022-09-03T06:07:36.904 DEBUG:boto:Token: None 2022-09-03T06:07:36.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA= 2022-09-03T06:07:36.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA=', 'Content-Length': '0'} 2022-09-03T06:07:36.908 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fcc3ad503244202d-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.909 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.909 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.910 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.911 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.911 DEBUG:boto:Method: GET 2022-09-03T06:07:36.912 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.912 DEBUG:boto:Data: 2022-09-03T06:07:36.913 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.913 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.914 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.914 DEBUG:boto:Params: {} 2022-09-03T06:07:36.915 DEBUG:boto:Token: None 2022-09-03T06:07:36.916 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:36.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA= 2022-09-03T06:07:36.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZN2Yojb4JRGmuyS/d9gcEONMpAA=', 'Content-Length': '0'} 2022-09-03T06:07:36.919 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000827f404a56b0c4c9-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.966 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.967 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.968 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.968 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.969 DEBUG:boto:Method: GET 2022-09-03T06:07:36.969 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.970 DEBUG:boto:Data: 2022-09-03T06:07:36.970 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.971 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.971 DEBUG:boto:Port: 8001 2022-09-03T06:07:36.972 DEBUG:boto:Params: {} 2022-09-03T06:07:36.973 DEBUG:boto:Token: None 2022-09-03T06:07:36.973 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.974 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.975 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.977 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005cebcc4e85b6a019-006312ef28-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.978 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.979 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:36.980 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:36.980 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.981 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:36.981 DEBUG:boto:Method: GET 2022-09-03T06:07:36.982 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:36.983 DEBUG:boto:Data: 2022-09-03T06:07:36.983 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.984 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:36.984 DEBUG:boto:Port: 8000 2022-09-03T06:07:36.985 DEBUG:boto:Params: {} 2022-09-03T06:07:36.985 DEBUG:boto:Token: None 2022-09-03T06:07:36.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:36 GMT /avysmw-19/ 2022-09-03T06:07:36.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ= 2022-09-03T06:07:36.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qnmk7VwXuEal+jKrr+3toMzgLFQ=', 'Content-Length': '0'} 2022-09-03T06:07:36.990 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017da8771387a956d-006312ef28-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:36.991 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:36.992 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:36.993 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:36.994 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:36.995 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:36.996 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:36.996 DEBUG:boto:Method: GET 2022-09-03T06:07:36.997 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:36.997 DEBUG:boto:Data: 2022-09-03T06:07:36.998 DEBUG:boto:Headers: {} 2022-09-03T06:07:36.998 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:36.999 DEBUG:boto:Port: 8001 2022-09-03T06:07:37.000 DEBUG:boto:Params: {} 2022-09-03T06:07:37.000 DEBUG:boto:Token: None 2022-09-03T06:07:37.001 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ll3CGhuwdKGIorG7qr7NECCSTfs= 2022-09-03T06:07:37.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ll3CGhuwdKGIorG7qr7NECCSTfs=', 'Content-Length': '0'} 2022-09-03T06:07:37.004 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000003b7b4229447d1cce-006312ef29-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.005 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:37.006 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:37.007 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.007 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.008 DEBUG:boto:Method: GET 2022-09-03T06:07:37.008 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.009 DEBUG:boto:Data: 2022-09-03T06:07:37.009 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.010 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:37.010 DEBUG:boto:Port: 8000 2022-09-03T06:07:37.011 DEBUG:boto:Params: {} 2022-09-03T06:07:37.011 DEBUG:boto:Token: None 2022-09-03T06:07:37.012 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ll3CGhuwdKGIorG7qr7NECCSTfs= 2022-09-03T06:07:37.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ll3CGhuwdKGIorG7qr7NECCSTfs=', 'Content-Length': '0'} 2022-09-03T06:07:37.016 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000992d74cf09ec197f-006312ef29-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.017 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:37.017 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:37.018 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.019 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.019 DEBUG:boto:Method: HEAD 2022-09-03T06:07:37.020 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.020 DEBUG:boto:Data: 2022-09-03T06:07:37.021 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.022 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:37.022 DEBUG:boto:Port: 8001 2022-09-03T06:07:37.023 DEBUG:boto:Params: {} 2022-09-03T06:07:37.023 DEBUG:boto:Token: None 2022-09-03T06:07:37.024 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6yIYRK6PnN24mibSOfbMQq533Is= 2022-09-03T06:07:37.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6yIYRK6PnN24mibSOfbMQq533Is=', 'Content-Length': '0'} 2022-09-03T06:07:37.027 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fc5266ed9a815fc0-006312ef29-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.028 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:37.029 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:37.029 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.030 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.030 DEBUG:boto:Method: HEAD 2022-09-03T06:07:37.031 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.031 DEBUG:boto:Data: 2022-09-03T06:07:37.032 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.032 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:37.033 DEBUG:boto:Port: 8000 2022-09-03T06:07:37.033 DEBUG:boto:Params: {} 2022-09-03T06:07:37.034 DEBUG:boto:Token: None 2022-09-03T06:07:37.034 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6yIYRK6PnN24mibSOfbMQq533Is= 2022-09-03T06:07:37.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6yIYRK6PnN24mibSOfbMQq533Is=', 'Content-Length': '0'} 2022-09-03T06:07:37.038 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000fb248b604be67f23-006312ef29-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.039 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:37.040 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:37.040 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:37.041 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.041 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.042 DEBUG:boto:Method: GET 2022-09-03T06:07:37.043 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.044 DEBUG:boto:Data: 2022-09-03T06:07:37.045 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.046 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:37.047 DEBUG:boto:Port: 8001 2022-09-03T06:07:37.048 DEBUG:boto:Params: {} 2022-09-03T06:07:37.049 DEBUG:boto:Token: None 2022-09-03T06:07:37.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ll3CGhuwdKGIorG7qr7NECCSTfs= 2022-09-03T06:07:37.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ll3CGhuwdKGIorG7qr7NECCSTfs=', 'Content-Length': '0'} 2022-09-03T06:07:37.054 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000005c80f9e4bd29d0ad-006312ef29-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.055 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:37.056 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:37.056 DEBUG:boto:path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.057 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.057 DEBUG:boto:Method: GET 2022-09-03T06:07:37.058 DEBUG:boto:Path: /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.059 DEBUG:boto:Data: 2022-09-03T06:07:37.059 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.060 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:37.061 DEBUG:boto:Port: 8000 2022-09-03T06:07:37.061 DEBUG:boto:Params: {} 2022-09-03T06:07:37.062 DEBUG:boto:Token: None 2022-09-03T06:07:37.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj?versionId=cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0 2022-09-03T06:07:37.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ll3CGhuwdKGIorG7qr7NECCSTfs= 2022-09-03T06:07:37.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ll3CGhuwdKGIorG7qr7NECCSTfs=', 'Content-Length': '0'} 2022-09-03T06:07:37.066 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:34 GMT'), ('x-amz-version-id', 'cF8EjhosWZOuCqqD1zZg8FJAt2pBaJ0'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx0000081d04aed9c8f81d9-006312ef29-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.110 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:37.111 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:37.111 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:37.112 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:37.112 DEBUG:boto:Method: GET 2022-09-03T06:07:37.113 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:37.113 DEBUG:boto:Data: 2022-09-03T06:07:37.114 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:37.115 DEBUG:boto:Port: 8001 2022-09-03T06:07:37.116 DEBUG:boto:Params: {} 2022-09-03T06:07:37.116 DEBUG:boto:Token: None 2022-09-03T06:07:37.117 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/ 2022-09-03T06:07:37.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WXGotOqOqj3Qqrj6Hr6cZv6Fqcg= 2022-09-03T06:07:37.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WXGotOqOqj3Qqrj6Hr6cZv6Fqcg=', 'Content-Length': '0'} 2022-09-03T06:07:37.122 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008672db99a3955c52-006312ef29-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.123 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:37.123 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:37.124 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:37.124 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:37.125 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:37.126 DEBUG:boto:Method: GET 2022-09-03T06:07:37.126 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:37.127 DEBUG:boto:Data: 2022-09-03T06:07:37.127 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.128 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:37.128 DEBUG:boto:Port: 8000 2022-09-03T06:07:37.129 DEBUG:boto:Params: {} 2022-09-03T06:07:37.129 DEBUG:boto:Token: None 2022-09-03T06:07:37.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/ 2022-09-03T06:07:37.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WXGotOqOqj3Qqrj6Hr6cZv6Fqcg= 2022-09-03T06:07:37.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WXGotOqOqj3Qqrj6Hr6cZv6Fqcg=', 'Content-Length': '0'} 2022-09-03T06:07:37.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001cd3f28fca642cd8-006312ef29-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.134 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:37.135 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:37.136 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:37.136 DEBUG:boto:Method: HEAD 2022-09-03T06:07:37.137 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:37.137 DEBUG:boto:Data: 2022-09-03T06:07:37.138 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.138 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:37.139 DEBUG:boto:Port: 8001 2022-09-03T06:07:37.139 DEBUG:boto:Params: {} 2022-09-03T06:07:37.140 DEBUG:boto:Token: None 2022-09-03T06:07:37.141 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj 2022-09-03T06:07:37.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pJl1UhX2kM2odzmhZEag1z1xZko= 2022-09-03T06:07:37.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pJl1UhX2kM2odzmhZEag1z1xZko=', 'Content-Length': '0'} 2022-09-03T06:07:37.146 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000356bdb2f225722ad-006312ef29-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.147 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:37.147 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:37.148 DEBUG:boto:Method: HEAD 2022-09-03T06:07:37.148 DEBUG:boto:Path: /avysmw-19/obj 2022-09-03T06:07:37.149 DEBUG:boto:Data: 2022-09-03T06:07:37.149 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.150 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:37.151 DEBUG:boto:Port: 8000 2022-09-03T06:07:37.151 DEBUG:boto:Params: {} 2022-09-03T06:07:37.152 DEBUG:boto:Token: None 2022-09-03T06:07:37.152 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj 2022-09-03T06:07:37.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pJl1UhX2kM2odzmhZEag1z1xZko= 2022-09-03T06:07:37.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pJl1UhX2kM2odzmhZEag1z1xZko=', 'Content-Length': '0'} 2022-09-03T06:07:37.160 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000f534151507dba4b8-006312ef29-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.161 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:07:37.162 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:37.163 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:37.164 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:37.165 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:37.165 DEBUG:boto:Method: GET 2022-09-03T06:07:37.166 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:37.167 DEBUG:boto:Data: 2022-09-03T06:07:37.167 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:37.169 DEBUG:boto:Port: 8001 2022-09-03T06:07:37.169 DEBUG:boto:Params: {} 2022-09-03T06:07:37.170 DEBUG:boto:Token: None 2022-09-03T06:07:37.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:37.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EHYBnNdGEFuQbciKAmrumE4Cxd8= 2022-09-03T06:07:37.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EHYBnNdGEFuQbciKAmrumE4Cxd8=', 'Content-Length': '0'} 2022-09-03T06:07:37.175 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx000009abd99d340881be7-006312ef29-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.176 DEBUG:boto:path=/avysmw-19/obj 2022-09-03T06:07:37.177 DEBUG:boto:auth_path=/avysmw-19/obj 2022-09-03T06:07:37.178 DEBUG:boto:path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:37.179 DEBUG:boto:auth_path=/avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:37.180 DEBUG:boto:Method: GET 2022-09-03T06:07:37.181 DEBUG:boto:Path: /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:37.182 DEBUG:boto:Data: 2022-09-03T06:07:37.183 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.183 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:37.184 DEBUG:boto:Port: 8000 2022-09-03T06:07:37.185 DEBUG:boto:Params: {} 2022-09-03T06:07:37.185 DEBUG:boto:Token: None 2022-09-03T06:07:37.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/obj?versionId=Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A 2022-09-03T06:07:37.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EHYBnNdGEFuQbciKAmrumE4Cxd8= 2022-09-03T06:07:37.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EHYBnNdGEFuQbciKAmrumE4Cxd8=', 'Content-Length': '0'} 2022-09-03T06:07:37.190 DEBUG:boto:Response headers: [('Content-Length', '8'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:04:35 GMT'), ('x-amz-version-id', 'Ph8NnAz2uekZeWU.bk2t0FNiid5Y89A'), ('x-rgw-object-type', 'Normal'), ('ETag', '"966634ebf2fc135707d6753692bf4b1e"'), ('x-amz-request-id', 'tx00000be25dd22571d63b4-006312ef29-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.234 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:37.235 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:37.235 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:37.236 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:37.236 DEBUG:boto:Method: GET 2022-09-03T06:07:37.237 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:37.237 DEBUG:boto:Data: 2022-09-03T06:07:37.238 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:07:37.239 DEBUG:boto:Port: 8001 2022-09-03T06:07:37.239 DEBUG:boto:Params: {} 2022-09-03T06:07:37.240 DEBUG:boto:Token: None 2022-09-03T06:07:37.241 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/ 2022-09-03T06:07:37.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WXGotOqOqj3Qqrj6Hr6cZv6Fqcg= 2022-09-03T06:07:37.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WXGotOqOqj3Qqrj6Hr6cZv6Fqcg=', 'Content-Length': '0'} 2022-09-03T06:07:37.245 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ba8ab43be65599e0-006312ef29-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.245 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:37.246 DEBUG:boto:path=/avysmw-19/ 2022-09-03T06:07:37.247 DEBUG:boto:auth_path=/avysmw-19/ 2022-09-03T06:07:37.247 DEBUG:boto:path=/avysmw-19/?prefix=obj 2022-09-03T06:07:37.248 DEBUG:boto:auth_path=/avysmw-19/?prefix=obj 2022-09-03T06:07:37.248 DEBUG:boto:Method: GET 2022-09-03T06:07:37.249 DEBUG:boto:Path: /avysmw-19/?prefix=obj 2022-09-03T06:07:37.249 DEBUG:boto:Data: 2022-09-03T06:07:37.250 DEBUG:boto:Headers: {} 2022-09-03T06:07:37.251 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:07:37.251 DEBUG:boto:Port: 8000 2022-09-03T06:07:37.252 DEBUG:boto:Params: {} 2022-09-03T06:07:37.252 DEBUG:boto:Token: None 2022-09-03T06:07:37.253 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:07:37 GMT /avysmw-19/ 2022-09-03T06:07:37.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WXGotOqOqj3Qqrj6Hr6cZv6Fqcg= 2022-09-03T06:07:37.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WXGotOqOqj3Qqrj6Hr6cZv6Fqcg=', 'Content-Length': '0'} 2022-09-03T06:07:37.257 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f25c56dbcf277a54-006312ef29-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:07:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:37.258 DEBUG:boto:b'avysmw-19obj1000falseobj2022-09-03T06:04:35.704Z"966634ebf2fc135707d6753692bf4b1e"8STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:07:37.259 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-19 zones={test-zone3, test-zone2} 2022-09-03T06:07:37.260 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone2 2022-09-03T06:07:37.261 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.262 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.263 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:37.324 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:37.324 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:37.387 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:07:37.388 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.388 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000001640", 2022-09-03T06:07:37.388 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:07:10.547376Z" 2022-09-03T06:07:37.389 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.389 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.389 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:37.389 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:07:37.390 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.390 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.390 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:37.390 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:07:37.391 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.391 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.391 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:37.391 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:07:37.392 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:37.392 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:07:37.393 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000001640', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:37.394 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2022-09-03T06:07:37.394 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.395 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.396 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:37.454 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:37.455 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:37.501 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:07:37.502 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:07:37.502 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:07:37.502 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:07:37.503 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:07:37.503 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 16975738441461511297 2022-09-03T06:07:37.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:37.504 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:07:37.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:37.504 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:07:37.505 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:37.505 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:37.505 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000001640", 2022-09-03T06:07:37.506 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:37.506 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:37.506 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:37.506 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:10.547376Z" 2022-09-03T06:07:37.507 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:37.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:37.507 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:37.508 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:07:37.508 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:37.508 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:37.509 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:37.509 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:37.509 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:37.510 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:37.510 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:07:37.510 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:37.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:37.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:37.511 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:07:37.511 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:37.512 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:37.512 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:07:37.512 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:37.513 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:37.513 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:37.513 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000" 2022-09-03T06:07:37.513 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:37.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:37.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:37.514 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:07:37.515 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:37.515 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:37.515 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:37.515 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:37.516 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:37.516 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:37.516 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:07:37.516 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:37.517 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:37.517 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:07:37.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:37.517 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:07:37.518 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:07:37.518 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:07:37.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:37.519 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:07:37Z" 2022-09-03T06:07:37.519 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:07:37.520 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 16975738441461511297 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000001640", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:07:10.547376Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.332361Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:07:37Z" } 2022-09-03T06:07:37.521 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000001640', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:07:10.547376Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.332361Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:07:37.521 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000001640', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:37.522 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000001640', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:37.522 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2022-09-03T06:07:37.523 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone3 2022-09-03T06:07:37.523 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.524 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.525 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:37.577 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:37.578 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:37.645 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:07:37.646 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.646 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000001640", 2022-09-03T06:07:37.646 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:07:10.547376Z" 2022-09-03T06:07:37.647 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.647 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.647 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:37.647 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:07:37.648 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.648 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.648 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:37.648 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:07:37.649 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.649 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.649 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:37.649 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:07:37.650 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:37.650 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:07:37.651 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000001640', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:37.652 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2022-09-03T06:07:37.652 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.653 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.654 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:37.706 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:37.707 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:37.774 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:07:37.775 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:07:37.775 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:07:37.776 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:07:37.776 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:07:37.776 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 11727837677587591728 2022-09-03T06:07:37.776 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.777 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:07:37.777 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.777 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:07:37.777 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:37.778 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:37.778 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000001640", 2022-09-03T06:07:37.778 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:37.778 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:37.779 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:37.779 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:10.547376Z" 2022-09-03T06:07:37.779 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:37.780 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.780 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.780 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:07:37.780 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:37.781 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:37.781 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:37.781 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:37.782 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:37.782 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:37.782 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:07:37.782 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:37.783 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.783 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.783 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:07:37.783 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:37.784 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:37.784 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:37.784 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:37.785 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:37.785 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:37.786 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000" 2022-09-03T06:07:37.787 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:37.787 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.788 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.788 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:07:37.789 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:37.789 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:37.790 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:37.790 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:37.791 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:37.791 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:37.792 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:07:37.792 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:37.794 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:37.794 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:07:37.794 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.795 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:07:37.795 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:07:37.795 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:07:37.795 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.796 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:07:37Z" 2022-09-03T06:07:37.796 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:07:37.797 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 11727837677587591728 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000001640", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:07:10.547376Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.332361Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:07:37Z" } 2022-09-03T06:07:37.798 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000001640', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:07:10.547376Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.332361Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:07:37.798 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000001640', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:37.799 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000001640', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:37.799 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2022-09-03T06:07:37.800 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone4 2022-09-03T06:07:37.800 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.801 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.802 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:37.851 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:37.852 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:37.899 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:07:37.899 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.900 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000001640", 2022-09-03T06:07:37.900 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:07:10.547376Z" 2022-09-03T06:07:37.900 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.900 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.901 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:37.901 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:07:37.901 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.901 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.902 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:37.902 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:07:37.902 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:37.903 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:37.903 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:37.903 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:07:37.903 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:37.904 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:07:37.905 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000001640', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:37.905 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2022-09-03T06:07:37.906 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.907 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:37.908 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:37.962 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:37.962 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:38.030 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:07:38.031 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:07:38.031 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:07:38.031 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:07:38.032 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:07:38.032 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 6281331864854677188 2022-09-03T06:07:38.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.032 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:07:38.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.033 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:07:38.033 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:38.034 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.034 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000001640", 2022-09-03T06:07:38.035 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:38.036 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:38.036 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:38.037 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:10.547376Z" 2022-09-03T06:07:38.037 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:38.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.039 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:07:38.039 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:38.040 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.040 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:38.041 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:38.041 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:38.042 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:38.042 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:07:38.043 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:38.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.044 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:07:38.044 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:38.044 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.045 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:07:38.045 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:38.045 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:38.045 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:38.046 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000" 2022-09-03T06:07:38.046 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:38.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.047 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:07:38.047 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:38.047 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.048 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:38.048 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:38.048 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:38.048 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:38.049 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:07:38.049 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:38.049 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:38.049 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:07:38.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.050 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:07:38.051 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:07:38.051 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:07:38.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.052 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:07:38Z" 2022-09-03T06:07:38.052 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:07:38.054 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 6281331864854677188 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000001640", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:07:10.547376Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.332361Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:07:38Z" } 2022-09-03T06:07:38.055 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000001640', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:07:10.547376Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.332361Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:07:38.056 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000001640', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.056 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000001640', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.057 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2022-09-03T06:07:38.057 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone1 2022-09-03T06:07:38.058 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.058 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.059 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:38.107 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:38.108 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:38.173 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:07:38.174 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.174 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000001358", 2022-09-03T06:07:38.175 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:05:33.374293Z" 2022-09-03T06:07:38.175 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.176 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.176 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:38.176 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:04:27.368903Z" 2022-09-03T06:07:38.177 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.177 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.177 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:07:38.178 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "0.000000" 2022-09-03T06:07:38.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.179 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.179 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:38.180 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:07:38.180 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:38.180 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:07:38.182 DEBUG:rgw_multi.tests:data markers for zone=test-zone2 markers={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.182 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2022-09-03T06:07:38.183 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.183 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.184 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:38.238 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:38.238 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:38.315 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:07:38.315 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:07:38.315 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:07:38.316 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:07:38.316 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:07:38.316 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 8557007329946676474 2022-09-03T06:07:38.316 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.317 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:07:38.317 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.317 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:07:38.317 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.318 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.318 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000001358", 2022-09-03T06:07:38.318 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.319 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.319 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.319 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:05:33.374293Z" 2022-09-03T06:07:38.319 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.319 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.320 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.320 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:07:38.320 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.320 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.321 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:38.321 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.321 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.321 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.322 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.368903Z" 2022-09-03T06:07:38.322 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.322 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.322 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.323 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:07:38.323 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.324 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.324 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:38.324 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.325 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.325 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.325 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000" 2022-09-03T06:07:38.325 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.326 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.326 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.326 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:07:38.326 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.327 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.327 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:38.328 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.328 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.328 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.328 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:07:38.329 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.329 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.329 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:07:38.329 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.330 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:07:38.330 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:07:38.330 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:07:38.330 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.331 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:07:38Z" 2022-09-03T06:07:38.331 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:07:38.332 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 8557007329946676474 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000001358", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:05:33.374293Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.368903Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:59:59.312039Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:07:38Z" } 2022-09-03T06:07:38.333 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000001358', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:05:33.374293Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.368903Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:59:59.312039Z'}}] 2022-09-03T06:07:38.333 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.334 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.334 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2022-09-03T06:07:38.335 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone3 2022-09-03T06:07:38.335 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.336 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.336 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:38.387 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:38.388 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:38.433 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:07:38.433 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.434 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000001358", 2022-09-03T06:07:38.434 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:05:33.374293Z" 2022-09-03T06:07:38.434 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.434 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.435 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:38.435 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:04:27.368903Z" 2022-09-03T06:07:38.435 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.435 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.436 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:07:38.436 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "0.000000" 2022-09-03T06:07:38.436 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:38.436 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:38.437 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:38.437 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:07:38.437 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:38.437 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:07:38.439 DEBUG:rgw_multi.tests:data markers for zone=test-zone2 markers={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.439 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2022-09-03T06:07:38.441 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.442 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.443 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:38.496 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:38.496 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:38.560 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:07:38.561 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:07:38.561 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:07:38.561 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:07:38.562 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:07:38.562 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 14782160398262406281 2022-09-03T06:07:38.562 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.563 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:07:38.563 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.563 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:07:38.563 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.564 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.564 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000001358", 2022-09-03T06:07:38.564 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.564 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.565 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.565 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:05:33.374293Z" 2022-09-03T06:07:38.566 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.566 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.567 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.567 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:07:38.568 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.568 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.568 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:07:38.569 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.570 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.570 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.570 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.368903Z" 2022-09-03T06:07:38.570 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.571 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.571 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.572 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:07:38.572 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.572 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.573 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:38.573 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.574 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.574 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.575 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000" 2022-09-03T06:07:38.575 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.576 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.576 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.577 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:07:38.578 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.579 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.579 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:38.580 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.581 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.581 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.582 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:07:38.582 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.583 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.583 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:07:38.584 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.584 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:07:38.585 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:07:38.585 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:07:38.586 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.586 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:07:38Z" 2022-09-03T06:07:38.586 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:07:38.588 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 14782160398262406281 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000001358", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:05:33.374293Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.368903Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:59:59.312039Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:07:38Z" } 2022-09-03T06:07:38.589 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000001358', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:05:33.374293Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.368903Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:59:59.312039Z'}}] 2022-09-03T06:07:38.589 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.590 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000006011', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.591 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2022-09-03T06:07:38.592 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone4 2022-09-03T06:07:38.592 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone1 2022-09-03T06:07:38.593 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.593 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.595 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:38.646 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:38.647 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:38.707 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:07:38.707 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.708 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000001358", 2022-09-03T06:07:38.708 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:05:50.706403Z" 2022-09-03T06:07:38.708 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.709 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.709 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:07:38.709 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:07:38.709 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.710 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.710 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:38.710 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:07:38.710 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.711 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.711 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:38.711 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:07:38.711 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.712 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:07:38.713 DEBUG:rgw_multi.tests:data markers for zone=test-zone3 markers={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000005729', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.714 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2022-09-03T06:07:38.714 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.715 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.715 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:38.768 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:38.768 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:38.827 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:07:38.827 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:07:38.827 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:07:38.828 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:07:38.828 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:07:38.828 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 13955113675122108513 2022-09-03T06:07:38.828 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.829 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:07:38.829 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.829 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:07:38.829 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.830 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.830 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000001358", 2022-09-03T06:07:38.830 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.830 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.831 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.831 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:05:50.706403Z" 2022-09-03T06:07:38.831 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.832 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.832 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.832 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:07:38.833 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.833 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.834 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:07:38.834 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.834 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.835 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.835 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:07:38.836 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.836 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.837 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.837 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:07:38.838 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.838 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.839 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:38.839 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.840 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.841 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.841 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000" 2022-09-03T06:07:38.842 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.842 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.843 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.843 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:07:38.844 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:38.844 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:07:38.845 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:38.845 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:38.846 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:38.846 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:38.846 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:07:38.847 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.847 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.847 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:07:38.848 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.848 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:07:38.848 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:07:38.848 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:07:38.849 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.849 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:07:38Z" 2022-09-03T06:07:38.849 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:07:38.851 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 13955113675122108513 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000001358", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:05:50.706403Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005729", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.402482Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:00:10.617872Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:07:38Z" } 2022-09-03T06:07:38.852 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000001358', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:05:50.706403Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005729', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.402482Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:00:10.617872Z'}}] 2022-09-03T06:07:38.852 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000005729', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.853 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000005729', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.853 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2022-09-03T06:07:38.854 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone2 2022-09-03T06:07:38.855 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.855 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:38.856 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:38.907 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:38.907 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:38.990 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:07:38.990 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.990 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000001358", 2022-09-03T06:07:38.991 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:05:50.706403Z" 2022-09-03T06:07:38.991 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.992 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.992 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:07:38.992 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:07:38.993 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.993 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.993 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:38.993 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:07:38.994 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:38.994 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:38.995 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:38.995 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:07:38.995 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:38.996 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:07:38.998 DEBUG:rgw_multi.tests:data markers for zone=test-zone3 markers={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000005729', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:38.999 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2022-09-03T06:07:39.000 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:39.001 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:39.002 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:39.054 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:39.055 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:39.125 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:07:39.126 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:07:39.126 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:07:39.127 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:07:39.127 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:07:39.127 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 16273219177139511486 2022-09-03T06:07:39.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.129 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:07:39.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:39.129 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:07:39.129 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:39.130 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:39.130 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000001358", 2022-09-03T06:07:39.130 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:39.131 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:39.131 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:39.131 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:05:50.706403Z" 2022-09-03T06:07:39.132 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:39.132 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:07:39.133 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:39.133 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:39.133 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:07:39.134 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:39.134 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:39.134 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:39.135 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:07:39.135 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.135 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:39.136 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:07:39.136 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:39.136 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:39.137 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:07:39.137 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:39.137 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:39.137 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:39.138 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000" 2022-09-03T06:07:39.138 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:39.139 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:07:39.139 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:39.139 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:07:39.140 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:07:39.140 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:39.140 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:39.140 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:39.141 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:07:39.141 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.141 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.142 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:07:39.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.142 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:07:39.142 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:07:39.143 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:07:39.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.143 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:07:39Z" 2022-09-03T06:07:39.143 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:07:39.145 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 16273219177139511486 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000001358", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:05:50.706403Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005729", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.402482Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:00:10.617872Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:07:39Z" } 2022-09-03T06:07:39.145 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000001358', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:05:50.706403Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005729', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.402482Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:00:10.617872Z'}}] 2022-09-03T06:07:39.146 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000005729', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:39.147 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000001358', 1: '00000000000000000000:00000000000000005729', 2: '', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:07:39.147 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2022-09-03T06:07:39.148 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone4 2022-09-03T06:07:39.149 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_concurrent_versioned_object_incremental_sync ... ok 2022-09-03T06:07:39.152 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:07:39.153 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:07:39.153 DEBUG:boto:Method: PUT 2022-09-03T06:07:39.154 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:07:39.154 DEBUG:boto:Data: 2022-09-03T06:07:39.155 DEBUG:boto:Headers: {} 2022-09-03T06:07:39.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:39.156 DEBUG:boto:Port: 8000 2022-09-03T06:07:39.156 DEBUG:boto:Params: {} 2022-09-03T06:07:39.157 DEBUG:boto:Token: None 2022-09-03T06:07:39.157 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:07:39 GMT /avysmw-20/ 2022-09-03T06:07:39.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2oure51IOmpeCxX/sWWiXy/s0NM= 2022-09-03T06:07:39.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2oure51IOmpeCxX/sWWiXy/s0NM=', 'Content-Length': '0'} 2022-09-03T06:07:39.172 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000a4a6dd74fb9198e6-006312ef2b-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:07:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:39.173 DEBUG:rgw_multi.tests:created bucket=avysmw-20 2022-09-03T06:07:39.173 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:39.174 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:39.175 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:39.226 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:39.226 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:39.255 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T06:07:39.269 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:07:39.269 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:39.270 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:07:39.270 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T06:07:39.270 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:39.271 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:39.271 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185259.169934_156.1", 2022-09-03T06:07:39.271 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:07:39.169934Z" 2022-09-03T06:07:39.271 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:39.272 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:39.272 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:39.272 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:07:39.273 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:39.273 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:39.274 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:39.274 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:07:39.275 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:39.275 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:07:39.278 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662185259.169934_156.1', 2: '', 3: ''} 2022-09-03T06:07:39.279 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:07:39.280 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:39.281 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:39.282 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:39.335 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:39.335 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:39.395 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:39.393+0000 7f1fac0d5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:07:39.395 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:39.394+0000 7f1fac0d5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:07:39.412 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:07:39.412 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:07:39.412 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:07:39.413 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:07:39.413 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:07:39.413 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:07:39.413 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:39.414 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.414 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:07:39.414 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:39.415 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:07:39.415 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:39.415 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:07:39.415 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:07:39.416 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:39.416 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:07:39.416 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:39.416 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:07:39.417 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:39.417 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.417 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.418 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:39.418 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:07:39.418 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:39.419 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:07:39.419 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185259.169934_156.1", 2022-09-03T06:07:39.420 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:39.420 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:39.421 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:39.421 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.169934Z", 2022-09-03T06:07:39.421 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:39.422 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.422 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.422 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:39.423 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:07:39.423 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:39.423 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:07:39.423 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:07:39.424 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:39.424 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:39.424 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:39.424 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:07:39.425 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:39.425 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.425 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.425 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:39.426 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:07:39.426 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:39.426 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:07:39.427 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:07:39.427 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:39.427 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:39.427 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:39.428 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:07:39.428 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:39.428 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.429 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:39.429 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:07:39.430 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.430 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:07:39.430 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:07:39.431 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:07:39.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:39.431 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:07:39Z" 2022-09-03T06:07:39.431 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:07:39.433 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185259.169934_156.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:07:39.169934Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:07:39Z" } 2022-09-03T06:07:39.433 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185259.169934_156.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:07:39.169934Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:07:39.434 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185259.169934_156.1', 2: '', 3: ''} 2022-09-03T06:07:39.435 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185259.169934_156.1', 2: '', 3: ''} 2022-09-03T06:07:39.435 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:07:39.436 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:07:39.436 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:39.437 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:39.438 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:39.491 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:39.491 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:39.537 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:39.534+0000 7fa8d93e5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:07:39.538 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:39.536+0000 7fa8d93e5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:07:39.558 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:07:39.559 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:07:39.559 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:07:39.560 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:07:39.560 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:07:39.561 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:07:39.561 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:39.561 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:39.562 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:07:39.562 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:39.562 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:07:39.563 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:39.563 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:07:39.564 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:07:39.564 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:39.565 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:07:39.565 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:39.566 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:07:39.566 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:39.567 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:39.567 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:39.567 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:39.568 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:07:39.568 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:39.569 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:07:39.569 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184997.201443_148.1", 2022-09-03T06:07:39.570 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:39.570 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:39.571 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:39.571 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.201443Z", 2022-09-03T06:07:39.572 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:39.573 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:39.573 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:39.573 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:39.574 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:07:39.575 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:39.575 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:07:39.575 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:39.576 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:39.576 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:39.577 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:39.577 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:07:39.578 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:39.578 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:39.579 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:39.579 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:39.580 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:07:39.580 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:39.580 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:07:39.581 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:39.581 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:39.582 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:39.582 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:39.583 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:07:39.583 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:39.583 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:39.583 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:39.584 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:07:39.584 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:39.585 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:07:39.585 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:07:39.585 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:07:39.585 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:39.586 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:07:39Z" 2022-09-03T06:07:39.586 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:07:39.587 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662184997.201443_148.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:03:17.201443Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:07:39Z" } 2022-09-03T06:07:39.588 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662184997.201443_148.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:03:17.201443Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:07:39.589 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185259.169934_156.1', 2: '', 3: ''} 2022-09-03T06:07:39.589 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662184997.201443_148.1', 2: '', 3: ''} 2022-09-03T06:07:39.590 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662185259.169934_156.1 target=1_1662184997.201443_148.1 2022-09-03T06:07:44.592 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:44.593 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:44.594 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:44.652 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:44.653 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:44.691 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:44.688+0000 7f885adbe600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:07:44.692 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:44.689+0000 7f885adbe600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:07:44.709 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:07:44.710 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:07:44.710 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:07:44.710 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:07:44.711 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:07:44.711 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:07:44.711 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:44.712 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:44.712 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:07:44.712 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:44.712 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:07:44.713 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:44.713 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:07:44.713 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:07:44.713 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:44.714 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:07:44.714 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:44.714 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:07:44.714 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:44.715 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:44.715 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:44.715 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:44.716 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:07:44.716 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:44.716 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:07:44.716 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185259.169934_156.1", 2022-09-03T06:07:44.717 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:44.717 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:44.717 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:44.717 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:39.169934Z", 2022-09-03T06:07:44.718 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:44.718 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:44.718 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:44.718 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:44.719 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:07:44.719 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:44.719 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:07:44.719 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:44.720 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:44.720 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:44.720 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:44.720 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:07:44.721 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:44.721 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:44.721 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:44.721 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:07:44.721 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:07:44.722 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:07:44.722 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:07:44.722 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:07:44.722 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:07:44.723 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:07:44.723 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:07:44.723 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:07:44.723 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:07:44.724 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:44.724 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:07:44.724 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:07:44.724 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:44.725 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:07:44.725 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:07:44.725 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:07:44.725 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:07:44.726 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:07:44Z" 2022-09-03T06:07:44.726 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:07:44.727 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185259.169934_156.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:07:39.169934Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:07:44Z" } 2022-09-03T06:07:44.728 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185259.169934_156.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:07:39.169934Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:07:44.728 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185259.169934_156.1', 2: '', 3: ''} 2022-09-03T06:07:44.729 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185259.169934_156.1', 2: '', 3: ''} 2022-09-03T06:07:44.729 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:07:44.731 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:07:44.732 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:44.732 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:07:44.733 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:07:44.791 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:44.791 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:44.821 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:44.819+0000 7f68996a2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:07:44.822 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:44.820+0000 7f68996a2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:07:44.839 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:07:44.840 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:07:44.841 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:07:44.841 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:07:44.842 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:07:44.842 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:07:44.843 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:44.843 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:44.844 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:07:44.844 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:44.845 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:07:44.845 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:44.846 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:07:44.846 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:07:44.846 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:44.847 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:07:44.847 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:44.848 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:07:44.848 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:44.849 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:44.849 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:44.850 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:44.850 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:07:44.850 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:44.850 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:07:44.851 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185259.169934_156.1", 2022-09-03T06:07:44.851 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:44.851 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:44.851 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:44.852 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.169934Z", 2022-09-03T06:07:44.852 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:44.852 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:44.852 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:44.853 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:44.853 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:07:44.853 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:44.854 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:07:44.854 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:07:44.854 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:44.854 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:44.855 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:44.855 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:07:44.855 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:44.855 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:44.856 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:44.856 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:07:44.856 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:07:44.857 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:07:44.857 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:07:44.858 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:07:44.858 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:07:44.858 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:07:44.859 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:07:44.859 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:07:44.859 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:07:44.860 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:44.860 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:07:44.861 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:07:44.861 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:44.862 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:07:44.863 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:07:44.863 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:07:44.864 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:07:44.864 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:07:44Z" 2022-09-03T06:07:44.865 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:07:44.867 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185259.169934_156.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:07:39.169934Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:07:44Z" } 2022-09-03T06:07:44.868 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185259.169934_156.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:07:39.169934Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:07:44.869 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185259.169934_156.1', 2: '', 3: ''} 2022-09-03T06:07:44.870 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185259.169934_156.1', 2: '', 3: ''} 2022-09-03T06:07:44.870 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:07:44.871 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:07:44.872 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:07:44.873 DEBUG:boto:Method: HEAD 2022-09-03T06:07:44.874 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:07:44.875 DEBUG:boto:Data: 2022-09-03T06:07:44.876 DEBUG:boto:Headers: {} 2022-09-03T06:07:44.877 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:44.878 DEBUG:boto:Port: 8000 2022-09-03T06:07:44.879 DEBUG:boto:Params: {} 2022-09-03T06:07:44.880 DEBUG:boto:Token: None 2022-09-03T06:07:44.880 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:07:44 GMT /avysmw-20/ 2022-09-03T06:07:44.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i3W7+cXe1/oZUWTVIeokcXoqPuI= 2022-09-03T06:07:44.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:07:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i3W7+cXe1/oZUWTVIeokcXoqPuI=', 'Content-Length': '0'} 2022-09-03T06:07:44.886 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008bdf878750f9deaf-006312ef30-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:44.887 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:07:44.888 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:07:44.889 DEBUG:boto:Method: PUT 2022-09-03T06:07:44.889 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:07:44.890 DEBUG:boto:Data: 2022-09-03T06:07:44.890 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:07:44.891 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:07:44.892 DEBUG:boto:Port: 8000 2022-09-03T06:07:44.893 DEBUG:boto:Params: {} 2022-09-03T06:07:44.893 DEBUG:boto:Token: None 2022-09-03T06:07:44.894 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:07:44 GMT /avysmw-20/obj 2022-09-03T06:07:44.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EJYi5icikIdpIQjFJbGdnputhVA= 2022-09-03T06:07:44.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:07:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EJYi5icikIdpIQjFJbGdnputhVA='} 2022-09-03T06:07:44.905 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000deb87c4848f07b66-006312ef30-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:07:44.906 DEBUG:rgw_multi.tests:created initial version id=None 2022-09-03T06:07:44.908 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-20 2022-09-03T06:07:44.908 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:44.909 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:44.909 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:07:44.967 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:44.968 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:45.023 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:45.021+0000 7ff2e895d600 1 waiting to reach incremental sync.. 2022-09-03T06:07:50.024 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:50.021+0000 7ff2e895d600 1 waiting to reach incremental sync.. 2022-09-03T06:07:55.025 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:55.023+0000 7ff2e895d600 1 bucket sync caught up with source: 2022-09-03T06:07:55.026 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:07:55.026 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:07:55.027 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:55.023+0000 7ff2e895d600 0 bucket checkpoint complete 2022-09-03T06:07:55.042 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-20 2022-09-03T06:07:55.043 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:55.044 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:55.044 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:07:55.110 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:55.110 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:55.156 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:55.153+0000 7f7620695600 1 bucket sync caught up with source: 2022-09-03T06:07:55.157 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:07:55.157 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:07:55.157 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:07:55.153+0000 7f7620695600 0 bucket checkpoint complete 2022-09-03T06:07:55.169 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-20 2022-09-03T06:07:55.170 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:55.171 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:07:55.172 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:07:55.237 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:07:55.237 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:07:55.273 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:07:55.272+0000 7fcb577ca600 1 waiting to reach incremental sync.. 2022-09-03T06:08:00.274 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:08:00.272+0000 7fcb577ca600 1 waiting to reach incremental sync.. 2022-09-03T06:08:05.277 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:08:05.274+0000 7fcb577ca600 1 bucket sync caught up with source: 2022-09-03T06:08:05.278 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:08:05.278 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:08:05.279 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:08:05.274+0000 7fcb577ca600 0 bucket checkpoint complete 2022-09-03T06:08:05.298 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-20 2022-09-03T06:08:05.299 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:08:05.301 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:08:05.302 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:08:05.371 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:08:05.371 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:08:05.420 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:05.419+0000 7fe62b7c3600 1 waiting to reach incremental sync.. 2022-09-03T06:08:10.421 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:10.419+0000 7fe62b7c3600 1 waiting to reach incremental sync.. 2022-09-03T06:08:15.423 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:15.421+0000 7fe62b7c3600 1 bucket sync caught up with source: 2022-09-03T06:08:15.424 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.20.5, , ] 2022-09-03T06:08:15.424 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.20.5, , ] 2022-09-03T06:08:15.424 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:15.421+0000 7fe62b7c3600 0 bucket checkpoint complete 2022-09-03T06:08:15.440 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-20 2022-09-03T06:08:15.442 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:08:15.442 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:08:15.443 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:08:15.507 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:08:15.507 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:08:15.548 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:15.546+0000 7fab07382600 1 bucket sync caught up with source: 2022-09-03T06:08:15.548 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.20.5, , ] 2022-09-03T06:08:15.549 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.20.5, , ] 2022-09-03T06:08:15.549 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:15.546+0000 7fab07382600 0 bucket checkpoint complete 2022-09-03T06:08:15.569 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-20 2022-09-03T06:08:15.570 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-20 2022-09-03T06:08:15.571 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:08:15.571 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:08:15.572 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:08:15.630 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:08:15.630 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:08:15.681 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:15.680+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:08:20.682 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:20.680+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:08:25.683 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:25.681+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:08:30.683 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:30.681+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:08:35.684 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:35.681+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:08:40.685 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:40.682+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:08:45.685 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:45.683+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:08:50.686 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:50.683+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:08:55.686 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:08:55.684+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:09:00.686 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:00.684+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:09:05.687 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:05.685+0000 7f573ab71600 1 waiting to reach incremental sync.. 2022-09-03T06:09:10.689 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:10.687+0000 7f573ab71600 1 bucket sync caught up with source: 2022-09-03T06:09:10.690 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.25.5, , ] 2022-09-03T06:09:10.691 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.25.5, , ] 2022-09-03T06:09:10.691 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:10.687+0000 7f573ab71600 0 bucket checkpoint complete 2022-09-03T06:09:10.708 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-20 2022-09-03T06:09:10.709 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:10.710 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:10.711 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:09:10.774 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:10.775 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:10.815 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:10.813+0000 7f3094d53600 1 bucket sync caught up with source: 2022-09-03T06:09:10.816 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.25.5, , ] 2022-09-03T06:09:10.816 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.25.5, , ] 2022-09-03T06:09:10.816 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:10.813+0000 7f3094d53600 0 bucket checkpoint complete 2022-09-03T06:09:10.828 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-20 2022-09-03T06:09:10.828 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-20 zones={test-zone2, test-zone1} 2022-09-03T06:09:10.829 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:10.830 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:10.830 DEBUG:boto:Method: HEAD 2022-09-03T06:09:10.831 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:09:10.831 DEBUG:boto:Data: 2022-09-03T06:09:10.832 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:10.833 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.834 DEBUG:boto:Params: {} 2022-09-03T06:09:10.836 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:09:10.837 DEBUG:boto:Token: None 2022-09-03T06:09:10.839 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/ 2022-09-03T06:09:10.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0A8MxGcKp40kgM/TjNzFlJq8CuY= 2022-09-03T06:09:10.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0A8MxGcKp40kgM/TjNzFlJq8CuY=', 'Content-Length': '0'} 2022-09-03T06:09:10.846 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ed14fe3fc62fc22c-006312ef86-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.847 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:10.847 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:10.848 DEBUG:boto:Method: HEAD 2022-09-03T06:09:10.848 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:09:10.849 DEBUG:boto:Data: 2022-09-03T06:09:10.849 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:10.850 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.851 DEBUG:boto:Params: {} 2022-09-03T06:09:10.852 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:09:10.852 DEBUG:boto:Token: None 2022-09-03T06:09:10.853 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/ 2022-09-03T06:09:10.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0A8MxGcKp40kgM/TjNzFlJq8CuY= 2022-09-03T06:09:10.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0A8MxGcKp40kgM/TjNzFlJq8CuY=', 'Content-Length': '0'} 2022-09-03T06:09:10.858 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000590bbc22bead36f-006312ef86-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.859 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:09:10.859 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:10.860 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:10.860 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.861 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.861 DEBUG:boto:Method: GET 2022-09-03T06:09:10.862 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.863 DEBUG:boto:Data: 2022-09-03T06:09:10.863 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.864 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:10.864 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.865 DEBUG:boto:Params: {} 2022-09-03T06:09:10.865 DEBUG:boto:Token: None 2022-09-03T06:09:10.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/?versions 2022-09-03T06:09:10.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ighO/sPaBVthJfjwdqO855brNNo= 2022-09-03T06:09:10.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ighO/sPaBVthJfjwdqO855brNNo=', 'Content-Length': '0'} 2022-09-03T06:09:10.870 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000522cc648aa69c557-006312ef86-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.871 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:10.872 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:09:10.873 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:09:10.873 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:10.874 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:10.874 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.875 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.875 DEBUG:boto:Method: GET 2022-09-03T06:09:10.876 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.876 DEBUG:boto:Data: 2022-09-03T06:09:10.877 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.877 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:10.878 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.879 DEBUG:boto:Params: {} 2022-09-03T06:09:10.879 DEBUG:boto:Token: None 2022-09-03T06:09:10.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/?versions 2022-09-03T06:09:10.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ighO/sPaBVthJfjwdqO855brNNo= 2022-09-03T06:09:10.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ighO/sPaBVthJfjwdqO855brNNo=', 'Content-Length': '0'} 2022-09-03T06:09:10.883 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003d218c0dd65e680c-006312ef86-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.884 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:10.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:09:10.886 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:10.886 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:10.887 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.887 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.888 DEBUG:boto:Method: GET 2022-09-03T06:09:10.888 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.889 DEBUG:boto:Data: 2022-09-03T06:09:10.889 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:10.890 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.891 DEBUG:boto:Params: {} 2022-09-03T06:09:10.892 DEBUG:boto:Token: None 2022-09-03T06:09:10.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/?versions 2022-09-03T06:09:10.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ighO/sPaBVthJfjwdqO855brNNo= 2022-09-03T06:09:10.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ighO/sPaBVthJfjwdqO855brNNo=', 'Content-Length': '0'} 2022-09-03T06:09:10.896 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001b0b76392ecff683-006312ef86-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.897 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:10.898 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:10.898 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:10.899 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.899 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.900 DEBUG:boto:Method: GET 2022-09-03T06:09:10.901 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:10.901 DEBUG:boto:Data: 2022-09-03T06:09:10.902 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.902 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:10.903 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.904 DEBUG:boto:Params: {} 2022-09-03T06:09:10.904 DEBUG:boto:Token: None 2022-09-03T06:09:10.905 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/?versions 2022-09-03T06:09:10.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ighO/sPaBVthJfjwdqO855brNNo= 2022-09-03T06:09:10.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ighO/sPaBVthJfjwdqO855brNNo=', 'Content-Length': '0'} 2022-09-03T06:09:10.909 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9b6c607762e8479-006312ef86-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.910 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:10.911 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:09:10.911 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:10.912 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:10.912 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.913 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.913 DEBUG:boto:Method: GET 2022-09-03T06:09:10.914 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.914 DEBUG:boto:Data: 2022-09-03T06:09:10.915 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.916 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:10.916 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.917 DEBUG:boto:Params: {} 2022-09-03T06:09:10.917 DEBUG:boto:Token: None 2022-09-03T06:09:10.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t2+uyDvbThhl2Tzia2t08tICK0U= 2022-09-03T06:09:10.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t2+uyDvbThhl2Tzia2t08tICK0U=', 'Content-Length': '0'} 2022-09-03T06:09:10.921 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000049ebee311919d01-006312ef86-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.922 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:10.923 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:10.924 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.924 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.925 DEBUG:boto:Method: GET 2022-09-03T06:09:10.925 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.926 DEBUG:boto:Data: 2022-09-03T06:09:10.927 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:10.928 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.928 DEBUG:boto:Params: {} 2022-09-03T06:09:10.929 DEBUG:boto:Token: None 2022-09-03T06:09:10.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t2+uyDvbThhl2Tzia2t08tICK0U= 2022-09-03T06:09:10.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t2+uyDvbThhl2Tzia2t08tICK0U=', 'Content-Length': '0'} 2022-09-03T06:09:10.933 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000056d7897765a742b7-006312ef86-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.934 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:10.934 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:10.935 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.935 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.936 DEBUG:boto:Method: HEAD 2022-09-03T06:09:10.936 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.937 DEBUG:boto:Data: 2022-09-03T06:09:10.937 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.938 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:10.939 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.939 DEBUG:boto:Params: {} 2022-09-03T06:09:10.940 DEBUG:boto:Token: None 2022-09-03T06:09:10.940 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7edKH7iagHbYFXsu5WYZbqm6E/M= 2022-09-03T06:09:10.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7edKH7iagHbYFXsu5WYZbqm6E/M=', 'Content-Length': '0'} 2022-09-03T06:09:10.943 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000aff07badead08bf4-006312ef86-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.944 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:10.945 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:10.945 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.946 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.947 DEBUG:boto:Method: HEAD 2022-09-03T06:09:10.948 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.949 DEBUG:boto:Data: 2022-09-03T06:09:10.950 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:10.952 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.953 DEBUG:boto:Params: {} 2022-09-03T06:09:10.953 DEBUG:boto:Token: None 2022-09-03T06:09:10.954 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7edKH7iagHbYFXsu5WYZbqm6E/M= 2022-09-03T06:09:10.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7edKH7iagHbYFXsu5WYZbqm6E/M=', 'Content-Length': '0'} 2022-09-03T06:09:10.957 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f3947975e3164a18-006312ef86-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.958 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:09:10.959 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:10.959 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:10.960 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.961 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.961 DEBUG:boto:Method: GET 2022-09-03T06:09:10.962 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.963 DEBUG:boto:Data: 2022-09-03T06:09:10.963 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.964 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:10.964 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.965 DEBUG:boto:Params: {} 2022-09-03T06:09:10.965 DEBUG:boto:Token: None 2022-09-03T06:09:10.966 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t2+uyDvbThhl2Tzia2t08tICK0U= 2022-09-03T06:09:10.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t2+uyDvbThhl2Tzia2t08tICK0U=', 'Content-Length': '0'} 2022-09-03T06:09:10.971 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000328d8e0c0c9bea9e-006312ef86-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.972 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:10.973 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:10.975 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.976 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:10.977 DEBUG:boto:Method: GET 2022-09-03T06:09:10.978 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.979 DEBUG:boto:Data: 2022-09-03T06:09:10.981 DEBUG:boto:Headers: {} 2022-09-03T06:09:10.982 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:10.983 DEBUG:boto:Port: 8000 2022-09-03T06:09:10.984 DEBUG:boto:Params: {} 2022-09-03T06:09:10.985 DEBUG:boto:Token: None 2022-09-03T06:09:10.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:10 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:10.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t2+uyDvbThhl2Tzia2t08tICK0U= 2022-09-03T06:09:10.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t2+uyDvbThhl2Tzia2t08tICK0U=', 'Content-Length': '0'} 2022-09-03T06:09:10.991 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000aeb56c2456cab072-006312ef86-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:10.992 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:10.993 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:10.994 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:10.995 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:10.996 DEBUG:boto:Method: GET 2022-09-03T06:09:10.997 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:10.998 DEBUG:boto:Data: 2022-09-03T06:09:10.999 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.000 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.001 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.002 DEBUG:boto:Params: {} 2022-09-03T06:09:11.003 DEBUG:boto:Token: None 2022-09-03T06:09:11.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.008 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044cc29dfdc72c644-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.009 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.011 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.012 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.013 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.014 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.015 DEBUG:boto:Method: GET 2022-09-03T06:09:11.016 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.017 DEBUG:boto:Data: 2022-09-03T06:09:11.018 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.019 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.020 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.020 DEBUG:boto:Params: {} 2022-09-03T06:09:11.021 DEBUG:boto:Token: None 2022-09-03T06:09:11.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.025 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b32de5364791ea3c-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.027 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.028 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.029 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.030 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.031 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.031 DEBUG:boto:Data: 2022-09-03T06:09:11.032 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.033 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.034 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.035 DEBUG:boto:Params: {} 2022-09-03T06:09:11.036 DEBUG:boto:Token: None 2022-09-03T06:09:11.037 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE= 2022-09-03T06:09:11.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE=', 'Content-Length': '0'} 2022-09-03T06:09:11.041 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001705f189b251c928-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.042 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.042 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.043 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.043 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.044 DEBUG:boto:Data: 2022-09-03T06:09:11.044 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.045 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.046 DEBUG:boto:Params: {} 2022-09-03T06:09:11.047 DEBUG:boto:Token: None 2022-09-03T06:09:11.049 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE= 2022-09-03T06:09:11.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE=', 'Content-Length': '0'} 2022-09-03T06:09:11.053 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ea1fb50e66d483fa-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.054 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:09:11.055 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.056 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.057 DEBUG:boto:Method: GET 2022-09-03T06:09:11.058 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.058 DEBUG:boto:Data: 2022-09-03T06:09:11.059 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.059 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.060 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.061 DEBUG:boto:Params: {} 2022-09-03T06:09:11.061 DEBUG:boto:Token: None 2022-09-03T06:09:11.062 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y= 2022-09-03T06:09:11.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y=', 'Content-Length': '0'} 2022-09-03T06:09:11.065 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005d3c45cecb268111-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.067 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.067 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.068 DEBUG:boto:Method: GET 2022-09-03T06:09:11.069 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.069 DEBUG:boto:Data: 2022-09-03T06:09:11.070 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.070 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.072 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.073 DEBUG:boto:Params: {} 2022-09-03T06:09:11.074 DEBUG:boto:Token: None 2022-09-03T06:09:11.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y= 2022-09-03T06:09:11.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y=', 'Content-Length': '0'} 2022-09-03T06:09:11.082 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001e239a96ee665536-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.083 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.084 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.085 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.086 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.087 DEBUG:boto:Method: GET 2022-09-03T06:09:11.088 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.089 DEBUG:boto:Data: 2022-09-03T06:09:11.090 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.091 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.092 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.093 DEBUG:boto:Params: {} 2022-09-03T06:09:11.094 DEBUG:boto:Token: None 2022-09-03T06:09:11.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.098 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000061e161d52037f336-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.099 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.100 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.101 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.101 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.102 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.102 DEBUG:boto:Method: GET 2022-09-03T06:09:11.103 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.103 DEBUG:boto:Data: 2022-09-03T06:09:11.104 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.105 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.106 DEBUG:boto:Params: {} 2022-09-03T06:09:11.106 DEBUG:boto:Token: None 2022-09-03T06:09:11.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.112 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028674efa8b47b0a6-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.113 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.114 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-20 zones={test-zone2, test-zone1} 2022-09-03T06:09:11.115 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-20 zones={test-zone3, test-zone1} 2022-09-03T06:09:11.115 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.116 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.118 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.118 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:09:11.119 DEBUG:boto:Data: 2022-09-03T06:09:11.120 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.121 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.122 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.123 DEBUG:boto:Params: {} 2022-09-03T06:09:11.125 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:09:11.126 DEBUG:boto:Token: None 2022-09-03T06:09:11.127 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UuUjov8rzR61SdaUm0BMJBJEnaY= 2022-09-03T06:09:11.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UuUjov8rzR61SdaUm0BMJBJEnaY=', 'Content-Length': '0'} 2022-09-03T06:09:11.133 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005c774b4ec6069747-006312ef87-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.134 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.135 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.136 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.137 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:09:11.138 DEBUG:boto:Data: 2022-09-03T06:09:11.138 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.139 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.140 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.140 DEBUG:boto:Params: {} 2022-09-03T06:09:11.141 DEBUG:boto:Token: None 2022-09-03T06:09:11.142 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UuUjov8rzR61SdaUm0BMJBJEnaY= 2022-09-03T06:09:11.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UuUjov8rzR61SdaUm0BMJBJEnaY=', 'Content-Length': '0'} 2022-09-03T06:09:11.145 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000707937ff8ab3a9f7-006312ef87-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.146 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:09:11.147 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.147 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.148 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.148 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.149 DEBUG:boto:Method: GET 2022-09-03T06:09:11.150 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.151 DEBUG:boto:Data: 2022-09-03T06:09:11.151 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.152 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.152 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.153 DEBUG:boto:Params: {} 2022-09-03T06:09:11.153 DEBUG:boto:Token: None 2022-09-03T06:09:11.154 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/?versions 2022-09-03T06:09:11.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4= 2022-09-03T06:09:11.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4=', 'Content-Length': '0'} 2022-09-03T06:09:11.158 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d66997e20348a49-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.158 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.159 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:09:11.160 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:09:11.160 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.161 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.162 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.162 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.163 DEBUG:boto:Method: GET 2022-09-03T06:09:11.164 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.164 DEBUG:boto:Data: 2022-09-03T06:09:11.165 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.166 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.166 DEBUG:boto:Params: {} 2022-09-03T06:09:11.167 DEBUG:boto:Token: None 2022-09-03T06:09:11.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/?versions 2022-09-03T06:09:11.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4= 2022-09-03T06:09:11.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4=', 'Content-Length': '0'} 2022-09-03T06:09:11.171 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f8fbb8aec848c11d-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.172 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.173 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:09:11.173 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.174 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.174 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.175 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.175 DEBUG:boto:Method: GET 2022-09-03T06:09:11.176 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.176 DEBUG:boto:Data: 2022-09-03T06:09:11.177 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.177 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.178 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.178 DEBUG:boto:Params: {} 2022-09-03T06:09:11.179 DEBUG:boto:Token: None 2022-09-03T06:09:11.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/?versions 2022-09-03T06:09:11.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4= 2022-09-03T06:09:11.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4=', 'Content-Length': '0'} 2022-09-03T06:09:11.183 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b8a90dd5e2f54c76-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.184 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.185 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.185 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.186 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.186 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.187 DEBUG:boto:Method: GET 2022-09-03T06:09:11.187 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.188 DEBUG:boto:Data: 2022-09-03T06:09:11.188 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.189 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.189 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.190 DEBUG:boto:Params: {} 2022-09-03T06:09:11.190 DEBUG:boto:Token: None 2022-09-03T06:09:11.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/?versions 2022-09-03T06:09:11.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4= 2022-09-03T06:09:11.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4=', 'Content-Length': '0'} 2022-09-03T06:09:11.195 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027f2d285cdebd75c-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.238 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.239 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:09:11.240 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.240 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.241 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.242 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.242 DEBUG:boto:Method: GET 2022-09-03T06:09:11.243 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.243 DEBUG:boto:Data: 2022-09-03T06:09:11.244 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.244 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.245 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.246 DEBUG:boto:Params: {} 2022-09-03T06:09:11.246 DEBUG:boto:Token: None 2022-09-03T06:09:11.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0= 2022-09-03T06:09:11.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0=', 'Content-Length': '0'} 2022-09-03T06:09:11.250 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f515730e52177b38-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.251 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.252 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.252 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.253 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.253 DEBUG:boto:Method: GET 2022-09-03T06:09:11.254 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.254 DEBUG:boto:Data: 2022-09-03T06:09:11.255 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.255 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.256 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.256 DEBUG:boto:Params: {} 2022-09-03T06:09:11.257 DEBUG:boto:Token: None 2022-09-03T06:09:11.258 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0= 2022-09-03T06:09:11.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0=', 'Content-Length': '0'} 2022-09-03T06:09:11.261 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000056d1b9e86ac3165b-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.262 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.263 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.263 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.264 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.264 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.265 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.265 DEBUG:boto:Data: 2022-09-03T06:09:11.265 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.266 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.266 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.267 DEBUG:boto:Params: {} 2022-09-03T06:09:11.268 DEBUG:boto:Token: None 2022-09-03T06:09:11.268 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hEbDBnCnEA5GXRJC5TwK8hhUOt8= 2022-09-03T06:09:11.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hEbDBnCnEA5GXRJC5TwK8hhUOt8=', 'Content-Length': '0'} 2022-09-03T06:09:11.272 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005e522e80d18edba2-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.273 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.273 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.274 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.275 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.275 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.276 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.276 DEBUG:boto:Data: 2022-09-03T06:09:11.276 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.278 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.278 DEBUG:boto:Params: {} 2022-09-03T06:09:11.279 DEBUG:boto:Token: None 2022-09-03T06:09:11.279 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hEbDBnCnEA5GXRJC5TwK8hhUOt8= 2022-09-03T06:09:11.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hEbDBnCnEA5GXRJC5TwK8hhUOt8=', 'Content-Length': '0'} 2022-09-03T06:09:11.282 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007ab222fc9f675a3b-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.283 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:09:11.284 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.284 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.285 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.285 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.286 DEBUG:boto:Method: GET 2022-09-03T06:09:11.286 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.287 DEBUG:boto:Data: 2022-09-03T06:09:11.287 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.288 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.289 DEBUG:boto:Params: {} 2022-09-03T06:09:11.290 DEBUG:boto:Token: None 2022-09-03T06:09:11.290 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0= 2022-09-03T06:09:11.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0=', 'Content-Length': '0'} 2022-09-03T06:09:11.294 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006b724e260d09bf70-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.294 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.295 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.295 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.296 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.297 DEBUG:boto:Method: GET 2022-09-03T06:09:11.297 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.298 DEBUG:boto:Data: 2022-09-03T06:09:11.298 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.299 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.300 DEBUG:boto:Params: {} 2022-09-03T06:09:11.300 DEBUG:boto:Token: None 2022-09-03T06:09:11.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0= 2022-09-03T06:09:11.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0=', 'Content-Length': '0'} 2022-09-03T06:09:11.305 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ec976dd160fbb7e0-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.306 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.307 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.308 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.309 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.310 DEBUG:boto:Method: GET 2022-09-03T06:09:11.311 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.312 DEBUG:boto:Data: 2022-09-03T06:09:11.312 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.314 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.315 DEBUG:boto:Params: {} 2022-09-03T06:09:11.317 DEBUG:boto:Token: None 2022-09-03T06:09:11.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.323 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000008be4521e460ac72-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.324 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.325 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.326 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.326 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.327 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.327 DEBUG:boto:Method: GET 2022-09-03T06:09:11.328 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.329 DEBUG:boto:Data: 2022-09-03T06:09:11.329 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.330 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.331 DEBUG:boto:Params: {} 2022-09-03T06:09:11.332 DEBUG:boto:Token: None 2022-09-03T06:09:11.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.337 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c59dd244e23b307d-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.337 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.339 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.339 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.340 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.341 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.342 DEBUG:boto:Data: 2022-09-03T06:09:11.343 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.345 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.346 DEBUG:boto:Params: {} 2022-09-03T06:09:11.347 DEBUG:boto:Token: None 2022-09-03T06:09:11.347 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE= 2022-09-03T06:09:11.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE=', 'Content-Length': '0'} 2022-09-03T06:09:11.352 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e6928f10a5968ae9-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.353 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.354 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.355 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.355 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.356 DEBUG:boto:Data: 2022-09-03T06:09:11.357 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.357 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.358 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.359 DEBUG:boto:Params: {} 2022-09-03T06:09:11.360 DEBUG:boto:Token: None 2022-09-03T06:09:11.361 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE= 2022-09-03T06:09:11.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE=', 'Content-Length': '0'} 2022-09-03T06:09:11.365 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ab6c8a11180958f0-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.367 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:09:11.368 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.369 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.370 DEBUG:boto:Method: GET 2022-09-03T06:09:11.370 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.371 DEBUG:boto:Data: 2022-09-03T06:09:11.372 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.374 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.375 DEBUG:boto:Params: {} 2022-09-03T06:09:11.376 DEBUG:boto:Token: None 2022-09-03T06:09:11.377 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y= 2022-09-03T06:09:11.379 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y=', 'Content-Length': '0'} 2022-09-03T06:09:11.381 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000116dffeed3c39b9d-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.382 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.382 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.383 DEBUG:boto:Method: GET 2022-09-03T06:09:11.383 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.384 DEBUG:boto:Data: 2022-09-03T06:09:11.385 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.385 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.386 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.387 DEBUG:boto:Params: {} 2022-09-03T06:09:11.387 DEBUG:boto:Token: None 2022-09-03T06:09:11.388 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y= 2022-09-03T06:09:11.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y=', 'Content-Length': '0'} 2022-09-03T06:09:11.391 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006d9819768300b869-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.392 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.393 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.393 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.394 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.395 DEBUG:boto:Method: GET 2022-09-03T06:09:11.395 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.396 DEBUG:boto:Data: 2022-09-03T06:09:11.397 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.397 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.398 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.398 DEBUG:boto:Params: {} 2022-09-03T06:09:11.399 DEBUG:boto:Token: None 2022-09-03T06:09:11.400 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.404 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000efb615b977c3fc23-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.404 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.405 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.406 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.406 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.407 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.407 DEBUG:boto:Method: GET 2022-09-03T06:09:11.408 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.409 DEBUG:boto:Data: 2022-09-03T06:09:11.409 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.410 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.410 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.411 DEBUG:boto:Params: {} 2022-09-03T06:09:11.411 DEBUG:boto:Token: None 2022-09-03T06:09:11.412 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.416 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d72d4023daea639c-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.417 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.418 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-20 zones={test-zone3, test-zone1} 2022-09-03T06:09:11.418 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-20 zones={test-zone3, test-zone2} 2022-09-03T06:09:11.419 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.420 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.420 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.421 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:09:11.421 DEBUG:boto:Data: 2022-09-03T06:09:11.422 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.422 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.423 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.424 DEBUG:boto:Params: {} 2022-09-03T06:09:11.424 DEBUG:boto:Token: None 2022-09-03T06:09:11.425 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UuUjov8rzR61SdaUm0BMJBJEnaY= 2022-09-03T06:09:11.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UuUjov8rzR61SdaUm0BMJBJEnaY=', 'Content-Length': '0'} 2022-09-03T06:09:11.429 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000292f90963ebd3ae6-006312ef87-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.430 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.431 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.432 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.433 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:09:11.434 DEBUG:boto:Data: 2022-09-03T06:09:11.435 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.436 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.437 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.437 DEBUG:boto:Params: {} 2022-09-03T06:09:11.439 DEBUG:boto:Token: None 2022-09-03T06:09:11.439 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UuUjov8rzR61SdaUm0BMJBJEnaY= 2022-09-03T06:09:11.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UuUjov8rzR61SdaUm0BMJBJEnaY=', 'Content-Length': '0'} 2022-09-03T06:09:11.444 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000059e7b5690ed08e2a-006312ef87-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.445 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:09:11.446 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.447 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.448 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.449 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.450 DEBUG:boto:Method: GET 2022-09-03T06:09:11.451 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.452 DEBUG:boto:Data: 2022-09-03T06:09:11.452 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.454 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.454 DEBUG:boto:Params: {} 2022-09-03T06:09:11.455 DEBUG:boto:Token: None 2022-09-03T06:09:11.455 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/?versions 2022-09-03T06:09:11.456 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4= 2022-09-03T06:09:11.457 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4=', 'Content-Length': '0'} 2022-09-03T06:09:11.459 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a9496110fbe47cbf-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.460 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.461 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:09:11.461 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:09:11.462 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.463 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.463 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.464 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.464 DEBUG:boto:Method: GET 2022-09-03T06:09:11.465 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.465 DEBUG:boto:Data: 2022-09-03T06:09:11.466 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.466 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.467 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.467 DEBUG:boto:Params: {} 2022-09-03T06:09:11.468 DEBUG:boto:Token: None 2022-09-03T06:09:11.469 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/?versions 2022-09-03T06:09:11.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4= 2022-09-03T06:09:11.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4=', 'Content-Length': '0'} 2022-09-03T06:09:11.472 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5ab5a5bf2c89e40-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.473 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.474 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:09:11.475 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.475 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.476 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.476 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.477 DEBUG:boto:Method: GET 2022-09-03T06:09:11.477 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.478 DEBUG:boto:Data: 2022-09-03T06:09:11.478 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.479 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.480 DEBUG:boto:Params: {} 2022-09-03T06:09:11.481 DEBUG:boto:Token: None 2022-09-03T06:09:11.481 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/?versions 2022-09-03T06:09:11.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4= 2022-09-03T06:09:11.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4=', 'Content-Length': '0'} 2022-09-03T06:09:11.485 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000006a9039a69b5f32-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.526 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.527 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.528 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.528 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.529 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.529 DEBUG:boto:Method: GET 2022-09-03T06:09:11.530 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:09:11.530 DEBUG:boto:Data: 2022-09-03T06:09:11.531 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.531 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.532 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.532 DEBUG:boto:Params: {} 2022-09-03T06:09:11.533 DEBUG:boto:Token: None 2022-09-03T06:09:11.534 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/?versions 2022-09-03T06:09:11.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4= 2022-09-03T06:09:11.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9d1WknU9HqNKlXte9m6rln3YTV4=', 'Content-Length': '0'} 2022-09-03T06:09:11.537 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008787814704550a15-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.538 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.539 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:09:11.540 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.540 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.541 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.541 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.542 DEBUG:boto:Method: GET 2022-09-03T06:09:11.542 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.543 DEBUG:boto:Data: 2022-09-03T06:09:11.543 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.544 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.544 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.545 DEBUG:boto:Params: {} 2022-09-03T06:09:11.546 DEBUG:boto:Token: None 2022-09-03T06:09:11.546 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0= 2022-09-03T06:09:11.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0=', 'Content-Length': '0'} 2022-09-03T06:09:11.550 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e4b20aa936ef4c91-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.551 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.551 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.552 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.552 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.553 DEBUG:boto:Method: GET 2022-09-03T06:09:11.553 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.554 DEBUG:boto:Data: 2022-09-03T06:09:11.554 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.555 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.555 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.556 DEBUG:boto:Params: {} 2022-09-03T06:09:11.557 DEBUG:boto:Token: None 2022-09-03T06:09:11.557 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0= 2022-09-03T06:09:11.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0=', 'Content-Length': '0'} 2022-09-03T06:09:11.560 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000002aa4defdb1c4dcd0-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.561 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.562 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.562 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.563 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.563 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.564 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.564 DEBUG:boto:Data: 2022-09-03T06:09:11.565 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.565 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.566 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.566 DEBUG:boto:Params: {} 2022-09-03T06:09:11.567 DEBUG:boto:Token: None 2022-09-03T06:09:11.567 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hEbDBnCnEA5GXRJC5TwK8hhUOt8= 2022-09-03T06:09:11.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hEbDBnCnEA5GXRJC5TwK8hhUOt8=', 'Content-Length': '0'} 2022-09-03T06:09:11.571 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000093dae251bf17c4e8-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.571 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.572 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.572 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.573 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.573 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.574 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.574 DEBUG:boto:Data: 2022-09-03T06:09:11.575 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.575 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.576 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.576 DEBUG:boto:Params: {} 2022-09-03T06:09:11.577 DEBUG:boto:Token: None 2022-09-03T06:09:11.578 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hEbDBnCnEA5GXRJC5TwK8hhUOt8= 2022-09-03T06:09:11.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hEbDBnCnEA5GXRJC5TwK8hhUOt8=', 'Content-Length': '0'} 2022-09-03T06:09:11.581 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005692d20fa7f04806-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.581 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:09:11.582 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.582 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.583 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.583 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.584 DEBUG:boto:Method: GET 2022-09-03T06:09:11.585 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.585 DEBUG:boto:Data: 2022-09-03T06:09:11.586 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.587 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.587 DEBUG:boto:Params: {} 2022-09-03T06:09:11.588 DEBUG:boto:Token: None 2022-09-03T06:09:11.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0= 2022-09-03T06:09:11.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0=', 'Content-Length': '0'} 2022-09-03T06:09:11.592 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009f9aeaf1e9042f35-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.592 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.593 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.594 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.594 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:09:11.595 DEBUG:boto:Method: GET 2022-09-03T06:09:11.595 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.596 DEBUG:boto:Data: 2022-09-03T06:09:11.596 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.597 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.597 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.598 DEBUG:boto:Params: {} 2022-09-03T06:09:11.598 DEBUG:boto:Token: None 2022-09-03T06:09:11.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:09:11.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0= 2022-09-03T06:09:11.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m3j2TrKAt6S/ZFzS1zCBRWx32S0=', 'Content-Length': '0'} 2022-09-03T06:09:11.602 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b1b27d93b3dd8a98-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.603 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.603 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.604 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.605 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.605 DEBUG:boto:Method: GET 2022-09-03T06:09:11.606 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.606 DEBUG:boto:Data: 2022-09-03T06:09:11.607 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.607 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.608 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.608 DEBUG:boto:Params: {} 2022-09-03T06:09:11.609 DEBUG:boto:Token: None 2022-09-03T06:09:11.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.620 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e8d442616b56d5a4-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.621 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.622 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.623 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.623 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.624 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.624 DEBUG:boto:Method: GET 2022-09-03T06:09:11.625 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.625 DEBUG:boto:Data: 2022-09-03T06:09:11.626 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.626 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.627 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.628 DEBUG:boto:Params: {} 2022-09-03T06:09:11.628 DEBUG:boto:Token: None 2022-09-03T06:09:11.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.633 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036372159e2016cec-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.674 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.676 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.677 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.679 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.680 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.680 DEBUG:boto:Data: 2022-09-03T06:09:11.681 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.681 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.682 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.682 DEBUG:boto:Params: {} 2022-09-03T06:09:11.683 DEBUG:boto:Token: None 2022-09-03T06:09:11.684 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE= 2022-09-03T06:09:11.685 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE=', 'Content-Length': '0'} 2022-09-03T06:09:11.688 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b5c5b2a7b5946434-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.689 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.689 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.690 DEBUG:boto:Method: HEAD 2022-09-03T06:09:11.691 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.691 DEBUG:boto:Data: 2022-09-03T06:09:11.692 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.693 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.693 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.694 DEBUG:boto:Params: {} 2022-09-03T06:09:11.695 DEBUG:boto:Token: None 2022-09-03T06:09:11.695 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.696 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE= 2022-09-03T06:09:11.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b6Z98Wpj1/C167LXs8GpZKTI1qE=', 'Content-Length': '0'} 2022-09-03T06:09:11.699 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c080759961529f2b-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.700 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:09:11.700 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.701 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.702 DEBUG:boto:Method: GET 2022-09-03T06:09:11.703 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.704 DEBUG:boto:Data: 2022-09-03T06:09:11.705 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.706 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.707 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.709 DEBUG:boto:Params: {} 2022-09-03T06:09:11.710 DEBUG:boto:Token: None 2022-09-03T06:09:11.710 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y= 2022-09-03T06:09:11.712 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y=', 'Content-Length': '0'} 2022-09-03T06:09:11.715 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000094fce4b1fa2395d1-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.716 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:11.716 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:11.717 DEBUG:boto:Method: GET 2022-09-03T06:09:11.718 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:11.719 DEBUG:boto:Data: 2022-09-03T06:09:11.720 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.721 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.722 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.723 DEBUG:boto:Params: {} 2022-09-03T06:09:11.724 DEBUG:boto:Token: None 2022-09-03T06:09:11.725 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/obj 2022-09-03T06:09:11.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y= 2022-09-03T06:09:11.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/P3kO23gJhygMPsOvnlEh1daU9Y=', 'Content-Length': '0'} 2022-09-03T06:09:11.729 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000078daed41f6b31f49-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.730 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.731 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.731 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.732 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.733 DEBUG:boto:Method: GET 2022-09-03T06:09:11.734 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.735 DEBUG:boto:Data: 2022-09-03T06:09:11.736 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:09:11.739 DEBUG:boto:Port: 8001 2022-09-03T06:09:11.740 DEBUG:boto:Params: {} 2022-09-03T06:09:11.741 DEBUG:boto:Token: None 2022-09-03T06:09:11.742 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.746 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d444fb0ca86b6155-006312ef87-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.790 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.791 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.792 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.793 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.794 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:09:11.794 DEBUG:boto:Method: GET 2022-09-03T06:09:11.795 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:09:11.796 DEBUG:boto:Data: 2022-09-03T06:09:11.796 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.797 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:09:11.798 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.798 DEBUG:boto:Params: {} 2022-09-03T06:09:11.799 DEBUG:boto:Token: None 2022-09-03T06:09:11.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/ 2022-09-03T06:09:11.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco= 2022-09-03T06:09:11.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2s+EeOOSmf9AXb5kcligyjGxco=', 'Content-Length': '0'} 2022-09-03T06:09:11.804 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1ce6f5668a8ea9c-006312ef87-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.805 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:09:11.806 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-20 zones={test-zone3, test-zone2} 2022-09-03T06:09:11.807 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:11.807 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:11.808 DEBUG:boto:path=/avysmw-20/?versioning 2022-09-03T06:09:11.808 DEBUG:boto:auth_path=/avysmw-20/?versioning 2022-09-03T06:09:11.809 DEBUG:boto:Method: PUT 2022-09-03T06:09:11.809 DEBUG:boto:Path: /avysmw-20/?versioning 2022-09-03T06:09:11.810 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:09:11.810 DEBUG:boto:Headers: {} 2022-09-03T06:09:11.811 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:11.811 DEBUG:boto:Port: 8000 2022-09-03T06:09:11.812 DEBUG:boto:Params: {} 2022-09-03T06:09:11.812 DEBUG:boto:Token: None 2022-09-03T06:09:11.813 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:09:11 GMT /avysmw-20/?versioning 2022-09-03T06:09:11.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cyR9JiaOSGrDid3BSs6ntsZenco= 2022-09-03T06:09:11.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cyR9JiaOSGrDid3BSs6ntsZenco=', 'Content-Length': '228'} 2022-09-03T06:09:11.822 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000004854de3a56413b8d-006312ef87-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:09:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:11.823 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:11.823 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:11.824 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:09:11.882 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:11.883 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:11.930 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T06:09:11.951 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:09:11.952 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:09:11.952 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:09:11.953 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T06:09:11.953 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:09:11.953 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:09:11.954 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185351.819576_158.1", 2022-09-03T06:09:11.954 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:09:11.819576Z" 2022-09-03T06:09:11.954 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:09:11.955 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:09:11.955 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:09:11.955 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:09:11.956 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:09:11.956 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:09:11.956 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:09:11.956 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:09:11.957 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:09:11.957 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:09:11.958 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662185351.819576_158.1', 2: '', 3: ''} 2022-09-03T06:09:11.959 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:09:11.960 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:11.965 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:11.966 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:09:12.026 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:12.027 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:12.081 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:12.079+0000 7fe7d41c6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:09:12.082 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:12.080+0000 7fe7d41c6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:09:12.105 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:09:12.106 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:09:12.106 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:09:12.107 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:09:12.107 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:09:12.107 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:09:12.107 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:12.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:12.108 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:09:12.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:12.108 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:09:12.109 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:12.109 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:12.109 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:09:12.109 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:12.110 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:09:12.110 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:12.110 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:09:12.111 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:12.111 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:12.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:12.112 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:12.113 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:09:12.113 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:12.114 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:12.114 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185259.169934_156.1", 2022-09-03T06:09:12.115 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:12.116 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:09:12.116 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:12.117 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.169934Z", 2022-09-03T06:09:12.117 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:12.118 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:12.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:12.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:12.119 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:09:12.120 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:12.120 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:12.121 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:09:12.121 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:12.122 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:09:12.122 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:12.123 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:09:12.123 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:12.124 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:12.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:12.125 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:12.125 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:09:12.126 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:12.126 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:12.127 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:09:12.127 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:12.128 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:09:12.128 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:12.129 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:09:12.129 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:12.130 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:12.130 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:12.131 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:09:12.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:12.132 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:09:12.132 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:09:12.133 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:09:12.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:12.134 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:09:12Z" 2022-09-03T06:09:12.134 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:09:12.137 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185259.169934_156.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:07:39.169934Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:09:12Z" } 2022-09-03T06:09:12.138 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185259.169934_156.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:07:39.169934Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:09:12.139 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185351.819576_158.1', 2: '', 3: ''} 2022-09-03T06:09:12.139 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185259.169934_156.1', 2: '', 3: ''} 2022-09-03T06:09:12.140 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662185351.819576_158.1 target=1_1662185259.169934_156.1 2022-09-03T06:09:17.142 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:17.143 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:17.144 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:09:17.202 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:17.202 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:17.246 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:17.244+0000 7f81dd588600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:09:17.246 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:17.245+0000 7f81dd588600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:09:17.267 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:09:17.268 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:09:17.268 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:09:17.268 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:09:17.269 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:09:17.269 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:09:17.269 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.270 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:09:17.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:17.270 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:09:17.271 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:17.271 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:17.271 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:09:17.271 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:17.272 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:09:17.272 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:17.272 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:09:17.273 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.273 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:17.276 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:09:17.276 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:17.277 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:17.277 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185351.819576_158.1", 2022-09-03T06:09:17.277 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:17.277 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:09:17.278 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:17.278 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:09:11.819576Z", 2022-09-03T06:09:17.278 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.279 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:17.279 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:09:17.280 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:17.280 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:17.280 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:09:17.281 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:17.281 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:09:17.281 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:17.282 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:09:17.282 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.282 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:17.283 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:09:17.283 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:17.283 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:17.284 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:09:17.284 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:17.284 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:09:17.284 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:17.285 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:09:17.285 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.286 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.286 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.286 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:09:17.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.287 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:09:17.287 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:09:17.287 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:09:17.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.288 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:09:17Z" 2022-09-03T06:09:17.288 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:09:17.289 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185351.819576_158.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:09:11.819576Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:09:17Z" } 2022-09-03T06:09:17.290 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185351.819576_158.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:09:11.819576Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:09:17.290 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185351.819576_158.1', 2: '', 3: ''} 2022-09-03T06:09:17.291 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185351.819576_158.1', 2: '', 3: ''} 2022-09-03T06:09:17.291 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:09:17.292 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:09:17.292 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:17.293 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:17.293 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:09:17.354 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:17.355 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:17.385 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:17.383+0000 7f8bbf1db600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:09:17.386 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:17.385+0000 7f8bbf1db600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:09:17.403 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:09:17.404 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:09:17.405 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:09:17.405 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:09:17.406 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:09:17.406 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:09:17.407 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:09:17.407 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:09:17.408 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:09:17.408 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:09:17.410 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:09:17.410 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:09:17.410 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:09:17.410 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:09:17.411 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:09:17.411 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:09:17.411 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:09:17.412 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:09:17.412 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:09:17.412 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:09:17.412 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:09:17.413 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:09:17.413 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:09:17.413 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:09:17.413 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:09:17.414 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185351.819576_158.1", 2022-09-03T06:09:17.414 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:09:17.414 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:09:17.414 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:09:17.415 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:09:11.819576Z", 2022-09-03T06:09:17.415 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:09:17.415 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:09:17.416 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:09:17.416 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:09:17.416 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:09:17.416 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:09:17.418 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:09:17.418 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:09:17.418 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:09:17.418 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:09:17.419 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:09:17.419 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:09:17.419 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:09:17.420 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:09:17.420 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:09:17.420 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:09:17.420 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:09:17.420 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:09:17.421 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:09:17.421 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:09:17.421 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:09:17.421 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:09:17.422 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:09:17.422 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:09:17.423 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:09:17.423 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:09:17.423 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:09:17.423 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:09:17.424 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:09:17.424 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:09:17.424 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:09:17.424 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:09:17.425 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:09:17.425 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:09:17Z" 2022-09-03T06:09:17.425 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:09:17.427 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185351.819576_158.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:09:11.819576Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:09:17Z" } 2022-09-03T06:09:17.428 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185351.819576_158.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:09:11.819576Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:09:17.429 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185351.819576_158.1', 2: '', 3: ''} 2022-09-03T06:09:17.429 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185351.819576_158.1', 2: '', 3: ''} 2022-09-03T06:09:17.430 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:09:17.430 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:09:17.431 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:17.432 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:09:17.433 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:09:17.486 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:17.487 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:17.521 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:17.519+0000 7f3193d80600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:09:17.522 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:17.520+0000 7f3193d80600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:09:17.533 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:09:17.534 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:09:17.534 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:09:17.534 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:09:17.535 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:09:17.535 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:09:17.535 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.535 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.536 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:09:17.536 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:17.537 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:09:17.537 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:17.537 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:17.537 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:09:17.538 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:17.538 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:09:17.538 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:17.539 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:09:17.539 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.539 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:17.541 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:09:17.541 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:17.541 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:17.542 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185351.819576_158.1", 2022-09-03T06:09:17.542 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:17.542 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:09:17.543 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:17.543 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:09:11.819576Z", 2022-09-03T06:09:17.543 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.543 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:17.544 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:09:17.544 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:17.544 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:17.545 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:09:17.545 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:17.545 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:09:17.545 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:17.546 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:09:17.546 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.546 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:09:17.547 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:09:17.547 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:09:17.547 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:09:17.548 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:09:17.548 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:09:17.548 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:09:17.548 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:09:17.549 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:09:17.549 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:09:17.549 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.550 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:09:17.550 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:09:17.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.550 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:09:17.551 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:09:17.551 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:09:17.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:09:17.551 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:09:17Z" 2022-09-03T06:09:17.552 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:09:17.553 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185351.819576_158.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:09:11.819576Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:09:17Z" } 2022-09-03T06:09:17.554 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185351.819576_158.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:09:11.819576Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:09:17.554 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185351.819576_158.1', 2: '', 3: ''} 2022-09-03T06:09:17.555 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185351.819576_158.1', 2: '', 3: ''} 2022-09-03T06:09:17.555 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:09:17.556 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:09:17.556 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:09:17.557 DEBUG:boto:Method: HEAD 2022-09-03T06:09:17.557 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:09:17.558 DEBUG:boto:Data: 2022-09-03T06:09:17.558 DEBUG:boto:Headers: {} 2022-09-03T06:09:17.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:17.559 DEBUG:boto:Port: 8000 2022-09-03T06:09:17.560 DEBUG:boto:Params: {} 2022-09-03T06:09:17.560 DEBUG:boto:Token: None 2022-09-03T06:09:17.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:09:17 GMT /avysmw-20/ 2022-09-03T06:09:17.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8zLfoTkY+SXxHM+4bVEOnIhLlOM= 2022-09-03T06:09:17.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:09:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8zLfoTkY+SXxHM+4bVEOnIhLlOM=', 'Content-Length': '0'} 2022-09-03T06:09:17.565 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007bfb7e5f085eebe0-006312ef8d-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:17.566 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:09:17.567 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:09:17.569 DEBUG:boto:Method: PUT 2022-09-03T06:09:17.569 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:09:17.570 DEBUG:boto:Data: 2022-09-03T06:09:17.571 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:09:17.572 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:09:17.573 DEBUG:boto:Port: 8000 2022-09-03T06:09:17.574 DEBUG:boto:Params: {} 2022-09-03T06:09:17.575 DEBUG:boto:Token: None 2022-09-03T06:09:17.576 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:09:17 GMT /avysmw-20/obj 2022-09-03T06:09:17.577 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g6pRRvccw63AlnRB+HdLcgsTXI0= 2022-09-03T06:09:17.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:09:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g6pRRvccw63AlnRB+HdLcgsTXI0='} 2022-09-03T06:09:17.595 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-amz-request-id', 'tx00000c714a25a9b0e9a5a-006312ef8d-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:09:17.595 DEBUG:rgw_multi.tests:created new version id=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:09:17.596 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-20 2022-09-03T06:09:17.596 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:17.597 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:17.598 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:09:17.655 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:17.655 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:17.700 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:17.697+0000 7f90ad8fb600 1 waiting for incremental sync to catch up: 2022-09-03T06:09:17.700 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:09:17.701 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:22.701 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:22.699+0000 7f90ad8fb600 1 waiting for incremental sync to catch up: 2022-09-03T06:09:22.701 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:09:22.702 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:27.703 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:27.700+0000 7f90ad8fb600 1 waiting for incremental sync to catch up: 2022-09-03T06:09:27.703 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:09:27.704 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:32.704 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:32.702+0000 7f90ad8fb600 1 bucket sync caught up with source: 2022-09-03T06:09:32.705 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:32.705 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:32.708 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:32.702+0000 7f90ad8fb600 0 bucket checkpoint complete 2022-09-03T06:09:32.721 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-20 2022-09-03T06:09:32.722 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:32.723 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:32.724 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:09:32.790 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:32.792 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:32.848 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:32.845+0000 7f702212b600 1 waiting for incremental sync to catch up: 2022-09-03T06:09:32.848 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:09:32.849 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:37.849 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:37.846+0000 7f702212b600 1 waiting for incremental sync to catch up: 2022-09-03T06:09:37.850 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:09:37.850 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:42.850 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:42.848+0000 7f702212b600 1 waiting for incremental sync to catch up: 2022-09-03T06:09:42.851 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:09:42.852 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:47.853 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:47.849+0000 7f702212b600 1 waiting for incremental sync to catch up: 2022-09-03T06:09:47.854 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.10.5, , ] 2022-09-03T06:09:47.854 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:52.854 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:52.852+0000 7f702212b600 1 bucket sync caught up with source: 2022-09-03T06:09:52.854 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:52.854 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:52.855 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:52.852+0000 7f702212b600 0 bucket checkpoint complete 2022-09-03T06:09:52.876 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-20 2022-09-03T06:09:52.877 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:52.878 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:52.879 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:09:52.942 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:52.943 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:52.998 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:52.996+0000 7f13d5e1f600 1 bucket sync caught up with source: 2022-09-03T06:09:52.999 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:52.999 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:09:53.000 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:09:52.996+0000 7f13d5e1f600 0 bucket checkpoint complete 2022-09-03T06:09:53.019 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-20 2022-09-03T06:09:53.020 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:53.021 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:53.022 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:09:53.087 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:53.087 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:53.134 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:53.133+0000 7f55c6123600 1 bucket sync caught up with source: 2022-09-03T06:09:53.135 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.25.12, , ] 2022-09-03T06:09:53.135 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.25.12, , ] 2022-09-03T06:09:53.136 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:53.133+0000 7f55c6123600 0 bucket checkpoint complete 2022-09-03T06:09:53.148 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-20 2022-09-03T06:09:53.149 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:53.149 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:53.150 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:09:53.208 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:53.209 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:53.246 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:53.244+0000 7f6afc7db600 1 bucket sync caught up with source: 2022-09-03T06:09:53.246 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.25.12, , ] 2022-09-03T06:09:53.246 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.25.12, , ] 2022-09-03T06:09:53.247 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:53.244+0000 7f6afc7db600 0 bucket checkpoint complete 2022-09-03T06:09:53.262 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-20 2022-09-03T06:09:53.263 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-20 2022-09-03T06:09:53.264 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:53.265 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:09:53.266 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:09:53.326 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:09:53.327 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:09:53.374 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:53.373+0000 7fbd71561600 1 waiting for incremental sync to catch up: 2022-09-03T06:09:53.375 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.25.5, , ] 2022-09-03T06:09:53.375 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000004.31.9, , ] 2022-09-03T06:09:58.376 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:09:58.374+0000 7fbd71561600 1 waiting for incremental sync to catch up: 2022-09-03T06:09:58.377 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.25.5, , ] 2022-09-03T06:09:58.378 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000004.31.9, , ] 2022-09-03T06:10:03.378 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:03.376+0000 7fbd71561600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:03.379 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.25.5, , ] 2022-09-03T06:10:03.380 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000004.31.9, , ] 2022-09-03T06:10:08.380 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:08.378+0000 7fbd71561600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:08.381 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.25.5, , ] 2022-09-03T06:10:08.381 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000004.31.9, , ] 2022-09-03T06:10:13.381 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:13.379+0000 7fbd71561600 1 bucket sync caught up with source: 2022-09-03T06:10:13.382 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000004.31.9, , ] 2022-09-03T06:10:13.382 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000004.31.9, , ] 2022-09-03T06:10:13.383 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:13.379+0000 7fbd71561600 0 bucket checkpoint complete 2022-09-03T06:10:13.399 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-20 2022-09-03T06:10:13.400 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:10:13.401 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:10:13.402 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:10:13.466 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:10:13.467 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:10:13.505 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:13.503+0000 7f89bf187600 1 bucket sync caught up with source: 2022-09-03T06:10:13.505 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000004.31.9, , ] 2022-09-03T06:10:13.505 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000004.31.9, , ] 2022-09-03T06:10:13.506 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:13.503+0000 7f89bf187600 0 bucket checkpoint complete 2022-09-03T06:10:13.519 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-20 2022-09-03T06:10:13.520 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-20 zones={test-zone2, test-zone1} 2022-09-03T06:10:13.520 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.521 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.522 DEBUG:boto:Method: HEAD 2022-09-03T06:10:13.523 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:10:13.524 DEBUG:boto:Data: 2022-09-03T06:10:13.525 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.526 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.527 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.528 DEBUG:boto:Params: {} 2022-09-03T06:10:13.529 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:10:13.530 DEBUG:boto:Token: None 2022-09-03T06:10:13.530 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/ 2022-09-03T06:10:13.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bMinf1MDUW6W0QtKjV7JXY4oIzM= 2022-09-03T06:10:13.532 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bMinf1MDUW6W0QtKjV7JXY4oIzM=', 'Content-Length': '0'} 2022-09-03T06:10:13.536 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000027b1f9100be5daa3-006312efc5-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.537 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.537 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.538 DEBUG:boto:Method: HEAD 2022-09-03T06:10:13.538 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:10:13.539 DEBUG:boto:Data: 2022-09-03T06:10:13.539 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.540 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.541 DEBUG:boto:Params: {} 2022-09-03T06:10:13.541 DEBUG:boto:Token: None 2022-09-03T06:10:13.542 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/ 2022-09-03T06:10:13.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bMinf1MDUW6W0QtKjV7JXY4oIzM= 2022-09-03T06:10:13.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bMinf1MDUW6W0QtKjV7JXY4oIzM=', 'Content-Length': '0'} 2022-09-03T06:10:13.546 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000417a71bc15995e1a-006312efc5-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.547 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:10:13.547 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.548 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.549 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.550 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.551 DEBUG:boto:Method: GET 2022-09-03T06:10:13.552 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.553 DEBUG:boto:Data: 2022-09-03T06:10:13.554 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.555 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.556 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.557 DEBUG:boto:Params: {} 2022-09-03T06:10:13.558 DEBUG:boto:Token: None 2022-09-03T06:10:13.559 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/?versions 2022-09-03T06:10:13.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZIJyUU8qo6v0k/MZhvV0gt2LKkQ= 2022-09-03T06:10:13.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZIJyUU8qo6v0k/MZhvV0gt2LKkQ=', 'Content-Length': '0'} 2022-09-03T06:10:13.563 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d6c2afc7b445460-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.564 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:13.565 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:13.565 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:13.566 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:10:13.566 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.567 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.567 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.568 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.568 DEBUG:boto:Method: GET 2022-09-03T06:10:13.569 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.569 DEBUG:boto:Data: 2022-09-03T06:10:13.570 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.571 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.571 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.572 DEBUG:boto:Params: {} 2022-09-03T06:10:13.572 DEBUG:boto:Token: None 2022-09-03T06:10:13.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/?versions 2022-09-03T06:10:13.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZIJyUU8qo6v0k/MZhvV0gt2LKkQ= 2022-09-03T06:10:13.574 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZIJyUU8qo6v0k/MZhvV0gt2LKkQ=', 'Content-Length': '0'} 2022-09-03T06:10:13.577 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a70452827d7b59ba-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.618 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:13.619 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:13.620 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:13.620 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.621 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.622 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.622 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.623 DEBUG:boto:Method: GET 2022-09-03T06:10:13.624 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.625 DEBUG:boto:Data: 2022-09-03T06:10:13.626 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.627 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.628 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.629 DEBUG:boto:Params: {} 2022-09-03T06:10:13.630 DEBUG:boto:Token: None 2022-09-03T06:10:13.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/?versions 2022-09-03T06:10:13.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZIJyUU8qo6v0k/MZhvV0gt2LKkQ= 2022-09-03T06:10:13.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZIJyUU8qo6v0k/MZhvV0gt2LKkQ=', 'Content-Length': '0'} 2022-09-03T06:10:13.636 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c8e6ac401cd772f-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.637 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:13.638 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.639 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.640 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.641 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.642 DEBUG:boto:Method: GET 2022-09-03T06:10:13.643 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:13.644 DEBUG:boto:Data: 2022-09-03T06:10:13.645 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.646 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.647 DEBUG:boto:Params: {} 2022-09-03T06:10:13.648 DEBUG:boto:Token: None 2022-09-03T06:10:13.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/?versions 2022-09-03T06:10:13.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZIJyUU8qo6v0k/MZhvV0gt2LKkQ= 2022-09-03T06:10:13.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZIJyUU8qo6v0k/MZhvV0gt2LKkQ=', 'Content-Length': '0'} 2022-09-03T06:10:13.653 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017c180bb62cf82b7-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.654 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:13.655 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:13.658 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.658 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.659 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.660 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.660 DEBUG:boto:Method: GET 2022-09-03T06:10:13.661 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.661 DEBUG:boto:Data: 2022-09-03T06:10:13.662 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.662 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.663 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.663 DEBUG:boto:Params: {} 2022-09-03T06:10:13.664 DEBUG:boto:Token: None 2022-09-03T06:10:13.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk= 2022-09-03T06:10:13.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk=', 'Content-Length': '0'} 2022-09-03T06:10:13.668 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000adb9cb1f4aa894fe-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.669 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.670 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.670 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.671 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.671 DEBUG:boto:Method: GET 2022-09-03T06:10:13.672 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.672 DEBUG:boto:Data: 2022-09-03T06:10:13.673 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.674 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.674 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.675 DEBUG:boto:Params: {} 2022-09-03T06:10:13.675 DEBUG:boto:Token: None 2022-09-03T06:10:13.676 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.677 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk= 2022-09-03T06:10:13.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk=', 'Content-Length': '0'} 2022-09-03T06:10:13.679 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ed0ff52231110720-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.680 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.681 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.681 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.682 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.682 DEBUG:boto:Method: HEAD 2022-09-03T06:10:13.683 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.683 DEBUG:boto:Data: 2022-09-03T06:10:13.684 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.684 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.685 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.685 DEBUG:boto:Params: {} 2022-09-03T06:10:13.686 DEBUG:boto:Token: None 2022-09-03T06:10:13.687 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:53o/6s+BNNp/WcZ7qkgPPvn3psM= 2022-09-03T06:10:13.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:53o/6s+BNNp/WcZ7qkgPPvn3psM=', 'Content-Length': '0'} 2022-09-03T06:10:13.690 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f09946c2fb88064b-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.691 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.691 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.692 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.692 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.693 DEBUG:boto:Method: HEAD 2022-09-03T06:10:13.693 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.694 DEBUG:boto:Data: 2022-09-03T06:10:13.695 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.696 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.696 DEBUG:boto:Params: {} 2022-09-03T06:10:13.697 DEBUG:boto:Token: None 2022-09-03T06:10:13.698 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:53o/6s+BNNp/WcZ7qkgPPvn3psM= 2022-09-03T06:10:13.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:53o/6s+BNNp/WcZ7qkgPPvn3psM=', 'Content-Length': '0'} 2022-09-03T06:10:13.701 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e6bbb3dcbf4f920b-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.702 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:13.703 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.703 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.704 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.705 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.705 DEBUG:boto:Method: GET 2022-09-03T06:10:13.706 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.706 DEBUG:boto:Data: 2022-09-03T06:10:13.707 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.708 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.708 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.709 DEBUG:boto:Params: {} 2022-09-03T06:10:13.709 DEBUG:boto:Token: None 2022-09-03T06:10:13.710 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk= 2022-09-03T06:10:13.712 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk=', 'Content-Length': '0'} 2022-09-03T06:10:13.714 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000cc6785b30f5aa0e5-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.715 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.715 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.716 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.716 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.717 DEBUG:boto:Method: GET 2022-09-03T06:10:13.717 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.718 DEBUG:boto:Data: 2022-09-03T06:10:13.719 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.720 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.721 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.721 DEBUG:boto:Params: {} 2022-09-03T06:10:13.722 DEBUG:boto:Token: None 2022-09-03T06:10:13.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk= 2022-09-03T06:10:13.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk=', 'Content-Length': '0'} 2022-09-03T06:10:13.726 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008f7f2d2d0ee374bd-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.727 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.728 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.729 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.729 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.730 DEBUG:boto:Method: GET 2022-09-03T06:10:13.730 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:13.731 DEBUG:boto:Data: 2022-09-03T06:10:13.732 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.732 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.733 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.733 DEBUG:boto:Params: {} 2022-09-03T06:10:13.734 DEBUG:boto:Token: None 2022-09-03T06:10:13.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/ 2022-09-03T06:10:13.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C6Sw6TL5XYk5GVO3Dlah4B1JraQ= 2022-09-03T06:10:13.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C6Sw6TL5XYk5GVO3Dlah4B1JraQ=', 'Content-Length': '0'} 2022-09-03T06:10:13.739 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5d9989229fc7db1-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.740 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:13.741 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.741 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.742 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.742 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.743 DEBUG:boto:Method: GET 2022-09-03T06:10:13.743 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:13.744 DEBUG:boto:Data: 2022-09-03T06:10:13.744 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.745 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.746 DEBUG:boto:Params: {} 2022-09-03T06:10:13.747 DEBUG:boto:Token: None 2022-09-03T06:10:13.747 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/ 2022-09-03T06:10:13.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C6Sw6TL5XYk5GVO3Dlah4B1JraQ= 2022-09-03T06:10:13.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C6Sw6TL5XYk5GVO3Dlah4B1JraQ=', 'Content-Length': '0'} 2022-09-03T06:10:13.759 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac7e6b5a0d7cd423-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.761 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:13.762 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.763 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.764 DEBUG:boto:Method: HEAD 2022-09-03T06:10:13.766 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:13.767 DEBUG:boto:Data: 2022-09-03T06:10:13.768 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.769 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.770 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.771 DEBUG:boto:Params: {} 2022-09-03T06:10:13.772 DEBUG:boto:Token: None 2022-09-03T06:10:13.773 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj 2022-09-03T06:10:13.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9pTTGjRALnakqHgo3Fwt5tUvQvw= 2022-09-03T06:10:13.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9pTTGjRALnakqHgo3Fwt5tUvQvw=', 'Content-Length': '0'} 2022-09-03T06:10:13.778 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f0b92fce5902463f-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.779 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.780 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.781 DEBUG:boto:Method: HEAD 2022-09-03T06:10:13.783 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:13.784 DEBUG:boto:Data: 2022-09-03T06:10:13.785 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.786 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.787 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.788 DEBUG:boto:Params: {} 2022-09-03T06:10:13.789 DEBUG:boto:Token: None 2022-09-03T06:10:13.790 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj 2022-09-03T06:10:13.791 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9pTTGjRALnakqHgo3Fwt5tUvQvw= 2022-09-03T06:10:13.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9pTTGjRALnakqHgo3Fwt5tUvQvw=', 'Content-Length': '0'} 2022-09-03T06:10:13.794 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000037a7fe6f0dc31578-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.796 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:13.797 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.799 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.800 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.801 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.802 DEBUG:boto:Method: GET 2022-09-03T06:10:13.803 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.804 DEBUG:boto:Data: 2022-09-03T06:10:13.805 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.807 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.808 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.810 DEBUG:boto:Params: {} 2022-09-03T06:10:13.811 DEBUG:boto:Token: None 2022-09-03T06:10:13.812 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk= 2022-09-03T06:10:13.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk=', 'Content-Length': '0'} 2022-09-03T06:10:13.817 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000570a329660d702e1-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.819 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.821 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.822 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.823 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.824 DEBUG:boto:Method: GET 2022-09-03T06:10:13.825 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.825 DEBUG:boto:Data: 2022-09-03T06:10:13.826 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.827 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.827 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.828 DEBUG:boto:Params: {} 2022-09-03T06:10:13.828 DEBUG:boto:Token: None 2022-09-03T06:10:13.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:13.830 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk= 2022-09-03T06:10:13.831 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uGIq8P8g84//5bAst2lhlmXUFKk=', 'Content-Length': '0'} 2022-09-03T06:10:13.834 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e5ad7fe3ac9c1ffc-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.836 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.837 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.838 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.839 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.840 DEBUG:boto:Method: GET 2022-09-03T06:10:13.841 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:13.842 DEBUG:boto:Data: 2022-09-03T06:10:13.843 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.844 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.845 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.846 DEBUG:boto:Params: {} 2022-09-03T06:10:13.848 DEBUG:boto:Token: None 2022-09-03T06:10:13.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/ 2022-09-03T06:10:13.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C6Sw6TL5XYk5GVO3Dlah4B1JraQ= 2022-09-03T06:10:13.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C6Sw6TL5XYk5GVO3Dlah4B1JraQ=', 'Content-Length': '0'} 2022-09-03T06:10:13.855 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000e90037be769c66e-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.856 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:13.858 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.859 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.860 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.861 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.862 DEBUG:boto:Method: GET 2022-09-03T06:10:13.863 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:13.864 DEBUG:boto:Data: 2022-09-03T06:10:13.865 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.866 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.867 DEBUG:boto:Params: {} 2022-09-03T06:10:13.868 DEBUG:boto:Token: None 2022-09-03T06:10:13.868 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/ 2022-09-03T06:10:13.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C6Sw6TL5XYk5GVO3Dlah4B1JraQ= 2022-09-03T06:10:13.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C6Sw6TL5XYk5GVO3Dlah4B1JraQ=', 'Content-Length': '0'} 2022-09-03T06:10:13.872 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d15d3aae39fd6789-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.918 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:13.919 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:13.920 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.921 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.921 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.922 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.922 DEBUG:boto:Method: GET 2022-09-03T06:10:13.923 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.923 DEBUG:boto:Data: 2022-09-03T06:10:13.924 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.924 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.925 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.925 DEBUG:boto:Params: {} 2022-09-03T06:10:13.926 DEBUG:boto:Token: None 2022-09-03T06:10:13.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2f7KcvYxKsigcTCmNVa9miqOBU= 2022-09-03T06:10:13.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2f7KcvYxKsigcTCmNVa9miqOBU=', 'Content-Length': '0'} 2022-09-03T06:10:13.931 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000063ba5c433116a2e7-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.932 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.933 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.933 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.934 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.934 DEBUG:boto:Method: GET 2022-09-03T06:10:13.935 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.936 DEBUG:boto:Data: 2022-09-03T06:10:13.938 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.941 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.942 DEBUG:boto:Params: {} 2022-09-03T06:10:13.944 DEBUG:boto:Token: None 2022-09-03T06:10:13.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2f7KcvYxKsigcTCmNVa9miqOBU= 2022-09-03T06:10:13.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2f7KcvYxKsigcTCmNVa9miqOBU=', 'Content-Length': '0'} 2022-09-03T06:10:13.950 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b34e5f5aef75e66a-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.951 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.952 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.952 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.953 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.953 DEBUG:boto:Method: HEAD 2022-09-03T06:10:13.954 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.954 DEBUG:boto:Data: 2022-09-03T06:10:13.955 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.955 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.956 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.956 DEBUG:boto:Params: {} 2022-09-03T06:10:13.957 DEBUG:boto:Token: None 2022-09-03T06:10:13.957 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8z9ytIaON8Uif+ROnjMe7IK+NBw= 2022-09-03T06:10:13.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8z9ytIaON8Uif+ROnjMe7IK+NBw=', 'Content-Length': '0'} 2022-09-03T06:10:13.961 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000db08acd940f8b488-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.962 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.963 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.963 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.964 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.965 DEBUG:boto:Method: HEAD 2022-09-03T06:10:13.965 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.966 DEBUG:boto:Data: 2022-09-03T06:10:13.966 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.967 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.968 DEBUG:boto:Params: {} 2022-09-03T06:10:13.969 DEBUG:boto:Token: None 2022-09-03T06:10:13.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8z9ytIaON8Uif+ROnjMe7IK+NBw= 2022-09-03T06:10:13.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8z9ytIaON8Uif+ROnjMe7IK+NBw=', 'Content-Length': '0'} 2022-09-03T06:10:13.972 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000018a470564505176e-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.973 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:13.974 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.974 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.975 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.975 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.976 DEBUG:boto:Method: GET 2022-09-03T06:10:13.976 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.977 DEBUG:boto:Data: 2022-09-03T06:10:13.978 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.978 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:13.979 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.979 DEBUG:boto:Params: {} 2022-09-03T06:10:13.980 DEBUG:boto:Token: None 2022-09-03T06:10:13.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2f7KcvYxKsigcTCmNVa9miqOBU= 2022-09-03T06:10:13.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2f7KcvYxKsigcTCmNVa9miqOBU=', 'Content-Length': '0'} 2022-09-03T06:10:13.984 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009afd12f7cddece35-006312efc5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.985 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:13.985 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:13.986 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.986 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:13.987 DEBUG:boto:Method: GET 2022-09-03T06:10:13.987 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.988 DEBUG:boto:Data: 2022-09-03T06:10:13.988 DEBUG:boto:Headers: {} 2022-09-03T06:10:13.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:13.989 DEBUG:boto:Port: 8000 2022-09-03T06:10:13.990 DEBUG:boto:Params: {} 2022-09-03T06:10:13.992 DEBUG:boto:Token: None 2022-09-03T06:10:13.992 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:13 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:13.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2f7KcvYxKsigcTCmNVa9miqOBU= 2022-09-03T06:10:13.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2f7KcvYxKsigcTCmNVa9miqOBU=', 'Content-Length': '0'} 2022-09-03T06:10:13.996 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000012852704fd752b17-006312efc5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:13.997 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:13.997 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:13.998 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.998 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:13.999 DEBUG:boto:Method: GET 2022-09-03T06:10:14.000 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.000 DEBUG:boto:Data: 2022-09-03T06:10:14.001 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.002 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.002 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.003 DEBUG:boto:Params: {} 2022-09-03T06:10:14.003 DEBUG:boto:Token: None 2022-09-03T06:10:14.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.008 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0288f2a0cfe919c-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.009 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.009 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.010 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.011 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.011 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.012 DEBUG:boto:Method: GET 2022-09-03T06:10:14.013 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.013 DEBUG:boto:Data: 2022-09-03T06:10:14.014 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.015 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.016 DEBUG:boto:Params: {} 2022-09-03T06:10:14.016 DEBUG:boto:Token: None 2022-09-03T06:10:14.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.022 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a26df489b7edc00b-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.023 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.024 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.025 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.025 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.026 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:14.026 DEBUG:boto:Data: 2022-09-03T06:10:14.027 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.027 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.028 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.028 DEBUG:boto:Params: {} 2022-09-03T06:10:14.029 DEBUG:boto:Token: None 2022-09-03T06:10:14.029 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj 2022-09-03T06:10:14.030 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8= 2022-09-03T06:10:14.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8=', 'Content-Length': '0'} 2022-09-03T06:10:14.033 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b44b8f62c839e1be-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.034 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.035 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.036 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.037 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:14.038 DEBUG:boto:Data: 2022-09-03T06:10:14.039 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.040 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.041 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.042 DEBUG:boto:Params: {} 2022-09-03T06:10:14.043 DEBUG:boto:Token: None 2022-09-03T06:10:14.044 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj 2022-09-03T06:10:14.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8= 2022-09-03T06:10:14.046 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8=', 'Content-Length': '0'} 2022-09-03T06:10:14.048 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000001ef05cf5f10de77-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.050 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.051 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.052 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.053 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.054 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.055 DEBUG:boto:Method: GET 2022-09-03T06:10:14.056 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.057 DEBUG:boto:Data: 2022-09-03T06:10:14.058 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.059 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.060 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.061 DEBUG:boto:Params: {} 2022-09-03T06:10:14.061 DEBUG:boto:Token: None 2022-09-03T06:10:14.062 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.066 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e26f8959878a3ac5-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.067 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.067 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.068 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.069 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.069 DEBUG:boto:Method: GET 2022-09-03T06:10:14.070 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.070 DEBUG:boto:Data: 2022-09-03T06:10:14.071 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.072 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.072 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.073 DEBUG:boto:Params: {} 2022-09-03T06:10:14.074 DEBUG:boto:Token: None 2022-09-03T06:10:14.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.078 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a213df1b84ec43e3-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.080 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.080 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.081 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.081 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.082 DEBUG:boto:Method: GET 2022-09-03T06:10:14.082 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.083 DEBUG:boto:Data: 2022-09-03T06:10:14.083 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.084 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.085 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.085 DEBUG:boto:Params: {} 2022-09-03T06:10:14.086 DEBUG:boto:Token: None 2022-09-03T06:10:14.087 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.087 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.091 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001e4d1c4eb7c6fc91-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.092 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.093 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.093 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.094 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.095 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.095 DEBUG:boto:Method: GET 2022-09-03T06:10:14.096 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.096 DEBUG:boto:Data: 2022-09-03T06:10:14.097 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.097 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.098 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.099 DEBUG:boto:Params: {} 2022-09-03T06:10:14.099 DEBUG:boto:Token: None 2022-09-03T06:10:14.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.104 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000579ce7e41e06145e-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.146 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.148 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-20 zones={test-zone2, test-zone1} 2022-09-03T06:10:14.149 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-20 zones={test-zone3, test-zone1} 2022-09-03T06:10:14.150 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.150 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.151 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.152 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:10:14.153 DEBUG:boto:Data: 2022-09-03T06:10:14.154 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.156 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.156 DEBUG:boto:Params: {} 2022-09-03T06:10:14.157 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:10:14.158 DEBUG:boto:Token: None 2022-09-03T06:10:14.159 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/DlLD/biGfVe93b2FaMzICVTkA= 2022-09-03T06:10:14.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/DlLD/biGfVe93b2FaMzICVTkA=', 'Content-Length': '0'} 2022-09-03T06:10:14.164 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000038d4894c2fb29334-006312efc6-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.166 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.167 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.168 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.169 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:10:14.170 DEBUG:boto:Data: 2022-09-03T06:10:14.171 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.172 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.173 DEBUG:boto:Params: {} 2022-09-03T06:10:14.174 DEBUG:boto:Token: None 2022-09-03T06:10:14.175 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/DlLD/biGfVe93b2FaMzICVTkA= 2022-09-03T06:10:14.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/DlLD/biGfVe93b2FaMzICVTkA=', 'Content-Length': '0'} 2022-09-03T06:10:14.180 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006bbf955daed1bc4a-006312efc6-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.181 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:10:14.181 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.182 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.183 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.183 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.184 DEBUG:boto:Method: GET 2022-09-03T06:10:14.184 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.185 DEBUG:boto:Data: 2022-09-03T06:10:14.185 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.186 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.186 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.187 DEBUG:boto:Params: {} 2022-09-03T06:10:14.187 DEBUG:boto:Token: None 2022-09-03T06:10:14.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/?versions 2022-09-03T06:10:14.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE= 2022-09-03T06:10:14.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE=', 'Content-Length': '0'} 2022-09-03T06:10:14.192 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da43b3a43c828b6c-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.192 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.193 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:14.194 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:14.194 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:10:14.195 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.195 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.196 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.196 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.197 DEBUG:boto:Method: GET 2022-09-03T06:10:14.198 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.198 DEBUG:boto:Data: 2022-09-03T06:10:14.199 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.199 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.200 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.200 DEBUG:boto:Params: {} 2022-09-03T06:10:14.201 DEBUG:boto:Token: None 2022-09-03T06:10:14.202 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/?versions 2022-09-03T06:10:14.202 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE= 2022-09-03T06:10:14.203 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE=', 'Content-Length': '0'} 2022-09-03T06:10:14.206 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069105bc9bdb458c1-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.207 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.209 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:14.210 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:14.210 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.211 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.212 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.213 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.214 DEBUG:boto:Method: GET 2022-09-03T06:10:14.215 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.216 DEBUG:boto:Data: 2022-09-03T06:10:14.216 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.217 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.218 DEBUG:boto:Params: {} 2022-09-03T06:10:14.218 DEBUG:boto:Token: None 2022-09-03T06:10:14.219 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/?versions 2022-09-03T06:10:14.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE= 2022-09-03T06:10:14.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE=', 'Content-Length': '0'} 2022-09-03T06:10:14.223 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ef11bda52f30080d-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.223 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.224 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.225 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.225 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.226 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.227 DEBUG:boto:Method: GET 2022-09-03T06:10:14.227 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.228 DEBUG:boto:Data: 2022-09-03T06:10:14.228 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.228 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.229 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.229 DEBUG:boto:Params: {} 2022-09-03T06:10:14.230 DEBUG:boto:Token: None 2022-09-03T06:10:14.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/?versions 2022-09-03T06:10:14.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE= 2022-09-03T06:10:14.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE=', 'Content-Length': '0'} 2022-09-03T06:10:14.234 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000257910f148aae265-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.235 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.236 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.237 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.237 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.238 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.238 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.239 DEBUG:boto:Method: GET 2022-09-03T06:10:14.239 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.240 DEBUG:boto:Data: 2022-09-03T06:10:14.240 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.241 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.241 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.242 DEBUG:boto:Params: {} 2022-09-03T06:10:14.243 DEBUG:boto:Token: None 2022-09-03T06:10:14.243 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.247 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008c35196bb35db6ee-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.247 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.248 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.248 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.249 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.249 DEBUG:boto:Method: GET 2022-09-03T06:10:14.250 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.251 DEBUG:boto:Data: 2022-09-03T06:10:14.251 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.252 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.252 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.253 DEBUG:boto:Params: {} 2022-09-03T06:10:14.253 DEBUG:boto:Token: None 2022-09-03T06:10:14.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.257 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007b1b11ce6ff197b5-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.258 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.259 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.259 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.260 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.260 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.261 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.261 DEBUG:boto:Data: 2022-09-03T06:10:14.262 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.263 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.263 DEBUG:boto:Params: {} 2022-09-03T06:10:14.264 DEBUG:boto:Token: None 2022-09-03T06:10:14.264 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Xjk64tSlWZqrwy7YY9UZaE7Fns= 2022-09-03T06:10:14.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Xjk64tSlWZqrwy7YY9UZaE7Fns=', 'Content-Length': '0'} 2022-09-03T06:10:14.268 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000046347d1b27324969-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.269 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.269 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.270 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.270 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.271 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.271 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.272 DEBUG:boto:Data: 2022-09-03T06:10:14.272 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.273 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.274 DEBUG:boto:Params: {} 2022-09-03T06:10:14.274 DEBUG:boto:Token: None 2022-09-03T06:10:14.275 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Xjk64tSlWZqrwy7YY9UZaE7Fns= 2022-09-03T06:10:14.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Xjk64tSlWZqrwy7YY9UZaE7Fns=', 'Content-Length': '0'} 2022-09-03T06:10:14.278 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f426d68e7308bd43-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.279 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.279 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.280 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.280 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.281 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.281 DEBUG:boto:Method: GET 2022-09-03T06:10:14.282 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.282 DEBUG:boto:Data: 2022-09-03T06:10:14.283 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.283 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.284 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.285 DEBUG:boto:Params: {} 2022-09-03T06:10:14.285 DEBUG:boto:Token: None 2022-09-03T06:10:14.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.289 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009cc10aa69842f59b-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.290 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.290 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.291 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.291 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.292 DEBUG:boto:Method: GET 2022-09-03T06:10:14.293 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.293 DEBUG:boto:Data: 2022-09-03T06:10:14.294 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.294 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.295 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.295 DEBUG:boto:Params: {} 2022-09-03T06:10:14.296 DEBUG:boto:Token: None 2022-09-03T06:10:14.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.300 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a87f021832cb9cf5-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.301 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.302 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.302 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.303 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.303 DEBUG:boto:Method: GET 2022-09-03T06:10:14.304 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.304 DEBUG:boto:Data: 2022-09-03T06:10:14.305 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.306 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.306 DEBUG:boto:Params: {} 2022-09-03T06:10:14.307 DEBUG:boto:Token: None 2022-09-03T06:10:14.307 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.312 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000adc1a9cc53e17a08-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.312 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.313 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.314 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.314 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.315 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.315 DEBUG:boto:Method: GET 2022-09-03T06:10:14.316 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.316 DEBUG:boto:Data: 2022-09-03T06:10:14.317 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.317 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.318 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.318 DEBUG:boto:Params: {} 2022-09-03T06:10:14.319 DEBUG:boto:Token: None 2022-09-03T06:10:14.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.323 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000240fd266053dabc5-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.366 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.367 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.368 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.368 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.369 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:14.369 DEBUG:boto:Data: 2022-09-03T06:10:14.370 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.371 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.371 DEBUG:boto:Params: {} 2022-09-03T06:10:14.372 DEBUG:boto:Token: None 2022-09-03T06:10:14.372 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj 2022-09-03T06:10:14.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8= 2022-09-03T06:10:14.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8=', 'Content-Length': '0'} 2022-09-03T06:10:14.376 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007cafb70ff579e2c3-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.377 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.377 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.378 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.379 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:14.379 DEBUG:boto:Data: 2022-09-03T06:10:14.380 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.380 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.381 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.381 DEBUG:boto:Params: {} 2022-09-03T06:10:14.382 DEBUG:boto:Token: None 2022-09-03T06:10:14.382 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj 2022-09-03T06:10:14.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8= 2022-09-03T06:10:14.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8=', 'Content-Length': '0'} 2022-09-03T06:10:14.386 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000021535e62cb665573-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.387 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.387 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.388 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.388 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.389 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.389 DEBUG:boto:Method: GET 2022-09-03T06:10:14.390 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.391 DEBUG:boto:Data: 2022-09-03T06:10:14.391 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.392 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.393 DEBUG:boto:Params: {} 2022-09-03T06:10:14.393 DEBUG:boto:Token: None 2022-09-03T06:10:14.394 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.395 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.398 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ee38228215e64610-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.399 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.399 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.400 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.400 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.401 DEBUG:boto:Method: GET 2022-09-03T06:10:14.401 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.402 DEBUG:boto:Data: 2022-09-03T06:10:14.402 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.403 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.403 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.404 DEBUG:boto:Params: {} 2022-09-03T06:10:14.405 DEBUG:boto:Token: None 2022-09-03T06:10:14.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.408 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000054e7f457bd170720-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.409 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.410 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.410 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.411 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.411 DEBUG:boto:Method: GET 2022-09-03T06:10:14.412 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.412 DEBUG:boto:Data: 2022-09-03T06:10:14.413 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.413 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.414 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.415 DEBUG:boto:Params: {} 2022-09-03T06:10:14.415 DEBUG:boto:Token: None 2022-09-03T06:10:14.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.419 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007792814488812e31-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.420 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.421 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.422 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.422 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.423 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.423 DEBUG:boto:Method: GET 2022-09-03T06:10:14.424 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.424 DEBUG:boto:Data: 2022-09-03T06:10:14.425 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.425 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.426 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.426 DEBUG:boto:Params: {} 2022-09-03T06:10:14.427 DEBUG:boto:Token: None 2022-09-03T06:10:14.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.431 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f382bd89cacc33ce-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.432 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.433 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.433 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.434 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.435 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.435 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.436 DEBUG:boto:Method: GET 2022-09-03T06:10:14.436 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.437 DEBUG:boto:Data: 2022-09-03T06:10:14.437 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.438 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.439 DEBUG:boto:Params: {} 2022-09-03T06:10:14.439 DEBUG:boto:Token: None 2022-09-03T06:10:14.440 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gA8gvQA/sXcilSuRBVaMfIHlwjk= 2022-09-03T06:10:14.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gA8gvQA/sXcilSuRBVaMfIHlwjk=', 'Content-Length': '0'} 2022-09-03T06:10:14.443 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009e6da62ee7be84a2-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.444 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.445 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.445 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.446 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.446 DEBUG:boto:Method: GET 2022-09-03T06:10:14.447 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.447 DEBUG:boto:Data: 2022-09-03T06:10:14.448 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.449 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.449 DEBUG:boto:Params: {} 2022-09-03T06:10:14.450 DEBUG:boto:Token: None 2022-09-03T06:10:14.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gA8gvQA/sXcilSuRBVaMfIHlwjk= 2022-09-03T06:10:14.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gA8gvQA/sXcilSuRBVaMfIHlwjk=', 'Content-Length': '0'} 2022-09-03T06:10:14.454 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000026c3d9916cf3039e-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.454 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.455 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.455 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.456 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.456 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.457 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.457 DEBUG:boto:Data: 2022-09-03T06:10:14.458 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.459 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.459 DEBUG:boto:Params: {} 2022-09-03T06:10:14.460 DEBUG:boto:Token: None 2022-09-03T06:10:14.461 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OHksESZEc/dcDN6MADM8PI3EGnE= 2022-09-03T06:10:14.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OHksESZEc/dcDN6MADM8PI3EGnE=', 'Content-Length': '0'} 2022-09-03T06:10:14.464 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004c864766555b6c38-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.465 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.465 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.466 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.466 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.467 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.467 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.468 DEBUG:boto:Data: 2022-09-03T06:10:14.468 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.469 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.470 DEBUG:boto:Params: {} 2022-09-03T06:10:14.470 DEBUG:boto:Token: None 2022-09-03T06:10:14.471 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OHksESZEc/dcDN6MADM8PI3EGnE= 2022-09-03T06:10:14.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OHksESZEc/dcDN6MADM8PI3EGnE=', 'Content-Length': '0'} 2022-09-03T06:10:14.474 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000785923dee7213f26-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.475 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.475 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.476 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.476 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.477 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.477 DEBUG:boto:Method: GET 2022-09-03T06:10:14.478 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.478 DEBUG:boto:Data: 2022-09-03T06:10:14.479 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.480 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.480 DEBUG:boto:Params: {} 2022-09-03T06:10:14.481 DEBUG:boto:Token: None 2022-09-03T06:10:14.482 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gA8gvQA/sXcilSuRBVaMfIHlwjk= 2022-09-03T06:10:14.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gA8gvQA/sXcilSuRBVaMfIHlwjk=', 'Content-Length': '0'} 2022-09-03T06:10:14.485 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005b25c778adfff811-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.486 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.486 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.487 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.487 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:14.488 DEBUG:boto:Method: GET 2022-09-03T06:10:14.488 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.489 DEBUG:boto:Data: 2022-09-03T06:10:14.489 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.491 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.491 DEBUG:boto:Params: {} 2022-09-03T06:10:14.492 DEBUG:boto:Token: None 2022-09-03T06:10:14.492 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:14.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gA8gvQA/sXcilSuRBVaMfIHlwjk= 2022-09-03T06:10:14.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gA8gvQA/sXcilSuRBVaMfIHlwjk=', 'Content-Length': '0'} 2022-09-03T06:10:14.496 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009f7dc6bdf7111c0f-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.497 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.497 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.498 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.498 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.499 DEBUG:boto:Method: GET 2022-09-03T06:10:14.499 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.500 DEBUG:boto:Data: 2022-09-03T06:10:14.500 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.501 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.501 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.502 DEBUG:boto:Params: {} 2022-09-03T06:10:14.502 DEBUG:boto:Token: None 2022-09-03T06:10:14.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.507 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed6310545001bb8f-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.550 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.551 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.552 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.552 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.553 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.554 DEBUG:boto:Method: GET 2022-09-03T06:10:14.554 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.555 DEBUG:boto:Data: 2022-09-03T06:10:14.555 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.556 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.557 DEBUG:boto:Params: {} 2022-09-03T06:10:14.557 DEBUG:boto:Token: None 2022-09-03T06:10:14.558 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.559 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.562 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8a9a23def120ea6-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.563 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.564 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.564 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.565 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.565 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:14.566 DEBUG:boto:Data: 2022-09-03T06:10:14.566 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.567 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.568 DEBUG:boto:Params: {} 2022-09-03T06:10:14.568 DEBUG:boto:Token: None 2022-09-03T06:10:14.569 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj 2022-09-03T06:10:14.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8= 2022-09-03T06:10:14.570 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8=', 'Content-Length': '0'} 2022-09-03T06:10:14.573 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000094a7a43a4181cc6f-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.574 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.574 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.575 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.575 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:14.576 DEBUG:boto:Data: 2022-09-03T06:10:14.576 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.577 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.578 DEBUG:boto:Params: {} 2022-09-03T06:10:14.578 DEBUG:boto:Token: None 2022-09-03T06:10:14.579 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj 2022-09-03T06:10:14.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8= 2022-09-03T06:10:14.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8=', 'Content-Length': '0'} 2022-09-03T06:10:14.583 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000fd86bb75bae3684c-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.584 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.585 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.587 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.588 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.588 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.589 DEBUG:boto:Method: GET 2022-09-03T06:10:14.590 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.591 DEBUG:boto:Data: 2022-09-03T06:10:14.592 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.593 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.594 DEBUG:boto:Params: {} 2022-09-03T06:10:14.594 DEBUG:boto:Token: None 2022-09-03T06:10:14.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.599 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b02c85ca3557cc49-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.600 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.601 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.602 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.603 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.604 DEBUG:boto:Method: GET 2022-09-03T06:10:14.605 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.606 DEBUG:boto:Data: 2022-09-03T06:10:14.607 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.609 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.609 DEBUG:boto:Params: {} 2022-09-03T06:10:14.610 DEBUG:boto:Token: None 2022-09-03T06:10:14.611 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.614 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000004b6c1f23a1317eb8-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.615 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.615 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.616 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.616 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.617 DEBUG:boto:Method: GET 2022-09-03T06:10:14.617 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.618 DEBUG:boto:Data: 2022-09-03T06:10:14.619 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.619 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.620 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.620 DEBUG:boto:Params: {} 2022-09-03T06:10:14.621 DEBUG:boto:Token: None 2022-09-03T06:10:14.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.625 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006086a3954ca23352-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.627 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.628 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.628 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.629 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.630 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.630 DEBUG:boto:Method: GET 2022-09-03T06:10:14.631 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.631 DEBUG:boto:Data: 2022-09-03T06:10:14.632 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.632 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:14.633 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.633 DEBUG:boto:Params: {} 2022-09-03T06:10:14.634 DEBUG:boto:Token: None 2022-09-03T06:10:14.634 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.638 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000026e1139c6a196bea-006312efc6-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.639 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.640 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-20 zones={test-zone3, test-zone1} 2022-09-03T06:10:14.641 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-20 zones={test-zone3, test-zone2} 2022-09-03T06:10:14.641 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.642 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.643 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.643 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:10:14.644 DEBUG:boto:Data: 2022-09-03T06:10:14.645 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.647 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.648 DEBUG:boto:Params: {} 2022-09-03T06:10:14.649 DEBUG:boto:Token: None 2022-09-03T06:10:14.650 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/DlLD/biGfVe93b2FaMzICVTkA= 2022-09-03T06:10:14.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/DlLD/biGfVe93b2FaMzICVTkA=', 'Content-Length': '0'} 2022-09-03T06:10:14.655 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000de02997f81501191-006312efc6-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.655 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.656 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.657 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.657 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:10:14.658 DEBUG:boto:Data: 2022-09-03T06:10:14.658 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.659 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.659 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.660 DEBUG:boto:Params: {} 2022-09-03T06:10:14.660 DEBUG:boto:Token: None 2022-09-03T06:10:14.661 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/DlLD/biGfVe93b2FaMzICVTkA= 2022-09-03T06:10:14.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/DlLD/biGfVe93b2FaMzICVTkA=', 'Content-Length': '0'} 2022-09-03T06:10:14.665 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bc59258d8fff536f-006312efc6-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.665 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:10:14.666 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.666 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.667 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.667 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.668 DEBUG:boto:Method: GET 2022-09-03T06:10:14.668 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.669 DEBUG:boto:Data: 2022-09-03T06:10:14.670 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.671 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.671 DEBUG:boto:Params: {} 2022-09-03T06:10:14.672 DEBUG:boto:Token: None 2022-09-03T06:10:14.672 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/?versions 2022-09-03T06:10:14.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE= 2022-09-03T06:10:14.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE=', 'Content-Length': '0'} 2022-09-03T06:10:14.677 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098f917b8af26eeee-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.722 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.724 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:14.724 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:14.725 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:10:14.726 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.726 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.727 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.727 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.728 DEBUG:boto:Method: GET 2022-09-03T06:10:14.728 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.729 DEBUG:boto:Data: 2022-09-03T06:10:14.730 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.730 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.731 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.731 DEBUG:boto:Params: {} 2022-09-03T06:10:14.732 DEBUG:boto:Token: None 2022-09-03T06:10:14.733 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/?versions 2022-09-03T06:10:14.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE= 2022-09-03T06:10:14.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE=', 'Content-Length': '0'} 2022-09-03T06:10:14.738 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000182d9fbcd5ee3e15-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.739 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.740 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:14.741 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:10:14.742 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.742 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.743 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.743 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.744 DEBUG:boto:Method: GET 2022-09-03T06:10:14.744 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.745 DEBUG:boto:Data: 2022-09-03T06:10:14.745 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.746 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.747 DEBUG:boto:Params: {} 2022-09-03T06:10:14.747 DEBUG:boto:Token: None 2022-09-03T06:10:14.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/?versions 2022-09-03T06:10:14.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE= 2022-09-03T06:10:14.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE=', 'Content-Length': '0'} 2022-09-03T06:10:14.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085c8bbd7e427e78f-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.753 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.755 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.756 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.757 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.758 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.759 DEBUG:boto:Method: GET 2022-09-03T06:10:14.760 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:10:14.760 DEBUG:boto:Data: 2022-09-03T06:10:14.761 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.762 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.762 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.763 DEBUG:boto:Params: {} 2022-09-03T06:10:14.764 DEBUG:boto:Token: None 2022-09-03T06:10:14.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/?versions 2022-09-03T06:10:14.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE= 2022-09-03T06:10:14.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:moSPCDXQcUJegzmdno52BwsdHBE=', 'Content-Length': '0'} 2022-09-03T06:10:14.768 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ee87a2fa5baa5b3-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.770 DEBUG:boto:b'avysmw-20999falseobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQtrue2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjnullfalse2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.771 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.772 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.773 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.774 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.775 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.775 DEBUG:boto:Method: GET 2022-09-03T06:10:14.776 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.777 DEBUG:boto:Data: 2022-09-03T06:10:14.778 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.779 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.780 DEBUG:boto:Params: {} 2022-09-03T06:10:14.781 DEBUG:boto:Token: None 2022-09-03T06:10:14.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.786 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000347810009ae59ac2-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.788 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.789 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.790 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.790 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.792 DEBUG:boto:Method: GET 2022-09-03T06:10:14.793 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.793 DEBUG:boto:Data: 2022-09-03T06:10:14.794 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.795 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.795 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.796 DEBUG:boto:Params: {} 2022-09-03T06:10:14.797 DEBUG:boto:Token: None 2022-09-03T06:10:14.797 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.802 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000046848013afd84f10-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.803 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.803 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.804 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.805 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.806 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.807 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.808 DEBUG:boto:Data: 2022-09-03T06:10:14.809 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.810 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.811 DEBUG:boto:Params: {} 2022-09-03T06:10:14.812 DEBUG:boto:Token: None 2022-09-03T06:10:14.813 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Xjk64tSlWZqrwy7YY9UZaE7Fns= 2022-09-03T06:10:14.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Xjk64tSlWZqrwy7YY9UZaE7Fns=', 'Content-Length': '0'} 2022-09-03T06:10:14.818 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000008eb56af2162d1de0-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.819 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.820 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.821 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.822 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.823 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.823 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.824 DEBUG:boto:Data: 2022-09-03T06:10:14.825 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.826 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.826 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.827 DEBUG:boto:Params: {} 2022-09-03T06:10:14.827 DEBUG:boto:Token: None 2022-09-03T06:10:14.828 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Xjk64tSlWZqrwy7YY9UZaE7Fns= 2022-09-03T06:10:14.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Xjk64tSlWZqrwy7YY9UZaE7Fns=', 'Content-Length': '0'} 2022-09-03T06:10:14.831 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007f54f68948b96b19-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.832 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.833 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.833 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.834 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.835 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.836 DEBUG:boto:Method: GET 2022-09-03T06:10:14.837 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.837 DEBUG:boto:Data: 2022-09-03T06:10:14.838 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.839 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.840 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.841 DEBUG:boto:Params: {} 2022-09-03T06:10:14.843 DEBUG:boto:Token: None 2022-09-03T06:10:14.844 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.849 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000084b41016004a3c05-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.850 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.851 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.852 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.853 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.853 DEBUG:boto:Method: GET 2022-09-03T06:10:14.854 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.854 DEBUG:boto:Data: 2022-09-03T06:10:14.855 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.855 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.856 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.857 DEBUG:boto:Params: {} 2022-09-03T06:10:14.858 DEBUG:boto:Token: None 2022-09-03T06:10:14.858 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.862 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000520e9d19329e8efd-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.863 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.863 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.864 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.864 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.865 DEBUG:boto:Method: GET 2022-09-03T06:10:14.865 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.866 DEBUG:boto:Data: 2022-09-03T06:10:14.866 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.867 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.867 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.868 DEBUG:boto:Params: {} 2022-09-03T06:10:14.869 DEBUG:boto:Token: None 2022-09-03T06:10:14.869 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.873 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063733babb6d9b053-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.874 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.875 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.875 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.876 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.876 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.877 DEBUG:boto:Method: GET 2022-09-03T06:10:14.877 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.878 DEBUG:boto:Data: 2022-09-03T06:10:14.878 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.879 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.880 DEBUG:boto:Params: {} 2022-09-03T06:10:14.881 DEBUG:boto:Token: None 2022-09-03T06:10:14.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.882 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.885 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008c2f0be518b1fa87-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.886 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:14.887 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.888 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.889 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.890 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:14.891 DEBUG:boto:Data: 2022-09-03T06:10:14.892 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.893 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.894 DEBUG:boto:Params: {} 2022-09-03T06:10:14.895 DEBUG:boto:Token: None 2022-09-03T06:10:14.895 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj 2022-09-03T06:10:14.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8= 2022-09-03T06:10:14.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8=', 'Content-Length': '0'} 2022-09-03T06:10:14.900 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000007534f6447b55fdc4-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.901 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.902 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.903 DEBUG:boto:Method: HEAD 2022-09-03T06:10:14.904 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:14.905 DEBUG:boto:Data: 2022-09-03T06:10:14.906 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.907 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.907 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.908 DEBUG:boto:Params: {} 2022-09-03T06:10:14.910 DEBUG:boto:Token: None 2022-09-03T06:10:14.911 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj 2022-09-03T06:10:14.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8= 2022-09-03T06:10:14.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8DQX97Dyl6iw8uCRqNUPrPgQg8=', 'Content-Length': '0'} 2022-09-03T06:10:14.916 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000000aa2d8d46807b77c-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.917 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:14.917 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.918 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.918 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.919 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.919 DEBUG:boto:Method: GET 2022-09-03T06:10:14.920 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.920 DEBUG:boto:Data: 2022-09-03T06:10:14.921 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.922 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.923 DEBUG:boto:Params: {} 2022-09-03T06:10:14.924 DEBUG:boto:Token: None 2022-09-03T06:10:14.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.929 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c6b3134892a508c2-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.930 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:14.931 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:14.931 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.932 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.933 DEBUG:boto:Method: GET 2022-09-03T06:10:14.934 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.935 DEBUG:boto:Data: 2022-09-03T06:10:14.936 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.937 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:14.938 DEBUG:boto:Port: 8000 2022-09-03T06:10:14.939 DEBUG:boto:Params: {} 2022-09-03T06:10:14.940 DEBUG:boto:Token: None 2022-09-03T06:10:14.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:14.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o= 2022-09-03T06:10:14.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0xDl//hggzbUITADg9m/tuV04o=', 'Content-Length': '0'} 2022-09-03T06:10:14.949 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000071723210baa2e886-006312efc6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:14.951 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:14.953 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:14.954 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.955 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:14.956 DEBUG:boto:Method: GET 2022-09-03T06:10:14.956 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:14.957 DEBUG:boto:Data: 2022-09-03T06:10:14.959 DEBUG:boto:Headers: {} 2022-09-03T06:10:14.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:14.960 DEBUG:boto:Port: 8001 2022-09-03T06:10:14.961 DEBUG:boto:Params: {} 2022-09-03T06:10:14.962 DEBUG:boto:Token: None 2022-09-03T06:10:14.963 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:14 GMT /avysmw-20/ 2022-09-03T06:10:14.964 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw= 2022-09-03T06:10:14.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPP6G5eJMdG2SgX9YjME/9YJXJw=', 'Content-Length': '0'} 2022-09-03T06:10:14.968 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ddebda7656fb3de0-006312efc6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.011 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:15.013 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:15.013 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:15.014 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.015 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.015 DEBUG:boto:Method: GET 2022-09-03T06:10:15.016 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:15.016 DEBUG:boto:Data: 2022-09-03T06:10:15.017 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.017 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:15.018 DEBUG:boto:Port: 8000 2022-09-03T06:10:15.019 DEBUG:boto:Params: {} 2022-09-03T06:10:15.019 DEBUG:boto:Token: None 2022-09-03T06:10:15.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/ 2022-09-03T06:10:15.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ= 2022-09-03T06:10:15.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ=', 'Content-Length': '0'} 2022-09-03T06:10:15.025 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ba4fb6a9a5869770-006312efc7-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.026 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:15.026 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:15.027 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.028 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.028 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.029 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.029 DEBUG:boto:Method: GET 2022-09-03T06:10:15.030 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.031 DEBUG:boto:Data: 2022-09-03T06:10:15.032 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:15.033 DEBUG:boto:Port: 8001 2022-09-03T06:10:15.033 DEBUG:boto:Params: {} 2022-09-03T06:10:15.035 DEBUG:boto:Token: None 2022-09-03T06:10:15.035 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:udSuQsPdh4efBdk7BmUaFnOOoqE= 2022-09-03T06:10:15.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:udSuQsPdh4efBdk7BmUaFnOOoqE=', 'Content-Length': '0'} 2022-09-03T06:10:15.039 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b11e3aa12ac01867-006312efc7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.040 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.040 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.041 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.041 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.042 DEBUG:boto:Method: GET 2022-09-03T06:10:15.042 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.043 DEBUG:boto:Data: 2022-09-03T06:10:15.043 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.044 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:15.044 DEBUG:boto:Port: 8000 2022-09-03T06:10:15.045 DEBUG:boto:Params: {} 2022-09-03T06:10:15.045 DEBUG:boto:Token: None 2022-09-03T06:10:15.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:udSuQsPdh4efBdk7BmUaFnOOoqE= 2022-09-03T06:10:15.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:udSuQsPdh4efBdk7BmUaFnOOoqE=', 'Content-Length': '0'} 2022-09-03T06:10:15.050 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000dc275becc76e4bd1-006312efc7-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.051 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.052 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.053 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.054 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.055 DEBUG:boto:Method: HEAD 2022-09-03T06:10:15.056 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.057 DEBUG:boto:Data: 2022-09-03T06:10:15.058 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:15.060 DEBUG:boto:Port: 8001 2022-09-03T06:10:15.061 DEBUG:boto:Params: {} 2022-09-03T06:10:15.062 DEBUG:boto:Token: None 2022-09-03T06:10:15.063 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cws7LjZDYXL+SdgJZUOnwP4XXwY= 2022-09-03T06:10:15.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cws7LjZDYXL+SdgJZUOnwP4XXwY=', 'Content-Length': '0'} 2022-09-03T06:10:15.067 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005deaa9953a9deca5-006312efc7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.069 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.070 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.070 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.071 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.072 DEBUG:boto:Method: HEAD 2022-09-03T06:10:15.073 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.074 DEBUG:boto:Data: 2022-09-03T06:10:15.075 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.076 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:15.077 DEBUG:boto:Port: 8000 2022-09-03T06:10:15.078 DEBUG:boto:Params: {} 2022-09-03T06:10:15.079 DEBUG:boto:Token: None 2022-09-03T06:10:15.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cws7LjZDYXL+SdgJZUOnwP4XXwY= 2022-09-03T06:10:15.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cws7LjZDYXL+SdgJZUOnwP4XXwY=', 'Content-Length': '0'} 2022-09-03T06:10:15.085 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000006720025b3078399-006312efc7-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.086 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:15.087 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.088 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.089 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.089 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.090 DEBUG:boto:Method: GET 2022-09-03T06:10:15.091 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.091 DEBUG:boto:Data: 2022-09-03T06:10:15.092 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:15.093 DEBUG:boto:Port: 8001 2022-09-03T06:10:15.093 DEBUG:boto:Params: {} 2022-09-03T06:10:15.094 DEBUG:boto:Token: None 2022-09-03T06:10:15.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:udSuQsPdh4efBdk7BmUaFnOOoqE= 2022-09-03T06:10:15.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:udSuQsPdh4efBdk7BmUaFnOOoqE=', 'Content-Length': '0'} 2022-09-03T06:10:15.098 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000bd5e317a708ecc57-006312efc7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.098 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.099 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.100 DEBUG:boto:path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.100 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=null 2022-09-03T06:10:15.101 DEBUG:boto:Method: GET 2022-09-03T06:10:15.101 DEBUG:boto:Path: /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.102 DEBUG:boto:Data: 2022-09-03T06:10:15.102 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.103 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:15.103 DEBUG:boto:Port: 8000 2022-09-03T06:10:15.104 DEBUG:boto:Params: {} 2022-09-03T06:10:15.104 DEBUG:boto:Token: None 2022-09-03T06:10:15.105 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj?versionId=null 2022-09-03T06:10:15.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:udSuQsPdh4efBdk7BmUaFnOOoqE= 2022-09-03T06:10:15.106 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:udSuQsPdh4efBdk7BmUaFnOOoqE=', 'Content-Length': '0'} 2022-09-03T06:10:15.108 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:07:44 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000009b559529b2154f4-006312efc7-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.109 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:15.110 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:15.110 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.111 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.111 DEBUG:boto:Method: GET 2022-09-03T06:10:15.112 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:15.112 DEBUG:boto:Data: 2022-09-03T06:10:15.113 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.113 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:15.114 DEBUG:boto:Port: 8001 2022-09-03T06:10:15.114 DEBUG:boto:Params: {} 2022-09-03T06:10:15.115 DEBUG:boto:Token: None 2022-09-03T06:10:15.115 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/ 2022-09-03T06:10:15.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ= 2022-09-03T06:10:15.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ=', 'Content-Length': '0'} 2022-09-03T06:10:15.119 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022611cb021c1016f-006312efc7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.120 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:15.121 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:15.121 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:15.122 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.122 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.123 DEBUG:boto:Method: GET 2022-09-03T06:10:15.123 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:15.124 DEBUG:boto:Data: 2022-09-03T06:10:15.124 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.125 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:15.125 DEBUG:boto:Port: 8000 2022-09-03T06:10:15.126 DEBUG:boto:Params: {} 2022-09-03T06:10:15.126 DEBUG:boto:Token: None 2022-09-03T06:10:15.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/ 2022-09-03T06:10:15.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ= 2022-09-03T06:10:15.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ=', 'Content-Length': '0'} 2022-09-03T06:10:15.132 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017534d915ec63a22-006312efc7-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.133 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:15.134 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.134 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.135 DEBUG:boto:Method: HEAD 2022-09-03T06:10:15.135 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:15.136 DEBUG:boto:Data: 2022-09-03T06:10:15.136 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:15.137 DEBUG:boto:Port: 8001 2022-09-03T06:10:15.138 DEBUG:boto:Params: {} 2022-09-03T06:10:15.138 DEBUG:boto:Token: None 2022-09-03T06:10:15.139 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj 2022-09-03T06:10:15.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44c/6dj0Lm4+ofA04f7zNE8zThE= 2022-09-03T06:10:15.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44c/6dj0Lm4+ofA04f7zNE8zThE=', 'Content-Length': '0'} 2022-09-03T06:10:15.143 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000043d76337d40960b9-006312efc7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.143 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.144 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.144 DEBUG:boto:Method: HEAD 2022-09-03T06:10:15.145 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:15.145 DEBUG:boto:Data: 2022-09-03T06:10:15.146 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.146 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:15.147 DEBUG:boto:Port: 8000 2022-09-03T06:10:15.147 DEBUG:boto:Params: {} 2022-09-03T06:10:15.148 DEBUG:boto:Token: None 2022-09-03T06:10:15.148 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj 2022-09-03T06:10:15.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44c/6dj0Lm4+ofA04f7zNE8zThE= 2022-09-03T06:10:15.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44c/6dj0Lm4+ofA04f7zNE8zThE=', 'Content-Length': '0'} 2022-09-03T06:10:15.152 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000072e8b26a4a91d065-006312efc7-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.153 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:10:15.153 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.154 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.154 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:15.155 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:15.155 DEBUG:boto:Method: GET 2022-09-03T06:10:15.156 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:15.156 DEBUG:boto:Data: 2022-09-03T06:10:15.157 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.157 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:15.158 DEBUG:boto:Port: 8001 2022-09-03T06:10:15.159 DEBUG:boto:Params: {} 2022-09-03T06:10:15.159 DEBUG:boto:Token: None 2022-09-03T06:10:15.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:15.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOn3GkcMDuO72aouGlaEeGPb16w= 2022-09-03T06:10:15.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOn3GkcMDuO72aouGlaEeGPb16w=', 'Content-Length': '0'} 2022-09-03T06:10:15.163 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000ad146f38fcd51033-006312efc7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.164 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:15.164 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:15.165 DEBUG:boto:path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:15.165 DEBUG:boto:auth_path=/avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:15.166 DEBUG:boto:Method: GET 2022-09-03T06:10:15.166 DEBUG:boto:Path: /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:15.167 DEBUG:boto:Data: 2022-09-03T06:10:15.167 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.168 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:15.168 DEBUG:boto:Port: 8000 2022-09-03T06:10:15.169 DEBUG:boto:Params: {} 2022-09-03T06:10:15.170 DEBUG:boto:Token: None 2022-09-03T06:10:15.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/obj?versionId=cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ 2022-09-03T06:10:15.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOn3GkcMDuO72aouGlaEeGPb16w= 2022-09-03T06:10:15.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOn3GkcMDuO72aouGlaEeGPb16w=', 'Content-Length': '0'} 2022-09-03T06:10:15.174 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:09:17 GMT'), ('x-amz-version-id', 'cEao242oFr0QvnFQ3AzgjYb9rMYHnCQ'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a37b854076e73b06-006312efc7-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.175 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:15.175 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:15.176 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.176 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.177 DEBUG:boto:Method: GET 2022-09-03T06:10:15.177 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:15.178 DEBUG:boto:Data: 2022-09-03T06:10:15.178 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.179 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:10:15.179 DEBUG:boto:Port: 8001 2022-09-03T06:10:15.180 DEBUG:boto:Params: {} 2022-09-03T06:10:15.180 DEBUG:boto:Token: None 2022-09-03T06:10:15.181 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/ 2022-09-03T06:10:15.181 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ= 2022-09-03T06:10:15.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ=', 'Content-Length': '0'} 2022-09-03T06:10:15.185 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c7ee7250cf70e29-006312efc7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.230 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:15.231 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:15.232 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:15.232 DEBUG:boto:path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.233 DEBUG:boto:auth_path=/avysmw-20/?prefix=obj 2022-09-03T06:10:15.233 DEBUG:boto:Method: GET 2022-09-03T06:10:15.234 DEBUG:boto:Path: /avysmw-20/?prefix=obj 2022-09-03T06:10:15.234 DEBUG:boto:Data: 2022-09-03T06:10:15.235 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.235 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:10:15.236 DEBUG:boto:Port: 8000 2022-09-03T06:10:15.236 DEBUG:boto:Params: {} 2022-09-03T06:10:15.237 DEBUG:boto:Token: None 2022-09-03T06:10:15.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/ 2022-09-03T06:10:15.238 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ= 2022-09-03T06:10:15.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06://hhXrIKT/iRhKl7EdWuFwN19nQ=', 'Content-Length': '0'} 2022-09-03T06:10:15.242 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000092008b6efb847c63-006312efc7-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.243 DEBUG:boto:b'avysmw-20obj1000falseobj2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:10:15.244 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-20 zones={test-zone3, test-zone2} 2022-09-03T06:10:15.244 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:15.245 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:15.245 DEBUG:boto:path=/avysmw-20/?versioning 2022-09-03T06:10:15.246 DEBUG:boto:auth_path=/avysmw-20/?versioning 2022-09-03T06:10:15.247 DEBUG:boto:Method: PUT 2022-09-03T06:10:15.247 DEBUG:boto:Path: /avysmw-20/?versioning 2022-09-03T06:10:15.248 DEBUG:boto:Data: Suspended Disabled 2022-09-03T06:10:15.248 DEBUG:boto:Headers: {} 2022-09-03T06:10:15.249 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:15.249 DEBUG:boto:Port: 8000 2022-09-03T06:10:15.250 DEBUG:boto:Params: {} 2022-09-03T06:10:15.251 DEBUG:boto:Token: None 2022-09-03T06:10:15.251 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:10:15 GMT /avysmw-20/?versioning 2022-09-03T06:10:15.252 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:16hYCs+Dn3PsYJg4LYWZ30P1QeM= 2022-09-03T06:10:15.252 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:16hYCs+Dn3PsYJg4LYWZ30P1QeM=', 'Content-Length': '230'} 2022-09-03T06:10:15.259 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ab6b975b9b904ff8-006312efc7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:10:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:15.260 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:15.261 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:15.262 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:10:15.320 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:10:15.320 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:10:15.353 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T06:10:15.366 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:10:15.367 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:15.367 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:10:15.367 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T06:10:15.367 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:15.368 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:15.368 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185415.257852_166.1", 2022-09-03T06:10:15.368 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:10:15.257852Z" 2022-09-03T06:10:15.369 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:15.369 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:15.369 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:10:15.369 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:10:15.370 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:15.370 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:15.370 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:10:15.370 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:10:15.371 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:15.371 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:10:15.373 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662185415.257852_166.1', 2: '', 3: ''} 2022-09-03T06:10:15.373 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:10:15.374 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:15.375 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:15.375 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:10:15.434 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:10:15.434 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:10:15.497 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:15.495+0000 7f16dfe75600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:10:15.498 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:15.496+0000 7f16dfe75600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:10:15.517 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:10:15.517 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:10:15.518 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:10:15.518 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:10:15.519 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:10:15.519 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:10:15.519 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:15.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:15.520 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:10:15.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:10:15.521 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:10:15.521 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:10:15.522 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:10:15.522 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:10:15.523 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:10:15.523 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:10:15.523 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:10:15.524 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:10:15.524 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:15.524 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:15.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:15.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:10:15.525 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:10:15.526 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:10:15.526 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:10:15.526 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185415.257852_166.1", 2022-09-03T06:10:15.526 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:10:15.527 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:10:15.527 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:10:15.527 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:10:15.257852Z", 2022-09-03T06:10:15.528 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:15.528 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:15.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:15.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:10:15.529 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:10:15.529 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:10:15.530 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:10:15.530 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:10:15.531 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:10:15.531 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:10:15.531 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:10:15.532 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:10:15.532 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:15.532 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:15.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:15.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:10:15.533 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:10:15.533 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:10:15.533 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:10:15.534 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:10:15.534 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:10:15.534 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:10:15.535 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:10:15.535 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:10:15.535 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:15.536 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:15.536 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:15.536 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:10:15.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:15.537 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:10:15.537 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:10:15.537 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:10:15.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:15.538 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:10:15Z" 2022-09-03T06:10:15.538 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:10:15.540 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185415.257852_166.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:10:15.257852Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:10:15Z" } 2022-09-03T06:10:15.541 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185415.257852_166.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:10:15.257852Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:10:15.542 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185415.257852_166.1', 2: '', 3: ''} 2022-09-03T06:10:15.543 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185415.257852_166.1', 2: '', 3: ''} 2022-09-03T06:10:15.544 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:10:15.544 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:10:15.545 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:15.546 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:15.547 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:10:15.600 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:10:15.601 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:10:15.634 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:15.632+0000 7f3f58324600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:10:15.635 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:15.633+0000 7f3f58324600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:10:15.651 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:10:15.652 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:10:15.652 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:10:15.653 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:10:15.653 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:10:15.654 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:10:15.654 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:15.655 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:15.655 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:10:15.655 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:15.656 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:10:15.656 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:10:15.657 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:10:15.657 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:10:15.658 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:10:15.658 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:10:15.658 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:10:15.659 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:10:15.659 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:15.659 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:15.660 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:15.660 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:15.660 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:10:15.660 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:10:15.661 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:10:15.661 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185351.819576_158.1", 2022-09-03T06:10:15.661 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:10:15.661 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:10:15.662 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:10:15.662 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:09:11.819576Z", 2022-09-03T06:10:15.662 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:15.662 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:15.663 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:15.663 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:15.663 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:10:15.663 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:10:15.664 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:10:15.664 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:10:15.665 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:10:15.665 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:10:15.665 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:10:15.665 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:10:15.666 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:15.666 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:15.666 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:15.667 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:15.667 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:10:15.667 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:10:15.668 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:10:15.668 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:10:15.668 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:10:15.668 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:10:15.669 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:10:15.669 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:10:15.670 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:15.670 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:15.670 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:15.671 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:10:15.671 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:15.671 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:10:15.671 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:10:15.672 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:10:15.672 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:15.672 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:10:15Z" 2022-09-03T06:10:15.672 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:10:15.674 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185351.819576_158.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:09:11.819576Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:10:15Z" } 2022-09-03T06:10:15.674 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185351.819576_158.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:09:11.819576Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:10:15.675 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185415.257852_166.1', 2: '', 3: ''} 2022-09-03T06:10:15.676 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185351.819576_158.1', 2: '', 3: ''} 2022-09-03T06:10:15.676 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662185415.257852_166.1 target=1_1662185351.819576_158.1 2022-09-03T06:10:20.678 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:20.679 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:20.680 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:10:20.740 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:10:20.741 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:10:20.781 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:20.779+0000 7fd950c37600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:10:20.781 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:20.780+0000 7fd950c37600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:10:20.794 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:10:20.794 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:10:20.795 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:10:20.795 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:10:20.795 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:10:20.796 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:10:20.796 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:20.796 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:20.796 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:10:20.797 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:20.797 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:10:20.797 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:10:20.797 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:10:20.798 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:10:20.798 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:10:20.798 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:10:20.799 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:10:20.799 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:10:20.799 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:20.799 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:20.800 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:20.800 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:20.800 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:10:20.800 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:10:20.801 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:10:20.801 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185415.257852_166.1", 2022-09-03T06:10:20.801 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:10:20.801 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:10:20.803 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:10:20.803 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:10:15.257852Z", 2022-09-03T06:10:20.803 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:20.804 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:20.804 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:20.804 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:20.805 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:10:20.805 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:10:20.805 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:10:20.805 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:10:20.806 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:10:20.806 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:10:20.806 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:10:20.807 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:10:20.807 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:20.807 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:20.808 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:20.808 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:10:20.808 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:10:20.808 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:10:20.809 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:10:20.809 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:10:20.809 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:10:20.809 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:10:20.810 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:10:20.810 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:10:20.810 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:10:20.811 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:20.811 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:10:20.811 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:10:20.811 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:20.812 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:10:20.812 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:10:20.812 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:10:20.813 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:10:20.813 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:10:20Z" 2022-09-03T06:10:20.814 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:10:20.815 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185415.257852_166.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:10:15.257852Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:10:20Z" } 2022-09-03T06:10:20.816 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185415.257852_166.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:10:15.257852Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:10:20.816 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185415.257852_166.1', 2: '', 3: ''} 2022-09-03T06:10:20.817 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185415.257852_166.1', 2: '', 3: ''} 2022-09-03T06:10:20.817 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:10:20.818 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:10:20.818 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:20.819 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:10:20.820 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:10:20.877 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:10:20.878 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:10:20.911 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:20.909+0000 7fe57de55600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:10:20.913 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:20.910+0000 7fe57de55600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:10:20.931 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:10:20.932 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:10:20.932 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:10:20.933 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:10:20.933 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:10:20.933 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:10:20.934 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:20.934 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:20.934 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:10:20.935 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:10:20.935 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:10:20.935 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:10:20.935 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:10:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:10:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:10:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:10:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:10:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:10:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:10:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:10:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:10:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:10:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185415.257852_166.1", 2022-09-03T06:10:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:10:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:10:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:10:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:10:15.257852Z", 2022-09-03T06:10:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:10:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:10:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:10:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:10:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:10:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:10:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:10:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:10:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:10:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:10:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:10:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:10:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:10:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:10:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:10:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:10:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:10:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:10:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:10:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:10:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:10:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:10:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:10:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:10:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:10:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:10:20Z" 2022-09-03T06:10:20.956 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:10:20.958 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185415.257852_166.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:10:15.257852Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:10:20Z" } 2022-09-03T06:10:20.959 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185415.257852_166.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:10:15.257852Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:10:20.960 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185415.257852_166.1', 2: '', 3: ''} 2022-09-03T06:10:20.961 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185415.257852_166.1', 2: '', 3: ''} 2022-09-03T06:10:20.961 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:10:20.962 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:10:20.963 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:10:20.964 DEBUG:boto:Method: HEAD 2022-09-03T06:10:20.965 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:10:20.966 DEBUG:boto:Data: 2022-09-03T06:10:20.967 DEBUG:boto:Headers: {} 2022-09-03T06:10:20.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:20.968 DEBUG:boto:Port: 8000 2022-09-03T06:10:20.969 DEBUG:boto:Params: {} 2022-09-03T06:10:20.970 DEBUG:boto:Token: None 2022-09-03T06:10:20.970 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:10:20 GMT /avysmw-20/ 2022-09-03T06:10:20.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ILbeOOAcKYh9niKVcrS1dB4A4dI= 2022-09-03T06:10:20.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:10:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ILbeOOAcKYh9niKVcrS1dB4A4dI=', 'Content-Length': '0'} 2022-09-03T06:10:20.974 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009070d33afcb69fc1-006312efcc-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:10:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:20.975 DEBUG:boto:path=/avysmw-20/obj 2022-09-03T06:10:20.976 DEBUG:boto:auth_path=/avysmw-20/obj 2022-09-03T06:10:20.976 DEBUG:boto:Method: PUT 2022-09-03T06:10:20.977 DEBUG:boto:Path: /avysmw-20/obj 2022-09-03T06:10:20.977 DEBUG:boto:Data: 2022-09-03T06:10:20.978 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:10:20.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:10:20.979 DEBUG:boto:Port: 8000 2022-09-03T06:10:20.979 DEBUG:boto:Params: {} 2022-09-03T06:10:20.980 DEBUG:boto:Token: None 2022-09-03T06:10:20.981 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:10:20 GMT /avysmw-20/obj 2022-09-03T06:10:20.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DuFK5/3gl44lVNYutPLQPUGVe38= 2022-09-03T06:10:20.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:10:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DuFK5/3gl44lVNYutPLQPUGVe38='} 2022-09-03T06:10:21.030 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000006140e329510150d9-006312efcc-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:10:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:10:21.031 DEBUG:rgw_multi.tests:created null version id=None 2022-09-03T06:10:21.032 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-20 2022-09-03T06:10:21.032 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:10:21.033 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:10:21.033 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:10:21.091 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:10:21.091 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:10:21.126 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:21.124+0000 7f88d9491600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:21.127 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:10:21.127 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:26.128 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:26.126+0000 7f88d9491600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:26.128 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:10:26.129 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:31.129 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:31.127+0000 7f88d9491600 1 bucket sync caught up with source: 2022-09-03T06:10:31.130 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:31.131 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:31.131 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:31.127+0000 7f88d9491600 0 bucket checkpoint complete 2022-09-03T06:10:31.154 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-20 2022-09-03T06:10:31.155 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:10:31.155 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:10:31.156 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:10:31.223 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:10:31.224 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:10:31.275 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:31.272+0000 7f5b285e9600 1 bucket sync caught up with source: 2022-09-03T06:10:31.276 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:31.277 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:31.277 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:10:31.272+0000 7f5b285e9600 0 bucket checkpoint complete 2022-09-03T06:10:31.289 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-20 2022-09-03T06:10:31.290 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:10:31.290 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:10:31.291 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:10:31.354 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:10:31.354 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:10:31.397 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:31.395+0000 7f5d4a3a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:31.397 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:10:31.397 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:36.398 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:36.396+0000 7f5d4a3a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:36.399 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:10:36.400 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:41.400 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:41.397+0000 7f5d4a3a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:41.400 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:10:41.401 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:46.401 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:46.399+0000 7f5d4a3a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:46.401 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:10:46.401 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:51.402 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:51.400+0000 7f5d4a3a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:51.402 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:10:51.403 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:10:56.404 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:10:56.402+0000 7f5d4a3a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:10:56.405 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.13.12, , ] 2022-09-03T06:10:56.405 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:11:01.406 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:01.403+0000 7f5d4a3a0600 1 bucket sync caught up with source: 2022-09-03T06:11:01.407 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:11:01.407 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000005.17.17, , ] 2022-09-03T06:11:01.408 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:01.403+0000 7f5d4a3a0600 0 bucket checkpoint complete 2022-09-03T06:11:01.422 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-20 2022-09-03T06:11:01.423 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:01.425 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:01.426 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:01.494 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:01.494 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:01.538 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:01.536+0000 7f932275d600 1 bucket sync caught up with source: 2022-09-03T06:11:01.539 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000006.32.14, , ] 2022-09-03T06:11:01.539 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000006.32.14, , ] 2022-09-03T06:11:01.540 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:01.536+0000 7f932275d600 0 bucket checkpoint complete 2022-09-03T06:11:01.556 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-20 2022-09-03T06:11:01.557 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:01.558 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:01.559 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:01.619 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:01.620 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:01.668 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:01.666+0000 7f8e61103600 1 bucket sync caught up with source: 2022-09-03T06:11:01.669 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000006.32.14, , ] 2022-09-03T06:11:01.669 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000006.32.14, , ] 2022-09-03T06:11:01.669 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:01.666+0000 7f8e61103600 0 bucket checkpoint complete 2022-09-03T06:11:01.687 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-20 2022-09-03T06:11:01.687 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-20 2022-09-03T06:11:01.688 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:01.688 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:01.689 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:01.747 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:01.748 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:01.798 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:01.797+0000 7f5489c26600 1 waiting for incremental sync to catch up: 2022-09-03T06:11:01.799 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000005.33.17, , ] 2022-09-03T06:11:01.799 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000006.37.14, , ] 2022-09-03T06:11:06.800 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:06.798+0000 7f5489c26600 1 waiting for incremental sync to catch up: 2022-09-03T06:11:06.800 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000005.33.17, , ] 2022-09-03T06:11:06.801 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000006.37.14, , ] 2022-09-03T06:11:11.802 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:11.800+0000 7f5489c26600 1 bucket sync caught up with source: 2022-09-03T06:11:11.803 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000006.37.14, , ] 2022-09-03T06:11:11.803 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000006.37.14, , ] 2022-09-03T06:11:11.804 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:11.800+0000 7f5489c26600 0 bucket checkpoint complete 2022-09-03T06:11:11.820 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-20 2022-09-03T06:11:11.821 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:11.822 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:11.824 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:11.892 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:11.893 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:11.954 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:11.951+0000 7f89fa1e7600 1 bucket sync caught up with source: 2022-09-03T06:11:11.955 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000006.37.14, , ] 2022-09-03T06:11:11.955 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000006.37.14, , ] 2022-09-03T06:11:11.956 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:11.951+0000 7f89fa1e7600 0 bucket checkpoint complete 2022-09-03T06:11:11.967 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-20 2022-09-03T06:11:11.967 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-20 zones={test-zone2, test-zone1} 2022-09-03T06:11:11.968 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:11:11.969 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:11:11.970 DEBUG:boto:Method: HEAD 2022-09-03T06:11:11.971 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:11:11.971 DEBUG:boto:Data: 2022-09-03T06:11:11.972 DEBUG:boto:Headers: {} 2022-09-03T06:11:11.972 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:11:11.974 DEBUG:boto:Port: 8000 2022-09-03T06:11:11.975 DEBUG:boto:Params: {} 2022-09-03T06:11:11.976 DEBUG:boto:Token: None 2022-09-03T06:11:11.977 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:11:11 GMT /avysmw-20/ 2022-09-03T06:11:11.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8bcutzSpj1S0a1uUFg0I3lqDf8s= 2022-09-03T06:11:11.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8bcutzSpj1S0a1uUFg0I3lqDf8s=', 'Content-Length': '0'} 2022-09-03T06:11:11.983 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ac2089cd67eebba6-006312efff-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:11.985 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:11:11.986 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:11:11.986 DEBUG:boto:Method: HEAD 2022-09-03T06:11:11.987 DEBUG:boto:Path: /avysmw-20/ 2022-09-03T06:11:11.988 DEBUG:boto:Data: 2022-09-03T06:11:11.989 DEBUG:boto:Headers: {} 2022-09-03T06:11:11.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:11:11.991 DEBUG:boto:Port: 8000 2022-09-03T06:11:11.992 DEBUG:boto:Params: {} 2022-09-03T06:11:11.993 DEBUG:boto:Token: None 2022-09-03T06:11:11.994 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:11:11 GMT /avysmw-20/ 2022-09-03T06:11:11.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8bcutzSpj1S0a1uUFg0I3lqDf8s= 2022-09-03T06:11:11.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8bcutzSpj1S0a1uUFg0I3lqDf8s=', 'Content-Length': '0'} 2022-09-03T06:11:12.000 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c8c2bcea2ef9b80e-006312efff-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.001 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:11:12.002 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:11:12.003 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:11:12.004 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.005 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.006 DEBUG:boto:Method: GET 2022-09-03T06:11:12.007 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.008 DEBUG:boto:Data: 2022-09-03T06:11:12.008 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.010 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:11:12.011 DEBUG:boto:Port: 8000 2022-09-03T06:11:12.012 DEBUG:boto:Params: {} 2022-09-03T06:11:12.013 DEBUG:boto:Token: None 2022-09-03T06:11:12.014 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:11:12 GMT /avysmw-20/?versions 2022-09-03T06:11:12.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IUlrjyTZ+0/yrpokX/98aUrQjFU= 2022-09-03T06:11:12.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IUlrjyTZ+0/yrpokX/98aUrQjFU=', 'Content-Length': '0'} 2022-09-03T06:11:12.019 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ecd091762ebd3327-006312f000-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.066 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQfalse2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:11:12.067 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:11:12.068 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:11:12.068 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:11:12.069 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:11:12.069 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:11:12.070 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.071 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.071 DEBUG:boto:Method: GET 2022-09-03T06:11:12.072 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.072 DEBUG:boto:Data: 2022-09-03T06:11:12.073 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.074 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:11:12.075 DEBUG:boto:Port: 8000 2022-09-03T06:11:12.076 DEBUG:boto:Params: {} 2022-09-03T06:11:12.076 DEBUG:boto:Token: None 2022-09-03T06:11:12.077 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:11:12 GMT /avysmw-20/?versions 2022-09-03T06:11:12.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IUlrjyTZ+0/yrpokX/98aUrQjFU= 2022-09-03T06:11:12.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IUlrjyTZ+0/yrpokX/98aUrQjFU=', 'Content-Length': '0'} 2022-09-03T06:11:12.082 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f5aca166da822ae8-006312f000-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.083 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:10:20.984Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQfalse2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:11:12.084 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:11:12.084 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:11:12.085 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:11:12.086 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:11:12.086 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.087 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.088 DEBUG:boto:Method: GET 2022-09-03T06:11:12.088 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.089 DEBUG:boto:Data: 2022-09-03T06:11:12.089 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.090 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:11:12.090 DEBUG:boto:Port: 8000 2022-09-03T06:11:12.091 DEBUG:boto:Params: {} 2022-09-03T06:11:12.092 DEBUG:boto:Token: None 2022-09-03T06:11:12.092 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:11:12 GMT /avysmw-20/?versions 2022-09-03T06:11:12.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IUlrjyTZ+0/yrpokX/98aUrQjFU= 2022-09-03T06:11:12.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IUlrjyTZ+0/yrpokX/98aUrQjFU=', 'Content-Length': '0'} 2022-09-03T06:11:12.098 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c1fe7a8b4c6396a-006312f000-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.099 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:07:44.897Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQfalse2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:11:12.101 DEBUG:boto:path=/avysmw-20/ 2022-09-03T06:11:12.102 DEBUG:boto:auth_path=/avysmw-20/ 2022-09-03T06:11:12.102 DEBUG:boto:path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.103 DEBUG:boto:auth_path=/avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.104 DEBUG:boto:Method: GET 2022-09-03T06:11:12.105 DEBUG:boto:Path: /avysmw-20/?versions&max-keys=999 2022-09-03T06:11:12.105 DEBUG:boto:Data: 2022-09-03T06:11:12.106 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:11:12.107 DEBUG:boto:Port: 8000 2022-09-03T06:11:12.107 DEBUG:boto:Params: {} 2022-09-03T06:11:12.108 DEBUG:boto:Token: None 2022-09-03T06:11:12.109 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:11:12 GMT /avysmw-20/?versions 2022-09-03T06:11:12.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IUlrjyTZ+0/yrpokX/98aUrQjFU= 2022-09-03T06:11:12.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IUlrjyTZ+0/yrpokX/98aUrQjFU=', 'Content-Length': '0'} 2022-09-03T06:11:12.117 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9f9160678492173-006312f000-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.118 DEBUG:boto:b'avysmw-20999falseobjnulltrue2022-09-03T06:10:20.984Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormalobjcEao242oFr0QvnFQ3AzgjYb9rMYHnCQfalse2022-09-03T06:09:17.581Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:11:12.119 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:11:12.125 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_version_suspended_incremental_sync ... FAIL 2022-09-03T06:11:12.126 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-21 2022-09-03T06:11:12.127 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:11:12.128 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:11:12.128 DEBUG:boto:Method: PUT 2022-09-03T06:11:12.129 DEBUG:boto:Path: /avysmw-21/ 2022-09-03T06:11:12.129 DEBUG:boto:Data: 2022-09-03T06:11:12.130 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.130 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:11:12.131 DEBUG:boto:Port: 8000 2022-09-03T06:11:12.131 DEBUG:boto:Params: {} 2022-09-03T06:11:12.132 DEBUG:boto:Token: None 2022-09-03T06:11:12.132 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:11:12 GMT /avysmw-21/ 2022-09-03T06:11:12.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ENIbj7nC9Nqyoro7wk6l5YznwU0= 2022-09-03T06:11:12.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ENIbj7nC9Nqyoro7wk6l5YznwU0=', 'Content-Length': '0'} 2022-09-03T06:11:12.152 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000007f88c3e559c8dbe6-006312f000-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.152 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-22 2022-09-03T06:11:12.153 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:11:12.153 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:11:12.154 DEBUG:boto:Method: PUT 2022-09-03T06:11:12.154 DEBUG:boto:Path: /avysmw-22/ 2022-09-03T06:11:12.155 DEBUG:boto:Data: 2022-09-03T06:11:12.155 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.156 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:11:12.156 DEBUG:boto:Port: 8000 2022-09-03T06:11:12.157 DEBUG:boto:Params: {} 2022-09-03T06:11:12.158 DEBUG:boto:Token: None 2022-09-03T06:11:12.158 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:11:12 GMT /avysmw-22/ 2022-09-03T06:11:12.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jcWOu7YoCsvX+FTPjW9stTRRs3Q= 2022-09-03T06:11:12.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jcWOu7YoCsvX+FTPjW9stTRRs3Q=', 'Content-Length': '0'} 2022-09-03T06:11:12.193 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c928fbf59ecad44a-006312f000-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.195 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-23 2022-09-03T06:11:12.196 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:11:12.197 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:11:12.198 DEBUG:boto:Method: PUT 2022-09-03T06:11:12.198 DEBUG:boto:Path: /avysmw-23/ 2022-09-03T06:11:12.200 DEBUG:boto:Data: 2022-09-03T06:11:12.202 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.202 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:11:12.203 DEBUG:boto:Port: 8001 2022-09-03T06:11:12.204 DEBUG:boto:Params: {} 2022-09-03T06:11:12.205 DEBUG:boto:Token: None 2022-09-03T06:11:12.206 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:11:12 GMT /avysmw-23/ 2022-09-03T06:11:12.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7JWmWfyKMBJIw8VCn5GD1gsC8YY= 2022-09-03T06:11:12.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7JWmWfyKMBJIw8VCn5GD1gsC8YY=', 'Content-Length': '0'} 2022-09-03T06:11:12.276 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000924672ef55302950-006312f000-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.278 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:11:12.279 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:11:12.280 DEBUG:boto:path=/avysmw-21/?versioning 2022-09-03T06:11:12.280 DEBUG:boto:auth_path=/avysmw-21/?versioning 2022-09-03T06:11:12.281 DEBUG:boto:Method: PUT 2022-09-03T06:11:12.282 DEBUG:boto:Path: /avysmw-21/?versioning 2022-09-03T06:11:12.282 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:11:12.284 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:11:12.285 DEBUG:boto:Port: 8000 2022-09-03T06:11:12.285 DEBUG:boto:Params: {} 2022-09-03T06:11:12.286 DEBUG:boto:Token: None 2022-09-03T06:11:12.286 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:11:12 GMT /avysmw-21/?versioning 2022-09-03T06:11:12.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tXmbwyoQ85Pp+kB5sllA72HFVi8= 2022-09-03T06:11:12.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tXmbwyoQ85Pp+kB5sllA72HFVi8=', 'Content-Length': '228'} 2022-09-03T06:11:12.293 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c1817a80b120819e-006312f000-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.294 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:11:12.295 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:11:12.296 DEBUG:boto:path=/avysmw-22/?versioning 2022-09-03T06:11:12.297 DEBUG:boto:auth_path=/avysmw-22/?versioning 2022-09-03T06:11:12.298 DEBUG:boto:Method: PUT 2022-09-03T06:11:12.298 DEBUG:boto:Path: /avysmw-22/?versioning 2022-09-03T06:11:12.299 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:11:12.299 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:11:12.300 DEBUG:boto:Port: 8000 2022-09-03T06:11:12.301 DEBUG:boto:Params: {} 2022-09-03T06:11:12.301 DEBUG:boto:Token: None 2022-09-03T06:11:12.302 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:11:12 GMT /avysmw-22/?versioning 2022-09-03T06:11:12.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fdRudkekQAY2gr5ydqLdP3pM0Iw= 2022-09-03T06:11:12.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fdRudkekQAY2gr5ydqLdP3pM0Iw=', 'Content-Length': '228'} 2022-09-03T06:11:12.311 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000003edf15d1d4a7887d-006312f000-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.312 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:11:12.313 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:11:12.313 DEBUG:boto:path=/avysmw-23/?versioning 2022-09-03T06:11:12.314 DEBUG:boto:auth_path=/avysmw-23/?versioning 2022-09-03T06:11:12.315 DEBUG:boto:Method: PUT 2022-09-03T06:11:12.315 DEBUG:boto:Path: /avysmw-23/?versioning 2022-09-03T06:11:12.316 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:11:12.317 DEBUG:boto:Headers: {} 2022-09-03T06:11:12.317 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:11:12.318 DEBUG:boto:Port: 8001 2022-09-03T06:11:12.318 DEBUG:boto:Params: {} 2022-09-03T06:11:12.319 DEBUG:boto:Token: None 2022-09-03T06:11:12.320 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:11:12 GMT /avysmw-23/?versioning 2022-09-03T06:11:12.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ev5EQ8ly5rT8P+I1JYXcTTVcluA= 2022-09-03T06:11:12.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ev5EQ8ly5rT8P+I1JYXcTTVcluA=', 'Content-Length': '228'} 2022-09-03T06:11:12.329 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000008624238d44a5c2e4-006312f000-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:12.330 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:12.331 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:12.331 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:11:12.383 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:12.384 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:12.413 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T06:11:12.457 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:11:12.458 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:11:12.458 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:11:12.459 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T06:11:12.459 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:11:12.460 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:11:12.460 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185472.326245_184.1", 2022-09-03T06:11:12.461 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:11:12.326245Z" 2022-09-03T06:11:12.461 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:11:12.461 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:11:12.462 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:11:12.462 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:11:12.463 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:11:12.464 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:11:12.464 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:11:12.465 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:11:12.465 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:11:12.466 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:11:12.467 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662185472.326245_184.1', 2: '', 3: ''} 2022-09-03T06:11:12.468 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:11:12.469 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:12.470 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:12.470 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:11:12.521 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:12.521 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:12.576 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:12.574+0000 7ff96f6c4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:12.577 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:12.575+0000 7ff96f6c4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:12.591 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:11:12.592 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:11:12.592 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:11:12.592 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:11:12.592 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:11:12.593 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:11:12.593 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:12.593 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:12.594 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:11:12.594 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:12.594 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:11:12.595 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:12.595 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:12.595 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:11:12.595 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:12.596 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:11:12.596 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:12.597 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:11:12.597 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:12.598 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:12.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:12.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:12.599 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:11:12.599 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:12.599 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:12.599 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185472.176058_173.1", 2022-09-03T06:11:12.600 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:12.600 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:12.600 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:12.600 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.176058Z", 2022-09-03T06:11:12.601 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:12.601 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:12.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:12.601 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:12.602 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:11:12.602 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:12.602 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:12.602 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:11:12.603 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:12.603 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:12.603 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:12.603 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:11:12.604 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:12.604 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:12.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:12.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:12.605 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:11:12.606 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:12.606 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:12.607 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:11:12.607 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:12.608 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:12.608 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:12.609 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:11:12.609 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:12.610 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:12.610 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:12.611 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:11:12.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:12.612 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:11:12.612 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:11:12.613 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:11:12.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:12.614 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:11:12Z" 2022-09-03T06:11:12.614 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:11:12.615 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185472.176058_173.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:11:12.176058Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:11:12Z" } 2022-09-03T06:11:12.616 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185472.176058_173.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:11:12.176058Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:11:12.617 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.326245_184.1', 2: '', 3: ''} 2022-09-03T06:11:12.618 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.176058_173.1', 2: '', 3: ''} 2022-09-03T06:11:12.619 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662185472.326245_184.1 target=1_1662185472.176058_173.1 2022-09-03T06:11:17.621 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:17.622 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:17.623 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:11:17.676 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:17.676 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:17.724 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:17.722+0000 7fd326b68600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:17.725 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:17.723+0000 7fd326b68600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:17.765 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:11:17.766 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:11:17.766 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:11:17.766 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:11:17.767 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:11:17.767 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:11:17.767 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:17.767 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:17.768 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:11:17.768 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:17.768 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:11:17.768 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:17.769 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:17.769 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:11:17.769 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:17.770 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:11:17.770 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:17.770 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:11:17.770 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:17.771 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:17.771 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:17.772 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:17.773 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:11:17.773 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:17.773 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:17.774 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185472.140367_167.1", 2022-09-03T06:11:17.774 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:17.774 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:17.774 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:17.775 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.140367Z", 2022-09-03T06:11:17.775 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:17.775 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:17.775 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:17.776 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:17.776 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:11:17.777 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:17.777 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:17.777 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:11:17.777 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:17.778 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:17.778 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:17.779 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:11:17.779 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:17.780 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:17.780 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:17.781 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:17.781 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:11:17.782 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:17.782 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:17.783 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:11:17.783 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:17.784 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:17.784 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:17.784 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:11:17.784 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:17.785 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:17.785 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:17.786 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:11:17.786 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:17.786 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:11:17.786 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:11:17.787 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:11:17.787 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:17.787 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:11:17Z" 2022-09-03T06:11:17.787 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:11:17.790 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185472.140367_167.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:11:12.140367Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:11:17Z" } 2022-09-03T06:11:17.791 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185472.140367_167.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:11:12.140367Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:11:17.792 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.326245_184.1', 2: '', 3: ''} 2022-09-03T06:11:17.792 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.140367_167.1', 2: '', 3: ''} 2022-09-03T06:11:17.794 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662185472.326245_184.1 target=1_1662185472.140367_167.1 2022-09-03T06:11:22.799 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:22.800 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:22.801 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:11:22.859 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:22.860 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:22.913 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:22.911+0000 7f41b8b75600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:22.914 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:22.913+0000 7f41b8b75600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:22.947 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:11:22.948 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:11:22.948 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:11:22.949 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:11:22.950 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:11:22.950 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:11:22.951 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:22.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:22.951 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:11:22.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:22.954 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:11:22.954 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:22.954 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:22.954 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:11:22.955 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:22.955 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:11:22.955 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:22.956 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:11:22.956 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:22.956 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:22.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:22.957 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:22.957 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:11:22.957 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:22.958 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:22.959 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185472.326245_184.1", 2022-09-03T06:11:22.959 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:22.959 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:22.960 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:22.960 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.326245Z", 2022-09-03T06:11:22.960 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:22.960 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:22.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:22.961 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:22.961 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:11:22.961 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:22.962 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:22.962 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:11:22.962 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:22.963 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:22.963 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:22.963 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:11:22.963 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:22.964 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:22.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:22.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:22.964 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:11:22.965 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:22.965 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:22.965 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:11:22.965 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:22.966 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:22.966 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:22.966 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:11:22.966 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:22.967 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:22.967 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:22.967 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:11:22.967 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:22.967 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:11:22.968 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:11:22.968 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:11:22.968 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:22.969 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:11:22Z" 2022-09-03T06:11:22.969 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:11:22.969 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185472.326245_184.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:11:12.326245Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:11:22Z" } 2022-09-03T06:11:22.970 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185472.326245_184.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:11:12.326245Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:11:22.971 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.326245_184.1', 2: '', 3: ''} 2022-09-03T06:11:22.972 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.326245_184.1', 2: '', 3: ''} 2022-09-03T06:11:22.972 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:11:22.973 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:11:22.973 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:22.974 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:22.975 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:11:23.034 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:23.035 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:23.075 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:23.069+0000 7feaa63b1600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:23.076 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:23.071+0000 7feaa63b1600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:23.092 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:11:23.093 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:11:23.094 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:11:23.095 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:11:23.096 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:11:23.097 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:11:23.097 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:11:23.098 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:11:23.098 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:11:23.099 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:11:23.099 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:11:23.100 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:11:23.100 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:11:23.101 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:11:23.101 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:11:23.102 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:11:23.103 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:11:23.103 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:11:23.104 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:11:23.104 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:11:23.105 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:11:23.105 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:11:23.106 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:11:23.106 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:11:23.107 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:11:23.108 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185472.326245_184.1", 2022-09-03T06:11:23.108 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:11:23.109 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:11:23.110 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:11:23.110 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.326245Z", 2022-09-03T06:11:23.111 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:11:23.111 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:11:23.112 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:11:23.112 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:11:23.113 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:11:23.114 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:11:23.114 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:11:23.115 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:11:23.115 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:11:23.116 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:11:23.116 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:11:23.117 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:11:23.117 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:11:23.119 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:11:23.119 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:11:23.119 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:11:23.120 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:11:23.120 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:11:23.120 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:11:23.120 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:11:23.121 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:11:23.121 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:11:23.121 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:11:23.121 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:11:23.122 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:11:23.122 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:11:23.122 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:11:23.122 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:11:23.123 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:11:23.123 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:11:23.123 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:11:23.123 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:11:23.124 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:11:23.124 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:11:23Z" 2022-09-03T06:11:23.125 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:11:23.126 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185472.326245_184.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:11:12.326245Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:11:23Z" } 2022-09-03T06:11:23.127 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185472.326245_184.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:11:12.326245Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:11:23.128 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.326245_184.1', 2: '', 3: ''} 2022-09-03T06:11:23.128 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.326245_184.1', 2: '', 3: ''} 2022-09-03T06:11:23.129 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:11:23.130 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:11:23.130 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:23.131 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:11:23.132 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:11:23.184 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:23.184 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:23.225 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:23.223+0000 7fa0f988f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:23.227 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:23.224+0000 7fa0f988f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:11:23.238 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:11:23.239 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:11:23.239 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:11:23.240 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:11:23.240 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:11:23.240 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:11:23.240 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:23.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:23.241 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:11:23.242 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:23.242 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:11:23.243 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:23.243 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:23.244 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:11:23.244 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:23.245 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:11:23.245 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:23.245 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:11:23.246 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:23.247 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:23.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:23.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:23.249 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:11:23.249 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:23.249 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:23.250 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185472.326245_184.1", 2022-09-03T06:11:23.250 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:23.251 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:23.251 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:23.252 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.326245Z", 2022-09-03T06:11:23.252 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:23.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:23.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:23.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:23.254 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:11:23.255 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:23.256 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:23.256 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:11:23.257 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:23.257 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:23.258 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:23.259 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:11:23.259 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:23.259 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:23.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:23.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:11:23.261 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:11:23.261 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:11:23.262 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:11:23.262 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:11:23.262 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:11:23.263 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:11:23.263 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:11:23.263 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:11:23.264 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:11:23.264 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:23.264 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:11:23.264 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:11:23.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:23.265 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:11:23.265 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:11:23.265 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:11:23.266 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:11:23.266 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:11:23Z" 2022-09-03T06:11:23.266 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:11:23.267 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185472.326245_184.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:11:12.326245Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:11:23Z" } 2022-09-03T06:11:23.269 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185472.326245_184.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:11:12.326245Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:11:23.270 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.326245_184.1', 2: '', 3: ''} 2022-09-03T06:11:23.271 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185472.326245_184.1', 2: '', 3: ''} 2022-09-03T06:11:23.271 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:11:23.272 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:11:23.273 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:11:23.274 DEBUG:boto:Method: HEAD 2022-09-03T06:11:23.275 DEBUG:boto:Path: /avysmw-21/ 2022-09-03T06:11:23.276 DEBUG:boto:Data: 2022-09-03T06:11:23.277 DEBUG:boto:Headers: {} 2022-09-03T06:11:23.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:11:23.279 DEBUG:boto:Port: 8000 2022-09-03T06:11:23.280 DEBUG:boto:Params: {} 2022-09-03T06:11:23.281 DEBUG:boto:Token: None 2022-09-03T06:11:23.282 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:11:23 GMT /avysmw-21/ 2022-09-03T06:11:23.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bhQX3m4wRra5JGW+jhJs6PwzOSI= 2022-09-03T06:11:23.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bhQX3m4wRra5JGW+jhJs6PwzOSI=', 'Content-Length': '0'} 2022-09-03T06:11:23.286 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000059ab3cb20b900406-006312f00b-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.287 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:11:23.287 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:11:23.288 DEBUG:boto:Method: PUT 2022-09-03T06:11:23.289 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:11:23.290 DEBUG:boto:Data: 2022-09-03T06:11:23.291 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:11:23.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:11:23.292 DEBUG:boto:Port: 8000 2022-09-03T06:11:23.293 DEBUG:boto:Params: {} 2022-09-03T06:11:23.294 DEBUG:boto:Token: None 2022-09-03T06:11:23.296 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:11:23 GMT /avysmw-21/obj 2022-09-03T06:11:23.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gGu3pASOzRTedL+hE6Ms37fNG58= 2022-09-03T06:11:23.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gGu3pASOzRTedL+hE6Ms37fNG58='} 2022-09-03T06:11:23.312 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-amz-request-id', 'tx00000548e568e47714e05-006312f00b-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.313 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:11:23.314 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:11:23.314 DEBUG:boto:Method: HEAD 2022-09-03T06:11:23.315 DEBUG:boto:Path: /avysmw-21/ 2022-09-03T06:11:23.315 DEBUG:boto:Data: 2022-09-03T06:11:23.316 DEBUG:boto:Headers: {} 2022-09-03T06:11:23.316 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:11:23.317 DEBUG:boto:Port: 8000 2022-09-03T06:11:23.317 DEBUG:boto:Params: {} 2022-09-03T06:11:23.318 DEBUG:boto:Token: None 2022-09-03T06:11:23.318 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:11:23 GMT /avysmw-21/ 2022-09-03T06:11:23.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bhQX3m4wRra5JGW+jhJs6PwzOSI= 2022-09-03T06:11:23.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bhQX3m4wRra5JGW+jhJs6PwzOSI=', 'Content-Length': '0'} 2022-09-03T06:11:23.322 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003fecdda12fe27a0d-006312f00b-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.323 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:11:23.323 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:11:23.324 DEBUG:boto:Method: DELETE 2022-09-03T06:11:23.324 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:11:23.325 DEBUG:boto:Data: 2022-09-03T06:11:23.325 DEBUG:boto:Headers: {} 2022-09-03T06:11:23.326 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:11:23.326 DEBUG:boto:Port: 8000 2022-09-03T06:11:23.327 DEBUG:boto:Params: {} 2022-09-03T06:11:23.328 DEBUG:boto:Token: None 2022-09-03T06:11:23.328 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 06:11:23 GMT /avysmw-21/obj 2022-09-03T06:11:23.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qql5c4Ch8dns3ZEoL9pPdhUTPPE= 2022-09-03T06:11:23.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qql5c4Ch8dns3ZEoL9pPdhUTPPE=', 'Content-Length': '0'} 2022-09-03T06:11:23.341 DEBUG:boto:Response headers: [('x-amz-version-id', '9dy1dF8OnUu5wef2Ui6TFdePOngJrs2'), ('x-amz-delete-marker', 'true'), ('x-amz-request-id', 'tx000004dab88773a6bada9-006312f00b-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.342 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:11:23.343 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:11:23.344 DEBUG:boto:Method: HEAD 2022-09-03T06:11:23.345 DEBUG:boto:Path: /avysmw-22/ 2022-09-03T06:11:23.346 DEBUG:boto:Data: 2022-09-03T06:11:23.347 DEBUG:boto:Headers: {} 2022-09-03T06:11:23.348 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:11:23.348 DEBUG:boto:Port: 8000 2022-09-03T06:11:23.348 DEBUG:boto:Params: {} 2022-09-03T06:11:23.349 DEBUG:boto:Token: None 2022-09-03T06:11:23.350 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:11:23 GMT /avysmw-22/ 2022-09-03T06:11:23.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3zJ6IPT/Na8Bc266cAM4Vgs+hXU= 2022-09-03T06:11:23.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3zJ6IPT/Na8Bc266cAM4Vgs+hXU=', 'Content-Length': '0'} 2022-09-03T06:11:23.354 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c60badc1394d586b-006312f00b-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.355 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:11:23.356 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:11:23.356 DEBUG:boto:Method: PUT 2022-09-03T06:11:23.357 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:11:23.357 DEBUG:boto:Data: 2022-09-03T06:11:23.358 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:11:23.358 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:11:23.359 DEBUG:boto:Port: 8000 2022-09-03T06:11:23.359 DEBUG:boto:Params: {} 2022-09-03T06:11:23.360 DEBUG:boto:Token: None 2022-09-03T06:11:23.361 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:11:23 GMT /avysmw-22/obj 2022-09-03T06:11:23.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SfVn2+eFGk2mktZxWpG+7bG5mcM= 2022-09-03T06:11:23.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SfVn2+eFGk2mktZxWpG+7bG5mcM='} 2022-09-03T06:11:23.376 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-amz-request-id', 'tx00000aadf9d3152163f9f-006312f00b-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.377 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:11:23.378 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:11:23.379 DEBUG:boto:Method: HEAD 2022-09-03T06:11:23.379 DEBUG:boto:Path: /avysmw-22/ 2022-09-03T06:11:23.380 DEBUG:boto:Data: 2022-09-03T06:11:23.380 DEBUG:boto:Headers: {} 2022-09-03T06:11:23.381 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:11:23.381 DEBUG:boto:Port: 8000 2022-09-03T06:11:23.382 DEBUG:boto:Params: {} 2022-09-03T06:11:23.382 DEBUG:boto:Token: None 2022-09-03T06:11:23.383 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:11:23 GMT /avysmw-22/ 2022-09-03T06:11:23.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3zJ6IPT/Na8Bc266cAM4Vgs+hXU= 2022-09-03T06:11:23.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3zJ6IPT/Na8Bc266cAM4Vgs+hXU=', 'Content-Length': '0'} 2022-09-03T06:11:23.386 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009f77e40af25bfb4b-006312f00b-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.387 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:11:23.388 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:11:23.388 DEBUG:boto:Method: DELETE 2022-09-03T06:11:23.389 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:11:23.389 DEBUG:boto:Data: 2022-09-03T06:11:23.390 DEBUG:boto:Headers: {} 2022-09-03T06:11:23.390 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:11:23.391 DEBUG:boto:Port: 8000 2022-09-03T06:11:23.391 DEBUG:boto:Params: {} 2022-09-03T06:11:23.392 DEBUG:boto:Token: None 2022-09-03T06:11:23.392 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 06:11:23 GMT /avysmw-22/obj 2022-09-03T06:11:23.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dNHwiZvXuHgDhuWQw6fSbbPHD9g= 2022-09-03T06:11:23.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dNHwiZvXuHgDhuWQw6fSbbPHD9g=', 'Content-Length': '0'} 2022-09-03T06:11:23.404 DEBUG:boto:Response headers: [('x-amz-version-id', 'oQrpTowIjTMzXOFDN9840QWSjpnmf.s'), ('x-amz-delete-marker', 'true'), ('x-amz-request-id', 'tx0000060e761d8e9f5bdb8-006312f00b-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.405 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:11:23.405 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:11:23.406 DEBUG:boto:Method: HEAD 2022-09-03T06:11:23.407 DEBUG:boto:Path: /avysmw-23/ 2022-09-03T06:11:23.407 DEBUG:boto:Data: 2022-09-03T06:11:23.408 DEBUG:boto:Headers: {} 2022-09-03T06:11:23.408 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:11:23.409 DEBUG:boto:Port: 8001 2022-09-03T06:11:23.409 DEBUG:boto:Params: {} 2022-09-03T06:11:23.410 DEBUG:boto:Token: None 2022-09-03T06:11:23.410 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:11:23 GMT /avysmw-23/ 2022-09-03T06:11:23.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ot8TF9WfFT+3yDklwGmtxDX8NGM= 2022-09-03T06:11:23.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ot8TF9WfFT+3yDklwGmtxDX8NGM=', 'Content-Length': '0'} 2022-09-03T06:11:23.414 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000045508f4b429e1a5f-006312f00b-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.415 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:11:23.416 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:11:23.416 DEBUG:boto:Method: PUT 2022-09-03T06:11:23.417 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:11:23.418 DEBUG:boto:Data: 2022-09-03T06:11:23.419 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:11:23.420 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:11:23.421 DEBUG:boto:Port: 8001 2022-09-03T06:11:23.426 DEBUG:boto:Params: {} 2022-09-03T06:11:23.427 DEBUG:boto:Token: None 2022-09-03T06:11:23.429 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:11:23 GMT /avysmw-23/obj 2022-09-03T06:11:23.429 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GyPc66DgYnJZ6SQeWjuMzgBdIQA= 2022-09-03T06:11:23.430 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GyPc66DgYnJZ6SQeWjuMzgBdIQA='} 2022-09-03T06:11:23.446 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-amz-request-id', 'tx00000c081f9a6a855b394-006312f00b-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.447 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:11:23.447 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:11:23.448 DEBUG:boto:Method: HEAD 2022-09-03T06:11:23.449 DEBUG:boto:Path: /avysmw-23/ 2022-09-03T06:11:23.450 DEBUG:boto:Data: 2022-09-03T06:11:23.451 DEBUG:boto:Headers: {} 2022-09-03T06:11:23.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:11:23.451 DEBUG:boto:Port: 8001 2022-09-03T06:11:23.452 DEBUG:boto:Params: {} 2022-09-03T06:11:23.453 DEBUG:boto:Token: None 2022-09-03T06:11:23.453 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:11:23 GMT /avysmw-23/ 2022-09-03T06:11:23.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ot8TF9WfFT+3yDklwGmtxDX8NGM= 2022-09-03T06:11:23.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ot8TF9WfFT+3yDklwGmtxDX8NGM=', 'Content-Length': '0'} 2022-09-03T06:11:23.457 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b552202370323bf0-006312f00b-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.458 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:11:23.459 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:11:23.460 DEBUG:boto:Method: DELETE 2022-09-03T06:11:23.461 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:11:23.461 DEBUG:boto:Data: 2022-09-03T06:11:23.462 DEBUG:boto:Headers: {} 2022-09-03T06:11:23.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:11:23.464 DEBUG:boto:Port: 8001 2022-09-03T06:11:23.465 DEBUG:boto:Params: {} 2022-09-03T06:11:23.466 DEBUG:boto:Token: None 2022-09-03T06:11:23.467 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 06:11:23 GMT /avysmw-23/obj 2022-09-03T06:11:23.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AQ7xwN5WcOjKDj2P3svyx+mPu+I= 2022-09-03T06:11:23.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:11:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AQ7xwN5WcOjKDj2P3svyx+mPu+I=', 'Content-Length': '0'} 2022-09-03T06:11:23.479 DEBUG:boto:Response headers: [('x-amz-version-id', 'rvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUt'), ('x-amz-delete-marker', 'true'), ('x-amz-request-id', 'tx00000a851cbc882ef3b4c-006312f00b-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:11:23.480 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-21 2022-09-03T06:11:23.481 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:23.481 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:23.482 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-21 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:23.539 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:23.539 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:23.598 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:23.596+0000 7fc2a5928600 1 waiting to reach incremental sync.. 2022-09-03T06:11:28.599 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:28.596+0000 7fc2a5928600 1 waiting to reach incremental sync.. 2022-09-03T06:11:33.601 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:33.598+0000 7fc2a5928600 1 bucket sync caught up with source: 2022-09-03T06:11:33.602 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.11.11, , ] 2022-09-03T06:11:33.602 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.11.11, , ] 2022-09-03T06:11:33.603 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:33.598+0000 7fc2a5928600 0 bucket checkpoint complete 2022-09-03T06:11:33.627 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-21 2022-09-03T06:11:33.628 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:33.629 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:33.629 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-21 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:33.699 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:33.699 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:33.763 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:33.748+0000 7f7f1d8a8600 1 waiting to reach incremental sync.. 2022-09-03T06:11:38.752 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:38.748+0000 7f7f1d8a8600 1 waiting to reach incremental sync.. 2022-09-03T06:11:43.752 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:43.749+0000 7f7f1d8a8600 1 waiting to reach incremental sync.. 2022-09-03T06:11:48.753 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:48.751+0000 7f7f1d8a8600 1 waiting to reach incremental sync.. 2022-09-03T06:11:53.755 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:53.753+0000 7f7f1d8a8600 1 bucket sync caught up with source: 2022-09-03T06:11:53.755 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.11.11, , ] 2022-09-03T06:11:53.756 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.11.11, , ] 2022-09-03T06:11:53.756 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:53.753+0000 7f7f1d8a8600 0 bucket checkpoint complete 2022-09-03T06:11:53.772 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-21 2022-09-03T06:11:53.773 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:53.773 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:53.775 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-21 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:53.840 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:53.840 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:53.894 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:53.891+0000 7f7f4e509600 1 bucket sync caught up with source: 2022-09-03T06:11:53.894 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.11.11, , ] 2022-09-03T06:11:53.895 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.11.11, , ] 2022-09-03T06:11:53.895 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:11:53.891+0000 7f7f4e509600 0 bucket checkpoint complete 2022-09-03T06:11:53.912 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-21 2022-09-03T06:11:53.913 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:53.915 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:53.916 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-21 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:53.983 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:53.984 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:54.023 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:54.022+0000 7f5ea60f0600 1 bucket sync caught up with source: 2022-09-03T06:11:54.024 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.75.5, , ] 2022-09-03T06:11:54.024 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000002.75.5, , ] 2022-09-03T06:11:54.025 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:54.022+0000 7f5ea60f0600 0 bucket checkpoint complete 2022-09-03T06:11:54.039 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-21 2022-09-03T06:11:54.040 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:54.041 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:54.041 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-21 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:54.101 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:54.101 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:54.154 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:54.150+0000 7f7501ac8600 1 bucket sync caught up with source: 2022-09-03T06:11:54.155 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.75.5, , ] 2022-09-03T06:11:54.156 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000002.75.5, , ] 2022-09-03T06:11:54.156 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:54.150+0000 7f7501ac8600 0 bucket checkpoint complete 2022-09-03T06:11:54.168 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-21 2022-09-03T06:11:54.169 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-21 2022-09-03T06:11:54.169 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:54.170 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:11:54.171 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-21 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:11:54.230 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:11:54.231 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:11:54.266 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:54.264+0000 7f09153e2600 1 waiting for incremental sync to catch up: 2022-09-03T06:11:54.266 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:11:54.267 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:11:59.268 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:11:59.266+0000 7f09153e2600 1 waiting for incremental sync to catch up: 2022-09-03T06:11:59.268 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:11:59.269 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:04.269 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:12:04.267+0000 7f09153e2600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:04.269 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:04.270 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:09.270 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:12:09.269+0000 7f09153e2600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:09.271 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:09.271 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:14.272 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:12:14.270+0000 7f09153e2600 1 bucket sync caught up with source: 2022-09-03T06:12:14.273 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:14.273 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:14.274 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:12:14.270+0000 7f09153e2600 0 bucket checkpoint complete 2022-09-03T06:12:14.293 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-21 2022-09-03T06:12:14.294 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:12:14.295 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-21', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:12:14.296 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-21 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:12:14.360 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:12:14.360 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:12:14.412 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:14.410+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:14.412 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:14.412 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:19.413 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:19.411+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:19.414 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:19.414 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:24.415 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:24.413+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:24.415 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:24.415 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:29.416 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:29.414+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:29.417 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:29.417 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:34.418 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:34.416+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:34.418 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:34.419 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:39.420 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:39.418+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:39.421 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:39.421 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:44.421 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:44.419+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:44.421 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:44.422 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:49.424 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:49.420+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:49.424 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:49.425 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:54.424 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:54.422+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:54.424 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:54.425 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:12:59.425 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:12:59.423+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:12:59.426 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:12:59.427 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:04.427 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:04.425+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:04.427 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:04.428 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:09.428 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:09.426+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:09.428 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:09.429 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:14.430 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:14.428+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:14.431 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:14.433 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:19.431 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:19.429+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:19.432 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:19.432 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:24.432 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:24.431+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:24.433 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:24.433 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:29.434 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:29.432+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:29.434 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:29.435 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:34.437 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:34.433+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:34.437 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:34.437 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:39.438 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:39.435+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:39.439 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:39.439 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:44.438 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:44.436+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:44.438 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:44.439 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:49.439 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:49.437+0000 7f5b53a3b600 1 waiting for incremental sync to catch up: 2022-09-03T06:13:49.440 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.61.10, , ] 2022-09-03T06:13:49.440 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:54.441 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:54.439+0000 7f5b53a3b600 1 bucket sync caught up with source: 2022-09-03T06:13:54.443 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:54.443 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.63.11, , ] 2022-09-03T06:13:54.444 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:54.439+0000 7f5b53a3b600 0 bucket checkpoint complete 2022-09-03T06:13:54.461 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-21 2022-09-03T06:13:54.462 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-21 zones={test-zone2, test-zone1} 2022-09-03T06:13:54.463 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.464 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.465 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.465 DEBUG:boto:Path: /avysmw-21/ 2022-09-03T06:13:54.466 DEBUG:boto:Data: 2022-09-03T06:13:54.466 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.467 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:54.468 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.468 DEBUG:boto:Params: {} 2022-09-03T06:13:54.469 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:13:54.470 DEBUG:boto:Token: None 2022-09-03T06:13:54.471 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/ 2022-09-03T06:13:54.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T+ihVmiVT4FY9InCyUsGyl3URaM= 2022-09-03T06:13:54.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T+ihVmiVT4FY9InCyUsGyl3URaM=', 'Content-Length': '0'} 2022-09-03T06:13:54.479 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f0adc369fd3d83a6-006312f0a2-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.481 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.481 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.482 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.483 DEBUG:boto:Path: /avysmw-21/ 2022-09-03T06:13:54.484 DEBUG:boto:Data: 2022-09-03T06:13:54.485 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.486 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.487 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.488 DEBUG:boto:Params: {} 2022-09-03T06:13:54.489 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:13:54.490 DEBUG:boto:Token: None 2022-09-03T06:13:54.491 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/ 2022-09-03T06:13:54.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T+ihVmiVT4FY9InCyUsGyl3URaM= 2022-09-03T06:13:54.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T+ihVmiVT4FY9InCyUsGyl3URaM=', 'Content-Length': '0'} 2022-09-03T06:13:54.497 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c70cbacb0d473c57-006312f0a2-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.499 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:13:54.500 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.501 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.502 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.503 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.504 DEBUG:boto:Method: GET 2022-09-03T06:13:54.505 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.506 DEBUG:boto:Data: 2022-09-03T06:13:54.508 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.508 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:54.509 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.510 DEBUG:boto:Params: {} 2022-09-03T06:13:54.510 DEBUG:boto:Token: None 2022-09-03T06:13:54.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/?versions 2022-09-03T06:13:54.512 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA= 2022-09-03T06:13:54.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA=', 'Content-Length': '0'} 2022-09-03T06:13:54.515 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c856190b77b77dce-006312f0a2-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.516 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:54.517 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:54.518 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:54.518 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:13:54.519 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.519 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.520 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.520 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.521 DEBUG:boto:Method: GET 2022-09-03T06:13:54.522 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.522 DEBUG:boto:Data: 2022-09-03T06:13:54.523 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.523 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.524 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.524 DEBUG:boto:Params: {} 2022-09-03T06:13:54.525 DEBUG:boto:Token: None 2022-09-03T06:13:54.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/?versions 2022-09-03T06:13:54.526 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA= 2022-09-03T06:13:54.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA=', 'Content-Length': '0'} 2022-09-03T06:13:54.530 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004821e520320cebd5-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.530 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:54.531 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:54.532 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:54.532 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.533 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.533 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.534 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.534 DEBUG:boto:Method: GET 2022-09-03T06:13:54.535 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.535 DEBUG:boto:Data: 2022-09-03T06:13:54.536 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.536 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:54.537 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.537 DEBUG:boto:Params: {} 2022-09-03T06:13:54.538 DEBUG:boto:Token: None 2022-09-03T06:13:54.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/?versions 2022-09-03T06:13:54.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA= 2022-09-03T06:13:54.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA=', 'Content-Length': '0'} 2022-09-03T06:13:54.542 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fac49e6619e8107d-006312f0a2-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.543 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:54.544 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.545 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.545 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.546 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.546 DEBUG:boto:Method: GET 2022-09-03T06:13:54.547 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.547 DEBUG:boto:Data: 2022-09-03T06:13:54.548 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.548 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.549 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.549 DEBUG:boto:Params: {} 2022-09-03T06:13:54.550 DEBUG:boto:Token: None 2022-09-03T06:13:54.551 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/?versions 2022-09-03T06:13:54.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA= 2022-09-03T06:13:54.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA=', 'Content-Length': '0'} 2022-09-03T06:13:54.554 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da50c726f145f6b2-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.555 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:54.556 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:54.556 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.557 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.558 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.558 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:54.559 DEBUG:boto:Data: 2022-09-03T06:13:54.559 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.560 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:54.561 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.561 DEBUG:boto:Params: {} 2022-09-03T06:13:54.562 DEBUG:boto:Token: None 2022-09-03T06:13:54.563 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj 2022-09-03T06:13:54.563 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y= 2022-09-03T06:13:54.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y=', 'Content-Length': '0'} 2022-09-03T06:13:54.566 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000a0607393b70894d4-006312f0a2-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.567 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.567 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.568 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.568 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:54.569 DEBUG:boto:Data: 2022-09-03T06:13:54.569 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.571 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.572 DEBUG:boto:Params: {} 2022-09-03T06:13:54.573 DEBUG:boto:Token: None 2022-09-03T06:13:54.574 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj 2022-09-03T06:13:54.575 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y= 2022-09-03T06:13:54.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y=', 'Content-Length': '0'} 2022-09-03T06:13:54.579 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000077ae6a1158b36090-006312f0a2-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.581 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:54.582 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.583 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.583 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.584 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.585 DEBUG:boto:Method: GET 2022-09-03T06:13:54.585 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.586 DEBUG:boto:Data: 2022-09-03T06:13:54.586 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.587 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:54.587 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.588 DEBUG:boto:Params: {} 2022-09-03T06:13:54.588 DEBUG:boto:Token: None 2022-09-03T06:13:54.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854= 2022-09-03T06:13:54.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854=', 'Content-Length': '0'} 2022-09-03T06:13:54.593 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000551a7c89a763ba22-006312f0a2-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.593 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.594 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.595 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.595 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.596 DEBUG:boto:Method: GET 2022-09-03T06:13:54.596 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.597 DEBUG:boto:Data: 2022-09-03T06:13:54.597 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.598 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.598 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.599 DEBUG:boto:Params: {} 2022-09-03T06:13:54.600 DEBUG:boto:Token: None 2022-09-03T06:13:54.600 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854= 2022-09-03T06:13:54.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854=', 'Content-Length': '0'} 2022-09-03T06:13:54.604 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005a7cbc10fc4c7353-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.605 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.605 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.606 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.606 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.607 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.607 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.608 DEBUG:boto:Data: 2022-09-03T06:13:54.608 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.609 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:54.609 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.610 DEBUG:boto:Params: {} 2022-09-03T06:13:54.611 DEBUG:boto:Token: None 2022-09-03T06:13:54.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kgkCWMZxi5lmwyJGON24tYyl6y4= 2022-09-03T06:13:54.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kgkCWMZxi5lmwyJGON24tYyl6y4=', 'Content-Length': '0'} 2022-09-03T06:13:54.616 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000046d6474b206d4bef-006312f0a2-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.617 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.618 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.619 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.619 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.620 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.620 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.621 DEBUG:boto:Data: 2022-09-03T06:13:54.621 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.622 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.623 DEBUG:boto:Params: {} 2022-09-03T06:13:54.623 DEBUG:boto:Token: None 2022-09-03T06:13:54.624 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kgkCWMZxi5lmwyJGON24tYyl6y4= 2022-09-03T06:13:54.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kgkCWMZxi5lmwyJGON24tYyl6y4=', 'Content-Length': '0'} 2022-09-03T06:13:54.628 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000002557b7052e03200b-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.628 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:54.629 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.630 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.630 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.631 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.631 DEBUG:boto:Method: GET 2022-09-03T06:13:54.632 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.633 DEBUG:boto:Data: 2022-09-03T06:13:54.633 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.634 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:54.634 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.635 DEBUG:boto:Params: {} 2022-09-03T06:13:54.636 DEBUG:boto:Token: None 2022-09-03T06:13:54.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854= 2022-09-03T06:13:54.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854=', 'Content-Length': '0'} 2022-09-03T06:13:54.640 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e8b64e0e8ea7bc87-006312f0a2-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.641 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.642 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.642 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.643 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.643 DEBUG:boto:Method: GET 2022-09-03T06:13:54.644 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.645 DEBUG:boto:Data: 2022-09-03T06:13:54.645 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.646 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.647 DEBUG:boto:Params: {} 2022-09-03T06:13:54.648 DEBUG:boto:Token: None 2022-09-03T06:13:54.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854= 2022-09-03T06:13:54.651 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854=', 'Content-Length': '0'} 2022-09-03T06:13:54.653 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f7e0d47eb3be3a1b-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.654 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.655 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.656 DEBUG:boto:path=/avysmw-21/?prefix=obj 2022-09-03T06:13:54.656 DEBUG:boto:auth_path=/avysmw-21/?prefix=obj 2022-09-03T06:13:54.657 DEBUG:boto:Method: GET 2022-09-03T06:13:54.658 DEBUG:boto:Path: /avysmw-21/?prefix=obj 2022-09-03T06:13:54.658 DEBUG:boto:Data: 2022-09-03T06:13:54.659 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.659 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:54.660 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.660 DEBUG:boto:Params: {} 2022-09-03T06:13:54.661 DEBUG:boto:Token: None 2022-09-03T06:13:54.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/ 2022-09-03T06:13:54.662 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qNKJ6QkEGdytKRZ7er0mtNt6Uw4= 2022-09-03T06:13:54.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qNKJ6QkEGdytKRZ7er0mtNt6Uw4=', 'Content-Length': '0'} 2022-09-03T06:13:54.665 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000afd1c240290ba808-006312f0a2-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.706 DEBUG:boto:b'avysmw-21obj1000false' 2022-09-03T06:13:54.707 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.708 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.709 DEBUG:boto:path=/avysmw-21/?prefix=obj 2022-09-03T06:13:54.710 DEBUG:boto:auth_path=/avysmw-21/?prefix=obj 2022-09-03T06:13:54.710 DEBUG:boto:Method: GET 2022-09-03T06:13:54.711 DEBUG:boto:Path: /avysmw-21/?prefix=obj 2022-09-03T06:13:54.712 DEBUG:boto:Data: 2022-09-03T06:13:54.712 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.713 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.714 DEBUG:boto:Params: {} 2022-09-03T06:13:54.715 DEBUG:boto:Token: None 2022-09-03T06:13:54.715 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/ 2022-09-03T06:13:54.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qNKJ6QkEGdytKRZ7er0mtNt6Uw4= 2022-09-03T06:13:54.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qNKJ6QkEGdytKRZ7er0mtNt6Uw4=', 'Content-Length': '0'} 2022-09-03T06:13:54.720 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e027ff35433ed80-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.721 DEBUG:boto:b'avysmw-21obj1000false' 2022-09-03T06:13:54.723 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.724 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.725 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.726 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:54.727 DEBUG:boto:Data: 2022-09-03T06:13:54.728 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.728 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:54.729 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.730 DEBUG:boto:Params: {} 2022-09-03T06:13:54.731 DEBUG:boto:Token: None 2022-09-03T06:13:54.732 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj 2022-09-03T06:13:54.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y= 2022-09-03T06:13:54.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y=', 'Content-Length': '0'} 2022-09-03T06:13:54.736 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000042b8cb85d991701f-006312f0a2-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.737 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.738 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.738 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.739 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:54.739 DEBUG:boto:Data: 2022-09-03T06:13:54.740 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.740 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.741 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.742 DEBUG:boto:Params: {} 2022-09-03T06:13:54.742 DEBUG:boto:Token: None 2022-09-03T06:13:54.743 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj 2022-09-03T06:13:54.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y= 2022-09-03T06:13:54.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y=', 'Content-Length': '0'} 2022-09-03T06:13:54.747 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000404aadef4c98dca0-006312f0a2-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.748 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-21 zones={test-zone2, test-zone1} 2022-09-03T06:13:54.749 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-21 zones={test-zone3, test-zone1} 2022-09-03T06:13:54.749 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.750 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.751 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.751 DEBUG:boto:Path: /avysmw-21/ 2022-09-03T06:13:54.752 DEBUG:boto:Data: 2022-09-03T06:13:54.752 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.753 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:54.753 DEBUG:boto:Port: 8001 2022-09-03T06:13:54.754 DEBUG:boto:Params: {} 2022-09-03T06:13:54.755 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:13:54.755 DEBUG:boto:Token: None 2022-09-03T06:13:54.756 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/ 2022-09-03T06:13:54.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T+ihVmiVT4FY9InCyUsGyl3URaM= 2022-09-03T06:13:54.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T+ihVmiVT4FY9InCyUsGyl3URaM=', 'Content-Length': '0'} 2022-09-03T06:13:54.762 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006819a6a8b7b4193a-006312f0a2-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.762 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.763 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.764 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.765 DEBUG:boto:Path: /avysmw-21/ 2022-09-03T06:13:54.765 DEBUG:boto:Data: 2022-09-03T06:13:54.766 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.767 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.768 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.769 DEBUG:boto:Params: {} 2022-09-03T06:13:54.770 DEBUG:boto:Token: None 2022-09-03T06:13:54.770 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/ 2022-09-03T06:13:54.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T+ihVmiVT4FY9InCyUsGyl3URaM= 2022-09-03T06:13:54.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T+ihVmiVT4FY9InCyUsGyl3URaM=', 'Content-Length': '0'} 2022-09-03T06:13:54.775 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003911ffbbb511de2e-006312f0a2-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.776 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:13:54.776 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.777 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.777 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.778 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.778 DEBUG:boto:Method: GET 2022-09-03T06:13:54.779 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.780 DEBUG:boto:Data: 2022-09-03T06:13:54.780 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.781 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:54.781 DEBUG:boto:Port: 8001 2022-09-03T06:13:54.782 DEBUG:boto:Params: {} 2022-09-03T06:13:54.783 DEBUG:boto:Token: None 2022-09-03T06:13:54.784 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/?versions 2022-09-03T06:13:54.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA= 2022-09-03T06:13:54.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA=', 'Content-Length': '0'} 2022-09-03T06:13:54.791 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d50a1235bc4708a-006312f0a2-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.793 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:54.794 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:54.795 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:54.795 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:13:54.796 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.797 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.797 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.798 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.798 DEBUG:boto:Method: GET 2022-09-03T06:13:54.799 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.799 DEBUG:boto:Data: 2022-09-03T06:13:54.800 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.800 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.801 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.801 DEBUG:boto:Params: {} 2022-09-03T06:13:54.802 DEBUG:boto:Token: None 2022-09-03T06:13:54.803 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/?versions 2022-09-03T06:13:54.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA= 2022-09-03T06:13:54.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA=', 'Content-Length': '0'} 2022-09-03T06:13:54.807 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d6e5a9b5a4412e0-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.850 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:54.851 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:54.852 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:54.853 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.854 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.855 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.856 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.857 DEBUG:boto:Method: GET 2022-09-03T06:13:54.858 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.859 DEBUG:boto:Data: 2022-09-03T06:13:54.860 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.860 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:54.861 DEBUG:boto:Port: 8001 2022-09-03T06:13:54.862 DEBUG:boto:Params: {} 2022-09-03T06:13:54.863 DEBUG:boto:Token: None 2022-09-03T06:13:54.864 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/?versions 2022-09-03T06:13:54.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA= 2022-09-03T06:13:54.866 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA=', 'Content-Length': '0'} 2022-09-03T06:13:54.869 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000217639652b8bebd0-006312f0a2-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.874 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:54.875 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.876 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.876 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.877 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.877 DEBUG:boto:Method: GET 2022-09-03T06:13:54.878 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:54.878 DEBUG:boto:Data: 2022-09-03T06:13:54.879 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.880 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.881 DEBUG:boto:Params: {} 2022-09-03T06:13:54.881 DEBUG:boto:Token: None 2022-09-03T06:13:54.882 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/?versions 2022-09-03T06:13:54.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA= 2022-09-03T06:13:54.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KvdhyGP2FA7gSGrW4wx7ECMWVhA=', 'Content-Length': '0'} 2022-09-03T06:13:54.886 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007fed59416f4a1ce5-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.887 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:54.888 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:54.888 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.889 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.890 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.890 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:54.891 DEBUG:boto:Data: 2022-09-03T06:13:54.891 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:54.892 DEBUG:boto:Port: 8001 2022-09-03T06:13:54.893 DEBUG:boto:Params: {} 2022-09-03T06:13:54.893 DEBUG:boto:Token: None 2022-09-03T06:13:54.894 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj 2022-09-03T06:13:54.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y= 2022-09-03T06:13:54.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y=', 'Content-Length': '0'} 2022-09-03T06:13:54.898 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000002fb2525b89b1552b-006312f0a2-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.899 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.899 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.900 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.900 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:54.901 DEBUG:boto:Data: 2022-09-03T06:13:54.901 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.902 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.902 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.903 DEBUG:boto:Params: {} 2022-09-03T06:13:54.903 DEBUG:boto:Token: None 2022-09-03T06:13:54.904 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj 2022-09-03T06:13:54.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y= 2022-09-03T06:13:54.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0JZShiHcIgrepJ3yaZ1eitwq/0Y=', 'Content-Length': '0'} 2022-09-03T06:13:54.908 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000065a248ecec3c0c00-006312f0a2-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.908 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:54.909 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.910 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.910 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.911 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.911 DEBUG:boto:Method: GET 2022-09-03T06:13:54.912 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.912 DEBUG:boto:Data: 2022-09-03T06:13:54.913 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.913 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:54.914 DEBUG:boto:Port: 8001 2022-09-03T06:13:54.915 DEBUG:boto:Params: {} 2022-09-03T06:13:54.915 DEBUG:boto:Token: None 2022-09-03T06:13:54.916 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854= 2022-09-03T06:13:54.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854=', 'Content-Length': '0'} 2022-09-03T06:13:54.919 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e588fc00f29118de-006312f0a2-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.920 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.920 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.921 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.922 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.922 DEBUG:boto:Method: GET 2022-09-03T06:13:54.923 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.923 DEBUG:boto:Data: 2022-09-03T06:13:54.924 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.925 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.925 DEBUG:boto:Params: {} 2022-09-03T06:13:54.926 DEBUG:boto:Token: None 2022-09-03T06:13:54.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854= 2022-09-03T06:13:54.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854=', 'Content-Length': '0'} 2022-09-03T06:13:54.931 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000002e84541a294a5086-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.932 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.933 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.933 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.934 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.934 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.935 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.935 DEBUG:boto:Data: 2022-09-03T06:13:54.936 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:54.937 DEBUG:boto:Port: 8001 2022-09-03T06:13:54.938 DEBUG:boto:Params: {} 2022-09-03T06:13:54.938 DEBUG:boto:Token: None 2022-09-03T06:13:54.939 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kgkCWMZxi5lmwyJGON24tYyl6y4= 2022-09-03T06:13:54.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kgkCWMZxi5lmwyJGON24tYyl6y4=', 'Content-Length': '0'} 2022-09-03T06:13:54.942 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d8cc0a8f1465a933-006312f0a2-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.943 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.944 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.945 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.945 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.946 DEBUG:boto:Method: HEAD 2022-09-03T06:13:54.947 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.948 DEBUG:boto:Data: 2022-09-03T06:13:54.949 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.950 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.951 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.952 DEBUG:boto:Params: {} 2022-09-03T06:13:54.953 DEBUG:boto:Token: None 2022-09-03T06:13:54.954 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kgkCWMZxi5lmwyJGON24tYyl6y4= 2022-09-03T06:13:54.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kgkCWMZxi5lmwyJGON24tYyl6y4=', 'Content-Length': '0'} 2022-09-03T06:13:54.957 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000f3f65f919822e0ef-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.958 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:54.959 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.959 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.960 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.961 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.961 DEBUG:boto:Method: GET 2022-09-03T06:13:54.962 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.963 DEBUG:boto:Data: 2022-09-03T06:13:54.964 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:54.967 DEBUG:boto:Port: 8001 2022-09-03T06:13:54.968 DEBUG:boto:Params: {} 2022-09-03T06:13:54.969 DEBUG:boto:Token: None 2022-09-03T06:13:54.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854= 2022-09-03T06:13:54.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854=', 'Content-Length': '0'} 2022-09-03T06:13:54.974 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009d27038ae41993eb-006312f0a2-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.976 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:54.976 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:54.978 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.979 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.980 DEBUG:boto:Method: GET 2022-09-03T06:13:54.981 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.982 DEBUG:boto:Data: 2022-09-03T06:13:54.983 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:54.985 DEBUG:boto:Port: 8000 2022-09-03T06:13:54.986 DEBUG:boto:Params: {} 2022-09-03T06:13:54.986 DEBUG:boto:Token: None 2022-09-03T06:13:54.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:54 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:54.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854= 2022-09-03T06:13:54.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+rdjNaq/1yLx+dQxbu/jm1n854=', 'Content-Length': '0'} 2022-09-03T06:13:54.991 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000521d61fac5a31e61-006312f0a2-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:54.992 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:54.993 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:54.994 DEBUG:boto:path=/avysmw-21/?prefix=obj 2022-09-03T06:13:54.995 DEBUG:boto:auth_path=/avysmw-21/?prefix=obj 2022-09-03T06:13:54.995 DEBUG:boto:Method: GET 2022-09-03T06:13:54.996 DEBUG:boto:Path: /avysmw-21/?prefix=obj 2022-09-03T06:13:54.997 DEBUG:boto:Data: 2022-09-03T06:13:54.998 DEBUG:boto:Headers: {} 2022-09-03T06:13:54.999 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.000 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.001 DEBUG:boto:Params: {} 2022-09-03T06:13:55.002 DEBUG:boto:Token: None 2022-09-03T06:13:55.003 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/ 2022-09-03T06:13:55.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zAK454GlLWa9IidOA1Ju3H4kUvs= 2022-09-03T06:13:55.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zAK454GlLWa9IidOA1Ju3H4kUvs=', 'Content-Length': '0'} 2022-09-03T06:13:55.008 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1e6c48f234d55b7-006312f0a3-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.009 DEBUG:boto:b'avysmw-21obj1000false' 2022-09-03T06:13:55.011 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:55.011 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:55.012 DEBUG:boto:path=/avysmw-21/?prefix=obj 2022-09-03T06:13:55.013 DEBUG:boto:auth_path=/avysmw-21/?prefix=obj 2022-09-03T06:13:55.014 DEBUG:boto:Method: GET 2022-09-03T06:13:55.015 DEBUG:boto:Path: /avysmw-21/?prefix=obj 2022-09-03T06:13:55.016 DEBUG:boto:Data: 2022-09-03T06:13:55.017 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:55.018 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.019 DEBUG:boto:Params: {} 2022-09-03T06:13:55.020 DEBUG:boto:Token: None 2022-09-03T06:13:55.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/ 2022-09-03T06:13:55.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zAK454GlLWa9IidOA1Ju3H4kUvs= 2022-09-03T06:13:55.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zAK454GlLWa9IidOA1Ju3H4kUvs=', 'Content-Length': '0'} 2022-09-03T06:13:55.027 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a62e5c01c27e1494-006312f0a3-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.070 DEBUG:boto:b'avysmw-21obj1000false' 2022-09-03T06:13:55.072 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.073 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.075 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.076 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:55.077 DEBUG:boto:Data: 2022-09-03T06:13:55.078 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.079 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.080 DEBUG:boto:Params: {} 2022-09-03T06:13:55.080 DEBUG:boto:Token: None 2022-09-03T06:13:55.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj 2022-09-03T06:13:55.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik= 2022-09-03T06:13:55.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik=', 'Content-Length': '0'} 2022-09-03T06:13:55.085 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000029e956370491bf2f-006312f0a3-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.086 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.087 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.088 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.088 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:55.089 DEBUG:boto:Data: 2022-09-03T06:13:55.090 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:55.093 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.094 DEBUG:boto:Params: {} 2022-09-03T06:13:55.095 DEBUG:boto:Token: None 2022-09-03T06:13:55.095 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj 2022-09-03T06:13:55.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik= 2022-09-03T06:13:55.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik=', 'Content-Length': '0'} 2022-09-03T06:13:55.099 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000005b0e2b21922c002d-006312f0a3-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.099 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-21 zones={test-zone3, test-zone1} 2022-09-03T06:13:55.100 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-21 zones={test-zone3, test-zone2} 2022-09-03T06:13:55.100 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:55.101 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:55.101 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.102 DEBUG:boto:Path: /avysmw-21/ 2022-09-03T06:13:55.102 DEBUG:boto:Data: 2022-09-03T06:13:55.103 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.104 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.104 DEBUG:boto:Params: {} 2022-09-03T06:13:55.105 DEBUG:boto:Token: None 2022-09-03T06:13:55.106 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/ 2022-09-03T06:13:55.106 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EifGHmmwuB284MHAD5I75yhQQGg= 2022-09-03T06:13:55.107 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EifGHmmwuB284MHAD5I75yhQQGg=', 'Content-Length': '0'} 2022-09-03T06:13:55.109 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000010316a27f79b79d2-006312f0a3-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.110 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:55.111 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:55.111 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.112 DEBUG:boto:Path: /avysmw-21/ 2022-09-03T06:13:55.112 DEBUG:boto:Data: 2022-09-03T06:13:55.113 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.113 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:55.114 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.114 DEBUG:boto:Params: {} 2022-09-03T06:13:55.115 DEBUG:boto:Token: None 2022-09-03T06:13:55.115 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/ 2022-09-03T06:13:55.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EifGHmmwuB284MHAD5I75yhQQGg= 2022-09-03T06:13:55.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EifGHmmwuB284MHAD5I75yhQQGg=', 'Content-Length': '0'} 2022-09-03T06:13:55.122 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000412667e97c4a1e4-006312f0a3-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.123 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:13:55.124 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:55.124 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:55.125 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.127 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.127 DEBUG:boto:Method: GET 2022-09-03T06:13:55.128 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.128 DEBUG:boto:Data: 2022-09-03T06:13:55.129 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.130 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.131 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.132 DEBUG:boto:Params: {} 2022-09-03T06:13:55.132 DEBUG:boto:Token: None 2022-09-03T06:13:55.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/?versions 2022-09-03T06:13:55.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mFulGJNDF3teMCv/2OwAHgWj9dk= 2022-09-03T06:13:55.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mFulGJNDF3teMCv/2OwAHgWj9dk=', 'Content-Length': '0'} 2022-09-03T06:13:55.138 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045bbd8cf483fe6f5-006312f0a3-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.139 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:55.141 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:55.141 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:55.142 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:13:55.143 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:55.144 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:55.145 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.145 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.146 DEBUG:boto:Method: GET 2022-09-03T06:13:55.147 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.148 DEBUG:boto:Data: 2022-09-03T06:13:55.149 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.149 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:55.150 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.150 DEBUG:boto:Params: {} 2022-09-03T06:13:55.151 DEBUG:boto:Token: None 2022-09-03T06:13:55.152 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/?versions 2022-09-03T06:13:55.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mFulGJNDF3teMCv/2OwAHgWj9dk= 2022-09-03T06:13:55.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mFulGJNDF3teMCv/2OwAHgWj9dk=', 'Content-Length': '0'} 2022-09-03T06:13:55.156 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000484b82f109d67d6a-006312f0a3-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.157 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:55.158 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:55.159 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:55.160 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:55.161 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:55.161 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.162 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.163 DEBUG:boto:Method: GET 2022-09-03T06:13:55.164 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.165 DEBUG:boto:Data: 2022-09-03T06:13:55.166 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.168 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.169 DEBUG:boto:Params: {} 2022-09-03T06:13:55.170 DEBUG:boto:Token: None 2022-09-03T06:13:55.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/?versions 2022-09-03T06:13:55.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mFulGJNDF3teMCv/2OwAHgWj9dk= 2022-09-03T06:13:55.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mFulGJNDF3teMCv/2OwAHgWj9dk=', 'Content-Length': '0'} 2022-09-03T06:13:55.176 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003db08cc0ad10355d-006312f0a3-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.177 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:55.178 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:55.179 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:55.180 DEBUG:boto:path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.181 DEBUG:boto:auth_path=/avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.182 DEBUG:boto:Method: GET 2022-09-03T06:13:55.183 DEBUG:boto:Path: /avysmw-21/?versions&max-keys=999 2022-09-03T06:13:55.184 DEBUG:boto:Data: 2022-09-03T06:13:55.185 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.186 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:55.187 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.188 DEBUG:boto:Params: {} 2022-09-03T06:13:55.189 DEBUG:boto:Token: None 2022-09-03T06:13:55.190 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/?versions 2022-09-03T06:13:55.191 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mFulGJNDF3teMCv/2OwAHgWj9dk= 2022-09-03T06:13:55.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mFulGJNDF3teMCv/2OwAHgWj9dk=', 'Content-Length': '0'} 2022-09-03T06:13:55.194 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ce36286b527fc7e-006312f0a3-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.195 DEBUG:boto:b'avysmw-21999falseobj9dy1dF8OnUu5wef2Ui6TFdePOngJrs2true2022-09-03T06:11:23.331Zrgw-multisite-test-userMultisite Test UserNormalobjcTdDyfLLFTPpTwC4NoFPPayUuEueuGkfalse2022-09-03T06:11:23.299Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:55.197 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:55.198 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.198 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.199 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.200 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:55.200 DEBUG:boto:Data: 2022-09-03T06:13:55.201 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.202 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.203 DEBUG:boto:Params: {} 2022-09-03T06:13:55.203 DEBUG:boto:Token: None 2022-09-03T06:13:55.204 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj 2022-09-03T06:13:55.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik= 2022-09-03T06:13:55.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik=', 'Content-Length': '0'} 2022-09-03T06:13:55.208 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000301b2096498523a5-006312f0a3-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.209 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.210 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.211 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.212 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:55.213 DEBUG:boto:Data: 2022-09-03T06:13:55.213 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.214 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:55.214 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.215 DEBUG:boto:Params: {} 2022-09-03T06:13:55.216 DEBUG:boto:Token: None 2022-09-03T06:13:55.216 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj 2022-09-03T06:13:55.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik= 2022-09-03T06:13:55.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik=', 'Content-Length': '0'} 2022-09-03T06:13:55.220 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000089c8470b5041830c-006312f0a3-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.221 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:55.221 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.222 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.222 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.223 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.223 DEBUG:boto:Method: GET 2022-09-03T06:13:55.224 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.224 DEBUG:boto:Data: 2022-09-03T06:13:55.225 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.226 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.226 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.227 DEBUG:boto:Params: {} 2022-09-03T06:13:55.228 DEBUG:boto:Token: None 2022-09-03T06:13:55.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4BPMRhE5SEA+HMcwDKNu/X9fmdY= 2022-09-03T06:13:55.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4BPMRhE5SEA+HMcwDKNu/X9fmdY=', 'Content-Length': '0'} 2022-09-03T06:13:55.232 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b463c2c731852a64-006312f0a3-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.234 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.234 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.235 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.236 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.236 DEBUG:boto:Method: GET 2022-09-03T06:13:55.237 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.238 DEBUG:boto:Data: 2022-09-03T06:13:55.239 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.240 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:55.241 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.242 DEBUG:boto:Params: {} 2022-09-03T06:13:55.242 DEBUG:boto:Token: None 2022-09-03T06:13:55.243 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4BPMRhE5SEA+HMcwDKNu/X9fmdY= 2022-09-03T06:13:55.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4BPMRhE5SEA+HMcwDKNu/X9fmdY=', 'Content-Length': '0'} 2022-09-03T06:13:55.247 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c8586bcca8bc2c9a-006312f0a3-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.249 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.249 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.250 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.251 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.252 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.253 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.254 DEBUG:boto:Data: 2022-09-03T06:13:55.255 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.255 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.257 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.258 DEBUG:boto:Params: {} 2022-09-03T06:13:55.259 DEBUG:boto:Token: None 2022-09-03T06:13:55.259 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jXnPZisPQgHP5XKJo4hrrFst5Bo= 2022-09-03T06:13:55.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jXnPZisPQgHP5XKJo4hrrFst5Bo=', 'Content-Length': '0'} 2022-09-03T06:13:55.264 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a4a27162b6fce408-006312f0a3-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.265 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.266 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.267 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.268 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.269 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.270 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.271 DEBUG:boto:Data: 2022-09-03T06:13:55.272 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.272 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:55.273 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.273 DEBUG:boto:Params: {} 2022-09-03T06:13:55.275 DEBUG:boto:Token: None 2022-09-03T06:13:55.276 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jXnPZisPQgHP5XKJo4hrrFst5Bo= 2022-09-03T06:13:55.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jXnPZisPQgHP5XKJo4hrrFst5Bo=', 'Content-Length': '0'} 2022-09-03T06:13:55.281 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000041efe9bc53f60e92-006312f0a3-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.283 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:55.284 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.285 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.286 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.287 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.288 DEBUG:boto:Method: GET 2022-09-03T06:13:55.289 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.290 DEBUG:boto:Data: 2022-09-03T06:13:55.291 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.293 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.294 DEBUG:boto:Params: {} 2022-09-03T06:13:55.295 DEBUG:boto:Token: None 2022-09-03T06:13:55.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4BPMRhE5SEA+HMcwDKNu/X9fmdY= 2022-09-03T06:13:55.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4BPMRhE5SEA+HMcwDKNu/X9fmdY=', 'Content-Length': '0'} 2022-09-03T06:13:55.302 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d88ca8dd93159723-006312f0a3-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.304 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.305 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.305 DEBUG:boto:path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.306 DEBUG:boto:auth_path=/avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.307 DEBUG:boto:Method: GET 2022-09-03T06:13:55.307 DEBUG:boto:Path: /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.308 DEBUG:boto:Data: 2022-09-03T06:13:55.308 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.309 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:55.309 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.310 DEBUG:boto:Params: {} 2022-09-03T06:13:55.310 DEBUG:boto:Token: None 2022-09-03T06:13:55.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj?versionId=cTdDyfLLFTPpTwC4NoFPPayUuEueuGk 2022-09-03T06:13:55.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4BPMRhE5SEA+HMcwDKNu/X9fmdY= 2022-09-03T06:13:55.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4BPMRhE5SEA+HMcwDKNu/X9fmdY=', 'Content-Length': '0'} 2022-09-03T06:13:55.315 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'cTdDyfLLFTPpTwC4NoFPPayUuEueuGk'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b913225aca1f4a55-006312f0a3-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.316 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:55.316 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:55.317 DEBUG:boto:path=/avysmw-21/?prefix=obj 2022-09-03T06:13:55.317 DEBUG:boto:auth_path=/avysmw-21/?prefix=obj 2022-09-03T06:13:55.318 DEBUG:boto:Method: GET 2022-09-03T06:13:55.318 DEBUG:boto:Path: /avysmw-21/?prefix=obj 2022-09-03T06:13:55.319 DEBUG:boto:Data: 2022-09-03T06:13:55.319 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.321 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.321 DEBUG:boto:Params: {} 2022-09-03T06:13:55.322 DEBUG:boto:Token: None 2022-09-03T06:13:55.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/ 2022-09-03T06:13:55.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zAK454GlLWa9IidOA1Ju3H4kUvs= 2022-09-03T06:13:55.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zAK454GlLWa9IidOA1Ju3H4kUvs=', 'Content-Length': '0'} 2022-09-03T06:13:55.327 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a35da30c23998b4-006312f0a3-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.328 DEBUG:boto:b'avysmw-21obj1000false' 2022-09-03T06:13:55.329 DEBUG:boto:path=/avysmw-21/ 2022-09-03T06:13:55.330 DEBUG:boto:auth_path=/avysmw-21/ 2022-09-03T06:13:55.330 DEBUG:boto:path=/avysmw-21/?prefix=obj 2022-09-03T06:13:55.331 DEBUG:boto:auth_path=/avysmw-21/?prefix=obj 2022-09-03T06:13:55.331 DEBUG:boto:Method: GET 2022-09-03T06:13:55.332 DEBUG:boto:Path: /avysmw-21/?prefix=obj 2022-09-03T06:13:55.332 DEBUG:boto:Data: 2022-09-03T06:13:55.333 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.334 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:55.334 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.335 DEBUG:boto:Params: {} 2022-09-03T06:13:55.335 DEBUG:boto:Token: None 2022-09-03T06:13:55.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/ 2022-09-03T06:13:55.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zAK454GlLWa9IidOA1Ju3H4kUvs= 2022-09-03T06:13:55.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zAK454GlLWa9IidOA1Ju3H4kUvs=', 'Content-Length': '0'} 2022-09-03T06:13:55.340 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000611f14347a541ee4-006312f0a3-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.382 DEBUG:boto:b'avysmw-21obj1000false' 2022-09-03T06:13:55.384 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.385 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.386 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.387 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:55.388 DEBUG:boto:Data: 2022-09-03T06:13:55.389 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:55.391 DEBUG:boto:Port: 8001 2022-09-03T06:13:55.392 DEBUG:boto:Params: {} 2022-09-03T06:13:55.393 DEBUG:boto:Token: None 2022-09-03T06:13:55.393 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj 2022-09-03T06:13:55.394 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik= 2022-09-03T06:13:55.395 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik=', 'Content-Length': '0'} 2022-09-03T06:13:55.398 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000ed484bdd15bbca68-006312f0a3-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.399 DEBUG:boto:path=/avysmw-21/obj 2022-09-03T06:13:55.400 DEBUG:boto:auth_path=/avysmw-21/obj 2022-09-03T06:13:55.401 DEBUG:boto:Method: HEAD 2022-09-03T06:13:55.402 DEBUG:boto:Path: /avysmw-21/obj 2022-09-03T06:13:55.403 DEBUG:boto:Data: 2022-09-03T06:13:55.403 DEBUG:boto:Headers: {} 2022-09-03T06:13:55.404 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:55.404 DEBUG:boto:Port: 8000 2022-09-03T06:13:55.405 DEBUG:boto:Params: {} 2022-09-03T06:13:55.406 DEBUG:boto:Token: None 2022-09-03T06:13:55.406 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:55 GMT /avysmw-21/obj 2022-09-03T06:13:55.407 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik= 2022-09-03T06:13:55.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VT6KemA/eh1XHk/c+pGIb/EdXik=', 'Content-Length': '0'} 2022-09-03T06:13:55.417 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000091b1db102087327-006312f0a3-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:55.418 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-21 zones={test-zone3, test-zone2} 2022-09-03T06:13:55.419 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-22 2022-09-03T06:13:55.419 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.420 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.421 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-22 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:55.486 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:55.486 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:55.535 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:55.533+0000 7fd60b47d600 1 bucket sync caught up with source: 2022-09-03T06:13:55.536 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.7.5, , ] 2022-09-03T06:13:55.536 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000002.7.5, , ] 2022-09-03T06:13:55.536 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:55.533+0000 7fd60b47d600 0 bucket checkpoint complete 2022-09-03T06:13:55.558 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-22 2022-09-03T06:13:55.559 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.561 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.562 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-22 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:55.628 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:55.629 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:55.669 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:55.667+0000 7f99e7ebc600 1 bucket sync caught up with source: 2022-09-03T06:13:55.669 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.7.5, , ] 2022-09-03T06:13:55.669 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000002.7.5, , ] 2022-09-03T06:13:55.670 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:55.667+0000 7f99e7ebc600 0 bucket checkpoint complete 2022-09-03T06:13:55.683 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-22 2022-09-03T06:13:55.684 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.685 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.686 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-22 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:55.745 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:55.746 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:55.805 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:55.803+0000 7fbb0a27c600 1 bucket sync caught up with source: 2022-09-03T06:13:55.805 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.7.5, , ] 2022-09-03T06:13:55.806 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000002.7.5, , ] 2022-09-03T06:13:55.806 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:55.803+0000 7fbb0a27c600 0 bucket checkpoint complete 2022-09-03T06:13:55.827 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-22 2022-09-03T06:13:55.828 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.829 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.831 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-22 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:55.891 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:55.892 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:55.929 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:55.925+0000 7f5deff86600 1 bucket sync caught up with source: 2022-09-03T06:13:55.930 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.74.11, , ] 2022-09-03T06:13:55.930 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.74.11, , ] 2022-09-03T06:13:55.931 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:55.925+0000 7f5deff86600 0 bucket checkpoint complete 2022-09-03T06:13:55.941 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-22 2022-09-03T06:13:55.941 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.943 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:55.943 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-22 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:56.001 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:56.001 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:56.046 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:56.042+0000 7fa89c24e600 1 bucket sync caught up with source: 2022-09-03T06:13:56.046 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.74.11, , ] 2022-09-03T06:13:56.046 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.74.11, , ] 2022-09-03T06:13:56.047 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:56.042+0000 7fa89c24e600 0 bucket checkpoint complete 2022-09-03T06:13:56.061 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-22 2022-09-03T06:13:56.062 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-22 2022-09-03T06:13:56.063 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:56.064 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:56.065 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-22 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:56.124 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:56.124 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:57.175 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:57.173+0000 7f5952064600 1 bucket sync caught up with source: 2022-09-03T06:13:57.176 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.59.5, , ] 2022-09-03T06:13:57.176 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000002.59.5, , ] 2022-09-03T06:13:57.176 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:57.173+0000 7f5952064600 0 bucket checkpoint complete 2022-09-03T06:13:57.197 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-22 2022-09-03T06:13:57.199 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:57.200 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-22', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:57.201 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-22 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:57.262 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:57.262 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:57.310 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:57.309+0000 7fa2f35cb600 1 bucket sync caught up with source: 2022-09-03T06:13:57.311 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.59.5, , ] 2022-09-03T06:13:57.311 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000002.59.5, , ] 2022-09-03T06:13:57.311 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:57.309+0000 7fa2f35cb600 0 bucket checkpoint complete 2022-09-03T06:13:57.332 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-22 2022-09-03T06:13:57.333 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-22 zones={test-zone2, test-zone1} 2022-09-03T06:13:57.333 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.335 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.335 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.336 DEBUG:boto:Path: /avysmw-22/ 2022-09-03T06:13:57.337 DEBUG:boto:Data: 2022-09-03T06:13:57.338 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.339 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.340 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.340 DEBUG:boto:Params: {} 2022-09-03T06:13:57.343 DEBUG:boto:Token: None 2022-09-03T06:13:57.343 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc= 2022-09-03T06:13:57.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc=', 'Content-Length': '0'} 2022-09-03T06:13:57.348 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000222c0ece6c50db3-006312f0a5-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.348 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.349 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.349 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.350 DEBUG:boto:Path: /avysmw-22/ 2022-09-03T06:13:57.350 DEBUG:boto:Data: 2022-09-03T06:13:57.351 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.352 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.352 DEBUG:boto:Params: {} 2022-09-03T06:13:57.353 DEBUG:boto:Token: None 2022-09-03T06:13:57.354 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc= 2022-09-03T06:13:57.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc=', 'Content-Length': '0'} 2022-09-03T06:13:57.357 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e0f271ca96533ce8-006312f0a5-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.358 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:13:57.359 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.359 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.360 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.360 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.361 DEBUG:boto:Method: GET 2022-09-03T06:13:57.361 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.362 DEBUG:boto:Data: 2022-09-03T06:13:57.362 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.363 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.363 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.364 DEBUG:boto:Params: {} 2022-09-03T06:13:57.365 DEBUG:boto:Token: None 2022-09-03T06:13:57.365 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/?versions 2022-09-03T06:13:57.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM= 2022-09-03T06:13:57.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM=', 'Content-Length': '0'} 2022-09-03T06:13:57.369 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000333da4f6db3ccc41-006312f0a5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.370 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:57.372 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:57.372 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:57.373 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:13:57.374 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.375 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.376 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.377 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.379 DEBUG:boto:Method: GET 2022-09-03T06:13:57.379 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.380 DEBUG:boto:Data: 2022-09-03T06:13:57.381 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.381 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.382 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.383 DEBUG:boto:Params: {} 2022-09-03T06:13:57.384 DEBUG:boto:Token: None 2022-09-03T06:13:57.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/?versions 2022-09-03T06:13:57.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM= 2022-09-03T06:13:57.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM=', 'Content-Length': '0'} 2022-09-03T06:13:57.389 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001eb1c2dcd619cc04-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.390 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:57.391 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:57.391 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:57.392 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.392 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.393 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.393 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.394 DEBUG:boto:Method: GET 2022-09-03T06:13:57.394 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.395 DEBUG:boto:Data: 2022-09-03T06:13:57.395 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.396 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.396 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.397 DEBUG:boto:Params: {} 2022-09-03T06:13:57.397 DEBUG:boto:Token: None 2022-09-03T06:13:57.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/?versions 2022-09-03T06:13:57.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM= 2022-09-03T06:13:57.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM=', 'Content-Length': '0'} 2022-09-03T06:13:57.402 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aff84005668813c7-006312f0a5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.403 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:57.403 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.404 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.405 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.405 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.406 DEBUG:boto:Method: GET 2022-09-03T06:13:57.406 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.407 DEBUG:boto:Data: 2022-09-03T06:13:57.407 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.408 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.408 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.409 DEBUG:boto:Params: {} 2022-09-03T06:13:57.410 DEBUG:boto:Token: None 2022-09-03T06:13:57.410 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/?versions 2022-09-03T06:13:57.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM= 2022-09-03T06:13:57.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM=', 'Content-Length': '0'} 2022-09-03T06:13:57.414 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002195e5d869f29e2e-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.415 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:57.416 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:57.416 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.417 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.417 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.418 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:57.418 DEBUG:boto:Data: 2022-09-03T06:13:57.419 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.419 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.420 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.420 DEBUG:boto:Params: {} 2022-09-03T06:13:57.421 DEBUG:boto:Token: None 2022-09-03T06:13:57.422 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj 2022-09-03T06:13:57.423 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs= 2022-09-03T06:13:57.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs=', 'Content-Length': '0'} 2022-09-03T06:13:57.425 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000b5c78bf72850a20b-006312f0a5-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.426 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.427 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.427 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.428 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:57.428 DEBUG:boto:Data: 2022-09-03T06:13:57.429 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.430 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.430 DEBUG:boto:Params: {} 2022-09-03T06:13:57.431 DEBUG:boto:Token: None 2022-09-03T06:13:57.431 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj 2022-09-03T06:13:57.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs= 2022-09-03T06:13:57.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs=', 'Content-Length': '0'} 2022-09-03T06:13:57.435 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000006769ed321900d8c5-006312f0a5-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.436 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:57.437 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.437 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.438 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.438 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.439 DEBUG:boto:Method: GET 2022-09-03T06:13:57.439 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.440 DEBUG:boto:Data: 2022-09-03T06:13:57.441 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.441 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.442 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.442 DEBUG:boto:Params: {} 2022-09-03T06:13:57.443 DEBUG:boto:Token: None 2022-09-03T06:13:57.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE= 2022-09-03T06:13:57.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE=', 'Content-Length': '0'} 2022-09-03T06:13:57.447 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006da00384920f8a7d-006312f0a5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.448 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.448 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.449 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.449 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.450 DEBUG:boto:Method: GET 2022-09-03T06:13:57.450 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.451 DEBUG:boto:Data: 2022-09-03T06:13:57.451 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.452 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.452 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.453 DEBUG:boto:Params: {} 2022-09-03T06:13:57.454 DEBUG:boto:Token: None 2022-09-03T06:13:57.454 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE= 2022-09-03T06:13:57.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE=', 'Content-Length': '0'} 2022-09-03T06:13:57.459 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003ecd1d279d36a947-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.460 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.461 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.461 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.462 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.462 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.463 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.463 DEBUG:boto:Data: 2022-09-03T06:13:57.464 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.465 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.465 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.466 DEBUG:boto:Params: {} 2022-09-03T06:13:57.466 DEBUG:boto:Token: None 2022-09-03T06:13:57.467 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xHlkGd7byu0ACAR7ONOyvulXI8c= 2022-09-03T06:13:57.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xHlkGd7byu0ACAR7ONOyvulXI8c=', 'Content-Length': '0'} 2022-09-03T06:13:57.471 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000d593e3886a320a85-006312f0a5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.471 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.472 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.472 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.473 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.474 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.474 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.475 DEBUG:boto:Data: 2022-09-03T06:13:57.475 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.476 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.477 DEBUG:boto:Params: {} 2022-09-03T06:13:57.478 DEBUG:boto:Token: None 2022-09-03T06:13:57.478 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xHlkGd7byu0ACAR7ONOyvulXI8c= 2022-09-03T06:13:57.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xHlkGd7byu0ACAR7ONOyvulXI8c=', 'Content-Length': '0'} 2022-09-03T06:13:57.482 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006705b861b502e299-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.483 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:57.483 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.484 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.484 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.485 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.485 DEBUG:boto:Method: GET 2022-09-03T06:13:57.486 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.487 DEBUG:boto:Data: 2022-09-03T06:13:57.487 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.488 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.489 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.489 DEBUG:boto:Params: {} 2022-09-03T06:13:57.490 DEBUG:boto:Token: None 2022-09-03T06:13:57.491 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE= 2022-09-03T06:13:57.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE=', 'Content-Length': '0'} 2022-09-03T06:13:57.494 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000073f939e00eaa19f3-006312f0a5-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.495 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.496 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.497 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.498 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.499 DEBUG:boto:Method: GET 2022-09-03T06:13:57.500 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.501 DEBUG:boto:Data: 2022-09-03T06:13:57.502 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.503 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.503 DEBUG:boto:Params: {} 2022-09-03T06:13:57.504 DEBUG:boto:Token: None 2022-09-03T06:13:57.505 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE= 2022-09-03T06:13:57.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE=', 'Content-Length': '0'} 2022-09-03T06:13:57.508 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001250ac5f9387068a-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.509 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.510 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.510 DEBUG:boto:path=/avysmw-22/?prefix=obj 2022-09-03T06:13:57.511 DEBUG:boto:auth_path=/avysmw-22/?prefix=obj 2022-09-03T06:13:57.511 DEBUG:boto:Method: GET 2022-09-03T06:13:57.512 DEBUG:boto:Path: /avysmw-22/?prefix=obj 2022-09-03T06:13:57.513 DEBUG:boto:Data: 2022-09-03T06:13:57.513 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.514 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.515 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.516 DEBUG:boto:Params: {} 2022-09-03T06:13:57.517 DEBUG:boto:Token: None 2022-09-03T06:13:57.518 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v7YQpfbvzRo11kVsSpHms5Lmqb8= 2022-09-03T06:13:57.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v7YQpfbvzRo11kVsSpHms5Lmqb8=', 'Content-Length': '0'} 2022-09-03T06:13:57.524 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000007650528d4d78070-006312f0a5-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.566 DEBUG:boto:b'avysmw-22obj1000false' 2022-09-03T06:13:57.567 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.568 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.569 DEBUG:boto:path=/avysmw-22/?prefix=obj 2022-09-03T06:13:57.570 DEBUG:boto:auth_path=/avysmw-22/?prefix=obj 2022-09-03T06:13:57.571 DEBUG:boto:Method: GET 2022-09-03T06:13:57.571 DEBUG:boto:Path: /avysmw-22/?prefix=obj 2022-09-03T06:13:57.572 DEBUG:boto:Data: 2022-09-03T06:13:57.572 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.573 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.573 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.575 DEBUG:boto:Params: {} 2022-09-03T06:13:57.576 DEBUG:boto:Token: None 2022-09-03T06:13:57.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v7YQpfbvzRo11kVsSpHms5Lmqb8= 2022-09-03T06:13:57.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v7YQpfbvzRo11kVsSpHms5Lmqb8=', 'Content-Length': '0'} 2022-09-03T06:13:57.582 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed4ef2f27f281726-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.583 DEBUG:boto:b'avysmw-22obj1000false' 2022-09-03T06:13:57.585 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.586 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.587 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.588 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:57.589 DEBUG:boto:Data: 2022-09-03T06:13:57.590 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.591 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.592 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.593 DEBUG:boto:Params: {} 2022-09-03T06:13:57.594 DEBUG:boto:Token: None 2022-09-03T06:13:57.596 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj 2022-09-03T06:13:57.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs= 2022-09-03T06:13:57.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs=', 'Content-Length': '0'} 2022-09-03T06:13:57.600 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000a6d298f7fd29d7fe-006312f0a5-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.601 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.601 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.603 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.603 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:57.604 DEBUG:boto:Data: 2022-09-03T06:13:57.605 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.606 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.608 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.609 DEBUG:boto:Params: {} 2022-09-03T06:13:57.610 DEBUG:boto:Token: None 2022-09-03T06:13:57.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj 2022-09-03T06:13:57.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs= 2022-09-03T06:13:57.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs=', 'Content-Length': '0'} 2022-09-03T06:13:57.616 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000f7438ccb0a966371-006312f0a5-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.617 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-22 zones={test-zone2, test-zone1} 2022-09-03T06:13:57.618 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-22 zones={test-zone3, test-zone1} 2022-09-03T06:13:57.619 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.620 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.621 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.622 DEBUG:boto:Path: /avysmw-22/ 2022-09-03T06:13:57.623 DEBUG:boto:Data: 2022-09-03T06:13:57.624 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.627 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.627 DEBUG:boto:Params: {} 2022-09-03T06:13:57.628 DEBUG:boto:Token: None 2022-09-03T06:13:57.629 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc= 2022-09-03T06:13:57.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc=', 'Content-Length': '0'} 2022-09-03T06:13:57.634 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000008ebeb71cdad6e1b-006312f0a5-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.636 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.637 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.638 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.639 DEBUG:boto:Path: /avysmw-22/ 2022-09-03T06:13:57.640 DEBUG:boto:Data: 2022-09-03T06:13:57.641 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.642 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.643 DEBUG:boto:Params: {} 2022-09-03T06:13:57.644 DEBUG:boto:Token: None 2022-09-03T06:13:57.644 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc= 2022-09-03T06:13:57.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc=', 'Content-Length': '0'} 2022-09-03T06:13:57.649 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ca33fe188e115c7a-006312f0a5-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.650 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:13:57.651 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.652 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.653 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.654 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.655 DEBUG:boto:Method: GET 2022-09-03T06:13:57.656 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.657 DEBUG:boto:Data: 2022-09-03T06:13:57.658 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.659 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.659 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.661 DEBUG:boto:Params: {} 2022-09-03T06:13:57.661 DEBUG:boto:Token: None 2022-09-03T06:13:57.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/?versions 2022-09-03T06:13:57.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM= 2022-09-03T06:13:57.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM=', 'Content-Length': '0'} 2022-09-03T06:13:57.667 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb177800b7345273-006312f0a5-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.714 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:57.716 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:57.717 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:57.717 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:13:57.718 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.719 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.719 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.720 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.720 DEBUG:boto:Method: GET 2022-09-03T06:13:57.721 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.721 DEBUG:boto:Data: 2022-09-03T06:13:57.722 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.723 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.723 DEBUG:boto:Params: {} 2022-09-03T06:13:57.724 DEBUG:boto:Token: None 2022-09-03T06:13:57.725 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/?versions 2022-09-03T06:13:57.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM= 2022-09-03T06:13:57.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM=', 'Content-Length': '0'} 2022-09-03T06:13:57.730 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000949854776743f501-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.732 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:57.733 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:57.734 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:57.735 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.736 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.736 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.737 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.738 DEBUG:boto:Method: GET 2022-09-03T06:13:57.739 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.740 DEBUG:boto:Data: 2022-09-03T06:13:57.741 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.742 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.743 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.743 DEBUG:boto:Params: {} 2022-09-03T06:13:57.744 DEBUG:boto:Token: None 2022-09-03T06:13:57.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/?versions 2022-09-03T06:13:57.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM= 2022-09-03T06:13:57.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM=', 'Content-Length': '0'} 2022-09-03T06:13:57.749 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000034e7f6107df84bd5-006312f0a5-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.749 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:57.750 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.751 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.751 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.752 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.753 DEBUG:boto:Method: GET 2022-09-03T06:13:57.753 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.754 DEBUG:boto:Data: 2022-09-03T06:13:57.754 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.756 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.756 DEBUG:boto:Params: {} 2022-09-03T06:13:57.757 DEBUG:boto:Token: None 2022-09-03T06:13:57.758 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/?versions 2022-09-03T06:13:57.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM= 2022-09-03T06:13:57.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM=', 'Content-Length': '0'} 2022-09-03T06:13:57.762 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e13e67c60b3bc95-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.762 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:57.763 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:57.764 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.764 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.765 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.765 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:57.766 DEBUG:boto:Data: 2022-09-03T06:13:57.766 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.767 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.767 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.768 DEBUG:boto:Params: {} 2022-09-03T06:13:57.768 DEBUG:boto:Token: None 2022-09-03T06:13:57.769 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj 2022-09-03T06:13:57.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs= 2022-09-03T06:13:57.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs=', 'Content-Length': '0'} 2022-09-03T06:13:57.772 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000000290d959dd17e4a6-006312f0a5-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.773 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.774 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.774 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.775 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:57.775 DEBUG:boto:Data: 2022-09-03T06:13:57.776 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.776 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.777 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.777 DEBUG:boto:Params: {} 2022-09-03T06:13:57.778 DEBUG:boto:Token: None 2022-09-03T06:13:57.779 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj 2022-09-03T06:13:57.779 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs= 2022-09-03T06:13:57.780 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs=', 'Content-Length': '0'} 2022-09-03T06:13:57.783 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000f7ae98b5f0a116cd-006312f0a5-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.784 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:57.785 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.786 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.786 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.787 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.788 DEBUG:boto:Method: GET 2022-09-03T06:13:57.789 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.790 DEBUG:boto:Data: 2022-09-03T06:13:57.791 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.792 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.793 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.794 DEBUG:boto:Params: {} 2022-09-03T06:13:57.795 DEBUG:boto:Token: None 2022-09-03T06:13:57.796 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE= 2022-09-03T06:13:57.798 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE=', 'Content-Length': '0'} 2022-09-03T06:13:57.801 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000023d181568fbf903-006312f0a5-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.803 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.804 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.804 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.805 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.805 DEBUG:boto:Method: GET 2022-09-03T06:13:57.806 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.806 DEBUG:boto:Data: 2022-09-03T06:13:57.807 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.808 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.809 DEBUG:boto:Params: {} 2022-09-03T06:13:57.809 DEBUG:boto:Token: None 2022-09-03T06:13:57.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE= 2022-09-03T06:13:57.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE=', 'Content-Length': '0'} 2022-09-03T06:13:57.813 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000743705283f8cf6a8-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.814 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.815 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.815 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.816 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.816 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.817 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.818 DEBUG:boto:Data: 2022-09-03T06:13:57.818 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.820 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.820 DEBUG:boto:Params: {} 2022-09-03T06:13:57.822 DEBUG:boto:Token: None 2022-09-03T06:13:57.823 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xHlkGd7byu0ACAR7ONOyvulXI8c= 2022-09-03T06:13:57.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xHlkGd7byu0ACAR7ONOyvulXI8c=', 'Content-Length': '0'} 2022-09-03T06:13:57.826 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000003dfe184571df5779-006312f0a5-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.827 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.828 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.828 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.829 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.830 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.830 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.831 DEBUG:boto:Data: 2022-09-03T06:13:57.831 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.833 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.834 DEBUG:boto:Params: {} 2022-09-03T06:13:57.835 DEBUG:boto:Token: None 2022-09-03T06:13:57.836 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xHlkGd7byu0ACAR7ONOyvulXI8c= 2022-09-03T06:13:57.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xHlkGd7byu0ACAR7ONOyvulXI8c=', 'Content-Length': '0'} 2022-09-03T06:13:57.840 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000504cfae11aa30100-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.841 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:57.842 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.842 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.843 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.843 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.844 DEBUG:boto:Method: GET 2022-09-03T06:13:57.844 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.845 DEBUG:boto:Data: 2022-09-03T06:13:57.845 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.847 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.847 DEBUG:boto:Params: {} 2022-09-03T06:13:57.848 DEBUG:boto:Token: None 2022-09-03T06:13:57.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE= 2022-09-03T06:13:57.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE=', 'Content-Length': '0'} 2022-09-03T06:13:57.852 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000002800aa5ee76088ca-006312f0a5-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.853 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.854 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.855 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.855 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.856 DEBUG:boto:Method: GET 2022-09-03T06:13:57.857 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.857 DEBUG:boto:Data: 2022-09-03T06:13:57.858 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.858 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.859 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.859 DEBUG:boto:Params: {} 2022-09-03T06:13:57.860 DEBUG:boto:Token: None 2022-09-03T06:13:57.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:57.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE= 2022-09-03T06:13:57.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qSo+tk7BjEn6Y2NrwxgwF4bRkvE=', 'Content-Length': '0'} 2022-09-03T06:13:57.866 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000569c61a18f2b3c9e-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.867 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.868 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.869 DEBUG:boto:path=/avysmw-22/?prefix=obj 2022-09-03T06:13:57.869 DEBUG:boto:auth_path=/avysmw-22/?prefix=obj 2022-09-03T06:13:57.870 DEBUG:boto:Method: GET 2022-09-03T06:13:57.870 DEBUG:boto:Path: /avysmw-22/?prefix=obj 2022-09-03T06:13:57.871 DEBUG:boto:Data: 2022-09-03T06:13:57.871 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.872 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.873 DEBUG:boto:Params: {} 2022-09-03T06:13:57.873 DEBUG:boto:Token: None 2022-09-03T06:13:57.874 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.875 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v7YQpfbvzRo11kVsSpHms5Lmqb8= 2022-09-03T06:13:57.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v7YQpfbvzRo11kVsSpHms5Lmqb8=', 'Content-Length': '0'} 2022-09-03T06:13:57.878 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1dc2d9c34ff5a6e-006312f0a5-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.879 DEBUG:boto:b'avysmw-22obj1000false' 2022-09-03T06:13:57.880 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.880 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.881 DEBUG:boto:path=/avysmw-22/?prefix=obj 2022-09-03T06:13:57.881 DEBUG:boto:auth_path=/avysmw-22/?prefix=obj 2022-09-03T06:13:57.882 DEBUG:boto:Method: GET 2022-09-03T06:13:57.882 DEBUG:boto:Path: /avysmw-22/?prefix=obj 2022-09-03T06:13:57.883 DEBUG:boto:Data: 2022-09-03T06:13:57.883 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.885 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.885 DEBUG:boto:Params: {} 2022-09-03T06:13:57.886 DEBUG:boto:Token: None 2022-09-03T06:13:57.887 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v7YQpfbvzRo11kVsSpHms5Lmqb8= 2022-09-03T06:13:57.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v7YQpfbvzRo11kVsSpHms5Lmqb8=', 'Content-Length': '0'} 2022-09-03T06:13:57.892 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044d79d8bbbf33a08-006312f0a5-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.934 DEBUG:boto:b'avysmw-22obj1000false' 2022-09-03T06:13:57.935 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.936 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.937 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.938 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:57.939 DEBUG:boto:Data: 2022-09-03T06:13:57.940 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.940 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.941 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.942 DEBUG:boto:Params: {} 2022-09-03T06:13:57.942 DEBUG:boto:Token: None 2022-09-03T06:13:57.943 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj 2022-09-03T06:13:57.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs= 2022-09-03T06:13:57.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs=', 'Content-Length': '0'} 2022-09-03T06:13:57.947 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000039ab96b4f9b8edc5-006312f0a5-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.948 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:57.950 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:57.951 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.952 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:57.953 DEBUG:boto:Data: 2022-09-03T06:13:57.954 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:57.955 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.956 DEBUG:boto:Params: {} 2022-09-03T06:13:57.956 DEBUG:boto:Token: None 2022-09-03T06:13:57.957 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/obj 2022-09-03T06:13:57.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs= 2022-09-03T06:13:57.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N1z+WDZp4Ai9oDTF29trlnu7Vvs=', 'Content-Length': '0'} 2022-09-03T06:13:57.961 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000008994b06d5b4560d1-006312f0a5-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.962 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-22 zones={test-zone3, test-zone1} 2022-09-03T06:13:57.963 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-22 zones={test-zone3, test-zone2} 2022-09-03T06:13:57.964 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.964 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.965 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.965 DEBUG:boto:Path: /avysmw-22/ 2022-09-03T06:13:57.966 DEBUG:boto:Data: 2022-09-03T06:13:57.966 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.967 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.968 DEBUG:boto:Params: {} 2022-09-03T06:13:57.968 DEBUG:boto:Token: None 2022-09-03T06:13:57.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc= 2022-09-03T06:13:57.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc=', 'Content-Length': '0'} 2022-09-03T06:13:57.973 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000575ebe883ce44d2c-006312f0a5-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.975 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.976 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.977 DEBUG:boto:Method: HEAD 2022-09-03T06:13:57.978 DEBUG:boto:Path: /avysmw-22/ 2022-09-03T06:13:57.979 DEBUG:boto:Data: 2022-09-03T06:13:57.980 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.980 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:57.981 DEBUG:boto:Port: 8000 2022-09-03T06:13:57.982 DEBUG:boto:Params: {} 2022-09-03T06:13:57.983 DEBUG:boto:Token: None 2022-09-03T06:13:57.984 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/ 2022-09-03T06:13:57.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc= 2022-09-03T06:13:57.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzWuciNh8YWiVjU+M+gwhO5ndNc=', 'Content-Length': '0'} 2022-09-03T06:13:57.988 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000532469c7aaa9060b-006312f0a5-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:57.988 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:13:57.989 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:57.989 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:57.990 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.990 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.991 DEBUG:boto:Method: GET 2022-09-03T06:13:57.992 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:57.992 DEBUG:boto:Data: 2022-09-03T06:13:57.993 DEBUG:boto:Headers: {} 2022-09-03T06:13:57.993 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:57.994 DEBUG:boto:Port: 8001 2022-09-03T06:13:57.994 DEBUG:boto:Params: {} 2022-09-03T06:13:57.995 DEBUG:boto:Token: None 2022-09-03T06:13:57.995 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:57 GMT /avysmw-22/?versions 2022-09-03T06:13:57.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM= 2022-09-03T06:13:57.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b8SEr4rtwkLxQ3azwzCIG3yy0RM=', 'Content-Length': '0'} 2022-09-03T06:13:58.000 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000075e8aac024a62ed-006312f0a5-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.042 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:58.044 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:58.044 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:58.045 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:13:58.046 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:58.047 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:58.048 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:58.048 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:58.049 DEBUG:boto:Method: GET 2022-09-03T06:13:58.049 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:58.050 DEBUG:boto:Data: 2022-09-03T06:13:58.051 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.051 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:58.053 DEBUG:boto:Port: 8000 2022-09-03T06:13:58.054 DEBUG:boto:Params: {} 2022-09-03T06:13:58.055 DEBUG:boto:Token: None 2022-09-03T06:13:58.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/?versions 2022-09-03T06:13:58.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p9WTBFZLeDLPLmMam0CC+IVYTLo= 2022-09-03T06:13:58.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p9WTBFZLeDLPLmMam0CC+IVYTLo=', 'Content-Length': '0'} 2022-09-03T06:13:58.060 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003463f687cc0d9459-006312f0a6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.061 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:58.062 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:58.063 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:58.064 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:58.065 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:58.065 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:58.066 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:58.067 DEBUG:boto:Method: GET 2022-09-03T06:13:58.068 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:58.068 DEBUG:boto:Data: 2022-09-03T06:13:58.069 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.070 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:58.071 DEBUG:boto:Port: 8001 2022-09-03T06:13:58.072 DEBUG:boto:Params: {} 2022-09-03T06:13:58.073 DEBUG:boto:Token: None 2022-09-03T06:13:58.074 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/?versions 2022-09-03T06:13:58.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p9WTBFZLeDLPLmMam0CC+IVYTLo= 2022-09-03T06:13:58.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p9WTBFZLeDLPLmMam0CC+IVYTLo=', 'Content-Length': '0'} 2022-09-03T06:13:58.079 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3b5ab6ac93dc165-006312f0a6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.080 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:58.081 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:58.081 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:58.082 DEBUG:boto:path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:58.082 DEBUG:boto:auth_path=/avysmw-22/?versions&max-keys=999 2022-09-03T06:13:58.083 DEBUG:boto:Method: GET 2022-09-03T06:13:58.083 DEBUG:boto:Path: /avysmw-22/?versions&max-keys=999 2022-09-03T06:13:58.084 DEBUG:boto:Data: 2022-09-03T06:13:58.084 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.085 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:58.086 DEBUG:boto:Port: 8000 2022-09-03T06:13:58.087 DEBUG:boto:Params: {} 2022-09-03T06:13:58.087 DEBUG:boto:Token: None 2022-09-03T06:13:58.088 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/?versions 2022-09-03T06:13:58.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p9WTBFZLeDLPLmMam0CC+IVYTLo= 2022-09-03T06:13:58.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p9WTBFZLeDLPLmMam0CC+IVYTLo=', 'Content-Length': '0'} 2022-09-03T06:13:58.092 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031657824b3d01e03-006312f0a6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.093 DEBUG:boto:b'avysmw-22999falseobjoQrpTowIjTMzXOFDN9840QWSjpnmf.strue2022-09-03T06:11:23.396Zrgw-multisite-test-userMultisite Test UserNormalobjU6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UCfalse2022-09-03T06:11:23.364Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:58.094 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:58.095 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.096 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.097 DEBUG:boto:Method: HEAD 2022-09-03T06:13:58.098 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:58.099 DEBUG:boto:Data: 2022-09-03T06:13:58.100 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.101 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:58.102 DEBUG:boto:Port: 8001 2022-09-03T06:13:58.103 DEBUG:boto:Params: {} 2022-09-03T06:13:58.104 DEBUG:boto:Token: None 2022-09-03T06:13:58.105 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj 2022-09-03T06:13:58.106 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5haa8mMzwJCGzC9oHassDXEdqH0= 2022-09-03T06:13:58.107 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5haa8mMzwJCGzC9oHassDXEdqH0=', 'Content-Length': '0'} 2022-09-03T06:13:58.112 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000aa8be1f76113b42b-006312f0a6-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.113 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.113 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.114 DEBUG:boto:Method: HEAD 2022-09-03T06:13:58.115 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:58.115 DEBUG:boto:Data: 2022-09-03T06:13:58.116 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.116 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:58.117 DEBUG:boto:Port: 8000 2022-09-03T06:13:58.117 DEBUG:boto:Params: {} 2022-09-03T06:13:58.118 DEBUG:boto:Token: None 2022-09-03T06:13:58.119 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj 2022-09-03T06:13:58.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5haa8mMzwJCGzC9oHassDXEdqH0= 2022-09-03T06:13:58.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5haa8mMzwJCGzC9oHassDXEdqH0=', 'Content-Length': '0'} 2022-09-03T06:13:58.122 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000009e0c4425766d50ff-006312f0a6-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.123 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:58.124 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.125 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.126 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.127 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.128 DEBUG:boto:Method: GET 2022-09-03T06:13:58.129 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.131 DEBUG:boto:Data: 2022-09-03T06:13:58.132 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:58.133 DEBUG:boto:Port: 8001 2022-09-03T06:13:58.134 DEBUG:boto:Params: {} 2022-09-03T06:13:58.135 DEBUG:boto:Token: None 2022-09-03T06:13:58.136 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SRAvSYbFansO05NkMLmATUMG3wc= 2022-09-03T06:13:58.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SRAvSYbFansO05NkMLmATUMG3wc=', 'Content-Length': '0'} 2022-09-03T06:13:58.141 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000006c2ee6fb45c435-006312f0a6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.142 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.143 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.144 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.144 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.145 DEBUG:boto:Method: GET 2022-09-03T06:13:58.147 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.148 DEBUG:boto:Data: 2022-09-03T06:13:58.149 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.149 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:58.150 DEBUG:boto:Port: 8000 2022-09-03T06:13:58.151 DEBUG:boto:Params: {} 2022-09-03T06:13:58.153 DEBUG:boto:Token: None 2022-09-03T06:13:58.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SRAvSYbFansO05NkMLmATUMG3wc= 2022-09-03T06:13:58.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SRAvSYbFansO05NkMLmATUMG3wc=', 'Content-Length': '0'} 2022-09-03T06:13:58.160 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000377a62a0a9d9a1be-006312f0a6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.161 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.162 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.163 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.164 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.165 DEBUG:boto:Method: HEAD 2022-09-03T06:13:58.165 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.166 DEBUG:boto:Data: 2022-09-03T06:13:58.167 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:58.168 DEBUG:boto:Port: 8001 2022-09-03T06:13:58.168 DEBUG:boto:Params: {} 2022-09-03T06:13:58.169 DEBUG:boto:Token: None 2022-09-03T06:13:58.170 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jL6IcXao/BJ0DR4igrI91lDiouY= 2022-09-03T06:13:58.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jL6IcXao/BJ0DR4igrI91lDiouY=', 'Content-Length': '0'} 2022-09-03T06:13:58.174 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000001d7f57edb0ce94b4-006312f0a6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.175 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.176 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.177 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.178 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.179 DEBUG:boto:Method: HEAD 2022-09-03T06:13:58.179 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.180 DEBUG:boto:Data: 2022-09-03T06:13:58.180 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.181 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:58.181 DEBUG:boto:Port: 8000 2022-09-03T06:13:58.182 DEBUG:boto:Params: {} 2022-09-03T06:13:58.182 DEBUG:boto:Token: None 2022-09-03T06:13:58.183 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jL6IcXao/BJ0DR4igrI91lDiouY= 2022-09-03T06:13:58.184 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jL6IcXao/BJ0DR4igrI91lDiouY=', 'Content-Length': '0'} 2022-09-03T06:13:58.186 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000005fb9bbf4b6e01c9b-006312f0a6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.187 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:58.188 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.188 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.189 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.189 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.190 DEBUG:boto:Method: GET 2022-09-03T06:13:58.190 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.191 DEBUG:boto:Data: 2022-09-03T06:13:58.191 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:58.192 DEBUG:boto:Port: 8001 2022-09-03T06:13:58.193 DEBUG:boto:Params: {} 2022-09-03T06:13:58.193 DEBUG:boto:Token: None 2022-09-03T06:13:58.194 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SRAvSYbFansO05NkMLmATUMG3wc= 2022-09-03T06:13:58.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SRAvSYbFansO05NkMLmATUMG3wc=', 'Content-Length': '0'} 2022-09-03T06:13:58.198 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000002436aaf9f14c02b5-006312f0a6-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.198 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.199 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.199 DEBUG:boto:path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.200 DEBUG:boto:auth_path=/avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.201 DEBUG:boto:Method: GET 2022-09-03T06:13:58.201 DEBUG:boto:Path: /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.202 DEBUG:boto:Data: 2022-09-03T06:13:58.202 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.203 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:58.207 DEBUG:boto:Port: 8000 2022-09-03T06:13:58.207 DEBUG:boto:Params: {} 2022-09-03T06:13:58.208 DEBUG:boto:Token: None 2022-09-03T06:13:58.208 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj?versionId=U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC 2022-09-03T06:13:58.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SRAvSYbFansO05NkMLmATUMG3wc= 2022-09-03T06:13:58.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SRAvSYbFansO05NkMLmATUMG3wc=', 'Content-Length': '0'} 2022-09-03T06:13:58.212 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'U6cmvQpLM8HZ1QG2KD1wUtt7Tdtl2UC'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000df8c4c60e9a72cd4-006312f0a6-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.212 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:58.213 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:58.214 DEBUG:boto:path=/avysmw-22/?prefix=obj 2022-09-03T06:13:58.214 DEBUG:boto:auth_path=/avysmw-22/?prefix=obj 2022-09-03T06:13:58.215 DEBUG:boto:Method: GET 2022-09-03T06:13:58.215 DEBUG:boto:Path: /avysmw-22/?prefix=obj 2022-09-03T06:13:58.216 DEBUG:boto:Data: 2022-09-03T06:13:58.216 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:58.217 DEBUG:boto:Port: 8001 2022-09-03T06:13:58.218 DEBUG:boto:Params: {} 2022-09-03T06:13:58.218 DEBUG:boto:Token: None 2022-09-03T06:13:58.219 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/ 2022-09-03T06:13:58.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0TtVu9o3RQ7Nrs9yhrsgqRxVluA= 2022-09-03T06:13:58.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0TtVu9o3RQ7Nrs9yhrsgqRxVluA=', 'Content-Length': '0'} 2022-09-03T06:13:58.223 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004849420761f83d69-006312f0a6-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.270 DEBUG:boto:b'avysmw-22obj1000false' 2022-09-03T06:13:58.271 DEBUG:boto:path=/avysmw-22/ 2022-09-03T06:13:58.272 DEBUG:boto:auth_path=/avysmw-22/ 2022-09-03T06:13:58.272 DEBUG:boto:path=/avysmw-22/?prefix=obj 2022-09-03T06:13:58.273 DEBUG:boto:auth_path=/avysmw-22/?prefix=obj 2022-09-03T06:13:58.273 DEBUG:boto:Method: GET 2022-09-03T06:13:58.274 DEBUG:boto:Path: /avysmw-22/?prefix=obj 2022-09-03T06:13:58.275 DEBUG:boto:Data: 2022-09-03T06:13:58.275 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.276 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:58.276 DEBUG:boto:Port: 8000 2022-09-03T06:13:58.277 DEBUG:boto:Params: {} 2022-09-03T06:13:58.277 DEBUG:boto:Token: None 2022-09-03T06:13:58.278 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/ 2022-09-03T06:13:58.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0TtVu9o3RQ7Nrs9yhrsgqRxVluA= 2022-09-03T06:13:58.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0TtVu9o3RQ7Nrs9yhrsgqRxVluA=', 'Content-Length': '0'} 2022-09-03T06:13:58.282 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087b216f3db3b939d-006312f0a6-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.283 DEBUG:boto:b'avysmw-22obj1000false' 2022-09-03T06:13:58.283 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.284 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.284 DEBUG:boto:Method: HEAD 2022-09-03T06:13:58.285 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:58.285 DEBUG:boto:Data: 2022-09-03T06:13:58.286 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.286 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:58.287 DEBUG:boto:Port: 8001 2022-09-03T06:13:58.288 DEBUG:boto:Params: {} 2022-09-03T06:13:58.288 DEBUG:boto:Token: None 2022-09-03T06:13:58.289 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj 2022-09-03T06:13:58.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5haa8mMzwJCGzC9oHassDXEdqH0= 2022-09-03T06:13:58.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5haa8mMzwJCGzC9oHassDXEdqH0=', 'Content-Length': '0'} 2022-09-03T06:13:58.292 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000028728ccacf28fed1-006312f0a6-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.293 DEBUG:boto:path=/avysmw-22/obj 2022-09-03T06:13:58.293 DEBUG:boto:auth_path=/avysmw-22/obj 2022-09-03T06:13:58.294 DEBUG:boto:Method: HEAD 2022-09-03T06:13:58.295 DEBUG:boto:Path: /avysmw-22/obj 2022-09-03T06:13:58.295 DEBUG:boto:Data: 2022-09-03T06:13:58.296 DEBUG:boto:Headers: {} 2022-09-03T06:13:58.296 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:58.296 DEBUG:boto:Port: 8000 2022-09-03T06:13:58.297 DEBUG:boto:Params: {} 2022-09-03T06:13:58.297 DEBUG:boto:Token: None 2022-09-03T06:13:58.298 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:58 GMT /avysmw-22/obj 2022-09-03T06:13:58.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5haa8mMzwJCGzC9oHassDXEdqH0= 2022-09-03T06:13:58.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5haa8mMzwJCGzC9oHassDXEdqH0=', 'Content-Length': '0'} 2022-09-03T06:13:58.301 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000d850dc5ee827bed5-006312f0a6-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:58.302 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-22 zones={test-zone3, test-zone2} 2022-09-03T06:13:58.302 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-23 2022-09-03T06:13:58.303 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.303 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.304 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-23 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:58.362 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:58.363 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:58.398 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:58.396+0000 7fc987644600 1 bucket sync caught up with source: 2022-09-03T06:13:58.398 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.23.5, , ] 2022-09-03T06:13:58.399 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000002.23.5, , ] 2022-09-03T06:13:58.399 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:58.396+0000 7fc987644600 0 bucket checkpoint complete 2022-09-03T06:13:58.411 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-23 2022-09-03T06:13:58.412 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.413 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.414 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-23 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:58.481 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:58.482 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:58.526 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:58.524+0000 7f343d5ac600 1 bucket sync caught up with source: 2022-09-03T06:13:58.526 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.23.5, , ] 2022-09-03T06:13:58.527 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000002.23.5, , ] 2022-09-03T06:13:58.527 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:58.524+0000 7f343d5ac600 0 bucket checkpoint complete 2022-09-03T06:13:58.539 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-23 2022-09-03T06:13:58.540 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.541 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.541 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-23 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:58.598 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:58.598 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:58.634 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:58.632+0000 7fe8ec3b0600 1 bucket sync caught up with source: 2022-09-03T06:13:58.634 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000002.23.5, , ] 2022-09-03T06:13:58.635 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000002.23.5, , ] 2022-09-03T06:13:58.635 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:58.632+0000 7fe8ec3b0600 0 bucket checkpoint complete 2022-09-03T06:13:58.655 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-23 2022-09-03T06:13:58.656 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.657 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.658 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-23 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:58.718 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:58.718 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:58.755 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:58.753+0000 7f1b5c3fc600 1 bucket sync caught up with source: 2022-09-03T06:13:58.756 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.85.5, , ] 2022-09-03T06:13:58.756 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000002.85.5, , ] 2022-09-03T06:13:58.757 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:58.753+0000 7f1b5c3fc600 0 bucket checkpoint complete 2022-09-03T06:13:58.775 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-23 2022-09-03T06:13:58.775 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.776 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.777 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-23 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:58.834 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:58.835 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:58.891 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:58.885+0000 7f45546b6600 1 bucket sync caught up with source: 2022-09-03T06:13:58.891 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000002.85.5, , ] 2022-09-03T06:13:58.891 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000002.85.5, , ] 2022-09-03T06:13:58.892 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:58.885+0000 7f45546b6600 0 bucket checkpoint complete 2022-09-03T06:13:58.907 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-23 2022-09-03T06:13:58.907 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-23 2022-09-03T06:13:58.908 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.909 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:58.909 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-23 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:58.968 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:58.969 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:59.023 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:59.021+0000 7ff1b6909600 1 bucket sync caught up with source: 2022-09-03T06:13:59.023 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.72.11, , ] 2022-09-03T06:13:59.024 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.72.11, , ] 2022-09-03T06:13:59.024 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:13:59.021+0000 7ff1b6909600 0 bucket checkpoint complete 2022-09-03T06:13:59.041 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-23 2022-09-03T06:13:59.042 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:59.043 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-23', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:13:59.044 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-23 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:13:59.104 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:13:59.104 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:13:59.173 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:59.171+0000 7fba08dec600 1 bucket sync caught up with source: 2022-09-03T06:13:59.173 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.72.11, , ] 2022-09-03T06:13:59.173 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.72.11, , ] 2022-09-03T06:13:59.174 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:13:59.171+0000 7fba08dec600 0 bucket checkpoint complete 2022-09-03T06:13:59.200 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-23 2022-09-03T06:13:59.201 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-23 zones={test-zone2, test-zone1} 2022-09-03T06:13:59.201 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.202 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.203 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.204 DEBUG:boto:Path: /avysmw-23/ 2022-09-03T06:13:59.205 DEBUG:boto:Data: 2022-09-03T06:13:59.205 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.206 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.208 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.208 DEBUG:boto:Params: {} 2022-09-03T06:13:59.209 DEBUG:boto:Token: None 2022-09-03T06:13:59.209 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI= 2022-09-03T06:13:59.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI=', 'Content-Length': '0'} 2022-09-03T06:13:59.214 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000060f832dc09c6f34d-006312f0a7-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.214 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.215 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.216 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.216 DEBUG:boto:Path: /avysmw-23/ 2022-09-03T06:13:59.217 DEBUG:boto:Data: 2022-09-03T06:13:59.217 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.218 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.219 DEBUG:boto:Params: {} 2022-09-03T06:13:59.220 DEBUG:boto:Token: None 2022-09-03T06:13:59.220 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI= 2022-09-03T06:13:59.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI=', 'Content-Length': '0'} 2022-09-03T06:13:59.224 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c0e896cdc86a1643-006312f0a7-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.225 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:13:59.225 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.226 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.227 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.227 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.228 DEBUG:boto:Method: GET 2022-09-03T06:13:59.229 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.230 DEBUG:boto:Data: 2022-09-03T06:13:59.230 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.231 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.231 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.232 DEBUG:boto:Params: {} 2022-09-03T06:13:59.232 DEBUG:boto:Token: None 2022-09-03T06:13:59.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.238 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a1506ebbc9eca75a-006312f0a7-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.239 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.241 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.242 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.242 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:13:59.243 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.245 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.245 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.246 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.247 DEBUG:boto:Method: GET 2022-09-03T06:13:59.248 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.249 DEBUG:boto:Data: 2022-09-03T06:13:59.250 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.252 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.252 DEBUG:boto:Params: {} 2022-09-03T06:13:59.253 DEBUG:boto:Token: None 2022-09-03T06:13:59.253 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.257 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004fd39b98dbcb9db4-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.258 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.259 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.260 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.260 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.261 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.262 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.263 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.264 DEBUG:boto:Method: GET 2022-09-03T06:13:59.264 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.265 DEBUG:boto:Data: 2022-09-03T06:13:59.266 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.268 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.269 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.270 DEBUG:boto:Params: {} 2022-09-03T06:13:59.271 DEBUG:boto:Token: None 2022-09-03T06:13:59.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.277 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e918cf4d93445622-006312f0a7-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.279 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.280 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.281 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.282 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.283 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.283 DEBUG:boto:Method: GET 2022-09-03T06:13:59.284 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.284 DEBUG:boto:Data: 2022-09-03T06:13:59.285 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.286 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.286 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.287 DEBUG:boto:Params: {} 2022-09-03T06:13:59.288 DEBUG:boto:Token: None 2022-09-03T06:13:59.289 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.290 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.294 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b41c53778ee90cfa-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.295 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.295 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:59.296 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.297 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.297 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.298 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.298 DEBUG:boto:Data: 2022-09-03T06:13:59.299 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.301 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.302 DEBUG:boto:Params: {} 2022-09-03T06:13:59.303 DEBUG:boto:Token: None 2022-09-03T06:13:59.304 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.310 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000cefc85bf0d72cb03-006312f0a7-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.311 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.312 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.313 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.315 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.316 DEBUG:boto:Data: 2022-09-03T06:13:59.317 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.318 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.318 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.319 DEBUG:boto:Params: {} 2022-09-03T06:13:59.320 DEBUG:boto:Token: None 2022-09-03T06:13:59.321 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.326 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000847acfed23e8b4f0-006312f0a7-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.327 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:59.327 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.328 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.329 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.329 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.330 DEBUG:boto:Method: GET 2022-09-03T06:13:59.334 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.335 DEBUG:boto:Data: 2022-09-03T06:13:59.335 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.336 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.336 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.337 DEBUG:boto:Params: {} 2022-09-03T06:13:59.338 DEBUG:boto:Token: None 2022-09-03T06:13:59.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.343 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000065bda425f3d29596-006312f0a7-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.344 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.344 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.345 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.345 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.346 DEBUG:boto:Method: GET 2022-09-03T06:13:59.347 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.347 DEBUG:boto:Data: 2022-09-03T06:13:59.348 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.348 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.349 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.349 DEBUG:boto:Params: {} 2022-09-03T06:13:59.350 DEBUG:boto:Token: None 2022-09-03T06:13:59.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.354 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000a3bc46ec2cb9f95b-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.355 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.355 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.356 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.356 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.357 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.357 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.358 DEBUG:boto:Data: 2022-09-03T06:13:59.358 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.359 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.359 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.360 DEBUG:boto:Params: {} 2022-09-03T06:13:59.361 DEBUG:boto:Token: None 2022-09-03T06:13:59.361 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo= 2022-09-03T06:13:59.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo=', 'Content-Length': '0'} 2022-09-03T06:13:59.364 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000965e3bba2fd01931-006312f0a7-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.365 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.366 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.366 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.367 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.368 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.369 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.369 DEBUG:boto:Data: 2022-09-03T06:13:59.370 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.371 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.371 DEBUG:boto:Params: {} 2022-09-03T06:13:59.372 DEBUG:boto:Token: None 2022-09-03T06:13:59.372 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo= 2022-09-03T06:13:59.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo=', 'Content-Length': '0'} 2022-09-03T06:13:59.376 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009d80fb1e07cb5dc5-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.376 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:59.377 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.378 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.378 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.379 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.379 DEBUG:boto:Method: GET 2022-09-03T06:13:59.380 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.380 DEBUG:boto:Data: 2022-09-03T06:13:59.381 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.381 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.382 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.382 DEBUG:boto:Params: {} 2022-09-03T06:13:59.383 DEBUG:boto:Token: None 2022-09-03T06:13:59.384 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.387 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000002a663d1f92d57864-006312f0a7-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.388 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.388 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.389 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.389 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.390 DEBUG:boto:Method: GET 2022-09-03T06:13:59.390 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.391 DEBUG:boto:Data: 2022-09-03T06:13:59.392 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.393 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.394 DEBUG:boto:Params: {} 2022-09-03T06:13:59.395 DEBUG:boto:Token: None 2022-09-03T06:13:59.396 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.401 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000076a2346fcbba3a4e-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.402 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.403 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.403 DEBUG:boto:path=/avysmw-23/?prefix=obj 2022-09-03T06:13:59.404 DEBUG:boto:auth_path=/avysmw-23/?prefix=obj 2022-09-03T06:13:59.404 DEBUG:boto:Method: GET 2022-09-03T06:13:59.405 DEBUG:boto:Path: /avysmw-23/?prefix=obj 2022-09-03T06:13:59.405 DEBUG:boto:Data: 2022-09-03T06:13:59.406 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.406 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.407 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.408 DEBUG:boto:Params: {} 2022-09-03T06:13:59.409 DEBUG:boto:Token: None 2022-09-03T06:13:59.410 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9BWpPBoBYcP+HH6F/iHGujqT4Rc= 2022-09-03T06:13:59.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9BWpPBoBYcP+HH6F/iHGujqT4Rc=', 'Content-Length': '0'} 2022-09-03T06:13:59.414 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003d7ebea322318823-006312f0a7-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.458 DEBUG:boto:b'avysmw-23obj1000false' 2022-09-03T06:13:59.459 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.460 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.460 DEBUG:boto:path=/avysmw-23/?prefix=obj 2022-09-03T06:13:59.461 DEBUG:boto:auth_path=/avysmw-23/?prefix=obj 2022-09-03T06:13:59.461 DEBUG:boto:Method: GET 2022-09-03T06:13:59.462 DEBUG:boto:Path: /avysmw-23/?prefix=obj 2022-09-03T06:13:59.463 DEBUG:boto:Data: 2022-09-03T06:13:59.463 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.464 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.464 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.465 DEBUG:boto:Params: {} 2022-09-03T06:13:59.465 DEBUG:boto:Token: None 2022-09-03T06:13:59.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9BWpPBoBYcP+HH6F/iHGujqT4Rc= 2022-09-03T06:13:59.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9BWpPBoBYcP+HH6F/iHGujqT4Rc=', 'Content-Length': '0'} 2022-09-03T06:13:59.470 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006464fee933a49992-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.473 DEBUG:boto:b'avysmw-23obj1000false' 2022-09-03T06:13:59.474 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.474 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.475 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.476 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.477 DEBUG:boto:Data: 2022-09-03T06:13:59.478 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.479 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.480 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.481 DEBUG:boto:Params: {} 2022-09-03T06:13:59.482 DEBUG:boto:Token: None 2022-09-03T06:13:59.483 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.487 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000f32be491c0aa93c5-006312f0a7-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.488 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.489 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.489 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.490 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.491 DEBUG:boto:Data: 2022-09-03T06:13:59.491 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.492 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.492 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.493 DEBUG:boto:Params: {} 2022-09-03T06:13:59.494 DEBUG:boto:Token: None 2022-09-03T06:13:59.495 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.499 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000ffcc891d9b4dd531-006312f0a7-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.499 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-23 zones={test-zone2, test-zone1} 2022-09-03T06:13:59.500 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-23 zones={test-zone3, test-zone1} 2022-09-03T06:13:59.501 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.501 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.502 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.502 DEBUG:boto:Path: /avysmw-23/ 2022-09-03T06:13:59.503 DEBUG:boto:Data: 2022-09-03T06:13:59.504 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.504 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.504 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.505 DEBUG:boto:Params: {} 2022-09-03T06:13:59.506 DEBUG:boto:Token: None 2022-09-03T06:13:59.506 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI= 2022-09-03T06:13:59.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI=', 'Content-Length': '0'} 2022-09-03T06:13:59.511 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a7a13a71f585f416-006312f0a7-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.513 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.513 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.514 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.514 DEBUG:boto:Path: /avysmw-23/ 2022-09-03T06:13:59.515 DEBUG:boto:Data: 2022-09-03T06:13:59.515 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.516 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.517 DEBUG:boto:Params: {} 2022-09-03T06:13:59.518 DEBUG:boto:Token: None 2022-09-03T06:13:59.518 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI= 2022-09-03T06:13:59.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI=', 'Content-Length': '0'} 2022-09-03T06:13:59.522 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001dc9801eb8cf462a-006312f0a7-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.523 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:13:59.524 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.524 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.525 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.525 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.526 DEBUG:boto:Method: GET 2022-09-03T06:13:59.526 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.527 DEBUG:boto:Data: 2022-09-03T06:13:59.527 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.528 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.528 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.529 DEBUG:boto:Params: {} 2022-09-03T06:13:59.529 DEBUG:boto:Token: None 2022-09-03T06:13:59.530 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.533 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000edb9c943c9b5bfa3-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.534 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.535 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.535 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.536 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:13:59.536 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.537 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.537 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.538 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.538 DEBUG:boto:Method: GET 2022-09-03T06:13:59.539 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.539 DEBUG:boto:Data: 2022-09-03T06:13:59.540 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.541 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.541 DEBUG:boto:Params: {} 2022-09-03T06:13:59.542 DEBUG:boto:Token: None 2022-09-03T06:13:59.542 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.546 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3550b9f9b8b63dd-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.594 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.595 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.596 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.596 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.598 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.598 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.599 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.599 DEBUG:boto:Method: GET 2022-09-03T06:13:59.600 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.600 DEBUG:boto:Data: 2022-09-03T06:13:59.601 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.601 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.602 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.602 DEBUG:boto:Params: {} 2022-09-03T06:13:59.603 DEBUG:boto:Token: None 2022-09-03T06:13:59.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.608 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000791a9ba11f5665a6-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.609 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.610 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.611 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.611 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.612 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.612 DEBUG:boto:Method: GET 2022-09-03T06:13:59.613 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.613 DEBUG:boto:Data: 2022-09-03T06:13:59.614 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.615 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.615 DEBUG:boto:Params: {} 2022-09-03T06:13:59.616 DEBUG:boto:Token: None 2022-09-03T06:13:59.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.621 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f78866d05a0e39cf-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.622 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.623 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:59.623 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.624 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.624 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.625 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.625 DEBUG:boto:Data: 2022-09-03T06:13:59.626 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.627 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.627 DEBUG:boto:Params: {} 2022-09-03T06:13:59.628 DEBUG:boto:Token: None 2022-09-03T06:13:59.629 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.633 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000019ca5fe7039349e0-006312f0a7-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.634 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.634 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.635 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.635 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.636 DEBUG:boto:Data: 2022-09-03T06:13:59.636 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.637 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.638 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.639 DEBUG:boto:Params: {} 2022-09-03T06:13:59.639 DEBUG:boto:Token: None 2022-09-03T06:13:59.640 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.644 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000008780d0f2962ad421-006312f0a7-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.644 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:59.645 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.645 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.646 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.647 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.647 DEBUG:boto:Method: GET 2022-09-03T06:13:59.648 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.648 DEBUG:boto:Data: 2022-09-03T06:13:59.649 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.650 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.650 DEBUG:boto:Params: {} 2022-09-03T06:13:59.651 DEBUG:boto:Token: None 2022-09-03T06:13:59.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.657 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000009f7abfb6e9987f06-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.659 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.659 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.660 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.660 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.661 DEBUG:boto:Method: GET 2022-09-03T06:13:59.662 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.662 DEBUG:boto:Data: 2022-09-03T06:13:59.663 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.663 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.664 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.664 DEBUG:boto:Params: {} 2022-09-03T06:13:59.665 DEBUG:boto:Token: None 2022-09-03T06:13:59.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.670 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e1414ada86312bd9-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.671 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.672 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.673 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.674 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.675 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.675 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.676 DEBUG:boto:Data: 2022-09-03T06:13:59.676 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.678 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.679 DEBUG:boto:Params: {} 2022-09-03T06:13:59.680 DEBUG:boto:Token: None 2022-09-03T06:13:59.681 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.682 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo= 2022-09-03T06:13:59.683 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo=', 'Content-Length': '0'} 2022-09-03T06:13:59.686 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx000006b185a126b2336f1-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.688 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.689 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.690 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.690 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.692 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.693 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.695 DEBUG:boto:Data: 2022-09-03T06:13:59.695 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.696 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.697 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.697 DEBUG:boto:Params: {} 2022-09-03T06:13:59.698 DEBUG:boto:Token: None 2022-09-03T06:13:59.700 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.701 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo= 2022-09-03T06:13:59.702 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo=', 'Content-Length': '0'} 2022-09-03T06:13:59.704 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000b0866f55d1d7f31e-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.706 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:59.707 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.708 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.708 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.710 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.711 DEBUG:boto:Method: GET 2022-09-03T06:13:59.711 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.712 DEBUG:boto:Data: 2022-09-03T06:13:59.713 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.715 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.716 DEBUG:boto:Params: {} 2022-09-03T06:13:59.717 DEBUG:boto:Token: None 2022-09-03T06:13:59.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.723 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000e689796f7ce4e756-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.724 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.725 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.726 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.726 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.727 DEBUG:boto:Method: GET 2022-09-03T06:13:59.728 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.729 DEBUG:boto:Data: 2022-09-03T06:13:59.730 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.732 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.732 DEBUG:boto:Params: {} 2022-09-03T06:13:59.734 DEBUG:boto:Token: None 2022-09-03T06:13:59.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.739 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000059db632d3d2c566b-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.740 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.741 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.741 DEBUG:boto:path=/avysmw-23/?prefix=obj 2022-09-03T06:13:59.742 DEBUG:boto:auth_path=/avysmw-23/?prefix=obj 2022-09-03T06:13:59.743 DEBUG:boto:Method: GET 2022-09-03T06:13:59.744 DEBUG:boto:Path: /avysmw-23/?prefix=obj 2022-09-03T06:13:59.745 DEBUG:boto:Data: 2022-09-03T06:13:59.746 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.748 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.748 DEBUG:boto:Params: {} 2022-09-03T06:13:59.749 DEBUG:boto:Token: None 2022-09-03T06:13:59.750 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9BWpPBoBYcP+HH6F/iHGujqT4Rc= 2022-09-03T06:13:59.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9BWpPBoBYcP+HH6F/iHGujqT4Rc=', 'Content-Length': '0'} 2022-09-03T06:13:59.762 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2dcb68765378272-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.806 DEBUG:boto:b'avysmw-23obj1000false' 2022-09-03T06:13:59.808 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.808 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.809 DEBUG:boto:path=/avysmw-23/?prefix=obj 2022-09-03T06:13:59.810 DEBUG:boto:auth_path=/avysmw-23/?prefix=obj 2022-09-03T06:13:59.811 DEBUG:boto:Method: GET 2022-09-03T06:13:59.812 DEBUG:boto:Path: /avysmw-23/?prefix=obj 2022-09-03T06:13:59.813 DEBUG:boto:Data: 2022-09-03T06:13:59.814 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.815 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.816 DEBUG:boto:Params: {} 2022-09-03T06:13:59.817 DEBUG:boto:Token: None 2022-09-03T06:13:59.818 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9BWpPBoBYcP+HH6F/iHGujqT4Rc= 2022-09-03T06:13:59.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9BWpPBoBYcP+HH6F/iHGujqT4Rc=', 'Content-Length': '0'} 2022-09-03T06:13:59.824 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9b6d284fc965940-006312f0a7-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.825 DEBUG:boto:b'avysmw-23obj1000false' 2022-09-03T06:13:59.826 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.827 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.828 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.828 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.829 DEBUG:boto:Data: 2022-09-03T06:13:59.830 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.832 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.833 DEBUG:boto:Params: {} 2022-09-03T06:13:59.835 DEBUG:boto:Token: None 2022-09-03T06:13:59.836 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.840 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000ef7220cb9dc96511-006312f0a7-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.842 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.843 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.844 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.844 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.845 DEBUG:boto:Data: 2022-09-03T06:13:59.846 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:13:59.847 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.848 DEBUG:boto:Params: {} 2022-09-03T06:13:59.849 DEBUG:boto:Token: None 2022-09-03T06:13:59.850 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.856 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000001f2c34a8a1ba157a-006312f0a7-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.857 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-23 zones={test-zone3, test-zone1} 2022-09-03T06:13:59.858 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-23 zones={test-zone3, test-zone2} 2022-09-03T06:13:59.859 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.859 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.860 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.861 DEBUG:boto:Path: /avysmw-23/ 2022-09-03T06:13:59.862 DEBUG:boto:Data: 2022-09-03T06:13:59.863 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.865 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.866 DEBUG:boto:Params: {} 2022-09-03T06:13:59.867 DEBUG:boto:Token: None 2022-09-03T06:13:59.868 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI= 2022-09-03T06:13:59.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI=', 'Content-Length': '0'} 2022-09-03T06:13:59.873 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000065f434c16f97f541-006312f0a7-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.875 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.875 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.876 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.876 DEBUG:boto:Path: /avysmw-23/ 2022-09-03T06:13:59.877 DEBUG:boto:Data: 2022-09-03T06:13:59.877 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.878 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.879 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.879 DEBUG:boto:Params: {} 2022-09-03T06:13:59.880 DEBUG:boto:Token: None 2022-09-03T06:13:59.880 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/ 2022-09-03T06:13:59.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI= 2022-09-03T06:13:59.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttFvt8WIXs1Bsgo6bH9CW01SoaI=', 'Content-Length': '0'} 2022-09-03T06:13:59.884 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000983d0ca6bf2184f9-006312f0a7-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.885 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:13:59.885 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.886 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.887 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.887 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.888 DEBUG:boto:Method: GET 2022-09-03T06:13:59.888 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.889 DEBUG:boto:Data: 2022-09-03T06:13:59.890 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.891 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.892 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.892 DEBUG:boto:Params: {} 2022-09-03T06:13:59.893 DEBUG:boto:Token: None 2022-09-03T06:13:59.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.898 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004d68de778a28cb0a-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.900 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.901 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.902 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.903 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:13:59.904 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.905 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.905 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.906 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.907 DEBUG:boto:Method: GET 2022-09-03T06:13:59.907 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.908 DEBUG:boto:Data: 2022-09-03T06:13:59.908 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.909 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.909 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.910 DEBUG:boto:Params: {} 2022-09-03T06:13:59.911 DEBUG:boto:Token: None 2022-09-03T06:13:59.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.917 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000054827b5732ea8a-006312f0a7-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.918 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:13:59.920 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.920 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.921 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.921 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.922 DEBUG:boto:Method: GET 2022-09-03T06:13:59.922 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.923 DEBUG:boto:Data: 2022-09-03T06:13:59.923 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.924 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.925 DEBUG:boto:Params: {} 2022-09-03T06:13:59.926 DEBUG:boto:Token: None 2022-09-03T06:13:59.926 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.930 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000862a6dcffe778349-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.931 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.932 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:13:59.932 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:13:59.933 DEBUG:boto:path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.933 DEBUG:boto:auth_path=/avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.934 DEBUG:boto:Method: GET 2022-09-03T06:13:59.934 DEBUG:boto:Path: /avysmw-23/?versions&max-keys=999 2022-09-03T06:13:59.935 DEBUG:boto:Data: 2022-09-03T06:13:59.935 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.936 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.936 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.937 DEBUG:boto:Params: {} 2022-09-03T06:13:59.938 DEBUG:boto:Token: None 2022-09-03T06:13:59.939 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/?versions 2022-09-03T06:13:59.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM= 2022-09-03T06:13:59.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Du2ybesAqYBzSJsg5C5u6bBDIYM=', 'Content-Length': '0'} 2022-09-03T06:13:59.943 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ffb687be52510e4-006312f0a7-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.944 DEBUG:boto:b'avysmw-23999falseobjrvSKphaVAl9DVI7CQ7iMcbm8Tr2PKUttrue2022-09-03T06:11:23.472Zrgw-multisite-test-userMultisite Test UserNormalobjnINUfDHU4YFVDIeNN1tLGFrdDbwGAdtfalse2022-09-03T06:11:23.432Z"d41d8cd98f00b204e9800998ecf8427e"0STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:13:59.944 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:59.945 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.946 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.946 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.947 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.947 DEBUG:boto:Data: 2022-09-03T06:13:59.948 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.949 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.949 DEBUG:boto:Params: {} 2022-09-03T06:13:59.950 DEBUG:boto:Token: None 2022-09-03T06:13:59.951 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.952 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.954 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000009c27e2aa0c86eedb-006312f0a7-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.955 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.955 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.956 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.956 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:13:59.957 DEBUG:boto:Data: 2022-09-03T06:13:59.957 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.958 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.958 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.959 DEBUG:boto:Params: {} 2022-09-03T06:13:59.959 DEBUG:boto:Token: None 2022-09-03T06:13:59.960 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj 2022-09-03T06:13:59.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw= 2022-09-03T06:13:59.961 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O6JwdNVG8ObX6SdauFYewJf4/Nw=', 'Content-Length': '0'} 2022-09-03T06:13:59.963 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000008882e98e735dc642-006312f0a7-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.964 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:13:59.964 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.965 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.965 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.966 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.966 DEBUG:boto:Method: GET 2022-09-03T06:13:59.967 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.967 DEBUG:boto:Data: 2022-09-03T06:13:59.968 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.969 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.969 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.970 DEBUG:boto:Params: {} 2022-09-03T06:13:59.970 DEBUG:boto:Token: None 2022-09-03T06:13:59.971 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.974 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000095ad23e1dd6b6a66-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.975 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.975 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.976 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.976 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.977 DEBUG:boto:Method: GET 2022-09-03T06:13:59.977 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.978 DEBUG:boto:Data: 2022-09-03T06:13:59.978 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.979 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.979 DEBUG:boto:Port: 8000 2022-09-03T06:13:59.980 DEBUG:boto:Params: {} 2022-09-03T06:13:59.981 DEBUG:boto:Token: None 2022-09-03T06:13:59.981 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ= 2022-09-03T06:13:59.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fDar1OcRHJt7FF+yb2+GgR24wwQ=', 'Content-Length': '0'} 2022-09-03T06:13:59.984 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000834ed54acbf11943-006312f0a7-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.985 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.986 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.986 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.987 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.987 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.988 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.988 DEBUG:boto:Data: 2022-09-03T06:13:59.989 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:13:59.990 DEBUG:boto:Port: 8001 2022-09-03T06:13:59.990 DEBUG:boto:Params: {} 2022-09-03T06:13:59.991 DEBUG:boto:Token: None 2022-09-03T06:13:59.991 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:13:59 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo= 2022-09-03T06:13:59.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:13:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SlZOyKRVy3jWc+PIMUkrwDviKSo=', 'Content-Length': '0'} 2022-09-03T06:13:59.994 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000dab21b7ddb90dd3c-006312f0a7-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:13:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:13:59.995 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:13:59.996 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:13:59.996 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.996 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.997 DEBUG:boto:Method: HEAD 2022-09-03T06:13:59.997 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:13:59.998 DEBUG:boto:Data: 2022-09-03T06:13:59.998 DEBUG:boto:Headers: {} 2022-09-03T06:13:59.999 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:13:59.999 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.000 DEBUG:boto:Params: {} 2022-09-03T06:14:00.000 DEBUG:boto:Token: None 2022-09-03T06:14:00.001 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:14:00 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:14:00.002 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qWxs4E2k5Ub6ek93YSachbTunYo= 2022-09-03T06:14:00.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qWxs4E2k5Ub6ek93YSachbTunYo=', 'Content-Length': '0'} 2022-09-03T06:14:00.004 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000029e6be93b0393f69-006312f0a8-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.005 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:14:00.005 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:14:00.006 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:14:00.006 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:14:00.007 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:14:00.007 DEBUG:boto:Method: GET 2022-09-03T06:14:00.008 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:14:00.008 DEBUG:boto:Data: 2022-09-03T06:14:00.009 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:00.010 DEBUG:boto:Port: 8001 2022-09-03T06:14:00.010 DEBUG:boto:Params: {} 2022-09-03T06:14:00.011 DEBUG:boto:Token: None 2022-09-03T06:14:00.012 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:14:00 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:14:00.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7JSpYl8pXBE9d8rV/5YFDhtTzpI= 2022-09-03T06:14:00.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7JSpYl8pXBE9d8rV/5YFDhtTzpI=', 'Content-Length': '0'} 2022-09-03T06:14:00.015 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx0000004444389aa4b2845-006312f0a8-10fa-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.016 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:14:00.016 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:14:00.017 DEBUG:boto:path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:14:00.017 DEBUG:boto:auth_path=/avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:14:00.018 DEBUG:boto:Method: GET 2022-09-03T06:14:00.018 DEBUG:boto:Path: /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:14:00.019 DEBUG:boto:Data: 2022-09-03T06:14:00.020 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.020 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:00.021 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.021 DEBUG:boto:Params: {} 2022-09-03T06:14:00.022 DEBUG:boto:Token: None 2022-09-03T06:14:00.022 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:14:00 GMT /avysmw-23/obj?versionId=nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt 2022-09-03T06:14:00.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7JSpYl8pXBE9d8rV/5YFDhtTzpI= 2022-09-03T06:14:00.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7JSpYl8pXBE9d8rV/5YFDhtTzpI=', 'Content-Length': '0'} 2022-09-03T06:14:00.025 DEBUG:boto:Response headers: [('Content-Length', '0'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:11:23 GMT'), ('x-amz-version-id', 'nINUfDHU4YFVDIeNN1tLGFrdDbwGAdt'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('x-amz-request-id', 'tx00000c5bc73261769164c-006312f0a8-10ee-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.026 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:14:00.027 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:14:00.027 DEBUG:boto:path=/avysmw-23/?prefix=obj 2022-09-03T06:14:00.028 DEBUG:boto:auth_path=/avysmw-23/?prefix=obj 2022-09-03T06:14:00.029 DEBUG:boto:Method: GET 2022-09-03T06:14:00.029 DEBUG:boto:Path: /avysmw-23/?prefix=obj 2022-09-03T06:14:00.030 DEBUG:boto:Data: 2022-09-03T06:14:00.030 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:00.031 DEBUG:boto:Port: 8001 2022-09-03T06:14:00.032 DEBUG:boto:Params: {} 2022-09-03T06:14:00.032 DEBUG:boto:Token: None 2022-09-03T06:14:00.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:14:00 GMT /avysmw-23/ 2022-09-03T06:14:00.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CtkH6igk681SxiB1d+HUaA7a7UY= 2022-09-03T06:14:00.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CtkH6igk681SxiB1d+HUaA7a7UY=', 'Content-Length': '0'} 2022-09-03T06:14:00.037 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aec4b1ceabdad1a5-006312f0a8-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.078 DEBUG:boto:b'avysmw-23obj1000false' 2022-09-03T06:14:00.079 DEBUG:boto:path=/avysmw-23/ 2022-09-03T06:14:00.080 DEBUG:boto:auth_path=/avysmw-23/ 2022-09-03T06:14:00.080 DEBUG:boto:path=/avysmw-23/?prefix=obj 2022-09-03T06:14:00.081 DEBUG:boto:auth_path=/avysmw-23/?prefix=obj 2022-09-03T06:14:00.081 DEBUG:boto:Method: GET 2022-09-03T06:14:00.082 DEBUG:boto:Path: /avysmw-23/?prefix=obj 2022-09-03T06:14:00.082 DEBUG:boto:Data: 2022-09-03T06:14:00.083 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.083 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:00.084 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.084 DEBUG:boto:Params: {} 2022-09-03T06:14:00.085 DEBUG:boto:Token: None 2022-09-03T06:14:00.086 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:14:00 GMT /avysmw-23/ 2022-09-03T06:14:00.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CtkH6igk681SxiB1d+HUaA7a7UY= 2022-09-03T06:14:00.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CtkH6igk681SxiB1d+HUaA7a7UY=', 'Content-Length': '0'} 2022-09-03T06:14:00.089 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003da20dc1bde30298-006312f0a8-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.090 DEBUG:boto:b'avysmw-23obj1000false' 2022-09-03T06:14:00.091 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:14:00.091 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:14:00.092 DEBUG:boto:Method: HEAD 2022-09-03T06:14:00.092 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:14:00.093 DEBUG:boto:Data: 2022-09-03T06:14:00.093 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.094 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:00.094 DEBUG:boto:Port: 8001 2022-09-03T06:14:00.095 DEBUG:boto:Params: {} 2022-09-03T06:14:00.096 DEBUG:boto:Token: None 2022-09-03T06:14:00.097 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:14:00 GMT /avysmw-23/obj 2022-09-03T06:14:00.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DJf+vzGvU5BmFlDT3M42yyg96WA= 2022-09-03T06:14:00.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DJf+vzGvU5BmFlDT3M42yyg96WA=', 'Content-Length': '0'} 2022-09-03T06:14:00.101 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000022497005fd8e1a33-006312f0a8-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.102 DEBUG:boto:path=/avysmw-23/obj 2022-09-03T06:14:00.102 DEBUG:boto:auth_path=/avysmw-23/obj 2022-09-03T06:14:00.103 DEBUG:boto:Method: HEAD 2022-09-03T06:14:00.104 DEBUG:boto:Path: /avysmw-23/obj 2022-09-03T06:14:00.104 DEBUG:boto:Data: 2022-09-03T06:14:00.105 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.106 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:00.107 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.107 DEBUG:boto:Params: {} 2022-09-03T06:14:00.108 DEBUG:boto:Token: None 2022-09-03T06:14:00.108 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:14:00 GMT /avysmw-23/obj 2022-09-03T06:14:00.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DJf+vzGvU5BmFlDT3M42yyg96WA= 2022-09-03T06:14:00.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DJf+vzGvU5BmFlDT3M42yyg96WA=', 'Content-Length': '0'} 2022-09-03T06:14:00.112 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000002e04de404b23f2a5-006312f0a8-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.112 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-23 zones={test-zone3, test-zone2} 2022-09-03T06:14:00.113 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_delete_marker_full_sync ... ok 2022-09-03T06:14:00.115 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-24 2022-09-03T06:14:00.116 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:14:00.116 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:14:00.117 DEBUG:boto:Method: PUT 2022-09-03T06:14:00.117 DEBUG:boto:Path: /avysmw-24/ 2022-09-03T06:14:00.118 DEBUG:boto:Data: 2022-09-03T06:14:00.118 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.119 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:14:00.120 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.120 DEBUG:boto:Params: {} 2022-09-03T06:14:00.121 DEBUG:boto:Token: None 2022-09-03T06:14:00.121 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:14:00 GMT /avysmw-24/ 2022-09-03T06:14:00.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:673aByy4rwlR7jl6rwMhtOvKhCY= 2022-09-03T06:14:00.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:673aByy4rwlR7jl6rwMhtOvKhCY=', 'Content-Length': '0'} 2022-09-03T06:14:00.139 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000001c62e08d4d7db02b-006312f0a8-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.140 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-25 2022-09-03T06:14:00.141 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:14:00.141 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:14:00.142 DEBUG:boto:Method: PUT 2022-09-03T06:14:00.142 DEBUG:boto:Path: /avysmw-25/ 2022-09-03T06:14:00.143 DEBUG:boto:Data: 2022-09-03T06:14:00.143 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.144 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:00.144 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.145 DEBUG:boto:Params: {} 2022-09-03T06:14:00.145 DEBUG:boto:Token: None 2022-09-03T06:14:00.146 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:14:00 GMT /avysmw-25/ 2022-09-03T06:14:00.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pcF//JJciFSjCfW65DE4+Yml38E= 2022-09-03T06:14:00.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pcF//JJciFSjCfW65DE4+Yml38E=', 'Content-Length': '0'} 2022-09-03T06:14:00.185 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000043660fc9b595f879-006312f0a8-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.186 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-26 2022-09-03T06:14:00.186 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:14:00.187 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:14:00.187 DEBUG:boto:Method: PUT 2022-09-03T06:14:00.188 DEBUG:boto:Path: /avysmw-26/ 2022-09-03T06:14:00.189 DEBUG:boto:Data: 2022-09-03T06:14:00.189 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.190 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:00.190 DEBUG:boto:Port: 8001 2022-09-03T06:14:00.191 DEBUG:boto:Params: {} 2022-09-03T06:14:00.191 DEBUG:boto:Token: None 2022-09-03T06:14:00.192 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:14:00 GMT /avysmw-26/ 2022-09-03T06:14:00.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Xx0CcKpYY7BijWjJq1mEHd0cpk= 2022-09-03T06:14:00.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Xx0CcKpYY7BijWjJq1mEHd0cpk=', 'Content-Length': '0'} 2022-09-03T06:14:00.231 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000bdc9a1f10a2a342c-006312f0a8-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.232 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:14:00.233 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:14:00.234 DEBUG:boto:path=/avysmw-24/?versioning 2022-09-03T06:14:00.235 DEBUG:boto:auth_path=/avysmw-24/?versioning 2022-09-03T06:14:00.236 DEBUG:boto:Method: PUT 2022-09-03T06:14:00.236 DEBUG:boto:Path: /avysmw-24/?versioning 2022-09-03T06:14:00.237 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:14:00.238 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:14:00.240 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.241 DEBUG:boto:Params: {} 2022-09-03T06:14:00.242 DEBUG:boto:Token: None 2022-09-03T06:14:00.243 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:14:00 GMT /avysmw-24/?versioning 2022-09-03T06:14:00.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VVW3Nl7cWqp+6hQuhg1aIc+n5xQ= 2022-09-03T06:14:00.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VVW3Nl7cWqp+6hQuhg1aIc+n5xQ=', 'Content-Length': '228'} 2022-09-03T06:14:00.253 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000f86c7b40d271b4be-006312f0a8-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.255 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:14:00.256 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:14:00.256 DEBUG:boto:path=/avysmw-24/?versioning 2022-09-03T06:14:00.257 DEBUG:boto:auth_path=/avysmw-24/?versioning 2022-09-03T06:14:00.257 DEBUG:boto:Method: PUT 2022-09-03T06:14:00.258 DEBUG:boto:Path: /avysmw-24/?versioning 2022-09-03T06:14:00.258 DEBUG:boto:Data: Suspended Disabled 2022-09-03T06:14:00.259 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.259 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:14:00.260 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.260 DEBUG:boto:Params: {} 2022-09-03T06:14:00.261 DEBUG:boto:Token: None 2022-09-03T06:14:00.262 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:14:00 GMT /avysmw-24/?versioning 2022-09-03T06:14:00.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VVW3Nl7cWqp+6hQuhg1aIc+n5xQ= 2022-09-03T06:14:00.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VVW3Nl7cWqp+6hQuhg1aIc+n5xQ=', 'Content-Length': '230'} 2022-09-03T06:14:00.273 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000d45ab04dca724adb-006312f0a8-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.273 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:14:00.274 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:14:00.275 DEBUG:boto:path=/avysmw-25/?versioning 2022-09-03T06:14:00.275 DEBUG:boto:auth_path=/avysmw-25/?versioning 2022-09-03T06:14:00.276 DEBUG:boto:Method: PUT 2022-09-03T06:14:00.276 DEBUG:boto:Path: /avysmw-25/?versioning 2022-09-03T06:14:00.277 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:14:00.277 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.277 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:00.278 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.279 DEBUG:boto:Params: {} 2022-09-03T06:14:00.279 DEBUG:boto:Token: None 2022-09-03T06:14:00.280 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:14:00 GMT /avysmw-25/?versioning 2022-09-03T06:14:00.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NZ+RhPkAJinQiUY4+s+4oo04Yk= 2022-09-03T06:14:00.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NZ+RhPkAJinQiUY4+s+4oo04Yk=', 'Content-Length': '228'} 2022-09-03T06:14:00.294 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000077aa66ca49dd1906-006312f0a8-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.295 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:14:00.296 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:14:00.297 DEBUG:boto:path=/avysmw-25/?versioning 2022-09-03T06:14:00.297 DEBUG:boto:auth_path=/avysmw-25/?versioning 2022-09-03T06:14:00.298 DEBUG:boto:Method: PUT 2022-09-03T06:14:00.298 DEBUG:boto:Path: /avysmw-25/?versioning 2022-09-03T06:14:00.299 DEBUG:boto:Data: Suspended Disabled 2022-09-03T06:14:00.299 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:00.300 DEBUG:boto:Port: 8000 2022-09-03T06:14:00.301 DEBUG:boto:Params: {} 2022-09-03T06:14:00.301 DEBUG:boto:Token: None 2022-09-03T06:14:00.302 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:14:00 GMT /avysmw-25/?versioning 2022-09-03T06:14:00.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NZ+RhPkAJinQiUY4+s+4oo04Yk= 2022-09-03T06:14:00.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NZ+RhPkAJinQiUY4+s+4oo04Yk=', 'Content-Length': '230'} 2022-09-03T06:14:00.329 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ef4622110db8537b-006312f0a8-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.331 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:14:00.331 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:14:00.332 DEBUG:boto:path=/avysmw-26/?versioning 2022-09-03T06:14:00.333 DEBUG:boto:auth_path=/avysmw-26/?versioning 2022-09-03T06:14:00.334 DEBUG:boto:Method: PUT 2022-09-03T06:14:00.334 DEBUG:boto:Path: /avysmw-26/?versioning 2022-09-03T06:14:00.335 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:14:00.335 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:00.336 DEBUG:boto:Port: 8001 2022-09-03T06:14:00.337 DEBUG:boto:Params: {} 2022-09-03T06:14:00.337 DEBUG:boto:Token: None 2022-09-03T06:14:00.339 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:14:00 GMT /avysmw-26/?versioning 2022-09-03T06:14:00.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wtXyZL9xzggFgFHfV0hAlALrb9g= 2022-09-03T06:14:00.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wtXyZL9xzggFgFHfV0hAlALrb9g=', 'Content-Length': '228'} 2022-09-03T06:14:00.368 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000016e6a1e62383d79f-006312f0a8-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.369 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:14:00.370 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:14:00.370 DEBUG:boto:path=/avysmw-26/?versioning 2022-09-03T06:14:00.371 DEBUG:boto:auth_path=/avysmw-26/?versioning 2022-09-03T06:14:00.371 DEBUG:boto:Method: PUT 2022-09-03T06:14:00.372 DEBUG:boto:Path: /avysmw-26/?versioning 2022-09-03T06:14:00.372 DEBUG:boto:Data: Suspended Disabled 2022-09-03T06:14:00.373 DEBUG:boto:Headers: {} 2022-09-03T06:14:00.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:00.377 DEBUG:boto:Port: 8001 2022-09-03T06:14:00.377 DEBUG:boto:Params: {} 2022-09-03T06:14:00.378 DEBUG:boto:Token: None 2022-09-03T06:14:00.378 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:14:00 GMT /avysmw-26/?versioning 2022-09-03T06:14:00.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wtXyZL9xzggFgFHfV0hAlALrb9g= 2022-09-03T06:14:00.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wtXyZL9xzggFgFHfV0hAlALrb9g=', 'Content-Length': '230'} 2022-09-03T06:14:00.392 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000005cedd77b920623c3-006312f0a8-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:00.393 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:00.394 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:00.396 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:14:00.449 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:00.449 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:00.482 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T06:14:00.501 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:14:00.501 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:14:00.502 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:14:00.502 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T06:14:00.502 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:14:00.503 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:14:00.503 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185640.386563_210.1", 2022-09-03T06:14:00.503 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:14:00.386563Z" 2022-09-03T06:14:00.504 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:14:00.504 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:14:00.504 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:14:00.505 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:14:00.505 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:14:00.505 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:14:00.506 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:14:00.506 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:14:00.506 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:14:00.506 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:14:00.507 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662185640.386563_210.1', 2: '', 3: ''} 2022-09-03T06:14:00.507 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:14:00.508 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:00.508 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:00.509 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:14:00.569 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:00.569 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:00.597 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:00.595+0000 7fb34c781600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:14:00.598 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:00.596+0000 7fb34c781600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:14:00.609 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:14:00.610 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:14:00.610 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:14:00.611 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:14:00.611 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:14:00.611 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:14:00.612 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:00.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:00.612 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:14:00.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:00.613 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:14:00.613 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:00.613 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:00.613 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:14:00.614 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:00.614 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:14:00.614 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:00.615 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:14:00.615 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:00.615 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:00.615 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:00.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:00.616 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:14:00.616 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:00.616 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:00.617 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185640.305370_205.1", 2022-09-03T06:14:00.617 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:00.617 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:14:00.618 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:00.619 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.305370Z", 2022-09-03T06:14:00.620 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:00.620 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:00.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:00.621 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:00.621 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:14:00.622 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:00.622 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:00.622 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:14:00.623 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:00.623 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:14:00.623 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:00.624 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:14:00.624 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:00.624 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:00.625 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:00.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:00.625 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:14:00.625 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:00.626 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:00.626 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:14:00.626 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:00.627 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:14:00.627 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:00.627 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:14:00.627 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:00.628 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:00.628 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:00.628 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:14:00.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:00.629 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:14:00.629 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:14:00.629 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:14:00.629 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:00.630 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:14:00Z" 2022-09-03T06:14:00.630 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:14:00.632 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185640.305370_205.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:14:00.305370Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:14:00Z" } 2022-09-03T06:14:00.633 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185640.305370_205.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:14:00.305370Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:14:00.634 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185640.386563_210.1', 2: '', 3: ''} 2022-09-03T06:14:00.635 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185640.305370_205.1', 2: '', 3: ''} 2022-09-03T06:14:00.636 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662185640.386563_210.1 target=1_1662185640.305370_205.1 2022-09-03T06:14:05.637 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:05.638 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:05.639 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:14:05.696 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:05.697 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:05.722 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:05.719+0000 7f0f775ad600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:14:05.723 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:05.720+0000 7f0f775ad600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:14:05.734 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:14:05.735 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:14:05.735 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:14:05.736 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:14:05.736 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:14:05.737 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:14:05.737 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:05.738 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:05.738 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:14:05.739 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:05.739 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:14:05.740 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:05.740 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:05.741 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:14:05.741 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:05.742 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:14:05.742 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:05.743 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:14:05.743 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:05.744 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:05.744 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:05.745 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:05.745 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:14:05.746 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:05.747 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:05.747 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185640.386563_210.1", 2022-09-03T06:14:05.748 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:05.748 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:14:05.749 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:05.749 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.386563Z", 2022-09-03T06:14:05.750 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:05.750 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:05.751 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:05.751 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:05.752 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:14:05.752 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:05.753 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:05.753 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:14:05.754 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:05.754 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:14:05.755 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:05.755 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:14:05.756 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:05.757 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:05.757 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:05.761 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:05.761 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:14:05.762 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:05.763 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:05.763 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:14:05.764 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:05.764 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:14:05.765 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:05.765 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:14:05.766 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:05.766 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:05.767 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:05.767 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:14:05.768 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:05.768 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:14:05.769 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:14:05.769 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:14:05.770 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:05.770 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:14:05Z" 2022-09-03T06:14:05.771 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:14:05.773 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185640.386563_210.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:14:00.386563Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:14:05Z" } 2022-09-03T06:14:05.774 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185640.386563_210.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:14:00.386563Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:14:05.775 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185640.386563_210.1', 2: '', 3: ''} 2022-09-03T06:14:05.776 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185640.386563_210.1', 2: '', 3: ''} 2022-09-03T06:14:05.777 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:14:05.778 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:14:05.779 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:05.780 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:05.781 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:14:05.841 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:05.842 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:05.883 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:05.881+0000 7f0354452600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:14:05.884 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:05.882+0000 7f0354452600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:14:05.905 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:14:05.905 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:14:05.906 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:14:05.906 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:14:05.906 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:14:05.907 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:14:05.907 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:14:05.907 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:14:05.908 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:14:05.908 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:14:05.908 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:14:05.908 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:14:05.909 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:14:05.909 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:14:05.909 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:14:05.909 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:14:05.910 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:14:05.910 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:14:05.910 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:14:05.911 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:14:05.911 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:14:05.911 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:14:05.911 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:14:05.912 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:14:05.912 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:14:05.912 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185640.386563_210.1", 2022-09-03T06:14:05.913 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:14:05.913 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:14:05.914 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:14:05.914 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.386563Z", 2022-09-03T06:14:05.914 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:14:05.915 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:14:05.915 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:14:05.915 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:14:05.916 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:14:05.916 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:14:05.916 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:14:05.916 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:14:05.917 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:14:05.917 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:14:05.917 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:14:05.918 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:14:05.918 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:14:05.919 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:14:05.919 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:14:05.920 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:14:05.920 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:14:05.921 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:14:05.921 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:14:05.922 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:14:05.922 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:14:05.923 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:14:05.924 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:14:05.925 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:14:05.926 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:14:05.926 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:14:05.927 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:14:05.927 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:14:05.928 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:14:05.928 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:14:05.928 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:14:05.929 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:14:05.929 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:14:05.930 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:14:05Z" 2022-09-03T06:14:05.930 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:14:05.932 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185640.386563_210.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:14:00.386563Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:14:05Z" } 2022-09-03T06:14:05.933 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185640.386563_210.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:14:00.386563Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:14:05.933 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185640.386563_210.1', 2: '', 3: ''} 2022-09-03T06:14:05.934 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185640.386563_210.1', 2: '', 3: ''} 2022-09-03T06:14:05.935 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:14:05.936 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:14:05.937 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:05.938 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:14:05.940 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:14:05.993 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:05.993 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:06.019 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:06.019+0000 7f0d5e93c600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:14:06.020 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:06.020+0000 7f0d5e93c600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:14:06.031 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:14:06.033 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:14:06.033 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:14:06.033 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:14:06.034 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:14:06.034 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:14:06.034 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:06.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:06.035 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:14:06.035 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:06.036 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:14:06.036 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:06.036 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:06.037 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:14:06.037 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:06.038 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:14:06.038 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:06.039 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:14:06.039 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:06.039 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:06.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:06.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:06.040 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:14:06.040 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:06.040 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:06.041 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185640.386563_210.1", 2022-09-03T06:14:06.041 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:06.041 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:14:06.042 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:06.042 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.386563Z", 2022-09-03T06:14:06.043 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:06.043 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:06.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:06.044 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:06.044 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:14:06.044 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:06.044 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:06.045 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:14:06.045 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:06.045 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:14:06.045 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:06.046 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:14:06.046 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:06.046 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:06.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:06.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:14:06.048 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:14:06.048 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:14:06.049 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:14:06.049 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:14:06.050 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:14:06.050 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:14:06.051 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:14:06.051 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:14:06.052 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:14:06.052 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:06.053 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:14:06.053 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:14:06.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:06.054 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:14:06.055 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:14:06.055 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:14:06.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:14:06.056 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:14:06Z" 2022-09-03T06:14:06.057 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:14:06.059 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185640.386563_210.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:14:00.386563Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:14:06Z" } 2022-09-03T06:14:06.060 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185640.386563_210.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:14:00.386563Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:14:06.061 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185640.386563_210.1', 2: '', 3: ''} 2022-09-03T06:14:06.062 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185640.386563_210.1', 2: '', 3: ''} 2022-09-03T06:14:06.062 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:14:06.063 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:14:06.064 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:14:06.065 DEBUG:boto:Method: HEAD 2022-09-03T06:14:06.066 DEBUG:boto:Path: /avysmw-24/ 2022-09-03T06:14:06.067 DEBUG:boto:Data: 2022-09-03T06:14:06.068 DEBUG:boto:Headers: {} 2022-09-03T06:14:06.069 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:14:06.070 DEBUG:boto:Port: 8000 2022-09-03T06:14:06.071 DEBUG:boto:Params: {} 2022-09-03T06:14:06.072 DEBUG:boto:Token: None 2022-09-03T06:14:06.073 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:14:06 GMT /avysmw-24/ 2022-09-03T06:14:06.074 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zUqkn5+P6Afn5RpkiYOUPoJRp3U= 2022-09-03T06:14:06.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zUqkn5+P6Afn5RpkiYOUPoJRp3U=', 'Content-Length': '0'} 2022-09-03T06:14:06.077 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d07363b14d620c1d-006312f0ae-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.078 DEBUG:boto:path=/avysmw-24/obj 2022-09-03T06:14:06.079 DEBUG:boto:auth_path=/avysmw-24/obj 2022-09-03T06:14:06.079 DEBUG:boto:Method: PUT 2022-09-03T06:14:06.080 DEBUG:boto:Path: /avysmw-24/obj 2022-09-03T06:14:06.080 DEBUG:boto:Data: 2022-09-03T06:14:06.081 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:14:06.081 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:14:06.082 DEBUG:boto:Port: 8000 2022-09-03T06:14:06.083 DEBUG:boto:Params: {} 2022-09-03T06:14:06.083 DEBUG:boto:Token: None 2022-09-03T06:14:06.084 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:14:06 GMT /avysmw-24/obj 2022-09-03T06:14:06.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fOfcyCzUcmiKZlPWx/TbnaMgSPQ= 2022-09-03T06:14:06.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fOfcyCzUcmiKZlPWx/TbnaMgSPQ='} 2022-09-03T06:14:06.096 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000008db1ad29e0fae906-006312f0ae-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.097 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:14:06.097 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:14:06.098 DEBUG:boto:Method: HEAD 2022-09-03T06:14:06.098 DEBUG:boto:Path: /avysmw-24/ 2022-09-03T06:14:06.099 DEBUG:boto:Data: 2022-09-03T06:14:06.100 DEBUG:boto:Headers: {} 2022-09-03T06:14:06.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:14:06.101 DEBUG:boto:Port: 8000 2022-09-03T06:14:06.101 DEBUG:boto:Params: {} 2022-09-03T06:14:06.102 DEBUG:boto:Token: None 2022-09-03T06:14:06.103 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:14:06 GMT /avysmw-24/ 2022-09-03T06:14:06.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zUqkn5+P6Afn5RpkiYOUPoJRp3U= 2022-09-03T06:14:06.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zUqkn5+P6Afn5RpkiYOUPoJRp3U=', 'Content-Length': '0'} 2022-09-03T06:14:06.107 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c6c9a5bea61dc0f5-006312f0ae-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.108 DEBUG:boto:path=/avysmw-24/obj 2022-09-03T06:14:06.108 DEBUG:boto:auth_path=/avysmw-24/obj 2022-09-03T06:14:06.109 DEBUG:boto:Method: DELETE 2022-09-03T06:14:06.109 DEBUG:boto:Path: /avysmw-24/obj 2022-09-03T06:14:06.110 DEBUG:boto:Data: 2022-09-03T06:14:06.111 DEBUG:boto:Headers: {} 2022-09-03T06:14:06.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:14:06.112 DEBUG:boto:Port: 8000 2022-09-03T06:14:06.112 DEBUG:boto:Params: {} 2022-09-03T06:14:06.113 DEBUG:boto:Token: None 2022-09-03T06:14:06.114 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 06:14:06 GMT /avysmw-24/obj 2022-09-03T06:14:06.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0IbFigg/KPUA7+ezf+bKxPuhR/k= 2022-09-03T06:14:06.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0IbFigg/KPUA7+ezf+bKxPuhR/k=', 'Content-Length': '0'} 2022-09-03T06:14:06.133 DEBUG:boto:Response headers: [('x-amz-version-id', 'null'), ('x-amz-delete-marker', 'true'), ('x-amz-request-id', 'tx00000361d08be7fdebd5e-006312f0ae-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.134 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:14:06.134 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:14:06.135 DEBUG:boto:Method: HEAD 2022-09-03T06:14:06.135 DEBUG:boto:Path: /avysmw-25/ 2022-09-03T06:14:06.136 DEBUG:boto:Data: 2022-09-03T06:14:06.136 DEBUG:boto:Headers: {} 2022-09-03T06:14:06.137 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:06.137 DEBUG:boto:Port: 8000 2022-09-03T06:14:06.138 DEBUG:boto:Params: {} 2022-09-03T06:14:06.139 DEBUG:boto:Token: None 2022-09-03T06:14:06.139 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:14:06 GMT /avysmw-25/ 2022-09-03T06:14:06.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9lykU4YQ1U/YFAzHspkIXhepcVM= 2022-09-03T06:14:06.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9lykU4YQ1U/YFAzHspkIXhepcVM=', 'Content-Length': '0'} 2022-09-03T06:14:06.143 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006fc187d7d6768750-006312f0ae-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.144 DEBUG:boto:path=/avysmw-25/obj 2022-09-03T06:14:06.145 DEBUG:boto:auth_path=/avysmw-25/obj 2022-09-03T06:14:06.146 DEBUG:boto:Method: PUT 2022-09-03T06:14:06.147 DEBUG:boto:Path: /avysmw-25/obj 2022-09-03T06:14:06.148 DEBUG:boto:Data: 2022-09-03T06:14:06.149 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:14:06.150 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:06.153 DEBUG:boto:Port: 8000 2022-09-03T06:14:06.154 DEBUG:boto:Params: {} 2022-09-03T06:14:06.155 DEBUG:boto:Token: None 2022-09-03T06:14:06.156 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:14:06 GMT /avysmw-25/obj 2022-09-03T06:14:06.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DnSVd1O2biwtaCIQO3So4ZthfCE= 2022-09-03T06:14:06.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DnSVd1O2biwtaCIQO3So4ZthfCE='} 2022-09-03T06:14:06.164 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f277e1e16f417a87-006312f0ae-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.165 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:14:06.165 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:14:06.166 DEBUG:boto:Method: HEAD 2022-09-03T06:14:06.167 DEBUG:boto:Path: /avysmw-25/ 2022-09-03T06:14:06.167 DEBUG:boto:Data: 2022-09-03T06:14:06.168 DEBUG:boto:Headers: {} 2022-09-03T06:14:06.169 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:06.169 DEBUG:boto:Port: 8000 2022-09-03T06:14:06.170 DEBUG:boto:Params: {} 2022-09-03T06:14:06.171 DEBUG:boto:Token: None 2022-09-03T06:14:06.172 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:14:06 GMT /avysmw-25/ 2022-09-03T06:14:06.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9lykU4YQ1U/YFAzHspkIXhepcVM= 2022-09-03T06:14:06.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9lykU4YQ1U/YFAzHspkIXhepcVM=', 'Content-Length': '0'} 2022-09-03T06:14:06.177 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000e08d2a892596e40-006312f0ae-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.178 DEBUG:boto:path=/avysmw-25/obj 2022-09-03T06:14:06.179 DEBUG:boto:auth_path=/avysmw-25/obj 2022-09-03T06:14:06.180 DEBUG:boto:Method: DELETE 2022-09-03T06:14:06.181 DEBUG:boto:Path: /avysmw-25/obj 2022-09-03T06:14:06.181 DEBUG:boto:Data: 2022-09-03T06:14:06.182 DEBUG:boto:Headers: {} 2022-09-03T06:14:06.184 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:14:06.185 DEBUG:boto:Port: 8000 2022-09-03T06:14:06.185 DEBUG:boto:Params: {} 2022-09-03T06:14:06.188 DEBUG:boto:Token: None 2022-09-03T06:14:06.189 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 06:14:06 GMT /avysmw-25/obj 2022-09-03T06:14:06.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rB1i37HLs1gX+psYRbPa6Zqg0ss= 2022-09-03T06:14:06.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rB1i37HLs1gX+psYRbPa6Zqg0ss=', 'Content-Length': '0'} 2022-09-03T06:14:06.202 DEBUG:boto:Response headers: [('x-amz-version-id', 'null'), ('x-amz-delete-marker', 'true'), ('x-amz-request-id', 'tx00000314d95d5741a4615-006312f0ae-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.203 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:14:06.204 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:14:06.205 DEBUG:boto:Method: HEAD 2022-09-03T06:14:06.206 DEBUG:boto:Path: /avysmw-26/ 2022-09-03T06:14:06.207 DEBUG:boto:Data: 2022-09-03T06:14:06.208 DEBUG:boto:Headers: {} 2022-09-03T06:14:06.208 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:06.209 DEBUG:boto:Port: 8001 2022-09-03T06:14:06.209 DEBUG:boto:Params: {} 2022-09-03T06:14:06.210 DEBUG:boto:Token: None 2022-09-03T06:14:06.210 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:14:06 GMT /avysmw-26/ 2022-09-03T06:14:06.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tDtNvaPLcz5AV2bJ0tC2zeNuA0o= 2022-09-03T06:14:06.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tDtNvaPLcz5AV2bJ0tC2zeNuA0o=', 'Content-Length': '0'} 2022-09-03T06:14:06.215 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009379341ea6b70f20-006312f0ae-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.216 DEBUG:boto:path=/avysmw-26/obj 2022-09-03T06:14:06.216 DEBUG:boto:auth_path=/avysmw-26/obj 2022-09-03T06:14:06.217 DEBUG:boto:Method: PUT 2022-09-03T06:14:06.217 DEBUG:boto:Path: /avysmw-26/obj 2022-09-03T06:14:06.218 DEBUG:boto:Data: 2022-09-03T06:14:06.219 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue'} 2022-09-03T06:14:06.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:06.221 DEBUG:boto:Port: 8001 2022-09-03T06:14:06.222 DEBUG:boto:Params: {} 2022-09-03T06:14:06.223 DEBUG:boto:Token: None 2022-09-03T06:14:06.224 DEBUG:boto:StringToSign: PUT 1B2M2Y8AsgTpgAmY7PhCfg== application/octet-stream Sat, 03 Sep 2022 06:14:06 GMT /avysmw-26/obj 2022-09-03T06:14:06.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tt6l6HLjMZ1T70ndb1Wp0Ag5Sk4= 2022-09-03T06:14:06.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Length': '0', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tt6l6HLjMZ1T70ndb1Wp0Ag5Sk4='} 2022-09-03T06:14:06.275 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"d41d8cd98f00b204e9800998ecf8427e"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000006a2f3455d96ed805-006312f0ae-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.277 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:14:06.278 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:14:06.279 DEBUG:boto:Method: HEAD 2022-09-03T06:14:06.279 DEBUG:boto:Path: /avysmw-26/ 2022-09-03T06:14:06.280 DEBUG:boto:Data: 2022-09-03T06:14:06.281 DEBUG:boto:Headers: {} 2022-09-03T06:14:06.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:06.283 DEBUG:boto:Port: 8001 2022-09-03T06:14:06.283 DEBUG:boto:Params: {} 2022-09-03T06:14:06.284 DEBUG:boto:Token: None 2022-09-03T06:14:06.285 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:14:06 GMT /avysmw-26/ 2022-09-03T06:14:06.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tDtNvaPLcz5AV2bJ0tC2zeNuA0o= 2022-09-03T06:14:06.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tDtNvaPLcz5AV2bJ0tC2zeNuA0o=', 'Content-Length': '0'} 2022-09-03T06:14:06.290 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000029da1dce364db5c5-006312f0ae-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.291 DEBUG:boto:path=/avysmw-26/obj 2022-09-03T06:14:06.292 DEBUG:boto:auth_path=/avysmw-26/obj 2022-09-03T06:14:06.292 DEBUG:boto:Method: DELETE 2022-09-03T06:14:06.293 DEBUG:boto:Path: /avysmw-26/obj 2022-09-03T06:14:06.294 DEBUG:boto:Data: 2022-09-03T06:14:06.295 DEBUG:boto:Headers: {} 2022-09-03T06:14:06.296 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:14:06.297 DEBUG:boto:Port: 8001 2022-09-03T06:14:06.298 DEBUG:boto:Params: {} 2022-09-03T06:14:06.299 DEBUG:boto:Token: None 2022-09-03T06:14:06.300 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 06:14:06 GMT /avysmw-26/obj 2022-09-03T06:14:06.301 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4bQTmm6EERqmGqZ+QKcJSmfPmEc= 2022-09-03T06:14:06.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:14:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4bQTmm6EERqmGqZ+QKcJSmfPmEc=', 'Content-Length': '0'} 2022-09-03T06:14:06.313 DEBUG:boto:Response headers: [('x-amz-version-id', 'null'), ('x-amz-delete-marker', 'true'), ('x-amz-request-id', 'tx000008b3fe0d8f088641a-006312f0ae-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:14:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:14:06.314 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-24 2022-09-03T06:14:06.316 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:06.316 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:06.317 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-24 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:14:06.376 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:06.376 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:06.424 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:06.423+0000 7f7b02290600 1 waiting to reach incremental sync.. 2022-09-03T06:14:11.426 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:11.424+0000 7f7b02290600 1 bucket sync caught up with source: 2022-09-03T06:14:11.426 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.22.6, , ] 2022-09-03T06:14:11.426 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.22.6, , ] 2022-09-03T06:14:11.427 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:11.424+0000 7f7b02290600 0 bucket checkpoint complete 2022-09-03T06:14:11.441 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-24 2022-09-03T06:14:11.442 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:11.442 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:11.443 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-24 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:14:11.504 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:11.505 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:11.555 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:11.554+0000 7fe258c96600 1 bucket sync caught up with source: 2022-09-03T06:14:11.556 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.22.6, , ] 2022-09-03T06:14:11.556 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.22.6, , ] 2022-09-03T06:14:11.557 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:11.554+0000 7fe258c96600 0 bucket checkpoint complete 2022-09-03T06:14:11.576 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-24 2022-09-03T06:14:11.577 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:11.578 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:11.579 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-24 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:14:11.643 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:11.643 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:11.694 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:11.690+0000 7f86723db600 1 waiting to reach incremental sync.. 2022-09-03T06:14:16.692 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:16.690+0000 7f86723db600 1 waiting to reach incremental sync.. 2022-09-03T06:14:21.694 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:21.692+0000 7f86723db600 1 bucket sync caught up with source: 2022-09-03T06:14:21.695 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.22.6, , ] 2022-09-03T06:14:21.695 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.22.6, , ] 2022-09-03T06:14:21.695 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:14:21.692+0000 7f86723db600 0 bucket checkpoint complete 2022-09-03T06:14:21.710 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-24 2022-09-03T06:14:21.711 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:21.713 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:21.714 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-24 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:14:21.781 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:21.782 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:21.821 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:21.819+0000 7f3d3811c600 1 waiting to reach incremental sync.. 2022-09-03T06:14:26.821 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:26.819+0000 7f3d3811c600 1 waiting to reach incremental sync.. 2022-09-03T06:14:31.827 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:31.821+0000 7f3d3811c600 1 bucket sync caught up with source: 2022-09-03T06:14:31.827 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.82.12, , ] 2022-09-03T06:14:31.828 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.82.12, , ] 2022-09-03T06:14:31.828 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:31.821+0000 7f3d3811c600 0 bucket checkpoint complete 2022-09-03T06:14:31.843 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-24 2022-09-03T06:14:31.845 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:31.845 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:14:31.850 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-24 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:14:31.916 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:14:31.917 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:14:31.956 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:31.952+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:14:36.956 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:36.952+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:14:41.956 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:41.953+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:14:46.956 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:46.955+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:14:51.957 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:51.955+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:14:56.959 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:14:56.956+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:01.960 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:01.957+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:06.961 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:06.957+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:11.961 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:11.958+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:16.961 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:16.959+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:21.962 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:21.958+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:26.963 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:26.959+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:31.967 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:31.960+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:36.964 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:36.960+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:41.964 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:41.962+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:46.964 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:46.962+0000 7f0094d7e600 1 waiting to reach incremental sync.. 2022-09-03T06:15:51.975 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:51.965+0000 7f0094d7e600 1 bucket sync caught up with source: 2022-09-03T06:15:51.976 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.82.12, , ] 2022-09-03T06:15:51.976 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.82.12, , ] 2022-09-03T06:15:51.976 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:51.965+0000 7f0094d7e600 0 bucket checkpoint complete 2022-09-03T06:15:51.992 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-24 2022-09-03T06:15:51.993 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-24 2022-09-03T06:15:51.994 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:51.995 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:51.996 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-24 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:52.061 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:52.062 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:52.124 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:52.119+0000 7fc9739ae600 1 bucket sync caught up with source: 2022-09-03T06:15:52.124 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.70.12, , ] 2022-09-03T06:15:52.125 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.70.12, , ] 2022-09-03T06:15:52.125 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:52.119+0000 7fc9739ae600 0 bucket checkpoint complete 2022-09-03T06:15:52.135 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-24 2022-09-03T06:15:52.136 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:52.137 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-24', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:52.148 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-24 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:52.221 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:52.222 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:52.261 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:52.258+0000 7f89276e6600 1 bucket sync caught up with source: 2022-09-03T06:15:52.266 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.70.12, , ] 2022-09-03T06:15:52.267 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.70.12, , ] 2022-09-03T06:15:52.268 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:52.258+0000 7f89276e6600 0 bucket checkpoint complete 2022-09-03T06:15:52.279 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-24 2022-09-03T06:15:52.290 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-24 zones={test-zone2, test-zone1} 2022-09-03T06:15:52.295 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.296 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.299 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.300 DEBUG:boto:Path: /avysmw-24/ 2022-09-03T06:15:52.301 DEBUG:boto:Data: 2022-09-03T06:15:52.302 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.304 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:52.305 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.305 DEBUG:boto:Params: {} 2022-09-03T06:15:52.306 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:15:52.307 DEBUG:boto:Token: None 2022-09-03T06:15:52.308 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/ 2022-09-03T06:15:52.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo= 2022-09-03T06:15:52.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo=', 'Content-Length': '0'} 2022-09-03T06:15:52.315 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000025b3b0fa5981c03b-006312f118-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.316 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.317 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.317 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.318 DEBUG:boto:Path: /avysmw-24/ 2022-09-03T06:15:52.319 DEBUG:boto:Data: 2022-09-03T06:15:52.320 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:52.323 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.324 DEBUG:boto:Params: {} 2022-09-03T06:15:52.324 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:15:52.325 DEBUG:boto:Token: None 2022-09-03T06:15:52.326 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/ 2022-09-03T06:15:52.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo= 2022-09-03T06:15:52.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo=', 'Content-Length': '0'} 2022-09-03T06:15:52.332 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000097809ccb9b6bdc44-006312f118-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.333 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:15:52.333 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.334 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.334 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.335 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.335 DEBUG:boto:Method: GET 2022-09-03T06:15:52.336 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.336 DEBUG:boto:Data: 2022-09-03T06:15:52.337 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.337 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:52.338 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.339 DEBUG:boto:Params: {} 2022-09-03T06:15:52.340 DEBUG:boto:Token: None 2022-09-03T06:15:52.340 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.349 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d27056ce50e82b89-006312f118-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.350 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.352 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:52.355 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:15:52.358 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.359 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.360 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.360 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.361 DEBUG:boto:Method: GET 2022-09-03T06:15:52.361 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.362 DEBUG:boto:Data: 2022-09-03T06:15:52.363 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:52.364 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.365 DEBUG:boto:Params: {} 2022-09-03T06:15:52.366 DEBUG:boto:Token: None 2022-09-03T06:15:52.367 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.368 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.371 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000073c7dbd7bd46d371-006312f118-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.373 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.376 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:52.377 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.378 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.379 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.380 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.381 DEBUG:boto:Method: GET 2022-09-03T06:15:52.381 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.382 DEBUG:boto:Data: 2022-09-03T06:15:52.383 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.384 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:52.386 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.387 DEBUG:boto:Params: {} 2022-09-03T06:15:52.388 DEBUG:boto:Token: None 2022-09-03T06:15:52.389 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.394 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006bdbaad015e98700-006312f118-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.395 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.396 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.397 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.398 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.398 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.399 DEBUG:boto:Method: GET 2022-09-03T06:15:52.400 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.401 DEBUG:boto:Data: 2022-09-03T06:15:52.402 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:52.403 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.404 DEBUG:boto:Params: {} 2022-09-03T06:15:52.405 DEBUG:boto:Token: None 2022-09-03T06:15:52.406 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.407 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.408 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.411 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000189012b8533a2dad-006312f118-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.412 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.413 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:15:52.414 DEBUG:boto:path=/avysmw-24/obj 2022-09-03T06:15:52.419 DEBUG:boto:auth_path=/avysmw-24/obj 2022-09-03T06:15:52.420 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.421 DEBUG:boto:Path: /avysmw-24/obj 2022-09-03T06:15:52.422 DEBUG:boto:Data: 2022-09-03T06:15:52.423 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.424 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:52.425 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.426 DEBUG:boto:Params: {} 2022-09-03T06:15:52.427 DEBUG:boto:Token: None 2022-09-03T06:15:52.428 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/obj 2022-09-03T06:15:52.429 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs= 2022-09-03T06:15:52.430 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs=', 'Content-Length': '0'} 2022-09-03T06:15:52.432 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000001aafae3d9cf274f2-006312f118-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.433 DEBUG:boto:path=/avysmw-24/obj 2022-09-03T06:15:52.434 DEBUG:boto:auth_path=/avysmw-24/obj 2022-09-03T06:15:52.434 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.435 DEBUG:boto:Path: /avysmw-24/obj 2022-09-03T06:15:52.435 DEBUG:boto:Data: 2022-09-03T06:15:52.436 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.436 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:52.437 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.437 DEBUG:boto:Params: {} 2022-09-03T06:15:52.438 DEBUG:boto:Token: None 2022-09-03T06:15:52.438 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/obj 2022-09-03T06:15:52.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs= 2022-09-03T06:15:52.440 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs=', 'Content-Length': '0'} 2022-09-03T06:15:52.442 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000003150d1bf1cb21417-006312f118-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.444 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-24 zones={test-zone2, test-zone1} 2022-09-03T06:15:52.445 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-24 zones={test-zone3, test-zone1} 2022-09-03T06:15:52.446 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.447 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.447 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.448 DEBUG:boto:Path: /avysmw-24/ 2022-09-03T06:15:52.449 DEBUG:boto:Data: 2022-09-03T06:15:52.450 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:52.451 DEBUG:boto:Port: 8001 2022-09-03T06:15:52.452 DEBUG:boto:Params: {} 2022-09-03T06:15:52.453 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:15:52.454 DEBUG:boto:Token: None 2022-09-03T06:15:52.454 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/ 2022-09-03T06:15:52.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo= 2022-09-03T06:15:52.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo=', 'Content-Length': '0'} 2022-09-03T06:15:52.468 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000847e6b8c7d964f8f-006312f118-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.469 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.470 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.471 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.472 DEBUG:boto:Path: /avysmw-24/ 2022-09-03T06:15:52.473 DEBUG:boto:Data: 2022-09-03T06:15:52.474 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:52.477 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.479 DEBUG:boto:Params: {} 2022-09-03T06:15:52.480 DEBUG:boto:Token: None 2022-09-03T06:15:52.482 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/ 2022-09-03T06:15:52.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo= 2022-09-03T06:15:52.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo=', 'Content-Length': '0'} 2022-09-03T06:15:52.488 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b502b04bb300a0d4-006312f118-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.494 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:15:52.495 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.497 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.498 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.499 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.500 DEBUG:boto:Method: GET 2022-09-03T06:15:52.501 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.503 DEBUG:boto:Data: 2022-09-03T06:15:52.504 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:52.506 DEBUG:boto:Port: 8001 2022-09-03T06:15:52.507 DEBUG:boto:Params: {} 2022-09-03T06:15:52.509 DEBUG:boto:Token: None 2022-09-03T06:15:52.510 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.515 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003a759c2794e8192d-006312f118-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.516 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.517 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:52.519 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:15:52.520 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.521 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.522 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.523 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.524 DEBUG:boto:Method: GET 2022-09-03T06:15:52.526 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.527 DEBUG:boto:Data: 2022-09-03T06:15:52.528 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:52.530 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.531 DEBUG:boto:Params: {} 2022-09-03T06:15:52.532 DEBUG:boto:Token: None 2022-09-03T06:15:52.533 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.541 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b40bc3611063e2e8-006312f118-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.542 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.543 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:52.544 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.545 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.546 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.546 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.548 DEBUG:boto:Method: GET 2022-09-03T06:15:52.549 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.552 DEBUG:boto:Data: 2022-09-03T06:15:52.553 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:52.556 DEBUG:boto:Port: 8001 2022-09-03T06:15:52.556 DEBUG:boto:Params: {} 2022-09-03T06:15:52.557 DEBUG:boto:Token: None 2022-09-03T06:15:52.558 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.564 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000abd30cf12aa3ce19-006312f118-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.566 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.567 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.568 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.569 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.570 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.571 DEBUG:boto:Method: GET 2022-09-03T06:15:52.572 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.573 DEBUG:boto:Data: 2022-09-03T06:15:52.574 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.575 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:52.576 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.577 DEBUG:boto:Params: {} 2022-09-03T06:15:52.579 DEBUG:boto:Token: None 2022-09-03T06:15:52.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.589 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036cea03713b73069-006312f118-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.590 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.592 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:15:52.592 DEBUG:boto:path=/avysmw-24/obj 2022-09-03T06:15:52.593 DEBUG:boto:auth_path=/avysmw-24/obj 2022-09-03T06:15:52.595 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.596 DEBUG:boto:Path: /avysmw-24/obj 2022-09-03T06:15:52.598 DEBUG:boto:Data: 2022-09-03T06:15:52.599 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:52.601 DEBUG:boto:Port: 8001 2022-09-03T06:15:52.602 DEBUG:boto:Params: {} 2022-09-03T06:15:52.603 DEBUG:boto:Token: None 2022-09-03T06:15:52.605 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/obj 2022-09-03T06:15:52.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs= 2022-09-03T06:15:52.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs=', 'Content-Length': '0'} 2022-09-03T06:15:52.621 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000023bcd2147415df61-006312f118-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.622 DEBUG:boto:path=/avysmw-24/obj 2022-09-03T06:15:52.623 DEBUG:boto:auth_path=/avysmw-24/obj 2022-09-03T06:15:52.623 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.624 DEBUG:boto:Path: /avysmw-24/obj 2022-09-03T06:15:52.624 DEBUG:boto:Data: 2022-09-03T06:15:52.625 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:52.626 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.626 DEBUG:boto:Params: {} 2022-09-03T06:15:52.627 DEBUG:boto:Token: None 2022-09-03T06:15:52.628 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/obj 2022-09-03T06:15:52.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs= 2022-09-03T06:15:52.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs=', 'Content-Length': '0'} 2022-09-03T06:15:52.631 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000009187b9f805680a81-006312f118-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.632 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-24 zones={test-zone3, test-zone1} 2022-09-03T06:15:52.633 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-24 zones={test-zone3, test-zone2} 2022-09-03T06:15:52.633 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.634 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.634 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.635 DEBUG:boto:Path: /avysmw-24/ 2022-09-03T06:15:52.635 DEBUG:boto:Data: 2022-09-03T06:15:52.636 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:52.637 DEBUG:boto:Port: 8001 2022-09-03T06:15:52.637 DEBUG:boto:Params: {} 2022-09-03T06:15:52.639 DEBUG:boto:Token: None 2022-09-03T06:15:52.640 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/ 2022-09-03T06:15:52.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo= 2022-09-03T06:15:52.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo=', 'Content-Length': '0'} 2022-09-03T06:15:52.645 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000652d1d8e552be511-006312f118-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.654 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.655 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.656 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.657 DEBUG:boto:Path: /avysmw-24/ 2022-09-03T06:15:52.657 DEBUG:boto:Data: 2022-09-03T06:15:52.658 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.659 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:52.660 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.661 DEBUG:boto:Params: {} 2022-09-03T06:15:52.662 DEBUG:boto:Token: None 2022-09-03T06:15:52.663 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/ 2022-09-03T06:15:52.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo= 2022-09-03T06:15:52.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzIWEz3ybiv/1HDLnSdG1a1opQo=', 'Content-Length': '0'} 2022-09-03T06:15:52.669 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c46b4b29ae7d5075-006312f118-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.671 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:15:52.671 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.672 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.672 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.673 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.673 DEBUG:boto:Method: GET 2022-09-03T06:15:52.674 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.674 DEBUG:boto:Data: 2022-09-03T06:15:52.675 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.675 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:52.676 DEBUG:boto:Port: 8001 2022-09-03T06:15:52.676 DEBUG:boto:Params: {} 2022-09-03T06:15:52.677 DEBUG:boto:Token: None 2022-09-03T06:15:52.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.682 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017b96592ba91ef7a-006312f118-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.683 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.684 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:52.684 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:15:52.685 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.685 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.686 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.686 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.687 DEBUG:boto:Method: GET 2022-09-03T06:15:52.688 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.688 DEBUG:boto:Data: 2022-09-03T06:15:52.689 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.689 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:52.690 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.692 DEBUG:boto:Params: {} 2022-09-03T06:15:52.693 DEBUG:boto:Token: None 2022-09-03T06:15:52.694 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.703 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000edfd17334e0bd5a3-006312f118-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.704 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.705 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:52.705 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.706 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.706 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.707 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.707 DEBUG:boto:Method: GET 2022-09-03T06:15:52.708 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.709 DEBUG:boto:Data: 2022-09-03T06:15:52.709 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.710 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:52.710 DEBUG:boto:Port: 8001 2022-09-03T06:15:52.711 DEBUG:boto:Params: {} 2022-09-03T06:15:52.712 DEBUG:boto:Token: None 2022-09-03T06:15:52.713 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.714 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.717 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085e80d8474010ab2-006312f118-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.718 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.719 DEBUG:boto:path=/avysmw-24/ 2022-09-03T06:15:52.719 DEBUG:boto:auth_path=/avysmw-24/ 2022-09-03T06:15:52.720 DEBUG:boto:path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.720 DEBUG:boto:auth_path=/avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.721 DEBUG:boto:Method: GET 2022-09-03T06:15:52.721 DEBUG:boto:Path: /avysmw-24/?versions&max-keys=999 2022-09-03T06:15:52.722 DEBUG:boto:Data: 2022-09-03T06:15:52.723 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.723 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:52.724 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.725 DEBUG:boto:Params: {} 2022-09-03T06:15:52.725 DEBUG:boto:Token: None 2022-09-03T06:15:52.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/?versions 2022-09-03T06:15:52.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY= 2022-09-03T06:15:52.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frB6NeGK21sRWMR5U114YGMVoSY=', 'Content-Length': '0'} 2022-09-03T06:15:52.737 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e6ec9879d26dfe9f-006312f118-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.774 DEBUG:boto:b'avysmw-24999falseobjnulltrue2022-09-03T06:14:06.116Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:52.776 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:15:52.777 DEBUG:boto:path=/avysmw-24/obj 2022-09-03T06:15:52.778 DEBUG:boto:auth_path=/avysmw-24/obj 2022-09-03T06:15:52.778 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.779 DEBUG:boto:Path: /avysmw-24/obj 2022-09-03T06:15:52.780 DEBUG:boto:Data: 2022-09-03T06:15:52.782 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:52.783 DEBUG:boto:Port: 8001 2022-09-03T06:15:52.784 DEBUG:boto:Params: {} 2022-09-03T06:15:52.785 DEBUG:boto:Token: None 2022-09-03T06:15:52.787 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/obj 2022-09-03T06:15:52.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs= 2022-09-03T06:15:52.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs=', 'Content-Length': '0'} 2022-09-03T06:15:52.798 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000d2f0d79fb7f89d1c-006312f118-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.799 DEBUG:boto:path=/avysmw-24/obj 2022-09-03T06:15:52.801 DEBUG:boto:auth_path=/avysmw-24/obj 2022-09-03T06:15:52.803 DEBUG:boto:Method: HEAD 2022-09-03T06:15:52.803 DEBUG:boto:Path: /avysmw-24/obj 2022-09-03T06:15:52.804 DEBUG:boto:Data: 2022-09-03T06:15:52.805 DEBUG:boto:Headers: {} 2022-09-03T06:15:52.805 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:52.806 DEBUG:boto:Port: 8000 2022-09-03T06:15:52.806 DEBUG:boto:Params: {} 2022-09-03T06:15:52.807 DEBUG:boto:Token: None 2022-09-03T06:15:52.808 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:52 GMT /avysmw-24/obj 2022-09-03T06:15:52.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs= 2022-09-03T06:15:52.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5Al1HoAfMzlG1KCRhFwakMAaSs=', 'Content-Length': '0'} 2022-09-03T06:15:52.811 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000022f57cff39ed45dd-006312f118-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:52.812 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-24 zones={test-zone3, test-zone2} 2022-09-03T06:15:52.813 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-25 2022-09-03T06:15:52.813 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:52.814 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:52.814 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-25 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:52.882 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:52.882 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:52.932 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:52.930+0000 7f8087fe8600 1 bucket sync caught up with source: 2022-09-03T06:15:52.932 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.24.12, , ] 2022-09-03T06:15:52.933 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.24.12, , ] 2022-09-03T06:15:52.933 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:52.930+0000 7f8087fe8600 0 bucket checkpoint complete 2022-09-03T06:15:52.949 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-25 2022-09-03T06:15:52.950 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:52.951 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:52.952 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-25 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:53.018 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:53.018 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:53.066 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:53.062+0000 7fe63888f600 1 bucket sync caught up with source: 2022-09-03T06:15:53.067 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.24.12, , ] 2022-09-03T06:15:53.067 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.24.12, , ] 2022-09-03T06:15:53.068 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:53.062+0000 7fe63888f600 0 bucket checkpoint complete 2022-09-03T06:15:53.083 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-25 2022-09-03T06:15:53.084 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.085 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.086 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-25 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:53.145 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:53.145 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:53.201 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:53.197+0000 7f1074c1b600 1 bucket sync caught up with source: 2022-09-03T06:15:53.202 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.24.12, , ] 2022-09-03T06:15:53.202 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.24.12, , ] 2022-09-03T06:15:53.203 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:53.197+0000 7f1074c1b600 0 bucket checkpoint complete 2022-09-03T06:15:53.213 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-25 2022-09-03T06:15:53.214 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.215 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.216 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-25 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:53.277 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:53.278 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:53.343 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:53.339+0000 7fbe41754600 1 bucket sync caught up with source: 2022-09-03T06:15:53.343 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.83.6, , ] 2022-09-03T06:15:53.344 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.83.6, , ] 2022-09-03T06:15:53.344 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:53.339+0000 7fbe41754600 0 bucket checkpoint complete 2022-09-03T06:15:53.365 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-25 2022-09-03T06:15:53.365 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.366 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.367 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-25 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:53.426 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:53.426 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:53.477 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:53.474+0000 7f45b94e0600 1 bucket sync caught up with source: 2022-09-03T06:15:53.478 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.83.6, , ] 2022-09-03T06:15:53.478 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.83.6, , ] 2022-09-03T06:15:53.479 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:53.474+0000 7f45b94e0600 0 bucket checkpoint complete 2022-09-03T06:15:53.492 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-25 2022-09-03T06:15:53.493 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-25 2022-09-03T06:15:53.494 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.496 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.498 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-25 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:53.559 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:53.560 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:53.599 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:53.596+0000 7f399fdba600 1 bucket sync caught up with source: 2022-09-03T06:15:53.599 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.68.12, , ] 2022-09-03T06:15:53.600 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.68.12, , ] 2022-09-03T06:15:53.600 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:53.596+0000 7f399fdba600 0 bucket checkpoint complete 2022-09-03T06:15:53.612 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-25 2022-09-03T06:15:53.612 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.615 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-25', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:53.615 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-25 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:53.674 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:53.674 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:53.709 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:53.707+0000 7fbafc31f600 1 bucket sync caught up with source: 2022-09-03T06:15:53.709 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.68.12, , ] 2022-09-03T06:15:53.710 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.68.12, , ] 2022-09-03T06:15:53.712 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:53.707+0000 7fbafc31f600 0 bucket checkpoint complete 2022-09-03T06:15:53.722 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-25 2022-09-03T06:15:53.723 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-25 zones={test-zone2, test-zone1} 2022-09-03T06:15:53.724 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.725 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.726 DEBUG:boto:Method: HEAD 2022-09-03T06:15:53.727 DEBUG:boto:Path: /avysmw-25/ 2022-09-03T06:15:53.728 DEBUG:boto:Data: 2022-09-03T06:15:53.729 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.729 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:53.730 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.731 DEBUG:boto:Params: {} 2022-09-03T06:15:53.732 DEBUG:boto:Token: None 2022-09-03T06:15:53.732 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/ 2022-09-03T06:15:53.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZF9RtRx1BXRcqoUdN5Sd2YKJbCE= 2022-09-03T06:15:53.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZF9RtRx1BXRcqoUdN5Sd2YKJbCE=', 'Content-Length': '0'} 2022-09-03T06:15:53.737 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000df2f18259cfe72e-006312f119-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.737 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.738 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.739 DEBUG:boto:Method: HEAD 2022-09-03T06:15:53.740 DEBUG:boto:Path: /avysmw-25/ 2022-09-03T06:15:53.741 DEBUG:boto:Data: 2022-09-03T06:15:53.742 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.742 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:53.743 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.743 DEBUG:boto:Params: {} 2022-09-03T06:15:53.744 DEBUG:boto:Token: None 2022-09-03T06:15:53.744 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/ 2022-09-03T06:15:53.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZF9RtRx1BXRcqoUdN5Sd2YKJbCE= 2022-09-03T06:15:53.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZF9RtRx1BXRcqoUdN5Sd2YKJbCE=', 'Content-Length': '0'} 2022-09-03T06:15:53.749 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f7abd73f86d58709-006312f119-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.751 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:15:53.751 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.752 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.753 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.754 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.754 DEBUG:boto:Method: GET 2022-09-03T06:15:53.755 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.755 DEBUG:boto:Data: 2022-09-03T06:15:53.756 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.757 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:53.757 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.758 DEBUG:boto:Params: {} 2022-09-03T06:15:53.759 DEBUG:boto:Token: None 2022-09-03T06:15:53.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/?versions 2022-09-03T06:15:53.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8= 2022-09-03T06:15:53.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8=', 'Content-Length': '0'} 2022-09-03T06:15:53.764 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008988a368ad200e52-006312f119-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.764 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:53.765 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:53.767 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:15:53.768 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.768 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.769 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.769 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.770 DEBUG:boto:Method: GET 2022-09-03T06:15:53.771 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.772 DEBUG:boto:Data: 2022-09-03T06:15:53.773 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:53.775 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.776 DEBUG:boto:Params: {} 2022-09-03T06:15:53.777 DEBUG:boto:Token: None 2022-09-03T06:15:53.778 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/?versions 2022-09-03T06:15:53.779 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8= 2022-09-03T06:15:53.780 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8=', 'Content-Length': '0'} 2022-09-03T06:15:53.783 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000276dec9c25dae4d5-006312f119-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.784 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:53.785 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:53.786 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.787 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.787 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.788 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.789 DEBUG:boto:Method: GET 2022-09-03T06:15:53.789 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.790 DEBUG:boto:Data: 2022-09-03T06:15:53.790 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.791 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:53.791 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.792 DEBUG:boto:Params: {} 2022-09-03T06:15:53.793 DEBUG:boto:Token: None 2022-09-03T06:15:53.794 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/?versions 2022-09-03T06:15:53.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8= 2022-09-03T06:15:53.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8=', 'Content-Length': '0'} 2022-09-03T06:15:53.798 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b07f3a741fd3d6c-006312f119-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.799 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:53.800 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.800 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.801 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.802 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.802 DEBUG:boto:Method: GET 2022-09-03T06:15:53.803 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.803 DEBUG:boto:Data: 2022-09-03T06:15:53.804 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:53.805 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.805 DEBUG:boto:Params: {} 2022-09-03T06:15:53.806 DEBUG:boto:Token: None 2022-09-03T06:15:53.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/?versions 2022-09-03T06:15:53.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8= 2022-09-03T06:15:53.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8=', 'Content-Length': '0'} 2022-09-03T06:15:53.811 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000342c51e042b64535-006312f119-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.854 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:53.855 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:15:53.856 DEBUG:boto:path=/avysmw-25/obj 2022-09-03T06:15:53.856 DEBUG:boto:auth_path=/avysmw-25/obj 2022-09-03T06:15:53.857 DEBUG:boto:Method: HEAD 2022-09-03T06:15:53.857 DEBUG:boto:Path: /avysmw-25/obj 2022-09-03T06:15:53.858 DEBUG:boto:Data: 2022-09-03T06:15:53.858 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.859 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:53.859 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.860 DEBUG:boto:Params: {} 2022-09-03T06:15:53.861 DEBUG:boto:Token: None 2022-09-03T06:15:53.861 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/obj 2022-09-03T06:15:53.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DJfv/1AOn42xsp4U6SpcChlascA= 2022-09-03T06:15:53.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DJfv/1AOn42xsp4U6SpcChlascA=', 'Content-Length': '0'} 2022-09-03T06:15:53.865 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000cc6bc6e63df292a7-006312f119-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.866 DEBUG:boto:path=/avysmw-25/obj 2022-09-03T06:15:53.866 DEBUG:boto:auth_path=/avysmw-25/obj 2022-09-03T06:15:53.867 DEBUG:boto:Method: HEAD 2022-09-03T06:15:53.867 DEBUG:boto:Path: /avysmw-25/obj 2022-09-03T06:15:53.868 DEBUG:boto:Data: 2022-09-03T06:15:53.868 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.869 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:53.869 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.870 DEBUG:boto:Params: {} 2022-09-03T06:15:53.871 DEBUG:boto:Token: None 2022-09-03T06:15:53.871 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/obj 2022-09-03T06:15:53.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DJfv/1AOn42xsp4U6SpcChlascA= 2022-09-03T06:15:53.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DJfv/1AOn42xsp4U6SpcChlascA=', 'Content-Length': '0'} 2022-09-03T06:15:53.876 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000037ea6b9c1ed4abbe-006312f119-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.877 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-25 zones={test-zone2, test-zone1} 2022-09-03T06:15:53.877 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-25 zones={test-zone3, test-zone1} 2022-09-03T06:15:53.878 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.878 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.879 DEBUG:boto:Method: HEAD 2022-09-03T06:15:53.880 DEBUG:boto:Path: /avysmw-25/ 2022-09-03T06:15:53.880 DEBUG:boto:Data: 2022-09-03T06:15:53.881 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:53.882 DEBUG:boto:Port: 8001 2022-09-03T06:15:53.882 DEBUG:boto:Params: {} 2022-09-03T06:15:53.883 DEBUG:boto:Token: None 2022-09-03T06:15:53.884 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/ 2022-09-03T06:15:53.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZF9RtRx1BXRcqoUdN5Sd2YKJbCE= 2022-09-03T06:15:53.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZF9RtRx1BXRcqoUdN5Sd2YKJbCE=', 'Content-Length': '0'} 2022-09-03T06:15:53.889 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005263e8fb711b664e-006312f119-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.890 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.891 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.892 DEBUG:boto:Method: HEAD 2022-09-03T06:15:53.893 DEBUG:boto:Path: /avysmw-25/ 2022-09-03T06:15:53.894 DEBUG:boto:Data: 2022-09-03T06:15:53.895 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:53.897 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.898 DEBUG:boto:Params: {} 2022-09-03T06:15:53.899 DEBUG:boto:Token: None 2022-09-03T06:15:53.900 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/ 2022-09-03T06:15:53.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZF9RtRx1BXRcqoUdN5Sd2YKJbCE= 2022-09-03T06:15:53.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZF9RtRx1BXRcqoUdN5Sd2YKJbCE=', 'Content-Length': '0'} 2022-09-03T06:15:53.905 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004d3dab93e2a1aa9b-006312f119-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.906 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:15:53.907 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.908 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.909 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.909 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.910 DEBUG:boto:Method: GET 2022-09-03T06:15:53.911 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.912 DEBUG:boto:Data: 2022-09-03T06:15:53.913 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:53.916 DEBUG:boto:Port: 8001 2022-09-03T06:15:53.916 DEBUG:boto:Params: {} 2022-09-03T06:15:53.917 DEBUG:boto:Token: None 2022-09-03T06:15:53.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/?versions 2022-09-03T06:15:53.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8= 2022-09-03T06:15:53.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8=', 'Content-Length': '0'} 2022-09-03T06:15:53.926 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007ca7b1b258697d34-006312f119-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.928 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:53.931 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:53.932 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:15:53.933 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.934 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.935 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.936 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.937 DEBUG:boto:Method: GET 2022-09-03T06:15:53.938 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.939 DEBUG:boto:Data: 2022-09-03T06:15:53.940 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.941 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:53.942 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.943 DEBUG:boto:Params: {} 2022-09-03T06:15:53.944 DEBUG:boto:Token: None 2022-09-03T06:15:53.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/?versions 2022-09-03T06:15:53.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8= 2022-09-03T06:15:53.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8=', 'Content-Length': '0'} 2022-09-03T06:15:53.954 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004adc5655e3e04a3e-006312f119-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.956 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:53.957 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:53.958 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.959 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.960 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.962 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.963 DEBUG:boto:Method: GET 2022-09-03T06:15:53.964 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.965 DEBUG:boto:Data: 2022-09-03T06:15:53.966 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:53.968 DEBUG:boto:Port: 8001 2022-09-03T06:15:53.969 DEBUG:boto:Params: {} 2022-09-03T06:15:53.970 DEBUG:boto:Token: None 2022-09-03T06:15:53.971 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/?versions 2022-09-03T06:15:53.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8= 2022-09-03T06:15:53.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8=', 'Content-Length': '0'} 2022-09-03T06:15:53.982 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a8628f731bf8f6f-006312f119-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:53.984 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:53.985 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:53.987 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:53.988 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.989 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.990 DEBUG:boto:Method: GET 2022-09-03T06:15:53.991 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:53.992 DEBUG:boto:Data: 2022-09-03T06:15:53.993 DEBUG:boto:Headers: {} 2022-09-03T06:15:53.994 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:53.995 DEBUG:boto:Port: 8000 2022-09-03T06:15:53.996 DEBUG:boto:Params: {} 2022-09-03T06:15:53.997 DEBUG:boto:Token: None 2022-09-03T06:15:53.998 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:53 GMT /avysmw-25/?versions 2022-09-03T06:15:53.999 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8= 2022-09-03T06:15:54.003 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wu6u5wUcYWlubuJoKKM956FNv8=', 'Content-Length': '0'} 2022-09-03T06:15:54.006 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000051b05b47dbb19533-006312f11a-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.007 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:54.008 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:15:54.009 DEBUG:boto:path=/avysmw-25/obj 2022-09-03T06:15:54.009 DEBUG:boto:auth_path=/avysmw-25/obj 2022-09-03T06:15:54.010 DEBUG:boto:Method: HEAD 2022-09-03T06:15:54.010 DEBUG:boto:Path: /avysmw-25/obj 2022-09-03T06:15:54.011 DEBUG:boto:Data: 2022-09-03T06:15:54.011 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.012 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:54.012 DEBUG:boto:Port: 8001 2022-09-03T06:15:54.013 DEBUG:boto:Params: {} 2022-09-03T06:15:54.013 DEBUG:boto:Token: None 2022-09-03T06:15:54.014 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/obj 2022-09-03T06:15:54.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/3H96zB8pyZR7tzdByADn5GPJA= 2022-09-03T06:15:54.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/3H96zB8pyZR7tzdByADn5GPJA=', 'Content-Length': '0'} 2022-09-03T06:15:54.024 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000f58ed867940b526e-006312f11a-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.025 DEBUG:boto:path=/avysmw-25/obj 2022-09-03T06:15:54.026 DEBUG:boto:auth_path=/avysmw-25/obj 2022-09-03T06:15:54.028 DEBUG:boto:Method: HEAD 2022-09-03T06:15:54.029 DEBUG:boto:Path: /avysmw-25/obj 2022-09-03T06:15:54.030 DEBUG:boto:Data: 2022-09-03T06:15:54.032 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.033 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:54.035 DEBUG:boto:Port: 8000 2022-09-03T06:15:54.036 DEBUG:boto:Params: {} 2022-09-03T06:15:54.040 DEBUG:boto:Token: None 2022-09-03T06:15:54.042 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/obj 2022-09-03T06:15:54.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/3H96zB8pyZR7tzdByADn5GPJA= 2022-09-03T06:15:54.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/3H96zB8pyZR7tzdByADn5GPJA=', 'Content-Length': '0'} 2022-09-03T06:15:54.046 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000007522a4651624f496-006312f11a-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.047 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-25 zones={test-zone3, test-zone1} 2022-09-03T06:15:54.048 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-25 zones={test-zone3, test-zone2} 2022-09-03T06:15:54.049 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:54.050 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:54.051 DEBUG:boto:Method: HEAD 2022-09-03T06:15:54.052 DEBUG:boto:Path: /avysmw-25/ 2022-09-03T06:15:54.052 DEBUG:boto:Data: 2022-09-03T06:15:54.053 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.053 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:54.054 DEBUG:boto:Port: 8001 2022-09-03T06:15:54.054 DEBUG:boto:Params: {} 2022-09-03T06:15:54.055 DEBUG:boto:Token: None 2022-09-03T06:15:54.055 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/ 2022-09-03T06:15:54.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BMvup5NH3KIgBtNVf1/GjjLDGTw= 2022-09-03T06:15:54.057 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BMvup5NH3KIgBtNVf1/GjjLDGTw=', 'Content-Length': '0'} 2022-09-03T06:15:54.062 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006d9a9b0d4c0a7312-006312f11a-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.069 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:54.070 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:54.071 DEBUG:boto:Method: HEAD 2022-09-03T06:15:54.072 DEBUG:boto:Path: /avysmw-25/ 2022-09-03T06:15:54.076 DEBUG:boto:Data: 2022-09-03T06:15:54.078 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.079 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:54.080 DEBUG:boto:Port: 8000 2022-09-03T06:15:54.081 DEBUG:boto:Params: {} 2022-09-03T06:15:54.082 DEBUG:boto:Token: None 2022-09-03T06:15:54.083 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/ 2022-09-03T06:15:54.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BMvup5NH3KIgBtNVf1/GjjLDGTw= 2022-09-03T06:15:54.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BMvup5NH3KIgBtNVf1/GjjLDGTw=', 'Content-Length': '0'} 2022-09-03T06:15:54.095 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000069a52d498fed685a-006312f11a-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.096 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:15:54.097 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:54.098 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:54.099 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.099 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.100 DEBUG:boto:Method: GET 2022-09-03T06:15:54.100 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.101 DEBUG:boto:Data: 2022-09-03T06:15:54.101 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.102 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:54.103 DEBUG:boto:Port: 8001 2022-09-03T06:15:54.103 DEBUG:boto:Params: {} 2022-09-03T06:15:54.104 DEBUG:boto:Token: None 2022-09-03T06:15:54.104 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/?versions 2022-09-03T06:15:54.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OFttqI76LVg3QuxUdh90qKYOWSg= 2022-09-03T06:15:54.106 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OFttqI76LVg3QuxUdh90qKYOWSg=', 'Content-Length': '0'} 2022-09-03T06:15:54.109 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a54fb94194dac830-006312f11a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.151 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:54.152 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:54.152 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:15:54.154 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:54.155 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:54.155 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.156 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.156 DEBUG:boto:Method: GET 2022-09-03T06:15:54.157 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.158 DEBUG:boto:Data: 2022-09-03T06:15:54.161 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.162 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:54.163 DEBUG:boto:Port: 8000 2022-09-03T06:15:54.164 DEBUG:boto:Params: {} 2022-09-03T06:15:54.165 DEBUG:boto:Token: None 2022-09-03T06:15:54.166 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/?versions 2022-09-03T06:15:54.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OFttqI76LVg3QuxUdh90qKYOWSg= 2022-09-03T06:15:54.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OFttqI76LVg3QuxUdh90qKYOWSg=', 'Content-Length': '0'} 2022-09-03T06:15:54.171 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b652a3c17db222a3-006312f11a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.173 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:54.176 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:54.177 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:54.179 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:54.180 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.182 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.184 DEBUG:boto:Method: GET 2022-09-03T06:15:54.185 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.187 DEBUG:boto:Data: 2022-09-03T06:15:54.189 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.190 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:54.193 DEBUG:boto:Port: 8001 2022-09-03T06:15:54.194 DEBUG:boto:Params: {} 2022-09-03T06:15:54.195 DEBUG:boto:Token: None 2022-09-03T06:15:54.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/?versions 2022-09-03T06:15:54.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OFttqI76LVg3QuxUdh90qKYOWSg= 2022-09-03T06:15:54.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OFttqI76LVg3QuxUdh90qKYOWSg=', 'Content-Length': '0'} 2022-09-03T06:15:54.212 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000077903d33df050fea-006312f11a-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.214 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:54.215 DEBUG:boto:path=/avysmw-25/ 2022-09-03T06:15:54.216 DEBUG:boto:auth_path=/avysmw-25/ 2022-09-03T06:15:54.217 DEBUG:boto:path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.219 DEBUG:boto:auth_path=/avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.220 DEBUG:boto:Method: GET 2022-09-03T06:15:54.221 DEBUG:boto:Path: /avysmw-25/?versions&max-keys=999 2022-09-03T06:15:54.225 DEBUG:boto:Data: 2022-09-03T06:15:54.225 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.226 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:54.226 DEBUG:boto:Port: 8000 2022-09-03T06:15:54.227 DEBUG:boto:Params: {} 2022-09-03T06:15:54.228 DEBUG:boto:Token: None 2022-09-03T06:15:54.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/?versions 2022-09-03T06:15:54.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OFttqI76LVg3QuxUdh90qKYOWSg= 2022-09-03T06:15:54.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OFttqI76LVg3QuxUdh90qKYOWSg=', 'Content-Length': '0'} 2022-09-03T06:15:54.232 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c0bcb85a48e6ae33-006312f11a-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.233 DEBUG:boto:b'avysmw-25999falseobjnulltrue2022-09-03T06:14:06.191Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:54.234 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:15:54.234 DEBUG:boto:path=/avysmw-25/obj 2022-09-03T06:15:54.235 DEBUG:boto:auth_path=/avysmw-25/obj 2022-09-03T06:15:54.236 DEBUG:boto:Method: HEAD 2022-09-03T06:15:54.237 DEBUG:boto:Path: /avysmw-25/obj 2022-09-03T06:15:54.237 DEBUG:boto:Data: 2022-09-03T06:15:54.238 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:54.239 DEBUG:boto:Port: 8001 2022-09-03T06:15:54.240 DEBUG:boto:Params: {} 2022-09-03T06:15:54.240 DEBUG:boto:Token: None 2022-09-03T06:15:54.241 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/obj 2022-09-03T06:15:54.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/3H96zB8pyZR7tzdByADn5GPJA= 2022-09-03T06:15:54.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/3H96zB8pyZR7tzdByADn5GPJA=', 'Content-Length': '0'} 2022-09-03T06:15:54.244 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000073c0bbf915e87ff3-006312f11a-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.245 DEBUG:boto:path=/avysmw-25/obj 2022-09-03T06:15:54.246 DEBUG:boto:auth_path=/avysmw-25/obj 2022-09-03T06:15:54.246 DEBUG:boto:Method: HEAD 2022-09-03T06:15:54.247 DEBUG:boto:Path: /avysmw-25/obj 2022-09-03T06:15:54.248 DEBUG:boto:Data: 2022-09-03T06:15:54.248 DEBUG:boto:Headers: {} 2022-09-03T06:15:54.249 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:54.249 DEBUG:boto:Port: 8000 2022-09-03T06:15:54.250 DEBUG:boto:Params: {} 2022-09-03T06:15:54.250 DEBUG:boto:Token: None 2022-09-03T06:15:54.251 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:54 GMT /avysmw-25/obj 2022-09-03T06:15:54.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/3H96zB8pyZR7tzdByADn5GPJA= 2022-09-03T06:15:54.252 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/3H96zB8pyZR7tzdByADn5GPJA=', 'Content-Length': '0'} 2022-09-03T06:15:54.254 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000004c6f3456ffeefb25-006312f11a-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:54.255 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-25 zones={test-zone3, test-zone2} 2022-09-03T06:15:54.255 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-26 2022-09-03T06:15:54.257 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.257 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.258 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-26 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:54.320 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:54.322 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:54.369 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:54.366+0000 7f7040541600 1 bucket sync caught up with source: 2022-09-03T06:15:54.369 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.27.12, , ] 2022-09-03T06:15:54.370 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.27.12, , ] 2022-09-03T06:15:54.370 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:54.366+0000 7f7040541600 0 bucket checkpoint complete 2022-09-03T06:15:54.393 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-26 2022-09-03T06:15:54.394 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.395 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.397 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-26 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:54.459 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:54.459 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:54.511 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:54.509+0000 7f669e755600 1 bucket sync caught up with source: 2022-09-03T06:15:54.511 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.27.12, , ] 2022-09-03T06:15:54.512 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.27.12, , ] 2022-09-03T06:15:54.512 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:54.509+0000 7f669e755600 0 bucket checkpoint complete 2022-09-03T06:15:54.532 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-26 2022-09-03T06:15:54.533 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.533 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.534 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-26 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:54.593 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:54.593 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:54.649 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:54.646+0000 7f8a67363600 1 bucket sync caught up with source: 2022-09-03T06:15:54.649 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.27.12, , ] 2022-09-03T06:15:54.650 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.27.12, , ] 2022-09-03T06:15:54.651 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:54.646+0000 7f8a67363600 0 bucket checkpoint complete 2022-09-03T06:15:54.668 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-26 2022-09-03T06:15:54.668 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.669 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.669 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-26 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:54.728 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:54.728 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:54.784 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:54.782+0000 7f04a5040600 1 bucket sync caught up with source: 2022-09-03T06:15:54.784 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.121.12, , ] 2022-09-03T06:15:54.785 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.121.12, , ] 2022-09-03T06:15:54.785 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:54.782+0000 7f04a5040600 0 bucket checkpoint complete 2022-09-03T06:15:54.816 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-26 2022-09-03T06:15:54.818 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.819 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.820 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-26 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:54.878 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:54.879 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:54.935 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:54.931+0000 7f1ba84c9600 1 bucket sync caught up with source: 2022-09-03T06:15:54.936 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.121.12, , ] 2022-09-03T06:15:54.937 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.121.12, , ] 2022-09-03T06:15:54.937 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:54.931+0000 7f1ba84c9600 0 bucket checkpoint complete 2022-09-03T06:15:54.949 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-26 2022-09-03T06:15:54.950 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-26 2022-09-03T06:15:54.951 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.951 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:54.952 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-26 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:55.009 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:55.010 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:55.060 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:55.052+0000 7f2d46fcf600 1 bucket sync caught up with source: 2022-09-03T06:15:55.061 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000003.121.6, , ] 2022-09-03T06:15:55.061 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000003.121.6, , ] 2022-09-03T06:15:55.062 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:15:55.052+0000 7f2d46fcf600 0 bucket checkpoint complete 2022-09-03T06:15:55.068 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-26 2022-09-03T06:15:55.070 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:55.071 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-26', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:15:55.071 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-26 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:15:55.129 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:55.130 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:55.173 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:55.171+0000 7fc9dacb8600 1 bucket sync caught up with source: 2022-09-03T06:15:55.173 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000003.121.6, , ] 2022-09-03T06:15:55.174 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000003.121.6, , ] 2022-09-03T06:15:55.175 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:55.171+0000 7fc9dacb8600 0 bucket checkpoint complete 2022-09-03T06:15:55.186 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-26 2022-09-03T06:15:55.190 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-26 zones={test-zone2, test-zone1} 2022-09-03T06:15:55.191 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.191 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.192 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.192 DEBUG:boto:Path: /avysmw-26/ 2022-09-03T06:15:55.193 DEBUG:boto:Data: 2022-09-03T06:15:55.193 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.195 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.195 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.196 DEBUG:boto:Params: {} 2022-09-03T06:15:55.197 DEBUG:boto:Token: None 2022-09-03T06:15:55.198 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/ 2022-09-03T06:15:55.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo= 2022-09-03T06:15:55.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo=', 'Content-Length': '0'} 2022-09-03T06:15:55.203 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004ec01ff993e741fe-006312f11b-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.204 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.204 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.205 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.205 DEBUG:boto:Path: /avysmw-26/ 2022-09-03T06:15:55.206 DEBUG:boto:Data: 2022-09-03T06:15:55.207 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.207 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.208 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.209 DEBUG:boto:Params: {} 2022-09-03T06:15:55.209 DEBUG:boto:Token: None 2022-09-03T06:15:55.210 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/ 2022-09-03T06:15:55.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo= 2022-09-03T06:15:55.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo=', 'Content-Length': '0'} 2022-09-03T06:15:55.214 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000733837aa92be1536-006312f11b-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.216 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:15:55.217 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.218 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.218 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.219 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.220 DEBUG:boto:Method: GET 2022-09-03T06:15:55.221 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.222 DEBUG:boto:Data: 2022-09-03T06:15:55.223 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.225 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.226 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.226 DEBUG:boto:Params: {} 2022-09-03T06:15:55.227 DEBUG:boto:Token: None 2022-09-03T06:15:55.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.232 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a0b4889ad63ad62-006312f11b-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.233 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.234 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:55.235 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:15:55.235 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.236 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.236 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.237 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.237 DEBUG:boto:Method: GET 2022-09-03T06:15:55.238 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.238 DEBUG:boto:Data: 2022-09-03T06:15:55.239 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.240 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.240 DEBUG:boto:Params: {} 2022-09-03T06:15:55.241 DEBUG:boto:Token: None 2022-09-03T06:15:55.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.246 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008549bb8842492192-006312f11b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.294 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.296 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:55.297 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.298 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.299 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.299 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.300 DEBUG:boto:Method: GET 2022-09-03T06:15:55.301 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.302 DEBUG:boto:Data: 2022-09-03T06:15:55.303 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.304 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.305 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.306 DEBUG:boto:Params: {} 2022-09-03T06:15:55.307 DEBUG:boto:Token: None 2022-09-03T06:15:55.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.313 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000be8181b456d1ac46-006312f11b-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.314 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.315 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.315 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.316 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.316 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.317 DEBUG:boto:Method: GET 2022-09-03T06:15:55.318 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.318 DEBUG:boto:Data: 2022-09-03T06:15:55.319 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.320 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.320 DEBUG:boto:Params: {} 2022-09-03T06:15:55.321 DEBUG:boto:Token: None 2022-09-03T06:15:55.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.327 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e81dad1c4841e3d-006312f11b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.329 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.330 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:15:55.331 DEBUG:boto:path=/avysmw-26/obj 2022-09-03T06:15:55.332 DEBUG:boto:auth_path=/avysmw-26/obj 2022-09-03T06:15:55.332 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.333 DEBUG:boto:Path: /avysmw-26/obj 2022-09-03T06:15:55.334 DEBUG:boto:Data: 2022-09-03T06:15:55.335 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.336 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.337 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.338 DEBUG:boto:Params: {} 2022-09-03T06:15:55.339 DEBUG:boto:Token: None 2022-09-03T06:15:55.339 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/obj 2022-09-03T06:15:55.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE= 2022-09-03T06:15:55.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE=', 'Content-Length': '0'} 2022-09-03T06:15:55.342 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000f36aa6adf023041d-006312f11b-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.344 DEBUG:boto:path=/avysmw-26/obj 2022-09-03T06:15:55.344 DEBUG:boto:auth_path=/avysmw-26/obj 2022-09-03T06:15:55.345 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.347 DEBUG:boto:Path: /avysmw-26/obj 2022-09-03T06:15:55.348 DEBUG:boto:Data: 2022-09-03T06:15:55.349 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.350 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.351 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.352 DEBUG:boto:Params: {} 2022-09-03T06:15:55.353 DEBUG:boto:Token: None 2022-09-03T06:15:55.357 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/obj 2022-09-03T06:15:55.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE= 2022-09-03T06:15:55.359 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE=', 'Content-Length': '0'} 2022-09-03T06:15:55.361 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000247efae8871a2970-006312f11b-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.363 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-26 zones={test-zone2, test-zone1} 2022-09-03T06:15:55.363 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-26 zones={test-zone3, test-zone1} 2022-09-03T06:15:55.364 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.365 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.366 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.367 DEBUG:boto:Path: /avysmw-26/ 2022-09-03T06:15:55.368 DEBUG:boto:Data: 2022-09-03T06:15:55.369 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.371 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.372 DEBUG:boto:Params: {} 2022-09-03T06:15:55.373 DEBUG:boto:Token: None 2022-09-03T06:15:55.374 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/ 2022-09-03T06:15:55.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo= 2022-09-03T06:15:55.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo=', 'Content-Length': '0'} 2022-09-03T06:15:55.378 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f861667123978238-006312f11b-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.379 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.380 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.381 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.382 DEBUG:boto:Path: /avysmw-26/ 2022-09-03T06:15:55.383 DEBUG:boto:Data: 2022-09-03T06:15:55.384 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.385 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.386 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.387 DEBUG:boto:Params: {} 2022-09-03T06:15:55.388 DEBUG:boto:Token: None 2022-09-03T06:15:55.389 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/ 2022-09-03T06:15:55.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo= 2022-09-03T06:15:55.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo=', 'Content-Length': '0'} 2022-09-03T06:15:55.395 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d2c6a6e894d8a165-006312f11b-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.396 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:15:55.397 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.399 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.400 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.401 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.402 DEBUG:boto:Method: GET 2022-09-03T06:15:55.403 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.404 DEBUG:boto:Data: 2022-09-03T06:15:55.405 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.407 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.408 DEBUG:boto:Params: {} 2022-09-03T06:15:55.409 DEBUG:boto:Token: None 2022-09-03T06:15:55.410 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.414 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000461d49fac153e06c-006312f11b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.415 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.416 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:55.417 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:15:55.418 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.419 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.420 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.421 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.422 DEBUG:boto:Method: GET 2022-09-03T06:15:55.434 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.435 DEBUG:boto:Data: 2022-09-03T06:15:55.436 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.437 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.438 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.439 DEBUG:boto:Params: {} 2022-09-03T06:15:55.440 DEBUG:boto:Token: None 2022-09-03T06:15:55.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.448 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b27c6b0942e1acc-006312f11b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.449 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.450 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:55.452 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.453 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.454 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.455 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.455 DEBUG:boto:Method: GET 2022-09-03T06:15:55.456 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.458 DEBUG:boto:Data: 2022-09-03T06:15:55.459 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.461 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.461 DEBUG:boto:Params: {} 2022-09-03T06:15:55.462 DEBUG:boto:Token: None 2022-09-03T06:15:55.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce6ada4d589c271f-006312f11b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.469 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.469 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.470 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.471 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.471 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.472 DEBUG:boto:Method: GET 2022-09-03T06:15:55.472 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.473 DEBUG:boto:Data: 2022-09-03T06:15:55.474 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.475 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.476 DEBUG:boto:Params: {} 2022-09-03T06:15:55.477 DEBUG:boto:Token: None 2022-09-03T06:15:55.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.483 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e79c924792209ee0-006312f11b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.530 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.531 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:15:55.532 DEBUG:boto:path=/avysmw-26/obj 2022-09-03T06:15:55.532 DEBUG:boto:auth_path=/avysmw-26/obj 2022-09-03T06:15:55.533 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.533 DEBUG:boto:Path: /avysmw-26/obj 2022-09-03T06:15:55.534 DEBUG:boto:Data: 2022-09-03T06:15:55.534 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.536 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.536 DEBUG:boto:Params: {} 2022-09-03T06:15:55.537 DEBUG:boto:Token: None 2022-09-03T06:15:55.537 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/obj 2022-09-03T06:15:55.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE= 2022-09-03T06:15:55.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE=', 'Content-Length': '0'} 2022-09-03T06:15:55.541 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000fea14b1924119931-006312f11b-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.541 DEBUG:boto:path=/avysmw-26/obj 2022-09-03T06:15:55.542 DEBUG:boto:auth_path=/avysmw-26/obj 2022-09-03T06:15:55.542 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.543 DEBUG:boto:Path: /avysmw-26/obj 2022-09-03T06:15:55.543 DEBUG:boto:Data: 2022-09-03T06:15:55.544 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.544 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.545 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.545 DEBUG:boto:Params: {} 2022-09-03T06:15:55.546 DEBUG:boto:Token: None 2022-09-03T06:15:55.547 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/obj 2022-09-03T06:15:55.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE= 2022-09-03T06:15:55.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE=', 'Content-Length': '0'} 2022-09-03T06:15:55.553 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000045018fbf3f039796-006312f11b-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.555 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-26 zones={test-zone3, test-zone1} 2022-09-03T06:15:55.556 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-26 zones={test-zone3, test-zone2} 2022-09-03T06:15:55.557 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.558 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.559 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.560 DEBUG:boto:Path: /avysmw-26/ 2022-09-03T06:15:55.561 DEBUG:boto:Data: 2022-09-03T06:15:55.561 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.562 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.564 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.565 DEBUG:boto:Params: {} 2022-09-03T06:15:55.566 DEBUG:boto:Token: None 2022-09-03T06:15:55.567 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/ 2022-09-03T06:15:55.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo= 2022-09-03T06:15:55.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo=', 'Content-Length': '0'} 2022-09-03T06:15:55.572 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000037459058930d7e-006312f11b-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.574 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.575 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.576 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.577 DEBUG:boto:Path: /avysmw-26/ 2022-09-03T06:15:55.578 DEBUG:boto:Data: 2022-09-03T06:15:55.579 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.580 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.580 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.581 DEBUG:boto:Params: {} 2022-09-03T06:15:55.583 DEBUG:boto:Token: None 2022-09-03T06:15:55.584 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/ 2022-09-03T06:15:55.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo= 2022-09-03T06:15:55.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ibq1VJE7leW1FnxJpdAjfzxPALo=', 'Content-Length': '0'} 2022-09-03T06:15:55.589 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000024b72806d66d8e55-006312f11b-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.591 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:15:55.591 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.592 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.593 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.594 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.595 DEBUG:boto:Method: GET 2022-09-03T06:15:55.596 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.597 DEBUG:boto:Data: 2022-09-03T06:15:55.597 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.598 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.599 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.600 DEBUG:boto:Params: {} 2022-09-03T06:15:55.600 DEBUG:boto:Token: None 2022-09-03T06:15:55.601 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.609 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3f0f562913bbfdb-006312f11b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.611 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.612 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:55.613 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:15:55.614 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.615 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.616 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.617 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.617 DEBUG:boto:Method: GET 2022-09-03T06:15:55.618 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.619 DEBUG:boto:Data: 2022-09-03T06:15:55.620 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.620 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.621 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.621 DEBUG:boto:Params: {} 2022-09-03T06:15:55.622 DEBUG:boto:Token: None 2022-09-03T06:15:55.623 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.628 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044a2e4ed323cc081-006312f11b-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.630 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.631 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2022-09-03T06:15:55.632 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.633 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.634 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.634 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.635 DEBUG:boto:Method: GET 2022-09-03T06:15:55.636 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.637 DEBUG:boto:Data: 2022-09-03T06:15:55.638 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.640 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.641 DEBUG:boto:Params: {} 2022-09-03T06:15:55.642 DEBUG:boto:Token: None 2022-09-03T06:15:55.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.649 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000ac17cf99ee2194d-006312f11b-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.650 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.652 DEBUG:boto:path=/avysmw-26/ 2022-09-03T06:15:55.653 DEBUG:boto:auth_path=/avysmw-26/ 2022-09-03T06:15:55.654 DEBUG:boto:path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.655 DEBUG:boto:auth_path=/avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.656 DEBUG:boto:Method: GET 2022-09-03T06:15:55.656 DEBUG:boto:Path: /avysmw-26/?versions&max-keys=999 2022-09-03T06:15:55.657 DEBUG:boto:Data: 2022-09-03T06:15:55.658 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.660 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.661 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.662 DEBUG:boto:Params: {} 2022-09-03T06:15:55.663 DEBUG:boto:Token: None 2022-09-03T06:15:55.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/?versions 2022-09-03T06:15:55.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY= 2022-09-03T06:15:55.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EWKPhSfd2Q8SlsBEL9Chr3/zyoY=', 'Content-Length': '0'} 2022-09-03T06:15:55.669 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000df763e3c9a0aaa56-006312f11b-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.671 DEBUG:boto:b'avysmw-26999falseobjnulltrue2022-09-03T06:14:06.304Zrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:15:55.672 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2022-09-03T06:15:55.673 DEBUG:boto:path=/avysmw-26/obj 2022-09-03T06:15:55.674 DEBUG:boto:auth_path=/avysmw-26/obj 2022-09-03T06:15:55.675 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.676 DEBUG:boto:Path: /avysmw-26/obj 2022-09-03T06:15:55.677 DEBUG:boto:Data: 2022-09-03T06:15:55.678 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.679 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.680 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.681 DEBUG:boto:Params: {} 2022-09-03T06:15:55.682 DEBUG:boto:Token: None 2022-09-03T06:15:55.684 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/obj 2022-09-03T06:15:55.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE= 2022-09-03T06:15:55.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE=', 'Content-Length': '0'} 2022-09-03T06:15:55.689 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx00000118500c01fd4cb1e-006312f11b-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.689 DEBUG:boto:path=/avysmw-26/obj 2022-09-03T06:15:55.690 DEBUG:boto:auth_path=/avysmw-26/obj 2022-09-03T06:15:55.690 DEBUG:boto:Method: HEAD 2022-09-03T06:15:55.691 DEBUG:boto:Path: /avysmw-26/obj 2022-09-03T06:15:55.691 DEBUG:boto:Data: 2022-09-03T06:15:55.692 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.692 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.693 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.694 DEBUG:boto:Params: {} 2022-09-03T06:15:55.694 DEBUG:boto:Token: None 2022-09-03T06:15:55.695 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:15:55 GMT /avysmw-26/obj 2022-09-03T06:15:55.695 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE= 2022-09-03T06:15:55.696 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+oJLII7eAL3JeorV0C8T5E95SE=', 'Content-Length': '0'} 2022-09-03T06:15:55.698 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx0000062f01f1b7955be2d-006312f11b-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.699 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-26 zones={test-zone3, test-zone2} 2022-09-03T06:15:55.700 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_suspended_delete_marker_full_sync ... ok 2022-09-03T06:15:55.702 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-27 2022-09-03T06:15:55.704 DEBUG:boto:path=/avysmw-27/ 2022-09-03T06:15:55.705 DEBUG:boto:auth_path=/avysmw-27/ 2022-09-03T06:15:55.705 DEBUG:boto:Method: PUT 2022-09-03T06:15:55.706 DEBUG:boto:Path: /avysmw-27/ 2022-09-03T06:15:55.712 DEBUG:boto:Data: 2022-09-03T06:15:55.713 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.715 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.716 DEBUG:boto:Params: {} 2022-09-03T06:15:55.717 DEBUG:boto:Token: None 2022-09-03T06:15:55.718 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:55 GMT /avysmw-27/ 2022-09-03T06:15:55.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hOTcnbgb1Al8FhPteNZdDsh3ubo= 2022-09-03T06:15:55.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hOTcnbgb1Al8FhPteNZdDsh3ubo=', 'Content-Length': '0'} 2022-09-03T06:15:55.748 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000fc14ed78939564af-006312f11b-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.749 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-28 2022-09-03T06:15:55.750 DEBUG:boto:path=/avysmw-28/ 2022-09-03T06:15:55.750 DEBUG:boto:auth_path=/avysmw-28/ 2022-09-03T06:15:55.751 DEBUG:boto:Method: PUT 2022-09-03T06:15:55.751 DEBUG:boto:Path: /avysmw-28/ 2022-09-03T06:15:55.752 DEBUG:boto:Data: 2022-09-03T06:15:55.753 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.753 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.754 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.754 DEBUG:boto:Params: {} 2022-09-03T06:15:55.755 DEBUG:boto:Token: None 2022-09-03T06:15:55.756 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:55 GMT /avysmw-28/ 2022-09-03T06:15:55.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SOpF1JClyv2iyFAkmcqzlW7khUk= 2022-09-03T06:15:55.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SOpF1JClyv2iyFAkmcqzlW7khUk=', 'Content-Length': '0'} 2022-09-03T06:15:55.810 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000008f34cde06a9fc2d4-006312f11b-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.812 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-29 2022-09-03T06:15:55.813 DEBUG:boto:path=/avysmw-29/ 2022-09-03T06:15:55.816 DEBUG:boto:auth_path=/avysmw-29/ 2022-09-03T06:15:55.817 DEBUG:boto:Method: PUT 2022-09-03T06:15:55.818 DEBUG:boto:Path: /avysmw-29/ 2022-09-03T06:15:55.819 DEBUG:boto:Data: 2022-09-03T06:15:55.820 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.821 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.823 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.824 DEBUG:boto:Params: {} 2022-09-03T06:15:55.825 DEBUG:boto:Token: None 2022-09-03T06:15:55.829 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:55 GMT /avysmw-29/ 2022-09-03T06:15:55.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b+l6hYT6fGMUKkmPPl+Nyw1m1eI= 2022-09-03T06:15:55.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b+l6hYT6fGMUKkmPPl+Nyw1m1eI=', 'Content-Length': '0'} 2022-09-03T06:15:55.876 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000b9e6a0e2410134d9-006312f11b-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.878 DEBUG:boto:path=/avysmw-27/ 2022-09-03T06:15:55.879 DEBUG:boto:auth_path=/avysmw-27/ 2022-09-03T06:15:55.880 DEBUG:boto:path=/avysmw-27/?versioning 2022-09-03T06:15:55.881 DEBUG:boto:auth_path=/avysmw-27/?versioning 2022-09-03T06:15:55.882 DEBUG:boto:Method: PUT 2022-09-03T06:15:55.883 DEBUG:boto:Path: /avysmw-27/?versioning 2022-09-03T06:15:55.883 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:15:55.884 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.887 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.888 DEBUG:boto:Params: {} 2022-09-03T06:15:55.889 DEBUG:boto:Token: None 2022-09-03T06:15:55.891 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:55 GMT /avysmw-27/?versioning 2022-09-03T06:15:55.892 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S6HIJZXtyvQ6+gmGP4Jg49koG3Y= 2022-09-03T06:15:55.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S6HIJZXtyvQ6+gmGP4Jg49koG3Y=', 'Content-Length': '228'} 2022-09-03T06:15:55.899 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ee1ca1787aeb1cb5-006312f11b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.900 DEBUG:boto:path=/avysmw-27/ 2022-09-03T06:15:55.900 DEBUG:boto:auth_path=/avysmw-27/ 2022-09-03T06:15:55.901 DEBUG:boto:path=/avysmw-27/?versioning 2022-09-03T06:15:55.901 DEBUG:boto:auth_path=/avysmw-27/?versioning 2022-09-03T06:15:55.902 DEBUG:boto:Method: GET 2022-09-03T06:15:55.903 DEBUG:boto:Path: /avysmw-27/?versioning 2022-09-03T06:15:55.903 DEBUG:boto:Data: 2022-09-03T06:15:55.904 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.904 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:55.905 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.905 DEBUG:boto:Params: {} 2022-09-03T06:15:55.906 DEBUG:boto:Token: None 2022-09-03T06:15:55.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-27/?versioning 2022-09-03T06:15:55.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MBtMUOWLh5lIEnUlhL0El1OOjHQ= 2022-09-03T06:15:55.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MBtMUOWLh5lIEnUlhL0El1OOjHQ=', 'Content-Length': '0'} 2022-09-03T06:15:55.909 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000093665137f727d3c-006312f11b-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '192'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.910 DEBUG:boto:EnabledDisabled 2022-09-03T06:15:55.911 DEBUG:boto:path=/avysmw-28/ 2022-09-03T06:15:55.912 DEBUG:boto:auth_path=/avysmw-28/ 2022-09-03T06:15:55.912 DEBUG:boto:path=/avysmw-28/?versioning 2022-09-03T06:15:55.913 DEBUG:boto:auth_path=/avysmw-28/?versioning 2022-09-03T06:15:55.913 DEBUG:boto:Method: PUT 2022-09-03T06:15:55.914 DEBUG:boto:Path: /avysmw-28/?versioning 2022-09-03T06:15:55.914 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:15:55.915 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.915 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.916 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.916 DEBUG:boto:Params: {} 2022-09-03T06:15:55.917 DEBUG:boto:Token: None 2022-09-03T06:15:55.918 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:55 GMT /avysmw-28/?versioning 2022-09-03T06:15:55.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kP01XNbVFFX76FvrB9O4iKyJ9Gk= 2022-09-03T06:15:55.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kP01XNbVFFX76FvrB9O4iKyJ9Gk=', 'Content-Length': '228'} 2022-09-03T06:15:55.927 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000515447d2a31ed45d-006312f11b-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.928 DEBUG:boto:path=/avysmw-28/ 2022-09-03T06:15:55.929 DEBUG:boto:auth_path=/avysmw-28/ 2022-09-03T06:15:55.931 DEBUG:boto:path=/avysmw-28/?versioning 2022-09-03T06:15:55.931 DEBUG:boto:auth_path=/avysmw-28/?versioning 2022-09-03T06:15:55.932 DEBUG:boto:Method: GET 2022-09-03T06:15:55.933 DEBUG:boto:Path: /avysmw-28/?versioning 2022-09-03T06:15:55.934 DEBUG:boto:Data: 2022-09-03T06:15:55.934 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.935 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:55.936 DEBUG:boto:Port: 8000 2022-09-03T06:15:55.937 DEBUG:boto:Params: {} 2022-09-03T06:15:55.937 DEBUG:boto:Token: None 2022-09-03T06:15:55.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:55 GMT /avysmw-28/?versioning 2022-09-03T06:15:55.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HQkpinrHHbNuloxkuyK2XlXDmZE= 2022-09-03T06:15:55.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HQkpinrHHbNuloxkuyK2XlXDmZE=', 'Content-Length': '0'} 2022-09-03T06:15:55.941 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000006a0d77340be5c60e-006312f11b-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '192'), ('Date', 'Sat, 03 Sep 2022 06:15:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:55.990 DEBUG:boto:EnabledDisabled 2022-09-03T06:15:55.991 DEBUG:boto:path=/avysmw-29/ 2022-09-03T06:15:55.992 DEBUG:boto:auth_path=/avysmw-29/ 2022-09-03T06:15:55.992 DEBUG:boto:path=/avysmw-29/?versioning 2022-09-03T06:15:55.992 DEBUG:boto:auth_path=/avysmw-29/?versioning 2022-09-03T06:15:55.993 DEBUG:boto:Method: PUT 2022-09-03T06:15:55.994 DEBUG:boto:Path: /avysmw-29/?versioning 2022-09-03T06:15:55.994 DEBUG:boto:Data: Enabled Disabled 2022-09-03T06:15:55.995 DEBUG:boto:Headers: {} 2022-09-03T06:15:55.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:55.996 DEBUG:boto:Port: 8001 2022-09-03T06:15:55.996 DEBUG:boto:Params: {} 2022-09-03T06:15:55.997 DEBUG:boto:Token: None 2022-09-03T06:15:55.997 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:55 GMT /avysmw-29/?versioning 2022-09-03T06:15:55.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:anjOrNKjLW0wmOEhsjVsp2h83LY= 2022-09-03T06:15:55.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:anjOrNKjLW0wmOEhsjVsp2h83LY=', 'Content-Length': '228'} 2022-09-03T06:15:56.017 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000cb30e20f3fd2663e-006312f11c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.019 DEBUG:boto:path=/avysmw-29/ 2022-09-03T06:15:56.020 DEBUG:boto:auth_path=/avysmw-29/ 2022-09-03T06:15:56.020 DEBUG:boto:path=/avysmw-29/?versioning 2022-09-03T06:15:56.021 DEBUG:boto:auth_path=/avysmw-29/?versioning 2022-09-03T06:15:56.022 DEBUG:boto:Method: GET 2022-09-03T06:15:56.023 DEBUG:boto:Path: /avysmw-29/?versioning 2022-09-03T06:15:56.025 DEBUG:boto:Data: 2022-09-03T06:15:56.026 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.027 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:56.027 DEBUG:boto:Port: 8001 2022-09-03T06:15:56.029 DEBUG:boto:Params: {} 2022-09-03T06:15:56.030 DEBUG:boto:Token: None 2022-09-03T06:15:56.030 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:56 GMT /avysmw-29/?versioning 2022-09-03T06:15:56.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LoEe7tTaEtbWRtrjAhL9FSZ9gOw= 2022-09-03T06:15:56.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LoEe7tTaEtbWRtrjAhL9FSZ9gOw=', 'Content-Length': '0'} 2022-09-03T06:15:56.033 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ca35c78ef8ad3c3d-006312f11c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '192'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.033 DEBUG:boto:EnabledDisabled 2022-09-03T06:15:56.034 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_versioning ... ok 2022-09-03T06:15:56.035 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-30 2022-09-03T06:15:56.036 DEBUG:boto:path=/avysmw-30/ 2022-09-03T06:15:56.037 DEBUG:boto:auth_path=/avysmw-30/ 2022-09-03T06:15:56.039 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.040 DEBUG:boto:Path: /avysmw-30/ 2022-09-03T06:15:56.042 DEBUG:boto:Data: 2022-09-03T06:15:56.043 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:56.045 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.045 DEBUG:boto:Params: {} 2022-09-03T06:15:56.046 DEBUG:boto:Token: None 2022-09-03T06:15:56.047 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:56 GMT /avysmw-30/ 2022-09-03T06:15:56.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EtcNaZR7EVGadjNUwmNj0paz0Ng= 2022-09-03T06:15:56.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EtcNaZR7EVGadjNUwmNj0paz0Ng=', 'Content-Length': '0'} 2022-09-03T06:15:56.076 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000159acbf7751ef3da-006312f11c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.077 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-31 2022-09-03T06:15:56.078 DEBUG:boto:path=/avysmw-31/ 2022-09-03T06:15:56.079 DEBUG:boto:auth_path=/avysmw-31/ 2022-09-03T06:15:56.080 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.081 DEBUG:boto:Path: /avysmw-31/ 2022-09-03T06:15:56.082 DEBUG:boto:Data: 2022-09-03T06:15:56.083 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.085 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:56.086 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.086 DEBUG:boto:Params: {} 2022-09-03T06:15:56.087 DEBUG:boto:Token: None 2022-09-03T06:15:56.087 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:56 GMT /avysmw-31/ 2022-09-03T06:15:56.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:USY0KrLYcDoGoIGKtiCRaNapycQ= 2022-09-03T06:15:56.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:USY0KrLYcDoGoIGKtiCRaNapycQ=', 'Content-Length': '0'} 2022-09-03T06:15:56.129 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c4788a6c79b0e2e1-006312f11c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.130 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-32 2022-09-03T06:15:56.131 DEBUG:boto:path=/avysmw-32/ 2022-09-03T06:15:56.131 DEBUG:boto:auth_path=/avysmw-32/ 2022-09-03T06:15:56.132 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.132 DEBUG:boto:Path: /avysmw-32/ 2022-09-03T06:15:56.133 DEBUG:boto:Data: 2022-09-03T06:15:56.133 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:56.134 DEBUG:boto:Port: 8001 2022-09-03T06:15:56.135 DEBUG:boto:Params: {} 2022-09-03T06:15:56.135 DEBUG:boto:Token: None 2022-09-03T06:15:56.136 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:56 GMT /avysmw-32/ 2022-09-03T06:15:56.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80ty2gElsEEn8WzNKwb+UlzqAqw= 2022-09-03T06:15:56.137 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80ty2gElsEEn8WzNKwb+UlzqAqw=', 'Content-Length': '0'} 2022-09-03T06:15:56.175 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000000c98a811c221d820-006312f11c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.176 DEBUG:boto:path=/avysmw-30/ 2022-09-03T06:15:56.177 DEBUG:boto:auth_path=/avysmw-30/ 2022-09-03T06:15:56.177 DEBUG:boto:path=/avysmw-30/?acl 2022-09-03T06:15:56.178 DEBUG:boto:auth_path=/avysmw-30/?acl 2022-09-03T06:15:56.179 DEBUG:boto:Method: GET 2022-09-03T06:15:56.182 DEBUG:boto:Path: /avysmw-30/?acl 2022-09-03T06:15:56.185 DEBUG:boto:Data: 2022-09-03T06:15:56.186 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:56.193 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.194 DEBUG:boto:Params: {} 2022-09-03T06:15:56.195 DEBUG:boto:Token: None 2022-09-03T06:15:56.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:56 GMT /avysmw-30/?acl 2022-09-03T06:15:56.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GONmlqVVTtR1PFd8m/FGq4zVkS0= 2022-09-03T06:15:56.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GONmlqVVTtR1PFd8m/FGq4zVkS0=', 'Content-Length': '0'} 2022-09-03T06:15:56.202 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000d6c2df5c4ca45304-006312f11c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '489'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.204 DEBUG:boto:path=/avysmw-30/ 2022-09-03T06:15:56.205 DEBUG:boto:auth_path=/avysmw-30/ 2022-09-03T06:15:56.205 DEBUG:boto:path=/avysmw-30/?acl 2022-09-03T06:15:56.206 DEBUG:boto:auth_path=/avysmw-30/?acl 2022-09-03T06:15:56.207 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.208 DEBUG:boto:Path: /avysmw-30/?acl 2022-09-03T06:15:56.209 DEBUG:boto:Data: 2022-09-03T06:15:56.211 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T06:15:56.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:56.213 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.215 DEBUG:boto:Params: {} 2022-09-03T06:15:56.216 DEBUG:boto:Token: None 2022-09-03T06:15:56.217 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:56 GMT x-amz-acl:public-read /avysmw-30/?acl 2022-09-03T06:15:56.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dU+m3sc3XfzW5urliylpPCoInfE= 2022-09-03T06:15:56.220 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dU+m3sc3XfzW5urliylpPCoInfE=', 'Content-Length': '0'} 2022-09-03T06:15:56.227 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000007434412ad47596de-006312f11c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.228 DEBUG:boto:path=/avysmw-30/ 2022-09-03T06:15:56.229 DEBUG:boto:auth_path=/avysmw-30/ 2022-09-03T06:15:56.231 DEBUG:boto:path=/avysmw-30/?acl 2022-09-03T06:15:56.233 DEBUG:boto:auth_path=/avysmw-30/?acl 2022-09-03T06:15:56.234 DEBUG:boto:Method: GET 2022-09-03T06:15:56.235 DEBUG:boto:Path: /avysmw-30/?acl 2022-09-03T06:15:56.237 DEBUG:boto:Data: 2022-09-03T06:15:56.238 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:56.240 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.241 DEBUG:boto:Params: {} 2022-09-03T06:15:56.242 DEBUG:boto:Token: None 2022-09-03T06:15:56.243 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:56 GMT /avysmw-30/?acl 2022-09-03T06:15:56.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GONmlqVVTtR1PFd8m/FGq4zVkS0= 2022-09-03T06:15:56.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GONmlqVVTtR1PFd8m/FGq4zVkS0=', 'Content-Length': '0'} 2022-09-03T06:15:56.247 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000004f492a3bdbc44144-006312f11c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '681'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.299 DEBUG:boto:path=/avysmw-31/ 2022-09-03T06:15:56.300 DEBUG:boto:auth_path=/avysmw-31/ 2022-09-03T06:15:56.301 DEBUG:boto:path=/avysmw-31/?acl 2022-09-03T06:15:56.301 DEBUG:boto:auth_path=/avysmw-31/?acl 2022-09-03T06:15:56.302 DEBUG:boto:Method: GET 2022-09-03T06:15:56.302 DEBUG:boto:Path: /avysmw-31/?acl 2022-09-03T06:15:56.303 DEBUG:boto:Data: 2022-09-03T06:15:56.303 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.303 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:56.304 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.304 DEBUG:boto:Params: {} 2022-09-03T06:15:56.305 DEBUG:boto:Token: None 2022-09-03T06:15:56.306 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:56 GMT /avysmw-31/?acl 2022-09-03T06:15:56.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wwe2MlHZTB97npZbN+aAXYpezA= 2022-09-03T06:15:56.308 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wwe2MlHZTB97npZbN+aAXYpezA=', 'Content-Length': '0'} 2022-09-03T06:15:56.310 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000a1f868daef78d1fc-006312f11c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '489'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.314 DEBUG:boto:path=/avysmw-31/ 2022-09-03T06:15:56.315 DEBUG:boto:auth_path=/avysmw-31/ 2022-09-03T06:15:56.315 DEBUG:boto:path=/avysmw-31/?acl 2022-09-03T06:15:56.316 DEBUG:boto:auth_path=/avysmw-31/?acl 2022-09-03T06:15:56.316 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.317 DEBUG:boto:Path: /avysmw-31/?acl 2022-09-03T06:15:56.317 DEBUG:boto:Data: 2022-09-03T06:15:56.318 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T06:15:56.319 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:56.319 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.319 DEBUG:boto:Params: {} 2022-09-03T06:15:56.320 DEBUG:boto:Token: None 2022-09-03T06:15:56.321 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:56 GMT x-amz-acl:public-read /avysmw-31/?acl 2022-09-03T06:15:56.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nbRFMMb8XD7hfoMAi6EEdXYIbns= 2022-09-03T06:15:56.322 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nbRFMMb8XD7hfoMAi6EEdXYIbns=', 'Content-Length': '0'} 2022-09-03T06:15:56.338 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000a068db30067b9d9a-006312f11c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.339 DEBUG:boto:path=/avysmw-31/ 2022-09-03T06:15:56.340 DEBUG:boto:auth_path=/avysmw-31/ 2022-09-03T06:15:56.340 DEBUG:boto:path=/avysmw-31/?acl 2022-09-03T06:15:56.341 DEBUG:boto:auth_path=/avysmw-31/?acl 2022-09-03T06:15:56.342 DEBUG:boto:Method: GET 2022-09-03T06:15:56.342 DEBUG:boto:Path: /avysmw-31/?acl 2022-09-03T06:15:56.343 DEBUG:boto:Data: 2022-09-03T06:15:56.343 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.344 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:56.344 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.345 DEBUG:boto:Params: {} 2022-09-03T06:15:56.345 DEBUG:boto:Token: None 2022-09-03T06:15:56.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:56 GMT /avysmw-31/?acl 2022-09-03T06:15:56.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6wwe2MlHZTB97npZbN+aAXYpezA= 2022-09-03T06:15:56.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6wwe2MlHZTB97npZbN+aAXYpezA=', 'Content-Length': '0'} 2022-09-03T06:15:56.352 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000000292ed49ede57808-006312f11c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '681'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.353 DEBUG:boto:path=/avysmw-32/ 2022-09-03T06:15:56.353 DEBUG:boto:auth_path=/avysmw-32/ 2022-09-03T06:15:56.354 DEBUG:boto:path=/avysmw-32/?acl 2022-09-03T06:15:56.354 DEBUG:boto:auth_path=/avysmw-32/?acl 2022-09-03T06:15:56.355 DEBUG:boto:Method: GET 2022-09-03T06:15:56.355 DEBUG:boto:Path: /avysmw-32/?acl 2022-09-03T06:15:56.357 DEBUG:boto:Data: 2022-09-03T06:15:56.358 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:56.362 DEBUG:boto:Port: 8001 2022-09-03T06:15:56.363 DEBUG:boto:Params: {} 2022-09-03T06:15:56.364 DEBUG:boto:Token: None 2022-09-03T06:15:56.365 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:56 GMT /avysmw-32/?acl 2022-09-03T06:15:56.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7s9GAhoXj/TLiKjAeTB04prxM/A= 2022-09-03T06:15:56.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7s9GAhoXj/TLiKjAeTB04prxM/A=', 'Content-Length': '0'} 2022-09-03T06:15:56.370 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000002dc14917224697e-006312f11c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '489'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.371 DEBUG:boto:path=/avysmw-32/ 2022-09-03T06:15:56.373 DEBUG:boto:auth_path=/avysmw-32/ 2022-09-03T06:15:56.374 DEBUG:boto:path=/avysmw-32/?acl 2022-09-03T06:15:56.374 DEBUG:boto:auth_path=/avysmw-32/?acl 2022-09-03T06:15:56.375 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.376 DEBUG:boto:Path: /avysmw-32/?acl 2022-09-03T06:15:56.377 DEBUG:boto:Data: 2022-09-03T06:15:56.378 DEBUG:boto:Headers: {'x-amz-acl': 'public-read'} 2022-09-03T06:15:56.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:56.380 DEBUG:boto:Port: 8001 2022-09-03T06:15:56.383 DEBUG:boto:Params: {} 2022-09-03T06:15:56.383 DEBUG:boto:Token: None 2022-09-03T06:15:56.384 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:56 GMT x-amz-acl:public-read /avysmw-32/?acl 2022-09-03T06:15:56.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eB1Erxr+g/VOtPZo/Q/JKxavmig= 2022-09-03T06:15:56.385 DEBUG:boto:Final headers: {'x-amz-acl': 'public-read', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eB1Erxr+g/VOtPZo/Q/JKxavmig=', 'Content-Length': '0'} 2022-09-03T06:15:56.396 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000048205f5aea2871e2-006312f11c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.397 DEBUG:boto:path=/avysmw-32/ 2022-09-03T06:15:56.397 DEBUG:boto:auth_path=/avysmw-32/ 2022-09-03T06:15:56.398 DEBUG:boto:path=/avysmw-32/?acl 2022-09-03T06:15:56.398 DEBUG:boto:auth_path=/avysmw-32/?acl 2022-09-03T06:15:56.399 DEBUG:boto:Method: GET 2022-09-03T06:15:56.399 DEBUG:boto:Path: /avysmw-32/?acl 2022-09-03T06:15:56.400 DEBUG:boto:Data: 2022-09-03T06:15:56.400 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:56.401 DEBUG:boto:Port: 8001 2022-09-03T06:15:56.402 DEBUG:boto:Params: {} 2022-09-03T06:15:56.403 DEBUG:boto:Token: None 2022-09-03T06:15:56.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:56 GMT /avysmw-32/?acl 2022-09-03T06:15:56.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7s9GAhoXj/TLiKjAeTB04prxM/A= 2022-09-03T06:15:56.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7s9GAhoXj/TLiKjAeTB04prxM/A=', 'Content-Length': '0'} 2022-09-03T06:15:56.408 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000043c7e4ad61cded20-006312f11c-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '681'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.410 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_acl ... ok 2022-09-03T06:15:56.411 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-33 2022-09-03T06:15:56.412 DEBUG:boto:path=/avysmw-33/ 2022-09-03T06:15:56.414 DEBUG:boto:auth_path=/avysmw-33/ 2022-09-03T06:15:56.415 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.418 DEBUG:boto:Path: /avysmw-33/ 2022-09-03T06:15:56.418 DEBUG:boto:Data: 2022-09-03T06:15:56.419 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:56.420 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.420 DEBUG:boto:Params: {} 2022-09-03T06:15:56.421 DEBUG:boto:Token: None 2022-09-03T06:15:56.422 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:56 GMT /avysmw-33/ 2022-09-03T06:15:56.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hY6aIXGW6eTWWqknImR35WiEP6s= 2022-09-03T06:15:56.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hY6aIXGW6eTWWqknImR35WiEP6s=', 'Content-Length': '0'} 2022-09-03T06:15:56.438 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c17d20e6f604cd15-006312f11c-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.439 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-34 2022-09-03T06:15:56.440 DEBUG:boto:path=/avysmw-34/ 2022-09-03T06:15:56.441 DEBUG:boto:auth_path=/avysmw-34/ 2022-09-03T06:15:56.442 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.443 DEBUG:boto:Path: /avysmw-34/ 2022-09-03T06:15:56.444 DEBUG:boto:Data: 2022-09-03T06:15:56.445 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.447 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:56.448 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.449 DEBUG:boto:Params: {} 2022-09-03T06:15:56.450 DEBUG:boto:Token: None 2022-09-03T06:15:56.451 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:56 GMT /avysmw-34/ 2022-09-03T06:15:56.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gk/Jej2lNrYRsxhSH4kkeceNCx4= 2022-09-03T06:15:56.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gk/Jej2lNrYRsxhSH4kkeceNCx4=', 'Content-Length': '0'} 2022-09-03T06:15:56.490 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000007115bc1f4f8e3bd8-006312f11c-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.491 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-35 2022-09-03T06:15:56.492 DEBUG:boto:path=/avysmw-35/ 2022-09-03T06:15:56.492 DEBUG:boto:auth_path=/avysmw-35/ 2022-09-03T06:15:56.493 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.493 DEBUG:boto:Path: /avysmw-35/ 2022-09-03T06:15:56.497 DEBUG:boto:Data: 2022-09-03T06:15:56.497 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:56.501 DEBUG:boto:Port: 8001 2022-09-03T06:15:56.501 DEBUG:boto:Params: {} 2022-09-03T06:15:56.502 DEBUG:boto:Token: None 2022-09-03T06:15:56.502 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:56 GMT /avysmw-35/ 2022-09-03T06:15:56.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2orYOd00L/pOvYoBj7XrzLaeq7o= 2022-09-03T06:15:56.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2orYOd00L/pOvYoBj7XrzLaeq7o=', 'Content-Length': '0'} 2022-09-03T06:15:56.545 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000006f1049b0706c58af-006312f11c-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.546 DEBUG:boto:path=/avysmw-33/ 2022-09-03T06:15:56.547 DEBUG:boto:auth_path=/avysmw-33/ 2022-09-03T06:15:56.548 DEBUG:boto:path=/avysmw-33/?cors 2022-09-03T06:15:56.549 DEBUG:boto:auth_path=/avysmw-33/?cors 2022-09-03T06:15:56.559 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.560 DEBUG:boto:Path: /avysmw-33/?cors 2022-09-03T06:15:56.560 DEBUG:boto:Data: DELETEhttps://www.example.com*3000 2022-09-03T06:15:56.561 DEBUG:boto:Headers: {'Content-MD5': 'c5gEzAIzM7zcw8WVE+AbrQ==', 'Content-Type': 'text/xml'} 2022-09-03T06:15:56.561 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:56.562 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.562 DEBUG:boto:Params: {} 2022-09-03T06:15:56.563 DEBUG:boto:Token: None 2022-09-03T06:15:56.563 DEBUG:boto:StringToSign: PUT c5gEzAIzM7zcw8WVE+AbrQ== text/xml Sat, 03 Sep 2022 06:15:56 GMT /avysmw-33/?cors 2022-09-03T06:15:56.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mQpbFJbtv/ZJoDei4A4BFirBkjo= 2022-09-03T06:15:56.564 DEBUG:boto:Final headers: {'Content-MD5': 'c5gEzAIzM7zcw8WVE+AbrQ==', 'Content-Type': 'text/xml', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mQpbFJbtv/ZJoDei4A4BFirBkjo=', 'Content-Length': '218'} 2022-09-03T06:15:56.574 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000a4d737f0c7d8f34a-006312f11c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.575 DEBUG:boto:path=/avysmw-33/ 2022-09-03T06:15:56.576 DEBUG:boto:auth_path=/avysmw-33/ 2022-09-03T06:15:56.577 DEBUG:boto:path=/avysmw-33/?cors 2022-09-03T06:15:56.578 DEBUG:boto:auth_path=/avysmw-33/?cors 2022-09-03T06:15:56.578 DEBUG:boto:Method: GET 2022-09-03T06:15:56.579 DEBUG:boto:Path: /avysmw-33/?cors 2022-09-03T06:15:56.580 DEBUG:boto:Data: 2022-09-03T06:15:56.583 DEBUG:boto:Headers: {} 2022-09-03T06:15:56.584 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:56.585 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.585 DEBUG:boto:Params: {} 2022-09-03T06:15:56.586 DEBUG:boto:Token: None 2022-09-03T06:15:56.587 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:56 GMT /avysmw-33/?cors 2022-09-03T06:15:56.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wKdRAK6QoSptjWYfTuIlO64O+Ws= 2022-09-03T06:15:56.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wKdRAK6QoSptjWYfTuIlO64O+Ws=', 'Content-Length': '0'} 2022-09-03T06:15:56.591 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000073f3c910445d4f73-006312f11c-10f6-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '266'), ('Date', 'Sat, 03 Sep 2022 06:15:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:56.591 DEBUG:boto:b'DELETEhttps://www.example.com*3000' 2022-09-03T06:15:56.592 DEBUG:boto:path=/avysmw-34/ 2022-09-03T06:15:56.594 DEBUG:boto:auth_path=/avysmw-34/ 2022-09-03T06:15:56.596 DEBUG:boto:path=/avysmw-34/?cors 2022-09-03T06:15:56.596 DEBUG:boto:auth_path=/avysmw-34/?cors 2022-09-03T06:15:56.597 DEBUG:boto:Method: PUT 2022-09-03T06:15:56.597 DEBUG:boto:Path: /avysmw-34/?cors 2022-09-03T06:15:56.597 DEBUG:boto:Data: DELETEhttps://www.example.com*3000 2022-09-03T06:15:56.598 DEBUG:boto:Headers: {'Content-MD5': 'c5gEzAIzM7zcw8WVE+AbrQ==', 'Content-Type': 'text/xml'} 2022-09-03T06:15:56.598 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:56.599 DEBUG:boto:Port: 8000 2022-09-03T06:15:56.599 DEBUG:boto:Params: {} 2022-09-03T06:15:56.601 DEBUG:boto:Token: None 2022-09-03T06:15:56.601 DEBUG:boto:StringToSign: PUT c5gEzAIzM7zcw8WVE+AbrQ== text/xml Sat, 03 Sep 2022 06:15:56 GMT /avysmw-34/?cors 2022-09-03T06:15:56.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fWpLyCo5rr/0WrNtQV9oTFDQz0U= 2022-09-03T06:15:56.602 DEBUG:boto:Final headers: {'Content-MD5': 'c5gEzAIzM7zcw8WVE+AbrQ==', 'Content-Type': 'text/xml', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fWpLyCo5rr/0WrNtQV9oTFDQz0U=', 'Content-Length': '218'} 2022-09-03T06:15:57.612 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000807fa4da7643845c-006312f11c-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:57.613 DEBUG:boto:path=/avysmw-34/ 2022-09-03T06:15:57.614 DEBUG:boto:auth_path=/avysmw-34/ 2022-09-03T06:15:57.614 DEBUG:boto:path=/avysmw-34/?cors 2022-09-03T06:15:57.615 DEBUG:boto:auth_path=/avysmw-34/?cors 2022-09-03T06:15:57.615 DEBUG:boto:Method: GET 2022-09-03T06:15:57.616 DEBUG:boto:Path: /avysmw-34/?cors 2022-09-03T06:15:57.616 DEBUG:boto:Data: 2022-09-03T06:15:57.617 DEBUG:boto:Headers: {} 2022-09-03T06:15:57.617 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:57.621 DEBUG:boto:Port: 8000 2022-09-03T06:15:57.623 DEBUG:boto:Params: {} 2022-09-03T06:15:57.624 DEBUG:boto:Token: None 2022-09-03T06:15:57.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:57 GMT /avysmw-34/?cors 2022-09-03T06:15:57.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W69x3YSWDMHlCEzl7lc/lJbcHpM= 2022-09-03T06:15:57.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W69x3YSWDMHlCEzl7lc/lJbcHpM=', 'Content-Length': '0'} 2022-09-03T06:15:57.629 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c2be1e2714660484-006312f11d-10ee-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '266'), ('Date', 'Sat, 03 Sep 2022 06:15:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:57.673 DEBUG:boto:b'DELETEhttps://www.example.com*3000' 2022-09-03T06:15:57.674 DEBUG:boto:path=/avysmw-35/ 2022-09-03T06:15:57.675 DEBUG:boto:auth_path=/avysmw-35/ 2022-09-03T06:15:57.677 DEBUG:boto:path=/avysmw-35/?cors 2022-09-03T06:15:57.678 DEBUG:boto:auth_path=/avysmw-35/?cors 2022-09-03T06:15:57.678 DEBUG:boto:Method: PUT 2022-09-03T06:15:57.680 DEBUG:boto:Path: /avysmw-35/?cors 2022-09-03T06:15:57.681 DEBUG:boto:Data: DELETEhttps://www.example.com*3000 2022-09-03T06:15:57.681 DEBUG:boto:Headers: {'Content-MD5': 'c5gEzAIzM7zcw8WVE+AbrQ==', 'Content-Type': 'text/xml'} 2022-09-03T06:15:57.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:57.684 DEBUG:boto:Port: 8001 2022-09-03T06:15:57.685 DEBUG:boto:Params: {} 2022-09-03T06:15:57.686 DEBUG:boto:Token: None 2022-09-03T06:15:57.687 DEBUG:boto:StringToSign: PUT c5gEzAIzM7zcw8WVE+AbrQ== text/xml Sat, 03 Sep 2022 06:15:57 GMT /avysmw-35/?cors 2022-09-03T06:15:57.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ixc9HkXAg1x96evR7IRzHEaY6RQ= 2022-09-03T06:15:57.689 DEBUG:boto:Final headers: {'Content-MD5': 'c5gEzAIzM7zcw8WVE+AbrQ==', 'Content-Type': 'text/xml', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ixc9HkXAg1x96evR7IRzHEaY6RQ=', 'Content-Length': '218'} 2022-09-03T06:15:58.705 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000899474e1bcbb8658-006312f11d-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:58.706 DEBUG:boto:path=/avysmw-35/ 2022-09-03T06:15:58.707 DEBUG:boto:auth_path=/avysmw-35/ 2022-09-03T06:15:58.707 DEBUG:boto:path=/avysmw-35/?cors 2022-09-03T06:15:58.708 DEBUG:boto:auth_path=/avysmw-35/?cors 2022-09-03T06:15:58.710 DEBUG:boto:Method: GET 2022-09-03T06:15:58.711 DEBUG:boto:Path: /avysmw-35/?cors 2022-09-03T06:15:58.712 DEBUG:boto:Data: 2022-09-03T06:15:58.712 DEBUG:boto:Headers: {} 2022-09-03T06:15:58.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:58.714 DEBUG:boto:Port: 8001 2022-09-03T06:15:58.715 DEBUG:boto:Params: {} 2022-09-03T06:15:58.716 DEBUG:boto:Token: None 2022-09-03T06:15:58.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:15:58 GMT /avysmw-35/?cors 2022-09-03T06:15:58.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ah5sBlqJlb4HKq2VzYXedWo29io= 2022-09-03T06:15:58.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ah5sBlqJlb4HKq2VzYXedWo29io=', 'Content-Length': '0'} 2022-09-03T06:15:58.722 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000053627a7dff273e8c-006312f11e-10fa-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '266'), ('Date', 'Sat, 03 Sep 2022 06:15:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:58.766 DEBUG:boto:b'DELETEhttps://www.example.com*3000' 2022-09-03T06:15:58.770 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_cors ... ok 2022-09-03T06:15:58.772 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-36 2022-09-03T06:15:58.773 DEBUG:boto:path=/avysmw-36/ 2022-09-03T06:15:58.774 DEBUG:boto:auth_path=/avysmw-36/ 2022-09-03T06:15:58.775 DEBUG:boto:Method: PUT 2022-09-03T06:15:58.776 DEBUG:boto:Path: /avysmw-36/ 2022-09-03T06:15:58.776 DEBUG:boto:Data: 2022-09-03T06:15:58.777 DEBUG:boto:Headers: {} 2022-09-03T06:15:58.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:15:58.791 DEBUG:boto:Port: 8000 2022-09-03T06:15:58.791 DEBUG:boto:Params: {} 2022-09-03T06:15:58.793 DEBUG:boto:Token: None 2022-09-03T06:15:58.794 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:58 GMT /avysmw-36/ 2022-09-03T06:15:58.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PUotXQEnrx3viTJpVJAqcp6gVxE= 2022-09-03T06:15:58.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PUotXQEnrx3viTJpVJAqcp6gVxE=', 'Content-Length': '0'} 2022-09-03T06:15:58.821 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000001a3faf872d3b703e-006312f11e-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:58.823 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-37 2022-09-03T06:15:58.823 DEBUG:boto:path=/avysmw-37/ 2022-09-03T06:15:58.824 DEBUG:boto:auth_path=/avysmw-37/ 2022-09-03T06:15:58.824 DEBUG:boto:Method: PUT 2022-09-03T06:15:58.825 DEBUG:boto:Path: /avysmw-37/ 2022-09-03T06:15:58.825 DEBUG:boto:Data: 2022-09-03T06:15:58.826 DEBUG:boto:Headers: {} 2022-09-03T06:15:58.826 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:15:58.827 DEBUG:boto:Port: 8000 2022-09-03T06:15:58.827 DEBUG:boto:Params: {} 2022-09-03T06:15:58.828 DEBUG:boto:Token: None 2022-09-03T06:15:58.829 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:58 GMT /avysmw-37/ 2022-09-03T06:15:58.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wQcNsR3IM28ugk4oHLwQF8Izgls= 2022-09-03T06:15:58.831 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wQcNsR3IM28ugk4oHLwQF8Izgls=', 'Content-Length': '0'} 2022-09-03T06:15:58.875 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000006d364f50dc4b18ae-006312f11e-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:58.876 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-38 2022-09-03T06:15:58.877 DEBUG:boto:path=/avysmw-38/ 2022-09-03T06:15:58.878 DEBUG:boto:auth_path=/avysmw-38/ 2022-09-03T06:15:58.879 DEBUG:boto:Method: PUT 2022-09-03T06:15:58.880 DEBUG:boto:Path: /avysmw-38/ 2022-09-03T06:15:58.881 DEBUG:boto:Data: 2022-09-03T06:15:58.882 DEBUG:boto:Headers: {} 2022-09-03T06:15:58.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:15:58.884 DEBUG:boto:Port: 8001 2022-09-03T06:15:58.884 DEBUG:boto:Params: {} 2022-09-03T06:15:58.885 DEBUG:boto:Token: None 2022-09-03T06:15:58.887 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:15:58 GMT /avysmw-38/ 2022-09-03T06:15:58.888 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5vaN4s7oY/2zxBCzg7IAR7jgNRw= 2022-09-03T06:15:58.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:15:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5vaN4s7oY/2zxBCzg7IAR7jgNRw=', 'Content-Length': '0'} 2022-09-03T06:15:58.951 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ab61ad6805fb150b-006312f11e-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:15:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:15:58.952 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:15:58.953 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:15:58.957 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:15:59.013 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:59.014 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:59.043 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T06:15:59.057 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:15:59.058 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:15:59.058 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:15:59.059 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T06:15:59.059 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:15:59.060 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:15:59.060 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185758.915547_281.1", 2022-09-03T06:15:59.061 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:15:58.915547Z" 2022-09-03T06:15:59.061 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:15:59.062 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:15:59.063 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:15:59.064 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:15:59.064 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:15:59.064 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:15:59.066 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:15:59.066 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:15:59.066 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:15:59.067 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:15:59.068 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:15:59.069 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:15:59.069 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:15:59.070 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:15:59.071 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:15:59.124 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:15:59.125 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:15:59.168 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:59.163+0000 7fcb840c4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:15:59.169 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:15:59.164+0000 7fcb840c4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:15:59.180 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:15:59.181 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:15:59.181 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:15:59.181 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:15:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:15:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:15:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:15:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:15:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:15:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:15:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:15:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:15:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:15:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:15:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:15:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:15:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:15:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:15:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:15:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:15:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:15:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:15:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:15:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:15:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:15:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.894939_228.1", 2022-09-03T06:15:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:15:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:15:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:15:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.894939Z", 2022-09-03T06:15:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:15:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:15:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:15:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:15:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:15:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:15:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:15:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:15:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:15:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:15:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:15:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:15:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:15:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:15:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:15:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:15:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:15:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:15:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:15:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:15:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:15:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:15:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:15:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:15:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:15:59.209 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:15:59.209 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:15:59.209 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:15:59.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:15:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:15:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:15:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:15:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:15:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:15:59Z" 2022-09-03T06:15:59.215 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:15:59.217 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.894939_228.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.894939Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:15:59Z" } 2022-09-03T06:15:59.218 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.894939_228.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.894939Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:15:59.219 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:15:59.219 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.894939_228.1', 2: '', 3: ''} 2022-09-03T06:15:59.220 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.894939_228.1 2022-09-03T06:16:04.224 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:04.231 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:04.231 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:04.282 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:04.283 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:04.323 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:04.320+0000 7fb5cf065600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:04.324 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:04.321+0000 7fb5cf065600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:04.340 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:16:04.340 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:16:04.340 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:16:04.341 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:16:04.341 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:16:04.341 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:04.341 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:04.342 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:04.343 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:16:04.343 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:04.343 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:16:04.343 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:04.344 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:04.344 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:04.344 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:04.344 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:16:04.345 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:04.345 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:04.345 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:04.345 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:04.346 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:04.346 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:04.346 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:16:04.346 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:04.347 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:04.347 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185758.915547_281.1", 2022-09-03T06:16:04.347 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:04.347 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:04.348 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:04.348 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.915547Z", 2022-09-03T06:16:04.348 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:04.348 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:04.349 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:04.349 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:04.349 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:16:04.350 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:04.350 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:04.350 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:04.350 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:04.351 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:04.351 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:04.351 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:04.351 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:04.352 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:04.352 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:04.353 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:04.353 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:16:04.353 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:04.354 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:04.354 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:04.354 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:04.354 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:04.355 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:04.355 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:04.355 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:04.356 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:04.356 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:04.356 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:16:04.356 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:04.357 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:16:04.357 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:16:04.357 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:16:04.358 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:04.358 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:16:04Z" 2022-09-03T06:16:04.358 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:16:04.358 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185758.915547_281.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:58.915547Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:04Z" } 2022-09-03T06:16:04.359 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185758.915547_281.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:58.915547Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:04.360 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:04.361 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:04.361 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:16:04.362 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:16:04.362 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:04.363 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:04.364 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:04.424 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:04.424 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:04.480 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:16:04.478+0000 7fef18556600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:04.482 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:16:04.481+0000 7fef18556600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:04.504 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:16:04.505 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:16:04.505 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:16:04.506 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:16:04.507 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:16:04.507 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:04.508 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:04.508 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:04.508 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:16:04.508 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:04.509 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:16:04.509 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:04.509 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:04.510 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:04.510 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:04.510 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:16:04.510 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:04.511 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:04.511 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:04.511 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:04.512 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:04.512 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:04.512 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:16:04.512 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:04.513 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:04.513 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185756.146357_242.1", 2022-09-03T06:16:04.513 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:04.513 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:04.514 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:04.514 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.146357Z", 2022-09-03T06:16:04.514 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:04.514 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:04.515 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:04.515 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:04.515 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:16:04.515 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:04.516 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:04.516 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:16:04.516 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:04.516 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:04.517 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:04.517 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:16:04.517 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:04.517 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:04.518 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:04.518 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:04.518 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:16:04.519 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:04.519 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:04.519 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:16:04.519 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:04.520 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:04.520 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:04.520 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:16:04.520 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:04.520 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:04.521 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:04.521 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:16:04.522 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:04.522 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:16:04.522 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:16:04.523 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:16:04.523 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:04.523 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:16:04Z" 2022-09-03T06:16:04.523 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:16:04.525 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.146357_242.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.146357Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:04Z" } 2022-09-03T06:16:04.526 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.146357_242.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.146357Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:04.527 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:04.528 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.146357_242.1', 2: '', 3: ''} 2022-09-03T06:16:04.528 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.146357_242.1 2022-09-03T06:16:09.530 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:09.531 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:09.532 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:09.604 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:09.605 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:09.640 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:16:09.638+0000 7fe9fb8fd600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:09.645 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:16:09.639+0000 7fe9fb8fd600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:09.656 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:16:09.656 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:16:09.657 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:16:09.657 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:16:09.658 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:16:09.658 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:09.660 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:09.660 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:09.661 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:16:09.661 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:09.662 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:16:09.662 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:09.663 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:09.663 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:09.664 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:09.664 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:16:09.665 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:09.666 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:09.666 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:09.671 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:09.671 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:09.672 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:09.672 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:16:09.672 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:09.672 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:09.673 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185756.146357_242.1", 2022-09-03T06:16:09.673 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:09.673 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:09.673 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:09.674 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.146357Z", 2022-09-03T06:16:09.674 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:09.674 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:09.674 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:09.675 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:09.675 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:16:09.675 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:09.675 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:09.682 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:16:09.682 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:09.682 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:09.683 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:09.683 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:16:09.683 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:09.683 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:09.684 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:09.684 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:09.684 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:16:09.684 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:09.685 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:09.685 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:16:09.685 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:09.685 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:09.688 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:09.688 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:16:09.689 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:09.690 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:09.691 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:09.693 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:16:09.693 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:09.694 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:16:09.694 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:16:09.695 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:16:09.695 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:09.696 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:16:09Z" 2022-09-03T06:16:09.696 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:16:09.698 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.146357_242.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.146357Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:09Z" } 2022-09-03T06:16:09.699 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.146357_242.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.146357Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:09.701 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:09.702 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.146357_242.1', 2: '', 3: ''} 2022-09-03T06:16:09.703 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.146357_242.1 2022-09-03T06:16:14.706 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:14.707 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:14.708 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:14.769 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:14.769 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:14.801 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:16:14.799+0000 7f72bc4b0600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:14.802 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:16:14.801+0000 7f72bc4b0600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:14.836 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:16:14.836 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:16:14.837 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:16:14.837 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:16:14.838 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:16:14.839 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:14.840 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:14.841 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:14.841 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:16:14.842 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:14.842 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:16:14.843 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:14.843 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:14.844 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:14.844 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:14.845 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:16:14.845 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:14.846 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:14.846 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:14.847 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:14.847 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:14.848 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:14.849 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:16:14.850 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:14.851 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:14.851 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185756.146357_242.1", 2022-09-03T06:16:14.851 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:14.852 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:14.852 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:14.852 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.146357Z", 2022-09-03T06:16:14.852 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:14.853 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:14.853 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:14.853 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:14.853 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:16:14.854 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:14.854 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:14.855 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:16:14.855 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:14.855 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:14.856 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:14.857 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:16:14.857 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:14.857 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:14.857 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:14.858 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:14.858 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:16:14.858 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:14.859 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:14.859 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:16:14.859 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:14.860 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:14.860 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:14.860 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:16:14.861 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:14.861 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:14.861 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:14.861 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:16:14.862 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:14.862 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:16:14.862 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:16:14.863 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:16:14.863 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:14.863 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:16:14Z" 2022-09-03T06:16:14.863 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:16:14.865 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.146357_242.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.146357Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:14Z" } 2022-09-03T06:16:14.866 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.146357_242.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.146357Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:14.866 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:14.867 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.146357_242.1', 2: '', 3: ''} 2022-09-03T06:16:14.868 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.146357_242.1 2022-09-03T06:16:19.870 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:19.872 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:19.875 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:19.937 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:19.938 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:19.988 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:16:19.985+0000 7f6f3e533600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:19.989 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:16:19.986+0000 7f6f3e533600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:20.005 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:16:20.006 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:16:20.006 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:16:20.006 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:16:20.006 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:16:20.008 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:20.008 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:20.008 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:20.008 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:16:20.009 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:20.009 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:16:20.009 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:20.009 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:20.010 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:20.010 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:20.010 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:16:20.010 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:20.011 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:20.011 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:20.012 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:20.012 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:20.012 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:20.013 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:16:20.013 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:20.013 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:20.014 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662185758.915547_281.1", 2022-09-03T06:16:20.014 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:20.014 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:20.014 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:20.015 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.915547Z", 2022-09-03T06:16:20.015 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:20.015 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:20.015 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:20.016 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:20.016 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:16:20.016 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:20.017 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:20.017 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:16:20.017 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:20.018 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:20.018 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:20.019 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:16:20.019 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:20.019 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:20.019 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:20.020 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:16:20.020 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:16:20.020 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:16:20.020 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:16:20.021 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:16:20.021 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:16:20.021 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:16:20.021 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:16:20.021 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:16:20.022 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:16:20.022 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:20.022 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:16:20.023 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:16:20.024 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:20.024 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:16:20.024 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:16:20.024 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:16:20.025 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:16:20.025 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:16:19Z" 2022-09-03T06:16:20.026 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:16:20.027 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185758.915547_281.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:58.915547Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:19Z" } 2022-09-03T06:16:20.028 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185758.915547_281.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:58.915547Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:20.029 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:20.030 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:20.031 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:16:20.031 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:16:20.032 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:20.032 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:20.033 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:20.086 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:20.087 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:20.139 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:20.134+0000 7fb887f3a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:20.139 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:20.135+0000 7fb887f3a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:20.150 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:16:20.151 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:16:20.151 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:16:20.152 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:16:20.152 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:16:20.154 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:20.154 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:20.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:20.154 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:16:20.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:20.155 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:16:20.155 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:20.156 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:20.156 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:20.157 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:20.157 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:16:20.158 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:20.158 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:20.159 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:20.159 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:20.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:20.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:20.160 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:16:20.160 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:20.160 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:20.161 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.778620_221.1", 2022-09-03T06:16:20.161 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:20.161 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:20.161 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:20.162 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.778620Z", 2022-09-03T06:16:20.162 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:20.163 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:20.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:20.164 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:20.165 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:16:20.165 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:20.166 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:20.167 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:20.167 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:20.168 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:20.168 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:20.169 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:20.169 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:20.170 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:20.170 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:20.171 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:20.171 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:16:20.177 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:20.178 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:20.178 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:20.178 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:20.178 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:20.179 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:20.179 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:20.179 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:20.179 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:20.180 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:20.180 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:16:20.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:20.181 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:16:20.181 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:16:20.182 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:16:20.182 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:20.182 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:16:20Z" 2022-09-03T06:16:20.183 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:16:20.184 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.778620_221.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.778620Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:20Z" } 2022-09-03T06:16:20.185 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.778620_221.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.778620Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:20.186 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:20.187 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.778620_221.1', 2: '', 3: ''} 2022-09-03T06:16:20.192 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.778620_221.1 2022-09-03T06:16:25.193 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:25.194 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:25.195 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:25.254 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:25.254 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:25.306 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:25.304+0000 7f38e2686600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:25.307 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:25.305+0000 7f38e2686600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:25.360 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:16:25.361 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:16:25.362 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:16:25.362 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:16:25.363 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:16:25.364 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:25.364 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:25.365 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:25.365 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:16:25.366 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:25.366 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:16:25.367 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:25.367 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:25.367 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:25.368 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:25.368 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:16:25.369 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:25.369 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:25.370 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:25.370 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:25.371 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:25.371 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:25.372 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:16:25.372 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:25.373 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:25.373 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.001910_232.1", 2022-09-03T06:16:25.374 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:25.374 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:25.374 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:25.375 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.001910Z", 2022-09-03T06:16:25.375 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:25.376 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:25.376 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:25.376 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:25.377 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:16:25.377 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:25.377 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:25.378 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:25.378 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:25.379 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:25.379 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:25.380 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:25.380 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:25.381 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:25.381 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:25.384 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:25.385 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:16:25.385 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:25.386 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:25.386 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:25.387 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:25.387 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:25.388 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:25.388 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:25.389 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:25.389 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:25.390 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:25.390 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:16:25.391 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:25.391 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:16:25.392 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:16:25.392 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:16:25.393 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:25.393 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:16:25Z" 2022-09-03T06:16:25.394 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:16:25.395 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.001910_232.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.001910Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:25Z" } 2022-09-03T06:16:25.396 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.001910_232.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.001910Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:25.397 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:25.398 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.001910_232.1', 2: '', 3: ''} 2022-09-03T06:16:25.399 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.001910_232.1 2022-09-03T06:16:30.400 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:30.401 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:30.401 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:30.454 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:30.454 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:30.494 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:30.490+0000 7f96b4883600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:30.494 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:30.491+0000 7f96b4883600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:30.504 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:16:30.504 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:16:30.504 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:16:30.505 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:16:30.505 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:16:30.505 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:30.505 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:30.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:30.506 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:16:30.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:30.506 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:16:30.507 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:30.507 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:30.507 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:30.508 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:30.508 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:16:30.509 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:30.509 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:30.510 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:30.510 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:30.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:30.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:30.512 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:16:30.512 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:30.514 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:30.514 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.850209_226.1", 2022-09-03T06:16:30.515 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:30.515 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:30.516 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:30.516 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209Z", 2022-09-03T06:16:30.517 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:30.517 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:30.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:30.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:30.518 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:16:30.519 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:30.519 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:30.520 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:30.520 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:30.521 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:30.521 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:30.521 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:30.522 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:30.522 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:30.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:30.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:30.526 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:16:30.526 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:30.527 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:30.527 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:30.527 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:30.527 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:30.528 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:30.528 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:30.528 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:30.528 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:30.529 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:30.529 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:16:30.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:30.530 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:16:30.530 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:16:30.530 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:16:30.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:30.531 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:16:30Z" 2022-09-03T06:16:30.531 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:16:30.532 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.850209_226.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.850209Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:30Z" } 2022-09-03T06:16:30.534 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.850209_226.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.850209Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:30.534 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:30.535 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.850209_226.1', 2: '', 3: ''} 2022-09-03T06:16:30.535 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.850209_226.1 2022-09-03T06:16:35.537 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:35.538 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:35.538 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:35.592 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:35.593 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:35.644 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:35.640+0000 7f47cc812600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:35.645 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:35.641+0000 7f47cc812600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:35.656 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:16:35.657 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:16:35.657 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:16:35.658 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:16:35.658 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:16:35.659 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:35.659 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:35.660 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:35.660 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:16:35.661 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:35.661 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:16:35.661 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:35.662 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:35.662 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:35.663 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:35.663 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:16:35.664 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:35.664 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:35.665 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:35.665 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:35.666 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:35.666 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:35.667 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:16:35.667 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:35.668 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:35.668 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.063873_235.1", 2022-09-03T06:16:35.669 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:35.669 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:35.669 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:35.670 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.063873Z", 2022-09-03T06:16:35.670 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:35.671 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:35.672 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:35.672 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:35.672 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:16:35.673 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:35.673 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:35.674 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:35.675 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:35.675 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:35.676 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:35.676 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:35.677 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:35.677 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:35.677 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:35.678 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:35.678 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:16:35.678 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:35.679 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:35.679 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:35.679 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:35.679 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:35.680 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:35.680 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:35.680 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:35.681 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:35.681 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:35.681 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:16:35.682 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:35.682 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:16:35.683 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:16:35.683 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:16:35.683 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:35.684 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:16:35Z" 2022-09-03T06:16:35.684 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:16:35.685 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.063873_235.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.063873Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:35Z" } 2022-09-03T06:16:35.686 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.063873_235.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.063873Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:35.687 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:35.688 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.063873_235.1', 2: '', 3: ''} 2022-09-03T06:16:35.689 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.063873_235.1 2022-09-03T06:16:40.691 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:40.692 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:40.692 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:40.745 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:40.746 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:40.786 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:40.784+0000 7fac8d847600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:40.789 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:40.785+0000 7fac8d847600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:40.798 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:16:40.799 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:16:40.799 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:16:40.800 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:16:40.800 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:16:40.800 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:40.801 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:40.801 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:40.801 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:16:40.802 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:40.802 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:16:40.803 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:40.803 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:40.803 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:40.805 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:40.805 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:16:40.806 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:40.806 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:40.807 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:40.807 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:40.808 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:40.808 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:40.808 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:16:40.809 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:40.809 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:40.810 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.842565_224.1", 2022-09-03T06:16:40.810 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:40.811 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:40.811 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:40.812 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.842565Z", 2022-09-03T06:16:40.813 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:40.813 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:40.813 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:40.814 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:40.814 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:16:40.815 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:40.815 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:40.815 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:40.815 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:40.816 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:40.816 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:40.817 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:40.817 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:40.817 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:40.817 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:40.818 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:40.818 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:16:40.818 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:40.819 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:40.819 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:40.821 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:40.821 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:40.822 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:40.822 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:40.823 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:40.823 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:40.824 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:40.824 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:16:40.825 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:40.825 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:16:40.826 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:16:40.826 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:16:40.827 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:40.827 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:16:40Z" 2022-09-03T06:16:40.828 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:16:40.831 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.842565_224.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.842565Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:40Z" } 2022-09-03T06:16:40.832 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.842565_224.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.842565Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:40.832 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:40.833 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.842565_224.1', 2: '', 3: ''} 2022-09-03T06:16:40.833 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.842565_224.1 2022-09-03T06:16:45.842 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:45.843 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:45.844 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:45.899 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:45.900 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:45.929 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:45.927+0000 7f5b227ed600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:45.931 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:45.928+0000 7f5b227ed600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:45.942 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:16:45.942 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:16:45.943 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:16:45.943 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:16:45.944 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:16:45.945 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:45.945 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:45.946 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:45.946 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:16:45.946 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:45.947 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:16:45.947 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:45.948 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:45.948 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:45.949 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:45.949 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:16:45.950 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:45.950 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:45.951 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:45.951 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:45.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:45.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:45.953 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:16:45.953 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:45.953 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:45.954 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.848053_225.1", 2022-09-03T06:16:45.954 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:45.955 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:45.955 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:45.956 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.848053Z", 2022-09-03T06:16:45.956 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:45.957 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:45.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:45.958 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:45.958 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:16:45.958 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:45.959 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:45.959 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:45.959 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:45.959 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:45.960 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:45.960 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:45.960 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:45.960 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:45.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:45.961 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:45.961 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:16:45.961 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:45.962 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:45.962 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:45.962 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:45.962 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:45.963 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:45.963 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:45.963 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:45.963 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:45.964 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:45.964 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:16:45.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:45.965 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:16:45.966 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:16:45.966 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:16:45.966 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:45.967 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:16:45Z" 2022-09-03T06:16:45.967 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:16:45.968 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.848053_225.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.848053Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:45Z" } 2022-09-03T06:16:45.969 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.848053_225.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.848053Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:45.970 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:45.970 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.848053_225.1', 2: '', 3: ''} 2022-09-03T06:16:45.971 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.848053_225.1 2022-09-03T06:16:50.972 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:50.974 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:50.975 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:51.027 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:51.028 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:51.062 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:51.055+0000 7f07cc715600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:51.062 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:51.056+0000 7f07cc715600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:51.070 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:16:51.070 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:16:51.071 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:16:51.071 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:16:51.072 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:16:51.072 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:51.073 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:51.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:51.074 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:16:51.074 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:51.075 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:16:51.075 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:51.076 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:51.076 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:51.076 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:51.077 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:16:51.078 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:51.078 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:51.079 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:51.079 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:51.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:51.080 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:51.081 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:16:51.081 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:51.081 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:51.081 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.894939_228.1", 2022-09-03T06:16:51.082 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:51.082 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:51.082 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:51.082 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.894939Z", 2022-09-03T06:16:51.083 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:51.083 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:51.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:51.083 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:51.084 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:16:51.084 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:51.084 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:51.084 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:51.085 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:51.085 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:51.085 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:51.085 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:51.086 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:51.086 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:51.086 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:51.087 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:51.087 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:16:51.087 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:51.087 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:51.088 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:51.088 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:51.088 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:51.088 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:51.089 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:51.089 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:51.089 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:51.089 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:51.090 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:16:51.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:51.090 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:16:51.090 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:16:51.091 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:16:51.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:51.091 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:16:51Z" 2022-09-03T06:16:51.091 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:16:51.093 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.894939_228.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.894939Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:51Z" } 2022-09-03T06:16:51.093 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.894939_228.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.894939Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:51.094 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:51.095 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.894939_228.1', 2: '', 3: ''} 2022-09-03T06:16:51.095 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.894939_228.1 2022-09-03T06:16:56.096 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:56.098 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:16:56.099 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:16:56.155 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:16:56.157 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:16:56.185 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:56.182+0000 7fc8e5f1c600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:56.186 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:16:56.183+0000 7fc8e5f1c600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:16:56.197 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:16:56.197 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:16:56.198 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:16:56.198 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:16:56.200 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:16:56.200 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:16:56.200 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:56.201 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:56.201 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:16:56.201 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:56.201 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:16:56.202 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:56.202 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:56.202 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:16:56.203 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:56.203 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:16:56.203 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:56.203 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:16:56.204 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:56.204 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:56.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:56.204 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:56.205 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:16:56.205 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:56.205 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:56.205 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.850209_226.1", 2022-09-03T06:16:56.207 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:56.207 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:56.208 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:56.208 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209Z", 2022-09-03T06:16:56.208 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:56.209 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:56.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:56.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:56.215 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:16:56.215 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:56.216 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:56.216 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:56.216 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:56.216 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:56.217 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:56.217 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:56.217 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:56.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:56.218 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:56.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:16:56.218 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:16:56.219 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:16:56.219 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:16:56.219 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:16:56.219 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:16:56.219 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:16:56.220 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:16:56.220 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:16:56.220 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:16:56.220 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:56.221 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:16:56.221 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:16:56.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:56.221 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:16:56.222 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:16:56.222 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:16:56.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:16:56.222 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:16:56Z" 2022-09-03T06:16:56.223 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:16:56.230 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.850209_226.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.850209Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:16:56Z" } 2022-09-03T06:16:56.231 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.850209_226.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.850209Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:16:56.232 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:16:56.232 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.850209_226.1', 2: '', 3: ''} 2022-09-03T06:16:56.233 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.850209_226.1 2022-09-03T06:17:01.235 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:01.236 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:01.237 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:01.290 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:01.291 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:01.319 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:01.318+0000 7fb851ef5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:01.323 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:01.319+0000 7fb851ef5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:01.332 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:01.332 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:01.333 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:01.333 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:01.334 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:01.334 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:01.334 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:01.334 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:01.335 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:01.335 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:01.336 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:01.336 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:01.337 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:01.337 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:01.338 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:01.338 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:01.338 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:01.339 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:01.339 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:01.340 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:01.341 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:01.341 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:01.341 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:01.341 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:01.342 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:01.342 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.779591_222.1", 2022-09-03T06:17:01.343 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:01.343 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:01.343 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:01.346 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.779591Z", 2022-09-03T06:17:01.346 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:01.347 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:01.350 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:01.350 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:01.351 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:01.351 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:01.351 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:01.352 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:01.352 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:01.353 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:01.353 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:01.353 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:01.353 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:01.354 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:01.354 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:01.354 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:01.354 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:01.355 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:01.355 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:01.355 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:01.355 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:01.356 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:01.356 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:01.356 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:01.357 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:01.357 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:01.358 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:01.358 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:01.360 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:01.360 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:01.360 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:01.360 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:01.361 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:01.361 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:01Z" 2022-09-03T06:17:01.362 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:01.363 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.779591_222.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.779591Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:01Z" } 2022-09-03T06:17:01.363 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.779591_222.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.779591Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:01.364 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:01.364 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.779591_222.1', 2: '', 3: ''} 2022-09-03T06:17:01.365 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.779591_222.1 2022-09-03T06:17:06.367 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:06.368 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:06.368 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:06.427 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:06.428 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:06.475 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:06.471+0000 7f9596608600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:06.475 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:06.472+0000 7f9596608600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:06.511 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:06.511 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:06.511 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:06.512 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:06.512 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:06.512 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:06.513 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:06.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:06.513 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:06.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:06.515 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:06.516 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:06.516 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:06.518 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:06.518 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:06.519 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:06.519 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:06.519 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:06.519 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:06.520 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:06.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:06.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:06.521 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:06.521 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:06.521 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:06.521 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.850209_226.1", 2022-09-03T06:17:06.522 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:06.523 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:06.523 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:06.523 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209Z", 2022-09-03T06:17:06.523 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:06.524 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:06.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:06.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:06.524 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:06.525 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:06.525 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:06.525 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:06.525 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:06.525 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:06.526 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:06.526 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:06.526 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:06.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:06.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:06.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:06.527 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:06.527 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:06.528 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:06.528 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:06.528 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:06.528 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:06.529 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:06.529 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:06.529 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:06.529 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:06.530 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:06.530 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:06.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:06.530 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:06.531 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:06.531 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:06.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:06.532 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:06Z" 2022-09-03T06:17:06.533 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:06.534 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.850209_226.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.850209Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:06Z" } 2022-09-03T06:17:06.535 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.850209_226.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.850209Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:06.535 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:06.536 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.850209_226.1', 2: '', 3: ''} 2022-09-03T06:17:06.536 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.850209_226.1 2022-09-03T06:17:11.537 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:11.539 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:11.540 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:11.591 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:11.592 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:11.622 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:11.618+0000 7f673f2a4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:11.623 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:11.619+0000 7f673f2a4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:11.632 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:11.633 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:11.633 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:11.633 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:11.634 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:11.634 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:11.635 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:11.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:11.636 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:11.636 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:11.637 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:11.638 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:11.638 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:11.639 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:11.639 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:11.639 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:11.640 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:11.640 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:11.641 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:11.641 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:11.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:11.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:11.642 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:11.643 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:11.643 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:11.643 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.001910_232.1", 2022-09-03T06:17:11.644 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:11.644 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:11.644 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:11.645 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.001910Z", 2022-09-03T06:17:11.645 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:11.645 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:11.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:11.646 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:11.647 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:11.647 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:11.647 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:11.648 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:11.648 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:11.648 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:11.649 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:11.649 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:11.650 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:11.650 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:11.651 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:11.651 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:11.651 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:11.652 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:11.652 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:11.652 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:11.652 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:11.653 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:11.653 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:11.653 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:11.653 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:11.654 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:11.654 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:11.654 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:11.654 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:11.655 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:11.655 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:11.655 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:11.655 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:11.656 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:11Z" 2022-09-03T06:17:11.656 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:11.665 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.001910_232.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.001910Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:11Z" } 2022-09-03T06:17:11.666 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.001910_232.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.001910Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:11.667 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:11.667 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.001910_232.1', 2: '', 3: ''} 2022-09-03T06:17:11.668 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.001910_232.1 2022-09-03T06:17:16.669 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:16.670 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:16.671 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:16.721 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:16.722 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:16.758 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:16.757+0000 7f7770b0a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:16.759 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:16.758+0000 7f7770b0a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:16.774 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:16.775 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:16.775 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:16.776 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:16.776 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:16.777 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:16.777 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:16.778 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:16.778 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:16.779 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:16.779 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:16.780 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:16.781 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:16.782 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:16.782 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:16.782 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:16.783 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:16.783 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:16.784 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:16.787 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:16.787 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:16.787 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:16.788 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:16.788 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:16.789 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:16.789 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.784101_223.1", 2022-09-03T06:17:16.789 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:16.789 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:16.790 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:16.790 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.784101Z", 2022-09-03T06:17:16.790 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:16.790 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:16.791 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:16.791 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:16.791 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:16.791 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:16.792 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:16.792 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:16.793 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:16.793 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:16.793 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:16.794 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:16.794 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:16.795 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:16.795 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:16.796 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:16.796 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:16.797 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:16.797 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:16.798 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:16.798 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:16.799 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:16.799 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:16.799 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:16.800 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:16.801 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:16.801 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:16.801 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:16.803 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:16.803 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:16.803 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:16.804 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:16.804 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:16.804 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:16Z" 2022-09-03T06:17:16.804 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:16.805 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.784101_223.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.784101Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:16Z" } 2022-09-03T06:17:16.806 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.784101_223.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.784101Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:16.806 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:16.807 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.784101_223.1', 2: '', 3: ''} 2022-09-03T06:17:16.807 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.784101_223.1 2022-09-03T06:17:21.808 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:21.810 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:21.811 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:21.863 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:21.864 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:21.902 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:21.897+0000 7f2f11266600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:21.903 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:21.898+0000 7f2f11266600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:21.911 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:21.912 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:21.912 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:21.913 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:21.913 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:21.913 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:21.914 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:21.914 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:21.915 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:21.915 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:21.916 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:21.916 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:21.917 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:21.920 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:21.921 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:21.921 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:21.922 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:21.922 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:21.923 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:21.923 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:21.924 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:21.924 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:21.925 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:21.926 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:21.926 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:21.927 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.779591_222.1", 2022-09-03T06:17:21.927 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:21.928 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:21.928 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:21.929 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.779591Z", 2022-09-03T06:17:21.930 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:21.930 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:21.931 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:21.931 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:21.932 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:21.932 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:21.933 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:21.933 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:21.934 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:21.935 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:21.935 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:21.936 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:21.936 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:21.937 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:21.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:21.939 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:21.939 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:21.940 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:21.940 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:21.941 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:21.941 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:21.942 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:21.942 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:21.943 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:21.943 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:21.944 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:21.944 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:21.945 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:21.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:21.946 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:21.946 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:21.947 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:21.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:21.948 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:21Z" 2022-09-03T06:17:21.948 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:21.951 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.779591_222.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.779591Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:21Z" } 2022-09-03T06:17:21.952 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.779591_222.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.779591Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:21.953 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:21.954 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.779591_222.1', 2: '', 3: ''} 2022-09-03T06:17:21.955 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.779591_222.1 2022-09-03T06:17:26.958 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:26.959 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:26.960 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:27.011 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:27.012 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:27.040 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:27.037+0000 7fa365c71600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:27.040 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:27.038+0000 7fa365c71600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:27.051 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:27.051 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:27.051 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:27.052 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:27.052 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:27.052 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:27.053 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:27.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:27.053 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:27.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:27.054 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:27.054 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:27.054 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:27.054 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:27.055 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:27.055 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:27.055 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:27.055 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:27.056 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:27.056 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:27.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:27.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:27.057 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:27.057 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:27.059 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:27.059 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.850209_226.1", 2022-09-03T06:17:27.059 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:27.060 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:27.060 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:27.061 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209Z", 2022-09-03T06:17:27.061 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:27.061 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:27.062 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:27.062 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:27.062 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:27.063 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:27.063 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:27.063 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:27.064 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:27.064 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:27.064 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:27.064 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:27.064 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:27.065 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:27.065 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:27.065 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:27.065 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:27.066 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:27.066 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:27.066 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:27.066 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:27.067 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:27.067 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:27.067 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:27.068 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:27.068 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:27.068 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:27.068 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:27.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:27.069 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:27.069 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:27.069 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:27.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:27.070 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:27Z" 2022-09-03T06:17:27.070 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:27.072 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.850209_226.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.850209Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:27Z" } 2022-09-03T06:17:27.073 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.850209_226.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.850209Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:27.073 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:27.074 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.850209_226.1', 2: '', 3: ''} 2022-09-03T06:17:27.074 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.850209_226.1 2022-09-03T06:17:32.078 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:32.080 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:32.083 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:32.141 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:32.142 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:32.180 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:32.178+0000 7f26a0b51600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:32.181 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:32.179+0000 7f26a0b51600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:32.193 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:32.194 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:32.195 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:32.195 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:32.196 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:32.196 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:32.197 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:32.197 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:32.197 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:32.197 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:32.198 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:32.199 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:32.200 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:32.200 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:32.201 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:32.201 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:32.202 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:32.202 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:32.203 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:32.203 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:32.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:32.205 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:32.205 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:32.206 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:32.207 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:32.207 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.850209_226.1", 2022-09-03T06:17:32.208 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:32.208 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:32.210 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:32.210 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209Z", 2022-09-03T06:17:32.211 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:32.211 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:32.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:32.212 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:32.212 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:32.213 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:32.213 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:32.213 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:32.214 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:32.214 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:32.215 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:32.215 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:32.215 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:32.216 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:32.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:32.217 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:32.217 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:32.218 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:32.218 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:32.219 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:32.219 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:32.221 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:32.222 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:32.222 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:32.223 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:32.223 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:32.224 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:32.224 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:32.225 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:32.225 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:32.226 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:32.227 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:32.227 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:32.228 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:32Z" 2022-09-03T06:17:32.228 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:32.230 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.850209_226.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.850209Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:32Z" } 2022-09-03T06:17:32.231 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.850209_226.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.850209Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:32.232 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:32.232 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.850209_226.1', 2: '', 3: ''} 2022-09-03T06:17:32.233 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.850209_226.1 2022-09-03T06:17:37.234 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:37.235 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:37.235 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:37.287 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:37.287 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:37.322 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:37.320+0000 7f55cff28600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:37.323 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:37.321+0000 7f55cff28600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:37.337 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:37.338 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:37.338 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:37.338 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:37.339 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:37.339 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:37.339 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:37.339 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:37.340 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:37.340 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:37.340 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:37.340 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:37.341 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:37.341 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:37.342 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:37.342 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:37.342 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:37.342 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:37.343 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:37.343 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:37.343 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:37.343 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:37.344 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:37.344 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:37.344 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:37.345 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.842565_224.1", 2022-09-03T06:17:37.345 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:37.345 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:37.345 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:37.345 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.842565Z", 2022-09-03T06:17:37.346 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:37.346 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:37.346 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:37.346 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:37.347 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:37.347 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:37.347 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:37.347 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:37.348 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:37.348 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:37.348 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:37.349 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:37.349 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:37.349 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:37.349 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:37.350 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:37.350 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:37.350 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:37.350 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:37.351 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:37.351 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:37.351 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:37.351 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:37.351 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:37.352 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:37.352 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:37.352 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:37.352 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:37.353 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:37.353 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:37.353 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:37.353 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:37.354 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:37.354 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:37Z" 2022-09-03T06:17:37.355 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:37.356 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.842565_224.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.842565Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:37Z" } 2022-09-03T06:17:37.357 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.842565_224.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.842565Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:37.357 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:37.358 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.842565_224.1', 2: '', 3: ''} 2022-09-03T06:17:37.358 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.842565_224.1 2022-09-03T06:17:42.359 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:42.360 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:42.361 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:42.413 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:42.413 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:42.448 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:42.446+0000 7f42ad6a8600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:42.449 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:42.447+0000 7f42ad6a8600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:42.465 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:42.466 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:42.466 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:42.466 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:42.467 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:42.467 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:42.467 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:42.468 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:42.468 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:42.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:42.468 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:42.469 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:42.469 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:42.469 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:42.470 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:42.470 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:42.470 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:42.470 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:42.471 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:42.471 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:42.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:42.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:42.473 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:42.473 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:42.474 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:42.474 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.848053_225.1", 2022-09-03T06:17:42.475 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:42.475 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:42.476 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:42.476 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.848053Z", 2022-09-03T06:17:42.477 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:42.478 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:42.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:42.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:42.480 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:42.481 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:42.481 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:42.482 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:42.482 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:42.483 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:42.483 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:42.484 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:42.484 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:42.485 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:42.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:42.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:42.486 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:42.487 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:42.488 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:42.488 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:42.489 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:42.489 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:42.490 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:42.491 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:42.491 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:42.492 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:42.492 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:42.494 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:42.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:42.495 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:42.495 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:42.496 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:42.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:42.496 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:42Z" 2022-09-03T06:17:42.497 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:42.497 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.848053_225.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.848053Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:42Z" } 2022-09-03T06:17:42.498 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.848053_225.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.848053Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:42.503 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:42.504 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.848053_225.1', 2: '', 3: ''} 2022-09-03T06:17:42.504 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.848053_225.1 2022-09-03T06:17:47.506 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:47.507 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:47.508 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:47.568 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:47.569 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:47.665 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:47.663+0000 7fa2bf1e4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:47.666 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:47.664+0000 7fa2bf1e4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:47.704 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:47.704 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:47.705 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:47.705 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:47.706 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:47.706 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:47.707 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:47.707 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:47.707 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:47.708 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:47.708 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:47.708 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:47.708 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:47.709 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:47.709 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:47.709 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:47.710 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:47.710 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:47.710 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:47.710 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:47.711 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:47.711 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:47.711 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:47.711 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:47.712 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:47.712 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.921086_230.1", 2022-09-03T06:17:47.712 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:47.712 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:47.713 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:47.713 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.921086Z", 2022-09-03T06:17:47.713 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:47.713 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:47.714 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:47.714 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:47.714 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:47.714 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:47.715 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:47.715 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:47.716 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:47.716 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:47.716 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:47.716 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:47.717 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:47.717 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:47.717 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:47.717 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:47.718 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:47.718 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:47.718 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:47.719 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:47.720 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:47.720 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:47.720 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:47.721 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:47.721 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:47.721 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:47.722 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:47.722 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:47.722 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:47.723 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:47.723 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:47.723 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:47.723 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:47.724 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:47Z" 2022-09-03T06:17:47.724 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:47.725 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.921086_230.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.921086Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:47Z" } 2022-09-03T06:17:47.727 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.921086_230.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.921086Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:47.728 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:47.729 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.921086_230.1', 2: '', 3: ''} 2022-09-03T06:17:47.730 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.921086_230.1 2022-09-03T06:17:52.731 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:52.732 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:52.733 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:52.790 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:52.791 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:52.820 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:52.816+0000 7f256f3e5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:52.821 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:52.817+0000 7f256f3e5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:52.832 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:52.833 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:52.834 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:52.834 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:52.835 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:52.835 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:52.836 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:52.836 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:52.837 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:52.837 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:52.839 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:52.839 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:52.840 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:52.841 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:52.842 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:52.843 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:52.844 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:52.844 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:52.844 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:52.845 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:52.847 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:52.847 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:52.848 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:52.848 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:52.854 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:52.855 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.098699_238.1", 2022-09-03T06:17:52.855 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:52.856 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:52.856 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:52.857 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.098699Z", 2022-09-03T06:17:52.857 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:52.858 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:52.858 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:52.859 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:52.860 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:52.861 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:52.864 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:52.865 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:52.865 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:52.866 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:52.867 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:52.867 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:52.868 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:52.868 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:52.869 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:52.869 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:52.870 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:52.870 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:52.871 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:52.871 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:52.872 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:52.872 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:52.873 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:52.873 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:52.874 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:52.874 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:52.875 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:52.875 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:52.876 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:52.876 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:52.876 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:52.877 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:52.877 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:52.878 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:52Z" 2022-09-03T06:17:52.878 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:52.881 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.098699_238.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.098699Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:52Z" } 2022-09-03T06:17:52.882 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.098699_238.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.098699Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:52.883 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:52.884 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.098699_238.1', 2: '', 3: ''} 2022-09-03T06:17:52.885 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.098699_238.1 2022-09-03T06:17:57.889 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:57.890 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:17:57.891 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:17:57.943 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:17:57.944 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:17:57.987 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:57.981+0000 7fa01c698600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:57.987 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:17:57.982+0000 7fa01c698600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:17:57.995 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:17:57.996 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:17:57.996 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:17:57.996 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:17:57.997 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:17:57.997 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:17:57.997 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:57.998 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:57.998 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:17:57.998 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:57.998 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:17:57.999 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:57.999 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:57.999 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:17:57.999 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:58.000 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:17:58.000 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:58.000 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:17:58.000 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:58.001 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:58.001 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:58.001 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:58.002 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:17:58.002 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:58.002 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:58.003 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.778620_221.1", 2022-09-03T06:17:58.003 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:58.004 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:58.004 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:58.005 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.778620Z", 2022-09-03T06:17:58.005 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:58.006 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:58.006 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:58.006 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:58.007 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:17:58.007 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:58.007 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:58.007 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:58.008 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:58.008 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:58.008 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:58.008 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:58.009 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:58.009 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:58.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:58.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:17:58.010 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:17:58.010 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:17:58.010 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:17:58.011 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:17:58.011 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:17:58.011 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:17:58.012 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:17:58.012 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:17:58.012 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:17:58.012 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:58.013 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:17:58.013 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:17:58.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:58.013 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:17:58.014 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:17:58.014 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:17:58.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:17:58.014 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:17:57Z" 2022-09-03T06:17:58.015 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:17:58.020 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.778620_221.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.778620Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:17:57Z" } 2022-09-03T06:17:58.021 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.778620_221.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.778620Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:17:58.021 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:17:58.022 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.778620_221.1', 2: '', 3: ''} 2022-09-03T06:17:58.022 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.778620_221.1 2022-09-03T06:18:03.026 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:03.028 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:03.029 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:03.081 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:03.081 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:03.109 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:03.107+0000 7fddc576c600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:03.111 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:03.108+0000 7fddc576c600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:03.121 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:03.124 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:03.125 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:03.125 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:03.126 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:03.126 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:03.127 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:03.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:03.128 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:03.128 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:03.128 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:03.129 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:03.129 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:03.130 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:03.130 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:03.131 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:03.131 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:03.132 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:03.132 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:03.133 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:03.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:03.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:03.135 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:03.135 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:03.136 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:03.136 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.779591_222.1", 2022-09-03T06:18:03.136 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:03.136 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:03.137 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:03.137 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.779591Z", 2022-09-03T06:18:03.137 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:03.138 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:03.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:03.138 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:03.138 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:03.139 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:03.139 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:03.139 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:03.140 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:03.140 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:03.140 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:03.141 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:03.141 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:03.141 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:03.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:03.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:03.142 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:03.142 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:03.142 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:03.143 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:03.143 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:03.143 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:03.143 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:03.144 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:03.144 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:03.144 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:03.144 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:03.145 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:03.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:03.146 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:03.146 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:03.146 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:03.147 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:03.147 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:03Z" 2022-09-03T06:18:03.147 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:03.148 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.779591_222.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.779591Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:03Z" } 2022-09-03T06:18:03.149 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.779591_222.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.779591Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:03.150 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:03.150 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.779591_222.1', 2: '', 3: ''} 2022-09-03T06:18:03.151 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.779591_222.1 2022-09-03T06:18:08.153 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:08.154 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:08.156 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:08.212 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:08.215 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:08.257 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:08.255+0000 7f21c3565600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:08.258 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:08.257+0000 7f21c3565600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:08.275 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:08.275 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:08.276 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:08.276 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:08.276 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:08.276 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:08.277 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:08.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:08.277 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:08.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:08.278 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:08.278 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:08.280 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:08.280 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:08.280 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:08.281 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:08.282 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:08.282 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:08.283 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:08.284 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:08.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:08.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:08.286 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:08.286 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:08.287 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:08.287 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.778620_221.1", 2022-09-03T06:18:08.288 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:08.288 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:08.289 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:08.289 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.778620Z", 2022-09-03T06:18:08.290 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:08.291 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:08.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:08.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:08.292 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:08.293 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:08.293 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:08.294 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:08.294 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:08.295 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:08.295 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:08.296 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:08.296 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:08.297 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:08.297 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:08.298 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:08.298 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:08.299 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:08.299 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:08.300 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:08.300 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:08.301 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:08.301 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:08.302 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:08.302 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:08.303 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:08.303 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:08.304 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:08.304 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:08.304 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:08.305 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:08.305 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:08.306 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:08.307 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:08Z" 2022-09-03T06:18:08.307 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:08.309 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.778620_221.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.778620Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:08Z" } 2022-09-03T06:18:08.311 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.778620_221.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.778620Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:08.312 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:08.313 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.778620_221.1', 2: '', 3: ''} 2022-09-03T06:18:08.314 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.778620_221.1 2022-09-03T06:18:13.316 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:13.317 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:13.317 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:13.382 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:13.382 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:13.412 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:13.410+0000 7f15760f6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:13.413 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:13.411+0000 7f15760f6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:13.425 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:13.425 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:13.426 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:13.426 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:13.426 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:13.427 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:13.427 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:13.427 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:13.427 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:13.428 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:13.428 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:13.428 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:13.428 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:13.428 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:13.429 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:13.429 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:13.429 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:13.430 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:13.430 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:13.430 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:13.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:13.431 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:13.431 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:13.431 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:13.431 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:13.432 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.064807_236.1", 2022-09-03T06:18:13.433 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:13.433 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:13.434 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:13.435 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.064807Z", 2022-09-03T06:18:13.435 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:13.436 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:13.436 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:13.437 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:13.437 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:13.438 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:13.438 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:13.439 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:13.439 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:13.440 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:13.440 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:13.441 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:13.441 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:13.442 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:13.442 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:13.443 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:13.443 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:13.444 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:13.444 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:13.446 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:13.446 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:13.447 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:13.447 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:13.448 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:13.448 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:13.449 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:13.449 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:13.450 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:13.450 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:13.451 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:13.451 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:13.452 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:13.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:13.453 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:13Z" 2022-09-03T06:18:13.453 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:13.455 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.064807_236.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.064807Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:13Z" } 2022-09-03T06:18:13.456 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.064807_236.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.064807Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:13.457 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:13.458 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.064807_236.1', 2: '', 3: ''} 2022-09-03T06:18:13.460 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.064807_236.1 2022-09-03T06:18:18.463 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:18.465 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:18.466 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:18.518 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:18.519 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:18.548 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:18.546+0000 7f105f6c0600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:18.548 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:18.547+0000 7f105f6c0600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:18.560 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:18.560 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:18.561 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:18.561 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:18.561 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:18.561 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:18.562 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:18.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:18.563 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:18.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:18.564 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:18.564 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:18.564 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:18.564 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:18.565 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:18.565 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:18.565 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:18.565 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:18.566 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:18.566 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:18.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:18.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:18.567 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:18.567 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:18.568 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:18.568 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.098699_238.1", 2022-09-03T06:18:18.568 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:18.568 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:18.569 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:18.569 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.098699Z", 2022-09-03T06:18:18.569 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:18.569 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:18.570 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:18.570 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:18.570 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:18.570 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:18.571 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:18.571 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:18.571 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:18.571 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:18.572 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:18.572 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:18.572 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:18.572 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:18.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:18.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:18.574 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:18.574 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:18.574 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:18.574 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:18.575 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:18.575 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:18.575 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:18.575 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:18.576 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:18.576 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:18.576 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:18.576 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:18.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:18.578 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:18.578 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:18.578 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:18.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:18.579 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:18Z" 2022-09-03T06:18:18.579 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:18.580 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.098699_238.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.098699Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:18Z" } 2022-09-03T06:18:18.581 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.098699_238.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.098699Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:18.581 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:18.582 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.098699_238.1', 2: '', 3: ''} 2022-09-03T06:18:18.582 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.098699_238.1 2022-09-03T06:18:23.584 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:23.585 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:23.585 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:23.638 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:23.639 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:23.676 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:23.674+0000 7fc7d1b97600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:23.677 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:23.675+0000 7fc7d1b97600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:23.688 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:23.689 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:23.689 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:23.689 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:23.689 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:23.690 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:23.690 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:23.690 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:23.691 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:23.692 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:23.692 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:23.693 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:23.694 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:23.694 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:23.695 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:23.695 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:23.696 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:23.697 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:23.697 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:23.698 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:23.698 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:23.699 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:23.700 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:23.701 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:23.701 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:23.702 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.104362_239.1", 2022-09-03T06:18:23.702 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:23.703 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:23.704 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:23.704 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.104362Z", 2022-09-03T06:18:23.704 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:23.705 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:23.705 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:23.706 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:23.706 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:23.707 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:23.708 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:23.708 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:23.708 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:23.709 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:23.710 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:23.710 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:23.710 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:23.711 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:23.711 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:23.712 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:23.712 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:23.713 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:23.713 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:23.714 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:23.714 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:23.715 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:23.715 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:23.716 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:23.716 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:23.717 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:23.717 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:23.718 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:23.719 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:23.719 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:23.720 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:23.720 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:23.721 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:23.721 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:23Z" 2022-09-03T06:18:23.722 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:23.724 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.104362_239.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.104362Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:23Z" } 2022-09-03T06:18:23.725 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.104362_239.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.104362Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:23.726 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:23.727 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.104362_239.1', 2: '', 3: ''} 2022-09-03T06:18:23.728 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.104362_239.1 2022-09-03T06:18:28.729 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:28.730 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:28.731 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:28.782 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:28.783 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:28.812 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:28.809+0000 7f2e46f04600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:28.813 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:28.810+0000 7f2e46f04600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:28.825 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:28.826 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:28.826 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:28.826 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:28.827 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:28.827 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:28.827 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:28.827 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:28.828 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:28.829 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:28.829 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:28.830 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:28.830 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:28.831 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:28.831 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:28.832 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:28.832 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:28.833 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:28.833 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:28.833 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:28.834 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:28.834 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:28.835 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:28.835 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:28.836 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:28.837 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.784101_223.1", 2022-09-03T06:18:28.837 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:28.838 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:28.839 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:28.839 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.784101Z", 2022-09-03T06:18:28.840 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:28.840 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:28.841 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:28.841 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:28.841 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:28.842 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:28.842 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:28.842 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:28.843 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:28.843 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:28.843 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:28.843 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:28.844 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:28.844 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:28.844 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:28.844 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:28.845 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:28.845 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:28.846 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:28.846 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:28.847 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:28.847 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:28.848 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:28.848 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:28.848 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:28.858 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:28.858 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:28.858 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:28.859 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:28.859 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:28.859 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:28.860 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:28.860 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:28.860 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:28Z" 2022-09-03T06:18:28.860 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:28.861 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.784101_223.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.784101Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:28Z" } 2022-09-03T06:18:28.861 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.784101_223.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.784101Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:28.862 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:28.863 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.784101_223.1', 2: '', 3: ''} 2022-09-03T06:18:28.863 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.784101_223.1 2022-09-03T06:18:33.865 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:33.867 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:33.868 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:33.922 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:33.922 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:33.965 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:33.963+0000 7f9c07ff1600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:33.965 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:33.964+0000 7f9c07ff1600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:33.977 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:33.978 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:33.978 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:33.978 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:33.978 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:33.979 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:33.979 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:33.979 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:33.979 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:33.980 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:33.980 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:33.980 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:33.980 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:33.981 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:33.981 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:33.981 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:33.981 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:33.982 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:33.982 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:33.982 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:33.982 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:33.983 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:33.983 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:33.983 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:33.984 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:33.984 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.921086_230.1", 2022-09-03T06:18:33.984 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:33.985 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:33.985 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:33.985 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.921086Z", 2022-09-03T06:18:33.986 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:33.986 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:33.986 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:33.987 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:33.987 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:33.987 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:33.988 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:33.988 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:33.988 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:33.988 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:33.990 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:33.990 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:33.990 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:33.990 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:33.991 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:33.991 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:33.991 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:33.991 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:33.992 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:33.992 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:33.992 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:33.992 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:33.992 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:33.993 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:33.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:33.993 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:33.993 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:33.994 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:33.994 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:33.995 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:33Z" 2022-09-03T06:18:33.995 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:33.998 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.921086_230.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.921086Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:33Z" } 2022-09-03T06:18:33.998 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.921086_230.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.921086Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:33.999 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:33.999 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.921086_230.1', 2: '', 3: ''} 2022-09-03T06:18:34.000 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.921086_230.1 2022-09-03T06:18:39.001 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:39.003 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:39.006 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:39.062 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:39.062 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:39.093 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:39.092+0000 7f1d7034f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:39.094 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:39.093+0000 7f1d7034f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:39.107 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:39.108 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:39.108 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:39.108 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:39.109 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:39.109 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:39.110 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:39.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:39.111 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:39.112 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:39.112 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:39.113 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:39.113 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:39.114 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:39.114 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:39.115 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:39.115 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:39.116 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:39.116 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:39.117 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:39.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:39.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:39.118 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:39.119 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:39.120 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:39.120 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.921086_230.1", 2022-09-03T06:18:39.121 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:39.121 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:39.122 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:39.122 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.921086Z", 2022-09-03T06:18:39.123 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:39.123 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:39.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:39.124 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:39.125 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:39.125 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:39.126 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:39.126 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:39.127 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:39.127 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:39.128 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:39.128 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:39.129 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:39.129 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:39.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:39.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:39.130 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:39.131 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:39.131 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:39.131 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:39.131 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:39.132 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:39.132 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:39.132 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:39.133 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:39.133 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:39.133 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:39.134 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:39.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:39.134 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:39.135 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:39.135 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:39.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:39.135 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:39Z" 2022-09-03T06:18:39.136 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:39.137 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.921086_230.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.921086Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:39Z" } 2022-09-03T06:18:39.138 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.921086_230.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.921086Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:39.138 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:39.139 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.921086_230.1', 2: '', 3: ''} 2022-09-03T06:18:39.139 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.921086_230.1 2022-09-03T06:18:44.140 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:44.142 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:44.144 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:44.204 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:44.205 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:44.236 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:44.233+0000 7f553b1e2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:44.237 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:44.234+0000 7f553b1e2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:44.248 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:44.248 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:44.248 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:44.249 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:44.249 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:44.250 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:44.250 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:44.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:44.251 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:44.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:44.252 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:44.252 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:44.253 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:44.253 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:44.253 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:44.253 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:44.254 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:44.254 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:44.254 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:44.254 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:44.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:44.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:44.255 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:44.255 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:44.256 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:44.256 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.390795_251.1", 2022-09-03T06:18:44.257 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:44.257 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:44.258 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:44.258 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.390795Z", 2022-09-03T06:18:44.259 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:44.259 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:44.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:44.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:44.261 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:44.261 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:44.262 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:44.263 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:44.263 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:44.264 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:44.264 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:44.264 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:44.265 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:44.265 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:44.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:44.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:44.266 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:44.266 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:44.266 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:44.267 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:44.267 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:44.267 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:44.268 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:44.268 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:44.268 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:44.268 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:44.269 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:44.269 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:44.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:44.270 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:44.270 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:44.270 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:44.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:44.271 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:44Z" 2022-09-03T06:18:44.271 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:44.272 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.390795_251.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.390795Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:44Z" } 2022-09-03T06:18:44.274 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.390795_251.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.390795Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:44.274 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:44.275 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.390795_251.1', 2: '', 3: ''} 2022-09-03T06:18:44.276 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.390795_251.1 2022-09-03T06:18:49.277 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:49.278 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:49.279 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:49.337 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:49.337 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:49.393 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:49.391+0000 7fcad3158600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:49.397 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:49.395+0000 7fcad3158600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:49.463 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:49.463 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:49.464 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:49.464 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:49.464 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:49.464 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:49.465 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:49.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:49.465 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:49.465 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:49.466 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:49.466 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:49.466 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:49.467 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:49.467 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:49.467 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:49.468 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:49.468 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:49.468 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:49.469 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:49.469 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:49.469 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:49.469 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:49.470 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:49.470 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:49.470 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.850209_226.1", 2022-09-03T06:18:49.470 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:49.471 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:49.471 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:49.471 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209Z", 2022-09-03T06:18:49.472 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:49.472 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:49.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:49.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:49.473 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:49.473 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:49.474 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:49.474 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:49.474 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:49.475 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:49.475 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:49.475 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:49.476 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:49.476 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:49.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:49.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:49.477 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:49.477 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:49.477 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:49.478 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:49.478 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:49.478 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:49.479 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:49.479 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:49.479 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:49.480 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:49.480 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:49.480 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:49.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:49.481 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:49.481 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:49.482 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:49.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:49.482 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:49Z" 2022-09-03T06:18:49.483 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:49.485 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.850209_226.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.850209Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:49Z" } 2022-09-03T06:18:49.485 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.850209_226.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.850209Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:49.487 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:49.487 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.850209_226.1', 2: '', 3: ''} 2022-09-03T06:18:49.488 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.850209_226.1 2022-09-03T06:18:54.489 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:54.490 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:54.491 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:54.543 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:54.544 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:54.581 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:54.579+0000 7fbecd9e5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:54.582 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:54.581+0000 7fbecd9e5600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:54.598 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:54.599 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:54.599 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:54.599 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:54.600 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:54.600 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:54.600 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:54.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:54.601 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:54.601 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:54.601 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:54.601 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:54.602 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:54.602 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:54.602 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:54.602 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:54.603 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:54.603 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:54.604 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:54.604 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:54.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:54.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:54.606 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:54.606 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:54.607 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:54.607 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.779591_222.1", 2022-09-03T06:18:54.607 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:54.608 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:54.609 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:54.610 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.779591Z", 2022-09-03T06:18:54.610 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:54.611 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:54.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:54.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:54.612 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:54.613 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:54.613 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:54.614 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:54.614 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:54.614 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:54.615 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:54.615 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:54.616 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:54.616 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:54.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:54.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:54.618 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:54.618 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:54.619 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:54.619 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:54.619 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:54.619 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:54.620 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:54.620 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:54.621 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:54.621 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:54.622 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:54.622 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:54.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:54.623 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:54.623 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:54.623 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:54.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:54.624 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:54Z" 2022-09-03T06:18:54.624 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:54.625 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.779591_222.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.779591Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:54Z" } 2022-09-03T06:18:54.625 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.779591_222.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.779591Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:54.626 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:54.627 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.779591_222.1', 2: '', 3: ''} 2022-09-03T06:18:54.627 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.779591_222.1 2022-09-03T06:18:59.630 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:59.631 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:18:59.632 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:18:59.682 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:18:59.683 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:18:59.735 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:59.731+0000 7f9b88424600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:59.736 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:18:59.732+0000 7f9b88424600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:18:59.754 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:18:59.755 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:18:59.756 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:18:59.756 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:18:59.757 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:18:59.757 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:18:59.758 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:59.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:59.760 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:18:59.760 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:59.761 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:18:59.761 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:59.762 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:59.762 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:18:59.763 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:59.763 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:18:59.764 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:59.764 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:18:59.765 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:59.765 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:59.778 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:59.779 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:59.779 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:18:59.780 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:59.781 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:59.781 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.850209_226.1", 2022-09-03T06:18:59.782 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:59.782 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:59.783 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:59.783 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209Z", 2022-09-03T06:18:59.784 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:59.786 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:59.788 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:59.789 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:59.789 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:18:59.790 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:59.790 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:59.791 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:59.791 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:59.792 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:59.792 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:59.792 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:59.793 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:59.793 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:59.794 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:59.794 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:18:59.795 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:18:59.795 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:18:59.795 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:18:59.796 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:18:59.796 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:18:59.797 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:18:59.797 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:18:59.798 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:18:59.798 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:18:59.799 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:59.799 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:18:59.800 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:18:59.800 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:59.801 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:18:59.801 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:18:59.802 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:18:59.802 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:18:59.803 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:18:59Z" 2022-09-03T06:18:59.803 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:18:59.805 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.850209_226.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.850209Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:18:59Z" } 2022-09-03T06:18:59.808 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.850209_226.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.850209Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:18:59.809 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:18:59.810 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.850209_226.1', 2: '', 3: ''} 2022-09-03T06:18:59.810 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.850209_226.1 2022-09-03T06:19:04.815 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:04.817 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:04.818 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:04.873 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:04.874 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:04.923 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:04.917+0000 7f6860b63600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:04.924 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:04.918+0000 7f6860b63600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:04.944 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:04.944 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:04.945 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:04.945 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:04.947 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:04.948 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:04.948 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:04.948 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:04.949 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:04.949 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:04.950 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:04.950 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:04.951 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:04.951 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:04.951 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:04.952 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:04.952 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:04.955 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:04.955 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:04.956 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:04.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:04.956 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:04.957 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:04.960 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:04.960 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:04.961 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.842565_224.1", 2022-09-03T06:19:04.961 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:04.961 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:04.962 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:04.962 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.842565Z", 2022-09-03T06:19:04.963 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:04.963 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:04.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:04.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:04.965 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:04.967 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:04.968 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:04.968 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:04.968 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:04.976 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:04.976 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:04.976 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:04.977 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:04.977 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:04.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:04.977 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:04.979 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:04.979 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:04.980 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:04.980 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:04.980 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:04.981 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:04.981 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:04.981 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:04.982 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:04.982 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:04.982 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:04.983 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:04.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:04.984 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:04.984 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:04.985 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:04.985 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:04.985 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:04Z" 2022-09-03T06:19:04.985 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:04.986 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.842565_224.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.842565Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:04Z" } 2022-09-03T06:19:04.987 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.842565_224.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.842565Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:04.987 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:04.988 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.842565_224.1', 2: '', 3: ''} 2022-09-03T06:19:04.988 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.842565_224.1 2022-09-03T06:19:09.990 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:09.991 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:09.992 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:10.058 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:10.059 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:10.112 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:10.109+0000 7f834a871600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:10.113 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:10.110+0000 7f834a871600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:10.133 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:10.134 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:10.135 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:10.135 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:10.136 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:10.136 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:10.137 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:10.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:10.139 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:10.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:10.140 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:10.140 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:10.141 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:10.141 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:10.142 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:10.142 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:10.144 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:10.145 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:10.145 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:10.146 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:10.147 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:10.147 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:10.147 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:10.148 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:10.148 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:10.149 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.778620_221.1", 2022-09-03T06:19:10.149 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:10.150 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:10.151 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:10.153 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.778620Z", 2022-09-03T06:19:10.153 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:10.154 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:10.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:10.154 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:10.155 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:10.155 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:10.155 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:10.156 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:10.156 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:10.156 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:10.157 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:10.157 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:10.157 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:10.157 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:10.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:10.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:10.158 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:10.158 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:10.159 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:10.159 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:10.159 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:10.160 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:10.160 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:10.161 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:10.161 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:10.161 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:10.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:10.162 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:10.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:10.165 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:10.165 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:10.166 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:10.167 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:10.167 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:10Z" 2022-09-03T06:19:10.168 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:10.170 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.778620_221.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.778620Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:10Z" } 2022-09-03T06:19:10.172 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.778620_221.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.778620Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:10.173 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:10.174 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.778620_221.1', 2: '', 3: ''} 2022-09-03T06:19:10.175 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.778620_221.1 2022-09-03T06:19:15.180 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:15.181 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:15.181 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:15.268 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:15.269 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:15.333 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:15.331+0000 7fa55de69600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:15.334 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:15.332+0000 7fa55de69600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:15.352 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:15.353 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:15.353 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:15.353 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:15.354 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:15.354 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:15.354 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:15.355 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:15.355 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:15.355 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:15.355 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:15.356 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:15.356 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:15.356 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:15.356 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:15.357 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:15.358 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:15.358 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:15.358 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:15.359 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:15.359 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:15.359 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:15.360 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:15.360 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:15.360 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:15.360 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.778620_221.1", 2022-09-03T06:19:15.361 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:15.361 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:15.361 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:15.361 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.778620Z", 2022-09-03T06:19:15.362 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:15.362 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:15.362 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:15.362 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:15.363 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:15.363 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:15.363 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:15.363 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:15.364 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:15.365 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:15.365 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:15.365 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:15.366 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:15.367 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:15.367 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:15.368 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:15.368 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:15.368 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:15.369 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:15.369 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:15.369 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:15.369 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:15.370 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:15.370 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:15.370 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:15.371 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:15.371 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:15.371 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:15.371 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:15.372 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:15.373 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:15.373 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:15.373 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:15.373 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:15Z" 2022-09-03T06:19:15.374 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:15.376 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.778620_221.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.778620Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:15Z" } 2022-09-03T06:19:15.377 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.778620_221.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.778620Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:15.377 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:15.378 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.778620_221.1', 2: '', 3: ''} 2022-09-03T06:19:15.379 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.778620_221.1 2022-09-03T06:19:20.380 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:20.381 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:20.382 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:20.434 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:20.435 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:20.475 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:20.473+0000 7fbf8abeb600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:20.476 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:20.474+0000 7fbf8abeb600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:20.490 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:20.490 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:20.491 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:20.491 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:20.491 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:20.492 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:20.494 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:20.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:20.495 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:20.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:20.496 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:20.497 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:20.497 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:20.498 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:20.498 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:20.499 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:20.500 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:20.500 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:20.501 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:20.501 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:20.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:20.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:20.512 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:20.512 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:20.512 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:20.512 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.059572_234.1", 2022-09-03T06:19:20.513 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:20.513 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:20.513 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:20.522 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.059572Z", 2022-09-03T06:19:20.522 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:20.523 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:20.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:20.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:20.524 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:20.524 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:20.527 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:20.528 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:20.529 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:20.530 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:20.531 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:20.531 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:20.532 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:20.532 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:20.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:20.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:20.535 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:20.535 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:20.538 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:20.539 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:20.539 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:20.540 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:20.541 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:20.542 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:20.542 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:20.543 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:20.543 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:20.544 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:20.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:20.547 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:20.547 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:20.550 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:20.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:20.551 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:20Z" 2022-09-03T06:19:20.552 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:20.556 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.059572_234.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.059572Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:20Z" } 2022-09-03T06:19:20.557 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.059572_234.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.059572Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:20.558 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:20.559 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.059572_234.1', 2: '', 3: ''} 2022-09-03T06:19:20.560 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.059572_234.1 2022-09-03T06:19:25.562 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:25.563 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:25.565 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:25.620 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:25.621 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:25.662 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:25.659+0000 7fb6b2567600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:25.663 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:25.661+0000 7fb6b2567600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:25.678 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:25.678 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:25.679 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:25.679 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:25.680 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:25.680 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:25.681 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:25.681 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:25.681 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:25.682 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:25.682 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:25.683 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:25.683 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:25.683 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:25.684 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:25.684 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:25.684 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:25.685 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:25.685 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:25.685 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:25.686 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:25.686 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:25.686 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:25.686 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:25.687 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:25.687 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.850209_226.1", 2022-09-03T06:19:25.687 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:25.687 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:25.688 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:25.688 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209Z", 2022-09-03T06:19:25.689 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:25.689 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:25.690 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:25.690 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:25.691 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:25.691 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:25.691 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:25.691 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:25.692 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:25.692 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:25.692 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:25.692 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:25.693 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:25.693 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:25.693 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:25.693 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:25.694 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:25.694 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:25.694 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:25.695 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:25.695 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:25.695 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:25.695 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:25.696 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:25.696 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:25.696 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:25.696 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:25.697 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:25.697 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:25.697 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:25.698 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:25.698 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:25.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:25.699 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:25Z" 2022-09-03T06:19:25.700 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:25.700 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.850209_226.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.850209Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:25Z" } 2022-09-03T06:19:25.701 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.850209_226.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.850209Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:25.702 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:25.703 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.850209_226.1', 2: '', 3: ''} 2022-09-03T06:19:25.704 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.850209_226.1 2022-09-03T06:19:30.705 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:30.707 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:30.709 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:30.770 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:30.771 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:30.818 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:30.816+0000 7f23ada1e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:30.820 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:30.817+0000 7f23ada1e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:30.849 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:30.850 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:30.850 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:30.850 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:30.851 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:30.851 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:30.851 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:30.852 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:30.852 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:30.852 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:30.852 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:30.853 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:30.853 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:30.853 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:30.854 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:30.854 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:30.854 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:30.855 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:30.855 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:30.855 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:30.856 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:30.856 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:30.857 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:30.858 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:30.858 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:30.858 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.779591_222.1", 2022-09-03T06:19:30.859 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:30.859 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:30.859 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:30.860 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.779591Z", 2022-09-03T06:19:30.860 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:30.860 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:30.860 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:30.861 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:30.861 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:30.861 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:30.861 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:30.862 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:30.862 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:30.862 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:30.862 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:30.863 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:30.863 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:30.863 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:30.863 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:30.864 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:30.864 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:30.864 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:30.864 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:30.865 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:30.865 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:30.865 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:30.865 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:30.866 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:30.866 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:30.866 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:30.866 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:30.867 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:30.867 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:30.867 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:30.867 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:30.867 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:30.868 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:30.868 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:30Z" 2022-09-03T06:19:30.868 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:30.869 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.779591_222.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.779591Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:30Z" } 2022-09-03T06:19:30.869 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.779591_222.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.779591Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:30.870 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:30.870 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.779591_222.1', 2: '', 3: ''} 2022-09-03T06:19:30.871 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.779591_222.1 2022-09-03T06:19:35.873 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:35.874 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:35.874 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:35.932 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:35.933 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:35.979 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:35.977+0000 7fab67534600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:35.979 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:35.978+0000 7fab67534600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:35.998 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:35.998 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:35.999 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:35.999 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:35.999 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:36.000 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:36.000 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:36.000 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:36.000 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:36.001 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:36.001 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:36.001 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:36.001 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:36.002 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:36.002 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:36.002 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:36.002 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:36.003 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:36.003 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:36.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:36.003 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:36.004 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:36.004 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:36.005 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:36.005 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:36.005 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.921086_230.1", 2022-09-03T06:19:36.005 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:36.006 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:36.006 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:36.006 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.921086Z", 2022-09-03T06:19:36.006 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:36.008 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:36.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:36.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:36.010 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:36.010 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:36.011 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:36.011 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:36.011 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:36.012 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:36.012 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:36.013 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:36.013 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:36.014 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:36.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:36.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:36.015 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:36.015 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:36.015 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:36.016 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:36.016 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:36.016 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:36.016 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:36.017 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:36.017 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:36.017 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:36.017 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:36.018 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:36.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:36.018 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:36.019 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:36.019 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:36.019 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:36.019 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:35Z" 2022-09-03T06:19:36.020 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:36.021 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.921086_230.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.921086Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:35Z" } 2022-09-03T06:19:36.022 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.921086_230.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.921086Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:36.023 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:36.023 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.921086_230.1', 2: '', 3: ''} 2022-09-03T06:19:36.024 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.921086_230.1 2022-09-03T06:19:41.025 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:41.026 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:41.027 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:41.100 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:41.101 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:41.183 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:41.180+0000 7faad0ab9600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:41.183 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:41.181+0000 7faad0ab9600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:41.204 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:41.205 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:41.205 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:41.206 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:41.206 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:41.207 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:41.207 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:41.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:41.209 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:41.210 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:41.210 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:41.211 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:41.211 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:41.212 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:41.212 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:41.214 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:41.214 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:41.215 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:41.215 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:41.215 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:41.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:41.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:41.216 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:41.216 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:41.217 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:41.226 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.921086_230.1", 2022-09-03T06:19:41.226 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:41.227 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:41.227 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:41.227 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.921086Z", 2022-09-03T06:19:41.228 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:41.228 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:41.228 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:41.228 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:41.229 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:41.229 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:41.230 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:41.230 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:41.230 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:41.231 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:41.231 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:41.231 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:41.232 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:41.232 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:41.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:41.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:41.233 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:41.233 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:41.233 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:41.234 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:41.234 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:41.234 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:41.235 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:41.235 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:41.235 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:41.235 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:41.236 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:41.236 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:41.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:41.237 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:41.237 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:41.238 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:41.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:41.238 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:41Z" 2022-09-03T06:19:41.238 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:41.239 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.921086_230.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.921086Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:41Z" } 2022-09-03T06:19:41.240 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.921086_230.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.921086Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:41.240 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:41.241 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.921086_230.1', 2: '', 3: ''} 2022-09-03T06:19:41.241 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.921086_230.1 2022-09-03T06:19:46.249 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:46.250 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:46.251 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:46.308 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:46.308 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:46.353 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:46.351+0000 7f5745237600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:46.353 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:46.352+0000 7f5745237600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:46.375 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:46.375 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:46.377 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:46.377 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:46.377 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:46.378 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:46.378 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:46.378 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:46.378 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:46.379 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:46.379 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:46.379 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:46.380 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:46.380 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:46.381 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:46.381 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:46.381 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:46.382 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:46.383 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:46.383 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:46.383 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:46.383 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:46.384 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:46.384 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:46.384 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:46.385 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.842565_224.1", 2022-09-03T06:19:46.385 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:46.385 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:46.385 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:46.386 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.842565Z", 2022-09-03T06:19:46.387 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:46.388 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:46.388 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:46.389 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:46.389 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:46.389 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:46.390 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:46.390 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:46.394 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:46.395 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:46.395 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:46.395 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:46.396 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:46.396 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:46.397 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:46.397 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:46.400 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:46.401 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:46.401 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:46.402 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:46.402 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:46.403 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:46.403 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:46.404 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:46.404 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:46.404 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:46.405 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:46.405 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:46.406 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:46.406 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:46.407 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:46.407 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:46.407 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:46.408 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:46Z" 2022-09-03T06:19:46.409 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:46.409 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.842565_224.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.842565Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:46Z" } 2022-09-03T06:19:46.410 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.842565_224.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.842565Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:46.411 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:46.411 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.842565_224.1', 2: '', 3: ''} 2022-09-03T06:19:46.412 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.842565_224.1 2022-09-03T06:19:51.414 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:51.415 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:51.416 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:51.466 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:51.467 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:51.492 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:51.491+0000 7f842a890600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:51.493 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:51.492+0000 7f842a890600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:51.505 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:51.506 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:51.506 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:51.507 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:51.507 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:51.508 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:51.508 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:51.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:51.509 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:51.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:51.510 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:51.510 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:51.511 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:51.511 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:51.512 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:51.512 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:51.513 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:51.513 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:51.514 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:51.514 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:51.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:51.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:51.516 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:51.516 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:51.516 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:51.517 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.784101_223.1", 2022-09-03T06:19:51.517 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:51.518 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:51.518 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:51.519 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.784101Z", 2022-09-03T06:19:51.519 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:51.519 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:51.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:51.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:51.520 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:51Z" 2022-09-03T06:19:51.528 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:51.530 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.784101_223.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.784101Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:51Z" } 2022-09-03T06:19:51.531 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.784101_223.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.784101Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:51.531 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:51.532 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.784101_223.1', 2: '', 3: ''} 2022-09-03T06:19:51.532 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.784101_223.1 2022-09-03T06:19:56.533 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:56.535 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:19:56.537 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:19:56.596 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:19:56.596 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:19:56.641 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:56.640+0000 7f0fc09cf600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:56.642 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:19:56.641+0000 7f0fc09cf600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:19:56.659 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:19:56.660 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:19:56.660 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:19:56.660 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:19:56.661 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:19:56.661 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:19:56.661 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:56.661 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:56.662 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:19:56.662 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:56.662 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:19:56.663 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:56.663 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:56.663 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:19:56.664 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:56.664 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:19:56.664 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:56.665 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:19:56.665 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:56.665 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:56.665 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:56.666 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:56.667 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:19:56.667 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:56.668 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:56.668 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.842565_224.1", 2022-09-03T06:19:56.668 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:56.669 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:56.669 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:56.670 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.842565Z", 2022-09-03T06:19:56.670 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:56.671 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:56.671 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:56.672 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:56.672 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:19:56.673 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:56.673 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:56.674 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:56.674 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:56.674 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:56.675 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:56.675 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:56.676 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:56.676 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:56.676 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:56.677 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:19:56.677 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:19:56.678 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:19:56.678 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:19:56.678 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:19:56.679 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:19:56.679 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:19:56.679 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:19:56.680 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:19:56.680 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:19:56.681 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:56.681 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:19:56.681 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:19:56.682 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:56.682 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:19:56.682 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:19:56.683 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:19:56.683 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:19:56.683 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:19:56Z" 2022-09-03T06:19:56.684 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:19:56.685 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.842565_224.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.842565Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:19:56Z" } 2022-09-03T06:19:56.686 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.842565_224.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.842565Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:19:56.687 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:19:56.688 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.842565_224.1', 2: '', 3: ''} 2022-09-03T06:19:56.689 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.842565_224.1 2022-09-03T06:20:01.691 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:01.692 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:01.693 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:01.748 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:01.748 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:01.783 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:01.781+0000 7f8c96f17600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:01.784 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:01.782+0000 7f8c96f17600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:01.796 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:01.797 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:01.798 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:01.799 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:01.799 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:01.799 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:01.800 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:01.800 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:01.800 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:01.800 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:01.801 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:01.801 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:01.801 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:01.801 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:01.802 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:01.802 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:01.802 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:01.802 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:01.803 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:01.803 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:01.803 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:01.804 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:01.804 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:01.804 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:01.804 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:01.805 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.842565_224.1", 2022-09-03T06:20:01.805 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:01.805 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:01.805 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:01.806 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.842565Z", 2022-09-03T06:20:01.806 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:01.806 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:01.806 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:01.807 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:01.807 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:01.807 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:01.807 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:01.808 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:01.808 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:01.808 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:01.809 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:01.810 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:01.810 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:01.810 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:01.811 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:01.811 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:01.811 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:01.812 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:01.812 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:01.812 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:01.813 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:01.813 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:01.813 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:01.813 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:01.814 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:01.814 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:01.814 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:01.815 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:01.815 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:01.815 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:01.816 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:01.816 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:01.816 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:01.816 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:01Z" 2022-09-03T06:20:01.817 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:01.826 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.842565_224.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.842565Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:01Z" } 2022-09-03T06:20:01.827 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.842565_224.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.842565Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:01.828 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:01.828 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.842565_224.1', 2: '', 3: ''} 2022-09-03T06:20:01.829 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.842565_224.1 2022-09-03T06:20:06.831 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:06.832 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:06.833 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:06.893 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:06.893 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:06.937 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:06.935+0000 7f525586d600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:06.938 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:06.936+0000 7f525586d600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:06.958 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:06.958 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:06.959 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:06.959 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:06.959 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:06.960 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:06.960 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:06.960 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:06.961 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:06.961 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:06.961 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:06.962 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:06.962 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:06.962 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:06.963 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:06.963 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:06.963 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:06.964 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:06.964 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:06.964 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:06.965 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:06.965 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:06.966 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:06.966 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:06.967 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:06.968 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.779591_222.1", 2022-09-03T06:20:06.969 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:06.969 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:06.970 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:06.970 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.779591Z", 2022-09-03T06:20:06.971 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:06.971 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:06.972 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:06.972 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:06.973 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:06.974 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:06.974 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:06.975 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:06.975 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:06.976 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:06.976 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:06.976 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:06.977 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:06.977 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:06.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:06.978 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:06.978 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:06.978 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:06.979 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:06.979 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:06.979 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:06.979 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:06.980 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:06.980 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:06.980 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:06.981 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:06.981 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:06.981 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:06.982 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:06.982 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:06.982 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:06.983 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:06.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:06.983 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:06Z" 2022-09-03T06:20:06.983 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:06.985 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.779591_222.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.779591Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:06Z" } 2022-09-03T06:20:06.986 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.779591_222.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.779591Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:06.986 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:06.987 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.779591_222.1', 2: '', 3: ''} 2022-09-03T06:20:06.987 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.779591_222.1 2022-09-03T06:20:11.989 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:11.990 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:11.991 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:12.041 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:12.042 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:12.091 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:12.089+0000 7fbfd7e4e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:12.092 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:12.090+0000 7fbfd7e4e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:12.106 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:12.106 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:12.107 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:12.107 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:12.107 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:12.107 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:12.108 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:12.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:12.108 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:12.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:12.109 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:12.109 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:12.109 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:12.109 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:12.110 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:12.110 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:12.110 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:12.110 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:12.111 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:12.111 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:12.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:12.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:12.112 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:12.112 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:12.112 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:12.112 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.063873_235.1", 2022-09-03T06:20:12.113 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:12.113 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:12.113 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:12.113 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.063873Z", 2022-09-03T06:20:12.114 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:12.114 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:12.114 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:12.114 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:12.115 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:12.115 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:12.115 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:12.115 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:12.116 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:12.116 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:12.116 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:12.116 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:12.117 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:12.117 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:12.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:12.117 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:12.117 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:12.118 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:12.118 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:12.118 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:12.118 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:12.119 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:12.119 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:12.119 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:12.119 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:12.120 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:12.120 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:12.120 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:12.120 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:12.121 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:12.121 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:12.121 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:12.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:12.122 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:12Z" 2022-09-03T06:20:12.122 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:12.123 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.063873_235.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.063873Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:12Z" } 2022-09-03T06:20:12.124 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.063873_235.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.063873Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:12.125 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:12.125 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.063873_235.1', 2: '', 3: ''} 2022-09-03T06:20:12.126 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.063873_235.1 2022-09-03T06:20:17.127 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:17.128 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:17.130 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:17.181 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:17.182 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:17.222 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:17.221+0000 7f3aa5bde600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:17.224 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:17.222+0000 7f3aa5bde600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:17.236 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:17.236 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:17.237 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:17.237 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:17.237 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:17.238 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:17.238 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:17.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:17.238 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:17.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:17.239 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:17.240 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:17.240 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:17.240 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:17.240 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:17.241 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:17.241 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:17.241 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:17.242 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:17.242 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:17.242 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:17.242 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:17.243 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:17.243 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:17.243 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:17.243 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.842565_224.1", 2022-09-03T06:20:17.244 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:17.244 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:17.244 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:17.244 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.842565Z", 2022-09-03T06:20:17.245 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:17.245 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:17.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:17.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:17.246 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:17.246 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:17.246 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:17.247 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:17.247 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:17.247 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:17.247 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:17.248 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:17.248 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:17.248 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:17.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:17.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:17.249 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:17.249 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:17.250 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:17.250 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:17.250 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:17.250 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:17.251 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:17.251 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:17.251 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:17.251 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:17.252 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:17.252 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:17.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:17.252 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:17.253 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:17.253 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:17.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:17.253 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:17Z" 2022-09-03T06:20:17.254 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:17.255 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.842565_224.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.842565Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:17Z" } 2022-09-03T06:20:17.256 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.842565_224.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.842565Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:17.257 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:17.257 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.842565_224.1', 2: '', 3: ''} 2022-09-03T06:20:17.258 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.842565_224.1 2022-09-03T06:20:22.259 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:22.260 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:22.261 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:22.316 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:22.317 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:22.383 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:22.381+0000 7f10e0a2a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:22.384 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:22.382+0000 7f10e0a2a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:22.421 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:22.422 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:22.423 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:22.424 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:22.424 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:22.424 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:22.424 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:22.425 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:22.425 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:22.425 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:22.425 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:22.426 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:22.426 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:22.426 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:22.426 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:22.427 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:22.427 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:22.427 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:22.427 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:22.428 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:22.428 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:22.428 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:22.428 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:22.429 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:22.429 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:22.429 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.897692_229.1", 2022-09-03T06:20:22.429 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:22.430 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:22.430 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:22.430 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.897692Z", 2022-09-03T06:20:22.430 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:22.431 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:22.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:22.431 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:22.431 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:22.432 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:22.432 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:22.432 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:22.432 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:22.433 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:22.433 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:22.434 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:22.434 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:22.435 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:22.435 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:22.435 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:22.435 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:22.436 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:22.436 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:22.436 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:22.436 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:22.437 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:22.437 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:22.437 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:22.438 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:22.438 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:22.438 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:22.438 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:22.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:22.439 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:22.441 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:22.441 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:22.441 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:22.441 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:22Z" 2022-09-03T06:20:22.442 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:22.443 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.897692_229.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.897692Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:22Z" } 2022-09-03T06:20:22.444 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.897692_229.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.897692Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:22.445 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:22.445 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.897692_229.1', 2: '', 3: ''} 2022-09-03T06:20:22.446 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.897692_229.1 2022-09-03T06:20:27.447 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:27.448 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:27.449 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:27.505 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:27.506 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:27.539 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:27.537+0000 7f3164b69600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:27.540 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:27.538+0000 7f3164b69600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:27.552 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:27.552 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:27.553 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:27.553 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:27.553 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:27.553 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:27.554 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:27.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:27.554 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:27.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:27.555 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:27.555 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:27.555 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:27.555 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:27.556 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:27.556 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:27.556 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:27.556 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:27.557 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:27.557 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:27.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:27.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:27.558 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:27.558 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:27.558 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:27.558 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.921086_230.1", 2022-09-03T06:20:27.559 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:27.559 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:27.559 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:27.559 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.921086Z", 2022-09-03T06:20:27.560 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:27.560 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:27.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:27.560 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:27.561 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:27.561 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:27.561 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:27.561 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:27.562 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:27.562 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:27.562 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:27.562 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:27.563 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:27.567 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:27.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:27.568 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:27.569 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:27.569 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:27.570 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:27.570 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:27.571 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:27.571 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:27.572 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:27.572 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:27.573 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:27.573 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:27.573 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:27.573 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:27.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:27.574 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:27.574 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:27.575 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:27.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:27.575 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:27Z" 2022-09-03T06:20:27.575 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:27.577 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.921086_230.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.921086Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:27Z" } 2022-09-03T06:20:27.578 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.921086_230.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.921086Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:27.579 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:27.579 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.921086_230.1', 2: '', 3: ''} 2022-09-03T06:20:27.580 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.921086_230.1 2022-09-03T06:20:32.582 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:32.583 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:32.583 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:32.640 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:32.640 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:32.672 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:32.670+0000 7fc6ed00d600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:32.672 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:32.671+0000 7fc6ed00d600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:32.685 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:32.686 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:32.686 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:32.688 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:32.688 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:32.689 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:32.689 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:32.690 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:32.690 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:32.690 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:32.690 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:32.691 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:32.691 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:32.691 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:32.691 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:32.692 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:32.692 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:32.693 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:32.693 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:32.694 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:32.694 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:32.694 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:32.694 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:32.695 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:32.695 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:32.695 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.894939_228.1", 2022-09-03T06:20:32.696 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:32.696 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:32.696 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:32.697 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.894939Z", 2022-09-03T06:20:32.697 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:32.697 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:32.697 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:32.698 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:32.698 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:32.698 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:32.699 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:32.699 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:32.700 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:32.700 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:32.701 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:32.701 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:32.702 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:32.702 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:32.703 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:32.703 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:32.704 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:32.704 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:32.705 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:32.705 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:32.706 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:32.707 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:32.707 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:32.708 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:32.708 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:32.709 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:32.709 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:32.710 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:32.710 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:32.710 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:32.711 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:32.712 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:32.712 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:32.713 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:32Z" 2022-09-03T06:20:32.713 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:32.715 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.894939_228.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.894939Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:32Z" } 2022-09-03T06:20:32.717 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.894939_228.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.894939Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:32.718 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:32.719 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.894939_228.1', 2: '', 3: ''} 2022-09-03T06:20:32.720 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.894939_228.1 2022-09-03T06:20:37.721 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:37.722 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:37.723 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:37.799 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:37.799 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:37.898 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:37.897+0000 7efe92ba6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:37.899 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:37.898+0000 7efe92ba6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:37.913 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:37.914 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:37.914 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:37.914 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:37.915 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:37.915 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:37.915 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:37.916 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:37.916 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:37.916 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:37.916 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:37.917 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:37.917 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:37.917 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:37.917 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:37.918 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:37.918 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:37.918 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:37.918 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:37.919 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:37.919 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:37.919 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:37.919 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:37.920 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:37.920 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:37.920 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185756.064807_236.1", 2022-09-03T06:20:37.920 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:37.921 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:37.921 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:37.921 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.064807Z", 2022-09-03T06:20:37.921 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:37.922 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:37.922 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:37.922 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:37.923 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:37.923 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:37.923 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:37.923 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:37.924 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:37.924 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:37.924 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:37.925 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:37.925 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:37.925 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:37.925 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:37.926 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:37.926 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:37.926 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:37.926 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:37.927 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:37.927 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:37.927 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:37.927 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:37.928 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:37.928 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:37.928 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:37.928 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:37.929 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:37.929 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:37.929 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:37.929 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:37.930 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:37.930 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:37.930 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:37Z" 2022-09-03T06:20:37.930 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:37.932 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185756.064807_236.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:56.064807Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:37Z" } 2022-09-03T06:20:37.933 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185756.064807_236.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:56.064807Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:37.933 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:37.934 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185756.064807_236.1', 2: '', 3: ''} 2022-09-03T06:20:37.934 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185756.064807_236.1 2022-09-03T06:20:42.936 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:42.937 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:42.937 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:42.988 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:42.988 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:43.017 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:43.015+0000 7f84de2c0600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:43.019 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:43.016+0000 7f84de2c0600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:43.032 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:43.032 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:43.033 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:43.033 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:43.033 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:43.034 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:43.034 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:43.034 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:43.035 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:43.035 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:43.035 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:43.035 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:43.036 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:43.036 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:43.036 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:43.036 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:43.037 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:43.037 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:43.037 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:43.038 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:43.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:43.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:43.038 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:43.039 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:43.039 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:43.039 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.778620_221.1", 2022-09-03T06:20:43.040 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:43.046 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:43.046 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:43.046 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.778620Z", 2022-09-03T06:20:43.047 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:43.047 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:43.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:43.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:43.048 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:43.048 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:43.049 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:43.049 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:43.049 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:43.049 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:43.050 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:43.050 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:43.050 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:43.050 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:43.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:43.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:43.051 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:43.051 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:43.052 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:43.052 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:43.052 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:43.053 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:43.053 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:43.053 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:43.054 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:43.054 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:43.055 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:43.055 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:43.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:43.057 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:43.057 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:43.058 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:43.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:43.058 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:43Z" 2022-09-03T06:20:43.058 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:43.059 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.778620_221.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.778620Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:43Z" } 2022-09-03T06:20:43.060 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.778620_221.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.778620Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:43.060 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:43.061 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.778620_221.1', 2: '', 3: ''} 2022-09-03T06:20:43.061 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.778620_221.1 2022-09-03T06:20:48.064 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:48.064 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:48.065 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:48.116 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:48.117 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:48.144 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:48.142+0000 7f8a6c992600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:48.145 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:48.143+0000 7f8a6c992600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:48.157 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:48.157 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:48.158 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:48.158 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:48.158 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:48.159 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:48.159 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:48.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:48.159 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:48.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:48.160 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:48.160 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:48.160 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:48.161 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:48.161 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:48.161 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:48.161 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:48.162 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:48.162 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:48.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:48.162 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:48.163 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:48.163 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:48.163 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:48.163 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:48.164 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.850209_226.1", 2022-09-03T06:20:48.164 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:48.164 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:48.164 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:48.164 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209Z", 2022-09-03T06:20:48.165 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:48.165 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:48.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:48.165 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:48.166 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:48.166 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:48.167 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:48.167 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:48.167 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:48.167 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:48.168 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:48.168 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:48.168 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:48.168 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:48.169 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:48.169 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:48.169 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:48.169 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:48.169 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:48.170 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:48.170 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:48.170 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:48.170 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:48.171 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:48.171 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:48.171 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:48.171 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:48.172 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:48.172 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:48.172 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:48.172 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:48.173 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:48.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:48.173 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:48Z" 2022-09-03T06:20:48.173 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:48.175 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.850209_226.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.850209Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:48Z" } 2022-09-03T06:20:48.175 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.850209_226.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.850209Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:48.176 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:48.177 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.850209_226.1', 2: '', 3: ''} 2022-09-03T06:20:48.177 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.850209_226.1 2022-09-03T06:20:53.179 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:53.180 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:53.181 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:53.237 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:53.238 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:53.272 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:53.270+0000 7f2a20ba1600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:53.273 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:53.271+0000 7f2a20ba1600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:53.286 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:53.286 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:53.287 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:53.287 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:53.287 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:53.287 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:53.288 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:53.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:53.288 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:53.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:53.289 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:53.289 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:53.293 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:53.293 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:53.293 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:53.294 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:53.294 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:53.294 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:53.294 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:53.295 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:53.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:53.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:53.295 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:53.296 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:53.296 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:53.296 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.779591_222.1", 2022-09-03T06:20:53.296 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:53.297 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:53.297 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:53.297 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.779591Z", 2022-09-03T06:20:53.297 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:53.298 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:53.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:53.298 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:53.298 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:53.299 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:53.299 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:53.299 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:53.299 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:53.300 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:53.300 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:53.300 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:53.300 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:53.301 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:53.301 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:53.301 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:53.301 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:53.302 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:53.302 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:53.302 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:53.302 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:53.303 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:53.303 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:53.303 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:53.303 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:53.304 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:53.304 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:53.304 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:53.304 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:53.305 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:53.305 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:53.305 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:53.305 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:53.306 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:53Z" 2022-09-03T06:20:53.306 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:53.307 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.779591_222.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.779591Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:53Z" } 2022-09-03T06:20:53.308 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.779591_222.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.779591Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:53.308 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:53.309 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.779591_222.1', 2: '', 3: ''} 2022-09-03T06:20:53.309 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.779591_222.1 2022-09-03T06:20:58.311 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:58.311 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:20:58.312 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:20:58.370 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:20:58.371 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:20:58.421 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:58.419+0000 7fb6faebf600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:58.422 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:20:58.420+0000 7fb6faebf600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:20:58.442 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:20:58.443 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:20:58.443 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:20:58.443 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:20:58.443 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:20:58.444 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:20:58.444 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:58.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:58.445 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:20:58.445 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:58.445 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:20:58.445 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:58.446 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:58.446 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:20:58.447 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:58.447 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:20:58.447 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:58.448 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:20:58.448 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:58.449 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:58.449 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:58.450 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:58.451 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:20:58.451 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:58.452 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:58.452 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185755.784101_223.1", 2022-09-03T06:20:58.453 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:58.453 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:58.454 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:58.454 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.784101Z", 2022-09-03T06:20:58.455 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:58.455 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:58.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:58.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:58.457 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:20:58.457 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:58.458 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:58.458 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:58.459 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:58.459 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:58.460 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:58.460 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:58.461 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:58.461 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:58.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:58.462 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:20:58.462 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:20:58.462 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:20:58.463 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:20:58.463 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:20:58.464 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:20:58.464 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:20:58.465 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:20:58.465 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:20:58.465 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:20:58.466 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:58.466 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:20:58.466 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:20:58.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:58.467 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:20:58.467 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:20:58.467 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:20:58.468 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:20:58.468 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:20:58Z" 2022-09-03T06:20:58.469 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:20:58.469 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185755.784101_223.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:55.784101Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:20:58Z" } 2022-09-03T06:20:58.470 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185755.784101_223.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:55.784101Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:20:58.470 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:20:58.471 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185755.784101_223.1', 2: '', 3: ''} 2022-09-03T06:20:58.472 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662185758.915547_281.1 target=1_1662185755.784101_223.1 2022-09-03T06:21:03.474 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:03.475 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:03.476 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:21:03.533 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:21:03.533 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:21:03.581 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:03.580+0000 7f16bd113600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:03.582 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:03.581+0000 7f16bd113600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:03.601 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:21:03.601 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:21:03.601 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:21:03.602 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:21:03.602 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:21:03.602 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:21:03.602 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:03.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:03.603 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:21:03.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:03.604 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:21:03.604 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:03.604 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:03.604 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:21:03.605 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:03.605 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:21:03.605 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:03.605 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:21:03.606 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:03.606 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:03.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:03.606 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:03.607 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:21:03.607 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:03.607 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:03.607 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662185758.915547_281.1", 2022-09-03T06:21:03.608 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:03.608 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:03.608 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:03.608 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.915547Z", 2022-09-03T06:21:03.609 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:03.609 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:03.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:03.609 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:03.610 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:21:03.610 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:03.610 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:03.610 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:21:03.611 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:03.611 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:03.611 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:03.611 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:21:03.612 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:03.612 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:03.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:03.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:03.613 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:21:03.613 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:03.613 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:03.614 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:21:03.614 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:03.614 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:03.615 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:03.615 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:21:03.615 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:03.615 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:03.616 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:03.616 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:03.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:03.616 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:21:03.616 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:21:03.617 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:21:03.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:03.617 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:21:03Z" 2022-09-03T06:21:03.617 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:21:03.618 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662185758.915547_281.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:15:58.915547Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:21:03Z" } 2022-09-03T06:21:03.618 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662185758.915547_281.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:15:58.915547Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:21:03.619 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:21:03.619 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662185758.915547_281.1', 2: '', 3: ''} 2022-09-03T06:21:03.620 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:21:03.621 DEBUG:boto:path=/avysmw-36/ 2022-09-03T06:21:03.622 DEBUG:boto:auth_path=/avysmw-36/ 2022-09-03T06:21:03.623 DEBUG:boto:Method: HEAD 2022-09-03T06:21:03.623 DEBUG:boto:Path: /avysmw-36/ 2022-09-03T06:21:03.624 DEBUG:boto:Data: 2022-09-03T06:21:03.624 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:21:03.626 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.626 DEBUG:boto:Params: {} 2022-09-03T06:21:03.627 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:21:03.628 DEBUG:boto:Token: None 2022-09-03T06:21:03.629 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:21:03 GMT /avysmw-36/ 2022-09-03T06:21:03.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KJUP6i5XEq+xuNtOis/86ZBMhc0= 2022-09-03T06:21:03.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KJUP6i5XEq+xuNtOis/86ZBMhc0=', 'Content-Length': '0'} 2022-09-03T06:21:03.636 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000052acc2a54d8c4fbd-006312f24f-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.637 DEBUG:boto:path=/avysmw-36/foo 2022-09-03T06:21:03.638 DEBUG:boto:auth_path=/avysmw-36/foo 2022-09-03T06:21:03.638 DEBUG:boto:Method: PUT 2022-09-03T06:21:03.639 DEBUG:boto:Path: /avysmw-36/foo 2022-09-03T06:21:03.640 DEBUG:boto:Data: 2022-09-03T06:21:03.640 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'N7UdGUp1E+RbVvZSTy1R8g==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:21:03.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:21:03.641 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.641 DEBUG:boto:Params: {} 2022-09-03T06:21:03.642 DEBUG:boto:Token: None 2022-09-03T06:21:03.643 DEBUG:boto:StringToSign: PUT N7UdGUp1E+RbVvZSTy1R8g== application/octet-stream Sat, 03 Sep 2022 06:21:03 GMT /avysmw-36/foo 2022-09-03T06:21:03.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gTDxJ/tP2l4vn3o2vnjCCFrF4GM= 2022-09-03T06:21:03.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'N7UdGUp1E+RbVvZSTy1R8g==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gTDxJ/tP2l4vn3o2vnjCCFrF4GM='} 2022-09-03T06:21:03.661 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"37b51d194a7513e45b56f6524f2d51f2"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000009fb1183648975299-006312f24f-10f6-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.662 DEBUG:boto:path=/avysmw-36/ 2022-09-03T06:21:03.663 DEBUG:boto:auth_path=/avysmw-36/ 2022-09-03T06:21:03.664 DEBUG:boto:Method: DELETE 2022-09-03T06:21:03.664 DEBUG:boto:Path: /avysmw-36/ 2022-09-03T06:21:03.665 DEBUG:boto:Data: 2022-09-03T06:21:03.666 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:21:03.667 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.668 DEBUG:boto:Params: {} 2022-09-03T06:21:03.668 DEBUG:boto:Token: None 2022-09-03T06:21:03.669 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 06:21:03 GMT /avysmw-36/ 2022-09-03T06:21:03.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FAT3oIMi16rj7qoCm2eyJJMF/gA= 2022-09-03T06:21:03.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FAT3oIMi16rj7qoCm2eyJJMF/gA=', 'Content-Length': '0'} 2022-09-03T06:21:03.679 DEBUG:boto:Response headers: [('Content-Length', '234'), ('x-amz-request-id', 'tx0000063b9a4d3d3038e2b-006312f24f-10f6-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.681 DEBUG:boto:path=/avysmw-37/ 2022-09-03T06:21:03.681 DEBUG:boto:auth_path=/avysmw-37/ 2022-09-03T06:21:03.682 DEBUG:boto:Method: HEAD 2022-09-03T06:21:03.682 DEBUG:boto:Path: /avysmw-37/ 2022-09-03T06:21:03.683 DEBUG:boto:Data: 2022-09-03T06:21:03.684 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.685 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:21:03.686 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.687 DEBUG:boto:Params: {} 2022-09-03T06:21:03.688 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:21:03.689 DEBUG:boto:Token: None 2022-09-03T06:21:03.690 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:21:03 GMT /avysmw-37/ 2022-09-03T06:21:03.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lWDc5zIayPZZrnvK0ntBBlFCpUI= 2022-09-03T06:21:03.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lWDc5zIayPZZrnvK0ntBBlFCpUI=', 'Content-Length': '0'} 2022-09-03T06:21:03.696 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009cf4e1f5ae6c9fa6-006312f24f-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.697 DEBUG:boto:path=/avysmw-37/foo 2022-09-03T06:21:03.697 DEBUG:boto:auth_path=/avysmw-37/foo 2022-09-03T06:21:03.698 DEBUG:boto:Method: PUT 2022-09-03T06:21:03.699 DEBUG:boto:Path: /avysmw-37/foo 2022-09-03T06:21:03.699 DEBUG:boto:Data: 2022-09-03T06:21:03.700 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'N7UdGUp1E+RbVvZSTy1R8g==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:21:03.700 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:21:03.700 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.701 DEBUG:boto:Params: {} 2022-09-03T06:21:03.702 DEBUG:boto:Token: None 2022-09-03T06:21:03.702 DEBUG:boto:StringToSign: PUT N7UdGUp1E+RbVvZSTy1R8g== application/octet-stream Sat, 03 Sep 2022 06:21:03 GMT /avysmw-37/foo 2022-09-03T06:21:03.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUhNOD4jzpQjAIbT9fkQhpLfRe0= 2022-09-03T06:21:03.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'N7UdGUp1E+RbVvZSTy1R8g==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUhNOD4jzpQjAIbT9fkQhpLfRe0='} 2022-09-03T06:21:03.717 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"37b51d194a7513e45b56f6524f2d51f2"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000e4996967954c2b09-006312f24f-10ee-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.718 DEBUG:boto:path=/avysmw-37/ 2022-09-03T06:21:03.719 DEBUG:boto:auth_path=/avysmw-37/ 2022-09-03T06:21:03.719 DEBUG:boto:Method: DELETE 2022-09-03T06:21:03.720 DEBUG:boto:Path: /avysmw-37/ 2022-09-03T06:21:03.720 DEBUG:boto:Data: 2022-09-03T06:21:03.722 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.722 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:21:03.723 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.723 DEBUG:boto:Params: {} 2022-09-03T06:21:03.724 DEBUG:boto:Token: None 2022-09-03T06:21:03.724 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 06:21:03 GMT /avysmw-37/ 2022-09-03T06:21:03.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RqtQdHpsY0haicCZr8j4ye8nafw= 2022-09-03T06:21:03.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RqtQdHpsY0haicCZr8j4ye8nafw=', 'Content-Length': '0'} 2022-09-03T06:21:03.733 DEBUG:boto:Response headers: [('Content-Length', '234'), ('x-amz-request-id', 'tx000006444c8932e86e295-006312f24f-10ee-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.734 DEBUG:boto:path=/avysmw-38/ 2022-09-03T06:21:03.734 DEBUG:boto:auth_path=/avysmw-38/ 2022-09-03T06:21:03.735 DEBUG:boto:Method: HEAD 2022-09-03T06:21:03.737 DEBUG:boto:Path: /avysmw-38/ 2022-09-03T06:21:03.737 DEBUG:boto:Data: 2022-09-03T06:21:03.738 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:21:03.739 DEBUG:boto:Port: 8001 2022-09-03T06:21:03.739 DEBUG:boto:Params: {} 2022-09-03T06:21:03.740 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:21:03.741 DEBUG:boto:Token: None 2022-09-03T06:21:03.742 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:21:03 GMT /avysmw-38/ 2022-09-03T06:21:03.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o2so79tovMFfthZzAr76gcWy8l8= 2022-09-03T06:21:03.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o2so79tovMFfthZzAr76gcWy8l8=', 'Content-Length': '0'} 2022-09-03T06:21:03.747 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000041c42ddce62e614a-006312f24f-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.748 DEBUG:boto:path=/avysmw-38/foo 2022-09-03T06:21:03.749 DEBUG:boto:auth_path=/avysmw-38/foo 2022-09-03T06:21:03.749 DEBUG:boto:Method: PUT 2022-09-03T06:21:03.750 DEBUG:boto:Path: /avysmw-38/foo 2022-09-03T06:21:03.751 DEBUG:boto:Data: 2022-09-03T06:21:03.751 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'N7UdGUp1E+RbVvZSTy1R8g==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:21:03.752 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:21:03.753 DEBUG:boto:Port: 8001 2022-09-03T06:21:03.753 DEBUG:boto:Params: {} 2022-09-03T06:21:03.754 DEBUG:boto:Token: None 2022-09-03T06:21:03.755 DEBUG:boto:StringToSign: PUT N7UdGUp1E+RbVvZSTy1R8g== application/octet-stream Sat, 03 Sep 2022 06:21:03 GMT /avysmw-38/foo 2022-09-03T06:21:03.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gRuFhdHcFhWLLdxe89hOg7oIUtQ= 2022-09-03T06:21:03.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'N7UdGUp1E+RbVvZSTy1R8g==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gRuFhdHcFhWLLdxe89hOg7oIUtQ='} 2022-09-03T06:21:03.773 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"37b51d194a7513e45b56f6524f2d51f2"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000008e6ce6bae58ec0b0-006312f24f-10fa-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.773 DEBUG:boto:path=/avysmw-38/ 2022-09-03T06:21:03.774 DEBUG:boto:auth_path=/avysmw-38/ 2022-09-03T06:21:03.775 DEBUG:boto:Method: DELETE 2022-09-03T06:21:03.776 DEBUG:boto:Path: /avysmw-38/ 2022-09-03T06:21:03.776 DEBUG:boto:Data: 2022-09-03T06:21:03.777 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:21:03.780 DEBUG:boto:Port: 8001 2022-09-03T06:21:03.781 DEBUG:boto:Params: {} 2022-09-03T06:21:03.782 DEBUG:boto:Token: None 2022-09-03T06:21:03.782 DEBUG:boto:StringToSign: DELETE Sat, 03 Sep 2022 06:21:03 GMT /avysmw-38/ 2022-09-03T06:21:03.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GPtHTylM17wSeTpTqhFoH1ZJ+JQ= 2022-09-03T06:21:03.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GPtHTylM17wSeTpTqhFoH1ZJ+JQ=', 'Content-Length': '0'} 2022-09-03T06:21:03.792 DEBUG:boto:Response headers: [('Content-Length', '234'), ('x-amz-request-id', 'tx00000cafe2ff3c006d797-006312f24f-10fa-test-zone3'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.793 DEBUG:boto:path=/avysmw-36/ 2022-09-03T06:21:03.793 DEBUG:boto:auth_path=/avysmw-36/ 2022-09-03T06:21:03.794 DEBUG:boto:Method: HEAD 2022-09-03T06:21:03.794 DEBUG:boto:Path: /avysmw-36/ 2022-09-03T06:21:03.795 DEBUG:boto:Data: 2022-09-03T06:21:03.795 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.796 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:21:03.796 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.797 DEBUG:boto:Params: {} 2022-09-03T06:21:03.797 DEBUG:boto:Token: None 2022-09-03T06:21:03.798 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:21:03 GMT /avysmw-36/ 2022-09-03T06:21:03.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KJUP6i5XEq+xuNtOis/86ZBMhc0= 2022-09-03T06:21:03.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KJUP6i5XEq+xuNtOis/86ZBMhc0=', 'Content-Length': '0'} 2022-09-03T06:21:03.802 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008e74a99ee087c2a0-006312f24f-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.802 DEBUG:boto:path=/avysmw-37/ 2022-09-03T06:21:03.803 DEBUG:boto:auth_path=/avysmw-37/ 2022-09-03T06:21:03.803 DEBUG:boto:Method: HEAD 2022-09-03T06:21:03.804 DEBUG:boto:Path: /avysmw-37/ 2022-09-03T06:21:03.804 DEBUG:boto:Data: 2022-09-03T06:21:03.805 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:21:03.806 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.806 DEBUG:boto:Params: {} 2022-09-03T06:21:03.807 DEBUG:boto:Token: None 2022-09-03T06:21:03.807 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:21:03 GMT /avysmw-37/ 2022-09-03T06:21:03.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lWDc5zIayPZZrnvK0ntBBlFCpUI= 2022-09-03T06:21:03.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lWDc5zIayPZZrnvK0ntBBlFCpUI=', 'Content-Length': '0'} 2022-09-03T06:21:03.811 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000041560eeeec85a60d-006312f24f-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.812 DEBUG:boto:path=/avysmw-38/ 2022-09-03T06:21:03.812 DEBUG:boto:auth_path=/avysmw-38/ 2022-09-03T06:21:03.813 DEBUG:boto:Method: HEAD 2022-09-03T06:21:03.814 DEBUG:boto:Path: /avysmw-38/ 2022-09-03T06:21:03.815 DEBUG:boto:Data: 2022-09-03T06:21:03.815 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:21:03.817 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.817 DEBUG:boto:Params: {} 2022-09-03T06:21:03.818 DEBUG:boto:Token: None 2022-09-03T06:21:03.818 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:21:03 GMT /avysmw-38/ 2022-09-03T06:21:03.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o2so79tovMFfthZzAr76gcWy8l8= 2022-09-03T06:21:03.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o2so79tovMFfthZzAr76gcWy8l8=', 'Content-Length': '0'} 2022-09-03T06:21:03.822 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000094d823fd20e74259-006312f24f-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.823 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_delete_notempty ... ok 2022-09-03T06:21:03.826 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-39 2022-09-03T06:21:03.827 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:21:03.827 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:21:03.828 DEBUG:boto:Method: PUT 2022-09-03T06:21:03.828 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:21:03.829 DEBUG:boto:Data: 2022-09-03T06:21:03.830 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:21:03.831 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.832 DEBUG:boto:Params: {} 2022-09-03T06:21:03.832 DEBUG:boto:Token: None 2022-09-03T06:21:03.833 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:21:03 GMT /avysmw-39/ 2022-09-03T06:21:03.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nK+dtVMiZSN0+6l5l6kEb2KfOY= 2022-09-03T06:21:03.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nK+dtVMiZSN0+6l5l6kEb2KfOY=', 'Content-Length': '0'} 2022-09-03T06:21:03.862 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000020918ab0e40b0d38-006312f24f-10f6-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.863 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-40 2022-09-03T06:21:03.864 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:21:03.865 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:21:03.866 DEBUG:boto:Method: PUT 2022-09-03T06:21:03.866 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:21:03.867 DEBUG:boto:Data: 2022-09-03T06:21:03.867 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.868 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:21:03.868 DEBUG:boto:Port: 8000 2022-09-03T06:21:03.869 DEBUG:boto:Params: {} 2022-09-03T06:21:03.870 DEBUG:boto:Token: None 2022-09-03T06:21:03.870 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:21:03 GMT /avysmw-40/ 2022-09-03T06:21:03.871 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c9FI7zHqJetwZbA53NLimT/HKgg= 2022-09-03T06:21:03.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c9FI7zHqJetwZbA53NLimT/HKgg=', 'Content-Length': '0'} 2022-09-03T06:21:03.927 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000060a502685ff2679-006312f24f-10ee-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.928 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-41 2022-09-03T06:21:03.928 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:21:03.929 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:21:03.929 DEBUG:boto:Method: PUT 2022-09-03T06:21:03.930 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:21:03.930 DEBUG:boto:Data: 2022-09-03T06:21:03.931 DEBUG:boto:Headers: {} 2022-09-03T06:21:03.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:21:03.932 DEBUG:boto:Port: 8001 2022-09-03T06:21:03.932 DEBUG:boto:Params: {} 2022-09-03T06:21:03.933 DEBUG:boto:Token: None 2022-09-03T06:21:03.933 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:21:03 GMT /avysmw-41/ 2022-09-03T06:21:03.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mi3wkthctc4qzK+z7hNjhWfKWRg= 2022-09-03T06:21:03.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:21:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mi3wkthctc4qzK+z7hNjhWfKWRg=', 'Content-Length': '0'} 2022-09-03T06:21:03.967 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000b9e2d5d1ceff1090-006312f24f-10fa-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:21:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:21:03.967 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:03.968 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:03.969 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:21:04.021 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:21:04.021 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:21:04.050 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=b84ccf39-78f5-4998-9db6-42362bc8c98b 2022-09-03T06:21:04.071 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:21:04.072 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:21:04.072 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:21:04.072 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T05:52:30.715807Z" 2022-09-03T06:21:04.073 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:21:04.073 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:21:04.073 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186063.955393_299.1", 2022-09-03T06:21:04.074 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:21:03.955393Z" 2022-09-03T06:21:04.074 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:21:04.074 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:21:04.074 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:21:04.075 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:21:04.075 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:21:04.075 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:21:04.075 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:21:04.076 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:21:04.076 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:21:04.076 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:21:04.078 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662184350.715807_109.1', 1: '1_1662186063.955393_299.1', 2: '', 3: ''} 2022-09-03T06:21:04.078 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:21:04.079 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:04.079 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:04.080 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:21:04.136 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:21:04.136 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:21:04.187 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:04.185+0000 7fb0fc07f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:04.188 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:04.186+0000 7fb0fc07f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:04.208 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:21:04.209 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:21:04.209 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:21:04.209 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:21:04.210 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:21:04.210 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:21:04.210 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.211 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:21:04.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:04.212 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:21:04.212 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:04.212 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:04.212 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:21:04.213 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:04.213 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:21:04.213 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:04.214 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:21:04.215 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.215 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:04.216 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:21:04.216 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:04.216 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:04.216 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186063.955393_299.1", 2022-09-03T06:21:04.217 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:04.217 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:04.217 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:04.217 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.955393Z", 2022-09-03T06:21:04.218 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.218 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.219 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:04.220 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:21:04.220 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:04.221 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:04.221 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:21:04.222 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:04.223 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:04.223 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:04.224 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:21:04.225 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.225 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.226 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.226 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:04.227 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:21:04.227 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:04.227 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:04.227 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:21:04.228 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:04.228 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:04.228 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:04.228 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:21:04.229 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.229 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.229 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.229 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:04.230 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.230 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:21:04.230 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:21:04.230 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:21:04.231 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.231 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:21:04Z" 2022-09-03T06:21:04.231 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:21:04.232 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186063.955393_299.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:21:03.955393Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:21:04Z" } 2022-09-03T06:21:04.233 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186063.955393_299.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:21:03.955393Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:21:04.234 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662186063.955393_299.1', 2: '', 3: ''} 2022-09-03T06:21:04.234 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662186063.955393_299.1', 2: '', 3: ''} 2022-09-03T06:21:04.235 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:21:04.235 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:21:04.236 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:04.236 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:04.237 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:21:04.287 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:21:04.287 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:21:04.333 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:21:04.331+0000 7f24a4e76600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:04.334 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:21:04.332+0000 7f24a4e76600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:04.357 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:21:04.358 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:21:04.358 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:21:04.358 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:21:04.359 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:21:04.359 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:21:04.359 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:21:04.360 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:21:04.361 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:21:04.361 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:21:04.361 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:21:04.361 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:21:04.362 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:21:04.362 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:21:04.362 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:21:04.363 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:21:04.363 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:21:04.363 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:21:04.363 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:21:04.364 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:21:04.364 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:21:04.364 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:21:04.364 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:21:04.365 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:21:04.365 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:21:04.365 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186063.955393_299.1", 2022-09-03T06:21:04.365 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:21:04.366 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:21:04.366 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:21:04.366 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.955393Z", 2022-09-03T06:21:04.366 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:21:04.367 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:21:04.367 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:21:04.367 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:21:04.367 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:21:04.368 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:21:04.368 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:21:04.368 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:21:04.369 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:21:04.369 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:21:04.369 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:21:04.369 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:21:04.370 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:21:04.370 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:21:04.370 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:21:04.370 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:21:04.371 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:21:04.371 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:21:04.371 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:21:04.371 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:21:04.372 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:21:04.372 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:21:04.372 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:21:04.372 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:21:04.373 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:21:04.373 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:21:04.373 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:21:04.373 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:21:04.374 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:21:04.374 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:21:04.374 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:21:04.374 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:21:04.375 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:21:04.375 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:21:04Z" 2022-09-03T06:21:04.375 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:21:04.376 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186063.955393_299.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:21:03.955393Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:21:04Z" } 2022-09-03T06:21:04.377 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186063.955393_299.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:21:03.955393Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:21:04.378 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662186063.955393_299.1', 2: '', 3: ''} 2022-09-03T06:21:04.378 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662186063.955393_299.1', 2: '', 3: ''} 2022-09-03T06:21:04.389 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:21:04.390 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:21:04.390 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:04.391 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:04.392 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:21:04.442 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:21:04.442 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:21:04.483 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:04.482+0000 7fa8c9d64600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:04.484 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:04.483+0000 7fa8c9d64600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:04.503 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:21:04.503 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:21:04.504 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:21:04.504 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:21:04.504 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:21:04.504 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:21:04.505 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.505 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:21:04.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:04.506 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:21:04.506 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:04.507 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:04.507 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:21:04.507 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:04.508 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:21:04.508 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:04.508 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:21:04.508 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.509 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:04.509 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:21:04.510 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:04.510 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:04.510 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186063.890194_294.1", 2022-09-03T06:21:04.510 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:04.511 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:04.511 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:04.511 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.890194Z", 2022-09-03T06:21:04.511 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.512 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:04.512 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:21:04.513 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:04.513 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:04.513 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:21:04.514 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:04.514 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:04.514 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:04.514 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:21:04.515 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.515 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:04.516 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:21:04.516 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:04.516 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:04.516 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:21:04.517 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:04.517 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:04.517 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:04.517 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:21:04.518 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:04.518 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.518 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:04.518 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:04.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.519 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:21:04.519 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:21:04.519 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:21:04.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:04.520 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:21:04Z" 2022-09-03T06:21:04.520 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:21:04.521 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186063.890194_294.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:21:03.890194Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:21:04Z" } 2022-09-03T06:21:04.523 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186063.890194_294.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:21:03.890194Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:21:04.523 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662186063.955393_299.1', 2: '', 3: ''} 2022-09-03T06:21:04.524 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662186063.890194_294.1', 2: '', 3: ''} 2022-09-03T06:21:04.524 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662186063.955393_299.1 target=1_1662186063.890194_294.1 2022-09-03T06:21:09.526 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:09.594 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:21:09.595 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:21:09.655 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:21:09.655 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:21:09.711 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:09.709+0000 7f621b40a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:09.713 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:09.711+0000 7f621b40a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:09.738 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:21:09.739 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:21:09.739 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:21:09.739 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:21:09.739 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:21:09.740 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:21:09.741 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:09.741 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:09.742 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:21:09.742 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:09.743 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:21:09.743 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:09.744 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:09.744 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:21:09.745 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:09.745 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:21:09.746 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:09.746 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:21:09.746 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:09.746 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:09.747 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:09.747 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:09.747 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:21:09.748 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:09.748 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:09.748 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186063.955393_299.1", 2022-09-03T06:21:09.749 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:09.749 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:09.750 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:09.750 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.955393Z", 2022-09-03T06:21:09.750 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:09.750 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:09.751 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:09.751 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:09.751 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:21:09.751 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:09.752 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:09.752 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:21:09.752 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:09.753 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:09.753 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:09.753 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:21:09.753 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:09.754 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:09.754 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:09.754 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:09.755 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:21:09.755 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:09.756 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:21:09.756 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:21:09.756 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:21:09.756 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:21:09.757 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:21:09.757 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:21:09.757 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:21:09.758 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:09.758 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:09.758 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:09.758 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:09.759 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:21:09.759 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:21:09.759 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:21:09.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:09.760 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:21:09Z" 2022-09-03T06:21:09.760 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:21:09.761 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186063.955393_299.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:21:03.955393Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:21:09Z" } 2022-09-03T06:21:09.762 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186063.955393_299.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:21:03.955393Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:21:09.763 DEBUG:rgw_multi.tests:log_status={0: '1_1662184350.715807_109.1', 1: '1_1662186063.955393_299.1', 2: '', 3: ''} 2022-09-03T06:21:09.763 DEBUG:rgw_multi.tests:sync_status={0: '1_1662184350.715807_109.1', 1: '1_1662186063.955393_299.1', 2: '', 3: ''} 2022-09-03T06:21:09.763 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:21:09.764 DEBUG:tasks.rgw.c1.client.1:waiting for process to exit 2022-09-03T06:21:09.765 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T06:21:09.766 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:21:09.762+0000 7fe94bc88700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm (PID: 113086) UID: 0 2022-09-03T06:21:09.766 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:21:09.762+0000 7fe94fef3600 -1 shutting down 2022-09-03T06:21:09.908 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:21:09.905+0000 7f50767ae700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5) 2022-09-03T06:21:13.933 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~0s 2022-09-03T06:21:15.868 INFO:tasks.rgw.c1.client.1:Stopped 2022-09-03T06:21:15.868 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--master', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:21:15.869 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--master', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:21:15.870 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone modify --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --master --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:21:15.926 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:21:15.927 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:21:15.950 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:15.948+0000 7eff2de1f600 0 NOTICE: overriding master zone: fca49d30-abba-4c2b-8462-66395ac063ba 2022-09-03T06:21:15.951 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:15.948+0000 7eff2de1f600 0 failed reading obj info from .rgw.root:zone_info.00aa9b6d-7a02-4715-9200-8a26b45eb25e: (2) No such file or directory 2022-09-03T06:21:15.951 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:15.948+0000 7eff2de1f600 0 WARNING: could not read zone params for zone id=00aa9b6d-7a02-4715-9200-8a26b45eb25e name=test-zone3 2022-09-03T06:21:15.952 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:15.951+0000 7eff2de1f600 0 failed reading obj info from .rgw.root:zone_info.fca49d30-abba-4c2b-8462-66395ac063ba: (2) No such file or directory 2022-09-03T06:21:15.953 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:15.951+0000 7eff2de1f600 0 WARNING: could not read zone params for zone id=fca49d30-abba-4c2b-8462-66395ac063ba name=test-zone1 2022-09-03T06:21:15.959 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:21:15.959 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:21:15.959 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:21:15.960 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "test-zone2.rgw.meta:root", 2022-09-03T06:21:15.960 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "test-zone2.rgw.control", 2022-09-03T06:21:15.960 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2022-09-03T06:21:15.960 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2022-09-03T06:21:15.961 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "test-zone2.rgw.log", 2022-09-03T06:21:15.961 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2022-09-03T06:21:15.961 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2022-09-03T06:21:15.961 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2022-09-03T06:21:15.962 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2022-09-03T06:21:15.962 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2022-09-03T06:21:15.962 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2022-09-03T06:21:15.962 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2022-09-03T06:21:15.963 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2022-09-03T06:21:15.963 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "test-zone2.rgw.otp", 2022-09-03T06:21:15.963 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T06:21:15.963 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T06:21:15.964 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T06:21:15.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:15.964 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T06:21:15.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:15.965 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T06:21:15.965 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:21:15.966 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2022-09-03T06:21:15.966 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T06:21:15.967 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T06:21:15.967 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "test-zone2.rgw.buckets.data", 2022-09-03T06:21:15.968 INFO:teuthology.orchestra.run.smithi087.stdout: "compression_type": "random" 2022-09-03T06:21:15.968 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:15.968 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:15.969 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2022-09-03T06:21:15.969 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T06:21:15.969 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:15.969 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:15.970 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:15.970 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:21:15.970 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2022-09-03T06:21:15.970 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:21:15.972 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:21:15.972 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:21:15.973 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:21:16.023 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:21:16.024 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:21:16.088 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:16.085+0000 7fd955070600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:16.088 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:21:16.087+0000 7fd955070600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:21:16.100 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:21:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:21:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 1, 2022-09-03T06:21:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "predecessor_uuid": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:21:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": [ 2022-09-03T06:21:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "1_1662184350.715807_109.1", 2022-09-03T06:21:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "1_1662186063.955393_299.1", 2022-09-03T06:21:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:21:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "" 2022-09-03T06:21:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "period_map": { 2022-09-03T06:21:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:21:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "zonegroups": [ 2022-09-03T06:21:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "test-zonegroup", 2022-09-03T06:21:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zonegroup", 2022-09-03T06:21:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "", 2022-09-03T06:21:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": true, 2022-09-03T06:21:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:21:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:21:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2022-09-03T06:21:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2022-09-03T06:21:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:21:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [ 2022-09-03T06:21:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:21:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone3", 2022-09-03T06:21:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:21:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:21:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:21:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:21:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:21:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:21:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:21:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:21:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:21:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:21:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:21:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:21:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:21:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone4", 2022-09-03T06:21:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:21:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:21:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:21:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:21:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:21:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:21:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "pubsub", 2022-09-03T06:21:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": false, 2022-09-03T06:21:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [ 2022-09-03T06:21:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "test-zone1" 2022-09-03T06:21:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:21:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:21:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:21:16.124 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:16.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:21:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:21:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:21:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:21:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:21:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:21:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:21:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:21:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:21:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:21:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:21:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:21:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:21:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:21:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:21:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone1", 2022-09-03T06:21:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:21:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:21:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:21:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:21:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:21:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:21:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:21:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:21:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:21:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:21:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:21:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:21:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [ 2022-09-03T06:21:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default-placement", 2022-09-03T06:21:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "tags": [], 2022-09-03T06:21:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": [ 2022-09-03T06:21:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD" 2022-09-03T06:21:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "default-placement", 2022-09-03T06:21:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:21:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2022-09-03T06:21:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2022-09-03T06:21:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled_features": [ 2022-09-03T06:21:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:21:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:21:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "short_zone_ids": [ 2022-09-03T06:21:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:21:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 700722763 2022-09-03T06:21:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:21:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 2398203312 2022-09-03T06:21:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:21:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 3737622162 2022-09-03T06:21:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:21:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:21:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 449119331 2022-09-03T06:21:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:21:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:21:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:21:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "period_config": { 2022-09-03T06:21:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_quota": { 2022-09-03T06:21:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:21:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:21:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:21:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:21:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:21:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "user_quota": { 2022-09-03T06:21:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:21:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:21:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:21:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:21:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:21:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "user_ratelimit": { 2022-09-03T06:21:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:21:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:21:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:21:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:21:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:21:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_ratelimit": { 2022-09-03T06:21:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:21:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:21:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:21:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:21:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:21:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "anonymous_ratelimit": { 2022-09-03T06:21:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:21:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:21:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:21:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:21:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:21:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:21:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:21:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:21:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_name": "test-realm", 2022-09-03T06:21:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 3 2022-09-03T06:21:16.158 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:21:16.159 INFO:rgw_multi.tests:Set master zone=test-zone2, waiting 60s for reconfiguration.. 2022-09-03T06:21:19.539 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 has been restored 2022-09-03T06:22:16.161 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-42 2022-09-03T06:22:16.162 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:22:16.163 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:22:16.163 DEBUG:boto:Method: PUT 2022-09-03T06:22:16.164 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:22:16.165 DEBUG:boto:Data: 2022-09-03T06:22:16.165 DEBUG:boto:Headers: {} 2022-09-03T06:22:16.166 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:22:16.166 DEBUG:boto:Port: 8000 2022-09-03T06:22:16.167 DEBUG:boto:Params: {} 2022-09-03T06:22:16.168 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:22:16.169 DEBUG:boto:Token: None 2022-09-03T06:22:16.170 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:22:16 GMT /avysmw-42/ 2022-09-03T06:22:16.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rDg/gIeFf0aekvHTBfT11UZSaBY= 2022-09-03T06:22:16.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:22:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rDg/gIeFf0aekvHTBfT11UZSaBY=', 'Content-Length': '0'} 2022-09-03T06:22:16.208 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000009e22aa86f34acdc5-006312f298-1428-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:22:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:22:16.209 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-43 2022-09-03T06:22:16.209 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:22:16.210 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:22:16.210 DEBUG:boto:Method: PUT 2022-09-03T06:22:16.211 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:22:16.211 DEBUG:boto:Data: 2022-09-03T06:22:16.212 DEBUG:boto:Headers: {} 2022-09-03T06:22:16.212 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:22:16.213 DEBUG:boto:Port: 8000 2022-09-03T06:22:16.214 DEBUG:boto:Params: {} 2022-09-03T06:22:16.214 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:22:16.215 DEBUG:boto:Token: None 2022-09-03T06:22:16.216 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:22:16 GMT /avysmw-43/ 2022-09-03T06:22:16.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4tnb8nQfn80DUDWG+9m06vexJE= 2022-09-03T06:22:16.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:22:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4tnb8nQfn80DUDWG+9m06vexJE=', 'Content-Length': '0'} 2022-09-03T06:22:16.242 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000f07b0e55fbebefaa-006312f298-1586-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:22:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:22:16.243 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:22:16.244 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:22:16.245 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:22:16.298 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:22:16.298 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:22:16.355 INFO:teuthology.orchestra.run.smithi087.stderr:No --period given, using current period=6a921abe-6ddc-4d12-8acc-a1151b4a5f12 2022-09-03T06:22:16.377 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:22:16.377 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:22:16.378 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:22:16.378 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "0.000000" 2022-09-03T06:22:16.378 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:22:16.379 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:22:16.379 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186136.239783_174.1", 2022-09-03T06:22:16.379 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:22:16.239783Z" 2022-09-03T06:22:16.379 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:22:16.380 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:22:16.380 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:22:16.380 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "0.000000" 2022-09-03T06:22:16.380 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:22:16.381 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:22:16.381 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:22:16.381 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "0.000000" 2022-09-03T06:22:16.381 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:22:16.382 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:22:16.383 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662186136.239783_174.1', 2: '', 3: ''} 2022-09-03T06:22:16.384 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone1 2022-09-03T06:22:16.384 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:22:16.385 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:22:16.385 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:22:16.444 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:22:16.445 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:22:16.489 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:16.486+0000 7f15f7e28600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:22:16.490 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:16.488+0000 7f15f7e28600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:22:16.507 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:22:16.508 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:22:16.508 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:22:16.508 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:22:16.509 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:22:16.509 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:22:16.509 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 3 2022-09-03T06:22:16.509 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:16.510 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:22:16.510 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:16.510 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:22:16.510 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:22:16.511 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 0, 2022-09-03T06:22:16.511 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "bucket.instance:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:22:16.513 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:22:16.513 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 81, 2022-09-03T06:22:16.513 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 7, 2022-09-03T06:22:16.513 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:22:16.514 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 0 2022-09-03T06:22:16.514 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:16.514 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:16.515 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:16.515 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:22:16.515 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:22:16.515 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:22:16.516 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186136.184096_167.1", 2022-09-03T06:22:16.516 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:22:16.516 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:22:16.522 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:22:16.522 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.184096Z", 2022-09-03T06:22:16.523 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 3 2022-09-03T06:22:16.523 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:16.523 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:16.523 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:16.524 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:22:16.524 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:22:16.525 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:22:16.525 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:22:16.526 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:22:16.526 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:22:16.526 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:22:16.527 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:22:16.527 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 3 2022-09-03T06:22:16.527 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:16.528 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:16.528 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:16.528 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:22:16.528 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:22:16.529 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:22:16.529 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:22:16.529 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:22:16.529 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:22:16.530 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:22:16.530 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:22:16.531 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 3 2022-09-03T06:22:16.531 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:16.531 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:16.531 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:22:16.532 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:16.532 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:22:16.532 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 81, 2022-09-03T06:22:16.533 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 7 2022-09-03T06:22:16.533 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:16.533 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:22:16Z" 2022-09-03T06:22:16.533 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:22:16.535 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", "realm_epoch": 3 }, "markers": [ { "key": 0, "val": { "state": 0, "marker": "bucket.instance:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", "next_step_marker": "", "total_entries": 81, "pos": 7, "timestamp": "0.000000", "realm_epoch": 0 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186136.184096_167.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:22:16.184096Z", "realm_epoch": 3 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 3 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 3 } } ] }, "full_sync": { "total": 81, "complete": 7 }, "current_time": "2022-09-03T06:22:16Z" } 2022-09-03T06:22:16.535 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 0, 'marker': 'bucket.instance:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16', 'next_step_marker': '', 'total_entries': 81, 'pos': 7, 'timestamp': '0.000000', 'realm_epoch': 0}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186136.184096_167.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:22:16.184096Z', 'realm_epoch': 3}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 3}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 3}}] 2022-09-03T06:22:16.536 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186136.239783_174.1', 2: '', 3: ''} 2022-09-03T06:22:16.536 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186136.184096_167.1', 2: '', 3: ''} 2022-09-03T06:22:16.537 WARNING:rgw_multi.tests:zone test-zone1 behind master: shard=1 master=1_1662186136.239783_174.1 target=1_1662186136.184096_167.1 2022-09-03T06:22:21.538 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:22:21.539 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:22:21.540 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:22:21.599 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:22:21.599 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:22:21.638 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:21.636+0000 7f2c9b48d600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:22:21.640 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:21.637+0000 7f2c9b48d600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:22:21.658 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:22:21.658 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:22:21.658 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:22:21.659 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:22:21.659 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:22:21.659 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:22:21.659 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 3 2022-09-03T06:22:21.660 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.660 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:22:21.660 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.660 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:22:21.661 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:22:21.661 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 0, 2022-09-03T06:22:21.661 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "bucket.instance:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:22:21.661 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:22:21.662 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 81, 2022-09-03T06:22:21.662 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 7, 2022-09-03T06:22:21.662 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:22:21.663 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 0 2022-09-03T06:22:21.663 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.663 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.663 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.664 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:22:21.664 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:22:21.664 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:22:21.664 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186136.239783_174.1", 2022-09-03T06:22:21.665 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:22:21.665 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:22:21.665 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:22:21.665 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.239783Z", 2022-09-03T06:22:21.666 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 3 2022-09-03T06:22:21.666 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.666 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.666 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.667 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:22:21.667 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:22:21.667 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:22:21.667 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:22:21.668 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:22:21.668 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:22:21.668 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:22:21.669 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:22:21.669 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 3 2022-09-03T06:22:21.669 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.669 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.670 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.670 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:22:21.670 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:22:21.670 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:22:21.671 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:22:21.671 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:22:21.671 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:22:21.671 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:22:21.672 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:22:21.672 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 3 2022-09-03T06:22:21.672 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.673 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.673 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:22:21.673 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.674 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:22:21.674 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 81, 2022-09-03T06:22:21.674 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 7 2022-09-03T06:22:21.674 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.675 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:22:21Z" 2022-09-03T06:22:21.675 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:22:21.676 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", "realm_epoch": 3 }, "markers": [ { "key": 0, "val": { "state": 0, "marker": "bucket.instance:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", "next_step_marker": "", "total_entries": 81, "pos": 7, "timestamp": "0.000000", "realm_epoch": 0 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186136.239783_174.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:22:16.239783Z", "realm_epoch": 3 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 3 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 3 } } ] }, "full_sync": { "total": 81, "complete": 7 }, "current_time": "2022-09-03T06:22:21Z" } 2022-09-03T06:22:21.677 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 0, 'marker': 'bucket.instance:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16', 'next_step_marker': '', 'total_entries': 81, 'pos': 7, 'timestamp': '0.000000', 'realm_epoch': 0}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186136.239783_174.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:22:16.239783Z', 'realm_epoch': 3}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 3}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 3}}] 2022-09-03T06:22:21.678 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186136.239783_174.1', 2: '', 3: ''} 2022-09-03T06:22:21.678 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186136.239783_174.1', 2: '', 3: ''} 2022-09-03T06:22:21.679 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone1 2022-09-03T06:22:21.680 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'modify', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--master', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:22:21.680 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'modify', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--master', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:22:21.681 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone modify --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --master --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:22:21.732 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:22:21.732 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:22:21.753 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:21.751+0000 7ff34e7bd600 0 NOTICE: overriding master zone: 9b3b3b7d-a26b-43ae-9603-22cc8f947364 2022-09-03T06:22:21.754 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:21.751+0000 7ff34e7bd600 0 failed reading obj info from .rgw.root:zone_info.31b17c08-52e2-4c88-9aba-17b28ca757e4: (2) No such file or directory 2022-09-03T06:22:21.755 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:21.751+0000 7ff34e7bd600 0 WARNING: could not read zone params for zone id=31b17c08-52e2-4c88-9aba-17b28ca757e4 name=test-zone4 2022-09-03T06:22:21.755 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:21.751+0000 7ff34e7bd600 0 failed reading obj info from .rgw.root:zone_info.9b3b3b7d-a26b-43ae-9603-22cc8f947364: (2) No such file or directory 2022-09-03T06:22:21.755 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:21.751+0000 7ff34e7bd600 0 WARNING: could not read zone params for zone id=9b3b3b7d-a26b-43ae-9603-22cc8f947364 name=test-zone2 2022-09-03T06:22:21.762 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:22:21.762 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:22:21.763 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T06:22:21.763 INFO:teuthology.orchestra.run.smithi062.stdout: "domain_root": "test-zone1.rgw.meta:root", 2022-09-03T06:22:21.764 INFO:teuthology.orchestra.run.smithi062.stdout: "control_pool": "test-zone1.rgw.control", 2022-09-03T06:22:21.764 INFO:teuthology.orchestra.run.smithi062.stdout: "gc_pool": "test-zone1.rgw.log:gc", 2022-09-03T06:22:21.765 INFO:teuthology.orchestra.run.smithi062.stdout: "lc_pool": "test-zone1.rgw.log:lc", 2022-09-03T06:22:21.765 INFO:teuthology.orchestra.run.smithi062.stdout: "log_pool": "test-zone1.rgw.log", 2022-09-03T06:22:21.765 INFO:teuthology.orchestra.run.smithi062.stdout: "intent_log_pool": "test-zone1.rgw.log:intent", 2022-09-03T06:22:21.766 INFO:teuthology.orchestra.run.smithi062.stdout: "usage_log_pool": "test-zone1.rgw.log:usage", 2022-09-03T06:22:21.766 INFO:teuthology.orchestra.run.smithi062.stdout: "roles_pool": "test-zone1.rgw.meta:roles", 2022-09-03T06:22:21.767 INFO:teuthology.orchestra.run.smithi062.stdout: "reshard_pool": "test-zone1.rgw.log:reshard", 2022-09-03T06:22:21.767 INFO:teuthology.orchestra.run.smithi062.stdout: "user_keys_pool": "test-zone1.rgw.meta:users.keys", 2022-09-03T06:22:21.768 INFO:teuthology.orchestra.run.smithi062.stdout: "user_email_pool": "test-zone1.rgw.meta:users.email", 2022-09-03T06:22:21.768 INFO:teuthology.orchestra.run.smithi062.stdout: "user_swift_pool": "test-zone1.rgw.meta:users.swift", 2022-09-03T06:22:21.769 INFO:teuthology.orchestra.run.smithi062.stdout: "user_uid_pool": "test-zone1.rgw.meta:users.uid", 2022-09-03T06:22:21.769 INFO:teuthology.orchestra.run.smithi062.stdout: "otp_pool": "test-zone1.rgw.otp", 2022-09-03T06:22:21.770 INFO:teuthology.orchestra.run.smithi062.stdout: "system_key": { 2022-09-03T06:22:21.770 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T06:22:21.771 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T06:22:21.771 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.772 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_pools": [ 2022-09-03T06:22:21.772 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.773 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "default-placement", 2022-09-03T06:22:21.773 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:22:21.774 INFO:teuthology.orchestra.run.smithi062.stdout: "index_pool": "test-zone1.rgw.buckets.index", 2022-09-03T06:22:21.774 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": { 2022-09-03T06:22:21.775 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD": { 2022-09-03T06:22:21.775 INFO:teuthology.orchestra.run.smithi062.stdout: "data_pool": "test-zone1.rgw.buckets.data", 2022-09-03T06:22:21.776 INFO:teuthology.orchestra.run.smithi062.stdout: "compression_type": "random" 2022-09-03T06:22:21.776 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.776 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.776 INFO:teuthology.orchestra.run.smithi062.stdout: "data_extra_pool": "test-zone1.rgw.buckets.non-ec", 2022-09-03T06:22:21.777 INFO:teuthology.orchestra.run.smithi062.stdout: "index_type": 0 2022-09-03T06:22:21.777 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.777 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.777 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.778 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:22:21.778 INFO:teuthology.orchestra.run.smithi062.stdout: "notif_pool": "test-zone1.rgw.log:notif" 2022-09-03T06:22:21.778 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:22:21.780 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:22:21.781 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:22:21.781 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:22:21.831 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:22:21.832 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:22:21.886 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:21.881+0000 7f7bedd12600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:22:21.886 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:22:21.882+0000 7f7bedd12600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:22:21.901 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:22:21.901 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:22:21.901 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 1, 2022-09-03T06:22:21.902 INFO:teuthology.orchestra.run.smithi062.stdout: "predecessor_uuid": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:22:21.902 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": [ 2022-09-03T06:22:21.903 INFO:teuthology.orchestra.run.smithi062.stdout: "", 2022-09-03T06:22:21.903 INFO:teuthology.orchestra.run.smithi062.stdout: "1_1662186136.239783_174.1", 2022-09-03T06:22:21.904 INFO:teuthology.orchestra.run.smithi062.stdout: "", 2022-09-03T06:22:21.904 INFO:teuthology.orchestra.run.smithi062.stdout: "" 2022-09-03T06:22:21.904 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.904 INFO:teuthology.orchestra.run.smithi062.stdout: "period_map": { 2022-09-03T06:22:21.905 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:22:21.905 INFO:teuthology.orchestra.run.smithi062.stdout: "zonegroups": [ 2022-09-03T06:22:21.906 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.906 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "test-zonegroup", 2022-09-03T06:22:21.906 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zonegroup", 2022-09-03T06:22:21.906 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "", 2022-09-03T06:22:21.907 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": true, 2022-09-03T06:22:21.907 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:22:21.907 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:22:21.907 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.908 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T06:22:21.908 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T06:22:21.908 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:22:21.909 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [ 2022-09-03T06:22:21.909 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.910 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:22:21.910 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone3", 2022-09-03T06:22:21.910 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:22:21.910 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:22:21.911 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.911 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:22:21.911 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:22:21.911 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:22:21.912 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:22:21.912 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:22:21.912 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:22:21.912 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:22:21.913 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:22:21.913 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:22:21.913 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:22:21.914 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:22:21.914 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.914 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.914 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:22:21.915 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone4", 2022-09-03T06:22:21.915 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:22:21.915 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:22:21.916 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.916 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:22:21.917 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:22:21.918 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:22:21.918 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:22:21.918 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "pubsub", 2022-09-03T06:22:21.919 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": false, 2022-09-03T06:22:21.919 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [ 2022-09-03T06:22:21.919 INFO:teuthology.orchestra.run.smithi062.stdout: "test-zone1" 2022-09-03T06:22:21.920 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.920 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:22:21.920 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:22:21.920 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:22:21.921 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:22:21.921 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.921 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.921 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:22:21.922 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone2", 2022-09-03T06:22:21.922 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:22:21.922 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:22:21.923 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.923 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:22:21.923 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:22:21.924 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:22:21.924 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:22:21.924 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:22:21.924 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:22:21.925 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:22:21.925 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:22:21.925 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:22:21.926 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:22:21.926 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:22:21.927 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.927 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.927 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:22:21.927 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T06:22:21.928 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:22:21.928 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:22:21.929 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.929 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:22:21.929 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:22:21.929 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:22:21.930 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:22:21.930 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:22:21.931 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:22:21.931 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:22:21.931 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:22:21.931 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:22:21.932 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:22:21.932 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:22:21.932 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.932 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.933 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [ 2022-09-03T06:22:21.933 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.933 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default-placement", 2022-09-03T06:22:21.935 INFO:teuthology.orchestra.run.smithi062.stdout: "tags": [], 2022-09-03T06:22:21.936 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": [ 2022-09-03T06:22:21.936 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD" 2022-09-03T06:22:21.937 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:22:21.937 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.938 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.938 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "default-placement", 2022-09-03T06:22:21.939 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:22:21.939 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T06:22:21.940 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T06:22:21.940 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.941 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T06:22:21.941 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:22:21.942 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:22:21.942 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.943 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:22:21.943 INFO:teuthology.orchestra.run.smithi062.stdout: "short_zone_ids": [ 2022-09-03T06:22:21.944 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.944 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:22:21.945 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 700722763 2022-09-03T06:22:21.945 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.946 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.946 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:22:21.947 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 2398203312 2022-09-03T06:22:21.947 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.948 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.948 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:22:21.949 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 3737622162 2022-09-03T06:22:21.949 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.950 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:22:21.951 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:22:21.952 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 449119331 2022-09-03T06:22:21.952 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.953 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:22:21.954 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.955 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:22:21.956 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:22:21.957 INFO:teuthology.orchestra.run.smithi062.stdout: "period_config": { 2022-09-03T06:22:21.958 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_quota": { 2022-09-03T06:22:21.959 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T06:22:21.960 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T06:22:21.961 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T06:22:21.963 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T06:22:21.965 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T06:22:21.965 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.966 INFO:teuthology.orchestra.run.smithi062.stdout: "user_quota": { 2022-09-03T06:22:21.966 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T06:22:21.967 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T06:22:21.968 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T06:22:21.969 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T06:22:21.970 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T06:22:21.971 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.971 INFO:teuthology.orchestra.run.smithi062.stdout: "user_ratelimit": { 2022-09-03T06:22:21.972 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:22:21.972 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:22:21.974 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:22:21.974 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:22:21.976 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:22:21.976 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.977 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_ratelimit": { 2022-09-03T06:22:21.977 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:22:21.978 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:22:21.978 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:22:21.979 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:22:21.980 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:22:21.980 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.981 INFO:teuthology.orchestra.run.smithi062.stdout: "anonymous_ratelimit": { 2022-09-03T06:22:21.982 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:22:21.982 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:22:21.983 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:22:21.983 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:22:21.984 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:22:21.985 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:22:21.985 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:22:21.986 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:22:21.987 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_name": "test-realm", 2022-09-03T06:22:21.987 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:22:21.988 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:22:21.989 INFO:rgw_multi.tests:Set master zone=test-zone1, waiting 60s for reconfiguration.. 2022-09-03T06:23:21.991 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-44 2022-09-03T06:23:21.995 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:21.995 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:21.996 DEBUG:boto:Method: PUT 2022-09-03T06:23:21.997 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:21.998 DEBUG:boto:Data: 2022-09-03T06:23:21.999 DEBUG:boto:Headers: {} 2022-09-03T06:23:21.999 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:22.000 DEBUG:boto:Port: 8000 2022-09-03T06:23:22.000 DEBUG:boto:Params: {} 2022-09-03T06:23:22.001 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:23:22.002 DEBUG:boto:Token: None 2022-09-03T06:23:22.003 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:23:22 GMT /avysmw-44/ 2022-09-03T06:23:22.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t8qLfWk+a+S5wrg3VHJTgZNoWak= 2022-09-03T06:23:22.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t8qLfWk+a+S5wrg3VHJTgZNoWak=', 'Content-Length': '0'} 2022-09-03T06:23:22.036 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c282ea4237bad515-006312f2da-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:22.038 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-45 2022-09-03T06:23:22.039 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:22.039 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:22.040 DEBUG:boto:Method: PUT 2022-09-03T06:23:22.041 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:22.042 DEBUG:boto:Data: 2022-09-03T06:23:22.043 DEBUG:boto:Headers: {} 2022-09-03T06:23:22.044 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:22.045 DEBUG:boto:Port: 8000 2022-09-03T06:23:22.046 DEBUG:boto:Params: {} 2022-09-03T06:23:22.048 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:23:22.049 DEBUG:boto:Token: None 2022-09-03T06:23:22.050 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:23:22 GMT /avysmw-45/ 2022-09-03T06:23:22.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:obpnEhRC0mgxUmTKuvFDE0gQDt8= 2022-09-03T06:23:22.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:obpnEhRC0mgxUmTKuvFDE0gQDt8=', 'Content-Length': '0'} 2022-09-03T06:23:22.097 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000001f636e7a744700c1-006312f2da-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:22.098 INFO:tasks.rgw.c1.client.1:Restarting daemon 2022-09-03T06:23:22.099 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c1.client.1.stdout 2>&1 2022-09-03T06:23:22.103 INFO:tasks.rgw.c1.client.1:Started 2022-09-03T06:23:22.104 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T06:23:22.132 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T06:23:22.132 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T06:23:22.132 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T06:23:23.134 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:23.161 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T06:23:23.161 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T06:23:23.162 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 208k 0 --:--:-- --:--:-- --:--:-- 208k 2022-09-03T06:23:23.163 INFO:teuthology.orchestra.run.smithi062.stdout:anonymous 2022-09-03T06:23:23.163 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:23.164 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:23.165 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:23.254 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:23.255 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:23.296 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:23:23.313 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:23.314 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:23.314 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:23.314 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:23:23.315 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:23.315 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:23.315 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186202.079851_158.1", 2022-09-03T06:23:23.315 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:22.079851Z" 2022-09-03T06:23:23.316 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:23.316 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:23.316 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:23.316 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:23:23.317 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:23.317 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:23.317 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:23.317 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:23:23.318 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:23.318 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:23.319 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662186202.079851_158.1', 2: '', 3: ''} 2022-09-03T06:23:23.320 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:23:23.320 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:23.321 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:23.322 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:23.383 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:23.383 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:23.445 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:23.444+0000 7f8f13524600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:23.446 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:23.445+0000 7f8f13524600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:23.470 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:23:23.471 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:23:23.471 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:23:23.472 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:23:23.472 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:23:23.473 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", 2022-09-03T06:23:23.473 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:23.474 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:23.474 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:23:23.475 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:23.475 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:23:23.476 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:23.476 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:23.477 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:23:23.477 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:23.478 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:23:23.478 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:23.479 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:23:23.479 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:23.480 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:23.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:23.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:23.481 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:23:23.481 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:23.482 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:23.482 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186063.955393_299.1", 2022-09-03T06:23:23.483 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:23.483 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:23.484 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:23.485 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.955393Z", 2022-09-03T06:23:23.485 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:23.485 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:23.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:23.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:23.487 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:23:23.487 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:23.488 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:23.488 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:23.489 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:23.489 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:23.490 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:23.490 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:23.491 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:23.491 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:23.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:23.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:23.493 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:23:23.494 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:23.494 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:23.495 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:23.495 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:23.496 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:23.496 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:23.497 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:23.497 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:23.498 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:23.498 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:23.499 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:23:23.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:23.500 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:23:23.501 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:23:23.501 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:23:23.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:23.502 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:23:23Z" 2022-09-03T06:23:23.503 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:23:23.505 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "b84ccf39-78f5-4998-9db6-42362bc8c98b", "realm_epoch": 2 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186063.955393_299.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:21:03.955393Z", "realm_epoch": 2 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:23:23Z" } 2022-09-03T06:23:23.506 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186063.955393_299.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:21:03.955393Z', 'realm_epoch': 2}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:23:23.507 WARNING:rgw_multi.tests:zone test-zone2 is syncing realm epoch=2, behind current realm epoch=4 2022-09-03T06:23:28.508 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:28.509 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:28.511 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:28.570 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:28.571 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:28.604 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:28.602+0000 7fd23579a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:28.605 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:28.603+0000 7fd23579a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:28.626 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:23:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:23:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:23:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:23:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:23:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:23:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:23:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:23:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:23:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:23:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:23:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:23:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186202.079851_158.1", 2022-09-03T06:23:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.079851Z", 2022-09-03T06:23:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.638 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:28.638 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:23:28.638 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:28.638 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:23:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:23:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:23:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:23:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:23:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:23:28Z" 2022-09-03T06:23:28.645 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:23:28.646 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186202.079851_158.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:22.079851Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:23:28Z" } 2022-09-03T06:23:28.647 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186202.079851_158.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:22.079851Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:23:28.647 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186202.079851_158.1', 2: '', 3: ''} 2022-09-03T06:23:28.648 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186202.079851_158.1', 2: '', 3: ''} 2022-09-03T06:23:28.648 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:23:28.649 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:23:28.649 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:28.650 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:28.650 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:28.710 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:28.710 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:28.741 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:23:28.739+0000 7f85e73dc600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:28.742 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:23:28.740+0000 7f85e73dc600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:28.755 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:23:28.756 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:23:28.756 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:23:28.757 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:23:28.757 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:23:28.757 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:23:28.757 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:23:28.758 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:28.758 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:23:28.758 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:28.759 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:23:28.759 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:28.759 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:23:28.759 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:23:28.760 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:28.760 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:23:28.760 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:28.760 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:23:28.761 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:23:28.761 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:28.762 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:28.762 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:28.763 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:23:28.763 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:28.764 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:23:28.764 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186202.079851_158.1", 2022-09-03T06:23:28.765 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:28.765 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:28.765 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:28.766 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.079851Z", 2022-09-03T06:23:28.766 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:23:28.767 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:28.767 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:28.768 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:28.768 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:23:28.768 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:28.769 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:23:28.769 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:28.769 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:28.770 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:28.770 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:28.770 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:23:28.770 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:23:28.771 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:28.771 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:28.771 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:28.771 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:23:28.772 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:28.772 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:23:28.773 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:28.773 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:28.773 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:28.774 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:28.774 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:23:28.775 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:23:28.775 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:28.775 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:28.775 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:23:28.776 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:28.776 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:23:28.776 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:23:28.777 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:23:28.777 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:28.777 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:23:28Z" 2022-09-03T06:23:28.778 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:23:28.779 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186202.079851_158.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:22.079851Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:23:28Z" } 2022-09-03T06:23:28.780 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186202.079851_158.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:22.079851Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:23:28.780 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186202.079851_158.1', 2: '', 3: ''} 2022-09-03T06:23:28.781 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186202.079851_158.1', 2: '', 3: ''} 2022-09-03T06:23:28.781 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:23:28.782 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:23:28.783 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:28.783 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:28.784 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:28.835 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:28.836 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:28.886 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:28.884+0000 7f639c434600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:28.887 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:28.885+0000 7f639c434600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:28.905 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:23:28.905 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:23:28.905 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:23:28.906 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:23:28.906 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:23:28.907 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:23:28.907 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:28.907 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.908 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:23:28.909 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:28.910 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:23:28.910 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:28.910 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:28.910 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:23:28.911 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:28.911 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:23:28.911 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:28.912 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:23:28.912 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:28.912 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.912 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.913 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:28.913 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:23:28.913 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:28.914 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:28.914 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186202.079851_158.1", 2022-09-03T06:23:28.914 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:28.914 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:28.915 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:28.915 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.079851Z", 2022-09-03T06:23:28.915 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:28.915 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.916 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.916 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:28.917 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:23:28.917 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:28.917 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:28.918 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:28.919 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:28.919 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:28.920 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:28.920 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:28.920 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:28.921 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.921 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.921 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:28.922 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:23:28.922 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:28.922 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:28.923 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:28.923 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:28.923 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:28.924 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:28.924 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:28.925 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:28.925 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.925 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:28.926 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:23:28.927 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.927 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:23:28.927 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:23:28.927 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:23:28.928 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:28.928 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:23:28Z" 2022-09-03T06:23:28.928 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:23:28.930 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186202.079851_158.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:22.079851Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:23:28Z" } 2022-09-03T06:23:28.930 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186202.079851_158.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:22.079851Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:23:28.931 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186202.079851_158.1', 2: '', 3: ''} 2022-09-03T06:23:28.931 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186202.079851_158.1', 2: '', 3: ''} 2022-09-03T06:23:28.932 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:23:28.932 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-39 zones={test-zone2, test-zone1} 2022-09-03T06:23:28.933 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:28.934 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:28.934 DEBUG:boto:Method: HEAD 2022-09-03T06:23:28.935 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:28.935 DEBUG:boto:Data: 2022-09-03T06:23:28.936 DEBUG:boto:Headers: {} 2022-09-03T06:23:28.936 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:28.937 DEBUG:boto:Port: 8000 2022-09-03T06:23:28.937 DEBUG:boto:Params: {} 2022-09-03T06:23:28.938 DEBUG:boto:Token: None 2022-09-03T06:23:28.939 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:28 GMT /avysmw-39/ 2022-09-03T06:23:28.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B5oUdLv0OLwkQWasEpI5geg5wFk= 2022-09-03T06:23:28.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B5oUdLv0OLwkQWasEpI5geg5wFk=', 'Content-Length': '0'} 2022-09-03T06:23:28.943 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b4e6e800f37a5a09-006312f2e0-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:28.944 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:28.945 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:28.946 DEBUG:boto:Method: HEAD 2022-09-03T06:23:28.947 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:28.948 DEBUG:boto:Data: 2022-09-03T06:23:28.948 DEBUG:boto:Headers: {} 2022-09-03T06:23:28.949 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:28.950 DEBUG:boto:Port: 8000 2022-09-03T06:23:28.951 DEBUG:boto:Params: {} 2022-09-03T06:23:28.952 DEBUG:boto:Token: None 2022-09-03T06:23:28.953 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:28 GMT /avysmw-39/ 2022-09-03T06:23:28.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B5oUdLv0OLwkQWasEpI5geg5wFk= 2022-09-03T06:23:28.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B5oUdLv0OLwkQWasEpI5geg5wFk=', 'Content-Length': '0'} 2022-09-03T06:23:28.959 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002dc39df6d461c0a3-006312f2e0-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:28.960 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:28.960 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:28.961 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:28.962 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:28.962 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:28.963 DEBUG:boto:Method: GET 2022-09-03T06:23:28.964 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:28.964 DEBUG:boto:Data: 2022-09-03T06:23:28.965 DEBUG:boto:Headers: {} 2022-09-03T06:23:28.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:28.966 DEBUG:boto:Port: 8000 2022-09-03T06:23:28.966 DEBUG:boto:Params: {} 2022-09-03T06:23:28.967 DEBUG:boto:Token: None 2022-09-03T06:23:28.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:28 GMT /avysmw-39/?versions 2022-09-03T06:23:28.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y2zfMZsUy7qTtr0TkgSaJwMU7h0= 2022-09-03T06:23:28.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y2zfMZsUy7qTtr0TkgSaJwMU7h0=', 'Content-Length': '0'} 2022-09-03T06:23:28.973 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e007cec04111bb6e-006312f2e0-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:28.975 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:28.976 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:28.977 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:28.977 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:28.978 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:28.978 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:28.979 DEBUG:boto:Method: GET 2022-09-03T06:23:28.979 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:28.980 DEBUG:boto:Data: 2022-09-03T06:23:28.980 DEBUG:boto:Headers: {} 2022-09-03T06:23:28.981 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:28.981 DEBUG:boto:Port: 8000 2022-09-03T06:23:28.982 DEBUG:boto:Params: {} 2022-09-03T06:23:28.982 DEBUG:boto:Token: None 2022-09-03T06:23:28.983 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:28 GMT /avysmw-39/?versions 2022-09-03T06:23:28.984 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y2zfMZsUy7qTtr0TkgSaJwMU7h0= 2022-09-03T06:23:28.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y2zfMZsUy7qTtr0TkgSaJwMU7h0=', 'Content-Length': '0'} 2022-09-03T06:23:28.987 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001aa75c2fd88d92dd-006312f2e0-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:28.987 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:28.988 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:28.989 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:28.990 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:28.991 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:28.992 DEBUG:boto:Method: GET 2022-09-03T06:23:28.993 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:28.994 DEBUG:boto:Data: 2022-09-03T06:23:28.995 DEBUG:boto:Headers: {} 2022-09-03T06:23:28.996 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:28.997 DEBUG:boto:Port: 8000 2022-09-03T06:23:28.998 DEBUG:boto:Params: {} 2022-09-03T06:23:28.999 DEBUG:boto:Token: None 2022-09-03T06:23:29.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.005 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000245779de44a67cbe-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.006 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.008 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.009 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.009 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.010 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.011 DEBUG:boto:Method: GET 2022-09-03T06:23:29.012 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.013 DEBUG:boto:Data: 2022-09-03T06:23:29.014 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.016 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.017 DEBUG:boto:Params: {} 2022-09-03T06:23:29.018 DEBUG:boto:Token: None 2022-09-03T06:23:29.019 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.024 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e3d76819168c9e11-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.025 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.026 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-39 zones={test-zone2, test-zone1} 2022-09-03T06:23:29.027 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-39 zones={test-zone3, test-zone1} 2022-09-03T06:23:29.028 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.028 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.029 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.029 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.030 DEBUG:boto:Data: 2022-09-03T06:23:29.030 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.032 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.033 DEBUG:boto:Params: {} 2022-09-03T06:23:29.034 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:23:29.035 DEBUG:boto:Token: None 2022-09-03T06:23:29.036 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.048 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008854081ee8748d57-006312f2e1-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.050 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.051 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.052 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.053 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.055 DEBUG:boto:Data: 2022-09-03T06:23:29.055 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.057 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.059 DEBUG:boto:Params: {} 2022-09-03T06:23:29.059 DEBUG:boto:Token: None 2022-09-03T06:23:29.060 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.063 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002c14085c7e1270bf-006312f2e1-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.064 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:29.065 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.065 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.066 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.066 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.067 DEBUG:boto:Method: GET 2022-09-03T06:23:29.067 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.068 DEBUG:boto:Data: 2022-09-03T06:23:29.069 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.069 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.070 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.070 DEBUG:boto:Params: {} 2022-09-03T06:23:29.071 DEBUG:boto:Token: None 2022-09-03T06:23:29.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.075 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c35ce7e683895502-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.076 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.077 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:29.077 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.078 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.078 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.079 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.079 DEBUG:boto:Method: GET 2022-09-03T06:23:29.080 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.080 DEBUG:boto:Data: 2022-09-03T06:23:29.081 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.081 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.082 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.083 DEBUG:boto:Params: {} 2022-09-03T06:23:29.083 DEBUG:boto:Token: None 2022-09-03T06:23:29.084 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.088 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d61893a2f6aa8236-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.089 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.089 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.090 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.090 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.091 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.091 DEBUG:boto:Method: GET 2022-09-03T06:23:29.092 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.093 DEBUG:boto:Data: 2022-09-03T06:23:29.093 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.094 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.094 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.095 DEBUG:boto:Params: {} 2022-09-03T06:23:29.096 DEBUG:boto:Token: None 2022-09-03T06:23:29.096 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.100 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b16721358dd0bd1f-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.101 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.102 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.102 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.103 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.103 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.104 DEBUG:boto:Method: GET 2022-09-03T06:23:29.104 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.105 DEBUG:boto:Data: 2022-09-03T06:23:29.105 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.106 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.107 DEBUG:boto:Params: {} 2022-09-03T06:23:29.108 DEBUG:boto:Token: None 2022-09-03T06:23:29.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.112 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000e6cb4db4bf8ea65-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.113 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.113 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-39 zones={test-zone3, test-zone1} 2022-09-03T06:23:29.114 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-39 zones={test-zone1, test-zone2} 2022-09-03T06:23:29.114 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.115 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.116 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.116 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.117 DEBUG:boto:Data: 2022-09-03T06:23:29.117 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.118 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.118 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.119 DEBUG:boto:Params: {} 2022-09-03T06:23:29.119 DEBUG:boto:Token: None 2022-09-03T06:23:29.120 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.123 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000043ef94f03c7929a7-006312f2e1-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.124 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.125 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.125 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.126 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.126 DEBUG:boto:Data: 2022-09-03T06:23:29.127 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.127 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.128 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.128 DEBUG:boto:Params: {} 2022-09-03T06:23:29.129 DEBUG:boto:Token: None 2022-09-03T06:23:29.130 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.134 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e229cf08c50aec13-006312f2e1-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.134 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:29.135 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.136 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.137 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.138 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.139 DEBUG:boto:Method: GET 2022-09-03T06:23:29.140 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.141 DEBUG:boto:Data: 2022-09-03T06:23:29.142 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.144 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.145 DEBUG:boto:Params: {} 2022-09-03T06:23:29.146 DEBUG:boto:Token: None 2022-09-03T06:23:29.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.152 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000046838ff3b4b285cf-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.154 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.155 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:29.156 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.157 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.158 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.159 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.160 DEBUG:boto:Method: GET 2022-09-03T06:23:29.161 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.161 DEBUG:boto:Data: 2022-09-03T06:23:29.162 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.163 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.164 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.165 DEBUG:boto:Params: {} 2022-09-03T06:23:29.166 DEBUG:boto:Token: None 2022-09-03T06:23:29.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.170 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.173 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb28118d3a939aa1-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.174 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.175 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.175 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.176 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.176 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.177 DEBUG:boto:Method: GET 2022-09-03T06:23:29.177 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.178 DEBUG:boto:Data: 2022-09-03T06:23:29.178 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.179 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.179 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.180 DEBUG:boto:Params: {} 2022-09-03T06:23:29.180 DEBUG:boto:Token: None 2022-09-03T06:23:29.181 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.185 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b311845ab9cd982c-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.185 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.186 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.187 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.187 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.188 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.188 DEBUG:boto:Method: GET 2022-09-03T06:23:29.189 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.189 DEBUG:boto:Data: 2022-09-03T06:23:29.190 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.190 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.191 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.191 DEBUG:boto:Params: {} 2022-09-03T06:23:29.192 DEBUG:boto:Token: None 2022-09-03T06:23:29.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.194 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.196 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cc7fe59a8f79d069-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.197 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.198 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-39 zones={test-zone1, test-zone2} 2022-09-03T06:23:29.198 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-39 zones={test-zone3, test-zone2} 2022-09-03T06:23:29.199 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.200 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.201 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.202 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.203 DEBUG:boto:Data: 2022-09-03T06:23:29.203 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.204 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.205 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.205 DEBUG:boto:Params: {} 2022-09-03T06:23:29.206 DEBUG:boto:Token: None 2022-09-03T06:23:29.207 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.212 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000030cb64712f319102-006312f2e1-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.213 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.213 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.214 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.215 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.215 DEBUG:boto:Data: 2022-09-03T06:23:29.216 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.216 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.217 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.217 DEBUG:boto:Params: {} 2022-09-03T06:23:29.218 DEBUG:boto:Token: None 2022-09-03T06:23:29.219 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.222 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f4085c7d51c2c13a-006312f2e1-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.223 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:29.223 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.224 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.224 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.225 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.226 DEBUG:boto:Method: GET 2022-09-03T06:23:29.227 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.227 DEBUG:boto:Data: 2022-09-03T06:23:29.228 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.229 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.229 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.230 DEBUG:boto:Params: {} 2022-09-03T06:23:29.230 DEBUG:boto:Token: None 2022-09-03T06:23:29.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.239 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008743806e243dc015-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.240 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.241 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:29.242 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.242 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.243 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.244 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.244 DEBUG:boto:Method: GET 2022-09-03T06:23:29.245 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.245 DEBUG:boto:Data: 2022-09-03T06:23:29.246 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.247 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.247 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.248 DEBUG:boto:Params: {} 2022-09-03T06:23:29.248 DEBUG:boto:Token: None 2022-09-03T06:23:29.249 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.250 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.253 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007164b0f14da876dd-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.254 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.255 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.256 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.257 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.258 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.259 DEBUG:boto:Method: GET 2022-09-03T06:23:29.260 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.261 DEBUG:boto:Data: 2022-09-03T06:23:29.262 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.264 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.265 DEBUG:boto:Params: {} 2022-09-03T06:23:29.266 DEBUG:boto:Token: None 2022-09-03T06:23:29.267 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.273 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000450133810fb533aa-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.314 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.315 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.316 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.317 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.317 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.318 DEBUG:boto:Method: GET 2022-09-03T06:23:29.319 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.320 DEBUG:boto:Data: 2022-09-03T06:23:29.321 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.322 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.323 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.324 DEBUG:boto:Params: {} 2022-09-03T06:23:29.325 DEBUG:boto:Token: None 2022-09-03T06:23:29.327 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.332 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000012ed14ea4a3c6059-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.334 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.335 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-39 zones={test-zone3, test-zone2} 2022-09-03T06:23:29.336 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-39 zones={test-zone1, test-zone3} 2022-09-03T06:23:29.336 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.337 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.338 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.339 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.340 DEBUG:boto:Data: 2022-09-03T06:23:29.341 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.342 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.343 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.344 DEBUG:boto:Params: {} 2022-09-03T06:23:29.345 DEBUG:boto:Token: None 2022-09-03T06:23:29.346 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.352 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007d31aba91b352428-006312f2e1-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.353 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.354 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.355 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.356 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.357 DEBUG:boto:Data: 2022-09-03T06:23:29.358 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.360 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.361 DEBUG:boto:Params: {} 2022-09-03T06:23:29.362 DEBUG:boto:Token: None 2022-09-03T06:23:29.363 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.368 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000027f7a91fc922ade5-006312f2e1-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.369 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:29.370 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.371 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.372 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.373 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.374 DEBUG:boto:Method: GET 2022-09-03T06:23:29.375 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.376 DEBUG:boto:Data: 2022-09-03T06:23:29.377 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.378 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.379 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.380 DEBUG:boto:Params: {} 2022-09-03T06:23:29.382 DEBUG:boto:Token: None 2022-09-03T06:23:29.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.389 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000804357e089719780-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.390 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.392 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:29.393 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.395 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.396 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.396 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.397 DEBUG:boto:Method: GET 2022-09-03T06:23:29.397 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.398 DEBUG:boto:Data: 2022-09-03T06:23:29.398 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.399 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.399 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.400 DEBUG:boto:Params: {} 2022-09-03T06:23:29.400 DEBUG:boto:Token: None 2022-09-03T06:23:29.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.405 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000df1dca18bd0aab93-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.406 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.407 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.407 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.408 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.408 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.409 DEBUG:boto:Method: GET 2022-09-03T06:23:29.409 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.410 DEBUG:boto:Data: 2022-09-03T06:23:29.411 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.411 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.412 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.412 DEBUG:boto:Params: {} 2022-09-03T06:23:29.413 DEBUG:boto:Token: None 2022-09-03T06:23:29.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.419 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c4ab269cee3eda9a-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.419 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.421 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.422 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.423 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.423 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.424 DEBUG:boto:Method: GET 2022-09-03T06:23:29.425 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.426 DEBUG:boto:Data: 2022-09-03T06:23:29.427 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.429 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.429 DEBUG:boto:Params: {} 2022-09-03T06:23:29.430 DEBUG:boto:Token: None 2022-09-03T06:23:29.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.435 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f02c4c6417f8adf-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.436 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.438 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-39 zones={test-zone1, test-zone3} 2022-09-03T06:23:29.438 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-39 zones={test-zone2, test-zone3} 2022-09-03T06:23:29.439 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.440 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.440 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.441 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.441 DEBUG:boto:Data: 2022-09-03T06:23:29.442 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.442 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.443 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.443 DEBUG:boto:Params: {} 2022-09-03T06:23:29.444 DEBUG:boto:Token: None 2022-09-03T06:23:29.445 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.448 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000010a4cbd0a3eeee7d-006312f2e1-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.449 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.449 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.450 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.450 DEBUG:boto:Path: /avysmw-39/ 2022-09-03T06:23:29.451 DEBUG:boto:Data: 2022-09-03T06:23:29.451 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.452 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.452 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.453 DEBUG:boto:Params: {} 2022-09-03T06:23:29.453 DEBUG:boto:Token: None 2022-09-03T06:23:29.454 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/ 2022-09-03T06:23:29.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs= 2022-09-03T06:23:29.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nIe52gXcspzMJVOX5N9f2D5RRPs=', 'Content-Length': '0'} 2022-09-03T06:23:29.458 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008fa629d40d71f22b-006312f2e1-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.458 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:29.459 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.459 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.460 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.460 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.461 DEBUG:boto:Method: GET 2022-09-03T06:23:29.461 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.462 DEBUG:boto:Data: 2022-09-03T06:23:29.462 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.463 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.464 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.464 DEBUG:boto:Params: {} 2022-09-03T06:23:29.465 DEBUG:boto:Token: None 2022-09-03T06:23:29.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.469 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003687a0c9aa7c939f-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.470 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.470 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:29.471 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.471 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.472 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.472 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.473 DEBUG:boto:Method: GET 2022-09-03T06:23:29.473 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.474 DEBUG:boto:Data: 2022-09-03T06:23:29.475 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.476 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.476 DEBUG:boto:Params: {} 2022-09-03T06:23:29.477 DEBUG:boto:Token: None 2022-09-03T06:23:29.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.481 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cbe9f486a9c4e407-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.482 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.482 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.483 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.483 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.484 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.484 DEBUG:boto:Method: GET 2022-09-03T06:23:29.485 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.486 DEBUG:boto:Data: 2022-09-03T06:23:29.486 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.487 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.487 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.488 DEBUG:boto:Params: {} 2022-09-03T06:23:29.488 DEBUG:boto:Token: None 2022-09-03T06:23:29.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.493 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006e24d2cdae85d11f-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.494 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.494 DEBUG:boto:path=/avysmw-39/ 2022-09-03T06:23:29.495 DEBUG:boto:auth_path=/avysmw-39/ 2022-09-03T06:23:29.495 DEBUG:boto:path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.496 DEBUG:boto:auth_path=/avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.497 DEBUG:boto:Method: GET 2022-09-03T06:23:29.497 DEBUG:boto:Path: /avysmw-39/?versions&max-keys=999 2022-09-03T06:23:29.498 DEBUG:boto:Data: 2022-09-03T06:23:29.498 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.499 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.500 DEBUG:boto:Params: {} 2022-09-03T06:23:29.501 DEBUG:boto:Token: None 2022-09-03T06:23:29.501 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-39/?versions 2022-09-03T06:23:29.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI= 2022-09-03T06:23:29.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dgmeo13D+fLRpxrOAg2fKUSsysI=', 'Content-Length': '0'} 2022-09-03T06:23:29.505 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c8d4cb27f17b900d-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.506 DEBUG:boto:b'avysmw-39999false' 2022-09-03T06:23:29.507 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-39 zones={test-zone2, test-zone3} 2022-09-03T06:23:29.507 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-40 zones={test-zone2, test-zone1} 2022-09-03T06:23:29.508 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.508 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.509 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.510 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:29.510 DEBUG:boto:Data: 2022-09-03T06:23:29.511 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.511 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.512 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.512 DEBUG:boto:Params: {} 2022-09-03T06:23:29.513 DEBUG:boto:Token: None 2022-09-03T06:23:29.514 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/ 2022-09-03T06:23:29.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk= 2022-09-03T06:23:29.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk=', 'Content-Length': '0'} 2022-09-03T06:23:29.518 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f851ea0cb2761f9b-006312f2e1-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.519 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.519 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.520 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.520 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:29.521 DEBUG:boto:Data: 2022-09-03T06:23:29.521 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.522 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.523 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.524 DEBUG:boto:Params: {} 2022-09-03T06:23:29.524 DEBUG:boto:Token: None 2022-09-03T06:23:29.526 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/ 2022-09-03T06:23:29.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk= 2022-09-03T06:23:29.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk=', 'Content-Length': '0'} 2022-09-03T06:23:29.532 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000062b4dfe896de6032-006312f2e1-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.532 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:29.533 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.533 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.534 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.534 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.535 DEBUG:boto:Method: GET 2022-09-03T06:23:29.535 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.536 DEBUG:boto:Data: 2022-09-03T06:23:29.536 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.537 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.538 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.538 DEBUG:boto:Params: {} 2022-09-03T06:23:29.539 DEBUG:boto:Token: None 2022-09-03T06:23:29.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.542 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.545 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002dbb5cac189709d-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.590 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.591 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:29.592 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.592 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.593 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.593 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.594 DEBUG:boto:Method: GET 2022-09-03T06:23:29.594 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.595 DEBUG:boto:Data: 2022-09-03T06:23:29.595 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.596 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.596 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.597 DEBUG:boto:Params: {} 2022-09-03T06:23:29.598 DEBUG:boto:Token: None 2022-09-03T06:23:29.598 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.604 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a9a544529bc4987-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.605 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.606 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.606 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.607 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.607 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.608 DEBUG:boto:Method: GET 2022-09-03T06:23:29.609 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.610 DEBUG:boto:Data: 2022-09-03T06:23:29.611 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.612 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.613 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.614 DEBUG:boto:Params: {} 2022-09-03T06:23:29.614 DEBUG:boto:Token: None 2022-09-03T06:23:29.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.618 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006cf4d17ec5fa0211-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.619 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.620 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.620 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.621 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.622 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.622 DEBUG:boto:Method: GET 2022-09-03T06:23:29.623 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.623 DEBUG:boto:Data: 2022-09-03T06:23:29.624 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.624 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.625 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.625 DEBUG:boto:Params: {} 2022-09-03T06:23:29.626 DEBUG:boto:Token: None 2022-09-03T06:23:29.627 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.631 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000003a0b6fc4a19e9c-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.678 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.679 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-40 zones={test-zone2, test-zone1} 2022-09-03T06:23:29.679 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-40 zones={test-zone3, test-zone1} 2022-09-03T06:23:29.680 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.680 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.681 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.682 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:29.682 DEBUG:boto:Data: 2022-09-03T06:23:29.683 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.684 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.684 DEBUG:boto:Params: {} 2022-09-03T06:23:29.685 DEBUG:boto:Token: None 2022-09-03T06:23:29.685 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/ 2022-09-03T06:23:29.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk= 2022-09-03T06:23:29.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk=', 'Content-Length': '0'} 2022-09-03T06:23:29.690 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000081d97189cdf67efa-006312f2e1-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.692 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.693 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.694 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.695 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:29.696 DEBUG:boto:Data: 2022-09-03T06:23:29.697 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.699 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.700 DEBUG:boto:Params: {} 2022-09-03T06:23:29.701 DEBUG:boto:Token: None 2022-09-03T06:23:29.702 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/ 2022-09-03T06:23:29.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk= 2022-09-03T06:23:29.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk=', 'Content-Length': '0'} 2022-09-03T06:23:29.708 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000fa7a97fb23397443-006312f2e1-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.709 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:29.710 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.711 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.712 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.713 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.714 DEBUG:boto:Method: GET 2022-09-03T06:23:29.715 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.716 DEBUG:boto:Data: 2022-09-03T06:23:29.717 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.719 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.720 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.721 DEBUG:boto:Params: {} 2022-09-03T06:23:29.722 DEBUG:boto:Token: None 2022-09-03T06:23:29.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.725 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.728 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7069792c77ad26b-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.729 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.731 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:29.732 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.733 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.734 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.734 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.735 DEBUG:boto:Method: GET 2022-09-03T06:23:29.736 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.737 DEBUG:boto:Data: 2022-09-03T06:23:29.738 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.740 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.741 DEBUG:boto:Params: {} 2022-09-03T06:23:29.742 DEBUG:boto:Token: None 2022-09-03T06:23:29.743 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.749 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000263cb8dfa028799f-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.750 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.751 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.752 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.753 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.754 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.755 DEBUG:boto:Method: GET 2022-09-03T06:23:29.756 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.757 DEBUG:boto:Data: 2022-09-03T06:23:29.758 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.761 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.762 DEBUG:boto:Params: {} 2022-09-03T06:23:29.763 DEBUG:boto:Token: None 2022-09-03T06:23:29.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.770 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007f61341d0fc5f3ee-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.771 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.772 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.773 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.774 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.775 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.776 DEBUG:boto:Method: GET 2022-09-03T06:23:29.777 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.778 DEBUG:boto:Data: 2022-09-03T06:23:29.779 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.780 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.781 DEBUG:boto:Params: {} 2022-09-03T06:23:29.782 DEBUG:boto:Token: None 2022-09-03T06:23:29.783 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.789 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000829ed5f58968cafd-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.790 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.791 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-40 zones={test-zone3, test-zone1} 2022-09-03T06:23:29.792 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-40 zones={test-zone1, test-zone2} 2022-09-03T06:23:29.793 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.794 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.795 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.796 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:29.797 DEBUG:boto:Data: 2022-09-03T06:23:29.798 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.799 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.800 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.801 DEBUG:boto:Params: {} 2022-09-03T06:23:29.803 DEBUG:boto:Token: None 2022-09-03T06:23:29.804 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/ 2022-09-03T06:23:29.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk= 2022-09-03T06:23:29.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk=', 'Content-Length': '0'} 2022-09-03T06:23:29.809 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ea839719205f3ae9-006312f2e1-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.809 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.810 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.810 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.811 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:29.811 DEBUG:boto:Data: 2022-09-03T06:23:29.812 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.812 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.813 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.813 DEBUG:boto:Params: {} 2022-09-03T06:23:29.814 DEBUG:boto:Token: None 2022-09-03T06:23:29.815 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/ 2022-09-03T06:23:29.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk= 2022-09-03T06:23:29.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk=', 'Content-Length': '0'} 2022-09-03T06:23:29.818 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000057dee8337742da1d-006312f2e1-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.830 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:29.832 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.833 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.834 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.835 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.836 DEBUG:boto:Method: GET 2022-09-03T06:23:29.837 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.839 DEBUG:boto:Data: 2022-09-03T06:23:29.840 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.841 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.842 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.843 DEBUG:boto:Params: {} 2022-09-03T06:23:29.844 DEBUG:boto:Token: None 2022-09-03T06:23:29.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.850 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004fbd47ce6876c594-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.852 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.853 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:29.854 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.854 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.855 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.856 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.856 DEBUG:boto:Method: GET 2022-09-03T06:23:29.857 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.857 DEBUG:boto:Data: 2022-09-03T06:23:29.858 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.859 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.859 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.860 DEBUG:boto:Params: {} 2022-09-03T06:23:29.860 DEBUG:boto:Token: None 2022-09-03T06:23:29.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.864 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028a56a07e5136b4d-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.865 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.866 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.866 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.867 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.867 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.868 DEBUG:boto:Method: GET 2022-09-03T06:23:29.868 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.869 DEBUG:boto:Data: 2022-09-03T06:23:29.869 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.870 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:29.870 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.871 DEBUG:boto:Params: {} 2022-09-03T06:23:29.871 DEBUG:boto:Token: None 2022-09-03T06:23:29.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.876 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001bd5400e2b6c4de4-006312f2e1-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.918 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.919 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.920 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.920 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.921 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.921 DEBUG:boto:Method: GET 2022-09-03T06:23:29.922 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.922 DEBUG:boto:Data: 2022-09-03T06:23:29.923 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.923 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.924 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.924 DEBUG:boto:Params: {} 2022-09-03T06:23:29.925 DEBUG:boto:Token: None 2022-09-03T06:23:29.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.929 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc5be0277646d6e8-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.930 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.930 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-40 zones={test-zone1, test-zone2} 2022-09-03T06:23:29.931 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-40 zones={test-zone3, test-zone2} 2022-09-03T06:23:29.931 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.932 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.932 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.933 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:29.933 DEBUG:boto:Data: 2022-09-03T06:23:29.934 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.935 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.935 DEBUG:boto:Params: {} 2022-09-03T06:23:29.936 DEBUG:boto:Token: None 2022-09-03T06:23:29.936 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/ 2022-09-03T06:23:29.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk= 2022-09-03T06:23:29.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk=', 'Content-Length': '0'} 2022-09-03T06:23:29.940 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004b51f64b36bae688-006312f2e1-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.941 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.941 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.942 DEBUG:boto:Method: HEAD 2022-09-03T06:23:29.942 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:29.943 DEBUG:boto:Data: 2022-09-03T06:23:29.943 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.944 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.944 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.945 DEBUG:boto:Params: {} 2022-09-03T06:23:29.946 DEBUG:boto:Token: None 2022-09-03T06:23:29.946 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/ 2022-09-03T06:23:29.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk= 2022-09-03T06:23:29.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4102Tg+AcWPAMA3D6AqsdKKnGkk=', 'Content-Length': '0'} 2022-09-03T06:23:29.950 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c5b9288ae747f2d5-006312f2e1-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.950 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:29.951 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.951 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.952 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.952 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.953 DEBUG:boto:Method: GET 2022-09-03T06:23:29.953 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.954 DEBUG:boto:Data: 2022-09-03T06:23:29.954 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.955 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.955 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.956 DEBUG:boto:Params: {} 2022-09-03T06:23:29.956 DEBUG:boto:Token: None 2022-09-03T06:23:29.957 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.961 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000054e8ee20bd31163b-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.961 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.962 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:29.963 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.963 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.964 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.964 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.965 DEBUG:boto:Method: GET 2022-09-03T06:23:29.965 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.966 DEBUG:boto:Data: 2022-09-03T06:23:29.966 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.967 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.967 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.968 DEBUG:boto:Params: {} 2022-09-03T06:23:29.968 DEBUG:boto:Token: None 2022-09-03T06:23:29.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.972 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000089baba3bdd48e229-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.973 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.974 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.974 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.975 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.975 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.976 DEBUG:boto:Method: GET 2022-09-03T06:23:29.977 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.977 DEBUG:boto:Data: 2022-09-03T06:23:29.978 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:29.979 DEBUG:boto:Port: 8001 2022-09-03T06:23:29.979 DEBUG:boto:Params: {} 2022-09-03T06:23:29.980 DEBUG:boto:Token: None 2022-09-03T06:23:29.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.984 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f0cbb65ad937106f-006312f2e1-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.985 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.985 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:29.986 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:29.987 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.987 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.988 DEBUG:boto:Method: GET 2022-09-03T06:23:29.988 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:29.989 DEBUG:boto:Data: 2022-09-03T06:23:29.990 DEBUG:boto:Headers: {} 2022-09-03T06:23:29.991 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:29.991 DEBUG:boto:Port: 8000 2022-09-03T06:23:29.992 DEBUG:boto:Params: {} 2022-09-03T06:23:29.993 DEBUG:boto:Token: None 2022-09-03T06:23:29.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:29 GMT /avysmw-40/?versions 2022-09-03T06:23:29.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM= 2022-09-03T06:23:29.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XECkrQdpQaBF9ivu9gSm2xqB4HM=', 'Content-Length': '0'} 2022-09-03T06:23:29.997 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7242bd032e3c8f0-006312f2e1-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:29.997 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:29.998 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-40 zones={test-zone3, test-zone2} 2022-09-03T06:23:29.999 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-40 zones={test-zone1, test-zone3} 2022-09-03T06:23:30.000 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.001 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.001 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.002 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:30.005 DEBUG:boto:Data: 2022-09-03T06:23:30.005 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.006 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.006 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.007 DEBUG:boto:Params: {} 2022-09-03T06:23:30.007 DEBUG:boto:Token: None 2022-09-03T06:23:30.008 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/ 2022-09-03T06:23:30.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wdzf0AoMVaWR2nRGsj8LQy4fifQ= 2022-09-03T06:23:30.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wdzf0AoMVaWR2nRGsj8LQy4fifQ=', 'Content-Length': '0'} 2022-09-03T06:23:30.011 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f4543b63bac123a9-006312f2e2-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.012 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.013 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.013 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.014 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:30.014 DEBUG:boto:Data: 2022-09-03T06:23:30.015 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.016 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.016 DEBUG:boto:Params: {} 2022-09-03T06:23:30.017 DEBUG:boto:Token: None 2022-09-03T06:23:30.017 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/ 2022-09-03T06:23:30.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wdzf0AoMVaWR2nRGsj8LQy4fifQ= 2022-09-03T06:23:30.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wdzf0AoMVaWR2nRGsj8LQy4fifQ=', 'Content-Length': '0'} 2022-09-03T06:23:30.021 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000030983b27f24746b4-006312f2e2-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.022 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.022 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.023 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.023 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.024 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.024 DEBUG:boto:Method: GET 2022-09-03T06:23:30.025 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.025 DEBUG:boto:Data: 2022-09-03T06:23:30.026 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.027 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.027 DEBUG:boto:Params: {} 2022-09-03T06:23:30.028 DEBUG:boto:Token: None 2022-09-03T06:23:30.029 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/?versions 2022-09-03T06:23:30.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo= 2022-09-03T06:23:30.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo=', 'Content-Length': '0'} 2022-09-03T06:23:30.032 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001b00f356cacc397f-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.033 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:30.034 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:30.034 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.035 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.035 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.036 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.036 DEBUG:boto:Method: GET 2022-09-03T06:23:30.037 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.037 DEBUG:boto:Data: 2022-09-03T06:23:30.038 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.039 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.039 DEBUG:boto:Params: {} 2022-09-03T06:23:30.040 DEBUG:boto:Token: None 2022-09-03T06:23:30.041 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/?versions 2022-09-03T06:23:30.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo= 2022-09-03T06:23:30.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo=', 'Content-Length': '0'} 2022-09-03T06:23:30.045 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031dfdae177b76023-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.045 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:30.046 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.047 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.047 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.048 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.048 DEBUG:boto:Method: GET 2022-09-03T06:23:30.049 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.049 DEBUG:boto:Data: 2022-09-03T06:23:30.050 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.051 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.051 DEBUG:boto:Params: {} 2022-09-03T06:23:30.052 DEBUG:boto:Token: None 2022-09-03T06:23:30.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/?versions 2022-09-03T06:23:30.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo= 2022-09-03T06:23:30.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo=', 'Content-Length': '0'} 2022-09-03T06:23:30.056 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000106eaffc640d66fe-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.057 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:30.057 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.058 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.058 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.059 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.059 DEBUG:boto:Method: GET 2022-09-03T06:23:30.060 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.060 DEBUG:boto:Data: 2022-09-03T06:23:30.061 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.062 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.062 DEBUG:boto:Params: {} 2022-09-03T06:23:30.063 DEBUG:boto:Token: None 2022-09-03T06:23:30.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/?versions 2022-09-03T06:23:30.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo= 2022-09-03T06:23:30.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo=', 'Content-Length': '0'} 2022-09-03T06:23:30.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e9a07186227c2482-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.070 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:30.071 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-40 zones={test-zone1, test-zone3} 2022-09-03T06:23:30.072 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-40 zones={test-zone2, test-zone3} 2022-09-03T06:23:30.073 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.074 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.076 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.077 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:30.078 DEBUG:boto:Data: 2022-09-03T06:23:30.079 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.080 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.084 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.087 DEBUG:boto:Params: {} 2022-09-03T06:23:30.088 DEBUG:boto:Token: None 2022-09-03T06:23:30.089 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/ 2022-09-03T06:23:30.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wdzf0AoMVaWR2nRGsj8LQy4fifQ= 2022-09-03T06:23:30.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wdzf0AoMVaWR2nRGsj8LQy4fifQ=', 'Content-Length': '0'} 2022-09-03T06:23:30.094 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000f0c3c2d1c54bd17-006312f2e2-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.096 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.097 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.098 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.099 DEBUG:boto:Path: /avysmw-40/ 2022-09-03T06:23:30.100 DEBUG:boto:Data: 2022-09-03T06:23:30.101 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.102 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.103 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.104 DEBUG:boto:Params: {} 2022-09-03T06:23:30.105 DEBUG:boto:Token: None 2022-09-03T06:23:30.106 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/ 2022-09-03T06:23:30.107 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wdzf0AoMVaWR2nRGsj8LQy4fifQ= 2022-09-03T06:23:30.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wdzf0AoMVaWR2nRGsj8LQy4fifQ=', 'Content-Length': '0'} 2022-09-03T06:23:30.110 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f053df4ae385b554-006312f2e2-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.111 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.111 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.112 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.112 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.113 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.113 DEBUG:boto:Method: GET 2022-09-03T06:23:30.114 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.114 DEBUG:boto:Data: 2022-09-03T06:23:30.115 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.115 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.116 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.116 DEBUG:boto:Params: {} 2022-09-03T06:23:30.117 DEBUG:boto:Token: None 2022-09-03T06:23:30.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/?versions 2022-09-03T06:23:30.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo= 2022-09-03T06:23:30.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo=', 'Content-Length': '0'} 2022-09-03T06:23:30.121 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ac1da5d6ec85b37-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.122 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:30.123 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:30.124 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.125 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.125 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.126 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.126 DEBUG:boto:Method: GET 2022-09-03T06:23:30.127 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.127 DEBUG:boto:Data: 2022-09-03T06:23:30.128 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.129 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.129 DEBUG:boto:Params: {} 2022-09-03T06:23:30.130 DEBUG:boto:Token: None 2022-09-03T06:23:30.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/?versions 2022-09-03T06:23:30.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo= 2022-09-03T06:23:30.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo=', 'Content-Length': '0'} 2022-09-03T06:23:30.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000919bdf9c5a7e19a3-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.178 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:30.179 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.180 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.180 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.181 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.181 DEBUG:boto:Method: GET 2022-09-03T06:23:30.182 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.183 DEBUG:boto:Data: 2022-09-03T06:23:30.183 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.184 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.184 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.185 DEBUG:boto:Params: {} 2022-09-03T06:23:30.185 DEBUG:boto:Token: None 2022-09-03T06:23:30.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/?versions 2022-09-03T06:23:30.186 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo= 2022-09-03T06:23:30.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo=', 'Content-Length': '0'} 2022-09-03T06:23:30.190 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001dc6cd8d56a73398-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.190 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:30.191 DEBUG:boto:path=/avysmw-40/ 2022-09-03T06:23:30.192 DEBUG:boto:auth_path=/avysmw-40/ 2022-09-03T06:23:30.192 DEBUG:boto:path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.193 DEBUG:boto:auth_path=/avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.193 DEBUG:boto:Method: GET 2022-09-03T06:23:30.194 DEBUG:boto:Path: /avysmw-40/?versions&max-keys=999 2022-09-03T06:23:30.194 DEBUG:boto:Data: 2022-09-03T06:23:30.195 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.195 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.196 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.196 DEBUG:boto:Params: {} 2022-09-03T06:23:30.197 DEBUG:boto:Token: None 2022-09-03T06:23:30.197 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-40/?versions 2022-09-03T06:23:30.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo= 2022-09-03T06:23:30.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdkt0JwcBWdg2mAB0Goa34JqHEo=', 'Content-Length': '0'} 2022-09-03T06:23:30.202 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000056d80a36a7ae7893-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.203 DEBUG:boto:b'avysmw-40999false' 2022-09-03T06:23:30.204 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-40 zones={test-zone2, test-zone3} 2022-09-03T06:23:30.205 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-41 zones={test-zone2, test-zone1} 2022-09-03T06:23:30.206 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.207 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.208 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.209 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.210 DEBUG:boto:Data: 2022-09-03T06:23:30.210 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.211 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.212 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.212 DEBUG:boto:Params: {} 2022-09-03T06:23:30.213 DEBUG:boto:Token: None 2022-09-03T06:23:30.213 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.217 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000fc3ab3c5f2fa1e17-006312f2e2-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.218 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.218 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.219 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.219 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.220 DEBUG:boto:Data: 2022-09-03T06:23:30.220 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.222 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.223 DEBUG:boto:Params: {} 2022-09-03T06:23:30.224 DEBUG:boto:Token: None 2022-09-03T06:23:30.225 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.230 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ec08b19fad6bbad9-006312f2e2-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.231 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.231 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.232 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.232 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.233 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.233 DEBUG:boto:Method: GET 2022-09-03T06:23:30.234 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.234 DEBUG:boto:Data: 2022-09-03T06:23:30.235 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.236 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.236 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.237 DEBUG:boto:Params: {} 2022-09-03T06:23:30.237 DEBUG:boto:Token: None 2022-09-03T06:23:30.238 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.241 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002f82abb419a3720f-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.242 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.243 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:30.243 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.244 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.245 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.246 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.247 DEBUG:boto:Method: GET 2022-09-03T06:23:30.247 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.248 DEBUG:boto:Data: 2022-09-03T06:23:30.249 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.251 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.252 DEBUG:boto:Params: {} 2022-09-03T06:23:30.253 DEBUG:boto:Token: None 2022-09-03T06:23:30.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.259 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031a6b57fa9116b94-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.260 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.261 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.261 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.262 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.263 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.264 DEBUG:boto:Method: GET 2022-09-03T06:23:30.264 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.265 DEBUG:boto:Data: 2022-09-03T06:23:30.266 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.267 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.268 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.268 DEBUG:boto:Params: {} 2022-09-03T06:23:30.269 DEBUG:boto:Token: None 2022-09-03T06:23:30.270 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.275 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000adb5a8642bb09846-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.276 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.277 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.278 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.279 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.279 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.280 DEBUG:boto:Method: GET 2022-09-03T06:23:30.280 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.281 DEBUG:boto:Data: 2022-09-03T06:23:30.282 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.283 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.283 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.284 DEBUG:boto:Params: {} 2022-09-03T06:23:30.285 DEBUG:boto:Token: None 2022-09-03T06:23:30.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.291 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fa79100d5a395f75-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.292 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.292 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-41 zones={test-zone2, test-zone1} 2022-09-03T06:23:30.293 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-41 zones={test-zone3, test-zone1} 2022-09-03T06:23:30.293 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.294 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.294 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.295 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.295 DEBUG:boto:Data: 2022-09-03T06:23:30.296 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.296 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.297 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.297 DEBUG:boto:Params: {} 2022-09-03T06:23:30.298 DEBUG:boto:Token: None 2022-09-03T06:23:30.299 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.303 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000540dbdf4542e62fd-006312f2e2-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.304 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.304 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.305 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.305 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.306 DEBUG:boto:Data: 2022-09-03T06:23:30.306 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.307 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.307 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.308 DEBUG:boto:Params: {} 2022-09-03T06:23:30.308 DEBUG:boto:Token: None 2022-09-03T06:23:30.309 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.313 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000091e8fce9e6c73ad9-006312f2e2-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.314 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.315 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.316 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.317 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.319 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.320 DEBUG:boto:Method: GET 2022-09-03T06:23:30.321 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.321 DEBUG:boto:Data: 2022-09-03T06:23:30.322 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.322 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.323 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.323 DEBUG:boto:Params: {} 2022-09-03T06:23:30.324 DEBUG:boto:Token: None 2022-09-03T06:23:30.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.329 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a2b5543b52873e23-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.329 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.330 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:30.331 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.331 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.332 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.332 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.333 DEBUG:boto:Method: GET 2022-09-03T06:23:30.333 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.334 DEBUG:boto:Data: 2022-09-03T06:23:30.334 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.335 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.336 DEBUG:boto:Params: {} 2022-09-03T06:23:30.336 DEBUG:boto:Token: None 2022-09-03T06:23:30.337 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.341 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c087fbd93d322db3-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.382 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.383 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.384 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.385 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.387 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.387 DEBUG:boto:Method: GET 2022-09-03T06:23:30.388 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.389 DEBUG:boto:Data: 2022-09-03T06:23:30.390 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.391 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.392 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.393 DEBUG:boto:Params: {} 2022-09-03T06:23:30.394 DEBUG:boto:Token: None 2022-09-03T06:23:30.395 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.396 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.400 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2ad1c8a7ba8881f-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.401 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.402 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.402 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.403 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.404 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.404 DEBUG:boto:Method: GET 2022-09-03T06:23:30.405 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.405 DEBUG:boto:Data: 2022-09-03T06:23:30.406 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.407 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.408 DEBUG:boto:Params: {} 2022-09-03T06:23:30.408 DEBUG:boto:Token: None 2022-09-03T06:23:30.409 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.413 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004fbe232258f227dd-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.414 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.415 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-41 zones={test-zone3, test-zone1} 2022-09-03T06:23:30.415 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-41 zones={test-zone1, test-zone2} 2022-09-03T06:23:30.416 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.416 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.417 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.418 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.419 DEBUG:boto:Data: 2022-09-03T06:23:30.419 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.420 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.420 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.421 DEBUG:boto:Params: {} 2022-09-03T06:23:30.421 DEBUG:boto:Token: None 2022-09-03T06:23:30.422 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.426 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cd039542986d8f80-006312f2e2-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.427 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.427 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.428 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.428 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.429 DEBUG:boto:Data: 2022-09-03T06:23:30.429 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.430 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.431 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.431 DEBUG:boto:Params: {} 2022-09-03T06:23:30.432 DEBUG:boto:Token: None 2022-09-03T06:23:30.433 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.436 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c01915aadd59d850-006312f2e2-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.437 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.438 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.438 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.439 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.439 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.440 DEBUG:boto:Method: GET 2022-09-03T06:23:30.440 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.441 DEBUG:boto:Data: 2022-09-03T06:23:30.441 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.443 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.443 DEBUG:boto:Params: {} 2022-09-03T06:23:30.444 DEBUG:boto:Token: None 2022-09-03T06:23:30.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.449 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ffc5107dc68f3f5-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.450 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.451 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:30.451 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.452 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.452 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.453 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.453 DEBUG:boto:Method: GET 2022-09-03T06:23:30.454 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.454 DEBUG:boto:Data: 2022-09-03T06:23:30.455 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.456 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.456 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.457 DEBUG:boto:Params: {} 2022-09-03T06:23:30.457 DEBUG:boto:Token: None 2022-09-03T06:23:30.458 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.462 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009579eadc186edc75-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.463 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.464 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.464 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.465 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.465 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.466 DEBUG:boto:Method: GET 2022-09-03T06:23:30.467 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.468 DEBUG:boto:Data: 2022-09-03T06:23:30.469 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.470 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.470 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.471 DEBUG:boto:Params: {} 2022-09-03T06:23:30.471 DEBUG:boto:Token: None 2022-09-03T06:23:30.472 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.478 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b414ef54c4ade42-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.479 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.480 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.480 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.481 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.481 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.482 DEBUG:boto:Method: GET 2022-09-03T06:23:30.482 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.483 DEBUG:boto:Data: 2022-09-03T06:23:30.483 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.484 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.484 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.485 DEBUG:boto:Params: {} 2022-09-03T06:23:30.486 DEBUG:boto:Token: None 2022-09-03T06:23:30.486 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.487 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.487 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.490 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c921e58b7c461b9f-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.534 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.535 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-41 zones={test-zone1, test-zone2} 2022-09-03T06:23:30.536 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-41 zones={test-zone3, test-zone2} 2022-09-03T06:23:30.536 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.537 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.537 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.538 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.538 DEBUG:boto:Data: 2022-09-03T06:23:30.539 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.539 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.540 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.540 DEBUG:boto:Params: {} 2022-09-03T06:23:30.541 DEBUG:boto:Token: None 2022-09-03T06:23:30.542 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.546 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d2e3844562773565-006312f2e2-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.548 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.549 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.550 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.551 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.552 DEBUG:boto:Data: 2022-09-03T06:23:30.553 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.554 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.555 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.556 DEBUG:boto:Params: {} 2022-09-03T06:23:30.557 DEBUG:boto:Token: None 2022-09-03T06:23:30.558 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.559 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.560 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.564 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000087d2b1a813d4f84-006312f2e2-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.565 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.566 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.567 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.568 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.569 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.570 DEBUG:boto:Method: GET 2022-09-03T06:23:30.571 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.572 DEBUG:boto:Data: 2022-09-03T06:23:30.573 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.574 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.575 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.576 DEBUG:boto:Params: {} 2022-09-03T06:23:30.577 DEBUG:boto:Token: None 2022-09-03T06:23:30.578 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.584 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020ba40e1450a88e7-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.626 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.627 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:30.627 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.628 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.628 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.629 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.629 DEBUG:boto:Method: GET 2022-09-03T06:23:30.630 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.631 DEBUG:boto:Data: 2022-09-03T06:23:30.632 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.632 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.633 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.634 DEBUG:boto:Params: {} 2022-09-03T06:23:30.635 DEBUG:boto:Token: None 2022-09-03T06:23:30.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.636 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.637 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.640 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e0f0e0ac48ac532-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.641 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.643 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.643 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.644 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.645 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.646 DEBUG:boto:Method: GET 2022-09-03T06:23:30.647 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.648 DEBUG:boto:Data: 2022-09-03T06:23:30.648 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.650 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.651 DEBUG:boto:Params: {} 2022-09-03T06:23:30.652 DEBUG:boto:Token: None 2022-09-03T06:23:30.653 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.657 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ec0d4514863339b0-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.658 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.659 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.659 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.660 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.660 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.661 DEBUG:boto:Method: GET 2022-09-03T06:23:30.661 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.662 DEBUG:boto:Data: 2022-09-03T06:23:30.663 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.663 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.664 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.664 DEBUG:boto:Params: {} 2022-09-03T06:23:30.665 DEBUG:boto:Token: None 2022-09-03T06:23:30.666 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.669 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000345d16cc4593f59f-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.670 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.671 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-41 zones={test-zone3, test-zone2} 2022-09-03T06:23:30.671 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-41 zones={test-zone1, test-zone3} 2022-09-03T06:23:30.672 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.672 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.673 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.673 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.674 DEBUG:boto:Data: 2022-09-03T06:23:30.675 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.676 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.677 DEBUG:boto:Params: {} 2022-09-03T06:23:30.677 DEBUG:boto:Token: None 2022-09-03T06:23:30.678 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.682 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d0c7eba0276062e4-006312f2e2-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.683 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.683 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.684 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.684 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.685 DEBUG:boto:Data: 2022-09-03T06:23:30.685 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.687 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.687 DEBUG:boto:Params: {} 2022-09-03T06:23:30.688 DEBUG:boto:Token: None 2022-09-03T06:23:30.689 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.693 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cb632da98b0c509b-006312f2e2-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.694 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.695 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.696 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.697 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.698 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.699 DEBUG:boto:Method: GET 2022-09-03T06:23:30.699 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.700 DEBUG:boto:Data: 2022-09-03T06:23:30.700 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.701 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.702 DEBUG:boto:Params: {} 2022-09-03T06:23:30.702 DEBUG:boto:Token: None 2022-09-03T06:23:30.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.707 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000458021ac28460d9a-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.707 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.708 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:30.709 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.709 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.710 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.711 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.712 DEBUG:boto:Method: GET 2022-09-03T06:23:30.713 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.713 DEBUG:boto:Data: 2022-09-03T06:23:30.714 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.715 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.716 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.717 DEBUG:boto:Params: {} 2022-09-03T06:23:30.718 DEBUG:boto:Token: None 2022-09-03T06:23:30.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.725 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000092cd32a9703fc39e-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.725 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.726 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.727 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.727 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.728 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.728 DEBUG:boto:Method: GET 2022-09-03T06:23:30.729 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.729 DEBUG:boto:Data: 2022-09-03T06:23:30.730 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.730 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.731 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.731 DEBUG:boto:Params: {} 2022-09-03T06:23:30.732 DEBUG:boto:Token: None 2022-09-03T06:23:30.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.736 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000074d46fc37a9b1917-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.737 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.738 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.738 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.739 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.739 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.740 DEBUG:boto:Method: GET 2022-09-03T06:23:30.740 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.741 DEBUG:boto:Data: 2022-09-03T06:23:30.741 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.742 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.742 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.743 DEBUG:boto:Params: {} 2022-09-03T06:23:30.744 DEBUG:boto:Token: None 2022-09-03T06:23:30.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.748 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f6f2764c74a3270d-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.749 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.749 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-41 zones={test-zone1, test-zone3} 2022-09-03T06:23:30.750 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-41 zones={test-zone2, test-zone3} 2022-09-03T06:23:30.750 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.751 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.751 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.752 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.752 DEBUG:boto:Data: 2022-09-03T06:23:30.753 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.753 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.754 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.754 DEBUG:boto:Params: {} 2022-09-03T06:23:30.755 DEBUG:boto:Token: None 2022-09-03T06:23:30.756 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.760 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000061cd88007202fc8f-006312f2e2-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.760 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.761 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.761 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.762 DEBUG:boto:Path: /avysmw-41/ 2022-09-03T06:23:30.762 DEBUG:boto:Data: 2022-09-03T06:23:30.763 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.764 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.764 DEBUG:boto:Params: {} 2022-09-03T06:23:30.765 DEBUG:boto:Token: None 2022-09-03T06:23:30.766 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/ 2022-09-03T06:23:30.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks= 2022-09-03T06:23:30.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KUbsY2gGFvefE/eMTQIeglqa1ks=', 'Content-Length': '0'} 2022-09-03T06:23:30.769 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000753ab34decba7f09-006312f2e2-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.770 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.771 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.772 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.772 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.773 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.773 DEBUG:boto:Method: GET 2022-09-03T06:23:30.774 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.774 DEBUG:boto:Data: 2022-09-03T06:23:30.775 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.775 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.776 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.776 DEBUG:boto:Params: {} 2022-09-03T06:23:30.777 DEBUG:boto:Token: None 2022-09-03T06:23:30.778 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.782 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000488c15eb27b6e7f0-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.783 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.783 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:30.784 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.784 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.785 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.785 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.786 DEBUG:boto:Method: GET 2022-09-03T06:23:30.786 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.787 DEBUG:boto:Data: 2022-09-03T06:23:30.787 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.788 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.789 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.789 DEBUG:boto:Params: {} 2022-09-03T06:23:30.790 DEBUG:boto:Token: None 2022-09-03T06:23:30.790 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.791 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.794 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000066f503c5a3c2d970-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.795 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.796 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.796 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.797 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.797 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.798 DEBUG:boto:Method: GET 2022-09-03T06:23:30.799 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.799 DEBUG:boto:Data: 2022-09-03T06:23:30.800 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.800 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.801 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.801 DEBUG:boto:Params: {} 2022-09-03T06:23:30.802 DEBUG:boto:Token: None 2022-09-03T06:23:30.803 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.806 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a1a9c8909cdb3cd-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.807 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.808 DEBUG:boto:path=/avysmw-41/ 2022-09-03T06:23:30.809 DEBUG:boto:auth_path=/avysmw-41/ 2022-09-03T06:23:30.809 DEBUG:boto:path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.810 DEBUG:boto:auth_path=/avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.811 DEBUG:boto:Method: GET 2022-09-03T06:23:30.812 DEBUG:boto:Path: /avysmw-41/?versions&max-keys=999 2022-09-03T06:23:30.813 DEBUG:boto:Data: 2022-09-03T06:23:30.814 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.816 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.817 DEBUG:boto:Params: {} 2022-09-03T06:23:30.818 DEBUG:boto:Token: None 2022-09-03T06:23:30.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-41/?versions 2022-09-03T06:23:30.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM= 2022-09-03T06:23:30.821 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:80Jwz2RCuR8AypSdqr5ycOmuKCM=', 'Content-Length': '0'} 2022-09-03T06:23:30.825 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000470dd5151e0aa68c-006312f2e2-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.826 DEBUG:boto:b'avysmw-41999false' 2022-09-03T06:23:30.827 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-41 zones={test-zone2, test-zone3} 2022-09-03T06:23:30.828 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-42 zones={test-zone2, test-zone1} 2022-09-03T06:23:30.829 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:30.830 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:30.831 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.832 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:30.833 DEBUG:boto:Data: 2022-09-03T06:23:30.834 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.834 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.835 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.835 DEBUG:boto:Params: {} 2022-09-03T06:23:30.836 DEBUG:boto:Token: None 2022-09-03T06:23:30.837 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-42/ 2022-09-03T06:23:30.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jEbTIVeQ1COsAFA+08uU0bLSq5w= 2022-09-03T06:23:30.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jEbTIVeQ1COsAFA+08uU0bLSq5w=', 'Content-Length': '0'} 2022-09-03T06:23:30.841 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000388597a5b3388542-006312f2e2-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.841 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:30.842 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:30.842 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.843 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:30.844 DEBUG:boto:Data: 2022-09-03T06:23:30.844 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.845 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.845 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.846 DEBUG:boto:Params: {} 2022-09-03T06:23:30.847 DEBUG:boto:Token: None 2022-09-03T06:23:30.847 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-42/ 2022-09-03T06:23:30.848 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jEbTIVeQ1COsAFA+08uU0bLSq5w= 2022-09-03T06:23:30.848 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jEbTIVeQ1COsAFA+08uU0bLSq5w=', 'Content-Length': '0'} 2022-09-03T06:23:30.851 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006c0b212aed324e7f-006312f2e2-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.852 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.853 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:30.854 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:30.855 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.856 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.856 DEBUG:boto:Method: GET 2022-09-03T06:23:30.857 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.857 DEBUG:boto:Data: 2022-09-03T06:23:30.858 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.858 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.859 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.859 DEBUG:boto:Params: {} 2022-09-03T06:23:30.860 DEBUG:boto:Token: None 2022-09-03T06:23:30.860 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-42/?versions 2022-09-03T06:23:30.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E5/ffT/Tt12S0+Gl8SRTnN8qtuo= 2022-09-03T06:23:30.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E5/ffT/Tt12S0+Gl8SRTnN8qtuo=', 'Content-Length': '0'} 2022-09-03T06:23:30.864 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007255f9d08bb1a54a-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.910 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:30.911 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:30.912 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:30.913 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:30.914 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.915 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.916 DEBUG:boto:Method: GET 2022-09-03T06:23:30.917 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.918 DEBUG:boto:Data: 2022-09-03T06:23:30.919 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.921 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.922 DEBUG:boto:Params: {} 2022-09-03T06:23:30.923 DEBUG:boto:Token: None 2022-09-03T06:23:30.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-42/?versions 2022-09-03T06:23:30.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E5/ffT/Tt12S0+Gl8SRTnN8qtuo= 2022-09-03T06:23:30.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E5/ffT/Tt12S0+Gl8SRTnN8qtuo=', 'Content-Length': '0'} 2022-09-03T06:23:30.928 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ba6cd6799538f808-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.928 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:30.929 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:30.930 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:30.931 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.933 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.934 DEBUG:boto:Method: GET 2022-09-03T06:23:30.935 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.936 DEBUG:boto:Data: 2022-09-03T06:23:30.936 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.937 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:30.938 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.939 DEBUG:boto:Params: {} 2022-09-03T06:23:30.940 DEBUG:boto:Token: None 2022-09-03T06:23:30.942 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-42/?versions 2022-09-03T06:23:30.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E5/ffT/Tt12S0+Gl8SRTnN8qtuo= 2022-09-03T06:23:30.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E5/ffT/Tt12S0+Gl8SRTnN8qtuo=', 'Content-Length': '0'} 2022-09-03T06:23:30.947 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000268c59b8767c03ab-006312f2e2-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.948 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:30.949 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:30.950 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:30.951 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.952 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.953 DEBUG:boto:Method: GET 2022-09-03T06:23:30.954 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.955 DEBUG:boto:Data: 2022-09-03T06:23:30.956 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.957 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.958 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.959 DEBUG:boto:Params: {} 2022-09-03T06:23:30.960 DEBUG:boto:Token: None 2022-09-03T06:23:30.961 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:30 GMT /avysmw-42/?versions 2022-09-03T06:23:30.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E5/ffT/Tt12S0+Gl8SRTnN8qtuo= 2022-09-03T06:23:30.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E5/ffT/Tt12S0+Gl8SRTnN8qtuo=', 'Content-Length': '0'} 2022-09-03T06:23:30.966 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8a7a7816b311ea7-006312f2e2-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.968 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:30.969 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-42 zones={test-zone2, test-zone1} 2022-09-03T06:23:30.970 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-42 zones={test-zone3, test-zone1} 2022-09-03T06:23:30.971 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:30.972 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:30.972 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.973 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:30.973 DEBUG:boto:Data: 2022-09-03T06:23:30.974 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.975 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.975 DEBUG:boto:Params: {} 2022-09-03T06:23:30.976 DEBUG:boto:Token: None 2022-09-03T06:23:30.976 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-42/ 2022-09-03T06:23:30.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jEbTIVeQ1COsAFA+08uU0bLSq5w= 2022-09-03T06:23:30.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jEbTIVeQ1COsAFA+08uU0bLSq5w=', 'Content-Length': '0'} 2022-09-03T06:23:30.980 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003bd8d15dbfd05fdb-006312f2e2-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.981 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:30.981 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:30.982 DEBUG:boto:Method: HEAD 2022-09-03T06:23:30.983 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:30.983 DEBUG:boto:Data: 2022-09-03T06:23:30.984 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:30.985 DEBUG:boto:Port: 8000 2022-09-03T06:23:30.985 DEBUG:boto:Params: {} 2022-09-03T06:23:30.986 DEBUG:boto:Token: None 2022-09-03T06:23:30.987 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:30 GMT /avysmw-42/ 2022-09-03T06:23:30.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jEbTIVeQ1COsAFA+08uU0bLSq5w= 2022-09-03T06:23:30.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jEbTIVeQ1COsAFA+08uU0bLSq5w=', 'Content-Length': '0'} 2022-09-03T06:23:30.991 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b57c5b5118af9fc2-006312f2e2-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:30.992 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:30.992 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:30.993 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:30.994 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.995 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.995 DEBUG:boto:Method: GET 2022-09-03T06:23:30.996 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:30.996 DEBUG:boto:Data: 2022-09-03T06:23:30.997 DEBUG:boto:Headers: {} 2022-09-03T06:23:30.997 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:30.998 DEBUG:boto:Port: 8001 2022-09-03T06:23:30.999 DEBUG:boto:Params: {} 2022-09-03T06:23:30.999 DEBUG:boto:Token: None 2022-09-03T06:23:31.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.004 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ec54a868a2a9ec41-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.005 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.006 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:31.006 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.007 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.007 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.008 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.008 DEBUG:boto:Method: GET 2022-09-03T06:23:31.009 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.010 DEBUG:boto:Data: 2022-09-03T06:23:31.010 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.011 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.012 DEBUG:boto:Params: {} 2022-09-03T06:23:31.012 DEBUG:boto:Token: None 2022-09-03T06:23:31.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.017 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001887496251a8f8c8-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.063 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.065 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.065 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.066 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.066 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.067 DEBUG:boto:Method: GET 2022-09-03T06:23:31.067 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.068 DEBUG:boto:Data: 2022-09-03T06:23:31.068 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.069 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.069 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.070 DEBUG:boto:Params: {} 2022-09-03T06:23:31.071 DEBUG:boto:Token: None 2022-09-03T06:23:31.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.075 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b53c48c7bcae28e7-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.076 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.077 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.077 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.078 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.078 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.079 DEBUG:boto:Method: GET 2022-09-03T06:23:31.079 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.080 DEBUG:boto:Data: 2022-09-03T06:23:31.080 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.081 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.081 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.081 DEBUG:boto:Params: {} 2022-09-03T06:23:31.082 DEBUG:boto:Token: None 2022-09-03T06:23:31.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.088 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008bf3fe124f8d024c-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.089 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.090 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-42 zones={test-zone3, test-zone1} 2022-09-03T06:23:31.090 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-42 zones={test-zone1, test-zone2} 2022-09-03T06:23:31.091 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.092 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.092 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.093 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:31.094 DEBUG:boto:Data: 2022-09-03T06:23:31.094 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.095 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.096 DEBUG:boto:Params: {} 2022-09-03T06:23:31.096 DEBUG:boto:Token: None 2022-09-03T06:23:31.097 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/ 2022-09-03T06:23:31.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E= 2022-09-03T06:23:31.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E=', 'Content-Length': '0'} 2022-09-03T06:23:31.101 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000029acc93f76ee58e6-006312f2e3-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.102 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.103 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.103 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.104 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:31.105 DEBUG:boto:Data: 2022-09-03T06:23:31.105 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.106 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.106 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.107 DEBUG:boto:Params: {} 2022-09-03T06:23:31.108 DEBUG:boto:Token: None 2022-09-03T06:23:31.108 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/ 2022-09-03T06:23:31.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E= 2022-09-03T06:23:31.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E=', 'Content-Length': '0'} 2022-09-03T06:23:31.112 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b2aedca5a1773ccf-006312f2e3-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.113 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:31.113 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.114 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.114 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.115 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.115 DEBUG:boto:Method: GET 2022-09-03T06:23:31.116 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.116 DEBUG:boto:Data: 2022-09-03T06:23:31.117 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.118 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.118 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.119 DEBUG:boto:Params: {} 2022-09-03T06:23:31.119 DEBUG:boto:Token: None 2022-09-03T06:23:31.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.124 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b49cff7f8c22dfce-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.125 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.125 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:31.126 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.127 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.127 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.128 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.128 DEBUG:boto:Method: GET 2022-09-03T06:23:31.129 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.129 DEBUG:boto:Data: 2022-09-03T06:23:31.130 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.130 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.131 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.132 DEBUG:boto:Params: {} 2022-09-03T06:23:31.132 DEBUG:boto:Token: None 2022-09-03T06:23:31.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.137 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8f990cbed3fae04-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.137 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.138 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.139 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.139 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.140 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.140 DEBUG:boto:Method: GET 2022-09-03T06:23:31.141 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.141 DEBUG:boto:Data: 2022-09-03T06:23:31.142 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.142 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.143 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.144 DEBUG:boto:Params: {} 2022-09-03T06:23:31.144 DEBUG:boto:Token: None 2022-09-03T06:23:31.145 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.145 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.146 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.149 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6448e21a48ef1eb-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.150 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.150 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.151 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.151 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.152 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.152 DEBUG:boto:Method: GET 2022-09-03T06:23:31.153 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.153 DEBUG:boto:Data: 2022-09-03T06:23:31.154 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.155 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.155 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.156 DEBUG:boto:Params: {} 2022-09-03T06:23:31.156 DEBUG:boto:Token: None 2022-09-03T06:23:31.157 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.161 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088877215c68a524c-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.162 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.162 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-42 zones={test-zone1, test-zone2} 2022-09-03T06:23:31.163 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-42 zones={test-zone3, test-zone2} 2022-09-03T06:23:31.163 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.164 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.165 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.165 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:31.166 DEBUG:boto:Data: 2022-09-03T06:23:31.166 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.167 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.168 DEBUG:boto:Params: {} 2022-09-03T06:23:31.169 DEBUG:boto:Token: None 2022-09-03T06:23:31.169 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/ 2022-09-03T06:23:31.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E= 2022-09-03T06:23:31.170 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E=', 'Content-Length': '0'} 2022-09-03T06:23:31.173 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000455b27a8b1a5fdf1-006312f2e3-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.174 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.174 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.175 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.175 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:31.176 DEBUG:boto:Data: 2022-09-03T06:23:31.176 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.177 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.177 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.178 DEBUG:boto:Params: {} 2022-09-03T06:23:31.179 DEBUG:boto:Token: None 2022-09-03T06:23:31.179 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/ 2022-09-03T06:23:31.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E= 2022-09-03T06:23:31.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E=', 'Content-Length': '0'} 2022-09-03T06:23:31.183 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c1f69db444000979-006312f2e3-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.184 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:31.184 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.185 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.185 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.186 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.186 DEBUG:boto:Method: GET 2022-09-03T06:23:31.187 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.187 DEBUG:boto:Data: 2022-09-03T06:23:31.188 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.188 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.189 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.189 DEBUG:boto:Params: {} 2022-09-03T06:23:31.190 DEBUG:boto:Token: None 2022-09-03T06:23:31.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.195 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb5321b0b0492418-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.196 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.197 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:31.198 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.198 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.199 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.200 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.200 DEBUG:boto:Method: GET 2022-09-03T06:23:31.201 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.201 DEBUG:boto:Data: 2022-09-03T06:23:31.202 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.202 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.203 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.203 DEBUG:boto:Params: {} 2022-09-03T06:23:31.204 DEBUG:boto:Token: None 2022-09-03T06:23:31.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.208 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ebcf247129b0d89-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.209 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.210 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.210 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.211 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.211 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.212 DEBUG:boto:Method: GET 2022-09-03T06:23:31.213 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.213 DEBUG:boto:Data: 2022-09-03T06:23:31.214 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.214 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.215 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.215 DEBUG:boto:Params: {} 2022-09-03T06:23:31.216 DEBUG:boto:Token: None 2022-09-03T06:23:31.216 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.220 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac226e6360739104-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.266 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.267 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.268 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.269 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.269 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.270 DEBUG:boto:Method: GET 2022-09-03T06:23:31.271 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.272 DEBUG:boto:Data: 2022-09-03T06:23:31.272 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.274 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.274 DEBUG:boto:Params: {} 2022-09-03T06:23:31.275 DEBUG:boto:Token: None 2022-09-03T06:23:31.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.280 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000071d199d82851bd6d-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.280 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.281 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-42 zones={test-zone3, test-zone2} 2022-09-03T06:23:31.282 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-42 zones={test-zone1, test-zone3} 2022-09-03T06:23:31.282 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.283 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.283 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.284 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:31.284 DEBUG:boto:Data: 2022-09-03T06:23:31.285 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.286 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.286 DEBUG:boto:Params: {} 2022-09-03T06:23:31.287 DEBUG:boto:Token: None 2022-09-03T06:23:31.288 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/ 2022-09-03T06:23:31.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E= 2022-09-03T06:23:31.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E=', 'Content-Length': '0'} 2022-09-03T06:23:31.293 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002f03e3141453934d-006312f2e3-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.294 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.295 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.296 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.296 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:31.298 DEBUG:boto:Data: 2022-09-03T06:23:31.298 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.300 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.301 DEBUG:boto:Params: {} 2022-09-03T06:23:31.301 DEBUG:boto:Token: None 2022-09-03T06:23:31.302 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/ 2022-09-03T06:23:31.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E= 2022-09-03T06:23:31.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E=', 'Content-Length': '0'} 2022-09-03T06:23:31.307 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c528b02c4feccb70-006312f2e3-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.308 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:31.308 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.309 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.309 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.310 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.311 DEBUG:boto:Method: GET 2022-09-03T06:23:31.311 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.312 DEBUG:boto:Data: 2022-09-03T06:23:31.313 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.314 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.314 DEBUG:boto:Params: {} 2022-09-03T06:23:31.315 DEBUG:boto:Token: None 2022-09-03T06:23:31.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.322 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d6cd431120ab895-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.370 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.371 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:31.372 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.373 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.373 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.374 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.375 DEBUG:boto:Method: GET 2022-09-03T06:23:31.375 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.376 DEBUG:boto:Data: 2022-09-03T06:23:31.377 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.378 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.378 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.379 DEBUG:boto:Params: {} 2022-09-03T06:23:31.380 DEBUG:boto:Token: None 2022-09-03T06:23:31.380 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.394 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004dfc287d939d435d-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.395 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.397 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.397 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.398 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.399 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.400 DEBUG:boto:Method: GET 2022-09-03T06:23:31.400 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.401 DEBUG:boto:Data: 2022-09-03T06:23:31.401 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.403 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.404 DEBUG:boto:Params: {} 2022-09-03T06:23:31.404 DEBUG:boto:Token: None 2022-09-03T06:23:31.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.410 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bdf3a4d115c7a917-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.411 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.412 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.412 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.413 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.414 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.415 DEBUG:boto:Method: GET 2022-09-03T06:23:31.415 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.416 DEBUG:boto:Data: 2022-09-03T06:23:31.416 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.417 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.417 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.418 DEBUG:boto:Params: {} 2022-09-03T06:23:31.419 DEBUG:boto:Token: None 2022-09-03T06:23:31.421 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.427 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc632494f064449e-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.428 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.429 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-42 zones={test-zone1, test-zone3} 2022-09-03T06:23:31.430 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-42 zones={test-zone2, test-zone3} 2022-09-03T06:23:31.431 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.432 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.432 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.433 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:31.434 DEBUG:boto:Data: 2022-09-03T06:23:31.434 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.435 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.436 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.437 DEBUG:boto:Params: {} 2022-09-03T06:23:31.438 DEBUG:boto:Token: None 2022-09-03T06:23:31.438 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/ 2022-09-03T06:23:31.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E= 2022-09-03T06:23:31.440 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E=', 'Content-Length': '0'} 2022-09-03T06:23:31.444 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009265b1445b1dad02-006312f2e3-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.445 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.446 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.446 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.447 DEBUG:boto:Path: /avysmw-42/ 2022-09-03T06:23:31.447 DEBUG:boto:Data: 2022-09-03T06:23:31.448 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.449 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.450 DEBUG:boto:Params: {} 2022-09-03T06:23:31.451 DEBUG:boto:Token: None 2022-09-03T06:23:31.451 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/ 2022-09-03T06:23:31.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E= 2022-09-03T06:23:31.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZjMYlTYkyJNaWMf/0YISaF/o8E=', 'Content-Length': '0'} 2022-09-03T06:23:31.456 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007f65665bb6b27583-006312f2e3-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.456 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:31.457 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.458 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.459 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.461 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.461 DEBUG:boto:Method: GET 2022-09-03T06:23:31.462 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.463 DEBUG:boto:Data: 2022-09-03T06:23:31.464 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.465 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.466 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.467 DEBUG:boto:Params: {} 2022-09-03T06:23:31.468 DEBUG:boto:Token: None 2022-09-03T06:23:31.468 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.473 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c3bc1a2055c14682-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.475 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.476 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:31.477 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.478 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.479 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.479 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.480 DEBUG:boto:Method: GET 2022-09-03T06:23:31.480 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.481 DEBUG:boto:Data: 2022-09-03T06:23:31.481 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.482 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.482 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.484 DEBUG:boto:Params: {} 2022-09-03T06:23:31.484 DEBUG:boto:Token: None 2022-09-03T06:23:31.485 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.489 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000040e435a8096dc3b8-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.535 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.536 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.537 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.537 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.538 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.538 DEBUG:boto:Method: GET 2022-09-03T06:23:31.539 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.539 DEBUG:boto:Data: 2022-09-03T06:23:31.540 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.541 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.541 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.542 DEBUG:boto:Params: {} 2022-09-03T06:23:31.544 DEBUG:boto:Token: None 2022-09-03T06:23:31.545 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.546 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.547 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.550 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088dfe06b208ff7de-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.551 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.553 DEBUG:boto:path=/avysmw-42/ 2022-09-03T06:23:31.555 DEBUG:boto:auth_path=/avysmw-42/ 2022-09-03T06:23:31.556 DEBUG:boto:path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.556 DEBUG:boto:auth_path=/avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.557 DEBUG:boto:Method: GET 2022-09-03T06:23:31.557 DEBUG:boto:Path: /avysmw-42/?versions&max-keys=999 2022-09-03T06:23:31.558 DEBUG:boto:Data: 2022-09-03T06:23:31.558 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.559 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.560 DEBUG:boto:Params: {} 2022-09-03T06:23:31.561 DEBUG:boto:Token: None 2022-09-03T06:23:31.562 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-42/?versions 2022-09-03T06:23:31.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ= 2022-09-03T06:23:31.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SI2xauvTzgb29UcPSd9n3/vPfiQ=', 'Content-Length': '0'} 2022-09-03T06:23:31.566 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d845d770418b263-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.566 DEBUG:boto:b'avysmw-42999false' 2022-09-03T06:23:31.567 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-42 zones={test-zone2, test-zone3} 2022-09-03T06:23:31.568 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-43 zones={test-zone2, test-zone1} 2022-09-03T06:23:31.568 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.569 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.570 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.570 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:31.571 DEBUG:boto:Data: 2022-09-03T06:23:31.572 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.573 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.574 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.575 DEBUG:boto:Params: {} 2022-09-03T06:23:31.576 DEBUG:boto:Token: None 2022-09-03T06:23:31.577 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/ 2022-09-03T06:23:31.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI= 2022-09-03T06:23:31.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI=', 'Content-Length': '0'} 2022-09-03T06:23:31.583 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000019582a9f714c4c40-006312f2e3-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.584 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.585 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.586 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.586 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:31.587 DEBUG:boto:Data: 2022-09-03T06:23:31.587 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.588 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.588 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.589 DEBUG:boto:Params: {} 2022-09-03T06:23:31.589 DEBUG:boto:Token: None 2022-09-03T06:23:31.591 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/ 2022-09-03T06:23:31.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI= 2022-09-03T06:23:31.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI=', 'Content-Length': '0'} 2022-09-03T06:23:31.595 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c54634f8f36a15d1-006312f2e3-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.596 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:31.597 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.598 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.599 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.600 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.602 DEBUG:boto:Method: GET 2022-09-03T06:23:31.603 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.603 DEBUG:boto:Data: 2022-09-03T06:23:31.604 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.605 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.606 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.608 DEBUG:boto:Params: {} 2022-09-03T06:23:31.609 DEBUG:boto:Token: None 2022-09-03T06:23:31.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.616 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007623f22b0b51c2da-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.658 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.659 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:31.660 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.660 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.661 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.662 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.662 DEBUG:boto:Method: GET 2022-09-03T06:23:31.663 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.663 DEBUG:boto:Data: 2022-09-03T06:23:31.664 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.665 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.665 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.666 DEBUG:boto:Params: {} 2022-09-03T06:23:31.667 DEBUG:boto:Token: None 2022-09-03T06:23:31.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.668 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.672 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d804545b286ea304-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.673 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.675 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.676 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.677 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.677 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.678 DEBUG:boto:Method: GET 2022-09-03T06:23:31.679 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.680 DEBUG:boto:Data: 2022-09-03T06:23:31.681 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.682 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.683 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.684 DEBUG:boto:Params: {} 2022-09-03T06:23:31.685 DEBUG:boto:Token: None 2022-09-03T06:23:31.686 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.692 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000362eb24d8ebdedc6-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.693 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.694 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.695 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.696 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.697 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.698 DEBUG:boto:Method: GET 2022-09-03T06:23:31.699 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.700 DEBUG:boto:Data: 2022-09-03T06:23:31.701 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.702 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.703 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.704 DEBUG:boto:Params: {} 2022-09-03T06:23:31.705 DEBUG:boto:Token: None 2022-09-03T06:23:31.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.713 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000007841387422c0361-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.715 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.716 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-43 zones={test-zone2, test-zone1} 2022-09-03T06:23:31.717 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-43 zones={test-zone3, test-zone1} 2022-09-03T06:23:31.718 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.719 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.720 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.721 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:31.722 DEBUG:boto:Data: 2022-09-03T06:23:31.723 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.724 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.725 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.726 DEBUG:boto:Params: {} 2022-09-03T06:23:31.727 DEBUG:boto:Token: None 2022-09-03T06:23:31.728 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/ 2022-09-03T06:23:31.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI= 2022-09-03T06:23:31.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI=', 'Content-Length': '0'} 2022-09-03T06:23:31.734 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000a0409ebf63eee24-006312f2e3-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.736 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.737 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.738 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.739 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:31.739 DEBUG:boto:Data: 2022-09-03T06:23:31.740 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.741 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.742 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.743 DEBUG:boto:Params: {} 2022-09-03T06:23:31.745 DEBUG:boto:Token: None 2022-09-03T06:23:31.746 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/ 2022-09-03T06:23:31.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI= 2022-09-03T06:23:31.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI=', 'Content-Length': '0'} 2022-09-03T06:23:31.751 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000907c0a463d1f29fa-006312f2e3-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.752 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:31.753 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.754 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.754 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.755 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.756 DEBUG:boto:Method: GET 2022-09-03T06:23:31.757 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.758 DEBUG:boto:Data: 2022-09-03T06:23:31.759 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.760 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.761 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.762 DEBUG:boto:Params: {} 2022-09-03T06:23:31.763 DEBUG:boto:Token: None 2022-09-03T06:23:31.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.769 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6414ca01918ce62-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.770 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.771 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:31.772 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.773 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.774 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.775 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.776 DEBUG:boto:Method: GET 2022-09-03T06:23:31.777 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.777 DEBUG:boto:Data: 2022-09-03T06:23:31.778 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.779 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.780 DEBUG:boto:Params: {} 2022-09-03T06:23:31.780 DEBUG:boto:Token: None 2022-09-03T06:23:31.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.786 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000184493eb1b845d78-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.787 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.788 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.789 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.789 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.790 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.790 DEBUG:boto:Method: GET 2022-09-03T06:23:31.791 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.791 DEBUG:boto:Data: 2022-09-03T06:23:31.792 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.793 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.794 DEBUG:boto:Params: {} 2022-09-03T06:23:31.794 DEBUG:boto:Token: None 2022-09-03T06:23:31.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.799 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091a87b8051e8d598-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.800 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.800 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.801 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.801 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.802 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.802 DEBUG:boto:Method: GET 2022-09-03T06:23:31.803 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.803 DEBUG:boto:Data: 2022-09-03T06:23:31.804 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.805 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.806 DEBUG:boto:Params: {} 2022-09-03T06:23:31.806 DEBUG:boto:Token: None 2022-09-03T06:23:31.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.811 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001714c40bcc5328cb-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.854 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.855 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-43 zones={test-zone3, test-zone1} 2022-09-03T06:23:31.856 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-43 zones={test-zone1, test-zone2} 2022-09-03T06:23:31.857 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.857 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.858 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.858 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:31.859 DEBUG:boto:Data: 2022-09-03T06:23:31.859 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.860 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.860 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.861 DEBUG:boto:Params: {} 2022-09-03T06:23:31.861 DEBUG:boto:Token: None 2022-09-03T06:23:31.862 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/ 2022-09-03T06:23:31.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI= 2022-09-03T06:23:31.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI=', 'Content-Length': '0'} 2022-09-03T06:23:31.866 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006e2ee0f556ac60ba-006312f2e3-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.867 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.867 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.868 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.868 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:31.869 DEBUG:boto:Data: 2022-09-03T06:23:31.869 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.870 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.870 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.871 DEBUG:boto:Params: {} 2022-09-03T06:23:31.872 DEBUG:boto:Token: None 2022-09-03T06:23:31.872 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/ 2022-09-03T06:23:31.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI= 2022-09-03T06:23:31.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI=', 'Content-Length': '0'} 2022-09-03T06:23:31.876 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000392bab955f6dd7a7-006312f2e3-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.877 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:31.877 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.878 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.878 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.879 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.880 DEBUG:boto:Method: GET 2022-09-03T06:23:31.880 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.881 DEBUG:boto:Data: 2022-09-03T06:23:31.882 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.883 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.884 DEBUG:boto:Params: {} 2022-09-03T06:23:31.884 DEBUG:boto:Token: None 2022-09-03T06:23:31.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007eaf753967b6194f-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.890 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.891 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:31.891 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.892 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.893 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.893 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.894 DEBUG:boto:Method: GET 2022-09-03T06:23:31.894 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.895 DEBUG:boto:Data: 2022-09-03T06:23:31.895 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.896 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.896 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.897 DEBUG:boto:Params: {} 2022-09-03T06:23:31.898 DEBUG:boto:Token: None 2022-09-03T06:23:31.898 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.899 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.900 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.902 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c940d3a9591008c5-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.903 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.904 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.905 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.905 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.906 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.906 DEBUG:boto:Method: GET 2022-09-03T06:23:31.907 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.907 DEBUG:boto:Data: 2022-09-03T06:23:31.908 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:31.909 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.910 DEBUG:boto:Params: {} 2022-09-03T06:23:31.910 DEBUG:boto:Token: None 2022-09-03T06:23:31.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.915 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001471048a3dfe8627-006312f2e3-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.916 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.918 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.921 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.921 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.922 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.923 DEBUG:boto:Method: GET 2022-09-03T06:23:31.924 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.926 DEBUG:boto:Data: 2022-09-03T06:23:31.927 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.928 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.928 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.929 DEBUG:boto:Params: {} 2022-09-03T06:23:31.930 DEBUG:boto:Token: None 2022-09-03T06:23:31.931 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.933 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.935 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e69bf21af5fb8b7c-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.936 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.937 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-43 zones={test-zone1, test-zone2} 2022-09-03T06:23:31.938 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-43 zones={test-zone3, test-zone2} 2022-09-03T06:23:31.938 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.939 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.939 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.940 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:31.940 DEBUG:boto:Data: 2022-09-03T06:23:31.941 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.941 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.942 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.942 DEBUG:boto:Params: {} 2022-09-03T06:23:31.943 DEBUG:boto:Token: None 2022-09-03T06:23:31.944 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/ 2022-09-03T06:23:31.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI= 2022-09-03T06:23:31.945 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI=', 'Content-Length': '0'} 2022-09-03T06:23:31.948 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000292281558c28d0b4-006312f2e3-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.949 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.949 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.950 DEBUG:boto:Method: HEAD 2022-09-03T06:23:31.950 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:31.951 DEBUG:boto:Data: 2022-09-03T06:23:31.952 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.952 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.953 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.953 DEBUG:boto:Params: {} 2022-09-03T06:23:31.954 DEBUG:boto:Token: None 2022-09-03T06:23:31.954 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/ 2022-09-03T06:23:31.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI= 2022-09-03T06:23:31.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UzUpobFlWWkA6k+zepkVJeKSsI=', 'Content-Length': '0'} 2022-09-03T06:23:31.958 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007b57c99093873dad-006312f2e3-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.959 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:31.960 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.960 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.961 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.962 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.962 DEBUG:boto:Method: GET 2022-09-03T06:23:31.963 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.964 DEBUG:boto:Data: 2022-09-03T06:23:31.965 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:31.967 DEBUG:boto:Port: 8001 2022-09-03T06:23:31.968 DEBUG:boto:Params: {} 2022-09-03T06:23:31.969 DEBUG:boto:Token: None 2022-09-03T06:23:31.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.976 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d6a4de406966accd-006312f2e3-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:31.977 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:31.978 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:31.979 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:31.980 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:31.981 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.982 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.983 DEBUG:boto:Method: GET 2022-09-03T06:23:31.984 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:31.985 DEBUG:boto:Data: 2022-09-03T06:23:31.985 DEBUG:boto:Headers: {} 2022-09-03T06:23:31.986 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:31.986 DEBUG:boto:Port: 8000 2022-09-03T06:23:31.987 DEBUG:boto:Params: {} 2022-09-03T06:23:31.987 DEBUG:boto:Token: None 2022-09-03T06:23:31.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:31 GMT /avysmw-43/?versions 2022-09-03T06:23:31.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk= 2022-09-03T06:23:31.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZohtrrUzO2Zufj58MNIkIZYGJSk=', 'Content-Length': '0'} 2022-09-03T06:23:31.992 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009c31f93138f8c891-006312f2e3-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.047 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.048 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.049 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.050 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.050 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.051 DEBUG:boto:Method: GET 2022-09-03T06:23:32.051 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.052 DEBUG:boto:Data: 2022-09-03T06:23:32.052 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.053 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.054 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.054 DEBUG:boto:Params: {} 2022-09-03T06:23:32.055 DEBUG:boto:Token: None 2022-09-03T06:23:32.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.059 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.062 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008788510feee4f530-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.063 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.064 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.065 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.066 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.067 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.068 DEBUG:boto:Method: GET 2022-09-03T06:23:32.069 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.069 DEBUG:boto:Data: 2022-09-03T06:23:32.070 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.070 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.071 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.071 DEBUG:boto:Params: {} 2022-09-03T06:23:32.072 DEBUG:boto:Token: None 2022-09-03T06:23:32.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.081 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000364603b78424b482-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.082 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.083 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-43 zones={test-zone3, test-zone2} 2022-09-03T06:23:32.084 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-43 zones={test-zone1, test-zone3} 2022-09-03T06:23:32.085 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.085 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.086 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.087 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:32.087 DEBUG:boto:Data: 2022-09-03T06:23:32.088 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.088 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.089 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.089 DEBUG:boto:Params: {} 2022-09-03T06:23:32.090 DEBUG:boto:Token: None 2022-09-03T06:23:32.091 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/ 2022-09-03T06:23:32.091 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:btzRqUO4zHMexPrIYW1zqh1yJ6w= 2022-09-03T06:23:32.092 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:btzRqUO4zHMexPrIYW1zqh1yJ6w=', 'Content-Length': '0'} 2022-09-03T06:23:32.094 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ea3f7677affb2774-006312f2e4-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.095 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.096 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.096 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.097 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:32.098 DEBUG:boto:Data: 2022-09-03T06:23:32.098 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.099 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.100 DEBUG:boto:Params: {} 2022-09-03T06:23:32.100 DEBUG:boto:Token: None 2022-09-03T06:23:32.101 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/ 2022-09-03T06:23:32.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:btzRqUO4zHMexPrIYW1zqh1yJ6w= 2022-09-03T06:23:32.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:btzRqUO4zHMexPrIYW1zqh1yJ6w=', 'Content-Length': '0'} 2022-09-03T06:23:32.106 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006552ace1185ecc36-006312f2e4-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.107 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:32.108 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.109 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.110 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.110 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.111 DEBUG:boto:Method: GET 2022-09-03T06:23:32.112 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.112 DEBUG:boto:Data: 2022-09-03T06:23:32.113 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.113 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.114 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.115 DEBUG:boto:Params: {} 2022-09-03T06:23:32.115 DEBUG:boto:Token: None 2022-09-03T06:23:32.116 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.120 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e6b61fabe2f2b00-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.166 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.168 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:32.169 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.170 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.170 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.171 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.172 DEBUG:boto:Method: GET 2022-09-03T06:23:32.173 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.174 DEBUG:boto:Data: 2022-09-03T06:23:32.175 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.176 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.177 DEBUG:boto:Params: {} 2022-09-03T06:23:32.178 DEBUG:boto:Token: None 2022-09-03T06:23:32.178 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.183 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000802258fe9d7a440a-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.184 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.184 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.185 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.186 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.187 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.188 DEBUG:boto:Method: GET 2022-09-03T06:23:32.189 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.190 DEBUG:boto:Data: 2022-09-03T06:23:32.191 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.193 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.193 DEBUG:boto:Params: {} 2022-09-03T06:23:32.194 DEBUG:boto:Token: None 2022-09-03T06:23:32.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.199 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006b90474d6c32fb7a-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.200 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.201 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.201 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.202 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.202 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.203 DEBUG:boto:Method: GET 2022-09-03T06:23:32.204 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.204 DEBUG:boto:Data: 2022-09-03T06:23:32.205 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.205 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.206 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.206 DEBUG:boto:Params: {} 2022-09-03T06:23:32.207 DEBUG:boto:Token: None 2022-09-03T06:23:32.207 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.211 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000322b63e3b4a90104-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.212 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.213 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-43 zones={test-zone1, test-zone3} 2022-09-03T06:23:32.213 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-43 zones={test-zone2, test-zone3} 2022-09-03T06:23:32.214 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.214 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.215 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.215 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:32.216 DEBUG:boto:Data: 2022-09-03T06:23:32.216 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.217 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.217 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.218 DEBUG:boto:Params: {} 2022-09-03T06:23:32.218 DEBUG:boto:Token: None 2022-09-03T06:23:32.219 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/ 2022-09-03T06:23:32.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:btzRqUO4zHMexPrIYW1zqh1yJ6w= 2022-09-03T06:23:32.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:btzRqUO4zHMexPrIYW1zqh1yJ6w=', 'Content-Length': '0'} 2022-09-03T06:23:32.223 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005ece2b7767d81ee2-006312f2e4-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.224 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.225 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.225 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.226 DEBUG:boto:Path: /avysmw-43/ 2022-09-03T06:23:32.226 DEBUG:boto:Data: 2022-09-03T06:23:32.227 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.227 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.228 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.228 DEBUG:boto:Params: {} 2022-09-03T06:23:32.229 DEBUG:boto:Token: None 2022-09-03T06:23:32.230 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/ 2022-09-03T06:23:32.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:btzRqUO4zHMexPrIYW1zqh1yJ6w= 2022-09-03T06:23:32.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:btzRqUO4zHMexPrIYW1zqh1yJ6w=', 'Content-Length': '0'} 2022-09-03T06:23:32.234 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006b8a5f3064b70c12-006312f2e4-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.236 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:32.237 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.237 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.238 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.239 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.240 DEBUG:boto:Method: GET 2022-09-03T06:23:32.241 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.242 DEBUG:boto:Data: 2022-09-03T06:23:32.243 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.243 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.244 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.244 DEBUG:boto:Params: {} 2022-09-03T06:23:32.245 DEBUG:boto:Token: None 2022-09-03T06:23:32.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.250 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022378865ceda043b-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.250 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.251 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:32.252 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.252 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.253 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.253 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.254 DEBUG:boto:Method: GET 2022-09-03T06:23:32.255 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.256 DEBUG:boto:Data: 2022-09-03T06:23:32.257 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.258 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.259 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.259 DEBUG:boto:Params: {} 2022-09-03T06:23:32.260 DEBUG:boto:Token: None 2022-09-03T06:23:32.261 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.266 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ab9cf1cf6b1dd1b3-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.314 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.315 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.316 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.316 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.317 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.317 DEBUG:boto:Method: GET 2022-09-03T06:23:32.318 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.318 DEBUG:boto:Data: 2022-09-03T06:23:32.319 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.319 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.320 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.321 DEBUG:boto:Params: {} 2022-09-03T06:23:32.322 DEBUG:boto:Token: None 2022-09-03T06:23:32.323 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.329 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a5f661f10d610b8-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.331 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.331 DEBUG:boto:path=/avysmw-43/ 2022-09-03T06:23:32.332 DEBUG:boto:auth_path=/avysmw-43/ 2022-09-03T06:23:32.333 DEBUG:boto:path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.334 DEBUG:boto:auth_path=/avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.335 DEBUG:boto:Method: GET 2022-09-03T06:23:32.336 DEBUG:boto:Path: /avysmw-43/?versions&max-keys=999 2022-09-03T06:23:32.337 DEBUG:boto:Data: 2022-09-03T06:23:32.337 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.338 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.339 DEBUG:boto:Params: {} 2022-09-03T06:23:32.340 DEBUG:boto:Token: None 2022-09-03T06:23:32.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-43/?versions 2022-09-03T06:23:32.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ= 2022-09-03T06:23:32.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HSZvgZIkJvIFscYC6WuqVquXLmQ=', 'Content-Length': '0'} 2022-09-03T06:23:32.347 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a2f225ca8358405d-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.348 DEBUG:boto:b'avysmw-43999false' 2022-09-03T06:23:32.349 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-43 zones={test-zone2, test-zone3} 2022-09-03T06:23:32.350 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-44 zones={test-zone2, test-zone1} 2022-09-03T06:23:32.350 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.351 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.352 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.352 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.353 DEBUG:boto:Data: 2022-09-03T06:23:32.353 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.354 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.355 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.355 DEBUG:boto:Params: {} 2022-09-03T06:23:32.356 DEBUG:boto:Token: None 2022-09-03T06:23:32.357 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.357 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.361 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000081cdfa1fec8abc4a-006312f2e4-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.362 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.363 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.363 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.364 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.365 DEBUG:boto:Data: 2022-09-03T06:23:32.366 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.368 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.369 DEBUG:boto:Params: {} 2022-09-03T06:23:32.370 DEBUG:boto:Token: None 2022-09-03T06:23:32.371 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.376 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a720f78b2e3667cc-006312f2e4-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.376 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:32.377 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.378 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.379 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.379 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.380 DEBUG:boto:Method: GET 2022-09-03T06:23:32.380 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.381 DEBUG:boto:Data: 2022-09-03T06:23:32.381 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.382 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.382 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.383 DEBUG:boto:Params: {} 2022-09-03T06:23:32.384 DEBUG:boto:Token: None 2022-09-03T06:23:32.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.389 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c75a4d18d5912f8d-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.390 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.391 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:32.392 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.392 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.393 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.393 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.394 DEBUG:boto:Method: GET 2022-09-03T06:23:32.394 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.395 DEBUG:boto:Data: 2022-09-03T06:23:32.396 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.396 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.397 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.397 DEBUG:boto:Params: {} 2022-09-03T06:23:32.398 DEBUG:boto:Token: None 2022-09-03T06:23:32.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.405 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f11d949e559ddac6-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.406 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.408 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.409 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.410 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.411 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.411 DEBUG:boto:Method: GET 2022-09-03T06:23:32.412 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.413 DEBUG:boto:Data: 2022-09-03T06:23:32.414 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.415 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.416 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.417 DEBUG:boto:Params: {} 2022-09-03T06:23:32.418 DEBUG:boto:Token: None 2022-09-03T06:23:32.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.425 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005f8d0dc3ccd2af43-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.426 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.427 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.428 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.428 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.429 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.429 DEBUG:boto:Method: GET 2022-09-03T06:23:32.430 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.430 DEBUG:boto:Data: 2022-09-03T06:23:32.431 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.431 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.432 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.432 DEBUG:boto:Params: {} 2022-09-03T06:23:32.433 DEBUG:boto:Token: None 2022-09-03T06:23:32.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.438 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006df4c69f0a667ffd-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.439 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.441 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-44 zones={test-zone2, test-zone1} 2022-09-03T06:23:32.441 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-44 zones={test-zone3, test-zone1} 2022-09-03T06:23:32.442 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.443 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.444 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.445 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.446 DEBUG:boto:Data: 2022-09-03T06:23:32.447 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.449 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.450 DEBUG:boto:Params: {} 2022-09-03T06:23:32.451 DEBUG:boto:Token: None 2022-09-03T06:23:32.452 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.457 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000030b0747a5e9d1026-006312f2e4-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.458 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.459 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.460 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.461 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.462 DEBUG:boto:Data: 2022-09-03T06:23:32.463 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.464 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.465 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.466 DEBUG:boto:Params: {} 2022-09-03T06:23:32.467 DEBUG:boto:Token: None 2022-09-03T06:23:32.468 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.474 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001c16479905145baf-006312f2e4-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.475 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:32.475 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.476 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.477 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.477 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.478 DEBUG:boto:Method: GET 2022-09-03T06:23:32.479 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.479 DEBUG:boto:Data: 2022-09-03T06:23:32.480 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.480 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.481 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.481 DEBUG:boto:Params: {} 2022-09-03T06:23:32.482 DEBUG:boto:Token: None 2022-09-03T06:23:32.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.487 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000000bcefcde553bc4e-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.488 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.489 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:32.490 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.491 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.492 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.493 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.494 DEBUG:boto:Method: GET 2022-09-03T06:23:32.495 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.496 DEBUG:boto:Data: 2022-09-03T06:23:32.497 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.497 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.499 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.502 DEBUG:boto:Params: {} 2022-09-03T06:23:32.503 DEBUG:boto:Token: None 2022-09-03T06:23:32.504 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.509 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e0bcf025a58d0c0-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.550 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.552 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.553 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.554 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.555 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.556 DEBUG:boto:Method: GET 2022-09-03T06:23:32.557 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.558 DEBUG:boto:Data: 2022-09-03T06:23:32.559 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.560 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.560 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.561 DEBUG:boto:Params: {} 2022-09-03T06:23:32.563 DEBUG:boto:Token: None 2022-09-03T06:23:32.564 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.565 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.566 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.569 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce930484f1d2c1d3-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.571 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.572 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.574 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.575 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.575 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.576 DEBUG:boto:Method: GET 2022-09-03T06:23:32.578 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.579 DEBUG:boto:Data: 2022-09-03T06:23:32.580 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.581 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.581 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.582 DEBUG:boto:Params: {} 2022-09-03T06:23:32.584 DEBUG:boto:Token: None 2022-09-03T06:23:32.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.591 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000043bc4d50efffd6d7-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.592 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.593 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-44 zones={test-zone3, test-zone1} 2022-09-03T06:23:32.594 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-44 zones={test-zone1, test-zone2} 2022-09-03T06:23:32.594 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.595 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.595 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.596 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.596 DEBUG:boto:Data: 2022-09-03T06:23:32.597 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.598 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.598 DEBUG:boto:Params: {} 2022-09-03T06:23:32.599 DEBUG:boto:Token: None 2022-09-03T06:23:32.600 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.605 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d3a4738b54ad3267-006312f2e4-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.605 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.606 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.607 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.607 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.608 DEBUG:boto:Data: 2022-09-03T06:23:32.608 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.609 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.609 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.610 DEBUG:boto:Params: {} 2022-09-03T06:23:32.610 DEBUG:boto:Token: None 2022-09-03T06:23:32.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.615 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004c4d93d3c1134a7f-006312f2e4-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.616 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:32.616 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.617 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.617 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.618 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.618 DEBUG:boto:Method: GET 2022-09-03T06:23:32.619 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.619 DEBUG:boto:Data: 2022-09-03T06:23:32.620 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.620 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.621 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.621 DEBUG:boto:Params: {} 2022-09-03T06:23:32.622 DEBUG:boto:Token: None 2022-09-03T06:23:32.623 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.627 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1469a573f66db5b-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.627 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.628 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:32.628 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.629 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.629 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.630 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.630 DEBUG:boto:Method: GET 2022-09-03T06:23:32.631 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.631 DEBUG:boto:Data: 2022-09-03T06:23:32.632 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.632 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.633 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.633 DEBUG:boto:Params: {} 2022-09-03T06:23:32.634 DEBUG:boto:Token: None 2022-09-03T06:23:32.635 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.638 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000f2b92f32ecb2473-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.686 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.687 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.688 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.688 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.689 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.689 DEBUG:boto:Method: GET 2022-09-03T06:23:32.690 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.690 DEBUG:boto:Data: 2022-09-03T06:23:32.691 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.692 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.692 DEBUG:boto:Params: {} 2022-09-03T06:23:32.693 DEBUG:boto:Token: None 2022-09-03T06:23:32.694 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.698 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000042465e952abf63f-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.699 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.700 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.700 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.701 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.701 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.702 DEBUG:boto:Method: GET 2022-09-03T06:23:32.703 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.703 DEBUG:boto:Data: 2022-09-03T06:23:32.704 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.704 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.705 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.705 DEBUG:boto:Params: {} 2022-09-03T06:23:32.706 DEBUG:boto:Token: None 2022-09-03T06:23:32.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.712 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c09a1868070b9c1c-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.713 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.713 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-44 zones={test-zone1, test-zone2} 2022-09-03T06:23:32.714 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-44 zones={test-zone3, test-zone2} 2022-09-03T06:23:32.714 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.715 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.716 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.716 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.717 DEBUG:boto:Data: 2022-09-03T06:23:32.717 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.718 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.719 DEBUG:boto:Params: {} 2022-09-03T06:23:32.719 DEBUG:boto:Token: None 2022-09-03T06:23:32.720 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.724 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000065c5b33115d98fcc-006312f2e4-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.725 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.725 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.726 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.727 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.727 DEBUG:boto:Data: 2022-09-03T06:23:32.728 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.728 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.729 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.730 DEBUG:boto:Params: {} 2022-09-03T06:23:32.730 DEBUG:boto:Token: None 2022-09-03T06:23:32.731 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.735 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000020ff183e13256110-006312f2e4-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.736 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:32.738 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.739 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.739 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.740 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.740 DEBUG:boto:Method: GET 2022-09-03T06:23:32.741 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.741 DEBUG:boto:Data: 2022-09-03T06:23:32.742 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.742 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.743 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.743 DEBUG:boto:Params: {} 2022-09-03T06:23:32.744 DEBUG:boto:Token: None 2022-09-03T06:23:32.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.749 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e5d2883daa969826-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.750 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.750 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:32.751 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.751 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.752 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.752 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.753 DEBUG:boto:Method: GET 2022-09-03T06:23:32.754 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.754 DEBUG:boto:Data: 2022-09-03T06:23:32.755 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.755 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.756 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.756 DEBUG:boto:Params: {} 2022-09-03T06:23:32.757 DEBUG:boto:Token: None 2022-09-03T06:23:32.758 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.762 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6e82cfd2e258295-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.762 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.763 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.764 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.764 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.765 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.765 DEBUG:boto:Method: GET 2022-09-03T06:23:32.766 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.766 DEBUG:boto:Data: 2022-09-03T06:23:32.767 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.768 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.769 DEBUG:boto:Params: {} 2022-09-03T06:23:32.769 DEBUG:boto:Token: None 2022-09-03T06:23:32.770 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.774 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dcfe3c7e7a75ab3c-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.775 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.776 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.776 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.777 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.777 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.778 DEBUG:boto:Method: GET 2022-09-03T06:23:32.778 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.779 DEBUG:boto:Data: 2022-09-03T06:23:32.780 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.780 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.781 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.781 DEBUG:boto:Params: {} 2022-09-03T06:23:32.782 DEBUG:boto:Token: None 2022-09-03T06:23:32.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.786 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a951f36ce143df60-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.787 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.787 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-44 zones={test-zone3, test-zone2} 2022-09-03T06:23:32.788 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-44 zones={test-zone1, test-zone3} 2022-09-03T06:23:32.789 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.789 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.790 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.790 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.791 DEBUG:boto:Data: 2022-09-03T06:23:32.791 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.792 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.793 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.793 DEBUG:boto:Params: {} 2022-09-03T06:23:32.794 DEBUG:boto:Token: None 2022-09-03T06:23:32.795 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.799 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f09115dac8232b33-006312f2e4-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.799 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.800 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.800 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.801 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.801 DEBUG:boto:Data: 2022-09-03T06:23:32.802 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.802 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.803 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.803 DEBUG:boto:Params: {} 2022-09-03T06:23:32.804 DEBUG:boto:Token: None 2022-09-03T06:23:32.805 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.808 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000afcf90e9892a2185-006312f2e4-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.809 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:32.809 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.810 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.810 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.811 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.812 DEBUG:boto:Method: GET 2022-09-03T06:23:32.812 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.813 DEBUG:boto:Data: 2022-09-03T06:23:32.813 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.814 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.815 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.815 DEBUG:boto:Params: {} 2022-09-03T06:23:32.816 DEBUG:boto:Token: None 2022-09-03T06:23:32.817 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.821 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000057feb16ac46a6ac9-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.866 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.867 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:32.868 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.868 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.869 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.869 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.870 DEBUG:boto:Method: GET 2022-09-03T06:23:32.870 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.871 DEBUG:boto:Data: 2022-09-03T06:23:32.871 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.873 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.873 DEBUG:boto:Params: {} 2022-09-03T06:23:32.874 DEBUG:boto:Token: None 2022-09-03T06:23:32.875 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.875 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.876 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.879 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f0eff4276f509331-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.879 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.880 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.881 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.881 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.882 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.882 DEBUG:boto:Method: GET 2022-09-03T06:23:32.883 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.883 DEBUG:boto:Data: 2022-09-03T06:23:32.884 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:32.885 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.886 DEBUG:boto:Params: {} 2022-09-03T06:23:32.886 DEBUG:boto:Token: None 2022-09-03T06:23:32.887 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.891 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006da9ea79a61bac59-006312f2e4-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.892 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.892 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.893 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.894 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.894 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.895 DEBUG:boto:Method: GET 2022-09-03T06:23:32.895 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.896 DEBUG:boto:Data: 2022-09-03T06:23:32.896 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.897 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.897 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.898 DEBUG:boto:Params: {} 2022-09-03T06:23:32.898 DEBUG:boto:Token: None 2022-09-03T06:23:32.899 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.900 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.903 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009480af64c6f5c610-006312f2e4-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.950 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.951 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-44 zones={test-zone1, test-zone3} 2022-09-03T06:23:32.952 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-44 zones={test-zone2, test-zone3} 2022-09-03T06:23:32.952 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.953 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.953 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.954 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.955 DEBUG:boto:Data: 2022-09-03T06:23:32.955 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.956 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.956 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.957 DEBUG:boto:Params: {} 2022-09-03T06:23:32.957 DEBUG:boto:Token: None 2022-09-03T06:23:32.958 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.962 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003e7a8f8046dd0007-006312f2e4-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.962 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.963 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.964 DEBUG:boto:Method: HEAD 2022-09-03T06:23:32.964 DEBUG:boto:Path: /avysmw-44/ 2022-09-03T06:23:32.965 DEBUG:boto:Data: 2022-09-03T06:23:32.966 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:32.967 DEBUG:boto:Port: 8001 2022-09-03T06:23:32.968 DEBUG:boto:Params: {} 2022-09-03T06:23:32.969 DEBUG:boto:Token: None 2022-09-03T06:23:32.970 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/ 2022-09-03T06:23:32.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY= 2022-09-03T06:23:32.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4ke02TaQ/qBvhzc/LOn4ZCJHMY=', 'Content-Length': '0'} 2022-09-03T06:23:32.976 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f3c22eacd407d12d-006312f2e4-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.977 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:32.978 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.978 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:32.979 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.980 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.981 DEBUG:boto:Method: GET 2022-09-03T06:23:32.982 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:32.983 DEBUG:boto:Data: 2022-09-03T06:23:32.984 DEBUG:boto:Headers: {} 2022-09-03T06:23:32.985 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:32.986 DEBUG:boto:Port: 8000 2022-09-03T06:23:32.987 DEBUG:boto:Params: {} 2022-09-03T06:23:32.988 DEBUG:boto:Token: None 2022-09-03T06:23:32.989 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:32 GMT /avysmw-44/?versions 2022-09-03T06:23:32.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0= 2022-09-03T06:23:32.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPB//axB9Xbp4y8tMhWNBG6LfE0=', 'Content-Length': '0'} 2022-09-03T06:23:32.993 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a00f32888b432653-006312f2e4-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:32.996 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:32.997 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:32.998 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:32.999 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:33.000 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:33.001 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:33.002 DEBUG:boto:Method: GET 2022-09-03T06:23:33.003 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:33.003 DEBUG:boto:Data: 2022-09-03T06:23:33.004 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.005 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.006 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.007 DEBUG:boto:Params: {} 2022-09-03T06:23:33.008 DEBUG:boto:Token: None 2022-09-03T06:23:33.009 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-44/?versions 2022-09-03T06:23:33.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S2lOWONZbReHXwwwF/xDLz0+oUM= 2022-09-03T06:23:33.011 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S2lOWONZbReHXwwwF/xDLz0+oUM=', 'Content-Length': '0'} 2022-09-03T06:23:33.014 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003cd230dc23c3b6ec-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.015 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:33.015 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:33.016 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:33.017 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:33.017 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:33.020 DEBUG:boto:Method: GET 2022-09-03T06:23:33.021 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:33.022 DEBUG:boto:Data: 2022-09-03T06:23:33.022 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.023 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.023 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.024 DEBUG:boto:Params: {} 2022-09-03T06:23:33.025 DEBUG:boto:Token: None 2022-09-03T06:23:33.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-44/?versions 2022-09-03T06:23:33.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S2lOWONZbReHXwwwF/xDLz0+oUM= 2022-09-03T06:23:33.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S2lOWONZbReHXwwwF/xDLz0+oUM=', 'Content-Length': '0'} 2022-09-03T06:23:33.029 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001eaeb6cbc60b9c17-006312f2e5-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.030 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:33.031 DEBUG:boto:path=/avysmw-44/ 2022-09-03T06:23:33.032 DEBUG:boto:auth_path=/avysmw-44/ 2022-09-03T06:23:33.032 DEBUG:boto:path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:33.033 DEBUG:boto:auth_path=/avysmw-44/?versions&max-keys=999 2022-09-03T06:23:33.033 DEBUG:boto:Method: GET 2022-09-03T06:23:33.034 DEBUG:boto:Path: /avysmw-44/?versions&max-keys=999 2022-09-03T06:23:33.034 DEBUG:boto:Data: 2022-09-03T06:23:33.035 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.036 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.036 DEBUG:boto:Params: {} 2022-09-03T06:23:33.037 DEBUG:boto:Token: None 2022-09-03T06:23:33.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-44/?versions 2022-09-03T06:23:33.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S2lOWONZbReHXwwwF/xDLz0+oUM= 2022-09-03T06:23:33.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S2lOWONZbReHXwwwF/xDLz0+oUM=', 'Content-Length': '0'} 2022-09-03T06:23:33.042 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da9e5e98bc3313f6-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.043 DEBUG:boto:b'avysmw-44999false' 2022-09-03T06:23:33.043 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-44 zones={test-zone2, test-zone3} 2022-09-03T06:23:33.044 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-45 zones={test-zone2, test-zone1} 2022-09-03T06:23:33.044 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.045 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.045 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.046 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.046 DEBUG:boto:Data: 2022-09-03T06:23:33.047 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.047 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.048 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.049 DEBUG:boto:Params: {} 2022-09-03T06:23:33.049 DEBUG:boto:Token: None 2022-09-03T06:23:33.050 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.054 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006df0a8dc6dfb5e40-006312f2e5-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.054 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.055 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.056 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.056 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.057 DEBUG:boto:Data: 2022-09-03T06:23:33.057 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.058 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.058 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.059 DEBUG:boto:Params: {} 2022-09-03T06:23:33.059 DEBUG:boto:Token: None 2022-09-03T06:23:33.060 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.064 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006320d7cdf5ee7978-006312f2e5-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.065 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:33.066 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.067 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.067 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.068 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.068 DEBUG:boto:Method: GET 2022-09-03T06:23:33.069 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.069 DEBUG:boto:Data: 2022-09-03T06:23:33.070 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.070 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.071 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.072 DEBUG:boto:Params: {} 2022-09-03T06:23:33.074 DEBUG:boto:Token: None 2022-09-03T06:23:33.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.080 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c674db1429e5ebd5-006312f2e5-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.081 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.083 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:33.083 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.084 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.085 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.085 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.086 DEBUG:boto:Method: GET 2022-09-03T06:23:33.086 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.087 DEBUG:boto:Data: 2022-09-03T06:23:33.087 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.088 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.088 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.089 DEBUG:boto:Params: {} 2022-09-03T06:23:33.089 DEBUG:boto:Token: None 2022-09-03T06:23:33.090 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.094 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c098d6f2e22f8fa3-006312f2e5-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.095 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.095 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.096 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.097 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.097 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.098 DEBUG:boto:Method: GET 2022-09-03T06:23:33.098 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.099 DEBUG:boto:Data: 2022-09-03T06:23:33.099 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.100 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.100 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.101 DEBUG:boto:Params: {} 2022-09-03T06:23:33.101 DEBUG:boto:Token: None 2022-09-03T06:23:33.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.106 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009de6c6fc3c8b27ff-006312f2e5-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.106 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.107 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.108 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.108 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.109 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.109 DEBUG:boto:Method: GET 2022-09-03T06:23:33.110 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.110 DEBUG:boto:Data: 2022-09-03T06:23:33.111 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.112 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.113 DEBUG:boto:Params: {} 2022-09-03T06:23:33.113 DEBUG:boto:Token: None 2022-09-03T06:23:33.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.118 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005208e34c42e3345b-006312f2e5-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.119 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.120 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-45 zones={test-zone2, test-zone1} 2022-09-03T06:23:33.121 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-45 zones={test-zone3, test-zone1} 2022-09-03T06:23:33.122 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.123 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.124 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.124 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.125 DEBUG:boto:Data: 2022-09-03T06:23:33.125 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.126 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.126 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.127 DEBUG:boto:Params: {} 2022-09-03T06:23:33.128 DEBUG:boto:Token: None 2022-09-03T06:23:33.128 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.129 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.132 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000266ce11d52ab9417-006312f2e5-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.133 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.133 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.134 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.134 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.135 DEBUG:boto:Data: 2022-09-03T06:23:33.135 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.136 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.136 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.137 DEBUG:boto:Params: {} 2022-09-03T06:23:33.137 DEBUG:boto:Token: None 2022-09-03T06:23:33.138 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.141 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000516f306b6fec5bb3-006312f2e5-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.142 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:33.143 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.143 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.144 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.144 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.145 DEBUG:boto:Method: GET 2022-09-03T06:23:33.145 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.146 DEBUG:boto:Data: 2022-09-03T06:23:33.146 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.147 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.147 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.148 DEBUG:boto:Params: {} 2022-09-03T06:23:33.149 DEBUG:boto:Token: None 2022-09-03T06:23:33.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.153 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7a7b057a5b96f22-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.154 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.155 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:33.155 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.156 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.157 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.157 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.158 DEBUG:boto:Method: GET 2022-09-03T06:23:33.159 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.160 DEBUG:boto:Data: 2022-09-03T06:23:33.161 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.163 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.163 DEBUG:boto:Params: {} 2022-09-03T06:23:33.164 DEBUG:boto:Token: None 2022-09-03T06:23:33.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.170 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000082348ca6872f408-006312f2e5-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.214 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.216 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.217 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.218 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.218 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.219 DEBUG:boto:Method: GET 2022-09-03T06:23:33.219 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.220 DEBUG:boto:Data: 2022-09-03T06:23:33.221 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.222 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.223 DEBUG:boto:Params: {} 2022-09-03T06:23:33.223 DEBUG:boto:Token: None 2022-09-03T06:23:33.224 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.229 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b4737469bdf4aaed-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.230 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.230 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.231 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.231 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.232 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.232 DEBUG:boto:Method: GET 2022-09-03T06:23:33.233 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.233 DEBUG:boto:Data: 2022-09-03T06:23:33.234 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.235 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.235 DEBUG:boto:Params: {} 2022-09-03T06:23:33.236 DEBUG:boto:Token: None 2022-09-03T06:23:33.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.242 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003cfd00229834b4f9-006312f2e5-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.243 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.244 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-45 zones={test-zone3, test-zone1} 2022-09-03T06:23:33.245 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-45 zones={test-zone1, test-zone2} 2022-09-03T06:23:33.246 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.246 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.247 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.247 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.248 DEBUG:boto:Data: 2022-09-03T06:23:33.249 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.250 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.251 DEBUG:boto:Params: {} 2022-09-03T06:23:33.251 DEBUG:boto:Token: None 2022-09-03T06:23:33.252 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.256 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009f7e99bf6347deff-006312f2e5-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.257 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.258 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.258 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.259 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.259 DEBUG:boto:Data: 2022-09-03T06:23:33.260 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.260 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.261 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.262 DEBUG:boto:Params: {} 2022-09-03T06:23:33.263 DEBUG:boto:Token: None 2022-09-03T06:23:33.264 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.269 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a2d94c58c4e40f4f-006312f2e5-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.270 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:33.270 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.271 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.272 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.273 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.273 DEBUG:boto:Method: GET 2022-09-03T06:23:33.274 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.274 DEBUG:boto:Data: 2022-09-03T06:23:33.275 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.275 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.276 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.277 DEBUG:boto:Params: {} 2022-09-03T06:23:33.278 DEBUG:boto:Token: None 2022-09-03T06:23:33.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.284 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058c5172303d0fac4-006312f2e5-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.285 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.286 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:33.286 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.287 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.287 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.288 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.289 DEBUG:boto:Method: GET 2022-09-03T06:23:33.289 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.290 DEBUG:boto:Data: 2022-09-03T06:23:33.290 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.292 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.293 DEBUG:boto:Params: {} 2022-09-03T06:23:33.294 DEBUG:boto:Token: None 2022-09-03T06:23:33.295 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003249f609e248f3c5-006312f2e5-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.301 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.302 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.303 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.303 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.304 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.304 DEBUG:boto:Method: GET 2022-09-03T06:23:33.305 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.305 DEBUG:boto:Data: 2022-09-03T06:23:33.306 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.306 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.307 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.307 DEBUG:boto:Params: {} 2022-09-03T06:23:33.308 DEBUG:boto:Token: None 2022-09-03T06:23:33.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.313 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a4418b47053de44-006312f2e5-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.314 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.315 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.316 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.316 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.317 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.317 DEBUG:boto:Method: GET 2022-09-03T06:23:33.318 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.318 DEBUG:boto:Data: 2022-09-03T06:23:33.319 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.319 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.320 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.320 DEBUG:boto:Params: {} 2022-09-03T06:23:33.321 DEBUG:boto:Token: None 2022-09-03T06:23:33.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.325 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e8db1d0d6f4c0ff-006312f2e5-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.326 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.327 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-45 zones={test-zone1, test-zone2} 2022-09-03T06:23:33.327 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-45 zones={test-zone3, test-zone2} 2022-09-03T06:23:33.328 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.328 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.329 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.329 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.330 DEBUG:boto:Data: 2022-09-03T06:23:33.330 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.331 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.331 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.332 DEBUG:boto:Params: {} 2022-09-03T06:23:33.332 DEBUG:boto:Token: None 2022-09-03T06:23:33.333 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.337 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000046a39af6a3983dce-006312f2e5-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.338 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.338 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.339 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.339 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.340 DEBUG:boto:Data: 2022-09-03T06:23:33.340 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.341 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.341 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.342 DEBUG:boto:Params: {} 2022-09-03T06:23:33.342 DEBUG:boto:Token: None 2022-09-03T06:23:33.343 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.343 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.347 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000be130c43b0cb27d0-006312f2e5-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.347 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:33.348 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.348 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.349 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.349 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.350 DEBUG:boto:Method: GET 2022-09-03T06:23:33.350 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.351 DEBUG:boto:Data: 2022-09-03T06:23:33.351 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.353 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.353 DEBUG:boto:Params: {} 2022-09-03T06:23:33.354 DEBUG:boto:Token: None 2022-09-03T06:23:33.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.360 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e87ac6d80741c309-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.361 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.362 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:33.363 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.363 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.364 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.364 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.365 DEBUG:boto:Method: GET 2022-09-03T06:23:33.365 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.366 DEBUG:boto:Data: 2022-09-03T06:23:33.366 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.367 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.367 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.368 DEBUG:boto:Params: {} 2022-09-03T06:23:33.368 DEBUG:boto:Token: None 2022-09-03T06:23:33.369 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.373 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003286bb0b0ca9694d-006312f2e5-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.418 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.419 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.420 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.420 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.421 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.421 DEBUG:boto:Method: GET 2022-09-03T06:23:33.422 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.422 DEBUG:boto:Data: 2022-09-03T06:23:33.423 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.424 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.424 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.425 DEBUG:boto:Params: {} 2022-09-03T06:23:33.425 DEBUG:boto:Token: None 2022-09-03T06:23:33.426 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.427 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.430 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000226899f1f903fc8b-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.431 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.433 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.434 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.435 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.435 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.436 DEBUG:boto:Method: GET 2022-09-03T06:23:33.437 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.438 DEBUG:boto:Data: 2022-09-03T06:23:33.439 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.440 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.441 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.442 DEBUG:boto:Params: {} 2022-09-03T06:23:33.443 DEBUG:boto:Token: None 2022-09-03T06:23:33.444 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.450 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d0bc8773cfd2ea21-006312f2e5-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.451 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.452 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-45 zones={test-zone3, test-zone2} 2022-09-03T06:23:33.453 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-45 zones={test-zone1, test-zone3} 2022-09-03T06:23:33.454 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.455 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.456 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.457 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.457 DEBUG:boto:Data: 2022-09-03T06:23:33.458 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.459 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.460 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.461 DEBUG:boto:Params: {} 2022-09-03T06:23:33.462 DEBUG:boto:Token: None 2022-09-03T06:23:33.463 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.469 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001c217cd3448eae24-006312f2e5-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.470 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.471 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.472 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.473 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.474 DEBUG:boto:Data: 2022-09-03T06:23:33.475 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.476 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.477 DEBUG:boto:Params: {} 2022-09-03T06:23:33.478 DEBUG:boto:Token: None 2022-09-03T06:23:33.479 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.484 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000047b3c54bb7edfa77-006312f2e5-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.486 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:33.486 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.487 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.488 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.489 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.490 DEBUG:boto:Method: GET 2022-09-03T06:23:33.490 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.491 DEBUG:boto:Data: 2022-09-03T06:23:33.491 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.492 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.492 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.493 DEBUG:boto:Params: {} 2022-09-03T06:23:33.494 DEBUG:boto:Token: None 2022-09-03T06:23:33.495 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.500 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001849627f129dd045-006312f2e5-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.501 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.502 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:33.504 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.504 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.505 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.505 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.506 DEBUG:boto:Method: GET 2022-09-03T06:23:33.507 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.507 DEBUG:boto:Data: 2022-09-03T06:23:33.508 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.509 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.509 DEBUG:boto:Params: {} 2022-09-03T06:23:33.510 DEBUG:boto:Token: None 2022-09-03T06:23:33.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.514 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dafc8206838a5a7d-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.515 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.516 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.517 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.517 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.518 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.518 DEBUG:boto:Method: GET 2022-09-03T06:23:33.519 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.519 DEBUG:boto:Data: 2022-09-03T06:23:33.520 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:33.521 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.521 DEBUG:boto:Params: {} 2022-09-03T06:23:33.522 DEBUG:boto:Token: None 2022-09-03T06:23:33.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.527 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000557a9eff74d012a1-006312f2e5-1444-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.528 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.528 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.529 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.529 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.530 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.530 DEBUG:boto:Method: GET 2022-09-03T06:23:33.531 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.531 DEBUG:boto:Data: 2022-09-03T06:23:33.532 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.533 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.533 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.534 DEBUG:boto:Params: {} 2022-09-03T06:23:33.535 DEBUG:boto:Token: None 2022-09-03T06:23:33.535 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.536 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.539 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000af066a04e01df52a-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.582 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.583 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-45 zones={test-zone1, test-zone3} 2022-09-03T06:23:33.584 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-45 zones={test-zone2, test-zone3} 2022-09-03T06:23:33.584 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.586 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.586 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.587 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.587 DEBUG:boto:Data: 2022-09-03T06:23:33.588 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.588 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.589 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.589 DEBUG:boto:Params: {} 2022-09-03T06:23:33.590 DEBUG:boto:Token: None 2022-09-03T06:23:33.591 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.594 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004f431a22bfc68d5e-006312f2e5-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.595 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.596 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.597 DEBUG:boto:Method: HEAD 2022-09-03T06:23:33.597 DEBUG:boto:Path: /avysmw-45/ 2022-09-03T06:23:33.598 DEBUG:boto:Data: 2022-09-03T06:23:33.598 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.599 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.600 DEBUG:boto:Params: {} 2022-09-03T06:23:33.600 DEBUG:boto:Token: None 2022-09-03T06:23:33.601 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/ 2022-09-03T06:23:33.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw= 2022-09-03T06:23:33.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SjOmYn0cZYVp7UONtmoLC0ou3Kw=', 'Content-Length': '0'} 2022-09-03T06:23:33.605 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001e02816c74f44f4b-006312f2e5-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.605 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:23:33.606 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.606 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.607 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.607 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.608 DEBUG:boto:Method: GET 2022-09-03T06:23:33.608 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.609 DEBUG:boto:Data: 2022-09-03T06:23:33.609 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.610 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.611 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.611 DEBUG:boto:Params: {} 2022-09-03T06:23:33.612 DEBUG:boto:Token: None 2022-09-03T06:23:33.612 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.617 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000523aeb5deb7a7e1d-006312f2e5-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.618 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.619 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:23:33.619 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.620 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.620 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.621 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.621 DEBUG:boto:Method: GET 2022-09-03T06:23:33.622 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.622 DEBUG:boto:Data: 2022-09-03T06:23:33.623 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.623 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.624 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.625 DEBUG:boto:Params: {} 2022-09-03T06:23:33.625 DEBUG:boto:Token: None 2022-09-03T06:23:33.626 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.631 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006041372a32161b1d-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.632 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.633 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.634 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.635 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.635 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.636 DEBUG:boto:Method: GET 2022-09-03T06:23:33.636 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.637 DEBUG:boto:Data: 2022-09-03T06:23:33.637 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.638 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:33.638 DEBUG:boto:Port: 8000 2022-09-03T06:23:33.639 DEBUG:boto:Params: {} 2022-09-03T06:23:33.640 DEBUG:boto:Token: None 2022-09-03T06:23:33.640 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.644 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000351dc101945e48b6-006312f2e5-1590-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.645 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.646 DEBUG:boto:path=/avysmw-45/ 2022-09-03T06:23:33.646 DEBUG:boto:auth_path=/avysmw-45/ 2022-09-03T06:23:33.647 DEBUG:boto:path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.647 DEBUG:boto:auth_path=/avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.648 DEBUG:boto:Method: GET 2022-09-03T06:23:33.648 DEBUG:boto:Path: /avysmw-45/?versions&max-keys=999 2022-09-03T06:23:33.649 DEBUG:boto:Data: 2022-09-03T06:23:33.649 DEBUG:boto:Headers: {} 2022-09-03T06:23:33.650 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:33.650 DEBUG:boto:Port: 8001 2022-09-03T06:23:33.651 DEBUG:boto:Params: {} 2022-09-03T06:23:33.651 DEBUG:boto:Token: None 2022-09-03T06:23:33.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:23:33 GMT /avysmw-45/?versions 2022-09-03T06:23:33.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI= 2022-09-03T06:23:33.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R68ggnR86e2j/Gbk92ajZ4d0fwI=', 'Content-Length': '0'} 2022-09-03T06:23:33.656 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000baa9420211b6e73b-006312f2e5-144a-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:23:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:33.656 DEBUG:boto:b'avysmw-45999false' 2022-09-03T06:23:33.657 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-45 zones={test-zone2, test-zone3} 2022-09-03T06:23:33.658 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:33.658 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:33.659 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period b84ccf39-78f5-4998-9db6-42362bc8c98b --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:33.716 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:33.717 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:33.759 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:33.760 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.760 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184222.657692_58.1", 2022-09-03T06:23:33.761 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "user", 2022-09-03T06:23:33.761 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "rgw-multisite-test-user", 2022-09-03T06:23:33.762 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:50:22.657692624Z", 2022-09-03T06:23:33.762 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.763 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.763 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.764 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.764 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.765 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.765 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6BN_OEAtimzMqhhbhGc7xsF", 2022-09-03T06:23:33.766 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.766 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.767 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.767 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.767 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.768 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.768 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.769 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.769 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184222.662031_59.1", 2022-09-03T06:23:33.769 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "user", 2022-09-03T06:23:33.770 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "rgw-multisite-test-user", 2022-09-03T06:23:33.770 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:50:22.662031116Z", 2022-09-03T06:23:33.770 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.770 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.771 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.772 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.772 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.772 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.773 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6BN_OEAtimzMqhhbhGc7xsF", 2022-09-03T06:23:33.773 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.774 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.774 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.774 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.775 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.775 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.775 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.775 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.776 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.219295_62.1", 2022-09-03T06:23:33.776 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.776 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1", 2022-09-03T06:23:33.777 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.219295443Z", 2022-09-03T06:23:33.777 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.777 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.777 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.778 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.778 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.778 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.779 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Y7JDAo_bsL14HyXDq4vsLc2", 2022-09-03T06:23:33.779 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.779 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.779 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.780 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.780 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.780 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.780 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.781 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.781 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.222658_63.1", 2022-09-03T06:23:33.782 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.782 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1", 2022-09-03T06:23:33.783 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.222658346Z", 2022-09-03T06:23:33.783 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.783 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.783 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.784 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.784 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.784 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.785 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Y7JDAo_bsL14HyXDq4vsLc2", 2022-09-03T06:23:33.785 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.785 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.785 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.786 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.786 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.787 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.787 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.788 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.788 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.223873_64.1", 2022-09-03T06:23:33.789 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.789 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-1", 2022-09-03T06:23:33.790 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.223873908Z", 2022-09-03T06:23:33.790 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.791 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.791 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.792 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.793 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.793 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.793 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_VlD_M0myhVj3j7F4hjNPDp1", 2022-09-03T06:23:33.794 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.794 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.795 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.795 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.795 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.796 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.796 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.796 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.797 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.226841_65.1", 2022-09-03T06:23:33.797 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.797 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-1", 2022-09-03T06:23:33.797 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.226841368Z", 2022-09-03T06:23:33.798 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.798 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.798 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.798 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.799 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.799 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.800 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_VlD_M0myhVj3j7F4hjNPDp1", 2022-09-03T06:23:33.800 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.800 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.801 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.801 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.801 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.801 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.802 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.802 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.802 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.252557_66.1", 2022-09-03T06:23:33.803 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.803 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2", 2022-09-03T06:23:33.804 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.252557878Z", 2022-09-03T06:23:33.804 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.804 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.804 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.805 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.805 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.805 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.805 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_eGqFRgXckOShhJ5K31PxBCB", 2022-09-03T06:23:33.806 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.806 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.806 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.807 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.807 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.808 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.808 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.808 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.808 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.258211_67.1", 2022-09-03T06:23:33.809 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.809 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2", 2022-09-03T06:23:33.809 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.258211354Z", 2022-09-03T06:23:33.809 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.810 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.810 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.810 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.810 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.811 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.811 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_eGqFRgXckOShhJ5K31PxBCB", 2022-09-03T06:23:33.811 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.811 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.812 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.812 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.812 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.812 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.813 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.813 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.814 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.262819_68.1", 2022-09-03T06:23:33.814 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.814 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-2", 2022-09-03T06:23:33.814 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.262819181Z", 2022-09-03T06:23:33.815 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.815 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.815 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.816 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.816 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.816 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.817 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lkn_J0TsdIPDWPcsSr8e5eP", 2022-09-03T06:23:33.817 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.817 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.817 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.818 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.818 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.818 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.819 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.819 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.819 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.267378_69.1", 2022-09-03T06:23:33.819 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.820 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-2", 2022-09-03T06:23:33.820 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.267378646Z", 2022-09-03T06:23:33.820 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.820 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.821 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.821 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.821 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.821 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.822 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lkn_J0TsdIPDWPcsSr8e5eP", 2022-09-03T06:23:33.822 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.822 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.822 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.823 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.823 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.823 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.824 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.825 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.825 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184342.345771_70.1", 2022-09-03T06:23:33.826 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.826 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3", 2022-09-03T06:23:33.827 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:22.345771278Z", 2022-09-03T06:23:33.827 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.828 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.828 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.829 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.829 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.830 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.830 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_eB0pPK7VCkudq9Jt1nUnfVO", 2022-09-03T06:23:33.831 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.831 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.831 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.832 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.832 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.833 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.833 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.834 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.834 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184342.350157_71.1", 2022-09-03T06:23:33.835 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.836 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3", 2022-09-03T06:23:33.836 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:22.350157638Z", 2022-09-03T06:23:33.836 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.837 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.837 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.838 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.838 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.839 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.839 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_eB0pPK7VCkudq9Jt1nUnfVO", 2022-09-03T06:23:33.839 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.840 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.840 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.840 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.841 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.841 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.841 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.842 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.842 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184342.351175_72.1", 2022-09-03T06:23:33.843 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.843 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-3", 2022-09-03T06:23:33.843 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:22.351175819Z", 2022-09-03T06:23:33.844 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.844 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.844 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.845 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.845 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.846 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.846 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_xQbDQL6LoF7KY81FCBsr-TR", 2022-09-03T06:23:33.846 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.846 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.847 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.847 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.847 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.847 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.848 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.848 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.848 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184342.357275_73.1", 2022-09-03T06:23:33.848 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.849 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-3", 2022-09-03T06:23:33.849 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:22.357275634Z", 2022-09-03T06:23:33.849 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.849 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.850 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.850 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.851 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.860 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.860 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_xQbDQL6LoF7KY81FCBsr-TR", 2022-09-03T06:23:33.861 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.861 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.862 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.862 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.863 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.863 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.864 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.864 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.865 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.193842_74.1", 2022-09-03T06:23:33.865 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.866 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4", 2022-09-03T06:23:33.866 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.193842881Z", 2022-09-03T06:23:33.867 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.867 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.868 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.868 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.869 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.869 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.869 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lw10L7BpVQy7VSU2U7UHl07", 2022-09-03T06:23:33.870 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.870 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.871 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.871 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.872 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.872 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.873 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.873 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.873 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.198704_75.1", 2022-09-03T06:23:33.873 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.874 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4", 2022-09-03T06:23:33.874 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.198704862Z", 2022-09-03T06:23:33.874 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.874 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.875 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.875 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.875 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.875 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.876 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lw10L7BpVQy7VSU2U7UHl07", 2022-09-03T06:23:33.876 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.876 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.877 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.877 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.877 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.878 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.878 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.878 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.878 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.200713_76.1", 2022-09-03T06:23:33.879 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.879 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-4", 2022-09-03T06:23:33.880 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.200713524Z", 2022-09-03T06:23:33.880 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.881 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.881 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.881 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.882 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.882 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.883 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2mGIDNrcwT5FJidyB-XgESn", 2022-09-03T06:23:33.883 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.884 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.884 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.884 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.885 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.885 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.885 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.886 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.886 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.204734_77.1", 2022-09-03T06:23:33.886 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.887 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-4", 2022-09-03T06:23:33.887 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.204734948Z", 2022-09-03T06:23:33.887 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.888 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.888 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.888 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.888 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.889 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.889 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2mGIDNrcwT5FJidyB-XgESn", 2022-09-03T06:23:33.889 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.889 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.890 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.890 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.890 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.890 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.891 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.891 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.891 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.224447_78.1", 2022-09-03T06:23:33.891 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.892 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5", 2022-09-03T06:23:33.892 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.224447947Z", 2022-09-03T06:23:33.892 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.892 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.893 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.893 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.893 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.894 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.894 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_xo2PU3_oh6eFpkPNkexfq7D", 2022-09-03T06:23:33.894 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.894 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.895 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.895 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.895 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.895 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.896 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.896 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.896 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.227473_79.1", 2022-09-03T06:23:33.896 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.897 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5", 2022-09-03T06:23:33.897 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.227473639Z", 2022-09-03T06:23:33.897 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.897 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.898 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.898 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.898 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.898 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.899 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_xo2PU3_oh6eFpkPNkexfq7D", 2022-09-03T06:23:33.899 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.899 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.899 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.900 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.900 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.900 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.900 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.901 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.901 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.228594_80.1", 2022-09-03T06:23:33.901 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.901 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-5", 2022-09-03T06:23:33.902 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.228594676Z", 2022-09-03T06:23:33.902 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.902 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.902 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.903 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.903 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.903 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.903 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_u7x8KJF8hnPDPr0uRDbixnY", 2022-09-03T06:23:33.904 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.904 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.904 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.904 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.905 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.905 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.905 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.905 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.906 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.231192_81.1", 2022-09-03T06:23:33.906 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.906 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-5", 2022-09-03T06:23:33.906 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.231192184Z", 2022-09-03T06:23:33.907 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.907 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.907 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.907 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.908 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.908 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.908 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_u7x8KJF8hnPDPr0uRDbixnY", 2022-09-03T06:23:33.908 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.909 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.909 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.909 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.909 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.909 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.910 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.910 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.910 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.255262_82.1", 2022-09-03T06:23:33.910 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.911 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6", 2022-09-03T06:23:33.911 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.255262164Z", 2022-09-03T06:23:33.911 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.911 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.912 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.912 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.912 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.912 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.913 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_7T2FDkJXu_-0lpyQJ4EERx0", 2022-09-03T06:23:33.913 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.913 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.913 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.914 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.914 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.914 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.914 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.915 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.915 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.258031_83.1", 2022-09-03T06:23:33.915 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.915 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6", 2022-09-03T06:23:33.916 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.258031985Z", 2022-09-03T06:23:33.916 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.916 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.916 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.917 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.917 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.917 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.917 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_7T2FDkJXu_-0lpyQJ4EERx0", 2022-09-03T06:23:33.917 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.918 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.918 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.918 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.918 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.919 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.919 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.919 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.919 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.259057_84.1", 2022-09-03T06:23:33.920 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.920 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-6", 2022-09-03T06:23:33.920 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.259057684Z", 2022-09-03T06:23:33.920 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.921 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.921 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.921 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.921 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.922 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.922 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_JcUmTGR0SXlRqG5W9Zl5D8p", 2022-09-03T06:23:33.922 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.922 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.923 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.923 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.923 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.923 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.923 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.924 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.924 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.261653_85.1", 2022-09-03T06:23:33.924 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.924 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-6", 2022-09-03T06:23:33.925 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.261653432Z", 2022-09-03T06:23:33.925 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.925 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.925 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.926 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.926 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.926 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.926 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_JcUmTGR0SXlRqG5W9Zl5D8p", 2022-09-03T06:23:33.927 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.927 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.927 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.927 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.928 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.928 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.928 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.928 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.929 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.855330_86.1", 2022-09-03T06:23:33.929 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.929 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:33.929 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.855330912Z", 2022-09-03T06:23:33.930 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.930 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.930 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.930 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.931 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.931 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.931 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:33.931 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.932 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.932 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.932 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.932 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.933 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.933 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.933 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.933 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.861502_87.1", 2022-09-03T06:23:33.934 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.934 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:33.934 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.861502176Z", 2022-09-03T06:23:33.934 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.935 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.935 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.935 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.935 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.936 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.936 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:33.936 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.936 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.937 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.937 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.937 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.937 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.938 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.938 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.938 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.862483_88.1", 2022-09-03T06:23:33.938 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.939 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7", 2022-09-03T06:23:33.939 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.862483283Z", 2022-09-03T06:23:33.939 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.939 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.939 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.940 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.940 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.940 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.940 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:33.941 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.941 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.941 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.941 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.942 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.942 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.942 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.942 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.943 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.867021_89.1", 2022-09-03T06:23:33.943 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.943 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7", 2022-09-03T06:23:33.943 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.867021552Z", 2022-09-03T06:23:33.944 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.944 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.944 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.944 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.945 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.945 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.945 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:33.945 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.946 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.946 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.946 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.946 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.947 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.947 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.947 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.947 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.888960_90.1", 2022-09-03T06:23:33.948 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.948 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:33.948 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.888960687Z", 2022-09-03T06:23:33.948 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.948 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.949 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.949 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.949 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.949 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.950 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:33.950 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.950 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.951 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.951 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.951 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.951 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.951 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.952 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.952 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.892373_91.1", 2022-09-03T06:23:33.952 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.952 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:33.953 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.892373633Z", 2022-09-03T06:23:33.953 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.953 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.953 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.954 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.954 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.954 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.954 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:33.955 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.955 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.955 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.956 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.956 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.956 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.956 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.957 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.957 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.893288_92.1", 2022-09-03T06:23:33.957 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.957 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8", 2022-09-03T06:23:33.958 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.893288080Z", 2022-09-03T06:23:33.958 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.958 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.958 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.959 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.959 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.959 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.959 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:33.959 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.960 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.960 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.960 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.960 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.961 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.961 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.961 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.961 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.895751_93.1", 2022-09-03T06:23:33.962 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.962 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8", 2022-09-03T06:23:33.962 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.895751491Z", 2022-09-03T06:23:33.962 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.963 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.963 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.963 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.963 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.964 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.964 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:33.964 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.964 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.964 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.965 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.965 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.965 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.965 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.966 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.966 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.922520_94.1", 2022-09-03T06:23:33.966 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.966 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:33.967 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.922520779Z", 2022-09-03T06:23:33.967 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.967 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.967 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.968 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.968 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.968 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.968 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:33.969 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.969 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.969 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.969 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.970 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.970 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.970 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.970 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.971 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.931081_95.1", 2022-09-03T06:23:33.971 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.971 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:33.971 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.931081374Z", 2022-09-03T06:23:33.972 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.972 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.972 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.972 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.973 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.973 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.973 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:33.973 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.974 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.974 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.974 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.974 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.975 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.975 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.975 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.975 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.933197_96.1", 2022-09-03T06:23:33.976 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.976 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9", 2022-09-03T06:23:33.976 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.933197726Z", 2022-09-03T06:23:33.976 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.977 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.977 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.977 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.977 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.978 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.978 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:33.978 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.978 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.979 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.979 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.979 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.979 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.980 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.980 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.980 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.938376_97.1", 2022-09-03T06:23:33.980 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.981 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9", 2022-09-03T06:23:33.981 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.938376383Z", 2022-09-03T06:23:33.981 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.981 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.982 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:33.982 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:33.982 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.982 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.983 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:33.983 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.983 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.983 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.984 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.984 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.984 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.984 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.985 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.985 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.570224_98.1", 2022-09-03T06:23:33.985 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.985 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:33.986 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.570224266Z", 2022-09-03T06:23:33.986 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.986 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.986 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:33.987 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.987 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.987 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.987 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:33.988 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:33.988 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.988 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.988 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:33.988 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.989 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.989 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.989 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.989 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.574509_99.1", 2022-09-03T06:23:33.990 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:33.990 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:33.990 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.574509413Z", 2022-09-03T06:23:33.990 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.991 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.991 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:33.991 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.991 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.992 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.992 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:33.992 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:33.992 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.993 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.993 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:33.993 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.993 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.993 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.994 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.994 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.584013_100.1", 2022-09-03T06:23:33.994 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.994 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7", 2022-09-03T06:23:33.995 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.584013358Z", 2022-09-03T06:23:33.995 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:33.995 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:33.996 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:33.996 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:33.996 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.996 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:33.997 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:33.997 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:33.997 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.997 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:33.997 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "remove" 2022-09-03T06:23:33.998 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.998 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:33.998 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:33.998 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:33.999 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.587599_101.1", 2022-09-03T06:23:33.999 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:33.999 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7", 2022-09-03T06:23:33.999 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.587599896Z", 2022-09-03T06:23:34.000 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.000 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.000 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:34.000 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.001 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.001 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.001 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:34.001 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.002 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.002 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.002 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.002 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.003 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.003 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.003 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.003 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.619234_102.1", 2022-09-03T06:23:34.003 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.004 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:34.004 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.619234589Z", 2022-09-03T06:23:34.004 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.004 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.005 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:34.005 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.005 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.005 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.006 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:34.006 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.006 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.006 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.007 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.007 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.007 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.007 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.008 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.008 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.625715_103.1", 2022-09-03T06:23:34.008 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.008 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:34.009 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.625715365Z", 2022-09-03T06:23:34.009 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.009 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.009 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:34.009 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.010 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.010 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.010 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:34.011 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.011 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.011 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.011 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.011 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.012 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.012 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.012 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.012 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.637706_104.1", 2022-09-03T06:23:34.013 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.013 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8", 2022-09-03T06:23:34.013 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.637706717Z", 2022-09-03T06:23:34.013 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.014 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.014 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:34.014 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.014 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.015 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.015 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:34.015 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.015 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.016 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.016 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "remove" 2022-09-03T06:23:34.016 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.016 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.017 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.017 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.017 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.639994_105.1", 2022-09-03T06:23:34.017 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.017 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8", 2022-09-03T06:23:34.018 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.639994804Z", 2022-09-03T06:23:34.018 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.018 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.019 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:34.019 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.019 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.019 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.019 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:34.020 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.020 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.020 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.020 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.021 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.021 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.021 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.021 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.022 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.697911_106.1", 2022-09-03T06:23:34.022 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.022 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:34.022 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.697911177Z", 2022-09-03T06:23:34.023 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.023 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.023 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:34.023 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.024 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.024 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.024 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:34.024 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.025 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.025 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.025 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.025 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.025 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.026 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.026 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.026 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.700455_107.1", 2022-09-03T06:23:34.026 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.027 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:34.030 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.700455333Z", 2022-09-03T06:23:34.030 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.030 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.031 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:34.031 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.031 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.031 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.032 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:34.032 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.032 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.032 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.033 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.033 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.033 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.033 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.034 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.034 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.712013_108.1", 2022-09-03T06:23:34.034 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.034 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9", 2022-09-03T06:23:34.035 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.712013759Z", 2022-09-03T06:23:34.035 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.035 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.035 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:34.036 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.036 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.036 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.036 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:34.037 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.037 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.037 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.037 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "remove" 2022-09-03T06:23:34.037 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.038 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.038 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.038 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.038 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.715807_109.1", 2022-09-03T06:23:34.039 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.039 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9", 2022-09-03T06:23:34.039 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807393Z", 2022-09-03T06:23:34.039 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.040 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.040 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:34.040 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.040 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.041 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.041 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:34.041 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.041 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.042 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.042 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.042 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.043 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.043 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.043 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.043 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.561415_64.1", 2022-09-03T06:23:34.043 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.044 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:23:34.044 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.561415905Z", 2022-09-03T06:23:34.044 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.044 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.045 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.045 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.045 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.045 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.046 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6ugH-NS-_bEeQKUdolNq7Oa", 2022-09-03T06:23:34.046 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.046 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.046 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.047 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.047 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.047 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.047 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.048 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.048 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.566930_66.1", 2022-09-03T06:23:34.048 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.048 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:23:34.049 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.566930651Z", 2022-09-03T06:23:34.049 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.049 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.049 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.050 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.050 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.050 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.050 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6ugH-NS-_bEeQKUdolNq7Oa", 2022-09-03T06:23:34.051 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.051 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.051 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.051 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.052 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.052 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.052 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.052 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.053 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.569237_67.1", 2022-09-03T06:23:34.053 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.053 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-10", 2022-09-03T06:23:34.053 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.569237568Z", 2022-09-03T06:23:34.054 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.054 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.054 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.054 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.055 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.055 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.055 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_htY0ij10LAX0ZNP6T_0mHNL", 2022-09-03T06:23:34.055 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.056 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.056 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.056 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.056 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.057 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.057 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.057 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.057 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.574987_68.1", 2022-09-03T06:23:34.058 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.058 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-10", 2022-09-03T06:23:34.058 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.574987235Z", 2022-09-03T06:23:34.058 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.059 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.059 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.059 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.059 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.060 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.060 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_htY0ij10LAX0ZNP6T_0mHNL", 2022-09-03T06:23:34.060 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.060 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.061 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.061 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.061 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.061 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.062 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.062 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.062 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.595637_69.1", 2022-09-03T06:23:34.063 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.063 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:23:34.063 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.595637832Z", 2022-09-03T06:23:34.063 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.063 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.064 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.064 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.064 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.064 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.065 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gtTH1QqAjuxzvv9DiFtpT7B", 2022-09-03T06:23:34.065 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.065 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.065 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.066 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.066 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.066 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.066 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.067 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.067 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.599901_70.1", 2022-09-03T06:23:34.067 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.067 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:23:34.068 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.599901361Z", 2022-09-03T06:23:34.068 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.068 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.068 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.069 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.069 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.069 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.069 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gtTH1QqAjuxzvv9DiFtpT7B", 2022-09-03T06:23:34.070 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.070 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.070 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.070 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.071 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.071 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.071 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.071 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.072 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.601762_71.1", 2022-09-03T06:23:34.072 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.072 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-11", 2022-09-03T06:23:34.072 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.601762485Z", 2022-09-03T06:23:34.073 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.073 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.073 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.073 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.073 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.074 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.074 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_umwMbGorUx0G8JCmxgLXcXQ", 2022-09-03T06:23:34.074 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.074 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.075 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.075 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.075 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.075 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.076 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.076 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.076 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.605809_72.1", 2022-09-03T06:23:34.076 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.077 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-11", 2022-09-03T06:23:34.077 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.605809267Z", 2022-09-03T06:23:34.077 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.077 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.078 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.078 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.078 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.078 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.079 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_umwMbGorUx0G8JCmxgLXcXQ", 2022-09-03T06:23:34.079 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.079 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.079 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.080 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.080 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.080 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.080 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.081 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.081 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.642680_73.1", 2022-09-03T06:23:34.081 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.081 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:23:34.082 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.642680630Z", 2022-09-03T06:23:34.082 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.082 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.082 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.083 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.083 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.083 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.083 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_pT_Dw9zXIA2-9UcP5Gn1GlK", 2022-09-03T06:23:34.084 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.084 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.084 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.084 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.085 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.085 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.085 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.085 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.086 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.645295_74.1", 2022-09-03T06:23:34.086 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.086 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:23:34.086 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.645295120Z", 2022-09-03T06:23:34.087 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.087 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.087 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.087 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.088 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.088 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.088 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_pT_Dw9zXIA2-9UcP5Gn1GlK", 2022-09-03T06:23:34.088 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.088 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.089 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.089 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.089 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.089 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.090 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.090 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.090 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.646235_75.1", 2022-09-03T06:23:34.090 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.091 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-12", 2022-09-03T06:23:34.091 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.646235577Z", 2022-09-03T06:23:34.091 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.091 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.092 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.092 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.092 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.092 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.092 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ig99b4m7-ssuB2Tw7htsmQp", 2022-09-03T06:23:34.093 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.093 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.093 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.093 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.094 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.094 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.094 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.094 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.095 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.648716_76.1", 2022-09-03T06:23:34.095 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.095 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-12", 2022-09-03T06:23:34.095 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.648716080Z", 2022-09-03T06:23:34.096 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.096 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.096 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.096 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.097 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.097 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.097 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ig99b4m7-ssuB2Tw7htsmQp", 2022-09-03T06:23:34.097 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.098 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.098 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.098 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.098 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.099 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.099 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.099 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.099 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.356235_99.1", 2022-09-03T06:23:34.100 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.100 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:23:34.100 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.356235827Z", 2022-09-03T06:23:34.100 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.101 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.101 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.101 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.101 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.101 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.102 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8iK3rbCB7fbjpNaGZ4Lk2f9", 2022-09-03T06:23:34.102 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.102 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.102 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.103 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.103 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.103 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.103 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.104 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.104 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.364532_100.1", 2022-09-03T06:23:34.104 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.104 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:23:34.105 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.364532891Z", 2022-09-03T06:23:34.105 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.105 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.106 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.106 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.106 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.106 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.107 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8iK3rbCB7fbjpNaGZ4Lk2f9", 2022-09-03T06:23:34.107 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.107 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.107 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.108 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.108 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.108 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.108 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.109 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.109 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.365724_101.1", 2022-09-03T06:23:34.109 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.109 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-13", 2022-09-03T06:23:34.110 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.365724056Z", 2022-09-03T06:23:34.110 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.110 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.110 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.111 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.111 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.111 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.111 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_hGY9xi1--Qa5DBRaR1Ev2Td", 2022-09-03T06:23:34.112 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.112 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.112 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.112 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.112 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.113 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.113 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.113 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.113 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.372032_102.1", 2022-09-03T06:23:34.114 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.114 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-13", 2022-09-03T06:23:34.114 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.372032718Z", 2022-09-03T06:23:34.114 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.115 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.115 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.115 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.115 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.116 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.116 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_hGY9xi1--Qa5DBRaR1Ev2Td", 2022-09-03T06:23:34.116 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.116 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.117 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.117 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.117 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.117 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.118 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.118 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.118 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.400096_103.1", 2022-09-03T06:23:34.119 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.119 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:23:34.119 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.400096996Z", 2022-09-03T06:23:34.119 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.120 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.120 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.120 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.120 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.121 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.121 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_CLeQ9451sE39DW-o5-rQ9Gb", 2022-09-03T06:23:34.121 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.121 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.122 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.122 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.122 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.122 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.123 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.123 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.123 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.406131_104.1", 2022-09-03T06:23:34.123 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.124 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:23:34.124 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.406131833Z", 2022-09-03T06:23:34.124 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.124 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.125 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.125 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.125 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.126 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.126 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_CLeQ9451sE39DW-o5-rQ9Gb", 2022-09-03T06:23:34.126 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.126 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.127 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.127 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.127 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.127 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.128 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.128 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.128 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.407235_105.1", 2022-09-03T06:23:34.128 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.129 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-14", 2022-09-03T06:23:34.129 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.407235283Z", 2022-09-03T06:23:34.129 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.129 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.129 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.130 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.130 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.130 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.130 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_E3QteIhTh8XKCT60Fs2visZ", 2022-09-03T06:23:34.131 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.131 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.131 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.131 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.132 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.132 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.132 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.132 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.133 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.413053_106.1", 2022-09-03T06:23:34.133 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.133 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-14", 2022-09-03T06:23:34.133 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.413053852Z", 2022-09-03T06:23:34.134 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.134 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.134 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.134 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.135 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.135 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.135 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_E3QteIhTh8XKCT60Fs2visZ", 2022-09-03T06:23:34.135 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.136 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.136 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.136 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.136 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.136 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.137 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.137 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.137 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.447727_107.1", 2022-09-03T06:23:34.137 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.138 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:23:34.138 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.447727339Z", 2022-09-03T06:23:34.138 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.138 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.139 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.139 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.139 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.139 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.140 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6mYNO_dnwhvNSPaHRDsVWkL", 2022-09-03T06:23:34.140 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.140 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.140 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.141 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.141 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.141 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.141 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.142 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.142 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.450607_108.1", 2022-09-03T06:23:34.142 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.142 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:23:34.143 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.450607653Z", 2022-09-03T06:23:34.143 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.143 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.143 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.143 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.144 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.144 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.144 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6mYNO_dnwhvNSPaHRDsVWkL", 2022-09-03T06:23:34.144 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.145 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.145 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.145 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.145 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.146 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.146 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.146 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.146 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.451654_109.1", 2022-09-03T06:23:34.147 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.147 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-15", 2022-09-03T06:23:34.147 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.451654129Z", 2022-09-03T06:23:34.147 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.148 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.148 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.148 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.148 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.149 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.149 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_iMvYVCIRH1guRCrQSTbYt62", 2022-09-03T06:23:34.149 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.149 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.150 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.150 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.150 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.150 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.151 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.151 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.151 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.454511_110.1", 2022-09-03T06:23:34.151 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.151 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-15", 2022-09-03T06:23:34.152 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.454511694Z", 2022-09-03T06:23:34.152 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.152 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.152 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.153 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.153 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.153 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.153 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_iMvYVCIRH1guRCrQSTbYt62", 2022-09-03T06:23:34.154 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.154 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.154 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.154 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.155 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.155 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.155 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.155 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.156 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184551.125768_119.1", 2022-09-03T06:23:34.156 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.156 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:34.156 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:51.125768074Z", 2022-09-03T06:23:34.157 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.157 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.157 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.157 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.158 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.158 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.158 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:34.158 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.159 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.159 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.159 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.159 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.160 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.160 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.160 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.160 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184551.133544_120.1", 2022-09-03T06:23:34.161 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.161 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:34.161 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:51.133544960Z", 2022-09-03T06:23:34.161 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.162 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.162 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.162 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.162 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.163 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.163 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:34.163 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.163 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.164 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.164 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.164 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.164 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.165 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.165 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.165 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184551.134772_121.1", 2022-09-03T06:23:34.165 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.166 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16", 2022-09-03T06:23:34.166 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:51.134772057Z", 2022-09-03T06:23:34.166 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.166 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.167 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.167 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.167 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.167 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.168 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yedFU2W-LnAQG28SX6Ff-RY", 2022-09-03T06:23:34.168 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.168 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.168 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.169 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.169 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.169 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.169 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.169 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.170 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184551.139229_122.1", 2022-09-03T06:23:34.170 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.170 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16", 2022-09-03T06:23:34.171 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:51.139229501Z", 2022-09-03T06:23:34.171 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.171 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.171 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.171 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.172 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.172 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.172 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yedFU2W-LnAQG28SX6Ff-RY", 2022-09-03T06:23:34.173 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.173 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.173 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.173 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.173 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.174 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.174 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.174 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.174 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.168524_123.1", 2022-09-03T06:23:34.175 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.175 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:34.175 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.168524769Z", 2022-09-03T06:23:34.175 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.176 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.176 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.176 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.176 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.177 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.177 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:34.177 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.177 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.178 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.178 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.178 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.179 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.179 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.179 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.179 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.172232_124.1", 2022-09-03T06:23:34.180 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.180 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:34.180 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.172232256Z", 2022-09-03T06:23:34.180 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.181 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.181 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.181 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.182 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.182 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.182 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:34.182 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.183 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.183 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.183 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.183 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.184 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.184 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.184 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.184 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.173517_125.1", 2022-09-03T06:23:34.185 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.185 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17", 2022-09-03T06:23:34.185 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.173517470Z", 2022-09-03T06:23:34.185 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.186 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.186 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.186 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.186 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.187 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.187 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_KGtuntqhz3tvT1e4k36oFKj", 2022-09-03T06:23:34.187 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.187 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.188 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.188 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.188 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.188 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.189 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.189 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.189 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.178102_126.1", 2022-09-03T06:23:34.189 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.190 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17", 2022-09-03T06:23:34.190 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.178102240Z", 2022-09-03T06:23:34.190 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.190 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.190 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.191 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.191 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.191 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.191 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_KGtuntqhz3tvT1e4k36oFKj", 2022-09-03T06:23:34.192 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.192 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.192 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.192 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.193 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.193 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.193 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.193 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.194 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.215736_127.1", 2022-09-03T06:23:34.194 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.194 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:34.194 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.215736666Z", 2022-09-03T06:23:34.195 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.195 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.195 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.195 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.196 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.196 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.196 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:34.196 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.197 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.197 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.197 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.197 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.197 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.198 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.198 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.198 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.219063_128.1", 2022-09-03T06:23:34.198 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.199 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:34.199 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.219063433Z", 2022-09-03T06:23:34.199 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.199 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.200 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.200 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.200 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.200 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.201 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:34.201 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.201 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.201 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.202 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.202 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.202 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.202 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.203 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.203 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.220690_129.1", 2022-09-03T06:23:34.203 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.203 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18", 2022-09-03T06:23:34.203 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.220690992Z", 2022-09-03T06:23:34.204 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.204 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.204 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.204 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.205 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.205 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.205 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm651SSphsn2Jx7Kwrq9oNR", 2022-09-03T06:23:34.205 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.206 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.206 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.206 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.206 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.207 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.207 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.207 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.207 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.226118_130.1", 2022-09-03T06:23:34.208 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.208 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18", 2022-09-03T06:23:34.208 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.226118188Z", 2022-09-03T06:23:34.208 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.209 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.209 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.209 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.209 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.209 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.210 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm651SSphsn2Jx7Kwrq9oNR", 2022-09-03T06:23:34.210 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.210 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.211 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.211 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.211 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.211 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.211 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.212 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.212 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.250228_131.1", 2022-09-03T06:23:34.212 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.212 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:34.213 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.250228835Z", 2022-09-03T06:23:34.213 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.213 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.213 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:34.214 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.214 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.214 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.215 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:34.215 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.215 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.215 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.215 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.216 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.216 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.216 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.216 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.217 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.253497_132.1", 2022-09-03T06:23:34.217 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.217 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:34.217 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.253497827Z", 2022-09-03T06:23:34.218 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.218 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.218 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:34.219 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.219 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.219 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.219 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:34.220 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.220 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.220 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.220 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.220 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.221 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.221 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.221 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.221 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.266389_133.1", 2022-09-03T06:23:34.222 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.222 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:34.222 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.266389718Z", 2022-09-03T06:23:34.223 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.223 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.223 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:34.223 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.224 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.224 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.224 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:34.224 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.225 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.225 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.225 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.225 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.226 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.226 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.226 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.226 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.270858_134.1", 2022-09-03T06:23:34.227 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.227 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:34.227 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.270858165Z", 2022-09-03T06:23:34.227 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.227 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.228 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:34.228 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.228 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.228 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.229 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:34.229 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.229 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.229 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.230 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.230 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.230 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.230 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.231 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.231 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.289855_135.1", 2022-09-03T06:23:34.231 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.232 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:34.232 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.289855850Z", 2022-09-03T06:23:34.232 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.232 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.233 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:34.233 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.233 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.233 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.234 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:34.234 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.234 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.234 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.235 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.235 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.235 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.235 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.236 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.236 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.296862_136.1", 2022-09-03T06:23:34.236 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.236 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:34.237 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.296862507Z", 2022-09-03T06:23:34.237 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.237 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.237 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:34.238 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.238 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.238 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.238 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:34.238 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.239 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.239 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.239 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.239 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.240 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.240 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.240 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.240 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.168986_143.1", 2022-09-03T06:23:34.241 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.241 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:34.241 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.168986972Z", 2022-09-03T06:23:34.241 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.242 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.242 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.242 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.242 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.243 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.243 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:34.243 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.243 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.244 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.244 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.244 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.244 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.245 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.245 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.245 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.174598_144.1", 2022-09-03T06:23:34.245 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.245 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:34.246 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.174598788Z", 2022-09-03T06:23:34.246 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.246 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.247 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.247 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.247 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.247 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.247 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:34.248 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.248 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.248 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.248 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.249 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.249 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.249 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.249 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.250 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.176005_145.1", 2022-09-03T06:23:34.250 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.250 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19", 2022-09-03T06:23:34.250 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.176005675Z", 2022-09-03T06:23:34.251 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.251 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.251 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.251 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.252 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.252 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.252 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yDXruVlqT4Xsg8Al8Plz6TA", 2022-09-03T06:23:34.252 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.252 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.253 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.253 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.253 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.253 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.254 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.254 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.254 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.179021_146.1", 2022-09-03T06:23:34.254 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.255 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19", 2022-09-03T06:23:34.255 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.179021577Z", 2022-09-03T06:23:34.255 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.255 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.256 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.256 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.256 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.256 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.257 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yDXruVlqT4Xsg8Al8Plz6TA", 2022-09-03T06:23:34.257 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.257 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.257 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.258 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.258 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.258 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.258 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.259 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.259 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.194719_147.1", 2022-09-03T06:23:34.259 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.259 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:34.260 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.194719304Z", 2022-09-03T06:23:34.260 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.260 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.260 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:34.261 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.261 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.261 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.261 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:34.262 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.262 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.262 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.262 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.263 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.263 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.263 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.263 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.263 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.201443_148.1", 2022-09-03T06:23:34.264 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.264 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:34.264 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.201443123Z", 2022-09-03T06:23:34.264 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.265 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.265 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:34.265 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.266 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.266 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.266 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:34.266 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.267 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.267 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.267 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.267 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.268 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.268 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.268 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.269 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185259.164178_150.1", 2022-09-03T06:23:34.269 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.269 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:34.270 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:39.164178377Z", 2022-09-03T06:23:34.270 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.270 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.270 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.271 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.271 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.271 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.271 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.272 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.272 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.272 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.272 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.273 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.273 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.273 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.273 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.273 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185259.166746_154.1", 2022-09-03T06:23:34.274 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.274 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:34.274 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:39.166746306Z", 2022-09-03T06:23:34.274 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.275 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.275 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.275 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.275 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.276 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.276 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.276 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.276 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.277 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.277 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.277 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.277 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.278 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.278 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.278 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185259.167637_155.1", 2022-09-03T06:23:34.278 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.279 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20", 2022-09-03T06:23:34.279 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:39.167637326Z", 2022-09-03T06:23:34.279 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.279 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.280 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.280 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.280 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.280 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.281 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZKafDPgftyjUAayY_VUswW5", 2022-09-03T06:23:34.281 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.281 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.281 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.282 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.282 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.282 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.282 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.283 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.283 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185259.169934_156.1", 2022-09-03T06:23:34.283 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.283 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20", 2022-09-03T06:23:34.284 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:39.169934736Z", 2022-09-03T06:23:34.284 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.284 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.284 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.285 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.285 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.285 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.285 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZKafDPgftyjUAayY_VUswW5", 2022-09-03T06:23:34.285 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.286 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.286 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.286 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.286 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.287 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.287 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.287 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.287 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185351.815540_157.1", 2022-09-03T06:23:34.288 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.288 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:34.288 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:09:11.815540177Z", 2022-09-03T06:23:34.288 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.289 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.289 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.289 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.289 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.290 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.290 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.290 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.290 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.291 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.291 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.291 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.291 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.292 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.292 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.292 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185351.819576_158.1", 2022-09-03T06:23:34.292 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.293 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:34.293 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:09:11.819576411Z", 2022-09-03T06:23:34.293 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.293 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.294 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.294 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.294 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.294 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.295 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.295 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.295 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.295 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.296 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.296 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.296 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.296 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.297 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.297 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185415.254080_165.1", 2022-09-03T06:23:34.297 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.297 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:34.298 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:10:15.254080988Z", 2022-09-03T06:23:34.298 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.298 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.298 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.299 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.299 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.299 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.299 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.300 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:34.300 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.300 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.300 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.301 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.301 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.301 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.301 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.302 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185415.257852_166.1", 2022-09-03T06:23:34.302 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.302 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:34.302 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:10:15.257852595Z", 2022-09-03T06:23:34.303 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.303 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.303 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.303 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.304 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.304 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.304 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:34.304 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:34.305 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.305 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.305 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.305 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.306 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.306 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.306 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.306 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.140367_167.1", 2022-09-03T06:23:34.307 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.307 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:34.307 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.140367923Z", 2022-09-03T06:23:34.307 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.308 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.308 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.308 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.308 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.308 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.309 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:34.309 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.309 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.309 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.310 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.310 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.310 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.311 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.311 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.311 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.143952_168.1", 2022-09-03T06:23:34.311 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.311 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:34.312 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.143952074Z", 2022-09-03T06:23:34.312 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.312 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.312 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.313 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.313 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.313 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.313 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:34.314 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.314 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.314 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.314 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.315 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.315 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.315 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.315 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.316 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.145226_169.1", 2022-09-03T06:23:34.316 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.316 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21", 2022-09-03T06:23:34.316 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.145226679Z", 2022-09-03T06:23:34.317 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.317 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.317 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.317 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.318 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.318 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.318 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WXbrqJZWiL3N6i3X5RrSf_L", 2022-09-03T06:23:34.318 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.319 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.319 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.319 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.319 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.320 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.320 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.320 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.320 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.148908_170.1", 2022-09-03T06:23:34.321 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.321 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21", 2022-09-03T06:23:34.321 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.148908181Z", 2022-09-03T06:23:34.321 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.322 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.322 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.322 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.322 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.323 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.323 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WXbrqJZWiL3N6i3X5RrSf_L", 2022-09-03T06:23:34.323 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.323 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.324 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.324 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.324 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.324 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.325 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.325 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.325 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.171153_171.1", 2022-09-03T06:23:34.326 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.326 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:34.326 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.171153368Z", 2022-09-03T06:23:34.326 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.327 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.327 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.327 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.327 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.328 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.328 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:34.328 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.328 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.329 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.329 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.329 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.329 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.330 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.330 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.330 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.174859_172.1", 2022-09-03T06:23:34.330 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.331 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:34.331 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.174859152Z", 2022-09-03T06:23:34.331 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.331 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.331 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.332 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.332 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.332 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.332 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:34.333 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.333 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.333 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.333 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.334 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.334 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.334 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.335 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.335 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.176058_173.1", 2022-09-03T06:23:34.335 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.335 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22", 2022-09-03T06:23:34.336 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.176058283Z", 2022-09-03T06:23:34.336 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.336 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.336 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.336 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.337 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.337 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.337 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_4TVYbHf2FAhumE819lO37xf", 2022-09-03T06:23:34.337 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.338 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.338 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.338 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.338 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.339 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.339 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.339 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.339 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.180790_174.1", 2022-09-03T06:23:34.340 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.340 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22", 2022-09-03T06:23:34.340 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.180790715Z", 2022-09-03T06:23:34.340 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.341 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.341 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.341 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.341 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.341 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.342 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_4TVYbHf2FAhumE819lO37xf", 2022-09-03T06:23:34.342 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.342 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.343 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.343 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.343 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.343 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.344 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.344 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.344 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.221697_175.1", 2022-09-03T06:23:34.344 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.345 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:34.345 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.221697751Z", 2022-09-03T06:23:34.345 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.345 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.346 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.346 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.346 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.346 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.347 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:34.347 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.347 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.347 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.347 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.348 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.348 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.348 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.349 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.349 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.229951_176.1", 2022-09-03T06:23:34.349 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.349 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:34.350 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.229951839Z", 2022-09-03T06:23:34.350 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.350 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.350 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.350 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.351 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.351 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.351 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:34.351 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.352 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.352 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.352 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.352 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.353 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.353 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.353 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.353 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.237269_177.1", 2022-09-03T06:23:34.354 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.354 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23", 2022-09-03T06:23:34.354 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.237269112Z", 2022-09-03T06:23:34.354 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.355 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.355 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.355 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.355 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.356 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.356 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_SUW5WcOi0sbRMVG7llLAyro", 2022-09-03T06:23:34.356 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.356 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.357 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.357 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.357 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.357 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.358 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.358 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.358 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.248604_178.1", 2022-09-03T06:23:34.358 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.359 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23", 2022-09-03T06:23:34.359 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.248604519Z", 2022-09-03T06:23:34.359 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.359 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.360 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.360 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.360 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.360 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.360 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_SUW5WcOi0sbRMVG7llLAyro", 2022-09-03T06:23:34.361 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.361 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.361 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.361 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.362 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.362 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.362 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.362 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.363 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.289134_179.1", 2022-09-03T06:23:34.363 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.363 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:34.363 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.289134872Z", 2022-09-03T06:23:34.364 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.364 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.364 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:34.364 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.365 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.365 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.365 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:34.365 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.366 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.366 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.366 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.366 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.367 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.367 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.367 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.367 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.292023_180.1", 2022-09-03T06:23:34.368 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.368 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:34.368 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.292023076Z", 2022-09-03T06:23:34.368 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.368 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.369 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:34.369 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.369 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.369 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.370 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:34.370 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.370 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.370 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.371 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.371 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.371 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.371 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.372 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.372 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.305537_181.1", 2022-09-03T06:23:34.372 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.372 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:34.373 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.305537948Z", 2022-09-03T06:23:34.373 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.373 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.373 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:34.374 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.374 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.374 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.374 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:34.374 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.375 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.375 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.375 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.375 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.376 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.376 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.376 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.376 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.308323_182.1", 2022-09-03T06:23:34.377 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.377 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:34.377 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.308323992Z", 2022-09-03T06:23:34.377 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.378 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.378 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:34.378 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.379 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.379 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.379 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:34.380 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.380 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.380 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.380 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.381 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.381 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.381 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.381 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.382 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.323485_183.1", 2022-09-03T06:23:34.382 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.382 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:34.383 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.323485482Z", 2022-09-03T06:23:34.383 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.383 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.383 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:34.384 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.384 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.384 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.384 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:34.385 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.385 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.385 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.385 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.386 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.386 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.386 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.386 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.387 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.326245_184.1", 2022-09-03T06:23:34.387 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.387 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:34.387 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.326245244Z", 2022-09-03T06:23:34.388 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.388 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.388 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:34.388 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.389 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.389 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.389 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:34.389 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.389 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.390 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.390 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.390 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.390 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.391 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.391 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.391 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.128610_187.1", 2022-09-03T06:23:34.391 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.392 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:34.392 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.128610110Z", 2022-09-03T06:23:34.392 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.392 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.393 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.393 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.393 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.393 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.394 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.394 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.394 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.394 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.395 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.395 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.395 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.395 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.396 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.396 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.132654_188.1", 2022-09-03T06:23:34.396 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.396 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:34.397 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.132654659Z", 2022-09-03T06:23:34.397 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.397 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.397 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.397 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.398 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.398 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.398 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.398 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.399 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.399 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.399 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.399 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.400 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.400 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.400 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.400 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.133913_189.1", 2022-09-03T06:23:34.401 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.401 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24", 2022-09-03T06:23:34.401 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.133913786Z", 2022-09-03T06:23:34.401 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.402 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.402 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.402 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.402 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.403 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.403 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_flubJt4QosOvedZNv50fpC6", 2022-09-03T06:23:34.403 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.403 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.404 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.404 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.404 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.404 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.405 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.405 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.405 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.136736_190.1", 2022-09-03T06:23:34.405 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.406 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24", 2022-09-03T06:23:34.406 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.136736736Z", 2022-09-03T06:23:34.406 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.406 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.407 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.407 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.407 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.407 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.407 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_flubJt4QosOvedZNv50fpC6", 2022-09-03T06:23:34.408 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.408 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.408 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.408 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.409 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.409 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.409 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.409 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.410 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.158636_191.1", 2022-09-03T06:23:34.410 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.410 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:34.410 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.158636011Z", 2022-09-03T06:23:34.411 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.411 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.411 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.411 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.412 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.412 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.412 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.412 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.413 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.413 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.413 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.413 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.413 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.414 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.414 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.414 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.162317_192.1", 2022-09-03T06:23:34.414 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.415 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:34.415 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.162317814Z", 2022-09-03T06:23:34.415 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.415 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.416 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.416 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.416 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.416 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.417 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.417 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.418 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.418 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.418 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.419 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.419 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.419 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.419 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.420 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.163511_193.1", 2022-09-03T06:23:34.420 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.420 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25", 2022-09-03T06:23:34.420 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.163511615Z", 2022-09-03T06:23:34.421 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.421 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.421 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.421 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.422 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.422 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.422 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tB02hvaj6C_E_OnwBMY8ywL", 2022-09-03T06:23:34.422 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.423 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.423 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.423 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.423 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.424 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.424 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.424 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.424 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.166349_194.1", 2022-09-03T06:23:34.425 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.425 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25", 2022-09-03T06:23:34.425 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.166349608Z", 2022-09-03T06:23:34.425 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.426 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.426 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.426 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.426 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.427 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.427 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tB02hvaj6C_E_OnwBMY8ywL", 2022-09-03T06:23:34.427 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.427 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.428 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.428 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.428 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.428 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.428 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.429 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.429 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.203237_195.1", 2022-09-03T06:23:34.429 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.429 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:34.430 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.203237512Z", 2022-09-03T06:23:34.430 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.430 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.430 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.431 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.431 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.431 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.431 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.432 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.432 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.432 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.432 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.433 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.433 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.433 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.433 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.433 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.206000_196.1", 2022-09-03T06:23:34.434 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.434 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:34.434 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.206000308Z", 2022-09-03T06:23:34.434 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.435 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.435 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.435 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.435 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.436 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.436 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.436 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.436 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.437 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.437 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.437 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.437 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.438 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.438 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.438 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.207006_197.1", 2022-09-03T06:23:34.438 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.439 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26", 2022-09-03T06:23:34.439 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.207006797Z", 2022-09-03T06:23:34.439 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.439 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.440 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.440 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.440 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.440 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.440 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_iK1lRBoYvVYLawtBAmt3Urm", 2022-09-03T06:23:34.441 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.441 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.441 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.441 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.442 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.442 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.442 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.442 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.443 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.211392_198.1", 2022-09-03T06:23:34.443 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.443 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26", 2022-09-03T06:23:34.443 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.211392135Z", 2022-09-03T06:23:34.444 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.444 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.444 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.444 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.445 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.445 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.445 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_iK1lRBoYvVYLawtBAmt3Urm", 2022-09-03T06:23:34.445 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.446 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.446 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.446 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.446 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.447 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.447 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.447 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.447 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.245835_199.1", 2022-09-03T06:23:34.448 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.448 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:34.448 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.245835749Z", 2022-09-03T06:23:34.448 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.448 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.449 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.449 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.449 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.449 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.450 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.450 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.450 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.450 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.451 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.451 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.451 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.451 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.452 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.452 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.250338_200.1", 2022-09-03T06:23:34.452 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.452 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:34.453 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.250338287Z", 2022-09-03T06:23:34.453 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.453 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.453 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.454 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.454 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.454 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.454 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.455 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.455 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.455 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.455 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.455 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.456 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.456 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.456 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.456 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.265702_201.1", 2022-09-03T06:23:34.457 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.457 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:34.457 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.265702416Z", 2022-09-03T06:23:34.457 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.458 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.458 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.458 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.458 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.459 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.459 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.459 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:34.459 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.460 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.460 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.460 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.460 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.460 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.461 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.461 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.270146_202.1", 2022-09-03T06:23:34.461 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.461 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:34.462 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.270146860Z", 2022-09-03T06:23:34.462 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.462 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.462 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.463 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.463 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.463 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.463 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:34.464 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:34.464 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.464 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.464 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.465 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.465 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.465 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.465 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.465 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.283449_203.1", 2022-09-03T06:23:34.466 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.466 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:34.466 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.283449642Z", 2022-09-03T06:23:34.466 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.467 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.467 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.467 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.467 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.468 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.468 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.468 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.468 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.469 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.469 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.469 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.469 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.470 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.470 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.470 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.286127_204.1", 2022-09-03T06:23:34.470 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.471 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:34.471 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.286127335Z", 2022-09-03T06:23:34.471 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.471 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.472 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.472 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.472 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.472 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.473 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.473 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.473 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.473 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.473 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.474 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.474 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.474 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.474 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.475 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.305370_205.1", 2022-09-03T06:23:34.475 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.475 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:34.475 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.305370975Z", 2022-09-03T06:23:34.476 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.476 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.476 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.476 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.477 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.477 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.477 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.477 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:34.478 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.478 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.478 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.478 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.479 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.479 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.479 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.479 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.312602_206.1", 2022-09-03T06:23:34.479 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.480 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:34.480 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.312602358Z", 2022-09-03T06:23:34.480 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.480 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.481 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.481 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.481 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.481 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.482 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:34.482 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:34.482 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.482 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.483 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.483 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.483 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.483 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.484 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.484 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.344134_207.1", 2022-09-03T06:23:34.484 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.484 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:34.485 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.344134550Z", 2022-09-03T06:23:34.485 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.485 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.485 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.486 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.486 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.486 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.486 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.487 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.487 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.487 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.487 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.487 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.488 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.488 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.488 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.488 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.355110_208.1", 2022-09-03T06:23:34.489 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.489 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:34.489 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.355110727Z", 2022-09-03T06:23:34.489 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.490 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.490 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.490 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.490 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.491 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.491 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.491 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.491 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.492 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.492 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.492 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.492 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.492 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.493 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.493 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.382118_209.1", 2022-09-03T06:23:34.493 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.493 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:34.494 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.382118897Z", 2022-09-03T06:23:34.494 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.494 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.494 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.495 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.495 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.495 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.495 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.496 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:34.496 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.496 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.496 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.497 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.497 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.497 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.497 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.498 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.386563_210.1", 2022-09-03T06:23:34.498 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.498 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:34.498 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.386563904Z", 2022-09-03T06:23:34.499 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.499 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.499 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.499 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.500 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.500 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.500 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:34.500 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:34.501 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.501 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.501 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.501 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.502 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.502 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.502 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.502 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.730358_216.1", 2022-09-03T06:23:34.502 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.503 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:34.503 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.730358281Z", 2022-09-03T06:23:34.503 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.503 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.504 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.504 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.504 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.504 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.505 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:34.505 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.505 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.505 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.506 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.506 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.506 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.506 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.507 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.507 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.735815_217.1", 2022-09-03T06:23:34.507 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.507 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:34.507 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.735815559Z", 2022-09-03T06:23:34.508 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.508 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.508 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.508 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.509 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.509 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.509 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:34.509 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.510 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.510 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.510 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.510 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.511 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.511 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.511 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.511 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.736742_218.1", 2022-09-03T06:23:34.512 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.512 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27", 2022-09-03T06:23:34.512 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.736742093Z", 2022-09-03T06:23:34.512 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.513 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.513 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.513 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.513 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.513 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.514 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_jZ9wkZPqzR2GWEIEfGTWCSz", 2022-09-03T06:23:34.514 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.514 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.514 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.515 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.515 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.515 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.515 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.516 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.516 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.742821_219.1", 2022-09-03T06:23:34.516 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.516 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27", 2022-09-03T06:23:34.517 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.742821039Z", 2022-09-03T06:23:34.517 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.517 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.517 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.518 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.518 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.518 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.518 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_jZ9wkZPqzR2GWEIEfGTWCSz", 2022-09-03T06:23:34.519 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.519 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.519 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.519 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.520 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.520 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.520 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.520 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.520 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.774385_220.1", 2022-09-03T06:23:34.521 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.521 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:34.521 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.774385987Z", 2022-09-03T06:23:34.521 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.522 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.522 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.522 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.522 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.523 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.523 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:34.523 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.523 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.524 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.524 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.524 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.524 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.525 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.525 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.525 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.778620_221.1", 2022-09-03T06:23:34.525 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.525 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:34.526 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.778620417Z", 2022-09-03T06:23:34.526 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.526 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.526 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.527 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.527 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.527 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.527 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:34.528 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.528 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.528 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.528 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.529 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.529 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.529 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.529 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.530 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.779591_222.1", 2022-09-03T06:23:34.530 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.530 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28", 2022-09-03T06:23:34.530 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.779591267Z", 2022-09-03T06:23:34.531 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.531 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.531 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.531 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.532 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.532 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.532 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ISaB31FGT1RJh5d1WEGeO7S", 2022-09-03T06:23:34.532 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.533 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.533 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.533 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.533 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.534 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.534 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.534 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.534 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.784101_223.1", 2022-09-03T06:23:34.535 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.535 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28", 2022-09-03T06:23:34.535 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.784101743Z", 2022-09-03T06:23:34.535 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.536 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.536 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.536 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.536 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.536 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.537 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ISaB31FGT1RJh5d1WEGeO7S", 2022-09-03T06:23:34.537 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.537 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.537 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.538 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.538 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.538 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.538 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.539 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.539 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.842565_224.1", 2022-09-03T06:23:34.539 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.539 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:34.540 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.842565677Z", 2022-09-03T06:23:34.540 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.540 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.540 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.541 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.541 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.541 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.541 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:34.542 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.542 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.542 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.542 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.543 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.543 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.543 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.543 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.544 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.848053_225.1", 2022-09-03T06:23:34.544 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.544 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:34.544 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.848053689Z", 2022-09-03T06:23:34.545 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.545 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.545 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.545 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.545 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.546 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.546 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:34.546 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.546 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.547 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.547 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.547 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.547 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.548 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.548 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.548 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.850209_226.1", 2022-09-03T06:23:34.548 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.549 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29", 2022-09-03T06:23:34.549 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.850209520Z", 2022-09-03T06:23:34.549 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.549 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.550 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.550 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.550 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.550 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.551 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_JaA8RlJaRxOPPcYHu0FpEzg", 2022-09-03T06:23:34.551 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.551 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.551 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.552 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.552 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.552 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.552 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.552 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.553 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.855964_227.1", 2022-09-03T06:23:34.553 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.553 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29", 2022-09-03T06:23:34.554 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.855964453Z", 2022-09-03T06:23:34.554 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.554 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.554 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.555 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.555 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.555 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.555 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_JaA8RlJaRxOPPcYHu0FpEzg", 2022-09-03T06:23:34.555 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.556 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.556 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.556 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.556 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.557 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.557 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.557 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.557 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.894939_228.1", 2022-09-03T06:23:34.558 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.558 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:34.558 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.894939418Z", 2022-09-03T06:23:34.558 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.559 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.559 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:34.559 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.559 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.560 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.560 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:34.560 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.560 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.561 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.561 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.561 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.561 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.562 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.562 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.562 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.897692_229.1", 2022-09-03T06:23:34.562 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.563 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:34.563 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.897692116Z", 2022-09-03T06:23:34.563 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.563 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.563 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:34.564 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.564 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.564 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.565 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:34.565 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.565 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.565 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.566 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.566 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.566 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.566 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.567 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.567 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.921086_230.1", 2022-09-03T06:23:34.567 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.567 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:34.567 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.921086404Z", 2022-09-03T06:23:34.568 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.568 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.568 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:34.568 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.569 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.569 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.569 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:34.569 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.570 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.570 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.570 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.570 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.571 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.571 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.571 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.571 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.923839_231.1", 2022-09-03T06:23:34.572 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.572 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:34.572 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.923839471Z", 2022-09-03T06:23:34.572 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.573 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.573 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:34.573 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.573 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.574 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.574 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:34.574 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.574 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.575 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.575 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.575 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.575 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.576 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.576 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.576 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.001910_232.1", 2022-09-03T06:23:34.576 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.577 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:34.577 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.001910722Z", 2022-09-03T06:23:34.577 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.577 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.577 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:34.578 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.578 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.578 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.578 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:34.579 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.579 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.579 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.579 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.580 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.580 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.580 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.580 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.581 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.011478_233.1", 2022-09-03T06:23:34.581 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.581 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:34.581 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.011478475Z", 2022-09-03T06:23:34.582 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.582 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.582 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:34.582 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.583 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.583 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.583 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:34.583 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.584 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.584 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.584 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.584 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.585 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.585 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.585 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.585 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.059572_234.1", 2022-09-03T06:23:34.586 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.586 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:34.586 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.059572375Z", 2022-09-03T06:23:34.587 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.587 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.587 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.588 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.588 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.588 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.588 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:34.589 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.589 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.589 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.590 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.590 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.590 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.590 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.591 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.591 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.063873_235.1", 2022-09-03T06:23:34.591 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.591 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:34.592 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.063873838Z", 2022-09-03T06:23:34.592 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.592 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.592 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.592 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.593 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.593 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.593 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:34.593 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.594 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.594 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.594 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.594 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.595 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.595 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.595 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.595 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.064807_236.1", 2022-09-03T06:23:34.596 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.596 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30", 2022-09-03T06:23:34.596 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.064807654Z", 2022-09-03T06:23:34.596 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.597 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.597 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.597 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.598 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.598 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.598 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_emDKTj1MXu1EbBl7W-BZDCI", 2022-09-03T06:23:34.599 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.599 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.599 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.599 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.600 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.600 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.600 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.600 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.601 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.069230_237.1", 2022-09-03T06:23:34.601 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.601 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30", 2022-09-03T06:23:34.601 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.069230157Z", 2022-09-03T06:23:34.602 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.602 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.602 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.602 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.603 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.603 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.603 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_emDKTj1MXu1EbBl7W-BZDCI", 2022-09-03T06:23:34.603 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.604 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.604 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.604 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.604 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.605 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.605 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.605 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.605 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.098699_238.1", 2022-09-03T06:23:34.606 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.606 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:34.606 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.098699181Z", 2022-09-03T06:23:34.606 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.607 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.607 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.607 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.607 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.608 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.608 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:34.608 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.608 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.609 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.609 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.609 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.609 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.610 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.610 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.610 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.104362_239.1", 2022-09-03T06:23:34.610 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.611 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:34.611 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.104362288Z", 2022-09-03T06:23:34.611 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.611 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.612 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.612 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.612 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.612 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.612 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:34.613 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.613 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.613 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.613 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.614 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.614 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.614 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.614 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.615 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.105238_240.1", 2022-09-03T06:23:34.615 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.615 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31", 2022-09-03T06:23:34.615 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.105238360Z", 2022-09-03T06:23:34.616 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.616 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.616 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.616 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.617 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.617 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.617 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Fx_iwLRI89zV-Ovfojlsmn_", 2022-09-03T06:23:34.617 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.618 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.618 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.618 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.618 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.619 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.619 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.619 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.619 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.109389_241.1", 2022-09-03T06:23:34.620 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.620 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31", 2022-09-03T06:23:34.620 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.109389408Z", 2022-09-03T06:23:34.620 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.621 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.621 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.621 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.621 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.622 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.622 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Fx_iwLRI89zV-Ovfojlsmn_", 2022-09-03T06:23:34.622 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.622 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.623 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.623 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.623 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.623 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.624 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.624 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.624 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.146357_242.1", 2022-09-03T06:23:34.624 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.624 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:34.625 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.146357921Z", 2022-09-03T06:23:34.625 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.625 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.625 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.626 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.626 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.626 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.626 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:34.627 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.627 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.627 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.627 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.628 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.628 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.628 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.628 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.629 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.149385_243.1", 2022-09-03T06:23:34.629 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.629 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:34.629 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.149385163Z", 2022-09-03T06:23:34.630 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.630 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.630 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.630 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.631 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.631 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.631 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:34.631 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.632 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.632 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.632 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.632 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.632 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.633 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.633 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.633 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.150696_244.1", 2022-09-03T06:23:34.633 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.634 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32", 2022-09-03T06:23:34.634 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.150696281Z", 2022-09-03T06:23:34.634 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.634 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.635 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.635 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.635 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.635 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.636 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_VJHe0Zrz6-lomZaJx40Mq8S", 2022-09-03T06:23:34.636 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.636 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.636 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.637 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.637 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.637 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.637 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.638 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.638 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.159044_245.1", 2022-09-03T06:23:34.638 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.638 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32", 2022-09-03T06:23:34.639 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.159044634Z", 2022-09-03T06:23:34.639 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.639 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.639 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.640 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.640 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.640 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.640 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_VJHe0Zrz6-lomZaJx40Mq8S", 2022-09-03T06:23:34.640 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.641 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.641 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.641 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.641 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.642 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.642 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.642 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.642 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.221962_246.1", 2022-09-03T06:23:34.643 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.643 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:34.643 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.221962483Z", 2022-09-03T06:23:34.643 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.644 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.644 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:34.644 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.644 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.645 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.645 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:34.645 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.645 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.646 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.646 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.646 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.646 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.647 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.647 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.647 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.225097_247.1", 2022-09-03T06:23:34.647 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.648 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:34.648 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.225097472Z", 2022-09-03T06:23:34.648 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.648 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.648 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:34.649 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.649 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.649 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.649 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:34.650 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.650 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.650 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.650 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.651 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.651 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.651 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.651 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.652 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.324010_248.1", 2022-09-03T06:23:34.652 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.652 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:34.652 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.324010891Z", 2022-09-03T06:23:34.653 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.653 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.653 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:34.653 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.654 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.654 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.654 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:34.654 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.655 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.655 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.655 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.655 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.656 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.656 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.656 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.656 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.326610_249.1", 2022-09-03T06:23:34.656 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.657 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:34.657 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.326610391Z", 2022-09-03T06:23:34.657 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.657 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.658 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:34.658 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.658 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.658 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.659 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:34.659 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.659 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.659 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.660 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.660 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.660 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.660 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.661 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.661 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.388152_250.1", 2022-09-03T06:23:34.661 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.661 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:34.662 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.388152023Z", 2022-09-03T06:23:34.662 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.662 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.662 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:34.662 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.663 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.663 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.663 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:34.663 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.664 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.664 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.664 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.664 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.665 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.665 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.665 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.665 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.390795_251.1", 2022-09-03T06:23:34.666 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.666 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:34.666 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.390795101Z", 2022-09-03T06:23:34.667 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.667 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.667 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:34.667 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.668 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.668 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.668 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:34.668 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.669 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.669 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.669 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.669 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.670 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.670 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.670 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.670 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.428775_252.1", 2022-09-03T06:23:34.671 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.671 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:34.671 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.428775853Z", 2022-09-03T06:23:34.671 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.672 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.672 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.672 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.673 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.673 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.673 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:34.674 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.674 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.674 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.675 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.675 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.676 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.676 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.676 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.676 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.432176_253.1", 2022-09-03T06:23:34.677 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.677 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:34.677 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.432176035Z", 2022-09-03T06:23:34.678 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.678 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.678 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.679 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.679 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.679 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.679 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:34.680 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.680 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.680 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.680 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.681 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.681 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.681 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.682 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.682 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.433149_254.1", 2022-09-03T06:23:34.682 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.682 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33", 2022-09-03T06:23:34.683 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.433149170Z", 2022-09-03T06:23:34.683 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.683 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.683 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.684 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.684 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.684 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.684 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_w0QD1jE_b482WNZp5Lj2ky4", 2022-09-03T06:23:34.685 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.685 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.685 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.685 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.686 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.686 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.686 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.686 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.687 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.435497_255.1", 2022-09-03T06:23:34.687 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.687 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33", 2022-09-03T06:23:34.687 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.435497913Z", 2022-09-03T06:23:34.688 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.688 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.688 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.688 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.689 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.689 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.689 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_w0QD1jE_b482WNZp5Lj2ky4", 2022-09-03T06:23:34.689 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.690 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.690 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.690 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.691 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.691 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.691 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.691 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.692 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.463906_256.1", 2022-09-03T06:23:34.692 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.692 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:34.693 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.463906896Z", 2022-09-03T06:23:34.693 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.693 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.693 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.694 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.694 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.694 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.695 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:34.695 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.695 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.695 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.696 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.696 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.696 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.697 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.697 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.697 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.466439_257.1", 2022-09-03T06:23:34.697 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.698 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:34.698 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.466439294Z", 2022-09-03T06:23:34.698 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.699 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.699 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.699 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.699 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.700 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.700 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:34.701 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.701 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.702 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.702 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.703 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.703 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.704 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.704 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.705 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.467396_258.1", 2022-09-03T06:23:34.705 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.705 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34", 2022-09-03T06:23:34.706 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.467396448Z", 2022-09-03T06:23:34.706 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.707 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.707 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.708 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.708 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.709 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.709 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G2EFJYOe8FOdjOCj6aBraUR", 2022-09-03T06:23:34.710 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.710 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.711 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.711 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.712 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.712 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.713 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.713 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.714 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.471551_259.1", 2022-09-03T06:23:34.714 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.714 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34", 2022-09-03T06:23:34.715 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.471551488Z", 2022-09-03T06:23:34.715 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.715 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.715 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.716 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.716 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.716 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.717 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G2EFJYOe8FOdjOCj6aBraUR", 2022-09-03T06:23:34.717 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.717 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.718 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.718 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.718 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.719 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.719 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.719 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.720 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.517947_260.1", 2022-09-03T06:23:34.720 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.721 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:34.721 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.517947959Z", 2022-09-03T06:23:34.722 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.722 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.723 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.723 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.724 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.724 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.725 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:34.725 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.726 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.726 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.727 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.727 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.728 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.728 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.729 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.729 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.522051_261.1", 2022-09-03T06:23:34.730 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.730 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:34.731 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.522051544Z", 2022-09-03T06:23:34.731 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.731 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.732 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.732 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.733 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.733 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.734 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:34.734 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.735 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.735 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.736 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.736 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.737 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.737 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.738 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.738 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.524288_262.1", 2022-09-03T06:23:34.739 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.739 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35", 2022-09-03T06:23:34.740 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.524288626Z", 2022-09-03T06:23:34.740 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.741 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.741 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.742 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.752 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.752 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.753 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G8oh7TOeWyD2_FvC8AIXccM", 2022-09-03T06:23:34.753 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.754 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.754 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.755 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.755 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.756 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.756 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.757 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.757 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.526709_263.1", 2022-09-03T06:23:34.758 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.758 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35", 2022-09-03T06:23:34.758 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.526709154Z", 2022-09-03T06:23:34.759 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.759 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.760 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.760 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.761 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.761 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.762 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G8oh7TOeWyD2_FvC8AIXccM", 2022-09-03T06:23:34.762 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.762 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.763 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.763 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.764 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.764 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.765 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.765 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.766 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.565852_264.1", 2022-09-03T06:23:34.766 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.766 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:34.767 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.565852141Z", 2022-09-03T06:23:34.767 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.768 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.768 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:34.769 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.769 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.770 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.770 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:34.770 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.770 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.771 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.771 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.771 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.771 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.772 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.772 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.772 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.568581_265.1", 2022-09-03T06:23:34.772 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.773 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:34.773 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.568581265Z", 2022-09-03T06:23:34.773 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.773 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.774 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:34.774 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.774 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.774 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.775 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:34.775 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.775 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.775 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.776 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.776 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.776 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.776 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.777 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.777 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185757.605699_266.1", 2022-09-03T06:23:34.777 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.777 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:34.778 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:57.605699761Z", 2022-09-03T06:23:34.778 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.778 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.778 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:34.779 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.779 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.779 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.779 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:34.780 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.780 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.780 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.780 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.781 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.781 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.781 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.781 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.782 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185757.608333_267.1", 2022-09-03T06:23:34.782 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.782 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:34.782 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:57.608333919Z", 2022-09-03T06:23:34.783 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.783 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.783 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:34.783 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.784 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.784 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.784 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:34.784 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.785 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.785 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.785 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.785 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.786 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.786 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.786 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.786 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.691847_268.1", 2022-09-03T06:23:34.787 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.787 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:34.787 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.691847123Z", 2022-09-03T06:23:34.787 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.788 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.788 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:34.788 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.788 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.789 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.789 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:34.789 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.789 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.790 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.790 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.790 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.790 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.791 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.791 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.791 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.696866_269.1", 2022-09-03T06:23:34.792 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.792 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:34.792 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.696866177Z", 2022-09-03T06:23:34.793 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.793 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.793 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:34.793 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.794 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.794 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.794 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:34.794 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:34.795 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.795 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.795 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.796 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.796 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.797 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.797 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.797 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.804943_270.1", 2022-09-03T06:23:34.798 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.798 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:23:34.799 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.804943552Z", 2022-09-03T06:23:34.799 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.799 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.800 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.800 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.800 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.800 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.801 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-F9Dl85tpDzK3gXJ-aezc_j", 2022-09-03T06:23:34.801 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.801 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.802 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.802 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.802 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.803 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.803 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.803 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.804 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.808467_271.1", 2022-09-03T06:23:34.804 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.805 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:23:34.805 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.808467134Z", 2022-09-03T06:23:34.806 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.806 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.807 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.807 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.807 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.808 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.808 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-F9Dl85tpDzK3gXJ-aezc_j", 2022-09-03T06:23:34.809 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.809 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.812 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.812 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.813 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.813 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.814 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.814 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.814 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.810847_272.1", 2022-09-03T06:23:34.814 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.815 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-36", 2022-09-03T06:23:34.815 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.810847136Z", 2022-09-03T06:23:34.815 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.816 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.816 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.816 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.816 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.817 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.817 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_5JS-wFAV9CrTmAifbKi9NjK", 2022-09-03T06:23:34.817 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.818 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.818 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.819 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.819 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.819 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.820 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.820 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.821 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.813462_273.1", 2022-09-03T06:23:34.821 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.821 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-36", 2022-09-03T06:23:34.821 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.813462243Z", 2022-09-03T06:23:34.822 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.822 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.822 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.822 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.823 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.823 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.823 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_5JS-wFAV9CrTmAifbKi9NjK", 2022-09-03T06:23:34.823 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.824 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.824 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.824 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.824 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.825 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.825 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.825 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.825 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.844464_274.1", 2022-09-03T06:23:34.826 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.826 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:23:34.826 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.844464322Z", 2022-09-03T06:23:34.826 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.827 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.827 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.828 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.828 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.828 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.829 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_brde0pgOsuZjrCySsOp9pmR", 2022-09-03T06:23:34.829 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.829 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.830 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.830 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.830 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.831 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.831 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.831 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.831 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.850329_275.1", 2022-09-03T06:23:34.832 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.832 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:23:34.832 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.850329433Z", 2022-09-03T06:23:34.832 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.832 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.833 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.833 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.833 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.834 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.834 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_brde0pgOsuZjrCySsOp9pmR", 2022-09-03T06:23:34.835 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.835 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.835 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.836 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.836 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.836 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.836 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.837 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.837 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.855256_276.1", 2022-09-03T06:23:34.837 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.837 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-37", 2022-09-03T06:23:34.838 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.855256692Z", 2022-09-03T06:23:34.838 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.838 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.838 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.839 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.839 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.839 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.839 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3HAVWKyp6rMdHTXkKukojjv", 2022-09-03T06:23:34.840 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.840 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.840 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.840 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.841 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.841 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.841 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.841 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.841 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.859644_277.1", 2022-09-03T06:23:34.842 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.842 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-37", 2022-09-03T06:23:34.842 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.859644317Z", 2022-09-03T06:23:34.842 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.843 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.843 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.843 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.843 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.844 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.844 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3HAVWKyp6rMdHTXkKukojjv", 2022-09-03T06:23:34.844 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.844 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.845 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.845 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.845 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.845 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.846 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.846 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.846 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.902177_278.1", 2022-09-03T06:23:34.846 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.847 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:23:34.847 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.902177035Z", 2022-09-03T06:23:34.847 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.847 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.848 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.848 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.848 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.848 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.848 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Iv3Ff3D6p1XbZ30mXZkhK63", 2022-09-03T06:23:34.849 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.849 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.849 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.849 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.850 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.850 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.850 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.850 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.851 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.907504_279.1", 2022-09-03T06:23:34.851 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.851 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:23:34.851 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.907504015Z", 2022-09-03T06:23:34.852 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.852 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.852 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.852 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.853 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.853 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.853 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Iv3Ff3D6p1XbZ30mXZkhK63", 2022-09-03T06:23:34.853 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.854 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.854 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.854 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.854 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.855 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.855 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.855 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.855 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.908555_280.1", 2022-09-03T06:23:34.856 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.856 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-38", 2022-09-03T06:23:34.856 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.908555860Z", 2022-09-03T06:23:34.856 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.856 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.857 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.857 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.857 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.857 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.858 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G9WP1PhzJZFD5vWOwiYS9Hb", 2022-09-03T06:23:34.858 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.858 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.858 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.859 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.859 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.859 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.859 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.860 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.860 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.915547_281.1", 2022-09-03T06:23:34.860 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.860 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-38", 2022-09-03T06:23:34.861 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.915547971Z", 2022-09-03T06:23:34.861 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.861 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.861 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.862 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.862 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.862 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.862 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G9WP1PhzJZFD5vWOwiYS9Hb", 2022-09-03T06:23:34.863 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.863 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.863 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.863 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.864 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.864 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.864 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.864 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.865 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.844866_288.1", 2022-09-03T06:23:34.865 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.866 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39", 2022-09-03T06:23:34.866 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.844866559Z", 2022-09-03T06:23:34.867 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.867 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.868 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.868 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.868 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.869 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.869 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lt1dYegSwDzpwX06X-fGkup", 2022-09-03T06:23:34.870 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.870 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.870 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.871 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.871 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.871 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.871 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.872 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.872 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.850612_289.1", 2022-09-03T06:23:34.872 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.873 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39", 2022-09-03T06:23:34.873 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.850612019Z", 2022-09-03T06:23:34.873 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.874 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.874 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.875 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.875 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.875 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.875 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lt1dYegSwDzpwX06X-fGkup", 2022-09-03T06:23:34.876 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.876 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.876 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.876 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.877 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.877 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.877 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.877 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.878 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.852628_290.1", 2022-09-03T06:23:34.878 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.878 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-39", 2022-09-03T06:23:34.879 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.852628518Z", 2022-09-03T06:23:34.879 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.879 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.879 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.880 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.880 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.880 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.880 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_O0mztKsloMen3wFjqSH6i9D", 2022-09-03T06:23:34.881 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.881 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.881 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.882 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.882 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.882 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.882 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.883 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.883 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.857180_291.1", 2022-09-03T06:23:34.883 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.884 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-39", 2022-09-03T06:23:34.884 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.857180178Z", 2022-09-03T06:23:34.884 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.884 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.885 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.885 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.885 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.885 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.886 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_O0mztKsloMen3wFjqSH6i9D", 2022-09-03T06:23:34.886 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.886 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.886 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.887 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.887 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.887 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.887 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.888 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.888 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.883799_292.1", 2022-09-03T06:23:34.888 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.888 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40", 2022-09-03T06:23:34.889 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.883799148Z", 2022-09-03T06:23:34.889 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.889 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.889 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.890 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.890 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.890 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.890 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_fswsShBMVZzHxqVULj3KzUU", 2022-09-03T06:23:34.891 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.891 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.891 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.891 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.892 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.892 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.892 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.893 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.893 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.889075_293.1", 2022-09-03T06:23:34.893 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.893 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40", 2022-09-03T06:23:34.893 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.889075582Z", 2022-09-03T06:23:34.894 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.894 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.894 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.895 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.895 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.895 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.895 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_fswsShBMVZzHxqVULj3KzUU", 2022-09-03T06:23:34.896 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.896 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.896 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.896 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.897 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.897 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.897 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.897 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.897 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.890194_294.1", 2022-09-03T06:23:34.898 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.898 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-40", 2022-09-03T06:23:34.898 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.890194639Z", 2022-09-03T06:23:34.899 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.899 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.899 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.900 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.900 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.900 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.900 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_A4T-tDc1XWVEtWeND6N-Hqq", 2022-09-03T06:23:34.901 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.901 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.901 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.901 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.902 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.902 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.902 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.902 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.903 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.896341_295.1", 2022-09-03T06:23:34.903 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.903 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-40", 2022-09-03T06:23:34.903 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.896341961Z", 2022-09-03T06:23:34.904 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.904 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.904 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.904 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.905 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.905 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.905 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_A4T-tDc1XWVEtWeND6N-Hqq", 2022-09-03T06:23:34.906 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.906 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.906 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.907 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.907 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.907 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.907 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.908 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.908 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.947010_296.1", 2022-09-03T06:23:34.908 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.908 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41", 2022-09-03T06:23:34.909 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.947010594Z", 2022-09-03T06:23:34.909 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.909 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.909 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.910 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.910 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.910 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.911 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yitoOgqNEqy39xdDDb8hNi-", 2022-09-03T06:23:34.911 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.911 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.911 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.912 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.912 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.912 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.912 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.913 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.913 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.950610_297.1", 2022-09-03T06:23:34.913 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:34.913 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41", 2022-09-03T06:23:34.914 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.950610217Z", 2022-09-03T06:23:34.914 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.914 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.914 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.915 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.915 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.915 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.916 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yitoOgqNEqy39xdDDb8hNi-", 2022-09-03T06:23:34.916 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.917 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.917 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.918 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.918 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.918 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.919 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.919 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.920 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.951802_298.1", 2022-09-03T06:23:34.920 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.920 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-41", 2022-09-03T06:23:34.921 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.951802745Z", 2022-09-03T06:23:34.921 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.922 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.922 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.923 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.923 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.923 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.924 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_rtPxlDQFqSiMUkKzGoM6IJb", 2022-09-03T06:23:34.924 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.925 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.925 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.926 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:34.926 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.926 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.927 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.927 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:34.928 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.955393_299.1", 2022-09-03T06:23:34.928 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:34.929 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-41", 2022-09-03T06:23:34.929 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.955393441Z", 2022-09-03T06:23:34.929 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:34.929 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:34.930 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:34.930 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:34.930 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.931 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:34.931 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_rtPxlDQFqSiMUkKzGoM6IJb", 2022-09-03T06:23:34.931 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:34.931 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:34.932 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:34.932 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:34.932 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.932 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.933 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:34.933 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:34.936 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:34.937 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:34.937 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period b84ccf39-78f5-4998-9db6-42362bc8c98b --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:34.993 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:34.994 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:35.032 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:23:35.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.033 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184222.657692_58.1", 2022-09-03T06:23:35.033 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "user", 2022-09-03T06:23:35.034 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "rgw-multisite-test-user", 2022-09-03T06:23:35.034 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:50:22.657692624Z", 2022-09-03T06:23:35.035 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.035 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.036 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.036 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.036 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.037 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6BN_OEAtimzMqhhbhGc7xsF", 2022-09-03T06:23:35.037 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.037 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.038 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.038 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.038 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.039 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184222.662031_59.1", 2022-09-03T06:23:35.039 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "user", 2022-09-03T06:23:35.040 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "rgw-multisite-test-user", 2022-09-03T06:23:35.040 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:50:22.662031116Z", 2022-09-03T06:23:35.040 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.040 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.041 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.041 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.041 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.042 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6BN_OEAtimzMqhhbhGc7xsF", 2022-09-03T06:23:35.042 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.042 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.043 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.043 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.043 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.044 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.044 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.219295_62.1", 2022-09-03T06:23:35.044 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.044 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1", 2022-09-03T06:23:35.045 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.219295443Z", 2022-09-03T06:23:35.045 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.045 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.046 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.046 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.046 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.047 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Y7JDAo_bsL14HyXDq4vsLc2", 2022-09-03T06:23:35.047 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.047 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.048 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.048 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.048 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.049 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.222658_63.1", 2022-09-03T06:23:35.049 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.049 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1", 2022-09-03T06:23:35.050 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.222658346Z", 2022-09-03T06:23:35.050 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.050 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.050 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.051 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.051 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.052 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Y7JDAo_bsL14HyXDq4vsLc2", 2022-09-03T06:23:35.052 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.052 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.053 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.053 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.053 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.054 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.223873_64.1", 2022-09-03T06:23:35.054 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.055 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-1", 2022-09-03T06:23:35.055 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.223873908Z", 2022-09-03T06:23:35.055 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.055 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.056 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.056 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.056 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.057 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_VlD_M0myhVj3j7F4hjNPDp1", 2022-09-03T06:23:35.057 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.057 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.058 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.058 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.058 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.059 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.059 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.226841_65.1", 2022-09-03T06:23:35.059 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.059 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-1", 2022-09-03T06:23:35.060 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.226841368Z", 2022-09-03T06:23:35.060 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.060 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.060 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.061 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.061 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.061 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.061 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_VlD_M0myhVj3j7F4hjNPDp1", 2022-09-03T06:23:35.062 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.062 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.062 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.063 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.063 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.063 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.063 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.064 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.064 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.252557_66.1", 2022-09-03T06:23:35.064 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.064 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2", 2022-09-03T06:23:35.065 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.252557878Z", 2022-09-03T06:23:35.065 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.065 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.065 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.065 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.066 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.066 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_eGqFRgXckOShhJ5K31PxBCB", 2022-09-03T06:23:35.067 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.067 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.067 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.067 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.068 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.068 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.068 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.068 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.069 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.258211_67.1", 2022-09-03T06:23:35.069 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.069 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2", 2022-09-03T06:23:35.069 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.258211354Z", 2022-09-03T06:23:35.070 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.070 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.070 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.070 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.071 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.071 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_eGqFRgXckOShhJ5K31PxBCB", 2022-09-03T06:23:35.071 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.072 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.072 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.072 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.072 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.073 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.074 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.262819_68.1", 2022-09-03T06:23:35.074 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.074 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-2", 2022-09-03T06:23:35.074 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.262819181Z", 2022-09-03T06:23:35.075 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.075 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.075 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.075 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.076 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.076 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lkn_J0TsdIPDWPcsSr8e5eP", 2022-09-03T06:23:35.076 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.077 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.077 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.077 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.077 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.078 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.078 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.078 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.078 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.267378_69.1", 2022-09-03T06:23:35.079 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.079 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-2", 2022-09-03T06:23:35.079 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.267378646Z", 2022-09-03T06:23:35.079 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.080 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.080 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.080 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.081 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.081 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lkn_J0TsdIPDWPcsSr8e5eP", 2022-09-03T06:23:35.081 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.082 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.082 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.082 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.082 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.083 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.083 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.083 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184342.345771_70.1", 2022-09-03T06:23:35.084 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.084 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3", 2022-09-03T06:23:35.084 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.345771278Z", 2022-09-03T06:23:35.084 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.085 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.085 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.085 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.085 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.085 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.086 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_eB0pPK7VCkudq9Jt1nUnfVO", 2022-09-03T06:23:35.086 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.086 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.087 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.087 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.087 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.087 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.088 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.088 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.088 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184342.350157_71.1", 2022-09-03T06:23:35.088 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.089 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3", 2022-09-03T06:23:35.089 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.350157638Z", 2022-09-03T06:23:35.089 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.089 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.090 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.090 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.090 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.091 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_eB0pPK7VCkudq9Jt1nUnfVO", 2022-09-03T06:23:35.091 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.091 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.092 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.092 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.092 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.093 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184342.351175_72.1", 2022-09-03T06:23:35.093 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.093 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-3", 2022-09-03T06:23:35.094 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.351175819Z", 2022-09-03T06:23:35.094 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.094 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.094 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.095 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.095 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.095 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.095 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_xQbDQL6LoF7KY81FCBsr-TR", 2022-09-03T06:23:35.096 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.096 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.096 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.097 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.097 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.097 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.098 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184342.357275_73.1", 2022-09-03T06:23:35.098 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.098 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-3", 2022-09-03T06:23:35.098 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.357275634Z", 2022-09-03T06:23:35.099 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.099 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.099 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.099 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.100 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.100 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_xQbDQL6LoF7KY81FCBsr-TR", 2022-09-03T06:23:35.100 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.101 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.101 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.101 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.101 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.102 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.102 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.102 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.193842_74.1", 2022-09-03T06:23:35.103 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.103 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4", 2022-09-03T06:23:35.103 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.193842881Z", 2022-09-03T06:23:35.103 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.104 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.104 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.104 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.105 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.105 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lw10L7BpVQy7VSU2U7UHl07", 2022-09-03T06:23:35.105 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.105 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.106 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.106 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.106 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.107 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.107 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.107 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.198704_75.1", 2022-09-03T06:23:35.108 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.108 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4", 2022-09-03T06:23:35.108 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.198704862Z", 2022-09-03T06:23:35.108 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.109 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.109 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.109 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.109 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.109 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.110 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lw10L7BpVQy7VSU2U7UHl07", 2022-09-03T06:23:35.110 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.111 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.111 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.111 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.112 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.113 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.113 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.200713_76.1", 2022-09-03T06:23:35.114 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.114 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-4", 2022-09-03T06:23:35.115 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.200713524Z", 2022-09-03T06:23:35.115 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.115 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.116 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.116 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.116 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.116 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.117 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2mGIDNrcwT5FJidyB-XgESn", 2022-09-03T06:23:35.117 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.118 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.118 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.118 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.119 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.119 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.120 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.204734_77.1", 2022-09-03T06:23:35.120 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.120 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-4", 2022-09-03T06:23:35.120 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.204734948Z", 2022-09-03T06:23:35.121 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.121 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.121 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.121 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.122 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.122 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2mGIDNrcwT5FJidyB-XgESn", 2022-09-03T06:23:35.123 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.123 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.123 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.124 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.124 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.124 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.125 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.224447_78.1", 2022-09-03T06:23:35.125 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.125 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5", 2022-09-03T06:23:35.126 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.224447947Z", 2022-09-03T06:23:35.126 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.126 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.126 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.127 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.127 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.128 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_xo2PU3_oh6eFpkPNkexfq7D", 2022-09-03T06:23:35.128 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.128 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.129 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.129 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.129 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.130 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.227473_79.1", 2022-09-03T06:23:35.130 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.131 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5", 2022-09-03T06:23:35.131 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.227473639Z", 2022-09-03T06:23:35.131 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.131 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.132 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.132 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.133 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.133 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_xo2PU3_oh6eFpkPNkexfq7D", 2022-09-03T06:23:35.133 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.134 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.134 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.134 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.135 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.135 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.135 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.228594_80.1", 2022-09-03T06:23:35.136 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.136 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-5", 2022-09-03T06:23:35.136 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.228594676Z", 2022-09-03T06:23:35.136 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.137 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.137 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.137 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.138 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.138 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_u7x8KJF8hnPDPr0uRDbixnY", 2022-09-03T06:23:35.138 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.139 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.139 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.139 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.139 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.140 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.231192_81.1", 2022-09-03T06:23:35.141 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.141 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-5", 2022-09-03T06:23:35.141 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.231192184Z", 2022-09-03T06:23:35.142 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.142 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.142 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.143 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.143 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.143 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_u7x8KJF8hnPDPr0uRDbixnY", 2022-09-03T06:23:35.144 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.144 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.144 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.145 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.145 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.145 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.146 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.255262_82.1", 2022-09-03T06:23:35.146 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.146 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6", 2022-09-03T06:23:35.146 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.255262164Z", 2022-09-03T06:23:35.147 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.147 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.147 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.147 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.148 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.148 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_7T2FDkJXu_-0lpyQJ4EERx0", 2022-09-03T06:23:35.148 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.149 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.149 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.149 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.150 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.150 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.150 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.151 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.258031_83.1", 2022-09-03T06:23:35.151 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.151 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6", 2022-09-03T06:23:35.152 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.258031985Z", 2022-09-03T06:23:35.152 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.152 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.152 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.153 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.153 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.153 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_7T2FDkJXu_-0lpyQJ4EERx0", 2022-09-03T06:23:35.154 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.154 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.154 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.155 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.155 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.156 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.259057_84.1", 2022-09-03T06:23:35.156 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.156 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-6", 2022-09-03T06:23:35.156 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.259057684Z", 2022-09-03T06:23:35.157 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.157 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.157 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.157 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.158 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.158 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_JcUmTGR0SXlRqG5W9Zl5D8p", 2022-09-03T06:23:35.159 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.159 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.160 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.160 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.160 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.161 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.161 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.261653_85.1", 2022-09-03T06:23:35.161 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.161 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-6", 2022-09-03T06:23:35.162 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.261653432Z", 2022-09-03T06:23:35.162 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.162 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.162 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.163 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.163 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.163 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.163 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_JcUmTGR0SXlRqG5W9Zl5D8p", 2022-09-03T06:23:35.164 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.164 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.165 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.165 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.165 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.166 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.166 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.855330_86.1", 2022-09-03T06:23:35.167 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.167 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:35.167 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.855330912Z", 2022-09-03T06:23:35.167 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.167 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.168 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.168 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.168 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.169 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.169 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:35.169 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.170 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.170 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.170 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.170 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.171 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.171 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.171 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.171 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.861502_87.1", 2022-09-03T06:23:35.172 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.172 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:35.172 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.861502176Z", 2022-09-03T06:23:35.172 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.173 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.173 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.173 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.174 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.174 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:35.174 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.175 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.175 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.175 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.175 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.176 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.176 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.176 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.862483_88.1", 2022-09-03T06:23:35.176 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.177 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7", 2022-09-03T06:23:35.177 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.862483283Z", 2022-09-03T06:23:35.177 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.177 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.178 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.178 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.178 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.179 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:35.179 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.179 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.179 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.180 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.180 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.180 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.181 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.181 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.867021_89.1", 2022-09-03T06:23:35.181 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.181 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7", 2022-09-03T06:23:35.182 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.867021552Z", 2022-09-03T06:23:35.182 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.182 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.182 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.183 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.183 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.183 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.183 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:35.184 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.184 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.184 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.184 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.185 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.185 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.185 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.185 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.186 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.888960_90.1", 2022-09-03T06:23:35.186 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.186 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:35.187 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.888960687Z", 2022-09-03T06:23:35.187 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.187 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.187 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.188 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.188 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.188 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.188 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:35.189 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.189 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.189 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.189 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.190 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.190 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.190 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.191 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.191 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.892373_91.1", 2022-09-03T06:23:35.191 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.192 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:35.192 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.892373633Z", 2022-09-03T06:23:35.192 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.193 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.193 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.193 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.194 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.194 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.194 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:35.194 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.195 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.195 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.195 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.195 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.196 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.196 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.196 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.196 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.893288_92.1", 2022-09-03T06:23:35.197 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.197 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8", 2022-09-03T06:23:35.197 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.893288080Z", 2022-09-03T06:23:35.198 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.198 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.198 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.198 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.199 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.199 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.199 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:35.199 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.200 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.200 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.200 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.200 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.201 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.201 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.201 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.201 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.895751_93.1", 2022-09-03T06:23:35.202 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.202 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8", 2022-09-03T06:23:35.202 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.895751491Z", 2022-09-03T06:23:35.203 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.203 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.203 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.203 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.204 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.204 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:35.205 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.205 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.205 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.205 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.206 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.206 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.206 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.207 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.207 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.922520_94.1", 2022-09-03T06:23:35.207 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.207 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:35.208 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.922520779Z", 2022-09-03T06:23:35.208 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.209 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.209 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.209 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.210 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.210 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:35.211 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.211 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.211 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.212 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.212 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.212 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.212 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.213 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.931081_95.1", 2022-09-03T06:23:35.213 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.213 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:35.213 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.931081374Z", 2022-09-03T06:23:35.214 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.214 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.214 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.215 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.215 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.215 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:35.216 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.216 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.216 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.218 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.933197_96.1", 2022-09-03T06:23:35.218 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.219 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9", 2022-09-03T06:23:35.219 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.933197726Z", 2022-09-03T06:23:35.219 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.220 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.220 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.220 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.221 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.221 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:35.221 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.222 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.222 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.222 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.223 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.224 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.224 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.225 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.938376_97.1", 2022-09-03T06:23:35.225 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.226 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9", 2022-09-03T06:23:35.226 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.938376383Z", 2022-09-03T06:23:35.227 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.227 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.228 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.228 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.229 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.230 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:35.230 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.231 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.231 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.232 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.232 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.233 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.234 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.570224_98.1", 2022-09-03T06:23:35.235 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.235 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:35.236 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.570224266Z", 2022-09-03T06:23:35.236 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.237 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.237 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:35.238 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.239 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.239 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:35.240 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.241 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.242 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.242 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.242 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.244 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.574509_99.1", 2022-09-03T06:23:35.244 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.245 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:35.245 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.574509413Z", 2022-09-03T06:23:35.245 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.245 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.246 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:35.246 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.247 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.247 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:35.247 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.248 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.248 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.248 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.248 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.249 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.584013_100.1", 2022-09-03T06:23:35.249 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.250 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7", 2022-09-03T06:23:35.250 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.584013358Z", 2022-09-03T06:23:35.250 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.250 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.251 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:35.251 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.251 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.252 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:35.252 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.253 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.253 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "remove" 2022-09-03T06:23:35.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.254 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.587599_101.1", 2022-09-03T06:23:35.255 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.255 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7", 2022-09-03T06:23:35.255 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.587599896Z", 2022-09-03T06:23:35.255 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.256 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.256 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:35.256 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.257 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.257 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:35.257 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.258 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.258 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.258 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.259 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.259 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.260 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.619234_102.1", 2022-09-03T06:23:35.260 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.260 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:35.260 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.619234589Z", 2022-09-03T06:23:35.261 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.261 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.261 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:35.262 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.262 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.262 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.262 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:35.263 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.263 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.264 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.264 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.264 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.265 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.625715_103.1", 2022-09-03T06:23:35.265 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.265 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:35.266 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.625715365Z", 2022-09-03T06:23:35.266 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.266 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.266 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:35.267 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.267 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.267 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:35.268 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.268 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.269 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.269 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.269 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.270 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.637706_104.1", 2022-09-03T06:23:35.270 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.271 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8", 2022-09-03T06:23:35.271 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.637706717Z", 2022-09-03T06:23:35.271 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.271 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.272 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:35.272 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.272 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.273 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.273 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:35.273 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.274 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.274 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "remove" 2022-09-03T06:23:35.274 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.274 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.275 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.639994_105.1", 2022-09-03T06:23:35.275 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.276 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8", 2022-09-03T06:23:35.276 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.639994804Z", 2022-09-03T06:23:35.276 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.276 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.277 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:35.277 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.277 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.278 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:35.278 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.279 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.279 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.279 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.279 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.280 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.697911_106.1", 2022-09-03T06:23:35.280 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.281 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:35.281 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.697911177Z", 2022-09-03T06:23:35.281 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.281 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.282 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:35.282 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.283 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.283 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:35.283 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.284 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.284 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.284 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.284 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.285 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.700455_107.1", 2022-09-03T06:23:35.285 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.286 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:35.289 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.700455333Z", 2022-09-03T06:23:35.289 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.290 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.290 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:35.290 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.291 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.292 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:35.292 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.293 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.293 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.293 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.294 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.294 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.295 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.712013_108.1", 2022-09-03T06:23:35.295 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.296 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9", 2022-09-03T06:23:35.296 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.712013759Z", 2022-09-03T06:23:35.296 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.297 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.297 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:35.297 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.298 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.298 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:35.298 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.299 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.299 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "remove" 2022-09-03T06:23:35.300 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.300 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.301 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.715807_109.1", 2022-09-03T06:23:35.301 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.301 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9", 2022-09-03T06:23:35.301 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807393Z", 2022-09-03T06:23:35.302 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.302 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.302 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:35.303 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.303 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.303 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.304 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:35.304 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.304 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.304 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.305 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.305 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.305 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.305 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.306 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.306 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.561415_64.1", 2022-09-03T06:23:35.306 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.307 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:23:35.307 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.561415905Z", 2022-09-03T06:23:35.307 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.307 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.308 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.308 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.308 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.309 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.309 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6ugH-NS-_bEeQKUdolNq7Oa", 2022-09-03T06:23:35.309 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.310 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.310 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.310 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.310 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.311 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.311 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.311 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.312 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.566930_66.1", 2022-09-03T06:23:35.312 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.312 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:23:35.312 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.566930651Z", 2022-09-03T06:23:35.313 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.313 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.313 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.314 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.314 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.314 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.314 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6ugH-NS-_bEeQKUdolNq7Oa", 2022-09-03T06:23:35.315 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.315 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.315 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.316 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.316 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.316 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.316 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.317 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.317 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.569237_67.1", 2022-09-03T06:23:35.317 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.318 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-10", 2022-09-03T06:23:35.318 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.569237568Z", 2022-09-03T06:23:35.318 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.318 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.319 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.319 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.319 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.320 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.320 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_htY0ij10LAX0ZNP6T_0mHNL", 2022-09-03T06:23:35.321 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.321 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.322 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.322 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.323 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.323 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.324 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.324 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.325 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.574987_68.1", 2022-09-03T06:23:35.325 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.326 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-10", 2022-09-03T06:23:35.326 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.574987235Z", 2022-09-03T06:23:35.327 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.327 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.328 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.329 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.329 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.330 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.330 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_htY0ij10LAX0ZNP6T_0mHNL", 2022-09-03T06:23:35.331 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.331 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.332 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.332 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.333 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.333 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.334 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.334 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.335 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.595637_69.1", 2022-09-03T06:23:35.335 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.336 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:23:35.336 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.595637832Z", 2022-09-03T06:23:35.337 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.337 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.338 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.338 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.339 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.339 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.340 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gtTH1QqAjuxzvv9DiFtpT7B", 2022-09-03T06:23:35.340 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.340 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.341 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.341 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.342 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.342 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.342 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.343 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.343 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.599901_70.1", 2022-09-03T06:23:35.343 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.344 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:23:35.344 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.599901361Z", 2022-09-03T06:23:35.344 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.345 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.345 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.346 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.346 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.346 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.347 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gtTH1QqAjuxzvv9DiFtpT7B", 2022-09-03T06:23:35.347 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.347 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.348 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.348 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.348 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.349 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.349 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.350 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.350 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.601762_71.1", 2022-09-03T06:23:35.351 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.351 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-11", 2022-09-03T06:23:35.352 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.601762485Z", 2022-09-03T06:23:35.352 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.353 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.353 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.354 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.354 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.355 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.355 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_umwMbGorUx0G8JCmxgLXcXQ", 2022-09-03T06:23:35.356 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.356 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.357 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.357 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.358 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.358 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.359 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.359 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.360 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.605809_72.1", 2022-09-03T06:23:35.360 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.361 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-11", 2022-09-03T06:23:35.361 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.605809267Z", 2022-09-03T06:23:35.362 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.362 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.363 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.363 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.364 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.364 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.365 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_umwMbGorUx0G8JCmxgLXcXQ", 2022-09-03T06:23:35.365 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.366 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.366 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.367 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.367 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.368 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.368 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.369 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.369 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.642680_73.1", 2022-09-03T06:23:35.370 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.370 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:23:35.371 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.642680630Z", 2022-09-03T06:23:35.371 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.372 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.372 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.373 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.373 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.373 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.374 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_pT_Dw9zXIA2-9UcP5Gn1GlK", 2022-09-03T06:23:35.374 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.374 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.374 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.375 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.375 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.375 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.376 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.376 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.377 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.645295_74.1", 2022-09-03T06:23:35.377 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.378 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:23:35.378 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.645295120Z", 2022-09-03T06:23:35.379 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.379 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.380 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.380 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.381 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.381 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.381 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_pT_Dw9zXIA2-9UcP5Gn1GlK", 2022-09-03T06:23:35.382 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.382 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.383 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.383 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.384 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.384 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.385 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.385 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.386 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.646235_75.1", 2022-09-03T06:23:35.386 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.387 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-12", 2022-09-03T06:23:35.387 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.646235577Z", 2022-09-03T06:23:35.388 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.388 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.389 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.389 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.390 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.390 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.391 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ig99b4m7-ssuB2Tw7htsmQp", 2022-09-03T06:23:35.391 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.392 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.392 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.393 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.393 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.394 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.394 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.395 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.395 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.648716_76.1", 2022-09-03T06:23:35.396 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.396 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-12", 2022-09-03T06:23:35.397 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.648716080Z", 2022-09-03T06:23:35.397 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.398 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.398 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.399 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.399 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.400 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.400 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ig99b4m7-ssuB2Tw7htsmQp", 2022-09-03T06:23:35.401 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.401 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.402 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.402 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.403 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.403 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.404 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.404 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.405 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.356235_99.1", 2022-09-03T06:23:35.405 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.406 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:23:35.406 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.356235827Z", 2022-09-03T06:23:35.407 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.407 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.408 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.408 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.409 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.409 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.410 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8iK3rbCB7fbjpNaGZ4Lk2f9", 2022-09-03T06:23:35.410 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.411 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.411 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.412 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.412 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.413 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.413 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.414 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.414 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.364532_100.1", 2022-09-03T06:23:35.415 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.415 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:23:35.416 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.364532891Z", 2022-09-03T06:23:35.416 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.417 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.417 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.418 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.418 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.419 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.419 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8iK3rbCB7fbjpNaGZ4Lk2f9", 2022-09-03T06:23:35.420 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.420 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.421 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.421 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.422 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.422 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.423 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.423 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.424 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.365724_101.1", 2022-09-03T06:23:35.424 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.425 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-13", 2022-09-03T06:23:35.425 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.365724056Z", 2022-09-03T06:23:35.426 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.426 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.427 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.427 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.428 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.428 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.429 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_hGY9xi1--Qa5DBRaR1Ev2Td", 2022-09-03T06:23:35.429 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.430 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.430 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.431 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.431 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.432 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.432 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.433 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.433 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.372032_102.1", 2022-09-03T06:23:35.433 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.434 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-13", 2022-09-03T06:23:35.434 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.372032718Z", 2022-09-03T06:23:35.435 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.435 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.436 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.436 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.437 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.437 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.438 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_hGY9xi1--Qa5DBRaR1Ev2Td", 2022-09-03T06:23:35.438 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.439 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.440 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.440 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.441 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.441 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.442 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.442 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.400096_103.1", 2022-09-03T06:23:35.442 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.443 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:23:35.443 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.400096996Z", 2022-09-03T06:23:35.444 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.444 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.445 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.445 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.446 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.447 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.447 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_CLeQ9451sE39DW-o5-rQ9Gb", 2022-09-03T06:23:35.448 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.448 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.449 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.449 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.450 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.450 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.451 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.451 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.406131_104.1", 2022-09-03T06:23:35.451 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.452 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:23:35.452 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.406131833Z", 2022-09-03T06:23:35.452 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.453 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.453 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.453 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.454 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.454 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.455 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_CLeQ9451sE39DW-o5-rQ9Gb", 2022-09-03T06:23:35.455 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.455 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.456 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.456 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.456 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.457 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.457 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.458 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.407235_105.1", 2022-09-03T06:23:35.458 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.458 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-14", 2022-09-03T06:23:35.459 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.407235283Z", 2022-09-03T06:23:35.459 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.459 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.460 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.460 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.460 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.460 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.461 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_E3QteIhTh8XKCT60Fs2visZ", 2022-09-03T06:23:35.461 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.462 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.462 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.462 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.462 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.463 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.413053_106.1", 2022-09-03T06:23:35.463 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.464 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-14", 2022-09-03T06:23:35.464 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.413053852Z", 2022-09-03T06:23:35.464 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.464 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.465 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.465 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.465 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.466 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_E3QteIhTh8XKCT60Fs2visZ", 2022-09-03T06:23:35.466 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.466 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.466 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.467 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.467 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.467 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.468 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.447727_107.1", 2022-09-03T06:23:35.468 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.468 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:23:35.469 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.447727339Z", 2022-09-03T06:23:35.469 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.469 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.469 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.470 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.470 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.470 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6mYNO_dnwhvNSPaHRDsVWkL", 2022-09-03T06:23:35.471 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.471 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.471 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.471 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.472 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.472 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.472 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.473 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.450607_108.1", 2022-09-03T06:23:35.473 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.473 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:23:35.473 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.450607653Z", 2022-09-03T06:23:35.474 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.474 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.474 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.474 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.475 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.475 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.475 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6mYNO_dnwhvNSPaHRDsVWkL", 2022-09-03T06:23:35.475 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.476 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.476 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.476 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.477 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.477 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.451654_109.1", 2022-09-03T06:23:35.478 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.478 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-15", 2022-09-03T06:23:35.478 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.451654129Z", 2022-09-03T06:23:35.478 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.479 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.479 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.479 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.480 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.480 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_iMvYVCIRH1guRCrQSTbYt62", 2022-09-03T06:23:35.480 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.481 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.481 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.482 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.482 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.483 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.454511_110.1", 2022-09-03T06:23:35.483 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.483 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-15", 2022-09-03T06:23:35.483 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.454511694Z", 2022-09-03T06:23:35.484 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.484 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.484 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.484 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.485 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.486 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_iMvYVCIRH1guRCrQSTbYt62", 2022-09-03T06:23:35.486 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.486 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.487 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.487 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.487 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.488 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184551.125768_119.1", 2022-09-03T06:23:35.488 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.489 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:35.489 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:51.125768074Z", 2022-09-03T06:23:35.489 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.490 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.490 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.490 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.491 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.491 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:35.491 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.492 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.492 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.494 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184551.133544_120.1", 2022-09-03T06:23:35.494 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.495 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:35.495 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:51.133544960Z", 2022-09-03T06:23:35.495 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.496 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.496 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.496 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.497 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.498 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:35.498 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.498 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.499 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.499 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.499 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.500 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184551.134772_121.1", 2022-09-03T06:23:35.501 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.501 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16", 2022-09-03T06:23:35.501 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:51.134772057Z", 2022-09-03T06:23:35.501 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.502 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.502 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.502 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.503 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.503 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yedFU2W-LnAQG28SX6Ff-RY", 2022-09-03T06:23:35.503 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.504 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.504 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.505 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.505 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.506 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184551.139229_122.1", 2022-09-03T06:23:35.506 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.506 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16", 2022-09-03T06:23:35.507 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:51.139229501Z", 2022-09-03T06:23:35.507 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.507 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.507 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.508 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.508 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.508 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yedFU2W-LnAQG28SX6Ff-RY", 2022-09-03T06:23:35.509 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.509 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.509 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.510 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.510 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.511 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.168524_123.1", 2022-09-03T06:23:35.511 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.511 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:35.512 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.168524769Z", 2022-09-03T06:23:35.512 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.512 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.513 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.513 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.514 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.514 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:35.514 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.515 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.515 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.515 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.516 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.516 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.172232_124.1", 2022-09-03T06:23:35.517 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.517 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:35.517 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.172232256Z", 2022-09-03T06:23:35.517 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.518 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.518 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.518 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.519 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.519 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:35.520 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.520 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.521 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.521 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.522 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.522 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.173517_125.1", 2022-09-03T06:23:35.523 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.523 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17", 2022-09-03T06:23:35.524 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.173517470Z", 2022-09-03T06:23:35.524 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.524 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.525 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.525 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.526 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.526 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_KGtuntqhz3tvT1e4k36oFKj", 2022-09-03T06:23:35.526 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.527 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.528 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.528 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.528 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.529 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.178102_126.1", 2022-09-03T06:23:35.530 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.530 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17", 2022-09-03T06:23:35.530 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.178102240Z", 2022-09-03T06:23:35.531 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.531 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.531 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.532 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.532 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.533 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_KGtuntqhz3tvT1e4k36oFKj", 2022-09-03T06:23:35.533 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.534 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.534 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.535 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.535 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.536 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.536 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.215736_127.1", 2022-09-03T06:23:35.537 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.537 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:35.538 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.215736666Z", 2022-09-03T06:23:35.538 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.539 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.539 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.540 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.541 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.541 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:35.541 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.542 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.542 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.542 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.543 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.543 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.544 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.219063_128.1", 2022-09-03T06:23:35.545 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.545 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:35.545 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.219063433Z", 2022-09-03T06:23:35.546 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.546 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.546 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.547 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.548 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.548 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:35.548 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.549 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.549 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.550 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.550 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.551 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.220690_129.1", 2022-09-03T06:23:35.552 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.552 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18", 2022-09-03T06:23:35.553 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.220690992Z", 2022-09-03T06:23:35.553 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.553 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.554 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.554 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.555 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.555 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm651SSphsn2Jx7Kwrq9oNR", 2022-09-03T06:23:35.556 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.556 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.557 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.557 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.558 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.559 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.226118_130.1", 2022-09-03T06:23:35.559 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.560 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18", 2022-09-03T06:23:35.560 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.226118188Z", 2022-09-03T06:23:35.561 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.561 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.561 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.562 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.562 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.563 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm651SSphsn2Jx7Kwrq9oNR", 2022-09-03T06:23:35.563 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.564 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.564 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.565 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.565 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.566 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.250228_131.1", 2022-09-03T06:23:35.566 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.567 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:35.567 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.250228835Z", 2022-09-03T06:23:35.567 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.568 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.568 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:35.569 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.569 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.569 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:35.570 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.570 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.571 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.571 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.571 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.572 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.573 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.253497_132.1", 2022-09-03T06:23:35.573 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.573 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:35.574 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.253497827Z", 2022-09-03T06:23:35.574 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.574 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.575 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:35.575 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.576 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.576 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:35.576 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.577 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.577 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.578 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.578 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.579 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.579 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.266389_133.1", 2022-09-03T06:23:35.579 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.580 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:35.580 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.266389718Z", 2022-09-03T06:23:35.580 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.581 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.581 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:35.582 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.582 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.583 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:35.583 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.583 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.584 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.584 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.585 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.585 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.586 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.270858_134.1", 2022-09-03T06:23:35.586 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.587 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:35.587 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.270858165Z", 2022-09-03T06:23:35.587 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.587 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.588 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:35.588 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.589 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.589 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:35.590 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.590 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.590 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.591 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.591 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.592 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.289855_135.1", 2022-09-03T06:23:35.592 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.593 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:35.593 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.289855850Z", 2022-09-03T06:23:35.593 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.594 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.594 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:35.594 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.595 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.595 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.595 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:35.596 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.596 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.596 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.597 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.597 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.597 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.598 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.296862_136.1", 2022-09-03T06:23:35.598 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.599 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:35.599 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.296862507Z", 2022-09-03T06:23:35.599 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.600 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.600 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:35.600 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.601 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.601 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:35.601 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.602 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.602 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.602 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.602 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.603 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.168986_143.1", 2022-09-03T06:23:35.604 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.604 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:35.604 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.168986972Z", 2022-09-03T06:23:35.605 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.605 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.606 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.606 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.607 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.607 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:35.607 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.608 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.608 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.608 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.609 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.609 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.610 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.610 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.174598_144.1", 2022-09-03T06:23:35.610 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.611 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:35.611 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.174598788Z", 2022-09-03T06:23:35.611 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.611 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.612 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.612 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.613 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.614 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:35.614 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.614 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.615 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.615 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.616 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.616 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.618 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.176005_145.1", 2022-09-03T06:23:35.618 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.619 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19", 2022-09-03T06:23:35.619 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.176005675Z", 2022-09-03T06:23:35.620 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.620 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.620 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.621 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.621 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.622 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yDXruVlqT4Xsg8Al8Plz6TA", 2022-09-03T06:23:35.622 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.623 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.623 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.624 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.624 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.625 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.179021_146.1", 2022-09-03T06:23:35.626 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.626 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19", 2022-09-03T06:23:35.626 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.179021577Z", 2022-09-03T06:23:35.627 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.627 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.628 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.628 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.629 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.629 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yDXruVlqT4Xsg8Al8Plz6TA", 2022-09-03T06:23:35.629 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.630 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.630 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.630 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.631 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.631 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.632 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.632 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.194719_147.1", 2022-09-03T06:23:35.632 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.633 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:35.633 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.194719304Z", 2022-09-03T06:23:35.634 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.634 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.634 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:35.635 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.635 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.636 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:35.636 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.637 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.637 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.638 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.638 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.638 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.639 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.201443_148.1", 2022-09-03T06:23:35.640 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.640 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:35.640 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.201443123Z", 2022-09-03T06:23:35.641 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.641 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.641 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:35.642 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.643 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.643 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:35.643 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.644 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.644 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.644 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.645 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.645 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.646 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.646 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185259.164178_150.1", 2022-09-03T06:23:35.647 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.647 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:35.647 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.164178377Z", 2022-09-03T06:23:35.648 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.648 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.649 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.649 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.650 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.650 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.650 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.651 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.651 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.651 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.652 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.652 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.652 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.653 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.653 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185259.166746_154.1", 2022-09-03T06:23:35.653 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.654 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:35.654 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.166746306Z", 2022-09-03T06:23:35.654 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.655 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.655 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.655 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.656 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.656 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.657 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.657 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.658 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.658 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.658 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.659 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.659 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.659 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185259.167637_155.1", 2022-09-03T06:23:35.660 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.660 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20", 2022-09-03T06:23:35.660 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.167637326Z", 2022-09-03T06:23:35.661 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.661 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.662 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.662 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.662 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.663 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.663 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZKafDPgftyjUAayY_VUswW5", 2022-09-03T06:23:35.664 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.664 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.664 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.665 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.665 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.665 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.666 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.666 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.667 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185259.169934_156.1", 2022-09-03T06:23:35.667 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.667 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20", 2022-09-03T06:23:35.668 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.169934736Z", 2022-09-03T06:23:35.668 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.668 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.669 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.669 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.670 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.670 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.670 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZKafDPgftyjUAayY_VUswW5", 2022-09-03T06:23:35.671 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.671 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.671 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.672 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.672 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.673 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.673 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.673 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.673 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185351.815540_157.1", 2022-09-03T06:23:35.674 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.674 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:35.674 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:09:11.815540177Z", 2022-09-03T06:23:35.675 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.675 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.676 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.676 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.676 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.677 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.677 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.677 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.678 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.678 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.678 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.679 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.679 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.680 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.680 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.680 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185351.819576_158.1", 2022-09-03T06:23:35.681 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.681 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:35.682 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:09:11.819576411Z", 2022-09-03T06:23:35.682 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.683 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.683 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.683 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.684 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.684 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.684 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.685 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.685 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.685 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.686 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.686 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.687 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.687 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.687 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.688 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185415.254080_165.1", 2022-09-03T06:23:35.688 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.688 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:35.689 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:10:15.254080988Z", 2022-09-03T06:23:35.689 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.690 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.690 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.690 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.691 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.691 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.692 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.692 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:35.692 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.692 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.693 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.693 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.693 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.694 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.694 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.694 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185415.257852_166.1", 2022-09-03T06:23:35.695 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.695 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:35.696 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:10:15.257852595Z", 2022-09-03T06:23:35.696 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.696 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.697 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.697 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.697 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.698 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.698 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:35.699 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:35.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.699 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.700 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.700 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.701 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.701 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.701 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.702 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.140367_167.1", 2022-09-03T06:23:35.702 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.702 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:35.703 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.140367923Z", 2022-09-03T06:23:35.703 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.704 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.704 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.704 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.705 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.705 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.706 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:35.706 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.706 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.706 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.707 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.707 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.708 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.708 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.708 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.709 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.143952_168.1", 2022-09-03T06:23:35.709 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.709 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:35.710 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.143952074Z", 2022-09-03T06:23:35.710 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.710 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.711 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.711 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.712 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.712 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.712 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:35.713 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.713 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.713 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.714 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.714 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.714 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.715 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.715 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.716 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.145226_169.1", 2022-09-03T06:23:35.716 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.716 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21", 2022-09-03T06:23:35.717 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.145226679Z", 2022-09-03T06:23:35.717 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.717 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.718 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.718 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.719 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.719 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.719 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WXbrqJZWiL3N6i3X5RrSf_L", 2022-09-03T06:23:35.720 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.720 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.721 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.721 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.721 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.722 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.722 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.723 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.723 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.148908_170.1", 2022-09-03T06:23:35.723 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.724 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21", 2022-09-03T06:23:35.724 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.148908181Z", 2022-09-03T06:23:35.725 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.725 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.725 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.726 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.726 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.727 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.727 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WXbrqJZWiL3N6i3X5RrSf_L", 2022-09-03T06:23:35.727 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.728 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.728 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.728 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.729 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.729 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.729 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.730 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.730 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.171153_171.1", 2022-09-03T06:23:35.730 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.731 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:35.731 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.171153368Z", 2022-09-03T06:23:35.732 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.732 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.732 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.733 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.733 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.734 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.734 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:35.734 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.735 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.735 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.736 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.736 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.736 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.737 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.737 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.737 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.174859_172.1", 2022-09-03T06:23:35.738 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.738 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:35.738 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.174859152Z", 2022-09-03T06:23:35.739 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.739 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.739 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.739 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.740 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.740 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.741 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:35.741 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.741 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.742 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.742 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.742 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.743 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.743 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.744 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.744 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.176058_173.1", 2022-09-03T06:23:35.744 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.745 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22", 2022-09-03T06:23:35.745 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.176058283Z", 2022-09-03T06:23:35.745 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.746 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.746 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.747 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.747 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.748 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.748 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_4TVYbHf2FAhumE819lO37xf", 2022-09-03T06:23:35.749 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.749 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.749 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.750 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.750 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.751 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.751 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.752 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.752 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.180790_174.1", 2022-09-03T06:23:35.752 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.753 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22", 2022-09-03T06:23:35.754 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.180790715Z", 2022-09-03T06:23:35.754 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.755 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.755 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.756 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.756 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.757 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.757 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_4TVYbHf2FAhumE819lO37xf", 2022-09-03T06:23:35.758 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.758 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.759 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.759 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.760 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.760 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.761 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.761 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.762 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.221697_175.1", 2022-09-03T06:23:35.762 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.763 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:35.763 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.221697751Z", 2022-09-03T06:23:35.763 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.764 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.764 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.765 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.765 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.766 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.766 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:35.767 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.767 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.768 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.768 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.769 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.769 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.769 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.770 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.770 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.229951_176.1", 2022-09-03T06:23:35.771 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.771 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:35.772 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.229951839Z", 2022-09-03T06:23:35.772 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.772 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.773 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.773 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.774 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.774 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.775 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:35.775 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.775 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.776 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.776 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.777 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.777 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.778 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.778 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.779 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.237269_177.1", 2022-09-03T06:23:35.779 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.780 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23", 2022-09-03T06:23:35.780 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.237269112Z", 2022-09-03T06:23:35.781 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.781 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.782 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.782 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.783 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.784 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.784 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_SUW5WcOi0sbRMVG7llLAyro", 2022-09-03T06:23:35.784 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.785 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.785 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.786 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.786 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.787 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.788 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.788 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.788 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.248604_178.1", 2022-09-03T06:23:35.789 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.789 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23", 2022-09-03T06:23:35.790 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.248604519Z", 2022-09-03T06:23:35.790 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.790 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.791 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.791 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.791 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.791 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.792 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_SUW5WcOi0sbRMVG7llLAyro", 2022-09-03T06:23:35.792 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.792 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.792 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.793 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.793 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.793 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.794 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.794 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.794 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.289134_179.1", 2022-09-03T06:23:35.795 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.795 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:35.796 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.289134872Z", 2022-09-03T06:23:35.796 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.796 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.796 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:35.797 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.797 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.797 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.797 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:35.798 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.798 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.798 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.799 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.799 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.799 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.799 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.800 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.800 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.292023_180.1", 2022-09-03T06:23:35.800 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.800 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:35.801 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.292023076Z", 2022-09-03T06:23:35.801 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.801 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.801 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:35.802 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.802 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.803 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.803 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:35.803 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.804 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.804 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.804 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.805 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.805 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.805 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.805 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.806 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.305537_181.1", 2022-09-03T06:23:35.806 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.806 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:35.807 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.305537948Z", 2022-09-03T06:23:35.807 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.808 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.808 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:35.808 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.809 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.809 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.809 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:35.809 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.810 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.810 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.810 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.811 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.811 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.811 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.812 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.812 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.308323_182.1", 2022-09-03T06:23:35.812 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.812 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:35.813 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.308323992Z", 2022-09-03T06:23:35.813 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.813 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.813 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:35.814 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.814 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.814 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.814 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:35.815 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.815 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.815 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.815 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.816 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.816 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.816 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.816 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.817 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.323485_183.1", 2022-09-03T06:23:35.817 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.817 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:35.817 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.323485482Z", 2022-09-03T06:23:35.818 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.818 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.818 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:35.818 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.819 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.819 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.820 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:35.820 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.820 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.820 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.821 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.821 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.821 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.821 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.822 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.822 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.326245_184.1", 2022-09-03T06:23:35.822 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.822 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:35.823 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.326245244Z", 2022-09-03T06:23:35.823 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.823 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.824 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:35.824 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.824 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.824 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.825 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:35.825 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.825 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.825 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.826 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.826 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.826 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.826 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.827 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.827 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.128610_187.1", 2022-09-03T06:23:35.827 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.827 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:35.828 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.128610110Z", 2022-09-03T06:23:35.828 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.829 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.829 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.829 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.829 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.830 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.830 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.830 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.830 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.831 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.831 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.831 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.831 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.832 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.832 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.832 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.132654_188.1", 2022-09-03T06:23:35.832 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.833 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:35.833 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.132654659Z", 2022-09-03T06:23:35.833 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.833 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.834 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.834 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.834 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.834 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.835 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.835 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.835 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.835 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.836 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.836 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.836 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.836 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.837 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.837 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.133913_189.1", 2022-09-03T06:23:35.838 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.838 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24", 2022-09-03T06:23:35.838 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.133913786Z", 2022-09-03T06:23:35.838 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.839 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.839 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.839 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.839 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.840 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.840 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_flubJt4QosOvedZNv50fpC6", 2022-09-03T06:23:35.840 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.840 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.841 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.841 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.841 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.841 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.842 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.842 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.842 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.136736_190.1", 2022-09-03T06:23:35.842 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.843 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24", 2022-09-03T06:23:35.843 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.136736736Z", 2022-09-03T06:23:35.843 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.844 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.844 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.844 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.844 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.845 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.845 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_flubJt4QosOvedZNv50fpC6", 2022-09-03T06:23:35.845 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.845 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.846 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.846 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.846 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.846 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.847 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.847 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.847 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.158636_191.1", 2022-09-03T06:23:35.847 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.848 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:35.848 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.158636011Z", 2022-09-03T06:23:35.848 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.848 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.849 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.849 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.849 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.849 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.850 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.850 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.850 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.850 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.851 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.851 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.851 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.851 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.852 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.852 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.162317_192.1", 2022-09-03T06:23:35.852 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.852 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:35.853 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.162317814Z", 2022-09-03T06:23:35.853 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.853 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.853 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.854 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.854 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.854 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.855 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.855 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.855 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.855 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.855 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.856 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.856 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.856 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.856 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.857 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.163511_193.1", 2022-09-03T06:23:35.857 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.857 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25", 2022-09-03T06:23:35.857 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.163511615Z", 2022-09-03T06:23:35.858 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.858 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.858 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.859 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.859 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.859 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.859 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tB02hvaj6C_E_OnwBMY8ywL", 2022-09-03T06:23:35.860 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.860 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.860 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.860 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.860 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.861 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.861 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.861 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.862 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.166349_194.1", 2022-09-03T06:23:35.862 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.862 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25", 2022-09-03T06:23:35.863 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.166349608Z", 2022-09-03T06:23:35.863 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.863 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.863 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.864 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.864 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.864 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.865 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tB02hvaj6C_E_OnwBMY8ywL", 2022-09-03T06:23:35.865 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.865 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.865 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.865 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.866 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.866 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.866 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.867 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.867 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.203237_195.1", 2022-09-03T06:23:35.876 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.877 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:35.877 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.203237512Z", 2022-09-03T06:23:35.877 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.878 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.878 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.878 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.878 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.879 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.879 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.879 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.879 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.880 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.880 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.880 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.881 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.881 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.881 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.881 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.206000_196.1", 2022-09-03T06:23:35.881 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.882 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:35.882 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.206000308Z", 2022-09-03T06:23:35.882 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.882 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.883 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.883 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.883 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.883 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.884 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.884 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.884 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.884 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.885 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.885 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.885 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.885 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.886 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.886 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.207006_197.1", 2022-09-03T06:23:35.886 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.887 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26", 2022-09-03T06:23:35.887 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.207006797Z", 2022-09-03T06:23:35.887 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.887 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.888 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.888 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.888 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.888 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.889 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_iK1lRBoYvVYLawtBAmt3Urm", 2022-09-03T06:23:35.889 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.889 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.889 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.890 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.890 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.890 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.890 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.891 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.891 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.211392_198.1", 2022-09-03T06:23:35.891 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.891 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26", 2022-09-03T06:23:35.892 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.211392135Z", 2022-09-03T06:23:35.892 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.892 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.892 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.893 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.893 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.893 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.893 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_iK1lRBoYvVYLawtBAmt3Urm", 2022-09-03T06:23:35.894 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.894 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.894 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.894 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.895 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.895 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.895 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.895 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.896 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.245835_199.1", 2022-09-03T06:23:35.896 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.896 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:35.896 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.245835749Z", 2022-09-03T06:23:35.897 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.897 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.897 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.897 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.898 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.898 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.898 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.899 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.899 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.899 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.899 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.900 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.900 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.900 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.900 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.901 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.250338_200.1", 2022-09-03T06:23:35.901 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.901 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:35.901 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.250338287Z", 2022-09-03T06:23:35.902 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.902 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.902 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.902 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.903 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.903 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.903 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.903 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.904 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.904 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.904 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.904 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.905 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.905 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.905 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.905 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.265702_201.1", 2022-09-03T06:23:35.906 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.906 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:35.906 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.265702416Z", 2022-09-03T06:23:35.907 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.907 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.907 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.907 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.908 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.908 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.908 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.908 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:35.909 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.909 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.909 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.909 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.910 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.910 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.910 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.910 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.270146_202.1", 2022-09-03T06:23:35.911 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.911 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:35.911 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.270146860Z", 2022-09-03T06:23:35.911 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.912 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.912 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.912 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.912 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.913 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.913 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:35.913 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:35.913 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.914 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.914 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.914 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.915 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.915 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.915 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.915 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.283449_203.1", 2022-09-03T06:23:35.916 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.916 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:35.916 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.283449642Z", 2022-09-03T06:23:35.916 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.917 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.917 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.917 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.917 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.918 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.918 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.918 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.919 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.919 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.919 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.919 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.920 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.920 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.920 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.920 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.286127_204.1", 2022-09-03T06:23:35.920 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.921 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:35.921 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.286127335Z", 2022-09-03T06:23:35.921 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.921 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.922 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.922 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.923 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.923 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.923 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.923 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.924 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.924 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.924 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.924 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.925 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.925 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.925 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.925 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.305370_205.1", 2022-09-03T06:23:35.926 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.926 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:35.926 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.305370975Z", 2022-09-03T06:23:35.926 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.927 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.927 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.927 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.927 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.928 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.928 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.928 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:35.928 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.929 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.929 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.929 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.929 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.930 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.930 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.930 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.312602_206.1", 2022-09-03T06:23:35.930 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.931 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:35.931 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.312602358Z", 2022-09-03T06:23:35.931 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.932 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.932 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.932 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.933 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.933 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.933 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:35.933 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:35.934 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.934 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.934 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.934 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.935 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.935 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.935 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.935 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.344134_207.1", 2022-09-03T06:23:35.936 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.936 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:35.936 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.344134550Z", 2022-09-03T06:23:35.936 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.937 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.937 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.937 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.938 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.939 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.939 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.940 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.940 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.941 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.941 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.941 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.941 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.942 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.355110_208.1", 2022-09-03T06:23:35.942 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.943 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:35.943 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.355110727Z", 2022-09-03T06:23:35.943 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.944 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.944 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.944 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.945 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.945 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.945 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.945 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.946 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.946 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.946 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.947 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.947 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.382118_209.1", 2022-09-03T06:23:35.947 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.948 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:35.948 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.382118897Z", 2022-09-03T06:23:35.948 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.948 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.949 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.949 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.949 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.949 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.950 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.950 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:35.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.950 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.951 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.951 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.951 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.952 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.386563_210.1", 2022-09-03T06:23:35.952 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.952 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:35.953 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.386563904Z", 2022-09-03T06:23:35.953 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.953 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.954 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.954 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:35.954 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.954 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.955 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:35.955 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:35.955 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.955 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.956 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.956 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.956 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.957 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.957 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.730358_216.1", 2022-09-03T06:23:35.957 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.957 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:35.958 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.730358281Z", 2022-09-03T06:23:35.958 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.958 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.959 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.960 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.960 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.960 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.960 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:35.961 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.962 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.962 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.962 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.962 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.963 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.963 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.964 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.735815_217.1", 2022-09-03T06:23:35.964 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.964 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:35.965 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.735815559Z", 2022-09-03T06:23:35.965 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.966 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.966 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.966 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.967 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.967 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.968 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:35.968 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.969 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.969 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.970 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.971 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.971 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.972 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.972 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.973 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.736742_218.1", 2022-09-03T06:23:35.973 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.973 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27", 2022-09-03T06:23:35.974 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.736742093Z", 2022-09-03T06:23:35.974 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.975 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.975 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.975 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.975 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.976 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.976 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_jZ9wkZPqzR2GWEIEfGTWCSz", 2022-09-03T06:23:35.976 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.977 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.977 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.978 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.978 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.979 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.979 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.979 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.742821_219.1", 2022-09-03T06:23:35.980 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.980 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27", 2022-09-03T06:23:35.981 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.742821039Z", 2022-09-03T06:23:35.981 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.982 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.982 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.982 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.982 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.983 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.983 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_jZ9wkZPqzR2GWEIEfGTWCSz", 2022-09-03T06:23:35.983 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.984 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.984 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.984 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.984 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.985 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.985 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.985 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.774385_220.1", 2022-09-03T06:23:35.985 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.985 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:35.986 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.774385987Z", 2022-09-03T06:23:35.986 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.986 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.987 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.987 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.987 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.987 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.987 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:35.988 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.988 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.988 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.988 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:35.989 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.989 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.989 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.989 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.990 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.778620_221.1", 2022-09-03T06:23:35.990 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:35.990 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:35.990 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.778620417Z", 2022-09-03T06:23:35.991 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.991 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.991 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:35.992 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:35.992 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.992 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:35.993 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:35.993 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:35.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.994 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:35.994 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:35.995 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.995 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:35.996 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:35.996 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:35.997 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.779591_222.1", 2022-09-03T06:23:35.997 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:35.998 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28", 2022-09-03T06:23:35.998 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.779591267Z", 2022-09-03T06:23:35.999 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:35.999 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:35.999 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.000 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.000 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.001 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.001 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ISaB31FGT1RJh5d1WEGeO7S", 2022-09-03T06:23:36.002 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.002 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.003 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.003 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.004 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.004 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.005 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.005 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.005 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.784101_223.1", 2022-09-03T06:23:36.006 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.006 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28", 2022-09-03T06:23:36.007 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.784101743Z", 2022-09-03T06:23:36.007 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.008 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.008 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.009 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.009 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.010 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ISaB31FGT1RJh5d1WEGeO7S", 2022-09-03T06:23:36.010 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.010 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.011 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.011 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.011 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.012 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.013 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.842565_224.1", 2022-09-03T06:23:36.013 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.013 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:36.013 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.842565677Z", 2022-09-03T06:23:36.014 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.014 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.014 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.014 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.015 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.015 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:36.015 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.016 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.016 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.016 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.017 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.017 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.017 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.017 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.848053_225.1", 2022-09-03T06:23:36.018 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.018 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:36.018 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.848053689Z", 2022-09-03T06:23:36.018 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.019 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.019 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.019 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.019 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.020 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.020 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:36.020 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.020 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.021 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.021 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.021 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.022 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.022 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.850209_226.1", 2022-09-03T06:23:36.022 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.022 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29", 2022-09-03T06:23:36.023 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209520Z", 2022-09-03T06:23:36.023 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.023 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.023 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.024 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.024 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.025 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_JaA8RlJaRxOPPcYHu0FpEzg", 2022-09-03T06:23:36.025 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.025 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.026 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.026 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.026 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.027 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.855964_227.1", 2022-09-03T06:23:36.027 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.027 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29", 2022-09-03T06:23:36.028 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.855964453Z", 2022-09-03T06:23:36.028 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.028 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.028 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.028 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.029 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.029 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.029 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_JaA8RlJaRxOPPcYHu0FpEzg", 2022-09-03T06:23:36.029 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.030 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.030 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.030 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.030 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.031 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.031 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.894939_228.1", 2022-09-03T06:23:36.032 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.032 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:36.032 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.894939418Z", 2022-09-03T06:23:36.032 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.033 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.033 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:36.033 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.034 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.034 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:36.034 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.035 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.035 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.035 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.036 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.036 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.036 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.897692_229.1", 2022-09-03T06:23:36.037 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.037 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:36.037 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.897692116Z", 2022-09-03T06:23:36.037 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.038 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.038 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:36.038 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.039 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.039 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:36.039 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.040 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.040 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.040 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.041 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.041 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.921086_230.1", 2022-09-03T06:23:36.042 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.042 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:36.042 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.921086404Z", 2022-09-03T06:23:36.043 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.043 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.043 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:36.043 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.044 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.044 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:36.044 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.045 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.045 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.045 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.046 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.046 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.923839_231.1", 2022-09-03T06:23:36.047 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.047 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:36.047 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.923839471Z", 2022-09-03T06:23:36.047 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.047 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.048 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:36.048 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.048 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.049 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:36.049 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.049 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.050 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.050 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.050 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.051 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.001910_232.1", 2022-09-03T06:23:36.051 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.051 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:36.052 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.001910722Z", 2022-09-03T06:23:36.052 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.052 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.052 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:36.053 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.053 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.053 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:36.054 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.054 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.054 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.055 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.055 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.055 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.055 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.056 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.011478_233.1", 2022-09-03T06:23:36.056 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.056 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:36.056 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.011478475Z", 2022-09-03T06:23:36.057 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.057 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.057 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:36.057 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.058 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.058 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:36.058 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.059 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.059 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.060 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.060 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.060 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.060 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.061 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.059572_234.1", 2022-09-03T06:23:36.061 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.061 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:36.061 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.059572375Z", 2022-09-03T06:23:36.062 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.062 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.062 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.063 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.063 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.063 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.064 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:36.064 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.064 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.065 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.065 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.065 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.066 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.066 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.063873_235.1", 2022-09-03T06:23:36.066 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.067 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:36.067 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.063873838Z", 2022-09-03T06:23:36.067 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.067 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.068 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.068 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.069 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.069 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:36.069 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.070 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.070 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.071 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.071 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.071 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.072 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.064807_236.1", 2022-09-03T06:23:36.072 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.072 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30", 2022-09-03T06:23:36.073 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.064807654Z", 2022-09-03T06:23:36.073 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.073 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.073 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.074 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.074 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.074 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_emDKTj1MXu1EbBl7W-BZDCI", 2022-09-03T06:23:36.075 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.075 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.075 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.076 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.076 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.076 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.077 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.077 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.077 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.069230_237.1", 2022-09-03T06:23:36.078 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.078 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30", 2022-09-03T06:23:36.078 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.069230157Z", 2022-09-03T06:23:36.079 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.079 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.079 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.080 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.080 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.080 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_emDKTj1MXu1EbBl7W-BZDCI", 2022-09-03T06:23:36.081 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.081 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.081 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.082 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.082 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.082 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.082 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.083 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.083 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.098699_238.1", 2022-09-03T06:23:36.083 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.083 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:36.084 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.098699181Z", 2022-09-03T06:23:36.084 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.084 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.085 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.085 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.085 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.085 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.086 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:36.086 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.086 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.086 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.087 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.087 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.088 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.088 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.088 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.089 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.104362_239.1", 2022-09-03T06:23:36.089 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.089 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:36.090 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.104362288Z", 2022-09-03T06:23:36.090 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.090 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.090 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.091 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.091 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.092 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:36.092 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.093 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.093 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.094 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.094 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.094 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.095 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.105238_240.1", 2022-09-03T06:23:36.095 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.095 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31", 2022-09-03T06:23:36.095 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.105238360Z", 2022-09-03T06:23:36.096 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.096 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.096 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.097 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.097 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.097 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Fx_iwLRI89zV-Ovfojlsmn_", 2022-09-03T06:23:36.098 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.098 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.098 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.099 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.099 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.100 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.109389_241.1", 2022-09-03T06:23:36.100 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.100 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31", 2022-09-03T06:23:36.101 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.109389408Z", 2022-09-03T06:23:36.101 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.101 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.101 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.102 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.102 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.103 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Fx_iwLRI89zV-Ovfojlsmn_", 2022-09-03T06:23:36.103 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.103 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.104 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.104 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.104 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.105 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.146357_242.1", 2022-09-03T06:23:36.105 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.105 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:36.106 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.146357921Z", 2022-09-03T06:23:36.106 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.106 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.107 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.107 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.107 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.108 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:36.108 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.108 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.109 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.109 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.109 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.109 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.110 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.110 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.149385_243.1", 2022-09-03T06:23:36.110 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.111 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:36.111 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.149385163Z", 2022-09-03T06:23:36.111 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.111 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.112 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.112 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.113 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.113 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:36.113 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.113 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.114 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.114 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.114 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.114 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.115 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.115 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.150696_244.1", 2022-09-03T06:23:36.115 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.116 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32", 2022-09-03T06:23:36.116 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.150696281Z", 2022-09-03T06:23:36.116 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.117 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.117 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.117 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.118 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.118 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_VJHe0Zrz6-lomZaJx40Mq8S", 2022-09-03T06:23:36.118 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.119 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.119 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.119 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.119 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.120 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.120 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.120 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.159044_245.1", 2022-09-03T06:23:36.121 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.121 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32", 2022-09-03T06:23:36.121 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.159044634Z", 2022-09-03T06:23:36.121 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.122 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.122 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.122 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.123 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.123 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_VJHe0Zrz6-lomZaJx40Mq8S", 2022-09-03T06:23:36.123 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.124 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.124 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.125 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.125 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.125 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.125 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.126 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.221962_246.1", 2022-09-03T06:23:36.126 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.126 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:36.126 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.221962483Z", 2022-09-03T06:23:36.127 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.127 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.127 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:36.128 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.128 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.128 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:36.129 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.129 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.129 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.130 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.130 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.131 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.225097_247.1", 2022-09-03T06:23:36.131 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.132 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:36.132 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.225097472Z", 2022-09-03T06:23:36.132 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.132 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.133 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:36.133 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.134 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.134 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:36.134 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.135 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.135 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.135 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.135 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.136 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.324010_248.1", 2022-09-03T06:23:36.137 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.137 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:36.137 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.324010891Z", 2022-09-03T06:23:36.137 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.138 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.138 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:36.138 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.139 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.139 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:36.139 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.140 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.140 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.140 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.141 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.141 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.326610_249.1", 2022-09-03T06:23:36.142 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.142 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:36.142 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.326610391Z", 2022-09-03T06:23:36.143 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.143 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.143 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:36.143 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.144 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.144 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:36.144 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.145 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.145 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.146 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.146 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.147 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.147 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.388152_250.1", 2022-09-03T06:23:36.147 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.147 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:36.148 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.388152023Z", 2022-09-03T06:23:36.148 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.148 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.148 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:36.149 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.149 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.149 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:36.150 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.150 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.150 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.151 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.151 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.151 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.152 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.390795_251.1", 2022-09-03T06:23:36.152 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.153 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:36.153 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.390795101Z", 2022-09-03T06:23:36.153 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.154 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.154 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:36.154 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.155 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.155 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:36.155 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.156 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.156 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.156 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.156 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.157 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.157 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.428775_252.1", 2022-09-03T06:23:36.157 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.158 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:36.158 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.428775853Z", 2022-09-03T06:23:36.158 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.158 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.159 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.159 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.160 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.160 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:36.161 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.161 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.161 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.162 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.162 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.163 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.163 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.432176_253.1", 2022-09-03T06:23:36.163 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.163 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:36.164 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.432176035Z", 2022-09-03T06:23:36.164 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.164 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.165 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.165 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.166 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.166 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:36.166 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.167 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.167 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.168 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.168 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.169 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.169 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.170 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.433149_254.1", 2022-09-03T06:23:36.170 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.171 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33", 2022-09-03T06:23:36.171 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.433149170Z", 2022-09-03T06:23:36.172 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.172 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.173 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.173 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.174 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.174 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_w0QD1jE_b482WNZp5Lj2ky4", 2022-09-03T06:23:36.175 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.175 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.176 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.176 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.177 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.177 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.177 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.178 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.178 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.435497_255.1", 2022-09-03T06:23:36.178 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.179 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33", 2022-09-03T06:23:36.179 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.435497913Z", 2022-09-03T06:23:36.179 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.179 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.180 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.180 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.180 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.181 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_w0QD1jE_b482WNZp5Lj2ky4", 2022-09-03T06:23:36.181 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.181 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.181 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.182 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.182 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.182 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.182 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.183 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.183 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.463906_256.1", 2022-09-03T06:23:36.183 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.183 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:36.184 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.463906896Z", 2022-09-03T06:23:36.184 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.184 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.184 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.185 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.185 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.185 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.185 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:36.186 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.186 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.186 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.186 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.187 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.187 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.187 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.187 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.188 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.466439_257.1", 2022-09-03T06:23:36.188 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.188 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:36.188 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.466439294Z", 2022-09-03T06:23:36.189 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.189 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.189 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.189 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.189 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.190 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.190 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:36.190 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.190 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.191 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.191 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.191 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.192 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.192 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.192 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.192 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.467396_258.1", 2022-09-03T06:23:36.193 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.193 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34", 2022-09-03T06:23:36.193 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.467396448Z", 2022-09-03T06:23:36.193 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.194 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.194 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.194 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.194 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.195 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.195 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G2EFJYOe8FOdjOCj6aBraUR", 2022-09-03T06:23:36.195 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.195 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.196 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.196 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.196 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.196 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.197 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.197 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.197 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.471551_259.1", 2022-09-03T06:23:36.197 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.198 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34", 2022-09-03T06:23:36.198 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.471551488Z", 2022-09-03T06:23:36.198 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.198 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.198 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.199 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.199 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.199 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.199 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G2EFJYOe8FOdjOCj6aBraUR", 2022-09-03T06:23:36.200 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.200 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.200 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.200 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.201 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.201 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.201 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.201 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.202 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.517947_260.1", 2022-09-03T06:23:36.202 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.202 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:36.202 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.517947959Z", 2022-09-03T06:23:36.203 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.203 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.203 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.203 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.204 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.204 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:36.204 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.205 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.205 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.205 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.205 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.206 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.206 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.206 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.522051_261.1", 2022-09-03T06:23:36.206 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.207 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:36.207 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.522051544Z", 2022-09-03T06:23:36.207 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.207 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.208 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.208 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.208 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.209 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:36.209 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.209 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.209 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.210 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.210 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.210 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.211 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.524288_262.1", 2022-09-03T06:23:36.211 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.211 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35", 2022-09-03T06:23:36.212 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.524288626Z", 2022-09-03T06:23:36.212 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.212 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.212 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.213 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.213 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.213 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.213 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G8oh7TOeWyD2_FvC8AIXccM", 2022-09-03T06:23:36.214 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.214 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.214 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.215 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.215 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.216 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.526709_263.1", 2022-09-03T06:23:36.216 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.216 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35", 2022-09-03T06:23:36.216 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.526709154Z", 2022-09-03T06:23:36.217 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.217 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.217 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.217 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.218 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.218 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G8oh7TOeWyD2_FvC8AIXccM", 2022-09-03T06:23:36.218 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.218 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.219 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.219 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.219 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.219 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.220 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.220 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.565852_264.1", 2022-09-03T06:23:36.220 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.221 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:36.221 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.565852141Z", 2022-09-03T06:23:36.221 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.221 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.222 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:36.222 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.222 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.223 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:36.223 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.223 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.223 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.568581_265.1", 2022-09-03T06:23:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:36.226 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.568581265Z", 2022-09-03T06:23:36.226 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.226 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.226 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:36.227 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.227 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.227 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.227 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:36.228 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.228 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.228 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.228 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.229 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.229 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.229 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.230 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185757.605699_266.1", 2022-09-03T06:23:36.230 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.230 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:36.230 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:57.605699761Z", 2022-09-03T06:23:36.231 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.231 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.231 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:36.231 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.232 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.232 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:36.232 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.233 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.233 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.233 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.233 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.234 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185757.608333_267.1", 2022-09-03T06:23:36.234 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.235 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:36.235 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:57.608333919Z", 2022-09-03T06:23:36.235 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.235 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.236 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:36.236 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.236 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.237 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:36.237 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.237 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.237 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.238 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.238 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.238 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.239 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.691847_268.1", 2022-09-03T06:23:36.239 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.239 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:36.239 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.691847123Z", 2022-09-03T06:23:36.240 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.240 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.240 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:36.240 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.241 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.241 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:36.241 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.242 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.242 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.242 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.242 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.243 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.243 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.696866_269.1", 2022-09-03T06:23:36.244 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.244 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:36.244 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.696866177Z", 2022-09-03T06:23:36.244 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.245 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.245 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:36.245 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.246 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.246 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:36.246 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:36.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.247 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.247 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.247 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.247 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.248 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.804943_270.1", 2022-09-03T06:23:36.248 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.248 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:23:36.249 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.804943552Z", 2022-09-03T06:23:36.249 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.249 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.249 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.250 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.250 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.250 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.250 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-F9Dl85tpDzK3gXJ-aezc_j", 2022-09-03T06:23:36.251 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.251 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.251 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.252 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.252 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.253 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.808467_271.1", 2022-09-03T06:23:36.253 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.253 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:23:36.253 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.808467134Z", 2022-09-03T06:23:36.254 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.254 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.254 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.254 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-F9Dl85tpDzK3gXJ-aezc_j", 2022-09-03T06:23:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.810847_272.1", 2022-09-03T06:23:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-36", 2022-09-03T06:23:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.810847136Z", 2022-09-03T06:23:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_5JS-wFAV9CrTmAifbKi9NjK", 2022-09-03T06:23:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.813462_273.1", 2022-09-03T06:23:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-36", 2022-09-03T06:23:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.813462243Z", 2022-09-03T06:23:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_5JS-wFAV9CrTmAifbKi9NjK", 2022-09-03T06:23:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.844464_274.1", 2022-09-03T06:23:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:23:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.844464322Z", 2022-09-03T06:23:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_brde0pgOsuZjrCySsOp9pmR", 2022-09-03T06:23:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.850329_275.1", 2022-09-03T06:23:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:23:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.850329433Z", 2022-09-03T06:23:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_brde0pgOsuZjrCySsOp9pmR", 2022-09-03T06:23:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.855256_276.1", 2022-09-03T06:23:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-37", 2022-09-03T06:23:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.855256692Z", 2022-09-03T06:23:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3HAVWKyp6rMdHTXkKukojjv", 2022-09-03T06:23:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.859644_277.1", 2022-09-03T06:23:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-37", 2022-09-03T06:23:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.859644317Z", 2022-09-03T06:23:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3HAVWKyp6rMdHTXkKukojjv", 2022-09-03T06:23:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.294 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.294 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.294 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.294 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.902177_278.1", 2022-09-03T06:23:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:23:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.902177035Z", 2022-09-03T06:23:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Iv3Ff3D6p1XbZ30mXZkhK63", 2022-09-03T06:23:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.907504_279.1", 2022-09-03T06:23:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:23:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.907504015Z", 2022-09-03T06:23:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Iv3Ff3D6p1XbZ30mXZkhK63", 2022-09-03T06:23:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.908555_280.1", 2022-09-03T06:23:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-38", 2022-09-03T06:23:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.908555860Z", 2022-09-03T06:23:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G9WP1PhzJZFD5vWOwiYS9Hb", 2022-09-03T06:23:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.308 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.308 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.308 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.308 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.309 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.915547_281.1", 2022-09-03T06:23:36.309 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.309 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-38", 2022-09-03T06:23:36.309 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.915547971Z", 2022-09-03T06:23:36.309 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.310 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.310 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.310 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.310 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.311 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.311 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G9WP1PhzJZFD5vWOwiYS9Hb", 2022-09-03T06:23:36.311 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.312 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.312 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.312 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.312 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.313 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.313 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.313 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.314 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.844866_288.1", 2022-09-03T06:23:36.314 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.314 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39", 2022-09-03T06:23:36.315 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.844866559Z", 2022-09-03T06:23:36.315 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.315 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.316 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.316 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.317 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.317 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.317 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lt1dYegSwDzpwX06X-fGkup", 2022-09-03T06:23:36.318 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.318 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.319 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.319 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.319 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.319 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.320 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.320 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.320 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.850612_289.1", 2022-09-03T06:23:36.321 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.321 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39", 2022-09-03T06:23:36.321 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.850612019Z", 2022-09-03T06:23:36.321 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.321 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.322 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.322 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.322 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.322 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.323 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lt1dYegSwDzpwX06X-fGkup", 2022-09-03T06:23:36.323 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.323 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.323 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.324 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.324 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.324 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.324 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.325 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.325 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.852628_290.1", 2022-09-03T06:23:36.325 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.325 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-39", 2022-09-03T06:23:36.326 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.852628518Z", 2022-09-03T06:23:36.326 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.326 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.326 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.327 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.327 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.327 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.327 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_O0mztKsloMen3wFjqSH6i9D", 2022-09-03T06:23:36.328 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.328 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.328 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.328 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.328 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.329 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.329 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.329 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.330 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.857180_291.1", 2022-09-03T06:23:36.330 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.330 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-39", 2022-09-03T06:23:36.330 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.857180178Z", 2022-09-03T06:23:36.331 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.331 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.331 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.332 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.332 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.332 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.332 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_O0mztKsloMen3wFjqSH6i9D", 2022-09-03T06:23:36.332 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.333 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.333 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.333 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.333 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.334 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.334 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.334 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.335 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.883799_292.1", 2022-09-03T06:23:36.335 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.335 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40", 2022-09-03T06:23:36.335 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.883799148Z", 2022-09-03T06:23:36.336 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.336 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.336 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.337 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.337 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.337 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.338 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_fswsShBMVZzHxqVULj3KzUU", 2022-09-03T06:23:36.338 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.338 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.338 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.339 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.339 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.339 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.339 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.340 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.340 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.889075_293.1", 2022-09-03T06:23:36.340 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.340 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40", 2022-09-03T06:23:36.341 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.889075582Z", 2022-09-03T06:23:36.341 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.341 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.341 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.342 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.342 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.342 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.342 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_fswsShBMVZzHxqVULj3KzUU", 2022-09-03T06:23:36.343 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.343 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.343 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.343 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.344 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.344 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.344 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.344 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.344 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.890194_294.1", 2022-09-03T06:23:36.345 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.345 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-40", 2022-09-03T06:23:36.345 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.890194639Z", 2022-09-03T06:23:36.346 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.346 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.346 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.347 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.347 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.347 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.347 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_A4T-tDc1XWVEtWeND6N-Hqq", 2022-09-03T06:23:36.348 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.348 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.348 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.348 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.349 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.349 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.349 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.349 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.350 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.896341_295.1", 2022-09-03T06:23:36.350 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.350 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-40", 2022-09-03T06:23:36.351 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.896341961Z", 2022-09-03T06:23:36.351 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.351 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.352 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.352 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.352 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.352 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.353 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_A4T-tDc1XWVEtWeND6N-Hqq", 2022-09-03T06:23:36.353 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.353 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.353 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.353 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.354 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.354 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.355 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.355 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.355 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.947010_296.1", 2022-09-03T06:23:36.356 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.356 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41", 2022-09-03T06:23:36.356 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.947010594Z", 2022-09-03T06:23:36.356 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.357 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.357 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.357 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.357 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.358 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.358 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yitoOgqNEqy39xdDDb8hNi-", 2022-09-03T06:23:36.358 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.358 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.359 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.359 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.359 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.359 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.360 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.360 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.360 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.950610_297.1", 2022-09-03T06:23:36.360 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.361 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41", 2022-09-03T06:23:36.361 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.950610217Z", 2022-09-03T06:23:36.361 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.361 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.362 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.362 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.362 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.363 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.363 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yitoOgqNEqy39xdDDb8hNi-", 2022-09-03T06:23:36.363 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.364 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.364 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.364 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.364 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.365 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.365 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.365 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.366 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.951802_298.1", 2022-09-03T06:23:36.366 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.366 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-41", 2022-09-03T06:23:36.366 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.951802745Z", 2022-09-03T06:23:36.367 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.367 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.367 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.368 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.368 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.368 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.368 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_rtPxlDQFqSiMUkKzGoM6IJb", 2022-09-03T06:23:36.369 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.369 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.369 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.369 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:36.370 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.370 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.370 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.371 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:36.371 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.955393_299.1", 2022-09-03T06:23:36.371 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:36.372 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-41", 2022-09-03T06:23:36.372 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.955393441Z", 2022-09-03T06:23:36.372 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:36.372 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:36.373 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:36.373 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:36.373 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.373 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:36.374 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_rtPxlDQFqSiMUkKzGoM6IJb", 2022-09-03T06:23:36.374 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:36.374 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:36.374 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:36.375 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:36.375 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.375 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.375 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:36.376 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:23:36.377 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:36.378 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:36.379 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period b84ccf39-78f5-4998-9db6-42362bc8c98b --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:36.441 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:36.442 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:36.484 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:36.485 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.485 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184222.657692_58.1", 2022-09-03T06:23:36.485 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "user", 2022-09-03T06:23:36.486 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "rgw-multisite-test-user", 2022-09-03T06:23:36.486 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:50:22.657692624Z", 2022-09-03T06:23:36.487 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.487 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.488 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.488 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.489 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.489 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.490 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6BN_OEAtimzMqhhbhGc7xsF", 2022-09-03T06:23:36.490 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.491 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.491 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.491 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.492 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.492 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.493 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.493 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.494 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184222.662031_59.1", 2022-09-03T06:23:36.494 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "user", 2022-09-03T06:23:36.495 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "rgw-multisite-test-user", 2022-09-03T06:23:36.495 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:50:22.662031116Z", 2022-09-03T06:23:36.496 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.496 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.497 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.497 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.498 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.498 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.499 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6BN_OEAtimzMqhhbhGc7xsF", 2022-09-03T06:23:36.499 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.500 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.500 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.501 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.501 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.502 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.502 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.503 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.503 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.219295_62.1", 2022-09-03T06:23:36.504 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.504 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1", 2022-09-03T06:23:36.505 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.219295443Z", 2022-09-03T06:23:36.505 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.506 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.506 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.507 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.507 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.508 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.508 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Y7JDAo_bsL14HyXDq4vsLc2", 2022-09-03T06:23:36.509 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.509 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.509 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.510 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.511 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.511 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.511 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.512 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.512 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.222658_63.1", 2022-09-03T06:23:36.513 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.513 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1", 2022-09-03T06:23:36.514 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.222658346Z", 2022-09-03T06:23:36.514 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.515 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.515 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.516 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.516 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.517 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.517 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Y7JDAo_bsL14HyXDq4vsLc2", 2022-09-03T06:23:36.518 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.518 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.519 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.519 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.519 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.520 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.520 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.520 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.521 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.223873_64.1", 2022-09-03T06:23:36.521 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.521 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-1", 2022-09-03T06:23:36.521 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.223873908Z", 2022-09-03T06:23:36.522 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.522 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.522 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.523 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.523 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.523 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.523 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_VlD_M0myhVj3j7F4hjNPDp1", 2022-09-03T06:23:36.524 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.524 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.524 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.524 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.525 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.525 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.525 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.526 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.526 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.226841_65.1", 2022-09-03T06:23:36.526 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.527 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-1", 2022-09-03T06:23:36.527 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.226841368Z", 2022-09-03T06:23:36.527 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.527 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.528 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.528 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.528 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.528 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.529 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_VlD_M0myhVj3j7F4hjNPDp1", 2022-09-03T06:23:36.529 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.529 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.529 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.530 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.530 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.530 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.531 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.531 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.531 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.252557_66.1", 2022-09-03T06:23:36.532 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.532 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2", 2022-09-03T06:23:36.532 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.252557878Z", 2022-09-03T06:23:36.532 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.533 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.533 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.533 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.533 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.534 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.534 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_eGqFRgXckOShhJ5K31PxBCB", 2022-09-03T06:23:36.534 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.535 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.535 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.535 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.535 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.536 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.536 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.536 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.536 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.258211_67.1", 2022-09-03T06:23:36.537 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.537 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2", 2022-09-03T06:23:36.537 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.258211354Z", 2022-09-03T06:23:36.538 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.538 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.538 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.538 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.539 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.539 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.539 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_eGqFRgXckOShhJ5K31PxBCB", 2022-09-03T06:23:36.539 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.540 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.540 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.541 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.541 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.542 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.542 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.543 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.543 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.262819_68.1", 2022-09-03T06:23:36.544 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.544 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-2", 2022-09-03T06:23:36.545 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.262819181Z", 2022-09-03T06:23:36.545 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.546 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.547 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.547 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.548 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.548 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.549 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lkn_J0TsdIPDWPcsSr8e5eP", 2022-09-03T06:23:36.549 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.549 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.549 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.550 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.550 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.550 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.551 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.551 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.551 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184339.267378_69.1", 2022-09-03T06:23:36.551 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.552 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-2", 2022-09-03T06:23:36.552 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:19.267378646Z", 2022-09-03T06:23:36.552 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.553 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.553 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.554 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.554 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.555 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.555 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lkn_J0TsdIPDWPcsSr8e5eP", 2022-09-03T06:23:36.556 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.556 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.557 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.557 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.559 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.559 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.559 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.560 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.560 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184342.345771_70.1", 2022-09-03T06:23:36.560 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.561 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3", 2022-09-03T06:23:36.561 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:22.345771278Z", 2022-09-03T06:23:36.561 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.561 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.562 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.562 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.562 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.563 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.563 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_eB0pPK7VCkudq9Jt1nUnfVO", 2022-09-03T06:23:36.564 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.564 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.564 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.564 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.565 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.565 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.565 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.566 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.566 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184342.350157_71.1", 2022-09-03T06:23:36.566 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.566 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3", 2022-09-03T06:23:36.567 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:22.350157638Z", 2022-09-03T06:23:36.567 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.568 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.568 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.568 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.569 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.569 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.569 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_eB0pPK7VCkudq9Jt1nUnfVO", 2022-09-03T06:23:36.569 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.570 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.570 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.570 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.571 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.571 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.571 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.572 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.572 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184342.351175_72.1", 2022-09-03T06:23:36.572 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.573 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-3", 2022-09-03T06:23:36.573 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:22.351175819Z", 2022-09-03T06:23:36.573 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.574 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.574 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.574 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.575 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.575 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.575 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_xQbDQL6LoF7KY81FCBsr-TR", 2022-09-03T06:23:36.575 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.576 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.576 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.576 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.576 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.577 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.577 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.577 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.577 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184342.357275_73.1", 2022-09-03T06:23:36.578 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.578 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-3", 2022-09-03T06:23:36.578 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:22.357275634Z", 2022-09-03T06:23:36.579 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.579 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.579 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.579 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.580 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.580 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.580 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_xQbDQL6LoF7KY81FCBsr-TR", 2022-09-03T06:23:36.581 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.581 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.581 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.581 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.582 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.582 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.583 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.583 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.583 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.193842_74.1", 2022-09-03T06:23:36.584 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.584 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4", 2022-09-03T06:23:36.584 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.193842881Z", 2022-09-03T06:23:36.584 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.585 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.585 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.585 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.586 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.586 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.586 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lw10L7BpVQy7VSU2U7UHl07", 2022-09-03T06:23:36.587 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.587 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.587 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.588 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.588 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.588 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.588 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.589 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.589 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.198704_75.1", 2022-09-03T06:23:36.589 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.589 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4", 2022-09-03T06:23:36.590 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.198704862Z", 2022-09-03T06:23:36.591 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.591 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.592 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.592 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.593 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.593 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.594 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lw10L7BpVQy7VSU2U7UHl07", 2022-09-03T06:23:36.594 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.595 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.595 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.596 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.596 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.597 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.597 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.598 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.598 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.200713_76.1", 2022-09-03T06:23:36.598 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.599 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-4", 2022-09-03T06:23:36.599 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.200713524Z", 2022-09-03T06:23:36.600 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.600 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.600 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.600 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.601 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.601 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.601 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2mGIDNrcwT5FJidyB-XgESn", 2022-09-03T06:23:36.602 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.602 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.602 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.602 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.603 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.603 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.603 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.604 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.604 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.204734_77.1", 2022-09-03T06:23:36.604 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.604 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-4", 2022-09-03T06:23:36.605 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.204734948Z", 2022-09-03T06:23:36.605 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.605 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.605 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.606 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.607 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.607 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.607 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2mGIDNrcwT5FJidyB-XgESn", 2022-09-03T06:23:36.608 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.608 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.608 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.608 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.609 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.609 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.609 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.610 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.610 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.224447_78.1", 2022-09-03T06:23:36.610 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.611 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5", 2022-09-03T06:23:36.611 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.224447947Z", 2022-09-03T06:23:36.611 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.611 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.612 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.612 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.612 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.612 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.613 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_xo2PU3_oh6eFpkPNkexfq7D", 2022-09-03T06:23:36.613 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.614 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.614 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.614 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.615 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.615 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.615 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.616 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.616 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.227473_79.1", 2022-09-03T06:23:36.616 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.617 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5", 2022-09-03T06:23:36.617 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.227473639Z", 2022-09-03T06:23:36.617 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.617 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.618 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.618 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.618 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.618 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.619 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_xo2PU3_oh6eFpkPNkexfq7D", 2022-09-03T06:23:36.619 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.619 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.619 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.620 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.620 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.620 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.621 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.621 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.621 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.228594_80.1", 2022-09-03T06:23:36.621 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.622 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-5", 2022-09-03T06:23:36.622 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.228594676Z", 2022-09-03T06:23:36.622 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.622 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.623 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.623 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.623 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.624 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.624 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_u7x8KJF8hnPDPr0uRDbixnY", 2022-09-03T06:23:36.624 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.624 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.625 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.625 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.625 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.625 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.626 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.626 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.626 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.231192_81.1", 2022-09-03T06:23:36.626 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.627 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-5", 2022-09-03T06:23:36.627 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.231192184Z", 2022-09-03T06:23:36.627 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.628 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.628 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.628 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.628 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.629 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.629 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_u7x8KJF8hnPDPr0uRDbixnY", 2022-09-03T06:23:36.629 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.629 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.630 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.630 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.630 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.630 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.631 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.631 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.631 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.255262_82.1", 2022-09-03T06:23:36.632 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.632 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6", 2022-09-03T06:23:36.632 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.255262164Z", 2022-09-03T06:23:36.632 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.633 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.633 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.633 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.633 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.634 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.634 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_7T2FDkJXu_-0lpyQJ4EERx0", 2022-09-03T06:23:36.634 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.635 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.635 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.635 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.635 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.636 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.636 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.636 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.636 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.258031_83.1", 2022-09-03T06:23:36.637 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.637 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6", 2022-09-03T06:23:36.637 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.258031985Z", 2022-09-03T06:23:36.637 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.638 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.638 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.638 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.639 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.639 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.639 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_7T2FDkJXu_-0lpyQJ4EERx0", 2022-09-03T06:23:36.639 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.640 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.640 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.640 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.640 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.641 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.641 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.641 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.641 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.259057_84.1", 2022-09-03T06:23:36.642 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.642 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-6", 2022-09-03T06:23:36.642 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.259057684Z", 2022-09-03T06:23:36.643 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.643 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.643 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.643 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.644 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.644 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.644 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_JcUmTGR0SXlRqG5W9Zl5D8p", 2022-09-03T06:23:36.644 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.645 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.645 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.645 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.645 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.646 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.646 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.646 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.647 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184348.261653_85.1", 2022-09-03T06:23:36.647 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.647 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-6", 2022-09-03T06:23:36.648 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:28.261653432Z", 2022-09-03T06:23:36.648 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.648 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.648 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.649 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.649 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.649 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.650 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_JcUmTGR0SXlRqG5W9Zl5D8p", 2022-09-03T06:23:36.650 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.650 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.650 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.651 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.651 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.651 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.651 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.652 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.652 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.855330_86.1", 2022-09-03T06:23:36.652 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.653 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:36.653 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.855330912Z", 2022-09-03T06:23:36.653 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.653 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.654 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.654 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.654 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.654 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.655 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:36.655 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.655 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.655 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.656 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.656 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.656 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.657 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.657 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.657 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.861502_87.1", 2022-09-03T06:23:36.657 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.658 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:36.658 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.861502176Z", 2022-09-03T06:23:36.658 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.658 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.659 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.659 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.659 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.660 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.660 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:36.660 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.660 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.661 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.661 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.661 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.661 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.662 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.662 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.662 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.862483_88.1", 2022-09-03T06:23:36.663 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.663 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7", 2022-09-03T06:23:36.663 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.862483283Z", 2022-09-03T06:23:36.663 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.664 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.664 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.664 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.664 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.665 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.665 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:36.665 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.665 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.666 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.666 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.666 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.667 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.667 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.667 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.667 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.867021_89.1", 2022-09-03T06:23:36.668 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.668 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7", 2022-09-03T06:23:36.668 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.867021552Z", 2022-09-03T06:23:36.669 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.669 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.669 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.669 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.670 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.670 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.670 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:36.670 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.671 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.671 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.671 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.672 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.672 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.672 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.672 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.673 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.888960_90.1", 2022-09-03T06:23:36.673 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.673 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:36.673 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.888960687Z", 2022-09-03T06:23:36.674 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.674 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.674 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.675 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.675 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.675 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.675 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:36.676 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.676 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.676 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.676 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.677 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.677 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.677 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.677 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.678 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.892373_91.1", 2022-09-03T06:23:36.678 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.678 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:36.678 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.892373633Z", 2022-09-03T06:23:36.679 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.679 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.679 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.680 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.680 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.680 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.680 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:36.681 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.681 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.681 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.681 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.682 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.682 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.682 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.683 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.683 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.893288_92.1", 2022-09-03T06:23:36.683 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.683 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8", 2022-09-03T06:23:36.684 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.893288080Z", 2022-09-03T06:23:36.684 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.684 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.684 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.685 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.685 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.685 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.685 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:36.686 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.686 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.686 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.687 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.687 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.687 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.687 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.688 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.688 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.895751_93.1", 2022-09-03T06:23:36.688 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.688 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8", 2022-09-03T06:23:36.689 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.895751491Z", 2022-09-03T06:23:36.689 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.689 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.689 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.690 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.690 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.690 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.691 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:36.691 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.691 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.691 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.692 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.692 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.692 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.692 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.693 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.693 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.922520_94.1", 2022-09-03T06:23:36.693 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.694 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:36.694 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.922520779Z", 2022-09-03T06:23:36.694 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.694 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.695 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.695 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.695 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.695 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.696 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:36.696 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.696 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.696 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.697 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.697 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.697 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.698 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.698 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.698 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.931081_95.1", 2022-09-03T06:23:36.698 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.699 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:36.699 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.931081374Z", 2022-09-03T06:23:36.699 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.699 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.700 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.700 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.700 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.700 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.701 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:36.701 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.701 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.702 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.702 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.702 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.702 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.703 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.703 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.703 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.933197_96.1", 2022-09-03T06:23:36.704 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.704 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9", 2022-09-03T06:23:36.704 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.933197726Z", 2022-09-03T06:23:36.705 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.705 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.705 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.706 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.706 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.706 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.707 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:36.707 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.707 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.707 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.708 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.708 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.708 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.709 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.709 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.709 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184349.938376_97.1", 2022-09-03T06:23:36.710 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.710 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9", 2022-09-03T06:23:36.711 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:29.938376383Z", 2022-09-03T06:23:36.711 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.711 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.712 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.712 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.712 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.713 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.713 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:36.713 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.714 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.714 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.714 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.715 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.715 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.715 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.716 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.716 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.570224_98.1", 2022-09-03T06:23:36.717 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.717 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:36.717 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.570224266Z", 2022-09-03T06:23:36.718 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.718 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.719 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:36.719 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.720 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.720 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.720 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:36.721 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.721 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.722 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.722 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.722 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.723 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.723 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.724 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.724 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.574509_99.1", 2022-09-03T06:23:36.725 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.725 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:36.725 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.574509413Z", 2022-09-03T06:23:36.726 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.726 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.726 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:36.727 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.727 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.727 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.728 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:36.728 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.728 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.728 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.729 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.729 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.729 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.729 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.730 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.730 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.584013_100.1", 2022-09-03T06:23:36.730 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.731 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7", 2022-09-03T06:23:36.731 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.584013358Z", 2022-09-03T06:23:36.731 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.731 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.732 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:36.732 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.732 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.732 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.733 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:36.733 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.733 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.733 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.734 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "remove" 2022-09-03T06:23:36.734 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.734 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.735 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.735 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.735 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.587599_101.1", 2022-09-03T06:23:36.735 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.736 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-7", 2022-09-03T06:23:36.736 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.587599896Z", 2022-09-03T06:23:36.736 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.736 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.737 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:36.737 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.737 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.737 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.738 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:36.738 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.738 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.739 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.739 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.739 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.739 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.740 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.740 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.740 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.619234_102.1", 2022-09-03T06:23:36.740 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.741 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:36.741 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.619234589Z", 2022-09-03T06:23:36.741 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.741 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.742 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:36.742 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.742 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.743 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.743 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:36.743 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.743 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.744 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.744 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.744 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.744 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.745 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.745 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.745 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.625715_103.1", 2022-09-03T06:23:36.745 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.746 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:36.746 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.625715365Z", 2022-09-03T06:23:36.746 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.746 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.747 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:36.747 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.747 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.748 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.748 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:36.748 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.748 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.749 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.749 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.749 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.749 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.750 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.750 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.750 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.637706_104.1", 2022-09-03T06:23:36.750 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.751 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8", 2022-09-03T06:23:36.751 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.637706717Z", 2022-09-03T06:23:36.751 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.751 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.752 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:36.752 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.752 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.752 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.753 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:36.753 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.753 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.754 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.754 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "remove" 2022-09-03T06:23:36.754 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.754 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.755 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.755 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.755 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.639994_105.1", 2022-09-03T06:23:36.755 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.756 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-8", 2022-09-03T06:23:36.756 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.639994804Z", 2022-09-03T06:23:36.756 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.756 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.757 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:36.757 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.757 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.758 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.758 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:36.758 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.758 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.759 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.759 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.759 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.759 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.760 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.760 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.760 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.697911_106.1", 2022-09-03T06:23:36.760 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.761 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:36.761 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.697911177Z", 2022-09-03T06:23:36.761 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.762 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.762 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:36.762 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.762 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.763 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.763 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:36.763 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.763 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.764 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.764 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.764 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.764 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.765 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.765 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.765 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.700455_107.1", 2022-09-03T06:23:36.765 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.766 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:36.769 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.700455333Z", 2022-09-03T06:23:36.769 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.769 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.769 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:36.770 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.770 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.770 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.771 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:36.771 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.771 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.771 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.772 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.772 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.772 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.772 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.773 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.773 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.712013_108.1", 2022-09-03T06:23:36.773 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.773 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9", 2022-09-03T06:23:36.774 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.712013759Z", 2022-09-03T06:23:36.774 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.774 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.774 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:36.775 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.775 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.775 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.776 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:36.776 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.776 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.776 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.777 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "remove" 2022-09-03T06:23:36.777 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.777 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.777 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.778 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.778 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184350.715807_109.1", 2022-09-03T06:23:36.778 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.778 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-9", 2022-09-03T06:23:36.779 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807393Z", 2022-09-03T06:23:36.779 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.779 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.779 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:36.780 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.780 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.780 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.780 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:36.781 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.781 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.781 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.782 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.782 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.782 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.782 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.783 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.783 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.561415_64.1", 2022-09-03T06:23:36.783 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.783 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:23:36.784 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.561415905Z", 2022-09-03T06:23:36.784 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.784 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.785 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.785 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.785 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.785 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.786 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6ugH-NS-_bEeQKUdolNq7Oa", 2022-09-03T06:23:36.786 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.786 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.786 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.787 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.787 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.787 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.787 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.788 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.788 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.566930_66.1", 2022-09-03T06:23:36.788 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.788 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:23:36.789 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.566930651Z", 2022-09-03T06:23:36.789 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.789 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.790 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.790 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.790 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.790 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.791 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6ugH-NS-_bEeQKUdolNq7Oa", 2022-09-03T06:23:36.791 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.791 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.791 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.792 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.792 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.792 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.792 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.793 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.793 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.569237_67.1", 2022-09-03T06:23:36.793 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.794 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-10", 2022-09-03T06:23:36.794 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.569237568Z", 2022-09-03T06:23:36.794 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.794 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.795 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.795 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.795 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.795 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.796 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_htY0ij10LAX0ZNP6T_0mHNL", 2022-09-03T06:23:36.796 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.796 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.796 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.797 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.797 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.797 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.797 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.798 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.798 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.574987_68.1", 2022-09-03T06:23:36.798 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.799 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-10", 2022-09-03T06:23:36.799 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.574987235Z", 2022-09-03T06:23:36.799 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.799 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.800 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.800 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.800 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.800 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.801 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_htY0ij10LAX0ZNP6T_0mHNL", 2022-09-03T06:23:36.801 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.801 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.802 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.802 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.802 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.802 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.803 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.803 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.803 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.595637_69.1", 2022-09-03T06:23:36.803 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.804 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:23:36.804 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.595637832Z", 2022-09-03T06:23:36.804 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.804 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.805 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.805 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.805 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.805 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.806 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gtTH1QqAjuxzvv9DiFtpT7B", 2022-09-03T06:23:36.806 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.806 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.806 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.807 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.807 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.807 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.807 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.808 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.808 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.599901_70.1", 2022-09-03T06:23:36.808 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.809 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:23:36.809 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.599901361Z", 2022-09-03T06:23:36.809 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.810 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.810 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.810 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.810 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.811 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.811 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gtTH1QqAjuxzvv9DiFtpT7B", 2022-09-03T06:23:36.811 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.811 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.812 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.812 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.812 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.812 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.813 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.813 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.813 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.601762_71.1", 2022-09-03T06:23:36.813 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.814 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-11", 2022-09-03T06:23:36.814 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.601762485Z", 2022-09-03T06:23:36.814 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.814 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.815 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.815 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.815 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.815 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.816 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_umwMbGorUx0G8JCmxgLXcXQ", 2022-09-03T06:23:36.816 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.816 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.816 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.817 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.817 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.817 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.817 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.818 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.818 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.605809_72.1", 2022-09-03T06:23:36.818 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.818 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-11", 2022-09-03T06:23:36.819 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.605809267Z", 2022-09-03T06:23:36.819 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.819 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.819 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.820 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.820 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.820 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.820 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_umwMbGorUx0G8JCmxgLXcXQ", 2022-09-03T06:23:36.821 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.821 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.821 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.822 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.822 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.823 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.823 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.823 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.823 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.642680_73.1", 2022-09-03T06:23:36.824 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.824 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:23:36.824 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.642680630Z", 2022-09-03T06:23:36.824 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.825 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.825 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.825 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.825 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.826 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.826 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_pT_Dw9zXIA2-9UcP5Gn1GlK", 2022-09-03T06:23:36.826 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.827 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.827 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.827 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.827 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.828 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.828 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.828 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.828 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.645295_74.1", 2022-09-03T06:23:36.829 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.829 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:23:36.829 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.645295120Z", 2022-09-03T06:23:36.829 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.830 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.830 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.830 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.830 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.831 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.831 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_pT_Dw9zXIA2-9UcP5Gn1GlK", 2022-09-03T06:23:36.831 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.831 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.832 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.832 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.832 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.832 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.832 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.833 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.833 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.646235_75.1", 2022-09-03T06:23:36.833 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.834 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-12", 2022-09-03T06:23:36.834 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.646235577Z", 2022-09-03T06:23:36.834 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.835 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.835 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.835 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.836 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.836 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.836 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ig99b4m7-ssuB2Tw7htsmQp", 2022-09-03T06:23:36.837 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.837 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.838 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.838 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.839 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.839 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.840 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.840 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.841 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184356.648716_76.1", 2022-09-03T06:23:36.841 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.842 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-12", 2022-09-03T06:23:36.842 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:36.648716080Z", 2022-09-03T06:23:36.843 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.843 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.844 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.844 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.844 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.845 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.845 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ig99b4m7-ssuB2Tw7htsmQp", 2022-09-03T06:23:36.845 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.845 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.846 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.846 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.846 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.846 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.847 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.847 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.847 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.356235_99.1", 2022-09-03T06:23:36.847 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.847 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:23:36.848 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.356235827Z", 2022-09-03T06:23:36.848 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.848 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.849 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.849 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.849 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.849 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.850 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8iK3rbCB7fbjpNaGZ4Lk2f9", 2022-09-03T06:23:36.850 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.850 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.851 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.851 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.851 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.851 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.852 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.852 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.852 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.364532_100.1", 2022-09-03T06:23:36.852 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.853 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:23:36.853 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.364532891Z", 2022-09-03T06:23:36.853 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.853 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.854 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.854 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.854 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.854 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.855 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8iK3rbCB7fbjpNaGZ4Lk2f9", 2022-09-03T06:23:36.855 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.855 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.856 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.856 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.856 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.856 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.857 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.857 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.857 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.365724_101.1", 2022-09-03T06:23:36.858 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.858 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-13", 2022-09-03T06:23:36.858 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.365724056Z", 2022-09-03T06:23:36.858 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.859 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.859 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.859 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.859 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.860 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.860 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_hGY9xi1--Qa5DBRaR1Ev2Td", 2022-09-03T06:23:36.861 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.861 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.861 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.861 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.862 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.862 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.863 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.863 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.864 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.372032_102.1", 2022-09-03T06:23:36.864 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.865 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-13", 2022-09-03T06:23:36.865 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.372032718Z", 2022-09-03T06:23:36.866 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.866 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.867 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.867 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.867 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.867 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.868 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_hGY9xi1--Qa5DBRaR1Ev2Td", 2022-09-03T06:23:36.868 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.868 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.868 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.869 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.869 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.869 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.869 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.870 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.870 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.400096_103.1", 2022-09-03T06:23:36.870 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.870 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:23:36.871 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.400096996Z", 2022-09-03T06:23:36.871 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.871 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.871 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.872 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.872 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.872 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.872 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_CLeQ9451sE39DW-o5-rQ9Gb", 2022-09-03T06:23:36.873 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.873 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.873 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.873 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.874 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.874 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.874 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.875 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.875 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.406131_104.1", 2022-09-03T06:23:36.875 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.875 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:23:36.875 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.406131833Z", 2022-09-03T06:23:36.876 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.876 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.876 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.876 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.877 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.877 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.877 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_CLeQ9451sE39DW-o5-rQ9Gb", 2022-09-03T06:23:36.877 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.878 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.878 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.878 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.878 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.879 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.879 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.879 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.879 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.407235_105.1", 2022-09-03T06:23:36.880 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.880 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-14", 2022-09-03T06:23:36.880 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.407235283Z", 2022-09-03T06:23:36.880 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.881 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.881 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.881 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.881 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.882 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.882 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_E3QteIhTh8XKCT60Fs2visZ", 2022-09-03T06:23:36.883 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.883 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.883 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.883 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.884 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.884 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.884 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.884 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.885 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.413053_106.1", 2022-09-03T06:23:36.885 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.885 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-14", 2022-09-03T06:23:36.885 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.413053852Z", 2022-09-03T06:23:36.886 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.886 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.886 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.886 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.887 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.887 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.887 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_E3QteIhTh8XKCT60Fs2visZ", 2022-09-03T06:23:36.887 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.888 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.888 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.888 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.888 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.889 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.889 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.889 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.889 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.447727_107.1", 2022-09-03T06:23:36.889 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.890 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:23:36.890 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.447727339Z", 2022-09-03T06:23:36.890 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.890 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.891 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.891 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.891 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.891 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.892 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6mYNO_dnwhvNSPaHRDsVWkL", 2022-09-03T06:23:36.892 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.892 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.892 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.893 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.893 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.893 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.894 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.894 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.894 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.450607_108.1", 2022-09-03T06:23:36.894 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.895 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:23:36.895 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.450607653Z", 2022-09-03T06:23:36.895 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.896 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.896 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.896 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.896 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.897 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.897 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_6mYNO_dnwhvNSPaHRDsVWkL", 2022-09-03T06:23:36.897 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.897 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.898 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.898 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.898 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.898 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.899 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.899 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.899 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.451654_109.1", 2022-09-03T06:23:36.899 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.900 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-15", 2022-09-03T06:23:36.900 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.451654129Z", 2022-09-03T06:23:36.900 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.901 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.901 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.901 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.901 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.902 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.902 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_iMvYVCIRH1guRCrQSTbYt62", 2022-09-03T06:23:36.902 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.902 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.903 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.903 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.903 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.903 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.904 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.904 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.904 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184450.454511_110.1", 2022-09-03T06:23:36.904 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.904 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-15", 2022-09-03T06:23:36.905 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:54:10.454511694Z", 2022-09-03T06:23:36.905 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.905 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.905 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.906 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.906 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.906 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.907 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_iMvYVCIRH1guRCrQSTbYt62", 2022-09-03T06:23:36.907 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.907 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.907 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.907 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.908 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.908 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.908 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.908 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.909 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184551.125768_119.1", 2022-09-03T06:23:36.909 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.910 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:36.910 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:51.125768074Z", 2022-09-03T06:23:36.911 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.911 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.912 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.912 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.912 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.913 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.913 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:36.914 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.914 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.915 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.915 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.915 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.916 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.916 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.916 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.917 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184551.133544_120.1", 2022-09-03T06:23:36.917 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.918 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:36.918 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:51.133544960Z", 2022-09-03T06:23:36.919 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.919 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.920 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.920 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.920 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.920 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.921 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:36.921 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.921 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.922 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.922 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.922 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.923 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.923 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.923 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.924 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184551.134772_121.1", 2022-09-03T06:23:36.924 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.924 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16", 2022-09-03T06:23:36.924 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:51.134772057Z", 2022-09-03T06:23:36.925 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.925 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.925 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.925 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.926 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.926 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.926 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yedFU2W-LnAQG28SX6Ff-RY", 2022-09-03T06:23:36.926 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.927 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.927 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.927 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.928 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.928 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.928 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.928 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.929 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184551.139229_122.1", 2022-09-03T06:23:36.929 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.929 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16", 2022-09-03T06:23:36.929 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:51.139229501Z", 2022-09-03T06:23:36.930 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.930 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.930 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.930 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.931 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.931 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.931 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yedFU2W-LnAQG28SX6Ff-RY", 2022-09-03T06:23:36.931 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.932 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.932 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.932 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.932 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.932 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.933 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.933 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.933 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.168524_123.1", 2022-09-03T06:23:36.933 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.934 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:36.934 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.168524769Z", 2022-09-03T06:23:36.935 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.935 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.935 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.936 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.936 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.937 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.937 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:36.937 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.938 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.938 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.939 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.939 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.939 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.940 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.940 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.940 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.172232_124.1", 2022-09-03T06:23:36.940 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.941 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:36.941 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.172232256Z", 2022-09-03T06:23:36.941 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.941 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.941 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.942 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.942 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.942 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.942 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:36.943 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.943 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.943 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.943 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.944 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.944 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.944 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.944 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.945 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.173517_125.1", 2022-09-03T06:23:36.945 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.945 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17", 2022-09-03T06:23:36.945 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.173517470Z", 2022-09-03T06:23:36.946 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.946 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.946 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.946 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.947 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.947 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.947 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_KGtuntqhz3tvT1e4k36oFKj", 2022-09-03T06:23:36.947 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.947 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.948 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.948 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.948 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.948 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.949 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.949 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.949 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.178102_126.1", 2022-09-03T06:23:36.949 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.950 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17", 2022-09-03T06:23:36.950 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.178102240Z", 2022-09-03T06:23:36.950 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.950 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.951 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.951 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.951 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.951 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.952 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_KGtuntqhz3tvT1e4k36oFKj", 2022-09-03T06:23:36.952 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.952 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.952 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.953 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.953 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.953 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.953 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.953 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.954 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.215736_127.1", 2022-09-03T06:23:36.954 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.954 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:36.954 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.215736666Z", 2022-09-03T06:23:36.955 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.955 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.955 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.955 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.956 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.956 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.956 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:36.956 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.957 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.957 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.957 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.957 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.958 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.958 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.958 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.958 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.219063_128.1", 2022-09-03T06:23:36.959 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.959 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:36.959 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.219063433Z", 2022-09-03T06:23:36.959 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.959 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.960 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.960 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.960 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.960 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.961 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:36.961 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.961 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.961 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.962 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.962 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.962 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.962 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.963 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.963 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.220690_129.1", 2022-09-03T06:23:36.963 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.963 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18", 2022-09-03T06:23:36.964 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.220690992Z", 2022-09-03T06:23:36.964 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.964 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.964 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.964 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.965 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.965 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.965 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm651SSphsn2Jx7Kwrq9oNR", 2022-09-03T06:23:36.965 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.966 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.966 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.966 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.966 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.967 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.967 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.967 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.967 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.226118_130.1", 2022-09-03T06:23:36.968 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:36.968 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18", 2022-09-03T06:23:36.968 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.226118188Z", 2022-09-03T06:23:36.968 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.969 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.969 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:36.969 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:36.969 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.969 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.970 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm651SSphsn2Jx7Kwrq9oNR", 2022-09-03T06:23:36.970 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.970 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.970 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.971 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.971 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.971 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.971 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.972 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.972 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.250228_131.1", 2022-09-03T06:23:36.972 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.972 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:36.973 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.250228835Z", 2022-09-03T06:23:36.973 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.973 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.973 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:36.974 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.974 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.974 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.974 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:36.974 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.975 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.975 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.975 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.975 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.976 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.976 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.976 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.976 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.253497_132.1", 2022-09-03T06:23:36.977 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.977 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:36.977 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.253497827Z", 2022-09-03T06:23:36.977 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.978 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.978 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:36.978 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.978 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.979 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.979 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:36.979 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.979 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.979 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.980 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.980 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.980 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.980 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.981 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.981 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.266389_133.1", 2022-09-03T06:23:36.981 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.981 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:36.982 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.266389718Z", 2022-09-03T06:23:36.982 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.982 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.982 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:36.983 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.983 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.983 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.983 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:36.983 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.984 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.984 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.984 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.984 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.985 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.985 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.985 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.985 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.270858_134.1", 2022-09-03T06:23:36.986 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.986 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:36.986 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.270858165Z", 2022-09-03T06:23:36.986 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.987 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.987 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:36.987 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.987 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.988 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.988 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:36.988 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.988 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.988 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.989 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.989 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.989 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.989 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.990 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.990 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.289855_135.1", 2022-09-03T06:23:36.990 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.990 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:36.991 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.289855850Z", 2022-09-03T06:23:36.991 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.991 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.991 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:36.991 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.992 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.992 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.992 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:36.992 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.993 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.993 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.993 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:36.993 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.994 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.994 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.994 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.994 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184552.296862_136.1", 2022-09-03T06:23:36.995 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.995 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:36.995 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:55:52.296862507Z", 2022-09-03T06:23:36.995 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:36.996 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:36.996 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:36.996 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:36.996 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.996 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:36.997 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:36.997 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:36.997 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.997 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:36.998 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:36.998 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.998 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:36.998 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:36.999 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:36.999 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.168986_143.1", 2022-09-03T06:23:36.999 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:36.999 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:37.000 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.168986972Z", 2022-09-03T06:23:37.000 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.000 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.000 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.001 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.001 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.001 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.001 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:37.001 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.002 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.002 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.002 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.002 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.003 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.003 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.003 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.003 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.174598_144.1", 2022-09-03T06:23:37.004 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.004 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:37.004 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.174598788Z", 2022-09-03T06:23:37.004 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.005 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.005 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.005 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.005 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.006 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.006 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:37.006 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.006 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.006 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.007 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.007 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.007 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.007 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.008 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.008 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.176005_145.1", 2022-09-03T06:23:37.008 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.008 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19", 2022-09-03T06:23:37.009 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.176005675Z", 2022-09-03T06:23:37.009 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.009 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.009 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.010 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.010 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.010 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.010 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yDXruVlqT4Xsg8Al8Plz6TA", 2022-09-03T06:23:37.011 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.011 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.011 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.011 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.011 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.012 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.012 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.012 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.012 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.179021_146.1", 2022-09-03T06:23:37.013 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.013 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19", 2022-09-03T06:23:37.013 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.179021577Z", 2022-09-03T06:23:37.013 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.014 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.014 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.014 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.014 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.015 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.015 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yDXruVlqT4Xsg8Al8Plz6TA", 2022-09-03T06:23:37.015 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.015 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.016 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.016 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.016 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.016 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.016 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.017 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.017 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.194719_147.1", 2022-09-03T06:23:37.017 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.017 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:37.018 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.194719304Z", 2022-09-03T06:23:37.018 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.018 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.018 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:37.019 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.019 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.019 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.019 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:37.020 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.020 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.020 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.020 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.021 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.021 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.021 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.021 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.022 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662184997.201443_148.1", 2022-09-03T06:23:37.022 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.022 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:37.022 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:03:17.201443123Z", 2022-09-03T06:23:37.023 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.023 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.023 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:37.023 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.023 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.024 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.024 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:37.024 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.024 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.025 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.025 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.026 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.026 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.026 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.026 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.027 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185259.164178_150.1", 2022-09-03T06:23:37.027 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.027 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:37.027 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:39.164178377Z", 2022-09-03T06:23:37.027 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.028 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.028 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.028 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.028 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.029 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.029 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.029 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.029 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.030 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.030 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.030 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.030 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.031 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.031 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.031 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185259.166746_154.1", 2022-09-03T06:23:37.031 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.031 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:37.032 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:39.166746306Z", 2022-09-03T06:23:37.032 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.032 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.032 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.033 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.033 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.033 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.034 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.034 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.034 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.034 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.035 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.035 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.035 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.035 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.036 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.036 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185259.167637_155.1", 2022-09-03T06:23:37.036 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.036 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20", 2022-09-03T06:23:37.037 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:39.167637326Z", 2022-09-03T06:23:37.037 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.037 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.037 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.038 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.038 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.038 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.038 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZKafDPgftyjUAayY_VUswW5", 2022-09-03T06:23:37.039 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.039 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.039 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.039 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.039 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.040 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.040 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.040 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.040 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185259.169934_156.1", 2022-09-03T06:23:37.041 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.041 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20", 2022-09-03T06:23:37.041 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:07:39.169934736Z", 2022-09-03T06:23:37.041 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.042 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.042 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.042 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.042 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.043 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.043 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZKafDPgftyjUAayY_VUswW5", 2022-09-03T06:23:37.043 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.043 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.044 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.044 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.044 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.044 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.045 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.045 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.045 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185351.815540_157.1", 2022-09-03T06:23:37.045 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.046 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:37.046 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:09:11.815540177Z", 2022-09-03T06:23:37.046 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.046 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.046 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.047 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.047 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.047 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.047 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.048 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.048 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.048 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.048 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.049 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.049 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.049 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.049 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.050 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185351.819576_158.1", 2022-09-03T06:23:37.050 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.050 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:37.050 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:09:11.819576411Z", 2022-09-03T06:23:37.051 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.051 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.051 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.051 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.051 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.052 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.052 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.052 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.052 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.053 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.053 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.053 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.053 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.054 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.054 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.054 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185415.254080_165.1", 2022-09-03T06:23:37.054 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.055 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:37.055 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:10:15.254080988Z", 2022-09-03T06:23:37.055 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.055 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.056 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.056 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.056 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.056 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.056 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.057 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:37.057 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.057 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.057 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.058 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.058 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.058 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.058 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.059 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185415.257852_166.1", 2022-09-03T06:23:37.059 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.059 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:37.059 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:10:15.257852595Z", 2022-09-03T06:23:37.060 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.060 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.060 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.060 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.061 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.061 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.061 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:37.061 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:37.061 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.062 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.062 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.062 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.062 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.063 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.063 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.063 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.140367_167.1", 2022-09-03T06:23:37.063 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.064 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:37.064 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.140367923Z", 2022-09-03T06:23:37.064 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.064 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.065 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.065 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.065 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.065 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.066 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:37.066 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.066 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.066 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.067 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.067 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.067 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.068 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.068 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.068 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.143952_168.1", 2022-09-03T06:23:37.069 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.069 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:37.070 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.143952074Z", 2022-09-03T06:23:37.070 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.070 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.071 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.071 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.071 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.072 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.072 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:37.073 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.073 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.073 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.074 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.074 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.074 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.075 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.075 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.075 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.145226_169.1", 2022-09-03T06:23:37.076 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.076 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21", 2022-09-03T06:23:37.077 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.145226679Z", 2022-09-03T06:23:37.077 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.077 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.078 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.078 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.078 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.079 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.079 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WXbrqJZWiL3N6i3X5RrSf_L", 2022-09-03T06:23:37.079 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.080 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.080 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.081 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.081 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.081 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.082 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.082 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.083 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.148908_170.1", 2022-09-03T06:23:37.083 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.084 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21", 2022-09-03T06:23:37.084 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.148908181Z", 2022-09-03T06:23:37.084 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.084 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.085 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.085 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.085 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.085 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.086 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WXbrqJZWiL3N6i3X5RrSf_L", 2022-09-03T06:23:37.086 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.086 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.086 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.087 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.087 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.087 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.087 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.088 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.088 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.171153_171.1", 2022-09-03T06:23:37.088 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.088 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:37.088 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.171153368Z", 2022-09-03T06:23:37.089 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.089 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.089 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.089 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.090 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.090 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.090 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:37.090 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.091 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.091 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.091 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.091 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.092 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.092 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.092 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.092 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.174859_172.1", 2022-09-03T06:23:37.093 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.093 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:37.093 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.174859152Z", 2022-09-03T06:23:37.093 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.093 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.094 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.094 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.094 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.094 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.095 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:37.095 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.095 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.095 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.096 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.096 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.096 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.096 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.097 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.097 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.176058_173.1", 2022-09-03T06:23:37.097 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.097 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22", 2022-09-03T06:23:37.098 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.176058283Z", 2022-09-03T06:23:37.098 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.098 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.098 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.099 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.099 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.099 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.099 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_4TVYbHf2FAhumE819lO37xf", 2022-09-03T06:23:37.100 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.100 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.100 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.100 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.100 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.101 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.101 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.101 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.101 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.180790_174.1", 2022-09-03T06:23:37.102 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.102 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22", 2022-09-03T06:23:37.102 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.180790715Z", 2022-09-03T06:23:37.102 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.103 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.103 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.103 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.103 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.104 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.104 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_4TVYbHf2FAhumE819lO37xf", 2022-09-03T06:23:37.104 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.104 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.105 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.105 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.105 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.105 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.106 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.106 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.106 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.221697_175.1", 2022-09-03T06:23:37.106 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.106 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:37.107 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.221697751Z", 2022-09-03T06:23:37.107 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.107 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.107 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.108 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.108 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.108 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.108 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:37.109 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.109 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.109 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.109 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.110 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.110 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.110 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.110 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.111 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.229951_176.1", 2022-09-03T06:23:37.111 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.111 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:37.111 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.229951839Z", 2022-09-03T06:23:37.111 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.112 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.112 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.112 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.112 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.113 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.113 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:37.113 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.113 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.114 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.114 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.114 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.114 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.115 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.115 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.115 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.237269_177.1", 2022-09-03T06:23:37.115 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.116 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23", 2022-09-03T06:23:37.116 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.237269112Z", 2022-09-03T06:23:37.116 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.116 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.116 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.117 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.117 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.117 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.117 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_SUW5WcOi0sbRMVG7llLAyro", 2022-09-03T06:23:37.118 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.118 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.118 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.118 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.119 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.119 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.119 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.119 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.120 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.248604_178.1", 2022-09-03T06:23:37.120 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.120 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23", 2022-09-03T06:23:37.120 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.248604519Z", 2022-09-03T06:23:37.120 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.121 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.121 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.121 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.122 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.122 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.122 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_SUW5WcOi0sbRMVG7llLAyro", 2022-09-03T06:23:37.122 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.123 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.123 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.123 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.123 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.123 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.124 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.124 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.124 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.289134_179.1", 2022-09-03T06:23:37.124 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.125 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:37.125 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.289134872Z", 2022-09-03T06:23:37.125 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.125 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.126 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:37.126 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.126 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.126 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.127 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:37.127 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.127 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.127 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.127 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.128 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.128 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.128 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.128 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.129 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.292023_180.1", 2022-09-03T06:23:37.129 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.129 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:37.129 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.292023076Z", 2022-09-03T06:23:37.130 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.130 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.130 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:37.130 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.131 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.131 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.131 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:37.131 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.131 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.132 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.132 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.132 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.132 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.133 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.133 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.133 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.305537_181.1", 2022-09-03T06:23:37.133 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.134 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:37.134 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.305537948Z", 2022-09-03T06:23:37.134 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.134 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.135 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:37.135 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.135 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.135 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.135 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:37.136 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.136 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.136 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.136 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.137 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.137 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.137 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.137 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.138 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.308323_182.1", 2022-09-03T06:23:37.138 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.138 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:37.138 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.308323992Z", 2022-09-03T06:23:37.139 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.139 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.139 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:37.139 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.139 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.140 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.140 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:37.140 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.140 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.141 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.141 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.141 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.141 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.142 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.142 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.142 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.323485_183.1", 2022-09-03T06:23:37.142 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.143 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:37.143 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.323485482Z", 2022-09-03T06:23:37.143 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.143 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.143 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:37.144 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.144 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.144 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.144 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:37.145 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.145 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.145 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.145 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.146 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.146 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.146 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.146 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.147 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185472.326245_184.1", 2022-09-03T06:23:37.147 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.147 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:37.147 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:11:12.326245244Z", 2022-09-03T06:23:37.148 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.148 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.148 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:37.148 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.149 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.149 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.149 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:37.149 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.150 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.150 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.150 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.150 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.151 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.151 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.151 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.151 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.128610_187.1", 2022-09-03T06:23:37.151 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.152 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:37.152 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.128610110Z", 2022-09-03T06:23:37.152 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.152 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.153 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.153 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.153 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.153 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.154 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.154 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.154 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.154 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.155 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.155 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.155 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.155 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.156 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.156 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.132654_188.1", 2022-09-03T06:23:37.156 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.156 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:37.156 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.132654659Z", 2022-09-03T06:23:37.157 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.157 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.157 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.157 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.158 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.158 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.158 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.158 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.159 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.159 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.159 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.159 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.160 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.160 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.160 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.160 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.133913_189.1", 2022-09-03T06:23:37.161 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.161 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24", 2022-09-03T06:23:37.161 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.133913786Z", 2022-09-03T06:23:37.161 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.161 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.163 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.163 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.164 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.164 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.165 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_flubJt4QosOvedZNv50fpC6", 2022-09-03T06:23:37.165 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.166 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.166 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.167 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.167 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.168 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.168 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.169 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.169 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.136736_190.1", 2022-09-03T06:23:37.170 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.170 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24", 2022-09-03T06:23:37.171 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.136736736Z", 2022-09-03T06:23:37.171 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.172 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.172 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.173 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.173 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.174 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.174 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_flubJt4QosOvedZNv50fpC6", 2022-09-03T06:23:37.175 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.175 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.176 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.176 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.177 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.178 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.178 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.179 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.179 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.158636_191.1", 2022-09-03T06:23:37.180 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.180 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:37.181 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.158636011Z", 2022-09-03T06:23:37.181 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.182 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.182 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.183 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.183 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.184 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.184 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.185 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.185 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.186 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.186 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.187 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.187 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.188 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.189 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.189 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.162317_192.1", 2022-09-03T06:23:37.189 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.190 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:37.191 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.162317814Z", 2022-09-03T06:23:37.191 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.192 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.192 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.193 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.193 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.194 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.194 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.195 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.195 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.196 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.196 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.197 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.197 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.198 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.198 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.199 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.163511_193.1", 2022-09-03T06:23:37.199 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.200 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25", 2022-09-03T06:23:37.200 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.163511615Z", 2022-09-03T06:23:37.201 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.201 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.202 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.202 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.203 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.203 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.204 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tB02hvaj6C_E_OnwBMY8ywL", 2022-09-03T06:23:37.204 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.205 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.205 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.206 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.206 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.207 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.207 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.208 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.208 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.166349_194.1", 2022-09-03T06:23:37.209 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.210 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25", 2022-09-03T06:23:37.210 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.166349608Z", 2022-09-03T06:23:37.211 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.211 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.212 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.212 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.213 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.213 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.214 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_tB02hvaj6C_E_OnwBMY8ywL", 2022-09-03T06:23:37.214 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.215 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.215 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.216 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.216 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.217 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.217 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.218 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.218 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.203237_195.1", 2022-09-03T06:23:37.219 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.219 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:37.220 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.203237512Z", 2022-09-03T06:23:37.220 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.221 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.221 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.222 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.222 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.223 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.223 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.224 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.224 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.225 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.225 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.226 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.226 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.227 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.227 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.228 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.206000_196.1", 2022-09-03T06:23:37.228 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.229 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:37.229 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.206000308Z", 2022-09-03T06:23:37.230 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.230 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.231 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.231 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.231 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.232 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.232 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.233 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.233 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.234 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.234 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.235 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.235 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.236 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.236 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.237 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.207006_197.1", 2022-09-03T06:23:37.237 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.238 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26", 2022-09-03T06:23:37.238 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.207006797Z", 2022-09-03T06:23:37.239 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.239 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.240 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.240 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.241 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.241 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.242 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_iK1lRBoYvVYLawtBAmt3Urm", 2022-09-03T06:23:37.242 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.243 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.243 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.244 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.244 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.245 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.246 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.246 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.247 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.211392_198.1", 2022-09-03T06:23:37.248 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.248 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26", 2022-09-03T06:23:37.249 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.211392135Z", 2022-09-03T06:23:37.249 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.250 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.250 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.251 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.251 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.252 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.252 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_iK1lRBoYvVYLawtBAmt3Urm", 2022-09-03T06:23:37.253 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.253 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.254 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.254 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.255 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.255 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.256 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.257 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.257 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.245835_199.1", 2022-09-03T06:23:37.258 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.258 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:37.259 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.245835749Z", 2022-09-03T06:23:37.259 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.260 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.260 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.261 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.261 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.262 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.262 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.263 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.263 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.264 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.264 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.265 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.265 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.266 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.266 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.267 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.250338_200.1", 2022-09-03T06:23:37.267 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.268 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:37.268 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.250338287Z", 2022-09-03T06:23:37.269 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.269 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.270 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.270 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.271 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.271 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.272 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.272 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.273 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.273 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.274 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.274 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.275 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.275 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.276 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.276 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.265702_201.1", 2022-09-03T06:23:37.277 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.277 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:37.278 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.265702416Z", 2022-09-03T06:23:37.278 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.279 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.279 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.280 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.280 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.281 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.281 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.282 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:37.282 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.283 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.283 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.284 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.285 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.285 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.286 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.286 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.270146_202.1", 2022-09-03T06:23:37.287 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.287 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:37.288 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.270146860Z", 2022-09-03T06:23:37.288 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.289 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.289 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.289 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.290 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.290 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.291 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:37.291 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:37.292 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.292 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.293 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.293 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.294 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.294 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.295 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.295 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.283449_203.1", 2022-09-03T06:23:37.296 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.296 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:37.297 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.283449642Z", 2022-09-03T06:23:37.297 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.298 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.298 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.299 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.299 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.300 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.300 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.301 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.301 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.302 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.302 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.303 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.303 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.304 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.304 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.305 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.286127_204.1", 2022-09-03T06:23:37.305 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.306 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:37.306 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.286127335Z", 2022-09-03T06:23:37.307 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.307 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.308 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.308 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.309 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.309 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.310 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.310 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.311 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.311 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.312 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.312 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.313 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.313 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.314 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.314 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.305370_205.1", 2022-09-03T06:23:37.315 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.315 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:37.316 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.305370975Z", 2022-09-03T06:23:37.316 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.317 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.317 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.318 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.318 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.319 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.319 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.320 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:37.320 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.321 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.321 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.322 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.322 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.323 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.323 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.324 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.312602_206.1", 2022-09-03T06:23:37.324 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.324 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:37.325 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.312602358Z", 2022-09-03T06:23:37.325 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.326 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.326 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.327 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.327 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.328 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.328 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:37.329 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:37.329 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.330 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.330 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.331 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.331 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.332 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.332 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.333 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.344134_207.1", 2022-09-03T06:23:37.333 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.334 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:37.334 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.344134550Z", 2022-09-03T06:23:37.335 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.335 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.336 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.336 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.337 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.337 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.338 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.338 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.339 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.339 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.340 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.340 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.341 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.341 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.341 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.342 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.355110_208.1", 2022-09-03T06:23:37.342 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.342 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:37.342 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.355110727Z", 2022-09-03T06:23:37.343 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.343 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.343 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.343 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.344 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.344 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.344 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.344 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.345 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.345 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.345 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.345 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.346 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.346 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.346 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.346 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.382118_209.1", 2022-09-03T06:23:37.347 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.347 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:37.347 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.382118897Z", 2022-09-03T06:23:37.347 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.348 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.348 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.348 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.348 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.349 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.349 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.349 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:37.349 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.350 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.350 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.350 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.350 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.351 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.351 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.351 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185640.386563_210.1", 2022-09-03T06:23:37.351 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.352 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:37.352 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:14:00.386563904Z", 2022-09-03T06:23:37.352 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.352 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.353 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.353 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.353 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.353 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.354 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:37.354 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 3 2022-09-03T06:23:37.354 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.354 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.355 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.355 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.355 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.355 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.356 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.356 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.730358_216.1", 2022-09-03T06:23:37.356 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.356 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:37.357 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.730358281Z", 2022-09-03T06:23:37.357 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.357 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.357 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.358 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.358 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.358 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.358 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:37.359 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.359 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.359 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.359 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.360 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.360 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.360 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.360 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.361 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.735815_217.1", 2022-09-03T06:23:37.361 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.361 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:37.361 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.735815559Z", 2022-09-03T06:23:37.362 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.362 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.362 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.362 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.363 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.363 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.363 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:37.363 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.364 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.364 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.364 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.364 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.365 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.365 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.365 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.365 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.736742_218.1", 2022-09-03T06:23:37.366 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.366 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27", 2022-09-03T06:23:37.366 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.736742093Z", 2022-09-03T06:23:37.366 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.367 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.367 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.367 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.367 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.368 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.368 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_jZ9wkZPqzR2GWEIEfGTWCSz", 2022-09-03T06:23:37.368 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.368 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.369 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.369 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.369 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.369 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.370 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.370 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.370 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.742821_219.1", 2022-09-03T06:23:37.370 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.371 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27", 2022-09-03T06:23:37.371 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.742821039Z", 2022-09-03T06:23:37.371 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.371 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.372 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.372 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.372 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.372 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.373 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_jZ9wkZPqzR2GWEIEfGTWCSz", 2022-09-03T06:23:37.373 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.373 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.373 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.374 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.374 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.374 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.374 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.375 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.375 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.774385_220.1", 2022-09-03T06:23:37.375 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.375 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:37.376 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.774385987Z", 2022-09-03T06:23:37.376 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.376 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.376 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.376 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.377 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.377 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.377 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:37.377 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.378 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.378 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.378 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.378 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.379 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.379 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.379 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.379 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.778620_221.1", 2022-09-03T06:23:37.380 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.380 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:37.380 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.778620417Z", 2022-09-03T06:23:37.380 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.381 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.381 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.381 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.381 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.382 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.382 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:37.382 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.382 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.383 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.383 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.383 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.383 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.384 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.384 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.384 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.779591_222.1", 2022-09-03T06:23:37.384 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.385 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28", 2022-09-03T06:23:37.385 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.779591267Z", 2022-09-03T06:23:37.385 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.385 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.386 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.386 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.386 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.386 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.387 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ISaB31FGT1RJh5d1WEGeO7S", 2022-09-03T06:23:37.387 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.387 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.387 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.388 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.388 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.388 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.388 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.389 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.389 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.784101_223.1", 2022-09-03T06:23:37.389 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.389 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28", 2022-09-03T06:23:37.389 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.784101743Z", 2022-09-03T06:23:37.390 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.390 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.390 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.391 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.391 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.391 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.391 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_ISaB31FGT1RJh5d1WEGeO7S", 2022-09-03T06:23:37.392 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.392 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.392 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.392 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.392 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.393 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.393 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.393 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.393 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.842565_224.1", 2022-09-03T06:23:37.394 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.394 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:37.394 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.842565677Z", 2022-09-03T06:23:37.394 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.395 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.395 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.395 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.395 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.396 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.396 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:37.396 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.396 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.397 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.397 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.397 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.397 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.398 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.398 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.398 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.848053_225.1", 2022-09-03T06:23:37.398 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.399 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:37.399 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.848053689Z", 2022-09-03T06:23:37.399 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.399 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.400 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.400 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.400 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.400 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.401 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:37.401 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.401 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.401 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.402 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.402 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.402 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.402 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.403 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.403 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.850209_226.1", 2022-09-03T06:23:37.403 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.403 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29", 2022-09-03T06:23:37.404 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.850209520Z", 2022-09-03T06:23:37.404 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.404 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.404 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.405 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.405 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.405 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.405 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_JaA8RlJaRxOPPcYHu0FpEzg", 2022-09-03T06:23:37.406 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.406 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.406 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.406 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.407 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.407 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.407 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.407 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.408 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.855964_227.1", 2022-09-03T06:23:37.408 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.408 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29", 2022-09-03T06:23:37.408 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.855964453Z", 2022-09-03T06:23:37.409 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.409 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.409 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.409 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.409 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.410 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.410 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_JaA8RlJaRxOPPcYHu0FpEzg", 2022-09-03T06:23:37.410 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.410 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.411 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.411 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.411 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.411 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.412 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.412 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.412 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.894939_228.1", 2022-09-03T06:23:37.412 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.413 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:37.413 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.894939418Z", 2022-09-03T06:23:37.413 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.413 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.414 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:37.414 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.414 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.414 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.415 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:37.415 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.415 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.415 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.416 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.416 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.416 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.416 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.417 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.417 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.897692_229.1", 2022-09-03T06:23:37.417 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.417 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:37.418 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.897692116Z", 2022-09-03T06:23:37.418 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.418 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.418 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:37.419 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.419 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.419 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.419 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:37.420 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.420 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.420 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.420 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.421 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.421 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.421 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.421 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.422 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.921086_230.1", 2022-09-03T06:23:37.422 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.422 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:37.422 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.921086404Z", 2022-09-03T06:23:37.423 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.423 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.423 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:37.423 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.424 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.424 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.424 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:37.424 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.425 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.425 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.425 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.425 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.426 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.426 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.426 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.426 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185755.923839_231.1", 2022-09-03T06:23:37.427 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.427 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:37.427 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:55.923839471Z", 2022-09-03T06:23:37.427 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.428 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.428 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:37.428 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.428 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.429 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.429 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:37.429 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.429 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.430 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.430 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.430 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.430 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.431 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.431 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.431 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.001910_232.1", 2022-09-03T06:23:37.431 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.432 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:37.432 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.001910722Z", 2022-09-03T06:23:37.432 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.432 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.433 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:37.433 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.433 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.433 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.434 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:37.434 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.434 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.434 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.435 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.435 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.435 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.435 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.436 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.436 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.011478_233.1", 2022-09-03T06:23:37.436 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.436 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:37.437 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.011478475Z", 2022-09-03T06:23:37.437 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.437 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.437 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:37.438 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.438 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.438 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.438 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:37.439 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.439 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.439 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.439 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.440 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.440 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.440 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.440 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.441 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.059572_234.1", 2022-09-03T06:23:37.441 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.441 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:37.441 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.059572375Z", 2022-09-03T06:23:37.442 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.442 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.442 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.442 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.443 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.443 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.443 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:37.443 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.444 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.444 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.444 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.444 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.445 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.445 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.445 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.445 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.063873_235.1", 2022-09-03T06:23:37.446 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.446 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:37.446 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.063873838Z", 2022-09-03T06:23:37.446 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.447 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.447 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.447 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.447 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.448 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.448 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:37.448 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.448 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.449 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.449 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.449 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.449 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.450 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.450 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.450 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.064807_236.1", 2022-09-03T06:23:37.450 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.451 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30", 2022-09-03T06:23:37.451 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.064807654Z", 2022-09-03T06:23:37.451 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.451 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.452 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.452 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.452 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.453 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.453 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_emDKTj1MXu1EbBl7W-BZDCI", 2022-09-03T06:23:37.453 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.454 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.454 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.454 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.455 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.455 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.455 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.455 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.456 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.069230_237.1", 2022-09-03T06:23:37.456 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.456 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30", 2022-09-03T06:23:37.456 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.069230157Z", 2022-09-03T06:23:37.457 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.457 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.457 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.457 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.458 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.458 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.458 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_emDKTj1MXu1EbBl7W-BZDCI", 2022-09-03T06:23:37.458 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.459 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.459 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.459 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.459 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.460 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.460 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.460 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.460 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.098699_238.1", 2022-09-03T06:23:37.460 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.461 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:37.461 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.098699181Z", 2022-09-03T06:23:37.461 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.461 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.462 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.462 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.462 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.463 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.463 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:37.463 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.463 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.464 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.464 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.464 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.464 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.465 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.465 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.465 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.104362_239.1", 2022-09-03T06:23:37.465 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.466 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:37.466 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.104362288Z", 2022-09-03T06:23:37.466 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.466 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.467 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.467 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.467 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.467 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.468 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:37.468 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.468 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.468 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.468 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.469 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.469 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.469 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.469 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.470 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.105238_240.1", 2022-09-03T06:23:37.470 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.470 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31", 2022-09-03T06:23:37.470 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.105238360Z", 2022-09-03T06:23:37.471 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.471 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.471 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.472 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.472 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.472 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.472 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Fx_iwLRI89zV-Ovfojlsmn_", 2022-09-03T06:23:37.472 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.473 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.473 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.473 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.473 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.474 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.474 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.474 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.475 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.109389_241.1", 2022-09-03T06:23:37.475 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.475 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31", 2022-09-03T06:23:37.475 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.109389408Z", 2022-09-03T06:23:37.475 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.476 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.476 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.476 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.476 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.477 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.477 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Fx_iwLRI89zV-Ovfojlsmn_", 2022-09-03T06:23:37.477 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.477 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.478 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.478 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.478 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.479 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.479 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.479 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.480 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.146357_242.1", 2022-09-03T06:23:37.480 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.480 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:37.480 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.146357921Z", 2022-09-03T06:23:37.481 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.481 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.481 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.481 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.482 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.482 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.482 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:37.482 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.483 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.483 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.483 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.483 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.484 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.484 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.484 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.484 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.149385_243.1", 2022-09-03T06:23:37.485 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.485 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:37.485 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.149385163Z", 2022-09-03T06:23:37.486 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.486 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.486 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.486 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.487 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.487 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.487 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:37.487 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.488 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.488 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.488 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.488 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.489 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.489 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.489 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.490 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.150696_244.1", 2022-09-03T06:23:37.490 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.490 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32", 2022-09-03T06:23:37.490 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.150696281Z", 2022-09-03T06:23:37.491 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.491 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.491 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.491 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.492 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.492 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.492 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_VJHe0Zrz6-lomZaJx40Mq8S", 2022-09-03T06:23:37.493 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.493 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.493 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.493 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.493 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.494 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.494 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.494 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.495 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.159044_245.1", 2022-09-03T06:23:37.495 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.495 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32", 2022-09-03T06:23:37.495 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.159044634Z", 2022-09-03T06:23:37.496 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.496 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.496 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.496 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.496 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.497 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.497 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_VJHe0Zrz6-lomZaJx40Mq8S", 2022-09-03T06:23:37.497 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.497 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.498 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.498 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.498 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.498 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.499 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.499 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.499 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.221962_246.1", 2022-09-03T06:23:37.499 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.500 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:37.500 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.221962483Z", 2022-09-03T06:23:37.500 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.500 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.501 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:37.501 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.501 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.501 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.502 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:37.502 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.502 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.502 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.503 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.503 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.503 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.503 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.504 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.504 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.225097_247.1", 2022-09-03T06:23:37.504 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.504 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:37.505 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.225097472Z", 2022-09-03T06:23:37.505 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.505 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.505 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:37.506 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.506 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.506 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.506 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:37.507 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.507 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.507 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.507 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.508 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.508 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.508 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.508 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.509 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.324010_248.1", 2022-09-03T06:23:37.509 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.509 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:37.509 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.324010891Z", 2022-09-03T06:23:37.510 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.510 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.510 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:37.510 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.511 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.511 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.511 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:37.511 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.512 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.512 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.512 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.512 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.513 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.513 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.513 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.513 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.326610_249.1", 2022-09-03T06:23:37.514 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.514 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:37.514 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.326610391Z", 2022-09-03T06:23:37.514 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.515 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.515 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:37.515 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.515 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.516 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.516 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:37.516 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.516 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.517 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.517 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.517 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.517 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.518 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.518 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.518 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.388152_250.1", 2022-09-03T06:23:37.518 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.519 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:37.519 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.388152023Z", 2022-09-03T06:23:37.519 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.519 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.520 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:37.520 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.520 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.520 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.521 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:37.521 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.521 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.521 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.522 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.522 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.522 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.522 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.523 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.523 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.390795_251.1", 2022-09-03T06:23:37.523 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.523 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:37.524 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.390795101Z", 2022-09-03T06:23:37.524 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.524 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.524 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:37.525 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.525 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.525 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.525 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:37.526 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.526 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.526 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.526 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.527 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.527 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.527 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.527 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.528 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.428775_252.1", 2022-09-03T06:23:37.528 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.528 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:37.528 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.428775853Z", 2022-09-03T06:23:37.529 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.529 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.529 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.529 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.530 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.530 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.530 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:37.530 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.531 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.531 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.531 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.531 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.532 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.532 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.532 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.532 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.432176_253.1", 2022-09-03T06:23:37.533 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.533 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:37.533 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.432176035Z", 2022-09-03T06:23:37.533 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.534 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.534 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.534 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.534 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.535 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.535 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:37.535 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.535 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.536 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.536 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.536 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.536 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.537 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.537 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.537 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.433149_254.1", 2022-09-03T06:23:37.537 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.538 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33", 2022-09-03T06:23:37.538 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.433149170Z", 2022-09-03T06:23:37.538 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.538 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.539 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.539 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.539 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.539 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.540 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_w0QD1jE_b482WNZp5Lj2ky4", 2022-09-03T06:23:37.540 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.540 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.540 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.541 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.541 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.541 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.541 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.542 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.542 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.435497_255.1", 2022-09-03T06:23:37.542 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.542 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33", 2022-09-03T06:23:37.543 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.435497913Z", 2022-09-03T06:23:37.543 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.543 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.543 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.544 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.544 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.544 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.544 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_w0QD1jE_b482WNZp5Lj2ky4", 2022-09-03T06:23:37.545 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.545 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.545 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.545 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.546 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.546 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.546 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.546 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.547 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.463906_256.1", 2022-09-03T06:23:37.547 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.547 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:37.547 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.463906896Z", 2022-09-03T06:23:37.548 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.548 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.548 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.548 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.549 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.549 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.549 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:37.549 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.550 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.550 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.550 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.550 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.551 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.551 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.551 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.551 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.466439_257.1", 2022-09-03T06:23:37.552 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.552 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:37.552 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.466439294Z", 2022-09-03T06:23:37.552 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.553 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.553 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.553 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.553 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.554 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.554 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:37.554 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.554 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.555 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.555 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.555 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.555 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.556 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.556 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.556 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.467396_258.1", 2022-09-03T06:23:37.556 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.557 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34", 2022-09-03T06:23:37.557 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.467396448Z", 2022-09-03T06:23:37.557 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.557 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.558 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.558 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.558 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.558 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.559 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G2EFJYOe8FOdjOCj6aBraUR", 2022-09-03T06:23:37.559 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.559 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.559 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.559 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.560 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.560 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.560 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.560 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.561 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.471551_259.1", 2022-09-03T06:23:37.561 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.561 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34", 2022-09-03T06:23:37.561 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.471551488Z", 2022-09-03T06:23:37.562 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.562 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.562 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.562 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.563 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.563 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.563 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G2EFJYOe8FOdjOCj6aBraUR", 2022-09-03T06:23:37.563 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.564 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.564 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.564 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.564 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.565 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.565 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.565 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.565 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.517947_260.1", 2022-09-03T06:23:37.566 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.566 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:37.566 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.517947959Z", 2022-09-03T06:23:37.566 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.567 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.567 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.567 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.567 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.568 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.568 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:37.568 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.568 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.569 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.569 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.569 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.569 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.570 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.570 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.570 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.522051_261.1", 2022-09-03T06:23:37.570 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.571 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:37.571 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.522051544Z", 2022-09-03T06:23:37.571 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.571 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.572 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.572 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.572 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.572 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.573 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:37.573 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.573 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.573 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.574 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.574 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.574 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.574 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.575 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.575 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.524288_262.1", 2022-09-03T06:23:37.575 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.575 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35", 2022-09-03T06:23:37.576 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.524288626Z", 2022-09-03T06:23:37.576 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.576 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.576 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.577 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.577 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.577 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.577 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G8oh7TOeWyD2_FvC8AIXccM", 2022-09-03T06:23:37.578 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.578 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.578 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.578 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.579 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.579 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.579 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.579 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.580 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.526709_263.1", 2022-09-03T06:23:37.580 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.580 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35", 2022-09-03T06:23:37.580 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.526709154Z", 2022-09-03T06:23:37.581 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.581 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.581 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.581 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.582 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.582 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.582 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G8oh7TOeWyD2_FvC8AIXccM", 2022-09-03T06:23:37.582 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.583 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.583 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.583 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.583 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.584 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.584 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.584 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.584 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.565852_264.1", 2022-09-03T06:23:37.585 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.585 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:37.585 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.565852141Z", 2022-09-03T06:23:37.585 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.586 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.586 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:37.586 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.586 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.587 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.587 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:37.587 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.587 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.588 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.588 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.588 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.588 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.589 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.589 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.589 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185756.568581_265.1", 2022-09-03T06:23:37.589 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.589 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:37.590 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:56.568581265Z", 2022-09-03T06:23:37.590 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.590 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.591 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:37.591 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.591 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.591 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.591 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:37.592 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.592 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.592 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.592 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.593 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.593 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.593 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.593 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.594 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185757.605699_266.1", 2022-09-03T06:23:37.594 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.594 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:37.594 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:57.605699761Z", 2022-09-03T06:23:37.595 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.595 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.595 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:37.595 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.596 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.596 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.596 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:37.596 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.597 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.597 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.597 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.597 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.598 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.598 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.598 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.598 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185757.608333_267.1", 2022-09-03T06:23:37.599 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.599 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:37.599 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:57.608333919Z", 2022-09-03T06:23:37.599 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.600 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.600 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:37.600 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.600 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.601 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.601 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:37.601 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.601 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.602 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.602 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.602 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.602 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.603 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.603 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.603 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.691847_268.1", 2022-09-03T06:23:37.603 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.604 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:37.604 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.691847123Z", 2022-09-03T06:23:37.604 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.604 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.604 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:37.605 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.605 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.605 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.605 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:37.606 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.606 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.606 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.606 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.607 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.607 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.607 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.607 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.608 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.696866_269.1", 2022-09-03T06:23:37.608 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.608 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:37.608 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.696866177Z", 2022-09-03T06:23:37.609 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.609 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.609 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:37.609 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.610 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.610 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.610 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:37.610 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 2 2022-09-03T06:23:37.611 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.611 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.611 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.611 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.612 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.612 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.612 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.612 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.804943_270.1", 2022-09-03T06:23:37.613 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.613 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:23:37.613 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.804943552Z", 2022-09-03T06:23:37.613 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.614 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.614 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.614 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.614 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.615 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.615 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-F9Dl85tpDzK3gXJ-aezc_j", 2022-09-03T06:23:37.615 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.615 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.616 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.616 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.616 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.616 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.617 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.617 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.617 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.808467_271.1", 2022-09-03T06:23:37.617 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.618 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:23:37.618 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.808467134Z", 2022-09-03T06:23:37.618 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.618 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.619 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.619 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.619 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.619 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.620 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-F9Dl85tpDzK3gXJ-aezc_j", 2022-09-03T06:23:37.620 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.620 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.620 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.621 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.621 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.621 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.621 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.622 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.622 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.810847_272.1", 2022-09-03T06:23:37.622 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.622 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-36", 2022-09-03T06:23:37.623 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.810847136Z", 2022-09-03T06:23:37.623 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.623 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.623 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.624 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.624 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.624 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.624 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_5JS-wFAV9CrTmAifbKi9NjK", 2022-09-03T06:23:37.625 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.625 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.625 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.625 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.626 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.626 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.626 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.626 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.627 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.813462_273.1", 2022-09-03T06:23:37.627 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.627 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-36", 2022-09-03T06:23:37.627 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.813462243Z", 2022-09-03T06:23:37.628 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.628 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.628 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.628 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.629 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.629 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.629 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_5JS-wFAV9CrTmAifbKi9NjK", 2022-09-03T06:23:37.629 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.630 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.630 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.630 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.630 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.631 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.631 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.631 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.631 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.844464_274.1", 2022-09-03T06:23:37.632 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.632 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:23:37.632 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.844464322Z", 2022-09-03T06:23:37.632 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.633 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.633 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.633 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.633 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.634 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.634 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_brde0pgOsuZjrCySsOp9pmR", 2022-09-03T06:23:37.634 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.634 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.635 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.635 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.635 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.635 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.636 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.636 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.636 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.850329_275.1", 2022-09-03T06:23:37.636 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.637 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:23:37.637 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.850329433Z", 2022-09-03T06:23:37.637 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.637 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.638 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.638 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.638 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.638 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.639 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_brde0pgOsuZjrCySsOp9pmR", 2022-09-03T06:23:37.639 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.639 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.639 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.640 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.640 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.640 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.640 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.641 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.641 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.855256_276.1", 2022-09-03T06:23:37.641 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.641 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-37", 2022-09-03T06:23:37.642 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.855256692Z", 2022-09-03T06:23:37.642 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.642 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.642 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.643 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.643 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.643 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.643 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3HAVWKyp6rMdHTXkKukojjv", 2022-09-03T06:23:37.644 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.644 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.644 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.644 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.645 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.645 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.645 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.646 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.646 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.859644_277.1", 2022-09-03T06:23:37.646 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.646 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-37", 2022-09-03T06:23:37.647 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.859644317Z", 2022-09-03T06:23:37.647 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.647 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.647 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.648 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.648 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.648 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.648 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_3HAVWKyp6rMdHTXkKukojjv", 2022-09-03T06:23:37.649 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.649 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.649 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.649 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.650 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.650 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.650 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.650 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.651 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.902177_278.1", 2022-09-03T06:23:37.651 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.651 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:23:37.651 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.902177035Z", 2022-09-03T06:23:37.652 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.652 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.652 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.653 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.653 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.653 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.653 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Iv3Ff3D6p1XbZ30mXZkhK63", 2022-09-03T06:23:37.654 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.654 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.654 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.654 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.655 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.655 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.655 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.655 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.656 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.907504_279.1", 2022-09-03T06:23:37.656 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.656 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:23:37.656 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.907504015Z", 2022-09-03T06:23:37.657 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.657 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.657 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.657 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.658 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.658 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.658 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Iv3Ff3D6p1XbZ30mXZkhK63", 2022-09-03T06:23:37.658 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.659 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.659 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.659 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.659 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.660 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.660 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.660 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.660 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.908555_280.1", 2022-09-03T06:23:37.661 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.661 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-38", 2022-09-03T06:23:37.661 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.908555860Z", 2022-09-03T06:23:37.662 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.662 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.662 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.663 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.663 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.663 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.663 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G9WP1PhzJZFD5vWOwiYS9Hb", 2022-09-03T06:23:37.664 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.664 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.664 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.664 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.665 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.665 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.665 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.665 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.666 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662185758.915547_281.1", 2022-09-03T06:23:37.666 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.666 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-38", 2022-09-03T06:23:37.666 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:15:58.915547971Z", 2022-09-03T06:23:37.667 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.667 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.667 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.667 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.668 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.668 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.668 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_G9WP1PhzJZFD5vWOwiYS9Hb", 2022-09-03T06:23:37.668 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.669 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.669 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.669 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.669 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.670 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.670 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.671 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.671 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.844866_288.1", 2022-09-03T06:23:37.671 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.671 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39", 2022-09-03T06:23:37.672 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.844866559Z", 2022-09-03T06:23:37.672 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.672 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.672 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.673 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.673 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.673 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.673 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lt1dYegSwDzpwX06X-fGkup", 2022-09-03T06:23:37.674 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.674 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.674 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.674 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.675 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.675 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.675 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.675 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.676 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.850612_289.1", 2022-09-03T06:23:37.676 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.676 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39", 2022-09-03T06:23:37.676 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.850612019Z", 2022-09-03T06:23:37.677 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.677 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.677 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.677 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.677 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.678 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.678 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Lt1dYegSwDzpwX06X-fGkup", 2022-09-03T06:23:37.678 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.678 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.679 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.679 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.680 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.680 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.680 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.680 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.681 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.852628_290.1", 2022-09-03T06:23:37.681 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.681 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-39", 2022-09-03T06:23:37.681 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.852628518Z", 2022-09-03T06:23:37.682 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.682 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.682 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.682 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.683 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.683 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.683 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_O0mztKsloMen3wFjqSH6i9D", 2022-09-03T06:23:37.684 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.684 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.684 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.684 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.685 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.685 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.685 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.685 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.686 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.857180_291.1", 2022-09-03T06:23:37.686 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.686 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-39", 2022-09-03T06:23:37.686 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.857180178Z", 2022-09-03T06:23:37.687 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.687 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.687 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.688 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.688 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.688 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.688 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_O0mztKsloMen3wFjqSH6i9D", 2022-09-03T06:23:37.689 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.689 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.689 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.689 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.690 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.690 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.690 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.690 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.691 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.883799_292.1", 2022-09-03T06:23:37.691 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.691 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40", 2022-09-03T06:23:37.691 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.883799148Z", 2022-09-03T06:23:37.692 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.692 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.692 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.692 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.693 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.693 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.693 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_fswsShBMVZzHxqVULj3KzUU", 2022-09-03T06:23:37.693 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.694 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.694 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.694 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.695 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.695 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.695 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.695 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.696 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.889075_293.1", 2022-09-03T06:23:37.696 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.696 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40", 2022-09-03T06:23:37.697 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.889075582Z", 2022-09-03T06:23:37.697 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.697 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.697 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.698 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.698 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.698 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.698 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_fswsShBMVZzHxqVULj3KzUU", 2022-09-03T06:23:37.699 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.699 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.699 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.699 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.700 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.700 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.700 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.700 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.701 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.890194_294.1", 2022-09-03T06:23:37.701 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.701 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-40", 2022-09-03T06:23:37.701 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.890194639Z", 2022-09-03T06:23:37.702 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.702 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.702 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.702 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.703 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.703 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.703 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_A4T-tDc1XWVEtWeND6N-Hqq", 2022-09-03T06:23:37.703 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.704 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.704 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.704 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.705 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.705 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.705 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.706 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.706 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.896341_295.1", 2022-09-03T06:23:37.706 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.706 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-40", 2022-09-03T06:23:37.707 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.896341961Z", 2022-09-03T06:23:37.707 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.707 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.707 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.708 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.708 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.708 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.708 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_A4T-tDc1XWVEtWeND6N-Hqq", 2022-09-03T06:23:37.709 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.709 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.709 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.709 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.710 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.710 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.710 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.710 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.711 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.947010_296.1", 2022-09-03T06:23:37.711 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.711 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41", 2022-09-03T06:23:37.711 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.947010594Z", 2022-09-03T06:23:37.712 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.712 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.712 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.713 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.713 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.713 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.713 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yitoOgqNEqy39xdDDb8hNi-", 2022-09-03T06:23:37.714 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.714 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.714 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.714 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.715 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.715 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.715 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.715 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.716 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.950610_297.1", 2022-09-03T06:23:37.716 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.716 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41", 2022-09-03T06:23:37.716 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.950610217Z", 2022-09-03T06:23:37.717 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.717 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.717 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.717 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.718 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.718 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.718 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yitoOgqNEqy39xdDDb8hNi-", 2022-09-03T06:23:37.718 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.719 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.719 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.719 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.719 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.720 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.720 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.720 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.720 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.951802_298.1", 2022-09-03T06:23:37.721 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.721 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-41", 2022-09-03T06:23:37.721 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.951802745Z", 2022-09-03T06:23:37.722 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.722 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.722 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.723 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.723 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.723 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.723 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_rtPxlDQFqSiMUkKzGoM6IJb", 2022-09-03T06:23:37.724 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.724 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.724 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.724 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:37.725 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.725 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.725 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.725 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:37.726 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186063.955393_299.1", 2022-09-03T06:23:37.726 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:37.726 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-41", 2022-09-03T06:23:37.726 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:21:03.955393441Z", 2022-09-03T06:23:37.727 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:37.727 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:37.727 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:37.727 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:37.728 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.728 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:37.728 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_rtPxlDQFqSiMUkKzGoM6IJb", 2022-09-03T06:23:37.728 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:37.729 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:37.729 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:37.729 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:37.729 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.730 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.730 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:37.730 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:37.732 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:37.733 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:37.733 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period b84ccf39-78f5-4998-9db6-42362bc8c98b --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:37.788 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:37.789 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:37.836 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:23:37.837 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.837 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184222.657692_58.1", 2022-09-03T06:23:37.838 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "user", 2022-09-03T06:23:37.838 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "rgw-multisite-test-user", 2022-09-03T06:23:37.839 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:50:22.657692624Z", 2022-09-03T06:23:37.839 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.840 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.840 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.841 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.841 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.842 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.842 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6BN_OEAtimzMqhhbhGc7xsF", 2022-09-03T06:23:37.843 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.843 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.844 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.845 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.845 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.846 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.846 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.847 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.847 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184222.662031_59.1", 2022-09-03T06:23:37.848 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "user", 2022-09-03T06:23:37.848 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "rgw-multisite-test-user", 2022-09-03T06:23:37.849 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:50:22.662031116Z", 2022-09-03T06:23:37.849 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.850 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.851 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.851 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.852 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.852 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.853 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6BN_OEAtimzMqhhbhGc7xsF", 2022-09-03T06:23:37.853 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.854 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.854 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.855 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.855 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.856 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.856 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.857 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.857 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.219295_62.1", 2022-09-03T06:23:37.858 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.858 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1", 2022-09-03T06:23:37.859 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.219295443Z", 2022-09-03T06:23:37.859 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.859 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.860 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.860 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.861 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.861 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.862 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Y7JDAo_bsL14HyXDq4vsLc2", 2022-09-03T06:23:37.862 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.863 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.863 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.864 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.864 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.865 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.865 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.866 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.866 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.222658_63.1", 2022-09-03T06:23:37.867 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.867 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1", 2022-09-03T06:23:37.867 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.222658346Z", 2022-09-03T06:23:37.867 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.868 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.868 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.868 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.868 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.869 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.869 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Y7JDAo_bsL14HyXDq4vsLc2", 2022-09-03T06:23:37.869 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.869 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.870 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.870 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.871 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.871 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.871 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.871 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.872 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.223873_64.1", 2022-09-03T06:23:37.872 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.872 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-1", 2022-09-03T06:23:37.873 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.223873908Z", 2022-09-03T06:23:37.873 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.873 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.873 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.874 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.874 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.874 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.874 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_VlD_M0myhVj3j7F4hjNPDp1", 2022-09-03T06:23:37.875 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.875 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.876 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.876 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.876 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.876 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.877 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.877 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.877 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.226841_65.1", 2022-09-03T06:23:37.877 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.878 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-1", 2022-09-03T06:23:37.878 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.226841368Z", 2022-09-03T06:23:37.878 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.879 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.879 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.879 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.880 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.880 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.880 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_VlD_M0myhVj3j7F4hjNPDp1", 2022-09-03T06:23:37.881 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.881 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.881 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.881 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.882 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.882 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.883 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.883 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.883 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.252557_66.1", 2022-09-03T06:23:37.884 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.884 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2", 2022-09-03T06:23:37.885 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.252557878Z", 2022-09-03T06:23:37.885 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.885 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.886 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.887 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.887 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.887 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.893 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_eGqFRgXckOShhJ5K31PxBCB", 2022-09-03T06:23:37.894 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.894 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.895 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.895 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.896 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.896 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.897 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.897 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.898 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.258211_67.1", 2022-09-03T06:23:37.898 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.899 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2", 2022-09-03T06:23:37.899 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.258211354Z", 2022-09-03T06:23:37.900 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.900 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.900 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.901 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.901 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.902 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.902 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_eGqFRgXckOShhJ5K31PxBCB", 2022-09-03T06:23:37.903 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.903 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.903 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.904 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.904 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.904 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.904 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.905 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.905 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.262819_68.1", 2022-09-03T06:23:37.905 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.906 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-2", 2022-09-03T06:23:37.906 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.262819181Z", 2022-09-03T06:23:37.906 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.906 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.907 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.907 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.907 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.908 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.908 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lkn_J0TsdIPDWPcsSr8e5eP", 2022-09-03T06:23:37.908 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.908 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.909 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.909 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.909 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.909 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.910 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.910 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.910 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184339.267378_69.1", 2022-09-03T06:23:37.911 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.911 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-2", 2022-09-03T06:23:37.911 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:19.267378646Z", 2022-09-03T06:23:37.912 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.912 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.912 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.912 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.913 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.913 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.913 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lkn_J0TsdIPDWPcsSr8e5eP", 2022-09-03T06:23:37.913 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.914 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.914 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.914 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.915 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.915 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.915 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.915 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.916 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184342.345771_70.1", 2022-09-03T06:23:37.916 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.916 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3", 2022-09-03T06:23:37.916 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.345771278Z", 2022-09-03T06:23:37.917 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.917 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.917 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.918 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.918 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.918 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.919 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_eB0pPK7VCkudq9Jt1nUnfVO", 2022-09-03T06:23:37.919 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.919 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.919 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.920 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.920 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.920 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.921 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.921 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.921 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184342.350157_71.1", 2022-09-03T06:23:37.921 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.922 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3", 2022-09-03T06:23:37.922 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.350157638Z", 2022-09-03T06:23:37.922 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.922 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.923 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.923 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.923 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.923 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.924 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_eB0pPK7VCkudq9Jt1nUnfVO", 2022-09-03T06:23:37.924 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.924 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.924 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.925 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.925 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.925 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.925 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.926 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.926 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184342.351175_72.1", 2022-09-03T06:23:37.926 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.927 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-3", 2022-09-03T06:23:37.927 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.351175819Z", 2022-09-03T06:23:37.927 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.928 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.928 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.928 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.929 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.929 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.929 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_xQbDQL6LoF7KY81FCBsr-TR", 2022-09-03T06:23:37.929 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.930 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.930 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.930 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.931 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.931 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.931 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.931 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.932 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184342.357275_73.1", 2022-09-03T06:23:37.932 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.932 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-3", 2022-09-03T06:23:37.933 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:22.357275634Z", 2022-09-03T06:23:37.933 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.933 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.933 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.934 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.934 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.934 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.934 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_xQbDQL6LoF7KY81FCBsr-TR", 2022-09-03T06:23:37.935 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.935 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.935 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.935 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.936 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.936 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.936 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.937 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.193842_74.1", 2022-09-03T06:23:37.937 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.937 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4", 2022-09-03T06:23:37.937 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.193842881Z", 2022-09-03T06:23:37.938 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.938 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.938 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.939 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.939 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.939 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lw10L7BpVQy7VSU2U7UHl07", 2022-09-03T06:23:37.940 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.940 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.940 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.941 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.941 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.941 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.941 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.942 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.198704_75.1", 2022-09-03T06:23:37.942 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.943 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4", 2022-09-03T06:23:37.943 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.198704862Z", 2022-09-03T06:23:37.943 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.944 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.944 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.945 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.946 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.946 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lw10L7BpVQy7VSU2U7UHl07", 2022-09-03T06:23:37.946 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.947 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.947 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.948 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.948 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.948 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.948 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.949 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.200713_76.1", 2022-09-03T06:23:37.949 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.949 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-4", 2022-09-03T06:23:37.949 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.200713524Z", 2022-09-03T06:23:37.950 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.950 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.950 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.951 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.951 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.951 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2mGIDNrcwT5FJidyB-XgESn", 2022-09-03T06:23:37.952 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.952 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.952 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.953 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.953 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.954 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.204734_77.1", 2022-09-03T06:23:37.954 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.954 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-4", 2022-09-03T06:23:37.954 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.204734948Z", 2022-09-03T06:23:37.955 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.955 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.955 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.955 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.956 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.956 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2mGIDNrcwT5FJidyB-XgESn", 2022-09-03T06:23:37.956 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.957 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.957 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.957 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.958 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.958 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.958 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.959 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.224447_78.1", 2022-09-03T06:23:37.959 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.959 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5", 2022-09-03T06:23:37.960 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.224447947Z", 2022-09-03T06:23:37.960 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.960 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.960 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.961 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.961 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.961 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_xo2PU3_oh6eFpkPNkexfq7D", 2022-09-03T06:23:37.962 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.962 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.962 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.962 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.963 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.963 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.963 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.963 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.964 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.227473_79.1", 2022-09-03T06:23:37.964 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.964 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5", 2022-09-03T06:23:37.964 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.227473639Z", 2022-09-03T06:23:37.965 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.965 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.965 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.965 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.966 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.966 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.966 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_xo2PU3_oh6eFpkPNkexfq7D", 2022-09-03T06:23:37.966 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.967 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.967 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.967 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.968 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.968 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.968 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.968 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.969 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.228594_80.1", 2022-09-03T06:23:37.969 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.969 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-5", 2022-09-03T06:23:37.969 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.228594676Z", 2022-09-03T06:23:37.970 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.970 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.970 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.971 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.971 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.972 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.972 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_u7x8KJF8hnPDPr0uRDbixnY", 2022-09-03T06:23:37.973 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.973 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.974 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.974 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.975 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.975 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.976 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.977 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.231192_81.1", 2022-09-03T06:23:37.977 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:37.978 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-5", 2022-09-03T06:23:37.978 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.231192184Z", 2022-09-03T06:23:37.979 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.979 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.980 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.980 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.981 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.981 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.981 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_u7x8KJF8hnPDPr0uRDbixnY", 2022-09-03T06:23:37.982 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.983 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.983 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:37.984 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.984 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.985 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.985 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.986 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.255262_82.1", 2022-09-03T06:23:37.986 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.987 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6", 2022-09-03T06:23:37.987 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.255262164Z", 2022-09-03T06:23:37.988 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.988 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.989 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.989 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.990 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.990 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:37.991 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_7T2FDkJXu_-0lpyQJ4EERx0", 2022-09-03T06:23:37.991 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:37.992 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.992 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:37.993 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:37.993 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.994 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:37.994 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:37.995 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:37.995 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.258031_83.1", 2022-09-03T06:23:37.996 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:37.996 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6", 2022-09-03T06:23:37.997 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.258031985Z", 2022-09-03T06:23:37.997 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:37.998 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:37.998 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:37.999 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:37.999 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.000 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.000 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_7T2FDkJXu_-0lpyQJ4EERx0", 2022-09-03T06:23:38.001 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.001 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.002 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.002 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.004 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.004 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.005 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.259057_84.1", 2022-09-03T06:23:38.005 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.006 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-6", 2022-09-03T06:23:38.006 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.259057684Z", 2022-09-03T06:23:38.007 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.007 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.008 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.008 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.009 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.010 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_JcUmTGR0SXlRqG5W9Zl5D8p", 2022-09-03T06:23:38.010 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.011 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.012 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.012 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.013 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.014 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184348.261653_85.1", 2022-09-03T06:23:38.015 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.015 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-6", 2022-09-03T06:23:38.016 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:28.261653432Z", 2022-09-03T06:23:38.016 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.017 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.017 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.018 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.019 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.019 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_JcUmTGR0SXlRqG5W9Zl5D8p", 2022-09-03T06:23:38.020 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.021 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.021 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.022 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.022 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.024 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.855330_86.1", 2022-09-03T06:23:38.024 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.025 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:38.025 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.855330912Z", 2022-09-03T06:23:38.026 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.026 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.027 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.027 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.028 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.029 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:38.029 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.030 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.030 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.031 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.031 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.032 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.034 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.861502_87.1", 2022-09-03T06:23:38.034 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.035 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:38.035 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.861502176Z", 2022-09-03T06:23:38.036 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.036 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.037 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.037 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.038 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.039 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:38.039 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.040 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.041 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.041 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.042 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.043 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.862483_88.1", 2022-09-03T06:23:38.044 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.044 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7", 2022-09-03T06:23:38.045 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.862483283Z", 2022-09-03T06:23:38.045 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.046 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.046 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.047 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.048 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.048 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:38.049 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.050 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.051 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.051 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.052 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.053 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.867021_89.1", 2022-09-03T06:23:38.054 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.054 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7", 2022-09-03T06:23:38.055 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.867021552Z", 2022-09-03T06:23:38.055 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.056 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.056 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.057 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.058 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.058 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:38.059 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.060 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.060 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.060 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.061 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.061 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.062 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.062 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.888960_90.1", 2022-09-03T06:23:38.062 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.062 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:38.063 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.888960687Z", 2022-09-03T06:23:38.063 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.063 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.063 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.064 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.064 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.064 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:38.065 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.065 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.065 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.065 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.066 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.066 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.066 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.067 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.892373_91.1", 2022-09-03T06:23:38.067 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.067 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:38.067 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.892373633Z", 2022-09-03T06:23:38.068 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.068 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.068 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.068 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.069 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.069 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:38.069 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.070 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.070 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.070 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.071 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.071 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.071 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.893288_92.1", 2022-09-03T06:23:38.072 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.072 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8", 2022-09-03T06:23:38.072 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.893288080Z", 2022-09-03T06:23:38.072 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.073 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.073 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.073 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.074 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.074 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:38.074 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.075 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.075 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.075 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.076 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.076 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.895751_93.1", 2022-09-03T06:23:38.076 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.077 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8", 2022-09-03T06:23:38.077 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.895751491Z", 2022-09-03T06:23:38.077 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.077 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.078 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.078 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.078 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.079 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.079 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:38.079 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.079 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.080 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.080 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.080 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.080 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.081 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.081 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.922520_94.1", 2022-09-03T06:23:38.081 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.081 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:38.082 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.922520779Z", 2022-09-03T06:23:38.082 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.082 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.082 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.083 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.083 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.083 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:38.084 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.084 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.084 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.084 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.085 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.085 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.085 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.085 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.086 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.931081_95.1", 2022-09-03T06:23:38.086 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.086 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:38.086 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.931081374Z", 2022-09-03T06:23:38.087 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.087 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.087 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.087 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.088 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.088 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.088 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:38.088 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.089 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.089 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.089 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.089 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.090 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.090 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.090 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.933197_96.1", 2022-09-03T06:23:38.091 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.091 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9", 2022-09-03T06:23:38.091 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.933197726Z", 2022-09-03T06:23:38.091 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.092 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.092 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.092 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.093 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.093 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:38.093 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.094 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.094 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.094 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.094 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.095 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.095 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.095 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184349.938376_97.1", 2022-09-03T06:23:38.095 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.096 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9", 2022-09-03T06:23:38.096 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:29.938376383Z", 2022-09-03T06:23:38.096 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.096 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.097 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.097 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.097 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.098 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:38.098 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.098 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.099 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.099 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.099 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.100 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.570224_98.1", 2022-09-03T06:23:38.100 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.100 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:38.101 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.570224266Z", 2022-09-03T06:23:38.101 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.101 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.101 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:38.102 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.102 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.102 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:38.103 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.103 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.103 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.104 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.104 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.105 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.574509_99.1", 2022-09-03T06:23:38.105 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.105 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7", 2022-09-03T06:23:38.105 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.574509413Z", 2022-09-03T06:23:38.106 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.106 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.106 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:38.106 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.107 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.107 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_cbgR5WS_jTPHXeLEqI97MWx", 2022-09-03T06:23:38.107 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.108 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.108 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.108 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.109 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.109 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.109 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.109 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.584013_100.1", 2022-09-03T06:23:38.110 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.110 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7", 2022-09-03T06:23:38.110 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.584013358Z", 2022-09-03T06:23:38.110 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.111 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.111 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:38.111 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.112 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.112 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:38.112 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.113 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.113 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "remove" 2022-09-03T06:23:38.113 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.113 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.114 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.114 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.114 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.587599_101.1", 2022-09-03T06:23:38.114 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.115 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-7", 2022-09-03T06:23:38.115 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.587599896Z", 2022-09-03T06:23:38.115 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.115 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.116 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:38.116 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.116 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.117 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.117 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_FeqZFN0dijQBiPCP3p8ZBht", 2022-09-03T06:23:38.117 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.118 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.118 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.118 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.119 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.119 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.120 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.619234_102.1", 2022-09-03T06:23:38.120 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.120 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:38.120 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.619234589Z", 2022-09-03T06:23:38.121 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.121 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.121 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:38.122 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.122 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.122 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:38.123 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.123 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.123 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.124 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.124 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.124 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.125 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.625715_103.1", 2022-09-03T06:23:38.125 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.125 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8", 2022-09-03T06:23:38.125 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.625715365Z", 2022-09-03T06:23:38.126 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.126 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.126 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:38.127 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.127 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.127 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_j6M-dtLZ2HjVFHah1CH4G5d", 2022-09-03T06:23:38.128 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.128 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.128 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.128 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.129 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.129 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.637706_104.1", 2022-09-03T06:23:38.130 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.130 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8", 2022-09-03T06:23:38.130 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.637706717Z", 2022-09-03T06:23:38.130 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.131 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.131 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:38.131 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.132 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.132 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:38.132 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.133 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.133 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "remove" 2022-09-03T06:23:38.133 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.133 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.134 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.639994_105.1", 2022-09-03T06:23:38.134 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.135 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-8", 2022-09-03T06:23:38.135 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.639994804Z", 2022-09-03T06:23:38.135 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.135 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.136 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:38.136 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.136 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.137 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AOYul6PUG-9nKLm0b2BlSeT", 2022-09-03T06:23:38.137 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.137 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.138 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.138 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.138 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.139 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.697911_106.1", 2022-09-03T06:23:38.139 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.139 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:38.140 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.697911177Z", 2022-09-03T06:23:38.140 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.140 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.140 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:38.141 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.141 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.141 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:38.142 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.142 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.142 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.143 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.143 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.143 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.144 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.700455_107.1", 2022-09-03T06:23:38.144 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.144 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9", 2022-09-03T06:23:38.147 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.700455333Z", 2022-09-03T06:23:38.147 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.147 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.148 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:38.148 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.148 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.149 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-z1Tpve3DmtIRrTUTgoikKH", 2022-09-03T06:23:38.149 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.149 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.150 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.150 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.150 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.150 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.151 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.712013_108.1", 2022-09-03T06:23:38.151 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.151 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9", 2022-09-03T06:23:38.152 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.712013759Z", 2022-09-03T06:23:38.152 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.152 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.152 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:38.153 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.153 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.153 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:38.154 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.154 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.154 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "remove" 2022-09-03T06:23:38.155 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.155 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.156 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184350.715807_109.1", 2022-09-03T06:23:38.156 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.156 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-9", 2022-09-03T06:23:38.156 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807393Z", 2022-09-03T06:23:38.157 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.157 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.157 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:38.157 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.158 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.158 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UomrayfNZa3c3CBJwrWCgno", 2022-09-03T06:23:38.158 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.159 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.159 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.159 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.160 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.160 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.561415_64.1", 2022-09-03T06:23:38.161 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.161 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:23:38.161 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.561415905Z", 2022-09-03T06:23:38.161 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.162 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.162 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.162 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.162 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.163 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.163 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6ugH-NS-_bEeQKUdolNq7Oa", 2022-09-03T06:23:38.163 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.163 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.164 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.164 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.164 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.165 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.165 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.165 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.566930_66.1", 2022-09-03T06:23:38.166 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.166 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:23:38.166 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.566930651Z", 2022-09-03T06:23:38.166 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.167 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.167 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.167 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.167 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.168 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.168 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6ugH-NS-_bEeQKUdolNq7Oa", 2022-09-03T06:23:38.168 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.168 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.169 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.169 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.169 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.169 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.170 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.170 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.170 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.569237_67.1", 2022-09-03T06:23:38.170 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.171 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-10", 2022-09-03T06:23:38.171 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.569237568Z", 2022-09-03T06:23:38.171 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.171 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.172 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.172 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.172 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.172 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.173 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_htY0ij10LAX0ZNP6T_0mHNL", 2022-09-03T06:23:38.173 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.173 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.174 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.174 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.174 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.175 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.175 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.574987_68.1", 2022-09-03T06:23:38.175 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.175 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-10", 2022-09-03T06:23:38.176 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.574987235Z", 2022-09-03T06:23:38.176 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.176 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.177 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.177 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.177 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.177 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.178 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_htY0ij10LAX0ZNP6T_0mHNL", 2022-09-03T06:23:38.178 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.178 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.179 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.179 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.179 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.179 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.180 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.180 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.595637_69.1", 2022-09-03T06:23:38.180 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.180 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:23:38.181 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.595637832Z", 2022-09-03T06:23:38.181 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.181 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.181 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.182 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.182 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.182 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.182 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gtTH1QqAjuxzvv9DiFtpT7B", 2022-09-03T06:23:38.183 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.183 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.183 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.183 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.184 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.184 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.184 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.184 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.185 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.599901_70.1", 2022-09-03T06:23:38.185 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.185 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:23:38.185 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.599901361Z", 2022-09-03T06:23:38.186 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.186 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.186 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.186 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.187 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.187 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.187 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gtTH1QqAjuxzvv9DiFtpT7B", 2022-09-03T06:23:38.187 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.188 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.188 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.188 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.188 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.189 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.189 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.189 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.189 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.601762_71.1", 2022-09-03T06:23:38.190 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.190 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-11", 2022-09-03T06:23:38.190 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.601762485Z", 2022-09-03T06:23:38.190 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.191 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.191 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.191 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.191 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.192 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.192 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_umwMbGorUx0G8JCmxgLXcXQ", 2022-09-03T06:23:38.192 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.192 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.193 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.193 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.193 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.193 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.194 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.194 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.194 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.605809_72.1", 2022-09-03T06:23:38.194 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.195 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-11", 2022-09-03T06:23:38.195 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.605809267Z", 2022-09-03T06:23:38.195 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.195 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.196 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.196 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.196 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.196 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.197 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_umwMbGorUx0G8JCmxgLXcXQ", 2022-09-03T06:23:38.197 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.197 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.197 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.198 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.198 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.198 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.198 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.199 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.199 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.642680_73.1", 2022-09-03T06:23:38.199 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.199 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:23:38.200 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.642680630Z", 2022-09-03T06:23:38.200 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.200 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.200 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.201 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.201 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.201 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.201 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_pT_Dw9zXIA2-9UcP5Gn1GlK", 2022-09-03T06:23:38.202 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.202 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.202 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.202 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.203 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.203 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.203 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.203 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.204 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.645295_74.1", 2022-09-03T06:23:38.204 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.204 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:23:38.204 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.645295120Z", 2022-09-03T06:23:38.205 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.205 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.205 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.205 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.206 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.206 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.206 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_pT_Dw9zXIA2-9UcP5Gn1GlK", 2022-09-03T06:23:38.206 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.207 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.207 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.207 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.207 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.208 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.208 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.208 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.646235_75.1", 2022-09-03T06:23:38.209 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.209 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-12", 2022-09-03T06:23:38.209 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.646235577Z", 2022-09-03T06:23:38.209 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.210 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.210 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.210 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.211 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.211 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ig99b4m7-ssuB2Tw7htsmQp", 2022-09-03T06:23:38.211 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.212 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.212 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.212 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.212 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.213 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.213 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.213 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184356.648716_76.1", 2022-09-03T06:23:38.213 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.214 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-12", 2022-09-03T06:23:38.214 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:36.648716080Z", 2022-09-03T06:23:38.214 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.214 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.215 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.215 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.215 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.216 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ig99b4m7-ssuB2Tw7htsmQp", 2022-09-03T06:23:38.216 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.216 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.217 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.218 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.356235_99.1", 2022-09-03T06:23:38.218 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.218 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:23:38.219 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.356235827Z", 2022-09-03T06:23:38.219 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.219 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.219 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.220 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.220 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.220 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8iK3rbCB7fbjpNaGZ4Lk2f9", 2022-09-03T06:23:38.221 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.221 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.221 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.222 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.222 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.222 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.223 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.364532_100.1", 2022-09-03T06:23:38.223 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.223 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:23:38.223 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.364532891Z", 2022-09-03T06:23:38.224 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.224 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.224 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.224 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.225 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.225 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.225 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8iK3rbCB7fbjpNaGZ4Lk2f9", 2022-09-03T06:23:38.225 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.226 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.226 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.226 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.226 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.227 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.227 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.227 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.227 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.365724_101.1", 2022-09-03T06:23:38.228 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.228 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-13", 2022-09-03T06:23:38.228 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.365724056Z", 2022-09-03T06:23:38.228 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.229 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.229 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.229 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.230 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.230 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_hGY9xi1--Qa5DBRaR1Ev2Td", 2022-09-03T06:23:38.230 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.230 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.231 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.231 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.231 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.231 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.232 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.372032_102.1", 2022-09-03T06:23:38.232 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.233 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-13", 2022-09-03T06:23:38.233 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.372032718Z", 2022-09-03T06:23:38.233 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.233 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.234 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.234 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.234 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.235 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_hGY9xi1--Qa5DBRaR1Ev2Td", 2022-09-03T06:23:38.235 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.235 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.235 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.236 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.236 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.236 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.237 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.237 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.400096_103.1", 2022-09-03T06:23:38.237 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.237 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:23:38.238 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.400096996Z", 2022-09-03T06:23:38.238 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.238 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.238 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.239 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.239 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.239 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.239 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_CLeQ9451sE39DW-o5-rQ9Gb", 2022-09-03T06:23:38.240 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.240 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.240 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.240 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.241 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.241 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.241 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.242 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.406131_104.1", 2022-09-03T06:23:38.242 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.242 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:23:38.242 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.406131833Z", 2022-09-03T06:23:38.243 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.243 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.243 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.243 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.244 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.244 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_CLeQ9451sE39DW-o5-rQ9Gb", 2022-09-03T06:23:38.244 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.245 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.245 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.245 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.246 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.246 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.407235_105.1", 2022-09-03T06:23:38.247 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.247 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-14", 2022-09-03T06:23:38.247 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.407235283Z", 2022-09-03T06:23:38.247 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.248 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.248 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.248 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.249 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.249 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_E3QteIhTh8XKCT60Fs2visZ", 2022-09-03T06:23:38.249 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.250 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.250 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.250 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.250 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.251 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.413053_106.1", 2022-09-03T06:23:38.251 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.252 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-14", 2022-09-03T06:23:38.252 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.413053852Z", 2022-09-03T06:23:38.252 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.252 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.253 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.253 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.253 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.254 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_E3QteIhTh8XKCT60Fs2visZ", 2022-09-03T06:23:38.254 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.254 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.255 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.255 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.255 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.256 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.447727_107.1", 2022-09-03T06:23:38.256 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.256 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:23:38.257 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.447727339Z", 2022-09-03T06:23:38.257 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.257 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.257 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.258 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.258 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.258 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6mYNO_dnwhvNSPaHRDsVWkL", 2022-09-03T06:23:38.259 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.259 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.259 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.260 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.260 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.261 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.450607_108.1", 2022-09-03T06:23:38.261 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.261 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:23:38.262 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.450607653Z", 2022-09-03T06:23:38.262 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.262 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.262 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.263 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.263 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.263 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6mYNO_dnwhvNSPaHRDsVWkL", 2022-09-03T06:23:38.264 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.264 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.264 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.265 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.265 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.266 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.451654_109.1", 2022-09-03T06:23:38.266 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.266 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-15", 2022-09-03T06:23:38.266 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.451654129Z", 2022-09-03T06:23:38.267 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.267 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.267 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.267 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.268 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.268 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_iMvYVCIRH1guRCrQSTbYt62", 2022-09-03T06:23:38.268 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.269 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.269 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.269 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.270 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.271 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184450.454511_110.1", 2022-09-03T06:23:38.271 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.271 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-15", 2022-09-03T06:23:38.272 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:54:10.454511694Z", 2022-09-03T06:23:38.272 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.272 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.272 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.273 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.273 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.273 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_iMvYVCIRH1guRCrQSTbYt62", 2022-09-03T06:23:38.274 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.274 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.274 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.275 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.275 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.276 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184551.125768_119.1", 2022-09-03T06:23:38.276 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.276 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:38.276 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:51.125768074Z", 2022-09-03T06:23:38.276 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.277 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.277 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.277 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.278 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.278 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:38.278 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.279 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.279 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.280 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.280 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.281 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184551.133544_120.1", 2022-09-03T06:23:38.281 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.281 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:38.281 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:51.133544960Z", 2022-09-03T06:23:38.282 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.282 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.282 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.282 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.283 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.283 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:38.283 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.284 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.284 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.284 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.284 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.285 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.285 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184551.134772_121.1", 2022-09-03T06:23:38.286 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.286 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16", 2022-09-03T06:23:38.286 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:51.134772057Z", 2022-09-03T06:23:38.286 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.287 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.287 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.287 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.288 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.288 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yedFU2W-LnAQG28SX6Ff-RY", 2022-09-03T06:23:38.289 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.289 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.289 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.290 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.290 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.290 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.291 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184551.139229_122.1", 2022-09-03T06:23:38.291 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.291 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16", 2022-09-03T06:23:38.291 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:51.139229501Z", 2022-09-03T06:23:38.292 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.292 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.292 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.292 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.293 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.293 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.293 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yedFU2W-LnAQG28SX6Ff-RY", 2022-09-03T06:23:38.293 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.294 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.294 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.294 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.294 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.295 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.295 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.168524_123.1", 2022-09-03T06:23:38.296 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.296 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:38.296 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.168524769Z", 2022-09-03T06:23:38.297 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.297 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.297 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.298 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.298 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.298 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:38.299 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.299 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.299 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.300 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.300 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.301 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.172232_124.1", 2022-09-03T06:23:38.301 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.301 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:38.301 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.172232256Z", 2022-09-03T06:23:38.302 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.302 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.302 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.303 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.303 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.303 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.303 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:38.303 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.304 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.304 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.304 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.304 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.305 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.306 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.306 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.306 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.173517_125.1", 2022-09-03T06:23:38.306 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.307 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17", 2022-09-03T06:23:38.307 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.173517470Z", 2022-09-03T06:23:38.307 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.307 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.308 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.308 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.308 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.308 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.309 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_KGtuntqhz3tvT1e4k36oFKj", 2022-09-03T06:23:38.309 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.309 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.309 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.310 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.310 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.310 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.310 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.311 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.311 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.178102_126.1", 2022-09-03T06:23:38.311 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.311 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17", 2022-09-03T06:23:38.312 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.178102240Z", 2022-09-03T06:23:38.312 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.312 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.312 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.313 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.313 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.313 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.314 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_KGtuntqhz3tvT1e4k36oFKj", 2022-09-03T06:23:38.314 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.314 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.315 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.315 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.315 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.315 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.316 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.316 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.316 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.215736_127.1", 2022-09-03T06:23:38.316 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.317 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:38.317 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.215736666Z", 2022-09-03T06:23:38.317 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.317 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.318 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.318 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.318 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.319 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.319 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:38.319 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.319 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.320 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.320 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.320 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.320 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.320 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.321 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.321 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.219063_128.1", 2022-09-03T06:23:38.321 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.321 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:38.322 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.219063433Z", 2022-09-03T06:23:38.323 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.323 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.323 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.323 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.324 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.324 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.324 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:38.324 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.325 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.325 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.325 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.325 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.326 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.326 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.326 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.326 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.220690_129.1", 2022-09-03T06:23:38.327 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.327 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18", 2022-09-03T06:23:38.327 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.220690992Z", 2022-09-03T06:23:38.327 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.328 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.328 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.328 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.328 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.329 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.329 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm651SSphsn2Jx7Kwrq9oNR", 2022-09-03T06:23:38.329 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.329 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.330 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.330 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.330 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.331 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.331 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.332 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.332 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.226118_130.1", 2022-09-03T06:23:38.332 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.332 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18", 2022-09-03T06:23:38.333 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.226118188Z", 2022-09-03T06:23:38.333 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.333 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.333 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.334 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.334 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.334 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.334 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm651SSphsn2Jx7Kwrq9oNR", 2022-09-03T06:23:38.335 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.335 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.335 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.335 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.336 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.336 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.336 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.336 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.337 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.250228_131.1", 2022-09-03T06:23:38.337 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.337 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:38.337 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.250228835Z", 2022-09-03T06:23:38.338 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.338 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.338 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:38.339 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.339 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.340 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.340 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:38.340 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.340 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.341 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.341 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.341 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.341 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.342 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.342 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.342 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.253497_132.1", 2022-09-03T06:23:38.342 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.343 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:23:38.343 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.253497827Z", 2022-09-03T06:23:38.343 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.343 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.344 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:38.344 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.344 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.344 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.345 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tlAG1oOShlH1GAhmDRSNAx1", 2022-09-03T06:23:38.345 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.345 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.345 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.346 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.346 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.346 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.346 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.347 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.347 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.266389_133.1", 2022-09-03T06:23:38.347 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.347 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:38.348 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.266389718Z", 2022-09-03T06:23:38.348 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.349 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.349 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:38.349 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.350 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.350 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.350 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:38.350 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.351 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.351 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.351 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.351 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.352 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.352 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.352 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.352 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.270858_134.1", 2022-09-03T06:23:38.353 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.353 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:23:38.353 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.270858165Z", 2022-09-03T06:23:38.353 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.354 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.354 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:38.354 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.354 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.355 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.355 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Yo8bYvs5mhVUHDN8n82ldcr", 2022-09-03T06:23:38.355 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.355 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.356 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.356 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.356 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.357 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.357 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.357 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.358 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.289855_135.1", 2022-09-03T06:23:38.358 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.358 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:38.358 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.289855850Z", 2022-09-03T06:23:38.359 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.359 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.359 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:38.359 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.360 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.360 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.360 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:38.360 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.361 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.361 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.361 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.361 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.362 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.362 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.362 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184552.296862_136.1", 2022-09-03T06:23:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:23:38.363 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:55:52.296862507Z", 2022-09-03T06:23:38.364 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.364 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.364 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:38.364 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.365 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.365 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.366 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_gmJdIk_1i8IwfYEeElijsOb", 2022-09-03T06:23:38.366 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.366 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.366 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.367 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.367 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.367 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.367 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.368 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.368 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.168986_143.1", 2022-09-03T06:23:38.368 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.368 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:38.369 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.168986972Z", 2022-09-03T06:23:38.369 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.369 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.369 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.370 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.370 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.370 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.371 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:38.371 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.371 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.371 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.372 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.372 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.372 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.372 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.373 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.373 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.174598_144.1", 2022-09-03T06:23:38.373 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.373 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:38.374 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.174598788Z", 2022-09-03T06:23:38.374 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.375 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.375 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.375 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.375 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.376 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.376 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:38.376 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.376 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.377 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.377 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.377 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.377 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.378 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.378 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.378 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.176005_145.1", 2022-09-03T06:23:38.378 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.379 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19", 2022-09-03T06:23:38.379 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.176005675Z", 2022-09-03T06:23:38.379 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.380 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.380 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.380 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.380 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.381 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.381 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yDXruVlqT4Xsg8Al8Plz6TA", 2022-09-03T06:23:38.381 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.381 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.382 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.382 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.382 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.382 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.383 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.383 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.383 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.179021_146.1", 2022-09-03T06:23:38.384 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.384 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19", 2022-09-03T06:23:38.384 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.179021577Z", 2022-09-03T06:23:38.385 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.385 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.385 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.385 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.385 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.386 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.386 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yDXruVlqT4Xsg8Al8Plz6TA", 2022-09-03T06:23:38.386 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.387 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.387 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.387 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.387 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.387 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.388 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.388 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.388 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.194719_147.1", 2022-09-03T06:23:38.388 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.389 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:38.389 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.194719304Z", 2022-09-03T06:23:38.389 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.389 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.390 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:38.390 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.390 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.390 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.391 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:38.391 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.391 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.391 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.392 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.392 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.392 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.392 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.393 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.393 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662184997.201443_148.1", 2022-09-03T06:23:38.393 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.393 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:23:38.394 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:03:17.201443123Z", 2022-09-03T06:23:38.394 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.394 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.394 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:38.395 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.395 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.395 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.395 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gm8NdX-Cco2xxS0RryuIAg1", 2022-09-03T06:23:38.396 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.396 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.396 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.396 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.397 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.397 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.397 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.397 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.398 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185259.164178_150.1", 2022-09-03T06:23:38.398 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.398 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:38.398 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.164178377Z", 2022-09-03T06:23:38.399 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.399 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.399 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.399 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.400 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.400 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.400 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.400 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.401 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.401 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.401 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.401 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.402 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.402 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.402 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.402 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185259.166746_154.1", 2022-09-03T06:23:38.403 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.403 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:38.403 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.166746306Z", 2022-09-03T06:23:38.403 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.404 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.404 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.404 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.404 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.405 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.405 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.405 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.405 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.406 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.406 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.406 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.406 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.407 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.407 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.407 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185259.167637_155.1", 2022-09-03T06:23:38.407 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.408 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20", 2022-09-03T06:23:38.408 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.167637326Z", 2022-09-03T06:23:38.408 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.408 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.409 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.409 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.409 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.409 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.410 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZKafDPgftyjUAayY_VUswW5", 2022-09-03T06:23:38.410 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.410 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.410 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.411 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.411 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.411 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.411 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.412 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.412 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185259.169934_156.1", 2022-09-03T06:23:38.412 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.412 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20", 2022-09-03T06:23:38.413 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:07:39.169934736Z", 2022-09-03T06:23:38.413 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.413 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.413 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.414 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.414 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.414 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.414 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZKafDPgftyjUAayY_VUswW5", 2022-09-03T06:23:38.415 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.415 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.415 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.415 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.416 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.416 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.416 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.416 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.417 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185351.815540_157.1", 2022-09-03T06:23:38.417 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.417 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:38.417 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:09:11.815540177Z", 2022-09-03T06:23:38.418 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.418 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.418 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.418 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.419 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.419 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.419 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.419 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.420 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.420 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.420 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.420 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.421 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.421 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.421 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.422 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185351.819576_158.1", 2022-09-03T06:23:38.422 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.423 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:38.423 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:09:11.819576411Z", 2022-09-03T06:23:38.424 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.424 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.425 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.425 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.425 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.426 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.426 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.427 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.427 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.428 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.428 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.429 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.429 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.430 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.430 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.430 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185415.254080_165.1", 2022-09-03T06:23:38.431 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.431 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:38.432 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:10:15.254080988Z", 2022-09-03T06:23:38.432 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.433 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.433 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.434 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.434 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.435 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.435 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.436 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:38.436 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.437 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.437 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.438 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.438 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.439 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.439 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185415.257852_166.1", 2022-09-03T06:23:38.440 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.440 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:23:38.441 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:10:15.257852595Z", 2022-09-03T06:23:38.441 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.441 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.442 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.442 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.443 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.443 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.443 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ygOgt33PCtyn0WDsBG0mn0p", 2022-09-03T06:23:38.444 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:38.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.445 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.445 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.445 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.446 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.446 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.446 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.447 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.140367_167.1", 2022-09-03T06:23:38.447 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.447 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:38.448 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.140367923Z", 2022-09-03T06:23:38.448 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.449 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.449 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.450 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.450 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.451 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.451 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:38.452 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.453 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.453 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.454 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.454 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.455 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.455 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.455 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.143952_168.1", 2022-09-03T06:23:38.456 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.456 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:38.457 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.143952074Z", 2022-09-03T06:23:38.457 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.457 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.457 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.458 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.458 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.459 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:38.459 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.459 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.460 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.460 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.460 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.460 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.461 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.145226_169.1", 2022-09-03T06:23:38.461 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.462 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21", 2022-09-03T06:23:38.462 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.145226679Z", 2022-09-03T06:23:38.462 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.462 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.463 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.463 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.463 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.464 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WXbrqJZWiL3N6i3X5RrSf_L", 2022-09-03T06:23:38.464 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.464 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.464 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.465 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.465 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.465 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.466 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.466 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.466 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.148908_170.1", 2022-09-03T06:23:38.467 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.467 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21", 2022-09-03T06:23:38.467 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.148908181Z", 2022-09-03T06:23:38.467 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.468 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.468 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.468 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.468 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.469 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.469 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WXbrqJZWiL3N6i3X5RrSf_L", 2022-09-03T06:23:38.469 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.469 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.470 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.470 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.470 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.470 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.471 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.471 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.471 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.171153_171.1", 2022-09-03T06:23:38.471 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.472 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:38.472 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.171153368Z", 2022-09-03T06:23:38.472 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.472 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.473 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.473 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.473 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.473 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.474 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:38.474 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.474 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.474 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.475 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.475 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.475 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.475 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.476 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.476 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.174859_172.1", 2022-09-03T06:23:38.476 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.476 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:38.477 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.174859152Z", 2022-09-03T06:23:38.477 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.477 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.477 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.478 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.478 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.478 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:38.479 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.479 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.479 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.480 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.480 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.481 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.176058_173.1", 2022-09-03T06:23:38.481 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.481 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22", 2022-09-03T06:23:38.481 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.176058283Z", 2022-09-03T06:23:38.482 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.482 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.482 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.482 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.483 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.483 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_4TVYbHf2FAhumE819lO37xf", 2022-09-03T06:23:38.484 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.484 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.484 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.484 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.485 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.485 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.486 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.180790_174.1", 2022-09-03T06:23:38.486 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.486 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22", 2022-09-03T06:23:38.486 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.180790715Z", 2022-09-03T06:23:38.487 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.487 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.487 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.487 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.488 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.488 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_4TVYbHf2FAhumE819lO37xf", 2022-09-03T06:23:38.488 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.489 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.489 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.489 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.490 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.490 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.221697_175.1", 2022-09-03T06:23:38.491 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.491 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:38.491 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.221697751Z", 2022-09-03T06:23:38.491 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.492 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.492 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.492 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.493 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.493 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:38.493 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.494 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.494 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.494 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.494 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.495 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.229951_176.1", 2022-09-03T06:23:38.495 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.496 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:38.496 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.229951839Z", 2022-09-03T06:23:38.496 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.496 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.497 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.497 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.497 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.498 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:38.498 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.498 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.499 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.499 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.499 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.500 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.237269_177.1", 2022-09-03T06:23:38.500 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.500 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23", 2022-09-03T06:23:38.501 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.237269112Z", 2022-09-03T06:23:38.501 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.501 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.501 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.502 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.502 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.502 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_SUW5WcOi0sbRMVG7llLAyro", 2022-09-03T06:23:38.503 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.503 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.503 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.504 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.504 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.505 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.248604_178.1", 2022-09-03T06:23:38.505 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.505 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23", 2022-09-03T06:23:38.505 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.248604519Z", 2022-09-03T06:23:38.506 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.506 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.506 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.506 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.507 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.507 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_SUW5WcOi0sbRMVG7llLAyro", 2022-09-03T06:23:38.507 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.508 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.508 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.508 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.509 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.509 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.289134_179.1", 2022-09-03T06:23:38.510 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.510 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:38.510 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.289134872Z", 2022-09-03T06:23:38.510 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.511 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.511 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:38.511 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.512 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.512 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:38.512 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.513 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.513 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.513 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.513 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.514 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.292023_180.1", 2022-09-03T06:23:38.514 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.515 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:23:38.515 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.292023076Z", 2022-09-03T06:23:38.515 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.515 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.516 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:38.516 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.516 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.517 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_bRXqvxum_g9BGPx0rsbonZt", 2022-09-03T06:23:38.517 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.517 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.518 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.518 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.518 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.519 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.305537_181.1", 2022-09-03T06:23:38.519 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.519 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:38.520 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.305537948Z", 2022-09-03T06:23:38.520 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.520 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.520 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:38.521 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.521 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.521 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:38.522 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.522 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.522 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.523 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.523 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.524 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.308323_182.1", 2022-09-03T06:23:38.524 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.524 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:23:38.524 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.308323992Z", 2022-09-03T06:23:38.525 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.525 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.525 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:38.525 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.526 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.526 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_WcYEuiMa0enLs2uvJT4Dxe-", 2022-09-03T06:23:38.526 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.527 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.527 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.527 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.527 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.528 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.323485_183.1", 2022-09-03T06:23:38.528 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.529 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:38.529 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.323485482Z", 2022-09-03T06:23:38.529 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.529 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.530 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:38.530 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.530 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.531 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:38.531 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.531 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.532 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.532 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.532 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.533 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185472.326245_184.1", 2022-09-03T06:23:38.533 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.533 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:23:38.534 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:11:12.326245244Z", 2022-09-03T06:23:38.534 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.534 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.534 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:38.535 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.535 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.535 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.535 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_uxx3bstSLjatDem5FzGACfu", 2022-09-03T06:23:38.536 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.536 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.536 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.537 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.537 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.538 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.128610_187.1", 2022-09-03T06:23:38.538 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.538 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:38.538 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.128610110Z", 2022-09-03T06:23:38.539 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.539 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.539 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.539 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.540 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.540 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.540 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.541 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.541 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.542 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.542 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.543 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.132654_188.1", 2022-09-03T06:23:38.544 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.544 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:38.545 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.132654659Z", 2022-09-03T06:23:38.545 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.545 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.545 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.546 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.546 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.546 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.547 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.547 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.547 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.548 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.548 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.549 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.133913_189.1", 2022-09-03T06:23:38.549 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.549 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24", 2022-09-03T06:23:38.549 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.133913786Z", 2022-09-03T06:23:38.550 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.550 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.550 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.551 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.551 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.552 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_flubJt4QosOvedZNv50fpC6", 2022-09-03T06:23:38.552 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.552 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.553 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.553 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.553 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.554 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.136736_190.1", 2022-09-03T06:23:38.554 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.554 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24", 2022-09-03T06:23:38.555 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.136736736Z", 2022-09-03T06:23:38.555 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.555 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.555 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.556 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.556 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.556 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_flubJt4QosOvedZNv50fpC6", 2022-09-03T06:23:38.557 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.557 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.558 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.558 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.558 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.559 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.158636_191.1", 2022-09-03T06:23:38.559 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.559 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:38.560 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.158636011Z", 2022-09-03T06:23:38.560 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.560 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.560 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.561 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.561 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.562 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.562 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.562 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.563 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.563 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.563 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.564 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.162317_192.1", 2022-09-03T06:23:38.565 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.565 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:38.565 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.162317814Z", 2022-09-03T06:23:38.566 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.566 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.566 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.566 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.567 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.567 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.567 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.568 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.568 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.568 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.569 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.570 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.163511_193.1", 2022-09-03T06:23:38.570 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.570 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25", 2022-09-03T06:23:38.570 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.163511615Z", 2022-09-03T06:23:38.571 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.571 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.571 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.572 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.572 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.572 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tB02hvaj6C_E_OnwBMY8ywL", 2022-09-03T06:23:38.573 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.573 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.574 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.574 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.574 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.575 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.166349_194.1", 2022-09-03T06:23:38.575 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.576 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25", 2022-09-03T06:23:38.576 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.166349608Z", 2022-09-03T06:23:38.576 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.576 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.577 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.577 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.577 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.578 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_tB02hvaj6C_E_OnwBMY8ywL", 2022-09-03T06:23:38.578 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.578 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.579 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.579 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.579 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.580 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.203237_195.1", 2022-09-03T06:23:38.580 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.581 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:38.581 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.203237512Z", 2022-09-03T06:23:38.581 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.582 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.582 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.582 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.583 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.583 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.583 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.584 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.584 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.584 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.585 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.585 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.585 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.206000_196.1", 2022-09-03T06:23:38.586 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.586 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:38.586 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.206000308Z", 2022-09-03T06:23:38.586 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.587 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.587 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.587 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.588 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.588 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.588 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.589 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.589 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.589 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.589 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.590 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.207006_197.1", 2022-09-03T06:23:38.590 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.591 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26", 2022-09-03T06:23:38.591 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.207006797Z", 2022-09-03T06:23:38.591 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.591 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.592 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.592 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.592 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.593 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_iK1lRBoYvVYLawtBAmt3Urm", 2022-09-03T06:23:38.593 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.593 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.593 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.594 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.594 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.594 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.594 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.595 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.595 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.211392_198.1", 2022-09-03T06:23:38.595 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.595 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26", 2022-09-03T06:23:38.596 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.211392135Z", 2022-09-03T06:23:38.596 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.596 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.596 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.597 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.597 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.597 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_iK1lRBoYvVYLawtBAmt3Urm", 2022-09-03T06:23:38.597 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.598 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.598 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.599 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.599 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.599 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.599 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.245835_199.1", 2022-09-03T06:23:38.600 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.600 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:38.600 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.245835749Z", 2022-09-03T06:23:38.600 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.601 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.601 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.601 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.602 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.602 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.602 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.603 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.603 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.603 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.604 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.604 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.250338_200.1", 2022-09-03T06:23:38.605 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.605 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:38.605 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.250338287Z", 2022-09-03T06:23:38.605 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.606 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.606 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.606 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.607 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.607 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.607 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.608 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.608 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.608 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.608 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.609 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.609 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.609 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.265702_201.1", 2022-09-03T06:23:38.609 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.610 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:38.610 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.265702416Z", 2022-09-03T06:23:38.610 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.611 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.611 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.611 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.612 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.612 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.612 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:38.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.613 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.613 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.613 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.613 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.614 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.614 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.614 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.270146_202.1", 2022-09-03T06:23:38.614 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.615 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:23:38.615 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.270146860Z", 2022-09-03T06:23:38.615 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.615 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.616 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.616 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.616 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.617 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2lz_VcGhI55LY8DBtFyh7Wg", 2022-09-03T06:23:38.617 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:38.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.617 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.618 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.618 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.618 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.619 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.283449_203.1", 2022-09-03T06:23:38.619 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.619 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:38.620 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.283449642Z", 2022-09-03T06:23:38.620 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.620 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.620 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.621 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.621 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.621 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.622 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.622 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.622 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.624 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.286127_204.1", 2022-09-03T06:23:38.624 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.624 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:38.625 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.286127335Z", 2022-09-03T06:23:38.625 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.625 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.625 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.626 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.626 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.626 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.627 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.627 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.627 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.628 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.628 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.629 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.305370_205.1", 2022-09-03T06:23:38.629 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.629 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:38.629 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.305370975Z", 2022-09-03T06:23:38.629 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.630 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.630 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.630 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.631 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.631 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.631 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:38.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.632 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.632 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.632 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.633 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.633 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.633 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.312602_206.1", 2022-09-03T06:23:38.634 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.634 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:23:38.634 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.312602358Z", 2022-09-03T06:23:38.634 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.635 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.635 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.635 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.636 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.636 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZZP7tea0o9AncGV_d6p8SPO", 2022-09-03T06:23:38.636 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:38.636 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.637 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.637 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.637 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.637 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.638 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.638 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.638 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.344134_207.1", 2022-09-03T06:23:38.638 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.639 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:38.639 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.344134550Z", 2022-09-03T06:23:38.639 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.639 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.640 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.640 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.640 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.641 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.641 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.641 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.641 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.642 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.642 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.642 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.643 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.643 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.355110_208.1", 2022-09-03T06:23:38.643 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.643 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:38.644 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.355110727Z", 2022-09-03T06:23:38.644 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.644 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.644 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.645 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.645 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.645 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.645 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.646 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.646 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.646 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.647 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.647 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.647 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.647 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.648 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.382118_209.1", 2022-09-03T06:23:38.648 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.648 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:38.648 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.382118897Z", 2022-09-03T06:23:38.649 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.649 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.649 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.649 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.650 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.650 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.650 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:38.651 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.651 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.651 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.651 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.651 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.652 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.652 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.652 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185640.386563_210.1", 2022-09-03T06:23:38.652 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.653 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:23:38.653 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:14:00.386563904Z", 2022-09-03T06:23:38.653 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.653 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.654 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.654 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.654 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.654 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.655 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Q75duLBm1Wt_VT0MK9Sr-DI", 2022-09-03T06:23:38.655 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 3 2022-09-03T06:23:38.655 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.655 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.656 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.656 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.656 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.657 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.657 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.730358_216.1", 2022-09-03T06:23:38.657 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.658 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:38.658 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.730358281Z", 2022-09-03T06:23:38.658 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.658 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.659 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.659 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.659 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.659 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.660 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:38.660 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.660 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.660 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.661 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.661 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.661 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.661 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.662 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.662 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.735815_217.1", 2022-09-03T06:23:38.662 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.662 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:38.663 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.735815559Z", 2022-09-03T06:23:38.663 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.663 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.663 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.664 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.664 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.664 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.664 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:38.665 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.665 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.665 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.665 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.666 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.666 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.666 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.666 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.667 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.736742_218.1", 2022-09-03T06:23:38.667 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.667 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27", 2022-09-03T06:23:38.667 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.736742093Z", 2022-09-03T06:23:38.668 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.668 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.668 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.669 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.669 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.669 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.670 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_jZ9wkZPqzR2GWEIEfGTWCSz", 2022-09-03T06:23:38.670 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.671 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.671 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.671 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.672 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.672 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.672 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.673 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.673 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.742821_219.1", 2022-09-03T06:23:38.674 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.674 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27", 2022-09-03T06:23:38.674 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.742821039Z", 2022-09-03T06:23:38.675 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.675 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.676 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.676 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.677 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.677 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.677 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_jZ9wkZPqzR2GWEIEfGTWCSz", 2022-09-03T06:23:38.678 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.678 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.679 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.679 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.679 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.680 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.680 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.681 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.681 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.774385_220.1", 2022-09-03T06:23:38.681 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.682 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:38.682 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.774385987Z", 2022-09-03T06:23:38.683 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.683 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.683 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.684 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.684 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.684 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.685 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:38.685 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.685 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.685 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.686 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.686 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.686 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.686 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.687 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.687 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.778620_221.1", 2022-09-03T06:23:38.687 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.687 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:38.688 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.778620417Z", 2022-09-03T06:23:38.688 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.688 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.688 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.689 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.689 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.689 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.689 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:38.690 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.690 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.690 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.690 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.691 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.691 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.691 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.692 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.692 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.779591_222.1", 2022-09-03T06:23:38.692 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.693 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28", 2022-09-03T06:23:38.693 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.779591267Z", 2022-09-03T06:23:38.693 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.693 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.694 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.694 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.694 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.694 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.695 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ISaB31FGT1RJh5d1WEGeO7S", 2022-09-03T06:23:38.695 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.695 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.695 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.696 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.696 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.696 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.696 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.697 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.697 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.784101_223.1", 2022-09-03T06:23:38.697 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.697 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28", 2022-09-03T06:23:38.698 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.784101743Z", 2022-09-03T06:23:38.698 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.698 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.699 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.699 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.700 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.700 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ISaB31FGT1RJh5d1WEGeO7S", 2022-09-03T06:23:38.700 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.700 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.701 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.701 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.701 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.701 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.702 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.702 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.702 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.842565_224.1", 2022-09-03T06:23:38.702 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.703 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:38.703 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.842565677Z", 2022-09-03T06:23:38.703 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.703 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.704 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.704 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.704 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.704 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.705 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:38.705 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.705 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.705 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.706 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.706 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.706 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.706 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.707 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.707 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.848053_225.1", 2022-09-03T06:23:38.707 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.708 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:38.708 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.848053689Z", 2022-09-03T06:23:38.708 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.709 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.709 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.709 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.710 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.710 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.710 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:38.710 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.711 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.711 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.711 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.711 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.712 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.712 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.712 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.712 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.850209_226.1", 2022-09-03T06:23:38.713 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.713 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29", 2022-09-03T06:23:38.713 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.850209520Z", 2022-09-03T06:23:38.713 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.714 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.714 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.714 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.714 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.715 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.715 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_JaA8RlJaRxOPPcYHu0FpEzg", 2022-09-03T06:23:38.715 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.715 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.716 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.716 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.716 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.717 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.717 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.717 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.718 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.855964_227.1", 2022-09-03T06:23:38.718 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.718 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29", 2022-09-03T06:23:38.718 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.855964453Z", 2022-09-03T06:23:38.719 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.719 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.719 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.719 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.720 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.720 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.720 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_JaA8RlJaRxOPPcYHu0FpEzg", 2022-09-03T06:23:38.720 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.721 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.721 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.721 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.721 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.722 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.722 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.722 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.722 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.894939_228.1", 2022-09-03T06:23:38.723 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.723 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:38.723 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.894939418Z", 2022-09-03T06:23:38.723 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.724 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.724 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:38.724 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.724 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.725 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.725 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:38.725 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.726 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.726 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.726 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.726 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.727 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.727 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.727 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.727 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.897692_229.1", 2022-09-03T06:23:38.728 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.728 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27", 2022-09-03T06:23:38.728 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.897692116Z", 2022-09-03T06:23:38.728 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.729 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.729 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:38.729 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.729 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.730 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.730 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_67tv2s2U16AUqogEBXlPZCM", 2022-09-03T06:23:38.730 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.730 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.731 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.731 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.731 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.731 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.732 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.732 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.732 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.921086_230.1", 2022-09-03T06:23:38.732 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.733 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:38.733 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.921086404Z", 2022-09-03T06:23:38.733 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.733 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.734 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:38.734 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.734 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.734 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.735 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:38.735 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.735 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.736 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.736 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.736 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.736 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.737 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.737 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.737 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185755.923839_231.1", 2022-09-03T06:23:38.737 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.738 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28", 2022-09-03T06:23:38.738 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:55.923839471Z", 2022-09-03T06:23:38.738 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.739 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.739 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:38.739 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.739 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.740 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.740 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8fO6ZdNKBIcvHZyM5f9Zaax", 2022-09-03T06:23:38.740 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.740 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.741 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.741 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.741 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.741 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.742 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.742 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.742 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.001910_232.1", 2022-09-03T06:23:38.742 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.743 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:38.743 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.001910722Z", 2022-09-03T06:23:38.743 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.743 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.744 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:38.744 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.744 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.744 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.745 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:38.745 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.745 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.745 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.746 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.746 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.746 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.746 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.747 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.747 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.011478_233.1", 2022-09-03T06:23:38.747 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.747 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29", 2022-09-03T06:23:38.748 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.011478475Z", 2022-09-03T06:23:38.748 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.748 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.748 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:38.749 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.749 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.749 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.749 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3wO1-nyGQPtpTzk_TBip32N", 2022-09-03T06:23:38.750 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.750 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.750 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.750 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.751 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.751 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.751 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.751 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.752 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.059572_234.1", 2022-09-03T06:23:38.752 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.752 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:38.752 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.059572375Z", 2022-09-03T06:23:38.753 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.753 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.753 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.753 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.754 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.754 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.754 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:38.754 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.755 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.755 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.755 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.755 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.756 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.756 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.756 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.756 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.063873_235.1", 2022-09-03T06:23:38.757 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.757 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:38.757 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.063873838Z", 2022-09-03T06:23:38.757 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.758 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.758 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.758 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.758 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.759 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.759 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:38.759 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.760 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.760 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.760 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.760 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.761 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.761 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.761 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.064807_236.1", 2022-09-03T06:23:38.761 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.762 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30", 2022-09-03T06:23:38.762 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.064807654Z", 2022-09-03T06:23:38.762 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.762 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.763 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.763 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.763 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.763 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.764 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_emDKTj1MXu1EbBl7W-BZDCI", 2022-09-03T06:23:38.764 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.764 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.764 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.765 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.765 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.765 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.765 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.766 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.766 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.069230_237.1", 2022-09-03T06:23:38.766 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.766 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30", 2022-09-03T06:23:38.767 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.069230157Z", 2022-09-03T06:23:38.767 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.767 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.767 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.768 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.768 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.768 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.768 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_emDKTj1MXu1EbBl7W-BZDCI", 2022-09-03T06:23:38.769 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.769 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.769 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.769 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.770 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.770 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.770 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.770 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.771 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.098699_238.1", 2022-09-03T06:23:38.771 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.771 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:38.772 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.098699181Z", 2022-09-03T06:23:38.772 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.772 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.772 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.772 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.773 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.773 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.773 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:38.774 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.774 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.774 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.774 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.775 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.775 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.775 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.775 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.776 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.104362_239.1", 2022-09-03T06:23:38.776 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.776 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:38.776 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.104362288Z", 2022-09-03T06:23:38.777 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.777 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.777 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.778 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.778 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.778 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.778 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:38.779 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.779 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.779 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.779 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.780 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.780 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.780 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.780 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.781 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.105238_240.1", 2022-09-03T06:23:38.781 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.781 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31", 2022-09-03T06:23:38.781 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.105238360Z", 2022-09-03T06:23:38.781 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.782 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.782 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.782 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.783 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.783 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.783 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Fx_iwLRI89zV-Ovfojlsmn_", 2022-09-03T06:23:38.783 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.784 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.784 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.784 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.784 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.785 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.785 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.785 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.785 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.109389_241.1", 2022-09-03T06:23:38.786 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.786 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31", 2022-09-03T06:23:38.786 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.109389408Z", 2022-09-03T06:23:38.786 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.787 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.787 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.787 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.787 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.788 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.788 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Fx_iwLRI89zV-Ovfojlsmn_", 2022-09-03T06:23:38.788 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.788 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.789 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.789 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.789 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.789 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.790 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.790 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.790 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.146357_242.1", 2022-09-03T06:23:38.790 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.791 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:38.791 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.146357921Z", 2022-09-03T06:23:38.791 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.791 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.792 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.792 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.792 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.792 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.793 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:38.793 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.793 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.793 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.794 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.794 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.794 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.794 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.795 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.795 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.149385_243.1", 2022-09-03T06:23:38.795 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.795 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:38.796 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.149385163Z", 2022-09-03T06:23:38.796 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.796 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.796 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.797 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.797 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.797 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.797 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:38.798 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.798 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.798 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.799 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.799 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.799 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.799 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.800 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.800 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.150696_244.1", 2022-09-03T06:23:38.800 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.800 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32", 2022-09-03T06:23:38.800 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.150696281Z", 2022-09-03T06:23:38.801 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.801 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.801 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.802 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.802 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.802 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.802 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_VJHe0Zrz6-lomZaJx40Mq8S", 2022-09-03T06:23:38.803 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.803 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.803 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.803 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.804 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.804 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.804 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.804 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.805 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.159044_245.1", 2022-09-03T06:23:38.805 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.805 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32", 2022-09-03T06:23:38.805 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.159044634Z", 2022-09-03T06:23:38.806 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.806 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.806 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.806 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.807 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.807 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.807 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_VJHe0Zrz6-lomZaJx40Mq8S", 2022-09-03T06:23:38.807 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.808 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.808 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.808 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.808 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.809 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.809 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.809 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.809 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.221962_246.1", 2022-09-03T06:23:38.810 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.810 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:38.810 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.221962483Z", 2022-09-03T06:23:38.810 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.811 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.811 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:38.811 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.811 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.812 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.812 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:38.812 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.812 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.813 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.813 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.813 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.813 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.814 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.814 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.814 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.225097_247.1", 2022-09-03T06:23:38.814 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.815 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30", 2022-09-03T06:23:38.815 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.225097472Z", 2022-09-03T06:23:38.815 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.815 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.816 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:38.816 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.816 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.816 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.817 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_kLepcdBDS4cpQi0GyavL59h", 2022-09-03T06:23:38.817 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.817 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.817 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.818 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.818 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.818 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.818 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.819 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.819 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.324010_248.1", 2022-09-03T06:23:38.819 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.819 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:38.820 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.324010891Z", 2022-09-03T06:23:38.820 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.820 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.820 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:38.821 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.821 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.821 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.821 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:38.822 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.822 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.822 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.822 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.823 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.823 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.823 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.823 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.824 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.326610_249.1", 2022-09-03T06:23:38.824 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.824 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31", 2022-09-03T06:23:38.824 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.326610391Z", 2022-09-03T06:23:38.825 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.825 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.825 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:38.825 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.826 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.826 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.826 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3c7_jX6p4SxhCxne7Glrt0j", 2022-09-03T06:23:38.827 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.827 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.827 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.827 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.828 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.828 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.828 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.828 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.829 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.388152_250.1", 2022-09-03T06:23:38.829 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.829 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:38.829 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.388152023Z", 2022-09-03T06:23:38.830 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.830 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.830 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:38.830 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.831 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.831 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.831 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:38.831 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.832 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.832 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.832 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.832 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.832 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.833 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.833 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.833 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.390795_251.1", 2022-09-03T06:23:38.833 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.834 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32", 2022-09-03T06:23:38.834 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.390795101Z", 2022-09-03T06:23:38.834 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.834 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.835 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:38.835 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.835 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.835 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.836 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_1buAshUbeIlXoh7hyEX_2fY", 2022-09-03T06:23:38.836 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.836 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.836 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.837 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.837 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.837 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.837 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.838 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.838 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.428775_252.1", 2022-09-03T06:23:38.838 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.839 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:38.839 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.428775853Z", 2022-09-03T06:23:38.839 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.840 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.840 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.840 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.840 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.841 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.841 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:38.841 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.841 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.842 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.842 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.842 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.842 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.843 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.843 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.843 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.432176_253.1", 2022-09-03T06:23:38.843 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.844 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:38.844 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.432176035Z", 2022-09-03T06:23:38.844 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.844 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.845 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.845 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.845 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.845 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.846 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:38.846 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.846 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.847 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.847 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.847 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.847 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.848 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.848 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.848 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.433149_254.1", 2022-09-03T06:23:38.848 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.849 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33", 2022-09-03T06:23:38.849 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.433149170Z", 2022-09-03T06:23:38.849 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.849 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.850 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.850 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.850 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.850 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.851 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_w0QD1jE_b482WNZp5Lj2ky4", 2022-09-03T06:23:38.851 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.851 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.851 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.852 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.852 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.852 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.853 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.853 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.853 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.435497_255.1", 2022-09-03T06:23:38.853 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.854 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33", 2022-09-03T06:23:38.854 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.435497913Z", 2022-09-03T06:23:38.854 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.854 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.855 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.855 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.855 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.855 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.856 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_w0QD1jE_b482WNZp5Lj2ky4", 2022-09-03T06:23:38.856 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.856 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.856 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.857 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.857 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.857 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.857 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.858 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.858 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.463906_256.1", 2022-09-03T06:23:38.858 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.858 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:38.859 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.463906896Z", 2022-09-03T06:23:38.859 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.859 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.859 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.860 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.860 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.860 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.860 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:38.861 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.861 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.861 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.861 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.862 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.862 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.862 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.862 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.863 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.466439_257.1", 2022-09-03T06:23:38.863 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.863 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:38.863 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.466439294Z", 2022-09-03T06:23:38.864 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.864 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.864 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.864 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.865 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.865 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.865 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:38.865 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.866 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.866 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.866 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.866 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.867 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.867 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.867 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.867 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.467396_258.1", 2022-09-03T06:23:38.868 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.868 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34", 2022-09-03T06:23:38.868 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.467396448Z", 2022-09-03T06:23:38.869 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.869 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.869 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.869 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.870 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.870 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.870 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G2EFJYOe8FOdjOCj6aBraUR", 2022-09-03T06:23:38.871 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.871 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.871 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.871 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.872 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.872 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.872 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.872 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.873 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.471551_259.1", 2022-09-03T06:23:38.873 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.873 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34", 2022-09-03T06:23:38.873 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.471551488Z", 2022-09-03T06:23:38.874 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.874 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.874 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.874 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.875 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.875 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.875 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G2EFJYOe8FOdjOCj6aBraUR", 2022-09-03T06:23:38.875 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.876 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.876 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.876 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.877 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.877 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.877 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.877 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.878 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.517947_260.1", 2022-09-03T06:23:38.878 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.878 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:38.878 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.517947959Z", 2022-09-03T06:23:38.879 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.879 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.879 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.879 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.880 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.880 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.880 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:38.880 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.881 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.881 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.881 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.881 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.882 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.882 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.882 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.883 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.522051_261.1", 2022-09-03T06:23:38.883 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.883 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:38.883 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.522051544Z", 2022-09-03T06:23:38.884 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.884 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.884 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.884 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.884 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.885 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.885 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:38.885 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.885 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.886 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.886 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.886 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.887 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.887 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.887 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.887 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.524288_262.1", 2022-09-03T06:23:38.888 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.888 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35", 2022-09-03T06:23:38.888 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.524288626Z", 2022-09-03T06:23:38.888 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.888 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.889 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.889 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.889 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.889 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.890 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G8oh7TOeWyD2_FvC8AIXccM", 2022-09-03T06:23:38.890 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.890 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.890 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.891 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.891 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.891 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.891 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.892 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.892 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.526709_263.1", 2022-09-03T06:23:38.892 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.892 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35", 2022-09-03T06:23:38.893 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.526709154Z", 2022-09-03T06:23:38.893 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.893 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.893 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.894 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.894 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.894 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.894 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G8oh7TOeWyD2_FvC8AIXccM", 2022-09-03T06:23:38.895 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.895 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.895 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.895 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.896 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.896 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.896 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.896 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.897 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.565852_264.1", 2022-09-03T06:23:38.897 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.897 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:38.897 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.565852141Z", 2022-09-03T06:23:38.898 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.898 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.898 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:38.899 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.899 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.899 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.899 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:38.900 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.900 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.900 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.900 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.900 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.901 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.901 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.901 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.901 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185756.568581_265.1", 2022-09-03T06:23:38.902 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.902 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33", 2022-09-03T06:23:38.902 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:56.568581265Z", 2022-09-03T06:23:38.902 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.903 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.903 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:38.903 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.903 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.904 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.904 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_lvb2g2-N3EBX954j0_eL9sd", 2022-09-03T06:23:38.904 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.904 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.905 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.905 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.905 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.905 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.906 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.906 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.906 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185757.605699_266.1", 2022-09-03T06:23:38.906 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.907 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:38.907 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:57.605699761Z", 2022-09-03T06:23:38.907 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.907 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.908 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:38.908 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.908 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.908 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.909 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:38.909 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.909 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.909 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.910 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.910 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.910 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.910 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.911 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.911 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185757.608333_267.1", 2022-09-03T06:23:38.911 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.911 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34", 2022-09-03T06:23:38.912 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:57.608333919Z", 2022-09-03T06:23:38.912 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.912 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.912 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:38.913 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.913 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.913 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.913 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_V-1-Xh7688A-hUaqIeR5ie6", 2022-09-03T06:23:38.914 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.914 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.914 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.914 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.915 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.915 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.915 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.915 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.916 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.691847_268.1", 2022-09-03T06:23:38.916 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.916 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:38.916 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.691847123Z", 2022-09-03T06:23:38.917 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.917 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.917 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:38.917 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.918 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.918 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.918 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:38.918 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.919 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.919 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.919 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.919 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.920 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.920 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.920 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.920 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.696866_269.1", 2022-09-03T06:23:38.921 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.921 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35", 2022-09-03T06:23:38.921 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.696866177Z", 2022-09-03T06:23:38.921 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.922 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.922 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:38.922 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.923 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.923 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.923 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8mG_kD5KnEGb-g_s70lufLw", 2022-09-03T06:23:38.923 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 2 2022-09-03T06:23:38.924 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.924 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.924 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.924 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.924 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.925 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.925 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.925 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.804943_270.1", 2022-09-03T06:23:38.925 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.926 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:23:38.926 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.804943552Z", 2022-09-03T06:23:38.926 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.926 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.927 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.927 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.927 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.927 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.928 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-F9Dl85tpDzK3gXJ-aezc_j", 2022-09-03T06:23:38.928 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.928 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.928 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.929 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.929 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.929 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.929 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.930 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.930 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.808467_271.1", 2022-09-03T06:23:38.930 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.930 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:23:38.931 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.808467134Z", 2022-09-03T06:23:38.931 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.931 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.931 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.932 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.932 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.932 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.932 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-F9Dl85tpDzK3gXJ-aezc_j", 2022-09-03T06:23:38.933 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.933 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.933 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.933 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.934 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.934 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.934 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.934 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.935 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.810847_272.1", 2022-09-03T06:23:38.935 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.935 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-36", 2022-09-03T06:23:38.936 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.810847136Z", 2022-09-03T06:23:38.936 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.936 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.936 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.937 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.937 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.937 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_5JS-wFAV9CrTmAifbKi9NjK", 2022-09-03T06:23:38.938 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.938 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.938 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.939 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.939 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.939 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.940 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.813462_273.1", 2022-09-03T06:23:38.940 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.940 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-36", 2022-09-03T06:23:38.941 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.813462243Z", 2022-09-03T06:23:38.941 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.941 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.941 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.942 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.942 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.942 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.942 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_5JS-wFAV9CrTmAifbKi9NjK", 2022-09-03T06:23:38.943 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.943 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.943 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.944 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.944 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.944 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.945 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.844464_274.1", 2022-09-03T06:23:38.945 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.945 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:23:38.945 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.844464322Z", 2022-09-03T06:23:38.946 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.946 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.946 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.946 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.947 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.947 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_brde0pgOsuZjrCySsOp9pmR", 2022-09-03T06:23:38.947 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.948 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.948 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.948 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.948 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.949 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.949 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.949 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.949 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.850329_275.1", 2022-09-03T06:23:38.950 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.950 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:23:38.950 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.850329433Z", 2022-09-03T06:23:38.950 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.951 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.951 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.951 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.952 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.952 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_brde0pgOsuZjrCySsOp9pmR", 2022-09-03T06:23:38.952 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.953 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.953 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.954 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.954 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.954 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.855256_276.1", 2022-09-03T06:23:38.955 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.955 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-37", 2022-09-03T06:23:38.955 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.855256692Z", 2022-09-03T06:23:38.955 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.956 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.956 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.956 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.956 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.957 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3HAVWKyp6rMdHTXkKukojjv", 2022-09-03T06:23:38.957 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.957 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.958 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.958 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.958 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.959 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.959 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.959 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.859644_277.1", 2022-09-03T06:23:38.959 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.959 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-37", 2022-09-03T06:23:38.960 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.859644317Z", 2022-09-03T06:23:38.960 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.960 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.960 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.961 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.961 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.961 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3HAVWKyp6rMdHTXkKukojjv", 2022-09-03T06:23:38.962 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.962 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.962 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.963 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.963 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.963 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.963 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.964 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.902177_278.1", 2022-09-03T06:23:38.964 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.964 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:23:38.965 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.902177035Z", 2022-09-03T06:23:38.965 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.965 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.965 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.966 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.966 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.966 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.966 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Iv3Ff3D6p1XbZ30mXZkhK63", 2022-09-03T06:23:38.967 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.967 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.967 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.967 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.968 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.968 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.968 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.968 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.968 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.907504_279.1", 2022-09-03T06:23:38.969 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.969 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:23:38.969 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.907504015Z", 2022-09-03T06:23:38.969 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.970 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.970 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.970 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.970 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.971 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.971 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Iv3Ff3D6p1XbZ30mXZkhK63", 2022-09-03T06:23:38.971 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.971 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.972 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.972 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.972 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.972 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.973 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.973 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.973 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.908555_280.1", 2022-09-03T06:23:38.973 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.974 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-38", 2022-09-03T06:23:38.974 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.908555860Z", 2022-09-03T06:23:38.974 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.975 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.975 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.975 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.975 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.976 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.976 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G9WP1PhzJZFD5vWOwiYS9Hb", 2022-09-03T06:23:38.976 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.976 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.977 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.977 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.977 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.978 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.978 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662185758.915547_281.1", 2022-09-03T06:23:38.978 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:38.978 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-38", 2022-09-03T06:23:38.979 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:15:58.915547971Z", 2022-09-03T06:23:38.979 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.979 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.979 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.980 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.980 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.980 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.981 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_G9WP1PhzJZFD5vWOwiYS9Hb", 2022-09-03T06:23:38.981 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.981 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.982 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.982 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.982 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.983 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.984 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.984 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.844866_288.1", 2022-09-03T06:23:38.984 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.985 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39", 2022-09-03T06:23:38.985 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.844866559Z", 2022-09-03T06:23:38.986 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.986 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.986 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.987 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.987 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.987 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.988 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lt1dYegSwDzpwX06X-fGkup", 2022-09-03T06:23:38.988 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.989 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.989 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.989 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:38.990 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.990 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.991 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.991 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.991 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.850612_289.1", 2022-09-03T06:23:38.992 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:38.992 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39", 2022-09-03T06:23:38.993 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.850612019Z", 2022-09-03T06:23:38.993 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:38.993 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:38.994 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:38.994 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:38.995 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.995 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:38.995 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Lt1dYegSwDzpwX06X-fGkup", 2022-09-03T06:23:38.996 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:38.996 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.997 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:38.997 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:38.997 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.998 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:38.998 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:38.999 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:38.999 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.852628_290.1", 2022-09-03T06:23:38.999 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.000 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-39", 2022-09-03T06:23:39.000 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.852628518Z", 2022-09-03T06:23:39.000 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.000 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.001 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.001 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.001 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.001 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.002 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_O0mztKsloMen3wFjqSH6i9D", 2022-09-03T06:23:39.002 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.002 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.002 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.003 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.003 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.004 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.004 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.857180_291.1", 2022-09-03T06:23:39.004 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.004 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-39", 2022-09-03T06:23:39.005 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.857180178Z", 2022-09-03T06:23:39.005 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.005 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.005 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.006 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.006 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.006 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.007 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_O0mztKsloMen3wFjqSH6i9D", 2022-09-03T06:23:39.007 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.007 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.008 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.008 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.008 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.008 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.009 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.883799_292.1", 2022-09-03T06:23:39.009 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.009 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40", 2022-09-03T06:23:39.010 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.883799148Z", 2022-09-03T06:23:39.010 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.010 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.010 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.011 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.011 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.011 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_fswsShBMVZzHxqVULj3KzUU", 2022-09-03T06:23:39.012 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.012 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.012 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.013 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.013 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.013 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.014 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.889075_293.1", 2022-09-03T06:23:39.014 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.014 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40", 2022-09-03T06:23:39.014 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.889075582Z", 2022-09-03T06:23:39.015 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.015 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.015 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.015 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.016 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.016 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_fswsShBMVZzHxqVULj3KzUU", 2022-09-03T06:23:39.016 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.017 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.017 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.017 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.017 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.018 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.018 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.018 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.890194_294.1", 2022-09-03T06:23:39.019 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.019 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-40", 2022-09-03T06:23:39.019 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.890194639Z", 2022-09-03T06:23:39.019 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.020 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.020 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.020 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.021 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.021 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_A4T-tDc1XWVEtWeND6N-Hqq", 2022-09-03T06:23:39.021 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.022 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.022 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.022 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.022 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.023 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.896341_295.1", 2022-09-03T06:23:39.023 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.024 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-40", 2022-09-03T06:23:39.024 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.896341961Z", 2022-09-03T06:23:39.024 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.024 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.025 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.025 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.025 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.026 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_A4T-tDc1XWVEtWeND6N-Hqq", 2022-09-03T06:23:39.026 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.026 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.027 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.027 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.027 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.028 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.947010_296.1", 2022-09-03T06:23:39.028 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.028 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41", 2022-09-03T06:23:39.029 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.947010594Z", 2022-09-03T06:23:39.029 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.029 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.029 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.030 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.030 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.030 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.030 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yitoOgqNEqy39xdDDb8hNi-", 2022-09-03T06:23:39.031 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.031 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.031 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.032 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.032 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.033 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.950610_297.1", 2022-09-03T06:23:39.033 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.033 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41", 2022-09-03T06:23:39.033 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.950610217Z", 2022-09-03T06:23:39.034 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.034 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.034 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.035 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.035 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.035 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yitoOgqNEqy39xdDDb8hNi-", 2022-09-03T06:23:39.036 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.036 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.036 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.036 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.037 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.037 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.951802_298.1", 2022-09-03T06:23:39.038 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.038 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-41", 2022-09-03T06:23:39.038 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.951802745Z", 2022-09-03T06:23:39.038 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.039 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.039 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.039 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.040 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.040 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_rtPxlDQFqSiMUkKzGoM6IJb", 2022-09-03T06:23:39.040 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.041 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.041 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.041 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.041 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.042 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.042 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186063.955393_299.1", 2022-09-03T06:23:39.042 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.043 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-41", 2022-09-03T06:23:39.043 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:21:03.955393441Z", 2022-09-03T06:23:39.043 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.043 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.044 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.044 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.044 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.045 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_rtPxlDQFqSiMUkKzGoM6IJb", 2022-09-03T06:23:39.045 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.045 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.046 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.046 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.046 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.046 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.047 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:23:39.048 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.049 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.050 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period 6a921abe-6ddc-4d12-8acc-a1151b4a5f12 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:39.105 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:39.106 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:39.155 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:39.156 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.156 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.184096_167.1", 2022-09-03T06:23:39.156 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.157 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1", 2022-09-03T06:23:39.157 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.184096095Z", 2022-09-03T06:23:39.157 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.158 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.158 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.158 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.158 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.159 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.159 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_oM9W7o0z_W0m_MnjhnJoUGA", 2022-09-03T06:23:39.159 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.159 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.160 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.160 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.160 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.160 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.161 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.161 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.162 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.187111_168.1", 2022-09-03T06:23:39.162 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.163 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1", 2022-09-03T06:23:39.163 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.187111725Z", 2022-09-03T06:23:39.164 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.164 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.164 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.164 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.165 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.165 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.165 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_oM9W7o0z_W0m_MnjhnJoUGA", 2022-09-03T06:23:39.166 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.166 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.166 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.166 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.167 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.167 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.167 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.167 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.168 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.188247_169.1", 2022-09-03T06:23:39.168 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.168 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-42", 2022-09-03T06:23:39.168 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.188247557Z", 2022-09-03T06:23:39.169 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.169 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.169 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.170 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.170 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.171 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.171 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yYNjlucJG1Bf1z6Ty3RVaD7", 2022-09-03T06:23:39.171 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.171 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.172 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.172 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.172 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.172 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.173 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.173 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.173 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.191060_170.1", 2022-09-03T06:23:39.174 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.174 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-42", 2022-09-03T06:23:39.174 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.191060022Z", 2022-09-03T06:23:39.175 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.175 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.175 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.175 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.176 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.176 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.176 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yYNjlucJG1Bf1z6Ty3RVaD7", 2022-09-03T06:23:39.176 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.177 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.177 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.178 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.178 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.179 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.179 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.180 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.180 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.227783_171.1", 2022-09-03T06:23:39.180 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.181 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2", 2022-09-03T06:23:39.181 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.227783307Z", 2022-09-03T06:23:39.181 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.182 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.182 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.182 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.182 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.183 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.183 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_p2cwSU070zqh19f6L0b33P5", 2022-09-03T06:23:39.183 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.183 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.184 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.184 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.184 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.185 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.185 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.185 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.185 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.233199_172.1", 2022-09-03T06:23:39.186 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.186 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2", 2022-09-03T06:23:39.186 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.233199159Z", 2022-09-03T06:23:39.187 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.187 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.187 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.187 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.188 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.188 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.188 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_p2cwSU070zqh19f6L0b33P5", 2022-09-03T06:23:39.188 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.189 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.189 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.189 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.189 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.190 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.190 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.190 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.190 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.234332_173.1", 2022-09-03T06:23:39.191 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.191 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-43", 2022-09-03T06:23:39.192 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.234332916Z", 2022-09-03T06:23:39.192 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.192 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.193 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.193 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.193 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.193 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.194 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_jlIB8ftIBR9w5ZBV0f0iTOf", 2022-09-03T06:23:39.194 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.194 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.194 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.195 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.195 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.195 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.196 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.196 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.196 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.239783_174.1", 2022-09-03T06:23:39.197 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.197 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-43", 2022-09-03T06:23:39.197 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.239783629Z", 2022-09-03T06:23:39.197 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.198 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.198 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.198 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.198 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.199 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.199 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_jlIB8ftIBR9w5ZBV0f0iTOf", 2022-09-03T06:23:39.199 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.200 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.200 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.201 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.202 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.202 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.202 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.203 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:39.203 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.204 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.204 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period 6a921abe-6ddc-4d12-8acc-a1151b4a5f12 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:39.259 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:39.260 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:39.309 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:23:39.310 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.311 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.184096_167.1", 2022-09-03T06:23:39.311 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.312 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1", 2022-09-03T06:23:39.312 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.184096095Z", 2022-09-03T06:23:39.313 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.313 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.314 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.314 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.315 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.315 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.316 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_oM9W7o0z_W0m_MnjhnJoUGA", 2022-09-03T06:23:39.316 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.317 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.318 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.319 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.319 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.319 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.320 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.320 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.321 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.187111_168.1", 2022-09-03T06:23:39.321 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.322 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1", 2022-09-03T06:23:39.322 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.187111725Z", 2022-09-03T06:23:39.323 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.323 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.323 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.324 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.324 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.324 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.325 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_oM9W7o0z_W0m_MnjhnJoUGA", 2022-09-03T06:23:39.325 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.326 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.326 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.326 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.326 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.327 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.327 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.327 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.328 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.188247_169.1", 2022-09-03T06:23:39.328 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.328 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-42", 2022-09-03T06:23:39.329 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.188247557Z", 2022-09-03T06:23:39.329 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.329 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.330 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.330 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.331 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.331 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.331 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yYNjlucJG1Bf1z6Ty3RVaD7", 2022-09-03T06:23:39.331 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.332 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.332 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.332 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.332 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.333 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.333 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.333 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.333 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.191060_170.1", 2022-09-03T06:23:39.334 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.334 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-42", 2022-09-03T06:23:39.334 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.191060022Z", 2022-09-03T06:23:39.335 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.335 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.335 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.335 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.336 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.336 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.336 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yYNjlucJG1Bf1z6Ty3RVaD7", 2022-09-03T06:23:39.336 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.337 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.337 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.337 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.338 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.338 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.338 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.339 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.339 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.227783_171.1", 2022-09-03T06:23:39.339 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.339 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2", 2022-09-03T06:23:39.340 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.227783307Z", 2022-09-03T06:23:39.340 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.340 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.340 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.341 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.341 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.341 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.342 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_p2cwSU070zqh19f6L0b33P5", 2022-09-03T06:23:39.342 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.342 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.343 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.343 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.343 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.343 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.344 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.344 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.344 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.233199_172.1", 2022-09-03T06:23:39.345 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.345 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2", 2022-09-03T06:23:39.345 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.233199159Z", 2022-09-03T06:23:39.345 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.346 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.346 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.346 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.346 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.347 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.347 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_p2cwSU070zqh19f6L0b33P5", 2022-09-03T06:23:39.347 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.348 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.348 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.348 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.348 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.349 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.349 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.349 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.349 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.234332_173.1", 2022-09-03T06:23:39.350 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.350 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-43", 2022-09-03T06:23:39.350 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.234332916Z", 2022-09-03T06:23:39.351 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.351 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.351 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.351 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.352 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.352 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.352 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_jlIB8ftIBR9w5ZBV0f0iTOf", 2022-09-03T06:23:39.353 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.353 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.353 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.353 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.355 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.355 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.356 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.356 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.356 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.239783_174.1", 2022-09-03T06:23:39.356 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.357 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-43", 2022-09-03T06:23:39.357 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.239783629Z", 2022-09-03T06:23:39.357 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.357 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.358 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.358 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.358 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.359 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.359 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_jlIB8ftIBR9w5ZBV0f0iTOf", 2022-09-03T06:23:39.359 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.360 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.360 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.361 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.361 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.362 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.362 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.362 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:23:39.363 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.363 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.364 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period 6a921abe-6ddc-4d12-8acc-a1151b4a5f12 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:39.414 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:39.414 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:39.444 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:39.444 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.444 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.184096_167.1", 2022-09-03T06:23:39.445 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.445 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1", 2022-09-03T06:23:39.445 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.184096095Z", 2022-09-03T06:23:39.445 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.446 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.446 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.446 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.447 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.447 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.447 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_oM9W7o0z_W0m_MnjhnJoUGA", 2022-09-03T06:23:39.448 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.448 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.448 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.448 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.449 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.449 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.449 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.449 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.450 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.187111_168.1", 2022-09-03T06:23:39.450 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.450 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1", 2022-09-03T06:23:39.451 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.187111725Z", 2022-09-03T06:23:39.451 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.451 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.451 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.452 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.452 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.452 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.452 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_oM9W7o0z_W0m_MnjhnJoUGA", 2022-09-03T06:23:39.453 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.453 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.453 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.453 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.454 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.455 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.455 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.455 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.455 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.188247_169.1", 2022-09-03T06:23:39.456 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.456 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-42", 2022-09-03T06:23:39.456 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.188247557Z", 2022-09-03T06:23:39.457 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.457 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.457 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.457 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.458 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.458 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.458 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yYNjlucJG1Bf1z6Ty3RVaD7", 2022-09-03T06:23:39.459 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.459 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.459 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.459 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.460 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.460 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.460 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.460 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.461 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.191060_170.1", 2022-09-03T06:23:39.461 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.461 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-42", 2022-09-03T06:23:39.461 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.191060022Z", 2022-09-03T06:23:39.462 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.462 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.462 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.463 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.463 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.463 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.463 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_yYNjlucJG1Bf1z6Ty3RVaD7", 2022-09-03T06:23:39.464 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.464 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.464 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.464 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.465 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.465 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.465 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.465 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.466 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.227783_171.1", 2022-09-03T06:23:39.466 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.466 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2", 2022-09-03T06:23:39.467 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.227783307Z", 2022-09-03T06:23:39.467 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.467 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.467 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.468 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.468 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.468 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.468 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_p2cwSU070zqh19f6L0b33P5", 2022-09-03T06:23:39.469 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.469 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.469 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.469 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.470 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.470 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.470 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.470 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.471 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.233199_172.1", 2022-09-03T06:23:39.471 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.471 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2", 2022-09-03T06:23:39.472 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.233199159Z", 2022-09-03T06:23:39.472 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.472 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.472 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.473 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.473 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.473 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.473 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_p2cwSU070zqh19f6L0b33P5", 2022-09-03T06:23:39.474 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.474 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.474 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.474 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.475 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.475 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.475 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.475 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.476 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.234332_173.1", 2022-09-03T06:23:39.476 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.476 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-43", 2022-09-03T06:23:39.476 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.234332916Z", 2022-09-03T06:23:39.477 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.477 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.477 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.478 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.478 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.478 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.478 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_jlIB8ftIBR9w5ZBV0f0iTOf", 2022-09-03T06:23:39.479 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.479 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.479 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.479 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.480 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.480 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.480 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.480 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.481 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186136.239783_174.1", 2022-09-03T06:23:39.481 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.481 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-43", 2022-09-03T06:23:39.481 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:22:16.239783629Z", 2022-09-03T06:23:39.482 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.482 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.482 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.483 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.483 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.483 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.483 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_jlIB8ftIBR9w5ZBV0f0iTOf", 2022-09-03T06:23:39.484 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.484 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.484 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.486 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.486 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.486 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.487 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.487 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:39.487 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.488 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.488 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period 6a921abe-6ddc-4d12-8acc-a1151b4a5f12 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:39.538 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:39.538 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:39.581 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:23:39.582 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.582 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.184096_167.1", 2022-09-03T06:23:39.582 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.583 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1", 2022-09-03T06:23:39.583 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.184096095Z", 2022-09-03T06:23:39.583 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.583 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.584 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.584 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.584 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.585 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_oM9W7o0z_W0m_MnjhnJoUGA", 2022-09-03T06:23:39.585 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.586 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.586 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.587 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.587 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.588 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.187111_168.1", 2022-09-03T06:23:39.588 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.588 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1", 2022-09-03T06:23:39.588 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.187111725Z", 2022-09-03T06:23:39.589 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.589 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.589 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.589 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.590 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.590 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_oM9W7o0z_W0m_MnjhnJoUGA", 2022-09-03T06:23:39.590 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.591 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.591 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.591 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.592 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.592 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.188247_169.1", 2022-09-03T06:23:39.593 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.593 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-42", 2022-09-03T06:23:39.593 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.188247557Z", 2022-09-03T06:23:39.593 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.594 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.594 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.594 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.594 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.595 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.595 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yYNjlucJG1Bf1z6Ty3RVaD7", 2022-09-03T06:23:39.595 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.595 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.596 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.596 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.596 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.597 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.597 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.191060_170.1", 2022-09-03T06:23:39.598 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.598 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-42", 2022-09-03T06:23:39.598 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.191060022Z", 2022-09-03T06:23:39.598 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.599 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.599 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.599 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.600 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.600 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_yYNjlucJG1Bf1z6Ty3RVaD7", 2022-09-03T06:23:39.600 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.601 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.601 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.601 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.601 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.602 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.602 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.602 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.227783_171.1", 2022-09-03T06:23:39.602 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.603 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2", 2022-09-03T06:23:39.603 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.227783307Z", 2022-09-03T06:23:39.603 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.603 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.604 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.604 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.605 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.605 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_p2cwSU070zqh19f6L0b33P5", 2022-09-03T06:23:39.605 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.606 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.606 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.606 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.607 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.607 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.607 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.233199_172.1", 2022-09-03T06:23:39.608 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.608 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2", 2022-09-03T06:23:39.608 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.233199159Z", 2022-09-03T06:23:39.608 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.609 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.609 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.609 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.610 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.610 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_p2cwSU070zqh19f6L0b33P5", 2022-09-03T06:23:39.610 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.611 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.611 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.611 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.611 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.612 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.234332_173.1", 2022-09-03T06:23:39.612 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.613 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-43", 2022-09-03T06:23:39.613 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.234332916Z", 2022-09-03T06:23:39.613 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.614 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.614 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.614 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.614 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.615 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.615 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_jlIB8ftIBR9w5ZBV0f0iTOf", 2022-09-03T06:23:39.615 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.615 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.616 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.616 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.616 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.616 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.617 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186136.239783_174.1", 2022-09-03T06:23:39.617 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.618 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-43", 2022-09-03T06:23:39.618 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:22:16.239783629Z", 2022-09-03T06:23:39.618 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.618 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.619 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.619 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.619 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.620 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_jlIB8ftIBR9w5ZBV0f0iTOf", 2022-09-03T06:23:39.620 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.620 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.622 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.622 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.623 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:23:39.624 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.624 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.625 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period 2962e93e-ca58-4758-b6d9-6fe2e67c2976 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:39.675 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:39.675 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:39.708 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:39.708 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.709 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.018196_151.1", 2022-09-03T06:23:39.709 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.709 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1", 2022-09-03T06:23:39.710 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.018196452Z", 2022-09-03T06:23:39.710 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.710 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.711 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.711 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.711 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.712 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.712 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_LxUudMu4CM-bwxE7Y-VjeuO", 2022-09-03T06:23:39.712 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.712 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.713 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.713 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.713 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.713 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.714 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.714 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.714 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.023060_152.1", 2022-09-03T06:23:39.715 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.715 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1", 2022-09-03T06:23:39.715 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.023060857Z", 2022-09-03T06:23:39.715 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.716 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.716 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.716 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.716 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.717 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.717 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_LxUudMu4CM-bwxE7Y-VjeuO", 2022-09-03T06:23:39.717 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.718 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.718 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.718 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.719 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.719 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.719 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.720 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.720 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.024958_153.1", 2022-09-03T06:23:39.721 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.721 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-44", 2022-09-03T06:23:39.721 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.024958028Z", 2022-09-03T06:23:39.722 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.722 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.722 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.722 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.723 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.723 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.723 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Zz-6QDH09979e3jImLmMSUm", 2022-09-03T06:23:39.723 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.724 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.724 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.724 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.724 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.725 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.725 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.725 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.726 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.031191_154.1", 2022-09-03T06:23:39.726 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.726 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-44", 2022-09-03T06:23:39.726 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.031191192Z", 2022-09-03T06:23:39.727 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.727 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.727 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.727 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.728 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.728 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.728 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Zz-6QDH09979e3jImLmMSUm", 2022-09-03T06:23:39.728 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.729 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.729 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.729 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.730 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.730 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.730 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.730 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.731 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.068210_155.1", 2022-09-03T06:23:39.731 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.731 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2", 2022-09-03T06:23:39.731 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.068210924Z", 2022-09-03T06:23:39.732 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.732 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.732 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.732 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.733 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.733 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.733 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-CAzyPJsufsmORlEfhsvqeY", 2022-09-03T06:23:39.734 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.734 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.734 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.734 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.735 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.735 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.735 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.735 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.736 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.073948_156.1", 2022-09-03T06:23:39.736 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.736 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2", 2022-09-03T06:23:39.736 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.073948683Z", 2022-09-03T06:23:39.737 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.737 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.737 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.737 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.738 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.738 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.738 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-CAzyPJsufsmORlEfhsvqeY", 2022-09-03T06:23:39.739 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.739 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.739 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.739 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.740 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.740 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.740 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.740 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.741 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.075500_157.1", 2022-09-03T06:23:39.741 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.741 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-45", 2022-09-03T06:23:39.741 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.075500240Z", 2022-09-03T06:23:39.742 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.742 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.742 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.742 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.743 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.743 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.743 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UkwaK0iSqKJtBxmfh0FjFSq", 2022-09-03T06:23:39.743 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.744 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.744 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.744 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.744 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.745 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.745 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.745 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.745 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.079851_158.1", 2022-09-03T06:23:39.746 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.746 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-45", 2022-09-03T06:23:39.746 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.079851457Z", 2022-09-03T06:23:39.746 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.747 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.747 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.747 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.748 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.748 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.748 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UkwaK0iSqKJtBxmfh0FjFSq", 2022-09-03T06:23:39.748 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.749 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.749 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.750 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.751 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.751 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.751 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.751 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:39.752 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.753 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.753 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period 2962e93e-ca58-4758-b6d9-6fe2e67c2976 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:39.801 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:39.802 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:39.840 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:23:39.840 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.841 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.018196_151.1", 2022-09-03T06:23:39.841 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.841 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1", 2022-09-03T06:23:39.841 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.018196452Z", 2022-09-03T06:23:39.842 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.842 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.842 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.843 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.843 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.843 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.844 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_LxUudMu4CM-bwxE7Y-VjeuO", 2022-09-03T06:23:39.844 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.844 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.844 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.845 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.845 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.845 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.846 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.846 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.846 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.023060_152.1", 2022-09-03T06:23:39.846 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.847 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1", 2022-09-03T06:23:39.847 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.023060857Z", 2022-09-03T06:23:39.847 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.847 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.848 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.848 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.848 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.849 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.849 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_LxUudMu4CM-bwxE7Y-VjeuO", 2022-09-03T06:23:39.849 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.849 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.850 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.850 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.850 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.851 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.851 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.851 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.851 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.024958_153.1", 2022-09-03T06:23:39.852 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.852 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-44", 2022-09-03T06:23:39.852 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.024958028Z", 2022-09-03T06:23:39.853 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.853 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.854 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.854 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.854 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.855 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.855 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Zz-6QDH09979e3jImLmMSUm", 2022-09-03T06:23:39.855 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.855 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.856 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.856 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.856 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.856 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.857 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.857 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.857 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.031191_154.1", 2022-09-03T06:23:39.857 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.858 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-44", 2022-09-03T06:23:39.858 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.031191192Z", 2022-09-03T06:23:39.858 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.859 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.859 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.860 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.860 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.861 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.861 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Zz-6QDH09979e3jImLmMSUm", 2022-09-03T06:23:39.861 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.862 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.862 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.863 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.863 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.863 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.864 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.864 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.864 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.068210_155.1", 2022-09-03T06:23:39.865 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.865 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2", 2022-09-03T06:23:39.866 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.068210924Z", 2022-09-03T06:23:39.866 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.866 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.867 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.867 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.868 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.868 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.868 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-CAzyPJsufsmORlEfhsvqeY", 2022-09-03T06:23:39.869 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.869 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.870 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.870 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.871 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.871 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.871 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.872 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.872 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.073948_156.1", 2022-09-03T06:23:39.873 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.873 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2", 2022-09-03T06:23:39.873 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.073948683Z", 2022-09-03T06:23:39.874 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.874 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.875 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.875 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.876 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.876 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.877 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-CAzyPJsufsmORlEfhsvqeY", 2022-09-03T06:23:39.877 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.877 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.878 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.878 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.879 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.879 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.880 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.880 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.881 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.075500_157.1", 2022-09-03T06:23:39.881 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.882 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-45", 2022-09-03T06:23:39.882 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.075500240Z", 2022-09-03T06:23:39.882 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.883 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.883 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.883 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.883 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.884 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.884 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UkwaK0iSqKJtBxmfh0FjFSq", 2022-09-03T06:23:39.884 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.884 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.885 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.885 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:39.885 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.885 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.886 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.886 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:39.886 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.079851_158.1", 2022-09-03T06:23:39.886 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:39.887 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-45", 2022-09-03T06:23:39.887 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.079851457Z", 2022-09-03T06:23:39.887 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:39.887 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:39.888 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:39.888 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:39.888 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.888 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:39.889 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UkwaK0iSqKJtBxmfh0FjFSq", 2022-09-03T06:23:39.889 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:39.889 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:39.889 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:39.891 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:39.891 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.891 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.892 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:39.892 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:23:39.892 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.893 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:39.894 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period 2962e93e-ca58-4758-b6d9-6fe2e67c2976 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:39.943 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:39.943 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:39.986 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:39.987 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.987 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.018196_151.1", 2022-09-03T06:23:39.987 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.987 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1", 2022-09-03T06:23:39.988 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.018196452Z", 2022-09-03T06:23:39.988 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.988 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.988 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.989 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.989 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.989 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.989 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_LxUudMu4CM-bwxE7Y-VjeuO", 2022-09-03T06:23:39.990 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.990 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.990 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.991 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:39.991 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.991 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.991 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.992 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.992 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.023060_152.1", 2022-09-03T06:23:39.992 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:39.992 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1", 2022-09-03T06:23:39.993 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.023060857Z", 2022-09-03T06:23:39.993 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.993 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.993 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.994 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.994 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.994 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:39.994 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_LxUudMu4CM-bwxE7Y-VjeuO", 2022-09-03T06:23:39.995 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:39.995 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.995 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:39.996 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:39.996 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.996 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:39.997 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:39.997 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:39.997 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.024958_153.1", 2022-09-03T06:23:39.998 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:39.998 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-44", 2022-09-03T06:23:39.998 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.024958028Z", 2022-09-03T06:23:39.998 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:39.999 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:39.999 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:39.999 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:39.999 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.000 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:40.000 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Zz-6QDH09979e3jImLmMSUm", 2022-09-03T06:23:40.000 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:40.000 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.000 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:40.001 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:40.001 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.001 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.002 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.002 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:40.002 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.031191_154.1", 2022-09-03T06:23:40.002 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:40.003 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-44", 2022-09-03T06:23:40.003 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.031191192Z", 2022-09-03T06:23:40.003 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:40.003 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:40.004 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:40.004 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:40.004 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.004 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:40.005 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_Zz-6QDH09979e3jImLmMSUm", 2022-09-03T06:23:40.005 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:40.005 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.005 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:40.006 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:40.006 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.006 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.007 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.007 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:40.007 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.068210_155.1", 2022-09-03T06:23:40.008 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:40.008 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2", 2022-09-03T06:23:40.008 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.068210924Z", 2022-09-03T06:23:40.008 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:40.009 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:40.009 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:40.009 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:40.009 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.010 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:40.010 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-CAzyPJsufsmORlEfhsvqeY", 2022-09-03T06:23:40.011 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:40.011 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.011 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:40.011 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:40.012 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.012 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.012 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.012 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:40.013 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.073948_156.1", 2022-09-03T06:23:40.013 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket.instance", 2022-09-03T06:23:40.013 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2", 2022-09-03T06:23:40.013 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.073948683Z", 2022-09-03T06:23:40.014 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:40.014 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:40.014 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:40.014 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:40.015 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.015 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:40.015 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_-CAzyPJsufsmORlEfhsvqeY", 2022-09-03T06:23:40.015 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:40.016 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.016 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:40.016 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:40.016 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.017 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.017 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.017 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:40.018 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.075500_157.1", 2022-09-03T06:23:40.018 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:40.018 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-45", 2022-09-03T06:23:40.018 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.075500240Z", 2022-09-03T06:23:40.019 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:40.019 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:40.019 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:40.019 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:40.020 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.020 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:40.021 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UkwaK0iSqKJtBxmfh0FjFSq", 2022-09-03T06:23:40.021 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:40.021 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.022 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:40.022 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "write" 2022-09-03T06:23:40.022 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.023 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.023 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.023 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:40.023 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "1_1662186202.079851_158.1", 2022-09-03T06:23:40.024 INFO:teuthology.orchestra.run.smithi062.stdout: "section": "bucket", 2022-09-03T06:23:40.024 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "avysmw-45", 2022-09-03T06:23:40.024 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.079851457Z", 2022-09-03T06:23:40.024 INFO:teuthology.orchestra.run.smithi062.stdout: "data": { 2022-09-03T06:23:40.025 INFO:teuthology.orchestra.run.smithi062.stdout: "read_version": { 2022-09-03T06:23:40.025 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "", 2022-09-03T06:23:40.025 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 0 2022-09-03T06:23:40.025 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.026 INFO:teuthology.orchestra.run.smithi062.stdout: "write_version": { 2022-09-03T06:23:40.026 INFO:teuthology.orchestra.run.smithi062.stdout: "tag": "_UkwaK0iSqKJtBxmfh0FjFSq", 2022-09-03T06:23:40.026 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": 1 2022-09-03T06:23:40.026 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:40.027 INFO:teuthology.orchestra.run.smithi062.stdout: "status": { 2022-09-03T06:23:40.028 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "complete" 2022-09-03T06:23:40.029 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.029 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.029 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:40.030 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:40.031 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:40.032 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:40.033 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period 2962e93e-ca58-4758-b6d9-6fe2e67c2976 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:40.081 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:40.082 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:40.119 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:23:40.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:40.119 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.018196_151.1", 2022-09-03T06:23:40.120 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:40.120 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1", 2022-09-03T06:23:40.120 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.018196452Z", 2022-09-03T06:23:40.120 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:40.121 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:40.121 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:40.121 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:40.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.122 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:40.122 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_LxUudMu4CM-bwxE7Y-VjeuO", 2022-09-03T06:23:40.122 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:40.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.123 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:40.123 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:40.124 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.124 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.124 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:40.125 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.023060_152.1", 2022-09-03T06:23:40.125 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:40.125 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1", 2022-09-03T06:23:40.125 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.023060857Z", 2022-09-03T06:23:40.126 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:40.126 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:40.126 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:40.126 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:40.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.127 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:40.127 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_LxUudMu4CM-bwxE7Y-VjeuO", 2022-09-03T06:23:40.128 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:40.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.128 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:40.128 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:40.129 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.129 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:40.130 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.024958_153.1", 2022-09-03T06:23:40.130 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:40.131 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-44", 2022-09-03T06:23:40.131 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.024958028Z", 2022-09-03T06:23:40.131 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:40.131 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:40.132 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:40.132 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:40.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.133 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:40.133 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Zz-6QDH09979e3jImLmMSUm", 2022-09-03T06:23:40.133 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:40.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.134 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:40.134 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:40.134 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.134 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.135 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:40.135 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.031191_154.1", 2022-09-03T06:23:40.135 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:40.136 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-44", 2022-09-03T06:23:40.136 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.031191192Z", 2022-09-03T06:23:40.136 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:40.136 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:40.137 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:40.137 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:40.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.137 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:40.138 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Zz-6QDH09979e3jImLmMSUm", 2022-09-03T06:23:40.138 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:40.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.139 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:40.139 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:40.139 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.139 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:40.140 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.068210_155.1", 2022-09-03T06:23:40.140 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:40.140 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2", 2022-09-03T06:23:40.141 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.068210924Z", 2022-09-03T06:23:40.141 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:40.141 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:40.141 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:40.142 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:40.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.142 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:40.142 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-CAzyPJsufsmORlEfhsvqeY", 2022-09-03T06:23:40.143 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:40.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.143 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:40.143 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:40.144 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.144 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:40.145 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.073948_156.1", 2022-09-03T06:23:40.145 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket.instance", 2022-09-03T06:23:40.145 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2", 2022-09-03T06:23:40.145 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.073948683Z", 2022-09-03T06:23:40.146 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:40.146 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:40.146 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:40.146 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:40.147 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.147 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:40.147 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_-CAzyPJsufsmORlEfhsvqeY", 2022-09-03T06:23:40.147 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:40.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.148 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:40.148 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:40.148 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.149 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.149 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:40.150 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.075500_157.1", 2022-09-03T06:23:40.150 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:40.150 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-45", 2022-09-03T06:23:40.150 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.075500240Z", 2022-09-03T06:23:40.151 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:40.151 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:40.151 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:40.151 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:40.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.152 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:40.152 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UkwaK0iSqKJtBxmfh0FjFSq", 2022-09-03T06:23:40.152 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:40.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.153 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:40.153 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "write" 2022-09-03T06:23:40.153 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.154 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.154 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:40.154 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "1_1662186202.079851_158.1", 2022-09-03T06:23:40.155 INFO:teuthology.orchestra.run.smithi087.stdout: "section": "bucket", 2022-09-03T06:23:40.155 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "avysmw-45", 2022-09-03T06:23:40.155 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.079851457Z", 2022-09-03T06:23:40.156 INFO:teuthology.orchestra.run.smithi087.stdout: "data": { 2022-09-03T06:23:40.156 INFO:teuthology.orchestra.run.smithi087.stdout: "read_version": { 2022-09-03T06:23:40.156 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "", 2022-09-03T06:23:40.156 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 0 2022-09-03T06:23:40.157 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.157 INFO:teuthology.orchestra.run.smithi087.stdout: "write_version": { 2022-09-03T06:23:40.157 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_UkwaK0iSqKJtBxmfh0FjFSq", 2022-09-03T06:23:40.157 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": 1 2022-09-03T06:23:40.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:40.158 INFO:teuthology.orchestra.run.smithi087.stdout: "status": { 2022-09-03T06:23:40.159 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "complete" 2022-09-03T06:23:40.160 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.160 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.160 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:40.161 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:23:40.161 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:23:40.162 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:23:40.162 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog autotrim --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:23:40.209 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:40.210 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:40.281 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'autotrim', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:23:40.282 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'autotrim', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:23:40.283 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog autotrim --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:23:40.328 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:40.328 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:40.513 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:23:40.514 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:23:40.515 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog autotrim --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:23:40.561 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:40.562 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:40.621 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'autotrim', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:23:40.622 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'autotrim', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:23:40.623 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog autotrim --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:23:40.669 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:40.669 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:40.814 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:40.814 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:40.815 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period b84ccf39-78f5-4998-9db6-42362bc8c98b --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:40.864 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:40.864 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:40.923 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:23:40.925 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:40.926 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:40.926 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period b84ccf39-78f5-4998-9db6-42362bc8c98b --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:40.974 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:40.974 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:41.053 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:23:41.055 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.055 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.056 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period b84ccf39-78f5-4998-9db6-42362bc8c98b --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:41.106 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:41.106 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:41.167 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:23:41.169 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.169 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', 'b84ccf39-78f5-4998-9db6-42362bc8c98b', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.170 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period b84ccf39-78f5-4998-9db6-42362bc8c98b --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:41.218 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:41.219 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:41.275 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:23:41.276 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.276 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.277 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period 6a921abe-6ddc-4d12-8acc-a1151b4a5f12 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:41.326 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:41.327 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:41.372 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:23:41.374 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.374 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.375 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period 6a921abe-6ddc-4d12-8acc-a1151b4a5f12 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:41.422 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:41.423 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:41.478 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:23:41.479 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.480 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.481 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period 6a921abe-6ddc-4d12-8acc-a1151b4a5f12 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:41.530 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:41.530 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:41.586 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:23:41.588 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.589 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', '6a921abe-6ddc-4d12-8acc-a1151b4a5f12', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.590 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period 6a921abe-6ddc-4d12-8acc-a1151b4a5f12 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:41.638 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:41.639 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:41.698 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:23:41.699 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.700 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.700 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period 2962e93e-ca58-4758-b6d9-6fe2e67c2976 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:41.749 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:41.750 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:41.803 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:23:41.805 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.806 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.807 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period 2962e93e-ca58-4758-b6d9-6fe2e67c2976 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:41.856 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:41.856 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:41.906 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:23:41.908 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.908 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:41.909 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog list --period 2962e93e-ca58-4758-b6d9-6fe2e67c2976 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:41.958 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:41.958 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:42.009 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:23:42.012 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:42.013 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'mdlog', 'list', '--period', '2962e93e-ca58-4758-b6d9-6fe2e67c2976', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:42.014 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 mdlog list --period 2962e93e-ca58-4758-b6d9-6fe2e67c2976 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:42.064 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:42.065 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:42.108 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:23:42.110 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multi_period_incremental_sync ... ok 2022-09-03T06:23:42.112 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-46 2022-09-03T06:23:42.113 DEBUG:boto:path=/avysmw-46/ 2022-09-03T06:23:42.113 DEBUG:boto:auth_path=/avysmw-46/ 2022-09-03T06:23:42.114 DEBUG:boto:Method: PUT 2022-09-03T06:23:42.114 DEBUG:boto:Path: /avysmw-46/ 2022-09-03T06:23:42.115 DEBUG:boto:Data: 2022-09-03T06:23:42.115 DEBUG:boto:Headers: {} 2022-09-03T06:23:42.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:42.116 DEBUG:boto:Port: 8000 2022-09-03T06:23:42.117 DEBUG:boto:Params: {} 2022-09-03T06:23:42.118 DEBUG:boto:Token: None 2022-09-03T06:23:42.118 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:23:42 GMT /avysmw-46/ 2022-09-03T06:23:42.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XnjPS431aPawS2ZYeYxxDbMa8W8= 2022-09-03T06:23:42.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XnjPS431aPawS2ZYeYxxDbMa8W8=', 'Content-Length': '0'} 2022-09-03T06:23:42.142 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ad77cf3773ae0baa-006312f2ee-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:42.143 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-47 2022-09-03T06:23:42.144 DEBUG:boto:path=/avysmw-47/ 2022-09-03T06:23:42.144 DEBUG:boto:auth_path=/avysmw-47/ 2022-09-03T06:23:42.145 DEBUG:boto:Method: PUT 2022-09-03T06:23:42.145 DEBUG:boto:Path: /avysmw-47/ 2022-09-03T06:23:42.146 DEBUG:boto:Data: 2022-09-03T06:23:42.146 DEBUG:boto:Headers: {} 2022-09-03T06:23:42.147 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:42.147 DEBUG:boto:Port: 8000 2022-09-03T06:23:42.148 DEBUG:boto:Params: {} 2022-09-03T06:23:42.148 DEBUG:boto:Token: None 2022-09-03T06:23:42.149 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:23:42 GMT /avysmw-47/ 2022-09-03T06:23:42.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UFeAD3PmfAQjZK7+m/JIlnXkuXM= 2022-09-03T06:23:42.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UFeAD3PmfAQjZK7+m/JIlnXkuXM=', 'Content-Length': '0'} 2022-09-03T06:23:42.194 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000031d047c57aef85fb-006312f2ee-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:42.195 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-48 2022-09-03T06:23:42.195 DEBUG:boto:path=/avysmw-48/ 2022-09-03T06:23:42.196 DEBUG:boto:auth_path=/avysmw-48/ 2022-09-03T06:23:42.196 DEBUG:boto:Method: PUT 2022-09-03T06:23:42.197 DEBUG:boto:Path: /avysmw-48/ 2022-09-03T06:23:42.197 DEBUG:boto:Data: 2022-09-03T06:23:42.198 DEBUG:boto:Headers: {} 2022-09-03T06:23:42.198 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:42.199 DEBUG:boto:Port: 8001 2022-09-03T06:23:42.199 DEBUG:boto:Params: {} 2022-09-03T06:23:42.200 DEBUG:boto:Token: None 2022-09-03T06:23:42.200 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:23:42 GMT /avysmw-48/ 2022-09-03T06:23:42.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pFTqltXL3V99vgdfAqQ3huQjl6A= 2022-09-03T06:23:42.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pFTqltXL3V99vgdfAqQ3huQjl6A=', 'Content-Length': '0'} 2022-09-03T06:23:42.258 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000d1b0f34ea9aae7c4-006312f2ee-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:42.259 DEBUG:boto:path=/avysmw-46/ 2022-09-03T06:23:42.259 DEBUG:boto:auth_path=/avysmw-46/ 2022-09-03T06:23:42.260 DEBUG:boto:Method: HEAD 2022-09-03T06:23:42.260 DEBUG:boto:Path: /avysmw-46/ 2022-09-03T06:23:42.261 DEBUG:boto:Data: 2022-09-03T06:23:42.262 DEBUG:boto:Headers: {} 2022-09-03T06:23:42.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:42.263 DEBUG:boto:Port: 8000 2022-09-03T06:23:42.264 DEBUG:boto:Params: {} 2022-09-03T06:23:42.264 DEBUG:boto:Token: None 2022-09-03T06:23:42.265 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:42 GMT /avysmw-46/ 2022-09-03T06:23:42.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IJBDqaVhQM3m7bX3OKyQRVr/UQI= 2022-09-03T06:23:42.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IJBDqaVhQM3m7bX3OKyQRVr/UQI=', 'Content-Length': '0'} 2022-09-03T06:23:42.268 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000028b452350a318f5e-006312f2ee-1444-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:42.270 DEBUG:boto:path=/avysmw-46/key 2022-09-03T06:23:42.270 DEBUG:boto:auth_path=/avysmw-46/key 2022-09-03T06:23:42.271 DEBUG:boto:Method: PUT 2022-09-03T06:23:42.272 DEBUG:boto:Path: /avysmw-46/key 2022-09-03T06:23:42.272 DEBUG:boto:Data: 2022-09-03T06:23:42.273 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'hBotaJrYa9FhFEdFPCLG/A==', 'Content-Length': '4', 'Expect': '100-Continue'} 2022-09-03T06:23:42.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:23:42.274 DEBUG:boto:Port: 8000 2022-09-03T06:23:42.274 DEBUG:boto:Params: {} 2022-09-03T06:23:42.275 DEBUG:boto:Token: None 2022-09-03T06:23:42.275 DEBUG:boto:StringToSign: PUT hBotaJrYa9FhFEdFPCLG/A== application/octet-stream Sat, 03 Sep 2022 06:23:42 GMT /avysmw-46/key 2022-09-03T06:23:42.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EPwbDHI/3NTQNrmOzj5lVY1BC+w= 2022-09-03T06:23:42.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'hBotaJrYa9FhFEdFPCLG/A==', 'Content-Length': '4', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:23:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EPwbDHI/3NTQNrmOzj5lVY1BC+w='} 2022-09-03T06:23:42.288 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"841a2d689ad86bd1611447453c22c6fc"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000001c6d1125074d4280-006312f2ee-1444-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:23:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:42.288 DEBUG:boto:path=/avysmw-47/ 2022-09-03T06:23:42.289 DEBUG:boto:auth_path=/avysmw-47/ 2022-09-03T06:23:42.289 DEBUG:boto:Method: HEAD 2022-09-03T06:23:42.290 DEBUG:boto:Path: /avysmw-47/ 2022-09-03T06:23:42.290 DEBUG:boto:Data: 2022-09-03T06:23:42.291 DEBUG:boto:Headers: {} 2022-09-03T06:23:42.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:42.292 DEBUG:boto:Port: 8000 2022-09-03T06:23:42.292 DEBUG:boto:Params: {} 2022-09-03T06:23:42.293 DEBUG:boto:Token: None 2022-09-03T06:23:42.293 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:42 GMT /avysmw-47/ 2022-09-03T06:23:42.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MRUMdM06S/qSvWunFhYC04KQEpA= 2022-09-03T06:23:42.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MRUMdM06S/qSvWunFhYC04KQEpA=', 'Content-Length': '0'} 2022-09-03T06:23:42.297 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000fea1af2a5763391b-006312f2ee-1590-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:42.298 DEBUG:boto:path=/avysmw-47/key 2022-09-03T06:23:42.298 DEBUG:boto:auth_path=/avysmw-47/key 2022-09-03T06:23:42.299 DEBUG:boto:Method: PUT 2022-09-03T06:23:42.299 DEBUG:boto:Path: /avysmw-47/key 2022-09-03T06:23:42.300 DEBUG:boto:Data: 2022-09-03T06:23:42.300 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'hBotaJrYa9FhFEdFPCLG/A==', 'Content-Length': '4', 'Expect': '100-Continue'} 2022-09-03T06:23:42.301 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:23:42.301 DEBUG:boto:Port: 8000 2022-09-03T06:23:42.301 DEBUG:boto:Params: {} 2022-09-03T06:23:42.302 DEBUG:boto:Token: None 2022-09-03T06:23:42.303 DEBUG:boto:StringToSign: PUT hBotaJrYa9FhFEdFPCLG/A== application/octet-stream Sat, 03 Sep 2022 06:23:42 GMT /avysmw-47/key 2022-09-03T06:23:42.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XPzi9xk3e4EtJWF9sJE27HdZSKA= 2022-09-03T06:23:42.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'hBotaJrYa9FhFEdFPCLG/A==', 'Content-Length': '4', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:23:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XPzi9xk3e4EtJWF9sJE27HdZSKA='} 2022-09-03T06:23:42.313 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"841a2d689ad86bd1611447453c22c6fc"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000003d68c878dee0a043-006312f2ee-1590-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:23:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:42.314 DEBUG:boto:path=/avysmw-48/ 2022-09-03T06:23:42.314 DEBUG:boto:auth_path=/avysmw-48/ 2022-09-03T06:23:42.315 DEBUG:boto:Method: HEAD 2022-09-03T06:23:42.316 DEBUG:boto:Path: /avysmw-48/ 2022-09-03T06:23:42.316 DEBUG:boto:Data: 2022-09-03T06:23:42.317 DEBUG:boto:Headers: {} 2022-09-03T06:23:42.318 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:42.318 DEBUG:boto:Port: 8001 2022-09-03T06:23:42.319 DEBUG:boto:Params: {} 2022-09-03T06:23:42.319 DEBUG:boto:Token: None 2022-09-03T06:23:42.320 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:23:42 GMT /avysmw-48/ 2022-09-03T06:23:42.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zYVKbjxsUHrRYnGDXG3eN4e3VhU= 2022-09-03T06:23:42.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:23:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zYVKbjxsUHrRYnGDXG3eN4e3VhU=', 'Content-Length': '0'} 2022-09-03T06:23:42.323 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ee39ca5cef45dc60-006312f2ee-144a-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:23:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:42.324 DEBUG:boto:path=/avysmw-48/key 2022-09-03T06:23:42.325 DEBUG:boto:auth_path=/avysmw-48/key 2022-09-03T06:23:42.326 DEBUG:boto:Method: PUT 2022-09-03T06:23:42.327 DEBUG:boto:Path: /avysmw-48/key 2022-09-03T06:23:42.327 DEBUG:boto:Data: 2022-09-03T06:23:42.328 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'hBotaJrYa9FhFEdFPCLG/A==', 'Content-Length': '4', 'Expect': '100-Continue'} 2022-09-03T06:23:42.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:23:42.330 DEBUG:boto:Port: 8001 2022-09-03T06:23:42.331 DEBUG:boto:Params: {} 2022-09-03T06:23:42.332 DEBUG:boto:Token: None 2022-09-03T06:23:42.333 DEBUG:boto:StringToSign: PUT hBotaJrYa9FhFEdFPCLG/A== application/octet-stream Sat, 03 Sep 2022 06:23:42 GMT /avysmw-48/key 2022-09-03T06:23:42.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tw/PQwGcIH7/ECVPnoNpS+Mso7k= 2022-09-03T06:23:42.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'hBotaJrYa9FhFEdFPCLG/A==', 'Content-Length': '4', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:23:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tw/PQwGcIH7/ECVPnoNpS+Mso7k='} 2022-09-03T06:23:42.378 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"841a2d689ad86bd1611447453c22c6fc"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000360e94241605f926-006312f2ee-144a-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:23:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:23:42.379 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:42.380 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:42.380 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:42.438 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:42.439 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:42.475 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:23:42.493 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:42.493 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:42.493 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:42.494 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:23:42.494 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:42.494 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:42.495 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:23:42.495 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:42.226042Z" 2022-09-03T06:23:42.495 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:42.495 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:42.496 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:42.496 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:23:42.496 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:42.497 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:42.497 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:42.497 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:23:42.498 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:42.498 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:42.499 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:23:42.500 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:23:42.501 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:42.501 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:42.502 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:42.551 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:42.552 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:42.580 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:42.578+0000 7ff5c611a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:42.580 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:42.579+0000 7ff5c611a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:42.601 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:23:42.601 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:23:42.602 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:23:42.602 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:23:42.602 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:23:42.603 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:23:42.603 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:42.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:42.603 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:23:42.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:42.604 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:23:42.604 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:42.604 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:42.605 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:23:42.605 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:42.605 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:23:42.605 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:42.606 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:23:42.606 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:42.606 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:42.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:42.607 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:42.607 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:23:42.607 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:42.608 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:42.608 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186222.127332_160.1", 2022-09-03T06:23:42.608 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:42.608 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:42.609 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:42.609 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.127332Z", 2022-09-03T06:23:42.609 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:42.609 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:42.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:42.610 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:42.610 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:23:42.611 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:42.611 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:42.611 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:42.611 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:42.612 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:42.612 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:42.612 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:42.612 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:42.613 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:42.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:42.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:42.613 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:23:42.614 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:42.614 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:42.614 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:42.614 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:42.615 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:42.615 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:42.615 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:42.615 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:42.616 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:42.616 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:42.616 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:23:42.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:42.617 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:23:42.617 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:23:42.617 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:23:42.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:42.618 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:23:42Z" 2022-09-03T06:23:42.618 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:23:42.619 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.127332_160.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.127332Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:23:42Z" } 2022-09-03T06:23:42.620 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.127332_160.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.127332Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:23:42.621 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:23:42.621 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.127332_160.1', 2: '', 3: ''} 2022-09-03T06:23:42.622 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186222.226042_172.1 target=1_1662186222.127332_160.1 2022-09-03T06:23:47.623 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:47.625 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:47.626 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:47.684 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:47.685 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:47.715 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:47.713+0000 7f295ac31600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:47.716 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:47.714+0000 7f295ac31600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:47.731 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:23:47.732 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:23:47.732 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:23:47.732 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:23:47.732 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:23:47.733 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:23:47.733 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:47.733 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:47.733 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:23:47.734 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:47.734 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:23:47.734 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:47.734 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:47.735 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:23:47.735 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:47.735 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:23:47.736 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:47.736 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:23:47.736 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:47.736 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:47.737 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:47.737 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:47.737 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:23:47.737 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:47.737 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:47.738 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:23:47.738 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:47.738 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:47.739 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:47.739 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:23:47.739 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:47.739 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:47.740 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:47.740 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:47.740 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:23:47.740 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:47.741 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:47.741 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:47.741 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:47.741 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:47.742 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:47.742 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:47.742 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:47.742 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:47.743 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:47.743 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:47.743 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:23:47.743 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:47.744 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:47.744 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:47.744 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:47.744 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:47.745 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:47.745 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:47.745 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:47.745 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:47.746 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:47.746 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:23:47.746 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:47.746 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:23:47.747 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:23:47.747 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:23:47.747 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:47.748 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:23:47Z" 2022-09-03T06:23:47.748 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:23:47.749 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:23:47Z" } 2022-09-03T06:23:47.750 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:23:47.750 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:23:47.751 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:23:47.751 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:23:47.752 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:23:47.752 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:47.753 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:47.753 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:47.809 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:47.810 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:47.840 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:23:47.838+0000 7f5d372eb600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:47.841 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:23:47.840+0000 7f5d372eb600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:47.858 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:23:47.858 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:23:47.859 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:23:47.859 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:23:47.859 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:23:47.859 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:23:47.860 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:23:47.860 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:47.860 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:23:47.860 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:47.861 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:23:47.861 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:47.861 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:23:47.861 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:23:47.862 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:47.864 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:23:47.864 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:47.865 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:23:47.865 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:23:47.865 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:47.865 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:47.866 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:47.866 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:23:47.866 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:47.867 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:23:47.867 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:23:47.867 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:47.868 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:47.869 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:47.869 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:23:47.870 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:23:47.870 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:47.870 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:47.871 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:47.871 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:23:47.871 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:47.871 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:23:47.872 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:47.872 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:47.872 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:47.872 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:47.873 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:23:47.873 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:23:47.873 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:47.873 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:47.874 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:47.874 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:23:47.874 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:47.874 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:23:47.875 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:23:47.875 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:47.875 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:47.875 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:47.876 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:23:47.876 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:23:47.876 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:47.876 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:47.877 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:23:47.877 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:47.877 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:23:47.877 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:23:47.878 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:23:47.878 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:47.878 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:23:47Z" 2022-09-03T06:23:47.878 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:23:47.879 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:23:47Z" } 2022-09-03T06:23:47.879 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:23:47.880 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:23:47.880 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:23:47.881 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:23:47.882 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:23:47.882 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:47.883 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:47.884 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:47.935 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:47.935 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:47.973 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:47.971+0000 7fa3cd392600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:47.974 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:23:47.972+0000 7fa3cd392600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:23:47.991 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:23:47.991 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:23:47.991 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:23:47.992 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:23:47.992 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:23:47.992 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:23:47.993 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:47.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:47.993 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:23:47.993 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:47.994 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:23:47.994 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:47.994 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:47.995 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:23:48.002 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:48.002 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:23:48.002 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:48.003 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:23:48.003 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:48.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.004 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.004 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:48.004 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:23:48.004 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:48.005 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:48.005 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:23:48.005 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:48.005 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:48.006 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:48.006 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:23:48.006 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:23:48.007 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.007 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:48.007 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:23:48.008 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:48.008 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:48.008 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:48.008 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:48.009 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:48.009 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:48.009 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:48.009 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:48.010 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.010 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.010 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:48.011 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:23:48.011 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:48.011 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:23:48.011 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:23:48.012 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:48.012 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:48.012 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:48.012 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:23:48.013 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:23:48.013 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.013 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.013 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:23:48.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.014 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:23:48.014 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:23:48.014 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:23:48.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.015 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:23:47Z" 2022-09-03T06:23:48.015 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:23:48.016 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:23:47Z" } 2022-09-03T06:23:48.017 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:23:48.018 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:23:48.018 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:23:48.019 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:23:48.019 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone2 2022-09-03T06:23:48.020 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:48.020 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:48.021 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:48.072 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:48.073 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:48.126 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:48.126 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.127 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004600", 2022-09-03T06:23:48.127 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:43.320632Z" 2022-09-03T06:23:48.128 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.128 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.129 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:23:48.129 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:23:48.130 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.130 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.130 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:23:48.131 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:23:48.131 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.132 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.132 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:23:48.132 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:23:48.132 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:48.133 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:48.134 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000004600', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:48.134 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2022-09-03T06:23:48.135 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:48.135 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:48.136 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:48.188 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:48.189 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:48.243 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:23:48.243 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:23:48.244 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:23:48.244 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:23:48.244 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:23:48.245 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 16975738441461511297 2022-09-03T06:23:48.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.245 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:23:48.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:48.246 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:23:48.246 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:48.247 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:23:48.247 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000004600", 2022-09-03T06:23:48.247 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:48.248 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:48.248 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:48.248 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:43.320632Z" 2022-09-03T06:23:48.248 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:48.249 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:23:48.249 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:48.250 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:23:48.250 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:23:48.250 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:48.250 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:48.251 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:48.251 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:23:48.251 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:48.252 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:23:48.252 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:48.253 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:23:48.253 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:23:48.253 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:48.253 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:48.254 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:48.254 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:23:48.254 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:23:48.255 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:23:48.255 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:23:48.255 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:23:48.256 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:23:48.256 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:23:48.256 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:23:48.257 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:23:48.257 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:23:48.257 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.257 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:23:48.258 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:23:48.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.258 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:23:48.258 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:23:48.259 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:23:48.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:23:48.259 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:23:48Z" 2022-09-03T06:23:48.259 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:23:48.261 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 16975738441461511297 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004600", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:43.320632Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.332361Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.293383Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:23:48Z" } 2022-09-03T06:23:48.261 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004600', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:43.320632Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.332361Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.293383Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:23:48.262 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004600', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:48.263 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004600', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:48.263 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2022-09-03T06:23:48.265 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone3 2022-09-03T06:23:48.265 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:48.266 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:48.266 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:48.316 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:48.317 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:48.384 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:23:48.384 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.385 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004600", 2022-09-03T06:23:48.385 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:43.320632Z" 2022-09-03T06:23:48.385 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.386 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.386 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:23:48.386 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:23:48.387 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.387 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.387 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:23:48.387 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:23:48.388 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.388 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.388 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:23:48.389 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:23:48.389 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:48.390 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:23:48.391 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000004600', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:48.392 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2022-09-03T06:23:48.392 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:48.393 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:48.394 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:48.446 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:48.447 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:48.508 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:23:48.509 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:23:48.509 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:23:48.511 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:23:48.511 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:23:48.512 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 11727837677587591728 2022-09-03T06:23:48.512 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.512 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:23:48.513 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.513 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:23:48.514 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:48.515 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:48.515 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004460", 2022-09-03T06:23:48.516 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:48.516 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:48.517 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:48.517 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.283184Z" 2022-09-03T06:23:48.517 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:48.517 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.518 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.518 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:23:48.518 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:48.518 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:48.519 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:23:48.519 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:48.519 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:48.519 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:48.520 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:23:48.520 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:48.520 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.521 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.522 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:23:48.522 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:48.523 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:48.523 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:23:48.524 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:48.524 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:48.525 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:48.525 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:23:48.526 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:48.527 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.527 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:48.528 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:23:48.528 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:48.529 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:48.529 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:23:48.530 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:48.530 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:48.531 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:48.532 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:23:48.532 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:48.533 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:48.533 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:23:48.534 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.534 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:23:48.535 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:23:48.535 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:23:48.536 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:48.536 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:23:48Z" 2022-09-03T06:23:48.537 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:23:48.539 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 11727837677587591728 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004460", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.283184Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.332361Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.293383Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:23:48Z" } 2022-09-03T06:23:48.540 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004460', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.283184Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.332361Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.293383Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:23:48.540 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004600', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:48.541 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004460', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:48.541 WARNING:rgw_multi.tests:data of zone test-zone3 behind zone test-zone1: shard=0 master=00000000000000000000:00000000000000004600 target=00000000000000000000:00000000000000004460 2022-09-03T06:23:53.542 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:53.543 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:53.544 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:53.605 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:53.605 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:53.675 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:23:53.675 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:23:53.676 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:23:53.676 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:23:53.676 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:23:53.676 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 11727837677587591728 2022-09-03T06:23:53.677 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:53.677 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:23:53.677 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:53.677 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:23:53.678 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:53.678 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:53.678 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004460", 2022-09-03T06:23:53.679 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:53.679 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:53.680 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:53.680 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.283184Z" 2022-09-03T06:23:53.680 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:53.681 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:53.681 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:53.681 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:23:53.681 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:53.682 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:53.682 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:23:53.682 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:53.682 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:53.683 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:53.683 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:23:53.683 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:53.683 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:53.684 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:53.684 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:23:53.684 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:53.684 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:53.685 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:23:53.685 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:53.685 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:53.685 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:53.686 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:23:53.686 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:53.686 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:53.687 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:53.687 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:23:53.687 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:53.687 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:53.688 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:23:53.688 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:53.688 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:53.688 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:53.689 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:23:53.689 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:53.689 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:53.689 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:23:53.690 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:53.690 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:23:53.690 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:23:53.690 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:23:53.691 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:53.691 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:23:53Z" 2022-09-03T06:23:53.691 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:23:53.692 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 11727837677587591728 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004460", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.283184Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.332361Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.293383Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:23:53Z" } 2022-09-03T06:23:53.693 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004460', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.283184Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.332361Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.293383Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:23:53.694 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004600', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:53.694 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004460', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:53.695 WARNING:rgw_multi.tests:data of zone test-zone3 behind zone test-zone1: shard=0 master=00000000000000000000:00000000000000004600 target=00000000000000000000:00000000000000004460 2022-09-03T06:23:58.696 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:58.697 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:23:58.698 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:23:58.758 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:23:58.758 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:23:58.826 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:23:58.827 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:23:58.827 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:23:58.828 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:23:58.828 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:23:58.828 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 11727837677587591728 2022-09-03T06:23:58.828 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:58.829 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:23:58.829 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:58.830 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:23:58.830 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:58.830 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:58.831 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004460", 2022-09-03T06:23:58.831 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:58.831 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:58.832 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:58.832 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.283184Z" 2022-09-03T06:23:58.832 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:58.833 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:58.833 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:58.833 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:23:58.833 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:58.834 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:58.834 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:23:58.835 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:58.835 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:58.835 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:58.836 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:23:58.836 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:58.836 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:58.836 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:58.837 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:23:58.837 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:58.837 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:58.837 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:23:58.838 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:58.838 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:58.838 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:58.839 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:23:58.839 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:58.839 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:58.840 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:23:58.840 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:23:58.841 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:23:58.841 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:23:58.841 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:23:58.841 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:23:58.842 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:23:58.842 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:23:58.842 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:23:58.843 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:58.843 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:23:58.843 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:23:58.844 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:58.844 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:23:58.844 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:23:58.844 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:23:58.845 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:23:58.845 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:23:58Z" 2022-09-03T06:23:58.845 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:23:58.846 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 11727837677587591728 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004460", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.283184Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.332361Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.293383Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:23:58Z" } 2022-09-03T06:23:58.847 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004460', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.283184Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.332361Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.293383Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:23:58.848 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004600', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:58.848 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004460', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:23:58.849 WARNING:rgw_multi.tests:data of zone test-zone3 behind zone test-zone1: shard=0 master=00000000000000000000:00000000000000004600 target=00000000000000000000:00000000000000004460 2022-09-03T06:24:03.851 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:03.852 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:03.854 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:03.917 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:03.919 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:03.983 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:24:03.984 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:24:03.984 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:24:03.984 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:24:03.985 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:24:03.985 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 11727837677587591728 2022-09-03T06:24:03.985 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:03.985 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:24:03.986 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:03.986 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:24:03.986 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:03.986 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:03.987 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004740", 2022-09-03T06:24:03.987 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:03.987 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:03.987 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:03.988 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:24:02.298566Z" 2022-09-03T06:24:03.988 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:03.989 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:03.989 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:03.989 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:24:03.989 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:03.990 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:03.990 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:03.990 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:03.990 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:03.991 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:03.991 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:24:03.991 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:03.991 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:03.992 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:03.992 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:24:03.992 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:03.992 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:03.993 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:03.993 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:03.993 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:03.993 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:03.994 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:24:03.994 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:03.994 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:03.995 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:03.995 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:24:03.995 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:03.995 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:03.996 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:03.996 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:03.997 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:03.997 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:03.997 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:24:03.997 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:03.998 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:03.998 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:24:03.998 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:03.999 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:24:03.999 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:24:03.999 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:24:03.999 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.000 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:24:03Z" 2022-09-03T06:24:04.000 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:24:04.002 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 11727837677587591728 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004740", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:24:02.298566Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.332361Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.293383Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:03Z" } 2022-09-03T06:24:04.003 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004740', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:24:02.298566Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.332361Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.293383Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:24:04.003 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004600', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.004 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004740', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.004 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2022-09-03T06:24:04.005 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone4 2022-09-03T06:24:04.006 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.006 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.007 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:04.058 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:04.059 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:04.127 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:24:04.128 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.128 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004740", 2022-09-03T06:24:04.128 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:24:02.298566Z" 2022-09-03T06:24:04.128 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.129 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.129 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:04.129 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:24:04.130 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.130 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.130 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:04.130 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:24:04.131 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.131 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.131 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:04.131 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:24:04.132 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.132 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:24:04.133 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000004740', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.134 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2022-09-03T06:24:04.134 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.135 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.135 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:04.202 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:04.203 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:04.269 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:24:04.270 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:24:04.270 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:24:04.270 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:24:04.271 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:24:04.271 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 6281331864854677188 2022-09-03T06:24:04.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.272 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:24:04.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.272 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:24:04.272 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:24:04.273 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.273 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000004740", 2022-09-03T06:24:04.273 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:24:04.273 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:24:04.274 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:24:04.274 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:24:02.298566Z" 2022-09-03T06:24:04.274 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:04.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.275 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:24:04.275 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:24:04.275 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.276 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:04.276 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:24:04.276 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:24:04.277 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:24:04.277 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:24:04.277 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:04.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.278 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:24:04.279 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:24:04.280 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.280 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:04.280 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:24:04.280 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:24:04.281 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:24:04.281 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:24:04.281 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:04.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.282 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:24:04.282 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:24:04.283 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.283 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:04.284 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:24:04.284 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:24:04.284 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:24:04.284 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:24:04.285 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:04.285 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:04.285 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:24:04.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.286 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:24:04.286 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:24:04.286 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:24:04.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.287 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:24:04Z" 2022-09-03T06:24:04.287 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:24:04.288 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 6281331864854677188 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004740", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:24:02.298566Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.332361Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.293383Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:04Z" } 2022-09-03T06:24:04.289 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004740', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:24:02.298566Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.332361Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.293383Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:24:04.290 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004740', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.290 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004740', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.291 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2022-09-03T06:24:04.291 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone1 2022-09-03T06:24:04.292 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.292 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.293 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:04.343 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:04.344 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:04.388 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:24:04.389 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.390 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000005022", 2022-09-03T06:24:04.390 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:23:44.218003Z" 2022-09-03T06:24:04.390 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.391 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.391 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:04.392 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:04:27.368903Z" 2022-09-03T06:24:04.392 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.392 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.393 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:04.393 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:23:24.194635Z" 2022-09-03T06:24:04.394 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.394 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.395 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:04.396 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:24:04.396 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:04.397 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:24:04.399 DEBUG:rgw_multi.tests:data markers for zone=test-zone2 markers={0: '00000000000000000000:00000000000000005022', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.400 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2022-09-03T06:24:04.401 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.402 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.403 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:04.458 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:04.458 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:04.503 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:24:04.504 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:24:04.504 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:24:04.504 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:24:04.505 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:24:04.505 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 8557007329946676474 2022-09-03T06:24:04.505 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.506 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:24:04.506 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.506 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:24:04.506 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:04.507 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.507 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005022", 2022-09-03T06:24:04.507 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:04.507 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:04.508 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:04.508 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:44.218003Z" 2022-09-03T06:24:04.508 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.508 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.509 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.509 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:24:04.509 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:04.510 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.510 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:04.510 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:04.510 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:04.511 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:04.511 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.368903Z" 2022-09-03T06:24:04.511 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.511 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.512 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.512 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:24:04.512 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:04.512 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.513 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:04.513 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:04.513 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:04.513 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:04.514 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:24.194635Z" 2022-09-03T06:24:04.514 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.514 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.515 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.515 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:24:04.515 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:04.516 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.516 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:04.516 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:04.516 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:04.517 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:04.517 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:24:04.517 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.517 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.518 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:24:04.518 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.518 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:24:04.518 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:24:04.519 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:24:04.519 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.519 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:24:04Z" 2022-09-03T06:24:04.519 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:24:04.521 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 8557007329946676474 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005022", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:44.218003Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.368903Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:24.194635Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:59:59.312039Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:04Z" } 2022-09-03T06:24:04.521 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005022', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:44.218003Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.368903Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:24.194635Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:59:59.312039Z'}}] 2022-09-03T06:24:04.522 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000005022', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.522 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000005022', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.523 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2022-09-03T06:24:04.523 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone3 2022-09-03T06:24:04.524 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.524 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.525 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:04.574 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:04.575 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:04.636 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:24:04.637 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.638 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000005022", 2022-09-03T06:24:04.638 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:23:44.218003Z" 2022-09-03T06:24:04.638 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.639 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:04.640 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:04:27.368903Z" 2022-09-03T06:24:04.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.641 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:04.642 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:23:24.194635Z" 2022-09-03T06:24:04.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:04.643 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:04.643 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:04.644 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:24:04.644 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:04.645 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:24:04.647 DEBUG:rgw_multi.tests:data markers for zone=test-zone2 markers={0: '00000000000000000000:00000000000000005022', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.648 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2022-09-03T06:24:04.648 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.649 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.650 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:04.703 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:04.704 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:04.746 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:24:04.747 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:24:04.747 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:24:04.748 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:24:04.748 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:24:04.748 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 14782160398262406281 2022-09-03T06:24:04.748 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.749 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:24:04.749 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.749 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:24:04.749 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:04.751 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.751 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005022", 2022-09-03T06:24:04.752 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:04.752 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:04.753 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:04.753 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:44.218003Z" 2022-09-03T06:24:04.754 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.754 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.755 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.755 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:24:04.756 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:04.756 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.757 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:04.757 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:04.758 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:04.759 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:04.759 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.368903Z" 2022-09-03T06:24:04.760 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.761 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.761 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.761 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:24:04.762 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:04.763 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.763 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:04.764 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:04.764 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:04.765 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:04.765 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:24.194635Z" 2022-09-03T06:24:04.766 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.766 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.767 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.767 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:24:04.768 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:04.769 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:04.769 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:04.770 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:04.770 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:04.771 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:04.772 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:24:04.772 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.773 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.773 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:24:04.774 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.774 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:24:04.775 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:24:04.775 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:24:04.776 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.776 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:24:04Z" 2022-09-03T06:24:04.777 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:24:04.780 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 14782160398262406281 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005022", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:44.218003Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006011", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.368903Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:24.194635Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:59:59.312039Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:04Z" } 2022-09-03T06:24:04.781 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005022', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:44.218003Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006011', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.368903Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:24.194635Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:59:59.312039Z'}}] 2022-09-03T06:24:04.782 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000005022', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.783 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000005022', 1: '00000000000000000000:00000000000000006011', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.784 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2022-09-03T06:24:04.784 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone4 2022-09-03T06:24:04.785 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone1 2022-09-03T06:24:04.785 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.786 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.787 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:04.839 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:04.840 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:04.896 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:24:04.896 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.897 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004458", 2022-09-03T06:24:04.897 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:24:02.456269Z" 2022-09-03T06:24:04.897 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.897 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.898 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:24:04.898 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:24:04.898 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.899 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.899 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:04.899 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:22.472302Z" 2022-09-03T06:24:04.900 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:04.900 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:04.900 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:04.900 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:24:04.901 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:04.901 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:24:04.903 DEBUG:rgw_multi.tests:data markers for zone=test-zone3 markers={0: '00000000000000000000:00000000000000004458', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:04.904 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2022-09-03T06:24:04.904 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.905 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:04.905 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:04.959 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:04.959 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:05.023 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:24:05.024 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:24:05.024 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:24:05.025 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:24:05.025 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:24:05.026 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 13955113675122108513 2022-09-03T06:24:05.026 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:05.027 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:24:05.027 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:05.028 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:24:05.029 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:05.029 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:05.030 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004318", 2022-09-03T06:24:05.030 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:05.031 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:05.031 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:05.032 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.458495Z" 2022-09-03T06:24:05.032 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:05.033 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:05.033 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:05.034 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:24:05.034 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:05.034 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:05.035 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:24:05.035 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:05.036 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:05.036 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:05.036 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:24:05.036 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:05.037 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:05.037 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:05.037 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:24:05.037 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:05.038 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:05.038 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:05.038 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:05.038 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:05.039 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:05.039 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.472302Z" 2022-09-03T06:24:05.039 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:05.039 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:05.040 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:05.040 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:24:05.040 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:05.041 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:05.041 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:05.041 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:05.041 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:05.042 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:05.042 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:24:05.042 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:05.043 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:05.043 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:24:05.043 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:05.043 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:24:05.044 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:24:05.044 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:24:05.044 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:05.044 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:24:05Z" 2022-09-03T06:24:05.045 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:24:05.046 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 13955113675122108513 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004318", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.458495Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005729", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.402482Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:22.472302Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:00:10.617872Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:05Z" } 2022-09-03T06:24:05.047 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004318', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.458495Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005729', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.402482Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:22.472302Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:00:10.617872Z'}}] 2022-09-03T06:24:05.047 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004458', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:05.048 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004318', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:05.048 WARNING:rgw_multi.tests:data of zone test-zone1 behind zone test-zone3: shard=0 master=00000000000000000000:00000000000000004458 target=00000000000000000000:00000000000000004318 2022-09-03T06:24:10.050 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:10.051 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:10.052 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:10.113 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:10.113 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:10.161 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:24:10.162 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:24:10.162 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:24:10.162 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:24:10.163 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:24:10.163 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 13955113675122108513 2022-09-03T06:24:10.163 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:10.163 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:24:10.164 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:10.164 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:24:10.164 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:10.165 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:10.165 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004318", 2022-09-03T06:24:10.165 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:10.165 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:10.166 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:10.166 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.458495Z" 2022-09-03T06:24:10.166 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:10.166 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:10.167 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:10.167 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:24:10.167 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:10.167 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:10.168 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:24:10.168 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:10.168 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:10.168 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:10.169 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:24:10.169 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:10.169 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:10.169 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:10.170 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:24:10.170 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:10.170 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:10.170 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:10.171 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:10.171 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:10.171 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:10.171 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.472302Z" 2022-09-03T06:24:10.172 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:10.172 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:10.172 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:10.172 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:24:10.173 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:10.173 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:10.173 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:10.173 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:10.174 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:10.174 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:10.174 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:24:10.174 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:10.175 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:10.175 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:24:10.175 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:10.176 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:24:10.176 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:24:10.176 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:24:10.177 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:10.177 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:24:10Z" 2022-09-03T06:24:10.177 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:24:10.178 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 13955113675122108513 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004318", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.458495Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005729", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.402482Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:22.472302Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:00:10.617872Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:10Z" } 2022-09-03T06:24:10.179 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004318', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.458495Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005729', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.402482Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:22.472302Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:00:10.617872Z'}}] 2022-09-03T06:24:10.180 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004458', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:10.180 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004318', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:10.181 WARNING:rgw_multi.tests:data of zone test-zone1 behind zone test-zone3: shard=0 master=00000000000000000000:00000000000000004458 target=00000000000000000000:00000000000000004318 2022-09-03T06:24:15.182 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:15.183 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:15.183 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:15.245 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:15.245 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:15.311 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:24:15.312 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:24:15.312 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:24:15.313 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:24:15.313 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:24:15.314 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 13955113675122108513 2022-09-03T06:24:15.314 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:15.314 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:24:15.315 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:15.315 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:24:15.315 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:15.315 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:15.316 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004318", 2022-09-03T06:24:15.316 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:15.316 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:15.317 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:15.317 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.458495Z" 2022-09-03T06:24:15.317 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:15.318 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:15.318 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:15.318 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:24:15.319 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:15.319 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:15.319 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:24:15.319 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:15.320 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:15.320 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:15.320 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:24:15.320 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:15.321 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:15.321 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:15.321 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:24:15.321 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:15.322 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:15.322 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:15.323 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:15.323 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:15.323 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:15.324 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.472302Z" 2022-09-03T06:24:15.324 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:15.324 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:15.325 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:15.325 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:24:15.325 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:15.325 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:15.326 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:15.326 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:15.326 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:15.327 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:15.327 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:24:15.328 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:15.328 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:15.328 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:24:15.329 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:15.329 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:24:15.329 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:24:15.329 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:24:15.330 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:15.330 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:24:15Z" 2022-09-03T06:24:15.330 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:24:15.332 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 13955113675122108513 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004318", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.458495Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005729", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.402482Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:22.472302Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:00:10.617872Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:15Z" } 2022-09-03T06:24:15.333 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004318', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.458495Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005729', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.402482Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:22.472302Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:00:10.617872Z'}}] 2022-09-03T06:24:15.333 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004458', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:15.334 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004318', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:15.335 WARNING:rgw_multi.tests:data of zone test-zone1 behind zone test-zone3: shard=0 master=00000000000000000000:00000000000000004458 target=00000000000000000000:00000000000000004318 2022-09-03T06:24:20.336 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:20.337 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:20.338 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:20.402 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:20.402 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:20.457 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:24:20.458 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:24:20.459 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:24:20.459 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:24:20.460 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:24:20.460 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 13955113675122108513 2022-09-03T06:24:20.461 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:20.461 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:24:20.462 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:20.462 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:24:20.463 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:20.463 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:20.464 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004318", 2022-09-03T06:24:20.464 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:20.464 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:20.464 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:20.465 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.458495Z" 2022-09-03T06:24:20.465 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:20.465 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:20.465 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:20.466 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:24:20.466 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:20.467 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:20.467 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:24:20.467 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:20.467 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:20.468 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:20.468 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:24:20.468 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:20.469 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:20.469 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:20.469 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:24:20.469 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:20.469 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:20.470 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:20.470 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:20.470 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:20.471 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:20.471 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.472302Z" 2022-09-03T06:24:20.471 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:20.471 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:20.472 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:20.472 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:24:20.472 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:20.472 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:20.473 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:20.473 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:20.473 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:20.473 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:20.474 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:24:20.474 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:20.475 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:20.475 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:24:20.475 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:20.476 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:24:20.476 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:24:20.477 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:24:20.477 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:20.478 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:24:20Z" 2022-09-03T06:24:20.478 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:24:20.479 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 13955113675122108513 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004318", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.458495Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005729", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.402482Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:22.472302Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:00:10.617872Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:20Z" } 2022-09-03T06:24:20.479 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004318', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.458495Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005729', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.402482Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:22.472302Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:00:10.617872Z'}}] 2022-09-03T06:24:20.480 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004458', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:20.480 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004318', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:20.481 WARNING:rgw_multi.tests:data of zone test-zone1 behind zone test-zone3: shard=0 master=00000000000000000000:00000000000000004458 target=00000000000000000000:00000000000000004318 2022-09-03T06:24:25.482 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:25.483 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:25.483 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:25.546 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:25.546 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:25.618 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:24:25.618 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:24:25.618 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:24:25.619 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:24:25.619 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:24:25.619 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 13955113675122108513 2022-09-03T06:24:25.619 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:25.620 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:24:25.620 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:25.621 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:24:25.621 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:25.621 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:25.621 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004598", 2022-09-03T06:24:25.622 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:25.625 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:25.626 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:25.626 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:24:06.230162Z" 2022-09-03T06:24:25.626 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:25.626 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:25.627 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:25.627 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:24:25.627 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:25.627 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:25.628 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:24:25.628 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:25.628 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:25.628 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:25.629 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:24:25.629 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:25.629 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:25.629 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:25.630 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:24:25.631 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:25.631 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:25.631 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:25.632 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:25.632 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:25.633 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:25.633 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:22.472302Z" 2022-09-03T06:24:25.633 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:25.634 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:25.634 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:25.635 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:24:25.635 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:24:25.636 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:24:25.636 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:25.636 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:24:25.637 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:24:25.637 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:24:25.637 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:24:25.638 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:25.638 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:25.638 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:24:25.639 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:25.639 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:24:25.640 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:24:25.640 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:24:25.641 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:25.641 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:24:25Z" 2022-09-03T06:24:25.642 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:24:25.645 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 13955113675122108513 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004598", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:24:06.230162Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005729", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.402482Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:22.472302Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:00:10.617872Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:25Z" } 2022-09-03T06:24:25.646 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004598', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:24:06.230162Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005729', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.402482Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:22.472302Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:00:10.617872Z'}}] 2022-09-03T06:24:25.648 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004458', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:25.649 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004598', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:25.650 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2022-09-03T06:24:25.651 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone2 2022-09-03T06:24:25.652 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:25.653 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:25.654 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:25.706 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:25.707 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:25.765 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:24:25.765 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:25.766 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004598", 2022-09-03T06:24:25.766 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:24:06.230162Z" 2022-09-03T06:24:25.766 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:25.767 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:25.767 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:24:25.767 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:24:25.767 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:25.768 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:25.768 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:25.768 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:22.472302Z" 2022-09-03T06:24:25.768 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:25.769 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:25.769 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:25.769 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:24:25.770 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:25.770 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:24:25.771 DEBUG:rgw_multi.tests:data markers for zone=test-zone3 markers={0: '00000000000000000000:00000000000000004598', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:25.772 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2022-09-03T06:24:25.772 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:25.773 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:25.773 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:25.832 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:25.832 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:25.895 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:24:25.895 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:24:25.895 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:24:25.896 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:24:25.896 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:24:25.896 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 16273219177139511486 2022-09-03T06:24:25.896 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:25.897 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:24:25.897 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:25.897 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:24:25.898 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:24:25.898 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:24:25.898 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000004598", 2022-09-03T06:24:25.899 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:24:25.899 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:24:25.899 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:24:25.900 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:24:06.230162Z" 2022-09-03T06:24:25.900 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:25.900 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:25.901 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:25.901 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:24:25.901 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:24:25.901 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:24:25.901 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000005729", 2022-09-03T06:24:25.902 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:24:25.902 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:24:25.903 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:24:25.903 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:04:27.402482Z" 2022-09-03T06:24:25.904 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:25.905 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:25.905 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:25.906 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:24:25.906 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:24:25.907 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:24:25.907 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:25.908 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:24:25.908 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:24:25.909 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:24:25.909 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:22.472302Z" 2022-09-03T06:24:25.910 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:25.911 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:25.912 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:25.912 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:24:25.913 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:24:25.913 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:24:25.914 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:25.914 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:24:25.915 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:24:25.916 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:24:25.916 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:00:10.617872Z" 2022-09-03T06:24:25.917 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:25.917 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:25.918 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:24:25.918 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:25.919 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:24:25.919 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:24:25.920 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:24:25.920 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:25.921 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:24:25Z" 2022-09-03T06:24:25.922 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:24:25.924 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 16273219177139511486 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004598", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:24:06.230162Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005729", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:04:27.402482Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:22.472302Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:00:10.617872Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:24:25Z" } 2022-09-03T06:24:25.924 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004598', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:24:06.230162Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005729', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:04:27.402482Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:22.472302Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:00:10.617872Z'}}] 2022-09-03T06:24:25.925 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000004598', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:25.926 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004598', 1: '00000000000000000000:00000000000000005729', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:24:25.927 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2022-09-03T06:24:25.928 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone4 2022-09-03T06:24:25.928 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:25.929 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:25.930 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:25.978 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:25.978 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:26.031 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:24:26.031 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:26.032 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004740", 2022-09-03T06:24:26.032 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:24:02.298566Z" 2022-09-03T06:24:26.033 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:26.033 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:26.034 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:26.034 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:24:26.035 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:26.035 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:26.036 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:26.037 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:24:26.037 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:26.037 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:26.038 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:26.038 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:24:26.039 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:26.039 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:24:26.041 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:26.043 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:26.044 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog autotrim --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:24:26.091 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:26.093 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:26.166 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.167 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.168 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 0 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:26.220 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:26.221 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:26.265 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:24:26.268 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.269 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.271 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 1 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:26.322 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:26.323 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:26.382 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:24:26.384 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.384 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.385 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 2 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:26.435 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:26.436 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:26.496 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:24:26.497 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.498 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.499 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 3 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:26.549 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:26.550 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:26.596 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:24:26.599 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.600 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:26.601 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:26.652 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:26.652 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:26.705 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:24:26.705 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:26.706 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000005022", 2022-09-03T06:24:26.706 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:23:44.218003Z" 2022-09-03T06:24:26.707 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:26.707 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:26.707 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:26.708 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:04:27.368903Z" 2022-09-03T06:24:26.708 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:26.708 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:26.709 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:26.709 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:23:24.194635Z" 2022-09-03T06:24:26.710 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:26.710 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:26.711 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:26.711 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:24:26.711 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:26.711 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:24:26.713 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'autotrim', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:26.713 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'autotrim', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:26.714 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog autotrim --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:24:26.759 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:26.760 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:27.844 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:27.845 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:27.846 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog list --shard-id 0 --max-entries 1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:27.939 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:27.939 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:28.004 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:24:28.005 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.006 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.007 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog list --shard-id 1 --max-entries 1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:28.057 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:28.057 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:28.097 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:24:28.099 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.100 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.101 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog list --shard-id 2 --max-entries 1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:28.152 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:28.152 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:28.203 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:24:28.205 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.207 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.207 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog list --shard-id 3 --max-entries 1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:28.258 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:28.259 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:28.328 INFO:teuthology.orchestra.run.smithi087.stdout:[] 2022-09-03T06:24:28.330 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.331 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.331 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:28.387 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:28.387 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:28.462 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:24:28.463 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:28.463 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000004740", 2022-09-03T06:24:28.463 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:24:02.298566Z" 2022-09-03T06:24:28.463 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:28.464 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:28.464 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006011", 2022-09-03T06:24:28.464 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:04:27.332361Z" 2022-09-03T06:24:28.464 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:28.465 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:28.465 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:24:28.465 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:24:28.465 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:24:28.466 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:24:28.466 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:24:28.467 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:24:28.467 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:24:28.467 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:24:28.469 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:28.469 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:28.470 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog autotrim --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:24:28.516 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:28.516 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:28.573 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.574 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.575 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 0 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:28.633 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:28.634 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:28.697 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:24:28.704 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.705 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.706 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 1 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:28.756 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:28.757 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:28.822 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:24:28.825 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.826 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.827 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 2 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:28.879 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:28.879 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:28.924 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:24:28.926 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.927 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:24:28.927 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 3 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:24:28.977 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:28.978 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:29.026 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:24:29.028 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_datalog_autotrim ... ok 2022-09-03T06:24:29.031 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--sync-from-all=false', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:29.032 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--sync-from-all=false', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:29.033 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone modify --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --sync-from-all=false --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:24:29.085 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:29.086 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:29.106 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:24:29.104+0000 7f0bf6256600 0 failed reading obj info from .rgw.root:zone_info.00aa9b6d-7a02-4715-9200-8a26b45eb25e: (2) No such file or directory 2022-09-03T06:24:29.107 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:24:29.104+0000 7f0bf6256600 0 WARNING: could not read zone params for zone id=00aa9b6d-7a02-4715-9200-8a26b45eb25e name=test-zone3 2022-09-03T06:24:29.110 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:24:29.108+0000 7f0bf6256600 0 failed reading obj info from .rgw.root:zone_info.fca49d30-abba-4c2b-8462-66395ac063ba: (2) No such file or directory 2022-09-03T06:24:29.110 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:24:29.108+0000 7f0bf6256600 0 WARNING: could not read zone params for zone id=fca49d30-abba-4c2b-8462-66395ac063ba name=test-zone1 2022-09-03T06:24:29.113 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:24:29.114 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:24:29.114 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:24:29.115 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "test-zone2.rgw.meta:root", 2022-09-03T06:24:29.115 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "test-zone2.rgw.control", 2022-09-03T06:24:29.116 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2022-09-03T06:24:29.116 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2022-09-03T06:24:29.116 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "test-zone2.rgw.log", 2022-09-03T06:24:29.116 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2022-09-03T06:24:29.117 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2022-09-03T06:24:29.117 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2022-09-03T06:24:29.118 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2022-09-03T06:24:29.118 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2022-09-03T06:24:29.118 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2022-09-03T06:24:29.119 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2022-09-03T06:24:29.119 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2022-09-03T06:24:29.119 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "test-zone2.rgw.otp", 2022-09-03T06:24:29.119 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T06:24:29.120 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T06:24:29.120 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T06:24:29.120 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.120 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T06:24:29.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.121 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T06:24:29.121 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:24:29.122 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2022-09-03T06:24:29.123 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T06:24:29.123 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T06:24:29.124 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "test-zone2.rgw.buckets.data", 2022-09-03T06:24:29.124 INFO:teuthology.orchestra.run.smithi087.stdout: "compression_type": "random" 2022-09-03T06:24:29.125 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:29.125 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.125 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2022-09-03T06:24:29.126 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T06:24:29.126 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:29.126 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:29.127 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.127 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:24:29.127 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2022-09-03T06:24:29.128 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:24:29.130 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:29.131 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:24:29.132 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:24:29.181 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:24:29.182 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:24:29.220 INFO:teuthology.orchestra.run.smithi087.stderr:Sending period to new master zone fca49d30-abba-4c2b-8462-66395ac063ba 2022-09-03T06:24:29.251 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:24:29.252 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:24:29.252 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 2, 2022-09-03T06:24:29.253 INFO:teuthology.orchestra.run.smithi087.stdout: "predecessor_uuid": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:24:29.253 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": [ 2022-09-03T06:24:29.254 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:24:29.254 INFO:teuthology.orchestra.run.smithi087.stdout: "1_1662186136.239783_174.1", 2022-09-03T06:24:29.255 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:24:29.255 INFO:teuthology.orchestra.run.smithi087.stdout: "" 2022-09-03T06:24:29.255 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.256 INFO:teuthology.orchestra.run.smithi087.stdout: "period_map": { 2022-09-03T06:24:29.256 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:24:29.257 INFO:teuthology.orchestra.run.smithi087.stdout: "zonegroups": [ 2022-09-03T06:24:29.257 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.258 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "test-zonegroup", 2022-09-03T06:24:29.258 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zonegroup", 2022-09-03T06:24:29.259 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "", 2022-09-03T06:24:29.259 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": true, 2022-09-03T06:24:29.260 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:24:29.261 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:24:29.262 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.262 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2022-09-03T06:24:29.263 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2022-09-03T06:24:29.263 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:24:29.264 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [ 2022-09-03T06:24:29.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.265 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:24:29.265 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone3", 2022-09-03T06:24:29.266 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:24:29.266 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:24:29.267 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.267 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:24:29.268 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:24:29.268 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:24:29.269 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:24:29.269 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:24:29.270 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:24:29.270 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:24:29.271 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:24:29.271 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:24:29.273 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:24:29.274 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:24:29.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.275 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:24:29.275 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone4", 2022-09-03T06:24:29.276 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:24:29.276 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:24:29.277 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.277 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:24:29.278 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:24:29.278 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:24:29.279 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:24:29.279 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "pubsub", 2022-09-03T06:24:29.280 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": false, 2022-09-03T06:24:29.280 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [ 2022-09-03T06:24:29.281 INFO:teuthology.orchestra.run.smithi087.stdout: "test-zone1" 2022-09-03T06:24:29.281 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.282 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:24:29.282 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:24:29.282 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:24:29.282 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:24:29.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.283 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:24:29.283 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:24:29.284 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:24:29.284 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:24:29.284 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.284 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:24:29.285 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:24:29.285 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:24:29.285 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:24:29.285 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:24:29.286 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": false, 2022-09-03T06:24:29.286 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:24:29.286 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:24:29.287 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:24:29.287 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:24:29.287 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:24:29.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.288 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:24:29.288 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone1", 2022-09-03T06:24:29.289 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:24:29.289 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:24:29.289 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.290 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:24:29.290 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:24:29.290 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:24:29.290 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:24:29.291 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:24:29.291 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:24:29.291 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:24:29.292 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:24:29.293 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:24:29.294 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:24:29.295 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:24:29.295 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:29.296 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.297 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [ 2022-09-03T06:24:29.297 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.298 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default-placement", 2022-09-03T06:24:29.300 INFO:teuthology.orchestra.run.smithi087.stdout: "tags": [], 2022-09-03T06:24:29.301 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": [ 2022-09-03T06:24:29.301 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD" 2022-09-03T06:24:29.302 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:24:29.302 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:29.303 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.303 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "default-placement", 2022-09-03T06:24:29.304 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:24:29.304 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2022-09-03T06:24:29.305 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2022-09-03T06:24:29.305 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.305 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled_features": [ 2022-09-03T06:24:29.306 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:24:29.306 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:24:29.307 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:29.308 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:24:29.308 INFO:teuthology.orchestra.run.smithi087.stdout: "short_zone_ids": [ 2022-09-03T06:24:29.308 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.309 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:24:29.309 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 700722763 2022-09-03T06:24:29.309 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.310 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.310 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:24:29.310 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 2398203312 2022-09-03T06:24:29.310 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.311 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.311 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:24:29.311 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 3737622162 2022-09-03T06:24:29.311 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.312 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:24:29.312 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:24:29.312 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 449119331 2022-09-03T06:24:29.312 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:29.313 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:24:29.313 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.314 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:24:29.314 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:24:29.315 INFO:teuthology.orchestra.run.smithi087.stdout: "period_config": { 2022-09-03T06:24:29.315 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_quota": { 2022-09-03T06:24:29.316 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:24:29.316 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:24:29.317 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:24:29.317 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:24:29.317 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:24:29.318 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.318 INFO:teuthology.orchestra.run.smithi087.stdout: "user_quota": { 2022-09-03T06:24:29.319 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:24:29.319 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:24:29.319 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:24:29.320 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:24:29.320 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:24:29.321 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.321 INFO:teuthology.orchestra.run.smithi087.stdout: "user_ratelimit": { 2022-09-03T06:24:29.321 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:24:29.321 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:24:29.322 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:24:29.322 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:24:29.322 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:24:29.323 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.323 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_ratelimit": { 2022-09-03T06:24:29.324 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:24:29.324 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:24:29.324 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:24:29.324 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:24:29.325 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:24:29.325 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.325 INFO:teuthology.orchestra.run.smithi087.stdout: "anonymous_ratelimit": { 2022-09-03T06:24:29.325 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:24:29.326 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:24:29.326 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:24:29.326 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:24:29.327 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:24:29.327 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:24:29.328 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:24:29.328 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:24:29.329 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_name": "test-realm", 2022-09-03T06:24:29.329 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:24:29.332 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:24:29.333 INFO:rgw_multi.tests:Set sync_from_all flag on zone test-zone2 to false 2022-09-03T06:25:29.334 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-49 2022-09-03T06:25:29.337 DEBUG:boto:path=/avysmw-49/ 2022-09-03T06:25:29.338 DEBUG:boto:auth_path=/avysmw-49/ 2022-09-03T06:25:29.338 DEBUG:boto:Method: PUT 2022-09-03T06:25:29.339 DEBUG:boto:Path: /avysmw-49/ 2022-09-03T06:25:29.340 DEBUG:boto:Data: 2022-09-03T06:25:29.340 DEBUG:boto:Headers: {} 2022-09-03T06:25:29.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:25:29.341 DEBUG:boto:Port: 8000 2022-09-03T06:25:29.341 DEBUG:boto:Params: {} 2022-09-03T06:25:29.342 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:25:29.343 DEBUG:boto:Token: None 2022-09-03T06:25:29.344 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:25:29 GMT /avysmw-49/ 2022-09-03T06:25:29.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:viviIn2WzeqvOd9kwGcRk3FJli4= 2022-09-03T06:25:29.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:25:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:viviIn2WzeqvOd9kwGcRk3FJli4=', 'Content-Length': '0'} 2022-09-03T06:25:29.372 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000004af0ee16737ba978-006312f359-1567-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:25:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:25:29.373 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:25:29.373 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:25:29.374 DEBUG:boto:Method: PUT 2022-09-03T06:25:29.375 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:25:29.375 DEBUG:boto:Data: 2022-09-03T06:25:29.376 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '3FCGuEcom6i4veFJuDiBdQ==', 'Content-Length': '512', 'Expect': '100-Continue'} 2022-09-03T06:25:29.376 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:25:29.377 DEBUG:boto:Port: 8000 2022-09-03T06:25:29.377 DEBUG:boto:Params: {} 2022-09-03T06:25:29.378 DEBUG:boto:Token: None 2022-09-03T06:25:29.379 DEBUG:boto:StringToSign: PUT 3FCGuEcom6i4veFJuDiBdQ== application/octet-stream Sat, 03 Sep 2022 06:25:29 GMT /avysmw-49/testredirect 2022-09-03T06:25:29.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SVRKYbgjpNgaE2R8jrLtr2FK+Ik= 2022-09-03T06:25:29.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '3FCGuEcom6i4veFJuDiBdQ==', 'Content-Length': '512', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:25:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SVRKYbgjpNgaE2R8jrLtr2FK+Ik='} 2022-09-03T06:25:29.389 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"dc5086b847289ba8b8bde149b8388175"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000a2edbfd941e83409-006312f359-1567-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:25:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:25:29.390 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:29.391 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:29.392 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:25:29.448 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:25:29.449 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:25:29.478 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:25:29.491 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:25:29.492 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:25:29.492 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:25:29.492 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:25:29.493 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:25:29.493 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:25:29.493 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:25:29.493 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:25:29.367916Z" 2022-09-03T06:25:29.494 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:25:29.494 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:25:29.494 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:25:29.494 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:25:29.495 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:25:29.495 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:25:29.495 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:25:29.496 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:25:29.496 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:25:29.496 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:25:29.498 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:25:29.498 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:25:29.499 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:29.500 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:29.500 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:25:29.558 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:25:29.559 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:25:29.601 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:29.599+0000 7f4774cf4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:29.602 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:29.600+0000 7f4774cf4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:29.623 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:25:29.624 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:25:29.625 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:25:29.625 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:25:29.625 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:25:29.626 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:25:29.626 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:29.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:29.627 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:25:29.627 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:29.628 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:25:29.628 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:29.629 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:29.629 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:25:29.630 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:29.631 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:25:29.631 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:29.632 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:25:29.632 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:29.632 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:29.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:29.633 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:29.635 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:25:29.635 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:29.636 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:29.636 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:25:29.637 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:29.638 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:29.638 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:29.639 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:25:29.639 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:29.640 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:29.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:29.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:29.641 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:25:29.642 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:29.642 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:29.643 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:29.643 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:29.644 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:29.644 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:29.645 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:29.645 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:29.646 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:29.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:29.647 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:29.647 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:25:29.648 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:29.648 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:29.649 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:29.649 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:29.650 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:29.650 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:29.651 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:29.651 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:29.652 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:29.652 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:29.653 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:25:29.653 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:29.654 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:25:29.654 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:25:29.655 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:25:29.655 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:29.656 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:25:29Z" 2022-09-03T06:25:29.656 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:25:29.658 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:25:29Z" } 2022-09-03T06:25:29.659 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:25:29.660 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:25:29.661 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:25:29.662 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:25:34.664 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:34.665 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:34.667 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:25:34.726 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:25:34.727 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:25:34.767 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:34.764+0000 7f22096ca600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:34.768 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:34.765+0000 7f22096ca600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:34.783 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:25:34.783 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:25:34.783 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:25:34.784 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:25:34.784 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:25:34.784 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:25:34.785 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:34.785 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:34.785 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:25:34.785 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:34.786 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:25:34.786 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:34.786 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:34.786 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:25:34.787 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:34.787 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:25:34.787 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:34.788 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:25:34.788 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:34.788 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:34.789 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:34.789 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:34.789 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:25:34.789 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:34.790 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:34.790 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:25:34.790 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:34.790 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:34.791 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:34.791 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:25:34.791 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:34.792 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:34.792 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:34.792 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:34.793 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:25:34.793 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:34.793 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:34.794 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:34.794 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:34.794 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:34.794 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:34.795 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:34.795 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:34.795 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:34.796 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:34.796 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:34.796 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:25:34.797 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:34.797 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:34.797 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:34.797 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:34.798 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:34.798 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:34.798 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:34.799 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:34.799 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:34.799 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:34.800 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:25:34.800 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:34.800 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:25:34.800 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:25:34.801 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:25:34.801 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:34.801 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:25:34Z" 2022-09-03T06:25:34.802 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:25:34.802 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:25:34Z" } 2022-09-03T06:25:34.803 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:25:34.804 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:25:34.804 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:25:34.805 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:25:39.806 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:39.807 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:39.807 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:25:39.866 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:25:39.867 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:25:39.905 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:39.903+0000 7f8969a64600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:39.907 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:39.904+0000 7f8969a64600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:39.926 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:25:39.927 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:25:39.927 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:25:39.928 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:25:39.928 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:25:39.928 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:25:39.929 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:39.929 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:39.929 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:25:39.930 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:39.930 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:25:39.930 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:39.930 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:39.931 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:25:39.931 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:39.931 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:25:39.933 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:39.933 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:25:39.934 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:39.934 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:39.934 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:39.935 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:39.935 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:25:39.935 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:39.935 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:39.936 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:25:39.936 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:39.936 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:39.937 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:39.937 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:25:39.937 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:39.943 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:39.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:39.944 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:39.944 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:25:39.944 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:39.945 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:39.945 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:39.945 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:39.945 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:39.946 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:39.946 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:39.946 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:39.947 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:39.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:39.947 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:39.947 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:25:39.948 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:39.948 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:39.948 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:39.949 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:39.949 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:39.949 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:39.950 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:39.950 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:39.950 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:39.951 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:39.951 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:25:39.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:39.952 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:25:39.952 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:25:39.952 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:25:39.953 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:39.953 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:25:39Z" 2022-09-03T06:25:39.953 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:25:39.955 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:25:39Z" } 2022-09-03T06:25:39.956 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:25:39.957 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:25:39.957 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:25:39.958 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:25:44.960 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:44.961 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:44.961 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:25:45.019 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:25:45.019 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:25:45.065 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:45.063+0000 7fcd2549a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:45.066 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:45.064+0000 7fcd2549a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:45.088 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:25:45.088 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:25:45.088 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:25:45.089 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:25:45.089 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:25:45.089 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:25:45.090 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:45.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:45.091 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:25:45.091 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:45.092 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:25:45.092 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:45.093 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:45.093 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:25:45.094 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:45.094 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:25:45.095 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:45.095 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:25:45.095 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:45.096 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:45.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:45.097 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:45.097 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:25:45.098 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:45.098 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:45.098 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:25:45.099 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:45.099 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:45.099 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:45.099 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:25:45.100 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:45.100 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:45.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:45.101 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:45.101 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:25:45.101 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:45.102 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:45.102 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:45.102 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:45.102 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:45.103 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:45.103 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:45.103 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:45.103 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:45.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:45.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:45.104 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:25:45.105 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:45.105 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:45.105 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:45.105 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:45.106 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:45.106 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:45.106 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:45.107 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:45.107 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:45.107 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:45.108 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:25:45.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:45.108 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:25:45.109 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:25:45.109 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:25:45.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:45.110 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:25:45Z" 2022-09-03T06:25:45.111 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:25:45.111 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:25:45Z" } 2022-09-03T06:25:45.112 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:25:45.113 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:25:45.114 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:25:45.115 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:25:50.116 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:50.117 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:50.118 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:25:50.182 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:25:50.183 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:25:50.226 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:50.224+0000 7f848f730600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:50.227 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:50.225+0000 7f848f730600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:50.241 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:25:50.241 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:25:50.242 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:25:50.242 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:25:50.242 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:25:50.243 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:25:50.243 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:50.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:50.244 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:25:50.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:50.244 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:25:50.244 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:50.245 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:50.245 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:25:50.245 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:50.246 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:25:50.246 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:50.246 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:25:50.246 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:50.247 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:50.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:50.247 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:50.248 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:25:50.248 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:50.248 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:50.248 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:25:50.249 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:50.249 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:50.249 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:50.249 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:25:50.250 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:50.250 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:50.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:50.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:50.251 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:25:50.251 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:50.252 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:50.252 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:50.252 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:50.252 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:50.253 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:50.253 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:50.253 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:50.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:50.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:50.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:50.254 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:25:50.254 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:50.255 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:50.255 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:50.255 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:50.255 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:50.256 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:50.256 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:50.256 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:50.256 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:50.257 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:50.257 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:25:50.257 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:50.257 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:25:50.258 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:25:50.258 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:25:50.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:50.258 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:25:50Z" 2022-09-03T06:25:50.259 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:25:50.260 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:25:50Z" } 2022-09-03T06:25:50.261 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:25:50.261 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:25:50.262 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:25:50.262 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:25:55.263 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:55.264 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:25:55.265 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:25:55.322 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:25:55.322 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:25:55.355 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:55.354+0000 7f58c2927600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:55.357 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:25:55.355+0000 7f58c2927600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:25:55.379 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:25:55.380 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:25:55.381 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:25:55.381 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:25:55.383 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:25:55.383 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:25:55.384 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:55.384 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:55.384 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:25:55.385 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:55.385 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:25:55.386 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:55.386 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:55.387 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:25:55.387 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:55.388 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:25:55.388 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:55.389 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:25:55.389 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:55.389 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:55.390 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:55.390 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:55.391 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:25:55.392 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:55.392 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:55.392 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:25:55.392 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:55.393 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:55.393 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:55.393 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:25:55.393 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:25:55.394 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:55.394 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:55.394 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:55.395 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:25:55.395 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:55.396 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:55.396 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:55.396 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:55.397 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:55.397 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:55.397 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:55.397 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:55.398 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:55.398 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:55.398 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:25:55.398 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:25:55.399 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:25:55.399 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:25:55.399 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:25:55.400 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:25:55.400 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:25:55.401 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:25:55.401 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:25:55.402 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:25:55.402 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:55.403 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:25:55.403 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:25:55.403 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:55.404 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:25:55.404 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:25:55.405 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:25:55.405 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:25:55.406 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:25:55Z" 2022-09-03T06:25:55.406 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:25:55.408 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:25:55Z" } 2022-09-03T06:25:55.409 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:25:55.410 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:25:55.411 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:25:55.411 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:26:00.413 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:00.415 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:00.415 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:26:00.474 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:00.475 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:00.518 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:00.515+0000 7fa43ef09600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:00.519 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:00.516+0000 7fa43ef09600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:00.531 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:26:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:26:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:26:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:26:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:26:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:26:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:26:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:26:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:26:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:26:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:26:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:26:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:26:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.360630_183.1", 2022-09-03T06:26:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:26:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.360630Z", 2022-09-03T06:26:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:26:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:26:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:26:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:26:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:26:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:26:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:26:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:26:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:26:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:26:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:26:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:26:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:26:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:26:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:26:00Z" 2022-09-03T06:26:00.556 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:26:00.558 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.360630_183.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.360630Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:26:00Z" } 2022-09-03T06:26:00.559 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.360630_183.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.360630Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:26:00.560 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:00.561 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.360630_183.1', 2: '', 3: ''} 2022-09-03T06:26:00.562 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186329.360630_183.1 2022-09-03T06:26:05.563 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:05.566 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:05.567 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:26:05.626 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:05.627 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:05.668 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:05.666+0000 7febd9b0b600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:05.669 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:05.667+0000 7febd9b0b600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:05.694 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:26:05.695 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:26:05.695 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:26:05.695 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:26:05.695 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:26:05.696 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:05.696 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:26:05.696 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:05.696 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:26:05.697 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:05.697 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:26:05.697 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:05.698 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:05.698 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:26:05.698 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:05.699 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:26:05.699 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:05.699 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:26:05.699 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:26:05.700 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:05.700 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:05.700 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:05.700 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:26:05.701 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:05.701 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:05.701 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:26:05.702 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:05.702 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:26:05.702 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:05.702 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:26:05.703 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:26:05.703 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:05.704 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:05.704 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:05.705 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:26:05.705 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:05.705 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:05.706 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:26:05.706 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:05.706 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:26:05.706 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:05.707 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:26:05.707 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:26:05.707 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:05.708 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:05.708 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:05.708 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:26:05.708 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:05.709 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:05.709 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:26:05.709 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:05.709 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:26:05.710 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:05.710 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:26:05.710 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:26:05.710 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:05.711 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:05.711 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:05.711 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:05.712 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:26:05.713 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:26:05.713 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:26:05.713 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:05.713 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:26:05Z" 2022-09-03T06:26:05.714 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:26:05.715 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:26:05Z" } 2022-09-03T06:26:05.716 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:26:05.716 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:05.717 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:05.717 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:26:05.718 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:26:05.718 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:05.719 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:05.720 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:26:05.779 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:05.779 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:05.827 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:05.824+0000 7efcd21d6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:05.828 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:05.825+0000 7efcd21d6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:05.850 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:26:05.851 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:26:05.851 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:26:05.851 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:26:05.852 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:26:05.852 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:05.852 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:05.852 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:05.853 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:26:05.853 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:05.853 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:26:05.854 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:05.854 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:05.855 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:26:05.855 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:05.856 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:26:05.857 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:05.857 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:26:05.857 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:05.857 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:05.858 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:05.858 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:05.858 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:26:05.859 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:05.859 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:05.859 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:26:05.860 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:05.860 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:05.860 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:05.860 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:26:05.861 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:05.861 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:05.861 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:05.862 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:05.862 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:26:05.862 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:05.863 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:05.863 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:05.863 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:05.863 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:05.864 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:05.864 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:05.864 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:05.865 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:05.865 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:05.865 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:05.865 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:26:05.866 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:05.866 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:05.866 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:05.866 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:05.867 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:05.867 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:05.867 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:05.868 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:05.868 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:05.868 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:05.868 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:26:05.869 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:05.869 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:26:05.869 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:26:05.870 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:26:05.870 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:05.870 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:26:05Z" 2022-09-03T06:26:05.870 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:26:05.871 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:26:05Z" } 2022-09-03T06:26:05.872 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:26:05.872 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:05.873 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:26:05.873 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:26:10.877 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:10.888 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:10.889 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:26:10.951 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:10.952 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:10.985 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:10.980+0000 7fb4b6c5a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:10.986 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:10.981+0000 7fb4b6c5a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:10.999 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:26:11.000 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:26:11.000 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:26:11.001 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:26:11.001 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:26:11.002 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:11.002 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:11.003 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:11.005 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:26:11.006 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:11.006 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:26:11.007 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:11.007 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:11.007 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:26:11.008 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:11.008 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:26:11.008 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:11.008 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:26:11.009 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:11.009 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:11.009 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:11.010 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:11.010 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:26:11.010 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:11.011 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:11.011 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:26:11.011 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:11.012 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:11.013 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:11.013 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:26:11.014 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:11.014 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:11.015 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:11.016 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:11.017 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:26:11.017 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:11.017 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:11.018 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:11.018 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:11.018 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:11.019 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:11.020 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:11.020 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:11.020 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:11.021 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:11.021 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:11.021 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:26:11.022 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:11.022 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:11.023 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:11.023 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:11.023 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:11.024 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:11.024 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:11.024 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:11.024 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:11.025 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:11.025 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:26:11.025 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:11.025 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:26:11.026 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:26:11.026 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:26:11.027 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:11.027 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:26:10Z" 2022-09-03T06:26:11.028 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:26:11.028 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:26:10Z" } 2022-09-03T06:26:11.029 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:26:11.031 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:11.031 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:26:11.032 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:26:16.033 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:16.037 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:16.047 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:26:16.109 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:16.113 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:16.153 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:16.146+0000 7f7446634600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:16.154 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:16.147+0000 7f7446634600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:16.170 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:26:16.170 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:26:16.171 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:26:16.171 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:26:16.172 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:26:16.172 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:16.173 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:16.173 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:16.174 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:26:16.175 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:16.175 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:26:16.176 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:16.176 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:16.177 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:26:16.177 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:16.179 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:26:16.180 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:16.181 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:26:16.181 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:16.184 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:16.185 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:16.186 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:16.187 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:26:16.187 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:16.188 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:16.188 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:26:16.189 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:16.189 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:16.190 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:16.192 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:26:16.192 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:16.193 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:16.193 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:16.193 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:16.194 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:26:16.195 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:16.196 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:16.197 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:16.197 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:16.197 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:16.198 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:16.198 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:16.198 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:16.199 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:16.199 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:16.199 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:16.200 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:26:16.200 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:16.200 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:16.201 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:16.201 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:16.203 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:16.203 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:16.204 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:16.204 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:16.205 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:16.205 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:16.206 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:26:16.214 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:16.215 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:26:16.215 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:26:16.216 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:26:16.217 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:16.217 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:26:16Z" 2022-09-03T06:26:16.218 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:26:16.219 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:26:16Z" } 2022-09-03T06:26:16.220 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:26:16.222 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:16.223 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:26:16.225 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:26:21.233 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:21.234 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:21.235 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:26:21.306 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:21.307 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:21.339 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:21.337+0000 7efcb0c2a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:21.340 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:21.338+0000 7efcb0c2a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:21.353 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:26:21.353 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:26:21.353 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:26:21.354 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:26:21.354 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:26:21.354 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:21.355 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:21.355 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:21.356 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:26:21.356 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:21.356 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:26:21.356 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:21.357 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:21.357 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:26:21.357 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:21.358 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:26:21.358 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:21.358 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:26:21.359 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:21.359 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:21.368 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:21.369 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:21.369 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:26:21.370 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:21.370 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:21.371 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:26:21.371 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:21.372 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:21.372 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:21.374 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:26:21.375 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:21.375 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:21.375 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:21.376 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:21.377 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:26:21.377 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:21.377 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:21.378 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:21.378 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:21.379 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:21.379 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:21.380 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:21.380 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:21.380 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:21.381 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:21.381 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:21.382 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:26:21.382 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:21.383 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:21.383 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:21.384 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:21.384 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:21.385 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:21.386 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:21.386 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:21.387 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:21.387 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:21.388 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:26:21.388 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:21.389 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:26:21.389 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:26:21.390 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:26:21.390 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:21.391 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:26:21Z" 2022-09-03T06:26:21.391 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:26:21.394 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:26:21Z" } 2022-09-03T06:26:21.395 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:26:21.396 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:21.397 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:26:21.398 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:26:26.400 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:26.401 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:26.403 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:26:26.468 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:26.468 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:26.505 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:26.496+0000 7fca1abb6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:26.505 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:26.497+0000 7fca1abb6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:26.514 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:26:26.515 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:26:26.515 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:26:26.516 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:26:26.516 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:26:26.516 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:26.517 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:26.517 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:26.517 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:26:26.517 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:26.518 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:26:26.518 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:26.519 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:26.519 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:26:26.520 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:26.520 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:26:26.520 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:26.521 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:26:26.521 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:26.521 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:26.522 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:26.522 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:26.523 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:26:26.523 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:26.524 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:26.524 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186222.226042_172.1", 2022-09-03T06:26:26.525 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:26.525 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:26.525 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:26.525 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:42.226042Z", 2022-09-03T06:26:26.526 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:26.527 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:26.527 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:26.527 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:26.528 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:26:26.528 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:26.528 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:26.529 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:26.529 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:26.530 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:26.530 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:26.531 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:26.531 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:26.543 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:26.543 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:26.544 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:26.544 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:26:26.545 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:26.545 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:26.546 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:26.546 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:26.547 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:26.547 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:26.548 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:26.548 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:26.549 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:26.549 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:26.550 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:26:26.550 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:26.551 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:26:26.551 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:26:26.552 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:26:26.552 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:26.553 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:26:26Z" 2022-09-03T06:26:26.553 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:26:26.567 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186222.226042_172.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.226042Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:26:26Z" } 2022-09-03T06:26:26.571 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186222.226042_172.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.226042Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:26:26.572 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:26.573 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186222.226042_172.1', 2: '', 3: ''} 2022-09-03T06:26:26.574 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186329.367916_186.1 target=1_1662186222.226042_172.1 2022-09-03T06:26:31.579 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:31.580 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:31.583 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:26:31.643 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:31.644 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:31.684 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:31.682+0000 7fb667092600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:31.685 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:26:31.683+0000 7fb667092600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:31.702 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:26:31.703 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:26:31.703 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:26:31.703 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:26:31.704 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:26:31.704 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:31.704 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:31.705 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:31.705 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:26:31.705 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:31.705 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:26:31.706 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:31.706 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:31.706 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:26:31.707 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:31.707 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:26:31.707 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:31.707 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:26:31.708 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:31.708 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:31.709 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:31.709 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:31.709 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:26:31.710 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:31.710 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:31.710 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:26:31.710 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:31.711 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:31.711 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:31.711 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:26:31.712 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:26:31.712 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:31.712 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:31.712 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:31.713 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:26:31.713 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:31.713 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:31.713 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:31.714 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:31.714 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:31.714 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:31.715 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:31.715 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:31.715 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:31.715 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:31.716 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:26:31.716 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:26:31.716 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:26:31.716 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:26:31.717 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:26:31.717 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:26:31.717 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:26:31.717 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:26:31.718 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:26:31.718 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:26:31.718 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:31.719 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:26:31.719 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:26:31.720 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:31.720 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:26:31.720 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:26:31.720 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:26:31.721 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:26:31.721 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:26:31Z" 2022-09-03T06:26:31.721 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:26:31.722 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:26:31Z" } 2022-09-03T06:26:31.724 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:26:31.725 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:31.726 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:31.727 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:26:31.727 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:26:31.728 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:31.728 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:26:31.729 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:26:31.792 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:31.793 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:31.838 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:31.835+0000 7f061fe4b600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:31.839 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:31.836+0000 7f061fe4b600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:26:31.863 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:26:31.864 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:26:31.864 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:26:31.865 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:26:31.865 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:26:31.866 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:31.866 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:26:31.867 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:31.867 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:26:31.868 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:31.868 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:26:31.868 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:31.869 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:31.869 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:26:31.870 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:31.870 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:26:31.870 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:31.871 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:26:31.871 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:26:31.872 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:31.872 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:31.873 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:31.873 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:26:31.873 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:31.873 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:31.874 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:26:31.874 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:31.874 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:26:31.874 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:31.875 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:26:31.875 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:26:31.875 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:31.876 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:31.876 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:31.877 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:26:31.877 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:31.877 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:31.878 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:26:31.878 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:31.879 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:26:31.879 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:31.880 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:26:31.880 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:26:31.881 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:31.881 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:31.882 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:31.882 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:26:31.883 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:31.883 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:26:31.884 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:26:31.884 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:26:31.885 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:26:31.885 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:26:31.885 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:26:31.886 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:26:31.886 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:31.887 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:31.888 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:31.888 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:31.889 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:26:31.889 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:26:31.889 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:26:31.890 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:31.890 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:26:31Z" 2022-09-03T06:26:31.891 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:26:31.891 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:26:31Z" } 2022-09-03T06:26:31.892 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:26:31.893 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:31.894 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:26:31.895 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:26:31.896 DEBUG:boto:path=/avysmw-49/ 2022-09-03T06:26:31.897 DEBUG:boto:auth_path=/avysmw-49/ 2022-09-03T06:26:31.898 DEBUG:boto:Method: HEAD 2022-09-03T06:26:31.899 DEBUG:boto:Path: /avysmw-49/ 2022-09-03T06:26:31.899 DEBUG:boto:Data: 2022-09-03T06:26:31.900 DEBUG:boto:Headers: {} 2022-09-03T06:26:31.901 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:26:31.902 DEBUG:boto:Port: 8000 2022-09-03T06:26:31.903 DEBUG:boto:Params: {} 2022-09-03T06:26:31.905 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:26:31.906 DEBUG:boto:Token: None 2022-09-03T06:26:31.907 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:26:31 GMT /avysmw-49/ 2022-09-03T06:26:31.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QrtbGZ0upmAryibGROiIWZC7faY= 2022-09-03T06:26:31.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:26:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QrtbGZ0upmAryibGROiIWZC7faY=', 'Content-Length': '0'} 2022-09-03T06:26:31.914 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002af3b8ed22156adc-006312f397-165e-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:26:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:26:31.916 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:26:31.918 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:26:31.919 DEBUG:boto:Method: HEAD 2022-09-03T06:26:31.920 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:26:31.920 DEBUG:boto:Data: 2022-09-03T06:26:31.921 DEBUG:boto:Headers: {} 2022-09-03T06:26:31.921 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:26:31.922 DEBUG:boto:Port: 8000 2022-09-03T06:26:31.923 DEBUG:boto:Params: {} 2022-09-03T06:26:31.923 DEBUG:boto:Token: None 2022-09-03T06:26:31.924 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:26:31 GMT /avysmw-49/testredirect 2022-09-03T06:26:31.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KeiDcom4BYQoqVyHMbMnPqC9+Ac= 2022-09-03T06:26:31.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:26:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KeiDcom4BYQoqVyHMbMnPqC9+Ac=', 'Content-Length': '0'} 2022-09-03T06:26:31.928 DEBUG:boto:Response headers: [('Content-Length', '229'), ('x-amz-request-id', 'tx000005d47531576236afd-006312f397-165e-test-zone2'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:26:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:26:31.928 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--redirect-zone=fca49d30-abba-4c2b-8462-66395ac063ba', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:26:31.929 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--redirect-zone=fca49d30-abba-4c2b-8462-66395ac063ba', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:26:31.930 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone modify --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --redirect-zone=fca49d30-abba-4c2b-8462-66395ac063ba --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:26:31.979 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:31.980 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:32.001 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:31.998+0000 7fe226b94600 0 failed reading obj info from .rgw.root:zone_info.00aa9b6d-7a02-4715-9200-8a26b45eb25e: (2) No such file or directory 2022-09-03T06:26:32.001 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:31.998+0000 7fe226b94600 0 WARNING: could not read zone params for zone id=00aa9b6d-7a02-4715-9200-8a26b45eb25e name=test-zone3 2022-09-03T06:26:32.002 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:32.000+0000 7fe226b94600 0 failed reading obj info from .rgw.root:zone_info.fca49d30-abba-4c2b-8462-66395ac063ba: (2) No such file or directory 2022-09-03T06:26:32.003 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:26:32.000+0000 7fe226b94600 0 WARNING: could not read zone params for zone id=fca49d30-abba-4c2b-8462-66395ac063ba name=test-zone1 2022-09-03T06:26:32.008 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:26:32.008 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:26:32.009 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:26:32.009 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "test-zone2.rgw.meta:root", 2022-09-03T06:26:32.009 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "test-zone2.rgw.control", 2022-09-03T06:26:32.010 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2022-09-03T06:26:32.010 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2022-09-03T06:26:32.011 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "test-zone2.rgw.log", 2022-09-03T06:26:32.012 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2022-09-03T06:26:32.012 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2022-09-03T06:26:32.013 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2022-09-03T06:26:32.013 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2022-09-03T06:26:32.014 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2022-09-03T06:26:32.015 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2022-09-03T06:26:32.015 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2022-09-03T06:26:32.016 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2022-09-03T06:26:32.016 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "test-zone2.rgw.otp", 2022-09-03T06:26:32.017 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T06:26:32.017 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T06:26:32.017 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T06:26:32.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.018 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T06:26:32.019 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.019 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T06:26:32.019 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:26:32.020 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2022-09-03T06:26:32.020 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T06:26:32.021 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T06:26:32.022 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "test-zone2.rgw.buckets.data", 2022-09-03T06:26:32.022 INFO:teuthology.orchestra.run.smithi087.stdout: "compression_type": "random" 2022-09-03T06:26:32.023 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:32.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.023 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2022-09-03T06:26:32.023 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T06:26:32.024 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:32.024 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:32.024 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.024 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:26:32.025 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2022-09-03T06:26:32.025 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:26:32.027 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:26:32.028 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:26:32.028 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:26:32.078 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:26:32.079 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:26:32.137 INFO:teuthology.orchestra.run.smithi087.stderr:Sending period to new master zone fca49d30-abba-4c2b-8462-66395ac063ba 2022-09-03T06:26:32.169 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:26:32.170 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:32.170 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 3, 2022-09-03T06:26:32.170 INFO:teuthology.orchestra.run.smithi087.stdout: "predecessor_uuid": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:26:32.171 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": [ 2022-09-03T06:26:32.171 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:26:32.171 INFO:teuthology.orchestra.run.smithi087.stdout: "1_1662186136.239783_174.1", 2022-09-03T06:26:32.171 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:26:32.172 INFO:teuthology.orchestra.run.smithi087.stdout: "" 2022-09-03T06:26:32.172 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.172 INFO:teuthology.orchestra.run.smithi087.stdout: "period_map": { 2022-09-03T06:26:32.172 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:26:32.173 INFO:teuthology.orchestra.run.smithi087.stdout: "zonegroups": [ 2022-09-03T06:26:32.173 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.173 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "test-zonegroup", 2022-09-03T06:26:32.173 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zonegroup", 2022-09-03T06:26:32.174 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "", 2022-09-03T06:26:32.174 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": true, 2022-09-03T06:26:32.174 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:26:32.174 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:26:32.175 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.175 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2022-09-03T06:26:32.175 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2022-09-03T06:26:32.175 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:26:32.176 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [ 2022-09-03T06:26:32.176 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.176 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:26:32.176 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone3", 2022-09-03T06:26:32.177 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:26:32.177 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:26:32.177 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.177 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:26:32.178 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:26:32.178 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:26:32.178 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:26:32.178 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:26:32.179 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:26:32.179 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:26:32.179 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:26:32.179 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:26:32.180 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:26:32.180 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:32.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.180 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.181 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:26:32.181 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone4", 2022-09-03T06:26:32.181 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:26:32.181 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:26:32.182 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.182 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:26:32.183 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:26:32.183 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:26:32.183 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:26:32.183 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "pubsub", 2022-09-03T06:26:32.184 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": false, 2022-09-03T06:26:32.184 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [ 2022-09-03T06:26:32.184 INFO:teuthology.orchestra.run.smithi087.stdout: "test-zone1" 2022-09-03T06:26:32.184 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.185 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:26:32.185 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:26:32.185 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:26:32.185 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:32.185 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.186 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.187 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:26:32.187 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:26:32.187 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:26:32.188 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:26:32.188 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.189 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:26:32.189 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:26:32.189 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:26:32.189 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:26:32.190 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:26:32.191 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": false, 2022-09-03T06:26:32.191 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:26:32.192 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:26:32.192 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:26:32.193 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:26:32.193 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:32.193 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.194 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.195 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:26:32.195 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone1", 2022-09-03T06:26:32.195 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:26:32.196 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:26:32.196 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.197 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:26:32.197 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:26:32.198 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:26:32.198 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:26:32.199 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:26:32.199 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:26:32.200 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:26:32.200 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:26:32.200 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:26:32.201 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:26:32.201 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:32.201 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:32.201 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.202 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [ 2022-09-03T06:26:32.202 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.203 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default-placement", 2022-09-03T06:26:32.204 INFO:teuthology.orchestra.run.smithi087.stdout: "tags": [], 2022-09-03T06:26:32.205 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": [ 2022-09-03T06:26:32.205 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD" 2022-09-03T06:26:32.206 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:32.206 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:32.207 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.207 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "default-placement", 2022-09-03T06:26:32.208 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:26:32.208 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2022-09-03T06:26:32.208 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2022-09-03T06:26:32.209 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.209 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled_features": [ 2022-09-03T06:26:32.209 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:26:32.209 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:32.210 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:32.210 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:26:32.210 INFO:teuthology.orchestra.run.smithi087.stdout: "short_zone_ids": [ 2022-09-03T06:26:32.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.211 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:26:32.211 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 700722763 2022-09-03T06:26:32.212 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.212 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.212 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:26:32.213 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 2398203312 2022-09-03T06:26:32.213 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.213 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.214 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:26:32.214 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 3737622162 2022-09-03T06:26:32.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:26:32.216 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:26:32.217 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 449119331 2022-09-03T06:26:32.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:32.218 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:26:32.218 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.218 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:26:32.219 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:26:32.219 INFO:teuthology.orchestra.run.smithi087.stdout: "period_config": { 2022-09-03T06:26:32.220 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_quota": { 2022-09-03T06:26:32.220 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:26:32.221 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:26:32.221 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:26:32.221 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:26:32.222 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:26:32.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.222 INFO:teuthology.orchestra.run.smithi087.stdout: "user_quota": { 2022-09-03T06:26:32.222 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:26:32.223 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:26:32.223 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:26:32.223 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:26:32.224 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:26:32.224 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.224 INFO:teuthology.orchestra.run.smithi087.stdout: "user_ratelimit": { 2022-09-03T06:26:32.224 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:26:32.225 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:26:32.225 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:26:32.226 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:26:32.226 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:26:32.227 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.227 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_ratelimit": { 2022-09-03T06:26:32.227 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:26:32.228 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:26:32.228 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:26:32.228 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:26:32.228 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:26:32.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.229 INFO:teuthology.orchestra.run.smithi087.stdout: "anonymous_ratelimit": { 2022-09-03T06:26:32.229 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:26:32.230 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:26:32.230 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:26:32.230 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:26:32.230 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:26:32.231 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:26:32.231 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:26:32.231 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:26:32.232 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_name": "test-realm", 2022-09-03T06:26:32.232 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:26:32.232 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:26:32.233 INFO:rgw_multi.tests:Set redirect_zone zone test-zone2 to "fca49d30-abba-4c2b-8462-66395ac063ba" 2022-09-03T06:27:32.235 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.237 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.238 DEBUG:boto:Method: HEAD 2022-09-03T06:27:32.240 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.241 DEBUG:boto:Data: 2022-09-03T06:27:32.242 DEBUG:boto:Headers: {} 2022-09-03T06:27:32.243 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:27:32.243 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.244 DEBUG:boto:Params: {} 2022-09-03T06:27:32.246 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:27:32.246 DEBUG:boto:Token: None 2022-09-03T06:27:32.247 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Sxm57ll4+OMN4fohhr57P5oISU= 2022-09-03T06:27:32.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Sxm57ll4+OMN4fohhr57P5oISU=', 'Content-Length': '0'} 2022-09-03T06:27:32.255 DEBUG:boto:Response headers: [('Location', 'http://smithi062.front.sepia.ceph.com:8000/avysmw-49/testredirect?'), ('x-amz-request-id', 'tx00000d7e679c858f43b67-006312f3d4-16ac-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.256 DEBUG:boto:Redirecting: http://smithi062.front.sepia.ceph.com/avysmw-49/testredirect 2022-09-03T06:27:32.257 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': '8000'} 2022-09-03T06:27:32.258 DEBUG:boto:Token: None 2022-09-03T06:27:32.259 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Sxm57ll4+OMN4fohhr57P5oISU= 2022-09-03T06:27:32.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Sxm57ll4+OMN4fohhr57P5oISU=', 'Content-Length': '0'} 2022-09-03T06:27:32.270 DEBUG:boto:Response headers: [('Content-Length', '512'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:25:29 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"dc5086b847289ba8b8bde149b8388175"'), ('x-amz-request-id', 'tx0000092e0b271f0464d27-006312f3d4-1597-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.272 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.273 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.274 DEBUG:boto:Method: GET 2022-09-03T06:27:32.275 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.276 DEBUG:boto:Data: 2022-09-03T06:27:32.277 DEBUG:boto:Headers: {} 2022-09-03T06:27:32.278 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:27:32.279 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.280 DEBUG:boto:Params: {} 2022-09-03T06:27:32.282 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:27:32.283 DEBUG:boto:Token: None 2022-09-03T06:27:32.284 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.285 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.291 DEBUG:boto:Response headers: [('Location', 'http://smithi062.front.sepia.ceph.com:8000/avysmw-49/testredirect?'), ('x-amz-request-id', 'tx0000014fa4d3ea5e1a6bb-006312f3d4-16ac-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.292 DEBUG:boto:Redirecting: http://smithi062.front.sepia.ceph.com/avysmw-49/testredirect 2022-09-03T06:27:32.293 DEBUG:boto:Token: None 2022-09-03T06:27:32.293 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.299 DEBUG:boto:Response headers: [('Content-Length', '512'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:25:29 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"dc5086b847289ba8b8bde149b8388175"'), ('x-amz-request-id', 'tx00000c25b30aa584666fc-006312f3d4-1597-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.300 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.300 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.301 DEBUG:boto:Method: PUT 2022-09-03T06:27:32.301 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.303 DEBUG:boto:Data: 2022-09-03T06:27:32.304 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'VpBzljOcorCZvRIkX5Nt3A==', 'Content-Length': '512', 'Expect': '100-Continue'} 2022-09-03T06:27:32.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:27:32.306 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.307 DEBUG:boto:Params: {} 2022-09-03T06:27:32.307 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:27:32.308 DEBUG:boto:Token: None 2022-09-03T06:27:32.309 DEBUG:boto:StringToSign: PUT VpBzljOcorCZvRIkX5Nt3A== application/octet-stream Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AH5fZA78xV1uudeSFM9xeGUP0DQ= 2022-09-03T06:27:32.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'VpBzljOcorCZvRIkX5Nt3A==', 'Content-Length': '512', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AH5fZA78xV1uudeSFM9xeGUP0DQ='} 2022-09-03T06:27:32.349 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"56907396339ca2b099bd12245f936ddc"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000ecd8c35762a202df-006312f3d4-1597-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.350 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.351 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.352 DEBUG:boto:Method: GET 2022-09-03T06:27:32.352 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.353 DEBUG:boto:Data: 2022-09-03T06:27:32.353 DEBUG:boto:Headers: {} 2022-09-03T06:27:32.354 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:27:32.355 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.355 DEBUG:boto:Params: {} 2022-09-03T06:27:32.356 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:27:32.356 DEBUG:boto:Token: None 2022-09-03T06:27:32.357 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.361 DEBUG:boto:Response headers: [('Location', 'http://smithi062.front.sepia.ceph.com:8000/avysmw-49/testredirect?'), ('x-amz-request-id', 'tx0000069ddfbf7b21c30e0-006312f3d4-16ac-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.362 DEBUG:boto:Redirecting: http://smithi062.front.sepia.ceph.com/avysmw-49/testredirect 2022-09-03T06:27:32.363 DEBUG:boto:Token: None 2022-09-03T06:27:32.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.377 DEBUG:boto:Response headers: [('Content-Length', '512'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"56907396339ca2b099bd12245f936ddc"'), ('x-amz-request-id', 'tx000005918bb8ee921c652-006312f3d4-1597-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.383 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.384 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.385 DEBUG:boto:Method: PUT 2022-09-03T06:27:32.386 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.386 DEBUG:boto:Data: 2022-09-03T06:27:32.387 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'uk9S5NXZfBvPq4jGr+LM5g==', 'Content-Length': '512', 'Expect': '100-Continue'} 2022-09-03T06:27:32.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:27:32.389 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.389 DEBUG:boto:Params: {} 2022-09-03T06:27:32.390 DEBUG:boto:Token: None 2022-09-03T06:27:32.391 DEBUG:boto:StringToSign: PUT uk9S5NXZfBvPq4jGr+LM5g== application/octet-stream Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9EFZNq0bxWJBKm56B6ImbozNGjM= 2022-09-03T06:27:32.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'uk9S5NXZfBvPq4jGr+LM5g==', 'Content-Length': '512', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9EFZNq0bxWJBKm56B6ImbozNGjM='} 2022-09-03T06:27:32.398 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"ba4f52e4d5d97c1bcfab88c6afe2cce6"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000056583e1f1d4d311f-006312f3d4-1597-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.399 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.400 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.401 DEBUG:boto:Method: GET 2022-09-03T06:27:32.403 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.403 DEBUG:boto:Data: 2022-09-03T06:27:32.404 DEBUG:boto:Headers: {} 2022-09-03T06:27:32.405 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:27:32.405 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.406 DEBUG:boto:Params: {} 2022-09-03T06:27:32.406 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:27:32.407 DEBUG:boto:Token: None 2022-09-03T06:27:32.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.413 DEBUG:boto:Response headers: [('Location', 'http://smithi062.front.sepia.ceph.com:8000/avysmw-49/testredirect?'), ('x-amz-request-id', 'tx00000ffd00bd868370e9d-006312f3d4-16ac-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.414 DEBUG:boto:Redirecting: http://smithi062.front.sepia.ceph.com/avysmw-49/testredirect 2022-09-03T06:27:32.414 DEBUG:boto:Token: None 2022-09-03T06:27:32.415 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.424 DEBUG:boto:Response headers: [('Content-Length', '512'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"ba4f52e4d5d97c1bcfab88c6afe2cce6"'), ('x-amz-request-id', 'tx000006fa2beed0cc03242-006312f3d4-1597-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.425 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.426 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.426 DEBUG:boto:Method: PUT 2022-09-03T06:27:32.427 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.428 DEBUG:boto:Data: 2022-09-03T06:27:32.428 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '6aKmvaJMgaK7Z2rfDZvVeg==', 'Content-Length': '512', 'Expect': '100-Continue'} 2022-09-03T06:27:32.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:27:32.429 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.430 DEBUG:boto:Params: {} 2022-09-03T06:27:32.430 DEBUG:boto:Token: None 2022-09-03T06:27:32.431 DEBUG:boto:StringToSign: PUT 6aKmvaJMgaK7Z2rfDZvVeg== application/octet-stream Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x7Cyt4NTtExJVW4JWvig6muKMMA= 2022-09-03T06:27:32.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '6aKmvaJMgaK7Z2rfDZvVeg==', 'Content-Length': '512', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x7Cyt4NTtExJVW4JWvig6muKMMA='} 2022-09-03T06:27:32.438 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"e9a2a6bda24c81a2bb676adf0d9bd57a"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000007f8cad355b4e3d0a-006312f3d4-1597-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.439 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.440 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.440 DEBUG:boto:Method: GET 2022-09-03T06:27:32.441 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.441 DEBUG:boto:Data: 2022-09-03T06:27:32.442 DEBUG:boto:Headers: {} 2022-09-03T06:27:32.442 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:27:32.444 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.445 DEBUG:boto:Params: {} 2022-09-03T06:27:32.445 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:27:32.447 DEBUG:boto:Token: None 2022-09-03T06:27:32.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.454 DEBUG:boto:Response headers: [('Location', 'http://smithi062.front.sepia.ceph.com:8000/avysmw-49/testredirect?'), ('x-amz-request-id', 'tx00000a96338de3ccac51b-006312f3d4-16ac-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.456 DEBUG:boto:Redirecting: http://smithi062.front.sepia.ceph.com/avysmw-49/testredirect 2022-09-03T06:27:32.457 DEBUG:boto:Token: None 2022-09-03T06:27:32.458 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.462 DEBUG:boto:Response headers: [('Content-Length', '512'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"e9a2a6bda24c81a2bb676adf0d9bd57a"'), ('x-amz-request-id', 'tx0000004924fa7b10e0341-006312f3d4-1597-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.463 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.464 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.464 DEBUG:boto:Method: PUT 2022-09-03T06:27:32.465 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.466 DEBUG:boto:Data: 2022-09-03T06:27:32.468 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'GbNgNQ5MAukCtgY+rsT5Xw==', 'Content-Length': '512', 'Expect': '100-Continue'} 2022-09-03T06:27:32.468 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:27:32.469 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.469 DEBUG:boto:Params: {} 2022-09-03T06:27:32.470 DEBUG:boto:Token: None 2022-09-03T06:27:32.470 DEBUG:boto:StringToSign: PUT GbNgNQ5MAukCtgY+rsT5Xw== application/octet-stream Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DKFQC7R1dlafhdwXCfeMDvmAabw= 2022-09-03T06:27:32.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'GbNgNQ5MAukCtgY+rsT5Xw==', 'Content-Length': '512', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DKFQC7R1dlafhdwXCfeMDvmAabw='} 2022-09-03T06:27:32.477 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"19b360350e4c02e902b6063eaec4f95f"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000376730ae51be1a11-006312f3d4-1597-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.478 DEBUG:boto:path=/avysmw-49/testredirect 2022-09-03T06:27:32.479 DEBUG:boto:auth_path=/avysmw-49/testredirect 2022-09-03T06:27:32.479 DEBUG:boto:Method: GET 2022-09-03T06:27:32.480 DEBUG:boto:Path: /avysmw-49/testredirect 2022-09-03T06:27:32.481 DEBUG:boto:Data: 2022-09-03T06:27:32.481 DEBUG:boto:Headers: {} 2022-09-03T06:27:32.482 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:27:32.482 DEBUG:boto:Port: 8000 2022-09-03T06:27:32.483 DEBUG:boto:Params: {} 2022-09-03T06:27:32.483 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:27:32.484 DEBUG:boto:Token: None 2022-09-03T06:27:32.485 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.491 DEBUG:boto:Response headers: [('Location', 'http://smithi062.front.sepia.ceph.com:8000/avysmw-49/testredirect?'), ('x-amz-request-id', 'tx00000571e5205b2be27b1-006312f3d4-16ac-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.492 DEBUG:boto:Redirecting: http://smithi062.front.sepia.ceph.com/avysmw-49/testredirect 2022-09-03T06:27:32.493 DEBUG:boto:Token: None 2022-09-03T06:27:32.495 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:27:32 GMT /avysmw-49/testredirect 2022-09-03T06:27:32.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE= 2022-09-03T06:27:32.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:27:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2/Gg1bN+wCm4Ef3HxYjC6eu/UE=', 'Content-Length': '0'} 2022-09-03T06:27:32.503 DEBUG:boto:Response headers: [('Content-Length', '512'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"19b360350e4c02e902b6063eaec4f95f"'), ('x-amz-request-id', 'tx00000d1b48d7902993791-006312f3d4-1597-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:27:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:27:32.511 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--sync-from-all=true', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:27:32.513 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--sync-from-all=true', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:27:32.515 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone modify --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --sync-from-all=true --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:27:32.577 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:27:32.578 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:27:32.598 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:27:32.596+0000 7f6476d45600 0 failed reading obj info from .rgw.root:zone_info.00aa9b6d-7a02-4715-9200-8a26b45eb25e: (2) No such file or directory 2022-09-03T06:27:32.599 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:27:32.596+0000 7f6476d45600 0 WARNING: could not read zone params for zone id=00aa9b6d-7a02-4715-9200-8a26b45eb25e name=test-zone3 2022-09-03T06:27:32.600 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:27:32.597+0000 7f6476d45600 0 failed reading obj info from .rgw.root:zone_info.fca49d30-abba-4c2b-8462-66395ac063ba: (2) No such file or directory 2022-09-03T06:27:32.601 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:27:32.597+0000 7f6476d45600 0 WARNING: could not read zone params for zone id=fca49d30-abba-4c2b-8462-66395ac063ba name=test-zone1 2022-09-03T06:27:32.605 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:27:32.605 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:27:32.606 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:27:32.607 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "test-zone2.rgw.meta:root", 2022-09-03T06:27:32.607 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "test-zone2.rgw.control", 2022-09-03T06:27:32.608 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2022-09-03T06:27:32.608 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2022-09-03T06:27:32.608 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "test-zone2.rgw.log", 2022-09-03T06:27:32.609 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2022-09-03T06:27:32.609 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2022-09-03T06:27:32.609 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2022-09-03T06:27:32.610 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2022-09-03T06:27:32.610 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2022-09-03T06:27:32.610 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2022-09-03T06:27:32.611 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2022-09-03T06:27:32.611 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2022-09-03T06:27:32.611 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "test-zone2.rgw.otp", 2022-09-03T06:27:32.612 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T06:27:32.612 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T06:27:32.612 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T06:27:32.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.613 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T06:27:32.614 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.614 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T06:27:32.616 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:27:32.616 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2022-09-03T06:27:32.617 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T06:27:32.617 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T06:27:32.618 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "test-zone2.rgw.buckets.data", 2022-09-03T06:27:32.619 INFO:teuthology.orchestra.run.smithi087.stdout: "compression_type": "random" 2022-09-03T06:27:32.620 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:27:32.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.621 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2022-09-03T06:27:32.622 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T06:27:32.622 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:27:32.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:27:32.623 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.626 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:27:32.626 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2022-09-03T06:27:32.626 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:27:32.629 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:27:32.630 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:27:32.631 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:27:32.682 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:27:32.683 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:27:32.723 INFO:teuthology.orchestra.run.smithi087.stderr:Sending period to new master zone fca49d30-abba-4c2b-8462-66395ac063ba 2022-09-03T06:27:32.752 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:27:32.753 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:27:32.753 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 4, 2022-09-03T06:27:32.754 INFO:teuthology.orchestra.run.smithi087.stdout: "predecessor_uuid": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:27:32.754 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": [ 2022-09-03T06:27:32.755 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:27:32.755 INFO:teuthology.orchestra.run.smithi087.stdout: "1_1662186136.239783_174.1", 2022-09-03T06:27:32.755 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:27:32.756 INFO:teuthology.orchestra.run.smithi087.stdout: "" 2022-09-03T06:27:32.756 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.758 INFO:teuthology.orchestra.run.smithi087.stdout: "period_map": { 2022-09-03T06:27:32.758 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:27:32.758 INFO:teuthology.orchestra.run.smithi087.stdout: "zonegroups": [ 2022-09-03T06:27:32.759 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.759 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "test-zonegroup", 2022-09-03T06:27:32.759 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zonegroup", 2022-09-03T06:27:32.760 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "", 2022-09-03T06:27:32.760 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": true, 2022-09-03T06:27:32.760 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:27:32.760 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:27:32.761 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.761 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2022-09-03T06:27:32.761 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2022-09-03T06:27:32.761 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:27:32.762 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [ 2022-09-03T06:27:32.762 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.763 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:27:32.764 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone3", 2022-09-03T06:27:32.764 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:27:32.764 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:27:32.764 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.765 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:27:32.765 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:27:32.765 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:27:32.765 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:27:32.766 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:27:32.766 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:27:32.767 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:27:32.767 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:27:32.767 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:27:32.767 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:27:32.768 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:27:32.768 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.768 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.769 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:27:32.769 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone4", 2022-09-03T06:27:32.769 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:27:32.769 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:27:32.770 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.770 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:27:32.770 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:27:32.771 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:27:32.771 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:27:32.771 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "pubsub", 2022-09-03T06:27:32.772 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": false, 2022-09-03T06:27:32.772 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [ 2022-09-03T06:27:32.772 INFO:teuthology.orchestra.run.smithi087.stdout: "test-zone1" 2022-09-03T06:27:32.772 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.773 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:27:32.773 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:27:32.773 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:27:32.773 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:27:32.775 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.776 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.776 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:27:32.776 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:27:32.777 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:27:32.777 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:27:32.778 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.784 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:27:32.785 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:27:32.786 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:27:32.787 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:27:32.788 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:27:32.788 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:27:32.789 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:27:32.789 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:27:32.790 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:27:32.790 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:27:32.791 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:27:32.791 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.792 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.792 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:27:32.793 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone1", 2022-09-03T06:27:32.793 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:27:32.794 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:27:32.794 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.795 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:27:32.795 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:27:32.796 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:27:32.797 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:27:32.798 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:27:32.799 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:27:32.800 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:27:32.801 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:27:32.801 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:27:32.802 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:27:32.802 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:27:32.803 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:27:32.803 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.803 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [ 2022-09-03T06:27:32.804 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.807 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default-placement", 2022-09-03T06:27:32.807 INFO:teuthology.orchestra.run.smithi087.stdout: "tags": [], 2022-09-03T06:27:32.807 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": [ 2022-09-03T06:27:32.808 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD" 2022-09-03T06:27:32.808 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:27:32.808 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:27:32.808 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.809 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "default-placement", 2022-09-03T06:27:32.809 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:27:32.809 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2022-09-03T06:27:32.809 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2022-09-03T06:27:32.810 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.810 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled_features": [ 2022-09-03T06:27:32.810 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:27:32.811 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:27:32.811 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:27:32.811 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:27:32.811 INFO:teuthology.orchestra.run.smithi087.stdout: "short_zone_ids": [ 2022-09-03T06:27:32.812 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.812 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:27:32.812 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 700722763 2022-09-03T06:27:32.812 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.813 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.813 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:27:32.813 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 2398203312 2022-09-03T06:27:32.813 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.814 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.814 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:27:32.814 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 3737622162 2022-09-03T06:27:32.814 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.815 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:27:32.815 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:27:32.815 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 449119331 2022-09-03T06:27:32.815 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:27:32.816 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:27:32.816 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.816 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:27:32.816 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:27:32.816 INFO:teuthology.orchestra.run.smithi087.stdout: "period_config": { 2022-09-03T06:27:32.817 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_quota": { 2022-09-03T06:27:32.817 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:27:32.817 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:27:32.818 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:27:32.818 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:27:32.818 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:27:32.818 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.819 INFO:teuthology.orchestra.run.smithi087.stdout: "user_quota": { 2022-09-03T06:27:32.819 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:27:32.819 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:27:32.819 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:27:32.820 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:27:32.820 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:27:32.820 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.820 INFO:teuthology.orchestra.run.smithi087.stdout: "user_ratelimit": { 2022-09-03T06:27:32.821 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:27:32.821 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:27:32.821 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:27:32.821 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:27:32.822 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:27:32.822 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.822 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_ratelimit": { 2022-09-03T06:27:32.822 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:27:32.823 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:27:32.823 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:27:32.823 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:27:32.823 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:27:32.824 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.824 INFO:teuthology.orchestra.run.smithi087.stdout: "anonymous_ratelimit": { 2022-09-03T06:27:32.824 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:27:32.827 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:27:32.827 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:27:32.828 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:27:32.828 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:27:32.829 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:27:32.829 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:27:32.830 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:27:32.831 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_name": "test-realm", 2022-09-03T06:27:32.832 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:27:32.832 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:27:32.834 INFO:rgw_multi.tests:Set sync_from_all flag on zone test-zone2 to true 2022-09-03T06:28:32.838 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--redirect-zone=', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:28:32.842 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--redirect-zone=', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:28:32.847 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone modify --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --redirect-zone= --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:28:32.908 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:28:32.909 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:28:32.931 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:28:32.929+0000 7feed08a2600 0 failed reading obj info from .rgw.root:zone_info.00aa9b6d-7a02-4715-9200-8a26b45eb25e: (2) No such file or directory 2022-09-03T06:28:32.932 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:28:32.929+0000 7feed08a2600 0 WARNING: could not read zone params for zone id=00aa9b6d-7a02-4715-9200-8a26b45eb25e name=test-zone3 2022-09-03T06:28:32.935 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:28:32.933+0000 7feed08a2600 0 failed reading obj info from .rgw.root:zone_info.fca49d30-abba-4c2b-8462-66395ac063ba: (2) No such file or directory 2022-09-03T06:28:32.936 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:28:32.933+0000 7feed08a2600 0 WARNING: could not read zone params for zone id=fca49d30-abba-4c2b-8462-66395ac063ba name=test-zone1 2022-09-03T06:28:32.940 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:28:32.940 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:28:32.941 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:28:32.941 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "test-zone2.rgw.meta:root", 2022-09-03T06:28:32.942 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "test-zone2.rgw.control", 2022-09-03T06:28:32.943 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2022-09-03T06:28:32.943 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2022-09-03T06:28:32.943 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "test-zone2.rgw.log", 2022-09-03T06:28:32.943 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2022-09-03T06:28:32.944 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2022-09-03T06:28:32.944 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2022-09-03T06:28:32.944 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2022-09-03T06:28:32.944 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2022-09-03T06:28:32.945 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2022-09-03T06:28:32.945 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2022-09-03T06:28:32.945 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2022-09-03T06:28:32.945 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "test-zone2.rgw.otp", 2022-09-03T06:28:32.946 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T06:28:32.946 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T06:28:32.946 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T06:28:32.946 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:32.947 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T06:28:32.947 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:32.947 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T06:28:32.947 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:28:32.948 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2022-09-03T06:28:32.949 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T06:28:32.949 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T06:28:32.950 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "test-zone2.rgw.buckets.data", 2022-09-03T06:28:32.950 INFO:teuthology.orchestra.run.smithi087.stdout: "compression_type": "random" 2022-09-03T06:28:32.950 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:28:32.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:32.951 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2022-09-03T06:28:32.951 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T06:28:32.951 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:28:32.952 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:28:32.952 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:32.952 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:28:32.955 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2022-09-03T06:28:32.955 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:28:32.958 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:28:32.959 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:28:32.959 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:28:33.011 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:28:33.012 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:28:33.072 INFO:teuthology.orchestra.run.smithi087.stderr:Sending period to new master zone fca49d30-abba-4c2b-8462-66395ac063ba 2022-09-03T06:28:33.107 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:28:33.108 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:28:33.108 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 5, 2022-09-03T06:28:33.109 INFO:teuthology.orchestra.run.smithi087.stdout: "predecessor_uuid": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:28:33.109 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": [ 2022-09-03T06:28:33.110 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:28:33.110 INFO:teuthology.orchestra.run.smithi087.stdout: "1_1662186136.239783_174.1", 2022-09-03T06:28:33.110 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:28:33.111 INFO:teuthology.orchestra.run.smithi087.stdout: "" 2022-09-03T06:28:33.111 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.112 INFO:teuthology.orchestra.run.smithi087.stdout: "period_map": { 2022-09-03T06:28:33.112 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:28:33.113 INFO:teuthology.orchestra.run.smithi087.stdout: "zonegroups": [ 2022-09-03T06:28:33.113 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.114 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "test-zonegroup", 2022-09-03T06:28:33.114 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zonegroup", 2022-09-03T06:28:33.114 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "", 2022-09-03T06:28:33.115 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": true, 2022-09-03T06:28:33.115 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:28:33.116 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:28:33.117 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.118 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2022-09-03T06:28:33.118 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2022-09-03T06:28:33.120 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:28:33.120 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [ 2022-09-03T06:28:33.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.121 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:28:33.122 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone3", 2022-09-03T06:28:33.122 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:28:33.122 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:28:33.122 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.123 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:28:33.123 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:28:33.123 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:28:33.123 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:28:33.124 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:28:33.124 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:28:33.125 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:28:33.125 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:28:33.125 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:28:33.125 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:28:33.126 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:28:33.126 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.126 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.127 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:28:33.127 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone4", 2022-09-03T06:28:33.127 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:28:33.127 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:28:33.128 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.128 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:28:33.128 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:28:33.129 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:28:33.129 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:28:33.129 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "pubsub", 2022-09-03T06:28:33.130 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": false, 2022-09-03T06:28:33.130 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [ 2022-09-03T06:28:33.130 INFO:teuthology.orchestra.run.smithi087.stdout: "test-zone1" 2022-09-03T06:28:33.130 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.131 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:28:33.131 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:28:33.131 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:28:33.131 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:28:33.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.132 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:28:33.132 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:28:33.133 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:28:33.134 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:28:33.134 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.135 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:28:33.135 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:28:33.136 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:28:33.137 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:28:33.137 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:28:33.138 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:28:33.138 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:28:33.139 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:28:33.139 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:28:33.140 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:28:33.140 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:28:33.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.141 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:28:33.142 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone1", 2022-09-03T06:28:33.142 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:28:33.142 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:28:33.142 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.143 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:28:33.143 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:28:33.144 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:28:33.144 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:28:33.144 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:28:33.144 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:28:33.145 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:28:33.145 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:28:33.145 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:28:33.146 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:28:33.146 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:28:33.146 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:28:33.147 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.147 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [ 2022-09-03T06:28:33.147 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.147 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default-placement", 2022-09-03T06:28:33.149 INFO:teuthology.orchestra.run.smithi087.stdout: "tags": [], 2022-09-03T06:28:33.149 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": [ 2022-09-03T06:28:33.149 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD" 2022-09-03T06:28:33.150 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:28:33.151 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:28:33.151 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.152 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "default-placement", 2022-09-03T06:28:33.152 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:28:33.152 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2022-09-03T06:28:33.152 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2022-09-03T06:28:33.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.153 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled_features": [ 2022-09-03T06:28:33.153 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:28:33.154 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:28:33.154 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:28:33.155 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:28:33.155 INFO:teuthology.orchestra.run.smithi087.stdout: "short_zone_ids": [ 2022-09-03T06:28:33.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.155 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:28:33.156 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 700722763 2022-09-03T06:28:33.156 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.157 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:28:33.157 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 2398203312 2022-09-03T06:28:33.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.159 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:28:33.159 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 3737622162 2022-09-03T06:28:33.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:28:33.160 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:28:33.161 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 449119331 2022-09-03T06:28:33.161 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:28:33.162 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:28:33.162 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.163 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:28:33.163 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:28:33.163 INFO:teuthology.orchestra.run.smithi087.stdout: "period_config": { 2022-09-03T06:28:33.164 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_quota": { 2022-09-03T06:28:33.164 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:28:33.164 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:28:33.165 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:28:33.165 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:28:33.165 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:28:33.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.166 INFO:teuthology.orchestra.run.smithi087.stdout: "user_quota": { 2022-09-03T06:28:33.166 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:28:33.167 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:28:33.167 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:28:33.167 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:28:33.167 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:28:33.168 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.168 INFO:teuthology.orchestra.run.smithi087.stdout: "user_ratelimit": { 2022-09-03T06:28:33.170 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:28:33.170 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:28:33.170 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:28:33.171 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:28:33.171 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:28:33.171 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.172 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_ratelimit": { 2022-09-03T06:28:33.172 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:28:33.172 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:28:33.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:28:33.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:28:33.174 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:28:33.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.175 INFO:teuthology.orchestra.run.smithi087.stdout: "anonymous_ratelimit": { 2022-09-03T06:28:33.175 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:28:33.175 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:28:33.176 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:28:33.176 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:28:33.176 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:28:33.177 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:28:33.181 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:28:33.182 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:28:33.182 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_name": "test-realm", 2022-09-03T06:28:33.182 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:28:33.183 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:28:33.184 INFO:rgw_multi.tests:Set redirect_zone zone test-zone2 to "" 2022-09-03T06:29:33.189 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multi_zone_redirect ... ok 2022-09-03T06:29:33.196 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:33.207 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:33.208 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:29:33.269 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:33.270 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:33.320 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:29:33.338 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:29:33.338 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:33.338 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:29:33.339 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:29:33.339 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:33.339 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:33.339 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:29:33.340 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:25:29.367916Z" 2022-09-03T06:29:33.340 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:33.340 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:33.340 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:29:33.341 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:29:33.341 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:33.341 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:33.342 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:29:33.342 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:29:33.342 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:33.342 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:29:33.343 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:33.344 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:29:33.344 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:33.345 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:33.345 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:29:33.406 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:33.406 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:33.456 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:33.452+0000 7f094dfd4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:33.456 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:33.453+0000 7f094dfd4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:33.475 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:29:33.476 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:29:33.476 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:29:33.476 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:29:33.477 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:29:33.477 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:33.478 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:33.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.479 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:29:33.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:33.479 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:29:33.479 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:33.480 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:33.480 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:29:33.480 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:33.481 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:29:33.481 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:33.481 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:29:33.481 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:33.482 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:33.483 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:29:33.483 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:33.483 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:33.483 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:29:33.484 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:33.484 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:33.484 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:33.485 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:29:33.485 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:33.486 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:33.487 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:29:33.487 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:33.487 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:33.488 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:33.488 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:33.488 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:33.488 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:33.489 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:33.489 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:33.489 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:33.490 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:29:33.490 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:33.491 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:33.491 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:33.491 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:33.492 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:33.492 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:33.492 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:33.492 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:33.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.493 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:33.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.494 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:29:33.495 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:29:33.495 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:29:33.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.495 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:29:33Z" 2022-09-03T06:29:33.496 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:29:33.496 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:29:33Z" } 2022-09-03T06:29:33.497 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:29:33.497 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:33.498 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:33.499 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:29:33.500 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:29:33.501 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:33.502 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:33.502 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:29:33.558 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:33.559 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:33.588 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:33.586+0000 7fabf7bfb600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:33.589 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:33.588+0000 7fabf7bfb600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:33.602 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:29:33.603 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:29:33.604 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:29:33.604 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:29:33.605 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:29:33.605 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:33.606 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:29:33.606 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:33.607 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:29:33.607 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:33.608 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:29:33.608 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:29:33.609 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:29:33.609 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:29:33.609 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:29:33.610 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:29:33.610 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:29:33.611 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:29:33.611 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:29:33.611 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:33.612 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:33.612 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:33.613 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:29:33.613 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:29:33.613 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:29:33.620 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:29:33.621 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:29:33.621 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:29:33.622 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:29:33.622 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:29:33.623 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:29:33.623 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:33.624 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:33.624 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:33.624 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:29:33.625 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:29:33.625 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:29:33.625 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:29:33.629 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:29:33.629 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:29:33.634 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:29:33.635 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:29:33.635 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:29:33.637 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:33.637 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:33.638 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:33.638 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:29:33.639 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:29:33.639 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:29:33.640 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:29:33.640 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:29:33.642 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:29:33.648 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:29:33.649 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:29:33.649 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:29:33.650 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:33.650 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:33.650 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:33.651 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:33.653 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:29:33.654 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:29:33.656 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:29:33.656 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:33.656 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:29:33Z" 2022-09-03T06:29:33.659 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:29:33.661 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:29:33Z" } 2022-09-03T06:29:33.663 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:29:33.667 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:33.680 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:33.692 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:29:33.694 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:29:33.696 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:33.712 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:33.715 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:29:33.790 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:33.790 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:33.832 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:33.824+0000 7f63b0d11600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:33.833 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:33.825+0000 7f63b0d11600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:33.858 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:29:33.859 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:29:33.859 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:29:33.860 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:29:33.860 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:29:33.861 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:33.861 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:33.863 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.864 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:29:33.871 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:33.876 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:29:33.876 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:33.877 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:33.877 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:29:33.878 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:33.879 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:29:33.879 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:33.880 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:29:33.880 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:33.881 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.881 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.882 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:33.894 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:29:33.895 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:33.895 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:33.896 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:29:33.896 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:33.897 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:33.897 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:33.903 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:29:33.904 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:33.905 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.905 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.906 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:33.906 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:29:33.907 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:33.907 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:33.908 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:33.909 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:33.909 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:33.910 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:33.911 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:33.912 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:33.913 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.913 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.914 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:33.915 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:29:33.915 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:33.915 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:33.916 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:33.916 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:33.917 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:33.917 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:33.918 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:33.918 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:33.919 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.919 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:33.920 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:33.921 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.921 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:29:33.921 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:29:33.922 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:29:33.922 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:33.923 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:29:33Z" 2022-09-03T06:29:33.926 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:29:33.927 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:29:33Z" } 2022-09-03T06:29:33.928 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:29:33.929 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:33.930 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:33.930 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:29:33.931 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:33.932 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'ND2IR1XIGKP3UN6B', '--secret', 'quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:33.934 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone create --rgw-zone remove --rgw-zonegroup test-zonegroup --rgw-realm test-realm --access-key ND2IR1XIGKP3UN6B --secret quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:33.988 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:33.988 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:34.012 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:34.010+0000 7fb6db35e600 0 failed reading obj info from .rgw.root:zone_info.00aa9b6d-7a02-4715-9200-8a26b45eb25e: (2) No such file or directory 2022-09-03T06:29:34.012 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:34.010+0000 7fb6db35e600 0 WARNING: could not read zone params for zone id=00aa9b6d-7a02-4715-9200-8a26b45eb25e name=test-zone3 2022-09-03T06:29:34.013 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:34.011+0000 7fb6db35e600 0 failed reading obj info from .rgw.root:zone_info.fca49d30-abba-4c2b-8462-66395ac063ba: (2) No such file or directory 2022-09-03T06:29:34.013 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:34.011+0000 7fb6db35e600 0 WARNING: could not read zone params for zone id=fca49d30-abba-4c2b-8462-66395ac063ba name=test-zone1 2022-09-03T06:29:34.020 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:29:34.020 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "4c38357c-e22a-4df7-84dc-ccfbee96ef90", 2022-09-03T06:29:34.021 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "remove", 2022-09-03T06:29:34.021 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "remove.rgw.meta:root", 2022-09-03T06:29:34.022 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "remove.rgw.control", 2022-09-03T06:29:34.022 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "remove.rgw.log:gc", 2022-09-03T06:29:34.022 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "remove.rgw.log:lc", 2022-09-03T06:29:34.023 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "remove.rgw.log", 2022-09-03T06:29:34.023 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "remove.rgw.log:intent", 2022-09-03T06:29:34.024 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "remove.rgw.log:usage", 2022-09-03T06:29:34.024 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "remove.rgw.meta:roles", 2022-09-03T06:29:34.024 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "remove.rgw.log:reshard", 2022-09-03T06:29:34.025 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "remove.rgw.meta:users.keys", 2022-09-03T06:29:34.025 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "remove.rgw.meta:users.email", 2022-09-03T06:29:34.025 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "remove.rgw.meta:users.swift", 2022-09-03T06:29:34.025 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "remove.rgw.meta:users.uid", 2022-09-03T06:29:34.026 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "remove.rgw.otp", 2022-09-03T06:29:34.026 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2022-09-03T06:29:34.026 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "ND2IR1XIGKP3UN6B", 2022-09-03T06:29:34.027 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "quLcRt3vmkDXItXtwHOhHkbB7Ox8gEfq" 2022-09-03T06:29:34.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:34.028 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2022-09-03T06:29:34.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:34.028 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2022-09-03T06:29:34.028 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:34.029 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "remove.rgw.buckets.index", 2022-09-03T06:29:34.029 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2022-09-03T06:29:34.030 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2022-09-03T06:29:34.030 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "remove.rgw.buckets.data" 2022-09-03T06:29:34.030 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:34.030 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:34.031 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "remove.rgw.buckets.non-ec", 2022-09-03T06:29:34.031 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2022-09-03T06:29:34.031 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:34.032 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:34.033 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:34.033 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:34.033 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "remove.rgw.log:notif" 2022-09-03T06:29:34.033 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:29:34.035 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'remove', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:34.036 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'remove', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:34.038 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone remove --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:34.086 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:34.087 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:34.096 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:34.093+0000 7f87e8309600 0 period (2962e93e-ca58-4758-b6d9-6fe2e67c2976 does not have zone 4c38357c-e22a-4df7-84dc-ccfbee96ef90 configured 2022-09-03T06:29:40.662 INFO:teuthology.orchestra.run.smithi087.stderr:Sending period to new master zone fca49d30-abba-4c2b-8462-66395ac063ba 2022-09-03T06:29:40.694 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:29:40.694 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:40.695 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 6, 2022-09-03T06:29:40.696 INFO:teuthology.orchestra.run.smithi087.stdout: "predecessor_uuid": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:29:40.696 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": [ 2022-09-03T06:29:40.697 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:29:40.697 INFO:teuthology.orchestra.run.smithi087.stdout: "1_1662186136.239783_174.1", 2022-09-03T06:29:40.702 INFO:teuthology.orchestra.run.smithi087.stdout: "", 2022-09-03T06:29:40.703 INFO:teuthology.orchestra.run.smithi087.stdout: "" 2022-09-03T06:29:40.703 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.704 INFO:teuthology.orchestra.run.smithi087.stdout: "period_map": { 2022-09-03T06:29:40.704 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:40.704 INFO:teuthology.orchestra.run.smithi087.stdout: "zonegroups": [ 2022-09-03T06:29:40.705 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.705 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "test-zonegroup", 2022-09-03T06:29:40.706 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zonegroup", 2022-09-03T06:29:40.709 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "", 2022-09-03T06:29:40.711 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": true, 2022-09-03T06:29:40.712 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:29:40.713 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:40.713 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.717 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2022-09-03T06:29:40.717 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2022-09-03T06:29:40.718 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:40.720 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [ 2022-09-03T06:29:40.720 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.721 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:29:40.722 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone3", 2022-09-03T06:29:40.722 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:29:40.724 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:29:40.725 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.725 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:29:40.726 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:29:40.727 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:40.727 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:29:40.728 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:29:40.728 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:29:40.728 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:29:40.729 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:29:40.729 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:29:40.730 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:29:40.730 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:40.730 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.730 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.731 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:29:40.731 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone4", 2022-09-03T06:29:40.731 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:29:40.731 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:29:40.732 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.732 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:29:40.732 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:29:40.733 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:40.733 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:29:40.733 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "pubsub", 2022-09-03T06:29:40.733 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": false, 2022-09-03T06:29:40.734 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [ 2022-09-03T06:29:40.734 INFO:teuthology.orchestra.run.smithi087.stdout: "test-zone1" 2022-09-03T06:29:40.734 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.734 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:29:40.735 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:29:40.735 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:29:40.735 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:40.735 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.736 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.736 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "4c38357c-e22a-4df7-84dc-ccfbee96ef90", 2022-09-03T06:29:40.736 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "remove", 2022-09-03T06:29:40.737 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [], 2022-09-03T06:29:40.737 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:29:40.737 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:29:40.737 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:40.738 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:29:40.738 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:29:40.738 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:29:40.739 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:29:40.739 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:29:40.739 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:29:40.739 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:29:40.740 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:40.740 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.741 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.741 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:29:40.741 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone2", 2022-09-03T06:29:40.741 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:29:40.742 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:29:40.742 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.742 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:29:40.743 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:29:40.743 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:40.744 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:29:40.744 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:29:40.745 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:29:40.746 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:29:40.746 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:29:40.747 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:29:40.747 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:29:40.748 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:40.748 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.749 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.749 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:40.749 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "test-zone1", 2022-09-03T06:29:40.750 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [ 2022-09-03T06:29:40.751 INFO:teuthology.orchestra.run.smithi087.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:40.753 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.754 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": false, 2022-09-03T06:29:40.754 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": true, 2022-09-03T06:29:40.755 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:40.755 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": false, 2022-09-03T06:29:40.755 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2022-09-03T06:29:40.756 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": true, 2022-09-03T06:29:40.756 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2022-09-03T06:29:40.756 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "", 2022-09-03T06:29:40.756 INFO:teuthology.orchestra.run.smithi087.stdout: "supported_features": [ 2022-09-03T06:29:40.757 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:29:40.757 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:40.757 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:40.757 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.758 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [ 2022-09-03T06:29:40.758 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.758 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default-placement", 2022-09-03T06:29:40.758 INFO:teuthology.orchestra.run.smithi087.stdout: "tags": [], 2022-09-03T06:29:40.760 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": [ 2022-09-03T06:29:40.761 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD" 2022-09-03T06:29:40.761 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:40.762 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:40.762 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.763 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "default-placement", 2022-09-03T06:29:40.763 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:40.764 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2022-09-03T06:29:40.764 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2022-09-03T06:29:40.765 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.765 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled_features": [ 2022-09-03T06:29:40.766 INFO:teuthology.orchestra.run.smithi087.stdout: "resharding" 2022-09-03T06:29:40.766 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:40.767 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:40.767 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2022-09-03T06:29:40.768 INFO:teuthology.orchestra.run.smithi087.stdout: "short_zone_ids": [ 2022-09-03T06:29:40.768 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.769 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:29:40.769 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 700722763 2022-09-03T06:29:40.770 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.770 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.771 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:29:40.771 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 2398203312 2022-09-03T06:29:40.772 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.772 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.772 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "4c38357c-e22a-4df7-84dc-ccfbee96ef90", 2022-09-03T06:29:40.773 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 3996576545 2022-09-03T06:29:40.773 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.774 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.774 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:29:40.775 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 3737622162 2022-09-03T06:29:40.775 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.776 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:40.776 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:40.777 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 449119331 2022-09-03T06:29:40.777 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:40.778 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:40.778 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.779 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:29:40.779 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:40.780 INFO:teuthology.orchestra.run.smithi087.stdout: "period_config": { 2022-09-03T06:29:40.780 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_quota": { 2022-09-03T06:29:40.781 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:29:40.781 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:29:40.781 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:29:40.782 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:29:40.782 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:29:40.783 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.783 INFO:teuthology.orchestra.run.smithi087.stdout: "user_quota": { 2022-09-03T06:29:40.784 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2022-09-03T06:29:40.784 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2022-09-03T06:29:40.785 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2022-09-03T06:29:40.785 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2022-09-03T06:29:40.785 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2022-09-03T06:29:40.786 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.786 INFO:teuthology.orchestra.run.smithi087.stdout: "user_ratelimit": { 2022-09-03T06:29:40.787 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:29:40.787 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:29:40.788 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:29:40.788 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:29:40.789 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:29:40.789 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.790 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_ratelimit": { 2022-09-03T06:29:40.790 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:29:40.790 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:29:40.790 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:29:40.791 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:29:40.791 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:29:40.791 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.791 INFO:teuthology.orchestra.run.smithi087.stdout: "anonymous_ratelimit": { 2022-09-03T06:29:40.792 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2022-09-03T06:29:40.792 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2022-09-03T06:29:40.792 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2022-09-03T06:29:40.792 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2022-09-03T06:29:40.793 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2022-09-03T06:29:40.793 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:40.793 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:40.793 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:40.794 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_name": "test-realm", 2022-09-03T06:29:40.794 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:40.794 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:29:40.795 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'remove', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:40.796 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'remove', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:40.796 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup remove --rgw-zonegroup test-zonegroup --rgw-realm test-realm --rgw-zone remove --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:40.857 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:40.858 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:40.875 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:40.872+0000 7f42a852b600 0 failed reading obj info from .rgw.root:zone_info.31b17c08-52e2-4c88-9aba-17b28ca757e4: (2) No such file or directory 2022-09-03T06:29:40.876 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:40.872+0000 7f42a852b600 0 WARNING: could not read zone params for zone id=31b17c08-52e2-4c88-9aba-17b28ca757e4 name=test-zone4 2022-09-03T06:29:40.876 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:40.872+0000 7f42a852b600 0 failed reading obj info from .rgw.root:zone_info.9b3b3b7d-a26b-43ae-9603-22cc8f947364: (2) No such file or directory 2022-09-03T06:29:40.877 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:40.872+0000 7f42a852b600 0 WARNING: could not read zone params for zone id=9b3b3b7d-a26b-43ae-9603-22cc8f947364 name=test-zone2 2022-09-03T06:29:40.884 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:29:40.888 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "test-zonegroup", 2022-09-03T06:29:40.889 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zonegroup", 2022-09-03T06:29:40.890 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "", 2022-09-03T06:29:40.891 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": true, 2022-09-03T06:29:40.891 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:40.892 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:40.892 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:40.893 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T06:29:40.893 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T06:29:40.894 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:40.895 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [ 2022-09-03T06:29:40.895 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:40.896 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:29:40.896 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone3", 2022-09-03T06:29:40.897 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:40.897 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:29:40.898 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:40.901 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:40.902 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:40.902 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:40.903 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:40.903 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:40.904 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:40.904 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:40.905 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:40.905 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:40.906 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:40.906 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:40.907 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:40.907 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:40.907 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:29:40.908 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone4", 2022-09-03T06:29:40.908 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:40.909 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:29:40.909 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:40.910 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:40.910 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:40.911 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:40.911 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:40.912 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "pubsub", 2022-09-03T06:29:40.913 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": false, 2022-09-03T06:29:40.914 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [ 2022-09-03T06:29:40.914 INFO:teuthology.orchestra.run.smithi062.stdout: "test-zone1" 2022-09-03T06:29:40.915 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:40.915 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:40.916 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:40.916 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:40.916 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:40.917 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:40.920 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:40.920 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:29:40.921 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone2", 2022-09-03T06:29:40.922 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:40.922 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:29:40.922 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:40.923 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:40.923 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:40.924 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:40.924 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:40.925 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:40.925 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:40.926 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:40.926 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:40.927 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:40.927 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:40.928 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:40.928 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:40.928 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:40.929 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:40.929 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T06:29:40.930 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:40.930 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:40.931 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:40.931 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:40.932 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:40.932 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:40.933 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:40.933 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:40.934 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:40.935 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:40.935 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:40.936 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:40.936 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:40.937 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:40.937 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:40.938 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:40.938 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [ 2022-09-03T06:29:40.939 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:40.939 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default-placement", 2022-09-03T06:29:40.940 INFO:teuthology.orchestra.run.smithi062.stdout: "tags": [], 2022-09-03T06:29:40.940 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": [ 2022-09-03T06:29:40.941 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD" 2022-09-03T06:29:40.941 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:40.942 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:40.942 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:40.943 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "default-placement", 2022-09-03T06:29:40.943 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:40.944 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T06:29:40.944 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T06:29:40.945 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:40.945 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T06:29:40.946 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:40.946 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:40.947 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:29:40.947 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'remove', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:40.948 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'remove', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:40.950 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup remove --rgw-zonegroup test-zonegroup --rgw-realm test-realm --rgw-zone remove --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:41.010 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:41.011 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:41.029 INFO:teuthology.orchestra.run.smithi062.stderr:zone name remove not found in zonegroup test-zonegroup 2022-09-03T06:29:41.034 DEBUG:teuthology.orchestra.run:got remote process result: 2 2022-09-03T06:29:41.036 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'delete', '--rgw-zone', 'remove', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.037 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'delete', '--rgw-zone', 'remove', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.039 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone delete --rgw-zone remove --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:41.107 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:41.108 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:41.149 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:41.146+0000 7fd38a147600 0 failed reading obj info from .rgw.root:zone_info.00aa9b6d-7a02-4715-9200-8a26b45eb25e: (2) No such file or directory 2022-09-03T06:29:41.150 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:41.146+0000 7fd38a147600 0 WARNING: could not read zone params for zone id=00aa9b6d-7a02-4715-9200-8a26b45eb25e name=test-zone3 2022-09-03T06:29:41.150 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:41.147+0000 7fd38a147600 0 failed reading obj info from .rgw.root:zone_info.fca49d30-abba-4c2b-8462-66395ac063ba: (2) No such file or directory 2022-09-03T06:29:41.151 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:41.147+0000 7fd38a147600 0 WARNING: could not read zone params for zone id=fca49d30-abba-4c2b-8462-66395ac063ba name=test-zone1 2022-09-03T06:29:41.158 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:41.156+0000 7fd38a147600 0 zone id 4c38357c-e22a-4df7-84dc-ccfbee96ef90 is not a part of zonegroup default 2022-09-03T06:29:41.187 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.188 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.189 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:41.250 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:41.251 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:41.273 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:41.268+0000 7f2ce5604600 0 WARNING: can't generate connection for zone 4c38357c-e22a-4df7-84dc-ccfbee96ef90 id remove: no endpoints defined 2022-09-03T06:29:41.352 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:29:41.352 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:41.353 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 7, 2022-09-03T06:29:41.353 INFO:teuthology.orchestra.run.smithi062.stdout: "predecessor_uuid": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:29:41.354 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": [ 2022-09-03T06:29:41.355 INFO:teuthology.orchestra.run.smithi062.stdout: "", 2022-09-03T06:29:41.355 INFO:teuthology.orchestra.run.smithi062.stdout: "1_1662186136.239783_174.1", 2022-09-03T06:29:41.356 INFO:teuthology.orchestra.run.smithi062.stdout: "", 2022-09-03T06:29:41.356 INFO:teuthology.orchestra.run.smithi062.stdout: "" 2022-09-03T06:29:41.357 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.358 INFO:teuthology.orchestra.run.smithi062.stdout: "period_map": { 2022-09-03T06:29:41.358 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:41.359 INFO:teuthology.orchestra.run.smithi062.stdout: "zonegroups": [ 2022-09-03T06:29:41.359 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.360 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "test-zonegroup", 2022-09-03T06:29:41.361 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zonegroup", 2022-09-03T06:29:41.361 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "", 2022-09-03T06:29:41.362 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": true, 2022-09-03T06:29:41.363 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.364 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:41.365 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.366 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T06:29:41.366 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T06:29:41.367 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:41.367 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [ 2022-09-03T06:29:41.367 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.367 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:29:41.368 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone3", 2022-09-03T06:29:41.368 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.368 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:29:41.368 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.369 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:41.369 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:41.369 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:41.369 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:41.371 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:41.372 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:41.373 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:41.373 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:41.374 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:41.374 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.375 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.375 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.376 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.377 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:29:41.377 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone4", 2022-09-03T06:29:41.378 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.378 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:29:41.379 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.380 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:41.381 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:41.381 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:41.382 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:41.382 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "pubsub", 2022-09-03T06:29:41.383 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": false, 2022-09-03T06:29:41.383 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [ 2022-09-03T06:29:41.384 INFO:teuthology.orchestra.run.smithi062.stdout: "test-zone1" 2022-09-03T06:29:41.384 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.384 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:41.386 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:41.386 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.387 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.387 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.387 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.388 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:29:41.388 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone2", 2022-09-03T06:29:41.388 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.389 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:29:41.389 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.389 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:41.390 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:41.390 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:41.390 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:41.390 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:41.391 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:41.391 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:41.391 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:41.391 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:41.392 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.392 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.392 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.393 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.393 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:41.393 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T06:29:41.394 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.394 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:41.395 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.395 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:41.396 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:41.397 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:41.397 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:41.398 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:41.398 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:41.399 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:41.399 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:41.400 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:41.400 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.401 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.401 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.401 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.402 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [ 2022-09-03T06:29:41.402 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.403 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default-placement", 2022-09-03T06:29:41.405 INFO:teuthology.orchestra.run.smithi062.stdout: "tags": [], 2022-09-03T06:29:41.406 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": [ 2022-09-03T06:29:41.406 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD" 2022-09-03T06:29:41.407 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.407 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.407 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.408 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "default-placement", 2022-09-03T06:29:41.408 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:41.408 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T06:29:41.409 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T06:29:41.409 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.409 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T06:29:41.411 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.411 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.412 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.412 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.413 INFO:teuthology.orchestra.run.smithi062.stdout: "short_zone_ids": [ 2022-09-03T06:29:41.413 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.413 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:29:41.414 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 700722763 2022-09-03T06:29:41.414 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.415 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.415 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:29:41.416 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 2398203312 2022-09-03T06:29:41.421 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.421 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.423 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:29:41.423 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 3737622162 2022-09-03T06:29:41.424 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.424 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.424 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:41.425 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 449119331 2022-09-03T06:29:41.425 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.426 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.427 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.427 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:29:41.428 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:41.428 INFO:teuthology.orchestra.run.smithi062.stdout: "period_config": { 2022-09-03T06:29:41.429 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_quota": { 2022-09-03T06:29:41.429 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T06:29:41.430 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T06:29:41.430 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T06:29:41.432 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T06:29:41.432 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T06:29:41.433 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.433 INFO:teuthology.orchestra.run.smithi062.stdout: "user_quota": { 2022-09-03T06:29:41.434 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T06:29:41.434 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T06:29:41.435 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T06:29:41.435 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T06:29:41.436 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T06:29:41.436 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.437 INFO:teuthology.orchestra.run.smithi062.stdout: "user_ratelimit": { 2022-09-03T06:29:41.437 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:29:41.437 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:29:41.438 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:29:41.438 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:29:41.439 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:29:41.439 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.439 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_ratelimit": { 2022-09-03T06:29:41.440 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:29:41.440 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:29:41.441 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:29:41.441 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:29:41.441 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:29:41.442 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.442 INFO:teuthology.orchestra.run.smithi062.stdout: "anonymous_ratelimit": { 2022-09-03T06:29:41.442 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:29:41.443 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:29:41.443 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:29:41.443 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:29:41.443 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:29:41.444 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.444 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.444 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:41.445 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_name": "test-realm", 2022-09-03T06:29:41.445 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:29:41.445 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:29:41.446 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_zonegroup_remove ... ok 2022-09-03T06:29:41.448 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'create', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.449 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'create', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.451 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup create --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:41.506 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:41.506 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:41.543 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:29:41.544 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "0ac38d08-b70c-4888-b109-f852acaad9f0", 2022-09-03T06:29:41.545 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "remove_zg", 2022-09-03T06:29:41.546 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "remove_zg", 2022-09-03T06:29:41.546 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": false, 2022-09-03T06:29:41.547 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [], 2022-09-03T06:29:41.548 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T06:29:41.548 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T06:29:41.549 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "", 2022-09-03T06:29:41.550 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [], 2022-09-03T06:29:41.550 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [], 2022-09-03T06:29:41.551 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "", 2022-09-03T06:29:41.551 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:41.552 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T06:29:41.552 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T06:29:41.554 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.555 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T06:29:41.556 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.556 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.557 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:29:41.559 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.560 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.561 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone remove --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:41.617 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:41.618 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:41.664 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:29:41.664 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "c7c273fc-2ca5-45b0-893f-fe9947e7703f", 2022-09-03T06:29:41.665 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "remove", 2022-09-03T06:29:41.665 INFO:teuthology.orchestra.run.smithi062.stdout: "domain_root": "remove.rgw.meta:root", 2022-09-03T06:29:41.665 INFO:teuthology.orchestra.run.smithi062.stdout: "control_pool": "remove.rgw.control", 2022-09-03T06:29:41.666 INFO:teuthology.orchestra.run.smithi062.stdout: "gc_pool": "remove.rgw.log:gc", 2022-09-03T06:29:41.666 INFO:teuthology.orchestra.run.smithi062.stdout: "lc_pool": "remove.rgw.log:lc", 2022-09-03T06:29:41.666 INFO:teuthology.orchestra.run.smithi062.stdout: "log_pool": "remove.rgw.log", 2022-09-03T06:29:41.666 INFO:teuthology.orchestra.run.smithi062.stdout: "intent_log_pool": "remove.rgw.log:intent", 2022-09-03T06:29:41.667 INFO:teuthology.orchestra.run.smithi062.stdout: "usage_log_pool": "remove.rgw.log:usage", 2022-09-03T06:29:41.667 INFO:teuthology.orchestra.run.smithi062.stdout: "roles_pool": "remove.rgw.meta:roles", 2022-09-03T06:29:41.667 INFO:teuthology.orchestra.run.smithi062.stdout: "reshard_pool": "remove.rgw.log:reshard", 2022-09-03T06:29:41.667 INFO:teuthology.orchestra.run.smithi062.stdout: "user_keys_pool": "remove.rgw.meta:users.keys", 2022-09-03T06:29:41.668 INFO:teuthology.orchestra.run.smithi062.stdout: "user_email_pool": "remove.rgw.meta:users.email", 2022-09-03T06:29:41.668 INFO:teuthology.orchestra.run.smithi062.stdout: "user_swift_pool": "remove.rgw.meta:users.swift", 2022-09-03T06:29:41.668 INFO:teuthology.orchestra.run.smithi062.stdout: "user_uid_pool": "remove.rgw.meta:users.uid", 2022-09-03T06:29:41.668 INFO:teuthology.orchestra.run.smithi062.stdout: "otp_pool": "remove.rgw.otp", 2022-09-03T06:29:41.669 INFO:teuthology.orchestra.run.smithi062.stdout: "system_key": { 2022-09-03T06:29:41.671 INFO:teuthology.orchestra.run.smithi062.stdout: "access_key": "", 2022-09-03T06:29:41.672 INFO:teuthology.orchestra.run.smithi062.stdout: "secret_key": "" 2022-09-03T06:29:41.672 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.672 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_pools": [ 2022-09-03T06:29:41.672 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.673 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "default-placement", 2022-09-03T06:29:41.674 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:29:41.675 INFO:teuthology.orchestra.run.smithi062.stdout: "index_pool": "remove.rgw.buckets.index", 2022-09-03T06:29:41.675 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": { 2022-09-03T06:29:41.676 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD": { 2022-09-03T06:29:41.676 INFO:teuthology.orchestra.run.smithi062.stdout: "data_pool": "remove.rgw.buckets.data" 2022-09-03T06:29:41.677 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.677 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.678 INFO:teuthology.orchestra.run.smithi062.stdout: "data_extra_pool": "remove.rgw.buckets.non-ec", 2022-09-03T06:29:41.679 INFO:teuthology.orchestra.run.smithi062.stdout: "index_type": 0 2022-09-03T06:29:41.679 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.680 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.680 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.681 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:41.681 INFO:teuthology.orchestra.run.smithi062.stdout: "notif_pool": "remove.rgw.log:notif" 2022-09-03T06:29:41.682 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:29:41.684 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.685 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.687 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:41.741 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:41.742 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:41.834 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:29:41.835 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:41.835 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 8, 2022-09-03T06:29:41.835 INFO:teuthology.orchestra.run.smithi062.stdout: "predecessor_uuid": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:29:41.836 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": [ 2022-09-03T06:29:41.836 INFO:teuthology.orchestra.run.smithi062.stdout: "", 2022-09-03T06:29:41.837 INFO:teuthology.orchestra.run.smithi062.stdout: "1_1662186136.239783_174.1", 2022-09-03T06:29:41.837 INFO:teuthology.orchestra.run.smithi062.stdout: "", 2022-09-03T06:29:41.838 INFO:teuthology.orchestra.run.smithi062.stdout: "" 2022-09-03T06:29:41.838 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.839 INFO:teuthology.orchestra.run.smithi062.stdout: "period_map": { 2022-09-03T06:29:41.839 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:41.840 INFO:teuthology.orchestra.run.smithi062.stdout: "zonegroups": [ 2022-09-03T06:29:41.840 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.840 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "0ac38d08-b70c-4888-b109-f852acaad9f0", 2022-09-03T06:29:41.841 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "remove_zg", 2022-09-03T06:29:41.842 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "remove_zg", 2022-09-03T06:29:41.842 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": false, 2022-09-03T06:29:41.843 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [], 2022-09-03T06:29:41.843 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T06:29:41.844 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T06:29:41.844 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "c7c273fc-2ca5-45b0-893f-fe9947e7703f", 2022-09-03T06:29:41.844 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [ 2022-09-03T06:29:41.845 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.845 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "c7c273fc-2ca5-45b0-893f-fe9947e7703f", 2022-09-03T06:29:41.846 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "remove", 2022-09-03T06:29:41.846 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [], 2022-09-03T06:29:41.847 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:41.847 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": false, 2022-09-03T06:29:41.848 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:41.848 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:41.848 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:41.848 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:41.849 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:41.849 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:41.849 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:41.850 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.850 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.850 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.850 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.851 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [ 2022-09-03T06:29:41.851 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.851 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default-placement", 2022-09-03T06:29:41.851 INFO:teuthology.orchestra.run.smithi062.stdout: "tags": [], 2022-09-03T06:29:41.852 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": [ 2022-09-03T06:29:41.852 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD" 2022-09-03T06:29:41.852 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.853 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.853 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.854 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "default-placement", 2022-09-03T06:29:41.854 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:41.855 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T06:29:41.855 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T06:29:41.856 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.856 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T06:29:41.857 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.857 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.858 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.858 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.859 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "test-zonegroup", 2022-09-03T06:29:41.859 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zonegroup", 2022-09-03T06:29:41.860 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "", 2022-09-03T06:29:41.860 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": true, 2022-09-03T06:29:41.861 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.861 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:41.862 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.862 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T06:29:41.863 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T06:29:41.863 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:41.864 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [ 2022-09-03T06:29:41.864 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.865 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:29:41.865 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone3", 2022-09-03T06:29:41.865 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.866 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:29:41.866 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.867 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:41.867 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:41.868 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:41.868 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:41.869 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:41.869 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:41.870 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:41.870 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:41.871 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:41.872 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.873 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.873 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.874 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.875 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:29:41.875 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone4", 2022-09-03T06:29:41.876 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.876 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:29:41.877 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.877 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:41.878 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:41.878 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:41.879 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:41.879 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "pubsub", 2022-09-03T06:29:41.879 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": false, 2022-09-03T06:29:41.880 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [ 2022-09-03T06:29:41.880 INFO:teuthology.orchestra.run.smithi062.stdout: "test-zone1" 2022-09-03T06:29:41.881 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.881 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:41.882 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:41.882 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.883 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.892 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.893 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.893 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:29:41.894 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone2", 2022-09-03T06:29:41.894 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.895 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:29:41.895 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.896 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:41.896 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:41.896 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:41.897 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:41.898 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:41.899 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:41.899 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:41.899 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:41.900 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:41.900 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.901 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.901 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.901 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.902 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:41.903 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T06:29:41.903 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:41.904 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:41.904 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.905 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:41.905 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:41.906 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:41.907 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:41.907 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:41.907 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:41.908 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:41.908 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:41.909 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:41.909 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.910 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.910 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.911 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.911 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [ 2022-09-03T06:29:41.911 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.911 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default-placement", 2022-09-03T06:29:41.912 INFO:teuthology.orchestra.run.smithi062.stdout: "tags": [], 2022-09-03T06:29:41.912 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": [ 2022-09-03T06:29:41.912 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD" 2022-09-03T06:29:41.913 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.913 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.913 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.913 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "default-placement", 2022-09-03T06:29:41.914 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:41.914 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T06:29:41.915 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T06:29:41.915 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.916 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T06:29:41.916 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:41.917 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.917 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.917 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:41.918 INFO:teuthology.orchestra.run.smithi062.stdout: "short_zone_ids": [ 2022-09-03T06:29:41.918 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.918 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:29:41.919 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 700722763 2022-09-03T06:29:41.919 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.920 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.920 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:29:41.921 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 2398203312 2022-09-03T06:29:41.921 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.922 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.922 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:29:41.923 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 3737622162 2022-09-03T06:29:41.924 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.924 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.925 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "c7c273fc-2ca5-45b0-893f-fe9947e7703f", 2022-09-03T06:29:41.925 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 3675394971 2022-09-03T06:29:41.926 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.927 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:41.927 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:41.928 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 449119331 2022-09-03T06:29:41.929 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.930 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:41.931 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.932 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:29:41.932 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:41.933 INFO:teuthology.orchestra.run.smithi062.stdout: "period_config": { 2022-09-03T06:29:41.933 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_quota": { 2022-09-03T06:29:41.934 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T06:29:41.934 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T06:29:41.935 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T06:29:41.936 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T06:29:41.936 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T06:29:41.937 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.937 INFO:teuthology.orchestra.run.smithi062.stdout: "user_quota": { 2022-09-03T06:29:41.938 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T06:29:41.938 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T06:29:41.939 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T06:29:41.939 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T06:29:41.940 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T06:29:41.940 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.941 INFO:teuthology.orchestra.run.smithi062.stdout: "user_ratelimit": { 2022-09-03T06:29:41.941 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:29:41.942 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:29:41.943 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:29:41.943 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:29:41.943 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:29:41.944 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.944 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_ratelimit": { 2022-09-03T06:29:41.944 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:29:41.945 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:29:41.946 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:29:41.953 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:29:41.953 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:29:41.954 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.954 INFO:teuthology.orchestra.run.smithi062.stdout: "anonymous_ratelimit": { 2022-09-03T06:29:41.955 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:29:41.955 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:29:41.956 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:29:41.956 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:29:41.956 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:29:41.957 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:41.958 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:41.958 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:41.959 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_name": "test-realm", 2022-09-03T06:29:41.959 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:29:41.959 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:29:41.960 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'delete', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.961 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'delete', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:41.962 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone delete --rgw-zone remove --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:42.021 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:42.022 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:42.071 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:42.069+0000 7fede7bf3600 0 zone id c7c273fc-2ca5-45b0-893f-fe9947e7703f is not a part of zonegroup test-zonegroup 2022-09-03T06:29:42.072 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:42.070+0000 7fede7bf3600 0 zone id c7c273fc-2ca5-45b0-893f-fe9947e7703f is not a part of zonegroup default 2022-09-03T06:29:42.091 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:42.092 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:42.093 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:42.166 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:42.167 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:42.206 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:29:42.203+0000 7fe31fe9a600 0 ERROR: zonegroup remove_zg has a non existent master zone 2022-09-03T06:29:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-09-03T06:29:42.213 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'delete', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:42.214 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'delete', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:42.215 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup delete --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:42.276 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:42.276 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:42.317 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:42.318 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:29:42.319 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:29:42.384 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:42.386 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:42.504 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:29:42.504 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:42.505 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 9, 2022-09-03T06:29:42.505 INFO:teuthology.orchestra.run.smithi062.stdout: "predecessor_uuid": "6a921abe-6ddc-4d12-8acc-a1151b4a5f12", 2022-09-03T06:29:42.506 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": [ 2022-09-03T06:29:42.506 INFO:teuthology.orchestra.run.smithi062.stdout: "", 2022-09-03T06:29:42.507 INFO:teuthology.orchestra.run.smithi062.stdout: "1_1662186136.239783_174.1", 2022-09-03T06:29:42.507 INFO:teuthology.orchestra.run.smithi062.stdout: "", 2022-09-03T06:29:42.507 INFO:teuthology.orchestra.run.smithi062.stdout: "" 2022-09-03T06:29:42.508 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.509 INFO:teuthology.orchestra.run.smithi062.stdout: "period_map": { 2022-09-03T06:29:42.509 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:42.511 INFO:teuthology.orchestra.run.smithi062.stdout: "zonegroups": [ 2022-09-03T06:29:42.511 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.514 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "test-zonegroup", 2022-09-03T06:29:42.514 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zonegroup", 2022-09-03T06:29:42.515 INFO:teuthology.orchestra.run.smithi062.stdout: "api_name": "", 2022-09-03T06:29:42.515 INFO:teuthology.orchestra.run.smithi062.stdout: "is_master": true, 2022-09-03T06:29:42.516 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:42.516 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:42.517 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.517 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames": [], 2022-09-03T06:29:42.518 INFO:teuthology.orchestra.run.smithi062.stdout: "hostnames_s3website": [], 2022-09-03T06:29:42.518 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:42.519 INFO:teuthology.orchestra.run.smithi062.stdout: "zones": [ 2022-09-03T06:29:42.519 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.520 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:29:42.524 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone3", 2022-09-03T06:29:42.525 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:42.525 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8001" 2022-09-03T06:29:42.525 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.525 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:42.531 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:42.531 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:42.532 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:42.532 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:42.533 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:42.533 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:42.534 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:42.535 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:42.535 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:42.536 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:42.536 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.537 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.537 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:29:42.539 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone4", 2022-09-03T06:29:42.539 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:42.540 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8001" 2022-09-03T06:29:42.540 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.541 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:42.541 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:42.542 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:42.542 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:42.543 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "pubsub", 2022-09-03T06:29:42.543 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": false, 2022-09-03T06:29:42.544 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [ 2022-09-03T06:29:42.544 INFO:teuthology.orchestra.run.smithi062.stdout: "test-zone1" 2022-09-03T06:29:42.545 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.545 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:42.546 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:42.546 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:42.546 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:42.547 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.547 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.547 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:29:42.547 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone2", 2022-09-03T06:29:42.548 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:42.548 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi087.front.sepia.ceph.com:8000" 2022-09-03T06:29:42.548 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.549 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:42.549 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:42.549 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:42.549 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:42.550 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:42.550 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:42.550 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:42.550 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:42.551 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:42.551 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:42.553 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:42.553 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.554 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.554 INFO:teuthology.orchestra.run.smithi062.stdout: "id": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:42.554 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "test-zone1", 2022-09-03T06:29:42.555 INFO:teuthology.orchestra.run.smithi062.stdout: "endpoints": [ 2022-09-03T06:29:42.555 INFO:teuthology.orchestra.run.smithi062.stdout: "http://smithi062.front.sepia.ceph.com:8000" 2022-09-03T06:29:42.555 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.556 INFO:teuthology.orchestra.run.smithi062.stdout: "log_meta": false, 2022-09-03T06:29:42.556 INFO:teuthology.orchestra.run.smithi062.stdout: "log_data": true, 2022-09-03T06:29:42.556 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_index_max_shards": 11, 2022-09-03T06:29:42.556 INFO:teuthology.orchestra.run.smithi062.stdout: "read_only": false, 2022-09-03T06:29:42.557 INFO:teuthology.orchestra.run.smithi062.stdout: "tier_type": "", 2022-09-03T06:29:42.557 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from_all": true, 2022-09-03T06:29:42.557 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_from": [], 2022-09-03T06:29:42.557 INFO:teuthology.orchestra.run.smithi062.stdout: "redirect_zone": "", 2022-09-03T06:29:42.558 INFO:teuthology.orchestra.run.smithi062.stdout: "supported_features": [ 2022-09-03T06:29:42.558 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:42.558 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:42.559 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:42.559 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.559 INFO:teuthology.orchestra.run.smithi062.stdout: "placement_targets": [ 2022-09-03T06:29:42.559 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.561 INFO:teuthology.orchestra.run.smithi062.stdout: "name": "default-placement", 2022-09-03T06:29:42.562 INFO:teuthology.orchestra.run.smithi062.stdout: "tags": [], 2022-09-03T06:29:42.563 INFO:teuthology.orchestra.run.smithi062.stdout: "storage_classes": [ 2022-09-03T06:29:42.563 INFO:teuthology.orchestra.run.smithi062.stdout: "STANDARD" 2022-09-03T06:29:42.563 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:42.564 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:42.564 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.564 INFO:teuthology.orchestra.run.smithi062.stdout: "default_placement": "default-placement", 2022-09-03T06:29:42.565 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:42.565 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_policy": { 2022-09-03T06:29:42.565 INFO:teuthology.orchestra.run.smithi062.stdout: "groups": [] 2022-09-03T06:29:42.566 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.566 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled_features": [ 2022-09-03T06:29:42.566 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding" 2022-09-03T06:29:42.567 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:42.567 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:42.569 INFO:teuthology.orchestra.run.smithi062.stdout: ], 2022-09-03T06:29:42.569 INFO:teuthology.orchestra.run.smithi062.stdout: "short_zone_ids": [ 2022-09-03T06:29:42.570 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.571 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:29:42.571 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 700722763 2022-09-03T06:29:42.572 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.572 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.572 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "31b17c08-52e2-4c88-9aba-17b28ca757e4", 2022-09-03T06:29:42.573 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 2398203312 2022-09-03T06:29:42.574 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.575 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.575 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:29:42.576 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 3737622162 2022-09-03T06:29:42.577 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.578 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:42.578 INFO:teuthology.orchestra.run.smithi062.stdout: "key": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:42.579 INFO:teuthology.orchestra.run.smithi062.stdout: "val": 449119331 2022-09-03T06:29:42.580 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:42.580 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:29:42.581 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.581 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zonegroup": "test-zonegroup", 2022-09-03T06:29:42.582 INFO:teuthology.orchestra.run.smithi062.stdout: "master_zone": "fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:29:42.582 INFO:teuthology.orchestra.run.smithi062.stdout: "period_config": { 2022-09-03T06:29:42.583 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_quota": { 2022-09-03T06:29:42.583 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T06:29:42.584 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T06:29:42.584 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T06:29:42.586 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T06:29:42.586 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T06:29:42.587 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.587 INFO:teuthology.orchestra.run.smithi062.stdout: "user_quota": { 2022-09-03T06:29:42.587 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false, 2022-09-03T06:29:42.587 INFO:teuthology.orchestra.run.smithi062.stdout: "check_on_raw": false, 2022-09-03T06:29:42.588 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size": -1, 2022-09-03T06:29:42.588 INFO:teuthology.orchestra.run.smithi062.stdout: "max_size_kb": 0, 2022-09-03T06:29:42.588 INFO:teuthology.orchestra.run.smithi062.stdout: "max_objects": -1 2022-09-03T06:29:42.589 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.589 INFO:teuthology.orchestra.run.smithi062.stdout: "user_ratelimit": { 2022-09-03T06:29:42.589 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:29:42.590 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:29:42.590 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:29:42.591 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:29:42.592 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:29:42.592 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.593 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket_ratelimit": { 2022-09-03T06:29:42.593 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:29:42.594 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:29:42.594 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:29:42.595 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:29:42.595 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:29:42.596 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.596 INFO:teuthology.orchestra.run.smithi062.stdout: "anonymous_ratelimit": { 2022-09-03T06:29:42.596 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_ops": 0, 2022-09-03T06:29:42.597 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_ops": 0, 2022-09-03T06:29:42.597 INFO:teuthology.orchestra.run.smithi062.stdout: "max_read_bytes": 0, 2022-09-03T06:29:42.598 INFO:teuthology.orchestra.run.smithi062.stdout: "max_write_bytes": 0, 2022-09-03T06:29:42.598 INFO:teuthology.orchestra.run.smithi062.stdout: "enabled": false 2022-09-03T06:29:42.599 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:42.599 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:42.600 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_id": "ec23be1e-4748-4838-8436-31585a76efae", 2022-09-03T06:29:42.600 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_name": "test-realm", 2022-09-03T06:29:42.601 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:29:42.601 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:29:42.602 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_zg_master_zone_delete ... ok 2022-09-03T06:29:42.604 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-50 2022-09-03T06:29:42.605 DEBUG:boto:path=/avysmw-50/ 2022-09-03T06:29:42.606 DEBUG:boto:auth_path=/avysmw-50/ 2022-09-03T06:29:42.608 DEBUG:boto:Method: PUT 2022-09-03T06:29:42.609 DEBUG:boto:Path: /avysmw-50/ 2022-09-03T06:29:42.610 DEBUG:boto:Data: 2022-09-03T06:29:42.611 DEBUG:boto:Headers: {} 2022-09-03T06:29:42.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:29:42.612 DEBUG:boto:Port: 8000 2022-09-03T06:29:42.613 DEBUG:boto:Params: {} 2022-09-03T06:29:42.614 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:29:42.615 DEBUG:boto:Token: None 2022-09-03T06:29:42.616 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:42 GMT /avysmw-50/ 2022-09-03T06:29:42.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wuAW67CcEseofaY4xyyHN36nuQ= 2022-09-03T06:29:42.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wuAW67CcEseofaY4xyyHN36nuQ=', 'Content-Length': '0'} 2022-09-03T06:29:43.122 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ae4063165153e7ee-006312f457-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:29:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:43.123 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-51 2022-09-03T06:29:43.124 DEBUG:boto:path=/avysmw-51/ 2022-09-03T06:29:43.134 DEBUG:boto:auth_path=/avysmw-51/ 2022-09-03T06:29:43.135 DEBUG:boto:Method: PUT 2022-09-03T06:29:43.136 DEBUG:boto:Path: /avysmw-51/ 2022-09-03T06:29:43.137 DEBUG:boto:Data: 2022-09-03T06:29:43.138 DEBUG:boto:Headers: {} 2022-09-03T06:29:43.138 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:29:43.139 DEBUG:boto:Port: 8000 2022-09-03T06:29:43.140 DEBUG:boto:Params: {} 2022-09-03T06:29:43.140 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:29:43.141 DEBUG:boto:Token: None 2022-09-03T06:29:43.146 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:43 GMT /avysmw-51/ 2022-09-03T06:29:43.147 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pJPbrN+lPvkFxxbNd2iIZGNaU3s= 2022-09-03T06:29:43.148 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pJPbrN+lPvkFxxbNd2iIZGNaU3s=', 'Content-Length': '0'} 2022-09-03T06:29:43.311 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ecf87ebb28b21842-006312f457-1707-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:29:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:43.312 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-52 2022-09-03T06:29:43.312 DEBUG:boto:path=/avysmw-52/ 2022-09-03T06:29:43.313 DEBUG:boto:auth_path=/avysmw-52/ 2022-09-03T06:29:43.313 DEBUG:boto:Method: PUT 2022-09-03T06:29:43.314 DEBUG:boto:Path: /avysmw-52/ 2022-09-03T06:29:43.314 DEBUG:boto:Data: 2022-09-03T06:29:43.317 DEBUG:boto:Headers: {} 2022-09-03T06:29:43.318 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:29:43.319 DEBUG:boto:Port: 8001 2022-09-03T06:29:43.320 DEBUG:boto:Params: {} 2022-09-03T06:29:43.322 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:29:43.323 DEBUG:boto:Token: None 2022-09-03T06:29:43.324 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:43 GMT /avysmw-52/ 2022-09-03T06:29:43.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TaNK4+vfttgCm8CWeTaan9I4osQ= 2022-09-03T06:29:43.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TaNK4+vfttgCm8CWeTaan9I4osQ=', 'Content-Length': '0'} 2022-09-03T06:29:43.390 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000fab55deac7803a3a-006312f457-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:29:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:43.391 DEBUG:boto:path=/avysmw-50/ 2022-09-03T06:29:43.392 DEBUG:boto:auth_path=/avysmw-50/ 2022-09-03T06:29:43.392 DEBUG:boto:path=/avysmw-50/?website 2022-09-03T06:29:43.393 DEBUG:boto:auth_path=/avysmw-50/?website 2022-09-03T06:29:43.395 DEBUG:boto:Method: PUT 2022-09-03T06:29:43.396 DEBUG:boto:Path: /avysmw-50/?website 2022-09-03T06:29:43.397 DEBUG:boto:Data: index.htmlerror.html 2022-09-03T06:29:43.397 DEBUG:boto:Headers: {} 2022-09-03T06:29:43.399 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:29:43.400 DEBUG:boto:Port: 8000 2022-09-03T06:29:43.400 DEBUG:boto:Params: {} 2022-09-03T06:29:43.401 DEBUG:boto:Token: None 2022-09-03T06:29:43.402 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:43 GMT /avysmw-50/?website 2022-09-03T06:29:43.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vHAo2rdnShp1otlaHuVEnwLG0bk= 2022-09-03T06:29:43.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vHAo2rdnShp1otlaHuVEnwLG0bk=', 'Content-Length': '241'} 2022-09-03T06:29:43.405 DEBUG:boto:Response headers: [('Content-Length', '202'), ('x-amz-request-id', 'tx00000b51f19e0c53e6503-006312f457-1604-test-zone1'), ('Accept-Ranges', 'bytes'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:29:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:43.407 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_set_bucket_website ... SKIP: test_set_bucket_website skipped. Requires rgw_enable_static_website = 1. 2022-09-03T06:29:43.411 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-53 2022-09-03T06:29:43.412 DEBUG:boto:path=/avysmw-53/ 2022-09-03T06:29:43.412 DEBUG:boto:auth_path=/avysmw-53/ 2022-09-03T06:29:43.413 DEBUG:boto:Method: PUT 2022-09-03T06:29:43.413 DEBUG:boto:Path: /avysmw-53/ 2022-09-03T06:29:43.414 DEBUG:boto:Data: 2022-09-03T06:29:43.415 DEBUG:boto:Headers: {} 2022-09-03T06:29:43.415 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:29:43.416 DEBUG:boto:Port: 8000 2022-09-03T06:29:43.417 DEBUG:boto:Params: {} 2022-09-03T06:29:43.418 DEBUG:boto:Token: None 2022-09-03T06:29:43.419 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:43 GMT /avysmw-53/ 2022-09-03T06:29:43.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n1JQudKOME0tEaLF9c4T1qF5irk= 2022-09-03T06:29:43.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n1JQudKOME0tEaLF9c4T1qF5irk=', 'Content-Length': '0'} 2022-09-03T06:29:43.467 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000775f7405d9f85e6c-006312f457-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:29:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:43.468 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-54 2022-09-03T06:29:43.469 DEBUG:boto:path=/avysmw-54/ 2022-09-03T06:29:43.470 DEBUG:boto:auth_path=/avysmw-54/ 2022-09-03T06:29:43.471 DEBUG:boto:Method: PUT 2022-09-03T06:29:43.471 DEBUG:boto:Path: /avysmw-54/ 2022-09-03T06:29:43.472 DEBUG:boto:Data: 2022-09-03T06:29:43.473 DEBUG:boto:Headers: {} 2022-09-03T06:29:43.474 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:29:43.475 DEBUG:boto:Port: 8000 2022-09-03T06:29:43.476 DEBUG:boto:Params: {} 2022-09-03T06:29:43.477 DEBUG:boto:Token: None 2022-09-03T06:29:43.478 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:43 GMT /avysmw-54/ 2022-09-03T06:29:43.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YHgpPlwCRbPAyvCozvHvUUahtcM= 2022-09-03T06:29:43.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YHgpPlwCRbPAyvCozvHvUUahtcM=', 'Content-Length': '0'} 2022-09-03T06:29:44.396 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000002b13a8e4831c6f08-006312f458-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:29:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:44.397 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-55 2022-09-03T06:29:44.398 DEBUG:boto:path=/avysmw-55/ 2022-09-03T06:29:44.399 DEBUG:boto:auth_path=/avysmw-55/ 2022-09-03T06:29:44.400 DEBUG:boto:Method: PUT 2022-09-03T06:29:44.401 DEBUG:boto:Path: /avysmw-55/ 2022-09-03T06:29:44.404 DEBUG:boto:Data: 2022-09-03T06:29:44.405 DEBUG:boto:Headers: {} 2022-09-03T06:29:44.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:29:44.407 DEBUG:boto:Port: 8001 2022-09-03T06:29:44.408 DEBUG:boto:Params: {} 2022-09-03T06:29:44.409 DEBUG:boto:Token: None 2022-09-03T06:29:44.409 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:44 GMT /avysmw-55/ 2022-09-03T06:29:44.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xhpCNvvhtz3TX/k3+GNdMUayFTs= 2022-09-03T06:29:44.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xhpCNvvhtz3TX/k3+GNdMUayFTs=', 'Content-Length': '0'} 2022-09-03T06:29:44.439 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000018a6e77088add2f4-006312f458-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:29:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:44.440 DEBUG:boto:path=/avysmw-53/ 2022-09-03T06:29:44.440 DEBUG:boto:auth_path=/avysmw-53/ 2022-09-03T06:29:44.441 DEBUG:boto:path=/avysmw-53/?policy 2022-09-03T06:29:44.441 DEBUG:boto:auth_path=/avysmw-53/?policy 2022-09-03T06:29:44.441 DEBUG:boto:Method: PUT 2022-09-03T06:29:44.442 DEBUG:boto:Path: /avysmw-53/?policy 2022-09-03T06:29:44.442 DEBUG:boto:Data: { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": "*" }] } 2022-09-03T06:29:44.443 DEBUG:boto:Headers: {} 2022-09-03T06:29:44.444 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:29:44.444 DEBUG:boto:Port: 8000 2022-09-03T06:29:44.445 DEBUG:boto:Params: {} 2022-09-03T06:29:44.445 DEBUG:boto:Token: None 2022-09-03T06:29:44.446 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:44 GMT /avysmw-53/?policy 2022-09-03T06:29:44.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t7lpke5s8hlaZfsgeRSCbP6isYo= 2022-09-03T06:29:44.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t7lpke5s8hlaZfsgeRSCbP6isYo=', 'Content-Length': '97'} 2022-09-03T06:29:44.454 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000c358b8075dd41fe2-006312f458-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:29:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:44.455 DEBUG:boto:path=/avysmw-53/ 2022-09-03T06:29:44.455 DEBUG:boto:auth_path=/avysmw-53/ 2022-09-03T06:29:44.456 DEBUG:boto:path=/avysmw-53/?policy 2022-09-03T06:29:44.456 DEBUG:boto:auth_path=/avysmw-53/?policy 2022-09-03T06:29:44.457 DEBUG:boto:Method: GET 2022-09-03T06:29:44.458 DEBUG:boto:Path: /avysmw-53/?policy 2022-09-03T06:29:44.458 DEBUG:boto:Data: 2022-09-03T06:29:44.459 DEBUG:boto:Headers: {} 2022-09-03T06:29:44.459 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:29:44.460 DEBUG:boto:Port: 8000 2022-09-03T06:29:44.460 DEBUG:boto:Params: {} 2022-09-03T06:29:44.461 DEBUG:boto:Token: None 2022-09-03T06:29:44.461 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:29:44 GMT /avysmw-53/?policy 2022-09-03T06:29:44.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wRUddMxr6tvrbyOKf1j1m1+BIbw= 2022-09-03T06:29:44.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wRUddMxr6tvrbyOKf1j1m1+BIbw=', 'Content-Length': '0'} 2022-09-03T06:29:44.464 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000001ab7ad14b6aa2420-006312f458-1604-test-zone1'), ('Content-Type', 'application/json'), ('Content-Length', '97'), ('Date', 'Sat, 03 Sep 2022 06:29:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:44.465 DEBUG:boto:path=/avysmw-54/ 2022-09-03T06:29:44.466 DEBUG:boto:auth_path=/avysmw-54/ 2022-09-03T06:29:44.466 DEBUG:boto:path=/avysmw-54/?policy 2022-09-03T06:29:44.467 DEBUG:boto:auth_path=/avysmw-54/?policy 2022-09-03T06:29:44.467 DEBUG:boto:Method: PUT 2022-09-03T06:29:44.468 DEBUG:boto:Path: /avysmw-54/?policy 2022-09-03T06:29:44.468 DEBUG:boto:Data: { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": "*" }] } 2022-09-03T06:29:44.469 DEBUG:boto:Headers: {} 2022-09-03T06:29:44.469 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:29:44.470 DEBUG:boto:Port: 8000 2022-09-03T06:29:44.470 DEBUG:boto:Params: {} 2022-09-03T06:29:44.471 DEBUG:boto:Token: None 2022-09-03T06:29:44.472 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:44 GMT /avysmw-54/?policy 2022-09-03T06:29:44.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fiKciOABpB3E8WecQyO2p/nmVv8= 2022-09-03T06:29:44.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fiKciOABpB3E8WecQyO2p/nmVv8=', 'Content-Length': '97'} 2022-09-03T06:29:45.487 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000aa1dda8f737b57aa-006312f458-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:29:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:45.490 DEBUG:boto:path=/avysmw-54/ 2022-09-03T06:29:45.491 DEBUG:boto:auth_path=/avysmw-54/ 2022-09-03T06:29:45.492 DEBUG:boto:path=/avysmw-54/?policy 2022-09-03T06:29:45.493 DEBUG:boto:auth_path=/avysmw-54/?policy 2022-09-03T06:29:45.493 DEBUG:boto:Method: GET 2022-09-03T06:29:45.494 DEBUG:boto:Path: /avysmw-54/?policy 2022-09-03T06:29:45.494 DEBUG:boto:Data: 2022-09-03T06:29:45.495 DEBUG:boto:Headers: {} 2022-09-03T06:29:45.495 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:29:45.497 DEBUG:boto:Port: 8000 2022-09-03T06:29:45.498 DEBUG:boto:Params: {} 2022-09-03T06:29:45.499 DEBUG:boto:Token: None 2022-09-03T06:29:45.500 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:29:45 GMT /avysmw-54/?policy 2022-09-03T06:29:45.501 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MEnfQSm4TpXNAwZe6vJVuExjUOU= 2022-09-03T06:29:45.501 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MEnfQSm4TpXNAwZe6vJVuExjUOU=', 'Content-Length': '0'} 2022-09-03T06:29:45.503 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000094e1c90593245058-006312f459-170f-test-zone2'), ('Content-Type', 'application/json'), ('Content-Length', '97'), ('Date', 'Sat, 03 Sep 2022 06:29:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:45.504 DEBUG:boto:path=/avysmw-55/ 2022-09-03T06:29:45.505 DEBUG:boto:auth_path=/avysmw-55/ 2022-09-03T06:29:45.505 DEBUG:boto:path=/avysmw-55/?policy 2022-09-03T06:29:45.506 DEBUG:boto:auth_path=/avysmw-55/?policy 2022-09-03T06:29:45.506 DEBUG:boto:Method: PUT 2022-09-03T06:29:45.507 DEBUG:boto:Path: /avysmw-55/?policy 2022-09-03T06:29:45.507 DEBUG:boto:Data: { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": "*" }] } 2022-09-03T06:29:45.508 DEBUG:boto:Headers: {} 2022-09-03T06:29:45.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:29:45.509 DEBUG:boto:Port: 8001 2022-09-03T06:29:45.510 DEBUG:boto:Params: {} 2022-09-03T06:29:45.511 DEBUG:boto:Token: None 2022-09-03T06:29:45.512 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:45 GMT /avysmw-55/?policy 2022-09-03T06:29:45.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q1qhQiehTCaBn/M/Q+v4tXUWhvM= 2022-09-03T06:29:45.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q1qhQiehTCaBn/M/Q+v4tXUWhvM=', 'Content-Length': '97'} 2022-09-03T06:29:46.527 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000054b564834edfb5ff-006312f459-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:29:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:46.528 DEBUG:boto:path=/avysmw-55/ 2022-09-03T06:29:46.528 DEBUG:boto:auth_path=/avysmw-55/ 2022-09-03T06:29:46.529 DEBUG:boto:path=/avysmw-55/?policy 2022-09-03T06:29:46.529 DEBUG:boto:auth_path=/avysmw-55/?policy 2022-09-03T06:29:46.530 DEBUG:boto:Method: GET 2022-09-03T06:29:46.531 DEBUG:boto:Path: /avysmw-55/?policy 2022-09-03T06:29:46.531 DEBUG:boto:Data: 2022-09-03T06:29:46.532 DEBUG:boto:Headers: {} 2022-09-03T06:29:46.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:29:46.535 DEBUG:boto:Port: 8001 2022-09-03T06:29:46.535 DEBUG:boto:Params: {} 2022-09-03T06:29:46.536 DEBUG:boto:Token: None 2022-09-03T06:29:46.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:29:46 GMT /avysmw-55/?policy 2022-09-03T06:29:46.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5WeM1JJqTp9+MBy7bth9q0xJssE= 2022-09-03T06:29:46.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5WeM1JJqTp9+MBy7bth9q0xJssE=', 'Content-Length': '0'} 2022-09-03T06:29:46.539 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000006ee83e9167284490-006312f45a-1608-test-zone3'), ('Content-Type', 'application/json'), ('Content-Length', '97'), ('Date', 'Sat, 03 Sep 2022 06:29:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:46.540 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_set_bucket_policy ... ok 2022-09-03T06:29:46.541 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-56 2022-09-03T06:29:46.541 DEBUG:boto:path=/avysmw-56/ 2022-09-03T06:29:46.542 DEBUG:boto:auth_path=/avysmw-56/ 2022-09-03T06:29:46.543 DEBUG:boto:Method: PUT 2022-09-03T06:29:46.543 DEBUG:boto:Path: /avysmw-56/ 2022-09-03T06:29:46.544 DEBUG:boto:Data: 2022-09-03T06:29:46.545 DEBUG:boto:Headers: {} 2022-09-03T06:29:46.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:29:46.546 DEBUG:boto:Port: 8000 2022-09-03T06:29:46.546 DEBUG:boto:Params: {} 2022-09-03T06:29:46.547 DEBUG:boto:Token: None 2022-09-03T06:29:46.547 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:46 GMT /avysmw-56/ 2022-09-03T06:29:46.548 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ch8yhRqW/OZCwqWck+4Prusyr3I= 2022-09-03T06:29:46.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ch8yhRqW/OZCwqWck+4Prusyr3I=', 'Content-Length': '0'} 2022-09-03T06:29:46.576 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000ebd1a552f2370704-006312f45a-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:29:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:46.577 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-57 2022-09-03T06:29:46.577 DEBUG:boto:path=/avysmw-57/ 2022-09-03T06:29:46.578 DEBUG:boto:auth_path=/avysmw-57/ 2022-09-03T06:29:46.579 DEBUG:boto:Method: PUT 2022-09-03T06:29:46.580 DEBUG:boto:Path: /avysmw-57/ 2022-09-03T06:29:46.581 DEBUG:boto:Data: 2022-09-03T06:29:46.582 DEBUG:boto:Headers: {} 2022-09-03T06:29:46.583 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:29:46.584 DEBUG:boto:Port: 8000 2022-09-03T06:29:46.585 DEBUG:boto:Params: {} 2022-09-03T06:29:46.586 DEBUG:boto:Token: None 2022-09-03T06:29:46.587 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:46 GMT /avysmw-57/ 2022-09-03T06:29:46.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zR0nqYrBXl/6Oi2LatXZniSAMCI= 2022-09-03T06:29:46.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zR0nqYrBXl/6Oi2LatXZniSAMCI=', 'Content-Length': '0'} 2022-09-03T06:29:46.628 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000063747cdfdb94255c-006312f45a-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:29:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:46.629 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-58 2022-09-03T06:29:46.630 DEBUG:boto:path=/avysmw-58/ 2022-09-03T06:29:46.631 DEBUG:boto:auth_path=/avysmw-58/ 2022-09-03T06:29:46.632 DEBUG:boto:Method: PUT 2022-09-03T06:29:46.633 DEBUG:boto:Path: /avysmw-58/ 2022-09-03T06:29:46.634 DEBUG:boto:Data: 2022-09-03T06:29:46.634 DEBUG:boto:Headers: {} 2022-09-03T06:29:46.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:29:46.637 DEBUG:boto:Port: 8001 2022-09-03T06:29:46.639 DEBUG:boto:Params: {} 2022-09-03T06:29:46.640 DEBUG:boto:Token: None 2022-09-03T06:29:46.641 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:29:46 GMT /avysmw-58/ 2022-09-03T06:29:46.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IeUeyMRjKIY7szaiIEKJbJ9C2Dk= 2022-09-03T06:29:46.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:29:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IeUeyMRjKIY7szaiIEKJbJ9C2Dk=', 'Content-Length': '0'} 2022-09-03T06:29:46.693 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000a997d8fbb9f02658-006312f45a-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:29:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:29:46.693 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:46.694 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:46.695 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:29:46.751 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:46.751 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:46.799 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:29:46.820 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:29:46.821 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:46.821 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:29:46.821 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:29:46.822 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:46.822 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:46.823 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186586.669522_282.1", 2022-09-03T06:29:46.823 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:29:46.669522Z" 2022-09-03T06:29:46.824 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:46.824 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:46.825 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:29:46.825 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:29:46.825 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:29:46.826 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:29:46.826 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:29:46.827 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:29:46.827 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:29:46.827 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:29:46.830 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:29:46.831 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:29:46.832 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:46.833 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:46.833 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:29:46.893 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:46.894 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:46.945 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:46.943+0000 7f2f682b2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:46.949 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:46.944+0000 7f2f682b2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:46.971 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:29:46.971 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:29:46.972 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:29:46.972 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:29:46.972 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:29:46.972 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:46.973 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:46.973 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:46.973 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:29:46.973 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:46.974 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:29:46.974 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:46.974 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:46.974 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:29:46.975 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:46.975 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:29:46.975 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:46.975 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:29:46.976 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:46.976 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:46.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:46.976 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:46.977 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:29:46.977 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:46.977 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:46.977 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:29:46.978 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:46.993 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:46.994 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:46.994 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:29:46.998 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:46.998 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:46.999 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:47.000 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:47.000 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:29:47.000 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:47.000 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:47.001 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:47.001 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:47.001 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:47.001 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:47.002 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:47.002 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:47.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:47.003 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:47.003 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:47.003 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:29:47.004 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:47.004 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:47.004 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:47.004 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:47.005 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:47.005 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:47.005 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:47.005 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:47.005 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:47.006 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:47.006 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:47.006 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:47.006 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:29:47.007 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:29:47.007 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:29:47.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:47.008 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:29:46Z" 2022-09-03T06:29:47.008 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:29:47.008 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:29:46Z" } 2022-09-03T06:29:47.009 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:29:47.010 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:29:47.010 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:47.011 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:29:52.012 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:52.013 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:52.015 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:29:52.075 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:52.076 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:52.121 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:52.119+0000 7feac7bd4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:52.122 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:52.120+0000 7feac7bd4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:52.145 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:29:52.146 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:29:52.147 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:29:52.147 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:29:52.148 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:29:52.148 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:52.150 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:52.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:52.151 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:29:52.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:52.152 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:29:52.152 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:52.152 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:52.152 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:29:52.153 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:52.153 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:29:52.153 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:52.153 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:29:52.154 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:52.154 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:52.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:52.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:52.155 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:29:52.155 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:52.155 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:52.156 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:29:52.156 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:52.156 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:52.156 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:52.157 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:29:52.157 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:52.157 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:52.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:52.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:52.158 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:29:52.159 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:52.159 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:52.159 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:52.159 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:52.160 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:52.160 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:52.160 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:52.161 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:52.161 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:52.161 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:52.162 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:52.162 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:29:52.162 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:52.162 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:52.163 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:52.163 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:52.163 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:52.164 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:52.164 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:52.164 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:52.165 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:52.165 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:52.166 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:52.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:52.167 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:29:52.168 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:29:52.168 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:29:52.168 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:52.168 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:29:52Z" 2022-09-03T06:29:52.169 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:29:52.170 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:29:52Z" } 2022-09-03T06:29:52.171 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:29:52.173 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:29:52.173 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:52.174 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:29:57.176 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:57.178 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:29:57.179 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:29:57.239 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:29:57.240 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:29:57.280 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:57.278+0000 7efec1f4b600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:57.281 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:29:57.279+0000 7efec1f4b600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:29:57.296 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:29:57.297 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:29:57.298 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:29:57.298 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:29:57.299 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:29:57.300 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:29:57.301 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:57.301 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:57.302 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:29:57.303 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:57.303 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:29:57.304 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:57.305 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:57.308 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:29:57.309 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:57.310 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:29:57.310 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:57.311 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:29:57.312 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:57.312 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:57.313 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:57.313 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:57.314 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:29:57.314 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:57.315 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:57.315 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:29:57.316 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:57.316 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:57.318 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:57.318 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:29:57.319 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:29:57.319 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:57.320 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:57.320 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:57.321 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:29:57.321 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:57.324 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:57.324 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:57.324 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:57.324 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:57.325 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:57.325 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:57.325 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:57.325 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:57.326 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:57.326 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:29:57.327 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:29:57.328 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:29:57.328 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:29:57.329 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:29:57.329 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:29:57.331 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:29:57.331 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:29:57.332 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:29:57.332 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:29:57.332 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:57.332 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:29:57.333 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:29:57.333 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:57.333 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:29:57.334 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:29:57.334 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:29:57.335 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:29:57.335 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:29:57Z" 2022-09-03T06:29:57.335 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:29:57.336 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:29:57Z" } 2022-09-03T06:29:57.337 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:29:57.337 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:29:57.338 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:29:57.339 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:30:02.342 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:02.344 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:02.345 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:02.403 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:02.403 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:02.458 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:02.455+0000 7f8a8abe6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:02.458 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:02.456+0000 7f8a8abe6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:02.476 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:30:02.477 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:30:02.477 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:30:02.477 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:30:02.477 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:30:02.478 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:02.478 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:02.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:02.479 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:30:02.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:02.480 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:30:02.480 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:02.481 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:02.482 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:02.483 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:02.483 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:30:02.484 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:02.484 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:02.484 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:02.485 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:02.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:02.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:02.486 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:30:02.487 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:02.487 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:02.488 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:30:02.488 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:02.488 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:02.489 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:02.490 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:30:02.491 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:02.502 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:02.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:02.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:02.503 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:30:02.504 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:02.504 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:02.505 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:02.505 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:02.509 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:02.510 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:02.512 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:02.513 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:02.513 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:02.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:02.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:02.514 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:30:02.514 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:02.515 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:02.515 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:02.515 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:02.515 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:02.516 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:02.516 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:02.517 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:02.517 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:02.522 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:02.523 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:30:02.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:02.525 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:30:02.525 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:30:02.526 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:30:02.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:02.528 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:30:02Z" 2022-09-03T06:30:02.529 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:30:02.530 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:02Z" } 2022-09-03T06:30:02.531 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:02.532 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:02.532 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:30:02.533 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:30:07.535 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:07.537 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:07.538 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:07.610 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:07.611 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:07.650 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:07.641+0000 7fd195029600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:07.651 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:07.642+0000 7fd195029600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:07.657 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:30:07.657 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:30:07.670 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:30:07.684 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:30:07.685 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:30:07.686 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:07.686 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:07.687 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:07.688 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:30:07.688 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:07.689 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:30:07.689 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:07.694 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:07.707 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:07.708 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:07.708 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:30:07.709 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:07.709 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:07.713 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:07.714 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:07.714 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:07.715 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:07.715 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:30:07.716 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:07.716 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:07.716 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:30:07.716 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:07.717 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:07.717 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:07.718 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:30:07.719 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:07.719 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:07.720 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:07.720 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:07.720 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:30:07.721 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:07.721 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:07.721 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:07.721 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:07.722 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:07.722 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:07.722 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:07.723 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:07.723 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:07.723 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:07.724 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:07.724 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:30:07.724 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:07.725 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:07.725 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:07.725 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:07.726 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:07.726 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:07.726 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:07.727 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:07.727 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:07.727 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:07.728 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:30:07.728 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:07.728 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:30:07.728 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:30:07.729 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:30:07.729 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:07.729 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:30:07Z" 2022-09-03T06:30:07.730 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:30:07.733 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:07Z" } 2022-09-03T06:30:07.733 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:07.734 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:07.735 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:30:07.735 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:30:12.737 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:12.739 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:12.740 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:12.798 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:12.798 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:12.835 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:12.829+0000 7fe823e51600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:12.836 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:12.830+0000 7fe823e51600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:12.845 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:30:12.846 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:30:12.849 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:30:12.849 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:30:12.850 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:30:12.851 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:12.851 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:12.852 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:12.852 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:30:12.853 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:12.853 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:30:12.854 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:12.854 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:12.855 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:12.855 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:12.856 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:30:12.856 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:12.857 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:12.859 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:12.860 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:12.862 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:12.862 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:12.863 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:30:12.863 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:12.864 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:12.864 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:30:12.865 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:12.865 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:12.866 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:12.867 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:30:12.867 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:12.868 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:12.870 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:12.871 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:12.871 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:30:12.871 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:12.872 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:12.872 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:12.873 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:12.873 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:12.874 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:12.874 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:12.874 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:12.874 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:12.875 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:12.875 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:12.875 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:30:12.875 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:12.876 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:12.876 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:12.877 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:12.877 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:12.877 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:12.878 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:12.878 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:12.878 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:12.878 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:12.879 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:30:12.879 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:12.879 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:30:12.879 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:30:12.880 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:30:12.880 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:12.880 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:30:12Z" 2022-09-03T06:30:12.881 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:30:12.886 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:12Z" } 2022-09-03T06:30:12.887 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:12.890 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:12.891 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:30:12.891 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:30:17.893 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:17.894 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:17.895 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:17.948 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:17.948 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:17.977 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:17.975+0000 7fa95f8cd600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:17.979 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:17.976+0000 7fa95f8cd600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:18.003 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:30:18.004 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:30:18.004 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:30:18.005 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:30:18.005 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:30:18.006 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:18.006 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:18.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:18.007 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:30:18.008 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:18.009 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:30:18.009 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:18.011 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:18.011 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:18.012 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:18.012 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:30:18.012 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:18.012 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:18.013 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:18.013 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:18.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:18.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:18.014 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:30:18.015 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:18.015 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:18.015 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186583.429819_251.1", 2022-09-03T06:30:18.016 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:18.016 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:18.016 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:18.017 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:29:43.429819Z", 2022-09-03T06:30:18.017 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:18.017 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:18.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:18.018 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:18.019 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:30:18.019 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:18.020 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:18.022 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:18.022 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:18.022 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:18.022 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:18.023 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:18.023 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:18.023 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:18.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:18.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:18.024 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:30:18.024 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:18.024 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:18.025 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:18.025 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:18.025 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:18.025 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:18.026 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:18.026 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:18.026 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:18.026 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:18.027 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:30:18.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:18.027 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:30:18.027 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:30:18.028 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:30:18.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:18.028 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:30:17Z" 2022-09-03T06:30:18.029 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:30:18.029 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186583.429819_251.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:29:43.429819Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:17Z" } 2022-09-03T06:30:18.030 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186583.429819_251.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:29:43.429819Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:18.030 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:18.031 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186583.429819_251.1', 2: '', 3: ''} 2022-09-03T06:30:18.032 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186583.429819_251.1 2022-09-03T06:30:23.033 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:23.034 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:23.036 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:23.091 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:23.093 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:23.133 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:23.129+0000 7f70efa54600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:23.134 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:23.130+0000 7f70efa54600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:23.145 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:30:23.146 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:30:23.147 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:30:23.147 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:30:23.148 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:30:23.148 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:23.149 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:23.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:23.150 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:30:23.150 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:23.151 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:30:23.151 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:23.152 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:23.152 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:23.152 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:23.153 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:30:23.153 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:23.154 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:23.154 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:23.155 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:23.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:23.156 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:23.156 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:30:23.158 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:23.158 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:23.159 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186583.348188_249.1", 2022-09-03T06:30:23.159 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:23.160 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:23.161 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:23.161 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:29:43.348188Z", 2022-09-03T06:30:23.161 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:23.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:23.163 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:23.163 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:23.164 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:30:23.164 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:23.165 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:23.166 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:23.166 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:23.167 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:23.167 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:23.168 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:23.168 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:23.168 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:23.168 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:23.169 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:23.169 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:30:23.170 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:23.170 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:23.170 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:23.170 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:23.171 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:23.171 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:23.171 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:23.171 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:23.172 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:23.172 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:23.173 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:30:23.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:23.174 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:30:23.175 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:30:23.175 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:30:23.176 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:23.176 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:30:23Z" 2022-09-03T06:30:23.176 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:30:23.179 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186583.348188_249.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:29:43.348188Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:23Z" } 2022-09-03T06:30:23.180 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186583.348188_249.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:29:43.348188Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:23.180 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:23.181 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186583.348188_249.1', 2: '', 3: ''} 2022-09-03T06:30:23.182 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186583.348188_249.1 2022-09-03T06:30:28.187 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:28.188 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:28.189 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:28.244 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:28.244 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:28.305 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:28.302+0000 7f8543b1a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:28.305 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:28.303+0000 7f8543b1a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:28.335 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:30:28.336 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:30:28.336 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:30:28.336 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:30:28.337 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:30:28.337 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:28.338 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:28.339 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:28.339 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:30:28.340 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:28.340 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:30:28.340 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:28.341 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:28.341 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:28.342 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:28.342 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:30:28.343 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:28.343 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:28.344 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:28.344 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:28.345 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:28.345 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:28.346 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:30:28.346 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:28.347 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:28.347 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186583.340953_247.1", 2022-09-03T06:30:28.347 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:28.348 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:28.348 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:28.349 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:29:43.340953Z", 2022-09-03T06:30:28.349 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:28.350 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:28.350 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:28.351 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:28.351 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:30:28.351 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:28.352 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:28.352 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:28.353 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:28.353 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:28.354 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:28.354 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:28.355 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:28.355 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:28.356 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:28.356 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:28.356 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:30:28.357 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:28.357 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:28.358 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:28.358 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:28.359 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:28.359 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:28.360 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:28.360 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:28.361 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:28.361 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:28.362 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:30:28.362 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:28.363 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:30:28.363 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:30:28.363 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:30:28.364 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:28.364 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:30:28Z" 2022-09-03T06:30:28.365 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:30:28.368 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186583.340953_247.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:29:43.340953Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:28Z" } 2022-09-03T06:30:28.369 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186583.340953_247.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:29:43.340953Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:28.370 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:28.371 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186583.340953_247.1', 2: '', 3: ''} 2022-09-03T06:30:28.372 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186583.340953_247.1 2022-09-03T06:30:33.378 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:33.379 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:33.380 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:33.442 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:33.444 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:33.473 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:33.471+0000 7f6dbf0ae600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:33.483 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:30:33.472+0000 7f6dbf0ae600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:33.494 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:30:33.495 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:30:33.495 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:30:33.495 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:30:33.496 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:30:33.496 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:33.497 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:33.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:33.498 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:30:33.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:33.498 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:30:33.499 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:33.499 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:33.499 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:33.500 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:33.500 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:30:33.500 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:33.500 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:33.501 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:33.501 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:33.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:33.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:33.502 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:30:33.504 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:33.504 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:33.505 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186586.669522_282.1", 2022-09-03T06:30:33.505 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:33.505 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:33.506 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:33.506 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:29:46.669522Z", 2022-09-03T06:30:33.507 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:30:33.508 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:33.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:33.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:33.508 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:30:33.509 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:33.509 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:33.509 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:33.511 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:33.511 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:33.512 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:33.512 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:33.513 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:33.515 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:33.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:33.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:30:33.517 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:30:33.517 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:30:33.518 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:30:33.519 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:30:33.519 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:30:33.520 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:30:33.520 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:30:33.521 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:30:33.521 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:30:33.522 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:33.522 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:30:33.523 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:30:33.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:33.524 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:30:33.524 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:30:33.525 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:30:33.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:30:33.526 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:30:33Z" 2022-09-03T06:30:33.526 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:30:33.527 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186586.669522_282.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:29:46.669522Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:33Z" } 2022-09-03T06:30:33.528 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186586.669522_282.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:29:46.669522Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:33.529 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:33.530 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:33.530 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:30:33.531 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:30:33.531 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:33.532 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:33.533 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:33.593 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:33.594 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:33.648 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:33.646+0000 7ff369e2f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:33.649 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:33.647+0000 7ff369e2f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:33.665 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:30:33.665 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:30:33.666 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:30:33.666 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:30:33.667 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:30:33.667 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:33.667 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:33.668 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:33.668 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:30:33.668 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:33.669 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:30:33.669 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:33.671 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:33.671 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:33.672 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:33.672 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:30:33.674 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:33.674 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:33.675 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:33.675 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:33.675 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:33.676 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:33.676 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:30:33.677 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:33.677 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:33.678 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:30:33.678 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:33.678 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:33.679 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:33.679 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:30:33.680 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:33.680 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:33.681 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:33.681 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:33.682 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:30:33.682 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:33.682 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:33.683 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:33.684 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:33.684 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:33.685 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:33.685 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:33.685 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:33.686 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:33.686 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:33.688 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:33.688 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:30:33.689 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:33.689 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:33.690 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:33.691 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:33.691 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:33.692 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:33.692 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:33.693 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:33.694 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:33.694 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:33.695 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:30:33.695 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:33.696 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:30:33.696 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:30:33.697 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:30:33.697 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:33.698 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:30:33Z" 2022-09-03T06:30:33.699 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:30:33.700 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:33Z" } 2022-09-03T06:30:33.701 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:33.702 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:33.703 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:30:33.704 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:30:38.707 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:38.709 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:38.710 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:38.775 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:38.776 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:38.812 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:38.809+0000 7fc1e5ad4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:38.813 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:38.810+0000 7fc1e5ad4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:38.841 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:30:38.841 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:30:38.842 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:30:38.842 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:30:38.843 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:30:38.844 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:38.844 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:38.845 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:38.845 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:30:38.848 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:38.848 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:30:38.849 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:38.849 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:38.850 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:38.850 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:38.851 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:30:38.851 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:38.851 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:38.852 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:38.852 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:38.853 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:38.853 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:38.853 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:30:38.863 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:38.863 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:38.864 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:30:38.864 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:38.865 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:38.865 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:38.866 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:30:38.866 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:38.867 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:38.867 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:38.868 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:38.868 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:30:38.869 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:38.869 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:38.870 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:38.871 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:38.871 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:38.872 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:38.872 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:38.874 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:38.874 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:38.874 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:38.875 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:38.875 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:30:38.876 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:38.876 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:38.877 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:38.877 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:38.878 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:38.878 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:38.879 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:38.879 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:38.880 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:38.880 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:38.881 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:30:38.881 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:38.882 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:30:38.882 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:30:38.882 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:30:38.883 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:38.883 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:30:38Z" 2022-09-03T06:30:38.884 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:30:38.884 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:38Z" } 2022-09-03T06:30:38.885 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:38.886 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:38.887 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:30:38.887 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:30:43.888 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:43.889 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:43.890 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:43.950 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:43.951 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:44.000 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:43.998+0000 7fa5b41b7600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:44.001 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:43.999+0000 7fa5b41b7600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:44.024 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:30:44.025 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:30:44.026 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:30:44.031 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:30:44.032 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:30:44.033 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:44.033 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:44.034 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:44.035 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:30:44.035 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:44.036 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:30:44.036 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:44.037 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:44.037 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:44.040 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:44.040 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:30:44.041 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:44.042 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:44.042 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:44.043 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:44.043 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:44.044 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:44.044 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:30:44.045 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:44.045 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:44.045 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:30:44.047 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:44.047 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:44.047 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:44.048 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:30:44.048 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:44.049 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:44.049 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:44.050 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:44.050 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:30:44.050 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:44.051 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:44.051 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:44.052 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:44.052 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:44.052 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:44.052 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:44.053 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:44.053 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:44.053 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:44.054 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:44.054 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:30:44.055 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:44.055 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:44.055 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:44.056 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:44.056 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:44.059 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:44.060 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:44.060 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:44.061 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:44.061 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:44.062 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:30:44.062 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:44.063 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:30:44.071 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:30:44.072 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:30:44.072 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:44.073 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:30:44Z" 2022-09-03T06:30:44.073 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:30:44.075 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:44Z" } 2022-09-03T06:30:44.076 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:44.078 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:44.079 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:30:44.080 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:30:49.084 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:49.086 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:49.087 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:49.156 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:49.157 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:49.212 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:49.204+0000 7f61af8fc600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:49.212 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:49.205+0000 7f61af8fc600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:49.231 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:30:49.231 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:30:49.232 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:30:49.232 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:30:49.233 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:30:49.233 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:49.234 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:49.234 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:49.235 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:30:49.235 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:49.236 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:30:49.237 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:49.237 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:49.237 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:49.238 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:49.239 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:30:49.239 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:49.240 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:49.241 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:49.241 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:49.242 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:49.243 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:49.243 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:30:49.244 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:49.244 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:49.245 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:30:49.245 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:49.246 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:49.246 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:49.247 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:30:49.247 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:49.248 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:49.248 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:49.249 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:49.249 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:30:49.250 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:49.250 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:49.251 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:49.251 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:49.252 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:49.252 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:49.253 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:49.253 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:49.257 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:49.257 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:49.258 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:49.258 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:30:49.259 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:49.259 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:49.260 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:49.260 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:49.260 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:49.262 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:49.262 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:49.263 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:49.263 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:49.264 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:49.264 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:30:49.265 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:49.265 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:30:49.266 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:30:49.267 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:30:49.267 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:49.268 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:30:49Z" 2022-09-03T06:30:49.268 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:30:49.269 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:49Z" } 2022-09-03T06:30:49.270 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:49.271 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:49.272 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:30:49.273 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:30:54.275 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:54.304 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:54.305 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:54.366 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:54.367 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:54.415 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:54.412+0000 7f41f8031600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:54.417 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:54.414+0000 7f41f8031600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:54.431 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:30:54.432 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:30:54.433 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:30:54.433 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:30:54.434 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:30:54.434 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:54.434 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:54.435 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:54.435 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:30:54.436 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:54.436 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:30:54.436 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:54.437 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:54.437 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:54.438 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:54.438 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:30:54.439 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:54.439 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:54.440 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:54.440 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:54.440 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:54.441 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:54.441 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:30:54.442 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:54.442 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:54.443 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:30:54.443 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:54.443 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:54.444 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:54.444 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:30:54.445 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:54.445 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:54.445 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:54.446 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:54.447 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:30:54.447 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:54.448 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:54.448 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:54.448 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:54.449 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:54.449 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:54.449 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:54.450 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:54.450 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:54.450 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:54.451 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:54.451 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:30:54.451 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:54.452 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:54.452 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:54.452 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:54.453 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:54.453 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:54.453 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:54.457 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:54.458 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:54.459 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:54.459 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:30:54.459 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:54.460 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:30:54.460 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:30:54.460 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:30:54.461 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:54.461 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:30:54Z" 2022-09-03T06:30:54.461 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:30:54.463 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:54Z" } 2022-09-03T06:30:54.464 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:54.464 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:54.465 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:30:54.465 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:30:59.467 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:59.469 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:30:59.470 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:30:59.531 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:30:59.532 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:30:59.563 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:59.560+0000 7f91029c6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:59.568 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:30:59.561+0000 7f91029c6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:30:59.586 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:30:59.587 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:30:59.588 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:30:59.588 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:30:59.589 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:30:59.589 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:30:59.590 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:59.591 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:59.591 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:30:59.591 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:59.592 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:30:59.593 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:59.594 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:59.598 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:30:59.599 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:59.601 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:30:59.604 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:59.604 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:30:59.605 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:59.605 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:59.606 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:59.606 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:59.607 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:30:59.607 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:59.609 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:59.609 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:30:59.610 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:59.610 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:59.611 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:59.611 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:30:59.612 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:30:59.612 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:59.612 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:59.613 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:59.613 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:30:59.614 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:59.615 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:59.615 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:59.616 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:59.616 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:59.623 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:59.623 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:59.624 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:59.624 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:59.625 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:59.625 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:30:59.625 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:30:59.626 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:30:59.628 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:30:59.629 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:30:59.629 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:30:59.629 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:30:59.630 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:30:59.630 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:30:59.631 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:30:59.631 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:59.631 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:30:59.632 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:30:59.632 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:59.632 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:30:59.632 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:30:59.633 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:30:59.633 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:30:59.633 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:30:59Z" 2022-09-03T06:30:59.633 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:30:59.635 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:30:59Z" } 2022-09-03T06:30:59.636 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:30:59.637 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:30:59.637 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:30:59.637 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:31:04.639 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:04.640 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:04.641 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:04.702 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:04.703 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:04.758 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:04.754+0000 7fc18ba2f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:04.759 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:04.755+0000 7fc18ba2f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:04.779 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:04.779 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:04.780 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:04.780 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:04.781 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:04.782 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:04.782 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:04.782 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:04.785 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:04.785 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:04.785 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:04.786 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:04.786 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:04.786 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:04.787 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:04.787 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:04.787 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:04.788 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:04.788 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:04.788 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:04.789 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:04.789 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:04.790 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:04.790 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:04.790 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:04.791 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:31:04.791 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:04.792 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:04.792 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:04.793 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:31:04.794 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:04.794 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:04.795 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:04.796 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:04.796 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:04.797 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:04.797 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:04.797 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:04.798 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:04.798 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:04.799 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:04.799 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:04.799 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:04.800 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:04.800 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:04.801 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:04.801 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:04.802 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:04.802 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:04.802 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:04.803 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:04.803 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:04.804 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:04.804 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:04.805 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:04.806 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:04.807 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:04.807 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:04.808 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:04.808 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:04.808 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:04.809 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:04.809 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:04.810 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:04Z" 2022-09-03T06:31:04.810 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:04.812 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:04Z" } 2022-09-03T06:31:04.814 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:04.814 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:04.815 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:31:04.816 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:31:09.818 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:09.819 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:09.820 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:09.879 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:09.880 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:09.919 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:09.912+0000 7f2771b25600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:09.919 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:09.913+0000 7f2771b25600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:09.929 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:09.930 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:09.930 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:09.930 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:09.931 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:09.931 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:09.931 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:09.932 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:09.932 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:09.932 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:09.933 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:09.933 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:09.933 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:09.934 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:09.934 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:09.935 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:09.935 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:09.935 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:09.936 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:09.936 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:09.936 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:09.937 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:09.937 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:09.938 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:09.938 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:09.938 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:31:09.939 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:09.939 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:09.939 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:09.939 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:31:09.940 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:09.940 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:09.940 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:09.940 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:09.941 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:09.941 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:09.941 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:09.941 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:09.942 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:09.942 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:09.942 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:09.942 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:09.943 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:09.943 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:09.943 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:09.943 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:09.944 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:09.944 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:09.944 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:09.945 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:09.945 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:09.945 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:09.945 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:09.946 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:09.946 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:09.946 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:09.946 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:09.947 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:09.947 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:09.947 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:09.948 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:09.948 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:09.948 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:09.949 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:09Z" 2022-09-03T06:31:09.949 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:09.952 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:09Z" } 2022-09-03T06:31:09.953 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:09.954 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:09.954 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:31:09.955 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:31:14.955 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:14.956 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:14.957 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:15.017 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:15.018 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:15.053 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:15.051+0000 7f713bfa6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:15.055 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:15.053+0000 7f713bfa6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:15.071 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:15.072 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:15.072 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:15.072 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:15.072 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:15.073 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:15.073 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:15.073 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:15.074 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:15.074 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:15.074 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:15.074 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:15.075 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:15.075 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:15.075 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:15.076 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:15.078 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:15.078 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:15.079 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:15.080 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:15.081 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:15.081 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:15.082 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:15.082 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:15.083 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:15.083 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:31:15.084 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:15.084 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:15.085 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:15.085 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:31:15.086 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:15.086 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:15.087 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:15.087 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:15.088 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:15.088 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:15.089 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:15.089 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:15.095 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:15.096 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:15.096 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:15.096 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:15.097 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:15.097 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:15.097 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:15.098 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:15.098 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:15.098 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:15.099 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:15.101 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:15.101 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:15.102 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:15.102 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:15.102 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:15.103 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:15.103 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:15.103 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:15.104 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:15.104 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:15.104 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:15.104 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:15.105 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:15.105 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:15.106 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:15Z" 2022-09-03T06:31:15.107 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:15.108 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:15Z" } 2022-09-03T06:31:15.109 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:15.110 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:15.110 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:31:15.112 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:31:20.113 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:20.115 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:20.116 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:20.178 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:20.179 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:20.212 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:20.209+0000 7fa7e25df600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:20.213 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:20.210+0000 7fa7e25df600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:20.229 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:20.235 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:20.236 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:20.236 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:20.237 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:20.237 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:20.239 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:20.240 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:20.241 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:20.241 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:20.241 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:20.242 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:20.242 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:20.243 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:20.243 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:20.244 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:20.244 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:20.245 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:20.246 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:20.246 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:20.246 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:20.247 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:20.247 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:20.247 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:20.248 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:20.248 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:31:20.248 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:20.248 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:20.249 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:20.249 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:31:20.249 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:20.250 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:20.250 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:20.250 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:20.251 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:20.251 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:20.251 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:20.251 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:20.252 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:20.252 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:20.252 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:20.253 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:20.253 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:20.253 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:20.253 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:20.254 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:20.254 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:20.254 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:20.255 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:20.255 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:20.255 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:20.256 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:20.256 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:20.256 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:20.256 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:20.257 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:20.257 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:20.257 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:20.257 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:20.258 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:20.258 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:20.258 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:20.259 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:20.259 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:20Z" 2022-09-03T06:31:20.259 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:20.262 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:20Z" } 2022-09-03T06:31:20.264 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:20.265 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:20.266 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:31:20.266 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:31:25.267 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:25.268 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:25.270 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:25.346 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:25.347 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:25.394 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:25.391+0000 7fe35332d600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:25.394 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:25.392+0000 7fe35332d600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:25.416 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:25.418 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:25.418 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:25.419 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:25.419 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:25.420 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:25.420 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:25.421 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:25.421 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:25.422 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:25.422 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:25.423 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:25.425 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:25.426 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:25.427 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:25.427 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:25.428 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:25.428 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:25.429 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:25.429 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:25.436 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:25.436 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:25.438 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:25.438 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:25.439 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:25.440 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:31:25.441 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:25.443 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:25.443 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:25.443 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:31:25.444 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:25.444 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:25.445 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:25.445 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:25.445 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:25.446 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:25.446 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:25.446 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:25.447 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:25.447 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:25.449 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:25.449 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:25.449 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:25.451 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:25.451 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:25.451 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:25.452 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:25.452 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:25.453 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:25.453 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:25.454 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:25.454 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:25.455 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:25.455 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:25.455 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:25.456 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:25.456 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:25.456 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:25.457 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:25.457 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:25.458 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:25.460 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:25.460 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:25.461 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:25Z" 2022-09-03T06:31:25.462 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:25.463 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:25Z" } 2022-09-03T06:31:25.466 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:25.468 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:25.469 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:31:25.473 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:31:30.477 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:30.478 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:30.478 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:30.560 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:30.560 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:30.592 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:30.588+0000 7fade9019600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:30.593 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:30.589+0000 7fade9019600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:30.610 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:30.610 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:30.611 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:30.613 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:30.613 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:30.614 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:30.614 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:30.615 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:30.616 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:30.616 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:30.617 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:30.617 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:30.618 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:30.618 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:30.618 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:30.619 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:30.619 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:30.620 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:30.620 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:30.621 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:30.621 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:30.622 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:30.622 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:30.623 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:30.623 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:30.624 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:31:30.624 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:30.625 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:30.625 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:30.627 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:31:30.627 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:30.629 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:30.633 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:30.634 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:30.634 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:30.635 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:30.635 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:30.636 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:30.636 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:30.636 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:30.637 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:30.637 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:30.638 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:30.639 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:30.639 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:30.640 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:30.641 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:30.641 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:30.645 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:30.647 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:30.647 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:30.648 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:30.650 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:30.650 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:30.651 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:30.652 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:30.652 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:30.652 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:30.653 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:30.653 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:30.653 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:30.654 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:30.654 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:30.655 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:30Z" 2022-09-03T06:31:30.656 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:30.657 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:30Z" } 2022-09-03T06:31:30.658 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:30.659 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:30.663 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:31:30.664 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:31:35.680 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:35.681 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:35.682 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:35.748 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:35.748 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:35.803 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:35.796+0000 7fd24ffa6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:35.804 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:35.797+0000 7fd24ffa6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:35.825 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:35.826 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:35.826 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:35.827 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:35.827 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:35.828 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:35.828 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:35.828 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:35.829 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:35.829 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:35.830 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:35.830 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:35.831 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:35.831 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:35.832 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:35.833 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:35.833 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:35.834 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:35.834 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:35.835 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:35.835 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:35.835 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:35.836 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:35.836 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:35.837 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:35.837 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:31:35.838 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:35.838 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:35.839 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:35.839 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:31:35.840 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:35.840 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:35.841 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:35.841 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:35.842 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:35.842 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:35.842 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:35.843 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:35.844 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:35.844 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:35.844 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:35.845 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:35.845 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:35.847 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:35.847 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:35.848 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:35.848 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:35.849 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:35.850 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:35.850 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:35.850 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:35.851 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:35.851 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:35.852 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:35.852 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:35.853 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:35.853 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:35.854 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:35.854 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:35.855 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:35.856 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:35.857 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:35.857 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:35.857 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:35Z" 2022-09-03T06:31:35.857 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:35.858 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:35Z" } 2022-09-03T06:31:35.859 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:35.859 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:35.860 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:31:35.861 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:31:40.863 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:40.864 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:40.865 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:40.926 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:40.927 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:40.964 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:40.955+0000 7f84903e7600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:40.964 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:40.956+0000 7f84903e7600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:40.973 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:40.974 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:40.974 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:40.975 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:40.975 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:40.976 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:40.976 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:40.977 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:40.978 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:40.978 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:40.979 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:40.979 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:40.980 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:40.980 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:40.981 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:40.981 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:40.981 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:40.982 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:40.982 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:40.992 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:40.993 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:40.994 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:40.995 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:40.995 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:40.996 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:40.996 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186329.367916_186.1", 2022-09-03T06:31:40.997 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:40.997 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:41.000 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:41.001 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:25:29.367916Z", 2022-09-03T06:31:41.001 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:41.001 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:41.002 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:41.003 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:41.003 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:41.004 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:41.004 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:41.005 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:41.005 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:41.006 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:41.006 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:41.007 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:41.007 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:41.008 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:41.008 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:41.009 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:41.009 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:41.010 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:41.010 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:41.011 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:41.011 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:41.012 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:41.012 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:41.013 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:41.013 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:41.014 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:41.014 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:41.015 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:41.016 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:41.016 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:41.017 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:41.017 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:41.018 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:41.018 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:40Z" 2022-09-03T06:31:41.019 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:41.021 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186329.367916_186.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:25:29.367916Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:40Z" } 2022-09-03T06:31:41.022 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186329.367916_186.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:25:29.367916Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:41.023 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:41.024 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186329.367916_186.1', 2: '', 3: ''} 2022-09-03T06:31:41.025 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186329.367916_186.1 2022-09-03T06:31:46.027 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:46.028 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:46.028 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:46.088 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:46.089 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:46.131 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:46.128+0000 7f6d6a2e7600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:46.132 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:46.129+0000 7f6d6a2e7600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:46.156 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:46.156 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:46.157 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:46.157 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:46.160 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:46.160 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:46.161 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:46.161 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:46.162 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:46.163 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:46.163 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:46.163 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:46.164 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:46.164 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:46.165 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:46.165 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:46.166 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:46.166 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:46.167 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:46.167 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:46.167 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:46.168 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:46.168 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:46.169 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:46.169 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:46.170 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186583.081419_232.1", 2022-09-03T06:31:46.170 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:46.171 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:46.171 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:46.171 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:29:43.081419Z", 2022-09-03T06:31:46.172 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:46.172 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:46.172 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:46.172 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:46.173 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:46.173 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:46.173 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:46.173 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:46.174 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:46.174 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:46.174 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:46.174 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:46.175 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:46.175 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:46.176 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:46.176 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:46.177 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:46.177 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:46.178 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:46.178 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:46.179 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:46.179 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:46.182 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:46.182 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:46.183 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:46.183 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:46.183 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:46.184 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:46.184 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:46.184 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:46.185 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:46.185 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:46.185 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:46.186 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:46Z" 2022-09-03T06:31:46.186 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:46.186 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186583.081419_232.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:29:43.081419Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:46Z" } 2022-09-03T06:31:46.187 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186583.081419_232.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:29:43.081419Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:46.188 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:46.188 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186583.081419_232.1', 2: '', 3: ''} 2022-09-03T06:31:46.189 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186583.081419_232.1 2022-09-03T06:31:51.190 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:51.191 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:51.192 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:51.252 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:51.254 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:51.295 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:51.293+0000 7fd39e71a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:51.297 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:31:51.295+0000 7fd39e71a600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:51.315 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:31:51.319 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:31:51.319 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:31:51.320 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:31:51.320 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:31:51.320 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:51.321 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:51.321 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:51.322 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:31:51.322 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:51.323 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:31:51.323 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:51.324 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:51.324 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:51.325 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:51.325 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:31:51.326 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:51.326 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:51.327 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:51.327 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:51.328 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:51.328 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:51.329 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:31:51.329 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:51.330 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:51.330 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186586.669522_282.1", 2022-09-03T06:31:51.331 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:51.331 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:51.332 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:51.332 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:29:46.669522Z", 2022-09-03T06:31:51.333 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:31:51.334 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:51.334 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:51.335 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:51.335 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:31:51.336 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:51.336 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:51.337 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:51.337 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:51.338 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:51.339 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:51.339 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:51.339 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:51.345 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:51.345 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:51.345 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:31:51.346 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:31:51.346 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:31:51.346 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:31:51.347 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:31:51.347 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:31:51.347 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:31:51.347 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:31:51.348 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:31:51.348 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:31:51.348 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:51.348 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:31:51.349 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:31:51.349 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:51.349 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:31:51.350 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:31:51.351 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:31:51.351 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:31:51.351 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:31:51Z" 2022-09-03T06:31:51.351 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:31:51.353 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186586.669522_282.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:29:46.669522Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:51Z" } 2022-09-03T06:31:51.354 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186586.669522_282.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:29:46.669522Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:51.355 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:51.355 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:51.356 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:31:51.356 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:31:51.357 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:51.358 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:51.358 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:51.415 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:51.416 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:51.443 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:31:51.439+0000 7ff0ae3c2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:51.443 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:31:51.440+0000 7ff0ae3c2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:51.468 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:31:51.470 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:31:51.470 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:31:51.471 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:31:51.471 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:31:51.471 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:51.472 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:31:51.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:51.472 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:31:51.472 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:31:51.473 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:31:51.473 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:31:51.473 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:31:51.474 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:51.481 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:31:51.481 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:31:51.481 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:31:51.482 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:51.482 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:31:51.482 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:51.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:51.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:31:51.483 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:31:51.484 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:31:51.484 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:31:51.484 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186583.348188_249.1", 2022-09-03T06:31:51.484 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:31:51.485 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:31:51.485 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:31:51.485 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:29:43.348188Z", 2022-09-03T06:31:51.485 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:31:51.486 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:51.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:51.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:31:51.487 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:31:51.487 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:31:51.488 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:31:51.488 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:31:51.488 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:31:51.488 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:31:51.489 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:31:51.489 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:31:51.489 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:31:51.489 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:51.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:51.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:31:51.490 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:31:51.490 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:31:51.491 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:31:51.491 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:31:51.491 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:31:51.491 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:31:51.492 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:31:51.492 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:31:51.492 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:31:51.492 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:51.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:51.493 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:31:51.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:51.494 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:31:51.494 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:31:51.494 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:31:51.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:51.495 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:31:51Z" 2022-09-03T06:31:51.495 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:31:51.496 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186583.348188_249.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:29:43.348188Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:51Z" } 2022-09-03T06:31:51.496 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186583.348188_249.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:29:43.348188Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:51.497 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:51.498 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186583.348188_249.1', 2: '', 3: ''} 2022-09-03T06:31:51.498 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186583.348188_249.1 2022-09-03T06:31:56.499 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:56.501 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:31:56.502 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:31:56.587 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:31:56.587 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:31:56.647 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:31:56.635+0000 7fc201945600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:56.647 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:31:56.636+0000 7fc201945600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:31:56.653 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:31:56.654 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:31:56.654 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:31:56.655 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:31:56.655 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:31:56.655 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:31:56.655 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:31:56.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:56.656 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:31:56.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:31:56.657 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:31:56.657 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:31:56.657 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:31:56.657 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:31:56.658 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:31:56.658 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:31:56.658 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:31:56.659 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:31:56.659 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:31:56.659 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:56.660 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:56.660 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:31:56.661 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:31:56.661 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:31:56.661 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:31:56.662 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186583.446360_253.1", 2022-09-03T06:31:56.662 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:31:56.663 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:31:56.663 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:31:56.663 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:29:43.446360Z", 2022-09-03T06:31:56.664 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:31:56.664 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:56.664 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:56.664 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:31:56.665 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:31:56.665 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:31:56.665 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:31:56.665 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:31:56.666 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:31:56.666 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:31:56.666 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:31:56.667 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:31:56.667 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:31:56.667 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:56.668 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:56.668 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:31:56.668 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:31:56.668 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:31:56.669 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:31:56.669 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:31:56.669 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:31:56.669 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:31:56.670 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:31:56.670 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:31:56.671 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:31:56.671 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:56.671 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:31:56.672 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:31:56.672 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:56.672 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:31:56.672 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:31:56.673 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:31:56.674 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:31:56.674 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:31:56Z" 2022-09-03T06:31:56.674 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:31:56.676 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186583.446360_253.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:29:43.446360Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:31:56Z" } 2022-09-03T06:31:56.676 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186583.446360_253.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:29:43.446360Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:31:56.677 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:31:56.678 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186583.446360_253.1', 2: '', 3: ''} 2022-09-03T06:31:56.678 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=1 master=1_1662186586.669522_282.1 target=1_1662186583.446360_253.1 2022-09-03T06:32:01.680 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:01.681 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:01.682 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:01.744 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:01.745 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:01.783 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:01.780+0000 7f1b9683e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:01.785 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:01.782+0000 7f1b9683e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:01.798 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:32:01.799 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:32:01.800 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:32:01.800 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:32:01.801 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:32:01.801 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:32:01.802 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:32:01.802 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:01.803 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:32:01.803 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:01.804 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:32:01.804 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:01.804 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:01.805 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:32:01.805 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:01.807 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:32:01.808 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:01.808 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:32:01.809 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:01.809 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:01.809 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:01.810 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:01.810 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:32:01.811 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:01.811 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:01.812 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186586.669522_282.1", 2022-09-03T06:32:01.812 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:01.812 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:01.812 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:01.813 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:29:46.669522Z", 2022-09-03T06:32:01.813 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:32:01.813 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:01.813 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:01.814 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:01.815 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:32:01.815 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:01.815 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:01.816 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:32:01.816 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:01.817 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:01.818 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:01.819 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:32:01.821 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:01.822 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:01.823 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:01.824 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:01.827 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:32:01.829 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:01.830 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:01.831 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:32:01.832 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:01.833 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:01.833 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:01.834 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:32:01.835 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:01.836 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:01.837 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:01.837 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:32:01.838 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:01.838 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:32:01.839 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:32:01.840 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:32:01.840 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:01.841 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:32:01Z" 2022-09-03T06:32:01.841 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:32:01.842 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186586.669522_282.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:29:46.669522Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:32:01Z" } 2022-09-03T06:32:01.843 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186586.669522_282.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:29:46.669522Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:32:01.844 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:32:01.845 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186586.669522_282.1', 2: '', 3: ''} 2022-09-03T06:32:01.846 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:32:01.849 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'avysmw-56', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:32:01.850 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'avysmw-56', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:32:01.851 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket avysmw-56 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:32:01.915 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:01.916 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:02.021 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'avysmw-57', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:32:02.022 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'avysmw-57', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:32:02.022 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket avysmw-57 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:32:02.075 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:02.076 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:02.171 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'avysmw-58', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:32:02.172 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'avysmw-58', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:32:02.174 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket avysmw-58 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:32:02.230 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:02.231 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:02.313 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['log', 'list', '--rgw-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:02.314 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'log', 'list', '--rgw-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:02.315 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 log list --rgw-zone test-zone1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:02.364 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:02.364 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:02.437 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:32:02.438 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000078", 2022-09-03T06:32:02.438 INFO:teuthology.orchestra.run.smithi062.stdout: "meta.history", 2022-09-03T06:32:02.439 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:4", 2022-09-03T06:32:02.439 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:5", 2022-09-03T06:32:02.439 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4", 2022-09-03T06:32:02.439 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:5", 2022-09-03T06:32:02.440 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:8", 2022-09-03T06:32:02.440 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.23", 2022-09-03T06:32:02.440 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000070", 2022-09-03T06:32:02.441 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000104", 2022-09-03T06:32:02.441 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000026", 2022-09-03T06:32:02.441 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:8", 2022-09-03T06:32:02.442 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:6", 2022-09-03T06:32:02.442 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:9", 2022-09-03T06:32:02.442 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:7", 2022-09-03T06:32:02.442 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:9", 2022-09-03T06:32:02.443 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:7", 2022-09-03T06:32:02.443 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.3.nelwPEmr1wm.0", 2022-09-03T06:32:02.443 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:3", 2022-09-03T06:32:02.444 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.8", 2022-09-03T06:32:02.444 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000028", 2022-09-03T06:32:02.445 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:2", 2022-09-03T06:32:02.445 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:1", 2022-09-03T06:32:02.446 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:1", 2022-09-03T06:32:02.446 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:8", 2022-09-03T06:32:02.446 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:10", 2022-09-03T06:32:02.447 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:4", 2022-09-03T06:32:02.447 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:8", 2022-09-03T06:32:02.447 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:8", 2022-09-03T06:32:02.448 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:7", 2022-09-03T06:32:02.448 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:32:02.448 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:9", 2022-09-03T06:32:02.448 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:10", 2022-09-03T06:32:02.449 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:8", 2022-09-03T06:32:02.449 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:9", 2022-09-03T06:32:02.450 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:2", 2022-09-03T06:32:02.451 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000040", 2022-09-03T06:32:02.451 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:9", 2022-09-03T06:32:02.452 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.6", 2022-09-03T06:32:02.452 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.2.retry", 2022-09-03T06:32:02.453 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:5", 2022-09-03T06:32:02.453 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:9", 2022-09-03T06:32:02.454 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:10", 2022-09-03T06:32:02.454 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:6", 2022-09-03T06:32:02.455 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000015", 2022-09-03T06:32:02.455 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000069", 2022-09-03T06:32:02.456 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:7", 2022-09-03T06:32:02.456 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:9", 2022-09-03T06:32:02.457 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000095", 2022-09-03T06:32:02.457 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.0.yRIzwNyFfnl.0", 2022-09-03T06:32:02.458 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:0", 2022-09-03T06:32:02.458 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000003", 2022-09-03T06:32:02.458 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000047", 2022-09-03T06:32:02.458 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:0", 2022-09-03T06:32:02.459 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:3", 2022-09-03T06:32:02.459 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:2", 2022-09-03T06:32:02.459 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:32:02.460 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:3", 2022-09-03T06:32:02.460 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000035", 2022-09-03T06:32:02.460 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:10", 2022-09-03T06:32:02.461 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:7", 2022-09-03T06:32:02.461 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000037", 2022-09-03T06:32:02.461 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:3", 2022-09-03T06:32:02.462 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:9", 2022-09-03T06:32:02.462 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:8", 2022-09-03T06:32:02.463 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:8", 2022-09-03T06:32:02.463 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:7", 2022-09-03T06:32:02.463 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:4", 2022-09-03T06:32:02.464 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:1", 2022-09-03T06:32:02.464 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:32:02.464 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:3", 2022-09-03T06:32:02.465 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000024", 2022-09-03T06:32:02.465 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:7", 2022-09-03T06:32:02.465 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:7", 2022-09-03T06:32:02.466 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:6", 2022-09-03T06:32:02.466 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:1", 2022-09-03T06:32:02.466 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:5", 2022-09-03T06:32:02.466 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000000", 2022-09-03T06:32:02.467 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:32:02.467 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:3", 2022-09-03T06:32:02.467 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000031", 2022-09-03T06:32:02.467 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:3", 2022-09-03T06:32:02.468 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:5", 2022-09-03T06:32:02.468 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:8", 2022-09-03T06:32:02.468 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.3", 2022-09-03T06:32:02.468 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:7", 2022-09-03T06:32:02.469 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:0", 2022-09-03T06:32:02.469 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:10", 2022-09-03T06:32:02.469 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:2", 2022-09-03T06:32:02.469 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:2", 2022-09-03T06:32:02.470 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:5", 2022-09-03T06:32:02.470 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:6", 2022-09-03T06:32:02.470 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:9", 2022-09-03T06:32:02.471 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:32:02.471 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:10", 2022-09-03T06:32:02.471 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:0", 2022-09-03T06:32:02.471 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:3", 2022-09-03T06:32:02.472 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:9", 2022-09-03T06:32:02.472 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:0", 2022-09-03T06:32:02.472 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:8", 2022-09-03T06:32:02.472 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:4", 2022-09-03T06:32:02.473 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:6", 2022-09-03T06:32:02.473 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:5", 2022-09-03T06:32:02.474 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:32:02.474 INFO:teuthology.orchestra.run.smithi062.stdout: "data_loggenerations_metadata", 2022-09-03T06:32:02.474 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:32:02.474 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:4", 2022-09-03T06:32:02.475 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:0", 2022-09-03T06:32:02.475 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:1", 2022-09-03T06:32:02.475 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:7", 2022-09-03T06:32:02.475 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:7", 2022-09-03T06:32:02.476 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000076", 2022-09-03T06:32:02.476 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:3", 2022-09-03T06:32:02.476 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000113", 2022-09-03T06:32:02.477 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:32:02.477 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000005", 2022-09-03T06:32:02.477 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:3", 2022-09-03T06:32:02.477 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:4", 2022-09-03T06:32:02.478 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000011", 2022-09-03T06:32:02.478 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000058", 2022-09-03T06:32:02.478 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:7", 2022-09-03T06:32:02.478 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000052", 2022-09-03T06:32:02.479 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:4", 2022-09-03T06:32:02.479 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:5", 2022-09-03T06:32:02.479 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:3", 2022-09-03T06:32:02.480 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:32:02.480 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:1", 2022-09-03T06:32:02.480 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.21", 2022-09-03T06:32:02.480 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:0", 2022-09-03T06:32:02.481 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:6", 2022-09-03T06:32:02.481 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:7", 2022-09-03T06:32:02.481 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:1", 2022-09-03T06:32:02.481 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:0", 2022-09-03T06:32:02.482 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:7", 2022-09-03T06:32:02.482 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:0", 2022-09-03T06:32:02.482 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:0", 2022-09-03T06:32:02.483 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:3", 2022-09-03T06:32:02.484 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:2", 2022-09-03T06:32:02.484 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:6", 2022-09-03T06:32:02.484 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:9", 2022-09-03T06:32:02.485 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:7", 2022-09-03T06:32:02.485 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:10", 2022-09-03T06:32:02.485 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:7", 2022-09-03T06:32:02.485 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000088", 2022-09-03T06:32:02.486 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:9", 2022-09-03T06:32:02.486 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000080", 2022-09-03T06:32:02.486 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:4", 2022-09-03T06:32:02.487 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000090", 2022-09-03T06:32:02.487 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:2", 2022-09-03T06:32:02.488 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:5", 2022-09-03T06:32:02.488 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:1", 2022-09-03T06:32:02.489 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000110", 2022-09-03T06:32:02.489 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:2", 2022-09-03T06:32:02.489 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:3", 2022-09-03T06:32:02.490 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:1", 2022-09-03T06:32:02.490 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:8", 2022-09-03T06:32:02.491 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000096", 2022-09-03T06:32:02.491 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:7", 2022-09-03T06:32:02.491 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:10", 2022-09-03T06:32:02.492 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:8", 2022-09-03T06:32:02.492 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:3", 2022-09-03T06:32:02.492 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:4", 2022-09-03T06:32:02.492 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000087", 2022-09-03T06:32:02.493 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:2", 2022-09-03T06:32:02.493 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:9", 2022-09-03T06:32:02.493 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000008", 2022-09-03T06:32:02.494 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1", 2022-09-03T06:32:02.494 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:6", 2022-09-03T06:32:02.494 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:5", 2022-09-03T06:32:02.495 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:4", 2022-09-03T06:32:02.495 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:5", 2022-09-03T06:32:02.495 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:5", 2022-09-03T06:32:02.496 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000006", 2022-09-03T06:32:02.496 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000029", 2022-09-03T06:32:02.496 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:7", 2022-09-03T06:32:02.497 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:5", 2022-09-03T06:32:02.497 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:2", 2022-09-03T06:32:02.497 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000089", 2022-09-03T06:32:02.498 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:1", 2022-09-03T06:32:02.499 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:4", 2022-09-03T06:32:02.499 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000022", 2022-09-03T06:32:02.500 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:1", 2022-09-03T06:32:02.500 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:32:02.500 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:2", 2022-09-03T06:32:02.501 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:5", 2022-09-03T06:32:02.501 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:6", 2022-09-03T06:32:02.502 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:10", 2022-09-03T06:32:02.502 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:7", 2022-09-03T06:32:02.503 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:2", 2022-09-03T06:32:02.503 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog.trim", 2022-09-03T06:32:02.504 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000016", 2022-09-03T06:32:02.505 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:2", 2022-09-03T06:32:02.505 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:0", 2022-09-03T06:32:02.506 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:8", 2022-09-03T06:32:02.506 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:2", 2022-09-03T06:32:02.507 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:0", 2022-09-03T06:32:02.507 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000041", 2022-09-03T06:32:02.508 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:4", 2022-09-03T06:32:02.508 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:9", 2022-09-03T06:32:02.509 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:0", 2022-09-03T06:32:02.509 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:9", 2022-09-03T06:32:02.510 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:5", 2022-09-03T06:32:02.510 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000018", 2022-09-03T06:32:02.511 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:32:02.511 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000092", 2022-09-03T06:32:02.512 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:32:02.512 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:2", 2022-09-03T06:32:02.513 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:2", 2022-09-03T06:32:02.513 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:6", 2022-09-03T06:32:02.514 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:6", 2022-09-03T06:32:02.515 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:10", 2022-09-03T06:32:02.515 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.27", 2022-09-03T06:32:02.516 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:6", 2022-09-03T06:32:02.516 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.4", 2022-09-03T06:32:02.517 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:8", 2022-09-03T06:32:02.517 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:6", 2022-09-03T06:32:02.518 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000014", 2022-09-03T06:32:02.518 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:2", 2022-09-03T06:32:02.519 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:4", 2022-09-03T06:32:02.519 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000112", 2022-09-03T06:32:02.520 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000007", 2022-09-03T06:32:02.520 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000021", 2022-09-03T06:32:02.521 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:10", 2022-09-03T06:32:02.521 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.16", 2022-09-03T06:32:02.522 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000064", 2022-09-03T06:32:02.522 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:4", 2022-09-03T06:32:02.523 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000071", 2022-09-03T06:32:02.523 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:1", 2022-09-03T06:32:02.524 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:3", 2022-09-03T06:32:02.524 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:6", 2022-09-03T06:32:02.525 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:4", 2022-09-03T06:32:02.525 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:1", 2022-09-03T06:32:02.526 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000074", 2022-09-03T06:32:02.526 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:0", 2022-09-03T06:32:02.527 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:4", 2022-09-03T06:32:02.527 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:1", 2022-09-03T06:32:02.528 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:0", 2022-09-03T06:32:02.529 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:8", 2022-09-03T06:32:02.529 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:5", 2022-09-03T06:32:02.530 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000081", 2022-09-03T06:32:02.530 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000009", 2022-09-03T06:32:02.531 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5", 2022-09-03T06:32:02.531 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:8", 2022-09-03T06:32:02.532 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000121", 2022-09-03T06:32:02.532 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.1", 2022-09-03T06:32:02.533 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:10", 2022-09-03T06:32:02.533 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:10", 2022-09-03T06:32:02.533 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.17", 2022-09-03T06:32:02.534 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:5", 2022-09-03T06:32:02.535 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000125", 2022-09-03T06:32:02.535 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:5", 2022-09-03T06:32:02.536 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000082", 2022-09-03T06:32:02.536 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.0", 2022-09-03T06:32:02.537 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:9", 2022-09-03T06:32:02.537 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:8", 2022-09-03T06:32:02.538 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:2", 2022-09-03T06:32:02.538 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:32:02.539 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000105", 2022-09-03T06:32:02.539 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000059", 2022-09-03T06:32:02.540 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:5", 2022-09-03T06:32:02.541 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000077", 2022-09-03T06:32:02.541 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:3", 2022-09-03T06:32:02.542 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000032", 2022-09-03T06:32:02.542 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:1", 2022-09-03T06:32:02.543 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:8", 2022-09-03T06:32:02.543 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:1", 2022-09-03T06:32:02.544 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:2", 2022-09-03T06:32:02.544 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:32:02.545 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:9", 2022-09-03T06:32:02.545 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:4", 2022-09-03T06:32:02.546 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:32:02.546 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:9", 2022-09-03T06:32:02.547 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:3", 2022-09-03T06:32:02.547 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:6", 2022-09-03T06:32:02.548 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:8", 2022-09-03T06:32:02.548 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.24", 2022-09-03T06:32:02.549 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:10", 2022-09-03T06:32:02.549 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:3", 2022-09-03T06:32:02.549 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:3", 2022-09-03T06:32:02.550 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:7", 2022-09-03T06:32:02.550 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:3", 2022-09-03T06:32:02.551 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:32:02.551 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:1", 2022-09-03T06:32:02.551 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:32:02.551 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.3", 2022-09-03T06:32:02.552 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:10", 2022-09-03T06:32:02.552 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:3", 2022-09-03T06:32:02.553 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000053", 2022-09-03T06:32:02.553 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:8", 2022-09-03T06:32:02.554 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:0", 2022-09-03T06:32:02.554 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.0.retry", 2022-09-03T06:32:02.555 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000091", 2022-09-03T06:32:02.555 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:7", 2022-09-03T06:32:02.556 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:5", 2022-09-03T06:32:02.556 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:5", 2022-09-03T06:32:02.557 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:8", 2022-09-03T06:32:02.557 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:9", 2022-09-03T06:32:02.558 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:2", 2022-09-03T06:32:02.558 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:1", 2022-09-03T06:32:02.559 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000065", 2022-09-03T06:32:02.559 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:8", 2022-09-03T06:32:02.560 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:3", 2022-09-03T06:32:02.560 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000083", 2022-09-03T06:32:02.561 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000010", 2022-09-03T06:32:02.561 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.2.e+J0tXhMknk.0", 2022-09-03T06:32:02.562 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:8", 2022-09-03T06:32:02.563 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:9", 2022-09-03T06:32:02.563 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:4", 2022-09-03T06:32:02.564 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:9", 2022-09-03T06:32:02.564 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:9", 2022-09-03T06:32:02.565 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.7", 2022-09-03T06:32:02.565 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.3", 2022-09-03T06:32:02.566 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:2", 2022-09-03T06:32:02.566 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:6", 2022-09-03T06:32:02.566 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:9", 2022-09-03T06:32:02.567 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:1", 2022-09-03T06:32:02.567 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:5", 2022-09-03T06:32:02.568 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:7", 2022-09-03T06:32:02.568 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:8", 2022-09-03T06:32:02.569 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:8", 2022-09-03T06:32:02.569 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:5", 2022-09-03T06:32:02.569 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:10", 2022-09-03T06:32:02.570 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:10", 2022-09-03T06:32:02.571 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status.shard.2", 2022-09-03T06:32:02.571 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:9", 2022-09-03T06:32:02.571 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:0", 2022-09-03T06:32:02.572 INFO:teuthology.orchestra.run.smithi062.stdout: "meta.log.2962e93e-ca58-4758-b6d9-6fe2e67c2976.1", 2022-09-03T06:32:02.572 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:7", 2022-09-03T06:32:02.572 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:9", 2022-09-03T06:32:02.572 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:4", 2022-09-03T06:32:02.573 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000045", 2022-09-03T06:32:02.573 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:10", 2022-09-03T06:32:02.574 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000002", 2022-09-03T06:32:02.574 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:0", 2022-09-03T06:32:02.575 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.12", 2022-09-03T06:32:02.575 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:10", 2022-09-03T06:32:02.576 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:32:02.577 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:5", 2022-09-03T06:32:02.577 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.0", 2022-09-03T06:32:02.578 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000116", 2022-09-03T06:32:02.578 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:7", 2022-09-03T06:32:02.579 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.25", 2022-09-03T06:32:02.579 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:2", 2022-09-03T06:32:02.580 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:2", 2022-09-03T06:32:02.580 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:1", 2022-09-03T06:32:02.580 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:4", 2022-09-03T06:32:02.581 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:10", 2022-09-03T06:32:02.581 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:0", 2022-09-03T06:32:02.582 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:10", 2022-09-03T06:32:02.582 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:1", 2022-09-03T06:32:02.583 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000034", 2022-09-03T06:32:02.583 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:6", 2022-09-03T06:32:02.583 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:4", 2022-09-03T06:32:02.584 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:1", 2022-09-03T06:32:02.584 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:9", 2022-09-03T06:32:02.585 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000101", 2022-09-03T06:32:02.585 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:1", 2022-09-03T06:32:02.585 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:5", 2022-09-03T06:32:02.586 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:1", 2022-09-03T06:32:02.586 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.13", 2022-09-03T06:32:02.586 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status.shard.1", 2022-09-03T06:32:02.587 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:7", 2022-09-03T06:32:02.587 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:8", 2022-09-03T06:32:02.587 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000079", 2022-09-03T06:32:02.587 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000049", 2022-09-03T06:32:02.588 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:2", 2022-09-03T06:32:02.588 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000117", 2022-09-03T06:32:02.588 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000044", 2022-09-03T06:32:02.589 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:32:02.589 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:3", 2022-09-03T06:32:02.589 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:0", 2022-09-03T06:32:02.590 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:3", 2022-09-03T06:32:02.590 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.14", 2022-09-03T06:32:02.590 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:4", 2022-09-03T06:32:02.590 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000066", 2022-09-03T06:32:02.591 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:10", 2022-09-03T06:32:02.591 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:0", 2022-09-03T06:32:02.592 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:5", 2022-09-03T06:32:02.592 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:10", 2022-09-03T06:32:02.593 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.2", 2022-09-03T06:32:02.593 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:1", 2022-09-03T06:32:02.594 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:6", 2022-09-03T06:32:02.594 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:7", 2022-09-03T06:32:02.594 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:8", 2022-09-03T06:32:02.595 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:6", 2022-09-03T06:32:02.595 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:8", 2022-09-03T06:32:02.596 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:10", 2022-09-03T06:32:02.596 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000068", 2022-09-03T06:32:02.596 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:8", 2022-09-03T06:32:02.597 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.1.zQIyxYGCP2A.0", 2022-09-03T06:32:02.597 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000085", 2022-09-03T06:32:02.598 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:8", 2022-09-03T06:32:02.598 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:1", 2022-09-03T06:32:02.599 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:5", 2022-09-03T06:32:02.599 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:6", 2022-09-03T06:32:02.600 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:8", 2022-09-03T06:32:02.600 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:10", 2022-09-03T06:32:02.601 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:4", 2022-09-03T06:32:02.601 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:0", 2022-09-03T06:32:02.601 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:3", 2022-09-03T06:32:02.601 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:3", 2022-09-03T06:32:02.602 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000073", 2022-09-03T06:32:02.602 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:7", 2022-09-03T06:32:02.602 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:2", 2022-09-03T06:32:02.603 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000038", 2022-09-03T06:32:02.603 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:9", 2022-09-03T06:32:02.603 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:8", 2022-09-03T06:32:02.603 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.10", 2022-09-03T06:32:02.604 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:5", 2022-09-03T06:32:02.604 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000118", 2022-09-03T06:32:02.604 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:3", 2022-09-03T06:32:02.605 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000036", 2022-09-03T06:32:02.605 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:6", 2022-09-03T06:32:02.605 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:1", 2022-09-03T06:32:02.606 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:4", 2022-09-03T06:32:02.606 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000103", 2022-09-03T06:32:02.606 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:5", 2022-09-03T06:32:02.607 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:6", 2022-09-03T06:32:02.607 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status.shard.3", 2022-09-03T06:32:02.608 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:0", 2022-09-03T06:32:02.608 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:10", 2022-09-03T06:32:02.609 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:1", 2022-09-03T06:32:02.609 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000119", 2022-09-03T06:32:02.610 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:8", 2022-09-03T06:32:02.610 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:6", 2022-09-03T06:32:02.610 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:0", 2022-09-03T06:32:02.610 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:6", 2022-09-03T06:32:02.611 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000098", 2022-09-03T06:32:02.611 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:32:02.611 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:32:02.612 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:32:02.612 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:10", 2022-09-03T06:32:02.612 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:32:02.613 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:2", 2022-09-03T06:32:02.613 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:6", 2022-09-03T06:32:02.613 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:2", 2022-09-03T06:32:02.614 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:10", 2022-09-03T06:32:02.614 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:5", 2022-09-03T06:32:02.615 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status.shard.0", 2022-09-03T06:32:02.615 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:2", 2022-09-03T06:32:02.616 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:9", 2022-09-03T06:32:02.616 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000027", 2022-09-03T06:32:02.617 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:8", 2022-09-03T06:32:02.618 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status", 2022-09-03T06:32:02.618 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000019", 2022-09-03T06:32:02.618 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:0", 2022-09-03T06:32:02.618 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:0", 2022-09-03T06:32:02.619 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1", 2022-09-03T06:32:02.619 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000039", 2022-09-03T06:32:02.619 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:8", 2022-09-03T06:32:02.620 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:8", 2022-09-03T06:32:02.620 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:8", 2022-09-03T06:32:02.620 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:0", 2022-09-03T06:32:02.620 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:10", 2022-09-03T06:32:02.621 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:32:02.621 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:9", 2022-09-03T06:32:02.621 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:4", 2022-09-03T06:32:02.621 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:4", 2022-09-03T06:32:02.622 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:2", 2022-09-03T06:32:02.622 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:7", 2022-09-03T06:32:02.622 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:0", 2022-09-03T06:32:02.622 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:0", 2022-09-03T06:32:02.623 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000100", 2022-09-03T06:32:02.623 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:32:02.623 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:4", 2022-09-03T06:32:02.623 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:0", 2022-09-03T06:32:02.624 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:3", 2022-09-03T06:32:02.624 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:32:02.624 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:9", 2022-09-03T06:32:02.624 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:8", 2022-09-03T06:32:02.625 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000093", 2022-09-03T06:32:02.625 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:0", 2022-09-03T06:32:02.625 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:3", 2022-09-03T06:32:02.626 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:7", 2022-09-03T06:32:02.626 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:9", 2022-09-03T06:32:02.626 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000004", 2022-09-03T06:32:02.627 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000063", 2022-09-03T06:32:02.627 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:2", 2022-09-03T06:32:02.627 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:3", 2022-09-03T06:32:02.627 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000122", 2022-09-03T06:32:02.628 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:1", 2022-09-03T06:32:02.628 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000057", 2022-09-03T06:32:02.628 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:4", 2022-09-03T06:32:02.628 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:7", 2022-09-03T06:32:02.629 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:7", 2022-09-03T06:32:02.629 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.1", 2022-09-03T06:32:02.629 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000054", 2022-09-03T06:32:02.629 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000114", 2022-09-03T06:32:02.630 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:10", 2022-09-03T06:32:02.630 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:6", 2022-09-03T06:32:02.630 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:3", 2022-09-03T06:32:02.631 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.3.retry", 2022-09-03T06:32:02.631 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.2", 2022-09-03T06:32:02.631 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.0", 2022-09-03T06:32:02.631 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:7", 2022-09-03T06:32:02.632 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:2", 2022-09-03T06:32:02.632 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000012", 2022-09-03T06:32:02.632 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:3", 2022-09-03T06:32:02.632 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:3", 2022-09-03T06:32:02.633 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:9", 2022-09-03T06:32:02.633 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:6", 2022-09-03T06:32:02.633 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:4", 2022-09-03T06:32:02.633 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.2", 2022-09-03T06:32:02.634 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:8", 2022-09-03T06:32:02.634 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:10", 2022-09-03T06:32:02.634 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:7", 2022-09-03T06:32:02.635 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:5", 2022-09-03T06:32:02.635 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:4", 2022-09-03T06:32:02.635 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:3", 2022-09-03T06:32:02.636 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000084", 2022-09-03T06:32:02.636 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000043", 2022-09-03T06:32:02.636 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:6", 2022-09-03T06:32:02.636 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:2", 2022-09-03T06:32:02.637 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:0", 2022-09-03T06:32:02.637 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:6", 2022-09-03T06:32:02.637 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:7", 2022-09-03T06:32:02.637 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:5", 2022-09-03T06:32:02.638 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.11", 2022-09-03T06:32:02.638 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:5", 2022-09-03T06:32:02.638 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:10", 2022-09-03T06:32:02.638 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:32:02.639 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:5", 2022-09-03T06:32:02.639 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:3", 2022-09-03T06:32:02.639 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.1.retry", 2022-09-03T06:32:02.639 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:32:02.640 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:10", 2022-09-03T06:32:02.640 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:5", 2022-09-03T06:32:02.640 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:6", 2022-09-03T06:32:02.641 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:8", 2022-09-03T06:32:02.641 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000111", 2022-09-03T06:32:02.641 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000048", 2022-09-03T06:32:02.641 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:2", 2022-09-03T06:32:02.642 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:8", 2022-09-03T06:32:02.642 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000020", 2022-09-03T06:32:02.642 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:1", 2022-09-03T06:32:02.642 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:4", 2022-09-03T06:32:02.643 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:2", 2022-09-03T06:32:02.643 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:4", 2022-09-03T06:32:02.643 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:5", 2022-09-03T06:32:02.644 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:8", 2022-09-03T06:32:02.644 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.3", 2022-09-03T06:32:02.644 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:1", 2022-09-03T06:32:02.644 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000099", 2022-09-03T06:32:02.645 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000056", 2022-09-03T06:32:02.645 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:3", 2022-09-03T06:32:02.645 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:5", 2022-09-03T06:32:02.645 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:2", 2022-09-03T06:32:02.646 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:32:02.646 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:7", 2022-09-03T06:32:02.646 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.19", 2022-09-03T06:32:02.647 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:32:02.647 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:7", 2022-09-03T06:32:02.647 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:10", 2022-09-03T06:32:02.647 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:32:02.648 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000072", 2022-09-03T06:32:02.648 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:9", 2022-09-03T06:32:02.648 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:0", 2022-09-03T06:32:02.648 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:4", 2022-09-03T06:32:02.649 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:10", 2022-09-03T06:32:02.649 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000062", 2022-09-03T06:32:02.649 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:9", 2022-09-03T06:32:02.649 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:10", 2022-09-03T06:32:02.650 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000115", 2022-09-03T06:32:02.650 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:3", 2022-09-03T06:32:02.650 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.1", 2022-09-03T06:32:02.651 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.1.retry", 2022-09-03T06:32:02.651 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:5", 2022-09-03T06:32:02.652 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000033", 2022-09-03T06:32:02.652 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:5", 2022-09-03T06:32:02.652 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:7", 2022-09-03T06:32:02.652 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:4", 2022-09-03T06:32:02.653 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:7", 2022-09-03T06:32:02.653 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:0", 2022-09-03T06:32:02.653 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:4", 2022-09-03T06:32:02.653 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:10", 2022-09-03T06:32:02.654 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:6", 2022-09-03T06:32:02.654 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:0", 2022-09-03T06:32:02.654 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:32:02.655 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000025", 2022-09-03T06:32:02.655 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:7", 2022-09-03T06:32:02.655 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:8", 2022-09-03T06:32:02.655 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:1", 2022-09-03T06:32:02.656 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000123", 2022-09-03T06:32:02.656 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:3", 2022-09-03T06:32:02.656 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:4", 2022-09-03T06:32:02.656 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.18", 2022-09-03T06:32:02.657 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000108", 2022-09-03T06:32:02.657 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:32:02.657 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000094", 2022-09-03T06:32:02.658 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:10", 2022-09-03T06:32:02.658 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:32:02.658 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:2", 2022-09-03T06:32:02.658 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000060", 2022-09-03T06:32:02.659 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:2", 2022-09-03T06:32:02.659 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:1", 2022-09-03T06:32:02.659 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000109", 2022-09-03T06:32:02.660 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.15", 2022-09-03T06:32:02.660 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:10", 2022-09-03T06:32:02.660 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:9", 2022-09-03T06:32:02.660 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:1", 2022-09-03T06:32:02.661 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:7", 2022-09-03T06:32:02.661 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:10", 2022-09-03T06:32:02.661 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000013", 2022-09-03T06:32:02.661 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.0.retry", 2022-09-03T06:32:02.662 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:32:02.662 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:0", 2022-09-03T06:32:02.662 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:32:02.663 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:2", 2022-09-03T06:32:02.663 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:0", 2022-09-03T06:32:02.663 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:10", 2022-09-03T06:32:02.663 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:6", 2022-09-03T06:32:02.664 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:7", 2022-09-03T06:32:02.664 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:9", 2022-09-03T06:32:02.664 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:1", 2022-09-03T06:32:02.664 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:6", 2022-09-03T06:32:02.665 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:2", 2022-09-03T06:32:02.665 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:1", 2022-09-03T06:32:02.665 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:6", 2022-09-03T06:32:02.665 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000051", 2022-09-03T06:32:02.666 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:1", 2022-09-03T06:32:02.666 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000106", 2022-09-03T06:32:02.666 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:10", 2022-09-03T06:32:02.666 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:3", 2022-09-03T06:32:02.667 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000017", 2022-09-03T06:32:02.667 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:4", 2022-09-03T06:32:02.667 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:1", 2022-09-03T06:32:02.667 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:6", 2022-09-03T06:32:02.668 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:9", 2022-09-03T06:32:02.668 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000023", 2022-09-03T06:32:02.668 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:0", 2022-09-03T06:32:02.668 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3", 2022-09-03T06:32:02.669 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:5", 2022-09-03T06:32:02.669 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:6", 2022-09-03T06:32:02.669 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3", 2022-09-03T06:32:02.669 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:4", 2022-09-03T06:32:02.670 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:4", 2022-09-03T06:32:02.670 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:2", 2022-09-03T06:32:02.670 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000107", 2022-09-03T06:32:02.670 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000046", 2022-09-03T06:32:02.671 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:4", 2022-09-03T06:32:02.671 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:9", 2022-09-03T06:32:02.671 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.26", 2022-09-03T06:32:02.671 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:10", 2022-09-03T06:32:02.672 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:1", 2022-09-03T06:32:02.672 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:6", 2022-09-03T06:32:02.672 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:7", 2022-09-03T06:32:02.672 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:2", 2022-09-03T06:32:02.673 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.22", 2022-09-03T06:32:02.673 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:32:02.673 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:6", 2022-09-03T06:32:02.674 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:6", 2022-09-03T06:32:02.674 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:32:02.674 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.1", 2022-09-03T06:32:02.675 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.2", 2022-09-03T06:32:02.675 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000067", 2022-09-03T06:32:02.676 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:0", 2022-09-03T06:32:02.676 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:5", 2022-09-03T06:32:02.676 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:1", 2022-09-03T06:32:02.677 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:7", 2022-09-03T06:32:02.677 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:3", 2022-09-03T06:32:02.677 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:5", 2022-09-03T06:32:02.678 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:6", 2022-09-03T06:32:02.678 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:3", 2022-09-03T06:32:02.678 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.3.retry", 2022-09-03T06:32:02.679 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:5", 2022-09-03T06:32:02.679 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:1", 2022-09-03T06:32:02.679 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:7", 2022-09-03T06:32:02.680 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:3", 2022-09-03T06:32:02.680 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:8", 2022-09-03T06:32:02.680 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.9", 2022-09-03T06:32:02.680 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000086", 2022-09-03T06:32:02.681 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:8", 2022-09-03T06:32:02.681 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.5", 2022-09-03T06:32:02.681 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:6", 2022-09-03T06:32:02.681 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:4", 2022-09-03T06:32:02.682 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:7", 2022-09-03T06:32:02.682 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:10", 2022-09-03T06:32:02.683 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:6", 2022-09-03T06:32:02.683 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:1", 2022-09-03T06:32:02.683 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:10", 2022-09-03T06:32:02.684 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:8", 2022-09-03T06:32:02.684 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:10", 2022-09-03T06:32:02.684 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:32:02.685 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:8", 2022-09-03T06:32:02.685 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:10", 2022-09-03T06:32:02.685 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:2", 2022-09-03T06:32:02.686 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:2", 2022-09-03T06:32:02.686 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.20", 2022-09-03T06:32:02.687 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:3", 2022-09-03T06:32:02.687 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:3", 2022-09-03T06:32:02.687 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:1", 2022-09-03T06:32:02.688 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:4", 2022-09-03T06:32:02.688 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:8", 2022-09-03T06:32:02.688 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:3", 2022-09-03T06:32:02.688 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:5", 2022-09-03T06:32:02.689 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:2", 2022-09-03T06:32:02.689 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:2", 2022-09-03T06:32:02.689 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:1", 2022-09-03T06:32:02.690 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:0", 2022-09-03T06:32:02.690 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:9", 2022-09-03T06:32:02.690 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:9", 2022-09-03T06:32:02.691 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:0", 2022-09-03T06:32:02.691 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:4", 2022-09-03T06:32:02.691 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:2", 2022-09-03T06:32:02.691 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:5", 2022-09-03T06:32:02.692 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:9", 2022-09-03T06:32:02.692 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000050", 2022-09-03T06:32:02.692 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:6", 2022-09-03T06:32:02.692 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:6", 2022-09-03T06:32:02.693 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000102", 2022-09-03T06:32:02.693 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:1", 2022-09-03T06:32:02.693 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:0", 2022-09-03T06:32:02.693 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:9", 2022-09-03T06:32:02.694 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:6", 2022-09-03T06:32:02.694 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:2", 2022-09-03T06:32:02.694 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:0", 2022-09-03T06:32:02.694 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:9", 2022-09-03T06:32:02.695 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:5", 2022-09-03T06:32:02.695 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:3", 2022-09-03T06:32:02.695 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000124", 2022-09-03T06:32:02.695 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:0", 2022-09-03T06:32:02.696 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:32:02.696 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:9", 2022-09-03T06:32:02.696 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:5", 2022-09-03T06:32:02.696 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000001", 2022-09-03T06:32:02.697 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:5", 2022-09-03T06:32:02.697 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:0", 2022-09-03T06:32:02.697 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:3", 2022-09-03T06:32:02.697 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:9", 2022-09-03T06:32:02.698 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000126", 2022-09-03T06:32:02.698 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:9", 2022-09-03T06:32:02.698 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:9", 2022-09-03T06:32:02.698 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:1", 2022-09-03T06:32:02.699 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:6", 2022-09-03T06:32:02.699 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:1", 2022-09-03T06:32:02.699 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000055", 2022-09-03T06:32:02.700 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000061", 2022-09-03T06:32:02.700 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:7", 2022-09-03T06:32:02.700 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:32:02.700 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:9", 2022-09-03T06:32:02.701 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:7", 2022-09-03T06:32:02.701 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:4", 2022-09-03T06:32:02.701 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000097", 2022-09-03T06:32:02.701 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:32:02.702 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:10", 2022-09-03T06:32:02.702 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5", 2022-09-03T06:32:02.702 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:1", 2022-09-03T06:32:02.702 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:2", 2022-09-03T06:32:02.703 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000042", 2022-09-03T06:32:02.703 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4", 2022-09-03T06:32:02.703 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:6", 2022-09-03T06:32:02.703 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:0", 2022-09-03T06:32:02.704 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:6", 2022-09-03T06:32:02.704 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000075", 2022-09-03T06:32:02.704 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:0", 2022-09-03T06:32:02.705 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000120", 2022-09-03T06:32:02.705 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000030", 2022-09-03T06:32:02.705 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:4", 2022-09-03T06:32:02.705 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:4", 2022-09-03T06:32:02.706 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:4", 2022-09-03T06:32:02.706 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:4", 2022-09-03T06:32:02.706 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:3", 2022-09-03T06:32:02.706 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:0", 2022-09-03T06:32:02.707 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:7" 2022-09-03T06:32:02.707 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:32:02.707 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T06:32:02.709 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['log', 'list', '--rgw-zone', 'test-zone2', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:02.710 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'log', 'list', '--rgw-zone', 'test-zone2', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:02.711 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 log list --rgw-zone test-zone2 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:02.764 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:02.765 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:02.843 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:32:02.843 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:10", 2022-09-03T06:32:02.844 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000078", 2022-09-03T06:32:02.844 INFO:teuthology.orchestra.run.smithi087.stdout: "meta.history", 2022-09-03T06:32:02.845 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:10", 2022-09-03T06:32:02.845 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:8", 2022-09-03T06:32:02.846 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:4", 2022-09-03T06:32:02.846 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:5", 2022-09-03T06:32:02.847 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4", 2022-09-03T06:32:02.847 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:5", 2022-09-03T06:32:02.848 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:8", 2022-09-03T06:32:02.849 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000070", 2022-09-03T06:32:02.849 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000104", 2022-09-03T06:32:02.850 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:4", 2022-09-03T06:32:02.850 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000026", 2022-09-03T06:32:02.851 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:32:02.852 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:7", 2022-09-03T06:32:02.852 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:9", 2022-09-03T06:32:02.853 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:7", 2022-09-03T06:32:02.853 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:0", 2022-09-03T06:32:02.854 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:4", 2022-09-03T06:32:02.854 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.8", 2022-09-03T06:32:02.855 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:32:02.855 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000028", 2022-09-03T06:32:02.856 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:2", 2022-09-03T06:32:02.856 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:1", 2022-09-03T06:32:02.857 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:1", 2022-09-03T06:32:02.857 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:32:02.857 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:10", 2022-09-03T06:32:02.858 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:7", 2022-09-03T06:32:02.858 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:32:02.859 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:10", 2022-09-03T06:32:02.859 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:7", 2022-09-03T06:32:02.860 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:8", 2022-09-03T06:32:02.860 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:2", 2022-09-03T06:32:02.861 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000040", 2022-09-03T06:32:02.861 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:0", 2022-09-03T06:32:02.861 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.6", 2022-09-03T06:32:02.861 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:5", 2022-09-03T06:32:02.862 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:10", 2022-09-03T06:32:02.862 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:10", 2022-09-03T06:32:02.862 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:32:02.862 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:8", 2022-09-03T06:32:02.863 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000015", 2022-09-03T06:32:02.863 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:6", 2022-09-03T06:32:02.863 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:32:02.863 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000069", 2022-09-03T06:32:02.864 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:7", 2022-09-03T06:32:02.864 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000095", 2022-09-03T06:32:02.864 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:0", 2022-09-03T06:32:02.864 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:0", 2022-09-03T06:32:02.865 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:5", 2022-09-03T06:32:02.865 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000003", 2022-09-03T06:32:02.865 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000047", 2022-09-03T06:32:02.865 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:0", 2022-09-03T06:32:02.866 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:1", 2022-09-03T06:32:02.866 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:8", 2022-09-03T06:32:02.866 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:5", 2022-09-03T06:32:02.867 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:3", 2022-09-03T06:32:02.867 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:32:02.868 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:3", 2022-09-03T06:32:02.868 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:9", 2022-09-03T06:32:02.869 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000035", 2022-09-03T06:32:02.869 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:3", 2022-09-03T06:32:02.870 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:7", 2022-09-03T06:32:02.871 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000037", 2022-09-03T06:32:02.871 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:1", 2022-09-03T06:32:02.871 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:10", 2022-09-03T06:32:02.872 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:3", 2022-09-03T06:32:02.872 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:9", 2022-09-03T06:32:02.872 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:8", 2022-09-03T06:32:02.872 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:32:02.873 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:8", 2022-09-03T06:32:02.873 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:1", 2022-09-03T06:32:02.873 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:7", 2022-09-03T06:32:02.873 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:4", 2022-09-03T06:32:02.874 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:3", 2022-09-03T06:32:02.874 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000024", 2022-09-03T06:32:02.874 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:7", 2022-09-03T06:32:02.875 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:7", 2022-09-03T06:32:02.875 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:6", 2022-09-03T06:32:02.875 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:1", 2022-09-03T06:32:02.876 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:5", 2022-09-03T06:32:02.876 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000000", 2022-09-03T06:32:02.876 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:32:02.876 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:3", 2022-09-03T06:32:02.877 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000031", 2022-09-03T06:32:02.877 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:3", 2022-09-03T06:32:02.877 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:5", 2022-09-03T06:32:02.878 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:32:02.878 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:8", 2022-09-03T06:32:02.878 INFO:teuthology.orchestra.run.smithi087.stdout: "data_log.3", 2022-09-03T06:32:02.878 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:0", 2022-09-03T06:32:02.879 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:3", 2022-09-03T06:32:02.879 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:10", 2022-09-03T06:32:02.879 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:32:02.880 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:10", 2022-09-03T06:32:02.880 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:5", 2022-09-03T06:32:02.880 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:32:02.880 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:3", 2022-09-03T06:32:02.881 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:8", 2022-09-03T06:32:02.881 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:5", 2022-09-03T06:32:02.881 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:9", 2022-09-03T06:32:02.881 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:8", 2022-09-03T06:32:02.882 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:6", 2022-09-03T06:32:02.882 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:9", 2022-09-03T06:32:02.882 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:5", 2022-09-03T06:32:02.882 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:9", 2022-09-03T06:32:02.883 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:7", 2022-09-03T06:32:02.883 INFO:teuthology.orchestra.run.smithi087.stdout: "data_loggenerations_metadata", 2022-09-03T06:32:02.883 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:3", 2022-09-03T06:32:02.884 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:2", 2022-09-03T06:32:02.884 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:10", 2022-09-03T06:32:02.884 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:32:02.885 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:7", 2022-09-03T06:32:02.885 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000076", 2022-09-03T06:32:02.885 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:1", 2022-09-03T06:32:02.886 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000113", 2022-09-03T06:32:02.886 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:32:02.886 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000005", 2022-09-03T06:32:02.886 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:10", 2022-09-03T06:32:02.887 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:4", 2022-09-03T06:32:02.887 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000011", 2022-09-03T06:32:02.887 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000058", 2022-09-03T06:32:02.887 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:7", 2022-09-03T06:32:02.888 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000052", 2022-09-03T06:32:02.888 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:6", 2022-09-03T06:32:02.888 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:4", 2022-09-03T06:32:02.889 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:1", 2022-09-03T06:32:02.889 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:32:02.889 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:1", 2022-09-03T06:32:02.889 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:0", 2022-09-03T06:32:02.890 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:1", 2022-09-03T06:32:02.890 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:0", 2022-09-03T06:32:02.890 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:0", 2022-09-03T06:32:02.891 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:7", 2022-09-03T06:32:02.891 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:0", 2022-09-03T06:32:02.891 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:3", 2022-09-03T06:32:02.892 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:8", 2022-09-03T06:32:02.892 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:2", 2022-09-03T06:32:02.892 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:5", 2022-09-03T06:32:02.892 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:6", 2022-09-03T06:32:02.893 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:32:02.893 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:9", 2022-09-03T06:32:02.893 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:7", 2022-09-03T06:32:02.894 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:7", 2022-09-03T06:32:02.894 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000088", 2022-09-03T06:32:02.894 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:9", 2022-09-03T06:32:02.894 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000080", 2022-09-03T06:32:02.895 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:4", 2022-09-03T06:32:02.895 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000090", 2022-09-03T06:32:02.895 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:10", 2022-09-03T06:32:02.896 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:1", 2022-09-03T06:32:02.896 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:5", 2022-09-03T06:32:02.896 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000110", 2022-09-03T06:32:02.897 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:2", 2022-09-03T06:32:02.897 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:7", 2022-09-03T06:32:02.897 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:3", 2022-09-03T06:32:02.898 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:8", 2022-09-03T06:32:02.898 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000096", 2022-09-03T06:32:02.898 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:4", 2022-09-03T06:32:02.898 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:0", 2022-09-03T06:32:02.899 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:10", 2022-09-03T06:32:02.899 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:9", 2022-09-03T06:32:02.899 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000087", 2022-09-03T06:32:02.899 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000008", 2022-09-03T06:32:02.900 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:4", 2022-09-03T06:32:02.900 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:1", 2022-09-03T06:32:02.900 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:7", 2022-09-03T06:32:02.901 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:6", 2022-09-03T06:32:02.901 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:6", 2022-09-03T06:32:02.901 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:4", 2022-09-03T06:32:02.902 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:5", 2022-09-03T06:32:02.902 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:3", 2022-09-03T06:32:02.902 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000006", 2022-09-03T06:32:02.903 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000029", 2022-09-03T06:32:02.903 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:5", 2022-09-03T06:32:02.903 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000089", 2022-09-03T06:32:02.903 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:7", 2022-09-03T06:32:02.904 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:4", 2022-09-03T06:32:02.904 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000022", 2022-09-03T06:32:02.904 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:1", 2022-09-03T06:32:02.904 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:9", 2022-09-03T06:32:02.905 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:1", 2022-09-03T06:32:02.905 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:32:02.905 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:10", 2022-09-03T06:32:02.906 INFO:teuthology.orchestra.run.smithi087.stdout: "data_log.2.9cQcNCNcrxJ.0", 2022-09-03T06:32:02.906 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:7", 2022-09-03T06:32:02.906 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:2", 2022-09-03T06:32:02.906 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:6", 2022-09-03T06:32:02.907 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:10", 2022-09-03T06:32:02.907 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:7", 2022-09-03T06:32:02.907 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:2", 2022-09-03T06:32:02.908 INFO:teuthology.orchestra.run.smithi087.stdout: "bilog.trim", 2022-09-03T06:32:02.908 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000016", 2022-09-03T06:32:02.908 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:32:02.909 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:0", 2022-09-03T06:32:02.909 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:8", 2022-09-03T06:32:02.909 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:2", 2022-09-03T06:32:02.909 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000041", 2022-09-03T06:32:02.910 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:9", 2022-09-03T06:32:02.910 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:32:02.910 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:8", 2022-09-03T06:32:02.911 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:6", 2022-09-03T06:32:02.911 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:1", 2022-09-03T06:32:02.911 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:3", 2022-09-03T06:32:02.912 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:5", 2022-09-03T06:32:02.912 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.0.retry", 2022-09-03T06:32:02.912 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000018", 2022-09-03T06:32:02.912 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:32:02.913 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000092", 2022-09-03T06:32:02.913 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:32:02.913 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:3", 2022-09-03T06:32:02.914 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:7", 2022-09-03T06:32:02.914 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:1", 2022-09-03T06:32:02.914 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:10", 2022-09-03T06:32:02.915 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:8", 2022-09-03T06:32:02.915 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:6", 2022-09-03T06:32:02.915 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.4", 2022-09-03T06:32:02.915 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.1.retry", 2022-09-03T06:32:02.916 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:7", 2022-09-03T06:32:02.916 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000014", 2022-09-03T06:32:02.916 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:9", 2022-09-03T06:32:02.917 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:4", 2022-09-03T06:32:02.917 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:1", 2022-09-03T06:32:02.917 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000112", 2022-09-03T06:32:02.918 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:2", 2022-09-03T06:32:02.918 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000007", 2022-09-03T06:32:02.918 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000021", 2022-09-03T06:32:02.919 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:10", 2022-09-03T06:32:02.919 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:4", 2022-09-03T06:32:02.919 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000064", 2022-09-03T06:32:02.920 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:8", 2022-09-03T06:32:02.920 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:10", 2022-09-03T06:32:02.920 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:4", 2022-09-03T06:32:02.921 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:6", 2022-09-03T06:32:02.921 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000071", 2022-09-03T06:32:02.921 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:1", 2022-09-03T06:32:02.922 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:5", 2022-09-03T06:32:02.922 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:6", 2022-09-03T06:32:02.922 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:1", 2022-09-03T06:32:02.923 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000074", 2022-09-03T06:32:02.924 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:0", 2022-09-03T06:32:02.924 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:0", 2022-09-03T06:32:02.925 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:1", 2022-09-03T06:32:02.925 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:2", 2022-09-03T06:32:02.926 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:7", 2022-09-03T06:32:02.926 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:4", 2022-09-03T06:32:02.927 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:2", 2022-09-03T06:32:02.927 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:8", 2022-09-03T06:32:02.928 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:6", 2022-09-03T06:32:02.928 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:5", 2022-09-03T06:32:02.929 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:3", 2022-09-03T06:32:02.929 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000081", 2022-09-03T06:32:02.929 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000009", 2022-09-03T06:32:02.930 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000121", 2022-09-03T06:32:02.930 INFO:teuthology.orchestra.run.smithi087.stdout: "data_log.1", 2022-09-03T06:32:02.931 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:10", 2022-09-03T06:32:02.931 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:10", 2022-09-03T06:32:02.932 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:10", 2022-09-03T06:32:02.932 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:1", 2022-09-03T06:32:02.932 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:5", 2022-09-03T06:32:02.933 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000125", 2022-09-03T06:32:02.933 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000082", 2022-09-03T06:32:02.934 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:5", 2022-09-03T06:32:02.934 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:9", 2022-09-03T06:32:02.935 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:7", 2022-09-03T06:32:02.935 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:8", 2022-09-03T06:32:02.936 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:2", 2022-09-03T06:32:02.936 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:4", 2022-09-03T06:32:02.937 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:0", 2022-09-03T06:32:02.937 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:9", 2022-09-03T06:32:02.938 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000105", 2022-09-03T06:32:02.938 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000059", 2022-09-03T06:32:02.939 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:4", 2022-09-03T06:32:02.940 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:9", 2022-09-03T06:32:02.940 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:9", 2022-09-03T06:32:02.941 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000077", 2022-09-03T06:32:02.941 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000032", 2022-09-03T06:32:02.942 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:7", 2022-09-03T06:32:02.942 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:8", 2022-09-03T06:32:02.943 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:1", 2022-09-03T06:32:02.944 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:5", 2022-09-03T06:32:02.944 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:2", 2022-09-03T06:32:02.945 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e", 2022-09-03T06:32:02.945 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:9", 2022-09-03T06:32:02.946 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:9", 2022-09-03T06:32:02.946 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:4", 2022-09-03T06:32:02.947 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:6", 2022-09-03T06:32:02.947 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:8", 2022-09-03T06:32:02.948 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:3", 2022-09-03T06:32:02.948 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:9", 2022-09-03T06:32:02.949 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:7", 2022-09-03T06:32:02.949 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.3", 2022-09-03T06:32:02.950 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:10", 2022-09-03T06:32:02.950 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:8", 2022-09-03T06:32:02.951 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:1", 2022-09-03T06:32:02.951 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000053", 2022-09-03T06:32:02.952 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:0", 2022-09-03T06:32:02.952 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:32:02.953 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.0.retry", 2022-09-03T06:32:02.953 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000091", 2022-09-03T06:32:02.954 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:10", 2022-09-03T06:32:02.954 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:5", 2022-09-03T06:32:02.955 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:9", 2022-09-03T06:32:02.955 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:2", 2022-09-03T06:32:02.956 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000065", 2022-09-03T06:32:02.956 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:8", 2022-09-03T06:32:02.957 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:3", 2022-09-03T06:32:02.957 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000083", 2022-09-03T06:32:02.958 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000010", 2022-09-03T06:32:02.959 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:2", 2022-09-03T06:32:02.959 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:8", 2022-09-03T06:32:02.960 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:9", 2022-09-03T06:32:02.960 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:4", 2022-09-03T06:32:02.961 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:9", 2022-09-03T06:32:02.961 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.7", 2022-09-03T06:32:02.962 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.3", 2022-09-03T06:32:02.962 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:1", 2022-09-03T06:32:02.963 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:6", 2022-09-03T06:32:02.963 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:7", 2022-09-03T06:32:02.964 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:8", 2022-09-03T06:32:02.964 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:8", 2022-09-03T06:32:02.965 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:8", 2022-09-03T06:32:02.965 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:7", 2022-09-03T06:32:02.966 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:5", 2022-09-03T06:32:02.966 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:5", 2022-09-03T06:32:02.967 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:10", 2022-09-03T06:32:02.967 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:10", 2022-09-03T06:32:02.968 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:5", 2022-09-03T06:32:02.968 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:0", 2022-09-03T06:32:02.969 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status.shard.2", 2022-09-03T06:32:02.969 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4", 2022-09-03T06:32:02.970 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:32:02.971 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:9", 2022-09-03T06:32:02.971 INFO:teuthology.orchestra.run.smithi087.stdout: "meta.log.2962e93e-ca58-4758-b6d9-6fe2e67c2976.1", 2022-09-03T06:32:02.972 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:1", 2022-09-03T06:32:02.972 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:9", 2022-09-03T06:32:02.973 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:4", 2022-09-03T06:32:02.973 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:0", 2022-09-03T06:32:02.974 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3", 2022-09-03T06:32:02.975 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:10", 2022-09-03T06:32:02.975 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000045", 2022-09-03T06:32:02.976 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000002", 2022-09-03T06:32:02.976 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:0", 2022-09-03T06:32:02.976 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:4", 2022-09-03T06:32:02.977 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.12", 2022-09-03T06:32:02.978 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:1", 2022-09-03T06:32:02.978 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:5", 2022-09-03T06:32:02.979 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.0", 2022-09-03T06:32:02.979 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000116", 2022-09-03T06:32:02.980 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:7", 2022-09-03T06:32:02.980 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:5", 2022-09-03T06:32:02.981 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:2", 2022-09-03T06:32:02.981 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:5", 2022-09-03T06:32:02.982 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:2", 2022-09-03T06:32:02.982 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:6", 2022-09-03T06:32:02.983 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:7", 2022-09-03T06:32:02.983 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:10", 2022-09-03T06:32:02.984 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000034", 2022-09-03T06:32:02.984 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:6", 2022-09-03T06:32:02.985 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:1", 2022-09-03T06:32:02.986 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000101", 2022-09-03T06:32:02.986 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:4", 2022-09-03T06:32:02.987 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:1", 2022-09-03T06:32:02.987 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:1", 2022-09-03T06:32:02.988 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.13", 2022-09-03T06:32:02.988 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:0", 2022-09-03T06:32:02.989 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status.shard.1", 2022-09-03T06:32:02.989 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:10", 2022-09-03T06:32:02.990 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5", 2022-09-03T06:32:02.990 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:4", 2022-09-03T06:32:02.991 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:2", 2022-09-03T06:32:02.991 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:8", 2022-09-03T06:32:02.992 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:2", 2022-09-03T06:32:02.992 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:3", 2022-09-03T06:32:02.993 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000079", 2022-09-03T06:32:02.993 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000049", 2022-09-03T06:32:02.994 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000117", 2022-09-03T06:32:02.994 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000044", 2022-09-03T06:32:02.995 INFO:teuthology.orchestra.run.smithi087.stdout: "data_log.1.zgD7oa3ns6n.0", 2022-09-03T06:32:02.995 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:9", 2022-09-03T06:32:02.996 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:3", 2022-09-03T06:32:02.996 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:3", 2022-09-03T06:32:02.997 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.14", 2022-09-03T06:32:02.997 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000066", 2022-09-03T06:32:02.998 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:10", 2022-09-03T06:32:02.998 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:8", 2022-09-03T06:32:02.999 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:7", 2022-09-03T06:32:02.999 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:1", 2022-09-03T06:32:03.000 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:8", 2022-09-03T06:32:03.000 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:7", 2022-09-03T06:32:03.001 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:7", 2022-09-03T06:32:03.001 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:6", 2022-09-03T06:32:03.002 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:8", 2022-09-03T06:32:03.002 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:1", 2022-09-03T06:32:03.003 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:10", 2022-09-03T06:32:03.003 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000068", 2022-09-03T06:32:03.004 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000085", 2022-09-03T06:32:03.004 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:6", 2022-09-03T06:32:03.005 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:8", 2022-09-03T06:32:03.005 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:5", 2022-09-03T06:32:03.006 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:8", 2022-09-03T06:32:03.006 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:5", 2022-09-03T06:32:03.007 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:8", 2022-09-03T06:32:03.007 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:10", 2022-09-03T06:32:03.008 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.3.retry", 2022-09-03T06:32:03.008 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:3", 2022-09-03T06:32:03.009 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000073", 2022-09-03T06:32:03.009 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:2", 2022-09-03T06:32:03.010 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000038", 2022-09-03T06:32:03.010 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:4", 2022-09-03T06:32:03.011 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:9", 2022-09-03T06:32:03.012 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:0", 2022-09-03T06:32:03.012 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.10", 2022-09-03T06:32:03.013 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000118", 2022-09-03T06:32:03.013 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:5", 2022-09-03T06:32:03.014 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:4", 2022-09-03T06:32:03.014 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:1", 2022-09-03T06:32:03.015 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:3", 2022-09-03T06:32:03.015 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000036", 2022-09-03T06:32:03.015 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:4", 2022-09-03T06:32:03.016 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000103", 2022-09-03T06:32:03.016 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:0", 2022-09-03T06:32:03.016 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:5", 2022-09-03T06:32:03.017 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status.shard.3", 2022-09-03T06:32:03.017 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:6", 2022-09-03T06:32:03.017 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:3", 2022-09-03T06:32:03.018 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000119", 2022-09-03T06:32:03.018 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:8", 2022-09-03T06:32:03.019 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:6", 2022-09-03T06:32:03.019 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:3", 2022-09-03T06:32:03.019 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000098", 2022-09-03T06:32:03.020 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:32:03.020 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:32:03.020 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:32:03.021 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:10", 2022-09-03T06:32:03.021 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:2", 2022-09-03T06:32:03.021 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:6", 2022-09-03T06:32:03.021 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:8", 2022-09-03T06:32:03.022 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:7", 2022-09-03T06:32:03.022 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:10", 2022-09-03T06:32:03.023 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:5", 2022-09-03T06:32:03.023 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status.shard.0", 2022-09-03T06:32:03.024 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:2", 2022-09-03T06:32:03.024 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000027", 2022-09-03T06:32:03.025 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status", 2022-09-03T06:32:03.025 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000019", 2022-09-03T06:32:03.026 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:9", 2022-09-03T06:32:03.026 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:9", 2022-09-03T06:32:03.027 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:0", 2022-09-03T06:32:03.027 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1", 2022-09-03T06:32:03.028 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000039", 2022-09-03T06:32:03.028 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:0", 2022-09-03T06:32:03.029 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:32:03.029 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:5", 2022-09-03T06:32:03.030 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:4", 2022-09-03T06:32:03.031 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:4", 2022-09-03T06:32:03.031 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:2", 2022-09-03T06:32:03.032 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:7", 2022-09-03T06:32:03.032 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:0", 2022-09-03T06:32:03.033 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:0", 2022-09-03T06:32:03.033 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000100", 2022-09-03T06:32:03.034 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:32:03.035 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:0", 2022-09-03T06:32:03.035 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:1", 2022-09-03T06:32:03.036 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:8", 2022-09-03T06:32:03.036 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:3", 2022-09-03T06:32:03.037 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000093", 2022-09-03T06:32:03.037 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:3", 2022-09-03T06:32:03.038 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:9", 2022-09-03T06:32:03.038 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:2", 2022-09-03T06:32:03.039 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000004", 2022-09-03T06:32:03.039 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000063", 2022-09-03T06:32:03.040 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:2", 2022-09-03T06:32:03.040 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000122", 2022-09-03T06:32:03.041 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000057", 2022-09-03T06:32:03.041 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:4", 2022-09-03T06:32:03.042 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:32:03.042 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:9", 2022-09-03T06:32:03.043 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000054", 2022-09-03T06:32:03.043 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000114", 2022-09-03T06:32:03.044 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:6", 2022-09-03T06:32:03.044 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:3", 2022-09-03T06:32:03.045 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.2", 2022-09-03T06:32:03.045 INFO:teuthology.orchestra.run.smithi087.stdout: "data_log.0", 2022-09-03T06:32:03.046 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:3", 2022-09-03T06:32:03.046 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:7", 2022-09-03T06:32:03.047 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:0", 2022-09-03T06:32:03.047 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:0", 2022-09-03T06:32:03.048 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:2", 2022-09-03T06:32:03.048 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000012", 2022-09-03T06:32:03.049 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:4", 2022-09-03T06:32:03.049 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:4", 2022-09-03T06:32:03.050 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.2", 2022-09-03T06:32:03.050 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:8", 2022-09-03T06:32:03.051 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:10", 2022-09-03T06:32:03.051 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:5", 2022-09-03T06:32:03.052 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000084", 2022-09-03T06:32:03.052 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000043", 2022-09-03T06:32:03.053 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:7", 2022-09-03T06:32:03.053 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:5", 2022-09-03T06:32:03.054 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.11", 2022-09-03T06:32:03.054 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:5", 2022-09-03T06:32:03.055 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:32:03.055 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:8", 2022-09-03T06:32:03.056 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:2", 2022-09-03T06:32:03.056 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:9", 2022-09-03T06:32:03.057 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.1.retry", 2022-09-03T06:32:03.057 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:10", 2022-09-03T06:32:03.058 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:32:03.058 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:6", 2022-09-03T06:32:03.059 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000111", 2022-09-03T06:32:03.059 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000048", 2022-09-03T06:32:03.060 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:8", 2022-09-03T06:32:03.060 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:2", 2022-09-03T06:32:03.061 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.2", 2022-09-03T06:32:03.061 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:2", 2022-09-03T06:32:03.062 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000020", 2022-09-03T06:32:03.062 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:9", 2022-09-03T06:32:03.063 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:1", 2022-09-03T06:32:03.063 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:2", 2022-09-03T06:32:03.063 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:7", 2022-09-03T06:32:03.064 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:4", 2022-09-03T06:32:03.064 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:7", 2022-09-03T06:32:03.065 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:1", 2022-09-03T06:32:03.065 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:9", 2022-09-03T06:32:03.066 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000099", 2022-09-03T06:32:03.066 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000056", 2022-09-03T06:32:03.067 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:0", 2022-09-03T06:32:03.067 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:3", 2022-09-03T06:32:03.068 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:5", 2022-09-03T06:32:03.068 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:7", 2022-09-03T06:32:03.069 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:7", 2022-09-03T06:32:03.069 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:7", 2022-09-03T06:32:03.070 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:0", 2022-09-03T06:32:03.070 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:6", 2022-09-03T06:32:03.071 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:6", 2022-09-03T06:32:03.071 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:0", 2022-09-03T06:32:03.071 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:0", 2022-09-03T06:32:03.072 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.3", 2022-09-03T06:32:03.072 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000072", 2022-09-03T06:32:03.073 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:0", 2022-09-03T06:32:03.073 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:9", 2022-09-03T06:32:03.074 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:4", 2022-09-03T06:32:03.074 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:9", 2022-09-03T06:32:03.075 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000062", 2022-09-03T06:32:03.075 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:0", 2022-09-03T06:32:03.076 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000115", 2022-09-03T06:32:03.076 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:3", 2022-09-03T06:32:03.077 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.1", 2022-09-03T06:32:03.077 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:5", 2022-09-03T06:32:03.078 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000033", 2022-09-03T06:32:03.078 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:5", 2022-09-03T06:32:03.079 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:4", 2022-09-03T06:32:03.079 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:5", 2022-09-03T06:32:03.080 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:0", 2022-09-03T06:32:03.080 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:4", 2022-09-03T06:32:03.081 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:6", 2022-09-03T06:32:03.081 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:3", 2022-09-03T06:32:03.082 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:7", 2022-09-03T06:32:03.082 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:0", 2022-09-03T06:32:03.082 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:1", 2022-09-03T06:32:03.083 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:1", 2022-09-03T06:32:03.083 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:3", 2022-09-03T06:32:03.084 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:0", 2022-09-03T06:32:03.084 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000025", 2022-09-03T06:32:03.085 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:1", 2022-09-03T06:32:03.085 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000123", 2022-09-03T06:32:03.086 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:3", 2022-09-03T06:32:03.086 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:4", 2022-09-03T06:32:03.087 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:10", 2022-09-03T06:32:03.087 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:4", 2022-09-03T06:32:03.088 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:5", 2022-09-03T06:32:03.088 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:10", 2022-09-03T06:32:03.089 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:4", 2022-09-03T06:32:03.089 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000108", 2022-09-03T06:32:03.090 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:32:03.090 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000094", 2022-09-03T06:32:03.091 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:5", 2022-09-03T06:32:03.091 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:2", 2022-09-03T06:32:03.092 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000060", 2022-09-03T06:32:03.092 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:1", 2022-09-03T06:32:03.093 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000109", 2022-09-03T06:32:03.093 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:4", 2022-09-03T06:32:03.094 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:9", 2022-09-03T06:32:03.094 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.2.retry", 2022-09-03T06:32:03.094 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:10", 2022-09-03T06:32:03.095 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:2", 2022-09-03T06:32:03.095 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000013", 2022-09-03T06:32:03.096 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:5", 2022-09-03T06:32:03.096 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:6", 2022-09-03T06:32:03.096 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:4", 2022-09-03T06:32:03.097 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:2", 2022-09-03T06:32:03.097 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:9", 2022-09-03T06:32:03.098 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:3", 2022-09-03T06:32:03.098 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:6", 2022-09-03T06:32:03.098 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:9", 2022-09-03T06:32:03.099 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:6", 2022-09-03T06:32:03.099 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:8", 2022-09-03T06:32:03.099 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:6", 2022-09-03T06:32:03.100 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:8", 2022-09-03T06:32:03.100 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:6", 2022-09-03T06:32:03.100 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000051", 2022-09-03T06:32:03.100 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:2", 2022-09-03T06:32:03.100 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000106", 2022-09-03T06:32:03.101 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:10", 2022-09-03T06:32:03.101 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:10", 2022-09-03T06:32:03.101 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:9", 2022-09-03T06:32:03.101 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:0", 2022-09-03T06:32:03.102 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:3", 2022-09-03T06:32:03.102 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000017", 2022-09-03T06:32:03.102 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:3", 2022-09-03T06:32:03.102 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:4", 2022-09-03T06:32:03.103 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:1", 2022-09-03T06:32:03.103 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:8", 2022-09-03T06:32:03.103 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:8", 2022-09-03T06:32:03.104 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000023", 2022-09-03T06:32:03.104 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:0", 2022-09-03T06:32:03.104 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3", 2022-09-03T06:32:03.105 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:5", 2022-09-03T06:32:03.105 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:6", 2022-09-03T06:32:03.105 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:4", 2022-09-03T06:32:03.106 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000107", 2022-09-03T06:32:03.106 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:32:03.107 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:9", 2022-09-03T06:32:03.107 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000046", 2022-09-03T06:32:03.108 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:1", 2022-09-03T06:32:03.108 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:0", 2022-09-03T06:32:03.109 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:3", 2022-09-03T06:32:03.109 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:9", 2022-09-03T06:32:03.110 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:10", 2022-09-03T06:32:03.110 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:2", 2022-09-03T06:32:03.111 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:10", 2022-09-03T06:32:03.111 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:0", 2022-09-03T06:32:03.112 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:1", 2022-09-03T06:32:03.112 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:2", 2022-09-03T06:32:03.113 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:3", 2022-09-03T06:32:03.113 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:32:03.113 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.1", 2022-09-03T06:32:03.114 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:3", 2022-09-03T06:32:03.114 INFO:teuthology.orchestra.run.smithi087.stdout: "data_log.2", 2022-09-03T06:32:03.115 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:1", 2022-09-03T06:32:03.115 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000067", 2022-09-03T06:32:03.116 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:9", 2022-09-03T06:32:03.116 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:0", 2022-09-03T06:32:03.117 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:10", 2022-09-03T06:32:03.117 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:5", 2022-09-03T06:32:03.117 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:6", 2022-09-03T06:32:03.117 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:3", 2022-09-03T06:32:03.118 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.00aa9b6d-7a02-4715-9200-8a26b45eb25e.3.retry", 2022-09-03T06:32:03.118 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:3", 2022-09-03T06:32:03.118 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:6", 2022-09-03T06:32:03.119 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:1", 2022-09-03T06:32:03.119 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:2", 2022-09-03T06:32:03.119 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:3", 2022-09-03T06:32:03.120 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.0", 2022-09-03T06:32:03.120 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:8", 2022-09-03T06:32:03.120 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.9", 2022-09-03T06:32:03.120 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000086", 2022-09-03T06:32:03.121 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:2", 2022-09-03T06:32:03.121 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:6", 2022-09-03T06:32:03.122 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.5", 2022-09-03T06:32:03.122 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:6", 2022-09-03T06:32:03.123 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:2", 2022-09-03T06:32:03.123 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:7", 2022-09-03T06:32:03.124 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:6", 2022-09-03T06:32:03.124 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:10", 2022-09-03T06:32:03.125 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:6", 2022-09-03T06:32:03.125 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1", 2022-09-03T06:32:03.126 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:4", 2022-09-03T06:32:03.126 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:2", 2022-09-03T06:32:03.127 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:10", 2022-09-03T06:32:03.127 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:10", 2022-09-03T06:32:03.128 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:2", 2022-09-03T06:32:03.128 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:8", 2022-09-03T06:32:03.129 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:6", 2022-09-03T06:32:03.129 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:3", 2022-09-03T06:32:03.129 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:1", 2022-09-03T06:32:03.130 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:5", 2022-09-03T06:32:03.130 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:2", 2022-09-03T06:32:03.131 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:2", 2022-09-03T06:32:03.131 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:9", 2022-09-03T06:32:03.132 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:0", 2022-09-03T06:32:03.132 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:8", 2022-09-03T06:32:03.133 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:2", 2022-09-03T06:32:03.133 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:5", 2022-09-03T06:32:03.134 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.1", 2022-09-03T06:32:03.134 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:6", 2022-09-03T06:32:03.135 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:6", 2022-09-03T06:32:03.135 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000050", 2022-09-03T06:32:03.136 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:6", 2022-09-03T06:32:03.136 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:6", 2022-09-03T06:32:03.136 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:6", 2022-09-03T06:32:03.137 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000102", 2022-09-03T06:32:03.137 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:8", 2022-09-03T06:32:03.138 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:0", 2022-09-03T06:32:03.138 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:1", 2022-09-03T06:32:03.139 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:32:03.139 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:8", 2022-09-03T06:32:03.140 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:2", 2022-09-03T06:32:03.140 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:0", 2022-09-03T06:32:03.140 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:9", 2022-09-03T06:32:03.141 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000124", 2022-09-03T06:32:03.141 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:32:03.142 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:32:03.142 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:9", 2022-09-03T06:32:03.142 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:7", 2022-09-03T06:32:03.143 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000001", 2022-09-03T06:32:03.143 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:0", 2022-09-03T06:32:03.144 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:3", 2022-09-03T06:32:03.144 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:5", 2022-09-03T06:32:03.145 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:9", 2022-09-03T06:32:03.145 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000126", 2022-09-03T06:32:03.146 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:9", 2022-09-03T06:32:03.146 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:9", 2022-09-03T06:32:03.146 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:2", 2022-09-03T06:32:03.147 INFO:teuthology.orchestra.run.smithi087.stdout: "data_log.0.oIPDmPVPiPO.0", 2022-09-03T06:32:03.147 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:6", 2022-09-03T06:32:03.148 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:1", 2022-09-03T06:32:03.148 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000055", 2022-09-03T06:32:03.149 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000061", 2022-09-03T06:32:03.149 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:7", 2022-09-03T06:32:03.150 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:32:03.150 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:7", 2022-09-03T06:32:03.151 INFO:teuthology.orchestra.run.smithi087.stdout: "data_log.3.VjKijllYa5g.0", 2022-09-03T06:32:03.151 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000097", 2022-09-03T06:32:03.152 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:32:03.152 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5", 2022-09-03T06:32:03.152 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:6", 2022-09-03T06:32:03.153 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:1", 2022-09-03T06:32:03.153 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000042", 2022-09-03T06:32:03.153 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:3", 2022-09-03T06:32:03.154 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:6", 2022-09-03T06:32:03.154 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:32:03.155 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:10", 2022-09-03T06:32:03.155 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:3", 2022-09-03T06:32:03.156 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:10", 2022-09-03T06:32:03.156 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:4", 2022-09-03T06:32:03.156 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:4", 2022-09-03T06:32:03.157 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000075", 2022-09-03T06:32:03.157 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:32:03.158 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:2", 2022-09-03T06:32:03.159 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000120", 2022-09-03T06:32:03.159 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000030", 2022-09-03T06:32:03.159 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:4", 2022-09-03T06:32:03.160 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:3", 2022-09-03T06:32:03.160 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:4", 2022-09-03T06:32:03.161 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:4", 2022-09-03T06:32:03.161 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:2", 2022-09-03T06:32:03.161 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:0", 2022-09-03T06:32:03.162 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.00aa9b6d-7a02-4715-9200-8a26b45eb25e:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:7" 2022-09-03T06:32:03.162 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:32:03.162 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T06:32:03.165 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['log', 'list', '--rgw-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:03.166 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'log', 'list', '--rgw-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:03.167 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 log list --rgw-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:03.221 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:03.222 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:03.277 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:32:03.278 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:10", 2022-09-03T06:32:03.278 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000078", 2022-09-03T06:32:03.279 INFO:teuthology.orchestra.run.smithi062.stdout: "meta.history", 2022-09-03T06:32:03.279 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:10", 2022-09-03T06:32:03.279 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:8", 2022-09-03T06:32:03.279 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000070", 2022-09-03T06:32:03.280 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000104", 2022-09-03T06:32:03.280 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:4", 2022-09-03T06:32:03.280 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000026", 2022-09-03T06:32:03.280 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:8", 2022-09-03T06:32:03.281 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:6", 2022-09-03T06:32:03.281 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:9", 2022-09-03T06:32:03.281 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:32:03.281 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:0", 2022-09-03T06:32:03.282 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:4", 2022-09-03T06:32:03.282 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:3", 2022-09-03T06:32:03.282 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.8", 2022-09-03T06:32:03.283 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:32:03.283 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000028", 2022-09-03T06:32:03.283 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:32:03.283 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:8", 2022-09-03T06:32:03.284 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:4", 2022-09-03T06:32:03.284 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.2.retry", 2022-09-03T06:32:03.284 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:8", 2022-09-03T06:32:03.284 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:8", 2022-09-03T06:32:03.285 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:9", 2022-09-03T06:32:03.285 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:10", 2022-09-03T06:32:03.285 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:10", 2022-09-03T06:32:03.285 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:7", 2022-09-03T06:32:03.286 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:9", 2022-09-03T06:32:03.286 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:2", 2022-09-03T06:32:03.286 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:2", 2022-09-03T06:32:03.286 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000040", 2022-09-03T06:32:03.287 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:9", 2022-09-03T06:32:03.287 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:0", 2022-09-03T06:32:03.287 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.6", 2022-09-03T06:32:03.288 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:9", 2022-09-03T06:32:03.288 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:6", 2022-09-03T06:32:03.288 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:10", 2022-09-03T06:32:03.289 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:32:03.289 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:8", 2022-09-03T06:32:03.289 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000015", 2022-09-03T06:32:03.289 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:6", 2022-09-03T06:32:03.290 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:32:03.290 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000069", 2022-09-03T06:32:03.290 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:9", 2022-09-03T06:32:03.291 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000095", 2022-09-03T06:32:03.291 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:0", 2022-09-03T06:32:03.291 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:5", 2022-09-03T06:32:03.291 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000003", 2022-09-03T06:32:03.292 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000047", 2022-09-03T06:32:03.292 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:1", 2022-09-03T06:32:03.292 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:8", 2022-09-03T06:32:03.293 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:5", 2022-09-03T06:32:03.293 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:2", 2022-09-03T06:32:03.293 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:9", 2022-09-03T06:32:03.293 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000035", 2022-09-03T06:32:03.294 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:10", 2022-09-03T06:32:03.294 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:3", 2022-09-03T06:32:03.294 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000037", 2022-09-03T06:32:03.294 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:1", 2022-09-03T06:32:03.295 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:10", 2022-09-03T06:32:03.295 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:32:03.295 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:1", 2022-09-03T06:32:03.296 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:4", 2022-09-03T06:32:03.296 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:4", 2022-09-03T06:32:03.296 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:1", 2022-09-03T06:32:03.296 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:32:03.297 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000024", 2022-09-03T06:32:03.297 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000000", 2022-09-03T06:32:03.297 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000031", 2022-09-03T06:32:03.297 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:32:03.298 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.3", 2022-09-03T06:32:03.298 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:7", 2022-09-03T06:32:03.298 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:3", 2022-09-03T06:32:03.298 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:2", 2022-09-03T06:32:03.299 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:2", 2022-09-03T06:32:03.299 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:5", 2022-09-03T06:32:03.299 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:6", 2022-09-03T06:32:03.299 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:9", 2022-09-03T06:32:03.300 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:5", 2022-09-03T06:32:03.300 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:32:03.300 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:0", 2022-09-03T06:32:03.300 INFO:teuthology.orchestra.run.smithi062.stdout: "meta.log.b84ccf39-78f5-4998-9db6-42362bc8c98b.1", 2022-09-03T06:32:03.301 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:3", 2022-09-03T06:32:03.301 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:8", 2022-09-03T06:32:03.301 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:3", 2022-09-03T06:32:03.301 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:5", 2022-09-03T06:32:03.302 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:0", 2022-09-03T06:32:03.302 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:6", 2022-09-03T06:32:03.302 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:4", 2022-09-03T06:32:03.303 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:6", 2022-09-03T06:32:03.303 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:9", 2022-09-03T06:32:03.303 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:32:03.303 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:9", 2022-09-03T06:32:03.304 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:7", 2022-09-03T06:32:03.304 INFO:teuthology.orchestra.run.smithi062.stdout: "data_loggenerations_metadata", 2022-09-03T06:32:03.304 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:3", 2022-09-03T06:32:03.304 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:2", 2022-09-03T06:32:03.305 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:32:03.305 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:4", 2022-09-03T06:32:03.305 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:10", 2022-09-03T06:32:03.305 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:32:03.306 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:0", 2022-09-03T06:32:03.306 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:1", 2022-09-03T06:32:03.306 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:7", 2022-09-03T06:32:03.306 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000076", 2022-09-03T06:32:03.307 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:1", 2022-09-03T06:32:03.307 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:3", 2022-09-03T06:32:03.307 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000113", 2022-09-03T06:32:03.307 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000005", 2022-09-03T06:32:03.308 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:10", 2022-09-03T06:32:03.308 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:3", 2022-09-03T06:32:03.308 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000011", 2022-09-03T06:32:03.309 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000058", 2022-09-03T06:32:03.309 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000052", 2022-09-03T06:32:03.309 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:6", 2022-09-03T06:32:03.310 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:1", 2022-09-03T06:32:03.310 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:5", 2022-09-03T06:32:03.310 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:3", 2022-09-03T06:32:03.310 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:6", 2022-09-03T06:32:03.311 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:7", 2022-09-03T06:32:03.311 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:0", 2022-09-03T06:32:03.311 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:0", 2022-09-03T06:32:03.311 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:8", 2022-09-03T06:32:03.312 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:5", 2022-09-03T06:32:03.312 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:32:03.312 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:10", 2022-09-03T06:32:03.312 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000088", 2022-09-03T06:32:03.313 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000080", 2022-09-03T06:32:03.313 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000090", 2022-09-03T06:32:03.313 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:2", 2022-09-03T06:32:03.313 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:5", 2022-09-03T06:32:03.314 INFO:teuthology.orchestra.run.smithi062.stdout: "meta.log.6a921abe-6ddc-4d12-8acc-a1151b4a5f12.1", 2022-09-03T06:32:03.314 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:10", 2022-09-03T06:32:03.314 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:5", 2022-09-03T06:32:03.314 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000110", 2022-09-03T06:32:03.315 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:7", 2022-09-03T06:32:03.315 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:1", 2022-09-03T06:32:03.315 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000096", 2022-09-03T06:32:03.315 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:4", 2022-09-03T06:32:03.316 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:0", 2022-09-03T06:32:03.316 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:7", 2022-09-03T06:32:03.316 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:8", 2022-09-03T06:32:03.316 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:3", 2022-09-03T06:32:03.317 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:4", 2022-09-03T06:32:03.317 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:9", 2022-09-03T06:32:03.317 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000087", 2022-09-03T06:32:03.317 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:2", 2022-09-03T06:32:03.318 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:9", 2022-09-03T06:32:03.318 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000008", 2022-09-03T06:32:03.319 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:4", 2022-09-03T06:32:03.319 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:1", 2022-09-03T06:32:03.319 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1", 2022-09-03T06:32:03.319 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:7", 2022-09-03T06:32:03.320 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:5", 2022-09-03T06:32:03.320 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:6", 2022-09-03T06:32:03.320 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:5", 2022-09-03T06:32:03.320 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:5", 2022-09-03T06:32:03.321 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:5", 2022-09-03T06:32:03.321 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:3", 2022-09-03T06:32:03.321 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000006", 2022-09-03T06:32:03.321 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000029", 2022-09-03T06:32:03.322 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:7", 2022-09-03T06:32:03.322 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:2", 2022-09-03T06:32:03.322 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000089", 2022-09-03T06:32:03.323 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:7", 2022-09-03T06:32:03.323 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:1", 2022-09-03T06:32:03.323 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000022", 2022-09-03T06:32:03.323 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:1", 2022-09-03T06:32:03.324 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:9", 2022-09-03T06:32:03.324 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:10", 2022-09-03T06:32:03.324 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:7", 2022-09-03T06:32:03.324 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:5", 2022-09-03T06:32:03.325 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog.trim", 2022-09-03T06:32:03.325 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000016", 2022-09-03T06:32:03.325 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:2", 2022-09-03T06:32:03.325 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:32:03.326 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:0", 2022-09-03T06:32:03.326 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000041", 2022-09-03T06:32:03.326 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:4", 2022-09-03T06:32:03.326 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:32:03.327 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:0", 2022-09-03T06:32:03.327 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:8", 2022-09-03T06:32:03.327 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:6", 2022-09-03T06:32:03.328 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:1", 2022-09-03T06:32:03.328 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:3", 2022-09-03T06:32:03.328 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:9", 2022-09-03T06:32:03.328 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.0.retry", 2022-09-03T06:32:03.329 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000018", 2022-09-03T06:32:03.329 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000092", 2022-09-03T06:32:03.329 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:3", 2022-09-03T06:32:03.329 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:2", 2022-09-03T06:32:03.330 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:7", 2022-09-03T06:32:03.330 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:1", 2022-09-03T06:32:03.330 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:2", 2022-09-03T06:32:03.330 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:6", 2022-09-03T06:32:03.331 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:6", 2022-09-03T06:32:03.331 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:8", 2022-09-03T06:32:03.331 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.4", 2022-09-03T06:32:03.332 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.1.retry", 2022-09-03T06:32:03.332 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:8", 2022-09-03T06:32:03.332 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:7", 2022-09-03T06:32:03.332 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:6", 2022-09-03T06:32:03.333 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000014", 2022-09-03T06:32:03.333 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:9", 2022-09-03T06:32:03.333 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:2", 2022-09-03T06:32:03.333 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:1", 2022-09-03T06:32:03.334 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000112", 2022-09-03T06:32:03.334 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:2", 2022-09-03T06:32:03.334 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000007", 2022-09-03T06:32:03.335 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000021", 2022-09-03T06:32:03.335 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:4", 2022-09-03T06:32:03.335 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000064", 2022-09-03T06:32:03.336 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:8", 2022-09-03T06:32:03.336 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:10", 2022-09-03T06:32:03.336 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:6", 2022-09-03T06:32:03.336 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000071", 2022-09-03T06:32:03.337 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:3", 2022-09-03T06:32:03.337 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:5", 2022-09-03T06:32:03.337 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:4", 2022-09-03T06:32:03.337 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000074", 2022-09-03T06:32:03.338 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:0", 2022-09-03T06:32:03.338 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:4", 2022-09-03T06:32:03.338 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:2", 2022-09-03T06:32:03.339 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:7", 2022-09-03T06:32:03.339 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:4", 2022-09-03T06:32:03.339 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:2", 2022-09-03T06:32:03.339 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:0", 2022-09-03T06:32:03.340 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.1.ztukkdnSZ1b.0", 2022-09-03T06:32:03.340 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:6", 2022-09-03T06:32:03.340 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:3", 2022-09-03T06:32:03.340 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000081", 2022-09-03T06:32:03.341 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000009", 2022-09-03T06:32:03.341 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5", 2022-09-03T06:32:03.341 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:8", 2022-09-03T06:32:03.341 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000121", 2022-09-03T06:32:03.342 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.1", 2022-09-03T06:32:03.342 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:10", 2022-09-03T06:32:03.342 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:10", 2022-09-03T06:32:03.343 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:10", 2022-09-03T06:32:03.343 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:1", 2022-09-03T06:32:03.343 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000125", 2022-09-03T06:32:03.343 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:5", 2022-09-03T06:32:03.344 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000082", 2022-09-03T06:32:03.344 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.0", 2022-09-03T06:32:03.344 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:5", 2022-09-03T06:32:03.344 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:7", 2022-09-03T06:32:03.345 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:4", 2022-09-03T06:32:03.345 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:32:03.345 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:0", 2022-09-03T06:32:03.346 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:9", 2022-09-03T06:32:03.346 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000105", 2022-09-03T06:32:03.346 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000059", 2022-09-03T06:32:03.346 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:4", 2022-09-03T06:32:03.347 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:9", 2022-09-03T06:32:03.347 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:9", 2022-09-03T06:32:03.347 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:5", 2022-09-03T06:32:03.347 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000077", 2022-09-03T06:32:03.348 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:3", 2022-09-03T06:32:03.348 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000032", 2022-09-03T06:32:03.348 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:1", 2022-09-03T06:32:03.348 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:7", 2022-09-03T06:32:03.349 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:5", 2022-09-03T06:32:03.349 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:4", 2022-09-03T06:32:03.349 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:32:03.349 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:4", 2022-09-03T06:32:03.350 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:3", 2022-09-03T06:32:03.350 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:10", 2022-09-03T06:32:03.350 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:3", 2022-09-03T06:32:03.350 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:7", 2022-09-03T06:32:03.351 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:3", 2022-09-03T06:32:03.351 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:9", 2022-09-03T06:32:03.351 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:32:03.351 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:1", 2022-09-03T06:32:03.352 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364", 2022-09-03T06:32:03.352 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:7", 2022-09-03T06:32:03.352 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:8", 2022-09-03T06:32:03.352 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:1", 2022-09-03T06:32:03.353 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:3", 2022-09-03T06:32:03.354 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000053", 2022-09-03T06:32:03.354 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:8", 2022-09-03T06:32:03.355 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:32:03.356 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000091", 2022-09-03T06:32:03.357 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:10", 2022-09-03T06:32:03.357 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:7", 2022-09-03T06:32:03.358 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:5", 2022-09-03T06:32:03.358 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:8", 2022-09-03T06:32:03.358 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:1", 2022-09-03T06:32:03.359 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000065", 2022-09-03T06:32:03.359 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000083", 2022-09-03T06:32:03.359 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000010", 2022-09-03T06:32:03.360 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:2", 2022-09-03T06:32:03.360 INFO:teuthology.orchestra.run.smithi062.stdout: "meta.log.b84ccf39-78f5-4998-9db6-42362bc8c98b.0", 2022-09-03T06:32:03.360 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:9", 2022-09-03T06:32:03.360 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.7", 2022-09-03T06:32:03.361 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.3", 2022-09-03T06:32:03.361 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:1", 2022-09-03T06:32:03.361 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:2", 2022-09-03T06:32:03.362 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:6", 2022-09-03T06:32:03.362 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:9", 2022-09-03T06:32:03.362 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:1", 2022-09-03T06:32:03.363 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:5", 2022-09-03T06:32:03.363 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:6", 2022-09-03T06:32:03.363 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:8", 2022-09-03T06:32:03.363 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.0.N7ePXaICeDU.0", 2022-09-03T06:32:03.364 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:7", 2022-09-03T06:32:03.364 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:5", 2022-09-03T06:32:03.364 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:5", 2022-09-03T06:32:03.364 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:0", 2022-09-03T06:32:03.365 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status.shard.2", 2022-09-03T06:32:03.365 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4", 2022-09-03T06:32:03.365 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:32:03.366 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:0", 2022-09-03T06:32:03.366 INFO:teuthology.orchestra.run.smithi062.stdout: "meta.log.2962e93e-ca58-4758-b6d9-6fe2e67c2976.1", 2022-09-03T06:32:03.366 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:7", 2022-09-03T06:32:03.366 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:1", 2022-09-03T06:32:03.367 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:0", 2022-09-03T06:32:03.367 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3", 2022-09-03T06:32:03.367 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:10", 2022-09-03T06:32:03.368 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000045", 2022-09-03T06:32:03.368 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:10", 2022-09-03T06:32:03.369 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000002", 2022-09-03T06:32:03.369 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:4", 2022-09-03T06:32:03.370 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:10", 2022-09-03T06:32:03.370 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:1", 2022-09-03T06:32:03.370 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:32:03.370 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000116", 2022-09-03T06:32:03.371 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:5", 2022-09-03T06:32:03.371 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:5", 2022-09-03T06:32:03.371 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:6", 2022-09-03T06:32:03.372 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:1", 2022-09-03T06:32:03.372 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:4", 2022-09-03T06:32:03.372 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:10", 2022-09-03T06:32:03.372 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:7", 2022-09-03T06:32:03.373 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:0", 2022-09-03T06:32:03.373 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:1", 2022-09-03T06:32:03.373 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000034", 2022-09-03T06:32:03.373 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:4", 2022-09-03T06:32:03.374 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:9", 2022-09-03T06:32:03.374 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000101", 2022-09-03T06:32:03.374 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:4", 2022-09-03T06:32:03.374 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:1", 2022-09-03T06:32:03.375 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:1", 2022-09-03T06:32:03.375 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:5", 2022-09-03T06:32:03.375 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:0", 2022-09-03T06:32:03.375 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status.shard.1", 2022-09-03T06:32:03.376 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:10", 2022-09-03T06:32:03.376 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:7", 2022-09-03T06:32:03.376 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5", 2022-09-03T06:32:03.376 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:4", 2022-09-03T06:32:03.377 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:2", 2022-09-03T06:32:03.377 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:2", 2022-09-03T06:32:03.378 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:3", 2022-09-03T06:32:03.378 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000079", 2022-09-03T06:32:03.378 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000049", 2022-09-03T06:32:03.378 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:2", 2022-09-03T06:32:03.379 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000117", 2022-09-03T06:32:03.379 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000044", 2022-09-03T06:32:03.379 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:9", 2022-09-03T06:32:03.380 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:32:03.380 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:0", 2022-09-03T06:32:03.380 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:4", 2022-09-03T06:32:03.380 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000066", 2022-09-03T06:32:03.381 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:0", 2022-09-03T06:32:03.381 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:8", 2022-09-03T06:32:03.381 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:7", 2022-09-03T06:32:03.381 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:5", 2022-09-03T06:32:03.381 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:10", 2022-09-03T06:32:03.382 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.2", 2022-09-03T06:32:03.382 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:6", 2022-09-03T06:32:03.382 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:8", 2022-09-03T06:32:03.382 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:7", 2022-09-03T06:32:03.383 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:8", 2022-09-03T06:32:03.383 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:1", 2022-09-03T06:32:03.384 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000068", 2022-09-03T06:32:03.384 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:8", 2022-09-03T06:32:03.384 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000085", 2022-09-03T06:32:03.384 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:6", 2022-09-03T06:32:03.385 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:5", 2022-09-03T06:32:03.385 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:1", 2022-09-03T06:32:03.385 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:8", 2022-09-03T06:32:03.385 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:5", 2022-09-03T06:32:03.386 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:5", 2022-09-03T06:32:03.386 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:6", 2022-09-03T06:32:03.386 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:4", 2022-09-03T06:32:03.387 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:0", 2022-09-03T06:32:03.387 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:3", 2022-09-03T06:32:03.387 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.3.retry", 2022-09-03T06:32:03.387 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000073", 2022-09-03T06:32:03.388 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:7", 2022-09-03T06:32:03.388 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000038", 2022-09-03T06:32:03.388 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:4", 2022-09-03T06:32:03.388 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:0", 2022-09-03T06:32:03.389 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:8", 2022-09-03T06:32:03.389 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.10", 2022-09-03T06:32:03.389 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:5", 2022-09-03T06:32:03.389 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000118", 2022-09-03T06:32:03.389 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:5", 2022-09-03T06:32:03.390 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:4", 2022-09-03T06:32:03.390 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:1", 2022-09-03T06:32:03.390 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000036", 2022-09-03T06:32:03.391 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:6", 2022-09-03T06:32:03.391 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:1", 2022-09-03T06:32:03.391 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000103", 2022-09-03T06:32:03.391 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:0", 2022-09-03T06:32:03.392 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:6", 2022-09-03T06:32:03.392 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status.shard.3", 2022-09-03T06:32:03.392 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:0", 2022-09-03T06:32:03.392 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:10", 2022-09-03T06:32:03.393 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:6", 2022-09-03T06:32:03.393 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:1", 2022-09-03T06:32:03.393 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:3", 2022-09-03T06:32:03.394 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000119", 2022-09-03T06:32:03.394 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:0", 2022-09-03T06:32:03.394 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:3", 2022-09-03T06:32:03.395 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:6", 2022-09-03T06:32:03.395 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000098", 2022-09-03T06:32:03.395 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:32:03.395 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:8", 2022-09-03T06:32:03.395 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:2", 2022-09-03T06:32:03.396 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:7", 2022-09-03T06:32:03.396 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status.shard.0", 2022-09-03T06:32:03.396 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:9", 2022-09-03T06:32:03.396 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000027", 2022-09-03T06:32:03.397 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:8", 2022-09-03T06:32:03.397 INFO:teuthology.orchestra.run.smithi062.stdout: "mdlog.sync-status", 2022-09-03T06:32:03.397 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000019", 2022-09-03T06:32:03.398 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:9", 2022-09-03T06:32:03.398 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:0", 2022-09-03T06:32:03.398 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:9", 2022-09-03T06:32:03.399 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000039", 2022-09-03T06:32:03.399 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:8", 2022-09-03T06:32:03.399 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:8", 2022-09-03T06:32:03.399 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:8", 2022-09-03T06:32:03.400 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:10", 2022-09-03T06:32:03.400 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:9", 2022-09-03T06:32:03.400 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:5", 2022-09-03T06:32:03.400 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:4", 2022-09-03T06:32:03.401 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:4", 2022-09-03T06:32:03.401 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000100", 2022-09-03T06:32:03.401 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:4", 2022-09-03T06:32:03.401 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:1", 2022-09-03T06:32:03.401 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:3", 2022-09-03T06:32:03.402 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:32:03.402 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:9", 2022-09-03T06:32:03.402 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:3", 2022-09-03T06:32:03.403 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000093", 2022-09-03T06:32:03.403 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:0", 2022-09-03T06:32:03.403 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:7", 2022-09-03T06:32:03.403 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:2", 2022-09-03T06:32:03.404 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000004", 2022-09-03T06:32:03.404 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000063", 2022-09-03T06:32:03.404 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:3", 2022-09-03T06:32:03.404 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000122", 2022-09-03T06:32:03.405 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:1", 2022-09-03T06:32:03.405 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000057", 2022-09-03T06:32:03.405 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:32:03.405 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:7", 2022-09-03T06:32:03.406 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:7", 2022-09-03T06:32:03.406 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.1", 2022-09-03T06:32:03.406 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:9", 2022-09-03T06:32:03.406 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000054", 2022-09-03T06:32:03.407 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000114", 2022-09-03T06:32:03.407 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:10", 2022-09-03T06:32:03.407 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.3.retry", 2022-09-03T06:32:03.407 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.2", 2022-09-03T06:32:03.408 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.0", 2022-09-03T06:32:03.408 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:3", 2022-09-03T06:32:03.408 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:0", 2022-09-03T06:32:03.408 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:0", 2022-09-03T06:32:03.409 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.3.St1Nz7cZQCy.0", 2022-09-03T06:32:03.409 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000012", 2022-09-03T06:32:03.409 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:3", 2022-09-03T06:32:03.409 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:3", 2022-09-03T06:32:03.410 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:9", 2022-09-03T06:32:03.410 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:6", 2022-09-03T06:32:03.410 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:4", 2022-09-03T06:32:03.410 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:7", 2022-09-03T06:32:03.411 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:4", 2022-09-03T06:32:03.411 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:3", 2022-09-03T06:32:03.411 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000084", 2022-09-03T06:32:03.411 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000043", 2022-09-03T06:32:03.412 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:6", 2022-09-03T06:32:03.412 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:2", 2022-09-03T06:32:03.412 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:0", 2022-09-03T06:32:03.412 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:6", 2022-09-03T06:32:03.413 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:10", 2022-09-03T06:32:03.413 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:8", 2022-09-03T06:32:03.414 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:5", 2022-09-03T06:32:03.414 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:2", 2022-09-03T06:32:03.415 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:9", 2022-09-03T06:32:03.415 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:3", 2022-09-03T06:32:03.416 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:32:03.416 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:10", 2022-09-03T06:32:03.417 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:10", 2022-09-03T06:32:03.417 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:5", 2022-09-03T06:32:03.418 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:32:03.418 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:8", 2022-09-03T06:32:03.419 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000111", 2022-09-03T06:32:03.419 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000048", 2022-09-03T06:32:03.419 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:8", 2022-09-03T06:32:03.420 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.2", 2022-09-03T06:32:03.420 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:8", 2022-09-03T06:32:03.420 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:2", 2022-09-03T06:32:03.421 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000020", 2022-09-03T06:32:03.421 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:9", 2022-09-03T06:32:03.422 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:4", 2022-09-03T06:32:03.422 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:7", 2022-09-03T06:32:03.423 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:5", 2022-09-03T06:32:03.423 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:7", 2022-09-03T06:32:03.424 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:8", 2022-09-03T06:32:03.424 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.3", 2022-09-03T06:32:03.425 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:9", 2022-09-03T06:32:03.425 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000099", 2022-09-03T06:32:03.425 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000056", 2022-09-03T06:32:03.426 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:0", 2022-09-03T06:32:03.427 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:2", 2022-09-03T06:32:03.427 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:32:03.428 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:7", 2022-09-03T06:32:03.428 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:7", 2022-09-03T06:32:03.429 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:0", 2022-09-03T06:32:03.429 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:6", 2022-09-03T06:32:03.430 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:32:03.430 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:6", 2022-09-03T06:32:03.431 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:7", 2022-09-03T06:32:03.431 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:0", 2022-09-03T06:32:03.432 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:0", 2022-09-03T06:32:03.432 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:10", 2022-09-03T06:32:03.433 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:32:03.433 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.3", 2022-09-03T06:32:03.434 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000072", 2022-09-03T06:32:03.434 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:9", 2022-09-03T06:32:03.435 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:0", 2022-09-03T06:32:03.435 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:0", 2022-09-03T06:32:03.435 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:9", 2022-09-03T06:32:03.436 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:10", 2022-09-03T06:32:03.436 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:9", 2022-09-03T06:32:03.437 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000062", 2022-09-03T06:32:03.437 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:9", 2022-09-03T06:32:03.437 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:10", 2022-09-03T06:32:03.437 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:0", 2022-09-03T06:32:03.438 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000115", 2022-09-03T06:32:03.438 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.1", 2022-09-03T06:32:03.438 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000033", 2022-09-03T06:32:03.438 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:7", 2022-09-03T06:32:03.439 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:5", 2022-09-03T06:32:03.439 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:7", 2022-09-03T06:32:03.439 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:10", 2022-09-03T06:32:03.440 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:3", 2022-09-03T06:32:03.440 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:7", 2022-09-03T06:32:03.440 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:1", 2022-09-03T06:32:03.441 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:1", 2022-09-03T06:32:03.441 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:32:03.442 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:3", 2022-09-03T06:32:03.442 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:0", 2022-09-03T06:32:03.442 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000025", 2022-09-03T06:32:03.443 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:7", 2022-09-03T06:32:03.443 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:8", 2022-09-03T06:32:03.443 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000123", 2022-09-03T06:32:03.444 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:4", 2022-09-03T06:32:03.444 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:10", 2022-09-03T06:32:03.445 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:4", 2022-09-03T06:32:03.445 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:5", 2022-09-03T06:32:03.445 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:10", 2022-09-03T06:32:03.446 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000108", 2022-09-03T06:32:03.446 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000094", 2022-09-03T06:32:03.446 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:10", 2022-09-03T06:32:03.447 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:32:03.447 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:5", 2022-09-03T06:32:03.447 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000060", 2022-09-03T06:32:03.448 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:2", 2022-09-03T06:32:03.448 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000109", 2022-09-03T06:32:03.448 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:4", 2022-09-03T06:32:03.448 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:10", 2022-09-03T06:32:03.449 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:1", 2022-09-03T06:32:03.449 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:7", 2022-09-03T06:32:03.449 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:2", 2022-09-03T06:32:03.449 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000013", 2022-09-03T06:32:03.449 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.9b3b3b7d-a26b-43ae-9603-22cc8f947364.0.retry", 2022-09-03T06:32:03.450 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:5", 2022-09-03T06:32:03.450 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:32:03.450 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:6", 2022-09-03T06:32:03.450 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:0", 2022-09-03T06:32:03.451 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:4", 2022-09-03T06:32:03.451 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:32:03.451 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:2", 2022-09-03T06:32:03.451 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:2", 2022-09-03T06:32:03.452 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:0", 2022-09-03T06:32:03.452 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:9", 2022-09-03T06:32:03.452 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:3", 2022-09-03T06:32:03.452 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:10", 2022-09-03T06:32:03.453 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:7", 2022-09-03T06:32:03.453 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:1", 2022-09-03T06:32:03.453 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:6", 2022-09-03T06:32:03.454 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:8", 2022-09-03T06:32:03.454 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:8", 2022-09-03T06:32:03.454 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:2", 2022-09-03T06:32:03.454 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:1", 2022-09-03T06:32:03.455 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000051", 2022-09-03T06:32:03.455 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:1", 2022-09-03T06:32:03.455 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:2", 2022-09-03T06:32:03.455 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000106", 2022-09-03T06:32:03.455 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:10", 2022-09-03T06:32:03.456 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:9", 2022-09-03T06:32:03.456 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:0", 2022-09-03T06:32:03.456 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000017", 2022-09-03T06:32:03.456 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:3", 2022-09-03T06:32:03.457 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:8", 2022-09-03T06:32:03.457 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:8", 2022-09-03T06:32:03.457 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:6", 2022-09-03T06:32:03.457 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:9", 2022-09-03T06:32:03.458 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000023", 2022-09-03T06:32:03.458 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3", 2022-09-03T06:32:03.458 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:4", 2022-09-03T06:32:03.459 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:2", 2022-09-03T06:32:03.459 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000107", 2022-09-03T06:32:03.459 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:32:03.459 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:9", 2022-09-03T06:32:03.460 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000046", 2022-09-03T06:32:03.460 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:1", 2022-09-03T06:32:03.460 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:4", 2022-09-03T06:32:03.460 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:0", 2022-09-03T06:32:03.461 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:3", 2022-09-03T06:32:03.461 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:2", 2022-09-03T06:32:03.461 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:10", 2022-09-03T06:32:03.461 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:0", 2022-09-03T06:32:03.462 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:6", 2022-09-03T06:32:03.462 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:7", 2022-09-03T06:32:03.462 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:2", 2022-09-03T06:32:03.462 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:2", 2022-09-03T06:32:03.463 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:32:03.463 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:3", 2022-09-03T06:32:03.463 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:6", 2022-09-03T06:32:03.463 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:6", 2022-09-03T06:32:03.464 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:3", 2022-09-03T06:32:03.464 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.2", 2022-09-03T06:32:03.464 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:1", 2022-09-03T06:32:03.464 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000067", 2022-09-03T06:32:03.465 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:9", 2022-09-03T06:32:03.465 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:5", 2022-09-03T06:32:03.465 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:10", 2022-09-03T06:32:03.465 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:5", 2022-09-03T06:32:03.465 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:1", 2022-09-03T06:32:03.466 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:7", 2022-09-03T06:32:03.466 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:3", 2022-09-03T06:32:03.466 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:5", 2022-09-03T06:32:03.466 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:3", 2022-09-03T06:32:03.467 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:5", 2022-09-03T06:32:03.467 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:6", 2022-09-03T06:32:03.467 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:2", 2022-09-03T06:32:03.467 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:7", 2022-09-03T06:32:03.468 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.0", 2022-09-03T06:32:03.468 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.9", 2022-09-03T06:32:03.468 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000086", 2022-09-03T06:32:03.468 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:2", 2022-09-03T06:32:03.469 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:8", 2022-09-03T06:32:03.469 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:6", 2022-09-03T06:32:03.469 INFO:teuthology.orchestra.run.smithi062.stdout: "sync.error-log.5", 2022-09-03T06:32:03.469 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:4", 2022-09-03T06:32:03.470 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:7", 2022-09-03T06:32:03.470 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:2", 2022-09-03T06:32:03.470 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:7", 2022-09-03T06:32:03.470 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:6", 2022-09-03T06:32:03.471 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:10", 2022-09-03T06:32:03.471 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:10", 2022-09-03T06:32:03.471 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:6", 2022-09-03T06:32:03.471 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1", 2022-09-03T06:32:03.472 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:6", 2022-09-03T06:32:03.472 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:1", 2022-09-03T06:32:03.472 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:4", 2022-09-03T06:32:03.472 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:10", 2022-09-03T06:32:03.473 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:8", 2022-09-03T06:32:03.473 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:2", 2022-09-03T06:32:03.473 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:32:03.473 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:8", 2022-09-03T06:32:03.474 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:8", 2022-09-03T06:32:03.474 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:2", 2022-09-03T06:32:03.474 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:6", 2022-09-03T06:32:03.474 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:3", 2022-09-03T06:32:03.475 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:4", 2022-09-03T06:32:03.475 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:8", 2022-09-03T06:32:03.475 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:3", 2022-09-03T06:32:03.475 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:1", 2022-09-03T06:32:03.475 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:0", 2022-09-03T06:32:03.476 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:9", 2022-09-03T06:32:03.476 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:4", 2022-09-03T06:32:03.476 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:8", 2022-09-03T06:32:03.476 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:2", 2022-09-03T06:32:03.477 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:2", 2022-09-03T06:32:03.477 INFO:teuthology.orchestra.run.smithi062.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.1", 2022-09-03T06:32:03.477 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:6", 2022-09-03T06:32:03.477 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:9", 2022-09-03T06:32:03.478 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:6", 2022-09-03T06:32:03.478 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000050", 2022-09-03T06:32:03.478 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:6", 2022-09-03T06:32:03.479 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000102", 2022-09-03T06:32:03.479 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:1", 2022-09-03T06:32:03.479 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:8", 2022-09-03T06:32:03.479 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:9", 2022-09-03T06:32:03.480 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:1", 2022-09-03T06:32:03.480 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:32:03.480 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:8", 2022-09-03T06:32:03.480 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:6", 2022-09-03T06:32:03.481 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:5", 2022-09-03T06:32:03.481 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:3", 2022-09-03T06:32:03.482 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000124", 2022-09-03T06:32:03.482 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:0", 2022-09-03T06:32:03.482 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:32:03.483 INFO:teuthology.orchestra.run.smithi062.stdout: "data_log.2.XOpGESTHOdl.0", 2022-09-03T06:32:03.483 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:7", 2022-09-03T06:32:03.483 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:5", 2022-09-03T06:32:03.484 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000001", 2022-09-03T06:32:03.484 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:5", 2022-09-03T06:32:03.485 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:5", 2022-09-03T06:32:03.485 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000126", 2022-09-03T06:32:03.485 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:9", 2022-09-03T06:32:03.486 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:9", 2022-09-03T06:32:03.486 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:1", 2022-09-03T06:32:03.487 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:2", 2022-09-03T06:32:03.487 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000055", 2022-09-03T06:32:03.487 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000061", 2022-09-03T06:32:03.487 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:9", 2022-09-03T06:32:03.488 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:4", 2022-09-03T06:32:03.488 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000097", 2022-09-03T06:32:03.488 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:10", 2022-09-03T06:32:03.488 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:6", 2022-09-03T06:32:03.489 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:2", 2022-09-03T06:32:03.489 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000042", 2022-09-03T06:32:03.489 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:3", 2022-09-03T06:32:03.489 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4", 2022-09-03T06:32:03.490 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:32:03.490 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:10", 2022-09-03T06:32:03.490 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:3", 2022-09-03T06:32:03.490 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:0", 2022-09-03T06:32:03.491 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:10", 2022-09-03T06:32:03.491 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:4", 2022-09-03T06:32:03.491 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:4", 2022-09-03T06:32:03.491 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:6", 2022-09-03T06:32:03.492 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000075", 2022-09-03T06:32:03.492 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:32:03.492 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:2", 2022-09-03T06:32:03.492 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:0", 2022-09-03T06:32:03.493 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000120", 2022-09-03T06:32:03.493 INFO:teuthology.orchestra.run.smithi062.stdout: "obj_delete_at_hint.0000000030", 2022-09-03T06:32:03.493 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:3", 2022-09-03T06:32:03.493 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:4", 2022-09-03T06:32:03.494 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.9b3b3b7d-a26b-43ae-9603-22cc8f947364:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:3", 2022-09-03T06:32:03.494 INFO:teuthology.orchestra.run.smithi062.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:2" 2022-09-03T06:32:03.494 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:32:03.494 INFO:teuthology.orchestra.run.smithi062.stdout: 2022-09-03T06:32:03.496 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['log', 'list', '--rgw-zone', 'test-zone4', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:03.496 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'log', 'list', '--rgw-zone', 'test-zone4', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:03.497 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 log list --rgw-zone test-zone4 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:03.545 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:03.546 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:03.612 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:32:03.612 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:10", 2022-09-03T06:32:03.613 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000078", 2022-09-03T06:32:03.613 INFO:teuthology.orchestra.run.smithi087.stdout: "meta.history", 2022-09-03T06:32:03.614 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:10", 2022-09-03T06:32:03.614 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:8", 2022-09-03T06:32:03.615 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000070", 2022-09-03T06:32:03.615 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000104", 2022-09-03T06:32:03.616 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:4", 2022-09-03T06:32:03.616 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000026", 2022-09-03T06:32:03.617 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16", 2022-09-03T06:32:03.617 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:0", 2022-09-03T06:32:03.620 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:4", 2022-09-03T06:32:03.620 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.8", 2022-09-03T06:32:03.621 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19", 2022-09-03T06:32:03.621 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000028", 2022-09-03T06:32:03.621 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13", 2022-09-03T06:32:03.621 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:10", 2022-09-03T06:32:03.622 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:7", 2022-09-03T06:32:03.622 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:2", 2022-09-03T06:32:03.622 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000040", 2022-09-03T06:32:03.622 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:0", 2022-09-03T06:32:03.623 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.6", 2022-09-03T06:32:03.623 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:10", 2022-09-03T06:32:03.623 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17", 2022-09-03T06:32:03.624 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:8", 2022-09-03T06:32:03.624 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000015", 2022-09-03T06:32:03.624 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:6", 2022-09-03T06:32:03.625 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14", 2022-09-03T06:32:03.625 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000069", 2022-09-03T06:32:03.625 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000095", 2022-09-03T06:32:03.625 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:0", 2022-09-03T06:32:03.626 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:5", 2022-09-03T06:32:03.626 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000003", 2022-09-03T06:32:03.626 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000047", 2022-09-03T06:32:03.626 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:1", 2022-09-03T06:32:03.627 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:8", 2022-09-03T06:32:03.627 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:5", 2022-09-03T06:32:03.627 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:9", 2022-09-03T06:32:03.627 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000035", 2022-09-03T06:32:03.628 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:3", 2022-09-03T06:32:03.628 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000037", 2022-09-03T06:32:03.628 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:1", 2022-09-03T06:32:03.628 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:10", 2022-09-03T06:32:03.629 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36", 2022-09-03T06:32:03.629 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:1", 2022-09-03T06:32:03.629 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:4", 2022-09-03T06:32:03.630 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000024", 2022-09-03T06:32:03.630 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000000", 2022-09-03T06:32:03.630 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000031", 2022-09-03T06:32:03.630 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37", 2022-09-03T06:32:03.631 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:3", 2022-09-03T06:32:03.631 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:5", 2022-09-03T06:32:03.631 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15", 2022-09-03T06:32:03.631 INFO:teuthology.orchestra.run.smithi087.stdout: "meta.log.b84ccf39-78f5-4998-9db6-42362bc8c98b.1", 2022-09-03T06:32:03.632 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:3", 2022-09-03T06:32:03.632 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:8", 2022-09-03T06:32:03.632 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:5", 2022-09-03T06:32:03.633 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:6", 2022-09-03T06:32:03.633 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:9", 2022-09-03T06:32:03.634 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:9", 2022-09-03T06:32:03.634 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:7", 2022-09-03T06:32:03.635 INFO:teuthology.orchestra.run.smithi087.stdout: "data_loggenerations_metadata", 2022-09-03T06:32:03.635 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:3", 2022-09-03T06:32:03.635 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:2", 2022-09-03T06:32:03.635 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:10", 2022-09-03T06:32:03.636 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21", 2022-09-03T06:32:03.636 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000076", 2022-09-03T06:32:03.636 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:1", 2022-09-03T06:32:03.637 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000113", 2022-09-03T06:32:03.637 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000005", 2022-09-03T06:32:03.637 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:10", 2022-09-03T06:32:03.637 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000011", 2022-09-03T06:32:03.638 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000058", 2022-09-03T06:32:03.638 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000052", 2022-09-03T06:32:03.638 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:6", 2022-09-03T06:32:03.638 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:1", 2022-09-03T06:32:03.639 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:0", 2022-09-03T06:32:03.639 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:8", 2022-09-03T06:32:03.639 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:5", 2022-09-03T06:32:03.639 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11", 2022-09-03T06:32:03.640 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000088", 2022-09-03T06:32:03.640 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000080", 2022-09-03T06:32:03.640 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000090", 2022-09-03T06:32:03.641 INFO:teuthology.orchestra.run.smithi087.stdout: "meta.log.6a921abe-6ddc-4d12-8acc-a1151b4a5f12.1", 2022-09-03T06:32:03.641 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:10", 2022-09-03T06:32:03.641 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:5", 2022-09-03T06:32:03.641 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000110", 2022-09-03T06:32:03.642 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:7", 2022-09-03T06:32:03.642 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000096", 2022-09-03T06:32:03.643 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:4", 2022-09-03T06:32:03.643 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:0", 2022-09-03T06:32:03.643 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:9", 2022-09-03T06:32:03.643 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000087", 2022-09-03T06:32:03.644 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000008", 2022-09-03T06:32:03.644 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:4", 2022-09-03T06:32:03.644 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:1", 2022-09-03T06:32:03.644 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:7", 2022-09-03T06:32:03.645 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:6", 2022-09-03T06:32:03.645 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:5", 2022-09-03T06:32:03.645 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:3", 2022-09-03T06:32:03.645 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000006", 2022-09-03T06:32:03.646 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000029", 2022-09-03T06:32:03.646 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000089", 2022-09-03T06:32:03.647 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:7", 2022-09-03T06:32:03.647 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000022", 2022-09-03T06:32:03.647 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:1", 2022-09-03T06:32:03.647 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:9", 2022-09-03T06:32:03.648 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:10", 2022-09-03T06:32:03.648 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:7", 2022-09-03T06:32:03.648 INFO:teuthology.orchestra.run.smithi087.stdout: "bilog.trim", 2022-09-03T06:32:03.648 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000016", 2022-09-03T06:32:03.649 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18", 2022-09-03T06:32:03.649 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000041", 2022-09-03T06:32:03.649 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25", 2022-09-03T06:32:03.649 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:8", 2022-09-03T06:32:03.650 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:6", 2022-09-03T06:32:03.650 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:1", 2022-09-03T06:32:03.650 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:3", 2022-09-03T06:32:03.650 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.0.retry", 2022-09-03T06:32:03.651 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000018", 2022-09-03T06:32:03.651 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000092", 2022-09-03T06:32:03.651 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:3", 2022-09-03T06:32:03.651 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:7", 2022-09-03T06:32:03.652 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:1", 2022-09-03T06:32:03.652 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:8", 2022-09-03T06:32:03.652 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.4", 2022-09-03T06:32:03.652 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.1.retry", 2022-09-03T06:32:03.653 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:7", 2022-09-03T06:32:03.653 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000014", 2022-09-03T06:32:03.653 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:9", 2022-09-03T06:32:03.653 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:1", 2022-09-03T06:32:03.656 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000112", 2022-09-03T06:32:03.656 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:2", 2022-09-03T06:32:03.657 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000007", 2022-09-03T06:32:03.657 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000021", 2022-09-03T06:32:03.658 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:4", 2022-09-03T06:32:03.659 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000064", 2022-09-03T06:32:03.659 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:8", 2022-09-03T06:32:03.660 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:10", 2022-09-03T06:32:03.660 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:6", 2022-09-03T06:32:03.661 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000071", 2022-09-03T06:32:03.661 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:5", 2022-09-03T06:32:03.662 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000074", 2022-09-03T06:32:03.662 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:0", 2022-09-03T06:32:03.663 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:2", 2022-09-03T06:32:03.664 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:7", 2022-09-03T06:32:03.664 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:4", 2022-09-03T06:32:03.665 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:2", 2022-09-03T06:32:03.666 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:6", 2022-09-03T06:32:03.666 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:3", 2022-09-03T06:32:03.667 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000081", 2022-09-03T06:32:03.667 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000009", 2022-09-03T06:32:03.668 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000121", 2022-09-03T06:32:03.668 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:10", 2022-09-03T06:32:03.669 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:10", 2022-09-03T06:32:03.669 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:1", 2022-09-03T06:32:03.669 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000125", 2022-09-03T06:32:03.670 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000082", 2022-09-03T06:32:03.672 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:5", 2022-09-03T06:32:03.673 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:7", 2022-09-03T06:32:03.673 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:4", 2022-09-03T06:32:03.674 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:0", 2022-09-03T06:32:03.674 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:9", 2022-09-03T06:32:03.675 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000105", 2022-09-03T06:32:03.675 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000059", 2022-09-03T06:32:03.675 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:4", 2022-09-03T06:32:03.676 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:9", 2022-09-03T06:32:03.676 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:9", 2022-09-03T06:32:03.676 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000077", 2022-09-03T06:32:03.676 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000032", 2022-09-03T06:32:03.677 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:7", 2022-09-03T06:32:03.677 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:5", 2022-09-03T06:32:03.678 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:4", 2022-09-03T06:32:03.690 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:9", 2022-09-03T06:32:03.690 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:7", 2022-09-03T06:32:03.691 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:8", 2022-09-03T06:32:03.692 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:1", 2022-09-03T06:32:03.692 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000053", 2022-09-03T06:32:03.693 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba", 2022-09-03T06:32:03.693 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000091", 2022-09-03T06:32:03.693 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:10", 2022-09-03T06:32:03.694 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000065", 2022-09-03T06:32:03.695 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000083", 2022-09-03T06:32:03.695 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000010", 2022-09-03T06:32:03.695 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:2", 2022-09-03T06:32:03.695 INFO:teuthology.orchestra.run.smithi087.stdout: "meta.log.b84ccf39-78f5-4998-9db6-42362bc8c98b.0", 2022-09-03T06:32:03.696 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.7", 2022-09-03T06:32:03.696 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.3", 2022-09-03T06:32:03.696 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:1", 2022-09-03T06:32:03.697 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:6", 2022-09-03T06:32:03.698 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:8", 2022-09-03T06:32:03.698 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:7", 2022-09-03T06:32:03.698 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:5", 2022-09-03T06:32:03.699 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:5", 2022-09-03T06:32:03.699 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:0", 2022-09-03T06:32:03.699 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status.shard.2", 2022-09-03T06:32:03.700 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4", 2022-09-03T06:32:03.700 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10", 2022-09-03T06:32:03.700 INFO:teuthology.orchestra.run.smithi087.stdout: "meta.log.2962e93e-ca58-4758-b6d9-6fe2e67c2976.1", 2022-09-03T06:32:03.700 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:1", 2022-09-03T06:32:03.701 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:0", 2022-09-03T06:32:03.701 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3", 2022-09-03T06:32:03.702 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:10", 2022-09-03T06:32:03.702 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000045", 2022-09-03T06:32:03.702 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000002", 2022-09-03T06:32:03.703 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:4", 2022-09-03T06:32:03.703 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:1", 2022-09-03T06:32:03.703 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000116", 2022-09-03T06:32:03.704 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:5", 2022-09-03T06:32:03.704 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:5", 2022-09-03T06:32:03.704 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:6", 2022-09-03T06:32:03.704 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:7", 2022-09-03T06:32:03.705 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000034", 2022-09-03T06:32:03.705 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000101", 2022-09-03T06:32:03.705 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:4", 2022-09-03T06:32:03.705 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:1", 2022-09-03T06:32:03.706 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:0", 2022-09-03T06:32:03.706 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status.shard.1", 2022-09-03T06:32:03.706 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:10", 2022-09-03T06:32:03.707 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5", 2022-09-03T06:32:03.707 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:4", 2022-09-03T06:32:03.709 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:2", 2022-09-03T06:32:03.709 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:2", 2022-09-03T06:32:03.709 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:3", 2022-09-03T06:32:03.710 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000079", 2022-09-03T06:32:03.710 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000049", 2022-09-03T06:32:03.710 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000117", 2022-09-03T06:32:03.711 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000044", 2022-09-03T06:32:03.711 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:9", 2022-09-03T06:32:03.711 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000066", 2022-09-03T06:32:03.712 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:8", 2022-09-03T06:32:03.712 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:7", 2022-09-03T06:32:03.713 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:8", 2022-09-03T06:32:03.714 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:7", 2022-09-03T06:32:03.714 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:1", 2022-09-03T06:32:03.715 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000068", 2022-09-03T06:32:03.715 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000085", 2022-09-03T06:32:03.715 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:6", 2022-09-03T06:32:03.715 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:5", 2022-09-03T06:32:03.716 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:8", 2022-09-03T06:32:03.716 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:5", 2022-09-03T06:32:03.716 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.3.retry", 2022-09-03T06:32:03.717 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000073", 2022-09-03T06:32:03.717 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000038", 2022-09-03T06:32:03.717 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:4", 2022-09-03T06:32:03.718 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:0", 2022-09-03T06:32:03.718 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.10", 2022-09-03T06:32:03.718 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000118", 2022-09-03T06:32:03.719 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:5", 2022-09-03T06:32:03.719 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:4", 2022-09-03T06:32:03.720 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:1", 2022-09-03T06:32:03.720 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000036", 2022-09-03T06:32:03.721 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000103", 2022-09-03T06:32:03.721 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:0", 2022-09-03T06:32:03.721 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status.shard.3", 2022-09-03T06:32:03.722 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:6", 2022-09-03T06:32:03.722 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:3", 2022-09-03T06:32:03.723 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000119", 2022-09-03T06:32:03.724 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:3", 2022-09-03T06:32:03.724 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000098", 2022-09-03T06:32:03.724 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:8", 2022-09-03T06:32:03.725 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:7", 2022-09-03T06:32:03.725 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status.shard.0", 2022-09-03T06:32:03.725 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000027", 2022-09-03T06:32:03.725 INFO:teuthology.orchestra.run.smithi087.stdout: "mdlog.sync-status", 2022-09-03T06:32:03.726 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000019", 2022-09-03T06:32:03.726 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:9", 2022-09-03T06:32:03.726 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:9", 2022-09-03T06:32:03.727 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000039", 2022-09-03T06:32:03.727 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:5", 2022-09-03T06:32:03.727 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:4", 2022-09-03T06:32:03.727 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000100", 2022-09-03T06:32:03.728 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:1", 2022-09-03T06:32:03.728 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1:3", 2022-09-03T06:32:03.728 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000093", 2022-09-03T06:32:03.728 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:2", 2022-09-03T06:32:03.729 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000004", 2022-09-03T06:32:03.729 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000063", 2022-09-03T06:32:03.729 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000122", 2022-09-03T06:32:03.730 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000057", 2022-09-03T06:32:03.730 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24", 2022-09-03T06:32:03.730 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:9", 2022-09-03T06:32:03.731 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000054", 2022-09-03T06:32:03.731 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000114", 2022-09-03T06:32:03.731 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.2", 2022-09-03T06:32:03.731 INFO:teuthology.orchestra.run.smithi087.stdout: "data_log.0", 2022-09-03T06:32:03.732 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:3", 2022-09-03T06:32:03.732 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:0", 2022-09-03T06:32:03.732 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:0", 2022-09-03T06:32:03.733 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000012", 2022-09-03T06:32:03.733 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:4", 2022-09-03T06:32:03.734 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000084", 2022-09-03T06:32:03.734 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000043", 2022-09-03T06:32:03.735 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:8", 2022-09-03T06:32:03.735 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:2", 2022-09-03T06:32:03.736 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:9", 2022-09-03T06:32:03.736 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:10", 2022-09-03T06:32:03.737 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12", 2022-09-03T06:32:03.737 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000111", 2022-09-03T06:32:03.737 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000048", 2022-09-03T06:32:03.738 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:8", 2022-09-03T06:32:03.739 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.2", 2022-09-03T06:32:03.739 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:2", 2022-09-03T06:32:03.739 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000020", 2022-09-03T06:32:03.740 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:9", 2022-09-03T06:32:03.740 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:7", 2022-09-03T06:32:03.741 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:7", 2022-09-03T06:32:03.741 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:9", 2022-09-03T06:32:03.742 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000099", 2022-09-03T06:32:03.742 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000056", 2022-09-03T06:32:03.743 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:0", 2022-09-03T06:32:03.743 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:7", 2022-09-03T06:32:03.744 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:7", 2022-09-03T06:32:03.744 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:0", 2022-09-03T06:32:03.744 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:6", 2022-09-03T06:32:03.745 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:6", 2022-09-03T06:32:03.745 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:0", 2022-09-03T06:32:03.746 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:0", 2022-09-03T06:32:03.746 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.3", 2022-09-03T06:32:03.746 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000072", 2022-09-03T06:32:03.747 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:0", 2022-09-03T06:32:03.747 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:9", 2022-09-03T06:32:03.747 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:9", 2022-09-03T06:32:03.748 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000062", 2022-09-03T06:32:03.748 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:0", 2022-09-03T06:32:03.749 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000115", 2022-09-03T06:32:03.749 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.1", 2022-09-03T06:32:03.749 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000033", 2022-09-03T06:32:03.749 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:5", 2022-09-03T06:32:03.750 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:3", 2022-09-03T06:32:03.750 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:7", 2022-09-03T06:32:03.750 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:1", 2022-09-03T06:32:03.751 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:1", 2022-09-03T06:32:03.751 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:3", 2022-09-03T06:32:03.752 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:0", 2022-09-03T06:32:03.752 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000025", 2022-09-03T06:32:03.753 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000123", 2022-09-03T06:32:03.753 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:4", 2022-09-03T06:32:03.753 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:10", 2022-09-03T06:32:03.754 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:4", 2022-09-03T06:32:03.754 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:5", 2022-09-03T06:32:03.754 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:10", 2022-09-03T06:32:03.755 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000108", 2022-09-03T06:32:03.755 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000094", 2022-09-03T06:32:03.755 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:5", 2022-09-03T06:32:03.755 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000060", 2022-09-03T06:32:03.756 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000109", 2022-09-03T06:32:03.756 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:4", 2022-09-03T06:32:03.756 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:2", 2022-09-03T06:32:03.756 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000013", 2022-09-03T06:32:03.757 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:5", 2022-09-03T06:32:03.757 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:6", 2022-09-03T06:32:03.757 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:4", 2022-09-03T06:32:03.757 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:2", 2022-09-03T06:32:03.758 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:9", 2022-09-03T06:32:03.758 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:3", 2022-09-03T06:32:03.758 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:6", 2022-09-03T06:32:03.758 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:8", 2022-09-03T06:32:03.759 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:8", 2022-09-03T06:32:03.759 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000051", 2022-09-03T06:32:03.759 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:2", 2022-09-03T06:32:03.759 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000106", 2022-09-03T06:32:03.760 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:10", 2022-09-03T06:32:03.760 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:9", 2022-09-03T06:32:03.760 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:0", 2022-09-03T06:32:03.760 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000017", 2022-09-03T06:32:03.761 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:3", 2022-09-03T06:32:03.761 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:8", 2022-09-03T06:32:03.761 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:8", 2022-09-03T06:32:03.761 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000023", 2022-09-03T06:32:03.762 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000107", 2022-09-03T06:32:03.762 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23", 2022-09-03T06:32:03.762 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:9", 2022-09-03T06:32:03.762 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000046", 2022-09-03T06:32:03.763 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:1", 2022-09-03T06:32:03.763 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:0", 2022-09-03T06:32:03.763 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:3", 2022-09-03T06:32:03.763 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:2", 2022-09-03T06:32:03.764 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-23:fca49d30-abba-4c2b-8462-66395ac063ba.4344.23:10", 2022-09-03T06:32:03.764 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:0", 2022-09-03T06:32:03.764 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:2", 2022-09-03T06:32:03.764 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:3", 2022-09-03T06:32:03.765 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:3", 2022-09-03T06:32:03.765 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:1", 2022-09-03T06:32:03.770 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000067", 2022-09-03T06:32:03.770 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:9", 2022-09-03T06:32:03.771 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-19:fca49d30-abba-4c2b-8462-66395ac063ba.4344.19:10", 2022-09-03T06:32:03.772 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-25:fca49d30-abba-4c2b-8462-66395ac063ba.4344.25:5", 2022-09-03T06:32:03.773 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:3", 2022-09-03T06:32:03.773 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:6", 2022-09-03T06:32:03.774 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:2", 2022-09-03T06:32:03.774 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.0", 2022-09-03T06:32:03.775 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.9", 2022-09-03T06:32:03.776 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000086", 2022-09-03T06:32:03.776 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-18:fca49d30-abba-4c2b-8462-66395ac063ba.4344.18:2", 2022-09-03T06:32:03.776 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:6", 2022-09-03T06:32:03.777 INFO:teuthology.orchestra.run.smithi087.stdout: "sync.error-log.5", 2022-09-03T06:32:03.777 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:2", 2022-09-03T06:32:03.778 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:7", 2022-09-03T06:32:03.778 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-36:fca49d30-abba-4c2b-8462-66395ac063ba.4344.36:6", 2022-09-03T06:32:03.779 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-12:fca49d30-abba-4c2b-8462-66395ac063ba.4344.12:10", 2022-09-03T06:32:03.779 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:6", 2022-09-03T06:32:03.780 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-49:fca49d30-abba-4c2b-8462-66395ac063ba.5481.1", 2022-09-03T06:32:03.780 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:4", 2022-09-03T06:32:03.781 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-11:fca49d30-abba-4c2b-8462-66395ac063ba.4344.11:2", 2022-09-03T06:32:03.781 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22:8", 2022-09-03T06:32:03.782 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:6", 2022-09-03T06:32:03.782 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-48:fca49d30-abba-4c2b-8462-66395ac063ba.5190.5:8", 2022-09-03T06:32:03.783 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-15:fca49d30-abba-4c2b-8462-66395ac063ba.4344.15:2", 2022-09-03T06:32:03.783 INFO:teuthology.orchestra.run.smithi087.stdout: "datalog.sync-status.shard.fca49d30-abba-4c2b-8462-66395ac063ba.1", 2022-09-03T06:32:03.784 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:6", 2022-09-03T06:32:03.784 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:6", 2022-09-03T06:32:03.785 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000050", 2022-09-03T06:32:03.785 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-46:fca49d30-abba-4c2b-8462-66395ac063ba.5190.3:6", 2022-09-03T06:32:03.786 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000102", 2022-09-03T06:32:03.786 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-17:fca49d30-abba-4c2b-8462-66395ac063ba.4344.17:8", 2022-09-03T06:32:03.787 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:1", 2022-09-03T06:32:03.787 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38", 2022-09-03T06:32:03.788 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-16:fca49d30-abba-4c2b-8462-66395ac063ba.4344.16:8", 2022-09-03T06:32:03.788 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000124", 2022-09-03T06:32:03.789 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20", 2022-09-03T06:32:03.789 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-38:fca49d30-abba-4c2b-8462-66395ac063ba.4344.38:7", 2022-09-03T06:32:03.790 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000001", 2022-09-03T06:32:03.790 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:5", 2022-09-03T06:32:03.791 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000126", 2022-09-03T06:32:03.791 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-20:fca49d30-abba-4c2b-8462-66395ac063ba.4344.20:9", 2022-09-03T06:32:03.791 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:2", 2022-09-03T06:32:03.792 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000055", 2022-09-03T06:32:03.793 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000061", 2022-09-03T06:32:03.793 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000097", 2022-09-03T06:32:03.793 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:6", 2022-09-03T06:32:03.794 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000042", 2022-09-03T06:32:03.794 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-21:fca49d30-abba-4c2b-8462-66395ac063ba.4344.21:3", 2022-09-03T06:32:03.795 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26", 2022-09-03T06:32:03.795 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-13:fca49d30-abba-4c2b-8462-66395ac063ba.4344.13:10", 2022-09-03T06:32:03.796 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-14:fca49d30-abba-4c2b-8462-66395ac063ba.4344.14:3", 2022-09-03T06:32:03.796 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:10", 2022-09-03T06:32:03.796 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-26:fca49d30-abba-4c2b-8462-66395ac063ba.4344.26:4", 2022-09-03T06:32:03.797 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-37:fca49d30-abba-4c2b-8462-66395ac063ba.4344.37:4", 2022-09-03T06:32:03.797 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000075", 2022-09-03T06:32:03.797 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.full-sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-22:fca49d30-abba-4c2b-8462-66395ac063ba.4344.22", 2022-09-03T06:32:03.797 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-10:fca49d30-abba-4c2b-8462-66395ac063ba.4344.10:2", 2022-09-03T06:32:03.798 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000120", 2022-09-03T06:32:03.798 INFO:teuthology.orchestra.run.smithi087.stdout: "obj_delete_at_hint.0000000030", 2022-09-03T06:32:03.798 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-47:fca49d30-abba-4c2b-8462-66395ac063ba.5190.4:3", 2022-09-03T06:32:03.799 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket.sync-status.fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-24:fca49d30-abba-4c2b-8462-66395ac063ba.4344.24:2" 2022-09-03T06:32:03.799 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:32:03.799 INFO:teuthology.orchestra.run.smithi087.stdout: 2022-09-03T06:32:03.801 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone2 2022-09-03T06:32:03.801 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:03.802 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:03.803 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:03.861 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:03.862 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:03.938 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:32:03.939 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:03.939 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006000", 2022-09-03T06:32:03.939 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.280943Z" 2022-09-03T06:32:03.939 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:03.940 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:03.940 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000007692", 2022-09-03T06:32:03.940 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.281950Z" 2022-09-03T06:32:03.940 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:03.941 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:03.941 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:03.941 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.283076Z" 2022-09-03T06:32:03.941 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:03.942 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:03.942 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:03.942 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.279922Z" 2022-09-03T06:32:03.942 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:03.943 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:32:03.943 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:03.944 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2022-09-03T06:32:03.944 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:03.945 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:03.946 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:04.001 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:04.002 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:04.046 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:32:04.047 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:32:04.047 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:32:04.047 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:32:04.048 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:32:04.048 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 16975738441461511297 2022-09-03T06:32:04.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:04.048 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:32:04.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:04.049 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:32:04.050 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:04.050 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:04.051 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000004740", 2022-09-03T06:32:04.052 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:04.052 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:04.053 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:04.053 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:24:02.298566Z" 2022-09-03T06:32:04.054 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:04.055 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:04.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:04.056 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:32:04.057 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:04.057 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:04.059 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006432", 2022-09-03T06:32:04.059 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:04.060 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:04.061 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:04.061 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:27:32.899702Z" 2022-09-03T06:32:04.061 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:04.062 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:04.062 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:04.063 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:32:04.063 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:04.064 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:04.064 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:32:04.064 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:04.065 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:04.066 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:04.066 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:23:42.293383Z" 2022-09-03T06:32:04.067 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:04.068 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:04.068 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:04.069 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:32:04.069 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:04.070 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:04.070 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:32:04.071 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:04.071 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:04.072 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:04.072 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:01:30.581084Z" 2022-09-03T06:32:04.073 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:04.073 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:04.075 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:32:04.075 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:04.076 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:32:04.077 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:32:04.077 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:32:04.078 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:04.078 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:32:04Z" 2022-09-03T06:32:04.079 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:32:04.081 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 16975738441461511297 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000004740", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:24:02.298566Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006432", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:27:32.899702Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:42.293383Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:01:30.581084Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:04Z" } 2022-09-03T06:32:04.082 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000004740', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:24:02.298566Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006432', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:27:32.899702Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:42.293383Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:01:30.581084Z'}}] 2022-09-03T06:32:04.083 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:04.084 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000004740', 1: '00000000000000000000:00000000000000006432', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:32:04.085 WARNING:rgw_multi.tests:data of zone test-zone2 behind zone test-zone1: shard=0 master=00000000000000000000:00000000000000006000 target=00000000000000000000:00000000000000004740, shard=1 master=00000000000000000000:00000000000000007692 target=00000000000000000000:00000000000000006432, shard=2 master=00000000000000000000:00000000000000002060 target=00000000000000000000:00000000000000000796, shard=3 master=00000000000000000000:00000000000000003327 target=00000000000000000000:00000000000000002486 2022-09-03T06:32:09.087 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.089 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.089 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:09.150 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:09.151 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:09.234 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:32:09.235 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:32:09.235 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:32:09.236 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:32:09.236 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:32:09.236 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 16975738441461511297 2022-09-03T06:32:09.237 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.237 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:32:09.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.238 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:32:09.239 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:09.240 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.240 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006000", 2022-09-03T06:32:09.241 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:09.241 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:09.242 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:09.242 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:02.280943Z" 2022-09-03T06:32:09.242 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.244 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:32:09.244 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:09.245 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.245 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000007692", 2022-09-03T06:32:09.245 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:09.246 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:09.247 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:09.247 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:02.281950Z" 2022-09-03T06:32:09.247 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.249 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:32:09.249 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:09.250 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.250 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:09.250 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:09.251 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:09.251 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:09.252 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:02.283076Z" 2022-09-03T06:32:09.252 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.253 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:32:09.253 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:09.253 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.254 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:09.254 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:09.254 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:09.254 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:09.255 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:02.279922Z" 2022-09-03T06:32:09.255 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.255 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.255 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:32:09.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.256 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:32:09.256 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:32:09.256 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:32:09.257 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.257 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:32:09Z" 2022-09-03T06:32:09.258 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:32:09.259 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 16975738441461511297 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006000", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.280943Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000007692", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.281950Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002060", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.283076Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000003327", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.279922Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:09Z" } 2022-09-03T06:32:09.260 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006000', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.280943Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000007692', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.281950Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002060', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.283076Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000003327', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.279922Z'}}] 2022-09-03T06:32:09.260 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:09.262 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:09.263 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2022-09-03T06:32:09.263 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone3 2022-09-03T06:32:09.264 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.264 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.265 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:09.334 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:09.335 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:09.416 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:32:09.417 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.417 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006000", 2022-09-03T06:32:09.418 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.280943Z" 2022-09-03T06:32:09.418 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.419 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.419 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000007692", 2022-09-03T06:32:09.420 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.281950Z" 2022-09-03T06:32:09.420 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.421 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.421 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:09.421 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.283076Z" 2022-09-03T06:32:09.422 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.422 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.423 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:09.423 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.279922Z" 2022-09-03T06:32:09.423 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:09.424 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:32:09.426 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:09.427 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2022-09-03T06:32:09.428 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.429 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.430 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:09.488 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:09.488 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:09.555 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:32:09.555 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:32:09.556 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:32:09.557 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:32:09.557 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:32:09.557 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 11727837677587591728 2022-09-03T06:32:09.558 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.558 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:32:09.558 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.559 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:32:09.559 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:09.559 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.560 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006000", 2022-09-03T06:32:09.560 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:09.560 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:09.560 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:09.561 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:02.280943Z" 2022-09-03T06:32:09.561 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:09.561 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.562 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.562 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:32:09.563 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:09.563 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.564 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000007692", 2022-09-03T06:32:09.564 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:09.565 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:09.565 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:09.565 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:02.281950Z" 2022-09-03T06:32:09.565 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:09.566 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.566 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.567 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:32:09.567 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:09.567 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.568 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:09.568 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:09.568 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:09.569 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:09.569 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:02.283076Z" 2022-09-03T06:32:09.569 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:09.570 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.570 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.570 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:32:09.570 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:09.571 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.571 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:09.571 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:09.572 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:09.573 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:09.573 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:02.279922Z" 2022-09-03T06:32:09.575 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:09.575 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:09.576 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:32:09.576 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.576 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:32:09.576 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:32:09.577 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:32:09.577 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.578 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:32:09Z" 2022-09-03T06:32:09.578 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:32:09.579 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 11727837677587591728 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006000", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.280943Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000007692", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.281950Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002060", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.283076Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000003327", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.279922Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:09Z" } 2022-09-03T06:32:09.579 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006000', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.280943Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000007692', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.281950Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002060', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.283076Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000003327', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.279922Z'}}] 2022-09-03T06:32:09.580 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:09.581 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:09.582 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2022-09-03T06:32:09.582 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone4 2022-09-03T06:32:09.583 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.584 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.585 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:09.640 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:09.641 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:09.694 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:32:09.694 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.695 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006000", 2022-09-03T06:32:09.695 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.280943Z" 2022-09-03T06:32:09.695 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.696 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.696 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000007692", 2022-09-03T06:32:09.696 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.281950Z" 2022-09-03T06:32:09.697 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.697 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.697 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:09.697 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.283076Z" 2022-09-03T06:32:09.702 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:09.702 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:09.702 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:09.703 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:02.279922Z" 2022-09-03T06:32:09.703 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:09.703 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:32:09.704 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:09.705 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2022-09-03T06:32:09.717 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.718 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.718 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:09.774 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:09.774 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:09.837 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:32:09.838 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:32:09.838 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:32:09.839 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:32:09.839 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:32:09.840 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 6281331864854677188 2022-09-03T06:32:09.843 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.844 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:32:09.846 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.847 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:32:09.847 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:09.847 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.848 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006000", 2022-09-03T06:32:09.848 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:09.849 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:09.849 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:09.850 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:02.280943Z" 2022-09-03T06:32:09.850 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.851 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.851 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.852 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:32:09.852 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:09.852 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.853 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000007692", 2022-09-03T06:32:09.853 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:09.854 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:09.855 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:09.855 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:02.281950Z" 2022-09-03T06:32:09.855 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.856 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.856 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.856 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:32:09.857 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:09.857 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.857 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:09.857 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:09.858 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:09.858 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:09.858 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:02.283076Z" 2022-09-03T06:32:09.858 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.859 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.859 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.859 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:32:09.859 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:09.860 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:09.860 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:09.860 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:09.861 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:09.861 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:09.861 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:02.279922Z" 2022-09-03T06:32:09.861 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.862 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.862 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:32:09.862 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.863 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:32:09.863 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:32:09.863 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:32:09.864 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.864 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:32:09Z" 2022-09-03T06:32:09.864 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:32:09.865 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 6281331864854677188 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006000", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.280943Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000007692", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.281950Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002060", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.283076Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000003327", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:02.279922Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:09Z" } 2022-09-03T06:32:09.865 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006000', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.280943Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000007692', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.281950Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002060', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.283076Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000003327', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:02.279922Z'}}] 2022-09-03T06:32:09.866 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:09.866 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000006000', 1: '00000000000000000000:00000000000000007692', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:09.867 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2022-09-03T06:32:09.867 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone1 2022-09-03T06:32:09.868 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.869 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.869 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:09.917 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:09.918 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:09.972 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:32:09.973 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.973 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006282", 2022-09-03T06:32:09.973 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:32:09.676575Z" 2022-09-03T06:32:09.974 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.974 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.974 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000007412", 2022-09-03T06:32:09.974 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:32:09.679452Z" 2022-09-03T06:32:09.975 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.975 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.975 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:09.976 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:32:09.682560Z" 2022-09-03T06:32:09.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:09.976 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:09.976 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:09.986 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:32:09.675548Z" 2022-09-03T06:32:09.986 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:09.987 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:32:09.988 DEBUG:rgw_multi.tests:data markers for zone=test-zone2 markers={0: '00000000000000000000:00000000000000006282', 1: '00000000000000000000:00000000000000007412', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:09.989 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2022-09-03T06:32:09.989 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.990 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:09.990 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:10.046 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:10.047 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:10.114 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:32:10.116 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:32:10.117 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:32:10.118 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:32:10.119 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:32:10.120 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 8557007329946676474 2022-09-03T06:32:10.120 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:10.121 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:32:10.121 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:10.122 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:32:10.123 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:10.123 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:10.123 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005022", 2022-09-03T06:32:10.124 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:10.124 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:10.125 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:10.125 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:44.218003Z" 2022-09-03T06:32:10.125 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:10.125 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:10.126 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:10.126 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:32:10.126 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:10.126 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:10.128 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006152", 2022-09-03T06:32:10.128 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:10.128 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:10.128 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:10.129 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:27:33.197639Z" 2022-09-03T06:32:10.129 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:10.130 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:10.130 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:10.130 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:32:10.131 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:10.131 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:10.131 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:32:10.131 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:10.132 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:10.132 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:10.132 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:24.194635Z" 2022-09-03T06:32:10.133 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:10.133 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:10.133 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:10.133 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:32:10.134 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:10.134 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:10.134 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:32:10.134 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:10.135 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:10.135 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:10.135 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:32:10.135 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:10.136 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:10.136 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:32:10.136 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:10.136 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:32:10.136 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:32:10.137 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:32:10.137 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:10.137 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:32:10Z" 2022-09-03T06:32:10.138 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:32:10.139 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 8557007329946676474 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005022", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:44.218003Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006152", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:27:33.197639Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:24.194635Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:59:59.312039Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:10Z" } 2022-09-03T06:32:10.140 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005022', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:44.218003Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006152', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:27:33.197639Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:24.194635Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:59:59.312039Z'}}] 2022-09-03T06:32:10.141 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000006282', 1: '00000000000000000000:00000000000000007412', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:10.141 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000005022', 1: '00000000000000000000:00000000000000006152', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:32:10.142 WARNING:rgw_multi.tests:data of zone test-zone1 behind zone test-zone2: shard=0 master=00000000000000000000:00000000000000006282 target=00000000000000000000:00000000000000005022, shard=1 master=00000000000000000000:00000000000000007412 target=00000000000000000000:00000000000000006152, shard=2 master=00000000000000000000:00000000000000002060 target=00000000000000000000:00000000000000000796, shard=3 master=00000000000000000000:00000000000000003327 target=00000000000000000000:00000000000000002486 2022-09-03T06:32:15.144 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:15.145 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:15.146 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:15.207 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:15.208 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:15.274 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:32:15.274 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:32:15.275 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:32:15.275 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:32:15.275 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:32:15.275 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 8557007329946676474 2022-09-03T06:32:15.276 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:15.276 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:32:15.276 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:15.277 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:32:15.277 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:15.277 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:15.277 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005022", 2022-09-03T06:32:15.278 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:15.287 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:15.288 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:15.288 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:44.218003Z" 2022-09-03T06:32:15.289 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:15.289 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:15.290 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:15.290 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:32:15.290 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:15.290 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:15.291 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006152", 2022-09-03T06:32:15.291 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:15.291 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:15.292 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:15.292 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:27:33.197639Z" 2022-09-03T06:32:15.292 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:15.292 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:15.293 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:15.293 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:32:15.293 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:15.293 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:15.294 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:32:15.294 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:15.294 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:15.294 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:15.295 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:24.194635Z" 2022-09-03T06:32:15.295 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:15.295 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:15.295 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:15.296 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:32:15.296 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:15.296 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:15.296 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:32:15.297 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:15.297 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:15.297 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:15.297 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:32:15.298 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:15.298 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:15.298 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:32:15.298 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:15.299 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:32:15.299 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:32:15.299 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:32:15.300 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:15.300 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:32:15Z" 2022-09-03T06:32:15.300 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:32:15.300 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 8557007329946676474 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005022", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:44.218003Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006152", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:27:33.197639Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:24.194635Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:59:59.312039Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:15Z" } 2022-09-03T06:32:15.301 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005022', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:44.218003Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006152', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:27:33.197639Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:24.194635Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:59:59.312039Z'}}] 2022-09-03T06:32:15.302 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000006282', 1: '00000000000000000000:00000000000000007412', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:15.302 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000005022', 1: '00000000000000000000:00000000000000006152', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:32:15.303 WARNING:rgw_multi.tests:data of zone test-zone1 behind zone test-zone2: shard=0 master=00000000000000000000:00000000000000006282 target=00000000000000000000:00000000000000005022, shard=1 master=00000000000000000000:00000000000000007412 target=00000000000000000000:00000000000000006152, shard=2 master=00000000000000000000:00000000000000002060 target=00000000000000000000:00000000000000000796, shard=3 master=00000000000000000000:00000000000000003327 target=00000000000000000000:00000000000000002486 2022-09-03T06:32:20.304 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:20.306 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:20.309 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:20.373 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:20.373 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:20.430 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:32:20.431 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:32:20.431 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:32:20.432 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:32:20.432 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:32:20.433 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 8557007329946676474 2022-09-03T06:32:20.433 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:20.434 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:32:20.434 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:20.435 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:32:20.438 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:20.439 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:20.440 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005022", 2022-09-03T06:32:20.440 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:20.441 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:20.441 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:20.441 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:44.218003Z" 2022-09-03T06:32:20.442 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:20.443 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:20.443 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:20.444 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:32:20.444 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:20.445 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:20.445 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006152", 2022-09-03T06:32:20.446 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:20.446 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:20.447 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:20.447 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:27:33.197639Z" 2022-09-03T06:32:20.448 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:20.448 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:20.449 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:20.449 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:32:20.450 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:20.450 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:20.451 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000000796", 2022-09-03T06:32:20.452 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:20.454 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:20.455 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:20.455 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:23:24.194635Z" 2022-09-03T06:32:20.456 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:20.456 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:20.457 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:20.457 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:32:20.458 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:20.458 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:20.459 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002486", 2022-09-03T06:32:20.459 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:20.460 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:20.460 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:20.461 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:59:59.312039Z" 2022-09-03T06:32:20.461 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:20.462 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:20.462 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:32:20.463 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:20.463 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:32:20.464 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:32:20.464 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:32:20.465 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:20.465 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:32:20Z" 2022-09-03T06:32:20.466 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:32:20.467 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 8557007329946676474 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005022", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:44.218003Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006152", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:27:33.197639Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000000796", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:23:24.194635Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002486", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T05:59:59.312039Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:20Z" } 2022-09-03T06:32:20.468 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005022', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:44.218003Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006152', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:27:33.197639Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000000796', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:23:24.194635Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002486', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T05:59:59.312039Z'}}] 2022-09-03T06:32:20.469 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000006282', 1: '00000000000000000000:00000000000000007412', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:20.470 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000005022', 1: '00000000000000000000:00000000000000006152', 2: '00000000000000000000:00000000000000000796', 3: '00000000000000000000:00000000000000002486'} 2022-09-03T06:32:20.471 WARNING:rgw_multi.tests:data of zone test-zone1 behind zone test-zone2: shard=0 master=00000000000000000000:00000000000000006282 target=00000000000000000000:00000000000000005022, shard=1 master=00000000000000000000:00000000000000007412 target=00000000000000000000:00000000000000006152, shard=2 master=00000000000000000000:00000000000000002060 target=00000000000000000000:00000000000000000796, shard=3 master=00000000000000000000:00000000000000003327 target=00000000000000000000:00000000000000002486 2022-09-03T06:32:25.473 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:25.474 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:25.475 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:25.533 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:25.533 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:25.594 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:32:25.594 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:32:25.595 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:32:25.595 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:32:25.596 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:32:25.596 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 8557007329946676474 2022-09-03T06:32:25.597 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.597 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:32:25.597 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:25.598 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:32:25.599 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:25.599 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:25.600 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006282", 2022-09-03T06:32:25.600 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:25.600 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:25.600 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:25.601 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.676575Z" 2022-09-03T06:32:25.601 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.602 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.602 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:25.602 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:32:25.603 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:25.603 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:25.603 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000007412", 2022-09-03T06:32:25.604 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:25.604 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:25.605 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:25.605 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.679452Z" 2022-09-03T06:32:25.606 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.606 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.607 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:25.607 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:32:25.608 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:25.608 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:25.609 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:25.610 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:25.610 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:25.611 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:25.612 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.682560Z" 2022-09-03T06:32:25.613 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.613 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.614 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:25.614 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:32:25.615 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:25.615 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:25.616 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:25.616 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:25.617 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:25.617 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:25.618 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.675548Z" 2022-09-03T06:32:25.618 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.619 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.619 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:32:25.620 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.620 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:32:25.622 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:32:25.623 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:32:25.623 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.623 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:32:25Z" 2022-09-03T06:32:25.624 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:32:25.631 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 8557007329946676474 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006282", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.676575Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000007412", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.679452Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002060", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.682560Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000003327", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.675548Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:25Z" } 2022-09-03T06:32:25.632 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006282', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.676575Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000007412', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.679452Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002060', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.682560Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000003327', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.675548Z'}}] 2022-09-03T06:32:25.632 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000006282', 1: '00000000000000000000:00000000000000007412', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:25.633 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000006282', 1: '00000000000000000000:00000000000000007412', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:25.633 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2022-09-03T06:32:25.634 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone3 2022-09-03T06:32:25.635 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:25.635 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:25.636 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:25.690 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:25.691 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:25.755 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T06:32:25.756 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:25.756 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006282", 2022-09-03T06:32:25.757 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:32:09.676575Z" 2022-09-03T06:32:25.757 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:25.758 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:25.758 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000007412", 2022-09-03T06:32:25.759 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:32:09.679452Z" 2022-09-03T06:32:25.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:25.760 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:25.760 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:25.761 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:32:09.682560Z" 2022-09-03T06:32:25.762 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:25.762 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:25.763 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:25.763 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T06:32:09.675548Z" 2022-09-03T06:32:25.765 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:25.765 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T06:32:25.767 DEBUG:rgw_multi.tests:data markers for zone=test-zone2 markers={0: '00000000000000000000:00000000000000006282', 1: '00000000000000000000:00000000000000007412', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:25.768 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2022-09-03T06:32:25.769 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:25.773 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:25.773 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:25.831 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:25.832 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:25.886 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:32:25.887 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:32:25.888 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:32:25.888 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:32:25.889 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:32:25.889 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 14782160398262406281 2022-09-03T06:32:25.889 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.889 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:32:25.890 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:25.891 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:32:25.891 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:25.891 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:25.891 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006282", 2022-09-03T06:32:25.892 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:25.892 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:25.892 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:25.892 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.676575Z" 2022-09-03T06:32:25.893 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.893 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.893 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:25.894 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:32:25.895 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:25.895 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:25.895 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000007412", 2022-09-03T06:32:25.895 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:25.896 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:25.896 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:25.898 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.679452Z" 2022-09-03T06:32:25.898 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.902 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.902 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:25.903 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:32:25.903 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:25.903 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:25.904 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:25.904 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:25.904 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:25.904 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:25.905 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.682560Z" 2022-09-03T06:32:25.905 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.905 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.905 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:25.906 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:32:25.906 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:25.907 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:25.907 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:25.907 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:25.907 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:25.908 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:25.908 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.675548Z" 2022-09-03T06:32:25.908 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.908 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:25.909 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:32:25.909 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.909 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:32:25.909 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:32:25.909 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:32:25.910 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:25.910 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:32:25Z" 2022-09-03T06:32:25.910 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:32:25.912 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 14782160398262406281 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006282", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.676575Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000007412", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.679452Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002060", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.682560Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000003327", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.675548Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:25Z" } 2022-09-03T06:32:25.913 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006282', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.676575Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000007412', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.679452Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002060', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.682560Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000003327', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.675548Z'}}] 2022-09-03T06:32:25.914 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000006282', 1: '00000000000000000000:00000000000000007412', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:25.914 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000006282', 1: '00000000000000000000:00000000000000007412', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:25.915 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2022-09-03T06:32:25.915 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone4 2022-09-03T06:32:25.916 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone1 2022-09-03T06:32:25.916 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:25.917 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:25.918 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:25.975 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:25.976 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:26.059 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:32:26.062 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.062 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005858", 2022-09-03T06:32:26.063 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:09.451269Z" 2022-09-03T06:32:26.063 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.063 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.064 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000007130", 2022-09-03T06:32:26.064 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:09.452734Z" 2022-09-03T06:32:26.064 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.064 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.065 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:26.065 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:09.457726Z" 2022-09-03T06:32:26.066 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.066 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.066 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:26.066 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:09.446150Z" 2022-09-03T06:32:26.067 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:26.067 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:32:26.067 DEBUG:rgw_multi.tests:data markers for zone=test-zone3 markers={0: '00000000000000000000:00000000000000005858', 1: '00000000000000000000:00000000000000007130', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:26.068 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2022-09-03T06:32:26.069 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:26.069 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:26.070 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:26.137 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:26.137 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:26.182 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:32:26.182 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:32:26.183 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:32:26.183 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:32:26.184 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:32:26.185 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 13955113675122108513 2022-09-03T06:32:26.185 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.185 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:32:26.186 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.187 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:32:26.187 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:26.187 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:26.188 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005858", 2022-09-03T06:32:26.189 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:26.189 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:26.189 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:26.190 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.451269Z" 2022-09-03T06:32:26.190 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:26.190 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.192 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.192 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:32:26.193 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:26.193 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:26.194 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000007130", 2022-09-03T06:32:26.194 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:26.194 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:26.194 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:26.195 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.452734Z" 2022-09-03T06:32:26.195 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:26.195 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.195 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.196 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:32:26.196 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:26.196 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:26.196 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:26.197 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:26.197 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:26.197 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:26.198 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.457726Z" 2022-09-03T06:32:26.198 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:26.199 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.199 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.199 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:32:26.200 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:26.200 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T06:32:26.201 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:26.201 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:26.204 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:26.204 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:26.205 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:09.446150Z" 2022-09-03T06:32:26.205 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:26.206 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:26.206 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:32:26.207 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.207 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:32:26.208 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T06:32:26.208 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T06:32:26.209 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.209 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:32:26Z" 2022-09-03T06:32:26.212 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:32:26.215 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 13955113675122108513 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005858", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.451269Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000007130", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.452734Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002060", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.457726Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000003327", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.446150Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:26Z" } 2022-09-03T06:32:26.217 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005858', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.451269Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000007130', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.452734Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002060', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.457726Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000003327', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.446150Z'}}] 2022-09-03T06:32:26.218 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000005858', 1: '00000000000000000000:00000000000000007130', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:26.219 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000005858', 1: '00000000000000000000:00000000000000007130', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:26.220 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2022-09-03T06:32:26.222 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone2 2022-09-03T06:32:26.223 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:26.223 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:26.225 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:26.277 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:26.277 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:26.345 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:32:26.345 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.346 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005858", 2022-09-03T06:32:26.346 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:09.451269Z" 2022-09-03T06:32:26.347 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.347 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.348 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000007130", 2022-09-03T06:32:26.348 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:09.452734Z" 2022-09-03T06:32:26.349 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.349 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.350 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:26.351 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:09.457726Z" 2022-09-03T06:32:26.352 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:26.352 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:26.353 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:26.353 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:09.446150Z" 2022-09-03T06:32:26.354 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:26.354 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:32:26.355 DEBUG:rgw_multi.tests:data markers for zone=test-zone3 markers={0: '00000000000000000000:00000000000000005858', 1: '00000000000000000000:00000000000000007130', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:26.356 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2022-09-03T06:32:26.357 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:26.358 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:26.360 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:26.415 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:26.415 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:26.461 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:32:26.461 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:32:26.461 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:32:26.462 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:32:26.462 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:32:26.462 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 16273219177139511486 2022-09-03T06:32:26.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:26.463 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:32:26.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:26.463 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:32:26.464 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:26.464 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:26.464 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000005858", 2022-09-03T06:32:26.464 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:26.465 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:26.465 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:26.466 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:09.451269Z" 2022-09-03T06:32:26.466 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:26.466 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:26.466 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:26.467 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:32:26.467 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:26.467 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:26.468 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000007130", 2022-09-03T06:32:26.468 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:26.468 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:26.468 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:26.469 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:09.452734Z" 2022-09-03T06:32:26.469 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:26.469 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:26.470 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:26.470 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:32:26.470 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:26.470 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:26.471 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000002060", 2022-09-03T06:32:26.471 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:26.471 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:26.471 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:26.472 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:09.457726Z" 2022-09-03T06:32:26.472 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:26.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:26.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:26.473 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:32:26.473 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:26.474 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T06:32:26.474 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000003327", 2022-09-03T06:32:26.475 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:26.475 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:26.476 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:26.476 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:09.446150Z" 2022-09-03T06:32:26.476 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:26.476 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:26.477 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:32:26.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:26.477 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:32:26.478 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T06:32:26.478 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T06:32:26.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:26.479 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:32:26Z" 2022-09-03T06:32:26.479 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:32:26.479 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 16273219177139511486 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005858", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.451269Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000007130", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.452734Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000002060", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.457726Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000003327", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:09.446150Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T06:32:26Z" } 2022-09-03T06:32:26.480 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005858', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.451269Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000007130', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.452734Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000002060', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.457726Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000003327', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:09.446150Z'}}] 2022-09-03T06:32:26.481 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000005858', 1: '00000000000000000000:00000000000000007130', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:26.481 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000005858', 1: '00000000000000000000:00000000000000007130', 2: '00000000000000000000:00000000000000002060', 3: '00000000000000000000:00000000000000003327'} 2022-09-03T06:32:26.482 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2022-09-03T06:32:26.482 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone4 2022-09-03T06:32:26.483 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_sync_disable ... ok 2022-09-03T06:32:26.486 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-59 2022-09-03T06:32:26.487 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:32:26.488 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:32:26.489 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.490 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:32:26.490 DEBUG:boto:Data: 2022-09-03T06:32:26.491 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:32:26.492 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.493 DEBUG:boto:Params: {} 2022-09-03T06:32:26.494 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:32:26.495 DEBUG:boto:Token: None 2022-09-03T06:32:26.496 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:32:26 GMT /avysmw-59/ 2022-09-03T06:32:26.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AblLERVG8NtQ/8LOexcBKb5rQ7k= 2022-09-03T06:32:26.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AblLERVG8NtQ/8LOexcBKb5rQ7k=', 'Content-Length': '0'} 2022-09-03T06:32:26.528 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000002d7b3d93b622f915-006312f4fa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.529 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-60 2022-09-03T06:32:26.529 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:32:26.530 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:32:26.530 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.531 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:32:26.531 DEBUG:boto:Data: 2022-09-03T06:32:26.534 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.536 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:32:26.536 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.537 DEBUG:boto:Params: {} 2022-09-03T06:32:26.538 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:32:26.539 DEBUG:boto:Token: None 2022-09-03T06:32:26.539 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:32:26 GMT /avysmw-60/ 2022-09-03T06:32:26.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xZ6ecT2XGAQeys6hieFKMQ4E4pc= 2022-09-03T06:32:26.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xZ6ecT2XGAQeys6hieFKMQ4E4pc=', 'Content-Length': '0'} 2022-09-03T06:32:26.577 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000356fba161e85c677-006312f4fa-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.578 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-61 2022-09-03T06:32:26.579 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:32:26.579 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:32:26.580 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.581 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:32:26.581 DEBUG:boto:Data: 2022-09-03T06:32:26.582 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:32:26.583 DEBUG:boto:Port: 8001 2022-09-03T06:32:26.583 DEBUG:boto:Params: {} 2022-09-03T06:32:26.584 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:32:26.585 DEBUG:boto:Token: None 2022-09-03T06:32:26.586 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:32:26 GMT /avysmw-61/ 2022-09-03T06:32:26.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8gIlRL5nkNtkGtIuP9cN9CsZ6ns= 2022-09-03T06:32:26.588 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8gIlRL5nkNtkGtIuP9cN9CsZ6ns=', 'Content-Length': '0'} 2022-09-03T06:32:26.634 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000080c2b338da0214a-006312f4fa-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.635 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:32:26.636 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:32:26.636 DEBUG:boto:Method: HEAD 2022-09-03T06:32:26.637 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:32:26.637 DEBUG:boto:Data: 2022-09-03T06:32:26.637 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.638 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:32:26.639 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.639 DEBUG:boto:Params: {} 2022-09-03T06:32:26.640 DEBUG:boto:Token: None 2022-09-03T06:32:26.641 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:26 GMT /avysmw-59/ 2022-09-03T06:32:26.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZPYettFl9nUSu11ixVJlyU22vw= 2022-09-03T06:32:26.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZPYettFl9nUSu11ixVJlyU22vw=', 'Content-Length': '0'} 2022-09-03T06:32:26.647 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009403ce61bbb831b2-006312f4fa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.651 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:32:26.652 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:32:26.652 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.653 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:32:26.653 DEBUG:boto:Data: 2022-09-03T06:32:26.655 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:26.656 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:32:26.657 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.657 DEBUG:boto:Params: {} 2022-09-03T06:32:26.659 DEBUG:boto:Token: None 2022-09-03T06:32:26.660 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:26 GMT /avysmw-59/obj1 2022-09-03T06:32:26.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oHk2xeNYymcTxjvVlrlDqSXx14U= 2022-09-03T06:32:26.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oHk2xeNYymcTxjvVlrlDqSXx14U='} 2022-09-03T06:32:26.675 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000869589f4ecd49569-006312f4fa-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.676 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:32:26.677 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:32:26.679 DEBUG:boto:Method: HEAD 2022-09-03T06:32:26.679 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:32:26.680 DEBUG:boto:Data: 2022-09-03T06:32:26.680 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.681 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:32:26.681 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.682 DEBUG:boto:Params: {} 2022-09-03T06:32:26.683 DEBUG:boto:Token: None 2022-09-03T06:32:26.683 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:26 GMT /avysmw-59/ 2022-09-03T06:32:26.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZPYettFl9nUSu11ixVJlyU22vw= 2022-09-03T06:32:26.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZPYettFl9nUSu11ixVJlyU22vw=', 'Content-Length': '0'} 2022-09-03T06:32:26.687 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ef8422aed9f55b67-006312f4fa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.688 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:32:26.688 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:32:26.689 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.689 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:32:26.690 DEBUG:boto:Data: 2022-09-03T06:32:26.690 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:26.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:32:26.691 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.692 DEBUG:boto:Params: {} 2022-09-03T06:32:26.692 DEBUG:boto:Token: None 2022-09-03T06:32:26.693 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:26 GMT /avysmw-59/obj2 2022-09-03T06:32:26.693 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k8QA6TKzTarFfyhYzZbgcwLGM1I= 2022-09-03T06:32:26.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k8QA6TKzTarFfyhYzZbgcwLGM1I='} 2022-09-03T06:32:26.739 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000db5abb68d62458d7-006312f4fa-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.740 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:32:26.740 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:32:26.741 DEBUG:boto:Method: HEAD 2022-09-03T06:32:26.741 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:32:26.743 DEBUG:boto:Data: 2022-09-03T06:32:26.747 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:32:26.749 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.749 DEBUG:boto:Params: {} 2022-09-03T06:32:26.750 DEBUG:boto:Token: None 2022-09-03T06:32:26.751 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:26 GMT /avysmw-59/ 2022-09-03T06:32:26.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZPYettFl9nUSu11ixVJlyU22vw= 2022-09-03T06:32:26.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZPYettFl9nUSu11ixVJlyU22vw=', 'Content-Length': '0'} 2022-09-03T06:32:26.757 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009587802a8fc330b4-006312f4fa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.758 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:32:26.759 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:32:26.760 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.760 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:32:26.761 DEBUG:boto:Data: 2022-09-03T06:32:26.762 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:26.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:32:26.763 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.764 DEBUG:boto:Params: {} 2022-09-03T06:32:26.765 DEBUG:boto:Token: None 2022-09-03T06:32:26.766 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:26 GMT /avysmw-59/obj3 2022-09-03T06:32:26.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LEd8aNT7qiHTuQVDLS12E34MRMo= 2022-09-03T06:32:26.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LEd8aNT7qiHTuQVDLS12E34MRMo='} 2022-09-03T06:32:26.776 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f7c9d4ebe8502085-006312f4fa-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.777 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:32:26.778 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:32:26.779 DEBUG:boto:Method: HEAD 2022-09-03T06:32:26.780 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:32:26.780 DEBUG:boto:Data: 2022-09-03T06:32:26.781 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:32:26.784 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.785 DEBUG:boto:Params: {} 2022-09-03T06:32:26.786 DEBUG:boto:Token: None 2022-09-03T06:32:26.786 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:26 GMT /avysmw-59/ 2022-09-03T06:32:26.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZPYettFl9nUSu11ixVJlyU22vw= 2022-09-03T06:32:26.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZPYettFl9nUSu11ixVJlyU22vw=', 'Content-Length': '0'} 2022-09-03T06:32:26.790 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ed1be7bcd1aa78fb-006312f4fa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.791 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:32:26.792 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:32:26.792 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.793 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:32:26.794 DEBUG:boto:Data: 2022-09-03T06:32:26.794 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:26.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:32:26.795 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.796 DEBUG:boto:Params: {} 2022-09-03T06:32:26.796 DEBUG:boto:Token: None 2022-09-03T06:32:26.797 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:26 GMT /avysmw-59/obj4 2022-09-03T06:32:26.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GPBMKvafW9wAKitz+dcRH/nRU/c= 2022-09-03T06:32:26.798 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GPBMKvafW9wAKitz+dcRH/nRU/c='} 2022-09-03T06:32:26.842 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000008b5fbd9747b36d26-006312f4fa-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.843 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:32:26.844 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:32:26.844 DEBUG:boto:Method: HEAD 2022-09-03T06:32:26.845 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:32:26.846 DEBUG:boto:Data: 2022-09-03T06:32:26.846 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.847 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:32:26.848 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.849 DEBUG:boto:Params: {} 2022-09-03T06:32:26.850 DEBUG:boto:Token: None 2022-09-03T06:32:26.851 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:26 GMT /avysmw-60/ 2022-09-03T06:32:26.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pFQmkN49k9X/72zETAIy4/gmWh8= 2022-09-03T06:32:26.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pFQmkN49k9X/72zETAIy4/gmWh8=', 'Content-Length': '0'} 2022-09-03T06:32:26.856 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000082c9b3c1054c0a7c-006312f4fa-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.859 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:32:26.860 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:32:26.861 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.862 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:32:26.863 DEBUG:boto:Data: 2022-09-03T06:32:26.863 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:26.864 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:32:26.865 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.865 DEBUG:boto:Params: {} 2022-09-03T06:32:26.866 DEBUG:boto:Token: None 2022-09-03T06:32:26.867 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:26 GMT /avysmw-60/obj1 2022-09-03T06:32:26.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eHShgRQaQ9MSxSVSvxe+vR3ijVQ= 2022-09-03T06:32:26.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eHShgRQaQ9MSxSVSvxe+vR3ijVQ='} 2022-09-03T06:32:26.881 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f4535083c24c9ee8-006312f4fa-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.882 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:32:26.883 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:32:26.883 DEBUG:boto:Method: HEAD 2022-09-03T06:32:26.884 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:32:26.884 DEBUG:boto:Data: 2022-09-03T06:32:26.886 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.887 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:32:26.887 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.888 DEBUG:boto:Params: {} 2022-09-03T06:32:26.889 DEBUG:boto:Token: None 2022-09-03T06:32:26.891 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:26 GMT /avysmw-60/ 2022-09-03T06:32:26.892 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pFQmkN49k9X/72zETAIy4/gmWh8= 2022-09-03T06:32:26.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pFQmkN49k9X/72zETAIy4/gmWh8=', 'Content-Length': '0'} 2022-09-03T06:32:26.895 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d207434fed8ecd0c-006312f4fa-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.896 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:32:26.897 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:32:26.897 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.898 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:32:26.898 DEBUG:boto:Data: 2022-09-03T06:32:26.899 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:26.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:32:26.900 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.901 DEBUG:boto:Params: {} 2022-09-03T06:32:26.902 DEBUG:boto:Token: None 2022-09-03T06:32:26.903 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:26 GMT /avysmw-60/obj2 2022-09-03T06:32:26.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oB9vl6qetS9b4qZzgyBTneIwIX4= 2022-09-03T06:32:26.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oB9vl6qetS9b4qZzgyBTneIwIX4='} 2022-09-03T06:32:26.952 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000d057c373a2660014-006312f4fa-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.953 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:32:26.954 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:32:26.954 DEBUG:boto:Method: HEAD 2022-09-03T06:32:26.955 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:32:26.955 DEBUG:boto:Data: 2022-09-03T06:32:26.956 DEBUG:boto:Headers: {} 2022-09-03T06:32:26.957 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:32:26.957 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.958 DEBUG:boto:Params: {} 2022-09-03T06:32:26.959 DEBUG:boto:Token: None 2022-09-03T06:32:26.959 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:26 GMT /avysmw-60/ 2022-09-03T06:32:26.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pFQmkN49k9X/72zETAIy4/gmWh8= 2022-09-03T06:32:26.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pFQmkN49k9X/72zETAIy4/gmWh8=', 'Content-Length': '0'} 2022-09-03T06:32:26.965 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000efa164ecdb530d9-006312f4fa-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.967 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:32:26.968 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:32:26.969 DEBUG:boto:Method: PUT 2022-09-03T06:32:26.970 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:32:26.971 DEBUG:boto:Data: 2022-09-03T06:32:26.972 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:26.973 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:32:26.974 DEBUG:boto:Port: 8000 2022-09-03T06:32:26.974 DEBUG:boto:Params: {} 2022-09-03T06:32:26.978 DEBUG:boto:Token: None 2022-09-03T06:32:26.978 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:26 GMT /avysmw-60/obj3 2022-09-03T06:32:26.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmQW33ZgWZ9jCYGOGFLFm9sIuMU= 2022-09-03T06:32:26.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:26 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmQW33ZgWZ9jCYGOGFLFm9sIuMU='} 2022-09-03T06:32:26.992 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000173a87f2aa7cfeb8-006312f4fa-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:26.993 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:32:26.995 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:32:26.996 DEBUG:boto:Method: HEAD 2022-09-03T06:32:26.997 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:32:26.998 DEBUG:boto:Data: 2022-09-03T06:32:26.998 DEBUG:boto:Headers: {} 2022-09-03T06:32:27.000 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:32:27.008 DEBUG:boto:Port: 8000 2022-09-03T06:32:27.011 DEBUG:boto:Params: {} 2022-09-03T06:32:27.012 DEBUG:boto:Token: None 2022-09-03T06:32:27.013 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:27 GMT /avysmw-60/ 2022-09-03T06:32:27.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjnqXFZuQlyqiQV8nP7aXRXcg3s= 2022-09-03T06:32:27.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjnqXFZuQlyqiQV8nP7aXRXcg3s=', 'Content-Length': '0'} 2022-09-03T06:32:27.018 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000122af6393c6aed68-006312f4fb-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.020 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:32:27.022 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:32:27.024 DEBUG:boto:Method: PUT 2022-09-03T06:32:27.025 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:32:27.028 DEBUG:boto:Data: 2022-09-03T06:32:27.029 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:27.031 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:32:27.032 DEBUG:boto:Port: 8000 2022-09-03T06:32:27.033 DEBUG:boto:Params: {} 2022-09-03T06:32:27.035 DEBUG:boto:Token: None 2022-09-03T06:32:27.036 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:27 GMT /avysmw-60/obj4 2022-09-03T06:32:27.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fsI1EpcGacF8Tm8ZxN2Mvc8+VNA= 2022-09-03T06:32:27.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fsI1EpcGacF8Tm8ZxN2Mvc8+VNA='} 2022-09-03T06:32:27.051 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000005c491bac164d6f9-006312f4fb-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.052 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:32:27.054 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:32:27.055 DEBUG:boto:Method: HEAD 2022-09-03T06:32:27.056 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:32:27.057 DEBUG:boto:Data: 2022-09-03T06:32:27.061 DEBUG:boto:Headers: {} 2022-09-03T06:32:27.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:32:27.063 DEBUG:boto:Port: 8001 2022-09-03T06:32:27.063 DEBUG:boto:Params: {} 2022-09-03T06:32:27.064 DEBUG:boto:Token: None 2022-09-03T06:32:27.064 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:27 GMT /avysmw-61/ 2022-09-03T06:32:27.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FDm/6nTUq44/CRU/ICwH/Us1OJA= 2022-09-03T06:32:27.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FDm/6nTUq44/CRU/ICwH/Us1OJA=', 'Content-Length': '0'} 2022-09-03T06:32:27.074 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005ed8c04e55112a13-006312f4fb-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.076 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:32:27.076 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:32:27.077 DEBUG:boto:Method: PUT 2022-09-03T06:32:27.077 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:32:27.085 DEBUG:boto:Data: 2022-09-03T06:32:27.086 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:27.086 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:32:27.087 DEBUG:boto:Port: 8001 2022-09-03T06:32:27.087 DEBUG:boto:Params: {} 2022-09-03T06:32:27.088 DEBUG:boto:Token: None 2022-09-03T06:32:27.088 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:27 GMT /avysmw-61/obj1 2022-09-03T06:32:27.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qotc5N0jrsh5Sk7PHNcA8dtIbtw= 2022-09-03T06:32:27.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qotc5N0jrsh5Sk7PHNcA8dtIbtw='} 2022-09-03T06:32:27.098 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000cb00e1c2a1cb2a70-006312f4fb-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.099 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:32:27.100 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:32:27.100 DEBUG:boto:Method: HEAD 2022-09-03T06:32:27.101 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:32:27.101 DEBUG:boto:Data: 2022-09-03T06:32:27.102 DEBUG:boto:Headers: {} 2022-09-03T06:32:27.102 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:32:27.103 DEBUG:boto:Port: 8001 2022-09-03T06:32:27.104 DEBUG:boto:Params: {} 2022-09-03T06:32:27.105 DEBUG:boto:Token: None 2022-09-03T06:32:27.106 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:27 GMT /avysmw-61/ 2022-09-03T06:32:27.106 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FDm/6nTUq44/CRU/ICwH/Us1OJA= 2022-09-03T06:32:27.107 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FDm/6nTUq44/CRU/ICwH/Us1OJA=', 'Content-Length': '0'} 2022-09-03T06:32:27.110 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000062d34e66b23861d-006312f4fb-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.111 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:32:27.111 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:32:27.112 DEBUG:boto:Method: PUT 2022-09-03T06:32:27.112 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:32:27.113 DEBUG:boto:Data: 2022-09-03T06:32:27.113 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:27.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:32:27.114 DEBUG:boto:Port: 8001 2022-09-03T06:32:27.115 DEBUG:boto:Params: {} 2022-09-03T06:32:27.115 DEBUG:boto:Token: None 2022-09-03T06:32:27.116 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:27 GMT /avysmw-61/obj2 2022-09-03T06:32:27.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VmR9HHsPa7OYfkc5Fl0FAKPdqIQ= 2022-09-03T06:32:27.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VmR9HHsPa7OYfkc5Fl0FAKPdqIQ='} 2022-09-03T06:32:27.125 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000fb99ef640003331e-006312f4fb-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.127 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:32:27.127 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:32:27.128 DEBUG:boto:Method: HEAD 2022-09-03T06:32:27.128 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:32:27.129 DEBUG:boto:Data: 2022-09-03T06:32:27.130 DEBUG:boto:Headers: {} 2022-09-03T06:32:27.130 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:32:27.131 DEBUG:boto:Port: 8001 2022-09-03T06:32:27.131 DEBUG:boto:Params: {} 2022-09-03T06:32:27.260 DEBUG:boto:Token: None 2022-09-03T06:32:27.261 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:27 GMT /avysmw-61/ 2022-09-03T06:32:27.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FDm/6nTUq44/CRU/ICwH/Us1OJA= 2022-09-03T06:32:27.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FDm/6nTUq44/CRU/ICwH/Us1OJA=', 'Content-Length': '0'} 2022-09-03T06:32:27.269 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001e883da38aed1e60-006312f4fb-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.271 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:32:27.272 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:32:27.274 DEBUG:boto:Method: PUT 2022-09-03T06:32:27.275 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:32:27.276 DEBUG:boto:Data: 2022-09-03T06:32:27.277 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:27.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:32:27.293 DEBUG:boto:Port: 8001 2022-09-03T06:32:27.294 DEBUG:boto:Params: {} 2022-09-03T06:32:27.296 DEBUG:boto:Token: None 2022-09-03T06:32:27.297 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:27 GMT /avysmw-61/obj3 2022-09-03T06:32:27.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9amcrOSNJgxA4wKvf1KdINTvQSs= 2022-09-03T06:32:27.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9amcrOSNJgxA4wKvf1KdINTvQSs='} 2022-09-03T06:32:27.322 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000007ffd78b9340d9da6-006312f4fb-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.327 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:32:27.328 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:32:27.328 DEBUG:boto:Method: HEAD 2022-09-03T06:32:27.329 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:32:27.329 DEBUG:boto:Data: 2022-09-03T06:32:27.330 DEBUG:boto:Headers: {} 2022-09-03T06:32:27.331 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:32:27.331 DEBUG:boto:Port: 8001 2022-09-03T06:32:27.332 DEBUG:boto:Params: {} 2022-09-03T06:32:27.333 DEBUG:boto:Token: None 2022-09-03T06:32:27.333 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:32:27 GMT /avysmw-61/ 2022-09-03T06:32:27.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FDm/6nTUq44/CRU/ICwH/Us1OJA= 2022-09-03T06:32:27.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FDm/6nTUq44/CRU/ICwH/Us1OJA=', 'Content-Length': '0'} 2022-09-03T06:32:27.340 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d109bdf1e90724dc-006312f4fb-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.342 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:32:27.343 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:32:27.344 DEBUG:boto:Method: PUT 2022-09-03T06:32:27.344 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:32:27.345 DEBUG:boto:Data: 2022-09-03T06:32:27.345 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:32:27.346 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:32:27.347 DEBUG:boto:Port: 8001 2022-09-03T06:32:27.347 DEBUG:boto:Params: {} 2022-09-03T06:32:27.348 DEBUG:boto:Token: None 2022-09-03T06:32:27.348 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:32:27 GMT /avysmw-61/obj4 2022-09-03T06:32:27.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jv6Pev8XTsEjnTI27cRpO8EkZTY= 2022-09-03T06:32:27.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:32:27 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jv6Pev8XTsEjnTI27cRpO8EkZTY='} 2022-09-03T06:32:27.399 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000009b3a11ef15f9287f-006312f4fb-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:32:27.400 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:27.401 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:27.404 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:27.462 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:27.462 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:27.497 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:32:27.516 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:32:27.516 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:27.517 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:32:27.517 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:32:27.518 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:27.518 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:27.519 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186746.617635_300.1", 2022-09-03T06:32:27.519 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:32:26.617635Z" 2022-09-03T06:32:27.519 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:27.520 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:27.520 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:32:27.521 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:32:27.521 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:27.521 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:27.527 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:32:27.528 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:32:27.528 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:27.529 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:32:27.529 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662186746.617635_300.1', 2: '', 3: ''} 2022-09-03T06:32:27.530 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:32:27.531 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:27.532 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:27.534 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:27.591 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:27.592 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:27.623 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:27.616+0000 7f2cc9215600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:27.623 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:27.617+0000 7f2cc9215600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:27.633 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:32:27.634 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:32:27.634 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:32:27.634 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:32:27.635 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:32:27.635 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:32:27.635 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:32:27.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:27.636 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:32:27.636 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:27.636 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:32:27.636 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:27.637 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:27.637 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:32:27.637 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:27.637 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:32:27.638 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:27.638 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:32:27.638 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:27.638 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:27.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:27.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:27.639 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:32:27.639 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:27.640 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:27.640 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186746.609489_298.1", 2022-09-03T06:32:27.640 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:27.640 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:27.641 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:27.641 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:26.609489Z", 2022-09-03T06:32:27.641 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:32:27.641 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:27.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:27.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:27.642 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:32:27.642 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:27.643 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:27.643 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:32:27.643 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:27.643 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:27.644 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:27.644 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:32:27.644 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:27.644 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:27.645 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:27.645 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:27.645 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:32:27.645 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:27.646 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:27.646 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:32:27.646 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:27.646 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:27.647 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:27.647 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:32:27.647 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:27.647 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:27.648 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:27.648 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:32:27.648 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:27.648 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:32:27.649 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:32:27.649 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:32:27.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:27.649 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:32:27Z" 2022-09-03T06:32:27.650 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:32:27.651 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186746.609489_298.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:26.609489Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:32:27Z" } 2022-09-03T06:32:27.652 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186746.609489_298.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:26.609489Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:32:27.652 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186746.617635_300.1', 2: '', 3: ''} 2022-09-03T06:32:27.653 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186746.609489_298.1', 2: '', 3: ''} 2022-09-03T06:32:27.653 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662186746.617635_300.1 target=1_1662186746.609489_298.1 2022-09-03T06:32:32.657 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:32.658 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:32.659 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:32.720 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:32.720 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:32.751 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:32.748+0000 7fc0c1e2f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:32.761 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:32.749+0000 7fc0c1e2f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:32.768 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:32:32.769 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:32:32.769 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:32:32.772 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:32:32.773 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:32:32.773 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:32:32.773 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:32:32.774 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:32.774 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:32:32.775 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:32.775 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:32:32.778 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:32.778 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:32.779 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:32:32.779 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:32.780 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:32:32.781 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:32.782 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:32:32.782 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:32.783 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:32.784 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:32.784 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:32.785 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:32:32.785 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:32.785 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:32.786 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186746.617635_300.1", 2022-09-03T06:32:32.788 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:32.788 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:32.789 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:32.789 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:26.617635Z", 2022-09-03T06:32:32.791 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:32:32.792 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:32.793 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:32.793 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:32.793 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:32:32.794 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:32.795 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:32.795 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:32:32.796 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:32.797 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:32.797 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:32.797 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:32:32.799 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:32.800 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:32.803 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:32.803 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:32.804 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:32:32.805 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:32.805 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:32.806 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:32:32.806 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:32.807 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:32.807 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:32.809 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:32:32.810 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:32.810 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:32.810 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:32.811 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:32:32.811 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:32.811 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:32:32.812 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:32:32.813 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:32:32.813 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:32.813 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:32:32Z" 2022-09-03T06:32:32.814 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:32:32.814 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186746.617635_300.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:26.617635Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:32:32Z" } 2022-09-03T06:32:32.815 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186746.617635_300.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:26.617635Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:32:32.816 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186746.617635_300.1', 2: '', 3: ''} 2022-09-03T06:32:32.817 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186746.617635_300.1', 2: '', 3: ''} 2022-09-03T06:32:32.819 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:32:32.821 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:32:32.823 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:32.824 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:32.826 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:32.888 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:32.889 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:32.920 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:32:32.917+0000 7faca4e61600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:32.921 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:32:32.918+0000 7faca4e61600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:32.939 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:32:32.939 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:32:32.940 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:32:32.941 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:32:32.942 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:32:32.942 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:32:32.943 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:32:32.944 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:32.944 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:32:32.945 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:32.945 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:32:32.946 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:32.948 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:32:32.948 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:32:32.948 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:32.948 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:32:32.949 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:32.949 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:32:32.949 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:32:32.949 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:32.950 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:32.950 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:32.950 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:32:32.950 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:32.951 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:32:32.951 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662186746.617635_300.1", 2022-09-03T06:32:32.951 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:32.951 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:32.952 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:32.952 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:32:26.617635Z", 2022-09-03T06:32:32.952 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:32:32.952 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:32.953 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:32.953 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:32.953 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:32:32.953 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:32.954 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:32:32.954 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:32:32.954 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:32.954 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:32.955 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:32.955 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:32:32.955 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:32:32.955 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:32.956 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:32.956 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:32:32.956 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:32:32.956 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:32:32.957 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:32:32.957 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:32:32.957 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:32:32.957 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:32:32.958 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:32:32.961 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:32:32.961 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:32:32.962 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:32.962 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:32:32.964 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:32:32.965 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:32.965 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:32:32.966 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:32:32.966 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:32:32.968 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:32:32.970 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:32:32Z" 2022-09-03T06:32:32.971 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:32:32.972 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186746.617635_300.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:26.617635Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:32:32Z" } 2022-09-03T06:32:32.973 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186746.617635_300.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:26.617635Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:32:32.975 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186746.617635_300.1', 2: '', 3: ''} 2022-09-03T06:32:32.976 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186746.617635_300.1', 2: '', 3: ''} 2022-09-03T06:32:32.977 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:32:32.978 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:32:32.980 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:32.980 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:32:32.982 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:32:33.038 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:33.038 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:33.073 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:33.064+0000 7f699bb45600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:33.073 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:33.065+0000 7f699bb45600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:32:33.095 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:32:33.096 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:32:33.096 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:32:33.097 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:32:33.097 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:32:33.098 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:32:33.107 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:32:33.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:33.108 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:32:33.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:33.108 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:32:33.108 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:33.109 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:33.109 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:32:33.109 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:33.110 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:32:33.110 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:33.110 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:32:33.110 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:33.111 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:33.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:33.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:33.111 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:32:33.112 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:33.112 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:33.112 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662186746.617635_300.1", 2022-09-03T06:32:33.112 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:33.113 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:33.113 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:33.113 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:32:26.617635Z", 2022-09-03T06:32:33.113 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:32:33.121 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:33.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:33.123 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:33.123 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:32:33.124 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:33.125 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:33.125 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:32:33.126 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:33.126 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:33.127 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:33.127 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:32:33.128 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:33.130 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:33.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:33.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:32:33.131 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:32:33.132 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:32:33.132 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:32:33.133 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:32:33.134 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:32:33.134 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:32:33.135 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:32:33.135 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:32:33.136 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:32:33.136 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:33.137 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:32:33.137 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:32:33.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:33.138 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:32:33.139 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:32:33.140 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:32:33.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:32:33.141 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:32:33Z" 2022-09-03T06:32:33.142 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:32:33.142 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662186746.617635_300.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:32:26.617635Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:32:33Z" } 2022-09-03T06:32:33.144 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662186746.617635_300.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:32:26.617635Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:32:33.144 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662186746.617635_300.1', 2: '', 3: ''} 2022-09-03T06:32:33.145 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662186746.617635_300.1', 2: '', 3: ''} 2022-09-03T06:32:33.146 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:32:33.147 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-59 2022-09-03T06:32:33.148 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:32:33.149 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:32:33.160 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:32:33.217 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:33.218 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:33.275 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:33.271+0000 7f774867a600 1 waiting to reach incremental sync.. 2022-09-03T06:32:38.273 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:38.271+0000 7f774867a600 1 waiting to reach incremental sync.. 2022-09-03T06:32:43.273 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:43.272+0000 7f774867a600 1 waiting to reach incremental sync.. 2022-09-03T06:32:48.277 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:48.274+0000 7f774867a600 1 bucket sync caught up with source: 2022-09-03T06:32:48.278 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:32:48.278 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:32:48.279 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:48.274+0000 7f774867a600 0 bucket checkpoint complete 2022-09-03T06:32:48.299 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-59 2022-09-03T06:32:48.300 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:32:48.301 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:32:48.312 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:32:48.379 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:48.380 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:48.423 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:32:48.420+0000 7f464dfc1600 1 bucket sync caught up with source: 2022-09-03T06:32:48.423 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:32:48.424 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:32:48.425 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:32:48.420+0000 7f464dfc1600 0 bucket checkpoint complete 2022-09-03T06:32:48.448 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-59 2022-09-03T06:32:48.450 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:32:48.451 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:32:48.452 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:32:48.521 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:32:48.521 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:32:48.589 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:48.580+0000 7f771f777600 1 waiting for incremental sync to catch up: 2022-09-03T06:32:48.589 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , 00000000001.26.5, , , , , , ] 2022-09-03T06:32:48.590 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:32:53.594 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:53.581+0000 7f771f777600 1 waiting for incremental sync to catch up: 2022-09-03T06:32:53.595 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , 00000000001.26.5, , , , , , ] 2022-09-03T06:32:53.596 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:32:58.585 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:32:58.583+0000 7f771f777600 1 waiting for incremental sync to catch up: 2022-09-03T06:32:58.586 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , 00000000001.26.5, , , , , , ] 2022-09-03T06:32:58.587 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:33:03.588 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:03.584+0000 7f771f777600 1 waiting for incremental sync to catch up: 2022-09-03T06:33:03.588 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , 00000000001.26.5, , , , , , ] 2022-09-03T06:33:03.589 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:33:08.588 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:08.586+0000 7f771f777600 1 bucket sync caught up with source: 2022-09-03T06:33:08.588 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:33:08.589 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:33:08.589 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:08.586+0000 7f771f777600 0 bucket checkpoint complete 2022-09-03T06:33:08.609 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-59 2022-09-03T06:33:08.614 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:08.624 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:08.627 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:08.694 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:08.695 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:08.747 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:08.735+0000 7f975a13e600 1 bucket sync caught up with source: 2022-09-03T06:33:08.748 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.206.5, , 00000000001.285.5, , 00000000001.158.5, , , 00000000001.284.5, , , ] 2022-09-03T06:33:08.748 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.206.5, , 00000000001.285.5, , 00000000001.158.5, , , 00000000001.284.5, , , ] 2022-09-03T06:33:08.749 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:08.735+0000 7f975a13e600 0 bucket checkpoint complete 2022-09-03T06:33:08.768 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-59 2022-09-03T06:33:08.770 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:08.775 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:08.776 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:08.854 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:08.854 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:08.899 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:08.896+0000 7fd9b1f1b600 1 bucket sync caught up with source: 2022-09-03T06:33:08.900 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.206.5, , 00000000001.285.5, , 00000000001.158.5, , , 00000000001.284.5, , , ] 2022-09-03T06:33:08.900 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.206.5, , 00000000001.285.5, , 00000000001.158.5, , , 00000000001.284.5, , , ] 2022-09-03T06:33:08.910 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:08.896+0000 7fd9b1f1b600 0 bucket checkpoint complete 2022-09-03T06:33:08.924 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-59 2022-09-03T06:33:08.925 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-59 2022-09-03T06:33:08.925 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:08.926 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:08.926 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:08.984 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:08.985 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:09.048 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:09.044+0000 7fd544a4d600 1 bucket sync caught up with source: 2022-09-03T06:33:09.049 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.234.5, , 00000000001.197.5, , 00000000001.185.5, , , 00000000001.248.5, , , ] 2022-09-03T06:33:09.049 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.234.5, , 00000000001.197.5, , 00000000001.185.5, , , 00000000001.248.5, , , ] 2022-09-03T06:33:09.049 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:09.044+0000 7fd544a4d600 0 bucket checkpoint complete 2022-09-03T06:33:09.076 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-59 2022-09-03T06:33:09.077 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:09.078 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:09.079 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:09.156 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:09.157 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:09.204 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:09.198+0000 7f1186b51600 1 bucket sync caught up with source: 2022-09-03T06:33:09.204 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.234.5, , 00000000001.197.5, , 00000000001.185.5, , , 00000000001.248.5, , , ] 2022-09-03T06:33:09.205 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.234.5, , 00000000001.197.5, , 00000000001.185.5, , , 00000000001.248.5, , , ] 2022-09-03T06:33:09.205 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:09.198+0000 7f1186b51600 0 bucket checkpoint complete 2022-09-03T06:33:09.220 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-59 2022-09-03T06:33:09.221 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-59 zones={test-zone2, test-zone1} 2022-09-03T06:33:09.222 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.223 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.224 DEBUG:boto:Method: HEAD 2022-09-03T06:33:09.228 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:33:09.229 DEBUG:boto:Data: 2022-09-03T06:33:09.234 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.235 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.236 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.237 DEBUG:boto:Params: {} 2022-09-03T06:33:09.238 DEBUG:boto:Token: None 2022-09-03T06:33:09.242 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/ 2022-09-03T06:33:09.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C5Quuv5XUwwiCX6uQX5ewCJ4h8U= 2022-09-03T06:33:09.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C5Quuv5XUwwiCX6uQX5ewCJ4h8U=', 'Content-Length': '0'} 2022-09-03T06:33:09.254 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000072abf35da33eef64-006312f525-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.255 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.255 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.256 DEBUG:boto:Method: HEAD 2022-09-03T06:33:09.257 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:33:09.257 DEBUG:boto:Data: 2022-09-03T06:33:09.258 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.265 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.266 DEBUG:boto:Params: {} 2022-09-03T06:33:09.269 DEBUG:boto:Token: None 2022-09-03T06:33:09.270 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/ 2022-09-03T06:33:09.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C5Quuv5XUwwiCX6uQX5ewCJ4h8U= 2022-09-03T06:33:09.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C5Quuv5XUwwiCX6uQX5ewCJ4h8U=', 'Content-Length': '0'} 2022-09-03T06:33:09.282 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003f4b57c159b026a7-006312f525-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.283 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:33:09.284 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.284 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.285 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.288 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.289 DEBUG:boto:Method: GET 2022-09-03T06:33:09.289 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.299 DEBUG:boto:Data: 2022-09-03T06:33:09.301 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.301 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.302 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.302 DEBUG:boto:Params: {} 2022-09-03T06:33:09.303 DEBUG:boto:Token: None 2022-09-03T06:33:09.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/?versions 2022-09-03T06:33:09.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3mowJSXKSf12ielguqsPNiBy1rE= 2022-09-03T06:33:09.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3mowJSXKSf12ielguqsPNiBy1rE=', 'Content-Length': '0'} 2022-09-03T06:33:09.311 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005f6d7de49d13fbad-006312f525-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.312 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:09.316 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:09.317 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:09.319 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:09.320 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:09.321 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:33:09.321 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.322 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.324 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.339 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.340 DEBUG:boto:Method: GET 2022-09-03T06:33:09.341 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.342 DEBUG:boto:Data: 2022-09-03T06:33:09.343 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.345 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.347 DEBUG:boto:Params: {} 2022-09-03T06:33:09.348 DEBUG:boto:Token: None 2022-09-03T06:33:09.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/?versions 2022-09-03T06:33:09.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3mowJSXKSf12ielguqsPNiBy1rE= 2022-09-03T06:33:09.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3mowJSXKSf12ielguqsPNiBy1rE=', 'Content-Length': '0'} 2022-09-03T06:33:09.371 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f76364c15f4df453-006312f525-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.373 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:09.375 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:09.379 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:09.380 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:09.381 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:09.382 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.383 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.385 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.385 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.386 DEBUG:boto:Method: GET 2022-09-03T06:33:09.386 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.387 DEBUG:boto:Data: 2022-09-03T06:33:09.388 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.388 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.389 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.391 DEBUG:boto:Params: {} 2022-09-03T06:33:09.392 DEBUG:boto:Token: None 2022-09-03T06:33:09.393 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/?versions 2022-09-03T06:33:09.394 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3mowJSXKSf12ielguqsPNiBy1rE= 2022-09-03T06:33:09.395 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3mowJSXKSf12ielguqsPNiBy1rE=', 'Content-Length': '0'} 2022-09-03T06:33:09.401 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ad43edc1d85c0e1-006312f525-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.403 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:09.404 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.407 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.408 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.408 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.420 DEBUG:boto:Method: GET 2022-09-03T06:33:09.420 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:09.421 DEBUG:boto:Data: 2022-09-03T06:33:09.424 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.426 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.427 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.428 DEBUG:boto:Params: {} 2022-09-03T06:33:09.431 DEBUG:boto:Token: None 2022-09-03T06:33:09.432 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/?versions 2022-09-03T06:33:09.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3mowJSXKSf12ielguqsPNiBy1rE= 2022-09-03T06:33:09.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3mowJSXKSf12ielguqsPNiBy1rE=', 'Content-Length': '0'} 2022-09-03T06:33:09.450 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ca6fd0d0bc306de7-006312f525-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.452 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:09.456 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:09.458 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.459 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.460 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.461 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.462 DEBUG:boto:Method: GET 2022-09-03T06:33:09.463 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.464 DEBUG:boto:Data: 2022-09-03T06:33:09.465 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.465 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.467 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.469 DEBUG:boto:Params: {} 2022-09-03T06:33:09.470 DEBUG:boto:Token: None 2022-09-03T06:33:09.471 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q6dsGKA8mI2qiKgxANVtr5iTSkE= 2022-09-03T06:33:09.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q6dsGKA8mI2qiKgxANVtr5iTSkE=', 'Content-Length': '0'} 2022-09-03T06:33:09.486 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e6450825e32349ab-006312f525-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.488 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.509 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.511 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.513 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.515 DEBUG:boto:Method: GET 2022-09-03T06:33:09.516 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.516 DEBUG:boto:Data: 2022-09-03T06:33:09.517 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.518 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.519 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.520 DEBUG:boto:Params: {} 2022-09-03T06:33:09.521 DEBUG:boto:Token: None 2022-09-03T06:33:09.522 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q6dsGKA8mI2qiKgxANVtr5iTSkE= 2022-09-03T06:33:09.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q6dsGKA8mI2qiKgxANVtr5iTSkE=', 'Content-Length': '0'} 2022-09-03T06:33:09.527 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006132dd621ffed045-006312f525-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.528 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.529 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.530 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.531 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.532 DEBUG:boto:Method: HEAD 2022-09-03T06:33:09.533 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.535 DEBUG:boto:Data: 2022-09-03T06:33:09.536 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.537 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.537 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.539 DEBUG:boto:Params: {} 2022-09-03T06:33:09.540 DEBUG:boto:Token: None 2022-09-03T06:33:09.542 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+hu9fX5kCN1wDyyKNb1l4p/G2LY= 2022-09-03T06:33:09.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+hu9fX5kCN1wDyyKNb1l4p/G2LY=', 'Content-Length': '0'} 2022-09-03T06:33:09.548 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d0cda90d98f82d11-006312f525-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.550 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.550 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.551 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.551 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.552 DEBUG:boto:Method: HEAD 2022-09-03T06:33:09.552 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.553 DEBUG:boto:Data: 2022-09-03T06:33:09.553 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.554 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.555 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.555 DEBUG:boto:Params: {} 2022-09-03T06:33:09.556 DEBUG:boto:Token: None 2022-09-03T06:33:09.557 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.557 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+hu9fX5kCN1wDyyKNb1l4p/G2LY= 2022-09-03T06:33:09.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+hu9fX5kCN1wDyyKNb1l4p/G2LY=', 'Content-Length': '0'} 2022-09-03T06:33:09.560 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a71c83ede5c3a566-006312f525-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.561 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:09.562 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.562 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.563 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.563 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.564 DEBUG:boto:Method: GET 2022-09-03T06:33:09.565 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.565 DEBUG:boto:Data: 2022-09-03T06:33:09.566 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.566 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.567 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.568 DEBUG:boto:Params: {} 2022-09-03T06:33:09.569 DEBUG:boto:Token: None 2022-09-03T06:33:09.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q6dsGKA8mI2qiKgxANVtr5iTSkE= 2022-09-03T06:33:09.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q6dsGKA8mI2qiKgxANVtr5iTSkE=', 'Content-Length': '0'} 2022-09-03T06:33:09.573 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dfb48f678f972204-006312f525-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.614 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.615 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.618 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.619 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.621 DEBUG:boto:Method: GET 2022-09-03T06:33:09.625 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.627 DEBUG:boto:Data: 2022-09-03T06:33:09.628 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.631 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.632 DEBUG:boto:Params: {} 2022-09-03T06:33:09.633 DEBUG:boto:Token: None 2022-09-03T06:33:09.637 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:09.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q6dsGKA8mI2qiKgxANVtr5iTSkE= 2022-09-03T06:33:09.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q6dsGKA8mI2qiKgxANVtr5iTSkE=', 'Content-Length': '0'} 2022-09-03T06:33:09.654 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ca02db5adb165901-006312f525-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.656 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.657 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.660 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:09.661 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:09.662 DEBUG:boto:Method: GET 2022-09-03T06:33:09.663 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:09.664 DEBUG:boto:Data: 2022-09-03T06:33:09.665 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.666 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.667 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.668 DEBUG:boto:Params: {} 2022-09-03T06:33:09.670 DEBUG:boto:Token: None 2022-09-03T06:33:09.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/ 2022-09-03T06:33:09.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Bzj1TA8HOaD09duHfhLJEVvUaQ= 2022-09-03T06:33:09.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Bzj1TA8HOaD09duHfhLJEVvUaQ=', 'Content-Length': '0'} 2022-09-03T06:33:09.678 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c3d685e7628d505-006312f525-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.681 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:09.685 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.686 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.688 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:09.689 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:09.689 DEBUG:boto:Method: GET 2022-09-03T06:33:09.691 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:09.692 DEBUG:boto:Data: 2022-09-03T06:33:09.693 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.696 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.697 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.698 DEBUG:boto:Params: {} 2022-09-03T06:33:09.699 DEBUG:boto:Token: None 2022-09-03T06:33:09.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/ 2022-09-03T06:33:09.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Bzj1TA8HOaD09duHfhLJEVvUaQ= 2022-09-03T06:33:09.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Bzj1TA8HOaD09duHfhLJEVvUaQ=', 'Content-Length': '0'} 2022-09-03T06:33:09.719 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000855ab0177455419f-006312f525-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.728 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:09.730 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.731 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.732 DEBUG:boto:Method: HEAD 2022-09-03T06:33:09.733 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:09.733 DEBUG:boto:Data: 2022-09-03T06:33:09.734 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.735 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.736 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.736 DEBUG:boto:Params: {} 2022-09-03T06:33:09.737 DEBUG:boto:Token: None 2022-09-03T06:33:09.737 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1 2022-09-03T06:33:09.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B/ZNnH10PkonVf2m+FHxEMhgdds= 2022-09-03T06:33:09.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B/ZNnH10PkonVf2m+FHxEMhgdds=', 'Content-Length': '0'} 2022-09-03T06:33:09.743 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000da45fa5c675b249e-006312f525-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.745 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.746 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.747 DEBUG:boto:Method: HEAD 2022-09-03T06:33:09.748 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:09.749 DEBUG:boto:Data: 2022-09-03T06:33:09.750 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.754 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.755 DEBUG:boto:Params: {} 2022-09-03T06:33:09.770 DEBUG:boto:Token: None 2022-09-03T06:33:09.771 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1 2022-09-03T06:33:09.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B/ZNnH10PkonVf2m+FHxEMhgdds= 2022-09-03T06:33:09.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B/ZNnH10PkonVf2m+FHxEMhgdds=', 'Content-Length': '0'} 2022-09-03T06:33:09.792 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000037b619a549c3d6c8-006312f525-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.794 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:09.803 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.805 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.806 DEBUG:boto:Method: GET 2022-09-03T06:33:09.807 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:09.815 DEBUG:boto:Data: 2022-09-03T06:33:09.816 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.817 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.820 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.821 DEBUG:boto:Params: {} 2022-09-03T06:33:09.823 DEBUG:boto:Token: None 2022-09-03T06:33:09.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1 2022-09-03T06:33:09.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eolOHSlCOHnB9osWWCeLsdF8tNg= 2022-09-03T06:33:09.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eolOHSlCOHnB9osWWCeLsdF8tNg=', 'Content-Length': '0'} 2022-09-03T06:33:09.848 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001edc737bbad23726-006312f525-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.849 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:09.850 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:09.850 DEBUG:boto:Method: GET 2022-09-03T06:33:09.851 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:09.852 DEBUG:boto:Data: 2022-09-03T06:33:09.853 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.853 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.854 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.855 DEBUG:boto:Params: {} 2022-09-03T06:33:09.855 DEBUG:boto:Token: None 2022-09-03T06:33:09.865 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj1 2022-09-03T06:33:09.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eolOHSlCOHnB9osWWCeLsdF8tNg= 2022-09-03T06:33:09.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eolOHSlCOHnB9osWWCeLsdF8tNg=', 'Content-Length': '0'} 2022-09-03T06:33:09.870 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004fbcf8eac0014bab-006312f525-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.872 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.873 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.877 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:09.878 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:09.886 DEBUG:boto:Method: GET 2022-09-03T06:33:09.888 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:09.889 DEBUG:boto:Data: 2022-09-03T06:33:09.890 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.891 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.892 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.894 DEBUG:boto:Params: {} 2022-09-03T06:33:09.895 DEBUG:boto:Token: None 2022-09-03T06:33:09.897 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/ 2022-09-03T06:33:09.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Bzj1TA8HOaD09duHfhLJEVvUaQ= 2022-09-03T06:33:09.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Bzj1TA8HOaD09duHfhLJEVvUaQ=', 'Content-Length': '0'} 2022-09-03T06:33:09.904 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000021a18fb4e53d8d77-006312f525-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.909 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:09.910 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:09.910 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:09.911 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:09.912 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:09.912 DEBUG:boto:Method: GET 2022-09-03T06:33:09.913 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:09.913 DEBUG:boto:Data: 2022-09-03T06:33:09.914 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.915 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.916 DEBUG:boto:Params: {} 2022-09-03T06:33:09.917 DEBUG:boto:Token: None 2022-09-03T06:33:09.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/ 2022-09-03T06:33:09.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Bzj1TA8HOaD09duHfhLJEVvUaQ= 2022-09-03T06:33:09.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Bzj1TA8HOaD09duHfhLJEVvUaQ=', 'Content-Length': '0'} 2022-09-03T06:33:09.923 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fac8d6bfc3f626e0-006312f525-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.925 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:09.927 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:09.934 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:09.935 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:09.935 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.936 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.938 DEBUG:boto:Method: GET 2022-09-03T06:33:09.940 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.941 DEBUG:boto:Data: 2022-09-03T06:33:09.942 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.943 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:09.944 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.945 DEBUG:boto:Params: {} 2022-09-03T06:33:09.950 DEBUG:boto:Token: None 2022-09-03T06:33:09.951 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DLHZmvk7kIAoIiwkeRNN7+XKXqQ= 2022-09-03T06:33:09.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DLHZmvk7kIAoIiwkeRNN7+XKXqQ=', 'Content-Length': '0'} 2022-09-03T06:33:09.958 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000018f3091913e8c777-006312f525-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.960 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:09.961 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:09.967 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.968 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.970 DEBUG:boto:Method: GET 2022-09-03T06:33:09.971 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.972 DEBUG:boto:Data: 2022-09-03T06:33:09.973 DEBUG:boto:Headers: {} 2022-09-03T06:33:09.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:09.975 DEBUG:boto:Port: 8000 2022-09-03T06:33:09.976 DEBUG:boto:Params: {} 2022-09-03T06:33:09.977 DEBUG:boto:Token: None 2022-09-03T06:33:09.979 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:09 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DLHZmvk7kIAoIiwkeRNN7+XKXqQ= 2022-09-03T06:33:09.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DLHZmvk7kIAoIiwkeRNN7+XKXqQ=', 'Content-Length': '0'} 2022-09-03T06:33:09.990 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001571368921a2363e-006312f525-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:09.992 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:09.995 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:09.996 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.997 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:09.998 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.000 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.001 DEBUG:boto:Data: 2022-09-03T06:33:10.002 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.003 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.003 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.004 DEBUG:boto:Params: {} 2022-09-03T06:33:10.006 DEBUG:boto:Token: None 2022-09-03T06:33:10.007 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YOkKP3NGeSaL9VUWupoSl5tRHA= 2022-09-03T06:33:10.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YOkKP3NGeSaL9VUWupoSl5tRHA=', 'Content-Length': '0'} 2022-09-03T06:33:10.013 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002efab0375ca40a07-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.015 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:10.015 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:10.016 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.018 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.019 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.020 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.021 DEBUG:boto:Data: 2022-09-03T06:33:10.027 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.028 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.028 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.029 DEBUG:boto:Params: {} 2022-09-03T06:33:10.029 DEBUG:boto:Token: None 2022-09-03T06:33:10.030 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YOkKP3NGeSaL9VUWupoSl5tRHA= 2022-09-03T06:33:10.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YOkKP3NGeSaL9VUWupoSl5tRHA=', 'Content-Length': '0'} 2022-09-03T06:33:10.042 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000370ac98a6e40dd4a-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.045 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:10.046 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:10.047 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:10.047 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.048 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.049 DEBUG:boto:Method: GET 2022-09-03T06:33:10.049 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.050 DEBUG:boto:Data: 2022-09-03T06:33:10.050 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.051 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.053 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.053 DEBUG:boto:Params: {} 2022-09-03T06:33:10.054 DEBUG:boto:Token: None 2022-09-03T06:33:10.055 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JZKa/JqWWdRZUmzJhvS+UVgSUVA= 2022-09-03T06:33:10.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JZKa/JqWWdRZUmzJhvS+UVgSUVA=', 'Content-Length': '0'} 2022-09-03T06:33:10.064 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003f204865c461e388-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.070 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:10.071 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:10.072 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.073 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.074 DEBUG:boto:Method: GET 2022-09-03T06:33:10.075 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.077 DEBUG:boto:Data: 2022-09-03T06:33:10.078 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.080 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.080 DEBUG:boto:Params: {} 2022-09-03T06:33:10.081 DEBUG:boto:Token: None 2022-09-03T06:33:10.082 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:10.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JZKa/JqWWdRZUmzJhvS+UVgSUVA= 2022-09-03T06:33:10.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JZKa/JqWWdRZUmzJhvS+UVgSUVA=', 'Content-Length': '0'} 2022-09-03T06:33:10.088 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bf90d6eefddd3fc5-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.089 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.090 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.090 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:10.091 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:10.091 DEBUG:boto:Method: GET 2022-09-03T06:33:10.092 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:10.092 DEBUG:boto:Data: 2022-09-03T06:33:10.093 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.093 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.094 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.095 DEBUG:boto:Params: {} 2022-09-03T06:33:10.095 DEBUG:boto:Token: None 2022-09-03T06:33:10.096 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.102 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b852da4eadac5218-006312f526-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.103 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.104 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.106 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.107 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:10.108 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:10.109 DEBUG:boto:Method: GET 2022-09-03T06:33:10.111 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:10.112 DEBUG:boto:Data: 2022-09-03T06:33:10.113 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.116 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.117 DEBUG:boto:Params: {} 2022-09-03T06:33:10.119 DEBUG:boto:Token: None 2022-09-03T06:33:10.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.127 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063c95924e0d320a5-006312f526-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.128 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.130 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:10.131 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:10.132 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.132 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:10.133 DEBUG:boto:Data: 2022-09-03T06:33:10.135 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.136 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.137 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.138 DEBUG:boto:Params: {} 2022-09-03T06:33:10.139 DEBUG:boto:Token: None 2022-09-03T06:33:10.140 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj2 2022-09-03T06:33:10.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UF/u3YwEVyQjeReGjWpIwN661EE= 2022-09-03T06:33:10.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UF/u3YwEVyQjeReGjWpIwN661EE=', 'Content-Length': '0'} 2022-09-03T06:33:10.147 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005f3c0f8ec354149e-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.148 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:10.149 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:10.149 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.150 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:10.150 DEBUG:boto:Data: 2022-09-03T06:33:10.151 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.151 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.152 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.152 DEBUG:boto:Params: {} 2022-09-03T06:33:10.153 DEBUG:boto:Token: None 2022-09-03T06:33:10.157 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj2 2022-09-03T06:33:10.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UF/u3YwEVyQjeReGjWpIwN661EE= 2022-09-03T06:33:10.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UF/u3YwEVyQjeReGjWpIwN661EE=', 'Content-Length': '0'} 2022-09-03T06:33:10.165 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000084081f8ad27f5081-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.167 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:10.168 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:10.169 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:10.170 DEBUG:boto:Method: GET 2022-09-03T06:33:10.171 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:10.172 DEBUG:boto:Data: 2022-09-03T06:33:10.173 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.174 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.174 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.176 DEBUG:boto:Params: {} 2022-09-03T06:33:10.177 DEBUG:boto:Token: None 2022-09-03T06:33:10.178 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj2 2022-09-03T06:33:10.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GA56zLGqR59cB3Lkr+sSG9wuQks= 2022-09-03T06:33:10.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GA56zLGqR59cB3Lkr+sSG9wuQks=', 'Content-Length': '0'} 2022-09-03T06:33:10.184 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000967a18b6f9f32fca-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.186 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:10.187 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:10.188 DEBUG:boto:Method: GET 2022-09-03T06:33:10.188 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:10.189 DEBUG:boto:Data: 2022-09-03T06:33:10.189 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.190 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.191 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.191 DEBUG:boto:Params: {} 2022-09-03T06:33:10.192 DEBUG:boto:Token: None 2022-09-03T06:33:10.193 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj2 2022-09-03T06:33:10.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GA56zLGqR59cB3Lkr+sSG9wuQks= 2022-09-03T06:33:10.194 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GA56zLGqR59cB3Lkr+sSG9wuQks=', 'Content-Length': '0'} 2022-09-03T06:33:10.197 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f5cbae5bc71d48da-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.243 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.244 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.245 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:10.246 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:10.247 DEBUG:boto:Method: GET 2022-09-03T06:33:10.248 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:10.249 DEBUG:boto:Data: 2022-09-03T06:33:10.250 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.251 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.252 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.253 DEBUG:boto:Params: {} 2022-09-03T06:33:10.254 DEBUG:boto:Token: None 2022-09-03T06:33:10.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.259 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3b0634193c9ea86-006312f526-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.259 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.261 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.263 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.264 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:10.265 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:10.266 DEBUG:boto:Method: GET 2022-09-03T06:33:10.267 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:10.268 DEBUG:boto:Data: 2022-09-03T06:33:10.269 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.270 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.272 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.273 DEBUG:boto:Params: {} 2022-09-03T06:33:10.274 DEBUG:boto:Token: None 2022-09-03T06:33:10.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.282 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001d139b857040a490-006312f526-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.283 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.284 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:10.285 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.287 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.288 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.289 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.291 DEBUG:boto:Method: GET 2022-09-03T06:33:10.291 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.292 DEBUG:boto:Data: 2022-09-03T06:33:10.293 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.294 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.295 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.296 DEBUG:boto:Params: {} 2022-09-03T06:33:10.297 DEBUG:boto:Token: None 2022-09-03T06:33:10.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MMSxtDS8lJBCJtz44XeIFehniq8= 2022-09-03T06:33:10.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MMSxtDS8lJBCJtz44XeIFehniq8=', 'Content-Length': '0'} 2022-09-03T06:33:10.304 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000668e337c697548ed-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.306 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.307 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.308 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.309 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.310 DEBUG:boto:Method: GET 2022-09-03T06:33:10.311 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.312 DEBUG:boto:Data: 2022-09-03T06:33:10.314 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.315 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.316 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.317 DEBUG:boto:Params: {} 2022-09-03T06:33:10.319 DEBUG:boto:Token: None 2022-09-03T06:33:10.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MMSxtDS8lJBCJtz44XeIFehniq8= 2022-09-03T06:33:10.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MMSxtDS8lJBCJtz44XeIFehniq8=', 'Content-Length': '0'} 2022-09-03T06:33:10.325 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000206123be915f7513-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.327 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.328 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.329 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.329 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.330 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.331 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.332 DEBUG:boto:Data: 2022-09-03T06:33:10.333 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.334 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.335 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.337 DEBUG:boto:Params: {} 2022-09-03T06:33:10.338 DEBUG:boto:Token: None 2022-09-03T06:33:10.340 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ir1ykeTeA7MEX9+UmObHEdUjpH0= 2022-09-03T06:33:10.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ir1ykeTeA7MEX9+UmObHEdUjpH0=', 'Content-Length': '0'} 2022-09-03T06:33:10.345 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fec244c80d5e52e9-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.346 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.347 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.348 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.350 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.351 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.352 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.353 DEBUG:boto:Data: 2022-09-03T06:33:10.354 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.355 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.355 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.356 DEBUG:boto:Params: {} 2022-09-03T06:33:10.358 DEBUG:boto:Token: None 2022-09-03T06:33:10.359 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ir1ykeTeA7MEX9+UmObHEdUjpH0= 2022-09-03T06:33:10.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ir1ykeTeA7MEX9+UmObHEdUjpH0=', 'Content-Length': '0'} 2022-09-03T06:33:10.365 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000935f167fc20263b5-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.367 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:10.368 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.369 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.371 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.371 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.372 DEBUG:boto:Method: GET 2022-09-03T06:33:10.372 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.373 DEBUG:boto:Data: 2022-09-03T06:33:10.374 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.374 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.375 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.376 DEBUG:boto:Params: {} 2022-09-03T06:33:10.377 DEBUG:boto:Token: None 2022-09-03T06:33:10.377 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MMSxtDS8lJBCJtz44XeIFehniq8= 2022-09-03T06:33:10.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MMSxtDS8lJBCJtz44XeIFehniq8=', 'Content-Length': '0'} 2022-09-03T06:33:10.386 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b794efb9c7bc4343-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.388 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.389 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.391 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.392 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.393 DEBUG:boto:Method: GET 2022-09-03T06:33:10.393 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.394 DEBUG:boto:Data: 2022-09-03T06:33:10.394 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.395 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.396 DEBUG:boto:Params: {} 2022-09-03T06:33:10.397 DEBUG:boto:Token: None 2022-09-03T06:33:10.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:10.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MMSxtDS8lJBCJtz44XeIFehniq8= 2022-09-03T06:33:10.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MMSxtDS8lJBCJtz44XeIFehniq8=', 'Content-Length': '0'} 2022-09-03T06:33:10.402 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bee968191fbff4e3-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.403 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.404 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.404 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:10.405 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:10.406 DEBUG:boto:Method: GET 2022-09-03T06:33:10.407 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:10.408 DEBUG:boto:Data: 2022-09-03T06:33:10.408 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.409 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.409 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.410 DEBUG:boto:Params: {} 2022-09-03T06:33:10.411 DEBUG:boto:Token: None 2022-09-03T06:33:10.412 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.414 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.416 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b6e16630347e1df-006312f526-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.460 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.462 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.464 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.465 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:10.466 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:10.467 DEBUG:boto:Method: GET 2022-09-03T06:33:10.468 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:10.469 DEBUG:boto:Data: 2022-09-03T06:33:10.470 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.473 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.474 DEBUG:boto:Params: {} 2022-09-03T06:33:10.475 DEBUG:boto:Token: None 2022-09-03T06:33:10.476 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.482 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037bda83da9eb1865-006312f526-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.484 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.485 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.487 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.488 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.489 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:10.490 DEBUG:boto:Data: 2022-09-03T06:33:10.493 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.493 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.495 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.496 DEBUG:boto:Params: {} 2022-09-03T06:33:10.497 DEBUG:boto:Token: None 2022-09-03T06:33:10.498 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3 2022-09-03T06:33:10.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OC4wGQEXvxs4k7BQh71WWLfRmzc= 2022-09-03T06:33:10.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OC4wGQEXvxs4k7BQh71WWLfRmzc=', 'Content-Length': '0'} 2022-09-03T06:33:10.505 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001b3b7b774f6034a0-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.506 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.507 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.509 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.510 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:10.513 DEBUG:boto:Data: 2022-09-03T06:33:10.513 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.521 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.523 DEBUG:boto:Params: {} 2022-09-03T06:33:10.523 DEBUG:boto:Token: None 2022-09-03T06:33:10.526 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3 2022-09-03T06:33:10.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OC4wGQEXvxs4k7BQh71WWLfRmzc= 2022-09-03T06:33:10.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OC4wGQEXvxs4k7BQh71WWLfRmzc=', 'Content-Length': '0'} 2022-09-03T06:33:10.534 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c4a4404b646450de-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.536 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:10.537 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.537 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.539 DEBUG:boto:Method: GET 2022-09-03T06:33:10.539 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:10.540 DEBUG:boto:Data: 2022-09-03T06:33:10.540 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.541 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.542 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.542 DEBUG:boto:Params: {} 2022-09-03T06:33:10.543 DEBUG:boto:Token: None 2022-09-03T06:33:10.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3 2022-09-03T06:33:10.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S4yEG/7H97xyS39vUJSj1PNABuc= 2022-09-03T06:33:10.546 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S4yEG/7H97xyS39vUJSj1PNABuc=', 'Content-Length': '0'} 2022-09-03T06:33:10.549 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007189c129cfac10d8-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.551 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:10.552 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:10.553 DEBUG:boto:Method: GET 2022-09-03T06:33:10.554 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:10.555 DEBUG:boto:Data: 2022-09-03T06:33:10.556 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.568 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.568 DEBUG:boto:Params: {} 2022-09-03T06:33:10.569 DEBUG:boto:Token: None 2022-09-03T06:33:10.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj3 2022-09-03T06:33:10.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S4yEG/7H97xyS39vUJSj1PNABuc= 2022-09-03T06:33:10.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S4yEG/7H97xyS39vUJSj1PNABuc=', 'Content-Length': '0'} 2022-09-03T06:33:10.574 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000044b687c2443cd127-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.575 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.575 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.576 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:10.577 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:10.577 DEBUG:boto:Method: GET 2022-09-03T06:33:10.578 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:10.578 DEBUG:boto:Data: 2022-09-03T06:33:10.579 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.580 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.580 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.581 DEBUG:boto:Params: {} 2022-09-03T06:33:10.582 DEBUG:boto:Token: None 2022-09-03T06:33:10.583 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.592 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000049e51bb51968f3b6-006312f526-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.594 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.595 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.595 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.596 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:10.596 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:10.597 DEBUG:boto:Method: GET 2022-09-03T06:33:10.598 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:10.598 DEBUG:boto:Data: 2022-09-03T06:33:10.599 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.600 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.601 DEBUG:boto:Params: {} 2022-09-03T06:33:10.602 DEBUG:boto:Token: None 2022-09-03T06:33:10.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.609 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000055d9b36f37c4a0d-006312f526-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.610 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.611 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:10.612 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.613 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.613 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.614 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.615 DEBUG:boto:Method: GET 2022-09-03T06:33:10.616 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.617 DEBUG:boto:Data: 2022-09-03T06:33:10.618 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.619 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.620 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.621 DEBUG:boto:Params: {} 2022-09-03T06:33:10.622 DEBUG:boto:Token: None 2022-09-03T06:33:10.623 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oZldJJ/CLH4jZPJgQiNkQkrP+y4= 2022-09-03T06:33:10.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oZldJJ/CLH4jZPJgQiNkQkrP+y4=', 'Content-Length': '0'} 2022-09-03T06:33:10.631 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f6a6a04f9036d49d-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.632 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.632 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.633 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.634 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.635 DEBUG:boto:Method: GET 2022-09-03T06:33:10.635 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.637 DEBUG:boto:Data: 2022-09-03T06:33:10.638 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.638 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.639 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.640 DEBUG:boto:Params: {} 2022-09-03T06:33:10.640 DEBUG:boto:Token: None 2022-09-03T06:33:10.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oZldJJ/CLH4jZPJgQiNkQkrP+y4= 2022-09-03T06:33:10.643 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oZldJJ/CLH4jZPJgQiNkQkrP+y4=', 'Content-Length': '0'} 2022-09-03T06:33:10.646 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003f76bad819ef907a-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.649 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.650 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.651 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.654 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.667 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.669 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.670 DEBUG:boto:Data: 2022-09-03T06:33:10.671 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.672 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.673 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.675 DEBUG:boto:Params: {} 2022-09-03T06:33:10.676 DEBUG:boto:Token: None 2022-09-03T06:33:10.677 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZVTrjJ1jeOFnrn4wmK7sXrwrTAg= 2022-09-03T06:33:10.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZVTrjJ1jeOFnrn4wmK7sXrwrTAg=', 'Content-Length': '0'} 2022-09-03T06:33:10.682 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000428a05ff5cf8838e-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.691 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.692 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.694 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.695 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.696 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.697 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.699 DEBUG:boto:Data: 2022-09-03T06:33:10.700 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.705 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.706 DEBUG:boto:Params: {} 2022-09-03T06:33:10.708 DEBUG:boto:Token: None 2022-09-03T06:33:10.709 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.710 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZVTrjJ1jeOFnrn4wmK7sXrwrTAg= 2022-09-03T06:33:10.711 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZVTrjJ1jeOFnrn4wmK7sXrwrTAg=', 'Content-Length': '0'} 2022-09-03T06:33:10.714 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000493f37d085d7257f-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.715 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:10.716 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.716 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.717 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.718 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.718 DEBUG:boto:Method: GET 2022-09-03T06:33:10.719 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.720 DEBUG:boto:Data: 2022-09-03T06:33:10.721 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.722 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.723 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.724 DEBUG:boto:Params: {} 2022-09-03T06:33:10.725 DEBUG:boto:Token: None 2022-09-03T06:33:10.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oZldJJ/CLH4jZPJgQiNkQkrP+y4= 2022-09-03T06:33:10.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oZldJJ/CLH4jZPJgQiNkQkrP+y4=', 'Content-Length': '0'} 2022-09-03T06:33:10.733 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000a853ab5b78e2567-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.735 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.736 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.737 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.737 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.739 DEBUG:boto:Method: GET 2022-09-03T06:33:10.740 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.740 DEBUG:boto:Data: 2022-09-03T06:33:10.741 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.742 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.743 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.745 DEBUG:boto:Params: {} 2022-09-03T06:33:10.746 DEBUG:boto:Token: None 2022-09-03T06:33:10.747 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:10.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oZldJJ/CLH4jZPJgQiNkQkrP+y4= 2022-09-03T06:33:10.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oZldJJ/CLH4jZPJgQiNkQkrP+y4=', 'Content-Length': '0'} 2022-09-03T06:33:10.757 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d5ea95c16074ba58-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.759 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.760 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.761 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:10.766 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:10.767 DEBUG:boto:Method: GET 2022-09-03T06:33:10.768 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:10.769 DEBUG:boto:Data: 2022-09-03T06:33:10.770 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.771 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.773 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.774 DEBUG:boto:Params: {} 2022-09-03T06:33:10.775 DEBUG:boto:Token: None 2022-09-03T06:33:10.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.782 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c98001833fc9d80a-006312f526-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.783 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.785 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.786 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.787 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:10.788 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:10.789 DEBUG:boto:Method: GET 2022-09-03T06:33:10.790 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:10.791 DEBUG:boto:Data: 2022-09-03T06:33:10.792 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.795 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.796 DEBUG:boto:Params: {} 2022-09-03T06:33:10.797 DEBUG:boto:Token: None 2022-09-03T06:33:10.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.814 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000081998cb70966a260-006312f526-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.816 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.819 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.820 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.821 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.822 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:10.823 DEBUG:boto:Data: 2022-09-03T06:33:10.824 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.825 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.826 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.830 DEBUG:boto:Params: {} 2022-09-03T06:33:10.831 DEBUG:boto:Token: None 2022-09-03T06:33:10.832 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4 2022-09-03T06:33:10.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jGKtdb4CBDy1iD6CwQxWOCQMy0I= 2022-09-03T06:33:10.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jGKtdb4CBDy1iD6CwQxWOCQMy0I=', 'Content-Length': '0'} 2022-09-03T06:33:10.837 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fcaa4d492ecac90d-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.839 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.840 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.841 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.842 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:10.843 DEBUG:boto:Data: 2022-09-03T06:33:10.844 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.845 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.847 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.848 DEBUG:boto:Params: {} 2022-09-03T06:33:10.849 DEBUG:boto:Token: None 2022-09-03T06:33:10.851 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4 2022-09-03T06:33:10.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jGKtdb4CBDy1iD6CwQxWOCQMy0I= 2022-09-03T06:33:10.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jGKtdb4CBDy1iD6CwQxWOCQMy0I=', 'Content-Length': '0'} 2022-09-03T06:33:10.858 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001e7b2b4f56c49465-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.861 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:10.862 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.862 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.863 DEBUG:boto:Method: GET 2022-09-03T06:33:10.863 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:10.864 DEBUG:boto:Data: 2022-09-03T06:33:10.864 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.865 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.865 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.866 DEBUG:boto:Params: {} 2022-09-03T06:33:10.867 DEBUG:boto:Token: None 2022-09-03T06:33:10.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4 2022-09-03T06:33:10.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZCFLjZyuCXmoG9+KyvGCm9T/y4I= 2022-09-03T06:33:10.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZCFLjZyuCXmoG9+KyvGCm9T/y4I=', 'Content-Length': '0'} 2022-09-03T06:33:10.871 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fccb8029b56c5f94-006312f526-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.872 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:10.873 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:10.873 DEBUG:boto:Method: GET 2022-09-03T06:33:10.881 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:10.882 DEBUG:boto:Data: 2022-09-03T06:33:10.883 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.885 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.886 DEBUG:boto:Params: {} 2022-09-03T06:33:10.888 DEBUG:boto:Token: None 2022-09-03T06:33:10.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/obj4 2022-09-03T06:33:10.892 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZCFLjZyuCXmoG9+KyvGCm9T/y4I= 2022-09-03T06:33:10.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZCFLjZyuCXmoG9+KyvGCm9T/y4I=', 'Content-Length': '0'} 2022-09-03T06:33:10.896 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000403ef4d7497e13a4-006312f526-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.897 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.898 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.899 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:10.900 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:10.902 DEBUG:boto:Method: GET 2022-09-03T06:33:10.903 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:10.904 DEBUG:boto:Data: 2022-09-03T06:33:10.905 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.906 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:10.908 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.909 DEBUG:boto:Params: {} 2022-09-03T06:33:10.909 DEBUG:boto:Token: None 2022-09-03T06:33:10.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.922 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000074b47730952a0578-006312f526-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.924 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.925 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.926 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.927 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:10.928 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:10.929 DEBUG:boto:Method: GET 2022-09-03T06:33:10.930 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:10.931 DEBUG:boto:Data: 2022-09-03T06:33:10.932 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.938 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.939 DEBUG:boto:Params: {} 2022-09-03T06:33:10.940 DEBUG:boto:Token: None 2022-09-03T06:33:10.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o= 2022-09-03T06:33:10.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzr/Z08fXU2PSEmUSXPjmglS84o=', 'Content-Length': '0'} 2022-09-03T06:33:10.950 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fe9a50088f9394e5-006312f526-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.951 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:10.952 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-59 zones={test-zone2, test-zone1} 2022-09-03T06:33:10.952 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-59 zones={test-zone3, test-zone1} 2022-09-03T06:33:10.953 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.953 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.954 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.955 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:33:10.956 DEBUG:boto:Data: 2022-09-03T06:33:10.956 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.957 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:10.958 DEBUG:boto:Port: 8001 2022-09-03T06:33:10.958 DEBUG:boto:Params: {} 2022-09-03T06:33:10.959 DEBUG:boto:Token: None 2022-09-03T06:33:10.960 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GXkLI4bgbNJcOTnPxJglkQl4scs= 2022-09-03T06:33:10.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GXkLI4bgbNJcOTnPxJglkQl4scs=', 'Content-Length': '0'} 2022-09-03T06:33:10.974 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000185de7be98bbd335-006312f526-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.976 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:10.979 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:10.980 DEBUG:boto:Method: HEAD 2022-09-03T06:33:10.981 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:33:10.982 DEBUG:boto:Data: 2022-09-03T06:33:10.983 DEBUG:boto:Headers: {} 2022-09-03T06:33:10.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:10.985 DEBUG:boto:Port: 8000 2022-09-03T06:33:10.986 DEBUG:boto:Params: {} 2022-09-03T06:33:10.988 DEBUG:boto:Token: None 2022-09-03T06:33:10.990 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:10 GMT /avysmw-59/ 2022-09-03T06:33:10.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GXkLI4bgbNJcOTnPxJglkQl4scs= 2022-09-03T06:33:10.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GXkLI4bgbNJcOTnPxJglkQl4scs=', 'Content-Length': '0'} 2022-09-03T06:33:10.996 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000079ba2cda74c68bc3-006312f526-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:10.998 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:33:10.999 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.000 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.001 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.001 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.002 DEBUG:boto:Method: GET 2022-09-03T06:33:11.003 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.003 DEBUG:boto:Data: 2022-09-03T06:33:11.004 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.005 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.005 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.006 DEBUG:boto:Params: {} 2022-09-03T06:33:11.007 DEBUG:boto:Token: None 2022-09-03T06:33:11.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/?versions 2022-09-03T06:33:11.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uqoiclm14WiBXHGHGBn/jtkEHRg= 2022-09-03T06:33:11.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uqoiclm14WiBXHGHGBn/jtkEHRg=', 'Content-Length': '0'} 2022-09-03T06:33:11.020 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb347f34da48a7b7-006312f527-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.021 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.024 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:11.025 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:11.026 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:11.027 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:11.027 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:33:11.028 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.029 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.030 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.031 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.032 DEBUG:boto:Method: GET 2022-09-03T06:33:11.033 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.033 DEBUG:boto:Data: 2022-09-03T06:33:11.034 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.036 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.037 DEBUG:boto:Params: {} 2022-09-03T06:33:11.039 DEBUG:boto:Token: None 2022-09-03T06:33:11.041 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/?versions 2022-09-03T06:33:11.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uqoiclm14WiBXHGHGBn/jtkEHRg= 2022-09-03T06:33:11.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uqoiclm14WiBXHGHGBn/jtkEHRg=', 'Content-Length': '0'} 2022-09-03T06:33:11.048 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090ac389cf2c31452-006312f527-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.049 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.052 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:11.053 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:11.054 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:11.056 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:11.057 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.066 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.067 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.069 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.070 DEBUG:boto:Method: GET 2022-09-03T06:33:11.070 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.078 DEBUG:boto:Data: 2022-09-03T06:33:11.079 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.080 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.081 DEBUG:boto:Params: {} 2022-09-03T06:33:11.081 DEBUG:boto:Token: None 2022-09-03T06:33:11.082 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/?versions 2022-09-03T06:33:11.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uqoiclm14WiBXHGHGBn/jtkEHRg= 2022-09-03T06:33:11.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uqoiclm14WiBXHGHGBn/jtkEHRg=', 'Content-Length': '0'} 2022-09-03T06:33:11.086 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f1580405cd96bdc-006312f527-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.089 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.091 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.091 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.092 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.093 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.093 DEBUG:boto:Method: GET 2022-09-03T06:33:11.094 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:11.094 DEBUG:boto:Data: 2022-09-03T06:33:11.095 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.096 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.097 DEBUG:boto:Params: {} 2022-09-03T06:33:11.097 DEBUG:boto:Token: None 2022-09-03T06:33:11.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/?versions 2022-09-03T06:33:11.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uqoiclm14WiBXHGHGBn/jtkEHRg= 2022-09-03T06:33:11.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uqoiclm14WiBXHGHGBn/jtkEHRg=', 'Content-Length': '0'} 2022-09-03T06:33:11.109 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cab566576bd1bc35-006312f527-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.111 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.113 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:11.114 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.116 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.118 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.120 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.121 DEBUG:boto:Method: GET 2022-09-03T06:33:11.122 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.123 DEBUG:boto:Data: 2022-09-03T06:33:11.124 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.125 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.126 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.127 DEBUG:boto:Params: {} 2022-09-03T06:33:11.128 DEBUG:boto:Token: None 2022-09-03T06:33:11.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1NOPF9FuexlINfYykRzDqfPmq4= 2022-09-03T06:33:11.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1NOPF9FuexlINfYykRzDqfPmq4=', 'Content-Length': '0'} 2022-09-03T06:33:11.138 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000002b1af5b341885b7-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.141 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.144 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.145 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.147 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.148 DEBUG:boto:Method: GET 2022-09-03T06:33:11.149 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.151 DEBUG:boto:Data: 2022-09-03T06:33:11.152 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.157 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.158 DEBUG:boto:Params: {} 2022-09-03T06:33:11.159 DEBUG:boto:Token: None 2022-09-03T06:33:11.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1NOPF9FuexlINfYykRzDqfPmq4= 2022-09-03T06:33:11.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1NOPF9FuexlINfYykRzDqfPmq4=', 'Content-Length': '0'} 2022-09-03T06:33:11.168 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000503ee8a7e33e9178-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.170 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.171 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.172 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.174 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.176 DEBUG:boto:Method: HEAD 2022-09-03T06:33:11.177 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.179 DEBUG:boto:Data: 2022-09-03T06:33:11.180 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.181 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.182 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.183 DEBUG:boto:Params: {} 2022-09-03T06:33:11.184 DEBUG:boto:Token: None 2022-09-03T06:33:11.189 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8pUSgTTxRWW3iZ7DJLFlSlFPjic= 2022-09-03T06:33:11.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8pUSgTTxRWW3iZ7DJLFlSlFPjic=', 'Content-Length': '0'} 2022-09-03T06:33:11.208 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ee708ae0839ccd1b-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.212 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.213 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.215 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.216 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.217 DEBUG:boto:Method: HEAD 2022-09-03T06:33:11.219 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.220 DEBUG:boto:Data: 2022-09-03T06:33:11.221 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.222 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.222 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.223 DEBUG:boto:Params: {} 2022-09-03T06:33:11.223 DEBUG:boto:Token: None 2022-09-03T06:33:11.224 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8pUSgTTxRWW3iZ7DJLFlSlFPjic= 2022-09-03T06:33:11.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8pUSgTTxRWW3iZ7DJLFlSlFPjic=', 'Content-Length': '0'} 2022-09-03T06:33:11.228 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000377104085bbc401f-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.229 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:11.229 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.230 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.231 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.232 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.233 DEBUG:boto:Method: GET 2022-09-03T06:33:11.234 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.235 DEBUG:boto:Data: 2022-09-03T06:33:11.236 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.237 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.238 DEBUG:boto:Params: {} 2022-09-03T06:33:11.240 DEBUG:boto:Token: None 2022-09-03T06:33:11.241 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1NOPF9FuexlINfYykRzDqfPmq4= 2022-09-03T06:33:11.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1NOPF9FuexlINfYykRzDqfPmq4=', 'Content-Length': '0'} 2022-09-03T06:33:11.249 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000064ea23f0fc16b54-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.251 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.252 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.253 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.254 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.256 DEBUG:boto:Method: GET 2022-09-03T06:33:11.259 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.260 DEBUG:boto:Data: 2022-09-03T06:33:11.262 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.264 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.266 DEBUG:boto:Params: {} 2022-09-03T06:33:11.267 DEBUG:boto:Token: None 2022-09-03T06:33:11.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:11.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1NOPF9FuexlINfYykRzDqfPmq4= 2022-09-03T06:33:11.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1NOPF9FuexlINfYykRzDqfPmq4=', 'Content-Length': '0'} 2022-09-03T06:33:11.279 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005fdd891713321039-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.280 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.281 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.283 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:11.284 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:11.292 DEBUG:boto:Method: GET 2022-09-03T06:33:11.315 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:11.317 DEBUG:boto:Data: 2022-09-03T06:33:11.320 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.322 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.323 DEBUG:boto:Params: {} 2022-09-03T06:33:11.324 DEBUG:boto:Token: None 2022-09-03T06:33:11.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/ 2022-09-03T06:33:11.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o= 2022-09-03T06:33:11.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o=', 'Content-Length': '0'} 2022-09-03T06:33:11.336 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b347688c5d26e332-006312f527-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.338 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.339 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.340 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.342 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:11.343 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:11.343 DEBUG:boto:Method: GET 2022-09-03T06:33:11.344 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:11.344 DEBUG:boto:Data: 2022-09-03T06:33:11.345 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.346 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.347 DEBUG:boto:Params: {} 2022-09-03T06:33:11.347 DEBUG:boto:Token: None 2022-09-03T06:33:11.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/ 2022-09-03T06:33:11.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o= 2022-09-03T06:33:11.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o=', 'Content-Length': '0'} 2022-09-03T06:33:11.357 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000464b516b5c7098a0-006312f527-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.359 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.360 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.361 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.361 DEBUG:boto:Method: HEAD 2022-09-03T06:33:11.362 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:11.362 DEBUG:boto:Data: 2022-09-03T06:33:11.363 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.364 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.364 DEBUG:boto:Params: {} 2022-09-03T06:33:11.365 DEBUG:boto:Token: None 2022-09-03T06:33:11.366 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1 2022-09-03T06:33:11.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lOr4/r60mDgMKHkqJe12TY6+LMc= 2022-09-03T06:33:11.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lOr4/r60mDgMKHkqJe12TY6+LMc=', 'Content-Length': '0'} 2022-09-03T06:33:11.370 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000177a18e9c40b4b5f-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.371 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.372 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.372 DEBUG:boto:Method: HEAD 2022-09-03T06:33:11.373 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:11.373 DEBUG:boto:Data: 2022-09-03T06:33:11.374 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.375 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.376 DEBUG:boto:Params: {} 2022-09-03T06:33:11.376 DEBUG:boto:Token: None 2022-09-03T06:33:11.377 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1 2022-09-03T06:33:11.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lOr4/r60mDgMKHkqJe12TY6+LMc= 2022-09-03T06:33:11.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lOr4/r60mDgMKHkqJe12TY6+LMc=', 'Content-Length': '0'} 2022-09-03T06:33:11.381 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000de51d3d72311f753-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.381 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:11.382 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.383 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.383 DEBUG:boto:Method: GET 2022-09-03T06:33:11.384 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:11.385 DEBUG:boto:Data: 2022-09-03T06:33:11.385 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.388 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.398 DEBUG:boto:Params: {} 2022-09-03T06:33:11.399 DEBUG:boto:Token: None 2022-09-03T06:33:11.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1 2022-09-03T06:33:11.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b4G2yaK8glKEOfhi7NC5BXc0KCg= 2022-09-03T06:33:11.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b4G2yaK8glKEOfhi7NC5BXc0KCg=', 'Content-Length': '0'} 2022-09-03T06:33:11.408 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001d424ee163889158-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.409 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:11.409 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:11.410 DEBUG:boto:Method: GET 2022-09-03T06:33:11.411 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:11.412 DEBUG:boto:Data: 2022-09-03T06:33:11.413 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.413 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.414 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.414 DEBUG:boto:Params: {} 2022-09-03T06:33:11.415 DEBUG:boto:Token: None 2022-09-03T06:33:11.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj1 2022-09-03T06:33:11.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b4G2yaK8glKEOfhi7NC5BXc0KCg= 2022-09-03T06:33:11.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b4G2yaK8glKEOfhi7NC5BXc0KCg=', 'Content-Length': '0'} 2022-09-03T06:33:11.420 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e720470e53b47341-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.422 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.423 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.423 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:11.424 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:11.424 DEBUG:boto:Method: GET 2022-09-03T06:33:11.426 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:11.426 DEBUG:boto:Data: 2022-09-03T06:33:11.427 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.428 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.430 DEBUG:boto:Params: {} 2022-09-03T06:33:11.431 DEBUG:boto:Token: None 2022-09-03T06:33:11.432 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/ 2022-09-03T06:33:11.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o= 2022-09-03T06:33:11.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o=', 'Content-Length': '0'} 2022-09-03T06:33:11.439 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002cdd409eb0067aa1-006312f527-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.440 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.442 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.444 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.445 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:11.446 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:11.447 DEBUG:boto:Method: GET 2022-09-03T06:33:11.448 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:11.450 DEBUG:boto:Data: 2022-09-03T06:33:11.451 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.456 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.457 DEBUG:boto:Params: {} 2022-09-03T06:33:11.458 DEBUG:boto:Token: None 2022-09-03T06:33:11.458 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/ 2022-09-03T06:33:11.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o= 2022-09-03T06:33:11.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o=', 'Content-Length': '0'} 2022-09-03T06:33:11.464 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f83501e0df588c3-006312f527-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.465 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.465 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:11.467 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.468 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.469 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.470 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.471 DEBUG:boto:Method: GET 2022-09-03T06:33:11.474 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.476 DEBUG:boto:Data: 2022-09-03T06:33:11.477 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.478 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.479 DEBUG:boto:Params: {} 2022-09-03T06:33:11.480 DEBUG:boto:Token: None 2022-09-03T06:33:11.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ldxbhcwlw4AypQ/RUa/AvXePTA8= 2022-09-03T06:33:11.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ldxbhcwlw4AypQ/RUa/AvXePTA8=', 'Content-Length': '0'} 2022-09-03T06:33:11.484 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000017f22bf9d881fe7a-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.530 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.531 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.532 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.533 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.534 DEBUG:boto:Method: GET 2022-09-03T06:33:11.535 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.536 DEBUG:boto:Data: 2022-09-03T06:33:11.538 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.539 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.540 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.540 DEBUG:boto:Params: {} 2022-09-03T06:33:11.541 DEBUG:boto:Token: None 2022-09-03T06:33:11.542 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ldxbhcwlw4AypQ/RUa/AvXePTA8= 2022-09-03T06:33:11.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ldxbhcwlw4AypQ/RUa/AvXePTA8=', 'Content-Length': '0'} 2022-09-03T06:33:11.548 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000453ac15013e91ebb-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.549 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.550 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.551 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.552 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.553 DEBUG:boto:Method: HEAD 2022-09-03T06:33:11.554 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.555 DEBUG:boto:Data: 2022-09-03T06:33:11.556 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.557 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.558 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.559 DEBUG:boto:Params: {} 2022-09-03T06:33:11.561 DEBUG:boto:Token: None 2022-09-03T06:33:11.570 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nuAno/sKXJCQ+E2LhizgvfGwOdI= 2022-09-03T06:33:11.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nuAno/sKXJCQ+E2LhizgvfGwOdI=', 'Content-Length': '0'} 2022-09-03T06:33:11.579 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f9cb8ef9df0514a0-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.582 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.591 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.594 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.595 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.596 DEBUG:boto:Method: HEAD 2022-09-03T06:33:11.597 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.598 DEBUG:boto:Data: 2022-09-03T06:33:11.599 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.600 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.601 DEBUG:boto:Params: {} 2022-09-03T06:33:11.602 DEBUG:boto:Token: None 2022-09-03T06:33:11.603 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nuAno/sKXJCQ+E2LhizgvfGwOdI= 2022-09-03T06:33:11.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nuAno/sKXJCQ+E2LhizgvfGwOdI=', 'Content-Length': '0'} 2022-09-03T06:33:11.613 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000157b1305304fa7c6-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.614 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:11.615 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.616 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.616 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.617 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.617 DEBUG:boto:Method: GET 2022-09-03T06:33:11.618 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.618 DEBUG:boto:Data: 2022-09-03T06:33:11.619 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.619 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.620 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.621 DEBUG:boto:Params: {} 2022-09-03T06:33:11.621 DEBUG:boto:Token: None 2022-09-03T06:33:11.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ldxbhcwlw4AypQ/RUa/AvXePTA8= 2022-09-03T06:33:11.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ldxbhcwlw4AypQ/RUa/AvXePTA8=', 'Content-Length': '0'} 2022-09-03T06:33:11.626 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000704434da5ac4d20f-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.626 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.627 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.628 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.628 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.629 DEBUG:boto:Method: GET 2022-09-03T06:33:11.629 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.630 DEBUG:boto:Data: 2022-09-03T06:33:11.630 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.632 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.633 DEBUG:boto:Params: {} 2022-09-03T06:33:11.633 DEBUG:boto:Token: None 2022-09-03T06:33:11.634 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:11.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ldxbhcwlw4AypQ/RUa/AvXePTA8= 2022-09-03T06:33:11.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ldxbhcwlw4AypQ/RUa/AvXePTA8=', 'Content-Length': '0'} 2022-09-03T06:33:11.638 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003c62c6f1805bd78b-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.639 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.640 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.643 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:11.644 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:11.645 DEBUG:boto:Method: GET 2022-09-03T06:33:11.646 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:11.662 DEBUG:boto:Data: 2022-09-03T06:33:11.664 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.665 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.666 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.667 DEBUG:boto:Params: {} 2022-09-03T06:33:11.668 DEBUG:boto:Token: None 2022-09-03T06:33:11.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/ 2022-09-03T06:33:11.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o= 2022-09-03T06:33:11.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o=', 'Content-Length': '0'} 2022-09-03T06:33:11.690 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063b8e2766f5586a3-006312f527-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.691 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.693 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.699 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.700 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:11.701 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:11.702 DEBUG:boto:Method: GET 2022-09-03T06:33:11.703 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:11.704 DEBUG:boto:Data: 2022-09-03T06:33:11.705 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.706 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.709 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.710 DEBUG:boto:Params: {} 2022-09-03T06:33:11.711 DEBUG:boto:Token: None 2022-09-03T06:33:11.712 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/ 2022-09-03T06:33:11.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o= 2022-09-03T06:33:11.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o=', 'Content-Length': '0'} 2022-09-03T06:33:11.719 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000010c9195fc21dc89e-006312f527-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.722 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.724 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.725 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.726 DEBUG:boto:Method: HEAD 2022-09-03T06:33:11.727 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:11.728 DEBUG:boto:Data: 2022-09-03T06:33:11.730 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.732 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.733 DEBUG:boto:Params: {} 2022-09-03T06:33:11.734 DEBUG:boto:Token: None 2022-09-03T06:33:11.735 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2 2022-09-03T06:33:11.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bz8p0TLC2ywDA6hN80zpib1lAeg= 2022-09-03T06:33:11.738 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bz8p0TLC2ywDA6hN80zpib1lAeg=', 'Content-Length': '0'} 2022-09-03T06:33:11.744 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000578465e58c6b4534-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.746 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.746 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.764 DEBUG:boto:Method: HEAD 2022-09-03T06:33:11.764 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:11.765 DEBUG:boto:Data: 2022-09-03T06:33:11.766 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.766 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.767 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.767 DEBUG:boto:Params: {} 2022-09-03T06:33:11.768 DEBUG:boto:Token: None 2022-09-03T06:33:11.769 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2 2022-09-03T06:33:11.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bz8p0TLC2ywDA6hN80zpib1lAeg= 2022-09-03T06:33:11.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bz8p0TLC2ywDA6hN80zpib1lAeg=', 'Content-Length': '0'} 2022-09-03T06:33:11.777 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004bb6ae2dddf1378e-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.779 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:11.780 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.784 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.792 DEBUG:boto:Method: GET 2022-09-03T06:33:11.793 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:11.794 DEBUG:boto:Data: 2022-09-03T06:33:11.796 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.801 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.802 DEBUG:boto:Params: {} 2022-09-03T06:33:11.803 DEBUG:boto:Token: None 2022-09-03T06:33:11.804 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2 2022-09-03T06:33:11.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JkgveKrTZlxpQ4X7aSY1DTLSx60= 2022-09-03T06:33:11.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JkgveKrTZlxpQ4X7aSY1DTLSx60=', 'Content-Length': '0'} 2022-09-03T06:33:11.811 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000668d90d19608c27e-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.814 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:11.817 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:11.818 DEBUG:boto:Method: GET 2022-09-03T06:33:11.819 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:11.820 DEBUG:boto:Data: 2022-09-03T06:33:11.821 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.823 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.824 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.824 DEBUG:boto:Params: {} 2022-09-03T06:33:11.825 DEBUG:boto:Token: None 2022-09-03T06:33:11.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj2 2022-09-03T06:33:11.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JkgveKrTZlxpQ4X7aSY1DTLSx60= 2022-09-03T06:33:11.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JkgveKrTZlxpQ4X7aSY1DTLSx60=', 'Content-Length': '0'} 2022-09-03T06:33:11.835 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fa5dbba656224493-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.839 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.841 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.842 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:11.848 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:11.853 DEBUG:boto:Method: GET 2022-09-03T06:33:11.853 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:11.854 DEBUG:boto:Data: 2022-09-03T06:33:11.854 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.855 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.855 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.856 DEBUG:boto:Params: {} 2022-09-03T06:33:11.857 DEBUG:boto:Token: None 2022-09-03T06:33:11.857 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/ 2022-09-03T06:33:11.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o= 2022-09-03T06:33:11.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o=', 'Content-Length': '0'} 2022-09-03T06:33:11.882 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000781e71f2cd1e0cb2-006312f527-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.884 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.885 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:11.901 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:11.902 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:11.903 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:11.904 DEBUG:boto:Method: GET 2022-09-03T06:33:11.905 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:11.918 DEBUG:boto:Data: 2022-09-03T06:33:11.919 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.919 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.920 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.920 DEBUG:boto:Params: {} 2022-09-03T06:33:11.921 DEBUG:boto:Token: None 2022-09-03T06:33:11.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/ 2022-09-03T06:33:11.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o= 2022-09-03T06:33:11.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X91+qow3czhYB3HhVPpftIR1Y6o=', 'Content-Length': '0'} 2022-09-03T06:33:11.935 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088251820d613f215-006312f527-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.935 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:11.936 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:11.937 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:11.938 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:11.939 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.940 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.941 DEBUG:boto:Method: GET 2022-09-03T06:33:11.942 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.943 DEBUG:boto:Data: 2022-09-03T06:33:11.944 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:11.945 DEBUG:boto:Port: 8001 2022-09-03T06:33:11.946 DEBUG:boto:Params: {} 2022-09-03T06:33:11.947 DEBUG:boto:Token: None 2022-09-03T06:33:11.948 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lgPUn3nbVUB6RAQJLyZg0/P9g5E= 2022-09-03T06:33:11.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lgPUn3nbVUB6RAQJLyZg0/P9g5E=', 'Content-Length': '0'} 2022-09-03T06:33:11.954 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dbb6265f384d67f6-006312f527-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.956 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:11.957 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:11.958 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.960 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.962 DEBUG:boto:Method: GET 2022-09-03T06:33:11.963 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.964 DEBUG:boto:Data: 2022-09-03T06:33:11.965 DEBUG:boto:Headers: {} 2022-09-03T06:33:11.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:11.975 DEBUG:boto:Port: 8000 2022-09-03T06:33:11.981 DEBUG:boto:Params: {} 2022-09-03T06:33:11.982 DEBUG:boto:Token: None 2022-09-03T06:33:11.983 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:11 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lgPUn3nbVUB6RAQJLyZg0/P9g5E= 2022-09-03T06:33:11.986 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lgPUn3nbVUB6RAQJLyZg0/P9g5E=', 'Content-Length': '0'} 2022-09-03T06:33:11.993 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000042b5a0f139dd23af-006312f527-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:11.994 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:11.995 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:11.996 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.996 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.997 DEBUG:boto:Method: HEAD 2022-09-03T06:33:11.998 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:11.998 DEBUG:boto:Data: 2022-09-03T06:33:11.999 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.000 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.000 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.001 DEBUG:boto:Params: {} 2022-09-03T06:33:12.002 DEBUG:boto:Token: None 2022-09-03T06:33:12.019 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QY9FNLML+W2ICqD+2vN1xxXRL0w= 2022-09-03T06:33:12.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QY9FNLML+W2ICqD+2vN1xxXRL0w=', 'Content-Length': '0'} 2022-09-03T06:33:12.033 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e5e58bc86ae5bf68-006312f528-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.035 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:12.036 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:12.037 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.039 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.040 DEBUG:boto:Method: HEAD 2022-09-03T06:33:12.040 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.041 DEBUG:boto:Data: 2022-09-03T06:33:12.043 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.045 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.046 DEBUG:boto:Params: {} 2022-09-03T06:33:12.047 DEBUG:boto:Token: None 2022-09-03T06:33:12.048 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QY9FNLML+W2ICqD+2vN1xxXRL0w= 2022-09-03T06:33:12.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QY9FNLML+W2ICqD+2vN1xxXRL0w=', 'Content-Length': '0'} 2022-09-03T06:33:12.069 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000de29cea39cd71bc7-006312f528-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.074 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:12.075 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:12.076 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:12.080 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.081 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.082 DEBUG:boto:Method: GET 2022-09-03T06:33:12.083 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.084 DEBUG:boto:Data: 2022-09-03T06:33:12.085 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.086 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.088 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.091 DEBUG:boto:Params: {} 2022-09-03T06:33:12.092 DEBUG:boto:Token: None 2022-09-03T06:33:12.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zOjETyeLISHIgg0yfJdJ5EFnASg= 2022-09-03T06:33:12.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zOjETyeLISHIgg0yfJdJ5EFnASg=', 'Content-Length': '0'} 2022-09-03T06:33:12.115 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009638e55089cce066-006312f528-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.118 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:12.119 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:12.120 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.121 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.122 DEBUG:boto:Method: GET 2022-09-03T06:33:12.125 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.127 DEBUG:boto:Data: 2022-09-03T06:33:12.128 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.129 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.130 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.132 DEBUG:boto:Params: {} 2022-09-03T06:33:12.133 DEBUG:boto:Token: None 2022-09-03T06:33:12.136 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:12.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zOjETyeLISHIgg0yfJdJ5EFnASg= 2022-09-03T06:33:12.141 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zOjETyeLISHIgg0yfJdJ5EFnASg=', 'Content-Length': '0'} 2022-09-03T06:33:12.161 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000778b75d46b9d8fca-006312f528-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.164 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.165 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.166 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:12.166 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:12.168 DEBUG:boto:Method: GET 2022-09-03T06:33:12.169 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:12.169 DEBUG:boto:Data: 2022-09-03T06:33:12.171 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.173 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.177 DEBUG:boto:Params: {} 2022-09-03T06:33:12.178 DEBUG:boto:Token: None 2022-09-03T06:33:12.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM= 2022-09-03T06:33:12.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM=', 'Content-Length': '0'} 2022-09-03T06:33:12.185 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cbf664178a7456fe-006312f528-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.186 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.188 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.189 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.191 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:12.193 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:12.195 DEBUG:boto:Method: GET 2022-09-03T06:33:12.196 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:12.197 DEBUG:boto:Data: 2022-09-03T06:33:12.197 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.198 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.199 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.199 DEBUG:boto:Params: {} 2022-09-03T06:33:12.201 DEBUG:boto:Token: None 2022-09-03T06:33:12.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM= 2022-09-03T06:33:12.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM=', 'Content-Length': '0'} 2022-09-03T06:33:12.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e453401b99e256c9-006312f528-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.220 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.222 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:12.223 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:12.224 DEBUG:boto:Method: HEAD 2022-09-03T06:33:12.225 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:12.226 DEBUG:boto:Data: 2022-09-03T06:33:12.227 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.228 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.229 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.230 DEBUG:boto:Params: {} 2022-09-03T06:33:12.231 DEBUG:boto:Token: None 2022-09-03T06:33:12.232 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj3 2022-09-03T06:33:12.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DIF9INAEgxw80SN0grSsZw0Efxw= 2022-09-03T06:33:12.236 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DIF9INAEgxw80SN0grSsZw0Efxw=', 'Content-Length': '0'} 2022-09-03T06:33:12.238 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001fd538aecfa451e3-006312f528-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.240 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:12.241 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:12.242 DEBUG:boto:Method: HEAD 2022-09-03T06:33:12.243 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:12.244 DEBUG:boto:Data: 2022-09-03T06:33:12.245 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.247 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.248 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.249 DEBUG:boto:Params: {} 2022-09-03T06:33:12.252 DEBUG:boto:Token: None 2022-09-03T06:33:12.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj3 2022-09-03T06:33:12.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DIF9INAEgxw80SN0grSsZw0Efxw= 2022-09-03T06:33:12.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DIF9INAEgxw80SN0grSsZw0Efxw=', 'Content-Length': '0'} 2022-09-03T06:33:12.263 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000034a1ae3872e3ff1d-006312f528-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.264 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:12.265 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:12.267 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:12.268 DEBUG:boto:Method: GET 2022-09-03T06:33:12.269 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:12.270 DEBUG:boto:Data: 2022-09-03T06:33:12.271 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.272 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.274 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.275 DEBUG:boto:Params: {} 2022-09-03T06:33:12.275 DEBUG:boto:Token: None 2022-09-03T06:33:12.276 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj3 2022-09-03T06:33:12.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zG7M7VDRq0h9Ofu1gWi/5o7XxM0= 2022-09-03T06:33:12.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zG7M7VDRq0h9Ofu1gWi/5o7XxM0=', 'Content-Length': '0'} 2022-09-03T06:33:12.289 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000566537bcc4c932d9-006312f528-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.292 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:12.293 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:12.294 DEBUG:boto:Method: GET 2022-09-03T06:33:12.295 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:12.296 DEBUG:boto:Data: 2022-09-03T06:33:12.297 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.304 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.305 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.305 DEBUG:boto:Params: {} 2022-09-03T06:33:12.306 DEBUG:boto:Token: None 2022-09-03T06:33:12.307 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj3 2022-09-03T06:33:12.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zG7M7VDRq0h9Ofu1gWi/5o7XxM0= 2022-09-03T06:33:12.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zG7M7VDRq0h9Ofu1gWi/5o7XxM0=', 'Content-Length': '0'} 2022-09-03T06:33:12.321 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e27d8fb338d7dd32-006312f528-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.323 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.323 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.324 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:12.325 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:12.325 DEBUG:boto:Method: GET 2022-09-03T06:33:12.326 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:12.327 DEBUG:boto:Data: 2022-09-03T06:33:12.328 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.330 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.330 DEBUG:boto:Params: {} 2022-09-03T06:33:12.332 DEBUG:boto:Token: None 2022-09-03T06:33:12.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM= 2022-09-03T06:33:12.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM=', 'Content-Length': '0'} 2022-09-03T06:33:12.339 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000083c2c579c15c9450-006312f528-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.341 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.348 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.349 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.350 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:12.350 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:12.351 DEBUG:boto:Method: GET 2022-09-03T06:33:12.351 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:12.352 DEBUG:boto:Data: 2022-09-03T06:33:12.353 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.353 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.354 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.354 DEBUG:boto:Params: {} 2022-09-03T06:33:12.355 DEBUG:boto:Token: None 2022-09-03T06:33:12.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM= 2022-09-03T06:33:12.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM=', 'Content-Length': '0'} 2022-09-03T06:33:12.360 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c99f455ec89f807-006312f528-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.403 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.404 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:12.405 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.406 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.421 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.423 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.424 DEBUG:boto:Method: GET 2022-09-03T06:33:12.425 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.426 DEBUG:boto:Data: 2022-09-03T06:33:12.427 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.429 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.430 DEBUG:boto:Params: {} 2022-09-03T06:33:12.431 DEBUG:boto:Token: None 2022-09-03T06:33:12.432 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9R0LturKE4jBBF0s4I1roJ+pTAI= 2022-09-03T06:33:12.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9R0LturKE4jBBF0s4I1roJ+pTAI=', 'Content-Length': '0'} 2022-09-03T06:33:12.437 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cfb2511c49106d45-006312f528-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.438 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.438 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.439 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.439 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.440 DEBUG:boto:Method: GET 2022-09-03T06:33:12.441 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.441 DEBUG:boto:Data: 2022-09-03T06:33:12.442 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.443 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.443 DEBUG:boto:Params: {} 2022-09-03T06:33:12.444 DEBUG:boto:Token: None 2022-09-03T06:33:12.444 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9R0LturKE4jBBF0s4I1roJ+pTAI= 2022-09-03T06:33:12.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9R0LturKE4jBBF0s4I1roJ+pTAI=', 'Content-Length': '0'} 2022-09-03T06:33:12.449 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000121a97805736e0dd-006312f528-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.451 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.452 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.453 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.454 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.455 DEBUG:boto:Method: HEAD 2022-09-03T06:33:12.456 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.457 DEBUG:boto:Data: 2022-09-03T06:33:12.458 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.459 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.460 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.461 DEBUG:boto:Params: {} 2022-09-03T06:33:12.462 DEBUG:boto:Token: None 2022-09-03T06:33:12.463 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:znU1NoZr6aobYEEsqtM7JKpc97s= 2022-09-03T06:33:12.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:znU1NoZr6aobYEEsqtM7JKpc97s=', 'Content-Length': '0'} 2022-09-03T06:33:12.468 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000026d729455450e814-006312f528-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.469 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.470 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.471 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.472 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.474 DEBUG:boto:Method: HEAD 2022-09-03T06:33:12.475 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.482 DEBUG:boto:Data: 2022-09-03T06:33:12.484 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.487 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.488 DEBUG:boto:Params: {} 2022-09-03T06:33:12.492 DEBUG:boto:Token: None 2022-09-03T06:33:12.495 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:znU1NoZr6aobYEEsqtM7JKpc97s= 2022-09-03T06:33:12.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:znU1NoZr6aobYEEsqtM7JKpc97s=', 'Content-Length': '0'} 2022-09-03T06:33:12.501 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006d7fb6128117d168-006312f528-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.505 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:12.506 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.507 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.507 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.509 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.510 DEBUG:boto:Method: GET 2022-09-03T06:33:12.510 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.511 DEBUG:boto:Data: 2022-09-03T06:33:12.511 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.524 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.524 DEBUG:boto:Params: {} 2022-09-03T06:33:12.525 DEBUG:boto:Token: None 2022-09-03T06:33:12.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9R0LturKE4jBBF0s4I1roJ+pTAI= 2022-09-03T06:33:12.529 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9R0LturKE4jBBF0s4I1roJ+pTAI=', 'Content-Length': '0'} 2022-09-03T06:33:12.533 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fa1abd55faa18f89-006312f528-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.534 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.535 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.536 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.536 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.537 DEBUG:boto:Method: GET 2022-09-03T06:33:12.538 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.539 DEBUG:boto:Data: 2022-09-03T06:33:12.540 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.541 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.542 DEBUG:boto:Params: {} 2022-09-03T06:33:12.543 DEBUG:boto:Token: None 2022-09-03T06:33:12.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:12.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9R0LturKE4jBBF0s4I1roJ+pTAI= 2022-09-03T06:33:12.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9R0LturKE4jBBF0s4I1roJ+pTAI=', 'Content-Length': '0'} 2022-09-03T06:33:12.551 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001b825402a4aa2fbc-006312f528-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.552 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.554 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.555 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:12.556 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:12.557 DEBUG:boto:Method: GET 2022-09-03T06:33:12.558 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:12.558 DEBUG:boto:Data: 2022-09-03T06:33:12.559 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.560 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.561 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.561 DEBUG:boto:Params: {} 2022-09-03T06:33:12.563 DEBUG:boto:Token: None 2022-09-03T06:33:12.564 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.565 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM= 2022-09-03T06:33:12.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM=', 'Content-Length': '0'} 2022-09-03T06:33:12.572 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a5fa419d89ed4777-006312f528-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.573 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.574 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.575 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.576 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:12.576 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:12.577 DEBUG:boto:Method: GET 2022-09-03T06:33:12.578 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:12.578 DEBUG:boto:Data: 2022-09-03T06:33:12.579 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.580 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.581 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.582 DEBUG:boto:Params: {} 2022-09-03T06:33:12.583 DEBUG:boto:Token: None 2022-09-03T06:33:12.584 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM= 2022-09-03T06:33:12.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM=', 'Content-Length': '0'} 2022-09-03T06:33:12.589 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d6ddb5770ff5c648-006312f528-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.590 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.591 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.592 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.592 DEBUG:boto:Method: HEAD 2022-09-03T06:33:12.593 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:12.593 DEBUG:boto:Data: 2022-09-03T06:33:12.594 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.594 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.595 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.595 DEBUG:boto:Params: {} 2022-09-03T06:33:12.596 DEBUG:boto:Token: None 2022-09-03T06:33:12.597 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4 2022-09-03T06:33:12.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BiC1mxkD3t/J/bPSruJQ5bFF4wA= 2022-09-03T06:33:12.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BiC1mxkD3t/J/bPSruJQ5bFF4wA=', 'Content-Length': '0'} 2022-09-03T06:33:12.601 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008bb7520931faac74-006312f528-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.602 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.602 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.603 DEBUG:boto:Method: HEAD 2022-09-03T06:33:12.604 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:12.604 DEBUG:boto:Data: 2022-09-03T06:33:12.605 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.605 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.606 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.606 DEBUG:boto:Params: {} 2022-09-03T06:33:12.607 DEBUG:boto:Token: None 2022-09-03T06:33:12.607 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4 2022-09-03T06:33:12.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BiC1mxkD3t/J/bPSruJQ5bFF4wA= 2022-09-03T06:33:12.610 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BiC1mxkD3t/J/bPSruJQ5bFF4wA=', 'Content-Length': '0'} 2022-09-03T06:33:12.616 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009f586e705015ed02-006312f528-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.617 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:12.623 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.623 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.624 DEBUG:boto:Method: GET 2022-09-03T06:33:12.625 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:12.625 DEBUG:boto:Data: 2022-09-03T06:33:12.633 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.634 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.635 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.635 DEBUG:boto:Params: {} 2022-09-03T06:33:12.636 DEBUG:boto:Token: None 2022-09-03T06:33:12.637 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4 2022-09-03T06:33:12.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hqWD+1iffUwKPgKJWP1fDZk0qNI= 2022-09-03T06:33:12.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hqWD+1iffUwKPgKJWP1fDZk0qNI=', 'Content-Length': '0'} 2022-09-03T06:33:12.645 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007cb609ae3394e01b-006312f528-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.683 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:12.692 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:12.692 DEBUG:boto:Method: GET 2022-09-03T06:33:12.696 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:12.696 DEBUG:boto:Data: 2022-09-03T06:33:12.697 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.697 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.735 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.737 DEBUG:boto:Params: {} 2022-09-03T06:33:12.737 DEBUG:boto:Token: None 2022-09-03T06:33:12.738 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/obj4 2022-09-03T06:33:12.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hqWD+1iffUwKPgKJWP1fDZk0qNI= 2022-09-03T06:33:12.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hqWD+1iffUwKPgKJWP1fDZk0qNI=', 'Content-Length': '0'} 2022-09-03T06:33:12.745 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005de6f1db5286a2dd-006312f528-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.747 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.747 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.748 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:12.748 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:12.749 DEBUG:boto:Method: GET 2022-09-03T06:33:12.751 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:12.752 DEBUG:boto:Data: 2022-09-03T06:33:12.752 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.753 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.753 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.754 DEBUG:boto:Params: {} 2022-09-03T06:33:12.754 DEBUG:boto:Token: None 2022-09-03T06:33:12.755 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM= 2022-09-03T06:33:12.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM=', 'Content-Length': '0'} 2022-09-03T06:33:12.765 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bc33415721437807-006312f528-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.766 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.767 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.768 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.768 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:12.769 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:12.769 DEBUG:boto:Method: GET 2022-09-03T06:33:12.771 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:12.772 DEBUG:boto:Data: 2022-09-03T06:33:12.773 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.773 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:12.774 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.774 DEBUG:boto:Params: {} 2022-09-03T06:33:12.775 DEBUG:boto:Token: None 2022-09-03T06:33:12.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM= 2022-09-03T06:33:12.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m2Q6cQLAg8d9U+bdDRu2cLEPsM=', 'Content-Length': '0'} 2022-09-03T06:33:12.785 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1bf8da406dd34a1-006312f528-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.795 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.796 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-59 zones={test-zone3, test-zone1} 2022-09-03T06:33:12.796 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-59 zones={test-zone3, test-zone2} 2022-09-03T06:33:12.797 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.797 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.810 DEBUG:boto:Method: HEAD 2022-09-03T06:33:12.811 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:33:12.813 DEBUG:boto:Data: 2022-09-03T06:33:12.814 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.818 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.819 DEBUG:boto:Params: {} 2022-09-03T06:33:12.820 DEBUG:boto:Token: None 2022-09-03T06:33:12.820 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fQ9UWdMHboVXCtrcYonEawGUi2U= 2022-09-03T06:33:12.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fQ9UWdMHboVXCtrcYonEawGUi2U=', 'Content-Length': '0'} 2022-09-03T06:33:12.826 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000032d3b55a87007317-006312f528-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.827 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.828 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.830 DEBUG:boto:Method: HEAD 2022-09-03T06:33:12.831 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:33:12.832 DEBUG:boto:Data: 2022-09-03T06:33:12.832 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.833 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:12.833 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.834 DEBUG:boto:Params: {} 2022-09-03T06:33:12.835 DEBUG:boto:Token: None 2022-09-03T06:33:12.836 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/ 2022-09-03T06:33:12.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fQ9UWdMHboVXCtrcYonEawGUi2U= 2022-09-03T06:33:12.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fQ9UWdMHboVXCtrcYonEawGUi2U=', 'Content-Length': '0'} 2022-09-03T06:33:12.844 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006c9b6b11ecb8e0d9-006312f528-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.845 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:33:12.845 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.846 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.847 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.848 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.849 DEBUG:boto:Method: GET 2022-09-03T06:33:12.849 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.850 DEBUG:boto:Data: 2022-09-03T06:33:12.850 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.851 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.852 DEBUG:boto:Params: {} 2022-09-03T06:33:12.853 DEBUG:boto:Token: None 2022-09-03T06:33:12.862 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/?versions 2022-09-03T06:33:12.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xdJf+TH6hPeIFBBdyTBp2/TVO5Q= 2022-09-03T06:33:12.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xdJf+TH6hPeIFBBdyTBp2/TVO5Q=', 'Content-Length': '0'} 2022-09-03T06:33:12.876 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000486fa626124cc9e-006312f528-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.878 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.880 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:12.881 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:12.881 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:12.882 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:12.883 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:33:12.885 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.886 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.887 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.888 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.890 DEBUG:boto:Method: GET 2022-09-03T06:33:12.891 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.892 DEBUG:boto:Data: 2022-09-03T06:33:12.892 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.893 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:12.894 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.896 DEBUG:boto:Params: {} 2022-09-03T06:33:12.897 DEBUG:boto:Token: None 2022-09-03T06:33:12.898 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/?versions 2022-09-03T06:33:12.899 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xdJf+TH6hPeIFBBdyTBp2/TVO5Q= 2022-09-03T06:33:12.900 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xdJf+TH6hPeIFBBdyTBp2/TVO5Q=', 'Content-Length': '0'} 2022-09-03T06:33:12.918 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000014c3d9e8391f4da8-006312f528-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.919 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.921 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:12.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:12.923 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:12.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:12.926 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.927 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.928 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.929 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.930 DEBUG:boto:Method: GET 2022-09-03T06:33:12.932 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.933 DEBUG:boto:Data: 2022-09-03T06:33:12.934 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.935 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.935 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.936 DEBUG:boto:Params: {} 2022-09-03T06:33:12.936 DEBUG:boto:Token: None 2022-09-03T06:33:12.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/?versions 2022-09-03T06:33:12.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xdJf+TH6hPeIFBBdyTBp2/TVO5Q= 2022-09-03T06:33:12.945 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xdJf+TH6hPeIFBBdyTBp2/TVO5Q=', 'Content-Length': '0'} 2022-09-03T06:33:12.949 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b1f3069fcca68df8-006312f528-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.952 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.954 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:12.955 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:12.957 DEBUG:boto:path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.958 DEBUG:boto:auth_path=/avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.960 DEBUG:boto:Method: GET 2022-09-03T06:33:12.961 DEBUG:boto:Path: /avysmw-59/?versions&max-keys=999 2022-09-03T06:33:12.962 DEBUG:boto:Data: 2022-09-03T06:33:12.964 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:12.966 DEBUG:boto:Port: 8000 2022-09-03T06:33:12.967 DEBUG:boto:Params: {} 2022-09-03T06:33:12.968 DEBUG:boto:Token: None 2022-09-03T06:33:12.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:12 GMT /avysmw-59/?versions 2022-09-03T06:33:12.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xdJf+TH6hPeIFBBdyTBp2/TVO5Q= 2022-09-03T06:33:12.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xdJf+TH6hPeIFBBdyTBp2/TVO5Q=', 'Content-Length': '0'} 2022-09-03T06:33:12.982 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fbfaaab19bd2420e-006312f528-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:12.983 DEBUG:boto:b'avysmw-59999falseobj1nulltrue2022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:12.985 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:12.986 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:12.987 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:12.988 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:12.989 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:12.990 DEBUG:boto:Method: GET 2022-09-03T06:33:12.992 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:12.993 DEBUG:boto:Data: 2022-09-03T06:33:12.994 DEBUG:boto:Headers: {} 2022-09-03T06:33:12.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:12.996 DEBUG:boto:Port: 8001 2022-09-03T06:33:12.998 DEBUG:boto:Params: {} 2022-09-03T06:33:12.999 DEBUG:boto:Token: None 2022-09-03T06:33:13.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.002 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D1AZEKsx2bie55Sku3sXrw8Q3Ic= 2022-09-03T06:33:13.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D1AZEKsx2bie55Sku3sXrw8Q3Ic=', 'Content-Length': '0'} 2022-09-03T06:33:13.010 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000062b761b86007e6d-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.012 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:13.013 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:13.014 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.015 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.017 DEBUG:boto:Method: GET 2022-09-03T06:33:13.018 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.020 DEBUG:boto:Data: 2022-09-03T06:33:13.021 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.022 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.023 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.024 DEBUG:boto:Params: {} 2022-09-03T06:33:13.026 DEBUG:boto:Token: None 2022-09-03T06:33:13.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D1AZEKsx2bie55Sku3sXrw8Q3Ic= 2022-09-03T06:33:13.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D1AZEKsx2bie55Sku3sXrw8Q3Ic=', 'Content-Length': '0'} 2022-09-03T06:33:13.033 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004ccc376e3366a52a-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.034 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:13.035 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:13.035 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.036 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.036 DEBUG:boto:Method: HEAD 2022-09-03T06:33:13.037 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.038 DEBUG:boto:Data: 2022-09-03T06:33:13.039 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.040 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.041 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.041 DEBUG:boto:Params: {} 2022-09-03T06:33:13.042 DEBUG:boto:Token: None 2022-09-03T06:33:13.043 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AJ7ZknjjRU2sgyjlLEvWIgi/KjI= 2022-09-03T06:33:13.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AJ7ZknjjRU2sgyjlLEvWIgi/KjI=', 'Content-Length': '0'} 2022-09-03T06:33:13.046 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cd099f56b154b54e-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.047 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:13.048 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:13.048 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.049 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.049 DEBUG:boto:Method: HEAD 2022-09-03T06:33:13.050 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.050 DEBUG:boto:Data: 2022-09-03T06:33:13.051 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.052 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.053 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.053 DEBUG:boto:Params: {} 2022-09-03T06:33:13.054 DEBUG:boto:Token: None 2022-09-03T06:33:13.055 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AJ7ZknjjRU2sgyjlLEvWIgi/KjI= 2022-09-03T06:33:13.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AJ7ZknjjRU2sgyjlLEvWIgi/KjI=', 'Content-Length': '0'} 2022-09-03T06:33:13.058 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c340504459ad586b-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.059 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:13.060 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:13.060 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:13.061 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.061 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.062 DEBUG:boto:Method: GET 2022-09-03T06:33:13.063 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.063 DEBUG:boto:Data: 2022-09-03T06:33:13.064 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.064 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.065 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.066 DEBUG:boto:Params: {} 2022-09-03T06:33:13.066 DEBUG:boto:Token: None 2022-09-03T06:33:13.067 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D1AZEKsx2bie55Sku3sXrw8Q3Ic= 2022-09-03T06:33:13.068 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D1AZEKsx2bie55Sku3sXrw8Q3Ic=', 'Content-Length': '0'} 2022-09-03T06:33:13.074 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e37e61036e304f59-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.075 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:13.076 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:13.077 DEBUG:boto:path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.077 DEBUG:boto:auth_path=/avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.078 DEBUG:boto:Method: GET 2022-09-03T06:33:13.078 DEBUG:boto:Path: /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.079 DEBUG:boto:Data: 2022-09-03T06:33:13.079 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.080 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.080 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.081 DEBUG:boto:Params: {} 2022-09-03T06:33:13.082 DEBUG:boto:Token: None 2022-09-03T06:33:13.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1?versionId=null 2022-09-03T06:33:13.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D1AZEKsx2bie55Sku3sXrw8Q3Ic= 2022-09-03T06:33:13.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D1AZEKsx2bie55Sku3sXrw8Q3Ic=', 'Content-Length': '0'} 2022-09-03T06:33:13.086 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c97139b42b452925-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.087 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:13.088 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:13.088 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:13.089 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:13.089 DEBUG:boto:Method: GET 2022-09-03T06:33:13.090 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:13.091 DEBUG:boto:Data: 2022-09-03T06:33:13.091 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.092 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.093 DEBUG:boto:Params: {} 2022-09-03T06:33:13.094 DEBUG:boto:Token: None 2022-09-03T06:33:13.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/ 2022-09-03T06:33:13.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE= 2022-09-03T06:33:13.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE=', 'Content-Length': '0'} 2022-09-03T06:33:13.102 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000059acbad84f8c2ea9-006312f529-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.146 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:13.147 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:13.152 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:13.153 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:13.153 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:13.154 DEBUG:boto:Method: GET 2022-09-03T06:33:13.155 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:13.155 DEBUG:boto:Data: 2022-09-03T06:33:13.156 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.157 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.157 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.158 DEBUG:boto:Params: {} 2022-09-03T06:33:13.158 DEBUG:boto:Token: None 2022-09-03T06:33:13.159 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/ 2022-09-03T06:33:13.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE= 2022-09-03T06:33:13.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE=', 'Content-Length': '0'} 2022-09-03T06:33:13.170 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000624516ee42b617d6-006312f529-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.172 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:13.173 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:13.173 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:13.174 DEBUG:boto:Method: HEAD 2022-09-03T06:33:13.175 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:13.175 DEBUG:boto:Data: 2022-09-03T06:33:13.176 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.177 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.177 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.178 DEBUG:boto:Params: {} 2022-09-03T06:33:13.178 DEBUG:boto:Token: None 2022-09-03T06:33:13.179 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1 2022-09-03T06:33:13.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R1VN+UQmqpdKkQjjK+NK2jyUHRk= 2022-09-03T06:33:13.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R1VN+UQmqpdKkQjjK+NK2jyUHRk=', 'Content-Length': '0'} 2022-09-03T06:33:13.188 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e9a9a8bfdc68c950-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.189 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:13.190 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:13.192 DEBUG:boto:Method: HEAD 2022-09-03T06:33:13.193 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:13.194 DEBUG:boto:Data: 2022-09-03T06:33:13.196 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.197 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.197 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.198 DEBUG:boto:Params: {} 2022-09-03T06:33:13.198 DEBUG:boto:Token: None 2022-09-03T06:33:13.199 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1 2022-09-03T06:33:13.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R1VN+UQmqpdKkQjjK+NK2jyUHRk= 2022-09-03T06:33:13.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R1VN+UQmqpdKkQjjK+NK2jyUHRk=', 'Content-Length': '0'} 2022-09-03T06:33:13.203 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000011792241098b7d5c-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.204 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:13.205 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:13.205 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:13.222 DEBUG:boto:Method: GET 2022-09-03T06:33:13.224 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:13.225 DEBUG:boto:Data: 2022-09-03T06:33:13.226 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.233 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.238 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.239 DEBUG:boto:Params: {} 2022-09-03T06:33:13.241 DEBUG:boto:Token: None 2022-09-03T06:33:13.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1 2022-09-03T06:33:13.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mTvnzYT8iiAYbjPAiagJZyPXgAo= 2022-09-03T06:33:13.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mTvnzYT8iiAYbjPAiagJZyPXgAo=', 'Content-Length': '0'} 2022-09-03T06:33:13.252 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009278c8b625a18864-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.264 DEBUG:boto:path=/avysmw-59/obj1 2022-09-03T06:33:13.275 DEBUG:boto:auth_path=/avysmw-59/obj1 2022-09-03T06:33:13.277 DEBUG:boto:Method: GET 2022-09-03T06:33:13.278 DEBUG:boto:Path: /avysmw-59/obj1 2022-09-03T06:33:13.279 DEBUG:boto:Data: 2022-09-03T06:33:13.280 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.281 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.282 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.283 DEBUG:boto:Params: {} 2022-09-03T06:33:13.284 DEBUG:boto:Token: None 2022-09-03T06:33:13.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj1 2022-09-03T06:33:13.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mTvnzYT8iiAYbjPAiagJZyPXgAo= 2022-09-03T06:33:13.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mTvnzYT8iiAYbjPAiagJZyPXgAo=', 'Content-Length': '0'} 2022-09-03T06:33:13.302 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b816a135cb0c05a0-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.307 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:13.308 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:13.309 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:13.315 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:13.316 DEBUG:boto:Method: GET 2022-09-03T06:33:13.317 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:13.322 DEBUG:boto:Data: 2022-09-03T06:33:13.323 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.324 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.325 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.326 DEBUG:boto:Params: {} 2022-09-03T06:33:13.327 DEBUG:boto:Token: None 2022-09-03T06:33:13.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/ 2022-09-03T06:33:13.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE= 2022-09-03T06:33:13.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE=', 'Content-Length': '0'} 2022-09-03T06:33:13.338 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000279231a4bfaa32ef-006312f529-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.345 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:13.347 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:13.348 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:13.349 DEBUG:boto:path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:13.350 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj1 2022-09-03T06:33:13.352 DEBUG:boto:Method: GET 2022-09-03T06:33:13.353 DEBUG:boto:Path: /avysmw-59/?prefix=obj1 2022-09-03T06:33:13.354 DEBUG:boto:Data: 2022-09-03T06:33:13.355 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.356 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.358 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.359 DEBUG:boto:Params: {} 2022-09-03T06:33:13.360 DEBUG:boto:Token: None 2022-09-03T06:33:13.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/ 2022-09-03T06:33:13.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE= 2022-09-03T06:33:13.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE=', 'Content-Length': '0'} 2022-09-03T06:33:13.373 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9e23cd7c08b8e6b-006312f529-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.375 DEBUG:boto:b'avysmw-59obj11000falseobj12022-09-03T06:32:26.663Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:13.377 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:13.380 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.384 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.392 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.393 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.394 DEBUG:boto:Method: GET 2022-09-03T06:33:13.394 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.395 DEBUG:boto:Data: 2022-09-03T06:33:13.398 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.404 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.405 DEBUG:boto:Params: {} 2022-09-03T06:33:13.406 DEBUG:boto:Token: None 2022-09-03T06:33:13.410 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KNFpcX76fdv3H33QeKjrU5xiPrY= 2022-09-03T06:33:13.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KNFpcX76fdv3H33QeKjrU5xiPrY=', 'Content-Length': '0'} 2022-09-03T06:33:13.424 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000abc8475e246c7890-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.425 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.427 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.429 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.431 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.433 DEBUG:boto:Method: GET 2022-09-03T06:33:13.439 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.440 DEBUG:boto:Data: 2022-09-03T06:33:13.441 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.445 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.447 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.448 DEBUG:boto:Params: {} 2022-09-03T06:33:13.449 DEBUG:boto:Token: None 2022-09-03T06:33:13.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KNFpcX76fdv3H33QeKjrU5xiPrY= 2022-09-03T06:33:13.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KNFpcX76fdv3H33QeKjrU5xiPrY=', 'Content-Length': '0'} 2022-09-03T06:33:13.457 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000387eaee0a81c6d49-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.460 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.461 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.468 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.469 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.483 DEBUG:boto:Method: HEAD 2022-09-03T06:33:13.484 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.485 DEBUG:boto:Data: 2022-09-03T06:33:13.485 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.487 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.488 DEBUG:boto:Params: {} 2022-09-03T06:33:13.492 DEBUG:boto:Token: None 2022-09-03T06:33:13.493 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C1e4Gq/RQEMvxKSbzdfXaZFpc3Q= 2022-09-03T06:33:13.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C1e4Gq/RQEMvxKSbzdfXaZFpc3Q=', 'Content-Length': '0'} 2022-09-03T06:33:13.497 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000783a00626ef84789-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.499 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.499 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.500 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.502 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.502 DEBUG:boto:Method: HEAD 2022-09-03T06:33:13.503 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.504 DEBUG:boto:Data: 2022-09-03T06:33:13.505 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.506 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.516 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.519 DEBUG:boto:Params: {} 2022-09-03T06:33:13.520 DEBUG:boto:Token: None 2022-09-03T06:33:13.520 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.521 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C1e4Gq/RQEMvxKSbzdfXaZFpc3Q= 2022-09-03T06:33:13.522 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C1e4Gq/RQEMvxKSbzdfXaZFpc3Q=', 'Content-Length': '0'} 2022-09-03T06:33:13.527 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000099ac99bfbe941aac-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.528 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:13.528 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.529 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.530 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.530 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.531 DEBUG:boto:Method: GET 2022-09-03T06:33:13.531 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.541 DEBUG:boto:Data: 2022-09-03T06:33:13.542 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.543 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.544 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.545 DEBUG:boto:Params: {} 2022-09-03T06:33:13.546 DEBUG:boto:Token: None 2022-09-03T06:33:13.548 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KNFpcX76fdv3H33QeKjrU5xiPrY= 2022-09-03T06:33:13.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KNFpcX76fdv3H33QeKjrU5xiPrY=', 'Content-Length': '0'} 2022-09-03T06:33:13.566 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004beb854cf74c43ba-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.569 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.570 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.572 DEBUG:boto:path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.572 DEBUG:boto:auth_path=/avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.573 DEBUG:boto:Method: GET 2022-09-03T06:33:13.574 DEBUG:boto:Path: /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.575 DEBUG:boto:Data: 2022-09-03T06:33:13.580 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.581 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.582 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.582 DEBUG:boto:Params: {} 2022-09-03T06:33:13.583 DEBUG:boto:Token: None 2022-09-03T06:33:13.584 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2?versionId=null 2022-09-03T06:33:13.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KNFpcX76fdv3H33QeKjrU5xiPrY= 2022-09-03T06:33:13.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KNFpcX76fdv3H33QeKjrU5xiPrY=', 'Content-Length': '0'} 2022-09-03T06:33:13.592 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009cbcea32fa76380e-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.593 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:13.594 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:13.595 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:13.596 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:13.597 DEBUG:boto:Method: GET 2022-09-03T06:33:13.602 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:13.603 DEBUG:boto:Data: 2022-09-03T06:33:13.604 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.605 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.606 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.607 DEBUG:boto:Params: {} 2022-09-03T06:33:13.608 DEBUG:boto:Token: None 2022-09-03T06:33:13.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/ 2022-09-03T06:33:13.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE= 2022-09-03T06:33:13.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE=', 'Content-Length': '0'} 2022-09-03T06:33:13.635 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d398393a0f4ca24b-006312f529-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.636 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:13.638 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:13.640 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:13.641 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:13.641 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:13.642 DEBUG:boto:Method: GET 2022-09-03T06:33:13.643 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:13.644 DEBUG:boto:Data: 2022-09-03T06:33:13.645 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.653 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.655 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.656 DEBUG:boto:Params: {} 2022-09-03T06:33:13.658 DEBUG:boto:Token: None 2022-09-03T06:33:13.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/ 2022-09-03T06:33:13.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE= 2022-09-03T06:33:13.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE=', 'Content-Length': '0'} 2022-09-03T06:33:13.686 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c4cfd9bd409ead6-006312f529-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.688 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:13.693 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.694 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.695 DEBUG:boto:Method: HEAD 2022-09-03T06:33:13.696 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:13.697 DEBUG:boto:Data: 2022-09-03T06:33:13.703 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.705 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.706 DEBUG:boto:Params: {} 2022-09-03T06:33:13.707 DEBUG:boto:Token: None 2022-09-03T06:33:13.708 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2 2022-09-03T06:33:13.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:msNmZzynMACrqRsdKoZ1eOTNYmQ= 2022-09-03T06:33:13.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:msNmZzynMACrqRsdKoZ1eOTNYmQ=', 'Content-Length': '0'} 2022-09-03T06:33:13.731 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fa9dd25cb4483916-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.732 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.732 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.733 DEBUG:boto:Method: HEAD 2022-09-03T06:33:13.734 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:13.735 DEBUG:boto:Data: 2022-09-03T06:33:13.735 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.739 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.743 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.744 DEBUG:boto:Params: {} 2022-09-03T06:33:13.745 DEBUG:boto:Token: None 2022-09-03T06:33:13.746 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2 2022-09-03T06:33:13.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:msNmZzynMACrqRsdKoZ1eOTNYmQ= 2022-09-03T06:33:13.758 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:msNmZzynMACrqRsdKoZ1eOTNYmQ=', 'Content-Length': '0'} 2022-09-03T06:33:13.769 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001fda64fd76cf4c19-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.770 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:13.771 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.772 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.773 DEBUG:boto:Method: GET 2022-09-03T06:33:13.784 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:13.789 DEBUG:boto:Data: 2022-09-03T06:33:13.790 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.797 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.798 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.799 DEBUG:boto:Params: {} 2022-09-03T06:33:13.800 DEBUG:boto:Token: None 2022-09-03T06:33:13.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2 2022-09-03T06:33:13.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cELez8B6VIiAv6KjXYRwU294yPQ= 2022-09-03T06:33:13.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cELez8B6VIiAv6KjXYRwU294yPQ=', 'Content-Length': '0'} 2022-09-03T06:33:13.806 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000046ceb635beff5b30-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.820 DEBUG:boto:path=/avysmw-59/obj2 2022-09-03T06:33:13.821 DEBUG:boto:auth_path=/avysmw-59/obj2 2022-09-03T06:33:13.822 DEBUG:boto:Method: GET 2022-09-03T06:33:13.823 DEBUG:boto:Path: /avysmw-59/obj2 2022-09-03T06:33:13.824 DEBUG:boto:Data: 2022-09-03T06:33:13.825 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.827 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.829 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.831 DEBUG:boto:Params: {} 2022-09-03T06:33:13.832 DEBUG:boto:Token: None 2022-09-03T06:33:13.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj2 2022-09-03T06:33:13.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cELez8B6VIiAv6KjXYRwU294yPQ= 2022-09-03T06:33:13.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cELez8B6VIiAv6KjXYRwU294yPQ=', 'Content-Length': '0'} 2022-09-03T06:33:13.838 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000083bf86e497663e3b-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.840 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:13.842 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:13.844 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:13.845 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:13.847 DEBUG:boto:Method: GET 2022-09-03T06:33:13.850 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:13.852 DEBUG:boto:Data: 2022-09-03T06:33:13.855 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.859 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.862 DEBUG:boto:Params: {} 2022-09-03T06:33:13.867 DEBUG:boto:Token: None 2022-09-03T06:33:13.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/ 2022-09-03T06:33:13.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE= 2022-09-03T06:33:13.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE=', 'Content-Length': '0'} 2022-09-03T06:33:13.875 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f64aa6fd65158b2-006312f529-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.877 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:13.879 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:13.880 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:13.880 DEBUG:boto:path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:13.881 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj2 2022-09-03T06:33:13.883 DEBUG:boto:Method: GET 2022-09-03T06:33:13.885 DEBUG:boto:Path: /avysmw-59/?prefix=obj2 2022-09-03T06:33:13.886 DEBUG:boto:Data: 2022-09-03T06:33:13.897 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.898 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.899 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.900 DEBUG:boto:Params: {} 2022-09-03T06:33:13.901 DEBUG:boto:Token: None 2022-09-03T06:33:13.908 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/ 2022-09-03T06:33:13.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE= 2022-09-03T06:33:13.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S5c4bJVSecT/yv2+rCb+SAKCnNE=', 'Content-Length': '0'} 2022-09-03T06:33:13.917 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000491403b95db251ad-006312f529-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.919 DEBUG:boto:b'avysmw-59obj21000falseobj22022-09-03T06:32:26.696Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:13.921 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:13.923 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:13.924 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:13.925 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.927 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.928 DEBUG:boto:Method: GET 2022-09-03T06:33:13.929 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.929 DEBUG:boto:Data: 2022-09-03T06:33:13.931 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.932 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.933 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.938 DEBUG:boto:Params: {} 2022-09-03T06:33:13.941 DEBUG:boto:Token: None 2022-09-03T06:33:13.942 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HQVWWvN0zc8yUr6Xjd50v7h2Q80= 2022-09-03T06:33:13.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HQVWWvN0zc8yUr6Xjd50v7h2Q80=', 'Content-Length': '0'} 2022-09-03T06:33:13.945 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003136af98b5aa815b-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.947 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:13.948 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:13.949 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.949 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.950 DEBUG:boto:Method: GET 2022-09-03T06:33:13.951 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.951 DEBUG:boto:Data: 2022-09-03T06:33:13.952 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.952 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:13.953 DEBUG:boto:Port: 8000 2022-09-03T06:33:13.953 DEBUG:boto:Params: {} 2022-09-03T06:33:13.954 DEBUG:boto:Token: None 2022-09-03T06:33:13.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HQVWWvN0zc8yUr6Xjd50v7h2Q80= 2022-09-03T06:33:13.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HQVWWvN0zc8yUr6Xjd50v7h2Q80=', 'Content-Length': '0'} 2022-09-03T06:33:13.958 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000045b86a7205c86910-006312f529-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:13.960 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:13.960 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:13.961 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.962 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.962 DEBUG:boto:Method: HEAD 2022-09-03T06:33:13.963 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.963 DEBUG:boto:Data: 2022-09-03T06:33:13.964 DEBUG:boto:Headers: {} 2022-09-03T06:33:13.964 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:13.965 DEBUG:boto:Port: 8001 2022-09-03T06:33:13.965 DEBUG:boto:Params: {} 2022-09-03T06:33:13.966 DEBUG:boto:Token: None 2022-09-03T06:33:13.967 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:13 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:13.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6+ZhTqG3zzr8KMwQVduJgN5oHW0= 2022-09-03T06:33:13.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6+ZhTqG3zzr8KMwQVduJgN5oHW0=', 'Content-Length': '0'} 2022-09-03T06:33:14.005 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a4eac898d086bbee-006312f529-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.007 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:14.008 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:14.009 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.018 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.019 DEBUG:boto:Method: HEAD 2022-09-03T06:33:14.019 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.020 DEBUG:boto:Data: 2022-09-03T06:33:14.021 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.021 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.022 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.022 DEBUG:boto:Params: {} 2022-09-03T06:33:14.024 DEBUG:boto:Token: None 2022-09-03T06:33:14.025 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:URdIAwAmrjLnU4Nn6CEiNTJFOR4= 2022-09-03T06:33:14.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:URdIAwAmrjLnU4Nn6CEiNTJFOR4=', 'Content-Length': '0'} 2022-09-03T06:33:14.035 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007f9c9b5586c3b400-006312f52a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.036 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:14.038 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:14.039 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:14.040 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.042 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.043 DEBUG:boto:Method: GET 2022-09-03T06:33:14.044 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.045 DEBUG:boto:Data: 2022-09-03T06:33:14.046 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.048 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.049 DEBUG:boto:Params: {} 2022-09-03T06:33:14.051 DEBUG:boto:Token: None 2022-09-03T06:33:14.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zUYLnf5/gKYMYkiuWp4KWua9XYQ= 2022-09-03T06:33:14.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zUYLnf5/gKYMYkiuWp4KWua9XYQ=', 'Content-Length': '0'} 2022-09-03T06:33:14.064 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000093b95cde0e1cd53d-006312f52a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.068 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:14.070 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:14.071 DEBUG:boto:path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.072 DEBUG:boto:auth_path=/avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.073 DEBUG:boto:Method: GET 2022-09-03T06:33:14.073 DEBUG:boto:Path: /avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.083 DEBUG:boto:Data: 2022-09-03T06:33:14.084 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.084 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.085 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.085 DEBUG:boto:Params: {} 2022-09-03T06:33:14.092 DEBUG:boto:Token: None 2022-09-03T06:33:14.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj3?versionId=null 2022-09-03T06:33:14.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zUYLnf5/gKYMYkiuWp4KWua9XYQ= 2022-09-03T06:33:14.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zUYLnf5/gKYMYkiuWp4KWua9XYQ=', 'Content-Length': '0'} 2022-09-03T06:33:14.099 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000065a56af0878bab86-006312f52a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.100 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:14.100 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:14.101 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:14.106 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:14.108 DEBUG:boto:Method: GET 2022-09-03T06:33:14.110 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:14.111 DEBUG:boto:Data: 2022-09-03T06:33:14.111 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.112 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.112 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.113 DEBUG:boto:Params: {} 2022-09-03T06:33:14.114 DEBUG:boto:Token: None 2022-09-03T06:33:14.115 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/ 2022-09-03T06:33:14.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA= 2022-09-03T06:33:14.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA=', 'Content-Length': '0'} 2022-09-03T06:33:14.139 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023064b075836ec3e-006312f52a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.143 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:14.144 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:14.145 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:14.145 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:14.173 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:14.174 DEBUG:boto:Method: GET 2022-09-03T06:33:14.177 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:14.184 DEBUG:boto:Data: 2022-09-03T06:33:14.185 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.187 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.188 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.189 DEBUG:boto:Params: {} 2022-09-03T06:33:14.190 DEBUG:boto:Token: None 2022-09-03T06:33:14.193 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/ 2022-09-03T06:33:14.194 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA= 2022-09-03T06:33:14.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA=', 'Content-Length': '0'} 2022-09-03T06:33:14.201 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b94e587ea7fb1b9-006312f52a-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.203 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:14.205 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:14.206 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:14.207 DEBUG:boto:Method: HEAD 2022-09-03T06:33:14.208 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:14.209 DEBUG:boto:Data: 2022-09-03T06:33:14.211 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.213 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.213 DEBUG:boto:Params: {} 2022-09-03T06:33:14.232 DEBUG:boto:Token: None 2022-09-03T06:33:14.232 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj3 2022-09-03T06:33:14.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dTFhAUsV2HxYcuZxJc82bhU6lSI= 2022-09-03T06:33:14.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dTFhAUsV2HxYcuZxJc82bhU6lSI=', 'Content-Length': '0'} 2022-09-03T06:33:14.249 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009d418f2b0138d046-006312f52a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.251 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:14.252 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:14.253 DEBUG:boto:Method: HEAD 2022-09-03T06:33:14.262 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:14.273 DEBUG:boto:Data: 2022-09-03T06:33:14.274 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.275 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.276 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.277 DEBUG:boto:Params: {} 2022-09-03T06:33:14.284 DEBUG:boto:Token: None 2022-09-03T06:33:14.285 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj3 2022-09-03T06:33:14.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dTFhAUsV2HxYcuZxJc82bhU6lSI= 2022-09-03T06:33:14.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dTFhAUsV2HxYcuZxJc82bhU6lSI=', 'Content-Length': '0'} 2022-09-03T06:33:14.299 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bb830b6dc2bd6553-006312f52a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.301 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:14.302 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:14.303 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:14.303 DEBUG:boto:Method: GET 2022-09-03T06:33:14.307 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:14.308 DEBUG:boto:Data: 2022-09-03T06:33:14.309 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.310 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.317 DEBUG:boto:Params: {} 2022-09-03T06:33:14.318 DEBUG:boto:Token: None 2022-09-03T06:33:14.319 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj3 2022-09-03T06:33:14.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:klzH8OfO05Y/kKHAKzvwS1rL6vo= 2022-09-03T06:33:14.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:klzH8OfO05Y/kKHAKzvwS1rL6vo=', 'Content-Length': '0'} 2022-09-03T06:33:14.334 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003c56b1214db4195f-006312f52a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.338 DEBUG:boto:path=/avysmw-59/obj3 2022-09-03T06:33:14.339 DEBUG:boto:auth_path=/avysmw-59/obj3 2022-09-03T06:33:14.340 DEBUG:boto:Method: GET 2022-09-03T06:33:14.341 DEBUG:boto:Path: /avysmw-59/obj3 2022-09-03T06:33:14.342 DEBUG:boto:Data: 2022-09-03T06:33:14.343 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.353 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.354 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.355 DEBUG:boto:Params: {} 2022-09-03T06:33:14.357 DEBUG:boto:Token: None 2022-09-03T06:33:14.359 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj3 2022-09-03T06:33:14.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:klzH8OfO05Y/kKHAKzvwS1rL6vo= 2022-09-03T06:33:14.360 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:klzH8OfO05Y/kKHAKzvwS1rL6vo=', 'Content-Length': '0'} 2022-09-03T06:33:14.370 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002d9aea4074a76a38-006312f52a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.371 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:14.371 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:14.372 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:14.373 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:14.373 DEBUG:boto:Method: GET 2022-09-03T06:33:14.374 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:14.381 DEBUG:boto:Data: 2022-09-03T06:33:14.382 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.383 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.384 DEBUG:boto:Params: {} 2022-09-03T06:33:14.384 DEBUG:boto:Token: None 2022-09-03T06:33:14.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/ 2022-09-03T06:33:14.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA= 2022-09-03T06:33:14.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA=', 'Content-Length': '0'} 2022-09-03T06:33:14.398 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000806aa4c231253557-006312f52a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.399 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:14.400 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:14.401 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:14.402 DEBUG:boto:path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:14.402 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj3 2022-09-03T06:33:14.403 DEBUG:boto:Method: GET 2022-09-03T06:33:14.404 DEBUG:boto:Path: /avysmw-59/?prefix=obj3 2022-09-03T06:33:14.404 DEBUG:boto:Data: 2022-09-03T06:33:14.405 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.405 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.406 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.407 DEBUG:boto:Params: {} 2022-09-03T06:33:14.407 DEBUG:boto:Token: None 2022-09-03T06:33:14.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/ 2022-09-03T06:33:14.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA= 2022-09-03T06:33:14.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA=', 'Content-Length': '0'} 2022-09-03T06:33:14.417 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000b64136900dca4af-006312f52a-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.420 DEBUG:boto:b'avysmw-59obj31000falseobj32022-09-03T06:32:26.769Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:14.421 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:14.423 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.437 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.439 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.440 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.441 DEBUG:boto:Method: GET 2022-09-03T06:33:14.441 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.442 DEBUG:boto:Data: 2022-09-03T06:33:14.443 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.444 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.444 DEBUG:boto:Params: {} 2022-09-03T06:33:14.445 DEBUG:boto:Token: None 2022-09-03T06:33:14.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/DJgPehp8r7/nWK1s82EBe0oeG8= 2022-09-03T06:33:14.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/DJgPehp8r7/nWK1s82EBe0oeG8=', 'Content-Length': '0'} 2022-09-03T06:33:14.452 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000017ded139eb7982e6-006312f52a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.453 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.453 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.454 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.455 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.455 DEBUG:boto:Method: GET 2022-09-03T06:33:14.456 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.456 DEBUG:boto:Data: 2022-09-03T06:33:14.457 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.457 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.465 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.467 DEBUG:boto:Params: {} 2022-09-03T06:33:14.468 DEBUG:boto:Token: None 2022-09-03T06:33:14.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/DJgPehp8r7/nWK1s82EBe0oeG8= 2022-09-03T06:33:14.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/DJgPehp8r7/nWK1s82EBe0oeG8=', 'Content-Length': '0'} 2022-09-03T06:33:14.479 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006e54683a15bd882d-006312f52a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.480 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.481 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.483 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.484 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.485 DEBUG:boto:Method: HEAD 2022-09-03T06:33:14.493 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.495 DEBUG:boto:Data: 2022-09-03T06:33:14.496 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.497 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.498 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.499 DEBUG:boto:Params: {} 2022-09-03T06:33:14.500 DEBUG:boto:Token: None 2022-09-03T06:33:14.501 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CqUOYU/jmFQxonD2cpnpyUx/avw= 2022-09-03T06:33:14.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CqUOYU/jmFQxonD2cpnpyUx/avw=', 'Content-Length': '0'} 2022-09-03T06:33:14.506 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b0d7f484208efcbd-006312f52a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.507 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.509 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.510 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.511 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.514 DEBUG:boto:Method: HEAD 2022-09-03T06:33:14.515 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.516 DEBUG:boto:Data: 2022-09-03T06:33:14.517 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.518 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.524 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.526 DEBUG:boto:Params: {} 2022-09-03T06:33:14.527 DEBUG:boto:Token: None 2022-09-03T06:33:14.528 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CqUOYU/jmFQxonD2cpnpyUx/avw= 2022-09-03T06:33:14.547 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CqUOYU/jmFQxonD2cpnpyUx/avw=', 'Content-Length': '0'} 2022-09-03T06:33:14.552 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b79b1f21f3c82f59-006312f52a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.554 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:14.555 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.557 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.558 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.559 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.560 DEBUG:boto:Method: GET 2022-09-03T06:33:14.562 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.563 DEBUG:boto:Data: 2022-09-03T06:33:14.564 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.565 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.568 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.569 DEBUG:boto:Params: {} 2022-09-03T06:33:14.570 DEBUG:boto:Token: None 2022-09-03T06:33:14.571 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/DJgPehp8r7/nWK1s82EBe0oeG8= 2022-09-03T06:33:14.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/DJgPehp8r7/nWK1s82EBe0oeG8=', 'Content-Length': '0'} 2022-09-03T06:33:14.586 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001d594394a96327d3-006312f52a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.588 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.589 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.591 DEBUG:boto:path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.592 DEBUG:boto:auth_path=/avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.593 DEBUG:boto:Method: GET 2022-09-03T06:33:14.594 DEBUG:boto:Path: /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.595 DEBUG:boto:Data: 2022-09-03T06:33:14.596 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.597 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.598 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.599 DEBUG:boto:Params: {} 2022-09-03T06:33:14.600 DEBUG:boto:Token: None 2022-09-03T06:33:14.601 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4?versionId=null 2022-09-03T06:33:14.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/DJgPehp8r7/nWK1s82EBe0oeG8= 2022-09-03T06:33:14.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/DJgPehp8r7/nWK1s82EBe0oeG8=', 'Content-Length': '0'} 2022-09-03T06:33:14.613 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e02ad6d6b8559593-006312f52a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.615 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:14.616 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:14.639 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:14.640 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:14.641 DEBUG:boto:Method: GET 2022-09-03T06:33:14.642 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:14.643 DEBUG:boto:Data: 2022-09-03T06:33:14.650 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.653 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.654 DEBUG:boto:Params: {} 2022-09-03T06:33:14.655 DEBUG:boto:Token: None 2022-09-03T06:33:14.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/ 2022-09-03T06:33:14.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA= 2022-09-03T06:33:14.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA=', 'Content-Length': '0'} 2022-09-03T06:33:14.670 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000f10a3695e6e8a19-006312f52a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.672 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:14.673 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:14.676 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:14.676 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:14.677 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:14.677 DEBUG:boto:Method: GET 2022-09-03T06:33:14.678 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:14.679 DEBUG:boto:Data: 2022-09-03T06:33:14.679 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.680 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.681 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.682 DEBUG:boto:Params: {} 2022-09-03T06:33:14.682 DEBUG:boto:Token: None 2022-09-03T06:33:14.683 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/ 2022-09-03T06:33:14.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA= 2022-09-03T06:33:14.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA=', 'Content-Length': '0'} 2022-09-03T06:33:14.691 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008794e90a644e1326-006312f52a-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.692 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:14.693 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.694 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.694 DEBUG:boto:Method: HEAD 2022-09-03T06:33:14.696 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:14.697 DEBUG:boto:Data: 2022-09-03T06:33:14.698 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.699 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.700 DEBUG:boto:Params: {} 2022-09-03T06:33:14.701 DEBUG:boto:Token: None 2022-09-03T06:33:14.702 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4 2022-09-03T06:33:14.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mkX8QOACEqXC2pL0bcMNshGiwMg= 2022-09-03T06:33:14.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mkX8QOACEqXC2pL0bcMNshGiwMg=', 'Content-Length': '0'} 2022-09-03T06:33:14.714 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a4cecb8275ed5176-006312f52a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.719 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.720 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.721 DEBUG:boto:Method: HEAD 2022-09-03T06:33:14.723 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:14.724 DEBUG:boto:Data: 2022-09-03T06:33:14.725 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.726 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.727 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.727 DEBUG:boto:Params: {} 2022-09-03T06:33:14.729 DEBUG:boto:Token: None 2022-09-03T06:33:14.730 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4 2022-09-03T06:33:14.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mkX8QOACEqXC2pL0bcMNshGiwMg= 2022-09-03T06:33:14.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mkX8QOACEqXC2pL0bcMNshGiwMg=', 'Content-Length': '0'} 2022-09-03T06:33:14.745 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000082adf1a8c8f365ca-006312f52a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.780 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:14.783 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.785 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.787 DEBUG:boto:Method: GET 2022-09-03T06:33:14.798 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:14.827 DEBUG:boto:Data: 2022-09-03T06:33:14.847 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.856 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.861 DEBUG:boto:Params: {} 2022-09-03T06:33:14.862 DEBUG:boto:Token: None 2022-09-03T06:33:14.863 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4 2022-09-03T06:33:14.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIXXDwYoV3bxzCYq41Xo6oxaud4= 2022-09-03T06:33:14.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIXXDwYoV3bxzCYq41Xo6oxaud4=', 'Content-Length': '0'} 2022-09-03T06:33:14.876 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000015a056d84cca3bea-006312f52a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.879 DEBUG:boto:path=/avysmw-59/obj4 2022-09-03T06:33:14.880 DEBUG:boto:auth_path=/avysmw-59/obj4 2022-09-03T06:33:14.881 DEBUG:boto:Method: GET 2022-09-03T06:33:14.882 DEBUG:boto:Path: /avysmw-59/obj4 2022-09-03T06:33:14.883 DEBUG:boto:Data: 2022-09-03T06:33:14.884 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.885 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:14.887 DEBUG:boto:Port: 8000 2022-09-03T06:33:14.895 DEBUG:boto:Params: {} 2022-09-03T06:33:14.899 DEBUG:boto:Token: None 2022-09-03T06:33:14.900 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/obj4 2022-09-03T06:33:14.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rIXXDwYoV3bxzCYq41Xo6oxaud4= 2022-09-03T06:33:14.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rIXXDwYoV3bxzCYq41Xo6oxaud4=', 'Content-Length': '0'} 2022-09-03T06:33:14.908 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002231b39251424492-006312f52a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.929 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:14.930 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:14.931 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:14.934 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:14.936 DEBUG:boto:Method: GET 2022-09-03T06:33:14.940 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:14.941 DEBUG:boto:Data: 2022-09-03T06:33:14.942 DEBUG:boto:Headers: {} 2022-09-03T06:33:14.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:14.945 DEBUG:boto:Port: 8001 2022-09-03T06:33:14.946 DEBUG:boto:Params: {} 2022-09-03T06:33:14.952 DEBUG:boto:Token: None 2022-09-03T06:33:14.953 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:14 GMT /avysmw-59/ 2022-09-03T06:33:14.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA= 2022-09-03T06:33:14.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cgUpPK2H+pKzzU92hehZkpycWxA=', 'Content-Length': '0'} 2022-09-03T06:33:14.982 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3d80133651026fd-006312f52a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:14.990 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:14.993 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:33:14.994 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:33:14.995 DEBUG:boto:path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:14.995 DEBUG:boto:auth_path=/avysmw-59/?prefix=obj4 2022-09-03T06:33:14.996 DEBUG:boto:Method: GET 2022-09-03T06:33:14.999 DEBUG:boto:Path: /avysmw-59/?prefix=obj4 2022-09-03T06:33:15.000 DEBUG:boto:Data: 2022-09-03T06:33:15.001 DEBUG:boto:Headers: {} 2022-09-03T06:33:15.002 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:15.003 DEBUG:boto:Port: 8000 2022-09-03T06:33:15.004 DEBUG:boto:Params: {} 2022-09-03T06:33:15.023 DEBUG:boto:Token: None 2022-09-03T06:33:15.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:15 GMT /avysmw-59/ 2022-09-03T06:33:15.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFoii7QxctxLOp6lRGfNBRKftak= 2022-09-03T06:33:15.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFoii7QxctxLOp6lRGfNBRKftak=', 'Content-Length': '0'} 2022-09-03T06:33:15.038 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004058db5d11f1487b-006312f52b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:15.039 DEBUG:boto:b'avysmw-59obj41000falseobj42022-09-03T06:32:26.799Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:15.041 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-59 zones={test-zone3, test-zone2} 2022-09-03T06:33:15.042 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-60 2022-09-03T06:33:15.044 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.045 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.054 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-60 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:15.132 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:15.132 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:15.184 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:15.178+0000 7fccbdf93600 1 bucket sync caught up with source: 2022-09-03T06:33:15.185 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.42.5, , 00000000001.72.5, , 00000000001.54.5, , , 00000000001.44.5, , , ] 2022-09-03T06:33:15.185 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.42.5, , 00000000001.72.5, , 00000000001.54.5, , , 00000000001.44.5, , , ] 2022-09-03T06:33:15.186 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:15.178+0000 7fccbdf93600 0 bucket checkpoint complete 2022-09-03T06:33:15.208 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-60 2022-09-03T06:33:15.209 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.215 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.216 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-60 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:15.290 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:15.291 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:15.347 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:15.344+0000 7ffb7c7e2600 1 bucket sync caught up with source: 2022-09-03T06:33:15.347 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.42.5, , 00000000001.72.5, , 00000000001.54.5, , , 00000000001.44.5, , , ] 2022-09-03T06:33:15.348 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.42.5, , 00000000001.72.5, , 00000000001.54.5, , , 00000000001.44.5, , , ] 2022-09-03T06:33:15.348 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:15.344+0000 7ffb7c7e2600 0 bucket checkpoint complete 2022-09-03T06:33:15.378 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-60 2022-09-03T06:33:15.379 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.380 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.380 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-60 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:15.450 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:15.451 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:15.508 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:15.505+0000 7efd6912f600 1 bucket sync caught up with source: 2022-09-03T06:33:15.509 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.42.5, , 00000000001.72.5, , 00000000001.54.5, , , 00000000001.44.5, , , ] 2022-09-03T06:33:15.509 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.42.5, , 00000000001.72.5, , 00000000001.54.5, , , 00000000001.44.5, , , ] 2022-09-03T06:33:15.509 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:15.505+0000 7efd6912f600 0 bucket checkpoint complete 2022-09-03T06:33:15.531 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-60 2022-09-03T06:33:15.532 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.533 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.552 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-60 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:15.624 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:15.624 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:15.670 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:15.664+0000 7feba8115600 1 bucket sync caught up with source: 2022-09-03T06:33:15.671 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.248.5, , 00000000001.206.5, , 00000000001.268.5, , , 00000000001.205.5, , , ] 2022-09-03T06:33:15.671 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.248.5, , 00000000001.206.5, , 00000000001.268.5, , , 00000000001.205.5, , , ] 2022-09-03T06:33:15.675 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:15.664+0000 7feba8115600 0 bucket checkpoint complete 2022-09-03T06:33:15.681 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-60 2022-09-03T06:33:15.690 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.707 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.709 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-60 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:15.781 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:15.782 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:15.816 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:15.812+0000 7ff285181600 1 bucket sync caught up with source: 2022-09-03T06:33:15.817 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.248.5, , 00000000001.206.5, , 00000000001.268.5, , , 00000000001.205.5, , , ] 2022-09-03T06:33:15.819 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.248.5, , 00000000001.206.5, , 00000000001.268.5, , , 00000000001.205.5, , , ] 2022-09-03T06:33:15.819 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:15.812+0000 7ff285181600 0 bucket checkpoint complete 2022-09-03T06:33:15.828 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-60 2022-09-03T06:33:15.829 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-60 2022-09-03T06:33:15.830 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.831 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.832 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-60 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:15.913 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:15.914 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:15.960 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:15.958+0000 7f2b75739600 1 bucket sync caught up with source: 2022-09-03T06:33:15.961 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.176.5, , 00000000001.195.5, , 00000000001.261.5, , , 00000000001.135.5, , , ] 2022-09-03T06:33:15.963 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.176.5, , 00000000001.195.5, , 00000000001.261.5, , , 00000000001.135.5, , , ] 2022-09-03T06:33:15.964 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:33:15.958+0000 7f2b75739600 0 bucket checkpoint complete 2022-09-03T06:33:15.976 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-60 2022-09-03T06:33:15.977 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.978 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-60', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:15.979 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-60 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:16.038 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:16.039 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:16.077 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:16.075+0000 7f29e7d6b600 1 bucket sync caught up with source: 2022-09-03T06:33:16.077 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.176.5, , 00000000001.195.5, , 00000000001.261.5, , , 00000000001.135.5, , , ] 2022-09-03T06:33:16.078 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.176.5, , 00000000001.195.5, , 00000000001.261.5, , , 00000000001.135.5, , , ] 2022-09-03T06:33:16.078 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:16.075+0000 7f29e7d6b600 0 bucket checkpoint complete 2022-09-03T06:33:16.091 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-60 2022-09-03T06:33:16.093 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-60 zones={test-zone2, test-zone1} 2022-09-03T06:33:16.094 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.095 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.095 DEBUG:boto:Method: HEAD 2022-09-03T06:33:16.096 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:33:16.097 DEBUG:boto:Data: 2022-09-03T06:33:16.098 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.100 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.101 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.104 DEBUG:boto:Params: {} 2022-09-03T06:33:16.105 DEBUG:boto:Token: None 2022-09-03T06:33:16.107 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/ 2022-09-03T06:33:16.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gjkxcSOn/G4Hu0y0i9irVYvW3vQ= 2022-09-03T06:33:16.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gjkxcSOn/G4Hu0y0i9irVYvW3vQ=', 'Content-Length': '0'} 2022-09-03T06:33:16.114 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007669cb0ddd7e9dc0-006312f52c-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.115 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.124 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.125 DEBUG:boto:Method: HEAD 2022-09-03T06:33:16.126 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:33:16.131 DEBUG:boto:Data: 2022-09-03T06:33:16.136 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.138 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.139 DEBUG:boto:Params: {} 2022-09-03T06:33:16.140 DEBUG:boto:Token: None 2022-09-03T06:33:16.141 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/ 2022-09-03T06:33:16.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gjkxcSOn/G4Hu0y0i9irVYvW3vQ= 2022-09-03T06:33:16.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gjkxcSOn/G4Hu0y0i9irVYvW3vQ=', 'Content-Length': '0'} 2022-09-03T06:33:16.148 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f61a3c8310905570-006312f52c-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.149 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:33:16.150 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.152 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.153 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.154 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.156 DEBUG:boto:Method: GET 2022-09-03T06:33:16.157 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.183 DEBUG:boto:Data: 2022-09-03T06:33:16.184 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.185 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.188 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.189 DEBUG:boto:Params: {} 2022-09-03T06:33:16.198 DEBUG:boto:Token: None 2022-09-03T06:33:16.200 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/?versions 2022-09-03T06:33:16.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R9QzMiyzXQmNE7fl7gcZVL8h5yU= 2022-09-03T06:33:16.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R9QzMiyzXQmNE7fl7gcZVL8h5yU=', 'Content-Length': '0'} 2022-09-03T06:33:16.208 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a8a7fff8388e9fdf-006312f52c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.209 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:16.211 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:16.211 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:16.213 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:16.214 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:16.215 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:33:16.216 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.217 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.218 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.219 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.220 DEBUG:boto:Method: GET 2022-09-03T06:33:16.222 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.235 DEBUG:boto:Data: 2022-09-03T06:33:16.236 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.238 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.238 DEBUG:boto:Params: {} 2022-09-03T06:33:16.239 DEBUG:boto:Token: None 2022-09-03T06:33:16.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/?versions 2022-09-03T06:33:16.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R9QzMiyzXQmNE7fl7gcZVL8h5yU= 2022-09-03T06:33:16.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R9QzMiyzXQmNE7fl7gcZVL8h5yU=', 'Content-Length': '0'} 2022-09-03T06:33:16.247 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000052bb830dde45678-006312f52c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.249 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:16.251 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:16.252 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:16.254 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:16.255 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:16.256 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.257 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.258 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.259 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.260 DEBUG:boto:Method: GET 2022-09-03T06:33:16.261 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.262 DEBUG:boto:Data: 2022-09-03T06:33:16.263 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.264 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.265 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.266 DEBUG:boto:Params: {} 2022-09-03T06:33:16.267 DEBUG:boto:Token: None 2022-09-03T06:33:16.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/?versions 2022-09-03T06:33:16.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R9QzMiyzXQmNE7fl7gcZVL8h5yU= 2022-09-03T06:33:16.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R9QzMiyzXQmNE7fl7gcZVL8h5yU=', 'Content-Length': '0'} 2022-09-03T06:33:16.274 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007caecc6cfd24a6a4-006312f52c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.275 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:16.277 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.278 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.279 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.280 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.281 DEBUG:boto:Method: GET 2022-09-03T06:33:16.281 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:16.282 DEBUG:boto:Data: 2022-09-03T06:33:16.283 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.285 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.286 DEBUG:boto:Params: {} 2022-09-03T06:33:16.287 DEBUG:boto:Token: None 2022-09-03T06:33:16.289 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/?versions 2022-09-03T06:33:16.290 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R9QzMiyzXQmNE7fl7gcZVL8h5yU= 2022-09-03T06:33:16.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R9QzMiyzXQmNE7fl7gcZVL8h5yU=', 'Content-Length': '0'} 2022-09-03T06:33:16.298 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000520f3aaf83701fff-006312f52c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.300 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:16.301 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:16.303 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.304 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.305 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.306 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.307 DEBUG:boto:Method: GET 2022-09-03T06:33:16.311 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.313 DEBUG:boto:Data: 2022-09-03T06:33:16.314 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.315 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.316 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.317 DEBUG:boto:Params: {} 2022-09-03T06:33:16.317 DEBUG:boto:Token: None 2022-09-03T06:33:16.319 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hwlYXKFhPdaighXk7CXKb48Pe80= 2022-09-03T06:33:16.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hwlYXKFhPdaighXk7CXKb48Pe80=', 'Content-Length': '0'} 2022-09-03T06:33:16.334 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000015d9f800daea8b5a-006312f52c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.336 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.337 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.338 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.340 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.341 DEBUG:boto:Method: GET 2022-09-03T06:33:16.342 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.343 DEBUG:boto:Data: 2022-09-03T06:33:16.344 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.346 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.347 DEBUG:boto:Params: {} 2022-09-03T06:33:16.348 DEBUG:boto:Token: None 2022-09-03T06:33:16.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hwlYXKFhPdaighXk7CXKb48Pe80= 2022-09-03T06:33:16.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hwlYXKFhPdaighXk7CXKb48Pe80=', 'Content-Length': '0'} 2022-09-03T06:33:16.355 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cca967d0591b1630-006312f52c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.361 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.362 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.363 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.364 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.365 DEBUG:boto:Method: HEAD 2022-09-03T06:33:16.367 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.368 DEBUG:boto:Data: 2022-09-03T06:33:16.369 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.369 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.370 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.371 DEBUG:boto:Params: {} 2022-09-03T06:33:16.372 DEBUG:boto:Token: None 2022-09-03T06:33:16.374 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EAlIJzTWfyQT9FJgAbkYEVgkDg= 2022-09-03T06:33:16.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EAlIJzTWfyQT9FJgAbkYEVgkDg=', 'Content-Length': '0'} 2022-09-03T06:33:16.379 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005115dae41f554bac-006312f52c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.380 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.381 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.382 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.383 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.384 DEBUG:boto:Method: HEAD 2022-09-03T06:33:16.385 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.386 DEBUG:boto:Data: 2022-09-03T06:33:16.387 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.388 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.389 DEBUG:boto:Params: {} 2022-09-03T06:33:16.395 DEBUG:boto:Token: None 2022-09-03T06:33:16.397 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EAlIJzTWfyQT9FJgAbkYEVgkDg= 2022-09-03T06:33:16.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EAlIJzTWfyQT9FJgAbkYEVgkDg=', 'Content-Length': '0'} 2022-09-03T06:33:16.414 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b2ceb130f46e0fc3-006312f52c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.415 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:16.417 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.418 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.418 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.419 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.419 DEBUG:boto:Method: GET 2022-09-03T06:33:16.420 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.420 DEBUG:boto:Data: 2022-09-03T06:33:16.421 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.421 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.423 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.424 DEBUG:boto:Params: {} 2022-09-03T06:33:16.426 DEBUG:boto:Token: None 2022-09-03T06:33:16.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hwlYXKFhPdaighXk7CXKb48Pe80= 2022-09-03T06:33:16.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hwlYXKFhPdaighXk7CXKb48Pe80=', 'Content-Length': '0'} 2022-09-03T06:33:16.431 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e9512c7c7b3e52f2-006312f52c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.435 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.436 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.437 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.438 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.439 DEBUG:boto:Method: GET 2022-09-03T06:33:16.440 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.441 DEBUG:boto:Data: 2022-09-03T06:33:16.442 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.444 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.444 DEBUG:boto:Params: {} 2022-09-03T06:33:16.446 DEBUG:boto:Token: None 2022-09-03T06:33:16.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:16.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hwlYXKFhPdaighXk7CXKb48Pe80= 2022-09-03T06:33:16.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hwlYXKFhPdaighXk7CXKb48Pe80=', 'Content-Length': '0'} 2022-09-03T06:33:16.462 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000947c77929d2cebb2-006312f52c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.468 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.469 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.470 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:16.471 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:16.472 DEBUG:boto:Method: GET 2022-09-03T06:33:16.473 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:16.474 DEBUG:boto:Data: 2022-09-03T06:33:16.475 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.476 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.477 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.478 DEBUG:boto:Params: {} 2022-09-03T06:33:16.479 DEBUG:boto:Token: None 2022-09-03T06:33:16.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/ 2022-09-03T06:33:16.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ACpguBgjN8fr67+S5173smqMtkI= 2022-09-03T06:33:16.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ACpguBgjN8fr67+S5173smqMtkI=', 'Content-Length': '0'} 2022-09-03T06:33:16.492 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b729ee68e7216e6b-006312f52c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.505 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:16.506 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.507 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.508 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:16.510 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:16.511 DEBUG:boto:Method: GET 2022-09-03T06:33:16.515 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:16.516 DEBUG:boto:Data: 2022-09-03T06:33:16.516 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.517 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.519 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.519 DEBUG:boto:Params: {} 2022-09-03T06:33:16.520 DEBUG:boto:Token: None 2022-09-03T06:33:16.520 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/ 2022-09-03T06:33:16.521 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ACpguBgjN8fr67+S5173smqMtkI= 2022-09-03T06:33:16.522 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ACpguBgjN8fr67+S5173smqMtkI=', 'Content-Length': '0'} 2022-09-03T06:33:16.557 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000608dc59d751c21a9-006312f52c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.566 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:16.567 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.568 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.568 DEBUG:boto:Method: HEAD 2022-09-03T06:33:16.569 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:16.574 DEBUG:boto:Data: 2022-09-03T06:33:16.575 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.575 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.576 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.577 DEBUG:boto:Params: {} 2022-09-03T06:33:16.578 DEBUG:boto:Token: None 2022-09-03T06:33:16.579 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1 2022-09-03T06:33:16.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4KoZI6MUUUIlB9vtkDfrENj6P/s= 2022-09-03T06:33:16.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4KoZI6MUUUIlB9vtkDfrENj6P/s=', 'Content-Length': '0'} 2022-09-03T06:33:16.603 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d95c23d0faf9bcdb-006312f52c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.604 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.605 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.606 DEBUG:boto:Method: HEAD 2022-09-03T06:33:16.620 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:16.621 DEBUG:boto:Data: 2022-09-03T06:33:16.631 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.632 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.633 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.639 DEBUG:boto:Params: {} 2022-09-03T06:33:16.640 DEBUG:boto:Token: None 2022-09-03T06:33:16.641 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1 2022-09-03T06:33:16.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4KoZI6MUUUIlB9vtkDfrENj6P/s= 2022-09-03T06:33:16.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4KoZI6MUUUIlB9vtkDfrENj6P/s=', 'Content-Length': '0'} 2022-09-03T06:33:16.647 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002c613d3d16729b73-006312f52c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.649 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:16.651 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.652 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.654 DEBUG:boto:Method: GET 2022-09-03T06:33:16.655 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:16.657 DEBUG:boto:Data: 2022-09-03T06:33:16.659 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.662 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.666 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.667 DEBUG:boto:Params: {} 2022-09-03T06:33:16.669 DEBUG:boto:Token: None 2022-09-03T06:33:16.670 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1 2022-09-03T06:33:16.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NCmUz2BkktUPaw+Uwu5Qg88D/7Q= 2022-09-03T06:33:16.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NCmUz2BkktUPaw+Uwu5Qg88D/7Q=', 'Content-Length': '0'} 2022-09-03T06:33:16.679 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000232782f2616e574e-006312f52c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.681 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:16.689 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:16.691 DEBUG:boto:Method: GET 2022-09-03T06:33:16.692 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:16.692 DEBUG:boto:Data: 2022-09-03T06:33:16.693 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.694 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.695 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.695 DEBUG:boto:Params: {} 2022-09-03T06:33:16.699 DEBUG:boto:Token: None 2022-09-03T06:33:16.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj1 2022-09-03T06:33:16.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NCmUz2BkktUPaw+Uwu5Qg88D/7Q= 2022-09-03T06:33:16.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NCmUz2BkktUPaw+Uwu5Qg88D/7Q=', 'Content-Length': '0'} 2022-09-03T06:33:16.708 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000000f1fbef19de62b4-006312f52c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.710 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.711 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.712 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:16.712 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:16.716 DEBUG:boto:Method: GET 2022-09-03T06:33:16.716 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:16.717 DEBUG:boto:Data: 2022-09-03T06:33:16.718 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.718 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.719 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.719 DEBUG:boto:Params: {} 2022-09-03T06:33:16.720 DEBUG:boto:Token: None 2022-09-03T06:33:16.721 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/ 2022-09-03T06:33:16.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ACpguBgjN8fr67+S5173smqMtkI= 2022-09-03T06:33:16.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ACpguBgjN8fr67+S5173smqMtkI=', 'Content-Length': '0'} 2022-09-03T06:33:16.725 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087e47e9034d29c8e-006312f52c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.726 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:16.735 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:16.745 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:16.745 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:16.746 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:16.747 DEBUG:boto:Method: GET 2022-09-03T06:33:16.747 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:16.748 DEBUG:boto:Data: 2022-09-03T06:33:16.748 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.749 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.758 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.759 DEBUG:boto:Params: {} 2022-09-03T06:33:16.761 DEBUG:boto:Token: None 2022-09-03T06:33:16.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/ 2022-09-03T06:33:16.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ACpguBgjN8fr67+S5173smqMtkI= 2022-09-03T06:33:16.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ACpguBgjN8fr67+S5173smqMtkI=', 'Content-Length': '0'} 2022-09-03T06:33:16.887 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2e9d1819dcf3cc3-006312f52c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.888 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:16.889 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:16.902 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:16.903 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:16.904 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:16.904 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:16.905 DEBUG:boto:Method: GET 2022-09-03T06:33:16.916 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:16.917 DEBUG:boto:Data: 2022-09-03T06:33:16.919 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.920 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:16.921 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.928 DEBUG:boto:Params: {} 2022-09-03T06:33:16.929 DEBUG:boto:Token: None 2022-09-03T06:33:16.930 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:16.931 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cv3WxZsSmurgW41t1RUaPBTovzI= 2022-09-03T06:33:16.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cv3WxZsSmurgW41t1RUaPBTovzI=', 'Content-Length': '0'} 2022-09-03T06:33:16.951 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000576eade6e8fcda0d-006312f52c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:16.952 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:16.984 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:16.985 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:16.986 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:16.987 DEBUG:boto:Method: GET 2022-09-03T06:33:16.989 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:16.990 DEBUG:boto:Data: 2022-09-03T06:33:16.991 DEBUG:boto:Headers: {} 2022-09-03T06:33:16.992 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:16.993 DEBUG:boto:Port: 8000 2022-09-03T06:33:16.995 DEBUG:boto:Params: {} 2022-09-03T06:33:16.996 DEBUG:boto:Token: None 2022-09-03T06:33:16.998 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:16 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:16.999 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cv3WxZsSmurgW41t1RUaPBTovzI= 2022-09-03T06:33:17.000 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cv3WxZsSmurgW41t1RUaPBTovzI=', 'Content-Length': '0'} 2022-09-03T06:33:17.003 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000eb301ab119940d8a-006312f52d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.005 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:17.005 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:17.016 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.017 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.025 DEBUG:boto:Method: HEAD 2022-09-03T06:33:17.028 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.029 DEBUG:boto:Data: 2022-09-03T06:33:17.030 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.031 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.031 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.032 DEBUG:boto:Params: {} 2022-09-03T06:33:17.034 DEBUG:boto:Token: None 2022-09-03T06:33:17.035 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TH/G3EM/vCmayRxdP+1eNzXIMz4= 2022-09-03T06:33:17.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TH/G3EM/vCmayRxdP+1eNzXIMz4=', 'Content-Length': '0'} 2022-09-03T06:33:17.048 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000041a7dd0ce0a4950d-006312f52d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.054 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:17.055 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:17.056 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.075 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.083 DEBUG:boto:Method: HEAD 2022-09-03T06:33:17.084 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.085 DEBUG:boto:Data: 2022-09-03T06:33:17.093 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.108 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.109 DEBUG:boto:Params: {} 2022-09-03T06:33:17.109 DEBUG:boto:Token: None 2022-09-03T06:33:17.110 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TH/G3EM/vCmayRxdP+1eNzXIMz4= 2022-09-03T06:33:17.124 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TH/G3EM/vCmayRxdP+1eNzXIMz4=', 'Content-Length': '0'} 2022-09-03T06:33:17.139 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000621be0f6202e95fc-006312f52d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.141 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:17.144 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:17.145 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:17.146 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.147 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.149 DEBUG:boto:Method: GET 2022-09-03T06:33:17.150 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.151 DEBUG:boto:Data: 2022-09-03T06:33:17.152 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.153 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.155 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.156 DEBUG:boto:Params: {} 2022-09-03T06:33:17.159 DEBUG:boto:Token: None 2022-09-03T06:33:17.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:abecpN9VsG3UQB67/Q3xOWwZ9F8= 2022-09-03T06:33:17.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:abecpN9VsG3UQB67/Q3xOWwZ9F8=', 'Content-Length': '0'} 2022-09-03T06:33:17.179 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004dd26815aa166d22-006312f52d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.180 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:17.181 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:17.182 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.182 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.183 DEBUG:boto:Method: GET 2022-09-03T06:33:17.183 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.194 DEBUG:boto:Data: 2022-09-03T06:33:17.195 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.204 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.205 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.206 DEBUG:boto:Params: {} 2022-09-03T06:33:17.207 DEBUG:boto:Token: None 2022-09-03T06:33:17.208 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:17.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:abecpN9VsG3UQB67/Q3xOWwZ9F8= 2022-09-03T06:33:17.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:abecpN9VsG3UQB67/Q3xOWwZ9F8=', 'Content-Length': '0'} 2022-09-03T06:33:17.215 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002b2e9b94d516e7a9-006312f52d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.217 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:17.218 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:17.220 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:17.221 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:17.222 DEBUG:boto:Method: GET 2022-09-03T06:33:17.223 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:17.224 DEBUG:boto:Data: 2022-09-03T06:33:17.227 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.228 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.229 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.230 DEBUG:boto:Params: {} 2022-09-03T06:33:17.231 DEBUG:boto:Token: None 2022-09-03T06:33:17.232 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/ 2022-09-03T06:33:17.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4= 2022-09-03T06:33:17.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4=', 'Content-Length': '0'} 2022-09-03T06:33:17.241 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ab31f37e0c31869-006312f52d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.243 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:17.245 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:17.246 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:17.247 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:17.247 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:17.249 DEBUG:boto:Method: GET 2022-09-03T06:33:17.251 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:17.252 DEBUG:boto:Data: 2022-09-03T06:33:17.253 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.254 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.255 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.256 DEBUG:boto:Params: {} 2022-09-03T06:33:17.257 DEBUG:boto:Token: None 2022-09-03T06:33:17.259 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/ 2022-09-03T06:33:17.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4= 2022-09-03T06:33:17.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4=', 'Content-Length': '0'} 2022-09-03T06:33:17.269 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000016ab386a2b5d2730-006312f52d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.273 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:17.275 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:17.276 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:17.284 DEBUG:boto:Method: HEAD 2022-09-03T06:33:17.285 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:17.286 DEBUG:boto:Data: 2022-09-03T06:33:17.287 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.288 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.289 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.295 DEBUG:boto:Params: {} 2022-09-03T06:33:17.296 DEBUG:boto:Token: None 2022-09-03T06:33:17.297 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj2 2022-09-03T06:33:17.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M9xEQ1zq9PrqAdFOZyDtvmWw14c= 2022-09-03T06:33:17.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M9xEQ1zq9PrqAdFOZyDtvmWw14c=', 'Content-Length': '0'} 2022-09-03T06:33:17.303 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d23c13788357b277-006312f52d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.304 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:17.305 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:17.306 DEBUG:boto:Method: HEAD 2022-09-03T06:33:17.307 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:17.308 DEBUG:boto:Data: 2022-09-03T06:33:17.309 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.310 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.311 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.312 DEBUG:boto:Params: {} 2022-09-03T06:33:17.313 DEBUG:boto:Token: None 2022-09-03T06:33:17.315 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj2 2022-09-03T06:33:17.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M9xEQ1zq9PrqAdFOZyDtvmWw14c= 2022-09-03T06:33:17.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M9xEQ1zq9PrqAdFOZyDtvmWw14c=', 'Content-Length': '0'} 2022-09-03T06:33:17.324 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fde5976ae0dd5c7f-006312f52d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.325 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:17.326 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:17.327 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:17.328 DEBUG:boto:Method: GET 2022-09-03T06:33:17.330 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:17.331 DEBUG:boto:Data: 2022-09-03T06:33:17.332 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.345 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.346 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.347 DEBUG:boto:Params: {} 2022-09-03T06:33:17.348 DEBUG:boto:Token: None 2022-09-03T06:33:17.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj2 2022-09-03T06:33:17.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NG4VOJqQkM0l0l+UQbKjFp/JgN8= 2022-09-03T06:33:17.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NG4VOJqQkM0l0l+UQbKjFp/JgN8=', 'Content-Length': '0'} 2022-09-03T06:33:17.374 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000733f5b12bbeccaf3-006312f52d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.376 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:17.377 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:17.382 DEBUG:boto:Method: GET 2022-09-03T06:33:17.383 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:17.384 DEBUG:boto:Data: 2022-09-03T06:33:17.385 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.392 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.394 DEBUG:boto:Params: {} 2022-09-03T06:33:17.395 DEBUG:boto:Token: None 2022-09-03T06:33:17.396 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj2 2022-09-03T06:33:17.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NG4VOJqQkM0l0l+UQbKjFp/JgN8= 2022-09-03T06:33:17.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NG4VOJqQkM0l0l+UQbKjFp/JgN8=', 'Content-Length': '0'} 2022-09-03T06:33:17.402 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000380f2a637713e80f-006312f52d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.404 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:17.405 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:17.406 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:17.407 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:17.407 DEBUG:boto:Method: GET 2022-09-03T06:33:17.408 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:17.409 DEBUG:boto:Data: 2022-09-03T06:33:17.410 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.411 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.412 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.412 DEBUG:boto:Params: {} 2022-09-03T06:33:17.413 DEBUG:boto:Token: None 2022-09-03T06:33:17.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/ 2022-09-03T06:33:17.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4= 2022-09-03T06:33:17.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4=', 'Content-Length': '0'} 2022-09-03T06:33:17.421 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006cbbd92c0e70027e-006312f52d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.422 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:17.424 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:17.425 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:17.426 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:17.427 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:17.428 DEBUG:boto:Method: GET 2022-09-03T06:33:17.429 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:17.432 DEBUG:boto:Data: 2022-09-03T06:33:17.433 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.435 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.436 DEBUG:boto:Params: {} 2022-09-03T06:33:17.437 DEBUG:boto:Token: None 2022-09-03T06:33:17.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/ 2022-09-03T06:33:17.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4= 2022-09-03T06:33:17.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4=', 'Content-Length': '0'} 2022-09-03T06:33:17.470 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009bd38ce54c429638-006312f52d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.472 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:17.473 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:17.483 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:17.485 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:17.486 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.487 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.488 DEBUG:boto:Method: GET 2022-09-03T06:33:17.490 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.491 DEBUG:boto:Data: 2022-09-03T06:33:17.492 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.493 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.495 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.496 DEBUG:boto:Params: {} 2022-09-03T06:33:17.497 DEBUG:boto:Token: None 2022-09-03T06:33:17.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gSiMx9W98diohx1WpYjTrFO8t4M= 2022-09-03T06:33:17.500 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gSiMx9W98diohx1WpYjTrFO8t4M=', 'Content-Length': '0'} 2022-09-03T06:33:17.511 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cf2c83aeaa43c0ef-006312f52d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.513 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:17.514 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:17.515 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.516 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.526 DEBUG:boto:Method: GET 2022-09-03T06:33:17.527 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.528 DEBUG:boto:Data: 2022-09-03T06:33:17.529 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.530 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.532 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.534 DEBUG:boto:Params: {} 2022-09-03T06:33:17.541 DEBUG:boto:Token: None 2022-09-03T06:33:17.542 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gSiMx9W98diohx1WpYjTrFO8t4M= 2022-09-03T06:33:17.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gSiMx9W98diohx1WpYjTrFO8t4M=', 'Content-Length': '0'} 2022-09-03T06:33:17.565 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000090aa6bb406629df6-006312f52d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.567 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:17.568 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:17.577 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.579 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.579 DEBUG:boto:Method: HEAD 2022-09-03T06:33:17.580 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.581 DEBUG:boto:Data: 2022-09-03T06:33:17.582 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.583 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.583 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.585 DEBUG:boto:Params: {} 2022-09-03T06:33:17.586 DEBUG:boto:Token: None 2022-09-03T06:33:17.586 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dNP24SjvKbjYXIR2yPTvYvpqeFA= 2022-09-03T06:33:17.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dNP24SjvKbjYXIR2yPTvYvpqeFA=', 'Content-Length': '0'} 2022-09-03T06:33:17.599 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002a3f9ac1c2946f10-006312f52d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.601 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:17.602 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:17.603 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.624 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.625 DEBUG:boto:Method: HEAD 2022-09-03T06:33:17.645 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.646 DEBUG:boto:Data: 2022-09-03T06:33:17.647 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.648 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.649 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.665 DEBUG:boto:Params: {} 2022-09-03T06:33:17.667 DEBUG:boto:Token: None 2022-09-03T06:33:17.668 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dNP24SjvKbjYXIR2yPTvYvpqeFA= 2022-09-03T06:33:17.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dNP24SjvKbjYXIR2yPTvYvpqeFA=', 'Content-Length': '0'} 2022-09-03T06:33:17.677 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000014f0df254fd20f1-006312f52d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.695 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:17.695 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:17.696 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:17.697 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.697 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.698 DEBUG:boto:Method: GET 2022-09-03T06:33:17.698 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.699 DEBUG:boto:Data: 2022-09-03T06:33:17.699 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.701 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.701 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.702 DEBUG:boto:Params: {} 2022-09-03T06:33:17.703 DEBUG:boto:Token: None 2022-09-03T06:33:17.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gSiMx9W98diohx1WpYjTrFO8t4M= 2022-09-03T06:33:17.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gSiMx9W98diohx1WpYjTrFO8t4M=', 'Content-Length': '0'} 2022-09-03T06:33:17.730 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000178f5d98ac56b30b-006312f52d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.732 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:17.733 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:17.734 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.736 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.737 DEBUG:boto:Method: GET 2022-09-03T06:33:17.738 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.741 DEBUG:boto:Data: 2022-09-03T06:33:17.743 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.744 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.745 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.749 DEBUG:boto:Params: {} 2022-09-03T06:33:17.752 DEBUG:boto:Token: None 2022-09-03T06:33:17.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:17.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gSiMx9W98diohx1WpYjTrFO8t4M= 2022-09-03T06:33:17.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gSiMx9W98diohx1WpYjTrFO8t4M=', 'Content-Length': '0'} 2022-09-03T06:33:17.780 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000016a6344b98ce1934-006312f52d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.782 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:17.782 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:17.783 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:17.783 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:17.784 DEBUG:boto:Method: GET 2022-09-03T06:33:17.785 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:17.786 DEBUG:boto:Data: 2022-09-03T06:33:17.787 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.788 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.789 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.797 DEBUG:boto:Params: {} 2022-09-03T06:33:17.798 DEBUG:boto:Token: None 2022-09-03T06:33:17.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/ 2022-09-03T06:33:17.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4= 2022-09-03T06:33:17.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4=', 'Content-Length': '0'} 2022-09-03T06:33:17.810 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dbe2646968d67507-006312f52d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.813 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:17.814 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:17.816 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:17.817 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:17.841 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:17.842 DEBUG:boto:Method: GET 2022-09-03T06:33:17.843 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:17.844 DEBUG:boto:Data: 2022-09-03T06:33:17.845 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.862 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.863 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.863 DEBUG:boto:Params: {} 2022-09-03T06:33:17.864 DEBUG:boto:Token: None 2022-09-03T06:33:17.865 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/ 2022-09-03T06:33:17.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4= 2022-09-03T06:33:17.868 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lc5eLt31e7KwpMLqBVGm/xDGby4=', 'Content-Length': '0'} 2022-09-03T06:33:17.888 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000504ec1c303ec2456-006312f52d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.889 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:17.898 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:17.918 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:17.919 DEBUG:boto:Method: HEAD 2022-09-03T06:33:17.919 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:17.920 DEBUG:boto:Data: 2022-09-03T06:33:17.920 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.921 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.921 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.929 DEBUG:boto:Params: {} 2022-09-03T06:33:17.929 DEBUG:boto:Token: None 2022-09-03T06:33:17.930 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj3 2022-09-03T06:33:17.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RvMqnuE0i7lqpnh4MMKJlkWYODE= 2022-09-03T06:33:17.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RvMqnuE0i7lqpnh4MMKJlkWYODE=', 'Content-Length': '0'} 2022-09-03T06:33:17.938 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000018a60ff79a2c9ef9-006312f52d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.939 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:17.940 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:17.940 DEBUG:boto:Method: HEAD 2022-09-03T06:33:17.941 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:17.941 DEBUG:boto:Data: 2022-09-03T06:33:17.946 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:17.950 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.951 DEBUG:boto:Params: {} 2022-09-03T06:33:17.951 DEBUG:boto:Token: None 2022-09-03T06:33:17.952 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:17 GMT /avysmw-60/obj3 2022-09-03T06:33:17.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RvMqnuE0i7lqpnh4MMKJlkWYODE= 2022-09-03T06:33:17.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RvMqnuE0i7lqpnh4MMKJlkWYODE=', 'Content-Length': '0'} 2022-09-03T06:33:17.960 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000db7e0d6bf35196d9-006312f52d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:17.961 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:17.962 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:17.970 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:17.972 DEBUG:boto:Method: GET 2022-09-03T06:33:17.973 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:17.973 DEBUG:boto:Data: 2022-09-03T06:33:17.982 DEBUG:boto:Headers: {} 2022-09-03T06:33:17.984 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:17.985 DEBUG:boto:Port: 8000 2022-09-03T06:33:17.986 DEBUG:boto:Params: {} 2022-09-03T06:33:17.999 DEBUG:boto:Token: None 2022-09-03T06:33:18.006 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj3 2022-09-03T06:33:18.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ifZ9vMuXjCMQADpOZdDPneVH7pc= 2022-09-03T06:33:18.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ifZ9vMuXjCMQADpOZdDPneVH7pc=', 'Content-Length': '0'} 2022-09-03T06:33:18.020 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009baf062c10c2b7f3-006312f52e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.023 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:18.024 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:18.025 DEBUG:boto:Method: GET 2022-09-03T06:33:18.034 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:18.035 DEBUG:boto:Data: 2022-09-03T06:33:18.036 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.037 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.038 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.039 DEBUG:boto:Params: {} 2022-09-03T06:33:18.041 DEBUG:boto:Token: None 2022-09-03T06:33:18.045 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj3 2022-09-03T06:33:18.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ifZ9vMuXjCMQADpOZdDPneVH7pc= 2022-09-03T06:33:18.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ifZ9vMuXjCMQADpOZdDPneVH7pc=', 'Content-Length': '0'} 2022-09-03T06:33:18.053 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000021cf51835ca43ed7-006312f52e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.061 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.068 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.074 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:18.075 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:18.086 DEBUG:boto:Method: GET 2022-09-03T06:33:18.100 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:18.107 DEBUG:boto:Data: 2022-09-03T06:33:18.112 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.114 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:18.116 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.117 DEBUG:boto:Params: {} 2022-09-03T06:33:18.119 DEBUG:boto:Token: None 2022-09-03T06:33:18.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/ 2022-09-03T06:33:18.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc= 2022-09-03T06:33:18.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc=', 'Content-Length': '0'} 2022-09-03T06:33:18.137 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d40e743e31acff92-006312f52e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.140 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.144 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.152 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.156 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:18.157 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:18.158 DEBUG:boto:Method: GET 2022-09-03T06:33:18.159 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:18.160 DEBUG:boto:Data: 2022-09-03T06:33:18.162 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.164 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.166 DEBUG:boto:Params: {} 2022-09-03T06:33:18.167 DEBUG:boto:Token: None 2022-09-03T06:33:18.168 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/ 2022-09-03T06:33:18.169 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc= 2022-09-03T06:33:18.170 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc=', 'Content-Length': '0'} 2022-09-03T06:33:18.174 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000038d2e789d9734979-006312f52e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.175 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.176 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:18.177 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.178 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.179 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.180 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.186 DEBUG:boto:Method: GET 2022-09-03T06:33:18.187 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.187 DEBUG:boto:Data: 2022-09-03T06:33:18.188 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.188 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:18.189 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.189 DEBUG:boto:Params: {} 2022-09-03T06:33:18.190 DEBUG:boto:Token: None 2022-09-03T06:33:18.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qgmGHhYr1o7Bori5UXkwcp7FbRA= 2022-09-03T06:33:18.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qgmGHhYr1o7Bori5UXkwcp7FbRA=', 'Content-Length': '0'} 2022-09-03T06:33:18.208 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003e98099ee4dfe876-006312f52e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.213 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.215 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.215 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.216 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.216 DEBUG:boto:Method: GET 2022-09-03T06:33:18.217 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.217 DEBUG:boto:Data: 2022-09-03T06:33:18.219 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.226 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.227 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.227 DEBUG:boto:Params: {} 2022-09-03T06:33:18.228 DEBUG:boto:Token: None 2022-09-03T06:33:18.229 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qgmGHhYr1o7Bori5UXkwcp7FbRA= 2022-09-03T06:33:18.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qgmGHhYr1o7Bori5UXkwcp7FbRA=', 'Content-Length': '0'} 2022-09-03T06:33:18.243 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b2b48a278ae4ea69-006312f52e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.244 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.244 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.245 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.253 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.254 DEBUG:boto:Method: HEAD 2022-09-03T06:33:18.255 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.255 DEBUG:boto:Data: 2022-09-03T06:33:18.256 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.256 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:18.257 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.257 DEBUG:boto:Params: {} 2022-09-03T06:33:18.258 DEBUG:boto:Token: None 2022-09-03T06:33:18.259 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4DAIP1Us+G42w77UwMVaI1Bro3A= 2022-09-03T06:33:18.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4DAIP1Us+G42w77UwMVaI1Bro3A=', 'Content-Length': '0'} 2022-09-03T06:33:18.270 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ba2d2deaef915316-006312f52e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.273 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.274 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.275 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.276 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.278 DEBUG:boto:Method: HEAD 2022-09-03T06:33:18.279 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.280 DEBUG:boto:Data: 2022-09-03T06:33:18.281 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.283 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.284 DEBUG:boto:Params: {} 2022-09-03T06:33:18.286 DEBUG:boto:Token: None 2022-09-03T06:33:18.287 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4DAIP1Us+G42w77UwMVaI1Bro3A= 2022-09-03T06:33:18.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4DAIP1Us+G42w77UwMVaI1Bro3A=', 'Content-Length': '0'} 2022-09-03T06:33:18.293 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003d2db0f110c17c5c-006312f52e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.295 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:18.295 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.296 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.297 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.297 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.298 DEBUG:boto:Method: GET 2022-09-03T06:33:18.307 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.309 DEBUG:boto:Data: 2022-09-03T06:33:18.310 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.311 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:18.312 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.313 DEBUG:boto:Params: {} 2022-09-03T06:33:18.315 DEBUG:boto:Token: None 2022-09-03T06:33:18.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qgmGHhYr1o7Bori5UXkwcp7FbRA= 2022-09-03T06:33:18.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qgmGHhYr1o7Bori5UXkwcp7FbRA=', 'Content-Length': '0'} 2022-09-03T06:33:18.329 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006259c810df55f364-006312f52e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.330 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.331 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.332 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.332 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.335 DEBUG:boto:Method: GET 2022-09-03T06:33:18.339 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.340 DEBUG:boto:Data: 2022-09-03T06:33:18.341 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.342 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.343 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.344 DEBUG:boto:Params: {} 2022-09-03T06:33:18.345 DEBUG:boto:Token: None 2022-09-03T06:33:18.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:18.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qgmGHhYr1o7Bori5UXkwcp7FbRA= 2022-09-03T06:33:18.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qgmGHhYr1o7Bori5UXkwcp7FbRA=', 'Content-Length': '0'} 2022-09-03T06:33:18.362 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003d1797533b423a99-006312f52e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.364 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.365 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.366 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:18.367 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:18.368 DEBUG:boto:Method: GET 2022-09-03T06:33:18.369 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:18.370 DEBUG:boto:Data: 2022-09-03T06:33:18.371 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.372 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:18.373 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.375 DEBUG:boto:Params: {} 2022-09-03T06:33:18.376 DEBUG:boto:Token: None 2022-09-03T06:33:18.377 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/ 2022-09-03T06:33:18.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc= 2022-09-03T06:33:18.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc=', 'Content-Length': '0'} 2022-09-03T06:33:18.396 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011433282e045f639-006312f52e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.397 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.398 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.399 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.399 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:18.400 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:18.400 DEBUG:boto:Method: GET 2022-09-03T06:33:18.401 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:18.401 DEBUG:boto:Data: 2022-09-03T06:33:18.402 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.403 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.404 DEBUG:boto:Params: {} 2022-09-03T06:33:18.404 DEBUG:boto:Token: None 2022-09-03T06:33:18.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/ 2022-09-03T06:33:18.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc= 2022-09-03T06:33:18.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc=', 'Content-Length': '0'} 2022-09-03T06:33:18.410 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c1e0bd260bd7705-006312f52e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.412 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.414 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.415 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.417 DEBUG:boto:Method: HEAD 2022-09-03T06:33:18.419 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:18.420 DEBUG:boto:Data: 2022-09-03T06:33:18.421 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.421 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:18.422 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.423 DEBUG:boto:Params: {} 2022-09-03T06:33:18.424 DEBUG:boto:Token: None 2022-09-03T06:33:18.425 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4 2022-09-03T06:33:18.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:icToq0wIkGRHj/1kN+ou+W4lp24= 2022-09-03T06:33:18.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:icToq0wIkGRHj/1kN+ou+W4lp24=', 'Content-Length': '0'} 2022-09-03T06:33:18.430 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e975a52e2f8c0ae6-006312f52e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.433 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.444 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.445 DEBUG:boto:Method: HEAD 2022-09-03T06:33:18.446 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:18.447 DEBUG:boto:Data: 2022-09-03T06:33:18.448 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.449 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.449 DEBUG:boto:Params: {} 2022-09-03T06:33:18.450 DEBUG:boto:Token: None 2022-09-03T06:33:18.451 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4 2022-09-03T06:33:18.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:icToq0wIkGRHj/1kN+ou+W4lp24= 2022-09-03T06:33:18.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:icToq0wIkGRHj/1kN+ou+W4lp24=', 'Content-Length': '0'} 2022-09-03T06:33:18.465 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003120d2e17f670f5f-006312f52e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.466 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:18.467 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.467 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.468 DEBUG:boto:Method: GET 2022-09-03T06:33:18.469 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:18.469 DEBUG:boto:Data: 2022-09-03T06:33:18.488 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.497 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:18.497 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.502 DEBUG:boto:Params: {} 2022-09-03T06:33:18.503 DEBUG:boto:Token: None 2022-09-03T06:33:18.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4 2022-09-03T06:33:18.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7sC5Pp82LOwDFJ9Bgbjcetm3Uj8= 2022-09-03T06:33:18.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7sC5Pp82LOwDFJ9Bgbjcetm3Uj8=', 'Content-Length': '0'} 2022-09-03T06:33:18.520 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000034ee5c354616c18e-006312f52e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.524 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:18.527 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:18.528 DEBUG:boto:Method: GET 2022-09-03T06:33:18.529 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:18.530 DEBUG:boto:Data: 2022-09-03T06:33:18.530 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.531 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.532 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.532 DEBUG:boto:Params: {} 2022-09-03T06:33:18.533 DEBUG:boto:Token: None 2022-09-03T06:33:18.534 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj4 2022-09-03T06:33:18.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7sC5Pp82LOwDFJ9Bgbjcetm3Uj8= 2022-09-03T06:33:18.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7sC5Pp82LOwDFJ9Bgbjcetm3Uj8=', 'Content-Length': '0'} 2022-09-03T06:33:18.541 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004d1a5c28c6c8b2e3-006312f52e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.547 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.548 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.549 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:18.550 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:18.551 DEBUG:boto:Method: GET 2022-09-03T06:33:18.552 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:18.553 DEBUG:boto:Data: 2022-09-03T06:33:18.555 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.556 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:18.557 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.559 DEBUG:boto:Params: {} 2022-09-03T06:33:18.560 DEBUG:boto:Token: None 2022-09-03T06:33:18.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/ 2022-09-03T06:33:18.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc= 2022-09-03T06:33:18.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc=', 'Content-Length': '0'} 2022-09-03T06:33:18.570 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000acf4be913c460782-006312f52e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.571 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.573 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.575 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.576 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:18.578 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:18.579 DEBUG:boto:Method: GET 2022-09-03T06:33:18.580 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:18.581 DEBUG:boto:Data: 2022-09-03T06:33:18.582 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.585 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.586 DEBUG:boto:Params: {} 2022-09-03T06:33:18.587 DEBUG:boto:Token: None 2022-09-03T06:33:18.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/ 2022-09-03T06:33:18.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc= 2022-09-03T06:33:18.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oOJlC1rrXSxGc/zxoZSTVBzGCMc=', 'Content-Length': '0'} 2022-09-03T06:33:18.598 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007dc77c83706311c5-006312f52e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.601 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.603 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-60 zones={test-zone2, test-zone1} 2022-09-03T06:33:18.604 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-60 zones={test-zone3, test-zone1} 2022-09-03T06:33:18.608 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.609 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.610 DEBUG:boto:Method: HEAD 2022-09-03T06:33:18.617 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:33:18.619 DEBUG:boto:Data: 2022-09-03T06:33:18.620 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.621 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:18.621 DEBUG:boto:Port: 8001 2022-09-03T06:33:18.622 DEBUG:boto:Params: {} 2022-09-03T06:33:18.623 DEBUG:boto:Token: None 2022-09-03T06:33:18.627 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/ 2022-09-03T06:33:18.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LCDXrtKu9y1+Fs0WpPP9nlGwVjk= 2022-09-03T06:33:18.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LCDXrtKu9y1+Fs0WpPP9nlGwVjk=', 'Content-Length': '0'} 2022-09-03T06:33:18.634 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f45028a5a3fc3d13-006312f52e-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.635 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.636 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.637 DEBUG:boto:Method: HEAD 2022-09-03T06:33:18.638 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:33:18.639 DEBUG:boto:Data: 2022-09-03T06:33:18.640 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.641 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.641 DEBUG:boto:Params: {} 2022-09-03T06:33:18.642 DEBUG:boto:Token: None 2022-09-03T06:33:18.643 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/ 2022-09-03T06:33:18.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LCDXrtKu9y1+Fs0WpPP9nlGwVjk= 2022-09-03T06:33:18.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LCDXrtKu9y1+Fs0WpPP9nlGwVjk=', 'Content-Length': '0'} 2022-09-03T06:33:18.651 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007077c11dca71e847-006312f52e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.652 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:33:18.652 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.653 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.653 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.654 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.654 DEBUG:boto:Method: GET 2022-09-03T06:33:18.655 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.655 DEBUG:boto:Data: 2022-09-03T06:33:18.656 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:18.658 DEBUG:boto:Port: 8001 2022-09-03T06:33:18.658 DEBUG:boto:Params: {} 2022-09-03T06:33:18.659 DEBUG:boto:Token: None 2022-09-03T06:33:18.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/?versions 2022-09-03T06:33:18.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qfEHQucOPjXa0yQenM0LDSYJouE= 2022-09-03T06:33:18.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qfEHQucOPjXa0yQenM0LDSYJouE=', 'Content-Length': '0'} 2022-09-03T06:33:18.666 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009c8688ac96d6751b-006312f52e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.667 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.668 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:18.669 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:18.669 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:18.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:18.670 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:33:18.671 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.672 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.672 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.673 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.673 DEBUG:boto:Method: GET 2022-09-03T06:33:18.674 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.674 DEBUG:boto:Data: 2022-09-03T06:33:18.675 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.676 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.677 DEBUG:boto:Params: {} 2022-09-03T06:33:18.678 DEBUG:boto:Token: None 2022-09-03T06:33:18.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/?versions 2022-09-03T06:33:18.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qfEHQucOPjXa0yQenM0LDSYJouE= 2022-09-03T06:33:18.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qfEHQucOPjXa0yQenM0LDSYJouE=', 'Content-Length': '0'} 2022-09-03T06:33:18.683 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000cb023574d000acf-006312f52e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.684 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.685 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:18.685 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:18.690 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:18.691 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:18.692 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.693 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.694 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.695 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.696 DEBUG:boto:Method: GET 2022-09-03T06:33:18.697 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.698 DEBUG:boto:Data: 2022-09-03T06:33:18.699 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.700 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:18.702 DEBUG:boto:Port: 8001 2022-09-03T06:33:18.703 DEBUG:boto:Params: {} 2022-09-03T06:33:18.704 DEBUG:boto:Token: None 2022-09-03T06:33:18.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/?versions 2022-09-03T06:33:18.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qfEHQucOPjXa0yQenM0LDSYJouE= 2022-09-03T06:33:18.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qfEHQucOPjXa0yQenM0LDSYJouE=', 'Content-Length': '0'} 2022-09-03T06:33:18.717 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b1e209d696625f27-006312f52e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.724 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.726 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:18.727 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:18.728 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.729 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.737 DEBUG:boto:Method: GET 2022-09-03T06:33:18.737 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:18.738 DEBUG:boto:Data: 2022-09-03T06:33:18.738 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.739 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.740 DEBUG:boto:Params: {} 2022-09-03T06:33:18.741 DEBUG:boto:Token: None 2022-09-03T06:33:18.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/?versions 2022-09-03T06:33:18.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qfEHQucOPjXa0yQenM0LDSYJouE= 2022-09-03T06:33:18.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qfEHQucOPjXa0yQenM0LDSYJouE=', 'Content-Length': '0'} 2022-09-03T06:33:18.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e4b73f0a623e13a0-006312f52e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.757 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:18.758 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:18.759 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:18.759 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:18.760 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.760 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.761 DEBUG:boto:Method: GET 2022-09-03T06:33:18.762 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.762 DEBUG:boto:Data: 2022-09-03T06:33:18.763 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:18.764 DEBUG:boto:Port: 8001 2022-09-03T06:33:18.764 DEBUG:boto:Params: {} 2022-09-03T06:33:18.765 DEBUG:boto:Token: None 2022-09-03T06:33:18.765 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CEwkeiWV9F83+kMBxj4NulWjqZY= 2022-09-03T06:33:18.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CEwkeiWV9F83+kMBxj4NulWjqZY=', 'Content-Length': '0'} 2022-09-03T06:33:18.772 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005fbe88cfad1764ee-006312f52e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.774 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:18.775 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:18.776 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.777 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.779 DEBUG:boto:Method: GET 2022-09-03T06:33:18.781 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.782 DEBUG:boto:Data: 2022-09-03T06:33:18.782 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.784 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.785 DEBUG:boto:Params: {} 2022-09-03T06:33:18.785 DEBUG:boto:Token: None 2022-09-03T06:33:18.786 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CEwkeiWV9F83+kMBxj4NulWjqZY= 2022-09-03T06:33:18.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CEwkeiWV9F83+kMBxj4NulWjqZY=', 'Content-Length': '0'} 2022-09-03T06:33:18.791 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008614673c32bbe06a-006312f52e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.792 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:18.792 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:18.793 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.793 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.794 DEBUG:boto:Method: HEAD 2022-09-03T06:33:18.795 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.795 DEBUG:boto:Data: 2022-09-03T06:33:18.796 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.796 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:18.797 DEBUG:boto:Port: 8001 2022-09-03T06:33:18.797 DEBUG:boto:Params: {} 2022-09-03T06:33:18.798 DEBUG:boto:Token: None 2022-09-03T06:33:18.799 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UPRVnUU+i7R574+aqZ0+jrWIMz0= 2022-09-03T06:33:18.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UPRVnUU+i7R574+aqZ0+jrWIMz0=', 'Content-Length': '0'} 2022-09-03T06:33:18.802 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f28e23675937fd5a-006312f52e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.803 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:18.803 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:18.804 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.804 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.805 DEBUG:boto:Method: HEAD 2022-09-03T06:33:18.805 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.808 DEBUG:boto:Data: 2022-09-03T06:33:18.808 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:18.810 DEBUG:boto:Port: 8000 2022-09-03T06:33:18.811 DEBUG:boto:Params: {} 2022-09-03T06:33:18.812 DEBUG:boto:Token: None 2022-09-03T06:33:18.812 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UPRVnUU+i7R574+aqZ0+jrWIMz0= 2022-09-03T06:33:18.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UPRVnUU+i7R574+aqZ0+jrWIMz0=', 'Content-Length': '0'} 2022-09-03T06:33:18.829 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c6d2cd58d8070182-006312f52e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.830 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:18.831 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:18.832 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:18.832 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.833 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.833 DEBUG:boto:Method: GET 2022-09-03T06:33:18.835 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.838 DEBUG:boto:Data: 2022-09-03T06:33:18.839 DEBUG:boto:Headers: {} 2022-09-03T06:33:18.840 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:18.858 DEBUG:boto:Port: 8001 2022-09-03T06:33:18.859 DEBUG:boto:Params: {} 2022-09-03T06:33:18.860 DEBUG:boto:Token: None 2022-09-03T06:33:18.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:18 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CEwkeiWV9F83+kMBxj4NulWjqZY= 2022-09-03T06:33:18.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CEwkeiWV9F83+kMBxj4NulWjqZY=', 'Content-Length': '0'} 2022-09-03T06:33:18.870 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c3f6789c44fa0a96-006312f52e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:18.871 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:18.872 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:18.967 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.970 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.971 DEBUG:boto:Method: GET 2022-09-03T06:33:18.972 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:18.989 DEBUG:boto:Data: 2022-09-03T06:33:19.001 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.012 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.013 DEBUG:boto:Params: {} 2022-09-03T06:33:19.014 DEBUG:boto:Token: None 2022-09-03T06:33:19.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:19.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Sd9e2zj8BD0pIAjsG/kcav884s= 2022-09-03T06:33:19.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Sd9e2zj8BD0pIAjsG/kcav884s=', 'Content-Length': '0'} 2022-09-03T06:33:19.025 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000048d1339819e10339-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.046 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.047 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.048 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:19.052 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:19.053 DEBUG:boto:Method: GET 2022-09-03T06:33:19.054 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:19.055 DEBUG:boto:Data: 2022-09-03T06:33:19.056 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.059 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.068 DEBUG:boto:Params: {} 2022-09-03T06:33:19.069 DEBUG:boto:Token: None 2022-09-03T06:33:19.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/ 2022-09-03T06:33:19.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc= 2022-09-03T06:33:19.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc=', 'Content-Length': '0'} 2022-09-03T06:33:19.079 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e6adb57d12192be2-006312f52f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.080 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:19.083 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.084 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.085 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:19.085 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:19.086 DEBUG:boto:Method: GET 2022-09-03T06:33:19.087 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:19.088 DEBUG:boto:Data: 2022-09-03T06:33:19.088 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.089 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.089 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.090 DEBUG:boto:Params: {} 2022-09-03T06:33:19.091 DEBUG:boto:Token: None 2022-09-03T06:33:19.092 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/ 2022-09-03T06:33:19.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc= 2022-09-03T06:33:19.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc=', 'Content-Length': '0'} 2022-09-03T06:33:19.113 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e547309b34771b1e-006312f52f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.114 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:19.115 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:19.116 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:19.116 DEBUG:boto:Method: HEAD 2022-09-03T06:33:19.117 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:19.117 DEBUG:boto:Data: 2022-09-03T06:33:19.118 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.118 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.119 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.119 DEBUG:boto:Params: {} 2022-09-03T06:33:19.120 DEBUG:boto:Token: None 2022-09-03T06:33:19.120 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj1 2022-09-03T06:33:19.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q26KwkS59JJK7drmwPEAIEtzmgc= 2022-09-03T06:33:19.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q26KwkS59JJK7drmwPEAIEtzmgc=', 'Content-Length': '0'} 2022-09-03T06:33:19.124 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f2c250559b4ac45b-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.124 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:19.125 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:19.126 DEBUG:boto:Method: HEAD 2022-09-03T06:33:19.127 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:19.127 DEBUG:boto:Data: 2022-09-03T06:33:19.128 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.133 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.134 DEBUG:boto:Params: {} 2022-09-03T06:33:19.135 DEBUG:boto:Token: None 2022-09-03T06:33:19.136 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj1 2022-09-03T06:33:19.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q26KwkS59JJK7drmwPEAIEtzmgc= 2022-09-03T06:33:19.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q26KwkS59JJK7drmwPEAIEtzmgc=', 'Content-Length': '0'} 2022-09-03T06:33:19.145 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007b575ccd5bbfc2a9-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.151 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:19.153 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:19.156 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:19.157 DEBUG:boto:Method: GET 2022-09-03T06:33:19.158 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:19.159 DEBUG:boto:Data: 2022-09-03T06:33:19.160 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.161 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.162 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.163 DEBUG:boto:Params: {} 2022-09-03T06:33:19.164 DEBUG:boto:Token: None 2022-09-03T06:33:19.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj1 2022-09-03T06:33:19.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FTFFO3YN8EhJBPlzoe83S8+nhTc= 2022-09-03T06:33:19.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FTFFO3YN8EhJBPlzoe83S8+nhTc=', 'Content-Length': '0'} 2022-09-03T06:33:19.171 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000046f86724a096cd71-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.173 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:19.173 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:19.174 DEBUG:boto:Method: GET 2022-09-03T06:33:19.174 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:19.175 DEBUG:boto:Data: 2022-09-03T06:33:19.176 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.177 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.177 DEBUG:boto:Params: {} 2022-09-03T06:33:19.178 DEBUG:boto:Token: None 2022-09-03T06:33:19.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj1 2022-09-03T06:33:19.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FTFFO3YN8EhJBPlzoe83S8+nhTc= 2022-09-03T06:33:19.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FTFFO3YN8EhJBPlzoe83S8+nhTc=', 'Content-Length': '0'} 2022-09-03T06:33:19.183 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dad31dc2494492ab-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.227 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.228 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.230 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:19.231 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:19.232 DEBUG:boto:Method: GET 2022-09-03T06:33:19.233 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:19.237 DEBUG:boto:Data: 2022-09-03T06:33:19.238 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.239 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.240 DEBUG:boto:Params: {} 2022-09-03T06:33:19.240 DEBUG:boto:Token: None 2022-09-03T06:33:19.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/ 2022-09-03T06:33:19.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc= 2022-09-03T06:33:19.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc=', 'Content-Length': '0'} 2022-09-03T06:33:19.253 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004dbb8f9c72b5ef11-006312f52f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.260 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:19.262 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.263 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.264 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:19.269 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:19.270 DEBUG:boto:Method: GET 2022-09-03T06:33:19.271 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:19.272 DEBUG:boto:Data: 2022-09-03T06:33:19.273 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.275 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.276 DEBUG:boto:Params: {} 2022-09-03T06:33:19.277 DEBUG:boto:Token: None 2022-09-03T06:33:19.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/ 2022-09-03T06:33:19.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc= 2022-09-03T06:33:19.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc=', 'Content-Length': '0'} 2022-09-03T06:33:19.285 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a615b949a01ae34b-006312f52f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.286 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:19.288 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:19.289 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.290 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.291 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.292 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.294 DEBUG:boto:Method: GET 2022-09-03T06:33:19.296 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.297 DEBUG:boto:Data: 2022-09-03T06:33:19.298 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.300 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.300 DEBUG:boto:Params: {} 2022-09-03T06:33:19.301 DEBUG:boto:Token: None 2022-09-03T06:33:19.302 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S3vwrJ5cKpcLrujYl3fOvFovn8c= 2022-09-03T06:33:19.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S3vwrJ5cKpcLrujYl3fOvFovn8c=', 'Content-Length': '0'} 2022-09-03T06:33:19.310 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007b712997970094cd-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.313 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.314 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.314 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.315 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.315 DEBUG:boto:Method: GET 2022-09-03T06:33:19.316 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.316 DEBUG:boto:Data: 2022-09-03T06:33:19.317 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.318 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.318 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.319 DEBUG:boto:Params: {} 2022-09-03T06:33:19.319 DEBUG:boto:Token: None 2022-09-03T06:33:19.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S3vwrJ5cKpcLrujYl3fOvFovn8c= 2022-09-03T06:33:19.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S3vwrJ5cKpcLrujYl3fOvFovn8c=', 'Content-Length': '0'} 2022-09-03T06:33:19.327 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000eaf5e1dbbdaa5f17-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.329 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.330 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.331 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.332 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.333 DEBUG:boto:Method: HEAD 2022-09-03T06:33:19.334 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.335 DEBUG:boto:Data: 2022-09-03T06:33:19.336 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.338 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.339 DEBUG:boto:Params: {} 2022-09-03T06:33:19.340 DEBUG:boto:Token: None 2022-09-03T06:33:19.341 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WjPIs9HGyJCXYHE5vdynm/Q1wEw= 2022-09-03T06:33:19.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WjPIs9HGyJCXYHE5vdynm/Q1wEw=', 'Content-Length': '0'} 2022-09-03T06:33:19.346 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000075c25a4f7e668a83-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.348 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.349 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.350 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.351 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.352 DEBUG:boto:Method: HEAD 2022-09-03T06:33:19.352 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.353 DEBUG:boto:Data: 2022-09-03T06:33:19.354 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.355 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.356 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.357 DEBUG:boto:Params: {} 2022-09-03T06:33:19.358 DEBUG:boto:Token: None 2022-09-03T06:33:19.359 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WjPIs9HGyJCXYHE5vdynm/Q1wEw= 2022-09-03T06:33:19.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WjPIs9HGyJCXYHE5vdynm/Q1wEw=', 'Content-Length': '0'} 2022-09-03T06:33:19.364 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009b7f4943954b64b3-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.366 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:19.367 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.368 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.369 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.369 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.370 DEBUG:boto:Method: GET 2022-09-03T06:33:19.371 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.372 DEBUG:boto:Data: 2022-09-03T06:33:19.373 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.374 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.375 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.376 DEBUG:boto:Params: {} 2022-09-03T06:33:19.377 DEBUG:boto:Token: None 2022-09-03T06:33:19.378 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S3vwrJ5cKpcLrujYl3fOvFovn8c= 2022-09-03T06:33:19.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S3vwrJ5cKpcLrujYl3fOvFovn8c=', 'Content-Length': '0'} 2022-09-03T06:33:19.387 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000077439c8412a48cbb-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.430 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.431 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.432 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.435 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.437 DEBUG:boto:Method: GET 2022-09-03T06:33:19.438 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.439 DEBUG:boto:Data: 2022-09-03T06:33:19.440 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.443 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.444 DEBUG:boto:Params: {} 2022-09-03T06:33:19.445 DEBUG:boto:Token: None 2022-09-03T06:33:19.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:19.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S3vwrJ5cKpcLrujYl3fOvFovn8c= 2022-09-03T06:33:19.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S3vwrJ5cKpcLrujYl3fOvFovn8c=', 'Content-Length': '0'} 2022-09-03T06:33:19.454 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000004ef53dfe519e3d0-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.456 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.457 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.458 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:19.459 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:19.460 DEBUG:boto:Method: GET 2022-09-03T06:33:19.460 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:19.461 DEBUG:boto:Data: 2022-09-03T06:33:19.462 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.465 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.465 DEBUG:boto:Params: {} 2022-09-03T06:33:19.466 DEBUG:boto:Token: None 2022-09-03T06:33:19.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/ 2022-09-03T06:33:19.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc= 2022-09-03T06:33:19.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc=', 'Content-Length': '0'} 2022-09-03T06:33:19.474 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000350e6655b488d3e-006312f52f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.476 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:19.477 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.479 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.480 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:19.480 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:19.481 DEBUG:boto:Method: GET 2022-09-03T06:33:19.482 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:19.483 DEBUG:boto:Data: 2022-09-03T06:33:19.484 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.487 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.488 DEBUG:boto:Params: {} 2022-09-03T06:33:19.489 DEBUG:boto:Token: None 2022-09-03T06:33:19.492 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/ 2022-09-03T06:33:19.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc= 2022-09-03T06:33:19.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc=', 'Content-Length': '0'} 2022-09-03T06:33:19.503 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a7604060a2ea5b6-006312f52f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.504 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:19.508 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.510 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.512 DEBUG:boto:Method: HEAD 2022-09-03T06:33:19.513 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:19.514 DEBUG:boto:Data: 2022-09-03T06:33:19.515 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.517 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.519 DEBUG:boto:Params: {} 2022-09-03T06:33:19.520 DEBUG:boto:Token: None 2022-09-03T06:33:19.520 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2 2022-09-03T06:33:19.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Etx3xubGM0mLE4mXS1t9faNR+uI= 2022-09-03T06:33:19.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Etx3xubGM0mLE4mXS1t9faNR+uI=', 'Content-Length': '0'} 2022-09-03T06:33:19.554 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007bd59633179b8ca7-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.556 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.556 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.557 DEBUG:boto:Method: HEAD 2022-09-03T06:33:19.559 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:19.560 DEBUG:boto:Data: 2022-09-03T06:33:19.561 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.563 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.564 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.564 DEBUG:boto:Params: {} 2022-09-03T06:33:19.565 DEBUG:boto:Token: None 2022-09-03T06:33:19.577 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2 2022-09-03T06:33:19.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Etx3xubGM0mLE4mXS1t9faNR+uI= 2022-09-03T06:33:19.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Etx3xubGM0mLE4mXS1t9faNR+uI=', 'Content-Length': '0'} 2022-09-03T06:33:19.603 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000027c91b0e68fae9cc-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.605 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:19.618 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.619 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.620 DEBUG:boto:Method: GET 2022-09-03T06:33:19.621 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:19.622 DEBUG:boto:Data: 2022-09-03T06:33:19.623 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.624 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.625 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.628 DEBUG:boto:Params: {} 2022-09-03T06:33:19.629 DEBUG:boto:Token: None 2022-09-03T06:33:19.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2 2022-09-03T06:33:19.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kR7ujvlui7dfX44XRfmIeAvHsXw= 2022-09-03T06:33:19.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kR7ujvlui7dfX44XRfmIeAvHsXw=', 'Content-Length': '0'} 2022-09-03T06:33:19.651 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000067f86eb92e7a5f5d-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.653 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:19.668 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:19.669 DEBUG:boto:Method: GET 2022-09-03T06:33:19.670 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:19.675 DEBUG:boto:Data: 2022-09-03T06:33:19.676 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.678 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.680 DEBUG:boto:Params: {} 2022-09-03T06:33:19.681 DEBUG:boto:Token: None 2022-09-03T06:33:19.683 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj2 2022-09-03T06:33:19.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kR7ujvlui7dfX44XRfmIeAvHsXw= 2022-09-03T06:33:19.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kR7ujvlui7dfX44XRfmIeAvHsXw=', 'Content-Length': '0'} 2022-09-03T06:33:19.691 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000aca4d014be2161b3-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.693 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.694 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.695 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:19.696 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:19.696 DEBUG:boto:Method: GET 2022-09-03T06:33:19.697 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:19.698 DEBUG:boto:Data: 2022-09-03T06:33:19.699 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.708 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.714 DEBUG:boto:Params: {} 2022-09-03T06:33:19.716 DEBUG:boto:Token: None 2022-09-03T06:33:19.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/ 2022-09-03T06:33:19.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc= 2022-09-03T06:33:19.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc=', 'Content-Length': '0'} 2022-09-03T06:33:19.728 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000de452741231751ea-006312f52f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.730 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:19.731 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.732 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.733 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:19.735 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:19.739 DEBUG:boto:Method: GET 2022-09-03T06:33:19.740 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:19.743 DEBUG:boto:Data: 2022-09-03T06:33:19.744 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.746 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.748 DEBUG:boto:Params: {} 2022-09-03T06:33:19.749 DEBUG:boto:Token: None 2022-09-03T06:33:19.751 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/ 2022-09-03T06:33:19.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc= 2022-09-03T06:33:19.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc=', 'Content-Length': '0'} 2022-09-03T06:33:19.762 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e49482d811e61709-006312f52f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.763 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:19.765 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:19.766 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:19.767 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:19.768 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.770 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.771 DEBUG:boto:Method: GET 2022-09-03T06:33:19.772 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.773 DEBUG:boto:Data: 2022-09-03T06:33:19.774 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.776 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.777 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.779 DEBUG:boto:Params: {} 2022-09-03T06:33:19.781 DEBUG:boto:Token: None 2022-09-03T06:33:19.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OQfL9enpohK/WUTP1xWyTUzt3jE= 2022-09-03T06:33:19.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OQfL9enpohK/WUTP1xWyTUzt3jE=', 'Content-Length': '0'} 2022-09-03T06:33:19.804 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001b785f6a099f3077-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.806 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:19.808 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:19.809 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.810 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.811 DEBUG:boto:Method: GET 2022-09-03T06:33:19.813 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.815 DEBUG:boto:Data: 2022-09-03T06:33:19.816 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.821 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.833 DEBUG:boto:Params: {} 2022-09-03T06:33:19.835 DEBUG:boto:Token: None 2022-09-03T06:33:19.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OQfL9enpohK/WUTP1xWyTUzt3jE= 2022-09-03T06:33:19.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OQfL9enpohK/WUTP1xWyTUzt3jE=', 'Content-Length': '0'} 2022-09-03T06:33:19.856 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dc179786315f12a0-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.857 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:19.858 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:19.859 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.860 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.860 DEBUG:boto:Method: HEAD 2022-09-03T06:33:19.861 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.861 DEBUG:boto:Data: 2022-09-03T06:33:19.862 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.862 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.863 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.863 DEBUG:boto:Params: {} 2022-09-03T06:33:19.864 DEBUG:boto:Token: None 2022-09-03T06:33:19.865 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6F+vIq43Bjl/M8MGA0j+MmNtlgw= 2022-09-03T06:33:19.866 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6F+vIq43Bjl/M8MGA0j+MmNtlgw=', 'Content-Length': '0'} 2022-09-03T06:33:19.886 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000eeb64bcc497f559b-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.888 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:19.888 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:19.889 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.889 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.890 DEBUG:boto:Method: HEAD 2022-09-03T06:33:19.892 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.892 DEBUG:boto:Data: 2022-09-03T06:33:19.893 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.894 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.895 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.895 DEBUG:boto:Params: {} 2022-09-03T06:33:19.896 DEBUG:boto:Token: None 2022-09-03T06:33:19.897 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6F+vIq43Bjl/M8MGA0j+MmNtlgw= 2022-09-03T06:33:19.898 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6F+vIq43Bjl/M8MGA0j+MmNtlgw=', 'Content-Length': '0'} 2022-09-03T06:33:19.903 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004cc145d5169b83e5-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.904 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:19.905 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:19.905 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:19.906 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.908 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.909 DEBUG:boto:Method: GET 2022-09-03T06:33:19.910 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.911 DEBUG:boto:Data: 2022-09-03T06:33:19.912 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.919 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.920 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.929 DEBUG:boto:Params: {} 2022-09-03T06:33:19.931 DEBUG:boto:Token: None 2022-09-03T06:33:19.932 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.933 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OQfL9enpohK/WUTP1xWyTUzt3jE= 2022-09-03T06:33:19.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OQfL9enpohK/WUTP1xWyTUzt3jE=', 'Content-Length': '0'} 2022-09-03T06:33:19.943 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bf4fc3181de6c002-006312f52f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.945 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:19.946 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:19.947 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.949 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.950 DEBUG:boto:Method: GET 2022-09-03T06:33:19.955 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.956 DEBUG:boto:Data: 2022-09-03T06:33:19.957 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.958 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.959 DEBUG:boto:Port: 8000 2022-09-03T06:33:19.961 DEBUG:boto:Params: {} 2022-09-03T06:33:19.964 DEBUG:boto:Token: None 2022-09-03T06:33:19.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:19.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OQfL9enpohK/WUTP1xWyTUzt3jE= 2022-09-03T06:33:19.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OQfL9enpohK/WUTP1xWyTUzt3jE=', 'Content-Length': '0'} 2022-09-03T06:33:19.970 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001989c056df3e79fe-006312f52f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.972 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.973 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.973 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:19.974 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:19.975 DEBUG:boto:Method: GET 2022-09-03T06:33:19.975 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:19.976 DEBUG:boto:Data: 2022-09-03T06:33:19.976 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.977 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:19.977 DEBUG:boto:Port: 8001 2022-09-03T06:33:19.978 DEBUG:boto:Params: {} 2022-09-03T06:33:19.979 DEBUG:boto:Token: None 2022-09-03T06:33:19.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:19 GMT /avysmw-60/ 2022-09-03T06:33:19.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc= 2022-09-03T06:33:19.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mfRQdGSr8yUksYTwfFP1iaMKJc=', 'Content-Length': '0'} 2022-09-03T06:33:19.984 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000052ccb34d2eed96c8-006312f52f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:19.985 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:19.986 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:19.986 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:19.987 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:19.987 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:19.988 DEBUG:boto:Method: GET 2022-09-03T06:33:19.989 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:19.989 DEBUG:boto:Data: 2022-09-03T06:33:19.990 DEBUG:boto:Headers: {} 2022-09-03T06:33:19.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:19.991 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.002 DEBUG:boto:Params: {} 2022-09-03T06:33:20.004 DEBUG:boto:Token: None 2022-09-03T06:33:20.006 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.015 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000000b4e7e4c6f537ec-006312f530-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.017 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.018 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:20.019 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:20.021 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.022 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:20.023 DEBUG:boto:Data: 2022-09-03T06:33:20.024 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.025 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.027 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.029 DEBUG:boto:Params: {} 2022-09-03T06:33:20.029 DEBUG:boto:Token: None 2022-09-03T06:33:20.030 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj3 2022-09-03T06:33:20.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dz+i0yqLjpUXbtgFWEGN1YwdNOw= 2022-09-03T06:33:20.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dz+i0yqLjpUXbtgFWEGN1YwdNOw=', 'Content-Length': '0'} 2022-09-03T06:33:20.033 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d0ec49a44dffddd7-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.034 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:20.035 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:20.035 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.037 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:20.038 DEBUG:boto:Data: 2022-09-03T06:33:20.038 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.039 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.041 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.042 DEBUG:boto:Params: {} 2022-09-03T06:33:20.044 DEBUG:boto:Token: None 2022-09-03T06:33:20.044 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj3 2022-09-03T06:33:20.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dz+i0yqLjpUXbtgFWEGN1YwdNOw= 2022-09-03T06:33:20.046 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dz+i0yqLjpUXbtgFWEGN1YwdNOw=', 'Content-Length': '0'} 2022-09-03T06:33:20.049 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f7abf19b1459f7fc-006312f530-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.050 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:20.051 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:20.052 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:20.053 DEBUG:boto:Method: GET 2022-09-03T06:33:20.054 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:20.054 DEBUG:boto:Data: 2022-09-03T06:33:20.055 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.056 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.059 DEBUG:boto:Params: {} 2022-09-03T06:33:20.060 DEBUG:boto:Token: None 2022-09-03T06:33:20.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj3 2022-09-03T06:33:20.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X86JXOkMzN0LInbFQBuX2D1jhOk= 2022-09-03T06:33:20.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X86JXOkMzN0LInbFQBuX2D1jhOk=', 'Content-Length': '0'} 2022-09-03T06:33:20.075 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003e1205f1aa4150eb-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.077 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:20.081 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:20.082 DEBUG:boto:Method: GET 2022-09-03T06:33:20.083 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:20.084 DEBUG:boto:Data: 2022-09-03T06:33:20.085 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.086 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.087 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.089 DEBUG:boto:Params: {} 2022-09-03T06:33:20.091 DEBUG:boto:Token: None 2022-09-03T06:33:20.092 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj3 2022-09-03T06:33:20.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X86JXOkMzN0LInbFQBuX2D1jhOk= 2022-09-03T06:33:20.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X86JXOkMzN0LInbFQBuX2D1jhOk=', 'Content-Length': '0'} 2022-09-03T06:33:20.111 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f0141a142f2b8e2c-006312f530-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.112 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.113 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.113 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:20.114 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:20.114 DEBUG:boto:Method: GET 2022-09-03T06:33:20.115 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:20.116 DEBUG:boto:Data: 2022-09-03T06:33:20.117 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.118 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.118 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.119 DEBUG:boto:Params: {} 2022-09-03T06:33:20.119 DEBUG:boto:Token: None 2022-09-03T06:33:20.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.125 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000019ac874435178349-006312f530-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.126 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.128 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.129 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.130 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:20.131 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:20.132 DEBUG:boto:Method: GET 2022-09-03T06:33:20.137 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:20.138 DEBUG:boto:Data: 2022-09-03T06:33:20.139 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.140 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.141 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.142 DEBUG:boto:Params: {} 2022-09-03T06:33:20.143 DEBUG:boto:Token: None 2022-09-03T06:33:20.144 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.145 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.152 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004b8e2159122782d2-006312f530-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.154 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.156 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:20.156 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.157 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.158 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.159 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.160 DEBUG:boto:Method: GET 2022-09-03T06:33:20.161 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.162 DEBUG:boto:Data: 2022-09-03T06:33:20.162 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.164 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.165 DEBUG:boto:Params: {} 2022-09-03T06:33:20.166 DEBUG:boto:Token: None 2022-09-03T06:33:20.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ou5BnKHbaukeaJQjq/++tWwgris= 2022-09-03T06:33:20.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ou5BnKHbaukeaJQjq/++tWwgris=', 'Content-Length': '0'} 2022-09-03T06:33:20.173 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009702170eb869bdc6-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.178 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.179 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.180 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.181 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.183 DEBUG:boto:Method: GET 2022-09-03T06:33:20.184 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.185 DEBUG:boto:Data: 2022-09-03T06:33:20.186 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.188 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.189 DEBUG:boto:Params: {} 2022-09-03T06:33:20.190 DEBUG:boto:Token: None 2022-09-03T06:33:20.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ou5BnKHbaukeaJQjq/++tWwgris= 2022-09-03T06:33:20.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ou5BnKHbaukeaJQjq/++tWwgris=', 'Content-Length': '0'} 2022-09-03T06:33:20.214 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f6a335e8cfc79f79-006312f530-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.215 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.216 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.217 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.217 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.218 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.219 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.219 DEBUG:boto:Data: 2022-09-03T06:33:20.220 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.222 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.224 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.225 DEBUG:boto:Params: {} 2022-09-03T06:33:20.225 DEBUG:boto:Token: None 2022-09-03T06:33:20.226 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.227 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TxAwdM4YMGUD7Rn73d0Izd/d+YM= 2022-09-03T06:33:20.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TxAwdM4YMGUD7Rn73d0Izd/d+YM=', 'Content-Length': '0'} 2022-09-03T06:33:20.230 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000aa7ba9bf0f9e8a45-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.231 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.232 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.232 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.233 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.233 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.234 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.235 DEBUG:boto:Data: 2022-09-03T06:33:20.236 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.238 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.239 DEBUG:boto:Params: {} 2022-09-03T06:33:20.239 DEBUG:boto:Token: None 2022-09-03T06:33:20.240 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TxAwdM4YMGUD7Rn73d0Izd/d+YM= 2022-09-03T06:33:20.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TxAwdM4YMGUD7Rn73d0Izd/d+YM=', 'Content-Length': '0'} 2022-09-03T06:33:20.246 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a85f7a20871aa8ca-006312f530-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.247 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:20.247 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.248 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.248 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.249 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.250 DEBUG:boto:Method: GET 2022-09-03T06:33:20.250 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.251 DEBUG:boto:Data: 2022-09-03T06:33:20.251 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.252 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.252 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.253 DEBUG:boto:Params: {} 2022-09-03T06:33:20.254 DEBUG:boto:Token: None 2022-09-03T06:33:20.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ou5BnKHbaukeaJQjq/++tWwgris= 2022-09-03T06:33:20.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ou5BnKHbaukeaJQjq/++tWwgris=', 'Content-Length': '0'} 2022-09-03T06:33:20.258 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008d78d914bc069a14-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.259 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.260 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.260 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.261 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.261 DEBUG:boto:Method: GET 2022-09-03T06:33:20.263 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.263 DEBUG:boto:Data: 2022-09-03T06:33:20.264 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.265 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.266 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.266 DEBUG:boto:Params: {} 2022-09-03T06:33:20.267 DEBUG:boto:Token: None 2022-09-03T06:33:20.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:20.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ou5BnKHbaukeaJQjq/++tWwgris= 2022-09-03T06:33:20.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ou5BnKHbaukeaJQjq/++tWwgris=', 'Content-Length': '0'} 2022-09-03T06:33:20.272 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005741d345e8254455-006312f530-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.274 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.274 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.275 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:20.276 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:20.276 DEBUG:boto:Method: GET 2022-09-03T06:33:20.277 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:20.277 DEBUG:boto:Data: 2022-09-03T06:33:20.278 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.279 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.279 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.280 DEBUG:boto:Params: {} 2022-09-03T06:33:20.280 DEBUG:boto:Token: None 2022-09-03T06:33:20.281 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.286 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000be656937ee2ae2ac-006312f530-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.287 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.288 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.289 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.289 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:20.290 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:20.291 DEBUG:boto:Method: GET 2022-09-03T06:33:20.291 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:20.292 DEBUG:boto:Data: 2022-09-03T06:33:20.292 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.294 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.294 DEBUG:boto:Params: {} 2022-09-03T06:33:20.295 DEBUG:boto:Token: None 2022-09-03T06:33:20.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.300 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017dfb65860fa5938-006312f530-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.301 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.302 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.303 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.304 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.305 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:20.307 DEBUG:boto:Data: 2022-09-03T06:33:20.308 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.310 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.311 DEBUG:boto:Params: {} 2022-09-03T06:33:20.312 DEBUG:boto:Token: None 2022-09-03T06:33:20.313 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4 2022-09-03T06:33:20.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tv6m/lCYZVpAGD+ovYa/ZCOak8s= 2022-09-03T06:33:20.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tv6m/lCYZVpAGD+ovYa/ZCOak8s=', 'Content-Length': '0'} 2022-09-03T06:33:20.319 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000074613ef0e7534448-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.320 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.321 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.322 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.323 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:20.324 DEBUG:boto:Data: 2022-09-03T06:33:20.325 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.326 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.327 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.328 DEBUG:boto:Params: {} 2022-09-03T06:33:20.328 DEBUG:boto:Token: None 2022-09-03T06:33:20.329 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4 2022-09-03T06:33:20.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tv6m/lCYZVpAGD+ovYa/ZCOak8s= 2022-09-03T06:33:20.332 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tv6m/lCYZVpAGD+ovYa/ZCOak8s=', 'Content-Length': '0'} 2022-09-03T06:33:20.334 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000099afaafdd149d09b-006312f530-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.335 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:20.336 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.337 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.339 DEBUG:boto:Method: GET 2022-09-03T06:33:20.340 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:20.341 DEBUG:boto:Data: 2022-09-03T06:33:20.342 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.343 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.344 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.345 DEBUG:boto:Params: {} 2022-09-03T06:33:20.346 DEBUG:boto:Token: None 2022-09-03T06:33:20.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4 2022-09-03T06:33:20.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OHTs3VDzJYeii/F9cnVtc/Aq+o8= 2022-09-03T06:33:20.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OHTs3VDzJYeii/F9cnVtc/Aq+o8=', 'Content-Length': '0'} 2022-09-03T06:33:20.353 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004d3491fe37e88335-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.395 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:20.397 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:20.401 DEBUG:boto:Method: GET 2022-09-03T06:33:20.403 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:20.404 DEBUG:boto:Data: 2022-09-03T06:33:20.405 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.407 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.408 DEBUG:boto:Params: {} 2022-09-03T06:33:20.409 DEBUG:boto:Token: None 2022-09-03T06:33:20.410 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj4 2022-09-03T06:33:20.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OHTs3VDzJYeii/F9cnVtc/Aq+o8= 2022-09-03T06:33:20.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OHTs3VDzJYeii/F9cnVtc/Aq+o8=', 'Content-Length': '0'} 2022-09-03T06:33:20.416 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003ada9fb762edd135-006312f530-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.417 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.419 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.420 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:20.421 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:20.422 DEBUG:boto:Method: GET 2022-09-03T06:33:20.423 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:20.424 DEBUG:boto:Data: 2022-09-03T06:33:20.425 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.426 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.427 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.428 DEBUG:boto:Params: {} 2022-09-03T06:33:20.429 DEBUG:boto:Token: None 2022-09-03T06:33:20.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.436 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f95074363cbd4d1f-006312f530-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.437 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.439 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.440 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.440 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:20.441 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:20.442 DEBUG:boto:Method: GET 2022-09-03T06:33:20.442 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:20.443 DEBUG:boto:Data: 2022-09-03T06:33:20.443 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.444 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:33:20.445 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.445 DEBUG:boto:Params: {} 2022-09-03T06:33:20.446 DEBUG:boto:Token: None 2022-09-03T06:33:20.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.451 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0cc3b117d45b9af-006312f530-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.452 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.453 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-60 zones={test-zone3, test-zone1} 2022-09-03T06:33:20.453 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-60 zones={test-zone3, test-zone2} 2022-09-03T06:33:20.454 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.454 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.455 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.455 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:33:20.456 DEBUG:boto:Data: 2022-09-03T06:33:20.457 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.457 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.458 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.458 DEBUG:boto:Params: {} 2022-09-03T06:33:20.459 DEBUG:boto:Token: None 2022-09-03T06:33:20.460 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.460 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u/lgad1xlVam9cir1Zt7R3yY9f8= 2022-09-03T06:33:20.461 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u/lgad1xlVam9cir1Zt7R3yY9f8=', 'Content-Length': '0'} 2022-09-03T06:33:20.464 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000afa2b24206420e6c-006312f530-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.465 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.466 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.469 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.469 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:33:20.470 DEBUG:boto:Data: 2022-09-03T06:33:20.471 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.471 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.472 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.473 DEBUG:boto:Params: {} 2022-09-03T06:33:20.474 DEBUG:boto:Token: None 2022-09-03T06:33:20.474 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.475 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u/lgad1xlVam9cir1Zt7R3yY9f8= 2022-09-03T06:33:20.476 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u/lgad1xlVam9cir1Zt7R3yY9f8=', 'Content-Length': '0'} 2022-09-03T06:33:20.479 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005848e9a754a5a212-006312f530-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.480 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:33:20.481 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.482 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.483 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.483 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.484 DEBUG:boto:Method: GET 2022-09-03T06:33:20.484 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.485 DEBUG:boto:Data: 2022-09-03T06:33:20.485 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.486 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.487 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.487 DEBUG:boto:Params: {} 2022-09-03T06:33:20.488 DEBUG:boto:Token: None 2022-09-03T06:33:20.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/?versions 2022-09-03T06:33:20.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1aWYN6vliMiquOVBJ90v6x64ORc= 2022-09-03T06:33:20.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1aWYN6vliMiquOVBJ90v6x64ORc=', 'Content-Length': '0'} 2022-09-03T06:33:20.495 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000caeb6a419f6f9c10-006312f530-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.496 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.497 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:20.498 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:20.498 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:20.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:20.499 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:33:20.500 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.501 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.501 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.502 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.502 DEBUG:boto:Method: GET 2022-09-03T06:33:20.503 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.503 DEBUG:boto:Data: 2022-09-03T06:33:20.504 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.504 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.505 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.506 DEBUG:boto:Params: {} 2022-09-03T06:33:20.506 DEBUG:boto:Token: None 2022-09-03T06:33:20.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/?versions 2022-09-03T06:33:20.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1aWYN6vliMiquOVBJ90v6x64ORc= 2022-09-03T06:33:20.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1aWYN6vliMiquOVBJ90v6x64ORc=', 'Content-Length': '0'} 2022-09-03T06:33:20.511 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000facc6069946dae62-006312f530-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.512 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.513 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:33:20.513 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:33:20.514 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:33:20.514 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:33:20.515 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.516 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.516 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.517 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.517 DEBUG:boto:Method: GET 2022-09-03T06:33:20.518 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.519 DEBUG:boto:Data: 2022-09-03T06:33:20.519 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.520 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.521 DEBUG:boto:Params: {} 2022-09-03T06:33:20.522 DEBUG:boto:Token: None 2022-09-03T06:33:20.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/?versions 2022-09-03T06:33:20.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1aWYN6vliMiquOVBJ90v6x64ORc= 2022-09-03T06:33:20.525 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1aWYN6vliMiquOVBJ90v6x64ORc=', 'Content-Length': '0'} 2022-09-03T06:33:20.529 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a1cce79e9fc3c278-006312f530-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.530 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.533 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.534 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.535 DEBUG:boto:path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.536 DEBUG:boto:auth_path=/avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.537 DEBUG:boto:Method: GET 2022-09-03T06:33:20.537 DEBUG:boto:Path: /avysmw-60/?versions&max-keys=999 2022-09-03T06:33:20.538 DEBUG:boto:Data: 2022-09-03T06:33:20.540 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.540 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.541 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.543 DEBUG:boto:Params: {} 2022-09-03T06:33:20.544 DEBUG:boto:Token: None 2022-09-03T06:33:20.545 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/?versions 2022-09-03T06:33:20.546 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1aWYN6vliMiquOVBJ90v6x64ORc= 2022-09-03T06:33:20.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1aWYN6vliMiquOVBJ90v6x64ORc=', 'Content-Length': '0'} 2022-09-03T06:33:20.551 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000070670b62b89e6d82-006312f530-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.553 DEBUG:boto:b'avysmw-60999falseobj1nulltrue2022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.555 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:20.556 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.557 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.558 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.560 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.561 DEBUG:boto:Method: GET 2022-09-03T06:33:20.562 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.564 DEBUG:boto:Data: 2022-09-03T06:33:20.565 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.565 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.566 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.567 DEBUG:boto:Params: {} 2022-09-03T06:33:20.568 DEBUG:boto:Token: None 2022-09-03T06:33:20.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WuyI0ObQriFOYqRt/DBGSj323Ok= 2022-09-03T06:33:20.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WuyI0ObQriFOYqRt/DBGSj323Ok=', 'Content-Length': '0'} 2022-09-03T06:33:20.582 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000003a34dad9df48208-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.584 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.584 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.585 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.585 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.586 DEBUG:boto:Method: GET 2022-09-03T06:33:20.587 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.587 DEBUG:boto:Data: 2022-09-03T06:33:20.588 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.589 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.589 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.590 DEBUG:boto:Params: {} 2022-09-03T06:33:20.591 DEBUG:boto:Token: None 2022-09-03T06:33:20.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WuyI0ObQriFOYqRt/DBGSj323Ok= 2022-09-03T06:33:20.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WuyI0ObQriFOYqRt/DBGSj323Ok=', 'Content-Length': '0'} 2022-09-03T06:33:20.598 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000015ec1d7aa956ceb1-006312f530-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.644 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.645 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.647 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.648 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.649 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.650 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.651 DEBUG:boto:Data: 2022-09-03T06:33:20.652 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.653 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.654 DEBUG:boto:Params: {} 2022-09-03T06:33:20.654 DEBUG:boto:Token: None 2022-09-03T06:33:20.655 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JU+lqG4On1co+huO62QoA6GiuJo= 2022-09-03T06:33:20.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JU+lqG4On1co+huO62QoA6GiuJo=', 'Content-Length': '0'} 2022-09-03T06:33:20.659 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007605ff96af87e7f4-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.660 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.661 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.663 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.663 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.664 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.665 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.665 DEBUG:boto:Data: 2022-09-03T06:33:20.666 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.666 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.667 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.668 DEBUG:boto:Params: {} 2022-09-03T06:33:20.669 DEBUG:boto:Token: None 2022-09-03T06:33:20.670 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JU+lqG4On1co+huO62QoA6GiuJo= 2022-09-03T06:33:20.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JU+lqG4On1co+huO62QoA6GiuJo=', 'Content-Length': '0'} 2022-09-03T06:33:20.678 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000aa540b5a3e1e188-006312f530-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.679 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:20.680 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.681 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.682 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.682 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.688 DEBUG:boto:Method: GET 2022-09-03T06:33:20.689 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.690 DEBUG:boto:Data: 2022-09-03T06:33:20.690 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.692 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.693 DEBUG:boto:Params: {} 2022-09-03T06:33:20.693 DEBUG:boto:Token: None 2022-09-03T06:33:20.695 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.695 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WuyI0ObQriFOYqRt/DBGSj323Ok= 2022-09-03T06:33:20.696 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WuyI0ObQriFOYqRt/DBGSj323Ok=', 'Content-Length': '0'} 2022-09-03T06:33:20.701 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000076bf968edd6e7462-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.704 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.705 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.707 DEBUG:boto:path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.709 DEBUG:boto:auth_path=/avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.710 DEBUG:boto:Method: GET 2022-09-03T06:33:20.711 DEBUG:boto:Path: /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.712 DEBUG:boto:Data: 2022-09-03T06:33:20.713 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.714 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.715 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.716 DEBUG:boto:Params: {} 2022-09-03T06:33:20.717 DEBUG:boto:Token: None 2022-09-03T06:33:20.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1?versionId=null 2022-09-03T06:33:20.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WuyI0ObQriFOYqRt/DBGSj323Ok= 2022-09-03T06:33:20.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WuyI0ObQriFOYqRt/DBGSj323Ok=', 'Content-Length': '0'} 2022-09-03T06:33:20.729 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003b661e19ececc1bd-006312f530-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.738 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.747 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.748 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:20.760 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:20.761 DEBUG:boto:Method: GET 2022-09-03T06:33:20.762 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:20.763 DEBUG:boto:Data: 2022-09-03T06:33:20.764 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.766 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.767 DEBUG:boto:Params: {} 2022-09-03T06:33:20.768 DEBUG:boto:Token: None 2022-09-03T06:33:20.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.780 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009bc17eb189c5df1d-006312f530-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.781 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.782 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.783 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.783 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:20.784 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:20.784 DEBUG:boto:Method: GET 2022-09-03T06:33:20.785 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:20.786 DEBUG:boto:Data: 2022-09-03T06:33:20.786 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.787 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.788 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.788 DEBUG:boto:Params: {} 2022-09-03T06:33:20.789 DEBUG:boto:Token: None 2022-09-03T06:33:20.791 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.791 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.792 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.800 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000038ba85fc587badf9-006312f530-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.802 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.803 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.803 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.804 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.804 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:20.805 DEBUG:boto:Data: 2022-09-03T06:33:20.805 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.807 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.807 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.808 DEBUG:boto:Params: {} 2022-09-03T06:33:20.809 DEBUG:boto:Token: None 2022-09-03T06:33:20.809 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1 2022-09-03T06:33:20.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gSLjEXFDplS+9Rz0EmgHQ8THXmg= 2022-09-03T06:33:20.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gSLjEXFDplS+9Rz0EmgHQ8THXmg=', 'Content-Length': '0'} 2022-09-03T06:33:20.817 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008d11340c949dcd5a-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.823 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.826 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.826 DEBUG:boto:Method: HEAD 2022-09-03T06:33:20.827 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:20.828 DEBUG:boto:Data: 2022-09-03T06:33:20.828 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.829 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.829 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.830 DEBUG:boto:Params: {} 2022-09-03T06:33:20.830 DEBUG:boto:Token: None 2022-09-03T06:33:20.831 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1 2022-09-03T06:33:20.832 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gSLjEXFDplS+9Rz0EmgHQ8THXmg= 2022-09-03T06:33:20.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gSLjEXFDplS+9Rz0EmgHQ8THXmg=', 'Content-Length': '0'} 2022-09-03T06:33:20.837 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d0d7c3f6c7a9711b-006312f530-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.838 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:33:20.839 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.839 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.840 DEBUG:boto:Method: GET 2022-09-03T06:33:20.841 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:20.841 DEBUG:boto:Data: 2022-09-03T06:33:20.842 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.843 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.843 DEBUG:boto:Params: {} 2022-09-03T06:33:20.844 DEBUG:boto:Token: None 2022-09-03T06:33:20.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1 2022-09-03T06:33:20.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RXl4/9OjKE/3Mp4P6CE+0CvxfHw= 2022-09-03T06:33:20.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RXl4/9OjKE/3Mp4P6CE+0CvxfHw=', 'Content-Length': '0'} 2022-09-03T06:33:20.849 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000068b437007c3edb49-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.850 DEBUG:boto:path=/avysmw-60/obj1 2022-09-03T06:33:20.851 DEBUG:boto:auth_path=/avysmw-60/obj1 2022-09-03T06:33:20.851 DEBUG:boto:Method: GET 2022-09-03T06:33:20.864 DEBUG:boto:Path: /avysmw-60/obj1 2022-09-03T06:33:20.865 DEBUG:boto:Data: 2022-09-03T06:33:20.875 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.876 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.877 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.878 DEBUG:boto:Params: {} 2022-09-03T06:33:20.879 DEBUG:boto:Token: None 2022-09-03T06:33:20.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj1 2022-09-03T06:33:20.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RXl4/9OjKE/3Mp4P6CE+0CvxfHw= 2022-09-03T06:33:20.891 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RXl4/9OjKE/3Mp4P6CE+0CvxfHw=', 'Content-Length': '0'} 2022-09-03T06:33:20.902 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000099dda508e589848-006312f530-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.904 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.905 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.913 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:20.915 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:20.916 DEBUG:boto:Method: GET 2022-09-03T06:33:20.917 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:20.918 DEBUG:boto:Data: 2022-09-03T06:33:20.919 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.921 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.922 DEBUG:boto:Params: {} 2022-09-03T06:33:20.924 DEBUG:boto:Token: None 2022-09-03T06:33:20.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.934 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000266810275a594af6-006312f530-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.937 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.939 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:20.940 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:20.941 DEBUG:boto:path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:20.943 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj1 2022-09-03T06:33:20.944 DEBUG:boto:Method: GET 2022-09-03T06:33:20.945 DEBUG:boto:Path: /avysmw-60/?prefix=obj1 2022-09-03T06:33:20.946 DEBUG:boto:Data: 2022-09-03T06:33:20.948 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:20.950 DEBUG:boto:Port: 8000 2022-09-03T06:33:20.951 DEBUG:boto:Params: {} 2022-09-03T06:33:20.952 DEBUG:boto:Token: None 2022-09-03T06:33:20.953 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/ 2022-09-03T06:33:20.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U= 2022-09-03T06:33:20.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZVg4vqG8SCVYfUX7+f1V6ZFN7U=', 'Content-Length': '0'} 2022-09-03T06:33:20.962 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d532c14299944fe5-006312f530-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.963 DEBUG:boto:b'avysmw-60obj11000falseobj12022-09-03T06:32:26.871Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:20.964 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:20.965 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:20.966 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:20.967 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:20.968 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:20.969 DEBUG:boto:Method: GET 2022-09-03T06:33:20.971 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:20.972 DEBUG:boto:Data: 2022-09-03T06:33:20.972 DEBUG:boto:Headers: {} 2022-09-03T06:33:20.973 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:20.974 DEBUG:boto:Port: 8001 2022-09-03T06:33:20.975 DEBUG:boto:Params: {} 2022-09-03T06:33:20.977 DEBUG:boto:Token: None 2022-09-03T06:33:20.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:20 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:20.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WDfEFbZ8X+/YV6GSDv1dwwffudk= 2022-09-03T06:33:20.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WDfEFbZ8X+/YV6GSDv1dwwffudk=', 'Content-Length': '0'} 2022-09-03T06:33:20.986 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c305c6cbd468f1da-006312f530-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:20.995 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:20.995 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:20.996 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:20.997 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:20.998 DEBUG:boto:Method: GET 2022-09-03T06:33:20.999 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:20.999 DEBUG:boto:Data: 2022-09-03T06:33:21.000 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.001 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.002 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.003 DEBUG:boto:Params: {} 2022-09-03T06:33:21.003 DEBUG:boto:Token: None 2022-09-03T06:33:21.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AU+47CtiWFoPbwlDBJD8WecxPLM= 2022-09-03T06:33:21.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AU+47CtiWFoPbwlDBJD8WecxPLM=', 'Content-Length': '0'} 2022-09-03T06:33:21.009 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006f6b5acaa47f9890-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.017 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:21.018 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:21.019 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.020 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.021 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.022 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.023 DEBUG:boto:Data: 2022-09-03T06:33:21.024 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.025 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.026 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.027 DEBUG:boto:Params: {} 2022-09-03T06:33:21.029 DEBUG:boto:Token: None 2022-09-03T06:33:21.030 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R3BZA7K2EviF5/E0s/c5OKfzRKo= 2022-09-03T06:33:21.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R3BZA7K2EviF5/E0s/c5OKfzRKo=', 'Content-Length': '0'} 2022-09-03T06:33:21.046 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f9d0e4229b23ad80-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.047 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:21.049 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:21.050 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.053 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.054 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.055 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.056 DEBUG:boto:Data: 2022-09-03T06:33:21.057 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.059 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.060 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.061 DEBUG:boto:Params: {} 2022-09-03T06:33:21.062 DEBUG:boto:Token: None 2022-09-03T06:33:21.062 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R3BZA7K2EviF5/E0s/c5OKfzRKo= 2022-09-03T06:33:21.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R3BZA7K2EviF5/E0s/c5OKfzRKo=', 'Content-Length': '0'} 2022-09-03T06:33:21.067 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000ad4ef6e1812cc8e-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.068 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:21.069 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:21.070 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:21.071 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.072 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.072 DEBUG:boto:Method: GET 2022-09-03T06:33:21.073 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.074 DEBUG:boto:Data: 2022-09-03T06:33:21.074 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.075 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.076 DEBUG:boto:Params: {} 2022-09-03T06:33:21.077 DEBUG:boto:Token: None 2022-09-03T06:33:21.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AU+47CtiWFoPbwlDBJD8WecxPLM= 2022-09-03T06:33:21.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AU+47CtiWFoPbwlDBJD8WecxPLM=', 'Content-Length': '0'} 2022-09-03T06:33:21.084 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ea28e2bf9bdd4c2d-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.085 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:21.086 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:21.087 DEBUG:boto:path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.088 DEBUG:boto:auth_path=/avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.089 DEBUG:boto:Method: GET 2022-09-03T06:33:21.090 DEBUG:boto:Path: /avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.091 DEBUG:boto:Data: 2022-09-03T06:33:21.092 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.093 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.094 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.095 DEBUG:boto:Params: {} 2022-09-03T06:33:21.096 DEBUG:boto:Token: None 2022-09-03T06:33:21.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj2?versionId=null 2022-09-03T06:33:21.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AU+47CtiWFoPbwlDBJD8WecxPLM= 2022-09-03T06:33:21.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AU+47CtiWFoPbwlDBJD8WecxPLM=', 'Content-Length': '0'} 2022-09-03T06:33:21.107 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000295ce9dc0087db51-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.108 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.109 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.109 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:21.110 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:21.110 DEBUG:boto:Method: GET 2022-09-03T06:33:21.111 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:21.111 DEBUG:boto:Data: 2022-09-03T06:33:21.112 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.112 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.113 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.114 DEBUG:boto:Params: {} 2022-09-03T06:33:21.116 DEBUG:boto:Token: None 2022-09-03T06:33:21.117 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.126 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb8d953daa4eab7e-006312f531-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.127 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.130 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.131 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.132 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:21.133 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:21.133 DEBUG:boto:Method: GET 2022-09-03T06:33:21.134 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:21.135 DEBUG:boto:Data: 2022-09-03T06:33:21.136 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.137 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.138 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.139 DEBUG:boto:Params: {} 2022-09-03T06:33:21.147 DEBUG:boto:Token: None 2022-09-03T06:33:21.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.157 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090efdc44230ed7dd-006312f531-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.159 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.161 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:21.163 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:21.165 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.167 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:21.168 DEBUG:boto:Data: 2022-09-03T06:33:21.169 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.170 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.171 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.173 DEBUG:boto:Params: {} 2022-09-03T06:33:21.174 DEBUG:boto:Token: None 2022-09-03T06:33:21.177 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj2 2022-09-03T06:33:21.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WLtsMzxRReR8G6ilYUGIMfR5N7o= 2022-09-03T06:33:21.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WLtsMzxRReR8G6ilYUGIMfR5N7o=', 'Content-Length': '0'} 2022-09-03T06:33:21.185 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e1c6f12e90f03caa-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.186 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:21.188 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:21.189 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.191 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:21.192 DEBUG:boto:Data: 2022-09-03T06:33:21.192 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.193 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.193 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.194 DEBUG:boto:Params: {} 2022-09-03T06:33:21.195 DEBUG:boto:Token: None 2022-09-03T06:33:21.196 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj2 2022-09-03T06:33:21.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WLtsMzxRReR8G6ilYUGIMfR5N7o= 2022-09-03T06:33:21.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WLtsMzxRReR8G6ilYUGIMfR5N7o=', 'Content-Length': '0'} 2022-09-03T06:33:21.210 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000840be0b6bf6eab3d-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.211 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:33:21.212 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:21.213 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:21.214 DEBUG:boto:Method: GET 2022-09-03T06:33:21.215 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:21.216 DEBUG:boto:Data: 2022-09-03T06:33:21.217 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.219 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.220 DEBUG:boto:Params: {} 2022-09-03T06:33:21.221 DEBUG:boto:Token: None 2022-09-03T06:33:21.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj2 2022-09-03T06:33:21.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NrjyjEcdkk7hVMahCoxJ/cpwtmk= 2022-09-03T06:33:21.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NrjyjEcdkk7hVMahCoxJ/cpwtmk=', 'Content-Length': '0'} 2022-09-03T06:33:21.231 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002e97affb16e702a4-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.232 DEBUG:boto:path=/avysmw-60/obj2 2022-09-03T06:33:21.233 DEBUG:boto:auth_path=/avysmw-60/obj2 2022-09-03T06:33:21.234 DEBUG:boto:Method: GET 2022-09-03T06:33:21.235 DEBUG:boto:Path: /avysmw-60/obj2 2022-09-03T06:33:21.236 DEBUG:boto:Data: 2022-09-03T06:33:21.237 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.238 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.239 DEBUG:boto:Params: {} 2022-09-03T06:33:21.240 DEBUG:boto:Token: None 2022-09-03T06:33:21.241 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj2 2022-09-03T06:33:21.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NrjyjEcdkk7hVMahCoxJ/cpwtmk= 2022-09-03T06:33:21.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NrjyjEcdkk7hVMahCoxJ/cpwtmk=', 'Content-Length': '0'} 2022-09-03T06:33:21.244 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009a5ebefa9b200f71-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.245 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.246 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.247 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:21.248 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:21.250 DEBUG:boto:Method: GET 2022-09-03T06:33:21.251 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:21.252 DEBUG:boto:Data: 2022-09-03T06:33:21.253 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.254 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.255 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.256 DEBUG:boto:Params: {} 2022-09-03T06:33:21.257 DEBUG:boto:Token: None 2022-09-03T06:33:21.258 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.265 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000143a11dd0fed9747-006312f531-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.302 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.304 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.305 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.306 DEBUG:boto:path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:21.307 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj2 2022-09-03T06:33:21.309 DEBUG:boto:Method: GET 2022-09-03T06:33:21.310 DEBUG:boto:Path: /avysmw-60/?prefix=obj2 2022-09-03T06:33:21.311 DEBUG:boto:Data: 2022-09-03T06:33:21.311 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.312 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.312 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.316 DEBUG:boto:Params: {} 2022-09-03T06:33:21.317 DEBUG:boto:Token: None 2022-09-03T06:33:21.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.324 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009f71136e73a39887-006312f531-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.326 DEBUG:boto:b'avysmw-60obj21000falseobj22022-09-03T06:32:26.906Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.333 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:21.334 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.335 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.336 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.337 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.338 DEBUG:boto:Method: GET 2022-09-03T06:33:21.339 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.340 DEBUG:boto:Data: 2022-09-03T06:33:21.341 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.342 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.343 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.344 DEBUG:boto:Params: {} 2022-09-03T06:33:21.346 DEBUG:boto:Token: None 2022-09-03T06:33:21.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6q8+3xHagZp5MC2GVN/868sdOzY= 2022-09-03T06:33:21.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6q8+3xHagZp5MC2GVN/868sdOzY=', 'Content-Length': '0'} 2022-09-03T06:33:21.352 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fa5aee89665d232e-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.368 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.369 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.370 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.371 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.378 DEBUG:boto:Method: GET 2022-09-03T06:33:21.379 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.380 DEBUG:boto:Data: 2022-09-03T06:33:21.381 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.387 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.388 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.389 DEBUG:boto:Params: {} 2022-09-03T06:33:21.397 DEBUG:boto:Token: None 2022-09-03T06:33:21.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6q8+3xHagZp5MC2GVN/868sdOzY= 2022-09-03T06:33:21.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6q8+3xHagZp5MC2GVN/868sdOzY=', 'Content-Length': '0'} 2022-09-03T06:33:21.405 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000006cf40e7861f820d-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.406 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.407 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.407 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.408 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.409 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.409 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.410 DEBUG:boto:Data: 2022-09-03T06:33:21.412 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.413 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.415 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.416 DEBUG:boto:Params: {} 2022-09-03T06:33:21.417 DEBUG:boto:Token: None 2022-09-03T06:33:21.418 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QIaT/0CirApfVaV8Mdy1kG7Ss5s= 2022-09-03T06:33:21.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QIaT/0CirApfVaV8Mdy1kG7Ss5s=', 'Content-Length': '0'} 2022-09-03T06:33:21.424 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f85d1788edf7b97e-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.426 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.427 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.428 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.429 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.430 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.431 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.432 DEBUG:boto:Data: 2022-09-03T06:33:21.433 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.435 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.436 DEBUG:boto:Params: {} 2022-09-03T06:33:21.437 DEBUG:boto:Token: None 2022-09-03T06:33:21.438 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QIaT/0CirApfVaV8Mdy1kG7Ss5s= 2022-09-03T06:33:21.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QIaT/0CirApfVaV8Mdy1kG7Ss5s=', 'Content-Length': '0'} 2022-09-03T06:33:21.458 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d2f950831047b7c2-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.460 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:21.462 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.474 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.475 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.476 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.477 DEBUG:boto:Method: GET 2022-09-03T06:33:21.478 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.482 DEBUG:boto:Data: 2022-09-03T06:33:21.485 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.486 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.487 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.488 DEBUG:boto:Params: {} 2022-09-03T06:33:21.489 DEBUG:boto:Token: None 2022-09-03T06:33:21.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6q8+3xHagZp5MC2GVN/868sdOzY= 2022-09-03T06:33:21.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6q8+3xHagZp5MC2GVN/868sdOzY=', 'Content-Length': '0'} 2022-09-03T06:33:21.516 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000064f9774f582b99a6-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.522 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.523 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.524 DEBUG:boto:path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.525 DEBUG:boto:auth_path=/avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.529 DEBUG:boto:Method: GET 2022-09-03T06:33:21.530 DEBUG:boto:Path: /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.531 DEBUG:boto:Data: 2022-09-03T06:33:21.535 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.544 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.545 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.546 DEBUG:boto:Params: {} 2022-09-03T06:33:21.547 DEBUG:boto:Token: None 2022-09-03T06:33:21.548 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3?versionId=null 2022-09-03T06:33:21.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6q8+3xHagZp5MC2GVN/868sdOzY= 2022-09-03T06:33:21.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6q8+3xHagZp5MC2GVN/868sdOzY=', 'Content-Length': '0'} 2022-09-03T06:33:21.557 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001c96e4c67ab013da-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.559 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.560 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.561 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:21.563 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:21.564 DEBUG:boto:Method: GET 2022-09-03T06:33:21.565 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:21.566 DEBUG:boto:Data: 2022-09-03T06:33:21.567 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.569 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.570 DEBUG:boto:Params: {} 2022-09-03T06:33:21.572 DEBUG:boto:Token: None 2022-09-03T06:33:21.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.575 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.591 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e8d370f37fef01d7-006312f531-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.592 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.600 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.601 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.602 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:21.603 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:21.604 DEBUG:boto:Method: GET 2022-09-03T06:33:21.611 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:21.613 DEBUG:boto:Data: 2022-09-03T06:33:21.615 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.616 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.617 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.619 DEBUG:boto:Params: {} 2022-09-03T06:33:21.620 DEBUG:boto:Token: None 2022-09-03T06:33:21.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.628 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004852a0b4301fe0cf-006312f531-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.629 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.630 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.631 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.632 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.633 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:21.634 DEBUG:boto:Data: 2022-09-03T06:33:21.635 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.638 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.639 DEBUG:boto:Params: {} 2022-09-03T06:33:21.639 DEBUG:boto:Token: None 2022-09-03T06:33:21.640 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3 2022-09-03T06:33:21.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/DDqnEc2eRPyJudIna9TDA0PZi8= 2022-09-03T06:33:21.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/DDqnEc2eRPyJudIna9TDA0PZi8=', 'Content-Length': '0'} 2022-09-03T06:33:21.643 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006c1cc180775667c8-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.644 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.645 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.645 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.646 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:21.655 DEBUG:boto:Data: 2022-09-03T06:33:21.664 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.664 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.665 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.665 DEBUG:boto:Params: {} 2022-09-03T06:33:21.666 DEBUG:boto:Token: None 2022-09-03T06:33:21.667 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3 2022-09-03T06:33:21.668 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/DDqnEc2eRPyJudIna9TDA0PZi8= 2022-09-03T06:33:21.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/DDqnEc2eRPyJudIna9TDA0PZi8=', 'Content-Length': '0'} 2022-09-03T06:33:21.672 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000000aff6c23e22dc6e-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.673 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:33:21.674 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.674 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.675 DEBUG:boto:Method: GET 2022-09-03T06:33:21.676 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:21.676 DEBUG:boto:Data: 2022-09-03T06:33:21.677 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.678 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.679 DEBUG:boto:Params: {} 2022-09-03T06:33:21.681 DEBUG:boto:Token: None 2022-09-03T06:33:21.682 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3 2022-09-03T06:33:21.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xb9IX6jk0GKIvtPFME7p50/TfsQ= 2022-09-03T06:33:21.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xb9IX6jk0GKIvtPFME7p50/TfsQ=', 'Content-Length': '0'} 2022-09-03T06:33:21.690 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cf79c97dcb531dba-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.695 DEBUG:boto:path=/avysmw-60/obj3 2022-09-03T06:33:21.696 DEBUG:boto:auth_path=/avysmw-60/obj3 2022-09-03T06:33:21.696 DEBUG:boto:Method: GET 2022-09-03T06:33:21.698 DEBUG:boto:Path: /avysmw-60/obj3 2022-09-03T06:33:21.699 DEBUG:boto:Data: 2022-09-03T06:33:21.699 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.700 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.701 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.702 DEBUG:boto:Params: {} 2022-09-03T06:33:21.703 DEBUG:boto:Token: None 2022-09-03T06:33:21.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj3 2022-09-03T06:33:21.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xb9IX6jk0GKIvtPFME7p50/TfsQ= 2022-09-03T06:33:21.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xb9IX6jk0GKIvtPFME7p50/TfsQ=', 'Content-Length': '0'} 2022-09-03T06:33:21.710 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:26 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000070b3529457199845-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.711 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.712 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.713 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:21.719 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:21.721 DEBUG:boto:Method: GET 2022-09-03T06:33:21.721 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:21.728 DEBUG:boto:Data: 2022-09-03T06:33:21.728 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.729 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.730 DEBUG:boto:Params: {} 2022-09-03T06:33:21.731 DEBUG:boto:Token: None 2022-09-03T06:33:21.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.736 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002e44d20a9763182-006312f531-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.737 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.739 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.740 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.740 DEBUG:boto:path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:21.741 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj3 2022-09-03T06:33:21.742 DEBUG:boto:Method: GET 2022-09-03T06:33:21.742 DEBUG:boto:Path: /avysmw-60/?prefix=obj3 2022-09-03T06:33:21.743 DEBUG:boto:Data: 2022-09-03T06:33:21.743 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.744 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.745 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.745 DEBUG:boto:Params: {} 2022-09-03T06:33:21.746 DEBUG:boto:Token: None 2022-09-03T06:33:21.747 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.759 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000080ab0b453837e077-006312f531-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.760 DEBUG:boto:b'avysmw-60obj31000falseobj32022-09-03T06:32:26.981Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.762 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:21.763 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:21.764 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:21.765 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.766 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.767 DEBUG:boto:Method: GET 2022-09-03T06:33:21.769 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.770 DEBUG:boto:Data: 2022-09-03T06:33:21.771 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.772 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.773 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.773 DEBUG:boto:Params: {} 2022-09-03T06:33:21.774 DEBUG:boto:Token: None 2022-09-03T06:33:21.775 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QDuvDouFhUxAbrK46ExeHzMZoaE= 2022-09-03T06:33:21.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QDuvDouFhUxAbrK46ExeHzMZoaE=', 'Content-Length': '0'} 2022-09-03T06:33:21.783 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000028a7c6141adc2ac7-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.784 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:21.784 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:21.785 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.786 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.787 DEBUG:boto:Method: GET 2022-09-03T06:33:21.788 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.789 DEBUG:boto:Data: 2022-09-03T06:33:21.790 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.791 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.792 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.793 DEBUG:boto:Params: {} 2022-09-03T06:33:21.794 DEBUG:boto:Token: None 2022-09-03T06:33:21.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QDuvDouFhUxAbrK46ExeHzMZoaE= 2022-09-03T06:33:21.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QDuvDouFhUxAbrK46ExeHzMZoaE=', 'Content-Length': '0'} 2022-09-03T06:33:21.800 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003489aa922214bd63-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.802 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:21.803 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:21.803 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.804 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.805 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.805 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.807 DEBUG:boto:Data: 2022-09-03T06:33:21.808 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.809 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.810 DEBUG:boto:Params: {} 2022-09-03T06:33:21.812 DEBUG:boto:Token: None 2022-09-03T06:33:21.813 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NYXPm1RLkrv9q1lAJhwTKxYp7kM= 2022-09-03T06:33:21.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NYXPm1RLkrv9q1lAJhwTKxYp7kM=', 'Content-Length': '0'} 2022-09-03T06:33:21.818 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000eee1ee332a36bae8-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.819 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:21.820 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:21.820 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.821 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.821 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.822 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.823 DEBUG:boto:Data: 2022-09-03T06:33:21.824 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.825 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.826 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.827 DEBUG:boto:Params: {} 2022-09-03T06:33:21.828 DEBUG:boto:Token: None 2022-09-03T06:33:21.829 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.830 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NYXPm1RLkrv9q1lAJhwTKxYp7kM= 2022-09-03T06:33:21.831 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NYXPm1RLkrv9q1lAJhwTKxYp7kM=', 'Content-Length': '0'} 2022-09-03T06:33:21.834 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002a88478d8802c653-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.835 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:21.836 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:21.837 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:21.838 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.839 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.840 DEBUG:boto:Method: GET 2022-09-03T06:33:21.841 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.842 DEBUG:boto:Data: 2022-09-03T06:33:21.843 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.844 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.845 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.847 DEBUG:boto:Params: {} 2022-09-03T06:33:21.848 DEBUG:boto:Token: None 2022-09-03T06:33:21.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QDuvDouFhUxAbrK46ExeHzMZoaE= 2022-09-03T06:33:21.866 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QDuvDouFhUxAbrK46ExeHzMZoaE=', 'Content-Length': '0'} 2022-09-03T06:33:21.875 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f98588a7758b08ff-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.879 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:21.880 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:21.881 DEBUG:boto:path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.882 DEBUG:boto:auth_path=/avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.883 DEBUG:boto:Method: GET 2022-09-03T06:33:21.884 DEBUG:boto:Path: /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.885 DEBUG:boto:Data: 2022-09-03T06:33:21.887 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.888 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.889 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.891 DEBUG:boto:Params: {} 2022-09-03T06:33:21.892 DEBUG:boto:Token: None 2022-09-03T06:33:21.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj4?versionId=null 2022-09-03T06:33:21.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QDuvDouFhUxAbrK46ExeHzMZoaE= 2022-09-03T06:33:21.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QDuvDouFhUxAbrK46ExeHzMZoaE=', 'Content-Length': '0'} 2022-09-03T06:33:21.898 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000e4a036c01259186-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.899 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.900 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.900 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:21.901 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:21.902 DEBUG:boto:Method: GET 2022-09-03T06:33:21.904 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:21.904 DEBUG:boto:Data: 2022-09-03T06:33:21.905 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.906 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.907 DEBUG:boto:Params: {} 2022-09-03T06:33:21.908 DEBUG:boto:Token: None 2022-09-03T06:33:21.908 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.914 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd0d0bc052ccbcd8-006312f531-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.915 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.917 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:21.917 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:21.918 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:21.918 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:21.919 DEBUG:boto:Method: GET 2022-09-03T06:33:21.919 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:21.920 DEBUG:boto:Data: 2022-09-03T06:33:21.921 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.921 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.922 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.922 DEBUG:boto:Params: {} 2022-09-03T06:33:21.924 DEBUG:boto:Token: None 2022-09-03T06:33:21.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/ 2022-09-03T06:33:21.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0= 2022-09-03T06:33:21.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ngkwer0Ljv2BOAw62weZzv5nHl0=', 'Content-Length': '0'} 2022-09-03T06:33:21.932 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008dbf018e370479fb-006312f531-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.934 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:21.935 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:21.936 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:21.937 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.943 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:21.945 DEBUG:boto:Data: 2022-09-03T06:33:21.952 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.956 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.957 DEBUG:boto:Params: {} 2022-09-03T06:33:21.958 DEBUG:boto:Token: None 2022-09-03T06:33:21.960 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj4 2022-09-03T06:33:21.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n7JQRH145a8wmyqbcsJZF0alFAg= 2022-09-03T06:33:21.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n7JQRH145a8wmyqbcsJZF0alFAg=', 'Content-Length': '0'} 2022-09-03T06:33:21.966 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e2684625ce0d31b5-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.970 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:21.970 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:21.971 DEBUG:boto:Method: HEAD 2022-09-03T06:33:21.972 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:21.972 DEBUG:boto:Data: 2022-09-03T06:33:21.973 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.974 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:21.974 DEBUG:boto:Port: 8000 2022-09-03T06:33:21.975 DEBUG:boto:Params: {} 2022-09-03T06:33:21.976 DEBUG:boto:Token: None 2022-09-03T06:33:21.977 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj4 2022-09-03T06:33:21.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n7JQRH145a8wmyqbcsJZF0alFAg= 2022-09-03T06:33:21.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n7JQRH145a8wmyqbcsJZF0alFAg=', 'Content-Length': '0'} 2022-09-03T06:33:21.983 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000095010a479a91f38c-006312f531-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:21.984 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:33:21.986 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:21.986 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:21.987 DEBUG:boto:Method: GET 2022-09-03T06:33:21.988 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:21.989 DEBUG:boto:Data: 2022-09-03T06:33:21.990 DEBUG:boto:Headers: {} 2022-09-03T06:33:21.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:21.991 DEBUG:boto:Port: 8001 2022-09-03T06:33:21.992 DEBUG:boto:Params: {} 2022-09-03T06:33:21.992 DEBUG:boto:Token: None 2022-09-03T06:33:21.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:21 GMT /avysmw-60/obj4 2022-09-03T06:33:21.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TNdTG7H77VgpG5JWHwlwzIvNLp0= 2022-09-03T06:33:21.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TNdTG7H77VgpG5JWHwlwzIvNLp0=', 'Content-Length': '0'} 2022-09-03T06:33:22.003 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002351f789a75d91c5-006312f531-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:22.005 DEBUG:boto:path=/avysmw-60/obj4 2022-09-03T06:33:22.006 DEBUG:boto:auth_path=/avysmw-60/obj4 2022-09-03T06:33:22.007 DEBUG:boto:Method: GET 2022-09-03T06:33:22.007 DEBUG:boto:Path: /avysmw-60/obj4 2022-09-03T06:33:22.008 DEBUG:boto:Data: 2022-09-03T06:33:22.009 DEBUG:boto:Headers: {} 2022-09-03T06:33:22.010 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:22.011 DEBUG:boto:Port: 8000 2022-09-03T06:33:22.011 DEBUG:boto:Params: {} 2022-09-03T06:33:22.012 DEBUG:boto:Token: None 2022-09-03T06:33:22.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:22 GMT /avysmw-60/obj4 2022-09-03T06:33:22.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uY39eMWZ75Xp6G+IAzwFKLdNsGI= 2022-09-03T06:33:22.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uY39eMWZ75Xp6G+IAzwFKLdNsGI=', 'Content-Length': '0'} 2022-09-03T06:33:22.024 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f339f81b78bb9612-006312f532-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:33:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:22.025 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:22.026 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:22.027 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:22.027 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:22.028 DEBUG:boto:Method: GET 2022-09-03T06:33:22.029 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:22.029 DEBUG:boto:Data: 2022-09-03T06:33:22.030 DEBUG:boto:Headers: {} 2022-09-03T06:33:22.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:33:22.031 DEBUG:boto:Port: 8001 2022-09-03T06:33:22.032 DEBUG:boto:Params: {} 2022-09-03T06:33:22.040 DEBUG:boto:Token: None 2022-09-03T06:33:22.041 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:22 GMT /avysmw-60/ 2022-09-03T06:33:22.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lisJyLvjaUiPbzMra2x9R2yvMhw= 2022-09-03T06:33:22.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lisJyLvjaUiPbzMra2x9R2yvMhw=', 'Content-Length': '0'} 2022-09-03T06:33:22.051 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000096e0bfbb9f16c500-006312f532-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:22.052 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:22.055 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:33:22.056 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:33:22.057 DEBUG:boto:path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:22.061 DEBUG:boto:auth_path=/avysmw-60/?prefix=obj4 2022-09-03T06:33:22.063 DEBUG:boto:Method: GET 2022-09-03T06:33:22.065 DEBUG:boto:Path: /avysmw-60/?prefix=obj4 2022-09-03T06:33:22.069 DEBUG:boto:Data: 2022-09-03T06:33:22.071 DEBUG:boto:Headers: {} 2022-09-03T06:33:22.073 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:33:22.075 DEBUG:boto:Port: 8000 2022-09-03T06:33:22.087 DEBUG:boto:Params: {} 2022-09-03T06:33:22.093 DEBUG:boto:Token: None 2022-09-03T06:33:22.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:33:22 GMT /avysmw-60/ 2022-09-03T06:33:22.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lisJyLvjaUiPbzMra2x9R2yvMhw= 2022-09-03T06:33:22.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:33:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lisJyLvjaUiPbzMra2x9R2yvMhw=', 'Content-Length': '0'} 2022-09-03T06:33:22.115 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e59382e158a0b5a8-006312f532-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:33:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:33:22.117 DEBUG:boto:b'avysmw-60obj41000falseobj42022-09-03T06:32:27.046Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:33:22.122 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-60 zones={test-zone3, test-zone2} 2022-09-03T06:33:22.123 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-61 2022-09-03T06:33:22.124 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:22.125 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:33:22.127 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-61 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:33:22.195 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:33:22.195 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:33:22.253 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:22.250+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:33:27.253 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:27.251+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:33:32.256 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:32.251+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:33:37.254 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:37.252+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:33:42.255 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:42.252+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:33:47.256 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:47.253+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:33:52.263 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:52.254+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:33:57.257 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:33:57.254+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:02.266 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:02.255+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:07.260 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:07.256+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:12.260 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:12.257+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:17.266 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:17.257+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:22.261 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:22.258+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:27.263 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:27.258+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:32.261 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:32.259+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:37.262 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:37.260+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:42.264 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:42.260+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:47.263 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:47.261+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:52.265 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:52.262+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:34:57.266 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:34:57.263+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:35:02.266 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:02.264+0000 7f88fccc0600 1 waiting to reach incremental sync.. 2022-09-03T06:35:07.268 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:07.265+0000 7f88fccc0600 1 bucket sync caught up with source: 2022-09-03T06:35:07.268 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.37.5, , 00000000001.49.5, , 00000000001.67.5, , , 00000000001.36.5, , , ] 2022-09-03T06:35:07.268 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.37.5, , 00000000001.49.5, , 00000000001.67.5, , , 00000000001.36.5, , , ] 2022-09-03T06:35:07.269 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:07.265+0000 7f88fccc0600 0 bucket checkpoint complete 2022-09-03T06:35:07.286 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-61 2022-09-03T06:35:07.288 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:07.288 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:07.290 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-61 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:35:07.356 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:07.357 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:07.399 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:07.396+0000 7f88d31fb600 1 bucket sync caught up with source: 2022-09-03T06:35:07.399 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.37.5, , 00000000001.49.5, , 00000000001.67.5, , , 00000000001.36.5, , , ] 2022-09-03T06:35:07.400 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.37.5, , 00000000001.49.5, , 00000000001.67.5, , , 00000000001.36.5, , , ] 2022-09-03T06:35:07.400 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:07.396+0000 7f88d31fb600 0 bucket checkpoint complete 2022-09-03T06:35:07.414 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-61 2022-09-03T06:35:07.414 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:07.417 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:07.418 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-61 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:35:07.486 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:07.487 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:07.520 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:07.518+0000 7f04ce574600 1 bucket sync caught up with source: 2022-09-03T06:35:07.521 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.37.5, , 00000000001.49.5, , 00000000001.67.5, , , 00000000001.36.5, , , ] 2022-09-03T06:35:07.523 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.37.5, , 00000000001.49.5, , 00000000001.67.5, , , 00000000001.36.5, , , ] 2022-09-03T06:35:07.523 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:07.518+0000 7f04ce574600 0 bucket checkpoint complete 2022-09-03T06:35:07.535 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-61 2022-09-03T06:35:07.537 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:07.537 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:07.541 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-61 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:35:07.603 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:07.604 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:07.657 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:07.652+0000 7fd6591a9600 1 bucket sync caught up with source: 2022-09-03T06:35:07.657 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.239.5, , 00000000001.208.5, , 00000000001.302.5, , , 00000000001.297.5, , , ] 2022-09-03T06:35:07.658 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.239.5, , 00000000001.208.5, , 00000000001.302.5, , , 00000000001.297.5, , , ] 2022-09-03T06:35:07.658 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:07.652+0000 7fd6591a9600 0 bucket checkpoint complete 2022-09-03T06:35:07.674 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-61 2022-09-03T06:35:07.675 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:07.676 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:07.677 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-61 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:35:07.738 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:07.739 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:07.788 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:07.786+0000 7fe333dcc600 1 waiting to reach incremental sync.. 2022-09-03T06:35:12.789 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:12.787+0000 7fe333dcc600 1 waiting to reach incremental sync.. 2022-09-03T06:35:17.790 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:17.788+0000 7fe333dcc600 1 waiting to reach incremental sync.. 2022-09-03T06:35:22.792 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:22.788+0000 7fe333dcc600 1 waiting to reach incremental sync.. 2022-09-03T06:35:27.792 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:27.790+0000 7fe333dcc600 1 bucket sync caught up with source: 2022-09-03T06:35:27.792 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.239.5, , 00000000001.208.5, , 00000000001.302.5, , , 00000000001.297.5, , , ] 2022-09-03T06:35:27.793 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.239.5, , 00000000001.208.5, , 00000000001.302.5, , , 00000000001.297.5, , , ] 2022-09-03T06:35:27.793 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:27.790+0000 7fe333dcc600 0 bucket checkpoint complete 2022-09-03T06:35:27.813 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-61 2022-09-03T06:35:27.815 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-61 2022-09-03T06:35:27.816 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:27.817 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:27.818 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-61 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:35:27.882 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:27.882 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:27.943 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:27.941+0000 7f271e406600 1 bucket sync caught up with source: 2022-09-03T06:35:27.944 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.138.5, , 00000000001.217.5, , 00000000001.133.5, , , 00000000001.325.5, , , ] 2022-09-03T06:35:27.944 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.138.5, , 00000000001.217.5, , 00000000001.133.5, , , 00000000001.325.5, , , ] 2022-09-03T06:35:27.944 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:27.941+0000 7f271e406600 0 bucket checkpoint complete 2022-09-03T06:35:27.963 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-61 2022-09-03T06:35:27.964 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:27.965 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-61', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:27.965 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-61 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:35:28.031 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:28.032 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:28.068 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:28.066+0000 7fea11b07600 1 bucket sync caught up with source: 2022-09-03T06:35:28.069 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.138.5, , 00000000001.217.5, , 00000000001.133.5, , , 00000000001.325.5, , , ] 2022-09-03T06:35:28.069 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.138.5, , 00000000001.217.5, , 00000000001.133.5, , , 00000000001.325.5, , , ] 2022-09-03T06:35:28.070 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:28.066+0000 7fea11b07600 0 bucket checkpoint complete 2022-09-03T06:35:28.083 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-61 2022-09-03T06:35:28.084 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-61 zones={test-zone2, test-zone1} 2022-09-03T06:35:28.086 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.087 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.088 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.089 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:28.090 DEBUG:boto:Data: 2022-09-03T06:35:28.091 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.092 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.093 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.094 DEBUG:boto:Params: {} 2022-09-03T06:35:28.096 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:35:28.098 DEBUG:boto:Token: None 2022-09-03T06:35:28.099 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t1YJkJhsSRaGddaGtBNqpWPy9xE= 2022-09-03T06:35:28.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t1YJkJhsSRaGddaGtBNqpWPy9xE=', 'Content-Length': '0'} 2022-09-03T06:35:28.111 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e4162631c3f61ae6-006312f5b0-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.113 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.114 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.115 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.116 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:28.117 DEBUG:boto:Data: 2022-09-03T06:35:28.118 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.122 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.123 DEBUG:boto:Params: {} 2022-09-03T06:35:28.123 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:35:28.124 DEBUG:boto:Token: None 2022-09-03T06:35:28.125 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t1YJkJhsSRaGddaGtBNqpWPy9xE= 2022-09-03T06:35:28.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t1YJkJhsSRaGddaGtBNqpWPy9xE=', 'Content-Length': '0'} 2022-09-03T06:35:28.130 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009ed4f5e3e3652e5e-006312f5b0-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.131 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:35:28.132 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.132 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.133 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.133 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.134 DEBUG:boto:Method: GET 2022-09-03T06:35:28.134 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.135 DEBUG:boto:Data: 2022-09-03T06:35:28.135 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.136 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.137 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.137 DEBUG:boto:Params: {} 2022-09-03T06:35:28.139 DEBUG:boto:Token: None 2022-09-03T06:35:28.140 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/?versions 2022-09-03T06:35:28.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YOQlGjdgrt97XrFDAjFR/XVny8= 2022-09-03T06:35:28.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YOQlGjdgrt97XrFDAjFR/XVny8=', 'Content-Length': '0'} 2022-09-03T06:35:28.145 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e06cd34e0f99b6e2-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.146 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.149 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:35:28.150 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:35:28.151 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:35:28.152 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:35:28.152 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:35:28.153 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.153 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.154 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.155 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.155 DEBUG:boto:Method: GET 2022-09-03T06:35:28.156 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.157 DEBUG:boto:Data: 2022-09-03T06:35:28.157 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.159 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.159 DEBUG:boto:Params: {} 2022-09-03T06:35:28.160 DEBUG:boto:Token: None 2022-09-03T06:35:28.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/?versions 2022-09-03T06:35:28.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YOQlGjdgrt97XrFDAjFR/XVny8= 2022-09-03T06:35:28.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YOQlGjdgrt97XrFDAjFR/XVny8=', 'Content-Length': '0'} 2022-09-03T06:35:28.164 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036c7001b2a94d5ef-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.166 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.168 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:35:28.169 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:35:28.170 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:35:28.170 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:35:28.172 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.172 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.173 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.174 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.175 DEBUG:boto:Method: GET 2022-09-03T06:35:28.176 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.177 DEBUG:boto:Data: 2022-09-03T06:35:28.178 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.179 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.180 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.181 DEBUG:boto:Params: {} 2022-09-03T06:35:28.182 DEBUG:boto:Token: None 2022-09-03T06:35:28.183 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/?versions 2022-09-03T06:35:28.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YOQlGjdgrt97XrFDAjFR/XVny8= 2022-09-03T06:35:28.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YOQlGjdgrt97XrFDAjFR/XVny8=', 'Content-Length': '0'} 2022-09-03T06:35:28.188 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000070b3a1df7fc3e4f3-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.188 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.189 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.190 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.191 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.191 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.192 DEBUG:boto:Method: GET 2022-09-03T06:35:28.192 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:28.193 DEBUG:boto:Data: 2022-09-03T06:35:28.193 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.194 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.195 DEBUG:boto:Params: {} 2022-09-03T06:35:28.195 DEBUG:boto:Token: None 2022-09-03T06:35:28.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/?versions 2022-09-03T06:35:28.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YOQlGjdgrt97XrFDAjFR/XVny8= 2022-09-03T06:35:28.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YOQlGjdgrt97XrFDAjFR/XVny8=', 'Content-Length': '0'} 2022-09-03T06:35:28.200 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002830b64f0b5fcc5b-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.201 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.202 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:35:28.202 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.203 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.204 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.204 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.205 DEBUG:boto:Method: GET 2022-09-03T06:35:28.205 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.206 DEBUG:boto:Data: 2022-09-03T06:35:28.206 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.207 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.208 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.208 DEBUG:boto:Params: {} 2022-09-03T06:35:28.209 DEBUG:boto:Token: None 2022-09-03T06:35:28.209 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Uls9W+zLzuN8D6I5zbX05D0w0w= 2022-09-03T06:35:28.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Uls9W+zLzuN8D6I5zbX05D0w0w=', 'Content-Length': '0'} 2022-09-03T06:35:28.215 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000061ea12d2954cd506-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.216 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.217 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.217 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.218 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.219 DEBUG:boto:Method: GET 2022-09-03T06:35:28.219 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.220 DEBUG:boto:Data: 2022-09-03T06:35:28.220 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.221 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.222 DEBUG:boto:Params: {} 2022-09-03T06:35:28.223 DEBUG:boto:Token: None 2022-09-03T06:35:28.224 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Uls9W+zLzuN8D6I5zbX05D0w0w= 2022-09-03T06:35:28.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Uls9W+zLzuN8D6I5zbX05D0w0w=', 'Content-Length': '0'} 2022-09-03T06:35:28.227 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c4988154de46fa11-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.228 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.229 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.230 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.230 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.231 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.231 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.232 DEBUG:boto:Data: 2022-09-03T06:35:28.232 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.233 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.233 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.235 DEBUG:boto:Params: {} 2022-09-03T06:35:28.235 DEBUG:boto:Token: None 2022-09-03T06:35:28.236 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:whZY4m5lWujTt9mQhgXOai6lDkc= 2022-09-03T06:35:28.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:whZY4m5lWujTt9mQhgXOai6lDkc=', 'Content-Length': '0'} 2022-09-03T06:35:28.239 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003f1a64523ef7fb5f-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.240 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.240 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.241 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.241 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.242 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.243 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.243 DEBUG:boto:Data: 2022-09-03T06:35:28.244 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.244 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.245 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.245 DEBUG:boto:Params: {} 2022-09-03T06:35:28.246 DEBUG:boto:Token: None 2022-09-03T06:35:28.247 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:whZY4m5lWujTt9mQhgXOai6lDkc= 2022-09-03T06:35:28.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:whZY4m5lWujTt9mQhgXOai6lDkc=', 'Content-Length': '0'} 2022-09-03T06:35:28.251 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003aaef192fc30ccfb-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.252 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:35:28.253 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.253 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.254 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.254 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.255 DEBUG:boto:Method: GET 2022-09-03T06:35:28.255 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.256 DEBUG:boto:Data: 2022-09-03T06:35:28.257 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.257 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.258 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.258 DEBUG:boto:Params: {} 2022-09-03T06:35:28.259 DEBUG:boto:Token: None 2022-09-03T06:35:28.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Uls9W+zLzuN8D6I5zbX05D0w0w= 2022-09-03T06:35:28.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Uls9W+zLzuN8D6I5zbX05D0w0w=', 'Content-Length': '0'} 2022-09-03T06:35:28.263 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000502edf86a9efc938-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.264 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.264 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.265 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.266 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.266 DEBUG:boto:Method: GET 2022-09-03T06:35:28.267 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.267 DEBUG:boto:Data: 2022-09-03T06:35:28.268 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.269 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.270 DEBUG:boto:Params: {} 2022-09-03T06:35:28.270 DEBUG:boto:Token: None 2022-09-03T06:35:28.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:28.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Uls9W+zLzuN8D6I5zbX05D0w0w= 2022-09-03T06:35:28.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Uls9W+zLzuN8D6I5zbX05D0w0w=', 'Content-Length': '0'} 2022-09-03T06:35:28.274 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d8c1412e0c3ad01e-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.276 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.276 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.277 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:28.278 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:28.278 DEBUG:boto:Method: GET 2022-09-03T06:35:28.279 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:28.280 DEBUG:boto:Data: 2022-09-03T06:35:28.281 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.282 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.283 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.284 DEBUG:boto:Params: {} 2022-09-03T06:35:28.285 DEBUG:boto:Token: None 2022-09-03T06:35:28.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.292 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003198606ecf656341-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.293 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.294 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.295 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.296 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:28.297 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:28.298 DEBUG:boto:Method: GET 2022-09-03T06:35:28.299 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:28.300 DEBUG:boto:Data: 2022-09-03T06:35:28.301 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.302 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.303 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.304 DEBUG:boto:Params: {} 2022-09-03T06:35:28.305 DEBUG:boto:Token: None 2022-09-03T06:35:28.305 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.306 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.307 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.310 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c30facfd682a758-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.311 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.312 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.313 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.313 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.314 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:28.315 DEBUG:boto:Data: 2022-09-03T06:35:28.315 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.315 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.316 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.316 DEBUG:boto:Params: {} 2022-09-03T06:35:28.317 DEBUG:boto:Token: None 2022-09-03T06:35:28.318 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1 2022-09-03T06:35:28.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DBrWYgviff6wtuFww+WesXQEdRk= 2022-09-03T06:35:28.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DBrWYgviff6wtuFww+WesXQEdRk=', 'Content-Length': '0'} 2022-09-03T06:35:28.321 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000058720a90ec60a23b-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.322 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.322 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.323 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.323 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:28.324 DEBUG:boto:Data: 2022-09-03T06:35:28.324 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.325 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.325 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.326 DEBUG:boto:Params: {} 2022-09-03T06:35:28.327 DEBUG:boto:Token: None 2022-09-03T06:35:28.327 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1 2022-09-03T06:35:28.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DBrWYgviff6wtuFww+WesXQEdRk= 2022-09-03T06:35:28.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DBrWYgviff6wtuFww+WesXQEdRk=', 'Content-Length': '0'} 2022-09-03T06:35:28.330 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000509393e9cf5ad8e3-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.331 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:35:28.332 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.332 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.333 DEBUG:boto:Method: GET 2022-09-03T06:35:28.334 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:28.334 DEBUG:boto:Data: 2022-09-03T06:35:28.335 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.335 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.336 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.336 DEBUG:boto:Params: {} 2022-09-03T06:35:28.337 DEBUG:boto:Token: None 2022-09-03T06:35:28.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1 2022-09-03T06:35:28.338 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fhi+b8J2Ew06o8NqdmEDqPrw1B8= 2022-09-03T06:35:28.339 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fhi+b8J2Ew06o8NqdmEDqPrw1B8=', 'Content-Length': '0'} 2022-09-03T06:35:28.342 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f57486491469fa02-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.343 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:28.343 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:28.344 DEBUG:boto:Method: GET 2022-09-03T06:35:28.344 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:28.345 DEBUG:boto:Data: 2022-09-03T06:35:28.345 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.346 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.347 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.347 DEBUG:boto:Params: {} 2022-09-03T06:35:28.348 DEBUG:boto:Token: None 2022-09-03T06:35:28.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj1 2022-09-03T06:35:28.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fhi+b8J2Ew06o8NqdmEDqPrw1B8= 2022-09-03T06:35:28.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fhi+b8J2Ew06o8NqdmEDqPrw1B8=', 'Content-Length': '0'} 2022-09-03T06:35:28.352 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000007feff1ca44184f82-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.400 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.401 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.401 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:28.402 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:28.402 DEBUG:boto:Method: GET 2022-09-03T06:35:28.403 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:28.404 DEBUG:boto:Data: 2022-09-03T06:35:28.404 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.405 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.406 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.406 DEBUG:boto:Params: {} 2022-09-03T06:35:28.407 DEBUG:boto:Token: None 2022-09-03T06:35:28.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.413 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc6cfa5ebba2d185-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.414 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.415 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.416 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.416 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:28.417 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:28.417 DEBUG:boto:Method: GET 2022-09-03T06:35:28.418 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:28.418 DEBUG:boto:Data: 2022-09-03T06:35:28.419 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.420 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.420 DEBUG:boto:Params: {} 2022-09-03T06:35:28.422 DEBUG:boto:Token: None 2022-09-03T06:35:28.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.429 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f03c8f28eec72568-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.430 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.431 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:35:28.432 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.432 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.433 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.433 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.434 DEBUG:boto:Method: GET 2022-09-03T06:35:28.434 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.435 DEBUG:boto:Data: 2022-09-03T06:35:28.436 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.436 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.437 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.437 DEBUG:boto:Params: {} 2022-09-03T06:35:28.438 DEBUG:boto:Token: None 2022-09-03T06:35:28.439 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZbU4A6ACd1TsweeOi8LGEvt4IOY= 2022-09-03T06:35:28.440 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZbU4A6ACd1TsweeOi8LGEvt4IOY=', 'Content-Length': '0'} 2022-09-03T06:35:28.443 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ddbc600f7d9f26e9-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.443 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.444 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.445 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.446 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.446 DEBUG:boto:Method: GET 2022-09-03T06:35:28.447 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.447 DEBUG:boto:Data: 2022-09-03T06:35:28.448 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.449 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.449 DEBUG:boto:Params: {} 2022-09-03T06:35:28.450 DEBUG:boto:Token: None 2022-09-03T06:35:28.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZbU4A6ACd1TsweeOi8LGEvt4IOY= 2022-09-03T06:35:28.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZbU4A6ACd1TsweeOi8LGEvt4IOY=', 'Content-Length': '0'} 2022-09-03T06:35:28.455 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000c2dcdeaf59d1512-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.456 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.457 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.458 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.459 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.459 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.460 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.461 DEBUG:boto:Data: 2022-09-03T06:35:28.462 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.463 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.463 DEBUG:boto:Params: {} 2022-09-03T06:35:28.464 DEBUG:boto:Token: None 2022-09-03T06:35:28.465 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L1YeUufdPxy8ac6QinDDnnkx+RA= 2022-09-03T06:35:28.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L1YeUufdPxy8ac6QinDDnnkx+RA=', 'Content-Length': '0'} 2022-09-03T06:35:28.470 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005bc6a9fcb06afede-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.471 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.471 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.472 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.472 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.473 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.474 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.474 DEBUG:boto:Data: 2022-09-03T06:35:28.475 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.476 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.477 DEBUG:boto:Params: {} 2022-09-03T06:35:28.477 DEBUG:boto:Token: None 2022-09-03T06:35:28.478 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L1YeUufdPxy8ac6QinDDnnkx+RA= 2022-09-03T06:35:28.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L1YeUufdPxy8ac6QinDDnnkx+RA=', 'Content-Length': '0'} 2022-09-03T06:35:28.481 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000053dcc62a373c2ff-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.482 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:35:28.483 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.484 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.484 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.485 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.486 DEBUG:boto:Method: GET 2022-09-03T06:35:28.486 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.487 DEBUG:boto:Data: 2022-09-03T06:35:28.487 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.488 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.488 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.489 DEBUG:boto:Params: {} 2022-09-03T06:35:28.490 DEBUG:boto:Token: None 2022-09-03T06:35:28.491 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZbU4A6ACd1TsweeOi8LGEvt4IOY= 2022-09-03T06:35:28.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZbU4A6ACd1TsweeOi8LGEvt4IOY=', 'Content-Length': '0'} 2022-09-03T06:35:28.495 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001e7dea96bf717593-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.496 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.497 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.497 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.498 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.499 DEBUG:boto:Method: GET 2022-09-03T06:35:28.499 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.500 DEBUG:boto:Data: 2022-09-03T06:35:28.500 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.501 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.501 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.502 DEBUG:boto:Params: {} 2022-09-03T06:35:28.503 DEBUG:boto:Token: None 2022-09-03T06:35:28.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:28.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZbU4A6ACd1TsweeOi8LGEvt4IOY= 2022-09-03T06:35:28.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZbU4A6ACd1TsweeOi8LGEvt4IOY=', 'Content-Length': '0'} 2022-09-03T06:35:28.507 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000b80c5d138736edc-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.508 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.509 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.511 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:28.511 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:28.512 DEBUG:boto:Method: GET 2022-09-03T06:35:28.513 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:28.514 DEBUG:boto:Data: 2022-09-03T06:35:28.515 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.515 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.516 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.516 DEBUG:boto:Params: {} 2022-09-03T06:35:28.517 DEBUG:boto:Token: None 2022-09-03T06:35:28.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.525 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ee74335463dedaa7-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.570 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.573 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.574 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.575 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:28.575 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:28.576 DEBUG:boto:Method: GET 2022-09-03T06:35:28.577 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:28.577 DEBUG:boto:Data: 2022-09-03T06:35:28.578 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.578 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.579 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.580 DEBUG:boto:Params: {} 2022-09-03T06:35:28.581 DEBUG:boto:Token: None 2022-09-03T06:35:28.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.586 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009088a8fcb5fc6e48-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.587 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.588 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.588 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.589 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.590 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:28.590 DEBUG:boto:Data: 2022-09-03T06:35:28.591 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.593 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.593 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.594 DEBUG:boto:Params: {} 2022-09-03T06:35:28.595 DEBUG:boto:Token: None 2022-09-03T06:35:28.595 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2 2022-09-03T06:35:28.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJGI4sdVLHVEuvcsDixT8LibwBc= 2022-09-03T06:35:28.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJGI4sdVLHVEuvcsDixT8LibwBc=', 'Content-Length': '0'} 2022-09-03T06:35:28.599 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e6a16dbd45d52e55-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.600 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.601 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.602 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.603 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:28.604 DEBUG:boto:Data: 2022-09-03T06:35:28.605 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.606 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.606 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.607 DEBUG:boto:Params: {} 2022-09-03T06:35:28.608 DEBUG:boto:Token: None 2022-09-03T06:35:28.609 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2 2022-09-03T06:35:28.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJGI4sdVLHVEuvcsDixT8LibwBc= 2022-09-03T06:35:28.610 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJGI4sdVLHVEuvcsDixT8LibwBc=', 'Content-Length': '0'} 2022-09-03T06:35:28.612 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ee57bc724aced736-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.614 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:35:28.615 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.615 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.616 DEBUG:boto:Method: GET 2022-09-03T06:35:28.616 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:28.617 DEBUG:boto:Data: 2022-09-03T06:35:28.618 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.618 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.619 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.620 DEBUG:boto:Params: {} 2022-09-03T06:35:28.621 DEBUG:boto:Token: None 2022-09-03T06:35:28.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2 2022-09-03T06:35:28.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FDm7HxBpCZNQy+0YN7rjiyi/qBA= 2022-09-03T06:35:28.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FDm7HxBpCZNQy+0YN7rjiyi/qBA=', 'Content-Length': '0'} 2022-09-03T06:35:28.629 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000ef061085323b6ee-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.631 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:28.632 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:28.633 DEBUG:boto:Method: GET 2022-09-03T06:35:28.634 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:28.636 DEBUG:boto:Data: 2022-09-03T06:35:28.637 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.637 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.638 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.639 DEBUG:boto:Params: {} 2022-09-03T06:35:28.640 DEBUG:boto:Token: None 2022-09-03T06:35:28.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj2 2022-09-03T06:35:28.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FDm7HxBpCZNQy+0YN7rjiyi/qBA= 2022-09-03T06:35:28.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FDm7HxBpCZNQy+0YN7rjiyi/qBA=', 'Content-Length': '0'} 2022-09-03T06:35:28.644 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002c30ed284f80f7d5-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.645 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.646 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.646 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:28.647 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:28.648 DEBUG:boto:Method: GET 2022-09-03T06:35:28.648 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:28.649 DEBUG:boto:Data: 2022-09-03T06:35:28.649 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.650 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.650 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.651 DEBUG:boto:Params: {} 2022-09-03T06:35:28.651 DEBUG:boto:Token: None 2022-09-03T06:35:28.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.656 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a4fdad658125853-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.698 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.700 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.701 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.702 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:28.703 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:28.704 DEBUG:boto:Method: GET 2022-09-03T06:35:28.705 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:28.706 DEBUG:boto:Data: 2022-09-03T06:35:28.707 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.708 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.709 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.710 DEBUG:boto:Params: {} 2022-09-03T06:35:28.711 DEBUG:boto:Token: None 2022-09-03T06:35:28.712 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.716 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000609e9582945aaa92-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.716 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.717 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:35:28.718 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.718 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.719 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.720 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.720 DEBUG:boto:Method: GET 2022-09-03T06:35:28.721 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.722 DEBUG:boto:Data: 2022-09-03T06:35:28.722 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.723 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.723 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.724 DEBUG:boto:Params: {} 2022-09-03T06:35:28.725 DEBUG:boto:Token: None 2022-09-03T06:35:28.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CWG/9HEUEnLboC2KTav/SYYJnew= 2022-09-03T06:35:28.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CWG/9HEUEnLboC2KTav/SYYJnew=', 'Content-Length': '0'} 2022-09-03T06:35:28.730 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001d5ca94b676af217-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.731 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.731 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.732 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.732 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.733 DEBUG:boto:Method: GET 2022-09-03T06:35:28.734 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.735 DEBUG:boto:Data: 2022-09-03T06:35:28.735 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.736 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.736 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.737 DEBUG:boto:Params: {} 2022-09-03T06:35:28.738 DEBUG:boto:Token: None 2022-09-03T06:35:28.739 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CWG/9HEUEnLboC2KTav/SYYJnew= 2022-09-03T06:35:28.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CWG/9HEUEnLboC2KTav/SYYJnew=', 'Content-Length': '0'} 2022-09-03T06:35:28.744 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000bfb458c8b394e0d-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.746 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.747 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.748 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.749 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.749 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.750 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.751 DEBUG:boto:Data: 2022-09-03T06:35:28.752 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.753 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.753 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.754 DEBUG:boto:Params: {} 2022-09-03T06:35:28.755 DEBUG:boto:Token: None 2022-09-03T06:35:28.756 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NVmfeTE7MgEF8wMOQdrUMCLKmRU= 2022-09-03T06:35:28.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NVmfeTE7MgEF8wMOQdrUMCLKmRU=', 'Content-Length': '0'} 2022-09-03T06:35:28.760 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000482d1b5b720b0b94-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.761 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.761 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.762 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.762 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.763 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.763 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.764 DEBUG:boto:Data: 2022-09-03T06:35:28.765 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.766 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.767 DEBUG:boto:Params: {} 2022-09-03T06:35:28.767 DEBUG:boto:Token: None 2022-09-03T06:35:28.768 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NVmfeTE7MgEF8wMOQdrUMCLKmRU= 2022-09-03T06:35:28.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NVmfeTE7MgEF8wMOQdrUMCLKmRU=', 'Content-Length': '0'} 2022-09-03T06:35:28.772 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d83fb4bf1beb6a83-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.773 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:35:28.774 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.775 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.776 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.776 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.777 DEBUG:boto:Method: GET 2022-09-03T06:35:28.778 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.779 DEBUG:boto:Data: 2022-09-03T06:35:28.779 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.780 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.780 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.781 DEBUG:boto:Params: {} 2022-09-03T06:35:28.782 DEBUG:boto:Token: None 2022-09-03T06:35:28.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CWG/9HEUEnLboC2KTav/SYYJnew= 2022-09-03T06:35:28.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CWG/9HEUEnLboC2KTav/SYYJnew=', 'Content-Length': '0'} 2022-09-03T06:35:28.786 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004cdaab819cfa2d5c-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.787 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.788 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.788 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.789 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.789 DEBUG:boto:Method: GET 2022-09-03T06:35:28.790 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.791 DEBUG:boto:Data: 2022-09-03T06:35:28.792 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.794 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.795 DEBUG:boto:Params: {} 2022-09-03T06:35:28.796 DEBUG:boto:Token: None 2022-09-03T06:35:28.797 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:28.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CWG/9HEUEnLboC2KTav/SYYJnew= 2022-09-03T06:35:28.798 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CWG/9HEUEnLboC2KTav/SYYJnew=', 'Content-Length': '0'} 2022-09-03T06:35:28.800 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ea4692465543dcac-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.801 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.802 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.802 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:28.803 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:28.803 DEBUG:boto:Method: GET 2022-09-03T06:35:28.804 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:28.804 DEBUG:boto:Data: 2022-09-03T06:35:28.805 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.806 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.806 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.807 DEBUG:boto:Params: {} 2022-09-03T06:35:28.807 DEBUG:boto:Token: None 2022-09-03T06:35:28.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.812 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f3d274d92cb0ab6-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.812 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.813 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.814 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.814 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:28.815 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:28.815 DEBUG:boto:Method: GET 2022-09-03T06:35:28.816 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:28.816 DEBUG:boto:Data: 2022-09-03T06:35:28.817 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.818 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.819 DEBUG:boto:Params: {} 2022-09-03T06:35:28.819 DEBUG:boto:Token: None 2022-09-03T06:35:28.820 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.821 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.826 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5c1e8ed04420741-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.827 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.829 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.830 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.831 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.832 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:28.833 DEBUG:boto:Data: 2022-09-03T06:35:28.834 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.835 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.836 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.837 DEBUG:boto:Params: {} 2022-09-03T06:35:28.838 DEBUG:boto:Token: None 2022-09-03T06:35:28.839 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3 2022-09-03T06:35:28.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kICoFqT+csjB9SzNIWiJM9LhNTk= 2022-09-03T06:35:28.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kICoFqT+csjB9SzNIWiJM9LhNTk=', 'Content-Length': '0'} 2022-09-03T06:35:28.844 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000376656ade1f0d46b-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.845 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.846 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.847 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.848 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:28.849 DEBUG:boto:Data: 2022-09-03T06:35:28.850 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.852 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.853 DEBUG:boto:Params: {} 2022-09-03T06:35:28.854 DEBUG:boto:Token: None 2022-09-03T06:35:28.855 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3 2022-09-03T06:35:28.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kICoFqT+csjB9SzNIWiJM9LhNTk= 2022-09-03T06:35:28.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kICoFqT+csjB9SzNIWiJM9LhNTk=', 'Content-Length': '0'} 2022-09-03T06:35:28.859 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008037f99a34ea4393-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.860 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:35:28.861 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.862 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.863 DEBUG:boto:Method: GET 2022-09-03T06:35:28.864 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:28.865 DEBUG:boto:Data: 2022-09-03T06:35:28.866 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.867 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.868 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.869 DEBUG:boto:Params: {} 2022-09-03T06:35:28.871 DEBUG:boto:Token: None 2022-09-03T06:35:28.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3 2022-09-03T06:35:28.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:07idT0uTMCVlzwRcCXjLQr3czmo= 2022-09-03T06:35:28.874 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:07idT0uTMCVlzwRcCXjLQr3czmo=', 'Content-Length': '0'} 2022-09-03T06:35:28.876 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b0471ad15749b710-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.877 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:28.878 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:28.878 DEBUG:boto:Method: GET 2022-09-03T06:35:28.879 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:28.880 DEBUG:boto:Data: 2022-09-03T06:35:28.880 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.882 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.882 DEBUG:boto:Params: {} 2022-09-03T06:35:28.883 DEBUG:boto:Token: None 2022-09-03T06:35:28.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj3 2022-09-03T06:35:28.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:07idT0uTMCVlzwRcCXjLQr3czmo= 2022-09-03T06:35:28.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:07idT0uTMCVlzwRcCXjLQr3czmo=', 'Content-Length': '0'} 2022-09-03T06:35:28.887 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000087f703b3344e01bf-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.930 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.931 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.932 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:28.932 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:28.933 DEBUG:boto:Method: GET 2022-09-03T06:35:28.933 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:28.934 DEBUG:boto:Data: 2022-09-03T06:35:28.934 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.935 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.936 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.937 DEBUG:boto:Params: {} 2022-09-03T06:35:28.937 DEBUG:boto:Token: None 2022-09-03T06:35:28.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.944 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002fd5646686102d7d-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.945 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.946 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:28.947 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:28.948 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:28.948 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:28.949 DEBUG:boto:Method: GET 2022-09-03T06:35:28.949 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:28.950 DEBUG:boto:Data: 2022-09-03T06:35:28.951 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.952 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.952 DEBUG:boto:Params: {} 2022-09-03T06:35:28.953 DEBUG:boto:Token: None 2022-09-03T06:35:28.953 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/ 2022-09-03T06:35:28.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI= 2022-09-03T06:35:28.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RWfXuvdyHJwnY1N4qaSFnSLRkVI=', 'Content-Length': '0'} 2022-09-03T06:35:28.966 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001654d6e2de7c9f2c-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.967 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:28.968 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:35:28.968 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:28.969 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:28.970 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.970 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.971 DEBUG:boto:Method: GET 2022-09-03T06:35:28.971 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.972 DEBUG:boto:Data: 2022-09-03T06:35:28.972 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.973 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:28.974 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.975 DEBUG:boto:Params: {} 2022-09-03T06:35:28.976 DEBUG:boto:Token: None 2022-09-03T06:35:28.977 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Do+Ee3TCK/qwX1xAG0+v8ishbEo= 2022-09-03T06:35:28.979 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Do+Ee3TCK/qwX1xAG0+v8ishbEo=', 'Content-Length': '0'} 2022-09-03T06:35:28.982 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000d68e9568fbb05e8-006312f5b0-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.983 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:28.983 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:28.984 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.985 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.985 DEBUG:boto:Method: GET 2022-09-03T06:35:28.986 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.986 DEBUG:boto:Data: 2022-09-03T06:35:28.987 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.987 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:28.988 DEBUG:boto:Port: 8000 2022-09-03T06:35:28.989 DEBUG:boto:Params: {} 2022-09-03T06:35:28.989 DEBUG:boto:Token: None 2022-09-03T06:35:28.990 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:28 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Do+Ee3TCK/qwX1xAG0+v8ishbEo= 2022-09-03T06:35:28.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:28 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Do+Ee3TCK/qwX1xAG0+v8ishbEo=', 'Content-Length': '0'} 2022-09-03T06:35:28.994 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c8a90e46032f508c-006312f5b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:28 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:28.994 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:28.995 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:28.996 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.996 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.997 DEBUG:boto:Method: HEAD 2022-09-03T06:35:28.997 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:28.998 DEBUG:boto:Data: 2022-09-03T06:35:28.998 DEBUG:boto:Headers: {} 2022-09-03T06:35:28.999 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:29.000 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.000 DEBUG:boto:Params: {} 2022-09-03T06:35:29.001 DEBUG:boto:Token: None 2022-09-03T06:35:29.002 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.002 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Vgv5U/gyrm/KvoevdNCV0tdMiQ= 2022-09-03T06:35:29.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Vgv5U/gyrm/KvoevdNCV0tdMiQ=', 'Content-Length': '0'} 2022-09-03T06:35:29.010 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000095876654828336d8-006312f5b1-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.011 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:29.012 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:29.013 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.014 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.015 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.016 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.017 DEBUG:boto:Data: 2022-09-03T06:35:29.018 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.019 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.020 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.021 DEBUG:boto:Params: {} 2022-09-03T06:35:29.022 DEBUG:boto:Token: None 2022-09-03T06:35:29.023 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Vgv5U/gyrm/KvoevdNCV0tdMiQ= 2022-09-03T06:35:29.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Vgv5U/gyrm/KvoevdNCV0tdMiQ=', 'Content-Length': '0'} 2022-09-03T06:35:29.027 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ac7d255b664d5d11-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.028 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:35:29.028 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:29.029 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:29.029 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.030 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.032 DEBUG:boto:Method: GET 2022-09-03T06:35:29.032 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.033 DEBUG:boto:Data: 2022-09-03T06:35:29.034 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.035 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:29.036 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.037 DEBUG:boto:Params: {} 2022-09-03T06:35:29.038 DEBUG:boto:Token: None 2022-09-03T06:35:29.039 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9JZUSEMx3gPDLyarsWkFuzwTfLQ= 2022-09-03T06:35:29.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9JZUSEMx3gPDLyarsWkFuzwTfLQ=', 'Content-Length': '0'} 2022-09-03T06:35:29.045 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a027e601363db0c1-006312f5b1-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.049 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:29.049 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:29.050 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.051 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.051 DEBUG:boto:Method: GET 2022-09-03T06:35:29.052 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.052 DEBUG:boto:Data: 2022-09-03T06:35:29.053 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.053 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.054 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.054 DEBUG:boto:Params: {} 2022-09-03T06:35:29.055 DEBUG:boto:Token: None 2022-09-03T06:35:29.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:29.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9JZUSEMx3gPDLyarsWkFuzwTfLQ= 2022-09-03T06:35:29.057 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9JZUSEMx3gPDLyarsWkFuzwTfLQ=', 'Content-Length': '0'} 2022-09-03T06:35:29.059 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004a760cb5eea22195-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.106 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.107 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.107 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:29.108 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:29.109 DEBUG:boto:Method: GET 2022-09-03T06:35:29.109 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:29.110 DEBUG:boto:Data: 2022-09-03T06:35:29.111 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.111 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:29.112 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.112 DEBUG:boto:Params: {} 2022-09-03T06:35:29.113 DEBUG:boto:Token: None 2022-09-03T06:35:29.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.117 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb3947358e0256e8-006312f5b1-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.118 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.119 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.120 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.120 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:29.121 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:29.121 DEBUG:boto:Method: GET 2022-09-03T06:35:29.122 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:29.122 DEBUG:boto:Data: 2022-09-03T06:35:29.123 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.124 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.124 DEBUG:boto:Params: {} 2022-09-03T06:35:29.125 DEBUG:boto:Token: None 2022-09-03T06:35:29.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.130 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f50aa3a146de8a7-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.130 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.131 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:29.132 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:29.132 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.133 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:29.134 DEBUG:boto:Data: 2022-09-03T06:35:29.134 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.135 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:29.135 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.136 DEBUG:boto:Params: {} 2022-09-03T06:35:29.136 DEBUG:boto:Token: None 2022-09-03T06:35:29.137 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj4 2022-09-03T06:35:29.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bHAM9lbP/CkZYfKTEdxtTW/l4VM= 2022-09-03T06:35:29.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bHAM9lbP/CkZYfKTEdxtTW/l4VM=', 'Content-Length': '0'} 2022-09-03T06:35:29.140 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000024107ed6ddad577-006312f5b1-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.141 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:29.142 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:29.142 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.143 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:29.143 DEBUG:boto:Data: 2022-09-03T06:35:29.144 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.145 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.146 DEBUG:boto:Params: {} 2022-09-03T06:35:29.146 DEBUG:boto:Token: None 2022-09-03T06:35:29.147 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj4 2022-09-03T06:35:29.147 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bHAM9lbP/CkZYfKTEdxtTW/l4VM= 2022-09-03T06:35:29.148 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bHAM9lbP/CkZYfKTEdxtTW/l4VM=', 'Content-Length': '0'} 2022-09-03T06:35:29.150 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000704119d1c30b73a-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.151 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:35:29.152 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:29.152 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:29.153 DEBUG:boto:Method: GET 2022-09-03T06:35:29.154 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:29.154 DEBUG:boto:Data: 2022-09-03T06:35:29.155 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.155 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:29.156 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.156 DEBUG:boto:Params: {} 2022-09-03T06:35:29.157 DEBUG:boto:Token: None 2022-09-03T06:35:29.158 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj4 2022-09-03T06:35:29.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tc6BwztksK/7qF1I69yenXOWlBA= 2022-09-03T06:35:29.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tc6BwztksK/7qF1I69yenXOWlBA=', 'Content-Length': '0'} 2022-09-03T06:35:29.161 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a279bd934335b956-006312f5b1-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.162 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:29.163 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:29.163 DEBUG:boto:Method: GET 2022-09-03T06:35:29.164 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:29.165 DEBUG:boto:Data: 2022-09-03T06:35:29.165 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.166 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.167 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.167 DEBUG:boto:Params: {} 2022-09-03T06:35:29.168 DEBUG:boto:Token: None 2022-09-03T06:35:29.168 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj4 2022-09-03T06:35:29.169 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tc6BwztksK/7qF1I69yenXOWlBA= 2022-09-03T06:35:29.170 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tc6BwztksK/7qF1I69yenXOWlBA=', 'Content-Length': '0'} 2022-09-03T06:35:29.172 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000312a597c62094937-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.173 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.173 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.174 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:29.175 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:29.175 DEBUG:boto:Method: GET 2022-09-03T06:35:29.176 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:29.176 DEBUG:boto:Data: 2022-09-03T06:35:29.177 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.177 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:29.178 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.179 DEBUG:boto:Params: {} 2022-09-03T06:35:29.179 DEBUG:boto:Token: None 2022-09-03T06:35:29.180 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.184 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c8f889b046503b81-006312f5b1-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.185 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.186 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.186 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.187 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:29.187 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:29.188 DEBUG:boto:Method: GET 2022-09-03T06:35:29.189 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:29.189 DEBUG:boto:Data: 2022-09-03T06:35:29.190 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.191 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.191 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.192 DEBUG:boto:Params: {} 2022-09-03T06:35:29.193 DEBUG:boto:Token: None 2022-09-03T06:35:29.193 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.194 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.198 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000723acd5c1c172e01-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.199 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.200 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-61 zones={test-zone2, test-zone1} 2022-09-03T06:35:29.201 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-61 zones={test-zone3, test-zone1} 2022-09-03T06:35:29.201 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.202 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.203 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.203 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:29.204 DEBUG:boto:Data: 2022-09-03T06:35:29.204 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.205 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.205 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.206 DEBUG:boto:Params: {} 2022-09-03T06:35:29.207 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:35:29.208 DEBUG:boto:Token: None 2022-09-03T06:35:29.208 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8acGRFNj1HiiWydD7pM1asQZh+I= 2022-09-03T06:35:29.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8acGRFNj1HiiWydD7pM1asQZh+I=', 'Content-Length': '0'} 2022-09-03T06:35:29.214 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000392d94aa5a40f6bb-006312f5b1-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.216 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.217 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.217 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.218 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:29.218 DEBUG:boto:Data: 2022-09-03T06:35:29.219 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.221 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.221 DEBUG:boto:Params: {} 2022-09-03T06:35:29.223 DEBUG:boto:Token: None 2022-09-03T06:35:29.224 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8acGRFNj1HiiWydD7pM1asQZh+I= 2022-09-03T06:35:29.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8acGRFNj1HiiWydD7pM1asQZh+I=', 'Content-Length': '0'} 2022-09-03T06:35:29.229 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d287195afd0a4dbd-006312f5b1-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.230 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:35:29.230 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.231 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.231 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.232 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.232 DEBUG:boto:Method: GET 2022-09-03T06:35:29.233 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.233 DEBUG:boto:Data: 2022-09-03T06:35:29.234 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.235 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.235 DEBUG:boto:Params: {} 2022-09-03T06:35:29.236 DEBUG:boto:Token: None 2022-09-03T06:35:29.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/?versions 2022-09-03T06:35:29.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QrgEpjWoGRnWAaMc54vLKPz9IHc= 2022-09-03T06:35:29.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QrgEpjWoGRnWAaMc54vLKPz9IHc=', 'Content-Length': '0'} 2022-09-03T06:35:29.241 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000642179ef93147cae-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.242 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.243 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:35:29.244 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:35:29.244 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:35:29.245 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:35:29.246 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:35:29.246 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.247 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.248 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.249 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.250 DEBUG:boto:Method: GET 2022-09-03T06:35:29.250 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.251 DEBUG:boto:Data: 2022-09-03T06:35:29.252 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.252 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.253 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.254 DEBUG:boto:Params: {} 2022-09-03T06:35:29.256 DEBUG:boto:Token: None 2022-09-03T06:35:29.257 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/?versions 2022-09-03T06:35:29.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QrgEpjWoGRnWAaMc54vLKPz9IHc= 2022-09-03T06:35:29.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QrgEpjWoGRnWAaMc54vLKPz9IHc=', 'Content-Length': '0'} 2022-09-03T06:35:29.263 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d306cfc17c4c405-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.264 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.266 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:35:29.267 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:35:29.268 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:35:29.268 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:35:29.269 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.269 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.270 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.271 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.272 DEBUG:boto:Method: GET 2022-09-03T06:35:29.273 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.274 DEBUG:boto:Data: 2022-09-03T06:35:29.275 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.276 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.277 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.278 DEBUG:boto:Params: {} 2022-09-03T06:35:29.278 DEBUG:boto:Token: None 2022-09-03T06:35:29.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/?versions 2022-09-03T06:35:29.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QrgEpjWoGRnWAaMc54vLKPz9IHc= 2022-09-03T06:35:29.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QrgEpjWoGRnWAaMc54vLKPz9IHc=', 'Content-Length': '0'} 2022-09-03T06:35:29.286 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004dc4fb6f479bd173-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.287 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.289 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.291 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.292 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.293 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.294 DEBUG:boto:Method: GET 2022-09-03T06:35:29.294 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:29.295 DEBUG:boto:Data: 2022-09-03T06:35:29.296 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.297 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.298 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.299 DEBUG:boto:Params: {} 2022-09-03T06:35:29.300 DEBUG:boto:Token: None 2022-09-03T06:35:29.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/?versions 2022-09-03T06:35:29.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QrgEpjWoGRnWAaMc54vLKPz9IHc= 2022-09-03T06:35:29.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QrgEpjWoGRnWAaMc54vLKPz9IHc=', 'Content-Length': '0'} 2022-09-03T06:35:29.306 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006eea4622719da77c-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.307 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.308 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:35:29.311 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.311 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.312 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.313 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.313 DEBUG:boto:Method: GET 2022-09-03T06:35:29.314 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.315 DEBUG:boto:Data: 2022-09-03T06:35:29.316 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.317 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.317 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.318 DEBUG:boto:Params: {} 2022-09-03T06:35:29.319 DEBUG:boto:Token: None 2022-09-03T06:35:29.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RuJPdE9iyJdhleUWI/d6nH8zWEc= 2022-09-03T06:35:29.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RuJPdE9iyJdhleUWI/d6nH8zWEc=', 'Content-Length': '0'} 2022-09-03T06:35:29.324 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ca3cb40f05b31944-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.325 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.325 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.326 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.327 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.327 DEBUG:boto:Method: GET 2022-09-03T06:35:29.328 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.328 DEBUG:boto:Data: 2022-09-03T06:35:29.329 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.330 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.331 DEBUG:boto:Params: {} 2022-09-03T06:35:29.331 DEBUG:boto:Token: None 2022-09-03T06:35:29.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RuJPdE9iyJdhleUWI/d6nH8zWEc= 2022-09-03T06:35:29.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RuJPdE9iyJdhleUWI/d6nH8zWEc=', 'Content-Length': '0'} 2022-09-03T06:35:29.336 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f296204271eb25b0-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.379 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.379 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.380 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.381 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.382 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.382 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.383 DEBUG:boto:Data: 2022-09-03T06:35:29.383 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.385 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.385 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.386 DEBUG:boto:Params: {} 2022-09-03T06:35:29.386 DEBUG:boto:Token: None 2022-09-03T06:35:29.388 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WJauky3JhvDGsPgUYiXoiVEEVrs= 2022-09-03T06:35:29.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WJauky3JhvDGsPgUYiXoiVEEVrs=', 'Content-Length': '0'} 2022-09-03T06:35:29.392 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009b4029da35f95373-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.394 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.395 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.396 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.398 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.399 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.399 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.400 DEBUG:boto:Data: 2022-09-03T06:35:29.401 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.403 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.404 DEBUG:boto:Params: {} 2022-09-03T06:35:29.404 DEBUG:boto:Token: None 2022-09-03T06:35:29.405 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WJauky3JhvDGsPgUYiXoiVEEVrs= 2022-09-03T06:35:29.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WJauky3JhvDGsPgUYiXoiVEEVrs=', 'Content-Length': '0'} 2022-09-03T06:35:29.410 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006742c8f106231ad3-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.411 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:35:29.412 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.412 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.413 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.413 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.414 DEBUG:boto:Method: GET 2022-09-03T06:35:29.415 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.415 DEBUG:boto:Data: 2022-09-03T06:35:29.416 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.417 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.417 DEBUG:boto:Params: {} 2022-09-03T06:35:29.418 DEBUG:boto:Token: None 2022-09-03T06:35:29.418 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RuJPdE9iyJdhleUWI/d6nH8zWEc= 2022-09-03T06:35:29.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RuJPdE9iyJdhleUWI/d6nH8zWEc=', 'Content-Length': '0'} 2022-09-03T06:35:29.422 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e3e213da8f86488b-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.423 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.425 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.426 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.426 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.427 DEBUG:boto:Method: GET 2022-09-03T06:35:29.428 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.429 DEBUG:boto:Data: 2022-09-03T06:35:29.430 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.431 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.432 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.432 DEBUG:boto:Params: {} 2022-09-03T06:35:29.433 DEBUG:boto:Token: None 2022-09-03T06:35:29.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:29.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RuJPdE9iyJdhleUWI/d6nH8zWEc= 2022-09-03T06:35:29.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RuJPdE9iyJdhleUWI/d6nH8zWEc=', 'Content-Length': '0'} 2022-09-03T06:35:29.438 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bbb1d73ac28e35c5-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.439 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.440 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.441 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:29.442 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:29.442 DEBUG:boto:Method: GET 2022-09-03T06:35:29.443 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:29.444 DEBUG:boto:Data: 2022-09-03T06:35:29.445 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.446 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.446 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.447 DEBUG:boto:Params: {} 2022-09-03T06:35:29.448 DEBUG:boto:Token: None 2022-09-03T06:35:29.449 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.450 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.454 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd84f229505d12ad-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.455 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.456 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.457 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.457 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:29.458 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:29.459 DEBUG:boto:Method: GET 2022-09-03T06:35:29.459 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:29.460 DEBUG:boto:Data: 2022-09-03T06:35:29.460 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.462 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.462 DEBUG:boto:Params: {} 2022-09-03T06:35:29.463 DEBUG:boto:Token: None 2022-09-03T06:35:29.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058ee295213fe4984-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.510 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.512 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.513 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.513 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.514 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:29.515 DEBUG:boto:Data: 2022-09-03T06:35:29.516 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.517 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.518 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.519 DEBUG:boto:Params: {} 2022-09-03T06:35:29.520 DEBUG:boto:Token: None 2022-09-03T06:35:29.521 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1 2022-09-03T06:35:29.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kIOVz5ki1hc4PwN80IimGOc/7gQ= 2022-09-03T06:35:29.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kIOVz5ki1hc4PwN80IimGOc/7gQ=', 'Content-Length': '0'} 2022-09-03T06:35:29.526 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002efa4b0628be7537-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.528 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.528 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.529 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.530 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:29.531 DEBUG:boto:Data: 2022-09-03T06:35:29.532 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.533 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.533 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.535 DEBUG:boto:Params: {} 2022-09-03T06:35:29.536 DEBUG:boto:Token: None 2022-09-03T06:35:29.537 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1 2022-09-03T06:35:29.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kIOVz5ki1hc4PwN80IimGOc/7gQ= 2022-09-03T06:35:29.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kIOVz5ki1hc4PwN80IimGOc/7gQ=', 'Content-Length': '0'} 2022-09-03T06:35:29.540 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d3d60fb7c5ef1cf7-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.541 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:35:29.542 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.543 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.544 DEBUG:boto:Method: GET 2022-09-03T06:35:29.544 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:29.545 DEBUG:boto:Data: 2022-09-03T06:35:29.546 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.546 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.547 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.548 DEBUG:boto:Params: {} 2022-09-03T06:35:29.548 DEBUG:boto:Token: None 2022-09-03T06:35:29.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1 2022-09-03T06:35:29.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OgxmT20i+AXw53l3P7yM+Z6i4sM= 2022-09-03T06:35:29.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OgxmT20i+AXw53l3P7yM+Z6i4sM=', 'Content-Length': '0'} 2022-09-03T06:35:29.553 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002db6ea07d0a27080-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.605 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:29.606 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:29.606 DEBUG:boto:Method: GET 2022-09-03T06:35:29.607 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:29.608 DEBUG:boto:Data: 2022-09-03T06:35:29.608 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.609 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.609 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.610 DEBUG:boto:Params: {} 2022-09-03T06:35:29.610 DEBUG:boto:Token: None 2022-09-03T06:35:29.611 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj1 2022-09-03T06:35:29.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OgxmT20i+AXw53l3P7yM+Z6i4sM= 2022-09-03T06:35:29.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OgxmT20i+AXw53l3P7yM+Z6i4sM=', 'Content-Length': '0'} 2022-09-03T06:35:29.615 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000420aef94afc388c3-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.617 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.617 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.618 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:29.619 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:29.619 DEBUG:boto:Method: GET 2022-09-03T06:35:29.620 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:29.621 DEBUG:boto:Data: 2022-09-03T06:35:29.621 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.622 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.623 DEBUG:boto:Params: {} 2022-09-03T06:35:29.624 DEBUG:boto:Token: None 2022-09-03T06:35:29.624 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.629 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003599778f868bb841-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.631 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.632 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.633 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.634 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:29.635 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:29.636 DEBUG:boto:Method: GET 2022-09-03T06:35:29.638 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:29.639 DEBUG:boto:Data: 2022-09-03T06:35:29.640 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.641 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.642 DEBUG:boto:Params: {} 2022-09-03T06:35:29.642 DEBUG:boto:Token: None 2022-09-03T06:35:29.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.647 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b123b5667a32a64-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.648 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.649 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:35:29.649 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.650 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.651 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.651 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.655 DEBUG:boto:Method: GET 2022-09-03T06:35:29.656 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.656 DEBUG:boto:Data: 2022-09-03T06:35:29.657 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.658 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.659 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.659 DEBUG:boto:Params: {} 2022-09-03T06:35:29.660 DEBUG:boto:Token: None 2022-09-03T06:35:29.661 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ugcuJ6BdBd/qHnO7TxLiLxUyBJ4= 2022-09-03T06:35:29.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ugcuJ6BdBd/qHnO7TxLiLxUyBJ4=', 'Content-Length': '0'} 2022-09-03T06:35:29.666 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a93dcd1b8bd5173c-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.667 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.667 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.668 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.668 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.669 DEBUG:boto:Method: GET 2022-09-03T06:35:29.670 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.670 DEBUG:boto:Data: 2022-09-03T06:35:29.671 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.673 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.674 DEBUG:boto:Params: {} 2022-09-03T06:35:29.675 DEBUG:boto:Token: None 2022-09-03T06:35:29.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ugcuJ6BdBd/qHnO7TxLiLxUyBJ4= 2022-09-03T06:35:29.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ugcuJ6BdBd/qHnO7TxLiLxUyBJ4=', 'Content-Length': '0'} 2022-09-03T06:35:29.679 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006d74cb4b67cc1ed7-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.726 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.727 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.728 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.728 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.729 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.729 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.730 DEBUG:boto:Data: 2022-09-03T06:35:29.731 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.732 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.733 DEBUG:boto:Params: {} 2022-09-03T06:35:29.734 DEBUG:boto:Token: None 2022-09-03T06:35:29.734 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wVaQ03IApNLZbGfs4Bx/0wREad4= 2022-09-03T06:35:29.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wVaQ03IApNLZbGfs4Bx/0wREad4=', 'Content-Length': '0'} 2022-09-03T06:35:29.740 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000a59f93c088c0b9c-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.743 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.744 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.744 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.745 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.745 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.746 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.746 DEBUG:boto:Data: 2022-09-03T06:35:29.747 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.748 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.748 DEBUG:boto:Params: {} 2022-09-03T06:35:29.749 DEBUG:boto:Token: None 2022-09-03T06:35:29.750 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wVaQ03IApNLZbGfs4Bx/0wREad4= 2022-09-03T06:35:29.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wVaQ03IApNLZbGfs4Bx/0wREad4=', 'Content-Length': '0'} 2022-09-03T06:35:29.754 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002d57e6591648d288-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.754 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:35:29.755 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.755 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.756 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.756 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.757 DEBUG:boto:Method: GET 2022-09-03T06:35:29.758 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.758 DEBUG:boto:Data: 2022-09-03T06:35:29.759 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.760 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.760 DEBUG:boto:Params: {} 2022-09-03T06:35:29.761 DEBUG:boto:Token: None 2022-09-03T06:35:29.762 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ugcuJ6BdBd/qHnO7TxLiLxUyBJ4= 2022-09-03T06:35:29.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ugcuJ6BdBd/qHnO7TxLiLxUyBJ4=', 'Content-Length': '0'} 2022-09-03T06:35:29.765 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009e8855321703e641-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.766 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.766 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.767 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.767 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.768 DEBUG:boto:Method: GET 2022-09-03T06:35:29.769 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.769 DEBUG:boto:Data: 2022-09-03T06:35:29.770 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.771 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.771 DEBUG:boto:Params: {} 2022-09-03T06:35:29.772 DEBUG:boto:Token: None 2022-09-03T06:35:29.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:29.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ugcuJ6BdBd/qHnO7TxLiLxUyBJ4= 2022-09-03T06:35:29.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ugcuJ6BdBd/qHnO7TxLiLxUyBJ4=', 'Content-Length': '0'} 2022-09-03T06:35:29.776 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000008c85e99ea543feff-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.777 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.778 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.779 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:29.780 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:29.780 DEBUG:boto:Method: GET 2022-09-03T06:35:29.781 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:29.781 DEBUG:boto:Data: 2022-09-03T06:35:29.782 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.782 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.783 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.784 DEBUG:boto:Params: {} 2022-09-03T06:35:29.784 DEBUG:boto:Token: None 2022-09-03T06:35:29.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.790 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000092e1c28e7bd38ae5-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.791 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.792 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.792 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.793 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:29.793 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:29.794 DEBUG:boto:Method: GET 2022-09-03T06:35:29.795 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:29.795 DEBUG:boto:Data: 2022-09-03T06:35:29.796 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.797 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.797 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.798 DEBUG:boto:Params: {} 2022-09-03T06:35:29.799 DEBUG:boto:Token: None 2022-09-03T06:35:29.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.804 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c76e386b0cbe6cfb-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.804 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.805 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.806 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.807 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.807 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:29.808 DEBUG:boto:Data: 2022-09-03T06:35:29.808 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.809 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.810 DEBUG:boto:Params: {} 2022-09-03T06:35:29.811 DEBUG:boto:Token: None 2022-09-03T06:35:29.812 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2 2022-09-03T06:35:29.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bv7+KcVpuj05Ups1pMzjkVBvQt8= 2022-09-03T06:35:29.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bv7+KcVpuj05Ups1pMzjkVBvQt8=', 'Content-Length': '0'} 2022-09-03T06:35:29.815 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bbfef2f64e092cc7-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.816 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.817 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.817 DEBUG:boto:Method: HEAD 2022-09-03T06:35:29.818 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:29.819 DEBUG:boto:Data: 2022-09-03T06:35:29.819 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.820 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.821 DEBUG:boto:Params: {} 2022-09-03T06:35:29.821 DEBUG:boto:Token: None 2022-09-03T06:35:29.822 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2 2022-09-03T06:35:29.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bv7+KcVpuj05Ups1pMzjkVBvQt8= 2022-09-03T06:35:29.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bv7+KcVpuj05Ups1pMzjkVBvQt8=', 'Content-Length': '0'} 2022-09-03T06:35:29.826 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b2c1fc40f02af256-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.827 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:35:29.827 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.828 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.828 DEBUG:boto:Method: GET 2022-09-03T06:35:29.829 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:29.830 DEBUG:boto:Data: 2022-09-03T06:35:29.830 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.831 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.832 DEBUG:boto:Params: {} 2022-09-03T06:35:29.833 DEBUG:boto:Token: None 2022-09-03T06:35:29.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2 2022-09-03T06:35:29.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KnbhCvf1fnSmhPbRFqnaqEdCR40= 2022-09-03T06:35:29.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KnbhCvf1fnSmhPbRFqnaqEdCR40=', 'Content-Length': '0'} 2022-09-03T06:35:29.837 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c7037e25d09303f6-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.880 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:29.881 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:29.882 DEBUG:boto:Method: GET 2022-09-03T06:35:29.883 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:29.884 DEBUG:boto:Data: 2022-09-03T06:35:29.885 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.887 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.888 DEBUG:boto:Params: {} 2022-09-03T06:35:29.889 DEBUG:boto:Token: None 2022-09-03T06:35:29.890 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj2 2022-09-03T06:35:29.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KnbhCvf1fnSmhPbRFqnaqEdCR40= 2022-09-03T06:35:29.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KnbhCvf1fnSmhPbRFqnaqEdCR40=', 'Content-Length': '0'} 2022-09-03T06:35:29.897 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000947a0c5d5f9dcf5e-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.899 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.900 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.901 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:29.902 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:29.903 DEBUG:boto:Method: GET 2022-09-03T06:35:29.904 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:29.904 DEBUG:boto:Data: 2022-09-03T06:35:29.905 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.906 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.906 DEBUG:boto:Params: {} 2022-09-03T06:35:29.907 DEBUG:boto:Token: None 2022-09-03T06:35:29.908 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.914 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fdde0f8904276f23-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.915 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.916 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:29.917 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:29.918 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:29.918 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:29.919 DEBUG:boto:Method: GET 2022-09-03T06:35:29.920 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:29.921 DEBUG:boto:Data: 2022-09-03T06:35:29.922 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.923 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.923 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.924 DEBUG:boto:Params: {} 2022-09-03T06:35:29.925 DEBUG:boto:Token: None 2022-09-03T06:35:29.926 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/ 2022-09-03T06:35:29.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY= 2022-09-03T06:35:29.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:598OQKLDxbSpC48gDrRVcytTrzY=', 'Content-Length': '0'} 2022-09-03T06:35:29.933 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009af1f39519985431-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.935 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:29.936 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:35:29.938 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:29.939 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:29.941 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:29.942 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:29.943 DEBUG:boto:Method: GET 2022-09-03T06:35:29.943 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:29.944 DEBUG:boto:Data: 2022-09-03T06:35:29.944 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.945 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:29.946 DEBUG:boto:Port: 8001 2022-09-03T06:35:29.947 DEBUG:boto:Params: {} 2022-09-03T06:35:29.948 DEBUG:boto:Token: None 2022-09-03T06:35:29.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:29.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5VHIG0taCTLKdAdcfEsH5kPHmnM= 2022-09-03T06:35:29.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5VHIG0taCTLKdAdcfEsH5kPHmnM=', 'Content-Length': '0'} 2022-09-03T06:35:29.954 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000116670b273483e98-006312f5b1-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:29.955 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:29.956 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:29.956 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:29.957 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:29.958 DEBUG:boto:Method: GET 2022-09-03T06:35:29.958 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:29.959 DEBUG:boto:Data: 2022-09-03T06:35:29.959 DEBUG:boto:Headers: {} 2022-09-03T06:35:29.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:29.961 DEBUG:boto:Port: 8000 2022-09-03T06:35:29.961 DEBUG:boto:Params: {} 2022-09-03T06:35:29.962 DEBUG:boto:Token: None 2022-09-03T06:35:29.963 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:29 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:29.964 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5VHIG0taCTLKdAdcfEsH5kPHmnM= 2022-09-03T06:35:29.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5VHIG0taCTLKdAdcfEsH5kPHmnM=', 'Content-Length': '0'} 2022-09-03T06:35:29.967 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003d567e74fdb72f08-006312f5b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.011 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:30.012 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:30.012 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.013 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.014 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.014 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.015 DEBUG:boto:Data: 2022-09-03T06:35:30.016 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.016 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.017 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.018 DEBUG:boto:Params: {} 2022-09-03T06:35:30.019 DEBUG:boto:Token: None 2022-09-03T06:35:30.020 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tfp3qrGCCIIIyjrwwMO5YtzPeBI= 2022-09-03T06:35:30.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tfp3qrGCCIIIyjrwwMO5YtzPeBI=', 'Content-Length': '0'} 2022-09-03T06:35:30.026 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000091389d3d72d64cc7-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.028 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:30.029 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:30.029 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.030 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.030 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.031 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.031 DEBUG:boto:Data: 2022-09-03T06:35:30.032 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.033 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.034 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.035 DEBUG:boto:Params: {} 2022-09-03T06:35:30.036 DEBUG:boto:Token: None 2022-09-03T06:35:30.037 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tfp3qrGCCIIIyjrwwMO5YtzPeBI= 2022-09-03T06:35:30.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tfp3qrGCCIIIyjrwwMO5YtzPeBI=', 'Content-Length': '0'} 2022-09-03T06:35:30.044 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009535496f65fdad52-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.045 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:35:30.045 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:30.046 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:30.047 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.047 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.048 DEBUG:boto:Method: GET 2022-09-03T06:35:30.048 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.049 DEBUG:boto:Data: 2022-09-03T06:35:30.049 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.051 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.051 DEBUG:boto:Params: {} 2022-09-03T06:35:30.052 DEBUG:boto:Token: None 2022-09-03T06:35:30.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nU+23HFh2+K216zy7kLnOh4mTkQ= 2022-09-03T06:35:30.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nU+23HFh2+K216zy7kLnOh4mTkQ=', 'Content-Length': '0'} 2022-09-03T06:35:30.056 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dfcbf3288e65bae6-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.057 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:30.058 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:30.058 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.059 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.060 DEBUG:boto:Method: GET 2022-09-03T06:35:30.060 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.061 DEBUG:boto:Data: 2022-09-03T06:35:30.062 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.063 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.063 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.064 DEBUG:boto:Params: {} 2022-09-03T06:35:30.065 DEBUG:boto:Token: None 2022-09-03T06:35:30.066 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:30.067 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nU+23HFh2+K216zy7kLnOh4mTkQ= 2022-09-03T06:35:30.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nU+23HFh2+K216zy7kLnOh4mTkQ=', 'Content-Length': '0'} 2022-09-03T06:35:30.069 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d3e8fc4cfa77bf1c-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.070 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.071 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.072 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:30.072 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:30.073 DEBUG:boto:Method: GET 2022-09-03T06:35:30.073 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:30.074 DEBUG:boto:Data: 2022-09-03T06:35:30.075 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.077 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.078 DEBUG:boto:Params: {} 2022-09-03T06:35:30.079 DEBUG:boto:Token: None 2022-09-03T06:35:30.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.085 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd7f55d08fbbe30b-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.086 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.088 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.089 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.089 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:30.090 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:30.091 DEBUG:boto:Method: GET 2022-09-03T06:35:30.092 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:30.093 DEBUG:boto:Data: 2022-09-03T06:35:30.094 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.096 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.096 DEBUG:boto:Params: {} 2022-09-03T06:35:30.097 DEBUG:boto:Token: None 2022-09-03T06:35:30.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.109 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000054e19df7fd12032-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.111 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.112 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:30.113 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:30.113 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.114 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:30.114 DEBUG:boto:Data: 2022-09-03T06:35:30.115 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.116 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.117 DEBUG:boto:Params: {} 2022-09-03T06:35:30.117 DEBUG:boto:Token: None 2022-09-03T06:35:30.118 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj3 2022-09-03T06:35:30.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MUqiff9yt0wSWxZsX2hOihtwqb4= 2022-09-03T06:35:30.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MUqiff9yt0wSWxZsX2hOihtwqb4=', 'Content-Length': '0'} 2022-09-03T06:35:30.123 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009e730bd1ac504ec6-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.124 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:30.125 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:30.126 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.126 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:30.127 DEBUG:boto:Data: 2022-09-03T06:35:30.128 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.129 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.129 DEBUG:boto:Params: {} 2022-09-03T06:35:30.130 DEBUG:boto:Token: None 2022-09-03T06:35:30.131 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj3 2022-09-03T06:35:30.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MUqiff9yt0wSWxZsX2hOihtwqb4= 2022-09-03T06:35:30.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MUqiff9yt0wSWxZsX2hOihtwqb4=', 'Content-Length': '0'} 2022-09-03T06:35:30.134 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000065029960873e673f-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.136 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:35:30.136 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:30.138 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:30.138 DEBUG:boto:Method: GET 2022-09-03T06:35:30.139 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:30.140 DEBUG:boto:Data: 2022-09-03T06:35:30.141 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.141 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.142 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.143 DEBUG:boto:Params: {} 2022-09-03T06:35:30.144 DEBUG:boto:Token: None 2022-09-03T06:35:30.144 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj3 2022-09-03T06:35:30.145 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HY6tpbEn07tO69834cuuJdF3SMw= 2022-09-03T06:35:30.146 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HY6tpbEn07tO69834cuuJdF3SMw=', 'Content-Length': '0'} 2022-09-03T06:35:30.149 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000006fe4d9caf39b90b2-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.151 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:30.152 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:30.153 DEBUG:boto:Method: GET 2022-09-03T06:35:30.153 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:30.154 DEBUG:boto:Data: 2022-09-03T06:35:30.155 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.157 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.158 DEBUG:boto:Params: {} 2022-09-03T06:35:30.159 DEBUG:boto:Token: None 2022-09-03T06:35:30.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj3 2022-09-03T06:35:30.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HY6tpbEn07tO69834cuuJdF3SMw= 2022-09-03T06:35:30.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HY6tpbEn07tO69834cuuJdF3SMw=', 'Content-Length': '0'} 2022-09-03T06:35:30.166 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ba7369c2747de629-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.210 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.212 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.213 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:30.214 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:30.215 DEBUG:boto:Method: GET 2022-09-03T06:35:30.216 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:30.217 DEBUG:boto:Data: 2022-09-03T06:35:30.219 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.221 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.222 DEBUG:boto:Params: {} 2022-09-03T06:35:30.223 DEBUG:boto:Token: None 2022-09-03T06:35:30.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.231 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000106bc8e0e6f38688-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.232 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.233 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.234 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.234 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:30.235 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:30.236 DEBUG:boto:Method: GET 2022-09-03T06:35:30.236 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:30.237 DEBUG:boto:Data: 2022-09-03T06:35:30.237 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.239 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.239 DEBUG:boto:Params: {} 2022-09-03T06:35:30.241 DEBUG:boto:Token: None 2022-09-03T06:35:30.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.248 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000043103dec3262f59a-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.248 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.249 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:35:30.250 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.250 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.251 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.252 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.253 DEBUG:boto:Method: GET 2022-09-03T06:35:30.254 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.255 DEBUG:boto:Data: 2022-09-03T06:35:30.255 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.256 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.257 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.259 DEBUG:boto:Params: {} 2022-09-03T06:35:30.260 DEBUG:boto:Token: None 2022-09-03T06:35:30.261 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q4LhwBuuFPVcQ/iu9MgqM4ITbaY= 2022-09-03T06:35:30.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q4LhwBuuFPVcQ/iu9MgqM4ITbaY=', 'Content-Length': '0'} 2022-09-03T06:35:30.266 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d3b438b5eb20f040-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.306 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.307 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.308 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.309 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.310 DEBUG:boto:Method: GET 2022-09-03T06:35:30.311 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.312 DEBUG:boto:Data: 2022-09-03T06:35:30.314 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.315 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.316 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.317 DEBUG:boto:Params: {} 2022-09-03T06:35:30.318 DEBUG:boto:Token: None 2022-09-03T06:35:30.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q4LhwBuuFPVcQ/iu9MgqM4ITbaY= 2022-09-03T06:35:30.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q4LhwBuuFPVcQ/iu9MgqM4ITbaY=', 'Content-Length': '0'} 2022-09-03T06:35:30.334 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000027ba3f124a1d1d3f-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.336 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.337 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.337 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.338 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.338 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.339 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.339 DEBUG:boto:Data: 2022-09-03T06:35:30.340 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.341 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.341 DEBUG:boto:Params: {} 2022-09-03T06:35:30.342 DEBUG:boto:Token: None 2022-09-03T06:35:30.343 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.343 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GkdlLAbJiVcgPxPOVNYfotcXTAA= 2022-09-03T06:35:30.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GkdlLAbJiVcgPxPOVNYfotcXTAA=', 'Content-Length': '0'} 2022-09-03T06:35:30.348 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ab10f00f1dbfcdfe-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.349 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.350 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.352 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.353 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.354 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.355 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.358 DEBUG:boto:Data: 2022-09-03T06:35:30.359 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.361 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.362 DEBUG:boto:Params: {} 2022-09-03T06:35:30.363 DEBUG:boto:Token: None 2022-09-03T06:35:30.364 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GkdlLAbJiVcgPxPOVNYfotcXTAA= 2022-09-03T06:35:30.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GkdlLAbJiVcgPxPOVNYfotcXTAA=', 'Content-Length': '0'} 2022-09-03T06:35:30.372 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000efae54d17dcf810b-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.373 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:35:30.374 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.379 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.380 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.381 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.382 DEBUG:boto:Method: GET 2022-09-03T06:35:30.383 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.384 DEBUG:boto:Data: 2022-09-03T06:35:30.385 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.387 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.388 DEBUG:boto:Params: {} 2022-09-03T06:35:30.388 DEBUG:boto:Token: None 2022-09-03T06:35:30.389 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q4LhwBuuFPVcQ/iu9MgqM4ITbaY= 2022-09-03T06:35:30.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q4LhwBuuFPVcQ/iu9MgqM4ITbaY=', 'Content-Length': '0'} 2022-09-03T06:35:30.393 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000184f14447e3b43e2-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.394 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.395 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.395 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.396 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.396 DEBUG:boto:Method: GET 2022-09-03T06:35:30.397 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.397 DEBUG:boto:Data: 2022-09-03T06:35:30.398 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.398 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.399 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.399 DEBUG:boto:Params: {} 2022-09-03T06:35:30.400 DEBUG:boto:Token: None 2022-09-03T06:35:30.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:30.401 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q4LhwBuuFPVcQ/iu9MgqM4ITbaY= 2022-09-03T06:35:30.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q4LhwBuuFPVcQ/iu9MgqM4ITbaY=', 'Content-Length': '0'} 2022-09-03T06:35:30.404 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000050bf432836a83afb-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.452 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.453 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.454 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:30.455 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:30.456 DEBUG:boto:Method: GET 2022-09-03T06:35:30.457 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:30.458 DEBUG:boto:Data: 2022-09-03T06:35:30.459 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.462 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.463 DEBUG:boto:Params: {} 2022-09-03T06:35:30.464 DEBUG:boto:Token: None 2022-09-03T06:35:30.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.472 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001b173f59ba23bb74-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.473 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.475 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.475 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.476 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:30.476 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:30.477 DEBUG:boto:Method: GET 2022-09-03T06:35:30.477 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:30.478 DEBUG:boto:Data: 2022-09-03T06:35:30.478 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.479 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.480 DEBUG:boto:Params: {} 2022-09-03T06:35:30.480 DEBUG:boto:Token: None 2022-09-03T06:35:30.481 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.487 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000451b18b2577986ee-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.487 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.489 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.490 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.491 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.491 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:30.492 DEBUG:boto:Data: 2022-09-03T06:35:30.492 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.494 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.495 DEBUG:boto:Params: {} 2022-09-03T06:35:30.496 DEBUG:boto:Token: None 2022-09-03T06:35:30.497 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4 2022-09-03T06:35:30.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:90y1lvg4T5UjwXVFJ3enp+iv8go= 2022-09-03T06:35:30.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:90y1lvg4T5UjwXVFJ3enp+iv8go=', 'Content-Length': '0'} 2022-09-03T06:35:30.501 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b79fe65fe71e814a-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.503 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.504 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.505 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.505 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:30.506 DEBUG:boto:Data: 2022-09-03T06:35:30.507 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.508 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.509 DEBUG:boto:Params: {} 2022-09-03T06:35:30.510 DEBUG:boto:Token: None 2022-09-03T06:35:30.510 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4 2022-09-03T06:35:30.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:90y1lvg4T5UjwXVFJ3enp+iv8go= 2022-09-03T06:35:30.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:90y1lvg4T5UjwXVFJ3enp+iv8go=', 'Content-Length': '0'} 2022-09-03T06:35:30.514 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001c2fde0124714f7f-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.515 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:35:30.515 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.517 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.517 DEBUG:boto:Method: GET 2022-09-03T06:35:30.519 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:30.519 DEBUG:boto:Data: 2022-09-03T06:35:30.520 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.521 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.522 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.523 DEBUG:boto:Params: {} 2022-09-03T06:35:30.525 DEBUG:boto:Token: None 2022-09-03T06:35:30.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4 2022-09-03T06:35:30.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H8mrppEf9/Hd/Sn5jJ/N8oCjqzE= 2022-09-03T06:35:30.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H8mrppEf9/Hd/Sn5jJ/N8oCjqzE=', 'Content-Length': '0'} 2022-09-03T06:35:30.530 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000263c96570e72a4aa-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.531 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:30.532 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:30.533 DEBUG:boto:Method: GET 2022-09-03T06:35:30.534 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:30.535 DEBUG:boto:Data: 2022-09-03T06:35:30.536 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.538 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.539 DEBUG:boto:Params: {} 2022-09-03T06:35:30.539 DEBUG:boto:Token: None 2022-09-03T06:35:30.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj4 2022-09-03T06:35:30.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H8mrppEf9/Hd/Sn5jJ/N8oCjqzE= 2022-09-03T06:35:30.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H8mrppEf9/Hd/Sn5jJ/N8oCjqzE=', 'Content-Length': '0'} 2022-09-03T06:35:30.546 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000319fb26a9076c87b-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.547 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.548 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.549 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:30.550 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:30.551 DEBUG:boto:Method: GET 2022-09-03T06:35:30.552 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:30.553 DEBUG:boto:Data: 2022-09-03T06:35:30.554 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.556 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.557 DEBUG:boto:Params: {} 2022-09-03T06:35:30.558 DEBUG:boto:Token: None 2022-09-03T06:35:30.559 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.559 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.560 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.566 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c6a58dea237d1fe3-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.569 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.571 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.572 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.573 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:30.574 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:30.575 DEBUG:boto:Method: GET 2022-09-03T06:35:30.576 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:30.577 DEBUG:boto:Data: 2022-09-03T06:35:30.577 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.578 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:30.580 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.581 DEBUG:boto:Params: {} 2022-09-03T06:35:30.582 DEBUG:boto:Token: None 2022-09-03T06:35:30.583 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.584 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.590 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008462442a0d4b8579-006312f5b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.591 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.593 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-61 zones={test-zone3, test-zone1} 2022-09-03T06:35:30.593 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-61 zones={test-zone3, test-zone2} 2022-09-03T06:35:30.594 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.595 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.595 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.596 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:30.596 DEBUG:boto:Data: 2022-09-03T06:35:30.597 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.598 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.599 DEBUG:boto:Params: {} 2022-09-03T06:35:30.600 DEBUG:boto:Token: None 2022-09-03T06:35:30.601 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0GRi7DdMfNdNpncBSj2qKeiDLGo= 2022-09-03T06:35:30.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0GRi7DdMfNdNpncBSj2qKeiDLGo=', 'Content-Length': '0'} 2022-09-03T06:35:30.615 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003db2450b0edd9a68-006312f5b2-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.616 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.617 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.617 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.618 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:30.619 DEBUG:boto:Data: 2022-09-03T06:35:30.619 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.620 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.620 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.621 DEBUG:boto:Params: {} 2022-09-03T06:35:30.621 DEBUG:boto:Token: None 2022-09-03T06:35:30.622 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0GRi7DdMfNdNpncBSj2qKeiDLGo= 2022-09-03T06:35:30.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0GRi7DdMfNdNpncBSj2qKeiDLGo=', 'Content-Length': '0'} 2022-09-03T06:35:30.627 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004870803d02259988-006312f5b2-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.629 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:35:30.629 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.630 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.631 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.632 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.633 DEBUG:boto:Method: GET 2022-09-03T06:35:30.633 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.634 DEBUG:boto:Data: 2022-09-03T06:35:30.635 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.636 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.637 DEBUG:boto:Params: {} 2022-09-03T06:35:30.638 DEBUG:boto:Token: None 2022-09-03T06:35:30.638 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/?versions 2022-09-03T06:35:30.639 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l33mXVFuwA34KfNSAIy/pb2RwFc= 2022-09-03T06:35:30.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l33mXVFuwA34KfNSAIy/pb2RwFc=', 'Content-Length': '0'} 2022-09-03T06:35:30.643 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007e060088000388d2-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.644 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.645 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:35:30.646 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:35:30.647 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:35:30.647 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:35:30.648 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:35:30.648 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.649 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.650 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.659 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.660 DEBUG:boto:Method: GET 2022-09-03T06:35:30.661 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.662 DEBUG:boto:Data: 2022-09-03T06:35:30.664 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.664 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.666 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.667 DEBUG:boto:Params: {} 2022-09-03T06:35:30.668 DEBUG:boto:Token: None 2022-09-03T06:35:30.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/?versions 2022-09-03T06:35:30.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l33mXVFuwA34KfNSAIy/pb2RwFc= 2022-09-03T06:35:30.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l33mXVFuwA34KfNSAIy/pb2RwFc=', 'Content-Length': '0'} 2022-09-03T06:35:30.674 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000060e189ba6aada961-006312f5b2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.678 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.679 DEBUG:tasks.rgw_multi.zone_rados:o=obj1 2022-09-03T06:35:30.680 DEBUG:tasks.rgw_multi.zone_rados:o=obj2 2022-09-03T06:35:30.681 DEBUG:tasks.rgw_multi.zone_rados:o=obj3 2022-09-03T06:35:30.682 DEBUG:tasks.rgw_multi.zone_rados:o=obj4 2022-09-03T06:35:30.683 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.684 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.685 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.686 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.687 DEBUG:boto:Method: GET 2022-09-03T06:35:30.688 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.689 DEBUG:boto:Data: 2022-09-03T06:35:30.690 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.692 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.693 DEBUG:boto:Params: {} 2022-09-03T06:35:30.695 DEBUG:boto:Token: None 2022-09-03T06:35:30.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/?versions 2022-09-03T06:35:30.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l33mXVFuwA34KfNSAIy/pb2RwFc= 2022-09-03T06:35:30.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l33mXVFuwA34KfNSAIy/pb2RwFc=', 'Content-Length': '0'} 2022-09-03T06:35:30.702 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000920e2bbff87e46db-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.703 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.705 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.705 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.707 DEBUG:boto:path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.708 DEBUG:boto:auth_path=/avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.709 DEBUG:boto:Method: GET 2022-09-03T06:35:30.710 DEBUG:boto:Path: /avysmw-61/?versions&max-keys=999 2022-09-03T06:35:30.710 DEBUG:boto:Data: 2022-09-03T06:35:30.711 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.712 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.713 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.714 DEBUG:boto:Params: {} 2022-09-03T06:35:30.716 DEBUG:boto:Token: None 2022-09-03T06:35:30.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/?versions 2022-09-03T06:35:30.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l33mXVFuwA34KfNSAIy/pb2RwFc= 2022-09-03T06:35:30.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l33mXVFuwA34KfNSAIy/pb2RwFc=', 'Content-Length': '0'} 2022-09-03T06:35:30.721 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000314ee2321e8dae53-006312f5b2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.722 DEBUG:boto:b'avysmw-61999falseobj1nulltrue2022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj2nulltrue2022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj3nulltrue2022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormalobj4nulltrue2022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.723 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:35:30.724 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.724 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.725 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.726 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.727 DEBUG:boto:Method: GET 2022-09-03T06:35:30.727 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.728 DEBUG:boto:Data: 2022-09-03T06:35:30.728 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.729 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.730 DEBUG:boto:Params: {} 2022-09-03T06:35:30.731 DEBUG:boto:Token: None 2022-09-03T06:35:30.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I6wVlC4m/UhBW4nXM2irNJLGCRs= 2022-09-03T06:35:30.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I6wVlC4m/UhBW4nXM2irNJLGCRs=', 'Content-Length': '0'} 2022-09-03T06:35:30.736 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000046c704bf4cf66b67-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.737 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.738 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.738 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.739 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.740 DEBUG:boto:Method: GET 2022-09-03T06:35:30.741 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.741 DEBUG:boto:Data: 2022-09-03T06:35:30.742 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.742 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.743 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.744 DEBUG:boto:Params: {} 2022-09-03T06:35:30.745 DEBUG:boto:Token: None 2022-09-03T06:35:30.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I6wVlC4m/UhBW4nXM2irNJLGCRs= 2022-09-03T06:35:30.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I6wVlC4m/UhBW4nXM2irNJLGCRs=', 'Content-Length': '0'} 2022-09-03T06:35:30.749 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000e5abf1f4e087cbe4-006312f5b2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.750 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.750 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.751 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.752 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.752 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.753 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.753 DEBUG:boto:Data: 2022-09-03T06:35:30.754 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.755 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.756 DEBUG:boto:Params: {} 2022-09-03T06:35:30.756 DEBUG:boto:Token: None 2022-09-03T06:35:30.757 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fXdDweZDoxsMZh/CixQsFvMSmz0= 2022-09-03T06:35:30.758 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fXdDweZDoxsMZh/CixQsFvMSmz0=', 'Content-Length': '0'} 2022-09-03T06:35:30.760 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002f7b965977a9f65f-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.761 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.762 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.763 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.764 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.765 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.766 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.767 DEBUG:boto:Data: 2022-09-03T06:35:30.767 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.768 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.769 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.770 DEBUG:boto:Params: {} 2022-09-03T06:35:30.771 DEBUG:boto:Token: None 2022-09-03T06:35:30.772 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fXdDweZDoxsMZh/CixQsFvMSmz0= 2022-09-03T06:35:30.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fXdDweZDoxsMZh/CixQsFvMSmz0=', 'Content-Length': '0'} 2022-09-03T06:35:30.777 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d87f0f5e999b3b5f-006312f5b2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.778 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:35:30.778 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.779 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.780 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.780 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.781 DEBUG:boto:Method: GET 2022-09-03T06:35:30.781 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.782 DEBUG:boto:Data: 2022-09-03T06:35:30.783 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.784 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.784 DEBUG:boto:Params: {} 2022-09-03T06:35:30.785 DEBUG:boto:Token: None 2022-09-03T06:35:30.786 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I6wVlC4m/UhBW4nXM2irNJLGCRs= 2022-09-03T06:35:30.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I6wVlC4m/UhBW4nXM2irNJLGCRs=', 'Content-Length': '0'} 2022-09-03T06:35:30.789 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f3e4a008fdfea9e8-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.790 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.792 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.792 DEBUG:boto:path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.793 DEBUG:boto:auth_path=/avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.793 DEBUG:boto:Method: GET 2022-09-03T06:35:30.794 DEBUG:boto:Path: /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.795 DEBUG:boto:Data: 2022-09-03T06:35:30.795 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.796 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.796 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.797 DEBUG:boto:Params: {} 2022-09-03T06:35:30.798 DEBUG:boto:Token: None 2022-09-03T06:35:30.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1?versionId=null 2022-09-03T06:35:30.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I6wVlC4m/UhBW4nXM2irNJLGCRs= 2022-09-03T06:35:30.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I6wVlC4m/UhBW4nXM2irNJLGCRs=', 'Content-Length': '0'} 2022-09-03T06:35:30.803 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000066c6c2d96b2b5109-006312f5b2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.804 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.805 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.805 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:30.806 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:30.806 DEBUG:boto:Method: GET 2022-09-03T06:35:30.809 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:30.809 DEBUG:boto:Data: 2022-09-03T06:35:30.810 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.811 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.812 DEBUG:boto:Params: {} 2022-09-03T06:35:30.812 DEBUG:boto:Token: None 2022-09-03T06:35:30.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.817 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ff842c547f0cd38-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.818 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.819 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.819 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.820 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:30.820 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:30.821 DEBUG:boto:Method: GET 2022-09-03T06:35:30.821 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:30.822 DEBUG:boto:Data: 2022-09-03T06:35:30.822 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.823 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.823 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.824 DEBUG:boto:Params: {} 2022-09-03T06:35:30.824 DEBUG:boto:Token: None 2022-09-03T06:35:30.825 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.829 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ccfb0ce4382dc8a6-006312f5b2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.874 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.875 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.876 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.876 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.877 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:30.877 DEBUG:boto:Data: 2022-09-03T06:35:30.878 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.879 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.879 DEBUG:boto:Params: {} 2022-09-03T06:35:30.880 DEBUG:boto:Token: None 2022-09-03T06:35:30.881 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1 2022-09-03T06:35:30.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2+LV780qO53fl4Pbn3oPqJcYT4g= 2022-09-03T06:35:30.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2+LV780qO53fl4Pbn3oPqJcYT4g=', 'Content-Length': '0'} 2022-09-03T06:35:30.887 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d2ee984590e6fd70-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.888 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.888 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.889 DEBUG:boto:Method: HEAD 2022-09-03T06:35:30.889 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:30.890 DEBUG:boto:Data: 2022-09-03T06:35:30.890 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.891 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.891 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.892 DEBUG:boto:Params: {} 2022-09-03T06:35:30.893 DEBUG:boto:Token: None 2022-09-03T06:35:30.893 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1 2022-09-03T06:35:30.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2+LV780qO53fl4Pbn3oPqJcYT4g= 2022-09-03T06:35:30.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2+LV780qO53fl4Pbn3oPqJcYT4g=', 'Content-Length': '0'} 2022-09-03T06:35:30.897 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cbfdef0368b9433e-006312f5b2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.899 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2022-09-03T06:35:30.900 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.902 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.910 DEBUG:boto:Method: GET 2022-09-03T06:35:30.911 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:30.912 DEBUG:boto:Data: 2022-09-03T06:35:30.912 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.913 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.913 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.914 DEBUG:boto:Params: {} 2022-09-03T06:35:30.914 DEBUG:boto:Token: None 2022-09-03T06:35:30.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1 2022-09-03T06:35:30.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2GuFcFEbk49wsO9auLXpPgdihE0= 2022-09-03T06:35:30.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2GuFcFEbk49wsO9auLXpPgdihE0=', 'Content-Length': '0'} 2022-09-03T06:35:30.919 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000001eaa4bf482187347-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.920 DEBUG:boto:path=/avysmw-61/obj1 2022-09-03T06:35:30.920 DEBUG:boto:auth_path=/avysmw-61/obj1 2022-09-03T06:35:30.921 DEBUG:boto:Method: GET 2022-09-03T06:35:30.922 DEBUG:boto:Path: /avysmw-61/obj1 2022-09-03T06:35:30.922 DEBUG:boto:Data: 2022-09-03T06:35:30.923 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.923 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.924 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.924 DEBUG:boto:Params: {} 2022-09-03T06:35:30.925 DEBUG:boto:Token: None 2022-09-03T06:35:30.926 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj1 2022-09-03T06:35:30.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2GuFcFEbk49wsO9auLXpPgdihE0= 2022-09-03T06:35:30.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2GuFcFEbk49wsO9auLXpPgdihE0=', 'Content-Length': '0'} 2022-09-03T06:35:30.930 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000b8d94eafa21799ac-006312f5b2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.931 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.931 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.932 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:30.933 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:30.933 DEBUG:boto:Method: GET 2022-09-03T06:35:30.934 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:30.934 DEBUG:boto:Data: 2022-09-03T06:35:30.935 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.935 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.936 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.936 DEBUG:boto:Params: {} 2022-09-03T06:35:30.937 DEBUG:boto:Token: None 2022-09-03T06:35:30.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.943 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a7bad7691857d38-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.943 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.944 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:30.945 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:30.946 DEBUG:boto:path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:30.946 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj1 2022-09-03T06:35:30.947 DEBUG:boto:Method: GET 2022-09-03T06:35:30.948 DEBUG:boto:Path: /avysmw-61/?prefix=obj1 2022-09-03T06:35:30.948 DEBUG:boto:Data: 2022-09-03T06:35:30.949 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.950 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:30.950 DEBUG:boto:Port: 8000 2022-09-03T06:35:30.951 DEBUG:boto:Params: {} 2022-09-03T06:35:30.951 DEBUG:boto:Token: None 2022-09-03T06:35:30.952 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/ 2022-09-03T06:35:30.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY= 2022-09-03T06:35:30.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUMeueNkczVMLaoz/asQVLSThHY=', 'Content-Length': '0'} 2022-09-03T06:35:30.967 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a62ce82103c6b42e-006312f5b2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.969 DEBUG:boto:b'avysmw-61obj11000falseobj12022-09-03T06:32:27.091Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:30.970 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:35:30.972 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:30.973 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:30.974 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:30.975 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:30.977 DEBUG:boto:Method: GET 2022-09-03T06:35:30.978 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:30.979 DEBUG:boto:Data: 2022-09-03T06:35:30.980 DEBUG:boto:Headers: {} 2022-09-03T06:35:30.981 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:30.982 DEBUG:boto:Port: 8001 2022-09-03T06:35:30.984 DEBUG:boto:Params: {} 2022-09-03T06:35:30.985 DEBUG:boto:Token: None 2022-09-03T06:35:30.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:30 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:30.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9kU7jVZq+zevQJAUxs2Yr+31SDo= 2022-09-03T06:35:30.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9kU7jVZq+zevQJAUxs2Yr+31SDo=', 'Content-Length': '0'} 2022-09-03T06:35:30.995 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000aa9cfeb127b8ee5b-006312f5b2-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:30.997 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:30.999 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:31.000 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.001 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.002 DEBUG:boto:Method: GET 2022-09-03T06:35:31.003 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.005 DEBUG:boto:Data: 2022-09-03T06:35:31.006 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.007 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.008 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.009 DEBUG:boto:Params: {} 2022-09-03T06:35:31.012 DEBUG:boto:Token: None 2022-09-03T06:35:31.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gHD7Kh+qd0Ry+AhtybDr+HKfVZg= 2022-09-03T06:35:31.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gHD7Kh+qd0Ry+AhtybDr+HKfVZg=', 'Content-Length': '0'} 2022-09-03T06:35:31.019 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000065984cf9cd6b9d12-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.020 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:31.021 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:31.022 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.022 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.023 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.023 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.024 DEBUG:boto:Data: 2022-09-03T06:35:31.024 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.025 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.027 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.027 DEBUG:boto:Params: {} 2022-09-03T06:35:31.028 DEBUG:boto:Token: None 2022-09-03T06:35:31.028 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dvDqlGD4TfuBUTyFOEbplV3m2HM= 2022-09-03T06:35:31.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dvDqlGD4TfuBUTyFOEbplV3m2HM=', 'Content-Length': '0'} 2022-09-03T06:35:31.041 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003bc8341d3a8ae9a5-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.043 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:31.044 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:31.044 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.047 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.052 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.054 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.055 DEBUG:boto:Data: 2022-09-03T06:35:31.056 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.057 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.058 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.059 DEBUG:boto:Params: {} 2022-09-03T06:35:31.060 DEBUG:boto:Token: None 2022-09-03T06:35:31.061 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dvDqlGD4TfuBUTyFOEbplV3m2HM= 2022-09-03T06:35:31.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dvDqlGD4TfuBUTyFOEbplV3m2HM=', 'Content-Length': '0'} 2022-09-03T06:35:31.070 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000075a19d5b343f0fda-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.072 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:35:31.073 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:31.075 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:31.076 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.077 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.079 DEBUG:boto:Method: GET 2022-09-03T06:35:31.080 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.081 DEBUG:boto:Data: 2022-09-03T06:35:31.083 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.084 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.085 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.087 DEBUG:boto:Params: {} 2022-09-03T06:35:31.088 DEBUG:boto:Token: None 2022-09-03T06:35:31.090 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gHD7Kh+qd0Ry+AhtybDr+HKfVZg= 2022-09-03T06:35:31.093 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gHD7Kh+qd0Ry+AhtybDr+HKfVZg=', 'Content-Length': '0'} 2022-09-03T06:35:31.107 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c3fa4daa3c1bcaa6-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.121 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:31.122 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:31.123 DEBUG:boto:path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.124 DEBUG:boto:auth_path=/avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.125 DEBUG:boto:Method: GET 2022-09-03T06:35:31.126 DEBUG:boto:Path: /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.129 DEBUG:boto:Data: 2022-09-03T06:35:31.130 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.131 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.133 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.134 DEBUG:boto:Params: {} 2022-09-03T06:35:31.136 DEBUG:boto:Token: None 2022-09-03T06:35:31.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj2?versionId=null 2022-09-03T06:35:31.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gHD7Kh+qd0Ry+AhtybDr+HKfVZg= 2022-09-03T06:35:31.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gHD7Kh+qd0Ry+AhtybDr+HKfVZg=', 'Content-Length': '0'} 2022-09-03T06:35:31.155 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000052cff309d97bba6e-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.157 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.158 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.159 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:31.160 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:31.162 DEBUG:boto:Method: GET 2022-09-03T06:35:31.163 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:31.163 DEBUG:boto:Data: 2022-09-03T06:35:31.164 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.164 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.165 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.166 DEBUG:boto:Params: {} 2022-09-03T06:35:31.166 DEBUG:boto:Token: None 2022-09-03T06:35:31.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.182 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d74bcd8d061bff15-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.186 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.187 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.188 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.189 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:31.190 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:31.191 DEBUG:boto:Method: GET 2022-09-03T06:35:31.192 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:31.194 DEBUG:boto:Data: 2022-09-03T06:35:31.194 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.195 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.196 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.196 DEBUG:boto:Params: {} 2022-09-03T06:35:31.197 DEBUG:boto:Token: None 2022-09-03T06:35:31.198 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.204 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b69092bec0eeaf4-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.205 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.207 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:31.208 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:31.209 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.209 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:31.210 DEBUG:boto:Data: 2022-09-03T06:35:31.211 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.213 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.214 DEBUG:boto:Params: {} 2022-09-03T06:35:31.215 DEBUG:boto:Token: None 2022-09-03T06:35:31.216 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj2 2022-09-03T06:35:31.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mm4O1uy9HEpql7A8thXOmC2kqxg= 2022-09-03T06:35:31.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mm4O1uy9HEpql7A8thXOmC2kqxg=', 'Content-Length': '0'} 2022-09-03T06:35:31.221 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c2c3d6ae837ab235-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.223 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:31.223 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:31.224 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.225 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:31.225 DEBUG:boto:Data: 2022-09-03T06:35:31.226 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.227 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.228 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.229 DEBUG:boto:Params: {} 2022-09-03T06:35:31.230 DEBUG:boto:Token: None 2022-09-03T06:35:31.230 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj2 2022-09-03T06:35:31.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mm4O1uy9HEpql7A8thXOmC2kqxg= 2022-09-03T06:35:31.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mm4O1uy9HEpql7A8thXOmC2kqxg=', 'Content-Length': '0'} 2022-09-03T06:35:31.243 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000005ebf266eb3ed50de-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.245 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2022-09-03T06:35:31.246 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:31.247 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:31.248 DEBUG:boto:Method: GET 2022-09-03T06:35:31.249 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:31.250 DEBUG:boto:Data: 2022-09-03T06:35:31.251 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.252 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.253 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.255 DEBUG:boto:Params: {} 2022-09-03T06:35:31.256 DEBUG:boto:Token: None 2022-09-03T06:35:31.257 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj2 2022-09-03T06:35:31.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gRif12pcbjDeXmdb+iUOJVvvi8Y= 2022-09-03T06:35:31.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gRif12pcbjDeXmdb+iUOJVvvi8Y=', 'Content-Length': '0'} 2022-09-03T06:35:31.262 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000cea25916ce87ea17-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.264 DEBUG:boto:path=/avysmw-61/obj2 2022-09-03T06:35:31.265 DEBUG:boto:auth_path=/avysmw-61/obj2 2022-09-03T06:35:31.266 DEBUG:boto:Method: GET 2022-09-03T06:35:31.267 DEBUG:boto:Path: /avysmw-61/obj2 2022-09-03T06:35:31.268 DEBUG:boto:Data: 2022-09-03T06:35:31.269 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.270 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.271 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.272 DEBUG:boto:Params: {} 2022-09-03T06:35:31.273 DEBUG:boto:Token: None 2022-09-03T06:35:31.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj2 2022-09-03T06:35:31.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gRif12pcbjDeXmdb+iUOJVvvi8Y= 2022-09-03T06:35:31.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gRif12pcbjDeXmdb+iUOJVvvi8Y=', 'Content-Length': '0'} 2022-09-03T06:35:31.281 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000216eafdb4dfc8be4-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.323 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.324 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.324 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:31.325 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:31.326 DEBUG:boto:Method: GET 2022-09-03T06:35:31.329 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:31.329 DEBUG:boto:Data: 2022-09-03T06:35:31.330 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.331 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.331 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.332 DEBUG:boto:Params: {} 2022-09-03T06:35:31.332 DEBUG:boto:Token: None 2022-09-03T06:35:31.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.337 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000005ab822ce730add3-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.338 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.339 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.339 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.340 DEBUG:boto:path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:31.340 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj2 2022-09-03T06:35:31.341 DEBUG:boto:Method: GET 2022-09-03T06:35:31.342 DEBUG:boto:Path: /avysmw-61/?prefix=obj2 2022-09-03T06:35:31.343 DEBUG:boto:Data: 2022-09-03T06:35:31.344 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.345 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.346 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.347 DEBUG:boto:Params: {} 2022-09-03T06:35:31.347 DEBUG:boto:Token: None 2022-09-03T06:35:31.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.359 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e9c14b397b72d9d-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.361 DEBUG:boto:b'avysmw-61obj21000falseobj22022-09-03T06:32:27.119Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.362 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:35:31.363 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.364 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.365 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.366 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.367 DEBUG:boto:Method: GET 2022-09-03T06:35:31.368 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.370 DEBUG:boto:Data: 2022-09-03T06:35:31.372 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.374 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.375 DEBUG:boto:Params: {} 2022-09-03T06:35:31.376 DEBUG:boto:Token: None 2022-09-03T06:35:31.378 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sWNlHZXyjBNuBRb1GZFyzhU07rY= 2022-09-03T06:35:31.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sWNlHZXyjBNuBRb1GZFyzhU07rY=', 'Content-Length': '0'} 2022-09-03T06:35:31.386 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000fb09e370204223a8-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.388 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.389 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.390 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.391 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.392 DEBUG:boto:Method: GET 2022-09-03T06:35:31.393 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.393 DEBUG:boto:Data: 2022-09-03T06:35:31.395 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.396 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.397 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.398 DEBUG:boto:Params: {} 2022-09-03T06:35:31.400 DEBUG:boto:Token: None 2022-09-03T06:35:31.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sWNlHZXyjBNuBRb1GZFyzhU07rY= 2022-09-03T06:35:31.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sWNlHZXyjBNuBRb1GZFyzhU07rY=', 'Content-Length': '0'} 2022-09-03T06:35:31.406 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000373932a13b720320-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.407 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.408 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.409 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.410 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.411 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.411 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.413 DEBUG:boto:Data: 2022-09-03T06:35:31.414 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.415 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.416 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.417 DEBUG:boto:Params: {} 2022-09-03T06:35:31.418 DEBUG:boto:Token: None 2022-09-03T06:35:31.419 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xZDV+VjiHczUarP+boJiZYYSONY= 2022-09-03T06:35:31.421 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xZDV+VjiHczUarP+boJiZYYSONY=', 'Content-Length': '0'} 2022-09-03T06:35:31.428 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009d7363fd0f880cd3-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.430 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.431 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.432 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.433 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.434 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.435 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.436 DEBUG:boto:Data: 2022-09-03T06:35:31.436 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.437 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.439 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.440 DEBUG:boto:Params: {} 2022-09-03T06:35:31.441 DEBUG:boto:Token: None 2022-09-03T06:35:31.442 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xZDV+VjiHczUarP+boJiZYYSONY= 2022-09-03T06:35:31.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xZDV+VjiHczUarP+boJiZYYSONY=', 'Content-Length': '0'} 2022-09-03T06:35:31.451 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000f66692fc36e1815e-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.453 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:35:31.454 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.455 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.456 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.457 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.457 DEBUG:boto:Method: GET 2022-09-03T06:35:31.458 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.460 DEBUG:boto:Data: 2022-09-03T06:35:31.461 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.463 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.463 DEBUG:boto:Params: {} 2022-09-03T06:35:31.464 DEBUG:boto:Token: None 2022-09-03T06:35:31.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sWNlHZXyjBNuBRb1GZFyzhU07rY= 2022-09-03T06:35:31.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sWNlHZXyjBNuBRb1GZFyzhU07rY=', 'Content-Length': '0'} 2022-09-03T06:35:31.476 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000009c3831e169c73b77-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.524 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.526 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.527 DEBUG:boto:path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.527 DEBUG:boto:auth_path=/avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.528 DEBUG:boto:Method: GET 2022-09-03T06:35:31.529 DEBUG:boto:Path: /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.531 DEBUG:boto:Data: 2022-09-03T06:35:31.534 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.536 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.539 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.540 DEBUG:boto:Params: {} 2022-09-03T06:35:31.542 DEBUG:boto:Token: None 2022-09-03T06:35:31.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3?versionId=null 2022-09-03T06:35:31.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sWNlHZXyjBNuBRb1GZFyzhU07rY= 2022-09-03T06:35:31.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sWNlHZXyjBNuBRb1GZFyzhU07rY=', 'Content-Length': '0'} 2022-09-03T06:35:31.548 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000055f15284d4fd4832-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.564 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.565 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.566 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:31.567 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:31.568 DEBUG:boto:Method: GET 2022-09-03T06:35:31.569 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:31.570 DEBUG:boto:Data: 2022-09-03T06:35:31.572 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.573 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.578 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.579 DEBUG:boto:Params: {} 2022-09-03T06:35:31.580 DEBUG:boto:Token: None 2022-09-03T06:35:31.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.596 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ec5e398d8ab3cffd-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.601 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.603 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.604 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.607 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:31.608 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:31.609 DEBUG:boto:Method: GET 2022-09-03T06:35:31.610 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:31.611 DEBUG:boto:Data: 2022-09-03T06:35:31.611 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.612 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.613 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.614 DEBUG:boto:Params: {} 2022-09-03T06:35:31.615 DEBUG:boto:Token: None 2022-09-03T06:35:31.618 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.628 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006d95d7efdad1f406-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.634 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.642 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.643 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.644 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.644 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:31.645 DEBUG:boto:Data: 2022-09-03T06:35:31.645 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.647 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.647 DEBUG:boto:Params: {} 2022-09-03T06:35:31.648 DEBUG:boto:Token: None 2022-09-03T06:35:31.648 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3 2022-09-03T06:35:31.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c/KLuFRT6c3/2oW+rq5CTVh1E9U= 2022-09-03T06:35:31.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c/KLuFRT6c3/2oW+rq5CTVh1E9U=', 'Content-Length': '0'} 2022-09-03T06:35:31.654 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000c405843dfc474d7a-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.655 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.656 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.657 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.657 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:31.658 DEBUG:boto:Data: 2022-09-03T06:35:31.658 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.659 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.659 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.660 DEBUG:boto:Params: {} 2022-09-03T06:35:31.660 DEBUG:boto:Token: None 2022-09-03T06:35:31.661 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3 2022-09-03T06:35:31.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c/KLuFRT6c3/2oW+rq5CTVh1E9U= 2022-09-03T06:35:31.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c/KLuFRT6c3/2oW+rq5CTVh1E9U=', 'Content-Length': '0'} 2022-09-03T06:35:31.667 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a72465387053e477-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.668 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2022-09-03T06:35:31.668 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.670 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.670 DEBUG:boto:Method: GET 2022-09-03T06:35:31.671 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:31.672 DEBUG:boto:Data: 2022-09-03T06:35:31.673 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.674 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.675 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.676 DEBUG:boto:Params: {} 2022-09-03T06:35:31.677 DEBUG:boto:Token: None 2022-09-03T06:35:31.679 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3 2022-09-03T06:35:31.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hEIKv6VddvA17+XsuQpFitgzbuM= 2022-09-03T06:35:31.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hEIKv6VddvA17+XsuQpFitgzbuM=', 'Content-Length': '0'} 2022-09-03T06:35:31.685 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000dc4bd5897c3488b3-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.688 DEBUG:boto:path=/avysmw-61/obj3 2022-09-03T06:35:31.689 DEBUG:boto:auth_path=/avysmw-61/obj3 2022-09-03T06:35:31.690 DEBUG:boto:Method: GET 2022-09-03T06:35:31.700 DEBUG:boto:Path: /avysmw-61/obj3 2022-09-03T06:35:31.703 DEBUG:boto:Data: 2022-09-03T06:35:31.703 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.704 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.704 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.705 DEBUG:boto:Params: {} 2022-09-03T06:35:31.706 DEBUG:boto:Token: None 2022-09-03T06:35:31.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj3 2022-09-03T06:35:31.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hEIKv6VddvA17+XsuQpFitgzbuM= 2022-09-03T06:35:31.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hEIKv6VddvA17+XsuQpFitgzbuM=', 'Content-Length': '0'} 2022-09-03T06:35:31.713 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a16555fa024c7b01-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.715 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.716 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.717 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:31.718 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:31.719 DEBUG:boto:Method: GET 2022-09-03T06:35:31.721 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:31.722 DEBUG:boto:Data: 2022-09-03T06:35:31.723 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.724 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.725 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.725 DEBUG:boto:Params: {} 2022-09-03T06:35:31.727 DEBUG:boto:Token: None 2022-09-03T06:35:31.729 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.734 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac6e10aeaa5cdbff-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.739 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.740 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.741 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.743 DEBUG:boto:path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:31.744 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj3 2022-09-03T06:35:31.744 DEBUG:boto:Method: GET 2022-09-03T06:35:31.745 DEBUG:boto:Path: /avysmw-61/?prefix=obj3 2022-09-03T06:35:31.746 DEBUG:boto:Data: 2022-09-03T06:35:31.747 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.747 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.748 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.749 DEBUG:boto:Params: {} 2022-09-03T06:35:31.751 DEBUG:boto:Token: None 2022-09-03T06:35:31.751 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.756 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076bc043608829ba4-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.757 DEBUG:boto:b'avysmw-61obj31000falseobj32022-09-03T06:32:27.313Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.758 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:35:31.760 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.760 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.761 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.762 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.762 DEBUG:boto:Method: GET 2022-09-03T06:35:31.763 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.764 DEBUG:boto:Data: 2022-09-03T06:35:31.764 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.765 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.766 DEBUG:boto:Params: {} 2022-09-03T06:35:31.767 DEBUG:boto:Token: None 2022-09-03T06:35:31.768 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:43OzmyTE9YGcV5Amx0Ukgfn3+eM= 2022-09-03T06:35:31.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:43OzmyTE9YGcV5Amx0Ukgfn3+eM=', 'Content-Length': '0'} 2022-09-03T06:35:31.772 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d3154b8fe9d5f0a1-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.774 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.775 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.775 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.776 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.776 DEBUG:boto:Method: GET 2022-09-03T06:35:31.777 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.778 DEBUG:boto:Data: 2022-09-03T06:35:31.778 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.779 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.780 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.781 DEBUG:boto:Params: {} 2022-09-03T06:35:31.781 DEBUG:boto:Token: None 2022-09-03T06:35:31.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:43OzmyTE9YGcV5Amx0Ukgfn3+eM= 2022-09-03T06:35:31.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:43OzmyTE9YGcV5Amx0Ukgfn3+eM=', 'Content-Length': '0'} 2022-09-03T06:35:31.787 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000ba613b6ba1863d96-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.788 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.789 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.790 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.790 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.791 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.791 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.792 DEBUG:boto:Data: 2022-09-03T06:35:31.793 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.794 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.794 DEBUG:boto:Params: {} 2022-09-03T06:35:31.795 DEBUG:boto:Token: None 2022-09-03T06:35:31.796 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BkPcrgNCRegnXfcrImxAsMrP08k= 2022-09-03T06:35:31.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BkPcrgNCRegnXfcrImxAsMrP08k=', 'Content-Length': '0'} 2022-09-03T06:35:31.799 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000d3b3503ef3e8d8d9-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.800 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.801 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.801 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.802 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.802 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.803 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.804 DEBUG:boto:Data: 2022-09-03T06:35:31.804 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.805 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.805 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.806 DEBUG:boto:Params: {} 2022-09-03T06:35:31.807 DEBUG:boto:Token: None 2022-09-03T06:35:31.807 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BkPcrgNCRegnXfcrImxAsMrP08k= 2022-09-03T06:35:31.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BkPcrgNCRegnXfcrImxAsMrP08k=', 'Content-Length': '0'} 2022-09-03T06:35:31.811 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000a755eb1bd627c147-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.811 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:35:31.812 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.813 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.813 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.814 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.815 DEBUG:boto:Method: GET 2022-09-03T06:35:31.815 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.816 DEBUG:boto:Data: 2022-09-03T06:35:31.817 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.818 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.819 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.819 DEBUG:boto:Params: {} 2022-09-03T06:35:31.820 DEBUG:boto:Token: None 2022-09-03T06:35:31.820 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:43OzmyTE9YGcV5Amx0Ukgfn3+eM= 2022-09-03T06:35:31.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:43OzmyTE9YGcV5Amx0Ukgfn3+eM=', 'Content-Length': '0'} 2022-09-03T06:35:31.824 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx00000bdd2b86db34c69b2-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.825 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.826 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.826 DEBUG:boto:path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.827 DEBUG:boto:auth_path=/avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.827 DEBUG:boto:Method: GET 2022-09-03T06:35:31.828 DEBUG:boto:Path: /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.828 DEBUG:boto:Data: 2022-09-03T06:35:31.829 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.829 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.830 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.831 DEBUG:boto:Params: {} 2022-09-03T06:35:31.831 DEBUG:boto:Token: None 2022-09-03T06:35:31.832 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4?versionId=null 2022-09-03T06:35:31.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:43OzmyTE9YGcV5Amx0Ukgfn3+eM= 2022-09-03T06:35:31.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:43OzmyTE9YGcV5Amx0Ukgfn3+eM=', 'Content-Length': '0'} 2022-09-03T06:35:31.835 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000004ffe453d3cef9347-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.836 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.837 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.837 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:31.838 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:31.838 DEBUG:boto:Method: GET 2022-09-03T06:35:31.839 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:31.839 DEBUG:boto:Data: 2022-09-03T06:35:31.840 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.841 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.841 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.842 DEBUG:boto:Params: {} 2022-09-03T06:35:31.842 DEBUG:boto:Token: None 2022-09-03T06:35:31.843 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.843 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.844 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.847 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6447397b6f2fa16-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.848 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.848 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.849 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.849 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:31.850 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:31.851 DEBUG:boto:Method: GET 2022-09-03T06:35:31.851 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:31.852 DEBUG:boto:Data: 2022-09-03T06:35:31.852 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.853 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.853 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.854 DEBUG:boto:Params: {} 2022-09-03T06:35:31.855 DEBUG:boto:Token: None 2022-09-03T06:35:31.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.859 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a412daf3138058ba-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.860 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.861 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.862 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.862 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.863 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:31.863 DEBUG:boto:Data: 2022-09-03T06:35:31.864 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.873 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.874 DEBUG:boto:Params: {} 2022-09-03T06:35:31.875 DEBUG:boto:Token: None 2022-09-03T06:35:31.876 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4 2022-09-03T06:35:31.877 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PnKMAx/tyiEFKf1aI8rStYLkcH8= 2022-09-03T06:35:31.878 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PnKMAx/tyiEFKf1aI8rStYLkcH8=', 'Content-Length': '0'} 2022-09-03T06:35:31.881 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000003b1be9fddecf5d59-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.882 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.883 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.884 DEBUG:boto:Method: HEAD 2022-09-03T06:35:31.884 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:31.885 DEBUG:boto:Data: 2022-09-03T06:35:31.885 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.886 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.886 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.887 DEBUG:boto:Params: {} 2022-09-03T06:35:31.888 DEBUG:boto:Token: None 2022-09-03T06:35:31.888 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4 2022-09-03T06:35:31.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PnKMAx/tyiEFKf1aI8rStYLkcH8= 2022-09-03T06:35:31.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PnKMAx/tyiEFKf1aI8rStYLkcH8=', 'Content-Length': '0'} 2022-09-03T06:35:31.892 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx0000049db038b6c20bb2d-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.893 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2022-09-03T06:35:31.893 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.894 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.896 DEBUG:boto:Method: GET 2022-09-03T06:35:31.897 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:31.898 DEBUG:boto:Data: 2022-09-03T06:35:31.899 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.900 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.901 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.902 DEBUG:boto:Params: {} 2022-09-03T06:35:31.903 DEBUG:boto:Token: None 2022-09-03T06:35:31.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4 2022-09-03T06:35:31.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oTw419smTi0HNjzkVDz75WK/hV8= 2022-09-03T06:35:31.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oTw419smTi0HNjzkVDz75WK/hV8=', 'Content-Length': '0'} 2022-09-03T06:35:31.910 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000002a7f3487e0a77633-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.911 DEBUG:boto:path=/avysmw-61/obj4 2022-09-03T06:35:31.912 DEBUG:boto:auth_path=/avysmw-61/obj4 2022-09-03T06:35:31.913 DEBUG:boto:Method: GET 2022-09-03T06:35:31.914 DEBUG:boto:Path: /avysmw-61/obj4 2022-09-03T06:35:31.915 DEBUG:boto:Data: 2022-09-03T06:35:31.916 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.917 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:31.918 DEBUG:boto:Port: 8000 2022-09-03T06:35:31.919 DEBUG:boto:Params: {} 2022-09-03T06:35:31.920 DEBUG:boto:Token: None 2022-09-03T06:35:31.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/obj4 2022-09-03T06:35:31.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oTw419smTi0HNjzkVDz75WK/hV8= 2022-09-03T06:35:31.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oTw419smTi0HNjzkVDz75WK/hV8=', 'Content-Length': '0'} 2022-09-03T06:35:31.927 DEBUG:boto:Response headers: [('Content-Length', '6'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:32:27 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('x-amz-request-id', 'tx000000a73548b899a1f31-006312f5b3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.974 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.976 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.976 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:31.977 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:31.977 DEBUG:boto:Method: GET 2022-09-03T06:35:31.978 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:31.979 DEBUG:boto:Data: 2022-09-03T06:35:31.980 DEBUG:boto:Headers: {} 2022-09-03T06:35:31.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:31.981 DEBUG:boto:Port: 8001 2022-09-03T06:35:31.981 DEBUG:boto:Params: {} 2022-09-03T06:35:31.982 DEBUG:boto:Token: None 2022-09-03T06:35:31.983 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:31 GMT /avysmw-61/ 2022-09-03T06:35:31.984 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0= 2022-09-03T06:35:31.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Al26/Kaw1djmnajADmSkk+wlrt0=', 'Content-Length': '0'} 2022-09-03T06:35:31.988 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093de6d0690183a98-006312f5b3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:31.989 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:31.991 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:31.992 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:31.994 DEBUG:boto:path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:31.995 DEBUG:boto:auth_path=/avysmw-61/?prefix=obj4 2022-09-03T06:35:31.996 DEBUG:boto:Method: GET 2022-09-03T06:35:31.997 DEBUG:boto:Path: /avysmw-61/?prefix=obj4 2022-09-03T06:35:31.998 DEBUG:boto:Data: 2022-09-03T06:35:31.999 DEBUG:boto:Headers: {} 2022-09-03T06:35:32.000 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:32.001 DEBUG:boto:Port: 8000 2022-09-03T06:35:32.001 DEBUG:boto:Params: {} 2022-09-03T06:35:32.003 DEBUG:boto:Token: None 2022-09-03T06:35:32.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:35:32 GMT /avysmw-61/ 2022-09-03T06:35:32.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1idHpF+1cKG139ODa0HciOGHkJU= 2022-09-03T06:35:32.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1idHpF+1cKG139ODa0HciOGHkJU=', 'Content-Length': '0'} 2022-09-03T06:35:32.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa1d36899f5cdbed-006312f5b4-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:35:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:32.011 DEBUG:boto:b'avysmw-61obj41000falseobj42022-09-03T06:32:27.351Z"a8f5f167f44f4964e6c998dee827110c"6STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:35:32.012 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-61 zones={test-zone3, test-zone2} 2022-09-03T06:35:32.013 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'avysmw-59', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.014 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'avysmw-59', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.015 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket avysmw-59 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:35:32.075 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:32.076 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:32.177 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'enable', '--bucket', 'avysmw-59', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.178 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'enable', '--bucket', 'avysmw-59', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.179 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync enable --bucket avysmw-59 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:35:32.232 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:32.232 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:32.313 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'avysmw-60', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.314 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'avysmw-60', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.315 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket avysmw-60 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:35:32.367 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:32.368 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:32.465 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'enable', '--bucket', 'avysmw-60', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.467 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'enable', '--bucket', 'avysmw-60', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.467 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync enable --bucket avysmw-60 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:35:32.520 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:32.520 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:32.622 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'avysmw-61', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.623 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'avysmw-61', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.627 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket avysmw-61 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:35:32.690 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:32.690 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:32.790 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'enable', '--bucket', 'avysmw-61', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.791 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'enable', '--bucket', 'avysmw-61', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:35:32.791 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync enable --bucket avysmw-61 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:35:32.846 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:32.847 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:32.935 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:35:32.937 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:35:32.938 DEBUG:boto:Method: HEAD 2022-09-03T06:35:32.939 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:35:32.940 DEBUG:boto:Data: 2022-09-03T06:35:32.940 DEBUG:boto:Headers: {} 2022-09-03T06:35:32.941 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:32.941 DEBUG:boto:Port: 8000 2022-09-03T06:35:32.942 DEBUG:boto:Params: {} 2022-09-03T06:35:32.943 DEBUG:boto:Token: None 2022-09-03T06:35:32.944 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:32 GMT /avysmw-59/ 2022-09-03T06:35:32.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N7IxkUNx+rfUmeyi+eZGz7EcTAk= 2022-09-03T06:35:32.945 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N7IxkUNx+rfUmeyi+eZGz7EcTAk=', 'Content-Length': '0'} 2022-09-03T06:35:32.948 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000138e430e73702366-006312f5b4-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:32.949 DEBUG:boto:path=/avysmw-59/obj5 2022-09-03T06:35:32.950 DEBUG:boto:auth_path=/avysmw-59/obj5 2022-09-03T06:35:32.950 DEBUG:boto:Method: PUT 2022-09-03T06:35:32.951 DEBUG:boto:Path: /avysmw-59/obj5 2022-09-03T06:35:32.952 DEBUG:boto:Data: 2022-09-03T06:35:32.952 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:32.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:32.953 DEBUG:boto:Port: 8000 2022-09-03T06:35:32.954 DEBUG:boto:Params: {} 2022-09-03T06:35:32.955 DEBUG:boto:Token: None 2022-09-03T06:35:32.955 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:32 GMT /avysmw-59/obj5 2022-09-03T06:35:32.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o5qQ9LKEZ81zfG3zVVKEEPFuBgQ= 2022-09-03T06:35:32.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o5qQ9LKEZ81zfG3zVVKEEPFuBgQ='} 2022-09-03T06:35:32.964 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000306ec802305f8b8d-006312f5b4-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:35:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:32.966 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:35:32.967 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:35:32.968 DEBUG:boto:Method: HEAD 2022-09-03T06:35:32.969 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:35:32.970 DEBUG:boto:Data: 2022-09-03T06:35:32.971 DEBUG:boto:Headers: {} 2022-09-03T06:35:32.973 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:32.974 DEBUG:boto:Port: 8000 2022-09-03T06:35:32.975 DEBUG:boto:Params: {} 2022-09-03T06:35:32.976 DEBUG:boto:Token: None 2022-09-03T06:35:32.978 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:32 GMT /avysmw-59/ 2022-09-03T06:35:32.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N7IxkUNx+rfUmeyi+eZGz7EcTAk= 2022-09-03T06:35:32.979 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N7IxkUNx+rfUmeyi+eZGz7EcTAk=', 'Content-Length': '0'} 2022-09-03T06:35:32.986 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000536c828c242cbca2-006312f5b4-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:32.987 DEBUG:boto:path=/avysmw-59/obj6 2022-09-03T06:35:32.988 DEBUG:boto:auth_path=/avysmw-59/obj6 2022-09-03T06:35:32.989 DEBUG:boto:Method: PUT 2022-09-03T06:35:32.989 DEBUG:boto:Path: /avysmw-59/obj6 2022-09-03T06:35:32.990 DEBUG:boto:Data: 2022-09-03T06:35:32.991 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:32.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:32.992 DEBUG:boto:Port: 8000 2022-09-03T06:35:32.992 DEBUG:boto:Params: {} 2022-09-03T06:35:32.993 DEBUG:boto:Token: None 2022-09-03T06:35:32.994 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:32 GMT /avysmw-59/obj6 2022-09-03T06:35:32.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WT9L13TA8yDj8ulEBO8X/v24/EI= 2022-09-03T06:35:32.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WT9L13TA8yDj8ulEBO8X/v24/EI='} 2022-09-03T06:35:33.042 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000a9c8ae55c53ae08e-006312f5b4-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.044 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:35:33.045 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:35:33.046 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.047 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:35:33.048 DEBUG:boto:Data: 2022-09-03T06:35:33.049 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:33.051 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.052 DEBUG:boto:Params: {} 2022-09-03T06:35:33.053 DEBUG:boto:Token: None 2022-09-03T06:35:33.054 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-59/ 2022-09-03T06:35:33.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OA2ebbqcE1NGRwahfpnkaudPXJk= 2022-09-03T06:35:33.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OA2ebbqcE1NGRwahfpnkaudPXJk=', 'Content-Length': '0'} 2022-09-03T06:35:33.059 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '36'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a9d090d5506997f1-006312f5b5-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.060 DEBUG:boto:path=/avysmw-59/obj7 2022-09-03T06:35:33.061 DEBUG:boto:auth_path=/avysmw-59/obj7 2022-09-03T06:35:33.061 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.062 DEBUG:boto:Path: /avysmw-59/obj7 2022-09-03T06:35:33.063 DEBUG:boto:Data: 2022-09-03T06:35:33.063 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.064 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:33.065 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.065 DEBUG:boto:Params: {} 2022-09-03T06:35:33.066 DEBUG:boto:Token: None 2022-09-03T06:35:33.067 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-59/obj7 2022-09-03T06:35:33.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qSHuN6/FeXMZAAL6nzTVwdZVkFA= 2022-09-03T06:35:33.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qSHuN6/FeXMZAAL6nzTVwdZVkFA='} 2022-09-03T06:35:33.076 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000098351e8dadffe6b7-006312f5b5-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.077 DEBUG:boto:path=/avysmw-59/ 2022-09-03T06:35:33.078 DEBUG:boto:auth_path=/avysmw-59/ 2022-09-03T06:35:33.079 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.080 DEBUG:boto:Path: /avysmw-59/ 2022-09-03T06:35:33.081 DEBUG:boto:Data: 2022-09-03T06:35:33.082 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.084 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:33.085 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.086 DEBUG:boto:Params: {} 2022-09-03T06:35:33.087 DEBUG:boto:Token: None 2022-09-03T06:35:33.088 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-59/ 2022-09-03T06:35:33.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OA2ebbqcE1NGRwahfpnkaudPXJk= 2022-09-03T06:35:33.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OA2ebbqcE1NGRwahfpnkaudPXJk=', 'Content-Length': '0'} 2022-09-03T06:35:33.093 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '42'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006ae78b7be3507bbe-006312f5b5-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.095 DEBUG:boto:path=/avysmw-59/obj8 2022-09-03T06:35:33.095 DEBUG:boto:auth_path=/avysmw-59/obj8 2022-09-03T06:35:33.096 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.096 DEBUG:boto:Path: /avysmw-59/obj8 2022-09-03T06:35:33.097 DEBUG:boto:Data: 2022-09-03T06:35:33.097 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.098 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:35:33.098 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.099 DEBUG:boto:Params: {} 2022-09-03T06:35:33.100 DEBUG:boto:Token: None 2022-09-03T06:35:33.100 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-59/obj8 2022-09-03T06:35:33.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c62sRmlfNFztbjvZe9bWhaZery8= 2022-09-03T06:35:33.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c62sRmlfNFztbjvZe9bWhaZery8='} 2022-09-03T06:35:33.146 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000063ce3f5d640c967f-006312f5b5-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.147 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:35:33.148 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:35:33.148 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.149 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:35:33.149 DEBUG:boto:Data: 2022-09-03T06:35:33.150 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.150 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:33.151 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.152 DEBUG:boto:Params: {} 2022-09-03T06:35:33.152 DEBUG:boto:Token: None 2022-09-03T06:35:33.153 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-60/ 2022-09-03T06:35:33.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3z2E4FnYyexPazPjcSfNXTt1u9w= 2022-09-03T06:35:33.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3z2E4FnYyexPazPjcSfNXTt1u9w=', 'Content-Length': '0'} 2022-09-03T06:35:33.157 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006ea507056e4d91d3-006312f5b5-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.159 DEBUG:boto:path=/avysmw-60/obj5 2022-09-03T06:35:33.160 DEBUG:boto:auth_path=/avysmw-60/obj5 2022-09-03T06:35:33.161 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.162 DEBUG:boto:Path: /avysmw-60/obj5 2022-09-03T06:35:33.163 DEBUG:boto:Data: 2022-09-03T06:35:33.164 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.164 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:33.165 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.165 DEBUG:boto:Params: {} 2022-09-03T06:35:33.166 DEBUG:boto:Token: None 2022-09-03T06:35:33.167 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-60/obj5 2022-09-03T06:35:33.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ss0679iU1XwEtpgfLbRrombXDLA= 2022-09-03T06:35:33.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ss0679iU1XwEtpgfLbRrombXDLA='} 2022-09-03T06:35:33.176 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000658677732800109f-006312f5b5-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.178 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:35:33.179 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:35:33.179 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.180 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:35:33.181 DEBUG:boto:Data: 2022-09-03T06:35:33.182 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.184 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:33.184 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.186 DEBUG:boto:Params: {} 2022-09-03T06:35:33.187 DEBUG:boto:Token: None 2022-09-03T06:35:33.187 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-60/ 2022-09-03T06:35:33.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3z2E4FnYyexPazPjcSfNXTt1u9w= 2022-09-03T06:35:33.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3z2E4FnYyexPazPjcSfNXTt1u9w=', 'Content-Length': '0'} 2022-09-03T06:35:33.192 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c0ce327f3f74ebf7-006312f5b5-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.194 DEBUG:boto:path=/avysmw-60/obj6 2022-09-03T06:35:33.195 DEBUG:boto:auth_path=/avysmw-60/obj6 2022-09-03T06:35:33.196 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.197 DEBUG:boto:Path: /avysmw-60/obj6 2022-09-03T06:35:33.199 DEBUG:boto:Data: 2022-09-03T06:35:33.200 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.203 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:33.204 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.205 DEBUG:boto:Params: {} 2022-09-03T06:35:33.206 DEBUG:boto:Token: None 2022-09-03T06:35:33.208 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-60/obj6 2022-09-03T06:35:33.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ms+Z5Ts1I6amjeFw98ljtOCBG18= 2022-09-03T06:35:33.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ms+Z5Ts1I6amjeFw98ljtOCBG18='} 2022-09-03T06:35:33.262 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000222cc49a498c48dd-006312f5b5-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.263 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:35:33.264 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:35:33.265 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.265 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:35:33.271 DEBUG:boto:Data: 2022-09-03T06:35:33.272 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:33.273 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.275 DEBUG:boto:Params: {} 2022-09-03T06:35:33.276 DEBUG:boto:Token: None 2022-09-03T06:35:33.277 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-60/ 2022-09-03T06:35:33.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3z2E4FnYyexPazPjcSfNXTt1u9w= 2022-09-03T06:35:33.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3z2E4FnYyexPazPjcSfNXTt1u9w=', 'Content-Length': '0'} 2022-09-03T06:35:33.283 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '36'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ade6e38c6b316bae-006312f5b5-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.285 DEBUG:boto:path=/avysmw-60/obj7 2022-09-03T06:35:33.285 DEBUG:boto:auth_path=/avysmw-60/obj7 2022-09-03T06:35:33.286 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.288 DEBUG:boto:Path: /avysmw-60/obj7 2022-09-03T06:35:33.289 DEBUG:boto:Data: 2022-09-03T06:35:33.290 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:33.292 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.292 DEBUG:boto:Params: {} 2022-09-03T06:35:33.294 DEBUG:boto:Token: None 2022-09-03T06:35:33.295 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-60/obj7 2022-09-03T06:35:33.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9E/0SJ9eYYo5o38ejr+8Mlhc8FM= 2022-09-03T06:35:33.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9E/0SJ9eYYo5o38ejr+8Mlhc8FM='} 2022-09-03T06:35:33.306 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000006b2f2f9320fdafad-006312f5b5-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.308 DEBUG:boto:path=/avysmw-60/ 2022-09-03T06:35:33.310 DEBUG:boto:auth_path=/avysmw-60/ 2022-09-03T06:35:33.311 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.312 DEBUG:boto:Path: /avysmw-60/ 2022-09-03T06:35:33.313 DEBUG:boto:Data: 2022-09-03T06:35:33.314 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.315 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:33.316 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.317 DEBUG:boto:Params: {} 2022-09-03T06:35:33.317 DEBUG:boto:Token: None 2022-09-03T06:35:33.318 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-60/ 2022-09-03T06:35:33.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3z2E4FnYyexPazPjcSfNXTt1u9w= 2022-09-03T06:35:33.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3z2E4FnYyexPazPjcSfNXTt1u9w=', 'Content-Length': '0'} 2022-09-03T06:35:33.323 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '42'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000028a3d3f214d0c6ef-006312f5b5-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.324 DEBUG:boto:path=/avysmw-60/obj8 2022-09-03T06:35:33.325 DEBUG:boto:auth_path=/avysmw-60/obj8 2022-09-03T06:35:33.325 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.326 DEBUG:boto:Path: /avysmw-60/obj8 2022-09-03T06:35:33.326 DEBUG:boto:Data: 2022-09-03T06:35:33.327 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.327 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:35:33.328 DEBUG:boto:Port: 8000 2022-09-03T06:35:33.328 DEBUG:boto:Params: {} 2022-09-03T06:35:33.329 DEBUG:boto:Token: None 2022-09-03T06:35:33.329 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-60/obj8 2022-09-03T06:35:33.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XV7NXAphSl2eLusn5kd9Jgl0kQ4= 2022-09-03T06:35:33.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XV7NXAphSl2eLusn5kd9Jgl0kQ4='} 2022-09-03T06:35:33.378 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000038f24e787433160a-006312f5b5-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.379 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:33.380 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:33.380 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.381 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:33.381 DEBUG:boto:Data: 2022-09-03T06:35:33.382 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:33.383 DEBUG:boto:Port: 8001 2022-09-03T06:35:33.384 DEBUG:boto:Params: {} 2022-09-03T06:35:33.384 DEBUG:boto:Token: None 2022-09-03T06:35:33.385 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-61/ 2022-09-03T06:35:33.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q25BttAiaxowMZlPdTcDUEMlyg0= 2022-09-03T06:35:33.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q25BttAiaxowMZlPdTcDUEMlyg0=', 'Content-Length': '0'} 2022-09-03T06:35:33.389 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001289aa5ecdfa3120-006312f5b5-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.390 DEBUG:boto:path=/avysmw-61/obj5 2022-09-03T06:35:33.390 DEBUG:boto:auth_path=/avysmw-61/obj5 2022-09-03T06:35:33.391 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.391 DEBUG:boto:Path: /avysmw-61/obj5 2022-09-03T06:35:33.392 DEBUG:boto:Data: 2022-09-03T06:35:33.392 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.393 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:33.393 DEBUG:boto:Port: 8001 2022-09-03T06:35:33.394 DEBUG:boto:Params: {} 2022-09-03T06:35:33.394 DEBUG:boto:Token: None 2022-09-03T06:35:33.395 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-61/obj5 2022-09-03T06:35:33.396 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TgCUPylbYd1Odt4aevED5NRrpy8= 2022-09-03T06:35:33.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TgCUPylbYd1Odt4aevED5NRrpy8='} 2022-09-03T06:35:33.404 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000c37868781e46f3ad-006312f5b5-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.405 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:33.406 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:33.407 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.408 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:33.410 DEBUG:boto:Data: 2022-09-03T06:35:33.411 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:33.412 DEBUG:boto:Port: 8001 2022-09-03T06:35:33.413 DEBUG:boto:Params: {} 2022-09-03T06:35:33.414 DEBUG:boto:Token: None 2022-09-03T06:35:33.416 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-61/ 2022-09-03T06:35:33.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q25BttAiaxowMZlPdTcDUEMlyg0= 2022-09-03T06:35:33.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q25BttAiaxowMZlPdTcDUEMlyg0=', 'Content-Length': '0'} 2022-09-03T06:35:33.424 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c501cabcdd466a8d-006312f5b5-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.425 DEBUG:boto:path=/avysmw-61/obj6 2022-09-03T06:35:33.425 DEBUG:boto:auth_path=/avysmw-61/obj6 2022-09-03T06:35:33.426 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.426 DEBUG:boto:Path: /avysmw-61/obj6 2022-09-03T06:35:33.427 DEBUG:boto:Data: 2022-09-03T06:35:33.427 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:33.428 DEBUG:boto:Port: 8001 2022-09-03T06:35:33.429 DEBUG:boto:Params: {} 2022-09-03T06:35:33.431 DEBUG:boto:Token: None 2022-09-03T06:35:33.431 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-61/obj6 2022-09-03T06:35:33.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eq+bExtycXIAqAU9p4f5eUG5GMM= 2022-09-03T06:35:33.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eq+bExtycXIAqAU9p4f5eUG5GMM='} 2022-09-03T06:35:33.484 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000021e4e9f1efe5ac8-006312f5b5-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.486 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:33.487 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:33.487 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.488 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:33.489 DEBUG:boto:Data: 2022-09-03T06:35:33.490 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:33.491 DEBUG:boto:Port: 8001 2022-09-03T06:35:33.492 DEBUG:boto:Params: {} 2022-09-03T06:35:33.493 DEBUG:boto:Token: None 2022-09-03T06:35:33.494 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-61/ 2022-09-03T06:35:33.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q25BttAiaxowMZlPdTcDUEMlyg0= 2022-09-03T06:35:33.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q25BttAiaxowMZlPdTcDUEMlyg0=', 'Content-Length': '0'} 2022-09-03T06:35:33.498 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '36'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005761643b9872cbec-006312f5b5-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.499 DEBUG:boto:path=/avysmw-61/obj7 2022-09-03T06:35:33.500 DEBUG:boto:auth_path=/avysmw-61/obj7 2022-09-03T06:35:33.501 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.501 DEBUG:boto:Path: /avysmw-61/obj7 2022-09-03T06:35:33.502 DEBUG:boto:Data: 2022-09-03T06:35:33.502 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:33.504 DEBUG:boto:Port: 8001 2022-09-03T06:35:33.505 DEBUG:boto:Params: {} 2022-09-03T06:35:33.505 DEBUG:boto:Token: None 2022-09-03T06:35:33.506 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-61/obj7 2022-09-03T06:35:33.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u53IbeION3+czsgOyqv6W1MJMqo= 2022-09-03T06:35:33.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u53IbeION3+czsgOyqv6W1MJMqo='} 2022-09-03T06:35:33.527 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000adab0b1aba22847c-006312f5b5-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.528 DEBUG:boto:path=/avysmw-61/ 2022-09-03T06:35:33.529 DEBUG:boto:auth_path=/avysmw-61/ 2022-09-03T06:35:33.530 DEBUG:boto:Method: HEAD 2022-09-03T06:35:33.531 DEBUG:boto:Path: /avysmw-61/ 2022-09-03T06:35:33.531 DEBUG:boto:Data: 2022-09-03T06:35:33.533 DEBUG:boto:Headers: {} 2022-09-03T06:35:33.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:33.539 DEBUG:boto:Port: 8001 2022-09-03T06:35:33.540 DEBUG:boto:Params: {} 2022-09-03T06:35:33.541 DEBUG:boto:Token: None 2022-09-03T06:35:33.542 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:35:33 GMT /avysmw-61/ 2022-09-03T06:35:33.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q25BttAiaxowMZlPdTcDUEMlyg0= 2022-09-03T06:35:33.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q25BttAiaxowMZlPdTcDUEMlyg0=', 'Content-Length': '0'} 2022-09-03T06:35:33.550 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '42'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000025f33dd370f3c3dc-006312f5b5-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.551 DEBUG:boto:path=/avysmw-61/obj8 2022-09-03T06:35:33.552 DEBUG:boto:auth_path=/avysmw-61/obj8 2022-09-03T06:35:33.553 DEBUG:boto:Method: PUT 2022-09-03T06:35:33.554 DEBUG:boto:Path: /avysmw-61/obj8 2022-09-03T06:35:33.555 DEBUG:boto:Data: 2022-09-03T06:35:33.556 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:35:33.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:35:33.557 DEBUG:boto:Port: 8001 2022-09-03T06:35:33.558 DEBUG:boto:Params: {} 2022-09-03T06:35:33.559 DEBUG:boto:Token: None 2022-09-03T06:35:33.560 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:35:33 GMT /avysmw-61/obj8 2022-09-03T06:35:33.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yObLnqpYHV0bRsFjfRn6YIsZBOs= 2022-09-03T06:35:33.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:35:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yObLnqpYHV0bRsFjfRn6YIsZBOs='} 2022-09-03T06:35:33.608 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000007bdec45dcce2a479-006312f5b5-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:35:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:35:33.609 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-59 2022-09-03T06:35:33.610 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:33.611 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:33.612 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:35:33.680 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:33.681 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:33.733 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:33.731+0000 7f448edf6600 1 waiting for incremental sync to catch up: 2022-09-03T06:35:33.733 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:35:33.734 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:35:38.734 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:38.732+0000 7f448edf6600 1 waiting for incremental sync to catch up: 2022-09-03T06:35:38.735 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:35:38.735 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:35:43.736 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:43.734+0000 7f448edf6600 1 waiting for incremental sync to catch up: 2022-09-03T06:35:43.736 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.47.5, , 00000000001.53.5, , 00000000001.26.5, , , 00000000001.52.5, , , ] 2022-09-03T06:35:43.737 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:35:48.740 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:48.737+0000 7f448edf6600 1 bucket sync caught up with source: 2022-09-03T06:35:48.740 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:35:48.741 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:35:48.741 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:35:48.737+0000 7f448edf6600 0 bucket checkpoint complete 2022-09-03T06:35:48.760 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-59 2022-09-03T06:35:48.762 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:48.763 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:35:48.764 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:35:48.830 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:35:48.831 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:35:49.884 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:49.881+0000 7fc20aef9600 1 waiting to reach incremental sync.. 2022-09-03T06:35:54.886 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:54.882+0000 7fc20aef9600 1 waiting to reach incremental sync.. 2022-09-03T06:35:59.886 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:35:59.882+0000 7fc20aef9600 1 waiting to reach incremental sync.. 2022-09-03T06:36:04.886 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:36:04.883+0000 7fc20aef9600 1 waiting to reach incremental sync.. 2022-09-03T06:36:09.886 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:36:09.884+0000 7fc20aef9600 1 waiting to reach incremental sync.. 2022-09-03T06:36:14.890 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:36:14.885+0000 7fc20aef9600 1 waiting to reach incremental sync.. 2022-09-03T06:36:19.888 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:36:19.885+0000 7fc20aef9600 1 waiting to reach incremental sync.. 2022-09-03T06:36:24.892 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:36:24.887+0000 7fc20aef9600 1 bucket sync caught up with source: 2022-09-03T06:36:24.893 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:36:24.893 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:36:24.894 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:36:24.887+0000 7fc20aef9600 0 bucket checkpoint complete 2022-09-03T06:36:24.912 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-59 2022-09-03T06:36:24.913 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:36:24.913 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-59', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:36:24.914 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-59 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:36:24.984 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:36:24.984 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:36:25.021 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:36:25.018+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:36:25.022 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:36:25.022 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:36:30.027 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:36:30.019+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:36:30.028 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:36:30.028 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:36:35.027 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:36:35.020+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:36:35.028 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:36:35.029 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:36:40.025 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:36:40.022+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:36:40.027 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:36:40.027 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:36:45.025 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:36:45.023+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:36:45.026 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:36:45.026 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:36:50.026 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:36:50.024+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:36:50.027 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:36:50.028 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:36:55.027 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:36:55.025+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:36:55.028 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:36:55.029 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:00.028 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:00.026+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:00.029 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:00.029 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:05.030 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:05.028+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:05.030 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:05.031 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:10.035 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:10.029+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:10.035 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:10.036 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:15.035 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:15.030+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:15.035 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:15.036 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:20.034 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:20.031+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:20.035 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:20.035 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:25.037 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:25.032+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:25.038 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:25.038 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:30.035 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:30.033+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:30.036 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:30.036 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:35.037 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:35.034+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:35.038 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:35.038 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:40.040 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:40.035+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:40.041 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:40.041 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:45.039 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:45.037+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:45.040 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:45.041 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:50.041 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:50.038+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:50.041 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:50.042 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:37:55.042 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:37:55.040+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:37:55.043 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:37:55.044 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:00.046 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:00.043+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:00.047 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:00.047 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:05.046 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:05.044+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:05.047 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:05.047 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:10.050 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:10.046+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:10.051 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:10.051 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:15.053 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:15.047+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:15.056 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:15.056 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:20.050 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:20.048+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:20.051 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:20.052 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:25.051 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:25.049+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:25.052 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:25.053 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:30.052 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:30.050+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:30.053 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:30.053 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:35.053 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:35.051+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:35.054 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:35.054 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:40.057 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:40.056+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:40.058 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:40.058 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:45.059 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:45.057+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:45.060 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:45.060 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:50.060 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:50.058+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:50.061 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:50.062 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:38:55.063 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:38:55.061+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:38:55.064 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:38:55.064 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:00.064 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:00.062+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:00.065 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:00.065 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:05.065 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:05.063+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:05.066 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:05.066 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:10.067 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:10.065+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:10.067 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:10.067 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:15.068 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:15.066+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:15.070 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:15.070 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:20.071 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:20.068+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:20.073 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:20.073 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:25.071 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:25.069+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:25.072 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:25.072 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:30.073 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:30.071+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:30.073 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:30.073 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:35.074 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:35.072+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:35.075 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:35.075 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:40.075 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:40.073+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:40.075 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:40.076 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:45.076 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:45.074+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:45.077 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:45.078 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:50.077 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:50.075+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:50.078 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:50.078 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:39:55.081 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:39:55.076+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:39:55.085 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:39:55.086 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:00.084 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:00.078+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:00.084 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:00.085 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:05.088 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:05.086+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:05.089 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:05.089 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:10.089 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:10.087+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:10.090 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:10.090 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:15.091 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:15.089+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:15.092 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:15.092 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:20.092 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:20.090+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:20.093 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:20.093 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:25.093 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:25.091+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:25.094 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:25.094 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:30.098 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:30.092+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:30.099 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:30.099 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:35.100 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:35.094+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:35.101 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:35.102 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:40.098 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:40.096+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:40.099 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:40.099 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:45.098 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:45.096+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:45.099 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:45.100 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:50.099 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:50.097+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:50.100 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:50.100 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:40:55.101 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:40:55.099+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:40:55.101 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:40:55.102 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:41:00.106 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:00.104+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:00.107 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:41:00.108 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:41:05.108 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:05.105+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:05.108 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:41:05.109 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:41:10.108 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:10.107+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:10.109 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:41:10.109 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:41:15.114 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:15.111+0000 7fd4fab8b600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:15.114 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , ] 2022-09-03T06:41:15.115 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000004.55.5, 00000000002.69.1, 00000000004.57.5, 00000000002.38.1, 00000000004.34.5, 00000000002.42.1, 00000000002.62.1, 00000000004.56.5, 00000000002.50.1, 00000000002.37.1, 00000000002.20.1] 2022-09-03T06:41:20.118 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:20.112+0000 7fd4fab8b600 0 bucket checkpoint timed out waiting for incremental sync to catch up 2022-09-03T06:41:20.119 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:20.112+0000 7fd4fab8b600 0 bucket sync checkpoint failed: (110) Connection timed out 2022-09-03T06:41:20.119 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:20.112+0000 7fd4fab8b600 -1 bucket sync checkpoint failed: (110) Connection timed out 2022-09-03T06:41:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 110 2022-09-03T06:41:20.186 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_sync_enable_right_after_disable ... FAIL 2022-09-03T06:41:20.188 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-62 2022-09-03T06:41:20.189 DEBUG:boto:path=/avysmw-62/ 2022-09-03T06:41:20.190 DEBUG:boto:auth_path=/avysmw-62/ 2022-09-03T06:41:20.191 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.191 DEBUG:boto:Path: /avysmw-62/ 2022-09-03T06:41:20.192 DEBUG:boto:Data: 2022-09-03T06:41:20.193 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.193 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:41:20.194 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.195 DEBUG:boto:Params: {} 2022-09-03T06:41:20.196 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:41:20.197 DEBUG:boto:Token: None 2022-09-03T06:41:20.199 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:41:20 GMT /avysmw-62/ 2022-09-03T06:41:20.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FzRi4bOxWbRczLDi2/PI9vMAzks= 2022-09-03T06:41:20.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FzRi4bOxWbRczLDi2/PI9vMAzks=', 'Content-Length': '0'} 2022-09-03T06:41:20.239 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000fd49b94e71269222-006312f710-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.241 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-63 2022-09-03T06:41:20.241 DEBUG:boto:path=/avysmw-63/ 2022-09-03T06:41:20.242 DEBUG:boto:auth_path=/avysmw-63/ 2022-09-03T06:41:20.243 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.243 DEBUG:boto:Path: /avysmw-63/ 2022-09-03T06:41:20.244 DEBUG:boto:Data: 2022-09-03T06:41:20.244 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.245 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:41:20.247 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.248 DEBUG:boto:Params: {} 2022-09-03T06:41:20.248 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:41:20.249 DEBUG:boto:Token: None 2022-09-03T06:41:20.250 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:41:20 GMT /avysmw-63/ 2022-09-03T06:41:20.250 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gpG8a38sLJlh7VnCH7lRookxTtE= 2022-09-03T06:41:20.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gpG8a38sLJlh7VnCH7lRookxTtE=', 'Content-Length': '0'} 2022-09-03T06:41:20.323 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000e140a497b0ecd37a-006312f710-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.324 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-64 2022-09-03T06:41:20.324 DEBUG:boto:path=/avysmw-64/ 2022-09-03T06:41:20.326 DEBUG:boto:auth_path=/avysmw-64/ 2022-09-03T06:41:20.327 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.328 DEBUG:boto:Path: /avysmw-64/ 2022-09-03T06:41:20.329 DEBUG:boto:Data: 2022-09-03T06:41:20.329 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.331 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:41:20.332 DEBUG:boto:Port: 8001 2022-09-03T06:41:20.332 DEBUG:boto:Params: {} 2022-09-03T06:41:20.333 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:41:20.334 DEBUG:boto:Token: None 2022-09-03T06:41:20.335 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:41:20 GMT /avysmw-64/ 2022-09-03T06:41:20.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9vD9NcWk66OSyD9KlZ3qyExzmbw= 2022-09-03T06:41:20.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9vD9NcWk66OSyD9KlZ3qyExzmbw=', 'Content-Length': '0'} 2022-09-03T06:41:20.375 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000fc5e017a0df1b043-006312f710-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.378 DEBUG:boto:path=/avysmw-62/ 2022-09-03T06:41:20.379 DEBUG:boto:auth_path=/avysmw-62/ 2022-09-03T06:41:20.380 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.381 DEBUG:boto:Path: /avysmw-62/ 2022-09-03T06:41:20.381 DEBUG:boto:Data: 2022-09-03T06:41:20.382 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:41:20.384 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.385 DEBUG:boto:Params: {} 2022-09-03T06:41:20.387 DEBUG:boto:Token: None 2022-09-03T06:41:20.388 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-62/ 2022-09-03T06:41:20.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fhilnoEtmmodowcgL4L+oH8dXhs= 2022-09-03T06:41:20.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fhilnoEtmmodowcgL4L+oH8dXhs=', 'Content-Length': '0'} 2022-09-03T06:41:20.393 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009a60f79bbebed0fd-006312f710-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.395 DEBUG:boto:path=/avysmw-62/obj1 2022-09-03T06:41:20.396 DEBUG:boto:auth_path=/avysmw-62/obj1 2022-09-03T06:41:20.397 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.398 DEBUG:boto:Path: /avysmw-62/obj1 2022-09-03T06:41:20.399 DEBUG:boto:Data: 2022-09-03T06:41:20.399 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:41:20.401 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.402 DEBUG:boto:Params: {} 2022-09-03T06:41:20.403 DEBUG:boto:Token: None 2022-09-03T06:41:20.404 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-62/obj1 2022-09-03T06:41:20.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BBHXRu3Mqg1X5lTXMpiH1gqt/dY= 2022-09-03T06:41:20.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BBHXRu3Mqg1X5lTXMpiH1gqt/dY='} 2022-09-03T06:41:20.416 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000befb9227f08588a6-006312f710-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.417 DEBUG:boto:path=/avysmw-62/ 2022-09-03T06:41:20.418 DEBUG:boto:auth_path=/avysmw-62/ 2022-09-03T06:41:20.419 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.420 DEBUG:boto:Path: /avysmw-62/ 2022-09-03T06:41:20.421 DEBUG:boto:Data: 2022-09-03T06:41:20.422 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.422 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:41:20.423 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.424 DEBUG:boto:Params: {} 2022-09-03T06:41:20.424 DEBUG:boto:Token: None 2022-09-03T06:41:20.425 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-62/ 2022-09-03T06:41:20.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fhilnoEtmmodowcgL4L+oH8dXhs= 2022-09-03T06:41:20.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fhilnoEtmmodowcgL4L+oH8dXhs=', 'Content-Length': '0'} 2022-09-03T06:41:20.428 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000088d3bfade84dca30-006312f710-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.430 DEBUG:boto:path=/avysmw-62/obj2 2022-09-03T06:41:20.431 DEBUG:boto:auth_path=/avysmw-62/obj2 2022-09-03T06:41:20.431 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.432 DEBUG:boto:Path: /avysmw-62/obj2 2022-09-03T06:41:20.433 DEBUG:boto:Data: 2022-09-03T06:41:20.433 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:41:20.435 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.436 DEBUG:boto:Params: {} 2022-09-03T06:41:20.437 DEBUG:boto:Token: None 2022-09-03T06:41:20.438 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-62/obj2 2022-09-03T06:41:20.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ytkt7r39cqng7KrvNLGERTfVkqw= 2022-09-03T06:41:20.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ytkt7r39cqng7KrvNLGERTfVkqw='} 2022-09-03T06:41:20.454 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000a6249584bf18bd68-006312f710-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.455 DEBUG:boto:path=/avysmw-62/ 2022-09-03T06:41:20.456 DEBUG:boto:auth_path=/avysmw-62/ 2022-09-03T06:41:20.457 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.457 DEBUG:boto:Path: /avysmw-62/ 2022-09-03T06:41:20.458 DEBUG:boto:Data: 2022-09-03T06:41:20.458 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.459 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:41:20.459 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.460 DEBUG:boto:Params: {} 2022-09-03T06:41:20.460 DEBUG:boto:Token: None 2022-09-03T06:41:20.461 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-62/ 2022-09-03T06:41:20.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fhilnoEtmmodowcgL4L+oH8dXhs= 2022-09-03T06:41:20.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fhilnoEtmmodowcgL4L+oH8dXhs=', 'Content-Length': '0'} 2022-09-03T06:41:20.469 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c66469db3b35064a-006312f710-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.471 DEBUG:boto:path=/avysmw-62/obj3 2022-09-03T06:41:20.472 DEBUG:boto:auth_path=/avysmw-62/obj3 2022-09-03T06:41:20.473 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.473 DEBUG:boto:Path: /avysmw-62/obj3 2022-09-03T06:41:20.479 DEBUG:boto:Data: 2022-09-03T06:41:20.480 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.480 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:41:20.481 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.481 DEBUG:boto:Params: {} 2022-09-03T06:41:20.482 DEBUG:boto:Token: None 2022-09-03T06:41:20.483 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-62/obj3 2022-09-03T06:41:20.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/pdjO5AFd/RkaA83JwUZyutSjog= 2022-09-03T06:41:20.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/pdjO5AFd/RkaA83JwUZyutSjog='} 2022-09-03T06:41:20.499 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000009aba2abf7db2c41b-006312f710-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.500 DEBUG:boto:path=/avysmw-62/ 2022-09-03T06:41:20.501 DEBUG:boto:auth_path=/avysmw-62/ 2022-09-03T06:41:20.501 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.502 DEBUG:boto:Path: /avysmw-62/ 2022-09-03T06:41:20.502 DEBUG:boto:Data: 2022-09-03T06:41:20.503 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:41:20.504 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.504 DEBUG:boto:Params: {} 2022-09-03T06:41:20.505 DEBUG:boto:Token: None 2022-09-03T06:41:20.505 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-62/ 2022-09-03T06:41:20.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fhilnoEtmmodowcgL4L+oH8dXhs= 2022-09-03T06:41:20.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fhilnoEtmmodowcgL4L+oH8dXhs=', 'Content-Length': '0'} 2022-09-03T06:41:20.509 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ccb76ec10ab62943-006312f710-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.510 DEBUG:boto:path=/avysmw-62/obj4 2022-09-03T06:41:20.511 DEBUG:boto:auth_path=/avysmw-62/obj4 2022-09-03T06:41:20.511 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.512 DEBUG:boto:Path: /avysmw-62/obj4 2022-09-03T06:41:20.512 DEBUG:boto:Data: 2022-09-03T06:41:20.513 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:41:20.514 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.514 DEBUG:boto:Params: {} 2022-09-03T06:41:20.515 DEBUG:boto:Token: None 2022-09-03T06:41:20.516 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-62/obj4 2022-09-03T06:41:20.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/7QkT+kMqlZ5dBMQJYNQvGE+boI= 2022-09-03T06:41:20.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/7QkT+kMqlZ5dBMQJYNQvGE+boI='} 2022-09-03T06:41:20.563 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000092b08404ba971a74-006312f710-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.565 DEBUG:boto:path=/avysmw-63/ 2022-09-03T06:41:20.565 DEBUG:boto:auth_path=/avysmw-63/ 2022-09-03T06:41:20.566 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.567 DEBUG:boto:Path: /avysmw-63/ 2022-09-03T06:41:20.568 DEBUG:boto:Data: 2022-09-03T06:41:20.568 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.570 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:41:20.570 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.571 DEBUG:boto:Params: {} 2022-09-03T06:41:20.571 DEBUG:boto:Token: None 2022-09-03T06:41:20.572 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-63/ 2022-09-03T06:41:20.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6aFOWKQHI2H7aMQVxcJVzV2fgfw= 2022-09-03T06:41:20.574 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6aFOWKQHI2H7aMQVxcJVzV2fgfw=', 'Content-Length': '0'} 2022-09-03T06:41:20.577 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a94e3112f3b9a358-006312f710-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.579 DEBUG:boto:path=/avysmw-63/obj1 2022-09-03T06:41:20.580 DEBUG:boto:auth_path=/avysmw-63/obj1 2022-09-03T06:41:20.580 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.581 DEBUG:boto:Path: /avysmw-63/obj1 2022-09-03T06:41:20.582 DEBUG:boto:Data: 2022-09-03T06:41:20.582 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.583 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:41:20.583 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.584 DEBUG:boto:Params: {} 2022-09-03T06:41:20.585 DEBUG:boto:Token: None 2022-09-03T06:41:20.586 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-63/obj1 2022-09-03T06:41:20.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uN0xGJXrShPENHuuJNiSZoRBNto= 2022-09-03T06:41:20.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uN0xGJXrShPENHuuJNiSZoRBNto='} 2022-09-03T06:41:20.599 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000070bf1f28e432b8be-006312f710-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.601 DEBUG:boto:path=/avysmw-63/ 2022-09-03T06:41:20.601 DEBUG:boto:auth_path=/avysmw-63/ 2022-09-03T06:41:20.602 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.603 DEBUG:boto:Path: /avysmw-63/ 2022-09-03T06:41:20.603 DEBUG:boto:Data: 2022-09-03T06:41:20.604 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:41:20.605 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.606 DEBUG:boto:Params: {} 2022-09-03T06:41:20.607 DEBUG:boto:Token: None 2022-09-03T06:41:20.608 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-63/ 2022-09-03T06:41:20.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6aFOWKQHI2H7aMQVxcJVzV2fgfw= 2022-09-03T06:41:20.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6aFOWKQHI2H7aMQVxcJVzV2fgfw=', 'Content-Length': '0'} 2022-09-03T06:41:20.611 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cf29819f1f5739f1-006312f710-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.613 DEBUG:boto:path=/avysmw-63/obj2 2022-09-03T06:41:20.613 DEBUG:boto:auth_path=/avysmw-63/obj2 2022-09-03T06:41:20.614 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.614 DEBUG:boto:Path: /avysmw-63/obj2 2022-09-03T06:41:20.615 DEBUG:boto:Data: 2022-09-03T06:41:20.615 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.618 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:41:20.619 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.621 DEBUG:boto:Params: {} 2022-09-03T06:41:20.622 DEBUG:boto:Token: None 2022-09-03T06:41:20.623 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-63/obj2 2022-09-03T06:41:20.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GraVV14iOCm86DTxletA+Qj+MEk= 2022-09-03T06:41:20.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GraVV14iOCm86DTxletA+Qj+MEk='} 2022-09-03T06:41:20.634 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000133b8597849179ab-006312f710-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.635 DEBUG:boto:path=/avysmw-63/ 2022-09-03T06:41:20.636 DEBUG:boto:auth_path=/avysmw-63/ 2022-09-03T06:41:20.637 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.638 DEBUG:boto:Path: /avysmw-63/ 2022-09-03T06:41:20.639 DEBUG:boto:Data: 2022-09-03T06:41:20.640 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.641 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:41:20.642 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.643 DEBUG:boto:Params: {} 2022-09-03T06:41:20.644 DEBUG:boto:Token: None 2022-09-03T06:41:20.645 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-63/ 2022-09-03T06:41:20.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6aFOWKQHI2H7aMQVxcJVzV2fgfw= 2022-09-03T06:41:20.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6aFOWKQHI2H7aMQVxcJVzV2fgfw=', 'Content-Length': '0'} 2022-09-03T06:41:20.651 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000051af3305c3c22c1e-006312f710-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.652 DEBUG:boto:path=/avysmw-63/obj3 2022-09-03T06:41:20.652 DEBUG:boto:auth_path=/avysmw-63/obj3 2022-09-03T06:41:20.653 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.653 DEBUG:boto:Path: /avysmw-63/obj3 2022-09-03T06:41:20.654 DEBUG:boto:Data: 2022-09-03T06:41:20.655 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.655 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:41:20.655 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.656 DEBUG:boto:Params: {} 2022-09-03T06:41:20.657 DEBUG:boto:Token: None 2022-09-03T06:41:20.658 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-63/obj3 2022-09-03T06:41:20.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JApp5FIJk2rdLpJmCG04BJQAJhI= 2022-09-03T06:41:20.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JApp5FIJk2rdLpJmCG04BJQAJhI='} 2022-09-03T06:41:20.714 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000d18103d49444404f-006312f710-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.715 DEBUG:boto:path=/avysmw-63/ 2022-09-03T06:41:20.716 DEBUG:boto:auth_path=/avysmw-63/ 2022-09-03T06:41:20.717 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.722 DEBUG:boto:Path: /avysmw-63/ 2022-09-03T06:41:20.723 DEBUG:boto:Data: 2022-09-03T06:41:20.724 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.725 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:41:20.726 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.727 DEBUG:boto:Params: {} 2022-09-03T06:41:20.728 DEBUG:boto:Token: None 2022-09-03T06:41:20.729 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-63/ 2022-09-03T06:41:20.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6aFOWKQHI2H7aMQVxcJVzV2fgfw= 2022-09-03T06:41:20.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6aFOWKQHI2H7aMQVxcJVzV2fgfw=', 'Content-Length': '0'} 2022-09-03T06:41:20.734 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000063bbf91f84d2d026-006312f710-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.736 DEBUG:boto:path=/avysmw-63/obj4 2022-09-03T06:41:20.736 DEBUG:boto:auth_path=/avysmw-63/obj4 2022-09-03T06:41:20.737 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.738 DEBUG:boto:Path: /avysmw-63/obj4 2022-09-03T06:41:20.739 DEBUG:boto:Data: 2022-09-03T06:41:20.740 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.741 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:41:20.742 DEBUG:boto:Port: 8000 2022-09-03T06:41:20.743 DEBUG:boto:Params: {} 2022-09-03T06:41:20.744 DEBUG:boto:Token: None 2022-09-03T06:41:20.745 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-63/obj4 2022-09-03T06:41:20.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Sk+fghmcdnqYwSaDQjM7vvyzC88= 2022-09-03T06:41:20.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Sk+fghmcdnqYwSaDQjM7vvyzC88='} 2022-09-03T06:41:20.758 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000041404623c079ba49-006312f710-170f-test-zone2'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.759 DEBUG:boto:path=/avysmw-64/ 2022-09-03T06:41:20.759 DEBUG:boto:auth_path=/avysmw-64/ 2022-09-03T06:41:20.760 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.760 DEBUG:boto:Path: /avysmw-64/ 2022-09-03T06:41:20.761 DEBUG:boto:Data: 2022-09-03T06:41:20.761 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.762 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:41:20.763 DEBUG:boto:Port: 8001 2022-09-03T06:41:20.764 DEBUG:boto:Params: {} 2022-09-03T06:41:20.765 DEBUG:boto:Token: None 2022-09-03T06:41:20.765 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-64/ 2022-09-03T06:41:20.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yi/FCnxTlL4AcdtLb9FalE1ToYA= 2022-09-03T06:41:20.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yi/FCnxTlL4AcdtLb9FalE1ToYA=', 'Content-Length': '0'} 2022-09-03T06:41:20.769 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bba6fc13b145e774-006312f710-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.771 DEBUG:boto:path=/avysmw-64/obj1 2022-09-03T06:41:20.773 DEBUG:boto:auth_path=/avysmw-64/obj1 2022-09-03T06:41:20.774 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.775 DEBUG:boto:Path: /avysmw-64/obj1 2022-09-03T06:41:20.778 DEBUG:boto:Data: 2022-09-03T06:41:20.779 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:41:20.781 DEBUG:boto:Port: 8001 2022-09-03T06:41:20.782 DEBUG:boto:Params: {} 2022-09-03T06:41:20.783 DEBUG:boto:Token: None 2022-09-03T06:41:20.784 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-64/obj1 2022-09-03T06:41:20.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PQblnKg3Ld+h2dTeui6D6R0ngS8= 2022-09-03T06:41:20.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PQblnKg3Ld+h2dTeui6D6R0ngS8='} 2022-09-03T06:41:20.798 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000fa8a670a3ebffba9-006312f710-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.799 DEBUG:boto:path=/avysmw-64/ 2022-09-03T06:41:20.800 DEBUG:boto:auth_path=/avysmw-64/ 2022-09-03T06:41:20.800 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.801 DEBUG:boto:Path: /avysmw-64/ 2022-09-03T06:41:20.801 DEBUG:boto:Data: 2022-09-03T06:41:20.802 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.802 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:41:20.803 DEBUG:boto:Port: 8001 2022-09-03T06:41:20.804 DEBUG:boto:Params: {} 2022-09-03T06:41:20.805 DEBUG:boto:Token: None 2022-09-03T06:41:20.806 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-64/ 2022-09-03T06:41:20.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yi/FCnxTlL4AcdtLb9FalE1ToYA= 2022-09-03T06:41:20.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yi/FCnxTlL4AcdtLb9FalE1ToYA=', 'Content-Length': '0'} 2022-09-03T06:41:20.812 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008b5dd1d54b810110-006312f710-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.814 DEBUG:boto:path=/avysmw-64/obj2 2022-09-03T06:41:20.815 DEBUG:boto:auth_path=/avysmw-64/obj2 2022-09-03T06:41:20.816 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.817 DEBUG:boto:Path: /avysmw-64/obj2 2022-09-03T06:41:20.818 DEBUG:boto:Data: 2022-09-03T06:41:20.819 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:41:20.821 DEBUG:boto:Port: 8001 2022-09-03T06:41:20.822 DEBUG:boto:Params: {} 2022-09-03T06:41:20.822 DEBUG:boto:Token: None 2022-09-03T06:41:20.823 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-64/obj2 2022-09-03T06:41:20.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fOWIYDKiGo4wYBgflaXuzb3oV3Q= 2022-09-03T06:41:20.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fOWIYDKiGo4wYBgflaXuzb3oV3Q='} 2022-09-03T06:41:20.836 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000019a8ac0e3acd83a9-006312f710-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.837 DEBUG:boto:path=/avysmw-64/ 2022-09-03T06:41:20.837 DEBUG:boto:auth_path=/avysmw-64/ 2022-09-03T06:41:20.838 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.839 DEBUG:boto:Path: /avysmw-64/ 2022-09-03T06:41:20.840 DEBUG:boto:Data: 2022-09-03T06:41:20.841 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.841 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:41:20.842 DEBUG:boto:Port: 8001 2022-09-03T06:41:20.843 DEBUG:boto:Params: {} 2022-09-03T06:41:20.844 DEBUG:boto:Token: None 2022-09-03T06:41:20.845 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-64/ 2022-09-03T06:41:20.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yi/FCnxTlL4AcdtLb9FalE1ToYA= 2022-09-03T06:41:20.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yi/FCnxTlL4AcdtLb9FalE1ToYA=', 'Content-Length': '0'} 2022-09-03T06:41:20.850 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000039d70d92441ca279-006312f710-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.852 DEBUG:boto:path=/avysmw-64/obj3 2022-09-03T06:41:20.852 DEBUG:boto:auth_path=/avysmw-64/obj3 2022-09-03T06:41:20.853 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.855 DEBUG:boto:Path: /avysmw-64/obj3 2022-09-03T06:41:20.856 DEBUG:boto:Data: 2022-09-03T06:41:20.857 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.858 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:41:20.859 DEBUG:boto:Port: 8001 2022-09-03T06:41:20.860 DEBUG:boto:Params: {} 2022-09-03T06:41:20.861 DEBUG:boto:Token: None 2022-09-03T06:41:20.862 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-64/obj3 2022-09-03T06:41:20.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Js1mUzQ2H2l9O52IeiTkS2nQ/MM= 2022-09-03T06:41:20.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Js1mUzQ2H2l9O52IeiTkS2nQ/MM='} 2022-09-03T06:41:20.873 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000167dbf4359392cbb-006312f710-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.875 DEBUG:boto:path=/avysmw-64/ 2022-09-03T06:41:20.875 DEBUG:boto:auth_path=/avysmw-64/ 2022-09-03T06:41:20.876 DEBUG:boto:Method: HEAD 2022-09-03T06:41:20.877 DEBUG:boto:Path: /avysmw-64/ 2022-09-03T06:41:20.877 DEBUG:boto:Data: 2022-09-03T06:41:20.878 DEBUG:boto:Headers: {} 2022-09-03T06:41:20.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:41:20.880 DEBUG:boto:Port: 8001 2022-09-03T06:41:20.882 DEBUG:boto:Params: {} 2022-09-03T06:41:20.883 DEBUG:boto:Token: None 2022-09-03T06:41:20.884 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:41:20 GMT /avysmw-64/ 2022-09-03T06:41:20.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yi/FCnxTlL4AcdtLb9FalE1ToYA= 2022-09-03T06:41:20.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yi/FCnxTlL4AcdtLb9FalE1ToYA=', 'Content-Length': '0'} 2022-09-03T06:41:20.889 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000017762ffcc2b8afd9-006312f710-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.891 DEBUG:boto:path=/avysmw-64/obj4 2022-09-03T06:41:20.892 DEBUG:boto:auth_path=/avysmw-64/obj4 2022-09-03T06:41:20.893 DEBUG:boto:Method: PUT 2022-09-03T06:41:20.894 DEBUG:boto:Path: /avysmw-64/obj4 2022-09-03T06:41:20.895 DEBUG:boto:Data: 2022-09-03T06:41:20.896 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue'} 2022-09-03T06:41:20.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:41:20.897 DEBUG:boto:Port: 8001 2022-09-03T06:41:20.898 DEBUG:boto:Params: {} 2022-09-03T06:41:20.899 DEBUG:boto:Token: None 2022-09-03T06:41:20.900 DEBUG:boto:StringToSign: PUT qPXxZ/RPSWTmyZje6CcRDA== application/octet-stream Sat, 03 Sep 2022 06:41:20 GMT /avysmw-64/obj4 2022-09-03T06:41:20.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FactaljACRXZZAkK+TuYJiMf4Mc= 2022-09-03T06:41:20.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'qPXxZ/RPSWTmyZje6CcRDA==', 'Content-Length': '6', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:41:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FactaljACRXZZAkK+TuYJiMf4Mc='} 2022-09-03T06:41:20.946 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"a8f5f167f44f4964e6c998dee827110c"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000fb59bffd95344d61-006312f710-1608-test-zone3'), ('Date', 'Sat, 03 Sep 2022 06:41:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:41:20.948 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:41:20.950 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:41:20.951 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:41:21.009 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:41:21.009 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:41:21.054 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:41:21.069 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:41:21.070 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:41:21.070 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:41:21.071 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:41:21.071 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:41:21.072 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:41:21.072 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187280.359597_346.1", 2022-09-03T06:41:21.073 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:41:20.359597Z" 2022-09-03T06:41:21.073 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:41:21.074 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:41:21.074 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:41:21.075 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:41:21.075 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:41:21.076 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:41:21.076 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:41:21.077 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:41:21.077 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:41:21.078 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:41:21.079 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662187280.359597_346.1', 2: '', 3: ''} 2022-09-03T06:41:21.080 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:41:21.082 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:41:21.083 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:41:21.084 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:41:21.144 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:41:21.144 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:41:21.198 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:21.191+0000 7fecf9e1e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:41:21.199 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:21.193+0000 7fecf9e1e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:41:21.239 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:41:21.239 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:41:21.239 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:41:21.240 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:41:21.240 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:41:21.240 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:41:21.240 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:41:21.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.241 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:41:21.241 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:41:21.242 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:41:21.242 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:41:21.242 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:41:21.243 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:41:21.243 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:41:21.243 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:41:21.244 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:41:21.244 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:41:21.244 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:41:21.245 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.245 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:41:21.246 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:41:21.246 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:41:21.249 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:41:21.251 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187280.359597_346.1", 2022-09-03T06:41:21.251 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:41:21.252 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:41:21.252 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:41:21.252 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:41:20.359597Z", 2022-09-03T06:41:21.252 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:41:21.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.253 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:41:21.253 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:41:21.254 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:41:21.254 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:41:21.254 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:41:21.255 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:41:21.255 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:41:21.255 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:41:21.255 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:41:21.256 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:41:21.256 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:41:21.256 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:41:21.257 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:41:21.257 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:41:21.257 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:41:21.257 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:41:21.258 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:41:21.258 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:41:21.258 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:41:21.258 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:41:21.259 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.259 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.259 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:41:21.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.260 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:41:21.260 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:41:21.260 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:41:21.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.261 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:41:21Z" 2022-09-03T06:41:21.261 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:41:21.262 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187280.359597_346.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:41:20.359597Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:41:21Z" } 2022-09-03T06:41:21.263 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187280.359597_346.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:41:20.359597Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:41:21.263 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187280.359597_346.1', 2: '', 3: ''} 2022-09-03T06:41:21.264 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187280.359597_346.1', 2: '', 3: ''} 2022-09-03T06:41:21.264 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:41:21.265 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:41:21.265 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:41:21.266 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:41:21.267 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:41:21.321 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:41:21.322 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:41:21.352 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:41:21.350+0000 7f6c3bdec600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:41:21.353 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:41:21.351+0000 7f6c3bdec600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:41:21.370 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:41:21.370 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:41:21.371 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:41:21.373 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:41:21.374 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:41:21.374 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:41:21.375 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:41:21.375 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:41:21.376 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:41:21.376 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:41:21.377 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:41:21.377 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:41:21.378 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:41:21.380 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:41:21.380 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:41:21.381 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:41:21.381 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:41:21.382 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:41:21.382 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:41:21.383 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:41:21.383 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:41:21.384 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:41:21.385 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:41:21.385 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:41:21.386 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:41:21.386 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187280.359597_346.1", 2022-09-03T06:41:21.387 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:41:21.388 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:41:21.388 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:41:21.389 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:41:20.359597Z", 2022-09-03T06:41:21.389 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:41:21.390 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:41:21.390 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:41:21.391 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:41:21.392 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:41:21.392 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:41:21.393 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:41:21.394 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:41:21.394 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:41:21.395 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:41:21.395 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:41:21.396 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:41:21.396 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:41:21.397 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:41:21.397 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:41:21.398 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:41:21.398 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:41:21.399 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:41:21.399 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:41:21.400 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:41:21.400 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:41:21.400 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:41:21.401 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:41:21.401 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:41:21.402 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:41:21.402 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:41:21.403 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:41:21.403 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:41:21.404 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:41:21.404 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:41:21.405 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:41:21.406 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:41:21.406 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:41:21.407 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:41:21Z" 2022-09-03T06:41:21.407 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:41:21.408 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187280.359597_346.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:41:20.359597Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:41:21Z" } 2022-09-03T06:41:21.409 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187280.359597_346.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:41:20.359597Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:41:21.410 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187280.359597_346.1', 2: '', 3: ''} 2022-09-03T06:41:21.411 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187280.359597_346.1', 2: '', 3: ''} 2022-09-03T06:41:21.412 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:41:21.413 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:41:21.414 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:41:21.415 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:41:21.416 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:41:21.468 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:41:21.469 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:41:21.501 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:21.500+0000 7ff8294f4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:41:21.502 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:21.501+0000 7ff8294f4600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:41:21.529 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:41:21.530 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:41:21.531 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:41:21.531 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:41:21.532 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:41:21.532 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:41:21.533 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:41:21.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.534 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:41:21.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:41:21.535 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:41:21.535 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:41:21.536 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:41:21.536 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:41:21.537 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:41:21.537 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:41:21.538 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:41:21.538 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:41:21.539 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:41:21.539 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:41:21.541 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:41:21.541 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:41:21.542 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:41:21.542 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187280.359597_346.1", 2022-09-03T06:41:21.543 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:41:21.543 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:41:21.544 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:41:21.544 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:41:20.359597Z", 2022-09-03T06:41:21.545 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:41:21.545 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:41:21.547 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:41:21.550 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:41:21.550 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:41:21.551 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:41:21.551 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:41:21.552 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:41:21.552 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:41:21.553 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:41:21.553 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:41:21.554 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:41:21.555 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:41:21.555 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:41:21.555 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:41:21.556 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:41:21.556 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:41:21.557 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:41:21.557 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:41:21.557 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:41:21.558 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:41:21.558 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.559 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:41:21.559 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:41:21.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.559 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:41:21.560 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:41:21.560 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:41:21.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:41:21.560 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:41:21Z" 2022-09-03T06:41:21.560 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:41:21.561 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187280.359597_346.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:41:20.359597Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:41:21Z" } 2022-09-03T06:41:21.562 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187280.359597_346.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:41:20.359597Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:41:21.562 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187280.359597_346.1', 2: '', 3: ''} 2022-09-03T06:41:21.563 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187280.359597_346.1', 2: '', 3: ''} 2022-09-03T06:41:21.563 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:41:21.564 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-62 2022-09-03T06:41:21.564 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-62', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:41:21.565 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-62', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:41:21.566 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-62 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:41:21.634 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:41:21.635 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:41:21.701 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:21.697+0000 7ff831064600 1 waiting to reach incremental sync.. 2022-09-03T06:41:26.700 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:26.698+0000 7ff831064600 1 bucket sync caught up with source: 2022-09-03T06:41:26.701 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:26.702 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:26.702 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:26.698+0000 7ff831064600 0 bucket checkpoint complete 2022-09-03T06:41:26.736 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-62 2022-09-03T06:41:26.738 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-62', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:41:26.738 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-62', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:41:26.739 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-62 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:41:26.798 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:41:26.798 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:41:26.850 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:41:26.846+0000 7fbcb0d7d600 1 bucket sync caught up with source: 2022-09-03T06:41:26.851 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:26.851 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:26.851 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:41:26.846+0000 7fbcb0d7d600 0 bucket checkpoint complete 2022-09-03T06:41:26.871 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-62 2022-09-03T06:41:26.874 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-62', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:41:26.875 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-62', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:41:26.876 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-62 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:41:26.936 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:41:26.936 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:41:26.995 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:26.993+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:26.995 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:41:26.996 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:31.996 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:31.994+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:31.997 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:41:31.997 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:36.997 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:36.995+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:36.998 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:41:36.999 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:41.998 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:41.996+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:41:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:46.999 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:46.997+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:47.000 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:41:47.000 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:52.000 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:51.998+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:52.001 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:41:52.001 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:41:57.003 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:41:56.999+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:41:57.003 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:41:57.004 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:02.003 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:02.001+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:02.003 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:42:02.004 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:07.006 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:07.004+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:07.008 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:42:07.008 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:12.009 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:12.007+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:12.010 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:42:12.010 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:17.014 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:17.009+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:17.015 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:42:17.016 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:22.012 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:22.010+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:22.013 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:42:22.014 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:27.013 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:27.011+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:27.013 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:42:27.014 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:32.014 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:32.012+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:32.015 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:42:32.016 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:37.015 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:37.013+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:37.016 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:42:37.017 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:42.018 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:42.016+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:42.019 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , , , , 00000000001.45.5, , , ] 2022-09-03T06:42:42.021 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:47.020 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:47.017+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:47.021 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:47.021 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:52.020 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:52.018+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:52.021 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:52.021 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:57.021 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:42:57.019+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:42:57.022 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:42:57.023 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:02.023 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:02.020+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:02.024 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:02.024 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:07.026 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:07.021+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:07.026 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:07.027 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:12.025 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:12.023+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:12.026 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:12.026 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:17.026 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:17.024+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:17.027 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:17.028 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:22.028 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:22.025+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:22.029 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:22.030 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:27.032 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:27.029+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:27.032 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:27.033 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:32.032 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:32.030+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:32.032 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:32.033 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:37.033 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:37.031+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:37.034 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:37.034 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:42.035 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:42.032+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:42.036 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:42.036 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:47.036 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:47.034+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:47.037 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:47.037 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:52.037 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:52.035+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:52.038 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:52.038 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:57.038 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:43:57.036+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:43:57.039 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:43:57.040 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:02.040 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:02.038+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:02.040 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:02.041 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:07.043 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:07.040+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:07.043 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:07.044 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:12.043 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:12.041+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:12.043 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:12.044 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:17.044 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:17.042+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:17.045 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:17.046 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:22.046 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:22.043+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:22.047 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:22.047 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:27.046 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:27.044+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:27.047 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:27.048 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:32.047 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:32.045+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:32.048 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:32.048 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:37.048 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:37.046+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:37.049 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:37.049 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:42.050 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:42.048+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:42.051 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:42.051 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:47.052 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:47.049+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:47.054 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:47.055 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:52.052 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:52.050+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:52.053 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:52.053 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:57.053 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:44:57.051+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:44:57.054 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:44:57.054 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:02.057 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:02.052+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:02.058 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:02.058 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:07.055 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:07.053+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:07.056 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:07.056 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:12.058 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:12.055+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:12.059 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:12.060 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:17.058 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:17.056+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:17.059 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:17.059 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:22.059 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:22.057+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:22.060 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:22.060 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:27.069 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:27.058+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:27.070 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:27.071 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:32.065 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:32.060+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:32.066 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:32.066 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:37.063 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:37.061+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:37.063 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:37.064 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:42.064 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:42.062+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:42.065 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:42.065 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:47.066 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:47.063+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:47.066 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:47.067 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:52.068 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:52.065+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:52.068 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:52.069 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:57.069 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:45:57.067+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:45:57.070 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:45:57.071 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:46:02.071 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:02.068+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:46:02.071 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:46:02.072 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:46:07.072 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:07.069+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:46:07.072 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:46:07.072 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:46:12.076 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:12.071+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:46:12.077 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:46:12.077 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:46:17.074 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:17.072+0000 7ff8dcf20600 1 waiting for incremental sync to catch up: 2022-09-03T06:46:17.074 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:46:17.074 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.41.5, , 00000000001.55.5, , 00000000001.28.5, , , 00000000001.45.5, , , ] 2022-09-03T06:46:22.075 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:22.073+0000 7ff8dcf20600 0 bucket checkpoint timed out waiting for incremental sync to catch up 2022-09-03T06:46:22.075 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:22.073+0000 7ff8dcf20600 0 bucket sync checkpoint failed: (110) Connection timed out 2022-09-03T06:46:22.076 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:22.073+0000 7ff8dcf20600 -1 bucket sync checkpoint failed: (110) Connection timed out 2022-09-03T06:46:22.088 DEBUG:teuthology.orchestra.run:got remote process result: 110 2022-09-03T06:46:22.097 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_sync_disable_enable ... FAIL 2022-09-03T06:46:22.099 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-65 2022-09-03T06:46:22.100 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:46:22.100 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:46:22.101 DEBUG:boto:Method: PUT 2022-09-03T06:46:22.102 DEBUG:boto:Path: /avysmw-65/ 2022-09-03T06:46:22.102 DEBUG:boto:Data: 2022-09-03T06:46:22.103 DEBUG:boto:Headers: {} 2022-09-03T06:46:22.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:22.104 DEBUG:boto:Port: 8000 2022-09-03T06:46:22.104 DEBUG:boto:Params: {} 2022-09-03T06:46:22.106 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:46:22.107 DEBUG:boto:Token: None 2022-09-03T06:46:22.108 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:46:22 GMT /avysmw-65/ 2022-09-03T06:46:22.110 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Q1tqErcs3YJdFr8AKPuFK892sU= 2022-09-03T06:46:22.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Q1tqErcs3YJdFr8AKPuFK892sU=', 'Content-Length': '0'} 2022-09-03T06:46:22.146 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000003a73c3e8a2b79b9b-006312f83e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:46:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:22.147 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=avysmw-66 2022-09-03T06:46:22.148 DEBUG:boto:path=/avysmw-66/ 2022-09-03T06:46:22.149 DEBUG:boto:auth_path=/avysmw-66/ 2022-09-03T06:46:22.151 DEBUG:boto:Method: PUT 2022-09-03T06:46:22.152 DEBUG:boto:Path: /avysmw-66/ 2022-09-03T06:46:22.153 DEBUG:boto:Data: 2022-09-03T06:46:22.154 DEBUG:boto:Headers: {} 2022-09-03T06:46:22.155 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:46:22.156 DEBUG:boto:Port: 8000 2022-09-03T06:46:22.157 DEBUG:boto:Params: {} 2022-09-03T06:46:22.159 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:46:22.164 DEBUG:boto:Token: None 2022-09-03T06:46:22.165 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:46:22 GMT /avysmw-66/ 2022-09-03T06:46:22.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MQFDJSD/4lad30BJwnaLqfNHsgE= 2022-09-03T06:46:22.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MQFDJSD/4lad30BJwnaLqfNHsgE=', 'Content-Length': '0'} 2022-09-03T06:46:22.256 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000768538f96eabe66f-006312f83e-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:46:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:22.257 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=avysmw-67 2022-09-03T06:46:22.257 DEBUG:boto:path=/avysmw-67/ 2022-09-03T06:46:22.258 DEBUG:boto:auth_path=/avysmw-67/ 2022-09-03T06:46:22.258 DEBUG:boto:Method: PUT 2022-09-03T06:46:22.259 DEBUG:boto:Path: /avysmw-67/ 2022-09-03T06:46:22.259 DEBUG:boto:Data: 2022-09-03T06:46:22.260 DEBUG:boto:Headers: {} 2022-09-03T06:46:22.261 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:46:22.261 DEBUG:boto:Port: 8001 2022-09-03T06:46:22.262 DEBUG:boto:Params: {} 2022-09-03T06:46:22.263 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:46:22.264 DEBUG:boto:Token: None 2022-09-03T06:46:22.265 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:46:22 GMT /avysmw-67/ 2022-09-03T06:46:22.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngSV/I4y4eciBz63/0umDZONPBs= 2022-09-03T06:46:22.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngSV/I4y4eciBz63/0umDZONPBs=', 'Content-Length': '0'} 2022-09-03T06:46:23.326 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000007c6841834fab025d-006312f83e-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:46:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:23.327 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:46:23.328 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:46:23.329 DEBUG:boto:path=/avysmw-65/MULTIPART?uploads 2022-09-03T06:46:23.330 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?uploads 2022-09-03T06:46:23.331 DEBUG:boto:Method: POST 2022-09-03T06:46:23.332 DEBUG:boto:Path: /avysmw-65/MULTIPART?uploads 2022-09-03T06:46:23.333 DEBUG:boto:Data: 2022-09-03T06:46:23.334 DEBUG:boto:Headers: {} 2022-09-03T06:46:23.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:23.336 DEBUG:boto:Port: 8000 2022-09-03T06:46:23.337 DEBUG:boto:Params: {} 2022-09-03T06:46:23.338 DEBUG:boto:Token: None 2022-09-03T06:46:23.339 DEBUG:boto:StringToSign: POST Sat, 03 Sep 2022 06:46:23 GMT /avysmw-65/MULTIPART?uploads 2022-09-03T06:46:23.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5OvKAcpz7UezE2cGcUa0h+5X0w8= 2022-09-03T06:46:23.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5OvKAcpz7UezE2cGcUa0h+5X0w8=', 'Content-Length': '0'} 2022-09-03T06:46:24.785 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000065bc072fa00912d3-006312f83f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '249'), ('Date', 'Sat, 03 Sep 2022 06:46:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:24.786 DEBUG:boto:b'avysmw-65MULTIPART2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW' 2022-09-03T06:46:24.845 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:46:24.853 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:46:24.853 DEBUG:boto:path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=1 2022-09-03T06:46:24.854 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=1 2022-09-03T06:46:24.855 DEBUG:boto:Method: PUT 2022-09-03T06:46:24.855 DEBUG:boto:Path: /avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=1 2022-09-03T06:46:24.856 DEBUG:boto:Data: 2022-09-03T06:46:24.856 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'ebKBBg0ze5srhMzzkK3PdA==', 'Content-Length': '5242880', 'Expect': '100-Continue'} 2022-09-03T06:46:24.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:24.858 DEBUG:boto:Port: 8000 2022-09-03T06:46:24.858 DEBUG:boto:Params: {} 2022-09-03T06:46:24.859 DEBUG:boto:Token: None 2022-09-03T06:46:24.860 DEBUG:boto:StringToSign: PUT ebKBBg0ze5srhMzzkK3PdA== application/octet-stream Sat, 03 Sep 2022 06:46:24 GMT /avysmw-65/MULTIPART?partNumber=1&uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:24.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UMTWbRzoYgW01k9yyZG/9Czkowg= 2022-09-03T06:46:24.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'ebKBBg0ze5srhMzzkK3PdA==', 'Content-Length': '5242880', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:46:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UMTWbRzoYgW01k9yyZG/9Czkowg='} 2022-09-03T06:46:24.958 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"79b281060d337b9b2b84ccf390adcf74"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000225e6a01bf8a0f7b-006312f840-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:46:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:25.006 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.007 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.008 DEBUG:boto:path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=2 2022-09-03T06:46:25.008 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=2 2022-09-03T06:46:25.009 DEBUG:boto:Method: PUT 2022-09-03T06:46:25.009 DEBUG:boto:Path: /avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=2 2022-09-03T06:46:25.010 DEBUG:boto:Data: 2022-09-03T06:46:25.011 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'dIQ6OrGTo4m87YmUAtmdXw==', 'Content-Length': '5242880', 'Expect': '100-Continue'} 2022-09-03T06:46:25.012 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:25.012 DEBUG:boto:Port: 8000 2022-09-03T06:46:25.013 DEBUG:boto:Params: {} 2022-09-03T06:46:25.013 DEBUG:boto:Token: None 2022-09-03T06:46:25.014 DEBUG:boto:StringToSign: PUT dIQ6OrGTo4m87YmUAtmdXw== application/octet-stream Sat, 03 Sep 2022 06:46:25 GMT /avysmw-65/MULTIPART?partNumber=2&uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BaSinY1BXQB4RabFgKO8EYlJ0aY= 2022-09-03T06:46:25.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'dIQ6OrGTo4m87YmUAtmdXw==', 'Content-Length': '5242880', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:46:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BaSinY1BXQB4RabFgKO8EYlJ0aY='} 2022-09-03T06:46:25.076 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"74843a3ab193a389bced899402d99d5f"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000fb9cfe439181de3c-006312f841-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:25.101 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.103 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.104 DEBUG:boto:path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=3 2022-09-03T06:46:25.104 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=3 2022-09-03T06:46:25.105 DEBUG:boto:Method: PUT 2022-09-03T06:46:25.106 DEBUG:boto:Path: /avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=3 2022-09-03T06:46:25.107 DEBUG:boto:Data: 2022-09-03T06:46:25.108 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'e4RW4edMN49Fhh9TYZ51tg==', 'Content-Length': '5242880', 'Expect': '100-Continue'} 2022-09-03T06:46:25.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:25.110 DEBUG:boto:Port: 8000 2022-09-03T06:46:25.110 DEBUG:boto:Params: {} 2022-09-03T06:46:25.111 DEBUG:boto:Token: None 2022-09-03T06:46:25.112 DEBUG:boto:StringToSign: PUT e4RW4edMN49Fhh9TYZ51tg== application/octet-stream Sat, 03 Sep 2022 06:46:25 GMT /avysmw-65/MULTIPART?partNumber=3&uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zzJpjUHqxzvsMEQUftpp2h7ADZ0= 2022-09-03T06:46:25.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'e4RW4edMN49Fhh9TYZ51tg==', 'Content-Length': '5242880', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:46:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zzJpjUHqxzvsMEQUftpp2h7ADZ0='} 2022-09-03T06:46:25.169 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"7b8456e1e74c378f45861f53619e75b6"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000001bba98117fc147a6-006312f841-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:25.194 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.195 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.195 DEBUG:boto:path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=4 2022-09-03T06:46:25.196 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=4 2022-09-03T06:46:25.196 DEBUG:boto:Method: PUT 2022-09-03T06:46:25.197 DEBUG:boto:Path: /avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW&partNumber=4 2022-09-03T06:46:25.197 DEBUG:boto:Data: 2022-09-03T06:46:25.198 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'l+jDi4yPcIlVvEiQwDQZTQ==', 'Content-Length': '5242880', 'Expect': '100-Continue'} 2022-09-03T06:46:25.199 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:25.199 DEBUG:boto:Port: 8000 2022-09-03T06:46:25.200 DEBUG:boto:Params: {} 2022-09-03T06:46:25.200 DEBUG:boto:Token: None 2022-09-03T06:46:25.201 DEBUG:boto:StringToSign: PUT l+jDi4yPcIlVvEiQwDQZTQ== application/octet-stream Sat, 03 Sep 2022 06:46:25 GMT /avysmw-65/MULTIPART?partNumber=4&uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hXSF5B+MwIb0jNKNQIlcdDoQoJY= 2022-09-03T06:46:25.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'l+jDi4yPcIlVvEiQwDQZTQ==', 'Content-Length': '5242880', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:46:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hXSF5B+MwIb0jNKNQIlcdDoQoJY='} 2022-09-03T06:46:25.262 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"97e8c38b8c8f708955bc4890c034194d"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000ea6aa09a5304fd19-006312f841-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:25.263 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.264 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.265 DEBUG:boto:path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.265 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.266 DEBUG:boto:Method: GET 2022-09-03T06:46:25.266 DEBUG:boto:Path: /avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.267 DEBUG:boto:Data: 2022-09-03T06:46:25.267 DEBUG:boto:Headers: {} 2022-09-03T06:46:25.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:25.269 DEBUG:boto:Port: 8000 2022-09-03T06:46:25.270 DEBUG:boto:Params: {} 2022-09-03T06:46:25.270 DEBUG:boto:Token: None 2022-09-03T06:46:25.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:46:25 GMT /avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8qcVnURXTWgJ3QJY3qZp6MburDI= 2022-09-03T06:46:25.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8qcVnURXTWgJ3QJY3qZp6MburDI=', 'Content-Length': '0'} 2022-09-03T06:46:25.276 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e5541bce54b2595-006312f841-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:25.279 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.295 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:46:25.295 DEBUG:boto:path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.296 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.296 DEBUG:boto:Method: POST 2022-09-03T06:46:25.297 DEBUG:boto:Path: /avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.297 DEBUG:boto:Data: 1 "79b281060d337b9b2b84ccf390adcf74" 2 "74843a3ab193a389bced899402d99d5f" 3 "7b8456e1e74c378f45861f53619e75b6" 4 "97e8c38b8c8f708955bc4890c034194d" 2022-09-03T06:46:25.298 DEBUG:boto:Headers: {'Content-Type': 'text/xml'} 2022-09-03T06:46:25.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:25.299 DEBUG:boto:Port: 8000 2022-09-03T06:46:25.300 DEBUG:boto:Params: {} 2022-09-03T06:46:25.300 DEBUG:boto:Token: None 2022-09-03T06:46:25.308 DEBUG:boto:StringToSign: POST text/xml Sat, 03 Sep 2022 06:46:25 GMT /avysmw-65/MULTIPART?uploadId=2~Yq4untBZXd9qb8JIrHhz7LUGzog_rNW 2022-09-03T06:46:25.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T1dVKw0rGBt8/YmDfCBkjLnnUAc= 2022-09-03T06:46:25.310 DEBUG:boto:Final headers: {'Content-Type': 'text/xml', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:25 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T1dVKw0rGBt8/YmDfCBkjLnnUAc=', 'Content-Length': '460'} 2022-09-03T06:46:25.326 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000bf45a66fee503dc2-006312f841-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '291'), ('Date', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:25.366 DEBUG:boto:http://smithi062.front.sepia.ceph.com:8000/avysmw-65/MULTIPARTavysmw-65MULTIPART 2022-09-03T06:46:25.369 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:46:25.370 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:46:25.372 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:46:25.431 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:25.432 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:25.478 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:46:25.496 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:46:25.497 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:46:25.497 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:46:25.497 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:46:25.498 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:46:25.498 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:46:25.498 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187583.311178_378.1", 2022-09-03T06:46:25.498 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:46:23.311178Z" 2022-09-03T06:46:25.499 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:46:25.499 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:46:25.499 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:46:25.499 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:46:25.500 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:46:25.500 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:46:25.500 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:46:25.501 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:46:25.501 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:46:25.501 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:46:25.503 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662187583.311178_378.1', 2: '', 3: ''} 2022-09-03T06:46:25.504 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:46:25.504 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:46:25.505 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:46:25.505 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:46:25.575 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:25.575 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:25.667 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:25.665+0000 7fa51a5c0600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:46:25.670 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:25.668+0000 7fa51a5c0600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:46:25.726 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:46:25.727 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:46:25.727 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:46:25.728 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:46:25.728 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:46:25.728 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:46:25.728 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:46:25.729 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:25.729 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:46:25.729 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:46:25.729 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:46:25.730 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:46:25.730 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:46:25.730 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:46:25.730 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:46:25.731 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:46:25.731 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:46:25.731 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:46:25.731 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:46:25.732 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:25.732 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:25.732 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:46:25.732 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:46:25.733 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:46:25.733 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:46:25.733 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187583.311178_378.1", 2022-09-03T06:46:25.733 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:46:25.734 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:46:25.734 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:46:25.734 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:46:23.311178Z", 2022-09-03T06:46:25.735 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:46:25.735 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:25.735 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:25.735 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:46:25.736 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:46:25.736 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:46:25.736 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:46:25.736 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:46:25.737 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:46:25.737 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:46:25.737 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:46:25.738 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:46:25.738 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:46:25.738 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:25.738 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:25.739 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:46:25.739 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:46:25.739 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:46:25.740 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:46:25.740 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:46:25.740 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:46:25.740 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:46:25.741 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:46:25.741 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:46:25.741 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:46:25.742 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:25.742 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:25.742 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:46:25.742 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:25.743 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:46:25.743 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:46:25.743 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:46:25.744 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:25.744 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:46:25Z" 2022-09-03T06:46:25.744 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:46:25.745 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187583.311178_378.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:46:23.311178Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:46:25Z" } 2022-09-03T06:46:25.746 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187583.311178_378.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:46:23.311178Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:46:25.747 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187583.311178_378.1', 2: '', 3: ''} 2022-09-03T06:46:25.748 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187583.311178_378.1', 2: '', 3: ''} 2022-09-03T06:46:25.748 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:46:25.749 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:46:25.750 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:46:25.750 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:46:25.751 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:46:25.810 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:25.810 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:25.856 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:25.854+0000 7fc0fd2d9600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:46:25.857 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:25.855+0000 7fc0fd2d9600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:46:25.873 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:46:25.873 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:46:25.874 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:46:25.874 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:46:25.874 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:46:25.875 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:46:25.875 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:46:25.875 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:46:25.875 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:46:25.876 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:46:25.876 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:46:25.876 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:46:25.877 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:46:25.877 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:46:25.877 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:46:25.878 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:46:25.879 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:46:25.879 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:46:25.880 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:46:25.880 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:46:25.881 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:46:25.881 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:46:25.881 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:46:25.882 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:46:25.882 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:46:25.883 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187583.311178_378.1", 2022-09-03T06:46:25.883 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:46:25.884 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:46:25.884 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:46:25.885 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:46:23.311178Z", 2022-09-03T06:46:25.885 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:46:25.886 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:46:25.886 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:46:25.886 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:46:25.887 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:46:25.887 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:46:25.888 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:46:25.888 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:46:25.888 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:46:25.889 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:46:25.889 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:46:25.890 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:46:25.890 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:46:25.891 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:46:25.891 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:46:25.892 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:46:25.893 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:46:25.893 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:46:25.894 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:46:25.894 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:46:25.894 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:46:25.895 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:46:25.895 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:46:25.896 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:46:25.896 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:46:25.897 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:46:25.897 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:46:25.897 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:46:25.898 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:46:25.898 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:46:25.899 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:46:25.899 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:46:25.900 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:46:25.900 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:46:25Z" 2022-09-03T06:46:25.901 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:46:25.902 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187583.311178_378.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:46:23.311178Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:46:25Z" } 2022-09-03T06:46:25.903 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187583.311178_378.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:46:23.311178Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:46:25.906 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187583.311178_378.1', 2: '', 3: ''} 2022-09-03T06:46:25.907 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187583.311178_378.1', 2: '', 3: ''} 2022-09-03T06:46:25.908 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:46:25.908 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:46:25.909 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:46:25.910 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:46:25.911 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:46:25.974 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:25.975 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:26.022 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:26.019+0000 7fed2cb73600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:46:26.026 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:26.024+0000 7fed2cb73600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:46:26.046 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:46:26.047 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:46:26.047 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:46:26.047 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:46:26.048 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:46:26.048 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:46:26.048 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:46:26.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:26.049 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:46:26.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:46:26.049 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:46:26.049 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:46:26.050 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:46:26.050 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:46:26.050 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:46:26.051 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:46:26.052 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:46:26.052 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:46:26.052 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:46:26.053 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:26.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:26.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:46:26.054 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:46:26.054 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:46:26.054 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:46:26.054 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187583.311178_378.1", 2022-09-03T06:46:26.055 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:46:26.055 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:46:26.055 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:46:26.055 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:46:23.311178Z", 2022-09-03T06:46:26.056 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:46:26.056 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:26.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:26.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:46:26.057 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:46:26.057 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:46:26.057 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:46:26.058 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:46:26.058 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:46:26.058 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:46:26.058 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:46:26.059 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:46:26.059 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:46:26.060 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:26.060 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:26.060 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:46:26.060 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:46:26.061 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:46:26.061 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:46:26.061 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:46:26.061 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:46:26.062 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:46:26.062 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:46:26.062 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:46:26.062 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:46:26.063 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:26.063 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:46:26.064 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:46:26.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:26.064 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:46:26.064 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:46:26.065 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:46:26.065 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:46:26.065 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:46:26Z" 2022-09-03T06:46:26.065 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:46:26.067 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187583.311178_378.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:46:23.311178Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:46:26Z" } 2022-09-03T06:46:26.067 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187583.311178_378.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:46:23.311178Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:46:26.069 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187583.311178_378.1', 2: '', 3: ''} 2022-09-03T06:46:26.069 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187583.311178_378.1', 2: '', 3: ''} 2022-09-03T06:46:26.070 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:46:26.070 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-65 2022-09-03T06:46:26.071 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:26.071 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:26.072 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-65 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:46:26.134 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:26.134 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:26.205 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:26.202+0000 7efed515b600 1 bucket sync caught up with source: 2022-09-03T06:46:26.206 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000011.49.5, , , , , ] 2022-09-03T06:46:26.207 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000011.49.5, , , , , ] 2022-09-03T06:46:26.208 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:26.202+0000 7efed515b600 0 bucket checkpoint complete 2022-09-03T06:46:26.219 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-65 2022-09-03T06:46:26.220 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:26.221 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:26.223 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-65 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:46:26.291 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:26.292 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:26.332 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:26.331+0000 7f609119c600 1 waiting to reach incremental sync.. 2022-09-03T06:46:31.341 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:31.331+0000 7f609119c600 1 waiting to reach incremental sync.. 2022-09-03T06:46:36.334 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:36.332+0000 7f609119c600 1 waiting to reach incremental sync.. 2022-09-03T06:46:41.334 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:41.332+0000 7f609119c600 1 waiting to reach incremental sync.. 2022-09-03T06:46:46.342 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:46.333+0000 7f609119c600 1 bucket sync caught up with source: 2022-09-03T06:46:46.346 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000011.49.5, , , , , ] 2022-09-03T06:46:46.347 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000011.49.5, , , , , ] 2022-09-03T06:46:46.347 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:46.333+0000 7f609119c600 0 bucket checkpoint complete 2022-09-03T06:46:46.358 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-65 2022-09-03T06:46:46.359 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:46.360 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:46.360 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-65 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:46:46.735 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:46.737 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:47.791 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:47.789+0000 7f2c4ab0f600 1 bucket sync caught up with source: 2022-09-03T06:46:47.792 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000011.49.5, , , , , ] 2022-09-03T06:46:47.792 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000011.49.5, , , , , ] 2022-09-03T06:46:47.793 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:47.789+0000 7f2c4ab0f600 0 bucket checkpoint complete 2022-09-03T06:46:47.821 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-65 2022-09-03T06:46:47.822 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:47.822 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:47.823 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-65 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:46:47.891 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:47.892 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:47.941 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:47.938+0000 7f89b0f36600 1 bucket sync caught up with source: 2022-09-03T06:46:47.941 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000001.346.5, , , , , ] 2022-09-03T06:46:47.941 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000001.346.5, , , , , ] 2022-09-03T06:46:47.942 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:47.938+0000 7f89b0f36600 0 bucket checkpoint complete 2022-09-03T06:46:47.959 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-65 2022-09-03T06:46:47.960 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:47.960 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:47.961 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-65 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:46:48.021 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:48.022 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:48.085 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:48.080+0000 7fa6ac5b0600 1 bucket sync caught up with source: 2022-09-03T06:46:48.086 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000001.346.5, , , , , ] 2022-09-03T06:46:48.086 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000001.346.5, , , , , ] 2022-09-03T06:46:48.087 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:48.080+0000 7fa6ac5b0600 0 bucket checkpoint complete 2022-09-03T06:46:48.103 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-65 2022-09-03T06:46:48.103 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-65 2022-09-03T06:46:48.104 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:48.104 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:48.105 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-65 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:46:48.163 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:48.164 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:49.206 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:49.203+0000 7f910a6b3600 1 waiting to reach incremental sync.. 2022-09-03T06:46:54.206 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:54.204+0000 7f910a6b3600 1 waiting to reach incremental sync.. 2022-09-03T06:46:59.209 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:59.205+0000 7f910a6b3600 1 bucket sync caught up with source: 2022-09-03T06:46:59.209 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000001.146.5, , , , , ] 2022-09-03T06:46:59.210 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000001.146.5, , , , , ] 2022-09-03T06:46:59.210 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:46:59.205+0000 7f910a6b3600 0 bucket checkpoint complete 2022-09-03T06:46:59.228 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-65 2022-09-03T06:46:59.229 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:59.230 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-65', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:46:59.231 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-65 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:46:59.352 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:46:59.353 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:46:59.483 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:59.481+0000 7f404f7da600 1 bucket sync caught up with source: 2022-09-03T06:46:59.484 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.146.5, , , , , ] 2022-09-03T06:46:59.485 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.146.5, , , , , ] 2022-09-03T06:46:59.485 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:46:59.481+0000 7f404f7da600 0 bucket checkpoint complete 2022-09-03T06:46:59.559 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-65 2022-09-03T06:46:59.561 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-65 zones={test-zone2, test-zone1} 2022-09-03T06:46:59.562 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:46:59.563 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:46:59.564 DEBUG:boto:Method: HEAD 2022-09-03T06:46:59.565 DEBUG:boto:Path: /avysmw-65/ 2022-09-03T06:46:59.565 DEBUG:boto:Data: 2022-09-03T06:46:59.566 DEBUG:boto:Headers: {} 2022-09-03T06:46:59.567 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:46:59.569 DEBUG:boto:Port: 8000 2022-09-03T06:46:59.570 DEBUG:boto:Params: {} 2022-09-03T06:46:59.580 DEBUG:boto:Token: None 2022-09-03T06:46:59.586 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:46:59 GMT /avysmw-65/ 2022-09-03T06:46:59.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v9svrvh/UvwSdvyzVTFtUylNnQw= 2022-09-03T06:46:59.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v9svrvh/UvwSdvyzVTFtUylNnQw=', 'Content-Length': '0'} 2022-09-03T06:46:59.596 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '20971520'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005b485699e73b3abe-006312f863-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:46:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:59.598 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:46:59.599 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:46:59.600 DEBUG:boto:Method: HEAD 2022-09-03T06:46:59.601 DEBUG:boto:Path: /avysmw-65/ 2022-09-03T06:46:59.602 DEBUG:boto:Data: 2022-09-03T06:46:59.604 DEBUG:boto:Headers: {} 2022-09-03T06:46:59.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:59.605 DEBUG:boto:Port: 8000 2022-09-03T06:46:59.606 DEBUG:boto:Params: {} 2022-09-03T06:46:59.606 DEBUG:boto:Token: None 2022-09-03T06:46:59.607 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:46:59 GMT /avysmw-65/ 2022-09-03T06:46:59.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v9svrvh/UvwSdvyzVTFtUylNnQw= 2022-09-03T06:46:59.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v9svrvh/UvwSdvyzVTFtUylNnQw=', 'Content-Length': '0'} 2022-09-03T06:46:59.612 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '20971520'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c9b2b5b640a807fc-006312f863-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:46:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:59.612 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:46:59.613 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:46:59.615 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:46:59.616 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.617 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.618 DEBUG:boto:Method: GET 2022-09-03T06:46:59.619 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.620 DEBUG:boto:Data: 2022-09-03T06:46:59.621 DEBUG:boto:Headers: {} 2022-09-03T06:46:59.622 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:46:59.623 DEBUG:boto:Port: 8000 2022-09-03T06:46:59.624 DEBUG:boto:Params: {} 2022-09-03T06:46:59.625 DEBUG:boto:Token: None 2022-09-03T06:46:59.626 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:46:59 GMT /avysmw-65/?versions 2022-09-03T06:46:59.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nC2fdNIQ19OqnGA3A80AlM90OMM= 2022-09-03T06:46:59.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nC2fdNIQ19OqnGA3A80AlM90OMM=', 'Content-Length': '0'} 2022-09-03T06:46:59.633 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000071cad97e31f01562-006312f863-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:46:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:59.634 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:46:59.636 DEBUG:tasks.rgw_multi.zone_rados:o=MULTIPART 2022-09-03T06:46:59.637 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:46:59.638 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:46:59.639 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:46:59.640 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.640 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.641 DEBUG:boto:Method: GET 2022-09-03T06:46:59.642 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.642 DEBUG:boto:Data: 2022-09-03T06:46:59.643 DEBUG:boto:Headers: {} 2022-09-03T06:46:59.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:59.644 DEBUG:boto:Port: 8000 2022-09-03T06:46:59.644 DEBUG:boto:Params: {} 2022-09-03T06:46:59.645 DEBUG:boto:Token: None 2022-09-03T06:46:59.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:46:59 GMT /avysmw-65/?versions 2022-09-03T06:46:59.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nC2fdNIQ19OqnGA3A80AlM90OMM= 2022-09-03T06:46:59.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nC2fdNIQ19OqnGA3A80AlM90OMM=', 'Content-Length': '0'} 2022-09-03T06:46:59.651 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bd1511a470211630-006312f863-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:46:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:59.652 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:46:59.654 DEBUG:tasks.rgw_multi.zone_rados:o=MULTIPART 2022-09-03T06:46:59.655 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:46:59.656 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:46:59.657 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.658 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.659 DEBUG:boto:Method: GET 2022-09-03T06:46:59.660 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.660 DEBUG:boto:Data: 2022-09-03T06:46:59.661 DEBUG:boto:Headers: {} 2022-09-03T06:46:59.661 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:46:59.662 DEBUG:boto:Port: 8000 2022-09-03T06:46:59.662 DEBUG:boto:Params: {} 2022-09-03T06:46:59.663 DEBUG:boto:Token: None 2022-09-03T06:46:59.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:46:59 GMT /avysmw-65/?versions 2022-09-03T06:46:59.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nC2fdNIQ19OqnGA3A80AlM90OMM= 2022-09-03T06:46:59.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nC2fdNIQ19OqnGA3A80AlM90OMM=', 'Content-Length': '0'} 2022-09-03T06:46:59.668 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cc72636ba1f421c5-006312f863-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:46:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:59.668 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:46:59.669 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:46:59.670 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:46:59.671 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.671 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.672 DEBUG:boto:Method: GET 2022-09-03T06:46:59.672 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:46:59.673 DEBUG:boto:Data: 2022-09-03T06:46:59.673 DEBUG:boto:Headers: {} 2022-09-03T06:46:59.674 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:59.675 DEBUG:boto:Port: 8000 2022-09-03T06:46:59.675 DEBUG:boto:Params: {} 2022-09-03T06:46:59.676 DEBUG:boto:Token: None 2022-09-03T06:46:59.677 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:46:59 GMT /avysmw-65/?versions 2022-09-03T06:46:59.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nC2fdNIQ19OqnGA3A80AlM90OMM= 2022-09-03T06:46:59.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nC2fdNIQ19OqnGA3A80AlM90OMM=', 'Content-Length': '0'} 2022-09-03T06:46:59.683 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008bfbcca9508640a0-006312f863-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:46:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:59.685 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:46:59.686 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2022-09-03T06:46:59.687 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:46:59.688 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:46:59.689 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:46:59.690 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:46:59.691 DEBUG:boto:Method: GET 2022-09-03T06:46:59.692 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:46:59.693 DEBUG:boto:Data: 2022-09-03T06:46:59.693 DEBUG:boto:Headers: {} 2022-09-03T06:46:59.694 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:46:59.695 DEBUG:boto:Port: 8000 2022-09-03T06:46:59.695 DEBUG:boto:Params: {} 2022-09-03T06:46:59.696 DEBUG:boto:Token: None 2022-09-03T06:46:59.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:46:59 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:46:59.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TpeOo4oThZxLhpp8AuvZEQGiRYY= 2022-09-03T06:46:59.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TpeOo4oThZxLhpp8AuvZEQGiRYY=', 'Content-Length': '0'} 2022-09-03T06:46:59.711 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000c7121f7bd15ca8e1-006312f863-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:46:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:46:59.841 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:46:59.842 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:46:59.843 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:46:59.843 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:46:59.844 DEBUG:boto:Method: GET 2022-09-03T06:46:59.844 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:46:59.845 DEBUG:boto:Data: 2022-09-03T06:46:59.845 DEBUG:boto:Headers: {} 2022-09-03T06:46:59.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:46:59.847 DEBUG:boto:Port: 8000 2022-09-03T06:46:59.848 DEBUG:boto:Params: {} 2022-09-03T06:46:59.848 DEBUG:boto:Token: None 2022-09-03T06:46:59.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:46:59 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:46:59.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TpeOo4oThZxLhpp8AuvZEQGiRYY= 2022-09-03T06:46:59.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:46:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TpeOo4oThZxLhpp8AuvZEQGiRYY=', 'Content-Length': '0'} 2022-09-03T06:46:59.855 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000d196d118429b620e-006312f863-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:46:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.003 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.004 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.005 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.006 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.006 DEBUG:boto:Method: HEAD 2022-09-03T06:47:00.007 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.007 DEBUG:boto:Data: 2022-09-03T06:47:00.008 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.008 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:00.009 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.010 DEBUG:boto:Params: {} 2022-09-03T06:47:00.010 DEBUG:boto:Token: None 2022-09-03T06:47:00.011 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hPnxC/pBYjm5aF+jmEZJMHP6AiE= 2022-09-03T06:47:00.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hPnxC/pBYjm5aF+jmEZJMHP6AiE=', 'Content-Length': '0'} 2022-09-03T06:47:00.014 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000a1ef15462701ab86-006312f864-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.015 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.016 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.016 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.017 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.017 DEBUG:boto:Method: HEAD 2022-09-03T06:47:00.018 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.018 DEBUG:boto:Data: 2022-09-03T06:47:00.019 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.019 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:00.020 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.020 DEBUG:boto:Params: {} 2022-09-03T06:47:00.021 DEBUG:boto:Token: None 2022-09-03T06:47:00.022 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hPnxC/pBYjm5aF+jmEZJMHP6AiE= 2022-09-03T06:47:00.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hPnxC/pBYjm5aF+jmEZJMHP6AiE=', 'Content-Length': '0'} 2022-09-03T06:47:00.025 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000fbd1abc095833d3a-006312f864-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.026 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2022-09-03T06:47:00.027 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.027 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.028 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.028 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.029 DEBUG:boto:Method: GET 2022-09-03T06:47:00.030 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.030 DEBUG:boto:Data: 2022-09-03T06:47:00.031 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.032 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:00.032 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.033 DEBUG:boto:Params: {} 2022-09-03T06:47:00.034 DEBUG:boto:Token: None 2022-09-03T06:47:00.035 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4mNv373qZhTKTwPIuzsonHCFpE8= 2022-09-03T06:47:00.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4mNv373qZhTKTwPIuzsonHCFpE8=', 'Content-Length': '0'} 2022-09-03T06:47:00.069 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx000001723f5da1733f866-006312f864-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.265 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.266 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.267 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.267 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.268 DEBUG:boto:Method: GET 2022-09-03T06:47:00.268 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.269 DEBUG:boto:Data: 2022-09-03T06:47:00.269 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.270 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:00.271 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.271 DEBUG:boto:Params: {} 2022-09-03T06:47:00.272 DEBUG:boto:Token: None 2022-09-03T06:47:00.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:00.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4mNv373qZhTKTwPIuzsonHCFpE8= 2022-09-03T06:47:00.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4mNv373qZhTKTwPIuzsonHCFpE8=', 'Content-Length': '0'} 2022-09-03T06:47:00.281 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx000003fb1e42071295786-006312f864-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.422 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.423 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.424 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.425 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.425 DEBUG:boto:Method: GET 2022-09-03T06:47:00.426 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.426 DEBUG:boto:Data: 2022-09-03T06:47:00.427 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.427 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:00.428 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.429 DEBUG:boto:Params: {} 2022-09-03T06:47:00.429 DEBUG:boto:Token: None 2022-09-03T06:47:00.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/ 2022-09-03T06:47:00.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RySlBm8tYU9aWytMHON71EkCVCg= 2022-09-03T06:47:00.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RySlBm8tYU9aWytMHON71EkCVCg=', 'Content-Length': '0'} 2022-09-03T06:47:00.435 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000810ce966e92d3b0a-006312f864-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.436 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:00.437 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.437 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.438 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.439 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.439 DEBUG:boto:Method: GET 2022-09-03T06:47:00.440 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.441 DEBUG:boto:Data: 2022-09-03T06:47:00.442 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:00.443 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.455 DEBUG:boto:Params: {} 2022-09-03T06:47:00.456 DEBUG:boto:Token: None 2022-09-03T06:47:00.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/ 2022-09-03T06:47:00.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RySlBm8tYU9aWytMHON71EkCVCg= 2022-09-03T06:47:00.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RySlBm8tYU9aWytMHON71EkCVCg=', 'Content-Length': '0'} 2022-09-03T06:47:00.461 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003305f1604a3cdf07-006312f864-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.462 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:00.462 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.463 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.464 DEBUG:boto:Method: HEAD 2022-09-03T06:47:00.465 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:00.466 DEBUG:boto:Data: 2022-09-03T06:47:00.466 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.467 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:00.467 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.468 DEBUG:boto:Params: {} 2022-09-03T06:47:00.469 DEBUG:boto:Token: None 2022-09-03T06:47:00.469 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:00.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SujzY52cAyK0AxhQHEUctnmELUw= 2022-09-03T06:47:00.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SujzY52cAyK0AxhQHEUctnmELUw=', 'Content-Length': '0'} 2022-09-03T06:47:00.473 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx0000008aa191c8c052246-006312f864-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.474 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.475 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.476 DEBUG:boto:Method: HEAD 2022-09-03T06:47:00.476 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:00.477 DEBUG:boto:Data: 2022-09-03T06:47:00.477 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:00.481 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.482 DEBUG:boto:Params: {} 2022-09-03T06:47:00.482 DEBUG:boto:Token: None 2022-09-03T06:47:00.483 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:00.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SujzY52cAyK0AxhQHEUctnmELUw= 2022-09-03T06:47:00.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SujzY52cAyK0AxhQHEUctnmELUw=', 'Content-Length': '0'} 2022-09-03T06:47:00.487 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000adda0cbdd58fa870-006312f864-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.488 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2022-09-03T06:47:00.488 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.489 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.489 DEBUG:boto:Method: GET 2022-09-03T06:47:00.490 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:00.491 DEBUG:boto:Data: 2022-09-03T06:47:00.491 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.492 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:00.493 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.494 DEBUG:boto:Params: {} 2022-09-03T06:47:00.495 DEBUG:boto:Token: None 2022-09-03T06:47:00.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:00.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DQZeo6nk0iisBO65qZsHZ1aV2hE= 2022-09-03T06:47:00.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DQZeo6nk0iisBO65qZsHZ1aV2hE=', 'Content-Length': '0'} 2022-09-03T06:47:00.513 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000d2cc7fa694728257-006312f864-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.687 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.688 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:00.689 DEBUG:boto:Method: GET 2022-09-03T06:47:00.690 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:00.690 DEBUG:boto:Data: 2022-09-03T06:47:00.691 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.692 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:00.693 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.694 DEBUG:boto:Params: {} 2022-09-03T06:47:00.695 DEBUG:boto:Token: None 2022-09-03T06:47:00.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:00.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DQZeo6nk0iisBO65qZsHZ1aV2hE= 2022-09-03T06:47:00.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DQZeo6nk0iisBO65qZsHZ1aV2hE=', 'Content-Length': '0'} 2022-09-03T06:47:00.707 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000d03f5fb4204e2c20-006312f864-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.868 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.869 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.870 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.870 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.871 DEBUG:boto:Method: GET 2022-09-03T06:47:00.871 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.872 DEBUG:boto:Data: 2022-09-03T06:47:00.872 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.873 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:00.873 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.874 DEBUG:boto:Params: {} 2022-09-03T06:47:00.874 DEBUG:boto:Token: None 2022-09-03T06:47:00.875 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/ 2022-09-03T06:47:00.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RySlBm8tYU9aWytMHON71EkCVCg= 2022-09-03T06:47:00.877 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RySlBm8tYU9aWytMHON71EkCVCg=', 'Content-Length': '0'} 2022-09-03T06:47:00.879 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000799b149a8260c923-006312f864-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.880 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:00.881 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.882 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.883 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.883 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.884 DEBUG:boto:Method: GET 2022-09-03T06:47:00.884 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:00.885 DEBUG:boto:Data: 2022-09-03T06:47:00.885 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:00.886 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.887 DEBUG:boto:Params: {} 2022-09-03T06:47:00.887 DEBUG:boto:Token: None 2022-09-03T06:47:00.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/ 2022-09-03T06:47:00.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RySlBm8tYU9aWytMHON71EkCVCg= 2022-09-03T06:47:00.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RySlBm8tYU9aWytMHON71EkCVCg=', 'Content-Length': '0'} 2022-09-03T06:47:00.898 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009da52e65988008bb-006312f864-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.900 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:00.901 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-65 zones={test-zone2, test-zone1} 2022-09-03T06:47:00.902 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-65 zones={test-zone3, test-zone1} 2022-09-03T06:47:00.904 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.905 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.906 DEBUG:boto:Method: HEAD 2022-09-03T06:47:00.907 DEBUG:boto:Path: /avysmw-65/ 2022-09-03T06:47:00.908 DEBUG:boto:Data: 2022-09-03T06:47:00.909 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.910 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:00.911 DEBUG:boto:Port: 8001 2022-09-03T06:47:00.912 DEBUG:boto:Params: {} 2022-09-03T06:47:00.913 DEBUG:boto:Token: None 2022-09-03T06:47:00.914 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/ 2022-09-03T06:47:00.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6iy/ewRU4bqqO03JyOEuSmSokck= 2022-09-03T06:47:00.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6iy/ewRU4bqqO03JyOEuSmSokck=', 'Content-Length': '0'} 2022-09-03T06:47:00.923 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '20971520'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000caa29eafca390d91-006312f864-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.924 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.925 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.925 DEBUG:boto:Method: HEAD 2022-09-03T06:47:00.926 DEBUG:boto:Path: /avysmw-65/ 2022-09-03T06:47:00.926 DEBUG:boto:Data: 2022-09-03T06:47:00.927 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:00.928 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.928 DEBUG:boto:Params: {} 2022-09-03T06:47:00.929 DEBUG:boto:Token: None 2022-09-03T06:47:00.930 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/ 2022-09-03T06:47:00.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6iy/ewRU4bqqO03JyOEuSmSokck= 2022-09-03T06:47:00.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6iy/ewRU4bqqO03JyOEuSmSokck=', 'Content-Length': '0'} 2022-09-03T06:47:00.934 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '20971520'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007d867419164864df-006312f864-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.936 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:47:00.937 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.938 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.939 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:00.940 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:00.941 DEBUG:boto:Method: GET 2022-09-03T06:47:00.942 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:47:00.943 DEBUG:boto:Data: 2022-09-03T06:47:00.946 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:00.947 DEBUG:boto:Port: 8001 2022-09-03T06:47:00.948 DEBUG:boto:Params: {} 2022-09-03T06:47:00.948 DEBUG:boto:Token: None 2022-09-03T06:47:00.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/?versions 2022-09-03T06:47:00.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:49qKGuyg1FYIdlvdRcWaJq6RA3E= 2022-09-03T06:47:00.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:49qKGuyg1FYIdlvdRcWaJq6RA3E=', 'Content-Length': '0'} 2022-09-03T06:47:00.958 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067d4be3ae0d8e0cd-006312f864-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.959 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:00.960 DEBUG:tasks.rgw_multi.zone_rados:o=MULTIPART 2022-09-03T06:47:00.960 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:47:00.961 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.962 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.964 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:00.965 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:00.966 DEBUG:boto:Method: GET 2022-09-03T06:47:00.967 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:47:00.968 DEBUG:boto:Data: 2022-09-03T06:47:00.969 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.970 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:00.971 DEBUG:boto:Port: 8000 2022-09-03T06:47:00.972 DEBUG:boto:Params: {} 2022-09-03T06:47:00.973 DEBUG:boto:Token: None 2022-09-03T06:47:00.974 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/?versions 2022-09-03T06:47:00.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:49qKGuyg1FYIdlvdRcWaJq6RA3E= 2022-09-03T06:47:00.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:49qKGuyg1FYIdlvdRcWaJq6RA3E=', 'Content-Length': '0'} 2022-09-03T06:47:00.979 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000599d416ddff19650-006312f864-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.980 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:00.981 DEBUG:tasks.rgw_multi.zone_rados:o=MULTIPART 2022-09-03T06:47:00.982 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.983 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.983 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:00.984 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:00.984 DEBUG:boto:Method: GET 2022-09-03T06:47:00.985 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:47:00.986 DEBUG:boto:Data: 2022-09-03T06:47:00.987 DEBUG:boto:Headers: {} 2022-09-03T06:47:00.987 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:00.988 DEBUG:boto:Port: 8001 2022-09-03T06:47:00.989 DEBUG:boto:Params: {} 2022-09-03T06:47:00.990 DEBUG:boto:Token: None 2022-09-03T06:47:00.990 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:00 GMT /avysmw-65/?versions 2022-09-03T06:47:00.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:49qKGuyg1FYIdlvdRcWaJq6RA3E= 2022-09-03T06:47:00.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:49qKGuyg1FYIdlvdRcWaJq6RA3E=', 'Content-Length': '0'} 2022-09-03T06:47:00.996 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000925813d05f9bcd6c-006312f864-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:00.997 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:00.998 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:00.999 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:00.999 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:01.000 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:01.000 DEBUG:boto:Method: GET 2022-09-03T06:47:01.001 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:47:01.001 DEBUG:boto:Data: 2022-09-03T06:47:01.002 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:01.003 DEBUG:boto:Port: 8000 2022-09-03T06:47:01.004 DEBUG:boto:Params: {} 2022-09-03T06:47:01.004 DEBUG:boto:Token: None 2022-09-03T06:47:01.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/?versions 2022-09-03T06:47:01.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rw2B6axcigvWHIEPTlkSwrtFFBE= 2022-09-03T06:47:01.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rw2B6axcigvWHIEPTlkSwrtFFBE=', 'Content-Length': '0'} 2022-09-03T06:47:01.017 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000057bff9951a36c4ac-006312f865-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.018 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:01.018 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2022-09-03T06:47:01.019 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.020 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.020 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.021 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.021 DEBUG:boto:Method: GET 2022-09-03T06:47:01.022 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.023 DEBUG:boto:Data: 2022-09-03T06:47:01.023 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.024 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:01.024 DEBUG:boto:Port: 8001 2022-09-03T06:47:01.025 DEBUG:boto:Params: {} 2022-09-03T06:47:01.025 DEBUG:boto:Token: None 2022-09-03T06:47:01.026 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ha09WxqeQvBXaz8QwoAIhv9pfz0= 2022-09-03T06:47:01.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ha09WxqeQvBXaz8QwoAIhv9pfz0=', 'Content-Length': '0'} 2022-09-03T06:47:01.036 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000098dcfe5d460d449-006312f865-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.184 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.185 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.185 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.186 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.187 DEBUG:boto:Method: GET 2022-09-03T06:47:01.187 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.188 DEBUG:boto:Data: 2022-09-03T06:47:01.188 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.189 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:01.190 DEBUG:boto:Port: 8000 2022-09-03T06:47:01.191 DEBUG:boto:Params: {} 2022-09-03T06:47:01.191 DEBUG:boto:Token: None 2022-09-03T06:47:01.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ha09WxqeQvBXaz8QwoAIhv9pfz0= 2022-09-03T06:47:01.194 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ha09WxqeQvBXaz8QwoAIhv9pfz0=', 'Content-Length': '0'} 2022-09-03T06:47:01.200 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx0000004b2ae094ab53011-006312f865-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.365 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.366 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.367 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.368 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.372 DEBUG:boto:Method: HEAD 2022-09-03T06:47:01.373 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.374 DEBUG:boto:Data: 2022-09-03T06:47:01.374 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:01.376 DEBUG:boto:Port: 8001 2022-09-03T06:47:01.377 DEBUG:boto:Params: {} 2022-09-03T06:47:01.378 DEBUG:boto:Token: None 2022-09-03T06:47:01.379 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1all/FIi98hkHnkQMfYB3dTiqWo= 2022-09-03T06:47:01.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1all/FIi98hkHnkQMfYB3dTiqWo=', 'Content-Length': '0'} 2022-09-03T06:47:01.384 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx000002f04f0029e3f5bbf-006312f865-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.385 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.386 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.386 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.387 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.387 DEBUG:boto:Method: HEAD 2022-09-03T06:47:01.388 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.388 DEBUG:boto:Data: 2022-09-03T06:47:01.389 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:01.390 DEBUG:boto:Port: 8000 2022-09-03T06:47:01.391 DEBUG:boto:Params: {} 2022-09-03T06:47:01.392 DEBUG:boto:Token: None 2022-09-03T06:47:01.393 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1all/FIi98hkHnkQMfYB3dTiqWo= 2022-09-03T06:47:01.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1all/FIi98hkHnkQMfYB3dTiqWo=', 'Content-Length': '0'} 2022-09-03T06:47:01.401 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000952209f486f162b0-006312f865-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.402 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2022-09-03T06:47:01.402 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.403 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.403 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.404 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.405 DEBUG:boto:Method: GET 2022-09-03T06:47:01.405 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.406 DEBUG:boto:Data: 2022-09-03T06:47:01.406 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:01.408 DEBUG:boto:Port: 8001 2022-09-03T06:47:01.409 DEBUG:boto:Params: {} 2022-09-03T06:47:01.411 DEBUG:boto:Token: None 2022-09-03T06:47:01.412 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ha09WxqeQvBXaz8QwoAIhv9pfz0= 2022-09-03T06:47:01.414 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ha09WxqeQvBXaz8QwoAIhv9pfz0=', 'Content-Length': '0'} 2022-09-03T06:47:01.420 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx000002871b820e218c952-006312f865-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.563 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.565 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.566 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.566 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.567 DEBUG:boto:Method: GET 2022-09-03T06:47:01.568 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.569 DEBUG:boto:Data: 2022-09-03T06:47:01.570 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.571 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:01.571 DEBUG:boto:Port: 8000 2022-09-03T06:47:01.572 DEBUG:boto:Params: {} 2022-09-03T06:47:01.574 DEBUG:boto:Token: None 2022-09-03T06:47:01.575 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:01.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ha09WxqeQvBXaz8QwoAIhv9pfz0= 2022-09-03T06:47:01.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ha09WxqeQvBXaz8QwoAIhv9pfz0=', 'Content-Length': '0'} 2022-09-03T06:47:01.582 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx000004b1c84044dcd88cb-006312f865-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.726 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:01.727 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:01.728 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:01.728 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:01.729 DEBUG:boto:Method: GET 2022-09-03T06:47:01.730 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:01.730 DEBUG:boto:Data: 2022-09-03T06:47:01.731 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:01.732 DEBUG:boto:Port: 8001 2022-09-03T06:47:01.732 DEBUG:boto:Params: {} 2022-09-03T06:47:01.733 DEBUG:boto:Token: None 2022-09-03T06:47:01.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/ 2022-09-03T06:47:01.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gA76cKT8CuNcYYQv4az31UDS5vA= 2022-09-03T06:47:01.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gA76cKT8CuNcYYQv4az31UDS5vA=', 'Content-Length': '0'} 2022-09-03T06:47:01.738 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c7a97b847135df17-006312f865-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.739 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:01.740 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:01.740 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:01.741 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:01.741 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:01.742 DEBUG:boto:Method: GET 2022-09-03T06:47:01.743 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:01.743 DEBUG:boto:Data: 2022-09-03T06:47:01.744 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.744 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:01.745 DEBUG:boto:Port: 8000 2022-09-03T06:47:01.745 DEBUG:boto:Params: {} 2022-09-03T06:47:01.746 DEBUG:boto:Token: None 2022-09-03T06:47:01.747 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/ 2022-09-03T06:47:01.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gA76cKT8CuNcYYQv4az31UDS5vA= 2022-09-03T06:47:01.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gA76cKT8CuNcYYQv4az31UDS5vA=', 'Content-Length': '0'} 2022-09-03T06:47:01.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bd8f339992cf8825-006312f865-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.753 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:01.754 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.754 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.755 DEBUG:boto:Method: HEAD 2022-09-03T06:47:01.756 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:01.756 DEBUG:boto:Data: 2022-09-03T06:47:01.757 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:01.759 DEBUG:boto:Port: 8001 2022-09-03T06:47:01.760 DEBUG:boto:Params: {} 2022-09-03T06:47:01.760 DEBUG:boto:Token: None 2022-09-03T06:47:01.761 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:01.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c5jr+P1tzWSXhzgmGK+tDjl+/9s= 2022-09-03T06:47:01.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c5jr+P1tzWSXhzgmGK+tDjl+/9s=', 'Content-Length': '0'} 2022-09-03T06:47:01.765 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx0000084f074af7939dd22-006312f865-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.765 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.766 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.766 DEBUG:boto:Method: HEAD 2022-09-03T06:47:01.767 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:01.768 DEBUG:boto:Data: 2022-09-03T06:47:01.768 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.769 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:01.769 DEBUG:boto:Port: 8000 2022-09-03T06:47:01.770 DEBUG:boto:Params: {} 2022-09-03T06:47:01.770 DEBUG:boto:Token: None 2022-09-03T06:47:01.771 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:01.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c5jr+P1tzWSXhzgmGK+tDjl+/9s= 2022-09-03T06:47:01.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c5jr+P1tzWSXhzgmGK+tDjl+/9s=', 'Content-Length': '0'} 2022-09-03T06:47:01.774 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx0000002bc7e31bf101343-006312f865-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.775 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2022-09-03T06:47:01.775 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.776 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.776 DEBUG:boto:Method: GET 2022-09-03T06:47:01.777 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:01.778 DEBUG:boto:Data: 2022-09-03T06:47:01.778 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:01.779 DEBUG:boto:Port: 8001 2022-09-03T06:47:01.780 DEBUG:boto:Params: {} 2022-09-03T06:47:01.780 DEBUG:boto:Token: None 2022-09-03T06:47:01.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:01.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ZOuLxWQ/pFCz2GiCYpchc/64J4= 2022-09-03T06:47:01.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ZOuLxWQ/pFCz2GiCYpchc/64J4=', 'Content-Length': '0'} 2022-09-03T06:47:01.790 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx0000080cb03a39d8e0769-006312f865-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:01.890 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.890 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:01.891 DEBUG:boto:Method: GET 2022-09-03T06:47:01.891 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:01.892 DEBUG:boto:Data: 2022-09-03T06:47:01.893 DEBUG:boto:Headers: {} 2022-09-03T06:47:01.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:01.894 DEBUG:boto:Port: 8000 2022-09-03T06:47:01.894 DEBUG:boto:Params: {} 2022-09-03T06:47:01.895 DEBUG:boto:Token: None 2022-09-03T06:47:01.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:01 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:01.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ZOuLxWQ/pFCz2GiCYpchc/64J4= 2022-09-03T06:47:01.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ZOuLxWQ/pFCz2GiCYpchc/64J4=', 'Content-Length': '0'} 2022-09-03T06:47:01.901 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx0000075b64fa8d17842d6-006312f865-1604-test-zone1'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.046 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.047 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.048 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.048 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.049 DEBUG:boto:Method: GET 2022-09-03T06:47:02.050 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.050 DEBUG:boto:Data: 2022-09-03T06:47:02.051 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.052 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.052 DEBUG:boto:Params: {} 2022-09-03T06:47:02.053 DEBUG:boto:Token: None 2022-09-03T06:47:02.054 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/ 2022-09-03T06:47:02.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l9qCcJd2pJjnhTzhm+HlZxdM61c= 2022-09-03T06:47:02.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l9qCcJd2pJjnhTzhm+HlZxdM61c=', 'Content-Length': '0'} 2022-09-03T06:47:02.059 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac333b6d7e3af3f9-006312f866-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.060 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:02.061 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.061 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.062 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.063 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.063 DEBUG:boto:Method: GET 2022-09-03T06:47:02.064 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.064 DEBUG:boto:Data: 2022-09-03T06:47:02.065 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.065 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:02.066 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.067 DEBUG:boto:Params: {} 2022-09-03T06:47:02.068 DEBUG:boto:Token: None 2022-09-03T06:47:02.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/ 2022-09-03T06:47:02.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l9qCcJd2pJjnhTzhm+HlZxdM61c= 2022-09-03T06:47:02.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l9qCcJd2pJjnhTzhm+HlZxdM61c=', 'Content-Length': '0'} 2022-09-03T06:47:02.072 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000135cc05e0f400aae-006312f866-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.073 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:02.074 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-65 zones={test-zone3, test-zone1} 2022-09-03T06:47:02.074 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-65 zones={test-zone3, test-zone2} 2022-09-03T06:47:02.075 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.075 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.076 DEBUG:boto:Method: HEAD 2022-09-03T06:47:02.077 DEBUG:boto:Path: /avysmw-65/ 2022-09-03T06:47:02.077 DEBUG:boto:Data: 2022-09-03T06:47:02.078 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.078 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.079 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.079 DEBUG:boto:Params: {} 2022-09-03T06:47:02.080 DEBUG:boto:Token: None 2022-09-03T06:47:02.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/ 2022-09-03T06:47:02.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YnbadZPQ8u0pm/wmXEDrLyD6LGw= 2022-09-03T06:47:02.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YnbadZPQ8u0pm/wmXEDrLyD6LGw=', 'Content-Length': '0'} 2022-09-03T06:47:02.084 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '20971520'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bbac1d4cd2c1acf3-006312f866-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.085 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.085 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.086 DEBUG:boto:Method: HEAD 2022-09-03T06:47:02.087 DEBUG:boto:Path: /avysmw-65/ 2022-09-03T06:47:02.088 DEBUG:boto:Data: 2022-09-03T06:47:02.089 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.089 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:02.090 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.091 DEBUG:boto:Params: {} 2022-09-03T06:47:02.092 DEBUG:boto:Token: None 2022-09-03T06:47:02.092 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/ 2022-09-03T06:47:02.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YnbadZPQ8u0pm/wmXEDrLyD6LGw= 2022-09-03T06:47:02.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YnbadZPQ8u0pm/wmXEDrLyD6LGw=', 'Content-Length': '0'} 2022-09-03T06:47:02.096 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '20971520'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b86e4cdc4139d79b-006312f866-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.097 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:47:02.098 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.099 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.099 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.100 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.100 DEBUG:boto:Method: GET 2022-09-03T06:47:02.101 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.101 DEBUG:boto:Data: 2022-09-03T06:47:02.102 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.102 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.103 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.103 DEBUG:boto:Params: {} 2022-09-03T06:47:02.104 DEBUG:boto:Token: None 2022-09-03T06:47:02.105 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/?versions 2022-09-03T06:47:02.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tn84EPVBRGthxqSir+RQVn2uzv0= 2022-09-03T06:47:02.106 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tn84EPVBRGthxqSir+RQVn2uzv0=', 'Content-Length': '0'} 2022-09-03T06:47:02.109 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c34da6a430bb7039-006312f866-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.109 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:02.110 DEBUG:tasks.rgw_multi.zone_rados:o=MULTIPART 2022-09-03T06:47:02.111 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:47:02.111 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.112 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.112 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.113 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.114 DEBUG:boto:Method: GET 2022-09-03T06:47:02.114 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.115 DEBUG:boto:Data: 2022-09-03T06:47:02.115 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.116 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:02.116 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.117 DEBUG:boto:Params: {} 2022-09-03T06:47:02.117 DEBUG:boto:Token: None 2022-09-03T06:47:02.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/?versions 2022-09-03T06:47:02.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tn84EPVBRGthxqSir+RQVn2uzv0= 2022-09-03T06:47:02.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tn84EPVBRGthxqSir+RQVn2uzv0=', 'Content-Length': '0'} 2022-09-03T06:47:02.122 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d14e7a4ba94d496-006312f866-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.123 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:02.125 DEBUG:tasks.rgw_multi.zone_rados:o=MULTIPART 2022-09-03T06:47:02.126 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.126 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.127 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.128 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.129 DEBUG:boto:Method: GET 2022-09-03T06:47:02.130 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.131 DEBUG:boto:Data: 2022-09-03T06:47:02.132 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.133 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.134 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.135 DEBUG:boto:Params: {} 2022-09-03T06:47:02.136 DEBUG:boto:Token: None 2022-09-03T06:47:02.137 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/?versions 2022-09-03T06:47:02.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tn84EPVBRGthxqSir+RQVn2uzv0= 2022-09-03T06:47:02.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tn84EPVBRGthxqSir+RQVn2uzv0=', 'Content-Length': '0'} 2022-09-03T06:47:02.143 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ffb3193c8fd806c-006312f866-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.144 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:02.156 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.157 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.158 DEBUG:boto:path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.159 DEBUG:boto:auth_path=/avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.160 DEBUG:boto:Method: GET 2022-09-03T06:47:02.160 DEBUG:boto:Path: /avysmw-65/?versions&max-keys=999 2022-09-03T06:47:02.161 DEBUG:boto:Data: 2022-09-03T06:47:02.162 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.163 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:02.164 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.165 DEBUG:boto:Params: {} 2022-09-03T06:47:02.167 DEBUG:boto:Token: None 2022-09-03T06:47:02.168 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/?versions 2022-09-03T06:47:02.169 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tn84EPVBRGthxqSir+RQVn2uzv0= 2022-09-03T06:47:02.170 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tn84EPVBRGthxqSir+RQVn2uzv0=', 'Content-Length': '0'} 2022-09-03T06:47:02.175 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f352e5b0c183c3e3-006312f866-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.175 DEBUG:boto:b'avysmw-65999falseMULTIPARTnulltrue2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:02.176 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2022-09-03T06:47:02.177 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.178 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.178 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.179 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.179 DEBUG:boto:Method: GET 2022-09-03T06:47:02.180 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.181 DEBUG:boto:Data: 2022-09-03T06:47:02.181 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.182 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.183 DEBUG:boto:Params: {} 2022-09-03T06:47:02.184 DEBUG:boto:Token: None 2022-09-03T06:47:02.184 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zIc/NIqLKz1qv07/2g9GDfLNlzY= 2022-09-03T06:47:02.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zIc/NIqLKz1qv07/2g9GDfLNlzY=', 'Content-Length': '0'} 2022-09-03T06:47:02.190 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000a1476e872c37baa4-006312f866-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.302 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.304 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.304 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.307 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.310 DEBUG:boto:Method: GET 2022-09-03T06:47:02.311 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.312 DEBUG:boto:Data: 2022-09-03T06:47:02.313 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.315 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:02.315 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.316 DEBUG:boto:Params: {} 2022-09-03T06:47:02.316 DEBUG:boto:Token: None 2022-09-03T06:47:02.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zIc/NIqLKz1qv07/2g9GDfLNlzY= 2022-09-03T06:47:02.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zIc/NIqLKz1qv07/2g9GDfLNlzY=', 'Content-Length': '0'} 2022-09-03T06:47:02.331 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000efc23a87321bc816-006312f866-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.451 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.452 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.452 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.453 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.453 DEBUG:boto:Method: HEAD 2022-09-03T06:47:02.454 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.455 DEBUG:boto:Data: 2022-09-03T06:47:02.455 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.457 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.458 DEBUG:boto:Params: {} 2022-09-03T06:47:02.459 DEBUG:boto:Token: None 2022-09-03T06:47:02.460 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dPjvFzYIXKuCruyl2Bn7CSId28M= 2022-09-03T06:47:02.461 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dPjvFzYIXKuCruyl2Bn7CSId28M=', 'Content-Length': '0'} 2022-09-03T06:47:02.474 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx0000063eccc150ee36bcf-006312f866-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.475 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.476 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.476 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.477 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.477 DEBUG:boto:Method: HEAD 2022-09-03T06:47:02.478 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.478 DEBUG:boto:Data: 2022-09-03T06:47:02.479 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.479 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:02.480 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.481 DEBUG:boto:Params: {} 2022-09-03T06:47:02.481 DEBUG:boto:Token: None 2022-09-03T06:47:02.482 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dPjvFzYIXKuCruyl2Bn7CSId28M= 2022-09-03T06:47:02.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dPjvFzYIXKuCruyl2Bn7CSId28M=', 'Content-Length': '0'} 2022-09-03T06:47:02.490 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000f583f210fd0679e2-006312f866-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.491 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2022-09-03T06:47:02.492 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.493 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.494 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.495 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.496 DEBUG:boto:Method: GET 2022-09-03T06:47:02.497 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.498 DEBUG:boto:Data: 2022-09-03T06:47:02.499 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.501 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.501 DEBUG:boto:Params: {} 2022-09-03T06:47:02.502 DEBUG:boto:Token: None 2022-09-03T06:47:02.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zIc/NIqLKz1qv07/2g9GDfLNlzY= 2022-09-03T06:47:02.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zIc/NIqLKz1qv07/2g9GDfLNlzY=', 'Content-Length': '0'} 2022-09-03T06:47:02.509 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000a113416dea0ea48f-006312f866-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.603 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.604 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.604 DEBUG:boto:path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.605 DEBUG:boto:auth_path=/avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.605 DEBUG:boto:Method: GET 2022-09-03T06:47:02.606 DEBUG:boto:Path: /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.606 DEBUG:boto:Data: 2022-09-03T06:47:02.607 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.607 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:02.608 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.608 DEBUG:boto:Params: {} 2022-09-03T06:47:02.609 DEBUG:boto:Token: None 2022-09-03T06:47:02.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART?versionId=null 2022-09-03T06:47:02.610 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zIc/NIqLKz1qv07/2g9GDfLNlzY= 2022-09-03T06:47:02.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zIc/NIqLKz1qv07/2g9GDfLNlzY=', 'Content-Length': '0'} 2022-09-03T06:47:02.621 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000ad165f02fb18be99-006312f866-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.722 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.723 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.724 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.725 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.726 DEBUG:boto:Method: GET 2022-09-03T06:47:02.727 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.728 DEBUG:boto:Data: 2022-09-03T06:47:02.729 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.730 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.731 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.732 DEBUG:boto:Params: {} 2022-09-03T06:47:02.732 DEBUG:boto:Token: None 2022-09-03T06:47:02.733 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/ 2022-09-03T06:47:02.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l9qCcJd2pJjnhTzhm+HlZxdM61c= 2022-09-03T06:47:02.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l9qCcJd2pJjnhTzhm+HlZxdM61c=', 'Content-Length': '0'} 2022-09-03T06:47:02.739 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000030174663cfc03988-006312f866-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.740 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:02.741 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:02.741 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:02.742 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.743 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.743 DEBUG:boto:Method: GET 2022-09-03T06:47:02.744 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:02.744 DEBUG:boto:Data: 2022-09-03T06:47:02.745 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.745 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:02.746 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.747 DEBUG:boto:Params: {} 2022-09-03T06:47:02.748 DEBUG:boto:Token: None 2022-09-03T06:47:02.749 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/ 2022-09-03T06:47:02.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l9qCcJd2pJjnhTzhm+HlZxdM61c= 2022-09-03T06:47:02.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l9qCcJd2pJjnhTzhm+HlZxdM61c=', 'Content-Length': '0'} 2022-09-03T06:47:02.786 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000b1a4136a4f60a4a-006312f866-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.787 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:02.789 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.789 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.791 DEBUG:boto:Method: HEAD 2022-09-03T06:47:02.792 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:02.793 DEBUG:boto:Data: 2022-09-03T06:47:02.794 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.794 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.795 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.796 DEBUG:boto:Params: {} 2022-09-03T06:47:02.797 DEBUG:boto:Token: None 2022-09-03T06:47:02.797 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:02.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:45lgbWXmQVLRip5Vt7hX1CL1bW4= 2022-09-03T06:47:02.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:45lgbWXmQVLRip5Vt7hX1CL1bW4=', 'Content-Length': '0'} 2022-09-03T06:47:02.801 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000752f63c81fc3f041-006312f866-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.801 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.802 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.803 DEBUG:boto:Method: HEAD 2022-09-03T06:47:02.803 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:02.804 DEBUG:boto:Data: 2022-09-03T06:47:02.804 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.805 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:02.805 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.806 DEBUG:boto:Params: {} 2022-09-03T06:47:02.806 DEBUG:boto:Token: None 2022-09-03T06:47:02.807 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:02.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:45lgbWXmQVLRip5Vt7hX1CL1bW4= 2022-09-03T06:47:02.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:45lgbWXmQVLRip5Vt7hX1CL1bW4=', 'Content-Length': '0'} 2022-09-03T06:47:02.810 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000cb71de0e938efcdc-006312f866-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.812 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2022-09-03T06:47:02.813 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.814 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.815 DEBUG:boto:Method: GET 2022-09-03T06:47:02.816 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:02.817 DEBUG:boto:Data: 2022-09-03T06:47:02.818 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:02.820 DEBUG:boto:Port: 8001 2022-09-03T06:47:02.820 DEBUG:boto:Params: {} 2022-09-03T06:47:02.821 DEBUG:boto:Token: None 2022-09-03T06:47:02.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:02.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JvChSSHkcMBFSVZ1Q8+8ING9LWU= 2022-09-03T06:47:02.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JvChSSHkcMBFSVZ1Q8+8ING9LWU=', 'Content-Length': '0'} 2022-09-03T06:47:02.829 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx00000557a549bcea9a75e-006312f866-1608-test-zone3'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:02.929 DEBUG:boto:path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.930 DEBUG:boto:auth_path=/avysmw-65/MULTIPART 2022-09-03T06:47:02.930 DEBUG:boto:Method: GET 2022-09-03T06:47:02.931 DEBUG:boto:Path: /avysmw-65/MULTIPART 2022-09-03T06:47:02.931 DEBUG:boto:Data: 2022-09-03T06:47:02.932 DEBUG:boto:Headers: {} 2022-09-03T06:47:02.933 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:02.933 DEBUG:boto:Port: 8000 2022-09-03T06:47:02.934 DEBUG:boto:Params: {} 2022-09-03T06:47:02.934 DEBUG:boto:Token: None 2022-09-03T06:47:02.935 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:02 GMT /avysmw-65/MULTIPART 2022-09-03T06:47:02.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JvChSSHkcMBFSVZ1Q8+8ING9LWU= 2022-09-03T06:47:02.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JvChSSHkcMBFSVZ1Q8+8ING9LWU=', 'Content-Length': '0'} 2022-09-03T06:47:02.945 DEBUG:boto:Response headers: [('Content-Length', '20971520'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:46:25 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"d07d411772772538cd4b657da660bfb5-4"'), ('x-amz-request-id', 'tx000001f08d0c87b6354f0-006312f866-170f-test-zone2'), ('Content-Type', 'binary/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:03.050 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:03.051 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:03.052 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:03.053 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:03.054 DEBUG:boto:Method: GET 2022-09-03T06:47:03.055 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:03.056 DEBUG:boto:Data: 2022-09-03T06:47:03.057 DEBUG:boto:Headers: {} 2022-09-03T06:47:03.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:47:03.058 DEBUG:boto:Port: 8001 2022-09-03T06:47:03.058 DEBUG:boto:Params: {} 2022-09-03T06:47:03.059 DEBUG:boto:Token: None 2022-09-03T06:47:03.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:03 GMT /avysmw-65/ 2022-09-03T06:47:03.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4n2rEJZfKuCn3A1xtfG2FH71umk= 2022-09-03T06:47:03.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4n2rEJZfKuCn3A1xtfG2FH71umk=', 'Content-Length': '0'} 2022-09-03T06:47:03.065 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000368136e9e07458d8-006312f867-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:03.066 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:03.067 DEBUG:boto:path=/avysmw-65/ 2022-09-03T06:47:03.068 DEBUG:boto:auth_path=/avysmw-65/ 2022-09-03T06:47:03.069 DEBUG:boto:path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:03.070 DEBUG:boto:auth_path=/avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:03.071 DEBUG:boto:Method: GET 2022-09-03T06:47:03.072 DEBUG:boto:Path: /avysmw-65/?prefix=MULTIPART 2022-09-03T06:47:03.073 DEBUG:boto:Data: 2022-09-03T06:47:03.074 DEBUG:boto:Headers: {} 2022-09-03T06:47:03.075 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:03.075 DEBUG:boto:Port: 8000 2022-09-03T06:47:03.076 DEBUG:boto:Params: {} 2022-09-03T06:47:03.077 DEBUG:boto:Token: None 2022-09-03T06:47:03.078 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:03 GMT /avysmw-65/ 2022-09-03T06:47:03.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4n2rEJZfKuCn3A1xtfG2FH71umk= 2022-09-03T06:47:03.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4n2rEJZfKuCn3A1xtfG2FH71umk=', 'Content-Length': '0'} 2022-09-03T06:47:03.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000003e32daa600976c6-006312f867-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:47:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:03.086 DEBUG:boto:b'avysmw-65MULTIPART1000falseMULTIPART2022-09-03T06:46:25.317Z"d07d411772772538cd4b657da660bfb5-4"20971520STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:47:03.087 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-65 zones={test-zone3, test-zone2} 2022-09-03T06:47:03.089 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multipart_object_sync ... ok 2022-09-03T06:47:03.091 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-68 2022-09-03T06:47:03.091 DEBUG:boto:path=/avysmw-68/ 2022-09-03T06:47:03.092 DEBUG:boto:auth_path=/avysmw-68/ 2022-09-03T06:47:03.093 DEBUG:boto:Method: PUT 2022-09-03T06:47:03.094 DEBUG:boto:Path: /avysmw-68/ 2022-09-03T06:47:03.094 DEBUG:boto:Data: 2022-09-03T06:47:03.097 DEBUG:boto:Headers: {} 2022-09-03T06:47:03.098 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:03.098 DEBUG:boto:Port: 8000 2022-09-03T06:47:03.099 DEBUG:boto:Params: {} 2022-09-03T06:47:03.100 DEBUG:boto:Token: None 2022-09-03T06:47:03.101 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:47:03 GMT /avysmw-68/ 2022-09-03T06:47:03.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jL+NKTiFNxMkGaje/GJxtZyeC3w= 2022-09-03T06:47:03.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jL+NKTiFNxMkGaje/GJxtZyeC3w=', 'Content-Length': '0'} 2022-09-03T06:47:03.135 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000f9f67e2fe3d6dc68-006312f867-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:03.137 DEBUG:boto:path=/avysmw-68/testobj-sse-c 2022-09-03T06:47:03.137 DEBUG:boto:auth_path=/avysmw-68/testobj-sse-c 2022-09-03T06:47:03.138 DEBUG:boto:Method: PUT 2022-09-03T06:47:03.139 DEBUG:boto:Path: /avysmw-68/testobj-sse-c 2022-09-03T06:47:03.139 DEBUG:boto:Data: 2022-09-03T06:47:03.140 DEBUG:boto:Headers: {'x-amz-server-side-encryption-customer-algorithm': 'AES256', 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '3FCGuEcom6i4veFJuDiBdQ==', 'Content-Length': '512', 'Expect': '100-Continue'} 2022-09-03T06:47:03.141 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:03.142 DEBUG:boto:Port: 8000 2022-09-03T06:47:03.143 DEBUG:boto:Params: {} 2022-09-03T06:47:03.144 DEBUG:boto:Token: None 2022-09-03T06:47:03.146 DEBUG:boto:StringToSign: PUT 3FCGuEcom6i4veFJuDiBdQ== application/octet-stream Sat, 03 Sep 2022 06:47:03 GMT x-amz-server-side-encryption-customer-algorithm:AES256 x-amz-server-side-encryption-customer-key:pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs= x-amz-server-side-encryption-customer-key-md5:DWygnHRtgiJ77HCm+1rvHw== /avysmw-68/testobj-sse-c 2022-09-03T06:47:03.147 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdsQ20xwwAXdXJPmzZOUmyn+W9E= 2022-09-03T06:47:03.148 DEBUG:boto:Final headers: {'x-amz-server-side-encryption-customer-algorithm': 'AES256', 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '3FCGuEcom6i4veFJuDiBdQ==', 'Content-Length': '512', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:47:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdsQ20xwwAXdXJPmzZOUmyn+W9E='} 2022-09-03T06:47:03.162 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"dc5086b847289ba8b8bde149b8388175"'), ('Accept-Ranges', 'bytes'), ('x-amz-server-side-encryption-customer-algorithm', 'AES256'), ('x-amz-server-side-encryption-customer-key-MD5', 'DWygnHRtgiJ77HCm+1rvHw=='), ('x-amz-request-id', 'tx0000064cfdaac00fce95e-006312f867-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:47:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:03.163 DEBUG:boto:path=/avysmw-68/testobj-sse-kms 2022-09-03T06:47:03.164 DEBUG:boto:auth_path=/avysmw-68/testobj-sse-kms 2022-09-03T06:47:03.164 DEBUG:boto:Method: PUT 2022-09-03T06:47:03.165 DEBUG:boto:Path: /avysmw-68/testobj-sse-kms 2022-09-03T06:47:03.165 DEBUG:boto:Data: 2022-09-03T06:47:03.166 DEBUG:boto:Headers: {'x-amz-server-side-encryption': 'aws:kms', 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '3FCGuEcom6i4veFJuDiBdQ==', 'Content-Length': '512', 'Expect': '100-Continue'} 2022-09-03T06:47:03.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:03.167 DEBUG:boto:Port: 8000 2022-09-03T06:47:03.168 DEBUG:boto:Params: {} 2022-09-03T06:47:03.168 DEBUG:boto:Token: None 2022-09-03T06:47:03.169 DEBUG:boto:StringToSign: PUT 3FCGuEcom6i4veFJuDiBdQ== application/octet-stream Sat, 03 Sep 2022 06:47:03 GMT x-amz-server-side-encryption:aws:kms x-amz-server-side-encryption-aws-kms-key-id:testkey-1 /avysmw-68/testobj-sse-kms 2022-09-03T06:47:03.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cy6IwBeTFWQ2YZDqUgHgIoHFYAc= 2022-09-03T06:47:03.170 DEBUG:boto:Final headers: {'x-amz-server-side-encryption': 'aws:kms', 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': '3FCGuEcom6i4veFJuDiBdQ==', 'Content-Length': '512', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:47:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cy6IwBeTFWQ2YZDqUgHgIoHFYAc='} 2022-09-03T06:47:03.218 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"dc5086b847289ba8b8bde149b8388175"'), ('Accept-Ranges', 'bytes'), ('x-amz-server-side-encryption', 'aws:kms'), ('x-amz-server-side-encryption-aws-kms-key-id', 'testkey-1'), ('x-amz-server-side-encryption-context', 'eyJhd3M6czM6YXJuIjoiYXJuOmF3czpzMzo6cmd3LW11bHRpc2l0ZS10ZXN0LXVzZXI6YXZ5c213LTY4L3Rlc3RvYmotc3NlLWttcyJ9'), ('x-amz-request-id', 'tx00000f1f871d214132709-006312f867-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:47:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:03.219 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:03.220 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:03.221 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:47:03.275 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:03.276 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:03.323 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:47:03.343 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:47:03.344 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:03.344 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:03.344 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:47:03.345 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:03.345 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:03.345 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187623.130327_382.1", 2022-09-03T06:47:03.346 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:47:03.130327Z" 2022-09-03T06:47:03.346 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:03.346 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:03.347 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:03.347 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:47:03.348 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:03.348 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:03.349 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:03.349 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:47:03.349 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:03.349 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:47:03.352 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662187623.130327_382.1', 2: '', 3: ''} 2022-09-03T06:47:03.352 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:47:03.353 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:03.353 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:03.354 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:47:03.449 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:03.450 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:03.504 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:03.502+0000 7f1373ce1600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:03.505 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:03.503+0000 7f1373ce1600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:03.531 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:47:03.531 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:47:03.532 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:47:03.532 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:47:03.532 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:47:03.533 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:47:03.533 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:47:03.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.534 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:47:03.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:03.535 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:47:03.536 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:03.536 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:03.537 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:47:03.537 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:03.538 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:47:03.538 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:03.539 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:47:03.539 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:03.539 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:03.541 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:47:03.541 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:03.541 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:03.541 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187623.130327_382.1", 2022-09-03T06:47:03.542 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:03.542 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:03.543 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:03.543 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:47:03.130327Z", 2022-09-03T06:47:03.544 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:47:03.545 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:03.546 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:47:03.546 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:03.547 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:03.547 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:47:03.547 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:03.547 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:03.548 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:03.548 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:47:03.549 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:03.550 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:03.551 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:47:03.552 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:03.552 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:03.553 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:47:03.553 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:03.554 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:03.554 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:03.555 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:47:03.555 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:03.556 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.556 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.557 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:47:03.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.558 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:47:03.558 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:47:03.559 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:47:03.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.560 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:47:03Z" 2022-09-03T06:47:03.560 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:47:03.561 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187623.130327_382.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:47:03.130327Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:47:03Z" } 2022-09-03T06:47:03.563 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187623.130327_382.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:47:03.130327Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:47:03.564 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187623.130327_382.1', 2: '', 3: ''} 2022-09-03T06:47:03.565 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187623.130327_382.1', 2: '', 3: ''} 2022-09-03T06:47:03.566 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:47:03.567 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:47:03.568 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:03.569 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:03.571 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:47:03.623 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:03.623 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:03.653 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:47:03.652+0000 7f003f155600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:03.654 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:47:03.653+0000 7f003f155600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:03.668 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:47:03.669 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:47:03.669 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:47:03.669 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:47:03.670 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:47:03.670 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:47:03.670 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:47:03.671 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:03.671 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:47:03.671 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:03.671 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:47:03.672 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:47:03.672 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:47:03.672 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:47:03.672 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:47:03.673 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:47:03.673 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:47:03.674 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:47:03.674 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:47:03.674 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:03.675 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:03.675 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:03.676 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:47:03.676 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:47:03.676 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:47:03.676 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187623.130327_382.1", 2022-09-03T06:47:03.677 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:47:03.678 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:47:03.678 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:47:03.678 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:47:03.130327Z", 2022-09-03T06:47:03.679 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:47:03.679 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:03.679 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:03.679 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:03.680 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:47:03.680 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:47:03.680 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:47:03.681 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:03.681 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:47:03.681 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:47:03.681 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:47:03.682 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:47:03.682 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:47:03.682 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:03.682 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:03.683 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:03.683 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:47:03.683 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:47:03.684 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:47:03.684 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:03.684 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:47:03.685 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:47:03.685 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:47:03.685 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:47:03.685 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:47:03.686 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:03.686 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:03.686 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:47:03.687 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:03.687 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:47:03.687 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:47:03.688 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:47:03.688 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:03.688 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:47:03Z" 2022-09-03T06:47:03.688 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:47:03.690 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187623.130327_382.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:47:03.130327Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:47:03Z" } 2022-09-03T06:47:03.691 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187623.130327_382.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:47:03.130327Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:47:03.692 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187623.130327_382.1', 2: '', 3: ''} 2022-09-03T06:47:03.692 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187623.130327_382.1', 2: '', 3: ''} 2022-09-03T06:47:03.693 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:47:03.693 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:47:03.694 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:03.694 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:03.695 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:47:03.798 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:03.799 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:03.845 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:03.841+0000 7f88f4c4e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:03.845 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:03.842+0000 7f88f4c4e600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:03.861 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:47:03.861 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:47:03.862 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:47:03.862 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:47:03.862 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:47:03.863 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:47:03.863 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:47:03.863 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.865 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:47:03.865 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:03.866 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:47:03.867 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:03.867 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:03.867 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:47:03.868 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:03.868 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:47:03.868 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:03.868 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:47:03.869 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:03.869 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.869 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.869 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:03.870 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:47:03.870 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:03.870 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:03.870 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187623.130327_382.1", 2022-09-03T06:47:03.871 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:03.871 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:03.871 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:03.871 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:47:03.130327Z", 2022-09-03T06:47:03.872 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:47:03.872 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.872 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.872 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:03.873 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:47:03.873 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:03.873 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:03.873 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:47:03.874 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:03.874 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:03.874 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:03.875 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:47:03.878 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:03.878 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.878 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.879 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:03.879 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:47:03.880 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:03.881 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:03.881 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:47:03.882 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:03.882 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:03.883 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:03.884 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:47:03.884 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:03.884 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.885 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:03.885 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:47:03.886 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.887 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:47:03.887 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:47:03.887 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:47:03.888 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:03.888 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:47:03Z" 2022-09-03T06:47:03.889 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:47:03.892 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187623.130327_382.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:47:03.130327Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:47:03Z" } 2022-09-03T06:47:03.893 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187623.130327_382.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:47:03.130327Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:47:03.893 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187623.130327_382.1', 2: '', 3: ''} 2022-09-03T06:47:03.894 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187623.130327_382.1', 2: '', 3: ''} 2022-09-03T06:47:03.895 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:47:03.895 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-68', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:47:03.896 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-68', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:47:03.897 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-68 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:47:03.958 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:03.959 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:05.025 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:05.021+0000 7f8737bc1600 1 waiting to reach incremental sync.. 2022-09-03T06:47:10.024 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:10.022+0000 7f8737bc1600 1 bucket sync caught up with source: 2022-09-03T06:47:10.025 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.50.5, , , , 00000000001.40.5, , , , , ] 2022-09-03T06:47:10.026 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.50.5, , , , 00000000001.40.5, , , , , ] 2022-09-03T06:47:10.026 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:10.022+0000 7f8737bc1600 0 bucket checkpoint complete 2022-09-03T06:47:10.042 DEBUG:boto:path=/avysmw-68/ 2022-09-03T06:47:10.043 DEBUG:boto:auth_path=/avysmw-68/ 2022-09-03T06:47:10.044 DEBUG:boto:Method: HEAD 2022-09-03T06:47:10.045 DEBUG:boto:Path: /avysmw-68/ 2022-09-03T06:47:10.046 DEBUG:boto:Data: 2022-09-03T06:47:10.047 DEBUG:boto:Headers: {} 2022-09-03T06:47:10.047 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:10.048 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.049 DEBUG:boto:Params: {} 2022-09-03T06:47:10.050 DEBUG:boto:Token: None 2022-09-03T06:47:10.050 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:10 GMT /avysmw-68/ 2022-09-03T06:47:10.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fwUXzpO0Mfj4gzSCZxLPMfNR3Cg= 2022-09-03T06:47:10.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fwUXzpO0Mfj4gzSCZxLPMfNR3Cg=', 'Content-Length': '0'} 2022-09-03T06:47:10.055 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '1024'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cd06fe80d844f715-006312f86e-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.055 DEBUG:boto:path=/avysmw-68/testobj-sse-c 2022-09-03T06:47:10.056 DEBUG:boto:auth_path=/avysmw-68/testobj-sse-c 2022-09-03T06:47:10.058 DEBUG:boto:Method: HEAD 2022-09-03T06:47:10.059 DEBUG:boto:Path: /avysmw-68/testobj-sse-c 2022-09-03T06:47:10.060 DEBUG:boto:Data: 2022-09-03T06:47:10.060 DEBUG:boto:Headers: {'x-amz-server-side-encryption-customer-algorithm': 'AES256', 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw=='} 2022-09-03T06:47:10.061 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:10.061 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.062 DEBUG:boto:Params: {} 2022-09-03T06:47:10.062 DEBUG:boto:Token: None 2022-09-03T06:47:10.063 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:10 GMT x-amz-server-side-encryption-customer-algorithm:AES256 x-amz-server-side-encryption-customer-key:pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs= x-amz-server-side-encryption-customer-key-md5:DWygnHRtgiJ77HCm+1rvHw== /avysmw-68/testobj-sse-c 2022-09-03T06:47:10.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YFffP5IsyufFCTbj541iEBZeA60= 2022-09-03T06:47:10.064 DEBUG:boto:Final headers: {'x-amz-server-side-encryption-customer-algorithm': 'AES256', 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YFffP5IsyufFCTbj541iEBZeA60=', 'Content-Length': '0'} 2022-09-03T06:47:10.066 DEBUG:boto:Response headers: [('Content-Length', '512'), ('x-amz-server-side-encryption-customer-algorithm', 'AES256'), ('x-amz-server-side-encryption-customer-key-MD5', 'DWygnHRtgiJ77HCm+1rvHw=='), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:03 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"dc5086b847289ba8b8bde149b8388175"'), ('x-amz-request-id', 'tx00000d4b3e53d8a5226f1-006312f86e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.067 DEBUG:boto:path=/avysmw-68/testobj-sse-c 2022-09-03T06:47:10.068 DEBUG:boto:auth_path=/avysmw-68/testobj-sse-c 2022-09-03T06:47:10.069 DEBUG:boto:Method: GET 2022-09-03T06:47:10.070 DEBUG:boto:Path: /avysmw-68/testobj-sse-c 2022-09-03T06:47:10.070 DEBUG:boto:Data: 2022-09-03T06:47:10.071 DEBUG:boto:Headers: {'x-amz-server-side-encryption-customer-algorithm': 'AES256', 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw=='} 2022-09-03T06:47:10.071 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:10.072 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.072 DEBUG:boto:Params: {} 2022-09-03T06:47:10.073 DEBUG:boto:Token: None 2022-09-03T06:47:10.074 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:10 GMT x-amz-server-side-encryption-customer-algorithm:AES256 x-amz-server-side-encryption-customer-key:pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs= x-amz-server-side-encryption-customer-key-md5:DWygnHRtgiJ77HCm+1rvHw== /avysmw-68/testobj-sse-c 2022-09-03T06:47:10.074 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G1NkuxL3kpr7snHuTyXo4J7kJ1Y= 2022-09-03T06:47:10.075 DEBUG:boto:Final headers: {'x-amz-server-side-encryption-customer-algorithm': 'AES256', 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==', 'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G1NkuxL3kpr7snHuTyXo4J7kJ1Y=', 'Content-Length': '0'} 2022-09-03T06:47:10.101 DEBUG:boto:Response headers: [('Content-Length', '512'), ('x-amz-server-side-encryption-customer-algorithm', 'AES256'), ('x-amz-server-side-encryption-customer-key-MD5', 'DWygnHRtgiJ77HCm+1rvHw=='), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:03 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"dc5086b847289ba8b8bde149b8388175"'), ('x-amz-request-id', 'tx00000118695a56f7b45f7-006312f86e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.103 DEBUG:boto:path=/avysmw-68/testobj-sse-kms 2022-09-03T06:47:10.104 DEBUG:boto:auth_path=/avysmw-68/testobj-sse-kms 2022-09-03T06:47:10.104 DEBUG:boto:Method: HEAD 2022-09-03T06:47:10.105 DEBUG:boto:Path: /avysmw-68/testobj-sse-kms 2022-09-03T06:47:10.106 DEBUG:boto:Data: 2022-09-03T06:47:10.106 DEBUG:boto:Headers: {} 2022-09-03T06:47:10.107 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:10.107 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.108 DEBUG:boto:Params: {} 2022-09-03T06:47:10.108 DEBUG:boto:Token: None 2022-09-03T06:47:10.109 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:10 GMT /avysmw-68/testobj-sse-kms 2022-09-03T06:47:10.110 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G92ssH76qBxPT1P7t5GwnuTsMNU= 2022-09-03T06:47:10.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G92ssH76qBxPT1P7t5GwnuTsMNU=', 'Content-Length': '0'} 2022-09-03T06:47:10.113 DEBUG:boto:Response headers: [('Content-Length', '512'), ('x-amz-server-side-encryption', 'aws:kms'), ('x-amz-server-side-encryption-aws-kms-key-id', 'testkey-1'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:03 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"dc5086b847289ba8b8bde149b8388175"'), ('x-amz-request-id', 'tx000002516517924252150-006312f86e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.114 DEBUG:boto:path=/avysmw-68/testobj-sse-kms 2022-09-03T06:47:10.115 DEBUG:boto:auth_path=/avysmw-68/testobj-sse-kms 2022-09-03T06:47:10.116 DEBUG:boto:Method: GET 2022-09-03T06:47:10.117 DEBUG:boto:Path: /avysmw-68/testobj-sse-kms 2022-09-03T06:47:10.118 DEBUG:boto:Data: 2022-09-03T06:47:10.119 DEBUG:boto:Headers: {} 2022-09-03T06:47:10.119 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:47:10.120 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.120 DEBUG:boto:Params: {} 2022-09-03T06:47:10.121 DEBUG:boto:Token: None 2022-09-03T06:47:10.122 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:47:10 GMT /avysmw-68/testobj-sse-kms 2022-09-03T06:47:10.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3TBTy1LAMUt3zhEqg/xEsW6S/TA= 2022-09-03T06:47:10.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3TBTy1LAMUt3zhEqg/xEsW6S/TA=', 'Content-Length': '0'} 2022-09-03T06:47:10.127 DEBUG:boto:Response headers: [('Content-Length', '512'), ('x-amz-server-side-encryption', 'aws:kms'), ('x-amz-server-side-encryption-aws-kms-key-id', 'testkey-1'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:03 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"dc5086b847289ba8b8bde149b8388175"'), ('x-amz-request-id', 'tx000005d99bd593a56eefa-006312f86e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.128 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_encrypted_object_sync ... ok 2022-09-03T06:47:10.129 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-69 2022-09-03T06:47:10.131 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:47:10.132 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:47:10.132 DEBUG:boto:Method: PUT 2022-09-03T06:47:10.133 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:47:10.133 DEBUG:boto:Data: 2022-09-03T06:47:10.134 DEBUG:boto:Headers: {} 2022-09-03T06:47:10.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:10.135 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.136 DEBUG:boto:Params: {} 2022-09-03T06:47:10.136 DEBUG:boto:Token: None 2022-09-03T06:47:10.137 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:47:10 GMT /avysmw-69/ 2022-09-03T06:47:10.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IsqkVPMfrEeDqs3bZeRVzGtgfIU= 2022-09-03T06:47:10.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IsqkVPMfrEeDqs3bZeRVzGtgfIU=', 'Content-Length': '0'} 2022-09-03T06:47:10.162 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000bf3f3d2aa49780ca-006312f86e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.163 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:47:10.163 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:47:10.164 DEBUG:boto:Method: HEAD 2022-09-03T06:47:10.165 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:47:10.165 DEBUG:boto:Data: 2022-09-03T06:47:10.166 DEBUG:boto:Headers: {} 2022-09-03T06:47:10.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:10.168 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.169 DEBUG:boto:Params: {} 2022-09-03T06:47:10.170 DEBUG:boto:Token: None 2022-09-03T06:47:10.171 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:10 GMT /avysmw-69/ 2022-09-03T06:47:10.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C0uEhMJo9JwPU8t5XUGoaGjLJEA= 2022-09-03T06:47:10.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C0uEhMJo9JwPU8t5XUGoaGjLJEA=', 'Content-Length': '0'} 2022-09-03T06:47:10.177 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cccea23470bf233e-006312f86e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.178 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:47:10.179 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:47:10.180 DEBUG:boto:Method: PUT 2022-09-03T06:47:10.181 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:47:10.181 DEBUG:boto:Data: 2022-09-03T06:47:10.182 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:47:10.183 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:10.184 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.185 DEBUG:boto:Params: {} 2022-09-03T06:47:10.185 DEBUG:boto:Token: None 2022-09-03T06:47:10.186 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:47:10 GMT /avysmw-69/a 2022-09-03T06:47:10.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xHryphx/mONWRlbbcUEdvAH5LXQ= 2022-09-03T06:47:10.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xHryphx/mONWRlbbcUEdvAH5LXQ='} 2022-09-03T06:47:10.198 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000fecdede388464768-006312f86e-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.199 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:47:10.200 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:47:10.201 DEBUG:boto:Method: HEAD 2022-09-03T06:47:10.202 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:47:10.202 DEBUG:boto:Data: 2022-09-03T06:47:10.203 DEBUG:boto:Headers: {} 2022-09-03T06:47:10.204 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:10.204 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.205 DEBUG:boto:Params: {} 2022-09-03T06:47:10.206 DEBUG:boto:Token: None 2022-09-03T06:47:10.206 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:10 GMT /avysmw-69/ 2022-09-03T06:47:10.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C0uEhMJo9JwPU8t5XUGoaGjLJEA= 2022-09-03T06:47:10.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C0uEhMJo9JwPU8t5XUGoaGjLJEA=', 'Content-Length': '0'} 2022-09-03T06:47:10.211 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004ba0879601ca4787-006312f86e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.211 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:47:10.212 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:47:10.212 DEBUG:boto:Method: PUT 2022-09-03T06:47:10.213 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:47:10.213 DEBUG:boto:Data: 2022-09-03T06:47:10.214 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:47:10.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:10.215 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.216 DEBUG:boto:Params: {} 2022-09-03T06:47:10.216 DEBUG:boto:Token: None 2022-09-03T06:47:10.217 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:47:10 GMT /avysmw-69/b 2022-09-03T06:47:10.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fZFy2Up8+NS0IvWNKSXSbEq07DU= 2022-09-03T06:47:10.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fZFy2Up8+NS0IvWNKSXSbEq07DU='} 2022-09-03T06:47:10.266 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000022cbe24563b95dee-006312f86e-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.267 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:47:10.269 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:47:10.269 DEBUG:boto:Method: HEAD 2022-09-03T06:47:10.271 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:47:10.271 DEBUG:boto:Data: 2022-09-03T06:47:10.272 DEBUG:boto:Headers: {} 2022-09-03T06:47:10.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:10.274 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.275 DEBUG:boto:Params: {} 2022-09-03T06:47:10.276 DEBUG:boto:Token: None 2022-09-03T06:47:10.277 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:10 GMT /avysmw-69/ 2022-09-03T06:47:10.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C0uEhMJo9JwPU8t5XUGoaGjLJEA= 2022-09-03T06:47:10.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C0uEhMJo9JwPU8t5XUGoaGjLJEA=', 'Content-Length': '0'} 2022-09-03T06:47:10.282 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000095d228bf599c2b13-006312f86e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.284 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:47:10.285 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:47:10.285 DEBUG:boto:Method: PUT 2022-09-03T06:47:10.286 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:47:10.286 DEBUG:boto:Data: 2022-09-03T06:47:10.287 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:47:10.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:10.288 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.289 DEBUG:boto:Params: {} 2022-09-03T06:47:10.290 DEBUG:boto:Token: None 2022-09-03T06:47:10.290 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:47:10 GMT /avysmw-69/c 2022-09-03T06:47:10.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ObTfhpV2xPXe3SEWGUxPxHGoyKo= 2022-09-03T06:47:10.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ObTfhpV2xPXe3SEWGUxPxHGoyKo='} 2022-09-03T06:47:10.302 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000cac0d4b6c89a40b6-006312f86e-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.303 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:47:10.304 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:47:10.305 DEBUG:boto:Method: HEAD 2022-09-03T06:47:10.305 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:47:10.306 DEBUG:boto:Data: 2022-09-03T06:47:10.307 DEBUG:boto:Headers: {} 2022-09-03T06:47:10.307 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:10.308 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.308 DEBUG:boto:Params: {} 2022-09-03T06:47:10.309 DEBUG:boto:Token: None 2022-09-03T06:47:10.310 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:47:10 GMT /avysmw-69/ 2022-09-03T06:47:10.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C0uEhMJo9JwPU8t5XUGoaGjLJEA= 2022-09-03T06:47:10.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C0uEhMJo9JwPU8t5XUGoaGjLJEA=', 'Content-Length': '0'} 2022-09-03T06:47:10.314 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '9'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007a3820530cc59699-006312f86e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.316 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:47:10.316 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:47:10.317 DEBUG:boto:Method: PUT 2022-09-03T06:47:10.318 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:47:10.319 DEBUG:boto:Data: 2022-09-03T06:47:10.320 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:47:10.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:47:10.321 DEBUG:boto:Port: 8000 2022-09-03T06:47:10.321 DEBUG:boto:Params: {} 2022-09-03T06:47:10.322 DEBUG:boto:Token: None 2022-09-03T06:47:10.323 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:47:10 GMT /avysmw-69/d 2022-09-03T06:47:10.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ettisal2DB4qpGI5vjsAyicvYq0= 2022-09-03T06:47:10.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:47:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ettisal2DB4qpGI5vjsAyicvYq0='} 2022-09-03T06:47:10.370 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000013010ced07eb58a6-006312f86e-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:47:10.371 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:10.372 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:10.373 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:47:10.431 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:10.432 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:10.476 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:47:10.493 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:47:10.493 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:10.493 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:10.494 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:47:10.494 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:10.494 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:10.495 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187630.158924_386.1", 2022-09-03T06:47:10.495 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:47:10.158924Z" 2022-09-03T06:47:10.495 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:10.495 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:10.496 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:10.496 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:47:10.496 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:10.496 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:10.497 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:10.497 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:47:10.497 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:10.497 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:47:10.499 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662187630.158924_386.1', 2: '', 3: ''} 2022-09-03T06:47:10.499 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:47:10.500 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:10.500 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:10.501 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:47:10.554 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:10.570 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:10.612 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:10.609+0000 7f9087694600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:10.613 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:10.610+0000 7f9087694600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:10.632 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:47:10.632 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:47:10.632 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:47:10.633 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:47:10.633 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:47:10.633 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:47:10.634 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:47:10.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.634 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:47:10.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:10.635 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:47:10.635 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:10.636 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:10.636 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:47:10.636 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:10.637 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:47:10.637 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:10.638 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:47:10.638 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:10.638 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:10.639 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:47:10.639 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:10.640 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:10.640 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187630.158924_386.1", 2022-09-03T06:47:10.640 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:10.640 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:10.641 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:10.641 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:47:10.158924Z", 2022-09-03T06:47:10.642 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:47:10.643 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.643 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.644 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:10.646 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:47:10.646 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:10.647 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:10.647 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:47:10.648 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:10.648 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:10.648 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:10.649 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:47:10.651 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:10.652 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.653 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.653 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:10.654 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:47:10.654 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:10.655 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:10.655 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:47:10.656 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:10.656 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:10.657 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:10.657 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:47:10.658 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:10.658 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.659 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.659 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:47:10.660 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.660 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:47:10.661 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:47:10.661 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:47:10.662 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.663 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:47:10Z" 2022-09-03T06:47:10.663 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:47:10.665 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187630.158924_386.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:47:10.158924Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:47:10Z" } 2022-09-03T06:47:10.666 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187630.158924_386.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:47:10.158924Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:47:10.667 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187630.158924_386.1', 2: '', 3: ''} 2022-09-03T06:47:10.667 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187630.158924_386.1', 2: '', 3: ''} 2022-09-03T06:47:10.668 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:47:10.669 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:47:10.669 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:10.670 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:10.671 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:47:10.724 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:10.724 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:10.766 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:47:10.764+0000 7fc2c03e6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:10.767 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:47:10.765+0000 7fc2c03e6600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:10.782 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:47:10.783 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:47:10.784 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:47:10.784 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:47:10.785 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:47:10.786 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:47:10.786 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:47:10.787 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:10.787 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:47:10.788 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:10.788 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:47:10.788 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:47:10.789 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:47:10.789 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:47:10.789 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:47:10.789 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:47:10.790 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:47:10.790 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:47:10.790 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:47:10.790 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:10.791 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:10.805 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:10.805 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:47:10.805 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:47:10.806 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:47:10.806 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187630.158924_386.1", 2022-09-03T06:47:10.806 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:47:10.806 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:47:10.807 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:47:10.807 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:47:10.158924Z", 2022-09-03T06:47:10.807 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:47:10.807 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:10.808 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:10.808 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:10.808 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:47:10.808 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:47:10.809 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:47:10.809 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:10.809 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:47:10.809 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:47:10.810 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:47:10.810 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:47:10.810 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:47:10.810 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:10.811 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:10.811 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:47:10.811 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:47:10.812 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:47:10.812 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:47:10.812 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:47:10.812 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:47:10.813 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:47:10.813 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:47:10.813 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:47:10.814 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:47:10.814 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:10.814 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:47:10.814 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:47:10.815 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:10.815 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:47:10.815 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:47:10.815 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:47:10.815 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:47:10.816 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:47:10Z" 2022-09-03T06:47:10.816 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:47:10.817 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187630.158924_386.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:47:10.158924Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:47:10Z" } 2022-09-03T06:47:10.819 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187630.158924_386.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:47:10.158924Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:47:10.819 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187630.158924_386.1', 2: '', 3: ''} 2022-09-03T06:47:10.820 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187630.158924_386.1', 2: '', 3: ''} 2022-09-03T06:47:10.820 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:47:10.821 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:47:10.821 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:10.822 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:47:10.822 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:47:10.874 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:10.874 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:10.915 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:10.913+0000 7fc53031f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:10.916 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:10.914+0000 7fc53031f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:47:10.932 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:47:10.933 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:47:10.933 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:47:10.935 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:47:10.935 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:47:10.936 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:47:10.936 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:47:10.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.937 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:47:10.937 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:10.938 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:47:10.938 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:10.938 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:10.939 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:47:10.939 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:10.939 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:47:10.939 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:10.940 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:47:10.940 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:10.940 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.941 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.941 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:10.941 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:47:10.941 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:10.942 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:10.942 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187630.158924_386.1", 2022-09-03T06:47:10.943 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:10.944 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:10.944 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:10.945 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:47:10.158924Z", 2022-09-03T06:47:10.945 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:47:10.946 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.947 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:10.948 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:47:10.949 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:10.949 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:10.950 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:47:10.950 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:10.951 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:10.951 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:10.951 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:47:10.952 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:10.952 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.954 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.954 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:47:10.955 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:47:10.955 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:47:10.955 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:47:10.956 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:47:10.956 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:47:10.957 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:47:10.957 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:47:10.957 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:47:10.959 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:47:10.960 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.960 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:47:10.960 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:47:10.960 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.961 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:47:10.961 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:47:10.961 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:47:10.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:47:10.962 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:47:10Z" 2022-09-03T06:47:10.962 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:47:10.965 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187630.158924_386.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:47:10.158924Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:47:10Z" } 2022-09-03T06:47:10.966 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187630.158924_386.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:47:10.158924Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:47:10.967 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187630.158924_386.1', 2: '', 3: ''} 2022-09-03T06:47:10.968 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187630.158924_386.1', 2: '', 3: ''} 2022-09-03T06:47:10.969 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:47:10.970 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-69 2022-09-03T06:47:10.971 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:47:10.972 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:47:10.974 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-69 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:47:11.039 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:11.039 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:11.087 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:11.085+0000 7f861a6ea600 1 waiting to reach incremental sync.. 2022-09-03T06:47:16.091 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:16.085+0000 7f861a6ea600 1 waiting to reach incremental sync.. 2022-09-03T06:47:21.093 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:21.086+0000 7f861a6ea600 1 waiting to reach incremental sync.. 2022-09-03T06:47:26.089 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:26.086+0000 7f861a6ea600 1 waiting to reach incremental sync.. 2022-09-03T06:47:31.089 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:31.087+0000 7f861a6ea600 1 waiting to reach incremental sync.. 2022-09-03T06:47:36.089 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:36.088+0000 7f861a6ea600 1 waiting to reach incremental sync.. 2022-09-03T06:47:41.090 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:41.088+0000 7f861a6ea600 1 waiting to reach incremental sync.. 2022-09-03T06:47:46.092 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:46.090+0000 7f861a6ea600 1 bucket sync caught up with source: 2022-09-03T06:47:46.094 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.61.5, , , , 00000000001.56.5, , , 00000000001.80.5, , 00000000001.67.5] 2022-09-03T06:47:46.094 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.61.5, , , , 00000000001.56.5, , , 00000000001.80.5, , 00000000001.67.5] 2022-09-03T06:47:46.095 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:47:46.090+0000 7f861a6ea600 0 bucket checkpoint complete 2022-09-03T06:47:46.127 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-69 2022-09-03T06:47:46.128 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:47:46.130 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:47:46.131 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-69 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:47:46.194 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:47:46.195 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:47:46.245 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:47:46.242+0000 7f3ae5aa1600 1 waiting to reach incremental sync.. 2022-09-03T06:47:51.245 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:47:51.243+0000 7f3ae5aa1600 1 waiting to reach incremental sync.. 2022-09-03T06:47:56.246 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:47:56.243+0000 7f3ae5aa1600 1 waiting to reach incremental sync.. 2022-09-03T06:48:01.247 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:01.245+0000 7f3ae5aa1600 1 waiting to reach incremental sync.. 2022-09-03T06:48:06.248 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:06.247+0000 7f3ae5aa1600 1 bucket sync caught up with source: 2022-09-03T06:48:06.249 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.61.5, , , , 00000000001.56.5, , , 00000000001.80.5, , 00000000001.67.5] 2022-09-03T06:48:06.249 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.61.5, , , , 00000000001.56.5, , , 00000000001.80.5, , 00000000001.67.5] 2022-09-03T06:48:06.250 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:06.247+0000 7f3ae5aa1600 0 bucket checkpoint complete 2022-09-03T06:48:06.268 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-69 2022-09-03T06:48:06.269 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.269 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.270 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-69 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:48:06.421 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:06.422 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:06.470 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:06.468+0000 7ff3a8b51600 1 bucket sync caught up with source: 2022-09-03T06:48:06.470 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.61.5, , , , 00000000001.56.5, , , 00000000001.80.5, , 00000000001.67.5] 2022-09-03T06:48:06.470 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.61.5, , , , 00000000001.56.5, , , 00000000001.80.5, , 00000000001.67.5] 2022-09-03T06:48:06.471 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:06.468+0000 7ff3a8b51600 0 bucket checkpoint complete 2022-09-03T06:48:06.484 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-69 2022-09-03T06:48:06.485 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.485 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.486 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-69 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:48:06.554 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:06.555 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:06.606 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:06.604+0000 7fc4cf0ac600 1 bucket sync caught up with source: 2022-09-03T06:48:06.606 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.293.5, , , , 00000000001.277.5, , , 00000000001.214.5, , 00000000001.231.5] 2022-09-03T06:48:06.606 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.293.5, , , , 00000000001.277.5, , , 00000000001.214.5, , 00000000001.231.5] 2022-09-03T06:48:06.607 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:06.604+0000 7fc4cf0ac600 0 bucket checkpoint complete 2022-09-03T06:48:06.631 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-69 2022-09-03T06:48:06.632 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.632 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.633 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-69 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:48:06.694 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:06.694 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:06.745 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:06.744+0000 7fe90d451600 1 bucket sync caught up with source: 2022-09-03T06:48:06.746 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.293.5, , , , 00000000001.277.5, , , 00000000001.214.5, , 00000000001.231.5] 2022-09-03T06:48:06.746 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.293.5, , , , 00000000001.277.5, , , 00000000001.214.5, , 00000000001.231.5] 2022-09-03T06:48:06.747 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:06.744+0000 7fe90d451600 0 bucket checkpoint complete 2022-09-03T06:48:06.774 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-69 2022-09-03T06:48:06.775 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-69 2022-09-03T06:48:06.776 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.777 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.778 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-69 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:48:06.838 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:06.839 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:06.882 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:06.881+0000 7fb02fdfe600 1 bucket sync caught up with source: 2022-09-03T06:48:06.883 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.208.5, , , , 00000000001.199.5, , , 00000000001.206.5, , 00000000001.271.5] 2022-09-03T06:48:06.883 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.208.5, , , , 00000000001.199.5, , , 00000000001.206.5, , 00000000001.271.5] 2022-09-03T06:48:06.884 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:06.881+0000 7fb02fdfe600 0 bucket checkpoint complete 2022-09-03T06:48:06.899 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-69 2022-09-03T06:48:06.900 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.900 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-69', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:06.901 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-69 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:48:06.962 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:06.963 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:07.020 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:07.006+0000 7f5aa8f23600 1 bucket sync caught up with source: 2022-09-03T06:48:07.021 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.208.5, , , , 00000000001.199.5, , , 00000000001.206.5, , 00000000001.271.5] 2022-09-03T06:48:07.021 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.208.5, , , , 00000000001.199.5, , , 00000000001.206.5, , 00000000001.271.5] 2022-09-03T06:48:07.021 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:07.006+0000 7f5aa8f23600 0 bucket checkpoint complete 2022-09-03T06:48:07.023 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-69 2022-09-03T06:48:07.024 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-69 zones={test-zone2, test-zone1} 2022-09-03T06:48:07.024 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.025 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.026 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.027 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:48:07.028 DEBUG:boto:Data: 2022-09-03T06:48:07.028 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.029 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.029 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.030 DEBUG:boto:Params: {} 2022-09-03T06:48:07.031 DEBUG:boto:Token: None 2022-09-03T06:48:07.031 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pn6/Pjiqiz4iigJWrTdbbzkBaEE= 2022-09-03T06:48:07.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pn6/Pjiqiz4iigJWrTdbbzkBaEE=', 'Content-Length': '0'} 2022-09-03T06:48:07.036 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ec155e55ad76c982-006312f8a7-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.038 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.038 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.039 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.040 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:48:07.040 DEBUG:boto:Data: 2022-09-03T06:48:07.041 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.042 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.042 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.043 DEBUG:boto:Params: {} 2022-09-03T06:48:07.044 DEBUG:boto:Token: None 2022-09-03T06:48:07.045 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pn6/Pjiqiz4iigJWrTdbbzkBaEE= 2022-09-03T06:48:07.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pn6/Pjiqiz4iigJWrTdbbzkBaEE=', 'Content-Length': '0'} 2022-09-03T06:48:07.050 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007a9b541fecf47741-006312f8a7-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.051 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:48:07.051 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.052 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.052 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.054 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.054 DEBUG:boto:Method: GET 2022-09-03T06:48:07.055 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.055 DEBUG:boto:Data: 2022-09-03T06:48:07.056 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.056 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.057 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.057 DEBUG:boto:Params: {} 2022-09-03T06:48:07.058 DEBUG:boto:Token: None 2022-09-03T06:48:07.059 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/?versions 2022-09-03T06:48:07.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QTRFHVRNnFMjPiDORe1868/3ZFY= 2022-09-03T06:48:07.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QTRFHVRNnFMjPiDORe1868/3ZFY=', 'Content-Length': '0'} 2022-09-03T06:48:07.065 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000029334902a7300fc7-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.066 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.068 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:48:07.069 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:48:07.070 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:48:07.071 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:48:07.072 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:48:07.073 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.075 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.077 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.078 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.079 DEBUG:boto:Method: GET 2022-09-03T06:48:07.080 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.081 DEBUG:boto:Data: 2022-09-03T06:48:07.082 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.083 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.084 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.085 DEBUG:boto:Params: {} 2022-09-03T06:48:07.087 DEBUG:boto:Token: None 2022-09-03T06:48:07.088 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/?versions 2022-09-03T06:48:07.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QTRFHVRNnFMjPiDORe1868/3ZFY= 2022-09-03T06:48:07.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QTRFHVRNnFMjPiDORe1868/3ZFY=', 'Content-Length': '0'} 2022-09-03T06:48:07.093 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000459e20298a2bd680-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.094 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.096 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:48:07.097 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:48:07.097 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:48:07.098 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:48:07.099 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.100 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.101 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.102 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.103 DEBUG:boto:Method: GET 2022-09-03T06:48:07.104 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.105 DEBUG:boto:Data: 2022-09-03T06:48:07.106 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.107 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.108 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.109 DEBUG:boto:Params: {} 2022-09-03T06:48:07.110 DEBUG:boto:Token: None 2022-09-03T06:48:07.111 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/?versions 2022-09-03T06:48:07.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QTRFHVRNnFMjPiDORe1868/3ZFY= 2022-09-03T06:48:07.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QTRFHVRNnFMjPiDORe1868/3ZFY=', 'Content-Length': '0'} 2022-09-03T06:48:07.117 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a6ea28c812aa7f0-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.118 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.120 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.121 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.122 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.123 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.124 DEBUG:boto:Method: GET 2022-09-03T06:48:07.125 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:07.126 DEBUG:boto:Data: 2022-09-03T06:48:07.127 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.129 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.130 DEBUG:boto:Params: {} 2022-09-03T06:48:07.131 DEBUG:boto:Token: None 2022-09-03T06:48:07.132 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/?versions 2022-09-03T06:48:07.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QTRFHVRNnFMjPiDORe1868/3ZFY= 2022-09-03T06:48:07.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QTRFHVRNnFMjPiDORe1868/3ZFY=', 'Content-Length': '0'} 2022-09-03T06:48:07.137 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000073bee81425f310bf-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.138 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.140 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:48:07.141 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.142 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.143 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.144 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.145 DEBUG:boto:Method: GET 2022-09-03T06:48:07.145 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:07.146 DEBUG:boto:Data: 2022-09-03T06:48:07.146 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.147 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.147 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.148 DEBUG:boto:Params: {} 2022-09-03T06:48:07.149 DEBUG:boto:Token: None 2022-09-03T06:48:07.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:07.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U5b+xPAsSBv8qbir3+oZa1GE5ug= 2022-09-03T06:48:07.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U5b+xPAsSBv8qbir3+oZa1GE5ug=', 'Content-Length': '0'} 2022-09-03T06:48:07.153 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c0b71cf368ddbf6-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.154 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.154 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.155 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.155 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.156 DEBUG:boto:Method: GET 2022-09-03T06:48:07.157 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:07.157 DEBUG:boto:Data: 2022-09-03T06:48:07.158 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.159 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.159 DEBUG:boto:Params: {} 2022-09-03T06:48:07.160 DEBUG:boto:Token: None 2022-09-03T06:48:07.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:07.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U5b+xPAsSBv8qbir3+oZa1GE5ug= 2022-09-03T06:48:07.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U5b+xPAsSBv8qbir3+oZa1GE5ug=', 'Content-Length': '0'} 2022-09-03T06:48:07.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007001f1815f28dc4-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.165 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.166 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.166 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.167 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.167 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.168 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:07.168 DEBUG:boto:Data: 2022-09-03T06:48:07.169 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.169 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.170 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.170 DEBUG:boto:Params: {} 2022-09-03T06:48:07.171 DEBUG:boto:Token: None 2022-09-03T06:48:07.171 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:07.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oN+QqpVKv6okVif2NcCfKhKwzSc= 2022-09-03T06:48:07.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oN+QqpVKv6okVif2NcCfKhKwzSc=', 'Content-Length': '0'} 2022-09-03T06:48:07.175 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c98cad2fbf4cbd74-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.175 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.176 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.176 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.177 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.177 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.178 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:07.178 DEBUG:boto:Data: 2022-09-03T06:48:07.179 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.179 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.180 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.180 DEBUG:boto:Params: {} 2022-09-03T06:48:07.181 DEBUG:boto:Token: None 2022-09-03T06:48:07.181 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:07.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oN+QqpVKv6okVif2NcCfKhKwzSc= 2022-09-03T06:48:07.183 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oN+QqpVKv6okVif2NcCfKhKwzSc=', 'Content-Length': '0'} 2022-09-03T06:48:07.184 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074750dcf0fe8aa88-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.185 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:48:07.186 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.186 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.187 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.187 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.188 DEBUG:boto:Method: GET 2022-09-03T06:48:07.188 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:07.189 DEBUG:boto:Data: 2022-09-03T06:48:07.190 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.190 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.191 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.191 DEBUG:boto:Params: {} 2022-09-03T06:48:07.192 DEBUG:boto:Token: None 2022-09-03T06:48:07.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:07.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U5b+xPAsSBv8qbir3+oZa1GE5ug= 2022-09-03T06:48:07.194 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U5b+xPAsSBv8qbir3+oZa1GE5ug=', 'Content-Length': '0'} 2022-09-03T06:48:07.196 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfd3a33fdfba562a-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.238 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.239 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.239 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.240 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:07.241 DEBUG:boto:Method: GET 2022-09-03T06:48:07.241 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:07.242 DEBUG:boto:Data: 2022-09-03T06:48:07.242 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.244 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.244 DEBUG:boto:Params: {} 2022-09-03T06:48:07.245 DEBUG:boto:Token: None 2022-09-03T06:48:07.246 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:07.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U5b+xPAsSBv8qbir3+oZa1GE5ug= 2022-09-03T06:48:07.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U5b+xPAsSBv8qbir3+oZa1GE5ug=', 'Content-Length': '0'} 2022-09-03T06:48:07.250 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff15374897702de3-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.251 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.251 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.252 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:07.252 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:07.253 DEBUG:boto:Method: GET 2022-09-03T06:48:07.254 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:07.254 DEBUG:boto:Data: 2022-09-03T06:48:07.255 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.255 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.256 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.257 DEBUG:boto:Params: {} 2022-09-03T06:48:07.258 DEBUG:boto:Token: None 2022-09-03T06:48:07.258 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.262 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020fce0b3f0157384-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.263 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.264 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.265 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.265 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:07.266 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:07.267 DEBUG:boto:Method: GET 2022-09-03T06:48:07.267 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:07.268 DEBUG:boto:Data: 2022-09-03T06:48:07.268 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.270 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.270 DEBUG:boto:Params: {} 2022-09-03T06:48:07.271 DEBUG:boto:Token: None 2022-09-03T06:48:07.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.275 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000707f39d11f5e650c-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.276 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.277 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.277 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.278 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.279 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:07.279 DEBUG:boto:Data: 2022-09-03T06:48:07.280 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.280 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.281 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.281 DEBUG:boto:Params: {} 2022-09-03T06:48:07.282 DEBUG:boto:Token: None 2022-09-03T06:48:07.283 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a 2022-09-03T06:48:07.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRpaeflF5/G3lzn9+gN+EUGC8jc= 2022-09-03T06:48:07.284 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRpaeflF5/G3lzn9+gN+EUGC8jc=', 'Content-Length': '0'} 2022-09-03T06:48:07.288 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008ad1e09286983bc8-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.288 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.289 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.290 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.290 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:07.291 DEBUG:boto:Data: 2022-09-03T06:48:07.292 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.293 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.293 DEBUG:boto:Params: {} 2022-09-03T06:48:07.294 DEBUG:boto:Token: None 2022-09-03T06:48:07.294 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a 2022-09-03T06:48:07.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRpaeflF5/G3lzn9+gN+EUGC8jc= 2022-09-03T06:48:07.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRpaeflF5/G3lzn9+gN+EUGC8jc=', 'Content-Length': '0'} 2022-09-03T06:48:07.302 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3ab6e7609180bc6-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.303 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:48:07.304 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.305 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.305 DEBUG:boto:Method: GET 2022-09-03T06:48:07.306 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:07.307 DEBUG:boto:Data: 2022-09-03T06:48:07.307 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.308 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.309 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.309 DEBUG:boto:Params: {} 2022-09-03T06:48:07.310 DEBUG:boto:Token: None 2022-09-03T06:48:07.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a 2022-09-03T06:48:07.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i41z2mPUdMfsxBw1Jtlrs3taMHo= 2022-09-03T06:48:07.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i41z2mPUdMfsxBw1Jtlrs3taMHo=', 'Content-Length': '0'} 2022-09-03T06:48:07.315 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000799f015585a6e835-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.358 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:07.359 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:07.360 DEBUG:boto:Method: GET 2022-09-03T06:48:07.360 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:07.361 DEBUG:boto:Data: 2022-09-03T06:48:07.361 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.362 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.363 DEBUG:boto:Params: {} 2022-09-03T06:48:07.364 DEBUG:boto:Token: None 2022-09-03T06:48:07.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/a 2022-09-03T06:48:07.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i41z2mPUdMfsxBw1Jtlrs3taMHo= 2022-09-03T06:48:07.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i41z2mPUdMfsxBw1Jtlrs3taMHo=', 'Content-Length': '0'} 2022-09-03T06:48:07.368 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc8457f93749efc6-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.369 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.369 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.370 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:07.371 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:07.371 DEBUG:boto:Method: GET 2022-09-03T06:48:07.372 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:07.372 DEBUG:boto:Data: 2022-09-03T06:48:07.373 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.374 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.374 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.375 DEBUG:boto:Params: {} 2022-09-03T06:48:07.375 DEBUG:boto:Token: None 2022-09-03T06:48:07.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.380 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000678eabb3f11d357-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.381 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.382 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.383 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.383 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:07.384 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:07.385 DEBUG:boto:Method: GET 2022-09-03T06:48:07.386 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:07.386 DEBUG:boto:Data: 2022-09-03T06:48:07.387 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.387 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.389 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.389 DEBUG:boto:Params: {} 2022-09-03T06:48:07.390 DEBUG:boto:Token: None 2022-09-03T06:48:07.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.394 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000081fd065d0d67bec2-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.395 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.396 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:48:07.397 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.398 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.398 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.399 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.399 DEBUG:boto:Method: GET 2022-09-03T06:48:07.400 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:07.400 DEBUG:boto:Data: 2022-09-03T06:48:07.401 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.402 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.402 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.403 DEBUG:boto:Params: {} 2022-09-03T06:48:07.404 DEBUG:boto:Token: None 2022-09-03T06:48:07.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:07.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EKdN1nyONUmp5vKySeN+K0w0GaM= 2022-09-03T06:48:07.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EKdN1nyONUmp5vKySeN+K0w0GaM=', 'Content-Length': '0'} 2022-09-03T06:48:07.408 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008090f9fe59d78cd6-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.408 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.409 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.409 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.410 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.411 DEBUG:boto:Method: GET 2022-09-03T06:48:07.411 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:07.412 DEBUG:boto:Data: 2022-09-03T06:48:07.412 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.413 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.413 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.414 DEBUG:boto:Params: {} 2022-09-03T06:48:07.415 DEBUG:boto:Token: None 2022-09-03T06:48:07.415 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:07.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EKdN1nyONUmp5vKySeN+K0w0GaM= 2022-09-03T06:48:07.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EKdN1nyONUmp5vKySeN+K0w0GaM=', 'Content-Length': '0'} 2022-09-03T06:48:07.419 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024a8dce201d8283a-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.419 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.420 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.421 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.421 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.422 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.422 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:07.423 DEBUG:boto:Data: 2022-09-03T06:48:07.424 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.424 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.425 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.426 DEBUG:boto:Params: {} 2022-09-03T06:48:07.426 DEBUG:boto:Token: None 2022-09-03T06:48:07.427 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:07.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U4V781opXu7DBLtArNjzh6hm15c= 2022-09-03T06:48:07.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U4V781opXu7DBLtArNjzh6hm15c=', 'Content-Length': '0'} 2022-09-03T06:48:07.430 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092a7a893afbc971b-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.431 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.432 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.433 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.433 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.434 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.434 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:07.435 DEBUG:boto:Data: 2022-09-03T06:48:07.436 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.436 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.437 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.437 DEBUG:boto:Params: {} 2022-09-03T06:48:07.438 DEBUG:boto:Token: None 2022-09-03T06:48:07.439 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:07.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U4V781opXu7DBLtArNjzh6hm15c= 2022-09-03T06:48:07.440 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U4V781opXu7DBLtArNjzh6hm15c=', 'Content-Length': '0'} 2022-09-03T06:48:07.442 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b5e65fb7779ed64-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.442 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:48:07.443 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.444 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.444 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.445 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.445 DEBUG:boto:Method: GET 2022-09-03T06:48:07.446 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:07.447 DEBUG:boto:Data: 2022-09-03T06:48:07.447 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.448 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.448 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.449 DEBUG:boto:Params: {} 2022-09-03T06:48:07.449 DEBUG:boto:Token: None 2022-09-03T06:48:07.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:07.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EKdN1nyONUmp5vKySeN+K0w0GaM= 2022-09-03T06:48:07.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EKdN1nyONUmp5vKySeN+K0w0GaM=', 'Content-Length': '0'} 2022-09-03T06:48:07.454 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d891f02b48191db5-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.455 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.455 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.456 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.456 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:07.457 DEBUG:boto:Method: GET 2022-09-03T06:48:07.458 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:07.458 DEBUG:boto:Data: 2022-09-03T06:48:07.459 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.459 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.460 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.461 DEBUG:boto:Params: {} 2022-09-03T06:48:07.461 DEBUG:boto:Token: None 2022-09-03T06:48:07.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:07.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EKdN1nyONUmp5vKySeN+K0w0GaM= 2022-09-03T06:48:07.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EKdN1nyONUmp5vKySeN+K0w0GaM=', 'Content-Length': '0'} 2022-09-03T06:48:07.466 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000efb24f4a8821d309-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.510 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.511 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.511 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:07.512 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:07.512 DEBUG:boto:Method: GET 2022-09-03T06:48:07.513 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:07.514 DEBUG:boto:Data: 2022-09-03T06:48:07.514 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.515 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.516 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.516 DEBUG:boto:Params: {} 2022-09-03T06:48:07.517 DEBUG:boto:Token: None 2022-09-03T06:48:07.518 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.522 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb1816cae8a44b5f-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.523 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.524 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.524 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.526 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:07.526 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:07.527 DEBUG:boto:Method: GET 2022-09-03T06:48:07.528 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:07.528 DEBUG:boto:Data: 2022-09-03T06:48:07.529 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.530 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.530 DEBUG:boto:Params: {} 2022-09-03T06:48:07.531 DEBUG:boto:Token: None 2022-09-03T06:48:07.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.532 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.536 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e4cd5fa73b53d819-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.536 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.537 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.538 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.539 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.539 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:07.540 DEBUG:boto:Data: 2022-09-03T06:48:07.540 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.541 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.541 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.542 DEBUG:boto:Params: {} 2022-09-03T06:48:07.542 DEBUG:boto:Token: None 2022-09-03T06:48:07.543 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b 2022-09-03T06:48:07.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E6R6pVaXd2+0FbES0bfwOmk7jv4= 2022-09-03T06:48:07.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E6R6pVaXd2+0FbES0bfwOmk7jv4=', 'Content-Length': '0'} 2022-09-03T06:48:07.546 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce5652bad14cff31-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.547 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.547 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.548 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.549 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:07.549 DEBUG:boto:Data: 2022-09-03T06:48:07.550 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.551 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.552 DEBUG:boto:Params: {} 2022-09-03T06:48:07.553 DEBUG:boto:Token: None 2022-09-03T06:48:07.553 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b 2022-09-03T06:48:07.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E6R6pVaXd2+0FbES0bfwOmk7jv4= 2022-09-03T06:48:07.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E6R6pVaXd2+0FbES0bfwOmk7jv4=', 'Content-Length': '0'} 2022-09-03T06:48:07.556 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f5b1e4d10d98140-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.557 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:48:07.558 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.559 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.559 DEBUG:boto:Method: GET 2022-09-03T06:48:07.560 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:07.560 DEBUG:boto:Data: 2022-09-03T06:48:07.561 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.562 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.562 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.563 DEBUG:boto:Params: {} 2022-09-03T06:48:07.563 DEBUG:boto:Token: None 2022-09-03T06:48:07.564 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b 2022-09-03T06:48:07.565 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j1Yl1RDmoyodGFoC/s9N4eZ/1yc= 2022-09-03T06:48:07.565 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j1Yl1RDmoyodGFoC/s9N4eZ/1yc=', 'Content-Length': '0'} 2022-09-03T06:48:07.567 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016df9f010b01d266-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.611 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:07.614 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:07.615 DEBUG:boto:Method: GET 2022-09-03T06:48:07.615 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:07.616 DEBUG:boto:Data: 2022-09-03T06:48:07.616 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.617 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.617 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.618 DEBUG:boto:Params: {} 2022-09-03T06:48:07.619 DEBUG:boto:Token: None 2022-09-03T06:48:07.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/b 2022-09-03T06:48:07.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j1Yl1RDmoyodGFoC/s9N4eZ/1yc= 2022-09-03T06:48:07.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j1Yl1RDmoyodGFoC/s9N4eZ/1yc=', 'Content-Length': '0'} 2022-09-03T06:48:07.624 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b881a6f12deaa0c1-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.625 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.625 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.626 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:07.627 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:07.627 DEBUG:boto:Method: GET 2022-09-03T06:48:07.628 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:07.628 DEBUG:boto:Data: 2022-09-03T06:48:07.629 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.629 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.630 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.631 DEBUG:boto:Params: {} 2022-09-03T06:48:07.631 DEBUG:boto:Token: None 2022-09-03T06:48:07.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.636 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002455e87df6ce56fd-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.636 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.637 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.638 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.639 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:07.639 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:07.640 DEBUG:boto:Method: GET 2022-09-03T06:48:07.641 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:07.642 DEBUG:boto:Data: 2022-09-03T06:48:07.643 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.645 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.646 DEBUG:boto:Params: {} 2022-09-03T06:48:07.648 DEBUG:boto:Token: None 2022-09-03T06:48:07.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.651 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.654 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f866817941e66a8d-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.656 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.658 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:48:07.659 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.660 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.661 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.662 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.663 DEBUG:boto:Method: GET 2022-09-03T06:48:07.664 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:07.665 DEBUG:boto:Data: 2022-09-03T06:48:07.666 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.667 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.668 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.669 DEBUG:boto:Params: {} 2022-09-03T06:48:07.670 DEBUG:boto:Token: None 2022-09-03T06:48:07.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:07.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T2e7MhhtRnQ1/SZSGXozQxsCb6E= 2022-09-03T06:48:07.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T2e7MhhtRnQ1/SZSGXozQxsCb6E=', 'Content-Length': '0'} 2022-09-03T06:48:07.676 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a15d740677654e70-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.678 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.679 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.680 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.682 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.683 DEBUG:boto:Method: GET 2022-09-03T06:48:07.684 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:07.684 DEBUG:boto:Data: 2022-09-03T06:48:07.685 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.687 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.688 DEBUG:boto:Params: {} 2022-09-03T06:48:07.689 DEBUG:boto:Token: None 2022-09-03T06:48:07.690 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:07.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T2e7MhhtRnQ1/SZSGXozQxsCb6E= 2022-09-03T06:48:07.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T2e7MhhtRnQ1/SZSGXozQxsCb6E=', 'Content-Length': '0'} 2022-09-03T06:48:07.694 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029bcd94333c4c479-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.695 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.696 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.697 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.697 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.698 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.698 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:07.699 DEBUG:boto:Data: 2022-09-03T06:48:07.699 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.700 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.700 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.701 DEBUG:boto:Params: {} 2022-09-03T06:48:07.701 DEBUG:boto:Token: None 2022-09-03T06:48:07.702 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:07.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tLQjZJAiMKY7Z5078KlIiNx4gDc= 2022-09-03T06:48:07.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tLQjZJAiMKY7Z5078KlIiNx4gDc=', 'Content-Length': '0'} 2022-09-03T06:48:07.705 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000691ec2692061a16b-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.706 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.707 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.708 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.708 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.709 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.710 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:07.711 DEBUG:boto:Data: 2022-09-03T06:48:07.712 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.713 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.713 DEBUG:boto:Params: {} 2022-09-03T06:48:07.714 DEBUG:boto:Token: None 2022-09-03T06:48:07.714 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:07.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tLQjZJAiMKY7Z5078KlIiNx4gDc= 2022-09-03T06:48:07.716 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tLQjZJAiMKY7Z5078KlIiNx4gDc=', 'Content-Length': '0'} 2022-09-03T06:48:07.718 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006aa42a0054f3d9a0-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.718 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:48:07.719 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.720 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.720 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.721 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.721 DEBUG:boto:Method: GET 2022-09-03T06:48:07.722 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:07.722 DEBUG:boto:Data: 2022-09-03T06:48:07.723 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.723 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.724 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.724 DEBUG:boto:Params: {} 2022-09-03T06:48:07.725 DEBUG:boto:Token: None 2022-09-03T06:48:07.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:07.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T2e7MhhtRnQ1/SZSGXozQxsCb6E= 2022-09-03T06:48:07.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T2e7MhhtRnQ1/SZSGXozQxsCb6E=', 'Content-Length': '0'} 2022-09-03T06:48:07.729 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a8ce87aa7b335f9-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.730 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.730 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.731 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.731 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:07.732 DEBUG:boto:Method: GET 2022-09-03T06:48:07.732 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:07.733 DEBUG:boto:Data: 2022-09-03T06:48:07.733 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.734 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.735 DEBUG:boto:Params: {} 2022-09-03T06:48:07.736 DEBUG:boto:Token: None 2022-09-03T06:48:07.736 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:07.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T2e7MhhtRnQ1/SZSGXozQxsCb6E= 2022-09-03T06:48:07.738 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T2e7MhhtRnQ1/SZSGXozQxsCb6E=', 'Content-Length': '0'} 2022-09-03T06:48:07.740 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf9c44b2fa56da2f-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.788 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.788 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.789 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:07.790 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:07.791 DEBUG:boto:Method: GET 2022-09-03T06:48:07.791 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:07.792 DEBUG:boto:Data: 2022-09-03T06:48:07.793 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.793 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.794 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.795 DEBUG:boto:Params: {} 2022-09-03T06:48:07.797 DEBUG:boto:Token: None 2022-09-03T06:48:07.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.803 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fee1ad28a8dbe06e-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.804 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.806 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.807 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.807 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:07.808 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:07.808 DEBUG:boto:Method: GET 2022-09-03T06:48:07.809 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:07.809 DEBUG:boto:Data: 2022-09-03T06:48:07.810 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.811 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.811 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.812 DEBUG:boto:Params: {} 2022-09-03T06:48:07.812 DEBUG:boto:Token: None 2022-09-03T06:48:07.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.817 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f952358fa32ffd80-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.817 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.818 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.819 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.819 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.820 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:07.820 DEBUG:boto:Data: 2022-09-03T06:48:07.821 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.821 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.822 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.822 DEBUG:boto:Params: {} 2022-09-03T06:48:07.823 DEBUG:boto:Token: None 2022-09-03T06:48:07.824 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c 2022-09-03T06:48:07.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rMgqbMoCc/U1jhNqk2Mgc3Z/w8k= 2022-09-03T06:48:07.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rMgqbMoCc/U1jhNqk2Mgc3Z/w8k=', 'Content-Length': '0'} 2022-09-03T06:48:07.827 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a1e51daf0a80ced-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.828 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.829 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.829 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.830 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:07.831 DEBUG:boto:Data: 2022-09-03T06:48:07.831 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.833 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.834 DEBUG:boto:Params: {} 2022-09-03T06:48:07.834 DEBUG:boto:Token: None 2022-09-03T06:48:07.835 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c 2022-09-03T06:48:07.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rMgqbMoCc/U1jhNqk2Mgc3Z/w8k= 2022-09-03T06:48:07.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rMgqbMoCc/U1jhNqk2Mgc3Z/w8k=', 'Content-Length': '0'} 2022-09-03T06:48:07.838 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013468b442ba86313-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.839 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:48:07.840 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.840 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.841 DEBUG:boto:Method: GET 2022-09-03T06:48:07.842 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:07.842 DEBUG:boto:Data: 2022-09-03T06:48:07.843 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.843 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.844 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.844 DEBUG:boto:Params: {} 2022-09-03T06:48:07.845 DEBUG:boto:Token: None 2022-09-03T06:48:07.846 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c 2022-09-03T06:48:07.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zMemqevmgqmZ0QMzvi6Gz0uBkc0= 2022-09-03T06:48:07.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zMemqevmgqmZ0QMzvi6Gz0uBkc0=', 'Content-Length': '0'} 2022-09-03T06:48:07.849 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090e9e4ca13b1e093-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.890 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:07.891 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:07.892 DEBUG:boto:Method: GET 2022-09-03T06:48:07.892 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:07.893 DEBUG:boto:Data: 2022-09-03T06:48:07.893 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.894 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.895 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.896 DEBUG:boto:Params: {} 2022-09-03T06:48:07.897 DEBUG:boto:Token: None 2022-09-03T06:48:07.897 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/c 2022-09-03T06:48:07.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zMemqevmgqmZ0QMzvi6Gz0uBkc0= 2022-09-03T06:48:07.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zMemqevmgqmZ0QMzvi6Gz0uBkc0=', 'Content-Length': '0'} 2022-09-03T06:48:07.901 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c504faac7fbe71f-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.902 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.904 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.905 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:07.905 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:07.906 DEBUG:boto:Method: GET 2022-09-03T06:48:07.906 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:07.907 DEBUG:boto:Data: 2022-09-03T06:48:07.907 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.909 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.912 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.912 DEBUG:boto:Params: {} 2022-09-03T06:48:07.913 DEBUG:boto:Token: None 2022-09-03T06:48:07.914 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.918 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff203e2d3f27197b-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.919 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.920 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:07.921 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:07.921 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:07.922 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:07.923 DEBUG:boto:Method: GET 2022-09-03T06:48:07.924 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:07.925 DEBUG:boto:Data: 2022-09-03T06:48:07.926 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.929 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.930 DEBUG:boto:Params: {} 2022-09-03T06:48:07.932 DEBUG:boto:Token: None 2022-09-03T06:48:07.933 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/ 2022-09-03T06:48:07.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4= 2022-09-03T06:48:07.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TIkcbszzEsAUSNN5T4eUOmpSEb4=', 'Content-Length': '0'} 2022-09-03T06:48:07.939 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000107db3ee97696101-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.940 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:07.942 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:48:07.943 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:07.943 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:07.944 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:07.945 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:07.946 DEBUG:boto:Method: GET 2022-09-03T06:48:07.948 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:07.949 DEBUG:boto:Data: 2022-09-03T06:48:07.949 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.950 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.951 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.951 DEBUG:boto:Params: {} 2022-09-03T06:48:07.952 DEBUG:boto:Token: None 2022-09-03T06:48:07.953 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:07.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cLBPMo+mTb8tQa5Pv7LUWbj1Fv8= 2022-09-03T06:48:07.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cLBPMo+mTb8tQa5Pv7LUWbj1Fv8=', 'Content-Length': '0'} 2022-09-03T06:48:07.957 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000011d0ffe9195e8846-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.958 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:07.959 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:07.960 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:07.961 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:07.962 DEBUG:boto:Method: GET 2022-09-03T06:48:07.963 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:07.965 DEBUG:boto:Data: 2022-09-03T06:48:07.966 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:07.968 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.969 DEBUG:boto:Params: {} 2022-09-03T06:48:07.970 DEBUG:boto:Token: None 2022-09-03T06:48:07.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:07.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cLBPMo+mTb8tQa5Pv7LUWbj1Fv8= 2022-09-03T06:48:07.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cLBPMo+mTb8tQa5Pv7LUWbj1Fv8=', 'Content-Length': '0'} 2022-09-03T06:48:07.978 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a74f4a3ff0006c9-006312f8a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.979 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:07.980 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:07.980 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:07.981 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:07.982 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.983 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:07.984 DEBUG:boto:Data: 2022-09-03T06:48:07.985 DEBUG:boto:Headers: {} 2022-09-03T06:48:07.986 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:07.987 DEBUG:boto:Port: 8000 2022-09-03T06:48:07.988 DEBUG:boto:Params: {} 2022-09-03T06:48:07.989 DEBUG:boto:Token: None 2022-09-03T06:48:07.990 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:07 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:07.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:arcFOtxr9Ecp7GTgNYYNBOa6SXg= 2022-09-03T06:48:07.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:arcFOtxr9Ecp7GTgNYYNBOa6SXg=', 'Content-Length': '0'} 2022-09-03T06:48:07.995 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042cd00ea36bb4cc1-006312f8a7-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:07.996 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:07.997 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:07.997 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:07.998 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:07.998 DEBUG:boto:Method: HEAD 2022-09-03T06:48:07.999 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:07.999 DEBUG:boto:Data: 2022-09-03T06:48:08.000 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.000 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.001 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.002 DEBUG:boto:Params: {} 2022-09-03T06:48:08.002 DEBUG:boto:Token: None 2022-09-03T06:48:08.003 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:08.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SBzZPTiDzk+g6vwt/PIdlvxEJWc= 2022-09-03T06:48:08.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SBzZPTiDzk+g6vwt/PIdlvxEJWc=', 'Content-Length': '0'} 2022-09-03T06:48:08.007 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d033fc04c900ce7f-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.008 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:48:08.009 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:08.010 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:08.011 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:08.012 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:08.013 DEBUG:boto:Method: GET 2022-09-03T06:48:08.014 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:08.015 DEBUG:boto:Data: 2022-09-03T06:48:08.016 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.017 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:08.018 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.018 DEBUG:boto:Params: {} 2022-09-03T06:48:08.019 DEBUG:boto:Token: None 2022-09-03T06:48:08.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:08.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4Ph2AKO44u3/i0zMZ9cBFxnD7vA= 2022-09-03T06:48:08.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4Ph2AKO44u3/i0zMZ9cBFxnD7vA=', 'Content-Length': '0'} 2022-09-03T06:48:08.023 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086f9ab07c6b29ca8-006312f8a8-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.070 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:08.071 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:08.071 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:08.072 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:08.073 DEBUG:boto:Method: GET 2022-09-03T06:48:08.073 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:08.075 DEBUG:boto:Data: 2022-09-03T06:48:08.075 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.077 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.078 DEBUG:boto:Params: {} 2022-09-03T06:48:08.079 DEBUG:boto:Token: None 2022-09-03T06:48:08.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:08.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4Ph2AKO44u3/i0zMZ9cBFxnD7vA= 2022-09-03T06:48:08.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4Ph2AKO44u3/i0zMZ9cBFxnD7vA=', 'Content-Length': '0'} 2022-09-03T06:48:08.087 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3bd547db6fa6083-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.088 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.089 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.090 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:08.090 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:08.091 DEBUG:boto:Method: GET 2022-09-03T06:48:08.092 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:08.092 DEBUG:boto:Data: 2022-09-03T06:48:08.093 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.093 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:08.094 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.095 DEBUG:boto:Params: {} 2022-09-03T06:48:08.096 DEBUG:boto:Token: None 2022-09-03T06:48:08.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.103 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085b057106188af4b-006312f8a8-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.105 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.106 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.107 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.108 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:08.109 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:08.110 DEBUG:boto:Method: GET 2022-09-03T06:48:08.111 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:08.112 DEBUG:boto:Data: 2022-09-03T06:48:08.113 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.115 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.116 DEBUG:boto:Params: {} 2022-09-03T06:48:08.117 DEBUG:boto:Token: None 2022-09-03T06:48:08.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.124 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f5f5d5b7e58896e-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.125 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.127 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:08.128 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:08.129 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.130 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:08.131 DEBUG:boto:Data: 2022-09-03T06:48:08.133 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.134 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:08.135 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.136 DEBUG:boto:Params: {} 2022-09-03T06:48:08.137 DEBUG:boto:Token: None 2022-09-03T06:48:08.138 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/d 2022-09-03T06:48:08.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5cvnRN0ZKBY/hqdbF0/cz66Qo1Y= 2022-09-03T06:48:08.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5cvnRN0ZKBY/hqdbF0/cz66Qo1Y=', 'Content-Length': '0'} 2022-09-03T06:48:08.143 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010d0cceb3537cfd3-006312f8a8-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.144 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:08.145 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:08.146 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.147 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:08.148 DEBUG:boto:Data: 2022-09-03T06:48:08.149 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.151 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.152 DEBUG:boto:Params: {} 2022-09-03T06:48:08.153 DEBUG:boto:Token: None 2022-09-03T06:48:08.154 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/d 2022-09-03T06:48:08.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5cvnRN0ZKBY/hqdbF0/cz66Qo1Y= 2022-09-03T06:48:08.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5cvnRN0ZKBY/hqdbF0/cz66Qo1Y=', 'Content-Length': '0'} 2022-09-03T06:48:08.158 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034fcfd55c6c4f682-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.160 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:48:08.161 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:08.162 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:08.162 DEBUG:boto:Method: GET 2022-09-03T06:48:08.163 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:08.164 DEBUG:boto:Data: 2022-09-03T06:48:08.165 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.166 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:08.167 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.168 DEBUG:boto:Params: {} 2022-09-03T06:48:08.169 DEBUG:boto:Token: None 2022-09-03T06:48:08.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/d 2022-09-03T06:48:08.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A0Q88kOYnmQqdP+BvGexINSGlLM= 2022-09-03T06:48:08.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A0Q88kOYnmQqdP+BvGexINSGlLM=', 'Content-Length': '0'} 2022-09-03T06:48:08.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013894c4ae7bac45e-006312f8a8-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.180 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:08.181 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:08.182 DEBUG:boto:Method: GET 2022-09-03T06:48:08.183 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:08.183 DEBUG:boto:Data: 2022-09-03T06:48:08.184 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.186 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.187 DEBUG:boto:Params: {} 2022-09-03T06:48:08.188 DEBUG:boto:Token: None 2022-09-03T06:48:08.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/d 2022-09-03T06:48:08.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A0Q88kOYnmQqdP+BvGexINSGlLM= 2022-09-03T06:48:08.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A0Q88kOYnmQqdP+BvGexINSGlLM=', 'Content-Length': '0'} 2022-09-03T06:48:08.193 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000379f4966969b677a-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.194 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.195 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.196 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:08.196 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:08.197 DEBUG:boto:Method: GET 2022-09-03T06:48:08.197 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:08.198 DEBUG:boto:Data: 2022-09-03T06:48:08.199 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.199 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:08.200 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.200 DEBUG:boto:Params: {} 2022-09-03T06:48:08.201 DEBUG:boto:Token: None 2022-09-03T06:48:08.201 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.202 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.203 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.210 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cfad874cd468e8c5-006312f8a8-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.246 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.248 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.249 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.250 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:08.251 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:08.252 DEBUG:boto:Method: GET 2022-09-03T06:48:08.253 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:08.254 DEBUG:boto:Data: 2022-09-03T06:48:08.256 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.256 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.257 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.257 DEBUG:boto:Params: {} 2022-09-03T06:48:08.258 DEBUG:boto:Token: None 2022-09-03T06:48:08.259 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.263 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000015176744129bfcf0-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.263 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.264 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-69 zones={test-zone2, test-zone1} 2022-09-03T06:48:08.265 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-69 zones={test-zone3, test-zone1} 2022-09-03T06:48:08.266 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.267 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.268 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.268 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:48:08.269 DEBUG:boto:Data: 2022-09-03T06:48:08.269 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.270 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.271 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.271 DEBUG:boto:Params: {} 2022-09-03T06:48:08.272 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:48:08.272 DEBUG:boto:Token: None 2022-09-03T06:48:08.273 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pcR5d2Ub84l7Vfx6IA+4gxwRWpw= 2022-09-03T06:48:08.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pcR5d2Ub84l7Vfx6IA+4gxwRWpw=', 'Content-Length': '0'} 2022-09-03T06:48:08.280 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a46a57f671c34697-006312f8a8-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.295 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.296 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.297 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.298 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:48:08.299 DEBUG:boto:Data: 2022-09-03T06:48:08.300 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.302 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.303 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.305 DEBUG:boto:Params: {} 2022-09-03T06:48:08.306 DEBUG:boto:Token: None 2022-09-03T06:48:08.307 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pcR5d2Ub84l7Vfx6IA+4gxwRWpw= 2022-09-03T06:48:08.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pcR5d2Ub84l7Vfx6IA+4gxwRWpw=', 'Content-Length': '0'} 2022-09-03T06:48:08.311 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002b06ef59fc7ced72-006312f8a8-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.312 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:48:08.312 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.313 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.313 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.314 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.315 DEBUG:boto:Method: GET 2022-09-03T06:48:08.315 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.316 DEBUG:boto:Data: 2022-09-03T06:48:08.316 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.317 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.318 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.318 DEBUG:boto:Params: {} 2022-09-03T06:48:08.319 DEBUG:boto:Token: None 2022-09-03T06:48:08.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/?versions 2022-09-03T06:48:08.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TegchBUa9q1t50shkZplxbandmE= 2022-09-03T06:48:08.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TegchBUa9q1t50shkZplxbandmE=', 'Content-Length': '0'} 2022-09-03T06:48:08.324 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027db4e883e94f77d-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.325 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.327 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:48:08.328 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:48:08.328 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:48:08.329 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:48:08.329 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:48:08.330 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.330 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.331 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.331 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.332 DEBUG:boto:Method: GET 2022-09-03T06:48:08.332 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.333 DEBUG:boto:Data: 2022-09-03T06:48:08.334 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.335 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.335 DEBUG:boto:Params: {} 2022-09-03T06:48:08.336 DEBUG:boto:Token: None 2022-09-03T06:48:08.337 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/?versions 2022-09-03T06:48:08.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TegchBUa9q1t50shkZplxbandmE= 2022-09-03T06:48:08.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TegchBUa9q1t50shkZplxbandmE=', 'Content-Length': '0'} 2022-09-03T06:48:08.341 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044bc2119a352eaf8-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.341 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.342 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:48:08.343 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:48:08.343 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:48:08.344 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:48:08.345 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.345 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.346 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.346 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.347 DEBUG:boto:Method: GET 2022-09-03T06:48:08.348 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.348 DEBUG:boto:Data: 2022-09-03T06:48:08.349 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.350 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.351 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.352 DEBUG:boto:Params: {} 2022-09-03T06:48:08.353 DEBUG:boto:Token: None 2022-09-03T06:48:08.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/?versions 2022-09-03T06:48:08.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TegchBUa9q1t50shkZplxbandmE= 2022-09-03T06:48:08.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TegchBUa9q1t50shkZplxbandmE=', 'Content-Length': '0'} 2022-09-03T06:48:08.360 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000161f018042bb88fc-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.361 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.363 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.364 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.364 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.365 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.365 DEBUG:boto:Method: GET 2022-09-03T06:48:08.366 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:08.366 DEBUG:boto:Data: 2022-09-03T06:48:08.367 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.369 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.369 DEBUG:boto:Params: {} 2022-09-03T06:48:08.370 DEBUG:boto:Token: None 2022-09-03T06:48:08.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/?versions 2022-09-03T06:48:08.371 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TegchBUa9q1t50shkZplxbandmE= 2022-09-03T06:48:08.372 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TegchBUa9q1t50shkZplxbandmE=', 'Content-Length': '0'} 2022-09-03T06:48:08.376 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007fd575462eb8c53f-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.377 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.378 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:48:08.379 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.380 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.380 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.381 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.382 DEBUG:boto:Method: GET 2022-09-03T06:48:08.382 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:08.384 DEBUG:boto:Data: 2022-09-03T06:48:08.385 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.385 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.386 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.387 DEBUG:boto:Params: {} 2022-09-03T06:48:08.387 DEBUG:boto:Token: None 2022-09-03T06:48:08.388 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:08.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZTpPQ4ncp9exvND1SJNG989aUR0= 2022-09-03T06:48:08.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZTpPQ4ncp9exvND1SJNG989aUR0=', 'Content-Length': '0'} 2022-09-03T06:48:08.391 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac3778ad2acc26d8-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.392 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.393 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.393 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.394 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.395 DEBUG:boto:Method: GET 2022-09-03T06:48:08.395 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:08.396 DEBUG:boto:Data: 2022-09-03T06:48:08.397 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.398 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.398 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.399 DEBUG:boto:Params: {} 2022-09-03T06:48:08.400 DEBUG:boto:Token: None 2022-09-03T06:48:08.400 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:08.401 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZTpPQ4ncp9exvND1SJNG989aUR0= 2022-09-03T06:48:08.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZTpPQ4ncp9exvND1SJNG989aUR0=', 'Content-Length': '0'} 2022-09-03T06:48:08.404 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a02bc144b09ec4a-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.446 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.447 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.447 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.448 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.449 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.450 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:08.451 DEBUG:boto:Data: 2022-09-03T06:48:08.452 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.454 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.455 DEBUG:boto:Params: {} 2022-09-03T06:48:08.456 DEBUG:boto:Token: None 2022-09-03T06:48:08.457 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:08.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ycly0DGHCUoKq2Z6ggoBeOHWvmc= 2022-09-03T06:48:08.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ycly0DGHCUoKq2Z6ggoBeOHWvmc=', 'Content-Length': '0'} 2022-09-03T06:48:08.461 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a589ea66ecd32dc3-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.462 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.463 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.464 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.464 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.465 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.466 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:08.466 DEBUG:boto:Data: 2022-09-03T06:48:08.467 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.468 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.468 DEBUG:boto:Params: {} 2022-09-03T06:48:08.469 DEBUG:boto:Token: None 2022-09-03T06:48:08.469 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:08.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ycly0DGHCUoKq2Z6ggoBeOHWvmc= 2022-09-03T06:48:08.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ycly0DGHCUoKq2Z6ggoBeOHWvmc=', 'Content-Length': '0'} 2022-09-03T06:48:08.473 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fe2e583d30a4d1b-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.474 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:48:08.474 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.475 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.475 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.476 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.477 DEBUG:boto:Method: GET 2022-09-03T06:48:08.477 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:08.478 DEBUG:boto:Data: 2022-09-03T06:48:08.478 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.479 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.480 DEBUG:boto:Params: {} 2022-09-03T06:48:08.480 DEBUG:boto:Token: None 2022-09-03T06:48:08.481 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:08.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZTpPQ4ncp9exvND1SJNG989aUR0= 2022-09-03T06:48:08.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZTpPQ4ncp9exvND1SJNG989aUR0=', 'Content-Length': '0'} 2022-09-03T06:48:08.484 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000280793b1f2c573cf-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.485 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.486 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.486 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.487 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:08.488 DEBUG:boto:Method: GET 2022-09-03T06:48:08.488 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:08.489 DEBUG:boto:Data: 2022-09-03T06:48:08.489 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.491 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.491 DEBUG:boto:Params: {} 2022-09-03T06:48:08.492 DEBUG:boto:Token: None 2022-09-03T06:48:08.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:08.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZTpPQ4ncp9exvND1SJNG989aUR0= 2022-09-03T06:48:08.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZTpPQ4ncp9exvND1SJNG989aUR0=', 'Content-Length': '0'} 2022-09-03T06:48:08.497 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b5f6bbebdba4d20-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.497 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.498 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.499 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:08.499 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:08.500 DEBUG:boto:Method: GET 2022-09-03T06:48:08.500 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:08.501 DEBUG:boto:Data: 2022-09-03T06:48:08.501 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.503 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.503 DEBUG:boto:Params: {} 2022-09-03T06:48:08.504 DEBUG:boto:Token: None 2022-09-03T06:48:08.504 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.509 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002169302a7c10a03-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.509 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.510 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.511 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.511 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:08.512 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:08.513 DEBUG:boto:Method: GET 2022-09-03T06:48:08.513 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:08.514 DEBUG:boto:Data: 2022-09-03T06:48:08.514 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.515 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.516 DEBUG:boto:Params: {} 2022-09-03T06:48:08.517 DEBUG:boto:Token: None 2022-09-03T06:48:08.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.522 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e4cf66a1457bca5f-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.522 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.523 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.524 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.524 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.525 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:08.525 DEBUG:boto:Data: 2022-09-03T06:48:08.526 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.526 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.527 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.527 DEBUG:boto:Params: {} 2022-09-03T06:48:08.528 DEBUG:boto:Token: None 2022-09-03T06:48:08.529 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a 2022-09-03T06:48:08.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cSJvjnyuYgTkzN+f+hJEDxW7G5M= 2022-09-03T06:48:08.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cSJvjnyuYgTkzN+f+hJEDxW7G5M=', 'Content-Length': '0'} 2022-09-03T06:48:08.532 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad4183ec9b74e6c4-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.533 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.534 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.535 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.536 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:08.537 DEBUG:boto:Data: 2022-09-03T06:48:08.538 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.539 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.540 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.541 DEBUG:boto:Params: {} 2022-09-03T06:48:08.542 DEBUG:boto:Token: None 2022-09-03T06:48:08.542 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a 2022-09-03T06:48:08.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cSJvjnyuYgTkzN+f+hJEDxW7G5M= 2022-09-03T06:48:08.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cSJvjnyuYgTkzN+f+hJEDxW7G5M=', 'Content-Length': '0'} 2022-09-03T06:48:08.546 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a18411cb4556845-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.547 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:48:08.548 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.548 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.549 DEBUG:boto:Method: GET 2022-09-03T06:48:08.550 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:08.550 DEBUG:boto:Data: 2022-09-03T06:48:08.551 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.552 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.553 DEBUG:boto:Params: {} 2022-09-03T06:48:08.553 DEBUG:boto:Token: None 2022-09-03T06:48:08.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a 2022-09-03T06:48:08.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ahcw5EFAsKfdPsxoQfyb9wZ6Tcw= 2022-09-03T06:48:08.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ahcw5EFAsKfdPsxoQfyb9wZ6Tcw=', 'Content-Length': '0'} 2022-09-03T06:48:08.558 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008708ef359affdc04-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.559 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:08.560 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:08.560 DEBUG:boto:Method: GET 2022-09-03T06:48:08.561 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:08.562 DEBUG:boto:Data: 2022-09-03T06:48:08.562 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.563 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.563 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.564 DEBUG:boto:Params: {} 2022-09-03T06:48:08.565 DEBUG:boto:Token: None 2022-09-03T06:48:08.565 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/a 2022-09-03T06:48:08.566 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ahcw5EFAsKfdPsxoQfyb9wZ6Tcw= 2022-09-03T06:48:08.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ahcw5EFAsKfdPsxoQfyb9wZ6Tcw=', 'Content-Length': '0'} 2022-09-03T06:48:08.568 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ebbee79f55b9568e-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.569 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.570 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.571 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:08.571 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:08.572 DEBUG:boto:Method: GET 2022-09-03T06:48:08.573 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:08.573 DEBUG:boto:Data: 2022-09-03T06:48:08.574 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.574 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.575 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.576 DEBUG:boto:Params: {} 2022-09-03T06:48:08.576 DEBUG:boto:Token: None 2022-09-03T06:48:08.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.581 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a8eaa184e47c79e6-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.581 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.582 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.583 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.584 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:08.584 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:08.585 DEBUG:boto:Method: GET 2022-09-03T06:48:08.585 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:08.586 DEBUG:boto:Data: 2022-09-03T06:48:08.586 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.587 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.588 DEBUG:boto:Params: {} 2022-09-03T06:48:08.588 DEBUG:boto:Token: None 2022-09-03T06:48:08.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.593 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c44efe07aad69e84-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.593 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.594 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:48:08.595 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.596 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.596 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.597 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.597 DEBUG:boto:Method: GET 2022-09-03T06:48:08.598 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:08.598 DEBUG:boto:Data: 2022-09-03T06:48:08.599 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.600 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.601 DEBUG:boto:Params: {} 2022-09-03T06:48:08.602 DEBUG:boto:Token: None 2022-09-03T06:48:08.602 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:08.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jekbPt/9yR07ik84O/2zeP5uu50= 2022-09-03T06:48:08.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jekbPt/9yR07ik84O/2zeP5uu50=', 'Content-Length': '0'} 2022-09-03T06:48:08.606 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7494327f88eb159-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.607 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.607 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.608 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.608 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.609 DEBUG:boto:Method: GET 2022-09-03T06:48:08.610 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:08.610 DEBUG:boto:Data: 2022-09-03T06:48:08.611 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.612 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.612 DEBUG:boto:Params: {} 2022-09-03T06:48:08.613 DEBUG:boto:Token: None 2022-09-03T06:48:08.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:08.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jekbPt/9yR07ik84O/2zeP5uu50= 2022-09-03T06:48:08.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jekbPt/9yR07ik84O/2zeP5uu50=', 'Content-Length': '0'} 2022-09-03T06:48:08.617 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000775394bbc346f4fc-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.618 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.619 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.619 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.620 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.620 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.621 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:08.621 DEBUG:boto:Data: 2022-09-03T06:48:08.622 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.623 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.623 DEBUG:boto:Params: {} 2022-09-03T06:48:08.624 DEBUG:boto:Token: None 2022-09-03T06:48:08.624 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:08.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cZ4l/KLFl9gzTdGstFh4CchQy38= 2022-09-03T06:48:08.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cZ4l/KLFl9gzTdGstFh4CchQy38=', 'Content-Length': '0'} 2022-09-03T06:48:08.627 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004dee11fb5a063bbb-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.628 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.629 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.629 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.630 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.630 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.631 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:08.631 DEBUG:boto:Data: 2022-09-03T06:48:08.632 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.632 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.633 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.633 DEBUG:boto:Params: {} 2022-09-03T06:48:08.634 DEBUG:boto:Token: None 2022-09-03T06:48:08.635 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:08.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cZ4l/KLFl9gzTdGstFh4CchQy38= 2022-09-03T06:48:08.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cZ4l/KLFl9gzTdGstFh4CchQy38=', 'Content-Length': '0'} 2022-09-03T06:48:08.638 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000279bec583d04055c-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.639 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:48:08.639 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.640 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.640 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.641 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.641 DEBUG:boto:Method: GET 2022-09-03T06:48:08.642 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:08.642 DEBUG:boto:Data: 2022-09-03T06:48:08.643 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.644 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.645 DEBUG:boto:Params: {} 2022-09-03T06:48:08.645 DEBUG:boto:Token: None 2022-09-03T06:48:08.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:08.647 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jekbPt/9yR07ik84O/2zeP5uu50= 2022-09-03T06:48:08.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jekbPt/9yR07ik84O/2zeP5uu50=', 'Content-Length': '0'} 2022-09-03T06:48:08.649 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e75622a2f2a32b8b-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.650 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.651 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.651 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.652 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:08.652 DEBUG:boto:Method: GET 2022-09-03T06:48:08.653 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:08.654 DEBUG:boto:Data: 2022-09-03T06:48:08.654 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.655 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.656 DEBUG:boto:Params: {} 2022-09-03T06:48:08.656 DEBUG:boto:Token: None 2022-09-03T06:48:08.657 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:08.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jekbPt/9yR07ik84O/2zeP5uu50= 2022-09-03T06:48:08.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jekbPt/9yR07ik84O/2zeP5uu50=', 'Content-Length': '0'} 2022-09-03T06:48:08.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091c4df12e111958e-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.661 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.662 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.663 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:08.663 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:08.664 DEBUG:boto:Method: GET 2022-09-03T06:48:08.664 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:08.665 DEBUG:boto:Data: 2022-09-03T06:48:08.665 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.666 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.667 DEBUG:boto:Params: {} 2022-09-03T06:48:08.667 DEBUG:boto:Token: None 2022-09-03T06:48:08.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.671 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000de26211139f4f7ea-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.672 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.673 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.674 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.674 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:08.675 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:08.675 DEBUG:boto:Method: GET 2022-09-03T06:48:08.676 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:08.676 DEBUG:boto:Data: 2022-09-03T06:48:08.677 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.678 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.679 DEBUG:boto:Params: {} 2022-09-03T06:48:08.679 DEBUG:boto:Token: None 2022-09-03T06:48:08.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.683 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce621dc05d997f54-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.684 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.685 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.686 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.686 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.687 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:08.687 DEBUG:boto:Data: 2022-09-03T06:48:08.688 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.689 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.689 DEBUG:boto:Params: {} 2022-09-03T06:48:08.690 DEBUG:boto:Token: None 2022-09-03T06:48:08.690 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b 2022-09-03T06:48:08.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pxKCKa4HFo79koXoDIltHYkQmj4= 2022-09-03T06:48:08.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pxKCKa4HFo79koXoDIltHYkQmj4=', 'Content-Length': '0'} 2022-09-03T06:48:08.693 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042e90928970daa6f-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.694 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.695 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.695 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.696 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:08.696 DEBUG:boto:Data: 2022-09-03T06:48:08.697 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.697 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.698 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.698 DEBUG:boto:Params: {} 2022-09-03T06:48:08.699 DEBUG:boto:Token: None 2022-09-03T06:48:08.700 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b 2022-09-03T06:48:08.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pxKCKa4HFo79koXoDIltHYkQmj4= 2022-09-03T06:48:08.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pxKCKa4HFo79koXoDIltHYkQmj4=', 'Content-Length': '0'} 2022-09-03T06:48:08.703 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d78e4d3220c2895-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.704 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:48:08.704 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.705 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.705 DEBUG:boto:Method: GET 2022-09-03T06:48:08.706 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:08.707 DEBUG:boto:Data: 2022-09-03T06:48:08.707 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.708 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.708 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.709 DEBUG:boto:Params: {} 2022-09-03T06:48:08.709 DEBUG:boto:Token: None 2022-09-03T06:48:08.710 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b 2022-09-03T06:48:08.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A7yWj+MhoKu2YE7domswHDhma6E= 2022-09-03T06:48:08.711 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A7yWj+MhoKu2YE7domswHDhma6E=', 'Content-Length': '0'} 2022-09-03T06:48:08.713 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e17074c4b64607ad-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.714 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:08.714 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:08.715 DEBUG:boto:Method: GET 2022-09-03T06:48:08.716 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:08.716 DEBUG:boto:Data: 2022-09-03T06:48:08.717 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.719 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.719 DEBUG:boto:Params: {} 2022-09-03T06:48:08.721 DEBUG:boto:Token: None 2022-09-03T06:48:08.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/b 2022-09-03T06:48:08.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A7yWj+MhoKu2YE7domswHDhma6E= 2022-09-03T06:48:08.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A7yWj+MhoKu2YE7domswHDhma6E=', 'Content-Length': '0'} 2022-09-03T06:48:08.726 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024b18738f383ea69-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.774 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.775 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.775 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:08.776 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:08.776 DEBUG:boto:Method: GET 2022-09-03T06:48:08.777 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:08.778 DEBUG:boto:Data: 2022-09-03T06:48:08.778 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.779 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.780 DEBUG:boto:Params: {} 2022-09-03T06:48:08.780 DEBUG:boto:Token: None 2022-09-03T06:48:08.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.785 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009d99378ef5baa988-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.786 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.787 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.787 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.788 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:08.788 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:08.789 DEBUG:boto:Method: GET 2022-09-03T06:48:08.789 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:08.790 DEBUG:boto:Data: 2022-09-03T06:48:08.790 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.791 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.792 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.792 DEBUG:boto:Params: {} 2022-09-03T06:48:08.793 DEBUG:boto:Token: None 2022-09-03T06:48:08.793 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.797 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d061f74edde25012-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.802 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.803 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:48:08.803 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.804 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.804 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.805 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.805 DEBUG:boto:Method: GET 2022-09-03T06:48:08.806 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:08.807 DEBUG:boto:Data: 2022-09-03T06:48:08.807 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.809 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.809 DEBUG:boto:Params: {} 2022-09-03T06:48:08.810 DEBUG:boto:Token: None 2022-09-03T06:48:08.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:08.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+hEJcxhr1+bfSYX/xLae4/najk= 2022-09-03T06:48:08.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+hEJcxhr1+bfSYX/xLae4/najk=', 'Content-Length': '0'} 2022-09-03T06:48:08.814 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ba810af5f1eeed6-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.815 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.815 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.816 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.816 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.817 DEBUG:boto:Method: GET 2022-09-03T06:48:08.817 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:08.818 DEBUG:boto:Data: 2022-09-03T06:48:08.818 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.819 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.820 DEBUG:boto:Params: {} 2022-09-03T06:48:08.820 DEBUG:boto:Token: None 2022-09-03T06:48:08.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:08.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+hEJcxhr1+bfSYX/xLae4/najk= 2022-09-03T06:48:08.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+hEJcxhr1+bfSYX/xLae4/najk=', 'Content-Length': '0'} 2022-09-03T06:48:08.824 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007782f904114ac9fc-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.825 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.826 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.826 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.827 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.827 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.828 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:08.828 DEBUG:boto:Data: 2022-09-03T06:48:08.829 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.830 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.831 DEBUG:boto:Params: {} 2022-09-03T06:48:08.831 DEBUG:boto:Token: None 2022-09-03T06:48:08.832 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:08.832 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KlCTl6UHOMvJ3VJuNaZIMlbIE+k= 2022-09-03T06:48:08.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KlCTl6UHOMvJ3VJuNaZIMlbIE+k=', 'Content-Length': '0'} 2022-09-03T06:48:08.836 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007027370c7c830a65-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.837 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.837 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.838 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.838 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.839 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.840 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:08.840 DEBUG:boto:Data: 2022-09-03T06:48:08.841 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.842 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.843 DEBUG:boto:Params: {} 2022-09-03T06:48:08.844 DEBUG:boto:Token: None 2022-09-03T06:48:08.845 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:08.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KlCTl6UHOMvJ3VJuNaZIMlbIE+k= 2022-09-03T06:48:08.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KlCTl6UHOMvJ3VJuNaZIMlbIE+k=', 'Content-Length': '0'} 2022-09-03T06:48:08.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005bea8a43168de6e4-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.849 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:48:08.850 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.850 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.851 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.852 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.852 DEBUG:boto:Method: GET 2022-09-03T06:48:08.853 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:08.854 DEBUG:boto:Data: 2022-09-03T06:48:08.854 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.855 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.856 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.856 DEBUG:boto:Params: {} 2022-09-03T06:48:08.858 DEBUG:boto:Token: None 2022-09-03T06:48:08.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:08.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+hEJcxhr1+bfSYX/xLae4/najk= 2022-09-03T06:48:08.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+hEJcxhr1+bfSYX/xLae4/najk=', 'Content-Length': '0'} 2022-09-03T06:48:08.864 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b1ba2f3da604393-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.865 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.866 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.867 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.868 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:08.869 DEBUG:boto:Method: GET 2022-09-03T06:48:08.870 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:08.871 DEBUG:boto:Data: 2022-09-03T06:48:08.872 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.873 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.874 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.875 DEBUG:boto:Params: {} 2022-09-03T06:48:08.877 DEBUG:boto:Token: None 2022-09-03T06:48:08.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:08.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G+hEJcxhr1+bfSYX/xLae4/najk= 2022-09-03T06:48:08.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G+hEJcxhr1+bfSYX/xLae4/najk=', 'Content-Length': '0'} 2022-09-03T06:48:08.882 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd243491b360b767-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.884 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.884 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.885 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:08.886 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:08.886 DEBUG:boto:Method: GET 2022-09-03T06:48:08.887 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:08.887 DEBUG:boto:Data: 2022-09-03T06:48:08.888 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.888 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.889 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.890 DEBUG:boto:Params: {} 2022-09-03T06:48:08.890 DEBUG:boto:Token: None 2022-09-03T06:48:08.891 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.895 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005a87290555110f8b-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.895 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.896 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:08.897 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:08.897 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:08.898 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:08.898 DEBUG:boto:Method: GET 2022-09-03T06:48:08.899 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:08.899 DEBUG:boto:Data: 2022-09-03T06:48:08.900 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.901 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.901 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.902 DEBUG:boto:Params: {} 2022-09-03T06:48:08.903 DEBUG:boto:Token: None 2022-09-03T06:48:08.903 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/ 2022-09-03T06:48:08.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4= 2022-09-03T06:48:08.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dXqpyk13/i707Wsu+ly06gvYHz4=', 'Content-Length': '0'} 2022-09-03T06:48:08.907 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c5e3f9fa5cd48bbc-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.908 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:08.908 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.909 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.910 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.910 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:08.911 DEBUG:boto:Data: 2022-09-03T06:48:08.911 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.912 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.912 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.913 DEBUG:boto:Params: {} 2022-09-03T06:48:08.913 DEBUG:boto:Token: None 2022-09-03T06:48:08.914 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/c 2022-09-03T06:48:08.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EUqRDi9fgkXDW19FarN151qogTs= 2022-09-03T06:48:08.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EUqRDi9fgkXDW19FarN151qogTs=', 'Content-Length': '0'} 2022-09-03T06:48:08.917 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000afa484c375812c5d-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.918 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.918 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.919 DEBUG:boto:Method: HEAD 2022-09-03T06:48:08.920 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:08.920 DEBUG:boto:Data: 2022-09-03T06:48:08.921 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:08.922 DEBUG:boto:Port: 8000 2022-09-03T06:48:08.923 DEBUG:boto:Params: {} 2022-09-03T06:48:08.923 DEBUG:boto:Token: None 2022-09-03T06:48:08.924 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/c 2022-09-03T06:48:08.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EUqRDi9fgkXDW19FarN151qogTs= 2022-09-03T06:48:08.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EUqRDi9fgkXDW19FarN151qogTs=', 'Content-Length': '0'} 2022-09-03T06:48:08.927 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020256e1572273556-006312f8a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.928 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:48:08.928 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.929 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.929 DEBUG:boto:Method: GET 2022-09-03T06:48:08.930 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:08.930 DEBUG:boto:Data: 2022-09-03T06:48:08.931 DEBUG:boto:Headers: {} 2022-09-03T06:48:08.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:08.932 DEBUG:boto:Port: 8001 2022-09-03T06:48:08.941 DEBUG:boto:Params: {} 2022-09-03T06:48:08.942 DEBUG:boto:Token: None 2022-09-03T06:48:08.943 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:08 GMT /avysmw-69/c 2022-09-03T06:48:08.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:waa5JFct78St9wYMVb6KKG9Jl0A= 2022-09-03T06:48:08.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:waa5JFct78St9wYMVb6KKG9Jl0A=', 'Content-Length': '0'} 2022-09-03T06:48:08.946 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000873a550ef0659996-006312f8a8-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:08.990 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:08.991 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:08.992 DEBUG:boto:Method: GET 2022-09-03T06:48:08.993 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:08.998 DEBUG:boto:Data: 2022-09-03T06:48:08.999 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.000 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:09.001 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.002 DEBUG:boto:Params: {} 2022-09-03T06:48:09.004 DEBUG:boto:Token: None 2022-09-03T06:48:09.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/c 2022-09-03T06:48:09.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4Mx9najd4Bm5aX7OREJh4xS+h9M= 2022-09-03T06:48:09.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4Mx9najd4Bm5aX7OREJh4xS+h9M=', 'Content-Length': '0'} 2022-09-03T06:48:09.010 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b1b5de056f56f09-006312f8a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.012 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.013 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.014 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:09.015 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:09.016 DEBUG:boto:Method: GET 2022-09-03T06:48:09.017 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:09.018 DEBUG:boto:Data: 2022-09-03T06:48:09.019 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.021 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.021 DEBUG:boto:Params: {} 2022-09-03T06:48:09.023 DEBUG:boto:Token: None 2022-09-03T06:48:09.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.028 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004b51b7ae169e7626-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.029 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.030 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.031 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.031 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:09.032 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:09.032 DEBUG:boto:Method: GET 2022-09-03T06:48:09.033 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:09.033 DEBUG:boto:Data: 2022-09-03T06:48:09.034 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.034 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:09.035 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.035 DEBUG:boto:Params: {} 2022-09-03T06:48:09.036 DEBUG:boto:Token: None 2022-09-03T06:48:09.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.040 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bef160ff366b9967-006312f8a9-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.041 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.042 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:48:09.042 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.043 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.044 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.044 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.045 DEBUG:boto:Method: GET 2022-09-03T06:48:09.045 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:09.046 DEBUG:boto:Data: 2022-09-03T06:48:09.047 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.048 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.048 DEBUG:boto:Params: {} 2022-09-03T06:48:09.049 DEBUG:boto:Token: None 2022-09-03T06:48:09.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:09.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uxGTTKBs3PcZFM2RToUfpeyRsKY= 2022-09-03T06:48:09.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uxGTTKBs3PcZFM2RToUfpeyRsKY=', 'Content-Length': '0'} 2022-09-03T06:48:09.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f91ea974b3abeeb-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.054 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.055 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.055 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.056 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.056 DEBUG:boto:Method: GET 2022-09-03T06:48:09.057 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:09.057 DEBUG:boto:Data: 2022-09-03T06:48:09.058 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.058 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:09.059 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.060 DEBUG:boto:Params: {} 2022-09-03T06:48:09.060 DEBUG:boto:Token: None 2022-09-03T06:48:09.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:09.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uxGTTKBs3PcZFM2RToUfpeyRsKY= 2022-09-03T06:48:09.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uxGTTKBs3PcZFM2RToUfpeyRsKY=', 'Content-Length': '0'} 2022-09-03T06:48:09.064 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000894dcb247f72be6e-006312f8a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.065 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.066 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.067 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.068 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.069 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.070 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:09.071 DEBUG:boto:Data: 2022-09-03T06:48:09.071 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.072 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.073 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.074 DEBUG:boto:Params: {} 2022-09-03T06:48:09.075 DEBUG:boto:Token: None 2022-09-03T06:48:09.076 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:09.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7prHxcdLj0jHwjnC8Sv+Qso2ar8= 2022-09-03T06:48:09.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7prHxcdLj0jHwjnC8Sv+Qso2ar8=', 'Content-Length': '0'} 2022-09-03T06:48:09.081 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec7d74600ed48876-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.082 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.082 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.083 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.084 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.084 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.085 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:09.085 DEBUG:boto:Data: 2022-09-03T06:48:09.086 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.086 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:09.087 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.088 DEBUG:boto:Params: {} 2022-09-03T06:48:09.088 DEBUG:boto:Token: None 2022-09-03T06:48:09.089 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:09.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7prHxcdLj0jHwjnC8Sv+Qso2ar8= 2022-09-03T06:48:09.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7prHxcdLj0jHwjnC8Sv+Qso2ar8=', 'Content-Length': '0'} 2022-09-03T06:48:09.092 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e83fea0dd766bad-006312f8a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.093 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:48:09.093 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.094 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.095 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.095 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.096 DEBUG:boto:Method: GET 2022-09-03T06:48:09.096 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:09.097 DEBUG:boto:Data: 2022-09-03T06:48:09.098 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.098 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.099 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.099 DEBUG:boto:Params: {} 2022-09-03T06:48:09.101 DEBUG:boto:Token: None 2022-09-03T06:48:09.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:09.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uxGTTKBs3PcZFM2RToUfpeyRsKY= 2022-09-03T06:48:09.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uxGTTKBs3PcZFM2RToUfpeyRsKY=', 'Content-Length': '0'} 2022-09-03T06:48:09.106 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006dde7e5e1cfc5a1c-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.106 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.107 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.108 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.108 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:09.109 DEBUG:boto:Method: GET 2022-09-03T06:48:09.109 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:09.110 DEBUG:boto:Data: 2022-09-03T06:48:09.110 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:09.111 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.112 DEBUG:boto:Params: {} 2022-09-03T06:48:09.112 DEBUG:boto:Token: None 2022-09-03T06:48:09.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:09.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uxGTTKBs3PcZFM2RToUfpeyRsKY= 2022-09-03T06:48:09.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uxGTTKBs3PcZFM2RToUfpeyRsKY=', 'Content-Length': '0'} 2022-09-03T06:48:09.116 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000758f4f4250fdb18d-006312f8a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.117 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.118 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.118 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:09.119 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:09.119 DEBUG:boto:Method: GET 2022-09-03T06:48:09.120 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:09.121 DEBUG:boto:Data: 2022-09-03T06:48:09.121 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.122 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.123 DEBUG:boto:Params: {} 2022-09-03T06:48:09.123 DEBUG:boto:Token: None 2022-09-03T06:48:09.124 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.129 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac9738c8c5c7c1ad-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.130 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.132 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.133 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.134 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:09.134 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:09.135 DEBUG:boto:Method: GET 2022-09-03T06:48:09.136 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:09.137 DEBUG:boto:Data: 2022-09-03T06:48:09.138 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.139 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:09.140 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.141 DEBUG:boto:Params: {} 2022-09-03T06:48:09.142 DEBUG:boto:Token: None 2022-09-03T06:48:09.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.147 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000053b9f0c959f81339-006312f8a9-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.148 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.149 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.150 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.151 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.151 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:09.152 DEBUG:boto:Data: 2022-09-03T06:48:09.153 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.153 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.154 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.154 DEBUG:boto:Params: {} 2022-09-03T06:48:09.155 DEBUG:boto:Token: None 2022-09-03T06:48:09.156 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d 2022-09-03T06:48:09.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gdykQZLUqpdU5NOWQuglYAEpAsc= 2022-09-03T06:48:09.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gdykQZLUqpdU5NOWQuglYAEpAsc=', 'Content-Length': '0'} 2022-09-03T06:48:09.159 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000631449b4059617e7-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.160 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.160 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.161 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.161 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:09.162 DEBUG:boto:Data: 2022-09-03T06:48:09.162 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:09.163 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.164 DEBUG:boto:Params: {} 2022-09-03T06:48:09.164 DEBUG:boto:Token: None 2022-09-03T06:48:09.165 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d 2022-09-03T06:48:09.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gdykQZLUqpdU5NOWQuglYAEpAsc= 2022-09-03T06:48:09.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gdykQZLUqpdU5NOWQuglYAEpAsc=', 'Content-Length': '0'} 2022-09-03T06:48:09.168 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d544d64bdfa84ef-006312f8a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.169 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:48:09.169 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.170 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.170 DEBUG:boto:Method: GET 2022-09-03T06:48:09.171 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:09.171 DEBUG:boto:Data: 2022-09-03T06:48:09.172 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.173 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.173 DEBUG:boto:Params: {} 2022-09-03T06:48:09.174 DEBUG:boto:Token: None 2022-09-03T06:48:09.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d 2022-09-03T06:48:09.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JdmiPV5CjMv41daghtEeGQIMHl8= 2022-09-03T06:48:09.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JdmiPV5CjMv41daghtEeGQIMHl8=', 'Content-Length': '0'} 2022-09-03T06:48:09.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000030ef9980c0d017c8-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.179 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:09.179 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:09.180 DEBUG:boto:Method: GET 2022-09-03T06:48:09.180 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:09.181 DEBUG:boto:Data: 2022-09-03T06:48:09.181 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:09.182 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.183 DEBUG:boto:Params: {} 2022-09-03T06:48:09.184 DEBUG:boto:Token: None 2022-09-03T06:48:09.184 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/d 2022-09-03T06:48:09.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JdmiPV5CjMv41daghtEeGQIMHl8= 2022-09-03T06:48:09.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JdmiPV5CjMv41daghtEeGQIMHl8=', 'Content-Length': '0'} 2022-09-03T06:48:09.187 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec2c4bbb6f929b16-006312f8a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.188 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.189 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.189 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:09.190 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:09.190 DEBUG:boto:Method: GET 2022-09-03T06:48:09.191 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:09.191 DEBUG:boto:Data: 2022-09-03T06:48:09.192 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.193 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.193 DEBUG:boto:Params: {} 2022-09-03T06:48:09.194 DEBUG:boto:Token: None 2022-09-03T06:48:09.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.198 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000784215971428463e-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.200 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.201 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.202 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.203 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:09.204 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:09.205 DEBUG:boto:Method: GET 2022-09-03T06:48:09.206 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:09.207 DEBUG:boto:Data: 2022-09-03T06:48:09.208 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:09.212 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.214 DEBUG:boto:Params: {} 2022-09-03T06:48:09.215 DEBUG:boto:Token: None 2022-09-03T06:48:09.216 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.222 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007bec0ae81bd1e810-006312f8a9-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.223 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.225 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-69 zones={test-zone3, test-zone1} 2022-09-03T06:48:09.226 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-69 zones={test-zone3, test-zone2} 2022-09-03T06:48:09.226 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.227 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.228 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.228 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:48:09.229 DEBUG:boto:Data: 2022-09-03T06:48:09.230 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.232 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.233 DEBUG:boto:Params: {} 2022-09-03T06:48:09.234 DEBUG:boto:Token: None 2022-09-03T06:48:09.235 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mRf2K9qTFsbSE4HErubl08IgNyA= 2022-09-03T06:48:09.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mRf2K9qTFsbSE4HErubl08IgNyA=', 'Content-Length': '0'} 2022-09-03T06:48:09.240 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009ea231bf5a59ce1d-006312f8a9-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.241 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.241 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.242 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.243 DEBUG:boto:Path: /avysmw-69/ 2022-09-03T06:48:09.243 DEBUG:boto:Data: 2022-09-03T06:48:09.244 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.244 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.245 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.246 DEBUG:boto:Params: {} 2022-09-03T06:48:09.246 DEBUG:boto:Token: None 2022-09-03T06:48:09.247 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mRf2K9qTFsbSE4HErubl08IgNyA= 2022-09-03T06:48:09.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mRf2K9qTFsbSE4HErubl08IgNyA=', 'Content-Length': '0'} 2022-09-03T06:48:09.252 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c29dccf88183c172-006312f8a9-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.252 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:48:09.253 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.253 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.254 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.254 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.255 DEBUG:boto:Method: GET 2022-09-03T06:48:09.256 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.256 DEBUG:boto:Data: 2022-09-03T06:48:09.257 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.257 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.258 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.258 DEBUG:boto:Params: {} 2022-09-03T06:48:09.259 DEBUG:boto:Token: None 2022-09-03T06:48:09.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/?versions 2022-09-03T06:48:09.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xesm9eAiq3pPE019Q4VTI/S2AVg= 2022-09-03T06:48:09.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xesm9eAiq3pPE019Q4VTI/S2AVg=', 'Content-Length': '0'} 2022-09-03T06:48:09.264 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000462caaf524669bb-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.264 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.266 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:48:09.267 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:48:09.267 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:48:09.268 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:48:09.268 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:48:09.269 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.269 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.270 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.270 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.271 DEBUG:boto:Method: GET 2022-09-03T06:48:09.271 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.272 DEBUG:boto:Data: 2022-09-03T06:48:09.272 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.273 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.274 DEBUG:boto:Params: {} 2022-09-03T06:48:09.275 DEBUG:boto:Token: None 2022-09-03T06:48:09.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/?versions 2022-09-03T06:48:09.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xesm9eAiq3pPE019Q4VTI/S2AVg= 2022-09-03T06:48:09.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xesm9eAiq3pPE019Q4VTI/S2AVg=', 'Content-Length': '0'} 2022-09-03T06:48:09.279 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000ea05abd48c983ee-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.280 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.282 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:48:09.282 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:48:09.284 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:48:09.284 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:48:09.286 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.287 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.288 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.288 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.289 DEBUG:boto:Method: GET 2022-09-03T06:48:09.290 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.291 DEBUG:boto:Data: 2022-09-03T06:48:09.292 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.294 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.295 DEBUG:boto:Params: {} 2022-09-03T06:48:09.296 DEBUG:boto:Token: None 2022-09-03T06:48:09.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/?versions 2022-09-03T06:48:09.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xesm9eAiq3pPE019Q4VTI/S2AVg= 2022-09-03T06:48:09.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xesm9eAiq3pPE019Q4VTI/S2AVg=', 'Content-Length': '0'} 2022-09-03T06:48:09.302 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003aa4e20a535575fc-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.303 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.305 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.306 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.307 DEBUG:boto:path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.307 DEBUG:boto:auth_path=/avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.308 DEBUG:boto:Method: GET 2022-09-03T06:48:09.308 DEBUG:boto:Path: /avysmw-69/?versions&max-keys=999 2022-09-03T06:48:09.309 DEBUG:boto:Data: 2022-09-03T06:48:09.310 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.310 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.311 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.311 DEBUG:boto:Params: {} 2022-09-03T06:48:09.312 DEBUG:boto:Token: None 2022-09-03T06:48:09.313 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/?versions 2022-09-03T06:48:09.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xesm9eAiq3pPE019Q4VTI/S2AVg= 2022-09-03T06:48:09.314 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xesm9eAiq3pPE019Q4VTI/S2AVg=', 'Content-Length': '0'} 2022-09-03T06:48:09.317 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c13729d3145c66ab-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.318 DEBUG:boto:b'avysmw-69999falseanulltrue2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.319 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:48:09.321 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.322 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.323 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.324 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.325 DEBUG:boto:Method: GET 2022-09-03T06:48:09.326 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:09.327 DEBUG:boto:Data: 2022-09-03T06:48:09.328 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.330 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.332 DEBUG:boto:Params: {} 2022-09-03T06:48:09.333 DEBUG:boto:Token: None 2022-09-03T06:48:09.334 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:09.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:huoz/YtTBtm73qB98XZybfesHwg= 2022-09-03T06:48:09.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:huoz/YtTBtm73qB98XZybfesHwg=', 'Content-Length': '0'} 2022-09-03T06:48:09.337 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b655743def321c23-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.338 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.338 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.339 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.339 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.340 DEBUG:boto:Method: GET 2022-09-03T06:48:09.341 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:09.341 DEBUG:boto:Data: 2022-09-03T06:48:09.342 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.342 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.343 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.343 DEBUG:boto:Params: {} 2022-09-03T06:48:09.344 DEBUG:boto:Token: None 2022-09-03T06:48:09.345 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:09.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:huoz/YtTBtm73qB98XZybfesHwg= 2022-09-03T06:48:09.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:huoz/YtTBtm73qB98XZybfesHwg=', 'Content-Length': '0'} 2022-09-03T06:48:09.348 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000aa1f12b713945a1-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.349 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.350 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.351 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.351 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.352 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.353 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:09.353 DEBUG:boto:Data: 2022-09-03T06:48:09.354 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.354 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.355 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.355 DEBUG:boto:Params: {} 2022-09-03T06:48:09.356 DEBUG:boto:Token: None 2022-09-03T06:48:09.357 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:09.357 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Aixx+tHsLeXRxqRV+jcLZTrLoSM= 2022-09-03T06:48:09.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Aixx+tHsLeXRxqRV+jcLZTrLoSM=', 'Content-Length': '0'} 2022-09-03T06:48:09.360 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f526a2052217320a-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.361 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.362 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.363 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.364 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.364 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.365 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:09.367 DEBUG:boto:Data: 2022-09-03T06:48:09.368 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.369 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.370 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.371 DEBUG:boto:Params: {} 2022-09-03T06:48:09.372 DEBUG:boto:Token: None 2022-09-03T06:48:09.373 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:09.374 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Aixx+tHsLeXRxqRV+jcLZTrLoSM= 2022-09-03T06:48:09.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Aixx+tHsLeXRxqRV+jcLZTrLoSM=', 'Content-Length': '0'} 2022-09-03T06:48:09.378 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d1fa68c3d6180a4-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.379 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:48:09.380 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.381 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.382 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.383 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.384 DEBUG:boto:Method: GET 2022-09-03T06:48:09.385 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:09.385 DEBUG:boto:Data: 2022-09-03T06:48:09.386 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.387 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.387 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.388 DEBUG:boto:Params: {} 2022-09-03T06:48:09.388 DEBUG:boto:Token: None 2022-09-03T06:48:09.389 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:09.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:huoz/YtTBtm73qB98XZybfesHwg= 2022-09-03T06:48:09.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:huoz/YtTBtm73qB98XZybfesHwg=', 'Content-Length': '0'} 2022-09-03T06:48:09.393 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005eb539ee1d09322f-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.394 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.394 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.395 DEBUG:boto:path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.395 DEBUG:boto:auth_path=/avysmw-69/a?versionId=null 2022-09-03T06:48:09.396 DEBUG:boto:Method: GET 2022-09-03T06:48:09.396 DEBUG:boto:Path: /avysmw-69/a?versionId=null 2022-09-03T06:48:09.397 DEBUG:boto:Data: 2022-09-03T06:48:09.398 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.398 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.399 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.399 DEBUG:boto:Params: {} 2022-09-03T06:48:09.400 DEBUG:boto:Token: None 2022-09-03T06:48:09.400 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a?versionId=null 2022-09-03T06:48:09.401 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:huoz/YtTBtm73qB98XZybfesHwg= 2022-09-03T06:48:09.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:huoz/YtTBtm73qB98XZybfesHwg=', 'Content-Length': '0'} 2022-09-03T06:48:09.407 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1a43eb3c592ad60-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.408 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.409 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.411 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:09.412 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:09.413 DEBUG:boto:Method: GET 2022-09-03T06:48:09.414 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:09.415 DEBUG:boto:Data: 2022-09-03T06:48:09.415 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.416 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.417 DEBUG:boto:Params: {} 2022-09-03T06:48:09.418 DEBUG:boto:Token: None 2022-09-03T06:48:09.418 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.422 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1ffdcee420760bf-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.422 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.423 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.424 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.424 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:09.425 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:09.425 DEBUG:boto:Method: GET 2022-09-03T06:48:09.426 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:09.426 DEBUG:boto:Data: 2022-09-03T06:48:09.427 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.427 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.428 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.429 DEBUG:boto:Params: {} 2022-09-03T06:48:09.429 DEBUG:boto:Token: None 2022-09-03T06:48:09.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.434 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a057bb4d3e5cad9-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.435 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.436 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.437 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.438 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.439 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:09.440 DEBUG:boto:Data: 2022-09-03T06:48:09.441 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.442 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.442 DEBUG:boto:Params: {} 2022-09-03T06:48:09.443 DEBUG:boto:Token: None 2022-09-03T06:48:09.443 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a 2022-09-03T06:48:09.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lZuT+7bAnZvsf1OG/f4pu2zk56Y= 2022-09-03T06:48:09.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lZuT+7bAnZvsf1OG/f4pu2zk56Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.446 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000fef05bbeb072e56-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.447 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.448 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.448 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.450 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:09.451 DEBUG:boto:Data: 2022-09-03T06:48:09.452 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.453 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.454 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.455 DEBUG:boto:Params: {} 2022-09-03T06:48:09.456 DEBUG:boto:Token: None 2022-09-03T06:48:09.457 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a 2022-09-03T06:48:09.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lZuT+7bAnZvsf1OG/f4pu2zk56Y= 2022-09-03T06:48:09.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lZuT+7bAnZvsf1OG/f4pu2zk56Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.461 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2a400db2fdba871-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.462 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:48:09.462 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.463 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.463 DEBUG:boto:Method: GET 2022-09-03T06:48:09.464 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:09.464 DEBUG:boto:Data: 2022-09-03T06:48:09.465 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.465 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.466 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.467 DEBUG:boto:Params: {} 2022-09-03T06:48:09.468 DEBUG:boto:Token: None 2022-09-03T06:48:09.468 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a 2022-09-03T06:48:09.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8n2O/Ltsz2pNX+kbDSEVuvQFRNg= 2022-09-03T06:48:09.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8n2O/Ltsz2pNX+kbDSEVuvQFRNg=', 'Content-Length': '0'} 2022-09-03T06:48:09.472 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000759793a28e5e9555-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.518 DEBUG:boto:path=/avysmw-69/a 2022-09-03T06:48:09.519 DEBUG:boto:auth_path=/avysmw-69/a 2022-09-03T06:48:09.520 DEBUG:boto:Method: GET 2022-09-03T06:48:09.521 DEBUG:boto:Path: /avysmw-69/a 2022-09-03T06:48:09.522 DEBUG:boto:Data: 2022-09-03T06:48:09.523 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.524 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.525 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.527 DEBUG:boto:Params: {} 2022-09-03T06:48:09.528 DEBUG:boto:Token: None 2022-09-03T06:48:09.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/a 2022-09-03T06:48:09.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8n2O/Ltsz2pNX+kbDSEVuvQFRNg= 2022-09-03T06:48:09.532 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8n2O/Ltsz2pNX+kbDSEVuvQFRNg=', 'Content-Length': '0'} 2022-09-03T06:48:09.535 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0a3560a444152f9-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.536 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.537 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.538 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:09.539 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:09.540 DEBUG:boto:Method: GET 2022-09-03T06:48:09.541 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:09.542 DEBUG:boto:Data: 2022-09-03T06:48:09.543 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.546 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.547 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.548 DEBUG:boto:Params: {} 2022-09-03T06:48:09.548 DEBUG:boto:Token: None 2022-09-03T06:48:09.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.552 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000095cd632d672158da-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.553 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.554 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.555 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.555 DEBUG:boto:path=/avysmw-69/?prefix=a 2022-09-03T06:48:09.556 DEBUG:boto:auth_path=/avysmw-69/?prefix=a 2022-09-03T06:48:09.557 DEBUG:boto:Method: GET 2022-09-03T06:48:09.557 DEBUG:boto:Path: /avysmw-69/?prefix=a 2022-09-03T06:48:09.558 DEBUG:boto:Data: 2022-09-03T06:48:09.558 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.559 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.559 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.560 DEBUG:boto:Params: {} 2022-09-03T06:48:09.560 DEBUG:boto:Token: None 2022-09-03T06:48:09.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.565 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000206072f8b6b709d6-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.566 DEBUG:boto:b'avysmw-69a1000falsea2022-09-03T06:47:10.189Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.567 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:48:09.568 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.569 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.569 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.570 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.570 DEBUG:boto:Method: GET 2022-09-03T06:48:09.571 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:09.571 DEBUG:boto:Data: 2022-09-03T06:48:09.572 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.574 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.575 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.576 DEBUG:boto:Params: {} 2022-09-03T06:48:09.577 DEBUG:boto:Token: None 2022-09-03T06:48:09.578 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:09.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YfOT+5cupUmdc/VbHjqIdJQU2XI= 2022-09-03T06:48:09.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YfOT+5cupUmdc/VbHjqIdJQU2XI=', 'Content-Length': '0'} 2022-09-03T06:48:09.583 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dbb2601348bd31fa-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.584 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.585 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.586 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.587 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.588 DEBUG:boto:Method: GET 2022-09-03T06:48:09.589 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:09.590 DEBUG:boto:Data: 2022-09-03T06:48:09.591 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.592 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.593 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.594 DEBUG:boto:Params: {} 2022-09-03T06:48:09.595 DEBUG:boto:Token: None 2022-09-03T06:48:09.596 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:09.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YfOT+5cupUmdc/VbHjqIdJQU2XI= 2022-09-03T06:48:09.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YfOT+5cupUmdc/VbHjqIdJQU2XI=', 'Content-Length': '0'} 2022-09-03T06:48:09.601 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e94dd8b8466cdd7-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.602 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.602 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.603 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.604 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.605 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.606 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:09.607 DEBUG:boto:Data: 2022-09-03T06:48:09.607 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.609 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.609 DEBUG:boto:Params: {} 2022-09-03T06:48:09.610 DEBUG:boto:Token: None 2022-09-03T06:48:09.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:09.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WNLLm5VlpgsUxQ8scjxhbsMQQx4= 2022-09-03T06:48:09.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WNLLm5VlpgsUxQ8scjxhbsMQQx4=', 'Content-Length': '0'} 2022-09-03T06:48:09.615 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001baa334897718c21-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.616 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.616 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.617 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.618 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.618 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.619 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:09.620 DEBUG:boto:Data: 2022-09-03T06:48:09.621 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.622 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.623 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.624 DEBUG:boto:Params: {} 2022-09-03T06:48:09.625 DEBUG:boto:Token: None 2022-09-03T06:48:09.626 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:09.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WNLLm5VlpgsUxQ8scjxhbsMQQx4= 2022-09-03T06:48:09.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WNLLm5VlpgsUxQ8scjxhbsMQQx4=', 'Content-Length': '0'} 2022-09-03T06:48:09.634 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f46613de61b066a4-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.635 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:48:09.636 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.637 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.637 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.638 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.639 DEBUG:boto:Method: GET 2022-09-03T06:48:09.640 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:09.641 DEBUG:boto:Data: 2022-09-03T06:48:09.642 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.645 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.646 DEBUG:boto:Params: {} 2022-09-03T06:48:09.647 DEBUG:boto:Token: None 2022-09-03T06:48:09.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:09.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YfOT+5cupUmdc/VbHjqIdJQU2XI= 2022-09-03T06:48:09.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YfOT+5cupUmdc/VbHjqIdJQU2XI=', 'Content-Length': '0'} 2022-09-03T06:48:09.653 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000267ed6cb8894e41b-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.698 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.699 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.700 DEBUG:boto:path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.701 DEBUG:boto:auth_path=/avysmw-69/b?versionId=null 2022-09-03T06:48:09.702 DEBUG:boto:Method: GET 2022-09-03T06:48:09.703 DEBUG:boto:Path: /avysmw-69/b?versionId=null 2022-09-03T06:48:09.703 DEBUG:boto:Data: 2022-09-03T06:48:09.704 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.704 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.705 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.706 DEBUG:boto:Params: {} 2022-09-03T06:48:09.706 DEBUG:boto:Token: None 2022-09-03T06:48:09.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b?versionId=null 2022-09-03T06:48:09.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YfOT+5cupUmdc/VbHjqIdJQU2XI= 2022-09-03T06:48:09.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YfOT+5cupUmdc/VbHjqIdJQU2XI=', 'Content-Length': '0'} 2022-09-03T06:48:09.712 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000cfbcab99985d516-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.714 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.714 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.715 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:09.716 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:09.716 DEBUG:boto:Method: GET 2022-09-03T06:48:09.717 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:09.717 DEBUG:boto:Data: 2022-09-03T06:48:09.718 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.719 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.720 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.721 DEBUG:boto:Params: {} 2022-09-03T06:48:09.722 DEBUG:boto:Token: None 2022-09-03T06:48:09.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.725 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.728 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000f97730181a15493-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.729 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.730 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.731 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.733 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:09.734 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:09.735 DEBUG:boto:Method: GET 2022-09-03T06:48:09.735 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:09.736 DEBUG:boto:Data: 2022-09-03T06:48:09.737 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.739 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.740 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.741 DEBUG:boto:Params: {} 2022-09-03T06:48:09.742 DEBUG:boto:Token: None 2022-09-03T06:48:09.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.751 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000047ffcf25b6f361c9-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.752 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.754 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.755 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.756 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.757 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:09.758 DEBUG:boto:Data: 2022-09-03T06:48:09.759 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.761 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.762 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.763 DEBUG:boto:Params: {} 2022-09-03T06:48:09.764 DEBUG:boto:Token: None 2022-09-03T06:48:09.771 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b 2022-09-03T06:48:09.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3FrADNBk7FfreMJZUDvXNc0QA3k= 2022-09-03T06:48:09.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3FrADNBk7FfreMJZUDvXNc0QA3k=', 'Content-Length': '0'} 2022-09-03T06:48:09.777 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000391637345eac9087-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.778 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.779 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.779 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.780 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:09.781 DEBUG:boto:Data: 2022-09-03T06:48:09.782 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.783 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.785 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.788 DEBUG:boto:Params: {} 2022-09-03T06:48:09.789 DEBUG:boto:Token: None 2022-09-03T06:48:09.790 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b 2022-09-03T06:48:09.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3FrADNBk7FfreMJZUDvXNc0QA3k= 2022-09-03T06:48:09.792 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3FrADNBk7FfreMJZUDvXNc0QA3k=', 'Content-Length': '0'} 2022-09-03T06:48:09.795 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df4ef666f4ed6451-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.796 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:48:09.798 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.799 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.801 DEBUG:boto:Method: GET 2022-09-03T06:48:09.802 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:09.802 DEBUG:boto:Data: 2022-09-03T06:48:09.803 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.805 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.805 DEBUG:boto:Params: {} 2022-09-03T06:48:09.806 DEBUG:boto:Token: None 2022-09-03T06:48:09.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b 2022-09-03T06:48:09.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oI3cZCgK6Yx21nQkuHspBgw/1m4= 2022-09-03T06:48:09.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oI3cZCgK6Yx21nQkuHspBgw/1m4=', 'Content-Length': '0'} 2022-09-03T06:48:09.812 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb02cb2b9cc95dcc-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.813 DEBUG:boto:path=/avysmw-69/b 2022-09-03T06:48:09.814 DEBUG:boto:auth_path=/avysmw-69/b 2022-09-03T06:48:09.815 DEBUG:boto:Method: GET 2022-09-03T06:48:09.816 DEBUG:boto:Path: /avysmw-69/b 2022-09-03T06:48:09.816 DEBUG:boto:Data: 2022-09-03T06:48:09.817 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.818 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.819 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.819 DEBUG:boto:Params: {} 2022-09-03T06:48:09.821 DEBUG:boto:Token: None 2022-09-03T06:48:09.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/b 2022-09-03T06:48:09.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oI3cZCgK6Yx21nQkuHspBgw/1m4= 2022-09-03T06:48:09.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oI3cZCgK6Yx21nQkuHspBgw/1m4=', 'Content-Length': '0'} 2022-09-03T06:48:09.829 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027058ecc9fae97ef-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.874 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.875 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.876 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:09.877 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:09.878 DEBUG:boto:Method: GET 2022-09-03T06:48:09.879 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:09.880 DEBUG:boto:Data: 2022-09-03T06:48:09.881 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.884 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.885 DEBUG:boto:Params: {} 2022-09-03T06:48:09.886 DEBUG:boto:Token: None 2022-09-03T06:48:09.887 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.888 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.893 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c7d76dd6817de0fa-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.894 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.896 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.897 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.898 DEBUG:boto:path=/avysmw-69/?prefix=b 2022-09-03T06:48:09.899 DEBUG:boto:auth_path=/avysmw-69/?prefix=b 2022-09-03T06:48:09.900 DEBUG:boto:Method: GET 2022-09-03T06:48:09.900 DEBUG:boto:Path: /avysmw-69/?prefix=b 2022-09-03T06:48:09.901 DEBUG:boto:Data: 2022-09-03T06:48:09.902 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.903 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.904 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.905 DEBUG:boto:Params: {} 2022-09-03T06:48:09.907 DEBUG:boto:Token: None 2022-09-03T06:48:09.908 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.912 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000015ae124457aaad30-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.913 DEBUG:boto:b'avysmw-69b1000falseb2022-09-03T06:47:10.221Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:09.914 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:48:09.914 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:09.915 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:09.916 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.916 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.917 DEBUG:boto:Method: GET 2022-09-03T06:48:09.917 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:09.918 DEBUG:boto:Data: 2022-09-03T06:48:09.918 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.919 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.919 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.920 DEBUG:boto:Params: {} 2022-09-03T06:48:09.921 DEBUG:boto:Token: None 2022-09-03T06:48:09.921 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:09.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2HFr4gCaALbQVk9h+Raxmp+nkw= 2022-09-03T06:48:09.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2HFr4gCaALbQVk9h+Raxmp+nkw=', 'Content-Length': '0'} 2022-09-03T06:48:09.925 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e2cb9323bb3d7b4-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.925 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:09.926 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:09.927 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.927 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.928 DEBUG:boto:Method: GET 2022-09-03T06:48:09.929 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:09.929 DEBUG:boto:Data: 2022-09-03T06:48:09.930 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.930 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.931 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.932 DEBUG:boto:Params: {} 2022-09-03T06:48:09.932 DEBUG:boto:Token: None 2022-09-03T06:48:09.933 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:09.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2HFr4gCaALbQVk9h+Raxmp+nkw= 2022-09-03T06:48:09.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2HFr4gCaALbQVk9h+Raxmp+nkw=', 'Content-Length': '0'} 2022-09-03T06:48:09.936 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004112973dd2c5799e-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.937 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:09.938 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:09.939 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.939 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.940 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.940 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:09.941 DEBUG:boto:Data: 2022-09-03T06:48:09.941 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.942 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.942 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.943 DEBUG:boto:Params: {} 2022-09-03T06:48:09.943 DEBUG:boto:Token: None 2022-09-03T06:48:09.944 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:09.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XpNKSQjBNhEHL4msyTdXSh3v4bo= 2022-09-03T06:48:09.945 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XpNKSQjBNhEHL4msyTdXSh3v4bo=', 'Content-Length': '0'} 2022-09-03T06:48:09.947 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f3c32adb555bfed-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.948 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:09.949 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:09.949 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.950 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.950 DEBUG:boto:Method: HEAD 2022-09-03T06:48:09.951 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:09.952 DEBUG:boto:Data: 2022-09-03T06:48:09.952 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.953 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.953 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.954 DEBUG:boto:Params: {} 2022-09-03T06:48:09.955 DEBUG:boto:Token: None 2022-09-03T06:48:09.956 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:09.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XpNKSQjBNhEHL4msyTdXSh3v4bo= 2022-09-03T06:48:09.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XpNKSQjBNhEHL4msyTdXSh3v4bo=', 'Content-Length': '0'} 2022-09-03T06:48:09.959 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026a132d3a79b968e-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.960 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:48:09.961 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:09.961 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:09.962 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.963 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.963 DEBUG:boto:Method: GET 2022-09-03T06:48:09.964 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:09.964 DEBUG:boto:Data: 2022-09-03T06:48:09.965 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.966 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.967 DEBUG:boto:Params: {} 2022-09-03T06:48:09.967 DEBUG:boto:Token: None 2022-09-03T06:48:09.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:09.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2HFr4gCaALbQVk9h+Raxmp+nkw= 2022-09-03T06:48:09.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2HFr4gCaALbQVk9h+Raxmp+nkw=', 'Content-Length': '0'} 2022-09-03T06:48:09.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ab3c4297558b001-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.972 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:09.973 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:09.974 DEBUG:boto:path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.974 DEBUG:boto:auth_path=/avysmw-69/c?versionId=null 2022-09-03T06:48:09.975 DEBUG:boto:Method: GET 2022-09-03T06:48:09.975 DEBUG:boto:Path: /avysmw-69/c?versionId=null 2022-09-03T06:48:09.976 DEBUG:boto:Data: 2022-09-03T06:48:09.976 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.977 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:09.977 DEBUG:boto:Port: 8000 2022-09-03T06:48:09.978 DEBUG:boto:Params: {} 2022-09-03T06:48:09.979 DEBUG:boto:Token: None 2022-09-03T06:48:09.979 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/c?versionId=null 2022-09-03T06:48:09.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E2HFr4gCaALbQVk9h+Raxmp+nkw= 2022-09-03T06:48:09.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E2HFr4gCaALbQVk9h+Raxmp+nkw=', 'Content-Length': '0'} 2022-09-03T06:48:09.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e7e7af1197924f91-006312f8a9-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:09.983 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:09.984 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:09.985 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:09.985 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:09.986 DEBUG:boto:Method: GET 2022-09-03T06:48:09.986 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:09.987 DEBUG:boto:Data: 2022-09-03T06:48:09.987 DEBUG:boto:Headers: {} 2022-09-03T06:48:09.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:09.988 DEBUG:boto:Port: 8001 2022-09-03T06:48:09.989 DEBUG:boto:Params: {} 2022-09-03T06:48:09.990 DEBUG:boto:Token: None 2022-09-03T06:48:09.990 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:09 GMT /avysmw-69/ 2022-09-03T06:48:09.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y= 2022-09-03T06:48:09.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LlmsSD2UwxQTnYzA7MnGKimwf3Y=', 'Content-Length': '0'} 2022-09-03T06:48:09.998 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085cfc4e8ffd47b26-006312f8a9-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.000 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:10.001 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:10.002 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:10.003 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:10.004 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:10.005 DEBUG:boto:Method: GET 2022-09-03T06:48:10.006 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:10.007 DEBUG:boto:Data: 2022-09-03T06:48:10.008 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.009 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.010 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.011 DEBUG:boto:Params: {} 2022-09-03T06:48:10.012 DEBUG:boto:Token: None 2022-09-03T06:48:10.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/ 2022-09-03T06:48:10.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w= 2022-09-03T06:48:10.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w=', 'Content-Length': '0'} 2022-09-03T06:48:10.018 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000452ebbf169aff438-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.019 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:10.020 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:10.021 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:10.021 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.022 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:10.023 DEBUG:boto:Data: 2022-09-03T06:48:10.023 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.024 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.025 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.025 DEBUG:boto:Params: {} 2022-09-03T06:48:10.026 DEBUG:boto:Token: None 2022-09-03T06:48:10.027 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/c 2022-09-03T06:48:10.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTfp97C4VbEIi8SnBg8RcuADs4I= 2022-09-03T06:48:10.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTfp97C4VbEIi8SnBg8RcuADs4I=', 'Content-Length': '0'} 2022-09-03T06:48:10.030 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fff6cde8c8b9990a-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.031 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:10.032 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:10.033 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.034 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:10.035 DEBUG:boto:Data: 2022-09-03T06:48:10.036 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.037 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.039 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.040 DEBUG:boto:Params: {} 2022-09-03T06:48:10.040 DEBUG:boto:Token: None 2022-09-03T06:48:10.041 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/c 2022-09-03T06:48:10.042 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nTfp97C4VbEIi8SnBg8RcuADs4I= 2022-09-03T06:48:10.043 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nTfp97C4VbEIi8SnBg8RcuADs4I=', 'Content-Length': '0'} 2022-09-03T06:48:10.045 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036f6de363712daee-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.047 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:48:10.048 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:10.049 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:10.050 DEBUG:boto:Method: GET 2022-09-03T06:48:10.051 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:10.051 DEBUG:boto:Data: 2022-09-03T06:48:10.052 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.053 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.054 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.055 DEBUG:boto:Params: {} 2022-09-03T06:48:10.056 DEBUG:boto:Token: None 2022-09-03T06:48:10.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/c 2022-09-03T06:48:10.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NozWkg7rdYBznf9vU1F57hE26rM= 2022-09-03T06:48:10.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NozWkg7rdYBznf9vU1F57hE26rM=', 'Content-Length': '0'} 2022-09-03T06:48:10.060 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000059ed34e8e0a5e930-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.061 DEBUG:boto:path=/avysmw-69/c 2022-09-03T06:48:10.062 DEBUG:boto:auth_path=/avysmw-69/c 2022-09-03T06:48:10.063 DEBUG:boto:Method: GET 2022-09-03T06:48:10.063 DEBUG:boto:Path: /avysmw-69/c 2022-09-03T06:48:10.064 DEBUG:boto:Data: 2022-09-03T06:48:10.064 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.065 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.066 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.067 DEBUG:boto:Params: {} 2022-09-03T06:48:10.067 DEBUG:boto:Token: None 2022-09-03T06:48:10.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/c 2022-09-03T06:48:10.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NozWkg7rdYBznf9vU1F57hE26rM= 2022-09-03T06:48:10.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NozWkg7rdYBznf9vU1F57hE26rM=', 'Content-Length': '0'} 2022-09-03T06:48:10.074 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000357050b1ebc32390-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.114 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:10.115 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:10.116 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:10.116 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:10.117 DEBUG:boto:Method: GET 2022-09-03T06:48:10.117 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:10.118 DEBUG:boto:Data: 2022-09-03T06:48:10.118 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.119 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.119 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.120 DEBUG:boto:Params: {} 2022-09-03T06:48:10.121 DEBUG:boto:Token: None 2022-09-03T06:48:10.121 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/ 2022-09-03T06:48:10.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w= 2022-09-03T06:48:10.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w=', 'Content-Length': '0'} 2022-09-03T06:48:10.126 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d87acd92eab51408-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.127 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:10.128 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:10.128 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:10.129 DEBUG:boto:path=/avysmw-69/?prefix=c 2022-09-03T06:48:10.129 DEBUG:boto:auth_path=/avysmw-69/?prefix=c 2022-09-03T06:48:10.130 DEBUG:boto:Method: GET 2022-09-03T06:48:10.131 DEBUG:boto:Path: /avysmw-69/?prefix=c 2022-09-03T06:48:10.131 DEBUG:boto:Data: 2022-09-03T06:48:10.132 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.132 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.133 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.134 DEBUG:boto:Params: {} 2022-09-03T06:48:10.135 DEBUG:boto:Token: None 2022-09-03T06:48:10.136 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/ 2022-09-03T06:48:10.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w= 2022-09-03T06:48:10.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w=', 'Content-Length': '0'} 2022-09-03T06:48:10.140 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000072421cb75bcff0fc-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.141 DEBUG:boto:b'avysmw-69c1000falsec2022-09-03T06:47:10.293Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:10.143 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:48:10.144 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.145 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.146 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.147 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.148 DEBUG:boto:Method: GET 2022-09-03T06:48:10.149 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:10.151 DEBUG:boto:Data: 2022-09-03T06:48:10.151 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.153 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.153 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.154 DEBUG:boto:Params: {} 2022-09-03T06:48:10.155 DEBUG:boto:Token: None 2022-09-03T06:48:10.156 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:10.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eqERBFglMJYlSumfjyl4AXBal+M= 2022-09-03T06:48:10.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eqERBFglMJYlSumfjyl4AXBal+M=', 'Content-Length': '0'} 2022-09-03T06:48:10.160 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a7a39093285bca6-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.161 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.162 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.163 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.163 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.164 DEBUG:boto:Method: GET 2022-09-03T06:48:10.164 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:10.165 DEBUG:boto:Data: 2022-09-03T06:48:10.165 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.167 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.167 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.168 DEBUG:boto:Params: {} 2022-09-03T06:48:10.168 DEBUG:boto:Token: None 2022-09-03T06:48:10.169 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:10.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eqERBFglMJYlSumfjyl4AXBal+M= 2022-09-03T06:48:10.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eqERBFglMJYlSumfjyl4AXBal+M=', 'Content-Length': '0'} 2022-09-03T06:48:10.173 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf7d72dd8c0d9fc8-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.174 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.175 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.176 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.176 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.177 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.178 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:10.179 DEBUG:boto:Data: 2022-09-03T06:48:10.180 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.180 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.182 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.183 DEBUG:boto:Params: {} 2022-09-03T06:48:10.184 DEBUG:boto:Token: None 2022-09-03T06:48:10.185 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:10.186 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mI9ZWAfuCvx13lnVTfVTDpqXv6E= 2022-09-03T06:48:10.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mI9ZWAfuCvx13lnVTfVTDpqXv6E=', 'Content-Length': '0'} 2022-09-03T06:48:10.189 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2ea2f40ede25e70-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.189 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.190 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.191 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.191 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.192 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.192 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:10.193 DEBUG:boto:Data: 2022-09-03T06:48:10.193 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.194 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.194 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.195 DEBUG:boto:Params: {} 2022-09-03T06:48:10.196 DEBUG:boto:Token: None 2022-09-03T06:48:10.196 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:10.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mI9ZWAfuCvx13lnVTfVTDpqXv6E= 2022-09-03T06:48:10.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mI9ZWAfuCvx13lnVTfVTDpqXv6E=', 'Content-Length': '0'} 2022-09-03T06:48:10.201 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004576e06ae934ac21-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.202 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:48:10.202 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.203 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.203 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.204 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.204 DEBUG:boto:Method: GET 2022-09-03T06:48:10.205 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:10.205 DEBUG:boto:Data: 2022-09-03T06:48:10.206 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.206 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.207 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.207 DEBUG:boto:Params: {} 2022-09-03T06:48:10.208 DEBUG:boto:Token: None 2022-09-03T06:48:10.209 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:10.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eqERBFglMJYlSumfjyl4AXBal+M= 2022-09-03T06:48:10.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eqERBFglMJYlSumfjyl4AXBal+M=', 'Content-Length': '0'} 2022-09-03T06:48:10.213 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070dd10633f4b6346-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.213 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.214 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.215 DEBUG:boto:path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.215 DEBUG:boto:auth_path=/avysmw-69/d?versionId=null 2022-09-03T06:48:10.216 DEBUG:boto:Method: GET 2022-09-03T06:48:10.216 DEBUG:boto:Path: /avysmw-69/d?versionId=null 2022-09-03T06:48:10.217 DEBUG:boto:Data: 2022-09-03T06:48:10.217 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.218 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.218 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.219 DEBUG:boto:Params: {} 2022-09-03T06:48:10.219 DEBUG:boto:Token: None 2022-09-03T06:48:10.220 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d?versionId=null 2022-09-03T06:48:10.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eqERBFglMJYlSumfjyl4AXBal+M= 2022-09-03T06:48:10.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eqERBFglMJYlSumfjyl4AXBal+M=', 'Content-Length': '0'} 2022-09-03T06:48:10.223 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039143c195a9b29b7-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.224 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:10.225 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:10.225 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:10.226 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:10.226 DEBUG:boto:Method: GET 2022-09-03T06:48:10.227 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:10.227 DEBUG:boto:Data: 2022-09-03T06:48:10.228 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.228 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.229 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.229 DEBUG:boto:Params: {} 2022-09-03T06:48:10.230 DEBUG:boto:Token: None 2022-09-03T06:48:10.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/ 2022-09-03T06:48:10.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w= 2022-09-03T06:48:10.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w=', 'Content-Length': '0'} 2022-09-03T06:48:10.234 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c295e9a975ea9f94-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.235 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:10.236 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:10.236 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:10.237 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:10.237 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:10.238 DEBUG:boto:Method: GET 2022-09-03T06:48:10.238 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:10.239 DEBUG:boto:Data: 2022-09-03T06:48:10.239 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.240 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.240 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.241 DEBUG:boto:Params: {} 2022-09-03T06:48:10.242 DEBUG:boto:Token: None 2022-09-03T06:48:10.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/ 2022-09-03T06:48:10.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w= 2022-09-03T06:48:10.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w=', 'Content-Length': '0'} 2022-09-03T06:48:10.246 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e233561927e96862-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.246 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:10.247 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.248 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.248 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.249 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:10.249 DEBUG:boto:Data: 2022-09-03T06:48:10.250 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.251 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.251 DEBUG:boto:Params: {} 2022-09-03T06:48:10.252 DEBUG:boto:Token: None 2022-09-03T06:48:10.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d 2022-09-03T06:48:10.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2iyx3rbgBRNLllcfgGDRWltu5gY= 2022-09-03T06:48:10.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2iyx3rbgBRNLllcfgGDRWltu5gY=', 'Content-Length': '0'} 2022-09-03T06:48:10.257 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000014736982f85adf7e-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.258 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.258 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.259 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.259 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:10.260 DEBUG:boto:Data: 2022-09-03T06:48:10.261 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.261 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.262 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.263 DEBUG:boto:Params: {} 2022-09-03T06:48:10.264 DEBUG:boto:Token: None 2022-09-03T06:48:10.265 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d 2022-09-03T06:48:10.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2iyx3rbgBRNLllcfgGDRWltu5gY= 2022-09-03T06:48:10.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2iyx3rbgBRNLllcfgGDRWltu5gY=', 'Content-Length': '0'} 2022-09-03T06:48:10.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007a2af13849db6963-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.271 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:48:10.272 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.273 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.274 DEBUG:boto:Method: GET 2022-09-03T06:48:10.275 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:10.275 DEBUG:boto:Data: 2022-09-03T06:48:10.276 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.276 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.277 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.277 DEBUG:boto:Params: {} 2022-09-03T06:48:10.278 DEBUG:boto:Token: None 2022-09-03T06:48:10.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d 2022-09-03T06:48:10.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o6KPTRj011NMj2hSDOTX7vF7M20= 2022-09-03T06:48:10.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o6KPTRj011NMj2hSDOTX7vF7M20=', 'Content-Length': '0'} 2022-09-03T06:48:10.282 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9cd975f69e66be1-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.330 DEBUG:boto:path=/avysmw-69/d 2022-09-03T06:48:10.331 DEBUG:boto:auth_path=/avysmw-69/d 2022-09-03T06:48:10.333 DEBUG:boto:Method: GET 2022-09-03T06:48:10.334 DEBUG:boto:Path: /avysmw-69/d 2022-09-03T06:48:10.335 DEBUG:boto:Data: 2022-09-03T06:48:10.336 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.337 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.338 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.339 DEBUG:boto:Params: {} 2022-09-03T06:48:10.340 DEBUG:boto:Token: None 2022-09-03T06:48:10.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/d 2022-09-03T06:48:10.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o6KPTRj011NMj2hSDOTX7vF7M20= 2022-09-03T06:48:10.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o6KPTRj011NMj2hSDOTX7vF7M20=', 'Content-Length': '0'} 2022-09-03T06:48:10.346 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:47:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6030f98ade19bb5-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.348 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:10.349 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:10.350 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:10.351 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:10.352 DEBUG:boto:Method: GET 2022-09-03T06:48:10.353 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:10.354 DEBUG:boto:Data: 2022-09-03T06:48:10.355 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:48:10.357 DEBUG:boto:Port: 8001 2022-09-03T06:48:10.358 DEBUG:boto:Params: {} 2022-09-03T06:48:10.359 DEBUG:boto:Token: None 2022-09-03T06:48:10.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/ 2022-09-03T06:48:10.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w= 2022-09-03T06:48:10.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w=', 'Content-Length': '0'} 2022-09-03T06:48:10.365 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002473f756e1176b30-006312f8aa-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.366 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:10.367 DEBUG:boto:path=/avysmw-69/ 2022-09-03T06:48:10.368 DEBUG:boto:auth_path=/avysmw-69/ 2022-09-03T06:48:10.368 DEBUG:boto:path=/avysmw-69/?prefix=d 2022-09-03T06:48:10.369 DEBUG:boto:auth_path=/avysmw-69/?prefix=d 2022-09-03T06:48:10.369 DEBUG:boto:Method: GET 2022-09-03T06:48:10.370 DEBUG:boto:Path: /avysmw-69/?prefix=d 2022-09-03T06:48:10.370 DEBUG:boto:Data: 2022-09-03T06:48:10.371 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.371 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:48:10.372 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.372 DEBUG:boto:Params: {} 2022-09-03T06:48:10.373 DEBUG:boto:Token: None 2022-09-03T06:48:10.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:48:10 GMT /avysmw-69/ 2022-09-03T06:48:10.374 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w= 2022-09-03T06:48:10.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2y2p6vmFiSOKs9f681p3wIaHC1w=', 'Content-Length': '0'} 2022-09-03T06:48:10.377 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a9ba7025774eb2a-006312f8aa-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.418 DEBUG:boto:b'avysmw-69d1000falsed2022-09-03T06:47:10.327Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:48:10.419 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-69 zones={test-zone3, test-zone2} 2022-09-03T06:48:10.420 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '0', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:48:10.420 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '0', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:48:10.421 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 0 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:48:10.470 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:10.471 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:10.531 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-70 2022-09-03T06:48:10.532 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:48:10.533 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:48:10.535 DEBUG:boto:Method: PUT 2022-09-03T06:48:10.535 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:48:10.536 DEBUG:boto:Data: 2022-09-03T06:48:10.537 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:10.539 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.540 DEBUG:boto:Params: {} 2022-09-03T06:48:10.541 DEBUG:boto:Token: None 2022-09-03T06:48:10.541 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:48:10 GMT /avysmw-70/ 2022-09-03T06:48:10.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MBsgWSYztms+ZmELBP+ntDSAChk= 2022-09-03T06:48:10.542 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MBsgWSYztms+ZmELBP+ntDSAChk=', 'Content-Length': '0'} 2022-09-03T06:48:10.565 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000b06e1e6a59680c72-006312f8aa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.567 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:48:10.568 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:48:10.569 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.570 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:48:10.571 DEBUG:boto:Data: 2022-09-03T06:48:10.572 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.573 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:10.574 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.575 DEBUG:boto:Params: {} 2022-09-03T06:48:10.576 DEBUG:boto:Token: None 2022-09-03T06:48:10.577 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-70/ 2022-09-03T06:48:10.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sj5IKvV4NXlijpXxwc9HpCKEN1s= 2022-09-03T06:48:10.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sj5IKvV4NXlijpXxwc9HpCKEN1s=', 'Content-Length': '0'} 2022-09-03T06:48:10.582 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000048e6246457270246-006312f8aa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.583 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:48:10.584 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:48:10.584 DEBUG:boto:Method: PUT 2022-09-03T06:48:10.585 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:48:10.585 DEBUG:boto:Data: 2022-09-03T06:48:10.586 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:48:10.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:10.587 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.588 DEBUG:boto:Params: {} 2022-09-03T06:48:10.588 DEBUG:boto:Token: None 2022-09-03T06:48:10.589 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:48:10 GMT /avysmw-70/a 2022-09-03T06:48:10.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+fZNSXCyIFxli/+7eYCkuDpptLU= 2022-09-03T06:48:10.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+fZNSXCyIFxli/+7eYCkuDpptLU='} 2022-09-03T06:48:10.635 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000009155e1c6f55e7924-006312f8aa-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.636 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:48:10.636 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:48:10.637 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.638 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:48:10.639 DEBUG:boto:Data: 2022-09-03T06:48:10.639 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:10.640 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.641 DEBUG:boto:Params: {} 2022-09-03T06:48:10.641 DEBUG:boto:Token: None 2022-09-03T06:48:10.642 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-70/ 2022-09-03T06:48:10.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sj5IKvV4NXlijpXxwc9HpCKEN1s= 2022-09-03T06:48:10.643 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sj5IKvV4NXlijpXxwc9HpCKEN1s=', 'Content-Length': '0'} 2022-09-03T06:48:10.646 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b531c295d6f7c868-006312f8aa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.648 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:48:10.649 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:48:10.650 DEBUG:boto:Method: PUT 2022-09-03T06:48:10.651 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:48:10.651 DEBUG:boto:Data: 2022-09-03T06:48:10.652 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:48:10.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:10.653 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.654 DEBUG:boto:Params: {} 2022-09-03T06:48:10.655 DEBUG:boto:Token: None 2022-09-03T06:48:10.656 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:48:10 GMT /avysmw-70/b 2022-09-03T06:48:10.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qg18CqReVjU+cDDUCpcvTNAYzeA= 2022-09-03T06:48:10.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qg18CqReVjU+cDDUCpcvTNAYzeA='} 2022-09-03T06:48:10.668 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000040337024e143257-006312f8aa-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.670 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:48:10.670 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:48:10.671 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.672 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:48:10.672 DEBUG:boto:Data: 2022-09-03T06:48:10.673 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.673 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:10.674 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.675 DEBUG:boto:Params: {} 2022-09-03T06:48:10.675 DEBUG:boto:Token: None 2022-09-03T06:48:10.676 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-70/ 2022-09-03T06:48:10.677 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sj5IKvV4NXlijpXxwc9HpCKEN1s= 2022-09-03T06:48:10.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sj5IKvV4NXlijpXxwc9HpCKEN1s=', 'Content-Length': '0'} 2022-09-03T06:48:10.680 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006ff699638dc7fc59-006312f8aa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.681 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:48:10.681 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:48:10.682 DEBUG:boto:Method: PUT 2022-09-03T06:48:10.683 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:48:10.683 DEBUG:boto:Data: 2022-09-03T06:48:10.684 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:48:10.684 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:10.685 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.685 DEBUG:boto:Params: {} 2022-09-03T06:48:10.686 DEBUG:boto:Token: None 2022-09-03T06:48:10.687 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:48:10 GMT /avysmw-70/c 2022-09-03T06:48:10.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KVtHnl+NAsgiXE8pfeFEHaQZJQ0= 2022-09-03T06:48:10.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KVtHnl+NAsgiXE8pfeFEHaQZJQ0='} 2022-09-03T06:48:10.734 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000005ecf9f46268685ba-006312f8aa-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.735 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:48:10.736 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:48:10.737 DEBUG:boto:Method: HEAD 2022-09-03T06:48:10.737 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:48:10.738 DEBUG:boto:Data: 2022-09-03T06:48:10.738 DEBUG:boto:Headers: {} 2022-09-03T06:48:10.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:10.739 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.740 DEBUG:boto:Params: {} 2022-09-03T06:48:10.740 DEBUG:boto:Token: None 2022-09-03T06:48:10.741 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:48:10 GMT /avysmw-70/ 2022-09-03T06:48:10.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sj5IKvV4NXlijpXxwc9HpCKEN1s= 2022-09-03T06:48:10.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sj5IKvV4NXlijpXxwc9HpCKEN1s=', 'Content-Length': '0'} 2022-09-03T06:48:10.745 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '9'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001dfd2a5437d7da56-006312f8aa-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.746 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:48:10.746 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:48:10.747 DEBUG:boto:Method: PUT 2022-09-03T06:48:10.748 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:48:10.748 DEBUG:boto:Data: 2022-09-03T06:48:10.749 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:48:10.749 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:48:10.750 DEBUG:boto:Port: 8000 2022-09-03T06:48:10.750 DEBUG:boto:Params: {} 2022-09-03T06:48:10.751 DEBUG:boto:Token: None 2022-09-03T06:48:10.752 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:48:10 GMT /avysmw-70/d 2022-09-03T06:48:10.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ITXp3DlySybBUuWW5Wv+JTeAegU= 2022-09-03T06:48:10.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:48:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ITXp3DlySybBUuWW5Wv+JTeAegU='} 2022-09-03T06:48:10.763 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000064c22cd0fd119702-006312f8aa-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:48:10.764 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:48:10.765 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:48:10.768 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:48:10.820 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:10.821 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:10.872 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:48:10.889 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:48:10.890 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:48:10.891 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:48:10.891 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:48:10.892 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:48:10.893 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:48:10.894 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187690.561538_391.1", 2022-09-03T06:48:10.895 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:48:10.561538Z" 2022-09-03T06:48:10.895 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:48:10.896 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:48:10.897 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:48:10.898 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:48:10.898 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:48:10.898 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:48:10.899 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:48:10.899 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:48:10.899 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:48:10.900 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:48:10.901 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662187690.561538_391.1', 2: '', 3: ''} 2022-09-03T06:48:10.902 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:48:10.903 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:48:10.903 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:48:10.904 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:48:10.957 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:10.957 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:11.008 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:11.001+0000 7f2b43462600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:48:11.008 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:11.002+0000 7f2b43462600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:48:11.024 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:48:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:48:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:48:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:48:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:48:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:48:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:48:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:48:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:48:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:48:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:48:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:48:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:48:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:48:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:48:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:48:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:48:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:48:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:48:11.045 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:48:11.045 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:48:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:48:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187690.561538_391.1", 2022-09-03T06:48:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:48:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:48:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:48:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:48:10.561538Z", 2022-09-03T06:48:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:48:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:48:11.051 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:48:11.051 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:48:11.051 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:48:11.052 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:48:11.052 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:48:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:48:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:48:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:48:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:48:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:48:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:48:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:48:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:48:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:48:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:48:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:48:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:48:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:48:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:48:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:48:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:48:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:48:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:48:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:48:11Z" 2022-09-03T06:48:11.064 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:48:11.064 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187690.561538_391.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:48:10.561538Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:48:11Z" } 2022-09-03T06:48:11.065 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187690.561538_391.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:48:10.561538Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:48:11.066 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187690.561538_391.1', 2: '', 3: ''} 2022-09-03T06:48:11.066 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187690.561538_391.1', 2: '', 3: ''} 2022-09-03T06:48:11.067 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:48:11.067 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:48:11.068 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:48:11.069 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:48:11.069 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:48:11.123 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:11.124 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:11.158 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:11.156+0000 7f5ab47f2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:48:11.159 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:11.157+0000 7f5ab47f2600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:48:11.174 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:48:11.174 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:48:11.175 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:48:11.176 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:48:11.176 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:48:11.176 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:48:11.177 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:48:11.177 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:48:11.177 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:48:11.177 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:48:11.178 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:48:11.178 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:48:11.178 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:48:11.178 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:48:11.179 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:48:11.179 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:48:11.179 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:48:11.179 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:48:11.180 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:48:11.180 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:48:11.180 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:48:11.180 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:48:11.181 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:48:11.181 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:48:11.181 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:48:11.181 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187690.561538_391.1", 2022-09-03T06:48:11.182 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:48:11.182 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:48:11.182 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:48:11.183 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:48:10.561538Z", 2022-09-03T06:48:11.183 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:48:11.183 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:48:11.184 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:48:11.184 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:48:11.184 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:48:11.184 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:48:11.185 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:48:11.185 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:48:11.185 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:48:11.186 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:48:11.186 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:48:11.186 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:48:11.186 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:48:11.187 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:48:11.188 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:48:11.188 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:48:11.189 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:48:11.189 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:48:11.189 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:48:11.190 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:48:11.190 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:48:11.191 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:48:11.191 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:48:11.191 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:48:11.192 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:48:11.192 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:48:11.192 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:48:11.193 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:48:11.193 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:48:11.193 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:48:11.194 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:48:11.194 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:48:11.195 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:48:11.195 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:48:11Z" 2022-09-03T06:48:11.195 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:48:11.197 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187690.561538_391.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:48:10.561538Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:48:11Z" } 2022-09-03T06:48:11.198 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187690.561538_391.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:48:10.561538Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:48:11.199 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187690.561538_391.1', 2: '', 3: ''} 2022-09-03T06:48:11.200 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187690.561538_391.1', 2: '', 3: ''} 2022-09-03T06:48:11.201 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:48:11.202 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:48:11.203 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:48:11.203 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:48:11.204 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:48:11.258 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:11.259 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:11.309 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:11.306+0000 7f32d0b21600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:48:11.316 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:48:11.317 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:48:11.317 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:48:11.317 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:48:11.318 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:48:11.318 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:48:11.318 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:48:11.319 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.319 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:48:11.320 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:48:11.320 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:48:11.321 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:48:11.321 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:48:11.321 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:48:11.321 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:48:11.322 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:48:11.322 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:48:11.322 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:48:11.323 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:48:11.323 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.323 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.323 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:48:11.324 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:48:11.324 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:48:11.325 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:48:11.325 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187690.561538_391.1", 2022-09-03T06:48:11.325 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:48:11.326 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:48:11.326 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:48:11.326 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:48:10.561538Z", 2022-09-03T06:48:11.327 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:48:11.327 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.328 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.328 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:48:11.329 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:48:11.329 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:48:11.330 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:48:11.330 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:48:11.331 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:48:11.331 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:48:11.331 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:48:11.332 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:48:11.332 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:48:11.333 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.333 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.335 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:48:11.335 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:48:11.336 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:48:11.336 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:48:11.336 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:48:11.336 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:48:11.337 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:48:11.337 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:48:11.337 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:48:11.337 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:48:11.338 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.338 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:48:11.338 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:48:11.339 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.339 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:48:11.339 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:48:11.339 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:48:11.340 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:48:11.340 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:48:11Z" 2022-09-03T06:48:11.340 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:48:11.342 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:11.307+0000 7f32d0b21600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:48:11.342 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187690.561538_391.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:48:10.561538Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:48:11Z" } 2022-09-03T06:48:11.343 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187690.561538_391.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:48:10.561538Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:48:11.343 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187690.561538_391.1', 2: '', 3: ''} 2022-09-03T06:48:11.344 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187690.561538_391.1', 2: '', 3: ''} 2022-09-03T06:48:11.344 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:48:11.345 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-70 2022-09-03T06:48:11.345 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:11.346 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:11.347 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-70 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:48:11.407 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:11.407 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:11.474 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:11.472+0000 7f2dab523600 1 waiting to reach incremental sync.. 2022-09-03T06:48:16.475 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:16.472+0000 7f2dab523600 1 waiting to reach incremental sync.. 2022-09-03T06:48:21.474 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:21.472+0000 7f2dab523600 1 waiting to reach incremental sync.. 2022-09-03T06:48:26.476 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:26.474+0000 7f2dab523600 1 bucket sync caught up with source: 2022-09-03T06:48:26.477 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.52.5, , , , 00000000001.56.5, , , 00000000001.54.5, , 00000000001.41.5] 2022-09-03T06:48:26.477 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.52.5, , , , 00000000001.56.5, , , 00000000001.54.5, , 00000000001.41.5] 2022-09-03T06:48:26.478 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:48:26.474+0000 7f2dab523600 0 bucket checkpoint complete 2022-09-03T06:48:26.495 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-70 2022-09-03T06:48:26.496 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:26.497 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:48:26.497 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-70 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:48:26.563 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:48:26.567 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:48:26.603 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:26.602+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:48:31.604 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:31.603+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:48:36.605 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:36.603+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:48:41.606 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:41.603+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:48:46.609 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:46.604+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:48:51.608 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:51.605+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:48:56.608 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:48:56.606+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:01.608 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:01.607+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:06.609 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:06.607+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:11.609 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:11.608+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:16.610 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:16.609+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:21.611 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:21.609+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:26.612 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:26.610+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:31.612 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:31.611+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:36.613 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:36.611+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:41.613 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:41.611+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:46.614 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:46.611+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:51.614 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:51.612+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:49:56.615 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:49:56.613+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:01.616 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:01.613+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:06.616 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:06.614+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:11.617 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:11.615+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:16.618 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:16.616+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:21.618 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:21.616+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:26.623 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:26.617+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:31.619 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:31.617+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:36.620 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:36.617+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:41.620 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:41.618+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:46.621 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:46.618+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:51.622 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:51.619+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:50:56.627 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:50:56.620+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:51:01.623 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:01.621+0000 7f5128b99600 1 waiting to reach incremental sync.. 2022-09-03T06:51:06.625 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:06.622+0000 7f5128b99600 1 bucket sync caught up with source: 2022-09-03T06:51:06.626 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.52.5, , , , 00000000001.56.5, , , 00000000001.54.5, , 00000000001.41.5] 2022-09-03T06:51:06.627 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.52.5, , , , 00000000001.56.5, , , 00000000001.54.5, , 00000000001.41.5] 2022-09-03T06:51:06.627 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:06.622+0000 7f5128b99600 0 bucket checkpoint complete 2022-09-03T06:51:06.647 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-70 2022-09-03T06:51:06.649 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:06.650 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:06.651 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-70 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:51:06.771 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:06.772 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:06.844 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:06.842+0000 7fe46dcd6600 1 bucket sync caught up with source: 2022-09-03T06:51:06.844 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.52.5, , , , 00000000001.56.5, , , 00000000001.54.5, , 00000000001.41.5] 2022-09-03T06:51:06.845 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.52.5, , , , 00000000001.56.5, , , 00000000001.54.5, , 00000000001.41.5] 2022-09-03T06:51:06.845 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:06.842+0000 7fe46dcd6600 0 bucket checkpoint complete 2022-09-03T06:51:06.870 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-70 2022-09-03T06:51:06.871 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:06.872 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:06.873 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-70 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:51:06.941 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:06.942 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:06.995 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:06.994+0000 7fe0bf98c600 1 bucket sync caught up with source: 2022-09-03T06:51:06.996 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.349.5, , , , 00000000001.246.5, , , 00000000001.247.5, , 00000000001.373.5] 2022-09-03T06:51:06.996 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.349.5, , , , 00000000001.246.5, , , 00000000001.247.5, , 00000000001.373.5] 2022-09-03T06:51:06.996 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:06.994+0000 7fe0bf98c600 0 bucket checkpoint complete 2022-09-03T06:51:07.014 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-70 2022-09-03T06:51:07.015 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:07.017 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:07.018 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-70 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:51:07.079 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:07.079 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:07.127 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:07.126+0000 7f7d3105e600 1 bucket sync caught up with source: 2022-09-03T06:51:07.128 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.349.5, , , , 00000000001.246.5, , , 00000000001.247.5, , 00000000001.373.5] 2022-09-03T06:51:07.129 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.349.5, , , , 00000000001.246.5, , , 00000000001.247.5, , 00000000001.373.5] 2022-09-03T06:51:07.129 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:07.126+0000 7f7d3105e600 0 bucket checkpoint complete 2022-09-03T06:51:07.141 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-70 2022-09-03T06:51:07.141 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-70 2022-09-03T06:51:07.142 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:07.143 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:07.143 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-70 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:51:07.202 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:07.202 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:07.241 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:07.239+0000 7f191c8ec600 1 bucket sync caught up with source: 2022-09-03T06:51:07.242 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.149.5, , , , 00000000001.131.5, , , 00000000001.129.5, , 00000000001.207.5] 2022-09-03T06:51:07.242 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.149.5, , , , 00000000001.131.5, , , 00000000001.129.5, , 00000000001.207.5] 2022-09-03T06:51:07.242 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:07.239+0000 7f191c8ec600 0 bucket checkpoint complete 2022-09-03T06:51:07.255 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-70 2022-09-03T06:51:07.256 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:07.256 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-70', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:07.257 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-70 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:51:07.319 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:07.319 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:07.391 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:07.389+0000 7f6114a3f600 1 bucket sync caught up with source: 2022-09-03T06:51:07.391 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.149.5, , , , 00000000001.131.5, , , 00000000001.129.5, , 00000000001.207.5] 2022-09-03T06:51:07.392 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.149.5, , , , 00000000001.131.5, , , 00000000001.129.5, , 00000000001.207.5] 2022-09-03T06:51:07.392 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:07.389+0000 7f6114a3f600 0 bucket checkpoint complete 2022-09-03T06:51:07.405 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-70 2022-09-03T06:51:07.406 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-70 zones={test-zone2, test-zone1} 2022-09-03T06:51:07.407 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.407 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.408 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.409 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:51:07.409 DEBUG:boto:Data: 2022-09-03T06:51:07.410 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.411 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.411 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.412 DEBUG:boto:Params: {} 2022-09-03T06:51:07.413 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:51:07.414 DEBUG:boto:Token: None 2022-09-03T06:51:07.416 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bnE3fDi7QhXMSY1zYf4aQjnGsfc= 2022-09-03T06:51:07.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bnE3fDi7QhXMSY1zYf4aQjnGsfc=', 'Content-Length': '0'} 2022-09-03T06:51:07.424 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002c950cd3a8a6a0a9-006312f95b-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.425 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.425 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.426 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.427 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:51:07.428 DEBUG:boto:Data: 2022-09-03T06:51:07.428 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.430 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.431 DEBUG:boto:Params: {} 2022-09-03T06:51:07.431 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:51:07.432 DEBUG:boto:Token: None 2022-09-03T06:51:07.433 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bnE3fDi7QhXMSY1zYf4aQjnGsfc= 2022-09-03T06:51:07.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bnE3fDi7QhXMSY1zYf4aQjnGsfc=', 'Content-Length': '0'} 2022-09-03T06:51:07.438 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d0caf532b6271ec0-006312f95b-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.439 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:51:07.440 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.440 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.441 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.442 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.442 DEBUG:boto:Method: GET 2022-09-03T06:51:07.443 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.444 DEBUG:boto:Data: 2022-09-03T06:51:07.444 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.445 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.445 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.446 DEBUG:boto:Params: {} 2022-09-03T06:51:07.446 DEBUG:boto:Token: None 2022-09-03T06:51:07.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/?versions 2022-09-03T06:51:07.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjn51ZWjO+KJstEXSUgxjJKDqOM= 2022-09-03T06:51:07.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjn51ZWjO+KJstEXSUgxjJKDqOM=', 'Content-Length': '0'} 2022-09-03T06:51:07.451 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf51f64cdebdfe67-006312f95b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.452 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.454 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:51:07.455 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:51:07.455 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:51:07.456 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:51:07.456 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:51:07.457 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.457 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.458 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.459 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.459 DEBUG:boto:Method: GET 2022-09-03T06:51:07.460 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.460 DEBUG:boto:Data: 2022-09-03T06:51:07.461 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.462 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.462 DEBUG:boto:Params: {} 2022-09-03T06:51:07.463 DEBUG:boto:Token: None 2022-09-03T06:51:07.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/?versions 2022-09-03T06:51:07.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjn51ZWjO+KJstEXSUgxjJKDqOM= 2022-09-03T06:51:07.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjn51ZWjO+KJstEXSUgxjJKDqOM=', 'Content-Length': '0'} 2022-09-03T06:51:07.467 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a88e711c9d1101d2-006312f95b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.468 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.469 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:51:07.470 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:51:07.470 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:51:07.471 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:51:07.471 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.472 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.473 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.473 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.474 DEBUG:boto:Method: GET 2022-09-03T06:51:07.474 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.475 DEBUG:boto:Data: 2022-09-03T06:51:07.476 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.476 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.477 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.478 DEBUG:boto:Params: {} 2022-09-03T06:51:07.479 DEBUG:boto:Token: None 2022-09-03T06:51:07.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/?versions 2022-09-03T06:51:07.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjn51ZWjO+KJstEXSUgxjJKDqOM= 2022-09-03T06:51:07.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjn51ZWjO+KJstEXSUgxjJKDqOM=', 'Content-Length': '0'} 2022-09-03T06:51:07.484 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4dc31cfd5264ac5-006312f95b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.485 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.486 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.486 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.487 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.487 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.488 DEBUG:boto:Method: GET 2022-09-03T06:51:07.488 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:07.489 DEBUG:boto:Data: 2022-09-03T06:51:07.490 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.491 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.492 DEBUG:boto:Params: {} 2022-09-03T06:51:07.492 DEBUG:boto:Token: None 2022-09-03T06:51:07.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/?versions 2022-09-03T06:51:07.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjn51ZWjO+KJstEXSUgxjJKDqOM= 2022-09-03T06:51:07.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjn51ZWjO+KJstEXSUgxjJKDqOM=', 'Content-Length': '0'} 2022-09-03T06:51:07.496 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005a146c70fd668ea8-006312f95b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.497 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.498 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:51:07.499 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.499 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.500 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.500 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.501 DEBUG:boto:Method: GET 2022-09-03T06:51:07.502 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:07.502 DEBUG:boto:Data: 2022-09-03T06:51:07.503 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.503 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.504 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.504 DEBUG:boto:Params: {} 2022-09-03T06:51:07.505 DEBUG:boto:Token: None 2022-09-03T06:51:07.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:07.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Arw7ZKWNuV3Ds1gTQPgBzVaYPGs= 2022-09-03T06:51:07.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Arw7ZKWNuV3Ds1gTQPgBzVaYPGs=', 'Content-Length': '0'} 2022-09-03T06:51:07.509 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d43c43e0302c32d-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.510 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.510 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.511 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.511 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.512 DEBUG:boto:Method: GET 2022-09-03T06:51:07.512 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:07.513 DEBUG:boto:Data: 2022-09-03T06:51:07.513 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.514 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.515 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.515 DEBUG:boto:Params: {} 2022-09-03T06:51:07.516 DEBUG:boto:Token: None 2022-09-03T06:51:07.516 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:07.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Arw7ZKWNuV3Ds1gTQPgBzVaYPGs= 2022-09-03T06:51:07.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Arw7ZKWNuV3Ds1gTQPgBzVaYPGs=', 'Content-Length': '0'} 2022-09-03T06:51:07.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9cc1deb2853faf0-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.521 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.521 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.522 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.522 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.523 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.523 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:07.524 DEBUG:boto:Data: 2022-09-03T06:51:07.524 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.525 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.525 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.526 DEBUG:boto:Params: {} 2022-09-03T06:51:07.527 DEBUG:boto:Token: None 2022-09-03T06:51:07.527 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:07.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T0oj0dlWQwPFqKqGQ6FtppfqYFk= 2022-09-03T06:51:07.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T0oj0dlWQwPFqKqGQ6FtppfqYFk=', 'Content-Length': '0'} 2022-09-03T06:51:07.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db216c92723bcb1b-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.531 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.531 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.532 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.532 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.533 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.533 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:07.534 DEBUG:boto:Data: 2022-09-03T06:51:07.535 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.536 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.537 DEBUG:boto:Params: {} 2022-09-03T06:51:07.537 DEBUG:boto:Token: None 2022-09-03T06:51:07.538 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:07.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T0oj0dlWQwPFqKqGQ6FtppfqYFk= 2022-09-03T06:51:07.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T0oj0dlWQwPFqKqGQ6FtppfqYFk=', 'Content-Length': '0'} 2022-09-03T06:51:07.541 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c67d6da18f7b283-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.542 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:51:07.542 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.543 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.544 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.544 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.545 DEBUG:boto:Method: GET 2022-09-03T06:51:07.545 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:07.546 DEBUG:boto:Data: 2022-09-03T06:51:07.546 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.547 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.547 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.548 DEBUG:boto:Params: {} 2022-09-03T06:51:07.548 DEBUG:boto:Token: None 2022-09-03T06:51:07.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:07.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Arw7ZKWNuV3Ds1gTQPgBzVaYPGs= 2022-09-03T06:51:07.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Arw7ZKWNuV3Ds1gTQPgBzVaYPGs=', 'Content-Length': '0'} 2022-09-03T06:51:07.552 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000108ee59a68ccc0cb-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.598 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.599 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.600 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.601 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:07.601 DEBUG:boto:Method: GET 2022-09-03T06:51:07.602 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:07.602 DEBUG:boto:Data: 2022-09-03T06:51:07.603 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.605 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.605 DEBUG:boto:Params: {} 2022-09-03T06:51:07.606 DEBUG:boto:Token: None 2022-09-03T06:51:07.607 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:07.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Arw7ZKWNuV3Ds1gTQPgBzVaYPGs= 2022-09-03T06:51:07.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Arw7ZKWNuV3Ds1gTQPgBzVaYPGs=', 'Content-Length': '0'} 2022-09-03T06:51:07.610 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b6b11f63724b287-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.612 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.613 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.614 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:07.615 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:07.615 DEBUG:boto:Method: GET 2022-09-03T06:51:07.616 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:07.616 DEBUG:boto:Data: 2022-09-03T06:51:07.617 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.617 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.618 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.619 DEBUG:boto:Params: {} 2022-09-03T06:51:07.619 DEBUG:boto:Token: None 2022-09-03T06:51:07.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ= 2022-09-03T06:51:07.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ=', 'Content-Length': '0'} 2022-09-03T06:51:07.624 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000b14a5a944bdaadd-006312f95b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.624 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.626 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.626 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.627 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:07.627 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:07.628 DEBUG:boto:Method: GET 2022-09-03T06:51:07.628 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:07.629 DEBUG:boto:Data: 2022-09-03T06:51:07.629 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.631 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.631 DEBUG:boto:Params: {} 2022-09-03T06:51:07.632 DEBUG:boto:Token: None 2022-09-03T06:51:07.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ= 2022-09-03T06:51:07.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ=', 'Content-Length': '0'} 2022-09-03T06:51:07.638 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000293e91e7f0c092aa-006312f95b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.639 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.640 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.641 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.642 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.643 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:07.643 DEBUG:boto:Data: 2022-09-03T06:51:07.644 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.644 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.645 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.646 DEBUG:boto:Params: {} 2022-09-03T06:51:07.646 DEBUG:boto:Token: None 2022-09-03T06:51:07.647 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a 2022-09-03T06:51:07.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dc//j2IdrVhaSFPZmNRgTEmuvaM= 2022-09-03T06:51:07.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dc//j2IdrVhaSFPZmNRgTEmuvaM=', 'Content-Length': '0'} 2022-09-03T06:51:07.651 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010f56be6fd28b8d1-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.652 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.652 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.653 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.653 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:07.654 DEBUG:boto:Data: 2022-09-03T06:51:07.654 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.655 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.656 DEBUG:boto:Params: {} 2022-09-03T06:51:07.656 DEBUG:boto:Token: None 2022-09-03T06:51:07.657 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a 2022-09-03T06:51:07.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dc//j2IdrVhaSFPZmNRgTEmuvaM= 2022-09-03T06:51:07.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dc//j2IdrVhaSFPZmNRgTEmuvaM=', 'Content-Length': '0'} 2022-09-03T06:51:07.660 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000691da1aa0c4c6d34-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.661 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:51:07.662 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.663 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.663 DEBUG:boto:Method: GET 2022-09-03T06:51:07.664 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:07.664 DEBUG:boto:Data: 2022-09-03T06:51:07.665 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.665 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.666 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.667 DEBUG:boto:Params: {} 2022-09-03T06:51:07.667 DEBUG:boto:Token: None 2022-09-03T06:51:07.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a 2022-09-03T06:51:07.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5IkFtN9a30oToHz9Ue8a86ytysg= 2022-09-03T06:51:07.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5IkFtN9a30oToHz9Ue8a86ytysg=', 'Content-Length': '0'} 2022-09-03T06:51:07.671 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6cfd49cf1e33936-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.672 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:07.673 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:07.673 DEBUG:boto:Method: GET 2022-09-03T06:51:07.674 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:07.674 DEBUG:boto:Data: 2022-09-03T06:51:07.675 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.676 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.677 DEBUG:boto:Params: {} 2022-09-03T06:51:07.677 DEBUG:boto:Token: None 2022-09-03T06:51:07.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/a 2022-09-03T06:51:07.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5IkFtN9a30oToHz9Ue8a86ytysg= 2022-09-03T06:51:07.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5IkFtN9a30oToHz9Ue8a86ytysg=', 'Content-Length': '0'} 2022-09-03T06:51:07.681 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f0a151821b70aaa-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.683 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.683 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.684 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:07.685 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:07.685 DEBUG:boto:Method: GET 2022-09-03T06:51:07.686 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:07.686 DEBUG:boto:Data: 2022-09-03T06:51:07.687 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.687 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.688 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.688 DEBUG:boto:Params: {} 2022-09-03T06:51:07.689 DEBUG:boto:Token: None 2022-09-03T06:51:07.690 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ= 2022-09-03T06:51:07.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ=', 'Content-Length': '0'} 2022-09-03T06:51:07.694 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069edf8d9e52331ea-006312f95b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.695 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.697 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.698 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.699 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:07.700 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:07.701 DEBUG:boto:Method: GET 2022-09-03T06:51:07.702 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:07.703 DEBUG:boto:Data: 2022-09-03T06:51:07.704 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.705 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.706 DEBUG:boto:Params: {} 2022-09-03T06:51:07.707 DEBUG:boto:Token: None 2022-09-03T06:51:07.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ= 2022-09-03T06:51:07.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ=', 'Content-Length': '0'} 2022-09-03T06:51:07.711 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003a5f642a2fd572f5-006312f95b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.712 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.714 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:51:07.715 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.715 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.716 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.716 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.717 DEBUG:boto:Method: GET 2022-09-03T06:51:07.717 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:07.718 DEBUG:boto:Data: 2022-09-03T06:51:07.719 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.719 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.720 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.720 DEBUG:boto:Params: {} 2022-09-03T06:51:07.721 DEBUG:boto:Token: None 2022-09-03T06:51:07.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:07.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MAJIhk1xnMOwVgPOahhYFT7kJTg= 2022-09-03T06:51:07.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MAJIhk1xnMOwVgPOahhYFT7kJTg=', 'Content-Length': '0'} 2022-09-03T06:51:07.725 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d390e1714b41a8cf-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.726 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.727 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.728 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.728 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.729 DEBUG:boto:Method: GET 2022-09-03T06:51:07.730 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:07.730 DEBUG:boto:Data: 2022-09-03T06:51:07.731 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.732 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.732 DEBUG:boto:Params: {} 2022-09-03T06:51:07.733 DEBUG:boto:Token: None 2022-09-03T06:51:07.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:07.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MAJIhk1xnMOwVgPOahhYFT7kJTg= 2022-09-03T06:51:07.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MAJIhk1xnMOwVgPOahhYFT7kJTg=', 'Content-Length': '0'} 2022-09-03T06:51:07.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000614fe7338469db72-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.738 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.739 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.739 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.740 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.740 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.741 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:07.741 DEBUG:boto:Data: 2022-09-03T06:51:07.742 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.742 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.743 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.743 DEBUG:boto:Params: {} 2022-09-03T06:51:07.744 DEBUG:boto:Token: None 2022-09-03T06:51:07.745 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:07.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7BZpbgVI46ucspi3e34Dv666jA= 2022-09-03T06:51:07.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7BZpbgVI46ucspi3e34Dv666jA=', 'Content-Length': '0'} 2022-09-03T06:51:07.748 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066bc06076971464a-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.749 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.750 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.750 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.751 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.751 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.752 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:07.753 DEBUG:boto:Data: 2022-09-03T06:51:07.754 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.756 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.757 DEBUG:boto:Params: {} 2022-09-03T06:51:07.758 DEBUG:boto:Token: None 2022-09-03T06:51:07.759 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:07.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7BZpbgVI46ucspi3e34Dv666jA= 2022-09-03T06:51:07.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7BZpbgVI46ucspi3e34Dv666jA=', 'Content-Length': '0'} 2022-09-03T06:51:07.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000552116b60a311086-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.764 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:51:07.765 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.766 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.767 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.767 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.768 DEBUG:boto:Method: GET 2022-09-03T06:51:07.769 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:07.770 DEBUG:boto:Data: 2022-09-03T06:51:07.771 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.772 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.773 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.773 DEBUG:boto:Params: {} 2022-09-03T06:51:07.774 DEBUG:boto:Token: None 2022-09-03T06:51:07.775 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:07.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MAJIhk1xnMOwVgPOahhYFT7kJTg= 2022-09-03T06:51:07.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MAJIhk1xnMOwVgPOahhYFT7kJTg=', 'Content-Length': '0'} 2022-09-03T06:51:07.779 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e24d94f21da89e37-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.781 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.782 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.783 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.784 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:07.785 DEBUG:boto:Method: GET 2022-09-03T06:51:07.785 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:07.786 DEBUG:boto:Data: 2022-09-03T06:51:07.787 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.788 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.789 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.790 DEBUG:boto:Params: {} 2022-09-03T06:51:07.790 DEBUG:boto:Token: None 2022-09-03T06:51:07.791 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:07.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MAJIhk1xnMOwVgPOahhYFT7kJTg= 2022-09-03T06:51:07.792 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MAJIhk1xnMOwVgPOahhYFT7kJTg=', 'Content-Length': '0'} 2022-09-03T06:51:07.795 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7c625374455965c-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.796 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.797 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.798 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:07.799 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:07.800 DEBUG:boto:Method: GET 2022-09-03T06:51:07.801 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:07.801 DEBUG:boto:Data: 2022-09-03T06:51:07.802 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.803 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.804 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.805 DEBUG:boto:Params: {} 2022-09-03T06:51:07.805 DEBUG:boto:Token: None 2022-09-03T06:51:07.806 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ= 2022-09-03T06:51:07.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ=', 'Content-Length': '0'} 2022-09-03T06:51:07.810 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005aca2e47b4219c89-006312f95b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.811 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.812 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.813 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.813 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:07.814 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:07.815 DEBUG:boto:Method: GET 2022-09-03T06:51:07.816 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:07.817 DEBUG:boto:Data: 2022-09-03T06:51:07.817 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.818 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.819 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.819 DEBUG:boto:Params: {} 2022-09-03T06:51:07.820 DEBUG:boto:Token: None 2022-09-03T06:51:07.820 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ= 2022-09-03T06:51:07.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ=', 'Content-Length': '0'} 2022-09-03T06:51:07.825 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001d06cf06dfe8cc72-006312f95b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.866 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.868 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.869 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.870 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.871 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:07.871 DEBUG:boto:Data: 2022-09-03T06:51:07.872 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.872 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.873 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.874 DEBUG:boto:Params: {} 2022-09-03T06:51:07.874 DEBUG:boto:Token: None 2022-09-03T06:51:07.875 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b 2022-09-03T06:51:07.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x5SrMVPBCLWFGoOrKhhNqZNiHrk= 2022-09-03T06:51:07.876 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x5SrMVPBCLWFGoOrKhhNqZNiHrk=', 'Content-Length': '0'} 2022-09-03T06:51:07.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036469fcd543114df-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.879 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.880 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.881 DEBUG:boto:Method: HEAD 2022-09-03T06:51:07.881 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:07.882 DEBUG:boto:Data: 2022-09-03T06:51:07.882 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.884 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.884 DEBUG:boto:Params: {} 2022-09-03T06:51:07.885 DEBUG:boto:Token: None 2022-09-03T06:51:07.886 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b 2022-09-03T06:51:07.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x5SrMVPBCLWFGoOrKhhNqZNiHrk= 2022-09-03T06:51:07.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x5SrMVPBCLWFGoOrKhhNqZNiHrk=', 'Content-Length': '0'} 2022-09-03T06:51:07.890 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dda2c6d55cd02e0b-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.891 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:51:07.892 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.893 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.894 DEBUG:boto:Method: GET 2022-09-03T06:51:07.895 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:07.896 DEBUG:boto:Data: 2022-09-03T06:51:07.897 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.900 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.901 DEBUG:boto:Params: {} 2022-09-03T06:51:07.902 DEBUG:boto:Token: None 2022-09-03T06:51:07.903 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b 2022-09-03T06:51:07.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EKVYkDHRvD3HwUAULog2iG6/fC8= 2022-09-03T06:51:07.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EKVYkDHRvD3HwUAULog2iG6/fC8=', 'Content-Length': '0'} 2022-09-03T06:51:07.910 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e642e9b8ccfadeb-006312f95b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.958 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:07.959 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:07.961 DEBUG:boto:Method: GET 2022-09-03T06:51:07.962 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:07.962 DEBUG:boto:Data: 2022-09-03T06:51:07.963 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.964 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.964 DEBUG:boto:Params: {} 2022-09-03T06:51:07.965 DEBUG:boto:Token: None 2022-09-03T06:51:07.966 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/b 2022-09-03T06:51:07.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EKVYkDHRvD3HwUAULog2iG6/fC8= 2022-09-03T06:51:07.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EKVYkDHRvD3HwUAULog2iG6/fC8=', 'Content-Length': '0'} 2022-09-03T06:51:07.969 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f65f5c3a03e73a4-006312f95b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.970 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.971 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.972 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:07.972 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:07.973 DEBUG:boto:Method: GET 2022-09-03T06:51:07.974 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:07.974 DEBUG:boto:Data: 2022-09-03T06:51:07.975 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.975 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:07.976 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.976 DEBUG:boto:Params: {} 2022-09-03T06:51:07.977 DEBUG:boto:Token: None 2022-09-03T06:51:07.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ= 2022-09-03T06:51:07.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ=', 'Content-Length': '0'} 2022-09-03T06:51:07.984 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005dc239b77daefc4f-006312f95b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.985 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:07.986 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:07.986 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:07.987 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:07.987 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:07.988 DEBUG:boto:Method: GET 2022-09-03T06:51:07.989 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:07.989 DEBUG:boto:Data: 2022-09-03T06:51:07.990 DEBUG:boto:Headers: {} 2022-09-03T06:51:07.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:07.991 DEBUG:boto:Port: 8000 2022-09-03T06:51:07.991 DEBUG:boto:Params: {} 2022-09-03T06:51:07.992 DEBUG:boto:Token: None 2022-09-03T06:51:07.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:07 GMT /avysmw-70/ 2022-09-03T06:51:07.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ= 2022-09-03T06:51:07.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7J/4DtzKRAtCfuYkickwrrCfmAQ=', 'Content-Length': '0'} 2022-09-03T06:51:07.997 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ccd39a12fd05a514-006312f95b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:07.999 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.000 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:51:08.000 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.001 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.002 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.003 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.004 DEBUG:boto:Method: GET 2022-09-03T06:51:08.005 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:08.006 DEBUG:boto:Data: 2022-09-03T06:51:08.007 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.008 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.009 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.009 DEBUG:boto:Params: {} 2022-09-03T06:51:08.010 DEBUG:boto:Token: None 2022-09-03T06:51:08.011 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:08.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FzWjiEHd5DymtDQJvvruHBBTgDk= 2022-09-03T06:51:08.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FzWjiEHd5DymtDQJvvruHBBTgDk=', 'Content-Length': '0'} 2022-09-03T06:51:08.014 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6f255d023f9ab17-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.016 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.017 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.018 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.019 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.020 DEBUG:boto:Method: GET 2022-09-03T06:51:08.021 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:08.022 DEBUG:boto:Data: 2022-09-03T06:51:08.023 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.024 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.025 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.025 DEBUG:boto:Params: {} 2022-09-03T06:51:08.026 DEBUG:boto:Token: None 2022-09-03T06:51:08.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:08.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FzWjiEHd5DymtDQJvvruHBBTgDk= 2022-09-03T06:51:08.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FzWjiEHd5DymtDQJvvruHBBTgDk=', 'Content-Length': '0'} 2022-09-03T06:51:08.030 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000def388c011637e53-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.032 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.032 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.033 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.033 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.034 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.035 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:08.035 DEBUG:boto:Data: 2022-09-03T06:51:08.036 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.037 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.037 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.038 DEBUG:boto:Params: {} 2022-09-03T06:51:08.038 DEBUG:boto:Token: None 2022-09-03T06:51:08.039 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:08.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:03bpPENW8DYlTVxkqkOfS5rrnic= 2022-09-03T06:51:08.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:03bpPENW8DYlTVxkqkOfS5rrnic=', 'Content-Length': '0'} 2022-09-03T06:51:08.043 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c9e6baa36e578eb-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.043 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.044 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.045 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.045 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.046 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.047 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:08.047 DEBUG:boto:Data: 2022-09-03T06:51:08.048 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.048 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.049 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.050 DEBUG:boto:Params: {} 2022-09-03T06:51:08.050 DEBUG:boto:Token: None 2022-09-03T06:51:08.051 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:08.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:03bpPENW8DYlTVxkqkOfS5rrnic= 2022-09-03T06:51:08.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:03bpPENW8DYlTVxkqkOfS5rrnic=', 'Content-Length': '0'} 2022-09-03T06:51:08.054 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001cdb70a422131c4d-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.056 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:51:08.056 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.057 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.058 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.058 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.059 DEBUG:boto:Method: GET 2022-09-03T06:51:08.060 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:08.061 DEBUG:boto:Data: 2022-09-03T06:51:08.061 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.062 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.063 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.063 DEBUG:boto:Params: {} 2022-09-03T06:51:08.064 DEBUG:boto:Token: None 2022-09-03T06:51:08.065 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:08.066 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FzWjiEHd5DymtDQJvvruHBBTgDk= 2022-09-03T06:51:08.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FzWjiEHd5DymtDQJvvruHBBTgDk=', 'Content-Length': '0'} 2022-09-03T06:51:08.069 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fb935b6b207f727-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.111 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.112 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.112 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.113 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:08.115 DEBUG:boto:Method: GET 2022-09-03T06:51:08.115 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:08.116 DEBUG:boto:Data: 2022-09-03T06:51:08.116 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.118 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.119 DEBUG:boto:Params: {} 2022-09-03T06:51:08.119 DEBUG:boto:Token: None 2022-09-03T06:51:08.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:08.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FzWjiEHd5DymtDQJvvruHBBTgDk= 2022-09-03T06:51:08.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FzWjiEHd5DymtDQJvvruHBBTgDk=', 'Content-Length': '0'} 2022-09-03T06:51:08.124 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003883d458cbc5a9fe-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.125 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.125 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.126 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:08.127 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:08.128 DEBUG:boto:Method: GET 2022-09-03T06:51:08.128 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:08.129 DEBUG:boto:Data: 2022-09-03T06:51:08.129 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.130 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.130 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.131 DEBUG:boto:Params: {} 2022-09-03T06:51:08.132 DEBUG:boto:Token: None 2022-09-03T06:51:08.132 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.133 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.136 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006005fd62d26650b5-006312f95c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.137 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.138 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.138 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.139 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:08.140 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:08.140 DEBUG:boto:Method: GET 2022-09-03T06:51:08.141 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:08.142 DEBUG:boto:Data: 2022-09-03T06:51:08.142 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.143 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.144 DEBUG:boto:Params: {} 2022-09-03T06:51:08.145 DEBUG:boto:Token: None 2022-09-03T06:51:08.145 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.149 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005c910dc07bf8e0b0-006312f95c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.150 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.151 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.152 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.152 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.153 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:08.154 DEBUG:boto:Data: 2022-09-03T06:51:08.154 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.155 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.155 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.156 DEBUG:boto:Params: {} 2022-09-03T06:51:08.157 DEBUG:boto:Token: None 2022-09-03T06:51:08.157 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c 2022-09-03T06:51:08.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ty0y5Qxam+2N1DLGVZEoC7D7uF0= 2022-09-03T06:51:08.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ty0y5Qxam+2N1DLGVZEoC7D7uF0=', 'Content-Length': '0'} 2022-09-03T06:51:08.161 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002667613d00a4fd52-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.162 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.163 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.163 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.164 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:08.164 DEBUG:boto:Data: 2022-09-03T06:51:08.165 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.166 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.167 DEBUG:boto:Params: {} 2022-09-03T06:51:08.168 DEBUG:boto:Token: None 2022-09-03T06:51:08.168 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c 2022-09-03T06:51:08.169 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ty0y5Qxam+2N1DLGVZEoC7D7uF0= 2022-09-03T06:51:08.170 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ty0y5Qxam+2N1DLGVZEoC7D7uF0=', 'Content-Length': '0'} 2022-09-03T06:51:08.172 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000926f677ed62c4975-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.172 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:51:08.173 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.174 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.175 DEBUG:boto:Method: GET 2022-09-03T06:51:08.175 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:08.176 DEBUG:boto:Data: 2022-09-03T06:51:08.176 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.177 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.178 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.178 DEBUG:boto:Params: {} 2022-09-03T06:51:08.179 DEBUG:boto:Token: None 2022-09-03T06:51:08.180 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c 2022-09-03T06:51:08.181 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pBMLCOjF2ymSFg8y8N4LhTcofR8= 2022-09-03T06:51:08.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pBMLCOjF2ymSFg8y8N4LhTcofR8=', 'Content-Length': '0'} 2022-09-03T06:51:08.183 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c35cb0a467325448-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.184 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:08.185 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:08.185 DEBUG:boto:Method: GET 2022-09-03T06:51:08.186 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:08.187 DEBUG:boto:Data: 2022-09-03T06:51:08.187 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.188 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.188 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.189 DEBUG:boto:Params: {} 2022-09-03T06:51:08.189 DEBUG:boto:Token: None 2022-09-03T06:51:08.190 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/c 2022-09-03T06:51:08.191 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pBMLCOjF2ymSFg8y8N4LhTcofR8= 2022-09-03T06:51:08.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pBMLCOjF2ymSFg8y8N4LhTcofR8=', 'Content-Length': '0'} 2022-09-03T06:51:08.193 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b53cccb88061579-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.194 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.195 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.196 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:08.196 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:08.197 DEBUG:boto:Method: GET 2022-09-03T06:51:08.197 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:08.198 DEBUG:boto:Data: 2022-09-03T06:51:08.198 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.199 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.200 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.200 DEBUG:boto:Params: {} 2022-09-03T06:51:08.201 DEBUG:boto:Token: None 2022-09-03T06:51:08.201 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.202 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.203 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.205 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004b9a8d92f12079a3-006312f95c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.206 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.207 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.208 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.208 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:08.209 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:08.209 DEBUG:boto:Method: GET 2022-09-03T06:51:08.210 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:08.210 DEBUG:boto:Data: 2022-09-03T06:51:08.211 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.212 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.213 DEBUG:boto:Params: {} 2022-09-03T06:51:08.213 DEBUG:boto:Token: None 2022-09-03T06:51:08.214 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.215 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.216 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.219 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008af2740f3b7b9366-006312f95c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.220 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.221 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:51:08.222 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.223 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.228 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.229 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.230 DEBUG:boto:Method: GET 2022-09-03T06:51:08.231 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:08.232 DEBUG:boto:Data: 2022-09-03T06:51:08.233 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.234 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.235 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.236 DEBUG:boto:Params: {} 2022-09-03T06:51:08.238 DEBUG:boto:Token: None 2022-09-03T06:51:08.239 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:08.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+G3dr4ZhDXfhdMqniqBYQh++xk= 2022-09-03T06:51:08.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+G3dr4ZhDXfhdMqniqBYQh++xk=', 'Content-Length': '0'} 2022-09-03T06:51:08.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c6d3821834b93f5e-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.246 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.246 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.247 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.248 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.248 DEBUG:boto:Method: GET 2022-09-03T06:51:08.251 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:08.251 DEBUG:boto:Data: 2022-09-03T06:51:08.252 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.253 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.254 DEBUG:boto:Params: {} 2022-09-03T06:51:08.256 DEBUG:boto:Token: None 2022-09-03T06:51:08.256 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:08.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+G3dr4ZhDXfhdMqniqBYQh++xk= 2022-09-03T06:51:08.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+G3dr4ZhDXfhdMqniqBYQh++xk=', 'Content-Length': '0'} 2022-09-03T06:51:08.263 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f361321696bdc9c8-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.311 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.313 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.314 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.315 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.316 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.316 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:08.317 DEBUG:boto:Data: 2022-09-03T06:51:08.318 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.319 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.320 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.322 DEBUG:boto:Params: {} 2022-09-03T06:51:08.323 DEBUG:boto:Token: None 2022-09-03T06:51:08.324 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:08.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZeo6ebc/jSqonlZKwj5hebuhYY= 2022-09-03T06:51:08.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZeo6ebc/jSqonlZKwj5hebuhYY=', 'Content-Length': '0'} 2022-09-03T06:51:08.328 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b66345a18578a8b-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.330 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.331 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.333 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.334 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.335 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.336 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:08.337 DEBUG:boto:Data: 2022-09-03T06:51:08.338 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.339 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.340 DEBUG:boto:Params: {} 2022-09-03T06:51:08.340 DEBUG:boto:Token: None 2022-09-03T06:51:08.344 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:08.346 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZeo6ebc/jSqonlZKwj5hebuhYY= 2022-09-03T06:51:08.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZeo6ebc/jSqonlZKwj5hebuhYY=', 'Content-Length': '0'} 2022-09-03T06:51:08.349 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d99c574be7f596d-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.350 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:51:08.351 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.352 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.352 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.353 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.353 DEBUG:boto:Method: GET 2022-09-03T06:51:08.354 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:08.354 DEBUG:boto:Data: 2022-09-03T06:51:08.355 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.355 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.356 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.357 DEBUG:boto:Params: {} 2022-09-03T06:51:08.357 DEBUG:boto:Token: None 2022-09-03T06:51:08.358 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:08.359 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+G3dr4ZhDXfhdMqniqBYQh++xk= 2022-09-03T06:51:08.360 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+G3dr4ZhDXfhdMqniqBYQh++xk=', 'Content-Length': '0'} 2022-09-03T06:51:08.361 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e54f0267b71e1dc-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.362 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.363 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.363 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.364 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:08.365 DEBUG:boto:Method: GET 2022-09-03T06:51:08.365 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:08.366 DEBUG:boto:Data: 2022-09-03T06:51:08.367 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.368 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.369 DEBUG:boto:Params: {} 2022-09-03T06:51:08.370 DEBUG:boto:Token: None 2022-09-03T06:51:08.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:08.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+G3dr4ZhDXfhdMqniqBYQh++xk= 2022-09-03T06:51:08.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+G3dr4ZhDXfhdMqniqBYQh++xk=', 'Content-Length': '0'} 2022-09-03T06:51:08.377 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ad3af9b140d70f8-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.378 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.379 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.381 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:08.382 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:08.383 DEBUG:boto:Method: GET 2022-09-03T06:51:08.384 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:08.385 DEBUG:boto:Data: 2022-09-03T06:51:08.386 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.387 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.388 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.389 DEBUG:boto:Params: {} 2022-09-03T06:51:08.390 DEBUG:boto:Token: None 2022-09-03T06:51:08.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.398 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000030fc5d784cc863e0-006312f95c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.399 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.400 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.401 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.402 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:08.403 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:08.404 DEBUG:boto:Method: GET 2022-09-03T06:51:08.405 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:08.406 DEBUG:boto:Data: 2022-09-03T06:51:08.408 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.408 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.409 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.410 DEBUG:boto:Params: {} 2022-09-03T06:51:08.411 DEBUG:boto:Token: None 2022-09-03T06:51:08.412 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.414 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.418 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005f5262ec3671063c-006312f95c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.419 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.421 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.422 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.423 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.424 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:08.425 DEBUG:boto:Data: 2022-09-03T06:51:08.426 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.427 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.427 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.428 DEBUG:boto:Params: {} 2022-09-03T06:51:08.430 DEBUG:boto:Token: None 2022-09-03T06:51:08.431 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d 2022-09-03T06:51:08.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HpinzxEd4LWL9cS/zvVspgFs9JE= 2022-09-03T06:51:08.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HpinzxEd4LWL9cS/zvVspgFs9JE=', 'Content-Length': '0'} 2022-09-03T06:51:08.439 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070d83d7ec5fb7fb4-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.442 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.443 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.445 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.446 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:08.447 DEBUG:boto:Data: 2022-09-03T06:51:08.447 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.449 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.450 DEBUG:boto:Params: {} 2022-09-03T06:51:08.450 DEBUG:boto:Token: None 2022-09-03T06:51:08.451 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d 2022-09-03T06:51:08.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HpinzxEd4LWL9cS/zvVspgFs9JE= 2022-09-03T06:51:08.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HpinzxEd4LWL9cS/zvVspgFs9JE=', 'Content-Length': '0'} 2022-09-03T06:51:08.458 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046faa481c6a32155-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.460 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:51:08.461 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.462 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.464 DEBUG:boto:Method: GET 2022-09-03T06:51:08.464 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:08.465 DEBUG:boto:Data: 2022-09-03T06:51:08.466 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.467 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.467 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.468 DEBUG:boto:Params: {} 2022-09-03T06:51:08.469 DEBUG:boto:Token: None 2022-09-03T06:51:08.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d 2022-09-03T06:51:08.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7ausCfxXcAmxRjaGTDU5yMCrPgs= 2022-09-03T06:51:08.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7ausCfxXcAmxRjaGTDU5yMCrPgs=', 'Content-Length': '0'} 2022-09-03T06:51:08.474 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cc463b3afcb01b9-006312f95c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.475 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:08.475 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:08.476 DEBUG:boto:Method: GET 2022-09-03T06:51:08.476 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:08.477 DEBUG:boto:Data: 2022-09-03T06:51:08.478 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.479 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.480 DEBUG:boto:Params: {} 2022-09-03T06:51:08.481 DEBUG:boto:Token: None 2022-09-03T06:51:08.482 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/d 2022-09-03T06:51:08.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7ausCfxXcAmxRjaGTDU5yMCrPgs= 2022-09-03T06:51:08.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7ausCfxXcAmxRjaGTDU5yMCrPgs=', 'Content-Length': '0'} 2022-09-03T06:51:08.488 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c892605ae1a1c987-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.530 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.531 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.532 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:08.532 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:08.533 DEBUG:boto:Method: GET 2022-09-03T06:51:08.533 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:08.534 DEBUG:boto:Data: 2022-09-03T06:51:08.534 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.535 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:08.536 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.536 DEBUG:boto:Params: {} 2022-09-03T06:51:08.537 DEBUG:boto:Token: None 2022-09-03T06:51:08.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.542 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c94543e79fc8765b-006312f95c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.543 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.543 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.544 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.545 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:08.545 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:08.546 DEBUG:boto:Method: GET 2022-09-03T06:51:08.547 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:08.548 DEBUG:boto:Data: 2022-09-03T06:51:08.548 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.549 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.549 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.550 DEBUG:boto:Params: {} 2022-09-03T06:51:08.551 DEBUG:boto:Token: None 2022-09-03T06:51:08.552 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.555 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001d5d08ba185c552e-006312f95c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.556 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.558 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-70 zones={test-zone2, test-zone1} 2022-09-03T06:51:08.559 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-70 zones={test-zone3, test-zone1} 2022-09-03T06:51:08.559 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.560 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.560 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.561 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:51:08.561 DEBUG:boto:Data: 2022-09-03T06:51:08.562 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.563 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.563 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.564 DEBUG:boto:Params: {} 2022-09-03T06:51:08.564 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:51:08.565 DEBUG:boto:Token: None 2022-09-03T06:51:08.566 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.566 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jlqu6ysp2tV9lZriBKnSL5eMlzo= 2022-09-03T06:51:08.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jlqu6ysp2tV9lZriBKnSL5eMlzo=', 'Content-Length': '0'} 2022-09-03T06:51:08.571 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d12b4a9f1db65600-006312f95c-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.572 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.572 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.573 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.574 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:51:08.575 DEBUG:boto:Data: 2022-09-03T06:51:08.575 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.577 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.577 DEBUG:boto:Params: {} 2022-09-03T06:51:08.578 DEBUG:boto:Token: None 2022-09-03T06:51:08.579 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jlqu6ysp2tV9lZriBKnSL5eMlzo= 2022-09-03T06:51:08.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jlqu6ysp2tV9lZriBKnSL5eMlzo=', 'Content-Length': '0'} 2022-09-03T06:51:08.582 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c30962cf2e49d561-006312f95c-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.583 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:51:08.584 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.584 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.585 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.585 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.587 DEBUG:boto:Method: GET 2022-09-03T06:51:08.588 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.588 DEBUG:boto:Data: 2022-09-03T06:51:08.589 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.590 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.591 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.592 DEBUG:boto:Params: {} 2022-09-03T06:51:08.592 DEBUG:boto:Token: None 2022-09-03T06:51:08.593 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/?versions 2022-09-03T06:51:08.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9CTP3ZVL7w1K6g6px0pcd4Fc5w= 2022-09-03T06:51:08.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9CTP3ZVL7w1K6g6px0pcd4Fc5w=', 'Content-Length': '0'} 2022-09-03T06:51:08.597 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc746bc38cd00619-006312f95c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.598 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.599 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:51:08.599 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:51:08.600 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:51:08.601 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:51:08.601 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:51:08.602 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.602 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.603 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.603 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.604 DEBUG:boto:Method: GET 2022-09-03T06:51:08.604 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.605 DEBUG:boto:Data: 2022-09-03T06:51:08.605 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.606 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.606 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.607 DEBUG:boto:Params: {} 2022-09-03T06:51:08.608 DEBUG:boto:Token: None 2022-09-03T06:51:08.608 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/?versions 2022-09-03T06:51:08.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9CTP3ZVL7w1K6g6px0pcd4Fc5w= 2022-09-03T06:51:08.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9CTP3ZVL7w1K6g6px0pcd4Fc5w=', 'Content-Length': '0'} 2022-09-03T06:51:08.612 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1fee0cd80b97b61-006312f95c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.613 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.614 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:51:08.614 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:51:08.615 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:51:08.615 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:51:08.616 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.616 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.617 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.617 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.618 DEBUG:boto:Method: GET 2022-09-03T06:51:08.618 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.619 DEBUG:boto:Data: 2022-09-03T06:51:08.619 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.620 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.621 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.621 DEBUG:boto:Params: {} 2022-09-03T06:51:08.622 DEBUG:boto:Token: None 2022-09-03T06:51:08.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/?versions 2022-09-03T06:51:08.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9CTP3ZVL7w1K6g6px0pcd4Fc5w= 2022-09-03T06:51:08.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9CTP3ZVL7w1K6g6px0pcd4Fc5w=', 'Content-Length': '0'} 2022-09-03T06:51:08.627 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bdbdb0a16a3a82ff-006312f95c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.628 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.630 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.631 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.632 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.633 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.634 DEBUG:boto:Method: GET 2022-09-03T06:51:08.634 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:08.635 DEBUG:boto:Data: 2022-09-03T06:51:08.635 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.636 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.637 DEBUG:boto:Params: {} 2022-09-03T06:51:08.638 DEBUG:boto:Token: None 2022-09-03T06:51:08.638 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/?versions 2022-09-03T06:51:08.639 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9CTP3ZVL7w1K6g6px0pcd4Fc5w= 2022-09-03T06:51:08.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9CTP3ZVL7w1K6g6px0pcd4Fc5w=', 'Content-Length': '0'} 2022-09-03T06:51:08.642 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b5264cddf66a992c-006312f95c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.643 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.644 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:51:08.644 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.645 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.645 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.646 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.647 DEBUG:boto:Method: GET 2022-09-03T06:51:08.648 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:08.649 DEBUG:boto:Data: 2022-09-03T06:51:08.649 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.650 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.650 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.651 DEBUG:boto:Params: {} 2022-09-03T06:51:08.652 DEBUG:boto:Token: None 2022-09-03T06:51:08.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:08.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:732IUWWPLpipWurbnZsYtsQPRB4= 2022-09-03T06:51:08.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:732IUWWPLpipWurbnZsYtsQPRB4=', 'Content-Length': '0'} 2022-09-03T06:51:08.659 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3d1180959c1496a-006312f95c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.661 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.662 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.663 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.664 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.665 DEBUG:boto:Method: GET 2022-09-03T06:51:08.666 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:08.666 DEBUG:boto:Data: 2022-09-03T06:51:08.667 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.667 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.668 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.669 DEBUG:boto:Params: {} 2022-09-03T06:51:08.670 DEBUG:boto:Token: None 2022-09-03T06:51:08.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:08.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:732IUWWPLpipWurbnZsYtsQPRB4= 2022-09-03T06:51:08.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:732IUWWPLpipWurbnZsYtsQPRB4=', 'Content-Length': '0'} 2022-09-03T06:51:08.677 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d625ebb4ce972d78-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.722 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.723 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.724 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.724 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.725 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.726 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:08.727 DEBUG:boto:Data: 2022-09-03T06:51:08.728 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.730 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.731 DEBUG:boto:Params: {} 2022-09-03T06:51:08.732 DEBUG:boto:Token: None 2022-09-03T06:51:08.733 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:08.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OZgT5eI21q1cSOr0jDiuw9coK1M= 2022-09-03T06:51:08.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OZgT5eI21q1cSOr0jDiuw9coK1M=', 'Content-Length': '0'} 2022-09-03T06:51:08.738 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b21f496fffa24e45-006312f95c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.739 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.740 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.741 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.742 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.743 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.743 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:08.744 DEBUG:boto:Data: 2022-09-03T06:51:08.744 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.745 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.746 DEBUG:boto:Params: {} 2022-09-03T06:51:08.747 DEBUG:boto:Token: None 2022-09-03T06:51:08.747 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:08.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OZgT5eI21q1cSOr0jDiuw9coK1M= 2022-09-03T06:51:08.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OZgT5eI21q1cSOr0jDiuw9coK1M=', 'Content-Length': '0'} 2022-09-03T06:51:08.750 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8bc11eaa38b399c-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.751 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:51:08.752 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.752 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.753 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.754 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.756 DEBUG:boto:Method: GET 2022-09-03T06:51:08.757 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:08.758 DEBUG:boto:Data: 2022-09-03T06:51:08.759 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.760 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.761 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.761 DEBUG:boto:Params: {} 2022-09-03T06:51:08.762 DEBUG:boto:Token: None 2022-09-03T06:51:08.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:08.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:732IUWWPLpipWurbnZsYtsQPRB4= 2022-09-03T06:51:08.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:732IUWWPLpipWurbnZsYtsQPRB4=', 'Content-Length': '0'} 2022-09-03T06:51:08.778 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b2b823e1e39ff16-006312f95c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.780 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.781 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.782 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.783 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:08.784 DEBUG:boto:Method: GET 2022-09-03T06:51:08.785 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:08.786 DEBUG:boto:Data: 2022-09-03T06:51:08.788 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.789 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.790 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.791 DEBUG:boto:Params: {} 2022-09-03T06:51:08.792 DEBUG:boto:Token: None 2022-09-03T06:51:08.793 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:08.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:732IUWWPLpipWurbnZsYtsQPRB4= 2022-09-03T06:51:08.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:732IUWWPLpipWurbnZsYtsQPRB4=', 'Content-Length': '0'} 2022-09-03T06:51:08.799 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bdd40bb6614c9628-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.801 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.802 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.803 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:08.804 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:08.805 DEBUG:boto:Method: GET 2022-09-03T06:51:08.806 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:08.806 DEBUG:boto:Data: 2022-09-03T06:51:08.807 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.808 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.811 DEBUG:boto:Params: {} 2022-09-03T06:51:08.812 DEBUG:boto:Token: None 2022-09-03T06:51:08.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.819 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d52ceb219b518755-006312f95c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.820 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.823 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.824 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.825 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:08.826 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:08.827 DEBUG:boto:Method: GET 2022-09-03T06:51:08.828 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:08.828 DEBUG:boto:Data: 2022-09-03T06:51:08.829 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.831 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.831 DEBUG:boto:Params: {} 2022-09-03T06:51:08.832 DEBUG:boto:Token: None 2022-09-03T06:51:08.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.837 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a1be573f27ad284e-006312f95c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.878 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.880 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.880 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.881 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.882 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:08.882 DEBUG:boto:Data: 2022-09-03T06:51:08.883 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.884 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.884 DEBUG:boto:Params: {} 2022-09-03T06:51:08.885 DEBUG:boto:Token: None 2022-09-03T06:51:08.885 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a 2022-09-03T06:51:08.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b+lXyzvakdYQZ+GXh9stEypQNHY= 2022-09-03T06:51:08.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b+lXyzvakdYQZ+GXh9stEypQNHY=', 'Content-Length': '0'} 2022-09-03T06:51:08.889 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000210c6c9569284743-006312f95c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.889 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.890 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.891 DEBUG:boto:Method: HEAD 2022-09-03T06:51:08.891 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:08.892 DEBUG:boto:Data: 2022-09-03T06:51:08.892 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.893 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.894 DEBUG:boto:Params: {} 2022-09-03T06:51:08.895 DEBUG:boto:Token: None 2022-09-03T06:51:08.895 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a 2022-09-03T06:51:08.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b+lXyzvakdYQZ+GXh9stEypQNHY= 2022-09-03T06:51:08.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b+lXyzvakdYQZ+GXh9stEypQNHY=', 'Content-Length': '0'} 2022-09-03T06:51:08.899 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc14bbbb7a1204cf-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.899 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:51:08.900 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.901 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.902 DEBUG:boto:Method: GET 2022-09-03T06:51:08.903 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:08.904 DEBUG:boto:Data: 2022-09-03T06:51:08.904 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.906 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.907 DEBUG:boto:Params: {} 2022-09-03T06:51:08.908 DEBUG:boto:Token: None 2022-09-03T06:51:08.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a 2022-09-03T06:51:08.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:917UqP/sF0CTa3RLE74wAAd6FEQ= 2022-09-03T06:51:08.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:917UqP/sF0CTa3RLE74wAAd6FEQ=', 'Content-Length': '0'} 2022-09-03T06:51:08.914 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000842750f651d0f1a-006312f95c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.917 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:08.917 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:08.918 DEBUG:boto:Method: GET 2022-09-03T06:51:08.919 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:08.920 DEBUG:boto:Data: 2022-09-03T06:51:08.921 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.922 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.923 DEBUG:boto:Params: {} 2022-09-03T06:51:08.924 DEBUG:boto:Token: None 2022-09-03T06:51:08.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/a 2022-09-03T06:51:08.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:917UqP/sF0CTa3RLE74wAAd6FEQ= 2022-09-03T06:51:08.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:917UqP/sF0CTa3RLE74wAAd6FEQ=', 'Content-Length': '0'} 2022-09-03T06:51:08.928 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007711f98f9de8545b-006312f95c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.929 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.930 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.931 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:08.931 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:08.932 DEBUG:boto:Method: GET 2022-09-03T06:51:08.932 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:08.933 DEBUG:boto:Data: 2022-09-03T06:51:08.933 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:08.934 DEBUG:boto:Port: 8001 2022-09-03T06:51:08.935 DEBUG:boto:Params: {} 2022-09-03T06:51:08.936 DEBUG:boto:Token: None 2022-09-03T06:51:08.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.941 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9743f3457f59237-006312f95c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.982 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.983 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:08.984 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:08.985 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:08.985 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:08.986 DEBUG:boto:Method: GET 2022-09-03T06:51:08.987 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:08.987 DEBUG:boto:Data: 2022-09-03T06:51:08.988 DEBUG:boto:Headers: {} 2022-09-03T06:51:08.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:08.989 DEBUG:boto:Port: 8000 2022-09-03T06:51:08.989 DEBUG:boto:Params: {} 2022-09-03T06:51:08.990 DEBUG:boto:Token: None 2022-09-03T06:51:08.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:08 GMT /avysmw-70/ 2022-09-03T06:51:08.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo= 2022-09-03T06:51:08.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oUw6VAOBuoHFQPSgnU6VC54vMGo=', 'Content-Length': '0'} 2022-09-03T06:51:08.995 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036cefa34c1de2dd6-006312f95c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:08.996 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:08.998 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:51:08.999 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.000 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.000 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.001 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.001 DEBUG:boto:Method: GET 2022-09-03T06:51:09.002 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:09.003 DEBUG:boto:Data: 2022-09-03T06:51:09.003 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.004 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.004 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.005 DEBUG:boto:Params: {} 2022-09-03T06:51:09.007 DEBUG:boto:Token: None 2022-09-03T06:51:09.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:09.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9u6g0vNduG00Dz4YU/Wrs8zTemk= 2022-09-03T06:51:09.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9u6g0vNduG00Dz4YU/Wrs8zTemk=', 'Content-Length': '0'} 2022-09-03T06:51:09.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e9d442b935a02d6-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.012 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.012 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.013 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.013 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.014 DEBUG:boto:Method: GET 2022-09-03T06:51:09.015 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:09.016 DEBUG:boto:Data: 2022-09-03T06:51:09.017 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.018 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.019 DEBUG:boto:Params: {} 2022-09-03T06:51:09.019 DEBUG:boto:Token: None 2022-09-03T06:51:09.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:09.021 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9u6g0vNduG00Dz4YU/Wrs8zTemk= 2022-09-03T06:51:09.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9u6g0vNduG00Dz4YU/Wrs8zTemk=', 'Content-Length': '0'} 2022-09-03T06:51:09.026 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af6aceea5cc49068-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.070 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.071 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.072 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.072 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.073 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.073 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:09.074 DEBUG:boto:Data: 2022-09-03T06:51:09.075 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.076 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.076 DEBUG:boto:Params: {} 2022-09-03T06:51:09.077 DEBUG:boto:Token: None 2022-09-03T06:51:09.078 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:09.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bR/ni7T37iPcDVmVXw0wfKQg2g0= 2022-09-03T06:51:09.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bR/ni7T37iPcDVmVXw0wfKQg2g0=', 'Content-Length': '0'} 2022-09-03T06:51:09.089 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1359e574989a0d1-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.090 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.091 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.091 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.092 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.093 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.093 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:09.094 DEBUG:boto:Data: 2022-09-03T06:51:09.094 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.095 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.096 DEBUG:boto:Params: {} 2022-09-03T06:51:09.097 DEBUG:boto:Token: None 2022-09-03T06:51:09.097 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:09.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bR/ni7T37iPcDVmVXw0wfKQg2g0= 2022-09-03T06:51:09.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bR/ni7T37iPcDVmVXw0wfKQg2g0=', 'Content-Length': '0'} 2022-09-03T06:51:09.102 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef8adee24e361cd1-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.105 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:51:09.106 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.107 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.109 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.110 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.111 DEBUG:boto:Method: GET 2022-09-03T06:51:09.111 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:09.113 DEBUG:boto:Data: 2022-09-03T06:51:09.114 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.115 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.116 DEBUG:boto:Params: {} 2022-09-03T06:51:09.116 DEBUG:boto:Token: None 2022-09-03T06:51:09.117 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:09.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9u6g0vNduG00Dz4YU/Wrs8zTemk= 2022-09-03T06:51:09.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9u6g0vNduG00Dz4YU/Wrs8zTemk=', 'Content-Length': '0'} 2022-09-03T06:51:09.121 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000036ecc5eb09bb3fc-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.123 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.124 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.124 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.125 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:09.125 DEBUG:boto:Method: GET 2022-09-03T06:51:09.126 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:09.126 DEBUG:boto:Data: 2022-09-03T06:51:09.127 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.128 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.129 DEBUG:boto:Params: {} 2022-09-03T06:51:09.129 DEBUG:boto:Token: None 2022-09-03T06:51:09.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:09.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9u6g0vNduG00Dz4YU/Wrs8zTemk= 2022-09-03T06:51:09.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9u6g0vNduG00Dz4YU/Wrs8zTemk=', 'Content-Length': '0'} 2022-09-03T06:51:09.135 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e4cf7defd16fb290-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.136 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.137 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.138 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:09.138 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:09.139 DEBUG:boto:Method: GET 2022-09-03T06:51:09.140 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:09.140 DEBUG:boto:Data: 2022-09-03T06:51:09.141 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.141 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.142 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.142 DEBUG:boto:Params: {} 2022-09-03T06:51:09.143 DEBUG:boto:Token: None 2022-09-03T06:51:09.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.146 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.149 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000572462f34f2894de-006312f95d-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.150 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.151 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.151 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.152 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:09.153 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:09.153 DEBUG:boto:Method: GET 2022-09-03T06:51:09.154 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:09.154 DEBUG:boto:Data: 2022-09-03T06:51:09.155 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.156 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.158 DEBUG:boto:Params: {} 2022-09-03T06:51:09.160 DEBUG:boto:Token: None 2022-09-03T06:51:09.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.170 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a9c653c3905a31f-006312f95d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.171 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.172 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.173 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.174 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.174 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:09.175 DEBUG:boto:Data: 2022-09-03T06:51:09.175 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.176 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.177 DEBUG:boto:Params: {} 2022-09-03T06:51:09.177 DEBUG:boto:Token: None 2022-09-03T06:51:09.178 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b 2022-09-03T06:51:09.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HKOr1OE656KASeCOHI11wagLjEo= 2022-09-03T06:51:09.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HKOr1OE656KASeCOHI11wagLjEo=', 'Content-Length': '0'} 2022-09-03T06:51:09.182 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000541d89edee2005ac-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.183 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.183 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.184 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.184 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:09.185 DEBUG:boto:Data: 2022-09-03T06:51:09.185 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.186 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.186 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.187 DEBUG:boto:Params: {} 2022-09-03T06:51:09.188 DEBUG:boto:Token: None 2022-09-03T06:51:09.188 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b 2022-09-03T06:51:09.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HKOr1OE656KASeCOHI11wagLjEo= 2022-09-03T06:51:09.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HKOr1OE656KASeCOHI11wagLjEo=', 'Content-Length': '0'} 2022-09-03T06:51:09.192 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000218f1474ead42ce4-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.193 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:51:09.193 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.194 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.195 DEBUG:boto:Method: GET 2022-09-03T06:51:09.196 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:09.196 DEBUG:boto:Data: 2022-09-03T06:51:09.197 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.198 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.199 DEBUG:boto:Params: {} 2022-09-03T06:51:09.199 DEBUG:boto:Token: None 2022-09-03T06:51:09.200 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b 2022-09-03T06:51:09.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8eaQXmR6yz1VGNaVfEtn2iMxsXo= 2022-09-03T06:51:09.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8eaQXmR6yz1VGNaVfEtn2iMxsXo=', 'Content-Length': '0'} 2022-09-03T06:51:09.204 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3b0140f6b2c2e6c-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.205 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:09.205 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:09.206 DEBUG:boto:Method: GET 2022-09-03T06:51:09.207 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:09.207 DEBUG:boto:Data: 2022-09-03T06:51:09.208 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.208 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.209 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.210 DEBUG:boto:Params: {} 2022-09-03T06:51:09.211 DEBUG:boto:Token: None 2022-09-03T06:51:09.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/b 2022-09-03T06:51:09.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8eaQXmR6yz1VGNaVfEtn2iMxsXo= 2022-09-03T06:51:09.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8eaQXmR6yz1VGNaVfEtn2iMxsXo=', 'Content-Length': '0'} 2022-09-03T06:51:09.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f820e9949127f7e-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.259 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.260 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.261 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:09.270 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:09.271 DEBUG:boto:Method: GET 2022-09-03T06:51:09.272 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:09.272 DEBUG:boto:Data: 2022-09-03T06:51:09.273 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.274 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.275 DEBUG:boto:Params: {} 2022-09-03T06:51:09.275 DEBUG:boto:Token: None 2022-09-03T06:51:09.276 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.281 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001eab8998a4ff7256-006312f95d-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.282 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.283 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.284 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.285 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:09.286 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:09.287 DEBUG:boto:Method: GET 2022-09-03T06:51:09.287 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:09.288 DEBUG:boto:Data: 2022-09-03T06:51:09.288 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.289 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.290 DEBUG:boto:Params: {} 2022-09-03T06:51:09.290 DEBUG:boto:Token: None 2022-09-03T06:51:09.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.296 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d60d2a359c6b0003-006312f95d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.298 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.300 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:51:09.301 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.302 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.303 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.304 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.305 DEBUG:boto:Method: GET 2022-09-03T06:51:09.307 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:09.308 DEBUG:boto:Data: 2022-09-03T06:51:09.308 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.310 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.311 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.312 DEBUG:boto:Params: {} 2022-09-03T06:51:09.313 DEBUG:boto:Token: None 2022-09-03T06:51:09.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:09.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Yz/cYwkJt4chccfYoRC69E/u5E= 2022-09-03T06:51:09.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Yz/cYwkJt4chccfYoRC69E/u5E=', 'Content-Length': '0'} 2022-09-03T06:51:09.327 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000028cf2e0cd0746244-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.328 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.328 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.329 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.329 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.330 DEBUG:boto:Method: GET 2022-09-03T06:51:09.331 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:09.331 DEBUG:boto:Data: 2022-09-03T06:51:09.332 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.333 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.334 DEBUG:boto:Params: {} 2022-09-03T06:51:09.335 DEBUG:boto:Token: None 2022-09-03T06:51:09.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:09.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Yz/cYwkJt4chccfYoRC69E/u5E= 2022-09-03T06:51:09.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Yz/cYwkJt4chccfYoRC69E/u5E=', 'Content-Length': '0'} 2022-09-03T06:51:09.340 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ad7baf50708bd2f-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.341 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.341 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.342 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.342 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.343 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.344 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:09.344 DEBUG:boto:Data: 2022-09-03T06:51:09.345 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.346 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.347 DEBUG:boto:Params: {} 2022-09-03T06:51:09.347 DEBUG:boto:Token: None 2022-09-03T06:51:09.348 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:09.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wbXkrluL2gzuqPkcezPoxHTbJpc= 2022-09-03T06:51:09.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wbXkrluL2gzuqPkcezPoxHTbJpc=', 'Content-Length': '0'} 2022-09-03T06:51:09.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060d520300d158971-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.352 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.353 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.353 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.354 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.354 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.355 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:09.355 DEBUG:boto:Data: 2022-09-03T06:51:09.356 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.357 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.357 DEBUG:boto:Params: {} 2022-09-03T06:51:09.358 DEBUG:boto:Token: None 2022-09-03T06:51:09.359 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:09.359 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wbXkrluL2gzuqPkcezPoxHTbJpc= 2022-09-03T06:51:09.360 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wbXkrluL2gzuqPkcezPoxHTbJpc=', 'Content-Length': '0'} 2022-09-03T06:51:09.362 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f51ea2f588428907-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.363 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:51:09.364 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.364 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.365 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.365 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.366 DEBUG:boto:Method: GET 2022-09-03T06:51:09.367 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:09.367 DEBUG:boto:Data: 2022-09-03T06:51:09.368 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.368 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.369 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.369 DEBUG:boto:Params: {} 2022-09-03T06:51:09.370 DEBUG:boto:Token: None 2022-09-03T06:51:09.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:09.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Yz/cYwkJt4chccfYoRC69E/u5E= 2022-09-03T06:51:09.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Yz/cYwkJt4chccfYoRC69E/u5E=', 'Content-Length': '0'} 2022-09-03T06:51:09.375 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de98e36414919d5e-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.418 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.419 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.419 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.420 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:09.421 DEBUG:boto:Method: GET 2022-09-03T06:51:09.421 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:09.422 DEBUG:boto:Data: 2022-09-03T06:51:09.422 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.423 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.424 DEBUG:boto:Params: {} 2022-09-03T06:51:09.425 DEBUG:boto:Token: None 2022-09-03T06:51:09.425 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:09.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Yz/cYwkJt4chccfYoRC69E/u5E= 2022-09-03T06:51:09.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Yz/cYwkJt4chccfYoRC69E/u5E=', 'Content-Length': '0'} 2022-09-03T06:51:09.430 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e72c5a548529c49-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.431 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.431 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.432 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:09.432 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:09.433 DEBUG:boto:Method: GET 2022-09-03T06:51:09.433 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:09.434 DEBUG:boto:Data: 2022-09-03T06:51:09.435 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.435 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.436 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.436 DEBUG:boto:Params: {} 2022-09-03T06:51:09.437 DEBUG:boto:Token: None 2022-09-03T06:51:09.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.442 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000059809b31ae98d4db-006312f95d-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.443 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.444 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.444 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.445 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:09.446 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:09.447 DEBUG:boto:Method: GET 2022-09-03T06:51:09.447 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:09.449 DEBUG:boto:Data: 2022-09-03T06:51:09.450 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.451 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.452 DEBUG:boto:Params: {} 2022-09-03T06:51:09.452 DEBUG:boto:Token: None 2022-09-03T06:51:09.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.457 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006d301f5f2ff866b3-006312f95d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.458 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.460 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.460 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.461 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.462 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:09.463 DEBUG:boto:Data: 2022-09-03T06:51:09.463 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.464 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.464 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.465 DEBUG:boto:Params: {} 2022-09-03T06:51:09.466 DEBUG:boto:Token: None 2022-09-03T06:51:09.467 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c 2022-09-03T06:51:09.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hy6UkjjC57ZTW3Q+I30HsJHiTrU= 2022-09-03T06:51:09.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hy6UkjjC57ZTW3Q+I30HsJHiTrU=', 'Content-Length': '0'} 2022-09-03T06:51:09.470 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfefe789ded7f1ab-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.471 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.471 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.472 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.473 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:09.473 DEBUG:boto:Data: 2022-09-03T06:51:09.474 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.475 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.476 DEBUG:boto:Params: {} 2022-09-03T06:51:09.476 DEBUG:boto:Token: None 2022-09-03T06:51:09.477 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c 2022-09-03T06:51:09.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hy6UkjjC57ZTW3Q+I30HsJHiTrU= 2022-09-03T06:51:09.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hy6UkjjC57ZTW3Q+I30HsJHiTrU=', 'Content-Length': '0'} 2022-09-03T06:51:09.480 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d1b7ead83c5a13b0-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.481 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:51:09.481 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.482 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.482 DEBUG:boto:Method: GET 2022-09-03T06:51:09.483 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:09.483 DEBUG:boto:Data: 2022-09-03T06:51:09.484 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.486 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.486 DEBUG:boto:Params: {} 2022-09-03T06:51:09.487 DEBUG:boto:Token: None 2022-09-03T06:51:09.488 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c 2022-09-03T06:51:09.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NrqLPZD+GpjvI/4b48GPKDqSJoE= 2022-09-03T06:51:09.489 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NrqLPZD+GpjvI/4b48GPKDqSJoE=', 'Content-Length': '0'} 2022-09-03T06:51:09.491 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005006661f681cfc49-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.492 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:09.493 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:09.493 DEBUG:boto:Method: GET 2022-09-03T06:51:09.494 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:09.494 DEBUG:boto:Data: 2022-09-03T06:51:09.495 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.496 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.497 DEBUG:boto:Params: {} 2022-09-03T06:51:09.497 DEBUG:boto:Token: None 2022-09-03T06:51:09.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/c 2022-09-03T06:51:09.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NrqLPZD+GpjvI/4b48GPKDqSJoE= 2022-09-03T06:51:09.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NrqLPZD+GpjvI/4b48GPKDqSJoE=', 'Content-Length': '0'} 2022-09-03T06:51:09.501 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001966f11b0522de82-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.502 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.503 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.503 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:09.504 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:09.505 DEBUG:boto:Method: GET 2022-09-03T06:51:09.505 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:09.506 DEBUG:boto:Data: 2022-09-03T06:51:09.506 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.507 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.507 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.508 DEBUG:boto:Params: {} 2022-09-03T06:51:09.509 DEBUG:boto:Token: None 2022-09-03T06:51:09.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.513 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a80c5497e2dcf580-006312f95d-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.514 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.515 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.516 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.517 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:09.517 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:09.518 DEBUG:boto:Method: GET 2022-09-03T06:51:09.519 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:09.519 DEBUG:boto:Data: 2022-09-03T06:51:09.520 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.521 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.521 DEBUG:boto:Params: {} 2022-09-03T06:51:09.523 DEBUG:boto:Token: None 2022-09-03T06:51:09.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.525 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.536 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d1143685c7ca73b-006312f95d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.537 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.539 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:51:09.540 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.541 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.542 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.543 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.544 DEBUG:boto:Method: GET 2022-09-03T06:51:09.545 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:09.546 DEBUG:boto:Data: 2022-09-03T06:51:09.547 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.548 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.549 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.553 DEBUG:boto:Params: {} 2022-09-03T06:51:09.554 DEBUG:boto:Token: None 2022-09-03T06:51:09.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:09.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPW5Bw3/RBee84nP7xixCW9rWd0= 2022-09-03T06:51:09.557 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPW5Bw3/RBee84nP7xixCW9rWd0=', 'Content-Length': '0'} 2022-09-03T06:51:09.560 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f06672dbea8cbf5-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.562 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.563 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.564 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.565 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.567 DEBUG:boto:Method: GET 2022-09-03T06:51:09.568 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:09.569 DEBUG:boto:Data: 2022-09-03T06:51:09.570 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.571 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.572 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.573 DEBUG:boto:Params: {} 2022-09-03T06:51:09.574 DEBUG:boto:Token: None 2022-09-03T06:51:09.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:09.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPW5Bw3/RBee84nP7xixCW9rWd0= 2022-09-03T06:51:09.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPW5Bw3/RBee84nP7xixCW9rWd0=', 'Content-Length': '0'} 2022-09-03T06:51:09.586 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d69b9165a4626475-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.588 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.589 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.590 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.590 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.591 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.592 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:09.593 DEBUG:boto:Data: 2022-09-03T06:51:09.594 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.595 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.596 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.597 DEBUG:boto:Params: {} 2022-09-03T06:51:09.598 DEBUG:boto:Token: None 2022-09-03T06:51:09.599 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:09.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1hSdbtNnleofIwZ0ycO2ndHsYsM= 2022-09-03T06:51:09.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1hSdbtNnleofIwZ0ycO2ndHsYsM=', 'Content-Length': '0'} 2022-09-03T06:51:09.605 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000775886d4cc97bb3b-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.606 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.607 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.608 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.609 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.610 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.611 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:09.612 DEBUG:boto:Data: 2022-09-03T06:51:09.613 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.615 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.617 DEBUG:boto:Params: {} 2022-09-03T06:51:09.618 DEBUG:boto:Token: None 2022-09-03T06:51:09.619 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:09.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1hSdbtNnleofIwZ0ycO2ndHsYsM= 2022-09-03T06:51:09.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1hSdbtNnleofIwZ0ycO2ndHsYsM=', 'Content-Length': '0'} 2022-09-03T06:51:09.625 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b89763de53e08459-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.627 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:51:09.627 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.629 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.631 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.632 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.633 DEBUG:boto:Method: GET 2022-09-03T06:51:09.635 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:09.636 DEBUG:boto:Data: 2022-09-03T06:51:09.637 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.638 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.639 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.640 DEBUG:boto:Params: {} 2022-09-03T06:51:09.642 DEBUG:boto:Token: None 2022-09-03T06:51:09.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:09.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPW5Bw3/RBee84nP7xixCW9rWd0= 2022-09-03T06:51:09.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPW5Bw3/RBee84nP7xixCW9rWd0=', 'Content-Length': '0'} 2022-09-03T06:51:09.655 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c043357077c91812-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.659 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.662 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.663 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.663 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:09.664 DEBUG:boto:Method: GET 2022-09-03T06:51:09.665 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:09.666 DEBUG:boto:Data: 2022-09-03T06:51:09.666 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.667 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.667 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.668 DEBUG:boto:Params: {} 2022-09-03T06:51:09.669 DEBUG:boto:Token: None 2022-09-03T06:51:09.670 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:09.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPW5Bw3/RBee84nP7xixCW9rWd0= 2022-09-03T06:51:09.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPW5Bw3/RBee84nP7xixCW9rWd0=', 'Content-Length': '0'} 2022-09-03T06:51:09.689 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8626a3ee8d461a7-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.696 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.697 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.698 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:09.698 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:09.699 DEBUG:boto:Method: GET 2022-09-03T06:51:09.701 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:09.702 DEBUG:boto:Data: 2022-09-03T06:51:09.703 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.706 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.707 DEBUG:boto:Params: {} 2022-09-03T06:51:09.708 DEBUG:boto:Token: None 2022-09-03T06:51:09.710 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.724 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ddee45af948ca902-006312f95d-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.725 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.734 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.734 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.735 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:09.735 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:09.736 DEBUG:boto:Method: GET 2022-09-03T06:51:09.737 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:09.737 DEBUG:boto:Data: 2022-09-03T06:51:09.738 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.739 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.740 DEBUG:boto:Params: {} 2022-09-03T06:51:09.740 DEBUG:boto:Token: None 2022-09-03T06:51:09.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.748 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ef7351473a14406-006312f95d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.751 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.754 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.755 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.756 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.757 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:09.758 DEBUG:boto:Data: 2022-09-03T06:51:09.759 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.760 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.761 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.762 DEBUG:boto:Params: {} 2022-09-03T06:51:09.763 DEBUG:boto:Token: None 2022-09-03T06:51:09.766 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d 2022-09-03T06:51:09.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oQ+1o7jdtotNmpffKQoe7BPlPTM= 2022-09-03T06:51:09.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oQ+1o7jdtotNmpffKQoe7BPlPTM=', 'Content-Length': '0'} 2022-09-03T06:51:09.782 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f2b6be2fac31cb5-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.787 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.788 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.789 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.789 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:09.790 DEBUG:boto:Data: 2022-09-03T06:51:09.792 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.807 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.808 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.809 DEBUG:boto:Params: {} 2022-09-03T06:51:09.812 DEBUG:boto:Token: None 2022-09-03T06:51:09.813 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d 2022-09-03T06:51:09.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oQ+1o7jdtotNmpffKQoe7BPlPTM= 2022-09-03T06:51:09.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oQ+1o7jdtotNmpffKQoe7BPlPTM=', 'Content-Length': '0'} 2022-09-03T06:51:09.840 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9f1021713c03228-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.841 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:51:09.842 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.843 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.844 DEBUG:boto:Method: GET 2022-09-03T06:51:09.844 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:09.845 DEBUG:boto:Data: 2022-09-03T06:51:09.845 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.846 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.847 DEBUG:boto:Params: {} 2022-09-03T06:51:09.848 DEBUG:boto:Token: None 2022-09-03T06:51:09.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d 2022-09-03T06:51:09.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1T61+dMNtlfdg1UiP++nX3L7NQI= 2022-09-03T06:51:09.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1T61+dMNtlfdg1UiP++nX3L7NQI=', 'Content-Length': '0'} 2022-09-03T06:51:09.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000034dd29b474476d7-006312f95d-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.863 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:09.864 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:09.864 DEBUG:boto:Method: GET 2022-09-03T06:51:09.865 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:09.865 DEBUG:boto:Data: 2022-09-03T06:51:09.866 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.867 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.867 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.868 DEBUG:boto:Params: {} 2022-09-03T06:51:09.868 DEBUG:boto:Token: None 2022-09-03T06:51:09.869 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/d 2022-09-03T06:51:09.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1T61+dMNtlfdg1UiP++nX3L7NQI= 2022-09-03T06:51:09.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1T61+dMNtlfdg1UiP++nX3L7NQI=', 'Content-Length': '0'} 2022-09-03T06:51:09.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000436b4dcfe5d6f83a-006312f95d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.880 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.882 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.886 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:09.888 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:09.889 DEBUG:boto:Method: GET 2022-09-03T06:51:09.890 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:09.891 DEBUG:boto:Data: 2022-09-03T06:51:09.892 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.894 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.895 DEBUG:boto:Params: {} 2022-09-03T06:51:09.898 DEBUG:boto:Token: None 2022-09-03T06:51:09.899 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.907 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000054e736035b7ac532-006312f95d-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.908 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.909 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.910 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.911 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:09.911 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:09.912 DEBUG:boto:Method: GET 2022-09-03T06:51:09.912 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:09.913 DEBUG:boto:Data: 2022-09-03T06:51:09.914 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:09.915 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.916 DEBUG:boto:Params: {} 2022-09-03T06:51:09.917 DEBUG:boto:Token: None 2022-09-03T06:51:09.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak= 2022-09-03T06:51:09.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+875Qcv/4JrHIPtQ+HI+zhmP2ak=', 'Content-Length': '0'} 2022-09-03T06:51:09.922 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cc4f8020bd8f0cf3-006312f95d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.922 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:09.923 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-70 zones={test-zone3, test-zone1} 2022-09-03T06:51:09.925 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-70 zones={test-zone3, test-zone2} 2022-09-03T06:51:09.926 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.927 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.927 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.928 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:51:09.929 DEBUG:boto:Data: 2022-09-03T06:51:09.930 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.931 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.932 DEBUG:boto:Params: {} 2022-09-03T06:51:09.933 DEBUG:boto:Token: None 2022-09-03T06:51:09.933 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6SPDq0sofHpTG2qmChCNvLTEyqY= 2022-09-03T06:51:09.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6SPDq0sofHpTG2qmChCNvLTEyqY=', 'Content-Length': '0'} 2022-09-03T06:51:09.939 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000044071a874ca20c65-006312f95d-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.942 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.943 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.945 DEBUG:boto:Method: HEAD 2022-09-03T06:51:09.946 DEBUG:boto:Path: /avysmw-70/ 2022-09-03T06:51:09.947 DEBUG:boto:Data: 2022-09-03T06:51:09.948 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.950 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:09.952 DEBUG:boto:Port: 8000 2022-09-03T06:51:09.953 DEBUG:boto:Params: {} 2022-09-03T06:51:09.954 DEBUG:boto:Token: None 2022-09-03T06:51:09.956 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/ 2022-09-03T06:51:09.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6SPDq0sofHpTG2qmChCNvLTEyqY= 2022-09-03T06:51:09.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6SPDq0sofHpTG2qmChCNvLTEyqY=', 'Content-Length': '0'} 2022-09-03T06:51:09.969 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000023cdabb49851a7b2-006312f95d-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:09.971 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:51:09.972 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:09.973 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:09.974 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:09.978 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:09.979 DEBUG:boto:Method: GET 2022-09-03T06:51:09.982 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:09.983 DEBUG:boto:Data: 2022-09-03T06:51:09.984 DEBUG:boto:Headers: {} 2022-09-03T06:51:09.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:09.987 DEBUG:boto:Port: 8001 2022-09-03T06:51:09.990 DEBUG:boto:Params: {} 2022-09-03T06:51:09.992 DEBUG:boto:Token: None 2022-09-03T06:51:09.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:09 GMT /avysmw-70/?versions 2022-09-03T06:51:09.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZYnncW7aAnUCkSIDAjVy0ON51zs= 2022-09-03T06:51:09.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZYnncW7aAnUCkSIDAjVy0ON51zs=', 'Content-Length': '0'} 2022-09-03T06:51:10.008 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8f61e20a8930793-006312f95e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.010 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.011 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:51:10.012 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:51:10.013 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:51:10.014 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:51:10.015 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:51:10.016 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.017 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.018 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:10.019 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:10.020 DEBUG:boto:Method: GET 2022-09-03T06:51:10.021 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:10.022 DEBUG:boto:Data: 2022-09-03T06:51:10.022 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.023 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.024 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.026 DEBUG:boto:Params: {} 2022-09-03T06:51:10.027 DEBUG:boto:Token: None 2022-09-03T06:51:10.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/?versions 2022-09-03T06:51:10.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p6/sG8pXkukLOac2VOjAKAkZ80o= 2022-09-03T06:51:10.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p6/sG8pXkukLOac2VOjAKAkZ80o=', 'Content-Length': '0'} 2022-09-03T06:51:10.037 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c98a9f23b18489fa-006312f95e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.039 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.040 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:51:10.041 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:51:10.042 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:51:10.043 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:51:10.044 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.045 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.046 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:10.047 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:10.048 DEBUG:boto:Method: GET 2022-09-03T06:51:10.048 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:10.049 DEBUG:boto:Data: 2022-09-03T06:51:10.050 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.051 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.052 DEBUG:boto:Params: {} 2022-09-03T06:51:10.053 DEBUG:boto:Token: None 2022-09-03T06:51:10.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/?versions 2022-09-03T06:51:10.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p6/sG8pXkukLOac2VOjAKAkZ80o= 2022-09-03T06:51:10.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p6/sG8pXkukLOac2VOjAKAkZ80o=', 'Content-Length': '0'} 2022-09-03T06:51:10.061 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000047ed6aadc838ef06-006312f95e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.067 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.069 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.071 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.073 DEBUG:boto:path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:10.074 DEBUG:boto:auth_path=/avysmw-70/?versions&max-keys=999 2022-09-03T06:51:10.074 DEBUG:boto:Method: GET 2022-09-03T06:51:10.075 DEBUG:boto:Path: /avysmw-70/?versions&max-keys=999 2022-09-03T06:51:10.076 DEBUG:boto:Data: 2022-09-03T06:51:10.077 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.077 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.078 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.079 DEBUG:boto:Params: {} 2022-09-03T06:51:10.079 DEBUG:boto:Token: None 2022-09-03T06:51:10.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/?versions 2022-09-03T06:51:10.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p6/sG8pXkukLOac2VOjAKAkZ80o= 2022-09-03T06:51:10.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p6/sG8pXkukLOac2VOjAKAkZ80o=', 'Content-Length': '0'} 2022-09-03T06:51:10.088 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005fbd3c29027e094c-006312f95e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.090 DEBUG:boto:b'avysmw-70999falseanulltrue2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.092 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:51:10.098 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.099 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.100 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.101 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.102 DEBUG:boto:Method: GET 2022-09-03T06:51:10.102 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:10.103 DEBUG:boto:Data: 2022-09-03T06:51:10.104 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.106 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.107 DEBUG:boto:Params: {} 2022-09-03T06:51:10.109 DEBUG:boto:Token: None 2022-09-03T06:51:10.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:10.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q24wP7Np81CB9wiGJlU9WZL8ktg= 2022-09-03T06:51:10.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q24wP7Np81CB9wiGJlU9WZL8ktg=', 'Content-Length': '0'} 2022-09-03T06:51:10.115 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b312dca66a5fc080-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.116 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.116 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.117 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.117 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.119 DEBUG:boto:Method: GET 2022-09-03T06:51:10.120 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:10.121 DEBUG:boto:Data: 2022-09-03T06:51:10.122 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.124 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.126 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.127 DEBUG:boto:Params: {} 2022-09-03T06:51:10.129 DEBUG:boto:Token: None 2022-09-03T06:51:10.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:10.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q24wP7Np81CB9wiGJlU9WZL8ktg= 2022-09-03T06:51:10.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q24wP7Np81CB9wiGJlU9WZL8ktg=', 'Content-Length': '0'} 2022-09-03T06:51:10.144 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000438eec3c2a34493c-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.146 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.147 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.148 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.149 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.150 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.151 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:10.152 DEBUG:boto:Data: 2022-09-03T06:51:10.153 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.156 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.157 DEBUG:boto:Params: {} 2022-09-03T06:51:10.158 DEBUG:boto:Token: None 2022-09-03T06:51:10.159 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:10.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1XChj8WDKVtn7gbzH6qd3v6gihI= 2022-09-03T06:51:10.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1XChj8WDKVtn7gbzH6qd3v6gihI=', 'Content-Length': '0'} 2022-09-03T06:51:10.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a321f66608ea2001-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.165 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.166 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.166 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.167 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.168 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.169 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:10.169 DEBUG:boto:Data: 2022-09-03T06:51:10.170 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.171 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.172 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.173 DEBUG:boto:Params: {} 2022-09-03T06:51:10.175 DEBUG:boto:Token: None 2022-09-03T06:51:10.176 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:10.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1XChj8WDKVtn7gbzH6qd3v6gihI= 2022-09-03T06:51:10.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1XChj8WDKVtn7gbzH6qd3v6gihI=', 'Content-Length': '0'} 2022-09-03T06:51:10.183 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c1fbeba3fc7a471-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.186 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:51:10.187 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.188 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.189 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.190 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.191 DEBUG:boto:Method: GET 2022-09-03T06:51:10.192 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:10.193 DEBUG:boto:Data: 2022-09-03T06:51:10.194 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.195 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.197 DEBUG:boto:Params: {} 2022-09-03T06:51:10.198 DEBUG:boto:Token: None 2022-09-03T06:51:10.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:10.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q24wP7Np81CB9wiGJlU9WZL8ktg= 2022-09-03T06:51:10.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q24wP7Np81CB9wiGJlU9WZL8ktg=', 'Content-Length': '0'} 2022-09-03T06:51:10.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001403b2732d62c46b-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.210 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.211 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.212 DEBUG:boto:path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.213 DEBUG:boto:auth_path=/avysmw-70/a?versionId=null 2022-09-03T06:51:10.214 DEBUG:boto:Method: GET 2022-09-03T06:51:10.215 DEBUG:boto:Path: /avysmw-70/a?versionId=null 2022-09-03T06:51:10.216 DEBUG:boto:Data: 2022-09-03T06:51:10.217 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.218 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.219 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.220 DEBUG:boto:Params: {} 2022-09-03T06:51:10.221 DEBUG:boto:Token: None 2022-09-03T06:51:10.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a?versionId=null 2022-09-03T06:51:10.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q24wP7Np81CB9wiGJlU9WZL8ktg= 2022-09-03T06:51:10.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q24wP7Np81CB9wiGJlU9WZL8ktg=', 'Content-Length': '0'} 2022-09-03T06:51:10.227 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a613e75231228b3e-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.229 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.230 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.231 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:10.232 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:10.233 DEBUG:boto:Method: GET 2022-09-03T06:51:10.234 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:10.235 DEBUG:boto:Data: 2022-09-03T06:51:10.236 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.238 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.239 DEBUG:boto:Params: {} 2022-09-03T06:51:10.240 DEBUG:boto:Token: None 2022-09-03T06:51:10.241 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.247 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000303c5f75d64daee2-006312f95e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.250 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.251 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.252 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.253 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:10.253 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:10.254 DEBUG:boto:Method: GET 2022-09-03T06:51:10.255 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:10.256 DEBUG:boto:Data: 2022-09-03T06:51:10.256 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.257 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.257 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.258 DEBUG:boto:Params: {} 2022-09-03T06:51:10.259 DEBUG:boto:Token: None 2022-09-03T06:51:10.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.265 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b55c74c8bc881c5-006312f95e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.266 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.268 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.269 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.269 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.270 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:10.271 DEBUG:boto:Data: 2022-09-03T06:51:10.272 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.274 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.275 DEBUG:boto:Params: {} 2022-09-03T06:51:10.275 DEBUG:boto:Token: None 2022-09-03T06:51:10.276 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a 2022-09-03T06:51:10.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OqTdxEFddeGhy+AW8M0q/HDgOsI= 2022-09-03T06:51:10.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OqTdxEFddeGhy+AW8M0q/HDgOsI=', 'Content-Length': '0'} 2022-09-03T06:51:10.284 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d7523ee7540d4e6-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.285 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.286 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.287 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.288 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:10.289 DEBUG:boto:Data: 2022-09-03T06:51:10.290 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.292 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.293 DEBUG:boto:Params: {} 2022-09-03T06:51:10.294 DEBUG:boto:Token: None 2022-09-03T06:51:10.295 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a 2022-09-03T06:51:10.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OqTdxEFddeGhy+AW8M0q/HDgOsI= 2022-09-03T06:51:10.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OqTdxEFddeGhy+AW8M0q/HDgOsI=', 'Content-Length': '0'} 2022-09-03T06:51:10.301 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b53326178d4d25c-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.302 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:51:10.303 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.304 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.305 DEBUG:boto:Method: GET 2022-09-03T06:51:10.306 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:10.307 DEBUG:boto:Data: 2022-09-03T06:51:10.308 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.310 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.311 DEBUG:boto:Params: {} 2022-09-03T06:51:10.312 DEBUG:boto:Token: None 2022-09-03T06:51:10.313 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a 2022-09-03T06:51:10.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9+Qbnq5vna+kUna9tOCs47fOTiQ= 2022-09-03T06:51:10.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9+Qbnq5vna+kUna9tOCs47fOTiQ=', 'Content-Length': '0'} 2022-09-03T06:51:10.318 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cbefb00e1a636fb8-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.320 DEBUG:boto:path=/avysmw-70/a 2022-09-03T06:51:10.321 DEBUG:boto:auth_path=/avysmw-70/a 2022-09-03T06:51:10.322 DEBUG:boto:Method: GET 2022-09-03T06:51:10.323 DEBUG:boto:Path: /avysmw-70/a 2022-09-03T06:51:10.324 DEBUG:boto:Data: 2022-09-03T06:51:10.325 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.326 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.327 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.328 DEBUG:boto:Params: {} 2022-09-03T06:51:10.329 DEBUG:boto:Token: None 2022-09-03T06:51:10.330 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/a 2022-09-03T06:51:10.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9+Qbnq5vna+kUna9tOCs47fOTiQ= 2022-09-03T06:51:10.332 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9+Qbnq5vna+kUna9tOCs47fOTiQ=', 'Content-Length': '0'} 2022-09-03T06:51:10.335 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098a0cb6be6957951-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.337 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.338 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.339 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:10.340 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:10.341 DEBUG:boto:Method: GET 2022-09-03T06:51:10.342 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:10.344 DEBUG:boto:Data: 2022-09-03T06:51:10.345 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.346 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.347 DEBUG:boto:Params: {} 2022-09-03T06:51:10.348 DEBUG:boto:Token: None 2022-09-03T06:51:10.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.356 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002edfea8421bc0988-006312f95e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.403 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.404 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.404 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.405 DEBUG:boto:path=/avysmw-70/?prefix=a 2022-09-03T06:51:10.406 DEBUG:boto:auth_path=/avysmw-70/?prefix=a 2022-09-03T06:51:10.406 DEBUG:boto:Method: GET 2022-09-03T06:51:10.407 DEBUG:boto:Path: /avysmw-70/?prefix=a 2022-09-03T06:51:10.407 DEBUG:boto:Data: 2022-09-03T06:51:10.408 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.409 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.409 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.410 DEBUG:boto:Params: {} 2022-09-03T06:51:10.411 DEBUG:boto:Token: None 2022-09-03T06:51:10.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.417 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f0d9c5d50f46336-006312f95e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.419 DEBUG:boto:b'avysmw-70a1000falsea2022-09-03T06:48:10.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.420 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:51:10.420 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.421 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.421 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.422 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.423 DEBUG:boto:Method: GET 2022-09-03T06:51:10.423 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:10.424 DEBUG:boto:Data: 2022-09-03T06:51:10.425 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.426 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.427 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.427 DEBUG:boto:Params: {} 2022-09-03T06:51:10.428 DEBUG:boto:Token: None 2022-09-03T06:51:10.429 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:10.429 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMtfo9aUeRBJyDkG02SmnSRgX0U= 2022-09-03T06:51:10.430 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMtfo9aUeRBJyDkG02SmnSRgX0U=', 'Content-Length': '0'} 2022-09-03T06:51:10.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000655501504a40014e-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.434 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.435 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.435 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.436 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.436 DEBUG:boto:Method: GET 2022-09-03T06:51:10.437 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:10.438 DEBUG:boto:Data: 2022-09-03T06:51:10.439 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.439 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.440 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.440 DEBUG:boto:Params: {} 2022-09-03T06:51:10.441 DEBUG:boto:Token: None 2022-09-03T06:51:10.442 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:10.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMtfo9aUeRBJyDkG02SmnSRgX0U= 2022-09-03T06:51:10.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMtfo9aUeRBJyDkG02SmnSRgX0U=', 'Content-Length': '0'} 2022-09-03T06:51:10.447 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e013c470e34c611-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.448 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.449 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.451 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.451 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.452 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.453 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:10.454 DEBUG:boto:Data: 2022-09-03T06:51:10.455 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.457 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.458 DEBUG:boto:Params: {} 2022-09-03T06:51:10.459 DEBUG:boto:Token: None 2022-09-03T06:51:10.460 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:10.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DGXinplI5O5Zlmyyd3HxJ6KzuvY= 2022-09-03T06:51:10.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DGXinplI5O5Zlmyyd3HxJ6KzuvY=', 'Content-Length': '0'} 2022-09-03T06:51:10.464 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000174f2159baf10b93-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.465 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.466 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.466 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.467 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.467 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.468 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:10.468 DEBUG:boto:Data: 2022-09-03T06:51:10.469 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.470 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.471 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.471 DEBUG:boto:Params: {} 2022-09-03T06:51:10.472 DEBUG:boto:Token: None 2022-09-03T06:51:10.473 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:10.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DGXinplI5O5Zlmyyd3HxJ6KzuvY= 2022-09-03T06:51:10.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DGXinplI5O5Zlmyyd3HxJ6KzuvY=', 'Content-Length': '0'} 2022-09-03T06:51:10.476 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9b52e34c506f3f0-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.477 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:51:10.478 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.479 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.479 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.480 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.480 DEBUG:boto:Method: GET 2022-09-03T06:51:10.481 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:10.482 DEBUG:boto:Data: 2022-09-03T06:51:10.483 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.484 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.485 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.485 DEBUG:boto:Params: {} 2022-09-03T06:51:10.487 DEBUG:boto:Token: None 2022-09-03T06:51:10.488 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:10.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMtfo9aUeRBJyDkG02SmnSRgX0U= 2022-09-03T06:51:10.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMtfo9aUeRBJyDkG02SmnSRgX0U=', 'Content-Length': '0'} 2022-09-03T06:51:10.493 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec21d7cb13390606-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.499 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.502 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.503 DEBUG:boto:path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.505 DEBUG:boto:auth_path=/avysmw-70/b?versionId=null 2022-09-03T06:51:10.506 DEBUG:boto:Method: GET 2022-09-03T06:51:10.507 DEBUG:boto:Path: /avysmw-70/b?versionId=null 2022-09-03T06:51:10.508 DEBUG:boto:Data: 2022-09-03T06:51:10.509 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.509 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.510 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.511 DEBUG:boto:Params: {} 2022-09-03T06:51:10.511 DEBUG:boto:Token: None 2022-09-03T06:51:10.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b?versionId=null 2022-09-03T06:51:10.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QMtfo9aUeRBJyDkG02SmnSRgX0U= 2022-09-03T06:51:10.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QMtfo9aUeRBJyDkG02SmnSRgX0U=', 'Content-Length': '0'} 2022-09-03T06:51:10.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5f0b0b0e200c00a-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.563 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.564 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.565 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:10.566 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:10.566 DEBUG:boto:Method: GET 2022-09-03T06:51:10.568 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:10.569 DEBUG:boto:Data: 2022-09-03T06:51:10.569 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.571 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.571 DEBUG:boto:Params: {} 2022-09-03T06:51:10.572 DEBUG:boto:Token: None 2022-09-03T06:51:10.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.575 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.585 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000096aced920b90c84c-006312f95e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.586 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.587 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.588 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.589 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:10.589 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:10.590 DEBUG:boto:Method: GET 2022-09-03T06:51:10.590 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:10.591 DEBUG:boto:Data: 2022-09-03T06:51:10.592 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.592 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.593 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.594 DEBUG:boto:Params: {} 2022-09-03T06:51:10.595 DEBUG:boto:Token: None 2022-09-03T06:51:10.596 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.603 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002f32358bd5ead22d-006312f95e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.608 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.609 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.610 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.610 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.611 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:10.611 DEBUG:boto:Data: 2022-09-03T06:51:10.612 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.613 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.613 DEBUG:boto:Params: {} 2022-09-03T06:51:10.614 DEBUG:boto:Token: None 2022-09-03T06:51:10.615 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b 2022-09-03T06:51:10.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mtoFL/jdyX+21wa7fx/Nu9802cc= 2022-09-03T06:51:10.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mtoFL/jdyX+21wa7fx/Nu9802cc=', 'Content-Length': '0'} 2022-09-03T06:51:10.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b933bb0301ba4fd-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.620 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.621 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.621 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.622 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:10.622 DEBUG:boto:Data: 2022-09-03T06:51:10.623 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.624 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.625 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.626 DEBUG:boto:Params: {} 2022-09-03T06:51:10.626 DEBUG:boto:Token: None 2022-09-03T06:51:10.627 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b 2022-09-03T06:51:10.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mtoFL/jdyX+21wa7fx/Nu9802cc= 2022-09-03T06:51:10.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mtoFL/jdyX+21wa7fx/Nu9802cc=', 'Content-Length': '0'} 2022-09-03T06:51:10.634 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008aaf90586d9f5ce1-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.635 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:51:10.636 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.637 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.638 DEBUG:boto:Method: GET 2022-09-03T06:51:10.639 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:10.640 DEBUG:boto:Data: 2022-09-03T06:51:10.641 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.643 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.644 DEBUG:boto:Params: {} 2022-09-03T06:51:10.645 DEBUG:boto:Token: None 2022-09-03T06:51:10.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b 2022-09-03T06:51:10.647 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0HzLG6ul2T48OVtHQPLb8BHF7i0= 2022-09-03T06:51:10.648 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0HzLG6ul2T48OVtHQPLb8BHF7i0=', 'Content-Length': '0'} 2022-09-03T06:51:10.650 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000290518ba772142c8-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.652 DEBUG:boto:path=/avysmw-70/b 2022-09-03T06:51:10.653 DEBUG:boto:auth_path=/avysmw-70/b 2022-09-03T06:51:10.654 DEBUG:boto:Method: GET 2022-09-03T06:51:10.655 DEBUG:boto:Path: /avysmw-70/b 2022-09-03T06:51:10.656 DEBUG:boto:Data: 2022-09-03T06:51:10.657 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.658 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.659 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.659 DEBUG:boto:Params: {} 2022-09-03T06:51:10.660 DEBUG:boto:Token: None 2022-09-03T06:51:10.661 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/b 2022-09-03T06:51:10.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0HzLG6ul2T48OVtHQPLb8BHF7i0= 2022-09-03T06:51:10.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0HzLG6ul2T48OVtHQPLb8BHF7i0=', 'Content-Length': '0'} 2022-09-03T06:51:10.664 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000807e7212d8eec030-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.665 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.666 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.666 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:10.667 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:10.667 DEBUG:boto:Method: GET 2022-09-03T06:51:10.668 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:10.669 DEBUG:boto:Data: 2022-09-03T06:51:10.669 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.670 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.671 DEBUG:boto:Params: {} 2022-09-03T06:51:10.671 DEBUG:boto:Token: None 2022-09-03T06:51:10.672 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.677 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076521f3eb9bc06d2-006312f95e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.678 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.679 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.680 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.681 DEBUG:boto:path=/avysmw-70/?prefix=b 2022-09-03T06:51:10.682 DEBUG:boto:auth_path=/avysmw-70/?prefix=b 2022-09-03T06:51:10.683 DEBUG:boto:Method: GET 2022-09-03T06:51:10.683 DEBUG:boto:Path: /avysmw-70/?prefix=b 2022-09-03T06:51:10.684 DEBUG:boto:Data: 2022-09-03T06:51:10.684 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.685 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.685 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.686 DEBUG:boto:Params: {} 2022-09-03T06:51:10.686 DEBUG:boto:Token: None 2022-09-03T06:51:10.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.691 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000113e5bafe4f5bcf-006312f95e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.692 DEBUG:boto:b'avysmw-70b1000falseb2022-09-03T06:48:10.662Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.692 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:51:10.693 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.694 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.694 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.695 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.695 DEBUG:boto:Method: GET 2022-09-03T06:51:10.696 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:10.697 DEBUG:boto:Data: 2022-09-03T06:51:10.697 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.699 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.699 DEBUG:boto:Params: {} 2022-09-03T06:51:10.700 DEBUG:boto:Token: None 2022-09-03T06:51:10.701 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:10.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XGKOZEMx2GU8MNYnWcNA76VXQ5Y= 2022-09-03T06:51:10.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XGKOZEMx2GU8MNYnWcNA76VXQ5Y=', 'Content-Length': '0'} 2022-09-03T06:51:10.706 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000296b4133db5f22d0-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.707 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.708 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.709 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.709 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.710 DEBUG:boto:Method: GET 2022-09-03T06:51:10.711 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:10.712 DEBUG:boto:Data: 2022-09-03T06:51:10.712 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.713 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.714 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.714 DEBUG:boto:Params: {} 2022-09-03T06:51:10.715 DEBUG:boto:Token: None 2022-09-03T06:51:10.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:10.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XGKOZEMx2GU8MNYnWcNA76VXQ5Y= 2022-09-03T06:51:10.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XGKOZEMx2GU8MNYnWcNA76VXQ5Y=', 'Content-Length': '0'} 2022-09-03T06:51:10.719 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000040a32df3afb02bc8-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.771 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.772 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.773 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.773 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.774 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.775 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:10.775 DEBUG:boto:Data: 2022-09-03T06:51:10.776 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.778 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.779 DEBUG:boto:Params: {} 2022-09-03T06:51:10.780 DEBUG:boto:Token: None 2022-09-03T06:51:10.780 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:10.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DuqTOliq3iAgWkL2hbOehyw8L0g= 2022-09-03T06:51:10.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DuqTOliq3iAgWkL2hbOehyw8L0g=', 'Content-Length': '0'} 2022-09-03T06:51:10.795 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dfce53a6c6761954-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.796 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.798 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.798 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.799 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.800 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.801 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:10.802 DEBUG:boto:Data: 2022-09-03T06:51:10.802 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.803 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.804 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.805 DEBUG:boto:Params: {} 2022-09-03T06:51:10.806 DEBUG:boto:Token: None 2022-09-03T06:51:10.807 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:10.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DuqTOliq3iAgWkL2hbOehyw8L0g= 2022-09-03T06:51:10.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DuqTOliq3iAgWkL2hbOehyw8L0g=', 'Content-Length': '0'} 2022-09-03T06:51:10.811 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c7f87ba5f51d0d97-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.812 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:51:10.813 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.814 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.815 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.816 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.817 DEBUG:boto:Method: GET 2022-09-03T06:51:10.818 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:10.819 DEBUG:boto:Data: 2022-09-03T06:51:10.820 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.821 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.822 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.823 DEBUG:boto:Params: {} 2022-09-03T06:51:10.826 DEBUG:boto:Token: None 2022-09-03T06:51:10.827 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:10.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XGKOZEMx2GU8MNYnWcNA76VXQ5Y= 2022-09-03T06:51:10.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XGKOZEMx2GU8MNYnWcNA76VXQ5Y=', 'Content-Length': '0'} 2022-09-03T06:51:10.832 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000067a28dca238d0cef-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.833 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.834 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.835 DEBUG:boto:path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.836 DEBUG:boto:auth_path=/avysmw-70/c?versionId=null 2022-09-03T06:51:10.838 DEBUG:boto:Method: GET 2022-09-03T06:51:10.841 DEBUG:boto:Path: /avysmw-70/c?versionId=null 2022-09-03T06:51:10.841 DEBUG:boto:Data: 2022-09-03T06:51:10.842 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.843 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.843 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.844 DEBUG:boto:Params: {} 2022-09-03T06:51:10.844 DEBUG:boto:Token: None 2022-09-03T06:51:10.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c?versionId=null 2022-09-03T06:51:10.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XGKOZEMx2GU8MNYnWcNA76VXQ5Y= 2022-09-03T06:51:10.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XGKOZEMx2GU8MNYnWcNA76VXQ5Y=', 'Content-Length': '0'} 2022-09-03T06:51:10.849 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008bc8bfe3a23b3d2f-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.850 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.851 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.851 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:10.852 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:10.853 DEBUG:boto:Method: GET 2022-09-03T06:51:10.853 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:10.854 DEBUG:boto:Data: 2022-09-03T06:51:10.854 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.855 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.855 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.856 DEBUG:boto:Params: {} 2022-09-03T06:51:10.856 DEBUG:boto:Token: None 2022-09-03T06:51:10.857 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.863 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a5ac11b4f0cf7472-006312f95e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.864 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.866 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.866 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.867 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:10.868 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:10.869 DEBUG:boto:Method: GET 2022-09-03T06:51:10.870 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:10.871 DEBUG:boto:Data: 2022-09-03T06:51:10.872 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.872 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.873 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.874 DEBUG:boto:Params: {} 2022-09-03T06:51:10.875 DEBUG:boto:Token: None 2022-09-03T06:51:10.876 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.882 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dcb8e270695478e9-006312f95e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.883 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.884 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.884 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.885 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.885 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:10.886 DEBUG:boto:Data: 2022-09-03T06:51:10.886 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.887 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.887 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.888 DEBUG:boto:Params: {} 2022-09-03T06:51:10.888 DEBUG:boto:Token: None 2022-09-03T06:51:10.889 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c 2022-09-03T06:51:10.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:melkhU8CvQdnvgjoV5saAAKskLI= 2022-09-03T06:51:10.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:melkhU8CvQdnvgjoV5saAAKskLI=', 'Content-Length': '0'} 2022-09-03T06:51:10.893 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9790e2f0f42f205-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.894 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.895 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.896 DEBUG:boto:Method: HEAD 2022-09-03T06:51:10.897 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:10.897 DEBUG:boto:Data: 2022-09-03T06:51:10.898 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.900 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.901 DEBUG:boto:Params: {} 2022-09-03T06:51:10.902 DEBUG:boto:Token: None 2022-09-03T06:51:10.903 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c 2022-09-03T06:51:10.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:melkhU8CvQdnvgjoV5saAAKskLI= 2022-09-03T06:51:10.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:melkhU8CvQdnvgjoV5saAAKskLI=', 'Content-Length': '0'} 2022-09-03T06:51:10.908 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dcd43296eb3d57a6-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.909 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:51:10.910 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.911 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.912 DEBUG:boto:Method: GET 2022-09-03T06:51:10.913 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:10.914 DEBUG:boto:Data: 2022-09-03T06:51:10.915 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.916 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.917 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.918 DEBUG:boto:Params: {} 2022-09-03T06:51:10.919 DEBUG:boto:Token: None 2022-09-03T06:51:10.920 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c 2022-09-03T06:51:10.921 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O45bHFW47m/6cMOBh40ZeEXDolk= 2022-09-03T06:51:10.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O45bHFW47m/6cMOBh40ZeEXDolk=', 'Content-Length': '0'} 2022-09-03T06:51:10.925 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076942eaf1abacd75-006312f95e-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.966 DEBUG:boto:path=/avysmw-70/c 2022-09-03T06:51:10.967 DEBUG:boto:auth_path=/avysmw-70/c 2022-09-03T06:51:10.968 DEBUG:boto:Method: GET 2022-09-03T06:51:10.969 DEBUG:boto:Path: /avysmw-70/c 2022-09-03T06:51:10.970 DEBUG:boto:Data: 2022-09-03T06:51:10.971 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.972 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:10.972 DEBUG:boto:Port: 8000 2022-09-03T06:51:10.973 DEBUG:boto:Params: {} 2022-09-03T06:51:10.974 DEBUG:boto:Token: None 2022-09-03T06:51:10.974 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/c 2022-09-03T06:51:10.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O45bHFW47m/6cMOBh40ZeEXDolk= 2022-09-03T06:51:10.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O45bHFW47m/6cMOBh40ZeEXDolk=', 'Content-Length': '0'} 2022-09-03T06:51:10.979 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000539732bf88ae0631-006312f95e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.980 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.980 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.981 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:10.982 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:10.982 DEBUG:boto:Method: GET 2022-09-03T06:51:10.983 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:10.983 DEBUG:boto:Data: 2022-09-03T06:51:10.984 DEBUG:boto:Headers: {} 2022-09-03T06:51:10.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:10.985 DEBUG:boto:Port: 8001 2022-09-03T06:51:10.986 DEBUG:boto:Params: {} 2022-09-03T06:51:10.987 DEBUG:boto:Token: None 2022-09-03T06:51:10.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:10 GMT /avysmw-70/ 2022-09-03T06:51:10.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew= 2022-09-03T06:51:10.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EJZDUUK5qOIAS2YVnE8cT9qhew=', 'Content-Length': '0'} 2022-09-03T06:51:10.992 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ccd1cf8611daf150-006312f95e-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:10.993 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:10.994 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:10.995 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:10.996 DEBUG:boto:path=/avysmw-70/?prefix=c 2022-09-03T06:51:10.998 DEBUG:boto:auth_path=/avysmw-70/?prefix=c 2022-09-03T06:51:10.999 DEBUG:boto:Method: GET 2022-09-03T06:51:11.003 DEBUG:boto:Path: /avysmw-70/?prefix=c 2022-09-03T06:51:11.004 DEBUG:boto:Data: 2022-09-03T06:51:11.005 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.006 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:11.007 DEBUG:boto:Port: 8000 2022-09-03T06:51:11.008 DEBUG:boto:Params: {} 2022-09-03T06:51:11.009 DEBUG:boto:Token: None 2022-09-03T06:51:11.010 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/ 2022-09-03T06:51:11.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w= 2022-09-03T06:51:11.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w=', 'Content-Length': '0'} 2022-09-03T06:51:11.015 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000041b095abd9a3500c-006312f95f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.060 DEBUG:boto:b'avysmw-70c1000falsec2022-09-03T06:48:10.689Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:11.061 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:51:11.063 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.064 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.065 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.066 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.067 DEBUG:boto:Method: GET 2022-09-03T06:51:11.068 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:11.070 DEBUG:boto:Data: 2022-09-03T06:51:11.071 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.072 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:11.072 DEBUG:boto:Port: 8001 2022-09-03T06:51:11.073 DEBUG:boto:Params: {} 2022-09-03T06:51:11.073 DEBUG:boto:Token: None 2022-09-03T06:51:11.074 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:11.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GMqkevySGeQY79cyz1xOVK5I5jU= 2022-09-03T06:51:11.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GMqkevySGeQY79cyz1xOVK5I5jU=', 'Content-Length': '0'} 2022-09-03T06:51:11.078 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d8d641fc51b5f27-006312f95f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.080 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.081 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.083 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.084 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.085 DEBUG:boto:Method: GET 2022-09-03T06:51:11.086 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:11.087 DEBUG:boto:Data: 2022-09-03T06:51:11.088 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.089 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:11.090 DEBUG:boto:Port: 8000 2022-09-03T06:51:11.091 DEBUG:boto:Params: {} 2022-09-03T06:51:11.093 DEBUG:boto:Token: None 2022-09-03T06:51:11.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:11.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GMqkevySGeQY79cyz1xOVK5I5jU= 2022-09-03T06:51:11.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GMqkevySGeQY79cyz1xOVK5I5jU=', 'Content-Length': '0'} 2022-09-03T06:51:11.098 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065434f2d36fa9a1e-006312f95f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.099 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.100 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.101 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.102 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.103 DEBUG:boto:Method: HEAD 2022-09-03T06:51:11.104 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:11.105 DEBUG:boto:Data: 2022-09-03T06:51:11.106 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.107 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:11.108 DEBUG:boto:Port: 8001 2022-09-03T06:51:11.109 DEBUG:boto:Params: {} 2022-09-03T06:51:11.111 DEBUG:boto:Token: None 2022-09-03T06:51:11.113 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:11.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nf+JEB+f+6vqUK55Oc+Wcs3R8co= 2022-09-03T06:51:11.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nf+JEB+f+6vqUK55Oc+Wcs3R8co=', 'Content-Length': '0'} 2022-09-03T06:51:11.120 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002886d87d37c337ff-006312f95f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.122 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.123 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.124 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.124 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.125 DEBUG:boto:Method: HEAD 2022-09-03T06:51:11.126 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:11.127 DEBUG:boto:Data: 2022-09-03T06:51:11.128 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.130 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:11.131 DEBUG:boto:Port: 8000 2022-09-03T06:51:11.131 DEBUG:boto:Params: {} 2022-09-03T06:51:11.132 DEBUG:boto:Token: None 2022-09-03T06:51:11.132 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:11.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nf+JEB+f+6vqUK55Oc+Wcs3R8co= 2022-09-03T06:51:11.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nf+JEB+f+6vqUK55Oc+Wcs3R8co=', 'Content-Length': '0'} 2022-09-03T06:51:11.139 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e19a200294f241c4-006312f95f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.140 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:51:11.141 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.142 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.143 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.143 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.144 DEBUG:boto:Method: GET 2022-09-03T06:51:11.145 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:11.145 DEBUG:boto:Data: 2022-09-03T06:51:11.146 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.146 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:11.147 DEBUG:boto:Port: 8001 2022-09-03T06:51:11.148 DEBUG:boto:Params: {} 2022-09-03T06:51:11.148 DEBUG:boto:Token: None 2022-09-03T06:51:11.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:11.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GMqkevySGeQY79cyz1xOVK5I5jU= 2022-09-03T06:51:11.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GMqkevySGeQY79cyz1xOVK5I5jU=', 'Content-Length': '0'} 2022-09-03T06:51:11.154 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9c2763c94b9de4e-006312f95f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.198 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.199 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.200 DEBUG:boto:path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.201 DEBUG:boto:auth_path=/avysmw-70/d?versionId=null 2022-09-03T06:51:11.202 DEBUG:boto:Method: GET 2022-09-03T06:51:11.203 DEBUG:boto:Path: /avysmw-70/d?versionId=null 2022-09-03T06:51:11.203 DEBUG:boto:Data: 2022-09-03T06:51:11.204 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.205 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:11.206 DEBUG:boto:Port: 8000 2022-09-03T06:51:11.207 DEBUG:boto:Params: {} 2022-09-03T06:51:11.208 DEBUG:boto:Token: None 2022-09-03T06:51:11.209 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d?versionId=null 2022-09-03T06:51:11.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GMqkevySGeQY79cyz1xOVK5I5jU= 2022-09-03T06:51:11.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GMqkevySGeQY79cyz1xOVK5I5jU=', 'Content-Length': '0'} 2022-09-03T06:51:11.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000878623b8d1101362-006312f95f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.217 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:11.218 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:11.219 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:11.220 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:11.221 DEBUG:boto:Method: GET 2022-09-03T06:51:11.222 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:11.223 DEBUG:boto:Data: 2022-09-03T06:51:11.224 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.225 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:11.226 DEBUG:boto:Port: 8001 2022-09-03T06:51:11.229 DEBUG:boto:Params: {} 2022-09-03T06:51:11.230 DEBUG:boto:Token: None 2022-09-03T06:51:11.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/ 2022-09-03T06:51:11.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w= 2022-09-03T06:51:11.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w=', 'Content-Length': '0'} 2022-09-03T06:51:11.240 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e342b6ed54e7e2c0-006312f95f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.242 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:11.244 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:11.245 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:11.246 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:11.247 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:11.248 DEBUG:boto:Method: GET 2022-09-03T06:51:11.249 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:11.252 DEBUG:boto:Data: 2022-09-03T06:51:11.253 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.254 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:11.255 DEBUG:boto:Port: 8000 2022-09-03T06:51:11.256 DEBUG:boto:Params: {} 2022-09-03T06:51:11.257 DEBUG:boto:Token: None 2022-09-03T06:51:11.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/ 2022-09-03T06:51:11.264 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w= 2022-09-03T06:51:11.265 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w=', 'Content-Length': '0'} 2022-09-03T06:51:11.271 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093c6db0af994554d-006312f95f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.272 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:11.286 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.287 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.288 DEBUG:boto:Method: HEAD 2022-09-03T06:51:11.289 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:11.291 DEBUG:boto:Data: 2022-09-03T06:51:11.292 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:11.293 DEBUG:boto:Port: 8001 2022-09-03T06:51:11.294 DEBUG:boto:Params: {} 2022-09-03T06:51:11.295 DEBUG:boto:Token: None 2022-09-03T06:51:11.296 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d 2022-09-03T06:51:11.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0ee1stx6TSRZLr6hUa2HqzsiLVY= 2022-09-03T06:51:11.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0ee1stx6TSRZLr6hUa2HqzsiLVY=', 'Content-Length': '0'} 2022-09-03T06:51:11.299 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000047519cc860d2f98f-006312f95f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.300 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.301 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.302 DEBUG:boto:Method: HEAD 2022-09-03T06:51:11.303 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:11.304 DEBUG:boto:Data: 2022-09-03T06:51:11.305 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.306 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:11.307 DEBUG:boto:Port: 8000 2022-09-03T06:51:11.308 DEBUG:boto:Params: {} 2022-09-03T06:51:11.309 DEBUG:boto:Token: None 2022-09-03T06:51:11.310 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d 2022-09-03T06:51:11.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0ee1stx6TSRZLr6hUa2HqzsiLVY= 2022-09-03T06:51:11.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0ee1stx6TSRZLr6hUa2HqzsiLVY=', 'Content-Length': '0'} 2022-09-03T06:51:11.315 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009168dbf030ddc411-006312f95f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.316 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:51:11.318 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.319 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.320 DEBUG:boto:Method: GET 2022-09-03T06:51:11.321 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:11.322 DEBUG:boto:Data: 2022-09-03T06:51:11.323 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.324 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:11.325 DEBUG:boto:Port: 8001 2022-09-03T06:51:11.326 DEBUG:boto:Params: {} 2022-09-03T06:51:11.327 DEBUG:boto:Token: None 2022-09-03T06:51:11.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d 2022-09-03T06:51:11.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRXMYMWyKpGA5Sj7nknyr9Cw+qg= 2022-09-03T06:51:11.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRXMYMWyKpGA5Sj7nknyr9Cw+qg=', 'Content-Length': '0'} 2022-09-03T06:51:11.334 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b220122b669446f9-006312f95f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.335 DEBUG:boto:path=/avysmw-70/d 2022-09-03T06:51:11.336 DEBUG:boto:auth_path=/avysmw-70/d 2022-09-03T06:51:11.337 DEBUG:boto:Method: GET 2022-09-03T06:51:11.340 DEBUG:boto:Path: /avysmw-70/d 2022-09-03T06:51:11.341 DEBUG:boto:Data: 2022-09-03T06:51:11.341 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.342 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:11.343 DEBUG:boto:Port: 8000 2022-09-03T06:51:11.343 DEBUG:boto:Params: {} 2022-09-03T06:51:11.344 DEBUG:boto:Token: None 2022-09-03T06:51:11.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/d 2022-09-03T06:51:11.346 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRXMYMWyKpGA5Sj7nknyr9Cw+qg= 2022-09-03T06:51:11.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRXMYMWyKpGA5Sj7nknyr9Cw+qg=', 'Content-Length': '0'} 2022-09-03T06:51:11.349 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:48:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd7f4e316bd2be5f-006312f95f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.403 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:11.404 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:11.405 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:11.406 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:11.407 DEBUG:boto:Method: GET 2022-09-03T06:51:11.409 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:11.410 DEBUG:boto:Data: 2022-09-03T06:51:11.411 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:51:11.414 DEBUG:boto:Port: 8001 2022-09-03T06:51:11.415 DEBUG:boto:Params: {} 2022-09-03T06:51:11.416 DEBUG:boto:Token: None 2022-09-03T06:51:11.418 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/ 2022-09-03T06:51:11.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w= 2022-09-03T06:51:11.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w=', 'Content-Length': '0'} 2022-09-03T06:51:11.427 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088b8a17780012ad3-006312f95f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.428 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:11.430 DEBUG:boto:path=/avysmw-70/ 2022-09-03T06:51:11.431 DEBUG:boto:auth_path=/avysmw-70/ 2022-09-03T06:51:11.433 DEBUG:boto:path=/avysmw-70/?prefix=d 2022-09-03T06:51:11.434 DEBUG:boto:auth_path=/avysmw-70/?prefix=d 2022-09-03T06:51:11.435 DEBUG:boto:Method: GET 2022-09-03T06:51:11.436 DEBUG:boto:Path: /avysmw-70/?prefix=d 2022-09-03T06:51:11.437 DEBUG:boto:Data: 2022-09-03T06:51:11.438 DEBUG:boto:Headers: {} 2022-09-03T06:51:11.439 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:51:11.440 DEBUG:boto:Port: 8000 2022-09-03T06:51:11.441 DEBUG:boto:Params: {} 2022-09-03T06:51:11.442 DEBUG:boto:Token: None 2022-09-03T06:51:11.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:51:11 GMT /avysmw-70/ 2022-09-03T06:51:11.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w= 2022-09-03T06:51:11.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IMoy2geiSBiOXBuOC8iiRsjlt4w=', 'Content-Length': '0'} 2022-09-03T06:51:11.449 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb23eddd7b1de7ac-006312f95f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:51:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:11.450 DEBUG:boto:b'avysmw-70d1000falsed2022-09-03T06:48:10.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:51:11.452 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-70 zones={test-zone3, test-zone2} 2022-09-03T06:51:11.453 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '1', '--rgw-sync-log-trim-min-cold-buckets', '0', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:51:11.454 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '1', '--rgw-sync-log-trim-min-cold-buckets', '0', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:51:11.455 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 1 --rgw-sync-log-trim-min-cold-buckets 0 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:51:11.510 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:11.511 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:11.607 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'list', '--bucket', 'avysmw-70', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:11.608 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'list', '--bucket', 'avysmw-70', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:11.609 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog list --bucket avysmw-70 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:51:11.658 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:11.659 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:11.711 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:51:11.713 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'list', '--bucket', 'avysmw-69', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:11.714 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'list', '--bucket', 'avysmw-69', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:11.715 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog list --bucket avysmw-69 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:51:11.768 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:11.768 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:11.828 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:51:11.829 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:11.829 INFO:teuthology.orchestra.run.smithi062.stdout: "op_id": "1#00000000001.61.5", 2022-09-03T06:51:11.830 INFO:teuthology.orchestra.run.smithi062.stdout: "op_tag": "fca49d30-abba-4c2b-8462-66395ac063ba.5636.18360592817359046504", 2022-09-03T06:51:11.833 INFO:teuthology.orchestra.run.smithi062.stdout: "op": "write", 2022-09-03T06:51:11.834 INFO:teuthology.orchestra.run.smithi062.stdout: "object": "a", 2022-09-03T06:51:11.834 INFO:teuthology.orchestra.run.smithi062.stdout: "instance": "", 2022-09-03T06:51:11.835 INFO:teuthology.orchestra.run.smithi062.stdout: "state": "complete", 2022-09-03T06:51:11.835 INFO:teuthology.orchestra.run.smithi062.stdout: "index_ver": 1, 2022-09-03T06:51:11.835 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:47:10.189611322Z", 2022-09-03T06:51:11.836 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": { 2022-09-03T06:51:11.836 INFO:teuthology.orchestra.run.smithi062.stdout: "pool": 9, 2022-09-03T06:51:11.837 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 5 2022-09-03T06:51:11.837 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:11.838 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog_flags": 0, 2022-09-03T06:51:11.838 INFO:teuthology.orchestra.run.smithi062.stdout: "versioned": false, 2022-09-03T06:51:11.839 INFO:teuthology.orchestra.run.smithi062.stdout: "owner": "", 2022-09-03T06:51:11.839 INFO:teuthology.orchestra.run.smithi062.stdout: "owner_display_name": "", 2022-09-03T06:51:11.840 INFO:teuthology.orchestra.run.smithi062.stdout: "zones_trace": [ 2022-09-03T06:51:11.841 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:11.841 INFO:teuthology.orchestra.run.smithi062.stdout: "entry": "fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-69:fca49d30-abba-4c2b-8462-66395ac063ba.5638.20" 2022-09-03T06:51:11.842 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:11.843 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:51:11.843 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:11.844 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:11.844 INFO:teuthology.orchestra.run.smithi062.stdout: "op_id": "5#00000000001.56.5", 2022-09-03T06:51:11.845 INFO:teuthology.orchestra.run.smithi062.stdout: "op_tag": "fca49d30-abba-4c2b-8462-66395ac063ba.5636.1369389973876988070", 2022-09-03T06:51:11.845 INFO:teuthology.orchestra.run.smithi062.stdout: "op": "write", 2022-09-03T06:51:11.845 INFO:teuthology.orchestra.run.smithi062.stdout: "object": "d", 2022-09-03T06:51:11.846 INFO:teuthology.orchestra.run.smithi062.stdout: "instance": "", 2022-09-03T06:51:11.846 INFO:teuthology.orchestra.run.smithi062.stdout: "state": "complete", 2022-09-03T06:51:11.846 INFO:teuthology.orchestra.run.smithi062.stdout: "index_ver": 1, 2022-09-03T06:51:11.847 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:47:10.327169121Z", 2022-09-03T06:51:11.847 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": { 2022-09-03T06:51:11.848 INFO:teuthology.orchestra.run.smithi062.stdout: "pool": 9, 2022-09-03T06:51:11.848 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 5 2022-09-03T06:51:11.849 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:11.849 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog_flags": 0, 2022-09-03T06:51:11.850 INFO:teuthology.orchestra.run.smithi062.stdout: "versioned": false, 2022-09-03T06:51:11.850 INFO:teuthology.orchestra.run.smithi062.stdout: "owner": "", 2022-09-03T06:51:11.850 INFO:teuthology.orchestra.run.smithi062.stdout: "owner_display_name": "", 2022-09-03T06:51:11.851 INFO:teuthology.orchestra.run.smithi062.stdout: "zones_trace": [ 2022-09-03T06:51:11.851 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:11.851 INFO:teuthology.orchestra.run.smithi062.stdout: "entry": "fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-69:fca49d30-abba-4c2b-8462-66395ac063ba.5638.20" 2022-09-03T06:51:11.852 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:11.852 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:51:11.853 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:11.853 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:11.854 INFO:teuthology.orchestra.run.smithi062.stdout: "op_id": "8#00000000001.80.5", 2022-09-03T06:51:11.854 INFO:teuthology.orchestra.run.smithi062.stdout: "op_tag": "fca49d30-abba-4c2b-8462-66395ac063ba.5636.14609911072704577718", 2022-09-03T06:51:11.854 INFO:teuthology.orchestra.run.smithi062.stdout: "op": "write", 2022-09-03T06:51:11.855 INFO:teuthology.orchestra.run.smithi062.stdout: "object": "c", 2022-09-03T06:51:11.855 INFO:teuthology.orchestra.run.smithi062.stdout: "instance": "", 2022-09-03T06:51:11.855 INFO:teuthology.orchestra.run.smithi062.stdout: "state": "complete", 2022-09-03T06:51:11.855 INFO:teuthology.orchestra.run.smithi062.stdout: "index_ver": 1, 2022-09-03T06:51:11.856 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:47:10.293975945Z", 2022-09-03T06:51:11.856 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": { 2022-09-03T06:51:11.856 INFO:teuthology.orchestra.run.smithi062.stdout: "pool": 9, 2022-09-03T06:51:11.857 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 3 2022-09-03T06:51:11.857 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:11.857 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog_flags": 0, 2022-09-03T06:51:11.858 INFO:teuthology.orchestra.run.smithi062.stdout: "versioned": false, 2022-09-03T06:51:11.858 INFO:teuthology.orchestra.run.smithi062.stdout: "owner": "", 2022-09-03T06:51:11.859 INFO:teuthology.orchestra.run.smithi062.stdout: "owner_display_name": "", 2022-09-03T06:51:11.859 INFO:teuthology.orchestra.run.smithi062.stdout: "zones_trace": [ 2022-09-03T06:51:11.860 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:11.860 INFO:teuthology.orchestra.run.smithi062.stdout: "entry": "fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-69:fca49d30-abba-4c2b-8462-66395ac063ba.5638.20" 2022-09-03T06:51:11.861 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:11.861 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:51:11.862 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:11.862 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:11.863 INFO:teuthology.orchestra.run.smithi062.stdout: "op_id": "10#00000000001.67.5", 2022-09-03T06:51:11.863 INFO:teuthology.orchestra.run.smithi062.stdout: "op_tag": "fca49d30-abba-4c2b-8462-66395ac063ba.5636.2507346405215526382", 2022-09-03T06:51:11.864 INFO:teuthology.orchestra.run.smithi062.stdout: "op": "write", 2022-09-03T06:51:11.864 INFO:teuthology.orchestra.run.smithi062.stdout: "object": "b", 2022-09-03T06:51:11.865 INFO:teuthology.orchestra.run.smithi062.stdout: "instance": "", 2022-09-03T06:51:11.865 INFO:teuthology.orchestra.run.smithi062.stdout: "state": "complete", 2022-09-03T06:51:11.866 INFO:teuthology.orchestra.run.smithi062.stdout: "index_ver": 1, 2022-09-03T06:51:11.867 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:47:10.221055532Z", 2022-09-03T06:51:11.868 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": { 2022-09-03T06:51:11.868 INFO:teuthology.orchestra.run.smithi062.stdout: "pool": 9, 2022-09-03T06:51:11.869 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 11 2022-09-03T06:51:11.869 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:11.870 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog_flags": 0, 2022-09-03T06:51:11.870 INFO:teuthology.orchestra.run.smithi062.stdout: "versioned": false, 2022-09-03T06:51:11.871 INFO:teuthology.orchestra.run.smithi062.stdout: "owner": "", 2022-09-03T06:51:11.871 INFO:teuthology.orchestra.run.smithi062.stdout: "owner_display_name": "", 2022-09-03T06:51:11.872 INFO:teuthology.orchestra.run.smithi062.stdout: "zones_trace": [ 2022-09-03T06:51:11.874 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:11.874 INFO:teuthology.orchestra.run.smithi062.stdout: "entry": "fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-69:fca49d30-abba-4c2b-8462-66395ac063ba.5638.20" 2022-09-03T06:51:11.875 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:11.875 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:51:11.876 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:11.876 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:51:11.879 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '999', '--rgw-sync-log-trim-min-cold-buckets', '999', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:51:11.880 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '999', '--rgw-sync-log-trim-min-cold-buckets', '999', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:51:11.881 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 999 --rgw-sync-log-trim-min-cold-buckets 999 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:51:11.930 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:11.931 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:12.002 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.000+0000 7fa89a9b4700 -1 req 17367285189639488061 0.003999906s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.004 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.000+0000 7f1708989700 -1 req 7604668191146651301 0.003999932s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.013 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.002+0000 7f4facdfa700 -1 req 15194658940403809546 0.005999860s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.017 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.015+0000 7f16f7967700 -1 req 4192036645524681033 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.018 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.016+0000 7fa872964700 -1 req 11587895995244314626 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.028 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.017+0000 7f4f7658d700 -1 req 449403023893410937 0.002999929s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.063 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.061+0000 7fa96b355700 -1 req 9003998000008567214 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.064 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.061+0000 7f4f935c7700 -1 req 2138003712202682162 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.065 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.062+0000 7f170f196700 -1 req 8971537849901845505 0.002999950s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.068 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.066+0000 7f16ce114700 -1 req 1480440587441634351 0.002999949s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.073 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.067+0000 7fa95c337700 -1 req 2774320575590687659 0.004999883s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.074 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.067+0000 7f4f9f5df700 -1 req 15974224913860489060 0.004999883s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.123 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.121+0000 7f16ba8ed700 -1 req 16474039991196040869 0.001999967s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.125 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.121+0000 7fa942303700 -1 req 8650248842513526733 0.001999954s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.126 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.121+0000 7f505d75b700 -1 req 3405626704088911637 0.001999954s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.157 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.155+0000 7f16d7927700 -1 req 7187106210159332185 0.001999967s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.158 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.155+0000 7fa93dafa700 -1 req 5911332762558334470 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.159 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.155+0000 7f4f79d94700 -1 req 7441364757618949855 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.168 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.165+0000 7f16ab8cf700 -1 req 11113757275146571351 0.002999949s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.169 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.166+0000 7f16cf917700 -1 req 17195669741790421926 0.001999967s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.170 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.165+0000 7fa9342e7700 -1 req 13363965311172334020 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.171 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.166+0000 7f4f8bdb8700 -1 req 16746096494126103491 0.003999906s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.172 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.166+0000 7fa933ae6700 -1 req 1486328389433977377 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.172 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.166+0000 7f5062765700 -1 req 8555891160143661141 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.201 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.199+0000 7fa92bad6700 -1 req 15321359835038229638 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.201 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.199+0000 7f4f7a595700 -1 req 8216140014049759059 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.202 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.199+0000 7f16b48e1700 -1 req 17177031523080178981 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.215 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.213+0000 7f168d893700 -1 req 11271077166816717319 0.002999949s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.216 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.213+0000 7fa91aab4700 -1 req 8058643743872390489 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.216 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.214+0000 7f50146c9700 -1 req 9686091708910697234 0.003999907s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.261 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.259+0000 7f16a48c1700 -1 req 7517394120626460379 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.262 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.261+0000 7f1760238700 -1 req 12581207396122152184 0.001999967s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.263 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.261+0000 7f5044f2a700 -1 req 7902640179784981719 0.004999883s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.264 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.261+0000 7fa906a8c700 -1 req 9965689367989467059 0.004999883s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.265 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.261+0000 7f5049733700 -1 req 7729568868363710646 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.266 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.262+0000 7fa8faa74700 -1 req 18140387985602402747 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.296 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.295+0000 7f167b06e700 -1 req 18061662244648442346 0.001999967s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.297 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.295+0000 7fa8f4a68700 -1 req 5322418181334911866 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.298 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.295+0000 7f503df1c700 -1 req 1543454770547895039 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.310 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.308+0000 7fa8e7a4e700 -1 req 6520302835481047741 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.311 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.308+0000 7f4fe8e72700 -1 req 11634556019647273087 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.312 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.309+0000 7f1746204700 -1 req 17044284293674957691 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.340 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.339+0000 7f175b22e700 -1 req 3138367968428706524 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.341 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.339+0000 7fa8dda3a700 -1 req 4795440408010401256 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.342 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.339+0000 7f500eebe700 -1 req 6463988439320979774 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-9:fca49d30-abba-4c2b-8462-66395ac063ba.4344.9,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.354 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.352+0000 7f17321dc700 -1 req 5037654776488645270 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.355 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.353+0000 7fa8d5229700 -1 req 11706339153631319604 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.359 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.356+0000 7f4fd4649700 -1 req 8283236077969798858 0.004999883s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-45:fca49d30-abba-4c2b-8462-66395ac063ba.5190.2,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.364 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.358+0000 7f175221c700 -1 req 14603563976200316158 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.364 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.359+0000 7fa8c9a12700 -1 req 12179637904696810141 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.365 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.359+0000 7f501fee0700 -1 req 12061684844568238335 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-8:fca49d30-abba-4c2b-8462-66395ac063ba.4344.8,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.385 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.383+0000 7f17301d8700 -1 req 864977010023758986 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.386 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.383+0000 7fa8c4a08700 -1 req 12051421486780117124 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.387 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.383+0000 7f4fee67d700 -1 req 15394209449959809974 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-39:fca49d30-abba-4c2b-8462-66395ac063ba.4344.39,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.403 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.400+0000 7f4fbce1a700 -1 req 6325462640533957303 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.404 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.400+0000 7f17231be700 -1 req 10695471857091597895 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.405 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.400+0000 7fa8bd1f9700 -1 req 9983356873544877720 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-44:fca49d30-abba-4c2b-8462-66395ac063ba.5190.1,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.406 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.402+0000 7fa8b89f0700 -1 req 3946301467233545471 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-66:fca49d30-abba-4c2b-8462-66395ac063ba.5638.17,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-66:fca49d30-abba-4c2b-8462-66395ac063ba.5638.17,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.406 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.402+0000 7f50046a9700 -1 req 7460376003600397065 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-66:fca49d30-abba-4c2b-8462-66395ac063ba.5638.17,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-66:fca49d30-abba-4c2b-8462-66395ac063ba.5638.17,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.408 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.402+0000 7f173a9ed700 -1 req 3964829905720614250 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-66:fca49d30-abba-4c2b-8462-66395ac063ba.5638.17,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-66:fca49d30-abba-4c2b-8462-66395ac063ba.5638.17,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.428 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.427+0000 7f1751a1b700 -1 req 4299314824500823691 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.429 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.427+0000 7f4fe666d700 -1 req 10077797039259642589 0.001999954s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.430 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.427+0000 7fa8ab1d5700 -1 req 12856422227968626772 0.001999954s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-27:fca49d30-abba-4c2b-8462-66395ac063ba.4344.27,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.449 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.447+0000 7f169309e700 -1 req 15662973746647686545 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-55:fca49d30-abba-4c2b-8462-66395ac063ba.5638.6,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-55:fca49d30-abba-4c2b-8462-66395ac063ba.5638.6,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.450 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.448+0000 7f50286f1700 -1 req 5641154603112112735 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-55:fca49d30-abba-4c2b-8462-66395ac063ba.5638.6,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-55:fca49d30-abba-4c2b-8462-66395ac063ba.5638.6,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.450 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.448+0000 7fa89f1bd700 -1 req 9484875089045658615 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-55:fca49d30-abba-4c2b-8462-66395ac063ba.5638.6,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-55:fca49d30-abba-4c2b-8462-66395ac063ba.5638.6,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.522 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.520+0000 7f1700979700 -1 req 8400389952247254461 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.523 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.521+0000 7f4fa25e5700 -1 req 7947975907231005093 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.524 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.521+0000 7fa87f17d700 -1 req 4562505464152640694 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-32:fca49d30-abba-4c2b-8462-66395ac063ba.4344.32,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.549 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.545+0000 7f4ff8e92700 -1 req 15237539123489218793 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.550 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.545+0000 7f16ec951700 -1 req 9741399541543370535 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.551 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.547+0000 7f16fc971700 -1 req 18297833982794166587 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.553 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.545+0000 7fa96db5a700 -1 req 4798886136912471824 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-1:fca49d30-abba-4c2b-8462-66395ac063ba.4344.1,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.554 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.547+0000 7fa971361700 -1 req 13393779863128317490 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.555 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.547+0000 7f4f7758f700 -1 req 5459440390499661927 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-40:fca49d30-abba-4c2b-8462-66395ac063ba.4344.40,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.567 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.565+0000 7f16e0939700 -1 req 14292748182491903016 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-54:fca49d30-abba-4c2b-8462-66395ac063ba.5638.5,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-54:fca49d30-abba-4c2b-8462-66395ac063ba.5638.5,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.571 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.569+0000 7f4fa85f1700 -1 req 6462340752418311064 0.004999883s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-54:fca49d30-abba-4c2b-8462-66395ac063ba.5638.5,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-54:fca49d30-abba-4c2b-8462-66395ac063ba.5638.5,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.572 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.569+0000 7fa963345700 -1 req 4798940232448882894 0.004999883s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-54:fca49d30-abba-4c2b-8462-66395ac063ba.5638.5,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-54:fca49d30-abba-4c2b-8462-66395ac063ba.5638.5,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.591 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.590+0000 7f16ce114700 -1 req 577264017600112930 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.592 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.590+0000 7f16d6925700 -1 req 10906671837193177420 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-52:fca49d30-abba-4c2b-8462-66395ac063ba.5638.3,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-52:fca49d30-abba-4c2b-8462-66395ac063ba.5638.3,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.592 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.590+0000 7f4fa6dee700 -1 req 6027231227398386320 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-52:fca49d30-abba-4c2b-8462-66395ac063ba.5638.3,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-52:fca49d30-abba-4c2b-8462-66395ac063ba.5638.3,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.593 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.590+0000 7fa95832f700 -1 req 14112844149551397616 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.593 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.590+0000 7fa95b335700 -1 req 396449893295079789 0.002999930s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-52:fca49d30-abba-4c2b-8462-66395ac063ba.5638.3,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-52:fca49d30-abba-4c2b-8462-66395ac063ba.5638.3,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.594 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.590+0000 7f5030f02700 -1 req 6953105122521642538 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-42:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.1,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.615 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.613+0000 7f16c18fb700 -1 req 7447075341431409962 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.616 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.614+0000 7fa94fb1e700 -1 req 6130799537064239863 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.617 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.614+0000 7f503a715700 -1 req 11169098525033810591 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-41:fca49d30-abba-4c2b-8462-66395ac063ba.4344.41,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.659 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.658+0000 7f16ca10c700 -1 req 7912149393396471143 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.661 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.657+0000 7f505674d700 -1 req 7673444193131179770 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.662 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.657+0000 7fa92e2db700 -1 req 9255897026442826200 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-35:fca49d30-abba-4c2b-8462-66395ac063ba.4344.35,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.689 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.685+0000 7f16cb10e700 -1 req 10135244209696245291 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.689 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.686+0000 7f16a10ba700 -1 req 12890642918274339600 0.001999967s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.690 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.685+0000 7fa92f2dd700 -1 req 11756079318606696754 0.001999954s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.691 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.685+0000 7fa92aad4700 -1 req 4559645545456693055 0.001999954s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.691 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.685+0000 7f4f6dd7c700 -1 req 14040404402271309566 0.001999954s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-6:fca49d30-abba-4c2b-8462-66395ac063ba.4344.6,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.692 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.685+0000 7f504cf3a700 -1 req 12135745016232727454 0.001999954s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-33:fca49d30-abba-4c2b-8462-66395ac063ba.4344.33,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.703 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:51:12.702+0000 7f16aa0cc700 -1 req 1044418089111573940 0.001999966s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-51:fca49d30-abba-4c2b-8462-66395ac063ba.5638.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-51:fca49d30-abba-4c2b-8462-66395ac063ba.5638.2,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:51:12.704 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:51:12.702+0000 7f4f71583700 -1 req 5782917417379662351 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-51:fca49d30-abba-4c2b-8462-66395ac063ba.5638.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-51:fca49d30-abba-4c2b-8462-66395ac063ba.5638.2,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:51:12.705 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:51:12.702+0000 7fa9162ab700 -1 req 9595278671461267843 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-51:fca49d30-abba-4c2b-8462-66395ac063ba.5638.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-51:fca49d30-abba-4c2b-8462-66395ac063ba.5638.2,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:51:12.857 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'list', '--bucket', 'avysmw-69', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:12.861 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'list', '--bucket', 'avysmw-69', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:12.863 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog list --bucket avysmw-69 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:51:12.948 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:12.949 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:13.016 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:51:13.019 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_index_log_trim ... ok 2022-09-03T06:51:13.022 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=avysmw-71 2022-09-03T06:51:13.023 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:51:13.024 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:51:13.025 DEBUG:boto:Method: PUT 2022-09-03T06:51:13.025 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:51:13.026 DEBUG:boto:Data: 2022-09-03T06:51:13.026 DEBUG:boto:Headers: {} 2022-09-03T06:51:13.027 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:13.027 DEBUG:boto:Port: 8000 2022-09-03T06:51:13.028 DEBUG:boto:Params: {} 2022-09-03T06:51:13.029 DEBUG:boto:Token: None 2022-09-03T06:51:13.029 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:51:13 GMT /avysmw-71/ 2022-09-03T06:51:13.030 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bl7YSG8VOmxhukPkx1eZGzBPudY= 2022-09-03T06:51:13.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bl7YSG8VOmxhukPkx1eZGzBPudY=', 'Content-Length': '0'} 2022-09-03T06:51:13.049 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000008dc1cc09a5703431-006312f961-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:13.051 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:51:13.052 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:51:13.053 DEBUG:boto:Method: HEAD 2022-09-03T06:51:13.054 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:51:13.055 DEBUG:boto:Data: 2022-09-03T06:51:13.056 DEBUG:boto:Headers: {} 2022-09-03T06:51:13.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:13.058 DEBUG:boto:Port: 8000 2022-09-03T06:51:13.059 DEBUG:boto:Params: {} 2022-09-03T06:51:13.060 DEBUG:boto:Token: None 2022-09-03T06:51:13.061 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:13 GMT /avysmw-71/ 2022-09-03T06:51:13.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4LMmUIs5vwewraT4trb/9aC+Ec0= 2022-09-03T06:51:13.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4LMmUIs5vwewraT4trb/9aC+Ec0=', 'Content-Length': '0'} 2022-09-03T06:51:13.066 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000061854d88e736ef3a-006312f961-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:13.068 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:51:13.069 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:51:13.070 DEBUG:boto:Method: PUT 2022-09-03T06:51:13.071 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:51:13.071 DEBUG:boto:Data: 2022-09-03T06:51:13.072 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:51:13.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:13.074 DEBUG:boto:Port: 8000 2022-09-03T06:51:13.075 DEBUG:boto:Params: {} 2022-09-03T06:51:13.076 DEBUG:boto:Token: None 2022-09-03T06:51:13.077 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:51:13 GMT /avysmw-71/a 2022-09-03T06:51:13.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nbFgd60uNDcgGQCKRCE8OCk02TQ= 2022-09-03T06:51:13.079 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:51:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nbFgd60uNDcgGQCKRCE8OCk02TQ='} 2022-09-03T06:51:13.126 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000003a7d1d316e1b8419-006312f961-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:13.128 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:51:13.129 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:51:13.131 DEBUG:boto:Method: HEAD 2022-09-03T06:51:13.131 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:51:13.132 DEBUG:boto:Data: 2022-09-03T06:51:13.132 DEBUG:boto:Headers: {} 2022-09-03T06:51:13.133 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:13.133 DEBUG:boto:Port: 8000 2022-09-03T06:51:13.133 DEBUG:boto:Params: {} 2022-09-03T06:51:13.134 DEBUG:boto:Token: None 2022-09-03T06:51:13.135 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:13 GMT /avysmw-71/ 2022-09-03T06:51:13.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4LMmUIs5vwewraT4trb/9aC+Ec0= 2022-09-03T06:51:13.137 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4LMmUIs5vwewraT4trb/9aC+Ec0=', 'Content-Length': '0'} 2022-09-03T06:51:13.140 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003a1cd3f10cca5591-006312f961-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:13.142 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:51:13.144 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:51:13.146 DEBUG:boto:Method: PUT 2022-09-03T06:51:13.146 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:51:13.147 DEBUG:boto:Data: 2022-09-03T06:51:13.148 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:51:13.148 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:13.149 DEBUG:boto:Port: 8000 2022-09-03T06:51:13.150 DEBUG:boto:Params: {} 2022-09-03T06:51:13.151 DEBUG:boto:Token: None 2022-09-03T06:51:13.152 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:51:13 GMT /avysmw-71/b 2022-09-03T06:51:13.152 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dOh1AmkYGX/w6s1jMn9678jzG9I= 2022-09-03T06:51:13.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:51:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dOh1AmkYGX/w6s1jMn9678jzG9I='} 2022-09-03T06:51:13.160 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000005391e756d18fc077-006312f961-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:13.161 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:51:13.162 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:51:13.163 DEBUG:boto:Method: HEAD 2022-09-03T06:51:13.163 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:51:13.164 DEBUG:boto:Data: 2022-09-03T06:51:13.164 DEBUG:boto:Headers: {} 2022-09-03T06:51:13.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:13.165 DEBUG:boto:Port: 8000 2022-09-03T06:51:13.167 DEBUG:boto:Params: {} 2022-09-03T06:51:13.169 DEBUG:boto:Token: None 2022-09-03T06:51:13.170 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:13 GMT /avysmw-71/ 2022-09-03T06:51:13.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4LMmUIs5vwewraT4trb/9aC+Ec0= 2022-09-03T06:51:13.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4LMmUIs5vwewraT4trb/9aC+Ec0=', 'Content-Length': '0'} 2022-09-03T06:51:13.178 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000501dc6ede568a392-006312f961-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:13.179 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:51:13.180 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:51:13.181 DEBUG:boto:Method: PUT 2022-09-03T06:51:13.182 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:51:13.182 DEBUG:boto:Data: 2022-09-03T06:51:13.183 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:51:13.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:13.186 DEBUG:boto:Port: 8000 2022-09-03T06:51:13.187 DEBUG:boto:Params: {} 2022-09-03T06:51:13.188 DEBUG:boto:Token: None 2022-09-03T06:51:13.189 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:51:13 GMT /avysmw-71/c 2022-09-03T06:51:13.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DwGjB3VC8nRMVl+rSMQR34O1U0c= 2022-09-03T06:51:13.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:51:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DwGjB3VC8nRMVl+rSMQR34O1U0c='} 2022-09-03T06:51:13.207 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000e71df6a023e0e973-006312f961-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:13.208 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:51:13.209 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:51:13.210 DEBUG:boto:Method: HEAD 2022-09-03T06:51:13.210 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:51:13.211 DEBUG:boto:Data: 2022-09-03T06:51:13.212 DEBUG:boto:Headers: {} 2022-09-03T06:51:13.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:13.213 DEBUG:boto:Port: 8000 2022-09-03T06:51:13.213 DEBUG:boto:Params: {} 2022-09-03T06:51:13.214 DEBUG:boto:Token: None 2022-09-03T06:51:13.215 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:51:13 GMT /avysmw-71/ 2022-09-03T06:51:13.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4LMmUIs5vwewraT4trb/9aC+Ec0= 2022-09-03T06:51:13.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:51:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4LMmUIs5vwewraT4trb/9aC+Ec0=', 'Content-Length': '0'} 2022-09-03T06:51:13.220 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '9'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f5e74cdc7134db9b-006312f961-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:13.221 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:51:13.222 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:51:13.222 DEBUG:boto:Method: PUT 2022-09-03T06:51:13.223 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:51:13.224 DEBUG:boto:Data: 2022-09-03T06:51:13.225 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:51:13.227 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:51:13.228 DEBUG:boto:Port: 8000 2022-09-03T06:51:13.230 DEBUG:boto:Params: {} 2022-09-03T06:51:13.231 DEBUG:boto:Token: None 2022-09-03T06:51:13.232 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:51:13 GMT /avysmw-71/d 2022-09-03T06:51:13.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fiTBNSe+7BJCbqFdBldIIBJ89HQ= 2022-09-03T06:51:13.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:51:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fiTBNSe+7BJCbqFdBldIIBJ89HQ='} 2022-09-03T06:51:13.279 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000e1424ef50b95eb03-006312f961-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:51:13.280 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:13.281 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:13.283 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:51:13.336 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:13.337 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:13.387 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:51:13.411 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:51:13.412 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:13.412 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:51:13.413 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:51:13.413 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:13.414 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:13.414 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187873.046707_405.1", 2022-09-03T06:51:13.415 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:51:13.046707Z" 2022-09-03T06:51:13.416 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:13.416 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:13.417 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:51:13.417 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:51:13.418 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:13.418 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:13.419 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:51:13.419 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:51:13.420 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:13.420 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:51:13.421 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662187873.046707_405.1', 2: '', 3: ''} 2022-09-03T06:51:13.422 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:51:13.423 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:13.429 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:13.430 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:51:13.485 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:13.486 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:13.540 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:13.538+0000 7fcd44c74600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:51:13.541 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:13.539+0000 7fcd44c74600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:51:13.561 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:51:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:51:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:51:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:51:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:51:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:51:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:51:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:51:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:51:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:51:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:51:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:51:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:51:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:51:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:51:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:51:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:51:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:51:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:51:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:51:13.570 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:51:13.570 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:51:13.570 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187873.046707_405.1", 2022-09-03T06:51:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:51:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:51:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:51:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:51:13.046707Z", 2022-09-03T06:51:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:51:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:51:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:51:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:51:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:51:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:51:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:51:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:51:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:51:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:51:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:51:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:51:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:51:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:51:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:51:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:51:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:51:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:51:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:51:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:51:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:51:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:51:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:51:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:51:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:51:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:51:13Z" 2022-09-03T06:51:13.583 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:51:13.583 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187873.046707_405.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:51:13.046707Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:51:13Z" } 2022-09-03T06:51:13.584 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187873.046707_405.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:51:13.046707Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:51:13.584 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187873.046707_405.1', 2: '', 3: ''} 2022-09-03T06:51:13.585 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187873.046707_405.1', 2: '', 3: ''} 2022-09-03T06:51:13.585 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:51:13.586 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:51:13.586 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:13.587 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:13.587 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:51:13.640 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:13.641 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:13.678 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:13.676+0000 7fc4de6cc600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:51:13.679 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:13.677+0000 7fc4de6cc600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:51:13.694 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:51:13.695 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:51:13.695 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:51:13.696 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:51:13.696 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:51:13.697 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:51:13.697 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:51:13.697 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:13.698 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:51:13.698 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:13.699 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:51:13.699 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:51:13.699 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:51:13.700 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:51:13.700 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:51:13.700 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:51:13.701 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:51:13.701 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:51:13.701 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:51:13.702 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:13.702 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:13.703 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:13.703 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:51:13.703 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:51:13.704 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:51:13.704 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662187873.046707_405.1", 2022-09-03T06:51:13.704 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:51:13.705 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:51:13.705 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:51:13.706 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:51:13.046707Z", 2022-09-03T06:51:13.706 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:51:13.706 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:13.707 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:13.707 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:13.707 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:51:13.708 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:51:13.709 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:51:13.709 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:51:13.709 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:51:13.710 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:51:13.710 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:51:13.710 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:51:13.711 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:51:13.711 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:13.712 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:13.712 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:51:13.712 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:51:13.713 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:51:13.713 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:51:13.714 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:51:13.714 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:51:13.714 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:51:13.715 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:51:13.715 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:51:13.715 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:51:13.716 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:13.716 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:51:13.717 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:51:13.718 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:13.718 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:51:13.718 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:51:13.718 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:51:13.719 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:51:13.719 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:51:13Z" 2022-09-03T06:51:13.719 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:51:13.722 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187873.046707_405.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:51:13.046707Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:51:13Z" } 2022-09-03T06:51:13.723 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187873.046707_405.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:51:13.046707Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:51:13.724 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187873.046707_405.1', 2: '', 3: ''} 2022-09-03T06:51:13.725 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187873.046707_405.1', 2: '', 3: ''} 2022-09-03T06:51:13.726 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:51:13.726 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:51:13.727 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:13.728 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:51:13.728 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:51:13.782 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:13.783 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:13.816 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:13.814+0000 7fe0d40eb600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:51:13.819 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:13.817+0000 7fe0d40eb600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:51:13.847 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:51:13.847 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:51:13.848 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:51:13.848 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:51:13.849 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:51:13.850 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:51:13.851 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:51:13.851 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.852 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:51:13.853 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:51:13.854 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:51:13.854 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:51:13.855 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:51:13.855 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:51:13.856 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:51:13.856 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:51:13.857 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:51:13.857 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:51:13.858 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:51:13.858 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.858 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.859 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:51:13.859 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:51:13.859 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:51:13.860 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:51:13.860 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662187873.046707_405.1", 2022-09-03T06:51:13.860 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:51:13.860 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:51:13.861 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:51:13.861 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:51:13.046707Z", 2022-09-03T06:51:13.861 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:51:13.861 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.862 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.862 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:51:13.862 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:51:13.862 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:51:13.863 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:51:13.863 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:51:13.863 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:51:13.864 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:51:13.864 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:51:13.864 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:51:13.865 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:51:13.865 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.865 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.865 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:51:13.866 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:51:13.866 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:51:13.866 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:51:13.867 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:51:13.867 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:51:13.867 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:51:13.867 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:51:13.868 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:51:13.868 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:51:13.868 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.868 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:51:13.869 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:51:13.869 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.869 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:51:13.869 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:51:13.870 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:51:13.870 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:51:13.870 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:51:13Z" 2022-09-03T06:51:13.870 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:51:13.872 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662187873.046707_405.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:51:13.046707Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:51:13Z" } 2022-09-03T06:51:13.872 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662187873.046707_405.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:51:13.046707Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:51:13.873 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662187873.046707_405.1', 2: '', 3: ''} 2022-09-03T06:51:13.874 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662187873.046707_405.1', 2: '', 3: ''} 2022-09-03T06:51:13.874 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:51:13.875 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-71 2022-09-03T06:51:13.875 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:13.876 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:13.876 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:51:13.939 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:13.939 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:13.993 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:13.991+0000 7f6ab9924600 1 waiting to reach incremental sync.. 2022-09-03T06:51:18.994 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:18.991+0000 7f6ab9924600 1 waiting to reach incremental sync.. 2022-09-03T06:51:23.994 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:23.992+0000 7f6ab9924600 1 waiting to reach incremental sync.. 2022-09-03T06:51:29.000 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:28.997+0000 7f6ab9924600 1 bucket sync caught up with source: 2022-09-03T06:51:29.002 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:29.003 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:29.005 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:28.997+0000 7f6ab9924600 0 bucket checkpoint complete 2022-09-03T06:51:29.030 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-71 2022-09-03T06:51:29.032 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:29.033 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:29.036 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:51:29.117 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:29.117 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:29.173 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:29.169+0000 7f2727809600 1 bucket sync caught up with source: 2022-09-03T06:51:29.174 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:29.175 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:29.175 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:51:29.169+0000 7f2727809600 0 bucket checkpoint complete 2022-09-03T06:51:29.192 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-71 2022-09-03T06:51:29.193 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:29.194 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:51:29.194 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:51:29.265 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:51:29.270 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:51:29.328 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:29.325+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:51:29.329 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:29.329 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:34.328 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:34.326+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:51:34.329 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:34.329 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:39.339 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:39.327+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:51:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:39.340 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:44.333 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:44.328+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:51:44.334 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:44.334 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:49.339 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:49.329+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:51:49.340 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:49.340 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:54.332 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:54.330+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:51:54.333 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:54.333 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:59.338 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:51:59.331+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:51:59.338 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:51:59.339 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:04.334 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:04.332+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:52:04.335 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:04.335 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:09.336 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:09.334+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:52:09.336 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:09.336 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:14.337 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:14.335+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:52:14.337 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:14.337 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:19.338 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:19.336+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:52:19.339 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:19.339 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:24.339 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:24.337+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:52:24.340 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:24.340 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:29.340 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:29.338+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:52:29.341 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:29.341 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:34.341 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:34.340+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:52:34.342 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:34.342 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:39.343 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:39.341+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:52:39.343 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:39.344 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:44.344 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:44.342+0000 7f6454ee3600 1 waiting for incremental sync to catch up: 2022-09-03T06:52:44.345 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:44.345 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:49.346 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:49.343+0000 7f6454ee3600 1 bucket sync caught up with source: 2022-09-03T06:52:49.347 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:49.347 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.49.5, , , , 00000000001.168.5, , , 00000000001.59.5, , 00000000001.50.5] 2022-09-03T06:52:49.347 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:49.343+0000 7f6454ee3600 0 bucket checkpoint complete 2022-09-03T06:52:49.368 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-71 2022-09-03T06:52:49.369 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:49.370 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:49.372 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:52:49.441 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:49.442 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:49.484 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:52:49.481+0000 7faa778ed600 1 bucket sync caught up with source: 2022-09-03T06:52:49.484 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.309.5, , , , 00000000001.359.5, , , 00000000001.353.5, , 00000000001.253.5] 2022-09-03T06:52:49.484 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.309.5, , , , 00000000001.359.5, , , 00000000001.353.5, , 00000000001.253.5] 2022-09-03T06:52:49.485 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:52:49.481+0000 7faa778ed600 0 bucket checkpoint complete 2022-09-03T06:52:49.503 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-71 2022-09-03T06:52:49.504 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:49.505 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:49.506 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:52:49.566 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:49.567 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:49.622 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:52:49.619+0000 7f66c73ab600 1 bucket sync caught up with source: 2022-09-03T06:52:49.622 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.309.5, , , , 00000000001.359.5, , , 00000000001.353.5, , 00000000001.253.5] 2022-09-03T06:52:49.622 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.309.5, , , , 00000000001.359.5, , , 00000000001.353.5, , 00000000001.253.5] 2022-09-03T06:52:49.623 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:52:49.619+0000 7f66c73ab600 0 bucket checkpoint complete 2022-09-03T06:52:49.640 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-71 2022-09-03T06:52:49.641 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-71 2022-09-03T06:52:49.642 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:49.643 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:49.643 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:52:49.702 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:49.702 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:49.743 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:52:49.741+0000 7f919c322600 1 bucket sync caught up with source: 2022-09-03T06:52:49.744 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.359.5, , , , 00000000001.375.5, , , 00000000001.153.5, , 00000000001.310.5] 2022-09-03T06:52:49.744 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.359.5, , , , 00000000001.375.5, , , 00000000001.153.5, , 00000000001.310.5] 2022-09-03T06:52:49.745 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:52:49.741+0000 7f919c322600 0 bucket checkpoint complete 2022-09-03T06:52:49.768 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-71 2022-09-03T06:52:49.769 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:49.769 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:49.770 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:52:49.838 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:49.838 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:49.895 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:49.893+0000 7f1626f3d600 1 bucket sync caught up with source: 2022-09-03T06:52:49.896 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.359.5, , , , 00000000001.375.5, , , 00000000001.153.5, , 00000000001.310.5] 2022-09-03T06:52:49.896 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.359.5, , , , 00000000001.375.5, , , 00000000001.153.5, , 00000000001.310.5] 2022-09-03T06:52:49.897 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:49.893+0000 7f1626f3d600 0 bucket checkpoint complete 2022-09-03T06:52:49.914 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-71 2022-09-03T06:52:49.915 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-71 zones={test-zone2, test-zone1} 2022-09-03T06:52:49.916 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:49.917 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:49.917 DEBUG:boto:Method: HEAD 2022-09-03T06:52:49.918 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:49.918 DEBUG:boto:Data: 2022-09-03T06:52:49.919 DEBUG:boto:Headers: {} 2022-09-03T06:52:49.919 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:49.920 DEBUG:boto:Port: 8000 2022-09-03T06:52:49.921 DEBUG:boto:Params: {} 2022-09-03T06:52:49.922 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:52:49.923 DEBUG:boto:Token: None 2022-09-03T06:52:49.924 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:49 GMT /avysmw-71/ 2022-09-03T06:52:49.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bjYZN6xudHuTiMFb49JY71/wQQ0= 2022-09-03T06:52:49.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bjYZN6xudHuTiMFb49JY71/wQQ0=', 'Content-Length': '0'} 2022-09-03T06:52:49.931 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c60d7a50257fa6c3-006312f9c1-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:49.932 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:49.933 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:49.933 DEBUG:boto:Method: HEAD 2022-09-03T06:52:49.934 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:49.934 DEBUG:boto:Data: 2022-09-03T06:52:49.935 DEBUG:boto:Headers: {} 2022-09-03T06:52:49.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:49.936 DEBUG:boto:Port: 8000 2022-09-03T06:52:49.937 DEBUG:boto:Params: {} 2022-09-03T06:52:49.937 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:52:49.938 DEBUG:boto:Token: None 2022-09-03T06:52:49.939 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:49 GMT /avysmw-71/ 2022-09-03T06:52:49.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bjYZN6xudHuTiMFb49JY71/wQQ0= 2022-09-03T06:52:49.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bjYZN6xudHuTiMFb49JY71/wQQ0=', 'Content-Length': '0'} 2022-09-03T06:52:49.943 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000fe3e69446dec5239-006312f9c1-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:49.944 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:52:49.945 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:49.945 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:49.946 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.946 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.947 DEBUG:boto:Method: GET 2022-09-03T06:52:49.947 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.948 DEBUG:boto:Data: 2022-09-03T06:52:49.948 DEBUG:boto:Headers: {} 2022-09-03T06:52:49.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:49.949 DEBUG:boto:Port: 8000 2022-09-03T06:52:49.950 DEBUG:boto:Params: {} 2022-09-03T06:52:49.951 DEBUG:boto:Token: None 2022-09-03T06:52:49.951 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:49 GMT /avysmw-71/?versions 2022-09-03T06:52:49.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dntDy5TxgX2vCrP3jVouL9AKX0= 2022-09-03T06:52:49.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dntDy5TxgX2vCrP3jVouL9AKX0=', 'Content-Length': '0'} 2022-09-03T06:52:49.955 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f61df8c5fafb09d2-006312f9c1-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:49.956 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:49.957 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:52:49.958 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:52:49.958 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:52:49.959 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:52:49.959 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:52:49.960 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:49.960 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:49.961 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.962 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.962 DEBUG:boto:Method: GET 2022-09-03T06:52:49.963 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.963 DEBUG:boto:Data: 2022-09-03T06:52:49.964 DEBUG:boto:Headers: {} 2022-09-03T06:52:49.964 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:49.965 DEBUG:boto:Port: 8000 2022-09-03T06:52:49.965 DEBUG:boto:Params: {} 2022-09-03T06:52:49.966 DEBUG:boto:Token: None 2022-09-03T06:52:49.967 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:49 GMT /avysmw-71/?versions 2022-09-03T06:52:49.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dntDy5TxgX2vCrP3jVouL9AKX0= 2022-09-03T06:52:49.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dntDy5TxgX2vCrP3jVouL9AKX0=', 'Content-Length': '0'} 2022-09-03T06:52:49.970 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002634b5e2044fe3d0-006312f9c1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:49.971 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:49.972 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:52:49.972 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:52:49.973 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:52:49.973 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:52:49.974 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:49.975 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:49.975 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.976 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.976 DEBUG:boto:Method: GET 2022-09-03T06:52:49.977 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.977 DEBUG:boto:Data: 2022-09-03T06:52:49.978 DEBUG:boto:Headers: {} 2022-09-03T06:52:49.978 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:49.979 DEBUG:boto:Port: 8000 2022-09-03T06:52:49.979 DEBUG:boto:Params: {} 2022-09-03T06:52:49.980 DEBUG:boto:Token: None 2022-09-03T06:52:49.981 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:49 GMT /avysmw-71/?versions 2022-09-03T06:52:49.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dntDy5TxgX2vCrP3jVouL9AKX0= 2022-09-03T06:52:49.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dntDy5TxgX2vCrP3jVouL9AKX0=', 'Content-Length': '0'} 2022-09-03T06:52:49.985 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000534a067774b66112-006312f9c1-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:49.986 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:49.987 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:49.987 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:49.988 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.988 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.989 DEBUG:boto:Method: GET 2022-09-03T06:52:49.989 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:49.990 DEBUG:boto:Data: 2022-09-03T06:52:49.990 DEBUG:boto:Headers: {} 2022-09-03T06:52:49.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:49.991 DEBUG:boto:Port: 8000 2022-09-03T06:52:49.992 DEBUG:boto:Params: {} 2022-09-03T06:52:49.993 DEBUG:boto:Token: None 2022-09-03T06:52:49.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:49 GMT /avysmw-71/?versions 2022-09-03T06:52:49.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dntDy5TxgX2vCrP3jVouL9AKX0= 2022-09-03T06:52:49.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dntDy5TxgX2vCrP3jVouL9AKX0=', 'Content-Length': '0'} 2022-09-03T06:52:49.997 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028f7ab9714668bf7-006312f9c1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:49.998 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:49.999 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:52:49.999 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.000 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.000 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.001 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.002 DEBUG:boto:Method: GET 2022-09-03T06:52:50.002 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:50.003 DEBUG:boto:Data: 2022-09-03T06:52:50.003 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.004 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.004 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.005 DEBUG:boto:Params: {} 2022-09-03T06:52:50.005 DEBUG:boto:Token: None 2022-09-03T06:52:50.006 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:50.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XBAgK4qT/As/zRu7cP+DlIPrOHU= 2022-09-03T06:52:50.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XBAgK4qT/As/zRu7cP+DlIPrOHU=', 'Content-Length': '0'} 2022-09-03T06:52:50.010 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075e997bc9e2d132d-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.010 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.011 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.012 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.012 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.013 DEBUG:boto:Method: GET 2022-09-03T06:52:50.013 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:50.014 DEBUG:boto:Data: 2022-09-03T06:52:50.014 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.015 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.016 DEBUG:boto:Params: {} 2022-09-03T06:52:50.017 DEBUG:boto:Token: None 2022-09-03T06:52:50.018 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:50.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XBAgK4qT/As/zRu7cP+DlIPrOHU= 2022-09-03T06:52:50.019 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XBAgK4qT/As/zRu7cP+DlIPrOHU=', 'Content-Length': '0'} 2022-09-03T06:52:50.021 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3643e35e6d306ee-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.022 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.022 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.023 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.023 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.024 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.024 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:50.025 DEBUG:boto:Data: 2022-09-03T06:52:50.025 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.026 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.027 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.027 DEBUG:boto:Params: {} 2022-09-03T06:52:50.028 DEBUG:boto:Token: None 2022-09-03T06:52:50.028 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:50.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xjStoFI9fjO0fHiSmA1eeKp8CgE= 2022-09-03T06:52:50.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xjStoFI9fjO0fHiSmA1eeKp8CgE=', 'Content-Length': '0'} 2022-09-03T06:52:50.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004a279b0fb3d0691-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.033 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.034 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.034 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.035 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.035 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.036 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:50.036 DEBUG:boto:Data: 2022-09-03T06:52:50.037 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.037 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.038 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.038 DEBUG:boto:Params: {} 2022-09-03T06:52:50.039 DEBUG:boto:Token: None 2022-09-03T06:52:50.040 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:50.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xjStoFI9fjO0fHiSmA1eeKp8CgE= 2022-09-03T06:52:50.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xjStoFI9fjO0fHiSmA1eeKp8CgE=', 'Content-Length': '0'} 2022-09-03T06:52:50.051 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe817119ce4ed6b5-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.052 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:52:50.053 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.054 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.055 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.056 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.056 DEBUG:boto:Method: GET 2022-09-03T06:52:50.057 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:50.057 DEBUG:boto:Data: 2022-09-03T06:52:50.058 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.059 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.060 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.061 DEBUG:boto:Params: {} 2022-09-03T06:52:50.062 DEBUG:boto:Token: None 2022-09-03T06:52:50.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:50.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XBAgK4qT/As/zRu7cP+DlIPrOHU= 2022-09-03T06:52:50.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XBAgK4qT/As/zRu7cP+DlIPrOHU=', 'Content-Length': '0'} 2022-09-03T06:52:50.068 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f23989c5f625efd4-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.070 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.071 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.071 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.072 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:50.073 DEBUG:boto:Method: GET 2022-09-03T06:52:50.074 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:50.075 DEBUG:boto:Data: 2022-09-03T06:52:50.076 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.077 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.078 DEBUG:boto:Params: {} 2022-09-03T06:52:50.079 DEBUG:boto:Token: None 2022-09-03T06:52:50.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:50.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XBAgK4qT/As/zRu7cP+DlIPrOHU= 2022-09-03T06:52:50.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XBAgK4qT/As/zRu7cP+DlIPrOHU=', 'Content-Length': '0'} 2022-09-03T06:52:50.083 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf85869f507fedf5-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.126 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.127 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.128 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:50.134 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:50.135 DEBUG:boto:Method: GET 2022-09-03T06:52:50.136 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:50.138 DEBUG:boto:Data: 2022-09-03T06:52:50.139 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.140 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.140 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.141 DEBUG:boto:Params: {} 2022-09-03T06:52:50.141 DEBUG:boto:Token: None 2022-09-03T06:52:50.142 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.149 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d2c523f3ce6b4d7-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.150 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.151 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.152 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.153 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:50.153 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:50.154 DEBUG:boto:Method: GET 2022-09-03T06:52:50.154 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:50.155 DEBUG:boto:Data: 2022-09-03T06:52:50.156 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.157 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.157 DEBUG:boto:Params: {} 2022-09-03T06:52:50.158 DEBUG:boto:Token: None 2022-09-03T06:52:50.158 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.163 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000181cc373b05535ea-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.164 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.165 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.166 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.166 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.167 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:50.168 DEBUG:boto:Data: 2022-09-03T06:52:50.168 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.170 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.171 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.172 DEBUG:boto:Params: {} 2022-09-03T06:52:50.173 DEBUG:boto:Token: None 2022-09-03T06:52:50.175 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a 2022-09-03T06:52:50.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:meWNpZ5yY1OHAFARM6gug9q+cXc= 2022-09-03T06:52:50.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:meWNpZ5yY1OHAFARM6gug9q+cXc=', 'Content-Length': '0'} 2022-09-03T06:52:50.182 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008fa40cd4f7ff1373-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.183 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.184 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.184 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.185 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:50.186 DEBUG:boto:Data: 2022-09-03T06:52:50.186 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.187 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.188 DEBUG:boto:Params: {} 2022-09-03T06:52:50.189 DEBUG:boto:Token: None 2022-09-03T06:52:50.189 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a 2022-09-03T06:52:50.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:meWNpZ5yY1OHAFARM6gug9q+cXc= 2022-09-03T06:52:50.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:meWNpZ5yY1OHAFARM6gug9q+cXc=', 'Content-Length': '0'} 2022-09-03T06:52:50.193 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b1cdade7a597a0d-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.195 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:52:50.196 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.197 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.198 DEBUG:boto:Method: GET 2022-09-03T06:52:50.199 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:50.200 DEBUG:boto:Data: 2022-09-03T06:52:50.201 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.202 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.203 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.205 DEBUG:boto:Params: {} 2022-09-03T06:52:50.206 DEBUG:boto:Token: None 2022-09-03T06:52:50.207 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a 2022-09-03T06:52:50.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pjL0nQUtNpefWXC9r5oyJQFKwqQ= 2022-09-03T06:52:50.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pjL0nQUtNpefWXC9r5oyJQFKwqQ=', 'Content-Length': '0'} 2022-09-03T06:52:50.211 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ac8fdba57f18d99-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.258 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:50.259 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:50.260 DEBUG:boto:Method: GET 2022-09-03T06:52:50.260 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:50.261 DEBUG:boto:Data: 2022-09-03T06:52:50.261 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.263 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.263 DEBUG:boto:Params: {} 2022-09-03T06:52:50.264 DEBUG:boto:Token: None 2022-09-03T06:52:50.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/a 2022-09-03T06:52:50.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pjL0nQUtNpefWXC9r5oyJQFKwqQ= 2022-09-03T06:52:50.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pjL0nQUtNpefWXC9r5oyJQFKwqQ=', 'Content-Length': '0'} 2022-09-03T06:52:50.268 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b72d342a14c55b38-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.269 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.270 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.270 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:50.271 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:50.271 DEBUG:boto:Method: GET 2022-09-03T06:52:50.272 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:50.272 DEBUG:boto:Data: 2022-09-03T06:52:50.273 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.274 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.275 DEBUG:boto:Params: {} 2022-09-03T06:52:50.276 DEBUG:boto:Token: None 2022-09-03T06:52:50.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.281 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006b45ef3d4ba514e8-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.282 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.284 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.285 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.286 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:50.287 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:50.288 DEBUG:boto:Method: GET 2022-09-03T06:52:50.289 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:50.290 DEBUG:boto:Data: 2022-09-03T06:52:50.291 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.292 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.293 DEBUG:boto:Params: {} 2022-09-03T06:52:50.295 DEBUG:boto:Token: None 2022-09-03T06:52:50.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.299 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c7ce2cd442a2dc7-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.300 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.302 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:52:50.303 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.304 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.305 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.306 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.307 DEBUG:boto:Method: GET 2022-09-03T06:52:50.308 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:50.309 DEBUG:boto:Data: 2022-09-03T06:52:50.310 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.311 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.312 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.313 DEBUG:boto:Params: {} 2022-09-03T06:52:50.314 DEBUG:boto:Token: None 2022-09-03T06:52:50.315 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:50.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Plz9uPXoRkuM9q0SrgL9CLQ3c+8= 2022-09-03T06:52:50.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Plz9uPXoRkuM9q0SrgL9CLQ3c+8=', 'Content-Length': '0'} 2022-09-03T06:52:50.319 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ddc7a7b6c68c93e0-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.321 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.322 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.324 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.325 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.325 DEBUG:boto:Method: GET 2022-09-03T06:52:50.326 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:50.327 DEBUG:boto:Data: 2022-09-03T06:52:50.328 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.328 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.329 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.329 DEBUG:boto:Params: {} 2022-09-03T06:52:50.330 DEBUG:boto:Token: None 2022-09-03T06:52:50.331 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:50.332 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Plz9uPXoRkuM9q0SrgL9CLQ3c+8= 2022-09-03T06:52:50.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Plz9uPXoRkuM9q0SrgL9CLQ3c+8=', 'Content-Length': '0'} 2022-09-03T06:52:50.335 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078a9c88e42b6a8b4-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.382 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.383 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.383 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.385 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.385 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.386 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:50.387 DEBUG:boto:Data: 2022-09-03T06:52:50.387 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.388 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.388 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.389 DEBUG:boto:Params: {} 2022-09-03T06:52:50.390 DEBUG:boto:Token: None 2022-09-03T06:52:50.391 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:50.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:taSpNQ3AeJZo2CGdaI8IBkJDoY8= 2022-09-03T06:52:50.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:taSpNQ3AeJZo2CGdaI8IBkJDoY8=', 'Content-Length': '0'} 2022-09-03T06:52:50.394 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b96ef0bb5f49b6b-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.395 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.396 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.397 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.397 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.398 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.398 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:50.399 DEBUG:boto:Data: 2022-09-03T06:52:50.399 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.400 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.401 DEBUG:boto:Params: {} 2022-09-03T06:52:50.401 DEBUG:boto:Token: None 2022-09-03T06:52:50.402 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:50.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:taSpNQ3AeJZo2CGdaI8IBkJDoY8= 2022-09-03T06:52:50.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:taSpNQ3AeJZo2CGdaI8IBkJDoY8=', 'Content-Length': '0'} 2022-09-03T06:52:50.406 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c511813eec3f2f2-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.407 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:52:50.407 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.408 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.408 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.409 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.409 DEBUG:boto:Method: GET 2022-09-03T06:52:50.410 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:50.411 DEBUG:boto:Data: 2022-09-03T06:52:50.411 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.412 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.412 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.413 DEBUG:boto:Params: {} 2022-09-03T06:52:50.413 DEBUG:boto:Token: None 2022-09-03T06:52:50.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:50.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Plz9uPXoRkuM9q0SrgL9CLQ3c+8= 2022-09-03T06:52:50.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Plz9uPXoRkuM9q0SrgL9CLQ3c+8=', 'Content-Length': '0'} 2022-09-03T06:52:50.419 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae3d771c51bac3a7-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.420 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.420 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.421 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.422 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:50.423 DEBUG:boto:Method: GET 2022-09-03T06:52:50.423 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:50.424 DEBUG:boto:Data: 2022-09-03T06:52:50.424 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.425 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.426 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.428 DEBUG:boto:Params: {} 2022-09-03T06:52:50.429 DEBUG:boto:Token: None 2022-09-03T06:52:50.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:50.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Plz9uPXoRkuM9q0SrgL9CLQ3c+8= 2022-09-03T06:52:50.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Plz9uPXoRkuM9q0SrgL9CLQ3c+8=', 'Content-Length': '0'} 2022-09-03T06:52:50.434 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfff446d3fd0f458-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.435 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.436 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.436 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:50.438 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:50.438 DEBUG:boto:Method: GET 2022-09-03T06:52:50.439 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:50.439 DEBUG:boto:Data: 2022-09-03T06:52:50.440 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.440 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.441 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.441 DEBUG:boto:Params: {} 2022-09-03T06:52:50.442 DEBUG:boto:Token: None 2022-09-03T06:52:50.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.448 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c19e048bd403c84-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.490 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.493 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.494 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.495 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:50.496 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:50.497 DEBUG:boto:Method: GET 2022-09-03T06:52:50.498 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:50.499 DEBUG:boto:Data: 2022-09-03T06:52:50.500 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.501 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.502 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.504 DEBUG:boto:Params: {} 2022-09-03T06:52:50.505 DEBUG:boto:Token: None 2022-09-03T06:52:50.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.511 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000089b77310aeed1a4-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.512 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.514 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.515 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.516 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.517 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:50.518 DEBUG:boto:Data: 2022-09-03T06:52:50.519 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.520 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.521 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.521 DEBUG:boto:Params: {} 2022-09-03T06:52:50.522 DEBUG:boto:Token: None 2022-09-03T06:52:50.523 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b 2022-09-03T06:52:50.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nAksjNElXLK3XeCdVAv3i/8xEYU= 2022-09-03T06:52:50.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nAksjNElXLK3XeCdVAv3i/8xEYU=', 'Content-Length': '0'} 2022-09-03T06:52:50.526 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000606038b118d4a143-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.527 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.527 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.528 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.528 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:50.529 DEBUG:boto:Data: 2022-09-03T06:52:50.529 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.530 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.530 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.531 DEBUG:boto:Params: {} 2022-09-03T06:52:50.532 DEBUG:boto:Token: None 2022-09-03T06:52:50.532 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b 2022-09-03T06:52:50.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nAksjNElXLK3XeCdVAv3i/8xEYU= 2022-09-03T06:52:50.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nAksjNElXLK3XeCdVAv3i/8xEYU=', 'Content-Length': '0'} 2022-09-03T06:52:50.535 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086d1512f150eda6a-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.536 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:52:50.537 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.537 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.538 DEBUG:boto:Method: GET 2022-09-03T06:52:50.538 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:50.539 DEBUG:boto:Data: 2022-09-03T06:52:50.540 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.541 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.542 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.543 DEBUG:boto:Params: {} 2022-09-03T06:52:50.544 DEBUG:boto:Token: None 2022-09-03T06:52:50.545 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b 2022-09-03T06:52:50.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XIdi24Sfka6cixjsEL/2w9+LB+8= 2022-09-03T06:52:50.547 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XIdi24Sfka6cixjsEL/2w9+LB+8=', 'Content-Length': '0'} 2022-09-03T06:52:50.549 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000010eaca3d095d47e-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.551 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:50.551 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:50.552 DEBUG:boto:Method: GET 2022-09-03T06:52:50.553 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:50.554 DEBUG:boto:Data: 2022-09-03T06:52:50.555 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.557 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.558 DEBUG:boto:Params: {} 2022-09-03T06:52:50.559 DEBUG:boto:Token: None 2022-09-03T06:52:50.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/b 2022-09-03T06:52:50.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XIdi24Sfka6cixjsEL/2w9+LB+8= 2022-09-03T06:52:50.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XIdi24Sfka6cixjsEL/2w9+LB+8=', 'Content-Length': '0'} 2022-09-03T06:52:50.565 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000786c7fafd4b5bca4-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.566 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.567 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.568 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:50.569 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:50.570 DEBUG:boto:Method: GET 2022-09-03T06:52:50.571 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:50.572 DEBUG:boto:Data: 2022-09-03T06:52:50.573 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.573 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.574 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.574 DEBUG:boto:Params: {} 2022-09-03T06:52:50.575 DEBUG:boto:Token: None 2022-09-03T06:52:50.576 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.582 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007e88ccce945aeadf-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.626 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.631 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.633 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.634 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:50.635 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:50.635 DEBUG:boto:Method: GET 2022-09-03T06:52:50.636 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:50.636 DEBUG:boto:Data: 2022-09-03T06:52:50.637 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.638 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.644 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.645 DEBUG:boto:Params: {} 2022-09-03T06:52:50.648 DEBUG:boto:Token: None 2022-09-03T06:52:50.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.656 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ed1451bbedc11d3-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.666 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.667 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:52:50.668 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.669 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.670 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.671 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.672 DEBUG:boto:Method: GET 2022-09-03T06:52:50.673 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:50.674 DEBUG:boto:Data: 2022-09-03T06:52:50.675 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.676 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.677 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.684 DEBUG:boto:Params: {} 2022-09-03T06:52:50.686 DEBUG:boto:Token: None 2022-09-03T06:52:50.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:50.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1R1e8urTtwYCOS0N5iWXwtGk3I= 2022-09-03T06:52:50.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1R1e8urTtwYCOS0N5iWXwtGk3I=', 'Content-Length': '0'} 2022-09-03T06:52:50.692 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000844bc7fa853f5127-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.694 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.695 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.696 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.697 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.698 DEBUG:boto:Method: GET 2022-09-03T06:52:50.699 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:50.700 DEBUG:boto:Data: 2022-09-03T06:52:50.701 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.702 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.703 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.704 DEBUG:boto:Params: {} 2022-09-03T06:52:50.706 DEBUG:boto:Token: None 2022-09-03T06:52:50.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:50.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1R1e8urTtwYCOS0N5iWXwtGk3I= 2022-09-03T06:52:50.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1R1e8urTtwYCOS0N5iWXwtGk3I=', 'Content-Length': '0'} 2022-09-03T06:52:50.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f05699adc9b03ef5-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.723 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.724 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.725 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.726 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.727 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.728 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:50.729 DEBUG:boto:Data: 2022-09-03T06:52:50.730 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.731 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.732 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.733 DEBUG:boto:Params: {} 2022-09-03T06:52:50.736 DEBUG:boto:Token: None 2022-09-03T06:52:50.737 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:50.738 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+7UCXRJ/PdS31jRYINHSbPFypx8= 2022-09-03T06:52:50.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+7UCXRJ/PdS31jRYINHSbPFypx8=', 'Content-Length': '0'} 2022-09-03T06:52:50.746 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002fcaa98f0559e4a7-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.748 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.749 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.750 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.751 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.752 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.753 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:50.753 DEBUG:boto:Data: 2022-09-03T06:52:50.755 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.757 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.758 DEBUG:boto:Params: {} 2022-09-03T06:52:50.759 DEBUG:boto:Token: None 2022-09-03T06:52:50.760 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:50.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+7UCXRJ/PdS31jRYINHSbPFypx8= 2022-09-03T06:52:50.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+7UCXRJ/PdS31jRYINHSbPFypx8=', 'Content-Length': '0'} 2022-09-03T06:52:50.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d1990d3d6807b3c2-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.767 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:52:50.768 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.769 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.770 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.771 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.772 DEBUG:boto:Method: GET 2022-09-03T06:52:50.774 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:50.778 DEBUG:boto:Data: 2022-09-03T06:52:50.779 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.779 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.780 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.780 DEBUG:boto:Params: {} 2022-09-03T06:52:50.781 DEBUG:boto:Token: None 2022-09-03T06:52:50.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:50.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1R1e8urTtwYCOS0N5iWXwtGk3I= 2022-09-03T06:52:50.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1R1e8urTtwYCOS0N5iWXwtGk3I=', 'Content-Length': '0'} 2022-09-03T06:52:50.792 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005529276b2d12c586-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.792 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.793 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.794 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.794 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:50.795 DEBUG:boto:Method: GET 2022-09-03T06:52:50.795 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:50.796 DEBUG:boto:Data: 2022-09-03T06:52:50.796 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.797 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.797 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.798 DEBUG:boto:Params: {} 2022-09-03T06:52:50.807 DEBUG:boto:Token: None 2022-09-03T06:52:50.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:50.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1R1e8urTtwYCOS0N5iWXwtGk3I= 2022-09-03T06:52:50.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1R1e8urTtwYCOS0N5iWXwtGk3I=', 'Content-Length': '0'} 2022-09-03T06:52:50.813 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023fef7c2607c8c4b-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.815 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.817 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.819 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:50.820 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:50.821 DEBUG:boto:Method: GET 2022-09-03T06:52:50.822 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:50.823 DEBUG:boto:Data: 2022-09-03T06:52:50.824 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.825 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.826 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.832 DEBUG:boto:Params: {} 2022-09-03T06:52:50.833 DEBUG:boto:Token: None 2022-09-03T06:52:50.834 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.849 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005b9ec6bc3b994c90-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.850 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.851 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.851 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.852 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:50.853 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:50.853 DEBUG:boto:Method: GET 2022-09-03T06:52:50.854 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:50.854 DEBUG:boto:Data: 2022-09-03T06:52:50.855 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.855 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.856 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.856 DEBUG:boto:Params: {} 2022-09-03T06:52:50.857 DEBUG:boto:Token: None 2022-09-03T06:52:50.858 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.861 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005d19765a70edebc0-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.862 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.863 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.864 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.864 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.865 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:50.865 DEBUG:boto:Data: 2022-09-03T06:52:50.866 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.866 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.867 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.867 DEBUG:boto:Params: {} 2022-09-03T06:52:50.868 DEBUG:boto:Token: None 2022-09-03T06:52:50.869 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c 2022-09-03T06:52:50.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xYRw1xLZQvPzC8sRnUrb3Fu35/c= 2022-09-03T06:52:50.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xYRw1xLZQvPzC8sRnUrb3Fu35/c=', 'Content-Length': '0'} 2022-09-03T06:52:50.872 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009cd8a745343b5704-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.873 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.873 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.874 DEBUG:boto:Method: HEAD 2022-09-03T06:52:50.874 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:50.875 DEBUG:boto:Data: 2022-09-03T06:52:50.875 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.876 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.876 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.877 DEBUG:boto:Params: {} 2022-09-03T06:52:50.877 DEBUG:boto:Token: None 2022-09-03T06:52:50.878 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c 2022-09-03T06:52:50.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xYRw1xLZQvPzC8sRnUrb3Fu35/c= 2022-09-03T06:52:50.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xYRw1xLZQvPzC8sRnUrb3Fu35/c=', 'Content-Length': '0'} 2022-09-03T06:52:50.881 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006571a9ad7cb084f3-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.882 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:52:50.883 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.884 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.884 DEBUG:boto:Method: GET 2022-09-03T06:52:50.885 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:50.885 DEBUG:boto:Data: 2022-09-03T06:52:50.886 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.887 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.887 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.888 DEBUG:boto:Params: {} 2022-09-03T06:52:50.888 DEBUG:boto:Token: None 2022-09-03T06:52:50.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c 2022-09-03T06:52:50.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wbXtZ5R4OnbQcnwMsH46DAD8XFs= 2022-09-03T06:52:50.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wbXtZ5R4OnbQcnwMsH46DAD8XFs=', 'Content-Length': '0'} 2022-09-03T06:52:50.893 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eee0aaf604711445-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.894 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:50.895 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:50.895 DEBUG:boto:Method: GET 2022-09-03T06:52:50.896 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:50.896 DEBUG:boto:Data: 2022-09-03T06:52:50.897 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.897 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.898 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.898 DEBUG:boto:Params: {} 2022-09-03T06:52:50.899 DEBUG:boto:Token: None 2022-09-03T06:52:50.900 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/c 2022-09-03T06:52:50.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wbXtZ5R4OnbQcnwMsH46DAD8XFs= 2022-09-03T06:52:50.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wbXtZ5R4OnbQcnwMsH46DAD8XFs=', 'Content-Length': '0'} 2022-09-03T06:52:50.903 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e496d55542c7e62d-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.904 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.904 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.905 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:50.905 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:50.906 DEBUG:boto:Method: GET 2022-09-03T06:52:50.907 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:50.907 DEBUG:boto:Data: 2022-09-03T06:52:50.908 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.908 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.909 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.909 DEBUG:boto:Params: {} 2022-09-03T06:52:50.910 DEBUG:boto:Token: None 2022-09-03T06:52:50.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.914 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000026bf8280b32425af-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.915 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.916 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:50.917 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:50.918 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:50.919 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:50.920 DEBUG:boto:Method: GET 2022-09-03T06:52:50.921 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:50.922 DEBUG:boto:Data: 2022-09-03T06:52:50.923 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:50.925 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.925 DEBUG:boto:Params: {} 2022-09-03T06:52:50.926 DEBUG:boto:Token: None 2022-09-03T06:52:50.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/ 2022-09-03T06:52:50.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs= 2022-09-03T06:52:50.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5QvqGzL2GT8YGIdmBbUzvRqqJs=', 'Content-Length': '0'} 2022-09-03T06:52:50.932 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000625f1ddf1bc96c9b-006312f9c2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.974 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:50.975 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:52:50.976 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:50.977 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:50.978 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:50.978 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:50.979 DEBUG:boto:Method: GET 2022-09-03T06:52:50.980 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:50.981 DEBUG:boto:Data: 2022-09-03T06:52:50.981 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.982 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:50.983 DEBUG:boto:Port: 8000 2022-09-03T06:52:50.984 DEBUG:boto:Params: {} 2022-09-03T06:52:50.985 DEBUG:boto:Token: None 2022-09-03T06:52:50.985 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:50 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:50.986 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G9JMlOHTX0sEag/zaULHN30T+i0= 2022-09-03T06:52:50.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G9JMlOHTX0sEag/zaULHN30T+i0=', 'Content-Length': '0'} 2022-09-03T06:52:50.990 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089d871911ab05fd9-006312f9c2-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:50.991 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:50.992 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:50.993 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:50.995 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:50.996 DEBUG:boto:Method: GET 2022-09-03T06:52:50.997 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:50.998 DEBUG:boto:Data: 2022-09-03T06:52:50.999 DEBUG:boto:Headers: {} 2022-09-03T06:52:50.999 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.000 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.001 DEBUG:boto:Params: {} 2022-09-03T06:52:51.003 DEBUG:boto:Token: None 2022-09-03T06:52:51.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:51.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sqp+tC/58K6CdLCczEj6CT7R2IE= 2022-09-03T06:52:51.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sqp+tC/58K6CdLCczEj6CT7R2IE=', 'Content-Length': '0'} 2022-09-03T06:52:51.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000712f89380af70e87-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.012 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:51.013 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:51.014 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:51.015 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:51.016 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.017 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:51.018 DEBUG:boto:Data: 2022-09-03T06:52:51.019 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.020 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:51.021 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.022 DEBUG:boto:Params: {} 2022-09-03T06:52:51.023 DEBUG:boto:Token: None 2022-09-03T06:52:51.024 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:51.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GjvWpCBwJ3gIIZ1WKy2KyhEDD2I= 2022-09-03T06:52:51.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GjvWpCBwJ3gIIZ1WKy2KyhEDD2I=', 'Content-Length': '0'} 2022-09-03T06:52:51.029 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd789acdfc17ffad-006312f9c3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.030 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:51.031 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:51.032 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:51.033 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:51.034 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.035 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:51.036 DEBUG:boto:Data: 2022-09-03T06:52:51.037 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.039 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.040 DEBUG:boto:Params: {} 2022-09-03T06:52:51.041 DEBUG:boto:Token: None 2022-09-03T06:52:51.042 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:51.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GjvWpCBwJ3gIIZ1WKy2KyhEDD2I= 2022-09-03T06:52:51.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GjvWpCBwJ3gIIZ1WKy2KyhEDD2I=', 'Content-Length': '0'} 2022-09-03T06:52:51.049 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc00aaaecf8c03ae-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.051 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:52:51.052 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:51.053 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:51.054 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:51.054 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:51.055 DEBUG:boto:Method: GET 2022-09-03T06:52:51.056 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:51.057 DEBUG:boto:Data: 2022-09-03T06:52:51.058 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.059 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:51.060 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.061 DEBUG:boto:Params: {} 2022-09-03T06:52:51.062 DEBUG:boto:Token: None 2022-09-03T06:52:51.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:51.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sqp+tC/58K6CdLCczEj6CT7R2IE= 2022-09-03T06:52:51.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sqp+tC/58K6CdLCczEj6CT7R2IE=', 'Content-Length': '0'} 2022-09-03T06:52:51.074 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ef1eb40fd5107a6-006312f9c3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.076 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:51.077 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:51.078 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:51.079 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:51.080 DEBUG:boto:Method: GET 2022-09-03T06:52:51.081 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:51.088 DEBUG:boto:Data: 2022-09-03T06:52:51.089 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.089 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.090 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.090 DEBUG:boto:Params: {} 2022-09-03T06:52:51.091 DEBUG:boto:Token: None 2022-09-03T06:52:51.092 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:51.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sqp+tC/58K6CdLCczEj6CT7R2IE= 2022-09-03T06:52:51.093 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sqp+tC/58K6CdLCczEj6CT7R2IE=', 'Content-Length': '0'} 2022-09-03T06:52:51.097 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbd23d95604d2227-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.139 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.140 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.141 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:51.141 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:51.142 DEBUG:boto:Method: GET 2022-09-03T06:52:51.142 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:51.143 DEBUG:boto:Data: 2022-09-03T06:52:51.144 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.144 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:51.145 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.146 DEBUG:boto:Params: {} 2022-09-03T06:52:51.148 DEBUG:boto:Token: None 2022-09-03T06:52:51.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.151 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.152 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.155 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007ea428dd1c5ef2ac-006312f9c3-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.157 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.158 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.159 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.160 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:51.160 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:51.162 DEBUG:boto:Method: GET 2022-09-03T06:52:51.163 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:51.164 DEBUG:boto:Data: 2022-09-03T06:52:51.164 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.166 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.166 DEBUG:boto:Params: {} 2022-09-03T06:52:51.167 DEBUG:boto:Token: None 2022-09-03T06:52:51.168 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.172 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001431281d7dbd0763-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.173 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.174 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:51.175 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:51.175 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.176 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:51.176 DEBUG:boto:Data: 2022-09-03T06:52:51.177 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.178 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:51.178 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.179 DEBUG:boto:Params: {} 2022-09-03T06:52:51.179 DEBUG:boto:Token: None 2022-09-03T06:52:51.180 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/d 2022-09-03T06:52:51.181 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D7egGmdFEyo9Ldi+2GYg3uGtLRw= 2022-09-03T06:52:51.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D7egGmdFEyo9Ldi+2GYg3uGtLRw=', 'Content-Length': '0'} 2022-09-03T06:52:51.185 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007cc482d91fc4e798-006312f9c3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.186 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:51.187 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:51.188 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.189 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:51.190 DEBUG:boto:Data: 2022-09-03T06:52:51.190 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.191 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.191 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.192 DEBUG:boto:Params: {} 2022-09-03T06:52:51.193 DEBUG:boto:Token: None 2022-09-03T06:52:51.194 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/d 2022-09-03T06:52:51.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D7egGmdFEyo9Ldi+2GYg3uGtLRw= 2022-09-03T06:52:51.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D7egGmdFEyo9Ldi+2GYg3uGtLRw=', 'Content-Length': '0'} 2022-09-03T06:52:51.202 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e7a8a80244bd5655-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.203 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:52:51.206 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:51.206 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:51.207 DEBUG:boto:Method: GET 2022-09-03T06:52:51.208 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:51.208 DEBUG:boto:Data: 2022-09-03T06:52:51.209 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.210 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:51.210 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.211 DEBUG:boto:Params: {} 2022-09-03T06:52:51.211 DEBUG:boto:Token: None 2022-09-03T06:52:51.212 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/d 2022-09-03T06:52:51.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gmj7oz5ITTtoqoRKyP6dRYlzik8= 2022-09-03T06:52:51.214 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gmj7oz5ITTtoqoRKyP6dRYlzik8=', 'Content-Length': '0'} 2022-09-03T06:52:51.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006999b8142fa32657-006312f9c3-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.265 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:51.266 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:51.267 DEBUG:boto:Method: GET 2022-09-03T06:52:51.268 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:51.269 DEBUG:boto:Data: 2022-09-03T06:52:51.270 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.272 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.273 DEBUG:boto:Params: {} 2022-09-03T06:52:51.274 DEBUG:boto:Token: None 2022-09-03T06:52:51.276 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/d 2022-09-03T06:52:51.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gmj7oz5ITTtoqoRKyP6dRYlzik8= 2022-09-03T06:52:51.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gmj7oz5ITTtoqoRKyP6dRYlzik8=', 'Content-Length': '0'} 2022-09-03T06:52:51.281 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e349d1fb79ce2079-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.282 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.284 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.285 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:51.286 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:51.287 DEBUG:boto:Method: GET 2022-09-03T06:52:51.287 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:51.288 DEBUG:boto:Data: 2022-09-03T06:52:51.289 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.294 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:51.295 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.295 DEBUG:boto:Params: {} 2022-09-03T06:52:51.296 DEBUG:boto:Token: None 2022-09-03T06:52:51.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.303 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a02a41026b7d820d-006312f9c3-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.304 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.305 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.306 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.307 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:51.307 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:51.308 DEBUG:boto:Method: GET 2022-09-03T06:52:51.309 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:51.310 DEBUG:boto:Data: 2022-09-03T06:52:51.311 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.313 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.314 DEBUG:boto:Params: {} 2022-09-03T06:52:51.315 DEBUG:boto:Token: None 2022-09-03T06:52:51.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.321 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f82dede2be05bd52-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.322 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.324 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-71 zones={test-zone2, test-zone1} 2022-09-03T06:52:51.326 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-71 zones={test-zone3, test-zone1} 2022-09-03T06:52:51.327 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.328 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.329 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.329 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:51.330 DEBUG:boto:Data: 2022-09-03T06:52:51.332 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.333 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.334 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.335 DEBUG:boto:Params: {} 2022-09-03T06:52:51.335 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:52:51.336 DEBUG:boto:Token: None 2022-09-03T06:52:51.337 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s21XcgRpPke4AO2ySSHrVJBirBI= 2022-09-03T06:52:51.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s21XcgRpPke4AO2ySSHrVJBirBI=', 'Content-Length': '0'} 2022-09-03T06:52:51.343 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000011174d4edd4c1904-006312f9c3-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.344 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.344 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.345 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.345 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:51.346 DEBUG:boto:Data: 2022-09-03T06:52:51.347 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.347 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.348 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.349 DEBUG:boto:Params: {} 2022-09-03T06:52:51.349 DEBUG:boto:Token: None 2022-09-03T06:52:51.350 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s21XcgRpPke4AO2ySSHrVJBirBI= 2022-09-03T06:52:51.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s21XcgRpPke4AO2ySSHrVJBirBI=', 'Content-Length': '0'} 2022-09-03T06:52:51.355 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bf9c4773dbbfd543-006312f9c3-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.356 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:52:51.360 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.361 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.363 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.364 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.365 DEBUG:boto:Method: GET 2022-09-03T06:52:51.365 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.367 DEBUG:boto:Data: 2022-09-03T06:52:51.368 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.369 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.370 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.371 DEBUG:boto:Params: {} 2022-09-03T06:52:51.372 DEBUG:boto:Token: None 2022-09-03T06:52:51.373 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/?versions 2022-09-03T06:52:51.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tXHuNnZ5ro2chkS5Qlsb/n/5a7o= 2022-09-03T06:52:51.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tXHuNnZ5ro2chkS5Qlsb/n/5a7o=', 'Content-Length': '0'} 2022-09-03T06:52:51.380 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b043d49e296b11a9-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.381 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.383 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:52:51.384 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:52:51.385 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:52:51.386 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:52:51.387 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:52:51.388 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.389 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.392 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.393 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.393 DEBUG:boto:Method: GET 2022-09-03T06:52:51.394 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.395 DEBUG:boto:Data: 2022-09-03T06:52:51.396 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.397 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.398 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.398 DEBUG:boto:Params: {} 2022-09-03T06:52:51.399 DEBUG:boto:Token: None 2022-09-03T06:52:51.400 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/?versions 2022-09-03T06:52:51.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tXHuNnZ5ro2chkS5Qlsb/n/5a7o= 2022-09-03T06:52:51.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tXHuNnZ5ro2chkS5Qlsb/n/5a7o=', 'Content-Length': '0'} 2022-09-03T06:52:51.406 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d72ce25daad2f660-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.407 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.409 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:52:51.410 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:52:51.411 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:52:51.411 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:52:51.412 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.414 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.414 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.415 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.415 DEBUG:boto:Method: GET 2022-09-03T06:52:51.416 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.417 DEBUG:boto:Data: 2022-09-03T06:52:51.417 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.424 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.424 DEBUG:boto:Params: {} 2022-09-03T06:52:51.425 DEBUG:boto:Token: None 2022-09-03T06:52:51.426 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/?versions 2022-09-03T06:52:51.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tXHuNnZ5ro2chkS5Qlsb/n/5a7o= 2022-09-03T06:52:51.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tXHuNnZ5ro2chkS5Qlsb/n/5a7o=', 'Content-Length': '0'} 2022-09-03T06:52:51.431 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f84b4bc6cd8e8d58-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.432 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.434 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.434 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.435 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.435 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.436 DEBUG:boto:Method: GET 2022-09-03T06:52:51.436 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:51.437 DEBUG:boto:Data: 2022-09-03T06:52:51.437 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.438 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.439 DEBUG:boto:Params: {} 2022-09-03T06:52:51.439 DEBUG:boto:Token: None 2022-09-03T06:52:51.440 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/?versions 2022-09-03T06:52:51.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tXHuNnZ5ro2chkS5Qlsb/n/5a7o= 2022-09-03T06:52:51.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tXHuNnZ5ro2chkS5Qlsb/n/5a7o=', 'Content-Length': '0'} 2022-09-03T06:52:51.449 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000afc3ae13f0a74f42-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.450 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.451 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:52:51.452 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.452 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.456 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.457 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.458 DEBUG:boto:Method: GET 2022-09-03T06:52:51.458 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:51.459 DEBUG:boto:Data: 2022-09-03T06:52:51.459 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.461 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.462 DEBUG:boto:Params: {} 2022-09-03T06:52:51.462 DEBUG:boto:Token: None 2022-09-03T06:52:51.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:51.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iMlnAMSBtKLSecQUGUFQJnySKuY= 2022-09-03T06:52:51.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iMlnAMSBtKLSecQUGUFQJnySKuY=', 'Content-Length': '0'} 2022-09-03T06:52:51.471 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036ad9321a0776790-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.472 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.472 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.473 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.473 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.474 DEBUG:boto:Method: GET 2022-09-03T06:52:51.475 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:51.475 DEBUG:boto:Data: 2022-09-03T06:52:51.476 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.477 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.478 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.478 DEBUG:boto:Params: {} 2022-09-03T06:52:51.479 DEBUG:boto:Token: None 2022-09-03T06:52:51.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:51.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iMlnAMSBtKLSecQUGUFQJnySKuY= 2022-09-03T06:52:51.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iMlnAMSBtKLSecQUGUFQJnySKuY=', 'Content-Length': '0'} 2022-09-03T06:52:51.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000040d07b9d6b865b75-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.484 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.485 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.485 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.486 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.486 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.487 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:51.487 DEBUG:boto:Data: 2022-09-03T06:52:51.488 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.489 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.490 DEBUG:boto:Params: {} 2022-09-03T06:52:51.490 DEBUG:boto:Token: None 2022-09-03T06:52:51.491 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:51.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wMDwGXTw0ILwOzX0k6RDnwQhSg= 2022-09-03T06:52:51.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wMDwGXTw0ILwOzX0k6RDnwQhSg=', 'Content-Length': '0'} 2022-09-03T06:52:51.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f85d530f8c90b54-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.496 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.497 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.498 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.499 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.499 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.500 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:51.500 DEBUG:boto:Data: 2022-09-03T06:52:51.501 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.501 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.502 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.502 DEBUG:boto:Params: {} 2022-09-03T06:52:51.503 DEBUG:boto:Token: None 2022-09-03T06:52:51.504 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:51.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wMDwGXTw0ILwOzX0k6RDnwQhSg= 2022-09-03T06:52:51.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wMDwGXTw0ILwOzX0k6RDnwQhSg=', 'Content-Length': '0'} 2022-09-03T06:52:51.508 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000059d53962940160b6-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.509 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:52:51.510 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.511 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.512 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.513 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.514 DEBUG:boto:Method: GET 2022-09-03T06:52:51.514 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:51.515 DEBUG:boto:Data: 2022-09-03T06:52:51.516 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.517 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.517 DEBUG:boto:Params: {} 2022-09-03T06:52:51.518 DEBUG:boto:Token: None 2022-09-03T06:52:51.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:51.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iMlnAMSBtKLSecQUGUFQJnySKuY= 2022-09-03T06:52:51.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iMlnAMSBtKLSecQUGUFQJnySKuY=', 'Content-Length': '0'} 2022-09-03T06:52:51.522 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d049485fb554936-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.523 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.524 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.525 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.525 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:51.526 DEBUG:boto:Method: GET 2022-09-03T06:52:51.526 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:51.527 DEBUG:boto:Data: 2022-09-03T06:52:51.527 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.528 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.528 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.529 DEBUG:boto:Params: {} 2022-09-03T06:52:51.530 DEBUG:boto:Token: None 2022-09-03T06:52:51.530 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:51.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iMlnAMSBtKLSecQUGUFQJnySKuY= 2022-09-03T06:52:51.532 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iMlnAMSBtKLSecQUGUFQJnySKuY=', 'Content-Length': '0'} 2022-09-03T06:52:51.534 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000624c092952ae9cc-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.536 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.536 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.537 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:51.538 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:51.538 DEBUG:boto:Method: GET 2022-09-03T06:52:51.539 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:51.540 DEBUG:boto:Data: 2022-09-03T06:52:51.540 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.541 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.542 DEBUG:boto:Params: {} 2022-09-03T06:52:51.543 DEBUG:boto:Token: None 2022-09-03T06:52:51.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.548 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000082d70b765d0903db-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.591 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.593 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.594 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.598 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:51.599 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:51.600 DEBUG:boto:Method: GET 2022-09-03T06:52:51.601 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:51.602 DEBUG:boto:Data: 2022-09-03T06:52:51.602 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.604 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.604 DEBUG:boto:Params: {} 2022-09-03T06:52:51.605 DEBUG:boto:Token: None 2022-09-03T06:52:51.606 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.612 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d8a9cd35853ec5d-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.613 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.615 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.616 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.617 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.618 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:51.619 DEBUG:boto:Data: 2022-09-03T06:52:51.620 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.621 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.623 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.624 DEBUG:boto:Params: {} 2022-09-03T06:52:51.625 DEBUG:boto:Token: None 2022-09-03T06:52:51.626 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a 2022-09-03T06:52:51.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lex/FjvR0XBugy2UH1MU+QgsLFk= 2022-09-03T06:52:51.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lex/FjvR0XBugy2UH1MU+QgsLFk=', 'Content-Length': '0'} 2022-09-03T06:52:51.631 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008200655a47fcaae7-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.632 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.633 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.634 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.635 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:51.637 DEBUG:boto:Data: 2022-09-03T06:52:51.638 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.640 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.641 DEBUG:boto:Params: {} 2022-09-03T06:52:51.643 DEBUG:boto:Token: None 2022-09-03T06:52:51.645 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a 2022-09-03T06:52:51.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lex/FjvR0XBugy2UH1MU+QgsLFk= 2022-09-03T06:52:51.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lex/FjvR0XBugy2UH1MU+QgsLFk=', 'Content-Length': '0'} 2022-09-03T06:52:51.651 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004bbe979c6cae3d07-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.652 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:52:51.652 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.653 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.653 DEBUG:boto:Method: GET 2022-09-03T06:52:51.654 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:51.655 DEBUG:boto:Data: 2022-09-03T06:52:51.655 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.656 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.656 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.657 DEBUG:boto:Params: {} 2022-09-03T06:52:51.657 DEBUG:boto:Token: None 2022-09-03T06:52:51.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a 2022-09-03T06:52:51.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fj0iX7PN9ga6JD7AU34uhicMcHg= 2022-09-03T06:52:51.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fj0iX7PN9ga6JD7AU34uhicMcHg=', 'Content-Length': '0'} 2022-09-03T06:52:51.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062cb5e1b1ff2b262-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.664 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:51.664 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:51.665 DEBUG:boto:Method: GET 2022-09-03T06:52:51.666 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:51.666 DEBUG:boto:Data: 2022-09-03T06:52:51.667 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.668 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.668 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.669 DEBUG:boto:Params: {} 2022-09-03T06:52:51.670 DEBUG:boto:Token: None 2022-09-03T06:52:51.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/a 2022-09-03T06:52:51.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fj0iX7PN9ga6JD7AU34uhicMcHg= 2022-09-03T06:52:51.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fj0iX7PN9ga6JD7AU34uhicMcHg=', 'Content-Length': '0'} 2022-09-03T06:52:51.677 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a24a67a5fc8e9ba4-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.718 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.719 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.719 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:51.720 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:51.720 DEBUG:boto:Method: GET 2022-09-03T06:52:51.721 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:51.722 DEBUG:boto:Data: 2022-09-03T06:52:51.722 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.723 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.724 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.725 DEBUG:boto:Params: {} 2022-09-03T06:52:51.726 DEBUG:boto:Token: None 2022-09-03T06:52:51.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.728 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.734 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000032797986404dab73-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.737 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.738 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.739 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.740 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:51.741 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:51.741 DEBUG:boto:Method: GET 2022-09-03T06:52:51.742 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:51.743 DEBUG:boto:Data: 2022-09-03T06:52:51.743 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.744 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.744 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.745 DEBUG:boto:Params: {} 2022-09-03T06:52:51.746 DEBUG:boto:Token: None 2022-09-03T06:52:51.746 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008bcb36d065cce4b4-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.754 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.756 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:52:51.757 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.758 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.759 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.760 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.761 DEBUG:boto:Method: GET 2022-09-03T06:52:51.763 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:51.767 DEBUG:boto:Data: 2022-09-03T06:52:51.768 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.769 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.770 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.770 DEBUG:boto:Params: {} 2022-09-03T06:52:51.772 DEBUG:boto:Token: None 2022-09-03T06:52:51.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:51.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vKAFjM8o28h10LNdE9njaKyx3aA= 2022-09-03T06:52:51.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vKAFjM8o28h10LNdE9njaKyx3aA=', 'Content-Length': '0'} 2022-09-03T06:52:51.783 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e482d9ed035b0209-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.785 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.787 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.789 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.790 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.791 DEBUG:boto:Method: GET 2022-09-03T06:52:51.791 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:51.792 DEBUG:boto:Data: 2022-09-03T06:52:51.793 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.794 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.795 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.798 DEBUG:boto:Params: {} 2022-09-03T06:52:51.799 DEBUG:boto:Token: None 2022-09-03T06:52:51.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:51.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vKAFjM8o28h10LNdE9njaKyx3aA= 2022-09-03T06:52:51.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vKAFjM8o28h10LNdE9njaKyx3aA=', 'Content-Length': '0'} 2022-09-03T06:52:51.806 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005758ab1e1f3330dc-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.807 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.808 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.809 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.810 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.811 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.812 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:51.813 DEBUG:boto:Data: 2022-09-03T06:52:51.814 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.816 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.817 DEBUG:boto:Params: {} 2022-09-03T06:52:51.818 DEBUG:boto:Token: None 2022-09-03T06:52:51.819 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:51.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FwJoR4cKU9KhkSr4Go61ikGYnRs= 2022-09-03T06:52:51.821 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FwJoR4cKU9KhkSr4Go61ikGYnRs=', 'Content-Length': '0'} 2022-09-03T06:52:51.823 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8fc652d60bef6b6-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.825 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.826 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.827 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.829 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.829 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.830 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:51.831 DEBUG:boto:Data: 2022-09-03T06:52:51.831 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.832 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.833 DEBUG:boto:Params: {} 2022-09-03T06:52:51.835 DEBUG:boto:Token: None 2022-09-03T06:52:51.836 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:51.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FwJoR4cKU9KhkSr4Go61ikGYnRs= 2022-09-03T06:52:51.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FwJoR4cKU9KhkSr4Go61ikGYnRs=', 'Content-Length': '0'} 2022-09-03T06:52:51.840 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007a3f211ea4343599-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.841 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:52:51.842 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.842 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.843 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.845 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.846 DEBUG:boto:Method: GET 2022-09-03T06:52:51.847 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:51.848 DEBUG:boto:Data: 2022-09-03T06:52:51.849 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.851 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.852 DEBUG:boto:Params: {} 2022-09-03T06:52:51.854 DEBUG:boto:Token: None 2022-09-03T06:52:51.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:51.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vKAFjM8o28h10LNdE9njaKyx3aA= 2022-09-03T06:52:51.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vKAFjM8o28h10LNdE9njaKyx3aA=', 'Content-Length': '0'} 2022-09-03T06:52:51.860 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc4f3030148fdfee-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.861 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.862 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.863 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.864 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:51.865 DEBUG:boto:Method: GET 2022-09-03T06:52:51.866 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:51.867 DEBUG:boto:Data: 2022-09-03T06:52:51.868 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.869 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.870 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.871 DEBUG:boto:Params: {} 2022-09-03T06:52:51.872 DEBUG:boto:Token: None 2022-09-03T06:52:51.873 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:51.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vKAFjM8o28h10LNdE9njaKyx3aA= 2022-09-03T06:52:51.874 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vKAFjM8o28h10LNdE9njaKyx3aA=', 'Content-Length': '0'} 2022-09-03T06:52:51.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000979675e74fd5f273-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.878 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.879 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.880 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:51.881 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:51.882 DEBUG:boto:Method: GET 2022-09-03T06:52:51.882 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:51.884 DEBUG:boto:Data: 2022-09-03T06:52:51.885 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.887 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.888 DEBUG:boto:Params: {} 2022-09-03T06:52:51.889 DEBUG:boto:Token: None 2022-09-03T06:52:51.891 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.892 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.897 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb2f7c0568347de0-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.899 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.900 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.900 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.901 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:51.902 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:51.903 DEBUG:boto:Method: GET 2022-09-03T06:52:51.903 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:51.904 DEBUG:boto:Data: 2022-09-03T06:52:51.905 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.907 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.908 DEBUG:boto:Params: {} 2022-09-03T06:52:51.909 DEBUG:boto:Token: None 2022-09-03T06:52:51.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.914 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a2b4d2508fe6b0da-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.915 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:51.916 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.917 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.918 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.918 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:51.919 DEBUG:boto:Data: 2022-09-03T06:52:51.919 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.920 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.921 DEBUG:boto:Params: {} 2022-09-03T06:52:51.921 DEBUG:boto:Token: None 2022-09-03T06:52:51.922 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b 2022-09-03T06:52:51.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5F6+fBmifWp+drNpw677zUVCEFg= 2022-09-03T06:52:51.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5F6+fBmifWp+drNpw677zUVCEFg=', 'Content-Length': '0'} 2022-09-03T06:52:51.929 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000348e65c58f3e34cb-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.930 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.931 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.932 DEBUG:boto:Method: HEAD 2022-09-03T06:52:51.932 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:51.933 DEBUG:boto:Data: 2022-09-03T06:52:51.934 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.935 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.936 DEBUG:boto:Params: {} 2022-09-03T06:52:51.936 DEBUG:boto:Token: None 2022-09-03T06:52:51.937 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b 2022-09-03T06:52:51.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5F6+fBmifWp+drNpw677zUVCEFg= 2022-09-03T06:52:51.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5F6+fBmifWp+drNpw677zUVCEFg=', 'Content-Length': '0'} 2022-09-03T06:52:51.941 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e85429e039c7ca0-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.942 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:52:51.943 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.944 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.945 DEBUG:boto:Method: GET 2022-09-03T06:52:51.946 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:51.947 DEBUG:boto:Data: 2022-09-03T06:52:51.948 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.949 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.949 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.950 DEBUG:boto:Params: {} 2022-09-03T06:52:51.951 DEBUG:boto:Token: None 2022-09-03T06:52:51.951 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b 2022-09-03T06:52:51.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9OFXqbtIWeYuC6wjsmrAbQfiDpc= 2022-09-03T06:52:51.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9OFXqbtIWeYuC6wjsmrAbQfiDpc=', 'Content-Length': '0'} 2022-09-03T06:52:51.955 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000584e927ebbaeb6e3-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.956 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:51.956 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:51.957 DEBUG:boto:Method: GET 2022-09-03T06:52:51.958 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:51.958 DEBUG:boto:Data: 2022-09-03T06:52:51.959 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:51.961 DEBUG:boto:Port: 8000 2022-09-03T06:52:51.962 DEBUG:boto:Params: {} 2022-09-03T06:52:51.963 DEBUG:boto:Token: None 2022-09-03T06:52:51.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/b 2022-09-03T06:52:51.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9OFXqbtIWeYuC6wjsmrAbQfiDpc= 2022-09-03T06:52:51.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9OFXqbtIWeYuC6wjsmrAbQfiDpc=', 'Content-Length': '0'} 2022-09-03T06:52:51.970 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0c2f2e8daab3b72-006312f9c3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:51.972 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:51.973 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:51.974 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:51.976 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:51.977 DEBUG:boto:Method: GET 2022-09-03T06:52:51.978 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:51.978 DEBUG:boto:Data: 2022-09-03T06:52:51.979 DEBUG:boto:Headers: {} 2022-09-03T06:52:51.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:51.981 DEBUG:boto:Port: 8001 2022-09-03T06:52:51.981 DEBUG:boto:Params: {} 2022-09-03T06:52:51.983 DEBUG:boto:Token: None 2022-09-03T06:52:51.984 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:51 GMT /avysmw-71/ 2022-09-03T06:52:51.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM= 2022-09-03T06:52:51.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUg2DadUqgry8328FqF09voKxfM=', 'Content-Length': '0'} 2022-09-03T06:52:51.990 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d52aac1f746d4abf-006312f9c3-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.030 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.032 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.033 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.034 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:52.035 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:52.036 DEBUG:boto:Method: GET 2022-09-03T06:52:52.037 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:52.038 DEBUG:boto:Data: 2022-09-03T06:52:52.039 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.042 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.042 DEBUG:boto:Params: {} 2022-09-03T06:52:52.043 DEBUG:boto:Token: None 2022-09-03T06:52:52.044 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.046 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.050 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a575da6522da24c-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.052 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.054 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:52:52.055 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.056 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.057 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.058 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.059 DEBUG:boto:Method: GET 2022-09-03T06:52:52.060 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:52.069 DEBUG:boto:Data: 2022-09-03T06:52:52.071 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.074 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.075 DEBUG:boto:Params: {} 2022-09-03T06:52:52.077 DEBUG:boto:Token: None 2022-09-03T06:52:52.078 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:52.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6QX1ey2yzCcZZc8IoAG9NWqTbdk= 2022-09-03T06:52:52.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6QX1ey2yzCcZZc8IoAG9NWqTbdk=', 'Content-Length': '0'} 2022-09-03T06:52:52.087 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f9829e77f6df579-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.089 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.090 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.091 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.093 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.094 DEBUG:boto:Method: GET 2022-09-03T06:52:52.095 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:52.096 DEBUG:boto:Data: 2022-09-03T06:52:52.098 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.100 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.100 DEBUG:boto:Params: {} 2022-09-03T06:52:52.101 DEBUG:boto:Token: None 2022-09-03T06:52:52.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:52.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6QX1ey2yzCcZZc8IoAG9NWqTbdk= 2022-09-03T06:52:52.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6QX1ey2yzCcZZc8IoAG9NWqTbdk=', 'Content-Length': '0'} 2022-09-03T06:52:52.107 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dba3db47d696ac98-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.108 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.109 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.110 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.111 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.111 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.112 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:52.113 DEBUG:boto:Data: 2022-09-03T06:52:52.114 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.116 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.117 DEBUG:boto:Params: {} 2022-09-03T06:52:52.118 DEBUG:boto:Token: None 2022-09-03T06:52:52.119 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:52.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YhHvPaMxs9RnEryzlcOC8FYWDvw= 2022-09-03T06:52:52.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YhHvPaMxs9RnEryzlcOC8FYWDvw=', 'Content-Length': '0'} 2022-09-03T06:52:52.124 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055c5120fa336858d-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.125 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.126 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.127 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.128 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.132 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.132 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:52.133 DEBUG:boto:Data: 2022-09-03T06:52:52.133 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.135 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.135 DEBUG:boto:Params: {} 2022-09-03T06:52:52.136 DEBUG:boto:Token: None 2022-09-03T06:52:52.137 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:52.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YhHvPaMxs9RnEryzlcOC8FYWDvw= 2022-09-03T06:52:52.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YhHvPaMxs9RnEryzlcOC8FYWDvw=', 'Content-Length': '0'} 2022-09-03T06:52:52.140 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000385f4042896b57d7-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.141 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:52:52.142 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.143 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.143 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.144 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.145 DEBUG:boto:Method: GET 2022-09-03T06:52:52.146 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:52.147 DEBUG:boto:Data: 2022-09-03T06:52:52.147 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.148 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.148 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.149 DEBUG:boto:Params: {} 2022-09-03T06:52:52.150 DEBUG:boto:Token: None 2022-09-03T06:52:52.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:52.151 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6QX1ey2yzCcZZc8IoAG9NWqTbdk= 2022-09-03T06:52:52.152 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6QX1ey2yzCcZZc8IoAG9NWqTbdk=', 'Content-Length': '0'} 2022-09-03T06:52:52.154 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000157a0d1da9fe3ab4-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.155 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.155 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.156 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.157 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:52.157 DEBUG:boto:Method: GET 2022-09-03T06:52:52.158 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:52.158 DEBUG:boto:Data: 2022-09-03T06:52:52.159 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.159 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.160 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.160 DEBUG:boto:Params: {} 2022-09-03T06:52:52.161 DEBUG:boto:Token: None 2022-09-03T06:52:52.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:52.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6QX1ey2yzCcZZc8IoAG9NWqTbdk= 2022-09-03T06:52:52.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6QX1ey2yzCcZZc8IoAG9NWqTbdk=', 'Content-Length': '0'} 2022-09-03T06:52:52.166 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b40ee3b42ad5b3e9-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.214 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.215 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.216 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:52.216 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:52.217 DEBUG:boto:Method: GET 2022-09-03T06:52:52.218 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:52.219 DEBUG:boto:Data: 2022-09-03T06:52:52.219 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.220 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.221 DEBUG:boto:Params: {} 2022-09-03T06:52:52.221 DEBUG:boto:Token: None 2022-09-03T06:52:52.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.227 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e01dd78e8df1f2b6-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.227 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.228 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.229 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.232 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:52.234 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:52.235 DEBUG:boto:Method: GET 2022-09-03T06:52:52.236 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:52.236 DEBUG:boto:Data: 2022-09-03T06:52:52.237 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.239 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.240 DEBUG:boto:Params: {} 2022-09-03T06:52:52.242 DEBUG:boto:Token: None 2022-09-03T06:52:52.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.246 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e9c0654485731534-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.247 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.248 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.248 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.249 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.250 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:52.250 DEBUG:boto:Data: 2022-09-03T06:52:52.251 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.254 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.255 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.255 DEBUG:boto:Params: {} 2022-09-03T06:52:52.257 DEBUG:boto:Token: None 2022-09-03T06:52:52.258 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c 2022-09-03T06:52:52.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4h73rfRrsToNh4UTe3I09YQvpic= 2022-09-03T06:52:52.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4h73rfRrsToNh4UTe3I09YQvpic=', 'Content-Length': '0'} 2022-09-03T06:52:52.262 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009220560fc965720e-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.264 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.265 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.266 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.267 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:52.267 DEBUG:boto:Data: 2022-09-03T06:52:52.268 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.269 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.270 DEBUG:boto:Params: {} 2022-09-03T06:52:52.271 DEBUG:boto:Token: None 2022-09-03T06:52:52.272 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c 2022-09-03T06:52:52.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4h73rfRrsToNh4UTe3I09YQvpic= 2022-09-03T06:52:52.274 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4h73rfRrsToNh4UTe3I09YQvpic=', 'Content-Length': '0'} 2022-09-03T06:52:52.276 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a98c95bbf3453772-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.277 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:52:52.278 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.278 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.279 DEBUG:boto:Method: GET 2022-09-03T06:52:52.279 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:52.280 DEBUG:boto:Data: 2022-09-03T06:52:52.280 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.281 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.282 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.282 DEBUG:boto:Params: {} 2022-09-03T06:52:52.283 DEBUG:boto:Token: None 2022-09-03T06:52:52.283 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c 2022-09-03T06:52:52.284 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WwMTXjTUic/LlsO8siDxkl/zcqo= 2022-09-03T06:52:52.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WwMTXjTUic/LlsO8siDxkl/zcqo=', 'Content-Length': '0'} 2022-09-03T06:52:52.287 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf2d4efa12862743-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.341 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:52.342 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:52.343 DEBUG:boto:Method: GET 2022-09-03T06:52:52.343 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:52.344 DEBUG:boto:Data: 2022-09-03T06:52:52.344 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.346 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.347 DEBUG:boto:Params: {} 2022-09-03T06:52:52.348 DEBUG:boto:Token: None 2022-09-03T06:52:52.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/c 2022-09-03T06:52:52.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WwMTXjTUic/LlsO8siDxkl/zcqo= 2022-09-03T06:52:52.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WwMTXjTUic/LlsO8siDxkl/zcqo=', 'Content-Length': '0'} 2022-09-03T06:52:52.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6ee05cd0dc84cac-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.353 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.354 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.354 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:52.355 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:52.355 DEBUG:boto:Method: GET 2022-09-03T06:52:52.356 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:52.356 DEBUG:boto:Data: 2022-09-03T06:52:52.357 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.359 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.359 DEBUG:boto:Params: {} 2022-09-03T06:52:52.361 DEBUG:boto:Token: None 2022-09-03T06:52:52.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.366 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001db146834c540c9e-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.367 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.367 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.368 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.369 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:52.369 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:52.370 DEBUG:boto:Method: GET 2022-09-03T06:52:52.370 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:52.371 DEBUG:boto:Data: 2022-09-03T06:52:52.371 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.372 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.373 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.373 DEBUG:boto:Params: {} 2022-09-03T06:52:52.374 DEBUG:boto:Token: None 2022-09-03T06:52:52.375 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.381 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a5695999394fe815-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.382 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.383 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:52:52.384 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.385 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.385 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.386 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.386 DEBUG:boto:Method: GET 2022-09-03T06:52:52.387 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:52.387 DEBUG:boto:Data: 2022-09-03T06:52:52.388 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.389 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.390 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.391 DEBUG:boto:Params: {} 2022-09-03T06:52:52.392 DEBUG:boto:Token: None 2022-09-03T06:52:52.393 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:52.394 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1YcmGJLA9jza3+TJvfPb0r7FdrA= 2022-09-03T06:52:52.395 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1YcmGJLA9jza3+TJvfPb0r7FdrA=', 'Content-Length': '0'} 2022-09-03T06:52:52.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006193c12b509083c2-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.398 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.399 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.400 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.401 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.403 DEBUG:boto:Method: GET 2022-09-03T06:52:52.404 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:52.405 DEBUG:boto:Data: 2022-09-03T06:52:52.406 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.408 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.409 DEBUG:boto:Params: {} 2022-09-03T06:52:52.410 DEBUG:boto:Token: None 2022-09-03T06:52:52.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:52.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1YcmGJLA9jza3+TJvfPb0r7FdrA= 2022-09-03T06:52:52.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1YcmGJLA9jza3+TJvfPb0r7FdrA=', 'Content-Length': '0'} 2022-09-03T06:52:52.416 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001eb98d59d4fb1d5-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.428 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.429 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.430 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.431 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.431 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.432 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:52.433 DEBUG:boto:Data: 2022-09-03T06:52:52.433 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.434 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.435 DEBUG:boto:Params: {} 2022-09-03T06:52:52.436 DEBUG:boto:Token: None 2022-09-03T06:52:52.437 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:52.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PhWf4BKPbh91NxAdWFfDjIbECoI= 2022-09-03T06:52:52.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PhWf4BKPbh91NxAdWFfDjIbECoI=', 'Content-Length': '0'} 2022-09-03T06:52:52.441 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000941ab9c07cc9969a-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.442 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.443 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.443 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.444 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.444 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.445 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:52.446 DEBUG:boto:Data: 2022-09-03T06:52:52.446 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.449 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.450 DEBUG:boto:Params: {} 2022-09-03T06:52:52.450 DEBUG:boto:Token: None 2022-09-03T06:52:52.451 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:52.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PhWf4BKPbh91NxAdWFfDjIbECoI= 2022-09-03T06:52:52.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PhWf4BKPbh91NxAdWFfDjIbECoI=', 'Content-Length': '0'} 2022-09-03T06:52:52.455 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004abaf6874633ab45-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.456 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:52:52.465 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.466 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.467 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.468 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.468 DEBUG:boto:Method: GET 2022-09-03T06:52:52.469 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:52.471 DEBUG:boto:Data: 2022-09-03T06:52:52.472 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.474 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.475 DEBUG:boto:Params: {} 2022-09-03T06:52:52.475 DEBUG:boto:Token: None 2022-09-03T06:52:52.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:52.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1YcmGJLA9jza3+TJvfPb0r7FdrA= 2022-09-03T06:52:52.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1YcmGJLA9jza3+TJvfPb0r7FdrA=', 'Content-Length': '0'} 2022-09-03T06:52:52.482 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aacd27eadabd8751-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.484 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.484 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.485 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.487 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:52.488 DEBUG:boto:Method: GET 2022-09-03T06:52:52.488 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:52.489 DEBUG:boto:Data: 2022-09-03T06:52:52.489 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.495 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.497 DEBUG:boto:Params: {} 2022-09-03T06:52:52.498 DEBUG:boto:Token: None 2022-09-03T06:52:52.499 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:52.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1YcmGJLA9jza3+TJvfPb0r7FdrA= 2022-09-03T06:52:52.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1YcmGJLA9jza3+TJvfPb0r7FdrA=', 'Content-Length': '0'} 2022-09-03T06:52:52.512 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd5bd9cadd0338d0-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.555 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.556 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.556 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:52.557 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:52.557 DEBUG:boto:Method: GET 2022-09-03T06:52:52.559 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:52.560 DEBUG:boto:Data: 2022-09-03T06:52:52.561 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.561 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.562 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.562 DEBUG:boto:Params: {} 2022-09-03T06:52:52.563 DEBUG:boto:Token: None 2022-09-03T06:52:52.563 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.565 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.569 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000907f90c70bf80e48-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.570 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.571 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.573 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.573 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:52.578 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:52.579 DEBUG:boto:Method: GET 2022-09-03T06:52:52.580 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:52.581 DEBUG:boto:Data: 2022-09-03T06:52:52.582 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.583 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.584 DEBUG:boto:Params: {} 2022-09-03T06:52:52.584 DEBUG:boto:Token: None 2022-09-03T06:52:52.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a87609a519106f6-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.597 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.598 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.598 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.599 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.600 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:52.600 DEBUG:boto:Data: 2022-09-03T06:52:52.601 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.601 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.602 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.602 DEBUG:boto:Params: {} 2022-09-03T06:52:52.603 DEBUG:boto:Token: None 2022-09-03T06:52:52.604 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d 2022-09-03T06:52:52.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uU009Wg1iTK1438idio9Pp4uBYw= 2022-09-03T06:52:52.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uU009Wg1iTK1438idio9Pp4uBYw=', 'Content-Length': '0'} 2022-09-03T06:52:52.608 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8df97bddb4ae1be-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.609 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.609 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.610 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.611 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:52.611 DEBUG:boto:Data: 2022-09-03T06:52:52.612 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.613 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.614 DEBUG:boto:Params: {} 2022-09-03T06:52:52.615 DEBUG:boto:Token: None 2022-09-03T06:52:52.615 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d 2022-09-03T06:52:52.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uU009Wg1iTK1438idio9Pp4uBYw= 2022-09-03T06:52:52.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uU009Wg1iTK1438idio9Pp4uBYw=', 'Content-Length': '0'} 2022-09-03T06:52:52.620 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b3905b88c696e67-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.621 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:52:52.621 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.622 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.622 DEBUG:boto:Method: GET 2022-09-03T06:52:52.623 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:52.624 DEBUG:boto:Data: 2022-09-03T06:52:52.624 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.625 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.626 DEBUG:boto:Params: {} 2022-09-03T06:52:52.626 DEBUG:boto:Token: None 2022-09-03T06:52:52.627 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d 2022-09-03T06:52:52.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:evyF07WSco91/uPxDfLQdTJ+eco= 2022-09-03T06:52:52.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:evyF07WSco91/uPxDfLQdTJ+eco=', 'Content-Length': '0'} 2022-09-03T06:52:52.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e985ad3fe84f962b-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.638 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:52.639 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:52.639 DEBUG:boto:Method: GET 2022-09-03T06:52:52.640 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:52.640 DEBUG:boto:Data: 2022-09-03T06:52:52.641 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.642 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.642 DEBUG:boto:Params: {} 2022-09-03T06:52:52.644 DEBUG:boto:Token: None 2022-09-03T06:52:52.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/d 2022-09-03T06:52:52.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:evyF07WSco91/uPxDfLQdTJ+eco= 2022-09-03T06:52:52.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:evyF07WSco91/uPxDfLQdTJ+eco=', 'Content-Length': '0'} 2022-09-03T06:52:52.649 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000387f9f34a35ce192-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.650 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.651 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.651 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:52.652 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:52.652 DEBUG:boto:Method: GET 2022-09-03T06:52:52.653 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:52.654 DEBUG:boto:Data: 2022-09-03T06:52:52.655 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.656 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.656 DEBUG:boto:Params: {} 2022-09-03T06:52:52.657 DEBUG:boto:Token: None 2022-09-03T06:52:52.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.665 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063235500bcb78193-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.666 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.667 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.668 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.668 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:52.669 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:52.669 DEBUG:boto:Method: GET 2022-09-03T06:52:52.670 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:52.671 DEBUG:boto:Data: 2022-09-03T06:52:52.671 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:52.672 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.673 DEBUG:boto:Params: {} 2022-09-03T06:52:52.673 DEBUG:boto:Token: None 2022-09-03T06:52:52.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.678 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f67b5354db4e6488-006312f9c4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.679 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.680 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-71 zones={test-zone3, test-zone1} 2022-09-03T06:52:52.681 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-71 zones={test-zone3, test-zone2} 2022-09-03T06:52:52.681 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.682 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.683 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.683 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:52.684 DEBUG:boto:Data: 2022-09-03T06:52:52.684 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.685 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.686 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.687 DEBUG:boto:Params: {} 2022-09-03T06:52:52.687 DEBUG:boto:Token: None 2022-09-03T06:52:52.688 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd1AGZKJXYCpkeMY+WwHt2Y0NkE= 2022-09-03T06:52:52.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd1AGZKJXYCpkeMY+WwHt2Y0NkE=', 'Content-Length': '0'} 2022-09-03T06:52:52.692 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b801d678a14ab3e9-006312f9c4-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.693 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.693 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.694 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.694 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:52.695 DEBUG:boto:Data: 2022-09-03T06:52:52.695 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.696 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:52.696 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.697 DEBUG:boto:Params: {} 2022-09-03T06:52:52.698 DEBUG:boto:Token: None 2022-09-03T06:52:52.699 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.699 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd1AGZKJXYCpkeMY+WwHt2Y0NkE= 2022-09-03T06:52:52.700 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd1AGZKJXYCpkeMY+WwHt2Y0NkE=', 'Content-Length': '0'} 2022-09-03T06:52:52.703 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009cb2078a655479ce-006312f9c4-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.704 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:52:52.704 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.705 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.705 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.706 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.707 DEBUG:boto:Method: GET 2022-09-03T06:52:52.707 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.708 DEBUG:boto:Data: 2022-09-03T06:52:52.708 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.709 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.709 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.710 DEBUG:boto:Params: {} 2022-09-03T06:52:52.711 DEBUG:boto:Token: None 2022-09-03T06:52:52.711 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/?versions 2022-09-03T06:52:52.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0QP8AHqBfPNDp/BwmyuI3ZZ+/h4= 2022-09-03T06:52:52.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0QP8AHqBfPNDp/BwmyuI3ZZ+/h4=', 'Content-Length': '0'} 2022-09-03T06:52:52.715 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e236798d43f603ed-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.716 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.717 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:52:52.718 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:52:52.718 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:52:52.719 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:52:52.719 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:52:52.720 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.720 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.721 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.722 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.722 DEBUG:boto:Method: GET 2022-09-03T06:52:52.723 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.723 DEBUG:boto:Data: 2022-09-03T06:52:52.724 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.724 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:52.725 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.725 DEBUG:boto:Params: {} 2022-09-03T06:52:52.726 DEBUG:boto:Token: None 2022-09-03T06:52:52.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/?versions 2022-09-03T06:52:52.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0QP8AHqBfPNDp/BwmyuI3ZZ+/h4= 2022-09-03T06:52:52.728 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0QP8AHqBfPNDp/BwmyuI3ZZ+/h4=', 'Content-Length': '0'} 2022-09-03T06:52:52.731 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b954ae81e7e67b7-006312f9c4-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.732 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.733 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:52:52.733 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:52:52.734 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:52:52.734 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:52:52.735 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.735 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.736 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.736 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.737 DEBUG:boto:Method: GET 2022-09-03T06:52:52.738 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.738 DEBUG:boto:Data: 2022-09-03T06:52:52.739 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.740 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.741 DEBUG:boto:Params: {} 2022-09-03T06:52:52.742 DEBUG:boto:Token: None 2022-09-03T06:52:52.743 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/?versions 2022-09-03T06:52:52.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0QP8AHqBfPNDp/BwmyuI3ZZ+/h4= 2022-09-03T06:52:52.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0QP8AHqBfPNDp/BwmyuI3ZZ+/h4=', 'Content-Length': '0'} 2022-09-03T06:52:52.747 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000000ed235c1da48f31-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.748 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.749 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.750 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.751 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.752 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.753 DEBUG:boto:Method: GET 2022-09-03T06:52:52.753 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:52:52.754 DEBUG:boto:Data: 2022-09-03T06:52:52.754 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.755 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:52.755 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.756 DEBUG:boto:Params: {} 2022-09-03T06:52:52.757 DEBUG:boto:Token: None 2022-09-03T06:52:52.757 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/?versions 2022-09-03T06:52:52.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0QP8AHqBfPNDp/BwmyuI3ZZ+/h4= 2022-09-03T06:52:52.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0QP8AHqBfPNDp/BwmyuI3ZZ+/h4=', 'Content-Length': '0'} 2022-09-03T06:52:52.762 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e6d2348449abdba-006312f9c4-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.763 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.764 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:52:52.765 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:52.766 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:52.767 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.768 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.769 DEBUG:boto:Method: GET 2022-09-03T06:52:52.771 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:52.771 DEBUG:boto:Data: 2022-09-03T06:52:52.774 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.775 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.775 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.776 DEBUG:boto:Params: {} 2022-09-03T06:52:52.777 DEBUG:boto:Token: None 2022-09-03T06:52:52.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:52.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FHbYRTGx5izY7XWvPjD9O/ZX4Vw= 2022-09-03T06:52:52.780 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FHbYRTGx5izY7XWvPjD9O/ZX4Vw=', 'Content-Length': '0'} 2022-09-03T06:52:52.782 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005170d9a3c9ff2f8a-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.826 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:52.828 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:52.830 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.831 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.832 DEBUG:boto:Method: GET 2022-09-03T06:52:52.832 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:52.833 DEBUG:boto:Data: 2022-09-03T06:52:52.833 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.834 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:52.835 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.836 DEBUG:boto:Params: {} 2022-09-03T06:52:52.837 DEBUG:boto:Token: None 2022-09-03T06:52:52.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:52.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FHbYRTGx5izY7XWvPjD9O/ZX4Vw= 2022-09-03T06:52:52.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FHbYRTGx5izY7XWvPjD9O/ZX4Vw=', 'Content-Length': '0'} 2022-09-03T06:52:52.844 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0871ef5abe955b9-006312f9c4-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.844 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:52.846 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:52.847 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.848 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.848 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.849 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:52.850 DEBUG:boto:Data: 2022-09-03T06:52:52.850 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.851 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.852 DEBUG:boto:Params: {} 2022-09-03T06:52:52.853 DEBUG:boto:Token: None 2022-09-03T06:52:52.853 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:52.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FiYJDIfxvb1CLZ/+CsJoaysFnB8= 2022-09-03T06:52:52.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FiYJDIfxvb1CLZ/+CsJoaysFnB8=', 'Content-Length': '0'} 2022-09-03T06:52:52.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003168f74f7ee5e175-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.859 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:52.860 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:52.861 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.862 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.862 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.863 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:52.863 DEBUG:boto:Data: 2022-09-03T06:52:52.864 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.864 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:52.865 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.866 DEBUG:boto:Params: {} 2022-09-03T06:52:52.867 DEBUG:boto:Token: None 2022-09-03T06:52:52.868 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:52.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FiYJDIfxvb1CLZ/+CsJoaysFnB8= 2022-09-03T06:52:52.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FiYJDIfxvb1CLZ/+CsJoaysFnB8=', 'Content-Length': '0'} 2022-09-03T06:52:52.871 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000205821aff2b72fb3-006312f9c4-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.872 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:52:52.873 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:52.873 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:52.874 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.875 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.875 DEBUG:boto:Method: GET 2022-09-03T06:52:52.876 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:52.876 DEBUG:boto:Data: 2022-09-03T06:52:52.877 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.877 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.878 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.878 DEBUG:boto:Params: {} 2022-09-03T06:52:52.879 DEBUG:boto:Token: None 2022-09-03T06:52:52.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:52.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FHbYRTGx5izY7XWvPjD9O/ZX4Vw= 2022-09-03T06:52:52.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FHbYRTGx5izY7XWvPjD9O/ZX4Vw=', 'Content-Length': '0'} 2022-09-03T06:52:52.889 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab4fa2e9c56550e7-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.890 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:52.892 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:52.893 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.894 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:52:52.895 DEBUG:boto:Method: GET 2022-09-03T06:52:52.896 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:52:52.897 DEBUG:boto:Data: 2022-09-03T06:52:52.898 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:52.900 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.901 DEBUG:boto:Params: {} 2022-09-03T06:52:52.902 DEBUG:boto:Token: None 2022-09-03T06:52:52.903 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/a?versionId=null 2022-09-03T06:52:52.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FHbYRTGx5izY7XWvPjD9O/ZX4Vw= 2022-09-03T06:52:52.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FHbYRTGx5izY7XWvPjD9O/ZX4Vw=', 'Content-Length': '0'} 2022-09-03T06:52:52.908 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dce91fa62698ccb7-006312f9c4-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.910 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.911 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.912 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:52.913 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:52.914 DEBUG:boto:Method: GET 2022-09-03T06:52:52.915 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:52.916 DEBUG:boto:Data: 2022-09-03T06:52:52.917 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.918 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:52.919 DEBUG:boto:Port: 8001 2022-09-03T06:52:52.920 DEBUG:boto:Params: {} 2022-09-03T06:52:52.921 DEBUG:boto:Token: None 2022-09-03T06:52:52.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.929 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085545ec136e27b17-006312f9c4-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.978 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.979 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:52.981 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:52.982 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:52.982 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:52.983 DEBUG:boto:Method: GET 2022-09-03T06:52:52.984 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:52.985 DEBUG:boto:Data: 2022-09-03T06:52:52.985 DEBUG:boto:Headers: {} 2022-09-03T06:52:52.986 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:52.987 DEBUG:boto:Port: 8000 2022-09-03T06:52:52.987 DEBUG:boto:Params: {} 2022-09-03T06:52:52.988 DEBUG:boto:Token: None 2022-09-03T06:52:52.989 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:52 GMT /avysmw-71/ 2022-09-03T06:52:52.990 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU= 2022-09-03T06:52:52.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XE3Bu9MQeI56BwFJ1M6OIrEVpKU=', 'Content-Length': '0'} 2022-09-03T06:52:52.994 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002757d0134edc8a24-006312f9c4-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:52.995 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:52.996 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:52.997 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:52.997 DEBUG:boto:Method: HEAD 2022-09-03T06:52:52.998 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:52.999 DEBUG:boto:Data: 2022-09-03T06:52:53.000 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.000 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.001 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.002 DEBUG:boto:Params: {} 2022-09-03T06:52:53.003 DEBUG:boto:Token: None 2022-09-03T06:52:53.004 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/a 2022-09-03T06:52:53.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b5BGWOWEQ31/4ODWlof69UFD4lg= 2022-09-03T06:52:53.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b5BGWOWEQ31/4ODWlof69UFD4lg=', 'Content-Length': '0'} 2022-09-03T06:52:53.007 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c43a5bf095f6752-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.008 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:53.009 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:53.009 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.010 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:53.010 DEBUG:boto:Data: 2022-09-03T06:52:53.012 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.012 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.013 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.014 DEBUG:boto:Params: {} 2022-09-03T06:52:53.015 DEBUG:boto:Token: None 2022-09-03T06:52:53.016 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/a 2022-09-03T06:52:53.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b5BGWOWEQ31/4ODWlof69UFD4lg= 2022-09-03T06:52:53.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b5BGWOWEQ31/4ODWlof69UFD4lg=', 'Content-Length': '0'} 2022-09-03T06:52:53.022 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab478ba31f2582fe-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.024 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:52:53.025 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:53.026 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:53.027 DEBUG:boto:Method: GET 2022-09-03T06:52:53.027 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:53.028 DEBUG:boto:Data: 2022-09-03T06:52:53.029 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.030 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.031 DEBUG:boto:Params: {} 2022-09-03T06:52:53.031 DEBUG:boto:Token: None 2022-09-03T06:52:53.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/a 2022-09-03T06:52:53.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oTw20Du/PQgK8LzfaX8uJGSSHH0= 2022-09-03T06:52:53.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oTw20Du/PQgK8LzfaX8uJGSSHH0=', 'Content-Length': '0'} 2022-09-03T06:52:53.038 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003572d670b6ee89c9-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.040 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:52:53.041 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:52:53.042 DEBUG:boto:Method: GET 2022-09-03T06:52:53.043 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:52:53.044 DEBUG:boto:Data: 2022-09-03T06:52:53.045 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.046 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.047 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.047 DEBUG:boto:Params: {} 2022-09-03T06:52:53.049 DEBUG:boto:Token: None 2022-09-03T06:52:53.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/a 2022-09-03T06:52:53.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oTw20Du/PQgK8LzfaX8uJGSSHH0= 2022-09-03T06:52:53.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oTw20Du/PQgK8LzfaX8uJGSSHH0=', 'Content-Length': '0'} 2022-09-03T06:52:53.056 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007b1bbfa1c9171159-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.103 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.105 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.106 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:53.108 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:53.108 DEBUG:boto:Method: GET 2022-09-03T06:52:53.109 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:53.109 DEBUG:boto:Data: 2022-09-03T06:52:53.110 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.112 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.112 DEBUG:boto:Params: {} 2022-09-03T06:52:53.113 DEBUG:boto:Token: None 2022-09-03T06:52:53.115 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.121 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1272267bbeef38f-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.127 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.128 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.129 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.130 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:52:53.131 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:52:53.132 DEBUG:boto:Method: GET 2022-09-03T06:52:53.133 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:52:53.134 DEBUG:boto:Data: 2022-09-03T06:52:53.135 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.137 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.138 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.139 DEBUG:boto:Params: {} 2022-09-03T06:52:53.140 DEBUG:boto:Token: None 2022-09-03T06:52:53.141 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.146 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c27e7c8af254a0b-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.146 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.147 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:52:53.148 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.149 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.150 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.151 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.152 DEBUG:boto:Method: GET 2022-09-03T06:52:53.153 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:53.153 DEBUG:boto:Data: 2022-09-03T06:52:53.154 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.156 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.156 DEBUG:boto:Params: {} 2022-09-03T06:52:53.158 DEBUG:boto:Token: None 2022-09-03T06:52:53.158 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:53.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EcCgW01C3J1QNw3YNzBy6eTgwQo= 2022-09-03T06:52:53.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EcCgW01C3J1QNw3YNzBy6eTgwQo=', 'Content-Length': '0'} 2022-09-03T06:52:53.163 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000acfc433d9756a6e7-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.164 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.165 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.166 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.167 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.167 DEBUG:boto:Method: GET 2022-09-03T06:52:53.168 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:53.169 DEBUG:boto:Data: 2022-09-03T06:52:53.169 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.170 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.170 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.171 DEBUG:boto:Params: {} 2022-09-03T06:52:53.172 DEBUG:boto:Token: None 2022-09-03T06:52:53.173 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:53.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EcCgW01C3J1QNw3YNzBy6eTgwQo= 2022-09-03T06:52:53.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EcCgW01C3J1QNw3YNzBy6eTgwQo=', 'Content-Length': '0'} 2022-09-03T06:52:53.176 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004b9c0b9eb50e89d-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.177 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.178 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.179 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.180 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.181 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.182 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:53.183 DEBUG:boto:Data: 2022-09-03T06:52:53.184 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.186 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.187 DEBUG:boto:Params: {} 2022-09-03T06:52:53.188 DEBUG:boto:Token: None 2022-09-03T06:52:53.189 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:53.191 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wg7JBBbifoVqY/l4tUa5kHWAuAs= 2022-09-03T06:52:53.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wg7JBBbifoVqY/l4tUa5kHWAuAs=', 'Content-Length': '0'} 2022-09-03T06:52:53.194 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af57381750ecfe56-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.195 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.196 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.196 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.197 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.197 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.198 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:53.199 DEBUG:boto:Data: 2022-09-03T06:52:53.200 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.202 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.203 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.203 DEBUG:boto:Params: {} 2022-09-03T06:52:53.205 DEBUG:boto:Token: None 2022-09-03T06:52:53.206 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:53.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wg7JBBbifoVqY/l4tUa5kHWAuAs= 2022-09-03T06:52:53.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wg7JBBbifoVqY/l4tUa5kHWAuAs=', 'Content-Length': '0'} 2022-09-03T06:52:53.211 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d959d9c7ce6dbb3f-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.212 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:52:53.213 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.214 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.215 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.216 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.218 DEBUG:boto:Method: GET 2022-09-03T06:52:53.219 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:53.220 DEBUG:boto:Data: 2022-09-03T06:52:53.221 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.222 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.223 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.225 DEBUG:boto:Params: {} 2022-09-03T06:52:53.227 DEBUG:boto:Token: None 2022-09-03T06:52:53.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:53.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EcCgW01C3J1QNw3YNzBy6eTgwQo= 2022-09-03T06:52:53.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EcCgW01C3J1QNw3YNzBy6eTgwQo=', 'Content-Length': '0'} 2022-09-03T06:52:53.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068c4186e21310d80-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.274 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.275 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.275 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.276 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:52:53.277 DEBUG:boto:Method: GET 2022-09-03T06:52:53.277 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:52:53.278 DEBUG:boto:Data: 2022-09-03T06:52:53.279 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.279 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.280 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.280 DEBUG:boto:Params: {} 2022-09-03T06:52:53.281 DEBUG:boto:Token: None 2022-09-03T06:52:53.283 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b?versionId=null 2022-09-03T06:52:53.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EcCgW01C3J1QNw3YNzBy6eTgwQo= 2022-09-03T06:52:53.284 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EcCgW01C3J1QNw3YNzBy6eTgwQo=', 'Content-Length': '0'} 2022-09-03T06:52:53.286 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000908a3910032a0e6d-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.287 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.288 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.288 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:53.289 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:53.290 DEBUG:boto:Method: GET 2022-09-03T06:52:53.290 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:53.291 DEBUG:boto:Data: 2022-09-03T06:52:53.292 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.294 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.294 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.295 DEBUG:boto:Params: {} 2022-09-03T06:52:53.295 DEBUG:boto:Token: None 2022-09-03T06:52:53.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000102ba63402aaf944-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.302 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.303 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.303 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.304 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:53.304 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:53.305 DEBUG:boto:Method: GET 2022-09-03T06:52:53.305 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:53.306 DEBUG:boto:Data: 2022-09-03T06:52:53.306 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.307 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.308 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.308 DEBUG:boto:Params: {} 2022-09-03T06:52:53.310 DEBUG:boto:Token: None 2022-09-03T06:52:53.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.317 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000019b200210385488b-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.326 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.328 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.329 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.329 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.330 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:53.330 DEBUG:boto:Data: 2022-09-03T06:52:53.331 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.332 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.333 DEBUG:boto:Params: {} 2022-09-03T06:52:53.333 DEBUG:boto:Token: None 2022-09-03T06:52:53.334 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b 2022-09-03T06:52:53.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kWlMHJUhR3BUHrLda1cy/IpiFGw= 2022-09-03T06:52:53.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kWlMHJUhR3BUHrLda1cy/IpiFGw=', 'Content-Length': '0'} 2022-09-03T06:52:53.338 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003221b30bbc5d9c30-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.339 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.340 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.342 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.343 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:53.344 DEBUG:boto:Data: 2022-09-03T06:52:53.344 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.345 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.346 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.346 DEBUG:boto:Params: {} 2022-09-03T06:52:53.347 DEBUG:boto:Token: None 2022-09-03T06:52:53.348 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b 2022-09-03T06:52:53.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kWlMHJUhR3BUHrLda1cy/IpiFGw= 2022-09-03T06:52:53.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kWlMHJUhR3BUHrLda1cy/IpiFGw=', 'Content-Length': '0'} 2022-09-03T06:52:53.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c47f3c39e8a31ba-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.353 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:52:53.354 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.355 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.357 DEBUG:boto:Method: GET 2022-09-03T06:52:53.358 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:53.359 DEBUG:boto:Data: 2022-09-03T06:52:53.360 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.361 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.362 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.363 DEBUG:boto:Params: {} 2022-09-03T06:52:53.364 DEBUG:boto:Token: None 2022-09-03T06:52:53.365 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b 2022-09-03T06:52:53.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZK0Z3WYC71SME8m9G5YofLTfuJg= 2022-09-03T06:52:53.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZK0Z3WYC71SME8m9G5YofLTfuJg=', 'Content-Length': '0'} 2022-09-03T06:52:53.370 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7c9350c565228f1-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.372 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:52:53.373 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:52:53.374 DEBUG:boto:Method: GET 2022-09-03T06:52:53.375 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:52:53.376 DEBUG:boto:Data: 2022-09-03T06:52:53.377 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.378 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.379 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.380 DEBUG:boto:Params: {} 2022-09-03T06:52:53.381 DEBUG:boto:Token: None 2022-09-03T06:52:53.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/b 2022-09-03T06:52:53.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZK0Z3WYC71SME8m9G5YofLTfuJg= 2022-09-03T06:52:53.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZK0Z3WYC71SME8m9G5YofLTfuJg=', 'Content-Length': '0'} 2022-09-03T06:52:53.395 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006cae527b1e9a8046-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.396 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.397 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.398 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:53.398 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:53.399 DEBUG:boto:Method: GET 2022-09-03T06:52:53.400 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:53.400 DEBUG:boto:Data: 2022-09-03T06:52:53.401 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.403 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.404 DEBUG:boto:Params: {} 2022-09-03T06:52:53.405 DEBUG:boto:Token: None 2022-09-03T06:52:53.406 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.407 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.412 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fcff701e5839d9f6-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.413 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.415 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.416 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.416 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:52:53.417 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:52:53.418 DEBUG:boto:Method: GET 2022-09-03T06:52:53.419 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:52:53.419 DEBUG:boto:Data: 2022-09-03T06:52:53.420 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.421 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.421 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.422 DEBUG:boto:Params: {} 2022-09-03T06:52:53.423 DEBUG:boto:Token: None 2022-09-03T06:52:53.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.429 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008adc8e2480801e07-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.430 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.431 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:52:53.432 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.433 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.434 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.436 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.436 DEBUG:boto:Method: GET 2022-09-03T06:52:53.438 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:53.438 DEBUG:boto:Data: 2022-09-03T06:52:53.439 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.440 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.441 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.442 DEBUG:boto:Params: {} 2022-09-03T06:52:53.443 DEBUG:boto:Token: None 2022-09-03T06:52:53.444 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:53.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g3kHLyfFLlgLZ9Ll7LDMfM8XKWE= 2022-09-03T06:52:53.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g3kHLyfFLlgLZ9Ll7LDMfM8XKWE=', 'Content-Length': '0'} 2022-09-03T06:52:53.449 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000159e3cc638d8000-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.490 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.492 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.493 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.494 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.495 DEBUG:boto:Method: GET 2022-09-03T06:52:53.496 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:53.496 DEBUG:boto:Data: 2022-09-03T06:52:53.497 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.498 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.498 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.499 DEBUG:boto:Params: {} 2022-09-03T06:52:53.500 DEBUG:boto:Token: None 2022-09-03T06:52:53.500 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:53.501 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g3kHLyfFLlgLZ9Ll7LDMfM8XKWE= 2022-09-03T06:52:53.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g3kHLyfFLlgLZ9Ll7LDMfM8XKWE=', 'Content-Length': '0'} 2022-09-03T06:52:53.504 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052bc9025684636cd-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.506 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.507 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.508 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.509 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.510 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.510 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:53.511 DEBUG:boto:Data: 2022-09-03T06:52:53.511 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.512 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.513 DEBUG:boto:Params: {} 2022-09-03T06:52:53.514 DEBUG:boto:Token: None 2022-09-03T06:52:53.514 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:53.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h1vVF8VV/wOmKfftM1cAGwhCMDo= 2022-09-03T06:52:53.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h1vVF8VV/wOmKfftM1cAGwhCMDo=', 'Content-Length': '0'} 2022-09-03T06:52:53.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075caec3ce38d41b3-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.519 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.520 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.520 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.521 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.521 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.522 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:53.523 DEBUG:boto:Data: 2022-09-03T06:52:53.523 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.524 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.524 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.525 DEBUG:boto:Params: {} 2022-09-03T06:52:53.526 DEBUG:boto:Token: None 2022-09-03T06:52:53.528 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:53.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:h1vVF8VV/wOmKfftM1cAGwhCMDo= 2022-09-03T06:52:53.529 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:h1vVF8VV/wOmKfftM1cAGwhCMDo=', 'Content-Length': '0'} 2022-09-03T06:52:53.531 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfaff924155ed5ed-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.532 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:52:53.533 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.534 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.534 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.535 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.535 DEBUG:boto:Method: GET 2022-09-03T06:52:53.536 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:53.536 DEBUG:boto:Data: 2022-09-03T06:52:53.537 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.538 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.539 DEBUG:boto:Params: {} 2022-09-03T06:52:53.540 DEBUG:boto:Token: None 2022-09-03T06:52:53.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:53.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g3kHLyfFLlgLZ9Ll7LDMfM8XKWE= 2022-09-03T06:52:53.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g3kHLyfFLlgLZ9Ll7LDMfM8XKWE=', 'Content-Length': '0'} 2022-09-03T06:52:53.545 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000698cea209e8c0367-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.547 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.547 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.548 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.548 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:52:53.549 DEBUG:boto:Method: GET 2022-09-03T06:52:53.550 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:52:53.551 DEBUG:boto:Data: 2022-09-03T06:52:53.552 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.553 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.554 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.555 DEBUG:boto:Params: {} 2022-09-03T06:52:53.555 DEBUG:boto:Token: None 2022-09-03T06:52:53.556 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c?versionId=null 2022-09-03T06:52:53.557 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g3kHLyfFLlgLZ9Ll7LDMfM8XKWE= 2022-09-03T06:52:53.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g3kHLyfFLlgLZ9Ll7LDMfM8XKWE=', 'Content-Length': '0'} 2022-09-03T06:52:53.560 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f2be3b17033da4c6-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.561 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.562 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.563 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:53.564 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:53.565 DEBUG:boto:Method: GET 2022-09-03T06:52:53.566 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:53.567 DEBUG:boto:Data: 2022-09-03T06:52:53.568 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.569 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.571 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.572 DEBUG:boto:Params: {} 2022-09-03T06:52:53.573 DEBUG:boto:Token: None 2022-09-03T06:52:53.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.578 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b541bd0908c6b82-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.579 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.580 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.581 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.581 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:53.582 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:53.582 DEBUG:boto:Method: GET 2022-09-03T06:52:53.583 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:53.583 DEBUG:boto:Data: 2022-09-03T06:52:53.584 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.584 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.585 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.585 DEBUG:boto:Params: {} 2022-09-03T06:52:53.586 DEBUG:boto:Token: None 2022-09-03T06:52:53.587 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.588 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.591 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e2c30e9fdce19356-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.591 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.592 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.593 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.593 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.594 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:53.594 DEBUG:boto:Data: 2022-09-03T06:52:53.595 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.595 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.596 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.596 DEBUG:boto:Params: {} 2022-09-03T06:52:53.597 DEBUG:boto:Token: None 2022-09-03T06:52:53.598 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c 2022-09-03T06:52:53.598 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FP29anUcNsOU1YuZJIBGcAw18ws= 2022-09-03T06:52:53.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FP29anUcNsOU1YuZJIBGcAw18ws=', 'Content-Length': '0'} 2022-09-03T06:52:53.601 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000664e3a3e1a6c411a-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.602 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.602 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.603 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.603 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:53.604 DEBUG:boto:Data: 2022-09-03T06:52:53.604 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.605 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.605 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.606 DEBUG:boto:Params: {} 2022-09-03T06:52:53.606 DEBUG:boto:Token: None 2022-09-03T06:52:53.607 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c 2022-09-03T06:52:53.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FP29anUcNsOU1YuZJIBGcAw18ws= 2022-09-03T06:52:53.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FP29anUcNsOU1YuZJIBGcAw18ws=', 'Content-Length': '0'} 2022-09-03T06:52:53.610 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c7ecdbb9ff11869-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.611 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:52:53.612 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.612 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.613 DEBUG:boto:Method: GET 2022-09-03T06:52:53.613 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:53.614 DEBUG:boto:Data: 2022-09-03T06:52:53.615 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.616 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.616 DEBUG:boto:Params: {} 2022-09-03T06:52:53.617 DEBUG:boto:Token: None 2022-09-03T06:52:53.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c 2022-09-03T06:52:53.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/D1O6w0ALZ4XUo+e3UDUnOUfRzw= 2022-09-03T06:52:53.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/D1O6w0ALZ4XUo+e3UDUnOUfRzw=', 'Content-Length': '0'} 2022-09-03T06:52:53.621 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe65611cea03c076-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.663 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:52:53.663 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:52:53.664 DEBUG:boto:Method: GET 2022-09-03T06:52:53.665 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:52:53.665 DEBUG:boto:Data: 2022-09-03T06:52:53.666 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.666 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.667 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.667 DEBUG:boto:Params: {} 2022-09-03T06:52:53.668 DEBUG:boto:Token: None 2022-09-03T06:52:53.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/c 2022-09-03T06:52:53.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/D1O6w0ALZ4XUo+e3UDUnOUfRzw= 2022-09-03T06:52:53.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/D1O6w0ALZ4XUo+e3UDUnOUfRzw=', 'Content-Length': '0'} 2022-09-03T06:52:53.672 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da5ca99b0996c846-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.673 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.673 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.674 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:53.674 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:53.675 DEBUG:boto:Method: GET 2022-09-03T06:52:53.675 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:53.676 DEBUG:boto:Data: 2022-09-03T06:52:53.676 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.678 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.678 DEBUG:boto:Params: {} 2022-09-03T06:52:53.679 DEBUG:boto:Token: None 2022-09-03T06:52:53.679 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.684 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006da1a6fa692e2c9b-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.684 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.685 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.686 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.686 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:52:53.687 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:52:53.688 DEBUG:boto:Method: GET 2022-09-03T06:52:53.688 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:52:53.689 DEBUG:boto:Data: 2022-09-03T06:52:53.689 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.690 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.691 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.691 DEBUG:boto:Params: {} 2022-09-03T06:52:53.692 DEBUG:boto:Token: None 2022-09-03T06:52:53.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.693 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.696 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd9c8a846a25ae13-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.697 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.698 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:52:53.698 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.699 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.700 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.700 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.701 DEBUG:boto:Method: GET 2022-09-03T06:52:53.701 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:53.702 DEBUG:boto:Data: 2022-09-03T06:52:53.703 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.704 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.704 DEBUG:boto:Params: {} 2022-09-03T06:52:53.705 DEBUG:boto:Token: None 2022-09-03T06:52:53.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:53.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR7y7Fb0Vy2FDFVhvoAsYsABpO8= 2022-09-03T06:52:53.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR7y7Fb0Vy2FDFVhvoAsYsABpO8=', 'Content-Length': '0'} 2022-09-03T06:52:53.710 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e18e115be5f69071-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.711 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.711 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.712 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.712 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.713 DEBUG:boto:Method: GET 2022-09-03T06:52:53.713 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:53.714 DEBUG:boto:Data: 2022-09-03T06:52:53.714 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.715 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.716 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.716 DEBUG:boto:Params: {} 2022-09-03T06:52:53.717 DEBUG:boto:Token: None 2022-09-03T06:52:53.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:53.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR7y7Fb0Vy2FDFVhvoAsYsABpO8= 2022-09-03T06:52:53.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR7y7Fb0Vy2FDFVhvoAsYsABpO8=', 'Content-Length': '0'} 2022-09-03T06:52:53.721 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ca45f104429ffd5-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.721 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.722 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.722 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.723 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.724 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.724 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:53.725 DEBUG:boto:Data: 2022-09-03T06:52:53.725 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.726 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.726 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.727 DEBUG:boto:Params: {} 2022-09-03T06:52:53.727 DEBUG:boto:Token: None 2022-09-03T06:52:53.728 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:53.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlPjSorGSHfcxaQwEwNl5TPhck= 2022-09-03T06:52:53.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlPjSorGSHfcxaQwEwNl5TPhck=', 'Content-Length': '0'} 2022-09-03T06:52:53.734 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000002790914a6a3eb8-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.736 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.737 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.738 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.739 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.740 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.741 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:53.742 DEBUG:boto:Data: 2022-09-03T06:52:53.743 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.744 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.745 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.746 DEBUG:boto:Params: {} 2022-09-03T06:52:53.747 DEBUG:boto:Token: None 2022-09-03T06:52:53.747 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:53.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlPjSorGSHfcxaQwEwNl5TPhck= 2022-09-03T06:52:53.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlPjSorGSHfcxaQwEwNl5TPhck=', 'Content-Length': '0'} 2022-09-03T06:52:53.751 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bdbc00a68cb07515-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.752 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:52:53.752 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.753 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.753 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.754 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.755 DEBUG:boto:Method: GET 2022-09-03T06:52:53.755 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:53.756 DEBUG:boto:Data: 2022-09-03T06:52:53.757 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.758 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.758 DEBUG:boto:Params: {} 2022-09-03T06:52:53.759 DEBUG:boto:Token: None 2022-09-03T06:52:53.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:53.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR7y7Fb0Vy2FDFVhvoAsYsABpO8= 2022-09-03T06:52:53.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR7y7Fb0Vy2FDFVhvoAsYsABpO8=', 'Content-Length': '0'} 2022-09-03T06:52:53.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000067a5964031c1e10e-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.812 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.813 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.814 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.815 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:52:53.815 DEBUG:boto:Method: GET 2022-09-03T06:52:53.816 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:52:53.817 DEBUG:boto:Data: 2022-09-03T06:52:53.817 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.818 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.819 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.819 DEBUG:boto:Params: {} 2022-09-03T06:52:53.820 DEBUG:boto:Token: None 2022-09-03T06:52:53.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d?versionId=null 2022-09-03T06:52:53.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR7y7Fb0Vy2FDFVhvoAsYsABpO8= 2022-09-03T06:52:53.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR7y7Fb0Vy2FDFVhvoAsYsABpO8=', 'Content-Length': '0'} 2022-09-03T06:52:53.825 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000739772b4697b447c-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.827 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.828 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.829 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:53.830 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:53.831 DEBUG:boto:Method: GET 2022-09-03T06:52:53.831 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:53.832 DEBUG:boto:Data: 2022-09-03T06:52:53.832 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.833 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.834 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.835 DEBUG:boto:Params: {} 2022-09-03T06:52:53.836 DEBUG:boto:Token: None 2022-09-03T06:52:53.836 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.840 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb1ffb1af4c13a87-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.842 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.843 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.843 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.844 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:53.846 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:53.847 DEBUG:boto:Method: GET 2022-09-03T06:52:53.847 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:53.849 DEBUG:boto:Data: 2022-09-03T06:52:53.850 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.851 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.852 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.853 DEBUG:boto:Params: {} 2022-09-03T06:52:53.854 DEBUG:boto:Token: None 2022-09-03T06:52:53.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.861 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044da067bf858a110-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.867 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.870 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.870 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.871 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.872 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:53.872 DEBUG:boto:Data: 2022-09-03T06:52:53.873 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.873 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.874 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.875 DEBUG:boto:Params: {} 2022-09-03T06:52:53.875 DEBUG:boto:Token: None 2022-09-03T06:52:53.876 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d 2022-09-03T06:52:53.877 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mPLyAa3qz1WaYbs7o8Hee+kBZS4= 2022-09-03T06:52:53.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mPLyAa3qz1WaYbs7o8Hee+kBZS4=', 'Content-Length': '0'} 2022-09-03T06:52:53.881 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026cd65f0d28a341d-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.883 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.884 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.888 DEBUG:boto:Method: HEAD 2022-09-03T06:52:53.889 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:53.890 DEBUG:boto:Data: 2022-09-03T06:52:53.890 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.891 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.891 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.892 DEBUG:boto:Params: {} 2022-09-03T06:52:53.892 DEBUG:boto:Token: None 2022-09-03T06:52:53.893 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d 2022-09-03T06:52:53.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mPLyAa3qz1WaYbs7o8Hee+kBZS4= 2022-09-03T06:52:53.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mPLyAa3qz1WaYbs7o8Hee+kBZS4=', 'Content-Length': '0'} 2022-09-03T06:52:53.897 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c63313736dc61c8-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.898 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:52:53.899 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.899 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.900 DEBUG:boto:Method: GET 2022-09-03T06:52:53.900 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:53.901 DEBUG:boto:Data: 2022-09-03T06:52:53.901 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.906 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.907 DEBUG:boto:Params: {} 2022-09-03T06:52:53.908 DEBUG:boto:Token: None 2022-09-03T06:52:53.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d 2022-09-03T06:52:53.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1T95/IgKlfZq6UfMcXCb9zWOxkg= 2022-09-03T06:52:53.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1T95/IgKlfZq6UfMcXCb9zWOxkg=', 'Content-Length': '0'} 2022-09-03T06:52:53.912 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000aecc3f4d0b70f07-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.913 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:52:53.914 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:52:53.914 DEBUG:boto:Method: GET 2022-09-03T06:52:53.915 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:52:53.916 DEBUG:boto:Data: 2022-09-03T06:52:53.916 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.917 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.917 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.918 DEBUG:boto:Params: {} 2022-09-03T06:52:53.919 DEBUG:boto:Token: None 2022-09-03T06:52:53.919 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/d 2022-09-03T06:52:53.920 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1T95/IgKlfZq6UfMcXCb9zWOxkg= 2022-09-03T06:52:53.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1T95/IgKlfZq6UfMcXCb9zWOxkg=', 'Content-Length': '0'} 2022-09-03T06:52:53.923 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000206600868fc19ee8-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.924 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.925 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.925 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:53.926 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:53.927 DEBUG:boto:Method: GET 2022-09-03T06:52:53.927 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:53.928 DEBUG:boto:Data: 2022-09-03T06:52:53.929 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.929 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:52:53.930 DEBUG:boto:Port: 8001 2022-09-03T06:52:53.931 DEBUG:boto:Params: {} 2022-09-03T06:52:53.931 DEBUG:boto:Token: None 2022-09-03T06:52:53.932 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.933 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.941 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa98920a62d852b6-006312f9c5-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.942 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.943 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:53.944 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:53.944 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:52:53.945 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:52:53.946 DEBUG:boto:Method: GET 2022-09-03T06:52:53.946 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:52:53.947 DEBUG:boto:Data: 2022-09-03T06:52:53.949 DEBUG:boto:Headers: {} 2022-09-03T06:52:53.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:52:53.950 DEBUG:boto:Port: 8000 2022-09-03T06:52:53.950 DEBUG:boto:Params: {} 2022-09-03T06:52:53.951 DEBUG:boto:Token: None 2022-09-03T06:52:53.952 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:52:53 GMT /avysmw-71/ 2022-09-03T06:52:53.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM= 2022-09-03T06:52:53.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pd5VZlUuZAgOIAVGNf77DMtLBoM=', 'Content-Length': '0'} 2022-09-03T06:52:53.959 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d63502040b5e0d5d-006312f9c5-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:52:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:53.960 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:52:53.962 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-71 zones={test-zone3, test-zone2} 2022-09-03T06:52:53.963 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:52:53.964 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:52:53.965 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket layout --bucket avysmw-71 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:52:54.024 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:54.026 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:54.105 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:52:54.106 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.107 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding": "None", 2022-09-03T06:52:54.107 INFO:teuthology.orchestra.run.smithi062.stdout: "current_index": { 2022-09-03T06:52:54.108 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 0, 2022-09-03T06:52:54.108 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.109 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "Normal", 2022-09-03T06:52:54.109 INFO:teuthology.orchestra.run.smithi062.stdout: "normal": { 2022-09-03T06:52:54.111 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 11, 2022-09-03T06:52:54.111 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:52:54.111 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.111 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.112 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.112 INFO:teuthology.orchestra.run.smithi062.stdout: "logs": [ 2022-09-03T06:52:54.112 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.112 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 0, 2022-09-03T06:52:54.113 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.113 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:52:54.114 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:52:54.114 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 0, 2022-09-03T06:52:54.114 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.115 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 11, 2022-09-03T06:52:54.115 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:52:54.115 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.115 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.116 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.116 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.117 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:52:54.117 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.117 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:52:54.122 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'list', '--bucket', 'avysmw-71', '--gen', '0', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:52:54.123 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'list', '--bucket', 'avysmw-71', '--gen', '0', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:52:54.124 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog list --bucket avysmw-71 --gen 0 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:52:54.176 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:54.176 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:54.227 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:52:54.228 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.228 INFO:teuthology.orchestra.run.smithi062.stdout: "op_id": "1#00000000001.49.5", 2022-09-03T06:52:54.229 INFO:teuthology.orchestra.run.smithi062.stdout: "op_tag": "fca49d30-abba-4c2b-8462-66395ac063ba.5636.4214556924426552345", 2022-09-03T06:52:54.229 INFO:teuthology.orchestra.run.smithi062.stdout: "op": "write", 2022-09-03T06:52:54.230 INFO:teuthology.orchestra.run.smithi062.stdout: "object": "a", 2022-09-03T06:52:54.230 INFO:teuthology.orchestra.run.smithi062.stdout: "instance": "", 2022-09-03T06:52:54.231 INFO:teuthology.orchestra.run.smithi062.stdout: "state": "complete", 2022-09-03T06:52:54.231 INFO:teuthology.orchestra.run.smithi062.stdout: "index_ver": 1, 2022-09-03T06:52:54.232 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:51:13.081784810Z", 2022-09-03T06:52:54.232 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": { 2022-09-03T06:52:54.233 INFO:teuthology.orchestra.run.smithi062.stdout: "pool": 9, 2022-09-03T06:52:54.234 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 2 2022-09-03T06:52:54.234 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.234 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog_flags": 0, 2022-09-03T06:52:54.235 INFO:teuthology.orchestra.run.smithi062.stdout: "versioned": false, 2022-09-03T06:52:54.235 INFO:teuthology.orchestra.run.smithi062.stdout: "owner": "", 2022-09-03T06:52:54.236 INFO:teuthology.orchestra.run.smithi062.stdout: "owner_display_name": "", 2022-09-03T06:52:54.236 INFO:teuthology.orchestra.run.smithi062.stdout: "zones_trace": [ 2022-09-03T06:52:54.237 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.237 INFO:teuthology.orchestra.run.smithi062.stdout: "entry": "fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-71:fca49d30-abba-4c2b-8462-66395ac063ba.5638.22" 2022-09-03T06:52:54.238 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.238 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:52:54.238 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.239 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.239 INFO:teuthology.orchestra.run.smithi062.stdout: "op_id": "5#00000000001.168.5", 2022-09-03T06:52:54.239 INFO:teuthology.orchestra.run.smithi062.stdout: "op_tag": "fca49d30-abba-4c2b-8462-66395ac063ba.5636.16231622821365017347", 2022-09-03T06:52:54.240 INFO:teuthology.orchestra.run.smithi062.stdout: "op": "write", 2022-09-03T06:52:54.241 INFO:teuthology.orchestra.run.smithi062.stdout: "object": "d", 2022-09-03T06:52:54.241 INFO:teuthology.orchestra.run.smithi062.stdout: "instance": "", 2022-09-03T06:52:54.242 INFO:teuthology.orchestra.run.smithi062.stdout: "state": "complete", 2022-09-03T06:52:54.243 INFO:teuthology.orchestra.run.smithi062.stdout: "index_ver": 1, 2022-09-03T06:52:54.243 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:51:13.236327765Z", 2022-09-03T06:52:54.244 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": { 2022-09-03T06:52:54.244 INFO:teuthology.orchestra.run.smithi062.stdout: "pool": 9, 2022-09-03T06:52:54.245 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 3 2022-09-03T06:52:54.245 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.246 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog_flags": 0, 2022-09-03T06:52:54.247 INFO:teuthology.orchestra.run.smithi062.stdout: "versioned": false, 2022-09-03T06:52:54.247 INFO:teuthology.orchestra.run.smithi062.stdout: "owner": "", 2022-09-03T06:52:54.247 INFO:teuthology.orchestra.run.smithi062.stdout: "owner_display_name": "", 2022-09-03T06:52:54.248 INFO:teuthology.orchestra.run.smithi062.stdout: "zones_trace": [ 2022-09-03T06:52:54.248 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.249 INFO:teuthology.orchestra.run.smithi062.stdout: "entry": "fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-71:fca49d30-abba-4c2b-8462-66395ac063ba.5638.22" 2022-09-03T06:52:54.249 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.251 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:52:54.251 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.252 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.253 INFO:teuthology.orchestra.run.smithi062.stdout: "op_id": "8#00000000001.59.5", 2022-09-03T06:52:54.253 INFO:teuthology.orchestra.run.smithi062.stdout: "op_tag": "fca49d30-abba-4c2b-8462-66395ac063ba.5636.16653738164743104883", 2022-09-03T06:52:54.254 INFO:teuthology.orchestra.run.smithi062.stdout: "op": "write", 2022-09-03T06:52:54.255 INFO:teuthology.orchestra.run.smithi062.stdout: "object": "c", 2022-09-03T06:52:54.256 INFO:teuthology.orchestra.run.smithi062.stdout: "instance": "", 2022-09-03T06:52:54.256 INFO:teuthology.orchestra.run.smithi062.stdout: "state": "complete", 2022-09-03T06:52:54.256 INFO:teuthology.orchestra.run.smithi062.stdout: "index_ver": 1, 2022-09-03T06:52:54.257 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:51:13.193832829Z", 2022-09-03T06:52:54.257 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": { 2022-09-03T06:52:54.257 INFO:teuthology.orchestra.run.smithi062.stdout: "pool": 9, 2022-09-03T06:52:54.258 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 1 2022-09-03T06:52:54.258 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.259 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog_flags": 0, 2022-09-03T06:52:54.260 INFO:teuthology.orchestra.run.smithi062.stdout: "versioned": false, 2022-09-03T06:52:54.260 INFO:teuthology.orchestra.run.smithi062.stdout: "owner": "", 2022-09-03T06:52:54.261 INFO:teuthology.orchestra.run.smithi062.stdout: "owner_display_name": "", 2022-09-03T06:52:54.261 INFO:teuthology.orchestra.run.smithi062.stdout: "zones_trace": [ 2022-09-03T06:52:54.262 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.262 INFO:teuthology.orchestra.run.smithi062.stdout: "entry": "fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-71:fca49d30-abba-4c2b-8462-66395ac063ba.5638.22" 2022-09-03T06:52:54.263 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.263 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:52:54.263 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.263 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.264 INFO:teuthology.orchestra.run.smithi062.stdout: "op_id": "10#00000000001.50.5", 2022-09-03T06:52:54.264 INFO:teuthology.orchestra.run.smithi062.stdout: "op_tag": "fca49d30-abba-4c2b-8462-66395ac063ba.5636.6021848536840126583", 2022-09-03T06:52:54.264 INFO:teuthology.orchestra.run.smithi062.stdout: "op": "write", 2022-09-03T06:52:54.264 INFO:teuthology.orchestra.run.smithi062.stdout: "object": "b", 2022-09-03T06:52:54.265 INFO:teuthology.orchestra.run.smithi062.stdout: "instance": "", 2022-09-03T06:52:54.265 INFO:teuthology.orchestra.run.smithi062.stdout: "state": "complete", 2022-09-03T06:52:54.266 INFO:teuthology.orchestra.run.smithi062.stdout: "index_ver": 1, 2022-09-03T06:52:54.266 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:51:13.154628505Z", 2022-09-03T06:52:54.266 INFO:teuthology.orchestra.run.smithi062.stdout: "ver": { 2022-09-03T06:52:54.267 INFO:teuthology.orchestra.run.smithi062.stdout: "pool": 9, 2022-09-03T06:52:54.267 INFO:teuthology.orchestra.run.smithi062.stdout: "epoch": 21 2022-09-03T06:52:54.267 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.267 INFO:teuthology.orchestra.run.smithi062.stdout: "bilog_flags": 0, 2022-09-03T06:52:54.268 INFO:teuthology.orchestra.run.smithi062.stdout: "versioned": false, 2022-09-03T06:52:54.268 INFO:teuthology.orchestra.run.smithi062.stdout: "owner": "", 2022-09-03T06:52:54.268 INFO:teuthology.orchestra.run.smithi062.stdout: "owner_display_name": "", 2022-09-03T06:52:54.269 INFO:teuthology.orchestra.run.smithi062.stdout: "zones_trace": [ 2022-09-03T06:52:54.269 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.269 INFO:teuthology.orchestra.run.smithi062.stdout: "entry": "fca49d30-abba-4c2b-8462-66395ac063ba:avysmw-71:fca49d30-abba-4c2b-8462-66395ac063ba.5638.22" 2022-09-03T06:52:54.269 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.270 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:52:54.270 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.270 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:52:54.271 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'avysmw-71', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:52:54.274 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'avysmw-71', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:52:54.275 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket avysmw-71 --num-shards 3 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:52:54.327 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:54.327 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:54.384 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T06:52:54.385 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-71 2022-09-03T06:52:54.390 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 4 2022-09-03T06:52:54.462 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:52:54.463 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:52:54.463 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket layout --bucket avysmw-71 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:52:54.511 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:54.512 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:54.567 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:52:54.568 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.568 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding": "None", 2022-09-03T06:52:54.568 INFO:teuthology.orchestra.run.smithi062.stdout: "current_index": { 2022-09-03T06:52:54.568 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 1, 2022-09-03T06:52:54.569 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.569 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "Normal", 2022-09-03T06:52:54.569 INFO:teuthology.orchestra.run.smithi062.stdout: "normal": { 2022-09-03T06:52:54.570 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:52:54.570 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:52:54.570 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.571 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.571 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.571 INFO:teuthology.orchestra.run.smithi062.stdout: "logs": [ 2022-09-03T06:52:54.572 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.572 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 0, 2022-09-03T06:52:54.572 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.573 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:52:54.573 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:52:54.574 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 0, 2022-09-03T06:52:54.574 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.574 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 11, 2022-09-03T06:52:54.575 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:52:54.575 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.576 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.576 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.576 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:52:54.576 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:52:54.577 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 1, 2022-09-03T06:52:54.577 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.578 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:52:54.578 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:52:54.578 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 1, 2022-09-03T06:52:54.579 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:52:54.579 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:52:54.579 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:52:54.580 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.580 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.580 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.581 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.581 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:52:54.581 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:52:54.581 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:52:54.583 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'list', '--bucket', 'avysmw-71', '--gen', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:52:54.583 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'list', '--bucket', 'avysmw-71', '--gen', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:52:54.584 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog list --bucket avysmw-71 --gen 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:52:54.635 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:54.635 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:54.691 INFO:teuthology.orchestra.run.smithi062.stdout:[] 2022-09-03T06:52:54.693 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:54.695 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:54.696 DEBUG:boto:Method: HEAD 2022-09-03T06:52:54.697 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:54.698 DEBUG:boto:Data: 2022-09-03T06:52:54.699 DEBUG:boto:Headers: {} 2022-09-03T06:52:54.700 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:54.700 DEBUG:boto:Port: 8000 2022-09-03T06:52:54.701 DEBUG:boto:Params: {} 2022-09-03T06:52:54.702 DEBUG:boto:Token: None 2022-09-03T06:52:54.703 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:54 GMT /avysmw-71/ 2022-09-03T06:52:54.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tFfyjGS/WB9CavAuehbmoG/KYwY= 2022-09-03T06:52:54.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tFfyjGS/WB9CavAuehbmoG/KYwY=', 'Content-Length': '0'} 2022-09-03T06:52:54.706 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b709d38ef01c1d46-006312f9c6-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:54.707 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:52:54.708 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:52:54.709 DEBUG:boto:Method: PUT 2022-09-03T06:52:54.710 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:52:54.710 DEBUG:boto:Data: 2022-09-03T06:52:54.711 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:52:54.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:54.713 DEBUG:boto:Port: 8000 2022-09-03T06:52:54.714 DEBUG:boto:Params: {} 2022-09-03T06:52:54.714 DEBUG:boto:Token: None 2022-09-03T06:52:54.715 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:52:54 GMT /avysmw-71/e 2022-09-03T06:52:54.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ayGh1ZrX/qwoDNt/AabwWp2VwpQ= 2022-09-03T06:52:54.716 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ayGh1ZrX/qwoDNt/AabwWp2VwpQ='} 2022-09-03T06:52:54.723 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000ec655750b0fd3362-006312f9c6-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:54.724 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:54.724 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:54.725 DEBUG:boto:Method: HEAD 2022-09-03T06:52:54.725 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:54.726 DEBUG:boto:Data: 2022-09-03T06:52:54.726 DEBUG:boto:Headers: {} 2022-09-03T06:52:54.727 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:54.727 DEBUG:boto:Port: 8000 2022-09-03T06:52:54.728 DEBUG:boto:Params: {} 2022-09-03T06:52:54.728 DEBUG:boto:Token: None 2022-09-03T06:52:54.729 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:54 GMT /avysmw-71/ 2022-09-03T06:52:54.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tFfyjGS/WB9CavAuehbmoG/KYwY= 2022-09-03T06:52:54.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tFfyjGS/WB9CavAuehbmoG/KYwY=', 'Content-Length': '0'} 2022-09-03T06:52:54.733 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000dbcfb33352c09958-006312f9c6-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:54.734 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:52:54.735 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:52:54.735 DEBUG:boto:Method: PUT 2022-09-03T06:52:54.736 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:52:54.736 DEBUG:boto:Data: 2022-09-03T06:52:54.737 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:52:54.737 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:54.738 DEBUG:boto:Port: 8000 2022-09-03T06:52:54.739 DEBUG:boto:Params: {} 2022-09-03T06:52:54.740 DEBUG:boto:Token: None 2022-09-03T06:52:54.740 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:52:54 GMT /avysmw-71/f 2022-09-03T06:52:54.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OANmUTiwCWSJXButm1OEbw44XLk= 2022-09-03T06:52:54.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OANmUTiwCWSJXButm1OEbw44XLk='} 2022-09-03T06:52:54.786 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000051681f3b5015d3cc-006312f9c6-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:54.787 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:54.788 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:54.789 DEBUG:boto:Method: HEAD 2022-09-03T06:52:54.789 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:54.790 DEBUG:boto:Data: 2022-09-03T06:52:54.790 DEBUG:boto:Headers: {} 2022-09-03T06:52:54.791 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:54.791 DEBUG:boto:Port: 8000 2022-09-03T06:52:54.792 DEBUG:boto:Params: {} 2022-09-03T06:52:54.792 DEBUG:boto:Token: None 2022-09-03T06:52:54.793 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:54 GMT /avysmw-71/ 2022-09-03T06:52:54.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tFfyjGS/WB9CavAuehbmoG/KYwY= 2022-09-03T06:52:54.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tFfyjGS/WB9CavAuehbmoG/KYwY=', 'Content-Length': '0'} 2022-09-03T06:52:54.797 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008262df17c8bfd493-006312f9c6-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:54.797 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:52:54.799 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:52:54.799 DEBUG:boto:Method: PUT 2022-09-03T06:52:54.800 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:52:54.801 DEBUG:boto:Data: 2022-09-03T06:52:54.802 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:52:54.802 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:54.803 DEBUG:boto:Port: 8000 2022-09-03T06:52:54.803 DEBUG:boto:Params: {} 2022-09-03T06:52:54.804 DEBUG:boto:Token: None 2022-09-03T06:52:54.804 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:52:54 GMT /avysmw-71/g 2022-09-03T06:52:54.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:13w0sArxbqKLwd+J13MYIyjC4Q8= 2022-09-03T06:52:54.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:13w0sArxbqKLwd+J13MYIyjC4Q8='} 2022-09-03T06:52:54.816 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000413774413a862747-006312f9c6-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:54.817 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:52:54.817 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:52:54.818 DEBUG:boto:Method: HEAD 2022-09-03T06:52:54.818 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:52:54.819 DEBUG:boto:Data: 2022-09-03T06:52:54.820 DEBUG:boto:Headers: {} 2022-09-03T06:52:54.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:54.821 DEBUG:boto:Port: 8000 2022-09-03T06:52:54.821 DEBUG:boto:Params: {} 2022-09-03T06:52:54.822 DEBUG:boto:Token: None 2022-09-03T06:52:54.823 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:52:54 GMT /avysmw-71/ 2022-09-03T06:52:54.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tFfyjGS/WB9CavAuehbmoG/KYwY= 2022-09-03T06:52:54.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tFfyjGS/WB9CavAuehbmoG/KYwY=', 'Content-Length': '0'} 2022-09-03T06:52:54.826 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '21'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000c94724e19674319-006312f9c6-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:54.827 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:52:54.827 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:52:54.828 DEBUG:boto:Method: PUT 2022-09-03T06:52:54.829 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:52:54.829 DEBUG:boto:Data: 2022-09-03T06:52:54.830 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:52:54.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:52:54.831 DEBUG:boto:Port: 8000 2022-09-03T06:52:54.834 DEBUG:boto:Params: {} 2022-09-03T06:52:54.835 DEBUG:boto:Token: None 2022-09-03T06:52:54.836 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:52:54 GMT /avysmw-71/h 2022-09-03T06:52:54.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2JJ/0XgI5G83vkS00vWi30uZJBQ= 2022-09-03T06:52:54.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:52:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2JJ/0XgI5G83vkS00vWi30uZJBQ='} 2022-09-03T06:52:54.844 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000140c1c4b7dfb2689-006312f9c6-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:52:54.844 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-71 2022-09-03T06:52:54.845 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:54.846 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:52:54.846 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:52:54.907 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:52:54.908 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:52:54.962 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:54.959+0000 7f7e2f4a0600 1 waiting to reach latest gen 1, on 0.. 2022-09-03T06:52:59.962 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:52:59.960+0000 7f7e2f4a0600 1 waiting to reach latest gen 1, on 0.. 2022-09-03T06:53:04.963 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:04.961+0000 7f7e2f4a0600 1 waiting to reach latest gen 1, on 0.. 2022-09-03T06:53:09.967 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:09.965+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:09.968 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:53:09.968 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:14.967 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:14.966+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:14.968 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:53:14.968 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:19.968 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:19.966+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:19.968 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:53:19.969 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:24.969 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:24.966+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:25.033 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:53:25.034 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:29.968 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:29.966+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:29.969 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, , 00000000002.70.5] 2022-09-03T06:53:29.969 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:34.969 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:34.967+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:34.970 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, , 00000000002.70.5] 2022-09-03T06:53:34.970 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:39.971 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:39.968+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:39.971 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, , 00000000002.70.5] 2022-09-03T06:53:39.972 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:44.971 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:44.969+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:44.972 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, , 00000000002.70.5] 2022-09-03T06:53:44.972 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:49.972 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:49.969+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:49.973 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, , 00000000002.70.5] 2022-09-03T06:53:49.973 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:54.972 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:54.970+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:54.972 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, , 00000000002.70.5] 2022-09-03T06:53:54.973 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:53:59.974 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:53:59.971+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:53:59.975 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, , 00000000002.70.5] 2022-09-03T06:53:59.975 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:54:04.974 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:04.972+0000 7f7e2f4a0600 1 waiting for incremental sync to catch up: 2022-09-03T06:54:04.975 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, , 00000000002.70.5] 2022-09-03T06:54:04.975 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:54:09.976 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:09.973+0000 7f7e2f4a0600 1 bucket sync caught up with source: 2022-09-03T06:54:09.976 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:54:09.977 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:54:09.977 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:09.973+0000 7f7e2f4a0600 0 bucket checkpoint complete 2022-09-03T06:54:10.003 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-71 2022-09-03T06:54:10.004 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:10.005 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:10.007 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:54:10.074 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:54:10.075 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:54:10.137 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:10.136+0000 7fee0d6ea600 1 bucket sync caught up with source: 2022-09-03T06:54:10.138 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:54:10.138 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:54:10.138 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:10.136+0000 7fee0d6ea600 0 bucket checkpoint complete 2022-09-03T06:54:10.157 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-71 2022-09-03T06:54:10.158 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:10.159 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:10.160 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:54:10.288 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:54:10.289 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:54:10.328 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:10.326+0000 7f096804c600 1 bucket sync caught up with source: 2022-09-03T06:54:10.329 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:54:10.329 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.67.5, 00000000002.52.5, 00000000002.70.5] 2022-09-03T06:54:10.330 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:10.326+0000 7f096804c600 0 bucket checkpoint complete 2022-09-03T06:54:10.359 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-71 2022-09-03T06:54:10.360 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:10.361 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:10.362 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:54:10.424 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:54:10.424 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:54:10.472 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:10.470+0000 7f206e198600 1 waiting for incremental sync to catch up: 2022-09-03T06:54:10.472 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.309.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:10.472 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:15.473 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:15.471+0000 7f206e198600 1 waiting for incremental sync to catch up: 2022-09-03T06:54:15.474 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.309.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:15.475 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:20.475 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:20.473+0000 7f206e198600 1 waiting for incremental sync to catch up: 2022-09-03T06:54:20.476 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.309.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:20.476 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:25.482 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:25.474+0000 7f206e198600 1 bucket sync caught up with source: 2022-09-03T06:54:25.483 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:25.483 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:25.483 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:25.474+0000 7f206e198600 0 bucket checkpoint complete 2022-09-03T06:54:25.499 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-71 2022-09-03T06:54:25.500 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:25.500 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:25.501 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:54:25.566 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:54:25.567 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:54:25.621 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:25.617+0000 7f3a36daf600 1 bucket sync caught up with source: 2022-09-03T06:54:25.622 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:25.622 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:54:25.623 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:25.617+0000 7f3a36daf600 0 bucket checkpoint complete 2022-09-03T06:54:25.638 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-71 2022-09-03T06:54:25.638 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-71 2022-09-03T06:54:25.639 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:25.640 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:25.641 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:54:25.701 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:54:25.701 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:54:25.749 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:25.747+0000 7fe501014600 1 bucket sync caught up with source: 2022-09-03T06:54:25.750 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:25.751 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:25.751 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:54:25.747+0000 7fe501014600 0 bucket checkpoint complete 2022-09-03T06:54:25.765 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-71 2022-09-03T06:54:25.766 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:25.767 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:25.767 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:54:25.824 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:54:25.824 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:54:25.895 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:25.893+0000 7fb883fbb600 1 waiting for incremental sync to catch up: 2022-09-03T06:54:25.895 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.373.5, , , , 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:25.896 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:30.901 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:30.894+0000 7fb883fbb600 1 waiting for incremental sync to catch up: 2022-09-03T06:54:30.902 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.373.5, , , , 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:30.903 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:35.897 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:35.895+0000 7fb883fbb600 1 waiting for incremental sync to catch up: 2022-09-03T06:54:35.898 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.373.5, , , , 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:35.898 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:40.899 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:40.896+0000 7fb883fbb600 1 waiting for incremental sync to catch up: 2022-09-03T06:54:40.899 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.373.5, , , , 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:40.899 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:45.905 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:45.898+0000 7fb883fbb600 1 bucket sync caught up with source: 2022-09-03T06:54:45.906 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:45.906 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:54:45.907 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:45.898+0000 7fb883fbb600 0 bucket checkpoint complete 2022-09-03T06:54:45.925 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-71 2022-09-03T06:54:45.926 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-71 zones={test-zone2, test-zone1} 2022-09-03T06:54:45.927 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:45.928 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:45.929 DEBUG:boto:Method: HEAD 2022-09-03T06:54:45.929 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:54:45.930 DEBUG:boto:Data: 2022-09-03T06:54:45.930 DEBUG:boto:Headers: {} 2022-09-03T06:54:45.931 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:45.932 DEBUG:boto:Port: 8000 2022-09-03T06:54:45.932 DEBUG:boto:Params: {} 2022-09-03T06:54:45.933 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:54:45.934 DEBUG:boto:Token: None 2022-09-03T06:54:45.936 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:45 GMT /avysmw-71/ 2022-09-03T06:54:45.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JDrgdL50irI2C9V3SO8xp4mjqL0= 2022-09-03T06:54:45.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JDrgdL50irI2C9V3SO8xp4mjqL0=', 'Content-Length': '0'} 2022-09-03T06:54:45.945 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007bd0629d6cb4e145-006312fa35-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:54:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:45.947 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:45.948 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:45.948 DEBUG:boto:Method: HEAD 2022-09-03T06:54:45.949 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:54:45.950 DEBUG:boto:Data: 2022-09-03T06:54:45.950 DEBUG:boto:Headers: {} 2022-09-03T06:54:45.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:45.951 DEBUG:boto:Port: 8000 2022-09-03T06:54:45.952 DEBUG:boto:Params: {} 2022-09-03T06:54:45.953 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:54:45.953 DEBUG:boto:Token: None 2022-09-03T06:54:45.954 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:45 GMT /avysmw-71/ 2022-09-03T06:54:45.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JDrgdL50irI2C9V3SO8xp4mjqL0= 2022-09-03T06:54:45.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JDrgdL50irI2C9V3SO8xp4mjqL0=', 'Content-Length': '0'} 2022-09-03T06:54:45.959 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cd6754834020da0b-006312fa35-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:54:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:45.960 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:54:45.961 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:45.961 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:45.962 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:45.963 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:45.963 DEBUG:boto:Method: GET 2022-09-03T06:54:45.964 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:45.964 DEBUG:boto:Data: 2022-09-03T06:54:45.965 DEBUG:boto:Headers: {} 2022-09-03T06:54:45.966 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:45.966 DEBUG:boto:Port: 8000 2022-09-03T06:54:45.967 DEBUG:boto:Params: {} 2022-09-03T06:54:45.968 DEBUG:boto:Token: None 2022-09-03T06:54:45.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:45 GMT /avysmw-71/?versions 2022-09-03T06:54:45.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gbtXWMQ4NIPD2J7rX+CViGZgG30= 2022-09-03T06:54:45.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gbtXWMQ4NIPD2J7rX+CViGZgG30=', 'Content-Length': '0'} 2022-09-03T06:54:45.973 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000818e97196a8bc8a8-006312fa35-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:45.974 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:45.976 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:54:45.976 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:54:45.977 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:54:45.978 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:54:45.978 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:54:45.979 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:54:45.979 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:54:45.980 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:54:45.980 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:54:45.981 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:45.982 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:45.982 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:45.983 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:45.983 DEBUG:boto:Method: GET 2022-09-03T06:54:45.984 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:45.984 DEBUG:boto:Data: 2022-09-03T06:54:45.985 DEBUG:boto:Headers: {} 2022-09-03T06:54:45.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:45.986 DEBUG:boto:Port: 8000 2022-09-03T06:54:45.986 DEBUG:boto:Params: {} 2022-09-03T06:54:45.987 DEBUG:boto:Token: None 2022-09-03T06:54:45.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:45 GMT /avysmw-71/?versions 2022-09-03T06:54:45.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gbtXWMQ4NIPD2J7rX+CViGZgG30= 2022-09-03T06:54:45.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gbtXWMQ4NIPD2J7rX+CViGZgG30=', 'Content-Length': '0'} 2022-09-03T06:54:45.992 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e56caf316f14b646-006312fa35-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:45.993 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:45.994 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:54:45.995 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:54:45.995 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:54:45.996 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:54:45.996 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:54:45.997 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:54:45.997 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:54:45.998 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:54:45.999 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:45.999 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.000 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:46.000 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:46.001 DEBUG:boto:Method: GET 2022-09-03T06:54:46.001 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:46.002 DEBUG:boto:Data: 2022-09-03T06:54:46.002 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.003 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.004 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.004 DEBUG:boto:Params: {} 2022-09-03T06:54:46.005 DEBUG:boto:Token: None 2022-09-03T06:54:46.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/?versions 2022-09-03T06:54:46.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xyA/eRZOWwHKqyd/2aH4+fmEI1g= 2022-09-03T06:54:46.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xyA/eRZOWwHKqyd/2aH4+fmEI1g=', 'Content-Length': '0'} 2022-09-03T06:54:46.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f9c6d636f324dce-006312fa36-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.011 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.012 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.013 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.014 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:46.014 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:46.016 DEBUG:boto:Method: GET 2022-09-03T06:54:46.017 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:46.017 DEBUG:boto:Data: 2022-09-03T06:54:46.018 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.018 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.019 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.019 DEBUG:boto:Params: {} 2022-09-03T06:54:46.020 DEBUG:boto:Token: None 2022-09-03T06:54:46.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/?versions 2022-09-03T06:54:46.021 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xyA/eRZOWwHKqyd/2aH4+fmEI1g= 2022-09-03T06:54:46.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xyA/eRZOWwHKqyd/2aH4+fmEI1g=', 'Content-Length': '0'} 2022-09-03T06:54:46.024 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000005aeeca4a7a8dc87-006312fa36-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.025 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.027 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:54:46.028 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.029 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.029 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.030 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.030 DEBUG:boto:Method: GET 2022-09-03T06:54:46.031 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:46.031 DEBUG:boto:Data: 2022-09-03T06:54:46.032 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.033 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.033 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.034 DEBUG:boto:Params: {} 2022-09-03T06:54:46.034 DEBUG:boto:Token: None 2022-09-03T06:54:46.035 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:46.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G4n7jDfdh8Si8GdnA+biu7zrueg= 2022-09-03T06:54:46.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G4n7jDfdh8Si8GdnA+biu7zrueg=', 'Content-Length': '0'} 2022-09-03T06:54:46.038 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc03505a4b16fbd3-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.039 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.040 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.040 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.041 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.042 DEBUG:boto:Method: GET 2022-09-03T06:54:46.042 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:46.043 DEBUG:boto:Data: 2022-09-03T06:54:46.043 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.045 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.045 DEBUG:boto:Params: {} 2022-09-03T06:54:46.046 DEBUG:boto:Token: None 2022-09-03T06:54:46.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:46.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G4n7jDfdh8Si8GdnA+biu7zrueg= 2022-09-03T06:54:46.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G4n7jDfdh8Si8GdnA+biu7zrueg=', 'Content-Length': '0'} 2022-09-03T06:54:46.050 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e43bbbd28a70071-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.051 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.052 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.053 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.053 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.054 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.055 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:46.055 DEBUG:boto:Data: 2022-09-03T06:54:46.056 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.057 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.057 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.058 DEBUG:boto:Params: {} 2022-09-03T06:54:46.058 DEBUG:boto:Token: None 2022-09-03T06:54:46.059 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:46.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Gm6PrQOfTMcTKiM1Y1NY8jZ6LE= 2022-09-03T06:54:46.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Gm6PrQOfTMcTKiM1Y1NY8jZ6LE=', 'Content-Length': '0'} 2022-09-03T06:54:46.063 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8fe822cd631c504-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.064 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.064 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.065 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.065 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.066 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.066 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:46.067 DEBUG:boto:Data: 2022-09-03T06:54:46.067 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.068 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.068 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.069 DEBUG:boto:Params: {} 2022-09-03T06:54:46.070 DEBUG:boto:Token: None 2022-09-03T06:54:46.070 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:46.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Gm6PrQOfTMcTKiM1Y1NY8jZ6LE= 2022-09-03T06:54:46.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Gm6PrQOfTMcTKiM1Y1NY8jZ6LE=', 'Content-Length': '0'} 2022-09-03T06:54:46.074 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057919c9e533dc1bc-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.074 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:54:46.075 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.076 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.076 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.077 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.077 DEBUG:boto:Method: GET 2022-09-03T06:54:46.078 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:46.078 DEBUG:boto:Data: 2022-09-03T06:54:46.079 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.079 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.080 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.081 DEBUG:boto:Params: {} 2022-09-03T06:54:46.081 DEBUG:boto:Token: None 2022-09-03T06:54:46.082 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:46.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G4n7jDfdh8Si8GdnA+biu7zrueg= 2022-09-03T06:54:46.086 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G4n7jDfdh8Si8GdnA+biu7zrueg=', 'Content-Length': '0'} 2022-09-03T06:54:46.089 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075bbab7e91030f12-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.134 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.135 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.135 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.136 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:46.137 DEBUG:boto:Method: GET 2022-09-03T06:54:46.137 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:46.138 DEBUG:boto:Data: 2022-09-03T06:54:46.138 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.139 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.140 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.140 DEBUG:boto:Params: {} 2022-09-03T06:54:46.141 DEBUG:boto:Token: None 2022-09-03T06:54:46.144 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:46.145 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G4n7jDfdh8Si8GdnA+biu7zrueg= 2022-09-03T06:54:46.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G4n7jDfdh8Si8GdnA+biu7zrueg=', 'Content-Length': '0'} 2022-09-03T06:54:46.148 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008bd7b8d04ff0371b-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.149 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.149 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.150 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:46.150 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:46.151 DEBUG:boto:Method: GET 2022-09-03T06:54:46.151 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:46.152 DEBUG:boto:Data: 2022-09-03T06:54:46.153 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.153 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.154 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.154 DEBUG:boto:Params: {} 2022-09-03T06:54:46.156 DEBUG:boto:Token: None 2022-09-03T06:54:46.157 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.161 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001210ae2241ee51a7-006312fa36-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.161 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.162 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.163 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.164 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:46.164 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:46.165 DEBUG:boto:Method: GET 2022-09-03T06:54:46.166 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:46.166 DEBUG:boto:Data: 2022-09-03T06:54:46.167 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.168 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.168 DEBUG:boto:Params: {} 2022-09-03T06:54:46.169 DEBUG:boto:Token: None 2022-09-03T06:54:46.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.173 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028a097a76c0739e3-006312fa36-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.174 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.175 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.175 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.176 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.177 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:46.177 DEBUG:boto:Data: 2022-09-03T06:54:46.178 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.179 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.180 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.181 DEBUG:boto:Params: {} 2022-09-03T06:54:46.182 DEBUG:boto:Token: None 2022-09-03T06:54:46.183 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a 2022-09-03T06:54:46.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9uBIms0YduFCywNo6Js7ocdzPg= 2022-09-03T06:54:46.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9uBIms0YduFCywNo6Js7ocdzPg=', 'Content-Length': '0'} 2022-09-03T06:54:46.188 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a7b6e107100f124-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.189 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.190 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.191 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.192 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:46.193 DEBUG:boto:Data: 2022-09-03T06:54:46.194 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.195 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.196 DEBUG:boto:Params: {} 2022-09-03T06:54:46.197 DEBUG:boto:Token: None 2022-09-03T06:54:46.198 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a 2022-09-03T06:54:46.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9uBIms0YduFCywNo6Js7ocdzPg= 2022-09-03T06:54:46.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9uBIms0YduFCywNo6Js7ocdzPg=', 'Content-Length': '0'} 2022-09-03T06:54:46.201 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006709798c21f17866-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.202 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:54:46.202 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.203 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.203 DEBUG:boto:Method: GET 2022-09-03T06:54:46.204 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:46.205 DEBUG:boto:Data: 2022-09-03T06:54:46.205 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.206 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.206 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.207 DEBUG:boto:Params: {} 2022-09-03T06:54:46.207 DEBUG:boto:Token: None 2022-09-03T06:54:46.208 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a 2022-09-03T06:54:46.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZepHAtIIj2/KNhijVGgwGEymqcs= 2022-09-03T06:54:46.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZepHAtIIj2/KNhijVGgwGEymqcs=', 'Content-Length': '0'} 2022-09-03T06:54:46.211 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008650ca60aaec36b5-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.212 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:46.212 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:46.213 DEBUG:boto:Method: GET 2022-09-03T06:54:46.213 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:46.214 DEBUG:boto:Data: 2022-09-03T06:54:46.214 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.216 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.216 DEBUG:boto:Params: {} 2022-09-03T06:54:46.217 DEBUG:boto:Token: None 2022-09-03T06:54:46.218 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/a 2022-09-03T06:54:46.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZepHAtIIj2/KNhijVGgwGEymqcs= 2022-09-03T06:54:46.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZepHAtIIj2/KNhijVGgwGEymqcs=', 'Content-Length': '0'} 2022-09-03T06:54:46.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b2133c8d5193094-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.226 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.227 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.227 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:46.228 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:46.228 DEBUG:boto:Method: GET 2022-09-03T06:54:46.229 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:46.229 DEBUG:boto:Data: 2022-09-03T06:54:46.230 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.230 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.231 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.231 DEBUG:boto:Params: {} 2022-09-03T06:54:46.232 DEBUG:boto:Token: None 2022-09-03T06:54:46.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.237 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d93bcf5b66693243-006312fa36-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.237 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.238 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.239 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.239 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:46.240 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:46.240 DEBUG:boto:Method: GET 2022-09-03T06:54:46.241 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:46.241 DEBUG:boto:Data: 2022-09-03T06:54:46.242 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.242 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.243 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.243 DEBUG:boto:Params: {} 2022-09-03T06:54:46.244 DEBUG:boto:Token: None 2022-09-03T06:54:46.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.246 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.248 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009f28b39455ab5539-006312fa36-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.249 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.249 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:54:46.250 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.251 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.251 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.252 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.252 DEBUG:boto:Method: GET 2022-09-03T06:54:46.253 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:46.253 DEBUG:boto:Data: 2022-09-03T06:54:46.254 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.254 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.255 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.255 DEBUG:boto:Params: {} 2022-09-03T06:54:46.256 DEBUG:boto:Token: None 2022-09-03T06:54:46.257 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:46.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKJTCPdyDZfuLVBQpW3bEVURXbc= 2022-09-03T06:54:46.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKJTCPdyDZfuLVBQpW3bEVURXbc=', 'Content-Length': '0'} 2022-09-03T06:54:46.260 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000846d1296980cb661-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.261 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.262 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.262 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.263 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.263 DEBUG:boto:Method: GET 2022-09-03T06:54:46.264 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:46.264 DEBUG:boto:Data: 2022-09-03T06:54:46.265 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.265 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.266 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.266 DEBUG:boto:Params: {} 2022-09-03T06:54:46.267 DEBUG:boto:Token: None 2022-09-03T06:54:46.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:46.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKJTCPdyDZfuLVBQpW3bEVURXbc= 2022-09-03T06:54:46.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKJTCPdyDZfuLVBQpW3bEVURXbc=', 'Content-Length': '0'} 2022-09-03T06:54:46.271 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000432ac0954ae9358f-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.272 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.272 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.273 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.273 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.274 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.274 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:46.275 DEBUG:boto:Data: 2022-09-03T06:54:46.275 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.276 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.276 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.277 DEBUG:boto:Params: {} 2022-09-03T06:54:46.277 DEBUG:boto:Token: None 2022-09-03T06:54:46.278 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:46.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XJg5mgRLtlyn6F7Q84hXqALj1sE= 2022-09-03T06:54:46.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XJg5mgRLtlyn6F7Q84hXqALj1sE=', 'Content-Length': '0'} 2022-09-03T06:54:46.281 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000991a2c7a2655f37f-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.282 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.284 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.285 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.285 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.286 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.286 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:46.287 DEBUG:boto:Data: 2022-09-03T06:54:46.287 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.289 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.290 DEBUG:boto:Params: {} 2022-09-03T06:54:46.290 DEBUG:boto:Token: None 2022-09-03T06:54:46.291 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:46.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XJg5mgRLtlyn6F7Q84hXqALj1sE= 2022-09-03T06:54:46.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XJg5mgRLtlyn6F7Q84hXqALj1sE=', 'Content-Length': '0'} 2022-09-03T06:54:46.296 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001cc6fc8ca4c186ed-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.297 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:54:46.298 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.299 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.300 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.300 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.301 DEBUG:boto:Method: GET 2022-09-03T06:54:46.301 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:46.302 DEBUG:boto:Data: 2022-09-03T06:54:46.303 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.304 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.305 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.306 DEBUG:boto:Params: {} 2022-09-03T06:54:46.307 DEBUG:boto:Token: None 2022-09-03T06:54:46.308 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:46.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKJTCPdyDZfuLVBQpW3bEVURXbc= 2022-09-03T06:54:46.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKJTCPdyDZfuLVBQpW3bEVURXbc=', 'Content-Length': '0'} 2022-09-03T06:54:46.312 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000183033af286219b1-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.313 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.314 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.315 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.315 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:46.316 DEBUG:boto:Method: GET 2022-09-03T06:54:46.316 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:46.317 DEBUG:boto:Data: 2022-09-03T06:54:46.317 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.318 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.319 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.319 DEBUG:boto:Params: {} 2022-09-03T06:54:46.320 DEBUG:boto:Token: None 2022-09-03T06:54:46.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:46.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OKJTCPdyDZfuLVBQpW3bEVURXbc= 2022-09-03T06:54:46.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OKJTCPdyDZfuLVBQpW3bEVURXbc=', 'Content-Length': '0'} 2022-09-03T06:54:46.324 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a40a41be2a9df05f-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.366 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.367 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.367 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:46.368 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:46.368 DEBUG:boto:Method: GET 2022-09-03T06:54:46.369 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:46.369 DEBUG:boto:Data: 2022-09-03T06:54:46.370 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.371 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.371 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.372 DEBUG:boto:Params: {} 2022-09-03T06:54:46.372 DEBUG:boto:Token: None 2022-09-03T06:54:46.373 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.377 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7dae24e27165ff8-006312fa36-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.377 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.378 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.379 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.379 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:46.380 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:46.380 DEBUG:boto:Method: GET 2022-09-03T06:54:46.381 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:46.382 DEBUG:boto:Data: 2022-09-03T06:54:46.382 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.383 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.385 DEBUG:boto:Params: {} 2022-09-03T06:54:46.385 DEBUG:boto:Token: None 2022-09-03T06:54:46.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.389 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002ecd8cd5367c67df-006312fa36-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.391 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.392 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.393 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.393 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.394 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:46.394 DEBUG:boto:Data: 2022-09-03T06:54:46.394 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.395 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.395 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.396 DEBUG:boto:Params: {} 2022-09-03T06:54:46.396 DEBUG:boto:Token: None 2022-09-03T06:54:46.397 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b 2022-09-03T06:54:46.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:REq/OhawyklAeqCzySp6v2q1iRE= 2022-09-03T06:54:46.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:REq/OhawyklAeqCzySp6v2q1iRE=', 'Content-Length': '0'} 2022-09-03T06:54:46.401 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002560ec1066d11642-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.402 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.402 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.403 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.403 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:46.404 DEBUG:boto:Data: 2022-09-03T06:54:46.404 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.405 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.406 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.407 DEBUG:boto:Params: {} 2022-09-03T06:54:46.408 DEBUG:boto:Token: None 2022-09-03T06:54:46.409 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b 2022-09-03T06:54:46.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:REq/OhawyklAeqCzySp6v2q1iRE= 2022-09-03T06:54:46.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:REq/OhawyklAeqCzySp6v2q1iRE=', 'Content-Length': '0'} 2022-09-03T06:54:46.414 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f016c7dc8edf63a4-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.415 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:54:46.415 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.416 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.417 DEBUG:boto:Method: GET 2022-09-03T06:54:46.418 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:46.419 DEBUG:boto:Data: 2022-09-03T06:54:46.420 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.421 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.421 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.422 DEBUG:boto:Params: {} 2022-09-03T06:54:46.423 DEBUG:boto:Token: None 2022-09-03T06:54:46.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b 2022-09-03T06:54:46.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8YyN8pkaNesAXuCXDdJ7zKZbT1s= 2022-09-03T06:54:46.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8YyN8pkaNesAXuCXDdJ7zKZbT1s=', 'Content-Length': '0'} 2022-09-03T06:54:46.427 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e93bfbe29d3f7858-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.428 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:46.429 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:46.429 DEBUG:boto:Method: GET 2022-09-03T06:54:46.430 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:46.430 DEBUG:boto:Data: 2022-09-03T06:54:46.431 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.432 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.433 DEBUG:boto:Params: {} 2022-09-03T06:54:46.433 DEBUG:boto:Token: None 2022-09-03T06:54:46.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/b 2022-09-03T06:54:46.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8YyN8pkaNesAXuCXDdJ7zKZbT1s= 2022-09-03T06:54:46.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8YyN8pkaNesAXuCXDdJ7zKZbT1s=', 'Content-Length': '0'} 2022-09-03T06:54:46.437 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3ef9fa701fa5736-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.439 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.440 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.440 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:46.441 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:46.441 DEBUG:boto:Method: GET 2022-09-03T06:54:46.442 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:46.443 DEBUG:boto:Data: 2022-09-03T06:54:46.443 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.444 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.444 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.445 DEBUG:boto:Params: {} 2022-09-03T06:54:46.446 DEBUG:boto:Token: None 2022-09-03T06:54:46.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.451 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d458cd29d2ab3671-006312fa36-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.452 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.453 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.453 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.454 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:46.454 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:46.455 DEBUG:boto:Method: GET 2022-09-03T06:54:46.456 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:46.457 DEBUG:boto:Data: 2022-09-03T06:54:46.457 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.459 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.459 DEBUG:boto:Params: {} 2022-09-03T06:54:46.460 DEBUG:boto:Token: None 2022-09-03T06:54:46.461 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.464 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ae35fd5dfc45170e-006312fa36-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.465 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.466 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:54:46.467 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.468 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.468 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.469 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.469 DEBUG:boto:Method: GET 2022-09-03T06:54:46.470 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:46.471 DEBUG:boto:Data: 2022-09-03T06:54:46.471 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.472 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.472 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.473 DEBUG:boto:Params: {} 2022-09-03T06:54:46.473 DEBUG:boto:Token: None 2022-09-03T06:54:46.474 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:46.475 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JACXFrqiEcGHeTRnvh/hKMfhbSc= 2022-09-03T06:54:46.476 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JACXFrqiEcGHeTRnvh/hKMfhbSc=', 'Content-Length': '0'} 2022-09-03T06:54:46.478 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075c5d59dc7815ab8-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.479 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.480 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.480 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.481 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.481 DEBUG:boto:Method: GET 2022-09-03T06:54:46.482 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:46.482 DEBUG:boto:Data: 2022-09-03T06:54:46.483 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.484 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.485 DEBUG:boto:Params: {} 2022-09-03T06:54:46.485 DEBUG:boto:Token: None 2022-09-03T06:54:46.486 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:46.486 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JACXFrqiEcGHeTRnvh/hKMfhbSc= 2022-09-03T06:54:46.487 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JACXFrqiEcGHeTRnvh/hKMfhbSc=', 'Content-Length': '0'} 2022-09-03T06:54:46.490 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c3a4e42dee83d27-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.490 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.491 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.492 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.492 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.493 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.493 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:46.494 DEBUG:boto:Data: 2022-09-03T06:54:46.494 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.495 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.495 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.496 DEBUG:boto:Params: {} 2022-09-03T06:54:46.496 DEBUG:boto:Token: None 2022-09-03T06:54:46.497 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:46.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KL9vE+DVHUI9dNpSQmrLoUUuCTQ= 2022-09-03T06:54:46.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KL9vE+DVHUI9dNpSQmrLoUUuCTQ=', 'Content-Length': '0'} 2022-09-03T06:54:46.500 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c67742e865cb5e8-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.501 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.502 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.502 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.503 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.503 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.504 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:46.504 DEBUG:boto:Data: 2022-09-03T06:54:46.505 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.506 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.506 DEBUG:boto:Params: {} 2022-09-03T06:54:46.507 DEBUG:boto:Token: None 2022-09-03T06:54:46.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:46.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KL9vE+DVHUI9dNpSQmrLoUUuCTQ= 2022-09-03T06:54:46.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KL9vE+DVHUI9dNpSQmrLoUUuCTQ=', 'Content-Length': '0'} 2022-09-03T06:54:46.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038603730080e40ff-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.512 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:54:46.512 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.513 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.513 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.514 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.515 DEBUG:boto:Method: GET 2022-09-03T06:54:46.515 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:46.516 DEBUG:boto:Data: 2022-09-03T06:54:46.516 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.517 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.517 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.518 DEBUG:boto:Params: {} 2022-09-03T06:54:46.519 DEBUG:boto:Token: None 2022-09-03T06:54:46.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:46.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JACXFrqiEcGHeTRnvh/hKMfhbSc= 2022-09-03T06:54:46.521 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JACXFrqiEcGHeTRnvh/hKMfhbSc=', 'Content-Length': '0'} 2022-09-03T06:54:46.523 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004960e8f1cd3a8b0-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.566 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.567 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.568 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.569 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:46.570 DEBUG:boto:Method: GET 2022-09-03T06:54:46.571 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:46.571 DEBUG:boto:Data: 2022-09-03T06:54:46.572 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.573 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.574 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.575 DEBUG:boto:Params: {} 2022-09-03T06:54:46.576 DEBUG:boto:Token: None 2022-09-03T06:54:46.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:46.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JACXFrqiEcGHeTRnvh/hKMfhbSc= 2022-09-03T06:54:46.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JACXFrqiEcGHeTRnvh/hKMfhbSc=', 'Content-Length': '0'} 2022-09-03T06:54:46.583 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e35f6c6f8e2e3d0-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.584 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.586 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.587 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:46.588 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:46.589 DEBUG:boto:Method: GET 2022-09-03T06:54:46.590 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:46.591 DEBUG:boto:Data: 2022-09-03T06:54:46.592 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.593 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.594 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.595 DEBUG:boto:Params: {} 2022-09-03T06:54:46.596 DEBUG:boto:Token: None 2022-09-03T06:54:46.597 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.598 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.604 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db7156646dfbd821-006312fa36-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.605 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.607 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.608 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.609 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:46.610 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:46.611 DEBUG:boto:Method: GET 2022-09-03T06:54:46.613 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:46.614 DEBUG:boto:Data: 2022-09-03T06:54:46.615 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.617 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.619 DEBUG:boto:Params: {} 2022-09-03T06:54:46.621 DEBUG:boto:Token: None 2022-09-03T06:54:46.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.627 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009f535f0eb7c0964e-006312fa36-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.628 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.629 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.630 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.631 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.632 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:46.633 DEBUG:boto:Data: 2022-09-03T06:54:46.634 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.635 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.636 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.637 DEBUG:boto:Params: {} 2022-09-03T06:54:46.638 DEBUG:boto:Token: None 2022-09-03T06:54:46.638 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c 2022-09-03T06:54:46.639 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4FlfICI/jIQa5EZwX94b33tKBiQ= 2022-09-03T06:54:46.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4FlfICI/jIQa5EZwX94b33tKBiQ=', 'Content-Length': '0'} 2022-09-03T06:54:46.646 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000059ef188558aac9ef-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.647 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.647 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.648 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.648 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:46.649 DEBUG:boto:Data: 2022-09-03T06:54:46.650 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.650 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.651 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.651 DEBUG:boto:Params: {} 2022-09-03T06:54:46.652 DEBUG:boto:Token: None 2022-09-03T06:54:46.653 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c 2022-09-03T06:54:46.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4FlfICI/jIQa5EZwX94b33tKBiQ= 2022-09-03T06:54:46.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4FlfICI/jIQa5EZwX94b33tKBiQ=', 'Content-Length': '0'} 2022-09-03T06:54:46.657 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e478217403dbc404-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.658 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:54:46.658 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.659 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.660 DEBUG:boto:Method: GET 2022-09-03T06:54:46.660 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:46.661 DEBUG:boto:Data: 2022-09-03T06:54:46.661 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.662 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.662 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.663 DEBUG:boto:Params: {} 2022-09-03T06:54:46.664 DEBUG:boto:Token: None 2022-09-03T06:54:46.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c 2022-09-03T06:54:46.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YkZb4NCFy5pUOpZQAFdG8mx2VbU= 2022-09-03T06:54:46.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YkZb4NCFy5pUOpZQAFdG8mx2VbU=', 'Content-Length': '0'} 2022-09-03T06:54:46.668 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3fb93bb835510a0-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.671 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:46.672 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:46.674 DEBUG:boto:Method: GET 2022-09-03T06:54:46.675 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:46.676 DEBUG:boto:Data: 2022-09-03T06:54:46.677 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.679 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.680 DEBUG:boto:Params: {} 2022-09-03T06:54:46.681 DEBUG:boto:Token: None 2022-09-03T06:54:46.682 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/c 2022-09-03T06:54:46.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YkZb4NCFy5pUOpZQAFdG8mx2VbU= 2022-09-03T06:54:46.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YkZb4NCFy5pUOpZQAFdG8mx2VbU=', 'Content-Length': '0'} 2022-09-03T06:54:46.689 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000021531964a8046a07-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.690 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.691 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.692 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:46.693 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:46.694 DEBUG:boto:Method: GET 2022-09-03T06:54:46.695 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:46.696 DEBUG:boto:Data: 2022-09-03T06:54:46.696 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.697 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.697 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.698 DEBUG:boto:Params: {} 2022-09-03T06:54:46.699 DEBUG:boto:Token: None 2022-09-03T06:54:46.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.706 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005c093372cffe1d3e-006312fa36-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.707 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.708 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.708 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.709 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:46.710 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:46.710 DEBUG:boto:Method: GET 2022-09-03T06:54:46.711 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:46.712 DEBUG:boto:Data: 2022-09-03T06:54:46.712 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.713 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.714 DEBUG:boto:Params: {} 2022-09-03T06:54:46.715 DEBUG:boto:Token: None 2022-09-03T06:54:46.715 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.719 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000352884d419af9b1a-006312fa36-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.719 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.720 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:54:46.721 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.721 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.722 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.722 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.723 DEBUG:boto:Method: GET 2022-09-03T06:54:46.724 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:46.724 DEBUG:boto:Data: 2022-09-03T06:54:46.725 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.725 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.726 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.726 DEBUG:boto:Params: {} 2022-09-03T06:54:46.727 DEBUG:boto:Token: None 2022-09-03T06:54:46.728 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:46.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Tr2OwLVKQgiOBKcmIT3JKmWTKA= 2022-09-03T06:54:46.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Tr2OwLVKQgiOBKcmIT3JKmWTKA=', 'Content-Length': '0'} 2022-09-03T06:54:46.731 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a0a1b99530bf3a4-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.732 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.733 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.733 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.734 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.734 DEBUG:boto:Method: GET 2022-09-03T06:54:46.735 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:46.735 DEBUG:boto:Data: 2022-09-03T06:54:46.736 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.736 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.737 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.738 DEBUG:boto:Params: {} 2022-09-03T06:54:46.738 DEBUG:boto:Token: None 2022-09-03T06:54:46.739 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:46.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Tr2OwLVKQgiOBKcmIT3JKmWTKA= 2022-09-03T06:54:46.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Tr2OwLVKQgiOBKcmIT3JKmWTKA=', 'Content-Length': '0'} 2022-09-03T06:54:46.742 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d5cc5cd360b1ebc-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.786 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.787 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.788 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.789 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.790 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.791 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:46.792 DEBUG:boto:Data: 2022-09-03T06:54:46.793 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.794 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.794 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.795 DEBUG:boto:Params: {} 2022-09-03T06:54:46.796 DEBUG:boto:Token: None 2022-09-03T06:54:46.797 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:46.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PUeHjUEhUuUH4tOsn2IcEyvf4VY= 2022-09-03T06:54:46.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PUeHjUEhUuUH4tOsn2IcEyvf4VY=', 'Content-Length': '0'} 2022-09-03T06:54:46.802 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000197f682711206c13-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.804 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.805 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.805 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.806 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.807 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.808 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:46.808 DEBUG:boto:Data: 2022-09-03T06:54:46.809 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.811 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.812 DEBUG:boto:Params: {} 2022-09-03T06:54:46.813 DEBUG:boto:Token: None 2022-09-03T06:54:46.814 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:46.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PUeHjUEhUuUH4tOsn2IcEyvf4VY= 2022-09-03T06:54:46.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PUeHjUEhUuUH4tOsn2IcEyvf4VY=', 'Content-Length': '0'} 2022-09-03T06:54:46.817 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c6f2f261d44b372-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.818 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:54:46.819 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.819 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.820 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.820 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.821 DEBUG:boto:Method: GET 2022-09-03T06:54:46.821 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:46.822 DEBUG:boto:Data: 2022-09-03T06:54:46.823 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.823 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.824 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.824 DEBUG:boto:Params: {} 2022-09-03T06:54:46.825 DEBUG:boto:Token: None 2022-09-03T06:54:46.825 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:46.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Tr2OwLVKQgiOBKcmIT3JKmWTKA= 2022-09-03T06:54:46.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Tr2OwLVKQgiOBKcmIT3JKmWTKA=', 'Content-Length': '0'} 2022-09-03T06:54:46.829 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008cbc9dc93345d300-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.874 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.875 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.876 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.876 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:46.877 DEBUG:boto:Method: GET 2022-09-03T06:54:46.877 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:46.878 DEBUG:boto:Data: 2022-09-03T06:54:46.878 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.880 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.880 DEBUG:boto:Params: {} 2022-09-03T06:54:46.881 DEBUG:boto:Token: None 2022-09-03T06:54:46.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:46.882 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Tr2OwLVKQgiOBKcmIT3JKmWTKA= 2022-09-03T06:54:46.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Tr2OwLVKQgiOBKcmIT3JKmWTKA=', 'Content-Length': '0'} 2022-09-03T06:54:46.885 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000347c8115e20c4b7e-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.886 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.887 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.888 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:46.888 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:46.889 DEBUG:boto:Method: GET 2022-09-03T06:54:46.889 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:46.891 DEBUG:boto:Data: 2022-09-03T06:54:46.892 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.893 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.893 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.894 DEBUG:boto:Params: {} 2022-09-03T06:54:46.895 DEBUG:boto:Token: None 2022-09-03T06:54:46.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.898 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.902 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004878f5ed58d0ae70-006312fa36-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.903 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.904 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:46.905 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:46.906 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:46.906 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:46.907 DEBUG:boto:Method: GET 2022-09-03T06:54:46.908 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:46.909 DEBUG:boto:Data: 2022-09-03T06:54:46.909 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.910 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.911 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.911 DEBUG:boto:Params: {} 2022-09-03T06:54:46.912 DEBUG:boto:Token: None 2022-09-03T06:54:46.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/ 2022-09-03T06:54:46.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0= 2022-09-03T06:54:46.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGq++B51b9jr66siNSUeaHG6Bv0=', 'Content-Length': '0'} 2022-09-03T06:54:46.916 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002eb705fd28f8e558-006312fa36-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.917 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:46.917 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.918 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.919 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.919 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:46.920 DEBUG:boto:Data: 2022-09-03T06:54:46.920 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.921 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.921 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.922 DEBUG:boto:Params: {} 2022-09-03T06:54:46.922 DEBUG:boto:Token: None 2022-09-03T06:54:46.923 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d 2022-09-03T06:54:46.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yOYdquJxwiZVAZyjdpTbXIrOtSU= 2022-09-03T06:54:46.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yOYdquJxwiZVAZyjdpTbXIrOtSU=', 'Content-Length': '0'} 2022-09-03T06:54:46.926 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023b1d5ded1d955b9-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.927 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.927 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.928 DEBUG:boto:Method: HEAD 2022-09-03T06:54:46.928 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:46.929 DEBUG:boto:Data: 2022-09-03T06:54:46.930 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.931 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.932 DEBUG:boto:Params: {} 2022-09-03T06:54:46.933 DEBUG:boto:Token: None 2022-09-03T06:54:46.934 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d 2022-09-03T06:54:46.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yOYdquJxwiZVAZyjdpTbXIrOtSU= 2022-09-03T06:54:46.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yOYdquJxwiZVAZyjdpTbXIrOtSU=', 'Content-Length': '0'} 2022-09-03T06:54:46.937 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000259c04d035907dff-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.938 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:54:46.939 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.939 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.940 DEBUG:boto:Method: GET 2022-09-03T06:54:46.940 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:46.941 DEBUG:boto:Data: 2022-09-03T06:54:46.941 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.942 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:46.943 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.943 DEBUG:boto:Params: {} 2022-09-03T06:54:46.944 DEBUG:boto:Token: None 2022-09-03T06:54:46.944 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d 2022-09-03T06:54:46.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4MjC/dHR5dhdQqRy8g/97FrIvik= 2022-09-03T06:54:46.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4MjC/dHR5dhdQqRy8g/97FrIvik=', 'Content-Length': '0'} 2022-09-03T06:54:46.948 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c7010b5b0f265d7-006312fa36-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:46.990 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:46.991 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:46.992 DEBUG:boto:Method: GET 2022-09-03T06:54:46.992 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:46.993 DEBUG:boto:Data: 2022-09-03T06:54:46.993 DEBUG:boto:Headers: {} 2022-09-03T06:54:46.994 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:46.994 DEBUG:boto:Port: 8000 2022-09-03T06:54:46.995 DEBUG:boto:Params: {} 2022-09-03T06:54:46.995 DEBUG:boto:Token: None 2022-09-03T06:54:46.996 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:46 GMT /avysmw-71/d 2022-09-03T06:54:46.997 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4MjC/dHR5dhdQqRy8g/97FrIvik= 2022-09-03T06:54:46.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4MjC/dHR5dhdQqRy8g/97FrIvik=', 'Content-Length': '0'} 2022-09-03T06:54:47.000 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001fc26edf2cf0cd55-006312fa36-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.001 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.001 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.002 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:47.003 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:47.003 DEBUG:boto:Method: GET 2022-09-03T06:54:47.004 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:47.004 DEBUG:boto:Data: 2022-09-03T06:54:47.005 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.005 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.006 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.006 DEBUG:boto:Params: {} 2022-09-03T06:54:47.007 DEBUG:boto:Token: None 2022-09-03T06:54:47.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.012 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d688cf44859f63d8-006312fa37-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.013 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.015 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.017 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.018 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:47.019 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:47.020 DEBUG:boto:Method: GET 2022-09-03T06:54:47.021 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:47.021 DEBUG:boto:Data: 2022-09-03T06:54:47.022 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.023 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.023 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.024 DEBUG:boto:Params: {} 2022-09-03T06:54:47.025 DEBUG:boto:Token: None 2022-09-03T06:54:47.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.029 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000097d220afe979ea25-006312fa37-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.030 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.031 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:54:47.032 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.032 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.033 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.033 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.034 DEBUG:boto:Method: GET 2022-09-03T06:54:47.035 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:47.036 DEBUG:boto:Data: 2022-09-03T06:54:47.037 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.038 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.039 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.040 DEBUG:boto:Params: {} 2022-09-03T06:54:47.041 DEBUG:boto:Token: None 2022-09-03T06:54:47.042 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:47.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aQDd8YHFYYJxtE0aggqbI7qiN5k= 2022-09-03T06:54:47.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aQDd8YHFYYJxtE0aggqbI7qiN5k=', 'Content-Length': '0'} 2022-09-03T06:54:47.047 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000135c681ae1fc3e50-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.048 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.049 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.051 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.052 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.053 DEBUG:boto:Method: GET 2022-09-03T06:54:47.054 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:47.055 DEBUG:boto:Data: 2022-09-03T06:54:47.056 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.058 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.059 DEBUG:boto:Params: {} 2022-09-03T06:54:47.060 DEBUG:boto:Token: None 2022-09-03T06:54:47.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:47.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aQDd8YHFYYJxtE0aggqbI7qiN5k= 2022-09-03T06:54:47.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aQDd8YHFYYJxtE0aggqbI7qiN5k=', 'Content-Length': '0'} 2022-09-03T06:54:47.066 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc00ff64fafde3f0-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.073 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.074 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.075 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.076 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.077 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.078 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:47.079 DEBUG:boto:Data: 2022-09-03T06:54:47.080 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.081 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.082 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.083 DEBUG:boto:Params: {} 2022-09-03T06:54:47.084 DEBUG:boto:Token: None 2022-09-03T06:54:47.085 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:47.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VPMt4PwGiMqAR//kDTn/VXPaCko= 2022-09-03T06:54:47.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VPMt4PwGiMqAR//kDTn/VXPaCko=', 'Content-Length': '0'} 2022-09-03T06:54:47.090 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a86107f9d323b485-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.091 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.092 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.093 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.094 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.095 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.096 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:47.097 DEBUG:boto:Data: 2022-09-03T06:54:47.098 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.098 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.099 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.100 DEBUG:boto:Params: {} 2022-09-03T06:54:47.101 DEBUG:boto:Token: None 2022-09-03T06:54:47.102 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:47.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VPMt4PwGiMqAR//kDTn/VXPaCko= 2022-09-03T06:54:47.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VPMt4PwGiMqAR//kDTn/VXPaCko=', 'Content-Length': '0'} 2022-09-03T06:54:47.107 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000790d9d64f57c344-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.108 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:54:47.109 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.110 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.111 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.113 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.114 DEBUG:boto:Method: GET 2022-09-03T06:54:47.115 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:47.116 DEBUG:boto:Data: 2022-09-03T06:54:47.116 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.117 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.119 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.119 DEBUG:boto:Params: {} 2022-09-03T06:54:47.120 DEBUG:boto:Token: None 2022-09-03T06:54:47.121 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:47.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aQDd8YHFYYJxtE0aggqbI7qiN5k= 2022-09-03T06:54:47.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aQDd8YHFYYJxtE0aggqbI7qiN5k=', 'Content-Length': '0'} 2022-09-03T06:54:47.125 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e875c619f66e5218-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.170 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.171 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.172 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.173 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:47.174 DEBUG:boto:Method: GET 2022-09-03T06:54:47.175 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:47.176 DEBUG:boto:Data: 2022-09-03T06:54:47.177 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.177 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.178 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.178 DEBUG:boto:Params: {} 2022-09-03T06:54:47.179 DEBUG:boto:Token: None 2022-09-03T06:54:47.180 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:47.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aQDd8YHFYYJxtE0aggqbI7qiN5k= 2022-09-03T06:54:47.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aQDd8YHFYYJxtE0aggqbI7qiN5k=', 'Content-Length': '0'} 2022-09-03T06:54:47.183 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003937a97785cfe107-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.184 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.184 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.185 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:47.185 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:47.186 DEBUG:boto:Method: GET 2022-09-03T06:54:47.187 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:47.187 DEBUG:boto:Data: 2022-09-03T06:54:47.188 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.188 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.189 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.189 DEBUG:boto:Params: {} 2022-09-03T06:54:47.190 DEBUG:boto:Token: None 2022-09-03T06:54:47.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.196 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f8a12b49d7c1328e-006312fa37-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.196 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.197 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.198 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.198 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:47.199 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:47.199 DEBUG:boto:Method: GET 2022-09-03T06:54:47.200 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:47.200 DEBUG:boto:Data: 2022-09-03T06:54:47.201 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.202 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.202 DEBUG:boto:Params: {} 2022-09-03T06:54:47.203 DEBUG:boto:Token: None 2022-09-03T06:54:47.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.208 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c64fedf6b9e348a8-006312fa37-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.209 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.210 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.210 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.211 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.211 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:47.212 DEBUG:boto:Data: 2022-09-03T06:54:47.212 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.213 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.214 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.215 DEBUG:boto:Params: {} 2022-09-03T06:54:47.216 DEBUG:boto:Token: None 2022-09-03T06:54:47.216 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e 2022-09-03T06:54:47.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t39t2wQQ8IBE9ncbjX1xRA3mYek= 2022-09-03T06:54:47.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t39t2wQQ8IBE9ncbjX1xRA3mYek=', 'Content-Length': '0'} 2022-09-03T06:54:47.226 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ed1d5f02922fed1-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.227 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.228 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.229 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.229 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:47.230 DEBUG:boto:Data: 2022-09-03T06:54:47.230 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.231 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.232 DEBUG:boto:Params: {} 2022-09-03T06:54:47.233 DEBUG:boto:Token: None 2022-09-03T06:54:47.234 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e 2022-09-03T06:54:47.235 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t39t2wQQ8IBE9ncbjX1xRA3mYek= 2022-09-03T06:54:47.236 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t39t2wQQ8IBE9ncbjX1xRA3mYek=', 'Content-Length': '0'} 2022-09-03T06:54:47.239 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7e9f29b20c92169-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.240 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:54:47.241 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.242 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.243 DEBUG:boto:Method: GET 2022-09-03T06:54:47.244 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:47.244 DEBUG:boto:Data: 2022-09-03T06:54:47.245 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.245 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.246 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.246 DEBUG:boto:Params: {} 2022-09-03T06:54:47.247 DEBUG:boto:Token: None 2022-09-03T06:54:47.248 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e 2022-09-03T06:54:47.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c2DPGyc3/3eEgYMm58MDflZkoA8= 2022-09-03T06:54:47.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c2DPGyc3/3eEgYMm58MDflZkoA8=', 'Content-Length': '0'} 2022-09-03T06:54:47.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000899deaf596e3e42a-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.256 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:47.257 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:47.260 DEBUG:boto:Method: GET 2022-09-03T06:54:47.261 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:47.263 DEBUG:boto:Data: 2022-09-03T06:54:47.264 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.265 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.265 DEBUG:boto:Params: {} 2022-09-03T06:54:47.266 DEBUG:boto:Token: None 2022-09-03T06:54:47.266 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/e 2022-09-03T06:54:47.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c2DPGyc3/3eEgYMm58MDflZkoA8= 2022-09-03T06:54:47.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c2DPGyc3/3eEgYMm58MDflZkoA8=', 'Content-Length': '0'} 2022-09-03T06:54:47.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6de668d8ac6c5c1-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.314 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.316 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.317 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:47.318 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:47.319 DEBUG:boto:Method: GET 2022-09-03T06:54:47.320 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:47.321 DEBUG:boto:Data: 2022-09-03T06:54:47.322 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.323 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.323 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.324 DEBUG:boto:Params: {} 2022-09-03T06:54:47.325 DEBUG:boto:Token: None 2022-09-03T06:54:47.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.329 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088a72b430b782214-006312fa37-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.330 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.331 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.332 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.332 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:47.333 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:47.333 DEBUG:boto:Method: GET 2022-09-03T06:54:47.334 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:47.334 DEBUG:boto:Data: 2022-09-03T06:54:47.335 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.336 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.337 DEBUG:boto:Params: {} 2022-09-03T06:54:47.337 DEBUG:boto:Token: None 2022-09-03T06:54:47.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.342 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ea7b1302c267135-006312fa37-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.343 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.343 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:54:47.344 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.345 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.345 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.346 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.347 DEBUG:boto:Method: GET 2022-09-03T06:54:47.347 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:47.348 DEBUG:boto:Data: 2022-09-03T06:54:47.349 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.350 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.350 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.351 DEBUG:boto:Params: {} 2022-09-03T06:54:47.352 DEBUG:boto:Token: None 2022-09-03T06:54:47.353 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:47.353 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RYHxvrD7mrxuv6rUQpAEaE98Tkg= 2022-09-03T06:54:47.354 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RYHxvrD7mrxuv6rUQpAEaE98Tkg=', 'Content-Length': '0'} 2022-09-03T06:54:47.356 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f3fe0e3d88c61ab-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.357 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.358 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.358 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.359 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.360 DEBUG:boto:Method: GET 2022-09-03T06:54:47.360 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:47.361 DEBUG:boto:Data: 2022-09-03T06:54:47.362 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.363 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.364 DEBUG:boto:Params: {} 2022-09-03T06:54:47.364 DEBUG:boto:Token: None 2022-09-03T06:54:47.365 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:47.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RYHxvrD7mrxuv6rUQpAEaE98Tkg= 2022-09-03T06:54:47.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RYHxvrD7mrxuv6rUQpAEaE98Tkg=', 'Content-Length': '0'} 2022-09-03T06:54:47.370 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d43c39ac2b0527a-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.371 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.371 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.372 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.373 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.373 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.374 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:47.374 DEBUG:boto:Data: 2022-09-03T06:54:47.375 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.376 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.376 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.377 DEBUG:boto:Params: {} 2022-09-03T06:54:47.377 DEBUG:boto:Token: None 2022-09-03T06:54:47.378 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:47.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0moVtpc+8OogWOgyjlPtrPYhvoA= 2022-09-03T06:54:47.379 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0moVtpc+8OogWOgyjlPtrPYhvoA=', 'Content-Length': '0'} 2022-09-03T06:54:47.381 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000329f67d33435b336-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.382 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.383 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.383 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.384 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.384 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.385 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:47.385 DEBUG:boto:Data: 2022-09-03T06:54:47.386 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.387 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.387 DEBUG:boto:Params: {} 2022-09-03T06:54:47.388 DEBUG:boto:Token: None 2022-09-03T06:54:47.389 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:47.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0moVtpc+8OogWOgyjlPtrPYhvoA= 2022-09-03T06:54:47.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0moVtpc+8OogWOgyjlPtrPYhvoA=', 'Content-Length': '0'} 2022-09-03T06:54:47.392 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6504560c6b7793c-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.393 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:54:47.394 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.395 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.397 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.397 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.398 DEBUG:boto:Method: GET 2022-09-03T06:54:47.399 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:47.400 DEBUG:boto:Data: 2022-09-03T06:54:47.401 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.402 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.403 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.404 DEBUG:boto:Params: {} 2022-09-03T06:54:47.405 DEBUG:boto:Token: None 2022-09-03T06:54:47.406 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:47.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RYHxvrD7mrxuv6rUQpAEaE98Tkg= 2022-09-03T06:54:47.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RYHxvrD7mrxuv6rUQpAEaE98Tkg=', 'Content-Length': '0'} 2022-09-03T06:54:47.409 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068234b3e29b62cfb-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.411 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.412 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.412 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.413 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:47.418 DEBUG:boto:Method: GET 2022-09-03T06:54:47.419 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:47.420 DEBUG:boto:Data: 2022-09-03T06:54:47.421 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.422 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.423 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.424 DEBUG:boto:Params: {} 2022-09-03T06:54:47.425 DEBUG:boto:Token: None 2022-09-03T06:54:47.426 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:47.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RYHxvrD7mrxuv6rUQpAEaE98Tkg= 2022-09-03T06:54:47.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RYHxvrD7mrxuv6rUQpAEaE98Tkg=', 'Content-Length': '0'} 2022-09-03T06:54:47.429 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a54e5b22614b7493-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.430 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.431 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.431 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:47.432 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:47.432 DEBUG:boto:Method: GET 2022-09-03T06:54:47.433 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:47.433 DEBUG:boto:Data: 2022-09-03T06:54:47.434 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.435 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.435 DEBUG:boto:Params: {} 2022-09-03T06:54:47.436 DEBUG:boto:Token: None 2022-09-03T06:54:47.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.441 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009817ba4c1b332584-006312fa37-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.442 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.443 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.444 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.444 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:47.445 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:47.445 DEBUG:boto:Method: GET 2022-09-03T06:54:47.446 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:47.446 DEBUG:boto:Data: 2022-09-03T06:54:47.447 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.447 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.448 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.448 DEBUG:boto:Params: {} 2022-09-03T06:54:47.449 DEBUG:boto:Token: None 2022-09-03T06:54:47.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.450 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.453 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c80c373913cd7f29-006312fa37-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.454 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.455 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.455 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.456 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.457 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:47.457 DEBUG:boto:Data: 2022-09-03T06:54:47.458 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.458 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.459 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.459 DEBUG:boto:Params: {} 2022-09-03T06:54:47.460 DEBUG:boto:Token: None 2022-09-03T06:54:47.461 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f 2022-09-03T06:54:47.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uBz56i/cjfYiqFtct1xR6ZuBz1c= 2022-09-03T06:54:47.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uBz56i/cjfYiqFtct1xR6ZuBz1c=', 'Content-Length': '0'} 2022-09-03T06:54:47.464 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000572a3317ca596cf1-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.465 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.465 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.466 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.466 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:47.467 DEBUG:boto:Data: 2022-09-03T06:54:47.467 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.468 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.468 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.469 DEBUG:boto:Params: {} 2022-09-03T06:54:47.469 DEBUG:boto:Token: None 2022-09-03T06:54:47.470 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f 2022-09-03T06:54:47.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uBz56i/cjfYiqFtct1xR6ZuBz1c= 2022-09-03T06:54:47.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uBz56i/cjfYiqFtct1xR6ZuBz1c=', 'Content-Length': '0'} 2022-09-03T06:54:47.473 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000058c0b6a4452d1fa-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.474 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:54:47.475 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.475 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.476 DEBUG:boto:Method: GET 2022-09-03T06:54:47.476 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:47.477 DEBUG:boto:Data: 2022-09-03T06:54:47.477 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.478 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.478 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.479 DEBUG:boto:Params: {} 2022-09-03T06:54:47.480 DEBUG:boto:Token: None 2022-09-03T06:54:47.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f 2022-09-03T06:54:47.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U2Vl6Lc0BEIozZ42fL7Gg4ALRNY= 2022-09-03T06:54:47.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U2Vl6Lc0BEIozZ42fL7Gg4ALRNY=', 'Content-Length': '0'} 2022-09-03T06:54:47.489 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f10f92a1268c8ef0-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.491 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:47.491 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:47.492 DEBUG:boto:Method: GET 2022-09-03T06:54:47.493 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:47.494 DEBUG:boto:Data: 2022-09-03T06:54:47.494 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.495 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.495 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.496 DEBUG:boto:Params: {} 2022-09-03T06:54:47.497 DEBUG:boto:Token: None 2022-09-03T06:54:47.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/f 2022-09-03T06:54:47.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U2Vl6Lc0BEIozZ42fL7Gg4ALRNY= 2022-09-03T06:54:47.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U2Vl6Lc0BEIozZ42fL7Gg4ALRNY=', 'Content-Length': '0'} 2022-09-03T06:54:47.501 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f59b381cec98dd72-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.503 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.503 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.504 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:47.504 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:47.505 DEBUG:boto:Method: GET 2022-09-03T06:54:47.505 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:47.506 DEBUG:boto:Data: 2022-09-03T06:54:47.507 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.507 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.508 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.509 DEBUG:boto:Params: {} 2022-09-03T06:54:47.510 DEBUG:boto:Token: None 2022-09-03T06:54:47.510 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.514 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006cf808946c183071-006312fa37-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.554 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.556 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.556 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.557 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:47.558 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:47.559 DEBUG:boto:Method: GET 2022-09-03T06:54:47.560 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:47.561 DEBUG:boto:Data: 2022-09-03T06:54:47.561 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.562 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.563 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.563 DEBUG:boto:Params: {} 2022-09-03T06:54:47.564 DEBUG:boto:Token: None 2022-09-03T06:54:47.565 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.566 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.569 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000783f1a5b0cbd3b2-006312fa37-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.571 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.572 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:54:47.573 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.574 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.575 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.576 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.576 DEBUG:boto:Method: GET 2022-09-03T06:54:47.577 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:47.578 DEBUG:boto:Data: 2022-09-03T06:54:47.579 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.580 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.581 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.582 DEBUG:boto:Params: {} 2022-09-03T06:54:47.583 DEBUG:boto:Token: None 2022-09-03T06:54:47.584 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:47.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HNWOa5C76uSdpt8irTO8fl69yLo= 2022-09-03T06:54:47.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HNWOa5C76uSdpt8irTO8fl69yLo=', 'Content-Length': '0'} 2022-09-03T06:54:47.588 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009841ca412d866327-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.588 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.589 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.590 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.590 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.591 DEBUG:boto:Method: GET 2022-09-03T06:54:47.591 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:47.592 DEBUG:boto:Data: 2022-09-03T06:54:47.592 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.593 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.594 DEBUG:boto:Params: {} 2022-09-03T06:54:47.594 DEBUG:boto:Token: None 2022-09-03T06:54:47.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:47.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HNWOa5C76uSdpt8irTO8fl69yLo= 2022-09-03T06:54:47.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HNWOa5C76uSdpt8irTO8fl69yLo=', 'Content-Length': '0'} 2022-09-03T06:54:47.603 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000040e703eef9cb779b-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.605 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.606 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.606 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.607 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.607 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.608 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:47.608 DEBUG:boto:Data: 2022-09-03T06:54:47.609 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.609 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.610 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.610 DEBUG:boto:Params: {} 2022-09-03T06:54:47.611 DEBUG:boto:Token: None 2022-09-03T06:54:47.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:47.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CfpkqMO1KnMz1Gq3/DPLstSUtB0= 2022-09-03T06:54:47.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CfpkqMO1KnMz1Gq3/DPLstSUtB0=', 'Content-Length': '0'} 2022-09-03T06:54:47.615 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6171c96b652f4bd-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.617 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.618 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.619 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.619 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.620 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.621 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:47.621 DEBUG:boto:Data: 2022-09-03T06:54:47.622 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.623 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.624 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.625 DEBUG:boto:Params: {} 2022-09-03T06:54:47.626 DEBUG:boto:Token: None 2022-09-03T06:54:47.627 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:47.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CfpkqMO1KnMz1Gq3/DPLstSUtB0= 2022-09-03T06:54:47.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CfpkqMO1KnMz1Gq3/DPLstSUtB0=', 'Content-Length': '0'} 2022-09-03T06:54:47.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026a3e13d07a1fd90-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.634 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:54:47.635 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.636 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.637 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.639 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.640 DEBUG:boto:Method: GET 2022-09-03T06:54:47.641 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:47.641 DEBUG:boto:Data: 2022-09-03T06:54:47.642 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.642 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.643 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.644 DEBUG:boto:Params: {} 2022-09-03T06:54:47.645 DEBUG:boto:Token: None 2022-09-03T06:54:47.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:47.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HNWOa5C76uSdpt8irTO8fl69yLo= 2022-09-03T06:54:47.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HNWOa5C76uSdpt8irTO8fl69yLo=', 'Content-Length': '0'} 2022-09-03T06:54:47.649 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b24e2122058cbde9-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.650 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.651 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.651 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.652 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:47.652 DEBUG:boto:Method: GET 2022-09-03T06:54:47.653 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:47.654 DEBUG:boto:Data: 2022-09-03T06:54:47.655 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.658 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.659 DEBUG:boto:Params: {} 2022-09-03T06:54:47.660 DEBUG:boto:Token: None 2022-09-03T06:54:47.661 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:47.662 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HNWOa5C76uSdpt8irTO8fl69yLo= 2022-09-03T06:54:47.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HNWOa5C76uSdpt8irTO8fl69yLo=', 'Content-Length': '0'} 2022-09-03T06:54:47.667 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065e743424c45e7d8-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.710 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.712 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.713 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:47.713 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:47.714 DEBUG:boto:Method: GET 2022-09-03T06:54:47.715 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:47.716 DEBUG:boto:Data: 2022-09-03T06:54:47.717 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.718 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.720 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.721 DEBUG:boto:Params: {} 2022-09-03T06:54:47.722 DEBUG:boto:Token: None 2022-09-03T06:54:47.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.729 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fa3df09888096866-006312fa37-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.730 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.731 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.732 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.733 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:47.734 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:47.734 DEBUG:boto:Method: GET 2022-09-03T06:54:47.735 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:47.736 DEBUG:boto:Data: 2022-09-03T06:54:47.736 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.737 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.737 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.738 DEBUG:boto:Params: {} 2022-09-03T06:54:47.738 DEBUG:boto:Token: None 2022-09-03T06:54:47.739 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.744 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006ac3cd428ea4127a-006312fa37-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.745 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.745 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.746 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.747 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.748 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:47.749 DEBUG:boto:Data: 2022-09-03T06:54:47.750 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.751 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.752 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.753 DEBUG:boto:Params: {} 2022-09-03T06:54:47.754 DEBUG:boto:Token: None 2022-09-03T06:54:47.755 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g 2022-09-03T06:54:47.755 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wmtf8CSmzYAsKtvvD8WYE2MTk+s= 2022-09-03T06:54:47.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wmtf8CSmzYAsKtvvD8WYE2MTk+s=', 'Content-Length': '0'} 2022-09-03T06:54:47.758 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009321245b7d31b768-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.760 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.761 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.762 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.763 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:47.763 DEBUG:boto:Data: 2022-09-03T06:54:47.764 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.764 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.765 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.765 DEBUG:boto:Params: {} 2022-09-03T06:54:47.766 DEBUG:boto:Token: None 2022-09-03T06:54:47.767 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g 2022-09-03T06:54:47.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wmtf8CSmzYAsKtvvD8WYE2MTk+s= 2022-09-03T06:54:47.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wmtf8CSmzYAsKtvvD8WYE2MTk+s=', 'Content-Length': '0'} 2022-09-03T06:54:47.771 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029b8aba98f3a0996-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.772 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:54:47.772 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.773 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.774 DEBUG:boto:Method: GET 2022-09-03T06:54:47.776 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:47.777 DEBUG:boto:Data: 2022-09-03T06:54:47.778 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.779 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.780 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.780 DEBUG:boto:Params: {} 2022-09-03T06:54:47.781 DEBUG:boto:Token: None 2022-09-03T06:54:47.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g 2022-09-03T06:54:47.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TmB0qKCUMyXbcg73V0D9vev644k= 2022-09-03T06:54:47.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TmB0qKCUMyXbcg73V0D9vev644k=', 'Content-Length': '0'} 2022-09-03T06:54:47.791 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000084ffe6b2e5085b0f-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.834 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:47.836 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:47.837 DEBUG:boto:Method: GET 2022-09-03T06:54:47.838 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:47.839 DEBUG:boto:Data: 2022-09-03T06:54:47.839 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.840 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.841 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.846 DEBUG:boto:Params: {} 2022-09-03T06:54:47.847 DEBUG:boto:Token: None 2022-09-03T06:54:47.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/g 2022-09-03T06:54:47.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TmB0qKCUMyXbcg73V0D9vev644k= 2022-09-03T06:54:47.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TmB0qKCUMyXbcg73V0D9vev644k=', 'Content-Length': '0'} 2022-09-03T06:54:47.856 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ed8dc38e3623b5c-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.858 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.859 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.860 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:47.860 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:47.861 DEBUG:boto:Method: GET 2022-09-03T06:54:47.861 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:47.862 DEBUG:boto:Data: 2022-09-03T06:54:47.862 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.863 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.863 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.864 DEBUG:boto:Params: {} 2022-09-03T06:54:47.865 DEBUG:boto:Token: None 2022-09-03T06:54:47.865 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.869 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023936cdcb952f506-006312fa37-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.870 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.870 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.871 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.872 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:47.872 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:47.873 DEBUG:boto:Method: GET 2022-09-03T06:54:47.873 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:47.874 DEBUG:boto:Data: 2022-09-03T06:54:47.882 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.884 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.885 DEBUG:boto:Params: {} 2022-09-03T06:54:47.886 DEBUG:boto:Token: None 2022-09-03T06:54:47.887 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/ 2022-09-03T06:54:47.888 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg= 2022-09-03T06:54:47.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3vhKWe+4RGE1h36I2uQYKNvazTg=', 'Content-Length': '0'} 2022-09-03T06:54:47.892 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c21e46171aa15f40-006312fa37-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.894 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:47.894 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:54:47.895 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:47.895 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:47.896 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.896 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.897 DEBUG:boto:Method: GET 2022-09-03T06:54:47.898 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:47.898 DEBUG:boto:Data: 2022-09-03T06:54:47.899 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.900 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.900 DEBUG:boto:Params: {} 2022-09-03T06:54:47.901 DEBUG:boto:Token: None 2022-09-03T06:54:47.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:47.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LOC/9Iind7KR0RetB67HshVcNas= 2022-09-03T06:54:47.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LOC/9Iind7KR0RetB67HshVcNas=', 'Content-Length': '0'} 2022-09-03T06:54:47.905 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057e8a62276450e3d-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.906 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:47.906 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:47.907 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.907 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.908 DEBUG:boto:Method: GET 2022-09-03T06:54:47.908 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:47.909 DEBUG:boto:Data: 2022-09-03T06:54:47.909 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.910 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.911 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.912 DEBUG:boto:Params: {} 2022-09-03T06:54:47.913 DEBUG:boto:Token: None 2022-09-03T06:54:47.913 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:47.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LOC/9Iind7KR0RetB67HshVcNas= 2022-09-03T06:54:47.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LOC/9Iind7KR0RetB67HshVcNas=', 'Content-Length': '0'} 2022-09-03T06:54:47.917 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2441ebe036f74ad-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.919 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:47.920 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:47.921 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.922 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.923 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.924 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:47.925 DEBUG:boto:Data: 2022-09-03T06:54:47.926 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.927 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.928 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.929 DEBUG:boto:Params: {} 2022-09-03T06:54:47.930 DEBUG:boto:Token: None 2022-09-03T06:54:47.931 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:47.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KWGtvDyGejZxtXxNmiyAe9zDcPA= 2022-09-03T06:54:47.933 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KWGtvDyGejZxtXxNmiyAe9zDcPA=', 'Content-Length': '0'} 2022-09-03T06:54:47.935 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089d24866f1d7fbaf-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.937 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:47.938 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:47.941 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.941 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.942 DEBUG:boto:Method: HEAD 2022-09-03T06:54:47.944 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:47.945 DEBUG:boto:Data: 2022-09-03T06:54:47.946 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.948 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.949 DEBUG:boto:Params: {} 2022-09-03T06:54:47.951 DEBUG:boto:Token: None 2022-09-03T06:54:47.952 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:47.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KWGtvDyGejZxtXxNmiyAe9zDcPA= 2022-09-03T06:54:47.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KWGtvDyGejZxtXxNmiyAe9zDcPA=', 'Content-Length': '0'} 2022-09-03T06:54:47.955 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7bd2a62e962b9de-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.956 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:54:47.956 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:47.957 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:47.958 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.958 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.959 DEBUG:boto:Method: GET 2022-09-03T06:54:47.959 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:47.960 DEBUG:boto:Data: 2022-09-03T06:54:47.962 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.962 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.963 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.964 DEBUG:boto:Params: {} 2022-09-03T06:54:47.964 DEBUG:boto:Token: None 2022-09-03T06:54:47.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:47.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LOC/9Iind7KR0RetB67HshVcNas= 2022-09-03T06:54:47.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LOC/9Iind7KR0RetB67HshVcNas=', 'Content-Length': '0'} 2022-09-03T06:54:47.973 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042e579b3c39d5399-006312fa37-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.974 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:47.975 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:47.975 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.976 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:47.976 DEBUG:boto:Method: GET 2022-09-03T06:54:47.977 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:47.981 DEBUG:boto:Data: 2022-09-03T06:54:47.983 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:47.985 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.986 DEBUG:boto:Params: {} 2022-09-03T06:54:47.987 DEBUG:boto:Token: None 2022-09-03T06:54:47.989 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:47 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:47.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LOC/9Iind7KR0RetB67HshVcNas= 2022-09-03T06:54:47.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LOC/9Iind7KR0RetB67HshVcNas=', 'Content-Length': '0'} 2022-09-03T06:54:47.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b295c28a41e0cc6-006312fa37-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:47.994 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:47.994 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:47.995 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:47.995 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:47.996 DEBUG:boto:Method: GET 2022-09-03T06:54:47.996 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:47.997 DEBUG:boto:Data: 2022-09-03T06:54:47.997 DEBUG:boto:Headers: {} 2022-09-03T06:54:47.998 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:47.999 DEBUG:boto:Port: 8000 2022-09-03T06:54:47.999 DEBUG:boto:Params: {} 2022-09-03T06:54:48.000 DEBUG:boto:Token: None 2022-09-03T06:54:48.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.004 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8776addae37ecb3-006312fa38-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.050 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.052 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.053 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.054 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:48.055 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:48.056 DEBUG:boto:Method: GET 2022-09-03T06:54:48.056 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:48.057 DEBUG:boto:Data: 2022-09-03T06:54:48.057 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.058 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.058 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.059 DEBUG:boto:Params: {} 2022-09-03T06:54:48.059 DEBUG:boto:Token: None 2022-09-03T06:54:48.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.067 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3712dfba941b209-006312fa38-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.068 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.069 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:48.069 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:48.070 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.074 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:48.075 DEBUG:boto:Data: 2022-09-03T06:54:48.076 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.077 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:48.078 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.078 DEBUG:boto:Params: {} 2022-09-03T06:54:48.080 DEBUG:boto:Token: None 2022-09-03T06:54:48.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/h 2022-09-03T06:54:48.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7mLwvUIHINchpd6NCWDDV0QolOg= 2022-09-03T06:54:48.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7mLwvUIHINchpd6NCWDDV0QolOg=', 'Content-Length': '0'} 2022-09-03T06:54:48.085 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000757bbe41c82285d4-006312fa38-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.086 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:48.088 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:48.089 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.090 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:48.090 DEBUG:boto:Data: 2022-09-03T06:54:48.091 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.093 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.093 DEBUG:boto:Params: {} 2022-09-03T06:54:48.094 DEBUG:boto:Token: None 2022-09-03T06:54:48.095 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/h 2022-09-03T06:54:48.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7mLwvUIHINchpd6NCWDDV0QolOg= 2022-09-03T06:54:48.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7mLwvUIHINchpd6NCWDDV0QolOg=', 'Content-Length': '0'} 2022-09-03T06:54:48.102 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9b2a6ee99141eb3-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.104 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:54:48.105 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:48.107 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:48.107 DEBUG:boto:Method: GET 2022-09-03T06:54:48.108 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:48.109 DEBUG:boto:Data: 2022-09-03T06:54:48.110 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.111 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:48.112 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.112 DEBUG:boto:Params: {} 2022-09-03T06:54:48.113 DEBUG:boto:Token: None 2022-09-03T06:54:48.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/h 2022-09-03T06:54:48.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z/8bC9Hi+Zo2Cc38CFzGrPtcLRE= 2022-09-03T06:54:48.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z/8bC9Hi+Zo2Cc38CFzGrPtcLRE=', 'Content-Length': '0'} 2022-09-03T06:54:48.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d0ec2b0e3e50447b-006312fa38-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.119 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:48.120 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:48.121 DEBUG:boto:Method: GET 2022-09-03T06:54:48.121 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:48.122 DEBUG:boto:Data: 2022-09-03T06:54:48.123 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.124 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.124 DEBUG:boto:Params: {} 2022-09-03T06:54:48.125 DEBUG:boto:Token: None 2022-09-03T06:54:48.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/h 2022-09-03T06:54:48.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z/8bC9Hi+Zo2Cc38CFzGrPtcLRE= 2022-09-03T06:54:48.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z/8bC9Hi+Zo2Cc38CFzGrPtcLRE=', 'Content-Length': '0'} 2022-09-03T06:54:48.131 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d80154fc1ee2f2e5-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.178 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.179 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.180 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:48.181 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:48.182 DEBUG:boto:Method: GET 2022-09-03T06:54:48.183 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:48.184 DEBUG:boto:Data: 2022-09-03T06:54:48.184 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.185 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:48.186 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.187 DEBUG:boto:Params: {} 2022-09-03T06:54:48.187 DEBUG:boto:Token: None 2022-09-03T06:54:48.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.192 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f3017357fc86dc9-006312fa38-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.193 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.194 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.195 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.195 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:48.196 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:48.196 DEBUG:boto:Method: GET 2022-09-03T06:54:48.197 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:48.197 DEBUG:boto:Data: 2022-09-03T06:54:48.198 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.199 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.200 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.200 DEBUG:boto:Params: {} 2022-09-03T06:54:48.201 DEBUG:boto:Token: None 2022-09-03T06:54:48.201 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.202 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.203 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.205 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007f49f492238974a8-006312fa38-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.206 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.207 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-71 zones={test-zone2, test-zone1} 2022-09-03T06:54:48.207 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-71 zones={test-zone3, test-zone1} 2022-09-03T06:54:48.208 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.209 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.209 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.210 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:54:48.210 DEBUG:boto:Data: 2022-09-03T06:54:48.211 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.211 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.212 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.212 DEBUG:boto:Params: {} 2022-09-03T06:54:48.213 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:54:48.213 DEBUG:boto:Token: None 2022-09-03T06:54:48.214 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.215 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HB++32HNAU7Xt0HNRwvNWleCqsU= 2022-09-03T06:54:48.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HB++32HNAU7Xt0HNRwvNWleCqsU=', 'Content-Length': '0'} 2022-09-03T06:54:48.220 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d358fcd5211d7dd8-006312fa38-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.221 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.221 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.222 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.223 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:54:48.224 DEBUG:boto:Data: 2022-09-03T06:54:48.224 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.225 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.226 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.226 DEBUG:boto:Params: {} 2022-09-03T06:54:48.227 DEBUG:boto:Token: None 2022-09-03T06:54:48.227 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HB++32HNAU7Xt0HNRwvNWleCqsU= 2022-09-03T06:54:48.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HB++32HNAU7Xt0HNRwvNWleCqsU=', 'Content-Length': '0'} 2022-09-03T06:54:48.232 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d21ab3b3e5b07608-006312fa38-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.233 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:54:48.234 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.235 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.235 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.236 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.237 DEBUG:boto:Method: GET 2022-09-03T06:54:48.238 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.238 DEBUG:boto:Data: 2022-09-03T06:54:48.239 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.240 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.240 DEBUG:boto:Params: {} 2022-09-03T06:54:48.241 DEBUG:boto:Token: None 2022-09-03T06:54:48.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/?versions 2022-09-03T06:54:48.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sLXS9anJ+cpDZr/eUV4lkje7zcA= 2022-09-03T06:54:48.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sLXS9anJ+cpDZr/eUV4lkje7zcA=', 'Content-Length': '0'} 2022-09-03T06:54:48.246 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b39874d7eddf13d8-006312fa38-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.247 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.248 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:54:48.249 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:54:48.249 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:54:48.249 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:54:48.250 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:54:48.251 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:54:48.251 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:54:48.252 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:54:48.252 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:54:48.253 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.253 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.254 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.254 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.255 DEBUG:boto:Method: GET 2022-09-03T06:54:48.255 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.256 DEBUG:boto:Data: 2022-09-03T06:54:48.256 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.257 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.258 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.258 DEBUG:boto:Params: {} 2022-09-03T06:54:48.259 DEBUG:boto:Token: None 2022-09-03T06:54:48.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/?versions 2022-09-03T06:54:48.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sLXS9anJ+cpDZr/eUV4lkje7zcA= 2022-09-03T06:54:48.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sLXS9anJ+cpDZr/eUV4lkje7zcA=', 'Content-Length': '0'} 2022-09-03T06:54:48.265 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000051f01c4b8de42af7-006312fa38-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.266 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.268 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:54:48.268 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:54:48.269 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:54:48.269 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:54:48.270 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:54:48.270 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:54:48.270 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:54:48.271 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:54:48.272 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.272 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.273 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.273 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.274 DEBUG:boto:Method: GET 2022-09-03T06:54:48.274 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.275 DEBUG:boto:Data: 2022-09-03T06:54:48.276 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.276 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.277 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.277 DEBUG:boto:Params: {} 2022-09-03T06:54:48.278 DEBUG:boto:Token: None 2022-09-03T06:54:48.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/?versions 2022-09-03T06:54:48.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sLXS9anJ+cpDZr/eUV4lkje7zcA= 2022-09-03T06:54:48.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sLXS9anJ+cpDZr/eUV4lkje7zcA=', 'Content-Length': '0'} 2022-09-03T06:54:48.283 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c13ecd0fff19cd55-006312fa38-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.289 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.291 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.292 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.292 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.293 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.294 DEBUG:boto:Method: GET 2022-09-03T06:54:48.294 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:48.295 DEBUG:boto:Data: 2022-09-03T06:54:48.296 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.296 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.297 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.297 DEBUG:boto:Params: {} 2022-09-03T06:54:48.298 DEBUG:boto:Token: None 2022-09-03T06:54:48.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/?versions 2022-09-03T06:54:48.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sLXS9anJ+cpDZr/eUV4lkje7zcA= 2022-09-03T06:54:48.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sLXS9anJ+cpDZr/eUV4lkje7zcA=', 'Content-Length': '0'} 2022-09-03T06:54:48.302 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e8dacde746a86dd-006312fa38-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.303 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.304 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:54:48.305 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.305 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.306 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.307 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.307 DEBUG:boto:Method: GET 2022-09-03T06:54:48.308 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:48.308 DEBUG:boto:Data: 2022-09-03T06:54:48.309 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.310 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.311 DEBUG:boto:Params: {} 2022-09-03T06:54:48.312 DEBUG:boto:Token: None 2022-09-03T06:54:48.313 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:48.313 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7m8YP5fCzJKPSgLDxDpNhePSyhY= 2022-09-03T06:54:48.314 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7m8YP5fCzJKPSgLDxDpNhePSyhY=', 'Content-Length': '0'} 2022-09-03T06:54:48.316 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d26d2a0130100f9e-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.358 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.359 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.359 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.360 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.360 DEBUG:boto:Method: GET 2022-09-03T06:54:48.361 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:48.362 DEBUG:boto:Data: 2022-09-03T06:54:48.362 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.363 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.364 DEBUG:boto:Params: {} 2022-09-03T06:54:48.365 DEBUG:boto:Token: None 2022-09-03T06:54:48.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:48.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7m8YP5fCzJKPSgLDxDpNhePSyhY= 2022-09-03T06:54:48.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7m8YP5fCzJKPSgLDxDpNhePSyhY=', 'Content-Length': '0'} 2022-09-03T06:54:48.369 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f73f4432d83ea65-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.370 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.370 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.371 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.371 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.372 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.372 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:48.373 DEBUG:boto:Data: 2022-09-03T06:54:48.373 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.374 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.374 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.375 DEBUG:boto:Params: {} 2022-09-03T06:54:48.375 DEBUG:boto:Token: None 2022-09-03T06:54:48.376 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:48.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIRxs4Oj0jmnk0CNflzsgKU4n9o= 2022-09-03T06:54:48.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIRxs4Oj0jmnk0CNflzsgKU4n9o=', 'Content-Length': '0'} 2022-09-03T06:54:48.379 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054af90f7d1f02d26-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.380 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.380 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.381 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.381 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.382 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.383 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:48.383 DEBUG:boto:Data: 2022-09-03T06:54:48.384 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.385 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.385 DEBUG:boto:Params: {} 2022-09-03T06:54:48.386 DEBUG:boto:Token: None 2022-09-03T06:54:48.386 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:48.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIRxs4Oj0jmnk0CNflzsgKU4n9o= 2022-09-03T06:54:48.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIRxs4Oj0jmnk0CNflzsgKU4n9o=', 'Content-Length': '0'} 2022-09-03T06:54:48.390 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6291a6cb06b3924-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.390 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:54:48.391 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.392 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.392 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.393 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.393 DEBUG:boto:Method: GET 2022-09-03T06:54:48.394 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:48.394 DEBUG:boto:Data: 2022-09-03T06:54:48.395 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.396 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.397 DEBUG:boto:Params: {} 2022-09-03T06:54:48.397 DEBUG:boto:Token: None 2022-09-03T06:54:48.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:48.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7m8YP5fCzJKPSgLDxDpNhePSyhY= 2022-09-03T06:54:48.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7m8YP5fCzJKPSgLDxDpNhePSyhY=', 'Content-Length': '0'} 2022-09-03T06:54:48.401 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ce22add75500b64-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.402 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.402 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.403 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.403 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:48.404 DEBUG:boto:Method: GET 2022-09-03T06:54:48.405 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:48.405 DEBUG:boto:Data: 2022-09-03T06:54:48.406 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.407 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.407 DEBUG:boto:Params: {} 2022-09-03T06:54:48.408 DEBUG:boto:Token: None 2022-09-03T06:54:48.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:48.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7m8YP5fCzJKPSgLDxDpNhePSyhY= 2022-09-03T06:54:48.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7m8YP5fCzJKPSgLDxDpNhePSyhY=', 'Content-Length': '0'} 2022-09-03T06:54:48.412 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c458b84a27f67d47-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.413 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.413 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.414 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:48.414 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:48.415 DEBUG:boto:Method: GET 2022-09-03T06:54:48.415 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:48.416 DEBUG:boto:Data: 2022-09-03T06:54:48.416 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.417 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.417 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.418 DEBUG:boto:Params: {} 2022-09-03T06:54:48.418 DEBUG:boto:Token: None 2022-09-03T06:54:48.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.424 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ca0bfdc20a54ffd0-006312fa38-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.425 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.426 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.427 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.428 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:48.429 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:48.430 DEBUG:boto:Method: GET 2022-09-03T06:54:48.431 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:48.432 DEBUG:boto:Data: 2022-09-03T06:54:48.433 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.435 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.436 DEBUG:boto:Params: {} 2022-09-03T06:54:48.437 DEBUG:boto:Token: None 2022-09-03T06:54:48.438 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.444 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000934bd78a633a902a-006312fa38-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.445 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.446 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.447 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.448 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.448 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:48.449 DEBUG:boto:Data: 2022-09-03T06:54:48.449 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.451 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.452 DEBUG:boto:Params: {} 2022-09-03T06:54:48.452 DEBUG:boto:Token: None 2022-09-03T06:54:48.453 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a 2022-09-03T06:54:48.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:79rlFBzVKL8Fyk9Lwq6YnMEA7MA= 2022-09-03T06:54:48.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:79rlFBzVKL8Fyk9Lwq6YnMEA7MA=', 'Content-Length': '0'} 2022-09-03T06:54:48.456 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087a4149c2bfc1fc4-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.457 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.458 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.458 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.459 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:48.460 DEBUG:boto:Data: 2022-09-03T06:54:48.460 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.462 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.462 DEBUG:boto:Params: {} 2022-09-03T06:54:48.463 DEBUG:boto:Token: None 2022-09-03T06:54:48.464 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a 2022-09-03T06:54:48.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:79rlFBzVKL8Fyk9Lwq6YnMEA7MA= 2022-09-03T06:54:48.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:79rlFBzVKL8Fyk9Lwq6YnMEA7MA=', 'Content-Length': '0'} 2022-09-03T06:54:48.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb4fb9d05124e9a0-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.468 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:54:48.469 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.469 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.470 DEBUG:boto:Method: GET 2022-09-03T06:54:48.471 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:48.471 DEBUG:boto:Data: 2022-09-03T06:54:48.472 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.473 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.474 DEBUG:boto:Params: {} 2022-09-03T06:54:48.475 DEBUG:boto:Token: None 2022-09-03T06:54:48.476 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a 2022-09-03T06:54:48.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XR236wJiEKKvVHpMCSuqzdhrlfA= 2022-09-03T06:54:48.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XR236wJiEKKvVHpMCSuqzdhrlfA=', 'Content-Length': '0'} 2022-09-03T06:54:48.481 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b27db622ec8d5364-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.482 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:48.483 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:48.484 DEBUG:boto:Method: GET 2022-09-03T06:54:48.485 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:48.485 DEBUG:boto:Data: 2022-09-03T06:54:48.486 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.486 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.487 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.487 DEBUG:boto:Params: {} 2022-09-03T06:54:48.488 DEBUG:boto:Token: None 2022-09-03T06:54:48.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/a 2022-09-03T06:54:48.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XR236wJiEKKvVHpMCSuqzdhrlfA= 2022-09-03T06:54:48.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XR236wJiEKKvVHpMCSuqzdhrlfA=', 'Content-Length': '0'} 2022-09-03T06:54:48.493 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003239f77a2e65ae14-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.495 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.495 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.496 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:48.497 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:48.498 DEBUG:boto:Method: GET 2022-09-03T06:54:48.498 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:48.499 DEBUG:boto:Data: 2022-09-03T06:54:48.499 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.500 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.501 DEBUG:boto:Params: {} 2022-09-03T06:54:48.502 DEBUG:boto:Token: None 2022-09-03T06:54:48.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.507 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c98e60111e53d725-006312fa38-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.508 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.508 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.509 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.510 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:48.510 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:48.511 DEBUG:boto:Method: GET 2022-09-03T06:54:48.511 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:48.512 DEBUG:boto:Data: 2022-09-03T06:54:48.512 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.513 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.514 DEBUG:boto:Params: {} 2022-09-03T06:54:48.515 DEBUG:boto:Token: None 2022-09-03T06:54:48.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.526 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000070eba54239b04725-006312fa38-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.527 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.529 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:54:48.530 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.531 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.532 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.533 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.534 DEBUG:boto:Method: GET 2022-09-03T06:54:48.535 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:48.536 DEBUG:boto:Data: 2022-09-03T06:54:48.537 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.539 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.540 DEBUG:boto:Params: {} 2022-09-03T06:54:48.541 DEBUG:boto:Token: None 2022-09-03T06:54:48.542 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:48.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HsAp3RUOcMmraDty1/nPCOC09L8= 2022-09-03T06:54:48.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HsAp3RUOcMmraDty1/nPCOC09L8=', 'Content-Length': '0'} 2022-09-03T06:54:48.549 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000435cc6e07297aa16-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.590 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.592 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.593 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.594 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.595 DEBUG:boto:Method: GET 2022-09-03T06:54:48.596 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:48.597 DEBUG:boto:Data: 2022-09-03T06:54:48.598 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.598 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.599 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.599 DEBUG:boto:Params: {} 2022-09-03T06:54:48.600 DEBUG:boto:Token: None 2022-09-03T06:54:48.601 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:48.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HsAp3RUOcMmraDty1/nPCOC09L8= 2022-09-03T06:54:48.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HsAp3RUOcMmraDty1/nPCOC09L8=', 'Content-Length': '0'} 2022-09-03T06:54:48.608 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5402eaedc7424c1-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.609 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.610 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.611 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.612 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.613 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.614 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:48.615 DEBUG:boto:Data: 2022-09-03T06:54:48.616 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.617 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.618 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.619 DEBUG:boto:Params: {} 2022-09-03T06:54:48.620 DEBUG:boto:Token: None 2022-09-03T06:54:48.621 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:48.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JXYwmFAbqR4RaoepJp41/qjAf3k= 2022-09-03T06:54:48.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JXYwmFAbqR4RaoepJp41/qjAf3k=', 'Content-Length': '0'} 2022-09-03T06:54:48.626 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dbcea728c949bece-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.628 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.629 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.630 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.630 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.631 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.632 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:48.633 DEBUG:boto:Data: 2022-09-03T06:54:48.634 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.636 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.637 DEBUG:boto:Params: {} 2022-09-03T06:54:48.638 DEBUG:boto:Token: None 2022-09-03T06:54:48.639 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:48.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JXYwmFAbqR4RaoepJp41/qjAf3k= 2022-09-03T06:54:48.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JXYwmFAbqR4RaoepJp41/qjAf3k=', 'Content-Length': '0'} 2022-09-03T06:54:48.644 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a620af533f2cde95-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.645 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:54:48.646 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.647 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.648 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.649 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.650 DEBUG:boto:Method: GET 2022-09-03T06:54:48.651 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:48.651 DEBUG:boto:Data: 2022-09-03T06:54:48.653 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.654 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.655 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.656 DEBUG:boto:Params: {} 2022-09-03T06:54:48.658 DEBUG:boto:Token: None 2022-09-03T06:54:48.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:48.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HsAp3RUOcMmraDty1/nPCOC09L8= 2022-09-03T06:54:48.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HsAp3RUOcMmraDty1/nPCOC09L8=', 'Content-Length': '0'} 2022-09-03T06:54:48.665 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000088969b240cee9c0d-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.666 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.667 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.667 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.669 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:48.669 DEBUG:boto:Method: GET 2022-09-03T06:54:48.670 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:48.671 DEBUG:boto:Data: 2022-09-03T06:54:48.672 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.673 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.674 DEBUG:boto:Params: {} 2022-09-03T06:54:48.675 DEBUG:boto:Token: None 2022-09-03T06:54:48.676 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:48.677 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HsAp3RUOcMmraDty1/nPCOC09L8= 2022-09-03T06:54:48.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HsAp3RUOcMmraDty1/nPCOC09L8=', 'Content-Length': '0'} 2022-09-03T06:54:48.680 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6894ddfe68cdac3-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.681 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.682 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.682 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:48.683 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:48.684 DEBUG:boto:Method: GET 2022-09-03T06:54:48.685 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:48.685 DEBUG:boto:Data: 2022-09-03T06:54:48.686 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.687 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.688 DEBUG:boto:Params: {} 2022-09-03T06:54:48.688 DEBUG:boto:Token: None 2022-09-03T06:54:48.689 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.694 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ee0942518bac722-006312fa38-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.696 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.697 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.702 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.703 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:48.704 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:48.704 DEBUG:boto:Method: GET 2022-09-03T06:54:48.705 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:48.706 DEBUG:boto:Data: 2022-09-03T06:54:48.707 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.708 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.709 DEBUG:boto:Params: {} 2022-09-03T06:54:48.710 DEBUG:boto:Token: None 2022-09-03T06:54:48.711 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.716 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4fb5b0ad04166f3-006312fa38-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.717 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.719 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.719 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.720 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.720 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:48.721 DEBUG:boto:Data: 2022-09-03T06:54:48.721 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.722 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.723 DEBUG:boto:Params: {} 2022-09-03T06:54:48.724 DEBUG:boto:Token: None 2022-09-03T06:54:48.724 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b 2022-09-03T06:54:48.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/oemqHtTlx5VVf40kydOYcNe8A= 2022-09-03T06:54:48.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/oemqHtTlx5VVf40kydOYcNe8A=', 'Content-Length': '0'} 2022-09-03T06:54:48.728 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c7e9b896630a0b6-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.729 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.730 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.731 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.732 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:48.733 DEBUG:boto:Data: 2022-09-03T06:54:48.733 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.735 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.736 DEBUG:boto:Params: {} 2022-09-03T06:54:48.737 DEBUG:boto:Token: None 2022-09-03T06:54:48.738 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b 2022-09-03T06:54:48.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/oemqHtTlx5VVf40kydOYcNe8A= 2022-09-03T06:54:48.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/oemqHtTlx5VVf40kydOYcNe8A=', 'Content-Length': '0'} 2022-09-03T06:54:48.746 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cedec1b5498743a6-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.749 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:54:48.751 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.753 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.754 DEBUG:boto:Method: GET 2022-09-03T06:54:48.755 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:48.756 DEBUG:boto:Data: 2022-09-03T06:54:48.757 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.759 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.760 DEBUG:boto:Params: {} 2022-09-03T06:54:48.762 DEBUG:boto:Token: None 2022-09-03T06:54:48.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b 2022-09-03T06:54:48.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lv3iLlvGEBlPkdn/g/w36H84B9M= 2022-09-03T06:54:48.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lv3iLlvGEBlPkdn/g/w36H84B9M=', 'Content-Length': '0'} 2022-09-03T06:54:48.773 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000738c4352ad006c1c-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.777 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:48.778 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:48.779 DEBUG:boto:Method: GET 2022-09-03T06:54:48.782 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:48.783 DEBUG:boto:Data: 2022-09-03T06:54:48.784 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.785 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.785 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.786 DEBUG:boto:Params: {} 2022-09-03T06:54:48.786 DEBUG:boto:Token: None 2022-09-03T06:54:48.787 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/b 2022-09-03T06:54:48.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lv3iLlvGEBlPkdn/g/w36H84B9M= 2022-09-03T06:54:48.788 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lv3iLlvGEBlPkdn/g/w36H84B9M=', 'Content-Length': '0'} 2022-09-03T06:54:48.790 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a0287188fdd5561-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.791 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.792 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.793 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:48.793 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:48.794 DEBUG:boto:Method: GET 2022-09-03T06:54:48.794 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:48.795 DEBUG:boto:Data: 2022-09-03T06:54:48.796 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.796 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.797 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.797 DEBUG:boto:Params: {} 2022-09-03T06:54:48.798 DEBUG:boto:Token: None 2022-09-03T06:54:48.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.803 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a50190397f43d390-006312fa38-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.851 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.852 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:48.852 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:48.853 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:48.854 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:48.854 DEBUG:boto:Method: GET 2022-09-03T06:54:48.855 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:48.855 DEBUG:boto:Data: 2022-09-03T06:54:48.856 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.856 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.857 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.857 DEBUG:boto:Params: {} 2022-09-03T06:54:48.858 DEBUG:boto:Token: None 2022-09-03T06:54:48.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/ 2022-09-03T06:54:48.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4= 2022-09-03T06:54:48.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+h150QBzbX2uWlVAgUqDzNM6Ud4=', 'Content-Length': '0'} 2022-09-03T06:54:48.863 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c036ca23995a164-006312fa38-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.864 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:48.866 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:54:48.867 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:48.868 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:48.869 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.870 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.872 DEBUG:boto:Method: GET 2022-09-03T06:54:48.873 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:48.874 DEBUG:boto:Data: 2022-09-03T06:54:48.875 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.876 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.877 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.877 DEBUG:boto:Params: {} 2022-09-03T06:54:48.878 DEBUG:boto:Token: None 2022-09-03T06:54:48.879 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:48.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8VVSvMSx3Bk1ERwD3fka2BUhWeM= 2022-09-03T06:54:48.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8VVSvMSx3Bk1ERwD3fka2BUhWeM=', 'Content-Length': '0'} 2022-09-03T06:54:48.883 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2126e76c7dd452f-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.884 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:48.885 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:48.885 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.886 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.886 DEBUG:boto:Method: GET 2022-09-03T06:54:48.887 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:48.887 DEBUG:boto:Data: 2022-09-03T06:54:48.888 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.888 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.889 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.890 DEBUG:boto:Params: {} 2022-09-03T06:54:48.890 DEBUG:boto:Token: None 2022-09-03T06:54:48.891 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:48.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8VVSvMSx3Bk1ERwD3fka2BUhWeM= 2022-09-03T06:54:48.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8VVSvMSx3Bk1ERwD3fka2BUhWeM=', 'Content-Length': '0'} 2022-09-03T06:54:48.898 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083ef0733f3ca03e7-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.946 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:48.947 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:48.948 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.949 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.950 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.951 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:48.952 DEBUG:boto:Data: 2022-09-03T06:54:48.953 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.958 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.959 DEBUG:boto:Params: {} 2022-09-03T06:54:48.959 DEBUG:boto:Token: None 2022-09-03T06:54:48.960 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:48.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lUs6b6joVerkDjCztWvp0IDsAu4= 2022-09-03T06:54:48.961 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lUs6b6joVerkDjCztWvp0IDsAu4=', 'Content-Length': '0'} 2022-09-03T06:54:48.964 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd7df16bb8f5e35f-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.964 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:48.965 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:48.965 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.966 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.966 DEBUG:boto:Method: HEAD 2022-09-03T06:54:48.967 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:48.969 DEBUG:boto:Data: 2022-09-03T06:54:48.970 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.970 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:48.971 DEBUG:boto:Port: 8000 2022-09-03T06:54:48.971 DEBUG:boto:Params: {} 2022-09-03T06:54:48.972 DEBUG:boto:Token: None 2022-09-03T06:54:48.973 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:48.974 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lUs6b6joVerkDjCztWvp0IDsAu4= 2022-09-03T06:54:48.975 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lUs6b6joVerkDjCztWvp0IDsAu4=', 'Content-Length': '0'} 2022-09-03T06:54:48.978 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000527111d7b0d51f6-006312fa38-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.980 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:54:48.980 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:48.981 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:48.982 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.983 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.984 DEBUG:boto:Method: GET 2022-09-03T06:54:48.985 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:48.986 DEBUG:boto:Data: 2022-09-03T06:54:48.987 DEBUG:boto:Headers: {} 2022-09-03T06:54:48.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:48.989 DEBUG:boto:Port: 8001 2022-09-03T06:54:48.990 DEBUG:boto:Params: {} 2022-09-03T06:54:48.992 DEBUG:boto:Token: None 2022-09-03T06:54:48.992 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:48 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:48.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8VVSvMSx3Bk1ERwD3fka2BUhWeM= 2022-09-03T06:54:48.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8VVSvMSx3Bk1ERwD3fka2BUhWeM=', 'Content-Length': '0'} 2022-09-03T06:54:48.996 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078bbc57d7c5812dc-006312fa38-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:48.997 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:48.998 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:48.999 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:48.999 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:49.000 DEBUG:boto:Method: GET 2022-09-03T06:54:49.000 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:49.001 DEBUG:boto:Data: 2022-09-03T06:54:49.002 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.005 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.008 DEBUG:boto:Params: {} 2022-09-03T06:54:49.009 DEBUG:boto:Token: None 2022-09-03T06:54:49.010 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:49.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RmWywrjfCVo/VmQudkEST02nlc8= 2022-09-03T06:54:49.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RmWywrjfCVo/VmQudkEST02nlc8=', 'Content-Length': '0'} 2022-09-03T06:54:49.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000259907b3ad2dbe3f-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.018 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.019 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.019 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:49.020 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:49.020 DEBUG:boto:Method: GET 2022-09-03T06:54:49.021 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:49.022 DEBUG:boto:Data: 2022-09-03T06:54:49.023 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.023 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.024 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.025 DEBUG:boto:Params: {} 2022-09-03T06:54:49.026 DEBUG:boto:Token: None 2022-09-03T06:54:49.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.033 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005847009559d40cdc-006312fa39-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.034 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.035 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.036 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.036 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:49.037 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:49.038 DEBUG:boto:Method: GET 2022-09-03T06:54:49.039 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:49.039 DEBUG:boto:Data: 2022-09-03T06:54:49.040 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.041 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.042 DEBUG:boto:Params: {} 2022-09-03T06:54:49.042 DEBUG:boto:Token: None 2022-09-03T06:54:49.043 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.044 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.047 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045419f2e196ddaee-006312fa39-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.048 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.049 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:49.049 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:49.050 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.050 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:49.051 DEBUG:boto:Data: 2022-09-03T06:54:49.052 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.053 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.053 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.054 DEBUG:boto:Params: {} 2022-09-03T06:54:49.054 DEBUG:boto:Token: None 2022-09-03T06:54:49.055 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/c 2022-09-03T06:54:49.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uiOsOQob8onQyB0EnIBcPiH9NpA= 2022-09-03T06:54:49.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uiOsOQob8onQyB0EnIBcPiH9NpA=', 'Content-Length': '0'} 2022-09-03T06:54:49.059 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000490e5402f3232366-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.059 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:49.060 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:49.061 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.061 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:49.062 DEBUG:boto:Data: 2022-09-03T06:54:49.062 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.063 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.063 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.064 DEBUG:boto:Params: {} 2022-09-03T06:54:49.064 DEBUG:boto:Token: None 2022-09-03T06:54:49.065 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/c 2022-09-03T06:54:49.066 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uiOsOQob8onQyB0EnIBcPiH9NpA= 2022-09-03T06:54:49.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uiOsOQob8onQyB0EnIBcPiH9NpA=', 'Content-Length': '0'} 2022-09-03T06:54:49.069 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000050dc38d790c00e54-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.070 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:54:49.071 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:49.072 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:49.073 DEBUG:boto:Method: GET 2022-09-03T06:54:49.074 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:49.075 DEBUG:boto:Data: 2022-09-03T06:54:49.076 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.077 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.078 DEBUG:boto:Params: {} 2022-09-03T06:54:49.079 DEBUG:boto:Token: None 2022-09-03T06:54:49.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/c 2022-09-03T06:54:49.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:da2HNtPcMCFU9agSI/3h/x8Z4KQ= 2022-09-03T06:54:49.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:da2HNtPcMCFU9agSI/3h/x8Z4KQ=', 'Content-Length': '0'} 2022-09-03T06:54:49.084 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070673fddd388cbb6-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.085 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:49.085 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:49.086 DEBUG:boto:Method: GET 2022-09-03T06:54:49.086 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:49.087 DEBUG:boto:Data: 2022-09-03T06:54:49.087 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.088 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.088 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.089 DEBUG:boto:Params: {} 2022-09-03T06:54:49.090 DEBUG:boto:Token: None 2022-09-03T06:54:49.090 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/c 2022-09-03T06:54:49.091 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:da2HNtPcMCFU9agSI/3h/x8Z4KQ= 2022-09-03T06:54:49.092 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:da2HNtPcMCFU9agSI/3h/x8Z4KQ=', 'Content-Length': '0'} 2022-09-03T06:54:49.094 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec1295c640076184-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.095 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.096 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.096 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:49.097 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:49.098 DEBUG:boto:Method: GET 2022-09-03T06:54:49.099 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:49.099 DEBUG:boto:Data: 2022-09-03T06:54:49.100 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.101 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.101 DEBUG:boto:Params: {} 2022-09-03T06:54:49.102 DEBUG:boto:Token: None 2022-09-03T06:54:49.103 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.107 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009d1ba920918d333c-006312fa39-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.107 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.108 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.109 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.109 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:49.110 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:49.110 DEBUG:boto:Method: GET 2022-09-03T06:54:49.111 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:49.111 DEBUG:boto:Data: 2022-09-03T06:54:49.112 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.113 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.113 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.114 DEBUG:boto:Params: {} 2022-09-03T06:54:49.114 DEBUG:boto:Token: None 2022-09-03T06:54:49.115 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.119 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001bf2b4974692189c-006312fa39-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.120 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.120 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:54:49.121 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.122 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.123 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.123 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.124 DEBUG:boto:Method: GET 2022-09-03T06:54:49.124 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:49.125 DEBUG:boto:Data: 2022-09-03T06:54:49.126 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.127 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.127 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.128 DEBUG:boto:Params: {} 2022-09-03T06:54:49.129 DEBUG:boto:Token: None 2022-09-03T06:54:49.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:49.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eO8owCyWdHuQzwK7q547YYA4rpc= 2022-09-03T06:54:49.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eO8owCyWdHuQzwK7q547YYA4rpc=', 'Content-Length': '0'} 2022-09-03T06:54:49.147 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000728d8b6f8b461a79-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.186 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.187 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.188 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.189 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.193 DEBUG:boto:Method: GET 2022-09-03T06:54:49.194 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:49.194 DEBUG:boto:Data: 2022-09-03T06:54:49.195 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.196 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.197 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.198 DEBUG:boto:Params: {} 2022-09-03T06:54:49.199 DEBUG:boto:Token: None 2022-09-03T06:54:49.200 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:49.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eO8owCyWdHuQzwK7q547YYA4rpc= 2022-09-03T06:54:49.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eO8owCyWdHuQzwK7q547YYA4rpc=', 'Content-Length': '0'} 2022-09-03T06:54:49.204 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5ab6c9d3c25cc28-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.205 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.205 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.206 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.206 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.208 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.209 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:49.209 DEBUG:boto:Data: 2022-09-03T06:54:49.210 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.211 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.211 DEBUG:boto:Params: {} 2022-09-03T06:54:49.212 DEBUG:boto:Token: None 2022-09-03T06:54:49.212 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:49.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:liQAAd3vc1wK3ktJMDXzAt1fKM4= 2022-09-03T06:54:49.214 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:liQAAd3vc1wK3ktJMDXzAt1fKM4=', 'Content-Length': '0'} 2022-09-03T06:54:49.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079b6454f74297c91-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.217 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.217 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.218 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.219 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.219 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.220 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:49.220 DEBUG:boto:Data: 2022-09-03T06:54:49.221 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.222 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.223 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.224 DEBUG:boto:Params: {} 2022-09-03T06:54:49.225 DEBUG:boto:Token: None 2022-09-03T06:54:49.225 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:49.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:liQAAd3vc1wK3ktJMDXzAt1fKM4= 2022-09-03T06:54:49.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:liQAAd3vc1wK3ktJMDXzAt1fKM4=', 'Content-Length': '0'} 2022-09-03T06:54:49.230 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000888b1bb7a7674124-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.231 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:54:49.232 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.232 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.234 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.235 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.237 DEBUG:boto:Method: GET 2022-09-03T06:54:49.238 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:49.238 DEBUG:boto:Data: 2022-09-03T06:54:49.239 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.240 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.240 DEBUG:boto:Params: {} 2022-09-03T06:54:49.241 DEBUG:boto:Token: None 2022-09-03T06:54:49.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:49.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eO8owCyWdHuQzwK7q547YYA4rpc= 2022-09-03T06:54:49.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eO8owCyWdHuQzwK7q547YYA4rpc=', 'Content-Length': '0'} 2022-09-03T06:54:49.245 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d14a457b81251fb-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.246 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.246 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.247 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.248 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:49.248 DEBUG:boto:Method: GET 2022-09-03T06:54:49.249 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:49.249 DEBUG:boto:Data: 2022-09-03T06:54:49.250 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.251 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.252 DEBUG:boto:Params: {} 2022-09-03T06:54:49.252 DEBUG:boto:Token: None 2022-09-03T06:54:49.253 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:49.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eO8owCyWdHuQzwK7q547YYA4rpc= 2022-09-03T06:54:49.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eO8owCyWdHuQzwK7q547YYA4rpc=', 'Content-Length': '0'} 2022-09-03T06:54:49.257 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c82ecdf3d0f3600-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.258 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.259 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.259 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:49.260 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:49.261 DEBUG:boto:Method: GET 2022-09-03T06:54:49.261 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:49.262 DEBUG:boto:Data: 2022-09-03T06:54:49.263 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.264 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.264 DEBUG:boto:Params: {} 2022-09-03T06:54:49.265 DEBUG:boto:Token: None 2022-09-03T06:54:49.266 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.271 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000513e1f7e46142fff-006312fa39-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.272 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.274 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.275 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.276 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:49.277 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:49.278 DEBUG:boto:Method: GET 2022-09-03T06:54:49.279 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:49.281 DEBUG:boto:Data: 2022-09-03T06:54:49.282 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.283 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.284 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.285 DEBUG:boto:Params: {} 2022-09-03T06:54:49.286 DEBUG:boto:Token: None 2022-09-03T06:54:49.287 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.293 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ccffc30322bc4aa1-006312fa39-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.294 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.295 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.296 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.296 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.297 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:49.298 DEBUG:boto:Data: 2022-09-03T06:54:49.299 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.300 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.301 DEBUG:boto:Params: {} 2022-09-03T06:54:49.301 DEBUG:boto:Token: None 2022-09-03T06:54:49.302 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d 2022-09-03T06:54:49.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ioFxQUXN13uxsGaq8u53QyUGAos= 2022-09-03T06:54:49.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ioFxQUXN13uxsGaq8u53QyUGAos=', 'Content-Length': '0'} 2022-09-03T06:54:49.305 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fabd2e18c949f18e-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.306 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.307 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.307 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.308 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:49.308 DEBUG:boto:Data: 2022-09-03T06:54:49.309 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.310 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.310 DEBUG:boto:Params: {} 2022-09-03T06:54:49.311 DEBUG:boto:Token: None 2022-09-03T06:54:49.312 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d 2022-09-03T06:54:49.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ioFxQUXN13uxsGaq8u53QyUGAos= 2022-09-03T06:54:49.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ioFxQUXN13uxsGaq8u53QyUGAos=', 'Content-Length': '0'} 2022-09-03T06:54:49.315 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c1545a780cb375f-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.315 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:54:49.316 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.317 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.317 DEBUG:boto:Method: GET 2022-09-03T06:54:49.318 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:49.318 DEBUG:boto:Data: 2022-09-03T06:54:49.319 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.320 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.321 DEBUG:boto:Params: {} 2022-09-03T06:54:49.321 DEBUG:boto:Token: None 2022-09-03T06:54:49.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d 2022-09-03T06:54:49.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6RPhz+lLSbzeplpS+RB8n6V/Nbs= 2022-09-03T06:54:49.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6RPhz+lLSbzeplpS+RB8n6V/Nbs=', 'Content-Length': '0'} 2022-09-03T06:54:49.325 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5bb1f5069209a14-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.326 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:49.327 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:49.327 DEBUG:boto:Method: GET 2022-09-03T06:54:49.328 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:49.329 DEBUG:boto:Data: 2022-09-03T06:54:49.329 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.330 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.331 DEBUG:boto:Params: {} 2022-09-03T06:54:49.331 DEBUG:boto:Token: None 2022-09-03T06:54:49.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/d 2022-09-03T06:54:49.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6RPhz+lLSbzeplpS+RB8n6V/Nbs= 2022-09-03T06:54:49.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6RPhz+lLSbzeplpS+RB8n6V/Nbs=', 'Content-Length': '0'} 2022-09-03T06:54:49.335 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d7f6a7a5a5228e0-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.378 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.379 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.380 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:49.380 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:49.381 DEBUG:boto:Method: GET 2022-09-03T06:54:49.381 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:49.382 DEBUG:boto:Data: 2022-09-03T06:54:49.383 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.384 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.384 DEBUG:boto:Params: {} 2022-09-03T06:54:49.385 DEBUG:boto:Token: None 2022-09-03T06:54:49.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.390 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000431ca6498f60f92e-006312fa39-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.391 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.392 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.392 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.393 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:49.394 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:49.394 DEBUG:boto:Method: GET 2022-09-03T06:54:49.395 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:49.396 DEBUG:boto:Data: 2022-09-03T06:54:49.396 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.397 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.397 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.398 DEBUG:boto:Params: {} 2022-09-03T06:54:49.399 DEBUG:boto:Token: None 2022-09-03T06:54:49.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.406 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007193c532f2a1d67c-006312fa39-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.409 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.410 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:54:49.411 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.412 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.413 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.414 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.416 DEBUG:boto:Method: GET 2022-09-03T06:54:49.417 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:49.417 DEBUG:boto:Data: 2022-09-03T06:54:49.419 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.420 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.420 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.421 DEBUG:boto:Params: {} 2022-09-03T06:54:49.422 DEBUG:boto:Token: None 2022-09-03T06:54:49.425 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:49.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fuidX45mStwhBk8HR5AFziikBJk= 2022-09-03T06:54:49.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fuidX45mStwhBk8HR5AFziikBJk=', 'Content-Length': '0'} 2022-09-03T06:54:49.431 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1e5864d4014ec82-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.474 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.475 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.475 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.476 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.477 DEBUG:boto:Method: GET 2022-09-03T06:54:49.477 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:49.478 DEBUG:boto:Data: 2022-09-03T06:54:49.478 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.480 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.481 DEBUG:boto:Params: {} 2022-09-03T06:54:49.483 DEBUG:boto:Token: None 2022-09-03T06:54:49.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:49.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fuidX45mStwhBk8HR5AFziikBJk= 2022-09-03T06:54:49.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fuidX45mStwhBk8HR5AFziikBJk=', 'Content-Length': '0'} 2022-09-03T06:54:49.489 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002838aac89fc9f97-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.491 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.492 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.493 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.494 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.495 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.496 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:49.496 DEBUG:boto:Data: 2022-09-03T06:54:49.497 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.497 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.498 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.498 DEBUG:boto:Params: {} 2022-09-03T06:54:49.499 DEBUG:boto:Token: None 2022-09-03T06:54:49.499 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:49.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F2j6xVkED08n68R/EBKZt0CnyHg= 2022-09-03T06:54:49.501 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F2j6xVkED08n68R/EBKZt0CnyHg=', 'Content-Length': '0'} 2022-09-03T06:54:49.503 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056dc38a74ca7769c-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.504 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.505 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.505 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.506 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.506 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.507 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:49.508 DEBUG:boto:Data: 2022-09-03T06:54:49.508 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.509 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.509 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.510 DEBUG:boto:Params: {} 2022-09-03T06:54:49.511 DEBUG:boto:Token: None 2022-09-03T06:54:49.512 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:49.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F2j6xVkED08n68R/EBKZt0CnyHg= 2022-09-03T06:54:49.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F2j6xVkED08n68R/EBKZt0CnyHg=', 'Content-Length': '0'} 2022-09-03T06:54:49.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005bc8b0373a30bb49-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.519 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:54:49.520 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.521 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.522 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.523 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.524 DEBUG:boto:Method: GET 2022-09-03T06:54:49.525 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:49.526 DEBUG:boto:Data: 2022-09-03T06:54:49.527 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.528 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.529 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.530 DEBUG:boto:Params: {} 2022-09-03T06:54:49.531 DEBUG:boto:Token: None 2022-09-03T06:54:49.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:49.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fuidX45mStwhBk8HR5AFziikBJk= 2022-09-03T06:54:49.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fuidX45mStwhBk8HR5AFziikBJk=', 'Content-Length': '0'} 2022-09-03T06:54:49.536 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006fda4d6c8abaa1d6-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.537 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.537 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.538 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.538 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:49.539 DEBUG:boto:Method: GET 2022-09-03T06:54:49.539 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:49.540 DEBUG:boto:Data: 2022-09-03T06:54:49.540 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.541 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.542 DEBUG:boto:Params: {} 2022-09-03T06:54:49.543 DEBUG:boto:Token: None 2022-09-03T06:54:49.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:49.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fuidX45mStwhBk8HR5AFziikBJk= 2022-09-03T06:54:49.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fuidX45mStwhBk8HR5AFziikBJk=', 'Content-Length': '0'} 2022-09-03T06:54:49.547 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0d8b98da59c61b3-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.548 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.549 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.550 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:49.550 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:49.551 DEBUG:boto:Method: GET 2022-09-03T06:54:49.551 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:49.552 DEBUG:boto:Data: 2022-09-03T06:54:49.552 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.553 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.553 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.554 DEBUG:boto:Params: {} 2022-09-03T06:54:49.554 DEBUG:boto:Token: None 2022-09-03T06:54:49.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.557 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.560 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f99435247c0cdc30-006312fa39-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.560 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.561 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.562 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.563 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:49.563 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:49.564 DEBUG:boto:Method: GET 2022-09-03T06:54:49.564 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:49.565 DEBUG:boto:Data: 2022-09-03T06:54:49.565 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.566 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.567 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.567 DEBUG:boto:Params: {} 2022-09-03T06:54:49.568 DEBUG:boto:Token: None 2022-09-03T06:54:49.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.574 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000010f2d76aaa6e1751-006312fa39-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.575 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.576 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.577 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.578 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.579 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:49.580 DEBUG:boto:Data: 2022-09-03T06:54:49.581 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.583 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.584 DEBUG:boto:Params: {} 2022-09-03T06:54:49.585 DEBUG:boto:Token: None 2022-09-03T06:54:49.585 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e 2022-09-03T06:54:49.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/ecJTzSDYWjviI/lIzXLw94yrI= 2022-09-03T06:54:49.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/ecJTzSDYWjviI/lIzXLw94yrI=', 'Content-Length': '0'} 2022-09-03T06:54:49.589 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000708534923069a3dc-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.591 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.592 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.593 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.594 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:49.595 DEBUG:boto:Data: 2022-09-03T06:54:49.596 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.596 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.597 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.597 DEBUG:boto:Params: {} 2022-09-03T06:54:49.598 DEBUG:boto:Token: None 2022-09-03T06:54:49.599 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e 2022-09-03T06:54:49.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/ecJTzSDYWjviI/lIzXLw94yrI= 2022-09-03T06:54:49.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/ecJTzSDYWjviI/lIzXLw94yrI=', 'Content-Length': '0'} 2022-09-03T06:54:49.602 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000239ddd9c14098aec-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.603 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:54:49.604 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.605 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.606 DEBUG:boto:Method: GET 2022-09-03T06:54:49.607 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:49.608 DEBUG:boto:Data: 2022-09-03T06:54:49.609 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.610 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.611 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.612 DEBUG:boto:Params: {} 2022-09-03T06:54:49.613 DEBUG:boto:Token: None 2022-09-03T06:54:49.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e 2022-09-03T06:54:49.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T4GbLyT5v3IAR7Jdt323F7EZTB0= 2022-09-03T06:54:49.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T4GbLyT5v3IAR7Jdt323F7EZTB0=', 'Content-Length': '0'} 2022-09-03T06:54:49.617 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052dab1025bcccdce-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.658 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:49.660 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:49.661 DEBUG:boto:Method: GET 2022-09-03T06:54:49.661 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:49.662 DEBUG:boto:Data: 2022-09-03T06:54:49.662 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.663 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.663 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.664 DEBUG:boto:Params: {} 2022-09-03T06:54:49.665 DEBUG:boto:Token: None 2022-09-03T06:54:49.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/e 2022-09-03T06:54:49.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T4GbLyT5v3IAR7Jdt323F7EZTB0= 2022-09-03T06:54:49.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T4GbLyT5v3IAR7Jdt323F7EZTB0=', 'Content-Length': '0'} 2022-09-03T06:54:49.670 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002647ba6fc5239d4a-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.671 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.671 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.672 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:49.672 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:49.673 DEBUG:boto:Method: GET 2022-09-03T06:54:49.674 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:49.674 DEBUG:boto:Data: 2022-09-03T06:54:49.675 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.675 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.676 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.676 DEBUG:boto:Params: {} 2022-09-03T06:54:49.677 DEBUG:boto:Token: None 2022-09-03T06:54:49.682 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.683 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.687 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db9f7fd19a86da78-006312fa39-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.688 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.690 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.691 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.692 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:49.693 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:49.694 DEBUG:boto:Method: GET 2022-09-03T06:54:49.695 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:49.696 DEBUG:boto:Data: 2022-09-03T06:54:49.698 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.699 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.700 DEBUG:boto:Params: {} 2022-09-03T06:54:49.701 DEBUG:boto:Token: None 2022-09-03T06:54:49.701 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.705 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cfdc73c8e4fcb8cd-006312fa39-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.707 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.709 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:54:49.710 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.711 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.712 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.713 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.714 DEBUG:boto:Method: GET 2022-09-03T06:54:49.715 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:49.716 DEBUG:boto:Data: 2022-09-03T06:54:49.717 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.718 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.719 DEBUG:boto:Params: {} 2022-09-03T06:54:49.720 DEBUG:boto:Token: None 2022-09-03T06:54:49.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:49.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LWerUa6WopWpf0ol35y57toaxUI= 2022-09-03T06:54:49.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LWerUa6WopWpf0ol35y57toaxUI=', 'Content-Length': '0'} 2022-09-03T06:54:49.724 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002541a6625d1ee7d6-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.725 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.726 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.726 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.727 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.727 DEBUG:boto:Method: GET 2022-09-03T06:54:49.728 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:49.728 DEBUG:boto:Data: 2022-09-03T06:54:49.729 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.730 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.731 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.731 DEBUG:boto:Params: {} 2022-09-03T06:54:49.732 DEBUG:boto:Token: None 2022-09-03T06:54:49.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:49.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LWerUa6WopWpf0ol35y57toaxUI= 2022-09-03T06:54:49.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LWerUa6WopWpf0ol35y57toaxUI=', 'Content-Length': '0'} 2022-09-03T06:54:49.736 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cad37f545b4e6c76-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.737 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.738 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.739 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.740 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.741 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.742 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:49.743 DEBUG:boto:Data: 2022-09-03T06:54:49.744 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.746 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.747 DEBUG:boto:Params: {} 2022-09-03T06:54:49.748 DEBUG:boto:Token: None 2022-09-03T06:54:49.749 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:49.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U1PSxs4tcVwPyrasMBmZpAKrigw= 2022-09-03T06:54:49.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U1PSxs4tcVwPyrasMBmZpAKrigw=', 'Content-Length': '0'} 2022-09-03T06:54:49.754 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000599b26c04baf17a7-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.756 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.760 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.761 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.762 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.763 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.764 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:49.765 DEBUG:boto:Data: 2022-09-03T06:54:49.766 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.767 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.768 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.769 DEBUG:boto:Params: {} 2022-09-03T06:54:49.771 DEBUG:boto:Token: None 2022-09-03T06:54:49.772 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:49.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U1PSxs4tcVwPyrasMBmZpAKrigw= 2022-09-03T06:54:49.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U1PSxs4tcVwPyrasMBmZpAKrigw=', 'Content-Length': '0'} 2022-09-03T06:54:49.778 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f66b40101ccdcbc-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.780 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:54:49.781 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.782 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.783 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.784 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.785 DEBUG:boto:Method: GET 2022-09-03T06:54:49.786 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:49.787 DEBUG:boto:Data: 2022-09-03T06:54:49.788 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.789 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.790 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.791 DEBUG:boto:Params: {} 2022-09-03T06:54:49.793 DEBUG:boto:Token: None 2022-09-03T06:54:49.793 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:49.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LWerUa6WopWpf0ol35y57toaxUI= 2022-09-03T06:54:49.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LWerUa6WopWpf0ol35y57toaxUI=', 'Content-Length': '0'} 2022-09-03T06:54:49.797 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db4c1fa2d834ff8c-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.843 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.844 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.844 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.845 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:49.845 DEBUG:boto:Method: GET 2022-09-03T06:54:49.846 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:49.847 DEBUG:boto:Data: 2022-09-03T06:54:49.848 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.849 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.850 DEBUG:boto:Params: {} 2022-09-03T06:54:49.851 DEBUG:boto:Token: None 2022-09-03T06:54:49.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:49.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LWerUa6WopWpf0ol35y57toaxUI= 2022-09-03T06:54:49.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LWerUa6WopWpf0ol35y57toaxUI=', 'Content-Length': '0'} 2022-09-03T06:54:49.859 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026db4f0774d659d2-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.865 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.866 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.867 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:49.868 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:49.868 DEBUG:boto:Method: GET 2022-09-03T06:54:49.869 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:49.870 DEBUG:boto:Data: 2022-09-03T06:54:49.871 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.873 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.874 DEBUG:boto:Params: {} 2022-09-03T06:54:49.875 DEBUG:boto:Token: None 2022-09-03T06:54:49.876 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.883 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b7b31ce1c197f71-006312fa39-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.884 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.886 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:49.886 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:49.887 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:49.888 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:49.888 DEBUG:boto:Method: GET 2022-09-03T06:54:49.889 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:49.890 DEBUG:boto:Data: 2022-09-03T06:54:49.891 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.893 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.893 DEBUG:boto:Params: {} 2022-09-03T06:54:49.894 DEBUG:boto:Token: None 2022-09-03T06:54:49.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/ 2022-09-03T06:54:49.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk= 2022-09-03T06:54:49.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UU5mtCKZzdmmaOSBYgVjfCb8uMk=', 'Content-Length': '0'} 2022-09-03T06:54:49.900 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008adc7256f23dec20-006312fa39-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.900 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:49.903 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.905 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.906 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.906 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:49.907 DEBUG:boto:Data: 2022-09-03T06:54:49.908 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.910 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.911 DEBUG:boto:Params: {} 2022-09-03T06:54:49.912 DEBUG:boto:Token: None 2022-09-03T06:54:49.913 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/f 2022-09-03T06:54:49.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8CVYhAiTHalttLQsBlg7Jnp5V3I= 2022-09-03T06:54:49.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8CVYhAiTHalttLQsBlg7Jnp5V3I=', 'Content-Length': '0'} 2022-09-03T06:54:49.919 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c53f2b63d779d012-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.920 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.921 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.922 DEBUG:boto:Method: HEAD 2022-09-03T06:54:49.923 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:49.926 DEBUG:boto:Data: 2022-09-03T06:54:49.927 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.928 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:49.929 DEBUG:boto:Port: 8000 2022-09-03T06:54:49.929 DEBUG:boto:Params: {} 2022-09-03T06:54:49.930 DEBUG:boto:Token: None 2022-09-03T06:54:49.931 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/f 2022-09-03T06:54:49.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8CVYhAiTHalttLQsBlg7Jnp5V3I= 2022-09-03T06:54:49.933 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8CVYhAiTHalttLQsBlg7Jnp5V3I=', 'Content-Length': '0'} 2022-09-03T06:54:49.936 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c895c4809eff0eb-006312fa39-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.937 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:54:49.938 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.939 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.940 DEBUG:boto:Method: GET 2022-09-03T06:54:49.941 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:49.942 DEBUG:boto:Data: 2022-09-03T06:54:49.943 DEBUG:boto:Headers: {} 2022-09-03T06:54:49.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:49.945 DEBUG:boto:Port: 8001 2022-09-03T06:54:49.945 DEBUG:boto:Params: {} 2022-09-03T06:54:49.946 DEBUG:boto:Token: None 2022-09-03T06:54:49.946 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:49 GMT /avysmw-71/f 2022-09-03T06:54:49.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kQtX+cQrVOfs8CRMzmBYZua+rZA= 2022-09-03T06:54:49.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kQtX+cQrVOfs8CRMzmBYZua+rZA=', 'Content-Length': '0'} 2022-09-03T06:54:49.950 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cb700efd2626aa9-006312fa39-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:49.994 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:49.996 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:49.997 DEBUG:boto:Method: GET 2022-09-03T06:54:49.998 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:49.999 DEBUG:boto:Data: 2022-09-03T06:54:50.000 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.000 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.001 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.002 DEBUG:boto:Params: {} 2022-09-03T06:54:50.003 DEBUG:boto:Token: None 2022-09-03T06:54:50.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/f 2022-09-03T06:54:50.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pcLajo/vNZKCgZA7a5qf8cGkBA0= 2022-09-03T06:54:50.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pcLajo/vNZKCgZA7a5qf8cGkBA0=', 'Content-Length': '0'} 2022-09-03T06:54:50.009 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af5520813e402952-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.010 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.012 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.012 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:50.013 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:50.014 DEBUG:boto:Method: GET 2022-09-03T06:54:50.015 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:50.015 DEBUG:boto:Data: 2022-09-03T06:54:50.016 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.016 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.017 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.018 DEBUG:boto:Params: {} 2022-09-03T06:54:50.019 DEBUG:boto:Token: None 2022-09-03T06:54:50.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.024 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000050d24a4278cad838-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.025 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.027 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.027 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.028 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:50.028 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:50.029 DEBUG:boto:Method: GET 2022-09-03T06:54:50.029 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:50.030 DEBUG:boto:Data: 2022-09-03T06:54:50.031 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.032 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.032 DEBUG:boto:Params: {} 2022-09-03T06:54:50.033 DEBUG:boto:Token: None 2022-09-03T06:54:50.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.038 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000136c3ba8ffd10f6c-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.039 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.040 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:54:50.040 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.041 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.042 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.042 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.043 DEBUG:boto:Method: GET 2022-09-03T06:54:50.044 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:50.045 DEBUG:boto:Data: 2022-09-03T06:54:50.045 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.046 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.047 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.047 DEBUG:boto:Params: {} 2022-09-03T06:54:50.048 DEBUG:boto:Token: None 2022-09-03T06:54:50.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:50.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AZ+fa896fQS+MqYxfUbqJO2YLJU= 2022-09-03T06:54:50.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AZ+fa896fQS+MqYxfUbqJO2YLJU=', 'Content-Length': '0'} 2022-09-03T06:54:50.055 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d4f569c8241556bd-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.056 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.057 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.059 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.059 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.060 DEBUG:boto:Method: GET 2022-09-03T06:54:50.061 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:50.062 DEBUG:boto:Data: 2022-09-03T06:54:50.063 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.064 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.065 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.066 DEBUG:boto:Params: {} 2022-09-03T06:54:50.067 DEBUG:boto:Token: None 2022-09-03T06:54:50.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:50.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AZ+fa896fQS+MqYxfUbqJO2YLJU= 2022-09-03T06:54:50.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AZ+fa896fQS+MqYxfUbqJO2YLJU=', 'Content-Length': '0'} 2022-09-03T06:54:50.071 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e1808fd9f8e0af2-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.072 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.073 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.073 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.074 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.075 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.075 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:50.076 DEBUG:boto:Data: 2022-09-03T06:54:50.076 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.078 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.078 DEBUG:boto:Params: {} 2022-09-03T06:54:50.079 DEBUG:boto:Token: None 2022-09-03T06:54:50.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:50.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tAEAaRlrL4eN98NVvChGrxe3Mv4= 2022-09-03T06:54:50.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tAEAaRlrL4eN98NVvChGrxe3Mv4=', 'Content-Length': '0'} 2022-09-03T06:54:50.083 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab84704659bf4500-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.084 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.085 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.085 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.086 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.086 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.087 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:50.087 DEBUG:boto:Data: 2022-09-03T06:54:50.088 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.088 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.089 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.090 DEBUG:boto:Params: {} 2022-09-03T06:54:50.090 DEBUG:boto:Token: None 2022-09-03T06:54:50.091 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:50.091 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tAEAaRlrL4eN98NVvChGrxe3Mv4= 2022-09-03T06:54:50.092 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tAEAaRlrL4eN98NVvChGrxe3Mv4=', 'Content-Length': '0'} 2022-09-03T06:54:50.098 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006173ae885dc1b783-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.100 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:54:50.101 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.102 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.102 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.103 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.103 DEBUG:boto:Method: GET 2022-09-03T06:54:50.104 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:50.104 DEBUG:boto:Data: 2022-09-03T06:54:50.105 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.106 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.107 DEBUG:boto:Params: {} 2022-09-03T06:54:50.107 DEBUG:boto:Token: None 2022-09-03T06:54:50.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:50.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AZ+fa896fQS+MqYxfUbqJO2YLJU= 2022-09-03T06:54:50.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AZ+fa896fQS+MqYxfUbqJO2YLJU=', 'Content-Length': '0'} 2022-09-03T06:54:50.112 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000811cc7370c68b0-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.113 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.113 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.114 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.115 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:50.115 DEBUG:boto:Method: GET 2022-09-03T06:54:50.116 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:50.116 DEBUG:boto:Data: 2022-09-03T06:54:50.117 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.118 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.118 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.119 DEBUG:boto:Params: {} 2022-09-03T06:54:50.119 DEBUG:boto:Token: None 2022-09-03T06:54:50.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:50.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AZ+fa896fQS+MqYxfUbqJO2YLJU= 2022-09-03T06:54:50.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AZ+fa896fQS+MqYxfUbqJO2YLJU=', 'Content-Length': '0'} 2022-09-03T06:54:50.124 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000859cd6556105c829-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.166 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.167 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.168 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:50.169 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:50.169 DEBUG:boto:Method: GET 2022-09-03T06:54:50.170 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:50.171 DEBUG:boto:Data: 2022-09-03T06:54:50.171 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.172 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.173 DEBUG:boto:Params: {} 2022-09-03T06:54:50.174 DEBUG:boto:Token: None 2022-09-03T06:54:50.174 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.178 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb458c8c5909e71b-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.179 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.180 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.181 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.181 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:50.182 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:50.182 DEBUG:boto:Method: GET 2022-09-03T06:54:50.183 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:50.183 DEBUG:boto:Data: 2022-09-03T06:54:50.184 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.186 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.187 DEBUG:boto:Params: {} 2022-09-03T06:54:50.188 DEBUG:boto:Token: None 2022-09-03T06:54:50.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.194 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b50225596bbe5f3a-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.195 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.196 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.196 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.197 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.198 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:50.199 DEBUG:boto:Data: 2022-09-03T06:54:50.200 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.200 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.201 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.201 DEBUG:boto:Params: {} 2022-09-03T06:54:50.202 DEBUG:boto:Token: None 2022-09-03T06:54:50.203 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g 2022-09-03T06:54:50.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u5aFSf/Vp9eAaHcTWVTltJLqMdM= 2022-09-03T06:54:50.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u5aFSf/Vp9eAaHcTWVTltJLqMdM=', 'Content-Length': '0'} 2022-09-03T06:54:50.208 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000740f37eb56056bdd-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.209 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.209 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.210 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.211 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:50.212 DEBUG:boto:Data: 2022-09-03T06:54:50.213 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.214 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.215 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.215 DEBUG:boto:Params: {} 2022-09-03T06:54:50.216 DEBUG:boto:Token: None 2022-09-03T06:54:50.217 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g 2022-09-03T06:54:50.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u5aFSf/Vp9eAaHcTWVTltJLqMdM= 2022-09-03T06:54:50.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u5aFSf/Vp9eAaHcTWVTltJLqMdM=', 'Content-Length': '0'} 2022-09-03T06:54:50.222 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1c728f04d87aa90-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.224 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:54:50.225 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.226 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.227 DEBUG:boto:Method: GET 2022-09-03T06:54:50.228 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:50.229 DEBUG:boto:Data: 2022-09-03T06:54:50.230 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.231 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.232 DEBUG:boto:Params: {} 2022-09-03T06:54:50.232 DEBUG:boto:Token: None 2022-09-03T06:54:50.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g 2022-09-03T06:54:50.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:60l0kBoe8I4XeIxC05pN2WBcgQo= 2022-09-03T06:54:50.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:60l0kBoe8I4XeIxC05pN2WBcgQo=', 'Content-Length': '0'} 2022-09-03T06:54:50.238 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052f1cf5737011178-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.278 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:50.280 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:50.281 DEBUG:boto:Method: GET 2022-09-03T06:54:50.282 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:50.283 DEBUG:boto:Data: 2022-09-03T06:54:50.284 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.285 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.286 DEBUG:boto:Params: {} 2022-09-03T06:54:50.288 DEBUG:boto:Token: None 2022-09-03T06:54:50.289 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/g 2022-09-03T06:54:50.290 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:60l0kBoe8I4XeIxC05pN2WBcgQo= 2022-09-03T06:54:50.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:60l0kBoe8I4XeIxC05pN2WBcgQo=', 'Content-Length': '0'} 2022-09-03T06:54:50.294 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a40144580e597eda-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.295 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.296 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.297 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:50.298 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:50.299 DEBUG:boto:Method: GET 2022-09-03T06:54:50.300 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:50.301 DEBUG:boto:Data: 2022-09-03T06:54:50.302 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.303 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.304 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.305 DEBUG:boto:Params: {} 2022-09-03T06:54:50.306 DEBUG:boto:Token: None 2022-09-03T06:54:50.307 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.314 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b312e8f870d4eb5-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.316 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.317 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.319 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.320 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:50.321 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:50.322 DEBUG:boto:Method: GET 2022-09-03T06:54:50.323 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:50.324 DEBUG:boto:Data: 2022-09-03T06:54:50.325 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.325 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.326 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.327 DEBUG:boto:Params: {} 2022-09-03T06:54:50.329 DEBUG:boto:Token: None 2022-09-03T06:54:50.330 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.332 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.335 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c678f9eec6e0a682-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.337 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.338 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:54:50.338 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.339 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.340 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.341 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.341 DEBUG:boto:Method: GET 2022-09-03T06:54:50.342 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:50.342 DEBUG:boto:Data: 2022-09-03T06:54:50.343 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.344 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.345 DEBUG:boto:Params: {} 2022-09-03T06:54:50.346 DEBUG:boto:Token: None 2022-09-03T06:54:50.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:50.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fn9wzosaw4Rabjy49nOC7oaPlaQ= 2022-09-03T06:54:50.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fn9wzosaw4Rabjy49nOC7oaPlaQ=', 'Content-Length': '0'} 2022-09-03T06:54:50.351 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dba9ac7fd254cece-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.352 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.352 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.353 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.354 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.354 DEBUG:boto:Method: GET 2022-09-03T06:54:50.355 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:50.355 DEBUG:boto:Data: 2022-09-03T06:54:50.356 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.357 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.358 DEBUG:boto:Params: {} 2022-09-03T06:54:50.358 DEBUG:boto:Token: None 2022-09-03T06:54:50.359 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:50.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fn9wzosaw4Rabjy49nOC7oaPlaQ= 2022-09-03T06:54:50.360 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fn9wzosaw4Rabjy49nOC7oaPlaQ=', 'Content-Length': '0'} 2022-09-03T06:54:50.363 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046f583b4255e7d69-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.406 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.407 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.408 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.408 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.409 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.409 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:50.410 DEBUG:boto:Data: 2022-09-03T06:54:50.410 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.411 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.412 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.412 DEBUG:boto:Params: {} 2022-09-03T06:54:50.413 DEBUG:boto:Token: None 2022-09-03T06:54:50.414 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:50.414 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q5NHbXR441+tjymLdhdhpQ8zgJQ= 2022-09-03T06:54:50.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q5NHbXR441+tjymLdhdhpQ8zgJQ=', 'Content-Length': '0'} 2022-09-03T06:54:50.417 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096c231bf7399c609-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.419 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.420 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.421 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.421 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.422 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.423 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:50.423 DEBUG:boto:Data: 2022-09-03T06:54:50.424 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.424 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.425 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.426 DEBUG:boto:Params: {} 2022-09-03T06:54:50.426 DEBUG:boto:Token: None 2022-09-03T06:54:50.427 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:50.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q5NHbXR441+tjymLdhdhpQ8zgJQ= 2022-09-03T06:54:50.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q5NHbXR441+tjymLdhdhpQ8zgJQ=', 'Content-Length': '0'} 2022-09-03T06:54:50.432 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac064072eccc37e2-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.433 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:54:50.433 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.434 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.434 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.435 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.436 DEBUG:boto:Method: GET 2022-09-03T06:54:50.436 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:50.437 DEBUG:boto:Data: 2022-09-03T06:54:50.437 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.438 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.439 DEBUG:boto:Params: {} 2022-09-03T06:54:50.440 DEBUG:boto:Token: None 2022-09-03T06:54:50.440 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:50.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fn9wzosaw4Rabjy49nOC7oaPlaQ= 2022-09-03T06:54:50.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fn9wzosaw4Rabjy49nOC7oaPlaQ=', 'Content-Length': '0'} 2022-09-03T06:54:50.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b497bdfb5a1763a1-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.445 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.445 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.446 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.446 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:50.447 DEBUG:boto:Method: GET 2022-09-03T06:54:50.447 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:50.448 DEBUG:boto:Data: 2022-09-03T06:54:50.448 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.450 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.450 DEBUG:boto:Params: {} 2022-09-03T06:54:50.451 DEBUG:boto:Token: None 2022-09-03T06:54:50.452 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:50.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fn9wzosaw4Rabjy49nOC7oaPlaQ= 2022-09-03T06:54:50.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fn9wzosaw4Rabjy49nOC7oaPlaQ=', 'Content-Length': '0'} 2022-09-03T06:54:50.455 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de90ec037ead784b-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.456 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.457 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.457 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:50.458 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:50.458 DEBUG:boto:Method: GET 2022-09-03T06:54:50.459 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:50.460 DEBUG:boto:Data: 2022-09-03T06:54:50.460 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.461 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.462 DEBUG:boto:Params: {} 2022-09-03T06:54:50.463 DEBUG:boto:Token: None 2022-09-03T06:54:50.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f9d8a01e12aa54f-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.468 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.469 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.470 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.470 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:50.471 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:50.472 DEBUG:boto:Method: GET 2022-09-03T06:54:50.472 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:50.473 DEBUG:boto:Data: 2022-09-03T06:54:50.473 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.474 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.474 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.475 DEBUG:boto:Params: {} 2022-09-03T06:54:50.475 DEBUG:boto:Token: None 2022-09-03T06:54:50.476 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.480 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000061382bdf58b1776c-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.481 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.482 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.482 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.483 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.484 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:50.484 DEBUG:boto:Data: 2022-09-03T06:54:50.485 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.486 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.487 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.487 DEBUG:boto:Params: {} 2022-09-03T06:54:50.488 DEBUG:boto:Token: None 2022-09-03T06:54:50.488 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h 2022-09-03T06:54:50.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oWIlT/s3ww2eIzfZs3oNGJc5h7w= 2022-09-03T06:54:50.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oWIlT/s3ww2eIzfZs3oNGJc5h7w=', 'Content-Length': '0'} 2022-09-03T06:54:50.492 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000094c6350bee72dd9d-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.493 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.494 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.494 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.495 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:50.495 DEBUG:boto:Data: 2022-09-03T06:54:50.496 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.497 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.498 DEBUG:boto:Params: {} 2022-09-03T06:54:50.498 DEBUG:boto:Token: None 2022-09-03T06:54:50.499 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h 2022-09-03T06:54:50.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oWIlT/s3ww2eIzfZs3oNGJc5h7w= 2022-09-03T06:54:50.500 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oWIlT/s3ww2eIzfZs3oNGJc5h7w=', 'Content-Length': '0'} 2022-09-03T06:54:50.505 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000380de5e3d0a8dcb3-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.507 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:54:50.508 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.509 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.509 DEBUG:boto:Method: GET 2022-09-03T06:54:50.510 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:50.511 DEBUG:boto:Data: 2022-09-03T06:54:50.511 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.512 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.513 DEBUG:boto:Params: {} 2022-09-03T06:54:50.513 DEBUG:boto:Token: None 2022-09-03T06:54:50.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h 2022-09-03T06:54:50.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TogBENRu/RanJr6TCmbNQG/SEHs= 2022-09-03T06:54:50.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TogBENRu/RanJr6TCmbNQG/SEHs=', 'Content-Length': '0'} 2022-09-03T06:54:50.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa49a407683bfc87-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.522 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:50.523 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:50.524 DEBUG:boto:Method: GET 2022-09-03T06:54:50.525 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:50.527 DEBUG:boto:Data: 2022-09-03T06:54:50.528 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.530 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.531 DEBUG:boto:Params: {} 2022-09-03T06:54:50.532 DEBUG:boto:Token: None 2022-09-03T06:54:50.533 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/h 2022-09-03T06:54:50.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TogBENRu/RanJr6TCmbNQG/SEHs= 2022-09-03T06:54:50.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TogBENRu/RanJr6TCmbNQG/SEHs=', 'Content-Length': '0'} 2022-09-03T06:54:50.539 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e574c7fa161a79c3-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.541 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.542 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.543 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:50.544 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:50.545 DEBUG:boto:Method: GET 2022-09-03T06:54:50.546 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:50.547 DEBUG:boto:Data: 2022-09-03T06:54:50.548 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.549 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.550 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.551 DEBUG:boto:Params: {} 2022-09-03T06:54:50.552 DEBUG:boto:Token: None 2022-09-03T06:54:50.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.565 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000026c671f941b0080b-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.566 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.567 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.568 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.568 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:50.570 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:50.571 DEBUG:boto:Method: GET 2022-09-03T06:54:50.571 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:50.572 DEBUG:boto:Data: 2022-09-03T06:54:50.573 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.574 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:54:50.575 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.576 DEBUG:boto:Params: {} 2022-09-03T06:54:50.577 DEBUG:boto:Token: None 2022-09-03T06:54:50.578 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.582 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f0d1a24ce2aa1a69-006312fa3a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.583 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.584 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-71 zones={test-zone3, test-zone1} 2022-09-03T06:54:50.584 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-71 zones={test-zone3, test-zone2} 2022-09-03T06:54:50.585 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.585 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.586 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.587 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:54:50.588 DEBUG:boto:Data: 2022-09-03T06:54:50.589 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.590 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.590 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.591 DEBUG:boto:Params: {} 2022-09-03T06:54:50.592 DEBUG:boto:Token: None 2022-09-03T06:54:50.593 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tHJqtFa9+Oprb+8AD0gv0cgXi0o= 2022-09-03T06:54:50.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tHJqtFa9+Oprb+8AD0gv0cgXi0o=', 'Content-Length': '0'} 2022-09-03T06:54:50.597 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000301b9f0c0a615c0-006312fa3a-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.597 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.600 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.601 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.602 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:54:50.603 DEBUG:boto:Data: 2022-09-03T06:54:50.604 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:50.605 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.605 DEBUG:boto:Params: {} 2022-09-03T06:54:50.606 DEBUG:boto:Token: None 2022-09-03T06:54:50.607 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tHJqtFa9+Oprb+8AD0gv0cgXi0o= 2022-09-03T06:54:50.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tHJqtFa9+Oprb+8AD0gv0cgXi0o=', 'Content-Length': '0'} 2022-09-03T06:54:50.612 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c85831618ac688ac-006312fa3a-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.613 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:54:50.613 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.614 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.615 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.615 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.616 DEBUG:boto:Method: GET 2022-09-03T06:54:50.616 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.617 DEBUG:boto:Data: 2022-09-03T06:54:50.617 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.618 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.619 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.620 DEBUG:boto:Params: {} 2022-09-03T06:54:50.620 DEBUG:boto:Token: None 2022-09-03T06:54:50.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/?versions 2022-09-03T06:54:50.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3MxKCSgqvwGvKjUdi7OdhxQ9Ra8= 2022-09-03T06:54:50.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3MxKCSgqvwGvKjUdi7OdhxQ9Ra8=', 'Content-Length': '0'} 2022-09-03T06:54:50.626 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6ef9bb67df42f16-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.627 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.629 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:54:50.629 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:54:50.630 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:54:50.630 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:54:50.631 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:54:50.631 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:54:50.632 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:54:50.632 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:54:50.633 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:54:50.633 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.634 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.635 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.635 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.636 DEBUG:boto:Method: GET 2022-09-03T06:54:50.636 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.637 DEBUG:boto:Data: 2022-09-03T06:54:50.637 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.639 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:50.639 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.640 DEBUG:boto:Params: {} 2022-09-03T06:54:50.640 DEBUG:boto:Token: None 2022-09-03T06:54:50.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/?versions 2022-09-03T06:54:50.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3MxKCSgqvwGvKjUdi7OdhxQ9Ra8= 2022-09-03T06:54:50.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3MxKCSgqvwGvKjUdi7OdhxQ9Ra8=', 'Content-Length': '0'} 2022-09-03T06:54:50.646 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b4bee5fa6ee0707f-006312fa3a-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.647 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.649 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:54:50.649 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:54:50.650 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:54:50.651 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:54:50.651 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:54:50.652 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:54:50.652 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:54:50.653 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:54:50.654 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.654 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.655 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.656 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.657 DEBUG:boto:Method: GET 2022-09-03T06:54:50.658 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.659 DEBUG:boto:Data: 2022-09-03T06:54:50.660 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.662 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.662 DEBUG:boto:Params: {} 2022-09-03T06:54:50.663 DEBUG:boto:Token: None 2022-09-03T06:54:50.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/?versions 2022-09-03T06:54:50.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3MxKCSgqvwGvKjUdi7OdhxQ9Ra8= 2022-09-03T06:54:50.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3MxKCSgqvwGvKjUdi7OdhxQ9Ra8=', 'Content-Length': '0'} 2022-09-03T06:54:50.669 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008877abc62f8050c1-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.670 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.672 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.673 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.673 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.674 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.674 DEBUG:boto:Method: GET 2022-09-03T06:54:50.675 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:54:50.676 DEBUG:boto:Data: 2022-09-03T06:54:50.676 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.677 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:50.678 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.678 DEBUG:boto:Params: {} 2022-09-03T06:54:50.679 DEBUG:boto:Token: None 2022-09-03T06:54:50.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/?versions 2022-09-03T06:54:50.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3MxKCSgqvwGvKjUdi7OdhxQ9Ra8= 2022-09-03T06:54:50.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3MxKCSgqvwGvKjUdi7OdhxQ9Ra8=', 'Content-Length': '0'} 2022-09-03T06:54:50.684 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b966813f0bc76492-006312fa3a-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.685 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.687 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:54:50.688 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.689 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.691 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.692 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.695 DEBUG:boto:Method: GET 2022-09-03T06:54:50.696 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:50.697 DEBUG:boto:Data: 2022-09-03T06:54:50.697 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.700 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.701 DEBUG:boto:Params: {} 2022-09-03T06:54:50.702 DEBUG:boto:Token: None 2022-09-03T06:54:50.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:50.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:viZ4knnRhxAxvpI2HVRPfy3Tvkw= 2022-09-03T06:54:50.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:viZ4knnRhxAxvpI2HVRPfy3Tvkw=', 'Content-Length': '0'} 2022-09-03T06:54:50.710 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a27675b7947da160-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.758 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.759 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.760 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.760 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.761 DEBUG:boto:Method: GET 2022-09-03T06:54:50.762 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:50.762 DEBUG:boto:Data: 2022-09-03T06:54:50.763 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.764 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:50.765 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.766 DEBUG:boto:Params: {} 2022-09-03T06:54:50.767 DEBUG:boto:Token: None 2022-09-03T06:54:50.768 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:50.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:viZ4knnRhxAxvpI2HVRPfy3Tvkw= 2022-09-03T06:54:50.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:viZ4knnRhxAxvpI2HVRPfy3Tvkw=', 'Content-Length': '0'} 2022-09-03T06:54:50.773 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000704509f7c8108517-006312fa3a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.775 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.776 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.777 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.777 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.778 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.779 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:50.779 DEBUG:boto:Data: 2022-09-03T06:54:50.780 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.781 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.781 DEBUG:boto:Params: {} 2022-09-03T06:54:50.782 DEBUG:boto:Token: None 2022-09-03T06:54:50.783 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:50.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TwQhCyg0aQLEioKzZ7PFbj8XerA= 2022-09-03T06:54:50.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TwQhCyg0aQLEioKzZ7PFbj8XerA=', 'Content-Length': '0'} 2022-09-03T06:54:50.786 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c52d532730ac663-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.787 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.788 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.788 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.789 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.789 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.790 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:50.791 DEBUG:boto:Data: 2022-09-03T06:54:50.791 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.792 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:50.793 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.794 DEBUG:boto:Params: {} 2022-09-03T06:54:50.795 DEBUG:boto:Token: None 2022-09-03T06:54:50.796 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:50.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TwQhCyg0aQLEioKzZ7PFbj8XerA= 2022-09-03T06:54:50.798 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TwQhCyg0aQLEioKzZ7PFbj8XerA=', 'Content-Length': '0'} 2022-09-03T06:54:50.801 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae17c8c07e9d48e8-006312fa3a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.802 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:54:50.803 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.804 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.805 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.806 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.807 DEBUG:boto:Method: GET 2022-09-03T06:54:50.808 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:50.809 DEBUG:boto:Data: 2022-09-03T06:54:50.810 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.811 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.812 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.813 DEBUG:boto:Params: {} 2022-09-03T06:54:50.814 DEBUG:boto:Token: None 2022-09-03T06:54:50.814 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:50.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:viZ4knnRhxAxvpI2HVRPfy3Tvkw= 2022-09-03T06:54:50.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:viZ4knnRhxAxvpI2HVRPfy3Tvkw=', 'Content-Length': '0'} 2022-09-03T06:54:50.818 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000272a971b1da32a46-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.819 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.820 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.821 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.821 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:54:50.822 DEBUG:boto:Method: GET 2022-09-03T06:54:50.823 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:54:50.824 DEBUG:boto:Data: 2022-09-03T06:54:50.825 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.826 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:50.827 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.827 DEBUG:boto:Params: {} 2022-09-03T06:54:50.828 DEBUG:boto:Token: None 2022-09-03T06:54:50.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a?versionId=null 2022-09-03T06:54:50.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:viZ4knnRhxAxvpI2HVRPfy3Tvkw= 2022-09-03T06:54:50.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:viZ4knnRhxAxvpI2HVRPfy3Tvkw=', 'Content-Length': '0'} 2022-09-03T06:54:50.835 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e16255227807245-006312fa3a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.837 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.838 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.839 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:50.840 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:50.841 DEBUG:boto:Method: GET 2022-09-03T06:54:50.842 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:50.842 DEBUG:boto:Data: 2022-09-03T06:54:50.843 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.844 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.845 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.846 DEBUG:boto:Params: {} 2022-09-03T06:54:50.847 DEBUG:boto:Token: None 2022-09-03T06:54:50.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.857 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000496b9b5b499bcb35-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.898 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.900 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.901 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.902 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:50.903 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:50.904 DEBUG:boto:Method: GET 2022-09-03T06:54:50.905 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:50.905 DEBUG:boto:Data: 2022-09-03T06:54:50.906 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.907 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:50.908 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.908 DEBUG:boto:Params: {} 2022-09-03T06:54:50.910 DEBUG:boto:Token: None 2022-09-03T06:54:50.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.916 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058fad54713a65d63-006312fa3a-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.917 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:50.918 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.918 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.919 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.919 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:50.920 DEBUG:boto:Data: 2022-09-03T06:54:50.920 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.922 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.923 DEBUG:boto:Params: {} 2022-09-03T06:54:50.923 DEBUG:boto:Token: None 2022-09-03T06:54:50.924 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a 2022-09-03T06:54:50.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vZHca0bNWufpt2rAw+lqoy0U6LM= 2022-09-03T06:54:50.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vZHca0bNWufpt2rAw+lqoy0U6LM=', 'Content-Length': '0'} 2022-09-03T06:54:50.927 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023706a94c5097412-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.928 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.928 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.929 DEBUG:boto:Method: HEAD 2022-09-03T06:54:50.929 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:50.930 DEBUG:boto:Data: 2022-09-03T06:54:50.930 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.931 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:50.931 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.932 DEBUG:boto:Params: {} 2022-09-03T06:54:50.933 DEBUG:boto:Token: None 2022-09-03T06:54:50.934 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a 2022-09-03T06:54:50.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vZHca0bNWufpt2rAw+lqoy0U6LM= 2022-09-03T06:54:50.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vZHca0bNWufpt2rAw+lqoy0U6LM=', 'Content-Length': '0'} 2022-09-03T06:54:50.937 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d5cf4e4ee3027ba3-006312fa3a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.938 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:54:50.938 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.941 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.942 DEBUG:boto:Method: GET 2022-09-03T06:54:50.942 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:50.943 DEBUG:boto:Data: 2022-09-03T06:54:50.944 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.945 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.946 DEBUG:boto:Params: {} 2022-09-03T06:54:50.947 DEBUG:boto:Token: None 2022-09-03T06:54:50.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a 2022-09-03T06:54:50.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PCn567aXxbNSMWA61j0iqJTbJkk= 2022-09-03T06:54:50.952 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PCn567aXxbNSMWA61j0iqJTbJkk=', 'Content-Length': '0'} 2022-09-03T06:54:50.957 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034882143585fe131-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.960 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:54:50.961 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:54:50.962 DEBUG:boto:Method: GET 2022-09-03T06:54:50.964 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:54:50.965 DEBUG:boto:Data: 2022-09-03T06:54:50.967 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.968 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:50.969 DEBUG:boto:Port: 8000 2022-09-03T06:54:50.970 DEBUG:boto:Params: {} 2022-09-03T06:54:50.971 DEBUG:boto:Token: None 2022-09-03T06:54:50.973 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/a 2022-09-03T06:54:50.974 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PCn567aXxbNSMWA61j0iqJTbJkk= 2022-09-03T06:54:50.975 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PCn567aXxbNSMWA61j0iqJTbJkk=', 'Content-Length': '0'} 2022-09-03T06:54:50.979 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070273cd4ff7a4fda-006312fa3a-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.981 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:50.982 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:50.984 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:50.984 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:50.985 DEBUG:boto:Method: GET 2022-09-03T06:54:50.986 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:50.986 DEBUG:boto:Data: 2022-09-03T06:54:50.987 DEBUG:boto:Headers: {} 2022-09-03T06:54:50.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:50.988 DEBUG:boto:Port: 8001 2022-09-03T06:54:50.989 DEBUG:boto:Params: {} 2022-09-03T06:54:50.991 DEBUG:boto:Token: None 2022-09-03T06:54:50.992 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:50 GMT /avysmw-71/ 2022-09-03T06:54:50.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA= 2022-09-03T06:54:50.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZhgC+/ULeP5xl/Ea2+GWZlpmEA=', 'Content-Length': '0'} 2022-09-03T06:54:50.998 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002f02e83e93afd115-006312fa3a-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:50.999 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.001 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.002 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.003 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:54:51.004 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:54:51.005 DEBUG:boto:Method: GET 2022-09-03T06:54:51.006 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:54:51.007 DEBUG:boto:Data: 2022-09-03T06:54:51.008 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.009 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.010 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.011 DEBUG:boto:Params: {} 2022-09-03T06:54:51.012 DEBUG:boto:Token: None 2022-09-03T06:54:51.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.018 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000094627393c59c8887-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.066 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.068 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:54:51.069 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.069 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.070 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.071 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.071 DEBUG:boto:Method: GET 2022-09-03T06:54:51.072 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:51.073 DEBUG:boto:Data: 2022-09-03T06:54:51.073 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.074 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.074 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.075 DEBUG:boto:Params: {} 2022-09-03T06:54:51.076 DEBUG:boto:Token: None 2022-09-03T06:54:51.076 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:51.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JtUM3Avxsnemk/7OPfYBmP7/PSs= 2022-09-03T06:54:51.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JtUM3Avxsnemk/7OPfYBmP7/PSs=', 'Content-Length': '0'} 2022-09-03T06:54:51.081 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e60ec3340e4bd53-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.082 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.082 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.083 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.083 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.084 DEBUG:boto:Method: GET 2022-09-03T06:54:51.085 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:51.086 DEBUG:boto:Data: 2022-09-03T06:54:51.086 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.087 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.087 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.088 DEBUG:boto:Params: {} 2022-09-03T06:54:51.089 DEBUG:boto:Token: None 2022-09-03T06:54:51.089 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:51.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JtUM3Avxsnemk/7OPfYBmP7/PSs= 2022-09-03T06:54:51.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JtUM3Avxsnemk/7OPfYBmP7/PSs=', 'Content-Length': '0'} 2022-09-03T06:54:51.092 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007327871f0393aa52-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.094 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.094 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.095 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.095 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.096 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.096 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:51.097 DEBUG:boto:Data: 2022-09-03T06:54:51.098 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.098 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.099 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.099 DEBUG:boto:Params: {} 2022-09-03T06:54:51.100 DEBUG:boto:Token: None 2022-09-03T06:54:51.101 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:51.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66HjNIrBs1I2yZVKR8CXLdKNwPI= 2022-09-03T06:54:51.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66HjNIrBs1I2yZVKR8CXLdKNwPI=', 'Content-Length': '0'} 2022-09-03T06:54:51.105 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f2bec6764ab7cce2-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.106 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.106 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.107 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.107 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.108 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.108 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:51.109 DEBUG:boto:Data: 2022-09-03T06:54:51.109 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.110 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.110 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.111 DEBUG:boto:Params: {} 2022-09-03T06:54:51.112 DEBUG:boto:Token: None 2022-09-03T06:54:51.112 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:51.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66HjNIrBs1I2yZVKR8CXLdKNwPI= 2022-09-03T06:54:51.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66HjNIrBs1I2yZVKR8CXLdKNwPI=', 'Content-Length': '0'} 2022-09-03T06:54:51.116 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054c4ab8ba4a3565e-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.117 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:54:51.118 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.118 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.119 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.120 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.120 DEBUG:boto:Method: GET 2022-09-03T06:54:51.121 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:51.121 DEBUG:boto:Data: 2022-09-03T06:54:51.122 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.124 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.124 DEBUG:boto:Params: {} 2022-09-03T06:54:51.126 DEBUG:boto:Token: None 2022-09-03T06:54:51.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:51.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JtUM3Avxsnemk/7OPfYBmP7/PSs= 2022-09-03T06:54:51.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JtUM3Avxsnemk/7OPfYBmP7/PSs=', 'Content-Length': '0'} 2022-09-03T06:54:51.131 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b86583b93fc2ca74-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.132 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.133 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.133 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.134 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:54:51.135 DEBUG:boto:Method: GET 2022-09-03T06:54:51.136 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:54:51.137 DEBUG:boto:Data: 2022-09-03T06:54:51.137 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.138 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.139 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.139 DEBUG:boto:Params: {} 2022-09-03T06:54:51.140 DEBUG:boto:Token: None 2022-09-03T06:54:51.140 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b?versionId=null 2022-09-03T06:54:51.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JtUM3Avxsnemk/7OPfYBmP7/PSs= 2022-09-03T06:54:51.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JtUM3Avxsnemk/7OPfYBmP7/PSs=', 'Content-Length': '0'} 2022-09-03T06:54:51.144 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060cec0a8c85b95bf-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.145 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.145 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.146 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:51.147 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:51.147 DEBUG:boto:Method: GET 2022-09-03T06:54:51.148 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:51.148 DEBUG:boto:Data: 2022-09-03T06:54:51.149 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.150 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.151 DEBUG:boto:Params: {} 2022-09-03T06:54:51.151 DEBUG:boto:Token: None 2022-09-03T06:54:51.152 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.152 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.156 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f8b8c82273ede3c-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.157 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.158 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.159 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.159 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:51.160 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:51.160 DEBUG:boto:Method: GET 2022-09-03T06:54:51.161 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:51.161 DEBUG:boto:Data: 2022-09-03T06:54:51.162 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.162 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.163 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.163 DEBUG:boto:Params: {} 2022-09-03T06:54:51.164 DEBUG:boto:Token: None 2022-09-03T06:54:51.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.168 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fbc1c13b6be72212-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.169 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.170 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.170 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.171 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.171 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:51.172 DEBUG:boto:Data: 2022-09-03T06:54:51.172 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.173 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.173 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.174 DEBUG:boto:Params: {} 2022-09-03T06:54:51.174 DEBUG:boto:Token: None 2022-09-03T06:54:51.175 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b 2022-09-03T06:54:51.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uMhjWYfTainBPWZVIiNTIbLj/5g= 2022-09-03T06:54:51.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uMhjWYfTainBPWZVIiNTIbLj/5g=', 'Content-Length': '0'} 2022-09-03T06:54:51.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af51408ac186488d-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.179 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.179 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.180 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.180 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:51.181 DEBUG:boto:Data: 2022-09-03T06:54:51.181 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.182 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.182 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.183 DEBUG:boto:Params: {} 2022-09-03T06:54:51.183 DEBUG:boto:Token: None 2022-09-03T06:54:51.184 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b 2022-09-03T06:54:51.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uMhjWYfTainBPWZVIiNTIbLj/5g= 2022-09-03T06:54:51.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uMhjWYfTainBPWZVIiNTIbLj/5g=', 'Content-Length': '0'} 2022-09-03T06:54:51.187 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ab02f4f5f24bf19-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.188 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:54:51.189 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.190 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.190 DEBUG:boto:Method: GET 2022-09-03T06:54:51.191 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:51.191 DEBUG:boto:Data: 2022-09-03T06:54:51.192 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.193 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.193 DEBUG:boto:Params: {} 2022-09-03T06:54:51.194 DEBUG:boto:Token: None 2022-09-03T06:54:51.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b 2022-09-03T06:54:51.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Sky2lygxv5QOaDx/OlQGvue+9s= 2022-09-03T06:54:51.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Sky2lygxv5QOaDx/OlQGvue+9s=', 'Content-Length': '0'} 2022-09-03T06:54:51.198 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f67cc1dd1761530-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.246 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:54:51.247 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:54:51.247 DEBUG:boto:Method: GET 2022-09-03T06:54:51.248 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:54:51.248 DEBUG:boto:Data: 2022-09-03T06:54:51.249 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.249 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.250 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.250 DEBUG:boto:Params: {} 2022-09-03T06:54:51.251 DEBUG:boto:Token: None 2022-09-03T06:54:51.252 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/b 2022-09-03T06:54:51.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Sky2lygxv5QOaDx/OlQGvue+9s= 2022-09-03T06:54:51.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Sky2lygxv5QOaDx/OlQGvue+9s=', 'Content-Length': '0'} 2022-09-03T06:54:51.255 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6070f637aedc949-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.256 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.257 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.257 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:51.258 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:51.258 DEBUG:boto:Method: GET 2022-09-03T06:54:51.259 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:51.259 DEBUG:boto:Data: 2022-09-03T06:54:51.260 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.261 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.261 DEBUG:boto:Params: {} 2022-09-03T06:54:51.262 DEBUG:boto:Token: None 2022-09-03T06:54:51.263 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.267 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a89816150f45db31-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.268 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.269 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.270 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.271 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:54:51.273 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:54:51.274 DEBUG:boto:Method: GET 2022-09-03T06:54:51.275 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:54:51.276 DEBUG:boto:Data: 2022-09-03T06:54:51.277 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.278 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.278 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.279 DEBUG:boto:Params: {} 2022-09-03T06:54:51.279 DEBUG:boto:Token: None 2022-09-03T06:54:51.280 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.284 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b738c29581fd88d7-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.284 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.285 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:54:51.286 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.286 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.287 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.287 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.288 DEBUG:boto:Method: GET 2022-09-03T06:54:51.288 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:51.289 DEBUG:boto:Data: 2022-09-03T06:54:51.289 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.290 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.290 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.291 DEBUG:boto:Params: {} 2022-09-03T06:54:51.292 DEBUG:boto:Token: None 2022-09-03T06:54:51.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:51.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fRoijKSX3YAw8b534Dql0hCSYcE= 2022-09-03T06:54:51.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fRoijKSX3YAw8b534Dql0hCSYcE=', 'Content-Length': '0'} 2022-09-03T06:54:51.296 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eaa100b535ea961f-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.297 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.297 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.298 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.298 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.299 DEBUG:boto:Method: GET 2022-09-03T06:54:51.299 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:51.300 DEBUG:boto:Data: 2022-09-03T06:54:51.300 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.301 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.301 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.302 DEBUG:boto:Params: {} 2022-09-03T06:54:51.303 DEBUG:boto:Token: None 2022-09-03T06:54:51.303 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:51.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fRoijKSX3YAw8b534Dql0hCSYcE= 2022-09-03T06:54:51.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fRoijKSX3YAw8b534Dql0hCSYcE=', 'Content-Length': '0'} 2022-09-03T06:54:51.307 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a64491eb268026aa-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.307 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.308 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.308 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.309 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.310 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.310 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:51.311 DEBUG:boto:Data: 2022-09-03T06:54:51.311 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.312 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.313 DEBUG:boto:Params: {} 2022-09-03T06:54:51.314 DEBUG:boto:Token: None 2022-09-03T06:54:51.315 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:51.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V+eYkBpKmSfgt5ClrPUKvFG+kVY= 2022-09-03T06:54:51.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V+eYkBpKmSfgt5ClrPUKvFG+kVY=', 'Content-Length': '0'} 2022-09-03T06:54:51.319 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052bef6e3238ae216-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.320 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.321 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.321 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.322 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.322 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.323 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:51.323 DEBUG:boto:Data: 2022-09-03T06:54:51.324 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.324 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.325 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.325 DEBUG:boto:Params: {} 2022-09-03T06:54:51.326 DEBUG:boto:Token: None 2022-09-03T06:54:51.327 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:51.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V+eYkBpKmSfgt5ClrPUKvFG+kVY= 2022-09-03T06:54:51.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V+eYkBpKmSfgt5ClrPUKvFG+kVY=', 'Content-Length': '0'} 2022-09-03T06:54:51.331 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba74bafa216f085b-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.332 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:54:51.332 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.333 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.334 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.334 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.335 DEBUG:boto:Method: GET 2022-09-03T06:54:51.335 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:51.336 DEBUG:boto:Data: 2022-09-03T06:54:51.336 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.337 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.338 DEBUG:boto:Params: {} 2022-09-03T06:54:51.338 DEBUG:boto:Token: None 2022-09-03T06:54:51.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:51.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fRoijKSX3YAw8b534Dql0hCSYcE= 2022-09-03T06:54:51.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fRoijKSX3YAw8b534Dql0hCSYcE=', 'Content-Length': '0'} 2022-09-03T06:54:51.343 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a13e5b4584d78ba-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.344 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.344 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.345 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.345 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:54:51.346 DEBUG:boto:Method: GET 2022-09-03T06:54:51.347 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:54:51.347 DEBUG:boto:Data: 2022-09-03T06:54:51.348 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.349 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.349 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.350 DEBUG:boto:Params: {} 2022-09-03T06:54:51.351 DEBUG:boto:Token: None 2022-09-03T06:54:51.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c?versionId=null 2022-09-03T06:54:51.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fRoijKSX3YAw8b534Dql0hCSYcE= 2022-09-03T06:54:51.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fRoijKSX3YAw8b534Dql0hCSYcE=', 'Content-Length': '0'} 2022-09-03T06:54:51.355 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de9d9957ff09d045-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.368 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.370 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.370 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:51.371 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:51.371 DEBUG:boto:Method: GET 2022-09-03T06:54:51.372 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:51.372 DEBUG:boto:Data: 2022-09-03T06:54:51.373 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.374 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.374 DEBUG:boto:Params: {} 2022-09-03T06:54:51.375 DEBUG:boto:Token: None 2022-09-03T06:54:51.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.380 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e3e7386332c9fa5-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.423 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.424 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.424 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.425 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:51.425 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:51.426 DEBUG:boto:Method: GET 2022-09-03T06:54:51.426 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:51.427 DEBUG:boto:Data: 2022-09-03T06:54:51.427 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.428 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.429 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.429 DEBUG:boto:Params: {} 2022-09-03T06:54:51.430 DEBUG:boto:Token: None 2022-09-03T06:54:51.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.434 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000092795916eface76f-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.435 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.436 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.436 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.437 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.438 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:51.438 DEBUG:boto:Data: 2022-09-03T06:54:51.439 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.440 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.440 DEBUG:boto:Params: {} 2022-09-03T06:54:51.441 DEBUG:boto:Token: None 2022-09-03T06:54:51.441 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c 2022-09-03T06:54:51.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QyxAb1xljwEyH4uIMCHfjrYUiuU= 2022-09-03T06:54:51.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QyxAb1xljwEyH4uIMCHfjrYUiuU=', 'Content-Length': '0'} 2022-09-03T06:54:51.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b86630c4988170a2-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.445 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.446 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.446 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.447 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:51.447 DEBUG:boto:Data: 2022-09-03T06:54:51.448 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.449 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.449 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.450 DEBUG:boto:Params: {} 2022-09-03T06:54:51.450 DEBUG:boto:Token: None 2022-09-03T06:54:51.451 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c 2022-09-03T06:54:51.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QyxAb1xljwEyH4uIMCHfjrYUiuU= 2022-09-03T06:54:51.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QyxAb1xljwEyH4uIMCHfjrYUiuU=', 'Content-Length': '0'} 2022-09-03T06:54:51.454 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007295c19d415f4dce-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.455 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:54:51.455 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.456 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.456 DEBUG:boto:Method: GET 2022-09-03T06:54:51.457 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:51.458 DEBUG:boto:Data: 2022-09-03T06:54:51.458 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.459 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.459 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.460 DEBUG:boto:Params: {} 2022-09-03T06:54:51.461 DEBUG:boto:Token: None 2022-09-03T06:54:51.461 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c 2022-09-03T06:54:51.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f7Ck/3c8y8MCLgHHupEjpik38UU= 2022-09-03T06:54:51.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f7Ck/3c8y8MCLgHHupEjpik38UU=', 'Content-Length': '0'} 2022-09-03T06:54:51.465 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e123b1a0d83a32f6-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.466 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:54:51.468 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:54:51.468 DEBUG:boto:Method: GET 2022-09-03T06:54:51.469 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:54:51.469 DEBUG:boto:Data: 2022-09-03T06:54:51.470 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.471 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.471 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.472 DEBUG:boto:Params: {} 2022-09-03T06:54:51.472 DEBUG:boto:Token: None 2022-09-03T06:54:51.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/c 2022-09-03T06:54:51.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f7Ck/3c8y8MCLgHHupEjpik38UU= 2022-09-03T06:54:51.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f7Ck/3c8y8MCLgHHupEjpik38UU=', 'Content-Length': '0'} 2022-09-03T06:54:51.476 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004892ce801a1f95d-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.518 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.519 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.520 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:51.520 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:51.521 DEBUG:boto:Method: GET 2022-09-03T06:54:51.521 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:51.522 DEBUG:boto:Data: 2022-09-03T06:54:51.523 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.523 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.524 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.524 DEBUG:boto:Params: {} 2022-09-03T06:54:51.526 DEBUG:boto:Token: None 2022-09-03T06:54:51.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.529 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.532 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007f03e17fb3857987-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.532 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.533 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.534 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.535 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:54:51.535 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:54:51.536 DEBUG:boto:Method: GET 2022-09-03T06:54:51.536 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:54:51.537 DEBUG:boto:Data: 2022-09-03T06:54:51.537 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.538 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.538 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.539 DEBUG:boto:Params: {} 2022-09-03T06:54:51.539 DEBUG:boto:Token: None 2022-09-03T06:54:51.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.544 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a7e893e88ced7420-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.545 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.546 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:54:51.547 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.547 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.548 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.548 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.549 DEBUG:boto:Method: GET 2022-09-03T06:54:51.549 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:51.550 DEBUG:boto:Data: 2022-09-03T06:54:51.551 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.552 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.553 DEBUG:boto:Params: {} 2022-09-03T06:54:51.553 DEBUG:boto:Token: None 2022-09-03T06:54:51.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:51.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IsrXoGzlZ6KkvTxldZ68nIIX8Mg= 2022-09-03T06:54:51.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IsrXoGzlZ6KkvTxldZ68nIIX8Mg=', 'Content-Length': '0'} 2022-09-03T06:54:51.558 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000978173f089c1fbf7-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.559 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.560 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.560 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.561 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.562 DEBUG:boto:Method: GET 2022-09-03T06:54:51.562 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:51.563 DEBUG:boto:Data: 2022-09-03T06:54:51.564 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.564 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.565 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.565 DEBUG:boto:Params: {} 2022-09-03T06:54:51.567 DEBUG:boto:Token: None 2022-09-03T06:54:51.568 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:51.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IsrXoGzlZ6KkvTxldZ68nIIX8Mg= 2022-09-03T06:54:51.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IsrXoGzlZ6KkvTxldZ68nIIX8Mg=', 'Content-Length': '0'} 2022-09-03T06:54:51.571 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f8cba05c9defc338-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.572 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.573 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.574 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.574 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.575 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.575 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:51.576 DEBUG:boto:Data: 2022-09-03T06:54:51.576 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.578 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.579 DEBUG:boto:Params: {} 2022-09-03T06:54:51.580 DEBUG:boto:Token: None 2022-09-03T06:54:51.580 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:51.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s2YzG06Hk+RIhB0mDI6S7tsg8M8= 2022-09-03T06:54:51.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s2YzG06Hk+RIhB0mDI6S7tsg8M8=', 'Content-Length': '0'} 2022-09-03T06:54:51.584 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003cda85ea69b27700-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.585 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.586 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.587 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.587 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.588 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.589 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:51.589 DEBUG:boto:Data: 2022-09-03T06:54:51.590 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.590 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.591 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.592 DEBUG:boto:Params: {} 2022-09-03T06:54:51.592 DEBUG:boto:Token: None 2022-09-03T06:54:51.593 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:51.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s2YzG06Hk+RIhB0mDI6S7tsg8M8= 2022-09-03T06:54:51.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s2YzG06Hk+RIhB0mDI6S7tsg8M8=', 'Content-Length': '0'} 2022-09-03T06:54:51.597 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045d2971df23556e3-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.598 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:54:51.599 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.599 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.600 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.601 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.602 DEBUG:boto:Method: GET 2022-09-03T06:54:51.603 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:51.603 DEBUG:boto:Data: 2022-09-03T06:54:51.604 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.605 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.605 DEBUG:boto:Params: {} 2022-09-03T06:54:51.606 DEBUG:boto:Token: None 2022-09-03T06:54:51.606 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:51.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IsrXoGzlZ6KkvTxldZ68nIIX8Mg= 2022-09-03T06:54:51.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IsrXoGzlZ6KkvTxldZ68nIIX8Mg=', 'Content-Length': '0'} 2022-09-03T06:54:51.611 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd2092efd69e188f-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.654 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.655 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.656 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.657 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:54:51.658 DEBUG:boto:Method: GET 2022-09-03T06:54:51.659 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:54:51.661 DEBUG:boto:Data: 2022-09-03T06:54:51.662 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.662 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.663 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.663 DEBUG:boto:Params: {} 2022-09-03T06:54:51.664 DEBUG:boto:Token: None 2022-09-03T06:54:51.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d?versionId=null 2022-09-03T06:54:51.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IsrXoGzlZ6KkvTxldZ68nIIX8Mg= 2022-09-03T06:54:51.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IsrXoGzlZ6KkvTxldZ68nIIX8Mg=', 'Content-Length': '0'} 2022-09-03T06:54:51.668 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007b7dbd390a2ea463-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.669 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.670 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.670 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:51.671 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:51.671 DEBUG:boto:Method: GET 2022-09-03T06:54:51.672 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:51.672 DEBUG:boto:Data: 2022-09-03T06:54:51.673 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.673 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.674 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.674 DEBUG:boto:Params: {} 2022-09-03T06:54:51.675 DEBUG:boto:Token: None 2022-09-03T06:54:51.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.679 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b5817db33ac3f7c1-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.680 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.681 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.682 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.682 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:51.683 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:51.683 DEBUG:boto:Method: GET 2022-09-03T06:54:51.684 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:51.684 DEBUG:boto:Data: 2022-09-03T06:54:51.685 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.685 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.686 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.686 DEBUG:boto:Params: {} 2022-09-03T06:54:51.687 DEBUG:boto:Token: None 2022-09-03T06:54:51.688 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.692 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000019684ed4b87e2448-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.693 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.695 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.695 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.696 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.697 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:51.698 DEBUG:boto:Data: 2022-09-03T06:54:51.698 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.700 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.700 DEBUG:boto:Params: {} 2022-09-03T06:54:51.701 DEBUG:boto:Token: None 2022-09-03T06:54:51.702 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d 2022-09-03T06:54:51.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MKWOPCJ2qapObVlZ5hoBFNWPck8= 2022-09-03T06:54:51.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MKWOPCJ2qapObVlZ5hoBFNWPck8=', 'Content-Length': '0'} 2022-09-03T06:54:51.705 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d30c189f321be3ed-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.706 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.707 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.708 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.709 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:51.710 DEBUG:boto:Data: 2022-09-03T06:54:51.711 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.712 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.712 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.714 DEBUG:boto:Params: {} 2022-09-03T06:54:51.715 DEBUG:boto:Token: None 2022-09-03T06:54:51.716 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d 2022-09-03T06:54:51.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MKWOPCJ2qapObVlZ5hoBFNWPck8= 2022-09-03T06:54:51.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MKWOPCJ2qapObVlZ5hoBFNWPck8=', 'Content-Length': '0'} 2022-09-03T06:54:51.720 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004fd8ea35948e6c76-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.721 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:54:51.723 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.723 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.724 DEBUG:boto:Method: GET 2022-09-03T06:54:51.725 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:51.726 DEBUG:boto:Data: 2022-09-03T06:54:51.727 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.729 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.730 DEBUG:boto:Params: {} 2022-09-03T06:54:51.731 DEBUG:boto:Token: None 2022-09-03T06:54:51.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d 2022-09-03T06:54:51.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vzk1SXR8yfsAlvd91e/cz0NxlgI= 2022-09-03T06:54:51.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vzk1SXR8yfsAlvd91e/cz0NxlgI=', 'Content-Length': '0'} 2022-09-03T06:54:51.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cbdfe3a78039e687-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.739 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:54:51.740 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:54:51.740 DEBUG:boto:Method: GET 2022-09-03T06:54:51.741 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:54:51.741 DEBUG:boto:Data: 2022-09-03T06:54:51.742 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.743 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.744 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.744 DEBUG:boto:Params: {} 2022-09-03T06:54:51.745 DEBUG:boto:Token: None 2022-09-03T06:54:51.746 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/d 2022-09-03T06:54:51.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vzk1SXR8yfsAlvd91e/cz0NxlgI= 2022-09-03T06:54:51.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vzk1SXR8yfsAlvd91e/cz0NxlgI=', 'Content-Length': '0'} 2022-09-03T06:54:51.750 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be71e926d73f2acf-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.751 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.752 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.752 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:51.753 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:51.753 DEBUG:boto:Method: GET 2022-09-03T06:54:51.754 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:51.755 DEBUG:boto:Data: 2022-09-03T06:54:51.755 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.756 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.757 DEBUG:boto:Params: {} 2022-09-03T06:54:51.757 DEBUG:boto:Token: None 2022-09-03T06:54:51.758 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.762 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f3e73865d2c9042-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.763 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.763 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.764 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.764 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:54:51.765 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:54:51.766 DEBUG:boto:Method: GET 2022-09-03T06:54:51.766 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:54:51.767 DEBUG:boto:Data: 2022-09-03T06:54:51.767 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.768 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.768 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.769 DEBUG:boto:Params: {} 2022-09-03T06:54:51.769 DEBUG:boto:Token: None 2022-09-03T06:54:51.770 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.773 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f0ae26b6b03cd3f-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.814 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.816 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:54:51.817 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:51.819 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:51.819 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.820 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.821 DEBUG:boto:Method: GET 2022-09-03T06:54:51.821 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:51.822 DEBUG:boto:Data: 2022-09-03T06:54:51.822 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.823 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.823 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.824 DEBUG:boto:Params: {} 2022-09-03T06:54:51.825 DEBUG:boto:Token: None 2022-09-03T06:54:51.825 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:51.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5pjVyvVSSw0L2tQmXINdFXXde8= 2022-09-03T06:54:51.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5pjVyvVSSw0L2tQmXINdFXXde8=', 'Content-Length': '0'} 2022-09-03T06:54:51.829 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ccf3449dba75a97-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.830 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:51.830 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:51.831 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.831 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.832 DEBUG:boto:Method: GET 2022-09-03T06:54:51.832 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:51.833 DEBUG:boto:Data: 2022-09-03T06:54:51.833 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.834 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.834 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.835 DEBUG:boto:Params: {} 2022-09-03T06:54:51.836 DEBUG:boto:Token: None 2022-09-03T06:54:51.836 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:51.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5pjVyvVSSw0L2tQmXINdFXXde8= 2022-09-03T06:54:51.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5pjVyvVSSw0L2tQmXINdFXXde8=', 'Content-Length': '0'} 2022-09-03T06:54:51.840 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001a067d68d810ad2-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.841 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:51.841 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:51.842 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.842 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.843 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.844 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:51.844 DEBUG:boto:Data: 2022-09-03T06:54:51.845 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.845 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.846 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.846 DEBUG:boto:Params: {} 2022-09-03T06:54:51.847 DEBUG:boto:Token: None 2022-09-03T06:54:51.848 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:51.848 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/xiMlUN/xUfuJsxU8lMZSkR/hns= 2022-09-03T06:54:51.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/xiMlUN/xUfuJsxU8lMZSkR/hns=', 'Content-Length': '0'} 2022-09-03T06:54:51.851 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000307df8327a6c4b22-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.853 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:51.854 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:51.854 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.855 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.855 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.856 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:51.857 DEBUG:boto:Data: 2022-09-03T06:54:51.857 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.858 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.858 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.859 DEBUG:boto:Params: {} 2022-09-03T06:54:51.859 DEBUG:boto:Token: None 2022-09-03T06:54:51.860 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:51.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/xiMlUN/xUfuJsxU8lMZSkR/hns= 2022-09-03T06:54:51.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/xiMlUN/xUfuJsxU8lMZSkR/hns=', 'Content-Length': '0'} 2022-09-03T06:54:51.864 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9e5061b4e5756da-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.864 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:54:51.865 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:51.865 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:51.866 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.867 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.867 DEBUG:boto:Method: GET 2022-09-03T06:54:51.868 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:51.868 DEBUG:boto:Data: 2022-09-03T06:54:51.869 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.870 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.870 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.871 DEBUG:boto:Params: {} 2022-09-03T06:54:51.871 DEBUG:boto:Token: None 2022-09-03T06:54:51.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:51.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5pjVyvVSSw0L2tQmXINdFXXde8= 2022-09-03T06:54:51.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5pjVyvVSSw0L2tQmXINdFXXde8=', 'Content-Length': '0'} 2022-09-03T06:54:51.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054ec204560c6424b-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.878 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:51.879 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:51.879 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.880 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:54:51.880 DEBUG:boto:Method: GET 2022-09-03T06:54:51.881 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:54:51.881 DEBUG:boto:Data: 2022-09-03T06:54:51.882 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.882 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.883 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.883 DEBUG:boto:Params: {} 2022-09-03T06:54:51.884 DEBUG:boto:Token: None 2022-09-03T06:54:51.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/e?versionId=null 2022-09-03T06:54:51.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P5pjVyvVSSw0L2tQmXINdFXXde8= 2022-09-03T06:54:51.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P5pjVyvVSSw0L2tQmXINdFXXde8=', 'Content-Length': '0'} 2022-09-03T06:54:51.888 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079528391cb2f82e4-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.889 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.890 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.890 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:51.891 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:51.891 DEBUG:boto:Method: GET 2022-09-03T06:54:51.892 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:51.892 DEBUG:boto:Data: 2022-09-03T06:54:51.893 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.894 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.894 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.895 DEBUG:boto:Params: {} 2022-09-03T06:54:51.896 DEBUG:boto:Token: None 2022-09-03T06:54:51.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.901 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009686a6486a059066-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.902 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.903 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:51.904 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:51.905 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:51.906 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:51.907 DEBUG:boto:Method: GET 2022-09-03T06:54:51.908 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:51.909 DEBUG:boto:Data: 2022-09-03T06:54:51.910 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.911 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.912 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.913 DEBUG:boto:Params: {} 2022-09-03T06:54:51.914 DEBUG:boto:Token: None 2022-09-03T06:54:51.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/ 2022-09-03T06:54:51.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk= 2022-09-03T06:54:51.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rKZhn0BvAG7Pb1RelsG1/UA2XGk=', 'Content-Length': '0'} 2022-09-03T06:54:51.921 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ab974ff407a9b5ba-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.923 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:51.924 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:51.925 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:51.926 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.927 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:51.928 DEBUG:boto:Data: 2022-09-03T06:54:51.929 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.931 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.932 DEBUG:boto:Params: {} 2022-09-03T06:54:51.933 DEBUG:boto:Token: None 2022-09-03T06:54:51.934 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/e 2022-09-03T06:54:51.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1L4QirOHuQVnSPGUYvddDnbKXo= 2022-09-03T06:54:51.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1L4QirOHuQVnSPGUYvddDnbKXo=', 'Content-Length': '0'} 2022-09-03T06:54:51.939 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df9612cded044f60-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.940 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:51.941 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:51.942 DEBUG:boto:Method: HEAD 2022-09-03T06:54:51.943 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:51.944 DEBUG:boto:Data: 2022-09-03T06:54:51.944 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.945 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:51.946 DEBUG:boto:Port: 8000 2022-09-03T06:54:51.946 DEBUG:boto:Params: {} 2022-09-03T06:54:51.947 DEBUG:boto:Token: None 2022-09-03T06:54:51.947 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/e 2022-09-03T06:54:51.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1L4QirOHuQVnSPGUYvddDnbKXo= 2022-09-03T06:54:51.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1L4QirOHuQVnSPGUYvddDnbKXo=', 'Content-Length': '0'} 2022-09-03T06:54:51.951 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056189bd47adc27d5-006312fa3b-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:51.952 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:54:51.953 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:51.954 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:51.954 DEBUG:boto:Method: GET 2022-09-03T06:54:51.955 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:51.956 DEBUG:boto:Data: 2022-09-03T06:54:51.957 DEBUG:boto:Headers: {} 2022-09-03T06:54:51.957 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:51.958 DEBUG:boto:Port: 8001 2022-09-03T06:54:51.959 DEBUG:boto:Params: {} 2022-09-03T06:54:51.959 DEBUG:boto:Token: None 2022-09-03T06:54:51.960 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:51 GMT /avysmw-71/e 2022-09-03T06:54:51.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wwlcNUIXqyM0YGSgIB9RD+42v74= 2022-09-03T06:54:51.961 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wwlcNUIXqyM0YGSgIB9RD+42v74=', 'Content-Length': '0'} 2022-09-03T06:54:51.964 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bbb6ec98ddf57cf6-006312fa3b-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.006 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:54:52.007 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:54:52.008 DEBUG:boto:Method: GET 2022-09-03T06:54:52.009 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:54:52.009 DEBUG:boto:Data: 2022-09-03T06:54:52.010 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.011 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.011 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.012 DEBUG:boto:Params: {} 2022-09-03T06:54:52.013 DEBUG:boto:Token: None 2022-09-03T06:54:52.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/e 2022-09-03T06:54:52.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kudSH9dF04nx/uPXjsEu38Gr1bA= 2022-09-03T06:54:52.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kudSH9dF04nx/uPXjsEu38Gr1bA=', 'Content-Length': '0'} 2022-09-03T06:54:52.018 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080de863d9f069601-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.020 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.021 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.022 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:52.023 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:52.023 DEBUG:boto:Method: GET 2022-09-03T06:54:52.024 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:52.025 DEBUG:boto:Data: 2022-09-03T06:54:52.026 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.027 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.028 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.029 DEBUG:boto:Params: {} 2022-09-03T06:54:52.030 DEBUG:boto:Token: None 2022-09-03T06:54:52.031 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.037 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000433c963ef066b520-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.038 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.039 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.039 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.040 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:54:52.040 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:54:52.042 DEBUG:boto:Method: GET 2022-09-03T06:54:52.042 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:54:52.043 DEBUG:boto:Data: 2022-09-03T06:54:52.044 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.044 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.045 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.045 DEBUG:boto:Params: {} 2022-09-03T06:54:52.046 DEBUG:boto:Token: None 2022-09-03T06:54:52.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.051 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001285802aab2d206e-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.052 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.053 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:54:52.053 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.054 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.055 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.055 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.057 DEBUG:boto:Method: GET 2022-09-03T06:54:52.058 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:52.059 DEBUG:boto:Data: 2022-09-03T06:54:52.060 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.061 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.062 DEBUG:boto:Params: {} 2022-09-03T06:54:52.063 DEBUG:boto:Token: None 2022-09-03T06:54:52.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:52.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bpOUn07aMMsEY6RFIur0zfxzWTM= 2022-09-03T06:54:52.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bpOUn07aMMsEY6RFIur0zfxzWTM=', 'Content-Length': '0'} 2022-09-03T06:54:52.072 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac6a9502e11259e9-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.073 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.074 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.074 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.075 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.075 DEBUG:boto:Method: GET 2022-09-03T06:54:52.076 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:52.077 DEBUG:boto:Data: 2022-09-03T06:54:52.077 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.078 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.078 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.079 DEBUG:boto:Params: {} 2022-09-03T06:54:52.079 DEBUG:boto:Token: None 2022-09-03T06:54:52.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:52.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bpOUn07aMMsEY6RFIur0zfxzWTM= 2022-09-03T06:54:52.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bpOUn07aMMsEY6RFIur0zfxzWTM=', 'Content-Length': '0'} 2022-09-03T06:54:52.085 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002299f1c9473a65e3-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.126 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.127 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.127 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.128 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.129 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.130 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:52.130 DEBUG:boto:Data: 2022-09-03T06:54:52.131 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.132 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.133 DEBUG:boto:Params: {} 2022-09-03T06:54:52.135 DEBUG:boto:Token: None 2022-09-03T06:54:52.136 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:52.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HFGcfD8FeuC2Kddn8ZLIDbfCDj8= 2022-09-03T06:54:52.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HFGcfD8FeuC2Kddn8ZLIDbfCDj8=', 'Content-Length': '0'} 2022-09-03T06:54:52.155 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000753b0d461ade68b9-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.156 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.157 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.158 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.159 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.159 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.160 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:52.161 DEBUG:boto:Data: 2022-09-03T06:54:52.161 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.162 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.162 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.163 DEBUG:boto:Params: {} 2022-09-03T06:54:52.164 DEBUG:boto:Token: None 2022-09-03T06:54:52.164 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:52.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HFGcfD8FeuC2Kddn8ZLIDbfCDj8= 2022-09-03T06:54:52.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HFGcfD8FeuC2Kddn8ZLIDbfCDj8=', 'Content-Length': '0'} 2022-09-03T06:54:52.168 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c81ba159fbb7870-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.169 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:54:52.169 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.170 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.171 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.171 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.172 DEBUG:boto:Method: GET 2022-09-03T06:54:52.172 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:52.173 DEBUG:boto:Data: 2022-09-03T06:54:52.173 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.176 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.177 DEBUG:boto:Params: {} 2022-09-03T06:54:52.178 DEBUG:boto:Token: None 2022-09-03T06:54:52.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:52.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bpOUn07aMMsEY6RFIur0zfxzWTM= 2022-09-03T06:54:52.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bpOUn07aMMsEY6RFIur0zfxzWTM=', 'Content-Length': '0'} 2022-09-03T06:54:52.185 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ef19b10f431dd7c-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.186 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.187 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.188 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.189 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:54:52.190 DEBUG:boto:Method: GET 2022-09-03T06:54:52.191 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:54:52.192 DEBUG:boto:Data: 2022-09-03T06:54:52.193 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.194 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.195 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.196 DEBUG:boto:Params: {} 2022-09-03T06:54:52.197 DEBUG:boto:Token: None 2022-09-03T06:54:52.198 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f?versionId=null 2022-09-03T06:54:52.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bpOUn07aMMsEY6RFIur0zfxzWTM= 2022-09-03T06:54:52.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bpOUn07aMMsEY6RFIur0zfxzWTM=', 'Content-Length': '0'} 2022-09-03T06:54:52.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062dc7b7fe6ef9f00-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.211 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.212 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.213 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:52.214 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:52.215 DEBUG:boto:Method: GET 2022-09-03T06:54:52.216 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:52.217 DEBUG:boto:Data: 2022-09-03T06:54:52.218 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.219 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.220 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.221 DEBUG:boto:Params: {} 2022-09-03T06:54:52.222 DEBUG:boto:Token: None 2022-09-03T06:54:52.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.230 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c130baaec7a09897-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.232 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.233 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.235 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.236 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:52.237 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:52.238 DEBUG:boto:Method: GET 2022-09-03T06:54:52.239 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:52.240 DEBUG:boto:Data: 2022-09-03T06:54:52.241 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.242 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.243 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.243 DEBUG:boto:Params: {} 2022-09-03T06:54:52.245 DEBUG:boto:Token: None 2022-09-03T06:54:52.246 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.252 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009fc86f1140c87eff-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.253 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.254 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.255 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.256 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.257 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:52.258 DEBUG:boto:Data: 2022-09-03T06:54:52.259 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.261 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.262 DEBUG:boto:Params: {} 2022-09-03T06:54:52.263 DEBUG:boto:Token: None 2022-09-03T06:54:52.263 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f 2022-09-03T06:54:52.264 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZAa64cgTrRZrZI0Ss6vI+eZEu8c= 2022-09-03T06:54:52.265 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZAa64cgTrRZrZI0Ss6vI+eZEu8c=', 'Content-Length': '0'} 2022-09-03T06:54:52.268 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008baa13eb675d9f60-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.269 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.270 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.271 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.272 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:52.273 DEBUG:boto:Data: 2022-09-03T06:54:52.273 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.274 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.275 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.276 DEBUG:boto:Params: {} 2022-09-03T06:54:52.276 DEBUG:boto:Token: None 2022-09-03T06:54:52.277 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f 2022-09-03T06:54:52.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZAa64cgTrRZrZI0Ss6vI+eZEu8c= 2022-09-03T06:54:52.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZAa64cgTrRZrZI0Ss6vI+eZEu8c=', 'Content-Length': '0'} 2022-09-03T06:54:52.280 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6e599593bd9971b-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.281 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:54:52.282 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.282 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.283 DEBUG:boto:Method: GET 2022-09-03T06:54:52.284 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:52.284 DEBUG:boto:Data: 2022-09-03T06:54:52.285 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.286 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.286 DEBUG:boto:Params: {} 2022-09-03T06:54:52.287 DEBUG:boto:Token: None 2022-09-03T06:54:52.288 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f 2022-09-03T06:54:52.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A1DE34VYks9Z94jFtBYUJkhD7UM= 2022-09-03T06:54:52.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A1DE34VYks9Z94jFtBYUJkhD7UM=', 'Content-Length': '0'} 2022-09-03T06:54:52.292 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1d1adf392a39ca7-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.338 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:54:52.339 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:54:52.341 DEBUG:boto:Method: GET 2022-09-03T06:54:52.342 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:54:52.343 DEBUG:boto:Data: 2022-09-03T06:54:52.344 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.345 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.346 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.347 DEBUG:boto:Params: {} 2022-09-03T06:54:52.348 DEBUG:boto:Token: None 2022-09-03T06:54:52.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/f 2022-09-03T06:54:52.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A1DE34VYks9Z94jFtBYUJkhD7UM= 2022-09-03T06:54:52.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A1DE34VYks9Z94jFtBYUJkhD7UM=', 'Content-Length': '0'} 2022-09-03T06:54:52.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b40b3b67caab3db-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.355 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.356 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.357 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:52.358 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:52.359 DEBUG:boto:Method: GET 2022-09-03T06:54:52.360 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:52.361 DEBUG:boto:Data: 2022-09-03T06:54:52.362 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.363 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.364 DEBUG:boto:Params: {} 2022-09-03T06:54:52.366 DEBUG:boto:Token: None 2022-09-03T06:54:52.367 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.368 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.372 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c79c21be987c522-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.373 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.375 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.375 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.376 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:54:52.376 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:54:52.377 DEBUG:boto:Method: GET 2022-09-03T06:54:52.378 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:54:52.378 DEBUG:boto:Data: 2022-09-03T06:54:52.379 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.379 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.380 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.381 DEBUG:boto:Params: {} 2022-09-03T06:54:52.381 DEBUG:boto:Token: None 2022-09-03T06:54:52.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.387 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ed3111b2ac4c830-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.388 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.389 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:54:52.389 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.390 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.390 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.391 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.392 DEBUG:boto:Method: GET 2022-09-03T06:54:52.393 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:52.393 DEBUG:boto:Data: 2022-09-03T06:54:52.394 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.396 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.396 DEBUG:boto:Params: {} 2022-09-03T06:54:52.397 DEBUG:boto:Token: None 2022-09-03T06:54:52.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:52.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FeBWVmv+IOADC5GYSFxqaHArPyw= 2022-09-03T06:54:52.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FeBWVmv+IOADC5GYSFxqaHArPyw=', 'Content-Length': '0'} 2022-09-03T06:54:52.402 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000055bf0699899c224-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.402 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.403 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.404 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.404 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.405 DEBUG:boto:Method: GET 2022-09-03T06:54:52.405 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:52.406 DEBUG:boto:Data: 2022-09-03T06:54:52.407 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.407 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.408 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.408 DEBUG:boto:Params: {} 2022-09-03T06:54:52.409 DEBUG:boto:Token: None 2022-09-03T06:54:52.410 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:52.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FeBWVmv+IOADC5GYSFxqaHArPyw= 2022-09-03T06:54:52.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FeBWVmv+IOADC5GYSFxqaHArPyw=', 'Content-Length': '0'} 2022-09-03T06:54:52.413 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032a8b105f759f4fe-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.458 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.459 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.459 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.460 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.460 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.461 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:52.462 DEBUG:boto:Data: 2022-09-03T06:54:52.462 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.463 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.464 DEBUG:boto:Params: {} 2022-09-03T06:54:52.464 DEBUG:boto:Token: None 2022-09-03T06:54:52.465 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:52.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pn9Y+ss43+p0j1pfABbb4udB2Lk= 2022-09-03T06:54:52.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pn9Y+ss43+p0j1pfABbb4udB2Lk=', 'Content-Length': '0'} 2022-09-03T06:54:52.468 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e787ddfdf3e2e50c-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.469 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.470 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.470 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.471 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.471 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.472 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:52.472 DEBUG:boto:Data: 2022-09-03T06:54:52.473 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.473 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.474 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.474 DEBUG:boto:Params: {} 2022-09-03T06:54:52.475 DEBUG:boto:Token: None 2022-09-03T06:54:52.475 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:52.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pn9Y+ss43+p0j1pfABbb4udB2Lk= 2022-09-03T06:54:52.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pn9Y+ss43+p0j1pfABbb4udB2Lk=', 'Content-Length': '0'} 2022-09-03T06:54:52.479 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a50be488cb6c928-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.479 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:54:52.480 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.481 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.481 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.482 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.482 DEBUG:boto:Method: GET 2022-09-03T06:54:52.483 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:52.483 DEBUG:boto:Data: 2022-09-03T06:54:52.484 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.485 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.486 DEBUG:boto:Params: {} 2022-09-03T06:54:52.486 DEBUG:boto:Token: None 2022-09-03T06:54:52.487 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:52.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FeBWVmv+IOADC5GYSFxqaHArPyw= 2022-09-03T06:54:52.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FeBWVmv+IOADC5GYSFxqaHArPyw=', 'Content-Length': '0'} 2022-09-03T06:54:52.491 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000accb1eaf636d6ba9-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.492 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.492 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.493 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.494 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:54:52.494 DEBUG:boto:Method: GET 2022-09-03T06:54:52.495 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:54:52.495 DEBUG:boto:Data: 2022-09-03T06:54:52.496 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.497 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.497 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.498 DEBUG:boto:Params: {} 2022-09-03T06:54:52.498 DEBUG:boto:Token: None 2022-09-03T06:54:52.499 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g?versionId=null 2022-09-03T06:54:52.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FeBWVmv+IOADC5GYSFxqaHArPyw= 2022-09-03T06:54:52.500 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FeBWVmv+IOADC5GYSFxqaHArPyw=', 'Content-Length': '0'} 2022-09-03T06:54:52.502 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f625db5c86490a2-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.503 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.503 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.504 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:52.505 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:52.505 DEBUG:boto:Method: GET 2022-09-03T06:54:52.506 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:52.506 DEBUG:boto:Data: 2022-09-03T06:54:52.507 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.508 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.509 DEBUG:boto:Params: {} 2022-09-03T06:54:52.509 DEBUG:boto:Token: None 2022-09-03T06:54:52.510 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.514 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b368d8728313edd0-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.515 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.516 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.516 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.517 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:52.517 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:52.518 DEBUG:boto:Method: GET 2022-09-03T06:54:52.518 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:52.519 DEBUG:boto:Data: 2022-09-03T06:54:52.520 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.520 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.521 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.521 DEBUG:boto:Params: {} 2022-09-03T06:54:52.522 DEBUG:boto:Token: None 2022-09-03T06:54:52.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.526 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d6769b8ab00035a-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.570 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.571 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.572 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.573 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.573 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:52.574 DEBUG:boto:Data: 2022-09-03T06:54:52.574 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.575 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.575 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.576 DEBUG:boto:Params: {} 2022-09-03T06:54:52.577 DEBUG:boto:Token: None 2022-09-03T06:54:52.578 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g 2022-09-03T06:54:52.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2vTLxqrxzQDYvFP43gPkD8DbGA= 2022-09-03T06:54:52.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2vTLxqrxzQDYvFP43gPkD8DbGA=', 'Content-Length': '0'} 2022-09-03T06:54:52.583 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bfa6c1583b26fedb-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.584 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.585 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.585 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.586 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:52.586 DEBUG:boto:Data: 2022-09-03T06:54:52.587 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.587 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.588 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.589 DEBUG:boto:Params: {} 2022-09-03T06:54:52.589 DEBUG:boto:Token: None 2022-09-03T06:54:52.590 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g 2022-09-03T06:54:52.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2vTLxqrxzQDYvFP43gPkD8DbGA= 2022-09-03T06:54:52.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2vTLxqrxzQDYvFP43gPkD8DbGA=', 'Content-Length': '0'} 2022-09-03T06:54:52.593 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000809e8dfed0809689-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.594 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:54:52.594 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.595 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.596 DEBUG:boto:Method: GET 2022-09-03T06:54:52.597 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:52.598 DEBUG:boto:Data: 2022-09-03T06:54:52.599 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.601 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.602 DEBUG:boto:Params: {} 2022-09-03T06:54:52.603 DEBUG:boto:Token: None 2022-09-03T06:54:52.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g 2022-09-03T06:54:52.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7dtcps6iHKAY6TL4OahXn12bxv0= 2022-09-03T06:54:52.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7dtcps6iHKAY6TL4OahXn12bxv0=', 'Content-Length': '0'} 2022-09-03T06:54:52.609 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001271c3f98c2ba730-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.650 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:54:52.651 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:54:52.652 DEBUG:boto:Method: GET 2022-09-03T06:54:52.653 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:54:52.653 DEBUG:boto:Data: 2022-09-03T06:54:52.654 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.655 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.656 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.657 DEBUG:boto:Params: {} 2022-09-03T06:54:52.658 DEBUG:boto:Token: None 2022-09-03T06:54:52.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/g 2022-09-03T06:54:52.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7dtcps6iHKAY6TL4OahXn12bxv0= 2022-09-03T06:54:52.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7dtcps6iHKAY6TL4OahXn12bxv0=', 'Content-Length': '0'} 2022-09-03T06:54:52.664 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2b8173abd1c5a81-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.665 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.667 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.667 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:52.668 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:52.669 DEBUG:boto:Method: GET 2022-09-03T06:54:52.669 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:52.670 DEBUG:boto:Data: 2022-09-03T06:54:52.670 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.671 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.671 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.673 DEBUG:boto:Params: {} 2022-09-03T06:54:52.674 DEBUG:boto:Token: None 2022-09-03T06:54:52.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.679 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7a94bdbb8c44b2c-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.680 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.681 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.681 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.682 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:54:52.682 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:54:52.683 DEBUG:boto:Method: GET 2022-09-03T06:54:52.684 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:54:52.684 DEBUG:boto:Data: 2022-09-03T06:54:52.685 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.685 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.686 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.687 DEBUG:boto:Params: {} 2022-09-03T06:54:52.688 DEBUG:boto:Token: None 2022-09-03T06:54:52.689 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.694 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a764e8558611a21b-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.695 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.697 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:54:52.697 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.698 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.699 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.700 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.700 DEBUG:boto:Method: GET 2022-09-03T06:54:52.701 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:52.701 DEBUG:boto:Data: 2022-09-03T06:54:52.702 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.706 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.706 DEBUG:boto:Params: {} 2022-09-03T06:54:52.707 DEBUG:boto:Token: None 2022-09-03T06:54:52.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:52.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e7SOX5PCUEevyT3FFYT5/oC4GJU= 2022-09-03T06:54:52.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e7SOX5PCUEevyT3FFYT5/oC4GJU=', 'Content-Length': '0'} 2022-09-03T06:54:52.711 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ff25376bcc267d1-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.712 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.713 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.713 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.714 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.716 DEBUG:boto:Method: GET 2022-09-03T06:54:52.716 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:52.717 DEBUG:boto:Data: 2022-09-03T06:54:52.717 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.718 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.718 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.719 DEBUG:boto:Params: {} 2022-09-03T06:54:52.720 DEBUG:boto:Token: None 2022-09-03T06:54:52.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:52.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e7SOX5PCUEevyT3FFYT5/oC4GJU= 2022-09-03T06:54:52.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e7SOX5PCUEevyT3FFYT5/oC4GJU=', 'Content-Length': '0'} 2022-09-03T06:54:52.723 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000545972b82790ae4d-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.724 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.725 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.726 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.727 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.727 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.728 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:52.728 DEBUG:boto:Data: 2022-09-03T06:54:52.729 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.730 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.730 DEBUG:boto:Params: {} 2022-09-03T06:54:52.731 DEBUG:boto:Token: None 2022-09-03T06:54:52.731 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:52.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6nA/9aBFXVi/6Oqi6xBsPsf70MY= 2022-09-03T06:54:52.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6nA/9aBFXVi/6Oqi6xBsPsf70MY=', 'Content-Length': '0'} 2022-09-03T06:54:52.735 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001293db908818bdf9-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.736 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.736 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.738 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.739 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.739 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.740 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:52.740 DEBUG:boto:Data: 2022-09-03T06:54:52.741 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.741 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.742 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.743 DEBUG:boto:Params: {} 2022-09-03T06:54:52.743 DEBUG:boto:Token: None 2022-09-03T06:54:52.744 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:52.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6nA/9aBFXVi/6Oqi6xBsPsf70MY= 2022-09-03T06:54:52.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6nA/9aBFXVi/6Oqi6xBsPsf70MY=', 'Content-Length': '0'} 2022-09-03T06:54:52.749 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3578b4583958753-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.750 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:54:52.750 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.751 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.751 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.752 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.752 DEBUG:boto:Method: GET 2022-09-03T06:54:52.753 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:52.753 DEBUG:boto:Data: 2022-09-03T06:54:52.754 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.754 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.755 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.755 DEBUG:boto:Params: {} 2022-09-03T06:54:52.756 DEBUG:boto:Token: None 2022-09-03T06:54:52.757 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:52.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e7SOX5PCUEevyT3FFYT5/oC4GJU= 2022-09-03T06:54:52.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e7SOX5PCUEevyT3FFYT5/oC4GJU=', 'Content-Length': '0'} 2022-09-03T06:54:52.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089967d19c7868e8a-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.807 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.808 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.809 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.809 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:54:52.810 DEBUG:boto:Method: GET 2022-09-03T06:54:52.811 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:54:52.811 DEBUG:boto:Data: 2022-09-03T06:54:52.812 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.813 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.813 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.814 DEBUG:boto:Params: {} 2022-09-03T06:54:52.814 DEBUG:boto:Token: None 2022-09-03T06:54:52.815 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h?versionId=null 2022-09-03T06:54:52.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e7SOX5PCUEevyT3FFYT5/oC4GJU= 2022-09-03T06:54:52.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e7SOX5PCUEevyT3FFYT5/oC4GJU=', 'Content-Length': '0'} 2022-09-03T06:54:52.818 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bef2cb20f0236b97-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.819 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.820 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.820 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:52.821 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:52.821 DEBUG:boto:Method: GET 2022-09-03T06:54:52.822 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:52.823 DEBUG:boto:Data: 2022-09-03T06:54:52.823 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.824 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.825 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.825 DEBUG:boto:Params: {} 2022-09-03T06:54:52.826 DEBUG:boto:Token: None 2022-09-03T06:54:52.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.831 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085810eefa1f53436-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.832 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.834 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.834 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.835 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:52.836 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:52.836 DEBUG:boto:Method: GET 2022-09-03T06:54:52.837 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:52.837 DEBUG:boto:Data: 2022-09-03T06:54:52.838 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.839 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.839 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.840 DEBUG:boto:Params: {} 2022-09-03T06:54:52.840 DEBUG:boto:Token: None 2022-09-03T06:54:52.841 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.845 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009323d8366002c5b9-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.886 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.887 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.888 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.888 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.889 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:52.889 DEBUG:boto:Data: 2022-09-03T06:54:52.890 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.890 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.891 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.892 DEBUG:boto:Params: {} 2022-09-03T06:54:52.893 DEBUG:boto:Token: None 2022-09-03T06:54:52.895 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h 2022-09-03T06:54:52.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n1jYiJD5B0yPD8hxCM+RtRSFzKI= 2022-09-03T06:54:52.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n1jYiJD5B0yPD8hxCM+RtRSFzKI=', 'Content-Length': '0'} 2022-09-03T06:54:52.900 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039cf868b67d7056b-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.901 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.903 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.904 DEBUG:boto:Method: HEAD 2022-09-03T06:54:52.905 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:52.906 DEBUG:boto:Data: 2022-09-03T06:54:52.907 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.908 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.909 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.910 DEBUG:boto:Params: {} 2022-09-03T06:54:52.912 DEBUG:boto:Token: None 2022-09-03T06:54:52.913 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h 2022-09-03T06:54:52.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n1jYiJD5B0yPD8hxCM+RtRSFzKI= 2022-09-03T06:54:52.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n1jYiJD5B0yPD8hxCM+RtRSFzKI=', 'Content-Length': '0'} 2022-09-03T06:54:52.918 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b76ff0da4e9e3802-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.919 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:54:52.923 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.924 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.925 DEBUG:boto:Method: GET 2022-09-03T06:54:52.925 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:52.926 DEBUG:boto:Data: 2022-09-03T06:54:52.926 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.927 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.928 DEBUG:boto:Params: {} 2022-09-03T06:54:52.929 DEBUG:boto:Token: None 2022-09-03T06:54:52.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h 2022-09-03T06:54:52.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cpGZq3wHW8RwUzvr/mTo7g7XjOI= 2022-09-03T06:54:52.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cpGZq3wHW8RwUzvr/mTo7g7XjOI=', 'Content-Length': '0'} 2022-09-03T06:54:52.933 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091e2dd0942b2226e-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.934 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:54:52.935 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:54:52.935 DEBUG:boto:Method: GET 2022-09-03T06:54:52.936 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:54:52.937 DEBUG:boto:Data: 2022-09-03T06:54:52.937 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.938 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.938 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.939 DEBUG:boto:Params: {} 2022-09-03T06:54:52.940 DEBUG:boto:Token: None 2022-09-03T06:54:52.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/h 2022-09-03T06:54:52.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cpGZq3wHW8RwUzvr/mTo7g7XjOI= 2022-09-03T06:54:52.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cpGZq3wHW8RwUzvr/mTo7g7XjOI=', 'Content-Length': '0'} 2022-09-03T06:54:52.944 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce8909385357f82c-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.945 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.945 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.946 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:52.947 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:52.948 DEBUG:boto:Method: GET 2022-09-03T06:54:52.949 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:52.950 DEBUG:boto:Data: 2022-09-03T06:54:52.951 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.952 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:54:52.953 DEBUG:boto:Port: 8001 2022-09-03T06:54:52.954 DEBUG:boto:Params: {} 2022-09-03T06:54:52.955 DEBUG:boto:Token: None 2022-09-03T06:54:52.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.962 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a09c74127ca17522-006312fa3c-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.963 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.964 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:54:52.964 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:54:52.965 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:54:52.966 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:54:52.966 DEBUG:boto:Method: GET 2022-09-03T06:54:52.967 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:54:52.968 DEBUG:boto:Data: 2022-09-03T06:54:52.968 DEBUG:boto:Headers: {} 2022-09-03T06:54:52.969 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:54:52.969 DEBUG:boto:Port: 8000 2022-09-03T06:54:52.970 DEBUG:boto:Params: {} 2022-09-03T06:54:52.971 DEBUG:boto:Token: None 2022-09-03T06:54:52.971 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:54:52 GMT /avysmw-71/ 2022-09-03T06:54:52.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c= 2022-09-03T06:54:52.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:54:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzNGqC0Is0yEHi6HUSvGhGqaW4c=', 'Content-Length': '0'} 2022-09-03T06:54:52.975 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5e8f12f08c4307d-006312fa3c-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:54:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:54:52.976 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:54:52.977 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-71 zones={test-zone3, test-zone2} 2022-09-03T06:54:52.978 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'avysmw-71', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:54:52.978 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'avysmw-71', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:54:52.979 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket avysmw-71 --num-shards 3 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:54:53.037 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:54:53.038 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:54:53.088 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T06:54:53.088 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-71 2022-09-03T06:54:53.089 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 8 2022-09-03T06:54:53.130 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:54:53.131 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:54:53.132 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket layout --bucket avysmw-71 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:54:53.185 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:54:53.186 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:54:53.255 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:54:53.256 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:54:53.256 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding": "None", 2022-09-03T06:54:53.256 INFO:teuthology.orchestra.run.smithi062.stdout: "current_index": { 2022-09-03T06:54:53.256 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 2, 2022-09-03T06:54:53.257 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:54:53.257 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "Normal", 2022-09-03T06:54:53.257 INFO:teuthology.orchestra.run.smithi062.stdout: "normal": { 2022-09-03T06:54:53.258 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:54:53.258 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:54:53.258 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.259 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.259 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:54:53.259 INFO:teuthology.orchestra.run.smithi062.stdout: "logs": [ 2022-09-03T06:54:53.259 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:54:53.260 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 0, 2022-09-03T06:54:53.260 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:54:53.260 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:54:53.262 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:54:53.262 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 0, 2022-09-03T06:54:53.262 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:54:53.262 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 11, 2022-09-03T06:54:53.263 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:54:53.263 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.263 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.264 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.264 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:54:53.264 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:54:53.264 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 1, 2022-09-03T06:54:53.265 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:54:53.265 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:54:53.265 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:54:53.265 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 1, 2022-09-03T06:54:53.266 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:54:53.266 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:54:53.266 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:54:53.267 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.267 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.267 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.267 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:54:53.268 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:54:53.268 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 2, 2022-09-03T06:54:53.268 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:54:53.269 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:54:53.269 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:54:53.269 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 2, 2022-09-03T06:54:53.269 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:54:53.270 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:54:53.270 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:54:53.270 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.270 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.271 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.271 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.271 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:54:53.271 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:54:53.272 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:54:53.273 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-71 2022-09-03T06:54:53.274 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:53.275 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:54:53.276 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:54:53.333 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:54:53.334 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:54:53.369 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:53.367+0000 7f77d929a600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:54:58.370 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:54:58.368+0000 7f77d929a600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:55:03.370 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:03.368+0000 7f77d929a600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:55:08.371 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:08.369+0000 7f77d929a600 1 bucket sync caught up with empty source 2022-09-03T06:55:08.372 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:08.369+0000 7f77d929a600 0 bucket checkpoint complete 2022-09-03T06:55:08.401 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-71 2022-09-03T06:55:08.402 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.404 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.405 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:08.475 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:08.475 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:08.525 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:08.523+0000 7f6aea9e9600 1 bucket sync caught up with empty source 2022-09-03T06:55:08.525 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:08.523+0000 7f6aea9e9600 0 bucket checkpoint complete 2022-09-03T06:55:08.544 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-71 2022-09-03T06:55:08.545 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.546 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.547 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:08.609 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:08.610 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:08.673 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:08.671+0000 7fdd2c5eb600 1 bucket sync caught up with empty source 2022-09-03T06:55:08.674 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:08.671+0000 7fdd2c5eb600 0 bucket checkpoint complete 2022-09-03T06:55:08.689 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-71 2022-09-03T06:55:08.690 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.691 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.692 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:08.757 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:08.758 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:08.812 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:08.805+0000 7fdd565d4600 1 bucket sync caught up with source: 2022-09-03T06:55:08.813 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:55:08.813 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:55:08.814 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:08.805+0000 7fdd565d4600 0 bucket checkpoint complete 2022-09-03T06:55:08.824 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-71 2022-09-03T06:55:08.826 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.827 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.828 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:08.888 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:08.888 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:08.926 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:08.924+0000 7f6ddf908600 1 bucket sync caught up with source: 2022-09-03T06:55:08.926 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:55:08.927 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.315.5, , , 00000000001.459.5, 00000000001.359.5, , , 00000000002.359.5, , 00000000002.257.5] 2022-09-03T06:55:08.927 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:08.924+0000 7f6ddf908600 0 bucket checkpoint complete 2022-09-03T06:55:08.940 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-71 2022-09-03T06:55:08.942 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-71 2022-09-03T06:55:08.942 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.943 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:08.944 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:09.003 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:09.004 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:09.054 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:09.052+0000 7f756cf2d600 1 bucket sync caught up with source: 2022-09-03T06:55:09.055 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:55:09.055 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:55:09.056 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:09.052+0000 7f756cf2d600 0 bucket checkpoint complete 2022-09-03T06:55:09.080 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-71 2022-09-03T06:55:09.081 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:09.082 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-71', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:09.083 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-71 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:09.141 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:09.141 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:09.200 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:09.198+0000 7f0c22b12600 1 bucket sync caught up with source: 2022-09-03T06:55:09.200 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:55:09.201 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.373.5, , , 00000000001.355.5, 00000000001.375.5, , , 00000000002.167.5, , 00000000002.318.5] 2022-09-03T06:55:09.201 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:09.198+0000 7f0c22b12600 0 bucket checkpoint complete 2022-09-03T06:55:09.214 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-71 2022-09-03T06:55:09.214 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-71 zones={test-zone2, test-zone1} 2022-09-03T06:55:09.215 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.215 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.216 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.217 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:55:09.218 DEBUG:boto:Data: 2022-09-03T06:55:09.219 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.219 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.220 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.221 DEBUG:boto:Params: {} 2022-09-03T06:55:09.222 DEBUG:boto:Token: None 2022-09-03T06:55:09.222 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hiOGsSXZqG6zxKAhz8l39cbhtMo= 2022-09-03T06:55:09.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hiOGsSXZqG6zxKAhz8l39cbhtMo=', 'Content-Length': '0'} 2022-09-03T06:55:09.226 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c97a883df1267b13-006312fa4d-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.227 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.228 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.228 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.229 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:55:09.230 DEBUG:boto:Data: 2022-09-03T06:55:09.231 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.232 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.233 DEBUG:boto:Params: {} 2022-09-03T06:55:09.234 DEBUG:boto:Token: None 2022-09-03T06:55:09.235 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hiOGsSXZqG6zxKAhz8l39cbhtMo= 2022-09-03T06:55:09.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hiOGsSXZqG6zxKAhz8l39cbhtMo=', 'Content-Length': '0'} 2022-09-03T06:55:09.240 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000669e15db0ee313ce-006312fa4d-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.241 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:55:09.242 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.243 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.244 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.245 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.246 DEBUG:boto:Method: GET 2022-09-03T06:55:09.247 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.248 DEBUG:boto:Data: 2022-09-03T06:55:09.249 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.250 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.251 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.252 DEBUG:boto:Params: {} 2022-09-03T06:55:09.253 DEBUG:boto:Token: None 2022-09-03T06:55:09.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/?versions 2022-09-03T06:55:09.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tTZuU0XGSlSUT6gIGxgvMmawYMk= 2022-09-03T06:55:09.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tTZuU0XGSlSUT6gIGxgvMmawYMk=', 'Content-Length': '0'} 2022-09-03T06:55:09.261 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000420d6a3ff5407f33-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.262 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.264 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:55:09.265 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:55:09.265 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:55:09.266 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:55:09.266 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:55:09.267 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:55:09.267 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:55:09.268 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:55:09.268 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:55:09.270 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.271 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.272 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.273 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.274 DEBUG:boto:Method: GET 2022-09-03T06:55:09.275 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.276 DEBUG:boto:Data: 2022-09-03T06:55:09.277 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.278 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.279 DEBUG:boto:Params: {} 2022-09-03T06:55:09.279 DEBUG:boto:Token: None 2022-09-03T06:55:09.280 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/?versions 2022-09-03T06:55:09.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tTZuU0XGSlSUT6gIGxgvMmawYMk= 2022-09-03T06:55:09.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tTZuU0XGSlSUT6gIGxgvMmawYMk=', 'Content-Length': '0'} 2022-09-03T06:55:09.288 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cfe5f10ec2d5c113-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.289 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.292 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:55:09.294 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:55:09.295 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:55:09.296 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:55:09.297 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:55:09.298 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:55:09.299 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:55:09.300 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:55:09.301 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.302 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.303 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.304 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.304 DEBUG:boto:Method: GET 2022-09-03T06:55:09.305 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.306 DEBUG:boto:Data: 2022-09-03T06:55:09.307 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.307 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.308 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.309 DEBUG:boto:Params: {} 2022-09-03T06:55:09.310 DEBUG:boto:Token: None 2022-09-03T06:55:09.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/?versions 2022-09-03T06:55:09.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tTZuU0XGSlSUT6gIGxgvMmawYMk= 2022-09-03T06:55:09.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tTZuU0XGSlSUT6gIGxgvMmawYMk=', 'Content-Length': '0'} 2022-09-03T06:55:09.317 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000ad6677bd8eeaa9c-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.319 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.321 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.322 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.323 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.324 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.325 DEBUG:boto:Method: GET 2022-09-03T06:55:09.326 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:09.327 DEBUG:boto:Data: 2022-09-03T06:55:09.328 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.329 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.330 DEBUG:boto:Params: {} 2022-09-03T06:55:09.332 DEBUG:boto:Token: None 2022-09-03T06:55:09.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/?versions 2022-09-03T06:55:09.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tTZuU0XGSlSUT6gIGxgvMmawYMk= 2022-09-03T06:55:09.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tTZuU0XGSlSUT6gIGxgvMmawYMk=', 'Content-Length': '0'} 2022-09-03T06:55:09.338 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000bb05f28a1ad88d8-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.339 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.340 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:55:09.341 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.341 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.342 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.343 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.343 DEBUG:boto:Method: GET 2022-09-03T06:55:09.344 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:09.344 DEBUG:boto:Data: 2022-09-03T06:55:09.345 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.345 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.346 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.347 DEBUG:boto:Params: {} 2022-09-03T06:55:09.348 DEBUG:boto:Token: None 2022-09-03T06:55:09.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:09.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5nQkyJU5vekyTb0oT1r8vr6mpxU= 2022-09-03T06:55:09.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5nQkyJU5vekyTb0oT1r8vr6mpxU=', 'Content-Length': '0'} 2022-09-03T06:55:09.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c21d1cb882200e6e-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.356 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.357 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.358 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.359 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.359 DEBUG:boto:Method: GET 2022-09-03T06:55:09.360 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:09.361 DEBUG:boto:Data: 2022-09-03T06:55:09.361 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.362 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.363 DEBUG:boto:Params: {} 2022-09-03T06:55:09.363 DEBUG:boto:Token: None 2022-09-03T06:55:09.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:09.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5nQkyJU5vekyTb0oT1r8vr6mpxU= 2022-09-03T06:55:09.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5nQkyJU5vekyTb0oT1r8vr6mpxU=', 'Content-Length': '0'} 2022-09-03T06:55:09.368 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001058aab098f9faef-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.369 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.369 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.370 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.370 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.371 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.372 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:09.372 DEBUG:boto:Data: 2022-09-03T06:55:09.373 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.373 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.374 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.374 DEBUG:boto:Params: {} 2022-09-03T06:55:09.375 DEBUG:boto:Token: None 2022-09-03T06:55:09.375 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:09.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5+lFEWnOV38gqOEpAHPGlwos69Q= 2022-09-03T06:55:09.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5+lFEWnOV38gqOEpAHPGlwos69Q=', 'Content-Length': '0'} 2022-09-03T06:55:09.378 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008752b5ce70f6a35-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.379 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.380 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.380 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.381 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.382 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.382 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:09.383 DEBUG:boto:Data: 2022-09-03T06:55:09.384 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.385 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.386 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.387 DEBUG:boto:Params: {} 2022-09-03T06:55:09.388 DEBUG:boto:Token: None 2022-09-03T06:55:09.389 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:09.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5+lFEWnOV38gqOEpAHPGlwos69Q= 2022-09-03T06:55:09.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5+lFEWnOV38gqOEpAHPGlwos69Q=', 'Content-Length': '0'} 2022-09-03T06:55:09.393 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042a58b467a20a0d6-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.394 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:55:09.395 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.396 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.397 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.398 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.398 DEBUG:boto:Method: GET 2022-09-03T06:55:09.399 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:09.399 DEBUG:boto:Data: 2022-09-03T06:55:09.400 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.401 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.401 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.403 DEBUG:boto:Params: {} 2022-09-03T06:55:09.404 DEBUG:boto:Token: None 2022-09-03T06:55:09.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:09.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5nQkyJU5vekyTb0oT1r8vr6mpxU= 2022-09-03T06:55:09.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5nQkyJU5vekyTb0oT1r8vr6mpxU=', 'Content-Length': '0'} 2022-09-03T06:55:09.410 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c71b85dca3771f0-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.450 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.451 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.452 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.453 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:09.454 DEBUG:boto:Method: GET 2022-09-03T06:55:09.455 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:09.456 DEBUG:boto:Data: 2022-09-03T06:55:09.457 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.459 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.460 DEBUG:boto:Params: {} 2022-09-03T06:55:09.461 DEBUG:boto:Token: None 2022-09-03T06:55:09.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:09.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5nQkyJU5vekyTb0oT1r8vr6mpxU= 2022-09-03T06:55:09.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5nQkyJU5vekyTb0oT1r8vr6mpxU=', 'Content-Length': '0'} 2022-09-03T06:55:09.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5499e7eb365e6eb-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.468 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.469 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.469 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:09.470 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:09.471 DEBUG:boto:Method: GET 2022-09-03T06:55:09.471 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:09.472 DEBUG:boto:Data: 2022-09-03T06:55:09.472 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.473 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.474 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.474 DEBUG:boto:Params: {} 2022-09-03T06:55:09.476 DEBUG:boto:Token: None 2022-09-03T06:55:09.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.482 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1ff7f1271e49abd-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.483 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.484 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.485 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.485 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:09.487 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:09.488 DEBUG:boto:Method: GET 2022-09-03T06:55:09.489 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:09.489 DEBUG:boto:Data: 2022-09-03T06:55:09.490 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.491 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.492 DEBUG:boto:Params: {} 2022-09-03T06:55:09.492 DEBUG:boto:Token: None 2022-09-03T06:55:09.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.498 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007142c102e6121049-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.499 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.500 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.500 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.501 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.502 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:09.502 DEBUG:boto:Data: 2022-09-03T06:55:09.503 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.504 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.504 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.505 DEBUG:boto:Params: {} 2022-09-03T06:55:09.505 DEBUG:boto:Token: None 2022-09-03T06:55:09.506 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a 2022-09-03T06:55:09.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r1K5rmFQld4kQ8bHruyKivC+9/I= 2022-09-03T06:55:09.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r1K5rmFQld4kQ8bHruyKivC+9/I=', 'Content-Length': '0'} 2022-09-03T06:55:09.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000006458732e448342d-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.511 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.511 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.512 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.513 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:09.514 DEBUG:boto:Data: 2022-09-03T06:55:09.515 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.516 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.517 DEBUG:boto:Params: {} 2022-09-03T06:55:09.518 DEBUG:boto:Token: None 2022-09-03T06:55:09.519 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a 2022-09-03T06:55:09.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r1K5rmFQld4kQ8bHruyKivC+9/I= 2022-09-03T06:55:09.522 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r1K5rmFQld4kQ8bHruyKivC+9/I=', 'Content-Length': '0'} 2022-09-03T06:55:09.524 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068252d1420d1f6d9-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.525 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:55:09.526 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.527 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.527 DEBUG:boto:Method: GET 2022-09-03T06:55:09.528 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:09.528 DEBUG:boto:Data: 2022-09-03T06:55:09.529 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.530 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.530 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.531 DEBUG:boto:Params: {} 2022-09-03T06:55:09.531 DEBUG:boto:Token: None 2022-09-03T06:55:09.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a 2022-09-03T06:55:09.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rl/k4Qrs92u88/O/qkBz7p77hu0= 2022-09-03T06:55:09.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rl/k4Qrs92u88/O/qkBz7p77hu0=', 'Content-Length': '0'} 2022-09-03T06:55:09.535 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee4ca2db99812c84-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.536 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:09.537 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:09.538 DEBUG:boto:Method: GET 2022-09-03T06:55:09.539 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:09.539 DEBUG:boto:Data: 2022-09-03T06:55:09.540 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.541 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.542 DEBUG:boto:Params: {} 2022-09-03T06:55:09.543 DEBUG:boto:Token: None 2022-09-03T06:55:09.543 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/a 2022-09-03T06:55:09.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rl/k4Qrs92u88/O/qkBz7p77hu0= 2022-09-03T06:55:09.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rl/k4Qrs92u88/O/qkBz7p77hu0=', 'Content-Length': '0'} 2022-09-03T06:55:09.547 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e99b86be1cfea70-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.590 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.591 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.592 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:09.593 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:09.594 DEBUG:boto:Method: GET 2022-09-03T06:55:09.595 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:09.597 DEBUG:boto:Data: 2022-09-03T06:55:09.598 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.599 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.600 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.601 DEBUG:boto:Params: {} 2022-09-03T06:55:09.602 DEBUG:boto:Token: None 2022-09-03T06:55:09.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.613 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000014db405ab94f8259-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.615 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.616 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.617 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.618 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:09.619 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:09.620 DEBUG:boto:Method: GET 2022-09-03T06:55:09.621 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:09.622 DEBUG:boto:Data: 2022-09-03T06:55:09.623 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.624 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.625 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.626 DEBUG:boto:Params: {} 2022-09-03T06:55:09.627 DEBUG:boto:Token: None 2022-09-03T06:55:09.628 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.633 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c20169d4cdae997-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.634 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.635 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:55:09.636 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.637 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.638 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.639 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.640 DEBUG:boto:Method: GET 2022-09-03T06:55:09.641 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:09.642 DEBUG:boto:Data: 2022-09-03T06:55:09.643 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.644 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.645 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.646 DEBUG:boto:Params: {} 2022-09-03T06:55:09.647 DEBUG:boto:Token: None 2022-09-03T06:55:09.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:09.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s0HIP7bRyqqZGwziglURCZsADdA= 2022-09-03T06:55:09.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s0HIP7bRyqqZGwziglURCZsADdA=', 'Content-Length': '0'} 2022-09-03T06:55:09.653 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027e12eadc1faf2da-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.655 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.656 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.660 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.661 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.662 DEBUG:boto:Method: GET 2022-09-03T06:55:09.663 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:09.664 DEBUG:boto:Data: 2022-09-03T06:55:09.665 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.667 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.668 DEBUG:boto:Params: {} 2022-09-03T06:55:09.675 DEBUG:boto:Token: None 2022-09-03T06:55:09.676 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:09.677 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s0HIP7bRyqqZGwziglURCZsADdA= 2022-09-03T06:55:09.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s0HIP7bRyqqZGwziglURCZsADdA=', 'Content-Length': '0'} 2022-09-03T06:55:09.681 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002cec6f9579727317-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.682 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.683 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.683 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.684 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.684 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.685 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:09.685 DEBUG:boto:Data: 2022-09-03T06:55:09.686 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.686 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.687 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.688 DEBUG:boto:Params: {} 2022-09-03T06:55:09.688 DEBUG:boto:Token: None 2022-09-03T06:55:09.689 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:09.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QuBsLvwRWd6o05Uaubv360Y64Jc= 2022-09-03T06:55:09.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QuBsLvwRWd6o05Uaubv360Y64Jc=', 'Content-Length': '0'} 2022-09-03T06:55:09.692 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f7c44f011341580-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.693 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.693 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.694 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.694 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.695 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.695 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:09.696 DEBUG:boto:Data: 2022-09-03T06:55:09.697 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.699 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.700 DEBUG:boto:Params: {} 2022-09-03T06:55:09.701 DEBUG:boto:Token: None 2022-09-03T06:55:09.702 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:09.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QuBsLvwRWd6o05Uaubv360Y64Jc= 2022-09-03T06:55:09.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QuBsLvwRWd6o05Uaubv360Y64Jc=', 'Content-Length': '0'} 2022-09-03T06:55:09.707 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fe50fd66f53c70e-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.708 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:55:09.709 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.710 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.711 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.712 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.713 DEBUG:boto:Method: GET 2022-09-03T06:55:09.714 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:09.715 DEBUG:boto:Data: 2022-09-03T06:55:09.716 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.717 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.718 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.719 DEBUG:boto:Params: {} 2022-09-03T06:55:09.720 DEBUG:boto:Token: None 2022-09-03T06:55:09.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:09.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s0HIP7bRyqqZGwziglURCZsADdA= 2022-09-03T06:55:09.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s0HIP7bRyqqZGwziglURCZsADdA=', 'Content-Length': '0'} 2022-09-03T06:55:09.726 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d99b131541b4579-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.727 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.728 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.728 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.729 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:09.730 DEBUG:boto:Method: GET 2022-09-03T06:55:09.731 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:09.731 DEBUG:boto:Data: 2022-09-03T06:55:09.732 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.733 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.734 DEBUG:boto:Params: {} 2022-09-03T06:55:09.735 DEBUG:boto:Token: None 2022-09-03T06:55:09.736 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:09.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s0HIP7bRyqqZGwziglURCZsADdA= 2022-09-03T06:55:09.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s0HIP7bRyqqZGwziglURCZsADdA=', 'Content-Length': '0'} 2022-09-03T06:55:09.739 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000db62e0b30393fe4-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.740 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.741 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.742 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:09.742 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:09.743 DEBUG:boto:Method: GET 2022-09-03T06:55:09.743 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:09.744 DEBUG:boto:Data: 2022-09-03T06:55:09.745 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.745 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.746 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.747 DEBUG:boto:Params: {} 2022-09-03T06:55:09.748 DEBUG:boto:Token: None 2022-09-03T06:55:09.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.757 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005b5f5d91b6742686-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.758 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.760 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.761 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.762 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:09.763 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:09.764 DEBUG:boto:Method: GET 2022-09-03T06:55:09.764 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:09.765 DEBUG:boto:Data: 2022-09-03T06:55:09.765 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.766 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.767 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.767 DEBUG:boto:Params: {} 2022-09-03T06:55:09.768 DEBUG:boto:Token: None 2022-09-03T06:55:09.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.773 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff2f58317dd4aab2-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.774 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.775 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.776 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.777 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.778 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:09.779 DEBUG:boto:Data: 2022-09-03T06:55:09.780 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.781 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.782 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.783 DEBUG:boto:Params: {} 2022-09-03T06:55:09.784 DEBUG:boto:Token: None 2022-09-03T06:55:09.784 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b 2022-09-03T06:55:09.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bsZCeXm1/vvs1zC87KSE705ymIE= 2022-09-03T06:55:09.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bsZCeXm1/vvs1zC87KSE705ymIE=', 'Content-Length': '0'} 2022-09-03T06:55:09.789 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a9fb2e34ab63859-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.790 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.790 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.791 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.791 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:09.792 DEBUG:boto:Data: 2022-09-03T06:55:09.792 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.794 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.794 DEBUG:boto:Params: {} 2022-09-03T06:55:09.795 DEBUG:boto:Token: None 2022-09-03T06:55:09.796 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b 2022-09-03T06:55:09.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bsZCeXm1/vvs1zC87KSE705ymIE= 2022-09-03T06:55:09.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bsZCeXm1/vvs1zC87KSE705ymIE=', 'Content-Length': '0'} 2022-09-03T06:55:09.800 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001708241752d2ce91-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.801 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:55:09.802 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.803 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.804 DEBUG:boto:Method: GET 2022-09-03T06:55:09.805 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:09.806 DEBUG:boto:Data: 2022-09-03T06:55:09.807 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.808 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.809 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.811 DEBUG:boto:Params: {} 2022-09-03T06:55:09.812 DEBUG:boto:Token: None 2022-09-03T06:55:09.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b 2022-09-03T06:55:09.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4uYYCetOMqBdanLkVLy+Bdd6tvE= 2022-09-03T06:55:09.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4uYYCetOMqBdanLkVLy+Bdd6tvE=', 'Content-Length': '0'} 2022-09-03T06:55:09.818 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c77aeb979019fd65-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.858 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:09.859 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:09.859 DEBUG:boto:Method: GET 2022-09-03T06:55:09.860 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:09.861 DEBUG:boto:Data: 2022-09-03T06:55:09.861 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.862 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.862 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.863 DEBUG:boto:Params: {} 2022-09-03T06:55:09.864 DEBUG:boto:Token: None 2022-09-03T06:55:09.864 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/b 2022-09-03T06:55:09.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4uYYCetOMqBdanLkVLy+Bdd6tvE= 2022-09-03T06:55:09.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4uYYCetOMqBdanLkVLy+Bdd6tvE=', 'Content-Length': '0'} 2022-09-03T06:55:09.867 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d2a5bb0ab2a1d917-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.868 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.869 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.869 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:09.870 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:09.870 DEBUG:boto:Method: GET 2022-09-03T06:55:09.871 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:09.871 DEBUG:boto:Data: 2022-09-03T06:55:09.872 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.872 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.873 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.873 DEBUG:boto:Params: {} 2022-09-03T06:55:09.875 DEBUG:boto:Token: None 2022-09-03T06:55:09.875 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.877 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.881 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000080945a219503819d-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.882 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.883 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.884 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.885 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:09.885 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:09.886 DEBUG:boto:Method: GET 2022-09-03T06:55:09.887 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:09.887 DEBUG:boto:Data: 2022-09-03T06:55:09.888 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.888 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.889 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.890 DEBUG:boto:Params: {} 2022-09-03T06:55:09.891 DEBUG:boto:Token: None 2022-09-03T06:55:09.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.892 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.895 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d21414ad0065e8fb-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.896 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.897 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:55:09.898 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:09.898 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:09.899 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.899 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.900 DEBUG:boto:Method: GET 2022-09-03T06:55:09.900 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:09.901 DEBUG:boto:Data: 2022-09-03T06:55:09.902 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.903 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.903 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.904 DEBUG:boto:Params: {} 2022-09-03T06:55:09.904 DEBUG:boto:Token: None 2022-09-03T06:55:09.905 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:09.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L2iCOMxCUL6VhgOvoqeNT3usS3U= 2022-09-03T06:55:09.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L2iCOMxCUL6VhgOvoqeNT3usS3U=', 'Content-Length': '0'} 2022-09-03T06:55:09.909 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004111fa15022c5269-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.910 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:09.911 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:09.911 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.912 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.913 DEBUG:boto:Method: GET 2022-09-03T06:55:09.913 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:09.914 DEBUG:boto:Data: 2022-09-03T06:55:09.914 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.915 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.916 DEBUG:boto:Params: {} 2022-09-03T06:55:09.917 DEBUG:boto:Token: None 2022-09-03T06:55:09.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:09.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L2iCOMxCUL6VhgOvoqeNT3usS3U= 2022-09-03T06:55:09.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L2iCOMxCUL6VhgOvoqeNT3usS3U=', 'Content-Length': '0'} 2022-09-03T06:55:09.921 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bca98a79cae143f9-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.921 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:09.922 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:09.923 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.923 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.924 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.924 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:09.925 DEBUG:boto:Data: 2022-09-03T06:55:09.925 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.926 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.927 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.927 DEBUG:boto:Params: {} 2022-09-03T06:55:09.928 DEBUG:boto:Token: None 2022-09-03T06:55:09.928 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:09.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:53atCs4wrxuEJODZlWmEgTwcUvA= 2022-09-03T06:55:09.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:53atCs4wrxuEJODZlWmEgTwcUvA=', 'Content-Length': '0'} 2022-09-03T06:55:09.932 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff90dd9901493181-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.932 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:09.933 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:09.933 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.934 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.935 DEBUG:boto:Method: HEAD 2022-09-03T06:55:09.935 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:09.936 DEBUG:boto:Data: 2022-09-03T06:55:09.936 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.937 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.937 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.938 DEBUG:boto:Params: {} 2022-09-03T06:55:09.938 DEBUG:boto:Token: None 2022-09-03T06:55:09.939 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:09.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:53atCs4wrxuEJODZlWmEgTwcUvA= 2022-09-03T06:55:09.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:53atCs4wrxuEJODZlWmEgTwcUvA=', 'Content-Length': '0'} 2022-09-03T06:55:09.944 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2859aa098c22be9-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.945 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:55:09.946 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:09.947 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:09.948 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.949 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.950 DEBUG:boto:Method: GET 2022-09-03T06:55:09.950 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:09.951 DEBUG:boto:Data: 2022-09-03T06:55:09.952 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.954 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.955 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.956 DEBUG:boto:Params: {} 2022-09-03T06:55:09.956 DEBUG:boto:Token: None 2022-09-03T06:55:09.957 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:09.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L2iCOMxCUL6VhgOvoqeNT3usS3U= 2022-09-03T06:55:09.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L2iCOMxCUL6VhgOvoqeNT3usS3U=', 'Content-Length': '0'} 2022-09-03T06:55:09.961 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000580059c41e0a9f84-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.961 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:09.962 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:09.963 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.963 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:09.964 DEBUG:boto:Method: GET 2022-09-03T06:55:09.964 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:09.965 DEBUG:boto:Data: 2022-09-03T06:55:09.965 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.966 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.967 DEBUG:boto:Params: {} 2022-09-03T06:55:09.968 DEBUG:boto:Token: None 2022-09-03T06:55:09.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:09.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L2iCOMxCUL6VhgOvoqeNT3usS3U= 2022-09-03T06:55:09.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L2iCOMxCUL6VhgOvoqeNT3usS3U=', 'Content-Length': '0'} 2022-09-03T06:55:09.971 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000041d8e23484738ad0-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.972 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.973 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.973 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:09.974 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:09.975 DEBUG:boto:Method: GET 2022-09-03T06:55:09.975 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:09.976 DEBUG:boto:Data: 2022-09-03T06:55:09.976 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.977 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:09.977 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.978 DEBUG:boto:Params: {} 2022-09-03T06:55:09.979 DEBUG:boto:Token: None 2022-09-03T06:55:09.979 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.983 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a351fd0daa33c31d-006312fa4d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.984 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:09.985 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:09.987 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:09.987 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:09.988 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:09.988 DEBUG:boto:Method: GET 2022-09-03T06:55:09.989 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:09.990 DEBUG:boto:Data: 2022-09-03T06:55:09.990 DEBUG:boto:Headers: {} 2022-09-03T06:55:09.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:09.991 DEBUG:boto:Port: 8000 2022-09-03T06:55:09.992 DEBUG:boto:Params: {} 2022-09-03T06:55:09.993 DEBUG:boto:Token: None 2022-09-03T06:55:09.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:09 GMT /avysmw-71/ 2022-09-03T06:55:09.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac= 2022-09-03T06:55:09.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x4i2m2NU6+zvqbgiQCg2+bl4jac=', 'Content-Length': '0'} 2022-09-03T06:55:09.998 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000010c5717fe227a1da-006312fa4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:09.999 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.000 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:10.001 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:10.001 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.002 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:10.002 DEBUG:boto:Data: 2022-09-03T06:55:10.003 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.003 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.004 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.005 DEBUG:boto:Params: {} 2022-09-03T06:55:10.005 DEBUG:boto:Token: None 2022-09-03T06:55:10.006 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/c 2022-09-03T06:55:10.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I00GQXar45NQjlCTR0XE0XvqAyk= 2022-09-03T06:55:10.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I00GQXar45NQjlCTR0XE0XvqAyk=', 'Content-Length': '0'} 2022-09-03T06:55:10.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074e27d45af94c85f-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.012 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:10.013 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:10.014 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.014 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:10.015 DEBUG:boto:Data: 2022-09-03T06:55:10.016 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.016 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.017 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.017 DEBUG:boto:Params: {} 2022-09-03T06:55:10.018 DEBUG:boto:Token: None 2022-09-03T06:55:10.019 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/c 2022-09-03T06:55:10.019 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I00GQXar45NQjlCTR0XE0XvqAyk= 2022-09-03T06:55:10.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I00GQXar45NQjlCTR0XE0XvqAyk=', 'Content-Length': '0'} 2022-09-03T06:55:10.023 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008caa03a4a90496e0-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.026 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:55:10.027 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:10.028 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:10.029 DEBUG:boto:Method: GET 2022-09-03T06:55:10.030 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:10.031 DEBUG:boto:Data: 2022-09-03T06:55:10.032 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.033 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.034 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.035 DEBUG:boto:Params: {} 2022-09-03T06:55:10.036 DEBUG:boto:Token: None 2022-09-03T06:55:10.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/c 2022-09-03T06:55:10.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Tkx19vBH4Qge8hrxb6XmrcdwiQ= 2022-09-03T06:55:10.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Tkx19vBH4Qge8hrxb6XmrcdwiQ=', 'Content-Length': '0'} 2022-09-03T06:55:10.040 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007d326a24fce5a80-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.041 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:10.042 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:10.042 DEBUG:boto:Method: GET 2022-09-03T06:55:10.043 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:10.043 DEBUG:boto:Data: 2022-09-03T06:55:10.044 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.046 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.047 DEBUG:boto:Params: {} 2022-09-03T06:55:10.048 DEBUG:boto:Token: None 2022-09-03T06:55:10.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/c 2022-09-03T06:55:10.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Tkx19vBH4Qge8hrxb6XmrcdwiQ= 2022-09-03T06:55:10.050 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Tkx19vBH4Qge8hrxb6XmrcdwiQ=', 'Content-Length': '0'} 2022-09-03T06:55:10.052 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003df9bc8235a80817-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.098 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.100 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.101 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:10.102 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:10.105 DEBUG:boto:Method: GET 2022-09-03T06:55:10.106 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:10.107 DEBUG:boto:Data: 2022-09-03T06:55:10.109 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.110 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.111 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.112 DEBUG:boto:Params: {} 2022-09-03T06:55:10.114 DEBUG:boto:Token: None 2022-09-03T06:55:10.115 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.140 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000820bc00b1aad751d-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.142 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.144 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.145 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.146 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:10.147 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:10.148 DEBUG:boto:Method: GET 2022-09-03T06:55:10.149 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:10.150 DEBUG:boto:Data: 2022-09-03T06:55:10.151 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.152 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.152 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.153 DEBUG:boto:Params: {} 2022-09-03T06:55:10.153 DEBUG:boto:Token: None 2022-09-03T06:55:10.154 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.159 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b44a518a2537f602-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.160 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.161 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:55:10.163 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.164 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.165 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.166 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.168 DEBUG:boto:Method: GET 2022-09-03T06:55:10.169 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:10.170 DEBUG:boto:Data: 2022-09-03T06:55:10.171 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.172 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.173 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.174 DEBUG:boto:Params: {} 2022-09-03T06:55:10.174 DEBUG:boto:Token: None 2022-09-03T06:55:10.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:10.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MZkwFiu2jOJLRukHNgG+vhEEqjk= 2022-09-03T06:55:10.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MZkwFiu2jOJLRukHNgG+vhEEqjk=', 'Content-Length': '0'} 2022-09-03T06:55:10.181 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb7920ef8ce1d766-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.183 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.184 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.185 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.186 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.187 DEBUG:boto:Method: GET 2022-09-03T06:55:10.188 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:10.189 DEBUG:boto:Data: 2022-09-03T06:55:10.189 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.191 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.192 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.193 DEBUG:boto:Params: {} 2022-09-03T06:55:10.194 DEBUG:boto:Token: None 2022-09-03T06:55:10.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:10.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MZkwFiu2jOJLRukHNgG+vhEEqjk= 2022-09-03T06:55:10.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MZkwFiu2jOJLRukHNgG+vhEEqjk=', 'Content-Length': '0'} 2022-09-03T06:55:10.201 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001fca219d9fa2b8e-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.202 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.203 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.204 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.205 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.206 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.206 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:10.207 DEBUG:boto:Data: 2022-09-03T06:55:10.208 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.209 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.210 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.211 DEBUG:boto:Params: {} 2022-09-03T06:55:10.212 DEBUG:boto:Token: None 2022-09-03T06:55:10.213 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:10.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cfzv5X/GtWky+vdbjW2HNGW6P48= 2022-09-03T06:55:10.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cfzv5X/GtWky+vdbjW2HNGW6P48=', 'Content-Length': '0'} 2022-09-03T06:55:10.222 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f81e9abbe20abefc-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.224 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.225 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.225 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.226 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.227 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.228 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:10.230 DEBUG:boto:Data: 2022-09-03T06:55:10.231 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.232 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.233 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.234 DEBUG:boto:Params: {} 2022-09-03T06:55:10.235 DEBUG:boto:Token: None 2022-09-03T06:55:10.236 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:10.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cfzv5X/GtWky+vdbjW2HNGW6P48= 2022-09-03T06:55:10.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cfzv5X/GtWky+vdbjW2HNGW6P48=', 'Content-Length': '0'} 2022-09-03T06:55:10.241 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c757e060173b58b6-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.243 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:55:10.243 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.244 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.245 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.246 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.247 DEBUG:boto:Method: GET 2022-09-03T06:55:10.248 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:10.249 DEBUG:boto:Data: 2022-09-03T06:55:10.251 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.252 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.253 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.254 DEBUG:boto:Params: {} 2022-09-03T06:55:10.255 DEBUG:boto:Token: None 2022-09-03T06:55:10.256 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:10.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MZkwFiu2jOJLRukHNgG+vhEEqjk= 2022-09-03T06:55:10.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MZkwFiu2jOJLRukHNgG+vhEEqjk=', 'Content-Length': '0'} 2022-09-03T06:55:10.262 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003565fd3e2012a7fd-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.264 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.265 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.266 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.267 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:10.268 DEBUG:boto:Method: GET 2022-09-03T06:55:10.268 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:10.269 DEBUG:boto:Data: 2022-09-03T06:55:10.270 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.270 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.271 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.272 DEBUG:boto:Params: {} 2022-09-03T06:55:10.273 DEBUG:boto:Token: None 2022-09-03T06:55:10.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:10.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MZkwFiu2jOJLRukHNgG+vhEEqjk= 2022-09-03T06:55:10.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MZkwFiu2jOJLRukHNgG+vhEEqjk=', 'Content-Length': '0'} 2022-09-03T06:55:10.277 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005090dc13ea7f3304-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.278 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.279 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.280 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:10.281 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:10.284 DEBUG:boto:Method: GET 2022-09-03T06:55:10.285 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:10.285 DEBUG:boto:Data: 2022-09-03T06:55:10.286 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.288 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.288 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.289 DEBUG:boto:Params: {} 2022-09-03T06:55:10.291 DEBUG:boto:Token: None 2022-09-03T06:55:10.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000646d106bd6ef70d3-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.303 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.304 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.305 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.306 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:10.307 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:10.308 DEBUG:boto:Method: GET 2022-09-03T06:55:10.309 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:10.310 DEBUG:boto:Data: 2022-09-03T06:55:10.311 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.313 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.314 DEBUG:boto:Params: {} 2022-09-03T06:55:10.315 DEBUG:boto:Token: None 2022-09-03T06:55:10.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.321 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000906c1ba591680e05-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.323 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.324 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.325 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.325 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.326 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:10.326 DEBUG:boto:Data: 2022-09-03T06:55:10.327 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.327 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.328 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.328 DEBUG:boto:Params: {} 2022-09-03T06:55:10.329 DEBUG:boto:Token: None 2022-09-03T06:55:10.330 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d 2022-09-03T06:55:10.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aCOJNPyzo2iSTzZrGpg5R9vzbwo= 2022-09-03T06:55:10.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aCOJNPyzo2iSTzZrGpg5R9vzbwo=', 'Content-Length': '0'} 2022-09-03T06:55:10.333 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ace101cd1cfecf0-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.334 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.334 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.335 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.336 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:10.337 DEBUG:boto:Data: 2022-09-03T06:55:10.339 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.341 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.342 DEBUG:boto:Params: {} 2022-09-03T06:55:10.343 DEBUG:boto:Token: None 2022-09-03T06:55:10.344 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d 2022-09-03T06:55:10.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aCOJNPyzo2iSTzZrGpg5R9vzbwo= 2022-09-03T06:55:10.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aCOJNPyzo2iSTzZrGpg5R9vzbwo=', 'Content-Length': '0'} 2022-09-03T06:55:10.350 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbc6feea1bfb83ae-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.353 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:55:10.354 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.356 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.357 DEBUG:boto:Method: GET 2022-09-03T06:55:10.358 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:10.360 DEBUG:boto:Data: 2022-09-03T06:55:10.361 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.362 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.363 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.365 DEBUG:boto:Params: {} 2022-09-03T06:55:10.366 DEBUG:boto:Token: None 2022-09-03T06:55:10.368 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d 2022-09-03T06:55:10.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HJc/pt863qvjL+cgwnv2TuphNv8= 2022-09-03T06:55:10.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HJc/pt863qvjL+cgwnv2TuphNv8=', 'Content-Length': '0'} 2022-09-03T06:55:10.374 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000095f6a2490e58a977-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.416 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:10.417 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:10.418 DEBUG:boto:Method: GET 2022-09-03T06:55:10.419 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:10.420 DEBUG:boto:Data: 2022-09-03T06:55:10.423 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.424 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.425 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.427 DEBUG:boto:Params: {} 2022-09-03T06:55:10.428 DEBUG:boto:Token: None 2022-09-03T06:55:10.429 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/d 2022-09-03T06:55:10.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HJc/pt863qvjL+cgwnv2TuphNv8= 2022-09-03T06:55:10.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HJc/pt863qvjL+cgwnv2TuphNv8=', 'Content-Length': '0'} 2022-09-03T06:55:10.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d00f303bea160b3f-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.446 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.447 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.448 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:10.449 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:10.450 DEBUG:boto:Method: GET 2022-09-03T06:55:10.451 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:10.452 DEBUG:boto:Data: 2022-09-03T06:55:10.456 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.458 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.459 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.460 DEBUG:boto:Params: {} 2022-09-03T06:55:10.460 DEBUG:boto:Token: None 2022-09-03T06:55:10.461 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000139614d2f60a53bb-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.470 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.471 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.471 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.472 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:10.472 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:10.473 DEBUG:boto:Method: GET 2022-09-03T06:55:10.474 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:10.474 DEBUG:boto:Data: 2022-09-03T06:55:10.475 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.476 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.476 DEBUG:boto:Params: {} 2022-09-03T06:55:10.477 DEBUG:boto:Token: None 2022-09-03T06:55:10.478 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.483 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004195c4f119138a88-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.484 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.484 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:55:10.485 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.486 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.487 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.487 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.488 DEBUG:boto:Method: GET 2022-09-03T06:55:10.488 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:10.489 DEBUG:boto:Data: 2022-09-03T06:55:10.490 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.490 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.491 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.491 DEBUG:boto:Params: {} 2022-09-03T06:55:10.492 DEBUG:boto:Token: None 2022-09-03T06:55:10.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:10.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hiEfa9egjIdhkVJjtio9OyhI4Xw= 2022-09-03T06:55:10.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hiEfa9egjIdhkVJjtio9OyhI4Xw=', 'Content-Length': '0'} 2022-09-03T06:55:10.499 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d1d0a0d868a0afc9-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.500 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.501 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.501 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.502 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.502 DEBUG:boto:Method: GET 2022-09-03T06:55:10.503 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:10.503 DEBUG:boto:Data: 2022-09-03T06:55:10.504 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.505 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.506 DEBUG:boto:Params: {} 2022-09-03T06:55:10.506 DEBUG:boto:Token: None 2022-09-03T06:55:10.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:10.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hiEfa9egjIdhkVJjtio9OyhI4Xw= 2022-09-03T06:55:10.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hiEfa9egjIdhkVJjtio9OyhI4Xw=', 'Content-Length': '0'} 2022-09-03T06:55:10.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fbb10d965b39429-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.512 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.512 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.513 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.513 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.514 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.514 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:10.515 DEBUG:boto:Data: 2022-09-03T06:55:10.516 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.516 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.517 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.518 DEBUG:boto:Params: {} 2022-09-03T06:55:10.518 DEBUG:boto:Token: None 2022-09-03T06:55:10.519 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:10.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i1czYnvXkTVGqxgS0hzGzEu36Z0= 2022-09-03T06:55:10.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i1czYnvXkTVGqxgS0hzGzEu36Z0=', 'Content-Length': '0'} 2022-09-03T06:55:10.522 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002894203c8750f3e8-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.524 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.525 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.526 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.527 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.528 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.530 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:10.531 DEBUG:boto:Data: 2022-09-03T06:55:10.533 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.535 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.536 DEBUG:boto:Params: {} 2022-09-03T06:55:10.537 DEBUG:boto:Token: None 2022-09-03T06:55:10.539 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:10.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i1czYnvXkTVGqxgS0hzGzEu36Z0= 2022-09-03T06:55:10.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i1czYnvXkTVGqxgS0hzGzEu36Z0=', 'Content-Length': '0'} 2022-09-03T06:55:10.544 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a1e39bd0449caaa-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.545 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:55:10.545 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.546 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.546 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.547 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.547 DEBUG:boto:Method: GET 2022-09-03T06:55:10.548 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:10.549 DEBUG:boto:Data: 2022-09-03T06:55:10.549 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.550 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.551 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.551 DEBUG:boto:Params: {} 2022-09-03T06:55:10.552 DEBUG:boto:Token: None 2022-09-03T06:55:10.553 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:10.553 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hiEfa9egjIdhkVJjtio9OyhI4Xw= 2022-09-03T06:55:10.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hiEfa9egjIdhkVJjtio9OyhI4Xw=', 'Content-Length': '0'} 2022-09-03T06:55:10.556 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000adab0f1e3c3177ff-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.598 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.600 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.601 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.602 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:10.603 DEBUG:boto:Method: GET 2022-09-03T06:55:10.604 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:10.605 DEBUG:boto:Data: 2022-09-03T06:55:10.606 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.607 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.608 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.609 DEBUG:boto:Params: {} 2022-09-03T06:55:10.610 DEBUG:boto:Token: None 2022-09-03T06:55:10.611 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:10.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hiEfa9egjIdhkVJjtio9OyhI4Xw= 2022-09-03T06:55:10.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hiEfa9egjIdhkVJjtio9OyhI4Xw=', 'Content-Length': '0'} 2022-09-03T06:55:10.619 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f017350f46f92ee-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.621 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.622 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.623 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:10.624 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:10.624 DEBUG:boto:Method: GET 2022-09-03T06:55:10.626 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:10.630 DEBUG:boto:Data: 2022-09-03T06:55:10.631 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.632 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.633 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.634 DEBUG:boto:Params: {} 2022-09-03T06:55:10.635 DEBUG:boto:Token: None 2022-09-03T06:55:10.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.642 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000055bc98714a8783d9-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.643 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.646 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.647 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.649 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:10.649 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:10.650 DEBUG:boto:Method: GET 2022-09-03T06:55:10.651 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:10.651 DEBUG:boto:Data: 2022-09-03T06:55:10.652 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.653 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.653 DEBUG:boto:Params: {} 2022-09-03T06:55:10.654 DEBUG:boto:Token: None 2022-09-03T06:55:10.655 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.660 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000097b4a01d482744e-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.660 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.662 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.662 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.663 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.663 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:10.664 DEBUG:boto:Data: 2022-09-03T06:55:10.666 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.666 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.667 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.668 DEBUG:boto:Params: {} 2022-09-03T06:55:10.670 DEBUG:boto:Token: None 2022-09-03T06:55:10.671 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e 2022-09-03T06:55:10.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IDUnltKPysQUGh9t7fpPiW0+jaU= 2022-09-03T06:55:10.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IDUnltKPysQUGh9t7fpPiW0+jaU=', 'Content-Length': '0'} 2022-09-03T06:55:10.676 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006252c26645681094-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.677 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.677 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.678 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.679 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:10.679 DEBUG:boto:Data: 2022-09-03T06:55:10.680 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.680 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.681 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.682 DEBUG:boto:Params: {} 2022-09-03T06:55:10.683 DEBUG:boto:Token: None 2022-09-03T06:55:10.685 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e 2022-09-03T06:55:10.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IDUnltKPysQUGh9t7fpPiW0+jaU= 2022-09-03T06:55:10.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IDUnltKPysQUGh9t7fpPiW0+jaU=', 'Content-Length': '0'} 2022-09-03T06:55:10.688 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e79c5a97f591b7a5-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.689 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:55:10.690 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.690 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.691 DEBUG:boto:Method: GET 2022-09-03T06:55:10.692 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:10.692 DEBUG:boto:Data: 2022-09-03T06:55:10.694 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.694 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.696 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.697 DEBUG:boto:Params: {} 2022-09-03T06:55:10.698 DEBUG:boto:Token: None 2022-09-03T06:55:10.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e 2022-09-03T06:55:10.701 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BY+XyamLfZSGXF3IHhnH52tMr3M= 2022-09-03T06:55:10.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BY+XyamLfZSGXF3IHhnH52tMr3M=', 'Content-Length': '0'} 2022-09-03T06:55:10.705 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ec098db71e4e324-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.706 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:10.707 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:10.707 DEBUG:boto:Method: GET 2022-09-03T06:55:10.709 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:10.710 DEBUG:boto:Data: 2022-09-03T06:55:10.711 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.711 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.712 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.712 DEBUG:boto:Params: {} 2022-09-03T06:55:10.713 DEBUG:boto:Token: None 2022-09-03T06:55:10.714 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/e 2022-09-03T06:55:10.714 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BY+XyamLfZSGXF3IHhnH52tMr3M= 2022-09-03T06:55:10.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BY+XyamLfZSGXF3IHhnH52tMr3M=', 'Content-Length': '0'} 2022-09-03T06:55:10.717 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f86d1c53a50cfd3d-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.718 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.719 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.719 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:10.720 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:10.720 DEBUG:boto:Method: GET 2022-09-03T06:55:10.721 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:10.721 DEBUG:boto:Data: 2022-09-03T06:55:10.722 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.722 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.723 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.723 DEBUG:boto:Params: {} 2022-09-03T06:55:10.724 DEBUG:boto:Token: None 2022-09-03T06:55:10.725 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.728 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085c0032f6b248ef7-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.729 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.731 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.731 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.732 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:10.733 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:10.734 DEBUG:boto:Method: GET 2022-09-03T06:55:10.735 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:10.736 DEBUG:boto:Data: 2022-09-03T06:55:10.737 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.739 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.740 DEBUG:boto:Params: {} 2022-09-03T06:55:10.741 DEBUG:boto:Token: None 2022-09-03T06:55:10.742 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.747 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000171605957139770b-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.748 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.750 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:55:10.751 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:10.752 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:10.753 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.753 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.754 DEBUG:boto:Method: GET 2022-09-03T06:55:10.754 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:10.755 DEBUG:boto:Data: 2022-09-03T06:55:10.756 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.756 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.757 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.757 DEBUG:boto:Params: {} 2022-09-03T06:55:10.758 DEBUG:boto:Token: None 2022-09-03T06:55:10.759 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:10.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P+0Sg0BHCmQDq7seFt/MHPpuOcg= 2022-09-03T06:55:10.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P+0Sg0BHCmQDq7seFt/MHPpuOcg=', 'Content-Length': '0'} 2022-09-03T06:55:10.762 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c12a8d1a1c846e24-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.763 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:10.764 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:10.765 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.765 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.766 DEBUG:boto:Method: GET 2022-09-03T06:55:10.768 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:10.768 DEBUG:boto:Data: 2022-09-03T06:55:10.770 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.772 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.773 DEBUG:boto:Params: {} 2022-09-03T06:55:10.774 DEBUG:boto:Token: None 2022-09-03T06:55:10.774 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:10.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P+0Sg0BHCmQDq7seFt/MHPpuOcg= 2022-09-03T06:55:10.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P+0Sg0BHCmQDq7seFt/MHPpuOcg=', 'Content-Length': '0'} 2022-09-03T06:55:10.778 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f349db6d1ec9a55-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.778 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:10.779 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:10.779 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.780 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.781 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.781 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:10.783 DEBUG:boto:Data: 2022-09-03T06:55:10.784 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.785 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.785 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.786 DEBUG:boto:Params: {} 2022-09-03T06:55:10.788 DEBUG:boto:Token: None 2022-09-03T06:55:10.789 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:10.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4rjLguiYNUD7tdQYQvxKryqmRgA= 2022-09-03T06:55:10.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4rjLguiYNUD7tdQYQvxKryqmRgA=', 'Content-Length': '0'} 2022-09-03T06:55:10.797 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fabbddc09ce1ff8-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.799 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:10.799 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:10.800 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.801 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.802 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.803 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:10.804 DEBUG:boto:Data: 2022-09-03T06:55:10.805 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.806 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.807 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.807 DEBUG:boto:Params: {} 2022-09-03T06:55:10.808 DEBUG:boto:Token: None 2022-09-03T06:55:10.809 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:10.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4rjLguiYNUD7tdQYQvxKryqmRgA= 2022-09-03T06:55:10.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4rjLguiYNUD7tdQYQvxKryqmRgA=', 'Content-Length': '0'} 2022-09-03T06:55:10.814 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003398297dc73830a4-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.816 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:55:10.817 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:10.818 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:10.818 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.819 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.820 DEBUG:boto:Method: GET 2022-09-03T06:55:10.821 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:10.821 DEBUG:boto:Data: 2022-09-03T06:55:10.822 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.823 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.823 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.824 DEBUG:boto:Params: {} 2022-09-03T06:55:10.825 DEBUG:boto:Token: None 2022-09-03T06:55:10.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:10.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P+0Sg0BHCmQDq7seFt/MHPpuOcg= 2022-09-03T06:55:10.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P+0Sg0BHCmQDq7seFt/MHPpuOcg=', 'Content-Length': '0'} 2022-09-03T06:55:10.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfdd83377030ba3f-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.870 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:10.871 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:10.871 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.872 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:10.873 DEBUG:boto:Method: GET 2022-09-03T06:55:10.873 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:10.874 DEBUG:boto:Data: 2022-09-03T06:55:10.874 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.876 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.876 DEBUG:boto:Params: {} 2022-09-03T06:55:10.877 DEBUG:boto:Token: None 2022-09-03T06:55:10.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:10.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P+0Sg0BHCmQDq7seFt/MHPpuOcg= 2022-09-03T06:55:10.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P+0Sg0BHCmQDq7seFt/MHPpuOcg=', 'Content-Length': '0'} 2022-09-03T06:55:10.883 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020d9f8d68958567f-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.884 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.885 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.886 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:10.887 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:10.887 DEBUG:boto:Method: GET 2022-09-03T06:55:10.888 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:10.889 DEBUG:boto:Data: 2022-09-03T06:55:10.889 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.891 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.891 DEBUG:boto:Params: {} 2022-09-03T06:55:10.892 DEBUG:boto:Token: None 2022-09-03T06:55:10.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.898 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003301caa92939c777-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.899 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.900 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:10.901 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:10.902 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:10.903 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:10.905 DEBUG:boto:Method: GET 2022-09-03T06:55:10.906 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:10.907 DEBUG:boto:Data: 2022-09-03T06:55:10.908 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.909 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.910 DEBUG:boto:Params: {} 2022-09-03T06:55:10.910 DEBUG:boto:Token: None 2022-09-03T06:55:10.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/ 2022-09-03T06:55:10.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8= 2022-09-03T06:55:10.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uv0GplayLHiU7otqDkbDYLTYVq8=', 'Content-Length': '0'} 2022-09-03T06:55:10.916 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ca76243e7ab5d5ba-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.916 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:10.917 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:10.918 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:10.919 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.919 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:10.920 DEBUG:boto:Data: 2022-09-03T06:55:10.920 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.921 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.921 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.923 DEBUG:boto:Params: {} 2022-09-03T06:55:10.924 DEBUG:boto:Token: None 2022-09-03T06:55:10.925 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/f 2022-09-03T06:55:10.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xrv+mUhyJC8d66J5sHA3miTejjo= 2022-09-03T06:55:10.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xrv+mUhyJC8d66J5sHA3miTejjo=', 'Content-Length': '0'} 2022-09-03T06:55:10.936 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000856ccc883e1f0bb6-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.938 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:10.939 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:10.940 DEBUG:boto:Method: HEAD 2022-09-03T06:55:10.941 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:10.942 DEBUG:boto:Data: 2022-09-03T06:55:10.943 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:10.945 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.946 DEBUG:boto:Params: {} 2022-09-03T06:55:10.947 DEBUG:boto:Token: None 2022-09-03T06:55:10.948 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/f 2022-09-03T06:55:10.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xrv+mUhyJC8d66J5sHA3miTejjo= 2022-09-03T06:55:10.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xrv+mUhyJC8d66J5sHA3miTejjo=', 'Content-Length': '0'} 2022-09-03T06:55:10.953 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075d749693acdf61d-006312fa4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:10.954 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:55:10.955 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:10.956 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:10.957 DEBUG:boto:Method: GET 2022-09-03T06:55:10.958 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:10.959 DEBUG:boto:Data: 2022-09-03T06:55:10.960 DEBUG:boto:Headers: {} 2022-09-03T06:55:10.961 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:10.962 DEBUG:boto:Port: 8000 2022-09-03T06:55:10.963 DEBUG:boto:Params: {} 2022-09-03T06:55:10.964 DEBUG:boto:Token: None 2022-09-03T06:55:10.966 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:10 GMT /avysmw-71/f 2022-09-03T06:55:10.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/QhjJ10628gPAA3P4qH+ryHQgkw= 2022-09-03T06:55:10.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/QhjJ10628gPAA3P4qH+ryHQgkw=', 'Content-Length': '0'} 2022-09-03T06:55:10.970 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009dc38490a75cf248-006312fa4e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.018 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:11.019 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:11.020 DEBUG:boto:Method: GET 2022-09-03T06:55:11.020 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:11.021 DEBUG:boto:Data: 2022-09-03T06:55:11.025 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.027 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.028 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.029 DEBUG:boto:Params: {} 2022-09-03T06:55:11.030 DEBUG:boto:Token: None 2022-09-03T06:55:11.031 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/f 2022-09-03T06:55:11.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zAiVWRxuAvQANrg+OchBIwo6GuY= 2022-09-03T06:55:11.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zAiVWRxuAvQANrg+OchBIwo6GuY=', 'Content-Length': '0'} 2022-09-03T06:55:11.036 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096a5c4525f635d6a-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.037 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.038 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.039 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:11.040 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:11.041 DEBUG:boto:Method: GET 2022-09-03T06:55:11.042 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:11.042 DEBUG:boto:Data: 2022-09-03T06:55:11.043 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.044 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.045 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.046 DEBUG:boto:Params: {} 2022-09-03T06:55:11.046 DEBUG:boto:Token: None 2022-09-03T06:55:11.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.051 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000052c877928323e35b-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.052 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.053 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.053 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.054 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:11.055 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:11.055 DEBUG:boto:Method: GET 2022-09-03T06:55:11.056 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:11.056 DEBUG:boto:Data: 2022-09-03T06:55:11.057 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.058 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.059 DEBUG:boto:Params: {} 2022-09-03T06:55:11.060 DEBUG:boto:Token: None 2022-09-03T06:55:11.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.064 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c43abba2c701ffe-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.065 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.066 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:55:11.066 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.067 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.068 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.068 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.069 DEBUG:boto:Method: GET 2022-09-03T06:55:11.069 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:11.070 DEBUG:boto:Data: 2022-09-03T06:55:11.071 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.071 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.072 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.073 DEBUG:boto:Params: {} 2022-09-03T06:55:11.074 DEBUG:boto:Token: None 2022-09-03T06:55:11.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:11.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cKh0u1RzhqDFp0OwHuTcoyXnuM4= 2022-09-03T06:55:11.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cKh0u1RzhqDFp0OwHuTcoyXnuM4=', 'Content-Length': '0'} 2022-09-03T06:55:11.078 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006df9afbd0f1c3cae-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.079 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.080 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.081 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.081 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.082 DEBUG:boto:Method: GET 2022-09-03T06:55:11.083 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:11.083 DEBUG:boto:Data: 2022-09-03T06:55:11.084 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.084 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.085 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.085 DEBUG:boto:Params: {} 2022-09-03T06:55:11.086 DEBUG:boto:Token: None 2022-09-03T06:55:11.087 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:11.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cKh0u1RzhqDFp0OwHuTcoyXnuM4= 2022-09-03T06:55:11.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cKh0u1RzhqDFp0OwHuTcoyXnuM4=', 'Content-Length': '0'} 2022-09-03T06:55:11.091 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003dba20929679661b-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.092 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.092 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.093 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.093 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.094 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.094 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:11.095 DEBUG:boto:Data: 2022-09-03T06:55:11.097 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.097 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.098 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.099 DEBUG:boto:Params: {} 2022-09-03T06:55:11.100 DEBUG:boto:Token: None 2022-09-03T06:55:11.100 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:11.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eBckRHDYu2hKHj2dFJXzkVnFtps= 2022-09-03T06:55:11.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eBckRHDYu2hKHj2dFJXzkVnFtps=', 'Content-Length': '0'} 2022-09-03T06:55:11.104 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f796e9d14b499b28-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.105 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.106 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.107 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.108 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.108 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.110 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:11.111 DEBUG:boto:Data: 2022-09-03T06:55:11.111 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.112 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.113 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.113 DEBUG:boto:Params: {} 2022-09-03T06:55:11.114 DEBUG:boto:Token: None 2022-09-03T06:55:11.115 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:11.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eBckRHDYu2hKHj2dFJXzkVnFtps= 2022-09-03T06:55:11.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eBckRHDYu2hKHj2dFJXzkVnFtps=', 'Content-Length': '0'} 2022-09-03T06:55:11.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3b523c3a3ac2688-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.120 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:55:11.121 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.123 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.124 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.125 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.126 DEBUG:boto:Method: GET 2022-09-03T06:55:11.126 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:11.127 DEBUG:boto:Data: 2022-09-03T06:55:11.127 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.128 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.128 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.129 DEBUG:boto:Params: {} 2022-09-03T06:55:11.130 DEBUG:boto:Token: None 2022-09-03T06:55:11.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:11.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cKh0u1RzhqDFp0OwHuTcoyXnuM4= 2022-09-03T06:55:11.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cKh0u1RzhqDFp0OwHuTcoyXnuM4=', 'Content-Length': '0'} 2022-09-03T06:55:11.134 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c05220c41e851a6a-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.178 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.179 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.180 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.181 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:11.182 DEBUG:boto:Method: GET 2022-09-03T06:55:11.183 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:11.184 DEBUG:boto:Data: 2022-09-03T06:55:11.185 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.186 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.187 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.188 DEBUG:boto:Params: {} 2022-09-03T06:55:11.189 DEBUG:boto:Token: None 2022-09-03T06:55:11.190 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:11.191 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cKh0u1RzhqDFp0OwHuTcoyXnuM4= 2022-09-03T06:55:11.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cKh0u1RzhqDFp0OwHuTcoyXnuM4=', 'Content-Length': '0'} 2022-09-03T06:55:11.195 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b463d2553983af7-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.197 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.197 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.198 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:11.199 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:11.199 DEBUG:boto:Method: GET 2022-09-03T06:55:11.200 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:11.200 DEBUG:boto:Data: 2022-09-03T06:55:11.201 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.201 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.202 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.203 DEBUG:boto:Params: {} 2022-09-03T06:55:11.204 DEBUG:boto:Token: None 2022-09-03T06:55:11.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.211 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a16df66ceee573a2-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.212 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.213 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.214 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.215 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:11.215 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:11.216 DEBUG:boto:Method: GET 2022-09-03T06:55:11.217 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:11.218 DEBUG:boto:Data: 2022-09-03T06:55:11.219 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.221 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.222 DEBUG:boto:Params: {} 2022-09-03T06:55:11.223 DEBUG:boto:Token: None 2022-09-03T06:55:11.224 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.230 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d10bab8d66500b7-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.231 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.233 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.234 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.235 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.236 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:11.237 DEBUG:boto:Data: 2022-09-03T06:55:11.237 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.239 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.241 DEBUG:boto:Params: {} 2022-09-03T06:55:11.242 DEBUG:boto:Token: None 2022-09-03T06:55:11.243 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g 2022-09-03T06:55:11.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v8hEcUv98b/cxpkp55+oXg36/uE= 2022-09-03T06:55:11.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v8hEcUv98b/cxpkp55+oXg36/uE=', 'Content-Length': '0'} 2022-09-03T06:55:11.247 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bc6b534562aeee0-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.249 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.250 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.251 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.253 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:11.254 DEBUG:boto:Data: 2022-09-03T06:55:11.255 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.257 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.258 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.259 DEBUG:boto:Params: {} 2022-09-03T06:55:11.260 DEBUG:boto:Token: None 2022-09-03T06:55:11.261 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g 2022-09-03T06:55:11.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v8hEcUv98b/cxpkp55+oXg36/uE= 2022-09-03T06:55:11.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v8hEcUv98b/cxpkp55+oXg36/uE=', 'Content-Length': '0'} 2022-09-03T06:55:11.282 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db5aca565dcc31fe-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.283 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:55:11.284 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.284 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.285 DEBUG:boto:Method: GET 2022-09-03T06:55:11.285 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:11.286 DEBUG:boto:Data: 2022-09-03T06:55:11.286 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.287 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.288 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.289 DEBUG:boto:Params: {} 2022-09-03T06:55:11.290 DEBUG:boto:Token: None 2022-09-03T06:55:11.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g 2022-09-03T06:55:11.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kfMBfGPDne+bi9MdMXUU0o+Ppdk= 2022-09-03T06:55:11.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kfMBfGPDne+bi9MdMXUU0o+Ppdk=', 'Content-Length': '0'} 2022-09-03T06:55:11.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d0142afa77d601d5-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.298 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:11.299 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:11.300 DEBUG:boto:Method: GET 2022-09-03T06:55:11.300 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:11.301 DEBUG:boto:Data: 2022-09-03T06:55:11.301 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.302 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.303 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.304 DEBUG:boto:Params: {} 2022-09-03T06:55:11.304 DEBUG:boto:Token: None 2022-09-03T06:55:11.305 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/g 2022-09-03T06:55:11.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kfMBfGPDne+bi9MdMXUU0o+Ppdk= 2022-09-03T06:55:11.308 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kfMBfGPDne+bi9MdMXUU0o+Ppdk=', 'Content-Length': '0'} 2022-09-03T06:55:11.312 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c04c4a4ca030e26-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.314 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.315 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.316 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:11.317 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:11.319 DEBUG:boto:Method: GET 2022-09-03T06:55:11.320 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:11.321 DEBUG:boto:Data: 2022-09-03T06:55:11.322 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.323 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.325 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.325 DEBUG:boto:Params: {} 2022-09-03T06:55:11.328 DEBUG:boto:Token: None 2022-09-03T06:55:11.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.335 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c255ad8c0496264b-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.378 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.380 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.382 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.383 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:11.385 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:11.387 DEBUG:boto:Method: GET 2022-09-03T06:55:11.389 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:11.393 DEBUG:boto:Data: 2022-09-03T06:55:11.394 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.397 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.399 DEBUG:boto:Params: {} 2022-09-03T06:55:11.400 DEBUG:boto:Token: None 2022-09-03T06:55:11.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.410 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc98f0ae07620f0f-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.411 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.412 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:55:11.414 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.414 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.415 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.416 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.416 DEBUG:boto:Method: GET 2022-09-03T06:55:11.417 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:11.417 DEBUG:boto:Data: 2022-09-03T06:55:11.418 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.419 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.420 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.421 DEBUG:boto:Params: {} 2022-09-03T06:55:11.423 DEBUG:boto:Token: None 2022-09-03T06:55:11.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:11.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y+LmGlhPQUbxB7jNLv+mamMnGds= 2022-09-03T06:55:11.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y+LmGlhPQUbxB7jNLv+mamMnGds=', 'Content-Length': '0'} 2022-09-03T06:55:11.432 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000069adbcd273da00fd-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.445 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.447 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.447 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.448 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.448 DEBUG:boto:Method: GET 2022-09-03T06:55:11.449 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:11.450 DEBUG:boto:Data: 2022-09-03T06:55:11.450 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.451 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.452 DEBUG:boto:Params: {} 2022-09-03T06:55:11.453 DEBUG:boto:Token: None 2022-09-03T06:55:11.454 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:11.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y+LmGlhPQUbxB7jNLv+mamMnGds= 2022-09-03T06:55:11.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y+LmGlhPQUbxB7jNLv+mamMnGds=', 'Content-Length': '0'} 2022-09-03T06:55:11.458 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000223bd4f803471e19-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.459 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.460 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.461 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.462 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.463 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.464 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:11.465 DEBUG:boto:Data: 2022-09-03T06:55:11.466 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.467 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.468 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.469 DEBUG:boto:Params: {} 2022-09-03T06:55:11.470 DEBUG:boto:Token: None 2022-09-03T06:55:11.471 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:11.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3F3qzsr6nPODOfkBVzKWVFJoX90= 2022-09-03T06:55:11.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3F3qzsr6nPODOfkBVzKWVFJoX90=', 'Content-Length': '0'} 2022-09-03T06:55:11.476 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000536ad80cc97276eb-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.477 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.478 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.479 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.480 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.481 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.481 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:11.482 DEBUG:boto:Data: 2022-09-03T06:55:11.482 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.484 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.485 DEBUG:boto:Params: {} 2022-09-03T06:55:11.487 DEBUG:boto:Token: None 2022-09-03T06:55:11.489 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:11.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3F3qzsr6nPODOfkBVzKWVFJoX90= 2022-09-03T06:55:11.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3F3qzsr6nPODOfkBVzKWVFJoX90=', 'Content-Length': '0'} 2022-09-03T06:55:11.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bb48b11f55a91ee6-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.497 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:55:11.499 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.500 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.501 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.502 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.503 DEBUG:boto:Method: GET 2022-09-03T06:55:11.504 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:11.504 DEBUG:boto:Data: 2022-09-03T06:55:11.505 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.505 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.506 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.506 DEBUG:boto:Params: {} 2022-09-03T06:55:11.507 DEBUG:boto:Token: None 2022-09-03T06:55:11.508 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:11.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y+LmGlhPQUbxB7jNLv+mamMnGds= 2022-09-03T06:55:11.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y+LmGlhPQUbxB7jNLv+mamMnGds=', 'Content-Length': '0'} 2022-09-03T06:55:11.512 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003fa633e8f29c00ef-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.514 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.515 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.516 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.517 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:11.520 DEBUG:boto:Method: GET 2022-09-03T06:55:11.521 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:11.522 DEBUG:boto:Data: 2022-09-03T06:55:11.523 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.525 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.526 DEBUG:boto:Params: {} 2022-09-03T06:55:11.527 DEBUG:boto:Token: None 2022-09-03T06:55:11.528 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:11.532 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y+LmGlhPQUbxB7jNLv+mamMnGds= 2022-09-03T06:55:11.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y+LmGlhPQUbxB7jNLv+mamMnGds=', 'Content-Length': '0'} 2022-09-03T06:55:11.538 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039b32980dc07c0c5-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.605 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.607 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.608 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:11.609 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:11.610 DEBUG:boto:Method: GET 2022-09-03T06:55:11.611 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:11.612 DEBUG:boto:Data: 2022-09-03T06:55:11.612 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.613 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.613 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.614 DEBUG:boto:Params: {} 2022-09-03T06:55:11.615 DEBUG:boto:Token: None 2022-09-03T06:55:11.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.620 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000061ada4d2978b08c8-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.621 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.623 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.623 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.624 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:11.625 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:11.625 DEBUG:boto:Method: GET 2022-09-03T06:55:11.626 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:11.626 DEBUG:boto:Data: 2022-09-03T06:55:11.627 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.628 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.628 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.629 DEBUG:boto:Params: {} 2022-09-03T06:55:11.630 DEBUG:boto:Token: None 2022-09-03T06:55:11.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.638 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001aa00d7c192af873-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.639 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.640 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.641 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.641 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.642 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:11.643 DEBUG:boto:Data: 2022-09-03T06:55:11.644 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.644 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.645 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.645 DEBUG:boto:Params: {} 2022-09-03T06:55:11.646 DEBUG:boto:Token: None 2022-09-03T06:55:11.647 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h 2022-09-03T06:55:11.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kXaqP72uHnN5LDBSWAZ3nZ8oKTE= 2022-09-03T06:55:11.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kXaqP72uHnN5LDBSWAZ3nZ8oKTE=', 'Content-Length': '0'} 2022-09-03T06:55:11.653 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001cc403f39eaf5fb0-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.655 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.655 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.656 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.658 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:11.659 DEBUG:boto:Data: 2022-09-03T06:55:11.660 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.661 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.661 DEBUG:boto:Params: {} 2022-09-03T06:55:11.663 DEBUG:boto:Token: None 2022-09-03T06:55:11.663 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h 2022-09-03T06:55:11.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kXaqP72uHnN5LDBSWAZ3nZ8oKTE= 2022-09-03T06:55:11.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kXaqP72uHnN5LDBSWAZ3nZ8oKTE=', 'Content-Length': '0'} 2022-09-03T06:55:11.668 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5bd103847f76808-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.668 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:55:11.669 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.670 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.670 DEBUG:boto:Method: GET 2022-09-03T06:55:11.671 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:11.672 DEBUG:boto:Data: 2022-09-03T06:55:11.673 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.673 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.674 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.674 DEBUG:boto:Params: {} 2022-09-03T06:55:11.675 DEBUG:boto:Token: None 2022-09-03T06:55:11.676 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h 2022-09-03T06:55:11.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QfnGGA5yYfeA5w5BsIbb08gb+/0= 2022-09-03T06:55:11.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QfnGGA5yYfeA5w5BsIbb08gb+/0=', 'Content-Length': '0'} 2022-09-03T06:55:11.680 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000068d83674f25f924-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.681 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:11.682 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:11.683 DEBUG:boto:Method: GET 2022-09-03T06:55:11.684 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:11.685 DEBUG:boto:Data: 2022-09-03T06:55:11.686 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.688 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.689 DEBUG:boto:Params: {} 2022-09-03T06:55:11.690 DEBUG:boto:Token: None 2022-09-03T06:55:11.691 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/h 2022-09-03T06:55:11.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QfnGGA5yYfeA5w5BsIbb08gb+/0= 2022-09-03T06:55:11.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QfnGGA5yYfeA5w5BsIbb08gb+/0=', 'Content-Length': '0'} 2022-09-03T06:55:11.696 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d3ef8f68d9a69fd-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.698 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.698 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.699 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:11.700 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:11.700 DEBUG:boto:Method: GET 2022-09-03T06:55:11.701 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:11.714 DEBUG:boto:Data: 2022-09-03T06:55:11.715 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.715 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:11.716 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.716 DEBUG:boto:Params: {} 2022-09-03T06:55:11.717 DEBUG:boto:Token: None 2022-09-03T06:55:11.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.721 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000047e644e79dd3b1c3-006312fa4f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.722 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.723 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.724 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.724 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:11.725 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:11.725 DEBUG:boto:Method: GET 2022-09-03T06:55:11.726 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:11.727 DEBUG:boto:Data: 2022-09-03T06:55:11.727 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.728 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.729 DEBUG:boto:Params: {} 2022-09-03T06:55:11.729 DEBUG:boto:Token: None 2022-09-03T06:55:11.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE= 2022-09-03T06:55:11.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Lvly8iGFd86PA15xkbD9Iz0HaDE=', 'Content-Length': '0'} 2022-09-03T06:55:11.735 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007acf3d140c2aa9fe-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.736 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.737 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-71 zones={test-zone2, test-zone1} 2022-09-03T06:55:11.738 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-71 zones={test-zone3, test-zone1} 2022-09-03T06:55:11.738 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.739 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.739 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.740 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:55:11.741 DEBUG:boto:Data: 2022-09-03T06:55:11.741 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.742 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:11.742 DEBUG:boto:Port: 8001 2022-09-03T06:55:11.743 DEBUG:boto:Params: {} 2022-09-03T06:55:11.743 DEBUG:boto:Token: None 2022-09-03T06:55:11.744 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10yKy+ej9LACLKG0bgpdXLKNwR0= 2022-09-03T06:55:11.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10yKy+ej9LACLKG0bgpdXLKNwR0=', 'Content-Length': '0'} 2022-09-03T06:55:11.748 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000adee53493b852db2-006312fa4f-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.749 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.750 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.751 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.751 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:55:11.752 DEBUG:boto:Data: 2022-09-03T06:55:11.753 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.754 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.755 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.756 DEBUG:boto:Params: {} 2022-09-03T06:55:11.757 DEBUG:boto:Token: None 2022-09-03T06:55:11.758 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/ 2022-09-03T06:55:11.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10yKy+ej9LACLKG0bgpdXLKNwR0= 2022-09-03T06:55:11.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10yKy+ej9LACLKG0bgpdXLKNwR0=', 'Content-Length': '0'} 2022-09-03T06:55:11.763 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000006e807c2bb0b0e74-006312fa4f-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.764 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:55:11.766 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.767 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.768 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.769 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.769 DEBUG:boto:Method: GET 2022-09-03T06:55:11.770 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.772 DEBUG:boto:Data: 2022-09-03T06:55:11.773 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:11.775 DEBUG:boto:Port: 8001 2022-09-03T06:55:11.776 DEBUG:boto:Params: {} 2022-09-03T06:55:11.777 DEBUG:boto:Token: None 2022-09-03T06:55:11.779 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/?versions 2022-09-03T06:55:11.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Nc6xpgof4ccPxn5qWdLyRS8Gw0= 2022-09-03T06:55:11.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Nc6xpgof4ccPxn5qWdLyRS8Gw0=', 'Content-Length': '0'} 2022-09-03T06:55:11.785 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb6a35ae062e5bb9-006312fa4f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.786 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.787 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:55:11.788 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:55:11.788 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:55:11.789 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:55:11.789 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:55:11.790 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:55:11.791 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:55:11.792 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:55:11.792 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:55:11.794 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.795 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.796 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.796 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.797 DEBUG:boto:Method: GET 2022-09-03T06:55:11.798 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.799 DEBUG:boto:Data: 2022-09-03T06:55:11.800 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.801 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.803 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.804 DEBUG:boto:Params: {} 2022-09-03T06:55:11.805 DEBUG:boto:Token: None 2022-09-03T06:55:11.805 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/?versions 2022-09-03T06:55:11.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Nc6xpgof4ccPxn5qWdLyRS8Gw0= 2022-09-03T06:55:11.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Nc6xpgof4ccPxn5qWdLyRS8Gw0=', 'Content-Length': '0'} 2022-09-03T06:55:11.810 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e7364f51204d1fb6-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.811 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.813 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:55:11.814 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:55:11.815 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:55:11.816 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:55:11.816 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:55:11.817 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:55:11.818 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:55:11.819 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:55:11.819 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.820 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.820 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.821 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.821 DEBUG:boto:Method: GET 2022-09-03T06:55:11.822 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.822 DEBUG:boto:Data: 2022-09-03T06:55:11.823 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.823 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:11.824 DEBUG:boto:Port: 8001 2022-09-03T06:55:11.825 DEBUG:boto:Params: {} 2022-09-03T06:55:11.825 DEBUG:boto:Token: None 2022-09-03T06:55:11.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/?versions 2022-09-03T06:55:11.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Nc6xpgof4ccPxn5qWdLyRS8Gw0= 2022-09-03T06:55:11.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Nc6xpgof4ccPxn5qWdLyRS8Gw0=', 'Content-Length': '0'} 2022-09-03T06:55:11.835 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005dd66f51daf48a8d-006312fa4f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.836 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.838 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:11.839 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:11.840 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.841 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.842 DEBUG:boto:Method: GET 2022-09-03T06:55:11.843 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:11.844 DEBUG:boto:Data: 2022-09-03T06:55:11.845 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.845 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.846 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.847 DEBUG:boto:Params: {} 2022-09-03T06:55:11.848 DEBUG:boto:Token: None 2022-09-03T06:55:11.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/?versions 2022-09-03T06:55:11.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Nc6xpgof4ccPxn5qWdLyRS8Gw0= 2022-09-03T06:55:11.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Nc6xpgof4ccPxn5qWdLyRS8Gw0=', 'Content-Length': '0'} 2022-09-03T06:55:11.858 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000195baa743cb93ed6-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.859 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:11.861 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:55:11.862 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:11.862 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:11.863 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.864 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.865 DEBUG:boto:Method: GET 2022-09-03T06:55:11.867 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:11.868 DEBUG:boto:Data: 2022-09-03T06:55:11.868 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.869 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:11.870 DEBUG:boto:Port: 8001 2022-09-03T06:55:11.871 DEBUG:boto:Params: {} 2022-09-03T06:55:11.872 DEBUG:boto:Token: None 2022-09-03T06:55:11.873 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:11.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GO3ZKBCdwCuQWJNP3vsRHxXUv9I= 2022-09-03T06:55:11.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GO3ZKBCdwCuQWJNP3vsRHxXUv9I=', 'Content-Length': '0'} 2022-09-03T06:55:11.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc601ca5c2745a76-006312fa4f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.879 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:11.880 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:11.880 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.881 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.881 DEBUG:boto:Method: GET 2022-09-03T06:55:11.882 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:11.883 DEBUG:boto:Data: 2022-09-03T06:55:11.883 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.884 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.885 DEBUG:boto:Params: {} 2022-09-03T06:55:11.885 DEBUG:boto:Token: None 2022-09-03T06:55:11.886 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:11.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GO3ZKBCdwCuQWJNP3vsRHxXUv9I= 2022-09-03T06:55:11.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GO3ZKBCdwCuQWJNP3vsRHxXUv9I=', 'Content-Length': '0'} 2022-09-03T06:55:11.894 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e7a1a8e9ae2e0fb9-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.896 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:11.897 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:11.900 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.901 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.902 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.903 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:11.904 DEBUG:boto:Data: 2022-09-03T06:55:11.905 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:11.907 DEBUG:boto:Port: 8001 2022-09-03T06:55:11.907 DEBUG:boto:Params: {} 2022-09-03T06:55:11.909 DEBUG:boto:Token: None 2022-09-03T06:55:11.910 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:11.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ytcG7pIIwyy4PKE2l0ryd9cIvfU= 2022-09-03T06:55:11.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ytcG7pIIwyy4PKE2l0ryd9cIvfU=', 'Content-Length': '0'} 2022-09-03T06:55:11.916 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032b926c3685bf656-006312fa4f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.920 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:11.921 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:11.922 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.923 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.924 DEBUG:boto:Method: HEAD 2022-09-03T06:55:11.926 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:11.926 DEBUG:boto:Data: 2022-09-03T06:55:11.928 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.929 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:11.935 DEBUG:boto:Port: 8000 2022-09-03T06:55:11.936 DEBUG:boto:Params: {} 2022-09-03T06:55:11.937 DEBUG:boto:Token: None 2022-09-03T06:55:11.939 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:11.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ytcG7pIIwyy4PKE2l0ryd9cIvfU= 2022-09-03T06:55:11.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ytcG7pIIwyy4PKE2l0ryd9cIvfU=', 'Content-Length': '0'} 2022-09-03T06:55:11.943 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf2000a79f98f9cc-006312fa4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:11.944 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:55:11.945 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:11.947 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:11.947 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.948 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:11.949 DEBUG:boto:Method: GET 2022-09-03T06:55:11.950 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:11.951 DEBUG:boto:Data: 2022-09-03T06:55:11.952 DEBUG:boto:Headers: {} 2022-09-03T06:55:11.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:11.954 DEBUG:boto:Port: 8001 2022-09-03T06:55:11.955 DEBUG:boto:Params: {} 2022-09-03T06:55:11.956 DEBUG:boto:Token: None 2022-09-03T06:55:11.958 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:11 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:11.959 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GO3ZKBCdwCuQWJNP3vsRHxXUv9I= 2022-09-03T06:55:11.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GO3ZKBCdwCuQWJNP3vsRHxXUv9I=', 'Content-Length': '0'} 2022-09-03T06:55:11.962 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e6659dd399269819-006312fa4f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.002 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:12.011 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:12.011 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:12.012 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:12.013 DEBUG:boto:Method: GET 2022-09-03T06:55:12.013 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:12.014 DEBUG:boto:Data: 2022-09-03T06:55:12.014 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.015 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.016 DEBUG:boto:Params: {} 2022-09-03T06:55:12.016 DEBUG:boto:Token: None 2022-09-03T06:55:12.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:12.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cTbS0PWrAjIJtwVic8BfdjnuC4g= 2022-09-03T06:55:12.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cTbS0PWrAjIJtwVic8BfdjnuC4g=', 'Content-Length': '0'} 2022-09-03T06:55:12.021 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c04a9313bef0443-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.022 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.023 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.023 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:12.024 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:12.024 DEBUG:boto:Method: GET 2022-09-03T06:55:12.025 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:12.025 DEBUG:boto:Data: 2022-09-03T06:55:12.026 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.027 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.028 DEBUG:boto:Params: {} 2022-09-03T06:55:12.028 DEBUG:boto:Token: None 2022-09-03T06:55:12.029 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.033 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058224ad906380eb9-006312fa50-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.034 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.035 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.036 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.036 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:12.037 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:12.037 DEBUG:boto:Method: GET 2022-09-03T06:55:12.038 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:12.039 DEBUG:boto:Data: 2022-09-03T06:55:12.039 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.040 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.040 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.041 DEBUG:boto:Params: {} 2022-09-03T06:55:12.042 DEBUG:boto:Token: None 2022-09-03T06:55:12.043 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.044 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.047 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f6d87e84236bb99-006312fa50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.048 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.049 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:12.049 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:12.050 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.050 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:12.051 DEBUG:boto:Data: 2022-09-03T06:55:12.051 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.052 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.052 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.053 DEBUG:boto:Params: {} 2022-09-03T06:55:12.054 DEBUG:boto:Token: None 2022-09-03T06:55:12.054 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/a 2022-09-03T06:55:12.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MpD2/FFMYUOFgWmBCIlROz487Vk= 2022-09-03T06:55:12.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MpD2/FFMYUOFgWmBCIlROz487Vk=', 'Content-Length': '0'} 2022-09-03T06:55:12.057 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000614211326be73a74-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.058 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:12.059 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:12.059 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.060 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:12.061 DEBUG:boto:Data: 2022-09-03T06:55:12.062 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.063 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.063 DEBUG:boto:Params: {} 2022-09-03T06:55:12.064 DEBUG:boto:Token: None 2022-09-03T06:55:12.065 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/a 2022-09-03T06:55:12.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MpD2/FFMYUOFgWmBCIlROz487Vk= 2022-09-03T06:55:12.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MpD2/FFMYUOFgWmBCIlROz487Vk=', 'Content-Length': '0'} 2022-09-03T06:55:12.068 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e0654ef6b4b7c2d-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.069 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:55:12.069 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:12.070 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:12.071 DEBUG:boto:Method: GET 2022-09-03T06:55:12.071 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:12.072 DEBUG:boto:Data: 2022-09-03T06:55:12.072 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.073 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.074 DEBUG:boto:Params: {} 2022-09-03T06:55:12.075 DEBUG:boto:Token: None 2022-09-03T06:55:12.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/a 2022-09-03T06:55:12.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i6N+7KQp+UFSmkSOx9ne+txjLKM= 2022-09-03T06:55:12.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i6N+7KQp+UFSmkSOx9ne+txjLKM=', 'Content-Length': '0'} 2022-09-03T06:55:12.079 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf5a4fba26ea9ef1-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.080 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:12.080 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:12.081 DEBUG:boto:Method: GET 2022-09-03T06:55:12.082 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:12.082 DEBUG:boto:Data: 2022-09-03T06:55:12.083 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.083 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.084 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.085 DEBUG:boto:Params: {} 2022-09-03T06:55:12.085 DEBUG:boto:Token: None 2022-09-03T06:55:12.086 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/a 2022-09-03T06:55:12.087 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i6N+7KQp+UFSmkSOx9ne+txjLKM= 2022-09-03T06:55:12.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i6N+7KQp+UFSmkSOx9ne+txjLKM=', 'Content-Length': '0'} 2022-09-03T06:55:12.090 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e0993e27571f705-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.092 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.092 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.093 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:12.093 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:12.094 DEBUG:boto:Method: GET 2022-09-03T06:55:12.095 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:12.095 DEBUG:boto:Data: 2022-09-03T06:55:12.096 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.097 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.098 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.099 DEBUG:boto:Params: {} 2022-09-03T06:55:12.100 DEBUG:boto:Token: None 2022-09-03T06:55:12.101 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.107 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005f4c47091467cd98-006312fa50-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.108 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.109 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.110 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.110 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:12.111 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:12.112 DEBUG:boto:Method: GET 2022-09-03T06:55:12.113 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:12.113 DEBUG:boto:Data: 2022-09-03T06:55:12.114 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.115 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.115 DEBUG:boto:Params: {} 2022-09-03T06:55:12.116 DEBUG:boto:Token: None 2022-09-03T06:55:12.117 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.120 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6e068ce9282af8d-006312fa50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.162 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.164 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:55:12.164 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.165 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.165 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.166 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.166 DEBUG:boto:Method: GET 2022-09-03T06:55:12.167 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:12.168 DEBUG:boto:Data: 2022-09-03T06:55:12.169 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.169 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.170 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.171 DEBUG:boto:Params: {} 2022-09-03T06:55:12.171 DEBUG:boto:Token: None 2022-09-03T06:55:12.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:12.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iD7aCJze88gVHKks3Vqy/WkkUZk= 2022-09-03T06:55:12.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iD7aCJze88gVHKks3Vqy/WkkUZk=', 'Content-Length': '0'} 2022-09-03T06:55:12.175 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008db5f9d815761d56-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.176 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.177 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.178 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.179 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.179 DEBUG:boto:Method: GET 2022-09-03T06:55:12.180 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:12.181 DEBUG:boto:Data: 2022-09-03T06:55:12.182 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.183 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.184 DEBUG:boto:Params: {} 2022-09-03T06:55:12.184 DEBUG:boto:Token: None 2022-09-03T06:55:12.185 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:12.186 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iD7aCJze88gVHKks3Vqy/WkkUZk= 2022-09-03T06:55:12.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iD7aCJze88gVHKks3Vqy/WkkUZk=', 'Content-Length': '0'} 2022-09-03T06:55:12.191 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000069b6f7f1afc5f4fc-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.192 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.193 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.194 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.195 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.196 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.197 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:12.198 DEBUG:boto:Data: 2022-09-03T06:55:12.199 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.200 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.201 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.202 DEBUG:boto:Params: {} 2022-09-03T06:55:12.203 DEBUG:boto:Token: None 2022-09-03T06:55:12.204 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:12.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:38nYh/TbGqq6HG/aDU6yzs4Sies= 2022-09-03T06:55:12.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:38nYh/TbGqq6HG/aDU6yzs4Sies=', 'Content-Length': '0'} 2022-09-03T06:55:12.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c99110897f68837d-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.210 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.211 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.212 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.213 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.214 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.214 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:12.215 DEBUG:boto:Data: 2022-09-03T06:55:12.216 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.218 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.219 DEBUG:boto:Params: {} 2022-09-03T06:55:12.220 DEBUG:boto:Token: None 2022-09-03T06:55:12.221 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:12.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:38nYh/TbGqq6HG/aDU6yzs4Sies= 2022-09-03T06:55:12.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:38nYh/TbGqq6HG/aDU6yzs4Sies=', 'Content-Length': '0'} 2022-09-03T06:55:12.228 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026207ed8a64e1e7c-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.229 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:55:12.230 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.231 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.233 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.234 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.235 DEBUG:boto:Method: GET 2022-09-03T06:55:12.236 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:12.237 DEBUG:boto:Data: 2022-09-03T06:55:12.238 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.240 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.241 DEBUG:boto:Params: {} 2022-09-03T06:55:12.242 DEBUG:boto:Token: None 2022-09-03T06:55:12.244 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:12.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iD7aCJze88gVHKks3Vqy/WkkUZk= 2022-09-03T06:55:12.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iD7aCJze88gVHKks3Vqy/WkkUZk=', 'Content-Length': '0'} 2022-09-03T06:55:12.249 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038a28fc88e91e6d8-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.250 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.250 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.251 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.251 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:12.252 DEBUG:boto:Method: GET 2022-09-03T06:55:12.252 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:12.253 DEBUG:boto:Data: 2022-09-03T06:55:12.253 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.254 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.255 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.255 DEBUG:boto:Params: {} 2022-09-03T06:55:12.256 DEBUG:boto:Token: None 2022-09-03T06:55:12.257 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:12.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iD7aCJze88gVHKks3Vqy/WkkUZk= 2022-09-03T06:55:12.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iD7aCJze88gVHKks3Vqy/WkkUZk=', 'Content-Length': '0'} 2022-09-03T06:55:12.262 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d66de35637364d6-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.306 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.307 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.308 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:12.308 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:12.309 DEBUG:boto:Method: GET 2022-09-03T06:55:12.310 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:12.311 DEBUG:boto:Data: 2022-09-03T06:55:12.312 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.313 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.315 DEBUG:boto:Params: {} 2022-09-03T06:55:12.316 DEBUG:boto:Token: None 2022-09-03T06:55:12.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.321 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b3b0889346f423d-006312fa50-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.322 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.323 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.324 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.324 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:12.325 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:12.326 DEBUG:boto:Method: GET 2022-09-03T06:55:12.327 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:12.328 DEBUG:boto:Data: 2022-09-03T06:55:12.329 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.330 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.331 DEBUG:boto:Params: {} 2022-09-03T06:55:12.331 DEBUG:boto:Token: None 2022-09-03T06:55:12.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.336 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff397a1e301ec03f-006312fa50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.336 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.337 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.338 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.338 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.339 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:12.339 DEBUG:boto:Data: 2022-09-03T06:55:12.340 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.341 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.341 DEBUG:boto:Params: {} 2022-09-03T06:55:12.342 DEBUG:boto:Token: None 2022-09-03T06:55:12.342 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b 2022-09-03T06:55:12.343 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1b9OWzVJti6zVj3t5ruT7LvmYHc= 2022-09-03T06:55:12.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1b9OWzVJti6zVj3t5ruT7LvmYHc=', 'Content-Length': '0'} 2022-09-03T06:55:12.346 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066bc03d1684f00ae-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.347 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.347 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.348 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.348 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:12.349 DEBUG:boto:Data: 2022-09-03T06:55:12.349 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.350 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.350 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.351 DEBUG:boto:Params: {} 2022-09-03T06:55:12.351 DEBUG:boto:Token: None 2022-09-03T06:55:12.352 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b 2022-09-03T06:55:12.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1b9OWzVJti6zVj3t5ruT7LvmYHc= 2022-09-03T06:55:12.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1b9OWzVJti6zVj3t5ruT7LvmYHc=', 'Content-Length': '0'} 2022-09-03T06:55:12.355 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000525a82cb9f194c64-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.356 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:55:12.357 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.358 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.358 DEBUG:boto:Method: GET 2022-09-03T06:55:12.359 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:12.359 DEBUG:boto:Data: 2022-09-03T06:55:12.360 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.361 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.361 DEBUG:boto:Params: {} 2022-09-03T06:55:12.362 DEBUG:boto:Token: None 2022-09-03T06:55:12.363 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b 2022-09-03T06:55:12.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lW0a9K26CfOR63i/etC7yhNHOJU= 2022-09-03T06:55:12.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lW0a9K26CfOR63i/etC7yhNHOJU=', 'Content-Length': '0'} 2022-09-03T06:55:12.366 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000184011bbcb98aede-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.367 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:12.367 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:12.368 DEBUG:boto:Method: GET 2022-09-03T06:55:12.368 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:12.369 DEBUG:boto:Data: 2022-09-03T06:55:12.369 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.370 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.371 DEBUG:boto:Params: {} 2022-09-03T06:55:12.372 DEBUG:boto:Token: None 2022-09-03T06:55:12.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/b 2022-09-03T06:55:12.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lW0a9K26CfOR63i/etC7yhNHOJU= 2022-09-03T06:55:12.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lW0a9K26CfOR63i/etC7yhNHOJU=', 'Content-Length': '0'} 2022-09-03T06:55:12.375 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e970dff83d62828-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.376 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.377 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.377 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:12.378 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:12.378 DEBUG:boto:Method: GET 2022-09-03T06:55:12.379 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:12.379 DEBUG:boto:Data: 2022-09-03T06:55:12.380 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.380 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.381 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.381 DEBUG:boto:Params: {} 2022-09-03T06:55:12.382 DEBUG:boto:Token: None 2022-09-03T06:55:12.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.386 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d497ce302267946d-006312fa50-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.430 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.431 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.432 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.432 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:12.433 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:12.433 DEBUG:boto:Method: GET 2022-09-03T06:55:12.434 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:12.434 DEBUG:boto:Data: 2022-09-03T06:55:12.435 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.435 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.436 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.436 DEBUG:boto:Params: {} 2022-09-03T06:55:12.437 DEBUG:boto:Token: None 2022-09-03T06:55:12.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.441 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb485dbc2ae55b48-006312fa50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.441 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.442 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:55:12.443 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.443 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.444 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.444 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.445 DEBUG:boto:Method: GET 2022-09-03T06:55:12.445 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:12.446 DEBUG:boto:Data: 2022-09-03T06:55:12.447 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.447 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.448 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.448 DEBUG:boto:Params: {} 2022-09-03T06:55:12.449 DEBUG:boto:Token: None 2022-09-03T06:55:12.449 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:12.450 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2s7XlJLED6qpr7W+krWw/mo+DxA= 2022-09-03T06:55:12.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2s7XlJLED6qpr7W+krWw/mo+DxA=', 'Content-Length': '0'} 2022-09-03T06:55:12.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006988fd8f9b5b6e79-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.454 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.454 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.455 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.455 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.456 DEBUG:boto:Method: GET 2022-09-03T06:55:12.456 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:12.457 DEBUG:boto:Data: 2022-09-03T06:55:12.457 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.458 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.459 DEBUG:boto:Params: {} 2022-09-03T06:55:12.460 DEBUG:boto:Token: None 2022-09-03T06:55:12.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:12.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2s7XlJLED6qpr7W+krWw/mo+DxA= 2022-09-03T06:55:12.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2s7XlJLED6qpr7W+krWw/mo+DxA=', 'Content-Length': '0'} 2022-09-03T06:55:12.464 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018c3ac7849d1df62-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.464 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.465 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.465 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.466 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.466 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.467 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:12.467 DEBUG:boto:Data: 2022-09-03T06:55:12.468 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.468 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.469 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.469 DEBUG:boto:Params: {} 2022-09-03T06:55:12.470 DEBUG:boto:Token: None 2022-09-03T06:55:12.470 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:12.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FETiEx5JXPIYbP+iLlcuzD0IBfE= 2022-09-03T06:55:12.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FETiEx5JXPIYbP+iLlcuzD0IBfE=', 'Content-Length': '0'} 2022-09-03T06:55:12.473 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e81c33e986bbc18-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.474 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.475 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.475 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.476 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.476 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.477 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:12.477 DEBUG:boto:Data: 2022-09-03T06:55:12.478 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.479 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.479 DEBUG:boto:Params: {} 2022-09-03T06:55:12.480 DEBUG:boto:Token: None 2022-09-03T06:55:12.480 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:12.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FETiEx5JXPIYbP+iLlcuzD0IBfE= 2022-09-03T06:55:12.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FETiEx5JXPIYbP+iLlcuzD0IBfE=', 'Content-Length': '0'} 2022-09-03T06:55:12.484 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000294dbe6938e97303-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.485 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:55:12.485 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.486 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.487 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.488 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.489 DEBUG:boto:Method: GET 2022-09-03T06:55:12.490 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:12.491 DEBUG:boto:Data: 2022-09-03T06:55:12.492 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.494 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.495 DEBUG:boto:Params: {} 2022-09-03T06:55:12.496 DEBUG:boto:Token: None 2022-09-03T06:55:12.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:12.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2s7XlJLED6qpr7W+krWw/mo+DxA= 2022-09-03T06:55:12.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2s7XlJLED6qpr7W+krWw/mo+DxA=', 'Content-Length': '0'} 2022-09-03T06:55:12.501 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000dcd9c70b47d19d3-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.502 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.502 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.503 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.503 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:12.504 DEBUG:boto:Method: GET 2022-09-03T06:55:12.504 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:12.505 DEBUG:boto:Data: 2022-09-03T06:55:12.505 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.506 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.506 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.507 DEBUG:boto:Params: {} 2022-09-03T06:55:12.507 DEBUG:boto:Token: None 2022-09-03T06:55:12.508 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:12.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2s7XlJLED6qpr7W+krWw/mo+DxA= 2022-09-03T06:55:12.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2s7XlJLED6qpr7W+krWw/mo+DxA=', 'Content-Length': '0'} 2022-09-03T06:55:12.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081621bf89aae8f96-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.512 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.512 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.513 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:12.513 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:12.514 DEBUG:boto:Method: GET 2022-09-03T06:55:12.514 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:12.515 DEBUG:boto:Data: 2022-09-03T06:55:12.516 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.517 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.517 DEBUG:boto:Params: {} 2022-09-03T06:55:12.518 DEBUG:boto:Token: None 2022-09-03T06:55:12.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.523 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000129a82517159b4ca-006312fa50-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.524 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.524 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.525 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.526 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:12.526 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:12.527 DEBUG:boto:Method: GET 2022-09-03T06:55:12.528 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:12.528 DEBUG:boto:Data: 2022-09-03T06:55:12.529 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.530 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.531 DEBUG:boto:Params: {} 2022-09-03T06:55:12.531 DEBUG:boto:Token: None 2022-09-03T06:55:12.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.535 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff2731f34fe7e2be-006312fa50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.536 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.537 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.537 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.538 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.538 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:12.539 DEBUG:boto:Data: 2022-09-03T06:55:12.540 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.541 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.541 DEBUG:boto:Params: {} 2022-09-03T06:55:12.542 DEBUG:boto:Token: None 2022-09-03T06:55:12.542 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c 2022-09-03T06:55:12.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P/oiCwAnhNxSlbYt1nq1NM/xsqQ= 2022-09-03T06:55:12.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P/oiCwAnhNxSlbYt1nq1NM/xsqQ=', 'Content-Length': '0'} 2022-09-03T06:55:12.546 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac72d8669df1bf34-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.546 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.547 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.547 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.548 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:12.548 DEBUG:boto:Data: 2022-09-03T06:55:12.549 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.549 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.550 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.550 DEBUG:boto:Params: {} 2022-09-03T06:55:12.551 DEBUG:boto:Token: None 2022-09-03T06:55:12.552 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c 2022-09-03T06:55:12.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P/oiCwAnhNxSlbYt1nq1NM/xsqQ= 2022-09-03T06:55:12.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P/oiCwAnhNxSlbYt1nq1NM/xsqQ=', 'Content-Length': '0'} 2022-09-03T06:55:12.555 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c93e5cda958e0592-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.555 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:55:12.556 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.556 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.557 DEBUG:boto:Method: GET 2022-09-03T06:55:12.557 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:12.558 DEBUG:boto:Data: 2022-09-03T06:55:12.559 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.560 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.560 DEBUG:boto:Params: {} 2022-09-03T06:55:12.561 DEBUG:boto:Token: None 2022-09-03T06:55:12.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c 2022-09-03T06:55:12.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZIhql7j6YvkT0wV3JCE7B0yGQrY= 2022-09-03T06:55:12.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZIhql7j6YvkT0wV3JCE7B0yGQrY=', 'Content-Length': '0'} 2022-09-03T06:55:12.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a443c6f62a7d981-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.565 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:12.566 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:12.566 DEBUG:boto:Method: GET 2022-09-03T06:55:12.567 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:12.567 DEBUG:boto:Data: 2022-09-03T06:55:12.568 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.569 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.570 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.571 DEBUG:boto:Params: {} 2022-09-03T06:55:12.572 DEBUG:boto:Token: None 2022-09-03T06:55:12.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/c 2022-09-03T06:55:12.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZIhql7j6YvkT0wV3JCE7B0yGQrY= 2022-09-03T06:55:12.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZIhql7j6YvkT0wV3JCE7B0yGQrY=', 'Content-Length': '0'} 2022-09-03T06:55:12.577 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008dd326408b9e6d29-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.579 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.580 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.580 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:12.581 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:12.581 DEBUG:boto:Method: GET 2022-09-03T06:55:12.582 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:12.583 DEBUG:boto:Data: 2022-09-03T06:55:12.584 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.584 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.585 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.586 DEBUG:boto:Params: {} 2022-09-03T06:55:12.587 DEBUG:boto:Token: None 2022-09-03T06:55:12.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.593 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a202560e06bbd13-006312fa50-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.594 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.594 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.595 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.596 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:12.596 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:12.597 DEBUG:boto:Method: GET 2022-09-03T06:55:12.598 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:12.599 DEBUG:boto:Data: 2022-09-03T06:55:12.600 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.601 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.601 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.602 DEBUG:boto:Params: {} 2022-09-03T06:55:12.603 DEBUG:boto:Token: None 2022-09-03T06:55:12.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.607 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009bd2828cc3aa4f22-006312fa50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.608 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.609 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:55:12.610 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.611 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.612 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.612 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.613 DEBUG:boto:Method: GET 2022-09-03T06:55:12.613 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:12.614 DEBUG:boto:Data: 2022-09-03T06:55:12.614 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.616 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.616 DEBUG:boto:Params: {} 2022-09-03T06:55:12.617 DEBUG:boto:Token: None 2022-09-03T06:55:12.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:12.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5exvDYDUx8s9B+Tk3s6LamBBILI= 2022-09-03T06:55:12.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5exvDYDUx8s9B+Tk3s6LamBBILI=', 'Content-Length': '0'} 2022-09-03T06:55:12.621 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009803abd5d2009554-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.622 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.622 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.623 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.623 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.624 DEBUG:boto:Method: GET 2022-09-03T06:55:12.625 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:12.626 DEBUG:boto:Data: 2022-09-03T06:55:12.626 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.628 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.628 DEBUG:boto:Params: {} 2022-09-03T06:55:12.629 DEBUG:boto:Token: None 2022-09-03T06:55:12.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:12.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5exvDYDUx8s9B+Tk3s6LamBBILI= 2022-09-03T06:55:12.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5exvDYDUx8s9B+Tk3s6LamBBILI=', 'Content-Length': '0'} 2022-09-03T06:55:12.633 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ab671ed7509c3ca-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.634 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.635 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.635 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.636 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.637 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.637 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:12.638 DEBUG:boto:Data: 2022-09-03T06:55:12.638 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.639 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.640 DEBUG:boto:Params: {} 2022-09-03T06:55:12.640 DEBUG:boto:Token: None 2022-09-03T06:55:12.641 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:12.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q3gGtTFcGNfbJZhvTDyK7Q9vs9M= 2022-09-03T06:55:12.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q3gGtTFcGNfbJZhvTDyK7Q9vs9M=', 'Content-Length': '0'} 2022-09-03T06:55:12.644 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ac68edc7880d069-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.645 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.646 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.646 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.647 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.647 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.648 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:12.648 DEBUG:boto:Data: 2022-09-03T06:55:12.649 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.650 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.650 DEBUG:boto:Params: {} 2022-09-03T06:55:12.651 DEBUG:boto:Token: None 2022-09-03T06:55:12.652 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:12.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q3gGtTFcGNfbJZhvTDyK7Q9vs9M= 2022-09-03T06:55:12.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q3gGtTFcGNfbJZhvTDyK7Q9vs9M=', 'Content-Length': '0'} 2022-09-03T06:55:12.655 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003339fa96f9fe6c7d-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.656 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:55:12.656 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.657 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.658 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.658 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.659 DEBUG:boto:Method: GET 2022-09-03T06:55:12.659 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:12.660 DEBUG:boto:Data: 2022-09-03T06:55:12.660 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.663 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.664 DEBUG:boto:Params: {} 2022-09-03T06:55:12.665 DEBUG:boto:Token: None 2022-09-03T06:55:12.666 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:12.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5exvDYDUx8s9B+Tk3s6LamBBILI= 2022-09-03T06:55:12.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5exvDYDUx8s9B+Tk3s6LamBBILI=', 'Content-Length': '0'} 2022-09-03T06:55:12.671 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ef1f7db43c75957-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.672 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.672 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.673 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.674 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:12.674 DEBUG:boto:Method: GET 2022-09-03T06:55:12.675 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:12.676 DEBUG:boto:Data: 2022-09-03T06:55:12.676 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.678 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.679 DEBUG:boto:Params: {} 2022-09-03T06:55:12.680 DEBUG:boto:Token: None 2022-09-03T06:55:12.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:12.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5exvDYDUx8s9B+Tk3s6LamBBILI= 2022-09-03T06:55:12.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5exvDYDUx8s9B+Tk3s6LamBBILI=', 'Content-Length': '0'} 2022-09-03T06:55:12.684 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d1728fde2ac84f8-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.685 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.685 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.686 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:12.687 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:12.688 DEBUG:boto:Method: GET 2022-09-03T06:55:12.689 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:12.690 DEBUG:boto:Data: 2022-09-03T06:55:12.690 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.692 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.692 DEBUG:boto:Params: {} 2022-09-03T06:55:12.693 DEBUG:boto:Token: None 2022-09-03T06:55:12.694 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.695 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.698 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e53dd956ea250f1-006312fa50-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.742 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.744 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.745 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.746 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:12.747 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:12.748 DEBUG:boto:Method: GET 2022-09-03T06:55:12.749 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:12.749 DEBUG:boto:Data: 2022-09-03T06:55:12.750 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.752 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.753 DEBUG:boto:Params: {} 2022-09-03T06:55:12.755 DEBUG:boto:Token: None 2022-09-03T06:55:12.756 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.758 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.760 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003594d2d124e69dab-006312fa50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.761 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.762 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.763 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.763 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.764 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:12.764 DEBUG:boto:Data: 2022-09-03T06:55:12.765 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.766 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.767 DEBUG:boto:Params: {} 2022-09-03T06:55:12.767 DEBUG:boto:Token: None 2022-09-03T06:55:12.768 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d 2022-09-03T06:55:12.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GtWFuvvoZT6JBM31DZjfK8QQZfg= 2022-09-03T06:55:12.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GtWFuvvoZT6JBM31DZjfK8QQZfg=', 'Content-Length': '0'} 2022-09-03T06:55:12.772 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed3c7a9106376664-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.773 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.774 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.775 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.775 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:12.776 DEBUG:boto:Data: 2022-09-03T06:55:12.777 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.779 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.779 DEBUG:boto:Params: {} 2022-09-03T06:55:12.780 DEBUG:boto:Token: None 2022-09-03T06:55:12.781 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d 2022-09-03T06:55:12.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GtWFuvvoZT6JBM31DZjfK8QQZfg= 2022-09-03T06:55:12.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GtWFuvvoZT6JBM31DZjfK8QQZfg=', 'Content-Length': '0'} 2022-09-03T06:55:12.786 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015a75a49e5944f40-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.787 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:55:12.788 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.789 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.790 DEBUG:boto:Method: GET 2022-09-03T06:55:12.790 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:12.791 DEBUG:boto:Data: 2022-09-03T06:55:12.792 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.794 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.794 DEBUG:boto:Params: {} 2022-09-03T06:55:12.795 DEBUG:boto:Token: None 2022-09-03T06:55:12.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d 2022-09-03T06:55:12.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dKhY7k/uROtbOBF9Bc5rxtEZ0tU= 2022-09-03T06:55:12.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dKhY7k/uROtbOBF9Bc5rxtEZ0tU=', 'Content-Length': '0'} 2022-09-03T06:55:12.799 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e568ecab01798dc-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.800 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:12.801 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:12.802 DEBUG:boto:Method: GET 2022-09-03T06:55:12.803 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:12.804 DEBUG:boto:Data: 2022-09-03T06:55:12.805 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.806 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.806 DEBUG:boto:Params: {} 2022-09-03T06:55:12.807 DEBUG:boto:Token: None 2022-09-03T06:55:12.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/d 2022-09-03T06:55:12.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dKhY7k/uROtbOBF9Bc5rxtEZ0tU= 2022-09-03T06:55:12.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dKhY7k/uROtbOBF9Bc5rxtEZ0tU=', 'Content-Length': '0'} 2022-09-03T06:55:12.811 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008bddc0e4e7d14c34-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.813 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.814 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.814 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:12.815 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:12.816 DEBUG:boto:Method: GET 2022-09-03T06:55:12.817 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:12.818 DEBUG:boto:Data: 2022-09-03T06:55:12.819 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.821 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.822 DEBUG:boto:Params: {} 2022-09-03T06:55:12.823 DEBUG:boto:Token: None 2022-09-03T06:55:12.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.829 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3e10957e6d82542-006312fa50-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.830 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.831 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.831 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.832 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:12.833 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:12.833 DEBUG:boto:Method: GET 2022-09-03T06:55:12.834 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:12.834 DEBUG:boto:Data: 2022-09-03T06:55:12.835 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.836 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.837 DEBUG:boto:Params: {} 2022-09-03T06:55:12.837 DEBUG:boto:Token: None 2022-09-03T06:55:12.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.843 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.846 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000de5d2755721193ca-006312fa50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.847 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.849 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:55:12.850 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:12.851 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:12.851 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.852 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.853 DEBUG:boto:Method: GET 2022-09-03T06:55:12.854 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:12.855 DEBUG:boto:Data: 2022-09-03T06:55:12.856 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.858 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.859 DEBUG:boto:Params: {} 2022-09-03T06:55:12.860 DEBUG:boto:Token: None 2022-09-03T06:55:12.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:12.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mDhpY+PG6a5X6CHXBggi764pzvI= 2022-09-03T06:55:12.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mDhpY+PG6a5X6CHXBggi764pzvI=', 'Content-Length': '0'} 2022-09-03T06:55:12.865 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065dd55c3b4df85bd-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.866 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:12.867 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:12.868 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.869 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.870 DEBUG:boto:Method: GET 2022-09-03T06:55:12.871 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:12.872 DEBUG:boto:Data: 2022-09-03T06:55:12.873 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.874 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.874 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.875 DEBUG:boto:Params: {} 2022-09-03T06:55:12.875 DEBUG:boto:Token: None 2022-09-03T06:55:12.876 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:12.877 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mDhpY+PG6a5X6CHXBggi764pzvI= 2022-09-03T06:55:12.878 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mDhpY+PG6a5X6CHXBggi764pzvI=', 'Content-Length': '0'} 2022-09-03T06:55:12.880 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000019a3787a0fe79d36-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.923 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:12.924 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:12.925 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.925 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.926 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.927 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:12.927 DEBUG:boto:Data: 2022-09-03T06:55:12.928 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.929 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.930 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.930 DEBUG:boto:Params: {} 2022-09-03T06:55:12.931 DEBUG:boto:Token: None 2022-09-03T06:55:12.932 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:12.933 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zq6VRAkDYH62UoCTwnL9f7Lbs9Q= 2022-09-03T06:55:12.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zq6VRAkDYH62UoCTwnL9f7Lbs9Q=', 'Content-Length': '0'} 2022-09-03T06:55:12.938 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015f6b49f17bdeb23-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.939 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:12.940 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:12.941 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.942 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.944 DEBUG:boto:Method: HEAD 2022-09-03T06:55:12.945 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:12.946 DEBUG:boto:Data: 2022-09-03T06:55:12.947 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.948 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.948 DEBUG:boto:Params: {} 2022-09-03T06:55:12.949 DEBUG:boto:Token: None 2022-09-03T06:55:12.950 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:12.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zq6VRAkDYH62UoCTwnL9f7Lbs9Q= 2022-09-03T06:55:12.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zq6VRAkDYH62UoCTwnL9f7Lbs9Q=', 'Content-Length': '0'} 2022-09-03T06:55:12.953 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b968fb1a50d9f088-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.954 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:55:12.955 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:12.956 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:12.957 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.957 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.958 DEBUG:boto:Method: GET 2022-09-03T06:55:12.958 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:12.959 DEBUG:boto:Data: 2022-09-03T06:55:12.960 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.961 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.962 DEBUG:boto:Params: {} 2022-09-03T06:55:12.963 DEBUG:boto:Token: None 2022-09-03T06:55:12.964 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:12.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mDhpY+PG6a5X6CHXBggi764pzvI= 2022-09-03T06:55:12.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mDhpY+PG6a5X6CHXBggi764pzvI=', 'Content-Length': '0'} 2022-09-03T06:55:12.967 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000466156e2a79a53c2-006312fa50-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.969 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:12.969 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:12.970 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.970 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:12.971 DEBUG:boto:Method: GET 2022-09-03T06:55:12.972 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:12.972 DEBUG:boto:Data: 2022-09-03T06:55:12.973 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:12.974 DEBUG:boto:Port: 8000 2022-09-03T06:55:12.975 DEBUG:boto:Params: {} 2022-09-03T06:55:12.975 DEBUG:boto:Token: None 2022-09-03T06:55:12.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:12.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mDhpY+PG6a5X6CHXBggi764pzvI= 2022-09-03T06:55:12.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mDhpY+PG6a5X6CHXBggi764pzvI=', 'Content-Length': '0'} 2022-09-03T06:55:12.980 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a173b46e60c92497-006312fa50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.981 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.981 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.982 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:12.982 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:12.983 DEBUG:boto:Method: GET 2022-09-03T06:55:12.984 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:12.985 DEBUG:boto:Data: 2022-09-03T06:55:12.985 DEBUG:boto:Headers: {} 2022-09-03T06:55:12.986 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:12.986 DEBUG:boto:Port: 8001 2022-09-03T06:55:12.987 DEBUG:boto:Params: {} 2022-09-03T06:55:12.987 DEBUG:boto:Token: None 2022-09-03T06:55:12.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:12 GMT /avysmw-71/ 2022-09-03T06:55:12.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0= 2022-09-03T06:55:12.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgpCtInEsj+22tXAs51sRar39L0=', 'Content-Length': '0'} 2022-09-03T06:55:12.992 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d8c753c0a98495c-006312fa50-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:12.993 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:12.995 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:12.996 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:12.997 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:12.998 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:12.999 DEBUG:boto:Method: GET 2022-09-03T06:55:13.000 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:13.001 DEBUG:boto:Data: 2022-09-03T06:55:13.002 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.004 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.005 DEBUG:boto:Params: {} 2022-09-03T06:55:13.007 DEBUG:boto:Token: None 2022-09-03T06:55:13.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.013 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004cb07b40fa17390c-006312fa51-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.014 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.015 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:13.016 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:13.017 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.018 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:13.018 DEBUG:boto:Data: 2022-09-03T06:55:13.019 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.020 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.021 DEBUG:boto:Params: {} 2022-09-03T06:55:13.021 DEBUG:boto:Token: None 2022-09-03T06:55:13.022 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/e 2022-09-03T06:55:13.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hS/zbsCrTU3pGiWVAxHyL8LhRRY= 2022-09-03T06:55:13.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hS/zbsCrTU3pGiWVAxHyL8LhRRY=', 'Content-Length': '0'} 2022-09-03T06:55:13.025 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d5064df64dbb17c0-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.027 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:13.028 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:13.029 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.029 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:13.030 DEBUG:boto:Data: 2022-09-03T06:55:13.030 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.031 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.032 DEBUG:boto:Params: {} 2022-09-03T06:55:13.033 DEBUG:boto:Token: None 2022-09-03T06:55:13.033 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/e 2022-09-03T06:55:13.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hS/zbsCrTU3pGiWVAxHyL8LhRRY= 2022-09-03T06:55:13.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hS/zbsCrTU3pGiWVAxHyL8LhRRY=', 'Content-Length': '0'} 2022-09-03T06:55:13.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032579dd263170d48-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.037 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:55:13.038 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:13.039 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:13.039 DEBUG:boto:Method: GET 2022-09-03T06:55:13.040 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:13.041 DEBUG:boto:Data: 2022-09-03T06:55:13.041 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.042 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.042 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.043 DEBUG:boto:Params: {} 2022-09-03T06:55:13.043 DEBUG:boto:Token: None 2022-09-03T06:55:13.044 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/e 2022-09-03T06:55:13.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gClzeU2x195z8SVU321mIusOASQ= 2022-09-03T06:55:13.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gClzeU2x195z8SVU321mIusOASQ=', 'Content-Length': '0'} 2022-09-03T06:55:13.048 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000047148d8d9061a2e8-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.048 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:13.049 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:13.050 DEBUG:boto:Method: GET 2022-09-03T06:55:13.050 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:13.051 DEBUG:boto:Data: 2022-09-03T06:55:13.051 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.052 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.052 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.053 DEBUG:boto:Params: {} 2022-09-03T06:55:13.054 DEBUG:boto:Token: None 2022-09-03T06:55:13.054 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/e 2022-09-03T06:55:13.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gClzeU2x195z8SVU321mIusOASQ= 2022-09-03T06:55:13.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gClzeU2x195z8SVU321mIusOASQ=', 'Content-Length': '0'} 2022-09-03T06:55:13.058 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000218ffaf5e10ad12f-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.059 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.060 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.061 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:13.061 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:13.062 DEBUG:boto:Method: GET 2022-09-03T06:55:13.063 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:13.063 DEBUG:boto:Data: 2022-09-03T06:55:13.064 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.065 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.066 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.066 DEBUG:boto:Params: {} 2022-09-03T06:55:13.067 DEBUG:boto:Token: None 2022-09-03T06:55:13.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.073 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7b2fe4231dc5057-006312fa51-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.074 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.075 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.076 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.076 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:13.077 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:13.077 DEBUG:boto:Method: GET 2022-09-03T06:55:13.078 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:13.079 DEBUG:boto:Data: 2022-09-03T06:55:13.080 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.081 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.082 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.083 DEBUG:boto:Params: {} 2022-09-03T06:55:13.084 DEBUG:boto:Token: None 2022-09-03T06:55:13.085 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.090 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac2909ff43fd1e7d-006312fa51-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.091 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.093 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:55:13.094 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.095 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.096 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.097 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.098 DEBUG:boto:Method: GET 2022-09-03T06:55:13.099 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:13.099 DEBUG:boto:Data: 2022-09-03T06:55:13.100 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.101 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.102 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.103 DEBUG:boto:Params: {} 2022-09-03T06:55:13.104 DEBUG:boto:Token: None 2022-09-03T06:55:13.105 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:13.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QbQOZMEkYG8VvWW2K3cghK1Dkjs= 2022-09-03T06:55:13.106 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QbQOZMEkYG8VvWW2K3cghK1Dkjs=', 'Content-Length': '0'} 2022-09-03T06:55:13.109 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048384a73ae5ff08f-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.110 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.111 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.112 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.113 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.114 DEBUG:boto:Method: GET 2022-09-03T06:55:13.115 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:13.115 DEBUG:boto:Data: 2022-09-03T06:55:13.116 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.118 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.118 DEBUG:boto:Params: {} 2022-09-03T06:55:13.119 DEBUG:boto:Token: None 2022-09-03T06:55:13.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:13.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QbQOZMEkYG8VvWW2K3cghK1Dkjs= 2022-09-03T06:55:13.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QbQOZMEkYG8VvWW2K3cghK1Dkjs=', 'Content-Length': '0'} 2022-09-03T06:55:13.124 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009bfab863e2d6e471-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.125 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.126 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.127 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.127 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.128 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.128 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:13.129 DEBUG:boto:Data: 2022-09-03T06:55:13.130 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.131 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.132 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.132 DEBUG:boto:Params: {} 2022-09-03T06:55:13.133 DEBUG:boto:Token: None 2022-09-03T06:55:13.133 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:13.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:caejNWPkLEqINj9CXQEdvTCY7Fs= 2022-09-03T06:55:13.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:caejNWPkLEqINj9CXQEdvTCY7Fs=', 'Content-Length': '0'} 2022-09-03T06:55:13.136 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d8f43d0db1ffa92-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.137 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.138 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.138 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.139 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.140 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.140 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:13.141 DEBUG:boto:Data: 2022-09-03T06:55:13.141 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.142 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.142 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.143 DEBUG:boto:Params: {} 2022-09-03T06:55:13.143 DEBUG:boto:Token: None 2022-09-03T06:55:13.144 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:13.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:caejNWPkLEqINj9CXQEdvTCY7Fs= 2022-09-03T06:55:13.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:caejNWPkLEqINj9CXQEdvTCY7Fs=', 'Content-Length': '0'} 2022-09-03T06:55:13.147 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5fdb9dbd176c9fe-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.148 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:55:13.148 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.149 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.149 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.150 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.150 DEBUG:boto:Method: GET 2022-09-03T06:55:13.151 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:13.152 DEBUG:boto:Data: 2022-09-03T06:55:13.152 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.153 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.153 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.154 DEBUG:boto:Params: {} 2022-09-03T06:55:13.155 DEBUG:boto:Token: None 2022-09-03T06:55:13.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:13.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QbQOZMEkYG8VvWW2K3cghK1Dkjs= 2022-09-03T06:55:13.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QbQOZMEkYG8VvWW2K3cghK1Dkjs=', 'Content-Length': '0'} 2022-09-03T06:55:13.158 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007080342f9fc51fde-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.159 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.160 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.160 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.161 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:13.161 DEBUG:boto:Method: GET 2022-09-03T06:55:13.162 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:13.162 DEBUG:boto:Data: 2022-09-03T06:55:13.163 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.164 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.164 DEBUG:boto:Params: {} 2022-09-03T06:55:13.165 DEBUG:boto:Token: None 2022-09-03T06:55:13.166 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:13.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QbQOZMEkYG8VvWW2K3cghK1Dkjs= 2022-09-03T06:55:13.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QbQOZMEkYG8VvWW2K3cghK1Dkjs=', 'Content-Length': '0'} 2022-09-03T06:55:13.169 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce3d24b7e77a2939-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.169 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.170 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.170 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:13.171 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:13.172 DEBUG:boto:Method: GET 2022-09-03T06:55:13.172 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:13.173 DEBUG:boto:Data: 2022-09-03T06:55:13.173 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.174 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.174 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.175 DEBUG:boto:Params: {} 2022-09-03T06:55:13.176 DEBUG:boto:Token: None 2022-09-03T06:55:13.176 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.180 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b8b1376a12bf07d-006312fa51-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.226 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.227 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.228 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.228 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:13.229 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:13.230 DEBUG:boto:Method: GET 2022-09-03T06:55:13.230 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:13.231 DEBUG:boto:Data: 2022-09-03T06:55:13.232 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.232 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.233 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.233 DEBUG:boto:Params: {} 2022-09-03T06:55:13.234 DEBUG:boto:Token: None 2022-09-03T06:55:13.234 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.235 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.236 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.238 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000459a9a59bbadb85c-006312fa51-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.239 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.240 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.240 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.241 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.241 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:13.242 DEBUG:boto:Data: 2022-09-03T06:55:13.242 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.244 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.244 DEBUG:boto:Params: {} 2022-09-03T06:55:13.245 DEBUG:boto:Token: None 2022-09-03T06:55:13.245 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f 2022-09-03T06:55:13.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YVxOwR5eDpmvQGNnRWj5/k2Nef0= 2022-09-03T06:55:13.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YVxOwR5eDpmvQGNnRWj5/k2Nef0=', 'Content-Length': '0'} 2022-09-03T06:55:13.249 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000143052b7904c25e6-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.249 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.250 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.251 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.251 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:13.252 DEBUG:boto:Data: 2022-09-03T06:55:13.252 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.253 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.254 DEBUG:boto:Params: {} 2022-09-03T06:55:13.254 DEBUG:boto:Token: None 2022-09-03T06:55:13.255 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f 2022-09-03T06:55:13.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YVxOwR5eDpmvQGNnRWj5/k2Nef0= 2022-09-03T06:55:13.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YVxOwR5eDpmvQGNnRWj5/k2Nef0=', 'Content-Length': '0'} 2022-09-03T06:55:13.259 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093b934f2f9a40a4f-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.260 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:55:13.260 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.261 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.261 DEBUG:boto:Method: GET 2022-09-03T06:55:13.262 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:13.263 DEBUG:boto:Data: 2022-09-03T06:55:13.263 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.264 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.265 DEBUG:boto:Params: {} 2022-09-03T06:55:13.265 DEBUG:boto:Token: None 2022-09-03T06:55:13.266 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f 2022-09-03T06:55:13.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6TD7ofILlT2dOD5KnEyaJ1xARcI= 2022-09-03T06:55:13.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6TD7ofILlT2dOD5KnEyaJ1xARcI=', 'Content-Length': '0'} 2022-09-03T06:55:13.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f2e4b0413c71990-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.271 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:13.272 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:13.272 DEBUG:boto:Method: GET 2022-09-03T06:55:13.273 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:13.273 DEBUG:boto:Data: 2022-09-03T06:55:13.274 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.275 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.275 DEBUG:boto:Params: {} 2022-09-03T06:55:13.276 DEBUG:boto:Token: None 2022-09-03T06:55:13.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/f 2022-09-03T06:55:13.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6TD7ofILlT2dOD5KnEyaJ1xARcI= 2022-09-03T06:55:13.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6TD7ofILlT2dOD5KnEyaJ1xARcI=', 'Content-Length': '0'} 2022-09-03T06:55:13.280 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d88826dfc3434b58-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.322 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.323 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.323 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:13.324 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:13.324 DEBUG:boto:Method: GET 2022-09-03T06:55:13.325 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:13.326 DEBUG:boto:Data: 2022-09-03T06:55:13.326 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.327 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.327 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.328 DEBUG:boto:Params: {} 2022-09-03T06:55:13.329 DEBUG:boto:Token: None 2022-09-03T06:55:13.330 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.332 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.335 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036ba465423d60bb6-006312fa51-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.336 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.338 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.339 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.340 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:13.340 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:13.341 DEBUG:boto:Method: GET 2022-09-03T06:55:13.342 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:13.343 DEBUG:boto:Data: 2022-09-03T06:55:13.343 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.345 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.346 DEBUG:boto:Params: {} 2022-09-03T06:55:13.347 DEBUG:boto:Token: None 2022-09-03T06:55:13.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.353 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000095c62cdab43f6d7a-006312fa51-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.354 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.355 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:55:13.356 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.356 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.357 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.358 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.359 DEBUG:boto:Method: GET 2022-09-03T06:55:13.360 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:13.361 DEBUG:boto:Data: 2022-09-03T06:55:13.362 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.364 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.365 DEBUG:boto:Params: {} 2022-09-03T06:55:13.366 DEBUG:boto:Token: None 2022-09-03T06:55:13.367 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:13.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BlXUA67k01sHXBY5Pw4OyctI7Ac= 2022-09-03T06:55:13.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BlXUA67k01sHXBY5Pw4OyctI7Ac=', 'Content-Length': '0'} 2022-09-03T06:55:13.372 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005008bc291ad0b78a-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.372 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.373 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.374 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.374 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.375 DEBUG:boto:Method: GET 2022-09-03T06:55:13.376 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:13.376 DEBUG:boto:Data: 2022-09-03T06:55:13.377 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.378 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.379 DEBUG:boto:Params: {} 2022-09-03T06:55:13.379 DEBUG:boto:Token: None 2022-09-03T06:55:13.380 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:13.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BlXUA67k01sHXBY5Pw4OyctI7Ac= 2022-09-03T06:55:13.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BlXUA67k01sHXBY5Pw4OyctI7Ac=', 'Content-Length': '0'} 2022-09-03T06:55:13.384 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fe6237ebcb20b25-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.385 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.385 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.386 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.387 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.387 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.388 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:13.388 DEBUG:boto:Data: 2022-09-03T06:55:13.389 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.390 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.391 DEBUG:boto:Params: {} 2022-09-03T06:55:13.392 DEBUG:boto:Token: None 2022-09-03T06:55:13.393 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:13.394 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N183NE+fZ6tagIrhoNwaFdrVIk4= 2022-09-03T06:55:13.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N183NE+fZ6tagIrhoNwaFdrVIk4=', 'Content-Length': '0'} 2022-09-03T06:55:13.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005712e97ad8f2ce77-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.397 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.398 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.398 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.399 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.399 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.400 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:13.401 DEBUG:boto:Data: 2022-09-03T06:55:13.401 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.402 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.403 DEBUG:boto:Params: {} 2022-09-03T06:55:13.403 DEBUG:boto:Token: None 2022-09-03T06:55:13.404 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:13.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N183NE+fZ6tagIrhoNwaFdrVIk4= 2022-09-03T06:55:13.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N183NE+fZ6tagIrhoNwaFdrVIk4=', 'Content-Length': '0'} 2022-09-03T06:55:13.407 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000043b3aa3f820f00e5-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.408 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:55:13.409 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.409 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.410 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.410 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.411 DEBUG:boto:Method: GET 2022-09-03T06:55:13.412 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:13.412 DEBUG:boto:Data: 2022-09-03T06:55:13.413 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.413 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.414 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.415 DEBUG:boto:Params: {} 2022-09-03T06:55:13.415 DEBUG:boto:Token: None 2022-09-03T06:55:13.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:13.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BlXUA67k01sHXBY5Pw4OyctI7Ac= 2022-09-03T06:55:13.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BlXUA67k01sHXBY5Pw4OyctI7Ac=', 'Content-Length': '0'} 2022-09-03T06:55:13.420 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000470234eaa7b89ab7-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.422 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.423 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.423 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.424 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:13.425 DEBUG:boto:Method: GET 2022-09-03T06:55:13.426 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:13.427 DEBUG:boto:Data: 2022-09-03T06:55:13.427 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.429 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.429 DEBUG:boto:Params: {} 2022-09-03T06:55:13.430 DEBUG:boto:Token: None 2022-09-03T06:55:13.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:13.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BlXUA67k01sHXBY5Pw4OyctI7Ac= 2022-09-03T06:55:13.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BlXUA67k01sHXBY5Pw4OyctI7Ac=', 'Content-Length': '0'} 2022-09-03T06:55:13.434 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b5cb0a237304f51-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.435 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.435 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.436 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:13.436 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:13.437 DEBUG:boto:Method: GET 2022-09-03T06:55:13.438 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:13.438 DEBUG:boto:Data: 2022-09-03T06:55:13.439 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.440 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.440 DEBUG:boto:Params: {} 2022-09-03T06:55:13.441 DEBUG:boto:Token: None 2022-09-03T06:55:13.442 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.447 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000112f642db7f690ed-006312fa51-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.448 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.450 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.451 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.452 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:13.453 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:13.454 DEBUG:boto:Method: GET 2022-09-03T06:55:13.455 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:13.456 DEBUG:boto:Data: 2022-09-03T06:55:13.457 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.459 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.460 DEBUG:boto:Params: {} 2022-09-03T06:55:13.461 DEBUG:boto:Token: None 2022-09-03T06:55:13.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.467 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f6705c21bba3d6bf-006312fa51-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.468 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.470 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.471 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.472 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.473 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:13.473 DEBUG:boto:Data: 2022-09-03T06:55:13.474 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.476 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.477 DEBUG:boto:Params: {} 2022-09-03T06:55:13.478 DEBUG:boto:Token: None 2022-09-03T06:55:13.478 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g 2022-09-03T06:55:13.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jrnKnrRDqS+O4VEVG8QLbs41hec= 2022-09-03T06:55:13.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jrnKnrRDqS+O4VEVG8QLbs41hec=', 'Content-Length': '0'} 2022-09-03T06:55:13.481 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b841470aaa5308b-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.482 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.483 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.484 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.485 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:13.486 DEBUG:boto:Data: 2022-09-03T06:55:13.487 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.489 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.490 DEBUG:boto:Params: {} 2022-09-03T06:55:13.491 DEBUG:boto:Token: None 2022-09-03T06:55:13.492 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g 2022-09-03T06:55:13.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jrnKnrRDqS+O4VEVG8QLbs41hec= 2022-09-03T06:55:13.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jrnKnrRDqS+O4VEVG8QLbs41hec=', 'Content-Length': '0'} 2022-09-03T06:55:13.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc7eda4f86652bba-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.496 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:55:13.496 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.497 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.498 DEBUG:boto:Method: GET 2022-09-03T06:55:13.498 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:13.499 DEBUG:boto:Data: 2022-09-03T06:55:13.499 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.500 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.501 DEBUG:boto:Params: {} 2022-09-03T06:55:13.502 DEBUG:boto:Token: None 2022-09-03T06:55:13.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g 2022-09-03T06:55:13.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZsQo5tDjY9BjmsxPXfE0QMcoV0Y= 2022-09-03T06:55:13.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZsQo5tDjY9BjmsxPXfE0QMcoV0Y=', 'Content-Length': '0'} 2022-09-03T06:55:13.506 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f874d17f9914b94-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.550 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:13.551 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:13.552 DEBUG:boto:Method: GET 2022-09-03T06:55:13.553 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:13.554 DEBUG:boto:Data: 2022-09-03T06:55:13.555 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.557 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.558 DEBUG:boto:Params: {} 2022-09-03T06:55:13.559 DEBUG:boto:Token: None 2022-09-03T06:55:13.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/g 2022-09-03T06:55:13.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZsQo5tDjY9BjmsxPXfE0QMcoV0Y= 2022-09-03T06:55:13.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZsQo5tDjY9BjmsxPXfE0QMcoV0Y=', 'Content-Length': '0'} 2022-09-03T06:55:13.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006c3a3a0014847b41-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.566 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.567 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.568 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:13.568 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:13.569 DEBUG:boto:Method: GET 2022-09-03T06:55:13.570 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:13.571 DEBUG:boto:Data: 2022-09-03T06:55:13.572 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.573 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.575 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.576 DEBUG:boto:Params: {} 2022-09-03T06:55:13.577 DEBUG:boto:Token: None 2022-09-03T06:55:13.578 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.582 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009d9880a017e9bd17-006312fa51-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.583 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.583 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.584 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.585 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:13.585 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:13.586 DEBUG:boto:Method: GET 2022-09-03T06:55:13.587 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:13.587 DEBUG:boto:Data: 2022-09-03T06:55:13.588 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.588 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.589 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.590 DEBUG:boto:Params: {} 2022-09-03T06:55:13.590 DEBUG:boto:Token: None 2022-09-03T06:55:13.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.602 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000319ba5a77cc34f19-006312fa51-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.603 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.604 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:55:13.604 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.605 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.606 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.607 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.608 DEBUG:boto:Method: GET 2022-09-03T06:55:13.609 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:13.610 DEBUG:boto:Data: 2022-09-03T06:55:13.611 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.613 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.613 DEBUG:boto:Params: {} 2022-09-03T06:55:13.614 DEBUG:boto:Token: None 2022-09-03T06:55:13.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:13.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s+gfAIZW+3oh8gIFt2bVbOgtlEY= 2022-09-03T06:55:13.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s+gfAIZW+3oh8gIFt2bVbOgtlEY=', 'Content-Length': '0'} 2022-09-03T06:55:13.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e17fa436fff15656-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.619 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.620 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.620 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.621 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.621 DEBUG:boto:Method: GET 2022-09-03T06:55:13.622 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:13.623 DEBUG:boto:Data: 2022-09-03T06:55:13.624 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.624 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.625 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.625 DEBUG:boto:Params: {} 2022-09-03T06:55:13.626 DEBUG:boto:Token: None 2022-09-03T06:55:13.627 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:13.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s+gfAIZW+3oh8gIFt2bVbOgtlEY= 2022-09-03T06:55:13.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s+gfAIZW+3oh8gIFt2bVbOgtlEY=', 'Content-Length': '0'} 2022-09-03T06:55:13.630 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009119a44f10804c36-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.631 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.632 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.632 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.633 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.634 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.634 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:13.635 DEBUG:boto:Data: 2022-09-03T06:55:13.635 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.636 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.637 DEBUG:boto:Params: {} 2022-09-03T06:55:13.638 DEBUG:boto:Token: None 2022-09-03T06:55:13.638 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:13.639 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PWMAFzk/CSVGtaFazC19F4D7g8s= 2022-09-03T06:55:13.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PWMAFzk/CSVGtaFazC19F4D7g8s=', 'Content-Length': '0'} 2022-09-03T06:55:13.642 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ebea8c4237961533-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.643 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.644 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.645 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.646 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.646 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.647 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:13.648 DEBUG:boto:Data: 2022-09-03T06:55:13.648 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.649 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.650 DEBUG:boto:Params: {} 2022-09-03T06:55:13.651 DEBUG:boto:Token: None 2022-09-03T06:55:13.651 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:13.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PWMAFzk/CSVGtaFazC19F4D7g8s= 2022-09-03T06:55:13.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PWMAFzk/CSVGtaFazC19F4D7g8s=', 'Content-Length': '0'} 2022-09-03T06:55:13.654 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e721de3124923ad-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.656 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:55:13.657 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.658 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.659 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.660 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.660 DEBUG:boto:Method: GET 2022-09-03T06:55:13.661 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:13.662 DEBUG:boto:Data: 2022-09-03T06:55:13.662 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.663 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.664 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.664 DEBUG:boto:Params: {} 2022-09-03T06:55:13.665 DEBUG:boto:Token: None 2022-09-03T06:55:13.666 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:13.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s+gfAIZW+3oh8gIFt2bVbOgtlEY= 2022-09-03T06:55:13.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s+gfAIZW+3oh8gIFt2bVbOgtlEY=', 'Content-Length': '0'} 2022-09-03T06:55:13.669 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000928ced0cfa0c8e9f-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.714 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.715 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.716 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.716 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:13.717 DEBUG:boto:Method: GET 2022-09-03T06:55:13.717 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:13.718 DEBUG:boto:Data: 2022-09-03T06:55:13.719 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.719 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.720 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.720 DEBUG:boto:Params: {} 2022-09-03T06:55:13.721 DEBUG:boto:Token: None 2022-09-03T06:55:13.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:13.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s+gfAIZW+3oh8gIFt2bVbOgtlEY= 2022-09-03T06:55:13.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s+gfAIZW+3oh8gIFt2bVbOgtlEY=', 'Content-Length': '0'} 2022-09-03T06:55:13.725 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031ef7c99fbfd8908-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.726 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.727 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.728 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:13.728 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:13.729 DEBUG:boto:Method: GET 2022-09-03T06:55:13.729 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:13.730 DEBUG:boto:Data: 2022-09-03T06:55:13.731 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.733 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.734 DEBUG:boto:Params: {} 2022-09-03T06:55:13.734 DEBUG:boto:Token: None 2022-09-03T06:55:13.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.740 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000eb6a406b6bb0e1e-006312fa51-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.740 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.741 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.742 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.742 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:13.743 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:13.743 DEBUG:boto:Method: GET 2022-09-03T06:55:13.744 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:13.745 DEBUG:boto:Data: 2022-09-03T06:55:13.745 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.746 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.747 DEBUG:boto:Params: {} 2022-09-03T06:55:13.747 DEBUG:boto:Token: None 2022-09-03T06:55:13.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f83c7585b1a15b7f-006312fa51-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.752 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.753 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.754 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.754 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.755 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:13.755 DEBUG:boto:Data: 2022-09-03T06:55:13.756 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.757 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.757 DEBUG:boto:Params: {} 2022-09-03T06:55:13.758 DEBUG:boto:Token: None 2022-09-03T06:55:13.759 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h 2022-09-03T06:55:13.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l23eGN/wNskXqLfU+duUxdc9iMY= 2022-09-03T06:55:13.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l23eGN/wNskXqLfU+duUxdc9iMY=', 'Content-Length': '0'} 2022-09-03T06:55:13.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000830e45692fa71d52-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.765 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.765 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.766 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.767 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:13.768 DEBUG:boto:Data: 2022-09-03T06:55:13.769 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.771 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.772 DEBUG:boto:Params: {} 2022-09-03T06:55:13.773 DEBUG:boto:Token: None 2022-09-03T06:55:13.774 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h 2022-09-03T06:55:13.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l23eGN/wNskXqLfU+duUxdc9iMY= 2022-09-03T06:55:13.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l23eGN/wNskXqLfU+duUxdc9iMY=', 'Content-Length': '0'} 2022-09-03T06:55:13.779 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c510ca080e3fd8f0-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.780 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:55:13.781 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.782 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.783 DEBUG:boto:Method: GET 2022-09-03T06:55:13.784 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:13.785 DEBUG:boto:Data: 2022-09-03T06:55:13.786 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.787 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.788 DEBUG:boto:Params: {} 2022-09-03T06:55:13.789 DEBUG:boto:Token: None 2022-09-03T06:55:13.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h 2022-09-03T06:55:13.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zHcr+9ycx4UvdHvQz+tm+4rfIDQ= 2022-09-03T06:55:13.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zHcr+9ycx4UvdHvQz+tm+4rfIDQ=', 'Content-Length': '0'} 2022-09-03T06:55:13.793 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000955cbe73a2468ca2-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.794 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:13.795 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:13.795 DEBUG:boto:Method: GET 2022-09-03T06:55:13.796 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:13.797 DEBUG:boto:Data: 2022-09-03T06:55:13.797 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.799 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.799 DEBUG:boto:Params: {} 2022-09-03T06:55:13.800 DEBUG:boto:Token: None 2022-09-03T06:55:13.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/h 2022-09-03T06:55:13.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zHcr+9ycx4UvdHvQz+tm+4rfIDQ= 2022-09-03T06:55:13.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zHcr+9ycx4UvdHvQz+tm+4rfIDQ=', 'Content-Length': '0'} 2022-09-03T06:55:13.805 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003acc16e66eb751bc-006312fa51-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.807 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.808 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.808 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:13.809 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:13.810 DEBUG:boto:Method: GET 2022-09-03T06:55:13.811 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:13.812 DEBUG:boto:Data: 2022-09-03T06:55:13.813 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.814 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.815 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.815 DEBUG:boto:Params: {} 2022-09-03T06:55:13.816 DEBUG:boto:Token: None 2022-09-03T06:55:13.816 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.821 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b61ed5ab6c434e12-006312fa51-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.822 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.823 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.824 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.825 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:13.825 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:13.826 DEBUG:boto:Method: GET 2022-09-03T06:55:13.826 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:13.827 DEBUG:boto:Data: 2022-09-03T06:55:13.827 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.828 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:13.828 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.829 DEBUG:boto:Params: {} 2022-09-03T06:55:13.829 DEBUG:boto:Token: None 2022-09-03T06:55:13.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs= 2022-09-03T06:55:13.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zuo1uL3wLxrHL7rxSpFeKnWrMqs=', 'Content-Length': '0'} 2022-09-03T06:55:13.834 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a573eda92622e4c0-006312fa51-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.835 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.836 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-71 zones={test-zone3, test-zone1} 2022-09-03T06:55:13.837 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-71 zones={test-zone3, test-zone2} 2022-09-03T06:55:13.838 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.840 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.841 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.842 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:55:13.842 DEBUG:boto:Data: 2022-09-03T06:55:13.843 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.843 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.844 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.844 DEBUG:boto:Params: {} 2022-09-03T06:55:13.845 DEBUG:boto:Token: None 2022-09-03T06:55:13.846 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DH6Msq1PfGnZTwhjaUXYMLO6RDI= 2022-09-03T06:55:13.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DH6Msq1PfGnZTwhjaUXYMLO6RDI=', 'Content-Length': '0'} 2022-09-03T06:55:13.850 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000709e5a89961c4401-006312fa51-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.850 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.851 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.851 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.852 DEBUG:boto:Path: /avysmw-71/ 2022-09-03T06:55:13.852 DEBUG:boto:Data: 2022-09-03T06:55:13.853 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.853 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:13.854 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.854 DEBUG:boto:Params: {} 2022-09-03T06:55:13.855 DEBUG:boto:Token: None 2022-09-03T06:55:13.856 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/ 2022-09-03T06:55:13.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DH6Msq1PfGnZTwhjaUXYMLO6RDI= 2022-09-03T06:55:13.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DH6Msq1PfGnZTwhjaUXYMLO6RDI=', 'Content-Length': '0'} 2022-09-03T06:55:13.862 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000061d6cb85323dd1b-006312fa51-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.862 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:55:13.863 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.863 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.864 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.864 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.865 DEBUG:boto:Method: GET 2022-09-03T06:55:13.865 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.867 DEBUG:boto:Data: 2022-09-03T06:55:13.868 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.868 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.869 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.869 DEBUG:boto:Params: {} 2022-09-03T06:55:13.870 DEBUG:boto:Token: None 2022-09-03T06:55:13.871 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/?versions 2022-09-03T06:55:13.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEu/cqAmX9DLjyu/8WU++anpoE0= 2022-09-03T06:55:13.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEu/cqAmX9DLjyu/8WU++anpoE0=', 'Content-Length': '0'} 2022-09-03T06:55:13.876 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c3a12c3106373251-006312fa51-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.877 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.878 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:55:13.879 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:55:13.879 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:55:13.880 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:55:13.880 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:55:13.881 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:55:13.881 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:55:13.882 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:55:13.882 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:55:13.883 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.883 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.884 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.884 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.885 DEBUG:boto:Method: GET 2022-09-03T06:55:13.886 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.886 DEBUG:boto:Data: 2022-09-03T06:55:13.887 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.887 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:13.888 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.889 DEBUG:boto:Params: {} 2022-09-03T06:55:13.889 DEBUG:boto:Token: None 2022-09-03T06:55:13.890 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/?versions 2022-09-03T06:55:13.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEu/cqAmX9DLjyu/8WU++anpoE0= 2022-09-03T06:55:13.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEu/cqAmX9DLjyu/8WU++anpoE0=', 'Content-Length': '0'} 2022-09-03T06:55:13.894 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000681328b0635dc784-006312fa51-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.895 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.896 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:55:13.897 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:55:13.897 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:55:13.898 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:55:13.898 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T06:55:13.899 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T06:55:13.899 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T06:55:13.900 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T06:55:13.900 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.901 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.901 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.903 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.904 DEBUG:boto:Method: GET 2022-09-03T06:55:13.905 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.906 DEBUG:boto:Data: 2022-09-03T06:55:13.907 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.908 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.909 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.910 DEBUG:boto:Params: {} 2022-09-03T06:55:13.912 DEBUG:boto:Token: None 2022-09-03T06:55:13.913 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/?versions 2022-09-03T06:55:13.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEu/cqAmX9DLjyu/8WU++anpoE0= 2022-09-03T06:55:13.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEu/cqAmX9DLjyu/8WU++anpoE0=', 'Content-Length': '0'} 2022-09-03T06:55:13.917 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c42915a939f9178c-006312fa51-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.918 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.919 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:13.920 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:13.921 DEBUG:boto:path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.921 DEBUG:boto:auth_path=/avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.922 DEBUG:boto:Method: GET 2022-09-03T06:55:13.923 DEBUG:boto:Path: /avysmw-71/?versions&max-keys=999 2022-09-03T06:55:13.923 DEBUG:boto:Data: 2022-09-03T06:55:13.924 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.925 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:13.925 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.926 DEBUG:boto:Params: {} 2022-09-03T06:55:13.927 DEBUG:boto:Token: None 2022-09-03T06:55:13.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/?versions 2022-09-03T06:55:13.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEu/cqAmX9DLjyu/8WU++anpoE0= 2022-09-03T06:55:13.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEu/cqAmX9DLjyu/8WU++anpoE0=', 'Content-Length': '0'} 2022-09-03T06:55:13.932 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000016affe729e170e0-006312fa51-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.933 DEBUG:boto:b'avysmw-71999falseanulltrue2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:13.934 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:55:13.934 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:13.935 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:13.935 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.936 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.936 DEBUG:boto:Method: GET 2022-09-03T06:55:13.937 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:13.937 DEBUG:boto:Data: 2022-09-03T06:55:13.938 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.939 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.940 DEBUG:boto:Params: {} 2022-09-03T06:55:13.940 DEBUG:boto:Token: None 2022-09-03T06:55:13.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:13.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JAUds5GmY4t/vvZtunmNWvqvS8I= 2022-09-03T06:55:13.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JAUds5GmY4t/vvZtunmNWvqvS8I=', 'Content-Length': '0'} 2022-09-03T06:55:13.944 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018a21d7593efff61-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.945 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:13.945 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:13.946 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.947 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.947 DEBUG:boto:Method: GET 2022-09-03T06:55:13.948 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:13.948 DEBUG:boto:Data: 2022-09-03T06:55:13.949 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:13.950 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.950 DEBUG:boto:Params: {} 2022-09-03T06:55:13.951 DEBUG:boto:Token: None 2022-09-03T06:55:13.951 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:13.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JAUds5GmY4t/vvZtunmNWvqvS8I= 2022-09-03T06:55:13.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JAUds5GmY4t/vvZtunmNWvqvS8I=', 'Content-Length': '0'} 2022-09-03T06:55:13.957 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006220d8352c234b37-006312fa51-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.958 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:13.958 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:13.959 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.959 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.960 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.960 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:13.961 DEBUG:boto:Data: 2022-09-03T06:55:13.961 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.962 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.962 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.963 DEBUG:boto:Params: {} 2022-09-03T06:55:13.963 DEBUG:boto:Token: None 2022-09-03T06:55:13.964 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:13.964 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hKYdfA3RChzjWZT/+1nHrtHVb5I= 2022-09-03T06:55:13.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hKYdfA3RChzjWZT/+1nHrtHVb5I=', 'Content-Length': '0'} 2022-09-03T06:55:13.967 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe381ebfb3f8fa66-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.967 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:13.968 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:13.968 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.969 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.969 DEBUG:boto:Method: HEAD 2022-09-03T06:55:13.970 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:13.970 DEBUG:boto:Data: 2022-09-03T06:55:13.971 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.971 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:13.972 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.972 DEBUG:boto:Params: {} 2022-09-03T06:55:13.973 DEBUG:boto:Token: None 2022-09-03T06:55:13.974 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:13.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hKYdfA3RChzjWZT/+1nHrtHVb5I= 2022-09-03T06:55:13.975 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hKYdfA3RChzjWZT/+1nHrtHVb5I=', 'Content-Length': '0'} 2022-09-03T06:55:13.977 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e8b385e7216f762-006312fa51-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.978 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:55:13.978 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:13.979 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:13.979 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.980 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.980 DEBUG:boto:Method: GET 2022-09-03T06:55:13.981 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:13.981 DEBUG:boto:Data: 2022-09-03T06:55:13.982 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.982 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:13.983 DEBUG:boto:Port: 8001 2022-09-03T06:55:13.983 DEBUG:boto:Params: {} 2022-09-03T06:55:13.984 DEBUG:boto:Token: None 2022-09-03T06:55:13.985 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:13.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JAUds5GmY4t/vvZtunmNWvqvS8I= 2022-09-03T06:55:13.986 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JAUds5GmY4t/vvZtunmNWvqvS8I=', 'Content-Length': '0'} 2022-09-03T06:55:13.988 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032eabcfbd6904c5e-006312fa51-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:13.989 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:13.989 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:13.990 DEBUG:boto:path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.990 DEBUG:boto:auth_path=/avysmw-71/a?versionId=null 2022-09-03T06:55:13.991 DEBUG:boto:Method: GET 2022-09-03T06:55:13.991 DEBUG:boto:Path: /avysmw-71/a?versionId=null 2022-09-03T06:55:13.992 DEBUG:boto:Data: 2022-09-03T06:55:13.992 DEBUG:boto:Headers: {} 2022-09-03T06:55:13.993 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:13.993 DEBUG:boto:Port: 8000 2022-09-03T06:55:13.994 DEBUG:boto:Params: {} 2022-09-03T06:55:13.994 DEBUG:boto:Token: None 2022-09-03T06:55:13.995 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:13 GMT /avysmw-71/a?versionId=null 2022-09-03T06:55:13.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JAUds5GmY4t/vvZtunmNWvqvS8I= 2022-09-03T06:55:13.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JAUds5GmY4t/vvZtunmNWvqvS8I=', 'Content-Length': '0'} 2022-09-03T06:55:13.998 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf4c431dd2669215-006312fa51-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.042 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.043 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.043 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:14.044 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:14.044 DEBUG:boto:Method: GET 2022-09-03T06:55:14.045 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:14.045 DEBUG:boto:Data: 2022-09-03T06:55:14.046 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.047 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.048 DEBUG:boto:Params: {} 2022-09-03T06:55:14.048 DEBUG:boto:Token: None 2022-09-03T06:55:14.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.054 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004340c12e1393ace0-006312fa52-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.055 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.056 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.057 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.057 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:14.058 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:14.059 DEBUG:boto:Method: GET 2022-09-03T06:55:14.059 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:14.060 DEBUG:boto:Data: 2022-09-03T06:55:14.060 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.061 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.061 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.062 DEBUG:boto:Params: {} 2022-09-03T06:55:14.063 DEBUG:boto:Token: None 2022-09-03T06:55:14.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000881307241229d386-006312fa52-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.070 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.072 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:14.073 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:14.074 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.075 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:14.076 DEBUG:boto:Data: 2022-09-03T06:55:14.077 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.078 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.079 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.079 DEBUG:boto:Params: {} 2022-09-03T06:55:14.080 DEBUG:boto:Token: None 2022-09-03T06:55:14.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/a 2022-09-03T06:55:14.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X7K2ThIASVrzvjRnFpUyWS+YgIs= 2022-09-03T06:55:14.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X7K2ThIASVrzvjRnFpUyWS+YgIs=', 'Content-Length': '0'} 2022-09-03T06:55:14.085 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e8b0f0b5ce58c64-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.085 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:14.086 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:14.086 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.087 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:14.087 DEBUG:boto:Data: 2022-09-03T06:55:14.088 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.089 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.090 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.091 DEBUG:boto:Params: {} 2022-09-03T06:55:14.092 DEBUG:boto:Token: None 2022-09-03T06:55:14.093 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/a 2022-09-03T06:55:14.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X7K2ThIASVrzvjRnFpUyWS+YgIs= 2022-09-03T06:55:14.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X7K2ThIASVrzvjRnFpUyWS+YgIs=', 'Content-Length': '0'} 2022-09-03T06:55:14.097 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd35c24c53fab298-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.098 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:55:14.099 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:14.099 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:14.100 DEBUG:boto:Method: GET 2022-09-03T06:55:14.100 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:14.101 DEBUG:boto:Data: 2022-09-03T06:55:14.101 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.102 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.102 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.103 DEBUG:boto:Params: {} 2022-09-03T06:55:14.104 DEBUG:boto:Token: None 2022-09-03T06:55:14.104 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/a 2022-09-03T06:55:14.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jlJI87/HBZLCaeF9u39Ft+aNbuU= 2022-09-03T06:55:14.106 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jlJI87/HBZLCaeF9u39Ft+aNbuU=', 'Content-Length': '0'} 2022-09-03T06:55:14.108 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a69eb0415e12ede2-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.150 DEBUG:boto:path=/avysmw-71/a 2022-09-03T06:55:14.151 DEBUG:boto:auth_path=/avysmw-71/a 2022-09-03T06:55:14.152 DEBUG:boto:Method: GET 2022-09-03T06:55:14.152 DEBUG:boto:Path: /avysmw-71/a 2022-09-03T06:55:14.153 DEBUG:boto:Data: 2022-09-03T06:55:14.154 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.154 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.155 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.155 DEBUG:boto:Params: {} 2022-09-03T06:55:14.156 DEBUG:boto:Token: None 2022-09-03T06:55:14.157 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/a 2022-09-03T06:55:14.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jlJI87/HBZLCaeF9u39Ft+aNbuU= 2022-09-03T06:55:14.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jlJI87/HBZLCaeF9u39Ft+aNbuU=', 'Content-Length': '0'} 2022-09-03T06:55:14.160 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df65ea54ba62b135-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.161 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.161 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.162 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:14.162 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:14.163 DEBUG:boto:Method: GET 2022-09-03T06:55:14.164 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:14.164 DEBUG:boto:Data: 2022-09-03T06:55:14.165 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.166 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.167 DEBUG:boto:Params: {} 2022-09-03T06:55:14.167 DEBUG:boto:Token: None 2022-09-03T06:55:14.168 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.169 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.172 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005fb1046c3e70359d-006312fa52-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.173 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.175 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.176 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.176 DEBUG:boto:path=/avysmw-71/?prefix=a 2022-09-03T06:55:14.177 DEBUG:boto:auth_path=/avysmw-71/?prefix=a 2022-09-03T06:55:14.177 DEBUG:boto:Method: GET 2022-09-03T06:55:14.179 DEBUG:boto:Path: /avysmw-71/?prefix=a 2022-09-03T06:55:14.179 DEBUG:boto:Data: 2022-09-03T06:55:14.180 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.180 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.181 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.181 DEBUG:boto:Params: {} 2022-09-03T06:55:14.182 DEBUG:boto:Token: None 2022-09-03T06:55:14.183 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.183 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.184 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.187 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079719f62bf69806e-006312fa52-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.188 DEBUG:boto:b'avysmw-71a1000falsea2022-09-03T06:51:13.081Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.189 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:55:14.189 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.190 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.190 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.191 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.191 DEBUG:boto:Method: GET 2022-09-03T06:55:14.192 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:14.192 DEBUG:boto:Data: 2022-09-03T06:55:14.193 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.193 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.194 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.194 DEBUG:boto:Params: {} 2022-09-03T06:55:14.195 DEBUG:boto:Token: None 2022-09-03T06:55:14.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:14.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mg5gkItLE8nALO0CwcdTWlh00gY= 2022-09-03T06:55:14.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mg5gkItLE8nALO0CwcdTWlh00gY=', 'Content-Length': '0'} 2022-09-03T06:55:14.199 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000051326f55478e4919-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.200 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.200 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.201 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.201 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.202 DEBUG:boto:Method: GET 2022-09-03T06:55:14.202 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:14.203 DEBUG:boto:Data: 2022-09-03T06:55:14.203 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.204 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.204 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.205 DEBUG:boto:Params: {} 2022-09-03T06:55:14.205 DEBUG:boto:Token: None 2022-09-03T06:55:14.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:14.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mg5gkItLE8nALO0CwcdTWlh00gY= 2022-09-03T06:55:14.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mg5gkItLE8nALO0CwcdTWlh00gY=', 'Content-Length': '0'} 2022-09-03T06:55:14.210 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000237b91927d36b38a-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.250 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.251 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.251 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.252 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.252 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.253 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:14.253 DEBUG:boto:Data: 2022-09-03T06:55:14.254 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.254 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.255 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.255 DEBUG:boto:Params: {} 2022-09-03T06:55:14.257 DEBUG:boto:Token: None 2022-09-03T06:55:14.258 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:14.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPTIM4Z9z4LTKRK0Xbxp7BDYQeI= 2022-09-03T06:55:14.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPTIM4Z9z4LTKRK0Xbxp7BDYQeI=', 'Content-Length': '0'} 2022-09-03T06:55:14.261 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8598296d7b6730e-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.262 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.262 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.263 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.263 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.264 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.264 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:14.265 DEBUG:boto:Data: 2022-09-03T06:55:14.265 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.266 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.267 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.267 DEBUG:boto:Params: {} 2022-09-03T06:55:14.268 DEBUG:boto:Token: None 2022-09-03T06:55:14.269 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:14.270 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPTIM4Z9z4LTKRK0Xbxp7BDYQeI= 2022-09-03T06:55:14.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPTIM4Z9z4LTKRK0Xbxp7BDYQeI=', 'Content-Length': '0'} 2022-09-03T06:55:14.274 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ae28d14b3aee44c-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.275 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:55:14.275 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.276 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.277 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.277 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.278 DEBUG:boto:Method: GET 2022-09-03T06:55:14.279 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:14.279 DEBUG:boto:Data: 2022-09-03T06:55:14.280 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.281 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.283 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.284 DEBUG:boto:Params: {} 2022-09-03T06:55:14.285 DEBUG:boto:Token: None 2022-09-03T06:55:14.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:14.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mg5gkItLE8nALO0CwcdTWlh00gY= 2022-09-03T06:55:14.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mg5gkItLE8nALO0CwcdTWlh00gY=', 'Content-Length': '0'} 2022-09-03T06:55:14.291 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5448e5f1aaf1f4b-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.292 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.293 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.293 DEBUG:boto:path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.294 DEBUG:boto:auth_path=/avysmw-71/b?versionId=null 2022-09-03T06:55:14.294 DEBUG:boto:Method: GET 2022-09-03T06:55:14.295 DEBUG:boto:Path: /avysmw-71/b?versionId=null 2022-09-03T06:55:14.296 DEBUG:boto:Data: 2022-09-03T06:55:14.297 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.297 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.298 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.299 DEBUG:boto:Params: {} 2022-09-03T06:55:14.299 DEBUG:boto:Token: None 2022-09-03T06:55:14.300 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b?versionId=null 2022-09-03T06:55:14.301 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mg5gkItLE8nALO0CwcdTWlh00gY= 2022-09-03T06:55:14.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mg5gkItLE8nALO0CwcdTWlh00gY=', 'Content-Length': '0'} 2022-09-03T06:55:14.305 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023e787c25b662c33-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.307 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.308 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.309 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:14.309 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:14.310 DEBUG:boto:Method: GET 2022-09-03T06:55:14.310 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:14.311 DEBUG:boto:Data: 2022-09-03T06:55:14.312 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.313 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.313 DEBUG:boto:Params: {} 2022-09-03T06:55:14.314 DEBUG:boto:Token: None 2022-09-03T06:55:14.314 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.318 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d348cb1452f99aa3-006312fa52-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.320 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.321 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.322 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.322 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:14.323 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:14.324 DEBUG:boto:Method: GET 2022-09-03T06:55:14.325 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:14.325 DEBUG:boto:Data: 2022-09-03T06:55:14.326 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.327 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.328 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.329 DEBUG:boto:Params: {} 2022-09-03T06:55:14.330 DEBUG:boto:Token: None 2022-09-03T06:55:14.331 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.332 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.336 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006b409a938fc7df64-006312fa52-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.337 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.338 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.339 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.339 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.340 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:14.341 DEBUG:boto:Data: 2022-09-03T06:55:14.341 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.342 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.342 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.343 DEBUG:boto:Params: {} 2022-09-03T06:55:14.343 DEBUG:boto:Token: None 2022-09-03T06:55:14.344 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b 2022-09-03T06:55:14.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sjWAe8NtkIbtQQzEhcWSfhTKylA= 2022-09-03T06:55:14.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sjWAe8NtkIbtQQzEhcWSfhTKylA=', 'Content-Length': '0'} 2022-09-03T06:55:14.347 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c520d338c7b86dc-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.348 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.348 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.349 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.349 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:14.350 DEBUG:boto:Data: 2022-09-03T06:55:14.350 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.351 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.351 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.352 DEBUG:boto:Params: {} 2022-09-03T06:55:14.353 DEBUG:boto:Token: None 2022-09-03T06:55:14.353 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b 2022-09-03T06:55:14.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sjWAe8NtkIbtQQzEhcWSfhTKylA= 2022-09-03T06:55:14.354 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sjWAe8NtkIbtQQzEhcWSfhTKylA=', 'Content-Length': '0'} 2022-09-03T06:55:14.357 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000577325393358de4-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.358 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:55:14.359 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.360 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.361 DEBUG:boto:Method: GET 2022-09-03T06:55:14.362 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:14.363 DEBUG:boto:Data: 2022-09-03T06:55:14.364 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.365 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.366 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.367 DEBUG:boto:Params: {} 2022-09-03T06:55:14.368 DEBUG:boto:Token: None 2022-09-03T06:55:14.368 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b 2022-09-03T06:55:14.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:965y8zcvIbZ41oa/nKLqdSPFf9o= 2022-09-03T06:55:14.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:965y8zcvIbZ41oa/nKLqdSPFf9o=', 'Content-Length': '0'} 2022-09-03T06:55:14.372 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000794cd40260b454e3-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.414 DEBUG:boto:path=/avysmw-71/b 2022-09-03T06:55:14.415 DEBUG:boto:auth_path=/avysmw-71/b 2022-09-03T06:55:14.416 DEBUG:boto:Method: GET 2022-09-03T06:55:14.416 DEBUG:boto:Path: /avysmw-71/b 2022-09-03T06:55:14.417 DEBUG:boto:Data: 2022-09-03T06:55:14.417 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.418 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.418 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.419 DEBUG:boto:Params: {} 2022-09-03T06:55:14.420 DEBUG:boto:Token: None 2022-09-03T06:55:14.420 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/b 2022-09-03T06:55:14.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:965y8zcvIbZ41oa/nKLqdSPFf9o= 2022-09-03T06:55:14.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:965y8zcvIbZ41oa/nKLqdSPFf9o=', 'Content-Length': '0'} 2022-09-03T06:55:14.424 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a4608c3cebcec62-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.425 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.426 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.426 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:14.427 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:14.428 DEBUG:boto:Method: GET 2022-09-03T06:55:14.429 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:14.429 DEBUG:boto:Data: 2022-09-03T06:55:14.430 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.430 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.431 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.431 DEBUG:boto:Params: {} 2022-09-03T06:55:14.432 DEBUG:boto:Token: None 2022-09-03T06:55:14.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.436 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000052745985df72cde8-006312fa52-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.437 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.438 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.439 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.439 DEBUG:boto:path=/avysmw-71/?prefix=b 2022-09-03T06:55:14.440 DEBUG:boto:auth_path=/avysmw-71/?prefix=b 2022-09-03T06:55:14.440 DEBUG:boto:Method: GET 2022-09-03T06:55:14.441 DEBUG:boto:Path: /avysmw-71/?prefix=b 2022-09-03T06:55:14.441 DEBUG:boto:Data: 2022-09-03T06:55:14.442 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.442 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.443 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.443 DEBUG:boto:Params: {} 2022-09-03T06:55:14.444 DEBUG:boto:Token: None 2022-09-03T06:55:14.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.448 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000873976997e4f2b1e-006312fa52-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.449 DEBUG:boto:b'avysmw-71b1000falseb2022-09-03T06:51:13.154Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.450 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:55:14.451 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.451 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.452 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.452 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.453 DEBUG:boto:Method: GET 2022-09-03T06:55:14.453 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:14.454 DEBUG:boto:Data: 2022-09-03T06:55:14.455 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.456 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.457 DEBUG:boto:Params: {} 2022-09-03T06:55:14.459 DEBUG:boto:Token: None 2022-09-03T06:55:14.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:14.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+c5/QLJVIhi+E9/uEN7EwnITxE= 2022-09-03T06:55:14.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+c5/QLJVIhi+E9/uEN7EwnITxE=', 'Content-Length': '0'} 2022-09-03T06:55:14.465 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e1751092135a9c3-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.466 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.466 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.467 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.467 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.468 DEBUG:boto:Method: GET 2022-09-03T06:55:14.468 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:14.469 DEBUG:boto:Data: 2022-09-03T06:55:14.469 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.470 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.470 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.471 DEBUG:boto:Params: {} 2022-09-03T06:55:14.471 DEBUG:boto:Token: None 2022-09-03T06:55:14.472 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:14.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+c5/QLJVIhi+E9/uEN7EwnITxE= 2022-09-03T06:55:14.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+c5/QLJVIhi+E9/uEN7EwnITxE=', 'Content-Length': '0'} 2022-09-03T06:55:14.475 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d262421e9840209a-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.476 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.477 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.477 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.478 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.478 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.479 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:14.480 DEBUG:boto:Data: 2022-09-03T06:55:14.480 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.481 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.481 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.482 DEBUG:boto:Params: {} 2022-09-03T06:55:14.482 DEBUG:boto:Token: None 2022-09-03T06:55:14.483 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:14.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pv9JGcQR3eKiD2SuFbM8IwbVdc0= 2022-09-03T06:55:14.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pv9JGcQR3eKiD2SuFbM8IwbVdc0=', 'Content-Length': '0'} 2022-09-03T06:55:14.486 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b53c7eb0d853f941-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.487 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.488 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.488 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.489 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.489 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.490 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:14.491 DEBUG:boto:Data: 2022-09-03T06:55:14.491 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.492 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.492 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.493 DEBUG:boto:Params: {} 2022-09-03T06:55:14.493 DEBUG:boto:Token: None 2022-09-03T06:55:14.494 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:14.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pv9JGcQR3eKiD2SuFbM8IwbVdc0= 2022-09-03T06:55:14.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pv9JGcQR3eKiD2SuFbM8IwbVdc0=', 'Content-Length': '0'} 2022-09-03T06:55:14.497 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d07fd033ac9b601-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.498 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:55:14.498 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.499 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.499 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.500 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.501 DEBUG:boto:Method: GET 2022-09-03T06:55:14.501 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:14.502 DEBUG:boto:Data: 2022-09-03T06:55:14.502 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.503 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.504 DEBUG:boto:Params: {} 2022-09-03T06:55:14.504 DEBUG:boto:Token: None 2022-09-03T06:55:14.505 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:14.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+c5/QLJVIhi+E9/uEN7EwnITxE= 2022-09-03T06:55:14.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+c5/QLJVIhi+E9/uEN7EwnITxE=', 'Content-Length': '0'} 2022-09-03T06:55:14.509 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000237b0a3fb7d01c8b-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.510 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.510 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.511 DEBUG:boto:path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.512 DEBUG:boto:auth_path=/avysmw-71/c?versionId=null 2022-09-03T06:55:14.512 DEBUG:boto:Method: GET 2022-09-03T06:55:14.513 DEBUG:boto:Path: /avysmw-71/c?versionId=null 2022-09-03T06:55:14.513 DEBUG:boto:Data: 2022-09-03T06:55:14.514 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.514 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.515 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.515 DEBUG:boto:Params: {} 2022-09-03T06:55:14.516 DEBUG:boto:Token: None 2022-09-03T06:55:14.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c?versionId=null 2022-09-03T06:55:14.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+c5/QLJVIhi+E9/uEN7EwnITxE= 2022-09-03T06:55:14.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+c5/QLJVIhi+E9/uEN7EwnITxE=', 'Content-Length': '0'} 2022-09-03T06:55:14.521 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b542dc029beba756-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.522 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.523 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.523 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:14.524 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:14.525 DEBUG:boto:Method: GET 2022-09-03T06:55:14.525 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:14.526 DEBUG:boto:Data: 2022-09-03T06:55:14.526 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.527 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.527 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.528 DEBUG:boto:Params: {} 2022-09-03T06:55:14.528 DEBUG:boto:Token: None 2022-09-03T06:55:14.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.533 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002641f6a5d048a6e0-006312fa52-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.534 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.535 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.535 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.536 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:14.536 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:14.537 DEBUG:boto:Method: GET 2022-09-03T06:55:14.537 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:14.538 DEBUG:boto:Data: 2022-09-03T06:55:14.538 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.539 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.539 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.540 DEBUG:boto:Params: {} 2022-09-03T06:55:14.541 DEBUG:boto:Token: None 2022-09-03T06:55:14.541 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.545 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a47591663365ef5d-006312fa52-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.546 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.547 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.548 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.549 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.549 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:14.550 DEBUG:boto:Data: 2022-09-03T06:55:14.550 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.551 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.552 DEBUG:boto:Params: {} 2022-09-03T06:55:14.552 DEBUG:boto:Token: None 2022-09-03T06:55:14.553 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c 2022-09-03T06:55:14.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzcA6EJzrunRMkyleervsN883NI= 2022-09-03T06:55:14.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzcA6EJzrunRMkyleervsN883NI=', 'Content-Length': '0'} 2022-09-03T06:55:14.556 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068dbb0583abfb53c-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.557 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.558 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.559 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.559 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:14.560 DEBUG:boto:Data: 2022-09-03T06:55:14.560 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.561 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.561 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.562 DEBUG:boto:Params: {} 2022-09-03T06:55:14.562 DEBUG:boto:Token: None 2022-09-03T06:55:14.563 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c 2022-09-03T06:55:14.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzcA6EJzrunRMkyleervsN883NI= 2022-09-03T06:55:14.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzcA6EJzrunRMkyleervsN883NI=', 'Content-Length': '0'} 2022-09-03T06:55:14.566 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007232ebe7917164cc-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.567 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:55:14.567 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.568 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.569 DEBUG:boto:Method: GET 2022-09-03T06:55:14.569 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:14.570 DEBUG:boto:Data: 2022-09-03T06:55:14.570 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.571 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.572 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.572 DEBUG:boto:Params: {} 2022-09-03T06:55:14.573 DEBUG:boto:Token: None 2022-09-03T06:55:14.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c 2022-09-03T06:55:14.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1jQPl9G7lCW/Jgx95qqUeKwPq5U= 2022-09-03T06:55:14.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1jQPl9G7lCW/Jgx95qqUeKwPq5U=', 'Content-Length': '0'} 2022-09-03T06:55:14.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c6d2522798d95258-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.580 DEBUG:boto:path=/avysmw-71/c 2022-09-03T06:55:14.580 DEBUG:boto:auth_path=/avysmw-71/c 2022-09-03T06:55:14.581 DEBUG:boto:Method: GET 2022-09-03T06:55:14.581 DEBUG:boto:Path: /avysmw-71/c 2022-09-03T06:55:14.582 DEBUG:boto:Data: 2022-09-03T06:55:14.583 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.584 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.585 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.586 DEBUG:boto:Params: {} 2022-09-03T06:55:14.587 DEBUG:boto:Token: None 2022-09-03T06:55:14.587 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/c 2022-09-03T06:55:14.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1jQPl9G7lCW/Jgx95qqUeKwPq5U= 2022-09-03T06:55:14.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1jQPl9G7lCW/Jgx95qqUeKwPq5U=', 'Content-Length': '0'} 2022-09-03T06:55:14.591 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000017796e27ec78a94a-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.634 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.635 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.635 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:14.636 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:14.636 DEBUG:boto:Method: GET 2022-09-03T06:55:14.637 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:14.638 DEBUG:boto:Data: 2022-09-03T06:55:14.638 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.639 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.640 DEBUG:boto:Params: {} 2022-09-03T06:55:14.641 DEBUG:boto:Token: None 2022-09-03T06:55:14.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.643 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.645 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000038c4406e55b3e8f7-006312fa52-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.646 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.647 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.647 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.648 DEBUG:boto:path=/avysmw-71/?prefix=c 2022-09-03T06:55:14.649 DEBUG:boto:auth_path=/avysmw-71/?prefix=c 2022-09-03T06:55:14.649 DEBUG:boto:Method: GET 2022-09-03T06:55:14.650 DEBUG:boto:Path: /avysmw-71/?prefix=c 2022-09-03T06:55:14.650 DEBUG:boto:Data: 2022-09-03T06:55:14.651 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.651 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.652 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.652 DEBUG:boto:Params: {} 2022-09-03T06:55:14.653 DEBUG:boto:Token: None 2022-09-03T06:55:14.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.654 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.655 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.657 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000038b3538327932bf8-006312fa52-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.658 DEBUG:boto:b'avysmw-71c1000falsec2022-09-03T06:51:13.193Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.659 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:55:14.659 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.660 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.660 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.661 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.662 DEBUG:boto:Method: GET 2022-09-03T06:55:14.662 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:14.663 DEBUG:boto:Data: 2022-09-03T06:55:14.663 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.664 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.664 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.665 DEBUG:boto:Params: {} 2022-09-03T06:55:14.665 DEBUG:boto:Token: None 2022-09-03T06:55:14.666 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:14.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nncoj3eETCQmQy6Y6JFajdE019g= 2022-09-03T06:55:14.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nncoj3eETCQmQy6Y6JFajdE019g=', 'Content-Length': '0'} 2022-09-03T06:55:14.670 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002056598288c44bb0-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.671 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.671 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.672 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.672 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.673 DEBUG:boto:Method: GET 2022-09-03T06:55:14.673 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:14.674 DEBUG:boto:Data: 2022-09-03T06:55:14.674 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.675 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.676 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.683 DEBUG:boto:Params: {} 2022-09-03T06:55:14.684 DEBUG:boto:Token: None 2022-09-03T06:55:14.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:14.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nncoj3eETCQmQy6Y6JFajdE019g= 2022-09-03T06:55:14.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nncoj3eETCQmQy6Y6JFajdE019g=', 'Content-Length': '0'} 2022-09-03T06:55:14.688 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d90484940b39641-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.689 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.690 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.691 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.691 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.692 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.693 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:14.693 DEBUG:boto:Data: 2022-09-03T06:55:14.694 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.694 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.695 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.696 DEBUG:boto:Params: {} 2022-09-03T06:55:14.696 DEBUG:boto:Token: None 2022-09-03T06:55:14.697 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:14.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m7AMtuq82epSPYg9L7s/18vnnOk= 2022-09-03T06:55:14.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m7AMtuq82epSPYg9L7s/18vnnOk=', 'Content-Length': '0'} 2022-09-03T06:55:14.700 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a58381ada361fb9d-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.701 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.701 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.702 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.703 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.703 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.704 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:14.704 DEBUG:boto:Data: 2022-09-03T06:55:14.705 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.705 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.706 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.707 DEBUG:boto:Params: {} 2022-09-03T06:55:14.707 DEBUG:boto:Token: None 2022-09-03T06:55:14.708 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:14.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m7AMtuq82epSPYg9L7s/18vnnOk= 2022-09-03T06:55:14.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m7AMtuq82epSPYg9L7s/18vnnOk=', 'Content-Length': '0'} 2022-09-03T06:55:14.711 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006bfc0d5ab1b2e2aa-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.712 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:55:14.712 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.713 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.714 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.714 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.715 DEBUG:boto:Method: GET 2022-09-03T06:55:14.715 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:14.716 DEBUG:boto:Data: 2022-09-03T06:55:14.716 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.717 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.718 DEBUG:boto:Params: {} 2022-09-03T06:55:14.719 DEBUG:boto:Token: None 2022-09-03T06:55:14.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:14.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nncoj3eETCQmQy6Y6JFajdE019g= 2022-09-03T06:55:14.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nncoj3eETCQmQy6Y6JFajdE019g=', 'Content-Length': '0'} 2022-09-03T06:55:14.723 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca48208a6183a783-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.723 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.724 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.725 DEBUG:boto:path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.725 DEBUG:boto:auth_path=/avysmw-71/d?versionId=null 2022-09-03T06:55:14.726 DEBUG:boto:Method: GET 2022-09-03T06:55:14.727 DEBUG:boto:Path: /avysmw-71/d?versionId=null 2022-09-03T06:55:14.727 DEBUG:boto:Data: 2022-09-03T06:55:14.728 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.728 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.729 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.729 DEBUG:boto:Params: {} 2022-09-03T06:55:14.730 DEBUG:boto:Token: None 2022-09-03T06:55:14.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d?versionId=null 2022-09-03T06:55:14.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nncoj3eETCQmQy6Y6JFajdE019g= 2022-09-03T06:55:14.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nncoj3eETCQmQy6Y6JFajdE019g=', 'Content-Length': '0'} 2022-09-03T06:55:14.734 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f9c42a259452f04-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.735 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.736 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.736 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:14.737 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:14.737 DEBUG:boto:Method: GET 2022-09-03T06:55:14.738 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:14.739 DEBUG:boto:Data: 2022-09-03T06:55:14.739 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.740 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.740 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.741 DEBUG:boto:Params: {} 2022-09-03T06:55:14.742 DEBUG:boto:Token: None 2022-09-03T06:55:14.742 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.746 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e031957d2dd7e53b-006312fa52-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.790 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.792 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.793 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.794 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:14.794 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:14.795 DEBUG:boto:Method: GET 2022-09-03T06:55:14.796 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:14.796 DEBUG:boto:Data: 2022-09-03T06:55:14.797 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.797 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.798 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.799 DEBUG:boto:Params: {} 2022-09-03T06:55:14.799 DEBUG:boto:Token: None 2022-09-03T06:55:14.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.805 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f02c213a01c61141-006312fa52-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.806 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.806 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.807 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.807 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.808 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:14.809 DEBUG:boto:Data: 2022-09-03T06:55:14.809 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.810 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.811 DEBUG:boto:Params: {} 2022-09-03T06:55:14.811 DEBUG:boto:Token: None 2022-09-03T06:55:14.812 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d 2022-09-03T06:55:14.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WySRVE9g2r0zy5iJygESqSefTdc= 2022-09-03T06:55:14.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WySRVE9g2r0zy5iJygESqSefTdc=', 'Content-Length': '0'} 2022-09-03T06:55:14.815 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000273a0b070a6c708e-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.816 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.816 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.817 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.817 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:14.818 DEBUG:boto:Data: 2022-09-03T06:55:14.819 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.819 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.820 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.820 DEBUG:boto:Params: {} 2022-09-03T06:55:14.821 DEBUG:boto:Token: None 2022-09-03T06:55:14.821 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d 2022-09-03T06:55:14.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WySRVE9g2r0zy5iJygESqSefTdc= 2022-09-03T06:55:14.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WySRVE9g2r0zy5iJygESqSefTdc=', 'Content-Length': '0'} 2022-09-03T06:55:14.826 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007b1cefa264ff32ff-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.827 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:55:14.828 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.828 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.829 DEBUG:boto:Method: GET 2022-09-03T06:55:14.829 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:14.830 DEBUG:boto:Data: 2022-09-03T06:55:14.831 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.832 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.833 DEBUG:boto:Params: {} 2022-09-03T06:55:14.833 DEBUG:boto:Token: None 2022-09-03T06:55:14.834 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d 2022-09-03T06:55:14.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a/c0iOMbX8qqEz042HqfNC819/U= 2022-09-03T06:55:14.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a/c0iOMbX8qqEz042HqfNC819/U=', 'Content-Length': '0'} 2022-09-03T06:55:14.838 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5b1fa4838888c15-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.839 DEBUG:boto:path=/avysmw-71/d 2022-09-03T06:55:14.839 DEBUG:boto:auth_path=/avysmw-71/d 2022-09-03T06:55:14.840 DEBUG:boto:Method: GET 2022-09-03T06:55:14.840 DEBUG:boto:Path: /avysmw-71/d 2022-09-03T06:55:14.841 DEBUG:boto:Data: 2022-09-03T06:55:14.841 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.842 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.843 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.843 DEBUG:boto:Params: {} 2022-09-03T06:55:14.844 DEBUG:boto:Token: None 2022-09-03T06:55:14.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/d 2022-09-03T06:55:14.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a/c0iOMbX8qqEz042HqfNC819/U= 2022-09-03T06:55:14.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a/c0iOMbX8qqEz042HqfNC819/U=', 'Content-Length': '0'} 2022-09-03T06:55:14.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:51:13 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004736bd05988dc3bb-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.849 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.850 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.850 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:14.851 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:14.851 DEBUG:boto:Method: GET 2022-09-03T06:55:14.852 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:14.852 DEBUG:boto:Data: 2022-09-03T06:55:14.853 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.854 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.854 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.855 DEBUG:boto:Params: {} 2022-09-03T06:55:14.855 DEBUG:boto:Token: None 2022-09-03T06:55:14.856 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.860 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000195a9d01b289c43c-006312fa52-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.860 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.861 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.862 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.863 DEBUG:boto:path=/avysmw-71/?prefix=d 2022-09-03T06:55:14.863 DEBUG:boto:auth_path=/avysmw-71/?prefix=d 2022-09-03T06:55:14.864 DEBUG:boto:Method: GET 2022-09-03T06:55:14.864 DEBUG:boto:Path: /avysmw-71/?prefix=d 2022-09-03T06:55:14.865 DEBUG:boto:Data: 2022-09-03T06:55:14.865 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.866 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.867 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.867 DEBUG:boto:Params: {} 2022-09-03T06:55:14.868 DEBUG:boto:Token: None 2022-09-03T06:55:14.868 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.872 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000685574ece38ac6c5-006312fa52-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.873 DEBUG:boto:b'avysmw-71d1000falsed2022-09-03T06:51:13.236Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:14.874 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:55:14.874 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:14.875 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:14.876 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.876 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.877 DEBUG:boto:Method: GET 2022-09-03T06:55:14.877 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:14.878 DEBUG:boto:Data: 2022-09-03T06:55:14.878 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.879 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.880 DEBUG:boto:Params: {} 2022-09-03T06:55:14.881 DEBUG:boto:Token: None 2022-09-03T06:55:14.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:14.882 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w3Yvco1ne7C9Tb30hDgprC0QUNI= 2022-09-03T06:55:14.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w3Yvco1ne7C9Tb30hDgprC0QUNI=', 'Content-Length': '0'} 2022-09-03T06:55:14.884 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ece238063c26c7c-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.885 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:14.886 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:14.886 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.887 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.887 DEBUG:boto:Method: GET 2022-09-03T06:55:14.888 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:14.889 DEBUG:boto:Data: 2022-09-03T06:55:14.889 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.890 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.891 DEBUG:boto:Params: {} 2022-09-03T06:55:14.891 DEBUG:boto:Token: None 2022-09-03T06:55:14.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:14.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w3Yvco1ne7C9Tb30hDgprC0QUNI= 2022-09-03T06:55:14.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w3Yvco1ne7C9Tb30hDgprC0QUNI=', 'Content-Length': '0'} 2022-09-03T06:55:14.896 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbdc75601dce23e0-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.896 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:14.897 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:14.898 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.898 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.899 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.899 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:14.900 DEBUG:boto:Data: 2022-09-03T06:55:14.900 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.901 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.902 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.902 DEBUG:boto:Params: {} 2022-09-03T06:55:14.903 DEBUG:boto:Token: None 2022-09-03T06:55:14.903 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:14.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zloZRoS2Z7t3LJ/kb3bWFfkAG6E= 2022-09-03T06:55:14.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zloZRoS2Z7t3LJ/kb3bWFfkAG6E=', 'Content-Length': '0'} 2022-09-03T06:55:14.906 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e724cf2e26ac2c86-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.907 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:14.908 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:14.908 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.909 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.910 DEBUG:boto:Method: HEAD 2022-09-03T06:55:14.910 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:14.911 DEBUG:boto:Data: 2022-09-03T06:55:14.911 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.912 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.913 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.913 DEBUG:boto:Params: {} 2022-09-03T06:55:14.914 DEBUG:boto:Token: None 2022-09-03T06:55:14.915 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:14.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zloZRoS2Z7t3LJ/kb3bWFfkAG6E= 2022-09-03T06:55:14.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zloZRoS2Z7t3LJ/kb3bWFfkAG6E=', 'Content-Length': '0'} 2022-09-03T06:55:14.920 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf67ab9f60cd0c20-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.920 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:55:14.921 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:14.921 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:14.922 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.923 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.923 DEBUG:boto:Method: GET 2022-09-03T06:55:14.924 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:14.924 DEBUG:boto:Data: 2022-09-03T06:55:14.925 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.926 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.926 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.927 DEBUG:boto:Params: {} 2022-09-03T06:55:14.928 DEBUG:boto:Token: None 2022-09-03T06:55:14.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:14.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w3Yvco1ne7C9Tb30hDgprC0QUNI= 2022-09-03T06:55:14.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w3Yvco1ne7C9Tb30hDgprC0QUNI=', 'Content-Length': '0'} 2022-09-03T06:55:14.932 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7a77f520c953165-006312fa52-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.932 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:14.933 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:14.934 DEBUG:boto:path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.934 DEBUG:boto:auth_path=/avysmw-71/e?versionId=null 2022-09-03T06:55:14.935 DEBUG:boto:Method: GET 2022-09-03T06:55:14.935 DEBUG:boto:Path: /avysmw-71/e?versionId=null 2022-09-03T06:55:14.936 DEBUG:boto:Data: 2022-09-03T06:55:14.937 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.937 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:14.938 DEBUG:boto:Port: 8000 2022-09-03T06:55:14.938 DEBUG:boto:Params: {} 2022-09-03T06:55:14.939 DEBUG:boto:Token: None 2022-09-03T06:55:14.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/e?versionId=null 2022-09-03T06:55:14.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w3Yvco1ne7C9Tb30hDgprC0QUNI= 2022-09-03T06:55:14.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w3Yvco1ne7C9Tb30hDgprC0QUNI=', 'Content-Length': '0'} 2022-09-03T06:55:14.943 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8dd4c688e144a1b-006312fa52-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.986 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:14.987 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:14.988 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:14.988 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:14.989 DEBUG:boto:Method: GET 2022-09-03T06:55:14.989 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:14.990 DEBUG:boto:Data: 2022-09-03T06:55:14.991 DEBUG:boto:Headers: {} 2022-09-03T06:55:14.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:14.992 DEBUG:boto:Port: 8001 2022-09-03T06:55:14.992 DEBUG:boto:Params: {} 2022-09-03T06:55:14.993 DEBUG:boto:Token: None 2022-09-03T06:55:14.994 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:14 GMT /avysmw-71/ 2022-09-03T06:55:14.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY= 2022-09-03T06:55:14.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0+xVZG2LU5EYVbFX4TDEeXi6OUY=', 'Content-Length': '0'} 2022-09-03T06:55:14.998 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000af2cd0149171644c-006312fa52-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:14.999 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.000 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.001 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.001 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:15.002 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:15.002 DEBUG:boto:Method: GET 2022-09-03T06:55:15.003 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:15.004 DEBUG:boto:Data: 2022-09-03T06:55:15.004 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.005 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.005 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.006 DEBUG:boto:Params: {} 2022-09-03T06:55:15.006 DEBUG:boto:Token: None 2022-09-03T06:55:15.007 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.011 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000019eed65db6bc6d6c-006312fa53-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.012 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.013 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:15.013 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:15.014 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.015 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:15.015 DEBUG:boto:Data: 2022-09-03T06:55:15.016 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.016 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.017 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.017 DEBUG:boto:Params: {} 2022-09-03T06:55:15.018 DEBUG:boto:Token: None 2022-09-03T06:55:15.018 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/e 2022-09-03T06:55:15.019 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+VK5o/1ZbScYLq0qTUaZ+3LJdRo= 2022-09-03T06:55:15.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+VK5o/1ZbScYLq0qTUaZ+3LJdRo=', 'Content-Length': '0'} 2022-09-03T06:55:15.022 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053b52d5368923118-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.022 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:15.023 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:15.024 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.024 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:15.025 DEBUG:boto:Data: 2022-09-03T06:55:15.025 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.026 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.026 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.027 DEBUG:boto:Params: {} 2022-09-03T06:55:15.028 DEBUG:boto:Token: None 2022-09-03T06:55:15.028 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/e 2022-09-03T06:55:15.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+VK5o/1ZbScYLq0qTUaZ+3LJdRo= 2022-09-03T06:55:15.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+VK5o/1ZbScYLq0qTUaZ+3LJdRo=', 'Content-Length': '0'} 2022-09-03T06:55:15.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016fbf00e10aae745-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.032 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T06:55:15.033 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:15.033 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:15.034 DEBUG:boto:Method: GET 2022-09-03T06:55:15.034 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:15.035 DEBUG:boto:Data: 2022-09-03T06:55:15.035 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.037 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.038 DEBUG:boto:Params: {} 2022-09-03T06:55:15.038 DEBUG:boto:Token: None 2022-09-03T06:55:15.039 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/e 2022-09-03T06:55:15.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+E0X9p8kkT0CZ2kZ3DmKjItYuuE= 2022-09-03T06:55:15.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+E0X9p8kkT0CZ2kZ3DmKjItYuuE=', 'Content-Length': '0'} 2022-09-03T06:55:15.043 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039fee3d2b24392c7-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.086 DEBUG:boto:path=/avysmw-71/e 2022-09-03T06:55:15.087 DEBUG:boto:auth_path=/avysmw-71/e 2022-09-03T06:55:15.088 DEBUG:boto:Method: GET 2022-09-03T06:55:15.089 DEBUG:boto:Path: /avysmw-71/e 2022-09-03T06:55:15.090 DEBUG:boto:Data: 2022-09-03T06:55:15.091 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.092 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.093 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.094 DEBUG:boto:Params: {} 2022-09-03T06:55:15.095 DEBUG:boto:Token: None 2022-09-03T06:55:15.096 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/e 2022-09-03T06:55:15.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+E0X9p8kkT0CZ2kZ3DmKjItYuuE= 2022-09-03T06:55:15.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+E0X9p8kkT0CZ2kZ3DmKjItYuuE=', 'Content-Length': '0'} 2022-09-03T06:55:15.101 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f98cf481c3e105b0-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.102 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.103 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.104 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:15.105 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:15.106 DEBUG:boto:Method: GET 2022-09-03T06:55:15.107 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:15.108 DEBUG:boto:Data: 2022-09-03T06:55:15.109 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.110 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.111 DEBUG:boto:Params: {} 2022-09-03T06:55:15.112 DEBUG:boto:Token: None 2022-09-03T06:55:15.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.119 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad32941c726c2495-006312fa53-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.120 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.121 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.122 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.123 DEBUG:boto:path=/avysmw-71/?prefix=e 2022-09-03T06:55:15.124 DEBUG:boto:auth_path=/avysmw-71/?prefix=e 2022-09-03T06:55:15.125 DEBUG:boto:Method: GET 2022-09-03T06:55:15.126 DEBUG:boto:Path: /avysmw-71/?prefix=e 2022-09-03T06:55:15.127 DEBUG:boto:Data: 2022-09-03T06:55:15.128 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.129 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.129 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.130 DEBUG:boto:Params: {} 2022-09-03T06:55:15.131 DEBUG:boto:Token: None 2022-09-03T06:55:15.132 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.138 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2afc12dd6abc9e5-006312fa53-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.139 DEBUG:boto:b'avysmw-71e1000falsee2022-09-03T06:52:54.718Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.141 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:55:15.141 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.142 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.143 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.144 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.145 DEBUG:boto:Method: GET 2022-09-03T06:55:15.146 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:15.147 DEBUG:boto:Data: 2022-09-03T06:55:15.148 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.149 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.150 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.151 DEBUG:boto:Params: {} 2022-09-03T06:55:15.152 DEBUG:boto:Token: None 2022-09-03T06:55:15.153 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:15.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sk3zJtiCc4KhOFHG2nZJ6Y3+mdY= 2022-09-03T06:55:15.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sk3zJtiCc4KhOFHG2nZJ6Y3+mdY=', 'Content-Length': '0'} 2022-09-03T06:55:15.157 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006bf9541a4c1d5482-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.158 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.159 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.159 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.160 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.160 DEBUG:boto:Method: GET 2022-09-03T06:55:15.161 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:15.161 DEBUG:boto:Data: 2022-09-03T06:55:15.162 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.162 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.163 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.163 DEBUG:boto:Params: {} 2022-09-03T06:55:15.164 DEBUG:boto:Token: None 2022-09-03T06:55:15.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:15.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sk3zJtiCc4KhOFHG2nZJ6Y3+mdY= 2022-09-03T06:55:15.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sk3zJtiCc4KhOFHG2nZJ6Y3+mdY=', 'Content-Length': '0'} 2022-09-03T06:55:15.168 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000325551ec35962924-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.211 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.211 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.212 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.213 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.213 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.214 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:15.214 DEBUG:boto:Data: 2022-09-03T06:55:15.215 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.216 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.216 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.217 DEBUG:boto:Params: {} 2022-09-03T06:55:15.217 DEBUG:boto:Token: None 2022-09-03T06:55:15.218 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:15.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ozn7BHOLKnupBwe1JG8MoNp0w4s= 2022-09-03T06:55:15.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ozn7BHOLKnupBwe1JG8MoNp0w4s=', 'Content-Length': '0'} 2022-09-03T06:55:15.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000596fe3b71be01be1-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.222 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.223 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.223 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.224 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.225 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.225 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:15.226 DEBUG:boto:Data: 2022-09-03T06:55:15.226 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.227 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.228 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.229 DEBUG:boto:Params: {} 2022-09-03T06:55:15.230 DEBUG:boto:Token: None 2022-09-03T06:55:15.231 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:15.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ozn7BHOLKnupBwe1JG8MoNp0w4s= 2022-09-03T06:55:15.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ozn7BHOLKnupBwe1JG8MoNp0w4s=', 'Content-Length': '0'} 2022-09-03T06:55:15.239 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071447a5848686382-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.240 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:55:15.241 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.241 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.242 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.243 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.244 DEBUG:boto:Method: GET 2022-09-03T06:55:15.245 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:15.246 DEBUG:boto:Data: 2022-09-03T06:55:15.247 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.248 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.249 DEBUG:boto:Params: {} 2022-09-03T06:55:15.249 DEBUG:boto:Token: None 2022-09-03T06:55:15.250 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:15.250 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sk3zJtiCc4KhOFHG2nZJ6Y3+mdY= 2022-09-03T06:55:15.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sk3zJtiCc4KhOFHG2nZJ6Y3+mdY=', 'Content-Length': '0'} 2022-09-03T06:55:15.253 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2b7bb0e0dd6f024-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.254 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.254 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.255 DEBUG:boto:path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.255 DEBUG:boto:auth_path=/avysmw-71/f?versionId=null 2022-09-03T06:55:15.256 DEBUG:boto:Method: GET 2022-09-03T06:55:15.256 DEBUG:boto:Path: /avysmw-71/f?versionId=null 2022-09-03T06:55:15.257 DEBUG:boto:Data: 2022-09-03T06:55:15.258 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.258 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.259 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.259 DEBUG:boto:Params: {} 2022-09-03T06:55:15.260 DEBUG:boto:Token: None 2022-09-03T06:55:15.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f?versionId=null 2022-09-03T06:55:15.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sk3zJtiCc4KhOFHG2nZJ6Y3+mdY= 2022-09-03T06:55:15.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sk3zJtiCc4KhOFHG2nZJ6Y3+mdY=', 'Content-Length': '0'} 2022-09-03T06:55:15.264 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c517c98c252690d-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.265 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.265 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.266 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:15.266 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:15.267 DEBUG:boto:Method: GET 2022-09-03T06:55:15.267 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:15.268 DEBUG:boto:Data: 2022-09-03T06:55:15.268 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.270 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.270 DEBUG:boto:Params: {} 2022-09-03T06:55:15.271 DEBUG:boto:Token: None 2022-09-03T06:55:15.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.275 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001b24d5f032204683-006312fa53-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.277 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.278 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.279 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.279 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:15.280 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:15.280 DEBUG:boto:Method: GET 2022-09-03T06:55:15.281 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:15.281 DEBUG:boto:Data: 2022-09-03T06:55:15.282 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.283 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.283 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.284 DEBUG:boto:Params: {} 2022-09-03T06:55:15.284 DEBUG:boto:Token: None 2022-09-03T06:55:15.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.289 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c7a813e938c959a2-006312fa53-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.289 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.290 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.291 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.291 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.292 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:15.292 DEBUG:boto:Data: 2022-09-03T06:55:15.293 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.294 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.294 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.295 DEBUG:boto:Params: {} 2022-09-03T06:55:15.295 DEBUG:boto:Token: None 2022-09-03T06:55:15.296 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f 2022-09-03T06:55:15.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:StK2ut+YqSH7XjW+vnrCTO/lqGU= 2022-09-03T06:55:15.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:StK2ut+YqSH7XjW+vnrCTO/lqGU=', 'Content-Length': '0'} 2022-09-03T06:55:15.299 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e89e67bf4ce47af3-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.300 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.301 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.301 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.302 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:15.303 DEBUG:boto:Data: 2022-09-03T06:55:15.303 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.304 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.304 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.305 DEBUG:boto:Params: {} 2022-09-03T06:55:15.305 DEBUG:boto:Token: None 2022-09-03T06:55:15.306 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f 2022-09-03T06:55:15.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:StK2ut+YqSH7XjW+vnrCTO/lqGU= 2022-09-03T06:55:15.307 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:StK2ut+YqSH7XjW+vnrCTO/lqGU=', 'Content-Length': '0'} 2022-09-03T06:55:15.309 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b1b2f9b9f38a27d-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.310 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T06:55:15.310 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.311 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.312 DEBUG:boto:Method: GET 2022-09-03T06:55:15.313 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:15.314 DEBUG:boto:Data: 2022-09-03T06:55:15.315 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.315 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.316 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.316 DEBUG:boto:Params: {} 2022-09-03T06:55:15.317 DEBUG:boto:Token: None 2022-09-03T06:55:15.318 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f 2022-09-03T06:55:15.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PdcocjZpIiqFnmgYpfkX4hPojFk= 2022-09-03T06:55:15.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PdcocjZpIiqFnmgYpfkX4hPojFk=', 'Content-Length': '0'} 2022-09-03T06:55:15.326 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a38249fe6f891253-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.373 DEBUG:boto:path=/avysmw-71/f 2022-09-03T06:55:15.374 DEBUG:boto:auth_path=/avysmw-71/f 2022-09-03T06:55:15.374 DEBUG:boto:Method: GET 2022-09-03T06:55:15.375 DEBUG:boto:Path: /avysmw-71/f 2022-09-03T06:55:15.375 DEBUG:boto:Data: 2022-09-03T06:55:15.376 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.376 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.377 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.377 DEBUG:boto:Params: {} 2022-09-03T06:55:15.378 DEBUG:boto:Token: None 2022-09-03T06:55:15.379 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/f 2022-09-03T06:55:15.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PdcocjZpIiqFnmgYpfkX4hPojFk= 2022-09-03T06:55:15.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PdcocjZpIiqFnmgYpfkX4hPojFk=', 'Content-Length': '0'} 2022-09-03T06:55:15.382 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c0ae8c33044b805-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.383 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.383 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.384 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:15.384 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:15.385 DEBUG:boto:Method: GET 2022-09-03T06:55:15.385 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:15.386 DEBUG:boto:Data: 2022-09-03T06:55:15.386 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.387 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.387 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.388 DEBUG:boto:Params: {} 2022-09-03T06:55:15.388 DEBUG:boto:Token: None 2022-09-03T06:55:15.390 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.394 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006113eaa040d35e11-006312fa53-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.395 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.396 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.396 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.397 DEBUG:boto:path=/avysmw-71/?prefix=f 2022-09-03T06:55:15.398 DEBUG:boto:auth_path=/avysmw-71/?prefix=f 2022-09-03T06:55:15.399 DEBUG:boto:Method: GET 2022-09-03T06:55:15.399 DEBUG:boto:Path: /avysmw-71/?prefix=f 2022-09-03T06:55:15.400 DEBUG:boto:Data: 2022-09-03T06:55:15.400 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.401 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.401 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.402 DEBUG:boto:Params: {} 2022-09-03T06:55:15.403 DEBUG:boto:Token: None 2022-09-03T06:55:15.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.407 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000252a110cc4e2fc87-006312fa53-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.408 DEBUG:boto:b'avysmw-71f1000falsef2022-09-03T06:52:54.743Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.408 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:55:15.409 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.411 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.411 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.413 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.414 DEBUG:boto:Method: GET 2022-09-03T06:55:15.415 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:15.416 DEBUG:boto:Data: 2022-09-03T06:55:15.417 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.418 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.418 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.419 DEBUG:boto:Params: {} 2022-09-03T06:55:15.419 DEBUG:boto:Token: None 2022-09-03T06:55:15.420 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:15.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GO+UMbsn8YZyFnGHWSEbdXk2YZ4= 2022-09-03T06:55:15.421 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GO+UMbsn8YZyFnGHWSEbdXk2YZ4=', 'Content-Length': '0'} 2022-09-03T06:55:15.424 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b4d3cce0a712e110-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.425 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.426 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.426 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.427 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.427 DEBUG:boto:Method: GET 2022-09-03T06:55:15.428 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:15.429 DEBUG:boto:Data: 2022-09-03T06:55:15.429 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.430 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.430 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.431 DEBUG:boto:Params: {} 2022-09-03T06:55:15.432 DEBUG:boto:Token: None 2022-09-03T06:55:15.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:15.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GO+UMbsn8YZyFnGHWSEbdXk2YZ4= 2022-09-03T06:55:15.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GO+UMbsn8YZyFnGHWSEbdXk2YZ4=', 'Content-Length': '0'} 2022-09-03T06:55:15.436 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5013a4e7334d3a3-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.478 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.479 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.480 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.481 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.482 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.483 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:15.484 DEBUG:boto:Data: 2022-09-03T06:55:15.485 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.486 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.487 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.488 DEBUG:boto:Params: {} 2022-09-03T06:55:15.489 DEBUG:boto:Token: None 2022-09-03T06:55:15.490 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:15.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pdMBOYpKyFxNI8TJLGhEgdNC6+0= 2022-09-03T06:55:15.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pdMBOYpKyFxNI8TJLGhEgdNC6+0=', 'Content-Length': '0'} 2022-09-03T06:55:15.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052b39e31d1e0abe1-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.497 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.498 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.499 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.500 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.501 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.502 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:15.503 DEBUG:boto:Data: 2022-09-03T06:55:15.504 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.504 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.505 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.506 DEBUG:boto:Params: {} 2022-09-03T06:55:15.506 DEBUG:boto:Token: None 2022-09-03T06:55:15.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:15.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pdMBOYpKyFxNI8TJLGhEgdNC6+0= 2022-09-03T06:55:15.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pdMBOYpKyFxNI8TJLGhEgdNC6+0=', 'Content-Length': '0'} 2022-09-03T06:55:15.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3fe7924bf90c01b-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.512 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:55:15.512 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.513 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.513 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.514 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.515 DEBUG:boto:Method: GET 2022-09-03T06:55:15.516 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:15.516 DEBUG:boto:Data: 2022-09-03T06:55:15.517 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.518 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.519 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.519 DEBUG:boto:Params: {} 2022-09-03T06:55:15.520 DEBUG:boto:Token: None 2022-09-03T06:55:15.521 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:15.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GO+UMbsn8YZyFnGHWSEbdXk2YZ4= 2022-09-03T06:55:15.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GO+UMbsn8YZyFnGHWSEbdXk2YZ4=', 'Content-Length': '0'} 2022-09-03T06:55:15.525 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097199b894800b8c7-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.526 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.526 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.527 DEBUG:boto:path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.527 DEBUG:boto:auth_path=/avysmw-71/g?versionId=null 2022-09-03T06:55:15.528 DEBUG:boto:Method: GET 2022-09-03T06:55:15.528 DEBUG:boto:Path: /avysmw-71/g?versionId=null 2022-09-03T06:55:15.529 DEBUG:boto:Data: 2022-09-03T06:55:15.530 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.530 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.531 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.532 DEBUG:boto:Params: {} 2022-09-03T06:55:15.533 DEBUG:boto:Token: None 2022-09-03T06:55:15.533 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g?versionId=null 2022-09-03T06:55:15.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GO+UMbsn8YZyFnGHWSEbdXk2YZ4= 2022-09-03T06:55:15.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GO+UMbsn8YZyFnGHWSEbdXk2YZ4=', 'Content-Length': '0'} 2022-09-03T06:55:15.537 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045659aec8da71ba4-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.538 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.540 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.541 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:15.542 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:15.543 DEBUG:boto:Method: GET 2022-09-03T06:55:15.543 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:15.545 DEBUG:boto:Data: 2022-09-03T06:55:15.545 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.546 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.547 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.548 DEBUG:boto:Params: {} 2022-09-03T06:55:15.549 DEBUG:boto:Token: None 2022-09-03T06:55:15.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.555 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000077cbce096b5a2e5e-006312fa53-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.556 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.557 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.557 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.558 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:15.559 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:15.559 DEBUG:boto:Method: GET 2022-09-03T06:55:15.560 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:15.560 DEBUG:boto:Data: 2022-09-03T06:55:15.561 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.561 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.562 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.562 DEBUG:boto:Params: {} 2022-09-03T06:55:15.563 DEBUG:boto:Token: None 2022-09-03T06:55:15.564 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.565 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.568 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009236790423ae68ac-006312fa53-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.568 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.569 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.570 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.570 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.571 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:15.571 DEBUG:boto:Data: 2022-09-03T06:55:15.572 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.572 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.573 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.573 DEBUG:boto:Params: {} 2022-09-03T06:55:15.574 DEBUG:boto:Token: None 2022-09-03T06:55:15.575 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g 2022-09-03T06:55:15.575 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8GcJs8AydQHQBtz+hajicR2/XiY= 2022-09-03T06:55:15.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8GcJs8AydQHQBtz+hajicR2/XiY=', 'Content-Length': '0'} 2022-09-03T06:55:15.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000025c7cf17aabb8d57-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.579 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.580 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.580 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.581 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:15.582 DEBUG:boto:Data: 2022-09-03T06:55:15.582 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.583 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.583 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.584 DEBUG:boto:Params: {} 2022-09-03T06:55:15.584 DEBUG:boto:Token: None 2022-09-03T06:55:15.585 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g 2022-09-03T06:55:15.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8GcJs8AydQHQBtz+hajicR2/XiY= 2022-09-03T06:55:15.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8GcJs8AydQHQBtz+hajicR2/XiY=', 'Content-Length': '0'} 2022-09-03T06:55:15.588 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c30cd052f051c7e7-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.589 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T06:55:15.590 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.590 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.591 DEBUG:boto:Method: GET 2022-09-03T06:55:15.591 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:15.592 DEBUG:boto:Data: 2022-09-03T06:55:15.592 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.593 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.594 DEBUG:boto:Params: {} 2022-09-03T06:55:15.595 DEBUG:boto:Token: None 2022-09-03T06:55:15.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g 2022-09-03T06:55:15.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b95D5d/g7q6ofJLz7mP7e4Q9L6o= 2022-09-03T06:55:15.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b95D5d/g7q6ofJLz7mP7e4Q9L6o=', 'Content-Length': '0'} 2022-09-03T06:55:15.598 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c558d5cf811a5765-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.642 DEBUG:boto:path=/avysmw-71/g 2022-09-03T06:55:15.643 DEBUG:boto:auth_path=/avysmw-71/g 2022-09-03T06:55:15.643 DEBUG:boto:Method: GET 2022-09-03T06:55:15.644 DEBUG:boto:Path: /avysmw-71/g 2022-09-03T06:55:15.645 DEBUG:boto:Data: 2022-09-03T06:55:15.645 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.646 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.646 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.647 DEBUG:boto:Params: {} 2022-09-03T06:55:15.647 DEBUG:boto:Token: None 2022-09-03T06:55:15.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/g 2022-09-03T06:55:15.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b95D5d/g7q6ofJLz7mP7e4Q9L6o= 2022-09-03T06:55:15.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b95D5d/g7q6ofJLz7mP7e4Q9L6o=', 'Content-Length': '0'} 2022-09-03T06:55:15.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023b64bd71b819913-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.653 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.654 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.655 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:15.656 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:15.658 DEBUG:boto:Method: GET 2022-09-03T06:55:15.659 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:15.659 DEBUG:boto:Data: 2022-09-03T06:55:15.660 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.661 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.662 DEBUG:boto:Params: {} 2022-09-03T06:55:15.663 DEBUG:boto:Token: None 2022-09-03T06:55:15.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.669 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff484c874a134da8-006312fa53-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.670 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.671 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.672 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.672 DEBUG:boto:path=/avysmw-71/?prefix=g 2022-09-03T06:55:15.673 DEBUG:boto:auth_path=/avysmw-71/?prefix=g 2022-09-03T06:55:15.674 DEBUG:boto:Method: GET 2022-09-03T06:55:15.675 DEBUG:boto:Path: /avysmw-71/?prefix=g 2022-09-03T06:55:15.676 DEBUG:boto:Data: 2022-09-03T06:55:15.676 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.677 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.677 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.678 DEBUG:boto:Params: {} 2022-09-03T06:55:15.679 DEBUG:boto:Token: None 2022-09-03T06:55:15.679 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.685 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ea3e75d3ac72133-006312fa53-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.685 DEBUG:boto:b'avysmw-71g1000falseg2022-09-03T06:52:54.807Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.686 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:55:15.687 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.687 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.688 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.688 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.689 DEBUG:boto:Method: GET 2022-09-03T06:55:15.689 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:15.690 DEBUG:boto:Data: 2022-09-03T06:55:15.690 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.692 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.692 DEBUG:boto:Params: {} 2022-09-03T06:55:15.693 DEBUG:boto:Token: None 2022-09-03T06:55:15.694 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:15.695 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xxFmet5zYH5TjW/9iiIgYNtiqLM= 2022-09-03T06:55:15.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xxFmet5zYH5TjW/9iiIgYNtiqLM=', 'Content-Length': '0'} 2022-09-03T06:55:15.698 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000800219d793d70da3-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.699 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.699 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.700 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.701 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.701 DEBUG:boto:Method: GET 2022-09-03T06:55:15.702 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:15.702 DEBUG:boto:Data: 2022-09-03T06:55:15.703 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.704 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.704 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.705 DEBUG:boto:Params: {} 2022-09-03T06:55:15.706 DEBUG:boto:Token: None 2022-09-03T06:55:15.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:15.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xxFmet5zYH5TjW/9iiIgYNtiqLM= 2022-09-03T06:55:15.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xxFmet5zYH5TjW/9iiIgYNtiqLM=', 'Content-Length': '0'} 2022-09-03T06:55:15.711 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d488824eca278254-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.712 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.712 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.713 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.713 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.714 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.714 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:15.715 DEBUG:boto:Data: 2022-09-03T06:55:15.716 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.717 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.717 DEBUG:boto:Params: {} 2022-09-03T06:55:15.718 DEBUG:boto:Token: None 2022-09-03T06:55:15.718 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:15.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IVVROYav4FKFpA8ZcmtfZf1rkSY= 2022-09-03T06:55:15.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IVVROYav4FKFpA8ZcmtfZf1rkSY=', 'Content-Length': '0'} 2022-09-03T06:55:15.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000015c69ff2ef463c1-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.722 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.723 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.724 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.724 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.725 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.725 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:15.726 DEBUG:boto:Data: 2022-09-03T06:55:15.726 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.727 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.727 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.728 DEBUG:boto:Params: {} 2022-09-03T06:55:15.728 DEBUG:boto:Token: None 2022-09-03T06:55:15.729 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:15.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IVVROYav4FKFpA8ZcmtfZf1rkSY= 2022-09-03T06:55:15.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IVVROYav4FKFpA8ZcmtfZf1rkSY=', 'Content-Length': '0'} 2022-09-03T06:55:15.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009ce33f95a33c8fd4-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.733 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:55:15.734 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.735 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.735 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.736 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.737 DEBUG:boto:Method: GET 2022-09-03T06:55:15.738 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:15.739 DEBUG:boto:Data: 2022-09-03T06:55:15.740 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.741 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.742 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.743 DEBUG:boto:Params: {} 2022-09-03T06:55:15.744 DEBUG:boto:Token: None 2022-09-03T06:55:15.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:15.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xxFmet5zYH5TjW/9iiIgYNtiqLM= 2022-09-03T06:55:15.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xxFmet5zYH5TjW/9iiIgYNtiqLM=', 'Content-Length': '0'} 2022-09-03T06:55:15.750 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009b3bd62c7de7192-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.751 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.751 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.752 DEBUG:boto:path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.752 DEBUG:boto:auth_path=/avysmw-71/h?versionId=null 2022-09-03T06:55:15.753 DEBUG:boto:Method: GET 2022-09-03T06:55:15.753 DEBUG:boto:Path: /avysmw-71/h?versionId=null 2022-09-03T06:55:15.754 DEBUG:boto:Data: 2022-09-03T06:55:15.754 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.755 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.756 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.756 DEBUG:boto:Params: {} 2022-09-03T06:55:15.757 DEBUG:boto:Token: None 2022-09-03T06:55:15.757 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h?versionId=null 2022-09-03T06:55:15.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xxFmet5zYH5TjW/9iiIgYNtiqLM= 2022-09-03T06:55:15.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xxFmet5zYH5TjW/9iiIgYNtiqLM=', 'Content-Length': '0'} 2022-09-03T06:55:15.761 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc6dfe38c0e575f2-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.802 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.803 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.804 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:15.805 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:15.805 DEBUG:boto:Method: GET 2022-09-03T06:55:15.806 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:15.807 DEBUG:boto:Data: 2022-09-03T06:55:15.807 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.808 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.809 DEBUG:boto:Params: {} 2022-09-03T06:55:15.809 DEBUG:boto:Token: None 2022-09-03T06:55:15.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.815 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000089cd0e74c3ae2410-006312fa53-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.815 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.816 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.817 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.817 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:15.818 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:15.819 DEBUG:boto:Method: GET 2022-09-03T06:55:15.819 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:15.820 DEBUG:boto:Data: 2022-09-03T06:55:15.820 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.821 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.821 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.822 DEBUG:boto:Params: {} 2022-09-03T06:55:15.823 DEBUG:boto:Token: None 2022-09-03T06:55:15.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.828 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f4092808ba25fa0e-006312fa53-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.828 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.830 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.830 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.831 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.832 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:15.832 DEBUG:boto:Data: 2022-09-03T06:55:15.833 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.833 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.834 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.834 DEBUG:boto:Params: {} 2022-09-03T06:55:15.835 DEBUG:boto:Token: None 2022-09-03T06:55:15.836 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h 2022-09-03T06:55:15.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hmONQK9AYl+QqTIOT4ps5njWL+o= 2022-09-03T06:55:15.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hmONQK9AYl+QqTIOT4ps5njWL+o=', 'Content-Length': '0'} 2022-09-03T06:55:15.839 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f86c91d95e2ebd9-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.840 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.841 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.842 DEBUG:boto:Method: HEAD 2022-09-03T06:55:15.842 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:15.843 DEBUG:boto:Data: 2022-09-03T06:55:15.844 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.845 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.846 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.846 DEBUG:boto:Params: {} 2022-09-03T06:55:15.847 DEBUG:boto:Token: None 2022-09-03T06:55:15.848 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h 2022-09-03T06:55:15.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hmONQK9AYl+QqTIOT4ps5njWL+o= 2022-09-03T06:55:15.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hmONQK9AYl+QqTIOT4ps5njWL+o=', 'Content-Length': '0'} 2022-09-03T06:55:15.852 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005cfc19d8621aa80e-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.854 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T06:55:15.854 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.855 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.856 DEBUG:boto:Method: GET 2022-09-03T06:55:15.856 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:15.857 DEBUG:boto:Data: 2022-09-03T06:55:15.858 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.858 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.859 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.859 DEBUG:boto:Params: {} 2022-09-03T06:55:15.860 DEBUG:boto:Token: None 2022-09-03T06:55:15.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h 2022-09-03T06:55:15.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T65kYzPcSR05KYfVQV6Mws+lDno= 2022-09-03T06:55:15.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T65kYzPcSR05KYfVQV6Mws+lDno=', 'Content-Length': '0'} 2022-09-03T06:55:15.866 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003af4b889c52cb5a9-006312fa53-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.867 DEBUG:boto:path=/avysmw-71/h 2022-09-03T06:55:15.867 DEBUG:boto:auth_path=/avysmw-71/h 2022-09-03T06:55:15.868 DEBUG:boto:Method: GET 2022-09-03T06:55:15.869 DEBUG:boto:Path: /avysmw-71/h 2022-09-03T06:55:15.869 DEBUG:boto:Data: 2022-09-03T06:55:15.870 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.870 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.871 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.871 DEBUG:boto:Params: {} 2022-09-03T06:55:15.873 DEBUG:boto:Token: None 2022-09-03T06:55:15.874 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/h 2022-09-03T06:55:15.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T65kYzPcSR05KYfVQV6Mws+lDno= 2022-09-03T06:55:15.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T65kYzPcSR05KYfVQV6Mws+lDno=', 'Content-Length': '0'} 2022-09-03T06:55:15.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:52:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005fb1f6f3bad2ade-006312fa53-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.878 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.878 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.879 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:15.879 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:15.880 DEBUG:boto:Method: GET 2022-09-03T06:55:15.880 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:15.881 DEBUG:boto:Data: 2022-09-03T06:55:15.881 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:55:15.882 DEBUG:boto:Port: 8001 2022-09-03T06:55:15.883 DEBUG:boto:Params: {} 2022-09-03T06:55:15.883 DEBUG:boto:Token: None 2022-09-03T06:55:15.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.888 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000438e8468f4f54d3e-006312fa53-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.930 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.931 DEBUG:boto:path=/avysmw-71/ 2022-09-03T06:55:15.932 DEBUG:boto:auth_path=/avysmw-71/ 2022-09-03T06:55:15.932 DEBUG:boto:path=/avysmw-71/?prefix=h 2022-09-03T06:55:15.933 DEBUG:boto:auth_path=/avysmw-71/?prefix=h 2022-09-03T06:55:15.934 DEBUG:boto:Method: GET 2022-09-03T06:55:15.934 DEBUG:boto:Path: /avysmw-71/?prefix=h 2022-09-03T06:55:15.935 DEBUG:boto:Data: 2022-09-03T06:55:15.936 DEBUG:boto:Headers: {} 2022-09-03T06:55:15.936 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:55:15.937 DEBUG:boto:Port: 8000 2022-09-03T06:55:15.937 DEBUG:boto:Params: {} 2022-09-03T06:55:15.938 DEBUG:boto:Token: None 2022-09-03T06:55:15.939 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:55:15 GMT /avysmw-71/ 2022-09-03T06:55:15.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI= 2022-09-03T06:55:15.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IHRf357dKjCNJPymGlD3yh6QpjI=', 'Content-Length': '0'} 2022-09-03T06:55:15.943 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000545205a8589c394b-006312fa53-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:55:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:15.943 DEBUG:boto:b'avysmw-71h1000falseh2022-09-03T06:52:54.839Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:55:15.944 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-71 zones={test-zone3, test-zone2} 2022-09-03T06:55:15.945 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:55:15.945 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:55:15.946 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:55:15.997 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:15.998 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:16.057 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.055+0000 7f170a98d700 -1 req 11277777707535569043 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.059 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.055+0000 7fa8d622b700 -1 req 7059158343616804983 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.060 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.055+0000 7f4ffb697700 -1 req 9423792249734418854 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-4:fca49d30-abba-4c2b-8462-66395ac063ba.4344.4,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.100 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.098+0000 7f5068f72700 -1 req 14626627834711885348 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.101 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.098+0000 7f16fc170700 -1 req 15124378897821930343 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.101 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.098+0000 7fa8ce21b700 -1 req 464061052378825680 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-3:fca49d30-abba-4c2b-8462-66395ac063ba.4344.3,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.105 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.103+0000 7f16eb14e700 -1 req 4001933922477212417 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.106 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.103+0000 7fa8c1201700 -1 req 18371913101855169331 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.107 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.103+0000 7f500debc700 -1 req 10535146199232221987 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-34:fca49d30-abba-4c2b-8462-66395ac063ba.4344.34,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.142 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.140+0000 7f16ed152700 -1 req 840261569043699344 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.144 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.140+0000 7f4fc4e2a700 -1 req 7338748490738990214 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.146 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.140+0000 7fa8ac9d8700 -1 req 10838154780813288497 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-2:fca49d30-abba-4c2b-8462-66395ac063ba.4344.2,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.227 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:55:16.228 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:55:16.229 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket layout --bucket avysmw-71 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:55:16.277 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:16.278 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:16.356 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:55:16.357 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.357 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding": "None", 2022-09-03T06:55:16.358 INFO:teuthology.orchestra.run.smithi062.stdout: "current_index": { 2022-09-03T06:55:16.359 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 2, 2022-09-03T06:55:16.359 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.360 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "Normal", 2022-09-03T06:55:16.360 INFO:teuthology.orchestra.run.smithi062.stdout: "normal": { 2022-09-03T06:55:16.361 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:55:16.361 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:55:16.362 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.362 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.363 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:16.363 INFO:teuthology.orchestra.run.smithi062.stdout: "logs": [ 2022-09-03T06:55:16.364 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:16.364 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 1, 2022-09-03T06:55:16.364 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.365 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:55:16.365 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:55:16.366 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 1, 2022-09-03T06:55:16.367 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.367 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:55:16.367 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:55:16.368 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.368 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.368 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.368 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:16.369 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:16.369 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 2, 2022-09-03T06:55:16.369 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.369 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:55:16.370 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:55:16.370 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 2, 2022-09-03T06:55:16.370 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.371 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:55:16.371 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:55:16.371 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.371 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.372 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.372 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.372 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:55:16.373 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.373 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:55:16.374 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:55:16.375 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:55:16.375 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:55:16.423 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:16.424 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:16.477 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.474+0000 7f4fc762f700 -1 req 9326305124476734599 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.478 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.474+0000 7f4fc3627700 -1 req 7350090413449677469 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.478 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.475+0000 7f4fd0641700 -1 req 879288026058440300 0.000999976s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.480 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.475+0000 7fa8951a9700 -1 req 18040450893042764000 0.000999976s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.481 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.476+0000 7fa89019f700 -1 req 240388142682913790 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.481 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.476+0000 7fa8cb215700 -1 req 3774755602694140636 0.001999953s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.482 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.478+0000 7fa88b195700 -1 req 15678672419492668258 0.000999976s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.483 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.474+0000 7f16ad8d3700 -1 req 12289301540061299171 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-5:fca49d30-abba-4c2b-8462-66395ac063ba.4344.5,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.483 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.474+0000 7f16d391f700 -1 req 11188057477231726798 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-28:fca49d30-abba-4c2b-8462-66395ac063ba.4344.28,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.484 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.475+0000 7f16b18db700 -1 req 16066607782731147293 0.000999984s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-29:fca49d30-abba-4c2b-8462-66395ac063ba.4344.29,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.484 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.478+0000 7f169b8af700 -1 req 1433470601413274876 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.484 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.479+0000 7f16980a8700 -1 req 10815052802030976 0.000999984s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.485 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.478+0000 7f4fe4669700 -1 req 2260892583077747576 0.000999976s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-31:fca49d30-abba-4c2b-8462-66395ac063ba.4344.31,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.486 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.478+0000 7fa87b976700 -1 req 1373527706882565611 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.487 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.478+0000 7f4f99dd4700 -1 req 13817659635828716528 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-7:fca49d30-abba-4c2b-8462-66395ac063ba.4344.7,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.523 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.522+0000 7fa8b21e3700 -1 req 7908614727910640033 0.000999976s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.524 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.522+0000 7f168b08e700 -1 req 6578819123985740546 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.525 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.522+0000 7f4fabdf8700 -1 req 14615228294564044103 0.000999976s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-53:fca49d30-abba-4c2b-8462-66395ac063ba.5638.4,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.528 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.527+0000 7f16db92f700 -1 req 17586864778786707709 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.529 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.527+0000 7f4f9d5db700 -1 req 17773050461007652235 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.530 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.527+0000 7fa968b50700 -1 req 250268244655156291 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-43:9b3b3b7d-a26b-43ae-9603-22cc8f947364.5512.2,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.566 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.564+0000 7f16cc911700 -1 req 15874901460965326863 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.567 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.564+0000 7fa96734d700 -1 req 10130173172993726726 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.567 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.564+0000 7f4fa95f3700 -1 req 3963590147266528767 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-67:fca49d30-abba-4c2b-8462-66395ac063ba.5638.18,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.570 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.568+0000 7f4fc9633700 -1 req 877948374473266263 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.570 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.568+0000 7f1690899700 -1 req 968740248305731339 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.571 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.568+0000 7fa95db3a700 -1 req 17905819420969957432 0.000999977s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-30:fca49d30-abba-4c2b-8462-66395ac063ba.4344.30,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.572 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T06:55:16.571+0000 7f16998ab700 -1 req 14157495316130484115 0.000999983s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=00aa9b6d-7a02-4715-9200-8a26b45eb25e,az=0}} returned ret=-2 2022-09-03T06:55:16.573 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T06:55:16.571+0000 7fa959331700 -1 req 18048270685227764306 0.000999976s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=31b17c08-52e2-4c88-9aba-17b28ca757e4,az=0}} returned ret=-2 2022-09-03T06:55:16.574 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T06:55:16.571+0000 7f4fb7e10700 -1 req 3974828017137766813 0.000999976s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=fca49d30-abba-4c2b-8462-66395ac063ba,az=0},d={b=avysmw-50:fca49d30-abba-4c2b-8462-66395ac063ba.5638.1,z=9b3b3b7d-a26b-43ae-9603-22cc8f947364,az=0}} returned ret=-2 2022-09-03T06:55:16.640 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:55:16.641 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'layout', '--bucket', 'avysmw-71', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:55:16.642 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket layout --bucket avysmw-71 --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:55:16.690 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:16.691 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:16.750 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:55:16.750 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.751 INFO:teuthology.orchestra.run.smithi062.stdout: "resharding": "None", 2022-09-03T06:55:16.751 INFO:teuthology.orchestra.run.smithi062.stdout: "current_index": { 2022-09-03T06:55:16.752 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 2, 2022-09-03T06:55:16.752 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.752 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "Normal", 2022-09-03T06:55:16.752 INFO:teuthology.orchestra.run.smithi062.stdout: "normal": { 2022-09-03T06:55:16.753 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:55:16.753 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:55:16.753 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.754 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.754 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:16.754 INFO:teuthology.orchestra.run.smithi062.stdout: "logs": [ 2022-09-03T06:55:16.755 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:16.755 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 1, 2022-09-03T06:55:16.755 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.755 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:55:16.756 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:55:16.756 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 1, 2022-09-03T06:55:16.756 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.756 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:55:16.757 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:55:16.757 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.757 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.757 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.758 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:16.758 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:16.758 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 2, 2022-09-03T06:55:16.759 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.759 INFO:teuthology.orchestra.run.smithi062.stdout: "type": "InIndex", 2022-09-03T06:55:16.759 INFO:teuthology.orchestra.run.smithi062.stdout: "in_index": { 2022-09-03T06:55:16.759 INFO:teuthology.orchestra.run.smithi062.stdout: "gen": 2, 2022-09-03T06:55:16.760 INFO:teuthology.orchestra.run.smithi062.stdout: "layout": { 2022-09-03T06:55:16.760 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 3, 2022-09-03T06:55:16.760 INFO:teuthology.orchestra.run.smithi062.stdout: "hash_type": "Mod" 2022-09-03T06:55:16.760 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.761 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.761 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.761 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.762 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:55:16.762 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.762 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:55:16.776 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_reshard_index_log_trim ... FAIL 2022-09-03T06:55:16.778 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:55:16.779 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:55:16.779 DEBUG:boto:Method: PUT 2022-09-03T06:55:16.780 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:55:16.781 DEBUG:boto:Data: 2022-09-03T06:55:16.781 DEBUG:boto:Headers: {} 2022-09-03T06:55:16.782 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:16.782 DEBUG:boto:Port: 8000 2022-09-03T06:55:16.783 DEBUG:boto:Params: {} 2022-09-03T06:55:16.783 DEBUG:boto:Token: None 2022-09-03T06:55:16.784 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 06:55:16 GMT /avysmw-72/ 2022-09-03T06:55:16.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QspBXPXbNkYk0BaqK2+0SvFPIuM= 2022-09-03T06:55:16.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QspBXPXbNkYk0BaqK2+0SvFPIuM=', 'Content-Length': '0'} 2022-09-03T06:55:16.804 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000007ef1a9605b8921d9-006312fa54-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:16.805 DEBUG:rgw_multi.tests:created bucket=avysmw-72 2022-09-03T06:55:16.806 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:55:16.806 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:55:16.807 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:55:16.862 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:16.862 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:16.903 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T06:55:16.919 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T06:55:16.919 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:16.920 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:55:16.920 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:55:16.920 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:16.920 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:16.921 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662188116.801696_424.1", 2022-09-03T06:55:16.921 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T06:55:16.801696Z" 2022-09-03T06:55:16.921 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:16.921 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:16.922 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:55:16.922 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:55:16.922 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:16.923 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:16.923 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:55:16.924 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T06:55:16.924 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:16.924 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T06:55:16.925 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662188116.801696_424.1', 2: '', 3: ''} 2022-09-03T06:55:16.925 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T06:55:16.926 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:55:16.926 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:55:16.927 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:55:16.979 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:16.979 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:17.046 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:17.044+0000 7ff239732600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:55:17.048 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:17.045+0000 7ff239732600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:55:17.063 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:55:17.064 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:55:17.064 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:55:17.066 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:55:17.066 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:55:17.066 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:55:17.066 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:55:17.067 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.067 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:55:17.067 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:55:17.067 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:55:17.068 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:55:17.068 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:55:17.068 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:55:17.069 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:55:17.069 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:55:17.069 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:55:17.069 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:55:17.069 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:55:17.070 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.070 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:55:17.071 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:55:17.071 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:55:17.071 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:55:17.071 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662188116.801696_424.1", 2022-09-03T06:55:17.072 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:55:17.072 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:55:17.072 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:55:17.072 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:55:16.801696Z", 2022-09-03T06:55:17.073 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:55:17.073 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:55:17.074 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:55:17.074 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:55:17.074 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:55:17.075 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:55:17.075 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:55:17.075 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:55:17.075 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:55:17.076 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:55:17.076 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:55:17.076 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.077 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:55:17.077 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:55:17.077 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:55:17.077 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:55:17.078 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:55:17.078 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:55:17.078 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:55:17.078 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:55:17.079 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:55:17.079 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:55:17.079 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.079 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.080 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:55:17.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.080 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:55:17.080 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:55:17.081 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:55:17.081 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.081 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:55:17Z" 2022-09-03T06:55:17.081 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:55:17.083 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188116.801696_424.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:55:16.801696Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:55:17Z" } 2022-09-03T06:55:17.083 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188116.801696_424.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:55:16.801696Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:55:17.084 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188116.801696_424.1', 2: '', 3: ''} 2022-09-03T06:55:17.084 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188116.801696_424.1', 2: '', 3: ''} 2022-09-03T06:55:17.085 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T06:55:17.085 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T06:55:17.086 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:55:17.087 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:55:17.088 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:55:17.140 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:17.141 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:17.172 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:17.171+0000 7f4b8901f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:55:17.173 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:17.172+0000 7f4b8901f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:55:17.189 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T06:55:17.189 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T06:55:17.189 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T06:55:17.190 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T06:55:17.190 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T06:55:17.190 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:55:17.191 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:55:17.191 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:17.191 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T06:55:17.192 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:17.192 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T06:55:17.192 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:55:17.192 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:55:17.193 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:55:17.193 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:55:17.193 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T06:55:17.195 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:55:17.195 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:55:17.196 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:55:17.196 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:17.197 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:17.197 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:17.198 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T06:55:17.198 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:55:17.198 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:55:17.199 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662188116.801696_424.1", 2022-09-03T06:55:17.199 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:55:17.199 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:55:17.200 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:55:17.200 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T06:55:16.801696Z", 2022-09-03T06:55:17.200 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T06:55:17.201 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:17.202 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:17.202 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:17.202 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T06:55:17.204 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:55:17.204 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:55:17.204 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:55:17.204 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:55:17.205 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:55:17.205 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:55:17.205 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:55:17.205 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:55:17.206 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:17.206 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:17.206 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T06:55:17.207 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T06:55:17.207 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T06:55:17.207 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T06:55:17.207 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T06:55:17.208 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T06:55:17.208 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T06:55:17.208 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T06:55:17.208 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T06:55:17.209 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T06:55:17.209 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:17.209 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T06:55:17.209 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T06:55:17.210 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:17.210 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T06:55:17.211 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T06:55:17.211 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T06:55:17.211 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T06:55:17.211 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T06:55:17Z" 2022-09-03T06:55:17.212 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T06:55:17.213 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188116.801696_424.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:55:16.801696Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:55:17Z" } 2022-09-03T06:55:17.214 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188116.801696_424.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:55:16.801696Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:55:17.215 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188116.801696_424.1', 2: '', 3: ''} 2022-09-03T06:55:17.215 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188116.801696_424.1', 2: '', 3: ''} 2022-09-03T06:55:17.216 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T06:55:17.216 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T06:55:17.216 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:55:17.218 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T06:55:17.218 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T06:55:17.268 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:17.269 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:17.311 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:17.308+0000 7fbcc468f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:55:17.315 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:17.313+0000 7fbcc468f600 20 RGW-SYNC:meta: read sync status 2022-09-03T06:55:17.344 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T06:55:17.345 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T06:55:17.345 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T06:55:17.345 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T06:55:17.345 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T06:55:17.346 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T06:55:17.346 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:55:17.346 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.347 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T06:55:17.348 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:55:17.348 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T06:55:17.349 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:55:17.349 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:55:17.349 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T06:55:17.350 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:55:17.350 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T06:55:17.350 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:55:17.350 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T06:55:17.351 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:55:17.351 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.351 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.351 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:55:17.352 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T06:55:17.352 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:55:17.352 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:55:17.353 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662188116.801696_424.1", 2022-09-03T06:55:17.353 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:55:17.353 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:55:17.353 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:55:17.354 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:55:16.801696Z", 2022-09-03T06:55:17.354 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T06:55:17.354 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.354 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.355 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:55:17.355 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T06:55:17.355 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:55:17.355 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:55:17.356 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:55:17.356 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:55:17.356 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:55:17.356 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:55:17.357 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:55:17.357 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:55:17.358 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.358 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.358 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T06:55:17.359 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T06:55:17.359 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T06:55:17.360 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T06:55:17.360 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T06:55:17.361 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T06:55:17.361 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T06:55:17.361 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T06:55:17.362 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T06:55:17.362 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T06:55:17.362 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.362 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T06:55:17.363 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T06:55:17.363 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.363 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T06:55:17.363 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T06:55:17.364 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T06:55:17.364 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T06:55:17.364 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T06:55:17Z" 2022-09-03T06:55:17.364 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T06:55:17.366 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188116.801696_424.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:55:16.801696Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T06:55:17Z" } 2022-09-03T06:55:17.366 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188116.801696_424.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:55:16.801696Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T06:55:17.367 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188116.801696_424.1', 2: '', 3: ''} 2022-09-03T06:55:17.367 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188116.801696_424.1', 2: '', 3: ''} 2022-09-03T06:55:17.368 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T06:55:17.368 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:55:17.369 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:55:17.369 DEBUG:boto:Method: HEAD 2022-09-03T06:55:17.370 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:55:17.370 DEBUG:boto:Data: 2022-09-03T06:55:17.371 DEBUG:boto:Headers: {} 2022-09-03T06:55:17.371 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:17.372 DEBUG:boto:Port: 8000 2022-09-03T06:55:17.372 DEBUG:boto:Params: {} 2022-09-03T06:55:17.373 DEBUG:boto:Token: None 2022-09-03T06:55:17.374 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:17 GMT /avysmw-72/ 2022-09-03T06:55:17.374 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MYfbe27h6taeDl3+AbOAHRgT9PU= 2022-09-03T06:55:17.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MYfbe27h6taeDl3+AbOAHRgT9PU=', 'Content-Length': '0'} 2022-09-03T06:55:17.377 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000071161a267fc6fede-006312fa55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:17.378 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:55:17.379 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:55:17.379 DEBUG:boto:Method: PUT 2022-09-03T06:55:17.380 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:55:17.380 DEBUG:boto:Data: 2022-09-03T06:55:17.381 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:55:17.381 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:17.382 DEBUG:boto:Port: 8000 2022-09-03T06:55:17.382 DEBUG:boto:Params: {} 2022-09-03T06:55:17.383 DEBUG:boto:Token: None 2022-09-03T06:55:17.383 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:55:17 GMT /avysmw-72/a 2022-09-03T06:55:17.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:blWzzTzSEUNH7x84s//w25nOWkA= 2022-09-03T06:55:17.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:55:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:blWzzTzSEUNH7x84s//w25nOWkA='} 2022-09-03T06:55:17.394 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000adb015ee502221ca-006312fa55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:17.394 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:55:17.395 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:55:17.396 DEBUG:boto:Method: HEAD 2022-09-03T06:55:17.396 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:55:17.397 DEBUG:boto:Data: 2022-09-03T06:55:17.398 DEBUG:boto:Headers: {} 2022-09-03T06:55:17.399 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:17.399 DEBUG:boto:Port: 8000 2022-09-03T06:55:17.400 DEBUG:boto:Params: {} 2022-09-03T06:55:17.401 DEBUG:boto:Token: None 2022-09-03T06:55:17.401 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:17 GMT /avysmw-72/ 2022-09-03T06:55:17.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MYfbe27h6taeDl3+AbOAHRgT9PU= 2022-09-03T06:55:17.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MYfbe27h6taeDl3+AbOAHRgT9PU=', 'Content-Length': '0'} 2022-09-03T06:55:17.405 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f8349fc92cfe1d7a-006312fa55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:17.406 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:55:17.407 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:55:17.408 DEBUG:boto:Method: PUT 2022-09-03T06:55:17.408 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:55:17.409 DEBUG:boto:Data: 2022-09-03T06:55:17.409 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:55:17.410 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:17.411 DEBUG:boto:Port: 8000 2022-09-03T06:55:17.411 DEBUG:boto:Params: {} 2022-09-03T06:55:17.412 DEBUG:boto:Token: None 2022-09-03T06:55:17.412 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:55:17 GMT /avysmw-72/b 2022-09-03T06:55:17.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAzql5b/m70zl7+F/NWd95Cs1hY= 2022-09-03T06:55:17.414 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:55:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAzql5b/m70zl7+F/NWd95Cs1hY='} 2022-09-03T06:55:17.462 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000746bc56fcd312209-006312fa55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:17.463 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:55:17.464 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:55:17.465 DEBUG:boto:Method: HEAD 2022-09-03T06:55:17.465 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:55:17.466 DEBUG:boto:Data: 2022-09-03T06:55:17.466 DEBUG:boto:Headers: {} 2022-09-03T06:55:17.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:17.467 DEBUG:boto:Port: 8000 2022-09-03T06:55:17.468 DEBUG:boto:Params: {} 2022-09-03T06:55:17.469 DEBUG:boto:Token: None 2022-09-03T06:55:17.469 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:17 GMT /avysmw-72/ 2022-09-03T06:55:17.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MYfbe27h6taeDl3+AbOAHRgT9PU= 2022-09-03T06:55:17.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MYfbe27h6taeDl3+AbOAHRgT9PU=', 'Content-Length': '0'} 2022-09-03T06:55:17.474 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e1768d9c56bf7c7e-006312fa55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:17.475 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:55:17.476 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:55:17.477 DEBUG:boto:Method: PUT 2022-09-03T06:55:17.477 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:55:17.478 DEBUG:boto:Data: 2022-09-03T06:55:17.479 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:55:17.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:17.480 DEBUG:boto:Port: 8000 2022-09-03T06:55:17.481 DEBUG:boto:Params: {} 2022-09-03T06:55:17.482 DEBUG:boto:Token: None 2022-09-03T06:55:17.483 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:55:17 GMT /avysmw-72/c 2022-09-03T06:55:17.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oTTi0MSyxGedwWctkDU9tcySYd4= 2022-09-03T06:55:17.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:55:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oTTi0MSyxGedwWctkDU9tcySYd4='} 2022-09-03T06:55:17.490 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f8e5e9ac190c6128-006312fa55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:17.491 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:55:17.492 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:55:17.492 DEBUG:boto:Method: HEAD 2022-09-03T06:55:17.493 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:55:17.493 DEBUG:boto:Data: 2022-09-03T06:55:17.494 DEBUG:boto:Headers: {} 2022-09-03T06:55:17.494 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:17.495 DEBUG:boto:Port: 8000 2022-09-03T06:55:17.495 DEBUG:boto:Params: {} 2022-09-03T06:55:17.496 DEBUG:boto:Token: None 2022-09-03T06:55:17.497 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:55:17 GMT /avysmw-72/ 2022-09-03T06:55:17.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MYfbe27h6taeDl3+AbOAHRgT9PU= 2022-09-03T06:55:17.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:55:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MYfbe27h6taeDl3+AbOAHRgT9PU=', 'Content-Length': '0'} 2022-09-03T06:55:17.500 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '9'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003d7ce3892e89e49d-006312fa55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:17.501 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:55:17.502 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:55:17.502 DEBUG:boto:Method: PUT 2022-09-03T06:55:17.503 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:55:17.504 DEBUG:boto:Data: 2022-09-03T06:55:17.505 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:55:17.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:55:17.506 DEBUG:boto:Port: 8000 2022-09-03T06:55:17.506 DEBUG:boto:Params: {} 2022-09-03T06:55:17.507 DEBUG:boto:Token: None 2022-09-03T06:55:17.508 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:55:17 GMT /avysmw-72/d 2022-09-03T06:55:17.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IQhREmgdAmdRitDoggPROQmXHrY= 2022-09-03T06:55:17.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:55:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IQhREmgdAmdRitDoggPROQmXHrY='} 2022-09-03T06:55:17.554 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000537dffcb1c522dea-006312fa55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:55:17.555 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-72 2022-09-03T06:55:17.556 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:17.556 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:17.557 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:17.623 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:17.624 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:17.683 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:17.680+0000 7f3eec644600 1 waiting to reach incremental sync.. 2022-09-03T06:55:22.683 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:22.681+0000 7f3eec644600 1 waiting to reach incremental sync.. 2022-09-03T06:55:27.685 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:27.683+0000 7f3eec644600 1 bucket sync caught up with source: 2022-09-03T06:55:27.686 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.35.5, , , , 00000000001.55.5, , , 00000000001.64.5, , 00000000001.37.5] 2022-09-03T06:55:27.686 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.35.5, , , , 00000000001.55.5, , , 00000000001.64.5, , 00000000001.37.5] 2022-09-03T06:55:27.686 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:27.683+0000 7f3eec644600 0 bucket checkpoint complete 2022-09-03T06:55:27.702 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-72 2022-09-03T06:55:27.703 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:27.703 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:27.705 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:27.782 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:27.782 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:27.826 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:27.824+0000 7fcd3707d600 1 bucket sync caught up with source: 2022-09-03T06:55:27.826 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.35.5, , , , 00000000001.55.5, , , 00000000001.64.5, , 00000000001.37.5] 2022-09-03T06:55:27.827 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.35.5, , , , 00000000001.55.5, , , 00000000001.64.5, , 00000000001.37.5] 2022-09-03T06:55:27.827 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:27.824+0000 7fcd3707d600 0 bucket checkpoint complete 2022-09-03T06:55:27.845 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-72 2022-09-03T06:55:27.846 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:27.846 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:27.847 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:27.910 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:27.911 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:27.975 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:27.973+0000 7fdff4be1600 1 bucket sync caught up with source: 2022-09-03T06:55:27.976 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.35.5, , , , 00000000001.55.5, , , 00000000001.64.5, , 00000000001.37.5] 2022-09-03T06:55:27.976 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.35.5, , , , 00000000001.55.5, , , 00000000001.64.5, , 00000000001.37.5] 2022-09-03T06:55:27.977 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:55:27.973+0000 7fdff4be1600 0 bucket checkpoint complete 2022-09-03T06:55:28.015 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-72 2022-09-03T06:55:28.016 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:28.017 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:28.018 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:28.081 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:28.081 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:28.119 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:28.118+0000 7f6e4ae0c600 1 waiting to reach incremental sync.. 2022-09-03T06:55:33.120 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:33.118+0000 7f6e4ae0c600 1 waiting to reach incremental sync.. 2022-09-03T06:55:38.121 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:38.118+0000 7f6e4ae0c600 1 waiting to reach incremental sync.. 2022-09-03T06:55:43.121 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:43.118+0000 7f6e4ae0c600 1 waiting to reach incremental sync.. 2022-09-03T06:55:48.123 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:48.120+0000 7f6e4ae0c600 1 bucket sync caught up with source: 2022-09-03T06:55:48.123 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.152.5, , , , 00000000002.262.5, , , 00000000001.295.5, , 00000000001.150.5] 2022-09-03T06:55:48.124 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.152.5, , , , 00000000002.262.5, , , 00000000001.295.5, , 00000000001.150.5] 2022-09-03T06:55:48.124 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:48.120+0000 7f6e4ae0c600 0 bucket checkpoint complete 2022-09-03T06:55:48.145 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-72 2022-09-03T06:55:48.146 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:48.148 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:48.150 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:48.217 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:48.218 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:48.274 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:48.270+0000 7f57ebf4a600 1 bucket sync caught up with source: 2022-09-03T06:55:48.275 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.152.5, , , , 00000000002.262.5, , , 00000000001.295.5, , 00000000001.150.5] 2022-09-03T06:55:48.275 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.152.5, , , , 00000000002.262.5, , , 00000000001.295.5, , 00000000001.150.5] 2022-09-03T06:55:48.275 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:48.270+0000 7f57ebf4a600 0 bucket checkpoint complete 2022-09-03T06:55:48.293 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-72 2022-09-03T06:55:48.293 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-72 2022-09-03T06:55:48.294 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:48.295 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:55:48.295 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:55:48.355 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:55:48.355 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:55:48.394 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:48.391+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:55:53.395 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:53.393+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:55:58.395 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:55:58.393+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:03.395 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:03.394+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:08.398 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:08.395+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:13.397 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:13.395+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:18.397 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:18.396+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:23.398 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:23.396+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:28.399 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:28.397+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:33.400 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:33.397+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:38.400 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:38.398+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:43.401 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:43.398+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:48.402 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:48.399+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:53.402 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:53.400+0000 7fba9014e600 1 waiting to reach incremental sync.. 2022-09-03T06:56:58.404 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:58.402+0000 7fba9014e600 1 bucket sync caught up with source: 2022-09-03T06:56:58.405 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.79.5, , , , 00000000001.193.5, , , 00000000001.267.5, , 00000000001.77.5] 2022-09-03T06:56:58.405 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.79.5, , , , 00000000001.193.5, , , 00000000001.267.5, , 00000000001.77.5] 2022-09-03T06:56:58.406 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:56:58.402+0000 7fba9014e600 0 bucket checkpoint complete 2022-09-03T06:56:58.430 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-72 2022-09-03T06:56:58.431 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:56:58.432 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:56:58.434 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:56:58.498 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:56:58.499 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:56:58.535 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:56:58.533+0000 7fa40aabb600 1 bucket sync caught up with source: 2022-09-03T06:56:58.536 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.79.5, , , , 00000000001.193.5, , , 00000000001.267.5, , 00000000001.77.5] 2022-09-03T06:56:58.536 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.79.5, , , , 00000000001.193.5, , , 00000000001.267.5, , 00000000001.77.5] 2022-09-03T06:56:58.537 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:56:58.533+0000 7fa40aabb600 0 bucket checkpoint complete 2022-09-03T06:56:58.552 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-72 2022-09-03T06:56:58.552 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-72 zones={test-zone2, test-zone1} 2022-09-03T06:56:58.553 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.564 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.565 DEBUG:boto:Method: HEAD 2022-09-03T06:56:58.566 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:56:58.566 DEBUG:boto:Data: 2022-09-03T06:56:58.567 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.567 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.568 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.568 DEBUG:boto:Params: {} 2022-09-03T06:56:58.570 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:56:58.570 DEBUG:boto:Token: None 2022-09-03T06:56:58.571 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/ 2022-09-03T06:56:58.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SZNb7i4JwY4vh6g7ievaMu1OCUw= 2022-09-03T06:56:58.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SZNb7i4JwY4vh6g7ievaMu1OCUw=', 'Content-Length': '0'} 2022-09-03T06:56:58.580 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000093e04c9e060d62e5-006312faba-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.581 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.582 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.582 DEBUG:boto:Method: HEAD 2022-09-03T06:56:58.583 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:56:58.583 DEBUG:boto:Data: 2022-09-03T06:56:58.584 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.584 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.585 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.585 DEBUG:boto:Params: {} 2022-09-03T06:56:58.586 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T06:56:58.587 DEBUG:boto:Token: None 2022-09-03T06:56:58.587 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/ 2022-09-03T06:56:58.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SZNb7i4JwY4vh6g7ievaMu1OCUw= 2022-09-03T06:56:58.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SZNb7i4JwY4vh6g7ievaMu1OCUw=', 'Content-Length': '0'} 2022-09-03T06:56:58.592 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007b5e559def166075-006312faba-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.593 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:56:58.594 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.594 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.595 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.595 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.596 DEBUG:boto:Method: GET 2022-09-03T06:56:58.596 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.597 DEBUG:boto:Data: 2022-09-03T06:56:58.597 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.598 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.599 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.599 DEBUG:boto:Params: {} 2022-09-03T06:56:58.600 DEBUG:boto:Token: None 2022-09-03T06:56:58.600 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/?versions 2022-09-03T06:56:58.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZZK4L3/7/CZHI4uj9+otzhJzsQw= 2022-09-03T06:56:58.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZZK4L3/7/CZHI4uj9+otzhJzsQw=', 'Content-Length': '0'} 2022-09-03T06:56:58.604 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d5872847e2cb09d-006312faba-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.605 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:58.607 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:56:58.608 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:56:58.608 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:56:58.609 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:56:58.609 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:56:58.610 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.611 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.611 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.612 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.612 DEBUG:boto:Method: GET 2022-09-03T06:56:58.613 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.613 DEBUG:boto:Data: 2022-09-03T06:56:58.614 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.615 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.616 DEBUG:boto:Params: {} 2022-09-03T06:56:58.616 DEBUG:boto:Token: None 2022-09-03T06:56:58.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/?versions 2022-09-03T06:56:58.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZZK4L3/7/CZHI4uj9+otzhJzsQw= 2022-09-03T06:56:58.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZZK4L3/7/CZHI4uj9+otzhJzsQw=', 'Content-Length': '0'} 2022-09-03T06:56:58.621 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000746b4e629022cfa1-006312faba-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.622 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:58.623 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:56:58.624 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:56:58.624 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:56:58.625 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:56:58.625 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.626 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.626 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.627 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.627 DEBUG:boto:Method: GET 2022-09-03T06:56:58.628 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.629 DEBUG:boto:Data: 2022-09-03T06:56:58.629 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.630 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.630 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.631 DEBUG:boto:Params: {} 2022-09-03T06:56:58.632 DEBUG:boto:Token: None 2022-09-03T06:56:58.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/?versions 2022-09-03T06:56:58.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZZK4L3/7/CZHI4uj9+otzhJzsQw= 2022-09-03T06:56:58.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZZK4L3/7/CZHI4uj9+otzhJzsQw=', 'Content-Length': '0'} 2022-09-03T06:56:58.636 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4f035c598f651f6-006312faba-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.636 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:58.637 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.638 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.639 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.639 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.640 DEBUG:boto:Method: GET 2022-09-03T06:56:58.640 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:56:58.641 DEBUG:boto:Data: 2022-09-03T06:56:58.641 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.643 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.643 DEBUG:boto:Params: {} 2022-09-03T06:56:58.644 DEBUG:boto:Token: None 2022-09-03T06:56:58.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/?versions 2022-09-03T06:56:58.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZZK4L3/7/CZHI4uj9+otzhJzsQw= 2022-09-03T06:56:58.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZZK4L3/7/CZHI4uj9+otzhJzsQw=', 'Content-Length': '0'} 2022-09-03T06:56:58.648 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f25ec930105d829e-006312faba-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.649 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:58.650 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:56:58.651 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.651 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.652 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.653 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.653 DEBUG:boto:Method: GET 2022-09-03T06:56:58.654 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:58.654 DEBUG:boto:Data: 2022-09-03T06:56:58.656 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.656 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.657 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.658 DEBUG:boto:Params: {} 2022-09-03T06:56:58.658 DEBUG:boto:Token: None 2022-09-03T06:56:58.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:58.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2pDkiToAV+ijXxBRxHuOlZsdzU0= 2022-09-03T06:56:58.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2pDkiToAV+ijXxBRxHuOlZsdzU0=', 'Content-Length': '0'} 2022-09-03T06:56:58.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f82beb4f361f842d-006312faba-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.663 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.664 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.665 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.665 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.666 DEBUG:boto:Method: GET 2022-09-03T06:56:58.666 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:58.667 DEBUG:boto:Data: 2022-09-03T06:56:58.668 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.668 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.669 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.669 DEBUG:boto:Params: {} 2022-09-03T06:56:58.670 DEBUG:boto:Token: None 2022-09-03T06:56:58.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:58.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2pDkiToAV+ijXxBRxHuOlZsdzU0= 2022-09-03T06:56:58.672 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2pDkiToAV+ijXxBRxHuOlZsdzU0=', 'Content-Length': '0'} 2022-09-03T06:56:58.678 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000615e9ab10e82f648-006312faba-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.679 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.680 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.681 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.682 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.682 DEBUG:boto:Method: HEAD 2022-09-03T06:56:58.683 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:58.684 DEBUG:boto:Data: 2022-09-03T06:56:58.685 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.686 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.686 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.687 DEBUG:boto:Params: {} 2022-09-03T06:56:58.688 DEBUG:boto:Token: None 2022-09-03T06:56:58.688 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:58.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ccpAfLaetn76+BbRDYLcOzG4TC8= 2022-09-03T06:56:58.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ccpAfLaetn76+BbRDYLcOzG4TC8=', 'Content-Length': '0'} 2022-09-03T06:56:58.692 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000067df093b28338ed7-006312faba-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.693 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.694 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.694 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.695 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.695 DEBUG:boto:Method: HEAD 2022-09-03T06:56:58.696 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:58.697 DEBUG:boto:Data: 2022-09-03T06:56:58.697 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.698 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.699 DEBUG:boto:Params: {} 2022-09-03T06:56:58.699 DEBUG:boto:Token: None 2022-09-03T06:56:58.700 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:58.701 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ccpAfLaetn76+BbRDYLcOzG4TC8= 2022-09-03T06:56:58.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ccpAfLaetn76+BbRDYLcOzG4TC8=', 'Content-Length': '0'} 2022-09-03T06:56:58.704 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000219eab26697290c3-006312faba-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.705 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:56:58.705 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.706 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.707 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.707 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.708 DEBUG:boto:Method: GET 2022-09-03T06:56:58.708 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:58.709 DEBUG:boto:Data: 2022-09-03T06:56:58.709 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.710 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.710 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.711 DEBUG:boto:Params: {} 2022-09-03T06:56:58.712 DEBUG:boto:Token: None 2022-09-03T06:56:58.712 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:58.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2pDkiToAV+ijXxBRxHuOlZsdzU0= 2022-09-03T06:56:58.714 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2pDkiToAV+ijXxBRxHuOlZsdzU0=', 'Content-Length': '0'} 2022-09-03T06:56:58.716 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e59e98eb881c743d-006312faba-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.716 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.717 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.718 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.718 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:58.719 DEBUG:boto:Method: GET 2022-09-03T06:56:58.719 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:58.720 DEBUG:boto:Data: 2022-09-03T06:56:58.721 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.721 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.722 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.722 DEBUG:boto:Params: {} 2022-09-03T06:56:58.723 DEBUG:boto:Token: None 2022-09-03T06:56:58.724 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:58.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2pDkiToAV+ijXxBRxHuOlZsdzU0= 2022-09-03T06:56:58.725 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2pDkiToAV+ijXxBRxHuOlZsdzU0=', 'Content-Length': '0'} 2022-09-03T06:56:58.729 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012510fb7c67a07a0-006312faba-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.731 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.732 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.733 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:56:58.734 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:56:58.736 DEBUG:boto:Method: GET 2022-09-03T06:56:58.737 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:56:58.738 DEBUG:boto:Data: 2022-09-03T06:56:58.738 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.739 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.739 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.740 DEBUG:boto:Params: {} 2022-09-03T06:56:58.741 DEBUG:boto:Token: None 2022-09-03T06:56:58.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/ 2022-09-03T06:56:58.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GbFKxVsgZ00NcARZ2ZS9wNEncrY= 2022-09-03T06:56:58.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GbFKxVsgZ00NcARZ2ZS9wNEncrY=', 'Content-Length': '0'} 2022-09-03T06:56:58.745 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b9cd958668f3d827-006312faba-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.746 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:58.747 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.748 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.748 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:56:58.749 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:56:58.750 DEBUG:boto:Method: GET 2022-09-03T06:56:58.750 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:56:58.751 DEBUG:boto:Data: 2022-09-03T06:56:58.751 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.752 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.753 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.753 DEBUG:boto:Params: {} 2022-09-03T06:56:58.754 DEBUG:boto:Token: None 2022-09-03T06:56:58.754 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/ 2022-09-03T06:56:58.755 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GbFKxVsgZ00NcARZ2ZS9wNEncrY= 2022-09-03T06:56:58.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GbFKxVsgZ00NcARZ2ZS9wNEncrY=', 'Content-Length': '0'} 2022-09-03T06:56:58.760 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c55ff71da934e1f8-006312faba-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.761 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:58.762 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.763 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.764 DEBUG:boto:Method: HEAD 2022-09-03T06:56:58.765 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:56:58.765 DEBUG:boto:Data: 2022-09-03T06:56:58.766 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.767 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.767 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.768 DEBUG:boto:Params: {} 2022-09-03T06:56:58.769 DEBUG:boto:Token: None 2022-09-03T06:56:58.770 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a 2022-09-03T06:56:58.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z8zgR+ji3ieu4QXyDUvB+oUZIZg= 2022-09-03T06:56:58.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z8zgR+ji3ieu4QXyDUvB+oUZIZg=', 'Content-Length': '0'} 2022-09-03T06:56:58.773 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e187716f803f4a3-006312faba-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.774 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.774 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.775 DEBUG:boto:Method: HEAD 2022-09-03T06:56:58.776 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:56:58.776 DEBUG:boto:Data: 2022-09-03T06:56:58.777 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.778 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.778 DEBUG:boto:Params: {} 2022-09-03T06:56:58.779 DEBUG:boto:Token: None 2022-09-03T06:56:58.780 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a 2022-09-03T06:56:58.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z8zgR+ji3ieu4QXyDUvB+oUZIZg= 2022-09-03T06:56:58.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z8zgR+ji3ieu4QXyDUvB+oUZIZg=', 'Content-Length': '0'} 2022-09-03T06:56:58.783 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e0e0568b4d25ddfa-006312faba-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.784 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:56:58.785 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.785 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.787 DEBUG:boto:Method: GET 2022-09-03T06:56:58.787 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:56:58.788 DEBUG:boto:Data: 2022-09-03T06:56:58.789 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.790 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.791 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.792 DEBUG:boto:Params: {} 2022-09-03T06:56:58.793 DEBUG:boto:Token: None 2022-09-03T06:56:58.794 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a 2022-09-03T06:56:58.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w9kPk8CEHTW6k92lcJeh49DGmOs= 2022-09-03T06:56:58.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w9kPk8CEHTW6k92lcJeh49DGmOs=', 'Content-Length': '0'} 2022-09-03T06:56:58.800 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9a4cbcdddf11557-006312faba-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.802 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:58.804 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:58.805 DEBUG:boto:Method: GET 2022-09-03T06:56:58.806 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:56:58.807 DEBUG:boto:Data: 2022-09-03T06:56:58.808 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.810 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.811 DEBUG:boto:Params: {} 2022-09-03T06:56:58.812 DEBUG:boto:Token: None 2022-09-03T06:56:58.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/a 2022-09-03T06:56:58.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w9kPk8CEHTW6k92lcJeh49DGmOs= 2022-09-03T06:56:58.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w9kPk8CEHTW6k92lcJeh49DGmOs=', 'Content-Length': '0'} 2022-09-03T06:56:58.817 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000012f3d518408b0ec-006312faba-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.818 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.819 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.820 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:56:58.820 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:56:58.821 DEBUG:boto:Method: GET 2022-09-03T06:56:58.822 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:56:58.823 DEBUG:boto:Data: 2022-09-03T06:56:58.824 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.825 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.826 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.827 DEBUG:boto:Params: {} 2022-09-03T06:56:58.829 DEBUG:boto:Token: None 2022-09-03T06:56:58.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/ 2022-09-03T06:56:58.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GbFKxVsgZ00NcARZ2ZS9wNEncrY= 2022-09-03T06:56:58.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GbFKxVsgZ00NcARZ2ZS9wNEncrY=', 'Content-Length': '0'} 2022-09-03T06:56:58.838 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085375e07d72158da-006312faba-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.839 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:58.840 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:58.840 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:58.841 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:56:58.841 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:56:58.842 DEBUG:boto:Method: GET 2022-09-03T06:56:58.843 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:56:58.843 DEBUG:boto:Data: 2022-09-03T06:56:58.844 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.845 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.846 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.847 DEBUG:boto:Params: {} 2022-09-03T06:56:58.847 DEBUG:boto:Token: None 2022-09-03T06:56:58.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/ 2022-09-03T06:56:58.848 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GbFKxVsgZ00NcARZ2ZS9wNEncrY= 2022-09-03T06:56:58.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GbFKxVsgZ00NcARZ2ZS9wNEncrY=', 'Content-Length': '0'} 2022-09-03T06:56:58.852 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad637a27864644f5-006312faba-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.853 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:58.854 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:56:58.855 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:58.855 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:58.856 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.857 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.858 DEBUG:boto:Method: GET 2022-09-03T06:56:58.858 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:56:58.859 DEBUG:boto:Data: 2022-09-03T06:56:58.859 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.860 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.860 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.861 DEBUG:boto:Params: {} 2022-09-03T06:56:58.862 DEBUG:boto:Token: None 2022-09-03T06:56:58.862 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/b?versionId=null 2022-09-03T06:56:58.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TSAn/FtQuviZ5BTBjgJGd9xQSwA= 2022-09-03T06:56:58.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TSAn/FtQuviZ5BTBjgJGd9xQSwA=', 'Content-Length': '0'} 2022-09-03T06:56:58.866 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091586cf1435172f0-006312faba-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.906 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:58.907 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:58.908 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.908 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.909 DEBUG:boto:Method: GET 2022-09-03T06:56:58.910 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:56:58.911 DEBUG:boto:Data: 2022-09-03T06:56:58.912 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.912 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.913 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.914 DEBUG:boto:Params: {} 2022-09-03T06:56:58.915 DEBUG:boto:Token: None 2022-09-03T06:56:58.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/b?versionId=null 2022-09-03T06:56:58.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TSAn/FtQuviZ5BTBjgJGd9xQSwA= 2022-09-03T06:56:58.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TSAn/FtQuviZ5BTBjgJGd9xQSwA=', 'Content-Length': '0'} 2022-09-03T06:56:58.925 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b40c4ec257f6679-006312faba-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.926 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:58.927 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:58.927 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.928 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.929 DEBUG:boto:Method: HEAD 2022-09-03T06:56:58.929 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:56:58.930 DEBUG:boto:Data: 2022-09-03T06:56:58.930 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.931 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.931 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.932 DEBUG:boto:Params: {} 2022-09-03T06:56:58.933 DEBUG:boto:Token: None 2022-09-03T06:56:58.933 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/b?versionId=null 2022-09-03T06:56:58.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zl+FtiISIMeQ9D5IsIX3tNjOYRg= 2022-09-03T06:56:58.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zl+FtiISIMeQ9D5IsIX3tNjOYRg=', 'Content-Length': '0'} 2022-09-03T06:56:58.938 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070c2a1a9b0fe3366-006312faba-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.939 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:58.940 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:58.940 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.941 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.941 DEBUG:boto:Method: HEAD 2022-09-03T06:56:58.942 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:56:58.943 DEBUG:boto:Data: 2022-09-03T06:56:58.944 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.946 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.947 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.948 DEBUG:boto:Params: {} 2022-09-03T06:56:58.949 DEBUG:boto:Token: None 2022-09-03T06:56:58.950 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/b?versionId=null 2022-09-03T06:56:58.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zl+FtiISIMeQ9D5IsIX3tNjOYRg= 2022-09-03T06:56:58.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zl+FtiISIMeQ9D5IsIX3tNjOYRg=', 'Content-Length': '0'} 2022-09-03T06:56:58.955 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000295c432f2b0d264e-006312faba-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.958 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:56:58.959 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:58.961 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:58.962 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.963 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.964 DEBUG:boto:Method: GET 2022-09-03T06:56:58.966 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:56:58.969 DEBUG:boto:Data: 2022-09-03T06:56:58.970 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.972 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:58.973 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.975 DEBUG:boto:Params: {} 2022-09-03T06:56:58.976 DEBUG:boto:Token: None 2022-09-03T06:56:58.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/b?versionId=null 2022-09-03T06:56:58.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TSAn/FtQuviZ5BTBjgJGd9xQSwA= 2022-09-03T06:56:58.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TSAn/FtQuviZ5BTBjgJGd9xQSwA=', 'Content-Length': '0'} 2022-09-03T06:56:58.984 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c1d172ec346818b-006312faba-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:58.985 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:58.986 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:58.987 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.987 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:56:58.988 DEBUG:boto:Method: GET 2022-09-03T06:56:58.989 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:56:58.990 DEBUG:boto:Data: 2022-09-03T06:56:58.990 DEBUG:boto:Headers: {} 2022-09-03T06:56:58.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:58.992 DEBUG:boto:Port: 8000 2022-09-03T06:56:58.992 DEBUG:boto:Params: {} 2022-09-03T06:56:58.993 DEBUG:boto:Token: None 2022-09-03T06:56:58.994 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:58 GMT /avysmw-72/b?versionId=null 2022-09-03T06:56:58.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TSAn/FtQuviZ5BTBjgJGd9xQSwA= 2022-09-03T06:56:58.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TSAn/FtQuviZ5BTBjgJGd9xQSwA=', 'Content-Length': '0'} 2022-09-03T06:56:58.998 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080d22f5acaff4f56-006312faba-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.042 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.043 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.044 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:56:59.045 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:56:59.046 DEBUG:boto:Method: GET 2022-09-03T06:56:59.048 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:56:59.049 DEBUG:boto:Data: 2022-09-03T06:56:59.050 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.051 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.052 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.052 DEBUG:boto:Params: {} 2022-09-03T06:56:59.054 DEBUG:boto:Token: None 2022-09-03T06:56:59.055 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.057 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.091 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed4df3a7fc3b2c1c-006312fabb-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.092 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.093 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.095 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.096 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:56:59.097 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:56:59.097 DEBUG:boto:Method: GET 2022-09-03T06:56:59.098 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:56:59.099 DEBUG:boto:Data: 2022-09-03T06:56:59.099 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.100 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.101 DEBUG:boto:Params: {} 2022-09-03T06:56:59.102 DEBUG:boto:Token: None 2022-09-03T06:56:59.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.108 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e2b3fceca277dc89-006312fabb-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.109 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.109 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:59.110 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:59.111 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.111 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:56:59.112 DEBUG:boto:Data: 2022-09-03T06:56:59.112 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.113 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.113 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.114 DEBUG:boto:Params: {} 2022-09-03T06:56:59.114 DEBUG:boto:Token: None 2022-09-03T06:56:59.116 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/b 2022-09-03T06:56:59.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XosRHvprcaX88Mp3DbTEMz3EfrE= 2022-09-03T06:56:59.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XosRHvprcaX88Mp3DbTEMz3EfrE=', 'Content-Length': '0'} 2022-09-03T06:56:59.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb84ec1f394433bc-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.120 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:59.121 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:59.121 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.122 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:56:59.122 DEBUG:boto:Data: 2022-09-03T06:56:59.123 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.124 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.125 DEBUG:boto:Params: {} 2022-09-03T06:56:59.125 DEBUG:boto:Token: None 2022-09-03T06:56:59.126 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/b 2022-09-03T06:56:59.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XosRHvprcaX88Mp3DbTEMz3EfrE= 2022-09-03T06:56:59.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XosRHvprcaX88Mp3DbTEMz3EfrE=', 'Content-Length': '0'} 2022-09-03T06:56:59.130 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000520732d36518465f-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.131 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:56:59.132 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:59.133 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:59.133 DEBUG:boto:Method: GET 2022-09-03T06:56:59.134 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:56:59.135 DEBUG:boto:Data: 2022-09-03T06:56:59.135 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.136 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.137 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.137 DEBUG:boto:Params: {} 2022-09-03T06:56:59.139 DEBUG:boto:Token: None 2022-09-03T06:56:59.140 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/b 2022-09-03T06:56:59.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1SxK0I7jeeu/pd4QoUPCSPte2RE= 2022-09-03T06:56:59.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1SxK0I7jeeu/pd4QoUPCSPte2RE=', 'Content-Length': '0'} 2022-09-03T06:56:59.144 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000fe050ade020ae3b-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.186 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:56:59.187 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:56:59.187 DEBUG:boto:Method: GET 2022-09-03T06:56:59.188 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:56:59.189 DEBUG:boto:Data: 2022-09-03T06:56:59.190 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.190 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.191 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.191 DEBUG:boto:Params: {} 2022-09-03T06:56:59.192 DEBUG:boto:Token: None 2022-09-03T06:56:59.193 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/b 2022-09-03T06:56:59.194 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1SxK0I7jeeu/pd4QoUPCSPte2RE= 2022-09-03T06:56:59.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1SxK0I7jeeu/pd4QoUPCSPte2RE=', 'Content-Length': '0'} 2022-09-03T06:56:59.197 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008cf2038895c63b12-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.198 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.198 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.199 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:56:59.200 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:56:59.200 DEBUG:boto:Method: GET 2022-09-03T06:56:59.201 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:56:59.202 DEBUG:boto:Data: 2022-09-03T06:56:59.202 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.203 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.204 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.204 DEBUG:boto:Params: {} 2022-09-03T06:56:59.205 DEBUG:boto:Token: None 2022-09-03T06:56:59.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.210 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c3a214eafd91a616-006312fabb-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.211 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.213 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.214 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.215 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:56:59.216 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:56:59.217 DEBUG:boto:Method: GET 2022-09-03T06:56:59.218 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:56:59.219 DEBUG:boto:Data: 2022-09-03T06:56:59.220 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.222 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.223 DEBUG:boto:Params: {} 2022-09-03T06:56:59.224 DEBUG:boto:Token: None 2022-09-03T06:56:59.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.227 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.233 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1261b3254f1c90f-006312fabb-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.237 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.238 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:56:59.239 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.240 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.241 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.241 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.242 DEBUG:boto:Method: GET 2022-09-03T06:56:59.243 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:56:59.244 DEBUG:boto:Data: 2022-09-03T06:56:59.244 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.245 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.245 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.246 DEBUG:boto:Params: {} 2022-09-03T06:56:59.247 DEBUG:boto:Token: None 2022-09-03T06:56:59.248 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c?versionId=null 2022-09-03T06:56:59.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ghjMsxP+dQWNNdnYoePE+RfzZjg= 2022-09-03T06:56:59.250 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ghjMsxP+dQWNNdnYoePE+RfzZjg=', 'Content-Length': '0'} 2022-09-03T06:56:59.253 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064f41f0c1a75cd43-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.254 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.255 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.256 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.257 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.258 DEBUG:boto:Method: GET 2022-09-03T06:56:59.259 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:56:59.260 DEBUG:boto:Data: 2022-09-03T06:56:59.261 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.264 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.264 DEBUG:boto:Params: {} 2022-09-03T06:56:59.266 DEBUG:boto:Token: None 2022-09-03T06:56:59.267 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c?versionId=null 2022-09-03T06:56:59.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ghjMsxP+dQWNNdnYoePE+RfzZjg= 2022-09-03T06:56:59.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ghjMsxP+dQWNNdnYoePE+RfzZjg=', 'Content-Length': '0'} 2022-09-03T06:56:59.277 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c302af4cbea3592-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.278 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.279 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.279 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.280 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.281 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.282 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:56:59.283 DEBUG:boto:Data: 2022-09-03T06:56:59.284 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.285 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.286 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.287 DEBUG:boto:Params: {} 2022-09-03T06:56:59.288 DEBUG:boto:Token: None 2022-09-03T06:56:59.288 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c?versionId=null 2022-09-03T06:56:59.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gRZU2RnMnocd4JVQ9SpTRF+KGnc= 2022-09-03T06:56:59.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gRZU2RnMnocd4JVQ9SpTRF+KGnc=', 'Content-Length': '0'} 2022-09-03T06:56:59.292 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000525b600e90928ac3-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.293 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.294 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.294 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.295 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.296 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.296 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:56:59.297 DEBUG:boto:Data: 2022-09-03T06:56:59.297 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.298 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.299 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.299 DEBUG:boto:Params: {} 2022-09-03T06:56:59.300 DEBUG:boto:Token: None 2022-09-03T06:56:59.300 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c?versionId=null 2022-09-03T06:56:59.301 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gRZU2RnMnocd4JVQ9SpTRF+KGnc= 2022-09-03T06:56:59.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gRZU2RnMnocd4JVQ9SpTRF+KGnc=', 'Content-Length': '0'} 2022-09-03T06:56:59.304 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000539cbe92e0d5abda-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.305 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:56:59.306 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.307 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.308 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.309 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.310 DEBUG:boto:Method: GET 2022-09-03T06:56:59.311 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:56:59.312 DEBUG:boto:Data: 2022-09-03T06:56:59.313 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.314 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.315 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.316 DEBUG:boto:Params: {} 2022-09-03T06:56:59.317 DEBUG:boto:Token: None 2022-09-03T06:56:59.319 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c?versionId=null 2022-09-03T06:56:59.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ghjMsxP+dQWNNdnYoePE+RfzZjg= 2022-09-03T06:56:59.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ghjMsxP+dQWNNdnYoePE+RfzZjg=', 'Content-Length': '0'} 2022-09-03T06:56:59.322 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d4d77239c5d42b5-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.323 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.324 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.325 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.326 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:56:59.327 DEBUG:boto:Method: GET 2022-09-03T06:56:59.327 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:56:59.328 DEBUG:boto:Data: 2022-09-03T06:56:59.329 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.332 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.333 DEBUG:boto:Params: {} 2022-09-03T06:56:59.334 DEBUG:boto:Token: None 2022-09-03T06:56:59.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c?versionId=null 2022-09-03T06:56:59.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ghjMsxP+dQWNNdnYoePE+RfzZjg= 2022-09-03T06:56:59.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ghjMsxP+dQWNNdnYoePE+RfzZjg=', 'Content-Length': '0'} 2022-09-03T06:56:59.341 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001798e4da8f4e5083-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.382 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.383 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.385 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:56:59.385 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:56:59.386 DEBUG:boto:Method: GET 2022-09-03T06:56:59.387 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:56:59.387 DEBUG:boto:Data: 2022-09-03T06:56:59.388 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.389 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.389 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.390 DEBUG:boto:Params: {} 2022-09-03T06:56:59.391 DEBUG:boto:Token: None 2022-09-03T06:56:59.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.395 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a08d32302981c3bf-006312fabb-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.396 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.397 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.398 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.398 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:56:59.399 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:56:59.399 DEBUG:boto:Method: GET 2022-09-03T06:56:59.400 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:56:59.400 DEBUG:boto:Data: 2022-09-03T06:56:59.401 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.402 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.403 DEBUG:boto:Params: {} 2022-09-03T06:56:59.403 DEBUG:boto:Token: None 2022-09-03T06:56:59.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.408 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000788fba5165e553f7-006312fabb-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.409 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.410 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.410 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.411 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.411 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:56:59.412 DEBUG:boto:Data: 2022-09-03T06:56:59.413 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.414 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.415 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.416 DEBUG:boto:Params: {} 2022-09-03T06:56:59.417 DEBUG:boto:Token: None 2022-09-03T06:56:59.418 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c 2022-09-03T06:56:59.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cGWRwtj6MRKv+JpfMHwfYRUlWOU= 2022-09-03T06:56:59.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cGWRwtj6MRKv+JpfMHwfYRUlWOU=', 'Content-Length': '0'} 2022-09-03T06:56:59.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb63ccdb1cb10be3-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.423 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.424 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.425 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.426 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:56:59.426 DEBUG:boto:Data: 2022-09-03T06:56:59.427 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.428 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.429 DEBUG:boto:Params: {} 2022-09-03T06:56:59.430 DEBUG:boto:Token: None 2022-09-03T06:56:59.431 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c 2022-09-03T06:56:59.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cGWRwtj6MRKv+JpfMHwfYRUlWOU= 2022-09-03T06:56:59.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cGWRwtj6MRKv+JpfMHwfYRUlWOU=', 'Content-Length': '0'} 2022-09-03T06:56:59.436 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000095fdc2ebb61b819d-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.436 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:56:59.437 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.438 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.438 DEBUG:boto:Method: GET 2022-09-03T06:56:59.439 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:56:59.439 DEBUG:boto:Data: 2022-09-03T06:56:59.440 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.440 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.441 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.442 DEBUG:boto:Params: {} 2022-09-03T06:56:59.442 DEBUG:boto:Token: None 2022-09-03T06:56:59.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c 2022-09-03T06:56:59.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ifu1CIjVDsneah7pEz2OPuLSqW8= 2022-09-03T06:56:59.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ifu1CIjVDsneah7pEz2OPuLSqW8=', 'Content-Length': '0'} 2022-09-03T06:56:59.447 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018fe51645d2a4fd6-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.447 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:56:59.448 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:56:59.449 DEBUG:boto:Method: GET 2022-09-03T06:56:59.449 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:56:59.450 DEBUG:boto:Data: 2022-09-03T06:56:59.450 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.451 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.452 DEBUG:boto:Params: {} 2022-09-03T06:56:59.452 DEBUG:boto:Token: None 2022-09-03T06:56:59.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/c 2022-09-03T06:56:59.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ifu1CIjVDsneah7pEz2OPuLSqW8= 2022-09-03T06:56:59.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ifu1CIjVDsneah7pEz2OPuLSqW8=', 'Content-Length': '0'} 2022-09-03T06:56:59.457 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2ef3f6f26a0e7d0-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.458 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.458 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.459 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:56:59.459 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:56:59.460 DEBUG:boto:Method: GET 2022-09-03T06:56:59.461 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:56:59.461 DEBUG:boto:Data: 2022-09-03T06:56:59.462 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.463 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.463 DEBUG:boto:Params: {} 2022-09-03T06:56:59.464 DEBUG:boto:Token: None 2022-09-03T06:56:59.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.469 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000910b57c43b330af9-006312fabb-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.516 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.518 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.519 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.519 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:56:59.520 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:56:59.520 DEBUG:boto:Method: GET 2022-09-03T06:56:59.521 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:56:59.522 DEBUG:boto:Data: 2022-09-03T06:56:59.523 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.523 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.524 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.525 DEBUG:boto:Params: {} 2022-09-03T06:56:59.526 DEBUG:boto:Token: None 2022-09-03T06:56:59.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.532 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.536 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d03f5af09de430e-006312fabb-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.537 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.539 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:56:59.539 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.540 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.541 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.541 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.542 DEBUG:boto:Method: GET 2022-09-03T06:56:59.543 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:56:59.543 DEBUG:boto:Data: 2022-09-03T06:56:59.544 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.545 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.545 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.546 DEBUG:boto:Params: {} 2022-09-03T06:56:59.546 DEBUG:boto:Token: None 2022-09-03T06:56:59.547 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d?versionId=null 2022-09-03T06:56:59.548 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HCTDSKEA4uQ9zZhdjN2nNSoqJ7o= 2022-09-03T06:56:59.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HCTDSKEA4uQ9zZhdjN2nNSoqJ7o=', 'Content-Length': '0'} 2022-09-03T06:56:59.551 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008896326ed73b7715-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.552 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.553 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.553 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.554 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.555 DEBUG:boto:Method: GET 2022-09-03T06:56:59.555 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:56:59.556 DEBUG:boto:Data: 2022-09-03T06:56:59.556 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.557 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.558 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.558 DEBUG:boto:Params: {} 2022-09-03T06:56:59.559 DEBUG:boto:Token: None 2022-09-03T06:56:59.559 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d?versionId=null 2022-09-03T06:56:59.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HCTDSKEA4uQ9zZhdjN2nNSoqJ7o= 2022-09-03T06:56:59.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HCTDSKEA4uQ9zZhdjN2nNSoqJ7o=', 'Content-Length': '0'} 2022-09-03T06:56:59.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055c834d3bee91fde-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.565 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.565 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.566 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.566 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.567 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.567 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:56:59.568 DEBUG:boto:Data: 2022-09-03T06:56:59.568 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.569 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.569 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.570 DEBUG:boto:Params: {} 2022-09-03T06:56:59.571 DEBUG:boto:Token: None 2022-09-03T06:56:59.572 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d?versionId=null 2022-09-03T06:56:59.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5b5+FpgAxP/WeGC7aawERy2cTdM= 2022-09-03T06:56:59.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5b5+FpgAxP/WeGC7aawERy2cTdM=', 'Content-Length': '0'} 2022-09-03T06:56:59.576 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce72527b2d442ccd-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.577 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.577 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.578 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.579 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.580 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.581 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:56:59.581 DEBUG:boto:Data: 2022-09-03T06:56:59.582 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.583 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.584 DEBUG:boto:Params: {} 2022-09-03T06:56:59.585 DEBUG:boto:Token: None 2022-09-03T06:56:59.586 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d?versionId=null 2022-09-03T06:56:59.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5b5+FpgAxP/WeGC7aawERy2cTdM= 2022-09-03T06:56:59.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5b5+FpgAxP/WeGC7aawERy2cTdM=', 'Content-Length': '0'} 2022-09-03T06:56:59.591 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f58cd4208b45c0e6-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.593 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:56:59.593 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.594 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.595 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.596 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.597 DEBUG:boto:Method: GET 2022-09-03T06:56:59.598 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:56:59.599 DEBUG:boto:Data: 2022-09-03T06:56:59.600 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.601 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.601 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.602 DEBUG:boto:Params: {} 2022-09-03T06:56:59.603 DEBUG:boto:Token: None 2022-09-03T06:56:59.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d?versionId=null 2022-09-03T06:56:59.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HCTDSKEA4uQ9zZhdjN2nNSoqJ7o= 2022-09-03T06:56:59.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HCTDSKEA4uQ9zZhdjN2nNSoqJ7o=', 'Content-Length': '0'} 2022-09-03T06:56:59.608 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086c36a99740bb2a5-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.608 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.610 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.611 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.612 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:56:59.613 DEBUG:boto:Method: GET 2022-09-03T06:56:59.613 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:56:59.614 DEBUG:boto:Data: 2022-09-03T06:56:59.614 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.616 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.617 DEBUG:boto:Params: {} 2022-09-03T06:56:59.618 DEBUG:boto:Token: None 2022-09-03T06:56:59.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d?versionId=null 2022-09-03T06:56:59.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HCTDSKEA4uQ9zZhdjN2nNSoqJ7o= 2022-09-03T06:56:59.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HCTDSKEA4uQ9zZhdjN2nNSoqJ7o=', 'Content-Length': '0'} 2022-09-03T06:56:59.625 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d57451de39a1d7b-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.666 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.667 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.667 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:56:59.668 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:56:59.669 DEBUG:boto:Method: GET 2022-09-03T06:56:59.669 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:56:59.670 DEBUG:boto:Data: 2022-09-03T06:56:59.670 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.671 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.672 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.673 DEBUG:boto:Params: {} 2022-09-03T06:56:59.673 DEBUG:boto:Token: None 2022-09-03T06:56:59.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.678 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006de8e99374dbb81e-006312fabb-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.679 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.680 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.680 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.681 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:56:59.682 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:56:59.683 DEBUG:boto:Method: GET 2022-09-03T06:56:59.683 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:56:59.684 DEBUG:boto:Data: 2022-09-03T06:56:59.684 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.685 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.685 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.686 DEBUG:boto:Params: {} 2022-09-03T06:56:59.687 DEBUG:boto:Token: None 2022-09-03T06:56:59.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.693 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098aba260f5b6fb2e-006312fabb-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.693 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.695 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.695 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.696 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.696 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:56:59.697 DEBUG:boto:Data: 2022-09-03T06:56:59.698 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.698 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.699 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.699 DEBUG:boto:Params: {} 2022-09-03T06:56:59.700 DEBUG:boto:Token: None 2022-09-03T06:56:59.701 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d 2022-09-03T06:56:59.701 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KIedBU681Hc1dRDxVoY4xMTpNzg= 2022-09-03T06:56:59.702 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KIedBU681Hc1dRDxVoY4xMTpNzg=', 'Content-Length': '0'} 2022-09-03T06:56:59.704 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9904f9f9c7901d8-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.705 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.705 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.706 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.707 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:56:59.707 DEBUG:boto:Data: 2022-09-03T06:56:59.708 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.708 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.709 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.709 DEBUG:boto:Params: {} 2022-09-03T06:56:59.710 DEBUG:boto:Token: None 2022-09-03T06:56:59.711 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d 2022-09-03T06:56:59.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KIedBU681Hc1dRDxVoY4xMTpNzg= 2022-09-03T06:56:59.712 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KIedBU681Hc1dRDxVoY4xMTpNzg=', 'Content-Length': '0'} 2022-09-03T06:56:59.714 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000061f750b95fcdad34-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.715 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:56:59.715 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.716 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.717 DEBUG:boto:Method: GET 2022-09-03T06:56:59.717 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:56:59.718 DEBUG:boto:Data: 2022-09-03T06:56:59.718 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.719 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.719 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.720 DEBUG:boto:Params: {} 2022-09-03T06:56:59.720 DEBUG:boto:Token: None 2022-09-03T06:56:59.721 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d 2022-09-03T06:56:59.722 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dwzw381hf+waJcnVa/fJ2Y9kf/c= 2022-09-03T06:56:59.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dwzw381hf+waJcnVa/fJ2Y9kf/c=', 'Content-Length': '0'} 2022-09-03T06:56:59.725 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009632a0bfd2e71d1b-006312fabb-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.726 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:56:59.726 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:56:59.727 DEBUG:boto:Method: GET 2022-09-03T06:56:59.728 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:56:59.729 DEBUG:boto:Data: 2022-09-03T06:56:59.730 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.732 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.732 DEBUG:boto:Params: {} 2022-09-03T06:56:59.733 DEBUG:boto:Token: None 2022-09-03T06:56:59.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/d 2022-09-03T06:56:59.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dwzw381hf+waJcnVa/fJ2Y9kf/c= 2022-09-03T06:56:59.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dwzw381hf+waJcnVa/fJ2Y9kf/c=', 'Content-Length': '0'} 2022-09-03T06:56:59.738 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042ade466c225c41e-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.739 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.741 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.742 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:56:59.743 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:56:59.743 DEBUG:boto:Method: GET 2022-09-03T06:56:59.744 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:56:59.745 DEBUG:boto:Data: 2022-09-03T06:56:59.745 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.746 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:56:59.746 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.747 DEBUG:boto:Params: {} 2022-09-03T06:56:59.747 DEBUG:boto:Token: None 2022-09-03T06:56:59.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.754 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000073bb22ee36f9e365-006312fabb-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.755 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.756 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.756 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.757 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:56:59.757 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:56:59.758 DEBUG:boto:Method: GET 2022-09-03T06:56:59.758 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:56:59.759 DEBUG:boto:Data: 2022-09-03T06:56:59.759 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.760 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.760 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.761 DEBUG:boto:Params: {} 2022-09-03T06:56:59.762 DEBUG:boto:Token: None 2022-09-03T06:56:59.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g= 2022-09-03T06:56:59.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gmwP5FBNfbW4EHRr7jMBuEkC43g=', 'Content-Length': '0'} 2022-09-03T06:56:59.768 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007de4bbdff043b820-006312fabb-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.810 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.812 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-72 zones={test-zone2, test-zone1} 2022-09-03T06:56:59.813 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-72 zones={test-zone3, test-zone1} 2022-09-03T06:56:59.814 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.816 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.817 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.818 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:56:59.819 DEBUG:boto:Data: 2022-09-03T06:56:59.820 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.821 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:56:59.822 DEBUG:boto:Port: 8001 2022-09-03T06:56:59.823 DEBUG:boto:Params: {} 2022-09-03T06:56:59.824 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T06:56:59.825 DEBUG:boto:Token: None 2022-09-03T06:56:59.827 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WTYjzzlukXRvtWkqKFiPJe04XsQ= 2022-09-03T06:56:59.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WTYjzzlukXRvtWkqKFiPJe04XsQ=', 'Content-Length': '0'} 2022-09-03T06:56:59.833 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f8a12331539a51f9-006312fabb-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.834 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.835 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.835 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.836 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:56:59.837 DEBUG:boto:Data: 2022-09-03T06:56:59.837 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.838 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.838 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.839 DEBUG:boto:Params: {} 2022-09-03T06:56:59.839 DEBUG:boto:Token: None 2022-09-03T06:56:59.840 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/ 2022-09-03T06:56:59.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WTYjzzlukXRvtWkqKFiPJe04XsQ= 2022-09-03T06:56:59.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WTYjzzlukXRvtWkqKFiPJe04XsQ=', 'Content-Length': '0'} 2022-09-03T06:56:59.854 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000026cb7d9fd81b99b1-006312fabb-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.854 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:56:59.855 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.855 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.856 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.856 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.857 DEBUG:boto:Method: GET 2022-09-03T06:56:59.857 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.858 DEBUG:boto:Data: 2022-09-03T06:56:59.859 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.859 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:56:59.860 DEBUG:boto:Port: 8001 2022-09-03T06:56:59.860 DEBUG:boto:Params: {} 2022-09-03T06:56:59.861 DEBUG:boto:Token: None 2022-09-03T06:56:59.862 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/?versions 2022-09-03T06:56:59.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:brEqG+ZQzKRHXnU9yp6io4GoCBU= 2022-09-03T06:56:59.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:brEqG+ZQzKRHXnU9yp6io4GoCBU=', 'Content-Length': '0'} 2022-09-03T06:56:59.866 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009f65020bc3412a82-006312fabb-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.867 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.868 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:56:59.869 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:56:59.869 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:56:59.870 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:56:59.870 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:56:59.871 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.871 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.872 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.873 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.874 DEBUG:boto:Method: GET 2022-09-03T06:56:59.875 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.876 DEBUG:boto:Data: 2022-09-03T06:56:59.878 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.880 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.881 DEBUG:boto:Params: {} 2022-09-03T06:56:59.882 DEBUG:boto:Token: None 2022-09-03T06:56:59.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/?versions 2022-09-03T06:56:59.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:brEqG+ZQzKRHXnU9yp6io4GoCBU= 2022-09-03T06:56:59.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:brEqG+ZQzKRHXnU9yp6io4GoCBU=', 'Content-Length': '0'} 2022-09-03T06:56:59.888 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9ecda59b2369c0a-006312fabb-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.889 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.890 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:56:59.891 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:56:59.892 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:56:59.892 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:56:59.894 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.895 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.895 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.896 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.897 DEBUG:boto:Method: GET 2022-09-03T06:56:59.897 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.898 DEBUG:boto:Data: 2022-09-03T06:56:59.898 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.899 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:56:59.900 DEBUG:boto:Port: 8001 2022-09-03T06:56:59.900 DEBUG:boto:Params: {} 2022-09-03T06:56:59.901 DEBUG:boto:Token: None 2022-09-03T06:56:59.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/?versions 2022-09-03T06:56:59.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:brEqG+ZQzKRHXnU9yp6io4GoCBU= 2022-09-03T06:56:59.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:brEqG+ZQzKRHXnU9yp6io4GoCBU=', 'Content-Length': '0'} 2022-09-03T06:56:59.907 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063217c1cca3223a9-006312fabb-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.908 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.909 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:56:59.909 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:56:59.910 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.910 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.911 DEBUG:boto:Method: GET 2022-09-03T06:56:59.911 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:56:59.912 DEBUG:boto:Data: 2022-09-03T06:56:59.912 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.913 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.913 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.915 DEBUG:boto:Params: {} 2022-09-03T06:56:59.915 DEBUG:boto:Token: None 2022-09-03T06:56:59.916 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/?versions 2022-09-03T06:56:59.917 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:brEqG+ZQzKRHXnU9yp6io4GoCBU= 2022-09-03T06:56:59.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:brEqG+ZQzKRHXnU9yp6io4GoCBU=', 'Content-Length': '0'} 2022-09-03T06:56:59.921 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7fa40ffbd9c93e7-006312fabb-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.921 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:56:59.922 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:56:59.923 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:59.923 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:59.924 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.926 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.927 DEBUG:boto:Method: GET 2022-09-03T06:56:59.927 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:59.928 DEBUG:boto:Data: 2022-09-03T06:56:59.928 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.929 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:56:59.929 DEBUG:boto:Port: 8001 2022-09-03T06:56:59.930 DEBUG:boto:Params: {} 2022-09-03T06:56:59.931 DEBUG:boto:Token: None 2022-09-03T06:56:59.931 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:59.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VC8AtrZ6NqGpsX1yNezkAZTaiBM= 2022-09-03T06:56:59.932 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VC8AtrZ6NqGpsX1yNezkAZTaiBM=', 'Content-Length': '0'} 2022-09-03T06:56:59.935 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000717902e63978451e-006312fabb-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.937 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:59.938 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:59.938 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.939 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.939 DEBUG:boto:Method: GET 2022-09-03T06:56:59.940 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:59.941 DEBUG:boto:Data: 2022-09-03T06:56:59.941 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.942 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.942 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.943 DEBUG:boto:Params: {} 2022-09-03T06:56:59.944 DEBUG:boto:Token: None 2022-09-03T06:56:59.944 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:59.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VC8AtrZ6NqGpsX1yNezkAZTaiBM= 2022-09-03T06:56:59.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VC8AtrZ6NqGpsX1yNezkAZTaiBM=', 'Content-Length': '0'} 2022-09-03T06:56:59.949 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa568bf726a7eef6-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.950 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:59.951 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:59.951 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.952 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.952 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.953 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:59.953 DEBUG:boto:Data: 2022-09-03T06:56:59.954 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:56:59.955 DEBUG:boto:Port: 8001 2022-09-03T06:56:59.955 DEBUG:boto:Params: {} 2022-09-03T06:56:59.957 DEBUG:boto:Token: None 2022-09-03T06:56:59.957 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:59.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBez4I8GDM7d2g4MXL7dr89cfCs= 2022-09-03T06:56:59.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBez4I8GDM7d2g4MXL7dr89cfCs=', 'Content-Length': '0'} 2022-09-03T06:56:59.961 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef0f3c0de6264d65-006312fabb-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.962 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:59.962 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:59.963 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.963 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.964 DEBUG:boto:Method: HEAD 2022-09-03T06:56:59.964 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:59.965 DEBUG:boto:Data: 2022-09-03T06:56:59.965 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:56:59.967 DEBUG:boto:Port: 8000 2022-09-03T06:56:59.968 DEBUG:boto:Params: {} 2022-09-03T06:56:59.968 DEBUG:boto:Token: None 2022-09-03T06:56:59.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:59.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBez4I8GDM7d2g4MXL7dr89cfCs= 2022-09-03T06:56:59.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBez4I8GDM7d2g4MXL7dr89cfCs=', 'Content-Length': '0'} 2022-09-03T06:56:59.973 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004dd7987753588f7c-006312fabb-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:56:59.974 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:56:59.975 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:56:59.976 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:56:59.979 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.980 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:56:59.981 DEBUG:boto:Method: GET 2022-09-03T06:56:59.982 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:56:59.983 DEBUG:boto:Data: 2022-09-03T06:56:59.984 DEBUG:boto:Headers: {} 2022-09-03T06:56:59.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:56:59.986 DEBUG:boto:Port: 8001 2022-09-03T06:56:59.986 DEBUG:boto:Params: {} 2022-09-03T06:56:59.987 DEBUG:boto:Token: None 2022-09-03T06:56:59.989 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:56:59 GMT /avysmw-72/a?versionId=null 2022-09-03T06:56:59.990 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VC8AtrZ6NqGpsX1yNezkAZTaiBM= 2022-09-03T06:56:59.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:56:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VC8AtrZ6NqGpsX1yNezkAZTaiBM=', 'Content-Length': '0'} 2022-09-03T06:56:59.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a46b9484941b71a4-006312fabb-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:56:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.041 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:00.042 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:00.042 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:57:00.043 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:57:00.043 DEBUG:boto:Method: GET 2022-09-03T06:57:00.044 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:57:00.044 DEBUG:boto:Data: 2022-09-03T06:57:00.045 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.046 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.047 DEBUG:boto:Params: {} 2022-09-03T06:57:00.048 DEBUG:boto:Token: None 2022-09-03T06:57:00.048 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/a?versionId=null 2022-09-03T06:57:00.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YtYicd+OG5Aepsks/lul8fC15Hg= 2022-09-03T06:57:00.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YtYicd+OG5Aepsks/lul8fC15Hg=', 'Content-Length': '0'} 2022-09-03T06:57:00.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cdd48fcd450adbdf-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.054 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.055 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.055 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:57:00.056 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:57:00.056 DEBUG:boto:Method: GET 2022-09-03T06:57:00.057 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:57:00.058 DEBUG:boto:Data: 2022-09-03T06:57:00.058 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.059 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.060 DEBUG:boto:Params: {} 2022-09-03T06:57:00.062 DEBUG:boto:Token: None 2022-09-03T06:57:00.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.068 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf202462d1ec9290-006312fabc-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.070 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.071 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.071 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.073 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:57:00.073 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:57:00.074 DEBUG:boto:Method: GET 2022-09-03T06:57:00.074 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:57:00.075 DEBUG:boto:Data: 2022-09-03T06:57:00.076 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.077 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.078 DEBUG:boto:Params: {} 2022-09-03T06:57:00.079 DEBUG:boto:Token: None 2022-09-03T06:57:00.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.085 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fdcf99ee99cba291-006312fabc-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.086 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.087 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:00.088 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:00.089 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.090 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:57:00.091 DEBUG:boto:Data: 2022-09-03T06:57:00.092 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.094 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.094 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.095 DEBUG:boto:Params: {} 2022-09-03T06:57:00.096 DEBUG:boto:Token: None 2022-09-03T06:57:00.097 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/a 2022-09-03T06:57:00.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3t4dh1/TRfjWSBZVeBrUCSnbyVU= 2022-09-03T06:57:00.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3t4dh1/TRfjWSBZVeBrUCSnbyVU=', 'Content-Length': '0'} 2022-09-03T06:57:00.101 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000727ff24a91f9e6b9-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.102 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:00.103 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:00.104 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.105 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:57:00.105 DEBUG:boto:Data: 2022-09-03T06:57:00.106 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.107 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.108 DEBUG:boto:Params: {} 2022-09-03T06:57:00.108 DEBUG:boto:Token: None 2022-09-03T06:57:00.109 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/a 2022-09-03T06:57:00.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3t4dh1/TRfjWSBZVeBrUCSnbyVU= 2022-09-03T06:57:00.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3t4dh1/TRfjWSBZVeBrUCSnbyVU=', 'Content-Length': '0'} 2022-09-03T06:57:00.112 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004749d4dde34671e4-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.113 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:57:00.114 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:00.114 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:00.115 DEBUG:boto:Method: GET 2022-09-03T06:57:00.116 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:57:00.116 DEBUG:boto:Data: 2022-09-03T06:57:00.117 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.118 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.119 DEBUG:boto:Params: {} 2022-09-03T06:57:00.119 DEBUG:boto:Token: None 2022-09-03T06:57:00.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/a 2022-09-03T06:57:00.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GC++fYOGC5WKpyeACAjGqIJ/Ukc= 2022-09-03T06:57:00.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GC++fYOGC5WKpyeACAjGqIJ/Ukc=', 'Content-Length': '0'} 2022-09-03T06:57:00.123 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fcb816a340d00b1f-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.124 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:00.125 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:00.125 DEBUG:boto:Method: GET 2022-09-03T06:57:00.126 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:57:00.127 DEBUG:boto:Data: 2022-09-03T06:57:00.127 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.128 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.129 DEBUG:boto:Params: {} 2022-09-03T06:57:00.130 DEBUG:boto:Token: None 2022-09-03T06:57:00.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/a 2022-09-03T06:57:00.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GC++fYOGC5WKpyeACAjGqIJ/Ukc= 2022-09-03T06:57:00.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GC++fYOGC5WKpyeACAjGqIJ/Ukc=', 'Content-Length': '0'} 2022-09-03T06:57:00.134 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd9d34884156df3c-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.135 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.135 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.136 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:57:00.137 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:57:00.138 DEBUG:boto:Method: GET 2022-09-03T06:57:00.138 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:57:00.139 DEBUG:boto:Data: 2022-09-03T06:57:00.140 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.140 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.141 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.141 DEBUG:boto:Params: {} 2022-09-03T06:57:00.142 DEBUG:boto:Token: None 2022-09-03T06:57:00.142 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.147 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eecd8bbc35ef5f6d-006312fabc-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.148 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.149 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.150 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.152 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:57:00.153 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:57:00.154 DEBUG:boto:Method: GET 2022-09-03T06:57:00.155 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:57:00.156 DEBUG:boto:Data: 2022-09-03T06:57:00.157 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.159 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.159 DEBUG:boto:Params: {} 2022-09-03T06:57:00.160 DEBUG:boto:Token: None 2022-09-03T06:57:00.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.165 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000242d95394889f5e1-006312fabc-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.166 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.167 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:57:00.167 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.168 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.168 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.169 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.169 DEBUG:boto:Method: GET 2022-09-03T06:57:00.170 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:00.171 DEBUG:boto:Data: 2022-09-03T06:57:00.172 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.173 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.173 DEBUG:boto:Params: {} 2022-09-03T06:57:00.174 DEBUG:boto:Token: None 2022-09-03T06:57:00.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:00.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:45UXQintIMNEAYDjqidRlNYP3gM= 2022-09-03T06:57:00.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:45UXQintIMNEAYDjqidRlNYP3gM=', 'Content-Length': '0'} 2022-09-03T06:57:00.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000897445f03b685fe8-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.179 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.180 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.180 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.181 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.181 DEBUG:boto:Method: GET 2022-09-03T06:57:00.182 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:00.182 DEBUG:boto:Data: 2022-09-03T06:57:00.183 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.183 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.184 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.185 DEBUG:boto:Params: {} 2022-09-03T06:57:00.185 DEBUG:boto:Token: None 2022-09-03T06:57:00.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:00.186 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:45UXQintIMNEAYDjqidRlNYP3gM= 2022-09-03T06:57:00.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:45UXQintIMNEAYDjqidRlNYP3gM=', 'Content-Length': '0'} 2022-09-03T06:57:00.189 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099b6cbcaf92232e4-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.190 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.191 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.191 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.192 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.192 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.193 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:00.193 DEBUG:boto:Data: 2022-09-03T06:57:00.194 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.195 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.195 DEBUG:boto:Params: {} 2022-09-03T06:57:00.196 DEBUG:boto:Token: None 2022-09-03T06:57:00.197 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:00.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p+wXFxPRVVU6WwfJkXz0H9PYXpQ= 2022-09-03T06:57:00.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p+wXFxPRVVU6WwfJkXz0H9PYXpQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.200 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b63d3ce603f391e-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.201 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.202 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.202 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.203 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.203 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.204 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:00.204 DEBUG:boto:Data: 2022-09-03T06:57:00.205 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.205 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.206 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.206 DEBUG:boto:Params: {} 2022-09-03T06:57:00.207 DEBUG:boto:Token: None 2022-09-03T06:57:00.207 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:00.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p+wXFxPRVVU6WwfJkXz0H9PYXpQ= 2022-09-03T06:57:00.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p+wXFxPRVVU6WwfJkXz0H9PYXpQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.211 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e1268a61ce1e150-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.212 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:57:00.212 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.213 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.213 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.214 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.214 DEBUG:boto:Method: GET 2022-09-03T06:57:00.215 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:00.215 DEBUG:boto:Data: 2022-09-03T06:57:00.216 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.216 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.217 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.218 DEBUG:boto:Params: {} 2022-09-03T06:57:00.218 DEBUG:boto:Token: None 2022-09-03T06:57:00.219 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:00.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:45UXQintIMNEAYDjqidRlNYP3gM= 2022-09-03T06:57:00.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:45UXQintIMNEAYDjqidRlNYP3gM=', 'Content-Length': '0'} 2022-09-03T06:57:00.223 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9cb9f7c077bed52-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.266 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.267 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.268 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.268 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:00.269 DEBUG:boto:Method: GET 2022-09-03T06:57:00.269 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:00.270 DEBUG:boto:Data: 2022-09-03T06:57:00.270 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.271 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.272 DEBUG:boto:Params: {} 2022-09-03T06:57:00.272 DEBUG:boto:Token: None 2022-09-03T06:57:00.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:00.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:45UXQintIMNEAYDjqidRlNYP3gM= 2022-09-03T06:57:00.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:45UXQintIMNEAYDjqidRlNYP3gM=', 'Content-Length': '0'} 2022-09-03T06:57:00.278 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032e9bfd4d4bc2a17-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.280 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.280 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.281 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:57:00.282 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:57:00.282 DEBUG:boto:Method: GET 2022-09-03T06:57:00.283 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:57:00.284 DEBUG:boto:Data: 2022-09-03T06:57:00.284 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.286 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.286 DEBUG:boto:Params: {} 2022-09-03T06:57:00.287 DEBUG:boto:Token: None 2022-09-03T06:57:00.288 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.292 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003487e82d3b191ed5-006312fabc-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.293 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.294 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.294 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.295 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:57:00.296 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:57:00.296 DEBUG:boto:Method: GET 2022-09-03T06:57:00.297 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:57:00.297 DEBUG:boto:Data: 2022-09-03T06:57:00.298 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.298 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.299 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.299 DEBUG:boto:Params: {} 2022-09-03T06:57:00.300 DEBUG:boto:Token: None 2022-09-03T06:57:00.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.301 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.305 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006287fee512c08a81-006312fabc-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.306 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.307 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.308 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.308 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.309 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:57:00.309 DEBUG:boto:Data: 2022-09-03T06:57:00.310 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.311 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.311 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.312 DEBUG:boto:Params: {} 2022-09-03T06:57:00.312 DEBUG:boto:Token: None 2022-09-03T06:57:00.313 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b 2022-09-03T06:57:00.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yWIgqgNQ+hh3NrT7foHnIPpp1n8= 2022-09-03T06:57:00.314 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yWIgqgNQ+hh3NrT7foHnIPpp1n8=', 'Content-Length': '0'} 2022-09-03T06:57:00.317 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e0042ad6e33c7ba-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.318 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.319 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.320 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.321 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:57:00.322 DEBUG:boto:Data: 2022-09-03T06:57:00.322 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.323 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.324 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.325 DEBUG:boto:Params: {} 2022-09-03T06:57:00.326 DEBUG:boto:Token: None 2022-09-03T06:57:00.326 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b 2022-09-03T06:57:00.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yWIgqgNQ+hh3NrT7foHnIPpp1n8= 2022-09-03T06:57:00.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yWIgqgNQ+hh3NrT7foHnIPpp1n8=', 'Content-Length': '0'} 2022-09-03T06:57:00.331 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008587df1820721972-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.332 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:57:00.332 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.333 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.334 DEBUG:boto:Method: GET 2022-09-03T06:57:00.334 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:57:00.335 DEBUG:boto:Data: 2022-09-03T06:57:00.335 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.336 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.337 DEBUG:boto:Params: {} 2022-09-03T06:57:00.338 DEBUG:boto:Token: None 2022-09-03T06:57:00.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b 2022-09-03T06:57:00.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:svy8wNwNl/DmO5fFPLABMfmJvlY= 2022-09-03T06:57:00.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:svy8wNwNl/DmO5fFPLABMfmJvlY=', 'Content-Length': '0'} 2022-09-03T06:57:00.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ddb1c2318bc52c58-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.343 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:00.344 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:00.344 DEBUG:boto:Method: GET 2022-09-03T06:57:00.345 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:57:00.346 DEBUG:boto:Data: 2022-09-03T06:57:00.347 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.347 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.348 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.349 DEBUG:boto:Params: {} 2022-09-03T06:57:00.349 DEBUG:boto:Token: None 2022-09-03T06:57:00.350 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/b 2022-09-03T06:57:00.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:svy8wNwNl/DmO5fFPLABMfmJvlY= 2022-09-03T06:57:00.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:svy8wNwNl/DmO5fFPLABMfmJvlY=', 'Content-Length': '0'} 2022-09-03T06:57:00.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3f26c84319e43a2-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.402 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.404 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.404 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:57:00.406 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:57:00.407 DEBUG:boto:Method: GET 2022-09-03T06:57:00.408 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:57:00.409 DEBUG:boto:Data: 2022-09-03T06:57:00.409 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.411 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.411 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.412 DEBUG:boto:Params: {} 2022-09-03T06:57:00.413 DEBUG:boto:Token: None 2022-09-03T06:57:00.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.422 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad4e45272605b13a-006312fabc-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.423 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.424 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.425 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.426 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:57:00.427 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:57:00.428 DEBUG:boto:Method: GET 2022-09-03T06:57:00.428 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:57:00.429 DEBUG:boto:Data: 2022-09-03T06:57:00.429 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.430 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.431 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.432 DEBUG:boto:Params: {} 2022-09-03T06:57:00.433 DEBUG:boto:Token: None 2022-09-03T06:57:00.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.439 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003d42eb6b73d0d1b5-006312fabc-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.440 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.441 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:57:00.442 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.443 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.444 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.444 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.445 DEBUG:boto:Method: GET 2022-09-03T06:57:00.445 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:00.446 DEBUG:boto:Data: 2022-09-03T06:57:00.447 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.447 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.448 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.448 DEBUG:boto:Params: {} 2022-09-03T06:57:00.449 DEBUG:boto:Token: None 2022-09-03T06:57:00.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:00.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nENjEtk7YKXP4NIblU8c2pXgFXU= 2022-09-03T06:57:00.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nENjEtk7YKXP4NIblU8c2pXgFXU=', 'Content-Length': '0'} 2022-09-03T06:57:00.454 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006011fe4ffa57dc43-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.455 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.456 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.456 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.457 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.458 DEBUG:boto:Method: GET 2022-09-03T06:57:00.458 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:00.459 DEBUG:boto:Data: 2022-09-03T06:57:00.460 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.461 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.462 DEBUG:boto:Params: {} 2022-09-03T06:57:00.463 DEBUG:boto:Token: None 2022-09-03T06:57:00.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:00.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nENjEtk7YKXP4NIblU8c2pXgFXU= 2022-09-03T06:57:00.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nENjEtk7YKXP4NIblU8c2pXgFXU=', 'Content-Length': '0'} 2022-09-03T06:57:00.468 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003747880a954b6e88-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.469 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.470 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.471 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.471 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.472 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.473 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:00.474 DEBUG:boto:Data: 2022-09-03T06:57:00.475 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.476 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.477 DEBUG:boto:Params: {} 2022-09-03T06:57:00.478 DEBUG:boto:Token: None 2022-09-03T06:57:00.479 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:00.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JXLtfqHNwwcLRKklvUolOAeUw+U= 2022-09-03T06:57:00.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JXLtfqHNwwcLRKklvUolOAeUw+U=', 'Content-Length': '0'} 2022-09-03T06:57:00.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008586b4dd077fb363-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.484 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.484 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.485 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.486 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.487 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.488 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:00.488 DEBUG:boto:Data: 2022-09-03T06:57:00.489 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.491 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.492 DEBUG:boto:Params: {} 2022-09-03T06:57:00.493 DEBUG:boto:Token: None 2022-09-03T06:57:00.494 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:00.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JXLtfqHNwwcLRKklvUolOAeUw+U= 2022-09-03T06:57:00.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JXLtfqHNwwcLRKklvUolOAeUw+U=', 'Content-Length': '0'} 2022-09-03T06:57:00.498 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aef118c4716076ef-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.499 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:57:00.500 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.501 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.502 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.502 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.503 DEBUG:boto:Method: GET 2022-09-03T06:57:00.504 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:00.505 DEBUG:boto:Data: 2022-09-03T06:57:00.505 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.506 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.507 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.508 DEBUG:boto:Params: {} 2022-09-03T06:57:00.508 DEBUG:boto:Token: None 2022-09-03T06:57:00.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:00.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nENjEtk7YKXP4NIblU8c2pXgFXU= 2022-09-03T06:57:00.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nENjEtk7YKXP4NIblU8c2pXgFXU=', 'Content-Length': '0'} 2022-09-03T06:57:00.515 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000326990c8979b2a76-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.558 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.560 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.560 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.562 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:00.563 DEBUG:boto:Method: GET 2022-09-03T06:57:00.565 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:00.566 DEBUG:boto:Data: 2022-09-03T06:57:00.585 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.589 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.590 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.591 DEBUG:boto:Params: {} 2022-09-03T06:57:00.593 DEBUG:boto:Token: None 2022-09-03T06:57:00.594 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:00.595 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nENjEtk7YKXP4NIblU8c2pXgFXU= 2022-09-03T06:57:00.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nENjEtk7YKXP4NIblU8c2pXgFXU=', 'Content-Length': '0'} 2022-09-03T06:57:00.600 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f129d1ab9c652794-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.601 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.602 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.603 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:57:00.604 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:57:00.604 DEBUG:boto:Method: GET 2022-09-03T06:57:00.605 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:57:00.606 DEBUG:boto:Data: 2022-09-03T06:57:00.607 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.610 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.611 DEBUG:boto:Params: {} 2022-09-03T06:57:00.613 DEBUG:boto:Token: None 2022-09-03T06:57:00.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.619 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093c6de3bfb375477-006312fabc-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.620 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.621 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.622 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.623 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:57:00.624 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:57:00.625 DEBUG:boto:Method: GET 2022-09-03T06:57:00.625 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:57:00.626 DEBUG:boto:Data: 2022-09-03T06:57:00.627 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.628 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.628 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.629 DEBUG:boto:Params: {} 2022-09-03T06:57:00.630 DEBUG:boto:Token: None 2022-09-03T06:57:00.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.637 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c15c6ed6317b9ab-006312fabc-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.638 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.641 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.642 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.642 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.643 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:57:00.644 DEBUG:boto:Data: 2022-09-03T06:57:00.645 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.647 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.648 DEBUG:boto:Params: {} 2022-09-03T06:57:00.649 DEBUG:boto:Token: None 2022-09-03T06:57:00.650 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c 2022-09-03T06:57:00.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WahfPWxGSdwBUfC+H1wqFbgU68w= 2022-09-03T06:57:00.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WahfPWxGSdwBUfC+H1wqFbgU68w=', 'Content-Length': '0'} 2022-09-03T06:57:00.659 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d362fb1fc1216091-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.661 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.661 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.662 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.663 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:57:00.664 DEBUG:boto:Data: 2022-09-03T06:57:00.665 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.667 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.668 DEBUG:boto:Params: {} 2022-09-03T06:57:00.669 DEBUG:boto:Token: None 2022-09-03T06:57:00.670 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c 2022-09-03T06:57:00.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WahfPWxGSdwBUfC+H1wqFbgU68w= 2022-09-03T06:57:00.672 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WahfPWxGSdwBUfC+H1wqFbgU68w=', 'Content-Length': '0'} 2022-09-03T06:57:00.675 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038568b1b92340588-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.676 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:57:00.677 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.677 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.678 DEBUG:boto:Method: GET 2022-09-03T06:57:00.679 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:57:00.679 DEBUG:boto:Data: 2022-09-03T06:57:00.680 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.680 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.681 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.682 DEBUG:boto:Params: {} 2022-09-03T06:57:00.682 DEBUG:boto:Token: None 2022-09-03T06:57:00.683 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c 2022-09-03T06:57:00.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mce1QHRMEiqP2+6+fLrRy1wLBDE= 2022-09-03T06:57:00.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mce1QHRMEiqP2+6+fLrRy1wLBDE=', 'Content-Length': '0'} 2022-09-03T06:57:00.689 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b3423426b1e8c75-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.694 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:00.695 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:00.695 DEBUG:boto:Method: GET 2022-09-03T06:57:00.699 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:57:00.700 DEBUG:boto:Data: 2022-09-03T06:57:00.701 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.702 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.703 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.703 DEBUG:boto:Params: {} 2022-09-03T06:57:00.704 DEBUG:boto:Token: None 2022-09-03T06:57:00.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/c 2022-09-03T06:57:00.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mce1QHRMEiqP2+6+fLrRy1wLBDE= 2022-09-03T06:57:00.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mce1QHRMEiqP2+6+fLrRy1wLBDE=', 'Content-Length': '0'} 2022-09-03T06:57:00.711 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000088c44f217d3f5c03-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.754 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.755 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.756 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:57:00.756 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:57:00.757 DEBUG:boto:Method: GET 2022-09-03T06:57:00.758 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:57:00.758 DEBUG:boto:Data: 2022-09-03T06:57:00.759 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.760 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.761 DEBUG:boto:Params: {} 2022-09-03T06:57:00.762 DEBUG:boto:Token: None 2022-09-03T06:57:00.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.763 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.767 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000083b5200bc1fd6d-006312fabc-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.768 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.769 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.769 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.770 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:57:00.770 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:57:00.771 DEBUG:boto:Method: GET 2022-09-03T06:57:00.772 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:57:00.773 DEBUG:boto:Data: 2022-09-03T06:57:00.774 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.775 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.776 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.777 DEBUG:boto:Params: {} 2022-09-03T06:57:00.778 DEBUG:boto:Token: None 2022-09-03T06:57:00.779 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.790 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000012b2901ac75bcda2-006312fabc-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.791 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.792 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:57:00.793 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.794 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.794 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.795 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.796 DEBUG:boto:Method: GET 2022-09-03T06:57:00.797 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:00.798 DEBUG:boto:Data: 2022-09-03T06:57:00.799 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.800 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.801 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.802 DEBUG:boto:Params: {} 2022-09-03T06:57:00.803 DEBUG:boto:Token: None 2022-09-03T06:57:00.804 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:00.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZguUL4O+SBHdfO1C5pDVzTpihHw= 2022-09-03T06:57:00.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZguUL4O+SBHdfO1C5pDVzTpihHw=', 'Content-Length': '0'} 2022-09-03T06:57:00.810 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c39591b092ccc7a0-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.810 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.811 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.811 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.812 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.812 DEBUG:boto:Method: GET 2022-09-03T06:57:00.813 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:00.814 DEBUG:boto:Data: 2022-09-03T06:57:00.814 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.815 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.816 DEBUG:boto:Params: {} 2022-09-03T06:57:00.816 DEBUG:boto:Token: None 2022-09-03T06:57:00.817 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:00.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZguUL4O+SBHdfO1C5pDVzTpihHw= 2022-09-03T06:57:00.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZguUL4O+SBHdfO1C5pDVzTpihHw=', 'Content-Length': '0'} 2022-09-03T06:57:00.821 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d651354b81c755f-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.822 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.822 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.823 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.823 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.824 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.824 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:00.825 DEBUG:boto:Data: 2022-09-03T06:57:00.825 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.826 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.827 DEBUG:boto:Params: {} 2022-09-03T06:57:00.827 DEBUG:boto:Token: None 2022-09-03T06:57:00.828 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:00.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kcntbeoqDkN0ON1/nAM0vb5Gxc4= 2022-09-03T06:57:00.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kcntbeoqDkN0ON1/nAM0vb5Gxc4=', 'Content-Length': '0'} 2022-09-03T06:57:00.831 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000094716bcdf2f5f75a-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.832 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.833 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.834 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.834 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.835 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.837 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:00.837 DEBUG:boto:Data: 2022-09-03T06:57:00.838 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.839 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.839 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.840 DEBUG:boto:Params: {} 2022-09-03T06:57:00.841 DEBUG:boto:Token: None 2022-09-03T06:57:00.841 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:00.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kcntbeoqDkN0ON1/nAM0vb5Gxc4= 2022-09-03T06:57:00.843 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kcntbeoqDkN0ON1/nAM0vb5Gxc4=', 'Content-Length': '0'} 2022-09-03T06:57:00.845 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6244f319b07f06a-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.846 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:57:00.846 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.847 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.848 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.848 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.849 DEBUG:boto:Method: GET 2022-09-03T06:57:00.849 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:00.850 DEBUG:boto:Data: 2022-09-03T06:57:00.850 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.852 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.852 DEBUG:boto:Params: {} 2022-09-03T06:57:00.853 DEBUG:boto:Token: None 2022-09-03T06:57:00.854 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:00.855 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZguUL4O+SBHdfO1C5pDVzTpihHw= 2022-09-03T06:57:00.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZguUL4O+SBHdfO1C5pDVzTpihHw=', 'Content-Length': '0'} 2022-09-03T06:57:00.857 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e247889603ba200-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.858 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.859 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.860 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.860 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:00.861 DEBUG:boto:Method: GET 2022-09-03T06:57:00.862 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:00.862 DEBUG:boto:Data: 2022-09-03T06:57:00.863 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.863 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.864 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.865 DEBUG:boto:Params: {} 2022-09-03T06:57:00.865 DEBUG:boto:Token: None 2022-09-03T06:57:00.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:00.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZguUL4O+SBHdfO1C5pDVzTpihHw= 2022-09-03T06:57:00.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZguUL4O+SBHdfO1C5pDVzTpihHw=', 'Content-Length': '0'} 2022-09-03T06:57:00.870 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f96006b1393295bd-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.871 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.872 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.873 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:57:00.874 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:57:00.875 DEBUG:boto:Method: GET 2022-09-03T06:57:00.875 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:57:00.876 DEBUG:boto:Data: 2022-09-03T06:57:00.877 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.879 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.879 DEBUG:boto:Params: {} 2022-09-03T06:57:00.880 DEBUG:boto:Token: None 2022-09-03T06:57:00.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.882 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.886 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017df805dc54b8f4f-006312fabc-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.887 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.888 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.889 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.890 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:57:00.891 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:57:00.891 DEBUG:boto:Method: GET 2022-09-03T06:57:00.892 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:57:00.893 DEBUG:boto:Data: 2022-09-03T06:57:00.893 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.894 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.894 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.895 DEBUG:boto:Params: {} 2022-09-03T06:57:00.895 DEBUG:boto:Token: None 2022-09-03T06:57:00.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.900 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000071912c831dbfa399-006312fabc-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.901 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.902 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.902 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.903 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.903 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:57:00.904 DEBUG:boto:Data: 2022-09-03T06:57:00.904 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.905 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.906 DEBUG:boto:Params: {} 2022-09-03T06:57:00.906 DEBUG:boto:Token: None 2022-09-03T06:57:00.907 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d 2022-09-03T06:57:00.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DSUquvu5tkDmC+Q7NrRbCHniTmU= 2022-09-03T06:57:00.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DSUquvu5tkDmC+Q7NrRbCHniTmU=', 'Content-Length': '0'} 2022-09-03T06:57:00.911 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fccb082cf7e33971-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.911 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.912 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.912 DEBUG:boto:Method: HEAD 2022-09-03T06:57:00.913 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:57:00.913 DEBUG:boto:Data: 2022-09-03T06:57:00.914 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.914 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.915 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.916 DEBUG:boto:Params: {} 2022-09-03T06:57:00.916 DEBUG:boto:Token: None 2022-09-03T06:57:00.917 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d 2022-09-03T06:57:00.917 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DSUquvu5tkDmC+Q7NrRbCHniTmU= 2022-09-03T06:57:00.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DSUquvu5tkDmC+Q7NrRbCHniTmU=', 'Content-Length': '0'} 2022-09-03T06:57:00.920 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009861b39a0b28205c-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.921 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:57:00.921 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.922 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.922 DEBUG:boto:Method: GET 2022-09-03T06:57:00.923 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:57:00.923 DEBUG:boto:Data: 2022-09-03T06:57:00.924 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.925 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.925 DEBUG:boto:Params: {} 2022-09-03T06:57:00.926 DEBUG:boto:Token: None 2022-09-03T06:57:00.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d 2022-09-03T06:57:00.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B4Gf4gjtTD56jpombSMYsmOI9Zs= 2022-09-03T06:57:00.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B4Gf4gjtTD56jpombSMYsmOI9Zs=', 'Content-Length': '0'} 2022-09-03T06:57:00.930 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6919faa2acedd2f-006312fabc-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.974 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:00.975 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:00.975 DEBUG:boto:Method: GET 2022-09-03T06:57:00.976 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:57:00.977 DEBUG:boto:Data: 2022-09-03T06:57:00.977 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:00.978 DEBUG:boto:Port: 8000 2022-09-03T06:57:00.979 DEBUG:boto:Params: {} 2022-09-03T06:57:00.979 DEBUG:boto:Token: None 2022-09-03T06:57:00.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/d 2022-09-03T06:57:00.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B4Gf4gjtTD56jpombSMYsmOI9Zs= 2022-09-03T06:57:00.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B4Gf4gjtTD56jpombSMYsmOI9Zs=', 'Content-Length': '0'} 2022-09-03T06:57:00.984 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007df54b906e9492b6-006312fabc-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.984 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.985 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.986 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:57:00.986 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:57:00.987 DEBUG:boto:Method: GET 2022-09-03T06:57:00.987 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:57:00.988 DEBUG:boto:Data: 2022-09-03T06:57:00.988 DEBUG:boto:Headers: {} 2022-09-03T06:57:00.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:00.990 DEBUG:boto:Port: 8001 2022-09-03T06:57:00.990 DEBUG:boto:Params: {} 2022-09-03T06:57:00.991 DEBUG:boto:Token: None 2022-09-03T06:57:00.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:00 GMT /avysmw-72/ 2022-09-03T06:57:00.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ= 2022-09-03T06:57:00.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GQa6Uzxc8WUjpUQLaNUqb602uUQ=', 'Content-Length': '0'} 2022-09-03T06:57:00.996 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f3b997bc6c4f1e1f-006312fabc-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:00.996 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:00.997 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:00.998 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:00.998 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:57:00.999 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:57:00.999 DEBUG:boto:Method: GET 2022-09-03T06:57:01.000 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:57:01.000 DEBUG:boto:Data: 2022-09-03T06:57:01.001 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.002 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:01.002 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.003 DEBUG:boto:Params: {} 2022-09-03T06:57:01.003 DEBUG:boto:Token: None 2022-09-03T06:57:01.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.008 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000363b80711fedae96-006312fabd-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.009 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.010 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-72 zones={test-zone3, test-zone1} 2022-09-03T06:57:01.010 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-72 zones={test-zone3, test-zone2} 2022-09-03T06:57:01.011 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.011 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.012 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.012 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:57:01.013 DEBUG:boto:Data: 2022-09-03T06:57:01.013 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.014 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.015 DEBUG:boto:Params: {} 2022-09-03T06:57:01.016 DEBUG:boto:Token: None 2022-09-03T06:57:01.016 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ifsieB4QRPU4ih6KEap3sYMfDRg= 2022-09-03T06:57:01.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ifsieB4QRPU4ih6KEap3sYMfDRg=', 'Content-Length': '0'} 2022-09-03T06:57:01.020 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000b5c416efe270463-006312fabd-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.021 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.021 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.022 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.023 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:57:01.023 DEBUG:boto:Data: 2022-09-03T06:57:01.024 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.024 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.025 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.025 DEBUG:boto:Params: {} 2022-09-03T06:57:01.026 DEBUG:boto:Token: None 2022-09-03T06:57:01.026 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ifsieB4QRPU4ih6KEap3sYMfDRg= 2022-09-03T06:57:01.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ifsieB4QRPU4ih6KEap3sYMfDRg=', 'Content-Length': '0'} 2022-09-03T06:57:01.030 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b6745af0c82e6501-006312fabd-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.031 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T06:57:01.031 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.032 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.033 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.033 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.034 DEBUG:boto:Method: GET 2022-09-03T06:57:01.034 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.035 DEBUG:boto:Data: 2022-09-03T06:57:01.035 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.036 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.037 DEBUG:boto:Params: {} 2022-09-03T06:57:01.038 DEBUG:boto:Token: None 2022-09-03T06:57:01.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/?versions 2022-09-03T06:57:01.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j7iksfc5DacdAx5pCnPVg2MK4v4= 2022-09-03T06:57:01.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j7iksfc5DacdAx5pCnPVg2MK4v4=', 'Content-Length': '0'} 2022-09-03T06:57:01.042 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e282b7be5ddb87c-006312fabd-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.043 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.044 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:57:01.044 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:57:01.045 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:57:01.045 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:57:01.046 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T06:57:01.047 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.047 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.048 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.048 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.049 DEBUG:boto:Method: GET 2022-09-03T06:57:01.049 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.050 DEBUG:boto:Data: 2022-09-03T06:57:01.050 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.051 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.051 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.052 DEBUG:boto:Params: {} 2022-09-03T06:57:01.052 DEBUG:boto:Token: None 2022-09-03T06:57:01.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/?versions 2022-09-03T06:57:01.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j7iksfc5DacdAx5pCnPVg2MK4v4= 2022-09-03T06:57:01.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j7iksfc5DacdAx5pCnPVg2MK4v4=', 'Content-Length': '0'} 2022-09-03T06:57:01.057 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000013d0e5499fb739b8-006312fabd-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.057 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.059 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T06:57:01.059 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T06:57:01.060 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T06:57:01.060 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T06:57:01.061 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.061 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.062 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.062 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.063 DEBUG:boto:Method: GET 2022-09-03T06:57:01.063 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.064 DEBUG:boto:Data: 2022-09-03T06:57:01.064 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.065 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.065 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.066 DEBUG:boto:Params: {} 2022-09-03T06:57:01.066 DEBUG:boto:Token: None 2022-09-03T06:57:01.067 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/?versions 2022-09-03T06:57:01.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j7iksfc5DacdAx5pCnPVg2MK4v4= 2022-09-03T06:57:01.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j7iksfc5DacdAx5pCnPVg2MK4v4=', 'Content-Length': '0'} 2022-09-03T06:57:01.072 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008e234476a23ccb17-006312fabd-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.073 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.074 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.074 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.075 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.075 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.076 DEBUG:boto:Method: GET 2022-09-03T06:57:01.076 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T06:57:01.077 DEBUG:boto:Data: 2022-09-03T06:57:01.077 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.078 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.078 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.079 DEBUG:boto:Params: {} 2022-09-03T06:57:01.080 DEBUG:boto:Token: None 2022-09-03T06:57:01.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/?versions 2022-09-03T06:57:01.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j7iksfc5DacdAx5pCnPVg2MK4v4= 2022-09-03T06:57:01.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j7iksfc5DacdAx5pCnPVg2MK4v4=', 'Content-Length': '0'} 2022-09-03T06:57:01.085 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000080b662e389554fd4-006312fabd-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.086 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.087 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:57:01.088 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.089 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.090 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.091 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.092 DEBUG:boto:Method: GET 2022-09-03T06:57:01.093 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:57:01.094 DEBUG:boto:Data: 2022-09-03T06:57:01.095 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.096 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.096 DEBUG:boto:Params: {} 2022-09-03T06:57:01.097 DEBUG:boto:Token: None 2022-09-03T06:57:01.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a?versionId=null 2022-09-03T06:57:01.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oYyqezSwLDCQG2Vx5VvUhzV/Et8= 2022-09-03T06:57:01.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oYyqezSwLDCQG2Vx5VvUhzV/Et8=', 'Content-Length': '0'} 2022-09-03T06:57:01.101 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048cc37fa0bae6d88-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.143 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.144 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.145 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.146 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.147 DEBUG:boto:Method: GET 2022-09-03T06:57:01.148 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:57:01.149 DEBUG:boto:Data: 2022-09-03T06:57:01.149 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.150 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.151 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.152 DEBUG:boto:Params: {} 2022-09-03T06:57:01.152 DEBUG:boto:Token: None 2022-09-03T06:57:01.153 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a?versionId=null 2022-09-03T06:57:01.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oYyqezSwLDCQG2Vx5VvUhzV/Et8= 2022-09-03T06:57:01.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oYyqezSwLDCQG2Vx5VvUhzV/Et8=', 'Content-Length': '0'} 2022-09-03T06:57:01.157 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000073547a6991111d89-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.158 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.159 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.160 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.160 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.161 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.161 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:57:01.162 DEBUG:boto:Data: 2022-09-03T06:57:01.163 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.164 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.164 DEBUG:boto:Params: {} 2022-09-03T06:57:01.165 DEBUG:boto:Token: None 2022-09-03T06:57:01.165 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a?versionId=null 2022-09-03T06:57:01.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4qtbeLOkS27Z6uCRrT7xaB9NceA= 2022-09-03T06:57:01.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4qtbeLOkS27Z6uCRrT7xaB9NceA=', 'Content-Length': '0'} 2022-09-03T06:57:01.169 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000077f84976bb5ee4b3-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.169 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.170 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.171 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.172 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.173 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.174 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:57:01.175 DEBUG:boto:Data: 2022-09-03T06:57:01.176 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.177 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.179 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.180 DEBUG:boto:Params: {} 2022-09-03T06:57:01.180 DEBUG:boto:Token: None 2022-09-03T06:57:01.181 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a?versionId=null 2022-09-03T06:57:01.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4qtbeLOkS27Z6uCRrT7xaB9NceA= 2022-09-03T06:57:01.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4qtbeLOkS27Z6uCRrT7xaB9NceA=', 'Content-Length': '0'} 2022-09-03T06:57:01.186 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd8151bbf437a3ea-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.187 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:57:01.187 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.188 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.188 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.189 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.189 DEBUG:boto:Method: GET 2022-09-03T06:57:01.190 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:57:01.191 DEBUG:boto:Data: 2022-09-03T06:57:01.191 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.192 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.193 DEBUG:boto:Params: {} 2022-09-03T06:57:01.193 DEBUG:boto:Token: None 2022-09-03T06:57:01.194 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a?versionId=null 2022-09-03T06:57:01.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oYyqezSwLDCQG2Vx5VvUhzV/Et8= 2022-09-03T06:57:01.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oYyqezSwLDCQG2Vx5VvUhzV/Et8=', 'Content-Length': '0'} 2022-09-03T06:57:01.198 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d19b82a12af6879-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.199 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.200 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.201 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.203 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T06:57:01.204 DEBUG:boto:Method: GET 2022-09-03T06:57:01.205 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T06:57:01.206 DEBUG:boto:Data: 2022-09-03T06:57:01.207 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.208 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.209 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.209 DEBUG:boto:Params: {} 2022-09-03T06:57:01.210 DEBUG:boto:Token: None 2022-09-03T06:57:01.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a?versionId=null 2022-09-03T06:57:01.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oYyqezSwLDCQG2Vx5VvUhzV/Et8= 2022-09-03T06:57:01.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oYyqezSwLDCQG2Vx5VvUhzV/Et8=', 'Content-Length': '0'} 2022-09-03T06:57:01.214 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000baaf17b20fbfbc21-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.215 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.216 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.216 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:57:01.217 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:57:01.217 DEBUG:boto:Method: GET 2022-09-03T06:57:01.218 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:57:01.219 DEBUG:boto:Data: 2022-09-03T06:57:01.219 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.221 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.222 DEBUG:boto:Params: {} 2022-09-03T06:57:01.223 DEBUG:boto:Token: None 2022-09-03T06:57:01.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.228 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002dd36e5f70997868-006312fabd-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.230 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.231 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.232 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.233 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:57:01.234 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:57:01.235 DEBUG:boto:Method: GET 2022-09-03T06:57:01.236 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:57:01.236 DEBUG:boto:Data: 2022-09-03T06:57:01.237 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.237 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.238 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.238 DEBUG:boto:Params: {} 2022-09-03T06:57:01.239 DEBUG:boto:Token: None 2022-09-03T06:57:01.239 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.243 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000507c0e25bf1ae223-006312fabd-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.244 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.245 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.245 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.246 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.246 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:57:01.247 DEBUG:boto:Data: 2022-09-03T06:57:01.248 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.249 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.249 DEBUG:boto:Params: {} 2022-09-03T06:57:01.250 DEBUG:boto:Token: None 2022-09-03T06:57:01.250 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a 2022-09-03T06:57:01.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n41JfVkGkmWh7+CYBY356T7ycr4= 2022-09-03T06:57:01.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n41JfVkGkmWh7+CYBY356T7ycr4=', 'Content-Length': '0'} 2022-09-03T06:57:01.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d8fd014a30e4ba3-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.254 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.255 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.255 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.256 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:57:01.256 DEBUG:boto:Data: 2022-09-03T06:57:01.257 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.257 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.258 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.258 DEBUG:boto:Params: {} 2022-09-03T06:57:01.259 DEBUG:boto:Token: None 2022-09-03T06:57:01.260 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a 2022-09-03T06:57:01.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n41JfVkGkmWh7+CYBY356T7ycr4= 2022-09-03T06:57:01.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n41JfVkGkmWh7+CYBY356T7ycr4=', 'Content-Length': '0'} 2022-09-03T06:57:01.263 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000493c699715801c5d-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.263 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T06:57:01.264 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.265 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.265 DEBUG:boto:Method: GET 2022-09-03T06:57:01.266 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:57:01.266 DEBUG:boto:Data: 2022-09-03T06:57:01.267 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.267 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.268 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.268 DEBUG:boto:Params: {} 2022-09-03T06:57:01.269 DEBUG:boto:Token: None 2022-09-03T06:57:01.269 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a 2022-09-03T06:57:01.270 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6+b04itArnK1QZ9HQCAls6GRgc4= 2022-09-03T06:57:01.271 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6+b04itArnK1QZ9HQCAls6GRgc4=', 'Content-Length': '0'} 2022-09-03T06:57:01.273 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1d3abe218270882-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.274 DEBUG:boto:path=/avysmw-72/a 2022-09-03T06:57:01.274 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T06:57:01.275 DEBUG:boto:Method: GET 2022-09-03T06:57:01.275 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T06:57:01.276 DEBUG:boto:Data: 2022-09-03T06:57:01.276 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.277 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.277 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.278 DEBUG:boto:Params: {} 2022-09-03T06:57:01.279 DEBUG:boto:Token: None 2022-09-03T06:57:01.280 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/a 2022-09-03T06:57:01.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6+b04itArnK1QZ9HQCAls6GRgc4= 2022-09-03T06:57:01.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6+b04itArnK1QZ9HQCAls6GRgc4=', 'Content-Length': '0'} 2022-09-03T06:57:01.286 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004977cc438a28e7f7-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.330 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.331 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.332 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:57:01.333 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:57:01.334 DEBUG:boto:Method: GET 2022-09-03T06:57:01.335 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:57:01.336 DEBUG:boto:Data: 2022-09-03T06:57:01.337 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.339 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.340 DEBUG:boto:Params: {} 2022-09-03T06:57:01.341 DEBUG:boto:Token: None 2022-09-03T06:57:01.343 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.349 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c42b77bd72d9eefc-006312fabd-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.350 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.351 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.352 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.353 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T06:57:01.354 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T06:57:01.355 DEBUG:boto:Method: GET 2022-09-03T06:57:01.356 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T06:57:01.357 DEBUG:boto:Data: 2022-09-03T06:57:01.358 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.359 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.359 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.360 DEBUG:boto:Params: {} 2022-09-03T06:57:01.362 DEBUG:boto:Token: None 2022-09-03T06:57:01.363 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.368 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000025b95cfbc22a3d30-006312fabd-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.370 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.371 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:57:01.372 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.373 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.374 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.375 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.376 DEBUG:boto:Method: GET 2022-09-03T06:57:01.377 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:01.378 DEBUG:boto:Data: 2022-09-03T06:57:01.378 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.380 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.381 DEBUG:boto:Params: {} 2022-09-03T06:57:01.382 DEBUG:boto:Token: None 2022-09-03T06:57:01.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:01.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:haUcRNoQqqiJgtueTv7v9CB2Ikg= 2022-09-03T06:57:01.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:haUcRNoQqqiJgtueTv7v9CB2Ikg=', 'Content-Length': '0'} 2022-09-03T06:57:01.387 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ef1e9bd7c8a4d23-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.388 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.389 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.390 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.391 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.392 DEBUG:boto:Method: GET 2022-09-03T06:57:01.392 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:01.393 DEBUG:boto:Data: 2022-09-03T06:57:01.393 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.394 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.394 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.395 DEBUG:boto:Params: {} 2022-09-03T06:57:01.395 DEBUG:boto:Token: None 2022-09-03T06:57:01.396 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:01.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:haUcRNoQqqiJgtueTv7v9CB2Ikg= 2022-09-03T06:57:01.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:haUcRNoQqqiJgtueTv7v9CB2Ikg=', 'Content-Length': '0'} 2022-09-03T06:57:01.400 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc177d0063c431c4-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.401 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.402 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.405 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.406 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.407 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.408 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:01.409 DEBUG:boto:Data: 2022-09-03T06:57:01.410 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.411 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.412 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.413 DEBUG:boto:Params: {} 2022-09-03T06:57:01.414 DEBUG:boto:Token: None 2022-09-03T06:57:01.415 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:01.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/x/RiMFATSvG/C0q1IQIo3WXJ9w= 2022-09-03T06:57:01.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/x/RiMFATSvG/C0q1IQIo3WXJ9w=', 'Content-Length': '0'} 2022-09-03T06:57:01.420 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001fa5652ac894a18a-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.421 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.422 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.423 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.424 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.425 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.426 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:01.426 DEBUG:boto:Data: 2022-09-03T06:57:01.427 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.428 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.428 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.429 DEBUG:boto:Params: {} 2022-09-03T06:57:01.429 DEBUG:boto:Token: None 2022-09-03T06:57:01.430 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:01.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/x/RiMFATSvG/C0q1IQIo3WXJ9w= 2022-09-03T06:57:01.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/x/RiMFATSvG/C0q1IQIo3WXJ9w=', 'Content-Length': '0'} 2022-09-03T06:57:01.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057d263530ba58fab-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.434 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:57:01.434 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.435 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.436 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.436 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.437 DEBUG:boto:Method: GET 2022-09-03T06:57:01.437 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:01.438 DEBUG:boto:Data: 2022-09-03T06:57:01.439 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.440 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.440 DEBUG:boto:Params: {} 2022-09-03T06:57:01.441 DEBUG:boto:Token: None 2022-09-03T06:57:01.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:01.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:haUcRNoQqqiJgtueTv7v9CB2Ikg= 2022-09-03T06:57:01.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:haUcRNoQqqiJgtueTv7v9CB2Ikg=', 'Content-Length': '0'} 2022-09-03T06:57:01.445 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c70e053fb6d39b2-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.446 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.447 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.448 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.449 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T06:57:01.450 DEBUG:boto:Method: GET 2022-09-03T06:57:01.451 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T06:57:01.451 DEBUG:boto:Data: 2022-09-03T06:57:01.452 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.452 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.453 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.454 DEBUG:boto:Params: {} 2022-09-03T06:57:01.455 DEBUG:boto:Token: None 2022-09-03T06:57:01.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b?versionId=null 2022-09-03T06:57:01.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:haUcRNoQqqiJgtueTv7v9CB2Ikg= 2022-09-03T06:57:01.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:haUcRNoQqqiJgtueTv7v9CB2Ikg=', 'Content-Length': '0'} 2022-09-03T06:57:01.461 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d26500728086f39-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.463 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.463 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.464 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:57:01.464 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:57:01.465 DEBUG:boto:Method: GET 2022-09-03T06:57:01.466 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:57:01.466 DEBUG:boto:Data: 2022-09-03T06:57:01.467 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.468 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.469 DEBUG:boto:Params: {} 2022-09-03T06:57:01.469 DEBUG:boto:Token: None 2022-09-03T06:57:01.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.474 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f98ee617c3109d0e-006312fabd-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.475 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.476 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.476 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.477 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:57:01.478 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:57:01.478 DEBUG:boto:Method: GET 2022-09-03T06:57:01.479 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:57:01.479 DEBUG:boto:Data: 2022-09-03T06:57:01.480 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.481 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.481 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.482 DEBUG:boto:Params: {} 2022-09-03T06:57:01.482 DEBUG:boto:Token: None 2022-09-03T06:57:01.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.487 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fa056e4e8d07e0c8-006312fabd-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.488 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.488 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.494 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.495 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.496 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:57:01.498 DEBUG:boto:Data: 2022-09-03T06:57:01.500 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.501 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.502 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.503 DEBUG:boto:Params: {} 2022-09-03T06:57:01.504 DEBUG:boto:Token: None 2022-09-03T06:57:01.505 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b 2022-09-03T06:57:01.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YDgoN4XhShTPRV2M2roYEcRJLEQ= 2022-09-03T06:57:01.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YDgoN4XhShTPRV2M2roYEcRJLEQ=', 'Content-Length': '0'} 2022-09-03T06:57:01.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001f980cebf092add-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.511 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.512 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.512 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.513 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:57:01.514 DEBUG:boto:Data: 2022-09-03T06:57:01.514 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.515 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.516 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.516 DEBUG:boto:Params: {} 2022-09-03T06:57:01.517 DEBUG:boto:Token: None 2022-09-03T06:57:01.518 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b 2022-09-03T06:57:01.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YDgoN4XhShTPRV2M2roYEcRJLEQ= 2022-09-03T06:57:01.521 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YDgoN4XhShTPRV2M2roYEcRJLEQ=', 'Content-Length': '0'} 2022-09-03T06:57:01.524 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022247e410b28681b-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.525 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T06:57:01.526 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.527 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.527 DEBUG:boto:Method: GET 2022-09-03T06:57:01.528 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:57:01.528 DEBUG:boto:Data: 2022-09-03T06:57:01.529 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.530 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.531 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.531 DEBUG:boto:Params: {} 2022-09-03T06:57:01.532 DEBUG:boto:Token: None 2022-09-03T06:57:01.533 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b 2022-09-03T06:57:01.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:twPcBN21IEL+wuhYUeXkBmh3KsI= 2022-09-03T06:57:01.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:twPcBN21IEL+wuhYUeXkBmh3KsI=', 'Content-Length': '0'} 2022-09-03T06:57:01.537 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000142bcf633fca339a-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.582 DEBUG:boto:path=/avysmw-72/b 2022-09-03T06:57:01.583 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T06:57:01.583 DEBUG:boto:Method: GET 2022-09-03T06:57:01.584 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T06:57:01.584 DEBUG:boto:Data: 2022-09-03T06:57:01.585 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.585 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.586 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.587 DEBUG:boto:Params: {} 2022-09-03T06:57:01.588 DEBUG:boto:Token: None 2022-09-03T06:57:01.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/b 2022-09-03T06:57:01.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:twPcBN21IEL+wuhYUeXkBmh3KsI= 2022-09-03T06:57:01.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:twPcBN21IEL+wuhYUeXkBmh3KsI=', 'Content-Length': '0'} 2022-09-03T06:57:01.591 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a8d36cf8fdeb56c-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.592 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.593 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.593 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:57:01.594 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:57:01.595 DEBUG:boto:Method: GET 2022-09-03T06:57:01.595 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:57:01.596 DEBUG:boto:Data: 2022-09-03T06:57:01.596 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.598 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.598 DEBUG:boto:Params: {} 2022-09-03T06:57:01.599 DEBUG:boto:Token: None 2022-09-03T06:57:01.600 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.604 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e4b598730984a6f9-006312fabd-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.605 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.605 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.606 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.607 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T06:57:01.607 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T06:57:01.608 DEBUG:boto:Method: GET 2022-09-03T06:57:01.608 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T06:57:01.609 DEBUG:boto:Data: 2022-09-03T06:57:01.609 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.610 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.610 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.611 DEBUG:boto:Params: {} 2022-09-03T06:57:01.611 DEBUG:boto:Token: None 2022-09-03T06:57:01.612 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.619 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000013d50eb6c21e3be4-006312fabd-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.620 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.621 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:57:01.621 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.623 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.623 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.624 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.624 DEBUG:boto:Method: GET 2022-09-03T06:57:01.625 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:01.625 DEBUG:boto:Data: 2022-09-03T06:57:01.626 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.627 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.628 DEBUG:boto:Params: {} 2022-09-03T06:57:01.628 DEBUG:boto:Token: None 2022-09-03T06:57:01.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:01.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RLONJEZik1DOjQLHbAfsHGV5mvc= 2022-09-03T06:57:01.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RLONJEZik1DOjQLHbAfsHGV5mvc=', 'Content-Length': '0'} 2022-09-03T06:57:01.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f148bd9bf81bf04-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.633 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.634 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.635 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.635 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.636 DEBUG:boto:Method: GET 2022-09-03T06:57:01.636 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:01.637 DEBUG:boto:Data: 2022-09-03T06:57:01.638 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.638 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.639 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.640 DEBUG:boto:Params: {} 2022-09-03T06:57:01.640 DEBUG:boto:Token: None 2022-09-03T06:57:01.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:01.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RLONJEZik1DOjQLHbAfsHGV5mvc= 2022-09-03T06:57:01.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RLONJEZik1DOjQLHbAfsHGV5mvc=', 'Content-Length': '0'} 2022-09-03T06:57:01.644 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079dfe82ace88e2af-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.690 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.692 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.693 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.695 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.696 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.696 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:01.697 DEBUG:boto:Data: 2022-09-03T06:57:01.698 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.700 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.701 DEBUG:boto:Params: {} 2022-09-03T06:57:01.702 DEBUG:boto:Token: None 2022-09-03T06:57:01.704 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:01.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8yYjArCqe1UgJmKFL8JTPQ4rlO0= 2022-09-03T06:57:01.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8yYjArCqe1UgJmKFL8JTPQ4rlO0=', 'Content-Length': '0'} 2022-09-03T06:57:01.707 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d89599f12914906-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.708 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.708 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.709 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.709 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.711 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.711 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:01.712 DEBUG:boto:Data: 2022-09-03T06:57:01.712 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.713 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.713 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.714 DEBUG:boto:Params: {} 2022-09-03T06:57:01.715 DEBUG:boto:Token: None 2022-09-03T06:57:01.715 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:01.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8yYjArCqe1UgJmKFL8JTPQ4rlO0= 2022-09-03T06:57:01.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8yYjArCqe1UgJmKFL8JTPQ4rlO0=', 'Content-Length': '0'} 2022-09-03T06:57:01.721 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ede2afefc3922fdf-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.722 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:57:01.723 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.723 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.724 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.725 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.725 DEBUG:boto:Method: GET 2022-09-03T06:57:01.726 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:01.726 DEBUG:boto:Data: 2022-09-03T06:57:01.727 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.729 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.730 DEBUG:boto:Params: {} 2022-09-03T06:57:01.732 DEBUG:boto:Token: None 2022-09-03T06:57:01.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:01.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RLONJEZik1DOjQLHbAfsHGV5mvc= 2022-09-03T06:57:01.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RLONJEZik1DOjQLHbAfsHGV5mvc=', 'Content-Length': '0'} 2022-09-03T06:57:01.736 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bddea787d219d542-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.737 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.738 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.739 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.741 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T06:57:01.742 DEBUG:boto:Method: GET 2022-09-03T06:57:01.743 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T06:57:01.744 DEBUG:boto:Data: 2022-09-03T06:57:01.745 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.746 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.746 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.747 DEBUG:boto:Params: {} 2022-09-03T06:57:01.748 DEBUG:boto:Token: None 2022-09-03T06:57:01.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c?versionId=null 2022-09-03T06:57:01.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RLONJEZik1DOjQLHbAfsHGV5mvc= 2022-09-03T06:57:01.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RLONJEZik1DOjQLHbAfsHGV5mvc=', 'Content-Length': '0'} 2022-09-03T06:57:01.753 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a9bd9ab11712206-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.754 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.755 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.755 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:57:01.756 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:57:01.757 DEBUG:boto:Method: GET 2022-09-03T06:57:01.758 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:57:01.759 DEBUG:boto:Data: 2022-09-03T06:57:01.760 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.761 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.762 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.763 DEBUG:boto:Params: {} 2022-09-03T06:57:01.764 DEBUG:boto:Token: None 2022-09-03T06:57:01.765 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.771 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cdc633ca67cefb98-006312fabd-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.772 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.774 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.775 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.777 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:57:01.778 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:57:01.779 DEBUG:boto:Method: GET 2022-09-03T06:57:01.779 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:57:01.780 DEBUG:boto:Data: 2022-09-03T06:57:01.781 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.781 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.782 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.783 DEBUG:boto:Params: {} 2022-09-03T06:57:01.785 DEBUG:boto:Token: None 2022-09-03T06:57:01.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.790 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e5568ad414125aa8-006312fabd-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.791 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.791 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.792 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.793 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.793 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:57:01.794 DEBUG:boto:Data: 2022-09-03T06:57:01.795 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.796 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.796 DEBUG:boto:Params: {} 2022-09-03T06:57:01.797 DEBUG:boto:Token: None 2022-09-03T06:57:01.798 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c 2022-09-03T06:57:01.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4IEnKJ6aQi96Tm1hcSgSg02Nc2Q= 2022-09-03T06:57:01.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4IEnKJ6aQi96Tm1hcSgSg02Nc2Q=', 'Content-Length': '0'} 2022-09-03T06:57:01.801 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6675f84ffc56352-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.802 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.803 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.803 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.804 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:57:01.805 DEBUG:boto:Data: 2022-09-03T06:57:01.805 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.806 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.807 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.807 DEBUG:boto:Params: {} 2022-09-03T06:57:01.808 DEBUG:boto:Token: None 2022-09-03T06:57:01.808 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c 2022-09-03T06:57:01.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4IEnKJ6aQi96Tm1hcSgSg02Nc2Q= 2022-09-03T06:57:01.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4IEnKJ6aQi96Tm1hcSgSg02Nc2Q=', 'Content-Length': '0'} 2022-09-03T06:57:01.813 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1a706d6b2cb7719-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.814 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T06:57:01.815 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.816 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.817 DEBUG:boto:Method: GET 2022-09-03T06:57:01.817 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:57:01.818 DEBUG:boto:Data: 2022-09-03T06:57:01.818 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.820 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.820 DEBUG:boto:Params: {} 2022-09-03T06:57:01.821 DEBUG:boto:Token: None 2022-09-03T06:57:01.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c 2022-09-03T06:57:01.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tCe1hGV+Y6fG9qVU153nRaKTHQQ= 2022-09-03T06:57:01.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tCe1hGV+Y6fG9qVU153nRaKTHQQ=', 'Content-Length': '0'} 2022-09-03T06:57:01.825 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003fe8cc1d99bf1f86-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.870 DEBUG:boto:path=/avysmw-72/c 2022-09-03T06:57:01.871 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T06:57:01.872 DEBUG:boto:Method: GET 2022-09-03T06:57:01.872 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T06:57:01.873 DEBUG:boto:Data: 2022-09-03T06:57:01.873 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.875 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.875 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.876 DEBUG:boto:Params: {} 2022-09-03T06:57:01.877 DEBUG:boto:Token: None 2022-09-03T06:57:01.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/c 2022-09-03T06:57:01.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tCe1hGV+Y6fG9qVU153nRaKTHQQ= 2022-09-03T06:57:01.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tCe1hGV+Y6fG9qVU153nRaKTHQQ=', 'Content-Length': '0'} 2022-09-03T06:57:01.882 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000561aea57a9b4245b-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.883 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.884 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.884 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:57:01.885 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:57:01.885 DEBUG:boto:Method: GET 2022-09-03T06:57:01.886 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:57:01.886 DEBUG:boto:Data: 2022-09-03T06:57:01.887 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.887 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.888 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.889 DEBUG:boto:Params: {} 2022-09-03T06:57:01.890 DEBUG:boto:Token: None 2022-09-03T06:57:01.891 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.895 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007141e6e6f636b030-006312fabd-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.896 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.897 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:01.898 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:01.898 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T06:57:01.899 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T06:57:01.899 DEBUG:boto:Method: GET 2022-09-03T06:57:01.900 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T06:57:01.900 DEBUG:boto:Data: 2022-09-03T06:57:01.902 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.902 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.903 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.904 DEBUG:boto:Params: {} 2022-09-03T06:57:01.904 DEBUG:boto:Token: None 2022-09-03T06:57:01.905 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/ 2022-09-03T06:57:01.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I= 2022-09-03T06:57:01.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nR1pzJkNeNwdZGZJ3gGkuiImE2I=', 'Content-Length': '0'} 2022-09-03T06:57:01.914 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091ed5ee3a62e397c-006312fabd-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.958 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:01.959 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:57:01.960 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:01.961 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:01.962 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:01.962 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:01.963 DEBUG:boto:Method: GET 2022-09-03T06:57:01.964 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:01.964 DEBUG:boto:Data: 2022-09-03T06:57:01.965 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.966 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.967 DEBUG:boto:Params: {} 2022-09-03T06:57:01.968 DEBUG:boto:Token: None 2022-09-03T06:57:01.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:01.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yhxH1VHVpVvwr3wrOibC4SOH1A4= 2022-09-03T06:57:01.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yhxH1VHVpVvwr3wrOibC4SOH1A4=', 'Content-Length': '0'} 2022-09-03T06:57:01.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d258b52e3268102-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.973 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:01.974 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:01.975 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:01.975 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:01.976 DEBUG:boto:Method: GET 2022-09-03T06:57:01.977 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:01.977 DEBUG:boto:Data: 2022-09-03T06:57:01.978 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.978 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:01.979 DEBUG:boto:Port: 8000 2022-09-03T06:57:01.979 DEBUG:boto:Params: {} 2022-09-03T06:57:01.980 DEBUG:boto:Token: None 2022-09-03T06:57:01.981 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:01.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yhxH1VHVpVvwr3wrOibC4SOH1A4= 2022-09-03T06:57:01.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yhxH1VHVpVvwr3wrOibC4SOH1A4=', 'Content-Length': '0'} 2022-09-03T06:57:01.984 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e48cfce8364247f-006312fabd-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.985 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:01.986 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:01.987 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:01.987 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:01.988 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.988 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:01.989 DEBUG:boto:Data: 2022-09-03T06:57:01.989 DEBUG:boto:Headers: {} 2022-09-03T06:57:01.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:01.990 DEBUG:boto:Port: 8001 2022-09-03T06:57:01.991 DEBUG:boto:Params: {} 2022-09-03T06:57:01.992 DEBUG:boto:Token: None 2022-09-03T06:57:01.992 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:01 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:01.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ImM0BcGM66iCep6DYKVFAsYkNCU= 2022-09-03T06:57:01.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ImM0BcGM66iCep6DYKVFAsYkNCU=', 'Content-Length': '0'} 2022-09-03T06:57:01.995 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002fd184ae89765a49-006312fabd-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:01.996 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:01.997 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:01.997 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:01.998 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:01.998 DEBUG:boto:Method: HEAD 2022-09-03T06:57:01.999 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:01.999 DEBUG:boto:Data: 2022-09-03T06:57:02.000 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.000 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:02.001 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.001 DEBUG:boto:Params: {} 2022-09-03T06:57:02.002 DEBUG:boto:Token: None 2022-09-03T06:57:02.002 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:02.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iX4Kyd0UGJnGaCHM8NnJx06K9cU= 2022-09-03T06:57:02.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iX4Kyd0UGJnGaCHM8NnJx06K9cU=', 'Content-Length': '0'} 2022-09-03T06:57:02.006 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063d4877b9cc50ad7-006312fabe-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.007 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:57:02.007 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:02.008 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:02.009 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:02.009 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:02.010 DEBUG:boto:Method: GET 2022-09-03T06:57:02.010 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:02.011 DEBUG:boto:Data: 2022-09-03T06:57:02.012 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.012 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:02.013 DEBUG:boto:Port: 8001 2022-09-03T06:57:02.013 DEBUG:boto:Params: {} 2022-09-03T06:57:02.014 DEBUG:boto:Token: None 2022-09-03T06:57:02.014 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:02.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0uQWXB42DBscCg68k2Oo0+jO018= 2022-09-03T06:57:02.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0uQWXB42DBscCg68k2Oo0+jO018=', 'Content-Length': '0'} 2022-09-03T06:57:02.018 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d584239f5670561b-006312fabe-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.019 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:02.020 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:02.020 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T06:57:02.021 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T06:57:02.021 DEBUG:boto:Method: GET 2022-09-03T06:57:02.022 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T06:57:02.023 DEBUG:boto:Data: 2022-09-03T06:57:02.023 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.024 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:02.024 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.025 DEBUG:boto:Params: {} 2022-09-03T06:57:02.025 DEBUG:boto:Token: None 2022-09-03T06:57:02.026 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/d?versionId=null 2022-09-03T06:57:02.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0uQWXB42DBscCg68k2Oo0+jO018= 2022-09-03T06:57:02.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0uQWXB42DBscCg68k2Oo0+jO018=', 'Content-Length': '0'} 2022-09-03T06:57:02.030 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab99df8bd7e1ffad-006312fabe-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.031 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:02.032 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:02.033 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:57:02.033 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:57:02.034 DEBUG:boto:Method: GET 2022-09-03T06:57:02.034 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:57:02.035 DEBUG:boto:Data: 2022-09-03T06:57:02.036 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.037 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:02.038 DEBUG:boto:Port: 8001 2022-09-03T06:57:02.038 DEBUG:boto:Params: {} 2022-09-03T06:57:02.039 DEBUG:boto:Token: None 2022-09-03T06:57:02.040 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/ 2022-09-03T06:57:02.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+3ViXXUcfk31CNs0Pdbn+Rr4OEg= 2022-09-03T06:57:02.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+3ViXXUcfk31CNs0Pdbn+Rr4OEg=', 'Content-Length': '0'} 2022-09-03T06:57:02.044 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000065c0d90c18d19215-006312fabe-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.045 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:02.046 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:02.047 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:02.047 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:57:02.048 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:57:02.048 DEBUG:boto:Method: GET 2022-09-03T06:57:02.049 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:57:02.050 DEBUG:boto:Data: 2022-09-03T06:57:02.050 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.051 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:02.051 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.052 DEBUG:boto:Params: {} 2022-09-03T06:57:02.052 DEBUG:boto:Token: None 2022-09-03T06:57:02.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/ 2022-09-03T06:57:02.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+3ViXXUcfk31CNs0Pdbn+Rr4OEg= 2022-09-03T06:57:02.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+3ViXXUcfk31CNs0Pdbn+Rr4OEg=', 'Content-Length': '0'} 2022-09-03T06:57:02.057 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b54321a9bfa48042-006312fabe-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.058 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:02.059 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:02.060 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:02.060 DEBUG:boto:Method: HEAD 2022-09-03T06:57:02.061 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:57:02.061 DEBUG:boto:Data: 2022-09-03T06:57:02.062 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:02.063 DEBUG:boto:Port: 8001 2022-09-03T06:57:02.063 DEBUG:boto:Params: {} 2022-09-03T06:57:02.064 DEBUG:boto:Token: None 2022-09-03T06:57:02.064 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/d 2022-09-03T06:57:02.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M3FVT+Aw5xy2QE3QhG5VnYdXidc= 2022-09-03T06:57:02.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M3FVT+Aw5xy2QE3QhG5VnYdXidc=', 'Content-Length': '0'} 2022-09-03T06:57:02.068 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000943ef4f4118e0e22-006312fabe-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.069 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:02.070 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:02.070 DEBUG:boto:Method: HEAD 2022-09-03T06:57:02.071 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:57:02.071 DEBUG:boto:Data: 2022-09-03T06:57:02.072 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.072 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:02.073 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.074 DEBUG:boto:Params: {} 2022-09-03T06:57:02.074 DEBUG:boto:Token: None 2022-09-03T06:57:02.075 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/d 2022-09-03T06:57:02.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M3FVT+Aw5xy2QE3QhG5VnYdXidc= 2022-09-03T06:57:02.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M3FVT+Aw5xy2QE3QhG5VnYdXidc=', 'Content-Length': '0'} 2022-09-03T06:57:02.078 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005da3cae5e22dbf21-006312fabe-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.079 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T06:57:02.079 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:02.080 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:02.080 DEBUG:boto:Method: GET 2022-09-03T06:57:02.081 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:57:02.082 DEBUG:boto:Data: 2022-09-03T06:57:02.082 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.083 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:02.083 DEBUG:boto:Port: 8001 2022-09-03T06:57:02.084 DEBUG:boto:Params: {} 2022-09-03T06:57:02.084 DEBUG:boto:Token: None 2022-09-03T06:57:02.085 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/d 2022-09-03T06:57:02.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YXnuFiCTxir1JVXXiI4zzBUwjoY= 2022-09-03T06:57:02.086 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YXnuFiCTxir1JVXXiI4zzBUwjoY=', 'Content-Length': '0'} 2022-09-03T06:57:02.088 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087e8c215e49bb34f-006312fabe-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.130 DEBUG:boto:path=/avysmw-72/d 2022-09-03T06:57:02.131 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T06:57:02.132 DEBUG:boto:Method: GET 2022-09-03T06:57:02.133 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T06:57:02.134 DEBUG:boto:Data: 2022-09-03T06:57:02.135 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.136 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:02.137 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.138 DEBUG:boto:Params: {} 2022-09-03T06:57:02.140 DEBUG:boto:Token: None 2022-09-03T06:57:02.141 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/d 2022-09-03T06:57:02.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YXnuFiCTxir1JVXXiI4zzBUwjoY= 2022-09-03T06:57:02.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YXnuFiCTxir1JVXXiI4zzBUwjoY=', 'Content-Length': '0'} 2022-09-03T06:57:02.149 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000094bae402a0b59452-006312fabe-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.150 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:02.151 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:02.152 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:57:02.152 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:57:02.153 DEBUG:boto:Method: GET 2022-09-03T06:57:02.154 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:57:02.155 DEBUG:boto:Data: 2022-09-03T06:57:02.155 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T06:57:02.156 DEBUG:boto:Port: 8001 2022-09-03T06:57:02.157 DEBUG:boto:Params: {} 2022-09-03T06:57:02.158 DEBUG:boto:Token: None 2022-09-03T06:57:02.159 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/ 2022-09-03T06:57:02.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+3ViXXUcfk31CNs0Pdbn+Rr4OEg= 2022-09-03T06:57:02.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+3ViXXUcfk31CNs0Pdbn+Rr4OEg=', 'Content-Length': '0'} 2022-09-03T06:57:02.165 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000794730a37c797e67-006312fabe-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.166 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:02.168 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:02.169 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:02.170 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T06:57:02.171 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T06:57:02.172 DEBUG:boto:Method: GET 2022-09-03T06:57:02.173 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T06:57:02.174 DEBUG:boto:Data: 2022-09-03T06:57:02.175 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.176 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T06:57:02.177 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.178 DEBUG:boto:Params: {} 2022-09-03T06:57:02.179 DEBUG:boto:Token: None 2022-09-03T06:57:02.181 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/ 2022-09-03T06:57:02.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+3ViXXUcfk31CNs0Pdbn+Rr4OEg= 2022-09-03T06:57:02.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+3ViXXUcfk31CNs0Pdbn+Rr4OEg=', 'Content-Length': '0'} 2022-09-03T06:57:02.188 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d012003fdb58386a-006312fabe-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.189 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T06:57:02.190 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-72 zones={test-zone3, test-zone2} 2022-09-03T06:57:02.190 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'avysmw-72', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:57:02.191 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'avysmw-72', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:57:02.191 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket avysmw-72 --num-shards 3 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:57:02.258 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:57:02.258 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:57:02.319 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T06:57:02.320 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-72 2022-09-03T06:57:02.325 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 4 2022-09-03T06:57:02.386 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'reshard', '--bucket', 'avysmw-72', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:57:02.387 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'reshard', '--bucket', 'avysmw-72', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:57:02.388 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket reshard --bucket avysmw-72 --num-shards 3 --yes-i-really-mean-it --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T06:57:02.438 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:57:02.439 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:57:02.630 INFO:teuthology.orchestra.run.smithi087.stdout:tenant: 2022-09-03T06:57:02.630 INFO:teuthology.orchestra.run.smithi087.stdout:bucket name: avysmw-72 2022-09-03T06:57:02.630 INFO:teuthology.orchestra.run.smithi087.stdout:total entries: 4 2022-09-03T06:57:02.677 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'avysmw-72', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:57:02.678 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'avysmw-72', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T06:57:02.678 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket avysmw-72 --num-shards 3 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T06:57:02.730 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:57:02.731 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:57:02.800 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T06:57:02.800 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-72 2022-09-03T06:57:02.801 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 4 2022-09-03T06:57:02.842 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:02.843 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:02.844 DEBUG:boto:Method: HEAD 2022-09-03T06:57:02.845 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:57:02.845 DEBUG:boto:Data: 2022-09-03T06:57:02.846 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.856 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:02.857 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.857 DEBUG:boto:Params: {} 2022-09-03T06:57:02.858 DEBUG:boto:Token: None 2022-09-03T06:57:02.858 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/ 2022-09-03T06:57:02.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ews1vGzMnc/g7hPi6GlbrTfziaE= 2022-09-03T06:57:02.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ews1vGzMnc/g7hPi6GlbrTfziaE=', 'Content-Length': '0'} 2022-09-03T06:57:02.862 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002ba09c35cc5d3802-006312fabe-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.863 DEBUG:boto:path=/avysmw-72/e 2022-09-03T06:57:02.863 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T06:57:02.864 DEBUG:boto:Method: PUT 2022-09-03T06:57:02.864 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T06:57:02.865 DEBUG:boto:Data: 2022-09-03T06:57:02.865 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:57:02.866 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:02.867 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.867 DEBUG:boto:Params: {} 2022-09-03T06:57:02.868 DEBUG:boto:Token: None 2022-09-03T06:57:02.869 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/e 2022-09-03T06:57:02.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rmCQlIioTLsYXWSF9AnPbE5xpsk= 2022-09-03T06:57:02.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rmCQlIioTLsYXWSF9AnPbE5xpsk='} 2022-09-03T06:57:02.877 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000003680ab9b9904b43-006312fabe-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.878 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:02.879 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:02.879 DEBUG:boto:Method: HEAD 2022-09-03T06:57:02.880 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:57:02.880 DEBUG:boto:Data: 2022-09-03T06:57:02.881 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:02.882 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.882 DEBUG:boto:Params: {} 2022-09-03T06:57:02.884 DEBUG:boto:Token: None 2022-09-03T06:57:02.885 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/ 2022-09-03T06:57:02.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ews1vGzMnc/g7hPi6GlbrTfziaE= 2022-09-03T06:57:02.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ews1vGzMnc/g7hPi6GlbrTfziaE=', 'Content-Length': '0'} 2022-09-03T06:57:02.888 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f05c88785498e504-006312fabe-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.889 DEBUG:boto:path=/avysmw-72/f 2022-09-03T06:57:02.890 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T06:57:02.890 DEBUG:boto:Method: PUT 2022-09-03T06:57:02.891 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T06:57:02.891 DEBUG:boto:Data: 2022-09-03T06:57:02.892 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:57:02.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:02.893 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.894 DEBUG:boto:Params: {} 2022-09-03T06:57:02.895 DEBUG:boto:Token: None 2022-09-03T06:57:02.895 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/f 2022-09-03T06:57:02.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zz3VgPRKelfiG/NKYG3iOPHbS8Q= 2022-09-03T06:57:02.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zz3VgPRKelfiG/NKYG3iOPHbS8Q='} 2022-09-03T06:57:02.903 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000e0a57c841e9142b8-006312fabe-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.903 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:02.904 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:02.904 DEBUG:boto:Method: HEAD 2022-09-03T06:57:02.905 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:57:02.905 DEBUG:boto:Data: 2022-09-03T06:57:02.906 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:02.907 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.907 DEBUG:boto:Params: {} 2022-09-03T06:57:02.908 DEBUG:boto:Token: None 2022-09-03T06:57:02.909 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/ 2022-09-03T06:57:02.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ews1vGzMnc/g7hPi6GlbrTfziaE= 2022-09-03T06:57:02.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ews1vGzMnc/g7hPi6GlbrTfziaE=', 'Content-Length': '0'} 2022-09-03T06:57:02.914 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000298f9e46818281ce-006312fabe-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.915 DEBUG:boto:path=/avysmw-72/g 2022-09-03T06:57:02.915 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T06:57:02.916 DEBUG:boto:Method: PUT 2022-09-03T06:57:02.916 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T06:57:02.917 DEBUG:boto:Data: 2022-09-03T06:57:02.918 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:57:02.918 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:02.919 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.919 DEBUG:boto:Params: {} 2022-09-03T06:57:02.920 DEBUG:boto:Token: None 2022-09-03T06:57:02.920 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/g 2022-09-03T06:57:02.921 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1h4y9LFGtxRbv02EU8E4nacvIqg= 2022-09-03T06:57:02.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1h4y9LFGtxRbv02EU8E4nacvIqg='} 2022-09-03T06:57:02.966 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000002b0138830e460e46-006312fabe-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.967 DEBUG:boto:path=/avysmw-72/ 2022-09-03T06:57:02.968 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T06:57:02.969 DEBUG:boto:Method: HEAD 2022-09-03T06:57:02.969 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T06:57:02.970 DEBUG:boto:Data: 2022-09-03T06:57:02.971 DEBUG:boto:Headers: {} 2022-09-03T06:57:02.971 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:02.974 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.975 DEBUG:boto:Params: {} 2022-09-03T06:57:02.975 DEBUG:boto:Token: None 2022-09-03T06:57:02.976 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/ 2022-09-03T06:57:02.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ews1vGzMnc/g7hPi6GlbrTfziaE= 2022-09-03T06:57:02.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ews1vGzMnc/g7hPi6GlbrTfziaE=', 'Content-Length': '0'} 2022-09-03T06:57:02.979 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '21'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009fbad7ce48737c93-006312fabe-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.980 DEBUG:boto:path=/avysmw-72/h 2022-09-03T06:57:02.981 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T06:57:02.982 DEBUG:boto:Method: PUT 2022-09-03T06:57:02.983 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T06:57:02.983 DEBUG:boto:Data: 2022-09-03T06:57:02.984 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T06:57:02.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T06:57:02.985 DEBUG:boto:Port: 8000 2022-09-03T06:57:02.985 DEBUG:boto:Params: {} 2022-09-03T06:57:02.986 DEBUG:boto:Token: None 2022-09-03T06:57:02.987 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 06:57:02 GMT /avysmw-72/h 2022-09-03T06:57:02.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:081qPTQJaqn4c+kBZN/mZMcF+1k= 2022-09-03T06:57:02.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 06:57:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:081qPTQJaqn4c+kBZN/mZMcF+1k='} 2022-09-03T06:57:02.994 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000fb10492fe82c8b89-006312fabe-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T06:57:02.995 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-72 2022-09-03T06:57:02.995 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:57:02.996 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:57:02.996 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:57:03.061 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:57:03.061 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:57:03.113 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:03.111+0000 7fb023927600 1 waiting to reach latest gen 2, on 0.. 2022-09-03T06:57:08.114 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:08.112+0000 7fb023927600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:57:13.114 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:13.112+0000 7fb023927600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:57:18.114 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:18.112+0000 7fb023927600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:57:23.115 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:23.113+0000 7fb023927600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:57:28.115 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:28.113+0000 7fb023927600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:57:33.116 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:33.114+0000 7fb023927600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:57:38.118 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:38.114+0000 7fb023927600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:57:43.117 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:43.115+0000 7fb023927600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T06:57:48.118 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:48.116+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:57:48.119 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:57:48.119 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:57:53.119 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:53.117+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:57:53.120 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:57:53.120 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:57:58.119 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:57:58.117+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:57:58.120 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:57:58.120 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:03.120 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:03.118+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:03.121 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:58:03.122 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:08.121 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:08.119+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:08.121 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:58:08.122 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:13.121 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:13.120+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:13.122 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:58:13.122 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:18.123 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:18.120+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:18.123 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:58:18.124 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:23.123 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:23.121+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:23.123 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , ] 2022-09-03T06:58:23.124 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:28.124 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:28.122+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:28.125 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.178.5] 2022-09-03T06:58:28.125 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:33.124 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:33.122+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:33.125 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.178.5] 2022-09-03T06:58:33.125 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:38.125 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:38.123+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:38.126 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.178.5] 2022-09-03T06:58:38.126 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:43.126 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:43.124+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:43.132 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.178.5] 2022-09-03T06:58:43.133 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:48.132 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:48.129+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:48.133 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:48.133 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:53.132 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:53.130+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:53.133 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:53.133 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:58.132 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:58:58.130+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:58:58.133 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:58:58.134 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:03.133 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:03.131+0000 7fb023927600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:03.135 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:03.135 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:08.136 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:08.132+0000 7fb023927600 1 bucket sync caught up with source: 2022-09-03T06:59:08.137 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:08.138 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:08.138 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:08.132+0000 7fb023927600 0 bucket checkpoint complete 2022-09-03T06:59:08.163 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-72 2022-09-03T06:59:08.166 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:59:08.167 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:59:08.169 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T06:59:08.237 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:59:08.239 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:59:08.292 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:59:08.290+0000 7fb31c54f600 1 bucket sync caught up with source: 2022-09-03T06:59:08.292 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:08.292 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:08.293 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T06:59:08.290+0000 7fb31c54f600 0 bucket checkpoint complete 2022-09-03T06:59:08.308 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-72 2022-09-03T06:59:08.310 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:59:08.311 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T06:59:08.312 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T06:59:08.404 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T06:59:08.405 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T06:59:08.461 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:08.459+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:08.462 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:08.462 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:13.462 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:13.460+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:13.463 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:13.463 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:18.463 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:18.461+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:18.463 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:18.463 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:23.463 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:23.461+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:23.463 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:23.464 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:28.470 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:28.462+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:28.470 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:28.471 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:33.465 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:33.463+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:33.465 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:33.466 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:38.465 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:38.463+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:38.466 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:38.466 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:43.467 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:43.465+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:43.468 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:43.468 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:48.472 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:48.465+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:48.473 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:48.473 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:53.468 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:53.465+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:53.468 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:53.469 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:58.468 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T06:59:58.466+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T06:59:58.469 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T06:59:58.469 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:03.469 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:03.467+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T07:00:03.469 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:03.470 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:08.470 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:08.467+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T07:00:08.471 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:08.471 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:13.470 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:13.468+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T07:00:13.471 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:13.471 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:18.471 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:18.469+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T07:00:18.471 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:18.472 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:23.471 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:23.469+0000 7f55c9d22600 1 waiting for incremental sync to catch up: 2022-09-03T07:00:23.472 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:23.473 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:28.472 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:28.470+0000 7f55c9d22600 1 bucket sync caught up with source: 2022-09-03T07:00:28.473 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:28.473 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.77.5, 00000000002.73.5, 00000000002.178.5] 2022-09-03T07:00:28.474 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:28.470+0000 7f55c9d22600 0 bucket checkpoint complete 2022-09-03T07:00:28.502 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-72 2022-09-03T07:00:28.504 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:00:28.505 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:00:28.505 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:00:28.594 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:28.594 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:28.654 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:00:28.653+0000 7f64aa645600 1 bucket sync caught up with source: 2022-09-03T07:00:28.655 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000003.224.5, 00000000002.372.5, 00000000002.259.5] 2022-09-03T07:00:28.655 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000003.224.5, 00000000002.372.5, 00000000002.259.5] 2022-09-03T07:00:28.655 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:00:28.653+0000 7f64aa645600 0 bucket checkpoint complete 2022-09-03T07:00:28.678 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-72 2022-09-03T07:00:28.679 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:00:28.680 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:00:28.680 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:00:28.739 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:28.740 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:28.795 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:00:28.793+0000 7f4f4db4b600 1 bucket sync caught up with source: 2022-09-03T07:00:28.795 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000003.224.5, 00000000002.372.5, 00000000002.259.5] 2022-09-03T07:00:28.795 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000003.224.5, 00000000002.372.5, 00000000002.259.5] 2022-09-03T07:00:28.796 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:00:28.793+0000 7f4f4db4b600 0 bucket checkpoint complete 2022-09-03T07:00:28.812 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-72 2022-09-03T07:00:28.813 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-72 2022-09-03T07:00:28.814 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:00:28.815 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:00:28.816 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:00:28.875 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:28.876 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:28.918 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:00:28.916+0000 7f014c5b0600 1 bucket sync caught up with source: 2022-09-03T07:00:28.918 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.95.5, , , 00000000001.56.5, 00000000001.193.5, , , 00000000002.283.5, , 00000000002.97.5] 2022-09-03T07:00:28.918 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.95.5, , , 00000000001.56.5, 00000000001.193.5, , , 00000000002.283.5, , 00000000002.97.5] 2022-09-03T07:00:28.919 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:00:28.916+0000 7f014c5b0600 0 bucket checkpoint complete 2022-09-03T07:00:28.933 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-72 2022-09-03T07:00:28.933 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:00:28.934 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-72', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:00:28.935 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-72 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:00:29.028 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:29.029 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:29.093 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:29.091+0000 7fc222217600 1 bucket sync caught up with source: 2022-09-03T07:00:29.093 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.95.5, , , 00000000001.56.5, 00000000001.193.5, , , 00000000002.283.5, , 00000000002.97.5] 2022-09-03T07:00:29.094 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.95.5, , , 00000000001.56.5, 00000000001.193.5, , , 00000000002.283.5, , 00000000002.97.5] 2022-09-03T07:00:29.095 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:29.091+0000 7fc222217600 0 bucket checkpoint complete 2022-09-03T07:00:29.114 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-72 2022-09-03T07:00:29.114 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-72 zones={test-zone2, test-zone1} 2022-09-03T07:00:29.116 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.116 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.117 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.118 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T07:00:29.118 DEBUG:boto:Data: 2022-09-03T07:00:29.119 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.119 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.120 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.120 DEBUG:boto:Params: {} 2022-09-03T07:00:29.122 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:00:29.123 DEBUG:boto:Token: None 2022-09-03T07:00:29.124 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s8sWjzEKi7DHx7KK2+U6/qSGLfE= 2022-09-03T07:00:29.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s8sWjzEKi7DHx7KK2+U6/qSGLfE=', 'Content-Length': '0'} 2022-09-03T07:00:29.131 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000061f3ef6aa4086100-006312fb8d-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.132 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.133 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.134 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.134 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T07:00:29.135 DEBUG:boto:Data: 2022-09-03T07:00:29.136 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.136 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.137 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.138 DEBUG:boto:Params: {} 2022-09-03T07:00:29.139 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:00:29.139 DEBUG:boto:Token: None 2022-09-03T07:00:29.140 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s8sWjzEKi7DHx7KK2+U6/qSGLfE= 2022-09-03T07:00:29.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s8sWjzEKi7DHx7KK2+U6/qSGLfE=', 'Content-Length': '0'} 2022-09-03T07:00:29.145 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008fb50b122f01c826-006312fb8d-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.146 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:00:29.148 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.148 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.150 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.151 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.152 DEBUG:boto:Method: GET 2022-09-03T07:00:29.153 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.154 DEBUG:boto:Data: 2022-09-03T07:00:29.155 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.156 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.156 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.157 DEBUG:boto:Params: {} 2022-09-03T07:00:29.158 DEBUG:boto:Token: None 2022-09-03T07:00:29.159 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/?versions 2022-09-03T07:00:29.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2HuUxYCqh20N6xvnPflpuknuCLA= 2022-09-03T07:00:29.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2HuUxYCqh20N6xvnPflpuknuCLA=', 'Content-Length': '0'} 2022-09-03T07:00:29.163 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f77e6b8f3b9955fb-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.164 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.167 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:00:29.168 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:00:29.168 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:00:29.169 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:00:29.169 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:00:29.170 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:00:29.170 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:00:29.171 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:00:29.171 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:00:29.172 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.172 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.173 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.173 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.174 DEBUG:boto:Method: GET 2022-09-03T07:00:29.175 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.175 DEBUG:boto:Data: 2022-09-03T07:00:29.176 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.177 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.177 DEBUG:boto:Params: {} 2022-09-03T07:00:29.178 DEBUG:boto:Token: None 2022-09-03T07:00:29.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/?versions 2022-09-03T07:00:29.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2HuUxYCqh20N6xvnPflpuknuCLA= 2022-09-03T07:00:29.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2HuUxYCqh20N6xvnPflpuknuCLA=', 'Content-Length': '0'} 2022-09-03T07:00:29.182 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003244273ee7c94c11-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.183 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.184 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:00:29.185 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:00:29.185 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:00:29.186 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:00:29.186 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:00:29.187 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:00:29.187 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:00:29.188 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:00:29.188 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.189 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.189 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.190 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.190 DEBUG:boto:Method: GET 2022-09-03T07:00:29.191 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.192 DEBUG:boto:Data: 2022-09-03T07:00:29.192 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.193 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.193 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.194 DEBUG:boto:Params: {} 2022-09-03T07:00:29.194 DEBUG:boto:Token: None 2022-09-03T07:00:29.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/?versions 2022-09-03T07:00:29.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2HuUxYCqh20N6xvnPflpuknuCLA= 2022-09-03T07:00:29.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2HuUxYCqh20N6xvnPflpuknuCLA=', 'Content-Length': '0'} 2022-09-03T07:00:29.199 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028ad4194eaa4e117-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.199 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.201 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.201 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.202 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.202 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.203 DEBUG:boto:Method: GET 2022-09-03T07:00:29.203 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:29.204 DEBUG:boto:Data: 2022-09-03T07:00:29.205 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.205 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.207 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.208 DEBUG:boto:Params: {} 2022-09-03T07:00:29.209 DEBUG:boto:Token: None 2022-09-03T07:00:29.210 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/?versions 2022-09-03T07:00:29.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2HuUxYCqh20N6xvnPflpuknuCLA= 2022-09-03T07:00:29.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2HuUxYCqh20N6xvnPflpuknuCLA=', 'Content-Length': '0'} 2022-09-03T07:00:29.214 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c97797966f40140-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.215 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:00:29.217 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.217 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.218 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.219 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.219 DEBUG:boto:Method: GET 2022-09-03T07:00:29.220 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:29.220 DEBUG:boto:Data: 2022-09-03T07:00:29.221 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.222 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.222 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.223 DEBUG:boto:Params: {} 2022-09-03T07:00:29.223 DEBUG:boto:Token: None 2022-09-03T07:00:29.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:29.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zYXHKyTeittLe0jFrlH9YepC/iM= 2022-09-03T07:00:29.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zYXHKyTeittLe0jFrlH9YepC/iM=', 'Content-Length': '0'} 2022-09-03T07:00:29.229 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d10d5864c21aa31e-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.230 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.230 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.231 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.231 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.232 DEBUG:boto:Method: GET 2022-09-03T07:00:29.233 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:29.233 DEBUG:boto:Data: 2022-09-03T07:00:29.234 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.235 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.236 DEBUG:boto:Params: {} 2022-09-03T07:00:29.236 DEBUG:boto:Token: None 2022-09-03T07:00:29.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:29.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zYXHKyTeittLe0jFrlH9YepC/iM= 2022-09-03T07:00:29.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zYXHKyTeittLe0jFrlH9YepC/iM=', 'Content-Length': '0'} 2022-09-03T07:00:29.242 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000028eb7ffc5691a01c-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.244 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.245 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.246 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.247 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.248 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.249 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:29.250 DEBUG:boto:Data: 2022-09-03T07:00:29.251 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.252 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.253 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.254 DEBUG:boto:Params: {} 2022-09-03T07:00:29.257 DEBUG:boto:Token: None 2022-09-03T07:00:29.257 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:29.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3aDDWzKWtDhotdBEwNla5qgV/z8= 2022-09-03T07:00:29.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3aDDWzKWtDhotdBEwNla5qgV/z8=', 'Content-Length': '0'} 2022-09-03T07:00:29.261 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f506baec459cdb55-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.262 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.263 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.263 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.264 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.265 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.266 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:29.267 DEBUG:boto:Data: 2022-09-03T07:00:29.268 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.269 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.270 DEBUG:boto:Params: {} 2022-09-03T07:00:29.270 DEBUG:boto:Token: None 2022-09-03T07:00:29.272 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:29.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3aDDWzKWtDhotdBEwNla5qgV/z8= 2022-09-03T07:00:29.274 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3aDDWzKWtDhotdBEwNla5qgV/z8=', 'Content-Length': '0'} 2022-09-03T07:00:29.277 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010bfeec78eab6eff-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.278 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:00:29.278 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.279 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.280 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.281 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.282 DEBUG:boto:Method: GET 2022-09-03T07:00:29.282 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:29.283 DEBUG:boto:Data: 2022-09-03T07:00:29.284 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.285 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.286 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.286 DEBUG:boto:Params: {} 2022-09-03T07:00:29.287 DEBUG:boto:Token: None 2022-09-03T07:00:29.288 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:29.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zYXHKyTeittLe0jFrlH9YepC/iM= 2022-09-03T07:00:29.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zYXHKyTeittLe0jFrlH9YepC/iM=', 'Content-Length': '0'} 2022-09-03T07:00:29.292 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fef07ec6c2685229-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.293 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.294 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.295 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.296 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:29.297 DEBUG:boto:Method: GET 2022-09-03T07:00:29.298 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:29.298 DEBUG:boto:Data: 2022-09-03T07:00:29.299 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.300 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.301 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.302 DEBUG:boto:Params: {} 2022-09-03T07:00:29.303 DEBUG:boto:Token: None 2022-09-03T07:00:29.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:29.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zYXHKyTeittLe0jFrlH9YepC/iM= 2022-09-03T07:00:29.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zYXHKyTeittLe0jFrlH9YepC/iM=', 'Content-Length': '0'} 2022-09-03T07:00:29.307 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002f90fedcf2567d9-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.308 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.309 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.310 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:29.310 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:29.311 DEBUG:boto:Method: GET 2022-09-03T07:00:29.312 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:29.312 DEBUG:boto:Data: 2022-09-03T07:00:29.313 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.314 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.314 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.315 DEBUG:boto:Params: {} 2022-09-03T07:00:29.316 DEBUG:boto:Token: None 2022-09-03T07:00:29.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.320 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad01828a483f1889-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.321 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.322 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.323 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.323 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:29.324 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:29.325 DEBUG:boto:Method: GET 2022-09-03T07:00:29.325 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:29.326 DEBUG:boto:Data: 2022-09-03T07:00:29.327 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.327 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.328 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.329 DEBUG:boto:Params: {} 2022-09-03T07:00:29.329 DEBUG:boto:Token: None 2022-09-03T07:00:29.330 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.333 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a76c67f4deab747-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.334 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.335 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.335 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.336 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.336 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:29.337 DEBUG:boto:Data: 2022-09-03T07:00:29.337 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.338 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.338 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.339 DEBUG:boto:Params: {} 2022-09-03T07:00:29.340 DEBUG:boto:Token: None 2022-09-03T07:00:29.340 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a 2022-09-03T07:00:29.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+zNArSL9OW4Myfbtqs3OaHHHOeU= 2022-09-03T07:00:29.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+zNArSL9OW4Myfbtqs3OaHHHOeU=', 'Content-Length': '0'} 2022-09-03T07:00:29.344 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa2b4324f3e12a70-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.345 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.345 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.346 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.346 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:29.347 DEBUG:boto:Data: 2022-09-03T07:00:29.347 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.348 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.349 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.349 DEBUG:boto:Params: {} 2022-09-03T07:00:29.350 DEBUG:boto:Token: None 2022-09-03T07:00:29.351 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a 2022-09-03T07:00:29.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+zNArSL9OW4Myfbtqs3OaHHHOeU= 2022-09-03T07:00:29.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+zNArSL9OW4Myfbtqs3OaHHHOeU=', 'Content-Length': '0'} 2022-09-03T07:00:29.355 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df07f11e08b1c394-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.356 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:00:29.356 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.357 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.358 DEBUG:boto:Method: GET 2022-09-03T07:00:29.358 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:29.359 DEBUG:boto:Data: 2022-09-03T07:00:29.359 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.360 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.360 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.361 DEBUG:boto:Params: {} 2022-09-03T07:00:29.362 DEBUG:boto:Token: None 2022-09-03T07:00:29.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a 2022-09-03T07:00:29.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oKSHkyvGszMiLPcweSe6V0vv5BU= 2022-09-03T07:00:29.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oKSHkyvGszMiLPcweSe6V0vv5BU=', 'Content-Length': '0'} 2022-09-03T07:00:29.366 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f4baa3d03fd7280-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.367 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:29.367 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:29.368 DEBUG:boto:Method: GET 2022-09-03T07:00:29.368 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:29.369 DEBUG:boto:Data: 2022-09-03T07:00:29.369 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.370 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.371 DEBUG:boto:Params: {} 2022-09-03T07:00:29.372 DEBUG:boto:Token: None 2022-09-03T07:00:29.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/a 2022-09-03T07:00:29.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oKSHkyvGszMiLPcweSe6V0vv5BU= 2022-09-03T07:00:29.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oKSHkyvGszMiLPcweSe6V0vv5BU=', 'Content-Length': '0'} 2022-09-03T07:00:29.376 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005df81fb121f4b3a-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.419 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.420 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.420 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:29.422 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:29.423 DEBUG:boto:Method: GET 2022-09-03T07:00:29.423 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:29.424 DEBUG:boto:Data: 2022-09-03T07:00:29.426 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.427 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.427 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.428 DEBUG:boto:Params: {} 2022-09-03T07:00:29.431 DEBUG:boto:Token: None 2022-09-03T07:00:29.432 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.436 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c25db53c22f29de-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.437 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.439 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.439 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.440 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:29.441 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:29.441 DEBUG:boto:Method: GET 2022-09-03T07:00:29.442 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:29.443 DEBUG:boto:Data: 2022-09-03T07:00:29.443 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.444 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.444 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.445 DEBUG:boto:Params: {} 2022-09-03T07:00:29.446 DEBUG:boto:Token: None 2022-09-03T07:00:29.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.451 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c6d181b2adf79ab-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.452 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.453 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:00:29.454 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.455 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.456 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.457 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.458 DEBUG:boto:Method: GET 2022-09-03T07:00:29.459 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:29.460 DEBUG:boto:Data: 2022-09-03T07:00:29.461 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.463 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.464 DEBUG:boto:Params: {} 2022-09-03T07:00:29.465 DEBUG:boto:Token: None 2022-09-03T07:00:29.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:29.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IPIv6/RPDSixOsaFG7n3BmsV90I= 2022-09-03T07:00:29.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IPIv6/RPDSixOsaFG7n3BmsV90I=', 'Content-Length': '0'} 2022-09-03T07:00:29.470 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000201be6350ede4e8d-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.471 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.472 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.472 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.473 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.474 DEBUG:boto:Method: GET 2022-09-03T07:00:29.475 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:29.476 DEBUG:boto:Data: 2022-09-03T07:00:29.477 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.477 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.478 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.480 DEBUG:boto:Params: {} 2022-09-03T07:00:29.481 DEBUG:boto:Token: None 2022-09-03T07:00:29.482 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:29.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IPIv6/RPDSixOsaFG7n3BmsV90I= 2022-09-03T07:00:29.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IPIv6/RPDSixOsaFG7n3BmsV90I=', 'Content-Length': '0'} 2022-09-03T07:00:29.486 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dac08d3b43f1b692-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.487 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.488 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.488 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.489 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.490 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.491 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:29.491 DEBUG:boto:Data: 2022-09-03T07:00:29.492 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.492 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.493 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.493 DEBUG:boto:Params: {} 2022-09-03T07:00:29.494 DEBUG:boto:Token: None 2022-09-03T07:00:29.495 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:29.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lIobNxr7kHQt51HttgtQU7e60vA= 2022-09-03T07:00:29.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lIobNxr7kHQt51HttgtQU7e60vA=', 'Content-Length': '0'} 2022-09-03T07:00:29.498 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f31c1affa7f6e85-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.499 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.500 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.501 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.501 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.502 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.503 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:29.504 DEBUG:boto:Data: 2022-09-03T07:00:29.504 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.505 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.506 DEBUG:boto:Params: {} 2022-09-03T07:00:29.506 DEBUG:boto:Token: None 2022-09-03T07:00:29.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:29.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lIobNxr7kHQt51HttgtQU7e60vA= 2022-09-03T07:00:29.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lIobNxr7kHQt51HttgtQU7e60vA=', 'Content-Length': '0'} 2022-09-03T07:00:29.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065576f8609597eed-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.511 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:00:29.511 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.512 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.513 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.513 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.514 DEBUG:boto:Method: GET 2022-09-03T07:00:29.514 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:29.515 DEBUG:boto:Data: 2022-09-03T07:00:29.516 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.516 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.517 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.518 DEBUG:boto:Params: {} 2022-09-03T07:00:29.518 DEBUG:boto:Token: None 2022-09-03T07:00:29.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:29.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IPIv6/RPDSixOsaFG7n3BmsV90I= 2022-09-03T07:00:29.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IPIv6/RPDSixOsaFG7n3BmsV90I=', 'Content-Length': '0'} 2022-09-03T07:00:29.522 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3e7155b60bd16bc-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.523 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.523 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.524 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.524 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:29.525 DEBUG:boto:Method: GET 2022-09-03T07:00:29.525 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:29.526 DEBUG:boto:Data: 2022-09-03T07:00:29.527 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.527 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.528 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.528 DEBUG:boto:Params: {} 2022-09-03T07:00:29.529 DEBUG:boto:Token: None 2022-09-03T07:00:29.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:29.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IPIv6/RPDSixOsaFG7n3BmsV90I= 2022-09-03T07:00:29.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IPIv6/RPDSixOsaFG7n3BmsV90I=', 'Content-Length': '0'} 2022-09-03T07:00:29.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dd54a507caee3fd4-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.574 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.575 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.576 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:29.576 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:29.577 DEBUG:boto:Method: GET 2022-09-03T07:00:29.577 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:29.578 DEBUG:boto:Data: 2022-09-03T07:00:29.578 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.579 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.579 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.580 DEBUG:boto:Params: {} 2022-09-03T07:00:29.581 DEBUG:boto:Token: None 2022-09-03T07:00:29.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.589 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1e2fae65203e1e8-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.590 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.592 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.593 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.594 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:29.595 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:29.596 DEBUG:boto:Method: GET 2022-09-03T07:00:29.597 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:29.598 DEBUG:boto:Data: 2022-09-03T07:00:29.599 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.601 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.601 DEBUG:boto:Params: {} 2022-09-03T07:00:29.602 DEBUG:boto:Token: None 2022-09-03T07:00:29.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.606 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6e21d86727b14bf-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.607 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.608 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.608 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.609 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.609 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:29.610 DEBUG:boto:Data: 2022-09-03T07:00:29.610 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.611 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.612 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.612 DEBUG:boto:Params: {} 2022-09-03T07:00:29.613 DEBUG:boto:Token: None 2022-09-03T07:00:29.613 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b 2022-09-03T07:00:29.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hw7skn5dWUlfVPCIFzhMPswUpyM= 2022-09-03T07:00:29.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hw7skn5dWUlfVPCIFzhMPswUpyM=', 'Content-Length': '0'} 2022-09-03T07:00:29.617 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e976e6f6549266b7-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.618 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.618 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.619 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.619 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:29.620 DEBUG:boto:Data: 2022-09-03T07:00:29.620 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.621 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.621 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.622 DEBUG:boto:Params: {} 2022-09-03T07:00:29.622 DEBUG:boto:Token: None 2022-09-03T07:00:29.623 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b 2022-09-03T07:00:29.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hw7skn5dWUlfVPCIFzhMPswUpyM= 2022-09-03T07:00:29.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hw7skn5dWUlfVPCIFzhMPswUpyM=', 'Content-Length': '0'} 2022-09-03T07:00:29.626 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e36abd0bdb9b433a-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.627 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:00:29.628 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.629 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.630 DEBUG:boto:Method: GET 2022-09-03T07:00:29.631 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:29.631 DEBUG:boto:Data: 2022-09-03T07:00:29.632 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.632 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.633 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.633 DEBUG:boto:Params: {} 2022-09-03T07:00:29.634 DEBUG:boto:Token: None 2022-09-03T07:00:29.635 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b 2022-09-03T07:00:29.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3RjQcqaM8PwSnGbIKYUi2sldDhI= 2022-09-03T07:00:29.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3RjQcqaM8PwSnGbIKYUi2sldDhI=', 'Content-Length': '0'} 2022-09-03T07:00:29.638 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dd547e33370448e7-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.639 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:29.639 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:29.640 DEBUG:boto:Method: GET 2022-09-03T07:00:29.641 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:29.641 DEBUG:boto:Data: 2022-09-03T07:00:29.642 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.643 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.643 DEBUG:boto:Params: {} 2022-09-03T07:00:29.644 DEBUG:boto:Token: None 2022-09-03T07:00:29.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/b 2022-09-03T07:00:29.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3RjQcqaM8PwSnGbIKYUi2sldDhI= 2022-09-03T07:00:29.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3RjQcqaM8PwSnGbIKYUi2sldDhI=', 'Content-Length': '0'} 2022-09-03T07:00:29.648 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046e81d94463c981b-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.694 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.696 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.696 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:29.697 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:29.698 DEBUG:boto:Method: GET 2022-09-03T07:00:29.699 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:29.700 DEBUG:boto:Data: 2022-09-03T07:00:29.701 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.703 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.704 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.705 DEBUG:boto:Params: {} 2022-09-03T07:00:29.706 DEBUG:boto:Token: None 2022-09-03T07:00:29.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.711 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031e063b93de57694-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.712 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.713 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.713 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.714 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:29.714 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:29.715 DEBUG:boto:Method: GET 2022-09-03T07:00:29.716 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:29.716 DEBUG:boto:Data: 2022-09-03T07:00:29.717 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.718 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.718 DEBUG:boto:Params: {} 2022-09-03T07:00:29.719 DEBUG:boto:Token: None 2022-09-03T07:00:29.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.725 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000078f58a5a9cc8539f-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.725 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.726 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:00:29.727 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.727 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.728 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.729 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.730 DEBUG:boto:Method: GET 2022-09-03T07:00:29.730 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:29.731 DEBUG:boto:Data: 2022-09-03T07:00:29.731 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.732 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.732 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.733 DEBUG:boto:Params: {} 2022-09-03T07:00:29.734 DEBUG:boto:Token: None 2022-09-03T07:00:29.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:29.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AZ6n9ClivvUqP4OXcQ6JR6Cjd4U= 2022-09-03T07:00:29.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AZ6n9ClivvUqP4OXcQ6JR6Cjd4U=', 'Content-Length': '0'} 2022-09-03T07:00:29.738 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0c902a6f67a0264-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.739 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.740 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.741 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.742 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.743 DEBUG:boto:Method: GET 2022-09-03T07:00:29.744 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:29.745 DEBUG:boto:Data: 2022-09-03T07:00:29.746 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.748 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.749 DEBUG:boto:Params: {} 2022-09-03T07:00:29.750 DEBUG:boto:Token: None 2022-09-03T07:00:29.751 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:29.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AZ6n9ClivvUqP4OXcQ6JR6Cjd4U= 2022-09-03T07:00:29.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AZ6n9ClivvUqP4OXcQ6JR6Cjd4U=', 'Content-Length': '0'} 2022-09-03T07:00:29.755 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dbd536a8269d7830-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.756 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.756 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.757 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.758 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.758 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.759 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:29.759 DEBUG:boto:Data: 2022-09-03T07:00:29.760 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.760 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.761 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.762 DEBUG:boto:Params: {} 2022-09-03T07:00:29.763 DEBUG:boto:Token: None 2022-09-03T07:00:29.764 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:29.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UVxbk/UuwWnvMBXxTTeRoy4APVs= 2022-09-03T07:00:29.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UVxbk/UuwWnvMBXxTTeRoy4APVs=', 'Content-Length': '0'} 2022-09-03T07:00:29.768 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c94e712ed49ae1ea-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.769 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.770 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.771 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.772 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.772 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.773 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:29.774 DEBUG:boto:Data: 2022-09-03T07:00:29.775 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.776 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.776 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.777 DEBUG:boto:Params: {} 2022-09-03T07:00:29.778 DEBUG:boto:Token: None 2022-09-03T07:00:29.779 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:29.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UVxbk/UuwWnvMBXxTTeRoy4APVs= 2022-09-03T07:00:29.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UVxbk/UuwWnvMBXxTTeRoy4APVs=', 'Content-Length': '0'} 2022-09-03T07:00:29.784 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e530880a15a86fc-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.786 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:00:29.787 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.788 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.789 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.790 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.791 DEBUG:boto:Method: GET 2022-09-03T07:00:29.791 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:29.792 DEBUG:boto:Data: 2022-09-03T07:00:29.792 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.793 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.793 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.794 DEBUG:boto:Params: {} 2022-09-03T07:00:29.795 DEBUG:boto:Token: None 2022-09-03T07:00:29.796 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:29.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AZ6n9ClivvUqP4OXcQ6JR6Cjd4U= 2022-09-03T07:00:29.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AZ6n9ClivvUqP4OXcQ6JR6Cjd4U=', 'Content-Length': '0'} 2022-09-03T07:00:29.800 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d1557539bdb3f7fa-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.801 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.801 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.802 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.802 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:29.803 DEBUG:boto:Method: GET 2022-09-03T07:00:29.803 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:29.804 DEBUG:boto:Data: 2022-09-03T07:00:29.805 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.806 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.807 DEBUG:boto:Params: {} 2022-09-03T07:00:29.807 DEBUG:boto:Token: None 2022-09-03T07:00:29.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:29.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AZ6n9ClivvUqP4OXcQ6JR6Cjd4U= 2022-09-03T07:00:29.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AZ6n9ClivvUqP4OXcQ6JR6Cjd4U=', 'Content-Length': '0'} 2022-09-03T07:00:29.812 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005c01f25f979ba88-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.815 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.816 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.817 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:29.819 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:29.820 DEBUG:boto:Method: GET 2022-09-03T07:00:29.821 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:29.823 DEBUG:boto:Data: 2022-09-03T07:00:29.824 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.825 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.827 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.828 DEBUG:boto:Params: {} 2022-09-03T07:00:29.829 DEBUG:boto:Token: None 2022-09-03T07:00:29.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.832 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.835 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006594b68876664d65-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.836 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.837 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.837 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.838 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:29.839 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:29.839 DEBUG:boto:Method: GET 2022-09-03T07:00:29.840 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:29.841 DEBUG:boto:Data: 2022-09-03T07:00:29.841 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.843 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.844 DEBUG:boto:Params: {} 2022-09-03T07:00:29.845 DEBUG:boto:Token: None 2022-09-03T07:00:29.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.850 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000068b3889da2cfc9aa-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.852 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.853 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.854 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.855 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.856 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:29.857 DEBUG:boto:Data: 2022-09-03T07:00:29.858 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.858 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.859 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.859 DEBUG:boto:Params: {} 2022-09-03T07:00:29.860 DEBUG:boto:Token: None 2022-09-03T07:00:29.861 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c 2022-09-03T07:00:29.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t/83TjidW5xEIhY5Zt5Og9q4ZlE= 2022-09-03T07:00:29.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t/83TjidW5xEIhY5Zt5Og9q4ZlE=', 'Content-Length': '0'} 2022-09-03T07:00:29.864 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f584ca21ebd5e37-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.865 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.865 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.867 DEBUG:boto:Method: HEAD 2022-09-03T07:00:29.867 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:29.868 DEBUG:boto:Data: 2022-09-03T07:00:29.868 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.869 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.869 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.870 DEBUG:boto:Params: {} 2022-09-03T07:00:29.870 DEBUG:boto:Token: None 2022-09-03T07:00:29.871 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c 2022-09-03T07:00:29.871 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t/83TjidW5xEIhY5Zt5Og9q4ZlE= 2022-09-03T07:00:29.872 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t/83TjidW5xEIhY5Zt5Og9q4ZlE=', 'Content-Length': '0'} 2022-09-03T07:00:29.874 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000795921aeb0020e4d-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.875 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:00:29.876 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.876 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.877 DEBUG:boto:Method: GET 2022-09-03T07:00:29.877 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:29.878 DEBUG:boto:Data: 2022-09-03T07:00:29.879 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.880 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.880 DEBUG:boto:Params: {} 2022-09-03T07:00:29.881 DEBUG:boto:Token: None 2022-09-03T07:00:29.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c 2022-09-03T07:00:29.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRLqPNfxlTVdrg6H2SRFIFqlL2A= 2022-09-03T07:00:29.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRLqPNfxlTVdrg6H2SRFIFqlL2A=', 'Content-Length': '0'} 2022-09-03T07:00:29.888 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003def099ab903daa1-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.889 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:29.890 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:29.890 DEBUG:boto:Method: GET 2022-09-03T07:00:29.891 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:29.891 DEBUG:boto:Data: 2022-09-03T07:00:29.892 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.893 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.894 DEBUG:boto:Params: {} 2022-09-03T07:00:29.894 DEBUG:boto:Token: None 2022-09-03T07:00:29.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/c 2022-09-03T07:00:29.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRLqPNfxlTVdrg6H2SRFIFqlL2A= 2022-09-03T07:00:29.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRLqPNfxlTVdrg6H2SRFIFqlL2A=', 'Content-Length': '0'} 2022-09-03T07:00:29.898 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d07dd3a4a7aefc7-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.946 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.948 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.949 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:29.949 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:29.950 DEBUG:boto:Method: GET 2022-09-03T07:00:29.951 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:29.951 DEBUG:boto:Data: 2022-09-03T07:00:29.952 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.953 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.953 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.954 DEBUG:boto:Params: {} 2022-09-03T07:00:29.955 DEBUG:boto:Token: None 2022-09-03T07:00:29.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.959 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a84b15ee91f01c41-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.960 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.961 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:29.962 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:29.963 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:29.964 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:29.964 DEBUG:boto:Method: GET 2022-09-03T07:00:29.965 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:29.966 DEBUG:boto:Data: 2022-09-03T07:00:29.966 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.968 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.968 DEBUG:boto:Params: {} 2022-09-03T07:00:29.969 DEBUG:boto:Token: None 2022-09-03T07:00:29.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/ 2022-09-03T07:00:29.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA= 2022-09-03T07:00:29.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngOpnLKLmeam+ninXBN6bk7s0BA=', 'Content-Length': '0'} 2022-09-03T07:00:29.974 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e876b2dbefdf6ac-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.974 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:29.975 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:00:29.976 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:29.977 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:29.977 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:29.978 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:29.979 DEBUG:boto:Method: GET 2022-09-03T07:00:29.979 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:29.980 DEBUG:boto:Data: 2022-09-03T07:00:29.980 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.981 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:29.982 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.982 DEBUG:boto:Params: {} 2022-09-03T07:00:29.983 DEBUG:boto:Token: None 2022-09-03T07:00:29.984 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:29.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kcRmuBbwe9DmqdnHhr3AiJ0eScU= 2022-09-03T07:00:29.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kcRmuBbwe9DmqdnHhr3AiJ0eScU=', 'Content-Length': '0'} 2022-09-03T07:00:29.987 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e44d7c0924e231ea-006312fb8d-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:29 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:29.988 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:29.989 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:29.989 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:29.990 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:29.991 DEBUG:boto:Method: GET 2022-09-03T07:00:29.991 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:29.992 DEBUG:boto:Data: 2022-09-03T07:00:29.993 DEBUG:boto:Headers: {} 2022-09-03T07:00:29.994 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:29.994 DEBUG:boto:Port: 8000 2022-09-03T07:00:29.995 DEBUG:boto:Params: {} 2022-09-03T07:00:29.996 DEBUG:boto:Token: None 2022-09-03T07:00:29.997 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:29 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:29.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kcRmuBbwe9DmqdnHhr3AiJ0eScU= 2022-09-03T07:00:29.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:29 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kcRmuBbwe9DmqdnHhr3AiJ0eScU=', 'Content-Length': '0'} 2022-09-03T07:00:30.001 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d58fd58855eeb6d-006312fb8d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.002 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:30.003 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:30.003 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:30.004 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:30.004 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.005 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:30.005 DEBUG:boto:Data: 2022-09-03T07:00:30.006 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.006 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.007 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.008 DEBUG:boto:Params: {} 2022-09-03T07:00:30.008 DEBUG:boto:Token: None 2022-09-03T07:00:30.009 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:30.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Eg2AVJQHEXHruY9Cr1OhTyRumGQ= 2022-09-03T07:00:30.011 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Eg2AVJQHEXHruY9Cr1OhTyRumGQ=', 'Content-Length': '0'} 2022-09-03T07:00:30.013 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f268918594bde3e-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.014 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:30.015 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:30.015 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:30.016 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:30.016 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.017 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:30.017 DEBUG:boto:Data: 2022-09-03T07:00:30.018 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.018 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.019 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.020 DEBUG:boto:Params: {} 2022-09-03T07:00:30.020 DEBUG:boto:Token: None 2022-09-03T07:00:30.021 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:30.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Eg2AVJQHEXHruY9Cr1OhTyRumGQ= 2022-09-03T07:00:30.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Eg2AVJQHEXHruY9Cr1OhTyRumGQ=', 'Content-Length': '0'} 2022-09-03T07:00:30.025 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c6af6737309bd47-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.026 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:00:30.027 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:30.027 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:30.028 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:30.029 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:30.029 DEBUG:boto:Method: GET 2022-09-03T07:00:30.030 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:30.030 DEBUG:boto:Data: 2022-09-03T07:00:30.031 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.031 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.032 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.032 DEBUG:boto:Params: {} 2022-09-03T07:00:30.033 DEBUG:boto:Token: None 2022-09-03T07:00:30.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:30.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NW1BjEgakkPn/snm8Gn5U4+3xEA= 2022-09-03T07:00:30.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NW1BjEgakkPn/snm8Gn5U4+3xEA=', 'Content-Length': '0'} 2022-09-03T07:00:30.038 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dbdcc110849eb247-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.039 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:30.039 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:30.040 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:30.041 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:30.041 DEBUG:boto:Method: GET 2022-09-03T07:00:30.042 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:30.043 DEBUG:boto:Data: 2022-09-03T07:00:30.043 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.044 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.045 DEBUG:boto:Params: {} 2022-09-03T07:00:30.045 DEBUG:boto:Token: None 2022-09-03T07:00:30.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:30.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NW1BjEgakkPn/snm8Gn5U4+3xEA= 2022-09-03T07:00:30.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NW1BjEgakkPn/snm8Gn5U4+3xEA=', 'Content-Length': '0'} 2022-09-03T07:00:30.049 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba845a05a455e43e-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.050 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.051 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.051 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:30.052 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:30.052 DEBUG:boto:Method: GET 2022-09-03T07:00:30.053 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:30.053 DEBUG:boto:Data: 2022-09-03T07:00:30.054 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.055 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.055 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.056 DEBUG:boto:Params: {} 2022-09-03T07:00:30.056 DEBUG:boto:Token: None 2022-09-03T07:00:30.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.060 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000008bfe12237abc473-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.102 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.103 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.104 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.104 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:30.105 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:30.105 DEBUG:boto:Method: GET 2022-09-03T07:00:30.106 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:30.107 DEBUG:boto:Data: 2022-09-03T07:00:30.107 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.108 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.108 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.109 DEBUG:boto:Params: {} 2022-09-03T07:00:30.109 DEBUG:boto:Token: None 2022-09-03T07:00:30.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.114 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000021d66cc481a4a44e-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.115 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.116 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:30.117 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:30.119 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.120 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:30.121 DEBUG:boto:Data: 2022-09-03T07:00:30.122 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.123 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.124 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.125 DEBUG:boto:Params: {} 2022-09-03T07:00:30.126 DEBUG:boto:Token: None 2022-09-03T07:00:30.127 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/d 2022-09-03T07:00:30.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7UE0X4bWiUA69yLGNo+iyx5cMno= 2022-09-03T07:00:30.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7UE0X4bWiUA69yLGNo+iyx5cMno=', 'Content-Length': '0'} 2022-09-03T07:00:30.131 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0b6916621308fa5-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.132 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:30.132 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:30.133 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.133 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:30.134 DEBUG:boto:Data: 2022-09-03T07:00:30.135 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.136 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.136 DEBUG:boto:Params: {} 2022-09-03T07:00:30.137 DEBUG:boto:Token: None 2022-09-03T07:00:30.137 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/d 2022-09-03T07:00:30.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7UE0X4bWiUA69yLGNo+iyx5cMno= 2022-09-03T07:00:30.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7UE0X4bWiUA69yLGNo+iyx5cMno=', 'Content-Length': '0'} 2022-09-03T07:00:30.147 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002da4fc221eb6f9f1-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.148 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:00:30.148 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:30.149 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:30.150 DEBUG:boto:Method: GET 2022-09-03T07:00:30.151 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:30.151 DEBUG:boto:Data: 2022-09-03T07:00:30.152 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.152 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.153 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.153 DEBUG:boto:Params: {} 2022-09-03T07:00:30.154 DEBUG:boto:Token: None 2022-09-03T07:00:30.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/d 2022-09-03T07:00:30.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/eb5aFYPoHKwx3PtSIgG6WZ2XI= 2022-09-03T07:00:30.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/eb5aFYPoHKwx3PtSIgG6WZ2XI=', 'Content-Length': '0'} 2022-09-03T07:00:30.160 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d71f6dc1444f390-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.163 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:30.164 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:30.165 DEBUG:boto:Method: GET 2022-09-03T07:00:30.165 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:30.166 DEBUG:boto:Data: 2022-09-03T07:00:30.167 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.169 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.170 DEBUG:boto:Params: {} 2022-09-03T07:00:30.172 DEBUG:boto:Token: None 2022-09-03T07:00:30.173 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/d 2022-09-03T07:00:30.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/eb5aFYPoHKwx3PtSIgG6WZ2XI= 2022-09-03T07:00:30.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/eb5aFYPoHKwx3PtSIgG6WZ2XI=', 'Content-Length': '0'} 2022-09-03T07:00:30.179 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0a97875c9a3f682-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.227 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.229 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.230 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:30.231 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:30.232 DEBUG:boto:Method: GET 2022-09-03T07:00:30.233 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:30.234 DEBUG:boto:Data: 2022-09-03T07:00:30.235 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.236 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.237 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.238 DEBUG:boto:Params: {} 2022-09-03T07:00:30.240 DEBUG:boto:Token: None 2022-09-03T07:00:30.241 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.247 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b4eef94c96ceaeb-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.248 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.250 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.251 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.252 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:30.252 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:30.253 DEBUG:boto:Method: GET 2022-09-03T07:00:30.255 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:30.256 DEBUG:boto:Data: 2022-09-03T07:00:30.256 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.257 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.258 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.259 DEBUG:boto:Params: {} 2022-09-03T07:00:30.259 DEBUG:boto:Token: None 2022-09-03T07:00:30.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.265 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000345d157e44e8ab13-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.266 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.267 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:00:30.268 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.268 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.269 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.269 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.270 DEBUG:boto:Method: GET 2022-09-03T07:00:30.271 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:30.271 DEBUG:boto:Data: 2022-09-03T07:00:30.272 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.273 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.274 DEBUG:boto:Params: {} 2022-09-03T07:00:30.275 DEBUG:boto:Token: None 2022-09-03T07:00:30.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:30.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M6yBiId62DIaV4s8xqgywjScZyI= 2022-09-03T07:00:30.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M6yBiId62DIaV4s8xqgywjScZyI=', 'Content-Length': '0'} 2022-09-03T07:00:30.279 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c5474e9683da6b4-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.280 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.280 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.281 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.281 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.282 DEBUG:boto:Method: GET 2022-09-03T07:00:30.282 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:30.283 DEBUG:boto:Data: 2022-09-03T07:00:30.283 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.285 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.285 DEBUG:boto:Params: {} 2022-09-03T07:00:30.286 DEBUG:boto:Token: None 2022-09-03T07:00:30.287 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:30.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M6yBiId62DIaV4s8xqgywjScZyI= 2022-09-03T07:00:30.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M6yBiId62DIaV4s8xqgywjScZyI=', 'Content-Length': '0'} 2022-09-03T07:00:30.292 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc980765efbeb7d5-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.293 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.294 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.295 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.295 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.296 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.297 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:30.297 DEBUG:boto:Data: 2022-09-03T07:00:30.298 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.299 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.299 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.300 DEBUG:boto:Params: {} 2022-09-03T07:00:30.301 DEBUG:boto:Token: None 2022-09-03T07:00:30.301 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:30.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCAZSaj4OA9ihV/fAPEjKAOm0Ms= 2022-09-03T07:00:30.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCAZSaj4OA9ihV/fAPEjKAOm0Ms=', 'Content-Length': '0'} 2022-09-03T07:00:30.305 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000114d75aaa39a962e-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.305 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.306 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.307 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.307 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.308 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.308 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:30.309 DEBUG:boto:Data: 2022-09-03T07:00:30.309 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.310 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.310 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.311 DEBUG:boto:Params: {} 2022-09-03T07:00:30.312 DEBUG:boto:Token: None 2022-09-03T07:00:30.313 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:30.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCAZSaj4OA9ihV/fAPEjKAOm0Ms= 2022-09-03T07:00:30.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCAZSaj4OA9ihV/fAPEjKAOm0Ms=', 'Content-Length': '0'} 2022-09-03T07:00:30.317 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004febc51340010910-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.318 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:00:30.318 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.319 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.319 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.320 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.321 DEBUG:boto:Method: GET 2022-09-03T07:00:30.321 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:30.322 DEBUG:boto:Data: 2022-09-03T07:00:30.322 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.323 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.324 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.324 DEBUG:boto:Params: {} 2022-09-03T07:00:30.325 DEBUG:boto:Token: None 2022-09-03T07:00:30.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:30.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M6yBiId62DIaV4s8xqgywjScZyI= 2022-09-03T07:00:30.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M6yBiId62DIaV4s8xqgywjScZyI=', 'Content-Length': '0'} 2022-09-03T07:00:30.329 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb6d663708f17485-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.371 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.372 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.373 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.374 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:30.376 DEBUG:boto:Method: GET 2022-09-03T07:00:30.377 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:30.378 DEBUG:boto:Data: 2022-09-03T07:00:30.379 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.380 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.381 DEBUG:boto:Params: {} 2022-09-03T07:00:30.381 DEBUG:boto:Token: None 2022-09-03T07:00:30.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:30.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M6yBiId62DIaV4s8xqgywjScZyI= 2022-09-03T07:00:30.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M6yBiId62DIaV4s8xqgywjScZyI=', 'Content-Length': '0'} 2022-09-03T07:00:30.387 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b4bfdf106ea7ac8-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.388 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.389 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.389 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:30.390 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:30.391 DEBUG:boto:Method: GET 2022-09-03T07:00:30.391 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:30.392 DEBUG:boto:Data: 2022-09-03T07:00:30.392 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.393 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.393 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.394 DEBUG:boto:Params: {} 2022-09-03T07:00:30.395 DEBUG:boto:Token: None 2022-09-03T07:00:30.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.402 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6c6a884ddab3c0e-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.402 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.403 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.404 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.405 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:30.405 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:30.406 DEBUG:boto:Method: GET 2022-09-03T07:00:30.407 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:30.408 DEBUG:boto:Data: 2022-09-03T07:00:30.409 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.409 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.410 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.410 DEBUG:boto:Params: {} 2022-09-03T07:00:30.411 DEBUG:boto:Token: None 2022-09-03T07:00:30.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.416 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c193266a413b5a8-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.417 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.418 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.419 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.419 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.420 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:30.421 DEBUG:boto:Data: 2022-09-03T07:00:30.421 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.422 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.423 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.423 DEBUG:boto:Params: {} 2022-09-03T07:00:30.424 DEBUG:boto:Token: None 2022-09-03T07:00:30.424 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e 2022-09-03T07:00:30.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TS+STaiBZaZWSPrWpqusLZ/HRSA= 2022-09-03T07:00:30.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TS+STaiBZaZWSPrWpqusLZ/HRSA=', 'Content-Length': '0'} 2022-09-03T07:00:30.428 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d2118c77b40f859-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.429 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.430 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.431 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.432 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:30.433 DEBUG:boto:Data: 2022-09-03T07:00:30.434 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.435 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.436 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.437 DEBUG:boto:Params: {} 2022-09-03T07:00:30.438 DEBUG:boto:Token: None 2022-09-03T07:00:30.439 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e 2022-09-03T07:00:30.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TS+STaiBZaZWSPrWpqusLZ/HRSA= 2022-09-03T07:00:30.440 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TS+STaiBZaZWSPrWpqusLZ/HRSA=', 'Content-Length': '0'} 2022-09-03T07:00:30.443 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055025195c26cb527-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.443 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:00:30.444 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.445 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.445 DEBUG:boto:Method: GET 2022-09-03T07:00:30.446 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:30.446 DEBUG:boto:Data: 2022-09-03T07:00:30.447 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.447 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.448 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.448 DEBUG:boto:Params: {} 2022-09-03T07:00:30.449 DEBUG:boto:Token: None 2022-09-03T07:00:30.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e 2022-09-03T07:00:30.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ic1qMdWVOFT7tBVDlXqKpQjUYoE= 2022-09-03T07:00:30.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ic1qMdWVOFT7tBVDlXqKpQjUYoE=', 'Content-Length': '0'} 2022-09-03T07:00:30.454 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a42e221b10cce5b-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.455 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:30.456 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:30.457 DEBUG:boto:Method: GET 2022-09-03T07:00:30.458 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:30.459 DEBUG:boto:Data: 2022-09-03T07:00:30.460 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.462 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.463 DEBUG:boto:Params: {} 2022-09-03T07:00:30.465 DEBUG:boto:Token: None 2022-09-03T07:00:30.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/e 2022-09-03T07:00:30.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ic1qMdWVOFT7tBVDlXqKpQjUYoE= 2022-09-03T07:00:30.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ic1qMdWVOFT7tBVDlXqKpQjUYoE=', 'Content-Length': '0'} 2022-09-03T07:00:30.472 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000326d0530dd2c4a2f-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.514 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.515 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.515 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:30.516 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:30.516 DEBUG:boto:Method: GET 2022-09-03T07:00:30.517 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:30.518 DEBUG:boto:Data: 2022-09-03T07:00:30.519 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.521 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.522 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.522 DEBUG:boto:Params: {} 2022-09-03T07:00:30.524 DEBUG:boto:Token: None 2022-09-03T07:00:30.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.542 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dd6158fb4eafdec4-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.543 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.544 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.545 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.546 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:30.548 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:30.549 DEBUG:boto:Method: GET 2022-09-03T07:00:30.549 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:30.550 DEBUG:boto:Data: 2022-09-03T07:00:30.551 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.552 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.552 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.553 DEBUG:boto:Params: {} 2022-09-03T07:00:30.553 DEBUG:boto:Token: None 2022-09-03T07:00:30.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.558 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000041aea92edd10fce8-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.559 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.560 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:00:30.561 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.561 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.562 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.562 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.563 DEBUG:boto:Method: GET 2022-09-03T07:00:30.564 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:30.564 DEBUG:boto:Data: 2022-09-03T07:00:30.565 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.565 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.566 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.567 DEBUG:boto:Params: {} 2022-09-03T07:00:30.567 DEBUG:boto:Token: None 2022-09-03T07:00:30.568 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:30.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V5Apn0IFik52Vzyr44CFwwGxxwQ= 2022-09-03T07:00:30.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V5Apn0IFik52Vzyr44CFwwGxxwQ=', 'Content-Length': '0'} 2022-09-03T07:00:30.571 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd864dd102a74101-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.572 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.573 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.573 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.574 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.575 DEBUG:boto:Method: GET 2022-09-03T07:00:30.575 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:30.576 DEBUG:boto:Data: 2022-09-03T07:00:30.577 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.578 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.578 DEBUG:boto:Params: {} 2022-09-03T07:00:30.579 DEBUG:boto:Token: None 2022-09-03T07:00:30.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:30.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V5Apn0IFik52Vzyr44CFwwGxxwQ= 2022-09-03T07:00:30.581 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V5Apn0IFik52Vzyr44CFwwGxxwQ=', 'Content-Length': '0'} 2022-09-03T07:00:30.583 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000426a35e88c4ad96d-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.584 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.585 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.586 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.586 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.587 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.588 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:30.588 DEBUG:boto:Data: 2022-09-03T07:00:30.589 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.589 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.590 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.590 DEBUG:boto:Params: {} 2022-09-03T07:00:30.591 DEBUG:boto:Token: None 2022-09-03T07:00:30.592 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:30.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:deKgfzkjY46PU+WkqxL0TTaLOWk= 2022-09-03T07:00:30.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:deKgfzkjY46PU+WkqxL0TTaLOWk=', 'Content-Length': '0'} 2022-09-03T07:00:30.595 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007040ede92c84bbbc-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.596 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.596 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.597 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.597 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.597 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.598 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:30.599 DEBUG:boto:Data: 2022-09-03T07:00:30.599 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.600 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.601 DEBUG:boto:Params: {} 2022-09-03T07:00:30.601 DEBUG:boto:Token: None 2022-09-03T07:00:30.602 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:30.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:deKgfzkjY46PU+WkqxL0TTaLOWk= 2022-09-03T07:00:30.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:deKgfzkjY46PU+WkqxL0TTaLOWk=', 'Content-Length': '0'} 2022-09-03T07:00:30.605 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003cd309f8931c0eaa-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.606 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:00:30.606 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.607 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.608 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.608 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.609 DEBUG:boto:Method: GET 2022-09-03T07:00:30.609 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:30.610 DEBUG:boto:Data: 2022-09-03T07:00:30.610 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.611 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.611 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.612 DEBUG:boto:Params: {} 2022-09-03T07:00:30.612 DEBUG:boto:Token: None 2022-09-03T07:00:30.613 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:30.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V5Apn0IFik52Vzyr44CFwwGxxwQ= 2022-09-03T07:00:30.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V5Apn0IFik52Vzyr44CFwwGxxwQ=', 'Content-Length': '0'} 2022-09-03T07:00:30.616 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6c9db02616f0307-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.617 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.617 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.618 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.618 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:30.619 DEBUG:boto:Method: GET 2022-09-03T07:00:30.619 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:30.620 DEBUG:boto:Data: 2022-09-03T07:00:30.620 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.621 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.622 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.622 DEBUG:boto:Params: {} 2022-09-03T07:00:30.623 DEBUG:boto:Token: None 2022-09-03T07:00:30.623 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:30.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V5Apn0IFik52Vzyr44CFwwGxxwQ= 2022-09-03T07:00:30.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V5Apn0IFik52Vzyr44CFwwGxxwQ=', 'Content-Length': '0'} 2022-09-03T07:00:30.627 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e1eb9b70deb0680-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.628 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.628 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.629 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:30.629 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:30.630 DEBUG:boto:Method: GET 2022-09-03T07:00:30.630 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:30.631 DEBUG:boto:Data: 2022-09-03T07:00:30.631 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.632 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.633 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.634 DEBUG:boto:Params: {} 2022-09-03T07:00:30.635 DEBUG:boto:Token: None 2022-09-03T07:00:30.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.642 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2bcbfed80c96bcd-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.643 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.645 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.646 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.647 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:30.647 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:30.648 DEBUG:boto:Method: GET 2022-09-03T07:00:30.649 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:30.649 DEBUG:boto:Data: 2022-09-03T07:00:30.650 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.650 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.651 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.651 DEBUG:boto:Params: {} 2022-09-03T07:00:30.652 DEBUG:boto:Token: None 2022-09-03T07:00:30.653 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.656 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000244c46af8638e5c2-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.657 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.658 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.658 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.659 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.659 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:30.660 DEBUG:boto:Data: 2022-09-03T07:00:30.660 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.661 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.661 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.662 DEBUG:boto:Params: {} 2022-09-03T07:00:30.662 DEBUG:boto:Token: None 2022-09-03T07:00:30.663 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f 2022-09-03T07:00:30.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r4WPUvk7WxxaL0+wsxIvSM1P5+c= 2022-09-03T07:00:30.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r4WPUvk7WxxaL0+wsxIvSM1P5+c=', 'Content-Length': '0'} 2022-09-03T07:00:30.666 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097ae069de5b32acc-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.667 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.667 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.668 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.668 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:30.669 DEBUG:boto:Data: 2022-09-03T07:00:30.670 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.672 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.673 DEBUG:boto:Params: {} 2022-09-03T07:00:30.674 DEBUG:boto:Token: None 2022-09-03T07:00:30.675 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f 2022-09-03T07:00:30.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r4WPUvk7WxxaL0+wsxIvSM1P5+c= 2022-09-03T07:00:30.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r4WPUvk7WxxaL0+wsxIvSM1P5+c=', 'Content-Length': '0'} 2022-09-03T07:00:30.678 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031d097ed240ae7e6-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.678 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:00:30.679 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.680 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.680 DEBUG:boto:Method: GET 2022-09-03T07:00:30.681 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:30.682 DEBUG:boto:Data: 2022-09-03T07:00:30.682 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.683 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.684 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.684 DEBUG:boto:Params: {} 2022-09-03T07:00:30.685 DEBUG:boto:Token: None 2022-09-03T07:00:30.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f 2022-09-03T07:00:30.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gx7Z7K25XbGoguu7Vv3VSN+F2eU= 2022-09-03T07:00:30.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gx7Z7K25XbGoguu7Vv3VSN+F2eU=', 'Content-Length': '0'} 2022-09-03T07:00:30.689 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e2ef977233e5ec8-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.689 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:30.690 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:30.691 DEBUG:boto:Method: GET 2022-09-03T07:00:30.691 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:30.692 DEBUG:boto:Data: 2022-09-03T07:00:30.693 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.693 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.694 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.694 DEBUG:boto:Params: {} 2022-09-03T07:00:30.695 DEBUG:boto:Token: None 2022-09-03T07:00:30.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/f 2022-09-03T07:00:30.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gx7Z7K25XbGoguu7Vv3VSN+F2eU= 2022-09-03T07:00:30.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gx7Z7K25XbGoguu7Vv3VSN+F2eU=', 'Content-Length': '0'} 2022-09-03T07:00:30.699 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9008ab14124bd5b-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.700 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.701 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.701 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:30.702 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:30.703 DEBUG:boto:Method: GET 2022-09-03T07:00:30.703 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:30.704 DEBUG:boto:Data: 2022-09-03T07:00:30.704 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.705 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.705 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.706 DEBUG:boto:Params: {} 2022-09-03T07:00:30.707 DEBUG:boto:Token: None 2022-09-03T07:00:30.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.711 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a786905ac8c6c2af-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.713 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.714 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.715 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.716 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:30.717 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:30.718 DEBUG:boto:Method: GET 2022-09-03T07:00:30.719 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:30.720 DEBUG:boto:Data: 2022-09-03T07:00:30.721 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.723 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.724 DEBUG:boto:Params: {} 2022-09-03T07:00:30.725 DEBUG:boto:Token: None 2022-09-03T07:00:30.725 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.730 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e31cd42466e7797f-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.731 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.732 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:00:30.733 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.734 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.735 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.736 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.737 DEBUG:boto:Method: GET 2022-09-03T07:00:30.738 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:30.739 DEBUG:boto:Data: 2022-09-03T07:00:30.740 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.740 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.741 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.741 DEBUG:boto:Params: {} 2022-09-03T07:00:30.742 DEBUG:boto:Token: None 2022-09-03T07:00:30.743 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:30.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ikizfn2bR11jYaDxjQ4S8iXSXfA= 2022-09-03T07:00:30.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ikizfn2bR11jYaDxjQ4S8iXSXfA=', 'Content-Length': '0'} 2022-09-03T07:00:30.746 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ecc9605f7e8c090-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.790 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.791 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.792 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.793 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.794 DEBUG:boto:Method: GET 2022-09-03T07:00:30.795 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:30.796 DEBUG:boto:Data: 2022-09-03T07:00:30.797 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.799 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.800 DEBUG:boto:Params: {} 2022-09-03T07:00:30.802 DEBUG:boto:Token: None 2022-09-03T07:00:30.803 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:30.804 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ikizfn2bR11jYaDxjQ4S8iXSXfA= 2022-09-03T07:00:30.805 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ikizfn2bR11jYaDxjQ4S8iXSXfA=', 'Content-Length': '0'} 2022-09-03T07:00:30.808 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006029b55e23df320d-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.809 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.810 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.811 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.811 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.812 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.812 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:30.813 DEBUG:boto:Data: 2022-09-03T07:00:30.814 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.814 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.815 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.815 DEBUG:boto:Params: {} 2022-09-03T07:00:30.816 DEBUG:boto:Token: None 2022-09-03T07:00:30.817 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:30.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GT53tgiKEUZi2cOEZyQ67qkWYd8= 2022-09-03T07:00:30.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GT53tgiKEUZi2cOEZyQ67qkWYd8=', 'Content-Length': '0'} 2022-09-03T07:00:30.821 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d25d0b12eb9354c2-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.822 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.823 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.824 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.825 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.825 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.826 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:30.827 DEBUG:boto:Data: 2022-09-03T07:00:30.827 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.828 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.828 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.829 DEBUG:boto:Params: {} 2022-09-03T07:00:30.829 DEBUG:boto:Token: None 2022-09-03T07:00:30.830 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:30.830 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GT53tgiKEUZi2cOEZyQ67qkWYd8= 2022-09-03T07:00:30.831 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GT53tgiKEUZi2cOEZyQ67qkWYd8=', 'Content-Length': '0'} 2022-09-03T07:00:30.833 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000348dbc38384a0b09-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.834 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:00:30.835 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.835 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.836 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.836 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.837 DEBUG:boto:Method: GET 2022-09-03T07:00:30.837 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:30.838 DEBUG:boto:Data: 2022-09-03T07:00:30.838 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.839 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.839 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.840 DEBUG:boto:Params: {} 2022-09-03T07:00:30.840 DEBUG:boto:Token: None 2022-09-03T07:00:30.841 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:30.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ikizfn2bR11jYaDxjQ4S8iXSXfA= 2022-09-03T07:00:30.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ikizfn2bR11jYaDxjQ4S8iXSXfA=', 'Content-Length': '0'} 2022-09-03T07:00:30.844 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f4ff6cd306893b6e-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.845 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.846 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.847 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.847 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:30.848 DEBUG:boto:Method: GET 2022-09-03T07:00:30.848 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:30.849 DEBUG:boto:Data: 2022-09-03T07:00:30.849 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.851 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.852 DEBUG:boto:Params: {} 2022-09-03T07:00:30.853 DEBUG:boto:Token: None 2022-09-03T07:00:30.854 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:30.855 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ikizfn2bR11jYaDxjQ4S8iXSXfA= 2022-09-03T07:00:30.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ikizfn2bR11jYaDxjQ4S8iXSXfA=', 'Content-Length': '0'} 2022-09-03T07:00:30.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005ce9d07681440a0-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.859 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.860 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.860 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:30.861 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:30.861 DEBUG:boto:Method: GET 2022-09-03T07:00:30.862 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:30.863 DEBUG:boto:Data: 2022-09-03T07:00:30.863 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.864 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.864 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.865 DEBUG:boto:Params: {} 2022-09-03T07:00:30.865 DEBUG:boto:Token: None 2022-09-03T07:00:30.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.868 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.870 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008159078ff29cee32-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.871 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.872 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.872 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.873 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:30.873 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:30.874 DEBUG:boto:Method: GET 2022-09-03T07:00:30.874 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:30.875 DEBUG:boto:Data: 2022-09-03T07:00:30.875 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.876 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.877 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.877 DEBUG:boto:Params: {} 2022-09-03T07:00:30.878 DEBUG:boto:Token: None 2022-09-03T07:00:30.879 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.882 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bbe554b92aa8380b-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.926 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.927 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.928 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.928 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.929 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:30.929 DEBUG:boto:Data: 2022-09-03T07:00:30.930 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.931 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.932 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.932 DEBUG:boto:Params: {} 2022-09-03T07:00:30.933 DEBUG:boto:Token: None 2022-09-03T07:00:30.934 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g 2022-09-03T07:00:30.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vK/gdFpP8DsRywrytEojr08HK5g= 2022-09-03T07:00:30.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vK/gdFpP8DsRywrytEojr08HK5g=', 'Content-Length': '0'} 2022-09-03T07:00:30.937 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007cfc64e148965621-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.938 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.938 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.939 DEBUG:boto:Method: HEAD 2022-09-03T07:00:30.939 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:30.940 DEBUG:boto:Data: 2022-09-03T07:00:30.940 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.941 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.941 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.942 DEBUG:boto:Params: {} 2022-09-03T07:00:30.943 DEBUG:boto:Token: None 2022-09-03T07:00:30.943 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g 2022-09-03T07:00:30.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vK/gdFpP8DsRywrytEojr08HK5g= 2022-09-03T07:00:30.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vK/gdFpP8DsRywrytEojr08HK5g=', 'Content-Length': '0'} 2022-09-03T07:00:30.946 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de276eee652d90cd-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.947 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:00:30.948 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.948 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.949 DEBUG:boto:Method: GET 2022-09-03T07:00:30.949 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:30.950 DEBUG:boto:Data: 2022-09-03T07:00:30.950 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.951 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.952 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.952 DEBUG:boto:Params: {} 2022-09-03T07:00:30.953 DEBUG:boto:Token: None 2022-09-03T07:00:30.954 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g 2022-09-03T07:00:30.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mc+1JjxplCKrajh97uuksGrLayY= 2022-09-03T07:00:30.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mc+1JjxplCKrajh97uuksGrLayY=', 'Content-Length': '0'} 2022-09-03T07:00:30.957 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ffca2fa866ef69c-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.958 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:30.959 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:30.959 DEBUG:boto:Method: GET 2022-09-03T07:00:30.960 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:30.960 DEBUG:boto:Data: 2022-09-03T07:00:30.961 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.962 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.962 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.963 DEBUG:boto:Params: {} 2022-09-03T07:00:30.963 DEBUG:boto:Token: None 2022-09-03T07:00:30.964 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/g 2022-09-03T07:00:30.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mc+1JjxplCKrajh97uuksGrLayY= 2022-09-03T07:00:30.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mc+1JjxplCKrajh97uuksGrLayY=', 'Content-Length': '0'} 2022-09-03T07:00:30.967 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020cb3998f9385483-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.968 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.969 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.970 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:30.970 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:30.971 DEBUG:boto:Method: GET 2022-09-03T07:00:30.971 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:30.972 DEBUG:boto:Data: 2022-09-03T07:00:30.972 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.973 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.973 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.974 DEBUG:boto:Params: {} 2022-09-03T07:00:30.975 DEBUG:boto:Token: None 2022-09-03T07:00:30.975 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.979 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001cff2f1af6dece32-006312fb8e-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.979 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.981 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:30.981 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:30.982 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:30.982 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:30.983 DEBUG:boto:Method: GET 2022-09-03T07:00:30.983 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:30.984 DEBUG:boto:Data: 2022-09-03T07:00:30.984 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:30.985 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.986 DEBUG:boto:Params: {} 2022-09-03T07:00:30.987 DEBUG:boto:Token: None 2022-09-03T07:00:30.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:30 GMT /avysmw-72/ 2022-09-03T07:00:30.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU= 2022-09-03T07:00:30.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiD9y14FWkmau4KUCEtUWufyJYU=', 'Content-Length': '0'} 2022-09-03T07:00:30.991 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a71d2f6e3ddca5c-006312fb8e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:30.992 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:30.993 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:00:30.993 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:30.994 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:30.994 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:30.995 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:30.995 DEBUG:boto:Method: GET 2022-09-03T07:00:30.996 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:30.996 DEBUG:boto:Data: 2022-09-03T07:00:30.997 DEBUG:boto:Headers: {} 2022-09-03T07:00:30.998 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:30.998 DEBUG:boto:Port: 8000 2022-09-03T07:00:30.999 DEBUG:boto:Params: {} 2022-09-03T07:00:30.999 DEBUG:boto:Token: None 2022-09-03T07:00:31.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:31.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bqtne0hkeiAEiWrB54fZVI8Qoj8= 2022-09-03T07:00:31.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bqtne0hkeiAEiWrB54fZVI8Qoj8=', 'Content-Length': '0'} 2022-09-03T07:00:31.003 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000188ecbe2ab9c7375-006312fb8f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.004 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:31.005 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:31.006 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.006 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.007 DEBUG:boto:Method: GET 2022-09-03T07:00:31.007 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:31.008 DEBUG:boto:Data: 2022-09-03T07:00:31.009 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.010 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.010 DEBUG:boto:Params: {} 2022-09-03T07:00:31.011 DEBUG:boto:Token: None 2022-09-03T07:00:31.011 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:31.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bqtne0hkeiAEiWrB54fZVI8Qoj8= 2022-09-03T07:00:31.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bqtne0hkeiAEiWrB54fZVI8Qoj8=', 'Content-Length': '0'} 2022-09-03T07:00:31.015 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8e50ff149624d37-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.058 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:31.059 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:31.060 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.061 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.062 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.063 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:31.063 DEBUG:boto:Data: 2022-09-03T07:00:31.064 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.066 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:31.067 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.068 DEBUG:boto:Params: {} 2022-09-03T07:00:31.069 DEBUG:boto:Token: None 2022-09-03T07:00:31.070 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:31.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:it+klBE2iMIz8jN8Xy0dexzPx1Q= 2022-09-03T07:00:31.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:it+klBE2iMIz8jN8Xy0dexzPx1Q=', 'Content-Length': '0'} 2022-09-03T07:00:31.075 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090bbaafcdec14422-006312fb8f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.077 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:31.078 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:31.079 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.080 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.083 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.084 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:31.085 DEBUG:boto:Data: 2022-09-03T07:00:31.086 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.087 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.090 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.091 DEBUG:boto:Params: {} 2022-09-03T07:00:31.092 DEBUG:boto:Token: None 2022-09-03T07:00:31.093 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:31.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:it+klBE2iMIz8jN8Xy0dexzPx1Q= 2022-09-03T07:00:31.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:it+klBE2iMIz8jN8Xy0dexzPx1Q=', 'Content-Length': '0'} 2022-09-03T07:00:31.096 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027dfbf02056be8c6-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.097 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:00:31.097 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:31.112 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:31.113 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.113 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.114 DEBUG:boto:Method: GET 2022-09-03T07:00:31.114 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:31.115 DEBUG:boto:Data: 2022-09-03T07:00:31.116 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.116 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:31.117 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.117 DEBUG:boto:Params: {} 2022-09-03T07:00:31.118 DEBUG:boto:Token: None 2022-09-03T07:00:31.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:31.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bqtne0hkeiAEiWrB54fZVI8Qoj8= 2022-09-03T07:00:31.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bqtne0hkeiAEiWrB54fZVI8Qoj8=', 'Content-Length': '0'} 2022-09-03T07:00:31.122 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000917c3c5020c8bfe5-006312fb8f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.123 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:31.124 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:31.124 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.125 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:31.126 DEBUG:boto:Method: GET 2022-09-03T07:00:31.126 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:31.127 DEBUG:boto:Data: 2022-09-03T07:00:31.127 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.128 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.129 DEBUG:boto:Params: {} 2022-09-03T07:00:31.130 DEBUG:boto:Token: None 2022-09-03T07:00:31.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:31.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bqtne0hkeiAEiWrB54fZVI8Qoj8= 2022-09-03T07:00:31.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bqtne0hkeiAEiWrB54fZVI8Qoj8=', 'Content-Length': '0'} 2022-09-03T07:00:31.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004887f11987ad0edd-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.139 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.140 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.141 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:31.142 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:31.143 DEBUG:boto:Method: GET 2022-09-03T07:00:31.144 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:31.145 DEBUG:boto:Data: 2022-09-03T07:00:31.145 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.146 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:31.146 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.147 DEBUG:boto:Params: {} 2022-09-03T07:00:31.148 DEBUG:boto:Token: None 2022-09-03T07:00:31.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.154 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000033d66315b0ae222-006312fb8f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.198 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.200 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.200 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.201 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:31.202 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:31.203 DEBUG:boto:Method: GET 2022-09-03T07:00:31.204 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:31.204 DEBUG:boto:Data: 2022-09-03T07:00:31.205 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.206 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.206 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.207 DEBUG:boto:Params: {} 2022-09-03T07:00:31.207 DEBUG:boto:Token: None 2022-09-03T07:00:31.208 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.212 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e70f6252a9c6627d-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.213 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.214 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:31.214 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:31.215 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.216 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:31.216 DEBUG:boto:Data: 2022-09-03T07:00:31.217 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.218 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:31.219 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.220 DEBUG:boto:Params: {} 2022-09-03T07:00:31.220 DEBUG:boto:Token: None 2022-09-03T07:00:31.221 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h 2022-09-03T07:00:31.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MIZCEEvHw5Cxcfm7vd9pwgieW4M= 2022-09-03T07:00:31.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MIZCEEvHw5Cxcfm7vd9pwgieW4M=', 'Content-Length': '0'} 2022-09-03T07:00:31.224 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029b63236ca5d1555-006312fb8f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.225 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:31.226 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:31.226 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.227 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:31.227 DEBUG:boto:Data: 2022-09-03T07:00:31.228 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.228 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.229 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.229 DEBUG:boto:Params: {} 2022-09-03T07:00:31.230 DEBUG:boto:Token: None 2022-09-03T07:00:31.230 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h 2022-09-03T07:00:31.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MIZCEEvHw5Cxcfm7vd9pwgieW4M= 2022-09-03T07:00:31.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MIZCEEvHw5Cxcfm7vd9pwgieW4M=', 'Content-Length': '0'} 2022-09-03T07:00:31.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb6a01604b5c495b-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.234 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:00:31.235 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:31.235 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:31.236 DEBUG:boto:Method: GET 2022-09-03T07:00:31.236 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:31.237 DEBUG:boto:Data: 2022-09-03T07:00:31.237 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:31.238 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.239 DEBUG:boto:Params: {} 2022-09-03T07:00:31.240 DEBUG:boto:Token: None 2022-09-03T07:00:31.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h 2022-09-03T07:00:31.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ccC7jffP6Wldfxgj5N0H+AjId78= 2022-09-03T07:00:31.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ccC7jffP6Wldfxgj5N0H+AjId78=', 'Content-Length': '0'} 2022-09-03T07:00:31.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092d600a0f430f3b2-006312fb8f-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.244 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:31.245 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:31.245 DEBUG:boto:Method: GET 2022-09-03T07:00:31.246 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:31.246 DEBUG:boto:Data: 2022-09-03T07:00:31.247 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.247 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.248 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.248 DEBUG:boto:Params: {} 2022-09-03T07:00:31.249 DEBUG:boto:Token: None 2022-09-03T07:00:31.250 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/h 2022-09-03T07:00:31.250 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ccC7jffP6Wldfxgj5N0H+AjId78= 2022-09-03T07:00:31.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ccC7jffP6Wldfxgj5N0H+AjId78=', 'Content-Length': '0'} 2022-09-03T07:00:31.253 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8e214b541f5f9dc-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.253 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.254 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.255 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:31.255 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:31.256 DEBUG:boto:Method: GET 2022-09-03T07:00:31.256 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:31.257 DEBUG:boto:Data: 2022-09-03T07:00:31.257 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.258 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:31.258 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.259 DEBUG:boto:Params: {} 2022-09-03T07:00:31.259 DEBUG:boto:Token: None 2022-09-03T07:00:31.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.263 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1dbbd2e1b56a10b-006312fb8f-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.264 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.265 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.265 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.266 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:31.266 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:31.267 DEBUG:boto:Method: GET 2022-09-03T07:00:31.267 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:31.268 DEBUG:boto:Data: 2022-09-03T07:00:31.268 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.269 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.270 DEBUG:boto:Params: {} 2022-09-03T07:00:31.271 DEBUG:boto:Token: None 2022-09-03T07:00:31.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.275 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da06bf6279e62a19-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.275 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.276 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-72 zones={test-zone2, test-zone1} 2022-09-03T07:00:31.277 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-72 zones={test-zone3, test-zone1} 2022-09-03T07:00:31.277 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.278 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.278 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.279 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T07:00:31.279 DEBUG:boto:Data: 2022-09-03T07:00:31.280 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.280 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.281 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.281 DEBUG:boto:Params: {} 2022-09-03T07:00:31.282 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:00:31.283 DEBUG:boto:Token: None 2022-09-03T07:00:31.283 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.284 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D7RTBbJCII7bmpPkNxo9piicYxk= 2022-09-03T07:00:31.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D7RTBbJCII7bmpPkNxo9piicYxk=', 'Content-Length': '0'} 2022-09-03T07:00:31.288 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000029be7466869ec73e-006312fb8f-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.289 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.289 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.290 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.290 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T07:00:31.291 DEBUG:boto:Data: 2022-09-03T07:00:31.291 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.292 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.293 DEBUG:boto:Params: {} 2022-09-03T07:00:31.293 DEBUG:boto:Token: None 2022-09-03T07:00:31.294 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D7RTBbJCII7bmpPkNxo9piicYxk= 2022-09-03T07:00:31.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D7RTBbJCII7bmpPkNxo9piicYxk=', 'Content-Length': '0'} 2022-09-03T07:00:31.297 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000093b1d142af7de7c2-006312fb8f-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.298 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:00:31.298 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.299 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.299 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.300 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.300 DEBUG:boto:Method: GET 2022-09-03T07:00:31.301 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.301 DEBUG:boto:Data: 2022-09-03T07:00:31.302 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.302 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.303 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.303 DEBUG:boto:Params: {} 2022-09-03T07:00:31.304 DEBUG:boto:Token: None 2022-09-03T07:00:31.305 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/?versions 2022-09-03T07:00:31.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hkDDgANWGrQdIZaaU6padf6c2TA= 2022-09-03T07:00:31.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hkDDgANWGrQdIZaaU6padf6c2TA=', 'Content-Length': '0'} 2022-09-03T07:00:31.309 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075be880708b0a80a-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.309 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.311 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:00:31.312 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:00:31.312 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:00:31.313 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:00:31.313 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:00:31.314 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:00:31.314 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:00:31.315 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:00:31.316 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:00:31.316 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.317 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.317 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.318 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.318 DEBUG:boto:Method: GET 2022-09-03T07:00:31.319 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.319 DEBUG:boto:Data: 2022-09-03T07:00:31.320 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.321 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.321 DEBUG:boto:Params: {} 2022-09-03T07:00:31.322 DEBUG:boto:Token: None 2022-09-03T07:00:31.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/?versions 2022-09-03T07:00:31.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hkDDgANWGrQdIZaaU6padf6c2TA= 2022-09-03T07:00:31.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hkDDgANWGrQdIZaaU6padf6c2TA=', 'Content-Length': '0'} 2022-09-03T07:00:31.326 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005adf83fe2528c6e0-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.327 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.328 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:00:31.329 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:00:31.329 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:00:31.330 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:00:31.330 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:00:31.331 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:00:31.331 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:00:31.332 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:00:31.332 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.333 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.333 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.334 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.334 DEBUG:boto:Method: GET 2022-09-03T07:00:31.335 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.336 DEBUG:boto:Data: 2022-09-03T07:00:31.337 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.339 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.340 DEBUG:boto:Params: {} 2022-09-03T07:00:31.341 DEBUG:boto:Token: None 2022-09-03T07:00:31.342 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/?versions 2022-09-03T07:00:31.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hkDDgANWGrQdIZaaU6padf6c2TA= 2022-09-03T07:00:31.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hkDDgANWGrQdIZaaU6padf6c2TA=', 'Content-Length': '0'} 2022-09-03T07:00:31.346 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004eed4482baa5ecc6-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.347 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.348 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.348 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.349 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.349 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.350 DEBUG:boto:Method: GET 2022-09-03T07:00:31.350 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:31.351 DEBUG:boto:Data: 2022-09-03T07:00:31.351 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.352 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.353 DEBUG:boto:Params: {} 2022-09-03T07:00:31.354 DEBUG:boto:Token: None 2022-09-03T07:00:31.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/?versions 2022-09-03T07:00:31.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hkDDgANWGrQdIZaaU6padf6c2TA= 2022-09-03T07:00:31.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hkDDgANWGrQdIZaaU6padf6c2TA=', 'Content-Length': '0'} 2022-09-03T07:00:31.358 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fbb6b8654331122b-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.359 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.360 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:00:31.360 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.361 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.362 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.363 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.363 DEBUG:boto:Method: GET 2022-09-03T07:00:31.364 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:31.364 DEBUG:boto:Data: 2022-09-03T07:00:31.365 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.366 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.367 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.368 DEBUG:boto:Params: {} 2022-09-03T07:00:31.369 DEBUG:boto:Token: None 2022-09-03T07:00:31.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:31.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yzV0LRiLgaFA7AU7qfxADYgILhQ= 2022-09-03T07:00:31.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yzV0LRiLgaFA7AU7qfxADYgILhQ=', 'Content-Length': '0'} 2022-09-03T07:00:31.376 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008742dfedd1ed996f-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.377 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.377 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.378 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.378 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.379 DEBUG:boto:Method: GET 2022-09-03T07:00:31.379 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:31.380 DEBUG:boto:Data: 2022-09-03T07:00:31.380 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.381 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.381 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.382 DEBUG:boto:Params: {} 2022-09-03T07:00:31.382 DEBUG:boto:Token: None 2022-09-03T07:00:31.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:31.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yzV0LRiLgaFA7AU7qfxADYgILhQ= 2022-09-03T07:00:31.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yzV0LRiLgaFA7AU7qfxADYgILhQ=', 'Content-Length': '0'} 2022-09-03T07:00:31.386 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000879690be3d99678a-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.430 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.431 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.431 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.432 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.432 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.433 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:31.434 DEBUG:boto:Data: 2022-09-03T07:00:31.434 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.435 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.435 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.436 DEBUG:boto:Params: {} 2022-09-03T07:00:31.436 DEBUG:boto:Token: None 2022-09-03T07:00:31.437 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:31.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GYkf58cKmnmo5TKNphaERdbOKX4= 2022-09-03T07:00:31.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GYkf58cKmnmo5TKNphaERdbOKX4=', 'Content-Length': '0'} 2022-09-03T07:00:31.441 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000768527e065e9bab-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.441 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.442 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.443 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.444 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.445 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.445 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:31.446 DEBUG:boto:Data: 2022-09-03T07:00:31.447 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.449 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.450 DEBUG:boto:Params: {} 2022-09-03T07:00:31.451 DEBUG:boto:Token: None 2022-09-03T07:00:31.452 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:31.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GYkf58cKmnmo5TKNphaERdbOKX4= 2022-09-03T07:00:31.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GYkf58cKmnmo5TKNphaERdbOKX4=', 'Content-Length': '0'} 2022-09-03T07:00:31.456 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000888daaec776cc579-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.457 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:00:31.457 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.458 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.458 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.459 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.459 DEBUG:boto:Method: GET 2022-09-03T07:00:31.460 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:31.460 DEBUG:boto:Data: 2022-09-03T07:00:31.461 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.462 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.463 DEBUG:boto:Params: {} 2022-09-03T07:00:31.463 DEBUG:boto:Token: None 2022-09-03T07:00:31.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:31.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yzV0LRiLgaFA7AU7qfxADYgILhQ= 2022-09-03T07:00:31.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yzV0LRiLgaFA7AU7qfxADYgILhQ=', 'Content-Length': '0'} 2022-09-03T07:00:31.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006cc15fe663fa63ed-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.468 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.469 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.469 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.470 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:31.470 DEBUG:boto:Method: GET 2022-09-03T07:00:31.471 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:31.471 DEBUG:boto:Data: 2022-09-03T07:00:31.472 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.473 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.473 DEBUG:boto:Params: {} 2022-09-03T07:00:31.474 DEBUG:boto:Token: None 2022-09-03T07:00:31.475 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:31.475 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yzV0LRiLgaFA7AU7qfxADYgILhQ= 2022-09-03T07:00:31.476 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yzV0LRiLgaFA7AU7qfxADYgILhQ=', 'Content-Length': '0'} 2022-09-03T07:00:31.478 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8870b98f1fb6fb1-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.479 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.480 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.480 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:31.481 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:31.481 DEBUG:boto:Method: GET 2022-09-03T07:00:31.482 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:31.482 DEBUG:boto:Data: 2022-09-03T07:00:31.483 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.484 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.484 DEBUG:boto:Params: {} 2022-09-03T07:00:31.485 DEBUG:boto:Token: None 2022-09-03T07:00:31.486 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.486 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.487 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.489 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009899dc4d5837bef9-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.490 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.491 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.492 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.492 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:31.493 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:31.493 DEBUG:boto:Method: GET 2022-09-03T07:00:31.494 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:31.494 DEBUG:boto:Data: 2022-09-03T07:00:31.495 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.495 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.496 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.496 DEBUG:boto:Params: {} 2022-09-03T07:00:31.497 DEBUG:boto:Token: None 2022-09-03T07:00:31.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.501 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000635f2a48ec2700a0-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.502 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.502 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.503 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.504 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.504 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:31.504 DEBUG:boto:Data: 2022-09-03T07:00:31.505 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.506 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.506 DEBUG:boto:Params: {} 2022-09-03T07:00:31.507 DEBUG:boto:Token: None 2022-09-03T07:00:31.508 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a 2022-09-03T07:00:31.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vSl+rzsMlN3QxVnUB7DxAUVOxvo= 2022-09-03T07:00:31.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vSl+rzsMlN3QxVnUB7DxAUVOxvo=', 'Content-Length': '0'} 2022-09-03T07:00:31.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005ab7c036407a954-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.511 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.512 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.512 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.513 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:31.513 DEBUG:boto:Data: 2022-09-03T07:00:31.514 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.514 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.515 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.515 DEBUG:boto:Params: {} 2022-09-03T07:00:31.516 DEBUG:boto:Token: None 2022-09-03T07:00:31.516 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a 2022-09-03T07:00:31.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vSl+rzsMlN3QxVnUB7DxAUVOxvo= 2022-09-03T07:00:31.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vSl+rzsMlN3QxVnUB7DxAUVOxvo=', 'Content-Length': '0'} 2022-09-03T07:00:31.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000749df5300a222fbc-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:00:31.521 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.522 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.522 DEBUG:boto:Method: GET 2022-09-03T07:00:31.523 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:31.523 DEBUG:boto:Data: 2022-09-03T07:00:31.524 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.525 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.526 DEBUG:boto:Params: {} 2022-09-03T07:00:31.526 DEBUG:boto:Token: None 2022-09-03T07:00:31.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a 2022-09-03T07:00:31.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rqyI++52Hm6MyrhmYanwRHUoRW8= 2022-09-03T07:00:31.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rqyI++52Hm6MyrhmYanwRHUoRW8=', 'Content-Length': '0'} 2022-09-03T07:00:31.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063158b522a86e994-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.531 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:31.531 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:31.532 DEBUG:boto:Method: GET 2022-09-03T07:00:31.532 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:31.533 DEBUG:boto:Data: 2022-09-03T07:00:31.533 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.535 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.535 DEBUG:boto:Params: {} 2022-09-03T07:00:31.536 DEBUG:boto:Token: None 2022-09-03T07:00:31.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/a 2022-09-03T07:00:31.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rqyI++52Hm6MyrhmYanwRHUoRW8= 2022-09-03T07:00:31.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rqyI++52Hm6MyrhmYanwRHUoRW8=', 'Content-Length': '0'} 2022-09-03T07:00:31.540 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009052661b6213f607-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.541 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.542 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.542 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:31.543 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:31.544 DEBUG:boto:Method: GET 2022-09-03T07:00:31.544 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:31.545 DEBUG:boto:Data: 2022-09-03T07:00:31.545 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.546 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.546 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.547 DEBUG:boto:Params: {} 2022-09-03T07:00:31.548 DEBUG:boto:Token: None 2022-09-03T07:00:31.548 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.552 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007792fbf8c76f03df-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.594 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.595 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.596 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.596 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:31.597 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:31.597 DEBUG:boto:Method: GET 2022-09-03T07:00:31.598 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:31.598 DEBUG:boto:Data: 2022-09-03T07:00:31.599 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.600 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.601 DEBUG:boto:Params: {} 2022-09-03T07:00:31.601 DEBUG:boto:Token: None 2022-09-03T07:00:31.602 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.606 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000097c038553a66706a-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.606 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.607 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:00:31.608 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.609 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.609 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.610 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.611 DEBUG:boto:Method: GET 2022-09-03T07:00:31.611 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:31.612 DEBUG:boto:Data: 2022-09-03T07:00:31.612 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.614 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.615 DEBUG:boto:Params: {} 2022-09-03T07:00:31.615 DEBUG:boto:Token: None 2022-09-03T07:00:31.616 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:31.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MOsQmiGoKdAYDR7K9xq2Id9/kOA= 2022-09-03T07:00:31.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MOsQmiGoKdAYDR7K9xq2Id9/kOA=', 'Content-Length': '0'} 2022-09-03T07:00:31.620 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000036a92bb85e4a1da-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.622 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.623 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.624 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.625 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.626 DEBUG:boto:Method: GET 2022-09-03T07:00:31.627 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:31.628 DEBUG:boto:Data: 2022-09-03T07:00:31.629 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.630 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.631 DEBUG:boto:Params: {} 2022-09-03T07:00:31.631 DEBUG:boto:Token: None 2022-09-03T07:00:31.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:31.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MOsQmiGoKdAYDR7K9xq2Id9/kOA= 2022-09-03T07:00:31.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MOsQmiGoKdAYDR7K9xq2Id9/kOA=', 'Content-Length': '0'} 2022-09-03T07:00:31.637 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001263f05c74e324c0-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.639 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.639 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.640 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.640 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.641 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.642 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:31.642 DEBUG:boto:Data: 2022-09-03T07:00:31.643 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.644 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.645 DEBUG:boto:Params: {} 2022-09-03T07:00:31.645 DEBUG:boto:Token: None 2022-09-03T07:00:31.646 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:31.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5aFlboo9HvrOzH9pPWy+3rsUsc0= 2022-09-03T07:00:31.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5aFlboo9HvrOzH9pPWy+3rsUsc0=', 'Content-Length': '0'} 2022-09-03T07:00:31.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c58cf131ba2d67ee-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.654 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.655 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.656 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.657 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.658 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.659 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:31.659 DEBUG:boto:Data: 2022-09-03T07:00:31.660 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.662 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.663 DEBUG:boto:Params: {} 2022-09-03T07:00:31.664 DEBUG:boto:Token: None 2022-09-03T07:00:31.665 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:31.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5aFlboo9HvrOzH9pPWy+3rsUsc0= 2022-09-03T07:00:31.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5aFlboo9HvrOzH9pPWy+3rsUsc0=', 'Content-Length': '0'} 2022-09-03T07:00:31.670 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ffa7ee10d578b24-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.671 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:00:31.672 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.673 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.674 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.674 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.675 DEBUG:boto:Method: GET 2022-09-03T07:00:31.676 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:31.677 DEBUG:boto:Data: 2022-09-03T07:00:31.678 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.679 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.680 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.681 DEBUG:boto:Params: {} 2022-09-03T07:00:31.683 DEBUG:boto:Token: None 2022-09-03T07:00:31.684 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:31.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MOsQmiGoKdAYDR7K9xq2Id9/kOA= 2022-09-03T07:00:31.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MOsQmiGoKdAYDR7K9xq2Id9/kOA=', 'Content-Length': '0'} 2022-09-03T07:00:31.690 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dd7f9f81d5871544-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.691 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.691 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.692 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.693 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:31.693 DEBUG:boto:Method: GET 2022-09-03T07:00:31.694 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:31.695 DEBUG:boto:Data: 2022-09-03T07:00:31.695 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.696 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.697 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.698 DEBUG:boto:Params: {} 2022-09-03T07:00:31.698 DEBUG:boto:Token: None 2022-09-03T07:00:31.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:31.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MOsQmiGoKdAYDR7K9xq2Id9/kOA= 2022-09-03T07:00:31.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MOsQmiGoKdAYDR7K9xq2Id9/kOA=', 'Content-Length': '0'} 2022-09-03T07:00:31.703 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046a425236a7535d1-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.704 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.705 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.707 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:31.708 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:31.709 DEBUG:boto:Method: GET 2022-09-03T07:00:31.710 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:31.711 DEBUG:boto:Data: 2022-09-03T07:00:31.712 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.714 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.715 DEBUG:boto:Params: {} 2022-09-03T07:00:31.716 DEBUG:boto:Token: None 2022-09-03T07:00:31.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.723 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dd56f5b17ae04d55-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.723 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.725 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.725 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.726 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:31.727 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:31.728 DEBUG:boto:Method: GET 2022-09-03T07:00:31.729 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:31.729 DEBUG:boto:Data: 2022-09-03T07:00:31.730 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.731 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.732 DEBUG:boto:Params: {} 2022-09-03T07:00:31.733 DEBUG:boto:Token: None 2022-09-03T07:00:31.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.738 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007e9f557a22ce2f09-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.739 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.740 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.741 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.741 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.742 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:31.743 DEBUG:boto:Data: 2022-09-03T07:00:31.744 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.744 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.745 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.746 DEBUG:boto:Params: {} 2022-09-03T07:00:31.747 DEBUG:boto:Token: None 2022-09-03T07:00:31.748 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b 2022-09-03T07:00:31.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fd2RPxMer5qyXx5luQxP9uBxPEw= 2022-09-03T07:00:31.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fd2RPxMer5qyXx5luQxP9uBxPEw=', 'Content-Length': '0'} 2022-09-03T07:00:31.752 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005cf5e3ab3295f806-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.753 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.754 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.754 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.755 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:31.756 DEBUG:boto:Data: 2022-09-03T07:00:31.757 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.758 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.759 DEBUG:boto:Params: {} 2022-09-03T07:00:31.760 DEBUG:boto:Token: None 2022-09-03T07:00:31.761 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b 2022-09-03T07:00:31.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fd2RPxMer5qyXx5luQxP9uBxPEw= 2022-09-03T07:00:31.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fd2RPxMer5qyXx5luQxP9uBxPEw=', 'Content-Length': '0'} 2022-09-03T07:00:31.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e4d2d63a08b66c4-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.767 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:00:31.768 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.769 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.769 DEBUG:boto:Method: GET 2022-09-03T07:00:31.771 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:31.771 DEBUG:boto:Data: 2022-09-03T07:00:31.772 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.773 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.773 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.774 DEBUG:boto:Params: {} 2022-09-03T07:00:31.775 DEBUG:boto:Token: None 2022-09-03T07:00:31.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b 2022-09-03T07:00:31.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oU/ffkFs9/Wx2e0Yx/gxwElx+HM= 2022-09-03T07:00:31.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oU/ffkFs9/Wx2e0Yx/gxwElx+HM=', 'Content-Length': '0'} 2022-09-03T07:00:31.780 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f87ff1614d95688-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.781 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:31.781 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:31.782 DEBUG:boto:Method: GET 2022-09-03T07:00:31.783 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:31.784 DEBUG:boto:Data: 2022-09-03T07:00:31.784 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.785 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.786 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.787 DEBUG:boto:Params: {} 2022-09-03T07:00:31.787 DEBUG:boto:Token: None 2022-09-03T07:00:31.788 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/b 2022-09-03T07:00:31.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oU/ffkFs9/Wx2e0Yx/gxwElx+HM= 2022-09-03T07:00:31.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oU/ffkFs9/Wx2e0Yx/gxwElx+HM=', 'Content-Length': '0'} 2022-09-03T07:00:31.792 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000436a8a31018450e9-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.834 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.835 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.836 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:31.836 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:31.837 DEBUG:boto:Method: GET 2022-09-03T07:00:31.837 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:31.838 DEBUG:boto:Data: 2022-09-03T07:00:31.838 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.839 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.839 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.840 DEBUG:boto:Params: {} 2022-09-03T07:00:31.840 DEBUG:boto:Token: None 2022-09-03T07:00:31.841 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.845 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ffdebbf1f093028b-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.846 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.847 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.848 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.848 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:31.849 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:31.849 DEBUG:boto:Method: GET 2022-09-03T07:00:31.850 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:31.850 DEBUG:boto:Data: 2022-09-03T07:00:31.851 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.852 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.852 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.853 DEBUG:boto:Params: {} 2022-09-03T07:00:31.853 DEBUG:boto:Token: None 2022-09-03T07:00:31.854 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.858 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b4f0f6c01bee5a49-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.858 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.859 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:00:31.860 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:31.860 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:31.861 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.861 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.862 DEBUG:boto:Method: GET 2022-09-03T07:00:31.862 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:31.863 DEBUG:boto:Data: 2022-09-03T07:00:31.863 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.864 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.865 DEBUG:boto:Params: {} 2022-09-03T07:00:31.865 DEBUG:boto:Token: None 2022-09-03T07:00:31.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:31.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aKfDHqoWu7godEBamjsH8K767fc= 2022-09-03T07:00:31.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aKfDHqoWu7godEBamjsH8K767fc=', 'Content-Length': '0'} 2022-09-03T07:00:31.870 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001992a3f26e48d77a-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.871 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:31.871 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:31.872 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.872 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.873 DEBUG:boto:Method: GET 2022-09-03T07:00:31.873 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:31.874 DEBUG:boto:Data: 2022-09-03T07:00:31.874 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.876 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.876 DEBUG:boto:Params: {} 2022-09-03T07:00:31.877 DEBUG:boto:Token: None 2022-09-03T07:00:31.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:31.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aKfDHqoWu7godEBamjsH8K767fc= 2022-09-03T07:00:31.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aKfDHqoWu7godEBamjsH8K767fc=', 'Content-Length': '0'} 2022-09-03T07:00:31.881 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe9b396a8744313a-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.882 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:31.883 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:31.883 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.884 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.885 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.885 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:31.886 DEBUG:boto:Data: 2022-09-03T07:00:31.886 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.887 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.887 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.888 DEBUG:boto:Params: {} 2022-09-03T07:00:31.888 DEBUG:boto:Token: None 2022-09-03T07:00:31.889 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:31.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P8MO9cw0JEQlnZloJmd9Z0gl1bc= 2022-09-03T07:00:31.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P8MO9cw0JEQlnZloJmd9Z0gl1bc=', 'Content-Length': '0'} 2022-09-03T07:00:31.892 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd9f3cad4c1828f0-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.893 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:31.894 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:31.894 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.895 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.895 DEBUG:boto:Method: HEAD 2022-09-03T07:00:31.896 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:31.897 DEBUG:boto:Data: 2022-09-03T07:00:31.897 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.898 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.898 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.899 DEBUG:boto:Params: {} 2022-09-03T07:00:31.899 DEBUG:boto:Token: None 2022-09-03T07:00:31.900 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:31.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P8MO9cw0JEQlnZloJmd9Z0gl1bc= 2022-09-03T07:00:31.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P8MO9cw0JEQlnZloJmd9Z0gl1bc=', 'Content-Length': '0'} 2022-09-03T07:00:31.903 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d9c965ad035bffb-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.904 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:00:31.905 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:31.905 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:31.906 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.906 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.907 DEBUG:boto:Method: GET 2022-09-03T07:00:31.907 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:31.908 DEBUG:boto:Data: 2022-09-03T07:00:31.909 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.910 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.910 DEBUG:boto:Params: {} 2022-09-03T07:00:31.911 DEBUG:boto:Token: None 2022-09-03T07:00:31.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:31.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aKfDHqoWu7godEBamjsH8K767fc= 2022-09-03T07:00:31.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aKfDHqoWu7godEBamjsH8K767fc=', 'Content-Length': '0'} 2022-09-03T07:00:31.915 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d2d0e72cd6b242bd-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.958 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:31.959 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:31.959 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.960 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:31.960 DEBUG:boto:Method: GET 2022-09-03T07:00:31.961 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:31.962 DEBUG:boto:Data: 2022-09-03T07:00:31.963 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.964 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.965 DEBUG:boto:Params: {} 2022-09-03T07:00:31.966 DEBUG:boto:Token: None 2022-09-03T07:00:31.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:31.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aKfDHqoWu7godEBamjsH8K767fc= 2022-09-03T07:00:31.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aKfDHqoWu7godEBamjsH8K767fc=', 'Content-Length': '0'} 2022-09-03T07:00:31.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000250a3f39f3519e0e-006312fb8f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.974 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.975 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.976 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:31.976 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:31.977 DEBUG:boto:Method: GET 2022-09-03T07:00:31.977 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:31.978 DEBUG:boto:Data: 2022-09-03T07:00:31.978 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:31.980 DEBUG:boto:Port: 8001 2022-09-03T07:00:31.980 DEBUG:boto:Params: {} 2022-09-03T07:00:31.981 DEBUG:boto:Token: None 2022-09-03T07:00:31.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:31.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:31.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:31.987 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036f4e62cb4944bad-006312fb8f-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:31.988 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:31.989 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:31.989 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:31.990 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:31.990 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:31.991 DEBUG:boto:Method: GET 2022-09-03T07:00:31.992 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:31.992 DEBUG:boto:Data: 2022-09-03T07:00:31.993 DEBUG:boto:Headers: {} 2022-09-03T07:00:31.994 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:31.995 DEBUG:boto:Port: 8000 2022-09-03T07:00:31.996 DEBUG:boto:Params: {} 2022-09-03T07:00:31.998 DEBUG:boto:Token: None 2022-09-03T07:00:31.999 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:31 GMT /avysmw-72/ 2022-09-03T07:00:32.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs= 2022-09-03T07:00:32.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bdI1vitukXIHeURK+DijuicxJHs=', 'Content-Length': '0'} 2022-09-03T07:00:32.004 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b90e91ab72c3a915-006312fb90-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.005 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.006 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:32.006 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:32.007 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.008 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:32.008 DEBUG:boto:Data: 2022-09-03T07:00:32.009 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.010 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.010 DEBUG:boto:Params: {} 2022-09-03T07:00:32.011 DEBUG:boto:Token: None 2022-09-03T07:00:32.011 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/c 2022-09-03T07:00:32.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3xgCCGKLbllox8NwE275HkjKM0w= 2022-09-03T07:00:32.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3xgCCGKLbllox8NwE275HkjKM0w=', 'Content-Length': '0'} 2022-09-03T07:00:32.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036ce876086fa3ee8-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.016 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:32.017 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:32.018 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.018 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:32.019 DEBUG:boto:Data: 2022-09-03T07:00:32.020 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.021 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.022 DEBUG:boto:Params: {} 2022-09-03T07:00:32.022 DEBUG:boto:Token: None 2022-09-03T07:00:32.023 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/c 2022-09-03T07:00:32.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3xgCCGKLbllox8NwE275HkjKM0w= 2022-09-03T07:00:32.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3xgCCGKLbllox8NwE275HkjKM0w=', 'Content-Length': '0'} 2022-09-03T07:00:32.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3baef37fc7689ea-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.028 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:00:32.028 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:32.029 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:32.029 DEBUG:boto:Method: GET 2022-09-03T07:00:32.030 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:32.030 DEBUG:boto:Data: 2022-09-03T07:00:32.031 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.032 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.033 DEBUG:boto:Params: {} 2022-09-03T07:00:32.033 DEBUG:boto:Token: None 2022-09-03T07:00:32.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/c 2022-09-03T07:00:32.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ED3QQv9q1aqpiQIV+7BOwIP0IRg= 2022-09-03T07:00:32.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ED3QQv9q1aqpiQIV+7BOwIP0IRg=', 'Content-Length': '0'} 2022-09-03T07:00:32.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec1890e0d140c859-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.038 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:32.039 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:32.040 DEBUG:boto:Method: GET 2022-09-03T07:00:32.040 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:32.041 DEBUG:boto:Data: 2022-09-03T07:00:32.041 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.042 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.042 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.043 DEBUG:boto:Params: {} 2022-09-03T07:00:32.044 DEBUG:boto:Token: None 2022-09-03T07:00:32.044 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/c 2022-09-03T07:00:32.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ED3QQv9q1aqpiQIV+7BOwIP0IRg= 2022-09-03T07:00:32.046 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ED3QQv9q1aqpiQIV+7BOwIP0IRg=', 'Content-Length': '0'} 2022-09-03T07:00:32.048 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063485da03ad59e54-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.049 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.050 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.050 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:32.051 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:32.051 DEBUG:boto:Method: GET 2022-09-03T07:00:32.052 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:32.052 DEBUG:boto:Data: 2022-09-03T07:00:32.053 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.053 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.054 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.054 DEBUG:boto:Params: {} 2022-09-03T07:00:32.055 DEBUG:boto:Token: None 2022-09-03T07:00:32.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.057 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.060 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003354c811b8e3c327-006312fb90-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.061 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.062 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.063 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.063 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:32.064 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:32.064 DEBUG:boto:Method: GET 2022-09-03T07:00:32.065 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:32.065 DEBUG:boto:Data: 2022-09-03T07:00:32.066 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.066 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.067 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.067 DEBUG:boto:Params: {} 2022-09-03T07:00:32.068 DEBUG:boto:Token: None 2022-09-03T07:00:32.069 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.073 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000062c9fa2ae967bd5b-006312fb90-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.074 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.074 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:00:32.075 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.076 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.076 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.077 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.077 DEBUG:boto:Method: GET 2022-09-03T07:00:32.078 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:32.078 DEBUG:boto:Data: 2022-09-03T07:00:32.079 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.080 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.080 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.081 DEBUG:boto:Params: {} 2022-09-03T07:00:32.081 DEBUG:boto:Token: None 2022-09-03T07:00:32.082 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:32.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rYRw2PPhGDuXvbgl07Oxa5ujkb0= 2022-09-03T07:00:32.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rYRw2PPhGDuXvbgl07Oxa5ujkb0=', 'Content-Length': '0'} 2022-09-03T07:00:32.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026f8b3bc07ddd39d-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.086 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.087 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.088 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.088 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.089 DEBUG:boto:Method: GET 2022-09-03T07:00:32.089 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:32.090 DEBUG:boto:Data: 2022-09-03T07:00:32.090 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.091 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.091 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.092 DEBUG:boto:Params: {} 2022-09-03T07:00:32.092 DEBUG:boto:Token: None 2022-09-03T07:00:32.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:32.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rYRw2PPhGDuXvbgl07Oxa5ujkb0= 2022-09-03T07:00:32.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rYRw2PPhGDuXvbgl07Oxa5ujkb0=', 'Content-Length': '0'} 2022-09-03T07:00:32.097 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4bfc39ec08b5d7e-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.098 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.098 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.099 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.099 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.100 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.100 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:32.101 DEBUG:boto:Data: 2022-09-03T07:00:32.101 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.102 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.102 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.103 DEBUG:boto:Params: {} 2022-09-03T07:00:32.104 DEBUG:boto:Token: None 2022-09-03T07:00:32.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:32.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ijpr6ZpCS+p6zB0zPY2d09qP30Q= 2022-09-03T07:00:32.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ijpr6ZpCS+p6zB0zPY2d09qP30Q=', 'Content-Length': '0'} 2022-09-03T07:00:32.107 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066082cbead441b77-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.108 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.109 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.109 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.110 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.111 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.111 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:32.112 DEBUG:boto:Data: 2022-09-03T07:00:32.112 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.113 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.113 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.114 DEBUG:boto:Params: {} 2022-09-03T07:00:32.115 DEBUG:boto:Token: None 2022-09-03T07:00:32.115 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:32.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ijpr6ZpCS+p6zB0zPY2d09qP30Q= 2022-09-03T07:00:32.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ijpr6ZpCS+p6zB0zPY2d09qP30Q=', 'Content-Length': '0'} 2022-09-03T07:00:32.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000403006eb217748-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.119 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:00:32.120 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.121 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.121 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.122 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.122 DEBUG:boto:Method: GET 2022-09-03T07:00:32.123 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:32.123 DEBUG:boto:Data: 2022-09-03T07:00:32.124 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.124 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.125 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.125 DEBUG:boto:Params: {} 2022-09-03T07:00:32.126 DEBUG:boto:Token: None 2022-09-03T07:00:32.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:32.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rYRw2PPhGDuXvbgl07Oxa5ujkb0= 2022-09-03T07:00:32.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rYRw2PPhGDuXvbgl07Oxa5ujkb0=', 'Content-Length': '0'} 2022-09-03T07:00:32.130 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ecbb285a33d9491-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.131 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.131 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.132 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.132 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:32.133 DEBUG:boto:Method: GET 2022-09-03T07:00:32.133 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:32.134 DEBUG:boto:Data: 2022-09-03T07:00:32.134 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.136 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.136 DEBUG:boto:Params: {} 2022-09-03T07:00:32.137 DEBUG:boto:Token: None 2022-09-03T07:00:32.138 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:32.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rYRw2PPhGDuXvbgl07Oxa5ujkb0= 2022-09-03T07:00:32.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rYRw2PPhGDuXvbgl07Oxa5ujkb0=', 'Content-Length': '0'} 2022-09-03T07:00:32.141 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d73821b5ef870c7-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.186 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.187 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.188 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:32.189 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:32.190 DEBUG:boto:Method: GET 2022-09-03T07:00:32.190 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:32.191 DEBUG:boto:Data: 2022-09-03T07:00:32.191 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.192 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.193 DEBUG:boto:Params: {} 2022-09-03T07:00:32.194 DEBUG:boto:Token: None 2022-09-03T07:00:32.194 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.198 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001f5010248567fb77-006312fb90-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.199 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.200 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.200 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.201 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:32.201 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:32.202 DEBUG:boto:Method: GET 2022-09-03T07:00:32.202 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:32.203 DEBUG:boto:Data: 2022-09-03T07:00:32.203 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.204 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.204 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.205 DEBUG:boto:Params: {} 2022-09-03T07:00:32.206 DEBUG:boto:Token: None 2022-09-03T07:00:32.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.210 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b580a6044c27122a-006312fb90-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.211 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.211 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.212 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.213 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.214 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:32.215 DEBUG:boto:Data: 2022-09-03T07:00:32.216 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.216 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.217 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.217 DEBUG:boto:Params: {} 2022-09-03T07:00:32.218 DEBUG:boto:Token: None 2022-09-03T07:00:32.219 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d 2022-09-03T07:00:32.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n1Yv1mZHcalf70st9Cqu6wRw4J0= 2022-09-03T07:00:32.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n1Yv1mZHcalf70st9Cqu6wRw4J0=', 'Content-Length': '0'} 2022-09-03T07:00:32.222 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000160214efed513599-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.223 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.224 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.225 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.225 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:32.226 DEBUG:boto:Data: 2022-09-03T07:00:32.227 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.227 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.228 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.228 DEBUG:boto:Params: {} 2022-09-03T07:00:32.229 DEBUG:boto:Token: None 2022-09-03T07:00:32.229 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d 2022-09-03T07:00:32.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n1Yv1mZHcalf70st9Cqu6wRw4J0= 2022-09-03T07:00:32.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n1Yv1mZHcalf70st9Cqu6wRw4J0=', 'Content-Length': '0'} 2022-09-03T07:00:32.234 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b3a7561a8f82671-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.235 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:00:32.236 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.236 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.237 DEBUG:boto:Method: GET 2022-09-03T07:00:32.237 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:32.239 DEBUG:boto:Data: 2022-09-03T07:00:32.239 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.240 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.241 DEBUG:boto:Params: {} 2022-09-03T07:00:32.241 DEBUG:boto:Token: None 2022-09-03T07:00:32.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d 2022-09-03T07:00:32.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1m3mpFjlJL+jPQgGydgk4fhQTk= 2022-09-03T07:00:32.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1m3mpFjlJL+jPQgGydgk4fhQTk=', 'Content-Length': '0'} 2022-09-03T07:00:32.246 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d2df0ef9193ebf6-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.286 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:32.287 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:32.287 DEBUG:boto:Method: GET 2022-09-03T07:00:32.288 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:32.289 DEBUG:boto:Data: 2022-09-03T07:00:32.290 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.290 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.291 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.291 DEBUG:boto:Params: {} 2022-09-03T07:00:32.292 DEBUG:boto:Token: None 2022-09-03T07:00:32.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/d 2022-09-03T07:00:32.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1m3mpFjlJL+jPQgGydgk4fhQTk= 2022-09-03T07:00:32.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1m3mpFjlJL+jPQgGydgk4fhQTk=', 'Content-Length': '0'} 2022-09-03T07:00:32.296 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089007711d712b7a0-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.297 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.297 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.298 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:32.298 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:32.299 DEBUG:boto:Method: GET 2022-09-03T07:00:32.300 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:32.300 DEBUG:boto:Data: 2022-09-03T07:00:32.301 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.301 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.302 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.302 DEBUG:boto:Params: {} 2022-09-03T07:00:32.303 DEBUG:boto:Token: None 2022-09-03T07:00:32.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.308 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bbfdd4e43b534592-006312fb90-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.309 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.310 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.310 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.311 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:32.312 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:32.312 DEBUG:boto:Method: GET 2022-09-03T07:00:32.313 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:32.313 DEBUG:boto:Data: 2022-09-03T07:00:32.314 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.315 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.315 DEBUG:boto:Params: {} 2022-09-03T07:00:32.316 DEBUG:boto:Token: None 2022-09-03T07:00:32.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.320 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000393a64bcec6e93a1-006312fb90-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.321 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.322 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:00:32.322 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.323 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.323 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.324 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.325 DEBUG:boto:Method: GET 2022-09-03T07:00:32.325 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:32.326 DEBUG:boto:Data: 2022-09-03T07:00:32.327 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.328 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.329 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.329 DEBUG:boto:Params: {} 2022-09-03T07:00:32.330 DEBUG:boto:Token: None 2022-09-03T07:00:32.331 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:32.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yi8Qz3FvDaJVWdGZ7PcK4uQwcbE= 2022-09-03T07:00:32.332 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yi8Qz3FvDaJVWdGZ7PcK4uQwcbE=', 'Content-Length': '0'} 2022-09-03T07:00:32.334 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f3d17f04aeb6be6-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.335 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.336 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.337 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.337 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.338 DEBUG:boto:Method: GET 2022-09-03T07:00:32.339 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:32.339 DEBUG:boto:Data: 2022-09-03T07:00:32.340 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.341 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.341 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.342 DEBUG:boto:Params: {} 2022-09-03T07:00:32.342 DEBUG:boto:Token: None 2022-09-03T07:00:32.343 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:32.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yi8Qz3FvDaJVWdGZ7PcK4uQwcbE= 2022-09-03T07:00:32.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yi8Qz3FvDaJVWdGZ7PcK4uQwcbE=', 'Content-Length': '0'} 2022-09-03T07:00:32.348 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008bc445c5d672d824-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.349 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.349 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.350 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.351 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.351 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.352 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:32.353 DEBUG:boto:Data: 2022-09-03T07:00:32.353 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.354 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.354 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.355 DEBUG:boto:Params: {} 2022-09-03T07:00:32.356 DEBUG:boto:Token: None 2022-09-03T07:00:32.357 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:32.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cIVDBNCCdUnlzJffAprjNts/ql4= 2022-09-03T07:00:32.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cIVDBNCCdUnlzJffAprjNts/ql4=', 'Content-Length': '0'} 2022-09-03T07:00:32.361 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed86ae1614075a22-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.362 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.363 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.364 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.364 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.365 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.366 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:32.366 DEBUG:boto:Data: 2022-09-03T07:00:32.367 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.368 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.369 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.370 DEBUG:boto:Params: {} 2022-09-03T07:00:32.373 DEBUG:boto:Token: None 2022-09-03T07:00:32.374 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:32.374 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cIVDBNCCdUnlzJffAprjNts/ql4= 2022-09-03T07:00:32.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cIVDBNCCdUnlzJffAprjNts/ql4=', 'Content-Length': '0'} 2022-09-03T07:00:32.377 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ffa42e8ce94b741d-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.378 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:00:32.379 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.379 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.380 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.381 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.381 DEBUG:boto:Method: GET 2022-09-03T07:00:32.382 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:32.383 DEBUG:boto:Data: 2022-09-03T07:00:32.384 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.385 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.387 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.387 DEBUG:boto:Params: {} 2022-09-03T07:00:32.388 DEBUG:boto:Token: None 2022-09-03T07:00:32.390 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:32.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yi8Qz3FvDaJVWdGZ7PcK4uQwcbE= 2022-09-03T07:00:32.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yi8Qz3FvDaJVWdGZ7PcK4uQwcbE=', 'Content-Length': '0'} 2022-09-03T07:00:32.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000554a6e97be1d1e05-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.398 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.400 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.401 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.401 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:32.402 DEBUG:boto:Method: GET 2022-09-03T07:00:32.403 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:32.403 DEBUG:boto:Data: 2022-09-03T07:00:32.404 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.404 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.405 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.405 DEBUG:boto:Params: {} 2022-09-03T07:00:32.406 DEBUG:boto:Token: None 2022-09-03T07:00:32.407 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:32.407 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yi8Qz3FvDaJVWdGZ7PcK4uQwcbE= 2022-09-03T07:00:32.408 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yi8Qz3FvDaJVWdGZ7PcK4uQwcbE=', 'Content-Length': '0'} 2022-09-03T07:00:32.412 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a83825415b5572f-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.454 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.455 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.456 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:32.457 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:32.458 DEBUG:boto:Method: GET 2022-09-03T07:00:32.459 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:32.459 DEBUG:boto:Data: 2022-09-03T07:00:32.460 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.462 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.463 DEBUG:boto:Params: {} 2022-09-03T07:00:32.464 DEBUG:boto:Token: None 2022-09-03T07:00:32.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.470 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d044837e431b242f-006312fb90-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.471 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.472 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.472 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.473 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:32.473 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:32.474 DEBUG:boto:Method: GET 2022-09-03T07:00:32.474 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:32.475 DEBUG:boto:Data: 2022-09-03T07:00:32.475 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.477 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.477 DEBUG:boto:Params: {} 2022-09-03T07:00:32.478 DEBUG:boto:Token: None 2022-09-03T07:00:32.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.484 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f6a13d9dcd2778c9-006312fb90-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.484 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.485 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.486 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.486 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.487 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:32.487 DEBUG:boto:Data: 2022-09-03T07:00:32.488 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.489 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.489 DEBUG:boto:Params: {} 2022-09-03T07:00:32.490 DEBUG:boto:Token: None 2022-09-03T07:00:32.491 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e 2022-09-03T07:00:32.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EynaDsL1zxwZeag2cHp2FxTTX8= 2022-09-03T07:00:32.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EynaDsL1zxwZeag2cHp2FxTTX8=', 'Content-Length': '0'} 2022-09-03T07:00:32.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001833c92dfeeb0bc5-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.496 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.496 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.497 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.498 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:32.498 DEBUG:boto:Data: 2022-09-03T07:00:32.499 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.501 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.501 DEBUG:boto:Params: {} 2022-09-03T07:00:32.502 DEBUG:boto:Token: None 2022-09-03T07:00:32.502 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e 2022-09-03T07:00:32.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EynaDsL1zxwZeag2cHp2FxTTX8= 2022-09-03T07:00:32.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EynaDsL1zxwZeag2cHp2FxTTX8=', 'Content-Length': '0'} 2022-09-03T07:00:32.506 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053490a857380b71d-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.507 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:00:32.507 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.508 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.508 DEBUG:boto:Method: GET 2022-09-03T07:00:32.509 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:32.510 DEBUG:boto:Data: 2022-09-03T07:00:32.510 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.511 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.511 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.512 DEBUG:boto:Params: {} 2022-09-03T07:00:32.512 DEBUG:boto:Token: None 2022-09-03T07:00:32.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e 2022-09-03T07:00:32.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ul8ekGSD+uYYdcVlCpwS06kgOKA= 2022-09-03T07:00:32.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ul8ekGSD+uYYdcVlCpwS06kgOKA=', 'Content-Length': '0'} 2022-09-03T07:00:32.517 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004fac104e94163a82-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.558 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:32.559 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:32.560 DEBUG:boto:Method: GET 2022-09-03T07:00:32.560 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:32.561 DEBUG:boto:Data: 2022-09-03T07:00:32.561 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.562 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.563 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.563 DEBUG:boto:Params: {} 2022-09-03T07:00:32.564 DEBUG:boto:Token: None 2022-09-03T07:00:32.564 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/e 2022-09-03T07:00:32.565 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ul8ekGSD+uYYdcVlCpwS06kgOKA= 2022-09-03T07:00:32.566 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ul8ekGSD+uYYdcVlCpwS06kgOKA=', 'Content-Length': '0'} 2022-09-03T07:00:32.569 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d34a8e403a7362a-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.570 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.570 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.571 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:32.572 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:32.573 DEBUG:boto:Method: GET 2022-09-03T07:00:32.573 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:32.574 DEBUG:boto:Data: 2022-09-03T07:00:32.574 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.575 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.575 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.576 DEBUG:boto:Params: {} 2022-09-03T07:00:32.576 DEBUG:boto:Token: None 2022-09-03T07:00:32.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.577 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.581 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008dcd76fe214abc6a-006312fb90-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.582 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.583 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.583 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.584 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:32.584 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:32.585 DEBUG:boto:Method: GET 2022-09-03T07:00:32.585 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:32.586 DEBUG:boto:Data: 2022-09-03T07:00:32.586 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.587 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.588 DEBUG:boto:Params: {} 2022-09-03T07:00:32.588 DEBUG:boto:Token: None 2022-09-03T07:00:32.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.593 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b801566d5f9fce15-006312fb90-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.594 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.594 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:00:32.595 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.595 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.596 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.597 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.597 DEBUG:boto:Method: GET 2022-09-03T07:00:32.598 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:32.598 DEBUG:boto:Data: 2022-09-03T07:00:32.599 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.600 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.600 DEBUG:boto:Params: {} 2022-09-03T07:00:32.601 DEBUG:boto:Token: None 2022-09-03T07:00:32.601 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:32.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dnI2PcsjcgmL+bSrOV84L8jap1s= 2022-09-03T07:00:32.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dnI2PcsjcgmL+bSrOV84L8jap1s=', 'Content-Length': '0'} 2022-09-03T07:00:32.605 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db0418cfe7e50d44-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.606 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.607 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.608 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.609 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.609 DEBUG:boto:Method: GET 2022-09-03T07:00:32.610 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:32.611 DEBUG:boto:Data: 2022-09-03T07:00:32.611 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.613 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.613 DEBUG:boto:Params: {} 2022-09-03T07:00:32.614 DEBUG:boto:Token: None 2022-09-03T07:00:32.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:32.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dnI2PcsjcgmL+bSrOV84L8jap1s= 2022-09-03T07:00:32.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dnI2PcsjcgmL+bSrOV84L8jap1s=', 'Content-Length': '0'} 2022-09-03T07:00:32.619 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c2ec18ff290f1c5-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.620 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.621 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.621 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.623 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.624 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.624 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:32.625 DEBUG:boto:Data: 2022-09-03T07:00:32.625 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.626 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.627 DEBUG:boto:Params: {} 2022-09-03T07:00:32.628 DEBUG:boto:Token: None 2022-09-03T07:00:32.628 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:32.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yJk6ds6c0iymfXfSw9gnjNceyL0= 2022-09-03T07:00:32.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yJk6ds6c0iymfXfSw9gnjNceyL0=', 'Content-Length': '0'} 2022-09-03T07:00:32.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3eff58ace05e0f0-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.633 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.633 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.634 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.634 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.635 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.635 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:32.636 DEBUG:boto:Data: 2022-09-03T07:00:32.636 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.637 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.637 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.639 DEBUG:boto:Params: {} 2022-09-03T07:00:32.640 DEBUG:boto:Token: None 2022-09-03T07:00:32.640 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:32.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yJk6ds6c0iymfXfSw9gnjNceyL0= 2022-09-03T07:00:32.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yJk6ds6c0iymfXfSw9gnjNceyL0=', 'Content-Length': '0'} 2022-09-03T07:00:32.644 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001bf88b2dc51123a3-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.645 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:00:32.645 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.646 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.646 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.647 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.648 DEBUG:boto:Method: GET 2022-09-03T07:00:32.649 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:32.650 DEBUG:boto:Data: 2022-09-03T07:00:32.651 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.651 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.652 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.653 DEBUG:boto:Params: {} 2022-09-03T07:00:32.654 DEBUG:boto:Token: None 2022-09-03T07:00:32.655 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:32.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dnI2PcsjcgmL+bSrOV84L8jap1s= 2022-09-03T07:00:32.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dnI2PcsjcgmL+bSrOV84L8jap1s=', 'Content-Length': '0'} 2022-09-03T07:00:32.660 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0ccbceaffde332e-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.662 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.663 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.663 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.664 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:32.665 DEBUG:boto:Method: GET 2022-09-03T07:00:32.666 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:32.667 DEBUG:boto:Data: 2022-09-03T07:00:32.668 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.669 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.670 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.671 DEBUG:boto:Params: {} 2022-09-03T07:00:32.672 DEBUG:boto:Token: None 2022-09-03T07:00:32.673 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:32.674 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dnI2PcsjcgmL+bSrOV84L8jap1s= 2022-09-03T07:00:32.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dnI2PcsjcgmL+bSrOV84L8jap1s=', 'Content-Length': '0'} 2022-09-03T07:00:32.678 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d913558ba4604a5-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.679 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.680 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.681 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:32.682 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:32.683 DEBUG:boto:Method: GET 2022-09-03T07:00:32.684 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:32.685 DEBUG:boto:Data: 2022-09-03T07:00:32.685 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.687 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.688 DEBUG:boto:Params: {} 2022-09-03T07:00:32.688 DEBUG:boto:Token: None 2022-09-03T07:00:32.689 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.693 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b72ae87fe7865e4-006312fb90-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.694 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.695 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.695 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.696 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:32.696 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:32.697 DEBUG:boto:Method: GET 2022-09-03T07:00:32.697 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:32.698 DEBUG:boto:Data: 2022-09-03T07:00:32.699 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.700 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.700 DEBUG:boto:Params: {} 2022-09-03T07:00:32.701 DEBUG:boto:Token: None 2022-09-03T07:00:32.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.706 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c63e75e344649873-006312fb90-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.707 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.708 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.709 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.710 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.710 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:32.711 DEBUG:boto:Data: 2022-09-03T07:00:32.711 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.712 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.713 DEBUG:boto:Params: {} 2022-09-03T07:00:32.714 DEBUG:boto:Token: None 2022-09-03T07:00:32.714 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f 2022-09-03T07:00:32.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/l5cY+ewjwHpjRMfNxqLrFi0BM= 2022-09-03T07:00:32.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/l5cY+ewjwHpjRMfNxqLrFi0BM=', 'Content-Length': '0'} 2022-09-03T07:00:32.718 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000881bdb5c99501564-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.718 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.719 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.720 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.721 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:32.721 DEBUG:boto:Data: 2022-09-03T07:00:32.722 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.723 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.723 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.724 DEBUG:boto:Params: {} 2022-09-03T07:00:32.725 DEBUG:boto:Token: None 2022-09-03T07:00:32.726 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f 2022-09-03T07:00:32.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/l5cY+ewjwHpjRMfNxqLrFi0BM= 2022-09-03T07:00:32.728 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/l5cY+ewjwHpjRMfNxqLrFi0BM=', 'Content-Length': '0'} 2022-09-03T07:00:32.731 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f2d67e39f49bf5fd-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.732 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:00:32.733 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.734 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.735 DEBUG:boto:Method: GET 2022-09-03T07:00:32.736 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:32.737 DEBUG:boto:Data: 2022-09-03T07:00:32.738 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.740 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.741 DEBUG:boto:Params: {} 2022-09-03T07:00:32.742 DEBUG:boto:Token: None 2022-09-03T07:00:32.743 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f 2022-09-03T07:00:32.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bP8vpzbVI2FDaG5R5WS7D93j/jk= 2022-09-03T07:00:32.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bP8vpzbVI2FDaG5R5WS7D93j/jk=', 'Content-Length': '0'} 2022-09-03T07:00:32.747 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055af70f00b82d3dc-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.748 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:32.749 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:32.750 DEBUG:boto:Method: GET 2022-09-03T07:00:32.750 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:32.751 DEBUG:boto:Data: 2022-09-03T07:00:32.751 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.752 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.752 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.753 DEBUG:boto:Params: {} 2022-09-03T07:00:32.753 DEBUG:boto:Token: None 2022-09-03T07:00:32.754 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/f 2022-09-03T07:00:32.755 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bP8vpzbVI2FDaG5R5WS7D93j/jk= 2022-09-03T07:00:32.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bP8vpzbVI2FDaG5R5WS7D93j/jk=', 'Content-Length': '0'} 2022-09-03T07:00:32.758 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010acdfc148790530-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.798 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.799 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.800 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:32.800 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:32.801 DEBUG:boto:Method: GET 2022-09-03T07:00:32.801 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:32.802 DEBUG:boto:Data: 2022-09-03T07:00:32.802 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.803 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.804 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.804 DEBUG:boto:Params: {} 2022-09-03T07:00:32.805 DEBUG:boto:Token: None 2022-09-03T07:00:32.805 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.810 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006324c22777fc03db-006312fb90-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.810 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.811 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.812 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.812 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:32.813 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:32.813 DEBUG:boto:Method: GET 2022-09-03T07:00:32.814 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:32.815 DEBUG:boto:Data: 2022-09-03T07:00:32.815 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.817 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.818 DEBUG:boto:Params: {} 2022-09-03T07:00:32.818 DEBUG:boto:Token: None 2022-09-03T07:00:32.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.823 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001dcd071407e3a7ff-006312fb90-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.824 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.825 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:00:32.825 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:32.826 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:32.827 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.827 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.828 DEBUG:boto:Method: GET 2022-09-03T07:00:32.828 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:32.829 DEBUG:boto:Data: 2022-09-03T07:00:32.829 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.831 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.831 DEBUG:boto:Params: {} 2022-09-03T07:00:32.832 DEBUG:boto:Token: None 2022-09-03T07:00:32.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:32.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U63rfzFxcoq17CmY/Jn/Y26aKo8= 2022-09-03T07:00:32.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U63rfzFxcoq17CmY/Jn/Y26aKo8=', 'Content-Length': '0'} 2022-09-03T07:00:32.836 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a7629939d712ac3-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.837 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:32.838 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:32.838 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.839 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.839 DEBUG:boto:Method: GET 2022-09-03T07:00:32.840 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:32.841 DEBUG:boto:Data: 2022-09-03T07:00:32.841 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.843 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.844 DEBUG:boto:Params: {} 2022-09-03T07:00:32.844 DEBUG:boto:Token: None 2022-09-03T07:00:32.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:32.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U63rfzFxcoq17CmY/Jn/Y26aKo8= 2022-09-03T07:00:32.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U63rfzFxcoq17CmY/Jn/Y26aKo8=', 'Content-Length': '0'} 2022-09-03T07:00:32.849 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e32ed3cca1f2b42-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.850 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:32.851 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:32.851 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.852 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.853 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.854 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:32.855 DEBUG:boto:Data: 2022-09-03T07:00:32.856 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.858 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.859 DEBUG:boto:Params: {} 2022-09-03T07:00:32.860 DEBUG:boto:Token: None 2022-09-03T07:00:32.861 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:32.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GvVn4P4X3l5t//FgqYnXMPCRx6Q= 2022-09-03T07:00:32.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GvVn4P4X3l5t//FgqYnXMPCRx6Q=', 'Content-Length': '0'} 2022-09-03T07:00:32.867 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005526468c332ba80d-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.868 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:32.869 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:32.870 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.871 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.872 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.873 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:32.874 DEBUG:boto:Data: 2022-09-03T07:00:32.875 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.876 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.877 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.878 DEBUG:boto:Params: {} 2022-09-03T07:00:32.879 DEBUG:boto:Token: None 2022-09-03T07:00:32.880 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:32.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GvVn4P4X3l5t//FgqYnXMPCRx6Q= 2022-09-03T07:00:32.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GvVn4P4X3l5t//FgqYnXMPCRx6Q=', 'Content-Length': '0'} 2022-09-03T07:00:32.885 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a94eda44ca7ee027-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.886 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:00:32.887 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:32.888 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:32.888 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.889 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.932 DEBUG:boto:Method: GET 2022-09-03T07:00:32.933 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:32.934 DEBUG:boto:Data: 2022-09-03T07:00:32.935 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.935 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.936 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.936 DEBUG:boto:Params: {} 2022-09-03T07:00:32.937 DEBUG:boto:Token: None 2022-09-03T07:00:32.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:32.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U63rfzFxcoq17CmY/Jn/Y26aKo8= 2022-09-03T07:00:32.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U63rfzFxcoq17CmY/Jn/Y26aKo8=', 'Content-Length': '0'} 2022-09-03T07:00:32.941 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091fa080324623b24-006312fb90-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.942 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:32.943 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:32.944 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.944 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:32.945 DEBUG:boto:Method: GET 2022-09-03T07:00:32.945 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:32.946 DEBUG:boto:Data: 2022-09-03T07:00:32.947 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.948 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.948 DEBUG:boto:Params: {} 2022-09-03T07:00:32.949 DEBUG:boto:Token: None 2022-09-03T07:00:32.950 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:32.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U63rfzFxcoq17CmY/Jn/Y26aKo8= 2022-09-03T07:00:32.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U63rfzFxcoq17CmY/Jn/Y26aKo8=', 'Content-Length': '0'} 2022-09-03T07:00:32.953 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002788e8affca5feb9-006312fb90-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.954 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.955 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.955 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:32.956 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:32.957 DEBUG:boto:Method: GET 2022-09-03T07:00:32.957 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:32.958 DEBUG:boto:Data: 2022-09-03T07:00:32.959 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.961 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.962 DEBUG:boto:Params: {} 2022-09-03T07:00:32.963 DEBUG:boto:Token: None 2022-09-03T07:00:32.964 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.969 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008013722cf90e1546-006312fb90-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.970 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.972 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:32.973 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:32.973 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:32.974 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:32.975 DEBUG:boto:Method: GET 2022-09-03T07:00:32.976 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:32.977 DEBUG:boto:Data: 2022-09-03T07:00:32.978 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:32.979 DEBUG:boto:Port: 8000 2022-09-03T07:00:32.980 DEBUG:boto:Params: {} 2022-09-03T07:00:32.981 DEBUG:boto:Token: None 2022-09-03T07:00:32.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/ 2022-09-03T07:00:32.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE= 2022-09-03T07:00:32.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYve9yhB7FeLzoHC0/xct4zVRvE=', 'Content-Length': '0'} 2022-09-03T07:00:32.987 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000642045a9f4fbd016-006312fb90-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:32.988 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:32.989 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:32.989 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:32.990 DEBUG:boto:Method: HEAD 2022-09-03T07:00:32.991 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:32.992 DEBUG:boto:Data: 2022-09-03T07:00:32.993 DEBUG:boto:Headers: {} 2022-09-03T07:00:32.993 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:32.995 DEBUG:boto:Port: 8001 2022-09-03T07:00:32.996 DEBUG:boto:Params: {} 2022-09-03T07:00:32.997 DEBUG:boto:Token: None 2022-09-03T07:00:32.998 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:32 GMT /avysmw-72/g 2022-09-03T07:00:32.999 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fCWG09R3G8u3aK6lwY/vh5kRYEM= 2022-09-03T07:00:33.000 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fCWG09R3G8u3aK6lwY/vh5kRYEM=', 'Content-Length': '0'} 2022-09-03T07:00:33.003 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000852c0856870100a5-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.004 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:33.005 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:33.006 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.007 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:33.007 DEBUG:boto:Data: 2022-09-03T07:00:33.008 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.009 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.010 DEBUG:boto:Params: {} 2022-09-03T07:00:33.011 DEBUG:boto:Token: None 2022-09-03T07:00:33.011 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/g 2022-09-03T07:00:33.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1fzAApTJVJmYY9hOoaghLQD0WoY= 2022-09-03T07:00:33.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1fzAApTJVJmYY9hOoaghLQD0WoY=', 'Content-Length': '0'} 2022-09-03T07:00:33.015 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000469a3e13565af981-006312fb91-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.016 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:00:33.017 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:33.018 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:33.018 DEBUG:boto:Method: GET 2022-09-03T07:00:33.019 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:33.020 DEBUG:boto:Data: 2022-09-03T07:00:33.020 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.021 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.022 DEBUG:boto:Params: {} 2022-09-03T07:00:33.022 DEBUG:boto:Token: None 2022-09-03T07:00:33.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/g 2022-09-03T07:00:33.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QJJKSGDSQ/BfcfDMMtwfFonDOG0= 2022-09-03T07:00:33.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QJJKSGDSQ/BfcfDMMtwfFonDOG0=', 'Content-Length': '0'} 2022-09-03T07:00:33.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093b6f9f454cec13c-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.028 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:33.028 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:33.029 DEBUG:boto:Method: GET 2022-09-03T07:00:33.029 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:33.030 DEBUG:boto:Data: 2022-09-03T07:00:33.031 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.032 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.032 DEBUG:boto:Params: {} 2022-09-03T07:00:33.033 DEBUG:boto:Token: None 2022-09-03T07:00:33.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/g 2022-09-03T07:00:33.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QJJKSGDSQ/BfcfDMMtwfFonDOG0= 2022-09-03T07:00:33.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QJJKSGDSQ/BfcfDMMtwfFonDOG0=', 'Content-Length': '0'} 2022-09-03T07:00:33.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007a3c08608da59304-006312fb91-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.078 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.079 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.079 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:33.080 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:33.081 DEBUG:boto:Method: GET 2022-09-03T07:00:33.081 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:33.082 DEBUG:boto:Data: 2022-09-03T07:00:33.083 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.083 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.084 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.085 DEBUG:boto:Params: {} 2022-09-03T07:00:33.086 DEBUG:boto:Token: None 2022-09-03T07:00:33.087 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.093 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e356d321253cf54-006312fb91-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.094 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.096 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.097 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.098 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:33.099 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:33.100 DEBUG:boto:Method: GET 2022-09-03T07:00:33.104 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:33.105 DEBUG:boto:Data: 2022-09-03T07:00:33.106 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.107 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.108 DEBUG:boto:Params: {} 2022-09-03T07:00:33.109 DEBUG:boto:Token: None 2022-09-03T07:00:33.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.115 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ffded5897db2eed6-006312fb91-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.116 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.117 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:00:33.118 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.119 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.120 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.121 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.122 DEBUG:boto:Method: GET 2022-09-03T07:00:33.122 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:33.123 DEBUG:boto:Data: 2022-09-03T07:00:33.124 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.124 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.125 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.126 DEBUG:boto:Params: {} 2022-09-03T07:00:33.127 DEBUG:boto:Token: None 2022-09-03T07:00:33.128 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:33.129 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Ymj0EvkqDPl/a86GcF/Q2Y3BYQ= 2022-09-03T07:00:33.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Ymj0EvkqDPl/a86GcF/Q2Y3BYQ=', 'Content-Length': '0'} 2022-09-03T07:00:33.132 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004901c7e903d155d5-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.134 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.135 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.136 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.136 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.137 DEBUG:boto:Method: GET 2022-09-03T07:00:33.138 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:33.138 DEBUG:boto:Data: 2022-09-03T07:00:33.139 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.139 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.140 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.141 DEBUG:boto:Params: {} 2022-09-03T07:00:33.142 DEBUG:boto:Token: None 2022-09-03T07:00:33.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:33.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Ymj0EvkqDPl/a86GcF/Q2Y3BYQ= 2022-09-03T07:00:33.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Ymj0EvkqDPl/a86GcF/Q2Y3BYQ=', 'Content-Length': '0'} 2022-09-03T07:00:33.148 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad126368f0323611-006312fb91-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.148 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.149 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.150 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.150 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.151 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.151 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:33.152 DEBUG:boto:Data: 2022-09-03T07:00:33.153 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.154 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.155 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.156 DEBUG:boto:Params: {} 2022-09-03T07:00:33.157 DEBUG:boto:Token: None 2022-09-03T07:00:33.158 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:33.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g5VWZccag64PilF7IVuzkePLMjE= 2022-09-03T07:00:33.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g5VWZccag64PilF7IVuzkePLMjE=', 'Content-Length': '0'} 2022-09-03T07:00:33.163 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000afc41a7aa83f1863-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.165 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.165 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.166 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.167 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.168 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.169 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:33.170 DEBUG:boto:Data: 2022-09-03T07:00:33.171 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.173 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.174 DEBUG:boto:Params: {} 2022-09-03T07:00:33.175 DEBUG:boto:Token: None 2022-09-03T07:00:33.176 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:33.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g5VWZccag64PilF7IVuzkePLMjE= 2022-09-03T07:00:33.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g5VWZccag64PilF7IVuzkePLMjE=', 'Content-Length': '0'} 2022-09-03T07:00:33.181 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000011d37759dd33e1f3-006312fb91-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.182 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:00:33.183 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.184 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.185 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.187 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.188 DEBUG:boto:Method: GET 2022-09-03T07:00:33.189 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:33.189 DEBUG:boto:Data: 2022-09-03T07:00:33.190 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.191 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.192 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.192 DEBUG:boto:Params: {} 2022-09-03T07:00:33.193 DEBUG:boto:Token: None 2022-09-03T07:00:33.194 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:33.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Ymj0EvkqDPl/a86GcF/Q2Y3BYQ= 2022-09-03T07:00:33.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Ymj0EvkqDPl/a86GcF/Q2Y3BYQ=', 'Content-Length': '0'} 2022-09-03T07:00:33.198 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032612ae2490903f2-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.238 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.239 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.239 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.240 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:33.241 DEBUG:boto:Method: GET 2022-09-03T07:00:33.241 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:33.242 DEBUG:boto:Data: 2022-09-03T07:00:33.242 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.243 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.244 DEBUG:boto:Params: {} 2022-09-03T07:00:33.245 DEBUG:boto:Token: None 2022-09-03T07:00:33.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:33.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Ymj0EvkqDPl/a86GcF/Q2Y3BYQ= 2022-09-03T07:00:33.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Ymj0EvkqDPl/a86GcF/Q2Y3BYQ=', 'Content-Length': '0'} 2022-09-03T07:00:33.249 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001cdf25c608d59c7f-006312fb91-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.250 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.251 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.251 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:33.252 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:33.253 DEBUG:boto:Method: GET 2022-09-03T07:00:33.253 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:33.254 DEBUG:boto:Data: 2022-09-03T07:00:33.254 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.255 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.256 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.256 DEBUG:boto:Params: {} 2022-09-03T07:00:33.257 DEBUG:boto:Token: None 2022-09-03T07:00:33.257 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.262 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000928b0d0e83e2938c-006312fb91-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.262 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.263 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.264 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.264 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:33.265 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:33.266 DEBUG:boto:Method: GET 2022-09-03T07:00:33.267 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:33.267 DEBUG:boto:Data: 2022-09-03T07:00:33.268 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.269 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.270 DEBUG:boto:Params: {} 2022-09-03T07:00:33.270 DEBUG:boto:Token: None 2022-09-03T07:00:33.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.275 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc145fdc4af55ff1-006312fb91-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.276 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.276 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.277 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.278 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.278 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:33.279 DEBUG:boto:Data: 2022-09-03T07:00:33.279 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.280 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.281 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.281 DEBUG:boto:Params: {} 2022-09-03T07:00:33.282 DEBUG:boto:Token: None 2022-09-03T07:00:33.283 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h 2022-09-03T07:00:33.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kvQoPO86N+WsqPftlAAFtNj0Eso= 2022-09-03T07:00:33.284 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kvQoPO86N+WsqPftlAAFtNj0Eso=', 'Content-Length': '0'} 2022-09-03T07:00:33.287 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af529344ef3da1f8-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.288 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.288 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.289 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.289 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:33.290 DEBUG:boto:Data: 2022-09-03T07:00:33.291 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.291 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.292 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.293 DEBUG:boto:Params: {} 2022-09-03T07:00:33.293 DEBUG:boto:Token: None 2022-09-03T07:00:33.294 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h 2022-09-03T07:00:33.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kvQoPO86N+WsqPftlAAFtNj0Eso= 2022-09-03T07:00:33.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kvQoPO86N+WsqPftlAAFtNj0Eso=', 'Content-Length': '0'} 2022-09-03T07:00:33.297 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ce51892f5cda8f6-006312fb91-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.298 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:00:33.299 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.299 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.300 DEBUG:boto:Method: GET 2022-09-03T07:00:33.300 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:33.301 DEBUG:boto:Data: 2022-09-03T07:00:33.301 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.302 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.303 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.303 DEBUG:boto:Params: {} 2022-09-03T07:00:33.304 DEBUG:boto:Token: None 2022-09-03T07:00:33.305 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h 2022-09-03T07:00:33.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YLiAe0HIgFPKYpScTPiSEVzSmuE= 2022-09-03T07:00:33.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YLiAe0HIgFPKYpScTPiSEVzSmuE=', 'Content-Length': '0'} 2022-09-03T07:00:33.308 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3466f8bbd73557d-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.309 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:33.310 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:33.311 DEBUG:boto:Method: GET 2022-09-03T07:00:33.311 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:33.312 DEBUG:boto:Data: 2022-09-03T07:00:33.312 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.313 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.314 DEBUG:boto:Params: {} 2022-09-03T07:00:33.315 DEBUG:boto:Token: None 2022-09-03T07:00:33.315 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/h 2022-09-03T07:00:33.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YLiAe0HIgFPKYpScTPiSEVzSmuE= 2022-09-03T07:00:33.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YLiAe0HIgFPKYpScTPiSEVzSmuE=', 'Content-Length': '0'} 2022-09-03T07:00:33.319 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa44f4838110b45a-006312fb91-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.320 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.320 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.321 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:33.321 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:33.322 DEBUG:boto:Method: GET 2022-09-03T07:00:33.322 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:33.323 DEBUG:boto:Data: 2022-09-03T07:00:33.324 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.324 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.325 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.325 DEBUG:boto:Params: {} 2022-09-03T07:00:33.326 DEBUG:boto:Token: None 2022-09-03T07:00:33.327 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.331 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017f56362fecabdd7-006312fb91-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.331 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.332 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.333 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.333 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:33.334 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:33.335 DEBUG:boto:Method: GET 2022-09-03T07:00:33.335 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:33.336 DEBUG:boto:Data: 2022-09-03T07:00:33.336 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:33.337 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.338 DEBUG:boto:Params: {} 2022-09-03T07:00:33.339 DEBUG:boto:Token: None 2022-09-03T07:00:33.340 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.344 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008e78eb5a8f65b557-006312fb91-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.347 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.348 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-72 zones={test-zone3, test-zone1} 2022-09-03T07:00:33.349 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-72 zones={test-zone3, test-zone2} 2022-09-03T07:00:33.349 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.350 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.351 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.352 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T07:00:33.353 DEBUG:boto:Data: 2022-09-03T07:00:33.354 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.355 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.356 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.357 DEBUG:boto:Params: {} 2022-09-03T07:00:33.358 DEBUG:boto:Token: None 2022-09-03T07:00:33.359 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lBES8sCezTSmJ7Fpb/mbEjZGtlw= 2022-09-03T07:00:33.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lBES8sCezTSmJ7Fpb/mbEjZGtlw=', 'Content-Length': '0'} 2022-09-03T07:00:33.364 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000026e58662b21deb2a-006312fb91-1608-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.364 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.365 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.366 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.367 DEBUG:boto:Path: /avysmw-72/ 2022-09-03T07:00:33.367 DEBUG:boto:Data: 2022-09-03T07:00:33.368 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.368 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.369 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.370 DEBUG:boto:Params: {} 2022-09-03T07:00:33.370 DEBUG:boto:Token: None 2022-09-03T07:00:33.371 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lBES8sCezTSmJ7Fpb/mbEjZGtlw= 2022-09-03T07:00:33.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lBES8sCezTSmJ7Fpb/mbEjZGtlw=', 'Content-Length': '0'} 2022-09-03T07:00:33.375 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e4a501ef07b398c0-006312fb91-170f-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.375 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:00:33.376 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.376 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.377 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.378 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.378 DEBUG:boto:Method: GET 2022-09-03T07:00:33.379 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.380 DEBUG:boto:Data: 2022-09-03T07:00:33.381 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.381 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.382 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.382 DEBUG:boto:Params: {} 2022-09-03T07:00:33.383 DEBUG:boto:Token: None 2022-09-03T07:00:33.384 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/?versions 2022-09-03T07:00:33.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/NfhvBOCFGt9cgrGv9CwjITBM2E= 2022-09-03T07:00:33.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/NfhvBOCFGt9cgrGv9CwjITBM2E=', 'Content-Length': '0'} 2022-09-03T07:00:33.388 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009321c3ef4f089431-006312fb91-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.388 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.390 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:00:33.390 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:00:33.391 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:00:33.392 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:00:33.393 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:00:33.394 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:00:33.395 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:00:33.396 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:00:33.396 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:00:33.397 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.398 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.399 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.400 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.401 DEBUG:boto:Method: GET 2022-09-03T07:00:33.401 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.402 DEBUG:boto:Data: 2022-09-03T07:00:33.403 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.403 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.404 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.404 DEBUG:boto:Params: {} 2022-09-03T07:00:33.405 DEBUG:boto:Token: None 2022-09-03T07:00:33.406 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/?versions 2022-09-03T07:00:33.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/NfhvBOCFGt9cgrGv9CwjITBM2E= 2022-09-03T07:00:33.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/NfhvBOCFGt9cgrGv9CwjITBM2E=', 'Content-Length': '0'} 2022-09-03T07:00:33.409 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a8c8ba43816fef0-006312fb91-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.410 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.412 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:00:33.412 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:00:33.413 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:00:33.413 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:00:33.414 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:00:33.414 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:00:33.415 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:00:33.415 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:00:33.416 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.416 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.417 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.417 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.418 DEBUG:boto:Method: GET 2022-09-03T07:00:33.419 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.419 DEBUG:boto:Data: 2022-09-03T07:00:33.420 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.421 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.422 DEBUG:boto:Params: {} 2022-09-03T07:00:33.423 DEBUG:boto:Token: None 2022-09-03T07:00:33.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/?versions 2022-09-03T07:00:33.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/NfhvBOCFGt9cgrGv9CwjITBM2E= 2022-09-03T07:00:33.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/NfhvBOCFGt9cgrGv9CwjITBM2E=', 'Content-Length': '0'} 2022-09-03T07:00:33.429 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a424f69e5ce4d93a-006312fb91-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.430 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.432 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.433 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.434 DEBUG:boto:path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.435 DEBUG:boto:auth_path=/avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.436 DEBUG:boto:Method: GET 2022-09-03T07:00:33.436 DEBUG:boto:Path: /avysmw-72/?versions&max-keys=999 2022-09-03T07:00:33.437 DEBUG:boto:Data: 2022-09-03T07:00:33.437 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.438 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.439 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.440 DEBUG:boto:Params: {} 2022-09-03T07:00:33.440 DEBUG:boto:Token: None 2022-09-03T07:00:33.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/?versions 2022-09-03T07:00:33.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/NfhvBOCFGt9cgrGv9CwjITBM2E= 2022-09-03T07:00:33.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/NfhvBOCFGt9cgrGv9CwjITBM2E=', 'Content-Length': '0'} 2022-09-03T07:00:33.445 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a536ed685d5f8eb7-006312fb91-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.446 DEBUG:boto:b'avysmw-72999falseanulltrue2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.447 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:00:33.448 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.449 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.449 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.450 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.450 DEBUG:boto:Method: GET 2022-09-03T07:00:33.451 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:33.452 DEBUG:boto:Data: 2022-09-03T07:00:33.453 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.454 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.455 DEBUG:boto:Params: {} 2022-09-03T07:00:33.455 DEBUG:boto:Token: None 2022-09-03T07:00:33.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:33.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cfpw22GJdEfIusY6RFjQV5dpIog= 2022-09-03T07:00:33.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cfpw22GJdEfIusY6RFjQV5dpIog=', 'Content-Length': '0'} 2022-09-03T07:00:33.460 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a630594447aa514f-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.502 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.503 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.504 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.505 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.506 DEBUG:boto:Method: GET 2022-09-03T07:00:33.507 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:33.508 DEBUG:boto:Data: 2022-09-03T07:00:33.509 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.510 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.511 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.512 DEBUG:boto:Params: {} 2022-09-03T07:00:33.513 DEBUG:boto:Token: None 2022-09-03T07:00:33.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:33.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cfpw22GJdEfIusY6RFjQV5dpIog= 2022-09-03T07:00:33.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cfpw22GJdEfIusY6RFjQV5dpIog=', 'Content-Length': '0'} 2022-09-03T07:00:33.519 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e9bfd84ebd4218e-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.522 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.523 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.524 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.525 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.526 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.527 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:33.528 DEBUG:boto:Data: 2022-09-03T07:00:33.530 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.531 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.531 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.532 DEBUG:boto:Params: {} 2022-09-03T07:00:33.532 DEBUG:boto:Token: None 2022-09-03T07:00:33.533 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:33.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sab2QX31IZzcopWfek4zG2l+ZNM= 2022-09-03T07:00:33.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sab2QX31IZzcopWfek4zG2l+ZNM=', 'Content-Length': '0'} 2022-09-03T07:00:33.537 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5539c0513045f37-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.538 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.539 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.540 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.541 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.542 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.542 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:33.543 DEBUG:boto:Data: 2022-09-03T07:00:33.544 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.545 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.546 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.548 DEBUG:boto:Params: {} 2022-09-03T07:00:33.549 DEBUG:boto:Token: None 2022-09-03T07:00:33.550 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:33.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sab2QX31IZzcopWfek4zG2l+ZNM= 2022-09-03T07:00:33.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sab2QX31IZzcopWfek4zG2l+ZNM=', 'Content-Length': '0'} 2022-09-03T07:00:33.554 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005af3b723012b3d01-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.554 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:00:33.555 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.556 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.557 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.558 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.558 DEBUG:boto:Method: GET 2022-09-03T07:00:33.559 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:33.560 DEBUG:boto:Data: 2022-09-03T07:00:33.560 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.561 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.561 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.562 DEBUG:boto:Params: {} 2022-09-03T07:00:33.562 DEBUG:boto:Token: None 2022-09-03T07:00:33.563 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:33.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cfpw22GJdEfIusY6RFjQV5dpIog= 2022-09-03T07:00:33.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cfpw22GJdEfIusY6RFjQV5dpIog=', 'Content-Length': '0'} 2022-09-03T07:00:33.566 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b2e76c9a23811ec-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.569 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.569 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.570 DEBUG:boto:path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.570 DEBUG:boto:auth_path=/avysmw-72/a?versionId=null 2022-09-03T07:00:33.571 DEBUG:boto:Method: GET 2022-09-03T07:00:33.571 DEBUG:boto:Path: /avysmw-72/a?versionId=null 2022-09-03T07:00:33.572 DEBUG:boto:Data: 2022-09-03T07:00:33.572 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.573 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.573 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.574 DEBUG:boto:Params: {} 2022-09-03T07:00:33.575 DEBUG:boto:Token: None 2022-09-03T07:00:33.575 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a?versionId=null 2022-09-03T07:00:33.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cfpw22GJdEfIusY6RFjQV5dpIog= 2022-09-03T07:00:33.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cfpw22GJdEfIusY6RFjQV5dpIog=', 'Content-Length': '0'} 2022-09-03T07:00:33.579 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004db1f39d851b5a51-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.580 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.581 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.581 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:33.582 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:33.582 DEBUG:boto:Method: GET 2022-09-03T07:00:33.583 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:33.583 DEBUG:boto:Data: 2022-09-03T07:00:33.584 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.584 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.585 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.585 DEBUG:boto:Params: {} 2022-09-03T07:00:33.586 DEBUG:boto:Token: None 2022-09-03T07:00:33.587 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.588 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.591 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031003c2848dad2c4-006312fb91-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.592 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.593 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.593 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.594 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:33.594 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:33.595 DEBUG:boto:Method: GET 2022-09-03T07:00:33.596 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:33.596 DEBUG:boto:Data: 2022-09-03T07:00:33.597 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.597 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.598 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.598 DEBUG:boto:Params: {} 2022-09-03T07:00:33.599 DEBUG:boto:Token: None 2022-09-03T07:00:33.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.603 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000078aa49f085113508-006312fb91-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.604 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.604 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.605 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.605 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.606 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:33.607 DEBUG:boto:Data: 2022-09-03T07:00:33.607 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.608 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.608 DEBUG:boto:Params: {} 2022-09-03T07:00:33.609 DEBUG:boto:Token: None 2022-09-03T07:00:33.610 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a 2022-09-03T07:00:33.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3gIdADTUWzBl1WC9cAYqEjysXf4= 2022-09-03T07:00:33.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3gIdADTUWzBl1WC9cAYqEjysXf4=', 'Content-Length': '0'} 2022-09-03T07:00:33.613 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e281d3e368f82316-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.614 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.614 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.615 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.615 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:33.616 DEBUG:boto:Data: 2022-09-03T07:00:33.616 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.617 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.617 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.618 DEBUG:boto:Params: {} 2022-09-03T07:00:33.618 DEBUG:boto:Token: None 2022-09-03T07:00:33.619 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a 2022-09-03T07:00:33.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3gIdADTUWzBl1WC9cAYqEjysXf4= 2022-09-03T07:00:33.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3gIdADTUWzBl1WC9cAYqEjysXf4=', 'Content-Length': '0'} 2022-09-03T07:00:33.622 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c15a06fbf59c0e03-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.623 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:00:33.623 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.624 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.624 DEBUG:boto:Method: GET 2022-09-03T07:00:33.625 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:33.625 DEBUG:boto:Data: 2022-09-03T07:00:33.626 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.627 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.628 DEBUG:boto:Params: {} 2022-09-03T07:00:33.629 DEBUG:boto:Token: None 2022-09-03T07:00:33.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a 2022-09-03T07:00:33.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B2wkgOv0sQ7pqC9YIwIa5IzWFv0= 2022-09-03T07:00:33.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B2wkgOv0sQ7pqC9YIwIa5IzWFv0=', 'Content-Length': '0'} 2022-09-03T07:00:33.633 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056632cec38a96112-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.674 DEBUG:boto:path=/avysmw-72/a 2022-09-03T07:00:33.675 DEBUG:boto:auth_path=/avysmw-72/a 2022-09-03T07:00:33.675 DEBUG:boto:Method: GET 2022-09-03T07:00:33.676 DEBUG:boto:Path: /avysmw-72/a 2022-09-03T07:00:33.676 DEBUG:boto:Data: 2022-09-03T07:00:33.677 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.677 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.678 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.678 DEBUG:boto:Params: {} 2022-09-03T07:00:33.679 DEBUG:boto:Token: None 2022-09-03T07:00:33.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/a 2022-09-03T07:00:33.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B2wkgOv0sQ7pqC9YIwIa5IzWFv0= 2022-09-03T07:00:33.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B2wkgOv0sQ7pqC9YIwIa5IzWFv0=', 'Content-Length': '0'} 2022-09-03T07:00:33.683 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa2dd8b9472b64e0-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.684 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.685 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.686 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:33.687 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:33.688 DEBUG:boto:Method: GET 2022-09-03T07:00:33.689 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:33.689 DEBUG:boto:Data: 2022-09-03T07:00:33.690 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.691 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.692 DEBUG:boto:Params: {} 2022-09-03T07:00:33.693 DEBUG:boto:Token: None 2022-09-03T07:00:33.693 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.698 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c7c9e9f01316cccb-006312fb91-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.698 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.699 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.700 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.700 DEBUG:boto:path=/avysmw-72/?prefix=a 2022-09-03T07:00:33.701 DEBUG:boto:auth_path=/avysmw-72/?prefix=a 2022-09-03T07:00:33.701 DEBUG:boto:Method: GET 2022-09-03T07:00:33.702 DEBUG:boto:Path: /avysmw-72/?prefix=a 2022-09-03T07:00:33.703 DEBUG:boto:Data: 2022-09-03T07:00:33.703 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.704 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.704 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.705 DEBUG:boto:Params: {} 2022-09-03T07:00:33.706 DEBUG:boto:Token: None 2022-09-03T07:00:33.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.710 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f7b8bd59a44f9a40-006312fb91-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.711 DEBUG:boto:b'avysmw-72a1000falsea2022-09-03T06:55:17.386Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.712 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:00:33.712 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.713 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.713 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.714 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.715 DEBUG:boto:Method: GET 2022-09-03T07:00:33.715 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:33.716 DEBUG:boto:Data: 2022-09-03T07:00:33.716 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.717 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.718 DEBUG:boto:Params: {} 2022-09-03T07:00:33.718 DEBUG:boto:Token: None 2022-09-03T07:00:33.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:33.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0wM8+BJpMgrkZKz4TgUFEWIgm1g= 2022-09-03T07:00:33.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0wM8+BJpMgrkZKz4TgUFEWIgm1g=', 'Content-Length': '0'} 2022-09-03T07:00:33.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000adb7602d0d3b9362-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.723 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.724 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.724 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.725 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.725 DEBUG:boto:Method: GET 2022-09-03T07:00:33.726 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:33.726 DEBUG:boto:Data: 2022-09-03T07:00:33.727 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.727 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.728 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.729 DEBUG:boto:Params: {} 2022-09-03T07:00:33.729 DEBUG:boto:Token: None 2022-09-03T07:00:33.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:33.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0wM8+BJpMgrkZKz4TgUFEWIgm1g= 2022-09-03T07:00:33.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0wM8+BJpMgrkZKz4TgUFEWIgm1g=', 'Content-Length': '0'} 2022-09-03T07:00:33.733 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7c1fab621614a0e-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.734 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.735 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.735 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.736 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.736 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.737 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:33.737 DEBUG:boto:Data: 2022-09-03T07:00:33.738 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.739 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.739 DEBUG:boto:Params: {} 2022-09-03T07:00:33.740 DEBUG:boto:Token: None 2022-09-03T07:00:33.741 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:33.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gIHMumo/ppv5WU2dNp8Cg++WQhI= 2022-09-03T07:00:33.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gIHMumo/ppv5WU2dNp8Cg++WQhI=', 'Content-Length': '0'} 2022-09-03T07:00:33.744 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009222c00706517876-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.745 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.745 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.746 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.746 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.747 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.747 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:33.748 DEBUG:boto:Data: 2022-09-03T07:00:33.749 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.749 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.750 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.751 DEBUG:boto:Params: {} 2022-09-03T07:00:33.752 DEBUG:boto:Token: None 2022-09-03T07:00:33.753 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:33.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gIHMumo/ppv5WU2dNp8Cg++WQhI= 2022-09-03T07:00:33.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gIHMumo/ppv5WU2dNp8Cg++WQhI=', 'Content-Length': '0'} 2022-09-03T07:00:33.756 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080cb4817c0a92cf1-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.757 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:00:33.757 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.758 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.758 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.759 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.759 DEBUG:boto:Method: GET 2022-09-03T07:00:33.760 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:33.760 DEBUG:boto:Data: 2022-09-03T07:00:33.761 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.762 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.762 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.763 DEBUG:boto:Params: {} 2022-09-03T07:00:33.764 DEBUG:boto:Token: None 2022-09-03T07:00:33.765 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:33.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0wM8+BJpMgrkZKz4TgUFEWIgm1g= 2022-09-03T07:00:33.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0wM8+BJpMgrkZKz4TgUFEWIgm1g=', 'Content-Length': '0'} 2022-09-03T07:00:33.768 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b20ca2f1ef32ae5-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.769 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.770 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.770 DEBUG:boto:path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.771 DEBUG:boto:auth_path=/avysmw-72/b?versionId=null 2022-09-03T07:00:33.771 DEBUG:boto:Method: GET 2022-09-03T07:00:33.772 DEBUG:boto:Path: /avysmw-72/b?versionId=null 2022-09-03T07:00:33.773 DEBUG:boto:Data: 2022-09-03T07:00:33.774 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.774 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.775 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.775 DEBUG:boto:Params: {} 2022-09-03T07:00:33.776 DEBUG:boto:Token: None 2022-09-03T07:00:33.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b?versionId=null 2022-09-03T07:00:33.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0wM8+BJpMgrkZKz4TgUFEWIgm1g= 2022-09-03T07:00:33.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0wM8+BJpMgrkZKz4TgUFEWIgm1g=', 'Content-Length': '0'} 2022-09-03T07:00:33.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed97ba9f525f5673-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.781 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.782 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.783 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:33.783 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:33.784 DEBUG:boto:Method: GET 2022-09-03T07:00:33.784 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:33.785 DEBUG:boto:Data: 2022-09-03T07:00:33.786 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.788 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.789 DEBUG:boto:Params: {} 2022-09-03T07:00:33.790 DEBUG:boto:Token: None 2022-09-03T07:00:33.791 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.797 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000055d9f58159a39a97-006312fb91-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.799 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.800 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.801 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.802 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:33.803 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:33.804 DEBUG:boto:Method: GET 2022-09-03T07:00:33.805 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:33.806 DEBUG:boto:Data: 2022-09-03T07:00:33.807 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.808 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.809 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.810 DEBUG:boto:Params: {} 2022-09-03T07:00:33.811 DEBUG:boto:Token: None 2022-09-03T07:00:33.812 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.816 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003328200051fa396a-006312fb91-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.817 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.818 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.819 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.820 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.821 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:33.821 DEBUG:boto:Data: 2022-09-03T07:00:33.822 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.823 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.823 DEBUG:boto:Params: {} 2022-09-03T07:00:33.824 DEBUG:boto:Token: None 2022-09-03T07:00:33.825 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b 2022-09-03T07:00:33.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jhvOfgyyxWJJCgb5ngvW/5QHUHU= 2022-09-03T07:00:33.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jhvOfgyyxWJJCgb5ngvW/5QHUHU=', 'Content-Length': '0'} 2022-09-03T07:00:33.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f9547aae4d490a8-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.831 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.832 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.833 DEBUG:boto:Method: HEAD 2022-09-03T07:00:33.834 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:33.835 DEBUG:boto:Data: 2022-09-03T07:00:33.836 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.837 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.837 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.838 DEBUG:boto:Params: {} 2022-09-03T07:00:33.840 DEBUG:boto:Token: None 2022-09-03T07:00:33.840 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b 2022-09-03T07:00:33.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jhvOfgyyxWJJCgb5ngvW/5QHUHU= 2022-09-03T07:00:33.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jhvOfgyyxWJJCgb5ngvW/5QHUHU=', 'Content-Length': '0'} 2022-09-03T07:00:33.843 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ca1a5017e2adc8b-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.844 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:00:33.845 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.845 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.846 DEBUG:boto:Method: GET 2022-09-03T07:00:33.846 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:33.847 DEBUG:boto:Data: 2022-09-03T07:00:33.848 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.849 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.849 DEBUG:boto:Params: {} 2022-09-03T07:00:33.850 DEBUG:boto:Token: None 2022-09-03T07:00:33.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b 2022-09-03T07:00:33.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7xHp9dAvQxVFTEmRz+sgGKM+Mik= 2022-09-03T07:00:33.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7xHp9dAvQxVFTEmRz+sgGKM+Mik=', 'Content-Length': '0'} 2022-09-03T07:00:33.854 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006fb6a4c4411da77c-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.856 DEBUG:boto:path=/avysmw-72/b 2022-09-03T07:00:33.858 DEBUG:boto:auth_path=/avysmw-72/b 2022-09-03T07:00:33.859 DEBUG:boto:Method: GET 2022-09-03T07:00:33.861 DEBUG:boto:Path: /avysmw-72/b 2022-09-03T07:00:33.861 DEBUG:boto:Data: 2022-09-03T07:00:33.862 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.863 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.863 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.864 DEBUG:boto:Params: {} 2022-09-03T07:00:33.864 DEBUG:boto:Token: None 2022-09-03T07:00:33.865 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/b 2022-09-03T07:00:33.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7xHp9dAvQxVFTEmRz+sgGKM+Mik= 2022-09-03T07:00:33.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7xHp9dAvQxVFTEmRz+sgGKM+Mik=', 'Content-Length': '0'} 2022-09-03T07:00:33.869 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003570136fcc58a87c-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.870 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.870 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.871 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:33.872 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:33.872 DEBUG:boto:Method: GET 2022-09-03T07:00:33.873 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:33.874 DEBUG:boto:Data: 2022-09-03T07:00:33.874 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.876 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.877 DEBUG:boto:Params: {} 2022-09-03T07:00:33.877 DEBUG:boto:Token: None 2022-09-03T07:00:33.878 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.884 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001f65d2ceae4f3eb1-006312fb91-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.930 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.931 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:33.932 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:33.933 DEBUG:boto:path=/avysmw-72/?prefix=b 2022-09-03T07:00:33.933 DEBUG:boto:auth_path=/avysmw-72/?prefix=b 2022-09-03T07:00:33.934 DEBUG:boto:Method: GET 2022-09-03T07:00:33.935 DEBUG:boto:Path: /avysmw-72/?prefix=b 2022-09-03T07:00:33.936 DEBUG:boto:Data: 2022-09-03T07:00:33.937 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.938 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.939 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.941 DEBUG:boto:Params: {} 2022-09-03T07:00:33.945 DEBUG:boto:Token: None 2022-09-03T07:00:33.946 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/ 2022-09-03T07:00:33.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04= 2022-09-03T07:00:33.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFYMTna1PcLtIdRNOIPAYZB/04=', 'Content-Length': '0'} 2022-09-03T07:00:33.952 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000050a26831ea450dc2-006312fb91-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.953 DEBUG:boto:b'avysmw-72b1000falseb2022-09-03T06:55:17.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:33.954 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:00:33.955 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:33.956 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:33.957 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:33.957 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:33.958 DEBUG:boto:Method: GET 2022-09-03T07:00:33.959 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:33.960 DEBUG:boto:Data: 2022-09-03T07:00:33.960 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.962 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:33.962 DEBUG:boto:Port: 8001 2022-09-03T07:00:33.963 DEBUG:boto:Params: {} 2022-09-03T07:00:33.964 DEBUG:boto:Token: None 2022-09-03T07:00:33.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:33.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5D3JJxVVpimNgP10E8guVlGwU7M= 2022-09-03T07:00:33.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5D3JJxVVpimNgP10E8guVlGwU7M=', 'Content-Length': '0'} 2022-09-03T07:00:33.969 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000723571865c8c61c5-006312fb91-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.970 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:33.971 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:33.973 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:33.973 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:33.974 DEBUG:boto:Method: GET 2022-09-03T07:00:33.975 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:33.978 DEBUG:boto:Data: 2022-09-03T07:00:33.980 DEBUG:boto:Headers: {} 2022-09-03T07:00:33.981 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:33.983 DEBUG:boto:Port: 8000 2022-09-03T07:00:33.984 DEBUG:boto:Params: {} 2022-09-03T07:00:33.986 DEBUG:boto:Token: None 2022-09-03T07:00:33.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:33 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:33.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5D3JJxVVpimNgP10E8guVlGwU7M= 2022-09-03T07:00:33.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5D3JJxVVpimNgP10E8guVlGwU7M=', 'Content-Length': '0'} 2022-09-03T07:00:33.994 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b063ee961fff8f4-006312fb91-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:33.996 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:33.997 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:33.998 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:33.999 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:34.000 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.001 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:34.002 DEBUG:boto:Data: 2022-09-03T07:00:34.003 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.004 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.006 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.007 DEBUG:boto:Params: {} 2022-09-03T07:00:34.008 DEBUG:boto:Token: None 2022-09-03T07:00:34.009 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:34.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8TFpEnIDbe4Ux5YCPznVzjvgsoQ= 2022-09-03T07:00:34.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8TFpEnIDbe4Ux5YCPznVzjvgsoQ=', 'Content-Length': '0'} 2022-09-03T07:00:34.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000434345a3de161759-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.018 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:34.019 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:34.020 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:34.021 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:34.022 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.023 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:34.024 DEBUG:boto:Data: 2022-09-03T07:00:34.025 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.026 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.027 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.028 DEBUG:boto:Params: {} 2022-09-03T07:00:34.029 DEBUG:boto:Token: None 2022-09-03T07:00:34.030 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:34.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8TFpEnIDbe4Ux5YCPznVzjvgsoQ= 2022-09-03T07:00:34.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8TFpEnIDbe4Ux5YCPznVzjvgsoQ=', 'Content-Length': '0'} 2022-09-03T07:00:34.035 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cdcef976e77f64fe-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.036 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:00:34.037 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:34.038 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:34.040 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:34.041 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:34.042 DEBUG:boto:Method: GET 2022-09-03T07:00:34.043 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:34.044 DEBUG:boto:Data: 2022-09-03T07:00:34.045 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.046 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.047 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.048 DEBUG:boto:Params: {} 2022-09-03T07:00:34.049 DEBUG:boto:Token: None 2022-09-03T07:00:34.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:34.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZRm34wXu2gJjsMeYN9ntaT1SYYY= 2022-09-03T07:00:34.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZRm34wXu2gJjsMeYN9ntaT1SYYY=', 'Content-Length': '0'} 2022-09-03T07:00:34.055 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d56e9deb4cd79ad-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.056 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:34.057 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:34.058 DEBUG:boto:path=/avysmw-72/c?versionId=null 2022-09-03T07:00:34.059 DEBUG:boto:auth_path=/avysmw-72/c?versionId=null 2022-09-03T07:00:34.060 DEBUG:boto:Method: GET 2022-09-03T07:00:34.061 DEBUG:boto:Path: /avysmw-72/c?versionId=null 2022-09-03T07:00:34.062 DEBUG:boto:Data: 2022-09-03T07:00:34.063 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.064 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.065 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.066 DEBUG:boto:Params: {} 2022-09-03T07:00:34.067 DEBUG:boto:Token: None 2022-09-03T07:00:34.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/c?versionId=null 2022-09-03T07:00:34.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZRm34wXu2gJjsMeYN9ntaT1SYYY= 2022-09-03T07:00:34.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZRm34wXu2gJjsMeYN9ntaT1SYYY=', 'Content-Length': '0'} 2022-09-03T07:00:34.072 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f26d328af47187fb-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.073 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.073 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.074 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:34.074 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:34.075 DEBUG:boto:Method: GET 2022-09-03T07:00:34.076 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:34.076 DEBUG:boto:Data: 2022-09-03T07:00:34.077 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.078 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.078 DEBUG:boto:Params: {} 2022-09-03T07:00:34.079 DEBUG:boto:Token: None 2022-09-03T07:00:34.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f93f857b8dbe803-006312fb92-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.085 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.087 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.087 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.088 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:34.088 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:34.089 DEBUG:boto:Method: GET 2022-09-03T07:00:34.090 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:34.091 DEBUG:boto:Data: 2022-09-03T07:00:34.092 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.093 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.094 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.095 DEBUG:boto:Params: {} 2022-09-03T07:00:34.096 DEBUG:boto:Token: None 2022-09-03T07:00:34.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.102 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e0ae9bc74c33951f-006312fb92-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.103 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.104 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:34.104 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:34.105 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.105 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:34.106 DEBUG:boto:Data: 2022-09-03T07:00:34.107 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.107 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.108 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.108 DEBUG:boto:Params: {} 2022-09-03T07:00:34.109 DEBUG:boto:Token: None 2022-09-03T07:00:34.110 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/c 2022-09-03T07:00:34.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bt5igpZEUZpjGLFCcMwkK35rkH8= 2022-09-03T07:00:34.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bt5igpZEUZpjGLFCcMwkK35rkH8=', 'Content-Length': '0'} 2022-09-03T07:00:34.116 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6e3a8fa9d4a74d1-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.117 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:34.118 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:34.119 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.120 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:34.121 DEBUG:boto:Data: 2022-09-03T07:00:34.122 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.123 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.124 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.125 DEBUG:boto:Params: {} 2022-09-03T07:00:34.126 DEBUG:boto:Token: None 2022-09-03T07:00:34.127 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/c 2022-09-03T07:00:34.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bt5igpZEUZpjGLFCcMwkK35rkH8= 2022-09-03T07:00:34.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bt5igpZEUZpjGLFCcMwkK35rkH8=', 'Content-Length': '0'} 2022-09-03T07:00:34.132 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000971b786c3c92bb67-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.133 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:00:34.133 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:34.134 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:34.135 DEBUG:boto:Method: GET 2022-09-03T07:00:34.135 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:34.136 DEBUG:boto:Data: 2022-09-03T07:00:34.136 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.137 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.138 DEBUG:boto:Params: {} 2022-09-03T07:00:34.139 DEBUG:boto:Token: None 2022-09-03T07:00:34.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/c 2022-09-03T07:00:34.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iUPSyk2duyh6EtMGznukJQcsGMs= 2022-09-03T07:00:34.141 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iUPSyk2duyh6EtMGznukJQcsGMs=', 'Content-Length': '0'} 2022-09-03T07:00:34.143 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000088e6639ff7960437-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.144 DEBUG:boto:path=/avysmw-72/c 2022-09-03T07:00:34.145 DEBUG:boto:auth_path=/avysmw-72/c 2022-09-03T07:00:34.145 DEBUG:boto:Method: GET 2022-09-03T07:00:34.146 DEBUG:boto:Path: /avysmw-72/c 2022-09-03T07:00:34.146 DEBUG:boto:Data: 2022-09-03T07:00:34.147 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.148 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.148 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.149 DEBUG:boto:Params: {} 2022-09-03T07:00:34.149 DEBUG:boto:Token: None 2022-09-03T07:00:34.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/c 2022-09-03T07:00:34.151 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iUPSyk2duyh6EtMGznukJQcsGMs= 2022-09-03T07:00:34.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iUPSyk2duyh6EtMGznukJQcsGMs=', 'Content-Length': '0'} 2022-09-03T07:00:34.154 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae992e355aef7c3f-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.155 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.155 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.156 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:34.156 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:34.157 DEBUG:boto:Method: GET 2022-09-03T07:00:34.158 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:34.158 DEBUG:boto:Data: 2022-09-03T07:00:34.159 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.160 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.160 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.161 DEBUG:boto:Params: {} 2022-09-03T07:00:34.161 DEBUG:boto:Token: None 2022-09-03T07:00:34.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.167 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ebd76713e606cbd4-006312fb92-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.210 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.212 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.213 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.214 DEBUG:boto:path=/avysmw-72/?prefix=c 2022-09-03T07:00:34.215 DEBUG:boto:auth_path=/avysmw-72/?prefix=c 2022-09-03T07:00:34.216 DEBUG:boto:Method: GET 2022-09-03T07:00:34.217 DEBUG:boto:Path: /avysmw-72/?prefix=c 2022-09-03T07:00:34.218 DEBUG:boto:Data: 2022-09-03T07:00:34.219 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.220 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.221 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.222 DEBUG:boto:Params: {} 2022-09-03T07:00:34.224 DEBUG:boto:Token: None 2022-09-03T07:00:34.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.228 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000078d796ea408d637b-006312fb92-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.229 DEBUG:boto:b'avysmw-72c1000falsec2022-09-03T06:55:17.486Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.230 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:00:34.231 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.231 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.232 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.232 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.233 DEBUG:boto:Method: GET 2022-09-03T07:00:34.234 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:34.234 DEBUG:boto:Data: 2022-09-03T07:00:34.235 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.236 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.237 DEBUG:boto:Params: {} 2022-09-03T07:00:34.237 DEBUG:boto:Token: None 2022-09-03T07:00:34.238 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:34.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rOrXmCSfpF93ex0w7RuwgyOoeRE= 2022-09-03T07:00:34.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rOrXmCSfpF93ex0w7RuwgyOoeRE=', 'Content-Length': '0'} 2022-09-03T07:00:34.242 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032069166d24e7362-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.243 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.244 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.245 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.246 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.247 DEBUG:boto:Method: GET 2022-09-03T07:00:34.248 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:34.249 DEBUG:boto:Data: 2022-09-03T07:00:34.250 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.251 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.252 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.253 DEBUG:boto:Params: {} 2022-09-03T07:00:34.254 DEBUG:boto:Token: None 2022-09-03T07:00:34.255 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:34.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rOrXmCSfpF93ex0w7RuwgyOoeRE= 2022-09-03T07:00:34.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rOrXmCSfpF93ex0w7RuwgyOoeRE=', 'Content-Length': '0'} 2022-09-03T07:00:34.258 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000193b8fabc5bce740-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.259 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.260 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.260 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.261 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.261 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.262 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:34.262 DEBUG:boto:Data: 2022-09-03T07:00:34.263 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.264 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.264 DEBUG:boto:Params: {} 2022-09-03T07:00:34.265 DEBUG:boto:Token: None 2022-09-03T07:00:34.266 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:34.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:21lKAzKjckiIVZBSohpVm5O7MYg= 2022-09-03T07:00:34.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:21lKAzKjckiIVZBSohpVm5O7MYg=', 'Content-Length': '0'} 2022-09-03T07:00:34.269 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a66fd36c55861920-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.270 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.270 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.271 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.272 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.272 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.273 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:34.274 DEBUG:boto:Data: 2022-09-03T07:00:34.274 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.275 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.275 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.276 DEBUG:boto:Params: {} 2022-09-03T07:00:34.276 DEBUG:boto:Token: None 2022-09-03T07:00:34.277 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:34.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:21lKAzKjckiIVZBSohpVm5O7MYg= 2022-09-03T07:00:34.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:21lKAzKjckiIVZBSohpVm5O7MYg=', 'Content-Length': '0'} 2022-09-03T07:00:34.280 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6d056157448ab6c-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.281 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:00:34.281 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.283 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.283 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.284 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.284 DEBUG:boto:Method: GET 2022-09-03T07:00:34.285 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:34.286 DEBUG:boto:Data: 2022-09-03T07:00:34.286 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.287 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.287 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.288 DEBUG:boto:Params: {} 2022-09-03T07:00:34.290 DEBUG:boto:Token: None 2022-09-03T07:00:34.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:34.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rOrXmCSfpF93ex0w7RuwgyOoeRE= 2022-09-03T07:00:34.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rOrXmCSfpF93ex0w7RuwgyOoeRE=', 'Content-Length': '0'} 2022-09-03T07:00:34.296 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002481c7d0f6c45e28-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.298 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.299 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.300 DEBUG:boto:path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.301 DEBUG:boto:auth_path=/avysmw-72/d?versionId=null 2022-09-03T07:00:34.302 DEBUG:boto:Method: GET 2022-09-03T07:00:34.303 DEBUG:boto:Path: /avysmw-72/d?versionId=null 2022-09-03T07:00:34.304 DEBUG:boto:Data: 2022-09-03T07:00:34.305 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.306 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.307 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.308 DEBUG:boto:Params: {} 2022-09-03T07:00:34.309 DEBUG:boto:Token: None 2022-09-03T07:00:34.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d?versionId=null 2022-09-03T07:00:34.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rOrXmCSfpF93ex0w7RuwgyOoeRE= 2022-09-03T07:00:34.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rOrXmCSfpF93ex0w7RuwgyOoeRE=', 'Content-Length': '0'} 2022-09-03T07:00:34.316 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8182ecb16f231b3-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.359 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.360 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.361 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:34.362 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:34.363 DEBUG:boto:Method: GET 2022-09-03T07:00:34.364 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:34.364 DEBUG:boto:Data: 2022-09-03T07:00:34.365 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.366 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.367 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.367 DEBUG:boto:Params: {} 2022-09-03T07:00:34.368 DEBUG:boto:Token: None 2022-09-03T07:00:34.369 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.371 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.374 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001aa90a46b51faa85-006312fb92-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.375 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.377 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.378 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.379 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:34.380 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:34.381 DEBUG:boto:Method: GET 2022-09-03T07:00:34.381 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:34.382 DEBUG:boto:Data: 2022-09-03T07:00:34.383 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.384 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.385 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.386 DEBUG:boto:Params: {} 2022-09-03T07:00:34.387 DEBUG:boto:Token: None 2022-09-03T07:00:34.387 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.393 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f4d274bc76a0432-006312fb92-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.393 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.394 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.395 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.396 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.396 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:34.397 DEBUG:boto:Data: 2022-09-03T07:00:34.397 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.398 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.398 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.399 DEBUG:boto:Params: {} 2022-09-03T07:00:34.400 DEBUG:boto:Token: None 2022-09-03T07:00:34.400 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d 2022-09-03T07:00:34.401 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/4RsBYDLtuOAtzxfxuSNjb54oVQ= 2022-09-03T07:00:34.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/4RsBYDLtuOAtzxfxuSNjb54oVQ=', 'Content-Length': '0'} 2022-09-03T07:00:34.406 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000045729a55b81becb-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.408 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.409 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.409 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.410 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:34.410 DEBUG:boto:Data: 2022-09-03T07:00:34.411 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.412 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.413 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.413 DEBUG:boto:Params: {} 2022-09-03T07:00:34.414 DEBUG:boto:Token: None 2022-09-03T07:00:34.415 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d 2022-09-03T07:00:34.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/4RsBYDLtuOAtzxfxuSNjb54oVQ= 2022-09-03T07:00:34.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/4RsBYDLtuOAtzxfxuSNjb54oVQ=', 'Content-Length': '0'} 2022-09-03T07:00:34.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d03570a7883e4b8-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.423 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:00:34.424 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.425 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.426 DEBUG:boto:Method: GET 2022-09-03T07:00:34.427 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:34.427 DEBUG:boto:Data: 2022-09-03T07:00:34.428 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.430 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.431 DEBUG:boto:Params: {} 2022-09-03T07:00:34.431 DEBUG:boto:Token: None 2022-09-03T07:00:34.432 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d 2022-09-03T07:00:34.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L7KvqwXhZenfM+AibwilqdyWsLI= 2022-09-03T07:00:34.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L7KvqwXhZenfM+AibwilqdyWsLI=', 'Content-Length': '0'} 2022-09-03T07:00:34.436 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fde4aeb9e9878654-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.482 DEBUG:boto:path=/avysmw-72/d 2022-09-03T07:00:34.483 DEBUG:boto:auth_path=/avysmw-72/d 2022-09-03T07:00:34.483 DEBUG:boto:Method: GET 2022-09-03T07:00:34.484 DEBUG:boto:Path: /avysmw-72/d 2022-09-03T07:00:34.485 DEBUG:boto:Data: 2022-09-03T07:00:34.485 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.486 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.487 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.487 DEBUG:boto:Params: {} 2022-09-03T07:00:34.488 DEBUG:boto:Token: None 2022-09-03T07:00:34.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/d 2022-09-03T07:00:34.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L7KvqwXhZenfM+AibwilqdyWsLI= 2022-09-03T07:00:34.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L7KvqwXhZenfM+AibwilqdyWsLI=', 'Content-Length': '0'} 2022-09-03T07:00:34.494 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:55:17 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d4da100f1d5d60f-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.495 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.496 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.497 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:34.497 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:34.498 DEBUG:boto:Method: GET 2022-09-03T07:00:34.499 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:34.500 DEBUG:boto:Data: 2022-09-03T07:00:34.501 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.503 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.504 DEBUG:boto:Params: {} 2022-09-03T07:00:34.505 DEBUG:boto:Token: None 2022-09-03T07:00:34.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.510 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000293a15fba40abac8-006312fb92-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.511 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.513 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.513 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.515 DEBUG:boto:path=/avysmw-72/?prefix=d 2022-09-03T07:00:34.515 DEBUG:boto:auth_path=/avysmw-72/?prefix=d 2022-09-03T07:00:34.516 DEBUG:boto:Method: GET 2022-09-03T07:00:34.517 DEBUG:boto:Path: /avysmw-72/?prefix=d 2022-09-03T07:00:34.518 DEBUG:boto:Data: 2022-09-03T07:00:34.518 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.519 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.520 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.521 DEBUG:boto:Params: {} 2022-09-03T07:00:34.522 DEBUG:boto:Token: None 2022-09-03T07:00:34.524 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.525 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.527 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000706de9bfbe648d11-006312fb92-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.529 DEBUG:boto:b'avysmw-72d1000falsed2022-09-03T06:55:17.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.531 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:00:34.532 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.532 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.533 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.533 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.534 DEBUG:boto:Method: GET 2022-09-03T07:00:34.535 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:34.535 DEBUG:boto:Data: 2022-09-03T07:00:34.536 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.537 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.538 DEBUG:boto:Params: {} 2022-09-03T07:00:34.539 DEBUG:boto:Token: None 2022-09-03T07:00:34.539 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:34.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FjGROwqfdk2p7g0+ieeZi2Pako8= 2022-09-03T07:00:34.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FjGROwqfdk2p7g0+ieeZi2Pako8=', 'Content-Length': '0'} 2022-09-03T07:00:34.544 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000426b4d3c9b8b93b6-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.545 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.545 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.546 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.546 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.547 DEBUG:boto:Method: GET 2022-09-03T07:00:34.548 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:34.548 DEBUG:boto:Data: 2022-09-03T07:00:34.549 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.549 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.550 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.550 DEBUG:boto:Params: {} 2022-09-03T07:00:34.551 DEBUG:boto:Token: None 2022-09-03T07:00:34.552 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:34.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FjGROwqfdk2p7g0+ieeZi2Pako8= 2022-09-03T07:00:34.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FjGROwqfdk2p7g0+ieeZi2Pako8=', 'Content-Length': '0'} 2022-09-03T07:00:34.555 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042d9c4324259979e-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.556 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.556 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.557 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.557 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.558 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.566 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:34.568 DEBUG:boto:Data: 2022-09-03T07:00:34.568 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.569 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.569 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.570 DEBUG:boto:Params: {} 2022-09-03T07:00:34.570 DEBUG:boto:Token: None 2022-09-03T07:00:34.571 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:34.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MScHv2nQ9egQy21Iys7UpY5Z8/w= 2022-09-03T07:00:34.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MScHv2nQ9egQy21Iys7UpY5Z8/w=', 'Content-Length': '0'} 2022-09-03T07:00:34.574 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c5fc2cf345fe3d4-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.575 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.576 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.576 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.576 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.577 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.577 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:34.578 DEBUG:boto:Data: 2022-09-03T07:00:34.578 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.579 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.579 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.580 DEBUG:boto:Params: {} 2022-09-03T07:00:34.580 DEBUG:boto:Token: None 2022-09-03T07:00:34.581 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:34.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MScHv2nQ9egQy21Iys7UpY5Z8/w= 2022-09-03T07:00:34.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MScHv2nQ9egQy21Iys7UpY5Z8/w=', 'Content-Length': '0'} 2022-09-03T07:00:34.584 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005067dde9e4963cb6-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.585 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:00:34.585 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.586 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.587 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.587 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.588 DEBUG:boto:Method: GET 2022-09-03T07:00:34.588 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:34.589 DEBUG:boto:Data: 2022-09-03T07:00:34.589 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.590 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.590 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.591 DEBUG:boto:Params: {} 2022-09-03T07:00:34.591 DEBUG:boto:Token: None 2022-09-03T07:00:34.592 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:34.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FjGROwqfdk2p7g0+ieeZi2Pako8= 2022-09-03T07:00:34.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FjGROwqfdk2p7g0+ieeZi2Pako8=', 'Content-Length': '0'} 2022-09-03T07:00:34.595 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c8a8aaf0503f219-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.642 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.643 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.643 DEBUG:boto:path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.644 DEBUG:boto:auth_path=/avysmw-72/e?versionId=null 2022-09-03T07:00:34.645 DEBUG:boto:Method: GET 2022-09-03T07:00:34.645 DEBUG:boto:Path: /avysmw-72/e?versionId=null 2022-09-03T07:00:34.646 DEBUG:boto:Data: 2022-09-03T07:00:34.646 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.647 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.647 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.648 DEBUG:boto:Params: {} 2022-09-03T07:00:34.648 DEBUG:boto:Token: None 2022-09-03T07:00:34.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e?versionId=null 2022-09-03T07:00:34.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FjGROwqfdk2p7g0+ieeZi2Pako8= 2022-09-03T07:00:34.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FjGROwqfdk2p7g0+ieeZi2Pako8=', 'Content-Length': '0'} 2022-09-03T07:00:34.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065a55e0a80ee3f0a-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.653 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.654 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.654 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:34.655 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:34.655 DEBUG:boto:Method: GET 2022-09-03T07:00:34.656 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:34.656 DEBUG:boto:Data: 2022-09-03T07:00:34.657 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.658 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.659 DEBUG:boto:Params: {} 2022-09-03T07:00:34.659 DEBUG:boto:Token: None 2022-09-03T07:00:34.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.669 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000efb543b18c0d2dd2-006312fb92-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.670 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.671 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.672 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.672 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:34.673 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:34.673 DEBUG:boto:Method: GET 2022-09-03T07:00:34.674 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:34.674 DEBUG:boto:Data: 2022-09-03T07:00:34.675 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.675 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.676 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.677 DEBUG:boto:Params: {} 2022-09-03T07:00:34.677 DEBUG:boto:Token: None 2022-09-03T07:00:34.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.681 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007fcd22ab6a0b9368-006312fb92-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.682 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.683 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.683 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.684 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.685 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:34.685 DEBUG:boto:Data: 2022-09-03T07:00:34.686 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.687 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.687 DEBUG:boto:Params: {} 2022-09-03T07:00:34.688 DEBUG:boto:Token: None 2022-09-03T07:00:34.688 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e 2022-09-03T07:00:34.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O5ymddFApr66sSb30UlFg9J5Wpk= 2022-09-03T07:00:34.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O5ymddFApr66sSb30UlFg9J5Wpk=', 'Content-Length': '0'} 2022-09-03T07:00:34.692 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000365a0b66c9ddbbd2-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.692 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.693 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.694 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.694 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:34.695 DEBUG:boto:Data: 2022-09-03T07:00:34.695 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.696 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.696 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.697 DEBUG:boto:Params: {} 2022-09-03T07:00:34.697 DEBUG:boto:Token: None 2022-09-03T07:00:34.698 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e 2022-09-03T07:00:34.699 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O5ymddFApr66sSb30UlFg9J5Wpk= 2022-09-03T07:00:34.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O5ymddFApr66sSb30UlFg9J5Wpk=', 'Content-Length': '0'} 2022-09-03T07:00:34.701 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c72da91a70becdf4-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.702 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:00:34.702 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.703 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.704 DEBUG:boto:Method: GET 2022-09-03T07:00:34.704 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:34.705 DEBUG:boto:Data: 2022-09-03T07:00:34.705 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.706 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.706 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.707 DEBUG:boto:Params: {} 2022-09-03T07:00:34.708 DEBUG:boto:Token: None 2022-09-03T07:00:34.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e 2022-09-03T07:00:34.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kQhLkIIJcSf3ppQMmKJlykMiMjc= 2022-09-03T07:00:34.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kQhLkIIJcSf3ppQMmKJlykMiMjc=', 'Content-Length': '0'} 2022-09-03T07:00:34.712 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c85f98fbd6ee4937-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.713 DEBUG:boto:path=/avysmw-72/e 2022-09-03T07:00:34.713 DEBUG:boto:auth_path=/avysmw-72/e 2022-09-03T07:00:34.714 DEBUG:boto:Method: GET 2022-09-03T07:00:34.714 DEBUG:boto:Path: /avysmw-72/e 2022-09-03T07:00:34.715 DEBUG:boto:Data: 2022-09-03T07:00:34.716 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.716 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.717 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.717 DEBUG:boto:Params: {} 2022-09-03T07:00:34.718 DEBUG:boto:Token: None 2022-09-03T07:00:34.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/e 2022-09-03T07:00:34.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kQhLkIIJcSf3ppQMmKJlykMiMjc= 2022-09-03T07:00:34.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kQhLkIIJcSf3ppQMmKJlykMiMjc=', 'Content-Length': '0'} 2022-09-03T07:00:34.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026489ca43925ddf8-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.723 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.723 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.724 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:34.725 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:34.725 DEBUG:boto:Method: GET 2022-09-03T07:00:34.726 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:34.727 DEBUG:boto:Data: 2022-09-03T07:00:34.727 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.728 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.729 DEBUG:boto:Params: {} 2022-09-03T07:00:34.729 DEBUG:boto:Token: None 2022-09-03T07:00:34.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.735 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ddc66338a0224d73-006312fb92-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.736 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.736 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.737 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.738 DEBUG:boto:path=/avysmw-72/?prefix=e 2022-09-03T07:00:34.738 DEBUG:boto:auth_path=/avysmw-72/?prefix=e 2022-09-03T07:00:34.739 DEBUG:boto:Method: GET 2022-09-03T07:00:34.739 DEBUG:boto:Path: /avysmw-72/?prefix=e 2022-09-03T07:00:34.740 DEBUG:boto:Data: 2022-09-03T07:00:34.740 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.741 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.741 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.742 DEBUG:boto:Params: {} 2022-09-03T07:00:34.743 DEBUG:boto:Token: None 2022-09-03T07:00:34.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.748 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000005d8ed4a3221bd09-006312fb92-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.795 DEBUG:boto:b'avysmw-72e1000falsee2022-09-03T06:57:02.872Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.797 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:00:34.798 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:34.800 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:34.801 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.802 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.803 DEBUG:boto:Method: GET 2022-09-03T07:00:34.804 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:34.804 DEBUG:boto:Data: 2022-09-03T07:00:34.805 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.806 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.806 DEBUG:boto:Params: {} 2022-09-03T07:00:34.807 DEBUG:boto:Token: None 2022-09-03T07:00:34.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:34.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mXilLreZbx633nfF4x/S37pT0WQ= 2022-09-03T07:00:34.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mXilLreZbx633nfF4x/S37pT0WQ=', 'Content-Length': '0'} 2022-09-03T07:00:34.818 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ee520642dcf2be4-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.821 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:34.821 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:34.822 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.823 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.824 DEBUG:boto:Method: GET 2022-09-03T07:00:34.824 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:34.825 DEBUG:boto:Data: 2022-09-03T07:00:34.826 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.826 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.827 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.827 DEBUG:boto:Params: {} 2022-09-03T07:00:34.828 DEBUG:boto:Token: None 2022-09-03T07:00:34.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:34.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mXilLreZbx633nfF4x/S37pT0WQ= 2022-09-03T07:00:34.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mXilLreZbx633nfF4x/S37pT0WQ=', 'Content-Length': '0'} 2022-09-03T07:00:34.833 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f9613c88d820590-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.834 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:34.835 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:34.835 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.836 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.837 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.837 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:34.838 DEBUG:boto:Data: 2022-09-03T07:00:34.839 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.840 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.841 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.841 DEBUG:boto:Params: {} 2022-09-03T07:00:34.842 DEBUG:boto:Token: None 2022-09-03T07:00:34.843 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:34.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jG5+i9XcfE+VIz9h7V7lmARaAkE= 2022-09-03T07:00:34.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jG5+i9XcfE+VIz9h7V7lmARaAkE=', 'Content-Length': '0'} 2022-09-03T07:00:34.850 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000180920ba9e633fb6-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.851 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:34.852 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:34.853 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.855 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.856 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.856 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:34.857 DEBUG:boto:Data: 2022-09-03T07:00:34.858 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.859 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.859 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.860 DEBUG:boto:Params: {} 2022-09-03T07:00:34.861 DEBUG:boto:Token: None 2022-09-03T07:00:34.861 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:34.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jG5+i9XcfE+VIz9h7V7lmARaAkE= 2022-09-03T07:00:34.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jG5+i9XcfE+VIz9h7V7lmARaAkE=', 'Content-Length': '0'} 2022-09-03T07:00:34.865 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000422b9f7d733a5b99-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.866 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:00:34.867 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:34.868 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:34.869 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.869 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.870 DEBUG:boto:Method: GET 2022-09-03T07:00:34.871 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:34.872 DEBUG:boto:Data: 2022-09-03T07:00:34.873 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.874 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.875 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.875 DEBUG:boto:Params: {} 2022-09-03T07:00:34.876 DEBUG:boto:Token: None 2022-09-03T07:00:34.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:34.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mXilLreZbx633nfF4x/S37pT0WQ= 2022-09-03T07:00:34.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mXilLreZbx633nfF4x/S37pT0WQ=', 'Content-Length': '0'} 2022-09-03T07:00:34.881 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aeaaaf0554e92871-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.922 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:34.923 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:34.924 DEBUG:boto:path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.924 DEBUG:boto:auth_path=/avysmw-72/f?versionId=null 2022-09-03T07:00:34.925 DEBUG:boto:Method: GET 2022-09-03T07:00:34.926 DEBUG:boto:Path: /avysmw-72/f?versionId=null 2022-09-03T07:00:34.927 DEBUG:boto:Data: 2022-09-03T07:00:34.928 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.928 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.929 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.929 DEBUG:boto:Params: {} 2022-09-03T07:00:34.930 DEBUG:boto:Token: None 2022-09-03T07:00:34.931 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/f?versionId=null 2022-09-03T07:00:34.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mXilLreZbx633nfF4x/S37pT0WQ= 2022-09-03T07:00:34.932 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mXilLreZbx633nfF4x/S37pT0WQ=', 'Content-Length': '0'} 2022-09-03T07:00:34.935 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000047729ab44c4d2a8-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.936 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.936 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.937 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:34.938 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:34.939 DEBUG:boto:Method: GET 2022-09-03T07:00:34.939 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:34.940 DEBUG:boto:Data: 2022-09-03T07:00:34.940 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.942 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.943 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.943 DEBUG:boto:Params: {} 2022-09-03T07:00:34.944 DEBUG:boto:Token: None 2022-09-03T07:00:34.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.953 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000440a8dfd11b751d-006312fb92-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.954 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.956 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:34.957 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:34.958 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:34.958 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:34.960 DEBUG:boto:Method: GET 2022-09-03T07:00:34.961 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:34.962 DEBUG:boto:Data: 2022-09-03T07:00:34.963 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.964 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.965 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.966 DEBUG:boto:Params: {} 2022-09-03T07:00:34.967 DEBUG:boto:Token: None 2022-09-03T07:00:34.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/ 2022-09-03T07:00:34.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc= 2022-09-03T07:00:34.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H3n/yJfU5tNWe/YAKEEONHcz3sc=', 'Content-Length': '0'} 2022-09-03T07:00:34.971 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da0771095501f960-006312fb92-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.972 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:34.973 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:34.973 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:34.974 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.975 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:34.976 DEBUG:boto:Data: 2022-09-03T07:00:34.976 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.977 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:34.978 DEBUG:boto:Port: 8001 2022-09-03T07:00:34.978 DEBUG:boto:Params: {} 2022-09-03T07:00:34.979 DEBUG:boto:Token: None 2022-09-03T07:00:34.980 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/f 2022-09-03T07:00:34.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iWjFG8izO3Okp7w6nHq4V7Yg+zM= 2022-09-03T07:00:34.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iWjFG8izO3Okp7w6nHq4V7Yg+zM=', 'Content-Length': '0'} 2022-09-03T07:00:34.984 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b628083123c63932-006312fb92-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.985 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:34.985 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:34.986 DEBUG:boto:Method: HEAD 2022-09-03T07:00:34.987 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:34.987 DEBUG:boto:Data: 2022-09-03T07:00:34.988 DEBUG:boto:Headers: {} 2022-09-03T07:00:34.988 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:34.989 DEBUG:boto:Port: 8000 2022-09-03T07:00:34.990 DEBUG:boto:Params: {} 2022-09-03T07:00:34.991 DEBUG:boto:Token: None 2022-09-03T07:00:34.991 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:34 GMT /avysmw-72/f 2022-09-03T07:00:34.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iWjFG8izO3Okp7w6nHq4V7Yg+zM= 2022-09-03T07:00:34.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iWjFG8izO3Okp7w6nHq4V7Yg+zM=', 'Content-Length': '0'} 2022-09-03T07:00:34.996 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e55e7abda62ae8b-006312fb92-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:34.998 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:00:34.999 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:35.000 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:35.001 DEBUG:boto:Method: GET 2022-09-03T07:00:35.002 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:35.002 DEBUG:boto:Data: 2022-09-03T07:00:35.003 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.004 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.005 DEBUG:boto:Params: {} 2022-09-03T07:00:35.005 DEBUG:boto:Token: None 2022-09-03T07:00:35.006 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/f 2022-09-03T07:00:35.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SG1GlIYJkcK0IxjVG4thq136WCk= 2022-09-03T07:00:35.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SG1GlIYJkcK0IxjVG4thq136WCk=', 'Content-Length': '0'} 2022-09-03T07:00:35.010 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012f891aeae66bd0c-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.011 DEBUG:boto:path=/avysmw-72/f 2022-09-03T07:00:35.012 DEBUG:boto:auth_path=/avysmw-72/f 2022-09-03T07:00:35.013 DEBUG:boto:Method: GET 2022-09-03T07:00:35.014 DEBUG:boto:Path: /avysmw-72/f 2022-09-03T07:00:35.015 DEBUG:boto:Data: 2022-09-03T07:00:35.016 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.017 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.018 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.018 DEBUG:boto:Params: {} 2022-09-03T07:00:35.019 DEBUG:boto:Token: None 2022-09-03T07:00:35.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/f 2022-09-03T07:00:35.021 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SG1GlIYJkcK0IxjVG4thq136WCk= 2022-09-03T07:00:35.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SG1GlIYJkcK0IxjVG4thq136WCk=', 'Content-Length': '0'} 2022-09-03T07:00:35.024 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab770ce1fc041c65-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.025 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.026 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.027 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:35.028 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:35.029 DEBUG:boto:Method: GET 2022-09-03T07:00:35.029 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:35.030 DEBUG:boto:Data: 2022-09-03T07:00:35.031 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.033 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.033 DEBUG:boto:Params: {} 2022-09-03T07:00:35.034 DEBUG:boto:Token: None 2022-09-03T07:00:35.035 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.042 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000024929bc19815b33e-006312fb93-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.082 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.084 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.085 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.087 DEBUG:boto:path=/avysmw-72/?prefix=f 2022-09-03T07:00:35.088 DEBUG:boto:auth_path=/avysmw-72/?prefix=f 2022-09-03T07:00:35.089 DEBUG:boto:Method: GET 2022-09-03T07:00:35.090 DEBUG:boto:Path: /avysmw-72/?prefix=f 2022-09-03T07:00:35.091 DEBUG:boto:Data: 2022-09-03T07:00:35.093 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.094 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.095 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.095 DEBUG:boto:Params: {} 2022-09-03T07:00:35.097 DEBUG:boto:Token: None 2022-09-03T07:00:35.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.102 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ca1c3bf5dedb96e9-006312fb93-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.104 DEBUG:boto:b'avysmw-72f1000falsef2022-09-03T06:57:02.898Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.106 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:00:35.107 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.107 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.110 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.110 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.111 DEBUG:boto:Method: GET 2022-09-03T07:00:35.112 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:35.113 DEBUG:boto:Data: 2022-09-03T07:00:35.114 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.116 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.118 DEBUG:boto:Params: {} 2022-09-03T07:00:35.119 DEBUG:boto:Token: None 2022-09-03T07:00:35.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:35.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:loe52SuIRMExj+xXoq2he5ViIco= 2022-09-03T07:00:35.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:loe52SuIRMExj+xXoq2he5ViIco=', 'Content-Length': '0'} 2022-09-03T07:00:35.126 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000daa492a16acc5f-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.128 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.129 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.130 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.131 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.132 DEBUG:boto:Method: GET 2022-09-03T07:00:35.133 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:35.134 DEBUG:boto:Data: 2022-09-03T07:00:35.135 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.136 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.137 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.138 DEBUG:boto:Params: {} 2022-09-03T07:00:35.139 DEBUG:boto:Token: None 2022-09-03T07:00:35.140 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:35.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:loe52SuIRMExj+xXoq2he5ViIco= 2022-09-03T07:00:35.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:loe52SuIRMExj+xXoq2he5ViIco=', 'Content-Length': '0'} 2022-09-03T07:00:35.145 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f7996bb5149f3d2-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.146 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.147 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.148 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.149 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.150 DEBUG:boto:Method: HEAD 2022-09-03T07:00:35.151 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:35.152 DEBUG:boto:Data: 2022-09-03T07:00:35.153 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.154 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.155 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.155 DEBUG:boto:Params: {} 2022-09-03T07:00:35.156 DEBUG:boto:Token: None 2022-09-03T07:00:35.158 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:35.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SEjidNxQe1d6mz6L6T19dpxo2sM= 2022-09-03T07:00:35.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SEjidNxQe1d6mz6L6T19dpxo2sM=', 'Content-Length': '0'} 2022-09-03T07:00:35.162 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087a8e74fa532e22c-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.164 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.164 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.165 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.166 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.167 DEBUG:boto:Method: HEAD 2022-09-03T07:00:35.168 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:35.169 DEBUG:boto:Data: 2022-09-03T07:00:35.170 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.171 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.172 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.173 DEBUG:boto:Params: {} 2022-09-03T07:00:35.175 DEBUG:boto:Token: None 2022-09-03T07:00:35.176 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:35.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SEjidNxQe1d6mz6L6T19dpxo2sM= 2022-09-03T07:00:35.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SEjidNxQe1d6mz6L6T19dpxo2sM=', 'Content-Length': '0'} 2022-09-03T07:00:35.180 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e536a4a93829271-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.181 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:00:35.182 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.183 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.184 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.185 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.185 DEBUG:boto:Method: GET 2022-09-03T07:00:35.186 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:35.186 DEBUG:boto:Data: 2022-09-03T07:00:35.187 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.188 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.189 DEBUG:boto:Params: {} 2022-09-03T07:00:35.189 DEBUG:boto:Token: None 2022-09-03T07:00:35.190 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:35.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:loe52SuIRMExj+xXoq2he5ViIco= 2022-09-03T07:00:35.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:loe52SuIRMExj+xXoq2he5ViIco=', 'Content-Length': '0'} 2022-09-03T07:00:35.193 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000404ef6fc7f37e30c-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.194 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.195 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.204 DEBUG:boto:path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.205 DEBUG:boto:auth_path=/avysmw-72/g?versionId=null 2022-09-03T07:00:35.205 DEBUG:boto:Method: GET 2022-09-03T07:00:35.206 DEBUG:boto:Path: /avysmw-72/g?versionId=null 2022-09-03T07:00:35.207 DEBUG:boto:Data: 2022-09-03T07:00:35.207 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.208 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.208 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.209 DEBUG:boto:Params: {} 2022-09-03T07:00:35.210 DEBUG:boto:Token: None 2022-09-03T07:00:35.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g?versionId=null 2022-09-03T07:00:35.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:loe52SuIRMExj+xXoq2he5ViIco= 2022-09-03T07:00:35.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:loe52SuIRMExj+xXoq2he5ViIco=', 'Content-Length': '0'} 2022-09-03T07:00:35.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b8891b8380c9b42-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.217 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.218 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.219 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:35.219 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:35.220 DEBUG:boto:Method: GET 2022-09-03T07:00:35.221 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:35.222 DEBUG:boto:Data: 2022-09-03T07:00:35.223 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.223 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.224 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.225 DEBUG:boto:Params: {} 2022-09-03T07:00:35.226 DEBUG:boto:Token: None 2022-09-03T07:00:35.227 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.232 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000077c0e170511f7f19-006312fb93-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.233 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.233 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.234 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.235 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:35.235 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:35.236 DEBUG:boto:Method: GET 2022-09-03T07:00:35.236 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:35.237 DEBUG:boto:Data: 2022-09-03T07:00:35.237 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.238 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.239 DEBUG:boto:Params: {} 2022-09-03T07:00:35.240 DEBUG:boto:Token: None 2022-09-03T07:00:35.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.244 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a5edb83a9c05cdb-006312fb93-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.244 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.245 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.246 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.247 DEBUG:boto:Method: HEAD 2022-09-03T07:00:35.248 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:35.249 DEBUG:boto:Data: 2022-09-03T07:00:35.250 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.251 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.252 DEBUG:boto:Params: {} 2022-09-03T07:00:35.252 DEBUG:boto:Token: None 2022-09-03T07:00:35.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g 2022-09-03T07:00:35.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s+mnotUMZPRBd0MSyf2Af+VFwmk= 2022-09-03T07:00:35.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s+mnotUMZPRBd0MSyf2Af+VFwmk=', 'Content-Length': '0'} 2022-09-03T07:00:35.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009163f57f2d875462-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.257 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.258 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.258 DEBUG:boto:Method: HEAD 2022-09-03T07:00:35.259 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:35.259 DEBUG:boto:Data: 2022-09-03T07:00:35.260 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.260 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.261 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.261 DEBUG:boto:Params: {} 2022-09-03T07:00:35.262 DEBUG:boto:Token: None 2022-09-03T07:00:35.263 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g 2022-09-03T07:00:35.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s+mnotUMZPRBd0MSyf2Af+VFwmk= 2022-09-03T07:00:35.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s+mnotUMZPRBd0MSyf2Af+VFwmk=', 'Content-Length': '0'} 2022-09-03T07:00:35.266 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000019d71463079ae9c5-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.267 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:00:35.267 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.268 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.268 DEBUG:boto:Method: GET 2022-09-03T07:00:35.269 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:35.270 DEBUG:boto:Data: 2022-09-03T07:00:35.270 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.271 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.272 DEBUG:boto:Params: {} 2022-09-03T07:00:35.273 DEBUG:boto:Token: None 2022-09-03T07:00:35.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g 2022-09-03T07:00:35.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zxvXZ2uzU3RaKxHJLtzpJCMb9CQ= 2022-09-03T07:00:35.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zxvXZ2uzU3RaKxHJLtzpJCMb9CQ=', 'Content-Length': '0'} 2022-09-03T07:00:35.277 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000684fdecb0619faeb-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.278 DEBUG:boto:path=/avysmw-72/g 2022-09-03T07:00:35.279 DEBUG:boto:auth_path=/avysmw-72/g 2022-09-03T07:00:35.279 DEBUG:boto:Method: GET 2022-09-03T07:00:35.280 DEBUG:boto:Path: /avysmw-72/g 2022-09-03T07:00:35.280 DEBUG:boto:Data: 2022-09-03T07:00:35.281 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.282 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.282 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.283 DEBUG:boto:Params: {} 2022-09-03T07:00:35.284 DEBUG:boto:Token: None 2022-09-03T07:00:35.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/g 2022-09-03T07:00:35.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zxvXZ2uzU3RaKxHJLtzpJCMb9CQ= 2022-09-03T07:00:35.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zxvXZ2uzU3RaKxHJLtzpJCMb9CQ=', 'Content-Length': '0'} 2022-09-03T07:00:35.291 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002aed36964b8138af-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.293 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.293 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.294 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:35.295 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:35.295 DEBUG:boto:Method: GET 2022-09-03T07:00:35.296 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:35.296 DEBUG:boto:Data: 2022-09-03T07:00:35.297 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.298 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.299 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.299 DEBUG:boto:Params: {} 2022-09-03T07:00:35.300 DEBUG:boto:Token: None 2022-09-03T07:00:35.300 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.301 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.304 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b98b170a4f01d40b-006312fb93-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.305 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.306 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.307 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.307 DEBUG:boto:path=/avysmw-72/?prefix=g 2022-09-03T07:00:35.308 DEBUG:boto:auth_path=/avysmw-72/?prefix=g 2022-09-03T07:00:35.308 DEBUG:boto:Method: GET 2022-09-03T07:00:35.309 DEBUG:boto:Path: /avysmw-72/?prefix=g 2022-09-03T07:00:35.310 DEBUG:boto:Data: 2022-09-03T07:00:35.310 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.311 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.311 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.312 DEBUG:boto:Params: {} 2022-09-03T07:00:35.313 DEBUG:boto:Token: None 2022-09-03T07:00:35.314 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.319 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a2a3aef9c5dcf4a2-006312fb93-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.362 DEBUG:boto:b'avysmw-72g1000falseg2022-09-03T06:57:02.923Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.363 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:00:35.364 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.364 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.365 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.365 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.366 DEBUG:boto:Method: GET 2022-09-03T07:00:35.366 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:35.367 DEBUG:boto:Data: 2022-09-03T07:00:35.367 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.368 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.368 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.369 DEBUG:boto:Params: {} 2022-09-03T07:00:35.370 DEBUG:boto:Token: None 2022-09-03T07:00:35.370 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:35.371 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzmHBIWAwc5uKp94DKsZNBTVhls= 2022-09-03T07:00:35.372 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzmHBIWAwc5uKp94DKsZNBTVhls=', 'Content-Length': '0'} 2022-09-03T07:00:35.374 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008028c56968d01a59-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.375 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.375 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.376 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.376 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.377 DEBUG:boto:Method: GET 2022-09-03T07:00:35.377 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:35.378 DEBUG:boto:Data: 2022-09-03T07:00:35.378 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.379 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.380 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.380 DEBUG:boto:Params: {} 2022-09-03T07:00:35.381 DEBUG:boto:Token: None 2022-09-03T07:00:35.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:35.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzmHBIWAwc5uKp94DKsZNBTVhls= 2022-09-03T07:00:35.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzmHBIWAwc5uKp94DKsZNBTVhls=', 'Content-Length': '0'} 2022-09-03T07:00:35.385 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079b2f839c2c333d7-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.386 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.386 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.387 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.387 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.388 DEBUG:boto:Method: HEAD 2022-09-03T07:00:35.388 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:35.389 DEBUG:boto:Data: 2022-09-03T07:00:35.389 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.391 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.391 DEBUG:boto:Params: {} 2022-09-03T07:00:35.392 DEBUG:boto:Token: None 2022-09-03T07:00:35.392 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:35.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vyfD5jUCdbP3q7q2hpWI6MDfDnM= 2022-09-03T07:00:35.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vyfD5jUCdbP3q7q2hpWI6MDfDnM=', 'Content-Length': '0'} 2022-09-03T07:00:35.396 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af5a63b571297405-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.396 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.397 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.397 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.398 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.398 DEBUG:boto:Method: HEAD 2022-09-03T07:00:35.399 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:35.399 DEBUG:boto:Data: 2022-09-03T07:00:35.400 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.400 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.401 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.401 DEBUG:boto:Params: {} 2022-09-03T07:00:35.402 DEBUG:boto:Token: None 2022-09-03T07:00:35.403 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:35.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vyfD5jUCdbP3q7q2hpWI6MDfDnM= 2022-09-03T07:00:35.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vyfD5jUCdbP3q7q2hpWI6MDfDnM=', 'Content-Length': '0'} 2022-09-03T07:00:35.406 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007bde5dc29843aaab-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.406 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:00:35.407 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.407 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.408 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.409 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.409 DEBUG:boto:Method: GET 2022-09-03T07:00:35.410 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:35.410 DEBUG:boto:Data: 2022-09-03T07:00:35.411 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.411 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.412 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.412 DEBUG:boto:Params: {} 2022-09-03T07:00:35.413 DEBUG:boto:Token: None 2022-09-03T07:00:35.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:35.414 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzmHBIWAwc5uKp94DKsZNBTVhls= 2022-09-03T07:00:35.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzmHBIWAwc5uKp94DKsZNBTVhls=', 'Content-Length': '0'} 2022-09-03T07:00:35.417 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6de64da09e76d5e-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.459 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.460 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.461 DEBUG:boto:path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.461 DEBUG:boto:auth_path=/avysmw-72/h?versionId=null 2022-09-03T07:00:35.462 DEBUG:boto:Method: GET 2022-09-03T07:00:35.462 DEBUG:boto:Path: /avysmw-72/h?versionId=null 2022-09-03T07:00:35.463 DEBUG:boto:Data: 2022-09-03T07:00:35.464 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.464 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.465 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.465 DEBUG:boto:Params: {} 2022-09-03T07:00:35.466 DEBUG:boto:Token: None 2022-09-03T07:00:35.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h?versionId=null 2022-09-03T07:00:35.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XzmHBIWAwc5uKp94DKsZNBTVhls= 2022-09-03T07:00:35.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XzmHBIWAwc5uKp94DKsZNBTVhls=', 'Content-Length': '0'} 2022-09-03T07:00:35.471 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1ba052c106ecccd-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.472 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.473 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.474 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:35.475 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:35.476 DEBUG:boto:Method: GET 2022-09-03T07:00:35.477 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:35.478 DEBUG:boto:Data: 2022-09-03T07:00:35.479 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.480 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.480 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.481 DEBUG:boto:Params: {} 2022-09-03T07:00:35.482 DEBUG:boto:Token: None 2022-09-03T07:00:35.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.488 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000b20f88596b47427-006312fb93-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.489 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.490 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.491 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.491 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:35.492 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:35.492 DEBUG:boto:Method: GET 2022-09-03T07:00:35.493 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:35.493 DEBUG:boto:Data: 2022-09-03T07:00:35.494 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.494 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.495 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.495 DEBUG:boto:Params: {} 2022-09-03T07:00:35.496 DEBUG:boto:Token: None 2022-09-03T07:00:35.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.500 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069ea451aff167fb0-006312fb93-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.501 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.502 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.502 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.503 DEBUG:boto:Method: HEAD 2022-09-03T07:00:35.503 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:35.504 DEBUG:boto:Data: 2022-09-03T07:00:35.504 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.505 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.506 DEBUG:boto:Params: {} 2022-09-03T07:00:35.506 DEBUG:boto:Token: None 2022-09-03T07:00:35.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h 2022-09-03T07:00:35.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HlNFxsLTf+QeBn9nSbKj8qsWmmk= 2022-09-03T07:00:35.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HlNFxsLTf+QeBn9nSbKj8qsWmmk=', 'Content-Length': '0'} 2022-09-03T07:00:35.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b5b446a1c66b12f-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.511 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.512 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.512 DEBUG:boto:Method: HEAD 2022-09-03T07:00:35.513 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:35.513 DEBUG:boto:Data: 2022-09-03T07:00:35.514 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.514 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.515 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.515 DEBUG:boto:Params: {} 2022-09-03T07:00:35.516 DEBUG:boto:Token: None 2022-09-03T07:00:35.517 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h 2022-09-03T07:00:35.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HlNFxsLTf+QeBn9nSbKj8qsWmmk= 2022-09-03T07:00:35.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HlNFxsLTf+QeBn9nSbKj8qsWmmk=', 'Content-Length': '0'} 2022-09-03T07:00:35.519 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a22fc0bd51709de-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.520 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:00:35.521 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.521 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.522 DEBUG:boto:Method: GET 2022-09-03T07:00:35.522 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:35.523 DEBUG:boto:Data: 2022-09-03T07:00:35.523 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.524 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.525 DEBUG:boto:Params: {} 2022-09-03T07:00:35.526 DEBUG:boto:Token: None 2022-09-03T07:00:35.526 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h 2022-09-03T07:00:35.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/888LnkNcX680w0I9ZJ4uHHNyuA= 2022-09-03T07:00:35.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/888LnkNcX680w0I9ZJ4uHHNyuA=', 'Content-Length': '0'} 2022-09-03T07:00:35.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006cfd6e9030d75da9-006312fb93-1608-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.574 DEBUG:boto:path=/avysmw-72/h 2022-09-03T07:00:35.575 DEBUG:boto:auth_path=/avysmw-72/h 2022-09-03T07:00:35.575 DEBUG:boto:Method: GET 2022-09-03T07:00:35.576 DEBUG:boto:Path: /avysmw-72/h 2022-09-03T07:00:35.577 DEBUG:boto:Data: 2022-09-03T07:00:35.577 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.578 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.578 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.579 DEBUG:boto:Params: {} 2022-09-03T07:00:35.580 DEBUG:boto:Token: None 2022-09-03T07:00:35.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/h 2022-09-03T07:00:35.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/888LnkNcX680w0I9ZJ4uHHNyuA= 2022-09-03T07:00:35.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/888LnkNcX680w0I9ZJ4uHHNyuA=', 'Content-Length': '0'} 2022-09-03T07:00:35.584 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 06:57:02 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c5040252005c7f9-006312fb93-170f-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.585 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.585 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.586 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:35.587 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:35.587 DEBUG:boto:Method: GET 2022-09-03T07:00:35.588 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:35.588 DEBUG:boto:Data: 2022-09-03T07:00:35.589 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.590 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:35.590 DEBUG:boto:Port: 8001 2022-09-03T07:00:35.591 DEBUG:boto:Params: {} 2022-09-03T07:00:35.591 DEBUG:boto:Token: None 2022-09-03T07:00:35.592 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.596 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006cfddf8deaca0116-006312fb93-1608-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.597 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.598 DEBUG:boto:path=/avysmw-72/ 2022-09-03T07:00:35.599 DEBUG:boto:auth_path=/avysmw-72/ 2022-09-03T07:00:35.599 DEBUG:boto:path=/avysmw-72/?prefix=h 2022-09-03T07:00:35.600 DEBUG:boto:auth_path=/avysmw-72/?prefix=h 2022-09-03T07:00:35.600 DEBUG:boto:Method: GET 2022-09-03T07:00:35.601 DEBUG:boto:Path: /avysmw-72/?prefix=h 2022-09-03T07:00:35.602 DEBUG:boto:Data: 2022-09-03T07:00:35.602 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.603 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:35.603 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.604 DEBUG:boto:Params: {} 2022-09-03T07:00:35.604 DEBUG:boto:Token: None 2022-09-03T07:00:35.605 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:00:35 GMT /avysmw-72/ 2022-09-03T07:00:35.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs= 2022-09-03T07:00:35.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VCMN949tAnjvHW3DL5Q7lxqEHrs=', 'Content-Length': '0'} 2022-09-03T07:00:35.609 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e4dcd38d281a0aa5-006312fb93-170f-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.609 DEBUG:boto:b'avysmw-72h1000falseh2022-09-03T06:57:02.991Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:00:35.610 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-72 zones={test-zone3, test-zone2} 2022-09-03T07:00:35.612 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_reshard_incremental ... ok 2022-09-03T07:00:35.614 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:00:35.615 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:00:35.615 DEBUG:boto:Method: PUT 2022-09-03T07:00:35.616 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:00:35.616 DEBUG:boto:Data: 2022-09-03T07:00:35.617 DEBUG:boto:Headers: {} 2022-09-03T07:00:35.617 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:35.618 DEBUG:boto:Port: 8000 2022-09-03T07:00:35.618 DEBUG:boto:Params: {} 2022-09-03T07:00:35.619 DEBUG:boto:Token: None 2022-09-03T07:00:35.619 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 07:00:35 GMT /avysmw-73/ 2022-09-03T07:00:35.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bVcJ3LdlZH6BOhrkGc/sIMD4mZg= 2022-09-03T07:00:35.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bVcJ3LdlZH6BOhrkGc/sIMD4mZg=', 'Content-Length': '0'} 2022-09-03T07:00:35.642 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000002a8042c86b409659-006312fb93-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:35.643 DEBUG:rgw_multi.tests:created bucket=avysmw-73 2022-09-03T07:00:35.644 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:35.645 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:35.645 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:00:35.703 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:35.704 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:35.735 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T07:00:35.751 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:00:35.751 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:00:35.752 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:00:35.752 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:00:35.752 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:00:35.752 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:00:35.753 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662188435.639499_451.1", 2022-09-03T07:00:35.753 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:00:35.639499Z" 2022-09-03T07:00:35.753 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:00:35.753 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:00:35.754 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:00:35.754 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:00:35.754 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:00:35.755 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:00:35.755 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:00:35.755 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:00:35.755 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:00:35.756 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:00:35.757 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662188435.639499_451.1', 2: '', 3: ''} 2022-09-03T07:00:35.758 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T07:00:35.758 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:35.759 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:35.759 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:00:35.813 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:35.814 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:35.856 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:35.854+0000 7feaeb0e6600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:00:35.857 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:35.855+0000 7feaeb0e6600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:00:35.883 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:00:35.884 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:00:35.885 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:00:35.885 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:00:35.885 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:00:35.887 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:00:35.887 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:00:35.888 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:35.888 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:00:35.888 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:35.889 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:00:35.889 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:35.889 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:35.890 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:00:35.890 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:35.890 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:00:35.890 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:35.891 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:00:35.891 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:00:35.891 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:35.892 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:35.892 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:35.893 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:00:35.893 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:35.893 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:35.893 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662188222.810607_438.1", 2022-09-03T07:00:35.894 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:35.894 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:00:35.895 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:35.895 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T06:57:02.810607Z", 2022-09-03T07:00:35.896 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:00:35.896 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:35.897 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:35.897 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:35.897 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:00:35.898 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:35.898 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:35.899 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:00:35.899 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:35.899 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:00:35.900 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:35.900 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:00:35.900 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:00:35.901 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:35.901 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:35.901 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:35.902 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:00:35.902 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:35.902 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:35.903 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:00:35.903 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:35.903 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:00:35.903 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:35.904 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:00:35.904 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:00:35.904 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:35.904 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:35.905 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:00:35.905 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:35.905 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:00:35.905 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:00:35.906 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:00:35.906 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:35.907 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:00:35Z" 2022-09-03T07:00:35.907 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:00:35.908 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188222.810607_438.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T06:57:02.810607Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:00:35Z" } 2022-09-03T07:00:35.909 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188222.810607_438.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T06:57:02.810607Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:00:35.909 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188435.639499_451.1', 2: '', 3: ''} 2022-09-03T07:00:35.910 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188222.810607_438.1', 2: '', 3: ''} 2022-09-03T07:00:35.911 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662188435.639499_451.1 target=1_1662188222.810607_438.1 2022-09-03T07:00:40.912 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:40.913 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:40.914 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:00:40.967 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:40.968 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:41.015 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:41.013+0000 7f10b35ab600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:00:41.016 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:41.014+0000 7f10b35ab600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:00:41.034 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:00:41.034 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:00:41.034 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:00:41.035 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:00:41.035 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:00:41.035 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:00:41.035 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:00:41.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.036 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:00:41.036 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:41.036 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:00:41.037 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:41.037 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:41.037 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:00:41.037 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:41.038 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:00:41.038 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:41.038 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:00:41.038 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:00:41.039 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:41.039 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:00:41.040 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:41.040 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:41.040 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662188435.639499_451.1", 2022-09-03T07:00:41.040 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:41.041 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:00:41.041 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:41.041 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:00:35.639499Z", 2022-09-03T07:00:41.041 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:00:41.042 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.042 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:41.042 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:00:41.043 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:41.043 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:41.043 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:00:41.043 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:41.044 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:00:41.044 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:41.044 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:00:41.044 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:00:41.045 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:41.045 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:00:41.046 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:41.046 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:41.046 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:00:41.046 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:41.047 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:00:41.047 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:41.047 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:00:41.047 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:00:41.048 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.048 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.048 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:00:41.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.049 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:00:41.049 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:00:41.049 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:00:41.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.050 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:00:41Z" 2022-09-03T07:00:41.050 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:00:41.051 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188435.639499_451.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:00:35.639499Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:00:41Z" } 2022-09-03T07:00:41.052 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188435.639499_451.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:00:35.639499Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:00:41.052 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188435.639499_451.1', 2: '', 3: ''} 2022-09-03T07:00:41.053 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188435.639499_451.1', 2: '', 3: ''} 2022-09-03T07:00:41.053 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T07:00:41.054 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T07:00:41.054 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:41.055 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:41.055 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:00:41.113 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:41.113 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:41.158 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:00:41.156+0000 7f7bbb004600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:00:41.159 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:00:41.157+0000 7f7bbb004600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:00:41.179 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:00:41.180 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:00:41.181 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:00:41.181 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:00:41.182 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:00:41.182 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:00:41.182 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:00:41.183 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:00:41.183 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:00:41.184 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:00:41.184 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:00:41.185 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:00:41.185 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:00:41.185 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:00:41.186 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:00:41.186 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T07:00:41.187 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:00:41.187 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:00:41.188 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:00:41.188 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:00:41.189 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:00:41.189 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:00:41.189 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:00:41.190 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:00:41.190 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:00:41.191 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662188435.639499_451.1", 2022-09-03T07:00:41.191 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:00:41.192 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:00:41.192 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:00:41.193 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:00:35.639499Z", 2022-09-03T07:00:41.193 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:00:41.193 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:00:41.194 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:00:41.194 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:00:41.195 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:00:41.195 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:00:41.196 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:00:41.196 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:00:41.197 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:00:41.197 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:00:41.197 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:00:41.198 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:00:41.198 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:00:41.199 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:00:41.199 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:00:41.200 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:00:41.200 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:00:41.201 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:00:41.201 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:00:41.202 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:00:41.202 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:00:41.202 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:00:41.203 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:00:41.204 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:00:41.204 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:00:41.204 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:00:41.205 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:00:41.205 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:00:41.206 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:00:41.206 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:00:41.207 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T07:00:41.207 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T07:00:41.208 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:00:41.208 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:00:41Z" 2022-09-03T07:00:41.208 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:00:41.210 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188435.639499_451.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:00:35.639499Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:00:41Z" } 2022-09-03T07:00:41.211 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188435.639499_451.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:00:35.639499Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:00:41.212 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188435.639499_451.1', 2: '', 3: ''} 2022-09-03T07:00:41.213 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188435.639499_451.1', 2: '', 3: ''} 2022-09-03T07:00:41.214 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T07:00:41.215 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T07:00:41.216 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:41.217 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:00:41.218 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:00:41.269 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:41.270 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:41.327 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:41.325+0000 7f57d03fb600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:00:41.329 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:00:41.328+0000 7f57d03fb600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:00:41.355 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:00:41.355 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:00:41.356 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:00:41.356 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:00:41.356 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:00:41.357 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:00:41.357 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:00:41.357 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.357 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:00:41.358 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:41.358 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:00:41.358 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:41.358 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:41.359 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:00:41.359 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:41.359 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:00:41.360 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:41.360 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:00:41.360 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:00:41.360 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.361 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.361 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:41.361 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:00:41.361 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:41.362 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:41.362 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662188435.639499_451.1", 2022-09-03T07:00:41.362 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:41.363 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:00:41.363 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:41.363 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:00:35.639499Z", 2022-09-03T07:00:41.363 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:00:41.364 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.364 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.364 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:41.364 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:00:41.364 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:41.365 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:41.365 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:00:41.365 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:41.365 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:00:41.366 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:41.366 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:00:41.366 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:00:41.367 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.367 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.367 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:00:41.367 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:00:41.368 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:00:41.368 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:00:41.368 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:00:41.368 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:00:41.369 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:00:41.369 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:00:41.369 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:00:41.369 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:00:41.370 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.370 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:00:41.370 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:00:41.370 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.371 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:00:41.371 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:00:41.371 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:00:41.372 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:00:41.372 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:00:41Z" 2022-09-03T07:00:41.372 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:00:41.373 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188435.639499_451.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:00:35.639499Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:00:41Z" } 2022-09-03T07:00:41.374 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188435.639499_451.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:00:35.639499Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:00:41.375 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188435.639499_451.1', 2: '', 3: ''} 2022-09-03T07:00:41.375 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188435.639499_451.1', 2: '', 3: ''} 2022-09-03T07:00:41.376 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T07:00:41.376 DEBUG:tasks.rgw.c2.client.0:waiting for process to exit 2022-09-03T07:00:41.377 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T07:00:41.378 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T07:00:41.375+0000 7f50941f0700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm (PID: 111489) UID: 0 2022-09-03T07:00:41.379 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T07:00:41.375+0000 7f509845b600 -1 shutting down 2022-09-03T07:00:44.295 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:00:44.292+0000 7f8d0d1f4700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16) 2022-09-03T07:00:44.308 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T07:00:44.305+0000 7f1779c93700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5) 2022-09-03T07:00:44.394 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T07:00:44.391+0000 7f1779c93700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16) 2022-09-03T07:00:44.395 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T07:00:44.391+0000 7f1779c93700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5) 2022-09-03T07:00:44.414 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:00:44.411+0000 7f8d0d1f4700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5) 2022-09-03T07:00:44.462 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:00:44.459+0000 7f8d0d1f4700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16) 2022-09-03T07:00:44.520 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T07:00:44.516+0000 7f1779c93700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16) 2022-09-03T07:00:44.550 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:00:44.548+0000 7f8d0d1f4700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16) 2022-09-03T07:00:46.925 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~0s 2022-09-03T07:00:47.480 INFO:tasks.rgw.c2.client.0:Stopped 2022-09-03T07:00:47.481 DEBUG:tasks.rgw.c1.client.1:waiting for process to exit 2022-09-03T07:00:47.482 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T07:00:47.484 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T07:00:47.481+0000 7f17974ce700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm (PID: 128221) UID: 0 2022-09-03T07:00:47.485 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T07:00:47.481+0000 7f179b739600 -1 shutting down 2022-09-03T07:00:51.032 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~0s 2022-09-03T07:00:52.533 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 has been restored 2022-09-03T07:00:53.585 INFO:tasks.rgw.c1.client.1:Stopped 2022-09-03T07:00:53.587 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:00:53.588 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:00:53.589 DEBUG:boto:Method: HEAD 2022-09-03T07:00:53.591 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:00:53.591 DEBUG:boto:Data: 2022-09-03T07:00:53.592 DEBUG:boto:Headers: {} 2022-09-03T07:00:53.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.594 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.596 DEBUG:boto:Params: {} 2022-09-03T07:00:53.597 DEBUG:boto:Token: None 2022-09-03T07:00:53.598 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/ 2022-09-03T07:00:53.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk= 2022-09-03T07:00:53.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk=', 'Content-Length': '0'} 2022-09-03T07:00:53.605 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000894fce0f8785d75e-006312fba5-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:53.607 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:00:53.608 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:00:53.609 DEBUG:boto:Method: PUT 2022-09-03T07:00:53.610 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:00:53.611 DEBUG:boto:Data: 2022-09-03T07:00:53.612 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:00:53.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.614 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.615 DEBUG:boto:Params: {} 2022-09-03T07:00:53.616 DEBUG:boto:Token: None 2022-09-03T07:00:53.618 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/a 2022-09-03T07:00:53.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VeRR1T6KBiP1jcXsi0BMnOexpaQ= 2022-09-03T07:00:53.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VeRR1T6KBiP1jcXsi0BMnOexpaQ='} 2022-09-03T07:00:53.630 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000bf8cacca5b1863b4-006312fba5-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:53.632 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:00:53.632 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:00:53.633 DEBUG:boto:Method: HEAD 2022-09-03T07:00:53.634 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:00:53.634 DEBUG:boto:Data: 2022-09-03T07:00:53.635 DEBUG:boto:Headers: {} 2022-09-03T07:00:53.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.637 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.638 DEBUG:boto:Params: {} 2022-09-03T07:00:53.639 DEBUG:boto:Token: None 2022-09-03T07:00:53.640 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/ 2022-09-03T07:00:53.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk= 2022-09-03T07:00:53.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk=', 'Content-Length': '0'} 2022-09-03T07:00:53.645 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c4d87989acc7d8a5-006312fba5-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:53.646 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:00:53.647 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:00:53.647 DEBUG:boto:Method: PUT 2022-09-03T07:00:53.648 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:00:53.649 DEBUG:boto:Data: 2022-09-03T07:00:53.649 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:00:53.650 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.651 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.652 DEBUG:boto:Params: {} 2022-09-03T07:00:53.652 DEBUG:boto:Token: None 2022-09-03T07:00:53.653 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/b 2022-09-03T07:00:53.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pu7iq1eF22wU84pN6aHLf4/1uLA= 2022-09-03T07:00:53.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pu7iq1eF22wU84pN6aHLf4/1uLA='} 2022-09-03T07:00:53.698 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000fdd1ad7eaa67fa33-006312fba5-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:53.699 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:00:53.700 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:00:53.700 DEBUG:boto:Method: HEAD 2022-09-03T07:00:53.701 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:00:53.701 DEBUG:boto:Data: 2022-09-03T07:00:53.702 DEBUG:boto:Headers: {} 2022-09-03T07:00:53.702 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.703 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.703 DEBUG:boto:Params: {} 2022-09-03T07:00:53.704 DEBUG:boto:Token: None 2022-09-03T07:00:53.705 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/ 2022-09-03T07:00:53.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk= 2022-09-03T07:00:53.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk=', 'Content-Length': '0'} 2022-09-03T07:00:53.710 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006cfc1b0a82b7e891-006312fba5-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:53.711 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:00:53.711 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:00:53.712 DEBUG:boto:Method: PUT 2022-09-03T07:00:53.712 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:00:53.713 DEBUG:boto:Data: 2022-09-03T07:00:53.713 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:00:53.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.715 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.715 DEBUG:boto:Params: {} 2022-09-03T07:00:53.716 DEBUG:boto:Token: None 2022-09-03T07:00:53.716 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/c 2022-09-03T07:00:53.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n4ki1ugNSlFh4j+pN23+xp7zpVw= 2022-09-03T07:00:53.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n4ki1ugNSlFh4j+pN23+xp7zpVw='} 2022-09-03T07:00:53.726 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000758dda0a1195e5b6-006312fba5-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:53.727 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:00:53.727 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:00:53.728 DEBUG:boto:Method: HEAD 2022-09-03T07:00:53.728 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:00:53.729 DEBUG:boto:Data: 2022-09-03T07:00:53.729 DEBUG:boto:Headers: {} 2022-09-03T07:00:53.730 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.731 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.731 DEBUG:boto:Params: {} 2022-09-03T07:00:53.732 DEBUG:boto:Token: None 2022-09-03T07:00:53.732 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/ 2022-09-03T07:00:53.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk= 2022-09-03T07:00:53.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk=', 'Content-Length': '0'} 2022-09-03T07:00:53.736 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '9'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000df02fa0afcd35d6b-006312fba5-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:53.737 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:00:53.738 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:00:53.738 DEBUG:boto:Method: PUT 2022-09-03T07:00:53.739 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:00:53.739 DEBUG:boto:Data: 2022-09-03T07:00:53.740 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:00:53.740 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.741 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.741 DEBUG:boto:Params: {} 2022-09-03T07:00:53.742 DEBUG:boto:Token: None 2022-09-03T07:00:53.743 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/d 2022-09-03T07:00:53.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqKlN7hfuMxJHNJXTXPVT4zbmBM= 2022-09-03T07:00:53.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqKlN7hfuMxJHNJXTXPVT4zbmBM='} 2022-09-03T07:00:53.755 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000030a99018d5c12770-006312fba5-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:53.756 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'avysmw-73', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:00:53.757 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'avysmw-73', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:00:53.757 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket avysmw-73 --num-shards 3 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:00:53.816 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:00:53.817 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:00:53.888 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:00:53.888 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-73 2022-09-03T07:00:53.893 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 4 2022-09-03T07:00:53.944 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:00:53.945 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:00:53.946 DEBUG:boto:Method: HEAD 2022-09-03T07:00:53.946 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:00:53.947 DEBUG:boto:Data: 2022-09-03T07:00:53.948 DEBUG:boto:Headers: {} 2022-09-03T07:00:53.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.949 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.949 DEBUG:boto:Params: {} 2022-09-03T07:00:53.950 DEBUG:boto:Token: None 2022-09-03T07:00:53.951 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/ 2022-09-03T07:00:53.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk= 2022-09-03T07:00:53.952 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pbQLQQoX0I/3te1lfkr9UKaemXk=', 'Content-Length': '0'} 2022-09-03T07:00:53.954 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000375731149d52697c-006312fba5-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:53.955 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:00:53.956 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:00:53.956 DEBUG:boto:Method: PUT 2022-09-03T07:00:53.957 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:00:53.958 DEBUG:boto:Data: 2022-09-03T07:00:53.958 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:00:53.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:53.959 DEBUG:boto:Port: 8000 2022-09-03T07:00:53.960 DEBUG:boto:Params: {} 2022-09-03T07:00:53.961 DEBUG:boto:Token: None 2022-09-03T07:00:53.961 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:00:53 GMT /avysmw-73/e 2022-09-03T07:00:53.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ujmyLRiVOTpUD2EX+fmQ3wMpyxQ= 2022-09-03T07:00:53.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:00:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ujmyLRiVOTpUD2EX+fmQ3wMpyxQ='} 2022-09-03T07:00:54.006 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000cc3220c92d47b70a-006312fba5-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:54.008 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:00:54.009 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:00:54.010 DEBUG:boto:Method: HEAD 2022-09-03T07:00:54.011 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:00:54.012 DEBUG:boto:Data: 2022-09-03T07:00:54.013 DEBUG:boto:Headers: {} 2022-09-03T07:00:54.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:54.015 DEBUG:boto:Port: 8000 2022-09-03T07:00:54.016 DEBUG:boto:Params: {} 2022-09-03T07:00:54.017 DEBUG:boto:Token: None 2022-09-03T07:00:54.018 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:54 GMT /avysmw-73/ 2022-09-03T07:00:54.019 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:edX0TxvOTONBPDWgF/5zS1ditwM= 2022-09-03T07:00:54.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:edX0TxvOTONBPDWgF/5zS1ditwM=', 'Content-Length': '0'} 2022-09-03T07:00:54.023 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000036ca03f3b66bbc07-006312fba6-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:54.025 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:00:54.026 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:00:54.026 DEBUG:boto:Method: PUT 2022-09-03T07:00:54.027 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:00:54.028 DEBUG:boto:Data: 2022-09-03T07:00:54.028 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:00:54.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:54.030 DEBUG:boto:Port: 8000 2022-09-03T07:00:54.030 DEBUG:boto:Params: {} 2022-09-03T07:00:54.031 DEBUG:boto:Token: None 2022-09-03T07:00:54.032 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:00:54 GMT /avysmw-73/f 2022-09-03T07:00:54.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RPl++SfR46XjlO7utXXsgGhjAk8= 2022-09-03T07:00:54.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:00:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RPl++SfR46XjlO7utXXsgGhjAk8='} 2022-09-03T07:00:54.043 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000002c9fef13bd92c734-006312fba6-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:54.044 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:00:54.044 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:00:54.045 DEBUG:boto:Method: HEAD 2022-09-03T07:00:54.045 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:00:54.046 DEBUG:boto:Data: 2022-09-03T07:00:54.046 DEBUG:boto:Headers: {} 2022-09-03T07:00:54.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:54.047 DEBUG:boto:Port: 8000 2022-09-03T07:00:54.048 DEBUG:boto:Params: {} 2022-09-03T07:00:54.049 DEBUG:boto:Token: None 2022-09-03T07:00:54.049 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:54 GMT /avysmw-73/ 2022-09-03T07:00:54.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:edX0TxvOTONBPDWgF/5zS1ditwM= 2022-09-03T07:00:54.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:edX0TxvOTONBPDWgF/5zS1ditwM=', 'Content-Length': '0'} 2022-09-03T07:00:54.053 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000469369e1d5ce9d8-006312fba6-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:54.053 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:00:54.054 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:00:54.055 DEBUG:boto:Method: PUT 2022-09-03T07:00:54.056 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:00:54.056 DEBUG:boto:Data: 2022-09-03T07:00:54.057 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:00:54.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:54.058 DEBUG:boto:Port: 8000 2022-09-03T07:00:54.059 DEBUG:boto:Params: {} 2022-09-03T07:00:54.059 DEBUG:boto:Token: None 2022-09-03T07:00:54.060 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:00:54 GMT /avysmw-73/g 2022-09-03T07:00:54.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:52X2iz6EvrR7XFxvE57eXKINRfQ= 2022-09-03T07:00:54.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:00:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:52X2iz6EvrR7XFxvE57eXKINRfQ='} 2022-09-03T07:00:54.106 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000002739bdd562d58a19-006312fba6-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:54.107 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:00:54.108 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:00:54.108 DEBUG:boto:Method: HEAD 2022-09-03T07:00:54.109 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:00:54.109 DEBUG:boto:Data: 2022-09-03T07:00:54.110 DEBUG:boto:Headers: {} 2022-09-03T07:00:54.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:54.112 DEBUG:boto:Port: 8000 2022-09-03T07:00:54.112 DEBUG:boto:Params: {} 2022-09-03T07:00:54.113 DEBUG:boto:Token: None 2022-09-03T07:00:54.113 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:00:54 GMT /avysmw-73/ 2022-09-03T07:00:54.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:edX0TxvOTONBPDWgF/5zS1ditwM= 2022-09-03T07:00:54.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:00:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:edX0TxvOTONBPDWgF/5zS1ditwM=', 'Content-Length': '0'} 2022-09-03T07:00:54.117 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '21'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000043be23e76156e1da-006312fba6-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:54.118 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:00:54.119 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:00:54.120 DEBUG:boto:Method: PUT 2022-09-03T07:00:54.121 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:00:54.122 DEBUG:boto:Data: 2022-09-03T07:00:54.122 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:00:54.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:00:54.124 DEBUG:boto:Port: 8000 2022-09-03T07:00:54.124 DEBUG:boto:Params: {} 2022-09-03T07:00:54.125 DEBUG:boto:Token: None 2022-09-03T07:00:54.126 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:00:54 GMT /avysmw-73/h 2022-09-03T07:00:54.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0kKc6icJTJv8lpZ//mJJtG6CdAA= 2022-09-03T07:00:54.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:00:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0kKc6icJTJv8lpZ//mJJtG6CdAA='} 2022-09-03T07:00:54.132 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000c8143bafb0676cf2-006312fba6-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:00:54.133 INFO:tasks.rgw.c2.client.0:Restarting daemon 2022-09-03T07:00:54.133 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2022-09-03T07:00:54.136 INFO:tasks.rgw.c2.client.0:Started 2022-09-03T07:00:54.137 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T07:00:54.170 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:00:54.170 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:00:54.170 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T07:00:54.240 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:00:54.238+0000 7f8d0e9f7700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16) 2022-09-03T07:00:54.241 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:00:54.238+0000 7f8d0e9f7700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5) 2022-09-03T07:00:54.242 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:00:54.241+0000 7f8d0e9f7700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5) 2022-09-03T07:00:55.173 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:00:55.200 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:00:55.201 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:00:55.202 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 208k 0 --:--:-- --:--:-- --:--:-- 208k 2022-09-03T07:00:55.203 INFO:teuthology.orchestra.run.smithi087.stdout:anonymous 2022-09-03T07:00:55.204 INFO:tasks.rgw.c1.client.1:Restarting daemon 2022-09-03T07:00:55.205 DEBUG:teuthology.orchestra.run.smithi062:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c1.client.1.stdout 2>&1 2022-09-03T07:00:55.213 INFO:tasks.rgw.c1.client.1:Started 2022-09-03T07:00:55.215 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T07:00:55.244 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:00:55.244 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:00:55.244 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T07:00:56.245 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:56.274 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T07:00:56.275 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:00:56.276 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:00:56.276 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T07:00:56.738 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 has been restored 2022-09-03T07:00:58.277 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001 2022-09-03T07:00:58.305 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:00:58.306 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:00:58.307 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T07:00:58.309 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi062.front.sepia.ceph.com port 8001: Connection refused 2022-09-03T07:01:02.310 DEBUG:teuthology.orchestra.run.smithi062:> curl http://smithi062.front.sepia.ceph.com:8001 2022-09-03T07:01:02.342 INFO:teuthology.orchestra.run.smithi062.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:01:02.342 INFO:teuthology.orchestra.run.smithi062.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:01:02.343 INFO:teuthology.orchestra.run.smithi062.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 208k 0 --:--:-- --:--:-- --:--:-- 208k 2022-09-03T07:01:02.345 INFO:teuthology.orchestra.run.smithi062.stdout:anonymous 2022-09-03T07:01:02.345 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-73 2022-09-03T07:01:02.346 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:01:02.347 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:01:02.348 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-73 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:01:02.460 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:01:02.462 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:01:02.522 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:02.520+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:07.523 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:07.521+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:12.529 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:12.522+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:17.524 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:17.522+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:22.525 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:22.522+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:27.525 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:27.523+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:32.525 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:32.523+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:37.526 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:37.524+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:42.529 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:42.525+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:47.528 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:47.525+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:52.527 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:52.525+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:01:57.528 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:01:57.526+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:02.528 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:02.526+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:07.529 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:07.527+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:12.530 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:12.528+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:17.530 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:17.528+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:22.531 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:22.529+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:27.531 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:27.529+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:32.532 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:32.529+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:37.533 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:37.530+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:42.533 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:42.531+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:47.534 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:47.532+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:52.534 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:52.532+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:02:57.535 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:02:57.533+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:03:02.535 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:02.533+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:03:07.536 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:07.534+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:03:12.537 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:12.535+0000 7f3d01d13600 1 waiting to reach incremental sync.. 2022-09-03T07:03:17.546 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:17.542+0000 7f3d01d13600 1 bucket sync caught up with source: 2022-09-03T07:03:17.547 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.63.5, 00000000002.50.5, 00000000002.186.5] 2022-09-03T07:03:17.547 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.63.5, 00000000002.50.5, 00000000002.186.5] 2022-09-03T07:03:17.548 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:17.542+0000 7f3d01d13600 0 bucket checkpoint complete 2022-09-03T07:03:17.573 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-73 2022-09-03T07:03:17.575 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:17.575 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:17.577 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-73 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:03:17.653 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:17.654 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:17.702 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:17.701+0000 7f849c6de600 1 bucket sync caught up with source: 2022-09-03T07:03:17.703 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000003.63.5, 00000000002.50.5, 00000000002.186.5] 2022-09-03T07:03:17.704 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000003.63.5, 00000000002.50.5, 00000000002.186.5] 2022-09-03T07:03:17.704 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:17.701+0000 7f849c6de600 0 bucket checkpoint complete 2022-09-03T07:03:17.724 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-73 2022-09-03T07:03:17.725 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:17.726 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:17.727 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-73 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:03:17.827 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:17.828 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:18.898 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:18.894+0000 7f83006ca600 1 bucket sync caught up with source: 2022-09-03T07:03:18.898 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000003.63.5, 00000000002.50.5, 00000000002.186.5] 2022-09-03T07:03:18.899 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000003.63.5, 00000000002.50.5, 00000000002.186.5] 2022-09-03T07:03:18.899 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:18.894+0000 7f83006ca600 0 bucket checkpoint complete 2022-09-03T07:03:18.914 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-73 2022-09-03T07:03:18.915 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:18.916 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:18.917 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-73 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:03:18.981 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:18.981 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:19.023 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:19.021+0000 7f9c9833a600 1 waiting for incremental sync to catch up: 2022-09-03T07:03:19.023 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.160.5, , , , 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:19.024 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.160.5, , , 00000000001.256.5, 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:24.025 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:24.023+0000 7f9c9833a600 1 waiting for incremental sync to catch up: 2022-09-03T07:03:24.026 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.160.5, , , , 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:24.026 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.160.5, , , 00000000001.256.5, 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:29.026 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:29.024+0000 7f9c9833a600 1 waiting for incremental sync to catch up: 2022-09-03T07:03:29.027 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.160.5, , , , 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:29.027 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.160.5, , , 00000000001.256.5, 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:34.030 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:34.025+0000 7f9c9833a600 1 waiting for incremental sync to catch up: 2022-09-03T07:03:34.031 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.160.5, , , , 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:34.032 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.160.5, , , 00000000001.256.5, 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:39.030 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:39.027+0000 7f9c9833a600 1 bucket sync caught up with source: 2022-09-03T07:03:39.031 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.160.5, , , 00000000001.256.5, 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:39.031 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.160.5, , , 00000000001.256.5, 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:39.031 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:39.027+0000 7f9c9833a600 0 bucket checkpoint complete 2022-09-03T07:03:39.053 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-73 2022-09-03T07:03:39.054 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:39.055 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:39.056 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-73 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:03:39.122 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:39.123 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:39.176 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:39.172+0000 7fb4db981600 1 bucket sync caught up with source: 2022-09-03T07:03:39.177 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.160.5, , , 00000000001.256.5, 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:39.178 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.160.5, , , 00000000001.256.5, 00000000001.263.5, , , 00000000002.284.5, , 00000000002.380.5] 2022-09-03T07:03:39.178 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:39.172+0000 7fb4db981600 0 bucket checkpoint complete 2022-09-03T07:03:39.195 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-73 2022-09-03T07:03:39.209 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-73 2022-09-03T07:03:39.211 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:39.212 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:39.213 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-73 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:03:39.278 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:39.279 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:39.317 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:39.312+0000 7ff8baa01600 1 bucket sync caught up with source: 2022-09-03T07:03:39.317 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.102.5, , , 00000000001.284.5, 00000000001.162.5, , , 00000000002.291.5, , 00000000002.391.5] 2022-09-03T07:03:39.318 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.102.5, , , 00000000001.284.5, 00000000001.162.5, , , 00000000002.291.5, , 00000000002.391.5] 2022-09-03T07:03:39.318 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:39.312+0000 7ff8baa01600 0 bucket checkpoint complete 2022-09-03T07:03:39.331 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-73 2022-09-03T07:03:39.332 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:39.333 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-73', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:39.335 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-73 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:03:39.497 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:39.498 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:39.560 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:39.558+0000 7fe6aad4e600 1 bucket sync caught up with source: 2022-09-03T07:03:39.561 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.102.5, , , 00000000001.284.5, 00000000001.162.5, , , 00000000002.291.5, , 00000000002.391.5] 2022-09-03T07:03:39.561 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.102.5, , , 00000000001.284.5, 00000000001.162.5, , , 00000000002.291.5, , 00000000002.391.5] 2022-09-03T07:03:39.562 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:39.558+0000 7fe6aad4e600 0 bucket checkpoint complete 2022-09-03T07:03:39.584 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-73 2022-09-03T07:03:39.585 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-73 zones={test-zone2, test-zone1} 2022-09-03T07:03:39.586 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:39.587 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:39.588 DEBUG:boto:Method: HEAD 2022-09-03T07:03:39.588 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:03:39.589 DEBUG:boto:Data: 2022-09-03T07:03:39.589 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.590 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:39.590 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.591 DEBUG:boto:Params: {} 2022-09-03T07:03:39.593 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:03:39.594 DEBUG:boto:Token: None 2022-09-03T07:03:39.595 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/ 2022-09-03T07:03:39.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2pOiGafo0QkHqlwvgUgBIpvO4yU= 2022-09-03T07:03:39.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2pOiGafo0QkHqlwvgUgBIpvO4yU=', 'Content-Length': '0'} 2022-09-03T07:03:39.602 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d7ece9dda62ba225-006312fc4b-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.603 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:39.604 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:39.604 DEBUG:boto:Method: HEAD 2022-09-03T07:03:39.605 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:03:39.605 DEBUG:boto:Data: 2022-09-03T07:03:39.606 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.607 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:39.607 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.608 DEBUG:boto:Params: {} 2022-09-03T07:03:39.610 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:03:39.611 DEBUG:boto:Token: None 2022-09-03T07:03:39.612 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/ 2022-09-03T07:03:39.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2pOiGafo0QkHqlwvgUgBIpvO4yU= 2022-09-03T07:03:39.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2pOiGafo0QkHqlwvgUgBIpvO4yU=', 'Content-Length': '0'} 2022-09-03T07:03:39.624 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000711077279bbd6422-006312fc4b-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.625 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:03:39.626 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:39.627 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:39.627 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.628 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.629 DEBUG:boto:Method: GET 2022-09-03T07:03:39.630 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.631 DEBUG:boto:Data: 2022-09-03T07:03:39.632 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.633 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:39.635 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.636 DEBUG:boto:Params: {} 2022-09-03T07:03:39.637 DEBUG:boto:Token: None 2022-09-03T07:03:39.639 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/?versions 2022-09-03T07:03:39.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HD3azinRpqO8rdBzervQFh766rw= 2022-09-03T07:03:39.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HD3azinRpqO8rdBzervQFh766rw=', 'Content-Length': '0'} 2022-09-03T07:03:39.644 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e27c36bd07f8445-006312fc4b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.645 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:39.647 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:03:39.648 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:03:39.648 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:03:39.649 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:03:39.649 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:03:39.649 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:03:39.650 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:03:39.650 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:03:39.651 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:03:39.651 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:39.652 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:39.653 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.654 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.654 DEBUG:boto:Method: GET 2022-09-03T07:03:39.655 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.655 DEBUG:boto:Data: 2022-09-03T07:03:39.656 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.656 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:39.657 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.657 DEBUG:boto:Params: {} 2022-09-03T07:03:39.658 DEBUG:boto:Token: None 2022-09-03T07:03:39.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/?versions 2022-09-03T07:03:39.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HD3azinRpqO8rdBzervQFh766rw= 2022-09-03T07:03:39.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HD3azinRpqO8rdBzervQFh766rw=', 'Content-Length': '0'} 2022-09-03T07:03:39.662 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000083f46dd438ddcfd-006312fc4b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.663 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:39.664 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:03:39.665 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:03:39.665 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:03:39.666 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:03:39.667 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:03:39.667 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:03:39.668 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:03:39.668 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:03:39.669 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:39.669 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:39.670 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.671 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.672 DEBUG:boto:Method: GET 2022-09-03T07:03:39.672 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.673 DEBUG:boto:Data: 2022-09-03T07:03:39.674 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.675 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:39.676 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.677 DEBUG:boto:Params: {} 2022-09-03T07:03:39.678 DEBUG:boto:Token: None 2022-09-03T07:03:39.679 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/?versions 2022-09-03T07:03:39.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HD3azinRpqO8rdBzervQFh766rw= 2022-09-03T07:03:39.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HD3azinRpqO8rdBzervQFh766rw=', 'Content-Length': '0'} 2022-09-03T07:03:39.684 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c6ff621c350a0cb3-006312fc4b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.686 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:39.689 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:39.689 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:39.690 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.691 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.692 DEBUG:boto:Method: GET 2022-09-03T07:03:39.693 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:39.693 DEBUG:boto:Data: 2022-09-03T07:03:39.695 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:39.696 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.696 DEBUG:boto:Params: {} 2022-09-03T07:03:39.697 DEBUG:boto:Token: None 2022-09-03T07:03:39.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/?versions 2022-09-03T07:03:39.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HD3azinRpqO8rdBzervQFh766rw= 2022-09-03T07:03:39.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HD3azinRpqO8rdBzervQFh766rw=', 'Content-Length': '0'} 2022-09-03T07:03:39.707 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb989557ee8a94c3-006312fc4b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.709 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:39.712 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:03:39.713 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.723 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.724 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.725 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.726 DEBUG:boto:Method: GET 2022-09-03T07:03:39.727 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:39.728 DEBUG:boto:Data: 2022-09-03T07:03:39.729 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.730 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:39.731 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.733 DEBUG:boto:Params: {} 2022-09-03T07:03:39.734 DEBUG:boto:Token: None 2022-09-03T07:03:39.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:39.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VLkfQfpbyXuIpJ/gl7XTJ8h7msc= 2022-09-03T07:03:39.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VLkfQfpbyXuIpJ/gl7XTJ8h7msc=', 'Content-Length': '0'} 2022-09-03T07:03:39.740 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d164ac870b6e900-006312fc4b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.742 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.744 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.745 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.746 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.747 DEBUG:boto:Method: GET 2022-09-03T07:03:39.753 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:39.755 DEBUG:boto:Data: 2022-09-03T07:03:39.757 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:39.759 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.766 DEBUG:boto:Params: {} 2022-09-03T07:03:39.768 DEBUG:boto:Token: None 2022-09-03T07:03:39.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:39.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VLkfQfpbyXuIpJ/gl7XTJ8h7msc= 2022-09-03T07:03:39.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VLkfQfpbyXuIpJ/gl7XTJ8h7msc=', 'Content-Length': '0'} 2022-09-03T07:03:39.776 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ac2b2a19db24efe-006312fc4b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.777 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.777 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.778 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.778 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.779 DEBUG:boto:Method: HEAD 2022-09-03T07:03:39.779 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:39.780 DEBUG:boto:Data: 2022-09-03T07:03:39.780 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.781 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:39.781 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.782 DEBUG:boto:Params: {} 2022-09-03T07:03:39.782 DEBUG:boto:Token: None 2022-09-03T07:03:39.783 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:39.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OeeuX57grmCr54uga+9mi++tC40= 2022-09-03T07:03:39.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OeeuX57grmCr54uga+9mi++tC40=', 'Content-Length': '0'} 2022-09-03T07:03:39.787 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f81433cb8c5bebec-006312fc4b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.788 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.789 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.789 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.790 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.790 DEBUG:boto:Method: HEAD 2022-09-03T07:03:39.791 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:39.791 DEBUG:boto:Data: 2022-09-03T07:03:39.792 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.792 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:39.793 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.793 DEBUG:boto:Params: {} 2022-09-03T07:03:39.794 DEBUG:boto:Token: None 2022-09-03T07:03:39.795 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:39.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OeeuX57grmCr54uga+9mi++tC40= 2022-09-03T07:03:39.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OeeuX57grmCr54uga+9mi++tC40=', 'Content-Length': '0'} 2022-09-03T07:03:39.802 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f02032c3e367ae2-006312fc4b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.803 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:03:39.804 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.804 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.805 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.805 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.806 DEBUG:boto:Method: GET 2022-09-03T07:03:39.806 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:39.807 DEBUG:boto:Data: 2022-09-03T07:03:39.808 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.809 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:39.809 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.812 DEBUG:boto:Params: {} 2022-09-03T07:03:39.814 DEBUG:boto:Token: None 2022-09-03T07:03:39.815 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:39.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VLkfQfpbyXuIpJ/gl7XTJ8h7msc= 2022-09-03T07:03:39.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VLkfQfpbyXuIpJ/gl7XTJ8h7msc=', 'Content-Length': '0'} 2022-09-03T07:03:39.824 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f7d54081c3cc109-006312fc4b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.826 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.827 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.832 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.833 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:39.835 DEBUG:boto:Method: GET 2022-09-03T07:03:39.836 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:39.837 DEBUG:boto:Data: 2022-09-03T07:03:39.838 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.839 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:39.839 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.840 DEBUG:boto:Params: {} 2022-09-03T07:03:39.841 DEBUG:boto:Token: None 2022-09-03T07:03:39.842 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:39.843 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VLkfQfpbyXuIpJ/gl7XTJ8h7msc= 2022-09-03T07:03:39.844 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VLkfQfpbyXuIpJ/gl7XTJ8h7msc=', 'Content-Length': '0'} 2022-09-03T07:03:39.849 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a48cd473a16f7b8-006312fc4b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.852 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:39.854 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:39.855 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:39.856 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:39.857 DEBUG:boto:Method: GET 2022-09-03T07:03:39.859 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:39.860 DEBUG:boto:Data: 2022-09-03T07:03:39.860 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.861 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:39.863 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.865 DEBUG:boto:Params: {} 2022-09-03T07:03:39.866 DEBUG:boto:Token: None 2022-09-03T07:03:39.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/ 2022-09-03T07:03:39.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:khUWkqcSDiQ5pnSmhOi4sOs2OAQ= 2022-09-03T07:03:39.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:khUWkqcSDiQ5pnSmhOi4sOs2OAQ=', 'Content-Length': '0'} 2022-09-03T07:03:39.874 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e09410cb2f68742c-006312fc4b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.883 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:39.886 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:39.893 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:39.895 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:39.896 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:39.897 DEBUG:boto:Method: GET 2022-09-03T07:03:39.898 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:39.900 DEBUG:boto:Data: 2022-09-03T07:03:39.901 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.902 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:39.903 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.904 DEBUG:boto:Params: {} 2022-09-03T07:03:39.905 DEBUG:boto:Token: None 2022-09-03T07:03:39.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/ 2022-09-03T07:03:39.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:khUWkqcSDiQ5pnSmhOi4sOs2OAQ= 2022-09-03T07:03:39.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:khUWkqcSDiQ5pnSmhOi4sOs2OAQ=', 'Content-Length': '0'} 2022-09-03T07:03:39.919 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005fcba52366604f9e-006312fc4b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.922 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:39.924 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.925 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.926 DEBUG:boto:Method: HEAD 2022-09-03T07:03:39.927 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:39.927 DEBUG:boto:Data: 2022-09-03T07:03:39.928 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.930 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:39.931 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.932 DEBUG:boto:Params: {} 2022-09-03T07:03:39.933 DEBUG:boto:Token: None 2022-09-03T07:03:39.934 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a 2022-09-03T07:03:39.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jhcnmJNweczYb/OHZMU1cvoyOqU= 2022-09-03T07:03:39.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jhcnmJNweczYb/OHZMU1cvoyOqU=', 'Content-Length': '0'} 2022-09-03T07:03:39.940 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098aaeae9179f8a7d-006312fc4b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.941 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.943 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.943 DEBUG:boto:Method: HEAD 2022-09-03T07:03:39.944 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:39.945 DEBUG:boto:Data: 2022-09-03T07:03:39.946 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:39.948 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.949 DEBUG:boto:Params: {} 2022-09-03T07:03:39.950 DEBUG:boto:Token: None 2022-09-03T07:03:39.951 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a 2022-09-03T07:03:39.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jhcnmJNweczYb/OHZMU1cvoyOqU= 2022-09-03T07:03:39.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jhcnmJNweczYb/OHZMU1cvoyOqU=', 'Content-Length': '0'} 2022-09-03T07:03:39.956 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000649cf068e464a726-006312fc4b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.958 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:03:39.960 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.961 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.962 DEBUG:boto:Method: GET 2022-09-03T07:03:39.963 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:39.965 DEBUG:boto:Data: 2022-09-03T07:03:39.966 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.967 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:39.969 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.970 DEBUG:boto:Params: {} 2022-09-03T07:03:39.971 DEBUG:boto:Token: None 2022-09-03T07:03:39.972 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a 2022-09-03T07:03:39.973 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rnNjse+wzEgulChOkKafK+0C4VA= 2022-09-03T07:03:39.974 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rnNjse+wzEgulChOkKafK+0C4VA=', 'Content-Length': '0'} 2022-09-03T07:03:39.976 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd68a672cd6fc5b7-006312fc4b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:39.977 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:39.978 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:39.978 DEBUG:boto:Method: GET 2022-09-03T07:03:39.979 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:39.980 DEBUG:boto:Data: 2022-09-03T07:03:39.981 DEBUG:boto:Headers: {} 2022-09-03T07:03:39.981 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:39.982 DEBUG:boto:Port: 8000 2022-09-03T07:03:39.983 DEBUG:boto:Params: {} 2022-09-03T07:03:39.985 DEBUG:boto:Token: None 2022-09-03T07:03:39.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:39 GMT /avysmw-73/a 2022-09-03T07:03:39.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rnNjse+wzEgulChOkKafK+0C4VA= 2022-09-03T07:03:39.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rnNjse+wzEgulChOkKafK+0C4VA=', 'Content-Length': '0'} 2022-09-03T07:03:39.991 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9191cee3b13e807-006312fc4b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.038 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.039 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.041 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:40.042 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:40.043 DEBUG:boto:Method: GET 2022-09-03T07:03:40.045 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:40.046 DEBUG:boto:Data: 2022-09-03T07:03:40.047 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.048 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.049 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.050 DEBUG:boto:Params: {} 2022-09-03T07:03:40.052 DEBUG:boto:Token: None 2022-09-03T07:03:40.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.059 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b1eebaae988c05a4-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.061 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.063 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.064 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.066 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:40.067 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:40.068 DEBUG:boto:Method: GET 2022-09-03T07:03:40.069 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:40.070 DEBUG:boto:Data: 2022-09-03T07:03:40.071 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.072 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.073 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.074 DEBUG:boto:Params: {} 2022-09-03T07:03:40.076 DEBUG:boto:Token: None 2022-09-03T07:03:40.077 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.083 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f94f989f4c1f8e78-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.084 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.085 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:03:40.086 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.087 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.088 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.089 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.091 DEBUG:boto:Method: GET 2022-09-03T07:03:40.092 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:40.093 DEBUG:boto:Data: 2022-09-03T07:03:40.094 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.096 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.097 DEBUG:boto:Params: {} 2022-09-03T07:03:40.099 DEBUG:boto:Token: None 2022-09-03T07:03:40.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:40.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w0pSvzblZJwu3Uy9RbCgcSjXKDY= 2022-09-03T07:03:40.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w0pSvzblZJwu3Uy9RbCgcSjXKDY=', 'Content-Length': '0'} 2022-09-03T07:03:40.108 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f8ba229e2588ee95-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.111 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.112 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.113 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.114 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.115 DEBUG:boto:Method: GET 2022-09-03T07:03:40.116 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:40.117 DEBUG:boto:Data: 2022-09-03T07:03:40.119 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.120 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.121 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.122 DEBUG:boto:Params: {} 2022-09-03T07:03:40.123 DEBUG:boto:Token: None 2022-09-03T07:03:40.125 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:40.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w0pSvzblZJwu3Uy9RbCgcSjXKDY= 2022-09-03T07:03:40.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w0pSvzblZJwu3Uy9RbCgcSjXKDY=', 'Content-Length': '0'} 2022-09-03T07:03:40.130 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa303597fd6018b3-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.132 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.133 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.134 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.135 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.136 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.137 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:40.138 DEBUG:boto:Data: 2022-09-03T07:03:40.140 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.141 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.142 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.143 DEBUG:boto:Params: {} 2022-09-03T07:03:40.144 DEBUG:boto:Token: None 2022-09-03T07:03:40.145 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:40.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Srkl4gpxfoikmUHsyKBAVwV6hVM= 2022-09-03T07:03:40.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Srkl4gpxfoikmUHsyKBAVwV6hVM=', 'Content-Length': '0'} 2022-09-03T07:03:40.150 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd2d8e9f1f29f9e8-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.151 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.152 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.153 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.154 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.155 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.156 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:40.157 DEBUG:boto:Data: 2022-09-03T07:03:40.158 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.159 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.160 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.162 DEBUG:boto:Params: {} 2022-09-03T07:03:40.163 DEBUG:boto:Token: None 2022-09-03T07:03:40.164 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:40.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Srkl4gpxfoikmUHsyKBAVwV6hVM= 2022-09-03T07:03:40.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Srkl4gpxfoikmUHsyKBAVwV6hVM=', 'Content-Length': '0'} 2022-09-03T07:03:40.170 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d8b356486909ef9-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.171 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:03:40.172 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.173 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.174 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.175 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.176 DEBUG:boto:Method: GET 2022-09-03T07:03:40.178 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:40.179 DEBUG:boto:Data: 2022-09-03T07:03:40.180 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.181 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.182 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.183 DEBUG:boto:Params: {} 2022-09-03T07:03:40.184 DEBUG:boto:Token: None 2022-09-03T07:03:40.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:40.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w0pSvzblZJwu3Uy9RbCgcSjXKDY= 2022-09-03T07:03:40.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w0pSvzblZJwu3Uy9RbCgcSjXKDY=', 'Content-Length': '0'} 2022-09-03T07:03:40.191 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000653eafd7581b893d-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.234 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.235 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.236 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.236 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:40.237 DEBUG:boto:Method: GET 2022-09-03T07:03:40.238 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:40.239 DEBUG:boto:Data: 2022-09-03T07:03:40.239 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.240 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.241 DEBUG:boto:Params: {} 2022-09-03T07:03:40.241 DEBUG:boto:Token: None 2022-09-03T07:03:40.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:40.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w0pSvzblZJwu3Uy9RbCgcSjXKDY= 2022-09-03T07:03:40.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w0pSvzblZJwu3Uy9RbCgcSjXKDY=', 'Content-Length': '0'} 2022-09-03T07:03:40.246 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be1b78df02cddda6-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.250 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.251 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.252 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:40.253 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:40.266 DEBUG:boto:Method: GET 2022-09-03T07:03:40.267 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:40.268 DEBUG:boto:Data: 2022-09-03T07:03:40.269 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.270 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.271 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.272 DEBUG:boto:Params: {} 2022-09-03T07:03:40.273 DEBUG:boto:Token: None 2022-09-03T07:03:40.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.286 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e4b83477601bcc23-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.287 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.288 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.288 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.289 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:40.290 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:40.290 DEBUG:boto:Method: GET 2022-09-03T07:03:40.291 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:40.292 DEBUG:boto:Data: 2022-09-03T07:03:40.292 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.293 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.294 DEBUG:boto:Params: {} 2022-09-03T07:03:40.294 DEBUG:boto:Token: None 2022-09-03T07:03:40.295 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b4e86a6739c7a10-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.302 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.304 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.305 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.306 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.308 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:40.309 DEBUG:boto:Data: 2022-09-03T07:03:40.313 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.315 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.316 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.318 DEBUG:boto:Params: {} 2022-09-03T07:03:40.320 DEBUG:boto:Token: None 2022-09-03T07:03:40.321 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b 2022-09-03T07:03:40.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GR8mlQ9Ee2FYYanz8yi0R6H8mus= 2022-09-03T07:03:40.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GR8mlQ9Ee2FYYanz8yi0R6H8mus=', 'Content-Length': '0'} 2022-09-03T07:03:40.327 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000926000acf01b1ab1-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.328 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.329 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.331 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.331 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:40.332 DEBUG:boto:Data: 2022-09-03T07:03:40.338 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.340 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.341 DEBUG:boto:Params: {} 2022-09-03T07:03:40.355 DEBUG:boto:Token: None 2022-09-03T07:03:40.356 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b 2022-09-03T07:03:40.357 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GR8mlQ9Ee2FYYanz8yi0R6H8mus= 2022-09-03T07:03:40.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GR8mlQ9Ee2FYYanz8yi0R6H8mus=', 'Content-Length': '0'} 2022-09-03T07:03:40.360 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d64e3b877b16f7fa-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.361 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:03:40.361 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.362 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.363 DEBUG:boto:Method: GET 2022-09-03T07:03:40.363 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:40.364 DEBUG:boto:Data: 2022-09-03T07:03:40.366 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.367 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.368 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.369 DEBUG:boto:Params: {} 2022-09-03T07:03:40.370 DEBUG:boto:Token: None 2022-09-03T07:03:40.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b 2022-09-03T07:03:40.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nnRN1Ek0Ce1Y0xE95q5YuAb5/5s= 2022-09-03T07:03:40.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nnRN1Ek0Ce1Y0xE95q5YuAb5/5s=', 'Content-Length': '0'} 2022-09-03T07:03:40.379 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078c8d4b3aee75c2b-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.380 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:40.381 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:40.381 DEBUG:boto:Method: GET 2022-09-03T07:03:40.386 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:40.387 DEBUG:boto:Data: 2022-09-03T07:03:40.388 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.389 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.389 DEBUG:boto:Params: {} 2022-09-03T07:03:40.390 DEBUG:boto:Token: None 2022-09-03T07:03:40.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/b 2022-09-03T07:03:40.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nnRN1Ek0Ce1Y0xE95q5YuAb5/5s= 2022-09-03T07:03:40.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nnRN1Ek0Ce1Y0xE95q5YuAb5/5s=', 'Content-Length': '0'} 2022-09-03T07:03:40.395 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f9387ca5f22a48b-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.396 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.397 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.398 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:40.399 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:40.400 DEBUG:boto:Method: GET 2022-09-03T07:03:40.401 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:40.402 DEBUG:boto:Data: 2022-09-03T07:03:40.404 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.404 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.405 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.414 DEBUG:boto:Params: {} 2022-09-03T07:03:40.416 DEBUG:boto:Token: None 2022-09-03T07:03:40.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.420 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000052f43010ef2ff399-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.424 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.425 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.425 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.426 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:40.426 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:40.427 DEBUG:boto:Method: GET 2022-09-03T07:03:40.428 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:40.429 DEBUG:boto:Data: 2022-09-03T07:03:40.429 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.430 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.430 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.431 DEBUG:boto:Params: {} 2022-09-03T07:03:40.432 DEBUG:boto:Token: None 2022-09-03T07:03:40.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.437 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.442 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ebe0717708f4757b-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.486 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.487 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:03:40.488 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.489 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.489 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.493 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.494 DEBUG:boto:Method: GET 2022-09-03T07:03:40.495 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:40.495 DEBUG:boto:Data: 2022-09-03T07:03:40.496 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.496 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.497 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.497 DEBUG:boto:Params: {} 2022-09-03T07:03:40.498 DEBUG:boto:Token: None 2022-09-03T07:03:40.499 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:40.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hq06ikpyrNNgu22IHMZR4CAC4WY= 2022-09-03T07:03:40.501 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hq06ikpyrNNgu22IHMZR4CAC4WY=', 'Content-Length': '0'} 2022-09-03T07:03:40.503 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fc979cd4992c6c0-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.504 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.504 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.505 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.506 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.508 DEBUG:boto:Method: GET 2022-09-03T07:03:40.509 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:40.510 DEBUG:boto:Data: 2022-09-03T07:03:40.510 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.511 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.512 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.512 DEBUG:boto:Params: {} 2022-09-03T07:03:40.513 DEBUG:boto:Token: None 2022-09-03T07:03:40.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:40.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hq06ikpyrNNgu22IHMZR4CAC4WY= 2022-09-03T07:03:40.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hq06ikpyrNNgu22IHMZR4CAC4WY=', 'Content-Length': '0'} 2022-09-03T07:03:40.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000714ac55f829ea88e-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.519 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.520 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.520 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.521 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.522 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.522 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:40.523 DEBUG:boto:Data: 2022-09-03T07:03:40.524 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.524 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.525 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.526 DEBUG:boto:Params: {} 2022-09-03T07:03:40.527 DEBUG:boto:Token: None 2022-09-03T07:03:40.528 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:40.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uvBAlbiKkkTJWVUzzD95VaHI2/c= 2022-09-03T07:03:40.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uvBAlbiKkkTJWVUzzD95VaHI2/c=', 'Content-Length': '0'} 2022-09-03T07:03:40.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005797a8c9ed3788d5-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.534 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.535 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.535 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.536 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.536 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.537 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:40.537 DEBUG:boto:Data: 2022-09-03T07:03:40.538 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.539 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.540 DEBUG:boto:Params: {} 2022-09-03T07:03:40.541 DEBUG:boto:Token: None 2022-09-03T07:03:40.550 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:40.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uvBAlbiKkkTJWVUzzD95VaHI2/c= 2022-09-03T07:03:40.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uvBAlbiKkkTJWVUzzD95VaHI2/c=', 'Content-Length': '0'} 2022-09-03T07:03:40.554 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001735817b16b9b802-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.555 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:03:40.556 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.557 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.558 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.559 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.561 DEBUG:boto:Method: GET 2022-09-03T07:03:40.562 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:40.563 DEBUG:boto:Data: 2022-09-03T07:03:40.564 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.564 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.565 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.565 DEBUG:boto:Params: {} 2022-09-03T07:03:40.566 DEBUG:boto:Token: None 2022-09-03T07:03:40.567 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:40.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hq06ikpyrNNgu22IHMZR4CAC4WY= 2022-09-03T07:03:40.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hq06ikpyrNNgu22IHMZR4CAC4WY=', 'Content-Length': '0'} 2022-09-03T07:03:40.574 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9a72e46d704d335-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.575 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.576 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.576 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.577 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:40.577 DEBUG:boto:Method: GET 2022-09-03T07:03:40.578 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:40.578 DEBUG:boto:Data: 2022-09-03T07:03:40.579 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.579 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.580 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.580 DEBUG:boto:Params: {} 2022-09-03T07:03:40.581 DEBUG:boto:Token: None 2022-09-03T07:03:40.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:40.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hq06ikpyrNNgu22IHMZR4CAC4WY= 2022-09-03T07:03:40.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hq06ikpyrNNgu22IHMZR4CAC4WY=', 'Content-Length': '0'} 2022-09-03T07:03:40.586 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000791d4ab7ffb585a9-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.587 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.587 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.588 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:40.588 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:40.589 DEBUG:boto:Method: GET 2022-09-03T07:03:40.590 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:40.590 DEBUG:boto:Data: 2022-09-03T07:03:40.591 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.592 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.593 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.593 DEBUG:boto:Params: {} 2022-09-03T07:03:40.594 DEBUG:boto:Token: None 2022-09-03T07:03:40.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.599 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a6e5c3efd1c3a27-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.600 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.601 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.601 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.602 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:40.603 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:40.604 DEBUG:boto:Method: GET 2022-09-03T07:03:40.604 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:40.605 DEBUG:boto:Data: 2022-09-03T07:03:40.605 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.606 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.606 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.607 DEBUG:boto:Params: {} 2022-09-03T07:03:40.608 DEBUG:boto:Token: None 2022-09-03T07:03:40.608 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.612 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3c226ce8ffe4ce3-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.613 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.614 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.615 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.616 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.617 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:40.618 DEBUG:boto:Data: 2022-09-03T07:03:40.618 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.619 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.620 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.620 DEBUG:boto:Params: {} 2022-09-03T07:03:40.621 DEBUG:boto:Token: None 2022-09-03T07:03:40.621 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c 2022-09-03T07:03:40.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6V2/YWH0vcwwQEzE6T3czYFkeuo= 2022-09-03T07:03:40.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6V2/YWH0vcwwQEzE6T3czYFkeuo=', 'Content-Length': '0'} 2022-09-03T07:03:40.625 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002aaa44d5f67216eb-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.626 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.627 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.628 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.628 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:40.629 DEBUG:boto:Data: 2022-09-03T07:03:40.629 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.630 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.631 DEBUG:boto:Params: {} 2022-09-03T07:03:40.632 DEBUG:boto:Token: None 2022-09-03T07:03:40.632 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c 2022-09-03T07:03:40.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6V2/YWH0vcwwQEzE6T3czYFkeuo= 2022-09-03T07:03:40.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6V2/YWH0vcwwQEzE6T3czYFkeuo=', 'Content-Length': '0'} 2022-09-03T07:03:40.640 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002de0697898d7d8af-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.641 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:03:40.642 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.643 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.643 DEBUG:boto:Method: GET 2022-09-03T07:03:40.644 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:40.645 DEBUG:boto:Data: 2022-09-03T07:03:40.645 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.646 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.646 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.647 DEBUG:boto:Params: {} 2022-09-03T07:03:40.647 DEBUG:boto:Token: None 2022-09-03T07:03:40.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c 2022-09-03T07:03:40.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tzJNLTJW2CSS3KGEZXfFGtS5A/A= 2022-09-03T07:03:40.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tzJNLTJW2CSS3KGEZXfFGtS5A/A=', 'Content-Length': '0'} 2022-09-03T07:03:40.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f057efb4f918b1fc-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.694 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:40.695 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:40.695 DEBUG:boto:Method: GET 2022-09-03T07:03:40.696 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:40.697 DEBUG:boto:Data: 2022-09-03T07:03:40.697 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.698 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.699 DEBUG:boto:Params: {} 2022-09-03T07:03:40.701 DEBUG:boto:Token: None 2022-09-03T07:03:40.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/c 2022-09-03T07:03:40.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tzJNLTJW2CSS3KGEZXfFGtS5A/A= 2022-09-03T07:03:40.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tzJNLTJW2CSS3KGEZXfFGtS5A/A=', 'Content-Length': '0'} 2022-09-03T07:03:40.708 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007009065100d90193-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.709 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.712 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.713 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:40.714 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:40.715 DEBUG:boto:Method: GET 2022-09-03T07:03:40.715 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:40.716 DEBUG:boto:Data: 2022-09-03T07:03:40.716 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.717 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.718 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.720 DEBUG:boto:Params: {} 2022-09-03T07:03:40.722 DEBUG:boto:Token: None 2022-09-03T07:03:40.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.732 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bf7cf2984a44fb30-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.733 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.736 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.737 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.741 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:40.743 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:40.743 DEBUG:boto:Method: GET 2022-09-03T07:03:40.744 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:40.746 DEBUG:boto:Data: 2022-09-03T07:03:40.747 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.748 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.748 DEBUG:boto:Params: {} 2022-09-03T07:03:40.749 DEBUG:boto:Token: None 2022-09-03T07:03:40.750 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.754 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000980baef8edec667a-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.755 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.756 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:03:40.757 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.758 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.759 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.760 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.761 DEBUG:boto:Method: GET 2022-09-03T07:03:40.761 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:40.762 DEBUG:boto:Data: 2022-09-03T07:03:40.763 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.763 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.764 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.764 DEBUG:boto:Params: {} 2022-09-03T07:03:40.765 DEBUG:boto:Token: None 2022-09-03T07:03:40.766 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:40.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uKOD2sr/BYMUBvS0BkDnRPTK3uc= 2022-09-03T07:03:40.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uKOD2sr/BYMUBvS0BkDnRPTK3uc=', 'Content-Length': '0'} 2022-09-03T07:03:40.771 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ef573caed34aaac-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.772 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.772 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.773 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.774 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.775 DEBUG:boto:Method: GET 2022-09-03T07:03:40.775 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:40.776 DEBUG:boto:Data: 2022-09-03T07:03:40.776 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.777 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.778 DEBUG:boto:Params: {} 2022-09-03T07:03:40.779 DEBUG:boto:Token: None 2022-09-03T07:03:40.779 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:40.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uKOD2sr/BYMUBvS0BkDnRPTK3uc= 2022-09-03T07:03:40.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uKOD2sr/BYMUBvS0BkDnRPTK3uc=', 'Content-Length': '0'} 2022-09-03T07:03:40.783 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000019d89e2e518fe11a-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.784 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.785 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.785 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.786 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.788 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.789 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:40.791 DEBUG:boto:Data: 2022-09-03T07:03:40.792 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.793 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.794 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.794 DEBUG:boto:Params: {} 2022-09-03T07:03:40.795 DEBUG:boto:Token: None 2022-09-03T07:03:40.796 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:40.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6xNx6HQVIaI4TKtf6aqJEcFkvd8= 2022-09-03T07:03:40.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6xNx6HQVIaI4TKtf6aqJEcFkvd8=', 'Content-Length': '0'} 2022-09-03T07:03:40.799 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037441bd1d3ed3cb0-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.800 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.801 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.802 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.803 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.804 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.805 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:40.806 DEBUG:boto:Data: 2022-09-03T07:03:40.807 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.807 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.808 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.809 DEBUG:boto:Params: {} 2022-09-03T07:03:40.810 DEBUG:boto:Token: None 2022-09-03T07:03:40.811 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:40.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6xNx6HQVIaI4TKtf6aqJEcFkvd8= 2022-09-03T07:03:40.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6xNx6HQVIaI4TKtf6aqJEcFkvd8=', 'Content-Length': '0'} 2022-09-03T07:03:40.815 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000067a177d81501defa-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.816 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:03:40.817 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.818 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.819 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.821 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.822 DEBUG:boto:Method: GET 2022-09-03T07:03:40.823 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:40.823 DEBUG:boto:Data: 2022-09-03T07:03:40.824 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.824 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.825 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.826 DEBUG:boto:Params: {} 2022-09-03T07:03:40.826 DEBUG:boto:Token: None 2022-09-03T07:03:40.827 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:40.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uKOD2sr/BYMUBvS0BkDnRPTK3uc= 2022-09-03T07:03:40.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uKOD2sr/BYMUBvS0BkDnRPTK3uc=', 'Content-Length': '0'} 2022-09-03T07:03:40.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005cc9ce1216ca6e9f-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.831 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.832 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.832 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.833 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:40.833 DEBUG:boto:Method: GET 2022-09-03T07:03:40.834 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:40.834 DEBUG:boto:Data: 2022-09-03T07:03:40.835 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.836 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.836 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.837 DEBUG:boto:Params: {} 2022-09-03T07:03:40.837 DEBUG:boto:Token: None 2022-09-03T07:03:40.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:40.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uKOD2sr/BYMUBvS0BkDnRPTK3uc= 2022-09-03T07:03:40.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uKOD2sr/BYMUBvS0BkDnRPTK3uc=', 'Content-Length': '0'} 2022-09-03T07:03:40.842 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000709025ca4d04d1a-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.888 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.889 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.890 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:40.890 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:40.891 DEBUG:boto:Method: GET 2022-09-03T07:03:40.892 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:40.892 DEBUG:boto:Data: 2022-09-03T07:03:40.893 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.894 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.895 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.895 DEBUG:boto:Params: {} 2022-09-03T07:03:40.896 DEBUG:boto:Token: None 2022-09-03T07:03:40.898 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.899 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.900 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.902 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc246d200662c004-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.904 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.905 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:40.905 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:40.906 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:40.906 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:40.907 DEBUG:boto:Method: GET 2022-09-03T07:03:40.908 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:40.908 DEBUG:boto:Data: 2022-09-03T07:03:40.909 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.910 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.910 DEBUG:boto:Params: {} 2022-09-03T07:03:40.911 DEBUG:boto:Token: None 2022-09-03T07:03:40.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/ 2022-09-03T07:03:40.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI= 2022-09-03T07:03:40.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CgEae7gAdXbgk2X2zJQKvWqp0pI=', 'Content-Length': '0'} 2022-09-03T07:03:40.916 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ffd1ae7a3ee8f773-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.916 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:40.917 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.918 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.918 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.919 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:40.919 DEBUG:boto:Data: 2022-09-03T07:03:40.920 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.920 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.921 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.921 DEBUG:boto:Params: {} 2022-09-03T07:03:40.922 DEBUG:boto:Token: None 2022-09-03T07:03:40.923 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/d 2022-09-03T07:03:40.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rHw7pagZhKb+lHdaS/XSHEiRHbw= 2022-09-03T07:03:40.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rHw7pagZhKb+lHdaS/XSHEiRHbw=', 'Content-Length': '0'} 2022-09-03T07:03:40.926 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000595a983db0191e2c-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.927 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.928 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.929 DEBUG:boto:Method: HEAD 2022-09-03T07:03:40.929 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:40.930 DEBUG:boto:Data: 2022-09-03T07:03:40.930 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.931 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.932 DEBUG:boto:Params: {} 2022-09-03T07:03:40.933 DEBUG:boto:Token: None 2022-09-03T07:03:40.933 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/d 2022-09-03T07:03:40.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rHw7pagZhKb+lHdaS/XSHEiRHbw= 2022-09-03T07:03:40.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rHw7pagZhKb+lHdaS/XSHEiRHbw=', 'Content-Length': '0'} 2022-09-03T07:03:40.938 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000041d688d2d0088e79-006312fc4c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.939 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:03:40.939 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.940 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.941 DEBUG:boto:Method: GET 2022-09-03T07:03:40.941 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:40.942 DEBUG:boto:Data: 2022-09-03T07:03:40.943 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.943 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:40.944 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.945 DEBUG:boto:Params: {} 2022-09-03T07:03:40.945 DEBUG:boto:Token: None 2022-09-03T07:03:40.946 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:40 GMT /avysmw-73/d 2022-09-03T07:03:40.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+lROTCAg7axtNZD/ng5u7ridfEE= 2022-09-03T07:03:40.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+lROTCAg7axtNZD/ng5u7ridfEE=', 'Content-Length': '0'} 2022-09-03T07:03:40.949 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015c71a20adecc887-006312fc4c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:40.990 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:40.991 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:40.993 DEBUG:boto:Method: GET 2022-09-03T07:03:40.993 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:40.994 DEBUG:boto:Data: 2022-09-03T07:03:40.995 DEBUG:boto:Headers: {} 2022-09-03T07:03:40.996 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:40.997 DEBUG:boto:Port: 8000 2022-09-03T07:03:40.998 DEBUG:boto:Params: {} 2022-09-03T07:03:40.999 DEBUG:boto:Token: None 2022-09-03T07:03:41.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/d 2022-09-03T07:03:41.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D4OdfUq+3M2ORnERPiPt89U5mGw= 2022-09-03T07:03:41.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D4OdfUq+3M2ORnERPiPt89U5mGw=', 'Content-Length': '0'} 2022-09-03T07:03:41.006 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d0303ea2de0e32a-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.007 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.007 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.008 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:41.008 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:41.009 DEBUG:boto:Method: GET 2022-09-03T07:03:41.010 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:41.010 DEBUG:boto:Data: 2022-09-03T07:03:41.011 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.011 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.012 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.012 DEBUG:boto:Params: {} 2022-09-03T07:03:41.013 DEBUG:boto:Token: None 2022-09-03T07:03:41.014 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.018 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b0202c71602aa96-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.019 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.020 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.021 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.022 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:41.022 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:41.023 DEBUG:boto:Method: GET 2022-09-03T07:03:41.023 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:41.024 DEBUG:boto:Data: 2022-09-03T07:03:41.025 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.025 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.026 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.026 DEBUG:boto:Params: {} 2022-09-03T07:03:41.027 DEBUG:boto:Token: None 2022-09-03T07:03:41.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.032 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ec800e59645fb718-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.034 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.035 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:03:41.036 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.040 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.041 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.042 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.044 DEBUG:boto:Method: GET 2022-09-03T07:03:41.045 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:41.046 DEBUG:boto:Data: 2022-09-03T07:03:41.050 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.051 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.052 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.053 DEBUG:boto:Params: {} 2022-09-03T07:03:41.054 DEBUG:boto:Token: None 2022-09-03T07:03:41.055 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:41.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IgrjMAsKkzSyyEtj8b2dTGLKza0= 2022-09-03T07:03:41.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IgrjMAsKkzSyyEtj8b2dTGLKza0=', 'Content-Length': '0'} 2022-09-03T07:03:41.059 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093670a68561ede72-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.060 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.061 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.062 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.063 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.064 DEBUG:boto:Method: GET 2022-09-03T07:03:41.065 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:41.066 DEBUG:boto:Data: 2022-09-03T07:03:41.068 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.068 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.069 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.070 DEBUG:boto:Params: {} 2022-09-03T07:03:41.072 DEBUG:boto:Token: None 2022-09-03T07:03:41.073 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:41.074 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IgrjMAsKkzSyyEtj8b2dTGLKza0= 2022-09-03T07:03:41.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IgrjMAsKkzSyyEtj8b2dTGLKza0=', 'Content-Length': '0'} 2022-09-03T07:03:41.078 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bb415e64943b69b-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.079 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.080 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.080 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.081 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.081 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.082 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:41.083 DEBUG:boto:Data: 2022-09-03T07:03:41.083 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.084 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.084 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.085 DEBUG:boto:Params: {} 2022-09-03T07:03:41.085 DEBUG:boto:Token: None 2022-09-03T07:03:41.087 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:41.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mDMSoJk1KnLOCEB6F+D8/9VcrdI= 2022-09-03T07:03:41.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mDMSoJk1KnLOCEB6F+D8/9VcrdI=', 'Content-Length': '0'} 2022-09-03T07:03:41.091 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089d8466c048ff827-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.093 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.094 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.095 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.096 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.096 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.097 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:41.099 DEBUG:boto:Data: 2022-09-03T07:03:41.100 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.101 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.102 DEBUG:boto:Params: {} 2022-09-03T07:03:41.103 DEBUG:boto:Token: None 2022-09-03T07:03:41.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:41.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mDMSoJk1KnLOCEB6F+D8/9VcrdI= 2022-09-03T07:03:41.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mDMSoJk1KnLOCEB6F+D8/9VcrdI=', 'Content-Length': '0'} 2022-09-03T07:03:41.107 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3873e695a0b7075-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.108 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:03:41.109 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.109 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.110 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.111 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.112 DEBUG:boto:Method: GET 2022-09-03T07:03:41.112 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:41.113 DEBUG:boto:Data: 2022-09-03T07:03:41.114 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.114 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.115 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.116 DEBUG:boto:Params: {} 2022-09-03T07:03:41.117 DEBUG:boto:Token: None 2022-09-03T07:03:41.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:41.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IgrjMAsKkzSyyEtj8b2dTGLKza0= 2022-09-03T07:03:41.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IgrjMAsKkzSyyEtj8b2dTGLKza0=', 'Content-Length': '0'} 2022-09-03T07:03:41.122 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007bb5e33dd5de310f-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.168 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.169 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.170 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.171 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:41.171 DEBUG:boto:Method: GET 2022-09-03T07:03:41.172 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:41.172 DEBUG:boto:Data: 2022-09-03T07:03:41.174 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.175 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.176 DEBUG:boto:Params: {} 2022-09-03T07:03:41.178 DEBUG:boto:Token: None 2022-09-03T07:03:41.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:41.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IgrjMAsKkzSyyEtj8b2dTGLKza0= 2022-09-03T07:03:41.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IgrjMAsKkzSyyEtj8b2dTGLKza0=', 'Content-Length': '0'} 2022-09-03T07:03:41.188 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ba314f850e04bdd-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.189 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.189 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.190 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:41.190 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:41.191 DEBUG:boto:Method: GET 2022-09-03T07:03:41.192 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:41.192 DEBUG:boto:Data: 2022-09-03T07:03:41.193 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.194 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.194 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.195 DEBUG:boto:Params: {} 2022-09-03T07:03:41.195 DEBUG:boto:Token: None 2022-09-03T07:03:41.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.200 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045c4d0d861c43498-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.201 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.202 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.203 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.203 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:41.204 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:41.205 DEBUG:boto:Method: GET 2022-09-03T07:03:41.206 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:41.206 DEBUG:boto:Data: 2022-09-03T07:03:41.207 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.208 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.209 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.211 DEBUG:boto:Params: {} 2022-09-03T07:03:41.212 DEBUG:boto:Token: None 2022-09-03T07:03:41.213 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.214 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.218 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bc97a6011dcb8f92-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.219 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.220 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.221 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.222 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.222 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:41.223 DEBUG:boto:Data: 2022-09-03T07:03:41.224 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.224 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.225 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.225 DEBUG:boto:Params: {} 2022-09-03T07:03:41.226 DEBUG:boto:Token: None 2022-09-03T07:03:41.227 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e 2022-09-03T07:03:41.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wwqYYF5Z+1ILJTUjhlNztGDK1Gk= 2022-09-03T07:03:41.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wwqYYF5Z+1ILJTUjhlNztGDK1Gk=', 'Content-Length': '0'} 2022-09-03T07:03:41.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef5d6bac7f1e2d06-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.234 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.234 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.235 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.236 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:41.236 DEBUG:boto:Data: 2022-09-03T07:03:41.237 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.238 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.238 DEBUG:boto:Params: {} 2022-09-03T07:03:41.239 DEBUG:boto:Token: None 2022-09-03T07:03:41.240 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e 2022-09-03T07:03:41.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wwqYYF5Z+1ILJTUjhlNztGDK1Gk= 2022-09-03T07:03:41.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wwqYYF5Z+1ILJTUjhlNztGDK1Gk=', 'Content-Length': '0'} 2022-09-03T07:03:41.249 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002abe103a1d3f0de7-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.251 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:03:41.251 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.252 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.253 DEBUG:boto:Method: GET 2022-09-03T07:03:41.254 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:41.255 DEBUG:boto:Data: 2022-09-03T07:03:41.256 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.257 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.258 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.259 DEBUG:boto:Params: {} 2022-09-03T07:03:41.259 DEBUG:boto:Token: None 2022-09-03T07:03:41.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e 2022-09-03T07:03:41.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Us1yqnjnJTIy6e+0R4vv2p6kLwE= 2022-09-03T07:03:41.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Us1yqnjnJTIy6e+0R4vv2p6kLwE=', 'Content-Length': '0'} 2022-09-03T07:03:41.265 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3591c5bd147b9b0-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.268 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:41.269 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:41.270 DEBUG:boto:Method: GET 2022-09-03T07:03:41.271 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:41.272 DEBUG:boto:Data: 2022-09-03T07:03:41.272 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.274 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.275 DEBUG:boto:Params: {} 2022-09-03T07:03:41.276 DEBUG:boto:Token: None 2022-09-03T07:03:41.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/e 2022-09-03T07:03:41.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Us1yqnjnJTIy6e+0R4vv2p6kLwE= 2022-09-03T07:03:41.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Us1yqnjnJTIy6e+0R4vv2p6kLwE=', 'Content-Length': '0'} 2022-09-03T07:03:41.282 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000674c5001e903a45-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.323 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.324 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.325 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:41.326 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:41.327 DEBUG:boto:Method: GET 2022-09-03T07:03:41.328 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:41.329 DEBUG:boto:Data: 2022-09-03T07:03:41.330 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.331 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.332 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.333 DEBUG:boto:Params: {} 2022-09-03T07:03:41.334 DEBUG:boto:Token: None 2022-09-03T07:03:41.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.339 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000050eb6dd25d41cff2-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.341 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.343 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.344 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.345 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:41.346 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:41.347 DEBUG:boto:Method: GET 2022-09-03T07:03:41.347 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:41.348 DEBUG:boto:Data: 2022-09-03T07:03:41.349 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.349 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.350 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.351 DEBUG:boto:Params: {} 2022-09-03T07:03:41.352 DEBUG:boto:Token: None 2022-09-03T07:03:41.353 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.358 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000899da9d6f57e75d9-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.359 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.360 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:03:41.361 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.362 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.363 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.365 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.366 DEBUG:boto:Method: GET 2022-09-03T07:03:41.367 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:41.368 DEBUG:boto:Data: 2022-09-03T07:03:41.369 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.370 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.371 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.372 DEBUG:boto:Params: {} 2022-09-03T07:03:41.373 DEBUG:boto:Token: None 2022-09-03T07:03:41.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:41.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9AuZ/JHD+0W77dwMGsnA9rRAHXM= 2022-09-03T07:03:41.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9AuZ/JHD+0W77dwMGsnA9rRAHXM=', 'Content-Length': '0'} 2022-09-03T07:03:41.378 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd62f0f85feba0c8-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.422 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.424 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.425 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.426 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.427 DEBUG:boto:Method: GET 2022-09-03T07:03:41.431 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:41.432 DEBUG:boto:Data: 2022-09-03T07:03:41.439 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.440 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.441 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.443 DEBUG:boto:Params: {} 2022-09-03T07:03:41.445 DEBUG:boto:Token: None 2022-09-03T07:03:41.449 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:41.450 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9AuZ/JHD+0W77dwMGsnA9rRAHXM= 2022-09-03T07:03:41.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9AuZ/JHD+0W77dwMGsnA9rRAHXM=', 'Content-Length': '0'} 2022-09-03T07:03:41.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000346c63131e08158f-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.454 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.454 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.455 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.455 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.456 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.457 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:41.457 DEBUG:boto:Data: 2022-09-03T07:03:41.458 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.458 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.459 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.459 DEBUG:boto:Params: {} 2022-09-03T07:03:41.460 DEBUG:boto:Token: None 2022-09-03T07:03:41.461 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:41.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TVQ62NV16nal0WQB28imXtUejZs= 2022-09-03T07:03:41.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TVQ62NV16nal0WQB28imXtUejZs=', 'Content-Length': '0'} 2022-09-03T07:03:41.465 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005639a76c6d5849e7-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.466 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.467 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.468 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.469 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.470 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.471 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:41.472 DEBUG:boto:Data: 2022-09-03T07:03:41.473 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.474 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.475 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.476 DEBUG:boto:Params: {} 2022-09-03T07:03:41.477 DEBUG:boto:Token: None 2022-09-03T07:03:41.478 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:41.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TVQ62NV16nal0WQB28imXtUejZs= 2022-09-03T07:03:41.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TVQ62NV16nal0WQB28imXtUejZs=', 'Content-Length': '0'} 2022-09-03T07:03:41.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a9795c5c1780759-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.489 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:03:41.491 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.492 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.492 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.493 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.494 DEBUG:boto:Method: GET 2022-09-03T07:03:41.495 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:41.496 DEBUG:boto:Data: 2022-09-03T07:03:41.496 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.500 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.501 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.502 DEBUG:boto:Params: {} 2022-09-03T07:03:41.503 DEBUG:boto:Token: None 2022-09-03T07:03:41.504 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:41.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9AuZ/JHD+0W77dwMGsnA9rRAHXM= 2022-09-03T07:03:41.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9AuZ/JHD+0W77dwMGsnA9rRAHXM=', 'Content-Length': '0'} 2022-09-03T07:03:41.507 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000688a32776fc77084-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.508 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.508 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.509 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.510 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:41.510 DEBUG:boto:Method: GET 2022-09-03T07:03:41.511 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:41.511 DEBUG:boto:Data: 2022-09-03T07:03:41.512 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.513 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.513 DEBUG:boto:Params: {} 2022-09-03T07:03:41.514 DEBUG:boto:Token: None 2022-09-03T07:03:41.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:41.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9AuZ/JHD+0W77dwMGsnA9rRAHXM= 2022-09-03T07:03:41.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9AuZ/JHD+0W77dwMGsnA9rRAHXM=', 'Content-Length': '0'} 2022-09-03T07:03:41.519 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f6c4d2420c86c58-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.521 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.522 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.522 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:41.523 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:41.524 DEBUG:boto:Method: GET 2022-09-03T07:03:41.524 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:41.525 DEBUG:boto:Data: 2022-09-03T07:03:41.526 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.526 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.527 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.527 DEBUG:boto:Params: {} 2022-09-03T07:03:41.528 DEBUG:boto:Token: None 2022-09-03T07:03:41.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.533 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000035e471547a07ba80-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.534 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.535 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.536 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.537 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:41.537 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:41.538 DEBUG:boto:Method: GET 2022-09-03T07:03:41.538 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:41.539 DEBUG:boto:Data: 2022-09-03T07:03:41.540 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.541 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.542 DEBUG:boto:Params: {} 2022-09-03T07:03:41.542 DEBUG:boto:Token: None 2022-09-03T07:03:41.543 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.547 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3bb063c0843367a-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.590 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.592 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.593 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.594 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.594 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:41.595 DEBUG:boto:Data: 2022-09-03T07:03:41.596 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.596 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.597 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.597 DEBUG:boto:Params: {} 2022-09-03T07:03:41.598 DEBUG:boto:Token: None 2022-09-03T07:03:41.599 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f 2022-09-03T07:03:41.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jbIJOh8zOSr8QQl+P//7rQK5b0g= 2022-09-03T07:03:41.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jbIJOh8zOSr8QQl+P//7rQK5b0g=', 'Content-Length': '0'} 2022-09-03T07:03:41.603 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b49325ff7f05f72-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.604 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.605 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.605 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.606 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:41.607 DEBUG:boto:Data: 2022-09-03T07:03:41.608 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.609 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.609 DEBUG:boto:Params: {} 2022-09-03T07:03:41.610 DEBUG:boto:Token: None 2022-09-03T07:03:41.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f 2022-09-03T07:03:41.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jbIJOh8zOSr8QQl+P//7rQK5b0g= 2022-09-03T07:03:41.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jbIJOh8zOSr8QQl+P//7rQK5b0g=', 'Content-Length': '0'} 2022-09-03T07:03:41.614 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ec8219f6f0f09e3-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.615 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:03:41.616 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.617 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.619 DEBUG:boto:Method: GET 2022-09-03T07:03:41.619 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:41.620 DEBUG:boto:Data: 2022-09-03T07:03:41.621 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.622 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.623 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.624 DEBUG:boto:Params: {} 2022-09-03T07:03:41.625 DEBUG:boto:Token: None 2022-09-03T07:03:41.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f 2022-09-03T07:03:41.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzI8r/dE+DM4mYaB5R1ff1/IESo= 2022-09-03T07:03:41.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzI8r/dE+DM4mYaB5R1ff1/IESo=', 'Content-Length': '0'} 2022-09-03T07:03:41.635 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dafe59597542117d-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.636 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:41.637 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:41.638 DEBUG:boto:Method: GET 2022-09-03T07:03:41.638 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:41.639 DEBUG:boto:Data: 2022-09-03T07:03:41.640 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.642 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.643 DEBUG:boto:Params: {} 2022-09-03T07:03:41.644 DEBUG:boto:Token: None 2022-09-03T07:03:41.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/f 2022-09-03T07:03:41.647 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wzI8r/dE+DM4mYaB5R1ff1/IESo= 2022-09-03T07:03:41.648 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wzI8r/dE+DM4mYaB5R1ff1/IESo=', 'Content-Length': '0'} 2022-09-03T07:03:41.651 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a0c91e6364bdef8-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.653 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.653 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.654 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:41.655 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:41.656 DEBUG:boto:Method: GET 2022-09-03T07:03:41.656 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:41.657 DEBUG:boto:Data: 2022-09-03T07:03:41.657 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.658 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.659 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.660 DEBUG:boto:Params: {} 2022-09-03T07:03:41.661 DEBUG:boto:Token: None 2022-09-03T07:03:41.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.667 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000580c3caeb698787f-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.667 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.668 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.669 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.669 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:41.670 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:41.671 DEBUG:boto:Method: GET 2022-09-03T07:03:41.672 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:41.672 DEBUG:boto:Data: 2022-09-03T07:03:41.673 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.673 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.675 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.675 DEBUG:boto:Params: {} 2022-09-03T07:03:41.676 DEBUG:boto:Token: None 2022-09-03T07:03:41.677 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.682 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000065047aa0fe0d8566-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.683 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.685 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:03:41.686 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.687 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.688 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.690 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.691 DEBUG:boto:Method: GET 2022-09-03T07:03:41.692 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:41.693 DEBUG:boto:Data: 2022-09-03T07:03:41.693 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.694 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.695 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.695 DEBUG:boto:Params: {} 2022-09-03T07:03:41.696 DEBUG:boto:Token: None 2022-09-03T07:03:41.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:41.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pBl/wR1yXfVyb2NEJFZCNfHWxKI= 2022-09-03T07:03:41.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pBl/wR1yXfVyb2NEJFZCNfHWxKI=', 'Content-Length': '0'} 2022-09-03T07:03:41.700 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000365e6f028a1b1949-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.701 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.701 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.702 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.703 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.704 DEBUG:boto:Method: GET 2022-09-03T07:03:41.704 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:41.705 DEBUG:boto:Data: 2022-09-03T07:03:41.705 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.708 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.709 DEBUG:boto:Params: {} 2022-09-03T07:03:41.710 DEBUG:boto:Token: None 2022-09-03T07:03:41.712 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:41.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pBl/wR1yXfVyb2NEJFZCNfHWxKI= 2022-09-03T07:03:41.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pBl/wR1yXfVyb2NEJFZCNfHWxKI=', 'Content-Length': '0'} 2022-09-03T07:03:41.716 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000429bade0eafb6f4f-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.758 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.759 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.760 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.760 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.761 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.762 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:41.763 DEBUG:boto:Data: 2022-09-03T07:03:41.764 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.764 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.765 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.766 DEBUG:boto:Params: {} 2022-09-03T07:03:41.766 DEBUG:boto:Token: None 2022-09-03T07:03:41.767 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:41.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nPylVZgVcknrqGVZ1HQHrucKUkI= 2022-09-03T07:03:41.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nPylVZgVcknrqGVZ1HQHrucKUkI=', 'Content-Length': '0'} 2022-09-03T07:03:41.772 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009369ecc0d0c7958a-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.774 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.776 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.777 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.778 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.779 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.780 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:41.781 DEBUG:boto:Data: 2022-09-03T07:03:41.782 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.782 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.783 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.783 DEBUG:boto:Params: {} 2022-09-03T07:03:41.784 DEBUG:boto:Token: None 2022-09-03T07:03:41.785 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:41.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nPylVZgVcknrqGVZ1HQHrucKUkI= 2022-09-03T07:03:41.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nPylVZgVcknrqGVZ1HQHrucKUkI=', 'Content-Length': '0'} 2022-09-03T07:03:41.790 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000723e7fb4816c051a-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.791 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:03:41.792 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.793 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.793 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.794 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.795 DEBUG:boto:Method: GET 2022-09-03T07:03:41.796 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:41.797 DEBUG:boto:Data: 2022-09-03T07:03:41.798 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.799 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.800 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.801 DEBUG:boto:Params: {} 2022-09-03T07:03:41.802 DEBUG:boto:Token: None 2022-09-03T07:03:41.803 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:41.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pBl/wR1yXfVyb2NEJFZCNfHWxKI= 2022-09-03T07:03:41.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pBl/wR1yXfVyb2NEJFZCNfHWxKI=', 'Content-Length': '0'} 2022-09-03T07:03:41.808 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e05ec1bc588aec9f-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.810 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.811 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.812 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.813 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:41.814 DEBUG:boto:Method: GET 2022-09-03T07:03:41.815 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:41.816 DEBUG:boto:Data: 2022-09-03T07:03:41.817 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.818 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.819 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.820 DEBUG:boto:Params: {} 2022-09-03T07:03:41.821 DEBUG:boto:Token: None 2022-09-03T07:03:41.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:41.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pBl/wR1yXfVyb2NEJFZCNfHWxKI= 2022-09-03T07:03:41.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pBl/wR1yXfVyb2NEJFZCNfHWxKI=', 'Content-Length': '0'} 2022-09-03T07:03:41.827 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f4db6b450f551e3d-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.829 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.830 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.832 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:41.833 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:41.834 DEBUG:boto:Method: GET 2022-09-03T07:03:41.835 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:41.836 DEBUG:boto:Data: 2022-09-03T07:03:41.837 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.838 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.839 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.840 DEBUG:boto:Params: {} 2022-09-03T07:03:41.841 DEBUG:boto:Token: None 2022-09-03T07:03:41.842 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.843 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.844 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.846 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a7f6d19fdaa8667-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.847 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.848 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.849 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.849 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:41.850 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:41.851 DEBUG:boto:Method: GET 2022-09-03T07:03:41.851 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:41.852 DEBUG:boto:Data: 2022-09-03T07:03:41.852 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.853 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.853 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.854 DEBUG:boto:Params: {} 2022-09-03T07:03:41.855 DEBUG:boto:Token: None 2022-09-03T07:03:41.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.859 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022d23307a0e03dae-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.860 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.860 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.861 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.861 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.862 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:41.863 DEBUG:boto:Data: 2022-09-03T07:03:41.863 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.864 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.864 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.865 DEBUG:boto:Params: {} 2022-09-03T07:03:41.865 DEBUG:boto:Token: None 2022-09-03T07:03:41.866 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g 2022-09-03T07:03:41.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bWBoKqeDRJWjyXKaVMSrCv5coDE= 2022-09-03T07:03:41.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bWBoKqeDRJWjyXKaVMSrCv5coDE=', 'Content-Length': '0'} 2022-09-03T07:03:41.872 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8a449f1d156ae7a-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.874 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.875 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.875 DEBUG:boto:Method: HEAD 2022-09-03T07:03:41.876 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:41.877 DEBUG:boto:Data: 2022-09-03T07:03:41.877 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.879 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.879 DEBUG:boto:Params: {} 2022-09-03T07:03:41.880 DEBUG:boto:Token: None 2022-09-03T07:03:41.881 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g 2022-09-03T07:03:41.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bWBoKqeDRJWjyXKaVMSrCv5coDE= 2022-09-03T07:03:41.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bWBoKqeDRJWjyXKaVMSrCv5coDE=', 'Content-Length': '0'} 2022-09-03T07:03:41.884 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f92c77e7c842a92-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.885 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:03:41.886 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.886 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.887 DEBUG:boto:Method: GET 2022-09-03T07:03:41.887 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:41.888 DEBUG:boto:Data: 2022-09-03T07:03:41.889 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.889 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.890 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.890 DEBUG:boto:Params: {} 2022-09-03T07:03:41.891 DEBUG:boto:Token: None 2022-09-03T07:03:41.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g 2022-09-03T07:03:41.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g7URgkhpy0v1YMMA7mUtF5f+DXg= 2022-09-03T07:03:41.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g7URgkhpy0v1YMMA7mUtF5f+DXg=', 'Content-Length': '0'} 2022-09-03T07:03:41.895 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c2f0b99988b4234-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.938 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:41.940 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:41.942 DEBUG:boto:Method: GET 2022-09-03T07:03:41.945 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:41.946 DEBUG:boto:Data: 2022-09-03T07:03:41.947 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.948 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.949 DEBUG:boto:Params: {} 2022-09-03T07:03:41.949 DEBUG:boto:Token: None 2022-09-03T07:03:41.950 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/g 2022-09-03T07:03:41.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g7URgkhpy0v1YMMA7mUtF5f+DXg= 2022-09-03T07:03:41.952 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g7URgkhpy0v1YMMA7mUtF5f+DXg=', 'Content-Length': '0'} 2022-09-03T07:03:41.954 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4290c92b54ff38b-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.955 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.956 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.956 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:41.957 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:41.957 DEBUG:boto:Method: GET 2022-09-03T07:03:41.958 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:41.958 DEBUG:boto:Data: 2022-09-03T07:03:41.959 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.959 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.960 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.961 DEBUG:boto:Params: {} 2022-09-03T07:03:41.961 DEBUG:boto:Token: None 2022-09-03T07:03:41.962 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.966 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b1dc8581c212e135-006312fc4d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.968 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.969 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:41.970 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:41.971 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:41.971 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:41.972 DEBUG:boto:Method: GET 2022-09-03T07:03:41.973 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:41.974 DEBUG:boto:Data: 2022-09-03T07:03:41.975 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.976 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:41.976 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.977 DEBUG:boto:Params: {} 2022-09-03T07:03:41.978 DEBUG:boto:Token: None 2022-09-03T07:03:41.979 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/ 2022-09-03T07:03:41.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4= 2022-09-03T07:03:41.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmuERlepoFbGx580Pikwab9UXE4=', 'Content-Length': '0'} 2022-09-03T07:03:41.984 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a77f195874dbfb3-006312fc4d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:41.985 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:41.986 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:03:41.987 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:41.988 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:41.989 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:41.990 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:41.991 DEBUG:boto:Method: GET 2022-09-03T07:03:41.992 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:41.993 DEBUG:boto:Data: 2022-09-03T07:03:41.994 DEBUG:boto:Headers: {} 2022-09-03T07:03:41.995 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:41.996 DEBUG:boto:Port: 8000 2022-09-03T07:03:41.997 DEBUG:boto:Params: {} 2022-09-03T07:03:41.998 DEBUG:boto:Token: None 2022-09-03T07:03:41.999 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:41 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:42.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Wub8OjiInigtVFNHxWgGNJtL4M= 2022-09-03T07:03:42.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Wub8OjiInigtVFNHxWgGNJtL4M=', 'Content-Length': '0'} 2022-09-03T07:03:42.004 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d81fef56d0db028-006312fc4e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.005 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:42.006 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:42.007 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.008 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.009 DEBUG:boto:Method: GET 2022-09-03T07:03:42.010 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:42.010 DEBUG:boto:Data: 2022-09-03T07:03:42.011 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.012 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.013 DEBUG:boto:Params: {} 2022-09-03T07:03:42.013 DEBUG:boto:Token: None 2022-09-03T07:03:42.014 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:42.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C6fb+CL3PRa2Of1+70VBWYGyC0c= 2022-09-03T07:03:42.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C6fb+CL3PRa2Of1+70VBWYGyC0c=', 'Content-Length': '0'} 2022-09-03T07:03:42.017 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b98d0c3850093d59-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.018 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:42.019 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:42.019 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.020 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.020 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.021 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:42.021 DEBUG:boto:Data: 2022-09-03T07:03:42.022 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.022 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:42.023 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.023 DEBUG:boto:Params: {} 2022-09-03T07:03:42.024 DEBUG:boto:Token: None 2022-09-03T07:03:42.024 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:42.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RW1Bmyo+3nXgIkWAzYnAkvFJukg= 2022-09-03T07:03:42.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RW1Bmyo+3nXgIkWAzYnAkvFJukg=', 'Content-Length': '0'} 2022-09-03T07:03:42.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038da06d50085b62e-006312fc4e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.028 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:42.029 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:42.029 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.030 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.030 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.031 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:42.031 DEBUG:boto:Data: 2022-09-03T07:03:42.032 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.033 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.033 DEBUG:boto:Params: {} 2022-09-03T07:03:42.034 DEBUG:boto:Token: None 2022-09-03T07:03:42.034 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:42.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RW1Bmyo+3nXgIkWAzYnAkvFJukg= 2022-09-03T07:03:42.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RW1Bmyo+3nXgIkWAzYnAkvFJukg=', 'Content-Length': '0'} 2022-09-03T07:03:42.038 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d9fb49061904df76-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.039 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:03:42.039 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:42.040 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:42.040 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.041 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.041 DEBUG:boto:Method: GET 2022-09-03T07:03:42.042 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:42.042 DEBUG:boto:Data: 2022-09-03T07:03:42.043 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.043 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:42.044 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.045 DEBUG:boto:Params: {} 2022-09-03T07:03:42.045 DEBUG:boto:Token: None 2022-09-03T07:03:42.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:42.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C6fb+CL3PRa2Of1+70VBWYGyC0c= 2022-09-03T07:03:42.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C6fb+CL3PRa2Of1+70VBWYGyC0c=', 'Content-Length': '0'} 2022-09-03T07:03:42.049 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000efde97a4bdf4cb3a-006312fc4e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.050 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:42.051 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:42.051 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.052 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:42.052 DEBUG:boto:Method: GET 2022-09-03T07:03:42.053 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:42.053 DEBUG:boto:Data: 2022-09-03T07:03:42.054 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.055 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.055 DEBUG:boto:Params: {} 2022-09-03T07:03:42.056 DEBUG:boto:Token: None 2022-09-03T07:03:42.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:42.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C6fb+CL3PRa2Of1+70VBWYGyC0c= 2022-09-03T07:03:42.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C6fb+CL3PRa2Of1+70VBWYGyC0c=', 'Content-Length': '0'} 2022-09-03T07:03:42.060 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a30d7bcc8842230d-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.061 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.061 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.062 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:42.062 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:42.063 DEBUG:boto:Method: GET 2022-09-03T07:03:42.063 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:42.064 DEBUG:boto:Data: 2022-09-03T07:03:42.064 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.065 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:42.066 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.066 DEBUG:boto:Params: {} 2022-09-03T07:03:42.067 DEBUG:boto:Token: None 2022-09-03T07:03:42.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.072 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e762348c235f597-006312fc4e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.073 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.075 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.076 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.077 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:42.078 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:42.079 DEBUG:boto:Method: GET 2022-09-03T07:03:42.080 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:42.081 DEBUG:boto:Data: 2022-09-03T07:03:42.081 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.083 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.083 DEBUG:boto:Params: {} 2022-09-03T07:03:42.084 DEBUG:boto:Token: None 2022-09-03T07:03:42.085 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.086 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.089 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000071cbc3b2aa0ca610-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.089 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.091 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:42.091 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:42.092 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.092 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:42.093 DEBUG:boto:Data: 2022-09-03T07:03:42.094 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.094 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:42.095 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.095 DEBUG:boto:Params: {} 2022-09-03T07:03:42.096 DEBUG:boto:Token: None 2022-09-03T07:03:42.096 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/h 2022-09-03T07:03:42.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KZZyJU5ClyAOFAfeZH6I+hgyr9U= 2022-09-03T07:03:42.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KZZyJU5ClyAOFAfeZH6I+hgyr9U=', 'Content-Length': '0'} 2022-09-03T07:03:42.100 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000095fbd145b6e3b59f-006312fc4e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.100 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:42.101 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:42.102 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.102 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:42.103 DEBUG:boto:Data: 2022-09-03T07:03:42.103 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.104 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.104 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.105 DEBUG:boto:Params: {} 2022-09-03T07:03:42.105 DEBUG:boto:Token: None 2022-09-03T07:03:42.106 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/h 2022-09-03T07:03:42.107 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KZZyJU5ClyAOFAfeZH6I+hgyr9U= 2022-09-03T07:03:42.107 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KZZyJU5ClyAOFAfeZH6I+hgyr9U=', 'Content-Length': '0'} 2022-09-03T07:03:42.112 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3361a63fd289883-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.113 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:03:42.113 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:42.114 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:42.114 DEBUG:boto:Method: GET 2022-09-03T07:03:42.115 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:42.115 DEBUG:boto:Data: 2022-09-03T07:03:42.116 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.117 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:42.117 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.118 DEBUG:boto:Params: {} 2022-09-03T07:03:42.119 DEBUG:boto:Token: None 2022-09-03T07:03:42.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/h 2022-09-03T07:03:42.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6TJLCuqL4hA+MUVgtxTIEH0MGwo= 2022-09-03T07:03:42.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6TJLCuqL4hA+MUVgtxTIEH0MGwo=', 'Content-Length': '0'} 2022-09-03T07:03:42.123 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3bd1ef2fb84b4eb-006312fc4e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.124 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:42.125 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:42.125 DEBUG:boto:Method: GET 2022-09-03T07:03:42.126 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:42.126 DEBUG:boto:Data: 2022-09-03T07:03:42.127 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.127 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.128 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.128 DEBUG:boto:Params: {} 2022-09-03T07:03:42.129 DEBUG:boto:Token: None 2022-09-03T07:03:42.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/h 2022-09-03T07:03:42.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6TJLCuqL4hA+MUVgtxTIEH0MGwo= 2022-09-03T07:03:42.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6TJLCuqL4hA+MUVgtxTIEH0MGwo=', 'Content-Length': '0'} 2022-09-03T07:03:42.134 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002da40760a1313581-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.135 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.135 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.136 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:42.136 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:42.137 DEBUG:boto:Method: GET 2022-09-03T07:03:42.137 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:42.138 DEBUG:boto:Data: 2022-09-03T07:03:42.138 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.139 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:42.140 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.140 DEBUG:boto:Params: {} 2022-09-03T07:03:42.141 DEBUG:boto:Token: None 2022-09-03T07:03:42.142 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.146 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000062a878bbba7c8623-006312fc4e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.194 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.196 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.197 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.198 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:42.198 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:42.199 DEBUG:boto:Method: GET 2022-09-03T07:03:42.199 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:42.200 DEBUG:boto:Data: 2022-09-03T07:03:42.201 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.202 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.203 DEBUG:boto:Params: {} 2022-09-03T07:03:42.203 DEBUG:boto:Token: None 2022-09-03T07:03:42.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c78c566c7ecdde09-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.210 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.212 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-73 zones={test-zone2, test-zone1} 2022-09-03T07:03:42.212 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-73 zones={test-zone3, test-zone1} 2022-09-03T07:03:42.214 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.215 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.216 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.216 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:03:42.217 DEBUG:boto:Data: 2022-09-03T07:03:42.218 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.219 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.220 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.221 DEBUG:boto:Params: {} 2022-09-03T07:03:42.222 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:03:42.223 DEBUG:boto:Token: None 2022-09-03T07:03:42.224 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mLjgjJKngC2wzZlK/YLLF02NbCo= 2022-09-03T07:03:42.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mLjgjJKngC2wzZlK/YLLF02NbCo=', 'Content-Length': '0'} 2022-09-03T07:03:42.231 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007ce75d767fb6701d-006312fc4e-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.232 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.233 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.235 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.236 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:03:42.237 DEBUG:boto:Data: 2022-09-03T07:03:42.237 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.239 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.240 DEBUG:boto:Params: {} 2022-09-03T07:03:42.241 DEBUG:boto:Token: None 2022-09-03T07:03:42.242 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mLjgjJKngC2wzZlK/YLLF02NbCo= 2022-09-03T07:03:42.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mLjgjJKngC2wzZlK/YLLF02NbCo=', 'Content-Length': '0'} 2022-09-03T07:03:42.246 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bdae01ad5f2b1467-006312fc4e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.247 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:03:42.248 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.249 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.250 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.251 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.252 DEBUG:boto:Method: GET 2022-09-03T07:03:42.253 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.253 DEBUG:boto:Data: 2022-09-03T07:03:42.255 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.255 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.256 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.256 DEBUG:boto:Params: {} 2022-09-03T07:03:42.257 DEBUG:boto:Token: None 2022-09-03T07:03:42.258 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/?versions 2022-09-03T07:03:42.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WcTQR2JVBZp+l9X2rdZ8nyROxzI= 2022-09-03T07:03:42.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WcTQR2JVBZp+l9X2rdZ8nyROxzI=', 'Content-Length': '0'} 2022-09-03T07:03:42.263 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ad1acff0c23e276-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.265 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.267 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:03:42.268 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:03:42.269 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:03:42.270 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:03:42.270 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:03:42.271 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:03:42.272 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:03:42.273 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:03:42.274 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:03:42.275 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.276 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.277 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.277 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.279 DEBUG:boto:Method: GET 2022-09-03T07:03:42.279 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.280 DEBUG:boto:Data: 2022-09-03T07:03:42.280 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.281 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.281 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.282 DEBUG:boto:Params: {} 2022-09-03T07:03:42.283 DEBUG:boto:Token: None 2022-09-03T07:03:42.283 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/?versions 2022-09-03T07:03:42.284 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WcTQR2JVBZp+l9X2rdZ8nyROxzI= 2022-09-03T07:03:42.284 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WcTQR2JVBZp+l9X2rdZ8nyROxzI=', 'Content-Length': '0'} 2022-09-03T07:03:42.287 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000057188172b7562e1-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.288 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.290 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:03:42.291 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:03:42.292 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:03:42.294 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:03:42.295 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:03:42.296 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:03:42.297 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:03:42.298 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:03:42.299 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.300 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.300 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.301 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.303 DEBUG:boto:Method: GET 2022-09-03T07:03:42.304 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.305 DEBUG:boto:Data: 2022-09-03T07:03:42.306 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.307 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.308 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.309 DEBUG:boto:Params: {} 2022-09-03T07:03:42.310 DEBUG:boto:Token: None 2022-09-03T07:03:42.312 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/?versions 2022-09-03T07:03:42.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WcTQR2JVBZp+l9X2rdZ8nyROxzI= 2022-09-03T07:03:42.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WcTQR2JVBZp+l9X2rdZ8nyROxzI=', 'Content-Length': '0'} 2022-09-03T07:03:42.321 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f7fa21932d7fffa2-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.323 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.325 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.326 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.327 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.328 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.329 DEBUG:boto:Method: GET 2022-09-03T07:03:42.329 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:42.330 DEBUG:boto:Data: 2022-09-03T07:03:42.331 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.333 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.334 DEBUG:boto:Params: {} 2022-09-03T07:03:42.335 DEBUG:boto:Token: None 2022-09-03T07:03:42.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/?versions 2022-09-03T07:03:42.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WcTQR2JVBZp+l9X2rdZ8nyROxzI= 2022-09-03T07:03:42.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WcTQR2JVBZp+l9X2rdZ8nyROxzI=', 'Content-Length': '0'} 2022-09-03T07:03:42.342 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b4244dbb2647ef05-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.343 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.345 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:03:42.346 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.347 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.348 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.349 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.350 DEBUG:boto:Method: GET 2022-09-03T07:03:42.351 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:42.352 DEBUG:boto:Data: 2022-09-03T07:03:42.353 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.354 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.356 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.356 DEBUG:boto:Params: {} 2022-09-03T07:03:42.357 DEBUG:boto:Token: None 2022-09-03T07:03:42.358 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:42.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pnsnihlMi+QDNpn92rHr6e4JQkA= 2022-09-03T07:03:42.359 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pnsnihlMi+QDNpn92rHr6e4JQkA=', 'Content-Length': '0'} 2022-09-03T07:03:42.362 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd90dc478c9ba683-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.373 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.379 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.379 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.380 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.381 DEBUG:boto:Method: GET 2022-09-03T07:03:42.381 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:42.382 DEBUG:boto:Data: 2022-09-03T07:03:42.382 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.384 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.386 DEBUG:boto:Params: {} 2022-09-03T07:03:42.387 DEBUG:boto:Token: None 2022-09-03T07:03:42.388 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:42.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pnsnihlMi+QDNpn92rHr6e4JQkA= 2022-09-03T07:03:42.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pnsnihlMi+QDNpn92rHr6e4JQkA=', 'Content-Length': '0'} 2022-09-03T07:03:42.391 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c9d7e8b7d48a7e2-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.392 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.393 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.394 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.395 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.395 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.396 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:42.396 DEBUG:boto:Data: 2022-09-03T07:03:42.397 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.398 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.398 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.399 DEBUG:boto:Params: {} 2022-09-03T07:03:42.399 DEBUG:boto:Token: None 2022-09-03T07:03:42.400 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:42.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIoGLM9xeh5IV2l1SYIWbAN1NBM= 2022-09-03T07:03:42.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIoGLM9xeh5IV2l1SYIWbAN1NBM=', 'Content-Length': '0'} 2022-09-03T07:03:42.403 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035f4b34ed07dfc7a-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.404 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.405 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.405 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.406 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.406 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.407 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:42.408 DEBUG:boto:Data: 2022-09-03T07:03:42.408 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.409 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.409 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.410 DEBUG:boto:Params: {} 2022-09-03T07:03:42.410 DEBUG:boto:Token: None 2022-09-03T07:03:42.411 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:42.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIoGLM9xeh5IV2l1SYIWbAN1NBM= 2022-09-03T07:03:42.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIoGLM9xeh5IV2l1SYIWbAN1NBM=', 'Content-Length': '0'} 2022-09-03T07:03:42.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d59f001c1d128ce-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.416 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:03:42.417 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.417 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.418 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.418 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.419 DEBUG:boto:Method: GET 2022-09-03T07:03:42.420 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:42.420 DEBUG:boto:Data: 2022-09-03T07:03:42.421 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.422 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.422 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.423 DEBUG:boto:Params: {} 2022-09-03T07:03:42.424 DEBUG:boto:Token: None 2022-09-03T07:03:42.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:42.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pnsnihlMi+QDNpn92rHr6e4JQkA= 2022-09-03T07:03:42.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pnsnihlMi+QDNpn92rHr6e4JQkA=', 'Content-Length': '0'} 2022-09-03T07:03:42.435 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b66c1c220da85022-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.436 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.437 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.438 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.438 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:42.439 DEBUG:boto:Method: GET 2022-09-03T07:03:42.439 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:42.440 DEBUG:boto:Data: 2022-09-03T07:03:42.441 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.442 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.442 DEBUG:boto:Params: {} 2022-09-03T07:03:42.443 DEBUG:boto:Token: None 2022-09-03T07:03:42.444 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:42.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pnsnihlMi+QDNpn92rHr6e4JQkA= 2022-09-03T07:03:42.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pnsnihlMi+QDNpn92rHr6e4JQkA=', 'Content-Length': '0'} 2022-09-03T07:03:42.448 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3a7fa1de38f72bc-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.490 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.491 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.492 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:42.493 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:42.493 DEBUG:boto:Method: GET 2022-09-03T07:03:42.494 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:42.495 DEBUG:boto:Data: 2022-09-03T07:03:42.495 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.497 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.498 DEBUG:boto:Params: {} 2022-09-03T07:03:42.499 DEBUG:boto:Token: None 2022-09-03T07:03:42.500 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.501 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.507 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d59423250ad18342-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.508 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.509 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.510 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.510 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:42.511 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:42.512 DEBUG:boto:Method: GET 2022-09-03T07:03:42.512 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:42.513 DEBUG:boto:Data: 2022-09-03T07:03:42.513 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.514 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.515 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.515 DEBUG:boto:Params: {} 2022-09-03T07:03:42.516 DEBUG:boto:Token: None 2022-09-03T07:03:42.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.521 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002647dc4e5682431b-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.522 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.523 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.524 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.525 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.526 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:42.527 DEBUG:boto:Data: 2022-09-03T07:03:42.528 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.529 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.530 DEBUG:boto:Params: {} 2022-09-03T07:03:42.531 DEBUG:boto:Token: None 2022-09-03T07:03:42.532 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a 2022-09-03T07:03:42.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qo1kFTWukMHxTbw7DFW5p9eDYy0= 2022-09-03T07:03:42.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qo1kFTWukMHxTbw7DFW5p9eDYy0=', 'Content-Length': '0'} 2022-09-03T07:03:42.536 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000302b129076f403f9-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.537 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.537 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.538 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.539 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:42.539 DEBUG:boto:Data: 2022-09-03T07:03:42.540 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.541 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.542 DEBUG:boto:Params: {} 2022-09-03T07:03:42.543 DEBUG:boto:Token: None 2022-09-03T07:03:42.544 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a 2022-09-03T07:03:42.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qo1kFTWukMHxTbw7DFW5p9eDYy0= 2022-09-03T07:03:42.546 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qo1kFTWukMHxTbw7DFW5p9eDYy0=', 'Content-Length': '0'} 2022-09-03T07:03:42.548 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000652a8b4903944e1d-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.549 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:03:42.550 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.551 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.552 DEBUG:boto:Method: GET 2022-09-03T07:03:42.553 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:42.554 DEBUG:boto:Data: 2022-09-03T07:03:42.555 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.557 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.558 DEBUG:boto:Params: {} 2022-09-03T07:03:42.559 DEBUG:boto:Token: None 2022-09-03T07:03:42.560 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a 2022-09-03T07:03:42.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nd3BtOe8kiVWSFZ4TJeLvRC/g4c= 2022-09-03T07:03:42.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nd3BtOe8kiVWSFZ4TJeLvRC/g4c=', 'Content-Length': '0'} 2022-09-03T07:03:42.565 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e5748b20f6fd2e0-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.566 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:42.567 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:42.568 DEBUG:boto:Method: GET 2022-09-03T07:03:42.569 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:42.570 DEBUG:boto:Data: 2022-09-03T07:03:42.570 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.571 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.572 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.573 DEBUG:boto:Params: {} 2022-09-03T07:03:42.574 DEBUG:boto:Token: None 2022-09-03T07:03:42.575 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/a 2022-09-03T07:03:42.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nd3BtOe8kiVWSFZ4TJeLvRC/g4c= 2022-09-03T07:03:42.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nd3BtOe8kiVWSFZ4TJeLvRC/g4c=', 'Content-Length': '0'} 2022-09-03T07:03:42.580 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001727ed8c2019a7de-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.581 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.581 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.582 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:42.583 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:42.583 DEBUG:boto:Method: GET 2022-09-03T07:03:42.584 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:42.584 DEBUG:boto:Data: 2022-09-03T07:03:42.585 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.586 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.587 DEBUG:boto:Params: {} 2022-09-03T07:03:42.587 DEBUG:boto:Token: None 2022-09-03T07:03:42.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.593 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007ab6b8cc7963d6ba-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.593 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.594 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.595 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.596 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:42.596 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:42.598 DEBUG:boto:Method: GET 2022-09-03T07:03:42.600 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:42.601 DEBUG:boto:Data: 2022-09-03T07:03:42.602 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.604 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.606 DEBUG:boto:Params: {} 2022-09-03T07:03:42.607 DEBUG:boto:Token: None 2022-09-03T07:03:42.609 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.610 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.615 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000adecdcd9fb7e249b-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.617 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.629 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:03:42.630 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.631 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.632 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.632 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.633 DEBUG:boto:Method: GET 2022-09-03T07:03:42.633 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:42.634 DEBUG:boto:Data: 2022-09-03T07:03:42.634 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.635 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.636 DEBUG:boto:Params: {} 2022-09-03T07:03:42.637 DEBUG:boto:Token: None 2022-09-03T07:03:42.637 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:42.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I8QtxiNJhXKUs9OH+O/jS/7mABM= 2022-09-03T07:03:42.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I8QtxiNJhXKUs9OH+O/jS/7mABM=', 'Content-Length': '0'} 2022-09-03T07:03:42.641 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034af730e02982562-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.642 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.643 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.644 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.644 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.645 DEBUG:boto:Method: GET 2022-09-03T07:03:42.646 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:42.646 DEBUG:boto:Data: 2022-09-03T07:03:42.647 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.648 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.649 DEBUG:boto:Params: {} 2022-09-03T07:03:42.649 DEBUG:boto:Token: None 2022-09-03T07:03:42.650 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:42.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I8QtxiNJhXKUs9OH+O/jS/7mABM= 2022-09-03T07:03:42.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I8QtxiNJhXKUs9OH+O/jS/7mABM=', 'Content-Length': '0'} 2022-09-03T07:03:42.655 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fdf9f1f65114eb67-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.657 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.658 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.659 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.660 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.661 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.662 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:42.663 DEBUG:boto:Data: 2022-09-03T07:03:42.663 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.665 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.666 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.667 DEBUG:boto:Params: {} 2022-09-03T07:03:42.668 DEBUG:boto:Token: None 2022-09-03T07:03:42.669 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:42.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wfidXZ6Ifp4qBVkBbuCjui+Ih7M= 2022-09-03T07:03:42.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wfidXZ6Ifp4qBVkBbuCjui+Ih7M=', 'Content-Length': '0'} 2022-09-03T07:03:42.674 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fcb4e5439dc5160-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.675 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.676 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.676 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.677 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.678 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.679 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:42.680 DEBUG:boto:Data: 2022-09-03T07:03:42.680 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.681 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.681 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.682 DEBUG:boto:Params: {} 2022-09-03T07:03:42.683 DEBUG:boto:Token: None 2022-09-03T07:03:42.684 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:42.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wfidXZ6Ifp4qBVkBbuCjui+Ih7M= 2022-09-03T07:03:42.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wfidXZ6Ifp4qBVkBbuCjui+Ih7M=', 'Content-Length': '0'} 2022-09-03T07:03:42.689 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060b4b0616a7e4a79-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.690 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:03:42.691 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.692 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.693 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.694 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.695 DEBUG:boto:Method: GET 2022-09-03T07:03:42.702 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:42.703 DEBUG:boto:Data: 2022-09-03T07:03:42.704 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.705 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.706 DEBUG:boto:Params: {} 2022-09-03T07:03:42.707 DEBUG:boto:Token: None 2022-09-03T07:03:42.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:42.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I8QtxiNJhXKUs9OH+O/jS/7mABM= 2022-09-03T07:03:42.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I8QtxiNJhXKUs9OH+O/jS/7mABM=', 'Content-Length': '0'} 2022-09-03T07:03:42.713 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a417c19f96f54c1c-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.758 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.760 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.761 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.762 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:42.763 DEBUG:boto:Method: GET 2022-09-03T07:03:42.764 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:42.765 DEBUG:boto:Data: 2022-09-03T07:03:42.766 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.767 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.768 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.769 DEBUG:boto:Params: {} 2022-09-03T07:03:42.770 DEBUG:boto:Token: None 2022-09-03T07:03:42.771 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:42.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I8QtxiNJhXKUs9OH+O/jS/7mABM= 2022-09-03T07:03:42.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I8QtxiNJhXKUs9OH+O/jS/7mABM=', 'Content-Length': '0'} 2022-09-03T07:03:42.776 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006886aeec227307ad-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.778 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.779 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.780 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:42.780 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:42.782 DEBUG:boto:Method: GET 2022-09-03T07:03:42.782 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:42.783 DEBUG:boto:Data: 2022-09-03T07:03:42.784 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.785 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.786 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.787 DEBUG:boto:Params: {} 2022-09-03T07:03:42.788 DEBUG:boto:Token: None 2022-09-03T07:03:42.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.793 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac94f1c69ac93517-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.794 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.796 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.797 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.798 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:42.799 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:42.800 DEBUG:boto:Method: GET 2022-09-03T07:03:42.801 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:42.802 DEBUG:boto:Data: 2022-09-03T07:03:42.803 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.803 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.804 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.805 DEBUG:boto:Params: {} 2022-09-03T07:03:42.805 DEBUG:boto:Token: None 2022-09-03T07:03:42.806 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.810 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d1100d23a566439-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.810 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.811 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.812 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.812 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.813 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:42.815 DEBUG:boto:Data: 2022-09-03T07:03:42.815 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.816 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.817 DEBUG:boto:Params: {} 2022-09-03T07:03:42.817 DEBUG:boto:Token: None 2022-09-03T07:03:42.818 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b 2022-09-03T07:03:42.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5w9uya4e9pM+lx/4fgv6bdGxbs= 2022-09-03T07:03:42.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5w9uya4e9pM+lx/4fgv6bdGxbs=', 'Content-Length': '0'} 2022-09-03T07:03:42.821 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003465ced03e4332d0-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.822 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.823 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.823 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.824 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:42.824 DEBUG:boto:Data: 2022-09-03T07:03:42.825 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.825 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.826 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.826 DEBUG:boto:Params: {} 2022-09-03T07:03:42.827 DEBUG:boto:Token: None 2022-09-03T07:03:42.828 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b 2022-09-03T07:03:42.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y5w9uya4e9pM+lx/4fgv6bdGxbs= 2022-09-03T07:03:42.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y5w9uya4e9pM+lx/4fgv6bdGxbs=', 'Content-Length': '0'} 2022-09-03T07:03:42.831 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000383b531c1cae1711-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.832 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:03:42.833 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.833 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.834 DEBUG:boto:Method: GET 2022-09-03T07:03:42.834 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:42.835 DEBUG:boto:Data: 2022-09-03T07:03:42.836 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.836 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.837 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.837 DEBUG:boto:Params: {} 2022-09-03T07:03:42.838 DEBUG:boto:Token: None 2022-09-03T07:03:42.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b 2022-09-03T07:03:42.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1YV7aURTpOo+w0b5VgwCHtVORsc= 2022-09-03T07:03:42.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1YV7aURTpOo+w0b5VgwCHtVORsc=', 'Content-Length': '0'} 2022-09-03T07:03:42.844 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a793ea2ea19a3605-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.889 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:42.890 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:42.891 DEBUG:boto:Method: GET 2022-09-03T07:03:42.892 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:42.892 DEBUG:boto:Data: 2022-09-03T07:03:42.893 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.894 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.894 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.895 DEBUG:boto:Params: {} 2022-09-03T07:03:42.896 DEBUG:boto:Token: None 2022-09-03T07:03:42.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/b 2022-09-03T07:03:42.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1YV7aURTpOo+w0b5VgwCHtVORsc= 2022-09-03T07:03:42.898 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1YV7aURTpOo+w0b5VgwCHtVORsc=', 'Content-Length': '0'} 2022-09-03T07:03:42.900 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e0869ebbca32525-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.902 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.902 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.903 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:42.903 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:42.904 DEBUG:boto:Method: GET 2022-09-03T07:03:42.904 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:42.905 DEBUG:boto:Data: 2022-09-03T07:03:42.905 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.907 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.908 DEBUG:boto:Params: {} 2022-09-03T07:03:42.908 DEBUG:boto:Token: None 2022-09-03T07:03:42.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.914 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed536c594a92a24b-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.915 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.916 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:42.917 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:42.917 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:42.918 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:42.919 DEBUG:boto:Method: GET 2022-09-03T07:03:42.920 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:42.921 DEBUG:boto:Data: 2022-09-03T07:03:42.922 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.923 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.924 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.925 DEBUG:boto:Params: {} 2022-09-03T07:03:42.926 DEBUG:boto:Token: None 2022-09-03T07:03:42.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/ 2022-09-03T07:03:42.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE= 2022-09-03T07:03:42.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zPAowU/hzsEyWyqVcY5UlAJ9YTE=', 'Content-Length': '0'} 2022-09-03T07:03:42.932 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000029c31e0aeac90b8-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.933 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:42.934 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:03:42.935 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:42.935 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:42.936 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.936 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.937 DEBUG:boto:Method: GET 2022-09-03T07:03:42.938 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:42.938 DEBUG:boto:Data: 2022-09-03T07:03:42.939 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.940 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.940 DEBUG:boto:Params: {} 2022-09-03T07:03:42.941 DEBUG:boto:Token: None 2022-09-03T07:03:42.942 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:42.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aQwZ2WTChGz4dsHtAzSL5tM7LXA= 2022-09-03T07:03:42.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aQwZ2WTChGz4dsHtAzSL5tM7LXA=', 'Content-Length': '0'} 2022-09-03T07:03:42.945 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab1cf5f5483e3486-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.946 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:42.947 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:42.947 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.948 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.949 DEBUG:boto:Method: GET 2022-09-03T07:03:42.949 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:42.950 DEBUG:boto:Data: 2022-09-03T07:03:42.950 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.952 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.952 DEBUG:boto:Params: {} 2022-09-03T07:03:42.953 DEBUG:boto:Token: None 2022-09-03T07:03:42.953 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:42.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aQwZ2WTChGz4dsHtAzSL5tM7LXA= 2022-09-03T07:03:42.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aQwZ2WTChGz4dsHtAzSL5tM7LXA=', 'Content-Length': '0'} 2022-09-03T07:03:42.957 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d9443e68b9b462d-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.958 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:42.958 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:42.959 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.959 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.960 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.960 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:42.961 DEBUG:boto:Data: 2022-09-03T07:03:42.961 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.962 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.962 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.963 DEBUG:boto:Params: {} 2022-09-03T07:03:42.963 DEBUG:boto:Token: None 2022-09-03T07:03:42.964 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:42.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hv8vzFjTQY9e7oLlN58Dfs97hpA= 2022-09-03T07:03:42.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hv8vzFjTQY9e7oLlN58Dfs97hpA=', 'Content-Length': '0'} 2022-09-03T07:03:42.967 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc9e5dbcafc1f05c-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.968 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:42.969 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:42.969 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.970 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.970 DEBUG:boto:Method: HEAD 2022-09-03T07:03:42.971 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:42.971 DEBUG:boto:Data: 2022-09-03T07:03:42.972 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.973 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.973 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.974 DEBUG:boto:Params: {} 2022-09-03T07:03:42.974 DEBUG:boto:Token: None 2022-09-03T07:03:42.975 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:42.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hv8vzFjTQY9e7oLlN58Dfs97hpA= 2022-09-03T07:03:42.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hv8vzFjTQY9e7oLlN58Dfs97hpA=', 'Content-Length': '0'} 2022-09-03T07:03:42.979 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000446b3fa4a892dfd3-006312fc4e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.980 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:03:42.980 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:42.981 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:42.982 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.982 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.983 DEBUG:boto:Method: GET 2022-09-03T07:03:42.984 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:42.984 DEBUG:boto:Data: 2022-09-03T07:03:42.985 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:42.986 DEBUG:boto:Port: 8001 2022-09-03T07:03:42.987 DEBUG:boto:Params: {} 2022-09-03T07:03:42.987 DEBUG:boto:Token: None 2022-09-03T07:03:42.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:42.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aQwZ2WTChGz4dsHtAzSL5tM7LXA= 2022-09-03T07:03:42.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aQwZ2WTChGz4dsHtAzSL5tM7LXA=', 'Content-Length': '0'} 2022-09-03T07:03:42.991 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008128a3b62187c4a2-006312fc4e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:42.992 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:42.993 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:42.993 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.994 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:42.995 DEBUG:boto:Method: GET 2022-09-03T07:03:42.995 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:42.996 DEBUG:boto:Data: 2022-09-03T07:03:42.996 DEBUG:boto:Headers: {} 2022-09-03T07:03:42.997 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:42.997 DEBUG:boto:Port: 8000 2022-09-03T07:03:42.998 DEBUG:boto:Params: {} 2022-09-03T07:03:42.999 DEBUG:boto:Token: None 2022-09-03T07:03:42.999 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:42 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:43.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aQwZ2WTChGz4dsHtAzSL5tM7LXA= 2022-09-03T07:03:43.000 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aQwZ2WTChGz4dsHtAzSL5tM7LXA=', 'Content-Length': '0'} 2022-09-03T07:03:43.003 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003fe4b4e300ad7919-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.004 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.004 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.005 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:43.005 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:43.006 DEBUG:boto:Method: GET 2022-09-03T07:03:43.007 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:43.007 DEBUG:boto:Data: 2022-09-03T07:03:43.008 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.009 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.009 DEBUG:boto:Params: {} 2022-09-03T07:03:43.010 DEBUG:boto:Token: None 2022-09-03T07:03:43.011 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.015 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000885a363213d197db-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.016 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.017 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.017 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.018 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:43.018 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:43.019 DEBUG:boto:Method: GET 2022-09-03T07:03:43.019 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:43.020 DEBUG:boto:Data: 2022-09-03T07:03:43.020 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.022 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.022 DEBUG:boto:Params: {} 2022-09-03T07:03:43.023 DEBUG:boto:Token: None 2022-09-03T07:03:43.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.027 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c363f708a2598aa-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.028 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.029 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:43.030 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:43.031 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.033 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:43.033 DEBUG:boto:Data: 2022-09-03T07:03:43.034 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.035 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.036 DEBUG:boto:Params: {} 2022-09-03T07:03:43.037 DEBUG:boto:Token: None 2022-09-03T07:03:43.038 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/c 2022-09-03T07:03:43.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jGmSSOtB+PhWCG86MUIXzA1DK0U= 2022-09-03T07:03:43.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jGmSSOtB+PhWCG86MUIXzA1DK0U=', 'Content-Length': '0'} 2022-09-03T07:03:43.042 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002296070344f536a3-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.043 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:43.044 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:43.044 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.045 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:43.046 DEBUG:boto:Data: 2022-09-03T07:03:43.047 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.048 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.049 DEBUG:boto:Params: {} 2022-09-03T07:03:43.050 DEBUG:boto:Token: None 2022-09-03T07:03:43.051 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/c 2022-09-03T07:03:43.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jGmSSOtB+PhWCG86MUIXzA1DK0U= 2022-09-03T07:03:43.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jGmSSOtB+PhWCG86MUIXzA1DK0U=', 'Content-Length': '0'} 2022-09-03T07:03:43.056 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f2b11dafc9676ff3-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.057 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:03:43.058 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:43.059 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:43.060 DEBUG:boto:Method: GET 2022-09-03T07:03:43.062 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:43.063 DEBUG:boto:Data: 2022-09-03T07:03:43.064 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.064 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.065 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.066 DEBUG:boto:Params: {} 2022-09-03T07:03:43.067 DEBUG:boto:Token: None 2022-09-03T07:03:43.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/c 2022-09-03T07:03:43.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOk3w7YkaMju/Gaq+ZEZZhR9DMk= 2022-09-03T07:03:43.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOk3w7YkaMju/Gaq+ZEZZhR9DMk=', 'Content-Length': '0'} 2022-09-03T07:03:43.072 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008065e11f3fa83f34-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.116 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:43.117 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:43.118 DEBUG:boto:Method: GET 2022-09-03T07:03:43.118 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:43.119 DEBUG:boto:Data: 2022-09-03T07:03:43.119 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.120 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.121 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.121 DEBUG:boto:Params: {} 2022-09-03T07:03:43.122 DEBUG:boto:Token: None 2022-09-03T07:03:43.123 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/c 2022-09-03T07:03:43.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOk3w7YkaMju/Gaq+ZEZZhR9DMk= 2022-09-03T07:03:43.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOk3w7YkaMju/Gaq+ZEZZhR9DMk=', 'Content-Length': '0'} 2022-09-03T07:03:43.127 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d4e04dc0d0a3a7a2-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.129 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.130 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.132 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:43.133 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:43.134 DEBUG:boto:Method: GET 2022-09-03T07:03:43.135 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:43.136 DEBUG:boto:Data: 2022-09-03T07:03:43.137 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.138 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.140 DEBUG:boto:Params: {} 2022-09-03T07:03:43.141 DEBUG:boto:Token: None 2022-09-03T07:03:43.144 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.149 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e1de2717f84f071-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.150 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.152 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.153 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.154 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:43.155 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:43.156 DEBUG:boto:Method: GET 2022-09-03T07:03:43.157 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:43.158 DEBUG:boto:Data: 2022-09-03T07:03:43.159 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.160 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.161 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.162 DEBUG:boto:Params: {} 2022-09-03T07:03:43.163 DEBUG:boto:Token: None 2022-09-03T07:03:43.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.179 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f2dda0081cf82590-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.189 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.192 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:03:43.193 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.195 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.196 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.197 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.198 DEBUG:boto:Method: GET 2022-09-03T07:03:43.208 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:43.209 DEBUG:boto:Data: 2022-09-03T07:03:43.209 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.212 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.212 DEBUG:boto:Params: {} 2022-09-03T07:03:43.213 DEBUG:boto:Token: None 2022-09-03T07:03:43.216 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:43.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DBEkMqVE0M/oJnZdkEIc/9uFkMg= 2022-09-03T07:03:43.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DBEkMqVE0M/oJnZdkEIc/9uFkMg=', 'Content-Length': '0'} 2022-09-03T07:03:43.222 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dcdc12f8f5565a02-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.224 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.225 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.226 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.229 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.230 DEBUG:boto:Method: GET 2022-09-03T07:03:43.231 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:43.232 DEBUG:boto:Data: 2022-09-03T07:03:43.233 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.235 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.236 DEBUG:boto:Params: {} 2022-09-03T07:03:43.237 DEBUG:boto:Token: None 2022-09-03T07:03:43.238 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:43.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DBEkMqVE0M/oJnZdkEIc/9uFkMg= 2022-09-03T07:03:43.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DBEkMqVE0M/oJnZdkEIc/9uFkMg=', 'Content-Length': '0'} 2022-09-03T07:03:43.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3cd0e90c7bcdc79-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.244 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.244 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.245 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.245 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.246 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.247 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:43.247 DEBUG:boto:Data: 2022-09-03T07:03:43.247 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.248 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.249 DEBUG:boto:Params: {} 2022-09-03T07:03:43.250 DEBUG:boto:Token: None 2022-09-03T07:03:43.250 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:43.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PA1gpTjGztTxPuZZPCU4h6zXjf0= 2022-09-03T07:03:43.252 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PA1gpTjGztTxPuZZPCU4h6zXjf0=', 'Content-Length': '0'} 2022-09-03T07:03:43.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c816a3336c2d557-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.257 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.259 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.260 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.261 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.262 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.263 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:43.263 DEBUG:boto:Data: 2022-09-03T07:03:43.264 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.265 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.266 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.267 DEBUG:boto:Params: {} 2022-09-03T07:03:43.268 DEBUG:boto:Token: None 2022-09-03T07:03:43.269 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:43.270 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PA1gpTjGztTxPuZZPCU4h6zXjf0= 2022-09-03T07:03:43.271 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PA1gpTjGztTxPuZZPCU4h6zXjf0=', 'Content-Length': '0'} 2022-09-03T07:03:43.279 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dbe634675d5f8e77-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.281 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:03:43.282 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.283 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.284 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.285 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.286 DEBUG:boto:Method: GET 2022-09-03T07:03:43.286 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:43.287 DEBUG:boto:Data: 2022-09-03T07:03:43.288 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.290 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.291 DEBUG:boto:Params: {} 2022-09-03T07:03:43.292 DEBUG:boto:Token: None 2022-09-03T07:03:43.293 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:43.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DBEkMqVE0M/oJnZdkEIc/9uFkMg= 2022-09-03T07:03:43.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DBEkMqVE0M/oJnZdkEIc/9uFkMg=', 'Content-Length': '0'} 2022-09-03T07:03:43.299 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029b0fe381f40d23f-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.343 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.344 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.352 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.353 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:43.354 DEBUG:boto:Method: GET 2022-09-03T07:03:43.355 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:43.356 DEBUG:boto:Data: 2022-09-03T07:03:43.357 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.364 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.365 DEBUG:boto:Params: {} 2022-09-03T07:03:43.366 DEBUG:boto:Token: None 2022-09-03T07:03:43.367 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:43.368 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DBEkMqVE0M/oJnZdkEIc/9uFkMg= 2022-09-03T07:03:43.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DBEkMqVE0M/oJnZdkEIc/9uFkMg=', 'Content-Length': '0'} 2022-09-03T07:03:43.372 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca99ab3cb30ed54a-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.374 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.377 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.378 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:43.379 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:43.380 DEBUG:boto:Method: GET 2022-09-03T07:03:43.381 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:43.382 DEBUG:boto:Data: 2022-09-03T07:03:43.383 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.385 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.386 DEBUG:boto:Params: {} 2022-09-03T07:03:43.387 DEBUG:boto:Token: None 2022-09-03T07:03:43.388 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.394 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004da4893ea3cbfdcd-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.395 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.398 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.399 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.400 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:43.401 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:43.401 DEBUG:boto:Method: GET 2022-09-03T07:03:43.402 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:43.403 DEBUG:boto:Data: 2022-09-03T07:03:43.404 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.407 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.407 DEBUG:boto:Params: {} 2022-09-03T07:03:43.408 DEBUG:boto:Token: None 2022-09-03T07:03:43.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.413 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000632b5550d73088bb-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.414 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.415 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.417 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.417 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.418 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:43.419 DEBUG:boto:Data: 2022-09-03T07:03:43.419 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.420 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.420 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.421 DEBUG:boto:Params: {} 2022-09-03T07:03:43.422 DEBUG:boto:Token: None 2022-09-03T07:03:43.424 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d 2022-09-03T07:03:43.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FF38+LXBiuTuTz3MkO5XGiVmCK4= 2022-09-03T07:03:43.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FF38+LXBiuTuTz3MkO5XGiVmCK4=', 'Content-Length': '0'} 2022-09-03T07:03:43.428 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000009b0ad9035e6fff-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.429 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.430 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.430 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.431 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:43.431 DEBUG:boto:Data: 2022-09-03T07:03:43.432 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.433 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.434 DEBUG:boto:Params: {} 2022-09-03T07:03:43.436 DEBUG:boto:Token: None 2022-09-03T07:03:43.436 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d 2022-09-03T07:03:43.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FF38+LXBiuTuTz3MkO5XGiVmCK4= 2022-09-03T07:03:43.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FF38+LXBiuTuTz3MkO5XGiVmCK4=', 'Content-Length': '0'} 2022-09-03T07:03:43.441 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f5ea4be1faa9366-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.442 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:03:43.442 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.443 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.443 DEBUG:boto:Method: GET 2022-09-03T07:03:43.444 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:43.444 DEBUG:boto:Data: 2022-09-03T07:03:43.445 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.446 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.447 DEBUG:boto:Params: {} 2022-09-03T07:03:43.447 DEBUG:boto:Token: None 2022-09-03T07:03:43.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d 2022-09-03T07:03:43.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pJqjDHcUbh8SPsa7PJVYiLOJOMc= 2022-09-03T07:03:43.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pJqjDHcUbh8SPsa7PJVYiLOJOMc=', 'Content-Length': '0'} 2022-09-03T07:03:43.452 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000897a732a66566258-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.453 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:43.453 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:43.454 DEBUG:boto:Method: GET 2022-09-03T07:03:43.454 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:43.455 DEBUG:boto:Data: 2022-09-03T07:03:43.455 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.456 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.457 DEBUG:boto:Params: {} 2022-09-03T07:03:43.458 DEBUG:boto:Token: None 2022-09-03T07:03:43.458 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/d 2022-09-03T07:03:43.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pJqjDHcUbh8SPsa7PJVYiLOJOMc= 2022-09-03T07:03:43.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pJqjDHcUbh8SPsa7PJVYiLOJOMc=', 'Content-Length': '0'} 2022-09-03T07:03:43.462 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000adbec671d8b4725c-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.463 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.464 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.464 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:43.465 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:43.465 DEBUG:boto:Method: GET 2022-09-03T07:03:43.466 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:43.466 DEBUG:boto:Data: 2022-09-03T07:03:43.467 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.468 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.468 DEBUG:boto:Params: {} 2022-09-03T07:03:43.469 DEBUG:boto:Token: None 2022-09-03T07:03:43.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.475 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005a4ff35604a2bc17-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.476 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.477 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.478 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.479 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:43.479 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:43.480 DEBUG:boto:Method: GET 2022-09-03T07:03:43.480 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:43.481 DEBUG:boto:Data: 2022-09-03T07:03:43.482 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.484 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.484 DEBUG:boto:Params: {} 2022-09-03T07:03:43.485 DEBUG:boto:Token: None 2022-09-03T07:03:43.486 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.487 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.491 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093e6f95cf2610f22-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.492 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.493 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:03:43.494 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.495 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.495 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.496 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.497 DEBUG:boto:Method: GET 2022-09-03T07:03:43.497 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:43.498 DEBUG:boto:Data: 2022-09-03T07:03:43.498 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.499 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.500 DEBUG:boto:Params: {} 2022-09-03T07:03:43.501 DEBUG:boto:Token: None 2022-09-03T07:03:43.501 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:43.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VG0Pjly+Q2Xjxc/5O+QSwEhykDQ= 2022-09-03T07:03:43.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VG0Pjly+Q2Xjxc/5O+QSwEhykDQ=', 'Content-Length': '0'} 2022-09-03T07:03:43.505 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ad0260843a1b51c-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.506 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.506 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.507 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.508 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.508 DEBUG:boto:Method: GET 2022-09-03T07:03:43.509 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:43.509 DEBUG:boto:Data: 2022-09-03T07:03:43.510 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.510 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.511 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.512 DEBUG:boto:Params: {} 2022-09-03T07:03:43.513 DEBUG:boto:Token: None 2022-09-03T07:03:43.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:43.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VG0Pjly+Q2Xjxc/5O+QSwEhykDQ= 2022-09-03T07:03:43.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VG0Pjly+Q2Xjxc/5O+QSwEhykDQ=', 'Content-Length': '0'} 2022-09-03T07:03:43.517 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b4c5c822f69889bb-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.558 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.563 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.564 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.565 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.566 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.566 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:43.567 DEBUG:boto:Data: 2022-09-03T07:03:43.568 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.569 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.570 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.570 DEBUG:boto:Params: {} 2022-09-03T07:03:43.572 DEBUG:boto:Token: None 2022-09-03T07:03:43.573 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:43.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OJiTrz310GqUB8Hh++jFFaq4PDo= 2022-09-03T07:03:43.574 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OJiTrz310GqUB8Hh++jFFaq4PDo=', 'Content-Length': '0'} 2022-09-03T07:03:43.576 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac886e12afc60702-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.577 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.578 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.578 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.579 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.579 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.580 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:43.580 DEBUG:boto:Data: 2022-09-03T07:03:43.581 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.581 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.582 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.582 DEBUG:boto:Params: {} 2022-09-03T07:03:43.583 DEBUG:boto:Token: None 2022-09-03T07:03:43.584 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:43.584 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OJiTrz310GqUB8Hh++jFFaq4PDo= 2022-09-03T07:03:43.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OJiTrz310GqUB8Hh++jFFaq4PDo=', 'Content-Length': '0'} 2022-09-03T07:03:43.587 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007be4636aaee64541-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.588 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:03:43.588 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.589 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.589 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.590 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.591 DEBUG:boto:Method: GET 2022-09-03T07:03:43.591 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:43.592 DEBUG:boto:Data: 2022-09-03T07:03:43.592 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.593 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.594 DEBUG:boto:Params: {} 2022-09-03T07:03:43.594 DEBUG:boto:Token: None 2022-09-03T07:03:43.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:43.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VG0Pjly+Q2Xjxc/5O+QSwEhykDQ= 2022-09-03T07:03:43.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VG0Pjly+Q2Xjxc/5O+QSwEhykDQ=', 'Content-Length': '0'} 2022-09-03T07:03:43.600 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fb382d285d83959-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.601 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.603 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.604 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.605 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:43.606 DEBUG:boto:Method: GET 2022-09-03T07:03:43.607 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:43.608 DEBUG:boto:Data: 2022-09-03T07:03:43.609 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.610 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.611 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.611 DEBUG:boto:Params: {} 2022-09-03T07:03:43.612 DEBUG:boto:Token: None 2022-09-03T07:03:43.613 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:43.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VG0Pjly+Q2Xjxc/5O+QSwEhykDQ= 2022-09-03T07:03:43.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VG0Pjly+Q2Xjxc/5O+QSwEhykDQ=', 'Content-Length': '0'} 2022-09-03T07:03:43.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a68d23255dcacfb8-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.620 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.621 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.623 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:43.624 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:43.625 DEBUG:boto:Method: GET 2022-09-03T07:03:43.626 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:43.627 DEBUG:boto:Data: 2022-09-03T07:03:43.627 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.628 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.628 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.629 DEBUG:boto:Params: {} 2022-09-03T07:03:43.629 DEBUG:boto:Token: None 2022-09-03T07:03:43.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.637 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005360700ff339f0be-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.638 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.639 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.640 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.640 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:43.641 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:43.642 DEBUG:boto:Method: GET 2022-09-03T07:03:43.642 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:43.643 DEBUG:boto:Data: 2022-09-03T07:03:43.643 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.644 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.645 DEBUG:boto:Params: {} 2022-09-03T07:03:43.645 DEBUG:boto:Token: None 2022-09-03T07:03:43.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.647 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.650 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a82672ae47f3d75-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.696 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.698 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.699 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.700 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.701 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:43.702 DEBUG:boto:Data: 2022-09-03T07:03:43.703 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.705 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.706 DEBUG:boto:Params: {} 2022-09-03T07:03:43.707 DEBUG:boto:Token: None 2022-09-03T07:03:43.708 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e 2022-09-03T07:03:43.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S1ylKhJmoHJSH/vEyigij/nVo1o= 2022-09-03T07:03:43.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S1ylKhJmoHJSH/vEyigij/nVo1o=', 'Content-Length': '0'} 2022-09-03T07:03:43.713 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c73c0a9a50cb3b5d-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.715 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.716 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.717 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.727 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:43.728 DEBUG:boto:Data: 2022-09-03T07:03:43.729 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.730 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.731 DEBUG:boto:Params: {} 2022-09-03T07:03:43.732 DEBUG:boto:Token: None 2022-09-03T07:03:43.733 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e 2022-09-03T07:03:43.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S1ylKhJmoHJSH/vEyigij/nVo1o= 2022-09-03T07:03:43.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S1ylKhJmoHJSH/vEyigij/nVo1o=', 'Content-Length': '0'} 2022-09-03T07:03:43.738 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c6ff04ec054a520-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.739 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:03:43.740 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.741 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.742 DEBUG:boto:Method: GET 2022-09-03T07:03:43.743 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:43.743 DEBUG:boto:Data: 2022-09-03T07:03:43.744 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.746 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.747 DEBUG:boto:Params: {} 2022-09-03T07:03:43.749 DEBUG:boto:Token: None 2022-09-03T07:03:43.750 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e 2022-09-03T07:03:43.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VQvzP23OlO9HF52PYiQNQod6RRk= 2022-09-03T07:03:43.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VQvzP23OlO9HF52PYiQNQod6RRk=', 'Content-Length': '0'} 2022-09-03T07:03:43.755 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be4344016b413d48-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.756 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:43.757 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:43.759 DEBUG:boto:Method: GET 2022-09-03T07:03:43.760 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:43.761 DEBUG:boto:Data: 2022-09-03T07:03:43.763 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.764 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.765 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.765 DEBUG:boto:Params: {} 2022-09-03T07:03:43.766 DEBUG:boto:Token: None 2022-09-03T07:03:43.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/e 2022-09-03T07:03:43.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VQvzP23OlO9HF52PYiQNQod6RRk= 2022-09-03T07:03:43.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VQvzP23OlO9HF52PYiQNQod6RRk=', 'Content-Length': '0'} 2022-09-03T07:03:43.770 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000099abcda755eb229-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.772 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.773 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.774 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:43.775 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:43.776 DEBUG:boto:Method: GET 2022-09-03T07:03:43.777 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:43.778 DEBUG:boto:Data: 2022-09-03T07:03:43.779 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.780 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.781 DEBUG:boto:Params: {} 2022-09-03T07:03:43.782 DEBUG:boto:Token: None 2022-09-03T07:03:43.784 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.790 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a437972aea6f276a-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.834 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.836 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.837 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.838 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:43.838 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:43.839 DEBUG:boto:Method: GET 2022-09-03T07:03:43.840 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:43.840 DEBUG:boto:Data: 2022-09-03T07:03:43.841 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.843 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.843 DEBUG:boto:Params: {} 2022-09-03T07:03:43.845 DEBUG:boto:Token: None 2022-09-03T07:03:43.847 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.848 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.858 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006532836999daa90a-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.859 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.860 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:03:43.860 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:43.861 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:43.861 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.862 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.863 DEBUG:boto:Method: GET 2022-09-03T07:03:43.863 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:43.864 DEBUG:boto:Data: 2022-09-03T07:03:43.864 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.865 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.866 DEBUG:boto:Params: {} 2022-09-03T07:03:43.867 DEBUG:boto:Token: None 2022-09-03T07:03:43.868 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:43.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cXTdGcm6xP+RBemJ9ZfJo9U25d0= 2022-09-03T07:03:43.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cXTdGcm6xP+RBemJ9ZfJo9U25d0=', 'Content-Length': '0'} 2022-09-03T07:03:43.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8bbd3180353179d-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.874 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:43.875 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:43.875 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.876 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.876 DEBUG:boto:Method: GET 2022-09-03T07:03:43.877 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:43.878 DEBUG:boto:Data: 2022-09-03T07:03:43.878 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.880 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.881 DEBUG:boto:Params: {} 2022-09-03T07:03:43.881 DEBUG:boto:Token: None 2022-09-03T07:03:43.882 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:43.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cXTdGcm6xP+RBemJ9ZfJo9U25d0= 2022-09-03T07:03:43.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cXTdGcm6xP+RBemJ9ZfJo9U25d0=', 'Content-Length': '0'} 2022-09-03T07:03:43.892 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf87569fbe760df6-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.894 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:43.895 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:43.896 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.896 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.897 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.898 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:43.899 DEBUG:boto:Data: 2022-09-03T07:03:43.900 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.901 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.902 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.904 DEBUG:boto:Params: {} 2022-09-03T07:03:43.905 DEBUG:boto:Token: None 2022-09-03T07:03:43.906 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:43.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pivzcKuNgVKbGNAi5VqJndlhsZ8= 2022-09-03T07:03:43.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pivzcKuNgVKbGNAi5VqJndlhsZ8=', 'Content-Length': '0'} 2022-09-03T07:03:43.910 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081a2cc8547b97df5-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.914 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:43.915 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:43.916 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.916 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.918 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.919 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:43.920 DEBUG:boto:Data: 2022-09-03T07:03:43.920 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.921 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.923 DEBUG:boto:Params: {} 2022-09-03T07:03:43.923 DEBUG:boto:Token: None 2022-09-03T07:03:43.924 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:43.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pivzcKuNgVKbGNAi5VqJndlhsZ8= 2022-09-03T07:03:43.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pivzcKuNgVKbGNAi5VqJndlhsZ8=', 'Content-Length': '0'} 2022-09-03T07:03:43.930 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f2ea6296284acb9-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.931 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:03:43.932 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:43.934 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:43.935 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.936 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.937 DEBUG:boto:Method: GET 2022-09-03T07:03:43.938 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:43.939 DEBUG:boto:Data: 2022-09-03T07:03:43.940 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.941 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.942 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.943 DEBUG:boto:Params: {} 2022-09-03T07:03:43.943 DEBUG:boto:Token: None 2022-09-03T07:03:43.944 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:43.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cXTdGcm6xP+RBemJ9ZfJo9U25d0= 2022-09-03T07:03:43.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cXTdGcm6xP+RBemJ9ZfJo9U25d0=', 'Content-Length': '0'} 2022-09-03T07:03:43.949 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000485f5ed239182f88-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.951 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:43.951 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:43.952 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.953 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:43.954 DEBUG:boto:Method: GET 2022-09-03T07:03:43.955 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:43.956 DEBUG:boto:Data: 2022-09-03T07:03:43.957 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.958 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.959 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.960 DEBUG:boto:Params: {} 2022-09-03T07:03:43.961 DEBUG:boto:Token: None 2022-09-03T07:03:43.961 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:43.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cXTdGcm6xP+RBemJ9ZfJo9U25d0= 2022-09-03T07:03:43.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cXTdGcm6xP+RBemJ9ZfJo9U25d0=', 'Content-Length': '0'} 2022-09-03T07:03:43.965 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf6b9a0664eeaaea-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.967 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.968 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.968 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:43.969 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:43.970 DEBUG:boto:Method: GET 2022-09-03T07:03:43.971 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:43.971 DEBUG:boto:Data: 2022-09-03T07:03:43.972 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.973 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.973 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.974 DEBUG:boto:Params: {} 2022-09-03T07:03:43.974 DEBUG:boto:Token: None 2022-09-03T07:03:43.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.980 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c6e3e6681ccb2e8f-006312fc4f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.981 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.982 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:43.983 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:43.983 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:43.984 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:43.984 DEBUG:boto:Method: GET 2022-09-03T07:03:43.985 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:43.986 DEBUG:boto:Data: 2022-09-03T07:03:43.986 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.987 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:43.987 DEBUG:boto:Port: 8000 2022-09-03T07:03:43.988 DEBUG:boto:Params: {} 2022-09-03T07:03:43.988 DEBUG:boto:Token: None 2022-09-03T07:03:43.989 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:43 GMT /avysmw-73/ 2022-09-03T07:03:43.990 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs= 2022-09-03T07:03:43.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uIu6QCZSUN96AABIhPS3Tj11fBs=', 'Content-Length': '0'} 2022-09-03T07:03:43.993 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f8d9e29c6df56821-006312fc4f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:43.993 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:43.995 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:43.995 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:43.996 DEBUG:boto:Method: HEAD 2022-09-03T07:03:43.996 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:43.997 DEBUG:boto:Data: 2022-09-03T07:03:43.997 DEBUG:boto:Headers: {} 2022-09-03T07:03:43.998 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:43.999 DEBUG:boto:Port: 8001 2022-09-03T07:03:43.999 DEBUG:boto:Params: {} 2022-09-03T07:03:44.000 DEBUG:boto:Token: None 2022-09-03T07:03:44.001 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/f 2022-09-03T07:03:44.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F0stekDsrdToor+ot2cf0BLyIxY= 2022-09-03T07:03:44.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F0stekDsrdToor+ot2cf0BLyIxY=', 'Content-Length': '0'} 2022-09-03T07:03:44.004 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef89f7c657affab8-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.005 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:44.006 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:44.007 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.008 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:44.009 DEBUG:boto:Data: 2022-09-03T07:03:44.010 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.012 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.013 DEBUG:boto:Params: {} 2022-09-03T07:03:44.014 DEBUG:boto:Token: None 2022-09-03T07:03:44.015 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/f 2022-09-03T07:03:44.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F0stekDsrdToor+ot2cf0BLyIxY= 2022-09-03T07:03:44.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F0stekDsrdToor+ot2cf0BLyIxY=', 'Content-Length': '0'} 2022-09-03T07:03:44.019 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cdb7757a58290869-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.020 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:03:44.021 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:44.022 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:44.023 DEBUG:boto:Method: GET 2022-09-03T07:03:44.024 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:44.025 DEBUG:boto:Data: 2022-09-03T07:03:44.027 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.027 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.028 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.029 DEBUG:boto:Params: {} 2022-09-03T07:03:44.030 DEBUG:boto:Token: None 2022-09-03T07:03:44.031 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/f 2022-09-03T07:03:44.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aSUaWowOQ3EGInTAp1gJ6ABoRZY= 2022-09-03T07:03:44.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aSUaWowOQ3EGInTAp1gJ6ABoRZY=', 'Content-Length': '0'} 2022-09-03T07:03:44.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe352f31b28a1157-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.039 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:44.040 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:44.041 DEBUG:boto:Method: GET 2022-09-03T07:03:44.042 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:44.043 DEBUG:boto:Data: 2022-09-03T07:03:44.044 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.046 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.047 DEBUG:boto:Params: {} 2022-09-03T07:03:44.048 DEBUG:boto:Token: None 2022-09-03T07:03:44.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/f 2022-09-03T07:03:44.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aSUaWowOQ3EGInTAp1gJ6ABoRZY= 2022-09-03T07:03:44.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aSUaWowOQ3EGInTAp1gJ6ABoRZY=', 'Content-Length': '0'} 2022-09-03T07:03:44.054 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000678517cf07a98143-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.098 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.099 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.100 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:44.100 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:44.101 DEBUG:boto:Method: GET 2022-09-03T07:03:44.102 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:44.104 DEBUG:boto:Data: 2022-09-03T07:03:44.104 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.105 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.106 DEBUG:boto:Params: {} 2022-09-03T07:03:44.107 DEBUG:boto:Token: None 2022-09-03T07:03:44.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.113 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ef0c4405e05e8312-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.114 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.116 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.118 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.119 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:44.120 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:44.120 DEBUG:boto:Method: GET 2022-09-03T07:03:44.121 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:44.122 DEBUG:boto:Data: 2022-09-03T07:03:44.123 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.124 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.125 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.126 DEBUG:boto:Params: {} 2022-09-03T07:03:44.127 DEBUG:boto:Token: None 2022-09-03T07:03:44.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.135 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001930406e78eb86ed-006312fc50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.137 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.140 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:03:44.141 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.141 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.142 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.143 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.144 DEBUG:boto:Method: GET 2022-09-03T07:03:44.145 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:44.146 DEBUG:boto:Data: 2022-09-03T07:03:44.147 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.148 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.149 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.150 DEBUG:boto:Params: {} 2022-09-03T07:03:44.151 DEBUG:boto:Token: None 2022-09-03T07:03:44.152 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:44.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CLO9wMkdz2Fj/ZG++TSE7SM9RKs= 2022-09-03T07:03:44.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CLO9wMkdz2Fj/ZG++TSE7SM9RKs=', 'Content-Length': '0'} 2022-09-03T07:03:44.158 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3361acc7d5e3038-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.160 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.161 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.161 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.162 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.163 DEBUG:boto:Method: GET 2022-09-03T07:03:44.164 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:44.165 DEBUG:boto:Data: 2022-09-03T07:03:44.166 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.169 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.171 DEBUG:boto:Params: {} 2022-09-03T07:03:44.174 DEBUG:boto:Token: None 2022-09-03T07:03:44.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:44.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CLO9wMkdz2Fj/ZG++TSE7SM9RKs= 2022-09-03T07:03:44.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CLO9wMkdz2Fj/ZG++TSE7SM9RKs=', 'Content-Length': '0'} 2022-09-03T07:03:44.181 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ccd108b62bedc13-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.182 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.183 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.184 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.184 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.186 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.190 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:44.190 DEBUG:boto:Data: 2022-09-03T07:03:44.191 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.193 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.196 DEBUG:boto:Params: {} 2022-09-03T07:03:44.197 DEBUG:boto:Token: None 2022-09-03T07:03:44.199 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:44.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:il3MPcZOdncsAAY04KOtk+lWfO8= 2022-09-03T07:03:44.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:il3MPcZOdncsAAY04KOtk+lWfO8=', 'Content-Length': '0'} 2022-09-03T07:03:44.203 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064fa502ac7629b9a-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.204 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.205 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.206 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.209 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.210 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.211 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:44.212 DEBUG:boto:Data: 2022-09-03T07:03:44.214 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.218 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.221 DEBUG:boto:Params: {} 2022-09-03T07:03:44.223 DEBUG:boto:Token: None 2022-09-03T07:03:44.224 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:44.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:il3MPcZOdncsAAY04KOtk+lWfO8= 2022-09-03T07:03:44.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:il3MPcZOdncsAAY04KOtk+lWfO8=', 'Content-Length': '0'} 2022-09-03T07:03:44.235 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8577612cab748e4-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.238 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:03:44.238 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.240 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.241 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.241 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.242 DEBUG:boto:Method: GET 2022-09-03T07:03:44.242 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:44.243 DEBUG:boto:Data: 2022-09-03T07:03:44.243 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.244 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.245 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.245 DEBUG:boto:Params: {} 2022-09-03T07:03:44.246 DEBUG:boto:Token: None 2022-09-03T07:03:44.246 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:44.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CLO9wMkdz2Fj/ZG++TSE7SM9RKs= 2022-09-03T07:03:44.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CLO9wMkdz2Fj/ZG++TSE7SM9RKs=', 'Content-Length': '0'} 2022-09-03T07:03:44.250 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7cdd20ef40035a7-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.251 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.251 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.252 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.253 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:44.253 DEBUG:boto:Method: GET 2022-09-03T07:03:44.254 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:44.255 DEBUG:boto:Data: 2022-09-03T07:03:44.255 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.256 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.256 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.257 DEBUG:boto:Params: {} 2022-09-03T07:03:44.258 DEBUG:boto:Token: None 2022-09-03T07:03:44.258 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:44.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CLO9wMkdz2Fj/ZG++TSE7SM9RKs= 2022-09-03T07:03:44.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CLO9wMkdz2Fj/ZG++TSE7SM9RKs=', 'Content-Length': '0'} 2022-09-03T07:03:44.262 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001802aeeed04b385d-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.263 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.264 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.265 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:44.265 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:44.266 DEBUG:boto:Method: GET 2022-09-03T07:03:44.267 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:44.268 DEBUG:boto:Data: 2022-09-03T07:03:44.269 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.270 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.271 DEBUG:boto:Params: {} 2022-09-03T07:03:44.271 DEBUG:boto:Token: None 2022-09-03T07:03:44.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.274 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.278 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000083a2c73e5904a47c-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.320 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.321 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.323 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.324 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:44.324 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:44.325 DEBUG:boto:Method: GET 2022-09-03T07:03:44.326 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:44.327 DEBUG:boto:Data: 2022-09-03T07:03:44.328 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.329 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.330 DEBUG:boto:Params: {} 2022-09-03T07:03:44.331 DEBUG:boto:Token: None 2022-09-03T07:03:44.331 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.332 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.335 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b1dc6efeb68714b1-006312fc50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.336 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.336 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.337 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.337 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.338 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:44.338 DEBUG:boto:Data: 2022-09-03T07:03:44.339 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.340 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.341 DEBUG:boto:Params: {} 2022-09-03T07:03:44.342 DEBUG:boto:Token: None 2022-09-03T07:03:44.343 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g 2022-09-03T07:03:44.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dP8IiG6w6QSBIgv17B1kSagjnbw= 2022-09-03T07:03:44.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dP8IiG6w6QSBIgv17B1kSagjnbw=', 'Content-Length': '0'} 2022-09-03T07:03:44.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f610135fda938848-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.357 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.358 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.359 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.360 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:44.360 DEBUG:boto:Data: 2022-09-03T07:03:44.361 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.361 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.362 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.362 DEBUG:boto:Params: {} 2022-09-03T07:03:44.363 DEBUG:boto:Token: None 2022-09-03T07:03:44.364 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g 2022-09-03T07:03:44.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dP8IiG6w6QSBIgv17B1kSagjnbw= 2022-09-03T07:03:44.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dP8IiG6w6QSBIgv17B1kSagjnbw=', 'Content-Length': '0'} 2022-09-03T07:03:44.368 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dd7c18ef86a53767-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.370 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:03:44.374 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.375 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.376 DEBUG:boto:Method: GET 2022-09-03T07:03:44.377 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:44.378 DEBUG:boto:Data: 2022-09-03T07:03:44.389 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.389 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.390 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.391 DEBUG:boto:Params: {} 2022-09-03T07:03:44.392 DEBUG:boto:Token: None 2022-09-03T07:03:44.393 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g 2022-09-03T07:03:44.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:um3F/EoRVbSkk7zUi/3HGuc76rE= 2022-09-03T07:03:44.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:um3F/EoRVbSkk7zUi/3HGuc76rE=', 'Content-Length': '0'} 2022-09-03T07:03:44.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c0358332f1aec4e-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.398 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:44.399 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:44.400 DEBUG:boto:Method: GET 2022-09-03T07:03:44.400 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:44.401 DEBUG:boto:Data: 2022-09-03T07:03:44.401 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.403 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.403 DEBUG:boto:Params: {} 2022-09-03T07:03:44.404 DEBUG:boto:Token: None 2022-09-03T07:03:44.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/g 2022-09-03T07:03:44.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:um3F/EoRVbSkk7zUi/3HGuc76rE= 2022-09-03T07:03:44.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:um3F/EoRVbSkk7zUi/3HGuc76rE=', 'Content-Length': '0'} 2022-09-03T07:03:44.410 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002b1019f5aff195e-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.450 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.451 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.452 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:44.452 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:44.453 DEBUG:boto:Method: GET 2022-09-03T07:03:44.453 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:44.454 DEBUG:boto:Data: 2022-09-03T07:03:44.455 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.457 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.458 DEBUG:boto:Params: {} 2022-09-03T07:03:44.459 DEBUG:boto:Token: None 2022-09-03T07:03:44.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.466 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cef58feffba6d026-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.467 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.468 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.469 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.469 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:44.470 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:44.470 DEBUG:boto:Method: GET 2022-09-03T07:03:44.471 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:44.473 DEBUG:boto:Data: 2022-09-03T07:03:44.474 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.476 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.478 DEBUG:boto:Params: {} 2022-09-03T07:03:44.479 DEBUG:boto:Token: None 2022-09-03T07:03:44.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.486 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000562733cdf6d8c283-006312fc50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.487 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.489 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:03:44.490 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.491 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.492 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.493 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.494 DEBUG:boto:Method: GET 2022-09-03T07:03:44.495 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:44.496 DEBUG:boto:Data: 2022-09-03T07:03:44.497 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.498 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.499 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.500 DEBUG:boto:Params: {} 2022-09-03T07:03:44.501 DEBUG:boto:Token: None 2022-09-03T07:03:44.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:44.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9T2OyAROn9wRHH6E3sJNXp5bj6k= 2022-09-03T07:03:44.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9T2OyAROn9wRHH6E3sJNXp5bj6k=', 'Content-Length': '0'} 2022-09-03T07:03:44.508 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007451aae56a109665-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.509 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.510 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.511 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.512 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.513 DEBUG:boto:Method: GET 2022-09-03T07:03:44.513 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:44.514 DEBUG:boto:Data: 2022-09-03T07:03:44.514 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.515 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.516 DEBUG:boto:Params: {} 2022-09-03T07:03:44.516 DEBUG:boto:Token: None 2022-09-03T07:03:44.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:44.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9T2OyAROn9wRHH6E3sJNXp5bj6k= 2022-09-03T07:03:44.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9T2OyAROn9wRHH6E3sJNXp5bj6k=', 'Content-Length': '0'} 2022-09-03T07:03:44.521 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7c466629f2dd040-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.522 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.523 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.523 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.524 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.524 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.525 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:44.526 DEBUG:boto:Data: 2022-09-03T07:03:44.527 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.527 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.528 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.529 DEBUG:boto:Params: {} 2022-09-03T07:03:44.529 DEBUG:boto:Token: None 2022-09-03T07:03:44.530 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:44.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PY/zWrOSYPazLdYK+Ewgmzz2EoQ= 2022-09-03T07:03:44.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PY/zWrOSYPazLdYK+Ewgmzz2EoQ=', 'Content-Length': '0'} 2022-09-03T07:03:44.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002010664ba916d882-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.534 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.535 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.535 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.536 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.537 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.537 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:44.538 DEBUG:boto:Data: 2022-09-03T07:03:44.538 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.539 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.539 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.540 DEBUG:boto:Params: {} 2022-09-03T07:03:44.541 DEBUG:boto:Token: None 2022-09-03T07:03:44.541 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:44.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PY/zWrOSYPazLdYK+Ewgmzz2EoQ= 2022-09-03T07:03:44.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PY/zWrOSYPazLdYK+Ewgmzz2EoQ=', 'Content-Length': '0'} 2022-09-03T07:03:44.545 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1f3a120fbe329d9-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.546 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:03:44.547 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.547 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.548 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.548 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.549 DEBUG:boto:Method: GET 2022-09-03T07:03:44.549 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:44.550 DEBUG:boto:Data: 2022-09-03T07:03:44.550 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.552 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.552 DEBUG:boto:Params: {} 2022-09-03T07:03:44.553 DEBUG:boto:Token: None 2022-09-03T07:03:44.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:44.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9T2OyAROn9wRHH6E3sJNXp5bj6k= 2022-09-03T07:03:44.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9T2OyAROn9wRHH6E3sJNXp5bj6k=', 'Content-Length': '0'} 2022-09-03T07:03:44.557 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004baeee73e4979dc2-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.558 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.559 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.559 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.560 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:44.561 DEBUG:boto:Method: GET 2022-09-03T07:03:44.561 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:44.562 DEBUG:boto:Data: 2022-09-03T07:03:44.562 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.563 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.564 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.564 DEBUG:boto:Params: {} 2022-09-03T07:03:44.565 DEBUG:boto:Token: None 2022-09-03T07:03:44.566 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:44.566 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9T2OyAROn9wRHH6E3sJNXp5bj6k= 2022-09-03T07:03:44.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9T2OyAROn9wRHH6E3sJNXp5bj6k=', 'Content-Length': '0'} 2022-09-03T07:03:44.570 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000487c47a00c94e532-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.571 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.572 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.572 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:44.573 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:44.573 DEBUG:boto:Method: GET 2022-09-03T07:03:44.574 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:44.575 DEBUG:boto:Data: 2022-09-03T07:03:44.576 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.578 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.579 DEBUG:boto:Params: {} 2022-09-03T07:03:44.580 DEBUG:boto:Token: None 2022-09-03T07:03:44.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.587 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fbdab1f9364df318-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.588 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.591 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.592 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.592 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:44.593 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:44.594 DEBUG:boto:Method: GET 2022-09-03T07:03:44.594 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:44.595 DEBUG:boto:Data: 2022-09-03T07:03:44.596 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.596 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.597 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.598 DEBUG:boto:Params: {} 2022-09-03T07:03:44.598 DEBUG:boto:Token: None 2022-09-03T07:03:44.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.607 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c5cb5b4a3c829cc3-006312fc50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.608 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.609 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.610 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.611 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.612 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:44.613 DEBUG:boto:Data: 2022-09-03T07:03:44.614 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.615 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.616 DEBUG:boto:Params: {} 2022-09-03T07:03:44.617 DEBUG:boto:Token: None 2022-09-03T07:03:44.618 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h 2022-09-03T07:03:44.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iHKF/XjciKhUErSl4n0Y4yLu6mM= 2022-09-03T07:03:44.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iHKF/XjciKhUErSl4n0Y4yLu6mM=', 'Content-Length': '0'} 2022-09-03T07:03:44.623 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000477bbf28fc988cc5-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.624 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.625 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.627 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.629 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:44.629 DEBUG:boto:Data: 2022-09-03T07:03:44.630 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.637 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.638 DEBUG:boto:Params: {} 2022-09-03T07:03:44.639 DEBUG:boto:Token: None 2022-09-03T07:03:44.640 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h 2022-09-03T07:03:44.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iHKF/XjciKhUErSl4n0Y4yLu6mM= 2022-09-03T07:03:44.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iHKF/XjciKhUErSl4n0Y4yLu6mM=', 'Content-Length': '0'} 2022-09-03T07:03:44.645 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078de7b9749900484-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.646 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:03:44.647 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.648 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.649 DEBUG:boto:Method: GET 2022-09-03T07:03:44.650 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:44.651 DEBUG:boto:Data: 2022-09-03T07:03:44.652 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.653 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.654 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.655 DEBUG:boto:Params: {} 2022-09-03T07:03:44.658 DEBUG:boto:Token: None 2022-09-03T07:03:44.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h 2022-09-03T07:03:44.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mr+IDrGxOrGDLg2yLW+iCaQmn/E= 2022-09-03T07:03:44.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mr+IDrGxOrGDLg2yLW+iCaQmn/E=', 'Content-Length': '0'} 2022-09-03T07:03:44.667 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3aa5534b8d9d7dd-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.672 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:44.674 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:44.675 DEBUG:boto:Method: GET 2022-09-03T07:03:44.676 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:44.679 DEBUG:boto:Data: 2022-09-03T07:03:44.680 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.681 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.682 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.683 DEBUG:boto:Params: {} 2022-09-03T07:03:44.684 DEBUG:boto:Token: None 2022-09-03T07:03:44.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/h 2022-09-03T07:03:44.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mr+IDrGxOrGDLg2yLW+iCaQmn/E= 2022-09-03T07:03:44.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mr+IDrGxOrGDLg2yLW+iCaQmn/E=', 'Content-Length': '0'} 2022-09-03T07:03:44.696 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075f0efa7e249ffc8-006312fc50-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.698 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.698 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.699 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:44.700 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:44.701 DEBUG:boto:Method: GET 2022-09-03T07:03:44.701 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:44.702 DEBUG:boto:Data: 2022-09-03T07:03:44.703 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.704 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.705 DEBUG:boto:Params: {} 2022-09-03T07:03:44.705 DEBUG:boto:Token: None 2022-09-03T07:03:44.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.711 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093fe9f0b12fefd86-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.754 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.755 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.756 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.756 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:44.757 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:44.757 DEBUG:boto:Method: GET 2022-09-03T07:03:44.758 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:44.759 DEBUG:boto:Data: 2022-09-03T07:03:44.759 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.760 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:44.760 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.761 DEBUG:boto:Params: {} 2022-09-03T07:03:44.761 DEBUG:boto:Token: None 2022-09-03T07:03:44.762 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.763 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4= 2022-09-03T07:03:44.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4YFVFnyROrGdRKxGS9b89vNWNZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.767 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045259894b9f365f2-006312fc50-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.768 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.770 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-73 zones={test-zone3, test-zone1} 2022-09-03T07:03:44.770 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-73 zones={test-zone3, test-zone2} 2022-09-03T07:03:44.771 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.772 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.773 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.774 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:03:44.775 DEBUG:boto:Data: 2022-09-03T07:03:44.776 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.776 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.778 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.779 DEBUG:boto:Params: {} 2022-09-03T07:03:44.780 DEBUG:boto:Token: None 2022-09-03T07:03:44.780 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRBRsdzcSqQjX1oRS4ZWA1+9/e4= 2022-09-03T07:03:44.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRBRsdzcSqQjX1oRS4ZWA1+9/e4=', 'Content-Length': '0'} 2022-09-03T07:03:44.786 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c6f49d3d2f2f0732-006312fc50-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.787 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.788 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.789 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.790 DEBUG:boto:Path: /avysmw-73/ 2022-09-03T07:03:44.791 DEBUG:boto:Data: 2022-09-03T07:03:44.792 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.793 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:44.794 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.795 DEBUG:boto:Params: {} 2022-09-03T07:03:44.796 DEBUG:boto:Token: None 2022-09-03T07:03:44.796 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/ 2022-09-03T07:03:44.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TRBRsdzcSqQjX1oRS4ZWA1+9/e4= 2022-09-03T07:03:44.798 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TRBRsdzcSqQjX1oRS4ZWA1+9/e4=', 'Content-Length': '0'} 2022-09-03T07:03:44.816 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e5a32688dde4baa3-006312fc50-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.817 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:03:44.818 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.819 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.820 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.821 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.822 DEBUG:boto:Method: GET 2022-09-03T07:03:44.823 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.824 DEBUG:boto:Data: 2022-09-03T07:03:44.825 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.827 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.828 DEBUG:boto:Params: {} 2022-09-03T07:03:44.829 DEBUG:boto:Token: None 2022-09-03T07:03:44.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/?versions 2022-09-03T07:03:44.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mQcku35mPdGTH0EPRyFhoN0kPeM= 2022-09-03T07:03:44.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mQcku35mPdGTH0EPRyFhoN0kPeM=', 'Content-Length': '0'} 2022-09-03T07:03:44.839 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000047d3839bff7e2f5d-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.840 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.842 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:03:44.843 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:03:44.844 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:03:44.845 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:03:44.846 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:03:44.847 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:03:44.847 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:03:44.848 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:03:44.849 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:03:44.850 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.851 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.851 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.852 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.852 DEBUG:boto:Method: GET 2022-09-03T07:03:44.853 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.854 DEBUG:boto:Data: 2022-09-03T07:03:44.854 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.855 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:44.856 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.857 DEBUG:boto:Params: {} 2022-09-03T07:03:44.858 DEBUG:boto:Token: None 2022-09-03T07:03:44.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/?versions 2022-09-03T07:03:44.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mQcku35mPdGTH0EPRyFhoN0kPeM= 2022-09-03T07:03:44.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mQcku35mPdGTH0EPRyFhoN0kPeM=', 'Content-Length': '0'} 2022-09-03T07:03:44.865 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf8d346b40a74d4f-006312fc50-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.866 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.868 DEBUG:tasks.rgw_multi.zone_rados:o=a 2022-09-03T07:03:44.869 DEBUG:tasks.rgw_multi.zone_rados:o=b 2022-09-03T07:03:44.871 DEBUG:tasks.rgw_multi.zone_rados:o=c 2022-09-03T07:03:44.871 DEBUG:tasks.rgw_multi.zone_rados:o=d 2022-09-03T07:03:44.872 DEBUG:tasks.rgw_multi.zone_rados:o=e 2022-09-03T07:03:44.873 DEBUG:tasks.rgw_multi.zone_rados:o=f 2022-09-03T07:03:44.874 DEBUG:tasks.rgw_multi.zone_rados:o=g 2022-09-03T07:03:44.876 DEBUG:tasks.rgw_multi.zone_rados:o=h 2022-09-03T07:03:44.878 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.879 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.879 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.880 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.880 DEBUG:boto:Method: GET 2022-09-03T07:03:44.881 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.882 DEBUG:boto:Data: 2022-09-03T07:03:44.882 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.884 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.884 DEBUG:boto:Params: {} 2022-09-03T07:03:44.885 DEBUG:boto:Token: None 2022-09-03T07:03:44.886 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/?versions 2022-09-03T07:03:44.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mQcku35mPdGTH0EPRyFhoN0kPeM= 2022-09-03T07:03:44.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mQcku35mPdGTH0EPRyFhoN0kPeM=', 'Content-Length': '0'} 2022-09-03T07:03:44.891 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f88c11ec05acc34c-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.892 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.893 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:44.894 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:44.894 DEBUG:boto:path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.895 DEBUG:boto:auth_path=/avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.896 DEBUG:boto:Method: GET 2022-09-03T07:03:44.897 DEBUG:boto:Path: /avysmw-73/?versions&max-keys=999 2022-09-03T07:03:44.898 DEBUG:boto:Data: 2022-09-03T07:03:44.899 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:44.900 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.902 DEBUG:boto:Params: {} 2022-09-03T07:03:44.904 DEBUG:boto:Token: None 2022-09-03T07:03:44.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/?versions 2022-09-03T07:03:44.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mQcku35mPdGTH0EPRyFhoN0kPeM= 2022-09-03T07:03:44.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mQcku35mPdGTH0EPRyFhoN0kPeM=', 'Content-Length': '0'} 2022-09-03T07:03:44.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000952c8c956ad38845-006312fc50-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.915 DEBUG:boto:b'avysmw-73999falseanulltrue2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalbnulltrue2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalcnulltrue2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormaldnulltrue2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalenulltrue2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalfnulltrue2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalgnulltrue2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalhnulltrue2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:44.917 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:03:44.918 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:44.919 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:44.919 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.920 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.922 DEBUG:boto:Method: GET 2022-09-03T07:03:44.923 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:44.924 DEBUG:boto:Data: 2022-09-03T07:03:44.926 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.928 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.930 DEBUG:boto:Params: {} 2022-09-03T07:03:44.932 DEBUG:boto:Token: None 2022-09-03T07:03:44.934 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:44.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bpc13nBOscnYZOaF6tzbF9a4pOs= 2022-09-03T07:03:44.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bpc13nBOscnYZOaF6tzbF9a4pOs=', 'Content-Length': '0'} 2022-09-03T07:03:44.939 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009cdc0353ed4f5eb8-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.940 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:44.942 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:44.943 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.945 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.946 DEBUG:boto:Method: GET 2022-09-03T07:03:44.947 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:44.948 DEBUG:boto:Data: 2022-09-03T07:03:44.949 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.950 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:44.951 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.953 DEBUG:boto:Params: {} 2022-09-03T07:03:44.954 DEBUG:boto:Token: None 2022-09-03T07:03:44.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:44.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bpc13nBOscnYZOaF6tzbF9a4pOs= 2022-09-03T07:03:44.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bpc13nBOscnYZOaF6tzbF9a4pOs=', 'Content-Length': '0'} 2022-09-03T07:03:44.962 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a176adba673e46b-006312fc50-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.963 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:44.963 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:44.964 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.964 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.965 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.965 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:44.966 DEBUG:boto:Data: 2022-09-03T07:03:44.966 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.967 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.968 DEBUG:boto:Params: {} 2022-09-03T07:03:44.969 DEBUG:boto:Token: None 2022-09-03T07:03:44.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:44.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UV/zX8xrQUGNS78gmZ3fpxq7tZ4= 2022-09-03T07:03:44.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UV/zX8xrQUGNS78gmZ3fpxq7tZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.973 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac1ce77a8ad156a4-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.974 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:44.975 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:44.975 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.976 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.976 DEBUG:boto:Method: HEAD 2022-09-03T07:03:44.977 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:44.977 DEBUG:boto:Data: 2022-09-03T07:03:44.978 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.979 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:44.980 DEBUG:boto:Port: 8000 2022-09-03T07:03:44.980 DEBUG:boto:Params: {} 2022-09-03T07:03:44.981 DEBUG:boto:Token: None 2022-09-03T07:03:44.983 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:44.984 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UV/zX8xrQUGNS78gmZ3fpxq7tZ4= 2022-09-03T07:03:44.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UV/zX8xrQUGNS78gmZ3fpxq7tZ4=', 'Content-Length': '0'} 2022-09-03T07:03:44.987 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa37c5485e16f340-006312fc50-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:44.987 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:03:44.988 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:44.988 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:44.989 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.990 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:44.990 DEBUG:boto:Method: GET 2022-09-03T07:03:44.991 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:44.991 DEBUG:boto:Data: 2022-09-03T07:03:44.992 DEBUG:boto:Headers: {} 2022-09-03T07:03:44.992 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:44.993 DEBUG:boto:Port: 8001 2022-09-03T07:03:44.994 DEBUG:boto:Params: {} 2022-09-03T07:03:44.995 DEBUG:boto:Token: None 2022-09-03T07:03:44.996 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:44 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:44.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bpc13nBOscnYZOaF6tzbF9a4pOs= 2022-09-03T07:03:44.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bpc13nBOscnYZOaF6tzbF9a4pOs=', 'Content-Length': '0'} 2022-09-03T07:03:45.000 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ba8c3f31b1005f9-006312fc50-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.001 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:45.002 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:45.003 DEBUG:boto:path=/avysmw-73/a?versionId=null 2022-09-03T07:03:45.005 DEBUG:boto:auth_path=/avysmw-73/a?versionId=null 2022-09-03T07:03:45.006 DEBUG:boto:Method: GET 2022-09-03T07:03:45.007 DEBUG:boto:Path: /avysmw-73/a?versionId=null 2022-09-03T07:03:45.008 DEBUG:boto:Data: 2022-09-03T07:03:45.009 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.010 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.011 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.012 DEBUG:boto:Params: {} 2022-09-03T07:03:45.013 DEBUG:boto:Token: None 2022-09-03T07:03:45.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/a?versionId=null 2022-09-03T07:03:45.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b25OSw0Eqo2Yo7N+CTrWDzcarDc= 2022-09-03T07:03:45.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b25OSw0Eqo2Yo7N+CTrWDzcarDc=', 'Content-Length': '0'} 2022-09-03T07:03:45.019 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a4c4ec6265eb581-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.021 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.022 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.023 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:45.023 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:45.024 DEBUG:boto:Method: GET 2022-09-03T07:03:45.024 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:45.025 DEBUG:boto:Data: 2022-09-03T07:03:45.025 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.027 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.027 DEBUG:boto:Params: {} 2022-09-03T07:03:45.028 DEBUG:boto:Token: None 2022-09-03T07:03:45.029 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.030 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.035 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e2c8b70ec3b51072-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.036 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.037 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.038 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.038 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:45.039 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:45.039 DEBUG:boto:Method: GET 2022-09-03T07:03:45.040 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:45.040 DEBUG:boto:Data: 2022-09-03T07:03:45.041 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.042 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.042 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.043 DEBUG:boto:Params: {} 2022-09-03T07:03:45.044 DEBUG:boto:Token: None 2022-09-03T07:03:45.045 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.050 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000033d51b6e88a431e7-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.051 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.052 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:45.052 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:45.053 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.053 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:45.054 DEBUG:boto:Data: 2022-09-03T07:03:45.055 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.057 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.058 DEBUG:boto:Params: {} 2022-09-03T07:03:45.059 DEBUG:boto:Token: None 2022-09-03T07:03:45.059 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/a 2022-09-03T07:03:45.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2n3IzyR3E0aqM5EG0ooAhV+UOKg= 2022-09-03T07:03:45.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2n3IzyR3E0aqM5EG0ooAhV+UOKg=', 'Content-Length': '0'} 2022-09-03T07:03:45.064 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d069cfc863ed0fb9-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.065 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:45.066 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:45.067 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.068 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:45.068 DEBUG:boto:Data: 2022-09-03T07:03:45.069 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.070 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.071 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.072 DEBUG:boto:Params: {} 2022-09-03T07:03:45.073 DEBUG:boto:Token: None 2022-09-03T07:03:45.074 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/a 2022-09-03T07:03:45.074 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2n3IzyR3E0aqM5EG0ooAhV+UOKg= 2022-09-03T07:03:45.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2n3IzyR3E0aqM5EG0ooAhV+UOKg=', 'Content-Length': '0'} 2022-09-03T07:03:45.077 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002d021ac51650440-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.078 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2022-09-03T07:03:45.078 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:45.079 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:45.080 DEBUG:boto:Method: GET 2022-09-03T07:03:45.080 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:45.082 DEBUG:boto:Data: 2022-09-03T07:03:45.083 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.084 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.085 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.085 DEBUG:boto:Params: {} 2022-09-03T07:03:45.086 DEBUG:boto:Token: None 2022-09-03T07:03:45.087 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/a 2022-09-03T07:03:45.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jezp/fmeHYTAL2FzlTfCxwjnlTg= 2022-09-03T07:03:45.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jezp/fmeHYTAL2FzlTfCxwjnlTg=', 'Content-Length': '0'} 2022-09-03T07:03:45.093 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000492a600f5568c2fa-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.138 DEBUG:boto:path=/avysmw-73/a 2022-09-03T07:03:45.139 DEBUG:boto:auth_path=/avysmw-73/a 2022-09-03T07:03:45.140 DEBUG:boto:Method: GET 2022-09-03T07:03:45.141 DEBUG:boto:Path: /avysmw-73/a 2022-09-03T07:03:45.142 DEBUG:boto:Data: 2022-09-03T07:03:45.143 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.144 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.145 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.145 DEBUG:boto:Params: {} 2022-09-03T07:03:45.147 DEBUG:boto:Token: None 2022-09-03T07:03:45.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/a 2022-09-03T07:03:45.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jezp/fmeHYTAL2FzlTfCxwjnlTg= 2022-09-03T07:03:45.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jezp/fmeHYTAL2FzlTfCxwjnlTg=', 'Content-Length': '0'} 2022-09-03T07:03:45.154 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b238952978b21e7e-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.155 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.156 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.157 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:45.159 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:45.160 DEBUG:boto:Method: GET 2022-09-03T07:03:45.161 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:45.161 DEBUG:boto:Data: 2022-09-03T07:03:45.162 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.163 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.164 DEBUG:boto:Params: {} 2022-09-03T07:03:45.165 DEBUG:boto:Token: None 2022-09-03T07:03:45.166 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.172 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000674d014d43ee7975-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.174 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.175 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.176 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.177 DEBUG:boto:path=/avysmw-73/?prefix=a 2022-09-03T07:03:45.178 DEBUG:boto:auth_path=/avysmw-73/?prefix=a 2022-09-03T07:03:45.178 DEBUG:boto:Method: GET 2022-09-03T07:03:45.179 DEBUG:boto:Path: /avysmw-73/?prefix=a 2022-09-03T07:03:45.180 DEBUG:boto:Data: 2022-09-03T07:03:45.181 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.182 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.183 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.184 DEBUG:boto:Params: {} 2022-09-03T07:03:45.185 DEBUG:boto:Token: None 2022-09-03T07:03:45.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.191 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008c04cb816d887319-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.192 DEBUG:boto:b'avysmw-73a1000falsea2022-09-03T07:00:53.624Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.193 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:03:45.193 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.194 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.195 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.195 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.196 DEBUG:boto:Method: GET 2022-09-03T07:03:45.196 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:45.197 DEBUG:boto:Data: 2022-09-03T07:03:45.198 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.199 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.200 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.201 DEBUG:boto:Params: {} 2022-09-03T07:03:45.202 DEBUG:boto:Token: None 2022-09-03T07:03:45.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:45.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+hiD6OsO8UmY/kIRyw2jxkr7MuI= 2022-09-03T07:03:45.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+hiD6OsO8UmY/kIRyw2jxkr7MuI=', 'Content-Length': '0'} 2022-09-03T07:03:45.208 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ebfe63fc64b6e0b1-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.209 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.210 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.211 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.211 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.212 DEBUG:boto:Method: GET 2022-09-03T07:03:45.212 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:45.213 DEBUG:boto:Data: 2022-09-03T07:03:45.214 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.215 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.216 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.217 DEBUG:boto:Params: {} 2022-09-03T07:03:45.218 DEBUG:boto:Token: None 2022-09-03T07:03:45.219 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:45.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+hiD6OsO8UmY/kIRyw2jxkr7MuI= 2022-09-03T07:03:45.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+hiD6OsO8UmY/kIRyw2jxkr7MuI=', 'Content-Length': '0'} 2022-09-03T07:03:45.224 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000477d230433187b14-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.270 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.271 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.271 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.272 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.272 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.273 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:45.274 DEBUG:boto:Data: 2022-09-03T07:03:45.274 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.275 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.276 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.277 DEBUG:boto:Params: {} 2022-09-03T07:03:45.278 DEBUG:boto:Token: None 2022-09-03T07:03:45.278 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:45.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUAQEx7u3BxW4DypKb5N2vdbFtU= 2022-09-03T07:03:45.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUAQEx7u3BxW4DypKb5N2vdbFtU=', 'Content-Length': '0'} 2022-09-03T07:03:45.284 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3fb41561912aaa9-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.285 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.286 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.286 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.287 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.287 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.288 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:45.288 DEBUG:boto:Data: 2022-09-03T07:03:45.289 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.289 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.290 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.290 DEBUG:boto:Params: {} 2022-09-03T07:03:45.291 DEBUG:boto:Token: None 2022-09-03T07:03:45.291 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:45.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eUAQEx7u3BxW4DypKb5N2vdbFtU= 2022-09-03T07:03:45.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eUAQEx7u3BxW4DypKb5N2vdbFtU=', 'Content-Length': '0'} 2022-09-03T07:03:45.302 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000052544ba0ef152b1-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.304 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:03:45.304 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.305 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.307 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.309 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.310 DEBUG:boto:Method: GET 2022-09-03T07:03:45.311 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:45.311 DEBUG:boto:Data: 2022-09-03T07:03:45.312 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.313 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.314 DEBUG:boto:Params: {} 2022-09-03T07:03:45.315 DEBUG:boto:Token: None 2022-09-03T07:03:45.315 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:45.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+hiD6OsO8UmY/kIRyw2jxkr7MuI= 2022-09-03T07:03:45.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+hiD6OsO8UmY/kIRyw2jxkr7MuI=', 'Content-Length': '0'} 2022-09-03T07:03:45.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7828419d55646f0-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.322 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.322 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.323 DEBUG:boto:path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.325 DEBUG:boto:auth_path=/avysmw-73/b?versionId=null 2022-09-03T07:03:45.326 DEBUG:boto:Method: GET 2022-09-03T07:03:45.327 DEBUG:boto:Path: /avysmw-73/b?versionId=null 2022-09-03T07:03:45.328 DEBUG:boto:Data: 2022-09-03T07:03:45.329 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.330 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.330 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.331 DEBUG:boto:Params: {} 2022-09-03T07:03:45.332 DEBUG:boto:Token: None 2022-09-03T07:03:45.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b?versionId=null 2022-09-03T07:03:45.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+hiD6OsO8UmY/kIRyw2jxkr7MuI= 2022-09-03T07:03:45.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+hiD6OsO8UmY/kIRyw2jxkr7MuI=', 'Content-Length': '0'} 2022-09-03T07:03:45.337 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039b562167fa09c7e-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.338 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.339 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.340 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:45.340 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:45.341 DEBUG:boto:Method: GET 2022-09-03T07:03:45.343 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:45.344 DEBUG:boto:Data: 2022-09-03T07:03:45.345 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.346 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.347 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.347 DEBUG:boto:Params: {} 2022-09-03T07:03:45.348 DEBUG:boto:Token: None 2022-09-03T07:03:45.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.354 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000060b0b9e66237d84-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.394 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.396 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.396 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.397 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:45.398 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:45.399 DEBUG:boto:Method: GET 2022-09-03T07:03:45.400 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:45.401 DEBUG:boto:Data: 2022-09-03T07:03:45.402 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.402 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.403 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.404 DEBUG:boto:Params: {} 2022-09-03T07:03:45.404 DEBUG:boto:Token: None 2022-09-03T07:03:45.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.409 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c18ead1375d21784-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.410 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.410 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.411 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.412 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.412 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:45.413 DEBUG:boto:Data: 2022-09-03T07:03:45.413 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.414 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.414 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.415 DEBUG:boto:Params: {} 2022-09-03T07:03:45.415 DEBUG:boto:Token: None 2022-09-03T07:03:45.416 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b 2022-09-03T07:03:45.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VbzwrB/ghY0zADeO/0tFYqghfyE= 2022-09-03T07:03:45.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VbzwrB/ghY0zADeO/0tFYqghfyE=', 'Content-Length': '0'} 2022-09-03T07:03:45.419 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3b3f00e45de3955-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.420 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.421 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.421 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.422 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:45.422 DEBUG:boto:Data: 2022-09-03T07:03:45.423 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.423 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.424 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.424 DEBUG:boto:Params: {} 2022-09-03T07:03:45.425 DEBUG:boto:Token: None 2022-09-03T07:03:45.425 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b 2022-09-03T07:03:45.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VbzwrB/ghY0zADeO/0tFYqghfyE= 2022-09-03T07:03:45.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VbzwrB/ghY0zADeO/0tFYqghfyE=', 'Content-Length': '0'} 2022-09-03T07:03:45.428 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056854a3fb78795d8-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.429 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2022-09-03T07:03:45.430 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.430 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.431 DEBUG:boto:Method: GET 2022-09-03T07:03:45.432 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:45.432 DEBUG:boto:Data: 2022-09-03T07:03:45.433 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.433 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.434 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.434 DEBUG:boto:Params: {} 2022-09-03T07:03:45.435 DEBUG:boto:Token: None 2022-09-03T07:03:45.435 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b 2022-09-03T07:03:45.436 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P7xtNGoyBklJBWqDA2nDkYPScSo= 2022-09-03T07:03:45.437 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P7xtNGoyBklJBWqDA2nDkYPScSo=', 'Content-Length': '0'} 2022-09-03T07:03:45.439 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f39c2cc0d60f7ece-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.440 DEBUG:boto:path=/avysmw-73/b 2022-09-03T07:03:45.441 DEBUG:boto:auth_path=/avysmw-73/b 2022-09-03T07:03:45.442 DEBUG:boto:Method: GET 2022-09-03T07:03:45.442 DEBUG:boto:Path: /avysmw-73/b 2022-09-03T07:03:45.443 DEBUG:boto:Data: 2022-09-03T07:03:45.443 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.444 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.444 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.445 DEBUG:boto:Params: {} 2022-09-03T07:03:45.445 DEBUG:boto:Token: None 2022-09-03T07:03:45.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/b 2022-09-03T07:03:45.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P7xtNGoyBklJBWqDA2nDkYPScSo= 2022-09-03T07:03:45.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P7xtNGoyBklJBWqDA2nDkYPScSo=', 'Content-Length': '0'} 2022-09-03T07:03:45.450 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce023c63318f816a-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.450 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.451 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.452 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:45.452 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:45.453 DEBUG:boto:Method: GET 2022-09-03T07:03:45.453 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:45.454 DEBUG:boto:Data: 2022-09-03T07:03:45.454 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.455 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.455 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.456 DEBUG:boto:Params: {} 2022-09-03T07:03:45.456 DEBUG:boto:Token: None 2022-09-03T07:03:45.457 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.461 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009da333e314bcd780-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.462 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.463 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.464 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.464 DEBUG:boto:path=/avysmw-73/?prefix=b 2022-09-03T07:03:45.465 DEBUG:boto:auth_path=/avysmw-73/?prefix=b 2022-09-03T07:03:45.465 DEBUG:boto:Method: GET 2022-09-03T07:03:45.466 DEBUG:boto:Path: /avysmw-73/?prefix=b 2022-09-03T07:03:45.466 DEBUG:boto:Data: 2022-09-03T07:03:45.467 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.467 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.468 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.468 DEBUG:boto:Params: {} 2022-09-03T07:03:45.469 DEBUG:boto:Token: None 2022-09-03T07:03:45.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.473 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4847262d4d42de9-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.474 DEBUG:boto:b'avysmw-73b1000falseb2022-09-03T07:00:53.656Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.475 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:03:45.475 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.476 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.476 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.477 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.477 DEBUG:boto:Method: GET 2022-09-03T07:03:45.478 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:45.478 DEBUG:boto:Data: 2022-09-03T07:03:45.479 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.480 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.480 DEBUG:boto:Params: {} 2022-09-03T07:03:45.481 DEBUG:boto:Token: None 2022-09-03T07:03:45.482 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:45.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Ay0E/fMf0gmQ7D25yt0RTCiJNw= 2022-09-03T07:03:45.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Ay0E/fMf0gmQ7D25yt0RTCiJNw=', 'Content-Length': '0'} 2022-09-03T07:03:45.485 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a05b18d0f80a8f70-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.486 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.487 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.487 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.488 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.488 DEBUG:boto:Method: GET 2022-09-03T07:03:45.489 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:45.489 DEBUG:boto:Data: 2022-09-03T07:03:45.490 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.490 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.491 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.491 DEBUG:boto:Params: {} 2022-09-03T07:03:45.492 DEBUG:boto:Token: None 2022-09-03T07:03:45.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:45.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Ay0E/fMf0gmQ7D25yt0RTCiJNw= 2022-09-03T07:03:45.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Ay0E/fMf0gmQ7D25yt0RTCiJNw=', 'Content-Length': '0'} 2022-09-03T07:03:45.497 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004c15a51d11c1695-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.498 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.499 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.500 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.501 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.501 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.502 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:45.502 DEBUG:boto:Data: 2022-09-03T07:03:45.503 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.504 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.505 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.506 DEBUG:boto:Params: {} 2022-09-03T07:03:45.506 DEBUG:boto:Token: None 2022-09-03T07:03:45.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:45.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dntm4SUids9rp5zkADrsPKoDoTA= 2022-09-03T07:03:45.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dntm4SUids9rp5zkADrsPKoDoTA=', 'Content-Length': '0'} 2022-09-03T07:03:45.512 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a7ae9ec7d3bcb55a-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.513 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.514 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.515 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.516 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.516 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.517 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:45.518 DEBUG:boto:Data: 2022-09-03T07:03:45.519 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.520 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.520 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.521 DEBUG:boto:Params: {} 2022-09-03T07:03:45.521 DEBUG:boto:Token: None 2022-09-03T07:03:45.522 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:45.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dntm4SUids9rp5zkADrsPKoDoTA= 2022-09-03T07:03:45.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dntm4SUids9rp5zkADrsPKoDoTA=', 'Content-Length': '0'} 2022-09-03T07:03:45.526 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015c751a1ca63d6d1-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.527 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:03:45.527 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.528 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.529 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.529 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.530 DEBUG:boto:Method: GET 2022-09-03T07:03:45.531 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:45.532 DEBUG:boto:Data: 2022-09-03T07:03:45.533 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.535 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.536 DEBUG:boto:Params: {} 2022-09-03T07:03:45.537 DEBUG:boto:Token: None 2022-09-03T07:03:45.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:45.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Ay0E/fMf0gmQ7D25yt0RTCiJNw= 2022-09-03T07:03:45.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Ay0E/fMf0gmQ7D25yt0RTCiJNw=', 'Content-Length': '0'} 2022-09-03T07:03:45.543 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a195de7de62f3bb5-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.544 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.545 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.546 DEBUG:boto:path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.546 DEBUG:boto:auth_path=/avysmw-73/c?versionId=null 2022-09-03T07:03:45.547 DEBUG:boto:Method: GET 2022-09-03T07:03:45.548 DEBUG:boto:Path: /avysmw-73/c?versionId=null 2022-09-03T07:03:45.557 DEBUG:boto:Data: 2022-09-03T07:03:45.558 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.559 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.559 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.560 DEBUG:boto:Params: {} 2022-09-03T07:03:45.560 DEBUG:boto:Token: None 2022-09-03T07:03:45.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c?versionId=null 2022-09-03T07:03:45.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Ay0E/fMf0gmQ7D25yt0RTCiJNw= 2022-09-03T07:03:45.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Ay0E/fMf0gmQ7D25yt0RTCiJNw=', 'Content-Length': '0'} 2022-09-03T07:03:45.566 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d4cb35867f24b723-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.570 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.571 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.571 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:45.572 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:45.572 DEBUG:boto:Method: GET 2022-09-03T07:03:45.573 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:45.573 DEBUG:boto:Data: 2022-09-03T07:03:45.574 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.574 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.575 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.576 DEBUG:boto:Params: {} 2022-09-03T07:03:45.576 DEBUG:boto:Token: None 2022-09-03T07:03:45.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.577 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.581 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001dbfe3f5cb1f2746-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.627 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.629 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.630 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.631 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:45.635 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:45.636 DEBUG:boto:Method: GET 2022-09-03T07:03:45.638 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:45.639 DEBUG:boto:Data: 2022-09-03T07:03:45.639 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.640 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.641 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.643 DEBUG:boto:Params: {} 2022-09-03T07:03:45.644 DEBUG:boto:Token: None 2022-09-03T07:03:45.647 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.655 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022b850dddaa95f6f-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.657 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.659 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.662 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.663 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.664 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:45.665 DEBUG:boto:Data: 2022-09-03T07:03:45.666 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.667 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.668 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.669 DEBUG:boto:Params: {} 2022-09-03T07:03:45.670 DEBUG:boto:Token: None 2022-09-03T07:03:45.671 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c 2022-09-03T07:03:45.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vRP0T+CP1HEELzxxxxSDsF8zUlk= 2022-09-03T07:03:45.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vRP0T+CP1HEELzxxxxSDsF8zUlk=', 'Content-Length': '0'} 2022-09-03T07:03:45.676 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b41f3560096aca6-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.677 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.678 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.679 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.680 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:45.681 DEBUG:boto:Data: 2022-09-03T07:03:45.682 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.683 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.684 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.685 DEBUG:boto:Params: {} 2022-09-03T07:03:45.686 DEBUG:boto:Token: None 2022-09-03T07:03:45.687 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c 2022-09-03T07:03:45.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vRP0T+CP1HEELzxxxxSDsF8zUlk= 2022-09-03T07:03:45.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vRP0T+CP1HEELzxxxxSDsF8zUlk=', 'Content-Length': '0'} 2022-09-03T07:03:45.693 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000051a3daa1f23f732a-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.695 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2022-09-03T07:03:45.696 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.697 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.698 DEBUG:boto:Method: GET 2022-09-03T07:03:45.699 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:45.699 DEBUG:boto:Data: 2022-09-03T07:03:45.700 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.702 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.703 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.704 DEBUG:boto:Params: {} 2022-09-03T07:03:45.705 DEBUG:boto:Token: None 2022-09-03T07:03:45.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c 2022-09-03T07:03:45.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GZkMtY+tX3Az7ya/1+MJKcPWNfU= 2022-09-03T07:03:45.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GZkMtY+tX3Az7ya/1+MJKcPWNfU=', 'Content-Length': '0'} 2022-09-03T07:03:45.712 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b98d4859eedfc388-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.717 DEBUG:boto:path=/avysmw-73/c 2022-09-03T07:03:45.718 DEBUG:boto:auth_path=/avysmw-73/c 2022-09-03T07:03:45.719 DEBUG:boto:Method: GET 2022-09-03T07:03:45.719 DEBUG:boto:Path: /avysmw-73/c 2022-09-03T07:03:45.720 DEBUG:boto:Data: 2022-09-03T07:03:45.721 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.721 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.722 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.723 DEBUG:boto:Params: {} 2022-09-03T07:03:45.723 DEBUG:boto:Token: None 2022-09-03T07:03:45.724 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/c 2022-09-03T07:03:45.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GZkMtY+tX3Az7ya/1+MJKcPWNfU= 2022-09-03T07:03:45.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GZkMtY+tX3Az7ya/1+MJKcPWNfU=', 'Content-Length': '0'} 2022-09-03T07:03:45.733 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000818fc472051cd2-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.735 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.737 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.739 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:45.740 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:45.742 DEBUG:boto:Method: GET 2022-09-03T07:03:45.743 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:45.743 DEBUG:boto:Data: 2022-09-03T07:03:45.745 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.749 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.749 DEBUG:boto:Params: {} 2022-09-03T07:03:45.750 DEBUG:boto:Token: None 2022-09-03T07:03:45.751 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.755 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003eeb1af6fec2f43b-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.756 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.757 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.757 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.758 DEBUG:boto:path=/avysmw-73/?prefix=c 2022-09-03T07:03:45.758 DEBUG:boto:auth_path=/avysmw-73/?prefix=c 2022-09-03T07:03:45.759 DEBUG:boto:Method: GET 2022-09-03T07:03:45.759 DEBUG:boto:Path: /avysmw-73/?prefix=c 2022-09-03T07:03:45.760 DEBUG:boto:Data: 2022-09-03T07:03:45.760 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.761 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.761 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.762 DEBUG:boto:Params: {} 2022-09-03T07:03:45.763 DEBUG:boto:Token: None 2022-09-03T07:03:45.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.767 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000016bf35a46bb90808-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.811 DEBUG:boto:b'avysmw-73c1000falsec2022-09-03T07:00:53.720Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.812 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:03:45.813 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:45.814 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:45.815 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.816 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.817 DEBUG:boto:Method: GET 2022-09-03T07:03:45.819 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:45.820 DEBUG:boto:Data: 2022-09-03T07:03:45.824 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.825 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.825 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.826 DEBUG:boto:Params: {} 2022-09-03T07:03:45.827 DEBUG:boto:Token: None 2022-09-03T07:03:45.827 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:45.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wCv3nCxIEi+c2heXjaItiuJn8wA= 2022-09-03T07:03:45.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wCv3nCxIEi+c2heXjaItiuJn8wA=', 'Content-Length': '0'} 2022-09-03T07:03:45.833 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f934175bb11129e-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.837 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:45.838 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:45.839 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.840 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.840 DEBUG:boto:Method: GET 2022-09-03T07:03:45.841 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:45.842 DEBUG:boto:Data: 2022-09-03T07:03:45.842 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.843 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.843 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.844 DEBUG:boto:Params: {} 2022-09-03T07:03:45.844 DEBUG:boto:Token: None 2022-09-03T07:03:45.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:45.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wCv3nCxIEi+c2heXjaItiuJn8wA= 2022-09-03T07:03:45.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wCv3nCxIEi+c2heXjaItiuJn8wA=', 'Content-Length': '0'} 2022-09-03T07:03:45.852 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9aebba3f68aeb3d-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.854 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:45.855 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:45.856 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.857 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.861 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.862 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:45.863 DEBUG:boto:Data: 2022-09-03T07:03:45.864 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.865 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.866 DEBUG:boto:Params: {} 2022-09-03T07:03:45.867 DEBUG:boto:Token: None 2022-09-03T07:03:45.869 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:45.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PXlT+04mehX3x1LxQGqvmVXXIA4= 2022-09-03T07:03:45.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PXlT+04mehX3x1LxQGqvmVXXIA4=', 'Content-Length': '0'} 2022-09-03T07:03:45.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083e3e499bddcf3b2-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.874 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:45.875 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:45.875 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.876 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.876 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.877 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:45.877 DEBUG:boto:Data: 2022-09-03T07:03:45.878 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.879 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.880 DEBUG:boto:Params: {} 2022-09-03T07:03:45.880 DEBUG:boto:Token: None 2022-09-03T07:03:45.881 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:45.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PXlT+04mehX3x1LxQGqvmVXXIA4= 2022-09-03T07:03:45.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PXlT+04mehX3x1LxQGqvmVXXIA4=', 'Content-Length': '0'} 2022-09-03T07:03:45.888 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ccc4bd7ddddd3a0-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.891 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:03:45.892 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:45.892 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:45.893 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.893 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.894 DEBUG:boto:Method: GET 2022-09-03T07:03:45.895 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:45.895 DEBUG:boto:Data: 2022-09-03T07:03:45.896 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.897 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.898 DEBUG:boto:Params: {} 2022-09-03T07:03:45.899 DEBUG:boto:Token: None 2022-09-03T07:03:45.899 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:45.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wCv3nCxIEi+c2heXjaItiuJn8wA= 2022-09-03T07:03:45.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wCv3nCxIEi+c2heXjaItiuJn8wA=', 'Content-Length': '0'} 2022-09-03T07:03:45.903 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003da6b70da1ddd695-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.904 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:45.905 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:45.906 DEBUG:boto:path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.907 DEBUG:boto:auth_path=/avysmw-73/d?versionId=null 2022-09-03T07:03:45.908 DEBUG:boto:Method: GET 2022-09-03T07:03:45.908 DEBUG:boto:Path: /avysmw-73/d?versionId=null 2022-09-03T07:03:45.909 DEBUG:boto:Data: 2022-09-03T07:03:45.909 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.910 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.911 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.912 DEBUG:boto:Params: {} 2022-09-03T07:03:45.913 DEBUG:boto:Token: None 2022-09-03T07:03:45.914 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/d?versionId=null 2022-09-03T07:03:45.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wCv3nCxIEi+c2heXjaItiuJn8wA= 2022-09-03T07:03:45.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wCv3nCxIEi+c2heXjaItiuJn8wA=', 'Content-Length': '0'} 2022-09-03T07:03:45.919 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c2acc596af339b0-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.964 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.965 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.966 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:45.967 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:45.967 DEBUG:boto:Method: GET 2022-09-03T07:03:45.968 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:45.968 DEBUG:boto:Data: 2022-09-03T07:03:45.969 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.969 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.970 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.970 DEBUG:boto:Params: {} 2022-09-03T07:03:45.971 DEBUG:boto:Token: None 2022-09-03T07:03:45.971 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.976 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017b53538149d6427-006312fc51-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.976 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.977 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:45.978 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:45.978 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:45.979 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:45.981 DEBUG:boto:Method: GET 2022-09-03T07:03:45.982 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:45.982 DEBUG:boto:Data: 2022-09-03T07:03:45.983 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.983 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:45.984 DEBUG:boto:Port: 8000 2022-09-03T07:03:45.985 DEBUG:boto:Params: {} 2022-09-03T07:03:45.985 DEBUG:boto:Token: None 2022-09-03T07:03:45.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/ 2022-09-03T07:03:45.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w= 2022-09-03T07:03:45.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPF4UaVccmHlj8Q3ai4NPo+Qh4w=', 'Content-Length': '0'} 2022-09-03T07:03:45.990 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cff060e79d15df32-006312fc51-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:45.991 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:45.992 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:45.992 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:45.993 DEBUG:boto:Method: HEAD 2022-09-03T07:03:45.993 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:45.994 DEBUG:boto:Data: 2022-09-03T07:03:45.995 DEBUG:boto:Headers: {} 2022-09-03T07:03:45.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:45.996 DEBUG:boto:Port: 8001 2022-09-03T07:03:45.996 DEBUG:boto:Params: {} 2022-09-03T07:03:45.997 DEBUG:boto:Token: None 2022-09-03T07:03:45.997 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:45 GMT /avysmw-73/d 2022-09-03T07:03:45.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NN/31AgAXVDBHaOn5R93zlR9EK8= 2022-09-03T07:03:45.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NN/31AgAXVDBHaOn5R93zlR9EK8=', 'Content-Length': '0'} 2022-09-03T07:03:46.005 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000bf8b8693c435747-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.006 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:46.006 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:46.007 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.008 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:46.008 DEBUG:boto:Data: 2022-09-03T07:03:46.009 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.009 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.010 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.011 DEBUG:boto:Params: {} 2022-09-03T07:03:46.012 DEBUG:boto:Token: None 2022-09-03T07:03:46.012 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/d 2022-09-03T07:03:46.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5c0czupIjYsccS9zA9KVtSyZMvk= 2022-09-03T07:03:46.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5c0czupIjYsccS9zA9KVtSyZMvk=', 'Content-Length': '0'} 2022-09-03T07:03:46.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001e91e1c57cb4bb3-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.017 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2022-09-03T07:03:46.017 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:46.018 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:46.018 DEBUG:boto:Method: GET 2022-09-03T07:03:46.019 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:46.020 DEBUG:boto:Data: 2022-09-03T07:03:46.020 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.021 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.022 DEBUG:boto:Params: {} 2022-09-03T07:03:46.022 DEBUG:boto:Token: None 2022-09-03T07:03:46.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/d 2022-09-03T07:03:46.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:70jM/e9aEOlX7wYDyjMvRwSO0Eo= 2022-09-03T07:03:46.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:70jM/e9aEOlX7wYDyjMvRwSO0Eo=', 'Content-Length': '0'} 2022-09-03T07:03:46.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001643507d61a5d543-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.074 DEBUG:boto:path=/avysmw-73/d 2022-09-03T07:03:46.075 DEBUG:boto:auth_path=/avysmw-73/d 2022-09-03T07:03:46.076 DEBUG:boto:Method: GET 2022-09-03T07:03:46.077 DEBUG:boto:Path: /avysmw-73/d 2022-09-03T07:03:46.079 DEBUG:boto:Data: 2022-09-03T07:03:46.084 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.085 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.086 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.087 DEBUG:boto:Params: {} 2022-09-03T07:03:46.088 DEBUG:boto:Token: None 2022-09-03T07:03:46.089 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/d 2022-09-03T07:03:46.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:70jM/e9aEOlX7wYDyjMvRwSO0Eo= 2022-09-03T07:03:46.092 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:70jM/e9aEOlX7wYDyjMvRwSO0Eo=', 'Content-Length': '0'} 2022-09-03T07:03:46.094 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c93d6f3d9bb1ff2f-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.095 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.096 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.097 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:46.097 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:46.098 DEBUG:boto:Method: GET 2022-09-03T07:03:46.099 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:46.099 DEBUG:boto:Data: 2022-09-03T07:03:46.100 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.101 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.102 DEBUG:boto:Params: {} 2022-09-03T07:03:46.102 DEBUG:boto:Token: None 2022-09-03T07:03:46.103 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.108 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e7cb9747af471110-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.110 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.112 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.112 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.113 DEBUG:boto:path=/avysmw-73/?prefix=d 2022-09-03T07:03:46.114 DEBUG:boto:auth_path=/avysmw-73/?prefix=d 2022-09-03T07:03:46.114 DEBUG:boto:Method: GET 2022-09-03T07:03:46.115 DEBUG:boto:Path: /avysmw-73/?prefix=d 2022-09-03T07:03:46.115 DEBUG:boto:Data: 2022-09-03T07:03:46.116 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.118 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.118 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.119 DEBUG:boto:Params: {} 2022-09-03T07:03:46.120 DEBUG:boto:Token: None 2022-09-03T07:03:46.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.124 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000066163f92e4f992b-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.167 DEBUG:boto:b'avysmw-73d1000falsed2022-09-03T07:00:53.745Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.170 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:03:46.171 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.173 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.174 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.175 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.176 DEBUG:boto:Method: GET 2022-09-03T07:03:46.177 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:46.178 DEBUG:boto:Data: 2022-09-03T07:03:46.179 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.180 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.180 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.181 DEBUG:boto:Params: {} 2022-09-03T07:03:46.182 DEBUG:boto:Token: None 2022-09-03T07:03:46.183 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:46.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZZMqd+Eow5x/s0lTsj3YCogJkcg= 2022-09-03T07:03:46.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZZMqd+Eow5x/s0lTsj3YCogJkcg=', 'Content-Length': '0'} 2022-09-03T07:03:46.188 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f915f23a3b66571-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.189 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.190 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.190 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.191 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.191 DEBUG:boto:Method: GET 2022-09-03T07:03:46.192 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:46.192 DEBUG:boto:Data: 2022-09-03T07:03:46.193 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.194 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.194 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.195 DEBUG:boto:Params: {} 2022-09-03T07:03:46.195 DEBUG:boto:Token: None 2022-09-03T07:03:46.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:46.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZZMqd+Eow5x/s0lTsj3YCogJkcg= 2022-09-03T07:03:46.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZZMqd+Eow5x/s0lTsj3YCogJkcg=', 'Content-Length': '0'} 2022-09-03T07:03:46.201 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096c5f9d064124df7-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.202 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.203 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.205 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.205 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.206 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.207 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:46.208 DEBUG:boto:Data: 2022-09-03T07:03:46.209 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.211 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.212 DEBUG:boto:Params: {} 2022-09-03T07:03:46.213 DEBUG:boto:Token: None 2022-09-03T07:03:46.214 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:46.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fb1JoJ9qTJw7UmcLs62C+lFXgeA= 2022-09-03T07:03:46.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fb1JoJ9qTJw7UmcLs62C+lFXgeA=', 'Content-Length': '0'} 2022-09-03T07:03:46.218 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000041d18e9fc29022b6-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.219 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.220 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.221 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.222 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.223 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.224 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:46.225 DEBUG:boto:Data: 2022-09-03T07:03:46.226 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.227 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.228 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.228 DEBUG:boto:Params: {} 2022-09-03T07:03:46.230 DEBUG:boto:Token: None 2022-09-03T07:03:46.231 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:46.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fb1JoJ9qTJw7UmcLs62C+lFXgeA= 2022-09-03T07:03:46.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fb1JoJ9qTJw7UmcLs62C+lFXgeA=', 'Content-Length': '0'} 2022-09-03T07:03:46.236 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002caa1ba5f552a118-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.238 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:03:46.239 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.240 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.241 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.242 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.243 DEBUG:boto:Method: GET 2022-09-03T07:03:46.243 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:46.244 DEBUG:boto:Data: 2022-09-03T07:03:46.244 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.245 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.245 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.246 DEBUG:boto:Params: {} 2022-09-03T07:03:46.247 DEBUG:boto:Token: None 2022-09-03T07:03:46.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:46.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZZMqd+Eow5x/s0lTsj3YCogJkcg= 2022-09-03T07:03:46.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZZMqd+Eow5x/s0lTsj3YCogJkcg=', 'Content-Length': '0'} 2022-09-03T07:03:46.251 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004faae3ad79e8f0f7-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.252 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.252 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.253 DEBUG:boto:path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.253 DEBUG:boto:auth_path=/avysmw-73/e?versionId=null 2022-09-03T07:03:46.254 DEBUG:boto:Method: GET 2022-09-03T07:03:46.255 DEBUG:boto:Path: /avysmw-73/e?versionId=null 2022-09-03T07:03:46.255 DEBUG:boto:Data: 2022-09-03T07:03:46.256 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.256 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.258 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.262 DEBUG:boto:Params: {} 2022-09-03T07:03:46.263 DEBUG:boto:Token: None 2022-09-03T07:03:46.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e?versionId=null 2022-09-03T07:03:46.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZZMqd+Eow5x/s0lTsj3YCogJkcg= 2022-09-03T07:03:46.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZZMqd+Eow5x/s0lTsj3YCogJkcg=', 'Content-Length': '0'} 2022-09-03T07:03:46.272 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e69c5651a96066bd-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.273 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.274 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.276 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:46.277 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:46.278 DEBUG:boto:Method: GET 2022-09-03T07:03:46.279 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:46.280 DEBUG:boto:Data: 2022-09-03T07:03:46.281 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.283 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.284 DEBUG:boto:Params: {} 2022-09-03T07:03:46.285 DEBUG:boto:Token: None 2022-09-03T07:03:46.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.289 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000673fb6c072f1074b-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.290 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.291 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.292 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.292 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:46.293 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:46.293 DEBUG:boto:Method: GET 2022-09-03T07:03:46.294 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:46.294 DEBUG:boto:Data: 2022-09-03T07:03:46.295 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.295 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.296 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.296 DEBUG:boto:Params: {} 2022-09-03T07:03:46.297 DEBUG:boto:Token: None 2022-09-03T07:03:46.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.302 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009dfbf07016df7f9d-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.302 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.303 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.304 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.304 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.305 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:46.306 DEBUG:boto:Data: 2022-09-03T07:03:46.307 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.308 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.309 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.310 DEBUG:boto:Params: {} 2022-09-03T07:03:46.311 DEBUG:boto:Token: None 2022-09-03T07:03:46.312 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e 2022-09-03T07:03:46.313 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:krPtMc66fYWKFAJvoxhzHKr3tj8= 2022-09-03T07:03:46.314 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:krPtMc66fYWKFAJvoxhzHKr3tj8=', 'Content-Length': '0'} 2022-09-03T07:03:46.316 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f240ce22303ac673-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.317 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.318 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.318 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.319 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:46.319 DEBUG:boto:Data: 2022-09-03T07:03:46.320 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.320 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.321 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.321 DEBUG:boto:Params: {} 2022-09-03T07:03:46.322 DEBUG:boto:Token: None 2022-09-03T07:03:46.322 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e 2022-09-03T07:03:46.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:krPtMc66fYWKFAJvoxhzHKr3tj8= 2022-09-03T07:03:46.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:krPtMc66fYWKFAJvoxhzHKr3tj8=', 'Content-Length': '0'} 2022-09-03T07:03:46.326 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001265f27efc8f6a3b-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.327 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2022-09-03T07:03:46.327 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.328 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.329 DEBUG:boto:Method: GET 2022-09-03T07:03:46.329 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:46.330 DEBUG:boto:Data: 2022-09-03T07:03:46.331 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.333 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.334 DEBUG:boto:Params: {} 2022-09-03T07:03:46.334 DEBUG:boto:Token: None 2022-09-03T07:03:46.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e 2022-09-03T07:03:46.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScoBghq+0hAhLu6P8Alom1+f5W4= 2022-09-03T07:03:46.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScoBghq+0hAhLu6P8Alom1+f5W4=', 'Content-Length': '0'} 2022-09-03T07:03:46.339 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3cdce1209f430f2-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.340 DEBUG:boto:path=/avysmw-73/e 2022-09-03T07:03:46.341 DEBUG:boto:auth_path=/avysmw-73/e 2022-09-03T07:03:46.341 DEBUG:boto:Method: GET 2022-09-03T07:03:46.342 DEBUG:boto:Path: /avysmw-73/e 2022-09-03T07:03:46.343 DEBUG:boto:Data: 2022-09-03T07:03:46.343 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.344 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.344 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.345 DEBUG:boto:Params: {} 2022-09-03T07:03:46.345 DEBUG:boto:Token: None 2022-09-03T07:03:46.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/e 2022-09-03T07:03:46.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ScoBghq+0hAhLu6P8Alom1+f5W4= 2022-09-03T07:03:46.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ScoBghq+0hAhLu6P8Alom1+f5W4=', 'Content-Length': '0'} 2022-09-03T07:03:46.350 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:53 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c242a85a4394870-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.394 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.395 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.395 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:46.396 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:46.397 DEBUG:boto:Method: GET 2022-09-03T07:03:46.398 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:46.399 DEBUG:boto:Data: 2022-09-03T07:03:46.399 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.407 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.408 DEBUG:boto:Params: {} 2022-09-03T07:03:46.409 DEBUG:boto:Token: None 2022-09-03T07:03:46.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.414 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.417 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a706b89c386be99b-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.418 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.420 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.421 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.422 DEBUG:boto:path=/avysmw-73/?prefix=e 2022-09-03T07:03:46.422 DEBUG:boto:auth_path=/avysmw-73/?prefix=e 2022-09-03T07:03:46.423 DEBUG:boto:Method: GET 2022-09-03T07:03:46.423 DEBUG:boto:Path: /avysmw-73/?prefix=e 2022-09-03T07:03:46.424 DEBUG:boto:Data: 2022-09-03T07:03:46.425 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.425 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.426 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.426 DEBUG:boto:Params: {} 2022-09-03T07:03:46.427 DEBUG:boto:Token: None 2022-09-03T07:03:46.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.432 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009d37cba37f57f491-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.433 DEBUG:boto:b'avysmw-73e1000falsee2022-09-03T07:00:53.964Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.436 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:03:46.437 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.438 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.438 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.439 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.440 DEBUG:boto:Method: GET 2022-09-03T07:03:46.440 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:46.441 DEBUG:boto:Data: 2022-09-03T07:03:46.441 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.443 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.443 DEBUG:boto:Params: {} 2022-09-03T07:03:46.444 DEBUG:boto:Token: None 2022-09-03T07:03:46.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:46.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+2jhNvLhA9tY+Sg0TKQmP+6Lv8Q= 2022-09-03T07:03:46.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+2jhNvLhA9tY+Sg0TKQmP+6Lv8Q=', 'Content-Length': '0'} 2022-09-03T07:03:46.448 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eebca239098dd67e-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.449 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.450 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.450 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.451 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.452 DEBUG:boto:Method: GET 2022-09-03T07:03:46.452 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:46.453 DEBUG:boto:Data: 2022-09-03T07:03:46.453 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.455 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.456 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.457 DEBUG:boto:Params: {} 2022-09-03T07:03:46.458 DEBUG:boto:Token: None 2022-09-03T07:03:46.459 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:46.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+2jhNvLhA9tY+Sg0TKQmP+6Lv8Q= 2022-09-03T07:03:46.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+2jhNvLhA9tY+Sg0TKQmP+6Lv8Q=', 'Content-Length': '0'} 2022-09-03T07:03:46.464 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083eca2b739ad00c2-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.466 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.467 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.468 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.468 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.469 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.470 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:46.471 DEBUG:boto:Data: 2022-09-03T07:03:46.472 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.474 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.475 DEBUG:boto:Params: {} 2022-09-03T07:03:46.476 DEBUG:boto:Token: None 2022-09-03T07:03:46.477 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:46.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EvEgz/Yd8wj4bv8dG+5E5jIxAg= 2022-09-03T07:03:46.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EvEgz/Yd8wj4bv8dG+5E5jIxAg=', 'Content-Length': '0'} 2022-09-03T07:03:46.482 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f88461af944c8db-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.483 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.484 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.485 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.486 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.487 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.488 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:46.488 DEBUG:boto:Data: 2022-09-03T07:03:46.489 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.490 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.491 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.492 DEBUG:boto:Params: {} 2022-09-03T07:03:46.493 DEBUG:boto:Token: None 2022-09-03T07:03:46.494 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:46.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EvEgz/Yd8wj4bv8dG+5E5jIxAg= 2022-09-03T07:03:46.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EvEgz/Yd8wj4bv8dG+5E5jIxAg=', 'Content-Length': '0'} 2022-09-03T07:03:46.499 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081ca7ae4a16cc439-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.500 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:03:46.501 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.502 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.503 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.504 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.504 DEBUG:boto:Method: GET 2022-09-03T07:03:46.505 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:46.507 DEBUG:boto:Data: 2022-09-03T07:03:46.508 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.509 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.510 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.511 DEBUG:boto:Params: {} 2022-09-03T07:03:46.512 DEBUG:boto:Token: None 2022-09-03T07:03:46.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:46.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+2jhNvLhA9tY+Sg0TKQmP+6Lv8Q= 2022-09-03T07:03:46.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+2jhNvLhA9tY+Sg0TKQmP+6Lv8Q=', 'Content-Length': '0'} 2022-09-03T07:03:46.519 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d95ae35056bd717-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.520 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.521 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.523 DEBUG:boto:path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.524 DEBUG:boto:auth_path=/avysmw-73/f?versionId=null 2022-09-03T07:03:46.524 DEBUG:boto:Method: GET 2022-09-03T07:03:46.525 DEBUG:boto:Path: /avysmw-73/f?versionId=null 2022-09-03T07:03:46.526 DEBUG:boto:Data: 2022-09-03T07:03:46.526 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.527 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.527 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.528 DEBUG:boto:Params: {} 2022-09-03T07:03:46.528 DEBUG:boto:Token: None 2022-09-03T07:03:46.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f?versionId=null 2022-09-03T07:03:46.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+2jhNvLhA9tY+Sg0TKQmP+6Lv8Q= 2022-09-03T07:03:46.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+2jhNvLhA9tY+Sg0TKQmP+6Lv8Q=', 'Content-Length': '0'} 2022-09-03T07:03:46.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076200a518a8c7177-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.534 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.535 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.536 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:46.536 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:46.537 DEBUG:boto:Method: GET 2022-09-03T07:03:46.538 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:46.538 DEBUG:boto:Data: 2022-09-03T07:03:46.539 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.539 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.540 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.540 DEBUG:boto:Params: {} 2022-09-03T07:03:46.541 DEBUG:boto:Token: None 2022-09-03T07:03:46.542 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.547 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e3399acfa8e2a9b0-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.588 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.590 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.591 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.592 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:46.593 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:46.594 DEBUG:boto:Method: GET 2022-09-03T07:03:46.595 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:46.596 DEBUG:boto:Data: 2022-09-03T07:03:46.597 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.598 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.599 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.600 DEBUG:boto:Params: {} 2022-09-03T07:03:46.601 DEBUG:boto:Token: None 2022-09-03T07:03:46.602 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.608 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3c41cea917fc4fd-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.609 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.611 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.612 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.613 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.614 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:46.614 DEBUG:boto:Data: 2022-09-03T07:03:46.616 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.617 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.618 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.619 DEBUG:boto:Params: {} 2022-09-03T07:03:46.620 DEBUG:boto:Token: None 2022-09-03T07:03:46.621 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f 2022-09-03T07:03:46.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TVINFRNPA1ungqWd9fFNp5XfOx0= 2022-09-03T07:03:46.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TVINFRNPA1ungqWd9fFNp5XfOx0=', 'Content-Length': '0'} 2022-09-03T07:03:46.625 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000423a3feaf38cc3cf-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.627 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.628 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.629 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.630 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:46.631 DEBUG:boto:Data: 2022-09-03T07:03:46.632 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.633 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.634 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.635 DEBUG:boto:Params: {} 2022-09-03T07:03:46.636 DEBUG:boto:Token: None 2022-09-03T07:03:46.637 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f 2022-09-03T07:03:46.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TVINFRNPA1ungqWd9fFNp5XfOx0= 2022-09-03T07:03:46.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TVINFRNPA1ungqWd9fFNp5XfOx0=', 'Content-Length': '0'} 2022-09-03T07:03:46.641 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a539a249c0b73c69-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.643 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2022-09-03T07:03:46.644 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.645 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.646 DEBUG:boto:Method: GET 2022-09-03T07:03:46.647 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:46.648 DEBUG:boto:Data: 2022-09-03T07:03:46.649 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.650 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.651 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.653 DEBUG:boto:Params: {} 2022-09-03T07:03:46.654 DEBUG:boto:Token: None 2022-09-03T07:03:46.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f 2022-09-03T07:03:46.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4lzQCBPBLAZ4rCOt/0uKeRX94fo= 2022-09-03T07:03:46.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4lzQCBPBLAZ4rCOt/0uKeRX94fo=', 'Content-Length': '0'} 2022-09-03T07:03:46.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b0eb1789710f7da-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.664 DEBUG:boto:path=/avysmw-73/f 2022-09-03T07:03:46.665 DEBUG:boto:auth_path=/avysmw-73/f 2022-09-03T07:03:46.666 DEBUG:boto:Method: GET 2022-09-03T07:03:46.667 DEBUG:boto:Path: /avysmw-73/f 2022-09-03T07:03:46.668 DEBUG:boto:Data: 2022-09-03T07:03:46.669 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.670 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.671 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.672 DEBUG:boto:Params: {} 2022-09-03T07:03:46.673 DEBUG:boto:Token: None 2022-09-03T07:03:46.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/f 2022-09-03T07:03:46.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4lzQCBPBLAZ4rCOt/0uKeRX94fo= 2022-09-03T07:03:46.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4lzQCBPBLAZ4rCOt/0uKeRX94fo=', 'Content-Length': '0'} 2022-09-03T07:03:46.678 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b87f2b76e8033fd5-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.680 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.680 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.681 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:46.681 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:46.682 DEBUG:boto:Method: GET 2022-09-03T07:03:46.683 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:46.684 DEBUG:boto:Data: 2022-09-03T07:03:46.684 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.685 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.685 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.686 DEBUG:boto:Params: {} 2022-09-03T07:03:46.686 DEBUG:boto:Token: None 2022-09-03T07:03:46.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.699 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008da887c7b034b303-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.702 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.704 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.705 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.707 DEBUG:boto:path=/avysmw-73/?prefix=f 2022-09-03T07:03:46.708 DEBUG:boto:auth_path=/avysmw-73/?prefix=f 2022-09-03T07:03:46.710 DEBUG:boto:Method: GET 2022-09-03T07:03:46.711 DEBUG:boto:Path: /avysmw-73/?prefix=f 2022-09-03T07:03:46.713 DEBUG:boto:Data: 2022-09-03T07:03:46.713 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.714 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.716 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.717 DEBUG:boto:Params: {} 2022-09-03T07:03:46.718 DEBUG:boto:Token: None 2022-09-03T07:03:46.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.725 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c868c45f0ac8be84-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.766 DEBUG:boto:b'avysmw-73f1000falsef2022-09-03T07:00:54.036Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.767 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:03:46.768 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:46.769 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:46.770 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.771 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.772 DEBUG:boto:Method: GET 2022-09-03T07:03:46.773 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:46.773 DEBUG:boto:Data: 2022-09-03T07:03:46.774 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.775 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.776 DEBUG:boto:Params: {} 2022-09-03T07:03:46.777 DEBUG:boto:Token: None 2022-09-03T07:03:46.778 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:46.779 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ivtnN86Favkxjv7NVoTx8nKKw/0= 2022-09-03T07:03:46.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ivtnN86Favkxjv7NVoTx8nKKw/0=', 'Content-Length': '0'} 2022-09-03T07:03:46.782 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000140b7c9fac17fad-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.786 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:46.788 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:46.789 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.790 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.790 DEBUG:boto:Method: GET 2022-09-03T07:03:46.791 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:46.792 DEBUG:boto:Data: 2022-09-03T07:03:46.793 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.793 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.794 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.795 DEBUG:boto:Params: {} 2022-09-03T07:03:46.796 DEBUG:boto:Token: None 2022-09-03T07:03:46.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:46.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ivtnN86Favkxjv7NVoTx8nKKw/0= 2022-09-03T07:03:46.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ivtnN86Favkxjv7NVoTx8nKKw/0=', 'Content-Length': '0'} 2022-09-03T07:03:46.804 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005a6405334f9586f-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.806 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:46.807 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:46.807 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.808 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.808 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.809 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:46.810 DEBUG:boto:Data: 2022-09-03T07:03:46.811 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.811 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.812 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.812 DEBUG:boto:Params: {} 2022-09-03T07:03:46.813 DEBUG:boto:Token: None 2022-09-03T07:03:46.813 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:46.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o2iQ8cgs+WWKFvS3bKDu+km7sAE= 2022-09-03T07:03:46.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o2iQ8cgs+WWKFvS3bKDu+km7sAE=', 'Content-Length': '0'} 2022-09-03T07:03:46.817 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d23c2e1d911b385-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.818 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:46.819 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:46.819 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.820 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.820 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.821 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:46.822 DEBUG:boto:Data: 2022-09-03T07:03:46.822 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.823 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.823 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.824 DEBUG:boto:Params: {} 2022-09-03T07:03:46.825 DEBUG:boto:Token: None 2022-09-03T07:03:46.825 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:46.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o2iQ8cgs+WWKFvS3bKDu+km7sAE= 2022-09-03T07:03:46.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o2iQ8cgs+WWKFvS3bKDu+km7sAE=', 'Content-Length': '0'} 2022-09-03T07:03:46.829 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3062373a3edc2d4-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.831 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:03:46.832 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:46.833 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:46.835 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.836 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.837 DEBUG:boto:Method: GET 2022-09-03T07:03:46.838 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:46.839 DEBUG:boto:Data: 2022-09-03T07:03:46.840 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.843 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.843 DEBUG:boto:Params: {} 2022-09-03T07:03:46.844 DEBUG:boto:Token: None 2022-09-03T07:03:46.844 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:46.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ivtnN86Favkxjv7NVoTx8nKKw/0= 2022-09-03T07:03:46.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ivtnN86Favkxjv7NVoTx8nKKw/0=', 'Content-Length': '0'} 2022-09-03T07:03:46.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1a0e9ab901a07a5-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.890 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:46.892 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:46.893 DEBUG:boto:path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.895 DEBUG:boto:auth_path=/avysmw-73/g?versionId=null 2022-09-03T07:03:46.895 DEBUG:boto:Method: GET 2022-09-03T07:03:46.896 DEBUG:boto:Path: /avysmw-73/g?versionId=null 2022-09-03T07:03:46.897 DEBUG:boto:Data: 2022-09-03T07:03:46.898 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.900 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.900 DEBUG:boto:Params: {} 2022-09-03T07:03:46.901 DEBUG:boto:Token: None 2022-09-03T07:03:46.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/g?versionId=null 2022-09-03T07:03:46.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ivtnN86Favkxjv7NVoTx8nKKw/0= 2022-09-03T07:03:46.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ivtnN86Favkxjv7NVoTx8nKKw/0=', 'Content-Length': '0'} 2022-09-03T07:03:46.911 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035d1235d138d8a61-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.912 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.912 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.913 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:46.914 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:46.915 DEBUG:boto:Method: GET 2022-09-03T07:03:46.915 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:46.916 DEBUG:boto:Data: 2022-09-03T07:03:46.916 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.921 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.922 DEBUG:boto:Params: {} 2022-09-03T07:03:46.922 DEBUG:boto:Token: None 2022-09-03T07:03:46.923 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.928 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b015c45589c54305-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.931 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.933 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:46.934 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:46.935 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:46.936 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:46.937 DEBUG:boto:Method: GET 2022-09-03T07:03:46.938 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:46.939 DEBUG:boto:Data: 2022-09-03T07:03:46.940 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.941 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.942 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.943 DEBUG:boto:Params: {} 2022-09-03T07:03:46.945 DEBUG:boto:Token: None 2022-09-03T07:03:46.947 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/ 2022-09-03T07:03:46.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI= 2022-09-03T07:03:46.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oviN38coeW8Pz85ArhaQBZzaZyI=', 'Content-Length': '0'} 2022-09-03T07:03:46.955 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093e2a32cb905953e-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.957 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:46.959 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:46.961 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:46.962 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.964 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:46.965 DEBUG:boto:Data: 2022-09-03T07:03:46.966 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:46.969 DEBUG:boto:Port: 8001 2022-09-03T07:03:46.970 DEBUG:boto:Params: {} 2022-09-03T07:03:46.971 DEBUG:boto:Token: None 2022-09-03T07:03:46.973 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/g 2022-09-03T07:03:46.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CNymKrdA/AVfdtSTPvLdD1uiXWM= 2022-09-03T07:03:46.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CNymKrdA/AVfdtSTPvLdD1uiXWM=', 'Content-Length': '0'} 2022-09-03T07:03:46.979 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ea39ad08383adeb3-006312fc52-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.980 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:46.981 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:46.982 DEBUG:boto:Method: HEAD 2022-09-03T07:03:46.982 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:46.983 DEBUG:boto:Data: 2022-09-03T07:03:46.984 DEBUG:boto:Headers: {} 2022-09-03T07:03:46.985 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:46.986 DEBUG:boto:Port: 8000 2022-09-03T07:03:46.986 DEBUG:boto:Params: {} 2022-09-03T07:03:46.987 DEBUG:boto:Token: None 2022-09-03T07:03:46.988 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:46 GMT /avysmw-73/g 2022-09-03T07:03:46.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CNymKrdA/AVfdtSTPvLdD1uiXWM= 2022-09-03T07:03:46.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CNymKrdA/AVfdtSTPvLdD1uiXWM=', 'Content-Length': '0'} 2022-09-03T07:03:46.995 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005b68e1dc17e68399-006312fc52-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:46.996 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2022-09-03T07:03:46.997 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:46.998 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:46.999 DEBUG:boto:Method: GET 2022-09-03T07:03:47.000 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:47.001 DEBUG:boto:Data: 2022-09-03T07:03:47.002 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:47.003 DEBUG:boto:Port: 8001 2022-09-03T07:03:47.004 DEBUG:boto:Params: {} 2022-09-03T07:03:47.004 DEBUG:boto:Token: None 2022-09-03T07:03:47.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/g 2022-09-03T07:03:47.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DU5/VSlSRKvxyNsnyEI1FbRKuJg= 2022-09-03T07:03:47.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DU5/VSlSRKvxyNsnyEI1FbRKuJg=', 'Content-Length': '0'} 2022-09-03T07:03:47.009 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b4863680dcba3c1-006312fc53-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.054 DEBUG:boto:path=/avysmw-73/g 2022-09-03T07:03:47.055 DEBUG:boto:auth_path=/avysmw-73/g 2022-09-03T07:03:47.056 DEBUG:boto:Method: GET 2022-09-03T07:03:47.057 DEBUG:boto:Path: /avysmw-73/g 2022-09-03T07:03:47.058 DEBUG:boto:Data: 2022-09-03T07:03:47.059 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.060 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:47.061 DEBUG:boto:Port: 8000 2022-09-03T07:03:47.062 DEBUG:boto:Params: {} 2022-09-03T07:03:47.063 DEBUG:boto:Token: None 2022-09-03T07:03:47.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/g 2022-09-03T07:03:47.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DU5/VSlSRKvxyNsnyEI1FbRKuJg= 2022-09-03T07:03:47.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DU5/VSlSRKvxyNsnyEI1FbRKuJg=', 'Content-Length': '0'} 2022-09-03T07:03:47.080 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000afbaa2fccd5d5593-006312fc53-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.082 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:47.083 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:47.084 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:47.085 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:47.086 DEBUG:boto:Method: GET 2022-09-03T07:03:47.087 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:47.088 DEBUG:boto:Data: 2022-09-03T07:03:47.089 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.090 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:47.091 DEBUG:boto:Port: 8001 2022-09-03T07:03:47.092 DEBUG:boto:Params: {} 2022-09-03T07:03:47.093 DEBUG:boto:Token: None 2022-09-03T07:03:47.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/ 2022-09-03T07:03:47.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk= 2022-09-03T07:03:47.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk=', 'Content-Length': '0'} 2022-09-03T07:03:47.100 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd01377761bf91d0-006312fc53-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.102 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:47.118 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:47.119 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:47.121 DEBUG:boto:path=/avysmw-73/?prefix=g 2022-09-03T07:03:47.123 DEBUG:boto:auth_path=/avysmw-73/?prefix=g 2022-09-03T07:03:47.126 DEBUG:boto:Method: GET 2022-09-03T07:03:47.127 DEBUG:boto:Path: /avysmw-73/?prefix=g 2022-09-03T07:03:47.128 DEBUG:boto:Data: 2022-09-03T07:03:47.129 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.130 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:47.131 DEBUG:boto:Port: 8000 2022-09-03T07:03:47.132 DEBUG:boto:Params: {} 2022-09-03T07:03:47.134 DEBUG:boto:Token: None 2022-09-03T07:03:47.137 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/ 2022-09-03T07:03:47.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk= 2022-09-03T07:03:47.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk=', 'Content-Length': '0'} 2022-09-03T07:03:47.142 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000082a36fad6566208c-006312fc53-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.143 DEBUG:boto:b'avysmw-73g1000falseg2022-09-03T07:00:54.062Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:47.144 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:03:47.145 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.145 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.151 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.152 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.153 DEBUG:boto:Method: GET 2022-09-03T07:03:47.154 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:47.157 DEBUG:boto:Data: 2022-09-03T07:03:47.158 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.159 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:47.159 DEBUG:boto:Port: 8001 2022-09-03T07:03:47.160 DEBUG:boto:Params: {} 2022-09-03T07:03:47.161 DEBUG:boto:Token: None 2022-09-03T07:03:47.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:47.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ElzWQcUws3yzajqvdxnCYS3uEOs= 2022-09-03T07:03:47.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ElzWQcUws3yzajqvdxnCYS3uEOs=', 'Content-Length': '0'} 2022-09-03T07:03:47.170 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000896b7fb13b6cd4a9-006312fc53-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.172 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.173 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.176 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.179 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.179 DEBUG:boto:Method: GET 2022-09-03T07:03:47.180 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:47.180 DEBUG:boto:Data: 2022-09-03T07:03:47.181 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.182 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:47.182 DEBUG:boto:Port: 8000 2022-09-03T07:03:47.183 DEBUG:boto:Params: {} 2022-09-03T07:03:47.183 DEBUG:boto:Token: None 2022-09-03T07:03:47.184 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:47.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ElzWQcUws3yzajqvdxnCYS3uEOs= 2022-09-03T07:03:47.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ElzWQcUws3yzajqvdxnCYS3uEOs=', 'Content-Length': '0'} 2022-09-03T07:03:47.188 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000370bce2bf3ead87a-006312fc53-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.189 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.189 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.190 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.191 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.191 DEBUG:boto:Method: HEAD 2022-09-03T07:03:47.192 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:47.192 DEBUG:boto:Data: 2022-09-03T07:03:47.193 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.193 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:47.194 DEBUG:boto:Port: 8001 2022-09-03T07:03:47.194 DEBUG:boto:Params: {} 2022-09-03T07:03:47.195 DEBUG:boto:Token: None 2022-09-03T07:03:47.196 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:47.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RISV0+fwUjLLXMXc5M88P6jZJ7Y= 2022-09-03T07:03:47.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RISV0+fwUjLLXMXc5M88P6jZJ7Y=', 'Content-Length': '0'} 2022-09-03T07:03:47.200 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f71a0efdc3381f05-006312fc53-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.201 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.202 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.203 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.204 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.205 DEBUG:boto:Method: HEAD 2022-09-03T07:03:47.206 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:47.207 DEBUG:boto:Data: 2022-09-03T07:03:47.208 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.209 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:47.210 DEBUG:boto:Port: 8000 2022-09-03T07:03:47.211 DEBUG:boto:Params: {} 2022-09-03T07:03:47.212 DEBUG:boto:Token: None 2022-09-03T07:03:47.213 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:47.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RISV0+fwUjLLXMXc5M88P6jZJ7Y= 2022-09-03T07:03:47.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RISV0+fwUjLLXMXc5M88P6jZJ7Y=', 'Content-Length': '0'} 2022-09-03T07:03:47.218 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3d0c1158d293a78-006312fc53-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.220 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:03:47.221 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.222 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.223 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.224 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.225 DEBUG:boto:Method: GET 2022-09-03T07:03:47.227 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:47.228 DEBUG:boto:Data: 2022-09-03T07:03:47.229 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.230 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:47.231 DEBUG:boto:Port: 8001 2022-09-03T07:03:47.232 DEBUG:boto:Params: {} 2022-09-03T07:03:47.233 DEBUG:boto:Token: None 2022-09-03T07:03:47.234 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:47.235 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ElzWQcUws3yzajqvdxnCYS3uEOs= 2022-09-03T07:03:47.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ElzWQcUws3yzajqvdxnCYS3uEOs=', 'Content-Length': '0'} 2022-09-03T07:03:47.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c7d16494e902af85-006312fc53-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.241 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.243 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.243 DEBUG:boto:path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.244 DEBUG:boto:auth_path=/avysmw-73/h?versionId=null 2022-09-03T07:03:47.244 DEBUG:boto:Method: GET 2022-09-03T07:03:47.245 DEBUG:boto:Path: /avysmw-73/h?versionId=null 2022-09-03T07:03:47.245 DEBUG:boto:Data: 2022-09-03T07:03:47.246 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.247 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:47.247 DEBUG:boto:Port: 8000 2022-09-03T07:03:47.249 DEBUG:boto:Params: {} 2022-09-03T07:03:47.250 DEBUG:boto:Token: None 2022-09-03T07:03:47.251 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h?versionId=null 2022-09-03T07:03:47.252 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ElzWQcUws3yzajqvdxnCYS3uEOs= 2022-09-03T07:03:47.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ElzWQcUws3yzajqvdxnCYS3uEOs=', 'Content-Length': '0'} 2022-09-03T07:03:47.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b09bcf954846440f-006312fc53-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.259 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:47.260 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:47.260 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:47.261 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:47.262 DEBUG:boto:Method: GET 2022-09-03T07:03:47.262 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:47.263 DEBUG:boto:Data: 2022-09-03T07:03:47.264 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:47.265 DEBUG:boto:Port: 8001 2022-09-03T07:03:47.267 DEBUG:boto:Params: {} 2022-09-03T07:03:47.268 DEBUG:boto:Token: None 2022-09-03T07:03:47.270 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/ 2022-09-03T07:03:47.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk= 2022-09-03T07:03:47.274 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk=', 'Content-Length': '0'} 2022-09-03T07:03:47.279 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f503bfdebe4860eb-006312fc53-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.280 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:47.282 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:47.283 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:47.284 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:47.285 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:47.286 DEBUG:boto:Method: GET 2022-09-03T07:03:47.287 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:47.287 DEBUG:boto:Data: 2022-09-03T07:03:47.288 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.288 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:47.289 DEBUG:boto:Port: 8000 2022-09-03T07:03:47.289 DEBUG:boto:Params: {} 2022-09-03T07:03:47.290 DEBUG:boto:Token: None 2022-09-03T07:03:47.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/ 2022-09-03T07:03:47.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk= 2022-09-03T07:03:47.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk=', 'Content-Length': '0'} 2022-09-03T07:03:47.296 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000224cee29481c7954-006312fc53-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.298 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:47.299 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.300 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.301 DEBUG:boto:Method: HEAD 2022-09-03T07:03:47.302 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:47.303 DEBUG:boto:Data: 2022-09-03T07:03:47.304 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:47.306 DEBUG:boto:Port: 8001 2022-09-03T07:03:47.307 DEBUG:boto:Params: {} 2022-09-03T07:03:47.308 DEBUG:boto:Token: None 2022-09-03T07:03:47.309 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h 2022-09-03T07:03:47.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:udRqfZzjBUnyJ+L6DQLjAD+7UEw= 2022-09-03T07:03:47.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:udRqfZzjBUnyJ+L6DQLjAD+7UEw=', 'Content-Length': '0'} 2022-09-03T07:03:47.313 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf6475443b0ec4cf-006312fc53-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.315 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.316 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.317 DEBUG:boto:Method: HEAD 2022-09-03T07:03:47.318 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:47.318 DEBUG:boto:Data: 2022-09-03T07:03:47.319 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.320 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:47.321 DEBUG:boto:Port: 8000 2022-09-03T07:03:47.322 DEBUG:boto:Params: {} 2022-09-03T07:03:47.323 DEBUG:boto:Token: None 2022-09-03T07:03:47.324 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h 2022-09-03T07:03:47.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:udRqfZzjBUnyJ+L6DQLjAD+7UEw= 2022-09-03T07:03:47.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:udRqfZzjBUnyJ+L6DQLjAD+7UEw=', 'Content-Length': '0'} 2022-09-03T07:03:47.331 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ea9d45974c16256f-006312fc53-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.332 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2022-09-03T07:03:47.333 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.334 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.335 DEBUG:boto:Method: GET 2022-09-03T07:03:47.336 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:47.336 DEBUG:boto:Data: 2022-09-03T07:03:47.337 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:47.339 DEBUG:boto:Port: 8001 2022-09-03T07:03:47.340 DEBUG:boto:Params: {} 2022-09-03T07:03:47.341 DEBUG:boto:Token: None 2022-09-03T07:03:47.342 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h 2022-09-03T07:03:47.343 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1wzIU6zV/p3axGt1mNnGNwX5OEk= 2022-09-03T07:03:47.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1wzIU6zV/p3axGt1mNnGNwX5OEk=', 'Content-Length': '0'} 2022-09-03T07:03:47.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022f75db8b07727fe-006312fc53-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.355 DEBUG:boto:path=/avysmw-73/h 2022-09-03T07:03:47.356 DEBUG:boto:auth_path=/avysmw-73/h 2022-09-03T07:03:47.357 DEBUG:boto:Method: GET 2022-09-03T07:03:47.358 DEBUG:boto:Path: /avysmw-73/h 2022-09-03T07:03:47.358 DEBUG:boto:Data: 2022-09-03T07:03:47.359 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.360 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:47.360 DEBUG:boto:Port: 8000 2022-09-03T07:03:47.361 DEBUG:boto:Params: {} 2022-09-03T07:03:47.362 DEBUG:boto:Token: None 2022-09-03T07:03:47.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/h 2022-09-03T07:03:47.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1wzIU6zV/p3axGt1mNnGNwX5OEk= 2022-09-03T07:03:47.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1wzIU6zV/p3axGt1mNnGNwX5OEk=', 'Content-Length': '0'} 2022-09-03T07:03:47.370 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:00:54 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031f1e0c1b5eeb2f5-006312fc53-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.372 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:47.373 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:47.375 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:47.376 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:47.377 DEBUG:boto:Method: GET 2022-09-03T07:03:47.381 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:47.382 DEBUG:boto:Data: 2022-09-03T07:03:47.383 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:47.385 DEBUG:boto:Port: 8001 2022-09-03T07:03:47.386 DEBUG:boto:Params: {} 2022-09-03T07:03:47.387 DEBUG:boto:Token: None 2022-09-03T07:03:47.388 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/ 2022-09-03T07:03:47.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk= 2022-09-03T07:03:47.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk=', 'Content-Length': '0'} 2022-09-03T07:03:47.394 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ec6652d2b204b57-006312fc53-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.395 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:47.397 DEBUG:boto:path=/avysmw-73/ 2022-09-03T07:03:47.398 DEBUG:boto:auth_path=/avysmw-73/ 2022-09-03T07:03:47.401 DEBUG:boto:path=/avysmw-73/?prefix=h 2022-09-03T07:03:47.402 DEBUG:boto:auth_path=/avysmw-73/?prefix=h 2022-09-03T07:03:47.403 DEBUG:boto:Method: GET 2022-09-03T07:03:47.404 DEBUG:boto:Path: /avysmw-73/?prefix=h 2022-09-03T07:03:47.405 DEBUG:boto:Data: 2022-09-03T07:03:47.406 DEBUG:boto:Headers: {} 2022-09-03T07:03:47.407 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:47.408 DEBUG:boto:Port: 8000 2022-09-03T07:03:47.409 DEBUG:boto:Params: {} 2022-09-03T07:03:47.409 DEBUG:boto:Token: None 2022-09-03T07:03:47.410 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:47 GMT /avysmw-73/ 2022-09-03T07:03:47.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk= 2022-09-03T07:03:47.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeYV+F+oBGH4JBgwWK9eveKVQdk=', 'Content-Length': '0'} 2022-09-03T07:03:47.416 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a5dac648e4b9f3e-006312fc53-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:47.417 DEBUG:boto:b'avysmw-73h1000falseh2022-09-03T07:00:54.129Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:03:47.419 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-73 zones={test-zone3, test-zone2} 2022-09-03T07:03:47.421 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_reshard_full ... ok 2022-09-03T07:03:47.423 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:47.424 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:47.425 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:03:47.485 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:47.486 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:47.535 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T07:03:47.557 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:03:47.557 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:47.558 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:47.558 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:03:47.558 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:47.559 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:47.559 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662188453.896599_455.1", 2022-09-03T07:03:47.559 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:00:53.896599Z" 2022-09-03T07:03:47.560 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:47.560 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:47.560 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:47.560 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:03:47.560 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:47.561 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:47.561 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:47.561 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:03:47.562 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:47.562 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:03:47.563 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662188453.896599_455.1', 2: '', 3: ''} 2022-09-03T07:03:47.564 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T07:03:47.565 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:47.566 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:47.566 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:03:47.639 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:47.640 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:47.681 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:47.680+0000 7f6179540600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:47.683 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:47.681+0000 7f6179540600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:47.709 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:03:47.709 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:03:47.710 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:03:47.710 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:03:47.712 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:03:47.712 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:03:47.713 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:03:47.713 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:47.713 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:03:47.715 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:47.715 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:03:47.716 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:47.716 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:47.717 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:03:47.717 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:47.718 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:03:47.718 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:47.719 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:03:47.719 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:47.720 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:47.720 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:47.721 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:47.721 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:03:47.721 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:47.723 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:47.724 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662188453.896599_455.1", 2022-09-03T07:03:47.724 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:47.725 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:47.726 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:47.726 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:00:53.896599Z", 2022-09-03T07:03:47.727 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:03:47.728 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:47.728 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:47.728 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:47.729 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:03:47.729 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:47.729 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:47.730 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:03:47.731 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:47.732 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:47.739 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:47.740 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:03:47.740 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:47.741 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:47.741 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:47.742 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:47.742 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:03:47.743 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:47.743 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:47.744 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:03:47.744 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:47.745 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:47.746 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:47.746 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:03:47.747 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:47.747 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:47.747 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:47.747 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:03:47.748 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:47.749 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:03:47.749 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:03:47.749 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:03:47.749 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:47.750 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:03:47Z" 2022-09-03T07:03:47.750 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:03:47.751 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188453.896599_455.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:00:53.896599Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:03:47Z" } 2022-09-03T07:03:47.751 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188453.896599_455.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:00:53.896599Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:03:47.755 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188453.896599_455.1', 2: '', 3: ''} 2022-09-03T07:03:47.756 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188453.896599_455.1', 2: '', 3: ''} 2022-09-03T07:03:47.757 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T07:03:47.758 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T07:03:47.759 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:47.760 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:47.761 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:03:47.815 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:47.816 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:47.864 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:47.862+0000 7f25e67cc600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:47.866 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:47.863+0000 7f25e67cc600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:47.882 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:03:47.883 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:03:47.883 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:03:47.883 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:03:47.884 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:03:47.884 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:03:47.884 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:03:47.885 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:47.885 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:03:47.886 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:47.887 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:03:47.887 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:03:47.887 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:03:47.887 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:03:47.888 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:03:47.888 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T07:03:47.888 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:03:47.889 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:03:47.889 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:03:47.889 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:47.889 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:47.890 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:47.890 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:03:47.890 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:03:47.891 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:03:47.891 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662188453.896599_455.1", 2022-09-03T07:03:47.891 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:03:47.891 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:03:47.892 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:03:47.892 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:00:53.896599Z", 2022-09-03T07:03:47.892 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:03:47.892 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:47.893 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:47.893 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:47.893 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:03:47.894 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:03:47.894 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:03:47.894 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:47.894 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:03:47.895 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:03:47.895 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:03:47.895 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:03:47.896 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:03:47.897 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:47.897 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:47.897 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:47.898 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:03:47.898 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:03:47.898 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:03:47.898 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:47.899 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:03:47.899 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:03:47.899 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:03:47.899 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:03:47.900 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:03:47.900 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:47.900 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:47.901 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:03:47.901 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:47.901 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:03:47.901 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T07:03:47.902 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T07:03:47.902 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:47.902 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:03:47Z" 2022-09-03T07:03:47.902 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:03:47.904 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188453.896599_455.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:00:53.896599Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:03:47Z" } 2022-09-03T07:03:47.904 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188453.896599_455.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:00:53.896599Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:03:47.905 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188453.896599_455.1', 2: '', 3: ''} 2022-09-03T07:03:47.906 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188453.896599_455.1', 2: '', 3: ''} 2022-09-03T07:03:47.906 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T07:03:47.908 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T07:03:47.908 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:47.909 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:47.910 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:03:47.963 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:47.964 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:48.014 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:48.012+0000 7efe2ab23600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:48.016 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:48.014+0000 7efe2ab23600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:48.041 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:03:48.041 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:03:48.042 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:03:48.042 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:03:48.045 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:03:48.046 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:03:48.046 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:03:48.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.048 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:03:48.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.050 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:03:48.051 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.052 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.052 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:03:48.053 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.053 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:03:48.054 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.055 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:03:48.055 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:48.055 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.057 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:03:48.057 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.058 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.058 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662188453.896599_455.1", 2022-09-03T07:03:48.059 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.059 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:48.059 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.060 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:00:53.896599Z", 2022-09-03T07:03:48.060 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:03:48.061 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.061 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.061 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.062 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:03:48.062 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.062 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.063 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:03:48.063 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.064 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:48.064 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.065 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:03:48.065 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:48.066 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.067 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.067 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:03:48.068 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.068 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.068 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:03:48.068 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.069 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:48.069 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.069 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:03:48.070 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:48.070 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.071 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.071 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:03:48.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.072 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:03:48.072 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:03:48.072 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:03:48.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.073 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:03:48Z" 2022-09-03T07:03:48.074 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:03:48.075 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188453.896599_455.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:00:53.896599Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:03:48Z" } 2022-09-03T07:03:48.076 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188453.896599_455.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:00:53.896599Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:03:48.077 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188453.896599_455.1', 2: '', 3: ''} 2022-09-03T07:03:48.078 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188453.896599_455.1', 2: '', 3: ''} 2022-09-03T07:03:48.079 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T07:03:48.079 DEBUG:boto:path=/ 2022-09-03T07:03:48.080 DEBUG:boto:auth_path=/ 2022-09-03T07:03:48.080 DEBUG:boto:Method: GET 2022-09-03T07:03:48.081 DEBUG:boto:Path: / 2022-09-03T07:03:48.081 DEBUG:boto:Data: 2022-09-03T07:03:48.082 DEBUG:boto:Headers: {} 2022-09-03T07:03:48.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:48.083 DEBUG:boto:Port: 8000 2022-09-03T07:03:48.083 DEBUG:boto:Params: {} 2022-09-03T07:03:48.084 DEBUG:boto:Token: None 2022-09-03T07:03:48.085 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:48 GMT / 2022-09-03T07:03:48.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2h0mvIesH0LqUL2zRr56lBUuZ0= 2022-09-03T07:03:48.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2h0mvIesH0LqUL2zRr56lBUuZ0=', 'Content-Length': '0'} 2022-09-03T07:03:48.092 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000109e7ccce7ad4cf4-006312fc54-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:48.095 DEBUG:boto:path=/ 2022-09-03T07:03:48.096 DEBUG:boto:auth_path=/ 2022-09-03T07:03:48.097 DEBUG:boto:Method: GET 2022-09-03T07:03:48.098 DEBUG:boto:Path: / 2022-09-03T07:03:48.099 DEBUG:boto:Data: 2022-09-03T07:03:48.100 DEBUG:boto:Headers: {} 2022-09-03T07:03:48.100 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:03:48.101 DEBUG:boto:Port: 8000 2022-09-03T07:03:48.103 DEBUG:boto:Params: {} 2022-09-03T07:03:48.108 DEBUG:boto:Token: None 2022-09-03T07:03:48.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:48 GMT / 2022-09-03T07:03:48.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2h0mvIesH0LqUL2zRr56lBUuZ0= 2022-09-03T07:03:48.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2h0mvIesH0LqUL2zRr56lBUuZ0=', 'Content-Length': '0'} 2022-09-03T07:03:48.114 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e6911fa4e9285355-006312fc54-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:48.116 DEBUG:boto:path=/ 2022-09-03T07:03:48.117 DEBUG:boto:auth_path=/ 2022-09-03T07:03:48.117 DEBUG:boto:Method: GET 2022-09-03T07:03:48.118 DEBUG:boto:Path: / 2022-09-03T07:03:48.119 DEBUG:boto:Data: 2022-09-03T07:03:48.119 DEBUG:boto:Headers: {} 2022-09-03T07:03:48.120 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:03:48.120 DEBUG:boto:Port: 8001 2022-09-03T07:03:48.121 DEBUG:boto:Params: {} 2022-09-03T07:03:48.121 DEBUG:boto:Token: None 2022-09-03T07:03:48.122 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:03:48 GMT / 2022-09-03T07:03:48.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2h0mvIesH0LqUL2zRr56lBUuZ0= 2022-09-03T07:03:48.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2h0mvIesH0LqUL2zRr56lBUuZ0=', 'Content-Length': '0'} 2022-09-03T07:03:48.127 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006578a7a812ad1ee3-006312fc54-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:03:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:48.129 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_creation_time ... ok 2022-09-03T07:03:48.131 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:48.132 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:48.132 DEBUG:boto:Method: PUT 2022-09-03T07:03:48.133 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:48.133 DEBUG:boto:Data: 2022-09-03T07:03:48.134 DEBUG:boto:Headers: {} 2022-09-03T07:03:48.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:48.135 DEBUG:boto:Port: 8000 2022-09-03T07:03:48.136 DEBUG:boto:Params: {} 2022-09-03T07:03:48.137 DEBUG:boto:Token: None 2022-09-03T07:03:48.138 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 07:03:48 GMT /avysmw-74/ 2022-09-03T07:03:48.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ng+Qt+Bg7T1bIasGnLosHJgrvFk= 2022-09-03T07:03:48.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ng+Qt+Bg7T1bIasGnLosHJgrvFk=', 'Content-Length': '0'} 2022-09-03T07:03:48.162 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000b164ef936dc87e49-006312fc54-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:48.164 DEBUG:rgw_multi.tests:created bucket=avysmw-74 2022-09-03T07:03:48.164 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.165 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.166 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:03:48.218 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:48.219 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:48.256 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T07:03:48.276 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:03:48.276 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:48.277 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:48.278 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:03:48.279 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:48.279 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:48.280 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662188628.157492_459.1", 2022-09-03T07:03:48.280 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:03:48.157492Z" 2022-09-03T07:03:48.280 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:48.280 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:48.281 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:48.281 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:03:48.281 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:48.282 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:48.282 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:48.282 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:03:48.283 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:48.283 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:03:48.286 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662188628.157492_459.1', 2: '', 3: ''} 2022-09-03T07:03:48.287 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T07:03:48.288 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.289 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.290 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:03:48.345 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:48.346 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:48.397 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:48.395+0000 7fd8651e5600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:48.398 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:48.396+0000 7fd8651e5600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:48.419 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:03:48.421 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:03:48.421 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:03:48.422 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:03:48.422 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:03:48.423 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:03:48.423 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:03:48.424 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.425 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:03:48.425 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.426 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:03:48.426 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.427 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.427 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:03:48.428 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.428 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:03:48.428 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.429 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:03:48.429 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:48.430 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.432 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.432 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.433 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:03:48.433 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.433 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.434 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662188628.157492_459.1", 2022-09-03T07:03:48.434 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.434 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:48.435 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.435 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:03:48.157492Z", 2022-09-03T07:03:48.435 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:03:48.436 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.436 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.436 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.436 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:03:48.437 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.437 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.437 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:03:48.437 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.438 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:48.438 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.438 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:03:48.438 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:48.439 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.439 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.439 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:03:48.440 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.440 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.440 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:03:48.441 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.441 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:48.441 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.441 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:03:48.442 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:48.442 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.442 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.442 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:03:48.443 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.443 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:03:48.443 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:03:48.444 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:03:48.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.444 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:03:48Z" 2022-09-03T07:03:48.444 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:03:48.445 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188628.157492_459.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:03:48.157492Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:03:48Z" } 2022-09-03T07:03:48.446 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188628.157492_459.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:03:48.157492Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:03:48.446 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188628.157492_459.1', 2: '', 3: ''} 2022-09-03T07:03:48.447 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188628.157492_459.1', 2: '', 3: ''} 2022-09-03T07:03:48.447 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T07:03:48.448 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T07:03:48.448 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.449 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.450 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:03:48.501 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:48.502 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:48.547 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:48.543+0000 7fa4ce5d4600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:48.547 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:48.544+0000 7fa4ce5d4600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:48.571 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:03:48.572 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:03:48.572 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:03:48.572 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:03:48.572 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:03:48.573 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:03:48.573 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:03:48.574 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:48.575 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:03:48.575 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:48.576 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:03:48.577 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:03:48.577 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:03:48.577 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:03:48.578 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:03:48.578 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T07:03:48.579 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:03:48.579 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:03:48.580 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:03:48.580 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:48.580 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:48.580 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:48.581 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:03:48.581 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:03:48.581 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:03:48.582 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662188628.157492_459.1", 2022-09-03T07:03:48.582 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:03:48.582 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:03:48.582 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:03:48.583 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:03:48.157492Z", 2022-09-03T07:03:48.583 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:03:48.583 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:48.584 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:48.584 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:48.584 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:03:48.585 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:03:48.585 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:03:48.585 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:48.586 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:03:48.586 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:03:48.586 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:03:48.587 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:03:48.587 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:03:48.588 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:48.588 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:48.589 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:03:48.589 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:03:48.589 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:03:48.589 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:03:48.590 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:03:48.590 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:03:48.590 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:03:48.590 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:03:48.591 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:03:48.591 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:03:48.591 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:48.592 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:03:48.592 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:03:48.592 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:48.592 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:03:48.592 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T07:03:48.593 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T07:03:48.593 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:03:48.593 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:03:48Z" 2022-09-03T07:03:48.593 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:03:48.594 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188628.157492_459.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:03:48.157492Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:03:48Z" } 2022-09-03T07:03:48.595 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188628.157492_459.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:03:48.157492Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:03:48.595 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188628.157492_459.1', 2: '', 3: ''} 2022-09-03T07:03:48.596 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188628.157492_459.1', 2: '', 3: ''} 2022-09-03T07:03:48.597 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T07:03:48.598 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T07:03:48.598 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.599 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.600 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:03:48.650 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:48.651 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:48.684 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:48.682+0000 7fa0a0a9a600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:48.686 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:48.683+0000 7fa0a0a9a600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:03:48.713 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:03:48.714 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:03:48.715 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:03:48.715 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:03:48.716 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:03:48.716 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:03:48.716 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:03:48.717 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.717 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:03:48.717 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.718 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:03:48.719 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.719 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.720 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:03:48.720 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.721 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:03:48.722 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.722 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:03:48.723 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:48.723 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.723 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.724 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.724 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:03:48.725 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.725 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.725 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662188628.157492_459.1", 2022-09-03T07:03:48.726 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.726 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:48.727 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.727 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:03:48.157492Z", 2022-09-03T07:03:48.728 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:03:48.728 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.729 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.730 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.730 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:03:48.731 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.731 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.732 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:03:48.732 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.733 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:48.733 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.734 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:03:48.734 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:48.735 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.735 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.735 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:03:48.736 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:03:48.736 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:03:48.737 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:03:48.737 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:03:48.738 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:03:48.738 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:03:48.739 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:03:48.739 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:03:48.739 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:03:48.740 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.740 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:03:48.741 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:03:48.741 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.741 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:03:48.742 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:03:48.742 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:03:48.743 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:03:48.744 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:03:48Z" 2022-09-03T07:03:48.746 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:03:48.746 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662188628.157492_459.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:03:48.157492Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:03:48Z" } 2022-09-03T07:03:48.747 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662188628.157492_459.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:03:48.157492Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:03:48.749 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662188628.157492_459.1', 2: '', 3: ''} 2022-09-03T07:03:48.749 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662188628.157492_459.1', 2: '', 3: ''} 2022-09-03T07:03:48.750 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T07:03:48.750 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'shard', 'objects', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--num-shards', '11', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.751 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'shard', 'objects', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--num-shards', '11', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:03:48.752 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket shard objects --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --num-shards 11 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:03:48.805 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:48.805 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:48.854 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:03:48.854 INFO:teuthology.orchestra.run.smithi062.stdout: "objs": [ 2022-09-03T07:03:48.855 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000001", 2022-09-03T07:03:48.855 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000005", 2022-09-03T07:03:48.856 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000012", 2022-09-03T07:03:48.856 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000004", 2022-09-03T07:03:48.856 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000011", 2022-09-03T07:03:48.857 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000000", 2022-09-03T07:03:48.857 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000010", 2022-09-03T07:03:48.857 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000003", 2022-09-03T07:03:48.857 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000013", 2022-09-03T07:03:48.858 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000002", 2022-09-03T07:03:48.858 INFO:teuthology.orchestra.run.smithi062.stdout: "obj00000000000000000006" 2022-09-03T07:03:48.859 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:03:48.859 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:03:48.861 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:48.862 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:48.862 DEBUG:boto:Method: HEAD 2022-09-03T07:03:48.863 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:48.864 DEBUG:boto:Data: 2022-09-03T07:03:48.864 DEBUG:boto:Headers: {} 2022-09-03T07:03:48.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:48.866 DEBUG:boto:Port: 8000 2022-09-03T07:03:48.866 DEBUG:boto:Params: {} 2022-09-03T07:03:48.867 DEBUG:boto:Token: None 2022-09-03T07:03:48.868 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:48 GMT /avysmw-74/ 2022-09-03T07:03:48.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pLWXtfh63DExFdK1P1yU3vfKHew= 2022-09-03T07:03:48.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pLWXtfh63DExFdK1P1yU3vfKHew=', 'Content-Length': '0'} 2022-09-03T07:03:48.874 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000af1447057c3718dd-006312fc54-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:48.877 DEBUG:boto:path=/avysmw-74/obj00000000000000000006 2022-09-03T07:03:48.878 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000006 2022-09-03T07:03:48.879 DEBUG:boto:Method: PUT 2022-09-03T07:03:48.880 DEBUG:boto:Path: /avysmw-74/obj00000000000000000006 2022-09-03T07:03:48.882 DEBUG:boto:Data: 2022-09-03T07:03:48.883 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:48.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:48.885 DEBUG:boto:Port: 8000 2022-09-03T07:03:48.886 DEBUG:boto:Params: {} 2022-09-03T07:03:48.886 DEBUG:boto:Token: None 2022-09-03T07:03:48.887 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:48 GMT /avysmw-74/obj00000000000000000006 2022-09-03T07:03:48.888 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TEdQCrOu0eXPqR7kGTEspuUDTw0= 2022-09-03T07:03:48.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TEdQCrOu0eXPqR7kGTEspuUDTw0='} 2022-09-03T07:03:48.935 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000004754d7fe890efad4-006312fc54-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:48.936 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:48.936 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:48.937 DEBUG:boto:Method: HEAD 2022-09-03T07:03:48.940 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:48.941 DEBUG:boto:Data: 2022-09-03T07:03:48.942 DEBUG:boto:Headers: {} 2022-09-03T07:03:48.942 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:48.943 DEBUG:boto:Port: 8000 2022-09-03T07:03:48.943 DEBUG:boto:Params: {} 2022-09-03T07:03:48.944 DEBUG:boto:Token: None 2022-09-03T07:03:48.945 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:48 GMT /avysmw-74/ 2022-09-03T07:03:48.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pLWXtfh63DExFdK1P1yU3vfKHew= 2022-09-03T07:03:48.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pLWXtfh63DExFdK1P1yU3vfKHew=', 'Content-Length': '0'} 2022-09-03T07:03:48.949 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006cd5e7efeb869341-006312fc54-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:48.951 DEBUG:boto:path=/avysmw-74/obj00000000000000000013 2022-09-03T07:03:48.952 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000013 2022-09-03T07:03:48.952 DEBUG:boto:Method: PUT 2022-09-03T07:03:48.953 DEBUG:boto:Path: /avysmw-74/obj00000000000000000013 2022-09-03T07:03:48.954 DEBUG:boto:Data: 2022-09-03T07:03:48.954 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:48.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:48.961 DEBUG:boto:Port: 8000 2022-09-03T07:03:48.962 DEBUG:boto:Params: {} 2022-09-03T07:03:48.963 DEBUG:boto:Token: None 2022-09-03T07:03:48.964 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:48 GMT /avysmw-74/obj00000000000000000013 2022-09-03T07:03:48.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6vUSA2EeTCtcCILEBwe5nZiW+ro= 2022-09-03T07:03:48.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6vUSA2EeTCtcCILEBwe5nZiW+ro='} 2022-09-03T07:03:48.980 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f64126fa05cb00fe-006312fc54-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:48.983 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:48.984 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:48.985 DEBUG:boto:Method: HEAD 2022-09-03T07:03:48.986 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:48.986 DEBUG:boto:Data: 2022-09-03T07:03:48.987 DEBUG:boto:Headers: {} 2022-09-03T07:03:48.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:48.989 DEBUG:boto:Port: 8000 2022-09-03T07:03:48.990 DEBUG:boto:Params: {} 2022-09-03T07:03:48.991 DEBUG:boto:Token: None 2022-09-03T07:03:48.995 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:48 GMT /avysmw-74/ 2022-09-03T07:03:48.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pLWXtfh63DExFdK1P1yU3vfKHew= 2022-09-03T07:03:48.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pLWXtfh63DExFdK1P1yU3vfKHew=', 'Content-Length': '0'} 2022-09-03T07:03:49.001 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b670d665fa56f3cf-006312fc54-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.002 DEBUG:boto:path=/avysmw-74/obj00000000000000000000 2022-09-03T07:03:49.005 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000000 2022-09-03T07:03:49.009 DEBUG:boto:Method: PUT 2022-09-03T07:03:49.010 DEBUG:boto:Path: /avysmw-74/obj00000000000000000000 2022-09-03T07:03:49.011 DEBUG:boto:Data: 2022-09-03T07:03:49.012 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:49.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.014 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.015 DEBUG:boto:Params: {} 2022-09-03T07:03:49.018 DEBUG:boto:Token: None 2022-09-03T07:03:49.019 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/obj00000000000000000000 2022-09-03T07:03:49.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y3ea/e+fO0g+zN/O3sWFiof40Hw= 2022-09-03T07:03:49.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y3ea/e+fO0g+zN/O3sWFiof40Hw='} 2022-09-03T07:03:49.029 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f454a69971bc81a0-006312fc55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.031 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:49.032 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:49.033 DEBUG:boto:Method: HEAD 2022-09-03T07:03:49.036 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:49.037 DEBUG:boto:Data: 2022-09-03T07:03:49.038 DEBUG:boto:Headers: {} 2022-09-03T07:03:49.039 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.039 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.040 DEBUG:boto:Params: {} 2022-09-03T07:03:49.043 DEBUG:boto:Token: None 2022-09-03T07:03:49.048 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/ 2022-09-03T07:03:49.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA= 2022-09-03T07:03:49.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA=', 'Content-Length': '0'} 2022-09-03T07:03:49.052 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '9'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000017bf62f6ccf75a68-006312fc55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.054 DEBUG:boto:path=/avysmw-74/obj00000000000000000003 2022-09-03T07:03:49.054 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000003 2022-09-03T07:03:49.055 DEBUG:boto:Method: PUT 2022-09-03T07:03:49.055 DEBUG:boto:Path: /avysmw-74/obj00000000000000000003 2022-09-03T07:03:49.056 DEBUG:boto:Data: 2022-09-03T07:03:49.056 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:49.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.057 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.058 DEBUG:boto:Params: {} 2022-09-03T07:03:49.059 DEBUG:boto:Token: None 2022-09-03T07:03:49.060 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/obj00000000000000000003 2022-09-03T07:03:49.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/vDYG41LRAJz83rFXeNp6JAINWY= 2022-09-03T07:03:49.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/vDYG41LRAJz83rFXeNp6JAINWY='} 2022-09-03T07:03:49.110 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000036d53f997930aa69-006312fc55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.111 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:49.112 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:49.112 DEBUG:boto:Method: HEAD 2022-09-03T07:03:49.113 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:49.113 DEBUG:boto:Data: 2022-09-03T07:03:49.114 DEBUG:boto:Headers: {} 2022-09-03T07:03:49.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.115 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.116 DEBUG:boto:Params: {} 2022-09-03T07:03:49.117 DEBUG:boto:Token: None 2022-09-03T07:03:49.118 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/ 2022-09-03T07:03:49.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA= 2022-09-03T07:03:49.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA=', 'Content-Length': '0'} 2022-09-03T07:03:49.128 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000fa899d32573c1b38-006312fc55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.132 DEBUG:boto:path=/avysmw-74/obj00000000000000000010 2022-09-03T07:03:49.133 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000010 2022-09-03T07:03:49.133 DEBUG:boto:Method: PUT 2022-09-03T07:03:49.134 DEBUG:boto:Path: /avysmw-74/obj00000000000000000010 2022-09-03T07:03:49.134 DEBUG:boto:Data: 2022-09-03T07:03:49.135 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:49.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.136 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.136 DEBUG:boto:Params: {} 2022-09-03T07:03:49.137 DEBUG:boto:Token: None 2022-09-03T07:03:49.138 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/obj00000000000000000010 2022-09-03T07:03:49.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JSan3iqSisU3nucfcDFH0gsPOBo= 2022-09-03T07:03:49.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JSan3iqSisU3nucfcDFH0gsPOBo='} 2022-09-03T07:03:49.146 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000001a1c42ab024922dc-006312fc55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.147 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:49.147 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:49.148 DEBUG:boto:Method: HEAD 2022-09-03T07:03:49.148 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:49.149 DEBUG:boto:Data: 2022-09-03T07:03:49.149 DEBUG:boto:Headers: {} 2022-09-03T07:03:49.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.150 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.151 DEBUG:boto:Params: {} 2022-09-03T07:03:49.151 DEBUG:boto:Token: None 2022-09-03T07:03:49.152 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/ 2022-09-03T07:03:49.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA= 2022-09-03T07:03:49.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA=', 'Content-Length': '0'} 2022-09-03T07:03:49.157 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000dadb2284660538c7-006312fc55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.158 DEBUG:boto:path=/avysmw-74/obj00000000000000000002 2022-09-03T07:03:49.159 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000002 2022-09-03T07:03:49.159 DEBUG:boto:Method: PUT 2022-09-03T07:03:49.160 DEBUG:boto:Path: /avysmw-74/obj00000000000000000002 2022-09-03T07:03:49.160 DEBUG:boto:Data: 2022-09-03T07:03:49.161 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:49.161 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.162 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.163 DEBUG:boto:Params: {} 2022-09-03T07:03:49.163 DEBUG:boto:Token: None 2022-09-03T07:03:49.164 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/obj00000000000000000002 2022-09-03T07:03:49.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K37/6uGLk/vCGQEGlfF0sBJCtjc= 2022-09-03T07:03:49.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K37/6uGLk/vCGQEGlfF0sBJCtjc='} 2022-09-03T07:03:49.214 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000b6c90ff7394d0cb9-006312fc55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.215 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:49.217 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:49.217 DEBUG:boto:Method: HEAD 2022-09-03T07:03:49.218 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:49.218 DEBUG:boto:Data: 2022-09-03T07:03:49.219 DEBUG:boto:Headers: {} 2022-09-03T07:03:49.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.220 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.221 DEBUG:boto:Params: {} 2022-09-03T07:03:49.222 DEBUG:boto:Token: None 2022-09-03T07:03:49.222 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/ 2022-09-03T07:03:49.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA= 2022-09-03T07:03:49.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA=', 'Content-Length': '0'} 2022-09-03T07:03:49.227 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007af21959ecf6285e-006312fc55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.228 DEBUG:boto:path=/avysmw-74/obj00000000000000000012 2022-09-03T07:03:49.228 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000012 2022-09-03T07:03:49.229 DEBUG:boto:Method: PUT 2022-09-03T07:03:49.229 DEBUG:boto:Path: /avysmw-74/obj00000000000000000012 2022-09-03T07:03:49.230 DEBUG:boto:Data: 2022-09-03T07:03:49.230 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:49.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.232 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.232 DEBUG:boto:Params: {} 2022-09-03T07:03:49.233 DEBUG:boto:Token: None 2022-09-03T07:03:49.234 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/obj00000000000000000012 2022-09-03T07:03:49.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8u2OFfxqG2BwTeZDnmYEbwRXJPc= 2022-09-03T07:03:49.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8u2OFfxqG2BwTeZDnmYEbwRXJPc='} 2022-09-03T07:03:49.242 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000d5b5d58b52d98886-006312fc55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.243 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:49.246 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:49.246 DEBUG:boto:Method: HEAD 2022-09-03T07:03:49.247 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:49.247 DEBUG:boto:Data: 2022-09-03T07:03:49.248 DEBUG:boto:Headers: {} 2022-09-03T07:03:49.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.249 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.251 DEBUG:boto:Params: {} 2022-09-03T07:03:49.251 DEBUG:boto:Token: None 2022-09-03T07:03:49.252 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/ 2022-09-03T07:03:49.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA= 2022-09-03T07:03:49.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA=', 'Content-Length': '0'} 2022-09-03T07:03:49.256 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '21'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d59b3785ac4b0a54-006312fc55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.257 DEBUG:boto:path=/avysmw-74/obj00000000000000000011 2022-09-03T07:03:49.257 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000011 2022-09-03T07:03:49.258 DEBUG:boto:Method: PUT 2022-09-03T07:03:49.259 DEBUG:boto:Path: /avysmw-74/obj00000000000000000011 2022-09-03T07:03:49.259 DEBUG:boto:Data: 2022-09-03T07:03:49.260 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:49.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.261 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.261 DEBUG:boto:Params: {} 2022-09-03T07:03:49.262 DEBUG:boto:Token: None 2022-09-03T07:03:49.263 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/obj00000000000000000011 2022-09-03T07:03:49.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:upcUG61JMbjGndaCsFlMz6jGObk= 2022-09-03T07:03:49.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:upcUG61JMbjGndaCsFlMz6jGObk='} 2022-09-03T07:03:49.307 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000ac86ed8a6a9147b1-006312fc55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.308 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:49.309 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:49.310 DEBUG:boto:Method: HEAD 2022-09-03T07:03:49.311 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:49.312 DEBUG:boto:Data: 2022-09-03T07:03:49.313 DEBUG:boto:Headers: {} 2022-09-03T07:03:49.316 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.317 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.318 DEBUG:boto:Params: {} 2022-09-03T07:03:49.319 DEBUG:boto:Token: None 2022-09-03T07:03:49.320 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/ 2022-09-03T07:03:49.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA= 2022-09-03T07:03:49.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA=', 'Content-Length': '0'} 2022-09-03T07:03:49.326 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000abd982bd7ac980bd-006312fc55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.328 DEBUG:boto:path=/avysmw-74/obj00000000000000000004 2022-09-03T07:03:49.329 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000004 2022-09-03T07:03:49.330 DEBUG:boto:Method: PUT 2022-09-03T07:03:49.331 DEBUG:boto:Path: /avysmw-74/obj00000000000000000004 2022-09-03T07:03:49.331 DEBUG:boto:Data: 2022-09-03T07:03:49.333 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:49.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.335 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.336 DEBUG:boto:Params: {} 2022-09-03T07:03:49.337 DEBUG:boto:Token: None 2022-09-03T07:03:49.338 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/obj00000000000000000004 2022-09-03T07:03:49.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LOsX9qeMwDcazsPTMsFsc/XbYgs= 2022-09-03T07:03:49.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LOsX9qeMwDcazsPTMsFsc/XbYgs='} 2022-09-03T07:03:49.352 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000d9c1b31490a4b146-006312fc55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.354 DEBUG:boto:path=/avysmw-74/ 2022-09-03T07:03:49.355 DEBUG:boto:auth_path=/avysmw-74/ 2022-09-03T07:03:49.356 DEBUG:boto:Method: HEAD 2022-09-03T07:03:49.357 DEBUG:boto:Path: /avysmw-74/ 2022-09-03T07:03:49.358 DEBUG:boto:Data: 2022-09-03T07:03:49.359 DEBUG:boto:Headers: {} 2022-09-03T07:03:49.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.360 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.361 DEBUG:boto:Params: {} 2022-09-03T07:03:49.362 DEBUG:boto:Token: None 2022-09-03T07:03:49.364 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/ 2022-09-03T07:03:49.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA= 2022-09-03T07:03:49.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4cP6+2kDRpuCKu5GAlCMbW/aoTA=', 'Content-Length': '0'} 2022-09-03T07:03:49.371 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '9'), ('X-RGW-Bytes-Used', '27'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c397591c7c5cb0da-006312fc55-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.372 DEBUG:boto:path=/avysmw-74/obj00000000000000000001 2022-09-03T07:03:49.372 DEBUG:boto:auth_path=/avysmw-74/obj00000000000000000001 2022-09-03T07:03:49.373 DEBUG:boto:Method: PUT 2022-09-03T07:03:49.374 DEBUG:boto:Path: /avysmw-74/obj00000000000000000001 2022-09-03T07:03:49.374 DEBUG:boto:Data: 2022-09-03T07:03:49.375 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:03:49.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:03:49.376 DEBUG:boto:Port: 8000 2022-09-03T07:03:49.376 DEBUG:boto:Params: {} 2022-09-03T07:03:49.377 DEBUG:boto:Token: None 2022-09-03T07:03:49.378 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:03:49 GMT /avysmw-74/obj00000000000000000001 2022-09-03T07:03:49.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PcwV+7WL3McjqLuUuU+Ffhl4+4E= 2022-09-03T07:03:49.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:03:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PcwV+7WL3McjqLuUuU+Ffhl4+4E='} 2022-09-03T07:03:49.392 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000d0ccd07f29e8a85f-006312fc55-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:03:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:03:49.393 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-74 2022-09-03T07:03:49.394 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-74', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:49.394 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-74', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:49.395 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-74 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:03:49.462 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:49.463 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:49.515 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:49.513+0000 7fa375bfb600 1 waiting to reach incremental sync.. 2022-09-03T07:03:54.516 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:54.514+0000 7fa375bfb600 1 waiting to reach incremental sync.. 2022-09-03T07:03:59.521 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:59.518+0000 7fa375bfb600 1 bucket sync caught up with source: 2022-09-03T07:03:59.522 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:03:59.522 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:03:59.523 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:03:59.518+0000 7fa375bfb600 0 bucket checkpoint complete 2022-09-03T07:03:59.546 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-74 2022-09-03T07:03:59.547 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-74', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:59.548 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-74', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:03:59.550 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-74 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:03:59.629 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:03:59.630 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:03:59.659 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:03:59.657+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:04.665 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:04.658+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:09.660 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:09.658+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:14.662 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:14.659+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:19.663 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:19.660+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:24.663 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:24.660+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:29.663 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:29.660+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:34.663 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:34.660+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:39.664 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:39.661+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:44.664 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:44.661+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:49.669 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:49.663+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:54.665 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:54.664+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:04:59.666 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:04:59.664+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:04.666 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:04.665+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:09.667 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:09.665+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:14.671 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:14.666+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:19.668 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:19.667+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:24.669 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:24.667+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:29.670 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:29.667+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:34.671 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:34.668+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:39.674 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:39.668+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:44.673 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:44.669+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:49.673 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:49.670+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:54.673 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:54.671+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:05:59.674 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:05:59.672+0000 7fb24d3e9600 1 waiting to reach incremental sync.. 2022-09-03T07:06:04.676 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:06:04.674+0000 7fb24d3e9600 1 bucket sync caught up with source: 2022-09-03T07:06:04.677 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:04.677 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:04.677 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:06:04.674+0000 7fb24d3e9600 0 bucket checkpoint complete 2022-09-03T07:06:04.697 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-74 2022-09-03T07:06:04.699 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-74', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:06:04.701 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-74', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:06:04.704 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-74 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:06:04.865 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:06:04.866 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:06:04.929 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:04.927+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:04.929 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , , 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:04.930 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:09.930 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:09.928+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:09.931 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:09.931 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:14.932 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:14.930+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:14.933 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:14.933 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:19.933 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:19.931+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:19.933 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:19.933 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:24.934 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:24.932+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:24.935 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:24.935 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:29.935 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:29.933+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:29.936 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:29.937 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:34.937 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:34.934+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:34.937 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:34.938 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:39.938 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:39.936+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:39.939 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:39.939 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:44.939 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:44.937+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:44.940 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:44.941 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:49.941 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:49.938+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:49.941 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:49.942 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:54.942 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:54.940+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:54.942 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:54.943 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:59.943 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:06:59.941+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:06:59.945 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:06:59.945 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:04.944 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:04.942+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:04.945 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:04.945 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:09.948 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:09.944+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:09.949 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:09.949 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:14.948 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:14.945+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:14.949 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:14.950 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:19.949 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:19.946+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:19.950 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:19.951 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:24.950 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:24.947+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:24.951 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:24.951 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:29.957 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:29.948+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:29.958 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:29.958 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:34.959 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:34.950+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:34.959 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:34.962 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:39.961 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:39.951+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:39.962 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:39.962 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:44.965 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:44.952+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:44.966 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:44.966 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:49.956 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:49.953+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:49.957 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:49.957 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:54.964 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:54.954+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:54.964 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:54.965 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:59.960 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:07:59.957+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:07:59.961 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:07:59.961 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:04.961 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:04.959+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:04.962 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:04.962 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:09.963 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:09.960+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:09.964 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:09.965 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:14.963 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:14.961+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:14.964 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:14.964 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:19.965 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:19.963+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:19.966 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:19.966 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:24.966 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:24.964+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:29.968 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:29.965+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:29.969 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:29.969 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:34.987 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:34.967+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:34.987 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:34.988 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:39.970 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:39.968+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:39.971 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:39.972 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:44.971 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:44.969+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:44.972 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:44.972 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:49.973 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:49.971+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:49.973 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:49.974 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:54.974 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:54.972+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:54.975 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:54.976 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:59.975 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:08:59.974+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:08:59.976 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:08:59.976 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:04.977 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:04.975+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:04.978 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:04.978 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:09.978 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:09.976+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:09.979 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:09.980 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:14.987 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:14.977+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:14.989 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:14.997 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:19.981 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:19.979+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:19.982 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:19.982 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:24.985 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:24.981+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:24.985 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:24.985 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:29.987 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:29.982+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:29.988 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:29.988 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:34.985 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:34.983+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:34.985 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:34.986 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:39.986 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:39.984+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:39.987 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:39.987 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:44.988 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:44.986+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:44.989 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:44.989 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:49.989 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:49.987+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:49.990 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:49.991 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:54.994 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:54.988+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:54.995 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:54.995 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:59.992 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:09:59.990+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:09:59.993 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:09:59.993 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:04.994 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:04.991+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:04.995 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:04.996 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:09.998 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:09.992+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:09.999 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:09.999 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:14.997 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:14.994+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:14.997 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:14.997 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:20.003 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:19.995+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:20.003 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:20.004 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:24.999 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:24.996+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:24.999 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:25.000 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:30.004 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:29.998+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:30.004 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:30.005 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:35.001 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:34.999+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:35.002 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:35.002 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:40.005 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:40.000+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:40.006 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:40.006 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:45.004 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:45.002+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:45.005 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:45.005 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:50.027 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:50.003+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:50.028 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:50.028 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:55.006 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:10:55.004+0000 7f5cb2646600 1 waiting for incremental sync to catch up: 2022-09-03T07:10:55.007 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.62.5, , , 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:10:55.007 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.62.5, , 00000000001.66.5, 00000000001.68.5, 00000000001.87.5, 00000000001.55.5, 00000000001.54.5, 00000000001.85.5, 00000000001.60.5, 00000000001.66.5, 00000000001.75.5] 2022-09-03T07:11:00.009 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:00.006+0000 7f5cb2646600 0 bucket checkpoint timed out waiting for incremental sync to catch up 2022-09-03T07:11:00.009 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:00.006+0000 7f5cb2646600 0 bucket sync checkpoint failed: (110) Connection timed out 2022-09-03T07:11:00.009 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:00.006+0000 7f5cb2646600 -1 bucket sync checkpoint failed: (110) Connection timed out 2022-09-03T07:11:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 110 2022-09-03T07:11:00.041 INFO:tasks.rgw_multisite_tests:Create several generations of objects, then run bucket sync ... FAIL 2022-09-03T07:11:00.044 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:11:00.044 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:11:00.051 DEBUG:boto:Method: PUT 2022-09-03T07:11:00.055 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:11:00.056 DEBUG:boto:Data: 2022-09-03T07:11:00.056 DEBUG:boto:Headers: {} 2022-09-03T07:11:00.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:00.057 DEBUG:boto:Port: 8000 2022-09-03T07:11:00.058 DEBUG:boto:Params: {} 2022-09-03T07:11:00.060 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:11:00.060 DEBUG:boto:Token: None 2022-09-03T07:11:00.061 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 07:11:00 GMT /avysmw-75/ 2022-09-03T07:11:00.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZIj6gyIC28zI0ntRUSd+hSvcW0M= 2022-09-03T07:11:00.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:11:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZIj6gyIC28zI0ntRUSd+hSvcW0M=', 'Content-Length': '0'} 2022-09-03T07:11:00.111 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000024648fb4fa9bd1f2-006312fe04-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:00.113 DEBUG:rgw_multi.tests:created bucket=avysmw-75 2022-09-03T07:11:00.114 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:11:00.115 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:11:00.121 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:11:00.187 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:11:00.189 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:11:00.217 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T07:11:00.232 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:11:00.232 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:11:00.233 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:11:00.233 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:11:00.233 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:11:00.234 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:11:00.234 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662189060.099677_469.1", 2022-09-03T07:11:00.234 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:11:00.099677Z" 2022-09-03T07:11:00.235 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:11:00.235 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:11:00.235 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:11:00.235 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:11:00.236 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:11:00.236 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:11:00.236 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:11:00.236 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:11:00.237 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:11:00.237 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:11:00.238 DEBUG:rgw_multi.tests:master meta markers={0: '', 1: '1_1662189060.099677_469.1', 2: '', 3: ''} 2022-09-03T07:11:00.239 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T07:11:00.240 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:11:00.242 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:11:00.244 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:11:00.406 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:11:00.407 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:11:00.462 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:00.457+0000 7f5455c4b600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:11:00.463 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:00.458+0000 7f5455c4b600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:11:00.475 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:11:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:11:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:11:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:11:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:11:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:11:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:11:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.478 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:11:00.478 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:11:00.478 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:11:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:11:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:11:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:11:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:11:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:11:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:11:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:11:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:11:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:11:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:11:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:11:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:11:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662189060.099677_469.1", 2022-09-03T07:11:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:11:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:11:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:11:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:11:00.099677Z", 2022-09-03T07:11:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:11:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:11:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:11:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:11:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:11:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:11:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:11:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:11:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:11:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:11:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:11:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:11:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:11:00.507 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:11:00.508 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:11:00.508 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:11:00.508 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:11:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:11:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:11:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:11:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:11:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:11:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:11:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:11:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:11:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:11:00Z" 2022-09-03T07:11:00.513 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:11:00.514 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662189060.099677_469.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:11:00.099677Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:11:00Z" } 2022-09-03T07:11:00.514 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662189060.099677_469.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:11:00.099677Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:11:00.515 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662189060.099677_469.1', 2: '', 3: ''} 2022-09-03T07:11:00.516 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662189060.099677_469.1', 2: '', 3: ''} 2022-09-03T07:11:00.516 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T07:11:00.517 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T07:11:00.517 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:11:00.518 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:11:00.519 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:11:00.577 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:11:00.577 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:11:00.643 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:00.640+0000 7fa890df8600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:11:00.644 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:00.641+0000 7fa890df8600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:11:00.659 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:11:00.660 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:11:00.661 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:11:00.661 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:11:00.663 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:11:00.664 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:11:00.671 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:11:00.672 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:11:00.672 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:11:00.672 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:11:00.673 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:11:00.675 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:11:00.675 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:11:00.676 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:11:00.676 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:11:00.677 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T07:11:00.677 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:11:00.677 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:11:00.677 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:11:00.678 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:11:00.678 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:11:00.678 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:11:00.679 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:11:00.679 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:11:00.680 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:11:00.680 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662189060.099677_469.1", 2022-09-03T07:11:00.680 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:11:00.680 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:11:00.681 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:11:00.681 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:11:00.099677Z", 2022-09-03T07:11:00.681 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:11:00.681 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:11:00.682 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:11:00.682 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:11:00.682 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:11:00.683 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:11:00.683 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:11:00.683 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:11:00.683 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:11:00.684 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:11:00.684 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:11:00.684 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:11:00.685 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:11:00.685 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:11:00.685 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:11:00.686 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:11:00.686 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:11:00.686 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:11:00.687 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:11:00.687 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:11:00.687 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:11:00.688 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:11:00.688 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:11:00.688 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:11:00.689 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:11:00.689 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:11:00.689 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:11:00.690 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:11:00.690 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:11:00.691 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:11:00.691 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T07:11:00.691 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T07:11:00.691 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:11:00.692 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:11:00Z" 2022-09-03T07:11:00.692 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:11:00.694 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662189060.099677_469.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:11:00.099677Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:11:00Z" } 2022-09-03T07:11:00.694 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662189060.099677_469.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:11:00.099677Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:11:00.695 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662189060.099677_469.1', 2: '', 3: ''} 2022-09-03T07:11:00.696 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662189060.099677_469.1', 2: '', 3: ''} 2022-09-03T07:11:00.696 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T07:11:00.697 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T07:11:00.697 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:11:00.698 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:11:00.699 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:11:00.752 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:11:00.752 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:11:00.805 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:00.801+0000 7f328e3e3600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:11:00.805 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:00.802+0000 7f328e3e3600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:11:00.821 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:11:00.821 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:11:00.821 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:11:00.829 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:11:00.830 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:11:00.830 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:11:00.831 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:11:00.831 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.831 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:11:00.831 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:11:00.832 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:11:00.832 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:11:00.832 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:11:00.832 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:11:00.833 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:11:00.833 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:11:00.834 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:11:00.851 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:11:00.852 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:11:00.853 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.853 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.854 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:11:00.854 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:11:00.855 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:11:00.855 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:11:00.856 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662189060.099677_469.1", 2022-09-03T07:11:00.856 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:11:00.857 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:11:00.857 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:11:00.858 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:11:00.099677Z", 2022-09-03T07:11:00.859 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:11:00.859 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.860 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.860 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:11:00.861 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:11:00.861 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:11:00.870 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:11:00.871 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:11:00.871 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:11:00.872 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:11:00.872 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:11:00.873 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:11:00.873 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:11:00.873 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.883 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.884 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:11:00.884 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:11:00.885 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:11:00.885 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:11:00.886 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:11:00.886 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:11:00.887 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:11:00.887 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:11:00.887 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:11:00.888 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:11:00.888 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.889 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:11:00.889 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:11:00.889 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.890 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:11:00.891 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:11:00.892 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:11:00.893 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:11:00.894 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:11:00Z" 2022-09-03T07:11:00.894 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:11:00.895 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662189060.099677_469.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:11:00.099677Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:11:00Z" } 2022-09-03T07:11:00.896 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662189060.099677_469.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:11:00.099677Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:11:00.897 DEBUG:rgw_multi.tests:log_status={0: '', 1: '1_1662189060.099677_469.1', 2: '', 3: ''} 2022-09-03T07:11:00.897 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662189060.099677_469.1', 2: '', 3: ''} 2022-09-03T07:11:00.898 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T07:11:00.898 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:11:00.899 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:11:00.899 DEBUG:boto:Method: HEAD 2022-09-03T07:11:00.900 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:11:00.900 DEBUG:boto:Data: 2022-09-03T07:11:00.901 DEBUG:boto:Headers: {} 2022-09-03T07:11:00.902 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:00.902 DEBUG:boto:Port: 8000 2022-09-03T07:11:00.903 DEBUG:boto:Params: {} 2022-09-03T07:11:00.903 DEBUG:boto:Token: None 2022-09-03T07:11:00.904 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:11:00 GMT /avysmw-75/ 2022-09-03T07:11:00.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1UvORSoUlGmWu7jXdyPjXLT570g= 2022-09-03T07:11:00.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:11:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1UvORSoUlGmWu7jXdyPjXLT570g=', 'Content-Length': '0'} 2022-09-03T07:11:00.908 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000008e7317a612647b1-006312fe04-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:00.909 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:11:00.909 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:11:00.910 DEBUG:boto:Method: PUT 2022-09-03T07:11:00.910 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:11:00.911 DEBUG:boto:Data: 2022-09-03T07:11:00.912 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:11:00.913 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:00.913 DEBUG:boto:Port: 8000 2022-09-03T07:11:00.914 DEBUG:boto:Params: {} 2022-09-03T07:11:00.915 DEBUG:boto:Token: None 2022-09-03T07:11:00.915 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:11:00 GMT /avysmw-75/obj11 2022-09-03T07:11:00.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ucc7KWth9qJSIOLfw1s3uyky/hE= 2022-09-03T07:11:00.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:11:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ucc7KWth9qJSIOLfw1s3uyky/hE='} 2022-09-03T07:11:00.926 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000c9deb6cf51f02463-006312fe04-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:00.927 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:11:00.927 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:11:00.928 DEBUG:boto:Method: HEAD 2022-09-03T07:11:00.928 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:11:00.929 DEBUG:boto:Data: 2022-09-03T07:11:00.929 DEBUG:boto:Headers: {} 2022-09-03T07:11:00.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:00.930 DEBUG:boto:Port: 8000 2022-09-03T07:11:00.931 DEBUG:boto:Params: {} 2022-09-03T07:11:00.932 DEBUG:boto:Token: None 2022-09-03T07:11:00.932 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:11:00 GMT /avysmw-75/ 2022-09-03T07:11:00.933 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1UvORSoUlGmWu7jXdyPjXLT570g= 2022-09-03T07:11:00.933 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:11:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1UvORSoUlGmWu7jXdyPjXLT570g=', 'Content-Length': '0'} 2022-09-03T07:11:00.936 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000b9bb4d5088b9220-006312fe04-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:00.937 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:11:00.938 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:11:00.938 DEBUG:boto:Method: PUT 2022-09-03T07:11:00.939 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:11:00.939 DEBUG:boto:Data: 2022-09-03T07:11:00.940 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:11:00.941 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:00.941 DEBUG:boto:Port: 8000 2022-09-03T07:11:00.942 DEBUG:boto:Params: {} 2022-09-03T07:11:00.947 DEBUG:boto:Token: None 2022-09-03T07:11:00.948 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:11:00 GMT /avysmw-75/obj22 2022-09-03T07:11:00.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ti+tZfw+e20zFohHE171BPH6f6k= 2022-09-03T07:11:00.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:11:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ti+tZfw+e20zFohHE171BPH6f6k='} 2022-09-03T07:11:00.956 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000aca4ee634dac918a-006312fe04-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:00.957 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:11:00.958 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:11:00.958 DEBUG:boto:Method: HEAD 2022-09-03T07:11:00.959 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:11:00.959 DEBUG:boto:Data: 2022-09-03T07:11:00.960 DEBUG:boto:Headers: {} 2022-09-03T07:11:00.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:00.961 DEBUG:boto:Port: 8000 2022-09-03T07:11:00.961 DEBUG:boto:Params: {} 2022-09-03T07:11:00.962 DEBUG:boto:Token: None 2022-09-03T07:11:00.963 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:11:00 GMT /avysmw-75/ 2022-09-03T07:11:00.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1UvORSoUlGmWu7jXdyPjXLT570g= 2022-09-03T07:11:00.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:11:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1UvORSoUlGmWu7jXdyPjXLT570g=', 'Content-Length': '0'} 2022-09-03T07:11:00.967 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e1b085149123af0f-006312fe04-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:00.967 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:11:00.968 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:11:00.969 DEBUG:boto:Method: PUT 2022-09-03T07:11:00.969 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:11:00.970 DEBUG:boto:Data: 2022-09-03T07:11:00.971 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:11:00.971 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:00.972 DEBUG:boto:Port: 8000 2022-09-03T07:11:00.973 DEBUG:boto:Params: {} 2022-09-03T07:11:00.973 DEBUG:boto:Token: None 2022-09-03T07:11:00.974 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:11:00 GMT /avysmw-75/obj33 2022-09-03T07:11:00.974 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jq9OHVaOmtA30ex7WshKD6wyCLQ= 2022-09-03T07:11:00.975 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:11:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jq9OHVaOmtA30ex7WshKD6wyCLQ='} 2022-09-03T07:11:01.024 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000005f8e5bb7f81713cc-006312fe04-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:01.025 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:11:01.025 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:11:01.026 DEBUG:boto:Method: HEAD 2022-09-03T07:11:01.026 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:11:01.028 DEBUG:boto:Data: 2022-09-03T07:11:01.028 DEBUG:boto:Headers: {} 2022-09-03T07:11:01.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:01.029 DEBUG:boto:Port: 8000 2022-09-03T07:11:01.036 DEBUG:boto:Params: {} 2022-09-03T07:11:01.037 DEBUG:boto:Token: None 2022-09-03T07:11:01.039 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:11:01 GMT /avysmw-75/ 2022-09-03T07:11:01.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9T5n1hleAzxkKbh8bT7DDvkLgc= 2022-09-03T07:11:01.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:11:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9T5n1hleAzxkKbh8bT7DDvkLgc=', 'Content-Length': '0'} 2022-09-03T07:11:01.047 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '9'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000031db65702e5458cf-006312fe05-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:01.049 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:11:01.050 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:11:01.051 DEBUG:boto:Method: PUT 2022-09-03T07:11:01.052 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:11:01.053 DEBUG:boto:Data: 2022-09-03T07:11:01.053 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:11:01.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:01.054 DEBUG:boto:Port: 8000 2022-09-03T07:11:01.055 DEBUG:boto:Params: {} 2022-09-03T07:11:01.056 DEBUG:boto:Token: None 2022-09-03T07:11:01.056 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:11:01 GMT /avysmw-75/obj44 2022-09-03T07:11:01.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9C7Ac3BO04t0VOtPs/QGT2wRofQ= 2022-09-03T07:11:01.057 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:11:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9C7Ac3BO04t0VOtPs/QGT2wRofQ='} 2022-09-03T07:11:01.073 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000034e2900db9358797-006312fe05-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:01.074 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:11:01.074 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:11:01.075 DEBUG:boto:Method: HEAD 2022-09-03T07:11:01.075 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:11:01.076 DEBUG:boto:Data: 2022-09-03T07:11:01.076 DEBUG:boto:Headers: {} 2022-09-03T07:11:01.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:01.077 DEBUG:boto:Port: 8000 2022-09-03T07:11:01.079 DEBUG:boto:Params: {} 2022-09-03T07:11:01.080 DEBUG:boto:Token: None 2022-09-03T07:11:01.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:11:01 GMT /avysmw-75/ 2022-09-03T07:11:01.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g9T5n1hleAzxkKbh8bT7DDvkLgc= 2022-09-03T07:11:01.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:11:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g9T5n1hleAzxkKbh8bT7DDvkLgc=', 'Content-Length': '0'} 2022-09-03T07:11:01.085 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000093fa9f0085d19af9-006312fe05-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:01.086 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:11:01.086 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:11:01.089 DEBUG:boto:Method: PUT 2022-09-03T07:11:01.091 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:11:01.092 DEBUG:boto:Data: 2022-09-03T07:11:01.092 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:11:01.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:11:01.093 DEBUG:boto:Port: 8000 2022-09-03T07:11:01.093 DEBUG:boto:Params: {} 2022-09-03T07:11:01.094 DEBUG:boto:Token: None 2022-09-03T07:11:01.095 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:11:01 GMT /avysmw-75/obj55 2022-09-03T07:11:01.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tSB6TPUIz/uKD2D8krRRpltihGU= 2022-09-03T07:11:01.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:11:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tSB6TPUIz/uKD2D8krRRpltihGU='} 2022-09-03T07:11:01.112 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000004ea8afaf0eae43f-006312fe05-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:11:01.113 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-75 2022-09-03T07:11:01.114 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:01.115 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:01.116 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:11:01.177 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:11:01.177 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:11:01.226 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:01.223+0000 7f45f3755600 1 waiting to reach incremental sync.. 2022-09-03T07:11:06.226 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:06.224+0000 7f45f3755600 1 waiting to reach incremental sync.. 2022-09-03T07:11:11.232 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:11.224+0000 7f45f3755600 1 waiting to reach incremental sync.. 2022-09-03T07:11:16.228 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:16.225+0000 7f45f3755600 1 bucket sync caught up with source: 2022-09-03T07:11:16.228 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.86.5, 00000000002.50.5, 00000000002.51.5] 2022-09-03T07:11:16.229 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.86.5, 00000000002.50.5, 00000000002.51.5] 2022-09-03T07:11:16.229 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:16.225+0000 7f45f3755600 0 bucket checkpoint complete 2022-09-03T07:11:16.265 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-75 2022-09-03T07:11:16.266 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:16.267 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:16.268 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:11:16.339 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:11:16.340 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:11:16.386 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:16.382+0000 7f81743c7600 1 waiting to reach incremental sync.. 2022-09-03T07:11:21.384 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:21.382+0000 7f81743c7600 1 waiting to reach incremental sync.. 2022-09-03T07:11:26.396 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:26.383+0000 7f81743c7600 1 bucket sync caught up with source: 2022-09-03T07:11:26.397 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.86.5, 00000000002.50.5, 00000000002.51.5] 2022-09-03T07:11:26.397 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.86.5, 00000000002.50.5, 00000000002.51.5] 2022-09-03T07:11:26.397 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:26.383+0000 7f81743c7600 0 bucket checkpoint complete 2022-09-03T07:11:26.411 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-75 2022-09-03T07:11:26.412 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:26.413 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:26.415 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:11:26.491 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:11:26.492 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:11:26.546 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:26.538+0000 7f149b299600 1 bucket sync caught up with source: 2022-09-03T07:11:26.546 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.86.5, 00000000002.50.5, 00000000002.51.5] 2022-09-03T07:11:26.547 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.86.5, 00000000002.50.5, 00000000002.51.5] 2022-09-03T07:11:26.547 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:11:26.538+0000 7f149b299600 0 bucket checkpoint complete 2022-09-03T07:11:26.586 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-75 2022-09-03T07:11:26.587 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:26.588 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:26.589 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:11:26.654 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:11:26.654 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:11:26.714 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:26.712+0000 7f255effe600 1 bucket sync caught up with source: 2022-09-03T07:11:26.715 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.370.5, 00000000002.291.5, 00000000002.211.5] 2022-09-03T07:11:26.715 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.370.5, 00000000002.291.5, 00000000002.211.5] 2022-09-03T07:11:26.715 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:26.712+0000 7f255effe600 0 bucket checkpoint complete 2022-09-03T07:11:26.735 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-75 2022-09-03T07:11:26.736 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:26.737 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:11:26.737 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:11:26.810 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:11:26.810 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:11:26.837 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:26.833+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:11:31.838 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:31.833+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:11:36.837 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:36.834+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:11:41.838 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:41.835+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:11:46.839 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:46.836+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:11:51.845 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:51.837+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:11:56.839 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:11:56.837+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:12:01.840 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:01.838+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:12:06.841 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:06.839+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:12:11.845 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:11.839+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:12:16.842 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:16.840+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:12:21.843 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:21.840+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:12:26.842 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:26.840+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:12:31.844 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:31.841+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:12:36.844 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:36.841+0000 7fe8ce307600 1 waiting to reach incremental sync.. 2022-09-03T07:12:41.848 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:41.843+0000 7fe8ce307600 1 bucket sync caught up with source: 2022-09-03T07:12:41.849 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.370.5, 00000000002.291.5, 00000000002.211.5] 2022-09-03T07:12:41.849 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.370.5, 00000000002.291.5, 00000000002.211.5] 2022-09-03T07:12:41.850 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:41.843+0000 7fe8ce307600 0 bucket checkpoint complete 2022-09-03T07:12:41.879 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-75 2022-09-03T07:12:41.881 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-75 2022-09-03T07:12:41.882 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:12:41.884 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:12:41.885 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:12:41.962 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:12:41.963 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:12:42.016 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:42.013+0000 7f8a1349f600 1 bucket sync caught up with source: 2022-09-03T07:12:42.017 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.279.5, 00000000002.298.5, 00000000002.127.5] 2022-09-03T07:12:42.017 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.279.5, 00000000002.298.5, 00000000002.127.5] 2022-09-03T07:12:42.018 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:12:42.013+0000 7f8a1349f600 0 bucket checkpoint complete 2022-09-03T07:12:42.041 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-75 2022-09-03T07:12:42.044 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:12:42.045 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:12:42.046 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:12:42.380 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:12:42.381 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:12:42.435 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:12:42.433+0000 7ff2bb33c600 1 bucket sync caught up with source: 2022-09-03T07:12:42.435 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.279.5, 00000000002.298.5, 00000000002.127.5] 2022-09-03T07:12:42.435 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.279.5, 00000000002.298.5, 00000000002.127.5] 2022-09-03T07:12:42.436 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:12:42.433+0000 7ff2bb33c600 0 bucket checkpoint complete 2022-09-03T07:12:42.470 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-75 2022-09-03T07:12:42.471 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:12:42.476 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.476 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.477 DEBUG:boto:Method: HEAD 2022-09-03T07:12:42.477 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:42.478 DEBUG:boto:Data: 2022-09-03T07:12:42.479 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.480 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.481 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.481 DEBUG:boto:Params: {} 2022-09-03T07:12:42.482 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:12:42.488 DEBUG:boto:Token: None 2022-09-03T07:12:42.489 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/ 2022-09-03T07:12:42.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cE0QAi9g1Zomul9cKKB/TRxbmpk= 2022-09-03T07:12:42.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cE0QAi9g1Zomul9cKKB/TRxbmpk=', 'Content-Length': '0'} 2022-09-03T07:12:42.511 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e4d80ee3b6167521-006312fe6a-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.512 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.512 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.513 DEBUG:boto:Method: HEAD 2022-09-03T07:12:42.513 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:42.514 DEBUG:boto:Data: 2022-09-03T07:12:42.515 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.516 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.516 DEBUG:boto:Params: {} 2022-09-03T07:12:42.517 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:12:42.518 DEBUG:boto:Token: None 2022-09-03T07:12:42.518 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/ 2022-09-03T07:12:42.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cE0QAi9g1Zomul9cKKB/TRxbmpk= 2022-09-03T07:12:42.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cE0QAi9g1Zomul9cKKB/TRxbmpk=', 'Content-Length': '0'} 2022-09-03T07:12:42.540 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bad6d79d7272b8dd-006312fe6a-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.544 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:12:42.545 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.545 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.546 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.547 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.547 DEBUG:boto:Method: GET 2022-09-03T07:12:42.548 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.549 DEBUG:boto:Data: 2022-09-03T07:12:42.549 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.550 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.550 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.551 DEBUG:boto:Params: {} 2022-09-03T07:12:42.552 DEBUG:boto:Token: None 2022-09-03T07:12:42.552 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/?versions 2022-09-03T07:12:42.553 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AUwDbZo8ZNsuJfthGMhILaVte/4= 2022-09-03T07:12:42.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AUwDbZo8ZNsuJfthGMhILaVte/4=', 'Content-Length': '0'} 2022-09-03T07:12:42.558 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003824981df46c0f8e-006312fe6a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.558 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:42.562 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:12:42.569 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:12:42.570 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:12:42.570 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:12:42.571 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:12:42.571 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:12:42.572 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.573 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.573 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.574 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.574 DEBUG:boto:Method: GET 2022-09-03T07:12:42.575 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.575 DEBUG:boto:Data: 2022-09-03T07:12:42.576 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.577 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.578 DEBUG:boto:Params: {} 2022-09-03T07:12:42.579 DEBUG:boto:Token: None 2022-09-03T07:12:42.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/?versions 2022-09-03T07:12:42.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AUwDbZo8ZNsuJfthGMhILaVte/4= 2022-09-03T07:12:42.581 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AUwDbZo8ZNsuJfthGMhILaVte/4=', 'Content-Length': '0'} 2022-09-03T07:12:42.591 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000892aa3d2782a6a96-006312fe6a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.592 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:42.594 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:12:42.595 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:12:42.595 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:12:42.595 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:12:42.596 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:12:42.596 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.597 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.598 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.598 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.603 DEBUG:boto:Method: GET 2022-09-03T07:12:42.613 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.614 DEBUG:boto:Data: 2022-09-03T07:12:42.615 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.616 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.617 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.618 DEBUG:boto:Params: {} 2022-09-03T07:12:42.619 DEBUG:boto:Token: None 2022-09-03T07:12:42.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/?versions 2022-09-03T07:12:42.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AUwDbZo8ZNsuJfthGMhILaVte/4= 2022-09-03T07:12:42.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AUwDbZo8ZNsuJfthGMhILaVte/4=', 'Content-Length': '0'} 2022-09-03T07:12:42.638 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd7144b738eb99a5-006312fe6a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.639 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:42.641 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.644 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.655 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.656 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.657 DEBUG:boto:Method: GET 2022-09-03T07:12:42.658 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:42.659 DEBUG:boto:Data: 2022-09-03T07:12:42.660 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.671 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.672 DEBUG:boto:Params: {} 2022-09-03T07:12:42.673 DEBUG:boto:Token: None 2022-09-03T07:12:42.690 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/?versions 2022-09-03T07:12:42.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AUwDbZo8ZNsuJfthGMhILaVte/4= 2022-09-03T07:12:42.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AUwDbZo8ZNsuJfthGMhILaVte/4=', 'Content-Length': '0'} 2022-09-03T07:12:42.706 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f5af50ba3b00938e-006312fe6a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.707 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:42.708 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:12:42.710 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.718 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.719 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.720 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.720 DEBUG:boto:Method: GET 2022-09-03T07:12:42.721 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.721 DEBUG:boto:Data: 2022-09-03T07:12:42.722 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.722 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.723 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.723 DEBUG:boto:Params: {} 2022-09-03T07:12:42.724 DEBUG:boto:Token: None 2022-09-03T07:12:42.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dinDJXiBtcS/BLdEwY1Fl/u8eRY= 2022-09-03T07:12:42.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dinDJXiBtcS/BLdEwY1Fl/u8eRY=', 'Content-Length': '0'} 2022-09-03T07:12:42.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078a3ca9c30086aba-006312fe6a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.733 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.733 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.734 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.735 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.735 DEBUG:boto:Method: GET 2022-09-03T07:12:42.736 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.737 DEBUG:boto:Data: 2022-09-03T07:12:42.737 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.739 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.739 DEBUG:boto:Params: {} 2022-09-03T07:12:42.740 DEBUG:boto:Token: None 2022-09-03T07:12:42.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dinDJXiBtcS/BLdEwY1Fl/u8eRY= 2022-09-03T07:12:42.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dinDJXiBtcS/BLdEwY1Fl/u8eRY=', 'Content-Length': '0'} 2022-09-03T07:12:42.745 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002251742d19ff4281-006312fe6a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.746 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.747 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.748 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.748 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.749 DEBUG:boto:Method: HEAD 2022-09-03T07:12:42.749 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.750 DEBUG:boto:Data: 2022-09-03T07:12:42.750 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.751 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.751 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.752 DEBUG:boto:Params: {} 2022-09-03T07:12:42.752 DEBUG:boto:Token: None 2022-09-03T07:12:42.753 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4eQvInd/+BO+/UShFTP05vX6V/g= 2022-09-03T07:12:42.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4eQvInd/+BO+/UShFTP05vX6V/g=', 'Content-Length': '0'} 2022-09-03T07:12:42.757 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004fde65dbe46a5e1d-006312fe6a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.758 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.759 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.760 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.761 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.761 DEBUG:boto:Method: HEAD 2022-09-03T07:12:42.762 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.762 DEBUG:boto:Data: 2022-09-03T07:12:42.763 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.764 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.764 DEBUG:boto:Params: {} 2022-09-03T07:12:42.765 DEBUG:boto:Token: None 2022-09-03T07:12:42.766 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4eQvInd/+BO+/UShFTP05vX6V/g= 2022-09-03T07:12:42.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4eQvInd/+BO+/UShFTP05vX6V/g=', 'Content-Length': '0'} 2022-09-03T07:12:42.770 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3ce44f206ea8993-006312fe6a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.771 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:12:42.772 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.772 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.773 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.774 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.774 DEBUG:boto:Method: GET 2022-09-03T07:12:42.775 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.775 DEBUG:boto:Data: 2022-09-03T07:12:42.776 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.777 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.777 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.778 DEBUG:boto:Params: {} 2022-09-03T07:12:42.779 DEBUG:boto:Token: None 2022-09-03T07:12:42.779 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dinDJXiBtcS/BLdEwY1Fl/u8eRY= 2022-09-03T07:12:42.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dinDJXiBtcS/BLdEwY1Fl/u8eRY=', 'Content-Length': '0'} 2022-09-03T07:12:42.783 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c04644db899fb78-006312fe6a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.784 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.784 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.785 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.785 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.786 DEBUG:boto:Method: GET 2022-09-03T07:12:42.787 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.788 DEBUG:boto:Data: 2022-09-03T07:12:42.789 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.789 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.790 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.790 DEBUG:boto:Params: {} 2022-09-03T07:12:42.791 DEBUG:boto:Token: None 2022-09-03T07:12:42.792 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:42.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dinDJXiBtcS/BLdEwY1Fl/u8eRY= 2022-09-03T07:12:42.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dinDJXiBtcS/BLdEwY1Fl/u8eRY=', 'Content-Length': '0'} 2022-09-03T07:12:42.797 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d0d0d141378ab78-006312fe6a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.798 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.798 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.799 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:42.800 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:42.801 DEBUG:boto:Method: GET 2022-09-03T07:12:42.801 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:42.802 DEBUG:boto:Data: 2022-09-03T07:12:42.802 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.804 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.805 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.806 DEBUG:boto:Params: {} 2022-09-03T07:12:42.807 DEBUG:boto:Token: None 2022-09-03T07:12:42.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/ 2022-09-03T07:12:42.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7y/rxjxaTVmOPqsxjwKdLlK++AQ= 2022-09-03T07:12:42.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7y/rxjxaTVmOPqsxjwKdLlK++AQ=', 'Content-Length': '0'} 2022-09-03T07:12:42.815 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ee2bf196d557e116-006312fe6a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.816 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:42.817 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.818 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.819 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:42.821 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:42.823 DEBUG:boto:Method: GET 2022-09-03T07:12:42.824 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:42.827 DEBUG:boto:Data: 2022-09-03T07:12:42.829 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.832 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.833 DEBUG:boto:Params: {} 2022-09-03T07:12:42.834 DEBUG:boto:Token: None 2022-09-03T07:12:42.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/ 2022-09-03T07:12:42.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7y/rxjxaTVmOPqsxjwKdLlK++AQ= 2022-09-03T07:12:42.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7y/rxjxaTVmOPqsxjwKdLlK++AQ=', 'Content-Length': '0'} 2022-09-03T07:12:42.846 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006bcc7aa44bdf1638-006312fe6a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.848 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:42.849 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.850 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.851 DEBUG:boto:Method: HEAD 2022-09-03T07:12:42.852 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:42.853 DEBUG:boto:Data: 2022-09-03T07:12:42.853 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.854 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.855 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.856 DEBUG:boto:Params: {} 2022-09-03T07:12:42.857 DEBUG:boto:Token: None 2022-09-03T07:12:42.858 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11 2022-09-03T07:12:42.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oC2r8s/rNZ+vIdYkiGPx2kGH1aM= 2022-09-03T07:12:42.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oC2r8s/rNZ+vIdYkiGPx2kGH1aM=', 'Content-Length': '0'} 2022-09-03T07:12:42.863 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a7af80c247a13744-006312fe6a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.864 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.865 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.866 DEBUG:boto:Method: HEAD 2022-09-03T07:12:42.866 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:42.867 DEBUG:boto:Data: 2022-09-03T07:12:42.868 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.869 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.870 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.871 DEBUG:boto:Params: {} 2022-09-03T07:12:42.872 DEBUG:boto:Token: None 2022-09-03T07:12:42.873 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11 2022-09-03T07:12:42.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oC2r8s/rNZ+vIdYkiGPx2kGH1aM= 2022-09-03T07:12:42.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oC2r8s/rNZ+vIdYkiGPx2kGH1aM=', 'Content-Length': '0'} 2022-09-03T07:12:42.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c209d4811e98c7cf-006312fe6a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.879 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:12:42.880 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.881 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.882 DEBUG:boto:Method: GET 2022-09-03T07:12:42.883 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:42.884 DEBUG:boto:Data: 2022-09-03T07:12:42.885 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.886 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.887 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.888 DEBUG:boto:Params: {} 2022-09-03T07:12:42.889 DEBUG:boto:Token: None 2022-09-03T07:12:42.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11 2022-09-03T07:12:42.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+P+R7SamJysTO/q2owjppyg7t38= 2022-09-03T07:12:42.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+P+R7SamJysTO/q2owjppyg7t38=', 'Content-Length': '0'} 2022-09-03T07:12:42.905 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d480ab9ec507636c-006312fe6a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.906 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:42.907 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:42.907 DEBUG:boto:Method: GET 2022-09-03T07:12:42.908 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:42.908 DEBUG:boto:Data: 2022-09-03T07:12:42.909 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.910 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.910 DEBUG:boto:Params: {} 2022-09-03T07:12:42.911 DEBUG:boto:Token: None 2022-09-03T07:12:42.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/obj11 2022-09-03T07:12:42.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+P+R7SamJysTO/q2owjppyg7t38= 2022-09-03T07:12:42.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+P+R7SamJysTO/q2owjppyg7t38=', 'Content-Length': '0'} 2022-09-03T07:12:42.919 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ba1edaf396f8f2a-006312fe6a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.920 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.920 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.921 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:42.922 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:42.923 DEBUG:boto:Method: GET 2022-09-03T07:12:42.923 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:42.924 DEBUG:boto:Data: 2022-09-03T07:12:42.924 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.925 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:42.925 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.926 DEBUG:boto:Params: {} 2022-09-03T07:12:42.927 DEBUG:boto:Token: None 2022-09-03T07:12:42.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/ 2022-09-03T07:12:42.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7y/rxjxaTVmOPqsxjwKdLlK++AQ= 2022-09-03T07:12:42.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7y/rxjxaTVmOPqsxjwKdLlK++AQ=', 'Content-Length': '0'} 2022-09-03T07:12:42.931 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e346bc9764f783e8-006312fe6a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:42.974 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:42.986 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:42.987 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:42.987 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:42.988 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:42.989 DEBUG:boto:Method: GET 2022-09-03T07:12:42.989 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:42.990 DEBUG:boto:Data: 2022-09-03T07:12:42.990 DEBUG:boto:Headers: {} 2022-09-03T07:12:42.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:42.991 DEBUG:boto:Port: 8000 2022-09-03T07:12:42.992 DEBUG:boto:Params: {} 2022-09-03T07:12:42.992 DEBUG:boto:Token: None 2022-09-03T07:12:42.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:42 GMT /avysmw-75/ 2022-09-03T07:12:42.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7y/rxjxaTVmOPqsxjwKdLlK++AQ= 2022-09-03T07:12:42.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7y/rxjxaTVmOPqsxjwKdLlK++AQ=', 'Content-Length': '0'} 2022-09-03T07:12:43.003 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e312f7285800146-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.004 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:43.006 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:12:43.007 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.008 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.009 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.010 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.011 DEBUG:boto:Method: GET 2022-09-03T07:12:43.013 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.014 DEBUG:boto:Data: 2022-09-03T07:12:43.015 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.016 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.017 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.018 DEBUG:boto:Params: {} 2022-09-03T07:12:43.019 DEBUG:boto:Token: None 2022-09-03T07:12:43.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tOhzxFOxHjArO/3JgRx144BtgV0= 2022-09-03T07:12:43.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tOhzxFOxHjArO/3JgRx144BtgV0=', 'Content-Length': '0'} 2022-09-03T07:12:43.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8342a5db0c5c76e-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.028 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.029 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.032 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.033 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.033 DEBUG:boto:Method: GET 2022-09-03T07:12:43.034 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.034 DEBUG:boto:Data: 2022-09-03T07:12:43.035 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.036 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.037 DEBUG:boto:Params: {} 2022-09-03T07:12:43.037 DEBUG:boto:Token: None 2022-09-03T07:12:43.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tOhzxFOxHjArO/3JgRx144BtgV0= 2022-09-03T07:12:43.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tOhzxFOxHjArO/3JgRx144BtgV0=', 'Content-Length': '0'} 2022-09-03T07:12:43.047 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cac9309692086f7f-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.082 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.084 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.084 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.085 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.088 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.089 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.093 DEBUG:boto:Data: 2022-09-03T07:12:43.094 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.096 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.097 DEBUG:boto:Params: {} 2022-09-03T07:12:43.097 DEBUG:boto:Token: None 2022-09-03T07:12:43.099 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB5OTCXfWZ4LUIryOarSK1NluI8= 2022-09-03T07:12:43.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB5OTCXfWZ4LUIryOarSK1NluI8=', 'Content-Length': '0'} 2022-09-03T07:12:43.104 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f06bec9c394a01d6-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.106 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.107 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.111 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.111 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.112 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.113 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.115 DEBUG:boto:Data: 2022-09-03T07:12:43.116 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.117 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.118 DEBUG:boto:Params: {} 2022-09-03T07:12:43.118 DEBUG:boto:Token: None 2022-09-03T07:12:43.119 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB5OTCXfWZ4LUIryOarSK1NluI8= 2022-09-03T07:12:43.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB5OTCXfWZ4LUIryOarSK1NluI8=', 'Content-Length': '0'} 2022-09-03T07:12:43.126 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005bbc11bc4171faae-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.127 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:12:43.128 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.129 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.130 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.131 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.132 DEBUG:boto:Method: GET 2022-09-03T07:12:43.133 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.134 DEBUG:boto:Data: 2022-09-03T07:12:43.138 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.139 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.141 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.142 DEBUG:boto:Params: {} 2022-09-03T07:12:43.156 DEBUG:boto:Token: None 2022-09-03T07:12:43.157 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tOhzxFOxHjArO/3JgRx144BtgV0= 2022-09-03T07:12:43.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tOhzxFOxHjArO/3JgRx144BtgV0=', 'Content-Length': '0'} 2022-09-03T07:12:43.169 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038dcfa95041b6bb1-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.171 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.172 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.173 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.174 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.175 DEBUG:boto:Method: GET 2022-09-03T07:12:43.176 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.177 DEBUG:boto:Data: 2022-09-03T07:12:43.178 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.179 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.180 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.181 DEBUG:boto:Params: {} 2022-09-03T07:12:43.195 DEBUG:boto:Token: None 2022-09-03T07:12:43.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:43.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tOhzxFOxHjArO/3JgRx144BtgV0= 2022-09-03T07:12:43.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tOhzxFOxHjArO/3JgRx144BtgV0=', 'Content-Length': '0'} 2022-09-03T07:12:43.206 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fccd1580854cce16-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.207 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:43.208 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:43.209 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:43.209 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:43.210 DEBUG:boto:Method: GET 2022-09-03T07:12:43.211 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:43.211 DEBUG:boto:Data: 2022-09-03T07:12:43.212 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.220 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.224 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.225 DEBUG:boto:Params: {} 2022-09-03T07:12:43.227 DEBUG:boto:Token: None 2022-09-03T07:12:43.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/ 2022-09-03T07:12:43.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg= 2022-09-03T07:12:43.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg=', 'Content-Length': '0'} 2022-09-03T07:12:43.245 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ab94b1b32fecb3ed-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.246 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:43.247 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:43.248 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:43.248 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:43.249 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:43.249 DEBUG:boto:Method: GET 2022-09-03T07:12:43.250 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:43.250 DEBUG:boto:Data: 2022-09-03T07:12:43.251 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.252 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.252 DEBUG:boto:Params: {} 2022-09-03T07:12:43.253 DEBUG:boto:Token: None 2022-09-03T07:12:43.256 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/ 2022-09-03T07:12:43.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg= 2022-09-03T07:12:43.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg=', 'Content-Length': '0'} 2022-09-03T07:12:43.271 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075712b075bbd0760-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.273 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:43.274 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.275 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.276 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.277 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:43.281 DEBUG:boto:Data: 2022-09-03T07:12:43.282 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.283 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.284 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.285 DEBUG:boto:Params: {} 2022-09-03T07:12:43.286 DEBUG:boto:Token: None 2022-09-03T07:12:43.287 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22 2022-09-03T07:12:43.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e52zo4feEGt2FcpWGUxyrj47fJM= 2022-09-03T07:12:43.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e52zo4feEGt2FcpWGUxyrj47fJM=', 'Content-Length': '0'} 2022-09-03T07:12:43.291 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d0f49b9f9263fb1-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.292 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.293 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.294 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.295 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:43.296 DEBUG:boto:Data: 2022-09-03T07:12:43.297 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.306 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.307 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.308 DEBUG:boto:Params: {} 2022-09-03T07:12:43.311 DEBUG:boto:Token: None 2022-09-03T07:12:43.313 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22 2022-09-03T07:12:43.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e52zo4feEGt2FcpWGUxyrj47fJM= 2022-09-03T07:12:43.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e52zo4feEGt2FcpWGUxyrj47fJM=', 'Content-Length': '0'} 2022-09-03T07:12:43.317 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079c529891740e1bf-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.319 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:12:43.320 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.321 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.322 DEBUG:boto:Method: GET 2022-09-03T07:12:43.323 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:43.324 DEBUG:boto:Data: 2022-09-03T07:12:43.325 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.325 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.326 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.327 DEBUG:boto:Params: {} 2022-09-03T07:12:43.327 DEBUG:boto:Token: None 2022-09-03T07:12:43.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22 2022-09-03T07:12:43.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPRE6eYjwjDC41Hn1LZOOlWV4GU= 2022-09-03T07:12:43.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPRE6eYjwjDC41Hn1LZOOlWV4GU=', 'Content-Length': '0'} 2022-09-03T07:12:43.331 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bee01e83dc692f22-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.332 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:43.333 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:43.333 DEBUG:boto:Method: GET 2022-09-03T07:12:43.334 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:43.334 DEBUG:boto:Data: 2022-09-03T07:12:43.335 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.343 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.344 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.345 DEBUG:boto:Params: {} 2022-09-03T07:12:43.346 DEBUG:boto:Token: None 2022-09-03T07:12:43.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj22 2022-09-03T07:12:43.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPRE6eYjwjDC41Hn1LZOOlWV4GU= 2022-09-03T07:12:43.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPRE6eYjwjDC41Hn1LZOOlWV4GU=', 'Content-Length': '0'} 2022-09-03T07:12:43.357 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007de67dc056fba487-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.358 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:43.358 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:43.359 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:43.360 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:43.360 DEBUG:boto:Method: GET 2022-09-03T07:12:43.361 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:43.361 DEBUG:boto:Data: 2022-09-03T07:12:43.362 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.363 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.364 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.364 DEBUG:boto:Params: {} 2022-09-03T07:12:43.365 DEBUG:boto:Token: None 2022-09-03T07:12:43.365 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/ 2022-09-03T07:12:43.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg= 2022-09-03T07:12:43.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg=', 'Content-Length': '0'} 2022-09-03T07:12:43.369 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0c586b80d0bcb49-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.370 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:43.371 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:43.371 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:43.372 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:43.372 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:43.373 DEBUG:boto:Method: GET 2022-09-03T07:12:43.373 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:43.374 DEBUG:boto:Data: 2022-09-03T07:12:43.374 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.375 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.376 DEBUG:boto:Params: {} 2022-09-03T07:12:43.378 DEBUG:boto:Token: None 2022-09-03T07:12:43.378 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/ 2022-09-03T07:12:43.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg= 2022-09-03T07:12:43.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg=', 'Content-Length': '0'} 2022-09-03T07:12:43.387 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc0cec4038a4be59-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.388 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:43.389 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:12:43.390 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.391 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.391 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.392 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.392 DEBUG:boto:Method: GET 2022-09-03T07:12:43.393 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.393 DEBUG:boto:Data: 2022-09-03T07:12:43.394 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.394 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.395 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.395 DEBUG:boto:Params: {} 2022-09-03T07:12:43.396 DEBUG:boto:Token: None 2022-09-03T07:12:43.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NtHWbC7Dx3TKmjQ2ak83mE4MRWQ= 2022-09-03T07:12:43.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NtHWbC7Dx3TKmjQ2ak83mE4MRWQ=', 'Content-Length': '0'} 2022-09-03T07:12:43.402 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000237b6a7769285316-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.404 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.405 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.406 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.407 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.408 DEBUG:boto:Method: GET 2022-09-03T07:12:43.409 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.410 DEBUG:boto:Data: 2022-09-03T07:12:43.411 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.413 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.414 DEBUG:boto:Params: {} 2022-09-03T07:12:43.415 DEBUG:boto:Token: None 2022-09-03T07:12:43.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NtHWbC7Dx3TKmjQ2ak83mE4MRWQ= 2022-09-03T07:12:43.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NtHWbC7Dx3TKmjQ2ak83mE4MRWQ=', 'Content-Length': '0'} 2022-09-03T07:12:43.426 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046dc5b7b3e83b1d5-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.428 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.429 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.431 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.431 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.432 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.432 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.433 DEBUG:boto:Data: 2022-09-03T07:12:43.433 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.443 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.443 DEBUG:boto:Params: {} 2022-09-03T07:12:43.444 DEBUG:boto:Token: None 2022-09-03T07:12:43.444 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cCGp0n57XTaMlaypZXlreJDR6q0= 2022-09-03T07:12:43.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cCGp0n57XTaMlaypZXlreJDR6q0=', 'Content-Length': '0'} 2022-09-03T07:12:43.449 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d09d62e4d71d2bab-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.449 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.450 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.451 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.452 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.452 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.453 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.453 DEBUG:boto:Data: 2022-09-03T07:12:43.454 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.454 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.455 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.456 DEBUG:boto:Params: {} 2022-09-03T07:12:43.456 DEBUG:boto:Token: None 2022-09-03T07:12:43.457 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cCGp0n57XTaMlaypZXlreJDR6q0= 2022-09-03T07:12:43.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cCGp0n57XTaMlaypZXlreJDR6q0=', 'Content-Length': '0'} 2022-09-03T07:12:43.460 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002974023d58ae8a72-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.461 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:12:43.463 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.464 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.464 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.465 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.466 DEBUG:boto:Method: GET 2022-09-03T07:12:43.466 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.467 DEBUG:boto:Data: 2022-09-03T07:12:43.467 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.468 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.468 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.469 DEBUG:boto:Params: {} 2022-09-03T07:12:43.469 DEBUG:boto:Token: None 2022-09-03T07:12:43.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NtHWbC7Dx3TKmjQ2ak83mE4MRWQ= 2022-09-03T07:12:43.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NtHWbC7Dx3TKmjQ2ak83mE4MRWQ=', 'Content-Length': '0'} 2022-09-03T07:12:43.474 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089de9ad143940c5f-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.477 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.478 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.479 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.480 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.481 DEBUG:boto:Method: GET 2022-09-03T07:12:43.482 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.483 DEBUG:boto:Data: 2022-09-03T07:12:43.484 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.484 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.485 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.485 DEBUG:boto:Params: {} 2022-09-03T07:12:43.493 DEBUG:boto:Token: None 2022-09-03T07:12:43.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:43.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NtHWbC7Dx3TKmjQ2ak83mE4MRWQ= 2022-09-03T07:12:43.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NtHWbC7Dx3TKmjQ2ak83mE4MRWQ=', 'Content-Length': '0'} 2022-09-03T07:12:43.499 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1ffd939c01fea7a-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.542 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:43.544 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:43.545 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:43.546 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:43.548 DEBUG:boto:Method: GET 2022-09-03T07:12:43.549 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:43.551 DEBUG:boto:Data: 2022-09-03T07:12:43.552 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.553 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.554 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.554 DEBUG:boto:Params: {} 2022-09-03T07:12:43.555 DEBUG:boto:Token: None 2022-09-03T07:12:43.556 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/ 2022-09-03T07:12:43.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg= 2022-09-03T07:12:43.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg=', 'Content-Length': '0'} 2022-09-03T07:12:43.563 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000032814249be49be7e-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.565 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:43.567 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:43.568 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:43.571 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:43.572 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:43.573 DEBUG:boto:Method: GET 2022-09-03T07:12:43.577 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:43.579 DEBUG:boto:Data: 2022-09-03T07:12:43.580 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.581 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.582 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.583 DEBUG:boto:Params: {} 2022-09-03T07:12:43.584 DEBUG:boto:Token: None 2022-09-03T07:12:43.586 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/ 2022-09-03T07:12:43.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg= 2022-09-03T07:12:43.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg=', 'Content-Length': '0'} 2022-09-03T07:12:43.599 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000039bef14d66adec6-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.601 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:43.603 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.604 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.604 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.605 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:43.606 DEBUG:boto:Data: 2022-09-03T07:12:43.606 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.607 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.607 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.608 DEBUG:boto:Params: {} 2022-09-03T07:12:43.610 DEBUG:boto:Token: None 2022-09-03T07:12:43.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33 2022-09-03T07:12:43.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FC90teqEV1PJ4RXWwGTosZFHC5Y= 2022-09-03T07:12:43.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FC90teqEV1PJ4RXWwGTosZFHC5Y=', 'Content-Length': '0'} 2022-09-03T07:12:43.619 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009cf176f86daff9db-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.629 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.630 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.631 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.632 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:43.633 DEBUG:boto:Data: 2022-09-03T07:12:43.635 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.636 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.637 DEBUG:boto:Params: {} 2022-09-03T07:12:43.638 DEBUG:boto:Token: None 2022-09-03T07:12:43.639 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33 2022-09-03T07:12:43.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FC90teqEV1PJ4RXWwGTosZFHC5Y= 2022-09-03T07:12:43.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FC90teqEV1PJ4RXWwGTosZFHC5Y=', 'Content-Length': '0'} 2022-09-03T07:12:43.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000183de59f2e65591f-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.653 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:12:43.654 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.654 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.655 DEBUG:boto:Method: GET 2022-09-03T07:12:43.655 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:43.656 DEBUG:boto:Data: 2022-09-03T07:12:43.674 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.675 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.677 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.678 DEBUG:boto:Params: {} 2022-09-03T07:12:43.679 DEBUG:boto:Token: None 2022-09-03T07:12:43.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33 2022-09-03T07:12:43.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LUhfW+p4W9a4T4Y+09k1GFw2ZLo= 2022-09-03T07:12:43.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LUhfW+p4W9a4T4Y+09k1GFw2ZLo=', 'Content-Length': '0'} 2022-09-03T07:12:43.699 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de1575a02d68962f-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.701 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:43.702 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:43.703 DEBUG:boto:Method: GET 2022-09-03T07:12:43.704 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:43.705 DEBUG:boto:Data: 2022-09-03T07:12:43.705 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.706 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.706 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.707 DEBUG:boto:Params: {} 2022-09-03T07:12:43.707 DEBUG:boto:Token: None 2022-09-03T07:12:43.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj33 2022-09-03T07:12:43.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LUhfW+p4W9a4T4Y+09k1GFw2ZLo= 2022-09-03T07:12:43.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LUhfW+p4W9a4T4Y+09k1GFw2ZLo=', 'Content-Length': '0'} 2022-09-03T07:12:43.721 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b09936317307c124-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.723 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:43.724 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:43.724 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:43.725 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:43.727 DEBUG:boto:Method: GET 2022-09-03T07:12:43.727 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:43.728 DEBUG:boto:Data: 2022-09-03T07:12:43.729 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.730 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.731 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.732 DEBUG:boto:Params: {} 2022-09-03T07:12:43.733 DEBUG:boto:Token: None 2022-09-03T07:12:43.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/ 2022-09-03T07:12:43.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg= 2022-09-03T07:12:43.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg=', 'Content-Length': '0'} 2022-09-03T07:12:43.751 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000050bb46ad2b4e4e69-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.760 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:43.761 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:43.761 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:43.762 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:43.762 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:43.763 DEBUG:boto:Method: GET 2022-09-03T07:12:43.763 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:43.764 DEBUG:boto:Data: 2022-09-03T07:12:43.764 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.766 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.766 DEBUG:boto:Params: {} 2022-09-03T07:12:43.767 DEBUG:boto:Token: None 2022-09-03T07:12:43.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/ 2022-09-03T07:12:43.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg= 2022-09-03T07:12:43.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg=', 'Content-Length': '0'} 2022-09-03T07:12:43.776 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d91672c1bd39e13e-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.778 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:43.779 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:12:43.782 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:43.783 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:43.783 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.784 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.785 DEBUG:boto:Method: GET 2022-09-03T07:12:43.786 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.787 DEBUG:boto:Data: 2022-09-03T07:12:43.788 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.789 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.791 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.793 DEBUG:boto:Params: {} 2022-09-03T07:12:43.794 DEBUG:boto:Token: None 2022-09-03T07:12:43.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5XnK2KuZX8F3w0FcWr3jQbgFBo= 2022-09-03T07:12:43.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5XnK2KuZX8F3w0FcWr3jQbgFBo=', 'Content-Length': '0'} 2022-09-03T07:12:43.806 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000118a04a6b246df44-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.808 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:43.810 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:43.811 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.812 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.819 DEBUG:boto:Method: GET 2022-09-03T07:12:43.819 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.820 DEBUG:boto:Data: 2022-09-03T07:12:43.820 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.821 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.821 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.822 DEBUG:boto:Params: {} 2022-09-03T07:12:43.823 DEBUG:boto:Token: None 2022-09-03T07:12:43.823 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5XnK2KuZX8F3w0FcWr3jQbgFBo= 2022-09-03T07:12:43.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5XnK2KuZX8F3w0FcWr3jQbgFBo=', 'Content-Length': '0'} 2022-09-03T07:12:43.828 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1b9e0b3fdb9d385-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.830 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:43.831 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:43.832 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.833 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.833 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.834 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.834 DEBUG:boto:Data: 2022-09-03T07:12:43.835 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.836 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.837 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.837 DEBUG:boto:Params: {} 2022-09-03T07:12:43.838 DEBUG:boto:Token: None 2022-09-03T07:12:43.839 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZGThFe4r7e3wdHYyWjghp40rsSk= 2022-09-03T07:12:43.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZGThFe4r7e3wdHYyWjghp40rsSk=', 'Content-Length': '0'} 2022-09-03T07:12:43.859 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005dd62ec439d97926-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.861 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:43.862 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:43.872 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.873 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.873 DEBUG:boto:Method: HEAD 2022-09-03T07:12:43.874 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.874 DEBUG:boto:Data: 2022-09-03T07:12:43.875 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.876 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.876 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.877 DEBUG:boto:Params: {} 2022-09-03T07:12:43.877 DEBUG:boto:Token: None 2022-09-03T07:12:43.878 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZGThFe4r7e3wdHYyWjghp40rsSk= 2022-09-03T07:12:43.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZGThFe4r7e3wdHYyWjghp40rsSk=', 'Content-Length': '0'} 2022-09-03T07:12:43.892 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097811624d9a7bca4-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.895 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:12:43.896 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:43.897 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:43.898 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.901 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.903 DEBUG:boto:Method: GET 2022-09-03T07:12:43.905 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.906 DEBUG:boto:Data: 2022-09-03T07:12:43.921 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.922 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.924 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.924 DEBUG:boto:Params: {} 2022-09-03T07:12:43.940 DEBUG:boto:Token: None 2022-09-03T07:12:43.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5XnK2KuZX8F3w0FcWr3jQbgFBo= 2022-09-03T07:12:43.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5XnK2KuZX8F3w0FcWr3jQbgFBo=', 'Content-Length': '0'} 2022-09-03T07:12:43.948 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1bce8396b72fe90-006312fe6b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.950 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:43.951 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:43.953 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.954 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.955 DEBUG:boto:Method: GET 2022-09-03T07:12:43.956 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.957 DEBUG:boto:Data: 2022-09-03T07:12:43.958 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:43.960 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.961 DEBUG:boto:Params: {} 2022-09-03T07:12:43.963 DEBUG:boto:Token: None 2022-09-03T07:12:43.964 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:43.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5XnK2KuZX8F3w0FcWr3jQbgFBo= 2022-09-03T07:12:43.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5XnK2KuZX8F3w0FcWr3jQbgFBo=', 'Content-Length': '0'} 2022-09-03T07:12:43.981 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009737a342aea2e105-006312fe6b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:43.983 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:43.984 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:43.985 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:43.989 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:43.990 DEBUG:boto:Method: GET 2022-09-03T07:12:43.991 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:43.992 DEBUG:boto:Data: 2022-09-03T07:12:43.993 DEBUG:boto:Headers: {} 2022-09-03T07:12:43.994 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:43.995 DEBUG:boto:Port: 8000 2022-09-03T07:12:43.996 DEBUG:boto:Params: {} 2022-09-03T07:12:43.997 DEBUG:boto:Token: None 2022-09-03T07:12:43.999 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:43 GMT /avysmw-75/ 2022-09-03T07:12:44.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg= 2022-09-03T07:12:44.000 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wBumTu1Sd5VHDcRD61Y4xiXBszg=', 'Content-Length': '0'} 2022-09-03T07:12:44.015 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dd102ce2ada895dc-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.017 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:44.019 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:44.020 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:44.022 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:44.023 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:44.025 DEBUG:boto:Method: GET 2022-09-03T07:12:44.026 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:44.027 DEBUG:boto:Data: 2022-09-03T07:12:44.027 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.028 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.029 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.031 DEBUG:boto:Params: {} 2022-09-03T07:12:44.032 DEBUG:boto:Token: None 2022-09-03T07:12:44.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/ 2022-09-03T07:12:44.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU= 2022-09-03T07:12:44.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU=', 'Content-Length': '0'} 2022-09-03T07:12:44.040 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d389fbaac1ca5be-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.042 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:44.044 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:44.045 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:44.053 DEBUG:boto:Method: HEAD 2022-09-03T07:12:44.055 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:44.056 DEBUG:boto:Data: 2022-09-03T07:12:44.057 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.058 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.059 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.060 DEBUG:boto:Params: {} 2022-09-03T07:12:44.061 DEBUG:boto:Token: None 2022-09-03T07:12:44.062 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj44 2022-09-03T07:12:44.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ChMbaFsbgtFDc9Xc0a14UpLLTk= 2022-09-03T07:12:44.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ChMbaFsbgtFDc9Xc0a14UpLLTk=', 'Content-Length': '0'} 2022-09-03T07:12:44.067 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de5c15a865b6f0c7-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.068 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:44.070 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:44.071 DEBUG:boto:Method: HEAD 2022-09-03T07:12:44.071 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:44.073 DEBUG:boto:Data: 2022-09-03T07:12:44.087 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.088 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.089 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.089 DEBUG:boto:Params: {} 2022-09-03T07:12:44.090 DEBUG:boto:Token: None 2022-09-03T07:12:44.092 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj44 2022-09-03T07:12:44.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ChMbaFsbgtFDc9Xc0a14UpLLTk= 2022-09-03T07:12:44.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ChMbaFsbgtFDc9Xc0a14UpLLTk=', 'Content-Length': '0'} 2022-09-03T07:12:44.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e991492e5c75b4a8-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.121 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:12:44.135 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:44.135 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:44.136 DEBUG:boto:Method: GET 2022-09-03T07:12:44.136 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:44.137 DEBUG:boto:Data: 2022-09-03T07:12:44.138 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.139 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.139 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.142 DEBUG:boto:Params: {} 2022-09-03T07:12:44.142 DEBUG:boto:Token: None 2022-09-03T07:12:44.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj44 2022-09-03T07:12:44.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XxOE47ClfEAUlH2ck5wTysb/GfE= 2022-09-03T07:12:44.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XxOE47ClfEAUlH2ck5wTysb/GfE=', 'Content-Length': '0'} 2022-09-03T07:12:44.150 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000418f104aa8b21ee0-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.151 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:44.152 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:44.152 DEBUG:boto:Method: GET 2022-09-03T07:12:44.153 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:44.154 DEBUG:boto:Data: 2022-09-03T07:12:44.155 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.156 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.157 DEBUG:boto:Params: {} 2022-09-03T07:12:44.158 DEBUG:boto:Token: None 2022-09-03T07:12:44.159 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj44 2022-09-03T07:12:44.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XxOE47ClfEAUlH2ck5wTysb/GfE= 2022-09-03T07:12:44.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XxOE47ClfEAUlH2ck5wTysb/GfE=', 'Content-Length': '0'} 2022-09-03T07:12:44.173 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004fdba01af86b7150-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.175 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:44.176 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:44.177 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:44.178 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:44.179 DEBUG:boto:Method: GET 2022-09-03T07:12:44.180 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:44.194 DEBUG:boto:Data: 2022-09-03T07:12:44.195 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.196 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.197 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.198 DEBUG:boto:Params: {} 2022-09-03T07:12:44.199 DEBUG:boto:Token: None 2022-09-03T07:12:44.200 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/ 2022-09-03T07:12:44.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU= 2022-09-03T07:12:44.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU=', 'Content-Length': '0'} 2022-09-03T07:12:44.228 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002dcaec5c139c18dc-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.229 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:44.235 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:44.236 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:44.238 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:44.239 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:44.240 DEBUG:boto:Method: GET 2022-09-03T07:12:44.241 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:44.242 DEBUG:boto:Data: 2022-09-03T07:12:44.243 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.259 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.261 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.262 DEBUG:boto:Params: {} 2022-09-03T07:12:44.264 DEBUG:boto:Token: None 2022-09-03T07:12:44.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/ 2022-09-03T07:12:44.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU= 2022-09-03T07:12:44.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU=', 'Content-Length': '0'} 2022-09-03T07:12:44.281 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000927f702b0f3ecf3d-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.315 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:44.316 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:12:44.317 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.317 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.318 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.318 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.319 DEBUG:boto:Method: GET 2022-09-03T07:12:44.319 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.320 DEBUG:boto:Data: 2022-09-03T07:12:44.321 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.321 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.343 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.343 DEBUG:boto:Params: {} 2022-09-03T07:12:44.344 DEBUG:boto:Token: None 2022-09-03T07:12:44.345 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W12DQ9EC16wZITWW3MtylRTcph8= 2022-09-03T07:12:44.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W12DQ9EC16wZITWW3MtylRTcph8=', 'Content-Length': '0'} 2022-09-03T07:12:44.364 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007472227b90a4e9e2-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.368 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.369 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.370 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.371 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.372 DEBUG:boto:Method: GET 2022-09-03T07:12:44.373 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.373 DEBUG:boto:Data: 2022-09-03T07:12:44.390 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.391 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.392 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.393 DEBUG:boto:Params: {} 2022-09-03T07:12:44.394 DEBUG:boto:Token: None 2022-09-03T07:12:44.396 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W12DQ9EC16wZITWW3MtylRTcph8= 2022-09-03T07:12:44.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W12DQ9EC16wZITWW3MtylRTcph8=', 'Content-Length': '0'} 2022-09-03T07:12:44.414 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000014e10c23324403d4-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.416 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.417 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.418 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.434 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.441 DEBUG:boto:Method: HEAD 2022-09-03T07:12:44.445 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.446 DEBUG:boto:Data: 2022-09-03T07:12:44.447 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.448 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.449 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.450 DEBUG:boto:Params: {} 2022-09-03T07:12:44.451 DEBUG:boto:Token: None 2022-09-03T07:12:44.462 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8qy2llhx5K5ztpNdwaMyZQZl+d8= 2022-09-03T07:12:44.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8qy2llhx5K5ztpNdwaMyZQZl+d8=', 'Content-Length': '0'} 2022-09-03T07:12:44.478 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004fc1b77aaee41150-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.479 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.481 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.482 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.483 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.484 DEBUG:boto:Method: HEAD 2022-09-03T07:12:44.484 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.485 DEBUG:boto:Data: 2022-09-03T07:12:44.486 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.488 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.489 DEBUG:boto:Params: {} 2022-09-03T07:12:44.492 DEBUG:boto:Token: None 2022-09-03T07:12:44.494 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8qy2llhx5K5ztpNdwaMyZQZl+d8= 2022-09-03T07:12:44.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8qy2llhx5K5ztpNdwaMyZQZl+d8=', 'Content-Length': '0'} 2022-09-03T07:12:44.506 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d136e302100b0baa-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.508 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:12:44.509 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.520 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.539 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.540 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.540 DEBUG:boto:Method: GET 2022-09-03T07:12:44.541 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.541 DEBUG:boto:Data: 2022-09-03T07:12:44.554 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.555 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.567 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.568 DEBUG:boto:Params: {} 2022-09-03T07:12:44.569 DEBUG:boto:Token: None 2022-09-03T07:12:44.586 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W12DQ9EC16wZITWW3MtylRTcph8= 2022-09-03T07:12:44.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W12DQ9EC16wZITWW3MtylRTcph8=', 'Content-Length': '0'} 2022-09-03T07:12:44.595 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e56ba8b76878275b-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.597 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.599 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.600 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.601 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.602 DEBUG:boto:Method: GET 2022-09-03T07:12:44.602 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.603 DEBUG:boto:Data: 2022-09-03T07:12:44.604 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.605 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.606 DEBUG:boto:Params: {} 2022-09-03T07:12:44.608 DEBUG:boto:Token: None 2022-09-03T07:12:44.609 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:44.610 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W12DQ9EC16wZITWW3MtylRTcph8= 2022-09-03T07:12:44.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W12DQ9EC16wZITWW3MtylRTcph8=', 'Content-Length': '0'} 2022-09-03T07:12:44.655 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ff4f5e756a55051-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.656 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:44.657 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:44.667 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:44.669 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:44.669 DEBUG:boto:Method: GET 2022-09-03T07:12:44.673 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:44.674 DEBUG:boto:Data: 2022-09-03T07:12:44.675 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.678 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.679 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.680 DEBUG:boto:Params: {} 2022-09-03T07:12:44.684 DEBUG:boto:Token: None 2022-09-03T07:12:44.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/ 2022-09-03T07:12:44.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU= 2022-09-03T07:12:44.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU=', 'Content-Length': '0'} 2022-09-03T07:12:44.698 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093d24fdd59abd1b2-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.700 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:44.702 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:44.703 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:44.713 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:44.714 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:44.715 DEBUG:boto:Method: GET 2022-09-03T07:12:44.716 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:44.717 DEBUG:boto:Data: 2022-09-03T07:12:44.726 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.729 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.737 DEBUG:boto:Params: {} 2022-09-03T07:12:44.739 DEBUG:boto:Token: None 2022-09-03T07:12:44.752 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/ 2022-09-03T07:12:44.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU= 2022-09-03T07:12:44.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU=', 'Content-Length': '0'} 2022-09-03T07:12:44.760 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2bb9a1faa1b7020-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.762 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:44.770 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.771 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.772 DEBUG:boto:Method: HEAD 2022-09-03T07:12:44.773 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:44.779 DEBUG:boto:Data: 2022-09-03T07:12:44.781 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.782 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.783 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.784 DEBUG:boto:Params: {} 2022-09-03T07:12:44.786 DEBUG:boto:Token: None 2022-09-03T07:12:44.787 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55 2022-09-03T07:12:44.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:age+rBd5ZESO1AF/2SIx/pXk9gE= 2022-09-03T07:12:44.788 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:age+rBd5ZESO1AF/2SIx/pXk9gE=', 'Content-Length': '0'} 2022-09-03T07:12:44.806 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7fdef8520fc5d9a-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.807 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.808 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.809 DEBUG:boto:Method: HEAD 2022-09-03T07:12:44.819 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:44.820 DEBUG:boto:Data: 2022-09-03T07:12:44.821 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.832 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.833 DEBUG:boto:Params: {} 2022-09-03T07:12:44.844 DEBUG:boto:Token: None 2022-09-03T07:12:44.845 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55 2022-09-03T07:12:44.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:age+rBd5ZESO1AF/2SIx/pXk9gE= 2022-09-03T07:12:44.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:age+rBd5ZESO1AF/2SIx/pXk9gE=', 'Content-Length': '0'} 2022-09-03T07:12:44.896 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dfb131b2d5aa6cae-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.897 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:12:44.900 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.901 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.902 DEBUG:boto:Method: GET 2022-09-03T07:12:44.903 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:44.904 DEBUG:boto:Data: 2022-09-03T07:12:44.905 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.905 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.906 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.907 DEBUG:boto:Params: {} 2022-09-03T07:12:44.908 DEBUG:boto:Token: None 2022-09-03T07:12:44.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55 2022-09-03T07:12:44.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qN01lOewvCmlnteZY07LhtnUjP0= 2022-09-03T07:12:44.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qN01lOewvCmlnteZY07LhtnUjP0=', 'Content-Length': '0'} 2022-09-03T07:12:44.921 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e14c4b0d2a9dbdb-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.922 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:44.931 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:44.932 DEBUG:boto:Method: GET 2022-09-03T07:12:44.933 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:44.934 DEBUG:boto:Data: 2022-09-03T07:12:44.941 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.945 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.947 DEBUG:boto:Params: {} 2022-09-03T07:12:44.948 DEBUG:boto:Token: None 2022-09-03T07:12:44.954 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/obj55 2022-09-03T07:12:44.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qN01lOewvCmlnteZY07LhtnUjP0= 2022-09-03T07:12:44.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qN01lOewvCmlnteZY07LhtnUjP0=', 'Content-Length': '0'} 2022-09-03T07:12:44.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009fc23b46927fb91-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.961 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:44.961 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:44.962 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:44.963 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:44.963 DEBUG:boto:Method: GET 2022-09-03T07:12:44.964 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:44.964 DEBUG:boto:Data: 2022-09-03T07:12:44.965 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:44.966 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.966 DEBUG:boto:Params: {} 2022-09-03T07:12:44.967 DEBUG:boto:Token: None 2022-09-03T07:12:44.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/ 2022-09-03T07:12:44.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU= 2022-09-03T07:12:44.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU=', 'Content-Length': '0'} 2022-09-03T07:12:44.973 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad1a628f8a12aaba-006312fe6c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:44.974 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:44.975 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:44.976 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:44.977 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:44.977 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:44.978 DEBUG:boto:Method: GET 2022-09-03T07:12:44.978 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:44.981 DEBUG:boto:Data: 2022-09-03T07:12:44.981 DEBUG:boto:Headers: {} 2022-09-03T07:12:44.982 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:44.982 DEBUG:boto:Port: 8000 2022-09-03T07:12:44.983 DEBUG:boto:Params: {} 2022-09-03T07:12:44.984 DEBUG:boto:Token: None 2022-09-03T07:12:44.984 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:44 GMT /avysmw-75/ 2022-09-03T07:12:44.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU= 2022-09-03T07:12:44.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vLGYkyuV/Iq1s6gwz6AdmwF/8nU=', 'Content-Length': '0'} 2022-09-03T07:12:45.000 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000599411e6a9f7c441-006312fe6c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.002 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.003 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:12:45.004 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:12:45.005 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.005 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.006 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.006 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:45.007 DEBUG:boto:Data: 2022-09-03T07:12:45.007 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.008 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.009 DEBUG:boto:Params: {} 2022-09-03T07:12:45.010 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:12:45.011 DEBUG:boto:Token: None 2022-09-03T07:12:45.013 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eQZyi2BAPvQWTN5ZKMzHYBiq5Dg= 2022-09-03T07:12:45.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eQZyi2BAPvQWTN5ZKMzHYBiq5Dg=', 'Content-Length': '0'} 2022-09-03T07:12:45.022 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ae316b79a44afd7c-006312fe6d-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.023 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.023 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.024 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.024 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:45.031 DEBUG:boto:Data: 2022-09-03T07:12:45.031 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.043 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.044 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.045 DEBUG:boto:Params: {} 2022-09-03T07:12:45.046 DEBUG:boto:Token: None 2022-09-03T07:12:45.047 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eQZyi2BAPvQWTN5ZKMzHYBiq5Dg= 2022-09-03T07:12:45.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eQZyi2BAPvQWTN5ZKMzHYBiq5Dg=', 'Content-Length': '0'} 2022-09-03T07:12:45.053 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000121868e7b8bd5f46-006312fe6d-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.054 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:12:45.055 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.056 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.057 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.058 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.059 DEBUG:boto:Method: GET 2022-09-03T07:12:45.060 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.061 DEBUG:boto:Data: 2022-09-03T07:12:45.062 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.063 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.064 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.066 DEBUG:boto:Params: {} 2022-09-03T07:12:45.067 DEBUG:boto:Token: None 2022-09-03T07:12:45.069 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/?versions 2022-09-03T07:12:45.079 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hlcu1oqXunp574pRy20wRjXa4VU= 2022-09-03T07:12:45.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hlcu1oqXunp574pRy20wRjXa4VU=', 'Content-Length': '0'} 2022-09-03T07:12:45.087 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000ec9f0b19c8ab8a0-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.088 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.090 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:12:45.099 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:12:45.100 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:12:45.101 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:12:45.102 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:12:45.102 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:12:45.103 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.104 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.106 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.109 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.110 DEBUG:boto:Method: GET 2022-09-03T07:12:45.122 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.123 DEBUG:boto:Data: 2022-09-03T07:12:45.124 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.124 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.125 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.125 DEBUG:boto:Params: {} 2022-09-03T07:12:45.126 DEBUG:boto:Token: None 2022-09-03T07:12:45.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/?versions 2022-09-03T07:12:45.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hlcu1oqXunp574pRy20wRjXa4VU= 2022-09-03T07:12:45.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hlcu1oqXunp574pRy20wRjXa4VU=', 'Content-Length': '0'} 2022-09-03T07:12:45.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017fc7a5da199d36d-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.135 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.136 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:12:45.136 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:12:45.137 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:12:45.138 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:12:45.138 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:12:45.162 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.164 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.165 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.166 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.167 DEBUG:boto:Method: GET 2022-09-03T07:12:45.168 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.169 DEBUG:boto:Data: 2022-09-03T07:12:45.170 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.172 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.173 DEBUG:boto:Params: {} 2022-09-03T07:12:45.175 DEBUG:boto:Token: None 2022-09-03T07:12:45.177 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/?versions 2022-09-03T07:12:45.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hlcu1oqXunp574pRy20wRjXa4VU= 2022-09-03T07:12:45.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hlcu1oqXunp574pRy20wRjXa4VU=', 'Content-Length': '0'} 2022-09-03T07:12:45.184 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc9397cb495a7611-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.195 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.196 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.197 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.197 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.199 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.199 DEBUG:boto:Method: GET 2022-09-03T07:12:45.200 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:45.200 DEBUG:boto:Data: 2022-09-03T07:12:45.201 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.202 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.203 DEBUG:boto:Params: {} 2022-09-03T07:12:45.203 DEBUG:boto:Token: None 2022-09-03T07:12:45.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/?versions 2022-09-03T07:12:45.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hlcu1oqXunp574pRy20wRjXa4VU= 2022-09-03T07:12:45.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hlcu1oqXunp574pRy20wRjXa4VU=', 'Content-Length': '0'} 2022-09-03T07:12:45.211 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4fd7380e360db11-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.213 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.215 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:12:45.216 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.217 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.218 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.219 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.220 DEBUG:boto:Method: GET 2022-09-03T07:12:45.225 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.226 DEBUG:boto:Data: 2022-09-03T07:12:45.228 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.232 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.245 DEBUG:boto:Params: {} 2022-09-03T07:12:45.247 DEBUG:boto:Token: None 2022-09-03T07:12:45.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vooBDVrB6wT4EVlqF7TrCNkBiDE= 2022-09-03T07:12:45.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vooBDVrB6wT4EVlqF7TrCNkBiDE=', 'Content-Length': '0'} 2022-09-03T07:12:45.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006836917f8c3c007e-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.256 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.257 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.258 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.259 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.260 DEBUG:boto:Method: GET 2022-09-03T07:12:45.261 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.262 DEBUG:boto:Data: 2022-09-03T07:12:45.263 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.265 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.266 DEBUG:boto:Params: {} 2022-09-03T07:12:45.267 DEBUG:boto:Token: None 2022-09-03T07:12:45.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vooBDVrB6wT4EVlqF7TrCNkBiDE= 2022-09-03T07:12:45.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vooBDVrB6wT4EVlqF7TrCNkBiDE=', 'Content-Length': '0'} 2022-09-03T07:12:45.276 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066ad36e15128b199-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.278 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.278 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.279 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.279 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.280 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.280 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.281 DEBUG:boto:Data: 2022-09-03T07:12:45.281 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.282 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.283 DEBUG:boto:Params: {} 2022-09-03T07:12:45.283 DEBUG:boto:Token: None 2022-09-03T07:12:45.284 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.285 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2VUV9rAkj/y1kqHgPYBvv7TDSFo= 2022-09-03T07:12:45.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2VUV9rAkj/y1kqHgPYBvv7TDSFo=', 'Content-Length': '0'} 2022-09-03T07:12:45.290 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7a4250c546b88cb-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.294 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.295 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.296 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.297 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.298 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.299 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.299 DEBUG:boto:Data: 2022-09-03T07:12:45.300 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.301 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.304 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.306 DEBUG:boto:Params: {} 2022-09-03T07:12:45.307 DEBUG:boto:Token: None 2022-09-03T07:12:45.308 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2VUV9rAkj/y1kqHgPYBvv7TDSFo= 2022-09-03T07:12:45.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2VUV9rAkj/y1kqHgPYBvv7TDSFo=', 'Content-Length': '0'} 2022-09-03T07:12:45.313 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000022a0e23a56484a1-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.316 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:12:45.317 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.318 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.319 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.320 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.321 DEBUG:boto:Method: GET 2022-09-03T07:12:45.322 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.323 DEBUG:boto:Data: 2022-09-03T07:12:45.324 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.327 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.328 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.329 DEBUG:boto:Params: {} 2022-09-03T07:12:45.331 DEBUG:boto:Token: None 2022-09-03T07:12:45.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vooBDVrB6wT4EVlqF7TrCNkBiDE= 2022-09-03T07:12:45.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vooBDVrB6wT4EVlqF7TrCNkBiDE=', 'Content-Length': '0'} 2022-09-03T07:12:45.339 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000122b04b143d68418-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.340 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.341 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.342 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.345 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.346 DEBUG:boto:Method: GET 2022-09-03T07:12:45.347 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.350 DEBUG:boto:Data: 2022-09-03T07:12:45.351 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.352 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.353 DEBUG:boto:Params: {} 2022-09-03T07:12:45.354 DEBUG:boto:Token: None 2022-09-03T07:12:45.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:45.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vooBDVrB6wT4EVlqF7TrCNkBiDE= 2022-09-03T07:12:45.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vooBDVrB6wT4EVlqF7TrCNkBiDE=', 'Content-Length': '0'} 2022-09-03T07:12:45.361 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000084df929ed0a9d126-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.363 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.363 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.364 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:45.365 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:45.365 DEBUG:boto:Method: GET 2022-09-03T07:12:45.366 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:45.367 DEBUG:boto:Data: 2022-09-03T07:12:45.367 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.368 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.369 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.372 DEBUG:boto:Params: {} 2022-09-03T07:12:45.373 DEBUG:boto:Token: None 2022-09-03T07:12:45.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o= 2022-09-03T07:12:45.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o=', 'Content-Length': '0'} 2022-09-03T07:12:45.380 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000021b2e70bc9f251af-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.381 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.382 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.382 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.383 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:45.383 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:45.384 DEBUG:boto:Method: GET 2022-09-03T07:12:45.384 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:45.391 DEBUG:boto:Data: 2022-09-03T07:12:45.391 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.394 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.394 DEBUG:boto:Params: {} 2022-09-03T07:12:45.397 DEBUG:boto:Token: None 2022-09-03T07:12:45.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o= 2022-09-03T07:12:45.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o=', 'Content-Length': '0'} 2022-09-03T07:12:45.404 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4f424311d1a3f8d-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.405 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.406 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.407 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.407 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.408 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:45.408 DEBUG:boto:Data: 2022-09-03T07:12:45.409 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.409 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.410 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.410 DEBUG:boto:Params: {} 2022-09-03T07:12:45.412 DEBUG:boto:Token: None 2022-09-03T07:12:45.414 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11 2022-09-03T07:12:45.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MvYNh/VaUhdbYGtppgR6C/htRa0= 2022-09-03T07:12:45.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MvYNh/VaUhdbYGtppgR6C/htRa0=', 'Content-Length': '0'} 2022-09-03T07:12:45.419 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000895369536104850a-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.420 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.420 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.421 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.422 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:45.422 DEBUG:boto:Data: 2022-09-03T07:12:45.423 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.424 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.424 DEBUG:boto:Params: {} 2022-09-03T07:12:45.425 DEBUG:boto:Token: None 2022-09-03T07:12:45.426 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11 2022-09-03T07:12:45.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MvYNh/VaUhdbYGtppgR6C/htRa0= 2022-09-03T07:12:45.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MvYNh/VaUhdbYGtppgR6C/htRa0=', 'Content-Length': '0'} 2022-09-03T07:12:45.429 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb80a90789b386c7-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.430 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:12:45.430 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.431 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.432 DEBUG:boto:Method: GET 2022-09-03T07:12:45.433 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:45.433 DEBUG:boto:Data: 2022-09-03T07:12:45.434 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.435 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.435 DEBUG:boto:Params: {} 2022-09-03T07:12:45.436 DEBUG:boto:Token: None 2022-09-03T07:12:45.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11 2022-09-03T07:12:45.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jry4wj8SZ3f4CuyGoGeYBtT+QCo= 2022-09-03T07:12:45.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jry4wj8SZ3f4CuyGoGeYBtT+QCo=', 'Content-Length': '0'} 2022-09-03T07:12:45.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000babecf6c1f4f479b-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.483 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:45.484 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:45.485 DEBUG:boto:Method: GET 2022-09-03T07:12:45.487 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:45.487 DEBUG:boto:Data: 2022-09-03T07:12:45.488 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.489 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.490 DEBUG:boto:Params: {} 2022-09-03T07:12:45.490 DEBUG:boto:Token: None 2022-09-03T07:12:45.491 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj11 2022-09-03T07:12:45.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jry4wj8SZ3f4CuyGoGeYBtT+QCo= 2022-09-03T07:12:45.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jry4wj8SZ3f4CuyGoGeYBtT+QCo=', 'Content-Length': '0'} 2022-09-03T07:12:45.499 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003da25c7a66422d6d-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.501 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.501 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.502 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:45.502 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:45.503 DEBUG:boto:Method: GET 2022-09-03T07:12:45.504 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:45.507 DEBUG:boto:Data: 2022-09-03T07:12:45.508 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.510 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.511 DEBUG:boto:Params: {} 2022-09-03T07:12:45.513 DEBUG:boto:Token: None 2022-09-03T07:12:45.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o= 2022-09-03T07:12:45.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o=', 'Content-Length': '0'} 2022-09-03T07:12:45.521 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000034d7e7b8c1fd5ffa-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.522 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.523 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.524 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.525 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:45.527 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:45.528 DEBUG:boto:Method: GET 2022-09-03T07:12:45.531 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:45.532 DEBUG:boto:Data: 2022-09-03T07:12:45.533 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.535 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.536 DEBUG:boto:Params: {} 2022-09-03T07:12:45.537 DEBUG:boto:Token: None 2022-09-03T07:12:45.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o= 2022-09-03T07:12:45.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o=', 'Content-Length': '0'} 2022-09-03T07:12:45.550 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf6761765ac534fc-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.553 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.555 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:12:45.556 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.557 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.563 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.564 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.565 DEBUG:boto:Method: GET 2022-09-03T07:12:45.565 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.566 DEBUG:boto:Data: 2022-09-03T07:12:45.567 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.568 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.569 DEBUG:boto:Params: {} 2022-09-03T07:12:45.570 DEBUG:boto:Token: None 2022-09-03T07:12:45.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gV1kqzLgXSM8Ys1zh0IObQciccU= 2022-09-03T07:12:45.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gV1kqzLgXSM8Ys1zh0IObQciccU=', 'Content-Length': '0'} 2022-09-03T07:12:45.585 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ad83758c6d39c72-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.586 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.587 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.587 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.588 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.588 DEBUG:boto:Method: GET 2022-09-03T07:12:45.589 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.589 DEBUG:boto:Data: 2022-09-03T07:12:45.590 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.590 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.591 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.591 DEBUG:boto:Params: {} 2022-09-03T07:12:45.592 DEBUG:boto:Token: None 2022-09-03T07:12:45.593 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gV1kqzLgXSM8Ys1zh0IObQciccU= 2022-09-03T07:12:45.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gV1kqzLgXSM8Ys1zh0IObQciccU=', 'Content-Length': '0'} 2022-09-03T07:12:45.600 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003ec7d9aa6a2a3d5-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.601 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.602 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.602 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.603 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.603 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.604 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.604 DEBUG:boto:Data: 2022-09-03T07:12:45.605 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.609 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.610 DEBUG:boto:Params: {} 2022-09-03T07:12:45.611 DEBUG:boto:Token: None 2022-09-03T07:12:45.612 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6l8U1c1fkE8w84e1sB0gx2QH6B8= 2022-09-03T07:12:45.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6l8U1c1fkE8w84e1sB0gx2QH6B8=', 'Content-Length': '0'} 2022-09-03T07:12:45.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091d88857765e90ef-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.627 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.628 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.628 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.634 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.634 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.635 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.635 DEBUG:boto:Data: 2022-09-03T07:12:45.636 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.637 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.637 DEBUG:boto:Params: {} 2022-09-03T07:12:45.638 DEBUG:boto:Token: None 2022-09-03T07:12:45.638 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.639 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6l8U1c1fkE8w84e1sB0gx2QH6B8= 2022-09-03T07:12:45.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6l8U1c1fkE8w84e1sB0gx2QH6B8=', 'Content-Length': '0'} 2022-09-03T07:12:45.642 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bb0814beb2f6893f-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.643 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:12:45.644 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.645 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.646 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.647 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.650 DEBUG:boto:Method: GET 2022-09-03T07:12:45.651 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.652 DEBUG:boto:Data: 2022-09-03T07:12:45.653 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.653 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.654 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.654 DEBUG:boto:Params: {} 2022-09-03T07:12:45.655 DEBUG:boto:Token: None 2022-09-03T07:12:45.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gV1kqzLgXSM8Ys1zh0IObQciccU= 2022-09-03T07:12:45.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gV1kqzLgXSM8Ys1zh0IObQciccU=', 'Content-Length': '0'} 2022-09-03T07:12:45.659 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006da0f4910dae9d9b-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.660 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.661 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.662 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.662 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.663 DEBUG:boto:Method: GET 2022-09-03T07:12:45.664 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.664 DEBUG:boto:Data: 2022-09-03T07:12:45.665 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.665 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.666 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.666 DEBUG:boto:Params: {} 2022-09-03T07:12:45.667 DEBUG:boto:Token: None 2022-09-03T07:12:45.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:45.668 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gV1kqzLgXSM8Ys1zh0IObQciccU= 2022-09-03T07:12:45.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gV1kqzLgXSM8Ys1zh0IObQciccU=', 'Content-Length': '0'} 2022-09-03T07:12:45.672 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000061eb73854a2e2020-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.673 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.673 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.674 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:45.675 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:45.675 DEBUG:boto:Method: GET 2022-09-03T07:12:45.676 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:45.676 DEBUG:boto:Data: 2022-09-03T07:12:45.677 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.678 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.678 DEBUG:boto:Params: {} 2022-09-03T07:12:45.679 DEBUG:boto:Token: None 2022-09-03T07:12:45.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o= 2022-09-03T07:12:45.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o=', 'Content-Length': '0'} 2022-09-03T07:12:45.684 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009d727edb9a05aca3-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.685 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.688 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.689 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.689 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:45.690 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:45.691 DEBUG:boto:Method: GET 2022-09-03T07:12:45.692 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:45.692 DEBUG:boto:Data: 2022-09-03T07:12:45.693 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.696 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.696 DEBUG:boto:Params: {} 2022-09-03T07:12:45.698 DEBUG:boto:Token: None 2022-09-03T07:12:45.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.699 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o= 2022-09-03T07:12:45.700 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o=', 'Content-Length': '0'} 2022-09-03T07:12:45.704 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000060d50a8081f67636-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.707 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.710 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.712 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.714 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.715 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:45.718 DEBUG:boto:Data: 2022-09-03T07:12:45.719 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.721 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.723 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.724 DEBUG:boto:Params: {} 2022-09-03T07:12:45.725 DEBUG:boto:Token: None 2022-09-03T07:12:45.727 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22 2022-09-03T07:12:45.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUq01BvgXHBgmnqYget/kvPa2wY= 2022-09-03T07:12:45.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUq01BvgXHBgmnqYget/kvPa2wY=', 'Content-Length': '0'} 2022-09-03T07:12:45.735 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a012760ab7c02fea-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.737 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.738 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.740 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.741 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:45.742 DEBUG:boto:Data: 2022-09-03T07:12:45.742 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.743 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.744 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.745 DEBUG:boto:Params: {} 2022-09-03T07:12:45.747 DEBUG:boto:Token: None 2022-09-03T07:12:45.747 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22 2022-09-03T07:12:45.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUq01BvgXHBgmnqYget/kvPa2wY= 2022-09-03T07:12:45.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUq01BvgXHBgmnqYget/kvPa2wY=', 'Content-Length': '0'} 2022-09-03T07:12:45.753 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000472f8a2ce774479f-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.754 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:12:45.754 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.755 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.756 DEBUG:boto:Method: GET 2022-09-03T07:12:45.757 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:45.758 DEBUG:boto:Data: 2022-09-03T07:12:45.758 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.759 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.760 DEBUG:boto:Params: {} 2022-09-03T07:12:45.761 DEBUG:boto:Token: None 2022-09-03T07:12:45.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22 2022-09-03T07:12:45.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xLOjtN4XpFg9twhvOwB+xpGtTBc= 2022-09-03T07:12:45.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xLOjtN4XpFg9twhvOwB+xpGtTBc=', 'Content-Length': '0'} 2022-09-03T07:12:45.780 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000845743bf68e6a57f-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.781 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:45.783 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:45.783 DEBUG:boto:Method: GET 2022-09-03T07:12:45.784 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:45.786 DEBUG:boto:Data: 2022-09-03T07:12:45.787 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.794 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.795 DEBUG:boto:Params: {} 2022-09-03T07:12:45.796 DEBUG:boto:Token: None 2022-09-03T07:12:45.797 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj22 2022-09-03T07:12:45.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xLOjtN4XpFg9twhvOwB+xpGtTBc= 2022-09-03T07:12:45.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xLOjtN4XpFg9twhvOwB+xpGtTBc=', 'Content-Length': '0'} 2022-09-03T07:12:45.812 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003edc5995ed8a6b22-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.816 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.817 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.818 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:45.819 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:45.819 DEBUG:boto:Method: GET 2022-09-03T07:12:45.820 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:45.821 DEBUG:boto:Data: 2022-09-03T07:12:45.822 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.823 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.824 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.837 DEBUG:boto:Params: {} 2022-09-03T07:12:45.844 DEBUG:boto:Token: None 2022-09-03T07:12:45.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o= 2022-09-03T07:12:45.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o=', 'Content-Length': '0'} 2022-09-03T07:12:45.874 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000176ac51367b64bce-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.875 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.876 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:45.876 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:45.877 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:45.877 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:45.878 DEBUG:boto:Method: GET 2022-09-03T07:12:45.878 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:45.879 DEBUG:boto:Data: 2022-09-03T07:12:45.879 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.880 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.882 DEBUG:boto:Params: {} 2022-09-03T07:12:45.883 DEBUG:boto:Token: None 2022-09-03T07:12:45.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/ 2022-09-03T07:12:45.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o= 2022-09-03T07:12:45.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzyWri077detxWsKMxWmxpM8p7o=', 'Content-Length': '0'} 2022-09-03T07:12:45.893 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000379c6a169d80ea57-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.894 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:45.896 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:12:45.896 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:45.897 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:45.897 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.904 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.905 DEBUG:boto:Method: GET 2022-09-03T07:12:45.905 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.906 DEBUG:boto:Data: 2022-09-03T07:12:45.907 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.908 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.909 DEBUG:boto:Params: {} 2022-09-03T07:12:45.909 DEBUG:boto:Token: None 2022-09-03T07:12:45.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qK6hIRaErW3d0BL13xT2n/QlXPs= 2022-09-03T07:12:45.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qK6hIRaErW3d0BL13xT2n/QlXPs=', 'Content-Length': '0'} 2022-09-03T07:12:45.918 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060e96d17f64e3af0-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.920 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:45.920 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:45.921 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.922 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.922 DEBUG:boto:Method: GET 2022-09-03T07:12:45.923 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.923 DEBUG:boto:Data: 2022-09-03T07:12:45.924 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.925 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.926 DEBUG:boto:Params: {} 2022-09-03T07:12:45.926 DEBUG:boto:Token: None 2022-09-03T07:12:45.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qK6hIRaErW3d0BL13xT2n/QlXPs= 2022-09-03T07:12:45.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qK6hIRaErW3d0BL13xT2n/QlXPs=', 'Content-Length': '0'} 2022-09-03T07:12:45.938 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098d5cd1095bf2094-006312fe6d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.979 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:45.980 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:45.980 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.981 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.981 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.982 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.982 DEBUG:boto:Data: 2022-09-03T07:12:45.983 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:45.984 DEBUG:boto:Port: 8001 2022-09-03T07:12:45.985 DEBUG:boto:Params: {} 2022-09-03T07:12:45.986 DEBUG:boto:Token: None 2022-09-03T07:12:45.987 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cUfPFiKWS0MP3lTvnmAPQvq1fBo= 2022-09-03T07:12:45.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cUfPFiKWS0MP3lTvnmAPQvq1fBo=', 'Content-Length': '0'} 2022-09-03T07:12:45.990 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008cbe3c4c07daade0-006312fe6d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:45.991 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:45.992 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:45.993 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.993 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.994 DEBUG:boto:Method: HEAD 2022-09-03T07:12:45.994 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:45.995 DEBUG:boto:Data: 2022-09-03T07:12:45.996 DEBUG:boto:Headers: {} 2022-09-03T07:12:45.997 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:45.997 DEBUG:boto:Port: 8000 2022-09-03T07:12:45.998 DEBUG:boto:Params: {} 2022-09-03T07:12:45.998 DEBUG:boto:Token: None 2022-09-03T07:12:45.999 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:45 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:46.000 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cUfPFiKWS0MP3lTvnmAPQvq1fBo= 2022-09-03T07:12:46.000 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cUfPFiKWS0MP3lTvnmAPQvq1fBo=', 'Content-Length': '0'} 2022-09-03T07:12:46.005 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098d58cbe74c292e0-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.007 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:12:46.008 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:46.009 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:46.011 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:46.012 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:46.013 DEBUG:boto:Method: GET 2022-09-03T07:12:46.014 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:46.015 DEBUG:boto:Data: 2022-09-03T07:12:46.015 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.016 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.017 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.018 DEBUG:boto:Params: {} 2022-09-03T07:12:46.019 DEBUG:boto:Token: None 2022-09-03T07:12:46.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:46.021 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x9N55PCXMc3pCx4fpGHIev3bmt4= 2022-09-03T07:12:46.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x9N55PCXMc3pCx4fpGHIev3bmt4=', 'Content-Length': '0'} 2022-09-03T07:12:46.025 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f820dd5d8acaa62d-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.026 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:46.027 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:46.028 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:46.029 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:46.030 DEBUG:boto:Method: GET 2022-09-03T07:12:46.031 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:46.032 DEBUG:boto:Data: 2022-09-03T07:12:46.033 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.033 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.034 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.035 DEBUG:boto:Params: {} 2022-09-03T07:12:46.037 DEBUG:boto:Token: None 2022-09-03T07:12:46.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:46.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x9N55PCXMc3pCx4fpGHIev3bmt4= 2022-09-03T07:12:46.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x9N55PCXMc3pCx4fpGHIev3bmt4=', 'Content-Length': '0'} 2022-09-03T07:12:46.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000498a01bb60c8c4ba-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.054 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.054 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.057 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:46.059 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:46.060 DEBUG:boto:Method: GET 2022-09-03T07:12:46.061 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:46.062 DEBUG:boto:Data: 2022-09-03T07:12:46.076 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.083 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.084 DEBUG:boto:Params: {} 2022-09-03T07:12:46.090 DEBUG:boto:Token: None 2022-09-03T07:12:46.091 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.091 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.092 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.098 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd8bca78d308482f-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.099 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.100 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.101 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.101 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:46.102 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:46.103 DEBUG:boto:Method: GET 2022-09-03T07:12:46.103 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:46.104 DEBUG:boto:Data: 2022-09-03T07:12:46.104 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.106 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.107 DEBUG:boto:Params: {} 2022-09-03T07:12:46.107 DEBUG:boto:Token: None 2022-09-03T07:12:46.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.117 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000086902838aa42df66-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.119 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.120 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:46.120 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:46.121 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.122 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:46.122 DEBUG:boto:Data: 2022-09-03T07:12:46.123 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.124 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.124 DEBUG:boto:Params: {} 2022-09-03T07:12:46.125 DEBUG:boto:Token: None 2022-09-03T07:12:46.125 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj33 2022-09-03T07:12:46.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u3UfciiveWNEjkIa6Ud77W3At4o= 2022-09-03T07:12:46.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u3UfciiveWNEjkIa6Ud77W3At4o=', 'Content-Length': '0'} 2022-09-03T07:12:46.131 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000edce4637cb92ba40-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.133 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:46.134 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:46.135 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.137 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:46.138 DEBUG:boto:Data: 2022-09-03T07:12:46.139 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.140 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.143 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.144 DEBUG:boto:Params: {} 2022-09-03T07:12:46.145 DEBUG:boto:Token: None 2022-09-03T07:12:46.149 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj33 2022-09-03T07:12:46.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u3UfciiveWNEjkIa6Ud77W3At4o= 2022-09-03T07:12:46.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u3UfciiveWNEjkIa6Ud77W3At4o=', 'Content-Length': '0'} 2022-09-03T07:12:46.165 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ef5e82a28e2eb22-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.172 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:12:46.174 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:46.175 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:46.183 DEBUG:boto:Method: GET 2022-09-03T07:12:46.189 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:46.190 DEBUG:boto:Data: 2022-09-03T07:12:46.193 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.195 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.195 DEBUG:boto:Params: {} 2022-09-03T07:12:46.196 DEBUG:boto:Token: None 2022-09-03T07:12:46.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj33 2022-09-03T07:12:46.203 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VTUHBlT7pN34drFIH3rpx9Wbsno= 2022-09-03T07:12:46.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VTUHBlT7pN34drFIH3rpx9Wbsno=', 'Content-Length': '0'} 2022-09-03T07:12:46.217 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048171926a88d492b-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.222 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:46.223 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:46.224 DEBUG:boto:Method: GET 2022-09-03T07:12:46.225 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:46.225 DEBUG:boto:Data: 2022-09-03T07:12:46.226 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.232 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.233 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.238 DEBUG:boto:Params: {} 2022-09-03T07:12:46.239 DEBUG:boto:Token: None 2022-09-03T07:12:46.239 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj33 2022-09-03T07:12:46.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VTUHBlT7pN34drFIH3rpx9Wbsno= 2022-09-03T07:12:46.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VTUHBlT7pN34drFIH3rpx9Wbsno=', 'Content-Length': '0'} 2022-09-03T07:12:46.248 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003b9381a5b3fcafb-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.250 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.251 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.251 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:46.254 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:46.255 DEBUG:boto:Method: GET 2022-09-03T07:12:46.256 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:46.257 DEBUG:boto:Data: 2022-09-03T07:12:46.257 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.258 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.259 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.260 DEBUG:boto:Params: {} 2022-09-03T07:12:46.262 DEBUG:boto:Token: None 2022-09-03T07:12:46.263 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.264 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.265 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.270 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000535b8c28f130daf3-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.271 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.272 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.273 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.279 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:46.280 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:46.280 DEBUG:boto:Method: GET 2022-09-03T07:12:46.281 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:46.281 DEBUG:boto:Data: 2022-09-03T07:12:46.282 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.283 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.283 DEBUG:boto:Params: {} 2022-09-03T07:12:46.284 DEBUG:boto:Token: None 2022-09-03T07:12:46.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.285 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.293 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000303908009f954667-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.294 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.295 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:12:46.295 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.296 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.296 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.297 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.297 DEBUG:boto:Method: GET 2022-09-03T07:12:46.298 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.298 DEBUG:boto:Data: 2022-09-03T07:12:46.299 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.300 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.300 DEBUG:boto:Params: {} 2022-09-03T07:12:46.301 DEBUG:boto:Token: None 2022-09-03T07:12:46.302 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YXIX64qN6rSiRYnW7j1BKi2lbKc= 2022-09-03T07:12:46.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YXIX64qN6rSiRYnW7j1BKi2lbKc=', 'Content-Length': '0'} 2022-09-03T07:12:46.306 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000123fd434700a890b-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.308 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.309 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.310 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.311 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.312 DEBUG:boto:Method: GET 2022-09-03T07:12:46.313 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.314 DEBUG:boto:Data: 2022-09-03T07:12:46.315 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.316 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.317 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.317 DEBUG:boto:Params: {} 2022-09-03T07:12:46.318 DEBUG:boto:Token: None 2022-09-03T07:12:46.319 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YXIX64qN6rSiRYnW7j1BKi2lbKc= 2022-09-03T07:12:46.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YXIX64qN6rSiRYnW7j1BKi2lbKc=', 'Content-Length': '0'} 2022-09-03T07:12:46.328 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e3c7c5d111fda22-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.370 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.371 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.372 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.372 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.373 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.374 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.374 DEBUG:boto:Data: 2022-09-03T07:12:46.375 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.376 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.377 DEBUG:boto:Params: {} 2022-09-03T07:12:46.379 DEBUG:boto:Token: None 2022-09-03T07:12:46.383 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VSFJ4bJO2zcE4gNwKvJfsrN/xJw= 2022-09-03T07:12:46.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VSFJ4bJO2zcE4gNwKvJfsrN/xJw=', 'Content-Length': '0'} 2022-09-03T07:12:46.393 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b78d8af07d0efe2-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.395 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.396 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.396 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.397 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.397 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.398 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.399 DEBUG:boto:Data: 2022-09-03T07:12:46.400 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.407 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.408 DEBUG:boto:Params: {} 2022-09-03T07:12:46.408 DEBUG:boto:Token: None 2022-09-03T07:12:46.409 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VSFJ4bJO2zcE4gNwKvJfsrN/xJw= 2022-09-03T07:12:46.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VSFJ4bJO2zcE4gNwKvJfsrN/xJw=', 'Content-Length': '0'} 2022-09-03T07:12:46.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c92c959bf77571ef-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.416 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:12:46.417 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.417 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.418 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.419 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.419 DEBUG:boto:Method: GET 2022-09-03T07:12:46.420 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.420 DEBUG:boto:Data: 2022-09-03T07:12:46.421 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.424 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.425 DEBUG:boto:Params: {} 2022-09-03T07:12:46.427 DEBUG:boto:Token: None 2022-09-03T07:12:46.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.429 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YXIX64qN6rSiRYnW7j1BKi2lbKc= 2022-09-03T07:12:46.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YXIX64qN6rSiRYnW7j1BKi2lbKc=', 'Content-Length': '0'} 2022-09-03T07:12:46.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3aa18a8d9cc0f45-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.436 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.437 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.437 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.438 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.439 DEBUG:boto:Method: GET 2022-09-03T07:12:46.439 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.440 DEBUG:boto:Data: 2022-09-03T07:12:46.441 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.442 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.442 DEBUG:boto:Params: {} 2022-09-03T07:12:46.443 DEBUG:boto:Token: None 2022-09-03T07:12:46.444 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:46.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YXIX64qN6rSiRYnW7j1BKi2lbKc= 2022-09-03T07:12:46.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YXIX64qN6rSiRYnW7j1BKi2lbKc=', 'Content-Length': '0'} 2022-09-03T07:12:46.462 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe4cecfadf84c889-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.464 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.464 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.465 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:46.466 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:46.467 DEBUG:boto:Method: GET 2022-09-03T07:12:46.469 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:46.469 DEBUG:boto:Data: 2022-09-03T07:12:46.470 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.470 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.471 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.471 DEBUG:boto:Params: {} 2022-09-03T07:12:46.472 DEBUG:boto:Token: None 2022-09-03T07:12:46.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.477 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000048ada56c5dd9a802-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.482 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.483 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.483 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.484 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:46.484 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:46.485 DEBUG:boto:Method: GET 2022-09-03T07:12:46.494 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:46.495 DEBUG:boto:Data: 2022-09-03T07:12:46.505 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.514 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.514 DEBUG:boto:Params: {} 2022-09-03T07:12:46.515 DEBUG:boto:Token: None 2022-09-03T07:12:46.516 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.526 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f007fb9e78149fa9-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.527 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.531 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.531 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.532 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.532 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:46.533 DEBUG:boto:Data: 2022-09-03T07:12:46.533 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.534 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.535 DEBUG:boto:Params: {} 2022-09-03T07:12:46.536 DEBUG:boto:Token: None 2022-09-03T07:12:46.538 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44 2022-09-03T07:12:46.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UV2wBFVfxxZl6M/KupUtH2XcEn4= 2022-09-03T07:12:46.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UV2wBFVfxxZl6M/KupUtH2XcEn4=', 'Content-Length': '0'} 2022-09-03T07:12:46.546 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0ad47625da11e73-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.548 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.548 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.549 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.549 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:46.550 DEBUG:boto:Data: 2022-09-03T07:12:46.550 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.552 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.552 DEBUG:boto:Params: {} 2022-09-03T07:12:46.553 DEBUG:boto:Token: None 2022-09-03T07:12:46.555 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44 2022-09-03T07:12:46.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UV2wBFVfxxZl6M/KupUtH2XcEn4= 2022-09-03T07:12:46.557 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UV2wBFVfxxZl6M/KupUtH2XcEn4=', 'Content-Length': '0'} 2022-09-03T07:12:46.563 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000011a6edbe85e98852-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.564 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:12:46.565 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.566 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.568 DEBUG:boto:Method: GET 2022-09-03T07:12:46.569 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:46.570 DEBUG:boto:Data: 2022-09-03T07:12:46.570 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.571 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.571 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.572 DEBUG:boto:Params: {} 2022-09-03T07:12:46.573 DEBUG:boto:Token: None 2022-09-03T07:12:46.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44 2022-09-03T07:12:46.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VmVMBqYcLivFpFZ4BCXZ6KlxKYg= 2022-09-03T07:12:46.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VmVMBqYcLivFpFZ4BCXZ6KlxKYg=', 'Content-Length': '0'} 2022-09-03T07:12:46.579 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cefd2d15e344af2a-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.581 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:46.582 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:46.583 DEBUG:boto:Method: GET 2022-09-03T07:12:46.583 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:46.585 DEBUG:boto:Data: 2022-09-03T07:12:46.585 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.586 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.587 DEBUG:boto:Params: {} 2022-09-03T07:12:46.589 DEBUG:boto:Token: None 2022-09-03T07:12:46.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj44 2022-09-03T07:12:46.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VmVMBqYcLivFpFZ4BCXZ6KlxKYg= 2022-09-03T07:12:46.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VmVMBqYcLivFpFZ4BCXZ6KlxKYg=', 'Content-Length': '0'} 2022-09-03T07:12:46.598 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e71ccd4b8b64b985-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.635 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.636 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.637 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:46.639 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:46.643 DEBUG:boto:Method: GET 2022-09-03T07:12:46.645 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:46.646 DEBUG:boto:Data: 2022-09-03T07:12:46.647 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.651 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.656 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.658 DEBUG:boto:Params: {} 2022-09-03T07:12:46.660 DEBUG:boto:Token: None 2022-09-03T07:12:46.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.680 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000300a095b85b13b85-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.681 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.691 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.694 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.694 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:46.695 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:46.695 DEBUG:boto:Method: GET 2022-09-03T07:12:46.696 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:46.696 DEBUG:boto:Data: 2022-09-03T07:12:46.697 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.698 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.699 DEBUG:boto:Params: {} 2022-09-03T07:12:46.704 DEBUG:boto:Token: None 2022-09-03T07:12:46.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.709 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d019dab3e266b9fa-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.710 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.711 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:12:46.711 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:46.712 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:46.713 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.713 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.714 DEBUG:boto:Method: GET 2022-09-03T07:12:46.715 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.716 DEBUG:boto:Data: 2022-09-03T07:12:46.716 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.717 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.718 DEBUG:boto:Params: {} 2022-09-03T07:12:46.718 DEBUG:boto:Token: None 2022-09-03T07:12:46.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zaK2mkxDjyCjN3mGTCMo88z+ApA= 2022-09-03T07:12:46.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zaK2mkxDjyCjN3mGTCMo88z+ApA=', 'Content-Length': '0'} 2022-09-03T07:12:46.725 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fe7b0314823abc7-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.768 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:46.769 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:46.771 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.772 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.773 DEBUG:boto:Method: GET 2022-09-03T07:12:46.775 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.776 DEBUG:boto:Data: 2022-09-03T07:12:46.777 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.779 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.781 DEBUG:boto:Params: {} 2022-09-03T07:12:46.782 DEBUG:boto:Token: None 2022-09-03T07:12:46.791 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zaK2mkxDjyCjN3mGTCMo88z+ApA= 2022-09-03T07:12:46.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zaK2mkxDjyCjN3mGTCMo88z+ApA=', 'Content-Length': '0'} 2022-09-03T07:12:46.806 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000daeb644abc60cfb9-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.808 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:46.809 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:46.810 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.811 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.815 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.816 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.816 DEBUG:boto:Data: 2022-09-03T07:12:46.817 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.819 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.822 DEBUG:boto:Params: {} 2022-09-03T07:12:46.823 DEBUG:boto:Token: None 2022-09-03T07:12:46.823 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0xrBKFy39CewnwCq+ZEutrJmr2o= 2022-09-03T07:12:46.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0xrBKFy39CewnwCq+ZEutrJmr2o=', 'Content-Length': '0'} 2022-09-03T07:12:46.827 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000030f9441635e5a5e3-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.828 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:46.828 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:46.829 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.829 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.830 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.830 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.831 DEBUG:boto:Data: 2022-09-03T07:12:46.831 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.832 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.833 DEBUG:boto:Params: {} 2022-09-03T07:12:46.833 DEBUG:boto:Token: None 2022-09-03T07:12:46.834 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0xrBKFy39CewnwCq+ZEutrJmr2o= 2022-09-03T07:12:46.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0xrBKFy39CewnwCq+ZEutrJmr2o=', 'Content-Length': '0'} 2022-09-03T07:12:46.837 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e475a83e3f1e1865-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.838 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:12:46.838 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:46.839 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:46.839 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.840 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.840 DEBUG:boto:Method: GET 2022-09-03T07:12:46.841 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.848 DEBUG:boto:Data: 2022-09-03T07:12:46.850 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.852 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.853 DEBUG:boto:Params: {} 2022-09-03T07:12:46.854 DEBUG:boto:Token: None 2022-09-03T07:12:46.854 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.855 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zaK2mkxDjyCjN3mGTCMo88z+ApA= 2022-09-03T07:12:46.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zaK2mkxDjyCjN3mGTCMo88z+ApA=', 'Content-Length': '0'} 2022-09-03T07:12:46.860 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7d91a973d6106a8-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.861 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:46.862 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:46.862 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.863 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.863 DEBUG:boto:Method: GET 2022-09-03T07:12:46.864 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.865 DEBUG:boto:Data: 2022-09-03T07:12:46.865 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.866 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.866 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.867 DEBUG:boto:Params: {} 2022-09-03T07:12:46.867 DEBUG:boto:Token: None 2022-09-03T07:12:46.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:46.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zaK2mkxDjyCjN3mGTCMo88z+ApA= 2022-09-03T07:12:46.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zaK2mkxDjyCjN3mGTCMo88z+ApA=', 'Content-Length': '0'} 2022-09-03T07:12:46.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be53f1f87918ebbf-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.880 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.881 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.882 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:46.884 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:46.885 DEBUG:boto:Method: GET 2022-09-03T07:12:46.886 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:46.887 DEBUG:boto:Data: 2022-09-03T07:12:46.888 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.889 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.890 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.891 DEBUG:boto:Params: {} 2022-09-03T07:12:46.892 DEBUG:boto:Token: None 2022-09-03T07:12:46.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.900 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c15f25aa95ca799f-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.901 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.902 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:46.903 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:46.903 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:46.904 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:46.904 DEBUG:boto:Method: GET 2022-09-03T07:12:46.905 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:46.905 DEBUG:boto:Data: 2022-09-03T07:12:46.906 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.910 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.911 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.912 DEBUG:boto:Params: {} 2022-09-03T07:12:46.913 DEBUG:boto:Token: None 2022-09-03T07:12:46.914 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/ 2022-09-03T07:12:46.917 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ= 2022-09-03T07:12:46.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q2atqRIIVpVXkBZ0xds+q5rvywQ=', 'Content-Length': '0'} 2022-09-03T07:12:46.922 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001554eae09ebd1dec-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.924 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:46.940 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:46.941 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:46.941 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.942 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:46.943 DEBUG:boto:Data: 2022-09-03T07:12:46.944 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:46.945 DEBUG:boto:Port: 8001 2022-09-03T07:12:46.945 DEBUG:boto:Params: {} 2022-09-03T07:12:46.946 DEBUG:boto:Token: None 2022-09-03T07:12:46.947 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj55 2022-09-03T07:12:46.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4C+HWV117840epEHc0JNJX6CDNo= 2022-09-03T07:12:46.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4C+HWV117840epEHc0JNJX6CDNo=', 'Content-Length': '0'} 2022-09-03T07:12:46.962 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f5e91d7f5fa1f73-006312fe6e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.965 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:46.966 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:46.966 DEBUG:boto:Method: HEAD 2022-09-03T07:12:46.967 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:46.967 DEBUG:boto:Data: 2022-09-03T07:12:46.968 DEBUG:boto:Headers: {} 2022-09-03T07:12:46.968 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:46.969 DEBUG:boto:Port: 8000 2022-09-03T07:12:46.970 DEBUG:boto:Params: {} 2022-09-03T07:12:46.970 DEBUG:boto:Token: None 2022-09-03T07:12:46.971 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:46 GMT /avysmw-75/obj55 2022-09-03T07:12:46.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4C+HWV117840epEHc0JNJX6CDNo= 2022-09-03T07:12:46.974 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4C+HWV117840epEHc0JNJX6CDNo=', 'Content-Length': '0'} 2022-09-03T07:12:46.977 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bae3e6a4a4ef8bd-006312fe6e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:46.979 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:12:46.980 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:46.981 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:46.998 DEBUG:boto:Method: GET 2022-09-03T07:12:47.011 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:47.013 DEBUG:boto:Data: 2022-09-03T07:12:47.026 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.027 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.028 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.031 DEBUG:boto:Params: {} 2022-09-03T07:12:47.032 DEBUG:boto:Token: None 2022-09-03T07:12:47.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj55 2022-09-03T07:12:47.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CF6yaKW4aEcViBDH+0NHu5auaAc= 2022-09-03T07:12:47.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CF6yaKW4aEcViBDH+0NHu5auaAc=', 'Content-Length': '0'} 2022-09-03T07:12:47.048 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a62beeb083da0828-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.050 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:47.051 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:47.094 DEBUG:boto:Method: GET 2022-09-03T07:12:47.095 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:47.097 DEBUG:boto:Data: 2022-09-03T07:12:47.097 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.098 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:47.098 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.101 DEBUG:boto:Params: {} 2022-09-03T07:12:47.102 DEBUG:boto:Token: None 2022-09-03T07:12:47.103 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj55 2022-09-03T07:12:47.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CF6yaKW4aEcViBDH+0NHu5auaAc= 2022-09-03T07:12:47.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CF6yaKW4aEcViBDH+0NHu5auaAc=', 'Content-Length': '0'} 2022-09-03T07:12:47.111 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d994634353b3abae-006312fe6f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.114 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.115 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.117 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:47.118 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:47.119 DEBUG:boto:Method: GET 2022-09-03T07:12:47.120 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:47.122 DEBUG:boto:Data: 2022-09-03T07:12:47.123 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.126 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.127 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.128 DEBUG:boto:Params: {} 2022-09-03T07:12:47.130 DEBUG:boto:Token: None 2022-09-03T07:12:47.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ= 2022-09-03T07:12:47.133 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.137 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004cfa1ad2bc06cde6-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.139 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.140 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.141 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.142 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:47.143 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:47.147 DEBUG:boto:Method: GET 2022-09-03T07:12:47.148 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:47.149 DEBUG:boto:Data: 2022-09-03T07:12:47.150 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.151 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:47.153 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.153 DEBUG:boto:Params: {} 2022-09-03T07:12:47.154 DEBUG:boto:Token: None 2022-09-03T07:12:47.156 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ= 2022-09-03T07:12:47.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.163 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0e2e60c9d519754-006312fe6f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.167 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.172 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:12:47.173 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:12:47.174 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.175 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.176 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.183 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:47.184 DEBUG:boto:Data: 2022-09-03T07:12:47.185 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.186 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.187 DEBUG:boto:Params: {} 2022-09-03T07:12:47.187 DEBUG:boto:Token: None 2022-09-03T07:12:47.188 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2xK8CEP3dJJAcrsRW1FiT62U3U= 2022-09-03T07:12:47.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2xK8CEP3dJJAcrsRW1FiT62U3U=', 'Content-Length': '0'} 2022-09-03T07:12:47.199 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e0993eded77bc054-006312fe6f-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.200 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.201 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.201 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.202 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:47.203 DEBUG:boto:Data: 2022-09-03T07:12:47.204 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.205 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.205 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.206 DEBUG:boto:Params: {} 2022-09-03T07:12:47.207 DEBUG:boto:Token: None 2022-09-03T07:12:47.207 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2xK8CEP3dJJAcrsRW1FiT62U3U= 2022-09-03T07:12:47.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2xK8CEP3dJJAcrsRW1FiT62U3U=', 'Content-Length': '0'} 2022-09-03T07:12:47.211 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007a9fa98025e586ab-006312fe6f-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.212 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:12:47.213 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.213 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.214 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.214 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.215 DEBUG:boto:Method: GET 2022-09-03T07:12:47.215 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.216 DEBUG:boto:Data: 2022-09-03T07:12:47.217 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.218 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.218 DEBUG:boto:Params: {} 2022-09-03T07:12:47.220 DEBUG:boto:Token: None 2022-09-03T07:12:47.220 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/?versions 2022-09-03T07:12:47.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUzV6xhyIZdId84G7llAcIU+L94= 2022-09-03T07:12:47.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUzV6xhyIZdId84G7llAcIU+L94=', 'Content-Length': '0'} 2022-09-03T07:12:47.229 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069326f52b78d197e-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.232 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.240 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:12:47.243 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:12:47.244 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:12:47.245 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:12:47.246 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:12:47.247 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:12:47.248 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.249 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.250 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.255 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.257 DEBUG:boto:Method: GET 2022-09-03T07:12:47.258 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.261 DEBUG:boto:Data: 2022-09-03T07:12:47.262 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.263 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.264 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.264 DEBUG:boto:Params: {} 2022-09-03T07:12:47.266 DEBUG:boto:Token: None 2022-09-03T07:12:47.267 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/?versions 2022-09-03T07:12:47.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUzV6xhyIZdId84G7llAcIU+L94= 2022-09-03T07:12:47.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUzV6xhyIZdId84G7llAcIU+L94=', 'Content-Length': '0'} 2022-09-03T07:12:47.273 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bc4b5615fad6f2b0-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.276 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.277 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:12:47.279 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:12:47.279 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:12:47.280 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:12:47.282 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:12:47.285 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.286 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.287 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.288 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.289 DEBUG:boto:Method: GET 2022-09-03T07:12:47.290 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.291 DEBUG:boto:Data: 2022-09-03T07:12:47.293 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.294 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.296 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.297 DEBUG:boto:Params: {} 2022-09-03T07:12:47.299 DEBUG:boto:Token: None 2022-09-03T07:12:47.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/?versions 2022-09-03T07:12:47.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUzV6xhyIZdId84G7llAcIU+L94= 2022-09-03T07:12:47.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUzV6xhyIZdId84G7llAcIU+L94=', 'Content-Length': '0'} 2022-09-03T07:12:47.310 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6996076a4b53dd4-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.311 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.313 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.323 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.324 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.325 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.325 DEBUG:boto:Method: GET 2022-09-03T07:12:47.331 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:12:47.332 DEBUG:boto:Data: 2022-09-03T07:12:47.333 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.335 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.335 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.337 DEBUG:boto:Params: {} 2022-09-03T07:12:47.338 DEBUG:boto:Token: None 2022-09-03T07:12:47.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/?versions 2022-09-03T07:12:47.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jUzV6xhyIZdId84G7llAcIU+L94= 2022-09-03T07:12:47.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jUzV6xhyIZdId84G7llAcIU+L94=', 'Content-Length': '0'} 2022-09-03T07:12:47.347 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f663e62ed7ae6574-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.348 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.349 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:12:47.351 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.352 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.353 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.354 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.355 DEBUG:boto:Method: GET 2022-09-03T07:12:47.356 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.357 DEBUG:boto:Data: 2022-09-03T07:12:47.358 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.360 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.361 DEBUG:boto:Params: {} 2022-09-03T07:12:47.362 DEBUG:boto:Token: None 2022-09-03T07:12:47.363 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GcurakEy1UsmZlyWRfrYZ5WeqUs= 2022-09-03T07:12:47.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GcurakEy1UsmZlyWRfrYZ5WeqUs=', 'Content-Length': '0'} 2022-09-03T07:12:47.366 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000072cbf5b28db11b17-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.367 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.368 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.368 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.369 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.369 DEBUG:boto:Method: GET 2022-09-03T07:12:47.370 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.370 DEBUG:boto:Data: 2022-09-03T07:12:47.371 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.372 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.372 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.373 DEBUG:boto:Params: {} 2022-09-03T07:12:47.373 DEBUG:boto:Token: None 2022-09-03T07:12:47.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GcurakEy1UsmZlyWRfrYZ5WeqUs= 2022-09-03T07:12:47.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GcurakEy1UsmZlyWRfrYZ5WeqUs=', 'Content-Length': '0'} 2022-09-03T07:12:47.380 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007099d122c27d18dc-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.381 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.382 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.383 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.383 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.384 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.384 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.385 DEBUG:boto:Data: 2022-09-03T07:12:47.385 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.387 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.387 DEBUG:boto:Params: {} 2022-09-03T07:12:47.388 DEBUG:boto:Token: None 2022-09-03T07:12:47.389 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:glnK6cJBY+r8Zkzy95Vj247OLnQ= 2022-09-03T07:12:47.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:glnK6cJBY+r8Zkzy95Vj247OLnQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.406 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001cd1a7719d4a1c2a-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.408 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.409 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.410 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.410 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.411 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.412 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.413 DEBUG:boto:Data: 2022-09-03T07:12:47.415 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.416 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.417 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.418 DEBUG:boto:Params: {} 2022-09-03T07:12:47.419 DEBUG:boto:Token: None 2022-09-03T07:12:47.420 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:glnK6cJBY+r8Zkzy95Vj247OLnQ= 2022-09-03T07:12:47.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:glnK6cJBY+r8Zkzy95Vj247OLnQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.425 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d9d0ab5abe203b3-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.427 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:12:47.428 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.428 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.429 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.433 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.434 DEBUG:boto:Method: GET 2022-09-03T07:12:47.436 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.437 DEBUG:boto:Data: 2022-09-03T07:12:47.438 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.440 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.442 DEBUG:boto:Params: {} 2022-09-03T07:12:47.443 DEBUG:boto:Token: None 2022-09-03T07:12:47.444 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GcurakEy1UsmZlyWRfrYZ5WeqUs= 2022-09-03T07:12:47.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GcurakEy1UsmZlyWRfrYZ5WeqUs=', 'Content-Length': '0'} 2022-09-03T07:12:47.450 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5a97fdd9f710c3c-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.455 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.457 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.458 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.459 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.459 DEBUG:boto:Method: GET 2022-09-03T07:12:47.460 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.460 DEBUG:boto:Data: 2022-09-03T07:12:47.461 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.462 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.463 DEBUG:boto:Params: {} 2022-09-03T07:12:47.463 DEBUG:boto:Token: None 2022-09-03T07:12:47.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:12:47.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GcurakEy1UsmZlyWRfrYZ5WeqUs= 2022-09-03T07:12:47.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GcurakEy1UsmZlyWRfrYZ5WeqUs=', 'Content-Length': '0'} 2022-09-03T07:12:47.470 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b4b298ad9fc3b130-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.471 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.472 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.473 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:47.473 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:47.484 DEBUG:boto:Method: GET 2022-09-03T07:12:47.485 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:47.493 DEBUG:boto:Data: 2022-09-03T07:12:47.495 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.497 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.498 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.499 DEBUG:boto:Params: {} 2022-09-03T07:12:47.500 DEBUG:boto:Token: None 2022-09-03T07:12:47.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ= 2022-09-03T07:12:47.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.511 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b5bcff213f32abca-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.558 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.560 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.561 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.562 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:47.563 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:47.563 DEBUG:boto:Method: GET 2022-09-03T07:12:47.564 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:47.564 DEBUG:boto:Data: 2022-09-03T07:12:47.565 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.565 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.567 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.568 DEBUG:boto:Params: {} 2022-09-03T07:12:47.569 DEBUG:boto:Token: None 2022-09-03T07:12:47.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ= 2022-09-03T07:12:47.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.580 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000047bc275984877c8c-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.581 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.581 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.582 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.583 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.583 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:47.584 DEBUG:boto:Data: 2022-09-03T07:12:47.584 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.585 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.585 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.598 DEBUG:boto:Params: {} 2022-09-03T07:12:47.599 DEBUG:boto:Token: None 2022-09-03T07:12:47.600 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11 2022-09-03T07:12:47.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MTob8kqOCBykR6r96+18nnITzjE= 2022-09-03T07:12:47.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MTob8kqOCBykR6r96+18nnITzjE=', 'Content-Length': '0'} 2022-09-03T07:12:47.603 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007eeabd744ecf2b9a-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.604 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.604 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.605 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.606 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:47.607 DEBUG:boto:Data: 2022-09-03T07:12:47.607 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.608 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.608 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.609 DEBUG:boto:Params: {} 2022-09-03T07:12:47.610 DEBUG:boto:Token: None 2022-09-03T07:12:47.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11 2022-09-03T07:12:47.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MTob8kqOCBykR6r96+18nnITzjE= 2022-09-03T07:12:47.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MTob8kqOCBykR6r96+18nnITzjE=', 'Content-Length': '0'} 2022-09-03T07:12:47.615 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009b24ee38be983db-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.616 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:12:47.617 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.617 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.618 DEBUG:boto:Method: GET 2022-09-03T07:12:47.618 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:47.619 DEBUG:boto:Data: 2022-09-03T07:12:47.620 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.620 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.629 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.632 DEBUG:boto:Params: {} 2022-09-03T07:12:47.633 DEBUG:boto:Token: None 2022-09-03T07:12:47.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11 2022-09-03T07:12:47.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u/fmKxsVhhED628YKNrSUPocPaQ= 2022-09-03T07:12:47.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u/fmKxsVhhED628YKNrSUPocPaQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.638 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006af1e16630440f6c-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.639 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:12:47.639 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:12:47.640 DEBUG:boto:Method: GET 2022-09-03T07:12:47.640 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:12:47.641 DEBUG:boto:Data: 2022-09-03T07:12:47.641 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.642 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.643 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.644 DEBUG:boto:Params: {} 2022-09-03T07:12:47.645 DEBUG:boto:Token: None 2022-09-03T07:12:47.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj11 2022-09-03T07:12:47.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u/fmKxsVhhED628YKNrSUPocPaQ= 2022-09-03T07:12:47.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u/fmKxsVhhED628YKNrSUPocPaQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.654 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa6dd94c19f6c8a0-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.701 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.701 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.702 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:47.703 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:47.703 DEBUG:boto:Method: GET 2022-09-03T07:12:47.704 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:47.704 DEBUG:boto:Data: 2022-09-03T07:12:47.706 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.706 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.713 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.719 DEBUG:boto:Params: {} 2022-09-03T07:12:47.721 DEBUG:boto:Token: None 2022-09-03T07:12:47.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ= 2022-09-03T07:12:47.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.735 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005698bb0c2fded689-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.736 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.737 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.737 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.738 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:47.739 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:12:47.739 DEBUG:boto:Method: GET 2022-09-03T07:12:47.740 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:12:47.740 DEBUG:boto:Data: 2022-09-03T07:12:47.741 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.741 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.742 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.742 DEBUG:boto:Params: {} 2022-09-03T07:12:47.743 DEBUG:boto:Token: None 2022-09-03T07:12:47.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ= 2022-09-03T07:12:47.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.759 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b797d1d0c24362ec-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.760 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.761 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:12:47.766 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:47.767 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:47.767 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.768 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.769 DEBUG:boto:Method: GET 2022-09-03T07:12:47.769 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.770 DEBUG:boto:Data: 2022-09-03T07:12:47.771 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.772 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.772 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.773 DEBUG:boto:Params: {} 2022-09-03T07:12:47.773 DEBUG:boto:Token: None 2022-09-03T07:12:47.774 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hLHRUDoa4BXbTuie2mK3XYY34/Q= 2022-09-03T07:12:47.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hLHRUDoa4BXbTuie2mK3XYY34/Q=', 'Content-Length': '0'} 2022-09-03T07:12:47.790 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8b865a7e155b8f0-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.794 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:47.795 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:47.803 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.804 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.805 DEBUG:boto:Method: GET 2022-09-03T07:12:47.806 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.807 DEBUG:boto:Data: 2022-09-03T07:12:47.808 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.810 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.810 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.811 DEBUG:boto:Params: {} 2022-09-03T07:12:47.812 DEBUG:boto:Token: None 2022-09-03T07:12:47.812 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hLHRUDoa4BXbTuie2mK3XYY34/Q= 2022-09-03T07:12:47.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hLHRUDoa4BXbTuie2mK3XYY34/Q=', 'Content-Length': '0'} 2022-09-03T07:12:47.834 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b8038500fe3a22b-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.835 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:47.837 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:47.838 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.839 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.840 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.841 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.843 DEBUG:boto:Data: 2022-09-03T07:12:47.844 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.845 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.846 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.847 DEBUG:boto:Params: {} 2022-09-03T07:12:47.849 DEBUG:boto:Token: None 2022-09-03T07:12:47.850 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XaJnSSYj6jem/QoICL9+dmr+Uqw= 2022-09-03T07:12:47.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XaJnSSYj6jem/QoICL9+dmr+Uqw=', 'Content-Length': '0'} 2022-09-03T07:12:47.856 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001176bc234e98e053-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.858 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:47.859 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:47.860 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.860 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.861 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.862 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.863 DEBUG:boto:Data: 2022-09-03T07:12:47.864 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.865 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.866 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.867 DEBUG:boto:Params: {} 2022-09-03T07:12:47.868 DEBUG:boto:Token: None 2022-09-03T07:12:47.869 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XaJnSSYj6jem/QoICL9+dmr+Uqw= 2022-09-03T07:12:47.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XaJnSSYj6jem/QoICL9+dmr+Uqw=', 'Content-Length': '0'} 2022-09-03T07:12:47.874 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b89d10253caca5e5-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.875 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:12:47.876 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:47.877 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:47.880 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.881 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.881 DEBUG:boto:Method: GET 2022-09-03T07:12:47.882 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.882 DEBUG:boto:Data: 2022-09-03T07:12:47.883 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.884 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.885 DEBUG:boto:Params: {} 2022-09-03T07:12:47.885 DEBUG:boto:Token: None 2022-09-03T07:12:47.886 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hLHRUDoa4BXbTuie2mK3XYY34/Q= 2022-09-03T07:12:47.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hLHRUDoa4BXbTuie2mK3XYY34/Q=', 'Content-Length': '0'} 2022-09-03T07:12:47.890 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f9736a3e2ad0fa4-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.891 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:47.891 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:47.892 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.892 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.893 DEBUG:boto:Method: GET 2022-09-03T07:12:47.894 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.898 DEBUG:boto:Data: 2022-09-03T07:12:47.899 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.900 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.900 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.901 DEBUG:boto:Params: {} 2022-09-03T07:12:47.901 DEBUG:boto:Token: None 2022-09-03T07:12:47.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:12:47.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hLHRUDoa4BXbTuie2mK3XYY34/Q= 2022-09-03T07:12:47.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hLHRUDoa4BXbTuie2mK3XYY34/Q=', 'Content-Length': '0'} 2022-09-03T07:12:47.907 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000789ddc52a561ecf5-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.908 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.909 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.910 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:47.911 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:47.911 DEBUG:boto:Method: GET 2022-09-03T07:12:47.912 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:47.912 DEBUG:boto:Data: 2022-09-03T07:12:47.913 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.914 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.914 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.915 DEBUG:boto:Params: {} 2022-09-03T07:12:47.915 DEBUG:boto:Token: None 2022-09-03T07:12:47.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ= 2022-09-03T07:12:47.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.926 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a2691dd6bcc37a7-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.927 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.936 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:47.937 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:47.937 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:47.938 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:47.938 DEBUG:boto:Method: GET 2022-09-03T07:12:47.939 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:47.939 DEBUG:boto:Data: 2022-09-03T07:12:47.940 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.941 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.943 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.944 DEBUG:boto:Params: {} 2022-09-03T07:12:47.946 DEBUG:boto:Token: None 2022-09-03T07:12:47.947 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/ 2022-09-03T07:12:47.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ= 2022-09-03T07:12:47.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SNY9T39Afa4j7aFo84D139TACfQ=', 'Content-Length': '0'} 2022-09-03T07:12:47.954 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000411faec38997d868-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.955 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:47.957 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:47.958 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:47.959 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.959 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:47.960 DEBUG:boto:Data: 2022-09-03T07:12:47.960 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.961 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:47.961 DEBUG:boto:Port: 8001 2022-09-03T07:12:47.962 DEBUG:boto:Params: {} 2022-09-03T07:12:47.963 DEBUG:boto:Token: None 2022-09-03T07:12:47.965 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj22 2022-09-03T07:12:47.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Frl9RzIbguE5P/HpvOmxfVAQB90= 2022-09-03T07:12:47.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Frl9RzIbguE5P/HpvOmxfVAQB90=', 'Content-Length': '0'} 2022-09-03T07:12:47.982 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000634016c9fc31a270-006312fe6f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:47.983 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:47.983 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:47.985 DEBUG:boto:Method: HEAD 2022-09-03T07:12:47.986 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:47.987 DEBUG:boto:Data: 2022-09-03T07:12:47.987 DEBUG:boto:Headers: {} 2022-09-03T07:12:47.988 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:47.989 DEBUG:boto:Port: 8000 2022-09-03T07:12:47.990 DEBUG:boto:Params: {} 2022-09-03T07:12:47.990 DEBUG:boto:Token: None 2022-09-03T07:12:47.991 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:47 GMT /avysmw-75/obj22 2022-09-03T07:12:47.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Frl9RzIbguE5P/HpvOmxfVAQB90= 2022-09-03T07:12:47.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Frl9RzIbguE5P/HpvOmxfVAQB90=', 'Content-Length': '0'} 2022-09-03T07:12:48.002 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aff05ecddc019334-006312fe6f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.004 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:12:48.005 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:48.006 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:48.008 DEBUG:boto:Method: GET 2022-09-03T07:12:48.018 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:48.019 DEBUG:boto:Data: 2022-09-03T07:12:48.020 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.023 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.024 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.025 DEBUG:boto:Params: {} 2022-09-03T07:12:48.026 DEBUG:boto:Token: None 2022-09-03T07:12:48.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj22 2022-09-03T07:12:48.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jO0k8BUeHMjgU8COb89AM027jNQ= 2022-09-03T07:12:48.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jO0k8BUeHMjgU8COb89AM027jNQ=', 'Content-Length': '0'} 2022-09-03T07:12:48.034 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cbaa54bd889cd8ac-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.035 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:12:48.036 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:12:48.038 DEBUG:boto:Method: GET 2022-09-03T07:12:48.048 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:12:48.050 DEBUG:boto:Data: 2022-09-03T07:12:48.051 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.051 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.052 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.052 DEBUG:boto:Params: {} 2022-09-03T07:12:48.053 DEBUG:boto:Token: None 2022-09-03T07:12:48.054 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj22 2022-09-03T07:12:48.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jO0k8BUeHMjgU8COb89AM027jNQ= 2022-09-03T07:12:48.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jO0k8BUeHMjgU8COb89AM027jNQ=', 'Content-Length': '0'} 2022-09-03T07:12:48.058 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3c5e4d29b4e9186-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.103 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.103 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.104 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:48.105 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:48.105 DEBUG:boto:Method: GET 2022-09-03T07:12:48.106 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:48.113 DEBUG:boto:Data: 2022-09-03T07:12:48.114 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.119 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.121 DEBUG:boto:Params: {} 2022-09-03T07:12:48.122 DEBUG:boto:Token: None 2022-09-03T07:12:48.124 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.138 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ab7955eea80e9f84-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.139 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.141 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.142 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.143 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:48.144 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:12:48.145 DEBUG:boto:Method: GET 2022-09-03T07:12:48.147 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:12:48.149 DEBUG:boto:Data: 2022-09-03T07:12:48.150 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.152 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.153 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.154 DEBUG:boto:Params: {} 2022-09-03T07:12:48.155 DEBUG:boto:Token: None 2022-09-03T07:12:48.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.169 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ae00e95fc3b213c6-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.170 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.171 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:12:48.177 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.177 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.178 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.178 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.179 DEBUG:boto:Method: GET 2022-09-03T07:12:48.179 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.180 DEBUG:boto:Data: 2022-09-03T07:12:48.181 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.181 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.193 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.195 DEBUG:boto:Params: {} 2022-09-03T07:12:48.196 DEBUG:boto:Token: None 2022-09-03T07:12:48.198 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rd08vKrSmO9mw9X+iZ5ek0jcaE4= 2022-09-03T07:12:48.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rd08vKrSmO9mw9X+iZ5ek0jcaE4=', 'Content-Length': '0'} 2022-09-03T07:12:48.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4bdcf2d3fe4a02d-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.217 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.218 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.218 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.219 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.227 DEBUG:boto:Method: GET 2022-09-03T07:12:48.229 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.231 DEBUG:boto:Data: 2022-09-03T07:12:48.232 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.233 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.233 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.234 DEBUG:boto:Params: {} 2022-09-03T07:12:48.235 DEBUG:boto:Token: None 2022-09-03T07:12:48.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rd08vKrSmO9mw9X+iZ5ek0jcaE4= 2022-09-03T07:12:48.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rd08vKrSmO9mw9X+iZ5ek0jcaE4=', 'Content-Length': '0'} 2022-09-03T07:12:48.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063f82345dfc8715f-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.240 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.241 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.241 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.242 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.243 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.244 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.244 DEBUG:boto:Data: 2022-09-03T07:12:48.245 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.246 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.246 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.247 DEBUG:boto:Params: {} 2022-09-03T07:12:48.247 DEBUG:boto:Token: None 2022-09-03T07:12:48.248 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wk55mWrWZM7/KKpikxTztu3Ph+g= 2022-09-03T07:12:48.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wk55mWrWZM7/KKpikxTztu3Ph+g=', 'Content-Length': '0'} 2022-09-03T07:12:48.252 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000283ccb92bdbf7c6a-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.253 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.254 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.258 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.259 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.262 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.262 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.263 DEBUG:boto:Data: 2022-09-03T07:12:48.263 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.264 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.265 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.277 DEBUG:boto:Params: {} 2022-09-03T07:12:48.279 DEBUG:boto:Token: None 2022-09-03T07:12:48.280 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wk55mWrWZM7/KKpikxTztu3Ph+g= 2022-09-03T07:12:48.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wk55mWrWZM7/KKpikxTztu3Ph+g=', 'Content-Length': '0'} 2022-09-03T07:12:48.287 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026acf138ad6e4ecc-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.288 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:12:48.289 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.289 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.290 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.290 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.293 DEBUG:boto:Method: GET 2022-09-03T07:12:48.294 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.300 DEBUG:boto:Data: 2022-09-03T07:12:48.301 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.301 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.302 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.302 DEBUG:boto:Params: {} 2022-09-03T07:12:48.303 DEBUG:boto:Token: None 2022-09-03T07:12:48.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rd08vKrSmO9mw9X+iZ5ek0jcaE4= 2022-09-03T07:12:48.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rd08vKrSmO9mw9X+iZ5ek0jcaE4=', 'Content-Length': '0'} 2022-09-03T07:12:48.322 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000082224120a53cc5eb-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.324 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.325 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.327 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.328 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.329 DEBUG:boto:Method: GET 2022-09-03T07:12:48.330 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.331 DEBUG:boto:Data: 2022-09-03T07:12:48.332 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.333 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.334 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.335 DEBUG:boto:Params: {} 2022-09-03T07:12:48.338 DEBUG:boto:Token: None 2022-09-03T07:12:48.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:12:48.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rd08vKrSmO9mw9X+iZ5ek0jcaE4= 2022-09-03T07:12:48.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rd08vKrSmO9mw9X+iZ5ek0jcaE4=', 'Content-Length': '0'} 2022-09-03T07:12:48.361 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3694c84be9bf942-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.370 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.371 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.371 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:48.372 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:48.373 DEBUG:boto:Method: GET 2022-09-03T07:12:48.373 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:48.379 DEBUG:boto:Data: 2022-09-03T07:12:48.380 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.380 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.381 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.381 DEBUG:boto:Params: {} 2022-09-03T07:12:48.382 DEBUG:boto:Token: None 2022-09-03T07:12:48.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.388 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b271d087d29ea38a-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.389 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.392 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.392 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.393 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:48.394 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:48.394 DEBUG:boto:Method: GET 2022-09-03T07:12:48.395 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:48.395 DEBUG:boto:Data: 2022-09-03T07:12:48.396 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.396 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.397 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.398 DEBUG:boto:Params: {} 2022-09-03T07:12:48.399 DEBUG:boto:Token: None 2022-09-03T07:12:48.400 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.401 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.420 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000017357e271505076-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.421 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.424 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.425 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.426 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.426 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:48.427 DEBUG:boto:Data: 2022-09-03T07:12:48.427 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.428 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.429 DEBUG:boto:Params: {} 2022-09-03T07:12:48.429 DEBUG:boto:Token: None 2022-09-03T07:12:48.431 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33 2022-09-03T07:12:48.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QkXBK++Oyr5C39tYy+4TLoFBCXs= 2022-09-03T07:12:48.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QkXBK++Oyr5C39tYy+4TLoFBCXs=', 'Content-Length': '0'} 2022-09-03T07:12:48.434 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db6d147a3f060c79-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.435 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.436 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.436 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.437 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:48.437 DEBUG:boto:Data: 2022-09-03T07:12:48.438 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.439 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.444 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.445 DEBUG:boto:Params: {} 2022-09-03T07:12:48.445 DEBUG:boto:Token: None 2022-09-03T07:12:48.446 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33 2022-09-03T07:12:48.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QkXBK++Oyr5C39tYy+4TLoFBCXs= 2022-09-03T07:12:48.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QkXBK++Oyr5C39tYy+4TLoFBCXs=', 'Content-Length': '0'} 2022-09-03T07:12:48.451 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b70f513140fc95c5-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.452 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:12:48.453 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.453 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.454 DEBUG:boto:Method: GET 2022-09-03T07:12:48.454 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:48.455 DEBUG:boto:Data: 2022-09-03T07:12:48.456 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.457 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.457 DEBUG:boto:Params: {} 2022-09-03T07:12:48.458 DEBUG:boto:Token: None 2022-09-03T07:12:48.459 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33 2022-09-03T07:12:48.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nFyCS+9J0Xeo6Wz72J0LxYLCoF8= 2022-09-03T07:12:48.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nFyCS+9J0Xeo6Wz72J0LxYLCoF8=', 'Content-Length': '0'} 2022-09-03T07:12:48.463 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000458b8839810c1b0d-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.506 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:12:48.508 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:12:48.508 DEBUG:boto:Method: GET 2022-09-03T07:12:48.509 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:12:48.509 DEBUG:boto:Data: 2022-09-03T07:12:48.510 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.511 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.512 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.513 DEBUG:boto:Params: {} 2022-09-03T07:12:48.513 DEBUG:boto:Token: None 2022-09-03T07:12:48.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj33 2022-09-03T07:12:48.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nFyCS+9J0Xeo6Wz72J0LxYLCoF8= 2022-09-03T07:12:48.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nFyCS+9J0Xeo6Wz72J0LxYLCoF8=', 'Content-Length': '0'} 2022-09-03T07:12:48.519 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008575098077b3d514-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.520 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.520 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.521 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:48.522 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:48.524 DEBUG:boto:Method: GET 2022-09-03T07:12:48.525 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:48.526 DEBUG:boto:Data: 2022-09-03T07:12:48.527 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.527 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.528 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.529 DEBUG:boto:Params: {} 2022-09-03T07:12:48.529 DEBUG:boto:Token: None 2022-09-03T07:12:48.530 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.534 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c393fe554403d9f-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.535 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.536 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.536 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.537 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:48.537 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:12:48.538 DEBUG:boto:Method: GET 2022-09-03T07:12:48.538 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:12:48.539 DEBUG:boto:Data: 2022-09-03T07:12:48.539 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.541 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.541 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.543 DEBUG:boto:Params: {} 2022-09-03T07:12:48.543 DEBUG:boto:Token: None 2022-09-03T07:12:48.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.546 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.554 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b9487513a2f9910-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.556 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.558 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:12:48.558 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.559 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.560 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.561 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.562 DEBUG:boto:Method: GET 2022-09-03T07:12:48.563 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.564 DEBUG:boto:Data: 2022-09-03T07:12:48.565 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.566 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.567 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.567 DEBUG:boto:Params: {} 2022-09-03T07:12:48.569 DEBUG:boto:Token: None 2022-09-03T07:12:48.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l5NPVa1XbVuw6z8vY4u5yZ79m/o= 2022-09-03T07:12:48.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l5NPVa1XbVuw6z8vY4u5yZ79m/o=', 'Content-Length': '0'} 2022-09-03T07:12:48.575 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007327959a0b00cae0-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.576 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.578 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.579 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.580 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.581 DEBUG:boto:Method: GET 2022-09-03T07:12:48.582 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.583 DEBUG:boto:Data: 2022-09-03T07:12:48.584 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.585 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.586 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.587 DEBUG:boto:Params: {} 2022-09-03T07:12:48.591 DEBUG:boto:Token: None 2022-09-03T07:12:48.594 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.595 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l5NPVa1XbVuw6z8vY4u5yZ79m/o= 2022-09-03T07:12:48.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l5NPVa1XbVuw6z8vY4u5yZ79m/o=', 'Content-Length': '0'} 2022-09-03T07:12:48.599 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000082fcb5732de89e24-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.643 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.644 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.645 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.646 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.648 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.648 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.649 DEBUG:boto:Data: 2022-09-03T07:12:48.650 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.651 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.652 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.652 DEBUG:boto:Params: {} 2022-09-03T07:12:48.653 DEBUG:boto:Token: None 2022-09-03T07:12:48.654 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KSfiqDcb/Rxz1L/2ORiFIkGt+Bk= 2022-09-03T07:12:48.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KSfiqDcb/Rxz1L/2ORiFIkGt+Bk=', 'Content-Length': '0'} 2022-09-03T07:12:48.667 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000596b6902cfca462f-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.668 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.669 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.671 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.672 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.673 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.674 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.676 DEBUG:boto:Data: 2022-09-03T07:12:48.677 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.677 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.678 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.678 DEBUG:boto:Params: {} 2022-09-03T07:12:48.679 DEBUG:boto:Token: None 2022-09-03T07:12:48.679 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KSfiqDcb/Rxz1L/2ORiFIkGt+Bk= 2022-09-03T07:12:48.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KSfiqDcb/Rxz1L/2ORiFIkGt+Bk=', 'Content-Length': '0'} 2022-09-03T07:12:48.683 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000767d175bc31e9660-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.683 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:12:48.684 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.685 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.685 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.686 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.686 DEBUG:boto:Method: GET 2022-09-03T07:12:48.687 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.687 DEBUG:boto:Data: 2022-09-03T07:12:48.688 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.689 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.689 DEBUG:boto:Params: {} 2022-09-03T07:12:48.690 DEBUG:boto:Token: None 2022-09-03T07:12:48.691 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l5NPVa1XbVuw6z8vY4u5yZ79m/o= 2022-09-03T07:12:48.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l5NPVa1XbVuw6z8vY4u5yZ79m/o=', 'Content-Length': '0'} 2022-09-03T07:12:48.697 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000305009f26c3aac73-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.698 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.699 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.700 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.700 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.701 DEBUG:boto:Method: GET 2022-09-03T07:12:48.702 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.703 DEBUG:boto:Data: 2022-09-03T07:12:48.704 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.705 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.707 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.708 DEBUG:boto:Params: {} 2022-09-03T07:12:48.709 DEBUG:boto:Token: None 2022-09-03T07:12:48.710 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:12:48.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l5NPVa1XbVuw6z8vY4u5yZ79m/o= 2022-09-03T07:12:48.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l5NPVa1XbVuw6z8vY4u5yZ79m/o=', 'Content-Length': '0'} 2022-09-03T07:12:48.718 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000269540f807645a8f-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.719 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.720 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.720 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:48.721 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:48.721 DEBUG:boto:Method: GET 2022-09-03T07:12:48.722 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:48.723 DEBUG:boto:Data: 2022-09-03T07:12:48.723 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.724 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.724 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.725 DEBUG:boto:Params: {} 2022-09-03T07:12:48.726 DEBUG:boto:Token: None 2022-09-03T07:12:48.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.728 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.734 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000143d25afa3f609a3-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.735 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.736 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.737 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.737 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:48.738 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:48.738 DEBUG:boto:Method: GET 2022-09-03T07:12:48.739 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:48.740 DEBUG:boto:Data: 2022-09-03T07:12:48.740 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.741 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.742 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.744 DEBUG:boto:Params: {} 2022-09-03T07:12:48.749 DEBUG:boto:Token: None 2022-09-03T07:12:48.749 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.755 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a46c40336c275d95-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.756 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.757 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.758 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.758 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.759 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:48.759 DEBUG:boto:Data: 2022-09-03T07:12:48.760 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.761 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.761 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.762 DEBUG:boto:Params: {} 2022-09-03T07:12:48.762 DEBUG:boto:Token: None 2022-09-03T07:12:48.763 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44 2022-09-03T07:12:48.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AhMZ9p1YrW2nHhtgkp/z7a88/Bg= 2022-09-03T07:12:48.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AhMZ9p1YrW2nHhtgkp/z7a88/Bg=', 'Content-Length': '0'} 2022-09-03T07:12:48.771 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a62499f6c1605c5c-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.772 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.772 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.773 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.773 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:48.774 DEBUG:boto:Data: 2022-09-03T07:12:48.776 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.777 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.778 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.778 DEBUG:boto:Params: {} 2022-09-03T07:12:48.779 DEBUG:boto:Token: None 2022-09-03T07:12:48.780 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44 2022-09-03T07:12:48.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AhMZ9p1YrW2nHhtgkp/z7a88/Bg= 2022-09-03T07:12:48.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AhMZ9p1YrW2nHhtgkp/z7a88/Bg=', 'Content-Length': '0'} 2022-09-03T07:12:48.785 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fd7360ed4e0b63d-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.786 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:12:48.787 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.788 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.789 DEBUG:boto:Method: GET 2022-09-03T07:12:48.789 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:48.790 DEBUG:boto:Data: 2022-09-03T07:12:48.790 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.791 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.792 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.792 DEBUG:boto:Params: {} 2022-09-03T07:12:48.794 DEBUG:boto:Token: None 2022-09-03T07:12:48.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44 2022-09-03T07:12:48.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3ldbgWGnQID0zcYuSgC21cS338A= 2022-09-03T07:12:48.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3ldbgWGnQID0zcYuSgC21cS338A=', 'Content-Length': '0'} 2022-09-03T07:12:48.802 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f35051906ef9579-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.846 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:12:48.847 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:12:48.848 DEBUG:boto:Method: GET 2022-09-03T07:12:48.849 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:12:48.849 DEBUG:boto:Data: 2022-09-03T07:12:48.850 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.850 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.851 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.851 DEBUG:boto:Params: {} 2022-09-03T07:12:48.852 DEBUG:boto:Token: None 2022-09-03T07:12:48.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj44 2022-09-03T07:12:48.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3ldbgWGnQID0zcYuSgC21cS338A= 2022-09-03T07:12:48.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3ldbgWGnQID0zcYuSgC21cS338A=', 'Content-Length': '0'} 2022-09-03T07:12:48.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b5bd59e8edc0fff5-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.859 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.859 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.860 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:48.860 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:48.861 DEBUG:boto:Method: GET 2022-09-03T07:12:48.862 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:48.862 DEBUG:boto:Data: 2022-09-03T07:12:48.863 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.864 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.865 DEBUG:boto:Params: {} 2022-09-03T07:12:48.865 DEBUG:boto:Token: None 2022-09-03T07:12:48.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.873 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac2e74950b18afdf-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.874 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.876 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:48.877 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:48.878 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:48.879 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:12:48.880 DEBUG:boto:Method: GET 2022-09-03T07:12:48.881 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:12:48.888 DEBUG:boto:Data: 2022-09-03T07:12:48.888 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.889 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.890 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.891 DEBUG:boto:Params: {} 2022-09-03T07:12:48.892 DEBUG:boto:Token: None 2022-09-03T07:12:48.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/ 2022-09-03T07:12:48.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM= 2022-09-03T07:12:48.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ODlCdmJhv0mAOewmmEfm1YjuCM=', 'Content-Length': '0'} 2022-09-03T07:12:48.900 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000deab59228c282202-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.901 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:48.902 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:12:48.903 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:48.912 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:48.913 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.914 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.915 DEBUG:boto:Method: GET 2022-09-03T07:12:48.917 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.918 DEBUG:boto:Data: 2022-09-03T07:12:48.919 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.921 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.922 DEBUG:boto:Params: {} 2022-09-03T07:12:48.923 DEBUG:boto:Token: None 2022-09-03T07:12:48.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fIhwjwiIXx9Rblad5GdwWNcWZq0= 2022-09-03T07:12:48.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fIhwjwiIXx9Rblad5GdwWNcWZq0=', 'Content-Length': '0'} 2022-09-03T07:12:48.932 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024b14ed8d69a3d03-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.934 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:48.935 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:48.936 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.937 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.938 DEBUG:boto:Method: GET 2022-09-03T07:12:48.939 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.941 DEBUG:boto:Data: 2022-09-03T07:12:48.942 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.943 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.943 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.944 DEBUG:boto:Params: {} 2022-09-03T07:12:48.945 DEBUG:boto:Token: None 2022-09-03T07:12:48.947 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fIhwjwiIXx9Rblad5GdwWNcWZq0= 2022-09-03T07:12:48.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fIhwjwiIXx9Rblad5GdwWNcWZq0=', 'Content-Length': '0'} 2022-09-03T07:12:48.953 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003690d447627eae6f-006312fe70-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.954 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:48.955 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:48.956 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.957 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.958 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.960 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.961 DEBUG:boto:Data: 2022-09-03T07:12:48.963 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.964 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:48.965 DEBUG:boto:Port: 8001 2022-09-03T07:12:48.982 DEBUG:boto:Params: {} 2022-09-03T07:12:48.983 DEBUG:boto:Token: None 2022-09-03T07:12:48.984 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:48 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:deX5zEUhPbCpbZcv/HdILVGg//g= 2022-09-03T07:12:48.986 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:deX5zEUhPbCpbZcv/HdILVGg//g=', 'Content-Length': '0'} 2022-09-03T07:12:48.989 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf8c167c2e035938-006312fe70-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:48.990 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:48.990 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:48.991 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.992 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.993 DEBUG:boto:Method: HEAD 2022-09-03T07:12:48.994 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:48.994 DEBUG:boto:Data: 2022-09-03T07:12:48.995 DEBUG:boto:Headers: {} 2022-09-03T07:12:48.995 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:48.998 DEBUG:boto:Port: 8000 2022-09-03T07:12:48.999 DEBUG:boto:Params: {} 2022-09-03T07:12:48.999 DEBUG:boto:Token: None 2022-09-03T07:12:49.000 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:49.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X/V5BwUxslNnM0i7uIg2ztzebIk= 2022-09-03T07:12:49.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X/V5BwUxslNnM0i7uIg2ztzebIk=', 'Content-Length': '0'} 2022-09-03T07:12:49.006 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8445c1bfee8626a-006312fe71-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.008 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:12:49.009 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:49.010 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:49.010 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:49.011 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:49.011 DEBUG:boto:Method: GET 2022-09-03T07:12:49.012 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:49.012 DEBUG:boto:Data: 2022-09-03T07:12:49.013 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:49.015 DEBUG:boto:Port: 8001 2022-09-03T07:12:49.015 DEBUG:boto:Params: {} 2022-09-03T07:12:49.016 DEBUG:boto:Token: None 2022-09-03T07:12:49.016 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:49.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:POV5hjNLiyDIEUD4jVKNVXr0/T4= 2022-09-03T07:12:49.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:POV5hjNLiyDIEUD4jVKNVXr0/T4=', 'Content-Length': '0'} 2022-09-03T07:12:49.021 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b83a0ed4f66212ba-006312fe71-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.022 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:49.023 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:49.024 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:49.024 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:12:49.025 DEBUG:boto:Method: GET 2022-09-03T07:12:49.025 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:12:49.026 DEBUG:boto:Data: 2022-09-03T07:12:49.027 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.027 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:49.028 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.029 DEBUG:boto:Params: {} 2022-09-03T07:12:49.030 DEBUG:boto:Token: None 2022-09-03T07:12:49.030 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:12:49.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:POV5hjNLiyDIEUD4jVKNVXr0/T4= 2022-09-03T07:12:49.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:POV5hjNLiyDIEUD4jVKNVXr0/T4=', 'Content-Length': '0'} 2022-09-03T07:12:49.045 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000067674cef403c3c64-006312fe71-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.049 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:49.049 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:49.051 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:49.051 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:49.052 DEBUG:boto:Method: GET 2022-09-03T07:12:49.055 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:49.056 DEBUG:boto:Data: 2022-09-03T07:12:49.057 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:49.058 DEBUG:boto:Port: 8001 2022-09-03T07:12:49.058 DEBUG:boto:Params: {} 2022-09-03T07:12:49.059 DEBUG:boto:Token: None 2022-09-03T07:12:49.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/ 2022-09-03T07:12:49.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V87HdKGQj96t0hj10odbMYqlZdI= 2022-09-03T07:12:49.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V87HdKGQj96t0hj10odbMYqlZdI=', 'Content-Length': '0'} 2022-09-03T07:12:49.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d0ae8555a0d456e-006312fe71-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.070 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:49.071 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:49.072 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:49.072 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:49.073 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:49.073 DEBUG:boto:Method: GET 2022-09-03T07:12:49.074 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:49.075 DEBUG:boto:Data: 2022-09-03T07:12:49.075 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.076 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:49.076 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.077 DEBUG:boto:Params: {} 2022-09-03T07:12:49.077 DEBUG:boto:Token: None 2022-09-03T07:12:49.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/ 2022-09-03T07:12:49.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V87HdKGQj96t0hj10odbMYqlZdI= 2022-09-03T07:12:49.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V87HdKGQj96t0hj10odbMYqlZdI=', 'Content-Length': '0'} 2022-09-03T07:12:49.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000029dcb7420b946d50-006312fe71-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.086 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:49.087 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:49.087 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:49.088 DEBUG:boto:Method: HEAD 2022-09-03T07:12:49.089 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:49.089 DEBUG:boto:Data: 2022-09-03T07:12:49.090 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.090 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:49.091 DEBUG:boto:Port: 8001 2022-09-03T07:12:49.093 DEBUG:boto:Params: {} 2022-09-03T07:12:49.094 DEBUG:boto:Token: None 2022-09-03T07:12:49.095 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj55 2022-09-03T07:12:49.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1G4vBQLP8WtfwFtMBFfireRx3os= 2022-09-03T07:12:49.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1G4vBQLP8WtfwFtMBFfireRx3os=', 'Content-Length': '0'} 2022-09-03T07:12:49.104 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005300abf7d632f96c-006312fe71-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.105 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:49.105 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:49.106 DEBUG:boto:Method: HEAD 2022-09-03T07:12:49.107 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:49.107 DEBUG:boto:Data: 2022-09-03T07:12:49.108 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.108 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:49.109 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.109 DEBUG:boto:Params: {} 2022-09-03T07:12:49.110 DEBUG:boto:Token: None 2022-09-03T07:12:49.111 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj55 2022-09-03T07:12:49.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1G4vBQLP8WtfwFtMBFfireRx3os= 2022-09-03T07:12:49.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1G4vBQLP8WtfwFtMBFfireRx3os=', 'Content-Length': '0'} 2022-09-03T07:12:49.116 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000719570b14ddf248f-006312fe71-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.117 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:12:49.118 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:49.120 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:49.121 DEBUG:boto:Method: GET 2022-09-03T07:12:49.122 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:49.122 DEBUG:boto:Data: 2022-09-03T07:12:49.123 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:49.124 DEBUG:boto:Port: 8001 2022-09-03T07:12:49.124 DEBUG:boto:Params: {} 2022-09-03T07:12:49.125 DEBUG:boto:Token: None 2022-09-03T07:12:49.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj55 2022-09-03T07:12:49.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:koUDiXFV8p8dfK7kM46Hn43LikE= 2022-09-03T07:12:49.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:koUDiXFV8p8dfK7kM46Hn43LikE=', 'Content-Length': '0'} 2022-09-03T07:12:49.131 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055bff1ee824a58fe-006312fe71-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.132 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:12:49.132 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:12:49.133 DEBUG:boto:Method: GET 2022-09-03T07:12:49.133 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:12:49.134 DEBUG:boto:Data: 2022-09-03T07:12:49.134 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.135 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:49.136 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.136 DEBUG:boto:Params: {} 2022-09-03T07:12:49.137 DEBUG:boto:Token: None 2022-09-03T07:12:49.137 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj55 2022-09-03T07:12:49.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:koUDiXFV8p8dfK7kM46Hn43LikE= 2022-09-03T07:12:49.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:koUDiXFV8p8dfK7kM46Hn43LikE=', 'Content-Length': '0'} 2022-09-03T07:12:49.141 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000871beaacd72a717d-006312fe71-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.142 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:49.143 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:49.143 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:49.144 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:49.144 DEBUG:boto:Method: GET 2022-09-03T07:12:49.145 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:49.146 DEBUG:boto:Data: 2022-09-03T07:12:49.146 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.147 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:12:49.147 DEBUG:boto:Port: 8001 2022-09-03T07:12:49.148 DEBUG:boto:Params: {} 2022-09-03T07:12:49.149 DEBUG:boto:Token: None 2022-09-03T07:12:49.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/ 2022-09-03T07:12:49.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V87HdKGQj96t0hj10odbMYqlZdI= 2022-09-03T07:12:49.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V87HdKGQj96t0hj10odbMYqlZdI=', 'Content-Length': '0'} 2022-09-03T07:12:49.157 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000066a3e0f987806625-006312fe71-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.199 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:49.201 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:49.203 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:49.204 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:49.205 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:12:49.206 DEBUG:boto:Method: GET 2022-09-03T07:12:49.207 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:12:49.209 DEBUG:boto:Data: 2022-09-03T07:12:49.210 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.217 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:12:49.218 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.218 DEBUG:boto:Params: {} 2022-09-03T07:12:49.219 DEBUG:boto:Token: None 2022-09-03T07:12:49.220 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/ 2022-09-03T07:12:49.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V87HdKGQj96t0hj10odbMYqlZdI= 2022-09-03T07:12:49.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V87HdKGQj96t0hj10odbMYqlZdI=', 'Content-Length': '0'} 2022-09-03T07:12:49.224 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b2b83723ed80487-006312fe71-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.225 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:12:49.226 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:12:49.226 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '17', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:12:49.227 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '17', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:12:49.228 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --num-shards 17 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:12:49.283 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:12:49.284 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:12:49.332 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:12:49.333 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-75 2022-09-03T07:12:49.337 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 5 2022-09-03T07:12:49.404 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:49.405 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:49.406 DEBUG:boto:Method: HEAD 2022-09-03T07:12:49.407 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:49.407 DEBUG:boto:Data: 2022-09-03T07:12:49.408 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.408 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.409 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.409 DEBUG:boto:Params: {} 2022-09-03T07:12:49.410 DEBUG:boto:Token: None 2022-09-03T07:12:49.410 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/ 2022-09-03T07:12:49.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E= 2022-09-03T07:12:49.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E=', 'Content-Length': '0'} 2022-09-03T07:12:49.419 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b001f4d360dbc15d-006312fe71-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.420 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:12:49.420 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:12:49.421 DEBUG:boto:Method: PUT 2022-09-03T07:12:49.422 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:12:49.422 DEBUG:boto:Data: 2022-09-03T07:12:49.423 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:12:49.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.435 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.436 DEBUG:boto:Params: {} 2022-09-03T07:12:49.437 DEBUG:boto:Token: None 2022-09-03T07:12:49.438 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj17 2022-09-03T07:12:49.439 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3QlpopUSHbMFLxUf4DaJysZ7uNU= 2022-09-03T07:12:49.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3QlpopUSHbMFLxUf4DaJysZ7uNU='} 2022-09-03T07:12:49.483 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000cc51eba0fb44e19b-006312fe71-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.484 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:49.485 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:49.486 DEBUG:boto:Method: HEAD 2022-09-03T07:12:49.487 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:49.488 DEBUG:boto:Data: 2022-09-03T07:12:49.489 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.491 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.492 DEBUG:boto:Params: {} 2022-09-03T07:12:49.493 DEBUG:boto:Token: None 2022-09-03T07:12:49.494 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/ 2022-09-03T07:12:49.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E= 2022-09-03T07:12:49.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E=', 'Content-Length': '0'} 2022-09-03T07:12:49.499 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bfbabcfd7ab32ba1-006312fe71-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.501 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:12:49.502 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:12:49.503 DEBUG:boto:Method: PUT 2022-09-03T07:12:49.503 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:12:49.504 DEBUG:boto:Data: 2022-09-03T07:12:49.505 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:12:49.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.506 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.506 DEBUG:boto:Params: {} 2022-09-03T07:12:49.507 DEBUG:boto:Token: None 2022-09-03T07:12:49.508 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj34 2022-09-03T07:12:49.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e17bKv+JjDMMt4Vqv2rBPx+MGKk= 2022-09-03T07:12:49.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e17bKv+JjDMMt4Vqv2rBPx+MGKk='} 2022-09-03T07:12:49.518 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000017146cbf9f8dc72d-006312fe71-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.522 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:49.523 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:49.524 DEBUG:boto:Method: HEAD 2022-09-03T07:12:49.524 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:49.525 DEBUG:boto:Data: 2022-09-03T07:12:49.526 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.527 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.528 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.529 DEBUG:boto:Params: {} 2022-09-03T07:12:49.529 DEBUG:boto:Token: None 2022-09-03T07:12:49.530 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/ 2022-09-03T07:12:49.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E= 2022-09-03T07:12:49.532 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E=', 'Content-Length': '0'} 2022-09-03T07:12:49.535 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '21'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000029a21a857c6ea917-006312fe71-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.536 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:12:49.536 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:12:49.537 DEBUG:boto:Method: PUT 2022-09-03T07:12:49.537 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:12:49.538 DEBUG:boto:Data: 2022-09-03T07:12:49.540 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:12:49.542 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.542 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.543 DEBUG:boto:Params: {} 2022-09-03T07:12:49.543 DEBUG:boto:Token: None 2022-09-03T07:12:49.544 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj51 2022-09-03T07:12:49.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zg2Q09wmm/WcGFfoTtt0HDHwRqo= 2022-09-03T07:12:49.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zg2Q09wmm/WcGFfoTtt0HDHwRqo='} 2022-09-03T07:12:49.595 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000013c5dde6f7bfafb4-006312fe71-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.596 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:49.597 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:49.597 DEBUG:boto:Method: HEAD 2022-09-03T07:12:49.598 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:49.599 DEBUG:boto:Data: 2022-09-03T07:12:49.600 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.604 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.605 DEBUG:boto:Params: {} 2022-09-03T07:12:49.607 DEBUG:boto:Token: None 2022-09-03T07:12:49.608 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/ 2022-09-03T07:12:49.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E= 2022-09-03T07:12:49.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E=', 'Content-Length': '0'} 2022-09-03T07:12:49.621 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000dbb78a746eeb1509-006312fe71-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.622 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:12:49.622 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:12:49.623 DEBUG:boto:Method: PUT 2022-09-03T07:12:49.624 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:12:49.624 DEBUG:boto:Data: 2022-09-03T07:12:49.625 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:12:49.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.626 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.626 DEBUG:boto:Params: {} 2022-09-03T07:12:49.627 DEBUG:boto:Token: None 2022-09-03T07:12:49.628 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj68 2022-09-03T07:12:49.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:thdxnXJGNWTUmivrUK06Jq/5aoo= 2022-09-03T07:12:49.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:thdxnXJGNWTUmivrUK06Jq/5aoo='} 2022-09-03T07:12:49.641 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000e454bf165e4b3ab3-006312fe71-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.642 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:12:49.643 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:12:49.643 DEBUG:boto:Method: HEAD 2022-09-03T07:12:49.644 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:12:49.644 DEBUG:boto:Data: 2022-09-03T07:12:49.645 DEBUG:boto:Headers: {} 2022-09-03T07:12:49.645 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.647 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.648 DEBUG:boto:Params: {} 2022-09-03T07:12:49.648 DEBUG:boto:Token: None 2022-09-03T07:12:49.649 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/ 2022-09-03T07:12:49.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E= 2022-09-03T07:12:49.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fX93g4MN8LTrWrdXcrOpfnFg5+E=', 'Content-Length': '0'} 2022-09-03T07:12:49.653 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '9'), ('X-RGW-Bytes-Used', '27'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b19294e05690adba-006312fe71-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.654 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:12:49.655 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:12:49.655 DEBUG:boto:Method: PUT 2022-09-03T07:12:49.656 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:12:49.656 DEBUG:boto:Data: 2022-09-03T07:12:49.657 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:12:49.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:12:49.658 DEBUG:boto:Port: 8000 2022-09-03T07:12:49.659 DEBUG:boto:Params: {} 2022-09-03T07:12:49.659 DEBUG:boto:Token: None 2022-09-03T07:12:49.660 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:12:49 GMT /avysmw-75/obj85 2022-09-03T07:12:49.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PDHiiyIzZ3ar5H7XlziF/6PUbgk= 2022-09-03T07:12:49.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:12:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PDHiiyIzZ3ar5H7XlziF/6PUbgk='} 2022-09-03T07:12:49.669 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000d4e4b2e491aa0e69-006312fe71-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:12:49.670 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-75 2022-09-03T07:12:49.671 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:12:49.672 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:12:49.673 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:12:49.738 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:12:49.739 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:12:49.787 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:12:49.783+0000 7f372f153600 1 waiting to reach latest gen 1, on 0.. 2022-09-03T07:12:54.785 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:12:54.783+0000 7f372f153600 1 waiting to reach latest gen 1, on 0.. 2022-09-03T07:12:59.788 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:12:59.786+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:12:59.788 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , ] 2022-09-03T07:12:59.788 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:04.789 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:04.787+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:04.790 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , ] 2022-09-03T07:13:04.790 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:09.792 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:09.788+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:09.793 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , ] 2022-09-03T07:13:09.794 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:14.792 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:14.789+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:14.793 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , ] 2022-09-03T07:13:14.794 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:19.802 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:19.791+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:19.803 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , , , , , , 00000000001.58.5, , , ] 2022-09-03T07:13:19.804 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:24.794 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:24.792+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:24.795 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , , , , , , 00000000001.58.5, , , ] 2022-09-03T07:13:24.795 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:29.795 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:29.793+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:29.796 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , , , , , , 00000000001.58.5, , , ] 2022-09-03T07:13:29.796 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:34.797 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:34.795+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:34.798 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , , , , , , 00000000001.58.5, , , ] 2022-09-03T07:13:34.798 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:39.799 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:39.797+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:39.800 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:39.800 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:44.803 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:44.798+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:44.803 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:44.804 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:49.806 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:49.800+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:49.807 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:49.807 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:54.811 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:54.801+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:54.812 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:54.812 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:59.805 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:13:59.803+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:13:59.806 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:13:59.807 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:04.808 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:04.804+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:04.808 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:04.809 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:09.812 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:09.808+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:09.813 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:09.813 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:14.811 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:14.809+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:14.812 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:14.812 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:19.815 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:19.810+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:19.815 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:19.815 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:24.814 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:24.812+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:24.815 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:24.815 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:29.822 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:29.813+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:29.823 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:29.823 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:34.817 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:34.814+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:34.817 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:34.820 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:39.817 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:39.815+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:39.818 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:39.821 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:44.819 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:44.816+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:44.820 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:44.820 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:49.820 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:49.818+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:49.821 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:49.821 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:54.831 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:54.820+0000 7f372f153600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:54.832 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:54.832 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:59.823 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:59.821+0000 7f372f153600 1 bucket sync caught up with source: 2022-09-03T07:14:59.824 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:59.824 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:59.825 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:14:59.821+0000 7f372f153600 0 bucket checkpoint complete 2022-09-03T07:14:59.845 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-75 2022-09-03T07:14:59.851 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:14:59.852 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:14:59.853 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:14:59.926 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:14:59.927 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:14:59.982 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:14:59.978+0000 7fea3059f600 1 waiting for incremental sync to catch up: 2022-09-03T07:14:59.983 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.36.5, , , , , , , , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:14:59.983 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:04.986 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:15:04.980+0000 7fea3059f600 1 bucket sync caught up with source: 2022-09-03T07:15:04.987 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:04.987 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:04.988 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:15:04.980+0000 7fea3059f600 0 bucket checkpoint complete 2022-09-03T07:15:05.009 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-75 2022-09-03T07:15:05.010 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:05.011 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:05.012 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:15:05.075 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:15:05.076 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:15:05.148 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:05.132+0000 7fd973d73600 1 waiting for incremental sync to catch up: 2022-09-03T07:15:05.148 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:05.150 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:10.138 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:10.133+0000 7fd973d73600 1 waiting for incremental sync to catch up: 2022-09-03T07:15:10.139 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:10.139 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:15.137 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:15.135+0000 7fd973d73600 1 waiting for incremental sync to catch up: 2022-09-03T07:15:15.137 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:15.143 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:20.151 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:20.136+0000 7fd973d73600 1 waiting for incremental sync to catch up: 2022-09-03T07:15:20.152 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:20.156 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:25.143 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:25.137+0000 7fd973d73600 1 waiting for incremental sync to catch up: 2022-09-03T07:15:25.144 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:25.144 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:30.146 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:30.139+0000 7fd973d73600 1 bucket sync caught up with source: 2022-09-03T07:15:30.148 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:30.148 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000001.36.5, , , , , , 00000000001.57.5, , 00000000001.69.5, , , , 00000000001.56.5, 00000000001.58.5, , , ] 2022-09-03T07:15:30.148 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:30.139+0000 7fd973d73600 0 bucket checkpoint complete 2022-09-03T07:15:30.164 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-75 2022-09-03T07:15:30.170 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:30.171 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:30.179 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:15:30.249 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:15:30.250 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:15:30.302 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:15:30.294+0000 7fe754dc8600 1 bucket sync caught up with source: 2022-09-03T07:15:30.303 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.114.5, 00000000001.325.5, , , 00000000001.252.5, , , 00000000001.370.5, 00000000003.295.5, 00000000003.215.5] 2022-09-03T07:15:30.303 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.114.5, 00000000001.325.5, , , 00000000001.252.5, , , 00000000001.370.5, 00000000003.295.5, 00000000003.215.5] 2022-09-03T07:15:30.304 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:15:30.294+0000 7fe754dc8600 0 bucket checkpoint complete 2022-09-03T07:15:30.318 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-75 2022-09-03T07:15:30.319 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:30.320 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:30.322 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:15:30.386 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:15:30.387 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:15:30.422 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:15:30.419+0000 7f38256e4600 1 bucket sync caught up with source: 2022-09-03T07:15:30.424 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.114.5, 00000000001.325.5, , , 00000000001.252.5, , , 00000000001.370.5, 00000000003.295.5, 00000000003.215.5] 2022-09-03T07:15:30.425 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.114.5, 00000000001.325.5, , , 00000000001.252.5, , , 00000000001.370.5, 00000000003.295.5, 00000000003.215.5] 2022-09-03T07:15:30.426 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:15:30.419+0000 7f38256e4600 0 bucket checkpoint complete 2022-09-03T07:15:30.451 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-75 2022-09-03T07:15:30.452 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-75 2022-09-03T07:15:30.452 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:30.453 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:30.453 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:15:30.513 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:15:30.513 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:15:30.566 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:15:30.563+0000 7fd27b7e2600 1 bucket sync caught up with source: 2022-09-03T07:15:30.567 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000001.110.5, 00000000001.175.5, , , 00000000001.136.5, , , 00000000001.279.5, 00000000003.302.5, 00000000003.131.5] 2022-09-03T07:15:30.567 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000001.110.5, 00000000001.175.5, , , 00000000001.136.5, , , 00000000001.279.5, 00000000003.302.5, 00000000003.131.5] 2022-09-03T07:15:30.568 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:15:30.563+0000 7fd27b7e2600 0 bucket checkpoint complete 2022-09-03T07:15:30.589 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-75 2022-09-03T07:15:30.590 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:30.591 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:30.592 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:15:30.660 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:15:30.661 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:15:30.734 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:30.732+0000 7faf76ff2600 1 bucket sync caught up with source: 2022-09-03T07:15:30.734 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000001.110.5, 00000000001.175.5, , , 00000000001.136.5, , , 00000000001.279.5, 00000000003.302.5, 00000000003.131.5] 2022-09-03T07:15:30.734 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000001.110.5, 00000000001.175.5, , , 00000000001.136.5, , , 00000000001.279.5, 00000000003.302.5, 00000000003.131.5] 2022-09-03T07:15:30.735 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:30.732+0000 7faf76ff2600 0 bucket checkpoint complete 2022-09-03T07:15:30.755 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-75 2022-09-03T07:15:30.756 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:15:30.757 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:30.758 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:30.758 DEBUG:boto:Method: HEAD 2022-09-03T07:15:30.759 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:30.760 DEBUG:boto:Data: 2022-09-03T07:15:30.760 DEBUG:boto:Headers: {} 2022-09-03T07:15:30.761 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:30.765 DEBUG:boto:Port: 8000 2022-09-03T07:15:30.766 DEBUG:boto:Params: {} 2022-09-03T07:15:30.769 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:15:30.771 DEBUG:boto:Token: None 2022-09-03T07:15:30.772 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:30 GMT /avysmw-75/ 2022-09-03T07:15:30.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LWDUEDNlFpRo3hS1yPFbG3rN1U8= 2022-09-03T07:15:30.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LWDUEDNlFpRo3hS1yPFbG3rN1U8=', 'Content-Length': '0'} 2022-09-03T07:15:30.790 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003b38c6cb2c24595e-006312ff12-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:30.802 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:30.804 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:30.805 DEBUG:boto:Method: HEAD 2022-09-03T07:15:30.809 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:30.809 DEBUG:boto:Data: 2022-09-03T07:15:30.810 DEBUG:boto:Headers: {} 2022-09-03T07:15:30.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:30.811 DEBUG:boto:Port: 8000 2022-09-03T07:15:30.811 DEBUG:boto:Params: {} 2022-09-03T07:15:30.812 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:15:30.813 DEBUG:boto:Token: None 2022-09-03T07:15:30.813 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:30 GMT /avysmw-75/ 2022-09-03T07:15:30.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LWDUEDNlFpRo3hS1yPFbG3rN1U8= 2022-09-03T07:15:30.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LWDUEDNlFpRo3hS1yPFbG3rN1U8=', 'Content-Length': '0'} 2022-09-03T07:15:30.822 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009384eb23a8c1859e-006312ff12-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:30.823 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:15:30.824 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:30.825 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:30.825 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.826 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.830 DEBUG:boto:Method: GET 2022-09-03T07:15:30.831 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.832 DEBUG:boto:Data: 2022-09-03T07:15:30.832 DEBUG:boto:Headers: {} 2022-09-03T07:15:30.833 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:30.833 DEBUG:boto:Port: 8000 2022-09-03T07:15:30.834 DEBUG:boto:Params: {} 2022-09-03T07:15:30.834 DEBUG:boto:Token: None 2022-09-03T07:15:30.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:30 GMT /avysmw-75/?versions 2022-09-03T07:15:30.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BcqDtT9xjhn9HOy2tXkBTYFuS7g= 2022-09-03T07:15:30.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BcqDtT9xjhn9HOy2tXkBTYFuS7g=', 'Content-Length': '0'} 2022-09-03T07:15:30.845 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001b2f2e3f004df8be-006312ff12-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:30.846 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:30.851 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:15:30.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:15:30.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:15:30.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:15:30.854 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:15:30.860 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:15:30.861 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:15:30.862 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:15:30.863 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:15:30.864 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:15:30.865 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:15:30.867 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:30.868 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:30.868 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.869 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.869 DEBUG:boto:Method: GET 2022-09-03T07:15:30.870 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.871 DEBUG:boto:Data: 2022-09-03T07:15:30.872 DEBUG:boto:Headers: {} 2022-09-03T07:15:30.873 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:30.875 DEBUG:boto:Port: 8000 2022-09-03T07:15:30.876 DEBUG:boto:Params: {} 2022-09-03T07:15:30.877 DEBUG:boto:Token: None 2022-09-03T07:15:30.879 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:30 GMT /avysmw-75/?versions 2022-09-03T07:15:30.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BcqDtT9xjhn9HOy2tXkBTYFuS7g= 2022-09-03T07:15:30.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BcqDtT9xjhn9HOy2tXkBTYFuS7g=', 'Content-Length': '0'} 2022-09-03T07:15:30.886 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098cde051054eb2b1-006312ff12-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:30.887 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:30.888 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:15:30.889 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:15:30.890 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:15:30.890 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:15:30.891 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:15:30.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:15:30.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:15:30.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:15:30.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:15:30.894 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:15:30.894 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:30.895 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:30.896 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.897 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.897 DEBUG:boto:Method: GET 2022-09-03T07:15:30.898 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.898 DEBUG:boto:Data: 2022-09-03T07:15:30.899 DEBUG:boto:Headers: {} 2022-09-03T07:15:30.900 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:30.900 DEBUG:boto:Port: 8000 2022-09-03T07:15:30.901 DEBUG:boto:Params: {} 2022-09-03T07:15:30.902 DEBUG:boto:Token: None 2022-09-03T07:15:30.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:30 GMT /avysmw-75/?versions 2022-09-03T07:15:30.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BcqDtT9xjhn9HOy2tXkBTYFuS7g= 2022-09-03T07:15:30.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BcqDtT9xjhn9HOy2tXkBTYFuS7g=', 'Content-Length': '0'} 2022-09-03T07:15:30.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e63fe0e73e90f615-006312ff12-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:30.913 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:30.919 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:30.919 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:30.920 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.920 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.921 DEBUG:boto:Method: GET 2022-09-03T07:15:30.921 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:30.923 DEBUG:boto:Data: 2022-09-03T07:15:30.923 DEBUG:boto:Headers: {} 2022-09-03T07:15:30.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:30.925 DEBUG:boto:Port: 8000 2022-09-03T07:15:30.926 DEBUG:boto:Params: {} 2022-09-03T07:15:30.927 DEBUG:boto:Token: None 2022-09-03T07:15:30.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:30 GMT /avysmw-75/?versions 2022-09-03T07:15:30.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BcqDtT9xjhn9HOy2tXkBTYFuS7g= 2022-09-03T07:15:30.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BcqDtT9xjhn9HOy2tXkBTYFuS7g=', 'Content-Length': '0'} 2022-09-03T07:15:30.934 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d7b2294e84c9abc-006312ff12-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:30.936 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:30.951 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:15:30.952 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:30.953 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:30.953 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:30.958 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:30.964 DEBUG:boto:Method: GET 2022-09-03T07:15:30.965 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:30.966 DEBUG:boto:Data: 2022-09-03T07:15:30.967 DEBUG:boto:Headers: {} 2022-09-03T07:15:30.968 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:30.969 DEBUG:boto:Port: 8000 2022-09-03T07:15:30.970 DEBUG:boto:Params: {} 2022-09-03T07:15:30.971 DEBUG:boto:Token: None 2022-09-03T07:15:30.972 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:30 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:30.974 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OTOww77hYOp7gTFOTy7kuhLrsmo= 2022-09-03T07:15:30.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:30 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OTOww77hYOp7gTFOTy7kuhLrsmo=', 'Content-Length': '0'} 2022-09-03T07:15:30.990 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9c189473b35177d-006312ff12-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:30 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:30.992 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:30.993 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:30.999 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.000 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.009 DEBUG:boto:Method: GET 2022-09-03T07:15:31.010 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.011 DEBUG:boto:Data: 2022-09-03T07:15:31.011 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.018 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.020 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.021 DEBUG:boto:Params: {} 2022-09-03T07:15:31.022 DEBUG:boto:Token: None 2022-09-03T07:15:31.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3brtKfwLd++jydqWILoQIunlEwY= 2022-09-03T07:15:31.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3brtKfwLd++jydqWILoQIunlEwY=', 'Content-Length': '0'} 2022-09-03T07:15:31.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009ec27b496950ae22-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.033 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:31.034 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:31.035 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.035 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.036 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.036 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.037 DEBUG:boto:Data: 2022-09-03T07:15:31.037 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.038 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.038 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.039 DEBUG:boto:Params: {} 2022-09-03T07:15:31.039 DEBUG:boto:Token: None 2022-09-03T07:15:31.042 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.042 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PUHC8p0i/lUYxfrYqVsx1v5Ouj8= 2022-09-03T07:15:31.043 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PUHC8p0i/lUYxfrYqVsx1v5Ouj8=', 'Content-Length': '0'} 2022-09-03T07:15:31.052 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018e0c81e3761cf7c-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.053 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:31.053 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:31.054 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.054 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.055 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.055 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.056 DEBUG:boto:Data: 2022-09-03T07:15:31.057 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.058 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.058 DEBUG:boto:Params: {} 2022-09-03T07:15:31.059 DEBUG:boto:Token: None 2022-09-03T07:15:31.059 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PUHC8p0i/lUYxfrYqVsx1v5Ouj8= 2022-09-03T07:15:31.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PUHC8p0i/lUYxfrYqVsx1v5Ouj8=', 'Content-Length': '0'} 2022-09-03T07:15:31.073 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024feeda3a54c0501-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.075 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:15:31.075 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:31.076 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:31.077 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.077 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.078 DEBUG:boto:Method: GET 2022-09-03T07:15:31.079 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.079 DEBUG:boto:Data: 2022-09-03T07:15:31.080 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.080 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.081 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.082 DEBUG:boto:Params: {} 2022-09-03T07:15:31.082 DEBUG:boto:Token: None 2022-09-03T07:15:31.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3brtKfwLd++jydqWILoQIunlEwY= 2022-09-03T07:15:31.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3brtKfwLd++jydqWILoQIunlEwY=', 'Content-Length': '0'} 2022-09-03T07:15:31.087 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9d1a574fe477e9c-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.088 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:31.089 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:31.090 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.091 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.092 DEBUG:boto:Method: GET 2022-09-03T07:15:31.093 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.094 DEBUG:boto:Data: 2022-09-03T07:15:31.095 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.096 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.097 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.098 DEBUG:boto:Params: {} 2022-09-03T07:15:31.099 DEBUG:boto:Token: None 2022-09-03T07:15:31.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:31.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3brtKfwLd++jydqWILoQIunlEwY= 2022-09-03T07:15:31.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3brtKfwLd++jydqWILoQIunlEwY=', 'Content-Length': '0'} 2022-09-03T07:15:31.105 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086f1588802138545-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.112 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.113 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.113 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:31.114 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:31.114 DEBUG:boto:Method: GET 2022-09-03T07:15:31.115 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:31.115 DEBUG:boto:Data: 2022-09-03T07:15:31.116 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.116 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.117 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.118 DEBUG:boto:Params: {} 2022-09-03T07:15:31.119 DEBUG:boto:Token: None 2022-09-03T07:15:31.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.124 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000043ef3e591010dd01-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.125 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:31.126 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.127 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.127 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:31.128 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:31.129 DEBUG:boto:Method: GET 2022-09-03T07:15:31.130 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:31.131 DEBUG:boto:Data: 2022-09-03T07:15:31.132 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.133 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.133 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.134 DEBUG:boto:Params: {} 2022-09-03T07:15:31.135 DEBUG:boto:Token: None 2022-09-03T07:15:31.136 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.149 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000421a9ffabb90b84f-006312ff13-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.155 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:31.157 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:31.158 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:31.159 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.160 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:31.160 DEBUG:boto:Data: 2022-09-03T07:15:31.161 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.162 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.163 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.164 DEBUG:boto:Params: {} 2022-09-03T07:15:31.165 DEBUG:boto:Token: None 2022-09-03T07:15:31.166 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj11 2022-09-03T07:15:31.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dHpGShA90nIUVFZNrSdWIWD2Y6I= 2022-09-03T07:15:31.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dHpGShA90nIUVFZNrSdWIWD2Y6I=', 'Content-Length': '0'} 2022-09-03T07:15:31.172 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b888ddc56113521-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.174 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:31.175 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:31.176 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.177 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:31.178 DEBUG:boto:Data: 2022-09-03T07:15:31.181 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.183 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.184 DEBUG:boto:Params: {} 2022-09-03T07:15:31.186 DEBUG:boto:Token: None 2022-09-03T07:15:31.187 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj11 2022-09-03T07:15:31.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dHpGShA90nIUVFZNrSdWIWD2Y6I= 2022-09-03T07:15:31.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dHpGShA90nIUVFZNrSdWIWD2Y6I=', 'Content-Length': '0'} 2022-09-03T07:15:31.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f16df2fb0d227755-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:15:31.217 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:31.218 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:31.218 DEBUG:boto:Method: GET 2022-09-03T07:15:31.219 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:31.220 DEBUG:boto:Data: 2022-09-03T07:15:31.220 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.221 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.221 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.222 DEBUG:boto:Params: {} 2022-09-03T07:15:31.223 DEBUG:boto:Token: None 2022-09-03T07:15:31.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj11 2022-09-03T07:15:31.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IRuZf64ygA2CowtoOXBlu2mFJo4= 2022-09-03T07:15:31.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IRuZf64ygA2CowtoOXBlu2mFJo4=', 'Content-Length': '0'} 2022-09-03T07:15:31.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a4f391e85216a8a-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.235 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:31.237 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:31.238 DEBUG:boto:Method: GET 2022-09-03T07:15:31.239 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:31.240 DEBUG:boto:Data: 2022-09-03T07:15:31.241 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.241 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.243 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.244 DEBUG:boto:Params: {} 2022-09-03T07:15:31.245 DEBUG:boto:Token: None 2022-09-03T07:15:31.246 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj11 2022-09-03T07:15:31.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IRuZf64ygA2CowtoOXBlu2mFJo4= 2022-09-03T07:15:31.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IRuZf64ygA2CowtoOXBlu2mFJo4=', 'Content-Length': '0'} 2022-09-03T07:15:31.253 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ccba9e3eacef1c0d-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.297 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.299 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.300 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:31.301 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:31.302 DEBUG:boto:Method: GET 2022-09-03T07:15:31.303 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:31.304 DEBUG:boto:Data: 2022-09-03T07:15:31.305 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.307 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.308 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.309 DEBUG:boto:Params: {} 2022-09-03T07:15:31.310 DEBUG:boto:Token: None 2022-09-03T07:15:31.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.319 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a7f8c7a15c13d105-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.320 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:31.321 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.322 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.330 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:31.330 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:31.331 DEBUG:boto:Method: GET 2022-09-03T07:15:31.332 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:31.332 DEBUG:boto:Data: 2022-09-03T07:15:31.333 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.333 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.334 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.335 DEBUG:boto:Params: {} 2022-09-03T07:15:31.335 DEBUG:boto:Token: None 2022-09-03T07:15:31.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.355 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b16bfea60c895129-006312ff13-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.356 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:31.358 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:15:31.359 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.360 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.360 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.361 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.362 DEBUG:boto:Method: GET 2022-09-03T07:15:31.363 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.364 DEBUG:boto:Data: 2022-09-03T07:15:31.365 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.367 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.368 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.369 DEBUG:boto:Params: {} 2022-09-03T07:15:31.370 DEBUG:boto:Token: None 2022-09-03T07:15:31.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H8d4O7ZpL7SCZY0mde7F4+GEBuM= 2022-09-03T07:15:31.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H8d4O7ZpL7SCZY0mde7F4+GEBuM=', 'Content-Length': '0'} 2022-09-03T07:15:31.376 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bb6e69042daa763f-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.377 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.378 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.378 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.379 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.380 DEBUG:boto:Method: GET 2022-09-03T07:15:31.380 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.381 DEBUG:boto:Data: 2022-09-03T07:15:31.381 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.382 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.383 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.383 DEBUG:boto:Params: {} 2022-09-03T07:15:31.385 DEBUG:boto:Token: None 2022-09-03T07:15:31.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H8d4O7ZpL7SCZY0mde7F4+GEBuM= 2022-09-03T07:15:31.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H8d4O7ZpL7SCZY0mde7F4+GEBuM=', 'Content-Length': '0'} 2022-09-03T07:15:31.389 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000011897b58ebdc865d-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.390 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.391 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.391 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.392 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.393 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.394 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.402 DEBUG:boto:Data: 2022-09-03T07:15:31.403 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.403 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.404 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.404 DEBUG:boto:Params: {} 2022-09-03T07:15:31.405 DEBUG:boto:Token: None 2022-09-03T07:15:31.406 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.407 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zc6PoXZcNhuuKH5e/Gw5wMNeGCw= 2022-09-03T07:15:31.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zc6PoXZcNhuuKH5e/Gw5wMNeGCw=', 'Content-Length': '0'} 2022-09-03T07:15:31.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a3668656b2780f8-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.416 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.417 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.418 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.418 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.419 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.419 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.420 DEBUG:boto:Data: 2022-09-03T07:15:31.420 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.421 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.422 DEBUG:boto:Params: {} 2022-09-03T07:15:31.422 DEBUG:boto:Token: None 2022-09-03T07:15:31.423 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zc6PoXZcNhuuKH5e/Gw5wMNeGCw= 2022-09-03T07:15:31.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zc6PoXZcNhuuKH5e/Gw5wMNeGCw=', 'Content-Length': '0'} 2022-09-03T07:15:31.428 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0de0c5740831a67-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.429 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:15:31.429 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.430 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.431 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.431 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.432 DEBUG:boto:Method: GET 2022-09-03T07:15:31.432 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.433 DEBUG:boto:Data: 2022-09-03T07:15:31.433 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.435 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.435 DEBUG:boto:Params: {} 2022-09-03T07:15:31.436 DEBUG:boto:Token: None 2022-09-03T07:15:31.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H8d4O7ZpL7SCZY0mde7F4+GEBuM= 2022-09-03T07:15:31.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H8d4O7ZpL7SCZY0mde7F4+GEBuM=', 'Content-Length': '0'} 2022-09-03T07:15:31.446 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005885cc9c0e625a7d-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.447 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.448 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.448 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.449 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.450 DEBUG:boto:Method: GET 2022-09-03T07:15:31.450 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.451 DEBUG:boto:Data: 2022-09-03T07:15:31.451 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.452 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.452 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.453 DEBUG:boto:Params: {} 2022-09-03T07:15:31.453 DEBUG:boto:Token: None 2022-09-03T07:15:31.455 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:31.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H8d4O7ZpL7SCZY0mde7F4+GEBuM= 2022-09-03T07:15:31.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H8d4O7ZpL7SCZY0mde7F4+GEBuM=', 'Content-Length': '0'} 2022-09-03T07:15:31.458 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008001fa5303ad23d6-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.459 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.459 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.460 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:31.460 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:31.461 DEBUG:boto:Method: GET 2022-09-03T07:15:31.461 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:31.462 DEBUG:boto:Data: 2022-09-03T07:15:31.463 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.463 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.464 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.465 DEBUG:boto:Params: {} 2022-09-03T07:15:31.466 DEBUG:boto:Token: None 2022-09-03T07:15:31.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.473 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088f4a122452d51d9-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.475 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:31.476 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.477 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.491 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:31.492 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:31.493 DEBUG:boto:Method: GET 2022-09-03T07:15:31.496 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:31.497 DEBUG:boto:Data: 2022-09-03T07:15:31.498 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.500 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.501 DEBUG:boto:Params: {} 2022-09-03T07:15:31.503 DEBUG:boto:Token: None 2022-09-03T07:15:31.508 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.523 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a040f72d11462b93-006312ff13-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.537 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:31.544 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.545 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.547 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.548 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:31.549 DEBUG:boto:Data: 2022-09-03T07:15:31.567 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.568 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.569 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.571 DEBUG:boto:Params: {} 2022-09-03T07:15:31.579 DEBUG:boto:Token: None 2022-09-03T07:15:31.579 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17 2022-09-03T07:15:31.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10TYBv0PxWpRooN0oJ7s0dFRHAY= 2022-09-03T07:15:31.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10TYBv0PxWpRooN0oJ7s0dFRHAY=', 'Content-Length': '0'} 2022-09-03T07:15:31.589 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000718b54f89ec3760a-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.598 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.599 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.600 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.600 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:31.601 DEBUG:boto:Data: 2022-09-03T07:15:31.601 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.602 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.603 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.603 DEBUG:boto:Params: {} 2022-09-03T07:15:31.604 DEBUG:boto:Token: None 2022-09-03T07:15:31.604 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17 2022-09-03T07:15:31.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:10TYBv0PxWpRooN0oJ7s0dFRHAY= 2022-09-03T07:15:31.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:10TYBv0PxWpRooN0oJ7s0dFRHAY=', 'Content-Length': '0'} 2022-09-03T07:15:31.617 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036c0033795270fff-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.626 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:15:31.627 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.628 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.631 DEBUG:boto:Method: GET 2022-09-03T07:15:31.631 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:31.632 DEBUG:boto:Data: 2022-09-03T07:15:31.632 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.633 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.634 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.634 DEBUG:boto:Params: {} 2022-09-03T07:15:31.635 DEBUG:boto:Token: None 2022-09-03T07:15:31.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17 2022-09-03T07:15:31.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rrk3QTm8MnKG2zHyapxlQsMhQCE= 2022-09-03T07:15:31.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rrk3QTm8MnKG2zHyapxlQsMhQCE=', 'Content-Length': '0'} 2022-09-03T07:15:31.641 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d49a18524f060c8-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.644 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:31.653 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:31.654 DEBUG:boto:Method: GET 2022-09-03T07:15:31.656 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:31.657 DEBUG:boto:Data: 2022-09-03T07:15:31.658 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.659 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.664 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.665 DEBUG:boto:Params: {} 2022-09-03T07:15:31.667 DEBUG:boto:Token: None 2022-09-03T07:15:31.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj17 2022-09-03T07:15:31.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rrk3QTm8MnKG2zHyapxlQsMhQCE= 2022-09-03T07:15:31.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rrk3QTm8MnKG2zHyapxlQsMhQCE=', 'Content-Length': '0'} 2022-09-03T07:15:31.674 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc40a5597f506d40-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.676 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.678 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.681 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:31.682 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:31.683 DEBUG:boto:Method: GET 2022-09-03T07:15:31.684 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:31.685 DEBUG:boto:Data: 2022-09-03T07:15:31.686 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.687 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.690 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.691 DEBUG:boto:Params: {} 2022-09-03T07:15:31.694 DEBUG:boto:Token: None 2022-09-03T07:15:31.695 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.696 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.705 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007aec815c0e913eec-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.710 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:31.712 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.713 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.722 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:31.724 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:31.725 DEBUG:boto:Method: GET 2022-09-03T07:15:31.731 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:31.732 DEBUG:boto:Data: 2022-09-03T07:15:31.741 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.741 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.742 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.743 DEBUG:boto:Params: {} 2022-09-03T07:15:31.743 DEBUG:boto:Token: None 2022-09-03T07:15:31.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.749 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004235d1ef2dfd4698-006312ff13-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.750 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:31.751 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:15:31.751 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:31.752 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:31.753 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.753 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.754 DEBUG:boto:Method: GET 2022-09-03T07:15:31.754 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.755 DEBUG:boto:Data: 2022-09-03T07:15:31.755 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.757 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.758 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.759 DEBUG:boto:Params: {} 2022-09-03T07:15:31.761 DEBUG:boto:Token: None 2022-09-03T07:15:31.762 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.763 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N/WuyM5dXpoSA8aaxpLzz5d3/bI= 2022-09-03T07:15:31.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N/WuyM5dXpoSA8aaxpLzz5d3/bI=', 'Content-Length': '0'} 2022-09-03T07:15:31.770 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc0692bdb70072fa-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.772 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:31.772 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:31.773 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.773 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.774 DEBUG:boto:Method: GET 2022-09-03T07:15:31.775 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.776 DEBUG:boto:Data: 2022-09-03T07:15:31.776 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.777 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.778 DEBUG:boto:Params: {} 2022-09-03T07:15:31.783 DEBUG:boto:Token: None 2022-09-03T07:15:31.784 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N/WuyM5dXpoSA8aaxpLzz5d3/bI= 2022-09-03T07:15:31.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N/WuyM5dXpoSA8aaxpLzz5d3/bI=', 'Content-Length': '0'} 2022-09-03T07:15:31.787 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a75ea35bcf014df-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.789 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:31.789 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:31.790 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.790 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.791 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.793 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.794 DEBUG:boto:Data: 2022-09-03T07:15:31.794 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.795 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.796 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.802 DEBUG:boto:Params: {} 2022-09-03T07:15:31.803 DEBUG:boto:Token: None 2022-09-03T07:15:31.804 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.804 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v+J8Z8eEQy25BdkYQpvDyOjMeXI= 2022-09-03T07:15:31.805 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v+J8Z8eEQy25BdkYQpvDyOjMeXI=', 'Content-Length': '0'} 2022-09-03T07:15:31.807 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b8c0f91c274a485-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.809 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:31.810 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:31.811 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.812 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.813 DEBUG:boto:Method: HEAD 2022-09-03T07:15:31.814 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.815 DEBUG:boto:Data: 2022-09-03T07:15:31.816 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.819 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.820 DEBUG:boto:Params: {} 2022-09-03T07:15:31.821 DEBUG:boto:Token: None 2022-09-03T07:15:31.824 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v+J8Z8eEQy25BdkYQpvDyOjMeXI= 2022-09-03T07:15:31.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v+J8Z8eEQy25BdkYQpvDyOjMeXI=', 'Content-Length': '0'} 2022-09-03T07:15:31.829 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d3ac5fe8650e3e2-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.830 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:15:31.831 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:31.832 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:31.834 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.835 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.836 DEBUG:boto:Method: GET 2022-09-03T07:15:31.837 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.839 DEBUG:boto:Data: 2022-09-03T07:15:31.844 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.853 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.853 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.854 DEBUG:boto:Params: {} 2022-09-03T07:15:31.855 DEBUG:boto:Token: None 2022-09-03T07:15:31.857 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N/WuyM5dXpoSA8aaxpLzz5d3/bI= 2022-09-03T07:15:31.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N/WuyM5dXpoSA8aaxpLzz5d3/bI=', 'Content-Length': '0'} 2022-09-03T07:15:31.862 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038c6e3742fe31ab6-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.864 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:31.865 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:31.867 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.868 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.872 DEBUG:boto:Method: GET 2022-09-03T07:15:31.873 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.874 DEBUG:boto:Data: 2022-09-03T07:15:31.877 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.885 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.900 DEBUG:boto:Params: {} 2022-09-03T07:15:31.903 DEBUG:boto:Token: None 2022-09-03T07:15:31.905 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:31.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N/WuyM5dXpoSA8aaxpLzz5d3/bI= 2022-09-03T07:15:31.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N/WuyM5dXpoSA8aaxpLzz5d3/bI=', 'Content-Length': '0'} 2022-09-03T07:15:31.922 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b263f4bdb724bd18-006312ff13-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.926 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.928 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.929 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:31.931 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:31.943 DEBUG:boto:Method: GET 2022-09-03T07:15:31.943 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:31.944 DEBUG:boto:Data: 2022-09-03T07:15:31.946 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.946 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:31.947 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.948 DEBUG:boto:Params: {} 2022-09-03T07:15:31.948 DEBUG:boto:Token: None 2022-09-03T07:15:31.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.953 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020e5be0e12aabf16-006312ff13-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.954 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:31.956 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:31.960 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:31.961 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:31.963 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:31.964 DEBUG:boto:Method: GET 2022-09-03T07:15:31.965 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:31.966 DEBUG:boto:Data: 2022-09-03T07:15:31.967 DEBUG:boto:Headers: {} 2022-09-03T07:15:31.970 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:31.971 DEBUG:boto:Port: 8000 2022-09-03T07:15:31.972 DEBUG:boto:Params: {} 2022-09-03T07:15:31.973 DEBUG:boto:Token: None 2022-09-03T07:15:31.974 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:31 GMT /avysmw-75/ 2022-09-03T07:15:31.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg= 2022-09-03T07:15:31.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:31 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wo6AZu/ETNPb1xpXd2r6DskvjVg=', 'Content-Length': '0'} 2022-09-03T07:15:31.994 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005cb761f9a426a5f7-006312ff13-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:31 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:31.996 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.021 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:32.032 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:32.033 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.034 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:32.035 DEBUG:boto:Data: 2022-09-03T07:15:32.039 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.041 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.042 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.045 DEBUG:boto:Params: {} 2022-09-03T07:15:32.048 DEBUG:boto:Token: None 2022-09-03T07:15:32.049 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj22 2022-09-03T07:15:32.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Ce2QV623R1edoZjlCYG5m4mnfk= 2022-09-03T07:15:32.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Ce2QV623R1edoZjlCYG5m4mnfk=', 'Content-Length': '0'} 2022-09-03T07:15:32.054 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8a28518c919b2ed-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.056 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:32.057 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:32.058 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.059 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:32.060 DEBUG:boto:Data: 2022-09-03T07:15:32.068 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.069 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.069 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.076 DEBUG:boto:Params: {} 2022-09-03T07:15:32.080 DEBUG:boto:Token: None 2022-09-03T07:15:32.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj22 2022-09-03T07:15:32.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Ce2QV623R1edoZjlCYG5m4mnfk= 2022-09-03T07:15:32.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Ce2QV623R1edoZjlCYG5m4mnfk=', 'Content-Length': '0'} 2022-09-03T07:15:32.093 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b4bea549cdb4729-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.094 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:15:32.094 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:32.095 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:32.097 DEBUG:boto:Method: GET 2022-09-03T07:15:32.098 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:32.099 DEBUG:boto:Data: 2022-09-03T07:15:32.102 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.103 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.104 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.105 DEBUG:boto:Params: {} 2022-09-03T07:15:32.106 DEBUG:boto:Token: None 2022-09-03T07:15:32.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj22 2022-09-03T07:15:32.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N43zm5Htib9/vIcqEQ4dwdOgJLU= 2022-09-03T07:15:32.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N43zm5Htib9/vIcqEQ4dwdOgJLU=', 'Content-Length': '0'} 2022-09-03T07:15:32.113 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b926248b09bc7c48-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.114 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:32.114 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:32.115 DEBUG:boto:Method: GET 2022-09-03T07:15:32.115 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:32.116 DEBUG:boto:Data: 2022-09-03T07:15:32.117 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.118 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.119 DEBUG:boto:Params: {} 2022-09-03T07:15:32.119 DEBUG:boto:Token: None 2022-09-03T07:15:32.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj22 2022-09-03T07:15:32.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N43zm5Htib9/vIcqEQ4dwdOgJLU= 2022-09-03T07:15:32.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N43zm5Htib9/vIcqEQ4dwdOgJLU=', 'Content-Length': '0'} 2022-09-03T07:15:32.123 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e0012de05ad49aa7-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.124 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.125 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.125 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:32.126 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:32.127 DEBUG:boto:Method: GET 2022-09-03T07:15:32.127 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:32.128 DEBUG:boto:Data: 2022-09-03T07:15:32.128 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.129 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.130 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.130 DEBUG:boto:Params: {} 2022-09-03T07:15:32.131 DEBUG:boto:Token: None 2022-09-03T07:15:32.132 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.144 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000033eaae1618ec4ce8-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.146 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.147 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.149 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.150 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:32.151 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:32.152 DEBUG:boto:Method: GET 2022-09-03T07:15:32.153 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:32.154 DEBUG:boto:Data: 2022-09-03T07:15:32.156 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.157 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.158 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.170 DEBUG:boto:Params: {} 2022-09-03T07:15:32.171 DEBUG:boto:Token: None 2022-09-03T07:15:32.176 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.182 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a6bf1707fc10ad4-006312ff14-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.183 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.184 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:15:32.185 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.186 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.186 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.187 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.187 DEBUG:boto:Method: GET 2022-09-03T07:15:32.188 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.189 DEBUG:boto:Data: 2022-09-03T07:15:32.190 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.191 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.191 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.192 DEBUG:boto:Params: {} 2022-09-03T07:15:32.198 DEBUG:boto:Token: None 2022-09-03T07:15:32.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7i/5L09VGpDXVy/iT0gmR9+e22g= 2022-09-03T07:15:32.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7i/5L09VGpDXVy/iT0gmR9+e22g=', 'Content-Length': '0'} 2022-09-03T07:15:32.205 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000082e807ca1310f035-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.206 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.207 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.207 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.208 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.209 DEBUG:boto:Method: GET 2022-09-03T07:15:32.209 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.210 DEBUG:boto:Data: 2022-09-03T07:15:32.211 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.211 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.212 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.213 DEBUG:boto:Params: {} 2022-09-03T07:15:32.213 DEBUG:boto:Token: None 2022-09-03T07:15:32.214 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7i/5L09VGpDXVy/iT0gmR9+e22g= 2022-09-03T07:15:32.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7i/5L09VGpDXVy/iT0gmR9+e22g=', 'Content-Length': '0'} 2022-09-03T07:15:32.217 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000743d2b52e6dbbe77-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.223 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.223 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.224 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.224 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.225 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.225 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.231 DEBUG:boto:Data: 2022-09-03T07:15:32.231 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.232 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.232 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.233 DEBUG:boto:Params: {} 2022-09-03T07:15:32.234 DEBUG:boto:Token: None 2022-09-03T07:15:32.235 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.235 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tw4H1i/Cq6ZFnCBedIc3Gf/Ot0c= 2022-09-03T07:15:32.236 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tw4H1i/Cq6ZFnCBedIc3Gf/Ot0c=', 'Content-Length': '0'} 2022-09-03T07:15:32.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bfc139404e296006-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.241 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.242 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.242 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.243 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.243 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.244 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.244 DEBUG:boto:Data: 2022-09-03T07:15:32.246 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.246 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.247 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.247 DEBUG:boto:Params: {} 2022-09-03T07:15:32.248 DEBUG:boto:Token: None 2022-09-03T07:15:32.249 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tw4H1i/Cq6ZFnCBedIc3Gf/Ot0c= 2022-09-03T07:15:32.250 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tw4H1i/Cq6ZFnCBedIc3Gf/Ot0c=', 'Content-Length': '0'} 2022-09-03T07:15:32.252 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a0f8fa126ff23181-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.254 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:15:32.254 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.255 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.256 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.256 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.257 DEBUG:boto:Method: GET 2022-09-03T07:15:32.257 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.258 DEBUG:boto:Data: 2022-09-03T07:15:32.259 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.259 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.260 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.260 DEBUG:boto:Params: {} 2022-09-03T07:15:32.262 DEBUG:boto:Token: None 2022-09-03T07:15:32.263 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7i/5L09VGpDXVy/iT0gmR9+e22g= 2022-09-03T07:15:32.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7i/5L09VGpDXVy/iT0gmR9+e22g=', 'Content-Length': '0'} 2022-09-03T07:15:32.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066748619be4f8134-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.272 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.272 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.273 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.273 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.274 DEBUG:boto:Method: GET 2022-09-03T07:15:32.276 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.276 DEBUG:boto:Data: 2022-09-03T07:15:32.277 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.279 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.280 DEBUG:boto:Params: {} 2022-09-03T07:15:32.281 DEBUG:boto:Token: None 2022-09-03T07:15:32.282 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:32.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7i/5L09VGpDXVy/iT0gmR9+e22g= 2022-09-03T07:15:32.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7i/5L09VGpDXVy/iT0gmR9+e22g=', 'Content-Length': '0'} 2022-09-03T07:15:32.287 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da66a24a26ba798c-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.289 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.290 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.291 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:32.292 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:32.293 DEBUG:boto:Method: GET 2022-09-03T07:15:32.294 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:32.294 DEBUG:boto:Data: 2022-09-03T07:15:32.295 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.296 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.298 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.299 DEBUG:boto:Params: {} 2022-09-03T07:15:32.300 DEBUG:boto:Token: None 2022-09-03T07:15:32.300 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.301 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.306 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037f7049e94c52e04-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.350 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.351 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.352 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.353 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:32.354 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:32.354 DEBUG:boto:Method: GET 2022-09-03T07:15:32.355 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:32.355 DEBUG:boto:Data: 2022-09-03T07:15:32.356 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.364 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.365 DEBUG:boto:Params: {} 2022-09-03T07:15:32.365 DEBUG:boto:Token: None 2022-09-03T07:15:32.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.372 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000352a1163e50fecbc-006312ff14-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.375 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.377 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.377 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.378 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.379 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:32.380 DEBUG:boto:Data: 2022-09-03T07:15:32.381 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.382 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.385 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.386 DEBUG:boto:Params: {} 2022-09-03T07:15:32.388 DEBUG:boto:Token: None 2022-09-03T07:15:32.395 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33 2022-09-03T07:15:32.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nRtKtihhLYPNphHHXmszoYhvUNs= 2022-09-03T07:15:32.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nRtKtihhLYPNphHHXmszoYhvUNs=', 'Content-Length': '0'} 2022-09-03T07:15:32.401 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066b52ae665540bb9-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.403 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.404 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.405 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.406 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:32.420 DEBUG:boto:Data: 2022-09-03T07:15:32.420 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.421 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.422 DEBUG:boto:Params: {} 2022-09-03T07:15:32.428 DEBUG:boto:Token: None 2022-09-03T07:15:32.430 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33 2022-09-03T07:15:32.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nRtKtihhLYPNphHHXmszoYhvUNs= 2022-09-03T07:15:32.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nRtKtihhLYPNphHHXmszoYhvUNs=', 'Content-Length': '0'} 2022-09-03T07:15:32.436 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e54d9d9d00239d6-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.437 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:15:32.437 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.438 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.439 DEBUG:boto:Method: GET 2022-09-03T07:15:32.439 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:32.440 DEBUG:boto:Data: 2022-09-03T07:15:32.440 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.441 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.442 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.442 DEBUG:boto:Params: {} 2022-09-03T07:15:32.443 DEBUG:boto:Token: None 2022-09-03T07:15:32.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33 2022-09-03T07:15:32.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYfctSiF9pMWDIE7Jmjoj/Kl9nc= 2022-09-03T07:15:32.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYfctSiF9pMWDIE7Jmjoj/Kl9nc=', 'Content-Length': '0'} 2022-09-03T07:15:32.448 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000149226973bff050b-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.450 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:32.450 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:32.451 DEBUG:boto:Method: GET 2022-09-03T07:15:32.452 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:32.452 DEBUG:boto:Data: 2022-09-03T07:15:32.453 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.454 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.454 DEBUG:boto:Params: {} 2022-09-03T07:15:32.455 DEBUG:boto:Token: None 2022-09-03T07:15:32.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj33 2022-09-03T07:15:32.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYfctSiF9pMWDIE7Jmjoj/Kl9nc= 2022-09-03T07:15:32.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYfctSiF9pMWDIE7Jmjoj/Kl9nc=', 'Content-Length': '0'} 2022-09-03T07:15:32.462 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc04acb1ddbc6336-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.464 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.465 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.466 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:32.467 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:32.468 DEBUG:boto:Method: GET 2022-09-03T07:15:32.469 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:32.469 DEBUG:boto:Data: 2022-09-03T07:15:32.470 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.471 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.472 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.475 DEBUG:boto:Params: {} 2022-09-03T07:15:32.476 DEBUG:boto:Token: None 2022-09-03T07:15:32.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.493 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e7231c7dc7be33ff-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.496 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.497 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.498 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.499 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:32.500 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:32.501 DEBUG:boto:Method: GET 2022-09-03T07:15:32.504 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:32.505 DEBUG:boto:Data: 2022-09-03T07:15:32.506 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.507 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.508 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.509 DEBUG:boto:Params: {} 2022-09-03T07:15:32.510 DEBUG:boto:Token: None 2022-09-03T07:15:32.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.520 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000bc0edee1f85de80-006312ff14-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.529 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.531 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:15:32.532 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.533 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.533 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.534 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.535 DEBUG:boto:Method: GET 2022-09-03T07:15:32.535 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.536 DEBUG:boto:Data: 2022-09-03T07:15:32.536 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.537 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.538 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.539 DEBUG:boto:Params: {} 2022-09-03T07:15:32.539 DEBUG:boto:Token: None 2022-09-03T07:15:32.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GlfzSfLruKSVyHDF/phk4eHw2HA= 2022-09-03T07:15:32.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GlfzSfLruKSVyHDF/phk4eHw2HA=', 'Content-Length': '0'} 2022-09-03T07:15:32.552 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007cdbd35d94b5fb7d-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.553 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.554 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.554 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.555 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.555 DEBUG:boto:Method: GET 2022-09-03T07:15:32.556 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.556 DEBUG:boto:Data: 2022-09-03T07:15:32.557 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.558 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.559 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.560 DEBUG:boto:Params: {} 2022-09-03T07:15:32.560 DEBUG:boto:Token: None 2022-09-03T07:15:32.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GlfzSfLruKSVyHDF/phk4eHw2HA= 2022-09-03T07:15:32.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GlfzSfLruKSVyHDF/phk4eHw2HA=', 'Content-Length': '0'} 2022-09-03T07:15:32.573 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c07dac04f35d5cbb-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.575 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.575 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.576 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.576 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.577 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.577 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.586 DEBUG:boto:Data: 2022-09-03T07:15:32.594 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.596 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.598 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.599 DEBUG:boto:Params: {} 2022-09-03T07:15:32.600 DEBUG:boto:Token: None 2022-09-03T07:15:32.601 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WIcoZ0PkRZlbdMB0C/r/XnYo+Fk= 2022-09-03T07:15:32.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WIcoZ0PkRZlbdMB0C/r/XnYo+Fk=', 'Content-Length': '0'} 2022-09-03T07:15:32.610 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e849411aa1a7158-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.616 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.617 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.618 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.619 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.620 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.621 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.621 DEBUG:boto:Data: 2022-09-03T07:15:32.623 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.626 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.628 DEBUG:boto:Params: {} 2022-09-03T07:15:32.629 DEBUG:boto:Token: None 2022-09-03T07:15:32.630 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WIcoZ0PkRZlbdMB0C/r/XnYo+Fk= 2022-09-03T07:15:32.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WIcoZ0PkRZlbdMB0C/r/XnYo+Fk=', 'Content-Length': '0'} 2022-09-03T07:15:32.645 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009843710d065abc7b-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.647 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:15:32.649 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.651 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.652 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.653 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.655 DEBUG:boto:Method: GET 2022-09-03T07:15:32.656 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.658 DEBUG:boto:Data: 2022-09-03T07:15:32.659 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.660 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.661 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.662 DEBUG:boto:Params: {} 2022-09-03T07:15:32.663 DEBUG:boto:Token: None 2022-09-03T07:15:32.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GlfzSfLruKSVyHDF/phk4eHw2HA= 2022-09-03T07:15:32.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GlfzSfLruKSVyHDF/phk4eHw2HA=', 'Content-Length': '0'} 2022-09-03T07:15:32.674 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071caeb993b3963d6-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.676 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.678 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.679 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.680 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.681 DEBUG:boto:Method: GET 2022-09-03T07:15:32.683 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.684 DEBUG:boto:Data: 2022-09-03T07:15:32.686 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.690 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.691 DEBUG:boto:Params: {} 2022-09-03T07:15:32.694 DEBUG:boto:Token: None 2022-09-03T07:15:32.695 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:32.696 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GlfzSfLruKSVyHDF/phk4eHw2HA= 2022-09-03T07:15:32.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GlfzSfLruKSVyHDF/phk4eHw2HA=', 'Content-Length': '0'} 2022-09-03T07:15:32.702 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f8db4a0e266649e-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.705 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.705 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.707 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:32.707 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:32.708 DEBUG:boto:Method: GET 2022-09-03T07:15:32.709 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:32.710 DEBUG:boto:Data: 2022-09-03T07:15:32.711 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.712 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.713 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.714 DEBUG:boto:Params: {} 2022-09-03T07:15:32.715 DEBUG:boto:Token: None 2022-09-03T07:15:32.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.728 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000019abb4ce49284b2c-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.729 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.730 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.731 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.731 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:32.732 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:32.733 DEBUG:boto:Method: GET 2022-09-03T07:15:32.735 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:32.735 DEBUG:boto:Data: 2022-09-03T07:15:32.736 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.737 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.739 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.741 DEBUG:boto:Params: {} 2022-09-03T07:15:32.742 DEBUG:boto:Token: None 2022-09-03T07:15:32.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.752 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000f16140c28a44097-006312ff14-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.753 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.756 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.757 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.758 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.759 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:32.760 DEBUG:boto:Data: 2022-09-03T07:15:32.761 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.763 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.764 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.765 DEBUG:boto:Params: {} 2022-09-03T07:15:32.766 DEBUG:boto:Token: None 2022-09-03T07:15:32.768 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34 2022-09-03T07:15:32.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QDmyshADH291AY55Zrh820Ig0i4= 2022-09-03T07:15:32.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QDmyshADH291AY55Zrh820Ig0i4=', 'Content-Length': '0'} 2022-09-03T07:15:32.780 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000511a1eb9cf47698d-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.781 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.783 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.784 DEBUG:boto:Method: HEAD 2022-09-03T07:15:32.785 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:32.787 DEBUG:boto:Data: 2022-09-03T07:15:32.787 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.788 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.791 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.793 DEBUG:boto:Params: {} 2022-09-03T07:15:32.795 DEBUG:boto:Token: None 2022-09-03T07:15:32.797 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34 2022-09-03T07:15:32.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QDmyshADH291AY55Zrh820Ig0i4= 2022-09-03T07:15:32.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QDmyshADH291AY55Zrh820Ig0i4=', 'Content-Length': '0'} 2022-09-03T07:15:32.804 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056c8c3f9d252f3e1-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.806 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:15:32.807 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.808 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.809 DEBUG:boto:Method: GET 2022-09-03T07:15:32.810 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:32.811 DEBUG:boto:Data: 2022-09-03T07:15:32.813 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.815 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.816 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.817 DEBUG:boto:Params: {} 2022-09-03T07:15:32.822 DEBUG:boto:Token: None 2022-09-03T07:15:32.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34 2022-09-03T07:15:32.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DjFkMBQjXaZmXYxCfWP1MP5ZBNo= 2022-09-03T07:15:32.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DjFkMBQjXaZmXYxCfWP1MP5ZBNo=', 'Content-Length': '0'} 2022-09-03T07:15:32.837 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dfd5f4522fe24d6f-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.851 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:32.852 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:32.853 DEBUG:boto:Method: GET 2022-09-03T07:15:32.855 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:32.856 DEBUG:boto:Data: 2022-09-03T07:15:32.856 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.858 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.859 DEBUG:boto:Params: {} 2022-09-03T07:15:32.861 DEBUG:boto:Token: None 2022-09-03T07:15:32.862 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj34 2022-09-03T07:15:32.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DjFkMBQjXaZmXYxCfWP1MP5ZBNo= 2022-09-03T07:15:32.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DjFkMBQjXaZmXYxCfWP1MP5ZBNo=', 'Content-Length': '0'} 2022-09-03T07:15:32.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fe002d71836e434-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.876 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.877 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.879 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:32.880 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:32.881 DEBUG:boto:Method: GET 2022-09-03T07:15:32.881 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:32.883 DEBUG:boto:Data: 2022-09-03T07:15:32.884 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.885 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.886 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.887 DEBUG:boto:Params: {} 2022-09-03T07:15:32.888 DEBUG:boto:Token: None 2022-09-03T07:15:32.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.911 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1bf89d3a4e87945-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.913 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.927 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:32.928 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:32.929 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:32.939 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:32.940 DEBUG:boto:Method: GET 2022-09-03T07:15:32.941 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:32.942 DEBUG:boto:Data: 2022-09-03T07:15:32.943 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.945 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.946 DEBUG:boto:Params: {} 2022-09-03T07:15:32.947 DEBUG:boto:Token: None 2022-09-03T07:15:32.948 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/ 2022-09-03T07:15:32.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM= 2022-09-03T07:15:32.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZT/utjsPu+ls1Sr71ZtADADZjM=', 'Content-Length': '0'} 2022-09-03T07:15:32.959 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007323cbda19ab395c-006312ff14-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.960 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:32.961 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:15:32.962 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:32.963 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:32.964 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:32.964 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:32.965 DEBUG:boto:Method: GET 2022-09-03T07:15:32.966 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:32.966 DEBUG:boto:Data: 2022-09-03T07:15:32.967 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.968 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:32.971 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.972 DEBUG:boto:Params: {} 2022-09-03T07:15:32.973 DEBUG:boto:Token: None 2022-09-03T07:15:32.974 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:32.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WnlruuZIRZqn+iIXPfjwVmkHN5s= 2022-09-03T07:15:32.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WnlruuZIRZqn+iIXPfjwVmkHN5s=', 'Content-Length': '0'} 2022-09-03T07:15:32.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002cdc46bbafdf1602-006312ff14-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:32.984 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:32.985 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:32.985 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:32.986 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:32.987 DEBUG:boto:Method: GET 2022-09-03T07:15:32.987 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:32.988 DEBUG:boto:Data: 2022-09-03T07:15:32.988 DEBUG:boto:Headers: {} 2022-09-03T07:15:32.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:32.989 DEBUG:boto:Port: 8000 2022-09-03T07:15:32.990 DEBUG:boto:Params: {} 2022-09-03T07:15:32.995 DEBUG:boto:Token: None 2022-09-03T07:15:32.997 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:32 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:32.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WnlruuZIRZqn+iIXPfjwVmkHN5s= 2022-09-03T07:15:32.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WnlruuZIRZqn+iIXPfjwVmkHN5s=', 'Content-Length': '0'} 2022-09-03T07:15:33.022 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a442332d5b75d76-006312ff14-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:32 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.023 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:33.023 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:33.024 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.025 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.025 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.026 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.026 DEBUG:boto:Data: 2022-09-03T07:15:33.027 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.027 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.028 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.028 DEBUG:boto:Params: {} 2022-09-03T07:15:33.029 DEBUG:boto:Token: None 2022-09-03T07:15:33.030 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XSRW314aGy2+C4CpQ9imbkr6AI8= 2022-09-03T07:15:33.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XSRW314aGy2+C4CpQ9imbkr6AI8=', 'Content-Length': '0'} 2022-09-03T07:15:33.033 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000158140445b3ff902-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.034 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:33.035 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:33.035 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.036 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.036 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.037 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.037 DEBUG:boto:Data: 2022-09-03T07:15:33.038 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.045 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.046 DEBUG:boto:Params: {} 2022-09-03T07:15:33.047 DEBUG:boto:Token: None 2022-09-03T07:15:33.047 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XSRW314aGy2+C4CpQ9imbkr6AI8= 2022-09-03T07:15:33.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XSRW314aGy2+C4CpQ9imbkr6AI8=', 'Content-Length': '0'} 2022-09-03T07:15:33.051 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a099c6b39d23fc4f-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.052 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:15:33.052 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:33.054 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:33.055 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.055 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.056 DEBUG:boto:Method: GET 2022-09-03T07:15:33.056 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.057 DEBUG:boto:Data: 2022-09-03T07:15:33.058 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.058 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.059 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.059 DEBUG:boto:Params: {} 2022-09-03T07:15:33.060 DEBUG:boto:Token: None 2022-09-03T07:15:33.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TBOy1jkRxx5PohgXOgT2SgXmhPg= 2022-09-03T07:15:33.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TBOy1jkRxx5PohgXOgT2SgXmhPg=', 'Content-Length': '0'} 2022-09-03T07:15:33.081 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d4d4c9146885f54-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.083 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:33.084 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:33.086 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.086 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.087 DEBUG:boto:Method: GET 2022-09-03T07:15:33.087 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.088 DEBUG:boto:Data: 2022-09-03T07:15:33.088 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.089 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.089 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.090 DEBUG:boto:Params: {} 2022-09-03T07:15:33.091 DEBUG:boto:Token: None 2022-09-03T07:15:33.092 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:33.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TBOy1jkRxx5PohgXOgT2SgXmhPg= 2022-09-03T07:15:33.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TBOy1jkRxx5PohgXOgT2SgXmhPg=', 'Content-Length': '0'} 2022-09-03T07:15:33.096 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003f607c78b199be1-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.097 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.097 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.098 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:33.098 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:33.099 DEBUG:boto:Method: GET 2022-09-03T07:15:33.100 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:33.100 DEBUG:boto:Data: 2022-09-03T07:15:33.101 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.101 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.102 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.102 DEBUG:boto:Params: {} 2022-09-03T07:15:33.103 DEBUG:boto:Token: None 2022-09-03T07:15:33.103 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.111 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028c7adeef7ba4136-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.112 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.113 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.113 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.114 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:33.115 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:33.116 DEBUG:boto:Method: GET 2022-09-03T07:15:33.117 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:33.118 DEBUG:boto:Data: 2022-09-03T07:15:33.119 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.120 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.122 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.123 DEBUG:boto:Params: {} 2022-09-03T07:15:33.124 DEBUG:boto:Token: None 2022-09-03T07:15:33.124 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007065cf2d1673a58a-006312ff15-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.135 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.136 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:33.136 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:33.137 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.137 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:33.138 DEBUG:boto:Data: 2022-09-03T07:15:33.138 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.139 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.139 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.140 DEBUG:boto:Params: {} 2022-09-03T07:15:33.141 DEBUG:boto:Token: None 2022-09-03T07:15:33.141 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj44 2022-09-03T07:15:33.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bf9lGAGFNN5ypxXS0lO0LiuQdhk= 2022-09-03T07:15:33.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bf9lGAGFNN5ypxXS0lO0LiuQdhk=', 'Content-Length': '0'} 2022-09-03T07:15:33.148 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035cec87ede5fac33-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.150 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:33.151 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:33.152 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.153 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:33.153 DEBUG:boto:Data: 2022-09-03T07:15:33.154 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.156 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.158 DEBUG:boto:Params: {} 2022-09-03T07:15:33.158 DEBUG:boto:Token: None 2022-09-03T07:15:33.159 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj44 2022-09-03T07:15:33.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bf9lGAGFNN5ypxXS0lO0LiuQdhk= 2022-09-03T07:15:33.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bf9lGAGFNN5ypxXS0lO0LiuQdhk=', 'Content-Length': '0'} 2022-09-03T07:15:33.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000744581b1ad771150-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.165 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:15:33.167 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:33.168 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:33.172 DEBUG:boto:Method: GET 2022-09-03T07:15:33.173 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:33.177 DEBUG:boto:Data: 2022-09-03T07:15:33.184 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.185 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.186 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.188 DEBUG:boto:Params: {} 2022-09-03T07:15:33.189 DEBUG:boto:Token: None 2022-09-03T07:15:33.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj44 2022-09-03T07:15:33.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1XDmiL/tWirHNokh7A61ysJH+a4= 2022-09-03T07:15:33.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1XDmiL/tWirHNokh7A61ysJH+a4=', 'Content-Length': '0'} 2022-09-03T07:15:33.195 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d928dcfb2ee80270-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.196 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:33.197 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:33.197 DEBUG:boto:Method: GET 2022-09-03T07:15:33.199 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:33.200 DEBUG:boto:Data: 2022-09-03T07:15:33.200 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.201 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.202 DEBUG:boto:Params: {} 2022-09-03T07:15:33.202 DEBUG:boto:Token: None 2022-09-03T07:15:33.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj44 2022-09-03T07:15:33.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1XDmiL/tWirHNokh7A61ysJH+a4= 2022-09-03T07:15:33.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1XDmiL/tWirHNokh7A61ysJH+a4=', 'Content-Length': '0'} 2022-09-03T07:15:33.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000104dbf187f4318f4-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.208 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.209 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.210 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:33.211 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:33.212 DEBUG:boto:Method: GET 2022-09-03T07:15:33.212 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:33.213 DEBUG:boto:Data: 2022-09-03T07:15:33.213 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.214 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.214 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.215 DEBUG:boto:Params: {} 2022-09-03T07:15:33.215 DEBUG:boto:Token: None 2022-09-03T07:15:33.216 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.221 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004539ff1b304fdc68-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.262 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.264 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.265 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.265 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:33.267 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:33.268 DEBUG:boto:Method: GET 2022-09-03T07:15:33.268 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:33.269 DEBUG:boto:Data: 2022-09-03T07:15:33.271 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.272 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.273 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.274 DEBUG:boto:Params: {} 2022-09-03T07:15:33.276 DEBUG:boto:Token: None 2022-09-03T07:15:33.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.283 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ef313546d2808302-006312ff15-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.284 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.286 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:15:33.287 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.288 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.289 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.290 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.291 DEBUG:boto:Method: GET 2022-09-03T07:15:33.292 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.294 DEBUG:boto:Data: 2022-09-03T07:15:33.295 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.296 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.297 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.298 DEBUG:boto:Params: {} 2022-09-03T07:15:33.299 DEBUG:boto:Token: None 2022-09-03T07:15:33.300 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZM7OjeTliicppGNM8BuoKj93ZOY= 2022-09-03T07:15:33.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZM7OjeTliicppGNM8BuoKj93ZOY=', 'Content-Length': '0'} 2022-09-03T07:15:33.306 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009940588839ebe6e-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.308 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.309 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.310 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.311 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.312 DEBUG:boto:Method: GET 2022-09-03T07:15:33.313 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.314 DEBUG:boto:Data: 2022-09-03T07:15:33.316 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.317 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.318 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.319 DEBUG:boto:Params: {} 2022-09-03T07:15:33.320 DEBUG:boto:Token: None 2022-09-03T07:15:33.321 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZM7OjeTliicppGNM8BuoKj93ZOY= 2022-09-03T07:15:33.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZM7OjeTliicppGNM8BuoKj93ZOY=', 'Content-Length': '0'} 2022-09-03T07:15:33.328 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004316c7930cb734e7-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.370 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.371 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.371 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.372 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.372 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.373 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.374 DEBUG:boto:Data: 2022-09-03T07:15:33.375 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.376 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.377 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.379 DEBUG:boto:Params: {} 2022-09-03T07:15:33.380 DEBUG:boto:Token: None 2022-09-03T07:15:33.380 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O53o3EUtiBdta1ovoCLEG4oXCrA= 2022-09-03T07:15:33.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O53o3EUtiBdta1ovoCLEG4oXCrA=', 'Content-Length': '0'} 2022-09-03T07:15:33.386 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000077e7078261a1b674-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.389 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.390 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.391 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.392 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.392 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.393 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.394 DEBUG:boto:Data: 2022-09-03T07:15:33.395 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.396 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.397 DEBUG:boto:Params: {} 2022-09-03T07:15:33.398 DEBUG:boto:Token: None 2022-09-03T07:15:33.399 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O53o3EUtiBdta1ovoCLEG4oXCrA= 2022-09-03T07:15:33.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O53o3EUtiBdta1ovoCLEG4oXCrA=', 'Content-Length': '0'} 2022-09-03T07:15:33.402 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab8e9f6d92d1db50-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.404 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:15:33.405 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.407 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.408 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.417 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.419 DEBUG:boto:Method: GET 2022-09-03T07:15:33.420 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.421 DEBUG:boto:Data: 2022-09-03T07:15:33.422 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.423 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.424 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.426 DEBUG:boto:Params: {} 2022-09-03T07:15:33.427 DEBUG:boto:Token: None 2022-09-03T07:15:33.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZM7OjeTliicppGNM8BuoKj93ZOY= 2022-09-03T07:15:33.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZM7OjeTliicppGNM8BuoKj93ZOY=', 'Content-Length': '0'} 2022-09-03T07:15:33.432 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d16e3380ee394d99-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.433 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.434 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.435 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.436 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.436 DEBUG:boto:Method: GET 2022-09-03T07:15:33.437 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.437 DEBUG:boto:Data: 2022-09-03T07:15:33.438 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.439 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.440 DEBUG:boto:Params: {} 2022-09-03T07:15:33.440 DEBUG:boto:Token: None 2022-09-03T07:15:33.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:33.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZM7OjeTliicppGNM8BuoKj93ZOY= 2022-09-03T07:15:33.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZM7OjeTliicppGNM8BuoKj93ZOY=', 'Content-Length': '0'} 2022-09-03T07:15:33.445 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079c176a331a3bae5-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.446 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.447 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.447 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:33.448 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:33.448 DEBUG:boto:Method: GET 2022-09-03T07:15:33.449 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:33.450 DEBUG:boto:Data: 2022-09-03T07:15:33.450 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.451 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.451 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.452 DEBUG:boto:Params: {} 2022-09-03T07:15:33.453 DEBUG:boto:Token: None 2022-09-03T07:15:33.455 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.463 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c668eea58c770858-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.464 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.465 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.467 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.467 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:33.468 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:33.469 DEBUG:boto:Method: GET 2022-09-03T07:15:33.470 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:33.471 DEBUG:boto:Data: 2022-09-03T07:15:33.472 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.475 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.475 DEBUG:boto:Params: {} 2022-09-03T07:15:33.476 DEBUG:boto:Token: None 2022-09-03T07:15:33.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.485 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f625730dd0b73aaa-006312ff15-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.486 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.489 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.490 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.491 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.492 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:33.493 DEBUG:boto:Data: 2022-09-03T07:15:33.495 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.496 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.497 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.498 DEBUG:boto:Params: {} 2022-09-03T07:15:33.499 DEBUG:boto:Token: None 2022-09-03T07:15:33.500 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51 2022-09-03T07:15:33.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:re7E0P6GrbH/5GVdnSYp6kOJYlI= 2022-09-03T07:15:33.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:re7E0P6GrbH/5GVdnSYp6kOJYlI=', 'Content-Length': '0'} 2022-09-03T07:15:33.508 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000021334a6ad7e8a7a2-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.509 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.509 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.510 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.510 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:33.511 DEBUG:boto:Data: 2022-09-03T07:15:33.511 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.513 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.514 DEBUG:boto:Params: {} 2022-09-03T07:15:33.515 DEBUG:boto:Token: None 2022-09-03T07:15:33.515 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51 2022-09-03T07:15:33.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:re7E0P6GrbH/5GVdnSYp6kOJYlI= 2022-09-03T07:15:33.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:re7E0P6GrbH/5GVdnSYp6kOJYlI=', 'Content-Length': '0'} 2022-09-03T07:15:33.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090cf0410caee3ffe-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.519 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:15:33.520 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.522 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.522 DEBUG:boto:Method: GET 2022-09-03T07:15:33.524 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:33.525 DEBUG:boto:Data: 2022-09-03T07:15:33.525 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.526 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.527 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.527 DEBUG:boto:Params: {} 2022-09-03T07:15:33.528 DEBUG:boto:Token: None 2022-09-03T07:15:33.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51 2022-09-03T07:15:33.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H8DQsn4hHO+vVjqbaMA7SGTGDQo= 2022-09-03T07:15:33.532 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H8DQsn4hHO+vVjqbaMA7SGTGDQo=', 'Content-Length': '0'} 2022-09-03T07:15:33.546 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053bc725da3004403-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.554 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:33.555 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:33.556 DEBUG:boto:Method: GET 2022-09-03T07:15:33.557 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:33.557 DEBUG:boto:Data: 2022-09-03T07:15:33.558 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.559 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.560 DEBUG:boto:Params: {} 2022-09-03T07:15:33.560 DEBUG:boto:Token: None 2022-09-03T07:15:33.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj51 2022-09-03T07:15:33.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H8DQsn4hHO+vVjqbaMA7SGTGDQo= 2022-09-03T07:15:33.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H8DQsn4hHO+vVjqbaMA7SGTGDQo=', 'Content-Length': '0'} 2022-09-03T07:15:33.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca82acafe8275b16-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.606 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.607 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.607 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:33.608 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:33.609 DEBUG:boto:Method: GET 2022-09-03T07:15:33.609 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:33.610 DEBUG:boto:Data: 2022-09-03T07:15:33.614 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.615 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.616 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.617 DEBUG:boto:Params: {} 2022-09-03T07:15:33.619 DEBUG:boto:Token: None 2022-09-03T07:15:33.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.626 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000083ab96625d8ff0ed-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.627 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.628 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.630 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.638 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:33.639 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:33.640 DEBUG:boto:Method: GET 2022-09-03T07:15:33.642 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:33.643 DEBUG:boto:Data: 2022-09-03T07:15:33.644 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.645 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.646 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.647 DEBUG:boto:Params: {} 2022-09-03T07:15:33.648 DEBUG:boto:Token: None 2022-09-03T07:15:33.650 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.659 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000057e85a8c4f8172e1-006312ff15-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.660 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.661 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:15:33.662 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.662 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.663 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.664 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.664 DEBUG:boto:Method: GET 2022-09-03T07:15:33.665 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.665 DEBUG:boto:Data: 2022-09-03T07:15:33.666 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.666 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.667 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.668 DEBUG:boto:Params: {} 2022-09-03T07:15:33.668 DEBUG:boto:Token: None 2022-09-03T07:15:33.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SxixBnDK6x3y/YD/HZ23OAZNPO8= 2022-09-03T07:15:33.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SxixBnDK6x3y/YD/HZ23OAZNPO8=', 'Content-Length': '0'} 2022-09-03T07:15:33.673 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c459f5634a95f182-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.675 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.676 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.677 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.678 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.679 DEBUG:boto:Method: GET 2022-09-03T07:15:33.680 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.681 DEBUG:boto:Data: 2022-09-03T07:15:33.682 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.684 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.685 DEBUG:boto:Params: {} 2022-09-03T07:15:33.686 DEBUG:boto:Token: None 2022-09-03T07:15:33.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SxixBnDK6x3y/YD/HZ23OAZNPO8= 2022-09-03T07:15:33.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SxixBnDK6x3y/YD/HZ23OAZNPO8=', 'Content-Length': '0'} 2022-09-03T07:15:33.693 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e1ce9f7e017e700-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.695 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.696 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.697 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.699 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.700 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.701 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.702 DEBUG:boto:Data: 2022-09-03T07:15:33.703 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.704 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.705 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.706 DEBUG:boto:Params: {} 2022-09-03T07:15:33.707 DEBUG:boto:Token: None 2022-09-03T07:15:33.708 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ohfi7Te7hQEucg77I2K+2AN1EBI= 2022-09-03T07:15:33.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ohfi7Te7hQEucg77I2K+2AN1EBI=', 'Content-Length': '0'} 2022-09-03T07:15:33.714 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005c2ff7af31a121d-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.717 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.718 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.718 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.719 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.719 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.720 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.720 DEBUG:boto:Data: 2022-09-03T07:15:33.721 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.722 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.723 DEBUG:boto:Params: {} 2022-09-03T07:15:33.723 DEBUG:boto:Token: None 2022-09-03T07:15:33.724 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ohfi7Te7hQEucg77I2K+2AN1EBI= 2022-09-03T07:15:33.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ohfi7Te7hQEucg77I2K+2AN1EBI=', 'Content-Length': '0'} 2022-09-03T07:15:33.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a33d5e79972f02f-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.740 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:15:33.741 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.744 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.745 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.746 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.747 DEBUG:boto:Method: GET 2022-09-03T07:15:33.748 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.749 DEBUG:boto:Data: 2022-09-03T07:15:33.750 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.752 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.753 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.756 DEBUG:boto:Params: {} 2022-09-03T07:15:33.758 DEBUG:boto:Token: None 2022-09-03T07:15:33.759 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SxixBnDK6x3y/YD/HZ23OAZNPO8= 2022-09-03T07:15:33.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SxixBnDK6x3y/YD/HZ23OAZNPO8=', 'Content-Length': '0'} 2022-09-03T07:15:33.766 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d946bfde02a57f87-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.768 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.769 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.772 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.773 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.774 DEBUG:boto:Method: GET 2022-09-03T07:15:33.775 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.776 DEBUG:boto:Data: 2022-09-03T07:15:33.777 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.779 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.781 DEBUG:boto:Params: {} 2022-09-03T07:15:33.791 DEBUG:boto:Token: None 2022-09-03T07:15:33.792 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:33.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SxixBnDK6x3y/YD/HZ23OAZNPO8= 2022-09-03T07:15:33.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SxixBnDK6x3y/YD/HZ23OAZNPO8=', 'Content-Length': '0'} 2022-09-03T07:15:33.802 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a899733d426105f-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.803 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.803 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.804 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:33.804 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:33.805 DEBUG:boto:Method: GET 2022-09-03T07:15:33.806 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:33.806 DEBUG:boto:Data: 2022-09-03T07:15:33.807 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.808 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.808 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.809 DEBUG:boto:Params: {} 2022-09-03T07:15:33.809 DEBUG:boto:Token: None 2022-09-03T07:15:33.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.814 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006813d50abbc1a788-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.818 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.819 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.820 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.821 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:33.822 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:33.823 DEBUG:boto:Method: GET 2022-09-03T07:15:33.824 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:33.825 DEBUG:boto:Data: 2022-09-03T07:15:33.826 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.832 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.833 DEBUG:boto:Params: {} 2022-09-03T07:15:33.834 DEBUG:boto:Token: None 2022-09-03T07:15:33.836 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.847 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000007b13bef6ae72b45-006312ff15-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.886 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.887 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.888 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.888 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.889 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:33.889 DEBUG:boto:Data: 2022-09-03T07:15:33.890 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.891 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.891 DEBUG:boto:Params: {} 2022-09-03T07:15:33.892 DEBUG:boto:Token: None 2022-09-03T07:15:33.892 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55 2022-09-03T07:15:33.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S76ZAG3hu7FXML0DJzKvSK1lXj4= 2022-09-03T07:15:33.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S76ZAG3hu7FXML0DJzKvSK1lXj4=', 'Content-Length': '0'} 2022-09-03T07:15:33.896 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd1564b9babd6cf8-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.896 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.897 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.898 DEBUG:boto:Method: HEAD 2022-09-03T07:15:33.898 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:33.899 DEBUG:boto:Data: 2022-09-03T07:15:33.899 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.903 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.904 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.904 DEBUG:boto:Params: {} 2022-09-03T07:15:33.906 DEBUG:boto:Token: None 2022-09-03T07:15:33.906 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55 2022-09-03T07:15:33.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S76ZAG3hu7FXML0DJzKvSK1lXj4= 2022-09-03T07:15:33.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S76ZAG3hu7FXML0DJzKvSK1lXj4=', 'Content-Length': '0'} 2022-09-03T07:15:33.909 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000219f0875194cf69a-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.910 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:15:33.911 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.911 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.915 DEBUG:boto:Method: GET 2022-09-03T07:15:33.916 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:33.918 DEBUG:boto:Data: 2022-09-03T07:15:33.919 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.922 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.923 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.923 DEBUG:boto:Params: {} 2022-09-03T07:15:33.925 DEBUG:boto:Token: None 2022-09-03T07:15:33.926 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55 2022-09-03T07:15:33.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1KhcJl6IMDc8aCZmML7DpqbQIYo= 2022-09-03T07:15:33.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1KhcJl6IMDc8aCZmML7DpqbQIYo=', 'Content-Length': '0'} 2022-09-03T07:15:33.933 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e938ca5d79ffaff-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.940 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:33.941 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:33.950 DEBUG:boto:Method: GET 2022-09-03T07:15:33.952 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:33.953 DEBUG:boto:Data: 2022-09-03T07:15:33.953 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:33.955 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.957 DEBUG:boto:Params: {} 2022-09-03T07:15:33.958 DEBUG:boto:Token: None 2022-09-03T07:15:33.959 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/obj55 2022-09-03T07:15:33.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1KhcJl6IMDc8aCZmML7DpqbQIYo= 2022-09-03T07:15:33.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1KhcJl6IMDc8aCZmML7DpqbQIYo=', 'Content-Length': '0'} 2022-09-03T07:15:33.964 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f5cfe86e72272e8-006312ff15-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.966 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.968 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.968 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:33.969 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:33.970 DEBUG:boto:Method: GET 2022-09-03T07:15:33.971 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:33.972 DEBUG:boto:Data: 2022-09-03T07:15:33.972 DEBUG:boto:Headers: {} 2022-09-03T07:15:33.973 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:33.974 DEBUG:boto:Port: 8000 2022-09-03T07:15:33.975 DEBUG:boto:Params: {} 2022-09-03T07:15:33.976 DEBUG:boto:Token: None 2022-09-03T07:15:33.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:33 GMT /avysmw-75/ 2022-09-03T07:15:33.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE= 2022-09-03T07:15:33.986 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:33 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q17D4HN8lf4ppZ9ba/uYl5xkpgE=', 'Content-Length': '0'} 2022-09-03T07:15:33.993 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045d5354e49e9a0c5-006312ff15-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:33 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:33.995 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:33.996 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:33.997 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:33.998 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:34.000 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:34.001 DEBUG:boto:Method: GET 2022-09-03T07:15:34.002 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:34.003 DEBUG:boto:Data: 2022-09-03T07:15:34.004 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.005 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.006 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.007 DEBUG:boto:Params: {} 2022-09-03T07:15:34.009 DEBUG:boto:Token: None 2022-09-03T07:15:34.010 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I= 2022-09-03T07:15:34.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I=', 'Content-Length': '0'} 2022-09-03T07:15:34.020 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005cfbd2ffbdc563f9-006312ff16-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.021 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.023 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:15:34.024 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.026 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.027 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.028 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.029 DEBUG:boto:Method: GET 2022-09-03T07:15:34.030 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.031 DEBUG:boto:Data: 2022-09-03T07:15:34.032 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.032 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.033 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.034 DEBUG:boto:Params: {} 2022-09-03T07:15:34.037 DEBUG:boto:Token: None 2022-09-03T07:15:34.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Tw+CKjAPMuxygFPjOWxOCYU/AI= 2022-09-03T07:15:34.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Tw+CKjAPMuxygFPjOWxOCYU/AI=', 'Content-Length': '0'} 2022-09-03T07:15:34.058 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000077aa437537be6374-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.059 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.060 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.060 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.061 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.062 DEBUG:boto:Method: GET 2022-09-03T07:15:34.063 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.063 DEBUG:boto:Data: 2022-09-03T07:15:34.064 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.064 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.065 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.065 DEBUG:boto:Params: {} 2022-09-03T07:15:34.070 DEBUG:boto:Token: None 2022-09-03T07:15:34.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Tw+CKjAPMuxygFPjOWxOCYU/AI= 2022-09-03T07:15:34.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Tw+CKjAPMuxygFPjOWxOCYU/AI=', 'Content-Length': '0'} 2022-09-03T07:15:34.078 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000824f176123d5b63e-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.080 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.083 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.084 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.094 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.095 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.096 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.097 DEBUG:boto:Data: 2022-09-03T07:15:34.097 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.100 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.101 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.104 DEBUG:boto:Params: {} 2022-09-03T07:15:34.105 DEBUG:boto:Token: None 2022-09-03T07:15:34.106 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.107 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nf+JDlKrM37WLcB8nNmeYEdJCmk= 2022-09-03T07:15:34.107 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nf+JDlKrM37WLcB8nNmeYEdJCmk=', 'Content-Length': '0'} 2022-09-03T07:15:34.109 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1eafe664e1f12e1-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.110 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.111 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.112 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.113 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.113 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.114 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.114 DEBUG:boto:Data: 2022-09-03T07:15:34.115 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.116 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.116 DEBUG:boto:Params: {} 2022-09-03T07:15:34.117 DEBUG:boto:Token: None 2022-09-03T07:15:34.118 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nf+JDlKrM37WLcB8nNmeYEdJCmk= 2022-09-03T07:15:34.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nf+JDlKrM37WLcB8nNmeYEdJCmk=', 'Content-Length': '0'} 2022-09-03T07:15:34.130 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a7340dd9f82547b6-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.131 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:15:34.132 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.134 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.135 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.136 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.137 DEBUG:boto:Method: GET 2022-09-03T07:15:34.138 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.139 DEBUG:boto:Data: 2022-09-03T07:15:34.139 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.140 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.141 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.142 DEBUG:boto:Params: {} 2022-09-03T07:15:34.144 DEBUG:boto:Token: None 2022-09-03T07:15:34.145 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.147 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Tw+CKjAPMuxygFPjOWxOCYU/AI= 2022-09-03T07:15:34.148 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Tw+CKjAPMuxygFPjOWxOCYU/AI=', 'Content-Length': '0'} 2022-09-03T07:15:34.154 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e8842464d4f05e9-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.156 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.161 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.162 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.163 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.163 DEBUG:boto:Method: GET 2022-09-03T07:15:34.165 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.166 DEBUG:boto:Data: 2022-09-03T07:15:34.167 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.169 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.170 DEBUG:boto:Params: {} 2022-09-03T07:15:34.171 DEBUG:boto:Token: None 2022-09-03T07:15:34.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:34.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Tw+CKjAPMuxygFPjOWxOCYU/AI= 2022-09-03T07:15:34.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Tw+CKjAPMuxygFPjOWxOCYU/AI=', 'Content-Length': '0'} 2022-09-03T07:15:34.180 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049cec2df237dc3c3-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.182 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.183 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.184 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:34.185 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:34.186 DEBUG:boto:Method: GET 2022-09-03T07:15:34.187 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:34.188 DEBUG:boto:Data: 2022-09-03T07:15:34.189 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.190 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.191 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.192 DEBUG:boto:Params: {} 2022-09-03T07:15:34.193 DEBUG:boto:Token: None 2022-09-03T07:15:34.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I= 2022-09-03T07:15:34.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I=', 'Content-Length': '0'} 2022-09-03T07:15:34.204 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d97369376a374014-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.207 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.208 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.209 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.210 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:34.211 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:34.212 DEBUG:boto:Method: GET 2022-09-03T07:15:34.213 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:34.215 DEBUG:boto:Data: 2022-09-03T07:15:34.216 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.219 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.219 DEBUG:boto:Params: {} 2022-09-03T07:15:34.220 DEBUG:boto:Token: None 2022-09-03T07:15:34.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I= 2022-09-03T07:15:34.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I=', 'Content-Length': '0'} 2022-09-03T07:15:34.228 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6733b5a14b0edf0-006312ff16-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.229 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.231 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.232 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.233 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.233 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:34.234 DEBUG:boto:Data: 2022-09-03T07:15:34.234 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.235 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.236 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.237 DEBUG:boto:Params: {} 2022-09-03T07:15:34.240 DEBUG:boto:Token: None 2022-09-03T07:15:34.241 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68 2022-09-03T07:15:34.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1fvc6m20o/9jLHiHMCypkEH6N5M= 2022-09-03T07:15:34.246 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1fvc6m20o/9jLHiHMCypkEH6N5M=', 'Content-Length': '0'} 2022-09-03T07:15:34.253 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ec19e9b1822e3a7-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.255 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.257 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.259 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.261 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:34.262 DEBUG:boto:Data: 2022-09-03T07:15:34.263 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.265 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.266 DEBUG:boto:Params: {} 2022-09-03T07:15:34.267 DEBUG:boto:Token: None 2022-09-03T07:15:34.268 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68 2022-09-03T07:15:34.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1fvc6m20o/9jLHiHMCypkEH6N5M= 2022-09-03T07:15:34.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1fvc6m20o/9jLHiHMCypkEH6N5M=', 'Content-Length': '0'} 2022-09-03T07:15:34.276 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053255f71a8a60730-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.278 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:15:34.279 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.283 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.284 DEBUG:boto:Method: GET 2022-09-03T07:15:34.286 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:34.287 DEBUG:boto:Data: 2022-09-03T07:15:34.288 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.289 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.290 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.291 DEBUG:boto:Params: {} 2022-09-03T07:15:34.293 DEBUG:boto:Token: None 2022-09-03T07:15:34.294 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68 2022-09-03T07:15:34.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zv9gC2BQsLwdSQMzkQXGJkzLp/A= 2022-09-03T07:15:34.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zv9gC2BQsLwdSQMzkQXGJkzLp/A=', 'Content-Length': '0'} 2022-09-03T07:15:34.305 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049babc0bde35d1b1-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.307 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:34.308 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:34.309 DEBUG:boto:Method: GET 2022-09-03T07:15:34.310 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:34.313 DEBUG:boto:Data: 2022-09-03T07:15:34.314 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.316 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.317 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.318 DEBUG:boto:Params: {} 2022-09-03T07:15:34.319 DEBUG:boto:Token: None 2022-09-03T07:15:34.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj68 2022-09-03T07:15:34.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zv9gC2BQsLwdSQMzkQXGJkzLp/A= 2022-09-03T07:15:34.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zv9gC2BQsLwdSQMzkQXGJkzLp/A=', 'Content-Length': '0'} 2022-09-03T07:15:34.337 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000fbbb35119049a33-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.338 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.339 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.340 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:34.340 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:34.341 DEBUG:boto:Method: GET 2022-09-03T07:15:34.341 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:34.342 DEBUG:boto:Data: 2022-09-03T07:15:34.342 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.343 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.344 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.345 DEBUG:boto:Params: {} 2022-09-03T07:15:34.345 DEBUG:boto:Token: None 2022-09-03T07:15:34.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I= 2022-09-03T07:15:34.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I=', 'Content-Length': '0'} 2022-09-03T07:15:34.350 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f654c25d93f9d655-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.351 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.354 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.355 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.356 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:34.358 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:34.359 DEBUG:boto:Method: GET 2022-09-03T07:15:34.360 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:34.361 DEBUG:boto:Data: 2022-09-03T07:15:34.362 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.364 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.365 DEBUG:boto:Params: {} 2022-09-03T07:15:34.369 DEBUG:boto:Token: None 2022-09-03T07:15:34.370 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.371 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I= 2022-09-03T07:15:34.372 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I=', 'Content-Length': '0'} 2022-09-03T07:15:34.389 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ebfa1d25333a1fe1-006312ff16-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.390 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.391 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:15:34.392 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.393 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.393 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.394 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.394 DEBUG:boto:Method: GET 2022-09-03T07:15:34.395 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.396 DEBUG:boto:Data: 2022-09-03T07:15:34.400 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.400 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.401 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.401 DEBUG:boto:Params: {} 2022-09-03T07:15:34.402 DEBUG:boto:Token: None 2022-09-03T07:15:34.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EDQ144uiArWGCegPkPD09BsFTZ4= 2022-09-03T07:15:34.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EDQ144uiArWGCegPkPD09BsFTZ4=', 'Content-Length': '0'} 2022-09-03T07:15:34.407 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000d86000f7ac4428-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.409 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.410 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.411 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.411 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.412 DEBUG:boto:Method: GET 2022-09-03T07:15:34.415 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.417 DEBUG:boto:Data: 2022-09-03T07:15:34.418 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.420 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.421 DEBUG:boto:Params: {} 2022-09-03T07:15:34.423 DEBUG:boto:Token: None 2022-09-03T07:15:34.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EDQ144uiArWGCegPkPD09BsFTZ4= 2022-09-03T07:15:34.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EDQ144uiArWGCegPkPD09BsFTZ4=', 'Content-Length': '0'} 2022-09-03T07:15:34.430 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe3850c6cda1056c-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.433 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.434 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.435 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.436 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.437 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.438 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.439 DEBUG:boto:Data: 2022-09-03T07:15:34.440 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.441 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.444 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.445 DEBUG:boto:Params: {} 2022-09-03T07:15:34.447 DEBUG:boto:Token: None 2022-09-03T07:15:34.448 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pyadXxDi7A3C29QueUPcBiaBpFQ= 2022-09-03T07:15:34.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pyadXxDi7A3C29QueUPcBiaBpFQ=', 'Content-Length': '0'} 2022-09-03T07:15:34.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a0525760fa865507-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.456 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.457 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.458 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.459 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.462 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.463 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.463 DEBUG:boto:Data: 2022-09-03T07:15:34.464 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.467 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.468 DEBUG:boto:Params: {} 2022-09-03T07:15:34.469 DEBUG:boto:Token: None 2022-09-03T07:15:34.470 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pyadXxDi7A3C29QueUPcBiaBpFQ= 2022-09-03T07:15:34.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pyadXxDi7A3C29QueUPcBiaBpFQ=', 'Content-Length': '0'} 2022-09-03T07:15:34.477 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e881a1020ae05134-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.478 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:15:34.479 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.480 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.481 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.482 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.483 DEBUG:boto:Method: GET 2022-09-03T07:15:34.484 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.488 DEBUG:boto:Data: 2022-09-03T07:15:34.489 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.491 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.492 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.494 DEBUG:boto:Params: {} 2022-09-03T07:15:34.494 DEBUG:boto:Token: None 2022-09-03T07:15:34.495 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EDQ144uiArWGCegPkPD09BsFTZ4= 2022-09-03T07:15:34.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EDQ144uiArWGCegPkPD09BsFTZ4=', 'Content-Length': '0'} 2022-09-03T07:15:34.502 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f31a56c8c010a91-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.504 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.505 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.506 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.507 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.508 DEBUG:boto:Method: GET 2022-09-03T07:15:34.509 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.510 DEBUG:boto:Data: 2022-09-03T07:15:34.511 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.514 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.515 DEBUG:boto:Params: {} 2022-09-03T07:15:34.515 DEBUG:boto:Token: None 2022-09-03T07:15:34.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:34.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EDQ144uiArWGCegPkPD09BsFTZ4= 2022-09-03T07:15:34.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EDQ144uiArWGCegPkPD09BsFTZ4=', 'Content-Length': '0'} 2022-09-03T07:15:34.527 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ecd20f4c729056aa-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.528 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.529 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.529 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:34.537 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:34.538 DEBUG:boto:Method: GET 2022-09-03T07:15:34.538 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:34.539 DEBUG:boto:Data: 2022-09-03T07:15:34.539 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.540 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.540 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.541 DEBUG:boto:Params: {} 2022-09-03T07:15:34.541 DEBUG:boto:Token: None 2022-09-03T07:15:34.542 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I= 2022-09-03T07:15:34.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I=', 'Content-Length': '0'} 2022-09-03T07:15:34.554 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1efd4c609388d1f-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.560 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.562 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.563 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.563 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:34.564 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:34.564 DEBUG:boto:Method: GET 2022-09-03T07:15:34.565 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:34.565 DEBUG:boto:Data: 2022-09-03T07:15:34.566 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.581 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.582 DEBUG:boto:Params: {} 2022-09-03T07:15:34.584 DEBUG:boto:Token: None 2022-09-03T07:15:34.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I= 2022-09-03T07:15:34.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I=', 'Content-Length': '0'} 2022-09-03T07:15:34.593 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091fb5ef354426e2b-006312ff16-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.594 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.596 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.598 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.599 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.600 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:34.601 DEBUG:boto:Data: 2022-09-03T07:15:34.603 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.604 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.605 DEBUG:boto:Params: {} 2022-09-03T07:15:34.605 DEBUG:boto:Token: None 2022-09-03T07:15:34.607 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85 2022-09-03T07:15:34.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yUstulD+iOlpyzRUEcfswFUp6ds= 2022-09-03T07:15:34.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yUstulD+iOlpyzRUEcfswFUp6ds=', 'Content-Length': '0'} 2022-09-03T07:15:34.619 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c3a3bcd5570c6b0-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.620 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.620 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.621 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.622 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:34.622 DEBUG:boto:Data: 2022-09-03T07:15:34.623 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.624 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.624 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.625 DEBUG:boto:Params: {} 2022-09-03T07:15:34.625 DEBUG:boto:Token: None 2022-09-03T07:15:34.626 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85 2022-09-03T07:15:34.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yUstulD+iOlpyzRUEcfswFUp6ds= 2022-09-03T07:15:34.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yUstulD+iOlpyzRUEcfswFUp6ds=', 'Content-Length': '0'} 2022-09-03T07:15:34.630 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6e9330a748a8d77-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.630 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:15:34.631 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.632 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.632 DEBUG:boto:Method: GET 2022-09-03T07:15:34.633 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:34.634 DEBUG:boto:Data: 2022-09-03T07:15:34.635 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.635 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.636 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.637 DEBUG:boto:Params: {} 2022-09-03T07:15:34.638 DEBUG:boto:Token: None 2022-09-03T07:15:34.639 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85 2022-09-03T07:15:34.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:naTj91hhO9AFSCEE9YDjwgcs3mc= 2022-09-03T07:15:34.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:naTj91hhO9AFSCEE9YDjwgcs3mc=', 'Content-Length': '0'} 2022-09-03T07:15:34.644 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064e06cc400896c5f-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.646 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:34.647 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:34.648 DEBUG:boto:Method: GET 2022-09-03T07:15:34.649 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:34.650 DEBUG:boto:Data: 2022-09-03T07:15:34.650 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.657 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.658 DEBUG:boto:Params: {} 2022-09-03T07:15:34.658 DEBUG:boto:Token: None 2022-09-03T07:15:34.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj85 2022-09-03T07:15:34.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:naTj91hhO9AFSCEE9YDjwgcs3mc= 2022-09-03T07:15:34.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:naTj91hhO9AFSCEE9YDjwgcs3mc=', 'Content-Length': '0'} 2022-09-03T07:15:34.665 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054825e4423ed68ef-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.711 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.712 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.713 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:34.715 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:34.716 DEBUG:boto:Method: GET 2022-09-03T07:15:34.717 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:34.718 DEBUG:boto:Data: 2022-09-03T07:15:34.719 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.720 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:34.720 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.721 DEBUG:boto:Params: {} 2022-09-03T07:15:34.722 DEBUG:boto:Token: None 2022-09-03T07:15:34.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I= 2022-09-03T07:15:34.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I=', 'Content-Length': '0'} 2022-09-03T07:15:34.734 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000080b2f313461f4c0b-006312ff16-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.735 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.736 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.737 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.738 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:34.739 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:34.740 DEBUG:boto:Method: GET 2022-09-03T07:15:34.742 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:34.744 DEBUG:boto:Data: 2022-09-03T07:15:34.746 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.747 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.748 DEBUG:boto:Params: {} 2022-09-03T07:15:34.749 DEBUG:boto:Token: None 2022-09-03T07:15:34.750 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I= 2022-09-03T07:15:34.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rJ+s4+yjDQOF3wZYX4+s+LWmC7I=', 'Content-Length': '0'} 2022-09-03T07:15:34.757 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e24fdc99c8fc0b24-006312ff16-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.758 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.759 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:15:34.760 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:15:34.761 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.761 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.762 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.762 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:34.763 DEBUG:boto:Data: 2022-09-03T07:15:34.764 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:34.765 DEBUG:boto:Port: 8001 2022-09-03T07:15:34.766 DEBUG:boto:Params: {} 2022-09-03T07:15:34.767 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:15:34.767 DEBUG:boto:Token: None 2022-09-03T07:15:34.768 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZfJwfeeU8XZM4GQsfYRpO2nTKRA= 2022-09-03T07:15:34.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZfJwfeeU8XZM4GQsfYRpO2nTKRA=', 'Content-Length': '0'} 2022-09-03T07:15:34.775 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c44fa3927fbc99b3-006312ff16-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.777 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.777 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.778 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.779 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:34.779 DEBUG:boto:Data: 2022-09-03T07:15:34.780 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.781 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.781 DEBUG:boto:Params: {} 2022-09-03T07:15:34.782 DEBUG:boto:Token: None 2022-09-03T07:15:34.783 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/ 2022-09-03T07:15:34.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZfJwfeeU8XZM4GQsfYRpO2nTKRA= 2022-09-03T07:15:34.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZfJwfeeU8XZM4GQsfYRpO2nTKRA=', 'Content-Length': '0'} 2022-09-03T07:15:34.790 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000081aa081cfd4f7fed-006312ff16-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.791 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:15:34.792 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.792 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.793 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.793 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.794 DEBUG:boto:Method: GET 2022-09-03T07:15:34.795 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.795 DEBUG:boto:Data: 2022-09-03T07:15:34.796 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.796 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:34.797 DEBUG:boto:Port: 8001 2022-09-03T07:15:34.798 DEBUG:boto:Params: {} 2022-09-03T07:15:34.798 DEBUG:boto:Token: None 2022-09-03T07:15:34.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/?versions 2022-09-03T07:15:34.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D2Nw153xSKDYqZa5pc5Eu3m36Vw= 2022-09-03T07:15:34.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D2Nw153xSKDYqZa5pc5Eu3m36Vw=', 'Content-Length': '0'} 2022-09-03T07:15:34.804 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9befc8dac751ae3-006312ff16-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.804 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.806 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:15:34.806 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:15:34.807 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:15:34.808 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:15:34.808 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:15:34.808 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:15:34.809 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:15:34.810 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:15:34.810 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:15:34.811 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:15:34.811 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:15:34.812 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.813 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.813 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.814 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.814 DEBUG:boto:Method: GET 2022-09-03T07:15:34.815 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.815 DEBUG:boto:Data: 2022-09-03T07:15:34.816 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.817 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.817 DEBUG:boto:Params: {} 2022-09-03T07:15:34.818 DEBUG:boto:Token: None 2022-09-03T07:15:34.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/?versions 2022-09-03T07:15:34.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D2Nw153xSKDYqZa5pc5Eu3m36Vw= 2022-09-03T07:15:34.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D2Nw153xSKDYqZa5pc5Eu3m36Vw=', 'Content-Length': '0'} 2022-09-03T07:15:34.823 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000523cb8f958367f92-006312ff16-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.824 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.825 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:15:34.826 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:15:34.826 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:15:34.827 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:15:34.827 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:15:34.828 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:15:34.828 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:15:34.829 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:15:34.829 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:15:34.830 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:15:34.830 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.831 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.832 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.832 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.833 DEBUG:boto:Method: GET 2022-09-03T07:15:34.833 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.834 DEBUG:boto:Data: 2022-09-03T07:15:34.834 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:34.835 DEBUG:boto:Port: 8001 2022-09-03T07:15:34.836 DEBUG:boto:Params: {} 2022-09-03T07:15:34.836 DEBUG:boto:Token: None 2022-09-03T07:15:34.837 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/?versions 2022-09-03T07:15:34.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D2Nw153xSKDYqZa5pc5Eu3m36Vw= 2022-09-03T07:15:34.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D2Nw153xSKDYqZa5pc5Eu3m36Vw=', 'Content-Length': '0'} 2022-09-03T07:15:34.843 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e49b4dbc0daf68f8-006312ff16-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.844 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.848 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:34.849 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:34.851 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.852 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.853 DEBUG:boto:Method: GET 2022-09-03T07:15:34.854 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:34.855 DEBUG:boto:Data: 2022-09-03T07:15:34.857 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.858 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.859 DEBUG:boto:Params: {} 2022-09-03T07:15:34.859 DEBUG:boto:Token: None 2022-09-03T07:15:34.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/?versions 2022-09-03T07:15:34.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D2Nw153xSKDYqZa5pc5Eu3m36Vw= 2022-09-03T07:15:34.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D2Nw153xSKDYqZa5pc5Eu3m36Vw=', 'Content-Length': '0'} 2022-09-03T07:15:34.866 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003959b9f4fe18ddf1-006312ff16-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.867 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:34.869 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:15:34.869 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:34.871 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:34.872 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.873 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.874 DEBUG:boto:Method: GET 2022-09-03T07:15:34.875 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.876 DEBUG:boto:Data: 2022-09-03T07:15:34.877 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:34.881 DEBUG:boto:Port: 8001 2022-09-03T07:15:34.882 DEBUG:boto:Params: {} 2022-09-03T07:15:34.883 DEBUG:boto:Token: None 2022-09-03T07:15:34.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pwf07gD5dBkiY61bsyfLLBw6gdU= 2022-09-03T07:15:34.891 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pwf07gD5dBkiY61bsyfLLBw6gdU=', 'Content-Length': '0'} 2022-09-03T07:15:34.894 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005cfa820190f2008c-006312ff16-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.895 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:34.896 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:34.897 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.897 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.898 DEBUG:boto:Method: GET 2022-09-03T07:15:34.899 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.901 DEBUG:boto:Data: 2022-09-03T07:15:34.903 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.904 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.905 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.905 DEBUG:boto:Params: {} 2022-09-03T07:15:34.906 DEBUG:boto:Token: None 2022-09-03T07:15:34.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pwf07gD5dBkiY61bsyfLLBw6gdU= 2022-09-03T07:15:34.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pwf07gD5dBkiY61bsyfLLBw6gdU=', 'Content-Length': '0'} 2022-09-03T07:15:34.910 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e6a73b003fa4ac9-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.913 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:34.914 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:34.915 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.916 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.917 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.917 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.918 DEBUG:boto:Data: 2022-09-03T07:15:34.918 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:34.921 DEBUG:boto:Port: 8001 2022-09-03T07:15:34.921 DEBUG:boto:Params: {} 2022-09-03T07:15:34.922 DEBUG:boto:Token: None 2022-09-03T07:15:34.922 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vbPDFv3FCAHFV1B2FFAcZ2DFWCw= 2022-09-03T07:15:34.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vbPDFv3FCAHFV1B2FFAcZ2DFWCw=', 'Content-Length': '0'} 2022-09-03T07:15:34.928 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053e3761b25315172-006312ff16-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.929 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:34.930 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:34.931 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.931 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.932 DEBUG:boto:Method: HEAD 2022-09-03T07:15:34.933 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.933 DEBUG:boto:Data: 2022-09-03T07:15:34.934 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.935 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.935 DEBUG:boto:Params: {} 2022-09-03T07:15:34.936 DEBUG:boto:Token: None 2022-09-03T07:15:34.937 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vbPDFv3FCAHFV1B2FFAcZ2DFWCw= 2022-09-03T07:15:34.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vbPDFv3FCAHFV1B2FFAcZ2DFWCw=', 'Content-Length': '0'} 2022-09-03T07:15:34.941 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b057ade45756f39f-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.944 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:15:34.945 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:34.946 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:34.947 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.948 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.949 DEBUG:boto:Method: GET 2022-09-03T07:15:34.959 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.960 DEBUG:boto:Data: 2022-09-03T07:15:34.961 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.962 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:34.963 DEBUG:boto:Port: 8001 2022-09-03T07:15:34.970 DEBUG:boto:Params: {} 2022-09-03T07:15:34.971 DEBUG:boto:Token: None 2022-09-03T07:15:34.973 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pwf07gD5dBkiY61bsyfLLBw6gdU= 2022-09-03T07:15:34.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pwf07gD5dBkiY61bsyfLLBw6gdU=', 'Content-Length': '0'} 2022-09-03T07:15:34.981 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f07a69f6279f4be9-006312ff16-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:34.982 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:34.982 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:34.983 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.984 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.985 DEBUG:boto:Method: GET 2022-09-03T07:15:34.986 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.987 DEBUG:boto:Data: 2022-09-03T07:15:34.989 DEBUG:boto:Headers: {} 2022-09-03T07:15:34.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:34.991 DEBUG:boto:Port: 8000 2022-09-03T07:15:34.992 DEBUG:boto:Params: {} 2022-09-03T07:15:34.993 DEBUG:boto:Token: None 2022-09-03T07:15:34.994 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:34 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:34.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pwf07gD5dBkiY61bsyfLLBw6gdU= 2022-09-03T07:15:34.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:34 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pwf07gD5dBkiY61bsyfLLBw6gdU=', 'Content-Length': '0'} 2022-09-03T07:15:35.000 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc63bb47fdadcfd5-006312ff16-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:34 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.001 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.002 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.004 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:35.005 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:35.006 DEBUG:boto:Method: GET 2022-09-03T07:15:35.007 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:35.007 DEBUG:boto:Data: 2022-09-03T07:15:35.009 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.018 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.019 DEBUG:boto:Params: {} 2022-09-03T07:15:35.020 DEBUG:boto:Token: None 2022-09-03T07:15:35.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.035 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e35705653eedf985-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.036 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.037 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.037 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.038 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:35.038 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:35.039 DEBUG:boto:Method: GET 2022-09-03T07:15:35.039 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:35.040 DEBUG:boto:Data: 2022-09-03T07:15:35.040 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.041 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.042 DEBUG:boto:Params: {} 2022-09-03T07:15:35.043 DEBUG:boto:Token: None 2022-09-03T07:15:35.043 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.044 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.050 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e81b0812ed1b810-006312ff17-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.051 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.052 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:35.052 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:35.053 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.054 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:35.054 DEBUG:boto:Data: 2022-09-03T07:15:35.055 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.057 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.058 DEBUG:boto:Params: {} 2022-09-03T07:15:35.059 DEBUG:boto:Token: None 2022-09-03T07:15:35.060 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj11 2022-09-03T07:15:35.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/D27e9O/k6z5AWjXpjFPWuoeHnQ= 2022-09-03T07:15:35.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/D27e9O/k6z5AWjXpjFPWuoeHnQ=', 'Content-Length': '0'} 2022-09-03T07:15:35.065 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000944f1bc65a16fe8c-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.068 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:35.069 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:35.069 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.070 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:35.071 DEBUG:boto:Data: 2022-09-03T07:15:35.072 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.074 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.074 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.075 DEBUG:boto:Params: {} 2022-09-03T07:15:35.076 DEBUG:boto:Token: None 2022-09-03T07:15:35.078 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj11 2022-09-03T07:15:35.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/D27e9O/k6z5AWjXpjFPWuoeHnQ= 2022-09-03T07:15:35.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/D27e9O/k6z5AWjXpjFPWuoeHnQ=', 'Content-Length': '0'} 2022-09-03T07:15:35.089 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000050ba95f0d0ab0411-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.090 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:15:35.090 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:35.091 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:35.096 DEBUG:boto:Method: GET 2022-09-03T07:15:35.096 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:35.111 DEBUG:boto:Data: 2022-09-03T07:15:35.111 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.112 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.113 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.113 DEBUG:boto:Params: {} 2022-09-03T07:15:35.114 DEBUG:boto:Token: None 2022-09-03T07:15:35.115 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj11 2022-09-03T07:15:35.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FZ9tT2sTBdypPsjtAO6jFDCqryc= 2022-09-03T07:15:35.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FZ9tT2sTBdypPsjtAO6jFDCqryc=', 'Content-Length': '0'} 2022-09-03T07:15:35.130 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000459e4fcaa298fe0a-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.132 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:35.133 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:35.133 DEBUG:boto:Method: GET 2022-09-03T07:15:35.134 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:35.134 DEBUG:boto:Data: 2022-09-03T07:15:35.135 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.136 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.136 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.137 DEBUG:boto:Params: {} 2022-09-03T07:15:35.137 DEBUG:boto:Token: None 2022-09-03T07:15:35.138 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj11 2022-09-03T07:15:35.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FZ9tT2sTBdypPsjtAO6jFDCqryc= 2022-09-03T07:15:35.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FZ9tT2sTBdypPsjtAO6jFDCqryc=', 'Content-Length': '0'} 2022-09-03T07:15:35.143 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a7d95ff6c0f772d-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.145 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.145 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.146 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:35.147 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:35.147 DEBUG:boto:Method: GET 2022-09-03T07:15:35.148 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:35.148 DEBUG:boto:Data: 2022-09-03T07:15:35.149 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.149 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.150 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.151 DEBUG:boto:Params: {} 2022-09-03T07:15:35.152 DEBUG:boto:Token: None 2022-09-03T07:15:35.152 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.158 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087f7ac960aeafc5e-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.158 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.159 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.160 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.161 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:35.161 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:35.162 DEBUG:boto:Method: GET 2022-09-03T07:15:35.163 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:35.164 DEBUG:boto:Data: 2022-09-03T07:15:35.164 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.165 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.166 DEBUG:boto:Params: {} 2022-09-03T07:15:35.166 DEBUG:boto:Token: None 2022-09-03T07:15:35.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.180 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2f56490cd6809ab-006312ff17-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.205 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.212 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:15:35.213 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.214 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.232 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.235 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.236 DEBUG:boto:Method: GET 2022-09-03T07:15:35.237 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.241 DEBUG:boto:Data: 2022-09-03T07:15:35.241 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.242 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.243 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.243 DEBUG:boto:Params: {} 2022-09-03T07:15:35.247 DEBUG:boto:Token: None 2022-09-03T07:15:35.248 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ioggt6uGowVYDT/apPPi5NEdisQ= 2022-09-03T07:15:35.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ioggt6uGowVYDT/apPPi5NEdisQ=', 'Content-Length': '0'} 2022-09-03T07:15:35.253 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054449495f3dd1264-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.255 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.255 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.256 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.257 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.263 DEBUG:boto:Method: GET 2022-09-03T07:15:35.264 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.265 DEBUG:boto:Data: 2022-09-03T07:15:35.266 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.267 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.267 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.268 DEBUG:boto:Params: {} 2022-09-03T07:15:35.268 DEBUG:boto:Token: None 2022-09-03T07:15:35.270 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ioggt6uGowVYDT/apPPi5NEdisQ= 2022-09-03T07:15:35.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ioggt6uGowVYDT/apPPi5NEdisQ=', 'Content-Length': '0'} 2022-09-03T07:15:35.274 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f84d03e12f435483-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.275 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.276 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.276 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.277 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.277 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.278 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.278 DEBUG:boto:Data: 2022-09-03T07:15:35.279 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.279 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.280 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.280 DEBUG:boto:Params: {} 2022-09-03T07:15:35.281 DEBUG:boto:Token: None 2022-09-03T07:15:35.282 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OhwuiayTcqxvNgMcj4IeZWY4XlA= 2022-09-03T07:15:35.284 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OhwuiayTcqxvNgMcj4IeZWY4XlA=', 'Content-Length': '0'} 2022-09-03T07:15:35.286 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000571aaf1ed554dbe7-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.287 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.288 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.288 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.289 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.289 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.290 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.290 DEBUG:boto:Data: 2022-09-03T07:15:35.291 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.291 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.292 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.292 DEBUG:boto:Params: {} 2022-09-03T07:15:35.293 DEBUG:boto:Token: None 2022-09-03T07:15:35.294 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OhwuiayTcqxvNgMcj4IeZWY4XlA= 2022-09-03T07:15:35.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OhwuiayTcqxvNgMcj4IeZWY4XlA=', 'Content-Length': '0'} 2022-09-03T07:15:35.299 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000191cd01900d14da2-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.300 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:15:35.301 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.302 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.306 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.307 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.308 DEBUG:boto:Method: GET 2022-09-03T07:15:35.308 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.309 DEBUG:boto:Data: 2022-09-03T07:15:35.309 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.310 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.310 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.311 DEBUG:boto:Params: {} 2022-09-03T07:15:35.312 DEBUG:boto:Token: None 2022-09-03T07:15:35.313 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.313 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ioggt6uGowVYDT/apPPi5NEdisQ= 2022-09-03T07:15:35.314 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ioggt6uGowVYDT/apPPi5NEdisQ=', 'Content-Length': '0'} 2022-09-03T07:15:35.319 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bb46197d8151640f-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.321 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.322 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.322 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.323 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.323 DEBUG:boto:Method: GET 2022-09-03T07:15:35.324 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.324 DEBUG:boto:Data: 2022-09-03T07:15:35.325 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.326 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.330 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.331 DEBUG:boto:Params: {} 2022-09-03T07:15:35.332 DEBUG:boto:Token: None 2022-09-03T07:15:35.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:35.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ioggt6uGowVYDT/apPPi5NEdisQ= 2022-09-03T07:15:35.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ioggt6uGowVYDT/apPPi5NEdisQ=', 'Content-Length': '0'} 2022-09-03T07:15:35.336 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000248cce2e9e360dba-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.379 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.381 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.383 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:35.388 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:35.389 DEBUG:boto:Method: GET 2022-09-03T07:15:35.396 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:35.397 DEBUG:boto:Data: 2022-09-03T07:15:35.397 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.398 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.399 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.399 DEBUG:boto:Params: {} 2022-09-03T07:15:35.401 DEBUG:boto:Token: None 2022-09-03T07:15:35.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.407 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad696a0011b96310-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.427 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.432 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.434 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.435 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:35.436 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:35.437 DEBUG:boto:Method: GET 2022-09-03T07:15:35.438 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:35.439 DEBUG:boto:Data: 2022-09-03T07:15:35.439 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.440 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.441 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.441 DEBUG:boto:Params: {} 2022-09-03T07:15:35.442 DEBUG:boto:Token: None 2022-09-03T07:15:35.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.449 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ee8ae49207208f05-006312ff17-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.449 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.450 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.451 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.452 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.455 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:35.456 DEBUG:boto:Data: 2022-09-03T07:15:35.457 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.459 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.460 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.466 DEBUG:boto:Params: {} 2022-09-03T07:15:35.467 DEBUG:boto:Token: None 2022-09-03T07:15:35.468 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17 2022-09-03T07:15:35.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6F6NyHAtnp2NI5Z0sbGgqQrk0wU= 2022-09-03T07:15:35.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6F6NyHAtnp2NI5Z0sbGgqQrk0wU=', 'Content-Length': '0'} 2022-09-03T07:15:35.475 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003031dc97f24c40ab-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.477 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.478 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.480 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.481 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:35.481 DEBUG:boto:Data: 2022-09-03T07:15:35.482 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.482 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.492 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.493 DEBUG:boto:Params: {} 2022-09-03T07:15:35.499 DEBUG:boto:Token: None 2022-09-03T07:15:35.501 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17 2022-09-03T07:15:35.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6F6NyHAtnp2NI5Z0sbGgqQrk0wU= 2022-09-03T07:15:35.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6F6NyHAtnp2NI5Z0sbGgqQrk0wU=', 'Content-Length': '0'} 2022-09-03T07:15:35.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d9ea0bcfc1bf022-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.511 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:15:35.512 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.514 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.515 DEBUG:boto:Method: GET 2022-09-03T07:15:35.516 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:35.517 DEBUG:boto:Data: 2022-09-03T07:15:35.518 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.519 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.520 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.521 DEBUG:boto:Params: {} 2022-09-03T07:15:35.522 DEBUG:boto:Token: None 2022-09-03T07:15:35.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17 2022-09-03T07:15:35.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lr4/xldIza6BOD200agtPkww1Io= 2022-09-03T07:15:35.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lr4/xldIza6BOD200agtPkww1Io=', 'Content-Length': '0'} 2022-09-03T07:15:35.535 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d599196916457eef-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.537 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:35.539 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:35.541 DEBUG:boto:Method: GET 2022-09-03T07:15:35.542 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:35.543 DEBUG:boto:Data: 2022-09-03T07:15:35.544 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.546 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.547 DEBUG:boto:Params: {} 2022-09-03T07:15:35.559 DEBUG:boto:Token: None 2022-09-03T07:15:35.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj17 2022-09-03T07:15:35.566 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lr4/xldIza6BOD200agtPkww1Io= 2022-09-03T07:15:35.568 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lr4/xldIza6BOD200agtPkww1Io=', 'Content-Length': '0'} 2022-09-03T07:15:35.571 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000456d1f67e89e5839-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.573 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.574 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.575 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:35.576 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:35.577 DEBUG:boto:Method: GET 2022-09-03T07:15:35.579 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:35.584 DEBUG:boto:Data: 2022-09-03T07:15:35.586 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.595 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.595 DEBUG:boto:Params: {} 2022-09-03T07:15:35.596 DEBUG:boto:Token: None 2022-09-03T07:15:35.597 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.607 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b776d86a591bd7a9-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.614 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.623 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.624 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.624 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:35.625 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:35.625 DEBUG:boto:Method: GET 2022-09-03T07:15:35.630 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:35.631 DEBUG:boto:Data: 2022-09-03T07:15:35.631 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.642 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.643 DEBUG:boto:Params: {} 2022-09-03T07:15:35.644 DEBUG:boto:Token: None 2022-09-03T07:15:35.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.660 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7f045b0df212626-006312ff17-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.662 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.663 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:15:35.664 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.665 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.666 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.666 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.667 DEBUG:boto:Method: GET 2022-09-03T07:15:35.668 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.669 DEBUG:boto:Data: 2022-09-03T07:15:35.677 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.678 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.679 DEBUG:boto:Params: {} 2022-09-03T07:15:35.679 DEBUG:boto:Token: None 2022-09-03T07:15:35.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Q09LgI1EHVUKxNcIDOqfXd+GAA= 2022-09-03T07:15:35.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Q09LgI1EHVUKxNcIDOqfXd+GAA=', 'Content-Length': '0'} 2022-09-03T07:15:35.697 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000463ac2375e5f9787-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.698 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.699 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.700 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.700 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.701 DEBUG:boto:Method: GET 2022-09-03T07:15:35.701 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.702 DEBUG:boto:Data: 2022-09-03T07:15:35.702 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.704 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.704 DEBUG:boto:Params: {} 2022-09-03T07:15:35.705 DEBUG:boto:Token: None 2022-09-03T07:15:35.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Q09LgI1EHVUKxNcIDOqfXd+GAA= 2022-09-03T07:15:35.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Q09LgI1EHVUKxNcIDOqfXd+GAA=', 'Content-Length': '0'} 2022-09-03T07:15:35.709 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004188d411761349ba-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.711 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.712 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.713 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.717 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.720 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.721 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.721 DEBUG:boto:Data: 2022-09-03T07:15:35.722 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.723 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.723 DEBUG:boto:Params: {} 2022-09-03T07:15:35.724 DEBUG:boto:Token: None 2022-09-03T07:15:35.724 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zOH7dhOC88ckcK7zCtx57lTYF14= 2022-09-03T07:15:35.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zOH7dhOC88ckcK7zCtx57lTYF14=', 'Content-Length': '0'} 2022-09-03T07:15:35.728 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f38348d5123bd87c-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.729 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.730 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.731 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.731 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.732 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.732 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.733 DEBUG:boto:Data: 2022-09-03T07:15:35.733 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.734 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.735 DEBUG:boto:Params: {} 2022-09-03T07:15:35.736 DEBUG:boto:Token: None 2022-09-03T07:15:35.736 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zOH7dhOC88ckcK7zCtx57lTYF14= 2022-09-03T07:15:35.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zOH7dhOC88ckcK7zCtx57lTYF14=', 'Content-Length': '0'} 2022-09-03T07:15:35.745 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3f7cf7c74d208fd-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.745 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:15:35.746 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.747 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.747 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.748 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.748 DEBUG:boto:Method: GET 2022-09-03T07:15:35.749 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.750 DEBUG:boto:Data: 2022-09-03T07:15:35.750 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.751 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.752 DEBUG:boto:Params: {} 2022-09-03T07:15:35.753 DEBUG:boto:Token: None 2022-09-03T07:15:35.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Q09LgI1EHVUKxNcIDOqfXd+GAA= 2022-09-03T07:15:35.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Q09LgI1EHVUKxNcIDOqfXd+GAA=', 'Content-Length': '0'} 2022-09-03T07:15:35.757 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef54a490feb18c8a-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.802 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.803 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.803 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.804 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.804 DEBUG:boto:Method: GET 2022-09-03T07:15:35.805 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.805 DEBUG:boto:Data: 2022-09-03T07:15:35.806 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.807 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.807 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.808 DEBUG:boto:Params: {} 2022-09-03T07:15:35.809 DEBUG:boto:Token: None 2022-09-03T07:15:35.809 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:35.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8Q09LgI1EHVUKxNcIDOqfXd+GAA= 2022-09-03T07:15:35.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8Q09LgI1EHVUKxNcIDOqfXd+GAA=', 'Content-Length': '0'} 2022-09-03T07:15:35.818 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000072337ddbdc4bb2fa-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.820 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.821 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.822 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:35.823 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:35.824 DEBUG:boto:Method: GET 2022-09-03T07:15:35.825 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:35.826 DEBUG:boto:Data: 2022-09-03T07:15:35.827 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.828 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.829 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.830 DEBUG:boto:Params: {} 2022-09-03T07:15:35.831 DEBUG:boto:Token: None 2022-09-03T07:15:35.832 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.838 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000097f6d1363a69c4ac-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.840 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.841 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.842 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.843 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:35.844 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:35.845 DEBUG:boto:Method: GET 2022-09-03T07:15:35.846 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:35.847 DEBUG:boto:Data: 2022-09-03T07:15:35.848 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.850 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.851 DEBUG:boto:Params: {} 2022-09-03T07:15:35.852 DEBUG:boto:Token: None 2022-09-03T07:15:35.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.857 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a2ab062289962184-006312ff17-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.858 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.859 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.859 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.860 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.861 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:35.861 DEBUG:boto:Data: 2022-09-03T07:15:35.862 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.862 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.863 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.863 DEBUG:boto:Params: {} 2022-09-03T07:15:35.864 DEBUG:boto:Token: None 2022-09-03T07:15:35.864 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22 2022-09-03T07:15:35.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ex/4/h5dpc77PAjQGOSeqHGFK0c= 2022-09-03T07:15:35.866 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ex/4/h5dpc77PAjQGOSeqHGFK0c=', 'Content-Length': '0'} 2022-09-03T07:15:35.868 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a0ac49cd9b942fe6-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.869 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.870 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.870 DEBUG:boto:Method: HEAD 2022-09-03T07:15:35.871 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:35.871 DEBUG:boto:Data: 2022-09-03T07:15:35.872 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.873 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.873 DEBUG:boto:Params: {} 2022-09-03T07:15:35.874 DEBUG:boto:Token: None 2022-09-03T07:15:35.875 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22 2022-09-03T07:15:35.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ex/4/h5dpc77PAjQGOSeqHGFK0c= 2022-09-03T07:15:35.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ex/4/h5dpc77PAjQGOSeqHGFK0c=', 'Content-Length': '0'} 2022-09-03T07:15:35.887 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d76004fa645c977-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.888 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:15:35.889 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.889 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.892 DEBUG:boto:Method: GET 2022-09-03T07:15:35.893 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:35.894 DEBUG:boto:Data: 2022-09-03T07:15:35.895 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.897 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.898 DEBUG:boto:Params: {} 2022-09-03T07:15:35.899 DEBUG:boto:Token: None 2022-09-03T07:15:35.900 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22 2022-09-03T07:15:35.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NOr6g7zRuTQJ3uOOtFUsJSdfc/o= 2022-09-03T07:15:35.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NOr6g7zRuTQJ3uOOtFUsJSdfc/o=', 'Content-Length': '0'} 2022-09-03T07:15:35.905 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020e5b918101f6dfa-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.906 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:35.906 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:35.907 DEBUG:boto:Method: GET 2022-09-03T07:15:35.907 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:35.908 DEBUG:boto:Data: 2022-09-03T07:15:35.909 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.910 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.910 DEBUG:boto:Params: {} 2022-09-03T07:15:35.911 DEBUG:boto:Token: None 2022-09-03T07:15:35.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj22 2022-09-03T07:15:35.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NOr6g7zRuTQJ3uOOtFUsJSdfc/o= 2022-09-03T07:15:35.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NOr6g7zRuTQJ3uOOtFUsJSdfc/o=', 'Content-Length': '0'} 2022-09-03T07:15:35.915 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f77ee97d7c31174-006312ff17-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.916 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.917 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.917 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:35.918 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:35.919 DEBUG:boto:Method: GET 2022-09-03T07:15:35.919 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:35.920 DEBUG:boto:Data: 2022-09-03T07:15:35.920 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.921 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.922 DEBUG:boto:Params: {} 2022-09-03T07:15:35.922 DEBUG:boto:Token: None 2022-09-03T07:15:35.923 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.930 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000066bad7b64e662e39-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.931 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.932 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:35.932 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:35.933 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:35.933 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:35.934 DEBUG:boto:Method: GET 2022-09-03T07:15:35.934 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:35.935 DEBUG:boto:Data: 2022-09-03T07:15:35.935 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.937 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:35.937 DEBUG:boto:Port: 8000 2022-09-03T07:15:35.938 DEBUG:boto:Params: {} 2022-09-03T07:15:35.939 DEBUG:boto:Token: None 2022-09-03T07:15:35.939 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/ 2022-09-03T07:15:35.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g= 2022-09-03T07:15:35.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:llu5Slz48PKxWi7cnsWnpcqaX8g=', 'Content-Length': '0'} 2022-09-03T07:15:35.944 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000007568ac9e809c591-006312ff17-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:35.945 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:35.947 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:15:35.948 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:35.948 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:35.949 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:35.950 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:35.951 DEBUG:boto:Method: GET 2022-09-03T07:15:35.951 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:35.952 DEBUG:boto:Data: 2022-09-03T07:15:35.953 DEBUG:boto:Headers: {} 2022-09-03T07:15:35.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:35.954 DEBUG:boto:Port: 8001 2022-09-03T07:15:35.954 DEBUG:boto:Params: {} 2022-09-03T07:15:35.955 DEBUG:boto:Token: None 2022-09-03T07:15:35.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:35 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:35.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Edgsq6dk2/VX1CV5ktyM3MLLRug= 2022-09-03T07:15:35.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:35 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Edgsq6dk2/VX1CV5ktyM3MLLRug=', 'Content-Length': '0'} 2022-09-03T07:15:35.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f97908070c351c3-006312ff17-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.007 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:36.008 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:36.009 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.009 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.011 DEBUG:boto:Method: GET 2022-09-03T07:15:36.011 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.012 DEBUG:boto:Data: 2022-09-03T07:15:36.012 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.013 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.014 DEBUG:boto:Params: {} 2022-09-03T07:15:36.015 DEBUG:boto:Token: None 2022-09-03T07:15:36.016 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/figsEc+ZC27aahwiae6fUSKSc= 2022-09-03T07:15:36.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/figsEc+ZC27aahwiae6fUSKSc=', 'Content-Length': '0'} 2022-09-03T07:15:36.024 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065c22e1247b637d2-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.025 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:36.026 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:36.027 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.027 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.028 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.028 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.029 DEBUG:boto:Data: 2022-09-03T07:15:36.030 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.031 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.032 DEBUG:boto:Params: {} 2022-09-03T07:15:36.032 DEBUG:boto:Token: None 2022-09-03T07:15:36.033 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvUM9igoK03QF6fciZxXXvn7OkI= 2022-09-03T07:15:36.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvUM9igoK03QF6fciZxXXvn7OkI=', 'Content-Length': '0'} 2022-09-03T07:15:36.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e6ff70617c722d8-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.038 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:36.038 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:36.039 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.039 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.040 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.040 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.041 DEBUG:boto:Data: 2022-09-03T07:15:36.041 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.046 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.047 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.047 DEBUG:boto:Params: {} 2022-09-03T07:15:36.051 DEBUG:boto:Token: None 2022-09-03T07:15:36.052 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HvUM9igoK03QF6fciZxXXvn7OkI= 2022-09-03T07:15:36.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HvUM9igoK03QF6fciZxXXvn7OkI=', 'Content-Length': '0'} 2022-09-03T07:15:36.056 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a7195bf2c764642f-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.057 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:15:36.057 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:36.058 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:36.059 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.059 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.060 DEBUG:boto:Method: GET 2022-09-03T07:15:36.060 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.061 DEBUG:boto:Data: 2022-09-03T07:15:36.061 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.063 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.063 DEBUG:boto:Params: {} 2022-09-03T07:15:36.064 DEBUG:boto:Token: None 2022-09-03T07:15:36.065 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/figsEc+ZC27aahwiae6fUSKSc= 2022-09-03T07:15:36.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/figsEc+ZC27aahwiae6fUSKSc=', 'Content-Length': '0'} 2022-09-03T07:15:36.074 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008bf1a7131c5b5f94-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.076 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:36.076 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:36.077 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.082 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.083 DEBUG:boto:Method: GET 2022-09-03T07:15:36.083 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.084 DEBUG:boto:Data: 2022-09-03T07:15:36.084 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.085 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.090 DEBUG:boto:Params: {} 2022-09-03T07:15:36.095 DEBUG:boto:Token: None 2022-09-03T07:15:36.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:36.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y/figsEc+ZC27aahwiae6fUSKSc= 2022-09-03T07:15:36.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y/figsEc+ZC27aahwiae6fUSKSc=', 'Content-Length': '0'} 2022-09-03T07:15:36.105 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e68733b4528dcca0-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.106 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.108 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.109 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:36.110 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:36.111 DEBUG:boto:Method: GET 2022-09-03T07:15:36.112 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:36.113 DEBUG:boto:Data: 2022-09-03T07:15:36.114 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.116 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.117 DEBUG:boto:Params: {} 2022-09-03T07:15:36.126 DEBUG:boto:Token: None 2022-09-03T07:15:36.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:36.132 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006608cb047e6fadb4-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.133 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:36.135 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.136 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.137 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:36.138 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:36.139 DEBUG:boto:Method: GET 2022-09-03T07:15:36.140 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:36.142 DEBUG:boto:Data: 2022-09-03T07:15:36.143 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.144 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.145 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.147 DEBUG:boto:Params: {} 2022-09-03T07:15:36.149 DEBUG:boto:Token: None 2022-09-03T07:15:36.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.151 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.152 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:36.156 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008045e147d67d0448-006312ff18-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.157 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:36.159 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:36.161 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:36.162 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.165 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:36.166 DEBUG:boto:Data: 2022-09-03T07:15:36.167 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.168 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.169 DEBUG:boto:Params: {} 2022-09-03T07:15:36.170 DEBUG:boto:Token: None 2022-09-03T07:15:36.171 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj33 2022-09-03T07:15:36.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mZmq7/+Ewz/xM7+eq5nYoCyrfKg= 2022-09-03T07:15:36.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mZmq7/+Ewz/xM7+eq5nYoCyrfKg=', 'Content-Length': '0'} 2022-09-03T07:15:36.175 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7fba68bf4fb98f8-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.176 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:36.177 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:36.178 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.179 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:36.180 DEBUG:boto:Data: 2022-09-03T07:15:36.181 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.184 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.185 DEBUG:boto:Params: {} 2022-09-03T07:15:36.186 DEBUG:boto:Token: None 2022-09-03T07:15:36.187 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj33 2022-09-03T07:15:36.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mZmq7/+Ewz/xM7+eq5nYoCyrfKg= 2022-09-03T07:15:36.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mZmq7/+Ewz/xM7+eq5nYoCyrfKg=', 'Content-Length': '0'} 2022-09-03T07:15:36.192 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c827b71e2e45a07-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.193 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:15:36.194 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:36.195 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:36.196 DEBUG:boto:Method: GET 2022-09-03T07:15:36.197 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:36.198 DEBUG:boto:Data: 2022-09-03T07:15:36.199 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.200 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.201 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.202 DEBUG:boto:Params: {} 2022-09-03T07:15:36.203 DEBUG:boto:Token: None 2022-09-03T07:15:36.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj33 2022-09-03T07:15:36.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SalcBQET1iBVoZNotJBq7Cyi/0Y= 2022-09-03T07:15:36.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SalcBQET1iBVoZNotJBq7Cyi/0Y=', 'Content-Length': '0'} 2022-09-03T07:15:36.208 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e0fd5a92aeb9b1c-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.250 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:36.252 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:36.253 DEBUG:boto:Method: GET 2022-09-03T07:15:36.255 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:36.256 DEBUG:boto:Data: 2022-09-03T07:15:36.257 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.258 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.259 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.259 DEBUG:boto:Params: {} 2022-09-03T07:15:36.260 DEBUG:boto:Token: None 2022-09-03T07:15:36.261 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj33 2022-09-03T07:15:36.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SalcBQET1iBVoZNotJBq7Cyi/0Y= 2022-09-03T07:15:36.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SalcBQET1iBVoZNotJBq7Cyi/0Y=', 'Content-Length': '0'} 2022-09-03T07:15:36.265 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009721675247897be9-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.266 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.267 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.267 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:36.268 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:36.268 DEBUG:boto:Method: GET 2022-09-03T07:15:36.269 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:36.269 DEBUG:boto:Data: 2022-09-03T07:15:36.270 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.270 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.271 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.271 DEBUG:boto:Params: {} 2022-09-03T07:15:36.272 DEBUG:boto:Token: None 2022-09-03T07:15:36.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:36.283 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd2705b8a5392848-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.288 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:36.289 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.289 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.290 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:36.290 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:36.291 DEBUG:boto:Method: GET 2022-09-03T07:15:36.292 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:36.293 DEBUG:boto:Data: 2022-09-03T07:15:36.295 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.295 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.296 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.297 DEBUG:boto:Params: {} 2022-09-03T07:15:36.297 DEBUG:boto:Token: None 2022-09-03T07:15:36.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:36.309 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c852f4215aead281-006312ff18-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.311 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:36.311 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:15:36.312 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.313 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.313 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.315 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.316 DEBUG:boto:Method: GET 2022-09-03T07:15:36.317 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.318 DEBUG:boto:Data: 2022-09-03T07:15:36.318 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.320 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.321 DEBUG:boto:Params: {} 2022-09-03T07:15:36.322 DEBUG:boto:Token: None 2022-09-03T07:15:36.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GtL+yCzP0kZEM3COhuHxX0Uiqtw= 2022-09-03T07:15:36.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GtL+yCzP0kZEM3COhuHxX0Uiqtw=', 'Content-Length': '0'} 2022-09-03T07:15:36.328 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081ea117e549ed938-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.335 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.336 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.340 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.341 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.342 DEBUG:boto:Method: GET 2022-09-03T07:15:36.343 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.346 DEBUG:boto:Data: 2022-09-03T07:15:36.347 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.349 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.360 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.361 DEBUG:boto:Params: {} 2022-09-03T07:15:36.362 DEBUG:boto:Token: None 2022-09-03T07:15:36.363 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GtL+yCzP0kZEM3COhuHxX0Uiqtw= 2022-09-03T07:15:36.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GtL+yCzP0kZEM3COhuHxX0Uiqtw=', 'Content-Length': '0'} 2022-09-03T07:15:36.368 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe70b4adb90d1b6c-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.373 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.374 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.375 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.376 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.377 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.378 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.379 DEBUG:boto:Data: 2022-09-03T07:15:36.380 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.380 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.381 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.382 DEBUG:boto:Params: {} 2022-09-03T07:15:36.383 DEBUG:boto:Token: None 2022-09-03T07:15:36.385 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lMMdYPSCPndVqmihZlydnHxKdr0= 2022-09-03T07:15:36.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lMMdYPSCPndVqmihZlydnHxKdr0=', 'Content-Length': '0'} 2022-09-03T07:15:36.393 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d52da3e8fa150781-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.394 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.395 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.396 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.397 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.397 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.398 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.399 DEBUG:boto:Data: 2022-09-03T07:15:36.400 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.401 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.402 DEBUG:boto:Params: {} 2022-09-03T07:15:36.403 DEBUG:boto:Token: None 2022-09-03T07:15:36.404 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lMMdYPSCPndVqmihZlydnHxKdr0= 2022-09-03T07:15:36.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lMMdYPSCPndVqmihZlydnHxKdr0=', 'Content-Length': '0'} 2022-09-03T07:15:36.419 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000894c9a34630fc90d-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.420 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:15:36.421 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.423 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.424 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.425 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.434 DEBUG:boto:Method: GET 2022-09-03T07:15:36.437 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.439 DEBUG:boto:Data: 2022-09-03T07:15:36.440 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.442 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.443 DEBUG:boto:Params: {} 2022-09-03T07:15:36.444 DEBUG:boto:Token: None 2022-09-03T07:15:36.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GtL+yCzP0kZEM3COhuHxX0Uiqtw= 2022-09-03T07:15:36.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GtL+yCzP0kZEM3COhuHxX0Uiqtw=', 'Content-Length': '0'} 2022-09-03T07:15:36.464 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006fca864773432214-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.466 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.467 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.468 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.468 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.469 DEBUG:boto:Method: GET 2022-09-03T07:15:36.474 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.475 DEBUG:boto:Data: 2022-09-03T07:15:36.477 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.484 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.485 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.486 DEBUG:boto:Params: {} 2022-09-03T07:15:36.498 DEBUG:boto:Token: None 2022-09-03T07:15:36.499 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:36.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GtL+yCzP0kZEM3COhuHxX0Uiqtw= 2022-09-03T07:15:36.501 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GtL+yCzP0kZEM3COhuHxX0Uiqtw=', 'Content-Length': '0'} 2022-09-03T07:15:36.513 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078cba950a2852890-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.515 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.516 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.517 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:36.518 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:36.519 DEBUG:boto:Method: GET 2022-09-03T07:15:36.521 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:36.522 DEBUG:boto:Data: 2022-09-03T07:15:36.532 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.533 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.534 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.535 DEBUG:boto:Params: {} 2022-09-03T07:15:36.536 DEBUG:boto:Token: None 2022-09-03T07:15:36.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:36.544 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004843ba860c07cd74-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.548 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:36.562 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.564 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.564 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:36.568 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:36.569 DEBUG:boto:Method: GET 2022-09-03T07:15:36.570 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:36.572 DEBUG:boto:Data: 2022-09-03T07:15:36.573 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.574 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.575 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.579 DEBUG:boto:Params: {} 2022-09-03T07:15:36.580 DEBUG:boto:Token: None 2022-09-03T07:15:36.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:36.589 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000080537363cc2d946e-006312ff18-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.591 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:36.592 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.593 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.594 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.595 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:36.596 DEBUG:boto:Data: 2022-09-03T07:15:36.597 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.598 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.598 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.600 DEBUG:boto:Params: {} 2022-09-03T07:15:36.601 DEBUG:boto:Token: None 2022-09-03T07:15:36.604 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34 2022-09-03T07:15:36.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UkdZOl6oJygTLb6oI0JjGf9pFtg= 2022-09-03T07:15:36.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UkdZOl6oJygTLb6oI0JjGf9pFtg=', 'Content-Length': '0'} 2022-09-03T07:15:36.619 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e432e19780297880-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.621 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.623 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.624 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.625 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:36.637 DEBUG:boto:Data: 2022-09-03T07:15:36.643 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.645 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.646 DEBUG:boto:Params: {} 2022-09-03T07:15:36.647 DEBUG:boto:Token: None 2022-09-03T07:15:36.648 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34 2022-09-03T07:15:36.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UkdZOl6oJygTLb6oI0JjGf9pFtg= 2022-09-03T07:15:36.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UkdZOl6oJygTLb6oI0JjGf9pFtg=', 'Content-Length': '0'} 2022-09-03T07:15:36.653 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001da69324cf72d7c2-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.654 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:15:36.655 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.656 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.657 DEBUG:boto:Method: GET 2022-09-03T07:15:36.659 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:36.660 DEBUG:boto:Data: 2022-09-03T07:15:36.661 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.663 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.664 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.665 DEBUG:boto:Params: {} 2022-09-03T07:15:36.667 DEBUG:boto:Token: None 2022-09-03T07:15:36.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34 2022-09-03T07:15:36.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2PNrbAZehIwinr99ba21XB9Wfz8= 2022-09-03T07:15:36.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2PNrbAZehIwinr99ba21XB9Wfz8=', 'Content-Length': '0'} 2022-09-03T07:15:36.683 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007df0e037aa5f419d-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.685 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:36.686 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:36.687 DEBUG:boto:Method: GET 2022-09-03T07:15:36.688 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:36.691 DEBUG:boto:Data: 2022-09-03T07:15:36.692 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.692 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.693 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.694 DEBUG:boto:Params: {} 2022-09-03T07:15:36.695 DEBUG:boto:Token: None 2022-09-03T07:15:36.695 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj34 2022-09-03T07:15:36.696 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2PNrbAZehIwinr99ba21XB9Wfz8= 2022-09-03T07:15:36.696 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2PNrbAZehIwinr99ba21XB9Wfz8=', 'Content-Length': '0'} 2022-09-03T07:15:36.701 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000528c9ddb516e4f38-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.702 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.703 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.704 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:36.704 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:36.705 DEBUG:boto:Method: GET 2022-09-03T07:15:36.705 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:36.706 DEBUG:boto:Data: 2022-09-03T07:15:36.707 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.708 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.708 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.710 DEBUG:boto:Params: {} 2022-09-03T07:15:36.712 DEBUG:boto:Token: None 2022-09-03T07:15:36.713 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.714 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.716 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:36.729 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aab666752f2a391d-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.730 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:36.731 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.731 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.732 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:36.733 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:36.733 DEBUG:boto:Method: GET 2022-09-03T07:15:36.734 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:36.735 DEBUG:boto:Data: 2022-09-03T07:15:36.735 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.736 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.737 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.737 DEBUG:boto:Params: {} 2022-09-03T07:15:36.739 DEBUG:boto:Token: None 2022-09-03T07:15:36.739 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:36.759 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023105c4781f3ea6e-006312ff18-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.760 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:36.760 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:15:36.761 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:36.762 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:36.763 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.763 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.764 DEBUG:boto:Method: GET 2022-09-03T07:15:36.764 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.765 DEBUG:boto:Data: 2022-09-03T07:15:36.766 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.767 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.768 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.769 DEBUG:boto:Params: {} 2022-09-03T07:15:36.770 DEBUG:boto:Token: None 2022-09-03T07:15:36.772 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+sfOdf5qmXGhEiHkZb1rPxD7NCQ= 2022-09-03T07:15:36.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+sfOdf5qmXGhEiHkZb1rPxD7NCQ=', 'Content-Length': '0'} 2022-09-03T07:15:36.778 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc101beadaf4c525-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.780 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:36.781 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:36.782 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.783 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.784 DEBUG:boto:Method: GET 2022-09-03T07:15:36.785 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.787 DEBUG:boto:Data: 2022-09-03T07:15:36.788 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.789 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.790 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.795 DEBUG:boto:Params: {} 2022-09-03T07:15:36.800 DEBUG:boto:Token: None 2022-09-03T07:15:36.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+sfOdf5qmXGhEiHkZb1rPxD7NCQ= 2022-09-03T07:15:36.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+sfOdf5qmXGhEiHkZb1rPxD7NCQ=', 'Content-Length': '0'} 2022-09-03T07:15:36.806 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dfc476b7b78def34-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.817 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:36.818 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:36.819 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.825 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.826 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.827 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.828 DEBUG:boto:Data: 2022-09-03T07:15:36.829 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.830 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.831 DEBUG:boto:Params: {} 2022-09-03T07:15:36.832 DEBUG:boto:Token: None 2022-09-03T07:15:36.833 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hnP5xDzJ6AWeFWPlN5/13LnKYKg= 2022-09-03T07:15:36.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hnP5xDzJ6AWeFWPlN5/13LnKYKg=', 'Content-Length': '0'} 2022-09-03T07:15:36.837 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c61a72ea4c765154-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.838 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:36.839 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:36.840 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.841 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.842 DEBUG:boto:Method: HEAD 2022-09-03T07:15:36.844 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.845 DEBUG:boto:Data: 2022-09-03T07:15:36.846 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.850 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.857 DEBUG:boto:Params: {} 2022-09-03T07:15:36.859 DEBUG:boto:Token: None 2022-09-03T07:15:36.860 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hnP5xDzJ6AWeFWPlN5/13LnKYKg= 2022-09-03T07:15:36.878 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hnP5xDzJ6AWeFWPlN5/13LnKYKg=', 'Content-Length': '0'} 2022-09-03T07:15:36.894 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008ca7a6ed72e424f8-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.895 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:15:36.896 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:36.896 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:36.897 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.899 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.900 DEBUG:boto:Method: GET 2022-09-03T07:15:36.900 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.901 DEBUG:boto:Data: 2022-09-03T07:15:36.901 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.904 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.904 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.905 DEBUG:boto:Params: {} 2022-09-03T07:15:36.907 DEBUG:boto:Token: None 2022-09-03T07:15:36.908 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+sfOdf5qmXGhEiHkZb1rPxD7NCQ= 2022-09-03T07:15:36.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+sfOdf5qmXGhEiHkZb1rPxD7NCQ=', 'Content-Length': '0'} 2022-09-03T07:15:36.919 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1c96e5697b37fc4-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.923 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:36.924 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:36.925 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.926 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.927 DEBUG:boto:Method: GET 2022-09-03T07:15:36.928 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.930 DEBUG:boto:Data: 2022-09-03T07:15:36.933 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.935 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.937 DEBUG:boto:Params: {} 2022-09-03T07:15:36.938 DEBUG:boto:Token: None 2022-09-03T07:15:36.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:36.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+sfOdf5qmXGhEiHkZb1rPxD7NCQ= 2022-09-03T07:15:36.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+sfOdf5qmXGhEiHkZb1rPxD7NCQ=', 'Content-Length': '0'} 2022-09-03T07:15:36.955 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc5666b0e789b4a9-006312ff18-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.956 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.958 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.959 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:36.960 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:36.961 DEBUG:boto:Method: GET 2022-09-03T07:15:36.962 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:36.962 DEBUG:boto:Data: 2022-09-03T07:15:36.963 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:36.968 DEBUG:boto:Port: 8001 2022-09-03T07:15:36.968 DEBUG:boto:Params: {} 2022-09-03T07:15:36.969 DEBUG:boto:Token: None 2022-09-03T07:15:36.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:36.977 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000096f0b436cdcea344-006312ff18-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:36 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:36.978 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:36.980 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:36.981 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:36.982 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:36.983 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:36.984 DEBUG:boto:Method: GET 2022-09-03T07:15:36.985 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:36.987 DEBUG:boto:Data: 2022-09-03T07:15:36.988 DEBUG:boto:Headers: {} 2022-09-03T07:15:36.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:36.990 DEBUG:boto:Port: 8000 2022-09-03T07:15:36.991 DEBUG:boto:Params: {} 2022-09-03T07:15:36.992 DEBUG:boto:Token: None 2022-09-03T07:15:36.994 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:36 GMT /avysmw-75/ 2022-09-03T07:15:36.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog= 2022-09-03T07:15:36.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:36 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:byLALFMCB5xOB31GYykkqhTawog=', 'Content-Length': '0'} 2022-09-03T07:15:37.008 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000b5794539837e7eb-006312ff19-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.009 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.011 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:37.012 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:37.013 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.013 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:37.014 DEBUG:boto:Data: 2022-09-03T07:15:37.015 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.016 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.017 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.018 DEBUG:boto:Params: {} 2022-09-03T07:15:37.019 DEBUG:boto:Token: None 2022-09-03T07:15:37.020 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj44 2022-09-03T07:15:37.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:93SyD9PIqZRQWGf3+9D1KqI8OBo= 2022-09-03T07:15:37.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:93SyD9PIqZRQWGf3+9D1KqI8OBo=', 'Content-Length': '0'} 2022-09-03T07:15:37.026 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037df2c90da51d185-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.028 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:37.029 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:37.030 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.031 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:37.032 DEBUG:boto:Data: 2022-09-03T07:15:37.033 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.034 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.035 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.036 DEBUG:boto:Params: {} 2022-09-03T07:15:37.037 DEBUG:boto:Token: None 2022-09-03T07:15:37.039 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj44 2022-09-03T07:15:37.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:93SyD9PIqZRQWGf3+9D1KqI8OBo= 2022-09-03T07:15:37.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:93SyD9PIqZRQWGf3+9D1KqI8OBo=', 'Content-Length': '0'} 2022-09-03T07:15:37.044 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e928bd033e93150-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.045 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:15:37.046 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:37.047 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:37.049 DEBUG:boto:Method: GET 2022-09-03T07:15:37.050 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:37.051 DEBUG:boto:Data: 2022-09-03T07:15:37.052 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.053 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.053 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.054 DEBUG:boto:Params: {} 2022-09-03T07:15:37.056 DEBUG:boto:Token: None 2022-09-03T07:15:37.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj44 2022-09-03T07:15:37.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lmuXjjJmJGQFrgsazmOm+YKPJgA= 2022-09-03T07:15:37.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lmuXjjJmJGQFrgsazmOm+YKPJgA=', 'Content-Length': '0'} 2022-09-03T07:15:37.067 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f696e8698090fa11-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.070 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:37.071 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:37.072 DEBUG:boto:Method: GET 2022-09-03T07:15:37.072 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:37.073 DEBUG:boto:Data: 2022-09-03T07:15:37.074 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.076 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.077 DEBUG:boto:Params: {} 2022-09-03T07:15:37.078 DEBUG:boto:Token: None 2022-09-03T07:15:37.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj44 2022-09-03T07:15:37.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lmuXjjJmJGQFrgsazmOm+YKPJgA= 2022-09-03T07:15:37.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lmuXjjJmJGQFrgsazmOm+YKPJgA=', 'Content-Length': '0'} 2022-09-03T07:15:37.084 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0f803d011a74cd6-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.086 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.087 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.088 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:37.094 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:37.095 DEBUG:boto:Method: GET 2022-09-03T07:15:37.096 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:37.097 DEBUG:boto:Data: 2022-09-03T07:15:37.098 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.100 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.101 DEBUG:boto:Params: {} 2022-09-03T07:15:37.101 DEBUG:boto:Token: None 2022-09-03T07:15:37.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.109 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009efbc6f04b9701a3-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.110 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.111 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.111 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.112 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:37.112 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:37.113 DEBUG:boto:Method: GET 2022-09-03T07:15:37.114 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:37.115 DEBUG:boto:Data: 2022-09-03T07:15:37.116 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.118 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.119 DEBUG:boto:Params: {} 2022-09-03T07:15:37.120 DEBUG:boto:Token: None 2022-09-03T07:15:37.122 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.123 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.124 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007dfa3d7df7cf09e2-006312ff19-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.135 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.136 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:15:37.136 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.137 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.138 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.138 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.139 DEBUG:boto:Method: GET 2022-09-03T07:15:37.139 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.140 DEBUG:boto:Data: 2022-09-03T07:15:37.140 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.141 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.142 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.142 DEBUG:boto:Params: {} 2022-09-03T07:15:37.143 DEBUG:boto:Token: None 2022-09-03T07:15:37.144 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6XCPazWqJaeE8MG7wY9CRqDCDCk= 2022-09-03T07:15:37.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6XCPazWqJaeE8MG7wY9CRqDCDCk=', 'Content-Length': '0'} 2022-09-03T07:15:37.147 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f2e6925033917cc-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.148 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.149 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.149 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.150 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.151 DEBUG:boto:Method: GET 2022-09-03T07:15:37.151 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.152 DEBUG:boto:Data: 2022-09-03T07:15:37.152 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.153 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.153 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.154 DEBUG:boto:Params: {} 2022-09-03T07:15:37.154 DEBUG:boto:Token: None 2022-09-03T07:15:37.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6XCPazWqJaeE8MG7wY9CRqDCDCk= 2022-09-03T07:15:37.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6XCPazWqJaeE8MG7wY9CRqDCDCk=', 'Content-Length': '0'} 2022-09-03T07:15:37.167 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d0d0eeab468a26c1-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.168 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.169 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.171 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.172 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.173 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.174 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.175 DEBUG:boto:Data: 2022-09-03T07:15:37.176 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.177 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.177 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.178 DEBUG:boto:Params: {} 2022-09-03T07:15:37.179 DEBUG:boto:Token: None 2022-09-03T07:15:37.181 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.183 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G1jNkhiMpW+sRAlxl4mhHkJU6zo= 2022-09-03T07:15:37.184 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G1jNkhiMpW+sRAlxl4mhHkJU6zo=', 'Content-Length': '0'} 2022-09-03T07:15:37.194 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a8218d9dc547282-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.195 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.202 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.203 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.204 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.204 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.205 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.205 DEBUG:boto:Data: 2022-09-03T07:15:37.206 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.207 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.208 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.208 DEBUG:boto:Params: {} 2022-09-03T07:15:37.209 DEBUG:boto:Token: None 2022-09-03T07:15:37.209 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G1jNkhiMpW+sRAlxl4mhHkJU6zo= 2022-09-03T07:15:37.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G1jNkhiMpW+sRAlxl4mhHkJU6zo=', 'Content-Length': '0'} 2022-09-03T07:15:37.214 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2e9d96ac1dd0c94-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:15:37.217 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.227 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.229 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.229 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.230 DEBUG:boto:Method: GET 2022-09-03T07:15:37.231 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.232 DEBUG:boto:Data: 2022-09-03T07:15:37.233 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.236 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.237 DEBUG:boto:Params: {} 2022-09-03T07:15:37.241 DEBUG:boto:Token: None 2022-09-03T07:15:37.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6XCPazWqJaeE8MG7wY9CRqDCDCk= 2022-09-03T07:15:37.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6XCPazWqJaeE8MG7wY9CRqDCDCk=', 'Content-Length': '0'} 2022-09-03T07:15:37.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be6ee026c8c390cc-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.256 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.257 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.258 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.259 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.260 DEBUG:boto:Method: GET 2022-09-03T07:15:37.261 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.262 DEBUG:boto:Data: 2022-09-03T07:15:37.263 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.269 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.270 DEBUG:boto:Params: {} 2022-09-03T07:15:37.272 DEBUG:boto:Token: None 2022-09-03T07:15:37.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:37.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6XCPazWqJaeE8MG7wY9CRqDCDCk= 2022-09-03T07:15:37.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6XCPazWqJaeE8MG7wY9CRqDCDCk=', 'Content-Length': '0'} 2022-09-03T07:15:37.291 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f817dbacec52d79f-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.293 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.294 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.295 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:37.296 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:37.306 DEBUG:boto:Method: GET 2022-09-03T07:15:37.307 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:37.308 DEBUG:boto:Data: 2022-09-03T07:15:37.309 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.310 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.311 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.313 DEBUG:boto:Params: {} 2022-09-03T07:15:37.314 DEBUG:boto:Token: None 2022-09-03T07:15:37.315 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.325 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045a400e90de0cc61-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.326 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.328 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.329 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.330 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:37.331 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:37.332 DEBUG:boto:Method: GET 2022-09-03T07:15:37.333 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:37.334 DEBUG:boto:Data: 2022-09-03T07:15:37.335 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.337 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.339 DEBUG:boto:Params: {} 2022-09-03T07:15:37.340 DEBUG:boto:Token: None 2022-09-03T07:15:37.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.354 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001fe4f19a5fd04886-006312ff19-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.355 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.356 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.357 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.357 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.359 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:37.359 DEBUG:boto:Data: 2022-09-03T07:15:37.360 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.361 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.362 DEBUG:boto:Params: {} 2022-09-03T07:15:37.362 DEBUG:boto:Token: None 2022-09-03T07:15:37.363 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51 2022-09-03T07:15:37.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3y7G9H+pPtg/1isxhx5Kg4FDnPA= 2022-09-03T07:15:37.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3y7G9H+pPtg/1isxhx5Kg4FDnPA=', 'Content-Length': '0'} 2022-09-03T07:15:37.372 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049cf9ddd93cd4cd5-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.373 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.373 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.374 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.375 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:37.375 DEBUG:boto:Data: 2022-09-03T07:15:37.376 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.376 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.378 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.378 DEBUG:boto:Params: {} 2022-09-03T07:15:37.379 DEBUG:boto:Token: None 2022-09-03T07:15:37.379 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51 2022-09-03T07:15:37.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3y7G9H+pPtg/1isxhx5Kg4FDnPA= 2022-09-03T07:15:37.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3y7G9H+pPtg/1isxhx5Kg4FDnPA=', 'Content-Length': '0'} 2022-09-03T07:15:37.383 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000427c2435be6b22c1-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.384 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:15:37.385 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.385 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.386 DEBUG:boto:Method: GET 2022-09-03T07:15:37.386 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:37.387 DEBUG:boto:Data: 2022-09-03T07:15:37.387 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.389 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.390 DEBUG:boto:Params: {} 2022-09-03T07:15:37.390 DEBUG:boto:Token: None 2022-09-03T07:15:37.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51 2022-09-03T07:15:37.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qmc8lFRUCPWnQwRd4TY+IlEFrUw= 2022-09-03T07:15:37.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qmc8lFRUCPWnQwRd4TY+IlEFrUw=', 'Content-Length': '0'} 2022-09-03T07:15:37.396 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aefd74bd480377f6-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.397 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:37.397 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:37.405 DEBUG:boto:Method: GET 2022-09-03T07:15:37.418 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:37.419 DEBUG:boto:Data: 2022-09-03T07:15:37.420 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.420 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.421 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.422 DEBUG:boto:Params: {} 2022-09-03T07:15:37.424 DEBUG:boto:Token: None 2022-09-03T07:15:37.425 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj51 2022-09-03T07:15:37.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qmc8lFRUCPWnQwRd4TY+IlEFrUw= 2022-09-03T07:15:37.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qmc8lFRUCPWnQwRd4TY+IlEFrUw=', 'Content-Length': '0'} 2022-09-03T07:15:37.439 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5121b25fb240816-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.440 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.441 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.441 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:37.442 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:37.443 DEBUG:boto:Method: GET 2022-09-03T07:15:37.443 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:37.444 DEBUG:boto:Data: 2022-09-03T07:15:37.445 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.446 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.447 DEBUG:boto:Params: {} 2022-09-03T07:15:37.448 DEBUG:boto:Token: None 2022-09-03T07:15:37.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.457 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069537a4829b5e517-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.458 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.459 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.459 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.462 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:37.463 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:37.464 DEBUG:boto:Method: GET 2022-09-03T07:15:37.465 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:37.467 DEBUG:boto:Data: 2022-09-03T07:15:37.468 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.470 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.471 DEBUG:boto:Params: {} 2022-09-03T07:15:37.472 DEBUG:boto:Token: None 2022-09-03T07:15:37.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.479 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069a1eb8ee459f8ee-006312ff19-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.480 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.482 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:15:37.483 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.484 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.485 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.488 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.489 DEBUG:boto:Method: GET 2022-09-03T07:15:37.490 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.491 DEBUG:boto:Data: 2022-09-03T07:15:37.492 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.495 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.500 DEBUG:boto:Params: {} 2022-09-03T07:15:37.502 DEBUG:boto:Token: None 2022-09-03T07:15:37.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fWAXHGSgvk9c9Aaif4jdgFGLvXw= 2022-09-03T07:15:37.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fWAXHGSgvk9c9Aaif4jdgFGLvXw=', 'Content-Length': '0'} 2022-09-03T07:15:37.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c550110401dd5209-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.512 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.513 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.514 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.514 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.515 DEBUG:boto:Method: GET 2022-09-03T07:15:37.515 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.516 DEBUG:boto:Data: 2022-09-03T07:15:37.516 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.517 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.518 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.518 DEBUG:boto:Params: {} 2022-09-03T07:15:37.519 DEBUG:boto:Token: None 2022-09-03T07:15:37.520 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fWAXHGSgvk9c9Aaif4jdgFGLvXw= 2022-09-03T07:15:37.521 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fWAXHGSgvk9c9Aaif4jdgFGLvXw=', 'Content-Length': '0'} 2022-09-03T07:15:37.531 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007307731d1010db9d-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.532 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.533 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.533 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.534 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.535 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.538 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.539 DEBUG:boto:Data: 2022-09-03T07:15:37.544 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.544 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.545 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.546 DEBUG:boto:Params: {} 2022-09-03T07:15:37.546 DEBUG:boto:Token: None 2022-09-03T07:15:37.547 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4lu2r2EIIBknEz5GsSAZiFJvM1A= 2022-09-03T07:15:37.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4lu2r2EIIBknEz5GsSAZiFJvM1A=', 'Content-Length': '0'} 2022-09-03T07:15:37.550 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9882c3fd0055c97-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.552 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.553 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.553 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.554 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.554 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.556 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.557 DEBUG:boto:Data: 2022-09-03T07:15:37.558 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.559 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.560 DEBUG:boto:Params: {} 2022-09-03T07:15:37.561 DEBUG:boto:Token: None 2022-09-03T07:15:37.562 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.563 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4lu2r2EIIBknEz5GsSAZiFJvM1A= 2022-09-03T07:15:37.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4lu2r2EIIBknEz5GsSAZiFJvM1A=', 'Content-Length': '0'} 2022-09-03T07:15:37.566 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004007ae2470560aad-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.567 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:15:37.567 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.568 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.568 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.569 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.569 DEBUG:boto:Method: GET 2022-09-03T07:15:37.570 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.570 DEBUG:boto:Data: 2022-09-03T07:15:37.571 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.572 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.572 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.573 DEBUG:boto:Params: {} 2022-09-03T07:15:37.573 DEBUG:boto:Token: None 2022-09-03T07:15:37.574 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.575 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fWAXHGSgvk9c9Aaif4jdgFGLvXw= 2022-09-03T07:15:37.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fWAXHGSgvk9c9Aaif4jdgFGLvXw=', 'Content-Length': '0'} 2022-09-03T07:15:37.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000adafd84430820f00-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.622 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.623 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.623 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.624 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.624 DEBUG:boto:Method: GET 2022-09-03T07:15:37.625 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.625 DEBUG:boto:Data: 2022-09-03T07:15:37.626 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.627 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.628 DEBUG:boto:Params: {} 2022-09-03T07:15:37.628 DEBUG:boto:Token: None 2022-09-03T07:15:37.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:37.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fWAXHGSgvk9c9Aaif4jdgFGLvXw= 2022-09-03T07:15:37.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fWAXHGSgvk9c9Aaif4jdgFGLvXw=', 'Content-Length': '0'} 2022-09-03T07:15:37.641 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d4f39bec373882c9-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.642 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.643 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.644 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:37.644 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:37.645 DEBUG:boto:Method: GET 2022-09-03T07:15:37.646 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:37.646 DEBUG:boto:Data: 2022-09-03T07:15:37.647 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.648 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.649 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.649 DEBUG:boto:Params: {} 2022-09-03T07:15:37.650 DEBUG:boto:Token: None 2022-09-03T07:15:37.651 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.655 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000086ac93813d5ffbf8-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.656 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.657 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.658 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.658 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:37.659 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:37.659 DEBUG:boto:Method: GET 2022-09-03T07:15:37.660 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:37.660 DEBUG:boto:Data: 2022-09-03T07:15:37.661 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.681 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.687 DEBUG:boto:Params: {} 2022-09-03T07:15:37.688 DEBUG:boto:Token: None 2022-09-03T07:15:37.688 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.693 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4d6525b521b1592-006312ff19-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.694 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.695 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.695 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.696 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.697 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:37.702 DEBUG:boto:Data: 2022-09-03T07:15:37.703 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.705 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.706 DEBUG:boto:Params: {} 2022-09-03T07:15:37.707 DEBUG:boto:Token: None 2022-09-03T07:15:37.708 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55 2022-09-03T07:15:37.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ykCjpQTmLDG6wQnZEeW6sXQit3s= 2022-09-03T07:15:37.711 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ykCjpQTmLDG6wQnZEeW6sXQit3s=', 'Content-Length': '0'} 2022-09-03T07:15:37.723 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009215a451cd59d717-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.725 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.726 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.727 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.728 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:37.730 DEBUG:boto:Data: 2022-09-03T07:15:37.731 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.733 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.734 DEBUG:boto:Params: {} 2022-09-03T07:15:37.735 DEBUG:boto:Token: None 2022-09-03T07:15:37.736 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55 2022-09-03T07:15:37.738 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ykCjpQTmLDG6wQnZEeW6sXQit3s= 2022-09-03T07:15:37.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ykCjpQTmLDG6wQnZEeW6sXQit3s=', 'Content-Length': '0'} 2022-09-03T07:15:37.746 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001bfd71c4c87b104e-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.748 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:15:37.750 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.752 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.753 DEBUG:boto:Method: GET 2022-09-03T07:15:37.754 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:37.756 DEBUG:boto:Data: 2022-09-03T07:15:37.758 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.761 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.763 DEBUG:boto:Params: {} 2022-09-03T07:15:37.763 DEBUG:boto:Token: None 2022-09-03T07:15:37.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55 2022-09-03T07:15:37.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O4xHXy93FrIkt4OpBuomlHMWugk= 2022-09-03T07:15:37.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O4xHXy93FrIkt4OpBuomlHMWugk=', 'Content-Length': '0'} 2022-09-03T07:15:37.768 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000415fcd436bbbe804-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.770 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:37.771 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:37.772 DEBUG:boto:Method: GET 2022-09-03T07:15:37.773 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:37.775 DEBUG:boto:Data: 2022-09-03T07:15:37.775 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.776 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.777 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.778 DEBUG:boto:Params: {} 2022-09-03T07:15:37.779 DEBUG:boto:Token: None 2022-09-03T07:15:37.779 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj55 2022-09-03T07:15:37.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O4xHXy93FrIkt4OpBuomlHMWugk= 2022-09-03T07:15:37.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O4xHXy93FrIkt4OpBuomlHMWugk=', 'Content-Length': '0'} 2022-09-03T07:15:37.785 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042a5550c5e51046c-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.786 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.787 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.788 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:37.789 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:37.790 DEBUG:boto:Method: GET 2022-09-03T07:15:37.791 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:37.791 DEBUG:boto:Data: 2022-09-03T07:15:37.792 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.792 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.793 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.794 DEBUG:boto:Params: {} 2022-09-03T07:15:37.794 DEBUG:boto:Token: None 2022-09-03T07:15:37.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.803 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c0ca7855561d9dcf-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.805 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.808 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.809 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.810 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:37.811 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:37.812 DEBUG:boto:Method: GET 2022-09-03T07:15:37.814 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:37.815 DEBUG:boto:Data: 2022-09-03T07:15:37.816 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.818 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.819 DEBUG:boto:Params: {} 2022-09-03T07:15:37.821 DEBUG:boto:Token: None 2022-09-03T07:15:37.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.830 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002bd60760b1a533a7-006312ff19-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.833 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.835 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:15:37.838 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:37.840 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:37.841 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.841 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.842 DEBUG:boto:Method: GET 2022-09-03T07:15:37.843 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.846 DEBUG:boto:Data: 2022-09-03T07:15:37.847 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.848 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.849 DEBUG:boto:Params: {} 2022-09-03T07:15:37.850 DEBUG:boto:Token: None 2022-09-03T07:15:37.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FnCBTSqXbrcy/qzOvIB/o+yBczc= 2022-09-03T07:15:37.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FnCBTSqXbrcy/qzOvIB/o+yBczc=', 'Content-Length': '0'} 2022-09-03T07:15:37.861 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091239f24aebe0e77-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.863 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:37.864 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:37.865 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.866 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.866 DEBUG:boto:Method: GET 2022-09-03T07:15:37.867 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.867 DEBUG:boto:Data: 2022-09-03T07:15:37.868 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.868 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.869 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.870 DEBUG:boto:Params: {} 2022-09-03T07:15:37.871 DEBUG:boto:Token: None 2022-09-03T07:15:37.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FnCBTSqXbrcy/qzOvIB/o+yBczc= 2022-09-03T07:15:37.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FnCBTSqXbrcy/qzOvIB/o+yBczc=', 'Content-Length': '0'} 2022-09-03T07:15:37.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037a3d9ba6dc81bc7-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.878 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:37.878 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:37.879 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.880 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.881 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.882 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.882 DEBUG:boto:Data: 2022-09-03T07:15:37.883 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.884 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.885 DEBUG:boto:Params: {} 2022-09-03T07:15:37.885 DEBUG:boto:Token: None 2022-09-03T07:15:37.886 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DypblezTy9G6AGYAgO6GmjCzNYQ= 2022-09-03T07:15:37.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DypblezTy9G6AGYAgO6GmjCzNYQ=', 'Content-Length': '0'} 2022-09-03T07:15:37.890 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015dd8971af5f6908-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.894 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:37.895 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:37.896 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.897 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.898 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.899 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.900 DEBUG:boto:Data: 2022-09-03T07:15:37.900 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.901 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.902 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.903 DEBUG:boto:Params: {} 2022-09-03T07:15:37.904 DEBUG:boto:Token: None 2022-09-03T07:15:37.905 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DypblezTy9G6AGYAgO6GmjCzNYQ= 2022-09-03T07:15:37.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DypblezTy9G6AGYAgO6GmjCzNYQ=', 'Content-Length': '0'} 2022-09-03T07:15:37.910 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ebe22a06eae57f1d-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.911 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:15:37.913 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:37.914 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:37.915 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.916 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.917 DEBUG:boto:Method: GET 2022-09-03T07:15:37.918 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.918 DEBUG:boto:Data: 2022-09-03T07:15:37.919 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.921 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.922 DEBUG:boto:Params: {} 2022-09-03T07:15:37.923 DEBUG:boto:Token: None 2022-09-03T07:15:37.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FnCBTSqXbrcy/qzOvIB/o+yBczc= 2022-09-03T07:15:37.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FnCBTSqXbrcy/qzOvIB/o+yBczc=', 'Content-Length': '0'} 2022-09-03T07:15:37.929 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6417931eff50ab6-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.930 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:37.931 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:37.931 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.932 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.932 DEBUG:boto:Method: GET 2022-09-03T07:15:37.933 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.934 DEBUG:boto:Data: 2022-09-03T07:15:37.935 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.935 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.936 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.936 DEBUG:boto:Params: {} 2022-09-03T07:15:37.937 DEBUG:boto:Token: None 2022-09-03T07:15:37.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:37.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FnCBTSqXbrcy/qzOvIB/o+yBczc= 2022-09-03T07:15:37.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FnCBTSqXbrcy/qzOvIB/o+yBczc=', 'Content-Length': '0'} 2022-09-03T07:15:37.945 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e38a6d541bf951d0-006312ff19-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.946 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.946 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.947 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:37.947 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:37.948 DEBUG:boto:Method: GET 2022-09-03T07:15:37.948 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:37.949 DEBUG:boto:Data: 2022-09-03T07:15:37.950 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.951 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.952 DEBUG:boto:Params: {} 2022-09-03T07:15:37.952 DEBUG:boto:Token: None 2022-09-03T07:15:37.953 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.958 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c5dbe0fdb5b41d37-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.959 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.961 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:37.962 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:37.963 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:37.964 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:37.965 DEBUG:boto:Method: GET 2022-09-03T07:15:37.966 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:37.967 DEBUG:boto:Data: 2022-09-03T07:15:37.968 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.970 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:37.970 DEBUG:boto:Port: 8000 2022-09-03T07:15:37.973 DEBUG:boto:Params: {} 2022-09-03T07:15:37.974 DEBUG:boto:Token: None 2022-09-03T07:15:37.975 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/ 2022-09-03T07:15:37.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU= 2022-09-03T07:15:37.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xzCMYUSf4LrD8IN1FwFszAXOQUU=', 'Content-Length': '0'} 2022-09-03T07:15:37.981 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036def3cef336226a-006312ff19-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.982 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:37.983 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:37.984 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:37.984 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.985 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:37.986 DEBUG:boto:Data: 2022-09-03T07:15:37.986 DEBUG:boto:Headers: {} 2022-09-03T07:15:37.987 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:37.987 DEBUG:boto:Port: 8001 2022-09-03T07:15:37.988 DEBUG:boto:Params: {} 2022-09-03T07:15:37.988 DEBUG:boto:Token: None 2022-09-03T07:15:37.989 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:37 GMT /avysmw-75/obj68 2022-09-03T07:15:37.990 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqYbiuxoErHEb76N4hVjaqif5yk= 2022-09-03T07:15:37.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:37 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqYbiuxoErHEb76N4hVjaqif5yk=', 'Content-Length': '0'} 2022-09-03T07:15:37.994 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d044577bac88e250-006312ff19-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:37 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:37.995 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:37.996 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:37.998 DEBUG:boto:Method: HEAD 2022-09-03T07:15:37.999 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:37.999 DEBUG:boto:Data: 2022-09-03T07:15:38.000 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.001 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.002 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.003 DEBUG:boto:Params: {} 2022-09-03T07:15:38.004 DEBUG:boto:Token: None 2022-09-03T07:15:38.004 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj68 2022-09-03T07:15:38.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a+pvZYsprV07sUPF16mpzTcWZQA= 2022-09-03T07:15:38.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a+pvZYsprV07sUPF16mpzTcWZQA=', 'Content-Length': '0'} 2022-09-03T07:15:38.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d17f256e1eeb832c-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.012 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:15:38.013 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:38.014 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:38.015 DEBUG:boto:Method: GET 2022-09-03T07:15:38.017 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:38.018 DEBUG:boto:Data: 2022-09-03T07:15:38.019 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.021 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.022 DEBUG:boto:Params: {} 2022-09-03T07:15:38.023 DEBUG:boto:Token: None 2022-09-03T07:15:38.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj68 2022-09-03T07:15:38.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieT34Q2lYAYKj9N6ybIEeFmGLXw= 2022-09-03T07:15:38.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieT34Q2lYAYKj9N6ybIEeFmGLXw=', 'Content-Length': '0'} 2022-09-03T07:15:38.030 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006c809d002d861989-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.071 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:38.074 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:38.075 DEBUG:boto:Method: GET 2022-09-03T07:15:38.076 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:38.076 DEBUG:boto:Data: 2022-09-03T07:15:38.077 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.078 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.079 DEBUG:boto:Params: {} 2022-09-03T07:15:38.079 DEBUG:boto:Token: None 2022-09-03T07:15:38.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj68 2022-09-03T07:15:38.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieT34Q2lYAYKj9N6ybIEeFmGLXw= 2022-09-03T07:15:38.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieT34Q2lYAYKj9N6ybIEeFmGLXw=', 'Content-Length': '0'} 2022-09-03T07:15:38.084 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000451d581f7e94e518-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.085 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.085 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.086 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:38.086 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:38.088 DEBUG:boto:Method: GET 2022-09-03T07:15:38.088 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:38.090 DEBUG:boto:Data: 2022-09-03T07:15:38.091 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.091 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.092 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.092 DEBUG:boto:Params: {} 2022-09-03T07:15:38.093 DEBUG:boto:Token: None 2022-09-03T07:15:38.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.098 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005a1510dad1b8534d-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.099 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.100 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.101 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.101 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:38.102 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:38.102 DEBUG:boto:Method: GET 2022-09-03T07:15:38.103 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:38.103 DEBUG:boto:Data: 2022-09-03T07:15:38.104 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.105 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.106 DEBUG:boto:Params: {} 2022-09-03T07:15:38.106 DEBUG:boto:Token: None 2022-09-03T07:15:38.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.112 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000056cfeaf67d576e65-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.113 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.114 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:15:38.115 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.115 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.116 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.117 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.117 DEBUG:boto:Method: GET 2022-09-03T07:15:38.118 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.119 DEBUG:boto:Data: 2022-09-03T07:15:38.119 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.120 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.120 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.121 DEBUG:boto:Params: {} 2022-09-03T07:15:38.122 DEBUG:boto:Token: None 2022-09-03T07:15:38.122 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.123 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tz2jWI5aQzw/oid6FL9EbhnorVs= 2022-09-03T07:15:38.124 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tz2jWI5aQzw/oid6FL9EbhnorVs=', 'Content-Length': '0'} 2022-09-03T07:15:38.126 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5fbdc025c665c9d-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.127 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.127 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.128 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.129 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.129 DEBUG:boto:Method: GET 2022-09-03T07:15:38.130 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.130 DEBUG:boto:Data: 2022-09-03T07:15:38.131 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.132 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.134 DEBUG:boto:Params: {} 2022-09-03T07:15:38.135 DEBUG:boto:Token: None 2022-09-03T07:15:38.136 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tz2jWI5aQzw/oid6FL9EbhnorVs= 2022-09-03T07:15:38.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tz2jWI5aQzw/oid6FL9EbhnorVs=', 'Content-Length': '0'} 2022-09-03T07:15:38.141 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c524d19f53ec4faa-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.143 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.144 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.145 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.146 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.147 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.148 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.148 DEBUG:boto:Data: 2022-09-03T07:15:38.149 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.149 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.150 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.151 DEBUG:boto:Params: {} 2022-09-03T07:15:38.151 DEBUG:boto:Token: None 2022-09-03T07:15:38.152 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uTJZDrGxcNQR1QY7O91LulmAvVk= 2022-09-03T07:15:38.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uTJZDrGxcNQR1QY7O91LulmAvVk=', 'Content-Length': '0'} 2022-09-03T07:15:38.157 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000574d7ee8645908da-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.159 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.160 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.161 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.162 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.164 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.165 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.166 DEBUG:boto:Data: 2022-09-03T07:15:38.167 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.169 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.170 DEBUG:boto:Params: {} 2022-09-03T07:15:38.171 DEBUG:boto:Token: None 2022-09-03T07:15:38.172 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uTJZDrGxcNQR1QY7O91LulmAvVk= 2022-09-03T07:15:38.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uTJZDrGxcNQR1QY7O91LulmAvVk=', 'Content-Length': '0'} 2022-09-03T07:15:38.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005bb7f1b3a3dc5eb5-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.179 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:15:38.181 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.182 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.183 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.183 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.184 DEBUG:boto:Method: GET 2022-09-03T07:15:38.185 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.186 DEBUG:boto:Data: 2022-09-03T07:15:38.187 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.188 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.189 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.190 DEBUG:boto:Params: {} 2022-09-03T07:15:38.191 DEBUG:boto:Token: None 2022-09-03T07:15:38.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tz2jWI5aQzw/oid6FL9EbhnorVs= 2022-09-03T07:15:38.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tz2jWI5aQzw/oid6FL9EbhnorVs=', 'Content-Length': '0'} 2022-09-03T07:15:38.197 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000069b4677ddf1cd77-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.199 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.200 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.201 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.202 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.203 DEBUG:boto:Method: GET 2022-09-03T07:15:38.204 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.205 DEBUG:boto:Data: 2022-09-03T07:15:38.206 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.206 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.207 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.207 DEBUG:boto:Params: {} 2022-09-03T07:15:38.208 DEBUG:boto:Token: None 2022-09-03T07:15:38.209 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:38.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tz2jWI5aQzw/oid6FL9EbhnorVs= 2022-09-03T07:15:38.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tz2jWI5aQzw/oid6FL9EbhnorVs=', 'Content-Length': '0'} 2022-09-03T07:15:38.214 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f2c6498ba50c772-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.215 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.216 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.216 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:38.217 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:38.217 DEBUG:boto:Method: GET 2022-09-03T07:15:38.218 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:38.218 DEBUG:boto:Data: 2022-09-03T07:15:38.219 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.220 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.221 DEBUG:boto:Params: {} 2022-09-03T07:15:38.221 DEBUG:boto:Token: None 2022-09-03T07:15:38.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.226 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005aa212652244a567-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.227 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.228 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.228 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.229 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:38.229 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:38.230 DEBUG:boto:Method: GET 2022-09-03T07:15:38.230 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:38.231 DEBUG:boto:Data: 2022-09-03T07:15:38.231 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.232 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.232 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.233 DEBUG:boto:Params: {} 2022-09-03T07:15:38.234 DEBUG:boto:Token: None 2022-09-03T07:15:38.234 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.235 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.236 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.239 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000014fffa058eef6d47-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.282 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.284 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.285 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.286 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.287 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:38.288 DEBUG:boto:Data: 2022-09-03T07:15:38.288 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.290 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.292 DEBUG:boto:Params: {} 2022-09-03T07:15:38.293 DEBUG:boto:Token: None 2022-09-03T07:15:38.294 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85 2022-09-03T07:15:38.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PfDEHhV8v3jXUkdySoF3pbCD/Mg= 2022-09-03T07:15:38.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PfDEHhV8v3jXUkdySoF3pbCD/Mg=', 'Content-Length': '0'} 2022-09-03T07:15:38.298 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c65c8884b0a2fbc-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.300 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.301 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.301 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.303 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:38.304 DEBUG:boto:Data: 2022-09-03T07:15:38.304 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.306 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.307 DEBUG:boto:Params: {} 2022-09-03T07:15:38.308 DEBUG:boto:Token: None 2022-09-03T07:15:38.309 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85 2022-09-03T07:15:38.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PfDEHhV8v3jXUkdySoF3pbCD/Mg= 2022-09-03T07:15:38.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PfDEHhV8v3jXUkdySoF3pbCD/Mg=', 'Content-Length': '0'} 2022-09-03T07:15:38.314 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b6dc28a18477952-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.315 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:15:38.316 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.317 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.317 DEBUG:boto:Method: GET 2022-09-03T07:15:38.318 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:38.319 DEBUG:boto:Data: 2022-09-03T07:15:38.320 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.322 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.323 DEBUG:boto:Params: {} 2022-09-03T07:15:38.324 DEBUG:boto:Token: None 2022-09-03T07:15:38.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85 2022-09-03T07:15:38.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/jLmvh7nWu1C/8vaL4NgyBXCoho= 2022-09-03T07:15:38.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/jLmvh7nWu1C/8vaL4NgyBXCoho=', 'Content-Length': '0'} 2022-09-03T07:15:38.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3ce458c473a69dc-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.378 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:38.379 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:38.379 DEBUG:boto:Method: GET 2022-09-03T07:15:38.380 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:38.381 DEBUG:boto:Data: 2022-09-03T07:15:38.382 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.382 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.383 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.383 DEBUG:boto:Params: {} 2022-09-03T07:15:38.384 DEBUG:boto:Token: None 2022-09-03T07:15:38.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj85 2022-09-03T07:15:38.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/jLmvh7nWu1C/8vaL4NgyBXCoho= 2022-09-03T07:15:38.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/jLmvh7nWu1C/8vaL4NgyBXCoho=', 'Content-Length': '0'} 2022-09-03T07:15:38.388 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000744a56bca7e14783-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.389 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.390 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.390 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:38.391 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:38.392 DEBUG:boto:Method: GET 2022-09-03T07:15:38.392 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:38.393 DEBUG:boto:Data: 2022-09-03T07:15:38.393 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.394 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.395 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.396 DEBUG:boto:Params: {} 2022-09-03T07:15:38.396 DEBUG:boto:Token: None 2022-09-03T07:15:38.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.402 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000970346edd440b13d-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.403 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.404 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.405 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.405 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:38.406 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:38.407 DEBUG:boto:Method: GET 2022-09-03T07:15:38.407 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:38.408 DEBUG:boto:Data: 2022-09-03T07:15:38.409 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.409 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:38.410 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.411 DEBUG:boto:Params: {} 2022-09-03T07:15:38.411 DEBUG:boto:Token: None 2022-09-03T07:15:38.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.423 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004cba05025e570f85-006312ff1a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.425 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.427 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:15:38.429 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:15:38.429 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.430 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.431 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.431 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:38.432 DEBUG:boto:Data: 2022-09-03T07:15:38.432 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.433 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.434 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.435 DEBUG:boto:Params: {} 2022-09-03T07:15:38.436 DEBUG:boto:Token: None 2022-09-03T07:15:38.437 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VSvOVWZJaTc8RFU1BCcSPbyANVo= 2022-09-03T07:15:38.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VSvOVWZJaTc8RFU1BCcSPbyANVo=', 'Content-Length': '0'} 2022-09-03T07:15:38.442 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000fa13f0447dba53f3-006312ff1a-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.443 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.444 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.445 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.446 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:38.447 DEBUG:boto:Data: 2022-09-03T07:15:38.448 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.449 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.450 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.451 DEBUG:boto:Params: {} 2022-09-03T07:15:38.452 DEBUG:boto:Token: None 2022-09-03T07:15:38.453 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VSvOVWZJaTc8RFU1BCcSPbyANVo= 2022-09-03T07:15:38.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VSvOVWZJaTc8RFU1BCcSPbyANVo=', 'Content-Length': '0'} 2022-09-03T07:15:38.458 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f0c93562107e5d72-006312ff1a-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.460 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:15:38.460 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.461 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.462 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.462 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.463 DEBUG:boto:Method: GET 2022-09-03T07:15:38.463 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.464 DEBUG:boto:Data: 2022-09-03T07:15:38.464 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.465 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.465 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.466 DEBUG:boto:Params: {} 2022-09-03T07:15:38.467 DEBUG:boto:Token: None 2022-09-03T07:15:38.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/?versions 2022-09-03T07:15:38.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V4Zs4Hrf+x4G44G7KwHcwL51HEg= 2022-09-03T07:15:38.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V4Zs4Hrf+x4G44G7KwHcwL51HEg=', 'Content-Length': '0'} 2022-09-03T07:15:38.475 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008da18e449c828a1e-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.476 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.477 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:15:38.478 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:15:38.478 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:15:38.479 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:15:38.479 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:15:38.480 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:15:38.480 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:15:38.481 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:15:38.481 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:15:38.482 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:15:38.483 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:15:38.483 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.484 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.484 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.485 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.486 DEBUG:boto:Method: GET 2022-09-03T07:15:38.486 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.487 DEBUG:boto:Data: 2022-09-03T07:15:38.487 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.488 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.488 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.489 DEBUG:boto:Params: {} 2022-09-03T07:15:38.489 DEBUG:boto:Token: None 2022-09-03T07:15:38.490 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/?versions 2022-09-03T07:15:38.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V4Zs4Hrf+x4G44G7KwHcwL51HEg= 2022-09-03T07:15:38.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V4Zs4Hrf+x4G44G7KwHcwL51HEg=', 'Content-Length': '0'} 2022-09-03T07:15:38.494 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002fcc2765492d082a-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.495 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.497 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:15:38.497 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:15:38.498 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:15:38.498 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:15:38.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:15:38.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:15:38.500 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:15:38.500 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:15:38.501 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:15:38.501 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:15:38.502 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.503 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.503 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.504 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.504 DEBUG:boto:Method: GET 2022-09-03T07:15:38.505 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.505 DEBUG:boto:Data: 2022-09-03T07:15:38.506 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.506 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.507 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.507 DEBUG:boto:Params: {} 2022-09-03T07:15:38.508 DEBUG:boto:Token: None 2022-09-03T07:15:38.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/?versions 2022-09-03T07:15:38.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V4Zs4Hrf+x4G44G7KwHcwL51HEg= 2022-09-03T07:15:38.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V4Zs4Hrf+x4G44G7KwHcwL51HEg=', 'Content-Length': '0'} 2022-09-03T07:15:38.513 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022be5cb6f478d72f-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.514 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.515 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.516 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.516 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.517 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.517 DEBUG:boto:Method: GET 2022-09-03T07:15:38.518 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:15:38.518 DEBUG:boto:Data: 2022-09-03T07:15:38.519 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.519 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.520 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.520 DEBUG:boto:Params: {} 2022-09-03T07:15:38.521 DEBUG:boto:Token: None 2022-09-03T07:15:38.522 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/?versions 2022-09-03T07:15:38.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V4Zs4Hrf+x4G44G7KwHcwL51HEg= 2022-09-03T07:15:38.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V4Zs4Hrf+x4G44G7KwHcwL51HEg=', 'Content-Length': '0'} 2022-09-03T07:15:38.525 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cbeaa8e20643d8fb-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.526 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.529 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:15:38.530 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.530 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.531 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.532 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.533 DEBUG:boto:Method: GET 2022-09-03T07:15:38.533 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.534 DEBUG:boto:Data: 2022-09-03T07:15:38.534 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.536 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.536 DEBUG:boto:Params: {} 2022-09-03T07:15:38.537 DEBUG:boto:Token: None 2022-09-03T07:15:38.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nZeJQwxIvUPwp25tjngkFnCY3zA= 2022-09-03T07:15:38.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nZeJQwxIvUPwp25tjngkFnCY3zA=', 'Content-Length': '0'} 2022-09-03T07:15:38.542 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000551db15c4c58f824-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.590 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.591 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.591 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.592 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.592 DEBUG:boto:Method: GET 2022-09-03T07:15:38.593 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.594 DEBUG:boto:Data: 2022-09-03T07:15:38.596 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.597 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.598 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.599 DEBUG:boto:Params: {} 2022-09-03T07:15:38.600 DEBUG:boto:Token: None 2022-09-03T07:15:38.602 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nZeJQwxIvUPwp25tjngkFnCY3zA= 2022-09-03T07:15:38.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nZeJQwxIvUPwp25tjngkFnCY3zA=', 'Content-Length': '0'} 2022-09-03T07:15:38.605 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b674396636fad378-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.607 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.608 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.608 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.609 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.610 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.611 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.612 DEBUG:boto:Data: 2022-09-03T07:15:38.613 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.615 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.616 DEBUG:boto:Params: {} 2022-09-03T07:15:38.617 DEBUG:boto:Token: None 2022-09-03T07:15:38.618 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FJAHMfr1m1QWwZktHqGBPrwulLE= 2022-09-03T07:15:38.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FJAHMfr1m1QWwZktHqGBPrwulLE=', 'Content-Length': '0'} 2022-09-03T07:15:38.624 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000324e18d58149251c-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.625 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.626 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.628 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.629 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.630 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.631 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.631 DEBUG:boto:Data: 2022-09-03T07:15:38.632 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.632 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.633 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.634 DEBUG:boto:Params: {} 2022-09-03T07:15:38.635 DEBUG:boto:Token: None 2022-09-03T07:15:38.636 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FJAHMfr1m1QWwZktHqGBPrwulLE= 2022-09-03T07:15:38.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FJAHMfr1m1QWwZktHqGBPrwulLE=', 'Content-Length': '0'} 2022-09-03T07:15:38.641 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cda42c44f41f7e5e-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.642 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:15:38.642 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.643 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.643 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.644 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.644 DEBUG:boto:Method: GET 2022-09-03T07:15:38.645 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.646 DEBUG:boto:Data: 2022-09-03T07:15:38.647 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.648 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.648 DEBUG:boto:Params: {} 2022-09-03T07:15:38.649 DEBUG:boto:Token: None 2022-09-03T07:15:38.651 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nZeJQwxIvUPwp25tjngkFnCY3zA= 2022-09-03T07:15:38.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nZeJQwxIvUPwp25tjngkFnCY3zA=', 'Content-Length': '0'} 2022-09-03T07:15:38.657 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b89702a02201ae6d-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.659 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.660 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.661 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.662 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.663 DEBUG:boto:Method: GET 2022-09-03T07:15:38.664 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.665 DEBUG:boto:Data: 2022-09-03T07:15:38.666 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.667 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.668 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.669 DEBUG:boto:Params: {} 2022-09-03T07:15:38.670 DEBUG:boto:Token: None 2022-09-03T07:15:38.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:15:38.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nZeJQwxIvUPwp25tjngkFnCY3zA= 2022-09-03T07:15:38.672 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nZeJQwxIvUPwp25tjngkFnCY3zA=', 'Content-Length': '0'} 2022-09-03T07:15:38.676 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007bf9f221bd292b3d-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.679 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.680 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.681 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:38.682 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:38.683 DEBUG:boto:Method: GET 2022-09-03T07:15:38.684 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:38.685 DEBUG:boto:Data: 2022-09-03T07:15:38.686 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.688 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.689 DEBUG:boto:Params: {} 2022-09-03T07:15:38.689 DEBUG:boto:Token: None 2022-09-03T07:15:38.691 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.697 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005de619f3e3ccc371-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.698 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.700 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.701 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.702 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:38.703 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:38.704 DEBUG:boto:Method: GET 2022-09-03T07:15:38.707 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:38.709 DEBUG:boto:Data: 2022-09-03T07:15:38.710 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.711 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.712 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.713 DEBUG:boto:Params: {} 2022-09-03T07:15:38.715 DEBUG:boto:Token: None 2022-09-03T07:15:38.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.722 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ffdb277e1a92fea5-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.723 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.724 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.725 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.726 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.726 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:38.727 DEBUG:boto:Data: 2022-09-03T07:15:38.729 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.732 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.733 DEBUG:boto:Params: {} 2022-09-03T07:15:38.734 DEBUG:boto:Token: None 2022-09-03T07:15:38.735 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11 2022-09-03T07:15:38.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6NdRf4Bk1MOufnxHkMrN/G9THiw= 2022-09-03T07:15:38.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6NdRf4Bk1MOufnxHkMrN/G9THiw=', 'Content-Length': '0'} 2022-09-03T07:15:38.740 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1147a522a861487-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.742 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.743 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.744 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.745 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:38.746 DEBUG:boto:Data: 2022-09-03T07:15:38.747 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.748 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.749 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.750 DEBUG:boto:Params: {} 2022-09-03T07:15:38.751 DEBUG:boto:Token: None 2022-09-03T07:15:38.752 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11 2022-09-03T07:15:38.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6NdRf4Bk1MOufnxHkMrN/G9THiw= 2022-09-03T07:15:38.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6NdRf4Bk1MOufnxHkMrN/G9THiw=', 'Content-Length': '0'} 2022-09-03T07:15:38.757 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008ecc2d464232463f-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.759 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:15:38.760 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.761 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.762 DEBUG:boto:Method: GET 2022-09-03T07:15:38.766 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:38.768 DEBUG:boto:Data: 2022-09-03T07:15:38.769 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.769 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.770 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.771 DEBUG:boto:Params: {} 2022-09-03T07:15:38.772 DEBUG:boto:Token: None 2022-09-03T07:15:38.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11 2022-09-03T07:15:38.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Vcjfb2S1WmU/8U/9NGQqhFgjMM= 2022-09-03T07:15:38.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Vcjfb2S1WmU/8U/9NGQqhFgjMM=', 'Content-Length': '0'} 2022-09-03T07:15:38.786 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2190dc02d49ecf5-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.787 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:15:38.788 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:15:38.789 DEBUG:boto:Method: GET 2022-09-03T07:15:38.789 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:15:38.790 DEBUG:boto:Data: 2022-09-03T07:15:38.790 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.791 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.791 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.792 DEBUG:boto:Params: {} 2022-09-03T07:15:38.793 DEBUG:boto:Token: None 2022-09-03T07:15:38.793 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj11 2022-09-03T07:15:38.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Vcjfb2S1WmU/8U/9NGQqhFgjMM= 2022-09-03T07:15:38.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Vcjfb2S1WmU/8U/9NGQqhFgjMM=', 'Content-Length': '0'} 2022-09-03T07:15:38.798 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039bef63f69366b62-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.800 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.801 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.802 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:38.802 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:38.803 DEBUG:boto:Method: GET 2022-09-03T07:15:38.806 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:38.807 DEBUG:boto:Data: 2022-09-03T07:15:38.808 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.810 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.811 DEBUG:boto:Params: {} 2022-09-03T07:15:38.812 DEBUG:boto:Token: None 2022-09-03T07:15:38.814 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.820 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf969db614411e07-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.862 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.864 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:38.866 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:38.867 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:38.868 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:15:38.869 DEBUG:boto:Method: GET 2022-09-03T07:15:38.870 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:15:38.871 DEBUG:boto:Data: 2022-09-03T07:15:38.872 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.872 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.874 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.875 DEBUG:boto:Params: {} 2022-09-03T07:15:38.876 DEBUG:boto:Token: None 2022-09-03T07:15:38.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/ 2022-09-03T07:15:38.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI= 2022-09-03T07:15:38.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EqcMlCTv8CXEpgcKh03Ol/CqBSI=', 'Content-Length': '0'} 2022-09-03T07:15:38.887 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003d27affafc871194-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.888 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:38.890 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:15:38.891 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:38.892 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:38.893 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.893 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.894 DEBUG:boto:Method: GET 2022-09-03T07:15:38.895 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.898 DEBUG:boto:Data: 2022-09-03T07:15:38.899 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.901 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.902 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.903 DEBUG:boto:Params: {} 2022-09-03T07:15:38.904 DEBUG:boto:Token: None 2022-09-03T07:15:38.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MU4sQx+yPt0VxuqH4+hzrKDzh7s= 2022-09-03T07:15:38.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MU4sQx+yPt0VxuqH4+hzrKDzh7s=', 'Content-Length': '0'} 2022-09-03T07:15:38.909 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf4736c1bb9deb78-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.911 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:38.912 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:38.913 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.914 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.916 DEBUG:boto:Method: GET 2022-09-03T07:15:38.917 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.918 DEBUG:boto:Data: 2022-09-03T07:15:38.919 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.921 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.922 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.923 DEBUG:boto:Params: {} 2022-09-03T07:15:38.924 DEBUG:boto:Token: None 2022-09-03T07:15:38.926 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MU4sQx+yPt0VxuqH4+hzrKDzh7s= 2022-09-03T07:15:38.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MU4sQx+yPt0VxuqH4+hzrKDzh7s=', 'Content-Length': '0'} 2022-09-03T07:15:38.931 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083a016b277e639a7-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.932 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:38.933 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:38.934 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.934 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.935 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.936 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.937 DEBUG:boto:Data: 2022-09-03T07:15:38.938 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.940 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.940 DEBUG:boto:Params: {} 2022-09-03T07:15:38.941 DEBUG:boto:Token: None 2022-09-03T07:15:38.941 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:veF0PwhDH0UN3GPtIumHVrbpbJ4= 2022-09-03T07:15:38.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:veF0PwhDH0UN3GPtIumHVrbpbJ4=', 'Content-Length': '0'} 2022-09-03T07:15:38.945 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa5f4949020e460c-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.947 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:38.948 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:38.949 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.950 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.951 DEBUG:boto:Method: HEAD 2022-09-03T07:15:38.951 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.952 DEBUG:boto:Data: 2022-09-03T07:15:38.952 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.953 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.954 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.955 DEBUG:boto:Params: {} 2022-09-03T07:15:38.956 DEBUG:boto:Token: None 2022-09-03T07:15:38.956 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:veF0PwhDH0UN3GPtIumHVrbpbJ4= 2022-09-03T07:15:38.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:veF0PwhDH0UN3GPtIumHVrbpbJ4=', 'Content-Length': '0'} 2022-09-03T07:15:38.961 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000869262664eacf90f-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.962 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:15:38.963 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:38.963 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:38.964 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.964 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.966 DEBUG:boto:Method: GET 2022-09-03T07:15:38.966 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.967 DEBUG:boto:Data: 2022-09-03T07:15:38.968 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.968 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:38.969 DEBUG:boto:Port: 8001 2022-09-03T07:15:38.971 DEBUG:boto:Params: {} 2022-09-03T07:15:38.972 DEBUG:boto:Token: None 2022-09-03T07:15:38.973 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.973 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MU4sQx+yPt0VxuqH4+hzrKDzh7s= 2022-09-03T07:15:38.974 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MU4sQx+yPt0VxuqH4+hzrKDzh7s=', 'Content-Length': '0'} 2022-09-03T07:15:38.976 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000077fc45464660dd32-006312ff1a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:38.977 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:38.978 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:38.978 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.979 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.979 DEBUG:boto:Method: GET 2022-09-03T07:15:38.980 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.981 DEBUG:boto:Data: 2022-09-03T07:15:38.982 DEBUG:boto:Headers: {} 2022-09-03T07:15:38.983 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:38.983 DEBUG:boto:Port: 8000 2022-09-03T07:15:38.984 DEBUG:boto:Params: {} 2022-09-03T07:15:38.985 DEBUG:boto:Token: None 2022-09-03T07:15:38.985 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:38 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:15:38.986 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MU4sQx+yPt0VxuqH4+hzrKDzh7s= 2022-09-03T07:15:38.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:38 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MU4sQx+yPt0VxuqH4+hzrKDzh7s=', 'Content-Length': '0'} 2022-09-03T07:15:38.992 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f824562c52c26c6-006312ff1a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:38 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.034 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.035 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.036 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:39.037 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:39.039 DEBUG:boto:Method: GET 2022-09-03T07:15:39.040 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:39.041 DEBUG:boto:Data: 2022-09-03T07:15:39.043 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.045 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.049 DEBUG:boto:Params: {} 2022-09-03T07:15:39.051 DEBUG:boto:Token: None 2022-09-03T07:15:39.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.062 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005523478d29d133ae-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.063 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.065 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.066 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.067 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:39.068 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:39.069 DEBUG:boto:Method: GET 2022-09-03T07:15:39.070 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:39.071 DEBUG:boto:Data: 2022-09-03T07:15:39.072 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.072 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.073 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.073 DEBUG:boto:Params: {} 2022-09-03T07:15:39.074 DEBUG:boto:Token: None 2022-09-03T07:15:39.077 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.090 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000054b3d6da02ef067e-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.092 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.094 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:39.095 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:39.095 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.096 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:39.097 DEBUG:boto:Data: 2022-09-03T07:15:39.098 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.100 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.101 DEBUG:boto:Params: {} 2022-09-03T07:15:39.106 DEBUG:boto:Token: None 2022-09-03T07:15:39.108 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj17 2022-09-03T07:15:39.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcPoYo9pOFpCcrcaSMp7ULT5rSw= 2022-09-03T07:15:39.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcPoYo9pOFpCcrcaSMp7ULT5rSw=', 'Content-Length': '0'} 2022-09-03T07:15:39.113 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002efc240c9b55849a-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.115 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:39.116 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:39.118 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.119 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:39.120 DEBUG:boto:Data: 2022-09-03T07:15:39.122 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.124 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.125 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.126 DEBUG:boto:Params: {} 2022-09-03T07:15:39.128 DEBUG:boto:Token: None 2022-09-03T07:15:39.129 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj17 2022-09-03T07:15:39.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcPoYo9pOFpCcrcaSMp7ULT5rSw= 2022-09-03T07:15:39.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcPoYo9pOFpCcrcaSMp7ULT5rSw=', 'Content-Length': '0'} 2022-09-03T07:15:39.140 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009010393471f8c5fd-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.141 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:15:39.146 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:39.148 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:39.150 DEBUG:boto:Method: GET 2022-09-03T07:15:39.151 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:39.151 DEBUG:boto:Data: 2022-09-03T07:15:39.152 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.152 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.153 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.153 DEBUG:boto:Params: {} 2022-09-03T07:15:39.154 DEBUG:boto:Token: None 2022-09-03T07:15:39.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj17 2022-09-03T07:15:39.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JF25w9t8ruT2Iu58pTh0lioCn+k= 2022-09-03T07:15:39.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JF25w9t8ruT2Iu58pTh0lioCn+k=', 'Content-Length': '0'} 2022-09-03T07:15:39.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000902d1c1a426e0e97-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.167 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:15:39.168 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:15:39.169 DEBUG:boto:Method: GET 2022-09-03T07:15:39.170 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:15:39.171 DEBUG:boto:Data: 2022-09-03T07:15:39.178 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.179 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.181 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.182 DEBUG:boto:Params: {} 2022-09-03T07:15:39.194 DEBUG:boto:Token: None 2022-09-03T07:15:39.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj17 2022-09-03T07:15:39.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JF25w9t8ruT2Iu58pTh0lioCn+k= 2022-09-03T07:15:39.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JF25w9t8ruT2Iu58pTh0lioCn+k=', 'Content-Length': '0'} 2022-09-03T07:15:39.205 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba51a3a52259caa5-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.207 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.208 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.232 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:39.233 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:39.234 DEBUG:boto:Method: GET 2022-09-03T07:15:39.235 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:39.237 DEBUG:boto:Data: 2022-09-03T07:15:39.238 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.250 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.251 DEBUG:boto:Params: {} 2022-09-03T07:15:39.258 DEBUG:boto:Token: None 2022-09-03T07:15:39.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.275 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e42c178e84d076df-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.285 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.289 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.291 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.292 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:39.293 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:15:39.293 DEBUG:boto:Method: GET 2022-09-03T07:15:39.295 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:15:39.296 DEBUG:boto:Data: 2022-09-03T07:15:39.297 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.298 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.299 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.301 DEBUG:boto:Params: {} 2022-09-03T07:15:39.302 DEBUG:boto:Token: None 2022-09-03T07:15:39.303 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.313 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb080fa8105b41c7-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.314 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.315 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:15:39.319 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.320 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.321 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.324 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.325 DEBUG:boto:Method: GET 2022-09-03T07:15:39.327 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.328 DEBUG:boto:Data: 2022-09-03T07:15:39.329 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.331 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.332 DEBUG:boto:Params: {} 2022-09-03T07:15:39.333 DEBUG:boto:Token: None 2022-09-03T07:15:39.334 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hygcp4mQ/jIySn1ag8Ri7X72kpY= 2022-09-03T07:15:39.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hygcp4mQ/jIySn1ag8Ri7X72kpY=', 'Content-Length': '0'} 2022-09-03T07:15:39.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fb9abfb2eafd894-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.345 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.346 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.347 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.348 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.350 DEBUG:boto:Method: GET 2022-09-03T07:15:39.351 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.352 DEBUG:boto:Data: 2022-09-03T07:15:39.353 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.354 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.355 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.356 DEBUG:boto:Params: {} 2022-09-03T07:15:39.357 DEBUG:boto:Token: None 2022-09-03T07:15:39.359 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hygcp4mQ/jIySn1ag8Ri7X72kpY= 2022-09-03T07:15:39.360 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hygcp4mQ/jIySn1ag8Ri7X72kpY=', 'Content-Length': '0'} 2022-09-03T07:15:39.365 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c52f8ea2d5bfbd38-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.366 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.367 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.367 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.368 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.368 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.369 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.369 DEBUG:boto:Data: 2022-09-03T07:15:39.370 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.371 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.372 DEBUG:boto:Params: {} 2022-09-03T07:15:39.373 DEBUG:boto:Token: None 2022-09-03T07:15:39.374 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ryy3yXskV1mNMSGlcqmmCetpmc4= 2022-09-03T07:15:39.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ryy3yXskV1mNMSGlcqmmCetpmc4=', 'Content-Length': '0'} 2022-09-03T07:15:39.381 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f5f52ae78deec6c-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.384 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.385 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.386 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.387 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.388 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.389 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.390 DEBUG:boto:Data: 2022-09-03T07:15:39.391 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.392 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.393 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.395 DEBUG:boto:Params: {} 2022-09-03T07:15:39.396 DEBUG:boto:Token: None 2022-09-03T07:15:39.397 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ryy3yXskV1mNMSGlcqmmCetpmc4= 2022-09-03T07:15:39.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ryy3yXskV1mNMSGlcqmmCetpmc4=', 'Content-Length': '0'} 2022-09-03T07:15:39.403 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000101950a09b93c20e-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.405 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:15:39.407 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.408 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.409 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.410 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.411 DEBUG:boto:Method: GET 2022-09-03T07:15:39.412 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.413 DEBUG:boto:Data: 2022-09-03T07:15:39.415 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.416 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.417 DEBUG:boto:Params: {} 2022-09-03T07:15:39.417 DEBUG:boto:Token: None 2022-09-03T07:15:39.418 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hygcp4mQ/jIySn1ag8Ri7X72kpY= 2022-09-03T07:15:39.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hygcp4mQ/jIySn1ag8Ri7X72kpY=', 'Content-Length': '0'} 2022-09-03T07:15:39.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000611dac6141cb22e5-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.423 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.424 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.424 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.425 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.425 DEBUG:boto:Method: GET 2022-09-03T07:15:39.426 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.427 DEBUG:boto:Data: 2022-09-03T07:15:39.427 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.428 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.428 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.429 DEBUG:boto:Params: {} 2022-09-03T07:15:39.429 DEBUG:boto:Token: None 2022-09-03T07:15:39.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:15:39.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hygcp4mQ/jIySn1ag8Ri7X72kpY= 2022-09-03T07:15:39.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hygcp4mQ/jIySn1ag8Ri7X72kpY=', 'Content-Length': '0'} 2022-09-03T07:15:39.434 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000046b2d511f2bd298-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.435 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.435 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.436 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:39.436 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:39.437 DEBUG:boto:Method: GET 2022-09-03T07:15:39.437 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:39.438 DEBUG:boto:Data: 2022-09-03T07:15:39.438 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.440 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.441 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.442 DEBUG:boto:Params: {} 2022-09-03T07:15:39.444 DEBUG:boto:Token: None 2022-09-03T07:15:39.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.453 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000073630f9d8ba1daa0-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.496 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.498 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.499 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.501 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:39.502 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:39.507 DEBUG:boto:Method: GET 2022-09-03T07:15:39.508 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:39.509 DEBUG:boto:Data: 2022-09-03T07:15:39.510 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.511 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.511 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.516 DEBUG:boto:Params: {} 2022-09-03T07:15:39.517 DEBUG:boto:Token: None 2022-09-03T07:15:39.518 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.530 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000098b49fb091f1efe-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.532 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.533 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.539 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.540 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.541 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:39.543 DEBUG:boto:Data: 2022-09-03T07:15:39.544 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.546 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.547 DEBUG:boto:Params: {} 2022-09-03T07:15:39.548 DEBUG:boto:Token: None 2022-09-03T07:15:39.549 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22 2022-09-03T07:15:39.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xVSJytpoG4JhbIWz1L3JZvVU158= 2022-09-03T07:15:39.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xVSJytpoG4JhbIWz1L3JZvVU158=', 'Content-Length': '0'} 2022-09-03T07:15:39.557 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc0966bd2a4e7ff0-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.558 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.559 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.560 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.561 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:39.562 DEBUG:boto:Data: 2022-09-03T07:15:39.563 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.564 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.565 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.566 DEBUG:boto:Params: {} 2022-09-03T07:15:39.567 DEBUG:boto:Token: None 2022-09-03T07:15:39.568 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22 2022-09-03T07:15:39.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xVSJytpoG4JhbIWz1L3JZvVU158= 2022-09-03T07:15:39.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xVSJytpoG4JhbIWz1L3JZvVU158=', 'Content-Length': '0'} 2022-09-03T07:15:39.576 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd7d7f9cfe981088-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.577 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:15:39.578 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.579 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.579 DEBUG:boto:Method: GET 2022-09-03T07:15:39.580 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:39.580 DEBUG:boto:Data: 2022-09-03T07:15:39.581 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.582 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.583 DEBUG:boto:Params: {} 2022-09-03T07:15:39.584 DEBUG:boto:Token: None 2022-09-03T07:15:39.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22 2022-09-03T07:15:39.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z562Tk/afsMcnsvli0UqUjEeMrU= 2022-09-03T07:15:39.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z562Tk/afsMcnsvli0UqUjEeMrU=', 'Content-Length': '0'} 2022-09-03T07:15:39.599 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090ed38e9c8b491d7-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.600 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:15:39.601 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:15:39.610 DEBUG:boto:Method: GET 2022-09-03T07:15:39.611 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:15:39.611 DEBUG:boto:Data: 2022-09-03T07:15:39.614 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.615 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.616 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.616 DEBUG:boto:Params: {} 2022-09-03T07:15:39.617 DEBUG:boto:Token: None 2022-09-03T07:15:39.618 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj22 2022-09-03T07:15:39.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z562Tk/afsMcnsvli0UqUjEeMrU= 2022-09-03T07:15:39.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z562Tk/afsMcnsvli0UqUjEeMrU=', 'Content-Length': '0'} 2022-09-03T07:15:39.631 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fff56bd41322c19c-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.633 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.633 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.634 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:39.646 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:39.648 DEBUG:boto:Method: GET 2022-09-03T07:15:39.649 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:39.650 DEBUG:boto:Data: 2022-09-03T07:15:39.651 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.653 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.654 DEBUG:boto:Params: {} 2022-09-03T07:15:39.655 DEBUG:boto:Token: None 2022-09-03T07:15:39.663 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.687 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000009d0dbbbb6d0c26a-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.688 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.689 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.690 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.690 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:39.691 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:15:39.691 DEBUG:boto:Method: GET 2022-09-03T07:15:39.692 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:15:39.693 DEBUG:boto:Data: 2022-09-03T07:15:39.693 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.694 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.694 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.695 DEBUG:boto:Params: {} 2022-09-03T07:15:39.696 DEBUG:boto:Token: None 2022-09-03T07:15:39.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.709 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000060c1215a4be65d80-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.710 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.712 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:15:39.716 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:39.717 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:39.718 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.719 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.720 DEBUG:boto:Method: GET 2022-09-03T07:15:39.721 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.722 DEBUG:boto:Data: 2022-09-03T07:15:39.723 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.724 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.725 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.727 DEBUG:boto:Params: {} 2022-09-03T07:15:39.728 DEBUG:boto:Token: None 2022-09-03T07:15:39.729 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Ebz2RR8OFmOzeQhJJMzwPv4LEw= 2022-09-03T07:15:39.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Ebz2RR8OFmOzeQhJJMzwPv4LEw=', 'Content-Length': '0'} 2022-09-03T07:15:39.736 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000006f6216b763ff45d-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.743 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:39.745 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:39.746 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.747 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.748 DEBUG:boto:Method: GET 2022-09-03T07:15:39.749 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.750 DEBUG:boto:Data: 2022-09-03T07:15:39.751 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.753 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.754 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.755 DEBUG:boto:Params: {} 2022-09-03T07:15:39.756 DEBUG:boto:Token: None 2022-09-03T07:15:39.757 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Ebz2RR8OFmOzeQhJJMzwPv4LEw= 2022-09-03T07:15:39.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Ebz2RR8OFmOzeQhJJMzwPv4LEw=', 'Content-Length': '0'} 2022-09-03T07:15:39.762 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb4ef07369e05f63-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.764 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:39.765 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:39.766 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.767 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.773 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.774 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.775 DEBUG:boto:Data: 2022-09-03T07:15:39.776 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.778 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.779 DEBUG:boto:Params: {} 2022-09-03T07:15:39.780 DEBUG:boto:Token: None 2022-09-03T07:15:39.781 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vPppSzn/WxGaViQe1fz+yOzTgHk= 2022-09-03T07:15:39.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vPppSzn/WxGaViQe1fz+yOzTgHk=', 'Content-Length': '0'} 2022-09-03T07:15:39.792 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004b145ec120640ae-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.793 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:39.796 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:39.797 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.798 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.800 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.801 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.802 DEBUG:boto:Data: 2022-09-03T07:15:39.803 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.804 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.807 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.808 DEBUG:boto:Params: {} 2022-09-03T07:15:39.809 DEBUG:boto:Token: None 2022-09-03T07:15:39.810 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vPppSzn/WxGaViQe1fz+yOzTgHk= 2022-09-03T07:15:39.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vPppSzn/WxGaViQe1fz+yOzTgHk=', 'Content-Length': '0'} 2022-09-03T07:15:39.820 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b948dd87d893535-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.823 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:15:39.824 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:39.825 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:39.826 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.830 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.831 DEBUG:boto:Method: GET 2022-09-03T07:15:39.832 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.833 DEBUG:boto:Data: 2022-09-03T07:15:39.834 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.836 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.844 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.847 DEBUG:boto:Params: {} 2022-09-03T07:15:39.852 DEBUG:boto:Token: None 2022-09-03T07:15:39.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Ebz2RR8OFmOzeQhJJMzwPv4LEw= 2022-09-03T07:15:39.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Ebz2RR8OFmOzeQhJJMzwPv4LEw=', 'Content-Length': '0'} 2022-09-03T07:15:39.860 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016d0329b22144ada-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.866 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:39.867 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:39.876 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.877 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.879 DEBUG:boto:Method: GET 2022-09-03T07:15:39.880 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.881 DEBUG:boto:Data: 2022-09-03T07:15:39.882 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.884 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.885 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.886 DEBUG:boto:Params: {} 2022-09-03T07:15:39.887 DEBUG:boto:Token: None 2022-09-03T07:15:39.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:15:39.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Ebz2RR8OFmOzeQhJJMzwPv4LEw= 2022-09-03T07:15:39.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Ebz2RR8OFmOzeQhJJMzwPv4LEw=', 'Content-Length': '0'} 2022-09-03T07:15:39.895 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074fecdbd34f13187-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.896 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.897 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.900 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:39.901 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:39.902 DEBUG:boto:Method: GET 2022-09-03T07:15:39.903 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:39.904 DEBUG:boto:Data: 2022-09-03T07:15:39.905 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.906 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.907 DEBUG:boto:Params: {} 2022-09-03T07:15:39.908 DEBUG:boto:Token: None 2022-09-03T07:15:39.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000005ddd51336289828-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.915 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.916 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:39.922 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:39.924 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:39.925 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:39.926 DEBUG:boto:Method: GET 2022-09-03T07:15:39.927 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:39.927 DEBUG:boto:Data: 2022-09-03T07:15:39.928 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.929 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.930 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.932 DEBUG:boto:Params: {} 2022-09-03T07:15:39.938 DEBUG:boto:Token: None 2022-09-03T07:15:39.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/ 2022-09-03T07:15:39.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM= 2022-09-03T07:15:39.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DzH8utNxLascfHi2M0/ypn9scM=', 'Content-Length': '0'} 2022-09-03T07:15:39.955 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000279b08e5c85eaec2-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.959 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:39.960 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:39.961 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:39.962 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.963 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:39.964 DEBUG:boto:Data: 2022-09-03T07:15:39.965 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.968 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:39.969 DEBUG:boto:Port: 8001 2022-09-03T07:15:39.969 DEBUG:boto:Params: {} 2022-09-03T07:15:39.970 DEBUG:boto:Token: None 2022-09-03T07:15:39.971 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj33 2022-09-03T07:15:39.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:37InkTgGKRdyCrV46734d+KWa8o= 2022-09-03T07:15:39.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:37InkTgGKRdyCrV46734d+KWa8o=', 'Content-Length': '0'} 2022-09-03T07:15:39.977 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009dfb651f4f6cc732-006312ff1b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:39.979 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:39.980 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:39.981 DEBUG:boto:Method: HEAD 2022-09-03T07:15:39.982 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:39.983 DEBUG:boto:Data: 2022-09-03T07:15:39.985 DEBUG:boto:Headers: {} 2022-09-03T07:15:39.986 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:39.987 DEBUG:boto:Port: 8000 2022-09-03T07:15:39.988 DEBUG:boto:Params: {} 2022-09-03T07:15:39.989 DEBUG:boto:Token: None 2022-09-03T07:15:39.990 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:39 GMT /avysmw-75/obj33 2022-09-03T07:15:39.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:37InkTgGKRdyCrV46734d+KWa8o= 2022-09-03T07:15:39.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:39 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:37InkTgGKRdyCrV46734d+KWa8o=', 'Content-Length': '0'} 2022-09-03T07:15:39.998 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f31c262fadc41db-006312ff1b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:39 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.001 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:15:40.002 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:40.003 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:40.005 DEBUG:boto:Method: GET 2022-09-03T07:15:40.006 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:40.007 DEBUG:boto:Data: 2022-09-03T07:15:40.008 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.010 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.011 DEBUG:boto:Params: {} 2022-09-03T07:15:40.012 DEBUG:boto:Token: None 2022-09-03T07:15:40.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj33 2022-09-03T07:15:40.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0H7Zeq419o9ZK9mnWKK9UrhWF2I= 2022-09-03T07:15:40.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0H7Zeq419o9ZK9mnWKK9UrhWF2I=', 'Content-Length': '0'} 2022-09-03T07:15:40.019 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f26cbd9db88e336-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.020 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:15:40.021 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:15:40.022 DEBUG:boto:Method: GET 2022-09-03T07:15:40.023 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:15:40.024 DEBUG:boto:Data: 2022-09-03T07:15:40.025 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.026 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.027 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.028 DEBUG:boto:Params: {} 2022-09-03T07:15:40.029 DEBUG:boto:Token: None 2022-09-03T07:15:40.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj33 2022-09-03T07:15:40.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0H7Zeq419o9ZK9mnWKK9UrhWF2I= 2022-09-03T07:15:40.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0H7Zeq419o9ZK9mnWKK9UrhWF2I=', 'Content-Length': '0'} 2022-09-03T07:15:40.036 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086156092b29b36ec-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.038 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.039 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.039 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:40.040 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:40.040 DEBUG:boto:Method: GET 2022-09-03T07:15:40.042 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:40.043 DEBUG:boto:Data: 2022-09-03T07:15:40.043 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.047 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.048 DEBUG:boto:Params: {} 2022-09-03T07:15:40.049 DEBUG:boto:Token: None 2022-09-03T07:15:40.051 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.062 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000caf875c45018b756-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.063 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.065 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.066 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.067 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:40.068 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:15:40.069 DEBUG:boto:Method: GET 2022-09-03T07:15:40.069 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:15:40.070 DEBUG:boto:Data: 2022-09-03T07:15:40.071 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.073 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.074 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.075 DEBUG:boto:Params: {} 2022-09-03T07:15:40.076 DEBUG:boto:Token: None 2022-09-03T07:15:40.077 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.079 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.085 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002016cbfcf0f9d0b2-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.087 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.088 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:15:40.089 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.089 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.090 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.090 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.091 DEBUG:boto:Method: GET 2022-09-03T07:15:40.093 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.094 DEBUG:boto:Data: 2022-09-03T07:15:40.095 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.096 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.097 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.098 DEBUG:boto:Params: {} 2022-09-03T07:15:40.099 DEBUG:boto:Token: None 2022-09-03T07:15:40.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ppsXa4x977SM2K65NV5T9yDkT2A= 2022-09-03T07:15:40.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ppsXa4x977SM2K65NV5T9yDkT2A=', 'Content-Length': '0'} 2022-09-03T07:15:40.110 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ef565f5a61ccced-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.118 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.121 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.121 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.123 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.125 DEBUG:boto:Method: GET 2022-09-03T07:15:40.126 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.127 DEBUG:boto:Data: 2022-09-03T07:15:40.128 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.129 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.130 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.132 DEBUG:boto:Params: {} 2022-09-03T07:15:40.133 DEBUG:boto:Token: None 2022-09-03T07:15:40.136 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ppsXa4x977SM2K65NV5T9yDkT2A= 2022-09-03T07:15:40.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ppsXa4x977SM2K65NV5T9yDkT2A=', 'Content-Length': '0'} 2022-09-03T07:15:40.148 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057e3a79a6693e2e8-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.151 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.152 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.153 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.163 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.164 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.165 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.166 DEBUG:boto:Data: 2022-09-03T07:15:40.167 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.169 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.171 DEBUG:boto:Params: {} 2022-09-03T07:15:40.172 DEBUG:boto:Token: None 2022-09-03T07:15:40.173 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:opPjCpRIhQphlWm1Kt8lM/H8ew0= 2022-09-03T07:15:40.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:opPjCpRIhQphlWm1Kt8lM/H8ew0=', 'Content-Length': '0'} 2022-09-03T07:15:40.179 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d5d804a95ad78155-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.181 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.182 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.183 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.184 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.185 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.186 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.197 DEBUG:boto:Data: 2022-09-03T07:15:40.198 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.198 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.199 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.199 DEBUG:boto:Params: {} 2022-09-03T07:15:40.200 DEBUG:boto:Token: None 2022-09-03T07:15:40.200 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:opPjCpRIhQphlWm1Kt8lM/H8ew0= 2022-09-03T07:15:40.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:opPjCpRIhQphlWm1Kt8lM/H8ew0=', 'Content-Length': '0'} 2022-09-03T07:15:40.204 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9db3a4fc8394b7d-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.205 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:15:40.206 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.206 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.207 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.207 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.208 DEBUG:boto:Method: GET 2022-09-03T07:15:40.208 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.209 DEBUG:boto:Data: 2022-09-03T07:15:40.209 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.211 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.211 DEBUG:boto:Params: {} 2022-09-03T07:15:40.213 DEBUG:boto:Token: None 2022-09-03T07:15:40.213 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ppsXa4x977SM2K65NV5T9yDkT2A= 2022-09-03T07:15:40.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ppsXa4x977SM2K65NV5T9yDkT2A=', 'Content-Length': '0'} 2022-09-03T07:15:40.217 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096abf8e90e179471-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.258 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.260 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.261 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.264 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.265 DEBUG:boto:Method: GET 2022-09-03T07:15:40.265 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.266 DEBUG:boto:Data: 2022-09-03T07:15:40.267 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.267 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.268 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.269 DEBUG:boto:Params: {} 2022-09-03T07:15:40.272 DEBUG:boto:Token: None 2022-09-03T07:15:40.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:15:40.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ppsXa4x977SM2K65NV5T9yDkT2A= 2022-09-03T07:15:40.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ppsXa4x977SM2K65NV5T9yDkT2A=', 'Content-Length': '0'} 2022-09-03T07:15:40.278 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024b6c033a12e9257-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.280 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.280 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.281 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:40.281 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:40.282 DEBUG:boto:Method: GET 2022-09-03T07:15:40.283 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:40.284 DEBUG:boto:Data: 2022-09-03T07:15:40.284 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.285 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.286 DEBUG:boto:Params: {} 2022-09-03T07:15:40.287 DEBUG:boto:Token: None 2022-09-03T07:15:40.287 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.292 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce7efc572bd8da72-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.293 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.296 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.298 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.299 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:40.300 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:40.301 DEBUG:boto:Method: GET 2022-09-03T07:15:40.302 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:40.303 DEBUG:boto:Data: 2022-09-03T07:15:40.304 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.305 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.306 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.307 DEBUG:boto:Params: {} 2022-09-03T07:15:40.315 DEBUG:boto:Token: None 2022-09-03T07:15:40.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.326 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a32d2adfc80177ab-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.326 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.327 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.328 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.328 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.329 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:40.329 DEBUG:boto:Data: 2022-09-03T07:15:40.330 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.331 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.332 DEBUG:boto:Params: {} 2022-09-03T07:15:40.332 DEBUG:boto:Token: None 2022-09-03T07:15:40.333 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34 2022-09-03T07:15:40.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V0ylHIFUXbtkpj9sVl6TQfjNVFc= 2022-09-03T07:15:40.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V0ylHIFUXbtkpj9sVl6TQfjNVFc=', 'Content-Length': '0'} 2022-09-03T07:15:40.338 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074f3aca4ffca16e6-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.340 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.341 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.342 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.343 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:40.343 DEBUG:boto:Data: 2022-09-03T07:15:40.344 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.344 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.345 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.345 DEBUG:boto:Params: {} 2022-09-03T07:15:40.346 DEBUG:boto:Token: None 2022-09-03T07:15:40.347 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34 2022-09-03T07:15:40.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V0ylHIFUXbtkpj9sVl6TQfjNVFc= 2022-09-03T07:15:40.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V0ylHIFUXbtkpj9sVl6TQfjNVFc=', 'Content-Length': '0'} 2022-09-03T07:15:40.350 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5da86e00432f82f-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.351 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:15:40.352 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.352 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.353 DEBUG:boto:Method: GET 2022-09-03T07:15:40.354 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:40.354 DEBUG:boto:Data: 2022-09-03T07:15:40.355 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.355 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.356 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.357 DEBUG:boto:Params: {} 2022-09-03T07:15:40.358 DEBUG:boto:Token: None 2022-09-03T07:15:40.359 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34 2022-09-03T07:15:40.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bckQnPw2s4plA31UEBcVzSTYfh0= 2022-09-03T07:15:40.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bckQnPw2s4plA31UEBcVzSTYfh0=', 'Content-Length': '0'} 2022-09-03T07:15:40.365 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c32641717c8f57f8-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.367 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:15:40.368 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:15:40.368 DEBUG:boto:Method: GET 2022-09-03T07:15:40.369 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:15:40.369 DEBUG:boto:Data: 2022-09-03T07:15:40.370 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.371 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.371 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.372 DEBUG:boto:Params: {} 2022-09-03T07:15:40.373 DEBUG:boto:Token: None 2022-09-03T07:15:40.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj34 2022-09-03T07:15:40.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bckQnPw2s4plA31UEBcVzSTYfh0= 2022-09-03T07:15:40.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bckQnPw2s4plA31UEBcVzSTYfh0=', 'Content-Length': '0'} 2022-09-03T07:15:40.378 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c6b9e548922c9ca-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.379 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.379 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.380 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:40.381 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:40.382 DEBUG:boto:Method: GET 2022-09-03T07:15:40.383 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:40.384 DEBUG:boto:Data: 2022-09-03T07:15:40.385 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.387 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.388 DEBUG:boto:Params: {} 2022-09-03T07:15:40.388 DEBUG:boto:Token: None 2022-09-03T07:15:40.389 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.394 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075f5234289e8f9f5-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.395 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.397 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.399 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.401 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:40.401 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:15:40.403 DEBUG:boto:Method: GET 2022-09-03T07:15:40.404 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:15:40.405 DEBUG:boto:Data: 2022-09-03T07:15:40.405 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.406 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.407 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.407 DEBUG:boto:Params: {} 2022-09-03T07:15:40.408 DEBUG:boto:Token: None 2022-09-03T07:15:40.409 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.413 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eecdadc747651941-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.413 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.414 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:15:40.415 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.415 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.416 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.416 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.417 DEBUG:boto:Method: GET 2022-09-03T07:15:40.418 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.418 DEBUG:boto:Data: 2022-09-03T07:15:40.419 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.420 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.420 DEBUG:boto:Params: {} 2022-09-03T07:15:40.421 DEBUG:boto:Token: None 2022-09-03T07:15:40.422 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NNQXalkANVTU94tUc+kUYnG4nXM= 2022-09-03T07:15:40.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NNQXalkANVTU94tUc+kUYnG4nXM=', 'Content-Length': '0'} 2022-09-03T07:15:40.426 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e6b9b915bb6cd69-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.427 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.427 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.428 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.428 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.429 DEBUG:boto:Method: GET 2022-09-03T07:15:40.429 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.430 DEBUG:boto:Data: 2022-09-03T07:15:40.430 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.431 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.432 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.432 DEBUG:boto:Params: {} 2022-09-03T07:15:40.433 DEBUG:boto:Token: None 2022-09-03T07:15:40.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NNQXalkANVTU94tUc+kUYnG4nXM= 2022-09-03T07:15:40.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NNQXalkANVTU94tUc+kUYnG4nXM=', 'Content-Length': '0'} 2022-09-03T07:15:40.437 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f1fca6fa3c8ea7c-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.439 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.440 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.440 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.441 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.441 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.442 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.443 DEBUG:boto:Data: 2022-09-03T07:15:40.443 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.444 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.444 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.445 DEBUG:boto:Params: {} 2022-09-03T07:15:40.446 DEBUG:boto:Token: None 2022-09-03T07:15:40.447 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1qkBbEYenFzMSYORO0pAPZ6RuUg= 2022-09-03T07:15:40.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1qkBbEYenFzMSYORO0pAPZ6RuUg=', 'Content-Length': '0'} 2022-09-03T07:15:40.455 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae1cc6ce8e0fe2f9-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.456 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.457 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.458 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.459 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.460 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.461 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.462 DEBUG:boto:Data: 2022-09-03T07:15:40.463 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.464 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.465 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.467 DEBUG:boto:Params: {} 2022-09-03T07:15:40.468 DEBUG:boto:Token: None 2022-09-03T07:15:40.469 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1qkBbEYenFzMSYORO0pAPZ6RuUg= 2022-09-03T07:15:40.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1qkBbEYenFzMSYORO0pAPZ6RuUg=', 'Content-Length': '0'} 2022-09-03T07:15:40.474 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6d0a9e19f434efc-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.475 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:15:40.475 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.476 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.476 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.477 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.478 DEBUG:boto:Method: GET 2022-09-03T07:15:40.479 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.479 DEBUG:boto:Data: 2022-09-03T07:15:40.480 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.480 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.481 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.481 DEBUG:boto:Params: {} 2022-09-03T07:15:40.482 DEBUG:boto:Token: None 2022-09-03T07:15:40.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NNQXalkANVTU94tUc+kUYnG4nXM= 2022-09-03T07:15:40.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NNQXalkANVTU94tUc+kUYnG4nXM=', 'Content-Length': '0'} 2022-09-03T07:15:40.486 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000851b74340668871b-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.537 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.538 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.538 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.539 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.539 DEBUG:boto:Method: GET 2022-09-03T07:15:40.540 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.540 DEBUG:boto:Data: 2022-09-03T07:15:40.541 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.541 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.542 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.543 DEBUG:boto:Params: {} 2022-09-03T07:15:40.543 DEBUG:boto:Token: None 2022-09-03T07:15:40.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:15:40.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NNQXalkANVTU94tUc+kUYnG4nXM= 2022-09-03T07:15:40.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NNQXalkANVTU94tUc+kUYnG4nXM=', 'Content-Length': '0'} 2022-09-03T07:15:40.549 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a7ff9874dae868af-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.551 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.551 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.552 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:40.552 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:40.553 DEBUG:boto:Method: GET 2022-09-03T07:15:40.553 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:40.554 DEBUG:boto:Data: 2022-09-03T07:15:40.554 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.555 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.556 DEBUG:boto:Params: {} 2022-09-03T07:15:40.557 DEBUG:boto:Token: None 2022-09-03T07:15:40.557 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.564 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000915844c6aeb7d581-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.565 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.566 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.567 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.567 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:40.568 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:40.569 DEBUG:boto:Method: GET 2022-09-03T07:15:40.569 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:40.570 DEBUG:boto:Data: 2022-09-03T07:15:40.571 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.571 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.572 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.573 DEBUG:boto:Params: {} 2022-09-03T07:15:40.573 DEBUG:boto:Token: None 2022-09-03T07:15:40.574 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.578 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044260c142b2f0647-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.579 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.581 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.582 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.583 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.584 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:40.586 DEBUG:boto:Data: 2022-09-03T07:15:40.587 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.588 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.589 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.590 DEBUG:boto:Params: {} 2022-09-03T07:15:40.591 DEBUG:boto:Token: None 2022-09-03T07:15:40.592 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44 2022-09-03T07:15:40.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N9fjxGqOZxiPBAx3pSq3ZJdwgKA= 2022-09-03T07:15:40.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N9fjxGqOZxiPBAx3pSq3ZJdwgKA=', 'Content-Length': '0'} 2022-09-03T07:15:40.597 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049c10b351a8aae91-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.599 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.600 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.601 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.602 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:40.603 DEBUG:boto:Data: 2022-09-03T07:15:40.604 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.605 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.607 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.608 DEBUG:boto:Params: {} 2022-09-03T07:15:40.609 DEBUG:boto:Token: None 2022-09-03T07:15:40.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44 2022-09-03T07:15:40.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N9fjxGqOZxiPBAx3pSq3ZJdwgKA= 2022-09-03T07:15:40.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N9fjxGqOZxiPBAx3pSq3ZJdwgKA=', 'Content-Length': '0'} 2022-09-03T07:15:40.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eae732f4b5eb9b42-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.619 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:15:40.621 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.621 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.622 DEBUG:boto:Method: GET 2022-09-03T07:15:40.623 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:40.625 DEBUG:boto:Data: 2022-09-03T07:15:40.625 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.627 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.628 DEBUG:boto:Params: {} 2022-09-03T07:15:40.630 DEBUG:boto:Token: None 2022-09-03T07:15:40.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44 2022-09-03T07:15:40.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qbjev8lPFy0l0EszGfk5V23n0Q8= 2022-09-03T07:15:40.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qbjev8lPFy0l0EszGfk5V23n0Q8=', 'Content-Length': '0'} 2022-09-03T07:15:40.636 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000025bac62bfb6676b7-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.637 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:15:40.638 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:15:40.639 DEBUG:boto:Method: GET 2022-09-03T07:15:40.640 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:15:40.641 DEBUG:boto:Data: 2022-09-03T07:15:40.642 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.643 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.644 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.645 DEBUG:boto:Params: {} 2022-09-03T07:15:40.646 DEBUG:boto:Token: None 2022-09-03T07:15:40.647 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj44 2022-09-03T07:15:40.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Qbjev8lPFy0l0EszGfk5V23n0Q8= 2022-09-03T07:15:40.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Qbjev8lPFy0l0EszGfk5V23n0Q8=', 'Content-Length': '0'} 2022-09-03T07:15:40.651 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068929e8bc184ea54-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.652 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.652 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.653 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:40.654 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:40.654 DEBUG:boto:Method: GET 2022-09-03T07:15:40.655 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:40.656 DEBUG:boto:Data: 2022-09-03T07:15:40.657 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.658 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.658 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.659 DEBUG:boto:Params: {} 2022-09-03T07:15:40.660 DEBUG:boto:Token: None 2022-09-03T07:15:40.661 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.662 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.675 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005442e97198cdec55-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.677 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.679 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.680 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.681 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:40.682 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:15:40.683 DEBUG:boto:Method: GET 2022-09-03T07:15:40.683 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:15:40.684 DEBUG:boto:Data: 2022-09-03T07:15:40.686 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.687 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.687 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.688 DEBUG:boto:Params: {} 2022-09-03T07:15:40.689 DEBUG:boto:Token: None 2022-09-03T07:15:40.689 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.693 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a95790014202de2d-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.694 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.695 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:15:40.696 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.697 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.697 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.698 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.698 DEBUG:boto:Method: GET 2022-09-03T07:15:40.699 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.700 DEBUG:boto:Data: 2022-09-03T07:15:40.700 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.701 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.702 DEBUG:boto:Params: {} 2022-09-03T07:15:40.703 DEBUG:boto:Token: None 2022-09-03T07:15:40.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.705 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BtkQElFi8HynYYkklC6zKu3dlKQ= 2022-09-03T07:15:40.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BtkQElFi8HynYYkklC6zKu3dlKQ=', 'Content-Length': '0'} 2022-09-03T07:15:40.716 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000289b305d269c3ed4-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.719 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.720 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.720 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.721 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.721 DEBUG:boto:Method: GET 2022-09-03T07:15:40.722 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.728 DEBUG:boto:Data: 2022-09-03T07:15:40.730 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.732 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.734 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.736 DEBUG:boto:Params: {} 2022-09-03T07:15:40.738 DEBUG:boto:Token: None 2022-09-03T07:15:40.739 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BtkQElFi8HynYYkklC6zKu3dlKQ= 2022-09-03T07:15:40.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BtkQElFi8HynYYkklC6zKu3dlKQ=', 'Content-Length': '0'} 2022-09-03T07:15:40.752 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000017f960ca8a9df446-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.753 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.753 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.754 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.754 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.755 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.755 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.756 DEBUG:boto:Data: 2022-09-03T07:15:40.756 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.757 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.758 DEBUG:boto:Params: {} 2022-09-03T07:15:40.759 DEBUG:boto:Token: None 2022-09-03T07:15:40.759 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lNCD11qz6j8JIVhfgX7bn1QwHV0= 2022-09-03T07:15:40.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lNCD11qz6j8JIVhfgX7bn1QwHV0=', 'Content-Length': '0'} 2022-09-03T07:15:40.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000627d40459019d1db-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.764 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.764 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.765 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.765 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.766 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.766 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.767 DEBUG:boto:Data: 2022-09-03T07:15:40.767 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.768 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.769 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.769 DEBUG:boto:Params: {} 2022-09-03T07:15:40.770 DEBUG:boto:Token: None 2022-09-03T07:15:40.771 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lNCD11qz6j8JIVhfgX7bn1QwHV0= 2022-09-03T07:15:40.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lNCD11qz6j8JIVhfgX7bn1QwHV0=', 'Content-Length': '0'} 2022-09-03T07:15:40.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd664fcb1a6b02ca-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.775 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:15:40.776 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.776 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.777 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.777 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.781 DEBUG:boto:Method: GET 2022-09-03T07:15:40.781 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.782 DEBUG:boto:Data: 2022-09-03T07:15:40.782 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.784 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.784 DEBUG:boto:Params: {} 2022-09-03T07:15:40.785 DEBUG:boto:Token: None 2022-09-03T07:15:40.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BtkQElFi8HynYYkklC6zKu3dlKQ= 2022-09-03T07:15:40.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BtkQElFi8HynYYkklC6zKu3dlKQ=', 'Content-Length': '0'} 2022-09-03T07:15:40.789 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000095511910a94714f8-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.830 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.831 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.832 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.832 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.833 DEBUG:boto:Method: GET 2022-09-03T07:15:40.834 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.834 DEBUG:boto:Data: 2022-09-03T07:15:40.835 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.836 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.836 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.837 DEBUG:boto:Params: {} 2022-09-03T07:15:40.838 DEBUG:boto:Token: None 2022-09-03T07:15:40.841 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:15:40.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BtkQElFi8HynYYkklC6zKu3dlKQ= 2022-09-03T07:15:40.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BtkQElFi8HynYYkklC6zKu3dlKQ=', 'Content-Length': '0'} 2022-09-03T07:15:40.845 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000803f8845951ac06-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.846 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.847 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.848 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:40.848 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:40.849 DEBUG:boto:Method: GET 2022-09-03T07:15:40.849 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:40.850 DEBUG:boto:Data: 2022-09-03T07:15:40.851 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.852 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.852 DEBUG:boto:Params: {} 2022-09-03T07:15:40.853 DEBUG:boto:Token: None 2022-09-03T07:15:40.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.858 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e1ba8d85b6805cd-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.859 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.860 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:40.860 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:40.861 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:40.862 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:40.862 DEBUG:boto:Method: GET 2022-09-03T07:15:40.863 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:40.868 DEBUG:boto:Data: 2022-09-03T07:15:40.869 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.870 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.871 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.872 DEBUG:boto:Params: {} 2022-09-03T07:15:40.873 DEBUG:boto:Token: None 2022-09-03T07:15:40.874 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/ 2022-09-03T07:15:40.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA= 2022-09-03T07:15:40.877 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wnQEitGuDm7XayMyvcd1n4pJJnA=', 'Content-Length': '0'} 2022-09-03T07:15:40.888 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000064b0331945163fbf-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.890 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:40.891 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.891 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.892 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.894 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:40.894 DEBUG:boto:Data: 2022-09-03T07:15:40.895 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.896 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.897 DEBUG:boto:Params: {} 2022-09-03T07:15:40.897 DEBUG:boto:Token: None 2022-09-03T07:15:40.898 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51 2022-09-03T07:15:40.899 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gNV6ZDpV2jqOphQKRSqAraWBuwc= 2022-09-03T07:15:40.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gNV6ZDpV2jqOphQKRSqAraWBuwc=', 'Content-Length': '0'} 2022-09-03T07:15:40.902 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000623435241a4a7561-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.903 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.904 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.905 DEBUG:boto:Method: HEAD 2022-09-03T07:15:40.905 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:40.906 DEBUG:boto:Data: 2022-09-03T07:15:40.907 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.907 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.908 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.908 DEBUG:boto:Params: {} 2022-09-03T07:15:40.909 DEBUG:boto:Token: None 2022-09-03T07:15:40.909 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51 2022-09-03T07:15:40.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gNV6ZDpV2jqOphQKRSqAraWBuwc= 2022-09-03T07:15:40.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gNV6ZDpV2jqOphQKRSqAraWBuwc=', 'Content-Length': '0'} 2022-09-03T07:15:40.916 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec4c47585692ad99-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.917 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:15:40.918 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.919 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.920 DEBUG:boto:Method: GET 2022-09-03T07:15:40.921 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:40.922 DEBUG:boto:Data: 2022-09-03T07:15:40.922 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.923 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:40.924 DEBUG:boto:Port: 8001 2022-09-03T07:15:40.925 DEBUG:boto:Params: {} 2022-09-03T07:15:40.926 DEBUG:boto:Token: None 2022-09-03T07:15:40.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51 2022-09-03T07:15:40.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zfz+VnsTX09U7ayxIg0b+pOlFgc= 2022-09-03T07:15:40.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zfz+VnsTX09U7ayxIg0b+pOlFgc=', 'Content-Length': '0'} 2022-09-03T07:15:40.936 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000000a486b4607f19e-006312ff1c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:40.938 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:15:40.939 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:15:40.940 DEBUG:boto:Method: GET 2022-09-03T07:15:40.940 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:15:40.941 DEBUG:boto:Data: 2022-09-03T07:15:40.942 DEBUG:boto:Headers: {} 2022-09-03T07:15:40.942 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:40.943 DEBUG:boto:Port: 8000 2022-09-03T07:15:40.945 DEBUG:boto:Params: {} 2022-09-03T07:15:40.945 DEBUG:boto:Token: None 2022-09-03T07:15:40.946 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:40 GMT /avysmw-75/obj51 2022-09-03T07:15:40.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zfz+VnsTX09U7ayxIg0b+pOlFgc= 2022-09-03T07:15:40.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:40 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zfz+VnsTX09U7ayxIg0b+pOlFgc=', 'Content-Length': '0'} 2022-09-03T07:15:40.951 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fab58c8c5f6a5969-006312ff1c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:40 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.001 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.002 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.003 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:41.003 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:41.005 DEBUG:boto:Method: GET 2022-09-03T07:15:41.006 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:41.006 DEBUG:boto:Data: 2022-09-03T07:15:41.007 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.009 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.010 DEBUG:boto:Params: {} 2022-09-03T07:15:41.012 DEBUG:boto:Token: None 2022-09-03T07:15:41.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.034 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000991658eb2ff89f25-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.035 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.036 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.037 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.038 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:41.039 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:15:41.043 DEBUG:boto:Method: GET 2022-09-03T07:15:41.045 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:15:41.046 DEBUG:boto:Data: 2022-09-03T07:15:41.048 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.050 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.051 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.053 DEBUG:boto:Params: {} 2022-09-03T07:15:41.054 DEBUG:boto:Token: None 2022-09-03T07:15:41.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.070 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067c0121746c8b24f-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.080 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.082 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:15:41.085 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.086 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.088 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.099 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.100 DEBUG:boto:Method: GET 2022-09-03T07:15:41.101 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.102 DEBUG:boto:Data: 2022-09-03T07:15:41.104 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.104 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.107 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.108 DEBUG:boto:Params: {} 2022-09-03T07:15:41.109 DEBUG:boto:Token: None 2022-09-03T07:15:41.109 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.110 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KD+BzXBCldkmk0jKRKZET0vZSfk= 2022-09-03T07:15:41.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KD+BzXBCldkmk0jKRKZET0vZSfk=', 'Content-Length': '0'} 2022-09-03T07:15:41.113 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000028d15fb2f07eefe8-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.114 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.115 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.116 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.117 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.117 DEBUG:boto:Method: GET 2022-09-03T07:15:41.122 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.123 DEBUG:boto:Data: 2022-09-03T07:15:41.124 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.124 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.133 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.134 DEBUG:boto:Params: {} 2022-09-03T07:15:41.135 DEBUG:boto:Token: None 2022-09-03T07:15:41.135 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.136 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KD+BzXBCldkmk0jKRKZET0vZSfk= 2022-09-03T07:15:41.137 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KD+BzXBCldkmk0jKRKZET0vZSfk=', 'Content-Length': '0'} 2022-09-03T07:15:41.146 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f78b264642d2f3ee-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.148 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.150 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.150 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.151 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.152 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.153 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.154 DEBUG:boto:Data: 2022-09-03T07:15:41.155 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.156 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.157 DEBUG:boto:Params: {} 2022-09-03T07:15:41.158 DEBUG:boto:Token: None 2022-09-03T07:15:41.159 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YcCSz12p1R+A4BuQ8QM3lb6qDs8= 2022-09-03T07:15:41.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YcCSz12p1R+A4BuQ8QM3lb6qDs8=', 'Content-Length': '0'} 2022-09-03T07:15:41.163 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed747194b9cb9820-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.164 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.165 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.169 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.174 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.176 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.177 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.182 DEBUG:boto:Data: 2022-09-03T07:15:41.183 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.184 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.185 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.186 DEBUG:boto:Params: {} 2022-09-03T07:15:41.187 DEBUG:boto:Token: None 2022-09-03T07:15:41.188 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YcCSz12p1R+A4BuQ8QM3lb6qDs8= 2022-09-03T07:15:41.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YcCSz12p1R+A4BuQ8QM3lb6qDs8=', 'Content-Length': '0'} 2022-09-03T07:15:41.225 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026585db9cd81aebf-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.226 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:15:41.227 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.228 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.228 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.233 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.233 DEBUG:boto:Method: GET 2022-09-03T07:15:41.234 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.235 DEBUG:boto:Data: 2022-09-03T07:15:41.236 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.238 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.238 DEBUG:boto:Params: {} 2022-09-03T07:15:41.239 DEBUG:boto:Token: None 2022-09-03T07:15:41.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KD+BzXBCldkmk0jKRKZET0vZSfk= 2022-09-03T07:15:41.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KD+BzXBCldkmk0jKRKZET0vZSfk=', 'Content-Length': '0'} 2022-09-03T07:15:41.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005706de4c2bbec77a-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.255 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.256 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.256 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.257 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.258 DEBUG:boto:Method: GET 2022-09-03T07:15:41.259 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.259 DEBUG:boto:Data: 2022-09-03T07:15:41.260 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.261 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.262 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.266 DEBUG:boto:Params: {} 2022-09-03T07:15:41.267 DEBUG:boto:Token: None 2022-09-03T07:15:41.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:15:41.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KD+BzXBCldkmk0jKRKZET0vZSfk= 2022-09-03T07:15:41.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KD+BzXBCldkmk0jKRKZET0vZSfk=', 'Content-Length': '0'} 2022-09-03T07:15:41.281 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f533f8bd7952aced-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.283 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.284 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.286 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:41.288 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:41.289 DEBUG:boto:Method: GET 2022-09-03T07:15:41.290 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:41.291 DEBUG:boto:Data: 2022-09-03T07:15:41.293 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.294 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.295 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.296 DEBUG:boto:Params: {} 2022-09-03T07:15:41.297 DEBUG:boto:Token: None 2022-09-03T07:15:41.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.306 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000508efb95f413d496-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.307 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.310 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.311 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.312 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:41.318 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:41.319 DEBUG:boto:Method: GET 2022-09-03T07:15:41.320 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:41.320 DEBUG:boto:Data: 2022-09-03T07:15:41.321 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.321 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.322 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.322 DEBUG:boto:Params: {} 2022-09-03T07:15:41.323 DEBUG:boto:Token: None 2022-09-03T07:15:41.324 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.327 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002fd7cb414166cbf-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.329 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.330 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.332 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.333 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.334 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:41.334 DEBUG:boto:Data: 2022-09-03T07:15:41.335 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.339 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.340 DEBUG:boto:Params: {} 2022-09-03T07:15:41.341 DEBUG:boto:Token: None 2022-09-03T07:15:41.341 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55 2022-09-03T07:15:41.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9vphTvu+lPv5rmbpWSggBYpOSo4= 2022-09-03T07:15:41.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9vphTvu+lPv5rmbpWSggBYpOSo4=', 'Content-Length': '0'} 2022-09-03T07:15:41.346 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000607b4c3dfba92d7b-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.347 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.348 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.352 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.353 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:41.354 DEBUG:boto:Data: 2022-09-03T07:15:41.354 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.355 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.355 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.356 DEBUG:boto:Params: {} 2022-09-03T07:15:41.357 DEBUG:boto:Token: None 2022-09-03T07:15:41.357 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55 2022-09-03T07:15:41.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9vphTvu+lPv5rmbpWSggBYpOSo4= 2022-09-03T07:15:41.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9vphTvu+lPv5rmbpWSggBYpOSo4=', 'Content-Length': '0'} 2022-09-03T07:15:41.362 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098b9d31cbcc14595-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.363 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:15:41.363 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.365 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.366 DEBUG:boto:Method: GET 2022-09-03T07:15:41.368 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:41.368 DEBUG:boto:Data: 2022-09-03T07:15:41.369 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.371 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.371 DEBUG:boto:Params: {} 2022-09-03T07:15:41.372 DEBUG:boto:Token: None 2022-09-03T07:15:41.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55 2022-09-03T07:15:41.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zUmKlbwlfKdLsNftB7DVyisSiy4= 2022-09-03T07:15:41.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zUmKlbwlfKdLsNftB7DVyisSiy4=', 'Content-Length': '0'} 2022-09-03T07:15:41.376 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000069995796fe274b35-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.377 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:15:41.377 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:15:41.378 DEBUG:boto:Method: GET 2022-09-03T07:15:41.378 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:15:41.379 DEBUG:boto:Data: 2022-09-03T07:15:41.379 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.380 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.381 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.381 DEBUG:boto:Params: {} 2022-09-03T07:15:41.382 DEBUG:boto:Token: None 2022-09-03T07:15:41.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj55 2022-09-03T07:15:41.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zUmKlbwlfKdLsNftB7DVyisSiy4= 2022-09-03T07:15:41.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zUmKlbwlfKdLsNftB7DVyisSiy4=', 'Content-Length': '0'} 2022-09-03T07:15:41.386 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068e3b054550da81c-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.387 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.388 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.388 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:41.388 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:41.389 DEBUG:boto:Method: GET 2022-09-03T07:15:41.390 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:41.390 DEBUG:boto:Data: 2022-09-03T07:15:41.391 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.392 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.393 DEBUG:boto:Params: {} 2022-09-03T07:15:41.393 DEBUG:boto:Token: None 2022-09-03T07:15:41.394 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.399 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003c3dbaf500842ae8-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.407 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.408 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.409 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.409 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:41.410 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:15:41.410 DEBUG:boto:Method: GET 2022-09-03T07:15:41.411 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:15:41.412 DEBUG:boto:Data: 2022-09-03T07:15:41.412 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.413 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.413 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.414 DEBUG:boto:Params: {} 2022-09-03T07:15:41.415 DEBUG:boto:Token: None 2022-09-03T07:15:41.415 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.428 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000767754cb4ee46715-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.429 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.431 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:15:41.432 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.433 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.433 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.434 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.435 DEBUG:boto:Method: GET 2022-09-03T07:15:41.436 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.437 DEBUG:boto:Data: 2022-09-03T07:15:41.438 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.440 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.441 DEBUG:boto:Params: {} 2022-09-03T07:15:41.442 DEBUG:boto:Token: None 2022-09-03T07:15:41.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IXYrHv8XJki5dm83q3kyedvEIJQ= 2022-09-03T07:15:41.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IXYrHv8XJki5dm83q3kyedvEIJQ=', 'Content-Length': '0'} 2022-09-03T07:15:41.448 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fefef2632bbed2f5-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.452 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.453 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.454 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.454 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.455 DEBUG:boto:Method: GET 2022-09-03T07:15:41.456 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.457 DEBUG:boto:Data: 2022-09-03T07:15:41.461 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.463 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.464 DEBUG:boto:Params: {} 2022-09-03T07:15:41.465 DEBUG:boto:Token: None 2022-09-03T07:15:41.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IXYrHv8XJki5dm83q3kyedvEIJQ= 2022-09-03T07:15:41.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IXYrHv8XJki5dm83q3kyedvEIJQ=', 'Content-Length': '0'} 2022-09-03T07:15:41.475 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000666b66b2109a76c0-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.477 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.478 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.479 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.481 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.484 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.485 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.486 DEBUG:boto:Data: 2022-09-03T07:15:41.487 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.492 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.494 DEBUG:boto:Params: {} 2022-09-03T07:15:41.495 DEBUG:boto:Token: None 2022-09-03T07:15:41.495 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:av8kuzSRpFCu/3LHRmBHLHe6BRs= 2022-09-03T07:15:41.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:av8kuzSRpFCu/3LHRmBHLHe6BRs=', 'Content-Length': '0'} 2022-09-03T07:15:41.505 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf8080a99cff00c3-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.506 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.507 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.508 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.508 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.510 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.511 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.512 DEBUG:boto:Data: 2022-09-03T07:15:41.512 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.513 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.514 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.515 DEBUG:boto:Params: {} 2022-09-03T07:15:41.516 DEBUG:boto:Token: None 2022-09-03T07:15:41.517 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:av8kuzSRpFCu/3LHRmBHLHe6BRs= 2022-09-03T07:15:41.521 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:av8kuzSRpFCu/3LHRmBHLHe6BRs=', 'Content-Length': '0'} 2022-09-03T07:15:41.531 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cebe116a85ddecd6-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.533 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:15:41.534 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.535 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.536 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.537 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.538 DEBUG:boto:Method: GET 2022-09-03T07:15:41.541 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.542 DEBUG:boto:Data: 2022-09-03T07:15:41.543 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.543 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.544 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.544 DEBUG:boto:Params: {} 2022-09-03T07:15:41.545 DEBUG:boto:Token: None 2022-09-03T07:15:41.545 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.546 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IXYrHv8XJki5dm83q3kyedvEIJQ= 2022-09-03T07:15:41.547 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IXYrHv8XJki5dm83q3kyedvEIJQ=', 'Content-Length': '0'} 2022-09-03T07:15:41.549 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000085b91cdc614981f-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.551 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.551 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.552 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.552 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.553 DEBUG:boto:Method: GET 2022-09-03T07:15:41.553 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.554 DEBUG:boto:Data: 2022-09-03T07:15:41.554 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.555 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.555 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.556 DEBUG:boto:Params: {} 2022-09-03T07:15:41.557 DEBUG:boto:Token: None 2022-09-03T07:15:41.557 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:15:41.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IXYrHv8XJki5dm83q3kyedvEIJQ= 2022-09-03T07:15:41.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IXYrHv8XJki5dm83q3kyedvEIJQ=', 'Content-Length': '0'} 2022-09-03T07:15:41.562 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee1dbb028b4e1c9a-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.564 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.566 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.572 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:41.573 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:41.576 DEBUG:boto:Method: GET 2022-09-03T07:15:41.578 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:41.579 DEBUG:boto:Data: 2022-09-03T07:15:41.582 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.584 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.585 DEBUG:boto:Params: {} 2022-09-03T07:15:41.586 DEBUG:boto:Token: None 2022-09-03T07:15:41.587 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.598 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ea3aa0e9b57439a-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.599 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.601 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.602 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.603 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:41.603 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:41.604 DEBUG:boto:Method: GET 2022-09-03T07:15:41.605 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:41.606 DEBUG:boto:Data: 2022-09-03T07:15:41.607 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.608 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.608 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.609 DEBUG:boto:Params: {} 2022-09-03T07:15:41.610 DEBUG:boto:Token: None 2022-09-03T07:15:41.612 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.617 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b8a5fc66502633b8-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.619 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.620 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.621 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.622 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.623 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:41.624 DEBUG:boto:Data: 2022-09-03T07:15:41.625 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.627 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.628 DEBUG:boto:Params: {} 2022-09-03T07:15:41.629 DEBUG:boto:Token: None 2022-09-03T07:15:41.630 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68 2022-09-03T07:15:41.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ikc+eXzzFBfo05gwrPNtyCUEEzs= 2022-09-03T07:15:41.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ikc+eXzzFBfo05gwrPNtyCUEEzs=', 'Content-Length': '0'} 2022-09-03T07:15:41.637 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064a1f28b75ef2e4a-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.638 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.642 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.644 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.645 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:41.646 DEBUG:boto:Data: 2022-09-03T07:15:41.647 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.648 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.649 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.649 DEBUG:boto:Params: {} 2022-09-03T07:15:41.650 DEBUG:boto:Token: None 2022-09-03T07:15:41.651 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68 2022-09-03T07:15:41.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ikc+eXzzFBfo05gwrPNtyCUEEzs= 2022-09-03T07:15:41.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ikc+eXzzFBfo05gwrPNtyCUEEzs=', 'Content-Length': '0'} 2022-09-03T07:15:41.655 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e787cf3d6c521fe7-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.656 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:15:41.656 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.657 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.657 DEBUG:boto:Method: GET 2022-09-03T07:15:41.659 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:41.660 DEBUG:boto:Data: 2022-09-03T07:15:41.661 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.664 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.665 DEBUG:boto:Params: {} 2022-09-03T07:15:41.668 DEBUG:boto:Token: None 2022-09-03T07:15:41.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68 2022-09-03T07:15:41.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yJ28DAuraXKWvADJiSJj7AoUd20= 2022-09-03T07:15:41.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yJ28DAuraXKWvADJiSJj7AoUd20=', 'Content-Length': '0'} 2022-09-03T07:15:41.675 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c40c766f0c3862ab-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.677 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:15:41.678 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:15:41.679 DEBUG:boto:Method: GET 2022-09-03T07:15:41.680 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:15:41.681 DEBUG:boto:Data: 2022-09-03T07:15:41.683 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.683 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.684 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.684 DEBUG:boto:Params: {} 2022-09-03T07:15:41.685 DEBUG:boto:Token: None 2022-09-03T07:15:41.688 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj68 2022-09-03T07:15:41.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yJ28DAuraXKWvADJiSJj7AoUd20= 2022-09-03T07:15:41.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yJ28DAuraXKWvADJiSJj7AoUd20=', 'Content-Length': '0'} 2022-09-03T07:15:41.699 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000050d8850baaacb158-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.700 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.701 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.703 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:41.704 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:41.705 DEBUG:boto:Method: GET 2022-09-03T07:15:41.705 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:41.707 DEBUG:boto:Data: 2022-09-03T07:15:41.708 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.709 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.710 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.711 DEBUG:boto:Params: {} 2022-09-03T07:15:41.712 DEBUG:boto:Token: None 2022-09-03T07:15:41.714 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.716 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.719 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a992f31636e1417c-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.720 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.721 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.722 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.723 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:41.724 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:15:41.724 DEBUG:boto:Method: GET 2022-09-03T07:15:41.725 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:15:41.725 DEBUG:boto:Data: 2022-09-03T07:15:41.726 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.727 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.727 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.728 DEBUG:boto:Params: {} 2022-09-03T07:15:41.729 DEBUG:boto:Token: None 2022-09-03T07:15:41.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.740 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a8095c495cf0c1a6-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.782 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.784 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:15:41.784 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:41.785 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:41.786 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.786 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.787 DEBUG:boto:Method: GET 2022-09-03T07:15:41.788 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.789 DEBUG:boto:Data: 2022-09-03T07:15:41.791 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.792 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.792 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.793 DEBUG:boto:Params: {} 2022-09-03T07:15:41.795 DEBUG:boto:Token: None 2022-09-03T07:15:41.796 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Nrb5G/hlwsxG6dQigAIaiW6LGM= 2022-09-03T07:15:41.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Nrb5G/hlwsxG6dQigAIaiW6LGM=', 'Content-Length': '0'} 2022-09-03T07:15:41.802 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000542738426521a7a3-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.803 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:41.804 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:41.804 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.805 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.806 DEBUG:boto:Method: GET 2022-09-03T07:15:41.806 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.808 DEBUG:boto:Data: 2022-09-03T07:15:41.809 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.810 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.811 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.812 DEBUG:boto:Params: {} 2022-09-03T07:15:41.813 DEBUG:boto:Token: None 2022-09-03T07:15:41.814 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Nrb5G/hlwsxG6dQigAIaiW6LGM= 2022-09-03T07:15:41.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Nrb5G/hlwsxG6dQigAIaiW6LGM=', 'Content-Length': '0'} 2022-09-03T07:15:41.820 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027311da97660fafe-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.821 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:41.822 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:41.822 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.823 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.823 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.824 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.824 DEBUG:boto:Data: 2022-09-03T07:15:41.825 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.825 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.826 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.827 DEBUG:boto:Params: {} 2022-09-03T07:15:41.828 DEBUG:boto:Token: None 2022-09-03T07:15:41.830 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Heaqse2U5OguBo4S/JwGshpT+w= 2022-09-03T07:15:41.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Heaqse2U5OguBo4S/JwGshpT+w=', 'Content-Length': '0'} 2022-09-03T07:15:41.835 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001dd2034a16647346-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.836 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:41.837 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:41.838 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.839 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.840 DEBUG:boto:Method: HEAD 2022-09-03T07:15:41.841 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.841 DEBUG:boto:Data: 2022-09-03T07:15:41.842 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.842 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.843 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.844 DEBUG:boto:Params: {} 2022-09-03T07:15:41.844 DEBUG:boto:Token: None 2022-09-03T07:15:41.845 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Heaqse2U5OguBo4S/JwGshpT+w= 2022-09-03T07:15:41.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Heaqse2U5OguBo4S/JwGshpT+w=', 'Content-Length': '0'} 2022-09-03T07:15:41.850 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f85a9ba2a7d93700-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.851 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:15:41.851 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:41.852 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:41.852 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.853 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.853 DEBUG:boto:Method: GET 2022-09-03T07:15:41.854 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.855 DEBUG:boto:Data: 2022-09-03T07:15:41.856 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.857 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.858 DEBUG:boto:Params: {} 2022-09-03T07:15:41.859 DEBUG:boto:Token: None 2022-09-03T07:15:41.860 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Nrb5G/hlwsxG6dQigAIaiW6LGM= 2022-09-03T07:15:41.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Nrb5G/hlwsxG6dQigAIaiW6LGM=', 'Content-Length': '0'} 2022-09-03T07:15:41.864 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000791000ac2efa3bc9-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.908 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:41.910 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:41.911 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.912 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.912 DEBUG:boto:Method: GET 2022-09-03T07:15:41.913 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.921 DEBUG:boto:Data: 2022-09-03T07:15:41.922 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.923 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.924 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.925 DEBUG:boto:Params: {} 2022-09-03T07:15:41.926 DEBUG:boto:Token: None 2022-09-03T07:15:41.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:15:41.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1Nrb5G/hlwsxG6dQigAIaiW6LGM= 2022-09-03T07:15:41.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1Nrb5G/hlwsxG6dQigAIaiW6LGM=', 'Content-Length': '0'} 2022-09-03T07:15:41.937 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a91f1b354decb6fb-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.939 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.940 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.941 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:41.943 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:41.945 DEBUG:boto:Method: GET 2022-09-03T07:15:41.952 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:41.953 DEBUG:boto:Data: 2022-09-03T07:15:41.956 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.956 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:41.957 DEBUG:boto:Port: 8001 2022-09-03T07:15:41.957 DEBUG:boto:Params: {} 2022-09-03T07:15:41.959 DEBUG:boto:Token: None 2022-09-03T07:15:41.960 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.961 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:41.965 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000276b9825f3f618b7-006312ff1d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:41.966 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:41.968 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:41.969 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:41.976 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:41.977 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:41.980 DEBUG:boto:Method: GET 2022-09-03T07:15:41.980 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:41.981 DEBUG:boto:Data: 2022-09-03T07:15:41.982 DEBUG:boto:Headers: {} 2022-09-03T07:15:41.983 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:41.983 DEBUG:boto:Port: 8000 2022-09-03T07:15:41.984 DEBUG:boto:Params: {} 2022-09-03T07:15:41.987 DEBUG:boto:Token: None 2022-09-03T07:15:41.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:41 GMT /avysmw-75/ 2022-09-03T07:15:41.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A= 2022-09-03T07:15:41.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:41 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dpC9/UHSM3eg7rlfgawSeEzA//A=', 'Content-Length': '0'} 2022-09-03T07:15:42.002 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f3a52f687505f9c2-006312ff1d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:41 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.015 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:42.016 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:42.017 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:42.018 DEBUG:boto:Method: HEAD 2022-09-03T07:15:42.020 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:42.021 DEBUG:boto:Data: 2022-09-03T07:15:42.022 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.023 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:42.024 DEBUG:boto:Port: 8001 2022-09-03T07:15:42.025 DEBUG:boto:Params: {} 2022-09-03T07:15:42.026 DEBUG:boto:Token: None 2022-09-03T07:15:42.027 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/obj85 2022-09-03T07:15:42.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5h140g1OGxfKkRSqaaUJfRE9EQE= 2022-09-03T07:15:42.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5h140g1OGxfKkRSqaaUJfRE9EQE=', 'Content-Length': '0'} 2022-09-03T07:15:42.035 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f599b608a6b05da-006312ff1e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.037 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:42.038 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:42.039 DEBUG:boto:Method: HEAD 2022-09-03T07:15:42.039 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:42.040 DEBUG:boto:Data: 2022-09-03T07:15:42.041 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.045 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:42.045 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.047 DEBUG:boto:Params: {} 2022-09-03T07:15:42.049 DEBUG:boto:Token: None 2022-09-03T07:15:42.049 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/obj85 2022-09-03T07:15:42.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5h140g1OGxfKkRSqaaUJfRE9EQE= 2022-09-03T07:15:42.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5h140g1OGxfKkRSqaaUJfRE9EQE=', 'Content-Length': '0'} 2022-09-03T07:15:42.066 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c19512f2bdcce8a6-006312ff1e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.068 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:15:42.072 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:42.073 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:42.073 DEBUG:boto:Method: GET 2022-09-03T07:15:42.074 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:42.075 DEBUG:boto:Data: 2022-09-03T07:15:42.075 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:42.077 DEBUG:boto:Port: 8001 2022-09-03T07:15:42.078 DEBUG:boto:Params: {} 2022-09-03T07:15:42.078 DEBUG:boto:Token: None 2022-09-03T07:15:42.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/obj85 2022-09-03T07:15:42.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KjaUhpN+St16RCDXNw4ykwip5Aw= 2022-09-03T07:15:42.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KjaUhpN+St16RCDXNw4ykwip5Aw=', 'Content-Length': '0'} 2022-09-03T07:15:42.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9920263bb705662-006312ff1e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.087 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:15:42.088 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:15:42.089 DEBUG:boto:Method: GET 2022-09-03T07:15:42.089 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:15:42.090 DEBUG:boto:Data: 2022-09-03T07:15:42.090 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.091 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:42.091 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.092 DEBUG:boto:Params: {} 2022-09-03T07:15:42.093 DEBUG:boto:Token: None 2022-09-03T07:15:42.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/obj85 2022-09-03T07:15:42.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KjaUhpN+St16RCDXNw4ykwip5Aw= 2022-09-03T07:15:42.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KjaUhpN+St16RCDXNw4ykwip5Aw=', 'Content-Length': '0'} 2022-09-03T07:15:42.100 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a5adf857ea8468c-006312ff1e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.101 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:42.102 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:42.102 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:42.103 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:42.103 DEBUG:boto:Method: GET 2022-09-03T07:15:42.104 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:42.104 DEBUG:boto:Data: 2022-09-03T07:15:42.105 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:15:42.112 DEBUG:boto:Port: 8001 2022-09-03T07:15:42.113 DEBUG:boto:Params: {} 2022-09-03T07:15:42.115 DEBUG:boto:Token: None 2022-09-03T07:15:42.116 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/ 2022-09-03T07:15:42.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QBlpy+z5L5EWS5cYXYCKLMqs+Qs= 2022-09-03T07:15:42.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QBlpy+z5L5EWS5cYXYCKLMqs+Qs=', 'Content-Length': '0'} 2022-09-03T07:15:42.130 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a65a3a7542017ca4-006312ff1e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.131 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:42.135 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:42.137 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:42.138 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:42.139 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:15:42.140 DEBUG:boto:Method: GET 2022-09-03T07:15:42.140 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:15:42.141 DEBUG:boto:Data: 2022-09-03T07:15:42.142 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.143 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:15:42.144 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.145 DEBUG:boto:Params: {} 2022-09-03T07:15:42.147 DEBUG:boto:Token: None 2022-09-03T07:15:42.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/ 2022-09-03T07:15:42.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QBlpy+z5L5EWS5cYXYCKLMqs+Qs= 2022-09-03T07:15:42.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QBlpy+z5L5EWS5cYXYCKLMqs+Qs=', 'Content-Length': '0'} 2022-09-03T07:15:42.153 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002fe7d8d9fcfcf173-006312ff1e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.155 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:15:42.156 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:15:42.158 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '19', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:15:42.159 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '19', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:15:42.161 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --num-shards 19 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:15:42.225 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:15:42.226 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:15:42.282 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:15:42.290 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-75 2022-09-03T07:15:42.292 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 10 2022-09-03T07:15:42.370 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:42.371 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:42.374 DEBUG:boto:Method: HEAD 2022-09-03T07:15:42.375 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:42.377 DEBUG:boto:Data: 2022-09-03T07:15:42.378 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.380 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.381 DEBUG:boto:Params: {} 2022-09-03T07:15:42.384 DEBUG:boto:Token: None 2022-09-03T07:15:42.385 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/ 2022-09-03T07:15:42.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw= 2022-09-03T07:15:42.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw=', 'Content-Length': '0'} 2022-09-03T07:15:42.390 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004f6cc14f9c5ce78b-006312ff1e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.392 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:15:42.393 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:15:42.403 DEBUG:boto:Method: PUT 2022-09-03T07:15:42.404 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:15:42.405 DEBUG:boto:Data: 2022-09-03T07:15:42.406 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:15:42.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.408 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.409 DEBUG:boto:Params: {} 2022-09-03T07:15:42.410 DEBUG:boto:Token: None 2022-09-03T07:15:42.411 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/obj19 2022-09-03T07:15:42.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tgYDqVU4V0tROWL0TtGj5AGFMPs= 2022-09-03T07:15:42.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tgYDqVU4V0tROWL0TtGj5AGFMPs='} 2022-09-03T07:15:42.421 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000003a52fa34d2063171-006312ff1e-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.422 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:42.423 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:42.424 DEBUG:boto:Method: HEAD 2022-09-03T07:15:42.425 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:42.426 DEBUG:boto:Data: 2022-09-03T07:15:42.429 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.430 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.431 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.432 DEBUG:boto:Params: {} 2022-09-03T07:15:42.434 DEBUG:boto:Token: None 2022-09-03T07:15:42.435 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/ 2022-09-03T07:15:42.436 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw= 2022-09-03T07:15:42.437 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw=', 'Content-Length': '0'} 2022-09-03T07:15:42.445 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '11'), ('X-RGW-Bytes-Used', '33'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001c40820f2d33e59c-006312ff1e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.447 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:15:42.448 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:15:42.449 DEBUG:boto:Method: PUT 2022-09-03T07:15:42.450 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:15:42.451 DEBUG:boto:Data: 2022-09-03T07:15:42.452 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:15:42.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.454 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.455 DEBUG:boto:Params: {} 2022-09-03T07:15:42.456 DEBUG:boto:Token: None 2022-09-03T07:15:42.457 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/obj38 2022-09-03T07:15:42.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HJVpPzHV9ZBoqQ2SG7lPtTkjJjI= 2022-09-03T07:15:42.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HJVpPzHV9ZBoqQ2SG7lPtTkjJjI='} 2022-09-03T07:15:42.508 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000097dfa3e6d786fd19-006312ff1e-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.509 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:42.509 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:42.510 DEBUG:boto:Method: HEAD 2022-09-03T07:15:42.510 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:42.511 DEBUG:boto:Data: 2022-09-03T07:15:42.512 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.513 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.514 DEBUG:boto:Params: {} 2022-09-03T07:15:42.515 DEBUG:boto:Token: None 2022-09-03T07:15:42.516 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/ 2022-09-03T07:15:42.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw= 2022-09-03T07:15:42.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw=', 'Content-Length': '0'} 2022-09-03T07:15:42.520 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '12'), ('X-RGW-Bytes-Used', '36'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000091e7dec3d40711dc-006312ff1e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.521 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:15:42.522 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:15:42.523 DEBUG:boto:Method: PUT 2022-09-03T07:15:42.524 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:15:42.524 DEBUG:boto:Data: 2022-09-03T07:15:42.525 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:15:42.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.526 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.527 DEBUG:boto:Params: {} 2022-09-03T07:15:42.528 DEBUG:boto:Token: None 2022-09-03T07:15:42.528 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/obj57 2022-09-03T07:15:42.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v+MdvIB7opxxZMDmEkmbUs2bZRc= 2022-09-03T07:15:42.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v+MdvIB7opxxZMDmEkmbUs2bZRc='} 2022-09-03T07:15:42.559 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000be89a84b04ebd74f-006312ff1e-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.560 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:42.561 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:42.561 DEBUG:boto:Method: HEAD 2022-09-03T07:15:42.562 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:42.562 DEBUG:boto:Data: 2022-09-03T07:15:42.563 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.565 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.566 DEBUG:boto:Params: {} 2022-09-03T07:15:42.566 DEBUG:boto:Token: None 2022-09-03T07:15:42.568 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/ 2022-09-03T07:15:42.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw= 2022-09-03T07:15:42.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw=', 'Content-Length': '0'} 2022-09-03T07:15:42.578 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '13'), ('X-RGW-Bytes-Used', '39'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cbe465c2501b11b4-006312ff1e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.579 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:15:42.581 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:15:42.582 DEBUG:boto:Method: PUT 2022-09-03T07:15:42.583 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:15:42.583 DEBUG:boto:Data: 2022-09-03T07:15:42.585 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:15:42.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.587 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.587 DEBUG:boto:Params: {} 2022-09-03T07:15:42.588 DEBUG:boto:Token: None 2022-09-03T07:15:42.589 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/obj76 2022-09-03T07:15:42.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ukX5OiqZOL4UNIKTPTWGs0iDvuE= 2022-09-03T07:15:42.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ukX5OiqZOL4UNIKTPTWGs0iDvuE='} 2022-09-03T07:15:42.635 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000003d80bd54a819fbf4-006312ff1e-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.637 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:15:42.639 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:15:42.641 DEBUG:boto:Method: HEAD 2022-09-03T07:15:42.641 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:15:42.642 DEBUG:boto:Data: 2022-09-03T07:15:42.644 DEBUG:boto:Headers: {} 2022-09-03T07:15:42.645 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.646 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.647 DEBUG:boto:Params: {} 2022-09-03T07:15:42.648 DEBUG:boto:Token: None 2022-09-03T07:15:42.649 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/ 2022-09-03T07:15:42.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw= 2022-09-03T07:15:42.651 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kBpZheWY0cxOJDjwcD/RTj487mw=', 'Content-Length': '0'} 2022-09-03T07:15:42.654 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '14'), ('X-RGW-Bytes-Used', '42'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f1888aefb094819d-006312ff1e-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.655 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:15:42.655 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:15:42.656 DEBUG:boto:Method: PUT 2022-09-03T07:15:42.656 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:15:42.657 DEBUG:boto:Data: 2022-09-03T07:15:42.657 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:15:42.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:15:42.661 DEBUG:boto:Port: 8000 2022-09-03T07:15:42.662 DEBUG:boto:Params: {} 2022-09-03T07:15:42.663 DEBUG:boto:Token: None 2022-09-03T07:15:42.664 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:15:42 GMT /avysmw-75/obj95 2022-09-03T07:15:42.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YXvQO4+d4/WpgFWdPvlXdOu0CSU= 2022-09-03T07:15:42.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:15:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YXvQO4+d4/WpgFWdPvlXdOu0CSU='} 2022-09-03T07:15:42.674 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f3423b43357c77ab-006312ff1e-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:15:42.675 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-75 2022-09-03T07:15:42.676 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:42.677 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:15:42.679 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:15:42.748 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:15:42.749 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:15:42.793 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:42.789+0000 7fb49d84f600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T07:15:47.800 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:47.790+0000 7fb49d84f600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T07:15:52.792 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:52.790+0000 7fb49d84f600 1 waiting to reach latest gen 2, on 1.. 2022-09-03T07:15:57.798 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:15:57.796+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:15:57.799 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , ] 2022-09-03T07:15:57.799 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:02.799 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:02.797+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:02.799 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , ] 2022-09-03T07:16:02.800 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:07.802 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:07.799+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:07.803 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , ] 2022-09-03T07:16:07.803 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:12.802 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:12.800+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:12.803 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , ] 2022-09-03T07:16:12.803 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:17.804 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:17.801+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:17.805 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:17.806 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:22.807 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:22.803+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:22.808 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:22.808 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:27.809 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:27.804+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:27.810 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:27.810 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:32.809 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:32.806+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:32.810 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:32.810 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:37.809 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:37.807+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:37.810 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:37.811 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:42.811 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:42.808+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:42.811 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:42.812 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:47.814 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:47.809+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:47.816 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:47.817 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:52.813 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:52.812+0000 7fb49d84f600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:52.814 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:52.815 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:57.816 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:57.812+0000 7fb49d84f600 1 bucket sync caught up with source: 2022-09-03T07:16:57.816 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:57.816 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:57.817 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:16:57.812+0000 7fb49d84f600 0 bucket checkpoint complete 2022-09-03T07:16:57.838 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-75 2022-09-03T07:16:57.839 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:16:57.840 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:16:57.855 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:16:57.923 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:16:57.924 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:16:57.968 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:16:57.967+0000 7f2e07410600 1 waiting for incremental sync to catch up: 2022-09-03T07:16:57.970 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:16:57.970 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:02.973 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:17:02.969+0000 7f2e07410600 1 bucket sync caught up with source: 2022-09-03T07:17:02.974 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:02.975 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:02.976 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:17:02.969+0000 7f2e07410600 0 bucket checkpoint complete 2022-09-03T07:17:02.994 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-75 2022-09-03T07:17:02.995 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:17:02.996 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:17:02.997 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:17:03.125 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:17:03.125 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:17:03.186 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:03.184+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:03.187 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , ] 2022-09-03T07:17:03.187 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:08.192 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:08.186+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:08.192 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:08.193 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:13.190 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:13.188+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:13.190 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:13.191 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:18.194 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:18.189+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:18.195 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:18.195 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:23.204 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:23.191+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:23.205 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:23.205 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:28.199 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:28.195+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:28.200 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:28.200 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:33.199 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:33.197+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:33.199 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:33.200 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:38.200 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:38.198+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:38.201 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:38.202 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:43.202 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:43.200+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:43.202 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:43.203 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:48.203 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:48.201+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:48.204 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:48.204 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:53.205 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:53.203+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:53.206 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:53.206 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:17:58.207 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:17:58.205+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:17:58.207 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:17:58.208 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:03.208 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:03.206+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:03.209 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:03.209 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:08.210 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:08.208+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:08.210 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:08.211 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:13.213 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:13.209+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:13.213 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:13.214 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:18.213 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:18.211+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:18.214 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:18.214 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:23.214 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:23.212+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:23.214 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:23.215 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:28.215 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:28.213+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:28.216 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:28.216 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:33.216 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:33.215+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:33.217 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:33.217 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:38.218 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:38.216+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:38.219 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:38.219 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:43.220 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:43.218+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:43.221 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:43.221 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:48.221 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:48.219+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:48.222 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:48.222 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:53.223 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:53.221+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:53.223 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:53.223 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:18:58.224 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:18:58.222+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:18:58.225 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:18:58.226 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:03.226 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:03.224+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:03.227 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:03.227 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:08.228 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:08.226+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:08.229 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:08.229 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:13.232 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:13.230+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:13.233 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:13.233 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:18.233 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:18.232+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:18.234 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:18.234 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:23.235 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:23.233+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:23.235 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:23.236 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:28.236 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:28.235+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:28.237 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:28.237 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:33.238 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:33.236+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:33.238 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:33.238 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:38.240 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:38.238+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:38.241 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:38.241 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:43.243 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:43.240+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:43.243 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:43.244 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:48.243 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:48.241+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:48.244 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:48.244 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:53.244 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:53.243+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:53.245 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:53.245 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:19:58.246 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:19:58.244+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:19:58.246 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:19:58.247 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:03.250 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:03.248+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:03.251 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:03.251 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:08.252 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:08.250+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:08.252 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:08.253 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:13.253 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:13.251+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:13.254 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:13.254 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:18.257 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:18.255+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:18.257 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:18.258 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:23.273 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:23.271+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:23.273 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:23.274 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:28.274 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:28.272+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:28.275 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:28.275 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:33.276 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:33.273+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:33.276 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:33.277 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:38.277 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:38.275+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:38.278 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:38.278 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:43.278 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:43.276+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:43.279 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:43.279 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:48.279 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:48.278+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:48.280 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:48.280 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:53.281 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:53.279+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:53.281 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:53.282 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:20:58.282 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:20:58.280+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:20:58.283 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:20:58.283 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:21:03.284 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:03.282+0000 7f4bef6fd600 1 waiting for incremental sync to catch up: 2022-09-03T07:21:03.284 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, , , , , , , , , , , , ] 2022-09-03T07:21:03.285 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:21:08.285 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:08.283+0000 7f4bef6fd600 1 bucket sync caught up with source: 2022-09-03T07:21:08.286 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:21:08.286 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.94.5, 00000000004.61.5, , , , , , , , , , , ] 2022-09-03T07:21:08.286 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:08.283+0000 7f4bef6fd600 0 bucket checkpoint complete 2022-09-03T07:21:08.323 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-75 2022-09-03T07:21:08.326 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:08.327 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:08.329 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:21:08.397 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:21:08.398 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:21:08.451 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:21:08.449+0000 7fcda2361600 1 bucket sync caught up with source: 2022-09-03T07:21:08.451 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.118.5, 00000000001.325.5, 00000000001.376.5, 00000000001.331.5, 00000000001.252.5, , , 00000000002.376.5, 00000000004.299.5, 00000000003.215.5] 2022-09-03T07:21:08.452 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.118.5, 00000000001.325.5, 00000000001.376.5, 00000000001.331.5, 00000000001.252.5, , , 00000000002.376.5, 00000000004.299.5, 00000000003.215.5] 2022-09-03T07:21:08.452 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:21:08.449+0000 7fcda2361600 0 bucket checkpoint complete 2022-09-03T07:21:08.466 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-75 2022-09-03T07:21:08.467 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:08.467 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:08.468 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:21:08.528 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:21:08.529 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:21:08.577 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:21:08.575+0000 7ff1bf7cd600 1 bucket sync caught up with source: 2022-09-03T07:21:08.577 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.118.5, 00000000001.325.5, 00000000001.376.5, 00000000001.331.5, 00000000001.252.5, , , 00000000002.376.5, 00000000004.299.5, 00000000003.215.5] 2022-09-03T07:21:08.577 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.118.5, 00000000001.325.5, 00000000001.376.5, 00000000001.331.5, 00000000001.252.5, , , 00000000002.376.5, 00000000004.299.5, 00000000003.215.5] 2022-09-03T07:21:08.578 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:21:08.575+0000 7ff1bf7cd600 0 bucket checkpoint complete 2022-09-03T07:21:08.599 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-75 2022-09-03T07:21:08.600 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-75 2022-09-03T07:21:08.601 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:08.602 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:08.603 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:21:08.661 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:21:08.662 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:21:08.715 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:21:08.714+0000 7f3f3d8fa600 1 bucket sync caught up with source: 2022-09-03T07:21:08.716 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.116.5, 00000000001.175.5, 00000000001.185.5, 00000000001.185.5, 00000000001.136.5, , , 00000000002.287.5, 00000000004.308.5, 00000000003.131.5] 2022-09-03T07:21:08.716 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.116.5, 00000000001.175.5, 00000000001.185.5, 00000000001.185.5, 00000000001.136.5, , , 00000000002.287.5, 00000000004.308.5, 00000000003.131.5] 2022-09-03T07:21:08.717 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:21:08.714+0000 7f3f3d8fa600 0 bucket checkpoint complete 2022-09-03T07:21:08.734 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-75 2022-09-03T07:21:08.734 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:08.735 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:08.736 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:21:08.928 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:21:08.929 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:21:09.984 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:09.982+0000 7f2231881600 1 bucket sync caught up with source: 2022-09-03T07:21:09.984 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.116.5, 00000000001.175.5, 00000000001.185.5, 00000000001.185.5, 00000000001.136.5, , , 00000000002.287.5, 00000000004.308.5, 00000000003.131.5] 2022-09-03T07:21:09.985 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.116.5, 00000000001.175.5, 00000000001.185.5, 00000000001.185.5, 00000000001.136.5, , , 00000000002.287.5, 00000000004.308.5, 00000000003.131.5] 2022-09-03T07:21:09.985 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:09.982+0000 7f2231881600 0 bucket checkpoint complete 2022-09-03T07:21:10.005 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-75 2022-09-03T07:21:10.006 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:21:10.007 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.007 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.008 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.009 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:10.009 DEBUG:boto:Data: 2022-09-03T07:21:10.010 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.010 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.011 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.011 DEBUG:boto:Params: {} 2022-09-03T07:21:10.013 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:21:10.013 DEBUG:boto:Token: None 2022-09-03T07:21:10.014 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ix4VJSmD1IrHQYm33BeZ5jt63Bg= 2022-09-03T07:21:10.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ix4VJSmD1IrHQYm33BeZ5jt63Bg=', 'Content-Length': '0'} 2022-09-03T07:21:10.023 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '15'), ('X-RGW-Bytes-Used', '45'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000012e636e0c4a15513-0063130066-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.024 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.025 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.026 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.026 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:10.027 DEBUG:boto:Data: 2022-09-03T07:21:10.027 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.028 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.029 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.029 DEBUG:boto:Params: {} 2022-09-03T07:21:10.030 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:21:10.031 DEBUG:boto:Token: None 2022-09-03T07:21:10.031 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ix4VJSmD1IrHQYm33BeZ5jt63Bg= 2022-09-03T07:21:10.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ix4VJSmD1IrHQYm33BeZ5jt63Bg=', 'Content-Length': '0'} 2022-09-03T07:21:10.039 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '15'), ('X-RGW-Bytes-Used', '45'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d668197f3f2709c5-0063130066-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.041 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:21:10.041 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.042 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.043 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.043 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.044 DEBUG:boto:Method: GET 2022-09-03T07:21:10.045 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.046 DEBUG:boto:Data: 2022-09-03T07:21:10.046 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.047 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.047 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.048 DEBUG:boto:Params: {} 2022-09-03T07:21:10.049 DEBUG:boto:Token: None 2022-09-03T07:21:10.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/?versions 2022-09-03T07:21:10.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YvkERuq1j+JsrXxqoegQ2wSNIOU= 2022-09-03T07:21:10.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YvkERuq1j+JsrXxqoegQ2wSNIOU=', 'Content-Length': '0'} 2022-09-03T07:21:10.053 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e5f65a6389806d8-0063130066-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.054 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.057 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:21:10.058 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:21:10.058 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:21:10.059 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:21:10.059 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:21:10.060 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:21:10.061 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:21:10.061 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:21:10.062 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:21:10.063 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:21:10.063 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:21:10.064 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:21:10.064 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:21:10.065 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:21:10.065 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:21:10.066 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:21:10.067 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.067 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.068 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.068 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.070 DEBUG:boto:Method: GET 2022-09-03T07:21:10.070 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.071 DEBUG:boto:Data: 2022-09-03T07:21:10.071 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.072 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.073 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.073 DEBUG:boto:Params: {} 2022-09-03T07:21:10.074 DEBUG:boto:Token: None 2022-09-03T07:21:10.074 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/?versions 2022-09-03T07:21:10.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YvkERuq1j+JsrXxqoegQ2wSNIOU= 2022-09-03T07:21:10.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YvkERuq1j+JsrXxqoegQ2wSNIOU=', 'Content-Length': '0'} 2022-09-03T07:21:10.079 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000837a35960ac191c3-0063130066-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.080 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.082 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:21:10.082 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:21:10.083 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:21:10.083 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:21:10.084 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:21:10.084 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:21:10.085 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:21:10.085 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:21:10.085 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:21:10.086 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:21:10.086 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:21:10.087 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:21:10.087 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:21:10.088 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:21:10.088 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:21:10.089 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.089 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.090 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.090 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.091 DEBUG:boto:Method: GET 2022-09-03T07:21:10.091 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.092 DEBUG:boto:Data: 2022-09-03T07:21:10.092 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.093 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.093 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.094 DEBUG:boto:Params: {} 2022-09-03T07:21:10.094 DEBUG:boto:Token: None 2022-09-03T07:21:10.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/?versions 2022-09-03T07:21:10.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YvkERuq1j+JsrXxqoegQ2wSNIOU= 2022-09-03T07:21:10.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YvkERuq1j+JsrXxqoegQ2wSNIOU=', 'Content-Length': '0'} 2022-09-03T07:21:10.099 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7529533003ba718-0063130066-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.099 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.101 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.101 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.102 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.102 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.103 DEBUG:boto:Method: GET 2022-09-03T07:21:10.103 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:10.104 DEBUG:boto:Data: 2022-09-03T07:21:10.104 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.105 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.106 DEBUG:boto:Params: {} 2022-09-03T07:21:10.107 DEBUG:boto:Token: None 2022-09-03T07:21:10.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/?versions 2022-09-03T07:21:10.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YvkERuq1j+JsrXxqoegQ2wSNIOU= 2022-09-03T07:21:10.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YvkERuq1j+JsrXxqoegQ2wSNIOU=', 'Content-Length': '0'} 2022-09-03T07:21:10.112 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a2752dc4ec63cb8-0063130066-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.112 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.114 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:21:10.115 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.116 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.116 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.117 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.117 DEBUG:boto:Method: GET 2022-09-03T07:21:10.118 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.118 DEBUG:boto:Data: 2022-09-03T07:21:10.119 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.120 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.120 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.121 DEBUG:boto:Params: {} 2022-09-03T07:21:10.121 DEBUG:boto:Token: None 2022-09-03T07:21:10.122 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.123 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pu2dKiuSdigJhcgDihj158rcKbw= 2022-09-03T07:21:10.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pu2dKiuSdigJhcgDihj158rcKbw=', 'Content-Length': '0'} 2022-09-03T07:21:10.126 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005a568914652b45a-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.127 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.128 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.129 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.129 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.130 DEBUG:boto:Method: GET 2022-09-03T07:21:10.130 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.131 DEBUG:boto:Data: 2022-09-03T07:21:10.132 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.133 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.133 DEBUG:boto:Params: {} 2022-09-03T07:21:10.134 DEBUG:boto:Token: None 2022-09-03T07:21:10.135 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pu2dKiuSdigJhcgDihj158rcKbw= 2022-09-03T07:21:10.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pu2dKiuSdigJhcgDihj158rcKbw=', 'Content-Length': '0'} 2022-09-03T07:21:10.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093b8103be385c868-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.182 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.236 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.237 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.238 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.238 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.239 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.240 DEBUG:boto:Data: 2022-09-03T07:21:10.240 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.241 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.241 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.242 DEBUG:boto:Params: {} 2022-09-03T07:21:10.244 DEBUG:boto:Token: None 2022-09-03T07:21:10.246 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QL/N2rUstFfLoUT0mnfxu0wiuuU= 2022-09-03T07:21:10.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QL/N2rUstFfLoUT0mnfxu0wiuuU=', 'Content-Length': '0'} 2022-09-03T07:21:10.252 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007207cdce7493e666-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.253 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.253 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.254 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.255 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.256 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.257 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.257 DEBUG:boto:Data: 2022-09-03T07:21:10.258 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.258 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.259 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.259 DEBUG:boto:Params: {} 2022-09-03T07:21:10.260 DEBUG:boto:Token: None 2022-09-03T07:21:10.261 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QL/N2rUstFfLoUT0mnfxu0wiuuU= 2022-09-03T07:21:10.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QL/N2rUstFfLoUT0mnfxu0wiuuU=', 'Content-Length': '0'} 2022-09-03T07:21:10.264 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000028927314ab396e56-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.265 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:21:10.266 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.267 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.268 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.269 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.269 DEBUG:boto:Method: GET 2022-09-03T07:21:10.270 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.271 DEBUG:boto:Data: 2022-09-03T07:21:10.272 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.272 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.273 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.273 DEBUG:boto:Params: {} 2022-09-03T07:21:10.274 DEBUG:boto:Token: None 2022-09-03T07:21:10.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pu2dKiuSdigJhcgDihj158rcKbw= 2022-09-03T07:21:10.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pu2dKiuSdigJhcgDihj158rcKbw=', 'Content-Length': '0'} 2022-09-03T07:21:10.280 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b98bb0101d7ded5-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.326 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.327 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.327 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.328 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.328 DEBUG:boto:Method: GET 2022-09-03T07:21:10.329 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.329 DEBUG:boto:Data: 2022-09-03T07:21:10.330 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.331 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.331 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.332 DEBUG:boto:Params: {} 2022-09-03T07:21:10.332 DEBUG:boto:Token: None 2022-09-03T07:21:10.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:10.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pu2dKiuSdigJhcgDihj158rcKbw= 2022-09-03T07:21:10.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pu2dKiuSdigJhcgDihj158rcKbw=', 'Content-Length': '0'} 2022-09-03T07:21:10.337 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b8d17a869e4d096-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.337 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.338 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.339 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:10.339 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:10.340 DEBUG:boto:Method: GET 2022-09-03T07:21:10.341 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:10.341 DEBUG:boto:Data: 2022-09-03T07:21:10.342 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.343 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.344 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.345 DEBUG:boto:Params: {} 2022-09-03T07:21:10.346 DEBUG:boto:Token: None 2022-09-03T07:21:10.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.352 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000054389247cb367dcc-0063130066-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.352 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.353 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.354 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.354 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:10.355 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:10.355 DEBUG:boto:Method: GET 2022-09-03T07:21:10.356 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:10.356 DEBUG:boto:Data: 2022-09-03T07:21:10.357 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.357 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.358 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.359 DEBUG:boto:Params: {} 2022-09-03T07:21:10.359 DEBUG:boto:Token: None 2022-09-03T07:21:10.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.364 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031808bfe16cc277e-0063130066-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.365 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.367 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.368 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.369 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.370 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:10.371 DEBUG:boto:Data: 2022-09-03T07:21:10.372 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.373 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.374 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.375 DEBUG:boto:Params: {} 2022-09-03T07:21:10.376 DEBUG:boto:Token: None 2022-09-03T07:21:10.377 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11 2022-09-03T07:21:10.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZ2uxK6DD3IIS5Cc11nyvzTm6sI= 2022-09-03T07:21:10.379 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZ2uxK6DD3IIS5Cc11nyvzTm6sI=', 'Content-Length': '0'} 2022-09-03T07:21:10.381 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012716c5d5f3d8018-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.383 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.384 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.384 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.385 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:10.386 DEBUG:boto:Data: 2022-09-03T07:21:10.387 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.389 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.390 DEBUG:boto:Params: {} 2022-09-03T07:21:10.391 DEBUG:boto:Token: None 2022-09-03T07:21:10.392 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11 2022-09-03T07:21:10.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TZ2uxK6DD3IIS5Cc11nyvzTm6sI= 2022-09-03T07:21:10.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TZ2uxK6DD3IIS5Cc11nyvzTm6sI=', 'Content-Length': '0'} 2022-09-03T07:21:10.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b61e8938c50e9d79-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.398 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:21:10.399 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.400 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.401 DEBUG:boto:Method: GET 2022-09-03T07:21:10.402 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:10.403 DEBUG:boto:Data: 2022-09-03T07:21:10.404 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.405 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.406 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.406 DEBUG:boto:Params: {} 2022-09-03T07:21:10.407 DEBUG:boto:Token: None 2022-09-03T07:21:10.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11 2022-09-03T07:21:10.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0LKk/DonJiTVdAkbk0bHNHHRAmI= 2022-09-03T07:21:10.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0LKk/DonJiTVdAkbk0bHNHHRAmI=', 'Content-Length': '0'} 2022-09-03T07:21:10.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c00fa92f881d8bb9-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.412 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:10.412 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:10.413 DEBUG:boto:Method: GET 2022-09-03T07:21:10.413 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:10.414 DEBUG:boto:Data: 2022-09-03T07:21:10.415 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.415 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.416 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.416 DEBUG:boto:Params: {} 2022-09-03T07:21:10.417 DEBUG:boto:Token: None 2022-09-03T07:21:10.417 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj11 2022-09-03T07:21:10.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0LKk/DonJiTVdAkbk0bHNHHRAmI= 2022-09-03T07:21:10.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0LKk/DonJiTVdAkbk0bHNHHRAmI=', 'Content-Length': '0'} 2022-09-03T07:21:10.421 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5fccc40779bda99-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.422 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.422 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.423 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:10.423 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:10.424 DEBUG:boto:Method: GET 2022-09-03T07:21:10.424 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:10.425 DEBUG:boto:Data: 2022-09-03T07:21:10.425 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.426 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.426 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.427 DEBUG:boto:Params: {} 2022-09-03T07:21:10.428 DEBUG:boto:Token: None 2022-09-03T07:21:10.429 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.429 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.430 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.432 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000237e2785d0b164ca-0063130066-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.433 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.434 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.434 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.435 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:10.436 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:10.437 DEBUG:boto:Method: GET 2022-09-03T07:21:10.438 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:10.439 DEBUG:boto:Data: 2022-09-03T07:21:10.440 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.442 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.443 DEBUG:boto:Params: {} 2022-09-03T07:21:10.444 DEBUG:boto:Token: None 2022-09-03T07:21:10.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.450 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8f8521349cb232e-0063130066-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.451 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.452 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:21:10.452 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.453 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.453 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.454 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.454 DEBUG:boto:Method: GET 2022-09-03T07:21:10.455 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.455 DEBUG:boto:Data: 2022-09-03T07:21:10.456 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.456 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.457 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.458 DEBUG:boto:Params: {} 2022-09-03T07:21:10.458 DEBUG:boto:Token: None 2022-09-03T07:21:10.459 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5XBs4Zk8lt8mjP0fO3udHPghZTQ= 2022-09-03T07:21:10.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5XBs4Zk8lt8mjP0fO3udHPghZTQ=', 'Content-Length': '0'} 2022-09-03T07:21:10.462 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006be927c900c31208-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.463 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.463 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.464 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.464 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.465 DEBUG:boto:Method: GET 2022-09-03T07:21:10.466 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.466 DEBUG:boto:Data: 2022-09-03T07:21:10.467 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.468 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.468 DEBUG:boto:Params: {} 2022-09-03T07:21:10.469 DEBUG:boto:Token: None 2022-09-03T07:21:10.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5XBs4Zk8lt8mjP0fO3udHPghZTQ= 2022-09-03T07:21:10.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5XBs4Zk8lt8mjP0fO3udHPghZTQ=', 'Content-Length': '0'} 2022-09-03T07:21:10.473 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000184c83002d277a38-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.474 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.474 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.475 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.475 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.476 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.476 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.477 DEBUG:boto:Data: 2022-09-03T07:21:10.477 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.478 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.478 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.479 DEBUG:boto:Params: {} 2022-09-03T07:21:10.479 DEBUG:boto:Token: None 2022-09-03T07:21:10.480 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pywsR6Lx5hHiiWUSuVqU80yO5zU= 2022-09-03T07:21:10.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pywsR6Lx5hHiiWUSuVqU80yO5zU=', 'Content-Length': '0'} 2022-09-03T07:21:10.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053f4299035da51fb-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.484 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.484 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.485 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.485 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.486 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.487 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.487 DEBUG:boto:Data: 2022-09-03T07:21:10.488 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.489 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.489 DEBUG:boto:Params: {} 2022-09-03T07:21:10.490 DEBUG:boto:Token: None 2022-09-03T07:21:10.490 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pywsR6Lx5hHiiWUSuVqU80yO5zU= 2022-09-03T07:21:10.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pywsR6Lx5hHiiWUSuVqU80yO5zU=', 'Content-Length': '0'} 2022-09-03T07:21:10.494 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c43c1f54fd6ce758-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.494 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:21:10.495 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.496 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.496 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.497 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.497 DEBUG:boto:Method: GET 2022-09-03T07:21:10.498 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.498 DEBUG:boto:Data: 2022-09-03T07:21:10.499 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.499 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.500 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.500 DEBUG:boto:Params: {} 2022-09-03T07:21:10.501 DEBUG:boto:Token: None 2022-09-03T07:21:10.501 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5XBs4Zk8lt8mjP0fO3udHPghZTQ= 2022-09-03T07:21:10.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5XBs4Zk8lt8mjP0fO3udHPghZTQ=', 'Content-Length': '0'} 2022-09-03T07:21:10.504 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000082800de740238b2c-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.546 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.547 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.548 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.549 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.550 DEBUG:boto:Method: GET 2022-09-03T07:21:10.551 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.551 DEBUG:boto:Data: 2022-09-03T07:21:10.552 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.553 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.554 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.555 DEBUG:boto:Params: {} 2022-09-03T07:21:10.556 DEBUG:boto:Token: None 2022-09-03T07:21:10.556 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:10.557 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5XBs4Zk8lt8mjP0fO3udHPghZTQ= 2022-09-03T07:21:10.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5XBs4Zk8lt8mjP0fO3udHPghZTQ=', 'Content-Length': '0'} 2022-09-03T07:21:10.560 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf66b82463eeb991-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.561 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.562 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.562 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:10.563 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:10.563 DEBUG:boto:Method: GET 2022-09-03T07:21:10.564 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:10.564 DEBUG:boto:Data: 2022-09-03T07:21:10.565 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.565 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.566 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.566 DEBUG:boto:Params: {} 2022-09-03T07:21:10.567 DEBUG:boto:Token: None 2022-09-03T07:21:10.567 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.574 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dbd40a1c0c7ab211-0063130066-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.575 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.576 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.577 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.578 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:10.578 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:10.579 DEBUG:boto:Method: GET 2022-09-03T07:21:10.579 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:10.581 DEBUG:boto:Data: 2022-09-03T07:21:10.581 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.582 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.583 DEBUG:boto:Params: {} 2022-09-03T07:21:10.583 DEBUG:boto:Token: None 2022-09-03T07:21:10.584 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.588 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000874596ffb664baa6-0063130066-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.589 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.591 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.591 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.592 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.592 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:10.593 DEBUG:boto:Data: 2022-09-03T07:21:10.593 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.594 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.595 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.596 DEBUG:boto:Params: {} 2022-09-03T07:21:10.597 DEBUG:boto:Token: None 2022-09-03T07:21:10.598 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17 2022-09-03T07:21:10.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QEboooE4uVr8FY0zyE1/44568vM= 2022-09-03T07:21:10.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QEboooE4uVr8FY0zyE1/44568vM=', 'Content-Length': '0'} 2022-09-03T07:21:10.603 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad9fdc737e07932b-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.604 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.605 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.606 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.607 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:10.608 DEBUG:boto:Data: 2022-09-03T07:21:10.609 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.609 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.610 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.611 DEBUG:boto:Params: {} 2022-09-03T07:21:10.612 DEBUG:boto:Token: None 2022-09-03T07:21:10.612 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17 2022-09-03T07:21:10.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QEboooE4uVr8FY0zyE1/44568vM= 2022-09-03T07:21:10.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QEboooE4uVr8FY0zyE1/44568vM=', 'Content-Length': '0'} 2022-09-03T07:21:10.616 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2fc45c454a27d15-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.617 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:21:10.617 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.618 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.618 DEBUG:boto:Method: GET 2022-09-03T07:21:10.619 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:10.619 DEBUG:boto:Data: 2022-09-03T07:21:10.620 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.620 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.621 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.622 DEBUG:boto:Params: {} 2022-09-03T07:21:10.622 DEBUG:boto:Token: None 2022-09-03T07:21:10.623 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17 2022-09-03T07:21:10.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3pVkqAfwizRfSP7oMwvruM7Fn/U= 2022-09-03T07:21:10.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3pVkqAfwizRfSP7oMwvruM7Fn/U=', 'Content-Length': '0'} 2022-09-03T07:21:10.626 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000905be15f3ee563b9-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.627 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:10.627 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:10.628 DEBUG:boto:Method: GET 2022-09-03T07:21:10.628 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:10.629 DEBUG:boto:Data: 2022-09-03T07:21:10.629 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.630 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.631 DEBUG:boto:Params: {} 2022-09-03T07:21:10.632 DEBUG:boto:Token: None 2022-09-03T07:21:10.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj17 2022-09-03T07:21:10.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3pVkqAfwizRfSP7oMwvruM7Fn/U= 2022-09-03T07:21:10.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3pVkqAfwizRfSP7oMwvruM7Fn/U=', 'Content-Length': '0'} 2022-09-03T07:21:10.636 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004605a631a1aead3a-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.637 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.638 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.638 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:10.639 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:10.639 DEBUG:boto:Method: GET 2022-09-03T07:21:10.640 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:10.641 DEBUG:boto:Data: 2022-09-03T07:21:10.641 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.642 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.642 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.643 DEBUG:boto:Params: {} 2022-09-03T07:21:10.644 DEBUG:boto:Token: None 2022-09-03T07:21:10.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.650 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a1dc01eb5e968224-0063130066-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.694 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.695 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.696 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.696 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:10.697 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:10.697 DEBUG:boto:Method: GET 2022-09-03T07:21:10.698 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:10.698 DEBUG:boto:Data: 2022-09-03T07:21:10.699 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.700 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.700 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.701 DEBUG:boto:Params: {} 2022-09-03T07:21:10.701 DEBUG:boto:Token: None 2022-09-03T07:21:10.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.706 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6d8463a86770526-0063130066-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.707 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.708 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:21:10.709 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.709 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.710 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.710 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.711 DEBUG:boto:Method: GET 2022-09-03T07:21:10.712 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.713 DEBUG:boto:Data: 2022-09-03T07:21:10.714 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.715 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.716 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.716 DEBUG:boto:Params: {} 2022-09-03T07:21:10.717 DEBUG:boto:Token: None 2022-09-03T07:21:10.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sArtPzo3JhrihoMn7M+B9jFlcuw= 2022-09-03T07:21:10.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sArtPzo3JhrihoMn7M+B9jFlcuw=', 'Content-Length': '0'} 2022-09-03T07:21:10.721 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003229c5fd6064aa87-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.722 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.722 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.723 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.723 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.724 DEBUG:boto:Method: GET 2022-09-03T07:21:10.724 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.725 DEBUG:boto:Data: 2022-09-03T07:21:10.725 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.726 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.727 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.727 DEBUG:boto:Params: {} 2022-09-03T07:21:10.728 DEBUG:boto:Token: None 2022-09-03T07:21:10.728 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sArtPzo3JhrihoMn7M+B9jFlcuw= 2022-09-03T07:21:10.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sArtPzo3JhrihoMn7M+B9jFlcuw=', 'Content-Length': '0'} 2022-09-03T07:21:10.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000416ebfcd162aa1f8-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.733 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.734 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.735 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.735 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.736 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.737 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.737 DEBUG:boto:Data: 2022-09-03T07:21:10.738 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.738 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.739 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.739 DEBUG:boto:Params: {} 2022-09-03T07:21:10.740 DEBUG:boto:Token: None 2022-09-03T07:21:10.741 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PmZk/s4TJrF0FBNeORVPpiyURm0= 2022-09-03T07:21:10.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PmZk/s4TJrF0FBNeORVPpiyURm0=', 'Content-Length': '0'} 2022-09-03T07:21:10.743 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2008cfe1c516ec8-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.744 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.745 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.745 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.746 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.746 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.747 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.747 DEBUG:boto:Data: 2022-09-03T07:21:10.748 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.749 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.749 DEBUG:boto:Params: {} 2022-09-03T07:21:10.750 DEBUG:boto:Token: None 2022-09-03T07:21:10.751 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PmZk/s4TJrF0FBNeORVPpiyURm0= 2022-09-03T07:21:10.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PmZk/s4TJrF0FBNeORVPpiyURm0=', 'Content-Length': '0'} 2022-09-03T07:21:10.753 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eac896f0833c75fb-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.754 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:21:10.755 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.755 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.756 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.757 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.757 DEBUG:boto:Method: GET 2022-09-03T07:21:10.758 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.759 DEBUG:boto:Data: 2022-09-03T07:21:10.760 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.761 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.762 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.762 DEBUG:boto:Params: {} 2022-09-03T07:21:10.763 DEBUG:boto:Token: None 2022-09-03T07:21:10.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sArtPzo3JhrihoMn7M+B9jFlcuw= 2022-09-03T07:21:10.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sArtPzo3JhrihoMn7M+B9jFlcuw=', 'Content-Length': '0'} 2022-09-03T07:21:10.768 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039ec1a4c0fb53cad-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.769 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.769 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.770 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.770 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.771 DEBUG:boto:Method: GET 2022-09-03T07:21:10.772 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.772 DEBUG:boto:Data: 2022-09-03T07:21:10.773 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.775 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.775 DEBUG:boto:Params: {} 2022-09-03T07:21:10.776 DEBUG:boto:Token: None 2022-09-03T07:21:10.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:10.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sArtPzo3JhrihoMn7M+B9jFlcuw= 2022-09-03T07:21:10.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sArtPzo3JhrihoMn7M+B9jFlcuw=', 'Content-Length': '0'} 2022-09-03T07:21:10.780 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3a9819d2c5593db-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.781 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.782 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.783 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:10.784 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:10.784 DEBUG:boto:Method: GET 2022-09-03T07:21:10.785 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:10.785 DEBUG:boto:Data: 2022-09-03T07:21:10.786 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.787 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.787 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.788 DEBUG:boto:Params: {} 2022-09-03T07:21:10.788 DEBUG:boto:Token: None 2022-09-03T07:21:10.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.793 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008fed31b67b03216d-0063130066-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.794 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.794 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.795 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.795 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:10.796 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:10.797 DEBUG:boto:Method: GET 2022-09-03T07:21:10.797 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:10.798 DEBUG:boto:Data: 2022-09-03T07:21:10.798 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.799 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.799 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.800 DEBUG:boto:Params: {} 2022-09-03T07:21:10.801 DEBUG:boto:Token: None 2022-09-03T07:21:10.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.806 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001349e733e4acc677-0063130066-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.808 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.809 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.810 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.811 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.811 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:10.812 DEBUG:boto:Data: 2022-09-03T07:21:10.812 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.813 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.813 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.814 DEBUG:boto:Params: {} 2022-09-03T07:21:10.814 DEBUG:boto:Token: None 2022-09-03T07:21:10.815 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19 2022-09-03T07:21:10.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9YVH89c8asf/TK3VnKv6hduq0HY= 2022-09-03T07:21:10.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9YVH89c8asf/TK3VnKv6hduq0HY=', 'Content-Length': '0'} 2022-09-03T07:21:10.818 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d33341cbc77fd218-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.819 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.819 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.820 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.820 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:10.821 DEBUG:boto:Data: 2022-09-03T07:21:10.821 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.823 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.823 DEBUG:boto:Params: {} 2022-09-03T07:21:10.824 DEBUG:boto:Token: None 2022-09-03T07:21:10.824 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19 2022-09-03T07:21:10.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9YVH89c8asf/TK3VnKv6hduq0HY= 2022-09-03T07:21:10.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9YVH89c8asf/TK3VnKv6hduq0HY=', 'Content-Length': '0'} 2022-09-03T07:21:10.828 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c06b2765f425a9b2-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.828 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:21:10.829 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.830 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.831 DEBUG:boto:Method: GET 2022-09-03T07:21:10.831 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:10.832 DEBUG:boto:Data: 2022-09-03T07:21:10.833 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.834 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.835 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.836 DEBUG:boto:Params: {} 2022-09-03T07:21:10.837 DEBUG:boto:Token: None 2022-09-03T07:21:10.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19 2022-09-03T07:21:10.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sHgOLzK3Mv8Tw+vHYk4rV+mrJAY= 2022-09-03T07:21:10.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sHgOLzK3Mv8Tw+vHYk4rV+mrJAY=', 'Content-Length': '0'} 2022-09-03T07:21:10.842 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000981a84d309a4c58d-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.843 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:10.843 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:10.844 DEBUG:boto:Method: GET 2022-09-03T07:21:10.844 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:10.845 DEBUG:boto:Data: 2022-09-03T07:21:10.845 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.847 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.847 DEBUG:boto:Params: {} 2022-09-03T07:21:10.848 DEBUG:boto:Token: None 2022-09-03T07:21:10.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj19 2022-09-03T07:21:10.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sHgOLzK3Mv8Tw+vHYk4rV+mrJAY= 2022-09-03T07:21:10.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sHgOLzK3Mv8Tw+vHYk4rV+mrJAY=', 'Content-Length': '0'} 2022-09-03T07:21:10.852 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000267a6a8c43e91b3b-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.853 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.854 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.855 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:10.856 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:10.857 DEBUG:boto:Method: GET 2022-09-03T07:21:10.857 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:10.858 DEBUG:boto:Data: 2022-09-03T07:21:10.858 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.859 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.859 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.860 DEBUG:boto:Params: {} 2022-09-03T07:21:10.861 DEBUG:boto:Token: None 2022-09-03T07:21:10.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.865 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000082a90bb045dac270-0063130066-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.865 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.866 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.867 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.867 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:10.868 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:10.868 DEBUG:boto:Method: GET 2022-09-03T07:21:10.869 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:10.869 DEBUG:boto:Data: 2022-09-03T07:21:10.870 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.870 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.871 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.871 DEBUG:boto:Params: {} 2022-09-03T07:21:10.872 DEBUG:boto:Token: None 2022-09-03T07:21:10.873 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/ 2022-09-03T07:21:10.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4= 2022-09-03T07:21:10.874 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LATWxacih6vylaMf9mt1tdiufs4=', 'Content-Length': '0'} 2022-09-03T07:21:10.877 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006411f07b1b402371-0063130066-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.878 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:10.878 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:21:10.879 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:10.880 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:10.880 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.881 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.881 DEBUG:boto:Method: GET 2022-09-03T07:21:10.882 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.882 DEBUG:boto:Data: 2022-09-03T07:21:10.883 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.883 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.884 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.884 DEBUG:boto:Params: {} 2022-09-03T07:21:10.885 DEBUG:boto:Token: None 2022-09-03T07:21:10.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qxh/isW3oS7/L9/IBfW6pZVxzzg= 2022-09-03T07:21:10.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qxh/isW3oS7/L9/IBfW6pZVxzzg=', 'Content-Length': '0'} 2022-09-03T07:21:10.889 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ba76d732b681be7-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.890 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:10.890 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:10.891 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.891 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.892 DEBUG:boto:Method: GET 2022-09-03T07:21:10.892 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.893 DEBUG:boto:Data: 2022-09-03T07:21:10.893 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.894 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.894 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.895 DEBUG:boto:Params: {} 2022-09-03T07:21:10.896 DEBUG:boto:Token: None 2022-09-03T07:21:10.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qxh/isW3oS7/L9/IBfW6pZVxzzg= 2022-09-03T07:21:10.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qxh/isW3oS7/L9/IBfW6pZVxzzg=', 'Content-Length': '0'} 2022-09-03T07:21:10.900 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000740b5a4f5a8088c1-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.901 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:10.902 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:10.902 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.903 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.903 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.904 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.904 DEBUG:boto:Data: 2022-09-03T07:21:10.905 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.905 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.906 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.906 DEBUG:boto:Params: {} 2022-09-03T07:21:10.907 DEBUG:boto:Token: None 2022-09-03T07:21:10.907 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wCRTyDaEd4PeKO6nxr0vD1xc6/w= 2022-09-03T07:21:10.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wCRTyDaEd4PeKO6nxr0vD1xc6/w=', 'Content-Length': '0'} 2022-09-03T07:21:10.911 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b703543b1b1352f7-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.911 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:10.912 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:10.912 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.913 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.913 DEBUG:boto:Method: HEAD 2022-09-03T07:21:10.914 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.914 DEBUG:boto:Data: 2022-09-03T07:21:10.915 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.916 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.916 DEBUG:boto:Params: {} 2022-09-03T07:21:10.917 DEBUG:boto:Token: None 2022-09-03T07:21:10.917 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wCRTyDaEd4PeKO6nxr0vD1xc6/w= 2022-09-03T07:21:10.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wCRTyDaEd4PeKO6nxr0vD1xc6/w=', 'Content-Length': '0'} 2022-09-03T07:21:10.921 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065ba931c61378aae-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.922 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:21:10.922 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:10.923 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:10.923 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.924 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.924 DEBUG:boto:Method: GET 2022-09-03T07:21:10.925 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.925 DEBUG:boto:Data: 2022-09-03T07:21:10.926 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.926 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:10.927 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.927 DEBUG:boto:Params: {} 2022-09-03T07:21:10.928 DEBUG:boto:Token: None 2022-09-03T07:21:10.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qxh/isW3oS7/L9/IBfW6pZVxzzg= 2022-09-03T07:21:10.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qxh/isW3oS7/L9/IBfW6pZVxzzg=', 'Content-Length': '0'} 2022-09-03T07:21:10.932 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003df246d362477679-0063130066-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.974 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:10.975 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:10.977 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.977 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.979 DEBUG:boto:Method: GET 2022-09-03T07:21:10.980 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.980 DEBUG:boto:Data: 2022-09-03T07:21:10.981 DEBUG:boto:Headers: {} 2022-09-03T07:21:10.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:10.984 DEBUG:boto:Port: 8000 2022-09-03T07:21:10.985 DEBUG:boto:Params: {} 2022-09-03T07:21:10.986 DEBUG:boto:Token: None 2022-09-03T07:21:10.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:10 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:10.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qxh/isW3oS7/L9/IBfW6pZVxzzg= 2022-09-03T07:21:10.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qxh/isW3oS7/L9/IBfW6pZVxzzg=', 'Content-Length': '0'} 2022-09-03T07:21:10.992 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d65574b3af6f445f-0063130066-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:10.994 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:10.995 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:10.996 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:10.997 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:10.998 DEBUG:boto:Method: GET 2022-09-03T07:21:10.999 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:11.000 DEBUG:boto:Data: 2022-09-03T07:21:11.001 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.002 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.003 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.004 DEBUG:boto:Params: {} 2022-09-03T07:21:11.005 DEBUG:boto:Token: None 2022-09-03T07:21:11.006 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.011 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f74b0130dd5a486-0063130067-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.012 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.013 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.014 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.015 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:11.016 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:11.017 DEBUG:boto:Method: GET 2022-09-03T07:21:11.018 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:11.019 DEBUG:boto:Data: 2022-09-03T07:21:11.020 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.022 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.023 DEBUG:boto:Params: {} 2022-09-03T07:21:11.024 DEBUG:boto:Token: None 2022-09-03T07:21:11.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.031 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f14d0ae22cda94f9-0063130067-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.033 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.034 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:11.035 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:11.036 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.037 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:11.038 DEBUG:boto:Data: 2022-09-03T07:21:11.039 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.039 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.040 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.041 DEBUG:boto:Params: {} 2022-09-03T07:21:11.041 DEBUG:boto:Token: None 2022-09-03T07:21:11.042 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj22 2022-09-03T07:21:11.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0KU1FQukkfJewCdIz3AJAJjla0M= 2022-09-03T07:21:11.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0KU1FQukkfJewCdIz3AJAJjla0M=', 'Content-Length': '0'} 2022-09-03T07:21:11.047 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000041b6cb3388f997-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.048 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:11.049 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:11.049 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.050 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:11.051 DEBUG:boto:Data: 2022-09-03T07:21:11.051 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.052 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.052 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.053 DEBUG:boto:Params: {} 2022-09-03T07:21:11.053 DEBUG:boto:Token: None 2022-09-03T07:21:11.054 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj22 2022-09-03T07:21:11.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0KU1FQukkfJewCdIz3AJAJjla0M= 2022-09-03T07:21:11.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0KU1FQukkfJewCdIz3AJAJjla0M=', 'Content-Length': '0'} 2022-09-03T07:21:11.057 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000946834bdb91edcb1-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.058 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:21:11.058 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:11.059 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:11.059 DEBUG:boto:Method: GET 2022-09-03T07:21:11.060 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:11.061 DEBUG:boto:Data: 2022-09-03T07:21:11.061 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.062 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.062 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.063 DEBUG:boto:Params: {} 2022-09-03T07:21:11.064 DEBUG:boto:Token: None 2022-09-03T07:21:11.065 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj22 2022-09-03T07:21:11.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EVAiPOoZNRyEqYVknl6XKKbldeA= 2022-09-03T07:21:11.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EVAiPOoZNRyEqYVknl6XKKbldeA=', 'Content-Length': '0'} 2022-09-03T07:21:11.068 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000095834b5d6786832-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.069 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:11.070 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:11.070 DEBUG:boto:Method: GET 2022-09-03T07:21:11.071 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:11.071 DEBUG:boto:Data: 2022-09-03T07:21:11.072 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.073 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.074 DEBUG:boto:Params: {} 2022-09-03T07:21:11.074 DEBUG:boto:Token: None 2022-09-03T07:21:11.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj22 2022-09-03T07:21:11.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EVAiPOoZNRyEqYVknl6XKKbldeA= 2022-09-03T07:21:11.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EVAiPOoZNRyEqYVknl6XKKbldeA=', 'Content-Length': '0'} 2022-09-03T07:21:11.078 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bcf0003f6d835fd8-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.122 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.123 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.124 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:11.124 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:11.125 DEBUG:boto:Method: GET 2022-09-03T07:21:11.125 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:11.126 DEBUG:boto:Data: 2022-09-03T07:21:11.126 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.127 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.128 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.128 DEBUG:boto:Params: {} 2022-09-03T07:21:11.129 DEBUG:boto:Token: None 2022-09-03T07:21:11.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000010c4c0b9878208c-0063130067-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.135 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.136 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.136 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.137 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:11.137 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:11.138 DEBUG:boto:Method: GET 2022-09-03T07:21:11.138 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:11.139 DEBUG:boto:Data: 2022-09-03T07:21:11.139 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.140 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.141 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.141 DEBUG:boto:Params: {} 2022-09-03T07:21:11.142 DEBUG:boto:Token: None 2022-09-03T07:21:11.142 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.147 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000064b6269e526bb337-0063130067-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.148 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.149 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:21:11.150 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.151 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.151 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.152 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.152 DEBUG:boto:Method: GET 2022-09-03T07:21:11.153 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.153 DEBUG:boto:Data: 2022-09-03T07:21:11.154 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.154 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.155 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.155 DEBUG:boto:Params: {} 2022-09-03T07:21:11.156 DEBUG:boto:Token: None 2022-09-03T07:21:11.157 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hxazYPOHZk8ZaLYHVJYm1jqwNJ8= 2022-09-03T07:21:11.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hxazYPOHZk8ZaLYHVJYm1jqwNJ8=', 'Content-Length': '0'} 2022-09-03T07:21:11.160 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000033841c1cfe93af85-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.202 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.203 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.204 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.205 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.206 DEBUG:boto:Method: GET 2022-09-03T07:21:11.207 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.207 DEBUG:boto:Data: 2022-09-03T07:21:11.208 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.209 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.210 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.211 DEBUG:boto:Params: {} 2022-09-03T07:21:11.211 DEBUG:boto:Token: None 2022-09-03T07:21:11.212 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hxazYPOHZk8ZaLYHVJYm1jqwNJ8= 2022-09-03T07:21:11.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hxazYPOHZk8ZaLYHVJYm1jqwNJ8=', 'Content-Length': '0'} 2022-09-03T07:21:11.220 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc2f38fdcfd3e5c3-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.223 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.223 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.224 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.224 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.225 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.226 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.226 DEBUG:boto:Data: 2022-09-03T07:21:11.227 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.228 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.229 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.230 DEBUG:boto:Params: {} 2022-09-03T07:21:11.231 DEBUG:boto:Token: None 2022-09-03T07:21:11.232 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jcAh+7RIorziIClauw6IfVg5Udk= 2022-09-03T07:21:11.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jcAh+7RIorziIClauw6IfVg5Udk=', 'Content-Length': '0'} 2022-09-03T07:21:11.235 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008cb6f2134ac40e5e-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.237 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.237 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.238 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.239 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.239 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.240 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.240 DEBUG:boto:Data: 2022-09-03T07:21:11.241 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.241 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.242 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.242 DEBUG:boto:Params: {} 2022-09-03T07:21:11.243 DEBUG:boto:Token: None 2022-09-03T07:21:11.244 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jcAh+7RIorziIClauw6IfVg5Udk= 2022-09-03T07:21:11.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jcAh+7RIorziIClauw6IfVg5Udk=', 'Content-Length': '0'} 2022-09-03T07:21:11.247 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074a7672a623051af-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.247 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:21:11.248 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.248 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.249 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.250 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.251 DEBUG:boto:Method: GET 2022-09-03T07:21:11.252 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.253 DEBUG:boto:Data: 2022-09-03T07:21:11.254 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.255 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.256 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.257 DEBUG:boto:Params: {} 2022-09-03T07:21:11.258 DEBUG:boto:Token: None 2022-09-03T07:21:11.259 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hxazYPOHZk8ZaLYHVJYm1jqwNJ8= 2022-09-03T07:21:11.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hxazYPOHZk8ZaLYHVJYm1jqwNJ8=', 'Content-Length': '0'} 2022-09-03T07:21:11.263 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000050fed25b4f4efaa0-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.264 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.265 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.265 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.266 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.267 DEBUG:boto:Method: GET 2022-09-03T07:21:11.267 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.268 DEBUG:boto:Data: 2022-09-03T07:21:11.268 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.269 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.270 DEBUG:boto:Params: {} 2022-09-03T07:21:11.271 DEBUG:boto:Token: None 2022-09-03T07:21:11.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:11.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hxazYPOHZk8ZaLYHVJYm1jqwNJ8= 2022-09-03T07:21:11.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hxazYPOHZk8ZaLYHVJYm1jqwNJ8=', 'Content-Length': '0'} 2022-09-03T07:21:11.275 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c8b9e3a72557467-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.322 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.323 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.323 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:11.324 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:11.325 DEBUG:boto:Method: GET 2022-09-03T07:21:11.325 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:11.326 DEBUG:boto:Data: 2022-09-03T07:21:11.327 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.328 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.329 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.330 DEBUG:boto:Params: {} 2022-09-03T07:21:11.331 DEBUG:boto:Token: None 2022-09-03T07:21:11.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.337 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c2a5df8d255b1de7-0063130067-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.338 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.340 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.340 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.341 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:11.342 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:11.343 DEBUG:boto:Method: GET 2022-09-03T07:21:11.344 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:11.345 DEBUG:boto:Data: 2022-09-03T07:21:11.346 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.346 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.347 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.348 DEBUG:boto:Params: {} 2022-09-03T07:21:11.348 DEBUG:boto:Token: None 2022-09-03T07:21:11.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.353 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001097dbf181727afa-0063130067-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.354 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.355 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.355 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.356 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.356 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:11.357 DEBUG:boto:Data: 2022-09-03T07:21:11.357 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.358 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.358 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.359 DEBUG:boto:Params: {} 2022-09-03T07:21:11.359 DEBUG:boto:Token: None 2022-09-03T07:21:11.360 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33 2022-09-03T07:21:11.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:evl7GWyECYJdeCvI9ndLg1RcwvI= 2022-09-03T07:21:11.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:evl7GWyECYJdeCvI9ndLg1RcwvI=', 'Content-Length': '0'} 2022-09-03T07:21:11.363 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000732f9cd0a903c429-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.364 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.364 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.365 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.366 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:11.366 DEBUG:boto:Data: 2022-09-03T07:21:11.367 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.368 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.368 DEBUG:boto:Params: {} 2022-09-03T07:21:11.369 DEBUG:boto:Token: None 2022-09-03T07:21:11.370 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33 2022-09-03T07:21:11.371 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:evl7GWyECYJdeCvI9ndLg1RcwvI= 2022-09-03T07:21:11.371 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:evl7GWyECYJdeCvI9ndLg1RcwvI=', 'Content-Length': '0'} 2022-09-03T07:21:11.373 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e2f0ff76ec6f6b6-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.375 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:21:11.375 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.376 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.376 DEBUG:boto:Method: GET 2022-09-03T07:21:11.377 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:11.378 DEBUG:boto:Data: 2022-09-03T07:21:11.378 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.379 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.380 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.380 DEBUG:boto:Params: {} 2022-09-03T07:21:11.381 DEBUG:boto:Token: None 2022-09-03T07:21:11.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33 2022-09-03T07:21:11.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xdavCKeEvl3w255xmRCQsCB+wdI= 2022-09-03T07:21:11.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xdavCKeEvl3w255xmRCQsCB+wdI=', 'Content-Length': '0'} 2022-09-03T07:21:11.385 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035e2bfe21bbcd7e5-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.386 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:11.386 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:11.387 DEBUG:boto:Method: GET 2022-09-03T07:21:11.387 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:11.388 DEBUG:boto:Data: 2022-09-03T07:21:11.388 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.389 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.390 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.390 DEBUG:boto:Params: {} 2022-09-03T07:21:11.392 DEBUG:boto:Token: None 2022-09-03T07:21:11.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj33 2022-09-03T07:21:11.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xdavCKeEvl3w255xmRCQsCB+wdI= 2022-09-03T07:21:11.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xdavCKeEvl3w255xmRCQsCB+wdI=', 'Content-Length': '0'} 2022-09-03T07:21:11.396 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8dc13ec6dd28242-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.397 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.398 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.399 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:11.400 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:11.401 DEBUG:boto:Method: GET 2022-09-03T07:21:11.401 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:11.402 DEBUG:boto:Data: 2022-09-03T07:21:11.403 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.404 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.405 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.405 DEBUG:boto:Params: {} 2022-09-03T07:21:11.406 DEBUG:boto:Token: None 2022-09-03T07:21:11.407 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.415 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce21020b7999ddbf-0063130067-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.416 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.417 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.417 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.418 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:11.418 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:11.419 DEBUG:boto:Method: GET 2022-09-03T07:21:11.419 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:11.420 DEBUG:boto:Data: 2022-09-03T07:21:11.421 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.422 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.422 DEBUG:boto:Params: {} 2022-09-03T07:21:11.423 DEBUG:boto:Token: None 2022-09-03T07:21:11.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.428 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d55c3c855fa53a6e-0063130067-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.428 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.429 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:21:11.430 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.430 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.431 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.431 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.432 DEBUG:boto:Method: GET 2022-09-03T07:21:11.432 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.433 DEBUG:boto:Data: 2022-09-03T07:21:11.434 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.435 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.435 DEBUG:boto:Params: {} 2022-09-03T07:21:11.436 DEBUG:boto:Token: None 2022-09-03T07:21:11.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yy747GtjhlreSqrf6N03SucHwUA= 2022-09-03T07:21:11.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yy747GtjhlreSqrf6N03SucHwUA=', 'Content-Length': '0'} 2022-09-03T07:21:11.440 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c25caa801a0c20da-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.441 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.441 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.442 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.442 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.443 DEBUG:boto:Method: GET 2022-09-03T07:21:11.444 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.444 DEBUG:boto:Data: 2022-09-03T07:21:11.445 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.446 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.446 DEBUG:boto:Params: {} 2022-09-03T07:21:11.447 DEBUG:boto:Token: None 2022-09-03T07:21:11.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yy747GtjhlreSqrf6N03SucHwUA= 2022-09-03T07:21:11.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yy747GtjhlreSqrf6N03SucHwUA=', 'Content-Length': '0'} 2022-09-03T07:21:11.451 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032da88970711781b-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.452 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.453 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.454 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.455 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.456 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.457 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.458 DEBUG:boto:Data: 2022-09-03T07:21:11.459 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.460 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.461 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.462 DEBUG:boto:Params: {} 2022-09-03T07:21:11.463 DEBUG:boto:Token: None 2022-09-03T07:21:11.464 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+8ZOjmEkpDz3a4N45L0dHbJGnu0= 2022-09-03T07:21:11.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+8ZOjmEkpDz3a4N45L0dHbJGnu0=', 'Content-Length': '0'} 2022-09-03T07:21:11.468 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037a0908e3e63710c-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.469 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.471 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.472 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.473 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.474 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.475 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.476 DEBUG:boto:Data: 2022-09-03T07:21:11.477 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.477 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.478 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.479 DEBUG:boto:Params: {} 2022-09-03T07:21:11.480 DEBUG:boto:Token: None 2022-09-03T07:21:11.481 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+8ZOjmEkpDz3a4N45L0dHbJGnu0= 2022-09-03T07:21:11.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+8ZOjmEkpDz3a4N45L0dHbJGnu0=', 'Content-Length': '0'} 2022-09-03T07:21:11.486 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cae3593bfc64dff1-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.488 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:21:11.489 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.490 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.491 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.492 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.493 DEBUG:boto:Method: GET 2022-09-03T07:21:11.494 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.495 DEBUG:boto:Data: 2022-09-03T07:21:11.496 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.497 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.498 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.499 DEBUG:boto:Params: {} 2022-09-03T07:21:11.501 DEBUG:boto:Token: None 2022-09-03T07:21:11.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yy747GtjhlreSqrf6N03SucHwUA= 2022-09-03T07:21:11.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yy747GtjhlreSqrf6N03SucHwUA=', 'Content-Length': '0'} 2022-09-03T07:21:11.507 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000240fa85aefada1f1-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.508 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.509 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.509 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.510 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.510 DEBUG:boto:Method: GET 2022-09-03T07:21:11.511 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.511 DEBUG:boto:Data: 2022-09-03T07:21:11.512 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.513 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.513 DEBUG:boto:Params: {} 2022-09-03T07:21:11.514 DEBUG:boto:Token: None 2022-09-03T07:21:11.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:11.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yy747GtjhlreSqrf6N03SucHwUA= 2022-09-03T07:21:11.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yy747GtjhlreSqrf6N03SucHwUA=', 'Content-Length': '0'} 2022-09-03T07:21:11.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005aa44f0147edfe81-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.566 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.567 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.568 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:11.568 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:11.569 DEBUG:boto:Method: GET 2022-09-03T07:21:11.570 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:11.570 DEBUG:boto:Data: 2022-09-03T07:21:11.571 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.571 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.572 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.572 DEBUG:boto:Params: {} 2022-09-03T07:21:11.573 DEBUG:boto:Token: None 2022-09-03T07:21:11.574 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.578 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ddf412378de36058-0063130067-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.579 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.580 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.580 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.581 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:11.581 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:11.582 DEBUG:boto:Method: GET 2022-09-03T07:21:11.583 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:11.583 DEBUG:boto:Data: 2022-09-03T07:21:11.584 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.584 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.585 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.585 DEBUG:boto:Params: {} 2022-09-03T07:21:11.586 DEBUG:boto:Token: None 2022-09-03T07:21:11.586 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.588 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.592 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fe4a931daadab7a5-0063130067-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.593 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.593 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.594 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.595 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.595 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:11.596 DEBUG:boto:Data: 2022-09-03T07:21:11.596 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.597 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.597 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.598 DEBUG:boto:Params: {} 2022-09-03T07:21:11.598 DEBUG:boto:Token: None 2022-09-03T07:21:11.599 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34 2022-09-03T07:21:11.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xJz95bIs/lkN+l8IjgzGv7CGfqc= 2022-09-03T07:21:11.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xJz95bIs/lkN+l8IjgzGv7CGfqc=', 'Content-Length': '0'} 2022-09-03T07:21:11.602 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e43a10f6af7ac6fd-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.603 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.604 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.604 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.605 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:11.605 DEBUG:boto:Data: 2022-09-03T07:21:11.606 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.606 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.607 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.607 DEBUG:boto:Params: {} 2022-09-03T07:21:11.608 DEBUG:boto:Token: None 2022-09-03T07:21:11.608 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34 2022-09-03T07:21:11.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xJz95bIs/lkN+l8IjgzGv7CGfqc= 2022-09-03T07:21:11.610 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xJz95bIs/lkN+l8IjgzGv7CGfqc=', 'Content-Length': '0'} 2022-09-03T07:21:11.612 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2c3a0bfe7d1ce1e-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.613 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:21:11.613 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.614 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.614 DEBUG:boto:Method: GET 2022-09-03T07:21:11.615 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:11.615 DEBUG:boto:Data: 2022-09-03T07:21:11.616 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.616 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.617 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.617 DEBUG:boto:Params: {} 2022-09-03T07:21:11.618 DEBUG:boto:Token: None 2022-09-03T07:21:11.619 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34 2022-09-03T07:21:11.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JfJBBBoXflzw/RQMiVg83n2/m0E= 2022-09-03T07:21:11.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JfJBBBoXflzw/RQMiVg83n2/m0E=', 'Content-Length': '0'} 2022-09-03T07:21:11.622 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8c1a5c19fcf4cd8-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.666 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:11.667 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:11.667 DEBUG:boto:Method: GET 2022-09-03T07:21:11.668 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:11.668 DEBUG:boto:Data: 2022-09-03T07:21:11.669 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.669 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.670 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.671 DEBUG:boto:Params: {} 2022-09-03T07:21:11.671 DEBUG:boto:Token: None 2022-09-03T07:21:11.672 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj34 2022-09-03T07:21:11.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JfJBBBoXflzw/RQMiVg83n2/m0E= 2022-09-03T07:21:11.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JfJBBBoXflzw/RQMiVg83n2/m0E=', 'Content-Length': '0'} 2022-09-03T07:21:11.677 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e7b4697a4c3f9eb-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.678 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.679 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.680 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:11.681 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:11.682 DEBUG:boto:Method: GET 2022-09-03T07:21:11.683 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:11.684 DEBUG:boto:Data: 2022-09-03T07:21:11.685 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.685 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.686 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.687 DEBUG:boto:Params: {} 2022-09-03T07:21:11.689 DEBUG:boto:Token: None 2022-09-03T07:21:11.690 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.695 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd359fdec25274cb-0063130067-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.695 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.696 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.697 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.697 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:11.698 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:11.698 DEBUG:boto:Method: GET 2022-09-03T07:21:11.699 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:11.699 DEBUG:boto:Data: 2022-09-03T07:21:11.700 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.700 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.701 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.701 DEBUG:boto:Params: {} 2022-09-03T07:21:11.702 DEBUG:boto:Token: None 2022-09-03T07:21:11.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.707 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000018f2f84c8b4b46da-0063130067-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.707 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.708 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:21:11.709 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.709 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.710 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.710 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.711 DEBUG:boto:Method: GET 2022-09-03T07:21:11.711 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.712 DEBUG:boto:Data: 2022-09-03T07:21:11.712 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.713 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.713 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.714 DEBUG:boto:Params: {} 2022-09-03T07:21:11.715 DEBUG:boto:Token: None 2022-09-03T07:21:11.715 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hiNsdgK2MvoD//UZ6WYTjWw2dE= 2022-09-03T07:21:11.716 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hiNsdgK2MvoD//UZ6WYTjWw2dE=', 'Content-Length': '0'} 2022-09-03T07:21:11.718 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eee153c05b017e4f-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.719 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.720 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.720 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.721 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.721 DEBUG:boto:Method: GET 2022-09-03T07:21:11.722 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.722 DEBUG:boto:Data: 2022-09-03T07:21:11.723 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.724 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.724 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.725 DEBUG:boto:Params: {} 2022-09-03T07:21:11.725 DEBUG:boto:Token: None 2022-09-03T07:21:11.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hiNsdgK2MvoD//UZ6WYTjWw2dE= 2022-09-03T07:21:11.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hiNsdgK2MvoD//UZ6WYTjWw2dE=', 'Content-Length': '0'} 2022-09-03T07:21:11.729 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000747f8e6ef2c70e4-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.774 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.775 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.775 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.776 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.776 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.777 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.777 DEBUG:boto:Data: 2022-09-03T07:21:11.778 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.778 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.779 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.779 DEBUG:boto:Params: {} 2022-09-03T07:21:11.780 DEBUG:boto:Token: None 2022-09-03T07:21:11.780 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PyOuMRUnP0Gg8Ft4uxRNGlvnIxg= 2022-09-03T07:21:11.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PyOuMRUnP0Gg8Ft4uxRNGlvnIxg=', 'Content-Length': '0'} 2022-09-03T07:21:11.784 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e27659dc06354805-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.785 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.786 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.787 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.788 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.789 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.790 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.790 DEBUG:boto:Data: 2022-09-03T07:21:11.791 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.792 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.792 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.793 DEBUG:boto:Params: {} 2022-09-03T07:21:11.794 DEBUG:boto:Token: None 2022-09-03T07:21:11.794 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PyOuMRUnP0Gg8Ft4uxRNGlvnIxg= 2022-09-03T07:21:11.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PyOuMRUnP0Gg8Ft4uxRNGlvnIxg=', 'Content-Length': '0'} 2022-09-03T07:21:11.798 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ca5edbe109f418b-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.798 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:21:11.799 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.799 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.800 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.800 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.801 DEBUG:boto:Method: GET 2022-09-03T07:21:11.801 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.802 DEBUG:boto:Data: 2022-09-03T07:21:11.803 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.803 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.804 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.804 DEBUG:boto:Params: {} 2022-09-03T07:21:11.805 DEBUG:boto:Token: None 2022-09-03T07:21:11.806 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hiNsdgK2MvoD//UZ6WYTjWw2dE= 2022-09-03T07:21:11.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hiNsdgK2MvoD//UZ6WYTjWw2dE=', 'Content-Length': '0'} 2022-09-03T07:21:11.809 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006607a93d69d3152d-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.810 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.810 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.811 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.812 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.813 DEBUG:boto:Method: GET 2022-09-03T07:21:11.814 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.815 DEBUG:boto:Data: 2022-09-03T07:21:11.815 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.817 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.817 DEBUG:boto:Params: {} 2022-09-03T07:21:11.818 DEBUG:boto:Token: None 2022-09-03T07:21:11.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:11.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hiNsdgK2MvoD//UZ6WYTjWw2dE= 2022-09-03T07:21:11.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hiNsdgK2MvoD//UZ6WYTjWw2dE=', 'Content-Length': '0'} 2022-09-03T07:21:11.822 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac3f09418bf7ef4d-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.823 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.823 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.824 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:11.825 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:11.825 DEBUG:boto:Method: GET 2022-09-03T07:21:11.826 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:11.826 DEBUG:boto:Data: 2022-09-03T07:21:11.827 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.827 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.828 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.828 DEBUG:boto:Params: {} 2022-09-03T07:21:11.829 DEBUG:boto:Token: None 2022-09-03T07:21:11.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.830 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.831 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.833 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b780e61701031110-0063130067-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.834 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.835 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.835 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.836 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:11.836 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:11.837 DEBUG:boto:Method: GET 2022-09-03T07:21:11.837 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:11.838 DEBUG:boto:Data: 2022-09-03T07:21:11.838 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.839 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.839 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.840 DEBUG:boto:Params: {} 2022-09-03T07:21:11.841 DEBUG:boto:Token: None 2022-09-03T07:21:11.841 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.846 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4ce52f1b818448b-0063130067-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.846 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.847 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.848 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.848 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.849 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:11.849 DEBUG:boto:Data: 2022-09-03T07:21:11.850 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.850 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.851 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.851 DEBUG:boto:Params: {} 2022-09-03T07:21:11.852 DEBUG:boto:Token: None 2022-09-03T07:21:11.852 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38 2022-09-03T07:21:11.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2i0+DiqN6+hB1ZrEyigTqMGy3SU= 2022-09-03T07:21:11.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2i0+DiqN6+hB1ZrEyigTqMGy3SU=', 'Content-Length': '0'} 2022-09-03T07:21:11.857 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3a11d155e7834c0-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.858 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.859 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.860 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.860 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:11.861 DEBUG:boto:Data: 2022-09-03T07:21:11.861 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.862 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.862 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.863 DEBUG:boto:Params: {} 2022-09-03T07:21:11.863 DEBUG:boto:Token: None 2022-09-03T07:21:11.864 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38 2022-09-03T07:21:11.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2i0+DiqN6+hB1ZrEyigTqMGy3SU= 2022-09-03T07:21:11.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2i0+DiqN6+hB1ZrEyigTqMGy3SU=', 'Content-Length': '0'} 2022-09-03T07:21:11.867 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000687ac7ac07ac6f21-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.868 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:21:11.868 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.869 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.869 DEBUG:boto:Method: GET 2022-09-03T07:21:11.870 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:11.870 DEBUG:boto:Data: 2022-09-03T07:21:11.871 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.872 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.872 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.873 DEBUG:boto:Params: {} 2022-09-03T07:21:11.873 DEBUG:boto:Token: None 2022-09-03T07:21:11.874 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38 2022-09-03T07:21:11.875 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BjpmlETVwBq910sZEGvU2yTbu50= 2022-09-03T07:21:11.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BjpmlETVwBq910sZEGvU2yTbu50=', 'Content-Length': '0'} 2022-09-03T07:21:11.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002aaac0f53619d4e1-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.878 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:11.879 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:11.879 DEBUG:boto:Method: GET 2022-09-03T07:21:11.880 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:11.880 DEBUG:boto:Data: 2022-09-03T07:21:11.881 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.882 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.883 DEBUG:boto:Params: {} 2022-09-03T07:21:11.884 DEBUG:boto:Token: None 2022-09-03T07:21:11.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj38 2022-09-03T07:21:11.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BjpmlETVwBq910sZEGvU2yTbu50= 2022-09-03T07:21:11.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BjpmlETVwBq910sZEGvU2yTbu50=', 'Content-Length': '0'} 2022-09-03T07:21:11.888 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000103ea603c1105b78-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.930 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.931 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.932 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:11.933 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:11.934 DEBUG:boto:Method: GET 2022-09-03T07:21:11.935 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:11.936 DEBUG:boto:Data: 2022-09-03T07:21:11.937 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.938 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.939 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.939 DEBUG:boto:Params: {} 2022-09-03T07:21:11.940 DEBUG:boto:Token: None 2022-09-03T07:21:11.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.944 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eec330d7666e2bd6-0063130067-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.945 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.946 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:11.947 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:11.947 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:11.948 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:11.948 DEBUG:boto:Method: GET 2022-09-03T07:21:11.949 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:11.950 DEBUG:boto:Data: 2022-09-03T07:21:11.950 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.951 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.952 DEBUG:boto:Params: {} 2022-09-03T07:21:11.953 DEBUG:boto:Token: None 2022-09-03T07:21:11.953 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/ 2022-09-03T07:21:11.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I= 2022-09-03T07:21:11.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AFO64iu/ObakXzFIc1o4gknwa/I=', 'Content-Length': '0'} 2022-09-03T07:21:11.957 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044fae54353d8dbb7-0063130067-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.958 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:11.959 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:21:11.959 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:11.960 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:11.961 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.961 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.962 DEBUG:boto:Method: GET 2022-09-03T07:21:11.962 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.963 DEBUG:boto:Data: 2022-09-03T07:21:11.963 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.964 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.964 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.965 DEBUG:boto:Params: {} 2022-09-03T07:21:11.966 DEBUG:boto:Token: None 2022-09-03T07:21:11.966 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MqHXYXGwYOpVpD8O71IlCasJX5g= 2022-09-03T07:21:11.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MqHXYXGwYOpVpD8O71IlCasJX5g=', 'Content-Length': '0'} 2022-09-03T07:21:11.970 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093f61ec636e6dc31-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.971 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:11.971 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:11.972 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.973 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.973 DEBUG:boto:Method: GET 2022-09-03T07:21:11.974 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.974 DEBUG:boto:Data: 2022-09-03T07:21:11.975 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.975 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.976 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.976 DEBUG:boto:Params: {} 2022-09-03T07:21:11.977 DEBUG:boto:Token: None 2022-09-03T07:21:11.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MqHXYXGwYOpVpD8O71IlCasJX5g= 2022-09-03T07:21:11.979 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MqHXYXGwYOpVpD8O71IlCasJX5g=', 'Content-Length': '0'} 2022-09-03T07:21:11.981 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000041bfd965adb511e0-0063130067-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.982 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:11.982 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:11.983 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.983 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.984 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.984 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.985 DEBUG:boto:Data: 2022-09-03T07:21:11.985 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.986 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:11.986 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.987 DEBUG:boto:Params: {} 2022-09-03T07:21:11.988 DEBUG:boto:Token: None 2022-09-03T07:21:11.989 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:11 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W1W+WaX54AG/Qf6ZU/1/KRRjx7o= 2022-09-03T07:21:11.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W1W+WaX54AG/Qf6ZU/1/KRRjx7o=', 'Content-Length': '0'} 2022-09-03T07:21:11.992 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001fbae4104ebac077-0063130067-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:11.993 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:11.993 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:11.994 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.994 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.995 DEBUG:boto:Method: HEAD 2022-09-03T07:21:11.996 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:11.996 DEBUG:boto:Data: 2022-09-03T07:21:11.997 DEBUG:boto:Headers: {} 2022-09-03T07:21:11.997 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:11.998 DEBUG:boto:Port: 8000 2022-09-03T07:21:11.999 DEBUG:boto:Params: {} 2022-09-03T07:21:11.999 DEBUG:boto:Token: None 2022-09-03T07:21:12.000 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:12.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WxC41yYxfnmRcWV6ouji9ohVM4s= 2022-09-03T07:21:12.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WxC41yYxfnmRcWV6ouji9ohVM4s=', 'Content-Length': '0'} 2022-09-03T07:21:12.003 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c45db726b7e6bd7-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.004 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:21:12.005 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:12.005 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:12.006 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:12.006 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:12.007 DEBUG:boto:Method: GET 2022-09-03T07:21:12.008 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:12.008 DEBUG:boto:Data: 2022-09-03T07:21:12.009 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.010 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.010 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.011 DEBUG:boto:Params: {} 2022-09-03T07:21:12.011 DEBUG:boto:Token: None 2022-09-03T07:21:12.012 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:12.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yk4Vn7wqulcciC1lDsmtac7KNmg= 2022-09-03T07:21:12.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yk4Vn7wqulcciC1lDsmtac7KNmg=', 'Content-Length': '0'} 2022-09-03T07:21:12.015 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc1d3c8a1671d1ce-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.062 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:12.063 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:12.063 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:12.064 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:12.065 DEBUG:boto:Method: GET 2022-09-03T07:21:12.065 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:12.066 DEBUG:boto:Data: 2022-09-03T07:21:12.066 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.067 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.068 DEBUG:boto:Params: {} 2022-09-03T07:21:12.068 DEBUG:boto:Token: None 2022-09-03T07:21:12.069 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:12.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yk4Vn7wqulcciC1lDsmtac7KNmg= 2022-09-03T07:21:12.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yk4Vn7wqulcciC1lDsmtac7KNmg=', 'Content-Length': '0'} 2022-09-03T07:21:12.072 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004050d6ef755bb6a6-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.073 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.074 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.074 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:12.075 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:12.075 DEBUG:boto:Method: GET 2022-09-03T07:21:12.076 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:12.076 DEBUG:boto:Data: 2022-09-03T07:21:12.077 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.078 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.078 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.079 DEBUG:boto:Params: {} 2022-09-03T07:21:12.079 DEBUG:boto:Token: None 2022-09-03T07:21:12.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009dcbc18ae0f28094-0063130068-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.085 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.085 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.086 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.087 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:12.087 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:12.088 DEBUG:boto:Method: GET 2022-09-03T07:21:12.088 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:12.089 DEBUG:boto:Data: 2022-09-03T07:21:12.089 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.090 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.090 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.091 DEBUG:boto:Params: {} 2022-09-03T07:21:12.092 DEBUG:boto:Token: None 2022-09-03T07:21:12.092 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.097 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000435210e0745d0e0b-0063130068-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.097 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.098 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:12.099 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:12.099 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.100 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:12.101 DEBUG:boto:Data: 2022-09-03T07:21:12.101 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.102 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.102 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.103 DEBUG:boto:Params: {} 2022-09-03T07:21:12.103 DEBUG:boto:Token: None 2022-09-03T07:21:12.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj44 2022-09-03T07:21:12.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NkQqXX3vg7UozsTARjl8FLHDvXc= 2022-09-03T07:21:12.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NkQqXX3vg7UozsTARjl8FLHDvXc=', 'Content-Length': '0'} 2022-09-03T07:21:12.107 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000085a0051b82e0e10e-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.108 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:12.108 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:12.109 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.109 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:12.110 DEBUG:boto:Data: 2022-09-03T07:21:12.110 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.111 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.112 DEBUG:boto:Params: {} 2022-09-03T07:21:12.113 DEBUG:boto:Token: None 2022-09-03T07:21:12.113 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj44 2022-09-03T07:21:12.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NkQqXX3vg7UozsTARjl8FLHDvXc= 2022-09-03T07:21:12.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NkQqXX3vg7UozsTARjl8FLHDvXc=', 'Content-Length': '0'} 2022-09-03T07:21:12.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9d036aac22d20cf-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.117 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:21:12.118 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:12.119 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:12.119 DEBUG:boto:Method: GET 2022-09-03T07:21:12.120 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:12.121 DEBUG:boto:Data: 2022-09-03T07:21:12.122 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.123 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.124 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.125 DEBUG:boto:Params: {} 2022-09-03T07:21:12.126 DEBUG:boto:Token: None 2022-09-03T07:21:12.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj44 2022-09-03T07:21:12.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9L+I9WQiabgl/v+8zMX2AtFtXzw= 2022-09-03T07:21:12.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9L+I9WQiabgl/v+8zMX2AtFtXzw=', 'Content-Length': '0'} 2022-09-03T07:21:12.132 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c89c8c2118f8c2cd-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.133 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:12.134 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:12.135 DEBUG:boto:Method: GET 2022-09-03T07:21:12.136 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:12.137 DEBUG:boto:Data: 2022-09-03T07:21:12.138 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.139 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.140 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.140 DEBUG:boto:Params: {} 2022-09-03T07:21:12.141 DEBUG:boto:Token: None 2022-09-03T07:21:12.142 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj44 2022-09-03T07:21:12.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9L+I9WQiabgl/v+8zMX2AtFtXzw= 2022-09-03T07:21:12.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9L+I9WQiabgl/v+8zMX2AtFtXzw=', 'Content-Length': '0'} 2022-09-03T07:21:12.145 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052379e5cba1eb8d0-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.147 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.148 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.148 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:12.149 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:12.149 DEBUG:boto:Method: GET 2022-09-03T07:21:12.150 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:12.151 DEBUG:boto:Data: 2022-09-03T07:21:12.151 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.152 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.152 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.153 DEBUG:boto:Params: {} 2022-09-03T07:21:12.154 DEBUG:boto:Token: None 2022-09-03T07:21:12.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.160 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004eac94348dbb0bf8-0063130068-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.160 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.161 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.162 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.162 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:12.163 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:12.163 DEBUG:boto:Method: GET 2022-09-03T07:21:12.164 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:12.164 DEBUG:boto:Data: 2022-09-03T07:21:12.165 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.166 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.166 DEBUG:boto:Params: {} 2022-09-03T07:21:12.167 DEBUG:boto:Token: None 2022-09-03T07:21:12.168 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.172 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9e9c0901b1f0b48-0063130068-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.173 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.173 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:21:12.174 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.174 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.175 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.175 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.176 DEBUG:boto:Method: GET 2022-09-03T07:21:12.176 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.177 DEBUG:boto:Data: 2022-09-03T07:21:12.178 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.178 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.179 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.179 DEBUG:boto:Params: {} 2022-09-03T07:21:12.180 DEBUG:boto:Token: None 2022-09-03T07:21:12.180 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.181 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uh35qGOlnrk3olyzfcqx0xI+kWQ= 2022-09-03T07:21:12.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uh35qGOlnrk3olyzfcqx0xI+kWQ=', 'Content-Length': '0'} 2022-09-03T07:21:12.185 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000033d9a9225e7843f4-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.186 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.186 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.187 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.187 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.188 DEBUG:boto:Method: GET 2022-09-03T07:21:12.188 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.189 DEBUG:boto:Data: 2022-09-03T07:21:12.189 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.190 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.191 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.191 DEBUG:boto:Params: {} 2022-09-03T07:21:12.192 DEBUG:boto:Token: None 2022-09-03T07:21:12.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uh35qGOlnrk3olyzfcqx0xI+kWQ= 2022-09-03T07:21:12.194 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uh35qGOlnrk3olyzfcqx0xI+kWQ=', 'Content-Length': '0'} 2022-09-03T07:21:12.196 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071a4759181b52d5e-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.197 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.197 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.198 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.198 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.199 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.199 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.200 DEBUG:boto:Data: 2022-09-03T07:21:12.200 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.201 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.201 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.202 DEBUG:boto:Params: {} 2022-09-03T07:21:12.202 DEBUG:boto:Token: None 2022-09-03T07:21:12.203 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.203 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0U39+mqtQrYtp9+sg78qb6c49Ho= 2022-09-03T07:21:12.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0U39+mqtQrYtp9+sg78qb6c49Ho=', 'Content-Length': '0'} 2022-09-03T07:21:12.206 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f7366a2c4e2ceae-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.207 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.207 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.208 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.208 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.209 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.209 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.210 DEBUG:boto:Data: 2022-09-03T07:21:12.210 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.211 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.211 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.212 DEBUG:boto:Params: {} 2022-09-03T07:21:12.212 DEBUG:boto:Token: None 2022-09-03T07:21:12.213 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0U39+mqtQrYtp9+sg78qb6c49Ho= 2022-09-03T07:21:12.214 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0U39+mqtQrYtp9+sg78qb6c49Ho=', 'Content-Length': '0'} 2022-09-03T07:21:12.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063ade412cbfb1355-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:21:12.217 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.217 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.218 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.218 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.219 DEBUG:boto:Method: GET 2022-09-03T07:21:12.219 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.220 DEBUG:boto:Data: 2022-09-03T07:21:12.220 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.221 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.222 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.222 DEBUG:boto:Params: {} 2022-09-03T07:21:12.223 DEBUG:boto:Token: None 2022-09-03T07:21:12.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uh35qGOlnrk3olyzfcqx0xI+kWQ= 2022-09-03T07:21:12.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uh35qGOlnrk3olyzfcqx0xI+kWQ=', 'Content-Length': '0'} 2022-09-03T07:21:12.226 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d7d4f4265c9b9af-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.227 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.228 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.228 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.229 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.229 DEBUG:boto:Method: GET 2022-09-03T07:21:12.230 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.230 DEBUG:boto:Data: 2022-09-03T07:21:12.231 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.232 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.232 DEBUG:boto:Params: {} 2022-09-03T07:21:12.233 DEBUG:boto:Token: None 2022-09-03T07:21:12.234 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:12.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Uh35qGOlnrk3olyzfcqx0xI+kWQ= 2022-09-03T07:21:12.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Uh35qGOlnrk3olyzfcqx0xI+kWQ=', 'Content-Length': '0'} 2022-09-03T07:21:12.237 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062bed60f32b53add-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.238 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.239 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.239 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:12.240 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:12.240 DEBUG:boto:Method: GET 2022-09-03T07:21:12.241 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:12.241 DEBUG:boto:Data: 2022-09-03T07:21:12.242 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.242 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.243 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.244 DEBUG:boto:Params: {} 2022-09-03T07:21:12.244 DEBUG:boto:Token: None 2022-09-03T07:21:12.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.246 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.249 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c459df5abf497b3-0063130068-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.294 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.295 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.296 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.296 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:12.297 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:12.297 DEBUG:boto:Method: GET 2022-09-03T07:21:12.298 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:12.298 DEBUG:boto:Data: 2022-09-03T07:21:12.299 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.300 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.300 DEBUG:boto:Params: {} 2022-09-03T07:21:12.301 DEBUG:boto:Token: None 2022-09-03T07:21:12.302 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.306 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2fdc7da1aa317cd-0063130068-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.306 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.307 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.308 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.308 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.309 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:12.309 DEBUG:boto:Data: 2022-09-03T07:21:12.310 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.310 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.312 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.312 DEBUG:boto:Params: {} 2022-09-03T07:21:12.313 DEBUG:boto:Token: None 2022-09-03T07:21:12.314 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51 2022-09-03T07:21:12.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xb5At5Q1O/M/1vfD3taTzWEtPLg= 2022-09-03T07:21:12.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xb5At5Q1O/M/1vfD3taTzWEtPLg=', 'Content-Length': '0'} 2022-09-03T07:21:12.317 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc166d81d887fbb2-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.318 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.318 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.319 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.319 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:12.320 DEBUG:boto:Data: 2022-09-03T07:21:12.320 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.321 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.322 DEBUG:boto:Params: {} 2022-09-03T07:21:12.323 DEBUG:boto:Token: None 2022-09-03T07:21:12.323 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51 2022-09-03T07:21:12.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xb5At5Q1O/M/1vfD3taTzWEtPLg= 2022-09-03T07:21:12.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xb5At5Q1O/M/1vfD3taTzWEtPLg=', 'Content-Length': '0'} 2022-09-03T07:21:12.327 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af9071e72a0ce30b-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.328 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:21:12.328 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.329 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.330 DEBUG:boto:Method: GET 2022-09-03T07:21:12.330 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:12.331 DEBUG:boto:Data: 2022-09-03T07:21:12.331 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.332 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.333 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.333 DEBUG:boto:Params: {} 2022-09-03T07:21:12.334 DEBUG:boto:Token: None 2022-09-03T07:21:12.334 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51 2022-09-03T07:21:12.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:apbswsBQ3X9wsHbO5d1dLwmnrZQ= 2022-09-03T07:21:12.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:apbswsBQ3X9wsHbO5d1dLwmnrZQ=', 'Content-Length': '0'} 2022-09-03T07:21:12.338 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f165b2a4d1e7181-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.339 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:12.339 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:12.340 DEBUG:boto:Method: GET 2022-09-03T07:21:12.340 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:12.341 DEBUG:boto:Data: 2022-09-03T07:21:12.341 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.342 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.342 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.343 DEBUG:boto:Params: {} 2022-09-03T07:21:12.343 DEBUG:boto:Token: None 2022-09-03T07:21:12.344 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj51 2022-09-03T07:21:12.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:apbswsBQ3X9wsHbO5d1dLwmnrZQ= 2022-09-03T07:21:12.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:apbswsBQ3X9wsHbO5d1dLwmnrZQ=', 'Content-Length': '0'} 2022-09-03T07:21:12.348 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8ca5af9570e714c-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.394 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.395 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.396 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:12.397 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:12.397 DEBUG:boto:Method: GET 2022-09-03T07:21:12.398 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:12.398 DEBUG:boto:Data: 2022-09-03T07:21:12.399 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.399 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.400 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.400 DEBUG:boto:Params: {} 2022-09-03T07:21:12.401 DEBUG:boto:Token: None 2022-09-03T07:21:12.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.408 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088bbb32292eec744-0063130068-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.409 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.411 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.412 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.413 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:12.414 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:12.415 DEBUG:boto:Method: GET 2022-09-03T07:21:12.416 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:12.417 DEBUG:boto:Data: 2022-09-03T07:21:12.418 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.420 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.422 DEBUG:boto:Params: {} 2022-09-03T07:21:12.423 DEBUG:boto:Token: None 2022-09-03T07:21:12.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.431 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db81dabce67b6347-0063130068-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.432 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.433 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:21:12.434 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.435 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.436 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.437 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.438 DEBUG:boto:Method: GET 2022-09-03T07:21:12.440 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.441 DEBUG:boto:Data: 2022-09-03T07:21:12.442 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.443 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.444 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.445 DEBUG:boto:Params: {} 2022-09-03T07:21:12.446 DEBUG:boto:Token: None 2022-09-03T07:21:12.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzncoijLSctw+CkGmp4n+mfL4f0= 2022-09-03T07:21:12.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzncoijLSctw+CkGmp4n+mfL4f0=', 'Content-Length': '0'} 2022-09-03T07:21:12.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022cd4f951f12e710-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.454 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.455 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.457 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.458 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.459 DEBUG:boto:Method: GET 2022-09-03T07:21:12.460 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.461 DEBUG:boto:Data: 2022-09-03T07:21:12.462 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.464 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.466 DEBUG:boto:Params: {} 2022-09-03T07:21:12.467 DEBUG:boto:Token: None 2022-09-03T07:21:12.468 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzncoijLSctw+CkGmp4n+mfL4f0= 2022-09-03T07:21:12.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzncoijLSctw+CkGmp4n+mfL4f0=', 'Content-Length': '0'} 2022-09-03T07:21:12.472 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002822e2e0742f22dc-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.473 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.474 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.474 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.475 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.475 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.476 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.476 DEBUG:boto:Data: 2022-09-03T07:21:12.477 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.477 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.478 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.478 DEBUG:boto:Params: {} 2022-09-03T07:21:12.479 DEBUG:boto:Token: None 2022-09-03T07:21:12.480 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zsoVLMFBy2lIacoguGGxoiu8ZwM= 2022-09-03T07:21:12.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zsoVLMFBy2lIacoguGGxoiu8ZwM=', 'Content-Length': '0'} 2022-09-03T07:21:12.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b544a2aa85170be8-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.484 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.484 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.485 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.485 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.486 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.486 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.487 DEBUG:boto:Data: 2022-09-03T07:21:12.487 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.489 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.489 DEBUG:boto:Params: {} 2022-09-03T07:21:12.490 DEBUG:boto:Token: None 2022-09-03T07:21:12.490 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zsoVLMFBy2lIacoguGGxoiu8ZwM= 2022-09-03T07:21:12.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zsoVLMFBy2lIacoguGGxoiu8ZwM=', 'Content-Length': '0'} 2022-09-03T07:21:12.494 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000190ce01354631b7b-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.495 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:21:12.495 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.496 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.496 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.497 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.497 DEBUG:boto:Method: GET 2022-09-03T07:21:12.498 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.499 DEBUG:boto:Data: 2022-09-03T07:21:12.499 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.500 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.500 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.501 DEBUG:boto:Params: {} 2022-09-03T07:21:12.501 DEBUG:boto:Token: None 2022-09-03T07:21:12.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzncoijLSctw+CkGmp4n+mfL4f0= 2022-09-03T07:21:12.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzncoijLSctw+CkGmp4n+mfL4f0=', 'Content-Length': '0'} 2022-09-03T07:21:12.505 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec1102dc71656d84-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.506 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.507 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.508 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.508 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.509 DEBUG:boto:Method: GET 2022-09-03T07:21:12.509 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.510 DEBUG:boto:Data: 2022-09-03T07:21:12.510 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.511 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.511 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.512 DEBUG:boto:Params: {} 2022-09-03T07:21:12.512 DEBUG:boto:Token: None 2022-09-03T07:21:12.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:12.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzncoijLSctw+CkGmp4n+mfL4f0= 2022-09-03T07:21:12.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzncoijLSctw+CkGmp4n+mfL4f0=', 'Content-Length': '0'} 2022-09-03T07:21:12.517 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000377ddb087177662c-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.519 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.520 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.520 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:12.521 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:12.524 DEBUG:boto:Method: GET 2022-09-03T07:21:12.524 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:12.525 DEBUG:boto:Data: 2022-09-03T07:21:12.526 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.526 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.527 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.527 DEBUG:boto:Params: {} 2022-09-03T07:21:12.528 DEBUG:boto:Token: None 2022-09-03T07:21:12.528 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.532 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000539d38c19f7ec8cb-0063130068-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.533 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.534 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.535 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.535 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:12.536 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:12.536 DEBUG:boto:Method: GET 2022-09-03T07:21:12.537 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:12.537 DEBUG:boto:Data: 2022-09-03T07:21:12.538 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.539 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.540 DEBUG:boto:Params: {} 2022-09-03T07:21:12.540 DEBUG:boto:Token: None 2022-09-03T07:21:12.541 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.542 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.546 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000967faed7576336b5-0063130068-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.546 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.547 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.548 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.549 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.549 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:12.550 DEBUG:boto:Data: 2022-09-03T07:21:12.550 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.551 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.551 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.552 DEBUG:boto:Params: {} 2022-09-03T07:21:12.553 DEBUG:boto:Token: None 2022-09-03T07:21:12.553 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55 2022-09-03T07:21:12.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qBElMR0sn4I94f4901KyHeQ0BGA= 2022-09-03T07:21:12.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qBElMR0sn4I94f4901KyHeQ0BGA=', 'Content-Length': '0'} 2022-09-03T07:21:12.556 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a7ee9f31a6797b7-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.558 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.558 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.559 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.559 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:12.560 DEBUG:boto:Data: 2022-09-03T07:21:12.560 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.561 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.561 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.562 DEBUG:boto:Params: {} 2022-09-03T07:21:12.563 DEBUG:boto:Token: None 2022-09-03T07:21:12.563 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55 2022-09-03T07:21:12.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qBElMR0sn4I94f4901KyHeQ0BGA= 2022-09-03T07:21:12.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qBElMR0sn4I94f4901KyHeQ0BGA=', 'Content-Length': '0'} 2022-09-03T07:21:12.567 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8d05a6d8a82f6e9-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.568 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:21:12.568 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.569 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.570 DEBUG:boto:Method: GET 2022-09-03T07:21:12.571 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:12.571 DEBUG:boto:Data: 2022-09-03T07:21:12.572 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.573 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.573 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.574 DEBUG:boto:Params: {} 2022-09-03T07:21:12.575 DEBUG:boto:Token: None 2022-09-03T07:21:12.575 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55 2022-09-03T07:21:12.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lotFYE5tnOvezFsZFGmQn6zt5dY= 2022-09-03T07:21:12.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lotFYE5tnOvezFsZFGmQn6zt5dY=', 'Content-Length': '0'} 2022-09-03T07:21:12.579 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009eaf60129cddecce-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.580 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:12.580 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:12.581 DEBUG:boto:Method: GET 2022-09-03T07:21:12.581 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:12.582 DEBUG:boto:Data: 2022-09-03T07:21:12.582 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.583 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.584 DEBUG:boto:Params: {} 2022-09-03T07:21:12.585 DEBUG:boto:Token: None 2022-09-03T07:21:12.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj55 2022-09-03T07:21:12.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lotFYE5tnOvezFsZFGmQn6zt5dY= 2022-09-03T07:21:12.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lotFYE5tnOvezFsZFGmQn6zt5dY=', 'Content-Length': '0'} 2022-09-03T07:21:12.589 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076ae5d0a0804c4de-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.630 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.631 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.631 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:12.632 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:12.633 DEBUG:boto:Method: GET 2022-09-03T07:21:12.633 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:12.634 DEBUG:boto:Data: 2022-09-03T07:21:12.635 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.635 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.636 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.636 DEBUG:boto:Params: {} 2022-09-03T07:21:12.637 DEBUG:boto:Token: None 2022-09-03T07:21:12.638 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.642 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e62b36a52577f8ec-0063130068-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.642 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.643 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.644 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.645 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:12.645 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:12.646 DEBUG:boto:Method: GET 2022-09-03T07:21:12.646 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:12.647 DEBUG:boto:Data: 2022-09-03T07:21:12.647 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.648 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.648 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.649 DEBUG:boto:Params: {} 2022-09-03T07:21:12.649 DEBUG:boto:Token: None 2022-09-03T07:21:12.650 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.655 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000be9896e12d53d260-0063130068-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.656 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.657 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:21:12.657 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.658 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.658 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.659 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.660 DEBUG:boto:Method: GET 2022-09-03T07:21:12.660 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.661 DEBUG:boto:Data: 2022-09-03T07:21:12.661 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.662 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.662 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.663 DEBUG:boto:Params: {} 2022-09-03T07:21:12.664 DEBUG:boto:Token: None 2022-09-03T07:21:12.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jRoK4sHIT/iAomWJT1q+XEORb7c= 2022-09-03T07:21:12.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jRoK4sHIT/iAomWJT1q+XEORb7c=', 'Content-Length': '0'} 2022-09-03T07:21:12.668 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a8edfb7a9c8e49d-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.669 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.669 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.670 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.670 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.671 DEBUG:boto:Method: GET 2022-09-03T07:21:12.671 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.672 DEBUG:boto:Data: 2022-09-03T07:21:12.672 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.673 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.673 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.674 DEBUG:boto:Params: {} 2022-09-03T07:21:12.675 DEBUG:boto:Token: None 2022-09-03T07:21:12.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jRoK4sHIT/iAomWJT1q+XEORb7c= 2022-09-03T07:21:12.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jRoK4sHIT/iAomWJT1q+XEORb7c=', 'Content-Length': '0'} 2022-09-03T07:21:12.679 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb52969308f02089-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.680 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.680 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.681 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.681 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.682 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.682 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.683 DEBUG:boto:Data: 2022-09-03T07:21:12.684 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.684 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.685 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.685 DEBUG:boto:Params: {} 2022-09-03T07:21:12.686 DEBUG:boto:Token: None 2022-09-03T07:21:12.687 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JZRoWCETxO27MJT6gA/FIwbMNU0= 2022-09-03T07:21:12.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JZRoWCETxO27MJT6gA/FIwbMNU0=', 'Content-Length': '0'} 2022-09-03T07:21:12.690 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2adfe64af7bbb04-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.691 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.691 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.692 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.693 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.693 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.694 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.694 DEBUG:boto:Data: 2022-09-03T07:21:12.695 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.696 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.696 DEBUG:boto:Params: {} 2022-09-03T07:21:12.697 DEBUG:boto:Token: None 2022-09-03T07:21:12.697 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JZRoWCETxO27MJT6gA/FIwbMNU0= 2022-09-03T07:21:12.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JZRoWCETxO27MJT6gA/FIwbMNU0=', 'Content-Length': '0'} 2022-09-03T07:21:12.707 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000197f555a077c15d1-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.708 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:21:12.709 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.709 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.710 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.710 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.711 DEBUG:boto:Method: GET 2022-09-03T07:21:12.712 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.712 DEBUG:boto:Data: 2022-09-03T07:21:12.713 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.713 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.714 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.714 DEBUG:boto:Params: {} 2022-09-03T07:21:12.715 DEBUG:boto:Token: None 2022-09-03T07:21:12.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jRoK4sHIT/iAomWJT1q+XEORb7c= 2022-09-03T07:21:12.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jRoK4sHIT/iAomWJT1q+XEORb7c=', 'Content-Length': '0'} 2022-09-03T07:21:12.719 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000075a0b1bb3b691a3-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.720 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.721 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.721 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.722 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.722 DEBUG:boto:Method: GET 2022-09-03T07:21:12.723 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.723 DEBUG:boto:Data: 2022-09-03T07:21:12.724 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.724 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.725 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.726 DEBUG:boto:Params: {} 2022-09-03T07:21:12.726 DEBUG:boto:Token: None 2022-09-03T07:21:12.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:12.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jRoK4sHIT/iAomWJT1q+XEORb7c= 2022-09-03T07:21:12.728 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jRoK4sHIT/iAomWJT1q+XEORb7c=', 'Content-Length': '0'} 2022-09-03T07:21:12.730 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2332f65b40f67d0-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.774 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.775 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.776 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:12.776 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:12.777 DEBUG:boto:Method: GET 2022-09-03T07:21:12.777 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:12.778 DEBUG:boto:Data: 2022-09-03T07:21:12.778 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.779 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.780 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.781 DEBUG:boto:Params: {} 2022-09-03T07:21:12.783 DEBUG:boto:Token: None 2022-09-03T07:21:12.784 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.788 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004eec132d1e0ee98e-0063130068-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.789 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.790 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.791 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.792 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:12.792 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:12.793 DEBUG:boto:Method: GET 2022-09-03T07:21:12.794 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:12.794 DEBUG:boto:Data: 2022-09-03T07:21:12.795 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.796 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.797 DEBUG:boto:Params: {} 2022-09-03T07:21:12.797 DEBUG:boto:Token: None 2022-09-03T07:21:12.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.803 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f34dff1fbe1ceaf4-0063130068-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.804 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.805 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.806 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.807 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.808 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:12.809 DEBUG:boto:Data: 2022-09-03T07:21:12.810 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.811 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.812 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.813 DEBUG:boto:Params: {} 2022-09-03T07:21:12.814 DEBUG:boto:Token: None 2022-09-03T07:21:12.815 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57 2022-09-03T07:21:12.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TTzihwKVeJBUcnQ8A65nUx9LzOs= 2022-09-03T07:21:12.817 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TTzihwKVeJBUcnQ8A65nUx9LzOs=', 'Content-Length': '0'} 2022-09-03T07:21:12.820 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e67bdeff82eef2e8-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.821 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.822 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.823 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.824 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:12.825 DEBUG:boto:Data: 2022-09-03T07:21:12.826 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.827 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.828 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.829 DEBUG:boto:Params: {} 2022-09-03T07:21:12.830 DEBUG:boto:Token: None 2022-09-03T07:21:12.832 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57 2022-09-03T07:21:12.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TTzihwKVeJBUcnQ8A65nUx9LzOs= 2022-09-03T07:21:12.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TTzihwKVeJBUcnQ8A65nUx9LzOs=', 'Content-Length': '0'} 2022-09-03T07:21:12.836 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002539de415f762b44-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.837 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:21:12.838 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.838 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.839 DEBUG:boto:Method: GET 2022-09-03T07:21:12.839 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:12.840 DEBUG:boto:Data: 2022-09-03T07:21:12.840 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.841 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.841 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.842 DEBUG:boto:Params: {} 2022-09-03T07:21:12.843 DEBUG:boto:Token: None 2022-09-03T07:21:12.843 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57 2022-09-03T07:21:12.844 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y3QRJRko7ENaurxW+PboTlVJX/E= 2022-09-03T07:21:12.845 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y3QRJRko7ENaurxW+PboTlVJX/E=', 'Content-Length': '0'} 2022-09-03T07:21:12.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aeedc96c3ed13dd6-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.850 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:12.851 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:12.852 DEBUG:boto:Method: GET 2022-09-03T07:21:12.853 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:12.854 DEBUG:boto:Data: 2022-09-03T07:21:12.855 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.855 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.856 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.857 DEBUG:boto:Params: {} 2022-09-03T07:21:12.857 DEBUG:boto:Token: None 2022-09-03T07:21:12.858 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj57 2022-09-03T07:21:12.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y3QRJRko7ENaurxW+PboTlVJX/E= 2022-09-03T07:21:12.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y3QRJRko7ENaurxW+PboTlVJX/E=', 'Content-Length': '0'} 2022-09-03T07:21:12.861 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a1b62556f3d7fd3-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.862 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.863 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.863 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:12.864 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:12.864 DEBUG:boto:Method: GET 2022-09-03T07:21:12.865 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:12.865 DEBUG:boto:Data: 2022-09-03T07:21:12.866 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.867 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.867 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.868 DEBUG:boto:Params: {} 2022-09-03T07:21:12.868 DEBUG:boto:Token: None 2022-09-03T07:21:12.869 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.873 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023750bb6a4d46deb-0063130068-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.874 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.875 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:12.876 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:12.877 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:12.878 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:12.879 DEBUG:boto:Method: GET 2022-09-03T07:21:12.880 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:12.881 DEBUG:boto:Data: 2022-09-03T07:21:12.882 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.883 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.884 DEBUG:boto:Params: {} 2022-09-03T07:21:12.884 DEBUG:boto:Token: None 2022-09-03T07:21:12.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/ 2022-09-03T07:21:12.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M= 2022-09-03T07:21:12.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmQh0WvD8jRfjaVaV7J34patI6M=', 'Content-Length': '0'} 2022-09-03T07:21:12.890 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007ea5f8518a0d87db-0063130068-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.890 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:12.891 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:21:12.892 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:12.892 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:12.893 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.893 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.894 DEBUG:boto:Method: GET 2022-09-03T07:21:12.895 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.895 DEBUG:boto:Data: 2022-09-03T07:21:12.896 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.897 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.897 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.898 DEBUG:boto:Params: {} 2022-09-03T07:21:12.898 DEBUG:boto:Token: None 2022-09-03T07:21:12.899 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VmvpqEtJ8fKVncAHdK/gru7sWXE= 2022-09-03T07:21:12.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VmvpqEtJ8fKVncAHdK/gru7sWXE=', 'Content-Length': '0'} 2022-09-03T07:21:12.903 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fafd7cb04418301b-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.904 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:12.904 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:12.905 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.905 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.906 DEBUG:boto:Method: GET 2022-09-03T07:21:12.907 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.907 DEBUG:boto:Data: 2022-09-03T07:21:12.908 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.908 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.909 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.910 DEBUG:boto:Params: {} 2022-09-03T07:21:12.910 DEBUG:boto:Token: None 2022-09-03T07:21:12.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VmvpqEtJ8fKVncAHdK/gru7sWXE= 2022-09-03T07:21:12.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VmvpqEtJ8fKVncAHdK/gru7sWXE=', 'Content-Length': '0'} 2022-09-03T07:21:12.915 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4a39c1b10304afe-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.916 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:12.916 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:12.917 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.917 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.918 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.918 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.919 DEBUG:boto:Data: 2022-09-03T07:21:12.919 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.920 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.920 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.921 DEBUG:boto:Params: {} 2022-09-03T07:21:12.921 DEBUG:boto:Token: None 2022-09-03T07:21:12.922 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:28wNLjp20J5rhztpBE4pLHil3hM= 2022-09-03T07:21:12.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:28wNLjp20J5rhztpBE4pLHil3hM=', 'Content-Length': '0'} 2022-09-03T07:21:12.925 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3ca7734738a0e77-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.926 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:12.927 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:12.927 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.928 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.928 DEBUG:boto:Method: HEAD 2022-09-03T07:21:12.929 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.929 DEBUG:boto:Data: 2022-09-03T07:21:12.930 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.931 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.932 DEBUG:boto:Params: {} 2022-09-03T07:21:12.933 DEBUG:boto:Token: None 2022-09-03T07:21:12.933 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:28wNLjp20J5rhztpBE4pLHil3hM= 2022-09-03T07:21:12.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:28wNLjp20J5rhztpBE4pLHil3hM=', 'Content-Length': '0'} 2022-09-03T07:21:12.937 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0eaf3045e484f76-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.938 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:21:12.939 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:12.940 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:12.940 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.941 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.942 DEBUG:boto:Method: GET 2022-09-03T07:21:12.943 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.944 DEBUG:boto:Data: 2022-09-03T07:21:12.945 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.946 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:12.946 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.947 DEBUG:boto:Params: {} 2022-09-03T07:21:12.948 DEBUG:boto:Token: None 2022-09-03T07:21:12.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VmvpqEtJ8fKVncAHdK/gru7sWXE= 2022-09-03T07:21:12.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VmvpqEtJ8fKVncAHdK/gru7sWXE=', 'Content-Length': '0'} 2022-09-03T07:21:12.954 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ed3ff6744fa1b47-0063130068-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:12.955 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:12.956 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:12.956 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.957 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.958 DEBUG:boto:Method: GET 2022-09-03T07:21:12.959 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.960 DEBUG:boto:Data: 2022-09-03T07:21:12.961 DEBUG:boto:Headers: {} 2022-09-03T07:21:12.962 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:12.962 DEBUG:boto:Port: 8000 2022-09-03T07:21:12.963 DEBUG:boto:Params: {} 2022-09-03T07:21:12.964 DEBUG:boto:Token: None 2022-09-03T07:21:12.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:12 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:12.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VmvpqEtJ8fKVncAHdK/gru7sWXE= 2022-09-03T07:21:12.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VmvpqEtJ8fKVncAHdK/gru7sWXE=', 'Content-Length': '0'} 2022-09-03T07:21:12.969 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff57548e7f8734bd-0063130068-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.014 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.015 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.016 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:13.017 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:13.018 DEBUG:boto:Method: GET 2022-09-03T07:21:13.019 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:13.019 DEBUG:boto:Data: 2022-09-03T07:21:13.020 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.021 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.022 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.023 DEBUG:boto:Params: {} 2022-09-03T07:21:13.024 DEBUG:boto:Token: None 2022-09-03T07:21:13.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.028 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f9fc6a836d7be711-0063130069-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.029 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.030 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.031 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.032 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:13.033 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:13.034 DEBUG:boto:Method: GET 2022-09-03T07:21:13.034 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:13.035 DEBUG:boto:Data: 2022-09-03T07:21:13.036 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.037 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.037 DEBUG:boto:Params: {} 2022-09-03T07:21:13.038 DEBUG:boto:Token: None 2022-09-03T07:21:13.039 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.043 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a662007bbe960e0d-0063130069-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.044 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.045 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:13.046 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:13.047 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.048 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:13.050 DEBUG:boto:Data: 2022-09-03T07:21:13.052 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.054 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.055 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.057 DEBUG:boto:Params: {} 2022-09-03T07:21:13.059 DEBUG:boto:Token: None 2022-09-03T07:21:13.061 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj68 2022-09-03T07:21:13.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q+cnf8NF70o6MY8oRDAOfHmA8rQ= 2022-09-03T07:21:13.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q+cnf8NF70o6MY8oRDAOfHmA8rQ=', 'Content-Length': '0'} 2022-09-03T07:21:13.065 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bead7c15e4b12c7-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.066 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:13.068 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:13.069 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.070 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:13.072 DEBUG:boto:Data: 2022-09-03T07:21:13.073 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.076 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.078 DEBUG:boto:Params: {} 2022-09-03T07:21:13.080 DEBUG:boto:Token: None 2022-09-03T07:21:13.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj68 2022-09-03T07:21:13.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q+cnf8NF70o6MY8oRDAOfHmA8rQ= 2022-09-03T07:21:13.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q+cnf8NF70o6MY8oRDAOfHmA8rQ=', 'Content-Length': '0'} 2022-09-03T07:21:13.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f89fd7e5a9bb620-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.087 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:21:13.088 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:13.089 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:13.089 DEBUG:boto:Method: GET 2022-09-03T07:21:13.090 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:13.091 DEBUG:boto:Data: 2022-09-03T07:21:13.092 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.093 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.094 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.094 DEBUG:boto:Params: {} 2022-09-03T07:21:13.095 DEBUG:boto:Token: None 2022-09-03T07:21:13.096 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj68 2022-09-03T07:21:13.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3NXNWTokqZ3ZLQj/536HGnBtw+E= 2022-09-03T07:21:13.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3NXNWTokqZ3ZLQj/536HGnBtw+E=', 'Content-Length': '0'} 2022-09-03T07:21:13.100 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007287577e5ce6c3c9-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.101 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:13.102 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:13.102 DEBUG:boto:Method: GET 2022-09-03T07:21:13.103 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:13.103 DEBUG:boto:Data: 2022-09-03T07:21:13.104 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.104 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.105 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.105 DEBUG:boto:Params: {} 2022-09-03T07:21:13.106 DEBUG:boto:Token: None 2022-09-03T07:21:13.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj68 2022-09-03T07:21:13.107 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3NXNWTokqZ3ZLQj/536HGnBtw+E= 2022-09-03T07:21:13.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3NXNWTokqZ3ZLQj/536HGnBtw+E=', 'Content-Length': '0'} 2022-09-03T07:21:13.110 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000662c34d4a7d1b3f8-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.158 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.159 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.159 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:13.160 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:13.161 DEBUG:boto:Method: GET 2022-09-03T07:21:13.161 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:13.162 DEBUG:boto:Data: 2022-09-03T07:21:13.162 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.163 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.163 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.164 DEBUG:boto:Params: {} 2022-09-03T07:21:13.164 DEBUG:boto:Token: None 2022-09-03T07:21:13.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.170 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d365f53247f5216d-0063130069-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.171 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.171 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.172 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.172 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:13.173 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:13.174 DEBUG:boto:Method: GET 2022-09-03T07:21:13.174 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:13.175 DEBUG:boto:Data: 2022-09-03T07:21:13.175 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.176 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.177 DEBUG:boto:Params: {} 2022-09-03T07:21:13.177 DEBUG:boto:Token: None 2022-09-03T07:21:13.178 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.182 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3ba55a1adb11a50-0063130069-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.183 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.184 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:21:13.184 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.185 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.185 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.186 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.186 DEBUG:boto:Method: GET 2022-09-03T07:21:13.187 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.187 DEBUG:boto:Data: 2022-09-03T07:21:13.188 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.189 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.189 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.190 DEBUG:boto:Params: {} 2022-09-03T07:21:13.190 DEBUG:boto:Token: None 2022-09-03T07:21:13.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WbM7cq+Y5dRKbP+NRJWu/teUoUA= 2022-09-03T07:21:13.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WbM7cq+Y5dRKbP+NRJWu/teUoUA=', 'Content-Length': '0'} 2022-09-03T07:21:13.194 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa86b16957bc181b-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.238 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.239 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.240 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.241 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.242 DEBUG:boto:Method: GET 2022-09-03T07:21:13.243 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.244 DEBUG:boto:Data: 2022-09-03T07:21:13.245 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.245 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.246 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.247 DEBUG:boto:Params: {} 2022-09-03T07:21:13.248 DEBUG:boto:Token: None 2022-09-03T07:21:13.248 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WbM7cq+Y5dRKbP+NRJWu/teUoUA= 2022-09-03T07:21:13.250 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WbM7cq+Y5dRKbP+NRJWu/teUoUA=', 'Content-Length': '0'} 2022-09-03T07:21:13.252 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef683bf68ad44075-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.253 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.254 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.255 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.255 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.256 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.256 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.257 DEBUG:boto:Data: 2022-09-03T07:21:13.257 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.258 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.258 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.259 DEBUG:boto:Params: {} 2022-09-03T07:21:13.259 DEBUG:boto:Token: None 2022-09-03T07:21:13.260 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eykQIaXQpM36ZC2FM1WUgUxF5wU= 2022-09-03T07:21:13.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eykQIaXQpM36ZC2FM1WUgUxF5wU=', 'Content-Length': '0'} 2022-09-03T07:21:13.263 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f21deaeccbe0f9fe-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.264 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.264 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.265 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.265 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.266 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.266 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.267 DEBUG:boto:Data: 2022-09-03T07:21:13.267 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.268 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.269 DEBUG:boto:Params: {} 2022-09-03T07:21:13.270 DEBUG:boto:Token: None 2022-09-03T07:21:13.270 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eykQIaXQpM36ZC2FM1WUgUxF5wU= 2022-09-03T07:21:13.271 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eykQIaXQpM36ZC2FM1WUgUxF5wU=', 'Content-Length': '0'} 2022-09-03T07:21:13.273 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d5b5899fabf2cbf-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.274 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:21:13.275 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.275 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.276 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.276 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.277 DEBUG:boto:Method: GET 2022-09-03T07:21:13.277 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.278 DEBUG:boto:Data: 2022-09-03T07:21:13.279 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.279 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.280 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.280 DEBUG:boto:Params: {} 2022-09-03T07:21:13.281 DEBUG:boto:Token: None 2022-09-03T07:21:13.282 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WbM7cq+Y5dRKbP+NRJWu/teUoUA= 2022-09-03T07:21:13.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WbM7cq+Y5dRKbP+NRJWu/teUoUA=', 'Content-Length': '0'} 2022-09-03T07:21:13.285 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ea9a57f6cba45387-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.286 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.287 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.287 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.288 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.288 DEBUG:boto:Method: GET 2022-09-03T07:21:13.289 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.290 DEBUG:boto:Data: 2022-09-03T07:21:13.291 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.291 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.292 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.292 DEBUG:boto:Params: {} 2022-09-03T07:21:13.293 DEBUG:boto:Token: None 2022-09-03T07:21:13.294 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:13.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WbM7cq+Y5dRKbP+NRJWu/teUoUA= 2022-09-03T07:21:13.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WbM7cq+Y5dRKbP+NRJWu/teUoUA=', 'Content-Length': '0'} 2022-09-03T07:21:13.297 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c0c9d479021ad3c-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.298 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.299 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.299 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:13.300 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:13.300 DEBUG:boto:Method: GET 2022-09-03T07:21:13.301 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:13.301 DEBUG:boto:Data: 2022-09-03T07:21:13.302 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.302 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.303 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.303 DEBUG:boto:Params: {} 2022-09-03T07:21:13.304 DEBUG:boto:Token: None 2022-09-03T07:21:13.305 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.308 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f5fc36e92b052a73-0063130069-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.309 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.310 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.311 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.311 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:13.312 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:13.312 DEBUG:boto:Method: GET 2022-09-03T07:21:13.313 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:13.313 DEBUG:boto:Data: 2022-09-03T07:21:13.314 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.315 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.315 DEBUG:boto:Params: {} 2022-09-03T07:21:13.316 DEBUG:boto:Token: None 2022-09-03T07:21:13.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.321 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000081121d8c145d7e67-0063130069-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.322 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.323 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.324 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.324 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.325 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:13.325 DEBUG:boto:Data: 2022-09-03T07:21:13.326 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.326 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.327 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.328 DEBUG:boto:Params: {} 2022-09-03T07:21:13.328 DEBUG:boto:Token: None 2022-09-03T07:21:13.329 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76 2022-09-03T07:21:13.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YwdwTqtCk1aYVP39B/EU1EyFiI= 2022-09-03T07:21:13.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YwdwTqtCk1aYVP39B/EU1EyFiI=', 'Content-Length': '0'} 2022-09-03T07:21:13.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000085cc0a461642747-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.333 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.333 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.334 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.334 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:13.335 DEBUG:boto:Data: 2022-09-03T07:21:13.335 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.336 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.337 DEBUG:boto:Params: {} 2022-09-03T07:21:13.337 DEBUG:boto:Token: None 2022-09-03T07:21:13.338 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76 2022-09-03T07:21:13.338 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YwdwTqtCk1aYVP39B/EU1EyFiI= 2022-09-03T07:21:13.339 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YwdwTqtCk1aYVP39B/EU1EyFiI=', 'Content-Length': '0'} 2022-09-03T07:21:13.341 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001073fd1b9c0c8ef9-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.341 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:21:13.342 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.343 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.343 DEBUG:boto:Method: GET 2022-09-03T07:21:13.344 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:13.344 DEBUG:boto:Data: 2022-09-03T07:21:13.345 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.345 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.346 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.346 DEBUG:boto:Params: {} 2022-09-03T07:21:13.347 DEBUG:boto:Token: None 2022-09-03T07:21:13.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76 2022-09-03T07:21:13.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rLrU2bbglZL3ZgTI1FCCEfh5GSQ= 2022-09-03T07:21:13.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rLrU2bbglZL3ZgTI1FCCEfh5GSQ=', 'Content-Length': '0'} 2022-09-03T07:21:13.351 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000520a82b797ef0a4a-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.351 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:13.352 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:13.352 DEBUG:boto:Method: GET 2022-09-03T07:21:13.353 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:13.353 DEBUG:boto:Data: 2022-09-03T07:21:13.354 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.354 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.355 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.355 DEBUG:boto:Params: {} 2022-09-03T07:21:13.356 DEBUG:boto:Token: None 2022-09-03T07:21:13.357 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj76 2022-09-03T07:21:13.357 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rLrU2bbglZL3ZgTI1FCCEfh5GSQ= 2022-09-03T07:21:13.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rLrU2bbglZL3ZgTI1FCCEfh5GSQ=', 'Content-Length': '0'} 2022-09-03T07:21:13.360 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7e9317dfdbe9241-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.361 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.361 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.362 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:13.362 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:13.363 DEBUG:boto:Method: GET 2022-09-03T07:21:13.363 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:13.364 DEBUG:boto:Data: 2022-09-03T07:21:13.364 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.365 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.365 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.366 DEBUG:boto:Params: {} 2022-09-03T07:21:13.366 DEBUG:boto:Token: None 2022-09-03T07:21:13.367 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.368 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.370 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000902f20cfcb6c4b90-0063130069-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.371 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.372 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.372 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.373 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:13.373 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:13.374 DEBUG:boto:Method: GET 2022-09-03T07:21:13.374 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:13.375 DEBUG:boto:Data: 2022-09-03T07:21:13.375 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.376 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.376 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.377 DEBUG:boto:Params: {} 2022-09-03T07:21:13.377 DEBUG:boto:Token: None 2022-09-03T07:21:13.378 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.379 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.382 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003eb8ca1ab0d70070-0063130069-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.383 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.384 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:21:13.384 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.385 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.385 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.386 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.387 DEBUG:boto:Method: GET 2022-09-03T07:21:13.387 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.388 DEBUG:boto:Data: 2022-09-03T07:21:13.388 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.389 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.389 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.390 DEBUG:boto:Params: {} 2022-09-03T07:21:13.390 DEBUG:boto:Token: None 2022-09-03T07:21:13.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lrHI+cB7SGtX8TDqw/ziLIKVPwk= 2022-09-03T07:21:13.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lrHI+cB7SGtX8TDqw/ziLIKVPwk=', 'Content-Length': '0'} 2022-09-03T07:21:13.395 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000836928bccc408e52-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.438 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.439 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.439 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.440 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.440 DEBUG:boto:Method: GET 2022-09-03T07:21:13.441 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.442 DEBUG:boto:Data: 2022-09-03T07:21:13.442 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.443 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.444 DEBUG:boto:Params: {} 2022-09-03T07:21:13.445 DEBUG:boto:Token: None 2022-09-03T07:21:13.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lrHI+cB7SGtX8TDqw/ziLIKVPwk= 2022-09-03T07:21:13.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lrHI+cB7SGtX8TDqw/ziLIKVPwk=', 'Content-Length': '0'} 2022-09-03T07:21:13.449 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090b8d960ecadd907-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.450 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.451 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.451 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.452 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.452 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.453 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.453 DEBUG:boto:Data: 2022-09-03T07:21:13.454 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.454 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.455 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.455 DEBUG:boto:Params: {} 2022-09-03T07:21:13.456 DEBUG:boto:Token: None 2022-09-03T07:21:13.457 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B3MCDusZPbGLPcF06N1z6bTSIco= 2022-09-03T07:21:13.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B3MCDusZPbGLPcF06N1z6bTSIco=', 'Content-Length': '0'} 2022-09-03T07:21:13.460 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057916ed63bc32dec-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.461 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.462 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.463 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.463 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.464 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.464 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.465 DEBUG:boto:Data: 2022-09-03T07:21:13.465 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.466 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.467 DEBUG:boto:Params: {} 2022-09-03T07:21:13.467 DEBUG:boto:Token: None 2022-09-03T07:21:13.468 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B3MCDusZPbGLPcF06N1z6bTSIco= 2022-09-03T07:21:13.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B3MCDusZPbGLPcF06N1z6bTSIco=', 'Content-Length': '0'} 2022-09-03T07:21:13.472 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000968ce9d42065793f-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.472 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:21:13.473 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.473 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.474 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.474 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.475 DEBUG:boto:Method: GET 2022-09-03T07:21:13.476 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.476 DEBUG:boto:Data: 2022-09-03T07:21:13.477 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.477 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.478 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.479 DEBUG:boto:Params: {} 2022-09-03T07:21:13.479 DEBUG:boto:Token: None 2022-09-03T07:21:13.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lrHI+cB7SGtX8TDqw/ziLIKVPwk= 2022-09-03T07:21:13.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lrHI+cB7SGtX8TDqw/ziLIKVPwk=', 'Content-Length': '0'} 2022-09-03T07:21:13.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1635186f9711db3-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.484 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.485 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.485 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.486 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.486 DEBUG:boto:Method: GET 2022-09-03T07:21:13.487 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.487 DEBUG:boto:Data: 2022-09-03T07:21:13.488 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.489 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.489 DEBUG:boto:Params: {} 2022-09-03T07:21:13.490 DEBUG:boto:Token: None 2022-09-03T07:21:13.491 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:13.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lrHI+cB7SGtX8TDqw/ziLIKVPwk= 2022-09-03T07:21:13.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lrHI+cB7SGtX8TDqw/ziLIKVPwk=', 'Content-Length': '0'} 2022-09-03T07:21:13.494 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009678ba47463ad50e-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.495 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.496 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.496 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:13.497 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:13.497 DEBUG:boto:Method: GET 2022-09-03T07:21:13.498 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:13.498 DEBUG:boto:Data: 2022-09-03T07:21:13.499 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.500 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.500 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.501 DEBUG:boto:Params: {} 2022-09-03T07:21:13.501 DEBUG:boto:Token: None 2022-09-03T07:21:13.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.506 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000869f90e691482bc2-0063130069-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.507 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.508 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.508 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.509 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:13.509 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:13.510 DEBUG:boto:Method: GET 2022-09-03T07:21:13.510 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:13.511 DEBUG:boto:Data: 2022-09-03T07:21:13.511 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.513 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.513 DEBUG:boto:Params: {} 2022-09-03T07:21:13.514 DEBUG:boto:Token: None 2022-09-03T07:21:13.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.519 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000042fb55f679a5480e-0063130069-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.520 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.521 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.522 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.522 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.523 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:13.523 DEBUG:boto:Data: 2022-09-03T07:21:13.524 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.524 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.524 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.525 DEBUG:boto:Params: {} 2022-09-03T07:21:13.526 DEBUG:boto:Token: None 2022-09-03T07:21:13.527 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85 2022-09-03T07:21:13.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zaabek3N9JsclR1U1uLjqTR7Q8c= 2022-09-03T07:21:13.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zaabek3N9JsclR1U1uLjqTR7Q8c=', 'Content-Length': '0'} 2022-09-03T07:21:13.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ed22f8ad71736a0-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.531 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.532 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.533 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.533 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:13.534 DEBUG:boto:Data: 2022-09-03T07:21:13.535 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.536 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.537 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.537 DEBUG:boto:Params: {} 2022-09-03T07:21:13.538 DEBUG:boto:Token: None 2022-09-03T07:21:13.538 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85 2022-09-03T07:21:13.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zaabek3N9JsclR1U1uLjqTR7Q8c= 2022-09-03T07:21:13.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zaabek3N9JsclR1U1uLjqTR7Q8c=', 'Content-Length': '0'} 2022-09-03T07:21:13.542 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b5bbf984b907f70c-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.542 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:21:13.543 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.543 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.544 DEBUG:boto:Method: GET 2022-09-03T07:21:13.544 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:13.545 DEBUG:boto:Data: 2022-09-03T07:21:13.546 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.546 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.547 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.547 DEBUG:boto:Params: {} 2022-09-03T07:21:13.548 DEBUG:boto:Token: None 2022-09-03T07:21:13.548 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85 2022-09-03T07:21:13.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UVD73QelCdMhXPBAayh1kEvX0KM= 2022-09-03T07:21:13.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UVD73QelCdMhXPBAayh1kEvX0KM=', 'Content-Length': '0'} 2022-09-03T07:21:13.551 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000efd0737cfd60f36-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.552 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:13.553 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:13.553 DEBUG:boto:Method: GET 2022-09-03T07:21:13.554 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:13.554 DEBUG:boto:Data: 2022-09-03T07:21:13.555 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.556 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.556 DEBUG:boto:Params: {} 2022-09-03T07:21:13.557 DEBUG:boto:Token: None 2022-09-03T07:21:13.558 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj85 2022-09-03T07:21:13.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UVD73QelCdMhXPBAayh1kEvX0KM= 2022-09-03T07:21:13.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UVD73QelCdMhXPBAayh1kEvX0KM=', 'Content-Length': '0'} 2022-09-03T07:21:13.561 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ffd807c3808a504-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.606 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.607 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.607 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:13.608 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:13.609 DEBUG:boto:Method: GET 2022-09-03T07:21:13.609 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:13.610 DEBUG:boto:Data: 2022-09-03T07:21:13.611 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.611 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.612 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.612 DEBUG:boto:Params: {} 2022-09-03T07:21:13.613 DEBUG:boto:Token: None 2022-09-03T07:21:13.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.619 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d38d16c7d974574c-0063130069-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.620 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.621 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.622 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.622 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:13.623 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:13.623 DEBUG:boto:Method: GET 2022-09-03T07:21:13.624 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:13.625 DEBUG:boto:Data: 2022-09-03T07:21:13.626 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.628 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.629 DEBUG:boto:Params: {} 2022-09-03T07:21:13.630 DEBUG:boto:Token: None 2022-09-03T07:21:13.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.638 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3232bc90795ec29-0063130069-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.639 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.640 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:21:13.641 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.642 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.643 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.644 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.645 DEBUG:boto:Method: GET 2022-09-03T07:21:13.646 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.647 DEBUG:boto:Data: 2022-09-03T07:21:13.648 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.649 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.650 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.651 DEBUG:boto:Params: {} 2022-09-03T07:21:13.653 DEBUG:boto:Token: None 2022-09-03T07:21:13.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uaLGZkCf/Fcb4HmlgNeh00JaCbY= 2022-09-03T07:21:13.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uaLGZkCf/Fcb4HmlgNeh00JaCbY=', 'Content-Length': '0'} 2022-09-03T07:21:13.658 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c0c43670209e140-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.702 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.704 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.705 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.706 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.706 DEBUG:boto:Method: GET 2022-09-03T07:21:13.707 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.708 DEBUG:boto:Data: 2022-09-03T07:21:13.709 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.711 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.712 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.712 DEBUG:boto:Params: {} 2022-09-03T07:21:13.713 DEBUG:boto:Token: None 2022-09-03T07:21:13.714 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.714 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uaLGZkCf/Fcb4HmlgNeh00JaCbY= 2022-09-03T07:21:13.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uaLGZkCf/Fcb4HmlgNeh00JaCbY=', 'Content-Length': '0'} 2022-09-03T07:21:13.717 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022b275ff59f83f98-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.718 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.719 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.719 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.720 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.720 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.721 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.721 DEBUG:boto:Data: 2022-09-03T07:21:13.722 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.722 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.723 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.724 DEBUG:boto:Params: {} 2022-09-03T07:21:13.724 DEBUG:boto:Token: None 2022-09-03T07:21:13.725 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oekc8eW2AWNCzcWDTMP2jbQtVzY= 2022-09-03T07:21:13.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oekc8eW2AWNCzcWDTMP2jbQtVzY=', 'Content-Length': '0'} 2022-09-03T07:21:13.731 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005dd5be32001df74c-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.732 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.733 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.733 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.734 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.734 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.735 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.736 DEBUG:boto:Data: 2022-09-03T07:21:13.736 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.737 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.738 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.739 DEBUG:boto:Params: {} 2022-09-03T07:21:13.740 DEBUG:boto:Token: None 2022-09-03T07:21:13.742 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oekc8eW2AWNCzcWDTMP2jbQtVzY= 2022-09-03T07:21:13.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oekc8eW2AWNCzcWDTMP2jbQtVzY=', 'Content-Length': '0'} 2022-09-03T07:21:13.747 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e7e7f91ebdaf3e8-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.748 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:21:13.749 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.750 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.751 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.752 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.752 DEBUG:boto:Method: GET 2022-09-03T07:21:13.754 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.755 DEBUG:boto:Data: 2022-09-03T07:21:13.756 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.756 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.757 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.758 DEBUG:boto:Params: {} 2022-09-03T07:21:13.759 DEBUG:boto:Token: None 2022-09-03T07:21:13.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uaLGZkCf/Fcb4HmlgNeh00JaCbY= 2022-09-03T07:21:13.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uaLGZkCf/Fcb4HmlgNeh00JaCbY=', 'Content-Length': '0'} 2022-09-03T07:21:13.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000381085742911edfb-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.764 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.764 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.765 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.766 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.766 DEBUG:boto:Method: GET 2022-09-03T07:21:13.767 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.767 DEBUG:boto:Data: 2022-09-03T07:21:13.768 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.769 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.769 DEBUG:boto:Params: {} 2022-09-03T07:21:13.770 DEBUG:boto:Token: None 2022-09-03T07:21:13.771 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:13.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uaLGZkCf/Fcb4HmlgNeh00JaCbY= 2022-09-03T07:21:13.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uaLGZkCf/Fcb4HmlgNeh00JaCbY=', 'Content-Length': '0'} 2022-09-03T07:21:13.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab5faf4f0060d6cf-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.775 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.775 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.776 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:13.777 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:13.777 DEBUG:boto:Method: GET 2022-09-03T07:21:13.778 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:13.778 DEBUG:boto:Data: 2022-09-03T07:21:13.779 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.779 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.780 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.780 DEBUG:boto:Params: {} 2022-09-03T07:21:13.781 DEBUG:boto:Token: None 2022-09-03T07:21:13.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.785 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c158c0424e12542-0063130069-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.790 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.791 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.791 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.792 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:13.792 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:13.793 DEBUG:boto:Method: GET 2022-09-03T07:21:13.794 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:13.794 DEBUG:boto:Data: 2022-09-03T07:21:13.795 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.796 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.796 DEBUG:boto:Params: {} 2022-09-03T07:21:13.797 DEBUG:boto:Token: None 2022-09-03T07:21:13.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.803 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ca30dda5738f2239-0063130069-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.850 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.851 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.852 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.852 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.853 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:13.853 DEBUG:boto:Data: 2022-09-03T07:21:13.854 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.854 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.855 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.855 DEBUG:boto:Params: {} 2022-09-03T07:21:13.856 DEBUG:boto:Token: None 2022-09-03T07:21:13.857 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95 2022-09-03T07:21:13.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2lBlvTOHNJkQkgFq3RwEXAR0Pc= 2022-09-03T07:21:13.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2lBlvTOHNJkQkgFq3RwEXAR0Pc=', 'Content-Length': '0'} 2022-09-03T07:21:13.860 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000094c85c4a0926ef8b-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.862 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.862 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.863 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.863 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:13.864 DEBUG:boto:Data: 2022-09-03T07:21:13.864 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.865 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.866 DEBUG:boto:Params: {} 2022-09-03T07:21:13.866 DEBUG:boto:Token: None 2022-09-03T07:21:13.867 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95 2022-09-03T07:21:13.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z2lBlvTOHNJkQkgFq3RwEXAR0Pc= 2022-09-03T07:21:13.868 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z2lBlvTOHNJkQkgFq3RwEXAR0Pc=', 'Content-Length': '0'} 2022-09-03T07:21:13.871 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000011dfbad22745caaf-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.872 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:21:13.873 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.874 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.875 DEBUG:boto:Method: GET 2022-09-03T07:21:13.876 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:13.876 DEBUG:boto:Data: 2022-09-03T07:21:13.877 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.877 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.878 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.878 DEBUG:boto:Params: {} 2022-09-03T07:21:13.879 DEBUG:boto:Token: None 2022-09-03T07:21:13.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95 2022-09-03T07:21:13.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a0uX4d4wFJ8yqjnAydKLOdSoTNg= 2022-09-03T07:21:13.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a0uX4d4wFJ8yqjnAydKLOdSoTNg=', 'Content-Length': '0'} 2022-09-03T07:21:13.883 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ccd3060031151e95-0063130069-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.883 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:13.884 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:13.885 DEBUG:boto:Method: GET 2022-09-03T07:21:13.885 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:13.886 DEBUG:boto:Data: 2022-09-03T07:21:13.886 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.887 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.887 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.888 DEBUG:boto:Params: {} 2022-09-03T07:21:13.889 DEBUG:boto:Token: None 2022-09-03T07:21:13.890 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/obj95 2022-09-03T07:21:13.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a0uX4d4wFJ8yqjnAydKLOdSoTNg= 2022-09-03T07:21:13.891 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a0uX4d4wFJ8yqjnAydKLOdSoTNg=', 'Content-Length': '0'} 2022-09-03T07:21:13.893 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e299f03e4675355-0063130069-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.894 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.895 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.896 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:13.896 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:13.897 DEBUG:boto:Method: GET 2022-09-03T07:21:13.897 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:13.898 DEBUG:boto:Data: 2022-09-03T07:21:13.898 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:13.899 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.900 DEBUG:boto:Params: {} 2022-09-03T07:21:13.900 DEBUG:boto:Token: None 2022-09-03T07:21:13.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.906 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003a12e9adf70c5529-0063130069-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.946 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.948 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.949 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.950 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:13.951 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:13.952 DEBUG:boto:Method: GET 2022-09-03T07:21:13.953 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:13.953 DEBUG:boto:Data: 2022-09-03T07:21:13.954 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.955 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.956 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.957 DEBUG:boto:Params: {} 2022-09-03T07:21:13.958 DEBUG:boto:Token: None 2022-09-03T07:21:13.959 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA= 2022-09-03T07:21:13.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rxh5L97EOiIY9f9GY8OrYU8RRsA=', 'Content-Length': '0'} 2022-09-03T07:21:13.966 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000030e4c438328a15a2-0063130069-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.967 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:13.968 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:21:13.969 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:21:13.970 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.971 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.972 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.973 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:13.974 DEBUG:boto:Data: 2022-09-03T07:21:13.975 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.976 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:13.976 DEBUG:boto:Port: 8001 2022-09-03T07:21:13.977 DEBUG:boto:Params: {} 2022-09-03T07:21:13.979 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:21:13.980 DEBUG:boto:Token: None 2022-09-03T07:21:13.981 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J82AHR46PvobtkKHMtSb3kPq4aM= 2022-09-03T07:21:13.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J82AHR46PvobtkKHMtSb3kPq4aM=', 'Content-Length': '0'} 2022-09-03T07:21:13.988 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '15'), ('X-RGW-Bytes-Used', '45'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001081ba7ca7edf011-0063130069-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:13.989 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:13.990 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:13.990 DEBUG:boto:Method: HEAD 2022-09-03T07:21:13.991 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:13.992 DEBUG:boto:Data: 2022-09-03T07:21:13.993 DEBUG:boto:Headers: {} 2022-09-03T07:21:13.994 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:13.995 DEBUG:boto:Port: 8000 2022-09-03T07:21:13.996 DEBUG:boto:Params: {} 2022-09-03T07:21:13.997 DEBUG:boto:Token: None 2022-09-03T07:21:13.998 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:13 GMT /avysmw-75/ 2022-09-03T07:21:13.999 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J82AHR46PvobtkKHMtSb3kPq4aM= 2022-09-03T07:21:13.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J82AHR46PvobtkKHMtSb3kPq4aM=', 'Content-Length': '0'} 2022-09-03T07:21:14.003 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '15'), ('X-RGW-Bytes-Used', '45'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000019c6ba02a4a1a9f5-0063130069-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.003 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:21:14.004 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.005 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.005 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.006 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.007 DEBUG:boto:Method: GET 2022-09-03T07:21:14.007 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.008 DEBUG:boto:Data: 2022-09-03T07:21:14.009 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.010 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.010 DEBUG:boto:Params: {} 2022-09-03T07:21:14.011 DEBUG:boto:Token: None 2022-09-03T07:21:14.011 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/?versions 2022-09-03T07:21:14.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ydgKXmSXrGfBlpw4AkxreYgZOV0= 2022-09-03T07:21:14.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ydgKXmSXrGfBlpw4AkxreYgZOV0=', 'Content-Length': '0'} 2022-09-03T07:21:14.015 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000517c069a053e1de-006313006a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.016 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.018 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:21:14.018 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:21:14.019 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:21:14.019 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:21:14.020 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:21:14.021 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:21:14.021 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:21:14.022 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:21:14.023 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:21:14.023 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:21:14.024 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:21:14.024 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:21:14.025 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:21:14.025 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:21:14.026 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:21:14.026 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:21:14.027 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.028 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.028 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.029 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.029 DEBUG:boto:Method: GET 2022-09-03T07:21:14.030 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.031 DEBUG:boto:Data: 2022-09-03T07:21:14.031 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.032 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.033 DEBUG:boto:Params: {} 2022-09-03T07:21:14.034 DEBUG:boto:Token: None 2022-09-03T07:21:14.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/?versions 2022-09-03T07:21:14.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ydgKXmSXrGfBlpw4AkxreYgZOV0= 2022-09-03T07:21:14.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ydgKXmSXrGfBlpw4AkxreYgZOV0=', 'Content-Length': '0'} 2022-09-03T07:21:14.039 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ef3feea4dfb1747-006313006a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.041 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.043 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:21:14.044 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:21:14.045 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:21:14.045 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:21:14.046 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:21:14.047 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:21:14.048 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:21:14.049 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:21:14.050 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:21:14.050 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:21:14.051 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:21:14.052 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:21:14.053 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:21:14.054 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:21:14.055 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:21:14.055 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.056 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.057 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.058 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.059 DEBUG:boto:Method: GET 2022-09-03T07:21:14.059 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.060 DEBUG:boto:Data: 2022-09-03T07:21:14.061 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.062 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.062 DEBUG:boto:Params: {} 2022-09-03T07:21:14.063 DEBUG:boto:Token: None 2022-09-03T07:21:14.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/?versions 2022-09-03T07:21:14.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ydgKXmSXrGfBlpw4AkxreYgZOV0= 2022-09-03T07:21:14.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ydgKXmSXrGfBlpw4AkxreYgZOV0=', 'Content-Length': '0'} 2022-09-03T07:21:14.068 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000015f39051fb44d80-006313006a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.068 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.070 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.070 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.071 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.071 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.072 DEBUG:boto:Method: GET 2022-09-03T07:21:14.072 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:14.073 DEBUG:boto:Data: 2022-09-03T07:21:14.074 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.074 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.075 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.075 DEBUG:boto:Params: {} 2022-09-03T07:21:14.076 DEBUG:boto:Token: None 2022-09-03T07:21:14.076 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/?versions 2022-09-03T07:21:14.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ydgKXmSXrGfBlpw4AkxreYgZOV0= 2022-09-03T07:21:14.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ydgKXmSXrGfBlpw4AkxreYgZOV0=', 'Content-Length': '0'} 2022-09-03T07:21:14.081 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000049d029dadb987c9d-006313006a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.082 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.083 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:21:14.084 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.084 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.085 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.085 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.086 DEBUG:boto:Method: GET 2022-09-03T07:21:14.086 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.087 DEBUG:boto:Data: 2022-09-03T07:21:14.087 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.088 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.088 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.089 DEBUG:boto:Params: {} 2022-09-03T07:21:14.090 DEBUG:boto:Token: None 2022-09-03T07:21:14.090 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.091 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q/6OSJA3Gix74CVm1KCJIFvFshs= 2022-09-03T07:21:14.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q/6OSJA3Gix74CVm1KCJIFvFshs=', 'Content-Length': '0'} 2022-09-03T07:21:14.094 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000079da73460223a7d-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.095 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.095 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.096 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.097 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.097 DEBUG:boto:Method: GET 2022-09-03T07:21:14.098 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.098 DEBUG:boto:Data: 2022-09-03T07:21:14.099 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.100 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.100 DEBUG:boto:Params: {} 2022-09-03T07:21:14.101 DEBUG:boto:Token: None 2022-09-03T07:21:14.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q/6OSJA3Gix74CVm1KCJIFvFshs= 2022-09-03T07:21:14.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q/6OSJA3Gix74CVm1KCJIFvFshs=', 'Content-Length': '0'} 2022-09-03T07:21:14.105 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a702a320a052bd0-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.106 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.107 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.107 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.108 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.108 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.109 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.109 DEBUG:boto:Data: 2022-09-03T07:21:14.110 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.111 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.111 DEBUG:boto:Params: {} 2022-09-03T07:21:14.112 DEBUG:boto:Token: None 2022-09-03T07:21:14.112 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RQnQwf9qluRuJz8gomh3hYeuEuc= 2022-09-03T07:21:14.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RQnQwf9qluRuJz8gomh3hYeuEuc=', 'Content-Length': '0'} 2022-09-03T07:21:14.116 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f777284b8214184b-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.117 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.118 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.118 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.119 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.120 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.120 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.121 DEBUG:boto:Data: 2022-09-03T07:21:14.121 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.122 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.123 DEBUG:boto:Params: {} 2022-09-03T07:21:14.123 DEBUG:boto:Token: None 2022-09-03T07:21:14.124 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RQnQwf9qluRuJz8gomh3hYeuEuc= 2022-09-03T07:21:14.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RQnQwf9qluRuJz8gomh3hYeuEuc=', 'Content-Length': '0'} 2022-09-03T07:21:14.127 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e8361986ceb901e-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.128 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:21:14.128 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.129 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.130 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.130 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.131 DEBUG:boto:Method: GET 2022-09-03T07:21:14.131 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.132 DEBUG:boto:Data: 2022-09-03T07:21:14.132 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.133 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.133 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.134 DEBUG:boto:Params: {} 2022-09-03T07:21:14.134 DEBUG:boto:Token: None 2022-09-03T07:21:14.135 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.136 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q/6OSJA3Gix74CVm1KCJIFvFshs= 2022-09-03T07:21:14.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q/6OSJA3Gix74CVm1KCJIFvFshs=', 'Content-Length': '0'} 2022-09-03T07:21:14.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c40a52b4f351e0e5-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.182 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.183 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.183 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.184 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.184 DEBUG:boto:Method: GET 2022-09-03T07:21:14.185 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.185 DEBUG:boto:Data: 2022-09-03T07:21:14.186 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.188 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.189 DEBUG:boto:Params: {} 2022-09-03T07:21:14.190 DEBUG:boto:Token: None 2022-09-03T07:21:14.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:14.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q/6OSJA3Gix74CVm1KCJIFvFshs= 2022-09-03T07:21:14.194 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q/6OSJA3Gix74CVm1KCJIFvFshs=', 'Content-Length': '0'} 2022-09-03T07:21:14.197 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d42960f980991ebb-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.198 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.199 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.199 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:14.200 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:14.201 DEBUG:boto:Method: GET 2022-09-03T07:21:14.202 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:14.202 DEBUG:boto:Data: 2022-09-03T07:21:14.203 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.204 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.204 DEBUG:boto:Params: {} 2022-09-03T07:21:14.205 DEBUG:boto:Token: None 2022-09-03T07:21:14.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1b2e9c22b593fa9-006313006a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.210 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.211 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.211 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.212 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:14.212 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:14.213 DEBUG:boto:Method: GET 2022-09-03T07:21:14.213 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:14.214 DEBUG:boto:Data: 2022-09-03T07:21:14.215 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.216 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.216 DEBUG:boto:Params: {} 2022-09-03T07:21:14.217 DEBUG:boto:Token: None 2022-09-03T07:21:14.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.222 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000064297b73ae2b78de-006313006a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.223 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.223 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.224 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.224 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.225 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:14.225 DEBUG:boto:Data: 2022-09-03T07:21:14.226 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.226 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.227 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.227 DEBUG:boto:Params: {} 2022-09-03T07:21:14.228 DEBUG:boto:Token: None 2022-09-03T07:21:14.228 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11 2022-09-03T07:21:14.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TqHSU9vif2WX/5i1VNDqlyJFZsU= 2022-09-03T07:21:14.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TqHSU9vif2WX/5i1VNDqlyJFZsU=', 'Content-Length': '0'} 2022-09-03T07:21:14.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef24f4cc09f6ee67-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.233 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.234 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.235 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.235 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:14.236 DEBUG:boto:Data: 2022-09-03T07:21:14.236 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.237 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.238 DEBUG:boto:Params: {} 2022-09-03T07:21:14.238 DEBUG:boto:Token: None 2022-09-03T07:21:14.239 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11 2022-09-03T07:21:14.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TqHSU9vif2WX/5i1VNDqlyJFZsU= 2022-09-03T07:21:14.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TqHSU9vif2WX/5i1VNDqlyJFZsU=', 'Content-Length': '0'} 2022-09-03T07:21:14.242 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ee820bdd7dc6aed-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.243 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:21:14.244 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.245 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.245 DEBUG:boto:Method: GET 2022-09-03T07:21:14.246 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:14.246 DEBUG:boto:Data: 2022-09-03T07:21:14.247 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.247 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.248 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.248 DEBUG:boto:Params: {} 2022-09-03T07:21:14.249 DEBUG:boto:Token: None 2022-09-03T07:21:14.249 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11 2022-09-03T07:21:14.250 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SLnJc3H4kE8Xg62kY9reDGGdsfM= 2022-09-03T07:21:14.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SLnJc3H4kE8Xg62kY9reDGGdsfM=', 'Content-Length': '0'} 2022-09-03T07:21:14.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005169b2b61f0224b6-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.254 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:14.255 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:14.256 DEBUG:boto:Method: GET 2022-09-03T07:21:14.256 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:14.257 DEBUG:boto:Data: 2022-09-03T07:21:14.257 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.258 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.258 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.259 DEBUG:boto:Params: {} 2022-09-03T07:21:14.260 DEBUG:boto:Token: None 2022-09-03T07:21:14.261 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj11 2022-09-03T07:21:14.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SLnJc3H4kE8Xg62kY9reDGGdsfM= 2022-09-03T07:21:14.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SLnJc3H4kE8Xg62kY9reDGGdsfM=', 'Content-Length': '0'} 2022-09-03T07:21:14.265 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075995077728029e9-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.306 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.307 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.308 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:14.309 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:14.310 DEBUG:boto:Method: GET 2022-09-03T07:21:14.310 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:14.311 DEBUG:boto:Data: 2022-09-03T07:21:14.311 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.312 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.313 DEBUG:boto:Params: {} 2022-09-03T07:21:14.313 DEBUG:boto:Token: None 2022-09-03T07:21:14.314 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.318 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022ecf0c757b415a4-006313006a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.319 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.320 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.320 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.321 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:14.321 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:14.322 DEBUG:boto:Method: GET 2022-09-03T07:21:14.322 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:14.323 DEBUG:boto:Data: 2022-09-03T07:21:14.324 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.325 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.325 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.326 DEBUG:boto:Params: {} 2022-09-03T07:21:14.326 DEBUG:boto:Token: None 2022-09-03T07:21:14.327 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.332 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d67cbb9d54134086-006313006a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.332 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.333 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:21:14.334 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.334 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.335 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.335 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.336 DEBUG:boto:Method: GET 2022-09-03T07:21:14.336 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.337 DEBUG:boto:Data: 2022-09-03T07:21:14.337 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.338 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.339 DEBUG:boto:Params: {} 2022-09-03T07:21:14.340 DEBUG:boto:Token: None 2022-09-03T07:21:14.340 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iL1RS8hO+9USSbS0cqMMCmTlqRg= 2022-09-03T07:21:14.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iL1RS8hO+9USSbS0cqMMCmTlqRg=', 'Content-Length': '0'} 2022-09-03T07:21:14.343 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015bdefdb7aa4ffe6-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.344 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.345 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.345 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.346 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.346 DEBUG:boto:Method: GET 2022-09-03T07:21:14.347 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.347 DEBUG:boto:Data: 2022-09-03T07:21:14.348 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.348 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.349 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.349 DEBUG:boto:Params: {} 2022-09-03T07:21:14.350 DEBUG:boto:Token: None 2022-09-03T07:21:14.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iL1RS8hO+9USSbS0cqMMCmTlqRg= 2022-09-03T07:21:14.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iL1RS8hO+9USSbS0cqMMCmTlqRg=', 'Content-Length': '0'} 2022-09-03T07:21:14.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a72f710e9a93d25e-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.355 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.355 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.356 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.356 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.357 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.357 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.358 DEBUG:boto:Data: 2022-09-03T07:21:14.358 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.359 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.360 DEBUG:boto:Params: {} 2022-09-03T07:21:14.360 DEBUG:boto:Token: None 2022-09-03T07:21:14.361 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZpxByho6RXwBErLgtM7sgqR73t4= 2022-09-03T07:21:14.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZpxByho6RXwBErLgtM7sgqR73t4=', 'Content-Length': '0'} 2022-09-03T07:21:14.364 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2607b8b6f1edcfb-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.365 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.365 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.366 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.366 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.367 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.367 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.368 DEBUG:boto:Data: 2022-09-03T07:21:14.368 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.369 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.369 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.370 DEBUG:boto:Params: {} 2022-09-03T07:21:14.371 DEBUG:boto:Token: None 2022-09-03T07:21:14.371 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZpxByho6RXwBErLgtM7sgqR73t4= 2022-09-03T07:21:14.372 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZpxByho6RXwBErLgtM7sgqR73t4=', 'Content-Length': '0'} 2022-09-03T07:21:14.375 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036b606b9f60fdb2e-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.375 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:21:14.376 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.376 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.377 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.377 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.378 DEBUG:boto:Method: GET 2022-09-03T07:21:14.378 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.379 DEBUG:boto:Data: 2022-09-03T07:21:14.380 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.380 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.381 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.381 DEBUG:boto:Params: {} 2022-09-03T07:21:14.382 DEBUG:boto:Token: None 2022-09-03T07:21:14.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iL1RS8hO+9USSbS0cqMMCmTlqRg= 2022-09-03T07:21:14.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iL1RS8hO+9USSbS0cqMMCmTlqRg=', 'Content-Length': '0'} 2022-09-03T07:21:14.386 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e8d7ae5dac9eb88-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.386 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.387 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.388 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.388 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.389 DEBUG:boto:Method: GET 2022-09-03T07:21:14.389 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.390 DEBUG:boto:Data: 2022-09-03T07:21:14.390 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.391 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.391 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.392 DEBUG:boto:Params: {} 2022-09-03T07:21:14.392 DEBUG:boto:Token: None 2022-09-03T07:21:14.393 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:14.394 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iL1RS8hO+9USSbS0cqMMCmTlqRg= 2022-09-03T07:21:14.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iL1RS8hO+9USSbS0cqMMCmTlqRg=', 'Content-Length': '0'} 2022-09-03T07:21:14.396 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b4f6ed70aae6ccf-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.438 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.439 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.439 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:14.440 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:14.440 DEBUG:boto:Method: GET 2022-09-03T07:21:14.441 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:14.441 DEBUG:boto:Data: 2022-09-03T07:21:14.442 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.443 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.443 DEBUG:boto:Params: {} 2022-09-03T07:21:14.444 DEBUG:boto:Token: None 2022-09-03T07:21:14.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.449 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000390177783a8ff01a-006313006a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.449 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.450 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.451 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.451 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:14.452 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:14.452 DEBUG:boto:Method: GET 2022-09-03T07:21:14.453 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:14.453 DEBUG:boto:Data: 2022-09-03T07:21:14.454 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.454 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.455 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.456 DEBUG:boto:Params: {} 2022-09-03T07:21:14.456 DEBUG:boto:Token: None 2022-09-03T07:21:14.457 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.461 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000015652831f2cf8c0c-006313006a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.462 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.463 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.463 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.464 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.464 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:14.465 DEBUG:boto:Data: 2022-09-03T07:21:14.465 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.466 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.467 DEBUG:boto:Params: {} 2022-09-03T07:21:14.467 DEBUG:boto:Token: None 2022-09-03T07:21:14.468 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17 2022-09-03T07:21:14.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uu8jXoZN9pkLFrx1cwzxVGZQ/5U= 2022-09-03T07:21:14.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uu8jXoZN9pkLFrx1cwzxVGZQ/5U=', 'Content-Length': '0'} 2022-09-03T07:21:14.471 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003316601637a401b4-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.472 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.472 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.473 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.473 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:14.474 DEBUG:boto:Data: 2022-09-03T07:21:14.474 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.475 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.476 DEBUG:boto:Params: {} 2022-09-03T07:21:14.476 DEBUG:boto:Token: None 2022-09-03T07:21:14.477 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17 2022-09-03T07:21:14.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uu8jXoZN9pkLFrx1cwzxVGZQ/5U= 2022-09-03T07:21:14.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uu8jXoZN9pkLFrx1cwzxVGZQ/5U=', 'Content-Length': '0'} 2022-09-03T07:21:14.480 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3092b2cf1c24d6f-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.481 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:21:14.481 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.482 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.483 DEBUG:boto:Method: GET 2022-09-03T07:21:14.483 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:14.484 DEBUG:boto:Data: 2022-09-03T07:21:14.484 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.485 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.486 DEBUG:boto:Params: {} 2022-09-03T07:21:14.486 DEBUG:boto:Token: None 2022-09-03T07:21:14.487 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17 2022-09-03T07:21:14.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TDpzqnhfyXvIZY5SH69+GYC6B24= 2022-09-03T07:21:14.489 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TDpzqnhfyXvIZY5SH69+GYC6B24=', 'Content-Length': '0'} 2022-09-03T07:21:14.490 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa381aa7e9de685a-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.491 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:14.492 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:14.492 DEBUG:boto:Method: GET 2022-09-03T07:21:14.493 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:14.493 DEBUG:boto:Data: 2022-09-03T07:21:14.494 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.494 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.495 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.496 DEBUG:boto:Params: {} 2022-09-03T07:21:14.496 DEBUG:boto:Token: None 2022-09-03T07:21:14.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj17 2022-09-03T07:21:14.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TDpzqnhfyXvIZY5SH69+GYC6B24= 2022-09-03T07:21:14.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TDpzqnhfyXvIZY5SH69+GYC6B24=', 'Content-Length': '0'} 2022-09-03T07:21:14.500 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9c42ad714a4397a-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.501 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.501 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.502 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:14.503 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:14.503 DEBUG:boto:Method: GET 2022-09-03T07:21:14.504 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:14.504 DEBUG:boto:Data: 2022-09-03T07:21:14.505 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.506 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.506 DEBUG:boto:Params: {} 2022-09-03T07:21:14.507 DEBUG:boto:Token: None 2022-09-03T07:21:14.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.511 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db569114b347208b-006313006a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.558 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.559 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.560 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.560 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:14.561 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:14.561 DEBUG:boto:Method: GET 2022-09-03T07:21:14.562 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:14.562 DEBUG:boto:Data: 2022-09-03T07:21:14.563 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.563 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.564 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.564 DEBUG:boto:Params: {} 2022-09-03T07:21:14.565 DEBUG:boto:Token: None 2022-09-03T07:21:14.566 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.566 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.570 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000064460e505594c238-006313006a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.571 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.571 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:21:14.572 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.573 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.573 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.574 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.574 DEBUG:boto:Method: GET 2022-09-03T07:21:14.575 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.575 DEBUG:boto:Data: 2022-09-03T07:21:14.576 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.577 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.577 DEBUG:boto:Params: {} 2022-09-03T07:21:14.578 DEBUG:boto:Token: None 2022-09-03T07:21:14.579 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aYAwZd+shxrIvK2FvVoVd/CcasA= 2022-09-03T07:21:14.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aYAwZd+shxrIvK2FvVoVd/CcasA=', 'Content-Length': '0'} 2022-09-03T07:21:14.582 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bb3bdd49418d60fd-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.583 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.583 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.584 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.585 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.585 DEBUG:boto:Method: GET 2022-09-03T07:21:14.586 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.586 DEBUG:boto:Data: 2022-09-03T07:21:14.587 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.588 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.588 DEBUG:boto:Params: {} 2022-09-03T07:21:14.589 DEBUG:boto:Token: None 2022-09-03T07:21:14.590 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aYAwZd+shxrIvK2FvVoVd/CcasA= 2022-09-03T07:21:14.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aYAwZd+shxrIvK2FvVoVd/CcasA=', 'Content-Length': '0'} 2022-09-03T07:21:14.593 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b43b2d5c615f3f2-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.594 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.595 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.595 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.596 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.596 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.597 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.597 DEBUG:boto:Data: 2022-09-03T07:21:14.598 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.598 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.599 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.599 DEBUG:boto:Params: {} 2022-09-03T07:21:14.600 DEBUG:boto:Token: None 2022-09-03T07:21:14.601 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5T/dSaIPG0t63B6PWWtdQ8UCDk8= 2022-09-03T07:21:14.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5T/dSaIPG0t63B6PWWtdQ8UCDk8=', 'Content-Length': '0'} 2022-09-03T07:21:14.604 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066ddddfac4bb0baa-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.605 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.605 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.606 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.606 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.607 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.607 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.608 DEBUG:boto:Data: 2022-09-03T07:21:14.608 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.609 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.609 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.610 DEBUG:boto:Params: {} 2022-09-03T07:21:14.610 DEBUG:boto:Token: None 2022-09-03T07:21:14.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5T/dSaIPG0t63B6PWWtdQ8UCDk8= 2022-09-03T07:21:14.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5T/dSaIPG0t63B6PWWtdQ8UCDk8=', 'Content-Length': '0'} 2022-09-03T07:21:14.614 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dff38472d92e9872-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.615 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:21:14.616 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.616 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.617 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.617 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.618 DEBUG:boto:Method: GET 2022-09-03T07:21:14.618 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.619 DEBUG:boto:Data: 2022-09-03T07:21:14.619 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.620 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.620 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.621 DEBUG:boto:Params: {} 2022-09-03T07:21:14.621 DEBUG:boto:Token: None 2022-09-03T07:21:14.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aYAwZd+shxrIvK2FvVoVd/CcasA= 2022-09-03T07:21:14.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aYAwZd+shxrIvK2FvVoVd/CcasA=', 'Content-Length': '0'} 2022-09-03T07:21:14.625 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000030d1cc907853c8a4-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.626 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.627 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.627 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.628 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.629 DEBUG:boto:Method: GET 2022-09-03T07:21:14.629 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.630 DEBUG:boto:Data: 2022-09-03T07:21:14.630 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.631 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.632 DEBUG:boto:Params: {} 2022-09-03T07:21:14.633 DEBUG:boto:Token: None 2022-09-03T07:21:14.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:14.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aYAwZd+shxrIvK2FvVoVd/CcasA= 2022-09-03T07:21:14.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aYAwZd+shxrIvK2FvVoVd/CcasA=', 'Content-Length': '0'} 2022-09-03T07:21:14.637 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007a18c861f9b5d7cf-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.678 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.679 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.679 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:14.680 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:14.681 DEBUG:boto:Method: GET 2022-09-03T07:21:14.681 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:14.682 DEBUG:boto:Data: 2022-09-03T07:21:14.682 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.683 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.684 DEBUG:boto:Params: {} 2022-09-03T07:21:14.684 DEBUG:boto:Token: None 2022-09-03T07:21:14.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.689 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000029b72444c6817fa-006313006a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.690 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.691 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.691 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.692 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:14.692 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:14.693 DEBUG:boto:Method: GET 2022-09-03T07:21:14.693 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:14.694 DEBUG:boto:Data: 2022-09-03T07:21:14.694 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.695 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.696 DEBUG:boto:Params: {} 2022-09-03T07:21:14.696 DEBUG:boto:Token: None 2022-09-03T07:21:14.697 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.702 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da6a6dc618f3bf68-006313006a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.703 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.704 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.705 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.705 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.706 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:14.706 DEBUG:boto:Data: 2022-09-03T07:21:14.707 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.708 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.708 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.709 DEBUG:boto:Params: {} 2022-09-03T07:21:14.710 DEBUG:boto:Token: None 2022-09-03T07:21:14.711 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19 2022-09-03T07:21:14.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bnhNrax9b/p26d8tshPvGLTiATo= 2022-09-03T07:21:14.712 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bnhNrax9b/p26d8tshPvGLTiATo=', 'Content-Length': '0'} 2022-09-03T07:21:14.714 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000817b753d9ab0aca2-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.722 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.723 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.724 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.724 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:14.725 DEBUG:boto:Data: 2022-09-03T07:21:14.726 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.727 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.728 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.729 DEBUG:boto:Params: {} 2022-09-03T07:21:14.730 DEBUG:boto:Token: None 2022-09-03T07:21:14.731 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19 2022-09-03T07:21:14.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bnhNrax9b/p26d8tshPvGLTiATo= 2022-09-03T07:21:14.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bnhNrax9b/p26d8tshPvGLTiATo=', 'Content-Length': '0'} 2022-09-03T07:21:14.736 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000890b2588d1700b11-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.738 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:21:14.739 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.740 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.741 DEBUG:boto:Method: GET 2022-09-03T07:21:14.742 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:14.743 DEBUG:boto:Data: 2022-09-03T07:21:14.744 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.746 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.747 DEBUG:boto:Params: {} 2022-09-03T07:21:14.749 DEBUG:boto:Token: None 2022-09-03T07:21:14.750 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19 2022-09-03T07:21:14.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CtS7KHnLmD9egwiWrpItdRu3WpE= 2022-09-03T07:21:14.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CtS7KHnLmD9egwiWrpItdRu3WpE=', 'Content-Length': '0'} 2022-09-03T07:21:14.755 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1015c915cc5f94f-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.798 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:14.799 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:14.800 DEBUG:boto:Method: GET 2022-09-03T07:21:14.801 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:14.803 DEBUG:boto:Data: 2022-09-03T07:21:14.804 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.805 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.806 DEBUG:boto:Params: {} 2022-09-03T07:21:14.808 DEBUG:boto:Token: None 2022-09-03T07:21:14.809 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj19 2022-09-03T07:21:14.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CtS7KHnLmD9egwiWrpItdRu3WpE= 2022-09-03T07:21:14.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CtS7KHnLmD9egwiWrpItdRu3WpE=', 'Content-Length': '0'} 2022-09-03T07:21:14.814 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3b03f2d21490f82-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.815 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.816 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.817 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:14.818 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:14.819 DEBUG:boto:Method: GET 2022-09-03T07:21:14.820 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:14.821 DEBUG:boto:Data: 2022-09-03T07:21:14.822 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.823 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.824 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.824 DEBUG:boto:Params: {} 2022-09-03T07:21:14.825 DEBUG:boto:Token: None 2022-09-03T07:21:14.825 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.829 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008aa2e0d6ec255ce5-006313006a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.830 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.831 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.831 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.832 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:14.832 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:14.833 DEBUG:boto:Method: GET 2022-09-03T07:21:14.833 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:14.834 DEBUG:boto:Data: 2022-09-03T07:21:14.834 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.836 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.836 DEBUG:boto:Params: {} 2022-09-03T07:21:14.837 DEBUG:boto:Token: None 2022-09-03T07:21:14.837 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.842 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000895b30d35e5ab012-006313006a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.843 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.843 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:21:14.844 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:14.844 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:14.845 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.846 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.846 DEBUG:boto:Method: GET 2022-09-03T07:21:14.847 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.847 DEBUG:boto:Data: 2022-09-03T07:21:14.848 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.849 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.849 DEBUG:boto:Params: {} 2022-09-03T07:21:14.850 DEBUG:boto:Token: None 2022-09-03T07:21:14.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:91Kqx0v5/g02TBqLXvGIqnuQ8yA= 2022-09-03T07:21:14.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:91Kqx0v5/g02TBqLXvGIqnuQ8yA=', 'Content-Length': '0'} 2022-09-03T07:21:14.854 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d901e526b2dad61-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.855 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:14.855 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:14.856 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.856 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.857 DEBUG:boto:Method: GET 2022-09-03T07:21:14.857 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.858 DEBUG:boto:Data: 2022-09-03T07:21:14.858 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.859 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.859 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.860 DEBUG:boto:Params: {} 2022-09-03T07:21:14.860 DEBUG:boto:Token: None 2022-09-03T07:21:14.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:91Kqx0v5/g02TBqLXvGIqnuQ8yA= 2022-09-03T07:21:14.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:91Kqx0v5/g02TBqLXvGIqnuQ8yA=', 'Content-Length': '0'} 2022-09-03T07:21:14.864 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef3dc57cc3edd4ab-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.906 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:14.907 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:14.908 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.909 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.910 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.911 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.912 DEBUG:boto:Data: 2022-09-03T07:21:14.912 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.913 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.914 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.915 DEBUG:boto:Params: {} 2022-09-03T07:21:14.916 DEBUG:boto:Token: None 2022-09-03T07:21:14.917 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oYTho2z7oZS+NI8BssSP06Y2oXU= 2022-09-03T07:21:14.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oYTho2z7oZS+NI8BssSP06Y2oXU=', 'Content-Length': '0'} 2022-09-03T07:21:14.922 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096ee16edf690d127-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.923 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:14.923 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:14.924 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.924 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.925 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.925 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.926 DEBUG:boto:Data: 2022-09-03T07:21:14.926 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.927 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.928 DEBUG:boto:Params: {} 2022-09-03T07:21:14.929 DEBUG:boto:Token: None 2022-09-03T07:21:14.929 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oYTho2z7oZS+NI8BssSP06Y2oXU= 2022-09-03T07:21:14.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oYTho2z7oZS+NI8BssSP06Y2oXU=', 'Content-Length': '0'} 2022-09-03T07:21:14.933 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000017694f5228087f03-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.934 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:21:14.934 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:14.935 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:14.936 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.936 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.937 DEBUG:boto:Method: GET 2022-09-03T07:21:14.938 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.939 DEBUG:boto:Data: 2022-09-03T07:21:14.939 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.940 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.940 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.941 DEBUG:boto:Params: {} 2022-09-03T07:21:14.941 DEBUG:boto:Token: None 2022-09-03T07:21:14.942 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:91Kqx0v5/g02TBqLXvGIqnuQ8yA= 2022-09-03T07:21:14.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:91Kqx0v5/g02TBqLXvGIqnuQ8yA=', 'Content-Length': '0'} 2022-09-03T07:21:14.946 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dff15fac3fbe638f-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.947 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:14.947 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:14.948 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.948 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.949 DEBUG:boto:Method: GET 2022-09-03T07:21:14.949 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.950 DEBUG:boto:Data: 2022-09-03T07:21:14.951 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.952 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.952 DEBUG:boto:Params: {} 2022-09-03T07:21:14.953 DEBUG:boto:Token: None 2022-09-03T07:21:14.954 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:14.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:91Kqx0v5/g02TBqLXvGIqnuQ8yA= 2022-09-03T07:21:14.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:91Kqx0v5/g02TBqLXvGIqnuQ8yA=', 'Content-Length': '0'} 2022-09-03T07:21:14.957 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e828308fac7b383f-006313006a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.958 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.959 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.959 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:14.960 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:14.960 DEBUG:boto:Method: GET 2022-09-03T07:21:14.961 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:14.962 DEBUG:boto:Data: 2022-09-03T07:21:14.963 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.964 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.964 DEBUG:boto:Params: {} 2022-09-03T07:21:14.965 DEBUG:boto:Token: None 2022-09-03T07:21:14.966 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.970 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058ef7cba420edf3b-006313006a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.971 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.972 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:14.973 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:14.974 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:14.975 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:14.975 DEBUG:boto:Method: GET 2022-09-03T07:21:14.976 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:14.976 DEBUG:boto:Data: 2022-09-03T07:21:14.977 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.977 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:14.978 DEBUG:boto:Port: 8000 2022-09-03T07:21:14.979 DEBUG:boto:Params: {} 2022-09-03T07:21:14.980 DEBUG:boto:Token: None 2022-09-03T07:21:14.981 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/ 2022-09-03T07:21:14.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU= 2022-09-03T07:21:14.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy+xcfW6rfAAkMomsnzibu6T9iU=', 'Content-Length': '0'} 2022-09-03T07:21:14.986 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a44d73c320b3e1e-006313006a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.987 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:14.988 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:14.989 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:14.989 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.990 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:14.990 DEBUG:boto:Data: 2022-09-03T07:21:14.991 DEBUG:boto:Headers: {} 2022-09-03T07:21:14.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:14.992 DEBUG:boto:Port: 8001 2022-09-03T07:21:14.992 DEBUG:boto:Params: {} 2022-09-03T07:21:14.993 DEBUG:boto:Token: None 2022-09-03T07:21:14.994 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:14 GMT /avysmw-75/obj22 2022-09-03T07:21:14.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ynMzALqTqOCwoQA3VFlJUh8FVvc= 2022-09-03T07:21:14.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ynMzALqTqOCwoQA3VFlJUh8FVvc=', 'Content-Length': '0'} 2022-09-03T07:21:14.997 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f5770742dd22eea-006313006a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:14.997 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:14.998 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:14.999 DEBUG:boto:Method: HEAD 2022-09-03T07:21:14.999 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:15.000 DEBUG:boto:Data: 2022-09-03T07:21:15.000 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.001 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.001 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.002 DEBUG:boto:Params: {} 2022-09-03T07:21:15.002 DEBUG:boto:Token: None 2022-09-03T07:21:15.003 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj22 2022-09-03T07:21:15.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dS5gFb2kmfaagj3e/GfH0ssozfw= 2022-09-03T07:21:15.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dS5gFb2kmfaagj3e/GfH0ssozfw=', 'Content-Length': '0'} 2022-09-03T07:21:15.006 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e0ee6116809a4cdc-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.007 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:21:15.007 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:15.008 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:15.008 DEBUG:boto:Method: GET 2022-09-03T07:21:15.009 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:15.009 DEBUG:boto:Data: 2022-09-03T07:21:15.010 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.011 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.012 DEBUG:boto:Params: {} 2022-09-03T07:21:15.012 DEBUG:boto:Token: None 2022-09-03T07:21:15.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj22 2022-09-03T07:21:15.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DZpMjdduN0SflfxM5W3FKxxu1/k= 2022-09-03T07:21:15.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DZpMjdduN0SflfxM5W3FKxxu1/k=', 'Content-Length': '0'} 2022-09-03T07:21:15.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007175f60428fee5c7-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.058 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:15.059 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:15.060 DEBUG:boto:Method: GET 2022-09-03T07:21:15.060 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:15.061 DEBUG:boto:Data: 2022-09-03T07:21:15.061 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.062 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.063 DEBUG:boto:Params: {} 2022-09-03T07:21:15.064 DEBUG:boto:Token: None 2022-09-03T07:21:15.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj22 2022-09-03T07:21:15.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DZpMjdduN0SflfxM5W3FKxxu1/k= 2022-09-03T07:21:15.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DZpMjdduN0SflfxM5W3FKxxu1/k=', 'Content-Length': '0'} 2022-09-03T07:21:15.068 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca7b9c1d48a15790-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.069 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.070 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.070 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:15.071 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:15.072 DEBUG:boto:Method: GET 2022-09-03T07:21:15.072 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:15.073 DEBUG:boto:Data: 2022-09-03T07:21:15.073 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.074 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.074 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.075 DEBUG:boto:Params: {} 2022-09-03T07:21:15.075 DEBUG:boto:Token: None 2022-09-03T07:21:15.076 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.080 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c42cb48d9557cbd-006313006b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.081 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.082 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.082 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.083 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:15.083 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:15.084 DEBUG:boto:Method: GET 2022-09-03T07:21:15.085 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:15.085 DEBUG:boto:Data: 2022-09-03T07:21:15.086 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.086 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.087 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.088 DEBUG:boto:Params: {} 2022-09-03T07:21:15.088 DEBUG:boto:Token: None 2022-09-03T07:21:15.089 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.093 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb27f97a0e3a586d-006313006b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.094 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.095 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:21:15.096 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.097 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.097 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.098 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.098 DEBUG:boto:Method: GET 2022-09-03T07:21:15.099 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.099 DEBUG:boto:Data: 2022-09-03T07:21:15.100 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.101 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.101 DEBUG:boto:Params: {} 2022-09-03T07:21:15.102 DEBUG:boto:Token: None 2022-09-03T07:21:15.103 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ryPokmhaDNWoWHNOvJaaCW1Ll64= 2022-09-03T07:21:15.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ryPokmhaDNWoWHNOvJaaCW1Ll64=', 'Content-Length': '0'} 2022-09-03T07:21:15.106 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bdc53739c647749c-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.107 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.107 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.108 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.108 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.109 DEBUG:boto:Method: GET 2022-09-03T07:21:15.109 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.110 DEBUG:boto:Data: 2022-09-03T07:21:15.110 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.112 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.112 DEBUG:boto:Params: {} 2022-09-03T07:21:15.113 DEBUG:boto:Token: None 2022-09-03T07:21:15.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ryPokmhaDNWoWHNOvJaaCW1Ll64= 2022-09-03T07:21:15.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ryPokmhaDNWoWHNOvJaaCW1Ll64=', 'Content-Length': '0'} 2022-09-03T07:21:15.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060bd8412f15f5051-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.118 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.119 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.119 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.120 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.120 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.121 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.121 DEBUG:boto:Data: 2022-09-03T07:21:15.122 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.123 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.124 DEBUG:boto:Params: {} 2022-09-03T07:21:15.124 DEBUG:boto:Token: None 2022-09-03T07:21:15.125 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qu/FaXrHxNaj9KyinsfrxQhHyjA= 2022-09-03T07:21:15.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qu/FaXrHxNaj9KyinsfrxQhHyjA=', 'Content-Length': '0'} 2022-09-03T07:21:15.128 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000137d412eba903a3a-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.129 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.129 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.130 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.131 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.131 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.132 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.133 DEBUG:boto:Data: 2022-09-03T07:21:15.133 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.134 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.135 DEBUG:boto:Params: {} 2022-09-03T07:21:15.136 DEBUG:boto:Token: None 2022-09-03T07:21:15.136 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qu/FaXrHxNaj9KyinsfrxQhHyjA= 2022-09-03T07:21:15.137 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qu/FaXrHxNaj9KyinsfrxQhHyjA=', 'Content-Length': '0'} 2022-09-03T07:21:15.139 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6b6a1044533a800-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.140 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:21:15.141 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.141 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.142 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.142 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.143 DEBUG:boto:Method: GET 2022-09-03T07:21:15.143 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.144 DEBUG:boto:Data: 2022-09-03T07:21:15.145 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.146 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.146 DEBUG:boto:Params: {} 2022-09-03T07:21:15.147 DEBUG:boto:Token: None 2022-09-03T07:21:15.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ryPokmhaDNWoWHNOvJaaCW1Ll64= 2022-09-03T07:21:15.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ryPokmhaDNWoWHNOvJaaCW1Ll64=', 'Content-Length': '0'} 2022-09-03T07:21:15.151 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be221893e871a6ff-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.152 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.152 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.153 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.153 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.154 DEBUG:boto:Method: GET 2022-09-03T07:21:15.155 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.155 DEBUG:boto:Data: 2022-09-03T07:21:15.156 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.157 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.158 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.158 DEBUG:boto:Params: {} 2022-09-03T07:21:15.159 DEBUG:boto:Token: None 2022-09-03T07:21:15.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:15.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ryPokmhaDNWoWHNOvJaaCW1Ll64= 2022-09-03T07:21:15.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ryPokmhaDNWoWHNOvJaaCW1Ll64=', 'Content-Length': '0'} 2022-09-03T07:21:15.163 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7c9680cf07a2896-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.164 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.164 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.165 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:15.165 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:15.166 DEBUG:boto:Method: GET 2022-09-03T07:21:15.167 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:15.167 DEBUG:boto:Data: 2022-09-03T07:21:15.168 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.169 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.169 DEBUG:boto:Params: {} 2022-09-03T07:21:15.170 DEBUG:boto:Token: None 2022-09-03T07:21:15.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.174 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c850f738d5ecbdca-006313006b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.222 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.223 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.224 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.224 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:15.225 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:15.226 DEBUG:boto:Method: GET 2022-09-03T07:21:15.226 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:15.227 DEBUG:boto:Data: 2022-09-03T07:21:15.228 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.229 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.230 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.230 DEBUG:boto:Params: {} 2022-09-03T07:21:15.231 DEBUG:boto:Token: None 2022-09-03T07:21:15.232 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.238 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011c860192d18e61d-006313006b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.239 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.240 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.240 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.241 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.241 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:15.242 DEBUG:boto:Data: 2022-09-03T07:21:15.243 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.244 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.244 DEBUG:boto:Params: {} 2022-09-03T07:21:15.245 DEBUG:boto:Token: None 2022-09-03T07:21:15.246 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33 2022-09-03T07:21:15.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zaby0wefSEnMomGFi+B9Ay3PYZw= 2022-09-03T07:21:15.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zaby0wefSEnMomGFi+B9Ay3PYZw=', 'Content-Length': '0'} 2022-09-03T07:21:15.249 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e2944e3162f1752-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.250 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.251 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.251 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.252 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:15.252 DEBUG:boto:Data: 2022-09-03T07:21:15.253 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.254 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.254 DEBUG:boto:Params: {} 2022-09-03T07:21:15.255 DEBUG:boto:Token: None 2022-09-03T07:21:15.255 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33 2022-09-03T07:21:15.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zaby0wefSEnMomGFi+B9Ay3PYZw= 2022-09-03T07:21:15.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zaby0wefSEnMomGFi+B9Ay3PYZw=', 'Content-Length': '0'} 2022-09-03T07:21:15.259 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022a0d1eecdea6141-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.260 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:21:15.260 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.261 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.261 DEBUG:boto:Method: GET 2022-09-03T07:21:15.262 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:15.263 DEBUG:boto:Data: 2022-09-03T07:21:15.263 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.264 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.265 DEBUG:boto:Params: {} 2022-09-03T07:21:15.265 DEBUG:boto:Token: None 2022-09-03T07:21:15.266 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33 2022-09-03T07:21:15.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1uTSio50eno0+1EJFMj4LNJoMjE= 2022-09-03T07:21:15.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1uTSio50eno0+1EJFMj4LNJoMjE=', 'Content-Length': '0'} 2022-09-03T07:21:15.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000feb38352fc1b4662-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.271 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:15.271 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:15.272 DEBUG:boto:Method: GET 2022-09-03T07:21:15.272 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:15.273 DEBUG:boto:Data: 2022-09-03T07:21:15.273 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.275 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.275 DEBUG:boto:Params: {} 2022-09-03T07:21:15.276 DEBUG:boto:Token: None 2022-09-03T07:21:15.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj33 2022-09-03T07:21:15.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1uTSio50eno0+1EJFMj4LNJoMjE= 2022-09-03T07:21:15.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1uTSio50eno0+1EJFMj4LNJoMjE=', 'Content-Length': '0'} 2022-09-03T07:21:15.280 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b38fe12887d466b-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.322 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.323 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.323 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:15.324 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:15.324 DEBUG:boto:Method: GET 2022-09-03T07:21:15.325 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:15.326 DEBUG:boto:Data: 2022-09-03T07:21:15.326 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.327 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.327 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.328 DEBUG:boto:Params: {} 2022-09-03T07:21:15.328 DEBUG:boto:Token: None 2022-09-03T07:21:15.329 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.333 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ab13523bd6ad3603-006313006b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.334 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.334 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.335 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.335 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:15.336 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:15.337 DEBUG:boto:Method: GET 2022-09-03T07:21:15.337 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:15.338 DEBUG:boto:Data: 2022-09-03T07:21:15.338 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.339 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.340 DEBUG:boto:Params: {} 2022-09-03T07:21:15.340 DEBUG:boto:Token: None 2022-09-03T07:21:15.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.346 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e4f716ce201bbe8a-006313006b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.347 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.347 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:21:15.348 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.349 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.349 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.350 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.351 DEBUG:boto:Method: GET 2022-09-03T07:21:15.351 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.352 DEBUG:boto:Data: 2022-09-03T07:21:15.352 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.353 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.354 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.355 DEBUG:boto:Params: {} 2022-09-03T07:21:15.356 DEBUG:boto:Token: None 2022-09-03T07:21:15.357 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.357 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OIVdvxVbqwP380qp1gH97J/G2G0= 2022-09-03T07:21:15.358 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OIVdvxVbqwP380qp1gH97J/G2G0=', 'Content-Length': '0'} 2022-09-03T07:21:15.360 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af38d498bb3a3e80-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.361 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.362 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.362 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.363 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.363 DEBUG:boto:Method: GET 2022-09-03T07:21:15.364 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.365 DEBUG:boto:Data: 2022-09-03T07:21:15.365 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.366 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.366 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.367 DEBUG:boto:Params: {} 2022-09-03T07:21:15.368 DEBUG:boto:Token: None 2022-09-03T07:21:15.368 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OIVdvxVbqwP380qp1gH97J/G2G0= 2022-09-03T07:21:15.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OIVdvxVbqwP380qp1gH97J/G2G0=', 'Content-Length': '0'} 2022-09-03T07:21:15.372 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc9c263fecb89d27-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.373 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.373 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.374 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.374 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.375 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.375 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.376 DEBUG:boto:Data: 2022-09-03T07:21:15.376 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.377 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.378 DEBUG:boto:Params: {} 2022-09-03T07:21:15.378 DEBUG:boto:Token: None 2022-09-03T07:21:15.379 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HeLVBiEIVO5oRUmUlFC3Pdo8Ddc= 2022-09-03T07:21:15.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HeLVBiEIVO5oRUmUlFC3Pdo8Ddc=', 'Content-Length': '0'} 2022-09-03T07:21:15.382 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5e36ec0956de09d-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.383 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.384 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.384 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.385 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.385 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.386 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.387 DEBUG:boto:Data: 2022-09-03T07:21:15.387 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.388 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.389 DEBUG:boto:Params: {} 2022-09-03T07:21:15.390 DEBUG:boto:Token: None 2022-09-03T07:21:15.390 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HeLVBiEIVO5oRUmUlFC3Pdo8Ddc= 2022-09-03T07:21:15.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HeLVBiEIVO5oRUmUlFC3Pdo8Ddc=', 'Content-Length': '0'} 2022-09-03T07:21:15.394 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005274b6598f0ddd8b-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.395 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:21:15.395 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.396 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.397 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.398 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.399 DEBUG:boto:Method: GET 2022-09-03T07:21:15.399 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.400 DEBUG:boto:Data: 2022-09-03T07:21:15.401 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.402 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.403 DEBUG:boto:Params: {} 2022-09-03T07:21:15.403 DEBUG:boto:Token: None 2022-09-03T07:21:15.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OIVdvxVbqwP380qp1gH97J/G2G0= 2022-09-03T07:21:15.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OIVdvxVbqwP380qp1gH97J/G2G0=', 'Content-Length': '0'} 2022-09-03T07:21:15.408 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef33717377f0ad61-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.409 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.409 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.410 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.411 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.412 DEBUG:boto:Method: GET 2022-09-03T07:21:15.412 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.413 DEBUG:boto:Data: 2022-09-03T07:21:15.414 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.414 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.415 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.416 DEBUG:boto:Params: {} 2022-09-03T07:21:15.417 DEBUG:boto:Token: None 2022-09-03T07:21:15.417 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:15.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OIVdvxVbqwP380qp1gH97J/G2G0= 2022-09-03T07:21:15.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OIVdvxVbqwP380qp1gH97J/G2G0=', 'Content-Length': '0'} 2022-09-03T07:21:15.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000585c8c23e6784fa3-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.423 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.424 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.424 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:15.425 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:15.426 DEBUG:boto:Method: GET 2022-09-03T07:21:15.426 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:15.427 DEBUG:boto:Data: 2022-09-03T07:21:15.428 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.429 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.430 DEBUG:boto:Params: {} 2022-09-03T07:21:15.431 DEBUG:boto:Token: None 2022-09-03T07:21:15.432 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.436 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b5b3bc857f7a1a3-006313006b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.437 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.438 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.439 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.439 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:15.440 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:15.441 DEBUG:boto:Method: GET 2022-09-03T07:21:15.442 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:15.442 DEBUG:boto:Data: 2022-09-03T07:21:15.443 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.444 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.445 DEBUG:boto:Params: {} 2022-09-03T07:21:15.445 DEBUG:boto:Token: None 2022-09-03T07:21:15.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.452 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d9d4b9fad2b2cb0-006313006b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.453 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.454 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.455 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.455 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.456 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:15.457 DEBUG:boto:Data: 2022-09-03T07:21:15.457 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.459 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.459 DEBUG:boto:Params: {} 2022-09-03T07:21:15.460 DEBUG:boto:Token: None 2022-09-03T07:21:15.461 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34 2022-09-03T07:21:15.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eSvwBR5fFuZQgjH9uFp3BBl/jRA= 2022-09-03T07:21:15.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eSvwBR5fFuZQgjH9uFp3BBl/jRA=', 'Content-Length': '0'} 2022-09-03T07:21:15.465 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000985931e1ae517924-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.465 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.466 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.466 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.467 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:15.468 DEBUG:boto:Data: 2022-09-03T07:21:15.468 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.469 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.470 DEBUG:boto:Params: {} 2022-09-03T07:21:15.471 DEBUG:boto:Token: None 2022-09-03T07:21:15.471 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34 2022-09-03T07:21:15.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eSvwBR5fFuZQgjH9uFp3BBl/jRA= 2022-09-03T07:21:15.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eSvwBR5fFuZQgjH9uFp3BBl/jRA=', 'Content-Length': '0'} 2022-09-03T07:21:15.474 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e6ae9fe30a4f1f5f-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.475 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:21:15.476 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.477 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.477 DEBUG:boto:Method: GET 2022-09-03T07:21:15.478 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:15.478 DEBUG:boto:Data: 2022-09-03T07:21:15.479 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.480 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.481 DEBUG:boto:Params: {} 2022-09-03T07:21:15.482 DEBUG:boto:Token: None 2022-09-03T07:21:15.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34 2022-09-03T07:21:15.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rouaFMEM2CHzwCwz7xbJABD7C+E= 2022-09-03T07:21:15.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rouaFMEM2CHzwCwz7xbJABD7C+E=', 'Content-Length': '0'} 2022-09-03T07:21:15.487 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f4312563bc7ad59-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.487 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:15.488 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:15.489 DEBUG:boto:Method: GET 2022-09-03T07:21:15.489 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:15.490 DEBUG:boto:Data: 2022-09-03T07:21:15.491 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.492 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.493 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.493 DEBUG:boto:Params: {} 2022-09-03T07:21:15.494 DEBUG:boto:Token: None 2022-09-03T07:21:15.494 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj34 2022-09-03T07:21:15.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rouaFMEM2CHzwCwz7xbJABD7C+E= 2022-09-03T07:21:15.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rouaFMEM2CHzwCwz7xbJABD7C+E=', 'Content-Length': '0'} 2022-09-03T07:21:15.498 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e1ba39ad8b702ee-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.543 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.544 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.545 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:15.546 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:15.547 DEBUG:boto:Method: GET 2022-09-03T07:21:15.548 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:15.550 DEBUG:boto:Data: 2022-09-03T07:21:15.550 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.551 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.552 DEBUG:boto:Params: {} 2022-09-03T07:21:15.553 DEBUG:boto:Token: None 2022-09-03T07:21:15.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.558 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001ba2eebcdd4417a1-006313006b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.559 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.560 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.560 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.561 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:15.561 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:15.562 DEBUG:boto:Method: GET 2022-09-03T07:21:15.562 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:15.563 DEBUG:boto:Data: 2022-09-03T07:21:15.563 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.565 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.566 DEBUG:boto:Params: {} 2022-09-03T07:21:15.566 DEBUG:boto:Token: None 2022-09-03T07:21:15.567 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.568 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.571 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c4c362775be0b5b6-006313006b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.572 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.573 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:21:15.574 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.574 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.575 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.575 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.576 DEBUG:boto:Method: GET 2022-09-03T07:21:15.576 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.577 DEBUG:boto:Data: 2022-09-03T07:21:15.577 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.578 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.578 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.579 DEBUG:boto:Params: {} 2022-09-03T07:21:15.580 DEBUG:boto:Token: None 2022-09-03T07:21:15.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nsMAfpbfxUynx4ADL8ZUI89J6YA= 2022-09-03T07:21:15.581 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nsMAfpbfxUynx4ADL8ZUI89J6YA=', 'Content-Length': '0'} 2022-09-03T07:21:15.584 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000841f7b1ecd1c8561-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.585 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.585 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.586 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.586 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.587 DEBUG:boto:Method: GET 2022-09-03T07:21:15.588 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.588 DEBUG:boto:Data: 2022-09-03T07:21:15.589 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.589 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.590 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.591 DEBUG:boto:Params: {} 2022-09-03T07:21:15.591 DEBUG:boto:Token: None 2022-09-03T07:21:15.592 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nsMAfpbfxUynx4ADL8ZUI89J6YA= 2022-09-03T07:21:15.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nsMAfpbfxUynx4ADL8ZUI89J6YA=', 'Content-Length': '0'} 2022-09-03T07:21:15.595 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000043ba0f0fc97e950b-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.596 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.597 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.597 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.598 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.598 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.599 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.599 DEBUG:boto:Data: 2022-09-03T07:21:15.600 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.601 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.601 DEBUG:boto:Params: {} 2022-09-03T07:21:15.602 DEBUG:boto:Token: None 2022-09-03T07:21:15.603 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gwdnz/phcyp6mVwUSXk1HO8cIyU= 2022-09-03T07:21:15.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gwdnz/phcyp6mVwUSXk1HO8cIyU=', 'Content-Length': '0'} 2022-09-03T07:21:15.607 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6cbcf84221e5770-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.608 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.609 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.610 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.611 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.612 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.613 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.613 DEBUG:boto:Data: 2022-09-03T07:21:15.614 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.616 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.617 DEBUG:boto:Params: {} 2022-09-03T07:21:15.618 DEBUG:boto:Token: None 2022-09-03T07:21:15.619 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gwdnz/phcyp6mVwUSXk1HO8cIyU= 2022-09-03T07:21:15.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gwdnz/phcyp6mVwUSXk1HO8cIyU=', 'Content-Length': '0'} 2022-09-03T07:21:15.624 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df4a7960a7a72e91-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.625 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:21:15.625 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.626 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.626 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.627 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.628 DEBUG:boto:Method: GET 2022-09-03T07:21:15.628 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.629 DEBUG:boto:Data: 2022-09-03T07:21:15.629 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.630 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.631 DEBUG:boto:Params: {} 2022-09-03T07:21:15.632 DEBUG:boto:Token: None 2022-09-03T07:21:15.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nsMAfpbfxUynx4ADL8ZUI89J6YA= 2022-09-03T07:21:15.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nsMAfpbfxUynx4ADL8ZUI89J6YA=', 'Content-Length': '0'} 2022-09-03T07:21:15.635 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fde23556a4b2c6b4-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.682 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.683 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.683 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.684 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.684 DEBUG:boto:Method: GET 2022-09-03T07:21:15.685 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.685 DEBUG:boto:Data: 2022-09-03T07:21:15.686 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.687 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.688 DEBUG:boto:Params: {} 2022-09-03T07:21:15.689 DEBUG:boto:Token: None 2022-09-03T07:21:15.689 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:15.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nsMAfpbfxUynx4ADL8ZUI89J6YA= 2022-09-03T07:21:15.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nsMAfpbfxUynx4ADL8ZUI89J6YA=', 'Content-Length': '0'} 2022-09-03T07:21:15.693 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003488ab809f450dd3-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.693 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.694 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.695 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:15.695 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:15.696 DEBUG:boto:Method: GET 2022-09-03T07:21:15.696 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:15.697 DEBUG:boto:Data: 2022-09-03T07:21:15.698 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.699 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.699 DEBUG:boto:Params: {} 2022-09-03T07:21:15.700 DEBUG:boto:Token: None 2022-09-03T07:21:15.700 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.701 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.702 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.704 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3170b90ef96fdbe-006313006b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.705 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.706 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.707 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.707 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:15.708 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:15.708 DEBUG:boto:Method: GET 2022-09-03T07:21:15.709 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:15.709 DEBUG:boto:Data: 2022-09-03T07:21:15.710 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.710 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.711 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.711 DEBUG:boto:Params: {} 2022-09-03T07:21:15.712 DEBUG:boto:Token: None 2022-09-03T07:21:15.713 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.714 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.718 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000191a8d9fd8262ba6-006313006b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.718 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.719 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.720 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.720 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.721 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:15.722 DEBUG:boto:Data: 2022-09-03T07:21:15.722 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.723 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.723 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.724 DEBUG:boto:Params: {} 2022-09-03T07:21:15.725 DEBUG:boto:Token: None 2022-09-03T07:21:15.725 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38 2022-09-03T07:21:15.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FWp87r1ySp6uB4UBXH22b35p9iM= 2022-09-03T07:21:15.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FWp87r1ySp6uB4UBXH22b35p9iM=', 'Content-Length': '0'} 2022-09-03T07:21:15.728 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfc541027f4c77d4-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.729 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.730 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.731 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.731 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:15.732 DEBUG:boto:Data: 2022-09-03T07:21:15.732 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.733 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.734 DEBUG:boto:Params: {} 2022-09-03T07:21:15.734 DEBUG:boto:Token: None 2022-09-03T07:21:15.735 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38 2022-09-03T07:21:15.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FWp87r1ySp6uB4UBXH22b35p9iM= 2022-09-03T07:21:15.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FWp87r1ySp6uB4UBXH22b35p9iM=', 'Content-Length': '0'} 2022-09-03T07:21:15.738 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008dedd8eabcc31068-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.739 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:21:15.740 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.740 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.741 DEBUG:boto:Method: GET 2022-09-03T07:21:15.741 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:15.742 DEBUG:boto:Data: 2022-09-03T07:21:15.742 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.743 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.744 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.744 DEBUG:boto:Params: {} 2022-09-03T07:21:15.745 DEBUG:boto:Token: None 2022-09-03T07:21:15.745 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38 2022-09-03T07:21:15.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Y2i12cZ6cCmcmZfFDiQb4hYIzA= 2022-09-03T07:21:15.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Y2i12cZ6cCmcmZfFDiQb4hYIzA=', 'Content-Length': '0'} 2022-09-03T07:21:15.749 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1fa27ed74c1dd7e-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.750 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:15.750 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:15.751 DEBUG:boto:Method: GET 2022-09-03T07:21:15.752 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:15.752 DEBUG:boto:Data: 2022-09-03T07:21:15.753 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.753 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.754 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.754 DEBUG:boto:Params: {} 2022-09-03T07:21:15.756 DEBUG:boto:Token: None 2022-09-03T07:21:15.756 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj38 2022-09-03T07:21:15.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Y2i12cZ6cCmcmZfFDiQb4hYIzA= 2022-09-03T07:21:15.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Y2i12cZ6cCmcmZfFDiQb4hYIzA=', 'Content-Length': '0'} 2022-09-03T07:21:15.760 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f261fbe4343c46a9-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.806 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.807 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.807 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:15.808 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:15.808 DEBUG:boto:Method: GET 2022-09-03T07:21:15.809 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:15.810 DEBUG:boto:Data: 2022-09-03T07:21:15.810 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.811 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.811 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.812 DEBUG:boto:Params: {} 2022-09-03T07:21:15.812 DEBUG:boto:Token: None 2022-09-03T07:21:15.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.818 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000046d2f3266b684bdf-006313006b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.818 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.819 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.820 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.820 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:15.821 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:15.821 DEBUG:boto:Method: GET 2022-09-03T07:21:15.822 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:15.823 DEBUG:boto:Data: 2022-09-03T07:21:15.824 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.824 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.826 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.827 DEBUG:boto:Params: {} 2022-09-03T07:21:15.827 DEBUG:boto:Token: None 2022-09-03T07:21:15.828 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.833 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002bb690ab0ece0290-006313006b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.833 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.834 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:21:15.835 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:15.835 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:15.836 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.837 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.837 DEBUG:boto:Method: GET 2022-09-03T07:21:15.838 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.838 DEBUG:boto:Data: 2022-09-03T07:21:15.839 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.840 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.840 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.841 DEBUG:boto:Params: {} 2022-09-03T07:21:15.841 DEBUG:boto:Token: None 2022-09-03T07:21:15.842 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.843 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEi2NpMoxrQhrdqTj9acO2pl0us= 2022-09-03T07:21:15.843 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEi2NpMoxrQhrdqTj9acO2pl0us=', 'Content-Length': '0'} 2022-09-03T07:21:15.845 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000857e397eadeaae89-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.846 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:15.847 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:15.847 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.848 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.848 DEBUG:boto:Method: GET 2022-09-03T07:21:15.849 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.850 DEBUG:boto:Data: 2022-09-03T07:21:15.850 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.851 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.852 DEBUG:boto:Params: {} 2022-09-03T07:21:15.853 DEBUG:boto:Token: None 2022-09-03T07:21:15.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEi2NpMoxrQhrdqTj9acO2pl0us= 2022-09-03T07:21:15.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEi2NpMoxrQhrdqTj9acO2pl0us=', 'Content-Length': '0'} 2022-09-03T07:21:15.857 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cee949c6b0334b9a-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.858 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:15.858 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:15.859 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.859 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.860 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.861 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.861 DEBUG:boto:Data: 2022-09-03T07:21:15.862 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.862 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.863 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.863 DEBUG:boto:Params: {} 2022-09-03T07:21:15.864 DEBUG:boto:Token: None 2022-09-03T07:21:15.864 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UkKWFkFSa+wddZqfjDYq920yt4Q= 2022-09-03T07:21:15.866 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UkKWFkFSa+wddZqfjDYq920yt4Q=', 'Content-Length': '0'} 2022-09-03T07:21:15.868 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f6226b4ac2ee5b2-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.869 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:15.869 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:15.870 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.871 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.871 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.872 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.872 DEBUG:boto:Data: 2022-09-03T07:21:15.873 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.873 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.874 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.874 DEBUG:boto:Params: {} 2022-09-03T07:21:15.875 DEBUG:boto:Token: None 2022-09-03T07:21:15.875 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UkKWFkFSa+wddZqfjDYq920yt4Q= 2022-09-03T07:21:15.877 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UkKWFkFSa+wddZqfjDYq920yt4Q=', 'Content-Length': '0'} 2022-09-03T07:21:15.879 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5fcc87e55ae2c76-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.880 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:21:15.880 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:15.881 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:15.881 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.882 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.882 DEBUG:boto:Method: GET 2022-09-03T07:21:15.883 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.883 DEBUG:boto:Data: 2022-09-03T07:21:15.884 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.885 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.885 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.886 DEBUG:boto:Params: {} 2022-09-03T07:21:15.886 DEBUG:boto:Token: None 2022-09-03T07:21:15.887 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.888 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEi2NpMoxrQhrdqTj9acO2pl0us= 2022-09-03T07:21:15.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEi2NpMoxrQhrdqTj9acO2pl0us=', 'Content-Length': '0'} 2022-09-03T07:21:15.890 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e0079dc2a7f1d9a7-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.891 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:15.892 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:15.892 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.893 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.893 DEBUG:boto:Method: GET 2022-09-03T07:21:15.894 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.895 DEBUG:boto:Data: 2022-09-03T07:21:15.895 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.896 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.897 DEBUG:boto:Params: {} 2022-09-03T07:21:15.897 DEBUG:boto:Token: None 2022-09-03T07:21:15.898 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:15.899 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEi2NpMoxrQhrdqTj9acO2pl0us= 2022-09-03T07:21:15.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEi2NpMoxrQhrdqTj9acO2pl0us=', 'Content-Length': '0'} 2022-09-03T07:21:15.901 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe4b95732f5724dd-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.902 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.903 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.903 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:15.904 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:15.904 DEBUG:boto:Method: GET 2022-09-03T07:21:15.905 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:15.906 DEBUG:boto:Data: 2022-09-03T07:21:15.906 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.907 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.908 DEBUG:boto:Params: {} 2022-09-03T07:21:15.908 DEBUG:boto:Token: None 2022-09-03T07:21:15.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000825dc007a85fa9ee-006313006b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.914 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.915 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:15.915 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:15.916 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:15.917 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:15.917 DEBUG:boto:Method: GET 2022-09-03T07:21:15.918 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:15.918 DEBUG:boto:Data: 2022-09-03T07:21:15.919 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.919 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.920 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.921 DEBUG:boto:Params: {} 2022-09-03T07:21:15.921 DEBUG:boto:Token: None 2022-09-03T07:21:15.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/ 2022-09-03T07:21:15.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI= 2022-09-03T07:21:15.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:275n5R+Nm0jqTFTmt0H+BeRirKI=', 'Content-Length': '0'} 2022-09-03T07:21:15.927 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000066e171ed1a78e747-006313006b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.928 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:15.929 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:15.929 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:15.930 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.930 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:15.931 DEBUG:boto:Data: 2022-09-03T07:21:15.932 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.933 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.934 DEBUG:boto:Params: {} 2022-09-03T07:21:15.934 DEBUG:boto:Token: None 2022-09-03T07:21:15.935 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj44 2022-09-03T07:21:15.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HKidSdk0vT+ZSBWZOQ1brcNCTXs= 2022-09-03T07:21:15.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HKidSdk0vT+ZSBWZOQ1brcNCTXs=', 'Content-Length': '0'} 2022-09-03T07:21:15.938 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bca1ff7aec92993b-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.939 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:15.940 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:15.940 DEBUG:boto:Method: HEAD 2022-09-03T07:21:15.941 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:15.942 DEBUG:boto:Data: 2022-09-03T07:21:15.943 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.943 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:15.944 DEBUG:boto:Port: 8000 2022-09-03T07:21:15.944 DEBUG:boto:Params: {} 2022-09-03T07:21:15.945 DEBUG:boto:Token: None 2022-09-03T07:21:15.946 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj44 2022-09-03T07:21:15.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HKidSdk0vT+ZSBWZOQ1brcNCTXs= 2022-09-03T07:21:15.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HKidSdk0vT+ZSBWZOQ1brcNCTXs=', 'Content-Length': '0'} 2022-09-03T07:21:15.949 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000826a5e2720ef7d96-006313006b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:15.950 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:21:15.950 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:15.951 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:15.952 DEBUG:boto:Method: GET 2022-09-03T07:21:15.952 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:15.953 DEBUG:boto:Data: 2022-09-03T07:21:15.953 DEBUG:boto:Headers: {} 2022-09-03T07:21:15.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:15.955 DEBUG:boto:Port: 8001 2022-09-03T07:21:15.955 DEBUG:boto:Params: {} 2022-09-03T07:21:15.956 DEBUG:boto:Token: None 2022-09-03T07:21:15.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:15 GMT /avysmw-75/obj44 2022-09-03T07:21:15.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JNHvZwYq30lSDZ++E/+FhkjCBUw= 2022-09-03T07:21:15.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JNHvZwYq30lSDZ++E/+FhkjCBUw=', 'Content-Length': '0'} 2022-09-03T07:21:15.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6338f1a0d257a30-006313006b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.006 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:16.007 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:16.008 DEBUG:boto:Method: GET 2022-09-03T07:21:16.008 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:16.009 DEBUG:boto:Data: 2022-09-03T07:21:16.009 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.010 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.011 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.011 DEBUG:boto:Params: {} 2022-09-03T07:21:16.012 DEBUG:boto:Token: None 2022-09-03T07:21:16.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj44 2022-09-03T07:21:16.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7s93a/WvJk2ekEG/pTm0KxHxVl8= 2022-09-03T07:21:16.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7s93a/WvJk2ekEG/pTm0KxHxVl8=', 'Content-Length': '0'} 2022-09-03T07:21:16.017 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023392283eb2debef-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.017 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.018 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.019 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:16.019 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:16.020 DEBUG:boto:Method: GET 2022-09-03T07:21:16.020 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:16.021 DEBUG:boto:Data: 2022-09-03T07:21:16.021 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.022 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.023 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.023 DEBUG:boto:Params: {} 2022-09-03T07:21:16.024 DEBUG:boto:Token: None 2022-09-03T07:21:16.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.029 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000393660065cdbd35c-006313006c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.030 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.030 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.031 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.032 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:16.033 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:16.033 DEBUG:boto:Method: GET 2022-09-03T07:21:16.034 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:16.034 DEBUG:boto:Data: 2022-09-03T07:21:16.035 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.036 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.037 DEBUG:boto:Params: {} 2022-09-03T07:21:16.037 DEBUG:boto:Token: None 2022-09-03T07:21:16.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.043 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d172fa2f6146078f-006313006c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.043 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.044 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:21:16.045 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.046 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.046 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.047 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.047 DEBUG:boto:Method: GET 2022-09-03T07:21:16.048 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.048 DEBUG:boto:Data: 2022-09-03T07:21:16.049 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.049 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.050 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.050 DEBUG:boto:Params: {} 2022-09-03T07:21:16.051 DEBUG:boto:Token: None 2022-09-03T07:21:16.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tTMj4PmgqmyZ1ioQMctuHFXzrZ0= 2022-09-03T07:21:16.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tTMj4PmgqmyZ1ioQMctuHFXzrZ0=', 'Content-Length': '0'} 2022-09-03T07:21:16.055 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000359f8c43a4e5d04f-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.056 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.056 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.057 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.057 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.058 DEBUG:boto:Method: GET 2022-09-03T07:21:16.058 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.059 DEBUG:boto:Data: 2022-09-03T07:21:16.059 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.060 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.060 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.061 DEBUG:boto:Params: {} 2022-09-03T07:21:16.062 DEBUG:boto:Token: None 2022-09-03T07:21:16.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tTMj4PmgqmyZ1ioQMctuHFXzrZ0= 2022-09-03T07:21:16.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tTMj4PmgqmyZ1ioQMctuHFXzrZ0=', 'Content-Length': '0'} 2022-09-03T07:21:16.067 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096271f3fec4f2e0f-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.068 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.069 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.070 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.071 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.072 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.073 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.074 DEBUG:boto:Data: 2022-09-03T07:21:16.075 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.077 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.078 DEBUG:boto:Params: {} 2022-09-03T07:21:16.079 DEBUG:boto:Token: None 2022-09-03T07:21:16.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NHJENdINIKCwlO0GpbuGUI3sCXI= 2022-09-03T07:21:16.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NHJENdINIKCwlO0GpbuGUI3sCXI=', 'Content-Length': '0'} 2022-09-03T07:21:16.084 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002dd849dc6e51e18f-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.086 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.087 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.088 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.089 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.090 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.090 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.091 DEBUG:boto:Data: 2022-09-03T07:21:16.092 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.094 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.095 DEBUG:boto:Params: {} 2022-09-03T07:21:16.095 DEBUG:boto:Token: None 2022-09-03T07:21:16.096 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NHJENdINIKCwlO0GpbuGUI3sCXI= 2022-09-03T07:21:16.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NHJENdINIKCwlO0GpbuGUI3sCXI=', 'Content-Length': '0'} 2022-09-03T07:21:16.099 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000f69ae8788dc1b2-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.100 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:21:16.101 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.101 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.102 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.102 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.103 DEBUG:boto:Method: GET 2022-09-03T07:21:16.103 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.104 DEBUG:boto:Data: 2022-09-03T07:21:16.104 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.105 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.106 DEBUG:boto:Params: {} 2022-09-03T07:21:16.106 DEBUG:boto:Token: None 2022-09-03T07:21:16.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tTMj4PmgqmyZ1ioQMctuHFXzrZ0= 2022-09-03T07:21:16.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tTMj4PmgqmyZ1ioQMctuHFXzrZ0=', 'Content-Length': '0'} 2022-09-03T07:21:16.110 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f4337ee6f0d9828-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.111 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.112 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.112 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.113 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.113 DEBUG:boto:Method: GET 2022-09-03T07:21:16.114 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.114 DEBUG:boto:Data: 2022-09-03T07:21:16.115 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.116 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.116 DEBUG:boto:Params: {} 2022-09-03T07:21:16.117 DEBUG:boto:Token: None 2022-09-03T07:21:16.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:16.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tTMj4PmgqmyZ1ioQMctuHFXzrZ0= 2022-09-03T07:21:16.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tTMj4PmgqmyZ1ioQMctuHFXzrZ0=', 'Content-Length': '0'} 2022-09-03T07:21:16.121 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a08c71a046c37de-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.122 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.123 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.123 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:16.124 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:16.124 DEBUG:boto:Method: GET 2022-09-03T07:21:16.125 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:16.125 DEBUG:boto:Data: 2022-09-03T07:21:16.126 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.127 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.127 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.128 DEBUG:boto:Params: {} 2022-09-03T07:21:16.128 DEBUG:boto:Token: None 2022-09-03T07:21:16.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.129 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.133 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004d0c81ddac133827-006313006c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.133 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.134 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.135 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.135 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:16.136 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:16.136 DEBUG:boto:Method: GET 2022-09-03T07:21:16.137 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:16.138 DEBUG:boto:Data: 2022-09-03T07:21:16.138 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.139 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.139 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.140 DEBUG:boto:Params: {} 2022-09-03T07:21:16.140 DEBUG:boto:Token: None 2022-09-03T07:21:16.141 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.145 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000001a502bf6a1acb8b-006313006c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.146 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.147 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.147 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.148 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.149 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:16.149 DEBUG:boto:Data: 2022-09-03T07:21:16.150 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.151 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.151 DEBUG:boto:Params: {} 2022-09-03T07:21:16.152 DEBUG:boto:Token: None 2022-09-03T07:21:16.152 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51 2022-09-03T07:21:16.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QqkZCmbmHai7csh8n8fsHQ7J64= 2022-09-03T07:21:16.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QqkZCmbmHai7csh8n8fsHQ7J64=', 'Content-Length': '0'} 2022-09-03T07:21:16.155 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080cdd2440e19e351-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.156 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.156 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.157 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.158 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:16.158 DEBUG:boto:Data: 2022-09-03T07:21:16.159 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.159 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.160 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.160 DEBUG:boto:Params: {} 2022-09-03T07:21:16.161 DEBUG:boto:Token: None 2022-09-03T07:21:16.161 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51 2022-09-03T07:21:16.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QqkZCmbmHai7csh8n8fsHQ7J64= 2022-09-03T07:21:16.163 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QqkZCmbmHai7csh8n8fsHQ7J64=', 'Content-Length': '0'} 2022-09-03T07:21:16.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013755fe0affdee00-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.165 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:21:16.166 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.166 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.167 DEBUG:boto:Method: GET 2022-09-03T07:21:16.168 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:16.168 DEBUG:boto:Data: 2022-09-03T07:21:16.169 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.169 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.170 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.170 DEBUG:boto:Params: {} 2022-09-03T07:21:16.171 DEBUG:boto:Token: None 2022-09-03T07:21:16.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51 2022-09-03T07:21:16.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zsH6p/vViMpOIzprubJiYcwCL1Y= 2022-09-03T07:21:16.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zsH6p/vViMpOIzprubJiYcwCL1Y=', 'Content-Length': '0'} 2022-09-03T07:21:16.175 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009382dc8e9ab6235-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.176 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:16.176 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:16.177 DEBUG:boto:Method: GET 2022-09-03T07:21:16.177 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:16.178 DEBUG:boto:Data: 2022-09-03T07:21:16.178 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.179 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.179 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.180 DEBUG:boto:Params: {} 2022-09-03T07:21:16.180 DEBUG:boto:Token: None 2022-09-03T07:21:16.181 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj51 2022-09-03T07:21:16.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zsH6p/vViMpOIzprubJiYcwCL1Y= 2022-09-03T07:21:16.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zsH6p/vViMpOIzprubJiYcwCL1Y=', 'Content-Length': '0'} 2022-09-03T07:21:16.184 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063b0fb3c61ba01d7-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.185 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.186 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.186 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:16.187 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:16.187 DEBUG:boto:Method: GET 2022-09-03T07:21:16.188 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:16.188 DEBUG:boto:Data: 2022-09-03T07:21:16.189 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.189 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.190 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.191 DEBUG:boto:Params: {} 2022-09-03T07:21:16.191 DEBUG:boto:Token: None 2022-09-03T07:21:16.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.195 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009f0b6cd352001e51-006313006c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.242 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.243 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.244 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.244 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:16.245 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:16.246 DEBUG:boto:Method: GET 2022-09-03T07:21:16.246 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:16.247 DEBUG:boto:Data: 2022-09-03T07:21:16.247 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.248 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.249 DEBUG:boto:Params: {} 2022-09-03T07:21:16.250 DEBUG:boto:Token: None 2022-09-03T07:21:16.250 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.254 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d9238837f30528d-006313006c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.255 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.257 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:21:16.257 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.258 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.258 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.259 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.260 DEBUG:boto:Method: GET 2022-09-03T07:21:16.260 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.261 DEBUG:boto:Data: 2022-09-03T07:21:16.262 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.263 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.263 DEBUG:boto:Params: {} 2022-09-03T07:21:16.264 DEBUG:boto:Token: None 2022-09-03T07:21:16.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xtp7oTERX3gbwUY3SH+y5SByjnw= 2022-09-03T07:21:16.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xtp7oTERX3gbwUY3SH+y5SByjnw=', 'Content-Length': '0'} 2022-09-03T07:21:16.268 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097423a889d1b2ba6-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.269 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.270 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.271 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.272 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.272 DEBUG:boto:Method: GET 2022-09-03T07:21:16.273 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.273 DEBUG:boto:Data: 2022-09-03T07:21:16.274 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.275 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.275 DEBUG:boto:Params: {} 2022-09-03T07:21:16.276 DEBUG:boto:Token: None 2022-09-03T07:21:16.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xtp7oTERX3gbwUY3SH+y5SByjnw= 2022-09-03T07:21:16.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xtp7oTERX3gbwUY3SH+y5SByjnw=', 'Content-Length': '0'} 2022-09-03T07:21:16.280 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a14791043df2f24-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.322 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.323 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.323 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.324 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.324 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.325 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.325 DEBUG:boto:Data: 2022-09-03T07:21:16.326 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.326 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.327 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.327 DEBUG:boto:Params: {} 2022-09-03T07:21:16.328 DEBUG:boto:Token: None 2022-09-03T07:21:16.328 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nQTesTI7+ZjKSahk65vX0fofHaA= 2022-09-03T07:21:16.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nQTesTI7+ZjKSahk65vX0fofHaA=', 'Content-Length': '0'} 2022-09-03T07:21:16.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003fabd8a1578807ca-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.332 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.333 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.333 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.334 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.334 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.335 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.335 DEBUG:boto:Data: 2022-09-03T07:21:16.336 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.337 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.337 DEBUG:boto:Params: {} 2022-09-03T07:21:16.338 DEBUG:boto:Token: None 2022-09-03T07:21:16.339 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nQTesTI7+ZjKSahk65vX0fofHaA= 2022-09-03T07:21:16.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nQTesTI7+ZjKSahk65vX0fofHaA=', 'Content-Length': '0'} 2022-09-03T07:21:16.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034633e52ed953606-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.343 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:21:16.344 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.345 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.345 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.346 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.347 DEBUG:boto:Method: GET 2022-09-03T07:21:16.347 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.348 DEBUG:boto:Data: 2022-09-03T07:21:16.348 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.349 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.349 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.350 DEBUG:boto:Params: {} 2022-09-03T07:21:16.351 DEBUG:boto:Token: None 2022-09-03T07:21:16.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xtp7oTERX3gbwUY3SH+y5SByjnw= 2022-09-03T07:21:16.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xtp7oTERX3gbwUY3SH+y5SByjnw=', 'Content-Length': '0'} 2022-09-03T07:21:16.355 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062817762b9859741-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.356 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.356 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.357 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.357 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.358 DEBUG:boto:Method: GET 2022-09-03T07:21:16.359 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.359 DEBUG:boto:Data: 2022-09-03T07:21:16.360 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.361 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.361 DEBUG:boto:Params: {} 2022-09-03T07:21:16.362 DEBUG:boto:Token: None 2022-09-03T07:21:16.363 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:16.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xtp7oTERX3gbwUY3SH+y5SByjnw= 2022-09-03T07:21:16.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xtp7oTERX3gbwUY3SH+y5SByjnw=', 'Content-Length': '0'} 2022-09-03T07:21:16.366 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d2130ede7a24e53-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.367 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.368 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.368 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:16.369 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:16.369 DEBUG:boto:Method: GET 2022-09-03T07:21:16.370 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:16.371 DEBUG:boto:Data: 2022-09-03T07:21:16.371 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.372 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.373 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.373 DEBUG:boto:Params: {} 2022-09-03T07:21:16.374 DEBUG:boto:Token: None 2022-09-03T07:21:16.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.378 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004697ad1d056f3063-006313006c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.379 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.380 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.380 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.381 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:16.381 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:16.382 DEBUG:boto:Method: GET 2022-09-03T07:21:16.382 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:16.383 DEBUG:boto:Data: 2022-09-03T07:21:16.384 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.385 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.385 DEBUG:boto:Params: {} 2022-09-03T07:21:16.386 DEBUG:boto:Token: None 2022-09-03T07:21:16.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.391 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000512dfc0b7f4617b0-006313006c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.438 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.439 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.439 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.440 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.441 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:16.441 DEBUG:boto:Data: 2022-09-03T07:21:16.442 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.443 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.443 DEBUG:boto:Params: {} 2022-09-03T07:21:16.444 DEBUG:boto:Token: None 2022-09-03T07:21:16.444 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55 2022-09-03T07:21:16.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L55FcY+vQlyfTb3ZIbKlo/I9RyI= 2022-09-03T07:21:16.446 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L55FcY+vQlyfTb3ZIbKlo/I9RyI=', 'Content-Length': '0'} 2022-09-03T07:21:16.448 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000165b08af0bc2a9d-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.449 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.449 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.450 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.450 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:16.451 DEBUG:boto:Data: 2022-09-03T07:21:16.451 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.452 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.452 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.453 DEBUG:boto:Params: {} 2022-09-03T07:21:16.454 DEBUG:boto:Token: None 2022-09-03T07:21:16.454 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55 2022-09-03T07:21:16.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L55FcY+vQlyfTb3ZIbKlo/I9RyI= 2022-09-03T07:21:16.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L55FcY+vQlyfTb3ZIbKlo/I9RyI=', 'Content-Length': '0'} 2022-09-03T07:21:16.457 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f42558e9af6610e-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.458 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:21:16.459 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.459 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.460 DEBUG:boto:Method: GET 2022-09-03T07:21:16.460 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:16.461 DEBUG:boto:Data: 2022-09-03T07:21:16.461 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.462 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.463 DEBUG:boto:Params: {} 2022-09-03T07:21:16.463 DEBUG:boto:Token: None 2022-09-03T07:21:16.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55 2022-09-03T07:21:16.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SJ8lEgDJ1mMd+pxA0igJeuJUZtU= 2022-09-03T07:21:16.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SJ8lEgDJ1mMd+pxA0igJeuJUZtU=', 'Content-Length': '0'} 2022-09-03T07:21:16.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081e38de1fd63fa1a-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.468 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:16.468 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:16.469 DEBUG:boto:Method: GET 2022-09-03T07:21:16.470 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:16.470 DEBUG:boto:Data: 2022-09-03T07:21:16.471 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.471 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.472 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.472 DEBUG:boto:Params: {} 2022-09-03T07:21:16.473 DEBUG:boto:Token: None 2022-09-03T07:21:16.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj55 2022-09-03T07:21:16.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SJ8lEgDJ1mMd+pxA0igJeuJUZtU= 2022-09-03T07:21:16.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SJ8lEgDJ1mMd+pxA0igJeuJUZtU=', 'Content-Length': '0'} 2022-09-03T07:21:16.477 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000577d88f1075e4900-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.477 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.478 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.478 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:16.479 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:16.479 DEBUG:boto:Method: GET 2022-09-03T07:21:16.480 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:16.480 DEBUG:boto:Data: 2022-09-03T07:21:16.481 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.482 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.482 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.483 DEBUG:boto:Params: {} 2022-09-03T07:21:16.483 DEBUG:boto:Token: None 2022-09-03T07:21:16.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.488 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e47c4171216b7a2a-006313006c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.488 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.489 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.490 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.490 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:16.491 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:16.491 DEBUG:boto:Method: GET 2022-09-03T07:21:16.492 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:16.492 DEBUG:boto:Data: 2022-09-03T07:21:16.493 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.494 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.494 DEBUG:boto:Params: {} 2022-09-03T07:21:16.495 DEBUG:boto:Token: None 2022-09-03T07:21:16.496 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.500 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d62c46c491215b8-006313006c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.501 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.502 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:21:16.502 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.503 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.503 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.504 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.504 DEBUG:boto:Method: GET 2022-09-03T07:21:16.505 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.505 DEBUG:boto:Data: 2022-09-03T07:21:16.506 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.506 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.507 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.507 DEBUG:boto:Params: {} 2022-09-03T07:21:16.508 DEBUG:boto:Token: None 2022-09-03T07:21:16.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cwBjQy+Lhqgz9aGTrJPQpEX+42Y= 2022-09-03T07:21:16.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cwBjQy+Lhqgz9aGTrJPQpEX+42Y=', 'Content-Length': '0'} 2022-09-03T07:21:16.512 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000417d384401fe596d-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.513 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.513 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.514 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.514 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.515 DEBUG:boto:Method: GET 2022-09-03T07:21:16.516 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.516 DEBUG:boto:Data: 2022-09-03T07:21:16.517 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.517 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.518 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.518 DEBUG:boto:Params: {} 2022-09-03T07:21:16.519 DEBUG:boto:Token: None 2022-09-03T07:21:16.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cwBjQy+Lhqgz9aGTrJPQpEX+42Y= 2022-09-03T07:21:16.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cwBjQy+Lhqgz9aGTrJPQpEX+42Y=', 'Content-Length': '0'} 2022-09-03T07:21:16.523 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007bab089244d538f1-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.523 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.524 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.524 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.525 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.525 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.526 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.526 DEBUG:boto:Data: 2022-09-03T07:21:16.527 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.527 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.528 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.528 DEBUG:boto:Params: {} 2022-09-03T07:21:16.529 DEBUG:boto:Token: None 2022-09-03T07:21:16.529 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TwQgOM5ctvcgSsPi3pYLCX73DkU= 2022-09-03T07:21:16.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TwQgOM5ctvcgSsPi3pYLCX73DkU=', 'Content-Length': '0'} 2022-09-03T07:21:16.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068c21dd01d5fc1d2-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.533 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.534 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.534 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.535 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.535 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.536 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.536 DEBUG:boto:Data: 2022-09-03T07:21:16.537 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.538 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.538 DEBUG:boto:Params: {} 2022-09-03T07:21:16.539 DEBUG:boto:Token: None 2022-09-03T07:21:16.539 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TwQgOM5ctvcgSsPi3pYLCX73DkU= 2022-09-03T07:21:16.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TwQgOM5ctvcgSsPi3pYLCX73DkU=', 'Content-Length': '0'} 2022-09-03T07:21:16.543 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000350aa83f492dda8f-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.543 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:21:16.544 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.544 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.545 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.545 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.546 DEBUG:boto:Method: GET 2022-09-03T07:21:16.546 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.547 DEBUG:boto:Data: 2022-09-03T07:21:16.547 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.548 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.548 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.549 DEBUG:boto:Params: {} 2022-09-03T07:21:16.550 DEBUG:boto:Token: None 2022-09-03T07:21:16.550 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cwBjQy+Lhqgz9aGTrJPQpEX+42Y= 2022-09-03T07:21:16.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cwBjQy+Lhqgz9aGTrJPQpEX+42Y=', 'Content-Length': '0'} 2022-09-03T07:21:16.554 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000298e325bc6e00ede-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.554 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.555 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.555 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.556 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.556 DEBUG:boto:Method: GET 2022-09-03T07:21:16.557 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.558 DEBUG:boto:Data: 2022-09-03T07:21:16.558 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.559 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.560 DEBUG:boto:Params: {} 2022-09-03T07:21:16.560 DEBUG:boto:Token: None 2022-09-03T07:21:16.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:16.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cwBjQy+Lhqgz9aGTrJPQpEX+42Y= 2022-09-03T07:21:16.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cwBjQy+Lhqgz9aGTrJPQpEX+42Y=', 'Content-Length': '0'} 2022-09-03T07:21:16.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000665acad28c0a0994-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.606 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.607 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.608 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:16.609 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:16.610 DEBUG:boto:Method: GET 2022-09-03T07:21:16.611 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:16.612 DEBUG:boto:Data: 2022-09-03T07:21:16.613 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.615 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.615 DEBUG:boto:Params: {} 2022-09-03T07:21:16.616 DEBUG:boto:Token: None 2022-09-03T07:21:16.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.620 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000958ddb12d21e7170-006313006c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.621 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.622 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.622 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.623 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:16.623 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:16.624 DEBUG:boto:Method: GET 2022-09-03T07:21:16.624 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:16.625 DEBUG:boto:Data: 2022-09-03T07:21:16.625 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.626 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.627 DEBUG:boto:Params: {} 2022-09-03T07:21:16.628 DEBUG:boto:Token: None 2022-09-03T07:21:16.628 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.633 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002183c980a035a651-006313006c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.633 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.634 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.635 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.635 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.636 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:16.636 DEBUG:boto:Data: 2022-09-03T07:21:16.637 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.637 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.638 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.638 DEBUG:boto:Params: {} 2022-09-03T07:21:16.639 DEBUG:boto:Token: None 2022-09-03T07:21:16.640 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57 2022-09-03T07:21:16.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mbrGp7uxUgeo7k93kDeexdXfZD0= 2022-09-03T07:21:16.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mbrGp7uxUgeo7k93kDeexdXfZD0=', 'Content-Length': '0'} 2022-09-03T07:21:16.643 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044d131dc70fd8c6c-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.644 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.644 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.645 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.645 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:16.646 DEBUG:boto:Data: 2022-09-03T07:21:16.646 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.647 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.648 DEBUG:boto:Params: {} 2022-09-03T07:21:16.649 DEBUG:boto:Token: None 2022-09-03T07:21:16.649 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57 2022-09-03T07:21:16.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mbrGp7uxUgeo7k93kDeexdXfZD0= 2022-09-03T07:21:16.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mbrGp7uxUgeo7k93kDeexdXfZD0=', 'Content-Length': '0'} 2022-09-03T07:21:16.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c83ad9bffe738f5f-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.653 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:21:16.654 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.654 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.655 DEBUG:boto:Method: GET 2022-09-03T07:21:16.655 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:16.656 DEBUG:boto:Data: 2022-09-03T07:21:16.657 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.658 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.659 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.660 DEBUG:boto:Params: {} 2022-09-03T07:21:16.661 DEBUG:boto:Token: None 2022-09-03T07:21:16.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57 2022-09-03T07:21:16.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sYMkvCZHmXt6ASxDBhmj62sN6OE= 2022-09-03T07:21:16.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sYMkvCZHmXt6ASxDBhmj62sN6OE=', 'Content-Length': '0'} 2022-09-03T07:21:16.666 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f494374b0b2729b0-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.714 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:16.715 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:16.715 DEBUG:boto:Method: GET 2022-09-03T07:21:16.716 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:16.717 DEBUG:boto:Data: 2022-09-03T07:21:16.717 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.718 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.719 DEBUG:boto:Params: {} 2022-09-03T07:21:16.719 DEBUG:boto:Token: None 2022-09-03T07:21:16.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj57 2022-09-03T07:21:16.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sYMkvCZHmXt6ASxDBhmj62sN6OE= 2022-09-03T07:21:16.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sYMkvCZHmXt6ASxDBhmj62sN6OE=', 'Content-Length': '0'} 2022-09-03T07:21:16.724 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000345ba55f081e2376-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.725 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.725 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.726 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:16.726 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:16.727 DEBUG:boto:Method: GET 2022-09-03T07:21:16.727 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:16.728 DEBUG:boto:Data: 2022-09-03T07:21:16.728 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.729 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.730 DEBUG:boto:Params: {} 2022-09-03T07:21:16.740 DEBUG:boto:Token: None 2022-09-03T07:21:16.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.745 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000489bda633dcf1ab0-006313006c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.745 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.746 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.747 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.747 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:16.748 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:16.748 DEBUG:boto:Method: GET 2022-09-03T07:21:16.749 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:16.749 DEBUG:boto:Data: 2022-09-03T07:21:16.750 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.751 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.752 DEBUG:boto:Params: {} 2022-09-03T07:21:16.752 DEBUG:boto:Token: None 2022-09-03T07:21:16.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.758 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bbf07e25e414b6f3-006313006c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.759 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.761 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:21:16.762 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.763 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.763 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.764 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.764 DEBUG:boto:Method: GET 2022-09-03T07:21:16.765 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.765 DEBUG:boto:Data: 2022-09-03T07:21:16.766 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.766 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.767 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.767 DEBUG:boto:Params: {} 2022-09-03T07:21:16.768 DEBUG:boto:Token: None 2022-09-03T07:21:16.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V7JsRvK6vB7zH4ExFImuJy1uEIQ= 2022-09-03T07:21:16.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V7JsRvK6vB7zH4ExFImuJy1uEIQ=', 'Content-Length': '0'} 2022-09-03T07:21:16.773 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f2cb7259ca56e732-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.774 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.775 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.776 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.776 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.777 DEBUG:boto:Method: GET 2022-09-03T07:21:16.777 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.778 DEBUG:boto:Data: 2022-09-03T07:21:16.778 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.779 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.780 DEBUG:boto:Params: {} 2022-09-03T07:21:16.780 DEBUG:boto:Token: None 2022-09-03T07:21:16.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V7JsRvK6vB7zH4ExFImuJy1uEIQ= 2022-09-03T07:21:16.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V7JsRvK6vB7zH4ExFImuJy1uEIQ=', 'Content-Length': '0'} 2022-09-03T07:21:16.785 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000704a16b7df0c749c-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.786 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.786 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.787 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.787 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.788 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.788 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.789 DEBUG:boto:Data: 2022-09-03T07:21:16.790 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.791 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.791 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.792 DEBUG:boto:Params: {} 2022-09-03T07:21:16.792 DEBUG:boto:Token: None 2022-09-03T07:21:16.793 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0GS5kH+Bpaj2MgZxpiFOPYj+AfM= 2022-09-03T07:21:16.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0GS5kH+Bpaj2MgZxpiFOPYj+AfM=', 'Content-Length': '0'} 2022-09-03T07:21:16.796 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e144b36845adcd43-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.797 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.798 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.798 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.799 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.799 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.800 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.800 DEBUG:boto:Data: 2022-09-03T07:21:16.801 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.802 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.803 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.803 DEBUG:boto:Params: {} 2022-09-03T07:21:16.804 DEBUG:boto:Token: None 2022-09-03T07:21:16.805 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0GS5kH+Bpaj2MgZxpiFOPYj+AfM= 2022-09-03T07:21:16.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0GS5kH+Bpaj2MgZxpiFOPYj+AfM=', 'Content-Length': '0'} 2022-09-03T07:21:16.809 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c685ca01d9353df-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.809 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:21:16.810 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.811 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.811 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.812 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.812 DEBUG:boto:Method: GET 2022-09-03T07:21:16.813 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.813 DEBUG:boto:Data: 2022-09-03T07:21:16.814 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.814 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.815 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.816 DEBUG:boto:Params: {} 2022-09-03T07:21:16.816 DEBUG:boto:Token: None 2022-09-03T07:21:16.817 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V7JsRvK6vB7zH4ExFImuJy1uEIQ= 2022-09-03T07:21:16.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V7JsRvK6vB7zH4ExFImuJy1uEIQ=', 'Content-Length': '0'} 2022-09-03T07:21:16.820 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef34ac0f9949727a-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.821 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.822 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.822 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.823 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.823 DEBUG:boto:Method: GET 2022-09-03T07:21:16.824 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.825 DEBUG:boto:Data: 2022-09-03T07:21:16.825 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.826 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.827 DEBUG:boto:Params: {} 2022-09-03T07:21:16.827 DEBUG:boto:Token: None 2022-09-03T07:21:16.828 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:16.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V7JsRvK6vB7zH4ExFImuJy1uEIQ= 2022-09-03T07:21:16.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V7JsRvK6vB7zH4ExFImuJy1uEIQ=', 'Content-Length': '0'} 2022-09-03T07:21:16.831 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1e5afac754dd79f-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.832 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.833 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.833 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:16.834 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:16.834 DEBUG:boto:Method: GET 2022-09-03T07:21:16.835 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:16.836 DEBUG:boto:Data: 2022-09-03T07:21:16.836 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.837 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.837 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.838 DEBUG:boto:Params: {} 2022-09-03T07:21:16.838 DEBUG:boto:Token: None 2022-09-03T07:21:16.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.843 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cab07eb9e7e13029-006313006c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.844 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.845 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.845 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.846 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:16.846 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:16.847 DEBUG:boto:Method: GET 2022-09-03T07:21:16.847 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:16.848 DEBUG:boto:Data: 2022-09-03T07:21:16.848 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.849 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.850 DEBUG:boto:Params: {} 2022-09-03T07:21:16.851 DEBUG:boto:Token: None 2022-09-03T07:21:16.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.856 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc8dcb31cffac52c-006313006c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.857 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.857 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.858 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.859 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.859 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:16.860 DEBUG:boto:Data: 2022-09-03T07:21:16.860 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.861 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.861 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.862 DEBUG:boto:Params: {} 2022-09-03T07:21:16.862 DEBUG:boto:Token: None 2022-09-03T07:21:16.863 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68 2022-09-03T07:21:16.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZtrYfEjFxT2IMLpnbKqcI1IpIo= 2022-09-03T07:21:16.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZtrYfEjFxT2IMLpnbKqcI1IpIo=', 'Content-Length': '0'} 2022-09-03T07:21:16.866 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d444df879a4e5de3-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.867 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.868 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.868 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.869 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:16.870 DEBUG:boto:Data: 2022-09-03T07:21:16.871 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.871 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.872 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.872 DEBUG:boto:Params: {} 2022-09-03T07:21:16.873 DEBUG:boto:Token: None 2022-09-03T07:21:16.874 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68 2022-09-03T07:21:16.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZtrYfEjFxT2IMLpnbKqcI1IpIo= 2022-09-03T07:21:16.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZtrYfEjFxT2IMLpnbKqcI1IpIo=', 'Content-Length': '0'} 2022-09-03T07:21:16.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a4448cba8fee291-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.878 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:21:16.878 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.879 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.879 DEBUG:boto:Method: GET 2022-09-03T07:21:16.880 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:16.880 DEBUG:boto:Data: 2022-09-03T07:21:16.881 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.882 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.883 DEBUG:boto:Params: {} 2022-09-03T07:21:16.883 DEBUG:boto:Token: None 2022-09-03T07:21:16.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68 2022-09-03T07:21:16.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:42oeJjJD2VIkrqFHOpwIilnQIVM= 2022-09-03T07:21:16.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:42oeJjJD2VIkrqFHOpwIilnQIVM=', 'Content-Length': '0'} 2022-09-03T07:21:16.888 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f545799bf353c8d0-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.930 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:16.931 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:16.931 DEBUG:boto:Method: GET 2022-09-03T07:21:16.932 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:16.933 DEBUG:boto:Data: 2022-09-03T07:21:16.934 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.935 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.936 DEBUG:boto:Params: {} 2022-09-03T07:21:16.936 DEBUG:boto:Token: None 2022-09-03T07:21:16.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj68 2022-09-03T07:21:16.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:42oeJjJD2VIkrqFHOpwIilnQIVM= 2022-09-03T07:21:16.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:42oeJjJD2VIkrqFHOpwIilnQIVM=', 'Content-Length': '0'} 2022-09-03T07:21:16.942 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb6a4ced921b992c-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.942 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.943 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.944 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:16.944 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:16.945 DEBUG:boto:Method: GET 2022-09-03T07:21:16.945 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:16.946 DEBUG:boto:Data: 2022-09-03T07:21:16.946 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.947 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.948 DEBUG:boto:Params: {} 2022-09-03T07:21:16.949 DEBUG:boto:Token: None 2022-09-03T07:21:16.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.954 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098e851f6951b3d23-006313006c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.955 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.955 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:16.956 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:16.956 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:16.957 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:16.958 DEBUG:boto:Method: GET 2022-09-03T07:21:16.958 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:16.959 DEBUG:boto:Data: 2022-09-03T07:21:16.959 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.960 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.961 DEBUG:boto:Params: {} 2022-09-03T07:21:16.961 DEBUG:boto:Token: None 2022-09-03T07:21:16.962 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/ 2022-09-03T07:21:16.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI= 2022-09-03T07:21:16.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpT8+P3EAmT0Y2JlWkrpN3D57zI=', 'Content-Length': '0'} 2022-09-03T07:21:16.967 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f6e3ba8ae64cdc8f-006313006c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.967 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:16.968 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:21:16.969 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:16.969 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:16.970 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.970 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.971 DEBUG:boto:Method: GET 2022-09-03T07:21:16.971 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.972 DEBUG:boto:Data: 2022-09-03T07:21:16.972 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.973 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.974 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.974 DEBUG:boto:Params: {} 2022-09-03T07:21:16.975 DEBUG:boto:Token: None 2022-09-03T07:21:16.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nEuT0MGyNs7BGLNPQgSkULURrtc= 2022-09-03T07:21:16.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nEuT0MGyNs7BGLNPQgSkULURrtc=', 'Content-Length': '0'} 2022-09-03T07:21:16.979 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034814d5e57f871f8-006313006c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.980 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:16.981 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:16.981 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.982 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.983 DEBUG:boto:Method: GET 2022-09-03T07:21:16.984 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.985 DEBUG:boto:Data: 2022-09-03T07:21:16.986 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.986 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:16.987 DEBUG:boto:Port: 8000 2022-09-03T07:21:16.987 DEBUG:boto:Params: {} 2022-09-03T07:21:16.988 DEBUG:boto:Token: None 2022-09-03T07:21:16.989 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:16 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nEuT0MGyNs7BGLNPQgSkULURrtc= 2022-09-03T07:21:16.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nEuT0MGyNs7BGLNPQgSkULURrtc=', 'Content-Length': '0'} 2022-09-03T07:21:16.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004002621eb8e74091-006313006c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:16.994 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:16.994 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:16.995 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.995 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.996 DEBUG:boto:Method: HEAD 2022-09-03T07:21:16.996 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:16.997 DEBUG:boto:Data: 2022-09-03T07:21:16.997 DEBUG:boto:Headers: {} 2022-09-03T07:21:16.998 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:16.998 DEBUG:boto:Port: 8001 2022-09-03T07:21:16.999 DEBUG:boto:Params: {} 2022-09-03T07:21:17.000 DEBUG:boto:Token: None 2022-09-03T07:21:17.000 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jezmbMNkEddSpqhHfXkxmaSDtqY= 2022-09-03T07:21:17.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jezmbMNkEddSpqhHfXkxmaSDtqY=', 'Content-Length': '0'} 2022-09-03T07:21:17.004 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093cd42b1c278c09a-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.004 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:17.005 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:17.005 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.006 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.007 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.007 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.008 DEBUG:boto:Data: 2022-09-03T07:21:17.008 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.009 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.010 DEBUG:boto:Params: {} 2022-09-03T07:21:17.010 DEBUG:boto:Token: None 2022-09-03T07:21:17.011 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jezmbMNkEddSpqhHfXkxmaSDtqY= 2022-09-03T07:21:17.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jezmbMNkEddSpqhHfXkxmaSDtqY=', 'Content-Length': '0'} 2022-09-03T07:21:17.014 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a752c4fc38bc9da9-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.015 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:21:17.016 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:17.016 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:17.017 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.017 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.018 DEBUG:boto:Method: GET 2022-09-03T07:21:17.018 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.019 DEBUG:boto:Data: 2022-09-03T07:21:17.019 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.021 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.021 DEBUG:boto:Params: {} 2022-09-03T07:21:17.022 DEBUG:boto:Token: None 2022-09-03T07:21:17.022 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gJPPJkQIUPe4AiGb4ZOVPmxGHqE= 2022-09-03T07:21:17.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gJPPJkQIUPe4AiGb4ZOVPmxGHqE=', 'Content-Length': '0'} 2022-09-03T07:21:17.026 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e62d2f71285a47e6-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.027 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:17.027 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:17.028 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.028 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.029 DEBUG:boto:Method: GET 2022-09-03T07:21:17.029 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.030 DEBUG:boto:Data: 2022-09-03T07:21:17.031 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.032 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.033 DEBUG:boto:Params: {} 2022-09-03T07:21:17.033 DEBUG:boto:Token: None 2022-09-03T07:21:17.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:17.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gJPPJkQIUPe4AiGb4ZOVPmxGHqE= 2022-09-03T07:21:17.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gJPPJkQIUPe4AiGb4ZOVPmxGHqE=', 'Content-Length': '0'} 2022-09-03T07:21:17.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049686f81a940d1d3-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.038 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.039 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.039 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:17.040 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:17.041 DEBUG:boto:Method: GET 2022-09-03T07:21:17.042 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:17.043 DEBUG:boto:Data: 2022-09-03T07:21:17.043 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.044 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.045 DEBUG:boto:Params: {} 2022-09-03T07:21:17.046 DEBUG:boto:Token: None 2022-09-03T07:21:17.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.050 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001f61d25ea0edce4d-006313006d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.094 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.096 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.097 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.098 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:17.099 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:17.100 DEBUG:boto:Method: GET 2022-09-03T07:21:17.101 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:17.101 DEBUG:boto:Data: 2022-09-03T07:21:17.102 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.104 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.105 DEBUG:boto:Params: {} 2022-09-03T07:21:17.107 DEBUG:boto:Token: None 2022-09-03T07:21:17.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.114 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dccccfbd21857840-006313006d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.115 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.116 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:17.117 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:17.117 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.118 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:17.119 DEBUG:boto:Data: 2022-09-03T07:21:17.119 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.119 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.120 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.120 DEBUG:boto:Params: {} 2022-09-03T07:21:17.121 DEBUG:boto:Token: None 2022-09-03T07:21:17.122 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj76 2022-09-03T07:21:17.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r25i3PTuWurUbr/Dzv59hgI6LH0= 2022-09-03T07:21:17.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r25i3PTuWurUbr/Dzv59hgI6LH0=', 'Content-Length': '0'} 2022-09-03T07:21:17.125 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a521051ba131ac3-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.126 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:17.126 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:17.127 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.127 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:17.128 DEBUG:boto:Data: 2022-09-03T07:21:17.128 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.129 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.129 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.130 DEBUG:boto:Params: {} 2022-09-03T07:21:17.130 DEBUG:boto:Token: None 2022-09-03T07:21:17.131 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj76 2022-09-03T07:21:17.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r25i3PTuWurUbr/Dzv59hgI6LH0= 2022-09-03T07:21:17.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r25i3PTuWurUbr/Dzv59hgI6LH0=', 'Content-Length': '0'} 2022-09-03T07:21:17.134 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f80aca4a38a66999-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.135 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:21:17.136 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:17.137 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:17.138 DEBUG:boto:Method: GET 2022-09-03T07:21:17.138 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:17.139 DEBUG:boto:Data: 2022-09-03T07:21:17.140 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.140 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.141 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.141 DEBUG:boto:Params: {} 2022-09-03T07:21:17.142 DEBUG:boto:Token: None 2022-09-03T07:21:17.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj76 2022-09-03T07:21:17.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YD1XROq0Mr5qPrs30oREUqh5z5M= 2022-09-03T07:21:17.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YD1XROq0Mr5qPrs30oREUqh5z5M=', 'Content-Length': '0'} 2022-09-03T07:21:17.146 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002d7f64c55765a88-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.147 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:17.147 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:17.148 DEBUG:boto:Method: GET 2022-09-03T07:21:17.148 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:17.149 DEBUG:boto:Data: 2022-09-03T07:21:17.149 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.151 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.151 DEBUG:boto:Params: {} 2022-09-03T07:21:17.152 DEBUG:boto:Token: None 2022-09-03T07:21:17.152 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj76 2022-09-03T07:21:17.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YD1XROq0Mr5qPrs30oREUqh5z5M= 2022-09-03T07:21:17.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YD1XROq0Mr5qPrs30oREUqh5z5M=', 'Content-Length': '0'} 2022-09-03T07:21:17.156 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d09b25ec314420ae-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.198 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.199 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.199 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:17.200 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:17.201 DEBUG:boto:Method: GET 2022-09-03T07:21:17.201 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:17.202 DEBUG:boto:Data: 2022-09-03T07:21:17.202 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.204 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.204 DEBUG:boto:Params: {} 2022-09-03T07:21:17.205 DEBUG:boto:Token: None 2022-09-03T07:21:17.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058b1220d47a221f9-006313006d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.210 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.211 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.212 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.212 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:17.213 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:17.213 DEBUG:boto:Method: GET 2022-09-03T07:21:17.214 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:17.214 DEBUG:boto:Data: 2022-09-03T07:21:17.215 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.216 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.216 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.217 DEBUG:boto:Params: {} 2022-09-03T07:21:17.217 DEBUG:boto:Token: None 2022-09-03T07:21:17.218 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.222 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005f72526597376bfc-006313006d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.223 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.224 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:21:17.224 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.225 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.226 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.226 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.227 DEBUG:boto:Method: GET 2022-09-03T07:21:17.227 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.228 DEBUG:boto:Data: 2022-09-03T07:21:17.228 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.229 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.229 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.230 DEBUG:boto:Params: {} 2022-09-03T07:21:17.230 DEBUG:boto:Token: None 2022-09-03T07:21:17.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PAPa79tTpBDBl25HpJC5Xos5fYo= 2022-09-03T07:21:17.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PAPa79tTpBDBl25HpJC5Xos5fYo=', 'Content-Length': '0'} 2022-09-03T07:21:17.235 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000031d59c3eb6b41da-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.236 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.236 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.237 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.237 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.238 DEBUG:boto:Method: GET 2022-09-03T07:21:17.238 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.239 DEBUG:boto:Data: 2022-09-03T07:21:17.239 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.240 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.241 DEBUG:boto:Params: {} 2022-09-03T07:21:17.242 DEBUG:boto:Token: None 2022-09-03T07:21:17.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PAPa79tTpBDBl25HpJC5Xos5fYo= 2022-09-03T07:21:17.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PAPa79tTpBDBl25HpJC5Xos5fYo=', 'Content-Length': '0'} 2022-09-03T07:21:17.246 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009443023ea587052f-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.246 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.247 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.247 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.248 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.249 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.249 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.250 DEBUG:boto:Data: 2022-09-03T07:21:17.250 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.251 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.252 DEBUG:boto:Params: {} 2022-09-03T07:21:17.252 DEBUG:boto:Token: None 2022-09-03T07:21:17.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MTzLP3DeWkPs7ZlqdLnfwHtqQ6M= 2022-09-03T07:21:17.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MTzLP3DeWkPs7ZlqdLnfwHtqQ6M=', 'Content-Length': '0'} 2022-09-03T07:21:17.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027f80f64fd95ce7d-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.257 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.257 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.258 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.258 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.259 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.259 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.260 DEBUG:boto:Data: 2022-09-03T07:21:17.261 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.261 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.262 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.262 DEBUG:boto:Params: {} 2022-09-03T07:21:17.263 DEBUG:boto:Token: None 2022-09-03T07:21:17.263 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.264 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MTzLP3DeWkPs7ZlqdLnfwHtqQ6M= 2022-09-03T07:21:17.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MTzLP3DeWkPs7ZlqdLnfwHtqQ6M=', 'Content-Length': '0'} 2022-09-03T07:21:17.272 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000acfd27bb0c84ebaf-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.272 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:21:17.273 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.274 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.274 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.275 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.275 DEBUG:boto:Method: GET 2022-09-03T07:21:17.276 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.276 DEBUG:boto:Data: 2022-09-03T07:21:17.277 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.278 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.278 DEBUG:boto:Params: {} 2022-09-03T07:21:17.279 DEBUG:boto:Token: None 2022-09-03T07:21:17.280 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PAPa79tTpBDBl25HpJC5Xos5fYo= 2022-09-03T07:21:17.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PAPa79tTpBDBl25HpJC5Xos5fYo=', 'Content-Length': '0'} 2022-09-03T07:21:17.283 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012b127118e339461-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.284 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.285 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.285 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.286 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.286 DEBUG:boto:Method: GET 2022-09-03T07:21:17.287 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.287 DEBUG:boto:Data: 2022-09-03T07:21:17.288 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.289 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.290 DEBUG:boto:Params: {} 2022-09-03T07:21:17.290 DEBUG:boto:Token: None 2022-09-03T07:21:17.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:17.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PAPa79tTpBDBl25HpJC5Xos5fYo= 2022-09-03T07:21:17.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PAPa79tTpBDBl25HpJC5Xos5fYo=', 'Content-Length': '0'} 2022-09-03T07:21:17.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000baa724717bfab23-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.338 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.339 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.339 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:17.340 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:17.341 DEBUG:boto:Method: GET 2022-09-03T07:21:17.341 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:17.342 DEBUG:boto:Data: 2022-09-03T07:21:17.342 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.343 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.344 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.345 DEBUG:boto:Params: {} 2022-09-03T07:21:17.345 DEBUG:boto:Token: None 2022-09-03T07:21:17.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.350 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c1b437eb36cf532-006313006d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.351 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.352 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.352 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.353 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:17.353 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:17.354 DEBUG:boto:Method: GET 2022-09-03T07:21:17.354 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:17.355 DEBUG:boto:Data: 2022-09-03T07:21:17.355 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.356 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.357 DEBUG:boto:Params: {} 2022-09-03T07:21:17.357 DEBUG:boto:Token: None 2022-09-03T07:21:17.358 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.359 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.363 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000816df58bae71ad20-006313006d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.363 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.364 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.365 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.365 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.366 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:17.366 DEBUG:boto:Data: 2022-09-03T07:21:17.367 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.368 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.368 DEBUG:boto:Params: {} 2022-09-03T07:21:17.369 DEBUG:boto:Token: None 2022-09-03T07:21:17.369 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85 2022-09-03T07:21:17.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fp5nFDRwTglGoHEa4bgxIR6Q158= 2022-09-03T07:21:17.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fp5nFDRwTglGoHEa4bgxIR6Q158=', 'Content-Length': '0'} 2022-09-03T07:21:17.373 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d31d8e6f267475e-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.373 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.374 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.374 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.375 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:17.376 DEBUG:boto:Data: 2022-09-03T07:21:17.376 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.377 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.377 DEBUG:boto:Params: {} 2022-09-03T07:21:17.378 DEBUG:boto:Token: None 2022-09-03T07:21:17.379 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85 2022-09-03T07:21:17.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fp5nFDRwTglGoHEa4bgxIR6Q158= 2022-09-03T07:21:17.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fp5nFDRwTglGoHEa4bgxIR6Q158=', 'Content-Length': '0'} 2022-09-03T07:21:17.382 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1a737d3cbea10a3-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.383 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:21:17.383 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.384 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.384 DEBUG:boto:Method: GET 2022-09-03T07:21:17.385 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:17.385 DEBUG:boto:Data: 2022-09-03T07:21:17.386 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.387 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.387 DEBUG:boto:Params: {} 2022-09-03T07:21:17.388 DEBUG:boto:Token: None 2022-09-03T07:21:17.388 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85 2022-09-03T07:21:17.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WkTkABXas4cpwjrLgitb35JcYsE= 2022-09-03T07:21:17.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WkTkABXas4cpwjrLgitb35JcYsE=', 'Content-Length': '0'} 2022-09-03T07:21:17.392 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e26d80cf7ca7184-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.434 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:17.435 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:17.435 DEBUG:boto:Method: GET 2022-09-03T07:21:17.436 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:17.437 DEBUG:boto:Data: 2022-09-03T07:21:17.437 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.438 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.439 DEBUG:boto:Params: {} 2022-09-03T07:21:17.439 DEBUG:boto:Token: None 2022-09-03T07:21:17.440 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj85 2022-09-03T07:21:17.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WkTkABXas4cpwjrLgitb35JcYsE= 2022-09-03T07:21:17.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WkTkABXas4cpwjrLgitb35JcYsE=', 'Content-Length': '0'} 2022-09-03T07:21:17.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003995db39d480cb21-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.445 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.445 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.446 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:17.446 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:17.447 DEBUG:boto:Method: GET 2022-09-03T07:21:17.448 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:17.448 DEBUG:boto:Data: 2022-09-03T07:21:17.449 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.450 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.450 DEBUG:boto:Params: {} 2022-09-03T07:21:17.451 DEBUG:boto:Token: None 2022-09-03T07:21:17.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.455 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000257a12294c94e496-006313006d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.456 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.457 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.458 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.458 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:17.459 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:17.459 DEBUG:boto:Method: GET 2022-09-03T07:21:17.460 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:17.460 DEBUG:boto:Data: 2022-09-03T07:21:17.461 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.462 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.463 DEBUG:boto:Params: {} 2022-09-03T07:21:17.463 DEBUG:boto:Token: None 2022-09-03T07:21:17.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000199023fba614b6b7-006313006d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.469 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.470 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:21:17.470 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.471 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.472 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.473 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.474 DEBUG:boto:Method: GET 2022-09-03T07:21:17.474 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.475 DEBUG:boto:Data: 2022-09-03T07:21:17.476 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.477 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.478 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.478 DEBUG:boto:Params: {} 2022-09-03T07:21:17.479 DEBUG:boto:Token: None 2022-09-03T07:21:17.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MjBRkBlQatUgng8OIljwU9ZW6xo= 2022-09-03T07:21:17.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MjBRkBlQatUgng8OIljwU9ZW6xo=', 'Content-Length': '0'} 2022-09-03T07:21:17.484 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009145a8f4a39a54ba-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.485 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.486 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.487 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.488 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.488 DEBUG:boto:Method: GET 2022-09-03T07:21:17.489 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.490 DEBUG:boto:Data: 2022-09-03T07:21:17.491 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.492 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.493 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.493 DEBUG:boto:Params: {} 2022-09-03T07:21:17.494 DEBUG:boto:Token: None 2022-09-03T07:21:17.495 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MjBRkBlQatUgng8OIljwU9ZW6xo= 2022-09-03T07:21:17.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MjBRkBlQatUgng8OIljwU9ZW6xo=', 'Content-Length': '0'} 2022-09-03T07:21:17.499 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d5b33fb87f7088c-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.500 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.501 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.501 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.502 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.503 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.503 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.504 DEBUG:boto:Data: 2022-09-03T07:21:17.504 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.505 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.506 DEBUG:boto:Params: {} 2022-09-03T07:21:17.506 DEBUG:boto:Token: None 2022-09-03T07:21:17.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VdqnrOgeUpQ1TUcu8u0EMNc/DJ0= 2022-09-03T07:21:17.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VdqnrOgeUpQ1TUcu8u0EMNc/DJ0=', 'Content-Length': '0'} 2022-09-03T07:21:17.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002770350ad1946048-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.511 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.512 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.512 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.513 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.514 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.515 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.516 DEBUG:boto:Data: 2022-09-03T07:21:17.517 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.518 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.519 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.520 DEBUG:boto:Params: {} 2022-09-03T07:21:17.521 DEBUG:boto:Token: None 2022-09-03T07:21:17.522 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VdqnrOgeUpQ1TUcu8u0EMNc/DJ0= 2022-09-03T07:21:17.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VdqnrOgeUpQ1TUcu8u0EMNc/DJ0=', 'Content-Length': '0'} 2022-09-03T07:21:17.526 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000050f95f2903c8d681-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.526 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:21:17.527 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.528 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.528 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.529 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.530 DEBUG:boto:Method: GET 2022-09-03T07:21:17.531 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.531 DEBUG:boto:Data: 2022-09-03T07:21:17.532 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.533 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.534 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.535 DEBUG:boto:Params: {} 2022-09-03T07:21:17.536 DEBUG:boto:Token: None 2022-09-03T07:21:17.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MjBRkBlQatUgng8OIljwU9ZW6xo= 2022-09-03T07:21:17.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MjBRkBlQatUgng8OIljwU9ZW6xo=', 'Content-Length': '0'} 2022-09-03T07:21:17.541 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a3a31539c10fd4e-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.582 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.583 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.583 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.584 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.585 DEBUG:boto:Method: GET 2022-09-03T07:21:17.585 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.587 DEBUG:boto:Data: 2022-09-03T07:21:17.588 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.589 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.590 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.591 DEBUG:boto:Params: {} 2022-09-03T07:21:17.592 DEBUG:boto:Token: None 2022-09-03T07:21:17.593 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:17.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MjBRkBlQatUgng8OIljwU9ZW6xo= 2022-09-03T07:21:17.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MjBRkBlQatUgng8OIljwU9ZW6xo=', 'Content-Length': '0'} 2022-09-03T07:21:17.597 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a724cbe224fa4df6-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.598 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.599 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.600 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:17.600 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:17.601 DEBUG:boto:Method: GET 2022-09-03T07:21:17.602 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:17.602 DEBUG:boto:Data: 2022-09-03T07:21:17.603 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.605 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.606 DEBUG:boto:Params: {} 2022-09-03T07:21:17.606 DEBUG:boto:Token: None 2022-09-03T07:21:17.607 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.612 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd8f4a323e666bca-006313006d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.612 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.613 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.614 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.614 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:17.615 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:17.616 DEBUG:boto:Method: GET 2022-09-03T07:21:17.616 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:17.617 DEBUG:boto:Data: 2022-09-03T07:21:17.617 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.618 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.619 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.620 DEBUG:boto:Params: {} 2022-09-03T07:21:17.620 DEBUG:boto:Token: None 2022-09-03T07:21:17.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.625 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e55f373b86cd271c-006313006d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.626 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.627 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.628 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.628 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.629 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:17.630 DEBUG:boto:Data: 2022-09-03T07:21:17.630 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.631 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.632 DEBUG:boto:Params: {} 2022-09-03T07:21:17.632 DEBUG:boto:Token: None 2022-09-03T07:21:17.633 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95 2022-09-03T07:21:17.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z/J03Ncat+RfHysatK17mpeizfY= 2022-09-03T07:21:17.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z/J03Ncat+RfHysatK17mpeizfY=', 'Content-Length': '0'} 2022-09-03T07:21:17.637 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f4c14975a89d8d2-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.638 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.639 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.640 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.640 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:17.641 DEBUG:boto:Data: 2022-09-03T07:21:17.641 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.642 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.643 DEBUG:boto:Params: {} 2022-09-03T07:21:17.643 DEBUG:boto:Token: None 2022-09-03T07:21:17.644 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95 2022-09-03T07:21:17.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z/J03Ncat+RfHysatK17mpeizfY= 2022-09-03T07:21:17.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z/J03Ncat+RfHysatK17mpeizfY=', 'Content-Length': '0'} 2022-09-03T07:21:17.647 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cbc65da9e94872a6-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.649 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:21:17.650 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.650 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.651 DEBUG:boto:Method: GET 2022-09-03T07:21:17.651 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:17.652 DEBUG:boto:Data: 2022-09-03T07:21:17.652 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.653 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.653 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.654 DEBUG:boto:Params: {} 2022-09-03T07:21:17.655 DEBUG:boto:Token: None 2022-09-03T07:21:17.655 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95 2022-09-03T07:21:17.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TXsWn0TIdXue6ZFsOuzjsTTOJ4Q= 2022-09-03T07:21:17.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TXsWn0TIdXue6ZFsOuzjsTTOJ4Q=', 'Content-Length': '0'} 2022-09-03T07:21:17.660 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9d5ac3a6ee97a66-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.661 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:17.662 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:17.662 DEBUG:boto:Method: GET 2022-09-03T07:21:17.663 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:17.663 DEBUG:boto:Data: 2022-09-03T07:21:17.664 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.664 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.665 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.665 DEBUG:boto:Params: {} 2022-09-03T07:21:17.666 DEBUG:boto:Token: None 2022-09-03T07:21:17.667 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj95 2022-09-03T07:21:17.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TXsWn0TIdXue6ZFsOuzjsTTOJ4Q= 2022-09-03T07:21:17.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TXsWn0TIdXue6ZFsOuzjsTTOJ4Q=', 'Content-Length': '0'} 2022-09-03T07:21:17.670 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079eb53e529f67624-006313006d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.671 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.672 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.672 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:17.673 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:17.673 DEBUG:boto:Method: GET 2022-09-03T07:21:17.674 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:17.674 DEBUG:boto:Data: 2022-09-03T07:21:17.675 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.676 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.677 DEBUG:boto:Params: {} 2022-09-03T07:21:17.677 DEBUG:boto:Token: None 2022-09-03T07:21:17.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.682 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007cda89206b590d34-006313006d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.683 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.683 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.684 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.684 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:17.685 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:17.686 DEBUG:boto:Method: GET 2022-09-03T07:21:17.686 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:17.687 DEBUG:boto:Data: 2022-09-03T07:21:17.687 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:17.688 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.689 DEBUG:boto:Params: {} 2022-09-03T07:21:17.690 DEBUG:boto:Token: None 2022-09-03T07:21:17.690 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.695 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e697d1e7584fa06f-006313006d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.738 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.739 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:21:17.740 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:21:17.740 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.741 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.741 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.742 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:17.743 DEBUG:boto:Data: 2022-09-03T07:21:17.743 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.744 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.744 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.745 DEBUG:boto:Params: {} 2022-09-03T07:21:17.745 DEBUG:boto:Token: None 2022-09-03T07:21:17.746 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tv3dVfEI5Lk1VQyBJOwzfPVBIVc= 2022-09-03T07:21:17.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tv3dVfEI5Lk1VQyBJOwzfPVBIVc=', 'Content-Length': '0'} 2022-09-03T07:21:17.750 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '15'), ('X-RGW-Bytes-Used', '45'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000556d9e6f49c438f4-006313006d-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.751 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.751 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.752 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.752 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:17.753 DEBUG:boto:Data: 2022-09-03T07:21:17.753 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.754 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:17.754 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.755 DEBUG:boto:Params: {} 2022-09-03T07:21:17.755 DEBUG:boto:Token: None 2022-09-03T07:21:17.756 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tv3dVfEI5Lk1VQyBJOwzfPVBIVc= 2022-09-03T07:21:17.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tv3dVfEI5Lk1VQyBJOwzfPVBIVc=', 'Content-Length': '0'} 2022-09-03T07:21:17.759 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '15'), ('X-RGW-Bytes-Used', '45'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000051e01da6d9f661da-006313006d-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.760 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:21:17.760 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.761 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.761 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.762 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.763 DEBUG:boto:Method: GET 2022-09-03T07:21:17.763 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.764 DEBUG:boto:Data: 2022-09-03T07:21:17.764 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.765 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.766 DEBUG:boto:Params: {} 2022-09-03T07:21:17.766 DEBUG:boto:Token: None 2022-09-03T07:21:17.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/?versions 2022-09-03T07:21:17.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmOFucB+yL5ZSahqMElL03N+xWI= 2022-09-03T07:21:17.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmOFucB+yL5ZSahqMElL03N+xWI=', 'Content-Length': '0'} 2022-09-03T07:21:17.771 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009d0743d69bd30dc3-006313006d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.772 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.773 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:21:17.774 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:21:17.775 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:21:17.775 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:21:17.776 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:21:17.776 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:21:17.777 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:21:17.777 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:21:17.778 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:21:17.778 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:21:17.779 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:21:17.779 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:21:17.780 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:21:17.780 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:21:17.781 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:21:17.782 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:21:17.782 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.783 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.783 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.784 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.784 DEBUG:boto:Method: GET 2022-09-03T07:21:17.785 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.786 DEBUG:boto:Data: 2022-09-03T07:21:17.786 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.787 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:17.787 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.788 DEBUG:boto:Params: {} 2022-09-03T07:21:17.788 DEBUG:boto:Token: None 2022-09-03T07:21:17.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/?versions 2022-09-03T07:21:17.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmOFucB+yL5ZSahqMElL03N+xWI= 2022-09-03T07:21:17.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmOFucB+yL5ZSahqMElL03N+xWI=', 'Content-Length': '0'} 2022-09-03T07:21:17.793 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000013fb5a8765fbd7d-006313006d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.794 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.795 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:21:17.796 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:21:17.796 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:21:17.797 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:21:17.797 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:21:17.798 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:21:17.798 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:21:17.799 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:21:17.800 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:21:17.800 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:21:17.801 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:21:17.801 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:21:17.802 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:21:17.802 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:21:17.803 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:21:17.804 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.804 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.805 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.805 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.806 DEBUG:boto:Method: GET 2022-09-03T07:21:17.806 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.807 DEBUG:boto:Data: 2022-09-03T07:21:17.807 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.808 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.809 DEBUG:boto:Params: {} 2022-09-03T07:21:17.810 DEBUG:boto:Token: None 2022-09-03T07:21:17.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/?versions 2022-09-03T07:21:17.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmOFucB+yL5ZSahqMElL03N+xWI= 2022-09-03T07:21:17.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmOFucB+yL5ZSahqMElL03N+xWI=', 'Content-Length': '0'} 2022-09-03T07:21:17.814 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a36967e82ca675de-006313006d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.815 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.816 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.817 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.817 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.818 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.819 DEBUG:boto:Method: GET 2022-09-03T07:21:17.819 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:21:17.820 DEBUG:boto:Data: 2022-09-03T07:21:17.820 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.821 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:17.821 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.822 DEBUG:boto:Params: {} 2022-09-03T07:21:17.823 DEBUG:boto:Token: None 2022-09-03T07:21:17.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/?versions 2022-09-03T07:21:17.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmOFucB+yL5ZSahqMElL03N+xWI= 2022-09-03T07:21:17.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmOFucB+yL5ZSahqMElL03N+xWI=', 'Content-Length': '0'} 2022-09-03T07:21:17.828 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000904d27d731b2e8b4-006313006d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.829 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.832 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:21:17.832 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:17.833 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:17.834 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.834 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.835 DEBUG:boto:Method: GET 2022-09-03T07:21:17.836 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.836 DEBUG:boto:Data: 2022-09-03T07:21:17.837 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.838 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.839 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.840 DEBUG:boto:Params: {} 2022-09-03T07:21:17.841 DEBUG:boto:Token: None 2022-09-03T07:21:17.842 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.843 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:48pPH5pqAGxpn30JUbdP1V8VbXI= 2022-09-03T07:21:17.844 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:48pPH5pqAGxpn30JUbdP1V8VbXI=', 'Content-Length': '0'} 2022-09-03T07:21:17.847 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb8b5ddca7545882-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.890 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:17.891 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:17.891 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.892 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.893 DEBUG:boto:Method: GET 2022-09-03T07:21:17.894 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.895 DEBUG:boto:Data: 2022-09-03T07:21:17.895 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.896 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:17.897 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.898 DEBUG:boto:Params: {} 2022-09-03T07:21:17.899 DEBUG:boto:Token: None 2022-09-03T07:21:17.900 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:48pPH5pqAGxpn30JUbdP1V8VbXI= 2022-09-03T07:21:17.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:48pPH5pqAGxpn30JUbdP1V8VbXI=', 'Content-Length': '0'} 2022-09-03T07:21:17.903 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e06bbeea01e0089-006313006d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.904 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:17.905 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:17.905 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.906 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.906 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.907 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.908 DEBUG:boto:Data: 2022-09-03T07:21:17.908 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.909 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.910 DEBUG:boto:Params: {} 2022-09-03T07:21:17.910 DEBUG:boto:Token: None 2022-09-03T07:21:17.911 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dp6fY/0X081nOPpTJxR3yM99PfU= 2022-09-03T07:21:17.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dp6fY/0X081nOPpTJxR3yM99PfU=', 'Content-Length': '0'} 2022-09-03T07:21:17.915 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae2e38bc80679bd7-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.915 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:17.916 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:17.917 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.917 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.918 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.918 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.919 DEBUG:boto:Data: 2022-09-03T07:21:17.919 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.920 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:17.920 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.921 DEBUG:boto:Params: {} 2022-09-03T07:21:17.922 DEBUG:boto:Token: None 2022-09-03T07:21:17.922 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dp6fY/0X081nOPpTJxR3yM99PfU= 2022-09-03T07:21:17.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dp6fY/0X081nOPpTJxR3yM99PfU=', 'Content-Length': '0'} 2022-09-03T07:21:17.926 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d74d2c2339e85a18-006313006d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.926 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:21:17.927 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:17.928 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:17.928 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.929 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.929 DEBUG:boto:Method: GET 2022-09-03T07:21:17.930 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.930 DEBUG:boto:Data: 2022-09-03T07:21:17.931 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.932 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.932 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.933 DEBUG:boto:Params: {} 2022-09-03T07:21:17.933 DEBUG:boto:Token: None 2022-09-03T07:21:17.934 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:48pPH5pqAGxpn30JUbdP1V8VbXI= 2022-09-03T07:21:17.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:48pPH5pqAGxpn30JUbdP1V8VbXI=', 'Content-Length': '0'} 2022-09-03T07:21:17.938 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055b0a015a6faa534-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.939 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:17.939 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:17.940 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.940 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.941 DEBUG:boto:Method: GET 2022-09-03T07:21:17.941 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.942 DEBUG:boto:Data: 2022-09-03T07:21:17.942 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.943 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:17.944 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.944 DEBUG:boto:Params: {} 2022-09-03T07:21:17.945 DEBUG:boto:Token: None 2022-09-03T07:21:17.946 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:21:17.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:48pPH5pqAGxpn30JUbdP1V8VbXI= 2022-09-03T07:21:17.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:48pPH5pqAGxpn30JUbdP1V8VbXI=', 'Content-Length': '0'} 2022-09-03T07:21:17.949 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e0d46c3a30ccaf84-006313006d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.950 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.951 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.951 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:17.952 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:17.953 DEBUG:boto:Method: GET 2022-09-03T07:21:17.953 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:17.954 DEBUG:boto:Data: 2022-09-03T07:21:17.954 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.955 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.956 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.956 DEBUG:boto:Params: {} 2022-09-03T07:21:17.957 DEBUG:boto:Token: None 2022-09-03T07:21:17.957 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.961 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001122a375ada8b3b0-006313006d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.962 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.963 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:17.963 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:17.964 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:17.965 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:17.965 DEBUG:boto:Method: GET 2022-09-03T07:21:17.966 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:17.966 DEBUG:boto:Data: 2022-09-03T07:21:17.967 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.967 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:17.968 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.968 DEBUG:boto:Params: {} 2022-09-03T07:21:17.969 DEBUG:boto:Token: None 2022-09-03T07:21:17.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/ 2022-09-03T07:21:17.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY= 2022-09-03T07:21:17.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lcT4Gidxm3Xa8Q2IIOaRJUZcMYY=', 'Content-Length': '0'} 2022-09-03T07:21:17.973 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000082a7b7fd97a43956-006313006d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.974 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:17.974 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:17.975 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:17.976 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.976 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:17.977 DEBUG:boto:Data: 2022-09-03T07:21:17.977 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.978 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.979 DEBUG:boto:Params: {} 2022-09-03T07:21:17.979 DEBUG:boto:Token: None 2022-09-03T07:21:17.980 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj11 2022-09-03T07:21:17.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z4xFzxW1UVGMv4R37zGaa8JkJUY= 2022-09-03T07:21:17.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z4xFzxW1UVGMv4R37zGaa8JkJUY=', 'Content-Length': '0'} 2022-09-03T07:21:17.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004bee341357825c9e-006313006d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.984 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:17.984 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:17.985 DEBUG:boto:Method: HEAD 2022-09-03T07:21:17.985 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:17.986 DEBUG:boto:Data: 2022-09-03T07:21:17.987 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.987 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:17.988 DEBUG:boto:Port: 8000 2022-09-03T07:21:17.989 DEBUG:boto:Params: {} 2022-09-03T07:21:17.989 DEBUG:boto:Token: None 2022-09-03T07:21:17.990 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:17 GMT /avysmw-75/obj11 2022-09-03T07:21:17.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z4xFzxW1UVGMv4R37zGaa8JkJUY= 2022-09-03T07:21:17.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z4xFzxW1UVGMv4R37zGaa8JkJUY=', 'Content-Length': '0'} 2022-09-03T07:21:17.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007971161f7904486a-006313006d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:17.994 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:21:17.994 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:17.995 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:17.996 DEBUG:boto:Method: GET 2022-09-03T07:21:17.996 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:17.997 DEBUG:boto:Data: 2022-09-03T07:21:17.997 DEBUG:boto:Headers: {} 2022-09-03T07:21:17.998 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:17.999 DEBUG:boto:Port: 8001 2022-09-03T07:21:17.999 DEBUG:boto:Params: {} 2022-09-03T07:21:18.000 DEBUG:boto:Token: None 2022-09-03T07:21:18.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj11 2022-09-03T07:21:18.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1RjcFgC4prJOJqADJDtYDm1sHMU= 2022-09-03T07:21:18.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1RjcFgC4prJOJqADJDtYDm1sHMU=', 'Content-Length': '0'} 2022-09-03T07:21:18.004 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090b293a4352df95e-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.005 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:21:18.005 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:21:18.006 DEBUG:boto:Method: GET 2022-09-03T07:21:18.007 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:21:18.007 DEBUG:boto:Data: 2022-09-03T07:21:18.008 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.008 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.009 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.009 DEBUG:boto:Params: {} 2022-09-03T07:21:18.010 DEBUG:boto:Token: None 2022-09-03T07:21:18.011 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj11 2022-09-03T07:21:18.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1RjcFgC4prJOJqADJDtYDm1sHMU= 2022-09-03T07:21:18.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1RjcFgC4prJOJqADJDtYDm1sHMU=', 'Content-Length': '0'} 2022-09-03T07:21:18.015 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bf39c0fdd47fd01-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.015 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.016 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.017 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:18.017 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:18.018 DEBUG:boto:Method: GET 2022-09-03T07:21:18.018 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:18.019 DEBUG:boto:Data: 2022-09-03T07:21:18.020 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.021 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.021 DEBUG:boto:Params: {} 2022-09-03T07:21:18.022 DEBUG:boto:Token: None 2022-09-03T07:21:18.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.027 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000182e80cc495a3b11-006313006e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.028 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.029 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.029 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.030 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:18.030 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:21:18.031 DEBUG:boto:Method: GET 2022-09-03T07:21:18.031 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:21:18.032 DEBUG:boto:Data: 2022-09-03T07:21:18.032 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.033 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.033 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.034 DEBUG:boto:Params: {} 2022-09-03T07:21:18.035 DEBUG:boto:Token: None 2022-09-03T07:21:18.035 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.039 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5281619a0e8db00-006313006e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.040 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.040 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:21:18.041 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.041 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.042 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.043 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.043 DEBUG:boto:Method: GET 2022-09-03T07:21:18.044 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.044 DEBUG:boto:Data: 2022-09-03T07:21:18.045 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.046 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.047 DEBUG:boto:Params: {} 2022-09-03T07:21:18.047 DEBUG:boto:Token: None 2022-09-03T07:21:18.048 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oNVmp54toQyP9ifCB3um+shV4jI= 2022-09-03T07:21:18.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oNVmp54toQyP9ifCB3um+shV4jI=', 'Content-Length': '0'} 2022-09-03T07:21:18.051 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bad57cf340a985b8-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.094 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.095 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.095 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.096 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.096 DEBUG:boto:Method: GET 2022-09-03T07:21:18.097 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.098 DEBUG:boto:Data: 2022-09-03T07:21:18.098 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.099 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.099 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.100 DEBUG:boto:Params: {} 2022-09-03T07:21:18.100 DEBUG:boto:Token: None 2022-09-03T07:21:18.101 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oNVmp54toQyP9ifCB3um+shV4jI= 2022-09-03T07:21:18.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oNVmp54toQyP9ifCB3um+shV4jI=', 'Content-Length': '0'} 2022-09-03T07:21:18.104 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007b4fcb36fa0fa9a0-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.105 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.106 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.106 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.107 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.108 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.108 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.109 DEBUG:boto:Data: 2022-09-03T07:21:18.109 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.110 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.111 DEBUG:boto:Params: {} 2022-09-03T07:21:18.111 DEBUG:boto:Token: None 2022-09-03T07:21:18.112 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tx8scGcU2r+l6pcgiKw0I+Ta794= 2022-09-03T07:21:18.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tx8scGcU2r+l6pcgiKw0I+Ta794=', 'Content-Length': '0'} 2022-09-03T07:21:18.116 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000954e3592ae081a2f-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.116 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.117 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.117 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.118 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.118 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.119 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.120 DEBUG:boto:Data: 2022-09-03T07:21:18.120 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.121 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.121 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.122 DEBUG:boto:Params: {} 2022-09-03T07:21:18.122 DEBUG:boto:Token: None 2022-09-03T07:21:18.123 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.123 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tx8scGcU2r+l6pcgiKw0I+Ta794= 2022-09-03T07:21:18.124 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tx8scGcU2r+l6pcgiKw0I+Ta794=', 'Content-Length': '0'} 2022-09-03T07:21:18.126 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003984a2f5ffbbbb67-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.127 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:21:18.127 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.128 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.128 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.129 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.129 DEBUG:boto:Method: GET 2022-09-03T07:21:18.130 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.131 DEBUG:boto:Data: 2022-09-03T07:21:18.131 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.132 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.133 DEBUG:boto:Params: {} 2022-09-03T07:21:18.133 DEBUG:boto:Token: None 2022-09-03T07:21:18.134 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oNVmp54toQyP9ifCB3um+shV4jI= 2022-09-03T07:21:18.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oNVmp54toQyP9ifCB3um+shV4jI=', 'Content-Length': '0'} 2022-09-03T07:21:18.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000724330ebe976a5aa-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.138 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.139 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.140 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.140 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.141 DEBUG:boto:Method: GET 2022-09-03T07:21:18.141 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.142 DEBUG:boto:Data: 2022-09-03T07:21:18.142 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.143 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.143 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.144 DEBUG:boto:Params: {} 2022-09-03T07:21:18.145 DEBUG:boto:Token: None 2022-09-03T07:21:18.145 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:21:18.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oNVmp54toQyP9ifCB3um+shV4jI= 2022-09-03T07:21:18.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oNVmp54toQyP9ifCB3um+shV4jI=', 'Content-Length': '0'} 2022-09-03T07:21:18.149 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c97024e59d4f79f3-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.149 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.150 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.151 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:18.151 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:18.152 DEBUG:boto:Method: GET 2022-09-03T07:21:18.152 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:18.153 DEBUG:boto:Data: 2022-09-03T07:21:18.153 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.154 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.154 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.155 DEBUG:boto:Params: {} 2022-09-03T07:21:18.155 DEBUG:boto:Token: None 2022-09-03T07:21:18.156 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.160 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db6cf45c2e8d69cd-006313006e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.161 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.163 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.163 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.164 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:18.165 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:18.166 DEBUG:boto:Method: GET 2022-09-03T07:21:18.166 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:18.167 DEBUG:boto:Data: 2022-09-03T07:21:18.167 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.168 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.168 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.169 DEBUG:boto:Params: {} 2022-09-03T07:21:18.169 DEBUG:boto:Token: None 2022-09-03T07:21:18.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.174 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b481a31583efeb0e-006313006e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.174 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.175 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.176 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.176 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.177 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:18.177 DEBUG:boto:Data: 2022-09-03T07:21:18.178 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.179 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.179 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.180 DEBUG:boto:Params: {} 2022-09-03T07:21:18.180 DEBUG:boto:Token: None 2022-09-03T07:21:18.181 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17 2022-09-03T07:21:18.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gYHgBGnOw5FjtcxcnT9VNfZwIs8= 2022-09-03T07:21:18.183 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gYHgBGnOw5FjtcxcnT9VNfZwIs8=', 'Content-Length': '0'} 2022-09-03T07:21:18.184 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026b8cd3ac270ed6c-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.185 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.186 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.186 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.187 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:18.187 DEBUG:boto:Data: 2022-09-03T07:21:18.188 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.188 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.189 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.189 DEBUG:boto:Params: {} 2022-09-03T07:21:18.190 DEBUG:boto:Token: None 2022-09-03T07:21:18.191 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17 2022-09-03T07:21:18.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gYHgBGnOw5FjtcxcnT9VNfZwIs8= 2022-09-03T07:21:18.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gYHgBGnOw5FjtcxcnT9VNfZwIs8=', 'Content-Length': '0'} 2022-09-03T07:21:18.194 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000067490841ba57999e-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.195 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:21:18.196 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.196 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.197 DEBUG:boto:Method: GET 2022-09-03T07:21:18.197 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:18.198 DEBUG:boto:Data: 2022-09-03T07:21:18.198 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.199 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.199 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.200 DEBUG:boto:Params: {} 2022-09-03T07:21:18.201 DEBUG:boto:Token: None 2022-09-03T07:21:18.201 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17 2022-09-03T07:21:18.202 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cF9r0VqQMIZjjZ5FAvJA3gZTIgc= 2022-09-03T07:21:18.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cF9r0VqQMIZjjZ5FAvJA3gZTIgc=', 'Content-Length': '0'} 2022-09-03T07:21:18.205 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c0de13896455f08-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.205 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:21:18.206 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:21:18.207 DEBUG:boto:Method: GET 2022-09-03T07:21:18.207 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:21:18.208 DEBUG:boto:Data: 2022-09-03T07:21:18.208 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.209 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.209 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.210 DEBUG:boto:Params: {} 2022-09-03T07:21:18.210 DEBUG:boto:Token: None 2022-09-03T07:21:18.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj17 2022-09-03T07:21:18.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cF9r0VqQMIZjjZ5FAvJA3gZTIgc= 2022-09-03T07:21:18.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cF9r0VqQMIZjjZ5FAvJA3gZTIgc=', 'Content-Length': '0'} 2022-09-03T07:21:18.214 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d107afe95b5d33d-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.258 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.259 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.260 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:18.260 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:18.261 DEBUG:boto:Method: GET 2022-09-03T07:21:18.262 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:18.263 DEBUG:boto:Data: 2022-09-03T07:21:18.263 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.264 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.265 DEBUG:boto:Params: {} 2022-09-03T07:21:18.265 DEBUG:boto:Token: None 2022-09-03T07:21:18.266 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.270 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000099cdc683c2135776-006313006e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.271 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.271 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.272 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.273 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:18.273 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:21:18.274 DEBUG:boto:Method: GET 2022-09-03T07:21:18.274 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:21:18.275 DEBUG:boto:Data: 2022-09-03T07:21:18.275 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.276 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.276 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.277 DEBUG:boto:Params: {} 2022-09-03T07:21:18.277 DEBUG:boto:Token: None 2022-09-03T07:21:18.278 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.282 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dfd3718a663c279c-006313006e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.283 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.283 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:21:18.284 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.284 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.285 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.285 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.286 DEBUG:boto:Method: GET 2022-09-03T07:21:18.287 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.287 DEBUG:boto:Data: 2022-09-03T07:21:18.288 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.289 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.289 DEBUG:boto:Params: {} 2022-09-03T07:21:18.290 DEBUG:boto:Token: None 2022-09-03T07:21:18.290 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XDOYdKrA3h1GtIqrstB4sK4Z+6c= 2022-09-03T07:21:18.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XDOYdKrA3h1GtIqrstB4sK4Z+6c=', 'Content-Length': '0'} 2022-09-03T07:21:18.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000961dae1394aab1a8-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.296 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.297 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.297 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.298 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.298 DEBUG:boto:Method: GET 2022-09-03T07:21:18.299 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.299 DEBUG:boto:Data: 2022-09-03T07:21:18.300 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.301 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.301 DEBUG:boto:Params: {} 2022-09-03T07:21:18.302 DEBUG:boto:Token: None 2022-09-03T07:21:18.303 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XDOYdKrA3h1GtIqrstB4sK4Z+6c= 2022-09-03T07:21:18.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XDOYdKrA3h1GtIqrstB4sK4Z+6c=', 'Content-Length': '0'} 2022-09-03T07:21:18.306 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000086d7055f01ec527-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.307 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.308 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.309 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.309 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.310 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.310 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.311 DEBUG:boto:Data: 2022-09-03T07:21:18.311 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.312 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.313 DEBUG:boto:Params: {} 2022-09-03T07:21:18.313 DEBUG:boto:Token: None 2022-09-03T07:21:18.314 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:04fatO/tFCzAzL2aesbze6ejbiY= 2022-09-03T07:21:18.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:04fatO/tFCzAzL2aesbze6ejbiY=', 'Content-Length': '0'} 2022-09-03T07:21:18.318 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3c2d9e7abd83f96-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.318 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.319 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.319 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.320 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.320 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.321 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.321 DEBUG:boto:Data: 2022-09-03T07:21:18.322 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.322 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.323 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.323 DEBUG:boto:Params: {} 2022-09-03T07:21:18.324 DEBUG:boto:Token: None 2022-09-03T07:21:18.324 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:04fatO/tFCzAzL2aesbze6ejbiY= 2022-09-03T07:21:18.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:04fatO/tFCzAzL2aesbze6ejbiY=', 'Content-Length': '0'} 2022-09-03T07:21:18.327 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e616b2b4200ae4c-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.328 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:21:18.329 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.329 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.330 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.330 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.331 DEBUG:boto:Method: GET 2022-09-03T07:21:18.331 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.332 DEBUG:boto:Data: 2022-09-03T07:21:18.332 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.333 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.333 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.334 DEBUG:boto:Params: {} 2022-09-03T07:21:18.334 DEBUG:boto:Token: None 2022-09-03T07:21:18.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XDOYdKrA3h1GtIqrstB4sK4Z+6c= 2022-09-03T07:21:18.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XDOYdKrA3h1GtIqrstB4sK4Z+6c=', 'Content-Length': '0'} 2022-09-03T07:21:18.338 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042b11e4ce9b5329d-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.382 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.383 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.383 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.384 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.384 DEBUG:boto:Method: GET 2022-09-03T07:21:18.385 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.385 DEBUG:boto:Data: 2022-09-03T07:21:18.386 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.386 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.387 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.388 DEBUG:boto:Params: {} 2022-09-03T07:21:18.388 DEBUG:boto:Token: None 2022-09-03T07:21:18.389 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:21:18.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XDOYdKrA3h1GtIqrstB4sK4Z+6c= 2022-09-03T07:21:18.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XDOYdKrA3h1GtIqrstB4sK4Z+6c=', 'Content-Length': '0'} 2022-09-03T07:21:18.392 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000327e2a35a8124b0d-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.393 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.393 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.394 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:18.394 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:18.395 DEBUG:boto:Method: GET 2022-09-03T07:21:18.395 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:18.396 DEBUG:boto:Data: 2022-09-03T07:21:18.397 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.397 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.398 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.398 DEBUG:boto:Params: {} 2022-09-03T07:21:18.399 DEBUG:boto:Token: None 2022-09-03T07:21:18.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.403 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005509f6802e066078-006313006e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.404 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.405 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.405 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.406 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:18.406 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:18.407 DEBUG:boto:Method: GET 2022-09-03T07:21:18.407 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:18.408 DEBUG:boto:Data: 2022-09-03T07:21:18.408 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.409 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.409 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.410 DEBUG:boto:Params: {} 2022-09-03T07:21:18.410 DEBUG:boto:Token: None 2022-09-03T07:21:18.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.415 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a5b65dca38d4ea2-006313006e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.415 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.416 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.416 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.417 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.417 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:18.418 DEBUG:boto:Data: 2022-09-03T07:21:18.418 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.419 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.420 DEBUG:boto:Params: {} 2022-09-03T07:21:18.421 DEBUG:boto:Token: None 2022-09-03T07:21:18.421 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19 2022-09-03T07:21:18.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lTeU+Gk/Oocwl86u6Sofx/ys0Es= 2022-09-03T07:21:18.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lTeU+Gk/Oocwl86u6Sofx/ys0Es=', 'Content-Length': '0'} 2022-09-03T07:21:18.424 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b26e6cf815b12be6-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.425 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.426 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.426 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.427 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:18.427 DEBUG:boto:Data: 2022-09-03T07:21:18.428 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.428 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.429 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.429 DEBUG:boto:Params: {} 2022-09-03T07:21:18.430 DEBUG:boto:Token: None 2022-09-03T07:21:18.430 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19 2022-09-03T07:21:18.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lTeU+Gk/Oocwl86u6Sofx/ys0Es= 2022-09-03T07:21:18.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lTeU+Gk/Oocwl86u6Sofx/ys0Es=', 'Content-Length': '0'} 2022-09-03T07:21:18.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0eecb76d7287eab-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.434 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:21:18.434 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.435 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.435 DEBUG:boto:Method: GET 2022-09-03T07:21:18.436 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:18.436 DEBUG:boto:Data: 2022-09-03T07:21:18.437 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.437 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.438 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.438 DEBUG:boto:Params: {} 2022-09-03T07:21:18.439 DEBUG:boto:Token: None 2022-09-03T07:21:18.440 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19 2022-09-03T07:21:18.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vjsUT6OCyOyRoTqTC2OYvrQ/VoQ= 2022-09-03T07:21:18.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vjsUT6OCyOyRoTqTC2OYvrQ/VoQ=', 'Content-Length': '0'} 2022-09-03T07:21:18.443 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7216b184979ebbd-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.444 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:21:18.444 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:21:18.445 DEBUG:boto:Method: GET 2022-09-03T07:21:18.445 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:21:18.446 DEBUG:boto:Data: 2022-09-03T07:21:18.446 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.447 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.447 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.448 DEBUG:boto:Params: {} 2022-09-03T07:21:18.448 DEBUG:boto:Token: None 2022-09-03T07:21:18.449 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj19 2022-09-03T07:21:18.450 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vjsUT6OCyOyRoTqTC2OYvrQ/VoQ= 2022-09-03T07:21:18.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vjsUT6OCyOyRoTqTC2OYvrQ/VoQ=', 'Content-Length': '0'} 2022-09-03T07:21:18.452 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074f91adb48c1d62d-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.453 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.454 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.454 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:18.455 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:18.455 DEBUG:boto:Method: GET 2022-09-03T07:21:18.456 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:18.456 DEBUG:boto:Data: 2022-09-03T07:21:18.457 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.457 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.458 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.458 DEBUG:boto:Params: {} 2022-09-03T07:21:18.459 DEBUG:boto:Token: None 2022-09-03T07:21:18.459 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.460 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.461 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.463 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a861774a8c3b19e-006313006e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.506 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.507 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.508 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.508 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:18.509 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:21:18.509 DEBUG:boto:Method: GET 2022-09-03T07:21:18.510 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:21:18.510 DEBUG:boto:Data: 2022-09-03T07:21:18.511 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.511 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.512 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.512 DEBUG:boto:Params: {} 2022-09-03T07:21:18.513 DEBUG:boto:Token: None 2022-09-03T07:21:18.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.517 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000074be296e8e2d456e-006313006e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.518 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.519 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:21:18.519 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.520 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.521 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.521 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.522 DEBUG:boto:Method: GET 2022-09-03T07:21:18.522 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.523 DEBUG:boto:Data: 2022-09-03T07:21:18.523 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.524 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.525 DEBUG:boto:Params: {} 2022-09-03T07:21:18.526 DEBUG:boto:Token: None 2022-09-03T07:21:18.526 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4xmmKarSSvbwc1D6H8ByBu4tGrU= 2022-09-03T07:21:18.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4xmmKarSSvbwc1D6H8ByBu4tGrU=', 'Content-Length': '0'} 2022-09-03T07:21:18.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e64aedf4e970e62d-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.531 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.531 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.532 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.532 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.533 DEBUG:boto:Method: GET 2022-09-03T07:21:18.533 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.534 DEBUG:boto:Data: 2022-09-03T07:21:18.534 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.535 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.535 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.536 DEBUG:boto:Params: {} 2022-09-03T07:21:18.537 DEBUG:boto:Token: None 2022-09-03T07:21:18.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4xmmKarSSvbwc1D6H8ByBu4tGrU= 2022-09-03T07:21:18.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4xmmKarSSvbwc1D6H8ByBu4tGrU=', 'Content-Length': '0'} 2022-09-03T07:21:18.540 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ace9e10fa9c8bb88-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.541 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.542 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.542 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.543 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.543 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.544 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.544 DEBUG:boto:Data: 2022-09-03T07:21:18.545 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.546 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.546 DEBUG:boto:Params: {} 2022-09-03T07:21:18.547 DEBUG:boto:Token: None 2022-09-03T07:21:18.547 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.548 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NZIUrWmaeNMy9sCuAGvpYR0vGP4= 2022-09-03T07:21:18.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NZIUrWmaeNMy9sCuAGvpYR0vGP4=', 'Content-Length': '0'} 2022-09-03T07:21:18.550 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f244b50b309b103-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.551 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.552 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.552 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.553 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.553 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.554 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.554 DEBUG:boto:Data: 2022-09-03T07:21:18.555 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.555 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.556 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.557 DEBUG:boto:Params: {} 2022-09-03T07:21:18.557 DEBUG:boto:Token: None 2022-09-03T07:21:18.558 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NZIUrWmaeNMy9sCuAGvpYR0vGP4= 2022-09-03T07:21:18.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NZIUrWmaeNMy9sCuAGvpYR0vGP4=', 'Content-Length': '0'} 2022-09-03T07:21:18.561 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c0de1e3369c1b48-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.562 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:21:18.562 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.563 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.563 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.564 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.565 DEBUG:boto:Method: GET 2022-09-03T07:21:18.565 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.566 DEBUG:boto:Data: 2022-09-03T07:21:18.566 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.567 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.568 DEBUG:boto:Params: {} 2022-09-03T07:21:18.568 DEBUG:boto:Token: None 2022-09-03T07:21:18.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4xmmKarSSvbwc1D6H8ByBu4tGrU= 2022-09-03T07:21:18.570 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4xmmKarSSvbwc1D6H8ByBu4tGrU=', 'Content-Length': '0'} 2022-09-03T07:21:18.573 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000423b19876326c08d-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.573 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.574 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.575 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.575 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.576 DEBUG:boto:Method: GET 2022-09-03T07:21:18.576 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.577 DEBUG:boto:Data: 2022-09-03T07:21:18.577 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.578 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.579 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.579 DEBUG:boto:Params: {} 2022-09-03T07:21:18.580 DEBUG:boto:Token: None 2022-09-03T07:21:18.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:21:18.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4xmmKarSSvbwc1D6H8ByBu4tGrU= 2022-09-03T07:21:18.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4xmmKarSSvbwc1D6H8ByBu4tGrU=', 'Content-Length': '0'} 2022-09-03T07:21:18.583 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de4921492d2fb03e-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.584 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.585 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.585 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:18.586 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:18.587 DEBUG:boto:Method: GET 2022-09-03T07:21:18.587 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:18.588 DEBUG:boto:Data: 2022-09-03T07:21:18.588 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.589 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.589 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.590 DEBUG:boto:Params: {} 2022-09-03T07:21:18.590 DEBUG:boto:Token: None 2022-09-03T07:21:18.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006843dbf0dd2b5835-006313006e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.596 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.597 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.597 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.598 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:18.598 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:18.599 DEBUG:boto:Method: GET 2022-09-03T07:21:18.599 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:18.600 DEBUG:boto:Data: 2022-09-03T07:21:18.601 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.601 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.602 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.602 DEBUG:boto:Params: {} 2022-09-03T07:21:18.603 DEBUG:boto:Token: None 2022-09-03T07:21:18.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.607 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb7ef01ba7939033-006313006e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.608 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.609 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.609 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.610 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.610 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:18.611 DEBUG:boto:Data: 2022-09-03T07:21:18.611 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.612 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.613 DEBUG:boto:Params: {} 2022-09-03T07:21:18.614 DEBUG:boto:Token: None 2022-09-03T07:21:18.614 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22 2022-09-03T07:21:18.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o08HnnwJx5HAWv/3uTEieBnJK4w= 2022-09-03T07:21:18.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o08HnnwJx5HAWv/3uTEieBnJK4w=', 'Content-Length': '0'} 2022-09-03T07:21:18.617 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a54cd6ed9a345fe5-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.618 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.619 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.619 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.620 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:18.620 DEBUG:boto:Data: 2022-09-03T07:21:18.621 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.621 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.622 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.622 DEBUG:boto:Params: {} 2022-09-03T07:21:18.623 DEBUG:boto:Token: None 2022-09-03T07:21:18.624 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22 2022-09-03T07:21:18.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o08HnnwJx5HAWv/3uTEieBnJK4w= 2022-09-03T07:21:18.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o08HnnwJx5HAWv/3uTEieBnJK4w=', 'Content-Length': '0'} 2022-09-03T07:21:18.627 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b4db23d66abbe67-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.627 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:21:18.628 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.628 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.629 DEBUG:boto:Method: GET 2022-09-03T07:21:18.630 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:18.630 DEBUG:boto:Data: 2022-09-03T07:21:18.631 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.632 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.632 DEBUG:boto:Params: {} 2022-09-03T07:21:18.633 DEBUG:boto:Token: None 2022-09-03T07:21:18.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22 2022-09-03T07:21:18.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xdDHiw4/Nt8ieD2gkiLEYmkQTBA= 2022-09-03T07:21:18.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xdDHiw4/Nt8ieD2gkiLEYmkQTBA=', 'Content-Length': '0'} 2022-09-03T07:21:18.637 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096066162a1aff366-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.678 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:21:18.679 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:21:18.680 DEBUG:boto:Method: GET 2022-09-03T07:21:18.680 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:21:18.681 DEBUG:boto:Data: 2022-09-03T07:21:18.681 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.682 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.682 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.683 DEBUG:boto:Params: {} 2022-09-03T07:21:18.683 DEBUG:boto:Token: None 2022-09-03T07:21:18.684 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj22 2022-09-03T07:21:18.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xdDHiw4/Nt8ieD2gkiLEYmkQTBA= 2022-09-03T07:21:18.685 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xdDHiw4/Nt8ieD2gkiLEYmkQTBA=', 'Content-Length': '0'} 2022-09-03T07:21:18.688 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008fe7864692a54307-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.689 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.689 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.690 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:18.690 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:18.691 DEBUG:boto:Method: GET 2022-09-03T07:21:18.691 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:18.692 DEBUG:boto:Data: 2022-09-03T07:21:18.692 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.693 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.694 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.694 DEBUG:boto:Params: {} 2022-09-03T07:21:18.695 DEBUG:boto:Token: None 2022-09-03T07:21:18.695 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.696 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.699 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075fa8d1450a006af-006313006e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.700 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.701 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.701 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.702 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:18.703 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:21:18.703 DEBUG:boto:Method: GET 2022-09-03T07:21:18.704 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:21:18.704 DEBUG:boto:Data: 2022-09-03T07:21:18.705 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.705 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.706 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.706 DEBUG:boto:Params: {} 2022-09-03T07:21:18.707 DEBUG:boto:Token: None 2022-09-03T07:21:18.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.712 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090e6c8c8fb774546-006313006e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.712 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.713 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:21:18.714 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.714 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.715 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.715 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.716 DEBUG:boto:Method: GET 2022-09-03T07:21:18.716 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.717 DEBUG:boto:Data: 2022-09-03T07:21:18.718 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.719 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.719 DEBUG:boto:Params: {} 2022-09-03T07:21:18.720 DEBUG:boto:Token: None 2022-09-03T07:21:18.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRifxmAiWDOqLVcpQ0jQak1T3xg= 2022-09-03T07:21:18.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRifxmAiWDOqLVcpQ0jQak1T3xg=', 'Content-Length': '0'} 2022-09-03T07:21:18.724 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab4205f24a234ace-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.725 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.725 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.726 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.726 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.727 DEBUG:boto:Method: GET 2022-09-03T07:21:18.728 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.728 DEBUG:boto:Data: 2022-09-03T07:21:18.729 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.729 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.730 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.730 DEBUG:boto:Params: {} 2022-09-03T07:21:18.731 DEBUG:boto:Token: None 2022-09-03T07:21:18.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRifxmAiWDOqLVcpQ0jQak1T3xg= 2022-09-03T07:21:18.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRifxmAiWDOqLVcpQ0jQak1T3xg=', 'Content-Length': '0'} 2022-09-03T07:21:18.735 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd4d39cb5ce93a66-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.736 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.736 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.737 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.737 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.738 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.738 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.739 DEBUG:boto:Data: 2022-09-03T07:21:18.739 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.740 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.740 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.741 DEBUG:boto:Params: {} 2022-09-03T07:21:18.741 DEBUG:boto:Token: None 2022-09-03T07:21:18.742 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oo1RZQ2PaeghUv5aVv3g6Xdn/yc= 2022-09-03T07:21:18.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oo1RZQ2PaeghUv5aVv3g6Xdn/yc=', 'Content-Length': '0'} 2022-09-03T07:21:18.745 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c334fa1c895fdf0-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.746 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.746 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.756 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.757 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.757 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.758 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.758 DEBUG:boto:Data: 2022-09-03T07:21:18.759 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.760 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.760 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.761 DEBUG:boto:Params: {} 2022-09-03T07:21:18.761 DEBUG:boto:Token: None 2022-09-03T07:21:18.762 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.763 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oo1RZQ2PaeghUv5aVv3g6Xdn/yc= 2022-09-03T07:21:18.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oo1RZQ2PaeghUv5aVv3g6Xdn/yc=', 'Content-Length': '0'} 2022-09-03T07:21:18.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001feccd706794566a-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.766 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:21:18.766 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.767 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.767 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.768 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.768 DEBUG:boto:Method: GET 2022-09-03T07:21:18.769 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.770 DEBUG:boto:Data: 2022-09-03T07:21:18.770 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.771 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.772 DEBUG:boto:Params: {} 2022-09-03T07:21:18.772 DEBUG:boto:Token: None 2022-09-03T07:21:18.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRifxmAiWDOqLVcpQ0jQak1T3xg= 2022-09-03T07:21:18.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRifxmAiWDOqLVcpQ0jQak1T3xg=', 'Content-Length': '0'} 2022-09-03T07:21:18.776 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e8d6b9412f0328b-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.777 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.778 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.778 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.779 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.779 DEBUG:boto:Method: GET 2022-09-03T07:21:18.780 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.780 DEBUG:boto:Data: 2022-09-03T07:21:18.781 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.781 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.782 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.782 DEBUG:boto:Params: {} 2022-09-03T07:21:18.783 DEBUG:boto:Token: None 2022-09-03T07:21:18.784 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:21:18.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRifxmAiWDOqLVcpQ0jQak1T3xg= 2022-09-03T07:21:18.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRifxmAiWDOqLVcpQ0jQak1T3xg=', 'Content-Length': '0'} 2022-09-03T07:21:18.787 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa41ed8f13a2ab2b-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.788 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.788 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.789 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:18.789 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:18.790 DEBUG:boto:Method: GET 2022-09-03T07:21:18.790 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:18.791 DEBUG:boto:Data: 2022-09-03T07:21:18.791 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.792 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.793 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.793 DEBUG:boto:Params: {} 2022-09-03T07:21:18.794 DEBUG:boto:Token: None 2022-09-03T07:21:18.794 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.798 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009d4f805427a8c2d4-006313006e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.799 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.800 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.800 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.801 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:18.802 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:18.802 DEBUG:boto:Method: GET 2022-09-03T07:21:18.803 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:18.803 DEBUG:boto:Data: 2022-09-03T07:21:18.804 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.804 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.805 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.805 DEBUG:boto:Params: {} 2022-09-03T07:21:18.806 DEBUG:boto:Token: None 2022-09-03T07:21:18.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.810 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b68f0f927a55404b-006313006e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.858 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.860 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.861 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.861 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.862 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:18.862 DEBUG:boto:Data: 2022-09-03T07:21:18.863 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.864 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.865 DEBUG:boto:Params: {} 2022-09-03T07:21:18.867 DEBUG:boto:Token: None 2022-09-03T07:21:18.868 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33 2022-09-03T07:21:18.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XFxQ9hsto85GU9/Mai3HkS2p59Y= 2022-09-03T07:21:18.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XFxQ9hsto85GU9/Mai3HkS2p59Y=', 'Content-Length': '0'} 2022-09-03T07:21:18.872 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000906be947779364d9-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.874 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.875 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.876 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.877 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:18.878 DEBUG:boto:Data: 2022-09-03T07:21:18.878 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.880 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.881 DEBUG:boto:Params: {} 2022-09-03T07:21:18.882 DEBUG:boto:Token: None 2022-09-03T07:21:18.883 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33 2022-09-03T07:21:18.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XFxQ9hsto85GU9/Mai3HkS2p59Y= 2022-09-03T07:21:18.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XFxQ9hsto85GU9/Mai3HkS2p59Y=', 'Content-Length': '0'} 2022-09-03T07:21:18.889 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f03c881b6b293301-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.890 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:21:18.891 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.892 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.893 DEBUG:boto:Method: GET 2022-09-03T07:21:18.894 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:18.895 DEBUG:boto:Data: 2022-09-03T07:21:18.896 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.897 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.898 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.899 DEBUG:boto:Params: {} 2022-09-03T07:21:18.900 DEBUG:boto:Token: None 2022-09-03T07:21:18.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33 2022-09-03T07:21:18.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:47fzNQ0gFIc8t+UPcjBEUQ784Es= 2022-09-03T07:21:18.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:47fzNQ0gFIc8t+UPcjBEUQ784Es=', 'Content-Length': '0'} 2022-09-03T07:21:18.906 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc62f7c53968d014-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.907 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:21:18.907 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:21:18.908 DEBUG:boto:Method: GET 2022-09-03T07:21:18.908 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:21:18.909 DEBUG:boto:Data: 2022-09-03T07:21:18.909 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.910 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.910 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.911 DEBUG:boto:Params: {} 2022-09-03T07:21:18.912 DEBUG:boto:Token: None 2022-09-03T07:21:18.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj33 2022-09-03T07:21:18.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:47fzNQ0gFIc8t+UPcjBEUQ784Es= 2022-09-03T07:21:18.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:47fzNQ0gFIc8t+UPcjBEUQ784Es=', 'Content-Length': '0'} 2022-09-03T07:21:18.915 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9e5653d1f218ebe-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.916 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.917 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.918 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:18.918 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:18.919 DEBUG:boto:Method: GET 2022-09-03T07:21:18.920 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:18.920 DEBUG:boto:Data: 2022-09-03T07:21:18.921 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.922 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.922 DEBUG:boto:Params: {} 2022-09-03T07:21:18.923 DEBUG:boto:Token: None 2022-09-03T07:21:18.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.929 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aef3be8fccfb8645-006313006e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.929 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.930 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:18.931 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:18.932 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:18.932 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:21:18.933 DEBUG:boto:Method: GET 2022-09-03T07:21:18.933 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:21:18.934 DEBUG:boto:Data: 2022-09-03T07:21:18.934 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.935 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.936 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.936 DEBUG:boto:Params: {} 2022-09-03T07:21:18.937 DEBUG:boto:Token: None 2022-09-03T07:21:18.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/ 2022-09-03T07:21:18.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4= 2022-09-03T07:21:18.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dctKhNdCHER3nJbBbQtGzjRWlc4=', 'Content-Length': '0'} 2022-09-03T07:21:18.941 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000aace97691736c5a-006313006e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.942 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:18.943 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:21:18.943 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:18.944 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:18.945 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.945 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.946 DEBUG:boto:Method: GET 2022-09-03T07:21:18.946 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.947 DEBUG:boto:Data: 2022-09-03T07:21:18.947 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.948 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.949 DEBUG:boto:Params: {} 2022-09-03T07:21:18.950 DEBUG:boto:Token: None 2022-09-03T07:21:18.950 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XBl+FADPbpkT1+OiZOW/k/aB/aY= 2022-09-03T07:21:18.952 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XBl+FADPbpkT1+OiZOW/k/aB/aY=', 'Content-Length': '0'} 2022-09-03T07:21:18.954 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eaf45a38f4d1029d-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.955 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:18.955 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:18.956 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.956 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.957 DEBUG:boto:Method: GET 2022-09-03T07:21:18.958 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.959 DEBUG:boto:Data: 2022-09-03T07:21:18.959 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.960 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.961 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.961 DEBUG:boto:Params: {} 2022-09-03T07:21:18.962 DEBUG:boto:Token: None 2022-09-03T07:21:18.962 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XBl+FADPbpkT1+OiZOW/k/aB/aY= 2022-09-03T07:21:18.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XBl+FADPbpkT1+OiZOW/k/aB/aY=', 'Content-Length': '0'} 2022-09-03T07:21:18.966 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089ecee0e1c505310-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.967 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:18.968 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:18.969 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.970 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.971 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.972 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.973 DEBUG:boto:Data: 2022-09-03T07:21:18.973 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:18.975 DEBUG:boto:Port: 8001 2022-09-03T07:21:18.976 DEBUG:boto:Params: {} 2022-09-03T07:21:18.977 DEBUG:boto:Token: None 2022-09-03T07:21:18.978 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tyFzfDfo4qIYctsAPNi3gOx36Yo= 2022-09-03T07:21:18.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tyFzfDfo4qIYctsAPNi3gOx36Yo=', 'Content-Length': '0'} 2022-09-03T07:21:18.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000167c1732ffbe2674-006313006e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.984 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:18.985 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:18.986 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.986 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.987 DEBUG:boto:Method: HEAD 2022-09-03T07:21:18.987 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.988 DEBUG:boto:Data: 2022-09-03T07:21:18.989 DEBUG:boto:Headers: {} 2022-09-03T07:21:18.990 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:18.990 DEBUG:boto:Port: 8000 2022-09-03T07:21:18.991 DEBUG:boto:Params: {} 2022-09-03T07:21:18.991 DEBUG:boto:Token: None 2022-09-03T07:21:18.992 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:18 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:18.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tyFzfDfo4qIYctsAPNi3gOx36Yo= 2022-09-03T07:21:18.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tyFzfDfo4qIYctsAPNi3gOx36Yo=', 'Content-Length': '0'} 2022-09-03T07:21:18.998 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087e60e8381efd373-006313006e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:18.998 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:21:18.999 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:19.000 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:19.000 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:19.001 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:19.001 DEBUG:boto:Method: GET 2022-09-03T07:21:19.002 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:19.002 DEBUG:boto:Data: 2022-09-03T07:21:19.003 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.004 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.004 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.005 DEBUG:boto:Params: {} 2022-09-03T07:21:19.005 DEBUG:boto:Token: None 2022-09-03T07:21:19.006 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:19.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vTHQXB9l7pwaB6rLTSphTWDGgMM= 2022-09-03T07:21:19.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vTHQXB9l7pwaB6rLTSphTWDGgMM=', 'Content-Length': '0'} 2022-09-03T07:21:19.009 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4360b0e5f4ab5cb-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.010 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:19.011 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:19.011 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:19.012 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:21:19.012 DEBUG:boto:Method: GET 2022-09-03T07:21:19.013 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:21:19.013 DEBUG:boto:Data: 2022-09-03T07:21:19.014 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.015 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.015 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.016 DEBUG:boto:Params: {} 2022-09-03T07:21:19.017 DEBUG:boto:Token: None 2022-09-03T07:21:19.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:21:19.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vTHQXB9l7pwaB6rLTSphTWDGgMM= 2022-09-03T07:21:19.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vTHQXB9l7pwaB6rLTSphTWDGgMM=', 'Content-Length': '0'} 2022-09-03T07:21:19.020 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007802c90e1db20b63-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.021 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.022 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.023 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:19.023 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:19.024 DEBUG:boto:Method: GET 2022-09-03T07:21:19.024 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:19.025 DEBUG:boto:Data: 2022-09-03T07:21:19.025 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.026 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.027 DEBUG:boto:Params: {} 2022-09-03T07:21:19.027 DEBUG:boto:Token: None 2022-09-03T07:21:19.028 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.032 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fe037a85b4b7d36c-006313006f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.032 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.033 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.034 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.034 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:19.035 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:19.035 DEBUG:boto:Method: GET 2022-09-03T07:21:19.036 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:19.036 DEBUG:boto:Data: 2022-09-03T07:21:19.037 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.037 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.038 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.039 DEBUG:boto:Params: {} 2022-09-03T07:21:19.039 DEBUG:boto:Token: None 2022-09-03T07:21:19.040 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.043 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003a5f0818009d5e1b-006313006f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.044 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.044 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:19.045 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:19.045 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.046 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:19.046 DEBUG:boto:Data: 2022-09-03T07:21:19.047 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.048 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.048 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.049 DEBUG:boto:Params: {} 2022-09-03T07:21:19.049 DEBUG:boto:Token: None 2022-09-03T07:21:19.050 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj34 2022-09-03T07:21:19.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YSjs7kigmqSy54edjCANY9oKYx0= 2022-09-03T07:21:19.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YSjs7kigmqSy54edjCANY9oKYx0=', 'Content-Length': '0'} 2022-09-03T07:21:19.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f54206f52ea9712c-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.054 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:19.054 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:19.055 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.055 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:19.056 DEBUG:boto:Data: 2022-09-03T07:21:19.056 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.057 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.057 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.058 DEBUG:boto:Params: {} 2022-09-03T07:21:19.058 DEBUG:boto:Token: None 2022-09-03T07:21:19.059 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj34 2022-09-03T07:21:19.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YSjs7kigmqSy54edjCANY9oKYx0= 2022-09-03T07:21:19.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YSjs7kigmqSy54edjCANY9oKYx0=', 'Content-Length': '0'} 2022-09-03T07:21:19.062 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053dbced0385fd662-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.063 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:21:19.064 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:19.064 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:19.065 DEBUG:boto:Method: GET 2022-09-03T07:21:19.065 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:19.066 DEBUG:boto:Data: 2022-09-03T07:21:19.066 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.067 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.068 DEBUG:boto:Params: {} 2022-09-03T07:21:19.068 DEBUG:boto:Token: None 2022-09-03T07:21:19.069 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj34 2022-09-03T07:21:19.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+/ZFbTZHHBZTi+TUcvXksIdiLMQ= 2022-09-03T07:21:19.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+/ZFbTZHHBZTi+TUcvXksIdiLMQ=', 'Content-Length': '0'} 2022-09-03T07:21:19.072 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d1a2a4d5d3608f61-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.073 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:21:19.074 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:21:19.074 DEBUG:boto:Method: GET 2022-09-03T07:21:19.075 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:21:19.075 DEBUG:boto:Data: 2022-09-03T07:21:19.076 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.076 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.077 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.077 DEBUG:boto:Params: {} 2022-09-03T07:21:19.078 DEBUG:boto:Token: None 2022-09-03T07:21:19.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj34 2022-09-03T07:21:19.079 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+/ZFbTZHHBZTi+TUcvXksIdiLMQ= 2022-09-03T07:21:19.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+/ZFbTZHHBZTi+TUcvXksIdiLMQ=', 'Content-Length': '0'} 2022-09-03T07:21:19.082 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062b65ae5593b22b0-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.122 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.123 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.124 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:19.125 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:19.126 DEBUG:boto:Method: GET 2022-09-03T07:21:19.126 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:19.127 DEBUG:boto:Data: 2022-09-03T07:21:19.128 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.129 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.129 DEBUG:boto:Params: {} 2022-09-03T07:21:19.130 DEBUG:boto:Token: None 2022-09-03T07:21:19.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090ab126cdbe37a71-006313006f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.135 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.137 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.138 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.139 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:19.140 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:21:19.141 DEBUG:boto:Method: GET 2022-09-03T07:21:19.141 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:21:19.142 DEBUG:boto:Data: 2022-09-03T07:21:19.143 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.144 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.145 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.145 DEBUG:boto:Params: {} 2022-09-03T07:21:19.146 DEBUG:boto:Token: None 2022-09-03T07:21:19.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.148 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.150 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000819253fed549d383-006313006f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.151 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.152 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:21:19.153 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.153 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.154 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.155 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.155 DEBUG:boto:Method: GET 2022-09-03T07:21:19.156 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.156 DEBUG:boto:Data: 2022-09-03T07:21:19.157 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.157 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.158 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.159 DEBUG:boto:Params: {} 2022-09-03T07:21:19.159 DEBUG:boto:Token: None 2022-09-03T07:21:19.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jXtd3aJLiOwPJ6Ce1fwf4cotyTw= 2022-09-03T07:21:19.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jXtd3aJLiOwPJ6Ce1fwf4cotyTw=', 'Content-Length': '0'} 2022-09-03T07:21:19.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c694b1997e06f7f-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.206 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.207 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.208 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.209 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.211 DEBUG:boto:Method: GET 2022-09-03T07:21:19.212 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.213 DEBUG:boto:Data: 2022-09-03T07:21:19.214 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.215 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.215 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.216 DEBUG:boto:Params: {} 2022-09-03T07:21:19.216 DEBUG:boto:Token: None 2022-09-03T07:21:19.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jXtd3aJLiOwPJ6Ce1fwf4cotyTw= 2022-09-03T07:21:19.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jXtd3aJLiOwPJ6Ce1fwf4cotyTw=', 'Content-Length': '0'} 2022-09-03T07:21:19.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000085f6388cb81e9310-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.221 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.222 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.223 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.223 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.224 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.225 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.225 DEBUG:boto:Data: 2022-09-03T07:21:19.226 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.226 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.227 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.227 DEBUG:boto:Params: {} 2022-09-03T07:21:19.228 DEBUG:boto:Token: None 2022-09-03T07:21:19.228 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iEvFCikabG7ssWkYxWCgwtj7rTM= 2022-09-03T07:21:19.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iEvFCikabG7ssWkYxWCgwtj7rTM=', 'Content-Length': '0'} 2022-09-03T07:21:19.232 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d58142bedc8873b3-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.232 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.233 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.233 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.234 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.234 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.235 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.235 DEBUG:boto:Data: 2022-09-03T07:21:19.236 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.236 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.237 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.237 DEBUG:boto:Params: {} 2022-09-03T07:21:19.238 DEBUG:boto:Token: None 2022-09-03T07:21:19.239 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iEvFCikabG7ssWkYxWCgwtj7rTM= 2022-09-03T07:21:19.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iEvFCikabG7ssWkYxWCgwtj7rTM=', 'Content-Length': '0'} 2022-09-03T07:21:19.242 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c9a12d1fc2c3bdf-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.242 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:21:19.243 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.243 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.244 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.244 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.245 DEBUG:boto:Method: GET 2022-09-03T07:21:19.246 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.246 DEBUG:boto:Data: 2022-09-03T07:21:19.247 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.249 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.250 DEBUG:boto:Params: {} 2022-09-03T07:21:19.251 DEBUG:boto:Token: None 2022-09-03T07:21:19.252 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jXtd3aJLiOwPJ6Ce1fwf4cotyTw= 2022-09-03T07:21:19.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jXtd3aJLiOwPJ6Ce1fwf4cotyTw=', 'Content-Length': '0'} 2022-09-03T07:21:19.257 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002dc4d348f3042058-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.258 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.259 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.259 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.260 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.260 DEBUG:boto:Method: GET 2022-09-03T07:21:19.261 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.262 DEBUG:boto:Data: 2022-09-03T07:21:19.262 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.263 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.263 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.264 DEBUG:boto:Params: {} 2022-09-03T07:21:19.265 DEBUG:boto:Token: None 2022-09-03T07:21:19.266 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:21:19.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jXtd3aJLiOwPJ6Ce1fwf4cotyTw= 2022-09-03T07:21:19.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jXtd3aJLiOwPJ6Ce1fwf4cotyTw=', 'Content-Length': '0'} 2022-09-03T07:21:19.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb6abbd4eafbabb6-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.271 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.272 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.273 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:19.274 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:19.275 DEBUG:boto:Method: GET 2022-09-03T07:21:19.275 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:19.276 DEBUG:boto:Data: 2022-09-03T07:21:19.276 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.277 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.278 DEBUG:boto:Params: {} 2022-09-03T07:21:19.278 DEBUG:boto:Token: None 2022-09-03T07:21:19.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.284 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006bd5489a4fcf815d-006313006f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.284 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.285 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.286 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.286 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:19.287 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:19.287 DEBUG:boto:Method: GET 2022-09-03T07:21:19.288 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:19.288 DEBUG:boto:Data: 2022-09-03T07:21:19.289 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.290 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.290 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.291 DEBUG:boto:Params: {} 2022-09-03T07:21:19.291 DEBUG:boto:Token: None 2022-09-03T07:21:19.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.295 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000083a7407f245bf44b-006313006f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.296 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.297 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.297 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.298 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.298 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:19.299 DEBUG:boto:Data: 2022-09-03T07:21:19.299 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.300 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.300 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.301 DEBUG:boto:Params: {} 2022-09-03T07:21:19.301 DEBUG:boto:Token: None 2022-09-03T07:21:19.302 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38 2022-09-03T07:21:19.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:leENLANyCdP0QROk44JN60lhanY= 2022-09-03T07:21:19.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:leENLANyCdP0QROk44JN60lhanY=', 'Content-Length': '0'} 2022-09-03T07:21:19.305 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e7cb56660e0a685-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.306 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.307 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.307 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.308 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:19.308 DEBUG:boto:Data: 2022-09-03T07:21:19.309 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.309 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.309 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.310 DEBUG:boto:Params: {} 2022-09-03T07:21:19.311 DEBUG:boto:Token: None 2022-09-03T07:21:19.311 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38 2022-09-03T07:21:19.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:leENLANyCdP0QROk44JN60lhanY= 2022-09-03T07:21:19.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:leENLANyCdP0QROk44JN60lhanY=', 'Content-Length': '0'} 2022-09-03T07:21:19.314 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fcf45804a7afd60c-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.315 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:21:19.315 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.316 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.316 DEBUG:boto:Method: GET 2022-09-03T07:21:19.317 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:19.317 DEBUG:boto:Data: 2022-09-03T07:21:19.318 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.319 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.320 DEBUG:boto:Params: {} 2022-09-03T07:21:19.320 DEBUG:boto:Token: None 2022-09-03T07:21:19.321 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38 2022-09-03T07:21:19.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwATsbOxUGpr5ninsBNBP2D7HgA= 2022-09-03T07:21:19.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwATsbOxUGpr5ninsBNBP2D7HgA=', 'Content-Length': '0'} 2022-09-03T07:21:19.325 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009606da6edfc45684-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.326 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:21:19.326 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:21:19.327 DEBUG:boto:Method: GET 2022-09-03T07:21:19.328 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:21:19.328 DEBUG:boto:Data: 2022-09-03T07:21:19.329 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.329 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.330 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.330 DEBUG:boto:Params: {} 2022-09-03T07:21:19.331 DEBUG:boto:Token: None 2022-09-03T07:21:19.331 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj38 2022-09-03T07:21:19.332 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwATsbOxUGpr5ninsBNBP2D7HgA= 2022-09-03T07:21:19.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwATsbOxUGpr5ninsBNBP2D7HgA=', 'Content-Length': '0'} 2022-09-03T07:21:19.339 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008ee1e0e56362d0f1-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.378 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.379 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.379 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:19.380 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:19.380 DEBUG:boto:Method: GET 2022-09-03T07:21:19.381 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:19.381 DEBUG:boto:Data: 2022-09-03T07:21:19.382 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.382 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.383 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.383 DEBUG:boto:Params: {} 2022-09-03T07:21:19.384 DEBUG:boto:Token: None 2022-09-03T07:21:19.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.389 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003bf52c9c4ab3bb89-006313006f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.390 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.390 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.391 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.392 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:19.392 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:21:19.393 DEBUG:boto:Method: GET 2022-09-03T07:21:19.393 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:21:19.394 DEBUG:boto:Data: 2022-09-03T07:21:19.394 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.395 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.395 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.396 DEBUG:boto:Params: {} 2022-09-03T07:21:19.396 DEBUG:boto:Token: None 2022-09-03T07:21:19.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.401 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000752087bd00deb404-006313006f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.401 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.402 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:21:19.403 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.403 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.404 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.404 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.405 DEBUG:boto:Method: GET 2022-09-03T07:21:19.405 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.406 DEBUG:boto:Data: 2022-09-03T07:21:19.406 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.408 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.408 DEBUG:boto:Params: {} 2022-09-03T07:21:19.409 DEBUG:boto:Token: None 2022-09-03T07:21:19.409 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:49LSPtd7dafLNWU4o2JREcAMYWc= 2022-09-03T07:21:19.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:49LSPtd7dafLNWU4o2JREcAMYWc=', 'Content-Length': '0'} 2022-09-03T07:21:19.413 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063ea367dbb97a781-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.414 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.414 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.415 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.415 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.416 DEBUG:boto:Method: GET 2022-09-03T07:21:19.417 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.417 DEBUG:boto:Data: 2022-09-03T07:21:19.418 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.418 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.419 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.419 DEBUG:boto:Params: {} 2022-09-03T07:21:19.420 DEBUG:boto:Token: None 2022-09-03T07:21:19.420 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:49LSPtd7dafLNWU4o2JREcAMYWc= 2022-09-03T07:21:19.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:49LSPtd7dafLNWU4o2JREcAMYWc=', 'Content-Length': '0'} 2022-09-03T07:21:19.424 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e31fe4eef96a284-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.424 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.425 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.425 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.426 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.426 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.427 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.427 DEBUG:boto:Data: 2022-09-03T07:21:19.428 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.429 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.429 DEBUG:boto:Params: {} 2022-09-03T07:21:19.430 DEBUG:boto:Token: None 2022-09-03T07:21:19.430 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tfppkz69eGYgfbXClcf60WoM2DU= 2022-09-03T07:21:19.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tfppkz69eGYgfbXClcf60WoM2DU=', 'Content-Length': '0'} 2022-09-03T07:21:19.434 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023121b2ce72447a1-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.435 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.436 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.436 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.436 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.437 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.437 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.438 DEBUG:boto:Data: 2022-09-03T07:21:19.438 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.439 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.439 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.440 DEBUG:boto:Params: {} 2022-09-03T07:21:19.441 DEBUG:boto:Token: None 2022-09-03T07:21:19.441 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tfppkz69eGYgfbXClcf60WoM2DU= 2022-09-03T07:21:19.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tfppkz69eGYgfbXClcf60WoM2DU=', 'Content-Length': '0'} 2022-09-03T07:21:19.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000826805426cc49c61-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.445 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:21:19.445 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.446 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.446 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.447 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.447 DEBUG:boto:Method: GET 2022-09-03T07:21:19.448 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.449 DEBUG:boto:Data: 2022-09-03T07:21:19.449 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.450 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.451 DEBUG:boto:Params: {} 2022-09-03T07:21:19.451 DEBUG:boto:Token: None 2022-09-03T07:21:19.452 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:49LSPtd7dafLNWU4o2JREcAMYWc= 2022-09-03T07:21:19.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:49LSPtd7dafLNWU4o2JREcAMYWc=', 'Content-Length': '0'} 2022-09-03T07:21:19.455 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec162f7b18b96eda-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.502 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.503 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.503 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.504 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.504 DEBUG:boto:Method: GET 2022-09-03T07:21:19.505 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.506 DEBUG:boto:Data: 2022-09-03T07:21:19.506 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.507 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.507 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.508 DEBUG:boto:Params: {} 2022-09-03T07:21:19.508 DEBUG:boto:Token: None 2022-09-03T07:21:19.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:21:19.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:49LSPtd7dafLNWU4o2JREcAMYWc= 2022-09-03T07:21:19.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:49LSPtd7dafLNWU4o2JREcAMYWc=', 'Content-Length': '0'} 2022-09-03T07:21:19.512 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe5c01321b390724-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.513 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.514 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.514 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:19.515 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:19.515 DEBUG:boto:Method: GET 2022-09-03T07:21:19.516 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:19.516 DEBUG:boto:Data: 2022-09-03T07:21:19.517 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.517 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.518 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.518 DEBUG:boto:Params: {} 2022-09-03T07:21:19.519 DEBUG:boto:Token: None 2022-09-03T07:21:19.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.521 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.524 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000089210a2f8bc5986e-006313006f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.525 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.526 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.527 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.528 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:19.529 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:19.530 DEBUG:boto:Method: GET 2022-09-03T07:21:19.531 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:19.532 DEBUG:boto:Data: 2022-09-03T07:21:19.533 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.534 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.535 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.536 DEBUG:boto:Params: {} 2022-09-03T07:21:19.537 DEBUG:boto:Token: None 2022-09-03T07:21:19.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.544 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000051873083bee8ec61-006313006f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.545 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.547 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.548 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.549 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.550 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:19.551 DEBUG:boto:Data: 2022-09-03T07:21:19.551 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.552 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.553 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.554 DEBUG:boto:Params: {} 2022-09-03T07:21:19.555 DEBUG:boto:Token: None 2022-09-03T07:21:19.556 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44 2022-09-03T07:21:19.557 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CrcouD7RBYrwsduA3V2I8SOGxfc= 2022-09-03T07:21:19.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CrcouD7RBYrwsduA3V2I8SOGxfc=', 'Content-Length': '0'} 2022-09-03T07:21:19.560 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c28bdd95cef88ae-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.561 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.561 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.562 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.563 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:19.563 DEBUG:boto:Data: 2022-09-03T07:21:19.564 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.564 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.565 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.565 DEBUG:boto:Params: {} 2022-09-03T07:21:19.566 DEBUG:boto:Token: None 2022-09-03T07:21:19.567 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44 2022-09-03T07:21:19.567 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CrcouD7RBYrwsduA3V2I8SOGxfc= 2022-09-03T07:21:19.568 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CrcouD7RBYrwsduA3V2I8SOGxfc=', 'Content-Length': '0'} 2022-09-03T07:21:19.570 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000977479ce62024816-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.570 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:21:19.571 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.571 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.572 DEBUG:boto:Method: GET 2022-09-03T07:21:19.572 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:19.573 DEBUG:boto:Data: 2022-09-03T07:21:19.574 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.574 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.575 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.575 DEBUG:boto:Params: {} 2022-09-03T07:21:19.576 DEBUG:boto:Token: None 2022-09-03T07:21:19.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44 2022-09-03T07:21:19.577 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CWJUZuLECWO63lQHjIVdr9YgvWY= 2022-09-03T07:21:19.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CWJUZuLECWO63lQHjIVdr9YgvWY=', 'Content-Length': '0'} 2022-09-03T07:21:19.580 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9f12e580a722f89-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.581 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:21:19.582 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:21:19.583 DEBUG:boto:Method: GET 2022-09-03T07:21:19.583 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:21:19.584 DEBUG:boto:Data: 2022-09-03T07:21:19.585 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.585 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.586 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.587 DEBUG:boto:Params: {} 2022-09-03T07:21:19.587 DEBUG:boto:Token: None 2022-09-03T07:21:19.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj44 2022-09-03T07:21:19.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CWJUZuLECWO63lQHjIVdr9YgvWY= 2022-09-03T07:21:19.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CWJUZuLECWO63lQHjIVdr9YgvWY=', 'Content-Length': '0'} 2022-09-03T07:21:19.592 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000363288ed86add80a-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.593 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.593 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.594 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:19.594 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:19.595 DEBUG:boto:Method: GET 2022-09-03T07:21:19.596 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:19.597 DEBUG:boto:Data: 2022-09-03T07:21:19.598 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.600 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.600 DEBUG:boto:Params: {} 2022-09-03T07:21:19.601 DEBUG:boto:Token: None 2022-09-03T07:21:19.602 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.606 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000113310421a179e8-006313006f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.607 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.608 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.608 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.609 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:19.609 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:21:19.610 DEBUG:boto:Method: GET 2022-09-03T07:21:19.611 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:21:19.611 DEBUG:boto:Data: 2022-09-03T07:21:19.612 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.612 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.613 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.613 DEBUG:boto:Params: {} 2022-09-03T07:21:19.614 DEBUG:boto:Token: None 2022-09-03T07:21:19.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.618 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009404b21fb084409f-006313006f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.619 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.620 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:21:19.621 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.622 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.623 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.623 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.624 DEBUG:boto:Method: GET 2022-09-03T07:21:19.625 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.625 DEBUG:boto:Data: 2022-09-03T07:21:19.626 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.627 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.627 DEBUG:boto:Params: {} 2022-09-03T07:21:19.628 DEBUG:boto:Token: None 2022-09-03T07:21:19.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf3SySCpQVeIHS8MALjJz9Bvt18= 2022-09-03T07:21:19.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf3SySCpQVeIHS8MALjJz9Bvt18=', 'Content-Length': '0'} 2022-09-03T07:21:19.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008124661ff4c9f39c-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.674 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.675 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.675 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.676 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.676 DEBUG:boto:Method: GET 2022-09-03T07:21:19.677 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.678 DEBUG:boto:Data: 2022-09-03T07:21:19.678 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.679 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.679 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.680 DEBUG:boto:Params: {} 2022-09-03T07:21:19.680 DEBUG:boto:Token: None 2022-09-03T07:21:19.681 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.682 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf3SySCpQVeIHS8MALjJz9Bvt18= 2022-09-03T07:21:19.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf3SySCpQVeIHS8MALjJz9Bvt18=', 'Content-Length': '0'} 2022-09-03T07:21:19.685 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e79ab6471e14fa4-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.687 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.688 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.688 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.689 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.690 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.691 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.692 DEBUG:boto:Data: 2022-09-03T07:21:19.693 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.693 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.694 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.695 DEBUG:boto:Params: {} 2022-09-03T07:21:19.696 DEBUG:boto:Token: None 2022-09-03T07:21:19.696 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I+EsjGt9JEawRQYKiFdaKU69tKM= 2022-09-03T07:21:19.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I+EsjGt9JEawRQYKiFdaKU69tKM=', 'Content-Length': '0'} 2022-09-03T07:21:19.700 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eecae37a591137fc-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.700 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.701 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.702 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.702 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.703 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.703 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.704 DEBUG:boto:Data: 2022-09-03T07:21:19.704 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.705 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.705 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.706 DEBUG:boto:Params: {} 2022-09-03T07:21:19.706 DEBUG:boto:Token: None 2022-09-03T07:21:19.707 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I+EsjGt9JEawRQYKiFdaKU69tKM= 2022-09-03T07:21:19.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I+EsjGt9JEawRQYKiFdaKU69tKM=', 'Content-Length': '0'} 2022-09-03T07:21:19.710 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e732e57de0426f9-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.711 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:21:19.711 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.712 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.712 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.713 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.713 DEBUG:boto:Method: GET 2022-09-03T07:21:19.714 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.714 DEBUG:boto:Data: 2022-09-03T07:21:19.715 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.716 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.717 DEBUG:boto:Params: {} 2022-09-03T07:21:19.717 DEBUG:boto:Token: None 2022-09-03T07:21:19.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf3SySCpQVeIHS8MALjJz9Bvt18= 2022-09-03T07:21:19.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf3SySCpQVeIHS8MALjJz9Bvt18=', 'Content-Length': '0'} 2022-09-03T07:21:19.721 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000528853251df552f2-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.722 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.723 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.723 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.724 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.724 DEBUG:boto:Method: GET 2022-09-03T07:21:19.725 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.725 DEBUG:boto:Data: 2022-09-03T07:21:19.726 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.726 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.727 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.727 DEBUG:boto:Params: {} 2022-09-03T07:21:19.728 DEBUG:boto:Token: None 2022-09-03T07:21:19.728 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:21:19.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tf3SySCpQVeIHS8MALjJz9Bvt18= 2022-09-03T07:21:19.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tf3SySCpQVeIHS8MALjJz9Bvt18=', 'Content-Length': '0'} 2022-09-03T07:21:19.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008757249436eaa525-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.733 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.733 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.734 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:19.734 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:19.735 DEBUG:boto:Method: GET 2022-09-03T07:21:19.735 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:19.736 DEBUG:boto:Data: 2022-09-03T07:21:19.736 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.737 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.737 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.738 DEBUG:boto:Params: {} 2022-09-03T07:21:19.738 DEBUG:boto:Token: None 2022-09-03T07:21:19.739 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.743 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000032617535ef273a95-006313006f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.744 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.745 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.745 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.746 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:19.746 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:19.747 DEBUG:boto:Method: GET 2022-09-03T07:21:19.747 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:19.748 DEBUG:boto:Data: 2022-09-03T07:21:19.748 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.749 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.749 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.750 DEBUG:boto:Params: {} 2022-09-03T07:21:19.750 DEBUG:boto:Token: None 2022-09-03T07:21:19.751 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.757 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c4b382ca37d486a4-006313006f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.758 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.760 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.761 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.762 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.763 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:19.764 DEBUG:boto:Data: 2022-09-03T07:21:19.765 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.766 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.767 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.768 DEBUG:boto:Params: {} 2022-09-03T07:21:19.770 DEBUG:boto:Token: None 2022-09-03T07:21:19.771 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51 2022-09-03T07:21:19.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mTkTu3sMWQBpFc5QNXc3bccn6EQ= 2022-09-03T07:21:19.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mTkTu3sMWQBpFc5QNXc3bccn6EQ=', 'Content-Length': '0'} 2022-09-03T07:21:19.776 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000858d5d8171b8a077-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.777 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.778 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.779 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.780 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:19.781 DEBUG:boto:Data: 2022-09-03T07:21:19.782 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.783 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.784 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.784 DEBUG:boto:Params: {} 2022-09-03T07:21:19.785 DEBUG:boto:Token: None 2022-09-03T07:21:19.786 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51 2022-09-03T07:21:19.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mTkTu3sMWQBpFc5QNXc3bccn6EQ= 2022-09-03T07:21:19.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mTkTu3sMWQBpFc5QNXc3bccn6EQ=', 'Content-Length': '0'} 2022-09-03T07:21:19.791 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb7fb5c3acf0f63c-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.791 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:21:19.792 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.792 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.793 DEBUG:boto:Method: GET 2022-09-03T07:21:19.794 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:19.794 DEBUG:boto:Data: 2022-09-03T07:21:19.795 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.796 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.796 DEBUG:boto:Params: {} 2022-09-03T07:21:19.797 DEBUG:boto:Token: None 2022-09-03T07:21:19.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51 2022-09-03T07:21:19.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F2hQ+jUtS6LsoeXBOB8WAUKc3KM= 2022-09-03T07:21:19.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F2hQ+jUtS6LsoeXBOB8WAUKc3KM=', 'Content-Length': '0'} 2022-09-03T07:21:19.804 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bdb023abf7344b0-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.805 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:21:19.806 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:21:19.807 DEBUG:boto:Method: GET 2022-09-03T07:21:19.807 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:21:19.808 DEBUG:boto:Data: 2022-09-03T07:21:19.809 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.810 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.811 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.812 DEBUG:boto:Params: {} 2022-09-03T07:21:19.813 DEBUG:boto:Token: None 2022-09-03T07:21:19.814 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj51 2022-09-03T07:21:19.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F2hQ+jUtS6LsoeXBOB8WAUKc3KM= 2022-09-03T07:21:19.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F2hQ+jUtS6LsoeXBOB8WAUKc3KM=', 'Content-Length': '0'} 2022-09-03T07:21:19.818 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3920ad163c3be90-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.866 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.867 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.867 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:19.868 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:19.868 DEBUG:boto:Method: GET 2022-09-03T07:21:19.869 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:19.869 DEBUG:boto:Data: 2022-09-03T07:21:19.870 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.871 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.871 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.872 DEBUG:boto:Params: {} 2022-09-03T07:21:19.872 DEBUG:boto:Token: None 2022-09-03T07:21:19.873 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.878 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000051837b1b580202a8-006313006f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.878 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.879 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.880 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.880 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:19.881 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:21:19.881 DEBUG:boto:Method: GET 2022-09-03T07:21:19.882 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:21:19.882 DEBUG:boto:Data: 2022-09-03T07:21:19.883 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.883 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.884 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.885 DEBUG:boto:Params: {} 2022-09-03T07:21:19.887 DEBUG:boto:Token: None 2022-09-03T07:21:19.887 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.888 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.893 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d082cfdf363299d3-006313006f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.894 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:19.895 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:21:19.896 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:19.897 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:19.898 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.899 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.900 DEBUG:boto:Method: GET 2022-09-03T07:21:19.900 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.901 DEBUG:boto:Data: 2022-09-03T07:21:19.902 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.903 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.904 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.905 DEBUG:boto:Params: {} 2022-09-03T07:21:19.906 DEBUG:boto:Token: None 2022-09-03T07:21:19.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ufLHeSju7bbJoUKCFGyTswhvrys= 2022-09-03T07:21:19.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ufLHeSju7bbJoUKCFGyTswhvrys=', 'Content-Length': '0'} 2022-09-03T07:21:19.912 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008905997279264ab0-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.914 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:19.914 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:19.915 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.916 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.917 DEBUG:boto:Method: GET 2022-09-03T07:21:19.918 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.918 DEBUG:boto:Data: 2022-09-03T07:21:19.919 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.919 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.920 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.920 DEBUG:boto:Params: {} 2022-09-03T07:21:19.921 DEBUG:boto:Token: None 2022-09-03T07:21:19.921 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ufLHeSju7bbJoUKCFGyTswhvrys= 2022-09-03T07:21:19.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ufLHeSju7bbJoUKCFGyTswhvrys=', 'Content-Length': '0'} 2022-09-03T07:21:19.925 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9ef1c04ba77c66b-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.926 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:19.927 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:19.927 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.928 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.928 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.929 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.930 DEBUG:boto:Data: 2022-09-03T07:21:19.930 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.932 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.932 DEBUG:boto:Params: {} 2022-09-03T07:21:19.933 DEBUG:boto:Token: None 2022-09-03T07:21:19.933 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xY3JaY5wLjGSC7eDXlMdl32gItI= 2022-09-03T07:21:19.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xY3JaY5wLjGSC7eDXlMdl32gItI=', 'Content-Length': '0'} 2022-09-03T07:21:19.937 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d5017887531d98a5-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.937 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:19.938 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:19.938 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.939 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.939 DEBUG:boto:Method: HEAD 2022-09-03T07:21:19.940 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.940 DEBUG:boto:Data: 2022-09-03T07:21:19.941 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.941 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.943 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.943 DEBUG:boto:Params: {} 2022-09-03T07:21:19.944 DEBUG:boto:Token: None 2022-09-03T07:21:19.945 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xY3JaY5wLjGSC7eDXlMdl32gItI= 2022-09-03T07:21:19.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xY3JaY5wLjGSC7eDXlMdl32gItI=', 'Content-Length': '0'} 2022-09-03T07:21:19.948 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ed2890704fa81e0-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.949 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:21:19.950 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:19.951 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:19.951 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.952 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.952 DEBUG:boto:Method: GET 2022-09-03T07:21:19.953 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.953 DEBUG:boto:Data: 2022-09-03T07:21:19.954 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.955 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.955 DEBUG:boto:Params: {} 2022-09-03T07:21:19.957 DEBUG:boto:Token: None 2022-09-03T07:21:19.958 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.959 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ufLHeSju7bbJoUKCFGyTswhvrys= 2022-09-03T07:21:19.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ufLHeSju7bbJoUKCFGyTswhvrys=', 'Content-Length': '0'} 2022-09-03T07:21:19.963 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e9ca291a0792c42-006313006f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.964 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:19.965 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:19.966 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.967 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.968 DEBUG:boto:Method: GET 2022-09-03T07:21:19.969 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.970 DEBUG:boto:Data: 2022-09-03T07:21:19.971 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.972 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:19.973 DEBUG:boto:Port: 8000 2022-09-03T07:21:19.974 DEBUG:boto:Params: {} 2022-09-03T07:21:19.975 DEBUG:boto:Token: None 2022-09-03T07:21:19.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:21:19.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ufLHeSju7bbJoUKCFGyTswhvrys= 2022-09-03T07:21:19.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ufLHeSju7bbJoUKCFGyTswhvrys=', 'Content-Length': '0'} 2022-09-03T07:21:19.981 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001fd9a5d214c508ab-006313006f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:19.983 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:19.984 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:19.984 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:19.985 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:19.986 DEBUG:boto:Method: GET 2022-09-03T07:21:19.987 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:19.987 DEBUG:boto:Data: 2022-09-03T07:21:19.988 DEBUG:boto:Headers: {} 2022-09-03T07:21:19.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:19.989 DEBUG:boto:Port: 8001 2022-09-03T07:21:19.989 DEBUG:boto:Params: {} 2022-09-03T07:21:19.990 DEBUG:boto:Token: None 2022-09-03T07:21:19.990 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:19 GMT /avysmw-75/ 2022-09-03T07:21:19.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY= 2022-09-03T07:21:19.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4VWmfvZTk+KX9JGJZ4QDLNR6fXY=', 'Content-Length': '0'} 2022-09-03T07:21:19.995 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090275cbf4fd92d43-006313006f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.038 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.039 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.040 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.040 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:20.041 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:20.041 DEBUG:boto:Method: GET 2022-09-03T07:21:20.042 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:20.042 DEBUG:boto:Data: 2022-09-03T07:21:20.043 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.043 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.044 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.044 DEBUG:boto:Params: {} 2022-09-03T07:21:20.045 DEBUG:boto:Token: None 2022-09-03T07:21:20.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.049 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000050d808026a8cf1be-0063130070-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.050 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.051 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:20.052 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:20.053 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.054 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:20.055 DEBUG:boto:Data: 2022-09-03T07:21:20.055 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.057 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.058 DEBUG:boto:Params: {} 2022-09-03T07:21:20.059 DEBUG:boto:Token: None 2022-09-03T07:21:20.060 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj55 2022-09-03T07:21:20.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aRjJrl39+Gy55QPkD6zUPwsV7HY= 2022-09-03T07:21:20.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aRjJrl39+Gy55QPkD6zUPwsV7HY=', 'Content-Length': '0'} 2022-09-03T07:21:20.065 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007928261311187f02-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.066 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:20.067 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:20.067 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.068 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:20.068 DEBUG:boto:Data: 2022-09-03T07:21:20.069 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.069 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.070 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.070 DEBUG:boto:Params: {} 2022-09-03T07:21:20.071 DEBUG:boto:Token: None 2022-09-03T07:21:20.071 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj55 2022-09-03T07:21:20.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aRjJrl39+Gy55QPkD6zUPwsV7HY= 2022-09-03T07:21:20.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aRjJrl39+Gy55QPkD6zUPwsV7HY=', 'Content-Length': '0'} 2022-09-03T07:21:20.074 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000269f65723afd53dd-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.075 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:21:20.076 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:20.076 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:20.077 DEBUG:boto:Method: GET 2022-09-03T07:21:20.077 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:20.078 DEBUG:boto:Data: 2022-09-03T07:21:20.078 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.079 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.080 DEBUG:boto:Params: {} 2022-09-03T07:21:20.081 DEBUG:boto:Token: None 2022-09-03T07:21:20.081 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj55 2022-09-03T07:21:20.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:erW898AtzEjlSP7haJhvXzAn3oc= 2022-09-03T07:21:20.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:erW898AtzEjlSP7haJhvXzAn3oc=', 'Content-Length': '0'} 2022-09-03T07:21:20.085 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cd96693f656b834-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.085 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:21:20.086 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:21:20.086 DEBUG:boto:Method: GET 2022-09-03T07:21:20.087 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:21:20.088 DEBUG:boto:Data: 2022-09-03T07:21:20.088 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.089 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.089 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.090 DEBUG:boto:Params: {} 2022-09-03T07:21:20.091 DEBUG:boto:Token: None 2022-09-03T07:21:20.091 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj55 2022-09-03T07:21:20.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:erW898AtzEjlSP7haJhvXzAn3oc= 2022-09-03T07:21:20.093 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:erW898AtzEjlSP7haJhvXzAn3oc=', 'Content-Length': '0'} 2022-09-03T07:21:20.095 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075d04ed0268cf936-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.095 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.096 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.097 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:20.097 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:20.098 DEBUG:boto:Method: GET 2022-09-03T07:21:20.098 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:20.099 DEBUG:boto:Data: 2022-09-03T07:21:20.099 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.100 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.101 DEBUG:boto:Params: {} 2022-09-03T07:21:20.101 DEBUG:boto:Token: None 2022-09-03T07:21:20.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.106 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000092cce666c923b0e5-0063130070-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.107 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.108 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.108 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.109 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:20.109 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:21:20.110 DEBUG:boto:Method: GET 2022-09-03T07:21:20.110 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:21:20.111 DEBUG:boto:Data: 2022-09-03T07:21:20.111 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.112 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.113 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.113 DEBUG:boto:Params: {} 2022-09-03T07:21:20.114 DEBUG:boto:Token: None 2022-09-03T07:21:20.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.118 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000744b2214102a9a4-0063130070-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.166 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.167 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:21:20.167 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.168 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.169 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.169 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.170 DEBUG:boto:Method: GET 2022-09-03T07:21:20.170 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.171 DEBUG:boto:Data: 2022-09-03T07:21:20.171 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.172 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.173 DEBUG:boto:Params: {} 2022-09-03T07:21:20.174 DEBUG:boto:Token: None 2022-09-03T07:21:20.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0b1XjLUp8Tf9XTb6dpA4Qu6Iac8= 2022-09-03T07:21:20.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0b1XjLUp8Tf9XTb6dpA4Qu6Iac8=', 'Content-Length': '0'} 2022-09-03T07:21:20.180 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b68d8f38b53859eb-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.182 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.183 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.183 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.184 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.185 DEBUG:boto:Method: GET 2022-09-03T07:21:20.186 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.186 DEBUG:boto:Data: 2022-09-03T07:21:20.187 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.188 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.188 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.189 DEBUG:boto:Params: {} 2022-09-03T07:21:20.189 DEBUG:boto:Token: None 2022-09-03T07:21:20.190 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0b1XjLUp8Tf9XTb6dpA4Qu6Iac8= 2022-09-03T07:21:20.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0b1XjLUp8Tf9XTb6dpA4Qu6Iac8=', 'Content-Length': '0'} 2022-09-03T07:21:20.193 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1ad33aa7d84f8d1-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.194 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.194 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.195 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.195 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.196 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.196 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.197 DEBUG:boto:Data: 2022-09-03T07:21:20.197 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.198 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.198 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.199 DEBUG:boto:Params: {} 2022-09-03T07:21:20.199 DEBUG:boto:Token: None 2022-09-03T07:21:20.200 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QWNLx5qcjBOgDHrADew0il2Z5dQ= 2022-09-03T07:21:20.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QWNLx5qcjBOgDHrADew0il2Z5dQ=', 'Content-Length': '0'} 2022-09-03T07:21:20.203 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f4d80347502b0e7-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.204 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.205 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.205 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.206 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.206 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.207 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.207 DEBUG:boto:Data: 2022-09-03T07:21:20.208 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.209 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.209 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.210 DEBUG:boto:Params: {} 2022-09-03T07:21:20.210 DEBUG:boto:Token: None 2022-09-03T07:21:20.211 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QWNLx5qcjBOgDHrADew0il2Z5dQ= 2022-09-03T07:21:20.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QWNLx5qcjBOgDHrADew0il2Z5dQ=', 'Content-Length': '0'} 2022-09-03T07:21:20.214 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003cc33a717757baee-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.214 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:21:20.215 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.216 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.216 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.217 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.217 DEBUG:boto:Method: GET 2022-09-03T07:21:20.218 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.218 DEBUG:boto:Data: 2022-09-03T07:21:20.219 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.219 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.220 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.220 DEBUG:boto:Params: {} 2022-09-03T07:21:20.221 DEBUG:boto:Token: None 2022-09-03T07:21:20.221 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0b1XjLUp8Tf9XTb6dpA4Qu6Iac8= 2022-09-03T07:21:20.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0b1XjLUp8Tf9XTb6dpA4Qu6Iac8=', 'Content-Length': '0'} 2022-09-03T07:21:20.225 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053f5b56680db3fd8-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.226 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.227 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.228 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.229 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.230 DEBUG:boto:Method: GET 2022-09-03T07:21:20.231 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.231 DEBUG:boto:Data: 2022-09-03T07:21:20.232 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.233 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.234 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.235 DEBUG:boto:Params: {} 2022-09-03T07:21:20.236 DEBUG:boto:Token: None 2022-09-03T07:21:20.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:21:20.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0b1XjLUp8Tf9XTb6dpA4Qu6Iac8= 2022-09-03T07:21:20.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0b1XjLUp8Tf9XTb6dpA4Qu6Iac8=', 'Content-Length': '0'} 2022-09-03T07:21:20.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dd84256127572664-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.241 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.241 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.242 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:20.242 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:20.243 DEBUG:boto:Method: GET 2022-09-03T07:21:20.244 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:20.244 DEBUG:boto:Data: 2022-09-03T07:21:20.245 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.245 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.246 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.246 DEBUG:boto:Params: {} 2022-09-03T07:21:20.247 DEBUG:boto:Token: None 2022-09-03T07:21:20.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.252 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b0dbd23b9c21fa4c-0063130070-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.252 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.253 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.254 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.254 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:20.255 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:20.256 DEBUG:boto:Method: GET 2022-09-03T07:21:20.257 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:20.258 DEBUG:boto:Data: 2022-09-03T07:21:20.259 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.260 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.261 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.262 DEBUG:boto:Params: {} 2022-09-03T07:21:20.263 DEBUG:boto:Token: None 2022-09-03T07:21:20.264 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.265 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.268 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c27af1c9214e453-0063130070-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.269 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.270 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.271 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.271 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.272 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:20.272 DEBUG:boto:Data: 2022-09-03T07:21:20.273 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.274 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.274 DEBUG:boto:Params: {} 2022-09-03T07:21:20.275 DEBUG:boto:Token: None 2022-09-03T07:21:20.276 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57 2022-09-03T07:21:20.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aCz8xOqVeBPoZMVdxL1aDhPokN0= 2022-09-03T07:21:20.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aCz8xOqVeBPoZMVdxL1aDhPokN0=', 'Content-Length': '0'} 2022-09-03T07:21:20.279 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bb0262edc7532fcf-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.280 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.280 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.281 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.281 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:20.282 DEBUG:boto:Data: 2022-09-03T07:21:20.282 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.283 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.283 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.284 DEBUG:boto:Params: {} 2022-09-03T07:21:20.284 DEBUG:boto:Token: None 2022-09-03T07:21:20.285 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57 2022-09-03T07:21:20.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aCz8xOqVeBPoZMVdxL1aDhPokN0= 2022-09-03T07:21:20.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aCz8xOqVeBPoZMVdxL1aDhPokN0=', 'Content-Length': '0'} 2022-09-03T07:21:20.288 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027f185a6a33581e5-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.289 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:21:20.289 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.290 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.291 DEBUG:boto:Method: GET 2022-09-03T07:21:20.292 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:20.292 DEBUG:boto:Data: 2022-09-03T07:21:20.293 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.294 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.295 DEBUG:boto:Params: {} 2022-09-03T07:21:20.296 DEBUG:boto:Token: None 2022-09-03T07:21:20.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57 2022-09-03T07:21:20.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yyE2UmqTAuTKYwkseH2Frr8E4sU= 2022-09-03T07:21:20.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yyE2UmqTAuTKYwkseH2Frr8E4sU=', 'Content-Length': '0'} 2022-09-03T07:21:20.302 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008ee97cf2ca719a5-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.342 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:21:20.343 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:21:20.344 DEBUG:boto:Method: GET 2022-09-03T07:21:20.344 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:21:20.345 DEBUG:boto:Data: 2022-09-03T07:21:20.346 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.346 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.347 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.347 DEBUG:boto:Params: {} 2022-09-03T07:21:20.348 DEBUG:boto:Token: None 2022-09-03T07:21:20.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj57 2022-09-03T07:21:20.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yyE2UmqTAuTKYwkseH2Frr8E4sU= 2022-09-03T07:21:20.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yyE2UmqTAuTKYwkseH2Frr8E4sU=', 'Content-Length': '0'} 2022-09-03T07:21:20.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000427320991ef701fb-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.353 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.354 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.354 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:20.355 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:20.355 DEBUG:boto:Method: GET 2022-09-03T07:21:20.356 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:20.356 DEBUG:boto:Data: 2022-09-03T07:21:20.357 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.357 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.358 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.359 DEBUG:boto:Params: {} 2022-09-03T07:21:20.360 DEBUG:boto:Token: None 2022-09-03T07:21:20.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.364 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087a8506857ceccc0-0063130070-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.365 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.366 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.366 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.367 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:20.367 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:21:20.368 DEBUG:boto:Method: GET 2022-09-03T07:21:20.369 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:21:20.369 DEBUG:boto:Data: 2022-09-03T07:21:20.370 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.370 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.371 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.371 DEBUG:boto:Params: {} 2022-09-03T07:21:20.372 DEBUG:boto:Token: None 2022-09-03T07:21:20.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.376 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c96624637c2dada-0063130070-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.377 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.377 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:21:20.378 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.379 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.379 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.380 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.380 DEBUG:boto:Method: GET 2022-09-03T07:21:20.381 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.381 DEBUG:boto:Data: 2022-09-03T07:21:20.382 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.382 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.383 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.383 DEBUG:boto:Params: {} 2022-09-03T07:21:20.384 DEBUG:boto:Token: None 2022-09-03T07:21:20.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IvcklGLhhWz3qmsgqvElOUA6CJc= 2022-09-03T07:21:20.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IvcklGLhhWz3qmsgqvElOUA6CJc=', 'Content-Length': '0'} 2022-09-03T07:21:20.388 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006bbafb9e52d08e10-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.389 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.389 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.390 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.391 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.391 DEBUG:boto:Method: GET 2022-09-03T07:21:20.392 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.392 DEBUG:boto:Data: 2022-09-03T07:21:20.393 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.393 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.394 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.394 DEBUG:boto:Params: {} 2022-09-03T07:21:20.395 DEBUG:boto:Token: None 2022-09-03T07:21:20.396 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.396 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IvcklGLhhWz3qmsgqvElOUA6CJc= 2022-09-03T07:21:20.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IvcklGLhhWz3qmsgqvElOUA6CJc=', 'Content-Length': '0'} 2022-09-03T07:21:20.399 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ef8b9a8bae78745-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.399 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.400 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.400 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.401 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.401 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.402 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.402 DEBUG:boto:Data: 2022-09-03T07:21:20.403 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.404 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.404 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.405 DEBUG:boto:Params: {} 2022-09-03T07:21:20.405 DEBUG:boto:Token: None 2022-09-03T07:21:20.406 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y+gKY60kz8HfZbefgUYOXb5MdMY= 2022-09-03T07:21:20.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y+gKY60kz8HfZbefgUYOXb5MdMY=', 'Content-Length': '0'} 2022-09-03T07:21:20.409 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab1736be55bc0b40-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.410 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.411 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.411 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.412 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.412 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.413 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.413 DEBUG:boto:Data: 2022-09-03T07:21:20.414 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.414 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.415 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.415 DEBUG:boto:Params: {} 2022-09-03T07:21:20.416 DEBUG:boto:Token: None 2022-09-03T07:21:20.416 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y+gKY60kz8HfZbefgUYOXb5MdMY= 2022-09-03T07:21:20.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y+gKY60kz8HfZbefgUYOXb5MdMY=', 'Content-Length': '0'} 2022-09-03T07:21:20.420 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e64657fc8f689e2-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.420 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:21:20.421 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.421 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.422 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.422 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.423 DEBUG:boto:Method: GET 2022-09-03T07:21:20.423 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.424 DEBUG:boto:Data: 2022-09-03T07:21:20.425 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.426 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.427 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.428 DEBUG:boto:Params: {} 2022-09-03T07:21:20.429 DEBUG:boto:Token: None 2022-09-03T07:21:20.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IvcklGLhhWz3qmsgqvElOUA6CJc= 2022-09-03T07:21:20.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IvcklGLhhWz3qmsgqvElOUA6CJc=', 'Content-Length': '0'} 2022-09-03T07:21:20.434 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a67e6124cc45595-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.436 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.437 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.438 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.439 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.440 DEBUG:boto:Method: GET 2022-09-03T07:21:20.441 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.441 DEBUG:boto:Data: 2022-09-03T07:21:20.442 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.443 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.444 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.445 DEBUG:boto:Params: {} 2022-09-03T07:21:20.446 DEBUG:boto:Token: None 2022-09-03T07:21:20.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:21:20.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IvcklGLhhWz3qmsgqvElOUA6CJc= 2022-09-03T07:21:20.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IvcklGLhhWz3qmsgqvElOUA6CJc=', 'Content-Length': '0'} 2022-09-03T07:21:20.452 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c491f75383ca78a-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.454 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.455 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.456 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:20.457 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:20.458 DEBUG:boto:Method: GET 2022-09-03T07:21:20.459 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:20.460 DEBUG:boto:Data: 2022-09-03T07:21:20.461 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.463 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.463 DEBUG:boto:Params: {} 2022-09-03T07:21:20.464 DEBUG:boto:Token: None 2022-09-03T07:21:20.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.470 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000054b46db1504ae74a-0063130070-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.471 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.472 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.472 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.473 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:20.473 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:20.474 DEBUG:boto:Method: GET 2022-09-03T07:21:20.474 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:20.475 DEBUG:boto:Data: 2022-09-03T07:21:20.475 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.476 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.476 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.477 DEBUG:boto:Params: {} 2022-09-03T07:21:20.477 DEBUG:boto:Token: None 2022-09-03T07:21:20.478 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.482 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a21ebe669858095-0063130070-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.482 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.483 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.484 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.484 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.485 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:20.485 DEBUG:boto:Data: 2022-09-03T07:21:20.486 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.486 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.486 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.487 DEBUG:boto:Params: {} 2022-09-03T07:21:20.488 DEBUG:boto:Token: None 2022-09-03T07:21:20.488 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68 2022-09-03T07:21:20.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3zhDhJtzgBcNtZMhHlWph67aoC4= 2022-09-03T07:21:20.489 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3zhDhJtzgBcNtZMhHlWph67aoC4=', 'Content-Length': '0'} 2022-09-03T07:21:20.491 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007b18ff5adfe727d0-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.492 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.492 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.493 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.493 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:20.494 DEBUG:boto:Data: 2022-09-03T07:21:20.494 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.495 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.495 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.496 DEBUG:boto:Params: {} 2022-09-03T07:21:20.496 DEBUG:boto:Token: None 2022-09-03T07:21:20.497 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68 2022-09-03T07:21:20.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3zhDhJtzgBcNtZMhHlWph67aoC4= 2022-09-03T07:21:20.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3zhDhJtzgBcNtZMhHlWph67aoC4=', 'Content-Length': '0'} 2022-09-03T07:21:20.500 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007668eb166aa239b4-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.500 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:21:20.501 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.502 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.502 DEBUG:boto:Method: GET 2022-09-03T07:21:20.503 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:20.503 DEBUG:boto:Data: 2022-09-03T07:21:20.504 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.504 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.505 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.505 DEBUG:boto:Params: {} 2022-09-03T07:21:20.506 DEBUG:boto:Token: None 2022-09-03T07:21:20.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68 2022-09-03T07:21:20.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jcnWaB6H+B5BjhGGxdN2gk4qw88= 2022-09-03T07:21:20.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jcnWaB6H+B5BjhGGxdN2gk4qw88=', 'Content-Length': '0'} 2022-09-03T07:21:20.509 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6624bd365399bd8-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.554 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:21:20.555 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:21:20.555 DEBUG:boto:Method: GET 2022-09-03T07:21:20.556 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:21:20.556 DEBUG:boto:Data: 2022-09-03T07:21:20.557 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.557 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.558 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.559 DEBUG:boto:Params: {} 2022-09-03T07:21:20.559 DEBUG:boto:Token: None 2022-09-03T07:21:20.560 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj68 2022-09-03T07:21:20.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jcnWaB6H+B5BjhGGxdN2gk4qw88= 2022-09-03T07:21:20.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jcnWaB6H+B5BjhGGxdN2gk4qw88=', 'Content-Length': '0'} 2022-09-03T07:21:20.563 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bfd166e31a30fa35-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.564 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.565 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.565 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:20.566 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:20.566 DEBUG:boto:Method: GET 2022-09-03T07:21:20.567 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:20.567 DEBUG:boto:Data: 2022-09-03T07:21:20.568 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.569 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.569 DEBUG:boto:Params: {} 2022-09-03T07:21:20.570 DEBUG:boto:Token: None 2022-09-03T07:21:20.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.575 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000593a543da3f9760b-0063130070-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.576 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.577 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.577 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.578 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:20.578 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:21:20.579 DEBUG:boto:Method: GET 2022-09-03T07:21:20.579 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:21:20.580 DEBUG:boto:Data: 2022-09-03T07:21:20.580 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.581 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.581 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.582 DEBUG:boto:Params: {} 2022-09-03T07:21:20.582 DEBUG:boto:Token: None 2022-09-03T07:21:20.583 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.584 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.587 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002bdb8f5907d2ab02-0063130070-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.587 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.588 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:21:20.589 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.589 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.590 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.590 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.591 DEBUG:boto:Method: GET 2022-09-03T07:21:20.591 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.592 DEBUG:boto:Data: 2022-09-03T07:21:20.592 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.593 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.594 DEBUG:boto:Params: {} 2022-09-03T07:21:20.595 DEBUG:boto:Token: None 2022-09-03T07:21:20.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oRX/KVc87dNm5vEKE8eD961TP4c= 2022-09-03T07:21:20.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oRX/KVc87dNm5vEKE8eD961TP4c=', 'Content-Length': '0'} 2022-09-03T07:21:20.599 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000465f5f17253f278-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.599 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.600 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.601 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.601 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.602 DEBUG:boto:Method: GET 2022-09-03T07:21:20.603 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.603 DEBUG:boto:Data: 2022-09-03T07:21:20.604 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.605 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.605 DEBUG:boto:Params: {} 2022-09-03T07:21:20.606 DEBUG:boto:Token: None 2022-09-03T07:21:20.606 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oRX/KVc87dNm5vEKE8eD961TP4c= 2022-09-03T07:21:20.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oRX/KVc87dNm5vEKE8eD961TP4c=', 'Content-Length': '0'} 2022-09-03T07:21:20.610 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005aa9e8dca747ca7a-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.650 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.651 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.652 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.653 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.654 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.655 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.655 DEBUG:boto:Data: 2022-09-03T07:21:20.656 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.656 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.657 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.657 DEBUG:boto:Params: {} 2022-09-03T07:21:20.658 DEBUG:boto:Token: None 2022-09-03T07:21:20.659 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:34Zz4cGpnqNjAsu/rb+tQOQ2ZUg= 2022-09-03T07:21:20.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:34Zz4cGpnqNjAsu/rb+tQOQ2ZUg=', 'Content-Length': '0'} 2022-09-03T07:21:20.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000078ea16aabd471af-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.663 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.663 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.664 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.664 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.665 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.665 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.666 DEBUG:boto:Data: 2022-09-03T07:21:20.667 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.667 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.668 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.668 DEBUG:boto:Params: {} 2022-09-03T07:21:20.669 DEBUG:boto:Token: None 2022-09-03T07:21:20.670 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:34Zz4cGpnqNjAsu/rb+tQOQ2ZUg= 2022-09-03T07:21:20.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:34Zz4cGpnqNjAsu/rb+tQOQ2ZUg=', 'Content-Length': '0'} 2022-09-03T07:21:20.673 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e30f48b5f4a3e1a6-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.673 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:21:20.674 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.675 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.676 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.676 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.677 DEBUG:boto:Method: GET 2022-09-03T07:21:20.677 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.678 DEBUG:boto:Data: 2022-09-03T07:21:20.679 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.680 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.680 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.681 DEBUG:boto:Params: {} 2022-09-03T07:21:20.682 DEBUG:boto:Token: None 2022-09-03T07:21:20.683 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oRX/KVc87dNm5vEKE8eD961TP4c= 2022-09-03T07:21:20.685 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oRX/KVc87dNm5vEKE8eD961TP4c=', 'Content-Length': '0'} 2022-09-03T07:21:20.687 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000433a4f0e45504f99-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.730 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.733 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.734 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.735 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.735 DEBUG:boto:Method: GET 2022-09-03T07:21:20.736 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.736 DEBUG:boto:Data: 2022-09-03T07:21:20.737 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.737 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.738 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.738 DEBUG:boto:Params: {} 2022-09-03T07:21:20.739 DEBUG:boto:Token: None 2022-09-03T07:21:20.740 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:21:20.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oRX/KVc87dNm5vEKE8eD961TP4c= 2022-09-03T07:21:20.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oRX/KVc87dNm5vEKE8eD961TP4c=', 'Content-Length': '0'} 2022-09-03T07:21:20.744 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d54ce51c019b491-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.745 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.746 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.746 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:20.747 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:20.748 DEBUG:boto:Method: GET 2022-09-03T07:21:20.748 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:20.749 DEBUG:boto:Data: 2022-09-03T07:21:20.750 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.750 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.751 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.751 DEBUG:boto:Params: {} 2022-09-03T07:21:20.752 DEBUG:boto:Token: None 2022-09-03T07:21:20.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.758 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6187036247ed994-0063130070-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.758 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.759 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.760 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.760 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:20.761 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:20.761 DEBUG:boto:Method: GET 2022-09-03T07:21:20.772 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:20.774 DEBUG:boto:Data: 2022-09-03T07:21:20.774 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.775 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.775 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.776 DEBUG:boto:Params: {} 2022-09-03T07:21:20.777 DEBUG:boto:Token: None 2022-09-03T07:21:20.778 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.782 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000007042e102ac7c978-0063130070-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.782 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.783 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.784 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.785 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.785 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:20.786 DEBUG:boto:Data: 2022-09-03T07:21:20.786 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.787 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.788 DEBUG:boto:Params: {} 2022-09-03T07:21:20.789 DEBUG:boto:Token: None 2022-09-03T07:21:20.789 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76 2022-09-03T07:21:20.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VHtPfeW/9zqAEfYGBAZwNJwkf/g= 2022-09-03T07:21:20.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VHtPfeW/9zqAEfYGBAZwNJwkf/g=', 'Content-Length': '0'} 2022-09-03T07:21:20.792 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e7ee944420ddc3b-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.793 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.794 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.795 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.796 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:20.797 DEBUG:boto:Data: 2022-09-03T07:21:20.798 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.799 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.800 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.801 DEBUG:boto:Params: {} 2022-09-03T07:21:20.802 DEBUG:boto:Token: None 2022-09-03T07:21:20.803 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76 2022-09-03T07:21:20.804 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VHtPfeW/9zqAEfYGBAZwNJwkf/g= 2022-09-03T07:21:20.805 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VHtPfeW/9zqAEfYGBAZwNJwkf/g=', 'Content-Length': '0'} 2022-09-03T07:21:20.808 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ed6e0ca9f259f6f-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.808 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:21:20.809 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.810 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.810 DEBUG:boto:Method: GET 2022-09-03T07:21:20.811 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:20.811 DEBUG:boto:Data: 2022-09-03T07:21:20.812 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.813 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.814 DEBUG:boto:Params: {} 2022-09-03T07:21:20.815 DEBUG:boto:Token: None 2022-09-03T07:21:20.815 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76 2022-09-03T07:21:20.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eRnBVFxqwYK4YABHakFZ3/Zbuso= 2022-09-03T07:21:20.817 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eRnBVFxqwYK4YABHakFZ3/Zbuso=', 'Content-Length': '0'} 2022-09-03T07:21:20.819 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000da9a8b0c45d642e-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.820 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:21:20.820 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:21:20.821 DEBUG:boto:Method: GET 2022-09-03T07:21:20.821 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:21:20.822 DEBUG:boto:Data: 2022-09-03T07:21:20.823 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.823 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.824 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.825 DEBUG:boto:Params: {} 2022-09-03T07:21:20.825 DEBUG:boto:Token: None 2022-09-03T07:21:20.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj76 2022-09-03T07:21:20.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eRnBVFxqwYK4YABHakFZ3/Zbuso= 2022-09-03T07:21:20.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eRnBVFxqwYK4YABHakFZ3/Zbuso=', 'Content-Length': '0'} 2022-09-03T07:21:20.829 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003303e5563d7c4621-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.830 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.831 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.831 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:20.832 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:20.833 DEBUG:boto:Method: GET 2022-09-03T07:21:20.833 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:20.834 DEBUG:boto:Data: 2022-09-03T07:21:20.834 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.835 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.836 DEBUG:boto:Params: {} 2022-09-03T07:21:20.837 DEBUG:boto:Token: None 2022-09-03T07:21:20.837 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.841 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000460e76dda70302fb-0063130070-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.842 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.843 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.843 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.844 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:20.844 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:21:20.845 DEBUG:boto:Method: GET 2022-09-03T07:21:20.845 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:21:20.846 DEBUG:boto:Data: 2022-09-03T07:21:20.847 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.847 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.848 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.848 DEBUG:boto:Params: {} 2022-09-03T07:21:20.849 DEBUG:boto:Token: None 2022-09-03T07:21:20.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.853 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e5c7cfc1ab15d043-0063130070-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.854 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.855 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:21:20.855 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:20.856 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:20.856 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.857 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.857 DEBUG:boto:Method: GET 2022-09-03T07:21:20.858 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.859 DEBUG:boto:Data: 2022-09-03T07:21:20.860 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.860 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.861 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.861 DEBUG:boto:Params: {} 2022-09-03T07:21:20.862 DEBUG:boto:Token: None 2022-09-03T07:21:20.862 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ttdDbSh2Y1jJjn/M93c+bjMgvI= 2022-09-03T07:21:20.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ttdDbSh2Y1jJjn/M93c+bjMgvI=', 'Content-Length': '0'} 2022-09-03T07:21:20.866 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3efcba7588aacbd-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.867 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:20.867 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:20.868 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.868 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.869 DEBUG:boto:Method: GET 2022-09-03T07:21:20.870 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.871 DEBUG:boto:Data: 2022-09-03T07:21:20.872 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.873 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.874 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.875 DEBUG:boto:Params: {} 2022-09-03T07:21:20.876 DEBUG:boto:Token: None 2022-09-03T07:21:20.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ttdDbSh2Y1jJjn/M93c+bjMgvI= 2022-09-03T07:21:20.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ttdDbSh2Y1jJjn/M93c+bjMgvI=', 'Content-Length': '0'} 2022-09-03T07:21:20.882 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aeef476f82bb755b-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.883 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:20.884 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:20.884 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.885 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.885 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.886 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.887 DEBUG:boto:Data: 2022-09-03T07:21:20.887 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.888 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.888 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.889 DEBUG:boto:Params: {} 2022-09-03T07:21:20.889 DEBUG:boto:Token: None 2022-09-03T07:21:20.890 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g6JF3Huq49ifAv5gQIkLMfV/10E= 2022-09-03T07:21:20.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g6JF3Huq49ifAv5gQIkLMfV/10E=', 'Content-Length': '0'} 2022-09-03T07:21:20.894 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fa84eee470c4528-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.894 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:20.895 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:20.895 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.896 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.897 DEBUG:boto:Method: HEAD 2022-09-03T07:21:20.897 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.898 DEBUG:boto:Data: 2022-09-03T07:21:20.899 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.900 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.901 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.902 DEBUG:boto:Params: {} 2022-09-03T07:21:20.902 DEBUG:boto:Token: None 2022-09-03T07:21:20.903 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g6JF3Huq49ifAv5gQIkLMfV/10E= 2022-09-03T07:21:20.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g6JF3Huq49ifAv5gQIkLMfV/10E=', 'Content-Length': '0'} 2022-09-03T07:21:20.906 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b4e96bd53daa32f9-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.907 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:21:20.909 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:20.910 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:20.911 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.912 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.913 DEBUG:boto:Method: GET 2022-09-03T07:21:20.914 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.915 DEBUG:boto:Data: 2022-09-03T07:21:20.916 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.917 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.918 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.918 DEBUG:boto:Params: {} 2022-09-03T07:21:20.919 DEBUG:boto:Token: None 2022-09-03T07:21:20.920 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.920 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ttdDbSh2Y1jJjn/M93c+bjMgvI= 2022-09-03T07:21:20.921 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ttdDbSh2Y1jJjn/M93c+bjMgvI=', 'Content-Length': '0'} 2022-09-03T07:21:20.923 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009cb5414d8cd1f52f-0063130070-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.966 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:20.967 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:20.969 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.970 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.971 DEBUG:boto:Method: GET 2022-09-03T07:21:20.972 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.973 DEBUG:boto:Data: 2022-09-03T07:21:20.974 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.975 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:20.976 DEBUG:boto:Port: 8000 2022-09-03T07:21:20.977 DEBUG:boto:Params: {} 2022-09-03T07:21:20.978 DEBUG:boto:Token: None 2022-09-03T07:21:20.979 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:21:20.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ttdDbSh2Y1jJjn/M93c+bjMgvI= 2022-09-03T07:21:20.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ttdDbSh2Y1jJjn/M93c+bjMgvI=', 'Content-Length': '0'} 2022-09-03T07:21:20.984 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000977955f7e013289a-0063130070-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.985 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.985 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.986 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:20.986 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:20.987 DEBUG:boto:Method: GET 2022-09-03T07:21:20.987 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:20.988 DEBUG:boto:Data: 2022-09-03T07:21:20.989 DEBUG:boto:Headers: {} 2022-09-03T07:21:20.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:20.990 DEBUG:boto:Port: 8001 2022-09-03T07:21:20.991 DEBUG:boto:Params: {} 2022-09-03T07:21:20.992 DEBUG:boto:Token: None 2022-09-03T07:21:20.992 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:20 GMT /avysmw-75/ 2022-09-03T07:21:20.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc= 2022-09-03T07:21:20.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VRuIUXxDN+3apxLsOiZZm0i8Bcc=', 'Content-Length': '0'} 2022-09-03T07:21:20.996 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aed338a39f0cca46-0063130070-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:20.997 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:20.998 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:20.999 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:20.999 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:21.000 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:21.000 DEBUG:boto:Method: GET 2022-09-03T07:21:21.001 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:21.001 DEBUG:boto:Data: 2022-09-03T07:21:21.002 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.002 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.003 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.003 DEBUG:boto:Params: {} 2022-09-03T07:21:21.004 DEBUG:boto:Token: None 2022-09-03T07:21:21.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0= 2022-09-03T07:21:21.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0=', 'Content-Length': '0'} 2022-09-03T07:21:21.008 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008876508d7d6e08eb-0063130071-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.009 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:21.009 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:21.010 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:21.011 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.011 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:21.012 DEBUG:boto:Data: 2022-09-03T07:21:21.012 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.012 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.013 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.013 DEBUG:boto:Params: {} 2022-09-03T07:21:21.014 DEBUG:boto:Token: None 2022-09-03T07:21:21.015 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj85 2022-09-03T07:21:21.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tgYdQvXp/zCcbc+0xbNgZcBpE9M= 2022-09-03T07:21:21.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tgYdQvXp/zCcbc+0xbNgZcBpE9M=', 'Content-Length': '0'} 2022-09-03T07:21:21.018 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b57483f113353dfa-0063130071-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.019 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:21.019 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:21.020 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.020 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:21.021 DEBUG:boto:Data: 2022-09-03T07:21:21.021 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.022 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.022 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.023 DEBUG:boto:Params: {} 2022-09-03T07:21:21.024 DEBUG:boto:Token: None 2022-09-03T07:21:21.024 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj85 2022-09-03T07:21:21.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tgYdQvXp/zCcbc+0xbNgZcBpE9M= 2022-09-03T07:21:21.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tgYdQvXp/zCcbc+0xbNgZcBpE9M=', 'Content-Length': '0'} 2022-09-03T07:21:21.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062fa9c5cda763601-0063130071-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.028 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:21:21.028 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:21.029 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:21.030 DEBUG:boto:Method: GET 2022-09-03T07:21:21.030 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:21.031 DEBUG:boto:Data: 2022-09-03T07:21:21.031 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.032 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.033 DEBUG:boto:Params: {} 2022-09-03T07:21:21.033 DEBUG:boto:Token: None 2022-09-03T07:21:21.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj85 2022-09-03T07:21:21.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e9wWmIsIXff+D/Vcyy3efGeGbiw= 2022-09-03T07:21:21.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e9wWmIsIXff+D/Vcyy3efGeGbiw=', 'Content-Length': '0'} 2022-09-03T07:21:21.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057c9d707a962a0f5-0063130071-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.038 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:21:21.039 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:21:21.039 DEBUG:boto:Method: GET 2022-09-03T07:21:21.040 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:21:21.040 DEBUG:boto:Data: 2022-09-03T07:21:21.041 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.041 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.042 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.042 DEBUG:boto:Params: {} 2022-09-03T07:21:21.043 DEBUG:boto:Token: None 2022-09-03T07:21:21.044 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj85 2022-09-03T07:21:21.044 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e9wWmIsIXff+D/Vcyy3efGeGbiw= 2022-09-03T07:21:21.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e9wWmIsIXff+D/Vcyy3efGeGbiw=', 'Content-Length': '0'} 2022-09-03T07:21:21.047 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002556ddf9beb38037-0063130071-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.047 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.048 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.048 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:21.049 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:21.050 DEBUG:boto:Method: GET 2022-09-03T07:21:21.050 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:21.051 DEBUG:boto:Data: 2022-09-03T07:21:21.051 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.052 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.052 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.053 DEBUG:boto:Params: {} 2022-09-03T07:21:21.053 DEBUG:boto:Token: None 2022-09-03T07:21:21.054 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0= 2022-09-03T07:21:21.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0=', 'Content-Length': '0'} 2022-09-03T07:21:21.058 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003412a62f986f10f0-0063130071-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.059 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:21.059 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.060 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.061 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:21.061 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:21:21.062 DEBUG:boto:Method: GET 2022-09-03T07:21:21.062 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:21:21.063 DEBUG:boto:Data: 2022-09-03T07:21:21.063 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.064 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.064 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.065 DEBUG:boto:Params: {} 2022-09-03T07:21:21.065 DEBUG:boto:Token: None 2022-09-03T07:21:21.066 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.067 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0= 2022-09-03T07:21:21.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0=', 'Content-Length': '0'} 2022-09-03T07:21:21.070 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005b0b7999308c16db-0063130071-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.110 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:21.111 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:21:21.112 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.112 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.113 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.113 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.114 DEBUG:boto:Method: GET 2022-09-03T07:21:21.114 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.115 DEBUG:boto:Data: 2022-09-03T07:21:21.115 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.116 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.117 DEBUG:boto:Params: {} 2022-09-03T07:21:21.117 DEBUG:boto:Token: None 2022-09-03T07:21:21.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MxmcrDy1mKztLgam7MDi3YpOmh8= 2022-09-03T07:21:21.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MxmcrDy1mKztLgam7MDi3YpOmh8=', 'Content-Length': '0'} 2022-09-03T07:21:21.122 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f71258a2ceacd7af-0063130071-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.122 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.123 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.123 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.124 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.125 DEBUG:boto:Method: GET 2022-09-03T07:21:21.125 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.126 DEBUG:boto:Data: 2022-09-03T07:21:21.126 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.127 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.127 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.128 DEBUG:boto:Params: {} 2022-09-03T07:21:21.129 DEBUG:boto:Token: None 2022-09-03T07:21:21.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MxmcrDy1mKztLgam7MDi3YpOmh8= 2022-09-03T07:21:21.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MxmcrDy1mKztLgam7MDi3YpOmh8=', 'Content-Length': '0'} 2022-09-03T07:21:21.132 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e7ea199754c76aaa-0063130071-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.133 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.134 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.134 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.135 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.135 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.136 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.136 DEBUG:boto:Data: 2022-09-03T07:21:21.137 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.138 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.139 DEBUG:boto:Params: {} 2022-09-03T07:21:21.139 DEBUG:boto:Token: None 2022-09-03T07:21:21.140 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KjtYloZqddt8X4dAYGd7dHO818k= 2022-09-03T07:21:21.141 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KjtYloZqddt8X4dAYGd7dHO818k=', 'Content-Length': '0'} 2022-09-03T07:21:21.143 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ec599a3e2beaccd-0063130071-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.144 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.144 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.145 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.145 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.146 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.146 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.147 DEBUG:boto:Data: 2022-09-03T07:21:21.147 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.148 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.148 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.149 DEBUG:boto:Params: {} 2022-09-03T07:21:21.149 DEBUG:boto:Token: None 2022-09-03T07:21:21.150 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KjtYloZqddt8X4dAYGd7dHO818k= 2022-09-03T07:21:21.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KjtYloZqddt8X4dAYGd7dHO818k=', 'Content-Length': '0'} 2022-09-03T07:21:21.153 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfa7bf325d578196-0063130071-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.153 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:21:21.154 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.155 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.155 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.156 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.156 DEBUG:boto:Method: GET 2022-09-03T07:21:21.157 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.157 DEBUG:boto:Data: 2022-09-03T07:21:21.158 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.159 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.159 DEBUG:boto:Params: {} 2022-09-03T07:21:21.160 DEBUG:boto:Token: None 2022-09-03T07:21:21.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MxmcrDy1mKztLgam7MDi3YpOmh8= 2022-09-03T07:21:21.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MxmcrDy1mKztLgam7MDi3YpOmh8=', 'Content-Length': '0'} 2022-09-03T07:21:21.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000866028ee8cc035d1-0063130071-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.164 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.165 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.166 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.166 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.167 DEBUG:boto:Method: GET 2022-09-03T07:21:21.167 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.168 DEBUG:boto:Data: 2022-09-03T07:21:21.168 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.169 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.169 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.170 DEBUG:boto:Params: {} 2022-09-03T07:21:21.170 DEBUG:boto:Token: None 2022-09-03T07:21:21.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:21:21.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MxmcrDy1mKztLgam7MDi3YpOmh8= 2022-09-03T07:21:21.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MxmcrDy1mKztLgam7MDi3YpOmh8=', 'Content-Length': '0'} 2022-09-03T07:21:21.174 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003eb593218ac7b173-0063130071-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.175 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.176 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.176 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:21.177 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:21.177 DEBUG:boto:Method: GET 2022-09-03T07:21:21.178 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:21.178 DEBUG:boto:Data: 2022-09-03T07:21:21.179 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.179 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.180 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.181 DEBUG:boto:Params: {} 2022-09-03T07:21:21.181 DEBUG:boto:Token: None 2022-09-03T07:21:21.182 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0= 2022-09-03T07:21:21.183 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0=', 'Content-Length': '0'} 2022-09-03T07:21:21.186 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ceb1f91bd1eeba99-0063130071-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.187 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:21.187 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.188 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.188 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:21.189 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:21.190 DEBUG:boto:Method: GET 2022-09-03T07:21:21.190 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:21.191 DEBUG:boto:Data: 2022-09-03T07:21:21.191 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.192 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.192 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.193 DEBUG:boto:Params: {} 2022-09-03T07:21:21.193 DEBUG:boto:Token: None 2022-09-03T07:21:21.194 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0= 2022-09-03T07:21:21.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0=', 'Content-Length': '0'} 2022-09-03T07:21:21.197 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003eb4ad97950fb98e-0063130071-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.198 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:21.199 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.199 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.200 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.200 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:21.201 DEBUG:boto:Data: 2022-09-03T07:21:21.201 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.202 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.202 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.203 DEBUG:boto:Params: {} 2022-09-03T07:21:21.203 DEBUG:boto:Token: None 2022-09-03T07:21:21.204 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95 2022-09-03T07:21:21.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ifg3PQ9feiz/EsJa/xY/wauTEE0= 2022-09-03T07:21:21.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ifg3PQ9feiz/EsJa/xY/wauTEE0=', 'Content-Length': '0'} 2022-09-03T07:21:21.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbe0c8d1d5c15733-0063130071-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.208 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.208 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.209 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.210 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:21.210 DEBUG:boto:Data: 2022-09-03T07:21:21.211 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.211 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.212 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.212 DEBUG:boto:Params: {} 2022-09-03T07:21:21.213 DEBUG:boto:Token: None 2022-09-03T07:21:21.213 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95 2022-09-03T07:21:21.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ifg3PQ9feiz/EsJa/xY/wauTEE0= 2022-09-03T07:21:21.214 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ifg3PQ9feiz/EsJa/xY/wauTEE0=', 'Content-Length': '0'} 2022-09-03T07:21:21.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006af0c0679c0e4a5a-0063130071-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.217 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:21:21.218 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.218 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.219 DEBUG:boto:Method: GET 2022-09-03T07:21:21.219 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:21.220 DEBUG:boto:Data: 2022-09-03T07:21:21.220 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.221 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.222 DEBUG:boto:Params: {} 2022-09-03T07:21:21.222 DEBUG:boto:Token: None 2022-09-03T07:21:21.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95 2022-09-03T07:21:21.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qJZICj6inNI1h0cUc4FKFA2+vuU= 2022-09-03T07:21:21.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qJZICj6inNI1h0cUc4FKFA2+vuU=', 'Content-Length': '0'} 2022-09-03T07:21:21.226 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002059a391389d1f64-0063130071-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.227 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:21:21.228 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:21:21.228 DEBUG:boto:Method: GET 2022-09-03T07:21:21.229 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:21:21.229 DEBUG:boto:Data: 2022-09-03T07:21:21.230 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.230 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.231 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.231 DEBUG:boto:Params: {} 2022-09-03T07:21:21.232 DEBUG:boto:Token: None 2022-09-03T07:21:21.232 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj95 2022-09-03T07:21:21.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qJZICj6inNI1h0cUc4FKFA2+vuU= 2022-09-03T07:21:21.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qJZICj6inNI1h0cUc4FKFA2+vuU=', 'Content-Length': '0'} 2022-09-03T07:21:21.236 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eed119ed08b98c9b-0063130071-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.236 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.237 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.238 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:21.238 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:21.239 DEBUG:boto:Method: GET 2022-09-03T07:21:21.239 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:21.240 DEBUG:boto:Data: 2022-09-03T07:21:21.240 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.241 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:21:21.241 DEBUG:boto:Port: 8001 2022-09-03T07:21:21.242 DEBUG:boto:Params: {} 2022-09-03T07:21:21.242 DEBUG:boto:Token: None 2022-09-03T07:21:21.243 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0= 2022-09-03T07:21:21.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0=', 'Content-Length': '0'} 2022-09-03T07:21:21.247 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000893b0272dd51e68c-0063130071-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.294 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:21.295 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.296 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.296 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:21.297 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:21:21.297 DEBUG:boto:Method: GET 2022-09-03T07:21:21.298 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:21:21.298 DEBUG:boto:Data: 2022-09-03T07:21:21.299 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.299 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:21:21.300 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.300 DEBUG:boto:Params: {} 2022-09-03T07:21:21.301 DEBUG:boto:Token: None 2022-09-03T07:21:21.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0= 2022-09-03T07:21:21.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2oI1VrBzvo39j5Q8hfRLlbELQJ0=', 'Content-Length': '0'} 2022-09-03T07:21:21.305 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000479148ba083783c5-0063130071-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.306 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:21:21.307 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:21:21.307 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '23', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:21:21.308 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '23', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:21:21.309 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --num-shards 23 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:21:21.371 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:21:21.372 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:21:21.425 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:21:21.426 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-75 2022-09-03T07:21:21.435 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 15 2022-09-03T07:21:21.521 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.522 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.522 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.523 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:21.523 DEBUG:boto:Data: 2022-09-03T07:21:21.524 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.525 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.525 DEBUG:boto:Params: {} 2022-09-03T07:21:21.526 DEBUG:boto:Token: None 2022-09-03T07:21:21.527 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo= 2022-09-03T07:21:21.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo=', 'Content-Length': '0'} 2022-09-03T07:21:21.532 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '15'), ('X-RGW-Bytes-Used', '45'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008e9d048e92ba9a84-0063130071-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.532 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:21:21.533 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:21:21.533 DEBUG:boto:Method: PUT 2022-09-03T07:21:21.534 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:21:21.534 DEBUG:boto:Data: 2022-09-03T07:21:21.535 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:21:21.536 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.536 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.537 DEBUG:boto:Params: {} 2022-09-03T07:21:21.537 DEBUG:boto:Token: None 2022-09-03T07:21:21.538 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj23 2022-09-03T07:21:21.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J2kVf9PKEpRmKkxmccchIeVaoOM= 2022-09-03T07:21:21.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J2kVf9PKEpRmKkxmccchIeVaoOM='} 2022-09-03T07:21:21.548 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000d6a25df32105ede3-0063130071-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.549 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.550 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.551 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.551 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:21.552 DEBUG:boto:Data: 2022-09-03T07:21:21.553 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.554 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.555 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.555 DEBUG:boto:Params: {} 2022-09-03T07:21:21.556 DEBUG:boto:Token: None 2022-09-03T07:21:21.556 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.557 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo= 2022-09-03T07:21:21.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo=', 'Content-Length': '0'} 2022-09-03T07:21:21.561 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '16'), ('X-RGW-Bytes-Used', '48'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c432f4dfca0c80d7-0063130071-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.561 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:21:21.562 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:21:21.562 DEBUG:boto:Method: PUT 2022-09-03T07:21:21.563 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:21:21.564 DEBUG:boto:Data: 2022-09-03T07:21:21.564 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:21:21.565 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.565 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.565 DEBUG:boto:Params: {} 2022-09-03T07:21:21.566 DEBUG:boto:Token: None 2022-09-03T07:21:21.567 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj46 2022-09-03T07:21:21.567 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3U+9QsUicFKQNPgkUBlB2x0Dosw= 2022-09-03T07:21:21.568 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3U+9QsUicFKQNPgkUBlB2x0Dosw='} 2022-09-03T07:21:21.582 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000009f8e613bfaec803-0063130071-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.582 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.583 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.583 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.584 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:21.584 DEBUG:boto:Data: 2022-09-03T07:21:21.585 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.585 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.586 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.586 DEBUG:boto:Params: {} 2022-09-03T07:21:21.587 DEBUG:boto:Token: None 2022-09-03T07:21:21.588 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo= 2022-09-03T07:21:21.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo=', 'Content-Length': '0'} 2022-09-03T07:21:21.592 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '17'), ('X-RGW-Bytes-Used', '51'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ca5a7707c9129eb7-0063130071-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.593 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:21:21.594 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:21:21.595 DEBUG:boto:Method: PUT 2022-09-03T07:21:21.596 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:21:21.597 DEBUG:boto:Data: 2022-09-03T07:21:21.598 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:21:21.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.599 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.600 DEBUG:boto:Params: {} 2022-09-03T07:21:21.601 DEBUG:boto:Token: None 2022-09-03T07:21:21.602 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj69 2022-09-03T07:21:21.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/ox9m2mRD5X97JExrknYJM0Kks= 2022-09-03T07:21:21.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/ox9m2mRD5X97JExrknYJM0Kks='} 2022-09-03T07:21:21.646 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f4c8929665013039-0063130071-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.647 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.648 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.648 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.649 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:21.649 DEBUG:boto:Data: 2022-09-03T07:21:21.649 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.650 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.650 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.651 DEBUG:boto:Params: {} 2022-09-03T07:21:21.652 DEBUG:boto:Token: None 2022-09-03T07:21:21.652 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo= 2022-09-03T07:21:21.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo=', 'Content-Length': '0'} 2022-09-03T07:21:21.657 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '18'), ('X-RGW-Bytes-Used', '54'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005f73cf885aae3f76-0063130071-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.658 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:21:21.658 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:21:21.659 DEBUG:boto:Method: PUT 2022-09-03T07:21:21.659 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:21:21.660 DEBUG:boto:Data: 2022-09-03T07:21:21.660 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:21:21.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.661 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.662 DEBUG:boto:Params: {} 2022-09-03T07:21:21.662 DEBUG:boto:Token: None 2022-09-03T07:21:21.663 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj92 2022-09-03T07:21:21.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S2eFFs+hDAj7b2gYaF5hsY6rGOA= 2022-09-03T07:21:21.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S2eFFs+hDAj7b2gYaF5hsY6rGOA='} 2022-09-03T07:21:21.671 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000006848ae1bd61c19a8-0063130071-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.672 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:21:21.672 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:21:21.673 DEBUG:boto:Method: HEAD 2022-09-03T07:21:21.673 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:21:21.674 DEBUG:boto:Data: 2022-09-03T07:21:21.674 DEBUG:boto:Headers: {} 2022-09-03T07:21:21.675 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.675 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.676 DEBUG:boto:Params: {} 2022-09-03T07:21:21.676 DEBUG:boto:Token: None 2022-09-03T07:21:21.677 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/ 2022-09-03T07:21:21.677 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo= 2022-09-03T07:21:21.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6mlIgrI0aBYQpyfNPK5QQqZ+cdo=', 'Content-Length': '0'} 2022-09-03T07:21:21.682 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '19'), ('X-RGW-Bytes-Used', '57'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000041ceaa4e46e46ea4-0063130071-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.683 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:21:21.684 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:21:21.685 DEBUG:boto:Method: PUT 2022-09-03T07:21:21.685 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:21:21.686 DEBUG:boto:Data: 2022-09-03T07:21:21.687 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:21:21.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:21:21.689 DEBUG:boto:Port: 8000 2022-09-03T07:21:21.690 DEBUG:boto:Params: {} 2022-09-03T07:21:21.691 DEBUG:boto:Token: None 2022-09-03T07:21:21.692 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:21:21 GMT /avysmw-75/obj115 2022-09-03T07:21:21.693 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1wHfyUhRpKvjibMk3ueK3nHYYKg= 2022-09-03T07:21:21.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:21:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1wHfyUhRpKvjibMk3ueK3nHYYKg='} 2022-09-03T07:21:21.701 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000003606b16c0aba47ee-0063130071-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:21:21.702 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-75 2022-09-03T07:21:21.703 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:21.704 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:21:21.705 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:21:21.763 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:21:21.763 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:21:21.823 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:21.821+0000 7ff43475f600 1 waiting to reach latest gen 3, on 2.. 2022-09-03T07:21:26.823 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:26.822+0000 7ff43475f600 1 waiting to reach latest gen 3, on 2.. 2022-09-03T07:21:31.824 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:31.822+0000 7ff43475f600 1 waiting to reach latest gen 3, on 2.. 2022-09-03T07:21:36.826 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:36.824+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:21:36.827 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:21:36.827 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:21:41.828 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:41.826+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:21:41.828 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:21:41.829 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:21:46.829 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:46.827+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:21:46.830 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:21:46.830 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:21:51.831 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:51.829+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:21:51.832 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:21:51.832 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:21:56.832 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:21:56.830+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:21:56.833 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , 00000000001.64.5, , , , ] 2022-09-03T07:21:56.834 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:01.834 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:01.832+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:01.834 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , 00000000001.64.5, , , , ] 2022-09-03T07:22:01.835 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:06.837 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:06.835+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:06.838 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , 00000000001.64.5, , , , ] 2022-09-03T07:22:06.838 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:11.838 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:11.836+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:11.839 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , 00000000001.64.5, , , , ] 2022-09-03T07:22:11.839 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:16.844 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:16.841+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:16.845 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , 00000000001.64.5, , , , ] 2022-09-03T07:22:16.845 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:21.844 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:21.842+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:21.845 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , 00000000001.64.5, , , , ] 2022-09-03T07:22:21.846 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:26.848 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:26.846+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:26.849 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , 00000000001.64.5, , , , ] 2022-09-03T07:22:26.849 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:31.850 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:31.848+0000 7ff43475f600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:31.850 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , 00000000001.64.5, , , , ] 2022-09-03T07:22:31.851 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:36.851 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:36.850+0000 7ff43475f600 1 bucket sync caught up with source: 2022-09-03T07:22:36.852 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:36.852 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:36.852 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:36.850+0000 7ff43475f600 0 bucket checkpoint complete 2022-09-03T07:22:36.891 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-75 2022-09-03T07:22:36.893 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:22:36.893 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:22:36.894 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:22:36.962 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:22:36.963 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:22:37.003 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:22:37.001+0000 7f2dd1ffb600 1 bucket sync caught up with source: 2022-09-03T07:22:37.003 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:37.004 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:37.004 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:22:37.001+0000 7f2dd1ffb600 0 bucket checkpoint complete 2022-09-03T07:22:37.018 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-75 2022-09-03T07:22:37.018 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:22:37.019 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:22:37.020 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:22:37.158 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:22:37.159 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:22:37.224 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:37.224+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:37.225 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:22:37.225 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:42.227 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:42.225+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:42.228 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:22:42.229 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:47.229 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:47.227+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:47.230 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , , , , , ] 2022-09-03T07:22:47.230 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:52.230 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:52.228+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:52.231 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , , , , , ] 2022-09-03T07:22:52.231 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:22:57.233 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:22:57.231+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:22:57.233 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , , , , , ] 2022-09-03T07:22:57.234 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:02.234 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:02.233+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:02.235 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , , , , , , , ] 2022-09-03T07:23:02.235 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:07.237 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:07.234+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:07.237 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:07.238 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:12.238 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:12.236+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:12.238 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:12.239 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:17.241 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:17.237+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:17.242 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:17.243 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:22.243 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:22.239+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:22.244 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:22.245 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:27.243 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:27.241+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:27.243 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:27.243 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:32.248 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:32.246+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:32.249 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:32.250 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:37.250 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:37.248+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:37.251 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:37.251 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:42.251 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:42.250+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:42.252 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:42.252 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:47.253 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:47.251+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:47.254 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:47.254 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:52.255 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:52.253+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:52.255 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:52.256 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:57.256 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:23:57.255+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:23:57.257 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:23:57.257 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:24:02.259 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:02.257+0000 7fa8f14cf600 1 waiting for incremental sync to catch up: 2022-09-03T07:24:02.261 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , , , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:24:02.262 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:24:07.266 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:07.263+0000 7fa8f14cf600 1 bucket sync caught up with source: 2022-09-03T07:24:07.266 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:24:07.267 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , 00000000001.191.5, , , 00000000002.45.5, , 00000000002.63.5, , , , , , 00000000001.65.5, , 00000000001.64.5, , , , ] 2022-09-03T07:24:07.267 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:07.263+0000 7fa8f14cf600 0 bucket checkpoint complete 2022-09-03T07:24:07.286 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-75 2022-09-03T07:24:07.288 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:07.290 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:07.291 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:24:07.360 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:24:07.361 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:24:07.423 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:24:07.421+0000 7f3c97854600 1 bucket sync caught up with source: 2022-09-03T07:24:07.423 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.118.5, 00000000002.339.5, 00000000001.376.5, 00000000002.337.5, 00000000002.258.5, , , 00000000002.376.5, 00000000005.303.5, 00000000004.221.5] 2022-09-03T07:24:07.423 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.118.5, 00000000002.339.5, 00000000001.376.5, 00000000002.337.5, 00000000002.258.5, , , 00000000002.376.5, 00000000005.303.5, 00000000004.221.5] 2022-09-03T07:24:07.424 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:24:07.421+0000 7f3c97854600 0 bucket checkpoint complete 2022-09-03T07:24:07.443 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-75 2022-09-03T07:24:07.443 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:07.444 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:07.445 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:24:07.506 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:24:07.507 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:24:07.549 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:24:07.547+0000 7f030c5f4600 1 bucket sync caught up with source: 2022-09-03T07:24:07.549 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.118.5, 00000000002.339.5, 00000000001.376.5, 00000000002.337.5, 00000000002.258.5, , , 00000000002.376.5, 00000000005.303.5, 00000000004.221.5] 2022-09-03T07:24:07.550 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.118.5, 00000000002.339.5, 00000000001.376.5, 00000000002.337.5, 00000000002.258.5, , , 00000000002.376.5, 00000000005.303.5, 00000000004.221.5] 2022-09-03T07:24:07.550 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:24:07.547+0000 7f030c5f4600 0 bucket checkpoint complete 2022-09-03T07:24:07.562 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-75 2022-09-03T07:24:07.563 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-75 2022-09-03T07:24:07.564 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:07.565 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:07.566 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:24:07.626 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:24:07.626 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:24:07.678 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:24:07.677+0000 7f0438e72600 1 bucket sync caught up with source: 2022-09-03T07:24:07.679 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.116.5, 00000000002.195.5, 00000000001.185.5, 00000000002.193.5, 00000000002.145.5, , , 00000000002.287.5, 00000000005.313.5, 00000000004.140.5] 2022-09-03T07:24:07.679 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.116.5, 00000000002.195.5, 00000000001.185.5, 00000000002.193.5, 00000000002.145.5, , , 00000000002.287.5, 00000000005.313.5, 00000000004.140.5] 2022-09-03T07:24:07.680 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:24:07.677+0000 7f0438e72600 0 bucket checkpoint complete 2022-09-03T07:24:07.705 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-75 2022-09-03T07:24:07.706 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:07.706 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:07.707 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:24:07.774 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:24:07.774 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:24:07.832 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:07.830+0000 7efe67408600 1 bucket sync caught up with source: 2022-09-03T07:24:07.833 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.116.5, 00000000002.195.5, 00000000001.185.5, 00000000002.193.5, 00000000002.145.5, , , 00000000002.287.5, 00000000005.313.5, 00000000004.140.5] 2022-09-03T07:24:07.833 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.116.5, 00000000002.195.5, 00000000001.185.5, 00000000002.193.5, 00000000002.145.5, , , 00000000002.287.5, 00000000005.313.5, 00000000004.140.5] 2022-09-03T07:24:07.834 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:07.830+0000 7efe67408600 0 bucket checkpoint complete 2022-09-03T07:24:07.847 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-75 2022-09-03T07:24:07.848 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:24:07.851 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:07.852 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:07.853 DEBUG:boto:Method: HEAD 2022-09-03T07:24:07.854 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:07.855 DEBUG:boto:Data: 2022-09-03T07:24:07.856 DEBUG:boto:Headers: {} 2022-09-03T07:24:07.857 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:07.857 DEBUG:boto:Port: 8000 2022-09-03T07:24:07.858 DEBUG:boto:Params: {} 2022-09-03T07:24:07.860 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:24:07.860 DEBUG:boto:Token: None 2022-09-03T07:24:07.861 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:07 GMT /avysmw-75/ 2022-09-03T07:24:07.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/szP21HgPAu4U4apRv5pTlSMEpM= 2022-09-03T07:24:07.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/szP21HgPAu4U4apRv5pTlSMEpM=', 'Content-Length': '0'} 2022-09-03T07:24:07.869 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '20'), ('X-RGW-Bytes-Used', '60'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f838202278aec237-0063130117-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:07.870 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:07.870 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:07.871 DEBUG:boto:Method: HEAD 2022-09-03T07:24:07.872 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:07.872 DEBUG:boto:Data: 2022-09-03T07:24:07.873 DEBUG:boto:Headers: {} 2022-09-03T07:24:07.873 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:07.874 DEBUG:boto:Port: 8000 2022-09-03T07:24:07.874 DEBUG:boto:Params: {} 2022-09-03T07:24:07.875 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:24:07.876 DEBUG:boto:Token: None 2022-09-03T07:24:07.877 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:07 GMT /avysmw-75/ 2022-09-03T07:24:07.877 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/szP21HgPAu4U4apRv5pTlSMEpM= 2022-09-03T07:24:07.878 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/szP21HgPAu4U4apRv5pTlSMEpM=', 'Content-Length': '0'} 2022-09-03T07:24:07.883 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '20'), ('X-RGW-Bytes-Used', '60'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f6e28244c626c652-0063130117-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:07.884 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:24:07.885 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:07.885 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:07.886 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.886 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.887 DEBUG:boto:Method: GET 2022-09-03T07:24:07.887 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.888 DEBUG:boto:Data: 2022-09-03T07:24:07.889 DEBUG:boto:Headers: {} 2022-09-03T07:24:07.889 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:07.890 DEBUG:boto:Port: 8000 2022-09-03T07:24:07.890 DEBUG:boto:Params: {} 2022-09-03T07:24:07.891 DEBUG:boto:Token: None 2022-09-03T07:24:07.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:07 GMT /avysmw-75/?versions 2022-09-03T07:24:07.892 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l7PUI2AB9rRAvBzUSrjxMsnWJBk= 2022-09-03T07:24:07.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l7PUI2AB9rRAvBzUSrjxMsnWJBk=', 'Content-Length': '0'} 2022-09-03T07:24:07.896 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d75a69d9845117b-0063130117-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:07.896 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:07.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:24:07.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:24:07.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:24:07.901 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:24:07.902 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:24:07.902 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:24:07.903 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:24:07.903 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:24:07.904 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:24:07.904 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:24:07.905 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:24:07.905 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:24:07.906 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:24:07.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:24:07.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:24:07.908 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:24:07.908 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:24:07.909 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:24:07.909 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:24:07.910 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:24:07.911 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:24:07.911 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:07.912 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:07.913 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.913 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.914 DEBUG:boto:Method: GET 2022-09-03T07:24:07.915 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.915 DEBUG:boto:Data: 2022-09-03T07:24:07.916 DEBUG:boto:Headers: {} 2022-09-03T07:24:07.917 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:07.917 DEBUG:boto:Port: 8000 2022-09-03T07:24:07.918 DEBUG:boto:Params: {} 2022-09-03T07:24:07.919 DEBUG:boto:Token: None 2022-09-03T07:24:07.920 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:07 GMT /avysmw-75/?versions 2022-09-03T07:24:07.920 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l7PUI2AB9rRAvBzUSrjxMsnWJBk= 2022-09-03T07:24:07.921 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l7PUI2AB9rRAvBzUSrjxMsnWJBk=', 'Content-Length': '0'} 2022-09-03T07:24:07.925 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e3ee742fe865a029-0063130117-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:07.926 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:07.928 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:24:07.929 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:24:07.929 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:24:07.930 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:24:07.931 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:24:07.931 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:24:07.932 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:24:07.932 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:24:07.933 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:24:07.933 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:24:07.934 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:24:07.934 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:24:07.935 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:24:07.936 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:24:07.936 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:24:07.937 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:24:07.937 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:24:07.938 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:24:07.938 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:24:07.939 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:24:07.939 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:07.940 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:07.941 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.941 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.942 DEBUG:boto:Method: GET 2022-09-03T07:24:07.942 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.943 DEBUG:boto:Data: 2022-09-03T07:24:07.944 DEBUG:boto:Headers: {} 2022-09-03T07:24:07.944 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:07.945 DEBUG:boto:Port: 8000 2022-09-03T07:24:07.945 DEBUG:boto:Params: {} 2022-09-03T07:24:07.946 DEBUG:boto:Token: None 2022-09-03T07:24:07.947 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:07 GMT /avysmw-75/?versions 2022-09-03T07:24:07.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l7PUI2AB9rRAvBzUSrjxMsnWJBk= 2022-09-03T07:24:07.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l7PUI2AB9rRAvBzUSrjxMsnWJBk=', 'Content-Length': '0'} 2022-09-03T07:24:07.951 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000064e2ec2d48753d6-0063130117-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:07.952 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:07.954 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:07.955 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:07.956 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.956 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.957 DEBUG:boto:Method: GET 2022-09-03T07:24:07.957 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:07.958 DEBUG:boto:Data: 2022-09-03T07:24:07.959 DEBUG:boto:Headers: {} 2022-09-03T07:24:07.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:07.960 DEBUG:boto:Port: 8000 2022-09-03T07:24:07.960 DEBUG:boto:Params: {} 2022-09-03T07:24:07.961 DEBUG:boto:Token: None 2022-09-03T07:24:07.962 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:07 GMT /avysmw-75/?versions 2022-09-03T07:24:07.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l7PUI2AB9rRAvBzUSrjxMsnWJBk= 2022-09-03T07:24:07.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l7PUI2AB9rRAvBzUSrjxMsnWJBk=', 'Content-Length': '0'} 2022-09-03T07:24:07.967 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c6f782acab6d67f0-0063130117-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:07.968 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:07.970 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:24:07.970 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:07.971 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:07.972 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.972 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.973 DEBUG:boto:Method: GET 2022-09-03T07:24:07.973 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.974 DEBUG:boto:Data: 2022-09-03T07:24:07.974 DEBUG:boto:Headers: {} 2022-09-03T07:24:07.975 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:07.975 DEBUG:boto:Port: 8000 2022-09-03T07:24:07.976 DEBUG:boto:Params: {} 2022-09-03T07:24:07.977 DEBUG:boto:Token: None 2022-09-03T07:24:07.977 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:07 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5P0I6uJmnfoabMPynMQcxYSJHc= 2022-09-03T07:24:07.979 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5P0I6uJmnfoabMPynMQcxYSJHc=', 'Content-Length': '0'} 2022-09-03T07:24:07.982 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c567fdcf46f6b99-0063130117-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:07.983 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:07.984 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:07.984 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.985 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.986 DEBUG:boto:Method: GET 2022-09-03T07:24:07.986 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.987 DEBUG:boto:Data: 2022-09-03T07:24:07.988 DEBUG:boto:Headers: {} 2022-09-03T07:24:07.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:07.989 DEBUG:boto:Port: 8000 2022-09-03T07:24:07.990 DEBUG:boto:Params: {} 2022-09-03T07:24:07.990 DEBUG:boto:Token: None 2022-09-03T07:24:07.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:07 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5P0I6uJmnfoabMPynMQcxYSJHc= 2022-09-03T07:24:07.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5P0I6uJmnfoabMPynMQcxYSJHc=', 'Content-Length': '0'} 2022-09-03T07:24:07.996 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020c5ae2b2144fe57-0063130117-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:07.997 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:07.998 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:07.998 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.999 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:07.999 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.000 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.000 DEBUG:boto:Data: 2022-09-03T07:24:08.001 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.001 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.002 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.002 DEBUG:boto:Params: {} 2022-09-03T07:24:08.003 DEBUG:boto:Token: None 2022-09-03T07:24:08.004 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wf/rgDakckZIrztXNDTLR9pH2pw= 2022-09-03T07:24:08.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wf/rgDakckZIrztXNDTLR9pH2pw=', 'Content-Length': '0'} 2022-09-03T07:24:08.007 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016e8a1f22a1e40bf-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.008 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:08.009 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:08.009 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.010 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.010 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.011 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.012 DEBUG:boto:Data: 2022-09-03T07:24:08.012 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.013 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.014 DEBUG:boto:Params: {} 2022-09-03T07:24:08.014 DEBUG:boto:Token: None 2022-09-03T07:24:08.015 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wf/rgDakckZIrztXNDTLR9pH2pw= 2022-09-03T07:24:08.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wf/rgDakckZIrztXNDTLR9pH2pw=', 'Content-Length': '0'} 2022-09-03T07:24:08.018 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034b002b4cc4420a9-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.019 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:24:08.019 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:08.020 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:08.021 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.021 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.022 DEBUG:boto:Method: GET 2022-09-03T07:24:08.022 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.023 DEBUG:boto:Data: 2022-09-03T07:24:08.023 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.024 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.024 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.025 DEBUG:boto:Params: {} 2022-09-03T07:24:08.026 DEBUG:boto:Token: None 2022-09-03T07:24:08.026 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tez9QTPtno5UpUEgRWK6gNrv4us= 2022-09-03T07:24:08.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tez9QTPtno5UpUEgRWK6gNrv4us=', 'Content-Length': '0'} 2022-09-03T07:24:08.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000847cd19eab5b06c1-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.032 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:08.033 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:08.033 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.034 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.034 DEBUG:boto:Method: GET 2022-09-03T07:24:08.035 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.036 DEBUG:boto:Data: 2022-09-03T07:24:08.037 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.037 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.038 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.038 DEBUG:boto:Params: {} 2022-09-03T07:24:08.039 DEBUG:boto:Token: None 2022-09-03T07:24:08.040 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:08.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tez9QTPtno5UpUEgRWK6gNrv4us= 2022-09-03T07:24:08.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tez9QTPtno5UpUEgRWK6gNrv4us=', 'Content-Length': '0'} 2022-09-03T07:24:08.043 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b243d6ac3e893f4-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.044 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.045 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.045 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:08.046 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:08.047 DEBUG:boto:Method: GET 2022-09-03T07:24:08.047 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:08.048 DEBUG:boto:Data: 2022-09-03T07:24:08.048 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.049 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.049 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.050 DEBUG:boto:Params: {} 2022-09-03T07:24:08.051 DEBUG:boto:Token: None 2022-09-03T07:24:08.051 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.055 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ad9697803d24878-0063130118-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.056 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.057 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.057 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.058 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:08.058 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:08.059 DEBUG:boto:Method: GET 2022-09-03T07:24:08.060 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:08.060 DEBUG:boto:Data: 2022-09-03T07:24:08.061 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.062 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.062 DEBUG:boto:Params: {} 2022-09-03T07:24:08.063 DEBUG:boto:Token: None 2022-09-03T07:24:08.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.068 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c8a8bbe758f0e415-0063130118-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.110 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.111 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:08.112 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:08.112 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.113 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:08.113 DEBUG:boto:Data: 2022-09-03T07:24:08.114 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.114 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.115 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.115 DEBUG:boto:Params: {} 2022-09-03T07:24:08.116 DEBUG:boto:Token: None 2022-09-03T07:24:08.117 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj11 2022-09-03T07:24:08.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiG1Ox1l7Vqzpdj/W6s6tAeMp/k= 2022-09-03T07:24:08.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiG1Ox1l7Vqzpdj/W6s6tAeMp/k=', 'Content-Length': '0'} 2022-09-03T07:24:08.120 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae75264c6078f7c0-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.121 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:08.121 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:08.122 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.123 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:08.124 DEBUG:boto:Data: 2022-09-03T07:24:08.125 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.126 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.127 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.129 DEBUG:boto:Params: {} 2022-09-03T07:24:08.130 DEBUG:boto:Token: None 2022-09-03T07:24:08.130 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj11 2022-09-03T07:24:08.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RiG1Ox1l7Vqzpdj/W6s6tAeMp/k= 2022-09-03T07:24:08.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RiG1Ox1l7Vqzpdj/W6s6tAeMp/k=', 'Content-Length': '0'} 2022-09-03T07:24:08.133 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5e1192622f26e52-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.134 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:24:08.135 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:08.135 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:08.136 DEBUG:boto:Method: GET 2022-09-03T07:24:08.136 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:08.137 DEBUG:boto:Data: 2022-09-03T07:24:08.138 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.138 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.139 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.141 DEBUG:boto:Params: {} 2022-09-03T07:24:08.142 DEBUG:boto:Token: None 2022-09-03T07:24:08.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj11 2022-09-03T07:24:08.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6zWKqqe1w+rdkrCmOqhTNFlUE0= 2022-09-03T07:24:08.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6zWKqqe1w+rdkrCmOqhTNFlUE0=', 'Content-Length': '0'} 2022-09-03T07:24:08.149 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2301616fd36db33-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.150 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:08.151 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:08.152 DEBUG:boto:Method: GET 2022-09-03T07:24:08.153 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:08.156 DEBUG:boto:Data: 2022-09-03T07:24:08.157 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.159 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.160 DEBUG:boto:Params: {} 2022-09-03T07:24:08.162 DEBUG:boto:Token: None 2022-09-03T07:24:08.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj11 2022-09-03T07:24:08.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6zWKqqe1w+rdkrCmOqhTNFlUE0= 2022-09-03T07:24:08.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6zWKqqe1w+rdkrCmOqhTNFlUE0=', 'Content-Length': '0'} 2022-09-03T07:24:08.168 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000cb76c8d89921818-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.170 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.171 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.172 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:08.173 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:08.174 DEBUG:boto:Method: GET 2022-09-03T07:24:08.175 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:08.176 DEBUG:boto:Data: 2022-09-03T07:24:08.177 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.178 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.179 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.180 DEBUG:boto:Params: {} 2022-09-03T07:24:08.181 DEBUG:boto:Token: None 2022-09-03T07:24:08.182 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.183 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.187 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f890e87000e0c7d-0063130118-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.188 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.189 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.190 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.190 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:08.191 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:08.191 DEBUG:boto:Method: GET 2022-09-03T07:24:08.192 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:08.192 DEBUG:boto:Data: 2022-09-03T07:24:08.193 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.193 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.194 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.195 DEBUG:boto:Params: {} 2022-09-03T07:24:08.195 DEBUG:boto:Token: None 2022-09-03T07:24:08.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.202 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c19a5d37f455af9d-0063130118-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.203 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.205 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:24:08.206 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.207 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.208 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.209 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.210 DEBUG:boto:Method: GET 2022-09-03T07:24:08.212 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.213 DEBUG:boto:Data: 2022-09-03T07:24:08.214 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.215 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.215 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.216 DEBUG:boto:Params: {} 2022-09-03T07:24:08.216 DEBUG:boto:Token: None 2022-09-03T07:24:08.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAn/8Sr+gj2KdCeeO8sy6K8M1SI= 2022-09-03T07:24:08.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAn/8Sr+gj2KdCeeO8sy6K8M1SI=', 'Content-Length': '0'} 2022-09-03T07:24:08.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c9ff98582eec648-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.222 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.222 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.223 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.223 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.224 DEBUG:boto:Method: GET 2022-09-03T07:24:08.225 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.225 DEBUG:boto:Data: 2022-09-03T07:24:08.226 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.226 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.227 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.228 DEBUG:boto:Params: {} 2022-09-03T07:24:08.228 DEBUG:boto:Token: None 2022-09-03T07:24:08.229 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAn/8Sr+gj2KdCeeO8sy6K8M1SI= 2022-09-03T07:24:08.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAn/8Sr+gj2KdCeeO8sy6K8M1SI=', 'Content-Length': '0'} 2022-09-03T07:24:08.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b5e5ffe94695590-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.234 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.234 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.235 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.235 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.236 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.236 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.237 DEBUG:boto:Data: 2022-09-03T07:24:08.237 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.239 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.239 DEBUG:boto:Params: {} 2022-09-03T07:24:08.240 DEBUG:boto:Token: None 2022-09-03T07:24:08.240 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IEPCmRskZ2zF56GUAADUGF75Pz0= 2022-09-03T07:24:08.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IEPCmRskZ2zF56GUAADUGF75Pz0=', 'Content-Length': '0'} 2022-09-03T07:24:08.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a810dc4cb59d8d31-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.244 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.245 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.246 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.248 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.249 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.250 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.250 DEBUG:boto:Data: 2022-09-03T07:24:08.251 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.252 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.253 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.254 DEBUG:boto:Params: {} 2022-09-03T07:24:08.254 DEBUG:boto:Token: None 2022-09-03T07:24:08.255 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IEPCmRskZ2zF56GUAADUGF75Pz0= 2022-09-03T07:24:08.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IEPCmRskZ2zF56GUAADUGF75Pz0=', 'Content-Length': '0'} 2022-09-03T07:24:08.259 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a71d790ce3791bb7-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.260 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:24:08.261 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.262 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.263 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.264 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.265 DEBUG:boto:Method: GET 2022-09-03T07:24:08.266 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.267 DEBUG:boto:Data: 2022-09-03T07:24:08.268 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.268 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.269 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.270 DEBUG:boto:Params: {} 2022-09-03T07:24:08.271 DEBUG:boto:Token: None 2022-09-03T07:24:08.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAn/8Sr+gj2KdCeeO8sy6K8M1SI= 2022-09-03T07:24:08.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAn/8Sr+gj2KdCeeO8sy6K8M1SI=', 'Content-Length': '0'} 2022-09-03T07:24:08.275 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c21bfe516f3e4e01-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.276 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.276 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.277 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.277 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.278 DEBUG:boto:Method: GET 2022-09-03T07:24:08.279 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.280 DEBUG:boto:Data: 2022-09-03T07:24:08.281 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.283 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.284 DEBUG:boto:Params: {} 2022-09-03T07:24:08.284 DEBUG:boto:Token: None 2022-09-03T07:24:08.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:08.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NAn/8Sr+gj2KdCeeO8sy6K8M1SI= 2022-09-03T07:24:08.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NAn/8Sr+gj2KdCeeO8sy6K8M1SI=', 'Content-Length': '0'} 2022-09-03T07:24:08.288 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f693df8cf24b39f-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.289 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.290 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.291 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:08.291 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:08.292 DEBUG:boto:Method: GET 2022-09-03T07:24:08.292 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:08.293 DEBUG:boto:Data: 2022-09-03T07:24:08.294 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.295 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.296 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.296 DEBUG:boto:Params: {} 2022-09-03T07:24:08.297 DEBUG:boto:Token: None 2022-09-03T07:24:08.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.303 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc1961ed9c768b0d-0063130118-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.346 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.347 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.348 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.348 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:08.349 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:08.349 DEBUG:boto:Method: GET 2022-09-03T07:24:08.350 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:08.350 DEBUG:boto:Data: 2022-09-03T07:24:08.351 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.352 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.352 DEBUG:boto:Params: {} 2022-09-03T07:24:08.353 DEBUG:boto:Token: None 2022-09-03T07:24:08.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.358 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bf490b492be6b125-0063130118-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.359 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.360 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.360 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.361 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.362 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:08.362 DEBUG:boto:Data: 2022-09-03T07:24:08.363 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.364 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.364 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.365 DEBUG:boto:Params: {} 2022-09-03T07:24:08.366 DEBUG:boto:Token: None 2022-09-03T07:24:08.366 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115 2022-09-03T07:24:08.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sPXkdvX/j4SHtgLMovuArgSSFnY= 2022-09-03T07:24:08.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sPXkdvX/j4SHtgLMovuArgSSFnY=', 'Content-Length': '0'} 2022-09-03T07:24:08.370 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b230b6a66f97bd4-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.371 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.372 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.372 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.373 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:08.373 DEBUG:boto:Data: 2022-09-03T07:24:08.374 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.374 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.375 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.375 DEBUG:boto:Params: {} 2022-09-03T07:24:08.376 DEBUG:boto:Token: None 2022-09-03T07:24:08.377 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115 2022-09-03T07:24:08.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sPXkdvX/j4SHtgLMovuArgSSFnY= 2022-09-03T07:24:08.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sPXkdvX/j4SHtgLMovuArgSSFnY=', 'Content-Length': '0'} 2022-09-03T07:24:08.389 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce3bbd76af3bcbc8-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.390 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:24:08.390 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.391 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.391 DEBUG:boto:Method: GET 2022-09-03T07:24:08.392 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:08.392 DEBUG:boto:Data: 2022-09-03T07:24:08.393 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.393 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.394 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.395 DEBUG:boto:Params: {} 2022-09-03T07:24:08.395 DEBUG:boto:Token: None 2022-09-03T07:24:08.396 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115 2022-09-03T07:24:08.396 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ifhYBishvom6RnKjBCkUtxpnHcU= 2022-09-03T07:24:08.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ifhYBishvom6RnKjBCkUtxpnHcU=', 'Content-Length': '0'} 2022-09-03T07:24:08.399 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c3330c6b8c4083a-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.400 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:08.400 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:08.401 DEBUG:boto:Method: GET 2022-09-03T07:24:08.401 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:08.402 DEBUG:boto:Data: 2022-09-03T07:24:08.403 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.403 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.404 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.404 DEBUG:boto:Params: {} 2022-09-03T07:24:08.405 DEBUG:boto:Token: None 2022-09-03T07:24:08.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj115 2022-09-03T07:24:08.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ifhYBishvom6RnKjBCkUtxpnHcU= 2022-09-03T07:24:08.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ifhYBishvom6RnKjBCkUtxpnHcU=', 'Content-Length': '0'} 2022-09-03T07:24:08.410 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec6c42c2bd78985d-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.454 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.455 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.456 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:08.457 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:08.459 DEBUG:boto:Method: GET 2022-09-03T07:24:08.460 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:08.461 DEBUG:boto:Data: 2022-09-03T07:24:08.462 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.463 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.464 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.465 DEBUG:boto:Params: {} 2022-09-03T07:24:08.466 DEBUG:boto:Token: None 2022-09-03T07:24:08.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.471 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d73ea9ac48f3bea3-0063130118-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.472 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.473 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.474 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.475 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:08.475 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:08.476 DEBUG:boto:Method: GET 2022-09-03T07:24:08.476 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:08.477 DEBUG:boto:Data: 2022-09-03T07:24:08.478 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.479 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.480 DEBUG:boto:Params: {} 2022-09-03T07:24:08.481 DEBUG:boto:Token: None 2022-09-03T07:24:08.482 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.489 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000284a7c7d433c9fc1-0063130118-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.491 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.492 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:24:08.493 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.494 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.494 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.496 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.497 DEBUG:boto:Method: GET 2022-09-03T07:24:08.498 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.498 DEBUG:boto:Data: 2022-09-03T07:24:08.499 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.500 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.500 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.501 DEBUG:boto:Params: {} 2022-09-03T07:24:08.501 DEBUG:boto:Token: None 2022-09-03T07:24:08.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fPL+EPnFaMmg++jTit+/jOdre40= 2022-09-03T07:24:08.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fPL+EPnFaMmg++jTit+/jOdre40=', 'Content-Length': '0'} 2022-09-03T07:24:08.506 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a42c67472db1e51c-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.507 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.507 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.508 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.509 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.509 DEBUG:boto:Method: GET 2022-09-03T07:24:08.510 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.510 DEBUG:boto:Data: 2022-09-03T07:24:08.511 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.511 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.512 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.513 DEBUG:boto:Params: {} 2022-09-03T07:24:08.513 DEBUG:boto:Token: None 2022-09-03T07:24:08.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fPL+EPnFaMmg++jTit+/jOdre40= 2022-09-03T07:24:08.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fPL+EPnFaMmg++jTit+/jOdre40=', 'Content-Length': '0'} 2022-09-03T07:24:08.517 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8726777dc071b12-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.518 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.519 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.519 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.520 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.520 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.521 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.521 DEBUG:boto:Data: 2022-09-03T07:24:08.522 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.523 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.523 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.524 DEBUG:boto:Params: {} 2022-09-03T07:24:08.524 DEBUG:boto:Token: None 2022-09-03T07:24:08.525 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.525 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kRu2i3Hp8EDBMR3ACaH7xygv9kE= 2022-09-03T07:24:08.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kRu2i3Hp8EDBMR3ACaH7xygv9kE=', 'Content-Length': '0'} 2022-09-03T07:24:08.528 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8f31ae03d091f3c-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.529 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.530 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.530 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.531 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.531 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.532 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.532 DEBUG:boto:Data: 2022-09-03T07:24:08.533 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.533 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.534 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.534 DEBUG:boto:Params: {} 2022-09-03T07:24:08.535 DEBUG:boto:Token: None 2022-09-03T07:24:08.535 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.536 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kRu2i3Hp8EDBMR3ACaH7xygv9kE= 2022-09-03T07:24:08.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kRu2i3Hp8EDBMR3ACaH7xygv9kE=', 'Content-Length': '0'} 2022-09-03T07:24:08.538 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb1020b9536c35c8-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.539 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:24:08.540 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.540 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.541 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.541 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.542 DEBUG:boto:Method: GET 2022-09-03T07:24:08.543 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.543 DEBUG:boto:Data: 2022-09-03T07:24:08.544 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.544 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.545 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.545 DEBUG:boto:Params: {} 2022-09-03T07:24:08.546 DEBUG:boto:Token: None 2022-09-03T07:24:08.546 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fPL+EPnFaMmg++jTit+/jOdre40= 2022-09-03T07:24:08.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fPL+EPnFaMmg++jTit+/jOdre40=', 'Content-Length': '0'} 2022-09-03T07:24:08.550 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002403df35e4578e33-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.551 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.551 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.552 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.552 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.553 DEBUG:boto:Method: GET 2022-09-03T07:24:08.553 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.554 DEBUG:boto:Data: 2022-09-03T07:24:08.555 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.556 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.556 DEBUG:boto:Params: {} 2022-09-03T07:24:08.557 DEBUG:boto:Token: None 2022-09-03T07:24:08.557 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:08.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fPL+EPnFaMmg++jTit+/jOdre40= 2022-09-03T07:24:08.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fPL+EPnFaMmg++jTit+/jOdre40=', 'Content-Length': '0'} 2022-09-03T07:24:08.561 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000909b18462a43361a-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.602 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.604 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.604 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:08.605 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:08.605 DEBUG:boto:Method: GET 2022-09-03T07:24:08.606 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:08.606 DEBUG:boto:Data: 2022-09-03T07:24:08.607 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.608 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.609 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.610 DEBUG:boto:Params: {} 2022-09-03T07:24:08.611 DEBUG:boto:Token: None 2022-09-03T07:24:08.612 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.620 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004b1722068b1a7cff-0063130118-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.621 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.621 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.622 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.623 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:08.624 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:08.624 DEBUG:boto:Method: GET 2022-09-03T07:24:08.625 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:08.625 DEBUG:boto:Data: 2022-09-03T07:24:08.626 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.627 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.627 DEBUG:boto:Params: {} 2022-09-03T07:24:08.628 DEBUG:boto:Token: None 2022-09-03T07:24:08.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.633 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001d1b9e998c250606-0063130118-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.634 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.635 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.635 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.636 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.636 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:08.637 DEBUG:boto:Data: 2022-09-03T07:24:08.637 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.638 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.638 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.639 DEBUG:boto:Params: {} 2022-09-03T07:24:08.640 DEBUG:boto:Token: None 2022-09-03T07:24:08.640 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17 2022-09-03T07:24:08.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N7nhJlxxeot4PVzRcY+TaRcNWG0= 2022-09-03T07:24:08.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N7nhJlxxeot4PVzRcY+TaRcNWG0=', 'Content-Length': '0'} 2022-09-03T07:24:08.643 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a72f99f4af8d56da-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.644 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.644 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.645 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.645 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:08.646 DEBUG:boto:Data: 2022-09-03T07:24:08.646 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.647 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.648 DEBUG:boto:Params: {} 2022-09-03T07:24:08.649 DEBUG:boto:Token: None 2022-09-03T07:24:08.649 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17 2022-09-03T07:24:08.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N7nhJlxxeot4PVzRcY+TaRcNWG0= 2022-09-03T07:24:08.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N7nhJlxxeot4PVzRcY+TaRcNWG0=', 'Content-Length': '0'} 2022-09-03T07:24:08.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079ba08d1624b8629-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.653 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:24:08.654 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.654 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.655 DEBUG:boto:Method: GET 2022-09-03T07:24:08.655 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:08.656 DEBUG:boto:Data: 2022-09-03T07:24:08.656 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.657 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.657 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.658 DEBUG:boto:Params: {} 2022-09-03T07:24:08.659 DEBUG:boto:Token: None 2022-09-03T07:24:08.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17 2022-09-03T07:24:08.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oEt/tvg9B4ANH8KArZkn5PG/R2A= 2022-09-03T07:24:08.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oEt/tvg9B4ANH8KArZkn5PG/R2A=', 'Content-Length': '0'} 2022-09-03T07:24:08.663 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ace977d5f3d1e469-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.710 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:08.711 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:08.712 DEBUG:boto:Method: GET 2022-09-03T07:24:08.712 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:08.713 DEBUG:boto:Data: 2022-09-03T07:24:08.713 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.714 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.715 DEBUG:boto:Params: {} 2022-09-03T07:24:08.716 DEBUG:boto:Token: None 2022-09-03T07:24:08.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj17 2022-09-03T07:24:08.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oEt/tvg9B4ANH8KArZkn5PG/R2A= 2022-09-03T07:24:08.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oEt/tvg9B4ANH8KArZkn5PG/R2A=', 'Content-Length': '0'} 2022-09-03T07:24:08.720 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d2335f5f0cce7d5-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.721 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.721 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.722 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:08.722 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:08.723 DEBUG:boto:Method: GET 2022-09-03T07:24:08.724 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:08.724 DEBUG:boto:Data: 2022-09-03T07:24:08.725 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.725 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.726 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.726 DEBUG:boto:Params: {} 2022-09-03T07:24:08.727 DEBUG:boto:Token: None 2022-09-03T07:24:08.728 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.732 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005c18b8c3c5b319b8-0063130118-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.733 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.733 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.734 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.735 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:08.735 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:08.736 DEBUG:boto:Method: GET 2022-09-03T07:24:08.736 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:08.737 DEBUG:boto:Data: 2022-09-03T07:24:08.737 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.738 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.739 DEBUG:boto:Params: {} 2022-09-03T07:24:08.740 DEBUG:boto:Token: None 2022-09-03T07:24:08.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.746 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b00818ea8719f509-0063130118-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.746 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.747 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:24:08.748 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.748 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.749 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.749 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.750 DEBUG:boto:Method: GET 2022-09-03T07:24:08.750 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.751 DEBUG:boto:Data: 2022-09-03T07:24:08.752 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.753 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.753 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.754 DEBUG:boto:Params: {} 2022-09-03T07:24:08.754 DEBUG:boto:Token: None 2022-09-03T07:24:08.755 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zjtb29+7VG6au8hm21wMWmjIGJo= 2022-09-03T07:24:08.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zjtb29+7VG6au8hm21wMWmjIGJo=', 'Content-Length': '0'} 2022-09-03T07:24:08.758 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001fb0fcde3a8f55f9-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.759 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.760 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.760 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.761 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.761 DEBUG:boto:Method: GET 2022-09-03T07:24:08.762 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.762 DEBUG:boto:Data: 2022-09-03T07:24:08.763 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.764 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.764 DEBUG:boto:Params: {} 2022-09-03T07:24:08.765 DEBUG:boto:Token: None 2022-09-03T07:24:08.766 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zjtb29+7VG6au8hm21wMWmjIGJo= 2022-09-03T07:24:08.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zjtb29+7VG6au8hm21wMWmjIGJo=', 'Content-Length': '0'} 2022-09-03T07:24:08.769 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a84f76fa50dd0c18-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.770 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.770 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.771 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.771 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.772 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.772 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.773 DEBUG:boto:Data: 2022-09-03T07:24:08.773 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.774 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.774 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.775 DEBUG:boto:Params: {} 2022-09-03T07:24:08.776 DEBUG:boto:Token: None 2022-09-03T07:24:08.776 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5z7QEFbCrVpsL6VcShHb5QNBfc= 2022-09-03T07:24:08.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5z7QEFbCrVpsL6VcShHb5QNBfc=', 'Content-Length': '0'} 2022-09-03T07:24:08.779 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6d466ed6a140177-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.780 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.781 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.781 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.782 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.782 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.783 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.783 DEBUG:boto:Data: 2022-09-03T07:24:08.784 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.784 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.785 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.785 DEBUG:boto:Params: {} 2022-09-03T07:24:08.786 DEBUG:boto:Token: None 2022-09-03T07:24:08.786 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5z7QEFbCrVpsL6VcShHb5QNBfc= 2022-09-03T07:24:08.788 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5z7QEFbCrVpsL6VcShHb5QNBfc=', 'Content-Length': '0'} 2022-09-03T07:24:08.790 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a78ac84e72a0cb3c-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.790 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:24:08.791 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.792 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.792 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.793 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.793 DEBUG:boto:Method: GET 2022-09-03T07:24:08.794 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.794 DEBUG:boto:Data: 2022-09-03T07:24:08.795 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.795 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.796 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.796 DEBUG:boto:Params: {} 2022-09-03T07:24:08.797 DEBUG:boto:Token: None 2022-09-03T07:24:08.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zjtb29+7VG6au8hm21wMWmjIGJo= 2022-09-03T07:24:08.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zjtb29+7VG6au8hm21wMWmjIGJo=', 'Content-Length': '0'} 2022-09-03T07:24:08.801 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000253e15f98934824-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.802 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.803 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.803 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.804 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.804 DEBUG:boto:Method: GET 2022-09-03T07:24:08.805 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.805 DEBUG:boto:Data: 2022-09-03T07:24:08.806 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.806 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.807 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.808 DEBUG:boto:Params: {} 2022-09-03T07:24:08.808 DEBUG:boto:Token: None 2022-09-03T07:24:08.809 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:08.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zjtb29+7VG6au8hm21wMWmjIGJo= 2022-09-03T07:24:08.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zjtb29+7VG6au8hm21wMWmjIGJo=', 'Content-Length': '0'} 2022-09-03T07:24:08.812 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000438355df38e93b4e-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.813 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.813 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.814 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:08.815 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:08.815 DEBUG:boto:Method: GET 2022-09-03T07:24:08.816 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:08.816 DEBUG:boto:Data: 2022-09-03T07:24:08.817 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.817 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.818 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.818 DEBUG:boto:Params: {} 2022-09-03T07:24:08.819 DEBUG:boto:Token: None 2022-09-03T07:24:08.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.821 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.823 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007297b213bffb7391-0063130118-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.824 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.825 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.826 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.826 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:08.827 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:08.828 DEBUG:boto:Method: GET 2022-09-03T07:24:08.828 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:08.829 DEBUG:boto:Data: 2022-09-03T07:24:08.829 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.830 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.831 DEBUG:boto:Params: {} 2022-09-03T07:24:08.832 DEBUG:boto:Token: None 2022-09-03T07:24:08.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.838 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eae5d83904196e8e-0063130118-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.838 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.839 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.840 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.840 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.841 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:08.841 DEBUG:boto:Data: 2022-09-03T07:24:08.842 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.843 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.843 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.844 DEBUG:boto:Params: {} 2022-09-03T07:24:08.844 DEBUG:boto:Token: None 2022-09-03T07:24:08.845 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19 2022-09-03T07:24:08.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:auJwgBudb3IPZCaX8BCl7irBTCc= 2022-09-03T07:24:08.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:auJwgBudb3IPZCaX8BCl7irBTCc=', 'Content-Length': '0'} 2022-09-03T07:24:08.850 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022711a1e550f8d4b-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.851 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.851 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.852 DEBUG:boto:Method: HEAD 2022-09-03T07:24:08.852 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:08.853 DEBUG:boto:Data: 2022-09-03T07:24:08.853 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.854 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.854 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.855 DEBUG:boto:Params: {} 2022-09-03T07:24:08.856 DEBUG:boto:Token: None 2022-09-03T07:24:08.858 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19 2022-09-03T07:24:08.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:auJwgBudb3IPZCaX8BCl7irBTCc= 2022-09-03T07:24:08.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:auJwgBudb3IPZCaX8BCl7irBTCc=', 'Content-Length': '0'} 2022-09-03T07:24:08.861 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af81bd761f76e752-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.862 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:24:08.863 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.864 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.865 DEBUG:boto:Method: GET 2022-09-03T07:24:08.866 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:08.867 DEBUG:boto:Data: 2022-09-03T07:24:08.868 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.869 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.870 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.871 DEBUG:boto:Params: {} 2022-09-03T07:24:08.872 DEBUG:boto:Token: None 2022-09-03T07:24:08.873 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19 2022-09-03T07:24:08.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A7X8nCQ5PcgT/BsLGZiB2ygk70U= 2022-09-03T07:24:08.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A7X8nCQ5PcgT/BsLGZiB2ygk70U=', 'Content-Length': '0'} 2022-09-03T07:24:08.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d20299240941680-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.918 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:08.919 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:08.920 DEBUG:boto:Method: GET 2022-09-03T07:24:08.921 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:08.922 DEBUG:boto:Data: 2022-09-03T07:24:08.923 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.925 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.926 DEBUG:boto:Params: {} 2022-09-03T07:24:08.927 DEBUG:boto:Token: None 2022-09-03T07:24:08.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj19 2022-09-03T07:24:08.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A7X8nCQ5PcgT/BsLGZiB2ygk70U= 2022-09-03T07:24:08.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A7X8nCQ5PcgT/BsLGZiB2ygk70U=', 'Content-Length': '0'} 2022-09-03T07:24:08.934 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000095d41ce25a1aa633-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.936 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.937 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.937 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:08.938 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:08.939 DEBUG:boto:Method: GET 2022-09-03T07:24:08.940 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:08.941 DEBUG:boto:Data: 2022-09-03T07:24:08.942 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.943 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.944 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.945 DEBUG:boto:Params: {} 2022-09-03T07:24:08.946 DEBUG:boto:Token: None 2022-09-03T07:24:08.947 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.951 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045bc1bc56a9765b7-0063130118-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.952 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.954 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:08.955 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:08.955 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:08.956 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:08.956 DEBUG:boto:Method: GET 2022-09-03T07:24:08.957 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:08.957 DEBUG:boto:Data: 2022-09-03T07:24:08.958 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.959 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.960 DEBUG:boto:Params: {} 2022-09-03T07:24:08.960 DEBUG:boto:Token: None 2022-09-03T07:24:08.961 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/ 2022-09-03T07:24:08.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo= 2022-09-03T07:24:08.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+O3D6MFZCMpXSVDal29KFK7ScRo=', 'Content-Length': '0'} 2022-09-03T07:24:08.967 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e7883d2aa4b22d69-0063130118-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.968 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:08.969 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:24:08.970 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:08.970 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:08.971 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:08.971 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:08.972 DEBUG:boto:Method: GET 2022-09-03T07:24:08.972 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:08.973 DEBUG:boto:Data: 2022-09-03T07:24:08.974 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.974 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:08.975 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.975 DEBUG:boto:Params: {} 2022-09-03T07:24:08.976 DEBUG:boto:Token: None 2022-09-03T07:24:08.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:08.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:amuxemMCrIqpCdZ10qT8DJtMznU= 2022-09-03T07:24:08.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:amuxemMCrIqpCdZ10qT8DJtMznU=', 'Content-Length': '0'} 2022-09-03T07:24:08.980 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007eb7a64f9b32cd59-0063130118-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:08.981 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:08.981 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:08.982 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:08.982 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:08.983 DEBUG:boto:Method: GET 2022-09-03T07:24:08.983 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:08.984 DEBUG:boto:Data: 2022-09-03T07:24:08.984 DEBUG:boto:Headers: {} 2022-09-03T07:24:08.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:08.985 DEBUG:boto:Port: 8000 2022-09-03T07:24:08.986 DEBUG:boto:Params: {} 2022-09-03T07:24:08.987 DEBUG:boto:Token: None 2022-09-03T07:24:08.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:08 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:08.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:amuxemMCrIqpCdZ10qT8DJtMznU= 2022-09-03T07:24:08.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:amuxemMCrIqpCdZ10qT8DJtMznU=', 'Content-Length': '0'} 2022-09-03T07:24:08.991 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae14adbbe6e42c18-0063130118-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.038 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:09.040 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:09.041 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.041 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.042 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.043 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.044 DEBUG:boto:Data: 2022-09-03T07:24:09.045 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.046 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.047 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.048 DEBUG:boto:Params: {} 2022-09-03T07:24:09.051 DEBUG:boto:Token: None 2022-09-03T07:24:09.052 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tjSvWwcYl7AKBFzhmVj27+J5fBE= 2022-09-03T07:24:09.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tjSvWwcYl7AKBFzhmVj27+J5fBE=', 'Content-Length': '0'} 2022-09-03T07:24:09.058 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f85f15fa73bf5acf-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.059 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:09.060 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:09.061 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.062 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.063 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.064 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.065 DEBUG:boto:Data: 2022-09-03T07:24:09.066 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.068 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.069 DEBUG:boto:Params: {} 2022-09-03T07:24:09.070 DEBUG:boto:Token: None 2022-09-03T07:24:09.071 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tjSvWwcYl7AKBFzhmVj27+J5fBE= 2022-09-03T07:24:09.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tjSvWwcYl7AKBFzhmVj27+J5fBE=', 'Content-Length': '0'} 2022-09-03T07:24:09.075 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f21ddf6d45fb68a3-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.076 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:24:09.077 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:09.077 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:09.078 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.078 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.079 DEBUG:boto:Method: GET 2022-09-03T07:24:09.080 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.080 DEBUG:boto:Data: 2022-09-03T07:24:09.081 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.081 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.082 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.083 DEBUG:boto:Params: {} 2022-09-03T07:24:09.084 DEBUG:boto:Token: None 2022-09-03T07:24:09.085 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ksyoBz+aLaRqlRPR4cVMz+W1eM4= 2022-09-03T07:24:09.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ksyoBz+aLaRqlRPR4cVMz+W1eM4=', 'Content-Length': '0'} 2022-09-03T07:24:09.089 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000019227abae8c5867c-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.090 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:09.091 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:09.092 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.092 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.093 DEBUG:boto:Method: GET 2022-09-03T07:24:09.093 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.094 DEBUG:boto:Data: 2022-09-03T07:24:09.094 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.095 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.096 DEBUG:boto:Params: {} 2022-09-03T07:24:09.096 DEBUG:boto:Token: None 2022-09-03T07:24:09.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:09.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ksyoBz+aLaRqlRPR4cVMz+W1eM4= 2022-09-03T07:24:09.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ksyoBz+aLaRqlRPR4cVMz+W1eM4=', 'Content-Length': '0'} 2022-09-03T07:24:09.100 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0d7d45db01291d2-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.101 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.102 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.102 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:09.103 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:09.103 DEBUG:boto:Method: GET 2022-09-03T07:24:09.104 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:09.104 DEBUG:boto:Data: 2022-09-03T07:24:09.105 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.105 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.106 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.106 DEBUG:boto:Params: {} 2022-09-03T07:24:09.107 DEBUG:boto:Token: None 2022-09-03T07:24:09.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.109 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.114 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c0dffc3c5ed4f3ab-0063130119-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.114 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.115 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.116 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.116 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:09.117 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:09.117 DEBUG:boto:Method: GET 2022-09-03T07:24:09.118 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:09.118 DEBUG:boto:Data: 2022-09-03T07:24:09.119 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.119 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.120 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.120 DEBUG:boto:Params: {} 2022-09-03T07:24:09.121 DEBUG:boto:Token: None 2022-09-03T07:24:09.122 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.126 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7f9212569fd335c-0063130119-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.127 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.127 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:09.128 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:09.128 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.129 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:09.129 DEBUG:boto:Data: 2022-09-03T07:24:09.130 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.131 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.131 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.132 DEBUG:boto:Params: {} 2022-09-03T07:24:09.132 DEBUG:boto:Token: None 2022-09-03T07:24:09.133 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj22 2022-09-03T07:24:09.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PI8NmvclL+p9a+AUoJXv0RFYzNo= 2022-09-03T07:24:09.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PI8NmvclL+p9a+AUoJXv0RFYzNo=', 'Content-Length': '0'} 2022-09-03T07:24:09.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ddd35ddb4823a65-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.139 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:09.140 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:09.141 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.142 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:09.143 DEBUG:boto:Data: 2022-09-03T07:24:09.144 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.146 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.147 DEBUG:boto:Params: {} 2022-09-03T07:24:09.147 DEBUG:boto:Token: None 2022-09-03T07:24:09.148 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj22 2022-09-03T07:24:09.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PI8NmvclL+p9a+AUoJXv0RFYzNo= 2022-09-03T07:24:09.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PI8NmvclL+p9a+AUoJXv0RFYzNo=', 'Content-Length': '0'} 2022-09-03T07:24:09.152 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000461aae4eeaea1e57-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.153 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:24:09.154 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:09.155 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:09.156 DEBUG:boto:Method: GET 2022-09-03T07:24:09.157 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:09.158 DEBUG:boto:Data: 2022-09-03T07:24:09.158 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.159 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.159 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.160 DEBUG:boto:Params: {} 2022-09-03T07:24:09.160 DEBUG:boto:Token: None 2022-09-03T07:24:09.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj22 2022-09-03T07:24:09.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LaEkdctd7YOYuB3e06CZVkpJjfg= 2022-09-03T07:24:09.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LaEkdctd7YOYuB3e06CZVkpJjfg=', 'Content-Length': '0'} 2022-09-03T07:24:09.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c912a052a76fee15-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.206 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:09.207 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:09.208 DEBUG:boto:Method: GET 2022-09-03T07:24:09.209 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:09.210 DEBUG:boto:Data: 2022-09-03T07:24:09.212 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.213 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.213 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.214 DEBUG:boto:Params: {} 2022-09-03T07:24:09.216 DEBUG:boto:Token: None 2022-09-03T07:24:09.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj22 2022-09-03T07:24:09.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LaEkdctd7YOYuB3e06CZVkpJjfg= 2022-09-03T07:24:09.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LaEkdctd7YOYuB3e06CZVkpJjfg=', 'Content-Length': '0'} 2022-09-03T07:24:09.222 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008843822bef227c08-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.224 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.225 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.226 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:09.227 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:09.228 DEBUG:boto:Method: GET 2022-09-03T07:24:09.229 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:09.229 DEBUG:boto:Data: 2022-09-03T07:24:09.230 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.231 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.232 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.233 DEBUG:boto:Params: {} 2022-09-03T07:24:09.235 DEBUG:boto:Token: None 2022-09-03T07:24:09.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.242 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d00ffb16521e7de9-0063130119-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.243 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.244 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.245 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.245 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:09.246 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:09.246 DEBUG:boto:Method: GET 2022-09-03T07:24:09.247 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:09.247 DEBUG:boto:Data: 2022-09-03T07:24:09.248 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.249 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.249 DEBUG:boto:Params: {} 2022-09-03T07:24:09.250 DEBUG:boto:Token: None 2022-09-03T07:24:09.251 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.252 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.255 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000461902ec0fbaeab7-0063130119-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.298 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.299 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:24:09.300 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.300 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.301 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.301 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.302 DEBUG:boto:Method: GET 2022-09-03T07:24:09.302 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.303 DEBUG:boto:Data: 2022-09-03T07:24:09.303 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.304 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.304 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.305 DEBUG:boto:Params: {} 2022-09-03T07:24:09.306 DEBUG:boto:Token: None 2022-09-03T07:24:09.306 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6jbw+txWlMdbtl/NimKv5jPC5SE= 2022-09-03T07:24:09.308 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6jbw+txWlMdbtl/NimKv5jPC5SE=', 'Content-Length': '0'} 2022-09-03T07:24:09.310 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003da87a86096700ec-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.311 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.311 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.312 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.312 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.313 DEBUG:boto:Method: GET 2022-09-03T07:24:09.313 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.314 DEBUG:boto:Data: 2022-09-03T07:24:09.314 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.315 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.315 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.316 DEBUG:boto:Params: {} 2022-09-03T07:24:09.317 DEBUG:boto:Token: None 2022-09-03T07:24:09.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6jbw+txWlMdbtl/NimKv5jPC5SE= 2022-09-03T07:24:09.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6jbw+txWlMdbtl/NimKv5jPC5SE=', 'Content-Length': '0'} 2022-09-03T07:24:09.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f545a3ed0907773a-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.321 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.322 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.323 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.323 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.324 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.325 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.325 DEBUG:boto:Data: 2022-09-03T07:24:09.326 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.326 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.327 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.327 DEBUG:boto:Params: {} 2022-09-03T07:24:09.328 DEBUG:boto:Token: None 2022-09-03T07:24:09.329 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y+mqz0mFiLKZVIaViBqqdgEp8/Q= 2022-09-03T07:24:09.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y+mqz0mFiLKZVIaViBqqdgEp8/Q=', 'Content-Length': '0'} 2022-09-03T07:24:09.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f6cc8e7a042e813-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.333 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.333 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.334 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.334 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.335 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.336 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.336 DEBUG:boto:Data: 2022-09-03T07:24:09.337 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.338 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.338 DEBUG:boto:Params: {} 2022-09-03T07:24:09.339 DEBUG:boto:Token: None 2022-09-03T07:24:09.340 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y+mqz0mFiLKZVIaViBqqdgEp8/Q= 2022-09-03T07:24:09.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y+mqz0mFiLKZVIaViBqqdgEp8/Q=', 'Content-Length': '0'} 2022-09-03T07:24:09.344 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053533489a9f0e84f-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.345 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:24:09.346 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.347 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.347 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.348 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.349 DEBUG:boto:Method: GET 2022-09-03T07:24:09.349 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.350 DEBUG:boto:Data: 2022-09-03T07:24:09.351 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.351 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.352 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.353 DEBUG:boto:Params: {} 2022-09-03T07:24:09.354 DEBUG:boto:Token: None 2022-09-03T07:24:09.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6jbw+txWlMdbtl/NimKv5jPC5SE= 2022-09-03T07:24:09.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6jbw+txWlMdbtl/NimKv5jPC5SE=', 'Content-Length': '0'} 2022-09-03T07:24:09.358 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002fc23742b06bb501-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.359 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.360 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.360 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.361 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.361 DEBUG:boto:Method: GET 2022-09-03T07:24:09.362 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.363 DEBUG:boto:Data: 2022-09-03T07:24:09.363 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.364 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.364 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.365 DEBUG:boto:Params: {} 2022-09-03T07:24:09.366 DEBUG:boto:Token: None 2022-09-03T07:24:09.367 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:09.368 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6jbw+txWlMdbtl/NimKv5jPC5SE= 2022-09-03T07:24:09.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6jbw+txWlMdbtl/NimKv5jPC5SE=', 'Content-Length': '0'} 2022-09-03T07:24:09.377 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000625dfcd2078c37d8-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.379 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.380 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.381 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:09.382 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:09.383 DEBUG:boto:Method: GET 2022-09-03T07:24:09.384 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:09.385 DEBUG:boto:Data: 2022-09-03T07:24:09.386 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.387 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.388 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.389 DEBUG:boto:Params: {} 2022-09-03T07:24:09.390 DEBUG:boto:Token: None 2022-09-03T07:24:09.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.396 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bf2872fb748ac077-0063130119-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.438 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.439 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.440 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.440 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:09.441 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:09.441 DEBUG:boto:Method: GET 2022-09-03T07:24:09.442 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:09.443 DEBUG:boto:Data: 2022-09-03T07:24:09.443 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.444 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.444 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.445 DEBUG:boto:Params: {} 2022-09-03T07:24:09.446 DEBUG:boto:Token: None 2022-09-03T07:24:09.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.453 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000cdef97b521e9f0d-0063130119-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.454 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.456 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.457 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.458 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.459 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:09.460 DEBUG:boto:Data: 2022-09-03T07:24:09.461 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.463 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.464 DEBUG:boto:Params: {} 2022-09-03T07:24:09.465 DEBUG:boto:Token: None 2022-09-03T07:24:09.466 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23 2022-09-03T07:24:09.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hhagbT3lQ2icJI5+51Bmtccav7A= 2022-09-03T07:24:09.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hhagbT3lQ2icJI5+51Bmtccav7A=', 'Content-Length': '0'} 2022-09-03T07:24:09.471 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003bfc12be5475a764-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.472 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.473 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.473 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.474 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:09.475 DEBUG:boto:Data: 2022-09-03T07:24:09.476 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.477 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.478 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.479 DEBUG:boto:Params: {} 2022-09-03T07:24:09.480 DEBUG:boto:Token: None 2022-09-03T07:24:09.481 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23 2022-09-03T07:24:09.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hhagbT3lQ2icJI5+51Bmtccav7A= 2022-09-03T07:24:09.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hhagbT3lQ2icJI5+51Bmtccav7A=', 'Content-Length': '0'} 2022-09-03T07:24:09.485 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f4e318c8ff5db85-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.487 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:24:09.488 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.488 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.489 DEBUG:boto:Method: GET 2022-09-03T07:24:09.489 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:09.490 DEBUG:boto:Data: 2022-09-03T07:24:09.490 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.491 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.491 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.492 DEBUG:boto:Params: {} 2022-09-03T07:24:09.493 DEBUG:boto:Token: None 2022-09-03T07:24:09.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23 2022-09-03T07:24:09.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XlZDqGp/JznujnfBskQWuSrKp78= 2022-09-03T07:24:09.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XlZDqGp/JznujnfBskQWuSrKp78=', 'Content-Length': '0'} 2022-09-03T07:24:09.498 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c7201b7efd5d4c73-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.498 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:09.499 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:09.500 DEBUG:boto:Method: GET 2022-09-03T07:24:09.500 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:09.501 DEBUG:boto:Data: 2022-09-03T07:24:09.501 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.502 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.503 DEBUG:boto:Params: {} 2022-09-03T07:24:09.503 DEBUG:boto:Token: None 2022-09-03T07:24:09.504 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj23 2022-09-03T07:24:09.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XlZDqGp/JznujnfBskQWuSrKp78= 2022-09-03T07:24:09.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XlZDqGp/JznujnfBskQWuSrKp78=', 'Content-Length': '0'} 2022-09-03T07:24:09.509 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000047dd690312bfbe1-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.510 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.511 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.511 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:09.512 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:09.513 DEBUG:boto:Method: GET 2022-09-03T07:24:09.513 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:09.514 DEBUG:boto:Data: 2022-09-03T07:24:09.514 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.515 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.516 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.516 DEBUG:boto:Params: {} 2022-09-03T07:24:09.517 DEBUG:boto:Token: None 2022-09-03T07:24:09.518 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.523 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098f1c403ed331d94-0063130119-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.523 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.524 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.525 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.525 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:09.526 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:09.527 DEBUG:boto:Method: GET 2022-09-03T07:24:09.527 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:09.528 DEBUG:boto:Data: 2022-09-03T07:24:09.528 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.529 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.531 DEBUG:boto:Params: {} 2022-09-03T07:24:09.531 DEBUG:boto:Token: None 2022-09-03T07:24:09.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.536 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b75a20441e92e8ea-0063130119-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.537 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.538 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:24:09.539 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.539 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.540 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.540 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.541 DEBUG:boto:Method: GET 2022-09-03T07:24:09.541 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.542 DEBUG:boto:Data: 2022-09-03T07:24:09.542 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.543 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.543 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.544 DEBUG:boto:Params: {} 2022-09-03T07:24:09.544 DEBUG:boto:Token: None 2022-09-03T07:24:09.545 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.546 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uzcAk+HKnDvEt1VdvUfikYZ3l0A= 2022-09-03T07:24:09.546 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uzcAk+HKnDvEt1VdvUfikYZ3l0A=', 'Content-Length': '0'} 2022-09-03T07:24:09.551 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dd666b7c689f3b13-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.598 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.599 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.599 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.600 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.600 DEBUG:boto:Method: GET 2022-09-03T07:24:09.601 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.602 DEBUG:boto:Data: 2022-09-03T07:24:09.602 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.604 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.604 DEBUG:boto:Params: {} 2022-09-03T07:24:09.605 DEBUG:boto:Token: None 2022-09-03T07:24:09.605 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uzcAk+HKnDvEt1VdvUfikYZ3l0A= 2022-09-03T07:24:09.607 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uzcAk+HKnDvEt1VdvUfikYZ3l0A=', 'Content-Length': '0'} 2022-09-03T07:24:09.609 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cdf191b325661a79-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.610 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.611 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.612 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.612 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.613 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.613 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.614 DEBUG:boto:Data: 2022-09-03T07:24:09.615 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.615 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.616 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.616 DEBUG:boto:Params: {} 2022-09-03T07:24:09.617 DEBUG:boto:Token: None 2022-09-03T07:24:09.617 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L1vjNt5Aktyy8GOYBsiDp+C/XJg= 2022-09-03T07:24:09.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L1vjNt5Aktyy8GOYBsiDp+C/XJg=', 'Content-Length': '0'} 2022-09-03T07:24:09.621 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b615b7e9bf88893-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.622 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.622 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.623 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.623 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.624 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.624 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.625 DEBUG:boto:Data: 2022-09-03T07:24:09.625 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.626 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.627 DEBUG:boto:Params: {} 2022-09-03T07:24:09.627 DEBUG:boto:Token: None 2022-09-03T07:24:09.628 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L1vjNt5Aktyy8GOYBsiDp+C/XJg= 2022-09-03T07:24:09.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L1vjNt5Aktyy8GOYBsiDp+C/XJg=', 'Content-Length': '0'} 2022-09-03T07:24:09.631 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000de15bd9d5256ae7-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.632 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:24:09.632 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.633 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.633 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.634 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.635 DEBUG:boto:Method: GET 2022-09-03T07:24:09.635 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.636 DEBUG:boto:Data: 2022-09-03T07:24:09.636 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.637 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.638 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.638 DEBUG:boto:Params: {} 2022-09-03T07:24:09.639 DEBUG:boto:Token: None 2022-09-03T07:24:09.639 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uzcAk+HKnDvEt1VdvUfikYZ3l0A= 2022-09-03T07:24:09.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uzcAk+HKnDvEt1VdvUfikYZ3l0A=', 'Content-Length': '0'} 2022-09-03T07:24:09.644 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006920ccb2ac3c372c-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.645 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.646 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.646 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.647 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.647 DEBUG:boto:Method: GET 2022-09-03T07:24:09.648 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.648 DEBUG:boto:Data: 2022-09-03T07:24:09.649 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.650 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.650 DEBUG:boto:Params: {} 2022-09-03T07:24:09.651 DEBUG:boto:Token: None 2022-09-03T07:24:09.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:09.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uzcAk+HKnDvEt1VdvUfikYZ3l0A= 2022-09-03T07:24:09.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uzcAk+HKnDvEt1VdvUfikYZ3l0A=', 'Content-Length': '0'} 2022-09-03T07:24:09.656 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000282f85e46f414e11-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.657 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.658 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.659 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:09.659 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:09.660 DEBUG:boto:Method: GET 2022-09-03T07:24:09.661 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:09.662 DEBUG:boto:Data: 2022-09-03T07:24:09.662 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.663 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.663 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.664 DEBUG:boto:Params: {} 2022-09-03T07:24:09.665 DEBUG:boto:Token: None 2022-09-03T07:24:09.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.677 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ea3fb6a850e686f9-0063130119-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.678 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.678 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.679 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.679 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:09.680 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:09.681 DEBUG:boto:Method: GET 2022-09-03T07:24:09.681 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:09.682 DEBUG:boto:Data: 2022-09-03T07:24:09.682 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.683 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.684 DEBUG:boto:Params: {} 2022-09-03T07:24:09.684 DEBUG:boto:Token: None 2022-09-03T07:24:09.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.690 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a66783d692659761-0063130119-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.691 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.691 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.692 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.693 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.693 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:09.694 DEBUG:boto:Data: 2022-09-03T07:24:09.694 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.695 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.695 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.696 DEBUG:boto:Params: {} 2022-09-03T07:24:09.696 DEBUG:boto:Token: None 2022-09-03T07:24:09.697 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33 2022-09-03T07:24:09.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dfxpI03pof8bkbNvx+4XYpc2aKM= 2022-09-03T07:24:09.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dfxpI03pof8bkbNvx+4XYpc2aKM=', 'Content-Length': '0'} 2022-09-03T07:24:09.700 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000be695ed6f602d02-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.701 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.702 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.702 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.703 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:09.703 DEBUG:boto:Data: 2022-09-03T07:24:09.704 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.705 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.705 DEBUG:boto:Params: {} 2022-09-03T07:24:09.706 DEBUG:boto:Token: None 2022-09-03T07:24:09.707 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33 2022-09-03T07:24:09.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dfxpI03pof8bkbNvx+4XYpc2aKM= 2022-09-03T07:24:09.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dfxpI03pof8bkbNvx+4XYpc2aKM=', 'Content-Length': '0'} 2022-09-03T07:24:09.710 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ddc964ae2ac228c-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.711 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:24:09.711 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.712 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.713 DEBUG:boto:Method: GET 2022-09-03T07:24:09.714 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:09.714 DEBUG:boto:Data: 2022-09-03T07:24:09.715 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.715 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.716 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.716 DEBUG:boto:Params: {} 2022-09-03T07:24:09.717 DEBUG:boto:Token: None 2022-09-03T07:24:09.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33 2022-09-03T07:24:09.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aO2FDwHlswAFKm2ZgW+AxgzNYVU= 2022-09-03T07:24:09.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aO2FDwHlswAFKm2ZgW+AxgzNYVU=', 'Content-Length': '0'} 2022-09-03T07:24:09.721 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b8aff8d96f3e6f3-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.722 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:09.723 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:09.724 DEBUG:boto:Method: GET 2022-09-03T07:24:09.724 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:09.725 DEBUG:boto:Data: 2022-09-03T07:24:09.726 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.726 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.727 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.727 DEBUG:boto:Params: {} 2022-09-03T07:24:09.728 DEBUG:boto:Token: None 2022-09-03T07:24:09.728 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj33 2022-09-03T07:24:09.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aO2FDwHlswAFKm2ZgW+AxgzNYVU= 2022-09-03T07:24:09.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aO2FDwHlswAFKm2ZgW+AxgzNYVU=', 'Content-Length': '0'} 2022-09-03T07:24:09.731 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f2c3345510386ab7-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.778 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.779 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.779 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:09.780 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:09.781 DEBUG:boto:Method: GET 2022-09-03T07:24:09.781 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:09.782 DEBUG:boto:Data: 2022-09-03T07:24:09.782 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.783 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.783 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.784 DEBUG:boto:Params: {} 2022-09-03T07:24:09.784 DEBUG:boto:Token: None 2022-09-03T07:24:09.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.789 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000febc24f3694f2050-0063130119-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.790 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.791 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.791 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.792 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:09.792 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:09.793 DEBUG:boto:Method: GET 2022-09-03T07:24:09.794 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:09.794 DEBUG:boto:Data: 2022-09-03T07:24:09.795 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.796 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.796 DEBUG:boto:Params: {} 2022-09-03T07:24:09.797 DEBUG:boto:Token: None 2022-09-03T07:24:09.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.803 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000086994017541aa140-0063130119-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.804 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.804 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:24:09.805 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:09.806 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:09.806 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.807 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.807 DEBUG:boto:Method: GET 2022-09-03T07:24:09.808 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.808 DEBUG:boto:Data: 2022-09-03T07:24:09.809 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.809 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.810 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.811 DEBUG:boto:Params: {} 2022-09-03T07:24:09.811 DEBUG:boto:Token: None 2022-09-03T07:24:09.812 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SpzwFWAcgqM/S9mW7XbdNSGDieI= 2022-09-03T07:24:09.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SpzwFWAcgqM/S9mW7XbdNSGDieI=', 'Content-Length': '0'} 2022-09-03T07:24:09.815 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb2f9ed8da10b91a-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.816 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:09.817 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:09.817 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.818 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.818 DEBUG:boto:Method: GET 2022-09-03T07:24:09.819 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.819 DEBUG:boto:Data: 2022-09-03T07:24:09.820 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.821 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.821 DEBUG:boto:Params: {} 2022-09-03T07:24:09.822 DEBUG:boto:Token: None 2022-09-03T07:24:09.823 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SpzwFWAcgqM/S9mW7XbdNSGDieI= 2022-09-03T07:24:09.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SpzwFWAcgqM/S9mW7XbdNSGDieI=', 'Content-Length': '0'} 2022-09-03T07:24:09.826 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3f27b24b91e9f87-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.827 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:09.828 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:09.828 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.829 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.829 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.830 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.830 DEBUG:boto:Data: 2022-09-03T07:24:09.831 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.831 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.832 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.832 DEBUG:boto:Params: {} 2022-09-03T07:24:09.833 DEBUG:boto:Token: None 2022-09-03T07:24:09.833 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BCZ5xIC3HMsdZESeogA6EkR/MXM= 2022-09-03T07:24:09.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BCZ5xIC3HMsdZESeogA6EkR/MXM=', 'Content-Length': '0'} 2022-09-03T07:24:09.836 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000747b754a58b6c6c2-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.837 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:09.838 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:09.838 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.839 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.839 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.840 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.840 DEBUG:boto:Data: 2022-09-03T07:24:09.841 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.841 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.842 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.842 DEBUG:boto:Params: {} 2022-09-03T07:24:09.843 DEBUG:boto:Token: None 2022-09-03T07:24:09.844 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.844 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BCZ5xIC3HMsdZESeogA6EkR/MXM= 2022-09-03T07:24:09.845 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BCZ5xIC3HMsdZESeogA6EkR/MXM=', 'Content-Length': '0'} 2022-09-03T07:24:09.847 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022679a90f60b45da-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.847 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:24:09.848 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:09.848 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:09.849 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.850 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.850 DEBUG:boto:Method: GET 2022-09-03T07:24:09.851 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.851 DEBUG:boto:Data: 2022-09-03T07:24:09.852 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.852 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.853 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.853 DEBUG:boto:Params: {} 2022-09-03T07:24:09.854 DEBUG:boto:Token: None 2022-09-03T07:24:09.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.855 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SpzwFWAcgqM/S9mW7XbdNSGDieI= 2022-09-03T07:24:09.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SpzwFWAcgqM/S9mW7XbdNSGDieI=', 'Content-Length': '0'} 2022-09-03T07:24:09.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be250bdfebcbdefa-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.859 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:09.859 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:09.860 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.861 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.861 DEBUG:boto:Method: GET 2022-09-03T07:24:09.862 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.862 DEBUG:boto:Data: 2022-09-03T07:24:09.863 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.863 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.864 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.864 DEBUG:boto:Params: {} 2022-09-03T07:24:09.865 DEBUG:boto:Token: None 2022-09-03T07:24:09.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:09.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SpzwFWAcgqM/S9mW7XbdNSGDieI= 2022-09-03T07:24:09.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SpzwFWAcgqM/S9mW7XbdNSGDieI=', 'Content-Length': '0'} 2022-09-03T07:24:09.869 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd85310f109e3ea2-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.914 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.915 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.915 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:09.916 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:09.917 DEBUG:boto:Method: GET 2022-09-03T07:24:09.917 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:09.918 DEBUG:boto:Data: 2022-09-03T07:24:09.918 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.919 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.919 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.920 DEBUG:boto:Params: {} 2022-09-03T07:24:09.920 DEBUG:boto:Token: None 2022-09-03T07:24:09.921 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.925 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000073a04cff73bebf33-0063130119-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.926 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.926 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:09.927 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:09.928 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:09.928 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:09.929 DEBUG:boto:Method: GET 2022-09-03T07:24:09.929 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:09.930 DEBUG:boto:Data: 2022-09-03T07:24:09.930 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.931 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.932 DEBUG:boto:Params: {} 2022-09-03T07:24:09.933 DEBUG:boto:Token: None 2022-09-03T07:24:09.933 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/ 2022-09-03T07:24:09.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM= 2022-09-03T07:24:09.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JqoXMcE4D8kBip7/zqGUf9fIZdM=', 'Content-Length': '0'} 2022-09-03T07:24:09.938 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004643cb320094a11f-0063130119-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.939 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:09.940 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:09.941 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:09.942 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.942 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:09.943 DEBUG:boto:Data: 2022-09-03T07:24:09.943 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.944 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.944 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.945 DEBUG:boto:Params: {} 2022-09-03T07:24:09.945 DEBUG:boto:Token: None 2022-09-03T07:24:09.946 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj34 2022-09-03T07:24:09.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1ivim3f3YTxDOILTJHn/tQX19xs= 2022-09-03T07:24:09.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1ivim3f3YTxDOILTJHn/tQX19xs=', 'Content-Length': '0'} 2022-09-03T07:24:09.950 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5cd3ad672717cc3-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.951 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:09.952 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:09.952 DEBUG:boto:Method: HEAD 2022-09-03T07:24:09.953 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:09.953 DEBUG:boto:Data: 2022-09-03T07:24:09.954 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.955 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:09.955 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.956 DEBUG:boto:Params: {} 2022-09-03T07:24:09.956 DEBUG:boto:Token: None 2022-09-03T07:24:09.957 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj34 2022-09-03T07:24:09.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1ivim3f3YTxDOILTJHn/tQX19xs= 2022-09-03T07:24:09.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1ivim3f3YTxDOILTJHn/tQX19xs=', 'Content-Length': '0'} 2022-09-03T07:24:09.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba245fc4eb884a11-0063130119-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:09.961 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:24:09.962 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:09.962 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:09.963 DEBUG:boto:Method: GET 2022-09-03T07:24:09.964 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:09.964 DEBUG:boto:Data: 2022-09-03T07:24:09.965 DEBUG:boto:Headers: {} 2022-09-03T07:24:09.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:09.966 DEBUG:boto:Port: 8000 2022-09-03T07:24:09.967 DEBUG:boto:Params: {} 2022-09-03T07:24:09.967 DEBUG:boto:Token: None 2022-09-03T07:24:09.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:09 GMT /avysmw-75/obj34 2022-09-03T07:24:09.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Jpq0w85sIb17He2buEbQ29/zGY= 2022-09-03T07:24:09.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Jpq0w85sIb17He2buEbQ29/zGY=', 'Content-Length': '0'} 2022-09-03T07:24:09.971 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045583739afc3b0d8-0063130119-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.019 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:10.019 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:10.020 DEBUG:boto:Method: GET 2022-09-03T07:24:10.021 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:10.021 DEBUG:boto:Data: 2022-09-03T07:24:10.022 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.022 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.023 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.023 DEBUG:boto:Params: {} 2022-09-03T07:24:10.024 DEBUG:boto:Token: None 2022-09-03T07:24:10.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj34 2022-09-03T07:24:10.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hA4ZazA5yC3/zSgEFm++6Wjh/og= 2022-09-03T07:24:10.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hA4ZazA5yC3/zSgEFm++6Wjh/og=', 'Content-Length': '0'} 2022-09-03T07:24:10.028 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9bbfe2de36cd598-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.030 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.030 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.031 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:10.031 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:10.032 DEBUG:boto:Method: GET 2022-09-03T07:24:10.032 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:10.033 DEBUG:boto:Data: 2022-09-03T07:24:10.033 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.034 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.035 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.035 DEBUG:boto:Params: {} 2022-09-03T07:24:10.036 DEBUG:boto:Token: None 2022-09-03T07:24:10.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.040 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058e7215178ddb7d8-006313011a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.041 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.042 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.043 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.043 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:10.044 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:10.045 DEBUG:boto:Method: GET 2022-09-03T07:24:10.045 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:10.046 DEBUG:boto:Data: 2022-09-03T07:24:10.047 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.048 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.048 DEBUG:boto:Params: {} 2022-09-03T07:24:10.049 DEBUG:boto:Token: None 2022-09-03T07:24:10.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.055 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd05324bec40d0c1-006313011a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.056 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.057 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:24:10.057 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.058 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.059 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.059 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.060 DEBUG:boto:Method: GET 2022-09-03T07:24:10.061 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.061 DEBUG:boto:Data: 2022-09-03T07:24:10.062 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.062 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.063 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.064 DEBUG:boto:Params: {} 2022-09-03T07:24:10.065 DEBUG:boto:Token: None 2022-09-03T07:24:10.066 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.066 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fT3mJ78bjaWuabZ4WC8MQBcbo+0= 2022-09-03T07:24:10.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fT3mJ78bjaWuabZ4WC8MQBcbo+0=', 'Content-Length': '0'} 2022-09-03T07:24:10.069 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e68ee503c49dead-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.070 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.071 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.071 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.072 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.073 DEBUG:boto:Method: GET 2022-09-03T07:24:10.074 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.074 DEBUG:boto:Data: 2022-09-03T07:24:10.075 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.076 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.077 DEBUG:boto:Params: {} 2022-09-03T07:24:10.078 DEBUG:boto:Token: None 2022-09-03T07:24:10.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fT3mJ78bjaWuabZ4WC8MQBcbo+0= 2022-09-03T07:24:10.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fT3mJ78bjaWuabZ4WC8MQBcbo+0=', 'Content-Length': '0'} 2022-09-03T07:24:10.082 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d80ad4ba7e005b80-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.126 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.128 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.129 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.129 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.130 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.131 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.132 DEBUG:boto:Data: 2022-09-03T07:24:10.133 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.134 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.135 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.136 DEBUG:boto:Params: {} 2022-09-03T07:24:10.138 DEBUG:boto:Token: None 2022-09-03T07:24:10.139 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCIwJzU+dslqoDtmaPzryVquqfo= 2022-09-03T07:24:10.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCIwJzU+dslqoDtmaPzryVquqfo=', 'Content-Length': '0'} 2022-09-03T07:24:10.145 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055e48a2146c070e2-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.146 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.147 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.147 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.148 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.149 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.150 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.152 DEBUG:boto:Data: 2022-09-03T07:24:10.153 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.154 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.155 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.155 DEBUG:boto:Params: {} 2022-09-03T07:24:10.156 DEBUG:boto:Token: None 2022-09-03T07:24:10.157 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCIwJzU+dslqoDtmaPzryVquqfo= 2022-09-03T07:24:10.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCIwJzU+dslqoDtmaPzryVquqfo=', 'Content-Length': '0'} 2022-09-03T07:24:10.161 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e9e69d5a9f3dd63-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.162 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:24:10.162 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.163 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.164 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.164 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.165 DEBUG:boto:Method: GET 2022-09-03T07:24:10.166 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.167 DEBUG:boto:Data: 2022-09-03T07:24:10.167 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.168 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.169 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.170 DEBUG:boto:Params: {} 2022-09-03T07:24:10.171 DEBUG:boto:Token: None 2022-09-03T07:24:10.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fT3mJ78bjaWuabZ4WC8MQBcbo+0= 2022-09-03T07:24:10.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fT3mJ78bjaWuabZ4WC8MQBcbo+0=', 'Content-Length': '0'} 2022-09-03T07:24:10.175 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000965c74b846768c9b-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.176 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.177 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.177 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.178 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.179 DEBUG:boto:Method: GET 2022-09-03T07:24:10.179 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.180 DEBUG:boto:Data: 2022-09-03T07:24:10.181 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.182 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.183 DEBUG:boto:Params: {} 2022-09-03T07:24:10.184 DEBUG:boto:Token: None 2022-09-03T07:24:10.184 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:10.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fT3mJ78bjaWuabZ4WC8MQBcbo+0= 2022-09-03T07:24:10.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fT3mJ78bjaWuabZ4WC8MQBcbo+0=', 'Content-Length': '0'} 2022-09-03T07:24:10.188 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b085092b1a2869ae-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.189 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.190 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.191 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:10.192 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:10.193 DEBUG:boto:Method: GET 2022-09-03T07:24:10.193 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:10.194 DEBUG:boto:Data: 2022-09-03T07:24:10.195 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.195 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.196 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.197 DEBUG:boto:Params: {} 2022-09-03T07:24:10.198 DEBUG:boto:Token: None 2022-09-03T07:24:10.198 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.203 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c8303ad00c0ccdf4-006313011a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.204 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.205 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.205 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.206 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:10.207 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:10.207 DEBUG:boto:Method: GET 2022-09-03T07:24:10.208 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:10.209 DEBUG:boto:Data: 2022-09-03T07:24:10.209 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.211 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.211 DEBUG:boto:Params: {} 2022-09-03T07:24:10.212 DEBUG:boto:Token: None 2022-09-03T07:24:10.213 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.218 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e8fa306f5f1ecfd7-006313011a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.219 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.220 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.221 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.222 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.223 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:10.224 DEBUG:boto:Data: 2022-09-03T07:24:10.225 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.226 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.227 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.227 DEBUG:boto:Params: {} 2022-09-03T07:24:10.229 DEBUG:boto:Token: None 2022-09-03T07:24:10.230 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38 2022-09-03T07:24:10.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:faMlew9mMcnCMdV/hIovaNX6muw= 2022-09-03T07:24:10.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:faMlew9mMcnCMdV/hIovaNX6muw=', 'Content-Length': '0'} 2022-09-03T07:24:10.234 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e03abb59dedcac81-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.235 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.235 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.236 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.236 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:10.237 DEBUG:boto:Data: 2022-09-03T07:24:10.237 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.238 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.239 DEBUG:boto:Params: {} 2022-09-03T07:24:10.239 DEBUG:boto:Token: None 2022-09-03T07:24:10.240 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38 2022-09-03T07:24:10.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:faMlew9mMcnCMdV/hIovaNX6muw= 2022-09-03T07:24:10.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:faMlew9mMcnCMdV/hIovaNX6muw=', 'Content-Length': '0'} 2022-09-03T07:24:10.250 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fbbf547dadb2e33-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.252 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:24:10.253 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.253 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.254 DEBUG:boto:Method: GET 2022-09-03T07:24:10.255 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:10.256 DEBUG:boto:Data: 2022-09-03T07:24:10.257 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.258 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.258 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.259 DEBUG:boto:Params: {} 2022-09-03T07:24:10.259 DEBUG:boto:Token: None 2022-09-03T07:24:10.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38 2022-09-03T07:24:10.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8AZUg5uoDWzA/0G8lUwn/C+MjKM= 2022-09-03T07:24:10.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8AZUg5uoDWzA/0G8lUwn/C+MjKM=', 'Content-Length': '0'} 2022-09-03T07:24:10.266 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8edc63ee130fa36-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.314 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:10.315 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:10.315 DEBUG:boto:Method: GET 2022-09-03T07:24:10.316 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:10.317 DEBUG:boto:Data: 2022-09-03T07:24:10.317 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.318 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.318 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.319 DEBUG:boto:Params: {} 2022-09-03T07:24:10.319 DEBUG:boto:Token: None 2022-09-03T07:24:10.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj38 2022-09-03T07:24:10.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8AZUg5uoDWzA/0G8lUwn/C+MjKM= 2022-09-03T07:24:10.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8AZUg5uoDWzA/0G8lUwn/C+MjKM=', 'Content-Length': '0'} 2022-09-03T07:24:10.324 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c93532f8629b5f7-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.325 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.326 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.326 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:10.327 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:10.329 DEBUG:boto:Method: GET 2022-09-03T07:24:10.330 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:10.331 DEBUG:boto:Data: 2022-09-03T07:24:10.332 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.333 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.334 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.334 DEBUG:boto:Params: {} 2022-09-03T07:24:10.335 DEBUG:boto:Token: None 2022-09-03T07:24:10.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.341 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0477c4869457667-006313011a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.342 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.343 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.343 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.344 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:10.345 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:10.346 DEBUG:boto:Method: GET 2022-09-03T07:24:10.346 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:10.347 DEBUG:boto:Data: 2022-09-03T07:24:10.347 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.348 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.348 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.349 DEBUG:boto:Params: {} 2022-09-03T07:24:10.350 DEBUG:boto:Token: None 2022-09-03T07:24:10.350 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.355 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006e641d94a41543e6-006313011a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.355 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.356 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:24:10.357 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.357 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.358 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.358 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.359 DEBUG:boto:Method: GET 2022-09-03T07:24:10.360 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.360 DEBUG:boto:Data: 2022-09-03T07:24:10.361 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.362 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.362 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.363 DEBUG:boto:Params: {} 2022-09-03T07:24:10.363 DEBUG:boto:Token: None 2022-09-03T07:24:10.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EoyN8wbZBa8K5CFX/GTsGNvdhqA= 2022-09-03T07:24:10.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EoyN8wbZBa8K5CFX/GTsGNvdhqA=', 'Content-Length': '0'} 2022-09-03T07:24:10.367 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3a1a8f1a6323658-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.368 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.369 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.369 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.370 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.370 DEBUG:boto:Method: GET 2022-09-03T07:24:10.371 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.371 DEBUG:boto:Data: 2022-09-03T07:24:10.372 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.373 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.374 DEBUG:boto:Params: {} 2022-09-03T07:24:10.374 DEBUG:boto:Token: None 2022-09-03T07:24:10.375 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EoyN8wbZBa8K5CFX/GTsGNvdhqA= 2022-09-03T07:24:10.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EoyN8wbZBa8K5CFX/GTsGNvdhqA=', 'Content-Length': '0'} 2022-09-03T07:24:10.378 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5f0009bc7b37d70-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.426 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.427 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.428 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.428 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.429 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.429 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.430 DEBUG:boto:Data: 2022-09-03T07:24:10.430 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.431 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.431 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.432 DEBUG:boto:Params: {} 2022-09-03T07:24:10.432 DEBUG:boto:Token: None 2022-09-03T07:24:10.433 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5peV9Mil121MEPAFYM+vYVUuBac= 2022-09-03T07:24:10.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5peV9Mil121MEPAFYM+vYVUuBac=', 'Content-Length': '0'} 2022-09-03T07:24:10.437 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d5d58cbe16cc0cc-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.438 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.439 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.439 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.441 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.441 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.442 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.442 DEBUG:boto:Data: 2022-09-03T07:24:10.443 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.444 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.445 DEBUG:boto:Params: {} 2022-09-03T07:24:10.446 DEBUG:boto:Token: None 2022-09-03T07:24:10.447 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5peV9Mil121MEPAFYM+vYVUuBac= 2022-09-03T07:24:10.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5peV9Mil121MEPAFYM+vYVUuBac=', 'Content-Length': '0'} 2022-09-03T07:24:10.450 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042ea9c6365d928d9-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.451 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:24:10.451 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.452 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.452 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.453 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.453 DEBUG:boto:Method: GET 2022-09-03T07:24:10.454 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.455 DEBUG:boto:Data: 2022-09-03T07:24:10.456 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.456 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.457 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.458 DEBUG:boto:Params: {} 2022-09-03T07:24:10.459 DEBUG:boto:Token: None 2022-09-03T07:24:10.459 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.460 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EoyN8wbZBa8K5CFX/GTsGNvdhqA= 2022-09-03T07:24:10.461 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EoyN8wbZBa8K5CFX/GTsGNvdhqA=', 'Content-Length': '0'} 2022-09-03T07:24:10.464 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7b9538bb156383c-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.465 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.465 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.466 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.467 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.467 DEBUG:boto:Method: GET 2022-09-03T07:24:10.468 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.468 DEBUG:boto:Data: 2022-09-03T07:24:10.469 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.470 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.471 DEBUG:boto:Params: {} 2022-09-03T07:24:10.471 DEBUG:boto:Token: None 2022-09-03T07:24:10.472 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:10.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EoyN8wbZBa8K5CFX/GTsGNvdhqA= 2022-09-03T07:24:10.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EoyN8wbZBa8K5CFX/GTsGNvdhqA=', 'Content-Length': '0'} 2022-09-03T07:24:10.476 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4cd3cd129753da5-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.478 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.479 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.479 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:10.480 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:10.480 DEBUG:boto:Method: GET 2022-09-03T07:24:10.481 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:10.482 DEBUG:boto:Data: 2022-09-03T07:24:10.482 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.483 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.483 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.484 DEBUG:boto:Params: {} 2022-09-03T07:24:10.484 DEBUG:boto:Token: None 2022-09-03T07:24:10.485 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.486 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.490 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000e1f2535050b3074-006313011a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.490 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.491 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.492 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.493 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:10.493 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:10.494 DEBUG:boto:Method: GET 2022-09-03T07:24:10.494 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:10.495 DEBUG:boto:Data: 2022-09-03T07:24:10.495 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.496 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.497 DEBUG:boto:Params: {} 2022-09-03T07:24:10.497 DEBUG:boto:Token: None 2022-09-03T07:24:10.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.503 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c084cc1f9af2d22-006313011a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.503 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.504 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.505 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.505 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.506 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:10.506 DEBUG:boto:Data: 2022-09-03T07:24:10.507 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.507 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.508 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.508 DEBUG:boto:Params: {} 2022-09-03T07:24:10.509 DEBUG:boto:Token: None 2022-09-03T07:24:10.509 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44 2022-09-03T07:24:10.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ypBMuMtvPNdzUwiP9KZiZS/jYF4= 2022-09-03T07:24:10.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ypBMuMtvPNdzUwiP9KZiZS/jYF4=', 'Content-Length': '0'} 2022-09-03T07:24:10.512 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9706aa4c1146029-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.513 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.514 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.514 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.515 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:10.515 DEBUG:boto:Data: 2022-09-03T07:24:10.516 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.517 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.517 DEBUG:boto:Params: {} 2022-09-03T07:24:10.518 DEBUG:boto:Token: None 2022-09-03T07:24:10.518 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44 2022-09-03T07:24:10.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ypBMuMtvPNdzUwiP9KZiZS/jYF4= 2022-09-03T07:24:10.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ypBMuMtvPNdzUwiP9KZiZS/jYF4=', 'Content-Length': '0'} 2022-09-03T07:24:10.522 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000718f4af726c9cd1f-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.522 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:24:10.523 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.524 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.524 DEBUG:boto:Method: GET 2022-09-03T07:24:10.525 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:10.525 DEBUG:boto:Data: 2022-09-03T07:24:10.526 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.526 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.527 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.527 DEBUG:boto:Params: {} 2022-09-03T07:24:10.528 DEBUG:boto:Token: None 2022-09-03T07:24:10.528 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44 2022-09-03T07:24:10.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z/WlBnEtJL7hlLEXpbHr76KEhRI= 2022-09-03T07:24:10.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z/WlBnEtJL7hlLEXpbHr76KEhRI=', 'Content-Length': '0'} 2022-09-03T07:24:10.532 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000153b255067315232-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.532 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:10.533 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:10.533 DEBUG:boto:Method: GET 2022-09-03T07:24:10.534 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:10.535 DEBUG:boto:Data: 2022-09-03T07:24:10.535 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.536 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.536 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.537 DEBUG:boto:Params: {} 2022-09-03T07:24:10.537 DEBUG:boto:Token: None 2022-09-03T07:24:10.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj44 2022-09-03T07:24:10.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z/WlBnEtJL7hlLEXpbHr76KEhRI= 2022-09-03T07:24:10.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z/WlBnEtJL7hlLEXpbHr76KEhRI=', 'Content-Length': '0'} 2022-09-03T07:24:10.541 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068fb840e0d4909f9-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.542 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.543 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.543 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:10.544 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:10.544 DEBUG:boto:Method: GET 2022-09-03T07:24:10.545 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:10.545 DEBUG:boto:Data: 2022-09-03T07:24:10.546 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.546 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.547 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.547 DEBUG:boto:Params: {} 2022-09-03T07:24:10.548 DEBUG:boto:Token: None 2022-09-03T07:24:10.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.552 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003c6ce8c532991ade-006313011a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.553 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.553 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.554 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.554 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:10.555 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:10.556 DEBUG:boto:Method: GET 2022-09-03T07:24:10.556 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:10.557 DEBUG:boto:Data: 2022-09-03T07:24:10.557 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.558 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.558 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.559 DEBUG:boto:Params: {} 2022-09-03T07:24:10.559 DEBUG:boto:Token: None 2022-09-03T07:24:10.560 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.564 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005d3f2a01c6c290d2-006313011a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.565 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.566 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:24:10.566 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.567 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.567 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.568 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.569 DEBUG:boto:Method: GET 2022-09-03T07:24:10.569 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.570 DEBUG:boto:Data: 2022-09-03T07:24:10.570 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.571 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.571 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.572 DEBUG:boto:Params: {} 2022-09-03T07:24:10.572 DEBUG:boto:Token: None 2022-09-03T07:24:10.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ddXJXqdynZ3Xgny4k5rUTLUR9Jg= 2022-09-03T07:24:10.574 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ddXJXqdynZ3Xgny4k5rUTLUR9Jg=', 'Content-Length': '0'} 2022-09-03T07:24:10.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008621e3a89d4d5d55-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.578 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.579 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.580 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.580 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.581 DEBUG:boto:Method: GET 2022-09-03T07:24:10.581 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.582 DEBUG:boto:Data: 2022-09-03T07:24:10.582 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.583 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.584 DEBUG:boto:Params: {} 2022-09-03T07:24:10.584 DEBUG:boto:Token: None 2022-09-03T07:24:10.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ddXJXqdynZ3Xgny4k5rUTLUR9Jg= 2022-09-03T07:24:10.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ddXJXqdynZ3Xgny4k5rUTLUR9Jg=', 'Content-Length': '0'} 2022-09-03T07:24:10.588 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e92113555077d6d5-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.589 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.590 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.590 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.591 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.591 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.592 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.592 DEBUG:boto:Data: 2022-09-03T07:24:10.593 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.593 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.593 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.594 DEBUG:boto:Params: {} 2022-09-03T07:24:10.595 DEBUG:boto:Token: None 2022-09-03T07:24:10.595 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ARMUuVEpVl/BFJmIuUpjZwNQ0MA= 2022-09-03T07:24:10.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ARMUuVEpVl/BFJmIuUpjZwNQ0MA=', 'Content-Length': '0'} 2022-09-03T07:24:10.598 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008caba8bf80fb21b0-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.599 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.599 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.600 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.600 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.601 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.601 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.602 DEBUG:boto:Data: 2022-09-03T07:24:10.602 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.603 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.604 DEBUG:boto:Params: {} 2022-09-03T07:24:10.604 DEBUG:boto:Token: None 2022-09-03T07:24:10.605 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ARMUuVEpVl/BFJmIuUpjZwNQ0MA= 2022-09-03T07:24:10.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ARMUuVEpVl/BFJmIuUpjZwNQ0MA=', 'Content-Length': '0'} 2022-09-03T07:24:10.608 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf3cfe4bfdc53751-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.609 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:24:10.609 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.610 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.610 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.611 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.612 DEBUG:boto:Method: GET 2022-09-03T07:24:10.612 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.613 DEBUG:boto:Data: 2022-09-03T07:24:10.613 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.614 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.614 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.615 DEBUG:boto:Params: {} 2022-09-03T07:24:10.615 DEBUG:boto:Token: None 2022-09-03T07:24:10.616 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ddXJXqdynZ3Xgny4k5rUTLUR9Jg= 2022-09-03T07:24:10.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ddXJXqdynZ3Xgny4k5rUTLUR9Jg=', 'Content-Length': '0'} 2022-09-03T07:24:10.619 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000583bde2023fd0a2-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.620 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.621 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.621 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.622 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.622 DEBUG:boto:Method: GET 2022-09-03T07:24:10.623 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.623 DEBUG:boto:Data: 2022-09-03T07:24:10.624 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.624 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.625 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.625 DEBUG:boto:Params: {} 2022-09-03T07:24:10.626 DEBUG:boto:Token: None 2022-09-03T07:24:10.627 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:10.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ddXJXqdynZ3Xgny4k5rUTLUR9Jg= 2022-09-03T07:24:10.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ddXJXqdynZ3Xgny4k5rUTLUR9Jg=', 'Content-Length': '0'} 2022-09-03T07:24:10.630 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034c567ba452524d6-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.631 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.631 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.632 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:10.632 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:10.633 DEBUG:boto:Method: GET 2022-09-03T07:24:10.633 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:10.634 DEBUG:boto:Data: 2022-09-03T07:24:10.634 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.635 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.635 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.636 DEBUG:boto:Params: {} 2022-09-03T07:24:10.636 DEBUG:boto:Token: None 2022-09-03T07:24:10.637 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.640 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009044f5df7dc2d0a1-006313011a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.641 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.642 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.642 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.643 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:10.643 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:10.644 DEBUG:boto:Method: GET 2022-09-03T07:24:10.644 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:10.645 DEBUG:boto:Data: 2022-09-03T07:24:10.645 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.647 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.647 DEBUG:boto:Params: {} 2022-09-03T07:24:10.648 DEBUG:boto:Token: None 2022-09-03T07:24:10.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.653 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb9fe158ac8ace6f-006313011a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.654 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.655 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.656 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.657 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.658 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:10.659 DEBUG:boto:Data: 2022-09-03T07:24:10.660 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.660 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.661 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.661 DEBUG:boto:Params: {} 2022-09-03T07:24:10.662 DEBUG:boto:Token: None 2022-09-03T07:24:10.663 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46 2022-09-03T07:24:10.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9PaAPxlMry6JXNyhKv0B4YSi9xw= 2022-09-03T07:24:10.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9PaAPxlMry6JXNyhKv0B4YSi9xw=', 'Content-Length': '0'} 2022-09-03T07:24:10.668 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009e2692d3744b9cb-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.669 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.670 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.670 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.671 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:10.671 DEBUG:boto:Data: 2022-09-03T07:24:10.672 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.673 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.673 DEBUG:boto:Params: {} 2022-09-03T07:24:10.674 DEBUG:boto:Token: None 2022-09-03T07:24:10.675 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46 2022-09-03T07:24:10.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9PaAPxlMry6JXNyhKv0B4YSi9xw= 2022-09-03T07:24:10.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9PaAPxlMry6JXNyhKv0B4YSi9xw=', 'Content-Length': '0'} 2022-09-03T07:24:10.678 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c59b211e00cd47da-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.679 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:24:10.680 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.680 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.681 DEBUG:boto:Method: GET 2022-09-03T07:24:10.681 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:10.682 DEBUG:boto:Data: 2022-09-03T07:24:10.682 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.683 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.683 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.684 DEBUG:boto:Params: {} 2022-09-03T07:24:10.684 DEBUG:boto:Token: None 2022-09-03T07:24:10.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46 2022-09-03T07:24:10.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JzMH5CMFQQ5vqiu/NK9baomIXkI= 2022-09-03T07:24:10.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JzMH5CMFQQ5vqiu/NK9baomIXkI=', 'Content-Length': '0'} 2022-09-03T07:24:10.690 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d5425268955ab6f1-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.691 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:10.692 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:10.692 DEBUG:boto:Method: GET 2022-09-03T07:24:10.693 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:10.694 DEBUG:boto:Data: 2022-09-03T07:24:10.694 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.695 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.696 DEBUG:boto:Params: {} 2022-09-03T07:24:10.696 DEBUG:boto:Token: None 2022-09-03T07:24:10.697 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj46 2022-09-03T07:24:10.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JzMH5CMFQQ5vqiu/NK9baomIXkI= 2022-09-03T07:24:10.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JzMH5CMFQQ5vqiu/NK9baomIXkI=', 'Content-Length': '0'} 2022-09-03T07:24:10.700 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026bbd1b44e9a650e-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.746 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.747 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.747 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:10.748 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:10.748 DEBUG:boto:Method: GET 2022-09-03T07:24:10.749 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:10.749 DEBUG:boto:Data: 2022-09-03T07:24:10.750 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.751 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.751 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.752 DEBUG:boto:Params: {} 2022-09-03T07:24:10.752 DEBUG:boto:Token: None 2022-09-03T07:24:10.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.758 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ba99fb523c20a9f1-006313011a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.760 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.761 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.762 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.763 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:10.764 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:10.765 DEBUG:boto:Method: GET 2022-09-03T07:24:10.766 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:10.767 DEBUG:boto:Data: 2022-09-03T07:24:10.768 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.769 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.770 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.771 DEBUG:boto:Params: {} 2022-09-03T07:24:10.772 DEBUG:boto:Token: None 2022-09-03T07:24:10.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.780 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001b0c39f693324225-006313011a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.781 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.782 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:24:10.783 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.784 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.785 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.786 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.787 DEBUG:boto:Method: GET 2022-09-03T07:24:10.788 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.789 DEBUG:boto:Data: 2022-09-03T07:24:10.790 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.791 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.792 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.793 DEBUG:boto:Params: {} 2022-09-03T07:24:10.794 DEBUG:boto:Token: None 2022-09-03T07:24:10.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OaBnkj6MqjDzBgPxMmjtNRFV4tQ= 2022-09-03T07:24:10.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OaBnkj6MqjDzBgPxMmjtNRFV4tQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.800 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000adb2a7d8218acbcb-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.801 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.802 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.802 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.803 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.803 DEBUG:boto:Method: GET 2022-09-03T07:24:10.804 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.804 DEBUG:boto:Data: 2022-09-03T07:24:10.805 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.806 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.806 DEBUG:boto:Params: {} 2022-09-03T07:24:10.807 DEBUG:boto:Token: None 2022-09-03T07:24:10.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OaBnkj6MqjDzBgPxMmjtNRFV4tQ= 2022-09-03T07:24:10.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OaBnkj6MqjDzBgPxMmjtNRFV4tQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.811 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f32a2c80df01985-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.812 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.813 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.813 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.814 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.814 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.815 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.815 DEBUG:boto:Data: 2022-09-03T07:24:10.816 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.816 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.817 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.817 DEBUG:boto:Params: {} 2022-09-03T07:24:10.818 DEBUG:boto:Token: None 2022-09-03T07:24:10.818 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LxhKHRyzMJUrHRNsWP+Hm3o0SsM= 2022-09-03T07:24:10.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LxhKHRyzMJUrHRNsWP+Hm3o0SsM=', 'Content-Length': '0'} 2022-09-03T07:24:10.821 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f087f0ab177699a-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.822 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.823 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.823 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.824 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.824 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.825 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.825 DEBUG:boto:Data: 2022-09-03T07:24:10.826 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.827 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.827 DEBUG:boto:Params: {} 2022-09-03T07:24:10.828 DEBUG:boto:Token: None 2022-09-03T07:24:10.828 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LxhKHRyzMJUrHRNsWP+Hm3o0SsM= 2022-09-03T07:24:10.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LxhKHRyzMJUrHRNsWP+Hm3o0SsM=', 'Content-Length': '0'} 2022-09-03T07:24:10.831 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000299161e4691ee77c-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.832 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:24:10.833 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.833 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.834 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.834 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.835 DEBUG:boto:Method: GET 2022-09-03T07:24:10.835 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.836 DEBUG:boto:Data: 2022-09-03T07:24:10.836 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.837 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.837 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.838 DEBUG:boto:Params: {} 2022-09-03T07:24:10.838 DEBUG:boto:Token: None 2022-09-03T07:24:10.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OaBnkj6MqjDzBgPxMmjtNRFV4tQ= 2022-09-03T07:24:10.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OaBnkj6MqjDzBgPxMmjtNRFV4tQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.843 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001106316297100115-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.843 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.844 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.844 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.845 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.845 DEBUG:boto:Method: GET 2022-09-03T07:24:10.846 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.847 DEBUG:boto:Data: 2022-09-03T07:24:10.847 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.849 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.849 DEBUG:boto:Params: {} 2022-09-03T07:24:10.850 DEBUG:boto:Token: None 2022-09-03T07:24:10.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:10.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OaBnkj6MqjDzBgPxMmjtNRFV4tQ= 2022-09-03T07:24:10.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OaBnkj6MqjDzBgPxMmjtNRFV4tQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.855 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000223b8e415f030832-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.856 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.856 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.857 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:10.857 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:10.858 DEBUG:boto:Method: GET 2022-09-03T07:24:10.858 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:10.859 DEBUG:boto:Data: 2022-09-03T07:24:10.860 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.861 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.862 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.863 DEBUG:boto:Params: {} 2022-09-03T07:24:10.863 DEBUG:boto:Token: None 2022-09-03T07:24:10.864 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.867 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000967c5b35a2a123dd-006313011a-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.910 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.911 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:10.912 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:10.912 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:10.913 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:10.914 DEBUG:boto:Method: GET 2022-09-03T07:24:10.915 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:10.916 DEBUG:boto:Data: 2022-09-03T07:24:10.917 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.918 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.919 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.920 DEBUG:boto:Params: {} 2022-09-03T07:24:10.920 DEBUG:boto:Token: None 2022-09-03T07:24:10.921 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/ 2022-09-03T07:24:10.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ= 2022-09-03T07:24:10.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Ds4txq+IOlfBlKgDGiY5mtWqTQ=', 'Content-Length': '0'} 2022-09-03T07:24:10.926 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a6e09fc72ed826f-006313011a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.926 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:10.927 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.928 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.929 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.932 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:10.932 DEBUG:boto:Data: 2022-09-03T07:24:10.933 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.934 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.934 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.935 DEBUG:boto:Params: {} 2022-09-03T07:24:10.935 DEBUG:boto:Token: None 2022-09-03T07:24:10.936 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51 2022-09-03T07:24:10.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iCwhcDOFsG0HYYald9muEn9lSG4= 2022-09-03T07:24:10.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iCwhcDOFsG0HYYald9muEn9lSG4=', 'Content-Length': '0'} 2022-09-03T07:24:10.939 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4437774c1a47efa-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.940 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.941 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.941 DEBUG:boto:Method: HEAD 2022-09-03T07:24:10.942 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:10.943 DEBUG:boto:Data: 2022-09-03T07:24:10.943 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.945 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.945 DEBUG:boto:Params: {} 2022-09-03T07:24:10.946 DEBUG:boto:Token: None 2022-09-03T07:24:10.946 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51 2022-09-03T07:24:10.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iCwhcDOFsG0HYYald9muEn9lSG4= 2022-09-03T07:24:10.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iCwhcDOFsG0HYYald9muEn9lSG4=', 'Content-Length': '0'} 2022-09-03T07:24:10.950 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ddbb5046f23cfd73-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.951 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:24:10.951 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.952 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.952 DEBUG:boto:Method: GET 2022-09-03T07:24:10.953 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:10.953 DEBUG:boto:Data: 2022-09-03T07:24:10.954 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.955 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:10.955 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.956 DEBUG:boto:Params: {} 2022-09-03T07:24:10.956 DEBUG:boto:Token: None 2022-09-03T07:24:10.957 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51 2022-09-03T07:24:10.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JAt8cq7pCMPNDq56/WAM36wS2fI= 2022-09-03T07:24:10.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JAt8cq7pCMPNDq56/WAM36wS2fI=', 'Content-Length': '0'} 2022-09-03T07:24:10.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9ec380d3efa3a5d-006313011a-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:10.961 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:10.961 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:10.962 DEBUG:boto:Method: GET 2022-09-03T07:24:10.963 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:10.963 DEBUG:boto:Data: 2022-09-03T07:24:10.964 DEBUG:boto:Headers: {} 2022-09-03T07:24:10.964 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:10.965 DEBUG:boto:Port: 8000 2022-09-03T07:24:10.965 DEBUG:boto:Params: {} 2022-09-03T07:24:10.966 DEBUG:boto:Token: None 2022-09-03T07:24:10.967 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:10 GMT /avysmw-75/obj51 2022-09-03T07:24:10.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JAt8cq7pCMPNDq56/WAM36wS2fI= 2022-09-03T07:24:10.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JAt8cq7pCMPNDq56/WAM36wS2fI=', 'Content-Length': '0'} 2022-09-03T07:24:10.971 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc21d43e5166b771-006313011a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.014 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.015 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.015 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:11.016 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:11.016 DEBUG:boto:Method: GET 2022-09-03T07:24:11.017 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:11.018 DEBUG:boto:Data: 2022-09-03T07:24:11.018 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.019 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.019 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.020 DEBUG:boto:Params: {} 2022-09-03T07:24:11.020 DEBUG:boto:Token: None 2022-09-03T07:24:11.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.025 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a71ca75df7f44e2-006313011b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.026 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.027 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.028 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.028 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:11.029 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:11.029 DEBUG:boto:Method: GET 2022-09-03T07:24:11.030 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:11.031 DEBUG:boto:Data: 2022-09-03T07:24:11.031 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.032 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.033 DEBUG:boto:Params: {} 2022-09-03T07:24:11.034 DEBUG:boto:Token: None 2022-09-03T07:24:11.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.039 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000133bd2172bccd506-006313011b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.040 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.040 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:24:11.041 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.041 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.042 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.043 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.043 DEBUG:boto:Method: GET 2022-09-03T07:24:11.044 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.045 DEBUG:boto:Data: 2022-09-03T07:24:11.045 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.046 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.046 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.047 DEBUG:boto:Params: {} 2022-09-03T07:24:11.048 DEBUG:boto:Token: None 2022-09-03T07:24:11.048 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k5paEWWxLPBva39x8TFLld/sedM= 2022-09-03T07:24:11.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k5paEWWxLPBva39x8TFLld/sedM=', 'Content-Length': '0'} 2022-09-03T07:24:11.051 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b51cb0c208c1250a-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.052 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.053 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.053 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.054 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.054 DEBUG:boto:Method: GET 2022-09-03T07:24:11.055 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.056 DEBUG:boto:Data: 2022-09-03T07:24:11.056 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.057 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.058 DEBUG:boto:Params: {} 2022-09-03T07:24:11.058 DEBUG:boto:Token: None 2022-09-03T07:24:11.059 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k5paEWWxLPBva39x8TFLld/sedM= 2022-09-03T07:24:11.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k5paEWWxLPBva39x8TFLld/sedM=', 'Content-Length': '0'} 2022-09-03T07:24:11.062 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012a5ef754d80ad3e-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.063 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.063 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.064 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.064 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.065 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.066 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.066 DEBUG:boto:Data: 2022-09-03T07:24:11.067 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.068 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.068 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.069 DEBUG:boto:Params: {} 2022-09-03T07:24:11.069 DEBUG:boto:Token: None 2022-09-03T07:24:11.070 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MMwirxP0pnh6eT5paO9YVHtszSk= 2022-09-03T07:24:11.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MMwirxP0pnh6eT5paO9YVHtszSk=', 'Content-Length': '0'} 2022-09-03T07:24:11.073 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000790ae2420eeef17a-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.074 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.074 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.075 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.075 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.076 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.076 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.077 DEBUG:boto:Data: 2022-09-03T07:24:11.077 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.078 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.079 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.079 DEBUG:boto:Params: {} 2022-09-03T07:24:11.080 DEBUG:boto:Token: None 2022-09-03T07:24:11.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MMwirxP0pnh6eT5paO9YVHtszSk= 2022-09-03T07:24:11.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MMwirxP0pnh6eT5paO9YVHtszSk=', 'Content-Length': '0'} 2022-09-03T07:24:11.083 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009bfe9f71e333d881-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.084 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:24:11.085 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.086 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.087 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.088 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.089 DEBUG:boto:Method: GET 2022-09-03T07:24:11.090 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.091 DEBUG:boto:Data: 2022-09-03T07:24:11.093 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.094 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.095 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.096 DEBUG:boto:Params: {} 2022-09-03T07:24:11.097 DEBUG:boto:Token: None 2022-09-03T07:24:11.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k5paEWWxLPBva39x8TFLld/sedM= 2022-09-03T07:24:11.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k5paEWWxLPBva39x8TFLld/sedM=', 'Content-Length': '0'} 2022-09-03T07:24:11.106 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007142b206de62c771-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.107 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.107 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.108 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.109 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.109 DEBUG:boto:Method: GET 2022-09-03T07:24:11.110 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.110 DEBUG:boto:Data: 2022-09-03T07:24:11.111 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.112 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.112 DEBUG:boto:Params: {} 2022-09-03T07:24:11.113 DEBUG:boto:Token: None 2022-09-03T07:24:11.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:11.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k5paEWWxLPBva39x8TFLld/sedM= 2022-09-03T07:24:11.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k5paEWWxLPBva39x8TFLld/sedM=', 'Content-Length': '0'} 2022-09-03T07:24:11.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd4049ecba6cc6e2-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.158 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.159 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.159 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:11.160 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:11.161 DEBUG:boto:Method: GET 2022-09-03T07:24:11.163 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:11.164 DEBUG:boto:Data: 2022-09-03T07:24:11.165 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.166 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.168 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.169 DEBUG:boto:Params: {} 2022-09-03T07:24:11.170 DEBUG:boto:Token: None 2022-09-03T07:24:11.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.177 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000024c7913fc10423e0-006313011b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.179 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.180 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.181 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.182 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:11.183 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:11.184 DEBUG:boto:Method: GET 2022-09-03T07:24:11.185 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:11.186 DEBUG:boto:Data: 2022-09-03T07:24:11.187 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.188 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.189 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.189 DEBUG:boto:Params: {} 2022-09-03T07:24:11.191 DEBUG:boto:Token: None 2022-09-03T07:24:11.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.194 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.198 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000770b539c35110317-006313011b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.198 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.199 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.200 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.200 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.201 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:11.201 DEBUG:boto:Data: 2022-09-03T07:24:11.202 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.202 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.203 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.203 DEBUG:boto:Params: {} 2022-09-03T07:24:11.204 DEBUG:boto:Token: None 2022-09-03T07:24:11.205 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55 2022-09-03T07:24:11.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NC55uOs5ZNeyxKk1ktQ7Eg8grgM= 2022-09-03T07:24:11.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NC55uOs5ZNeyxKk1ktQ7Eg8grgM=', 'Content-Length': '0'} 2022-09-03T07:24:11.208 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006c0d9ee8b4429fc8-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.209 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.209 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.210 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.210 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:11.211 DEBUG:boto:Data: 2022-09-03T07:24:11.211 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.212 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.213 DEBUG:boto:Params: {} 2022-09-03T07:24:11.214 DEBUG:boto:Token: None 2022-09-03T07:24:11.214 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55 2022-09-03T07:24:11.215 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NC55uOs5ZNeyxKk1ktQ7Eg8grgM= 2022-09-03T07:24:11.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NC55uOs5ZNeyxKk1ktQ7Eg8grgM=', 'Content-Length': '0'} 2022-09-03T07:24:11.217 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000981c0ce3d2b6cd79-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.218 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:24:11.219 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.219 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.220 DEBUG:boto:Method: GET 2022-09-03T07:24:11.220 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:11.221 DEBUG:boto:Data: 2022-09-03T07:24:11.222 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.222 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.223 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.223 DEBUG:boto:Params: {} 2022-09-03T07:24:11.224 DEBUG:boto:Token: None 2022-09-03T07:24:11.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55 2022-09-03T07:24:11.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LWrPMMlck5aZwVpOT7nZMWLynjs= 2022-09-03T07:24:11.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LWrPMMlck5aZwVpOT7nZMWLynjs=', 'Content-Length': '0'} 2022-09-03T07:24:11.229 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ad181045595ec24-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.230 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:11.230 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:11.231 DEBUG:boto:Method: GET 2022-09-03T07:24:11.232 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:11.233 DEBUG:boto:Data: 2022-09-03T07:24:11.234 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.236 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.237 DEBUG:boto:Params: {} 2022-09-03T07:24:11.238 DEBUG:boto:Token: None 2022-09-03T07:24:11.239 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj55 2022-09-03T07:24:11.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LWrPMMlck5aZwVpOT7nZMWLynjs= 2022-09-03T07:24:11.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LWrPMMlck5aZwVpOT7nZMWLynjs=', 'Content-Length': '0'} 2022-09-03T07:24:11.244 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000adf6846aef07c78f-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.245 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.246 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.247 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:11.248 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:11.249 DEBUG:boto:Method: GET 2022-09-03T07:24:11.250 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:11.251 DEBUG:boto:Data: 2022-09-03T07:24:11.252 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.253 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.254 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.254 DEBUG:boto:Params: {} 2022-09-03T07:24:11.255 DEBUG:boto:Token: None 2022-09-03T07:24:11.256 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.260 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000acd5d6dfe2c77b39-006313011b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.261 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.262 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.262 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.263 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:11.263 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:11.264 DEBUG:boto:Method: GET 2022-09-03T07:24:11.264 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:11.265 DEBUG:boto:Data: 2022-09-03T07:24:11.265 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.266 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.266 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.267 DEBUG:boto:Params: {} 2022-09-03T07:24:11.268 DEBUG:boto:Token: None 2022-09-03T07:24:11.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.273 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ef27f7b5155300c4-006313011b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.274 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.275 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:24:11.275 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.276 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.276 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.277 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.277 DEBUG:boto:Method: GET 2022-09-03T07:24:11.278 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.278 DEBUG:boto:Data: 2022-09-03T07:24:11.279 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.280 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.280 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.281 DEBUG:boto:Params: {} 2022-09-03T07:24:11.281 DEBUG:boto:Token: None 2022-09-03T07:24:11.282 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TM2tNBcRGO01h8u46pcA27PouuQ= 2022-09-03T07:24:11.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TM2tNBcRGO01h8u46pcA27PouuQ=', 'Content-Length': '0'} 2022-09-03T07:24:11.286 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9917f891855caba-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.286 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.287 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.288 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.288 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.289 DEBUG:boto:Method: GET 2022-09-03T07:24:11.289 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.290 DEBUG:boto:Data: 2022-09-03T07:24:11.290 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.291 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.291 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.292 DEBUG:boto:Params: {} 2022-09-03T07:24:11.293 DEBUG:boto:Token: None 2022-09-03T07:24:11.293 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TM2tNBcRGO01h8u46pcA27PouuQ= 2022-09-03T07:24:11.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TM2tNBcRGO01h8u46pcA27PouuQ=', 'Content-Length': '0'} 2022-09-03T07:24:11.297 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d4dfd94e150f8f7-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.298 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.298 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.299 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.299 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.300 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.301 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.302 DEBUG:boto:Data: 2022-09-03T07:24:11.303 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.303 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.304 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.304 DEBUG:boto:Params: {} 2022-09-03T07:24:11.305 DEBUG:boto:Token: None 2022-09-03T07:24:11.305 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.306 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPK174h1MhOciCifLYpZYmWa9uc= 2022-09-03T07:24:11.307 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPK174h1MhOciCifLYpZYmWa9uc=', 'Content-Length': '0'} 2022-09-03T07:24:11.309 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d9d8e036f379a02-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.310 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.310 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.311 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.311 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.312 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.312 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.313 DEBUG:boto:Data: 2022-09-03T07:24:11.313 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.314 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.315 DEBUG:boto:Params: {} 2022-09-03T07:24:11.315 DEBUG:boto:Token: None 2022-09-03T07:24:11.316 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPK174h1MhOciCifLYpZYmWa9uc= 2022-09-03T07:24:11.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPK174h1MhOciCifLYpZYmWa9uc=', 'Content-Length': '0'} 2022-09-03T07:24:11.319 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000139f4bdd2fde5aa8-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.320 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:24:11.320 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.321 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.321 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.322 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.322 DEBUG:boto:Method: GET 2022-09-03T07:24:11.323 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.323 DEBUG:boto:Data: 2022-09-03T07:24:11.324 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.324 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.325 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.326 DEBUG:boto:Params: {} 2022-09-03T07:24:11.326 DEBUG:boto:Token: None 2022-09-03T07:24:11.327 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TM2tNBcRGO01h8u46pcA27PouuQ= 2022-09-03T07:24:11.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TM2tNBcRGO01h8u46pcA27PouuQ=', 'Content-Length': '0'} 2022-09-03T07:24:11.330 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c83a250fa1eba16-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.331 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.332 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.332 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.333 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.333 DEBUG:boto:Method: GET 2022-09-03T07:24:11.334 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.334 DEBUG:boto:Data: 2022-09-03T07:24:11.335 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.336 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.337 DEBUG:boto:Params: {} 2022-09-03T07:24:11.337 DEBUG:boto:Token: None 2022-09-03T07:24:11.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:11.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TM2tNBcRGO01h8u46pcA27PouuQ= 2022-09-03T07:24:11.339 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TM2tNBcRGO01h8u46pcA27PouuQ=', 'Content-Length': '0'} 2022-09-03T07:24:11.341 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cea083afddcc854f-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.342 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.343 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.343 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:11.344 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:11.345 DEBUG:boto:Method: GET 2022-09-03T07:24:11.345 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:11.346 DEBUG:boto:Data: 2022-09-03T07:24:11.346 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.347 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.347 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.348 DEBUG:boto:Params: {} 2022-09-03T07:24:11.348 DEBUG:boto:Token: None 2022-09-03T07:24:11.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.353 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d24680e2dc5efa5-006313011b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.353 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.354 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.355 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.355 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:11.356 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:11.356 DEBUG:boto:Method: GET 2022-09-03T07:24:11.357 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:11.357 DEBUG:boto:Data: 2022-09-03T07:24:11.358 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.359 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.359 DEBUG:boto:Params: {} 2022-09-03T07:24:11.360 DEBUG:boto:Token: None 2022-09-03T07:24:11.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.365 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e69a71abb0e342f-006313011b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.366 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.367 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.367 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.368 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.368 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:11.369 DEBUG:boto:Data: 2022-09-03T07:24:11.369 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.370 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.370 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.371 DEBUG:boto:Params: {} 2022-09-03T07:24:11.371 DEBUG:boto:Token: None 2022-09-03T07:24:11.372 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57 2022-09-03T07:24:11.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7rYoFB6bv1KG7lzKftciTJDGtp0= 2022-09-03T07:24:11.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7rYoFB6bv1KG7lzKftciTJDGtp0=', 'Content-Length': '0'} 2022-09-03T07:24:11.375 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001eed51d361794302-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.376 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.376 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.377 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.377 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:11.378 DEBUG:boto:Data: 2022-09-03T07:24:11.379 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.380 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.380 DEBUG:boto:Params: {} 2022-09-03T07:24:11.381 DEBUG:boto:Token: None 2022-09-03T07:24:11.381 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57 2022-09-03T07:24:11.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7rYoFB6bv1KG7lzKftciTJDGtp0= 2022-09-03T07:24:11.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7rYoFB6bv1KG7lzKftciTJDGtp0=', 'Content-Length': '0'} 2022-09-03T07:24:11.385 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005505621454c0353b-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.385 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:24:11.386 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.387 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.387 DEBUG:boto:Method: GET 2022-09-03T07:24:11.388 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:11.389 DEBUG:boto:Data: 2022-09-03T07:24:11.389 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.390 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.391 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.392 DEBUG:boto:Params: {} 2022-09-03T07:24:11.392 DEBUG:boto:Token: None 2022-09-03T07:24:11.393 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57 2022-09-03T07:24:11.394 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iaw79gvNmZiH9NwFa0M5ougIosU= 2022-09-03T07:24:11.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iaw79gvNmZiH9NwFa0M5ougIosU=', 'Content-Length': '0'} 2022-09-03T07:24:11.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b5c4d779452c3d42-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.398 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:11.398 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:11.399 DEBUG:boto:Method: GET 2022-09-03T07:24:11.399 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:11.400 DEBUG:boto:Data: 2022-09-03T07:24:11.400 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.402 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.402 DEBUG:boto:Params: {} 2022-09-03T07:24:11.403 DEBUG:boto:Token: None 2022-09-03T07:24:11.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj57 2022-09-03T07:24:11.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iaw79gvNmZiH9NwFa0M5ougIosU= 2022-09-03T07:24:11.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iaw79gvNmZiH9NwFa0M5ougIosU=', 'Content-Length': '0'} 2022-09-03T07:24:11.407 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed7eff07233a12c4-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.407 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.408 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.408 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:11.409 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:11.409 DEBUG:boto:Method: GET 2022-09-03T07:24:11.410 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:11.411 DEBUG:boto:Data: 2022-09-03T07:24:11.411 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.412 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.412 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.413 DEBUG:boto:Params: {} 2022-09-03T07:24:11.413 DEBUG:boto:Token: None 2022-09-03T07:24:11.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.418 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b66c69ef358ef60a-006313011b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.419 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.420 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.420 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.421 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:11.421 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:11.422 DEBUG:boto:Method: GET 2022-09-03T07:24:11.422 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:11.423 DEBUG:boto:Data: 2022-09-03T07:24:11.423 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.424 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.425 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.425 DEBUG:boto:Params: {} 2022-09-03T07:24:11.426 DEBUG:boto:Token: None 2022-09-03T07:24:11.426 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.427 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.431 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027afa853dbea0ea3-006313011b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.432 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.433 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:24:11.433 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.434 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.435 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.436 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.436 DEBUG:boto:Method: GET 2022-09-03T07:24:11.437 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.437 DEBUG:boto:Data: 2022-09-03T07:24:11.438 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.438 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.439 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.439 DEBUG:boto:Params: {} 2022-09-03T07:24:11.440 DEBUG:boto:Token: None 2022-09-03T07:24:11.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RvEkK8UVfcqCp719m6l2EDA6mNc= 2022-09-03T07:24:11.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RvEkK8UVfcqCp719m6l2EDA6mNc=', 'Content-Length': '0'} 2022-09-03T07:24:11.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007b9826ba95cff25-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.490 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.491 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.492 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.493 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.494 DEBUG:boto:Method: GET 2022-09-03T07:24:11.496 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.496 DEBUG:boto:Data: 2022-09-03T07:24:11.498 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.500 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.501 DEBUG:boto:Params: {} 2022-09-03T07:24:11.502 DEBUG:boto:Token: None 2022-09-03T07:24:11.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RvEkK8UVfcqCp719m6l2EDA6mNc= 2022-09-03T07:24:11.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RvEkK8UVfcqCp719m6l2EDA6mNc=', 'Content-Length': '0'} 2022-09-03T07:24:11.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000393b6841f186a714-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.511 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.511 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.512 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.512 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.513 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.513 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.514 DEBUG:boto:Data: 2022-09-03T07:24:11.514 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.515 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.515 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.516 DEBUG:boto:Params: {} 2022-09-03T07:24:11.516 DEBUG:boto:Token: None 2022-09-03T07:24:11.517 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n3B77ZL8Ym2i8HHiPB9fi4AOI1c= 2022-09-03T07:24:11.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n3B77ZL8Ym2i8HHiPB9fi4AOI1c=', 'Content-Length': '0'} 2022-09-03T07:24:11.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f99f9b8654bc9a90-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.521 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.522 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.522 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.523 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.523 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.524 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.525 DEBUG:boto:Data: 2022-09-03T07:24:11.526 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.527 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.528 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.529 DEBUG:boto:Params: {} 2022-09-03T07:24:11.530 DEBUG:boto:Token: None 2022-09-03T07:24:11.531 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.532 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n3B77ZL8Ym2i8HHiPB9fi4AOI1c= 2022-09-03T07:24:11.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n3B77ZL8Ym2i8HHiPB9fi4AOI1c=', 'Content-Length': '0'} 2022-09-03T07:24:11.536 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a31648f78bca7284-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.537 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:24:11.537 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.538 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.539 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.539 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.540 DEBUG:boto:Method: GET 2022-09-03T07:24:11.540 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.541 DEBUG:boto:Data: 2022-09-03T07:24:11.541 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.542 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.543 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.543 DEBUG:boto:Params: {} 2022-09-03T07:24:11.544 DEBUG:boto:Token: None 2022-09-03T07:24:11.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RvEkK8UVfcqCp719m6l2EDA6mNc= 2022-09-03T07:24:11.546 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RvEkK8UVfcqCp719m6l2EDA6mNc=', 'Content-Length': '0'} 2022-09-03T07:24:11.548 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1d956a0a02af61a-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.549 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.549 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.550 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.551 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.552 DEBUG:boto:Method: GET 2022-09-03T07:24:11.553 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.554 DEBUG:boto:Data: 2022-09-03T07:24:11.554 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.556 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.556 DEBUG:boto:Params: {} 2022-09-03T07:24:11.557 DEBUG:boto:Token: None 2022-09-03T07:24:11.558 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:11.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RvEkK8UVfcqCp719m6l2EDA6mNc= 2022-09-03T07:24:11.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RvEkK8UVfcqCp719m6l2EDA6mNc=', 'Content-Length': '0'} 2022-09-03T07:24:11.562 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000047e9455527e9d9ed-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.563 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.564 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.565 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:11.566 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:11.567 DEBUG:boto:Method: GET 2022-09-03T07:24:11.567 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:11.568 DEBUG:boto:Data: 2022-09-03T07:24:11.569 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.569 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.570 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.570 DEBUG:boto:Params: {} 2022-09-03T07:24:11.571 DEBUG:boto:Token: None 2022-09-03T07:24:11.572 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.575 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e10ce6e427e522e2-006313011b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.576 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.577 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.577 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.578 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:11.578 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:11.579 DEBUG:boto:Method: GET 2022-09-03T07:24:11.579 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:11.580 DEBUG:boto:Data: 2022-09-03T07:24:11.580 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.581 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.582 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.582 DEBUG:boto:Params: {} 2022-09-03T07:24:11.583 DEBUG:boto:Token: None 2022-09-03T07:24:11.583 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.584 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.588 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003041a31c09712ee3-006313011b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.588 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.589 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.590 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.590 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.591 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:11.591 DEBUG:boto:Data: 2022-09-03T07:24:11.592 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.592 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.593 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.593 DEBUG:boto:Params: {} 2022-09-03T07:24:11.594 DEBUG:boto:Token: None 2022-09-03T07:24:11.595 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68 2022-09-03T07:24:11.595 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GZy+NhXEH7Z19rqkdcsA7QWx7FY= 2022-09-03T07:24:11.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GZy+NhXEH7Z19rqkdcsA7QWx7FY=', 'Content-Length': '0'} 2022-09-03T07:24:11.598 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eecf6468a85846cc-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.599 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.599 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.600 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.600 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:11.601 DEBUG:boto:Data: 2022-09-03T07:24:11.601 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.602 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.602 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.603 DEBUG:boto:Params: {} 2022-09-03T07:24:11.603 DEBUG:boto:Token: None 2022-09-03T07:24:11.604 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68 2022-09-03T07:24:11.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GZy+NhXEH7Z19rqkdcsA7QWx7FY= 2022-09-03T07:24:11.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GZy+NhXEH7Z19rqkdcsA7QWx7FY=', 'Content-Length': '0'} 2022-09-03T07:24:11.607 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f8af45349317afe5-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.608 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:24:11.609 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.609 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.610 DEBUG:boto:Method: GET 2022-09-03T07:24:11.610 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:11.611 DEBUG:boto:Data: 2022-09-03T07:24:11.611 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.612 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.613 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.613 DEBUG:boto:Params: {} 2022-09-03T07:24:11.614 DEBUG:boto:Token: None 2022-09-03T07:24:11.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68 2022-09-03T07:24:11.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XgcNjg+3raxDBKAKFZ5zbKkUSpg= 2022-09-03T07:24:11.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XgcNjg+3raxDBKAKFZ5zbKkUSpg=', 'Content-Length': '0'} 2022-09-03T07:24:11.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1c63fc855f4ccff-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.618 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:11.619 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:11.620 DEBUG:boto:Method: GET 2022-09-03T07:24:11.620 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:11.621 DEBUG:boto:Data: 2022-09-03T07:24:11.621 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.622 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.623 DEBUG:boto:Params: {} 2022-09-03T07:24:11.624 DEBUG:boto:Token: None 2022-09-03T07:24:11.624 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj68 2022-09-03T07:24:11.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XgcNjg+3raxDBKAKFZ5zbKkUSpg= 2022-09-03T07:24:11.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XgcNjg+3raxDBKAKFZ5zbKkUSpg=', 'Content-Length': '0'} 2022-09-03T07:24:11.627 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a548428340eef997-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.670 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.671 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.671 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:11.672 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:11.672 DEBUG:boto:Method: GET 2022-09-03T07:24:11.673 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:11.673 DEBUG:boto:Data: 2022-09-03T07:24:11.674 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.674 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.675 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.676 DEBUG:boto:Params: {} 2022-09-03T07:24:11.676 DEBUG:boto:Token: None 2022-09-03T07:24:11.677 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.677 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.681 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f49426ca02b06166-006313011b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.682 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.683 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.683 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.684 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:11.685 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:11.685 DEBUG:boto:Method: GET 2022-09-03T07:24:11.686 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:11.686 DEBUG:boto:Data: 2022-09-03T07:24:11.687 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.688 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.688 DEBUG:boto:Params: {} 2022-09-03T07:24:11.689 DEBUG:boto:Token: None 2022-09-03T07:24:11.690 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.695 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003fef3441e2e3a5f4-006313011b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.696 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.697 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:24:11.698 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.699 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.700 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.701 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.702 DEBUG:boto:Method: GET 2022-09-03T07:24:11.702 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.703 DEBUG:boto:Data: 2022-09-03T07:24:11.703 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.704 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.705 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.706 DEBUG:boto:Params: {} 2022-09-03T07:24:11.707 DEBUG:boto:Token: None 2022-09-03T07:24:11.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q0d2K3CWxcwprhpax0tzHk+CV58= 2022-09-03T07:24:11.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q0d2K3CWxcwprhpax0tzHk+CV58=', 'Content-Length': '0'} 2022-09-03T07:24:11.712 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b32a88cb0e4e30a5-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.713 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.713 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.714 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.715 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.715 DEBUG:boto:Method: GET 2022-09-03T07:24:11.716 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.716 DEBUG:boto:Data: 2022-09-03T07:24:11.717 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.718 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.719 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.719 DEBUG:boto:Params: {} 2022-09-03T07:24:11.720 DEBUG:boto:Token: None 2022-09-03T07:24:11.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q0d2K3CWxcwprhpax0tzHk+CV58= 2022-09-03T07:24:11.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q0d2K3CWxcwprhpax0tzHk+CV58=', 'Content-Length': '0'} 2022-09-03T07:24:11.724 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe4c14a0f0903084-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.725 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.725 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.726 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.727 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.727 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.728 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.728 DEBUG:boto:Data: 2022-09-03T07:24:11.729 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.729 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.730 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.731 DEBUG:boto:Params: {} 2022-09-03T07:24:11.731 DEBUG:boto:Token: None 2022-09-03T07:24:11.732 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a9T2WvG++DFf+O9Xa6sO2AeCMOU= 2022-09-03T07:24:11.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a9T2WvG++DFf+O9Xa6sO2AeCMOU=', 'Content-Length': '0'} 2022-09-03T07:24:11.736 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003eadfd1d26d2cb9-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.737 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.738 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.738 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.739 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.739 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.740 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.740 DEBUG:boto:Data: 2022-09-03T07:24:11.741 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.741 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.742 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.743 DEBUG:boto:Params: {} 2022-09-03T07:24:11.744 DEBUG:boto:Token: None 2022-09-03T07:24:11.745 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a9T2WvG++DFf+O9Xa6sO2AeCMOU= 2022-09-03T07:24:11.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a9T2WvG++DFf+O9Xa6sO2AeCMOU=', 'Content-Length': '0'} 2022-09-03T07:24:11.749 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057441a321318888f-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.751 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:24:11.752 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.752 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.753 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.754 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.755 DEBUG:boto:Method: GET 2022-09-03T07:24:11.756 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.757 DEBUG:boto:Data: 2022-09-03T07:24:11.757 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.758 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.759 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.759 DEBUG:boto:Params: {} 2022-09-03T07:24:11.760 DEBUG:boto:Token: None 2022-09-03T07:24:11.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q0d2K3CWxcwprhpax0tzHk+CV58= 2022-09-03T07:24:11.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q0d2K3CWxcwprhpax0tzHk+CV58=', 'Content-Length': '0'} 2022-09-03T07:24:11.764 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000042aa7ec8842d5c9-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.765 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.766 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.766 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.767 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.767 DEBUG:boto:Method: GET 2022-09-03T07:24:11.768 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.768 DEBUG:boto:Data: 2022-09-03T07:24:11.769 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.769 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.770 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.770 DEBUG:boto:Params: {} 2022-09-03T07:24:11.771 DEBUG:boto:Token: None 2022-09-03T07:24:11.772 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:11.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q0d2K3CWxcwprhpax0tzHk+CV58= 2022-09-03T07:24:11.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q0d2K3CWxcwprhpax0tzHk+CV58=', 'Content-Length': '0'} 2022-09-03T07:24:11.775 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac0d96af300bd941-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.776 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.776 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.777 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:11.777 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:11.778 DEBUG:boto:Method: GET 2022-09-03T07:24:11.779 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:11.779 DEBUG:boto:Data: 2022-09-03T07:24:11.780 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.780 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.781 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.781 DEBUG:boto:Params: {} 2022-09-03T07:24:11.782 DEBUG:boto:Token: None 2022-09-03T07:24:11.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.786 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4eecea9e0cb2fdc-006313011b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.787 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.788 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.788 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.789 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:11.789 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:11.790 DEBUG:boto:Method: GET 2022-09-03T07:24:11.790 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:11.791 DEBUG:boto:Data: 2022-09-03T07:24:11.791 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.792 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.792 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.793 DEBUG:boto:Params: {} 2022-09-03T07:24:11.793 DEBUG:boto:Token: None 2022-09-03T07:24:11.794 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.798 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007941ac51a82b8a53-006313011b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.799 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.800 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.800 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.801 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.802 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:11.803 DEBUG:boto:Data: 2022-09-03T07:24:11.803 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.804 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.805 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.806 DEBUG:boto:Params: {} 2022-09-03T07:24:11.807 DEBUG:boto:Token: None 2022-09-03T07:24:11.808 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69 2022-09-03T07:24:11.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GA+RHXnOxMwscJWuqzV7C1A6PPU= 2022-09-03T07:24:11.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GA+RHXnOxMwscJWuqzV7C1A6PPU=', 'Content-Length': '0'} 2022-09-03T07:24:11.811 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ade9a3c4d459e586-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.812 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.813 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.813 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.814 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:11.814 DEBUG:boto:Data: 2022-09-03T07:24:11.815 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.816 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.816 DEBUG:boto:Params: {} 2022-09-03T07:24:11.817 DEBUG:boto:Token: None 2022-09-03T07:24:11.817 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69 2022-09-03T07:24:11.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GA+RHXnOxMwscJWuqzV7C1A6PPU= 2022-09-03T07:24:11.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GA+RHXnOxMwscJWuqzV7C1A6PPU=', 'Content-Length': '0'} 2022-09-03T07:24:11.820 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000885011d4e0477def-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.821 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:24:11.821 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.822 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.823 DEBUG:boto:Method: GET 2022-09-03T07:24:11.823 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:11.824 DEBUG:boto:Data: 2022-09-03T07:24:11.824 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.825 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.825 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.826 DEBUG:boto:Params: {} 2022-09-03T07:24:11.826 DEBUG:boto:Token: None 2022-09-03T07:24:11.827 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69 2022-09-03T07:24:11.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FG6T15rzMZdZRUHZJK4ix/ei+A8= 2022-09-03T07:24:11.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FG6T15rzMZdZRUHZJK4ix/ei+A8=', 'Content-Length': '0'} 2022-09-03T07:24:11.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fac1e7c435aa5fa6-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.874 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:11.875 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:11.875 DEBUG:boto:Method: GET 2022-09-03T07:24:11.876 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:11.877 DEBUG:boto:Data: 2022-09-03T07:24:11.877 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.878 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.879 DEBUG:boto:Params: {} 2022-09-03T07:24:11.880 DEBUG:boto:Token: None 2022-09-03T07:24:11.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj69 2022-09-03T07:24:11.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FG6T15rzMZdZRUHZJK4ix/ei+A8= 2022-09-03T07:24:11.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FG6T15rzMZdZRUHZJK4ix/ei+A8=', 'Content-Length': '0'} 2022-09-03T07:24:11.883 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cbd6b2dbc9996785-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.884 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.885 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.886 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:11.887 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:11.888 DEBUG:boto:Method: GET 2022-09-03T07:24:11.888 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:11.889 DEBUG:boto:Data: 2022-09-03T07:24:11.889 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.891 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.891 DEBUG:boto:Params: {} 2022-09-03T07:24:11.892 DEBUG:boto:Token: None 2022-09-03T07:24:11.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.897 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa5729ee57a9e65b-006313011b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.897 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.898 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.899 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.900 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:11.901 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:11.901 DEBUG:boto:Method: GET 2022-09-03T07:24:11.902 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:11.902 DEBUG:boto:Data: 2022-09-03T07:24:11.903 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.903 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.904 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.904 DEBUG:boto:Params: {} 2022-09-03T07:24:11.905 DEBUG:boto:Token: None 2022-09-03T07:24:11.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:11.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:11.910 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000373e9721ff6ed1ca-006313011b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.911 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:11.912 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:24:11.913 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:11.913 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:11.914 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.914 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.915 DEBUG:boto:Method: GET 2022-09-03T07:24:11.916 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.916 DEBUG:boto:Data: 2022-09-03T07:24:11.917 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.918 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.919 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.920 DEBUG:boto:Params: {} 2022-09-03T07:24:11.921 DEBUG:boto:Token: None 2022-09-03T07:24:11.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D34pArT48sb/lpiF6BuyvIXMn9A= 2022-09-03T07:24:11.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D34pArT48sb/lpiF6BuyvIXMn9A=', 'Content-Length': '0'} 2022-09-03T07:24:11.927 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076d1970798e040c9-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.928 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:11.929 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:11.930 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.931 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.932 DEBUG:boto:Method: GET 2022-09-03T07:24:11.933 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.934 DEBUG:boto:Data: 2022-09-03T07:24:11.934 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.935 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.935 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.936 DEBUG:boto:Params: {} 2022-09-03T07:24:11.936 DEBUG:boto:Token: None 2022-09-03T07:24:11.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D34pArT48sb/lpiF6BuyvIXMn9A= 2022-09-03T07:24:11.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D34pArT48sb/lpiF6BuyvIXMn9A=', 'Content-Length': '0'} 2022-09-03T07:24:11.940 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4a31b638e2c8b53-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.941 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:11.941 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:11.942 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.942 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.943 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.943 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.944 DEBUG:boto:Data: 2022-09-03T07:24:11.944 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.945 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.945 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.946 DEBUG:boto:Params: {} 2022-09-03T07:24:11.946 DEBUG:boto:Token: None 2022-09-03T07:24:11.947 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8g8JA64B1a9E2UzsrWigCb3EF/U= 2022-09-03T07:24:11.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8g8JA64B1a9E2UzsrWigCb3EF/U=', 'Content-Length': '0'} 2022-09-03T07:24:11.950 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a91e5daa6cb80d72-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.951 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:11.951 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:11.952 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.952 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.953 DEBUG:boto:Method: HEAD 2022-09-03T07:24:11.953 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.954 DEBUG:boto:Data: 2022-09-03T07:24:11.954 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.955 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.955 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.956 DEBUG:boto:Params: {} 2022-09-03T07:24:11.956 DEBUG:boto:Token: None 2022-09-03T07:24:11.957 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8g8JA64B1a9E2UzsrWigCb3EF/U= 2022-09-03T07:24:11.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8g8JA64B1a9E2UzsrWigCb3EF/U=', 'Content-Length': '0'} 2022-09-03T07:24:11.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e8dbfbffe1a6724-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.961 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:24:11.961 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:11.962 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:11.962 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.964 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.965 DEBUG:boto:Method: GET 2022-09-03T07:24:11.966 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.967 DEBUG:boto:Data: 2022-09-03T07:24:11.968 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.969 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.970 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.971 DEBUG:boto:Params: {} 2022-09-03T07:24:11.972 DEBUG:boto:Token: None 2022-09-03T07:24:11.973 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D34pArT48sb/lpiF6BuyvIXMn9A= 2022-09-03T07:24:11.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D34pArT48sb/lpiF6BuyvIXMn9A=', 'Content-Length': '0'} 2022-09-03T07:24:11.978 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000932e1a6ea2dc91f3-006313011b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.979 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:11.980 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:11.981 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.982 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.982 DEBUG:boto:Method: GET 2022-09-03T07:24:11.983 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.984 DEBUG:boto:Data: 2022-09-03T07:24:11.985 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:11.986 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.986 DEBUG:boto:Params: {} 2022-09-03T07:24:11.987 DEBUG:boto:Token: None 2022-09-03T07:24:11.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:11.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D34pArT48sb/lpiF6BuyvIXMn9A= 2022-09-03T07:24:11.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D34pArT48sb/lpiF6BuyvIXMn9A=', 'Content-Length': '0'} 2022-09-03T07:24:11.991 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1657260d0ddc841-006313011b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:11.992 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:11.992 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:11.993 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:11.993 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:11.994 DEBUG:boto:Method: GET 2022-09-03T07:24:11.994 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:11.995 DEBUG:boto:Data: 2022-09-03T07:24:11.996 DEBUG:boto:Headers: {} 2022-09-03T07:24:11.996 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:11.997 DEBUG:boto:Port: 8000 2022-09-03T07:24:11.997 DEBUG:boto:Params: {} 2022-09-03T07:24:11.998 DEBUG:boto:Token: None 2022-09-03T07:24:11.999 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:11 GMT /avysmw-75/ 2022-09-03T07:24:11.999 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s= 2022-09-03T07:24:12.000 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UxmObsJvFyLoGoZ1dhntfUbiK8s=', 'Content-Length': '0'} 2022-09-03T07:24:12.002 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f0214166ee186538-006313011c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.046 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.048 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.048 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.049 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:12.050 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:12.051 DEBUG:boto:Method: GET 2022-09-03T07:24:12.052 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:12.052 DEBUG:boto:Data: 2022-09-03T07:24:12.053 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.054 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.056 DEBUG:boto:Params: {} 2022-09-03T07:24:12.057 DEBUG:boto:Token: None 2022-09-03T07:24:12.058 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.059 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.062 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003487e47ab933b5bc-006313011c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.063 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.065 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:12.066 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:12.067 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.068 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:12.069 DEBUG:boto:Data: 2022-09-03T07:24:12.069 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.070 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.071 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.072 DEBUG:boto:Params: {} 2022-09-03T07:24:12.073 DEBUG:boto:Token: None 2022-09-03T07:24:12.074 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj76 2022-09-03T07:24:12.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gRx+dCGmczd38AB22eJOpJfi/nY= 2022-09-03T07:24:12.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gRx+dCGmczd38AB22eJOpJfi/nY=', 'Content-Length': '0'} 2022-09-03T07:24:12.078 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008cd07b5a636553a4-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.079 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:12.079 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:12.080 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.081 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:12.081 DEBUG:boto:Data: 2022-09-03T07:24:12.082 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.083 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.083 DEBUG:boto:Params: {} 2022-09-03T07:24:12.084 DEBUG:boto:Token: None 2022-09-03T07:24:12.084 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj76 2022-09-03T07:24:12.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gRx+dCGmczd38AB22eJOpJfi/nY= 2022-09-03T07:24:12.086 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gRx+dCGmczd38AB22eJOpJfi/nY=', 'Content-Length': '0'} 2022-09-03T07:24:12.088 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000303e6c79e38e7e0c-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.088 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:24:12.089 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:12.089 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:12.090 DEBUG:boto:Method: GET 2022-09-03T07:24:12.091 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:12.092 DEBUG:boto:Data: 2022-09-03T07:24:12.093 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.094 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.095 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.096 DEBUG:boto:Params: {} 2022-09-03T07:24:12.096 DEBUG:boto:Token: None 2022-09-03T07:24:12.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj76 2022-09-03T07:24:12.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y9hM8qY9eLv7M8EaxGt+MTgNCeA= 2022-09-03T07:24:12.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y9hM8qY9eLv7M8EaxGt+MTgNCeA=', 'Content-Length': '0'} 2022-09-03T07:24:12.101 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a322e3d782afd751-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.102 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:12.103 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:12.104 DEBUG:boto:Method: GET 2022-09-03T07:24:12.104 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:12.105 DEBUG:boto:Data: 2022-09-03T07:24:12.105 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.106 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.107 DEBUG:boto:Params: {} 2022-09-03T07:24:12.108 DEBUG:boto:Token: None 2022-09-03T07:24:12.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj76 2022-09-03T07:24:12.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y9hM8qY9eLv7M8EaxGt+MTgNCeA= 2022-09-03T07:24:12.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y9hM8qY9eLv7M8EaxGt+MTgNCeA=', 'Content-Length': '0'} 2022-09-03T07:24:12.112 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034201f15013173d5-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.114 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.115 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.116 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:12.117 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:12.118 DEBUG:boto:Method: GET 2022-09-03T07:24:12.119 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:12.120 DEBUG:boto:Data: 2022-09-03T07:24:12.121 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.122 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.123 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.124 DEBUG:boto:Params: {} 2022-09-03T07:24:12.125 DEBUG:boto:Token: None 2022-09-03T07:24:12.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.130 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb7ca03e78091b11-006313011c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.131 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.132 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.133 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.133 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:12.134 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:12.134 DEBUG:boto:Method: GET 2022-09-03T07:24:12.135 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:12.135 DEBUG:boto:Data: 2022-09-03T07:24:12.136 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.137 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.138 DEBUG:boto:Params: {} 2022-09-03T07:24:12.138 DEBUG:boto:Token: None 2022-09-03T07:24:12.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.143 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e10ff2d6b7e3ae7e-006313011c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.144 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.145 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:24:12.146 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.146 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.147 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.148 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.148 DEBUG:boto:Method: GET 2022-09-03T07:24:12.149 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.150 DEBUG:boto:Data: 2022-09-03T07:24:12.151 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.152 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.153 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.154 DEBUG:boto:Params: {} 2022-09-03T07:24:12.155 DEBUG:boto:Token: None 2022-09-03T07:24:12.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JTVh+Dq0rb0Teq4/YsxH0G+PUbY= 2022-09-03T07:24:12.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JTVh+Dq0rb0Teq4/YsxH0G+PUbY=', 'Content-Length': '0'} 2022-09-03T07:24:12.159 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb92af23e8096db5-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.202 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.203 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.204 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.205 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.205 DEBUG:boto:Method: GET 2022-09-03T07:24:12.206 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.207 DEBUG:boto:Data: 2022-09-03T07:24:12.207 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.208 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.209 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.209 DEBUG:boto:Params: {} 2022-09-03T07:24:12.210 DEBUG:boto:Token: None 2022-09-03T07:24:12.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JTVh+Dq0rb0Teq4/YsxH0G+PUbY= 2022-09-03T07:24:12.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JTVh+Dq0rb0Teq4/YsxH0G+PUbY=', 'Content-Length': '0'} 2022-09-03T07:24:12.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc5d1c855886019d-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.217 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.217 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.218 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.218 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.219 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.220 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.220 DEBUG:boto:Data: 2022-09-03T07:24:12.221 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.222 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.223 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.223 DEBUG:boto:Params: {} 2022-09-03T07:24:12.224 DEBUG:boto:Token: None 2022-09-03T07:24:12.225 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u4aInn/j8HuRfFIgrjH+L+bHze0= 2022-09-03T07:24:12.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u4aInn/j8HuRfFIgrjH+L+bHze0=', 'Content-Length': '0'} 2022-09-03T07:24:12.228 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000088eb74dd90df7883-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.229 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.230 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.230 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.231 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.232 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.232 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.233 DEBUG:boto:Data: 2022-09-03T07:24:12.233 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.234 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.235 DEBUG:boto:Params: {} 2022-09-03T07:24:12.235 DEBUG:boto:Token: None 2022-09-03T07:24:12.236 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u4aInn/j8HuRfFIgrjH+L+bHze0= 2022-09-03T07:24:12.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u4aInn/j8HuRfFIgrjH+L+bHze0=', 'Content-Length': '0'} 2022-09-03T07:24:12.239 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026f78c8af07a6c72-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.240 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:24:12.240 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.241 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.242 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.243 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.243 DEBUG:boto:Method: GET 2022-09-03T07:24:12.244 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.245 DEBUG:boto:Data: 2022-09-03T07:24:12.245 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.246 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.247 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.248 DEBUG:boto:Params: {} 2022-09-03T07:24:12.249 DEBUG:boto:Token: None 2022-09-03T07:24:12.250 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JTVh+Dq0rb0Teq4/YsxH0G+PUbY= 2022-09-03T07:24:12.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JTVh+Dq0rb0Teq4/YsxH0G+PUbY=', 'Content-Length': '0'} 2022-09-03T07:24:12.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c83a5a1a3cfcb762-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.256 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.257 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.259 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.260 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.261 DEBUG:boto:Method: GET 2022-09-03T07:24:12.262 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.264 DEBUG:boto:Data: 2022-09-03T07:24:12.265 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.267 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.268 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.269 DEBUG:boto:Params: {} 2022-09-03T07:24:12.271 DEBUG:boto:Token: None 2022-09-03T07:24:12.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:12.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JTVh+Dq0rb0Teq4/YsxH0G+PUbY= 2022-09-03T07:24:12.274 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JTVh+Dq0rb0Teq4/YsxH0G+PUbY=', 'Content-Length': '0'} 2022-09-03T07:24:12.277 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c49fcd1e9c4e625-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.280 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.281 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.282 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:12.283 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:12.284 DEBUG:boto:Method: GET 2022-09-03T07:24:12.285 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:12.287 DEBUG:boto:Data: 2022-09-03T07:24:12.288 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.289 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.290 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.291 DEBUG:boto:Params: {} 2022-09-03T07:24:12.292 DEBUG:boto:Token: None 2022-09-03T07:24:12.293 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.298 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c6828f92313f3d2-006313011c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.299 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.300 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.300 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.301 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:12.302 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:12.303 DEBUG:boto:Method: GET 2022-09-03T07:24:12.304 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:12.304 DEBUG:boto:Data: 2022-09-03T07:24:12.305 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.306 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.307 DEBUG:boto:Params: {} 2022-09-03T07:24:12.308 DEBUG:boto:Token: None 2022-09-03T07:24:12.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.315 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000042c470e0528c8bd4-006313011c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.316 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.318 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.318 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.319 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.320 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:12.321 DEBUG:boto:Data: 2022-09-03T07:24:12.321 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.322 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.323 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.323 DEBUG:boto:Params: {} 2022-09-03T07:24:12.324 DEBUG:boto:Token: None 2022-09-03T07:24:12.325 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85 2022-09-03T07:24:12.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7PpaHtplMB40k+IZ8UP1C4CPo1E= 2022-09-03T07:24:12.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7PpaHtplMB40k+IZ8UP1C4CPo1E=', 'Content-Length': '0'} 2022-09-03T07:24:12.330 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1545b2d4d0e1b45-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.331 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.332 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.334 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.335 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:12.336 DEBUG:boto:Data: 2022-09-03T07:24:12.337 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.338 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.339 DEBUG:boto:Params: {} 2022-09-03T07:24:12.339 DEBUG:boto:Token: None 2022-09-03T07:24:12.340 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85 2022-09-03T07:24:12.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7PpaHtplMB40k+IZ8UP1C4CPo1E= 2022-09-03T07:24:12.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7PpaHtplMB40k+IZ8UP1C4CPo1E=', 'Content-Length': '0'} 2022-09-03T07:24:12.344 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a312cdf5c2f0c46-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.345 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:24:12.345 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.346 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.347 DEBUG:boto:Method: GET 2022-09-03T07:24:12.348 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:12.349 DEBUG:boto:Data: 2022-09-03T07:24:12.350 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.350 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.351 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.352 DEBUG:boto:Params: {} 2022-09-03T07:24:12.353 DEBUG:boto:Token: None 2022-09-03T07:24:12.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85 2022-09-03T07:24:12.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LrylCmQeZR+dk5sUGTSPJsd4OCA= 2022-09-03T07:24:12.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LrylCmQeZR+dk5sUGTSPJsd4OCA=', 'Content-Length': '0'} 2022-09-03T07:24:12.358 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3e6b6d8b044fd83-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.359 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:12.360 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:12.360 DEBUG:boto:Method: GET 2022-09-03T07:24:12.361 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:12.361 DEBUG:boto:Data: 2022-09-03T07:24:12.362 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.363 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.363 DEBUG:boto:Params: {} 2022-09-03T07:24:12.364 DEBUG:boto:Token: None 2022-09-03T07:24:12.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj85 2022-09-03T07:24:12.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LrylCmQeZR+dk5sUGTSPJsd4OCA= 2022-09-03T07:24:12.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LrylCmQeZR+dk5sUGTSPJsd4OCA=', 'Content-Length': '0'} 2022-09-03T07:24:12.368 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008675c57d221c20f-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.421 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.422 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.423 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:12.423 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:12.424 DEBUG:boto:Method: GET 2022-09-03T07:24:12.424 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:12.425 DEBUG:boto:Data: 2022-09-03T07:24:12.425 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.426 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.426 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.427 DEBUG:boto:Params: {} 2022-09-03T07:24:12.428 DEBUG:boto:Token: None 2022-09-03T07:24:12.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.429 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.432 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008694377abf3e0e7e-006313011c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.433 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.434 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.434 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.435 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:12.435 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:12.436 DEBUG:boto:Method: GET 2022-09-03T07:24:12.436 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:12.437 DEBUG:boto:Data: 2022-09-03T07:24:12.438 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.439 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.440 DEBUG:boto:Params: {} 2022-09-03T07:24:12.440 DEBUG:boto:Token: None 2022-09-03T07:24:12.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.446 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1af661a7df6ccca-006313011c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.447 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.448 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:24:12.449 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.450 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.450 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.451 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.451 DEBUG:boto:Method: GET 2022-09-03T07:24:12.452 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.452 DEBUG:boto:Data: 2022-09-03T07:24:12.453 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.453 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.454 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.455 DEBUG:boto:Params: {} 2022-09-03T07:24:12.455 DEBUG:boto:Token: None 2022-09-03T07:24:12.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy3cA2JlpdNcsRzEJaB+iPgyAXE= 2022-09-03T07:24:12.457 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy3cA2JlpdNcsRzEJaB+iPgyAXE=', 'Content-Length': '0'} 2022-09-03T07:24:12.460 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f222a5e338eb28c4-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.461 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.461 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.462 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.463 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.463 DEBUG:boto:Method: GET 2022-09-03T07:24:12.464 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.465 DEBUG:boto:Data: 2022-09-03T07:24:12.465 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.467 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.467 DEBUG:boto:Params: {} 2022-09-03T07:24:12.468 DEBUG:boto:Token: None 2022-09-03T07:24:12.468 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy3cA2JlpdNcsRzEJaB+iPgyAXE= 2022-09-03T07:24:12.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy3cA2JlpdNcsRzEJaB+iPgyAXE=', 'Content-Length': '0'} 2022-09-03T07:24:12.472 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bdf6dee5bd6ef087-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.473 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.473 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.474 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.475 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.475 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.476 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.477 DEBUG:boto:Data: 2022-09-03T07:24:12.477 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.478 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.478 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.479 DEBUG:boto:Params: {} 2022-09-03T07:24:12.479 DEBUG:boto:Token: None 2022-09-03T07:24:12.480 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0xJ6nyIc1TlC7jd1wYqVMJbH99Q= 2022-09-03T07:24:12.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0xJ6nyIc1TlC7jd1wYqVMJbH99Q=', 'Content-Length': '0'} 2022-09-03T07:24:12.484 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c44bac29770867c-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.485 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.486 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.487 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.488 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.489 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.489 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.490 DEBUG:boto:Data: 2022-09-03T07:24:12.491 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.492 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.493 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.493 DEBUG:boto:Params: {} 2022-09-03T07:24:12.494 DEBUG:boto:Token: None 2022-09-03T07:24:12.495 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0xJ6nyIc1TlC7jd1wYqVMJbH99Q= 2022-09-03T07:24:12.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0xJ6nyIc1TlC7jd1wYqVMJbH99Q=', 'Content-Length': '0'} 2022-09-03T07:24:12.500 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ec841341f4280f7-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.501 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:24:12.501 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.502 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.503 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.503 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.504 DEBUG:boto:Method: GET 2022-09-03T07:24:12.505 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.506 DEBUG:boto:Data: 2022-09-03T07:24:12.507 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.507 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.508 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.509 DEBUG:boto:Params: {} 2022-09-03T07:24:12.510 DEBUG:boto:Token: None 2022-09-03T07:24:12.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.512 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy3cA2JlpdNcsRzEJaB+iPgyAXE= 2022-09-03T07:24:12.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy3cA2JlpdNcsRzEJaB+iPgyAXE=', 'Content-Length': '0'} 2022-09-03T07:24:12.514 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af827378fccf0d31-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.515 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.516 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.516 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.517 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.517 DEBUG:boto:Method: GET 2022-09-03T07:24:12.518 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.519 DEBUG:boto:Data: 2022-09-03T07:24:12.520 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.521 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.522 DEBUG:boto:Params: {} 2022-09-03T07:24:12.523 DEBUG:boto:Token: None 2022-09-03T07:24:12.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:12.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jy3cA2JlpdNcsRzEJaB+iPgyAXE= 2022-09-03T07:24:12.525 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jy3cA2JlpdNcsRzEJaB+iPgyAXE=', 'Content-Length': '0'} 2022-09-03T07:24:12.526 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007209f0c30d151e95-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.527 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.528 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.528 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:12.529 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:12.530 DEBUG:boto:Method: GET 2022-09-03T07:24:12.531 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:12.532 DEBUG:boto:Data: 2022-09-03T07:24:12.532 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.533 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.534 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.535 DEBUG:boto:Params: {} 2022-09-03T07:24:12.536 DEBUG:boto:Token: None 2022-09-03T07:24:12.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.541 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e38f585121a4b5f7-006313011c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.586 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.587 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.588 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.588 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:12.589 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:12.589 DEBUG:boto:Method: GET 2022-09-03T07:24:12.590 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:12.590 DEBUG:boto:Data: 2022-09-03T07:24:12.591 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.592 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.592 DEBUG:boto:Params: {} 2022-09-03T07:24:12.593 DEBUG:boto:Token: None 2022-09-03T07:24:12.594 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.599 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006ef3fa13eb803fd5-006313011c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.600 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.600 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.601 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.601 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.602 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:12.603 DEBUG:boto:Data: 2022-09-03T07:24:12.603 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.604 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.605 DEBUG:boto:Params: {} 2022-09-03T07:24:12.605 DEBUG:boto:Token: None 2022-09-03T07:24:12.606 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92 2022-09-03T07:24:12.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2J30zrmhBMdk3QKEW9iOQezKbjQ= 2022-09-03T07:24:12.607 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2J30zrmhBMdk3QKEW9iOQezKbjQ=', 'Content-Length': '0'} 2022-09-03T07:24:12.609 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d21073c6a40372d6-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.610 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.610 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.611 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.611 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:12.612 DEBUG:boto:Data: 2022-09-03T07:24:12.612 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.613 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.614 DEBUG:boto:Params: {} 2022-09-03T07:24:12.614 DEBUG:boto:Token: None 2022-09-03T07:24:12.615 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92 2022-09-03T07:24:12.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2J30zrmhBMdk3QKEW9iOQezKbjQ= 2022-09-03T07:24:12.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2J30zrmhBMdk3QKEW9iOQezKbjQ=', 'Content-Length': '0'} 2022-09-03T07:24:12.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3dfc1e05ee7669f-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.619 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:24:12.619 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.620 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.620 DEBUG:boto:Method: GET 2022-09-03T07:24:12.622 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:12.622 DEBUG:boto:Data: 2022-09-03T07:24:12.623 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.624 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.624 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.625 DEBUG:boto:Params: {} 2022-09-03T07:24:12.625 DEBUG:boto:Token: None 2022-09-03T07:24:12.626 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92 2022-09-03T07:24:12.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m2ZI+qw36Ld8Ck/A7cv4ZSXM7bE= 2022-09-03T07:24:12.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m2ZI+qw36Ld8Ck/A7cv4ZSXM7bE=', 'Content-Length': '0'} 2022-09-03T07:24:12.629 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004938c18d35746b75-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.630 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:12.631 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:12.631 DEBUG:boto:Method: GET 2022-09-03T07:24:12.632 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:12.632 DEBUG:boto:Data: 2022-09-03T07:24:12.633 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.634 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.635 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.635 DEBUG:boto:Params: {} 2022-09-03T07:24:12.636 DEBUG:boto:Token: None 2022-09-03T07:24:12.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj92 2022-09-03T07:24:12.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m2ZI+qw36Ld8Ck/A7cv4ZSXM7bE= 2022-09-03T07:24:12.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m2ZI+qw36Ld8Ck/A7cv4ZSXM7bE=', 'Content-Length': '0'} 2022-09-03T07:24:12.640 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d30fd5ad9ceef5a2-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.682 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.683 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.683 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:12.684 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:12.686 DEBUG:boto:Method: GET 2022-09-03T07:24:12.687 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:12.687 DEBUG:boto:Data: 2022-09-03T07:24:12.688 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.688 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.689 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.689 DEBUG:boto:Params: {} 2022-09-03T07:24:12.690 DEBUG:boto:Token: None 2022-09-03T07:24:12.691 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.695 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000018a9386e38cb3663-006313011c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.696 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.696 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.697 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.698 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:12.698 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:12.699 DEBUG:boto:Method: GET 2022-09-03T07:24:12.699 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:12.700 DEBUG:boto:Data: 2022-09-03T07:24:12.700 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.701 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.702 DEBUG:boto:Params: {} 2022-09-03T07:24:12.703 DEBUG:boto:Token: None 2022-09-03T07:24:12.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.707 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003d00442044f114c3-006313011c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.708 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.709 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:24:12.709 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.710 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.711 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.711 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.712 DEBUG:boto:Method: GET 2022-09-03T07:24:12.713 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.713 DEBUG:boto:Data: 2022-09-03T07:24:12.714 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.715 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.715 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.716 DEBUG:boto:Params: {} 2022-09-03T07:24:12.717 DEBUG:boto:Token: None 2022-09-03T07:24:12.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/7Rinz1G4ZUORPf6HKjFYWMLI7A= 2022-09-03T07:24:12.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/7Rinz1G4ZUORPf6HKjFYWMLI7A=', 'Content-Length': '0'} 2022-09-03T07:24:12.720 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b558d182deeb045-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.721 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.722 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.722 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.723 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.724 DEBUG:boto:Method: GET 2022-09-03T07:24:12.725 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.726 DEBUG:boto:Data: 2022-09-03T07:24:12.727 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.729 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.730 DEBUG:boto:Params: {} 2022-09-03T07:24:12.730 DEBUG:boto:Token: None 2022-09-03T07:24:12.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/7Rinz1G4ZUORPf6HKjFYWMLI7A= 2022-09-03T07:24:12.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/7Rinz1G4ZUORPf6HKjFYWMLI7A=', 'Content-Length': '0'} 2022-09-03T07:24:12.735 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f3860e95fe3e1ab-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.736 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.737 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.738 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.739 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.740 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.741 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.742 DEBUG:boto:Data: 2022-09-03T07:24:12.743 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.744 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.745 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.746 DEBUG:boto:Params: {} 2022-09-03T07:24:12.747 DEBUG:boto:Token: None 2022-09-03T07:24:12.748 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1FPmAW5oCiBV1tVCUZlkBg27hzM= 2022-09-03T07:24:12.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1FPmAW5oCiBV1tVCUZlkBg27hzM=', 'Content-Length': '0'} 2022-09-03T07:24:12.752 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dafbaab5951debb8-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.754 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.755 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.756 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.757 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.758 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.759 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.760 DEBUG:boto:Data: 2022-09-03T07:24:12.761 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.761 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.761 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.762 DEBUG:boto:Params: {} 2022-09-03T07:24:12.763 DEBUG:boto:Token: None 2022-09-03T07:24:12.763 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1FPmAW5oCiBV1tVCUZlkBg27hzM= 2022-09-03T07:24:12.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1FPmAW5oCiBV1tVCUZlkBg27hzM=', 'Content-Length': '0'} 2022-09-03T07:24:12.766 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000864772bd606036e8-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.768 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:24:12.769 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.769 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.770 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.771 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.772 DEBUG:boto:Method: GET 2022-09-03T07:24:12.773 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.774 DEBUG:boto:Data: 2022-09-03T07:24:12.775 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.775 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.776 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.776 DEBUG:boto:Params: {} 2022-09-03T07:24:12.777 DEBUG:boto:Token: None 2022-09-03T07:24:12.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/7Rinz1G4ZUORPf6HKjFYWMLI7A= 2022-09-03T07:24:12.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/7Rinz1G4ZUORPf6HKjFYWMLI7A=', 'Content-Length': '0'} 2022-09-03T07:24:12.780 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f11c180912960ff-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.781 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.782 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.783 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.783 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.784 DEBUG:boto:Method: GET 2022-09-03T07:24:12.784 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.785 DEBUG:boto:Data: 2022-09-03T07:24:12.785 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.786 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.787 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.787 DEBUG:boto:Params: {} 2022-09-03T07:24:12.788 DEBUG:boto:Token: None 2022-09-03T07:24:12.788 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:12.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/7Rinz1G4ZUORPf6HKjFYWMLI7A= 2022-09-03T07:24:12.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/7Rinz1G4ZUORPf6HKjFYWMLI7A=', 'Content-Length': '0'} 2022-09-03T07:24:12.792 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001285ae7c42ee7a23-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.834 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.835 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.836 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:12.837 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:12.838 DEBUG:boto:Method: GET 2022-09-03T07:24:12.839 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:12.840 DEBUG:boto:Data: 2022-09-03T07:24:12.841 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.842 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.843 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.844 DEBUG:boto:Params: {} 2022-09-03T07:24:12.845 DEBUG:boto:Token: None 2022-09-03T07:24:12.846 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.847 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.848 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.851 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008ce313c86cc5ad83-006313011c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.852 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.854 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.855 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.857 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:12.858 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:12.859 DEBUG:boto:Method: GET 2022-09-03T07:24:12.860 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:12.861 DEBUG:boto:Data: 2022-09-03T07:24:12.862 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.863 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.864 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.865 DEBUG:boto:Params: {} 2022-09-03T07:24:12.866 DEBUG:boto:Token: None 2022-09-03T07:24:12.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.873 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb6839db81bfdff6-006313011c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.874 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.876 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.877 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.878 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.879 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:12.879 DEBUG:boto:Data: 2022-09-03T07:24:12.880 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.881 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.882 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.883 DEBUG:boto:Params: {} 2022-09-03T07:24:12.884 DEBUG:boto:Token: None 2022-09-03T07:24:12.885 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95 2022-09-03T07:24:12.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VcdywezuSjINPnEULPMmXfslHtc= 2022-09-03T07:24:12.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VcdywezuSjINPnEULPMmXfslHtc=', 'Content-Length': '0'} 2022-09-03T07:24:12.890 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f5163c0139266dc-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.891 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.892 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.893 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.894 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:12.895 DEBUG:boto:Data: 2022-09-03T07:24:12.896 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.897 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.898 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.899 DEBUG:boto:Params: {} 2022-09-03T07:24:12.900 DEBUG:boto:Token: None 2022-09-03T07:24:12.901 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95 2022-09-03T07:24:12.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VcdywezuSjINPnEULPMmXfslHtc= 2022-09-03T07:24:12.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VcdywezuSjINPnEULPMmXfslHtc=', 'Content-Length': '0'} 2022-09-03T07:24:12.905 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000040b4e6cbc666224a-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.907 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:24:12.908 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.909 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.910 DEBUG:boto:Method: GET 2022-09-03T07:24:12.911 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:12.912 DEBUG:boto:Data: 2022-09-03T07:24:12.913 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.914 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.915 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.916 DEBUG:boto:Params: {} 2022-09-03T07:24:12.917 DEBUG:boto:Token: None 2022-09-03T07:24:12.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95 2022-09-03T07:24:12.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qes1Q9N9oarl0ZQDwWdj4XVJZXk= 2022-09-03T07:24:12.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qes1Q9N9oarl0ZQDwWdj4XVJZXk=', 'Content-Length': '0'} 2022-09-03T07:24:12.923 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c4276de2e0b9d0c-006313011c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.925 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:12.925 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:12.926 DEBUG:boto:Method: GET 2022-09-03T07:24:12.927 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:12.928 DEBUG:boto:Data: 2022-09-03T07:24:12.929 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.931 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.932 DEBUG:boto:Params: {} 2022-09-03T07:24:12.933 DEBUG:boto:Token: None 2022-09-03T07:24:12.934 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/obj95 2022-09-03T07:24:12.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qes1Q9N9oarl0ZQDwWdj4XVJZXk= 2022-09-03T07:24:12.936 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qes1Q9N9oarl0ZQDwWdj4XVJZXk=', 'Content-Length': '0'} 2022-09-03T07:24:12.939 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037feedee23728f2d-006313011c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.940 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.940 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.941 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:12.941 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:12.942 DEBUG:boto:Method: GET 2022-09-03T07:24:12.943 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:12.944 DEBUG:boto:Data: 2022-09-03T07:24:12.945 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.946 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:12.947 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.947 DEBUG:boto:Params: {} 2022-09-03T07:24:12.948 DEBUG:boto:Token: None 2022-09-03T07:24:12.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.954 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5ba06c8f9423cd1-006313011c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.955 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.956 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.957 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.958 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:12.959 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:12.960 DEBUG:boto:Method: GET 2022-09-03T07:24:12.961 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:12.962 DEBUG:boto:Data: 2022-09-03T07:24:12.963 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.964 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.965 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.966 DEBUG:boto:Params: {} 2022-09-03T07:24:12.967 DEBUG:boto:Token: None 2022-09-03T07:24:12.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8= 2022-09-03T07:24:12.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7v886Df2EQn+am7gSC78zYPrL+8=', 'Content-Length': '0'} 2022-09-03T07:24:12.974 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d608f292f2bac386-006313011c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.975 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:12.978 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:24:12.979 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:24:12.980 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.980 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.981 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.981 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:12.982 DEBUG:boto:Data: 2022-09-03T07:24:12.983 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:12.984 DEBUG:boto:Port: 8001 2022-09-03T07:24:12.985 DEBUG:boto:Params: {} 2022-09-03T07:24:12.985 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:24:12.986 DEBUG:boto:Token: None 2022-09-03T07:24:12.987 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6pivKCYRdEeUi1oRy/kL5HhMgSY= 2022-09-03T07:24:12.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6pivKCYRdEeUi1oRy/kL5HhMgSY=', 'Content-Length': '0'} 2022-09-03T07:24:12.992 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '20'), ('X-RGW-Bytes-Used', '60'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006448db60455338db-006313011c-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:12.992 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:12.993 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:12.993 DEBUG:boto:Method: HEAD 2022-09-03T07:24:12.994 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:12.994 DEBUG:boto:Data: 2022-09-03T07:24:12.995 DEBUG:boto:Headers: {} 2022-09-03T07:24:12.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:12.996 DEBUG:boto:Port: 8000 2022-09-03T07:24:12.996 DEBUG:boto:Params: {} 2022-09-03T07:24:12.997 DEBUG:boto:Token: None 2022-09-03T07:24:12.997 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:12 GMT /avysmw-75/ 2022-09-03T07:24:12.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6pivKCYRdEeUi1oRy/kL5HhMgSY= 2022-09-03T07:24:12.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6pivKCYRdEeUi1oRy/kL5HhMgSY=', 'Content-Length': '0'} 2022-09-03T07:24:13.002 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '20'), ('X-RGW-Bytes-Used', '60'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000084de4967a7d3823f-006313011c-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.003 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:24:13.004 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.005 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.005 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.006 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.006 DEBUG:boto:Method: GET 2022-09-03T07:24:13.007 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.007 DEBUG:boto:Data: 2022-09-03T07:24:13.008 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.009 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.010 DEBUG:boto:Params: {} 2022-09-03T07:24:13.010 DEBUG:boto:Token: None 2022-09-03T07:24:13.011 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/?versions 2022-09-03T07:24:13.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7qV5HNJ1gnK8Zsm0FiGz8/IPH7o= 2022-09-03T07:24:13.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7qV5HNJ1gnK8Zsm0FiGz8/IPH7o=', 'Content-Length': '0'} 2022-09-03T07:24:13.015 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f91b20b9e39a756b-006313011d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.016 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.018 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:24:13.019 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:24:13.019 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:24:13.020 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:24:13.020 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:24:13.021 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:24:13.021 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:24:13.022 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:24:13.022 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:24:13.023 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:24:13.024 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:24:13.024 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:24:13.025 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:24:13.025 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:24:13.026 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:24:13.026 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:24:13.027 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:24:13.028 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:24:13.028 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:24:13.029 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:24:13.029 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:24:13.030 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.031 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.031 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.032 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.032 DEBUG:boto:Method: GET 2022-09-03T07:24:13.033 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.034 DEBUG:boto:Data: 2022-09-03T07:24:13.034 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.035 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.036 DEBUG:boto:Params: {} 2022-09-03T07:24:13.037 DEBUG:boto:Token: None 2022-09-03T07:24:13.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/?versions 2022-09-03T07:24:13.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7qV5HNJ1gnK8Zsm0FiGz8/IPH7o= 2022-09-03T07:24:13.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7qV5HNJ1gnK8Zsm0FiGz8/IPH7o=', 'Content-Length': '0'} 2022-09-03T07:24:13.042 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000033c63a8d6744d2c-006313011d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.043 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.045 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:24:13.045 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:24:13.046 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:24:13.046 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:24:13.047 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:24:13.047 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:24:13.048 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:24:13.048 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:24:13.048 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:24:13.049 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:24:13.049 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:24:13.050 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:24:13.050 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:24:13.051 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:24:13.052 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:24:13.052 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:24:13.053 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:24:13.053 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:24:13.054 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:24:13.054 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:24:13.055 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.055 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.056 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.056 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.057 DEBUG:boto:Method: GET 2022-09-03T07:24:13.058 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.058 DEBUG:boto:Data: 2022-09-03T07:24:13.059 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.060 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.060 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.061 DEBUG:boto:Params: {} 2022-09-03T07:24:13.061 DEBUG:boto:Token: None 2022-09-03T07:24:13.062 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/?versions 2022-09-03T07:24:13.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7qV5HNJ1gnK8Zsm0FiGz8/IPH7o= 2022-09-03T07:24:13.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7qV5HNJ1gnK8Zsm0FiGz8/IPH7o=', 'Content-Length': '0'} 2022-09-03T07:24:13.066 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6192ae5643c4cc1-006313011d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.067 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.069 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.069 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.070 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.070 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.071 DEBUG:boto:Method: GET 2022-09-03T07:24:13.072 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:13.072 DEBUG:boto:Data: 2022-09-03T07:24:13.073 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.074 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.075 DEBUG:boto:Params: {} 2022-09-03T07:24:13.076 DEBUG:boto:Token: None 2022-09-03T07:24:13.076 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/?versions 2022-09-03T07:24:13.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7qV5HNJ1gnK8Zsm0FiGz8/IPH7o= 2022-09-03T07:24:13.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7qV5HNJ1gnK8Zsm0FiGz8/IPH7o=', 'Content-Length': '0'} 2022-09-03T07:24:13.081 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000094b037dd2788a0f0-006313011d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.082 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.084 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:24:13.084 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.085 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.085 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.086 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.086 DEBUG:boto:Method: GET 2022-09-03T07:24:13.087 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.087 DEBUG:boto:Data: 2022-09-03T07:24:13.088 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.088 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.089 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.090 DEBUG:boto:Params: {} 2022-09-03T07:24:13.091 DEBUG:boto:Token: None 2022-09-03T07:24:13.091 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4U1d+lVq30rliDZ1r0jHHDXT/dA= 2022-09-03T07:24:13.093 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4U1d+lVq30rliDZ1r0jHHDXT/dA=', 'Content-Length': '0'} 2022-09-03T07:24:13.095 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005487225815a452bc-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.096 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.096 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.097 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.097 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.098 DEBUG:boto:Method: GET 2022-09-03T07:24:13.099 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.099 DEBUG:boto:Data: 2022-09-03T07:24:13.100 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.101 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.102 DEBUG:boto:Params: {} 2022-09-03T07:24:13.102 DEBUG:boto:Token: None 2022-09-03T07:24:13.103 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4U1d+lVq30rliDZ1r0jHHDXT/dA= 2022-09-03T07:24:13.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4U1d+lVq30rliDZ1r0jHHDXT/dA=', 'Content-Length': '0'} 2022-09-03T07:24:13.106 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000287d66025f73710d-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.150 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.151 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.151 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.152 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.152 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.153 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.153 DEBUG:boto:Data: 2022-09-03T07:24:13.154 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.154 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.155 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.155 DEBUG:boto:Params: {} 2022-09-03T07:24:13.156 DEBUG:boto:Token: None 2022-09-03T07:24:13.156 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ej8Ej5SMYZmbgCLwl+Q9p+Uer1Y= 2022-09-03T07:24:13.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ej8Ej5SMYZmbgCLwl+Q9p+Uer1Y=', 'Content-Length': '0'} 2022-09-03T07:24:13.160 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008815ffea18530ec5-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.161 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.161 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.162 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.162 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.163 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.163 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.164 DEBUG:boto:Data: 2022-09-03T07:24:13.164 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.165 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.166 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.166 DEBUG:boto:Params: {} 2022-09-03T07:24:13.167 DEBUG:boto:Token: None 2022-09-03T07:24:13.167 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ej8Ej5SMYZmbgCLwl+Q9p+Uer1Y= 2022-09-03T07:24:13.169 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ej8Ej5SMYZmbgCLwl+Q9p+Uer1Y=', 'Content-Length': '0'} 2022-09-03T07:24:13.171 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e25142d000b17a67-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.171 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:24:13.172 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.172 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.173 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.173 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.174 DEBUG:boto:Method: GET 2022-09-03T07:24:13.175 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.175 DEBUG:boto:Data: 2022-09-03T07:24:13.176 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.177 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.177 DEBUG:boto:Params: {} 2022-09-03T07:24:13.178 DEBUG:boto:Token: None 2022-09-03T07:24:13.178 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4U1d+lVq30rliDZ1r0jHHDXT/dA= 2022-09-03T07:24:13.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4U1d+lVq30rliDZ1r0jHHDXT/dA=', 'Content-Length': '0'} 2022-09-03T07:24:13.181 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000410d59dd22d95a34-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.182 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.183 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.183 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.184 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.184 DEBUG:boto:Method: GET 2022-09-03T07:24:13.185 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.186 DEBUG:boto:Data: 2022-09-03T07:24:13.186 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.187 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.188 DEBUG:boto:Params: {} 2022-09-03T07:24:13.189 DEBUG:boto:Token: None 2022-09-03T07:24:13.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:13.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4U1d+lVq30rliDZ1r0jHHDXT/dA= 2022-09-03T07:24:13.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4U1d+lVq30rliDZ1r0jHHDXT/dA=', 'Content-Length': '0'} 2022-09-03T07:24:13.192 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ad78df86ff686b2-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.193 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.194 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.194 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:13.195 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:13.195 DEBUG:boto:Method: GET 2022-09-03T07:24:13.196 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:13.196 DEBUG:boto:Data: 2022-09-03T07:24:13.197 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.198 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.198 DEBUG:boto:Params: {} 2022-09-03T07:24:13.199 DEBUG:boto:Token: None 2022-09-03T07:24:13.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.203 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2aee2176442c9e3-006313011d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.204 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.205 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.205 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.206 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:13.206 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:13.207 DEBUG:boto:Method: GET 2022-09-03T07:24:13.207 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:13.208 DEBUG:boto:Data: 2022-09-03T07:24:13.208 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.209 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.209 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.210 DEBUG:boto:Params: {} 2022-09-03T07:24:13.211 DEBUG:boto:Token: None 2022-09-03T07:24:13.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.216 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e01aa40269a42695-006313011d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.217 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.217 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.218 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.218 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.219 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:13.220 DEBUG:boto:Data: 2022-09-03T07:24:13.220 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.221 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.222 DEBUG:boto:Params: {} 2022-09-03T07:24:13.222 DEBUG:boto:Token: None 2022-09-03T07:24:13.223 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11 2022-09-03T07:24:13.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZHzFxPwZQ/wreTPYTy6foUCVi1g= 2022-09-03T07:24:13.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZHzFxPwZQ/wreTPYTy6foUCVi1g=', 'Content-Length': '0'} 2022-09-03T07:24:13.226 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000134bb23550133968-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.227 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.227 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.228 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.228 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:13.229 DEBUG:boto:Data: 2022-09-03T07:24:13.229 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.230 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.231 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.231 DEBUG:boto:Params: {} 2022-09-03T07:24:13.232 DEBUG:boto:Token: None 2022-09-03T07:24:13.232 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11 2022-09-03T07:24:13.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZHzFxPwZQ/wreTPYTy6foUCVi1g= 2022-09-03T07:24:13.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZHzFxPwZQ/wreTPYTy6foUCVi1g=', 'Content-Length': '0'} 2022-09-03T07:24:13.235 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000040385a3f66799bad-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.236 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:24:13.237 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.237 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.238 DEBUG:boto:Method: GET 2022-09-03T07:24:13.239 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:13.239 DEBUG:boto:Data: 2022-09-03T07:24:13.240 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.241 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.241 DEBUG:boto:Params: {} 2022-09-03T07:24:13.242 DEBUG:boto:Token: None 2022-09-03T07:24:13.243 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11 2022-09-03T07:24:13.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KXaqoMAMOMlOihMg9SQzJQunPKQ= 2022-09-03T07:24:13.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KXaqoMAMOMlOihMg9SQzJQunPKQ=', 'Content-Length': '0'} 2022-09-03T07:24:13.246 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000601f2d773d48b43c-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.290 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:13.291 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:13.291 DEBUG:boto:Method: GET 2022-09-03T07:24:13.292 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:13.292 DEBUG:boto:Data: 2022-09-03T07:24:13.293 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.294 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.294 DEBUG:boto:Params: {} 2022-09-03T07:24:13.295 DEBUG:boto:Token: None 2022-09-03T07:24:13.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj11 2022-09-03T07:24:13.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KXaqoMAMOMlOihMg9SQzJQunPKQ= 2022-09-03T07:24:13.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KXaqoMAMOMlOihMg9SQzJQunPKQ=', 'Content-Length': '0'} 2022-09-03T07:24:13.299 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d9990630348c267-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.300 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.301 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.301 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:13.302 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:13.303 DEBUG:boto:Method: GET 2022-09-03T07:24:13.304 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:13.305 DEBUG:boto:Data: 2022-09-03T07:24:13.306 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.307 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.308 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.309 DEBUG:boto:Params: {} 2022-09-03T07:24:13.310 DEBUG:boto:Token: None 2022-09-03T07:24:13.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.317 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000068137f278fb5aaea-006313011d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.317 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.318 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.319 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.320 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:13.320 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:13.321 DEBUG:boto:Method: GET 2022-09-03T07:24:13.322 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:13.322 DEBUG:boto:Data: 2022-09-03T07:24:13.323 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.323 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.324 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.325 DEBUG:boto:Params: {} 2022-09-03T07:24:13.326 DEBUG:boto:Token: None 2022-09-03T07:24:13.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.331 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000017782324c2b6716b-006313011d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.331 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.332 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:24:13.333 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.333 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.334 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.335 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.335 DEBUG:boto:Method: GET 2022-09-03T07:24:13.336 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.336 DEBUG:boto:Data: 2022-09-03T07:24:13.337 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.338 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.338 DEBUG:boto:Params: {} 2022-09-03T07:24:13.339 DEBUG:boto:Token: None 2022-09-03T07:24:13.340 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1IBD0ytf0aK0qDUIYWUkn9jDJ6A= 2022-09-03T07:24:13.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1IBD0ytf0aK0qDUIYWUkn9jDJ6A=', 'Content-Length': '0'} 2022-09-03T07:24:13.344 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f8bb0d119dcb311-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.345 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.345 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.346 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.346 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.347 DEBUG:boto:Method: GET 2022-09-03T07:24:13.347 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.348 DEBUG:boto:Data: 2022-09-03T07:24:13.348 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.349 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.349 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.350 DEBUG:boto:Params: {} 2022-09-03T07:24:13.351 DEBUG:boto:Token: None 2022-09-03T07:24:13.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1IBD0ytf0aK0qDUIYWUkn9jDJ6A= 2022-09-03T07:24:13.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1IBD0ytf0aK0qDUIYWUkn9jDJ6A=', 'Content-Length': '0'} 2022-09-03T07:24:13.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e3fb9fada99a16b-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.355 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.356 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.356 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.357 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.358 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.358 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.359 DEBUG:boto:Data: 2022-09-03T07:24:13.360 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.361 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.361 DEBUG:boto:Params: {} 2022-09-03T07:24:13.362 DEBUG:boto:Token: None 2022-09-03T07:24:13.363 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bTpXkmQnZxMClpvskNatNyaSmBc= 2022-09-03T07:24:13.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bTpXkmQnZxMClpvskNatNyaSmBc=', 'Content-Length': '0'} 2022-09-03T07:24:13.366 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f549bd6da36801f-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.367 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.368 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.369 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.370 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.371 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.372 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.373 DEBUG:boto:Data: 2022-09-03T07:24:13.374 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.376 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.377 DEBUG:boto:Params: {} 2022-09-03T07:24:13.378 DEBUG:boto:Token: None 2022-09-03T07:24:13.379 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bTpXkmQnZxMClpvskNatNyaSmBc= 2022-09-03T07:24:13.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bTpXkmQnZxMClpvskNatNyaSmBc=', 'Content-Length': '0'} 2022-09-03T07:24:13.383 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000695cf175846ea0c8-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.383 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:24:13.384 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.385 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.385 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.386 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.386 DEBUG:boto:Method: GET 2022-09-03T07:24:13.387 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.387 DEBUG:boto:Data: 2022-09-03T07:24:13.388 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.389 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.389 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.390 DEBUG:boto:Params: {} 2022-09-03T07:24:13.390 DEBUG:boto:Token: None 2022-09-03T07:24:13.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1IBD0ytf0aK0qDUIYWUkn9jDJ6A= 2022-09-03T07:24:13.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1IBD0ytf0aK0qDUIYWUkn9jDJ6A=', 'Content-Length': '0'} 2022-09-03T07:24:13.394 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075d362b3ba9e6522-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.395 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.396 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.396 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.397 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.397 DEBUG:boto:Method: GET 2022-09-03T07:24:13.398 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.399 DEBUG:boto:Data: 2022-09-03T07:24:13.399 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.401 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.401 DEBUG:boto:Params: {} 2022-09-03T07:24:13.403 DEBUG:boto:Token: None 2022-09-03T07:24:13.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:13.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1IBD0ytf0aK0qDUIYWUkn9jDJ6A= 2022-09-03T07:24:13.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1IBD0ytf0aK0qDUIYWUkn9jDJ6A=', 'Content-Length': '0'} 2022-09-03T07:24:13.409 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa12015fffec5864-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.454 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.455 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.455 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:13.456 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:13.458 DEBUG:boto:Method: GET 2022-09-03T07:24:13.458 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:13.459 DEBUG:boto:Data: 2022-09-03T07:24:13.460 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.461 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.461 DEBUG:boto:Params: {} 2022-09-03T07:24:13.462 DEBUG:boto:Token: None 2022-09-03T07:24:13.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c769ed5ad91748b1-006313011d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.469 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.470 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.471 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.471 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:13.472 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:13.472 DEBUG:boto:Method: GET 2022-09-03T07:24:13.473 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:13.474 DEBUG:boto:Data: 2022-09-03T07:24:13.475 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.476 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.477 DEBUG:boto:Params: {} 2022-09-03T07:24:13.478 DEBUG:boto:Token: None 2022-09-03T07:24:13.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.486 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004b9f4f4ce1a2a76c-006313011d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.486 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.487 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.488 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.488 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.489 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:13.489 DEBUG:boto:Data: 2022-09-03T07:24:13.490 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.491 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.491 DEBUG:boto:Params: {} 2022-09-03T07:24:13.492 DEBUG:boto:Token: None 2022-09-03T07:24:13.492 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115 2022-09-03T07:24:13.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WzV37/A2G/PjsczPOzH6YD9YY5E= 2022-09-03T07:24:13.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WzV37/A2G/PjsczPOzH6YD9YY5E=', 'Content-Length': '0'} 2022-09-03T07:24:13.496 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000263712953a6e4890-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.496 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.497 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.497 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.498 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:13.498 DEBUG:boto:Data: 2022-09-03T07:24:13.499 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.500 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.500 DEBUG:boto:Params: {} 2022-09-03T07:24:13.501 DEBUG:boto:Token: None 2022-09-03T07:24:13.502 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115 2022-09-03T07:24:13.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WzV37/A2G/PjsczPOzH6YD9YY5E= 2022-09-03T07:24:13.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WzV37/A2G/PjsczPOzH6YD9YY5E=', 'Content-Length': '0'} 2022-09-03T07:24:13.505 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba6d1d603995abcf-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.506 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:24:13.506 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.507 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.507 DEBUG:boto:Method: GET 2022-09-03T07:24:13.508 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:13.508 DEBUG:boto:Data: 2022-09-03T07:24:13.509 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.509 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.510 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.510 DEBUG:boto:Params: {} 2022-09-03T07:24:13.511 DEBUG:boto:Token: None 2022-09-03T07:24:13.512 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115 2022-09-03T07:24:13.512 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DGaXFCxKYFC3iyBn47sX1ntCF7E= 2022-09-03T07:24:13.513 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DGaXFCxKYFC3iyBn47sX1ntCF7E=', 'Content-Length': '0'} 2022-09-03T07:24:13.515 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000073d54a6f20c01024-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.558 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:13.559 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:13.560 DEBUG:boto:Method: GET 2022-09-03T07:24:13.561 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:13.562 DEBUG:boto:Data: 2022-09-03T07:24:13.563 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.565 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.566 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.567 DEBUG:boto:Params: {} 2022-09-03T07:24:13.568 DEBUG:boto:Token: None 2022-09-03T07:24:13.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj115 2022-09-03T07:24:13.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DGaXFCxKYFC3iyBn47sX1ntCF7E= 2022-09-03T07:24:13.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DGaXFCxKYFC3iyBn47sX1ntCF7E=', 'Content-Length': '0'} 2022-09-03T07:24:13.574 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096f8fd1a8d801f8f-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.575 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.576 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.576 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:13.577 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:13.578 DEBUG:boto:Method: GET 2022-09-03T07:24:13.578 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:13.579 DEBUG:boto:Data: 2022-09-03T07:24:13.579 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.580 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.580 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.581 DEBUG:boto:Params: {} 2022-09-03T07:24:13.582 DEBUG:boto:Token: None 2022-09-03T07:24:13.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.587 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000073feb132dc4e10f7-006313011d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.588 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.589 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.590 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.590 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:13.591 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:13.591 DEBUG:boto:Method: GET 2022-09-03T07:24:13.592 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:13.592 DEBUG:boto:Data: 2022-09-03T07:24:13.593 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.594 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.594 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.596 DEBUG:boto:Params: {} 2022-09-03T07:24:13.597 DEBUG:boto:Token: None 2022-09-03T07:24:13.598 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.603 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ee071f74732b7b77-006313011d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.603 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.604 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:24:13.606 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.607 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.607 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.608 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.608 DEBUG:boto:Method: GET 2022-09-03T07:24:13.609 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.609 DEBUG:boto:Data: 2022-09-03T07:24:13.610 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.611 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.612 DEBUG:boto:Params: {} 2022-09-03T07:24:13.612 DEBUG:boto:Token: None 2022-09-03T07:24:13.613 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJydgvE2Y4B0ZsHCNqqpSmRBHkg= 2022-09-03T07:24:13.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJydgvE2Y4B0ZsHCNqqpSmRBHkg=', 'Content-Length': '0'} 2022-09-03T07:24:13.617 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000755b5ba42c4f6bac-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.618 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.618 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.619 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.619 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.620 DEBUG:boto:Method: GET 2022-09-03T07:24:13.620 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.621 DEBUG:boto:Data: 2022-09-03T07:24:13.621 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.622 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.623 DEBUG:boto:Params: {} 2022-09-03T07:24:13.623 DEBUG:boto:Token: None 2022-09-03T07:24:13.624 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJydgvE2Y4B0ZsHCNqqpSmRBHkg= 2022-09-03T07:24:13.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJydgvE2Y4B0ZsHCNqqpSmRBHkg=', 'Content-Length': '0'} 2022-09-03T07:24:13.627 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080f3a2407275f64f-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.674 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.675 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.675 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.676 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.676 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.677 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.677 DEBUG:boto:Data: 2022-09-03T07:24:13.678 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.679 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.679 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.680 DEBUG:boto:Params: {} 2022-09-03T07:24:13.680 DEBUG:boto:Token: None 2022-09-03T07:24:13.681 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7DNERt3EgBe/gTCH/hzZ/7vtloE= 2022-09-03T07:24:13.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7DNERt3EgBe/gTCH/hzZ/7vtloE=', 'Content-Length': '0'} 2022-09-03T07:24:13.684 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000047a1922be28214ca-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.685 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.685 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.686 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.686 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.687 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.688 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.688 DEBUG:boto:Data: 2022-09-03T07:24:13.689 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.690 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.691 DEBUG:boto:Params: {} 2022-09-03T07:24:13.691 DEBUG:boto:Token: None 2022-09-03T07:24:13.692 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7DNERt3EgBe/gTCH/hzZ/7vtloE= 2022-09-03T07:24:13.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7DNERt3EgBe/gTCH/hzZ/7vtloE=', 'Content-Length': '0'} 2022-09-03T07:24:13.695 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b166f5af777e9663-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.696 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:24:13.696 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.697 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.698 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.698 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.699 DEBUG:boto:Method: GET 2022-09-03T07:24:13.699 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.700 DEBUG:boto:Data: 2022-09-03T07:24:13.700 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.702 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.702 DEBUG:boto:Params: {} 2022-09-03T07:24:13.703 DEBUG:boto:Token: None 2022-09-03T07:24:13.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJydgvE2Y4B0ZsHCNqqpSmRBHkg= 2022-09-03T07:24:13.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJydgvE2Y4B0ZsHCNqqpSmRBHkg=', 'Content-Length': '0'} 2022-09-03T07:24:13.706 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ba7b2942edba862-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.707 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.708 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.708 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.709 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.709 DEBUG:boto:Method: GET 2022-09-03T07:24:13.710 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.711 DEBUG:boto:Data: 2022-09-03T07:24:13.712 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.714 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.715 DEBUG:boto:Params: {} 2022-09-03T07:24:13.716 DEBUG:boto:Token: None 2022-09-03T07:24:13.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:13.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fJydgvE2Y4B0ZsHCNqqpSmRBHkg= 2022-09-03T07:24:13.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fJydgvE2Y4B0ZsHCNqqpSmRBHkg=', 'Content-Length': '0'} 2022-09-03T07:24:13.720 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048b687876c1ed4d9-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.720 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.721 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.721 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:13.722 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:13.723 DEBUG:boto:Method: GET 2022-09-03T07:24:13.723 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:13.724 DEBUG:boto:Data: 2022-09-03T07:24:13.724 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.725 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.725 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.726 DEBUG:boto:Params: {} 2022-09-03T07:24:13.727 DEBUG:boto:Token: None 2022-09-03T07:24:13.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.731 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003873fcffabfedf34-006313011d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.774 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.776 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.777 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.778 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:13.779 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:13.780 DEBUG:boto:Method: GET 2022-09-03T07:24:13.781 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:13.782 DEBUG:boto:Data: 2022-09-03T07:24:13.783 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.784 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.785 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.786 DEBUG:boto:Params: {} 2022-09-03T07:24:13.787 DEBUG:boto:Token: None 2022-09-03T07:24:13.788 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.794 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000432a41e309ef801b-006313011d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.795 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.796 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.797 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.798 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.798 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:13.799 DEBUG:boto:Data: 2022-09-03T07:24:13.799 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.800 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.800 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.801 DEBUG:boto:Params: {} 2022-09-03T07:24:13.801 DEBUG:boto:Token: None 2022-09-03T07:24:13.802 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17 2022-09-03T07:24:13.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RqRLAwgzHkDBx5Og3Xe+KpwxHTk= 2022-09-03T07:24:13.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RqRLAwgzHkDBx5Og3Xe+KpwxHTk=', 'Content-Length': '0'} 2022-09-03T07:24:13.805 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000519de89b65af720-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.806 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.807 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.807 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.808 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:13.808 DEBUG:boto:Data: 2022-09-03T07:24:13.809 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.810 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.810 DEBUG:boto:Params: {} 2022-09-03T07:24:13.811 DEBUG:boto:Token: None 2022-09-03T07:24:13.812 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17 2022-09-03T07:24:13.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RqRLAwgzHkDBx5Og3Xe+KpwxHTk= 2022-09-03T07:24:13.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RqRLAwgzHkDBx5Og3Xe+KpwxHTk=', 'Content-Length': '0'} 2022-09-03T07:24:13.815 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006514253486de4e99-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.815 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:24:13.816 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.816 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.817 DEBUG:boto:Method: GET 2022-09-03T07:24:13.818 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:13.818 DEBUG:boto:Data: 2022-09-03T07:24:13.819 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.820 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.820 DEBUG:boto:Params: {} 2022-09-03T07:24:13.821 DEBUG:boto:Token: None 2022-09-03T07:24:13.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17 2022-09-03T07:24:13.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DdojWpt7s0qYGSCdPt5VUb47+bY= 2022-09-03T07:24:13.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DdojWpt7s0qYGSCdPt5VUb47+bY=', 'Content-Length': '0'} 2022-09-03T07:24:13.825 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c00205c7898e179b-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.826 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:13.826 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:13.827 DEBUG:boto:Method: GET 2022-09-03T07:24:13.828 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:13.828 DEBUG:boto:Data: 2022-09-03T07:24:13.829 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.829 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.830 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.831 DEBUG:boto:Params: {} 2022-09-03T07:24:13.831 DEBUG:boto:Token: None 2022-09-03T07:24:13.834 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj17 2022-09-03T07:24:13.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DdojWpt7s0qYGSCdPt5VUb47+bY= 2022-09-03T07:24:13.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DdojWpt7s0qYGSCdPt5VUb47+bY=', 'Content-Length': '0'} 2022-09-03T07:24:13.838 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064ba5586f7a27d96-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.882 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.884 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.884 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:13.885 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:13.886 DEBUG:boto:Method: GET 2022-09-03T07:24:13.887 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:13.887 DEBUG:boto:Data: 2022-09-03T07:24:13.888 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.889 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.890 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.891 DEBUG:boto:Params: {} 2022-09-03T07:24:13.892 DEBUG:boto:Token: None 2022-09-03T07:24:13.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.899 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000064af51cfd18c83ff-006313011d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.901 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.902 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.903 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.903 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:13.904 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:13.904 DEBUG:boto:Method: GET 2022-09-03T07:24:13.905 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:13.905 DEBUG:boto:Data: 2022-09-03T07:24:13.906 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.908 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.908 DEBUG:boto:Params: {} 2022-09-03T07:24:13.910 DEBUG:boto:Token: None 2022-09-03T07:24:13.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/ 2022-09-03T07:24:13.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w= 2022-09-03T07:24:13.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZqC/VFqdgs2l61iAfM+X0yqEK2w=', 'Content-Length': '0'} 2022-09-03T07:24:13.915 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008dd72a641ae5c9f0-006313011d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.916 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:13.916 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:24:13.917 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:13.918 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:13.918 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.919 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.919 DEBUG:boto:Method: GET 2022-09-03T07:24:13.920 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.920 DEBUG:boto:Data: 2022-09-03T07:24:13.921 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.922 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.923 DEBUG:boto:Params: {} 2022-09-03T07:24:13.924 DEBUG:boto:Token: None 2022-09-03T07:24:13.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dn6GXAmcleLWfiwdK7cTuQ6NjO0= 2022-09-03T07:24:13.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dn6GXAmcleLWfiwdK7cTuQ6NjO0=', 'Content-Length': '0'} 2022-09-03T07:24:13.928 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d3b9389cfbaa04d-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.929 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:13.929 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:13.930 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.931 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.932 DEBUG:boto:Method: GET 2022-09-03T07:24:13.933 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.934 DEBUG:boto:Data: 2022-09-03T07:24:13.935 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.936 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.937 DEBUG:boto:Params: {} 2022-09-03T07:24:13.939 DEBUG:boto:Token: None 2022-09-03T07:24:13.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dn6GXAmcleLWfiwdK7cTuQ6NjO0= 2022-09-03T07:24:13.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dn6GXAmcleLWfiwdK7cTuQ6NjO0=', 'Content-Length': '0'} 2022-09-03T07:24:13.944 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057eb3f1c4b4d3484-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.946 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:13.947 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:13.948 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.949 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.950 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.950 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.951 DEBUG:boto:Data: 2022-09-03T07:24:13.952 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.953 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.955 DEBUG:boto:Params: {} 2022-09-03T07:24:13.956 DEBUG:boto:Token: None 2022-09-03T07:24:13.956 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mey1QsycAfvnt88/LOgoER8/O/0= 2022-09-03T07:24:13.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mey1QsycAfvnt88/LOgoER8/O/0=', 'Content-Length': '0'} 2022-09-03T07:24:13.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec3b0e2f0157eb08-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.962 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:13.963 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:13.963 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.964 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.964 DEBUG:boto:Method: HEAD 2022-09-03T07:24:13.965 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.965 DEBUG:boto:Data: 2022-09-03T07:24:13.966 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.967 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.968 DEBUG:boto:Params: {} 2022-09-03T07:24:13.969 DEBUG:boto:Token: None 2022-09-03T07:24:13.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mey1QsycAfvnt88/LOgoER8/O/0= 2022-09-03T07:24:13.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mey1QsycAfvnt88/LOgoER8/O/0=', 'Content-Length': '0'} 2022-09-03T07:24:13.973 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a32fa13d0b498735-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.974 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:24:13.974 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:13.975 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:13.976 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.977 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.977 DEBUG:boto:Method: GET 2022-09-03T07:24:13.978 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.978 DEBUG:boto:Data: 2022-09-03T07:24:13.979 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:13.980 DEBUG:boto:Port: 8001 2022-09-03T07:24:13.980 DEBUG:boto:Params: {} 2022-09-03T07:24:13.981 DEBUG:boto:Token: None 2022-09-03T07:24:13.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dn6GXAmcleLWfiwdK7cTuQ6NjO0= 2022-09-03T07:24:13.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dn6GXAmcleLWfiwdK7cTuQ6NjO0=', 'Content-Length': '0'} 2022-09-03T07:24:13.985 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083d2887556b1a393-006313011d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.986 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:13.987 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:13.988 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.988 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.989 DEBUG:boto:Method: GET 2022-09-03T07:24:13.989 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.990 DEBUG:boto:Data: 2022-09-03T07:24:13.990 DEBUG:boto:Headers: {} 2022-09-03T07:24:13.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:13.992 DEBUG:boto:Port: 8000 2022-09-03T07:24:13.992 DEBUG:boto:Params: {} 2022-09-03T07:24:13.993 DEBUG:boto:Token: None 2022-09-03T07:24:13.994 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:13 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:13.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dn6GXAmcleLWfiwdK7cTuQ6NjO0= 2022-09-03T07:24:13.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dn6GXAmcleLWfiwdK7cTuQ6NjO0=', 'Content-Length': '0'} 2022-09-03T07:24:13.997 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f72154657de91f7-006313011d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:13.998 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:13.998 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:13.999 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:14.000 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:14.000 DEBUG:boto:Method: GET 2022-09-03T07:24:14.001 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:14.001 DEBUG:boto:Data: 2022-09-03T07:24:14.002 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.003 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.004 DEBUG:boto:Params: {} 2022-09-03T07:24:14.005 DEBUG:boto:Token: None 2022-09-03T07:24:14.006 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000bc446c9d6f6221c-006313011e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.010 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.011 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.012 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.012 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:14.013 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:14.014 DEBUG:boto:Method: GET 2022-09-03T07:24:14.014 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:14.015 DEBUG:boto:Data: 2022-09-03T07:24:14.015 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.016 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.017 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.017 DEBUG:boto:Params: {} 2022-09-03T07:24:14.018 DEBUG:boto:Token: None 2022-09-03T07:24:14.018 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.019 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.023 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e93d1032aaf44779-006313011e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.024 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.025 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:14.025 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:14.026 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.026 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:14.027 DEBUG:boto:Data: 2022-09-03T07:24:14.027 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.028 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.028 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.029 DEBUG:boto:Params: {} 2022-09-03T07:24:14.030 DEBUG:boto:Token: None 2022-09-03T07:24:14.030 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj19 2022-09-03T07:24:14.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uKZsj4OaWh1JQkc0uE0k+akXqp4= 2022-09-03T07:24:14.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uKZsj4OaWh1JQkc0uE0k+akXqp4=', 'Content-Length': '0'} 2022-09-03T07:24:14.034 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fab34c2f37d42785-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.035 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:14.035 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:14.036 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.036 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:14.037 DEBUG:boto:Data: 2022-09-03T07:24:14.037 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.039 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.039 DEBUG:boto:Params: {} 2022-09-03T07:24:14.040 DEBUG:boto:Token: None 2022-09-03T07:24:14.040 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj19 2022-09-03T07:24:14.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uKZsj4OaWh1JQkc0uE0k+akXqp4= 2022-09-03T07:24:14.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uKZsj4OaWh1JQkc0uE0k+akXqp4=', 'Content-Length': '0'} 2022-09-03T07:24:14.043 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e93c9f177105bea-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.044 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:24:14.045 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:14.045 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:14.046 DEBUG:boto:Method: GET 2022-09-03T07:24:14.047 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:14.047 DEBUG:boto:Data: 2022-09-03T07:24:14.048 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.048 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.049 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.049 DEBUG:boto:Params: {} 2022-09-03T07:24:14.050 DEBUG:boto:Token: None 2022-09-03T07:24:14.051 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj19 2022-09-03T07:24:14.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:65Jj+0mJJWq6fjeANt+8sjp1E8A= 2022-09-03T07:24:14.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:65Jj+0mJJWq6fjeANt+8sjp1E8A=', 'Content-Length': '0'} 2022-09-03T07:24:14.054 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000030d04033ad0d9ec0-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.055 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:14.055 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:14.056 DEBUG:boto:Method: GET 2022-09-03T07:24:14.057 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:14.057 DEBUG:boto:Data: 2022-09-03T07:24:14.058 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.058 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.059 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.059 DEBUG:boto:Params: {} 2022-09-03T07:24:14.060 DEBUG:boto:Token: None 2022-09-03T07:24:14.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj19 2022-09-03T07:24:14.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:65Jj+0mJJWq6fjeANt+8sjp1E8A= 2022-09-03T07:24:14.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:65Jj+0mJJWq6fjeANt+8sjp1E8A=', 'Content-Length': '0'} 2022-09-03T07:24:14.064 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000acff5de22ede87f7-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.106 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.107 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.108 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:14.109 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:14.110 DEBUG:boto:Method: GET 2022-09-03T07:24:14.111 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:14.112 DEBUG:boto:Data: 2022-09-03T07:24:14.113 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.115 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.116 DEBUG:boto:Params: {} 2022-09-03T07:24:14.117 DEBUG:boto:Token: None 2022-09-03T07:24:14.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.124 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063024ce6cc05dbed-006313011e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.125 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.127 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.128 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.129 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:14.130 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:14.131 DEBUG:boto:Method: GET 2022-09-03T07:24:14.132 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:14.133 DEBUG:boto:Data: 2022-09-03T07:24:14.133 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.135 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.136 DEBUG:boto:Params: {} 2022-09-03T07:24:14.138 DEBUG:boto:Token: None 2022-09-03T07:24:14.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.141 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.145 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c97375e650b8259b-006313011e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.146 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.147 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:24:14.148 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.148 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.149 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.149 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.150 DEBUG:boto:Method: GET 2022-09-03T07:24:14.151 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.151 DEBUG:boto:Data: 2022-09-03T07:24:14.152 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.152 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.153 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.153 DEBUG:boto:Params: {} 2022-09-03T07:24:14.154 DEBUG:boto:Token: None 2022-09-03T07:24:14.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oHyWUtwYutLLN1t+/rgMUVKls+w= 2022-09-03T07:24:14.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oHyWUtwYutLLN1t+/rgMUVKls+w=', 'Content-Length': '0'} 2022-09-03T07:24:14.158 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2ed86d324a9a64d-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.160 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.161 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.162 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.163 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.164 DEBUG:boto:Method: GET 2022-09-03T07:24:14.165 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.166 DEBUG:boto:Data: 2022-09-03T07:24:14.167 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.169 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.170 DEBUG:boto:Params: {} 2022-09-03T07:24:14.171 DEBUG:boto:Token: None 2022-09-03T07:24:14.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oHyWUtwYutLLN1t+/rgMUVKls+w= 2022-09-03T07:24:14.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oHyWUtwYutLLN1t+/rgMUVKls+w=', 'Content-Length': '0'} 2022-09-03T07:24:14.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d88430183aeebf0-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.180 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.181 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.181 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.182 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.183 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.184 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.185 DEBUG:boto:Data: 2022-09-03T07:24:14.186 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.187 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.189 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.190 DEBUG:boto:Params: {} 2022-09-03T07:24:14.191 DEBUG:boto:Token: None 2022-09-03T07:24:14.192 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BbDLp98dgFPT9D9d/wrN3YwZOaI= 2022-09-03T07:24:14.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BbDLp98dgFPT9D9d/wrN3YwZOaI=', 'Content-Length': '0'} 2022-09-03T07:24:14.203 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a71590fdd5705a59-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.204 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.205 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.205 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.206 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.206 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.207 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.207 DEBUG:boto:Data: 2022-09-03T07:24:14.208 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.209 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.210 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.211 DEBUG:boto:Params: {} 2022-09-03T07:24:14.211 DEBUG:boto:Token: None 2022-09-03T07:24:14.212 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BbDLp98dgFPT9D9d/wrN3YwZOaI= 2022-09-03T07:24:14.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BbDLp98dgFPT9D9d/wrN3YwZOaI=', 'Content-Length': '0'} 2022-09-03T07:24:14.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065a4181ad9c637a8-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:24:14.217 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.217 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.218 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.219 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.219 DEBUG:boto:Method: GET 2022-09-03T07:24:14.220 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.220 DEBUG:boto:Data: 2022-09-03T07:24:14.221 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.222 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.223 DEBUG:boto:Params: {} 2022-09-03T07:24:14.223 DEBUG:boto:Token: None 2022-09-03T07:24:14.224 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oHyWUtwYutLLN1t+/rgMUVKls+w= 2022-09-03T07:24:14.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oHyWUtwYutLLN1t+/rgMUVKls+w=', 'Content-Length': '0'} 2022-09-03T07:24:14.228 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed64ac37a33df42e-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.275 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.276 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.277 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.278 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.279 DEBUG:boto:Method: GET 2022-09-03T07:24:14.279 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.280 DEBUG:boto:Data: 2022-09-03T07:24:14.281 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.283 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.284 DEBUG:boto:Params: {} 2022-09-03T07:24:14.284 DEBUG:boto:Token: None 2022-09-03T07:24:14.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:14.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oHyWUtwYutLLN1t+/rgMUVKls+w= 2022-09-03T07:24:14.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oHyWUtwYutLLN1t+/rgMUVKls+w=', 'Content-Length': '0'} 2022-09-03T07:24:14.289 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1a59239f3b36ab5-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.290 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.291 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.292 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:14.292 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:14.293 DEBUG:boto:Method: GET 2022-09-03T07:24:14.294 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:14.295 DEBUG:boto:Data: 2022-09-03T07:24:14.295 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.296 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.296 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.297 DEBUG:boto:Params: {} 2022-09-03T07:24:14.298 DEBUG:boto:Token: None 2022-09-03T07:24:14.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.302 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004b18c0fedfeb9a56-006313011e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.303 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.305 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.305 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.306 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:14.306 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:14.307 DEBUG:boto:Method: GET 2022-09-03T07:24:14.307 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:14.308 DEBUG:boto:Data: 2022-09-03T07:24:14.308 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.309 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.310 DEBUG:boto:Params: {} 2022-09-03T07:24:14.311 DEBUG:boto:Token: None 2022-09-03T07:24:14.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.316 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c366d7ee1c3c8db7-006313011e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.317 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.318 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.318 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.319 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.319 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:14.320 DEBUG:boto:Data: 2022-09-03T07:24:14.321 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.322 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.323 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.324 DEBUG:boto:Params: {} 2022-09-03T07:24:14.325 DEBUG:boto:Token: None 2022-09-03T07:24:14.325 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22 2022-09-03T07:24:14.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W0ZgenAizdtVbHUQvTIpZtUGhe0= 2022-09-03T07:24:14.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W0ZgenAizdtVbHUQvTIpZtUGhe0=', 'Content-Length': '0'} 2022-09-03T07:24:14.330 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f3104309950aacf-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.331 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.331 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.332 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.332 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:14.333 DEBUG:boto:Data: 2022-09-03T07:24:14.333 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.335 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.335 DEBUG:boto:Params: {} 2022-09-03T07:24:14.336 DEBUG:boto:Token: None 2022-09-03T07:24:14.336 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22 2022-09-03T07:24:14.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W0ZgenAizdtVbHUQvTIpZtUGhe0= 2022-09-03T07:24:14.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W0ZgenAizdtVbHUQvTIpZtUGhe0=', 'Content-Length': '0'} 2022-09-03T07:24:14.339 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c190ffeea6e45d7a-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.340 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:24:14.341 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.341 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.342 DEBUG:boto:Method: GET 2022-09-03T07:24:14.343 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:14.343 DEBUG:boto:Data: 2022-09-03T07:24:14.344 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.345 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.346 DEBUG:boto:Params: {} 2022-09-03T07:24:14.347 DEBUG:boto:Token: None 2022-09-03T07:24:14.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22 2022-09-03T07:24:14.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgLeGxZjjY8s+Evo2AnUCbMV4V8= 2022-09-03T07:24:14.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgLeGxZjjY8s+Evo2AnUCbMV4V8=', 'Content-Length': '0'} 2022-09-03T07:24:14.351 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000263f2d46e26ed467-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.352 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:14.353 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:14.354 DEBUG:boto:Method: GET 2022-09-03T07:24:14.355 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:14.356 DEBUG:boto:Data: 2022-09-03T07:24:14.357 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.359 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.359 DEBUG:boto:Params: {} 2022-09-03T07:24:14.360 DEBUG:boto:Token: None 2022-09-03T07:24:14.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj22 2022-09-03T07:24:14.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xgLeGxZjjY8s+Evo2AnUCbMV4V8= 2022-09-03T07:24:14.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xgLeGxZjjY8s+Evo2AnUCbMV4V8=', 'Content-Length': '0'} 2022-09-03T07:24:14.364 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b87ae1dc351ba80-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.365 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.366 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.367 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:14.368 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:14.369 DEBUG:boto:Method: GET 2022-09-03T07:24:14.370 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:14.371 DEBUG:boto:Data: 2022-09-03T07:24:14.372 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.374 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.375 DEBUG:boto:Params: {} 2022-09-03T07:24:14.376 DEBUG:boto:Token: None 2022-09-03T07:24:14.377 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.379 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.382 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff0d630a1390d84a-006313011e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.384 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.385 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.386 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.387 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:14.387 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:14.388 DEBUG:boto:Method: GET 2022-09-03T07:24:14.388 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:14.389 DEBUG:boto:Data: 2022-09-03T07:24:14.389 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.391 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.391 DEBUG:boto:Params: {} 2022-09-03T07:24:14.392 DEBUG:boto:Token: None 2022-09-03T07:24:14.393 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.397 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000082a57dc67334fec8-006313011e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.398 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.399 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:24:14.400 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.401 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.402 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.403 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.404 DEBUG:boto:Method: GET 2022-09-03T07:24:14.405 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.406 DEBUG:boto:Data: 2022-09-03T07:24:14.407 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.408 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.409 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.410 DEBUG:boto:Params: {} 2022-09-03T07:24:14.410 DEBUG:boto:Token: None 2022-09-03T07:24:14.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EgEohVimcbSM0BRgvZxzzPSR3WM= 2022-09-03T07:24:14.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EgEohVimcbSM0BRgvZxzzPSR3WM=', 'Content-Length': '0'} 2022-09-03T07:24:14.414 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ea9d34d943801483-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.458 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.462 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.463 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.464 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.465 DEBUG:boto:Method: GET 2022-09-03T07:24:14.466 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.468 DEBUG:boto:Data: 2022-09-03T07:24:14.469 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.470 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.471 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.472 DEBUG:boto:Params: {} 2022-09-03T07:24:14.473 DEBUG:boto:Token: None 2022-09-03T07:24:14.474 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.475 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EgEohVimcbSM0BRgvZxzzPSR3WM= 2022-09-03T07:24:14.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EgEohVimcbSM0BRgvZxzzPSR3WM=', 'Content-Length': '0'} 2022-09-03T07:24:14.480 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c46052038dd2d96e-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.481 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.482 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.483 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.484 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.485 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.486 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.487 DEBUG:boto:Data: 2022-09-03T07:24:14.488 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.492 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.493 DEBUG:boto:Params: {} 2022-09-03T07:24:14.494 DEBUG:boto:Token: None 2022-09-03T07:24:14.495 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EbmCMKLh4NxHunIyZ/C2lY8MmAk= 2022-09-03T07:24:14.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EbmCMKLh4NxHunIyZ/C2lY8MmAk=', 'Content-Length': '0'} 2022-09-03T07:24:14.500 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff94e3cb119ed39f-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.501 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.501 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.502 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.502 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.503 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.504 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.504 DEBUG:boto:Data: 2022-09-03T07:24:14.505 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.506 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.506 DEBUG:boto:Params: {} 2022-09-03T07:24:14.507 DEBUG:boto:Token: None 2022-09-03T07:24:14.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EbmCMKLh4NxHunIyZ/C2lY8MmAk= 2022-09-03T07:24:14.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EbmCMKLh4NxHunIyZ/C2lY8MmAk=', 'Content-Length': '0'} 2022-09-03T07:24:14.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000231b95dfc3ab5869-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.511 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:24:14.512 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.512 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.513 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.513 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.514 DEBUG:boto:Method: GET 2022-09-03T07:24:14.515 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.515 DEBUG:boto:Data: 2022-09-03T07:24:14.516 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.516 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.517 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.517 DEBUG:boto:Params: {} 2022-09-03T07:24:14.518 DEBUG:boto:Token: None 2022-09-03T07:24:14.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EgEohVimcbSM0BRgvZxzzPSR3WM= 2022-09-03T07:24:14.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EgEohVimcbSM0BRgvZxzzPSR3WM=', 'Content-Length': '0'} 2022-09-03T07:24:14.522 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b63b1ad1ebd1fed0-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.523 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.524 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.525 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.525 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.526 DEBUG:boto:Method: GET 2022-09-03T07:24:14.526 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.527 DEBUG:boto:Data: 2022-09-03T07:24:14.527 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.528 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.528 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.529 DEBUG:boto:Params: {} 2022-09-03T07:24:14.529 DEBUG:boto:Token: None 2022-09-03T07:24:14.530 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:14.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EgEohVimcbSM0BRgvZxzzPSR3WM= 2022-09-03T07:24:14.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EgEohVimcbSM0BRgvZxzzPSR3WM=', 'Content-Length': '0'} 2022-09-03T07:24:14.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008854cdd84ddb5fd1-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.574 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.575 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.576 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:14.577 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:14.578 DEBUG:boto:Method: GET 2022-09-03T07:24:14.579 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:14.579 DEBUG:boto:Data: 2022-09-03T07:24:14.580 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.581 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.582 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.583 DEBUG:boto:Params: {} 2022-09-03T07:24:14.584 DEBUG:boto:Token: None 2022-09-03T07:24:14.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.590 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d59be364848fa23-006313011e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.591 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.593 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.594 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.595 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:14.595 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:14.597 DEBUG:boto:Method: GET 2022-09-03T07:24:14.597 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:14.598 DEBUG:boto:Data: 2022-09-03T07:24:14.599 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.601 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.602 DEBUG:boto:Params: {} 2022-09-03T07:24:14.604 DEBUG:boto:Token: None 2022-09-03T07:24:14.605 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.607 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.611 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005bd1099482552733-006313011e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.612 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.614 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.615 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.616 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.616 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:14.617 DEBUG:boto:Data: 2022-09-03T07:24:14.617 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.618 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.618 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.619 DEBUG:boto:Params: {} 2022-09-03T07:24:14.620 DEBUG:boto:Token: None 2022-09-03T07:24:14.620 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23 2022-09-03T07:24:14.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRHj9iUdW+uWyQc3RGwmqFi2cPg= 2022-09-03T07:24:14.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRHj9iUdW+uWyQc3RGwmqFi2cPg=', 'Content-Length': '0'} 2022-09-03T07:24:14.623 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000006713cf14eeb38eb-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.624 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.625 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.625 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.626 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:14.626 DEBUG:boto:Data: 2022-09-03T07:24:14.627 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.628 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.628 DEBUG:boto:Params: {} 2022-09-03T07:24:14.629 DEBUG:boto:Token: None 2022-09-03T07:24:14.629 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23 2022-09-03T07:24:14.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRHj9iUdW+uWyQc3RGwmqFi2cPg= 2022-09-03T07:24:14.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRHj9iUdW+uWyQc3RGwmqFi2cPg=', 'Content-Length': '0'} 2022-09-03T07:24:14.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ebed9196c431ea5-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.633 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:24:14.634 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.635 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.635 DEBUG:boto:Method: GET 2022-09-03T07:24:14.636 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:14.636 DEBUG:boto:Data: 2022-09-03T07:24:14.637 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.637 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.638 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.638 DEBUG:boto:Params: {} 2022-09-03T07:24:14.639 DEBUG:boto:Token: None 2022-09-03T07:24:14.640 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23 2022-09-03T07:24:14.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBM0g+1WmM1sPo8YIFUOidfsblc= 2022-09-03T07:24:14.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBM0g+1WmM1sPo8YIFUOidfsblc=', 'Content-Length': '0'} 2022-09-03T07:24:14.643 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d849ceaa4a1a6b21-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.643 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:14.644 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:14.645 DEBUG:boto:Method: GET 2022-09-03T07:24:14.645 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:14.646 DEBUG:boto:Data: 2022-09-03T07:24:14.646 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.647 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.648 DEBUG:boto:Params: {} 2022-09-03T07:24:14.648 DEBUG:boto:Token: None 2022-09-03T07:24:14.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj23 2022-09-03T07:24:14.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBM0g+1WmM1sPo8YIFUOidfsblc= 2022-09-03T07:24:14.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBM0g+1WmM1sPo8YIFUOidfsblc=', 'Content-Length': '0'} 2022-09-03T07:24:14.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b47c3f7a4068cf3b-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.653 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.653 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.654 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:14.654 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:14.655 DEBUG:boto:Method: GET 2022-09-03T07:24:14.655 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:14.656 DEBUG:boto:Data: 2022-09-03T07:24:14.657 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.658 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.658 DEBUG:boto:Params: {} 2022-09-03T07:24:14.659 DEBUG:boto:Token: None 2022-09-03T07:24:14.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.663 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000095ae5a307c8f4898-006313011e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.663 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.664 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.665 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.665 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:14.666 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:14.666 DEBUG:boto:Method: GET 2022-09-03T07:24:14.667 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:14.667 DEBUG:boto:Data: 2022-09-03T07:24:14.668 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.668 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.669 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.669 DEBUG:boto:Params: {} 2022-09-03T07:24:14.670 DEBUG:boto:Token: None 2022-09-03T07:24:14.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.672 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.675 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007cd3226b9fa6ff80-006313011e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.676 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.677 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:24:14.677 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.678 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.678 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.679 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.680 DEBUG:boto:Method: GET 2022-09-03T07:24:14.681 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.682 DEBUG:boto:Data: 2022-09-03T07:24:14.683 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.684 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.685 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.686 DEBUG:boto:Params: {} 2022-09-03T07:24:14.687 DEBUG:boto:Token: None 2022-09-03T07:24:14.688 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3wxfIZfRHRYsEV1U1FQmHFnEl8= 2022-09-03T07:24:14.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3wxfIZfRHRYsEV1U1FQmHFnEl8=', 'Content-Length': '0'} 2022-09-03T07:24:14.693 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7d8b1c1c79d15f8-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.734 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.736 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.736 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.737 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.738 DEBUG:boto:Method: GET 2022-09-03T07:24:14.739 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.740 DEBUG:boto:Data: 2022-09-03T07:24:14.740 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.741 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.742 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.742 DEBUG:boto:Params: {} 2022-09-03T07:24:14.743 DEBUG:boto:Token: None 2022-09-03T07:24:14.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3wxfIZfRHRYsEV1U1FQmHFnEl8= 2022-09-03T07:24:14.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3wxfIZfRHRYsEV1U1FQmHFnEl8=', 'Content-Length': '0'} 2022-09-03T07:24:14.749 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbcd1b1a3e2c3994-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.751 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.752 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.752 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.753 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.753 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.754 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.755 DEBUG:boto:Data: 2022-09-03T07:24:14.755 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.757 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.757 DEBUG:boto:Params: {} 2022-09-03T07:24:14.758 DEBUG:boto:Token: None 2022-09-03T07:24:14.759 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ms7m7+P/THp2b8cpmdThZwb2f/w= 2022-09-03T07:24:14.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ms7m7+P/THp2b8cpmdThZwb2f/w=', 'Content-Length': '0'} 2022-09-03T07:24:14.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ead34b3480935dae-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.766 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.767 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.768 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.769 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.770 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.771 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.772 DEBUG:boto:Data: 2022-09-03T07:24:14.773 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.775 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.776 DEBUG:boto:Params: {} 2022-09-03T07:24:14.777 DEBUG:boto:Token: None 2022-09-03T07:24:14.778 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.779 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ms7m7+P/THp2b8cpmdThZwb2f/w= 2022-09-03T07:24:14.780 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ms7m7+P/THp2b8cpmdThZwb2f/w=', 'Content-Length': '0'} 2022-09-03T07:24:14.783 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e4e6dfbf4f3f2723-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.784 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:24:14.785 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.786 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.787 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.789 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.790 DEBUG:boto:Method: GET 2022-09-03T07:24:14.791 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.792 DEBUG:boto:Data: 2022-09-03T07:24:14.794 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.796 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.797 DEBUG:boto:Params: {} 2022-09-03T07:24:14.799 DEBUG:boto:Token: None 2022-09-03T07:24:14.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3wxfIZfRHRYsEV1U1FQmHFnEl8= 2022-09-03T07:24:14.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3wxfIZfRHRYsEV1U1FQmHFnEl8=', 'Content-Length': '0'} 2022-09-03T07:24:14.805 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000021640ee85a8dac9b-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.806 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.807 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.807 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.808 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.809 DEBUG:boto:Method: GET 2022-09-03T07:24:14.810 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.811 DEBUG:boto:Data: 2022-09-03T07:24:14.811 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.814 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.815 DEBUG:boto:Params: {} 2022-09-03T07:24:14.817 DEBUG:boto:Token: None 2022-09-03T07:24:14.818 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:14.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3wxfIZfRHRYsEV1U1FQmHFnEl8= 2022-09-03T07:24:14.821 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3wxfIZfRHRYsEV1U1FQmHFnEl8=', 'Content-Length': '0'} 2022-09-03T07:24:14.824 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004b44f61a0be1d28-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.825 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.827 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.827 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:14.828 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:14.829 DEBUG:boto:Method: GET 2022-09-03T07:24:14.829 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:14.830 DEBUG:boto:Data: 2022-09-03T07:24:14.831 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.832 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.833 DEBUG:boto:Params: {} 2022-09-03T07:24:14.834 DEBUG:boto:Token: None 2022-09-03T07:24:14.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.840 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067a314dfe8d45b4c-006313011e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.841 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.842 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.843 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.844 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:14.844 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:14.845 DEBUG:boto:Method: GET 2022-09-03T07:24:14.846 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:14.846 DEBUG:boto:Data: 2022-09-03T07:24:14.847 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.849 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.849 DEBUG:boto:Params: {} 2022-09-03T07:24:14.850 DEBUG:boto:Token: None 2022-09-03T07:24:14.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.856 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000043ac50073cc40308-006313011e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.857 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.858 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.858 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.859 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.859 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:14.860 DEBUG:boto:Data: 2022-09-03T07:24:14.860 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.861 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.861 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.862 DEBUG:boto:Params: {} 2022-09-03T07:24:14.863 DEBUG:boto:Token: None 2022-09-03T07:24:14.863 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33 2022-09-03T07:24:14.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A/Vlo18CBBuPIPgorKLTF4rZygI= 2022-09-03T07:24:14.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A/Vlo18CBBuPIPgorKLTF4rZygI=', 'Content-Length': '0'} 2022-09-03T07:24:14.867 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000882e109d6fd3bd41-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.868 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.868 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.869 DEBUG:boto:Method: HEAD 2022-09-03T07:24:14.869 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:14.870 DEBUG:boto:Data: 2022-09-03T07:24:14.871 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.871 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.872 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.872 DEBUG:boto:Params: {} 2022-09-03T07:24:14.873 DEBUG:boto:Token: None 2022-09-03T07:24:14.874 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33 2022-09-03T07:24:14.875 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:A/Vlo18CBBuPIPgorKLTF4rZygI= 2022-09-03T07:24:14.876 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:A/Vlo18CBBuPIPgorKLTF4rZygI=', 'Content-Length': '0'} 2022-09-03T07:24:14.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008640af4d6869aa27-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.878 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:24:14.879 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.880 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.880 DEBUG:boto:Method: GET 2022-09-03T07:24:14.881 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:14.881 DEBUG:boto:Data: 2022-09-03T07:24:14.882 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.883 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.883 DEBUG:boto:Params: {} 2022-09-03T07:24:14.884 DEBUG:boto:Token: None 2022-09-03T07:24:14.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33 2022-09-03T07:24:14.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DdxY1k2CphKaSV+h689GyxLyinI= 2022-09-03T07:24:14.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DdxY1k2CphKaSV+h689GyxLyinI=', 'Content-Length': '0'} 2022-09-03T07:24:14.888 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc19e483bf6eba5a-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.889 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:14.889 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:14.890 DEBUG:boto:Method: GET 2022-09-03T07:24:14.890 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:14.891 DEBUG:boto:Data: 2022-09-03T07:24:14.892 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.893 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.893 DEBUG:boto:Params: {} 2022-09-03T07:24:14.894 DEBUG:boto:Token: None 2022-09-03T07:24:14.894 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj33 2022-09-03T07:24:14.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DdxY1k2CphKaSV+h689GyxLyinI= 2022-09-03T07:24:14.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DdxY1k2CphKaSV+h689GyxLyinI=', 'Content-Length': '0'} 2022-09-03T07:24:14.897 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005da55df630e3b368-006313011e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.942 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.943 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.944 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:14.944 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:14.945 DEBUG:boto:Method: GET 2022-09-03T07:24:14.945 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:14.946 DEBUG:boto:Data: 2022-09-03T07:24:14.947 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.948 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.949 DEBUG:boto:Params: {} 2022-09-03T07:24:14.949 DEBUG:boto:Token: None 2022-09-03T07:24:14.950 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.954 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007413f7cc459505ec-006313011e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.954 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.955 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:14.956 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:14.957 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:14.957 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:14.958 DEBUG:boto:Method: GET 2022-09-03T07:24:14.958 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:14.959 DEBUG:boto:Data: 2022-09-03T07:24:14.959 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:14.961 DEBUG:boto:Port: 8000 2022-09-03T07:24:14.962 DEBUG:boto:Params: {} 2022-09-03T07:24:14.963 DEBUG:boto:Token: None 2022-09-03T07:24:14.964 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/ 2022-09-03T07:24:14.964 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0= 2022-09-03T07:24:14.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ashRkma1vOTFb9BUG/v1yrFsnM0=', 'Content-Length': '0'} 2022-09-03T07:24:14.969 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fde1d445f2eb04c9-006313011e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:14.969 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:14.970 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:24:14.971 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:14.972 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:14.972 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:14.973 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:14.974 DEBUG:boto:Method: GET 2022-09-03T07:24:14.974 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:14.975 DEBUG:boto:Data: 2022-09-03T07:24:14.975 DEBUG:boto:Headers: {} 2022-09-03T07:24:14.976 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:14.976 DEBUG:boto:Port: 8001 2022-09-03T07:24:14.977 DEBUG:boto:Params: {} 2022-09-03T07:24:14.978 DEBUG:boto:Token: None 2022-09-03T07:24:14.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:14 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:14.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zoysoy1uLHs9zSDg5wRyhDCEHRg= 2022-09-03T07:24:14.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zoysoy1uLHs9zSDg5wRyhDCEHRg=', 'Content-Length': '0'} 2022-09-03T07:24:14.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000959d8dea5f2c6a4b-006313011e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.030 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:15.031 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:15.032 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.033 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.033 DEBUG:boto:Method: GET 2022-09-03T07:24:15.034 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.035 DEBUG:boto:Data: 2022-09-03T07:24:15.035 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.037 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.037 DEBUG:boto:Params: {} 2022-09-03T07:24:15.038 DEBUG:boto:Token: None 2022-09-03T07:24:15.039 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:71zWaNoJR6C09wpt7r/ZwelaTVc= 2022-09-03T07:24:15.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:71zWaNoJR6C09wpt7r/ZwelaTVc=', 'Content-Length': '0'} 2022-09-03T07:24:15.042 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8261ea5fcb73c2f-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.043 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:15.044 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:15.044 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.045 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.046 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.046 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.047 DEBUG:boto:Data: 2022-09-03T07:24:15.047 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.048 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.048 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.049 DEBUG:boto:Params: {} 2022-09-03T07:24:15.049 DEBUG:boto:Token: None 2022-09-03T07:24:15.050 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bs2FsSGrp8IDtFEHlmB+Wnha23E= 2022-09-03T07:24:15.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bs2FsSGrp8IDtFEHlmB+Wnha23E=', 'Content-Length': '0'} 2022-09-03T07:24:15.054 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bbaac951a465f564-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.055 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:15.055 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:15.056 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.056 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.057 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.057 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.058 DEBUG:boto:Data: 2022-09-03T07:24:15.058 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.059 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.060 DEBUG:boto:Params: {} 2022-09-03T07:24:15.060 DEBUG:boto:Token: None 2022-09-03T07:24:15.061 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bs2FsSGrp8IDtFEHlmB+Wnha23E= 2022-09-03T07:24:15.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bs2FsSGrp8IDtFEHlmB+Wnha23E=', 'Content-Length': '0'} 2022-09-03T07:24:15.064 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d85ee8811c6ca6b7-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.065 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:24:15.065 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:15.066 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:15.067 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.067 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.068 DEBUG:boto:Method: GET 2022-09-03T07:24:15.068 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.069 DEBUG:boto:Data: 2022-09-03T07:24:15.069 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.070 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.070 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.071 DEBUG:boto:Params: {} 2022-09-03T07:24:15.071 DEBUG:boto:Token: None 2022-09-03T07:24:15.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:71zWaNoJR6C09wpt7r/ZwelaTVc= 2022-09-03T07:24:15.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:71zWaNoJR6C09wpt7r/ZwelaTVc=', 'Content-Length': '0'} 2022-09-03T07:24:15.075 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008fe95a1b82927d9b-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.076 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:15.077 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:15.077 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.078 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.078 DEBUG:boto:Method: GET 2022-09-03T07:24:15.079 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.079 DEBUG:boto:Data: 2022-09-03T07:24:15.080 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.080 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.081 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.081 DEBUG:boto:Params: {} 2022-09-03T07:24:15.082 DEBUG:boto:Token: None 2022-09-03T07:24:15.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:15.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:71zWaNoJR6C09wpt7r/ZwelaTVc= 2022-09-03T07:24:15.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:71zWaNoJR6C09wpt7r/ZwelaTVc=', 'Content-Length': '0'} 2022-09-03T07:24:15.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b6bf721ecac9613-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.087 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.087 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.088 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:15.088 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:15.089 DEBUG:boto:Method: GET 2022-09-03T07:24:15.090 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:15.090 DEBUG:boto:Data: 2022-09-03T07:24:15.091 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.091 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.092 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.092 DEBUG:boto:Params: {} 2022-09-03T07:24:15.093 DEBUG:boto:Token: None 2022-09-03T07:24:15.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.097 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003d15e57e5cc4d405-006313011f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.098 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.099 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.100 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.100 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:15.101 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:15.101 DEBUG:boto:Method: GET 2022-09-03T07:24:15.102 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:15.102 DEBUG:boto:Data: 2022-09-03T07:24:15.103 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.104 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.104 DEBUG:boto:Params: {} 2022-09-03T07:24:15.105 DEBUG:boto:Token: None 2022-09-03T07:24:15.105 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.106 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.107 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.110 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000984ff0bd6f4ef70d-006313011f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.111 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.112 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:15.112 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:15.113 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.113 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:15.114 DEBUG:boto:Data: 2022-09-03T07:24:15.114 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.115 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.116 DEBUG:boto:Params: {} 2022-09-03T07:24:15.117 DEBUG:boto:Token: None 2022-09-03T07:24:15.117 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj34 2022-09-03T07:24:15.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dYxRGS0RFcPLG7sT1IkOX/nuHSY= 2022-09-03T07:24:15.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dYxRGS0RFcPLG7sT1IkOX/nuHSY=', 'Content-Length': '0'} 2022-09-03T07:24:15.121 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae9354bedaf6fc78-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.121 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:15.122 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:15.123 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.123 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:15.124 DEBUG:boto:Data: 2022-09-03T07:24:15.124 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.124 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.125 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.125 DEBUG:boto:Params: {} 2022-09-03T07:24:15.126 DEBUG:boto:Token: None 2022-09-03T07:24:15.127 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj34 2022-09-03T07:24:15.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dYxRGS0RFcPLG7sT1IkOX/nuHSY= 2022-09-03T07:24:15.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dYxRGS0RFcPLG7sT1IkOX/nuHSY=', 'Content-Length': '0'} 2022-09-03T07:24:15.130 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb5d432562556d4f-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.130 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:24:15.131 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:15.132 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:15.132 DEBUG:boto:Method: GET 2022-09-03T07:24:15.133 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:15.133 DEBUG:boto:Data: 2022-09-03T07:24:15.134 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.135 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.135 DEBUG:boto:Params: {} 2022-09-03T07:24:15.136 DEBUG:boto:Token: None 2022-09-03T07:24:15.136 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj34 2022-09-03T07:24:15.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kdhq02gE92FZOSqMIZxRAnO9nfw= 2022-09-03T07:24:15.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kdhq02gE92FZOSqMIZxRAnO9nfw=', 'Content-Length': '0'} 2022-09-03T07:24:15.139 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000851a9340130aa492-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.140 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:15.141 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:15.141 DEBUG:boto:Method: GET 2022-09-03T07:24:15.142 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:15.142 DEBUG:boto:Data: 2022-09-03T07:24:15.143 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.144 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.144 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.145 DEBUG:boto:Params: {} 2022-09-03T07:24:15.145 DEBUG:boto:Token: None 2022-09-03T07:24:15.146 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj34 2022-09-03T07:24:15.146 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kdhq02gE92FZOSqMIZxRAnO9nfw= 2022-09-03T07:24:15.147 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kdhq02gE92FZOSqMIZxRAnO9nfw=', 'Content-Length': '0'} 2022-09-03T07:24:15.149 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a87366afbf03171-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.194 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.195 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.196 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:15.196 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:15.197 DEBUG:boto:Method: GET 2022-09-03T07:24:15.198 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:15.200 DEBUG:boto:Data: 2022-09-03T07:24:15.200 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.202 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.203 DEBUG:boto:Params: {} 2022-09-03T07:24:15.204 DEBUG:boto:Token: None 2022-09-03T07:24:15.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.211 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d58318de4da27750-006313011f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.212 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.214 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.216 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.217 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:15.218 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:15.219 DEBUG:boto:Method: GET 2022-09-03T07:24:15.220 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:15.221 DEBUG:boto:Data: 2022-09-03T07:24:15.222 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.223 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.224 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.225 DEBUG:boto:Params: {} 2022-09-03T07:24:15.226 DEBUG:boto:Token: None 2022-09-03T07:24:15.227 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.232 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005aaa4b62d9ced736-006313011f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.233 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.234 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:24:15.234 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.235 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.235 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.236 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.237 DEBUG:boto:Method: GET 2022-09-03T07:24:15.237 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.238 DEBUG:boto:Data: 2022-09-03T07:24:15.238 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.239 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.240 DEBUG:boto:Params: {} 2022-09-03T07:24:15.241 DEBUG:boto:Token: None 2022-09-03T07:24:15.241 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jGDBqXhNPBZC/NRoTlnCzJoJ08E= 2022-09-03T07:24:15.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jGDBqXhNPBZC/NRoTlnCzJoJ08E=', 'Content-Length': '0'} 2022-09-03T07:24:15.245 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9a8e18ff6409ed7-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.245 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.246 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.247 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.247 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.248 DEBUG:boto:Method: GET 2022-09-03T07:24:15.248 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.249 DEBUG:boto:Data: 2022-09-03T07:24:15.249 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.250 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.251 DEBUG:boto:Params: {} 2022-09-03T07:24:15.251 DEBUG:boto:Token: None 2022-09-03T07:24:15.252 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jGDBqXhNPBZC/NRoTlnCzJoJ08E= 2022-09-03T07:24:15.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jGDBqXhNPBZC/NRoTlnCzJoJ08E=', 'Content-Length': '0'} 2022-09-03T07:24:15.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009eff049175103530-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.256 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.257 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.258 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.258 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.259 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.259 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.260 DEBUG:boto:Data: 2022-09-03T07:24:15.260 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.261 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.261 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.262 DEBUG:boto:Params: {} 2022-09-03T07:24:15.262 DEBUG:boto:Token: None 2022-09-03T07:24:15.263 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.264 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4OpInLRGivUMLN6JxTcI7VflC3s= 2022-09-03T07:24:15.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4OpInLRGivUMLN6JxTcI7VflC3s=', 'Content-Length': '0'} 2022-09-03T07:24:15.266 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005caa5cc3f6b8824d-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.267 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.267 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.268 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.268 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.269 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.269 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.270 DEBUG:boto:Data: 2022-09-03T07:24:15.270 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.271 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.272 DEBUG:boto:Params: {} 2022-09-03T07:24:15.272 DEBUG:boto:Token: None 2022-09-03T07:24:15.273 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4OpInLRGivUMLN6JxTcI7VflC3s= 2022-09-03T07:24:15.274 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4OpInLRGivUMLN6JxTcI7VflC3s=', 'Content-Length': '0'} 2022-09-03T07:24:15.276 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000703aa0ce713de28d-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.277 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:24:15.277 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.278 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.279 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.279 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.280 DEBUG:boto:Method: GET 2022-09-03T07:24:15.280 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.281 DEBUG:boto:Data: 2022-09-03T07:24:15.281 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.282 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.283 DEBUG:boto:Params: {} 2022-09-03T07:24:15.284 DEBUG:boto:Token: None 2022-09-03T07:24:15.284 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.285 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jGDBqXhNPBZC/NRoTlnCzJoJ08E= 2022-09-03T07:24:15.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jGDBqXhNPBZC/NRoTlnCzJoJ08E=', 'Content-Length': '0'} 2022-09-03T07:24:15.288 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026a62ebbb5f41fa3-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.330 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.331 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.332 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.332 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.333 DEBUG:boto:Method: GET 2022-09-03T07:24:15.334 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.334 DEBUG:boto:Data: 2022-09-03T07:24:15.335 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.336 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.337 DEBUG:boto:Params: {} 2022-09-03T07:24:15.337 DEBUG:boto:Token: None 2022-09-03T07:24:15.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:15.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jGDBqXhNPBZC/NRoTlnCzJoJ08E= 2022-09-03T07:24:15.339 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jGDBqXhNPBZC/NRoTlnCzJoJ08E=', 'Content-Length': '0'} 2022-09-03T07:24:15.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e39bee66961518bb-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.343 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.343 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.344 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:15.345 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:15.345 DEBUG:boto:Method: GET 2022-09-03T07:24:15.346 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:15.346 DEBUG:boto:Data: 2022-09-03T07:24:15.347 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.347 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.348 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.349 DEBUG:boto:Params: {} 2022-09-03T07:24:15.349 DEBUG:boto:Token: None 2022-09-03T07:24:15.350 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.354 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d30c3a26434deb99-006313011f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.355 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.356 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.357 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.358 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:15.359 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:15.360 DEBUG:boto:Method: GET 2022-09-03T07:24:15.361 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:15.363 DEBUG:boto:Data: 2022-09-03T07:24:15.364 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.365 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.366 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.367 DEBUG:boto:Params: {} 2022-09-03T07:24:15.368 DEBUG:boto:Token: None 2022-09-03T07:24:15.369 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.371 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.374 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c31d6fc076b6700-006313011f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.376 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.377 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.378 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.379 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.380 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:15.381 DEBUG:boto:Data: 2022-09-03T07:24:15.382 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.384 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.385 DEBUG:boto:Params: {} 2022-09-03T07:24:15.386 DEBUG:boto:Token: None 2022-09-03T07:24:15.387 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38 2022-09-03T07:24:15.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3mkd2Usy2e1oX2B7waDlCopONMI= 2022-09-03T07:24:15.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3mkd2Usy2e1oX2B7waDlCopONMI=', 'Content-Length': '0'} 2022-09-03T07:24:15.391 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e5c3643566d90fc-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.392 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.392 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.393 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.393 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:15.394 DEBUG:boto:Data: 2022-09-03T07:24:15.394 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.395 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.396 DEBUG:boto:Params: {} 2022-09-03T07:24:15.397 DEBUG:boto:Token: None 2022-09-03T07:24:15.398 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38 2022-09-03T07:24:15.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3mkd2Usy2e1oX2B7waDlCopONMI= 2022-09-03T07:24:15.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3mkd2Usy2e1oX2B7waDlCopONMI=', 'Content-Length': '0'} 2022-09-03T07:24:15.401 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af88283869f34b12-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.401 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:24:15.402 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.402 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.403 DEBUG:boto:Method: GET 2022-09-03T07:24:15.404 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:15.404 DEBUG:boto:Data: 2022-09-03T07:24:15.405 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.406 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.407 DEBUG:boto:Params: {} 2022-09-03T07:24:15.407 DEBUG:boto:Token: None 2022-09-03T07:24:15.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38 2022-09-03T07:24:15.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I4f4SK809fqKgz9vHrLvEgvAiao= 2022-09-03T07:24:15.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I4f4SK809fqKgz9vHrLvEgvAiao=', 'Content-Length': '0'} 2022-09-03T07:24:15.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c6d8200012dc9c73-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.412 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:15.413 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:15.414 DEBUG:boto:Method: GET 2022-09-03T07:24:15.415 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:15.415 DEBUG:boto:Data: 2022-09-03T07:24:15.416 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.417 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.418 DEBUG:boto:Params: {} 2022-09-03T07:24:15.418 DEBUG:boto:Token: None 2022-09-03T07:24:15.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj38 2022-09-03T07:24:15.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I4f4SK809fqKgz9vHrLvEgvAiao= 2022-09-03T07:24:15.421 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I4f4SK809fqKgz9vHrLvEgvAiao=', 'Content-Length': '0'} 2022-09-03T07:24:15.423 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063fb020458984ff3-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.466 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.467 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.468 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:15.468 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:15.469 DEBUG:boto:Method: GET 2022-09-03T07:24:15.469 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:15.470 DEBUG:boto:Data: 2022-09-03T07:24:15.471 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.471 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.472 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.473 DEBUG:boto:Params: {} 2022-09-03T07:24:15.473 DEBUG:boto:Token: None 2022-09-03T07:24:15.474 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.478 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000034f6cf8e9b8ea450-006313011f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.479 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.480 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.481 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.482 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:15.483 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:15.484 DEBUG:boto:Method: GET 2022-09-03T07:24:15.485 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:15.485 DEBUG:boto:Data: 2022-09-03T07:24:15.486 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.488 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.489 DEBUG:boto:Params: {} 2022-09-03T07:24:15.489 DEBUG:boto:Token: None 2022-09-03T07:24:15.490 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.495 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ebcb79a7c815a4ea-006313011f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.496 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.497 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:24:15.498 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.498 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.499 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.500 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.500 DEBUG:boto:Method: GET 2022-09-03T07:24:15.501 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.502 DEBUG:boto:Data: 2022-09-03T07:24:15.502 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.504 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.504 DEBUG:boto:Params: {} 2022-09-03T07:24:15.505 DEBUG:boto:Token: None 2022-09-03T07:24:15.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JWTGH11tJYCTGThKG2uBuAGCYDQ= 2022-09-03T07:24:15.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JWTGH11tJYCTGThKG2uBuAGCYDQ=', 'Content-Length': '0'} 2022-09-03T07:24:15.509 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000089fd740495a42fe-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.554 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.555 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.555 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.556 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.557 DEBUG:boto:Method: GET 2022-09-03T07:24:15.557 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.558 DEBUG:boto:Data: 2022-09-03T07:24:15.558 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.560 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.561 DEBUG:boto:Params: {} 2022-09-03T07:24:15.562 DEBUG:boto:Token: None 2022-09-03T07:24:15.563 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JWTGH11tJYCTGThKG2uBuAGCYDQ= 2022-09-03T07:24:15.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JWTGH11tJYCTGThKG2uBuAGCYDQ=', 'Content-Length': '0'} 2022-09-03T07:24:15.567 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e01782204e65b53e-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.568 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.568 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.569 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.569 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.570 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.570 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.571 DEBUG:boto:Data: 2022-09-03T07:24:15.571 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.572 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.572 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.573 DEBUG:boto:Params: {} 2022-09-03T07:24:15.574 DEBUG:boto:Token: None 2022-09-03T07:24:15.574 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.575 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DgY5e+5x9bk8gY7vBwaUSOSFgcU= 2022-09-03T07:24:15.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DgY5e+5x9bk8gY7vBwaUSOSFgcU=', 'Content-Length': '0'} 2022-09-03T07:24:15.577 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1deec46fd5c6bc1-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.578 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.579 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.580 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.580 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.581 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.582 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.583 DEBUG:boto:Data: 2022-09-03T07:24:15.583 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.584 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.585 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.586 DEBUG:boto:Params: {} 2022-09-03T07:24:15.587 DEBUG:boto:Token: None 2022-09-03T07:24:15.588 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DgY5e+5x9bk8gY7vBwaUSOSFgcU= 2022-09-03T07:24:15.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DgY5e+5x9bk8gY7vBwaUSOSFgcU=', 'Content-Length': '0'} 2022-09-03T07:24:15.593 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000225781f234cb0c79-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.595 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:24:15.596 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.597 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.598 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.599 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.600 DEBUG:boto:Method: GET 2022-09-03T07:24:15.601 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.602 DEBUG:boto:Data: 2022-09-03T07:24:15.603 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.605 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.606 DEBUG:boto:Params: {} 2022-09-03T07:24:15.607 DEBUG:boto:Token: None 2022-09-03T07:24:15.608 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JWTGH11tJYCTGThKG2uBuAGCYDQ= 2022-09-03T07:24:15.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JWTGH11tJYCTGThKG2uBuAGCYDQ=', 'Content-Length': '0'} 2022-09-03T07:24:15.611 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ead21a8164565bc-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.612 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.612 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.613 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.613 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.614 DEBUG:boto:Method: GET 2022-09-03T07:24:15.614 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.615 DEBUG:boto:Data: 2022-09-03T07:24:15.616 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.617 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.617 DEBUG:boto:Params: {} 2022-09-03T07:24:15.618 DEBUG:boto:Token: None 2022-09-03T07:24:15.618 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:15.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JWTGH11tJYCTGThKG2uBuAGCYDQ= 2022-09-03T07:24:15.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JWTGH11tJYCTGThKG2uBuAGCYDQ=', 'Content-Length': '0'} 2022-09-03T07:24:15.622 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045df4ce376e5f287-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.623 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.623 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.624 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:15.624 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:15.625 DEBUG:boto:Method: GET 2022-09-03T07:24:15.625 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:15.626 DEBUG:boto:Data: 2022-09-03T07:24:15.626 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.627 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.628 DEBUG:boto:Params: {} 2022-09-03T07:24:15.628 DEBUG:boto:Token: None 2022-09-03T07:24:15.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.633 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000064813b94aed9e5c2-006313011f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.633 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.634 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.635 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.635 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:15.636 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:15.636 DEBUG:boto:Method: GET 2022-09-03T07:24:15.637 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:15.638 DEBUG:boto:Data: 2022-09-03T07:24:15.638 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.639 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.640 DEBUG:boto:Params: {} 2022-09-03T07:24:15.641 DEBUG:boto:Token: None 2022-09-03T07:24:15.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.646 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f7adaf84f9099ba2-006313011f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.646 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.647 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.648 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.648 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.649 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:15.650 DEBUG:boto:Data: 2022-09-03T07:24:15.651 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.652 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.653 DEBUG:boto:Params: {} 2022-09-03T07:24:15.654 DEBUG:boto:Token: None 2022-09-03T07:24:15.654 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44 2022-09-03T07:24:15.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+VFWokmgy2RYte+xCEL+DiNgzJw= 2022-09-03T07:24:15.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+VFWokmgy2RYte+xCEL+DiNgzJw=', 'Content-Length': '0'} 2022-09-03T07:24:15.658 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000613c06eb52e5ece8-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.659 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.659 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.660 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.660 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:15.661 DEBUG:boto:Data: 2022-09-03T07:24:15.662 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.663 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.663 DEBUG:boto:Params: {} 2022-09-03T07:24:15.664 DEBUG:boto:Token: None 2022-09-03T07:24:15.665 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44 2022-09-03T07:24:15.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+VFWokmgy2RYte+xCEL+DiNgzJw= 2022-09-03T07:24:15.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+VFWokmgy2RYte+xCEL+DiNgzJw=', 'Content-Length': '0'} 2022-09-03T07:24:15.668 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e46872c57914641-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.669 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:24:15.670 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.670 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.671 DEBUG:boto:Method: GET 2022-09-03T07:24:15.671 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:15.672 DEBUG:boto:Data: 2022-09-03T07:24:15.672 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.673 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.673 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.674 DEBUG:boto:Params: {} 2022-09-03T07:24:15.675 DEBUG:boto:Token: None 2022-09-03T07:24:15.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44 2022-09-03T07:24:15.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+XNgTltMW25ECB4eNj6oN4cGdN0= 2022-09-03T07:24:15.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+XNgTltMW25ECB4eNj6oN4cGdN0=', 'Content-Length': '0'} 2022-09-03T07:24:15.678 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e4aa9cbd67ad677-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.679 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:15.680 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:15.681 DEBUG:boto:Method: GET 2022-09-03T07:24:15.681 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:15.682 DEBUG:boto:Data: 2022-09-03T07:24:15.682 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.684 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.684 DEBUG:boto:Params: {} 2022-09-03T07:24:15.685 DEBUG:boto:Token: None 2022-09-03T07:24:15.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj44 2022-09-03T07:24:15.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+XNgTltMW25ECB4eNj6oN4cGdN0= 2022-09-03T07:24:15.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+XNgTltMW25ECB4eNj6oN4cGdN0=', 'Content-Length': '0'} 2022-09-03T07:24:15.688 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037ebf0646637a735-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.690 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.690 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.691 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:15.691 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:15.692 DEBUG:boto:Method: GET 2022-09-03T07:24:15.692 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:15.693 DEBUG:boto:Data: 2022-09-03T07:24:15.693 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.694 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.694 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.695 DEBUG:boto:Params: {} 2022-09-03T07:24:15.695 DEBUG:boto:Token: None 2022-09-03T07:24:15.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.697 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.700 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000731f82e0e716b65e-006313011f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.746 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.748 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.748 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.749 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:15.750 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:15.750 DEBUG:boto:Method: GET 2022-09-03T07:24:15.751 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:15.751 DEBUG:boto:Data: 2022-09-03T07:24:15.752 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.752 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.753 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.753 DEBUG:boto:Params: {} 2022-09-03T07:24:15.754 DEBUG:boto:Token: None 2022-09-03T07:24:15.755 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.761 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006df37acae03918b4-006313011f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.762 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.763 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:24:15.763 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:15.764 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:15.765 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.765 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.766 DEBUG:boto:Method: GET 2022-09-03T07:24:15.766 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.767 DEBUG:boto:Data: 2022-09-03T07:24:15.767 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.769 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.770 DEBUG:boto:Params: {} 2022-09-03T07:24:15.772 DEBUG:boto:Token: None 2022-09-03T07:24:15.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r1vQN7zeaNnKupt5q1E1mSb1O2c= 2022-09-03T07:24:15.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r1vQN7zeaNnKupt5q1E1mSb1O2c=', 'Content-Length': '0'} 2022-09-03T07:24:15.785 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f037c7adfb6a6ab-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.786 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:15.787 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:15.788 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.788 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.789 DEBUG:boto:Method: GET 2022-09-03T07:24:15.790 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.790 DEBUG:boto:Data: 2022-09-03T07:24:15.791 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.792 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.792 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.793 DEBUG:boto:Params: {} 2022-09-03T07:24:15.794 DEBUG:boto:Token: None 2022-09-03T07:24:15.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r1vQN7zeaNnKupt5q1E1mSb1O2c= 2022-09-03T07:24:15.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r1vQN7zeaNnKupt5q1E1mSb1O2c=', 'Content-Length': '0'} 2022-09-03T07:24:15.803 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001698469b74265446-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.804 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:15.805 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:15.806 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.807 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.808 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.809 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.810 DEBUG:boto:Data: 2022-09-03T07:24:15.811 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.812 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.813 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.814 DEBUG:boto:Params: {} 2022-09-03T07:24:15.815 DEBUG:boto:Token: None 2022-09-03T07:24:15.816 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JV6P1NWDTtt//YOVtBtmp1wk6S4= 2022-09-03T07:24:15.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JV6P1NWDTtt//YOVtBtmp1wk6S4=', 'Content-Length': '0'} 2022-09-03T07:24:15.821 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002aec3cd34b6a3d50-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.823 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:15.824 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:15.825 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.826 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.827 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.828 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.830 DEBUG:boto:Data: 2022-09-03T07:24:15.830 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.832 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.833 DEBUG:boto:Params: {} 2022-09-03T07:24:15.833 DEBUG:boto:Token: None 2022-09-03T07:24:15.834 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JV6P1NWDTtt//YOVtBtmp1wk6S4= 2022-09-03T07:24:15.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JV6P1NWDTtt//YOVtBtmp1wk6S4=', 'Content-Length': '0'} 2022-09-03T07:24:15.838 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081f810ee8261414c-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.839 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:24:15.840 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:15.840 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:15.841 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.842 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.842 DEBUG:boto:Method: GET 2022-09-03T07:24:15.843 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.844 DEBUG:boto:Data: 2022-09-03T07:24:15.846 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.848 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.849 DEBUG:boto:Params: {} 2022-09-03T07:24:15.851 DEBUG:boto:Token: None 2022-09-03T07:24:15.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r1vQN7zeaNnKupt5q1E1mSb1O2c= 2022-09-03T07:24:15.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r1vQN7zeaNnKupt5q1E1mSb1O2c=', 'Content-Length': '0'} 2022-09-03T07:24:15.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a9a946e4aa5dd91-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.859 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:15.860 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:15.860 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.861 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.862 DEBUG:boto:Method: GET 2022-09-03T07:24:15.863 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.863 DEBUG:boto:Data: 2022-09-03T07:24:15.864 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.865 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.866 DEBUG:boto:Params: {} 2022-09-03T07:24:15.867 DEBUG:boto:Token: None 2022-09-03T07:24:15.868 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:15.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r1vQN7zeaNnKupt5q1E1mSb1O2c= 2022-09-03T07:24:15.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r1vQN7zeaNnKupt5q1E1mSb1O2c=', 'Content-Length': '0'} 2022-09-03T07:24:15.872 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000073323d3201d804db-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.920 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.921 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.921 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:15.922 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:15.922 DEBUG:boto:Method: GET 2022-09-03T07:24:15.923 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:15.924 DEBUG:boto:Data: 2022-09-03T07:24:15.924 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.925 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.925 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.926 DEBUG:boto:Params: {} 2022-09-03T07:24:15.927 DEBUG:boto:Token: None 2022-09-03T07:24:15.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.935 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e6895d436e723e9e-006313011f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.938 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.940 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:15.940 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:15.941 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:15.941 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:15.942 DEBUG:boto:Method: GET 2022-09-03T07:24:15.943 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:15.943 DEBUG:boto:Data: 2022-09-03T07:24:15.944 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.945 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.946 DEBUG:boto:Params: {} 2022-09-03T07:24:15.947 DEBUG:boto:Token: None 2022-09-03T07:24:15.947 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/ 2022-09-03T07:24:15.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE= 2022-09-03T07:24:15.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5yDPwL0o0iAscb7q7n8BmhkLNE=', 'Content-Length': '0'} 2022-09-03T07:24:15.954 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e5fb993a3a26b6cd-006313011f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.955 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:15.956 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:15.957 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:15.957 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.958 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:15.958 DEBUG:boto:Data: 2022-09-03T07:24:15.959 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.960 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.960 DEBUG:boto:Params: {} 2022-09-03T07:24:15.961 DEBUG:boto:Token: None 2022-09-03T07:24:15.962 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj46 2022-09-03T07:24:15.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hUQ+aMTCOknaYhvDNYst96RhISE= 2022-09-03T07:24:15.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hUQ+aMTCOknaYhvDNYst96RhISE=', 'Content-Length': '0'} 2022-09-03T07:24:15.967 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e0f46dc09ad4f406-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.969 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:15.970 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:15.971 DEBUG:boto:Method: HEAD 2022-09-03T07:24:15.971 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:15.972 DEBUG:boto:Data: 2022-09-03T07:24:15.973 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:15.975 DEBUG:boto:Port: 8000 2022-09-03T07:24:15.976 DEBUG:boto:Params: {} 2022-09-03T07:24:15.977 DEBUG:boto:Token: None 2022-09-03T07:24:15.978 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj46 2022-09-03T07:24:15.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hUQ+aMTCOknaYhvDNYst96RhISE= 2022-09-03T07:24:15.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hUQ+aMTCOknaYhvDNYst96RhISE=', 'Content-Length': '0'} 2022-09-03T07:24:15.984 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000082f9f036422d637f-006313011f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:15.985 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:24:15.985 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:15.986 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:15.987 DEBUG:boto:Method: GET 2022-09-03T07:24:15.987 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:15.988 DEBUG:boto:Data: 2022-09-03T07:24:15.988 DEBUG:boto:Headers: {} 2022-09-03T07:24:15.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:15.989 DEBUG:boto:Port: 8001 2022-09-03T07:24:15.990 DEBUG:boto:Params: {} 2022-09-03T07:24:15.991 DEBUG:boto:Token: None 2022-09-03T07:24:15.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:15 GMT /avysmw-75/obj46 2022-09-03T07:24:15.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qQLEKxL1j4ew3Xx8yvED1c3IMas= 2022-09-03T07:24:15.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qQLEKxL1j4ew3Xx8yvED1c3IMas=', 'Content-Length': '0'} 2022-09-03T07:24:15.996 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080f6d97d98a8669a-006313011f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.042 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:16.043 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:16.044 DEBUG:boto:Method: GET 2022-09-03T07:24:16.045 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:16.045 DEBUG:boto:Data: 2022-09-03T07:24:16.046 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.047 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.048 DEBUG:boto:Params: {} 2022-09-03T07:24:16.048 DEBUG:boto:Token: None 2022-09-03T07:24:16.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj46 2022-09-03T07:24:16.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9EZGQEtJ86oVCPCw8oZjLerdNXY= 2022-09-03T07:24:16.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9EZGQEtJ86oVCPCw8oZjLerdNXY=', 'Content-Length': '0'} 2022-09-03T07:24:16.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075fdb993714fe068-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.054 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.054 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.055 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:16.055 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:16.056 DEBUG:boto:Method: GET 2022-09-03T07:24:16.057 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:16.058 DEBUG:boto:Data: 2022-09-03T07:24:16.059 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.060 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.061 DEBUG:boto:Params: {} 2022-09-03T07:24:16.062 DEBUG:boto:Token: None 2022-09-03T07:24:16.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cfbc201bd0d301aa-0063130120-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.070 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.071 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.072 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.072 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:16.073 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:16.073 DEBUG:boto:Method: GET 2022-09-03T07:24:16.074 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:16.074 DEBUG:boto:Data: 2022-09-03T07:24:16.075 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.076 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.077 DEBUG:boto:Params: {} 2022-09-03T07:24:16.078 DEBUG:boto:Token: None 2022-09-03T07:24:16.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009acc51114c8dcde2-0063130120-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.085 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.087 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:24:16.087 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.088 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.088 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.089 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.090 DEBUG:boto:Method: GET 2022-09-03T07:24:16.090 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.091 DEBUG:boto:Data: 2022-09-03T07:24:16.091 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.093 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.093 DEBUG:boto:Params: {} 2022-09-03T07:24:16.094 DEBUG:boto:Token: None 2022-09-03T07:24:16.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PqA4qJoUUNEloYcC1+tS49nCnmw= 2022-09-03T07:24:16.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PqA4qJoUUNEloYcC1+tS49nCnmw=', 'Content-Length': '0'} 2022-09-03T07:24:16.099 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ea14385e02b77cd-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.100 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.101 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.102 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.103 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.103 DEBUG:boto:Method: GET 2022-09-03T07:24:16.104 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.105 DEBUG:boto:Data: 2022-09-03T07:24:16.106 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.107 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.109 DEBUG:boto:Params: {} 2022-09-03T07:24:16.110 DEBUG:boto:Token: None 2022-09-03T07:24:16.111 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PqA4qJoUUNEloYcC1+tS49nCnmw= 2022-09-03T07:24:16.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PqA4qJoUUNEloYcC1+tS49nCnmw=', 'Content-Length': '0'} 2022-09-03T07:24:16.115 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000047a29b50ea7ca87d-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.116 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.117 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.118 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.118 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.119 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.119 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.120 DEBUG:boto:Data: 2022-09-03T07:24:16.121 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.121 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.122 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.122 DEBUG:boto:Params: {} 2022-09-03T07:24:16.123 DEBUG:boto:Token: None 2022-09-03T07:24:16.123 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TGyTHCXoH57T8JcCoMLiSmG8rM0= 2022-09-03T07:24:16.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TGyTHCXoH57T8JcCoMLiSmG8rM0=', 'Content-Length': '0'} 2022-09-03T07:24:16.127 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa35458915ff0762-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.127 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.128 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.129 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.129 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.129 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.130 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.130 DEBUG:boto:Data: 2022-09-03T07:24:16.131 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.131 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.132 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.132 DEBUG:boto:Params: {} 2022-09-03T07:24:16.133 DEBUG:boto:Token: None 2022-09-03T07:24:16.134 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TGyTHCXoH57T8JcCoMLiSmG8rM0= 2022-09-03T07:24:16.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TGyTHCXoH57T8JcCoMLiSmG8rM0=', 'Content-Length': '0'} 2022-09-03T07:24:16.137 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d54708b1000ff18-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.137 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:24:16.138 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.139 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.139 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.140 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.140 DEBUG:boto:Method: GET 2022-09-03T07:24:16.141 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.141 DEBUG:boto:Data: 2022-09-03T07:24:16.142 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.143 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.144 DEBUG:boto:Params: {} 2022-09-03T07:24:16.144 DEBUG:boto:Token: None 2022-09-03T07:24:16.145 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.145 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PqA4qJoUUNEloYcC1+tS49nCnmw= 2022-09-03T07:24:16.146 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PqA4qJoUUNEloYcC1+tS49nCnmw=', 'Content-Length': '0'} 2022-09-03T07:24:16.148 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071bd0b2b1d5df856-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.194 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.195 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.197 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.198 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.199 DEBUG:boto:Method: GET 2022-09-03T07:24:16.199 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.200 DEBUG:boto:Data: 2022-09-03T07:24:16.201 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.202 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.203 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.204 DEBUG:boto:Params: {} 2022-09-03T07:24:16.205 DEBUG:boto:Token: None 2022-09-03T07:24:16.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:16.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PqA4qJoUUNEloYcC1+tS49nCnmw= 2022-09-03T07:24:16.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PqA4qJoUUNEloYcC1+tS49nCnmw=', 'Content-Length': '0'} 2022-09-03T07:24:16.210 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e58f2f8aa0cbc4a-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.211 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.212 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.213 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:16.214 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:16.215 DEBUG:boto:Method: GET 2022-09-03T07:24:16.216 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:16.217 DEBUG:boto:Data: 2022-09-03T07:24:16.218 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.219 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.220 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.221 DEBUG:boto:Params: {} 2022-09-03T07:24:16.222 DEBUG:boto:Token: None 2022-09-03T07:24:16.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.229 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f4ae6e487cdec792-0063130120-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.230 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.231 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.232 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.233 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:16.234 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:16.235 DEBUG:boto:Method: GET 2022-09-03T07:24:16.236 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:16.236 DEBUG:boto:Data: 2022-09-03T07:24:16.237 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.238 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.239 DEBUG:boto:Params: {} 2022-09-03T07:24:16.239 DEBUG:boto:Token: None 2022-09-03T07:24:16.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.245 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000b9d005ab290b738-0063130120-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.286 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.287 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.288 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.288 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.289 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:16.289 DEBUG:boto:Data: 2022-09-03T07:24:16.289 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.290 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.291 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.291 DEBUG:boto:Params: {} 2022-09-03T07:24:16.292 DEBUG:boto:Token: None 2022-09-03T07:24:16.293 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51 2022-09-03T07:24:16.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vbpeYlQ7gBBuMDzzzS+FTVbVF0A= 2022-09-03T07:24:16.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vbpeYlQ7gBBuMDzzzS+FTVbVF0A=', 'Content-Length': '0'} 2022-09-03T07:24:16.296 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039f86748dea8f4dc-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.297 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.297 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.298 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.298 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:16.299 DEBUG:boto:Data: 2022-09-03T07:24:16.299 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.300 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.301 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.302 DEBUG:boto:Params: {} 2022-09-03T07:24:16.303 DEBUG:boto:Token: None 2022-09-03T07:24:16.304 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51 2022-09-03T07:24:16.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vbpeYlQ7gBBuMDzzzS+FTVbVF0A= 2022-09-03T07:24:16.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vbpeYlQ7gBBuMDzzzS+FTVbVF0A=', 'Content-Length': '0'} 2022-09-03T07:24:16.307 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009538c79ab0906869-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.308 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:24:16.308 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.309 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.310 DEBUG:boto:Method: GET 2022-09-03T07:24:16.310 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:16.311 DEBUG:boto:Data: 2022-09-03T07:24:16.311 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.312 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.313 DEBUG:boto:Params: {} 2022-09-03T07:24:16.314 DEBUG:boto:Token: None 2022-09-03T07:24:16.314 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51 2022-09-03T07:24:16.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3SmDBsNpaeSBddG+aLldLjur9Zs= 2022-09-03T07:24:16.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3SmDBsNpaeSBddG+aLldLjur9Zs=', 'Content-Length': '0'} 2022-09-03T07:24:16.317 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068b037546f41ef8d-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.318 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:16.319 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:16.319 DEBUG:boto:Method: GET 2022-09-03T07:24:16.320 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:16.321 DEBUG:boto:Data: 2022-09-03T07:24:16.321 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.322 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.322 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.323 DEBUG:boto:Params: {} 2022-09-03T07:24:16.323 DEBUG:boto:Token: None 2022-09-03T07:24:16.324 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj51 2022-09-03T07:24:16.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3SmDBsNpaeSBddG+aLldLjur9Zs= 2022-09-03T07:24:16.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3SmDBsNpaeSBddG+aLldLjur9Zs=', 'Content-Length': '0'} 2022-09-03T07:24:16.327 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa78a84fa5056a2e-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.328 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.329 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.329 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:16.330 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:16.330 DEBUG:boto:Method: GET 2022-09-03T07:24:16.331 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:16.331 DEBUG:boto:Data: 2022-09-03T07:24:16.332 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.333 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.334 DEBUG:boto:Params: {} 2022-09-03T07:24:16.334 DEBUG:boto:Token: None 2022-09-03T07:24:16.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.339 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069158fd368d681ab-0063130120-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.339 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.340 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.341 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.341 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:16.342 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:16.342 DEBUG:boto:Method: GET 2022-09-03T07:24:16.343 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:16.343 DEBUG:boto:Data: 2022-09-03T07:24:16.344 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.345 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.346 DEBUG:boto:Params: {} 2022-09-03T07:24:16.346 DEBUG:boto:Token: None 2022-09-03T07:24:16.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.351 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b91772888065c000-0063130120-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.352 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.353 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:24:16.354 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.354 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.355 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.355 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.356 DEBUG:boto:Method: GET 2022-09-03T07:24:16.356 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.357 DEBUG:boto:Data: 2022-09-03T07:24:16.357 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.359 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.359 DEBUG:boto:Params: {} 2022-09-03T07:24:16.360 DEBUG:boto:Token: None 2022-09-03T07:24:16.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LrPZOeJ5hivz0o+QElejYlCRZck= 2022-09-03T07:24:16.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LrPZOeJ5hivz0o+QElejYlCRZck=', 'Content-Length': '0'} 2022-09-03T07:24:16.364 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e2621b6da481d20-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.365 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.365 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.366 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.366 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.367 DEBUG:boto:Method: GET 2022-09-03T07:24:16.368 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.369 DEBUG:boto:Data: 2022-09-03T07:24:16.370 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.371 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.372 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.373 DEBUG:boto:Params: {} 2022-09-03T07:24:16.374 DEBUG:boto:Token: None 2022-09-03T07:24:16.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LrPZOeJ5hivz0o+QElejYlCRZck= 2022-09-03T07:24:16.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LrPZOeJ5hivz0o+QElejYlCRZck=', 'Content-Length': '0'} 2022-09-03T07:24:16.378 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000592c28435f5f9013-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.379 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.380 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.380 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.381 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.381 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.382 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.383 DEBUG:boto:Data: 2022-09-03T07:24:16.383 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.384 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.385 DEBUG:boto:Params: {} 2022-09-03T07:24:16.385 DEBUG:boto:Token: None 2022-09-03T07:24:16.386 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D1RU7MURGsVah8j2lRgJhC0aUQM= 2022-09-03T07:24:16.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D1RU7MURGsVah8j2lRgJhC0aUQM=', 'Content-Length': '0'} 2022-09-03T07:24:16.389 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6c04bd7fa68e134-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.390 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.390 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.391 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.391 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.392 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.392 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.393 DEBUG:boto:Data: 2022-09-03T07:24:16.393 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.394 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.394 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.395 DEBUG:boto:Params: {} 2022-09-03T07:24:16.396 DEBUG:boto:Token: None 2022-09-03T07:24:16.396 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D1RU7MURGsVah8j2lRgJhC0aUQM= 2022-09-03T07:24:16.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D1RU7MURGsVah8j2lRgJhC0aUQM=', 'Content-Length': '0'} 2022-09-03T07:24:16.399 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b101de5c1f561d58-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.400 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:24:16.401 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.401 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.402 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.402 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.403 DEBUG:boto:Method: GET 2022-09-03T07:24:16.404 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.404 DEBUG:boto:Data: 2022-09-03T07:24:16.405 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.405 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.406 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.406 DEBUG:boto:Params: {} 2022-09-03T07:24:16.407 DEBUG:boto:Token: None 2022-09-03T07:24:16.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LrPZOeJ5hivz0o+QElejYlCRZck= 2022-09-03T07:24:16.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LrPZOeJ5hivz0o+QElejYlCRZck=', 'Content-Length': '0'} 2022-09-03T07:24:16.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037a95b5040af5329-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.412 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.412 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.413 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.414 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.414 DEBUG:boto:Method: GET 2022-09-03T07:24:16.415 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.415 DEBUG:boto:Data: 2022-09-03T07:24:16.416 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.417 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.418 DEBUG:boto:Params: {} 2022-09-03T07:24:16.418 DEBUG:boto:Token: None 2022-09-03T07:24:16.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:16.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LrPZOeJ5hivz0o+QElejYlCRZck= 2022-09-03T07:24:16.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LrPZOeJ5hivz0o+QElejYlCRZck=', 'Content-Length': '0'} 2022-09-03T07:24:16.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e78073e8a6753a4d-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.466 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.467 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.468 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:16.469 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:16.470 DEBUG:boto:Method: GET 2022-09-03T07:24:16.471 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:16.472 DEBUG:boto:Data: 2022-09-03T07:24:16.473 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.474 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.474 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.475 DEBUG:boto:Params: {} 2022-09-03T07:24:16.476 DEBUG:boto:Token: None 2022-09-03T07:24:16.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.482 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000081bf577eeb1756b1-0063130120-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.482 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.483 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.484 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.484 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:16.485 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:16.485 DEBUG:boto:Method: GET 2022-09-03T07:24:16.486 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:16.487 DEBUG:boto:Data: 2022-09-03T07:24:16.487 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.488 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.489 DEBUG:boto:Params: {} 2022-09-03T07:24:16.490 DEBUG:boto:Token: None 2022-09-03T07:24:16.490 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.495 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027f0f2bbd7dcc8f6-0063130120-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.496 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.498 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.499 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.500 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.501 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:16.501 DEBUG:boto:Data: 2022-09-03T07:24:16.502 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.504 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.505 DEBUG:boto:Params: {} 2022-09-03T07:24:16.506 DEBUG:boto:Token: None 2022-09-03T07:24:16.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55 2022-09-03T07:24:16.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WsUYjuWY1fB7qMieuGB9bI4HBmc= 2022-09-03T07:24:16.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WsUYjuWY1fB7qMieuGB9bI4HBmc=', 'Content-Length': '0'} 2022-09-03T07:24:16.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3dc8987d8242596-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.512 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.512 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.513 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.513 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:16.514 DEBUG:boto:Data: 2022-09-03T07:24:16.514 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.516 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.516 DEBUG:boto:Params: {} 2022-09-03T07:24:16.517 DEBUG:boto:Token: None 2022-09-03T07:24:16.517 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55 2022-09-03T07:24:16.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WsUYjuWY1fB7qMieuGB9bI4HBmc= 2022-09-03T07:24:16.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WsUYjuWY1fB7qMieuGB9bI4HBmc=', 'Content-Length': '0'} 2022-09-03T07:24:16.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e703a12da28a9ebf-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:24:16.522 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.522 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.523 DEBUG:boto:Method: GET 2022-09-03T07:24:16.524 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:16.524 DEBUG:boto:Data: 2022-09-03T07:24:16.525 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.526 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.526 DEBUG:boto:Params: {} 2022-09-03T07:24:16.527 DEBUG:boto:Token: None 2022-09-03T07:24:16.528 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55 2022-09-03T07:24:16.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0XH47cFfm5/5B/qjAPW1dkqd03k= 2022-09-03T07:24:16.529 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0XH47cFfm5/5B/qjAPW1dkqd03k=', 'Content-Length': '0'} 2022-09-03T07:24:16.531 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dca4d8862276dd31-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.531 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:16.532 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:16.533 DEBUG:boto:Method: GET 2022-09-03T07:24:16.533 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:16.534 DEBUG:boto:Data: 2022-09-03T07:24:16.534 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.535 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.536 DEBUG:boto:Params: {} 2022-09-03T07:24:16.537 DEBUG:boto:Token: None 2022-09-03T07:24:16.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj55 2022-09-03T07:24:16.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0XH47cFfm5/5B/qjAPW1dkqd03k= 2022-09-03T07:24:16.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0XH47cFfm5/5B/qjAPW1dkqd03k=', 'Content-Length': '0'} 2022-09-03T07:24:16.540 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042d7a562abf88121-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.541 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.542 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.542 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:16.543 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:16.543 DEBUG:boto:Method: GET 2022-09-03T07:24:16.544 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:16.544 DEBUG:boto:Data: 2022-09-03T07:24:16.545 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.546 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.546 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.547 DEBUG:boto:Params: {} 2022-09-03T07:24:16.547 DEBUG:boto:Token: None 2022-09-03T07:24:16.548 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.552 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000089fa53bb7b377fc5-0063130120-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.594 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.595 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.596 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.596 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:16.597 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:16.597 DEBUG:boto:Method: GET 2022-09-03T07:24:16.598 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:16.598 DEBUG:boto:Data: 2022-09-03T07:24:16.599 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.600 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.600 DEBUG:boto:Params: {} 2022-09-03T07:24:16.601 DEBUG:boto:Token: None 2022-09-03T07:24:16.602 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.606 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000369acfff813099a9-0063130120-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.607 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.608 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:24:16.608 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.609 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.610 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.610 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.611 DEBUG:boto:Method: GET 2022-09-03T07:24:16.611 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.612 DEBUG:boto:Data: 2022-09-03T07:24:16.612 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.613 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.614 DEBUG:boto:Params: {} 2022-09-03T07:24:16.615 DEBUG:boto:Token: None 2022-09-03T07:24:16.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mw0zhU6ltN/Z8p/fFYo9BylvpRI= 2022-09-03T07:24:16.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mw0zhU6ltN/Z8p/fFYo9BylvpRI=', 'Content-Length': '0'} 2022-09-03T07:24:16.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ac82f92ce366706-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.619 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.620 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.620 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.621 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.622 DEBUG:boto:Method: GET 2022-09-03T07:24:16.622 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.623 DEBUG:boto:Data: 2022-09-03T07:24:16.623 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.624 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.624 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.625 DEBUG:boto:Params: {} 2022-09-03T07:24:16.626 DEBUG:boto:Token: None 2022-09-03T07:24:16.626 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mw0zhU6ltN/Z8p/fFYo9BylvpRI= 2022-09-03T07:24:16.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mw0zhU6ltN/Z8p/fFYo9BylvpRI=', 'Content-Length': '0'} 2022-09-03T07:24:16.630 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ccc428a411830dee-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.674 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.675 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.675 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.676 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.676 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.677 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.678 DEBUG:boto:Data: 2022-09-03T07:24:16.678 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.679 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.679 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.680 DEBUG:boto:Params: {} 2022-09-03T07:24:16.681 DEBUG:boto:Token: None 2022-09-03T07:24:16.682 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aWDm1RV4jcpqURe1C+k6GcOmlZM= 2022-09-03T07:24:16.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aWDm1RV4jcpqURe1C+k6GcOmlZM=', 'Content-Length': '0'} 2022-09-03T07:24:16.685 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007b044052bc26d76b-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.686 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.687 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.687 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.688 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.688 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.689 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.689 DEBUG:boto:Data: 2022-09-03T07:24:16.690 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.690 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.691 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.691 DEBUG:boto:Params: {} 2022-09-03T07:24:16.692 DEBUG:boto:Token: None 2022-09-03T07:24:16.693 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.693 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aWDm1RV4jcpqURe1C+k6GcOmlZM= 2022-09-03T07:24:16.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aWDm1RV4jcpqURe1C+k6GcOmlZM=', 'Content-Length': '0'} 2022-09-03T07:24:16.696 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a7f52d490316e39-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.696 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:24:16.697 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.698 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.698 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.699 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.699 DEBUG:boto:Method: GET 2022-09-03T07:24:16.700 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.700 DEBUG:boto:Data: 2022-09-03T07:24:16.701 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.702 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.702 DEBUG:boto:Params: {} 2022-09-03T07:24:16.703 DEBUG:boto:Token: None 2022-09-03T07:24:16.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mw0zhU6ltN/Z8p/fFYo9BylvpRI= 2022-09-03T07:24:16.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mw0zhU6ltN/Z8p/fFYo9BylvpRI=', 'Content-Length': '0'} 2022-09-03T07:24:16.707 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064d69963f6883295-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.708 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.708 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.709 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.709 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.710 DEBUG:boto:Method: GET 2022-09-03T07:24:16.710 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.711 DEBUG:boto:Data: 2022-09-03T07:24:16.711 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.712 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.713 DEBUG:boto:Params: {} 2022-09-03T07:24:16.714 DEBUG:boto:Token: None 2022-09-03T07:24:16.714 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:16.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mw0zhU6ltN/Z8p/fFYo9BylvpRI= 2022-09-03T07:24:16.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mw0zhU6ltN/Z8p/fFYo9BylvpRI=', 'Content-Length': '0'} 2022-09-03T07:24:16.717 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e673fce177f23c34-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.718 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.719 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.719 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:16.720 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:16.720 DEBUG:boto:Method: GET 2022-09-03T07:24:16.721 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:16.721 DEBUG:boto:Data: 2022-09-03T07:24:16.722 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.723 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.723 DEBUG:boto:Params: {} 2022-09-03T07:24:16.724 DEBUG:boto:Token: None 2022-09-03T07:24:16.724 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.728 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b5af54b73d0b2b18-0063130120-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.729 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.730 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.730 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.731 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:16.731 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:16.732 DEBUG:boto:Method: GET 2022-09-03T07:24:16.732 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:16.733 DEBUG:boto:Data: 2022-09-03T07:24:16.733 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.734 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.735 DEBUG:boto:Params: {} 2022-09-03T07:24:16.736 DEBUG:boto:Token: None 2022-09-03T07:24:16.736 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.741 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001fb584ff8d87d42d-0063130120-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.742 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.743 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.743 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.744 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.744 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:16.745 DEBUG:boto:Data: 2022-09-03T07:24:16.745 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.747 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.748 DEBUG:boto:Params: {} 2022-09-03T07:24:16.748 DEBUG:boto:Token: None 2022-09-03T07:24:16.749 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57 2022-09-03T07:24:16.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zwCd1kR4DoQV+co/qrHFGvocGSc= 2022-09-03T07:24:16.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zwCd1kR4DoQV+co/qrHFGvocGSc=', 'Content-Length': '0'} 2022-09-03T07:24:16.752 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd42192d080123eb-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.753 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.754 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.754 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.755 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:16.755 DEBUG:boto:Data: 2022-09-03T07:24:16.756 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.758 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.759 DEBUG:boto:Params: {} 2022-09-03T07:24:16.760 DEBUG:boto:Token: None 2022-09-03T07:24:16.760 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57 2022-09-03T07:24:16.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zwCd1kR4DoQV+co/qrHFGvocGSc= 2022-09-03T07:24:16.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zwCd1kR4DoQV+co/qrHFGvocGSc=', 'Content-Length': '0'} 2022-09-03T07:24:16.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087d4597ded2faf4a-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.764 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:24:16.765 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.765 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.766 DEBUG:boto:Method: GET 2022-09-03T07:24:16.766 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:16.767 DEBUG:boto:Data: 2022-09-03T07:24:16.767 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.769 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.769 DEBUG:boto:Params: {} 2022-09-03T07:24:16.770 DEBUG:boto:Token: None 2022-09-03T07:24:16.770 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57 2022-09-03T07:24:16.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCBFGcXq7fnJpt8Dz/C4bMId/Og= 2022-09-03T07:24:16.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCBFGcXq7fnJpt8Dz/C4bMId/Og=', 'Content-Length': '0'} 2022-09-03T07:24:16.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc33c9760713000e-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.774 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:16.775 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:16.775 DEBUG:boto:Method: GET 2022-09-03T07:24:16.776 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:16.776 DEBUG:boto:Data: 2022-09-03T07:24:16.777 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.778 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.779 DEBUG:boto:Params: {} 2022-09-03T07:24:16.779 DEBUG:boto:Token: None 2022-09-03T07:24:16.780 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj57 2022-09-03T07:24:16.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCBFGcXq7fnJpt8Dz/C4bMId/Og= 2022-09-03T07:24:16.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCBFGcXq7fnJpt8Dz/C4bMId/Og=', 'Content-Length': '0'} 2022-09-03T07:24:16.783 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008e4ad9496ed7532-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.783 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.784 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.784 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:16.785 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:16.785 DEBUG:boto:Method: GET 2022-09-03T07:24:16.786 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:16.787 DEBUG:boto:Data: 2022-09-03T07:24:16.787 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.788 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.788 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.789 DEBUG:boto:Params: {} 2022-09-03T07:24:16.789 DEBUG:boto:Token: None 2022-09-03T07:24:16.790 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.793 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004422ce1a81886fa5-0063130120-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.794 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.795 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.795 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.796 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:16.796 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:16.797 DEBUG:boto:Method: GET 2022-09-03T07:24:16.797 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:16.798 DEBUG:boto:Data: 2022-09-03T07:24:16.798 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.799 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.799 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.800 DEBUG:boto:Params: {} 2022-09-03T07:24:16.800 DEBUG:boto:Token: None 2022-09-03T07:24:16.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.806 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e28dbb0483af1752-0063130120-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.806 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.807 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:24:16.808 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.808 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.809 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.809 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.810 DEBUG:boto:Method: GET 2022-09-03T07:24:16.810 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.811 DEBUG:boto:Data: 2022-09-03T07:24:16.811 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.812 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.812 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.813 DEBUG:boto:Params: {} 2022-09-03T07:24:16.814 DEBUG:boto:Token: None 2022-09-03T07:24:16.814 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lvN8vEDD68z5zgiImZtRwrPhj/A= 2022-09-03T07:24:16.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lvN8vEDD68z5zgiImZtRwrPhj/A=', 'Content-Length': '0'} 2022-09-03T07:24:16.817 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac435e5844c30b52-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.818 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.819 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.819 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.820 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.820 DEBUG:boto:Method: GET 2022-09-03T07:24:16.821 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.821 DEBUG:boto:Data: 2022-09-03T07:24:16.822 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.823 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.823 DEBUG:boto:Params: {} 2022-09-03T07:24:16.824 DEBUG:boto:Token: None 2022-09-03T07:24:16.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lvN8vEDD68z5zgiImZtRwrPhj/A= 2022-09-03T07:24:16.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lvN8vEDD68z5zgiImZtRwrPhj/A=', 'Content-Length': '0'} 2022-09-03T07:24:16.828 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039e5afa64547b850-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.829 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.829 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.830 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.830 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.831 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.831 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.832 DEBUG:boto:Data: 2022-09-03T07:24:16.832 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.833 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.833 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.834 DEBUG:boto:Params: {} 2022-09-03T07:24:16.835 DEBUG:boto:Token: None 2022-09-03T07:24:16.835 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G5RS1wzH8fOBi4Qp4893EmYqc8s= 2022-09-03T07:24:16.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G5RS1wzH8fOBi4Qp4893EmYqc8s=', 'Content-Length': '0'} 2022-09-03T07:24:16.838 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dbdf23bfbdcf0785-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.839 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.839 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.840 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.840 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.841 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.841 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.842 DEBUG:boto:Data: 2022-09-03T07:24:16.842 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.843 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.843 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.844 DEBUG:boto:Params: {} 2022-09-03T07:24:16.844 DEBUG:boto:Token: None 2022-09-03T07:24:16.845 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G5RS1wzH8fOBi4Qp4893EmYqc8s= 2022-09-03T07:24:16.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G5RS1wzH8fOBi4Qp4893EmYqc8s=', 'Content-Length': '0'} 2022-09-03T07:24:16.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016316626b18dd4b0-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.849 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:24:16.850 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.850 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.851 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.851 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.852 DEBUG:boto:Method: GET 2022-09-03T07:24:16.852 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.853 DEBUG:boto:Data: 2022-09-03T07:24:16.853 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.854 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.854 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.855 DEBUG:boto:Params: {} 2022-09-03T07:24:16.855 DEBUG:boto:Token: None 2022-09-03T07:24:16.856 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lvN8vEDD68z5zgiImZtRwrPhj/A= 2022-09-03T07:24:16.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lvN8vEDD68z5zgiImZtRwrPhj/A=', 'Content-Length': '0'} 2022-09-03T07:24:16.859 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097bd096218744383-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.906 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.907 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.907 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.908 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.909 DEBUG:boto:Method: GET 2022-09-03T07:24:16.909 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.910 DEBUG:boto:Data: 2022-09-03T07:24:16.910 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.911 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.911 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.912 DEBUG:boto:Params: {} 2022-09-03T07:24:16.912 DEBUG:boto:Token: None 2022-09-03T07:24:16.913 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:16.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lvN8vEDD68z5zgiImZtRwrPhj/A= 2022-09-03T07:24:16.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lvN8vEDD68z5zgiImZtRwrPhj/A=', 'Content-Length': '0'} 2022-09-03T07:24:16.916 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aba7c2dbf737a4db-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.917 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.918 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.918 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:16.919 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:16.919 DEBUG:boto:Method: GET 2022-09-03T07:24:16.920 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:16.920 DEBUG:boto:Data: 2022-09-03T07:24:16.921 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.922 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.923 DEBUG:boto:Params: {} 2022-09-03T07:24:16.923 DEBUG:boto:Token: None 2022-09-03T07:24:16.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.927 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087a0fa6014a9d669-0063130120-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.928 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.929 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:16.929 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:16.930 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:16.931 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:16.931 DEBUG:boto:Method: GET 2022-09-03T07:24:16.932 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:16.932 DEBUG:boto:Data: 2022-09-03T07:24:16.933 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.934 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.934 DEBUG:boto:Params: {} 2022-09-03T07:24:16.935 DEBUG:boto:Token: None 2022-09-03T07:24:16.935 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/ 2022-09-03T07:24:16.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0= 2022-09-03T07:24:16.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a1pFWFS1eiewxkDeD70SKZTn/T0=', 'Content-Length': '0'} 2022-09-03T07:24:16.940 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000043c08b741d05f79f-0063130120-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.941 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:16.942 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.942 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.943 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.943 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:16.944 DEBUG:boto:Data: 2022-09-03T07:24:16.944 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.945 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.945 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.946 DEBUG:boto:Params: {} 2022-09-03T07:24:16.947 DEBUG:boto:Token: None 2022-09-03T07:24:16.947 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68 2022-09-03T07:24:16.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m0b/I5bGLetU8SaEvVUBS8haX3k= 2022-09-03T07:24:16.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m0b/I5bGLetU8SaEvVUBS8haX3k=', 'Content-Length': '0'} 2022-09-03T07:24:16.950 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf7fd9044704da6b-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.951 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.951 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.952 DEBUG:boto:Method: HEAD 2022-09-03T07:24:16.953 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:16.953 DEBUG:boto:Data: 2022-09-03T07:24:16.954 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.955 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.955 DEBUG:boto:Params: {} 2022-09-03T07:24:16.956 DEBUG:boto:Token: None 2022-09-03T07:24:16.956 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68 2022-09-03T07:24:16.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m0b/I5bGLetU8SaEvVUBS8haX3k= 2022-09-03T07:24:16.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m0b/I5bGLetU8SaEvVUBS8haX3k=', 'Content-Length': '0'} 2022-09-03T07:24:16.959 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013164a3fec708bba-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.960 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:24:16.960 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.961 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.962 DEBUG:boto:Method: GET 2022-09-03T07:24:16.962 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:16.963 DEBUG:boto:Data: 2022-09-03T07:24:16.963 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.964 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:16.964 DEBUG:boto:Port: 8001 2022-09-03T07:24:16.965 DEBUG:boto:Params: {} 2022-09-03T07:24:16.965 DEBUG:boto:Token: None 2022-09-03T07:24:16.966 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68 2022-09-03T07:24:16.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1IIwkOIxK6RBww9iugcfOWM3FS0= 2022-09-03T07:24:16.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1IIwkOIxK6RBww9iugcfOWM3FS0=', 'Content-Length': '0'} 2022-09-03T07:24:16.969 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000531361f6eebd1235-0063130120-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:16.970 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:16.971 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:16.971 DEBUG:boto:Method: GET 2022-09-03T07:24:16.972 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:16.973 DEBUG:boto:Data: 2022-09-03T07:24:16.973 DEBUG:boto:Headers: {} 2022-09-03T07:24:16.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:16.974 DEBUG:boto:Port: 8000 2022-09-03T07:24:16.975 DEBUG:boto:Params: {} 2022-09-03T07:24:16.975 DEBUG:boto:Token: None 2022-09-03T07:24:16.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:16 GMT /avysmw-75/obj68 2022-09-03T07:24:16.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1IIwkOIxK6RBww9iugcfOWM3FS0= 2022-09-03T07:24:16.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1IIwkOIxK6RBww9iugcfOWM3FS0=', 'Content-Length': '0'} 2022-09-03T07:24:16.979 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ea40e33eeb16a2b-0063130120-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.022 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.023 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.024 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:17.025 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:17.026 DEBUG:boto:Method: GET 2022-09-03T07:24:17.026 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:17.027 DEBUG:boto:Data: 2022-09-03T07:24:17.028 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.030 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.031 DEBUG:boto:Params: {} 2022-09-03T07:24:17.032 DEBUG:boto:Token: None 2022-09-03T07:24:17.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.038 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076cf9dbe95bdd6ad-0063130121-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.039 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.040 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.041 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.041 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:17.042 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:17.042 DEBUG:boto:Method: GET 2022-09-03T07:24:17.043 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:17.044 DEBUG:boto:Data: 2022-09-03T07:24:17.045 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.046 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.047 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.048 DEBUG:boto:Params: {} 2022-09-03T07:24:17.048 DEBUG:boto:Token: None 2022-09-03T07:24:17.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.055 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005c5845e15d47fd00-0063130121-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.056 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.057 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:24:17.058 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.059 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.060 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.061 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.062 DEBUG:boto:Method: GET 2022-09-03T07:24:17.063 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.063 DEBUG:boto:Data: 2022-09-03T07:24:17.064 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.065 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.066 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.067 DEBUG:boto:Params: {} 2022-09-03T07:24:17.068 DEBUG:boto:Token: None 2022-09-03T07:24:17.070 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LJvO1gHHwlKZK9vbFWvMh17CmMA= 2022-09-03T07:24:17.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LJvO1gHHwlKZK9vbFWvMh17CmMA=', 'Content-Length': '0'} 2022-09-03T07:24:17.076 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a28cc0310313068-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.118 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.119 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.120 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.121 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.122 DEBUG:boto:Method: GET 2022-09-03T07:24:17.123 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.124 DEBUG:boto:Data: 2022-09-03T07:24:17.125 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.126 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.127 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.127 DEBUG:boto:Params: {} 2022-09-03T07:24:17.128 DEBUG:boto:Token: None 2022-09-03T07:24:17.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LJvO1gHHwlKZK9vbFWvMh17CmMA= 2022-09-03T07:24:17.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LJvO1gHHwlKZK9vbFWvMh17CmMA=', 'Content-Length': '0'} 2022-09-03T07:24:17.133 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003cb0a1dfede8c36-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.134 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.135 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.135 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.136 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.137 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.138 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.138 DEBUG:boto:Data: 2022-09-03T07:24:17.141 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.142 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.143 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.145 DEBUG:boto:Params: {} 2022-09-03T07:24:17.146 DEBUG:boto:Token: None 2022-09-03T07:24:17.148 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I5Qu8NmY2RWpoLaiWdUaNfTzfvA= 2022-09-03T07:24:17.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I5Qu8NmY2RWpoLaiWdUaNfTzfvA=', 'Content-Length': '0'} 2022-09-03T07:24:17.156 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ea64c52d521e043-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.158 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.159 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.161 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.162 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.163 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.164 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.165 DEBUG:boto:Data: 2022-09-03T07:24:17.166 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.168 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.170 DEBUG:boto:Params: {} 2022-09-03T07:24:17.171 DEBUG:boto:Token: None 2022-09-03T07:24:17.172 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I5Qu8NmY2RWpoLaiWdUaNfTzfvA= 2022-09-03T07:24:17.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I5Qu8NmY2RWpoLaiWdUaNfTzfvA=', 'Content-Length': '0'} 2022-09-03T07:24:17.179 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009eb63980201bcefc-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.180 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:24:17.180 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.181 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.182 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.183 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.183 DEBUG:boto:Method: GET 2022-09-03T07:24:17.184 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.185 DEBUG:boto:Data: 2022-09-03T07:24:17.185 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.186 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.187 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.187 DEBUG:boto:Params: {} 2022-09-03T07:24:17.188 DEBUG:boto:Token: None 2022-09-03T07:24:17.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LJvO1gHHwlKZK9vbFWvMh17CmMA= 2022-09-03T07:24:17.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LJvO1gHHwlKZK9vbFWvMh17CmMA=', 'Content-Length': '0'} 2022-09-03T07:24:17.192 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000171190d1a82fac04-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.193 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.194 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.195 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.196 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.196 DEBUG:boto:Method: GET 2022-09-03T07:24:17.197 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.198 DEBUG:boto:Data: 2022-09-03T07:24:17.198 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.199 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.200 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.201 DEBUG:boto:Params: {} 2022-09-03T07:24:17.201 DEBUG:boto:Token: None 2022-09-03T07:24:17.202 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:17.203 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LJvO1gHHwlKZK9vbFWvMh17CmMA= 2022-09-03T07:24:17.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LJvO1gHHwlKZK9vbFWvMh17CmMA=', 'Content-Length': '0'} 2022-09-03T07:24:17.206 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003878d291a6ae516d-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.207 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.208 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.208 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:17.209 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:17.210 DEBUG:boto:Method: GET 2022-09-03T07:24:17.211 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:17.211 DEBUG:boto:Data: 2022-09-03T07:24:17.212 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.213 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.214 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.214 DEBUG:boto:Params: {} 2022-09-03T07:24:17.215 DEBUG:boto:Token: None 2022-09-03T07:24:17.216 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.221 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000054ba515ad57e72cf-0063130121-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.221 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.223 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.223 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.224 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:17.225 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:17.225 DEBUG:boto:Method: GET 2022-09-03T07:24:17.226 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:17.227 DEBUG:boto:Data: 2022-09-03T07:24:17.228 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.228 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.229 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.230 DEBUG:boto:Params: {} 2022-09-03T07:24:17.230 DEBUG:boto:Token: None 2022-09-03T07:24:17.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.236 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000006f23a18f5bfdf00-0063130121-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.282 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.283 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.284 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.285 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.286 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:17.287 DEBUG:boto:Data: 2022-09-03T07:24:17.287 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.289 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.290 DEBUG:boto:Params: {} 2022-09-03T07:24:17.290 DEBUG:boto:Token: None 2022-09-03T07:24:17.291 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69 2022-09-03T07:24:17.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7CO/C5CD0GaPm4/c/94UAqsitI4= 2022-09-03T07:24:17.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7CO/C5CD0GaPm4/c/94UAqsitI4=', 'Content-Length': '0'} 2022-09-03T07:24:17.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b0891aab304f9cf-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.296 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.296 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.297 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.297 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:17.298 DEBUG:boto:Data: 2022-09-03T07:24:17.298 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.299 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.300 DEBUG:boto:Params: {} 2022-09-03T07:24:17.301 DEBUG:boto:Token: None 2022-09-03T07:24:17.301 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69 2022-09-03T07:24:17.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7CO/C5CD0GaPm4/c/94UAqsitI4= 2022-09-03T07:24:17.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7CO/C5CD0GaPm4/c/94UAqsitI4=', 'Content-Length': '0'} 2022-09-03T07:24:17.304 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001466981fe580afd3-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.305 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:24:17.306 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.306 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.307 DEBUG:boto:Method: GET 2022-09-03T07:24:17.307 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:17.308 DEBUG:boto:Data: 2022-09-03T07:24:17.308 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.310 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.310 DEBUG:boto:Params: {} 2022-09-03T07:24:17.311 DEBUG:boto:Token: None 2022-09-03T07:24:17.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69 2022-09-03T07:24:17.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VKZwsmIhzHqj7dkRfn/M633KRSY= 2022-09-03T07:24:17.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VKZwsmIhzHqj7dkRfn/M633KRSY=', 'Content-Length': '0'} 2022-09-03T07:24:17.315 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de5b84069b073398-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.316 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:17.316 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:17.317 DEBUG:boto:Method: GET 2022-09-03T07:24:17.317 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:17.318 DEBUG:boto:Data: 2022-09-03T07:24:17.318 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.319 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.320 DEBUG:boto:Params: {} 2022-09-03T07:24:17.321 DEBUG:boto:Token: None 2022-09-03T07:24:17.321 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj69 2022-09-03T07:24:17.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VKZwsmIhzHqj7dkRfn/M633KRSY= 2022-09-03T07:24:17.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VKZwsmIhzHqj7dkRfn/M633KRSY=', 'Content-Length': '0'} 2022-09-03T07:24:17.325 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bdc257744cf0b502-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.325 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.326 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.326 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:17.327 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:17.328 DEBUG:boto:Method: GET 2022-09-03T07:24:17.328 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:17.329 DEBUG:boto:Data: 2022-09-03T07:24:17.329 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.330 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.331 DEBUG:boto:Params: {} 2022-09-03T07:24:17.331 DEBUG:boto:Token: None 2022-09-03T07:24:17.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.333 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.336 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e3765f6b5314f74-0063130121-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.378 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.379 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.380 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.380 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:17.381 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:17.381 DEBUG:boto:Method: GET 2022-09-03T07:24:17.382 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:17.382 DEBUG:boto:Data: 2022-09-03T07:24:17.383 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.384 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.385 DEBUG:boto:Params: {} 2022-09-03T07:24:17.385 DEBUG:boto:Token: None 2022-09-03T07:24:17.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.390 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000019d994834fe494ad-0063130121-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.391 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.392 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:24:17.392 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.393 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.393 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.394 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.395 DEBUG:boto:Method: GET 2022-09-03T07:24:17.395 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.396 DEBUG:boto:Data: 2022-09-03T07:24:17.396 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.397 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.397 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.398 DEBUG:boto:Params: {} 2022-09-03T07:24:17.398 DEBUG:boto:Token: None 2022-09-03T07:24:17.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0J8ClO2j/CpL5HQFn+Ijtp8ba/0= 2022-09-03T07:24:17.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0J8ClO2j/CpL5HQFn+Ijtp8ba/0=', 'Content-Length': '0'} 2022-09-03T07:24:17.402 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e8d56d03847e0e79-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.403 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.404 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.404 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.405 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.405 DEBUG:boto:Method: GET 2022-09-03T07:24:17.406 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.406 DEBUG:boto:Data: 2022-09-03T07:24:17.407 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.408 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.409 DEBUG:boto:Params: {} 2022-09-03T07:24:17.409 DEBUG:boto:Token: None 2022-09-03T07:24:17.410 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0J8ClO2j/CpL5HQFn+Ijtp8ba/0= 2022-09-03T07:24:17.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0J8ClO2j/CpL5HQFn+Ijtp8ba/0=', 'Content-Length': '0'} 2022-09-03T07:24:17.413 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbdef799e72b3501-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.414 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.415 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.415 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.416 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.416 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.417 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.417 DEBUG:boto:Data: 2022-09-03T07:24:17.418 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.418 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.419 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.419 DEBUG:boto:Params: {} 2022-09-03T07:24:17.420 DEBUG:boto:Token: None 2022-09-03T07:24:17.420 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DY8wnFyiYPLYhPHJ+YfUIDyuweQ= 2022-09-03T07:24:17.421 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DY8wnFyiYPLYhPHJ+YfUIDyuweQ=', 'Content-Length': '0'} 2022-09-03T07:24:17.423 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6652934bd5774b9-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.424 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.424 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.425 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.426 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.426 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.427 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.427 DEBUG:boto:Data: 2022-09-03T07:24:17.428 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.429 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.429 DEBUG:boto:Params: {} 2022-09-03T07:24:17.430 DEBUG:boto:Token: None 2022-09-03T07:24:17.431 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DY8wnFyiYPLYhPHJ+YfUIDyuweQ= 2022-09-03T07:24:17.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DY8wnFyiYPLYhPHJ+YfUIDyuweQ=', 'Content-Length': '0'} 2022-09-03T07:24:17.434 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071475ffa44c0aa59-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.434 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:24:17.435 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.435 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.436 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.436 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.437 DEBUG:boto:Method: GET 2022-09-03T07:24:17.437 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.438 DEBUG:boto:Data: 2022-09-03T07:24:17.438 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.440 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.440 DEBUG:boto:Params: {} 2022-09-03T07:24:17.441 DEBUG:boto:Token: None 2022-09-03T07:24:17.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0J8ClO2j/CpL5HQFn+Ijtp8ba/0= 2022-09-03T07:24:17.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0J8ClO2j/CpL5HQFn+Ijtp8ba/0=', 'Content-Length': '0'} 2022-09-03T07:24:17.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3e0dad92effa722-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.445 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.446 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.446 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.447 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.447 DEBUG:boto:Method: GET 2022-09-03T07:24:17.448 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.448 DEBUG:boto:Data: 2022-09-03T07:24:17.449 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.450 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.451 DEBUG:boto:Params: {} 2022-09-03T07:24:17.451 DEBUG:boto:Token: None 2022-09-03T07:24:17.452 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:17.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0J8ClO2j/CpL5HQFn+Ijtp8ba/0= 2022-09-03T07:24:17.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0J8ClO2j/CpL5HQFn+Ijtp8ba/0=', 'Content-Length': '0'} 2022-09-03T07:24:17.455 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000936808a358808c81-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.456 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.456 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.457 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:17.457 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:17.458 DEBUG:boto:Method: GET 2022-09-03T07:24:17.459 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:17.459 DEBUG:boto:Data: 2022-09-03T07:24:17.460 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.461 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.461 DEBUG:boto:Params: {} 2022-09-03T07:24:17.462 DEBUG:boto:Token: None 2022-09-03T07:24:17.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.467 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006cc76af9fd649216-0063130121-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.468 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.468 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.469 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.470 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:17.470 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:17.471 DEBUG:boto:Method: GET 2022-09-03T07:24:17.471 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:17.472 DEBUG:boto:Data: 2022-09-03T07:24:17.473 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.474 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.474 DEBUG:boto:Params: {} 2022-09-03T07:24:17.475 DEBUG:boto:Token: None 2022-09-03T07:24:17.475 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.480 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000756cf18fb11c4451-0063130121-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.481 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.481 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.482 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.483 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.483 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:17.483 DEBUG:boto:Data: 2022-09-03T07:24:17.484 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.484 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.485 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.485 DEBUG:boto:Params: {} 2022-09-03T07:24:17.486 DEBUG:boto:Token: None 2022-09-03T07:24:17.487 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76 2022-09-03T07:24:17.487 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V5xwbnkOMHpDS0Pa8IevyLJeAv8= 2022-09-03T07:24:17.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V5xwbnkOMHpDS0Pa8IevyLJeAv8=', 'Content-Length': '0'} 2022-09-03T07:24:17.490 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a47f44fead3f8eb-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.491 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.491 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.492 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.492 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:17.493 DEBUG:boto:Data: 2022-09-03T07:24:17.493 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.494 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.494 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.495 DEBUG:boto:Params: {} 2022-09-03T07:24:17.496 DEBUG:boto:Token: None 2022-09-03T07:24:17.496 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76 2022-09-03T07:24:17.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V5xwbnkOMHpDS0Pa8IevyLJeAv8= 2022-09-03T07:24:17.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V5xwbnkOMHpDS0Pa8IevyLJeAv8=', 'Content-Length': '0'} 2022-09-03T07:24:17.499 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab153e1cd2ef4669-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.500 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:24:17.500 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.501 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.502 DEBUG:boto:Method: GET 2022-09-03T07:24:17.502 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:17.503 DEBUG:boto:Data: 2022-09-03T07:24:17.503 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.504 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.504 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.505 DEBUG:boto:Params: {} 2022-09-03T07:24:17.506 DEBUG:boto:Token: None 2022-09-03T07:24:17.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76 2022-09-03T07:24:17.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DqrT+vxMPtQIJhDIMes7St1HhN0= 2022-09-03T07:24:17.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DqrT+vxMPtQIJhDIMes7St1HhN0=', 'Content-Length': '0'} 2022-09-03T07:24:17.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ac131af9846d090-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.562 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:17.564 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:17.565 DEBUG:boto:Method: GET 2022-09-03T07:24:17.566 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:17.567 DEBUG:boto:Data: 2022-09-03T07:24:17.568 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.569 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.570 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.571 DEBUG:boto:Params: {} 2022-09-03T07:24:17.572 DEBUG:boto:Token: None 2022-09-03T07:24:17.572 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj76 2022-09-03T07:24:17.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DqrT+vxMPtQIJhDIMes7St1HhN0= 2022-09-03T07:24:17.574 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DqrT+vxMPtQIJhDIMes7St1HhN0=', 'Content-Length': '0'} 2022-09-03T07:24:17.576 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc082a87c823557f-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.577 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.578 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.578 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:17.579 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:17.580 DEBUG:boto:Method: GET 2022-09-03T07:24:17.580 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:17.581 DEBUG:boto:Data: 2022-09-03T07:24:17.582 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.583 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.584 DEBUG:boto:Params: {} 2022-09-03T07:24:17.585 DEBUG:boto:Token: None 2022-09-03T07:24:17.586 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.588 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.590 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000bbd2f1b33cabc7e-0063130121-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.591 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.592 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.593 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.594 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:17.594 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:17.595 DEBUG:boto:Method: GET 2022-09-03T07:24:17.595 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:17.596 DEBUG:boto:Data: 2022-09-03T07:24:17.597 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.598 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.598 DEBUG:boto:Params: {} 2022-09-03T07:24:17.599 DEBUG:boto:Token: None 2022-09-03T07:24:17.600 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.604 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007bdcd961511581fa-0063130121-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.605 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.606 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:24:17.606 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.607 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.607 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.608 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.608 DEBUG:boto:Method: GET 2022-09-03T07:24:17.609 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.610 DEBUG:boto:Data: 2022-09-03T07:24:17.611 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.612 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.613 DEBUG:boto:Params: {} 2022-09-03T07:24:17.613 DEBUG:boto:Token: None 2022-09-03T07:24:17.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KzCIFH4EMN+jO56CgLa2DVvKYUI= 2022-09-03T07:24:17.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KzCIFH4EMN+jO56CgLa2DVvKYUI=', 'Content-Length': '0'} 2022-09-03T07:24:17.617 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070c439bb82455e9c-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.618 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.619 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.619 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.620 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.620 DEBUG:boto:Method: GET 2022-09-03T07:24:17.621 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.621 DEBUG:boto:Data: 2022-09-03T07:24:17.622 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.623 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.624 DEBUG:boto:Params: {} 2022-09-03T07:24:17.624 DEBUG:boto:Token: None 2022-09-03T07:24:17.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KzCIFH4EMN+jO56CgLa2DVvKYUI= 2022-09-03T07:24:17.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KzCIFH4EMN+jO56CgLa2DVvKYUI=', 'Content-Length': '0'} 2022-09-03T07:24:17.628 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d989148b8726c272-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.629 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.630 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.630 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.631 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.631 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.632 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.632 DEBUG:boto:Data: 2022-09-03T07:24:17.633 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.633 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.634 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.635 DEBUG:boto:Params: {} 2022-09-03T07:24:17.635 DEBUG:boto:Token: None 2022-09-03T07:24:17.636 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBF8VFCjM/v/kkbcRS/Tgu1+KTg= 2022-09-03T07:24:17.637 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBF8VFCjM/v/kkbcRS/Tgu1+KTg=', 'Content-Length': '0'} 2022-09-03T07:24:17.639 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010806cf67ce1f554-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.640 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.640 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.641 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.642 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.642 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.643 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.643 DEBUG:boto:Data: 2022-09-03T07:24:17.644 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.645 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.645 DEBUG:boto:Params: {} 2022-09-03T07:24:17.646 DEBUG:boto:Token: None 2022-09-03T07:24:17.647 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBF8VFCjM/v/kkbcRS/Tgu1+KTg= 2022-09-03T07:24:17.648 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBF8VFCjM/v/kkbcRS/Tgu1+KTg=', 'Content-Length': '0'} 2022-09-03T07:24:17.650 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c078d4f277e8b9dd-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.651 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:24:17.652 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.652 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.653 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.653 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.654 DEBUG:boto:Method: GET 2022-09-03T07:24:17.654 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.655 DEBUG:boto:Data: 2022-09-03T07:24:17.655 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.656 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.656 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.657 DEBUG:boto:Params: {} 2022-09-03T07:24:17.658 DEBUG:boto:Token: None 2022-09-03T07:24:17.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KzCIFH4EMN+jO56CgLa2DVvKYUI= 2022-09-03T07:24:17.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KzCIFH4EMN+jO56CgLa2DVvKYUI=', 'Content-Length': '0'} 2022-09-03T07:24:17.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004dff5a0036e26e6-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.662 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.663 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.663 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.664 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.664 DEBUG:boto:Method: GET 2022-09-03T07:24:17.665 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.666 DEBUG:boto:Data: 2022-09-03T07:24:17.666 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.667 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.667 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.668 DEBUG:boto:Params: {} 2022-09-03T07:24:17.669 DEBUG:boto:Token: None 2022-09-03T07:24:17.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:17.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KzCIFH4EMN+jO56CgLa2DVvKYUI= 2022-09-03T07:24:17.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KzCIFH4EMN+jO56CgLa2DVvKYUI=', 'Content-Length': '0'} 2022-09-03T07:24:17.672 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7d4b5edbc98ceac-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.673 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.674 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.675 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:17.675 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:17.676 DEBUG:boto:Method: GET 2022-09-03T07:24:17.676 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:17.677 DEBUG:boto:Data: 2022-09-03T07:24:17.677 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.679 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.679 DEBUG:boto:Params: {} 2022-09-03T07:24:17.680 DEBUG:boto:Token: None 2022-09-03T07:24:17.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.684 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3058d91f70f569c-0063130121-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.685 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.685 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.686 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.687 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:17.687 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:17.688 DEBUG:boto:Method: GET 2022-09-03T07:24:17.688 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:17.689 DEBUG:boto:Data: 2022-09-03T07:24:17.689 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.690 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.690 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.691 DEBUG:boto:Params: {} 2022-09-03T07:24:17.692 DEBUG:boto:Token: None 2022-09-03T07:24:17.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.693 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.697 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000f9e6d15959dda62-0063130121-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.697 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.698 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.699 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.699 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.700 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:17.700 DEBUG:boto:Data: 2022-09-03T07:24:17.701 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.702 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.702 DEBUG:boto:Params: {} 2022-09-03T07:24:17.703 DEBUG:boto:Token: None 2022-09-03T07:24:17.704 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85 2022-09-03T07:24:17.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yb4a/10wsXr+8oTo/QozMb5/GlA= 2022-09-03T07:24:17.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yb4a/10wsXr+8oTo/QozMb5/GlA=', 'Content-Length': '0'} 2022-09-03T07:24:17.706 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096c891283844156a-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.707 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.708 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.708 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.709 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:17.709 DEBUG:boto:Data: 2022-09-03T07:24:17.710 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.710 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.711 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.711 DEBUG:boto:Params: {} 2022-09-03T07:24:17.712 DEBUG:boto:Token: None 2022-09-03T07:24:17.712 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85 2022-09-03T07:24:17.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yb4a/10wsXr+8oTo/QozMb5/GlA= 2022-09-03T07:24:17.714 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yb4a/10wsXr+8oTo/QozMb5/GlA=', 'Content-Length': '0'} 2022-09-03T07:24:17.715 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b658936a090ca2de-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.716 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:24:17.717 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.717 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.718 DEBUG:boto:Method: GET 2022-09-03T07:24:17.718 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:17.719 DEBUG:boto:Data: 2022-09-03T07:24:17.720 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.720 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.721 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.721 DEBUG:boto:Params: {} 2022-09-03T07:24:17.722 DEBUG:boto:Token: None 2022-09-03T07:24:17.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85 2022-09-03T07:24:17.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AsQ6d7jq/WQmKok/O0ioNX1i2fc= 2022-09-03T07:24:17.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AsQ6d7jq/WQmKok/O0ioNX1i2fc=', 'Content-Length': '0'} 2022-09-03T07:24:17.726 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c49028cd16900c5e-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.766 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:17.767 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:17.767 DEBUG:boto:Method: GET 2022-09-03T07:24:17.768 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:17.768 DEBUG:boto:Data: 2022-09-03T07:24:17.769 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.770 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.771 DEBUG:boto:Params: {} 2022-09-03T07:24:17.771 DEBUG:boto:Token: None 2022-09-03T07:24:17.772 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj85 2022-09-03T07:24:17.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AsQ6d7jq/WQmKok/O0ioNX1i2fc= 2022-09-03T07:24:17.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AsQ6d7jq/WQmKok/O0ioNX1i2fc=', 'Content-Length': '0'} 2022-09-03T07:24:17.777 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092bdcb8d8a08a176-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.777 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.778 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.779 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:17.779 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:17.780 DEBUG:boto:Method: GET 2022-09-03T07:24:17.780 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:17.781 DEBUG:boto:Data: 2022-09-03T07:24:17.781 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.782 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.782 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.783 DEBUG:boto:Params: {} 2022-09-03T07:24:17.783 DEBUG:boto:Token: None 2022-09-03T07:24:17.784 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.788 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000e4655289de8a77f-0063130121-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.789 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.790 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.791 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.792 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:17.792 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:17.793 DEBUG:boto:Method: GET 2022-09-03T07:24:17.793 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:17.794 DEBUG:boto:Data: 2022-09-03T07:24:17.794 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.796 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.796 DEBUG:boto:Params: {} 2022-09-03T07:24:17.797 DEBUG:boto:Token: None 2022-09-03T07:24:17.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.802 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000729f57de2c07bdfc-0063130121-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.803 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.804 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:24:17.804 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.805 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.806 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.807 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.808 DEBUG:boto:Method: GET 2022-09-03T07:24:17.808 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.809 DEBUG:boto:Data: 2022-09-03T07:24:17.809 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.810 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.811 DEBUG:boto:Params: {} 2022-09-03T07:24:17.812 DEBUG:boto:Token: None 2022-09-03T07:24:17.812 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jiXHvPhxzfj2PHOt4Oukn4YYLeI= 2022-09-03T07:24:17.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jiXHvPhxzfj2PHOt4Oukn4YYLeI=', 'Content-Length': '0'} 2022-09-03T07:24:17.815 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000077d2b615f617a782-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.816 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.817 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.817 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.818 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.819 DEBUG:boto:Method: GET 2022-09-03T07:24:17.819 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.820 DEBUG:boto:Data: 2022-09-03T07:24:17.820 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.821 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.821 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.822 DEBUG:boto:Params: {} 2022-09-03T07:24:17.823 DEBUG:boto:Token: None 2022-09-03T07:24:17.823 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jiXHvPhxzfj2PHOt4Oukn4YYLeI= 2022-09-03T07:24:17.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jiXHvPhxzfj2PHOt4Oukn4YYLeI=', 'Content-Length': '0'} 2022-09-03T07:24:17.826 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e49644b5789c7992-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.827 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.828 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.828 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.829 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.829 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.830 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.831 DEBUG:boto:Data: 2022-09-03T07:24:17.831 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.832 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.833 DEBUG:boto:Params: {} 2022-09-03T07:24:17.833 DEBUG:boto:Token: None 2022-09-03T07:24:17.834 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rEkly0ueCUK1RnjGNUZsjneZ1VM= 2022-09-03T07:24:17.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rEkly0ueCUK1RnjGNUZsjneZ1VM=', 'Content-Length': '0'} 2022-09-03T07:24:17.838 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b86e10252216a5b5-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.839 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.840 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.840 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.841 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.841 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.842 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.842 DEBUG:boto:Data: 2022-09-03T07:24:17.843 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.843 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.844 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.844 DEBUG:boto:Params: {} 2022-09-03T07:24:17.845 DEBUG:boto:Token: None 2022-09-03T07:24:17.846 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.847 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rEkly0ueCUK1RnjGNUZsjneZ1VM= 2022-09-03T07:24:17.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rEkly0ueCUK1RnjGNUZsjneZ1VM=', 'Content-Length': '0'} 2022-09-03T07:24:17.849 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000129761c456ca2505-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.850 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:24:17.851 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.851 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.852 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.852 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.853 DEBUG:boto:Method: GET 2022-09-03T07:24:17.854 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.854 DEBUG:boto:Data: 2022-09-03T07:24:17.855 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.856 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.857 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.858 DEBUG:boto:Params: {} 2022-09-03T07:24:17.858 DEBUG:boto:Token: None 2022-09-03T07:24:17.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jiXHvPhxzfj2PHOt4Oukn4YYLeI= 2022-09-03T07:24:17.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jiXHvPhxzfj2PHOt4Oukn4YYLeI=', 'Content-Length': '0'} 2022-09-03T07:24:17.864 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006c9bd6a1a3e266d4-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.865 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.866 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.867 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.868 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.868 DEBUG:boto:Method: GET 2022-09-03T07:24:17.869 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.870 DEBUG:boto:Data: 2022-09-03T07:24:17.871 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.873 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.873 DEBUG:boto:Params: {} 2022-09-03T07:24:17.875 DEBUG:boto:Token: None 2022-09-03T07:24:17.875 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:17.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jiXHvPhxzfj2PHOt4Oukn4YYLeI= 2022-09-03T07:24:17.877 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jiXHvPhxzfj2PHOt4Oukn4YYLeI=', 'Content-Length': '0'} 2022-09-03T07:24:17.879 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000bf7275cee9645ce-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.922 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.923 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.923 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:17.924 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:17.925 DEBUG:boto:Method: GET 2022-09-03T07:24:17.925 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:17.926 DEBUG:boto:Data: 2022-09-03T07:24:17.927 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.928 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.929 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.929 DEBUG:boto:Params: {} 2022-09-03T07:24:17.930 DEBUG:boto:Token: None 2022-09-03T07:24:17.930 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.931 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.932 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.935 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e59419cd260719e1-0063130121-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.935 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.936 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:17.937 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:17.937 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:17.938 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:17.939 DEBUG:boto:Method: GET 2022-09-03T07:24:17.939 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:17.940 DEBUG:boto:Data: 2022-09-03T07:24:17.940 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.941 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.941 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.942 DEBUG:boto:Params: {} 2022-09-03T07:24:17.943 DEBUG:boto:Token: None 2022-09-03T07:24:17.943 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/ 2022-09-03T07:24:17.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU= 2022-09-03T07:24:17.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PzasvoldXBNS2+2W7pwH74a5meU=', 'Content-Length': '0'} 2022-09-03T07:24:17.947 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090c8fd86b3d77c6e-0063130121-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.948 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:17.949 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.950 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.951 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.952 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:17.952 DEBUG:boto:Data: 2022-09-03T07:24:17.954 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.955 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.956 DEBUG:boto:Params: {} 2022-09-03T07:24:17.957 DEBUG:boto:Token: None 2022-09-03T07:24:17.958 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92 2022-09-03T07:24:17.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z55cZ2xfr+fH2QS6Y/Sa1E6N/ns= 2022-09-03T07:24:17.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z55cZ2xfr+fH2QS6Y/Sa1E6N/ns=', 'Content-Length': '0'} 2022-09-03T07:24:17.962 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b715215bbc308194-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.962 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.963 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.964 DEBUG:boto:Method: HEAD 2022-09-03T07:24:17.964 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:17.965 DEBUG:boto:Data: 2022-09-03T07:24:17.965 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.966 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.967 DEBUG:boto:Params: {} 2022-09-03T07:24:17.967 DEBUG:boto:Token: None 2022-09-03T07:24:17.968 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92 2022-09-03T07:24:17.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z55cZ2xfr+fH2QS6Y/Sa1E6N/ns= 2022-09-03T07:24:17.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z55cZ2xfr+fH2QS6Y/Sa1E6N/ns=', 'Content-Length': '0'} 2022-09-03T07:24:17.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e116c519b774bee3-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.973 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:24:17.974 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.974 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.975 DEBUG:boto:Method: GET 2022-09-03T07:24:17.976 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:17.977 DEBUG:boto:Data: 2022-09-03T07:24:17.978 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:17.980 DEBUG:boto:Port: 8001 2022-09-03T07:24:17.980 DEBUG:boto:Params: {} 2022-09-03T07:24:17.981 DEBUG:boto:Token: None 2022-09-03T07:24:17.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92 2022-09-03T07:24:17.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hwZiXAbvsePGiTUOiZVt1njKjfU= 2022-09-03T07:24:17.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hwZiXAbvsePGiTUOiZVt1njKjfU=', 'Content-Length': '0'} 2022-09-03T07:24:17.985 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000094e6054cbf3611e-0063130121-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:17.986 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:17.987 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:17.987 DEBUG:boto:Method: GET 2022-09-03T07:24:17.988 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:17.989 DEBUG:boto:Data: 2022-09-03T07:24:17.990 DEBUG:boto:Headers: {} 2022-09-03T07:24:17.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:17.992 DEBUG:boto:Port: 8000 2022-09-03T07:24:17.992 DEBUG:boto:Params: {} 2022-09-03T07:24:17.993 DEBUG:boto:Token: None 2022-09-03T07:24:17.994 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:17 GMT /avysmw-75/obj92 2022-09-03T07:24:17.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hwZiXAbvsePGiTUOiZVt1njKjfU= 2022-09-03T07:24:17.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hwZiXAbvsePGiTUOiZVt1njKjfU=', 'Content-Length': '0'} 2022-09-03T07:24:17.999 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045ac1522b2901188-0063130121-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.000 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.001 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.001 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:18.002 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:18.003 DEBUG:boto:Method: GET 2022-09-03T07:24:18.003 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:18.004 DEBUG:boto:Data: 2022-09-03T07:24:18.004 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.005 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.005 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.006 DEBUG:boto:Params: {} 2022-09-03T07:24:18.007 DEBUG:boto:Token: None 2022-09-03T07:24:18.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.012 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c36c5708edef8ca-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.012 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.013 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.014 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.014 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:18.015 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:18.016 DEBUG:boto:Method: GET 2022-09-03T07:24:18.017 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:18.018 DEBUG:boto:Data: 2022-09-03T07:24:18.018 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.019 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:18.019 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.020 DEBUG:boto:Params: {} 2022-09-03T07:24:18.021 DEBUG:boto:Token: None 2022-09-03T07:24:18.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.026 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f745d80bb7c2673e-0063130122-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.070 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.072 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:24:18.073 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.073 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.074 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.075 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.076 DEBUG:boto:Method: GET 2022-09-03T07:24:18.077 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.078 DEBUG:boto:Data: 2022-09-03T07:24:18.079 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.081 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.081 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.082 DEBUG:boto:Params: {} 2022-09-03T07:24:18.083 DEBUG:boto:Token: None 2022-09-03T07:24:18.084 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J35i0RHh53gDJ+sgQ90GgTMprx8= 2022-09-03T07:24:18.086 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J35i0RHh53gDJ+sgQ90GgTMprx8=', 'Content-Length': '0'} 2022-09-03T07:24:18.089 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b129fa1bb121215c-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.091 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.092 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.093 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.094 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.095 DEBUG:boto:Method: GET 2022-09-03T07:24:18.096 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.097 DEBUG:boto:Data: 2022-09-03T07:24:18.098 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:18.100 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.101 DEBUG:boto:Params: {} 2022-09-03T07:24:18.103 DEBUG:boto:Token: None 2022-09-03T07:24:18.104 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J35i0RHh53gDJ+sgQ90GgTMprx8= 2022-09-03T07:24:18.106 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J35i0RHh53gDJ+sgQ90GgTMprx8=', 'Content-Length': '0'} 2022-09-03T07:24:18.109 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bb1f8d103c510566-0063130122-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.110 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.111 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.112 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.113 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.114 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.114 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.115 DEBUG:boto:Data: 2022-09-03T07:24:18.115 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.117 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.117 DEBUG:boto:Params: {} 2022-09-03T07:24:18.118 DEBUG:boto:Token: None 2022-09-03T07:24:18.118 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TdMLr/VFdHVWA7Z229tUiU5ju20= 2022-09-03T07:24:18.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TdMLr/VFdHVWA7Z229tUiU5ju20=', 'Content-Length': '0'} 2022-09-03T07:24:18.125 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007eee3dda1a867b1a-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.127 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.128 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.129 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.129 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.130 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.130 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.131 DEBUG:boto:Data: 2022-09-03T07:24:18.131 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:18.133 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.133 DEBUG:boto:Params: {} 2022-09-03T07:24:18.134 DEBUG:boto:Token: None 2022-09-03T07:24:18.135 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TdMLr/VFdHVWA7Z229tUiU5ju20= 2022-09-03T07:24:18.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TdMLr/VFdHVWA7Z229tUiU5ju20=', 'Content-Length': '0'} 2022-09-03T07:24:18.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ff86b2b1ca80895-0063130122-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.139 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:24:18.139 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.140 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.140 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.141 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.141 DEBUG:boto:Method: GET 2022-09-03T07:24:18.142 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.143 DEBUG:boto:Data: 2022-09-03T07:24:18.144 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.145 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.146 DEBUG:boto:Params: {} 2022-09-03T07:24:18.146 DEBUG:boto:Token: None 2022-09-03T07:24:18.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J35i0RHh53gDJ+sgQ90GgTMprx8= 2022-09-03T07:24:18.148 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J35i0RHh53gDJ+sgQ90GgTMprx8=', 'Content-Length': '0'} 2022-09-03T07:24:18.150 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079f43594b8048168-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.152 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.152 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.153 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.153 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.154 DEBUG:boto:Method: GET 2022-09-03T07:24:18.155 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.156 DEBUG:boto:Data: 2022-09-03T07:24:18.157 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:18.159 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.159 DEBUG:boto:Params: {} 2022-09-03T07:24:18.160 DEBUG:boto:Token: None 2022-09-03T07:24:18.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:18.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J35i0RHh53gDJ+sgQ90GgTMprx8= 2022-09-03T07:24:18.163 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J35i0RHh53gDJ+sgQ90GgTMprx8=', 'Content-Length': '0'} 2022-09-03T07:24:18.165 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002665915892bcf7c3-0063130122-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.166 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.167 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.167 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:18.168 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:18.168 DEBUG:boto:Method: GET 2022-09-03T07:24:18.169 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:18.169 DEBUG:boto:Data: 2022-09-03T07:24:18.170 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.171 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.172 DEBUG:boto:Params: {} 2022-09-03T07:24:18.172 DEBUG:boto:Token: None 2022-09-03T07:24:18.173 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.177 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ef62c32d3c7039b0-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.218 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.219 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.220 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.220 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:18.221 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:18.221 DEBUG:boto:Method: GET 2022-09-03T07:24:18.222 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:18.222 DEBUG:boto:Data: 2022-09-03T07:24:18.223 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.223 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:18.224 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.224 DEBUG:boto:Params: {} 2022-09-03T07:24:18.225 DEBUG:boto:Token: None 2022-09-03T07:24:18.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.230 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b3e511afb846f297-0063130122-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.231 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.232 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.232 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.233 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.233 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:18.234 DEBUG:boto:Data: 2022-09-03T07:24:18.234 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.235 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.236 DEBUG:boto:Params: {} 2022-09-03T07:24:18.236 DEBUG:boto:Token: None 2022-09-03T07:24:18.237 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95 2022-09-03T07:24:18.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Th1fyZo39pT1LcdH9SmstCuTG1Y= 2022-09-03T07:24:18.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Th1fyZo39pT1LcdH9SmstCuTG1Y=', 'Content-Length': '0'} 2022-09-03T07:24:18.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000095899aa6a3ec4c8e-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.240 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.241 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.241 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.242 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:18.242 DEBUG:boto:Data: 2022-09-03T07:24:18.243 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:18.244 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.245 DEBUG:boto:Params: {} 2022-09-03T07:24:18.245 DEBUG:boto:Token: None 2022-09-03T07:24:18.246 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95 2022-09-03T07:24:18.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Th1fyZo39pT1LcdH9SmstCuTG1Y= 2022-09-03T07:24:18.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Th1fyZo39pT1LcdH9SmstCuTG1Y=', 'Content-Length': '0'} 2022-09-03T07:24:18.249 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004eb6970d3014a817-0063130122-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.249 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:24:18.250 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.250 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.251 DEBUG:boto:Method: GET 2022-09-03T07:24:18.251 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:18.252 DEBUG:boto:Data: 2022-09-03T07:24:18.253 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.254 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.254 DEBUG:boto:Params: {} 2022-09-03T07:24:18.255 DEBUG:boto:Token: None 2022-09-03T07:24:18.255 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95 2022-09-03T07:24:18.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:URBVfzfC8+3vL+dghsvCAM/enwY= 2022-09-03T07:24:18.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:URBVfzfC8+3vL+dghsvCAM/enwY=', 'Content-Length': '0'} 2022-09-03T07:24:18.258 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027d9ff049283b0f9-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.259 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:18.260 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:18.260 DEBUG:boto:Method: GET 2022-09-03T07:24:18.261 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:18.262 DEBUG:boto:Data: 2022-09-03T07:24:18.262 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:18.263 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.264 DEBUG:boto:Params: {} 2022-09-03T07:24:18.265 DEBUG:boto:Token: None 2022-09-03T07:24:18.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj95 2022-09-03T07:24:18.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:URBVfzfC8+3vL+dghsvCAM/enwY= 2022-09-03T07:24:18.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:URBVfzfC8+3vL+dghsvCAM/enwY=', 'Content-Length': '0'} 2022-09-03T07:24:18.269 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f4b3486479ec699-0063130122-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.269 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.270 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.271 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:18.271 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:18.272 DEBUG:boto:Method: GET 2022-09-03T07:24:18.272 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:18.273 DEBUG:boto:Data: 2022-09-03T07:24:18.273 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.275 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.275 DEBUG:boto:Params: {} 2022-09-03T07:24:18.276 DEBUG:boto:Token: None 2022-09-03T07:24:18.276 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.281 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e4ebb174c7fde80-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.281 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.282 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.283 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.283 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:18.284 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:18.284 DEBUG:boto:Method: GET 2022-09-03T07:24:18.285 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:18.285 DEBUG:boto:Data: 2022-09-03T07:24:18.286 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.286 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:18.287 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.288 DEBUG:boto:Params: {} 2022-09-03T07:24:18.288 DEBUG:boto:Token: None 2022-09-03T07:24:18.289 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.290 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.294 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da1de852da8b1136-0063130122-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.295 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.296 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:24:18.297 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:24:18.297 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.298 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.299 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.300 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:18.300 DEBUG:boto:Data: 2022-09-03T07:24:18.301 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.302 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.303 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.304 DEBUG:boto:Params: {} 2022-09-03T07:24:18.305 DEBUG:boto:Token: None 2022-09-03T07:24:18.306 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E4U1/pFn2iq0OB2IrW5oooHE04A= 2022-09-03T07:24:18.308 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E4U1/pFn2iq0OB2IrW5oooHE04A=', 'Content-Length': '0'} 2022-09-03T07:24:18.311 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '20'), ('X-RGW-Bytes-Used', '60'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000971dba0b64ce105f-0063130122-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.312 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.313 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.314 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.314 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:18.315 DEBUG:boto:Data: 2022-09-03T07:24:18.315 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.316 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.317 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.318 DEBUG:boto:Params: {} 2022-09-03T07:24:18.319 DEBUG:boto:Token: None 2022-09-03T07:24:18.320 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E4U1/pFn2iq0OB2IrW5oooHE04A= 2022-09-03T07:24:18.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E4U1/pFn2iq0OB2IrW5oooHE04A=', 'Content-Length': '0'} 2022-09-03T07:24:18.325 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '20'), ('X-RGW-Bytes-Used', '60'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000003e853b183fc4b77-0063130122-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.326 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:24:18.327 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.328 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.329 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.330 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.331 DEBUG:boto:Method: GET 2022-09-03T07:24:18.331 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.332 DEBUG:boto:Data: 2022-09-03T07:24:18.332 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.333 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.333 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.334 DEBUG:boto:Params: {} 2022-09-03T07:24:18.335 DEBUG:boto:Token: None 2022-09-03T07:24:18.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/?versions 2022-09-03T07:24:18.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m1O+HfsytwmchK23BPy1+KsfeUA= 2022-09-03T07:24:18.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m1O+HfsytwmchK23BPy1+KsfeUA=', 'Content-Length': '0'} 2022-09-03T07:24:18.339 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004da068d90606aff2-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.340 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.342 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:24:18.343 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:24:18.344 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:24:18.345 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:24:18.346 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:24:18.346 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:24:18.347 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:24:18.348 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:24:18.349 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:24:18.350 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:24:18.351 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:24:18.352 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:24:18.353 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:24:18.354 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:24:18.355 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:24:18.356 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:24:18.356 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:24:18.357 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:24:18.357 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:24:18.357 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:24:18.358 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:24:18.358 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.359 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.360 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.360 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.360 DEBUG:boto:Method: GET 2022-09-03T07:24:18.361 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.362 DEBUG:boto:Data: 2022-09-03T07:24:18.362 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.363 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.363 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.364 DEBUG:boto:Params: {} 2022-09-03T07:24:18.364 DEBUG:boto:Token: None 2022-09-03T07:24:18.365 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/?versions 2022-09-03T07:24:18.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m1O+HfsytwmchK23BPy1+KsfeUA= 2022-09-03T07:24:18.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m1O+HfsytwmchK23BPy1+KsfeUA=', 'Content-Length': '0'} 2022-09-03T07:24:18.369 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000834f37527e378794-0063130122-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.369 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.371 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:24:18.372 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:24:18.372 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:24:18.373 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:24:18.373 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:24:18.374 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:24:18.374 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:24:18.375 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:24:18.375 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:24:18.376 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:24:18.376 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:24:18.377 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:24:18.377 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:24:18.378 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:24:18.378 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:24:18.379 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:24:18.379 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:24:18.380 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:24:18.380 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:24:18.381 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:24:18.381 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.382 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.382 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.383 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.383 DEBUG:boto:Method: GET 2022-09-03T07:24:18.384 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.385 DEBUG:boto:Data: 2022-09-03T07:24:18.385 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.386 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.387 DEBUG:boto:Params: {} 2022-09-03T07:24:18.387 DEBUG:boto:Token: None 2022-09-03T07:24:18.388 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/?versions 2022-09-03T07:24:18.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m1O+HfsytwmchK23BPy1+KsfeUA= 2022-09-03T07:24:18.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m1O+HfsytwmchK23BPy1+KsfeUA=', 'Content-Length': '0'} 2022-09-03T07:24:18.392 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b40bb514092a0ba-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.392 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.394 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.395 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.395 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.396 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.396 DEBUG:boto:Method: GET 2022-09-03T07:24:18.397 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:24:18.397 DEBUG:boto:Data: 2022-09-03T07:24:18.398 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.398 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.399 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.399 DEBUG:boto:Params: {} 2022-09-03T07:24:18.400 DEBUG:boto:Token: None 2022-09-03T07:24:18.400 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/?versions 2022-09-03T07:24:18.401 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m1O+HfsytwmchK23BPy1+KsfeUA= 2022-09-03T07:24:18.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m1O+HfsytwmchK23BPy1+KsfeUA=', 'Content-Length': '0'} 2022-09-03T07:24:18.404 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005df3bf51e35daaab-0063130122-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.406 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.409 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:24:18.410 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.411 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.412 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.413 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.414 DEBUG:boto:Method: GET 2022-09-03T07:24:18.415 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.416 DEBUG:boto:Data: 2022-09-03T07:24:18.417 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.418 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.419 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.420 DEBUG:boto:Params: {} 2022-09-03T07:24:18.422 DEBUG:boto:Token: None 2022-09-03T07:24:18.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bri3dI2jSVdg+oH5p8aB98RdNdo= 2022-09-03T07:24:18.424 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bri3dI2jSVdg+oH5p8aB98RdNdo=', 'Content-Length': '0'} 2022-09-03T07:24:18.426 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e47d4a23292b139f-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.427 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.428 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.429 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.429 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.430 DEBUG:boto:Method: GET 2022-09-03T07:24:18.430 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.431 DEBUG:boto:Data: 2022-09-03T07:24:18.432 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.432 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.433 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.434 DEBUG:boto:Params: {} 2022-09-03T07:24:18.435 DEBUG:boto:Token: None 2022-09-03T07:24:18.436 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bri3dI2jSVdg+oH5p8aB98RdNdo= 2022-09-03T07:24:18.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bri3dI2jSVdg+oH5p8aB98RdNdo=', 'Content-Length': '0'} 2022-09-03T07:24:18.441 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c683fd7bece115f5-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.442 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.443 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.443 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.444 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.445 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.446 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.447 DEBUG:boto:Data: 2022-09-03T07:24:18.447 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.449 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.450 DEBUG:boto:Params: {} 2022-09-03T07:24:18.451 DEBUG:boto:Token: None 2022-09-03T07:24:18.451 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYSbpmVYBkPmxo/Vt8M4iFjnXcE= 2022-09-03T07:24:18.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYSbpmVYBkPmxo/Vt8M4iFjnXcE=', 'Content-Length': '0'} 2022-09-03T07:24:18.454 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000963b4f7bc6d3d37c-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.455 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.456 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.456 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.457 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.457 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.458 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.458 DEBUG:boto:Data: 2022-09-03T07:24:18.459 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.459 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.460 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.460 DEBUG:boto:Params: {} 2022-09-03T07:24:18.461 DEBUG:boto:Token: None 2022-09-03T07:24:18.462 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jYSbpmVYBkPmxo/Vt8M4iFjnXcE= 2022-09-03T07:24:18.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jYSbpmVYBkPmxo/Vt8M4iFjnXcE=', 'Content-Length': '0'} 2022-09-03T07:24:18.472 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008862ea128a617849-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.473 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:24:18.474 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.474 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.475 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.476 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.476 DEBUG:boto:Method: GET 2022-09-03T07:24:18.477 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.477 DEBUG:boto:Data: 2022-09-03T07:24:18.478 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.479 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.479 DEBUG:boto:Params: {} 2022-09-03T07:24:18.480 DEBUG:boto:Token: None 2022-09-03T07:24:18.481 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bri3dI2jSVdg+oH5p8aB98RdNdo= 2022-09-03T07:24:18.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bri3dI2jSVdg+oH5p8aB98RdNdo=', 'Content-Length': '0'} 2022-09-03T07:24:18.484 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087b9cc424f3d27d6-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.485 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.485 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.486 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.486 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.487 DEBUG:boto:Method: GET 2022-09-03T07:24:18.487 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.488 DEBUG:boto:Data: 2022-09-03T07:24:18.488 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.489 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.490 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.490 DEBUG:boto:Params: {} 2022-09-03T07:24:18.491 DEBUG:boto:Token: None 2022-09-03T07:24:18.491 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:24:18.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bri3dI2jSVdg+oH5p8aB98RdNdo= 2022-09-03T07:24:18.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bri3dI2jSVdg+oH5p8aB98RdNdo=', 'Content-Length': '0'} 2022-09-03T07:24:18.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e62fbe54f83913f-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.496 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.496 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.497 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:18.498 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:18.498 DEBUG:boto:Method: GET 2022-09-03T07:24:18.499 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:18.499 DEBUG:boto:Data: 2022-09-03T07:24:18.500 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.501 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.501 DEBUG:boto:Params: {} 2022-09-03T07:24:18.502 DEBUG:boto:Token: None 2022-09-03T07:24:18.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.506 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007e3234be2b828441-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.507 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.508 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.508 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.509 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:18.509 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:18.510 DEBUG:boto:Method: GET 2022-09-03T07:24:18.510 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:18.511 DEBUG:boto:Data: 2022-09-03T07:24:18.511 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.512 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.513 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.513 DEBUG:boto:Params: {} 2022-09-03T07:24:18.514 DEBUG:boto:Token: None 2022-09-03T07:24:18.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.518 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000070ea55008d3d8fb5-0063130122-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.562 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.563 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.564 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.564 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.565 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:18.565 DEBUG:boto:Data: 2022-09-03T07:24:18.566 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.567 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.568 DEBUG:boto:Params: {} 2022-09-03T07:24:18.568 DEBUG:boto:Token: None 2022-09-03T07:24:18.569 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11 2022-09-03T07:24:18.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i1W+TbobQi9GjTRV0tCKzjT6tWg= 2022-09-03T07:24:18.570 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i1W+TbobQi9GjTRV0tCKzjT6tWg=', 'Content-Length': '0'} 2022-09-03T07:24:18.572 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be6e83ab9955e43e-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.573 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.573 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.574 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.574 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:18.575 DEBUG:boto:Data: 2022-09-03T07:24:18.575 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.576 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.577 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.577 DEBUG:boto:Params: {} 2022-09-03T07:24:18.578 DEBUG:boto:Token: None 2022-09-03T07:24:18.578 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11 2022-09-03T07:24:18.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i1W+TbobQi9GjTRV0tCKzjT6tWg= 2022-09-03T07:24:18.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i1W+TbobQi9GjTRV0tCKzjT6tWg=', 'Content-Length': '0'} 2022-09-03T07:24:18.581 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c36fe032be1440b-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.582 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:24:18.583 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.583 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.584 DEBUG:boto:Method: GET 2022-09-03T07:24:18.584 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:18.585 DEBUG:boto:Data: 2022-09-03T07:24:18.585 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.587 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.587 DEBUG:boto:Params: {} 2022-09-03T07:24:18.588 DEBUG:boto:Token: None 2022-09-03T07:24:18.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11 2022-09-03T07:24:18.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PK4Sg6JdJsX8fCaByJ11vn+ou1Y= 2022-09-03T07:24:18.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PK4Sg6JdJsX8fCaByJ11vn+ou1Y=', 'Content-Length': '0'} 2022-09-03T07:24:18.592 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0a89333f4113d7d-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.593 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:24:18.593 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:24:18.594 DEBUG:boto:Method: GET 2022-09-03T07:24:18.595 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:24:18.595 DEBUG:boto:Data: 2022-09-03T07:24:18.596 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.596 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.597 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.598 DEBUG:boto:Params: {} 2022-09-03T07:24:18.598 DEBUG:boto:Token: None 2022-09-03T07:24:18.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj11 2022-09-03T07:24:18.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PK4Sg6JdJsX8fCaByJ11vn+ou1Y= 2022-09-03T07:24:18.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PK4Sg6JdJsX8fCaByJ11vn+ou1Y=', 'Content-Length': '0'} 2022-09-03T07:24:18.603 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023e6f936124b53bb-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.605 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.606 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.606 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:18.607 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:18.607 DEBUG:boto:Method: GET 2022-09-03T07:24:18.608 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:18.609 DEBUG:boto:Data: 2022-09-03T07:24:18.609 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.610 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.611 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.611 DEBUG:boto:Params: {} 2022-09-03T07:24:18.612 DEBUG:boto:Token: None 2022-09-03T07:24:18.613 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.617 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000047b0386c65d124fe-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.618 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.619 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.619 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.620 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:18.620 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:24:18.621 DEBUG:boto:Method: GET 2022-09-03T07:24:18.621 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:24:18.622 DEBUG:boto:Data: 2022-09-03T07:24:18.622 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.623 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.623 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.624 DEBUG:boto:Params: {} 2022-09-03T07:24:18.624 DEBUG:boto:Token: None 2022-09-03T07:24:18.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.629 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008429a03e010a5f07-0063130122-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.629 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.630 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:24:18.631 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.631 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.632 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.632 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.633 DEBUG:boto:Method: GET 2022-09-03T07:24:18.633 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.634 DEBUG:boto:Data: 2022-09-03T07:24:18.634 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.635 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.636 DEBUG:boto:Params: {} 2022-09-03T07:24:18.637 DEBUG:boto:Token: None 2022-09-03T07:24:18.637 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WWartrZ5hgrF1Rdo6sXIRvb/Aq4= 2022-09-03T07:24:18.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WWartrZ5hgrF1Rdo6sXIRvb/Aq4=', 'Content-Length': '0'} 2022-09-03T07:24:18.640 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009906176c59c1b35e-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.641 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.642 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.642 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.643 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.643 DEBUG:boto:Method: GET 2022-09-03T07:24:18.644 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.644 DEBUG:boto:Data: 2022-09-03T07:24:18.645 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.645 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.646 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.647 DEBUG:boto:Params: {} 2022-09-03T07:24:18.647 DEBUG:boto:Token: None 2022-09-03T07:24:18.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WWartrZ5hgrF1Rdo6sXIRvb/Aq4= 2022-09-03T07:24:18.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WWartrZ5hgrF1Rdo6sXIRvb/Aq4=', 'Content-Length': '0'} 2022-09-03T07:24:18.651 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065773fecda30302e-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.652 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.652 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.653 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.653 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.654 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.654 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.655 DEBUG:boto:Data: 2022-09-03T07:24:18.655 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.656 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.656 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.657 DEBUG:boto:Params: {} 2022-09-03T07:24:18.657 DEBUG:boto:Token: None 2022-09-03T07:24:18.658 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1wN+oSEyp0J+cvHxeVoB0y/zRtA= 2022-09-03T07:24:18.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1wN+oSEyp0J+cvHxeVoB0y/zRtA=', 'Content-Length': '0'} 2022-09-03T07:24:18.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ffe680d2ec9c157-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.662 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.662 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.663 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.663 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.664 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.664 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.665 DEBUG:boto:Data: 2022-09-03T07:24:18.665 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.666 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.666 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.667 DEBUG:boto:Params: {} 2022-09-03T07:24:18.667 DEBUG:boto:Token: None 2022-09-03T07:24:18.668 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1wN+oSEyp0J+cvHxeVoB0y/zRtA= 2022-09-03T07:24:18.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1wN+oSEyp0J+cvHxeVoB0y/zRtA=', 'Content-Length': '0'} 2022-09-03T07:24:18.672 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6cf4d8dac9f81e0-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.672 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:24:18.673 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.674 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.675 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.675 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.676 DEBUG:boto:Method: GET 2022-09-03T07:24:18.677 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.678 DEBUG:boto:Data: 2022-09-03T07:24:18.679 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.679 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.680 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.681 DEBUG:boto:Params: {} 2022-09-03T07:24:18.681 DEBUG:boto:Token: None 2022-09-03T07:24:18.682 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WWartrZ5hgrF1Rdo6sXIRvb/Aq4= 2022-09-03T07:24:18.683 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WWartrZ5hgrF1Rdo6sXIRvb/Aq4=', 'Content-Length': '0'} 2022-09-03T07:24:18.685 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003eacbcd0828e095a-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.686 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.687 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.687 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.688 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.689 DEBUG:boto:Method: GET 2022-09-03T07:24:18.689 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.690 DEBUG:boto:Data: 2022-09-03T07:24:18.690 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.691 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.691 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.692 DEBUG:boto:Params: {} 2022-09-03T07:24:18.693 DEBUG:boto:Token: None 2022-09-03T07:24:18.693 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:24:18.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WWartrZ5hgrF1Rdo6sXIRvb/Aq4= 2022-09-03T07:24:18.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WWartrZ5hgrF1Rdo6sXIRvb/Aq4=', 'Content-Length': '0'} 2022-09-03T07:24:18.698 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099448b52611de61a-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.699 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.700 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.700 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:18.701 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:18.701 DEBUG:boto:Method: GET 2022-09-03T07:24:18.702 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:18.703 DEBUG:boto:Data: 2022-09-03T07:24:18.703 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.704 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.705 DEBUG:boto:Params: {} 2022-09-03T07:24:18.705 DEBUG:boto:Token: None 2022-09-03T07:24:18.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.710 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090596a40fb3a5965-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.711 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.711 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.712 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.712 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:18.713 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:18.713 DEBUG:boto:Method: GET 2022-09-03T07:24:18.714 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:18.715 DEBUG:boto:Data: 2022-09-03T07:24:18.715 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.716 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.717 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.717 DEBUG:boto:Params: {} 2022-09-03T07:24:18.718 DEBUG:boto:Token: None 2022-09-03T07:24:18.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.723 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ffa64583fba67c85-0063130122-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.770 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.771 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.772 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.773 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.773 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:18.774 DEBUG:boto:Data: 2022-09-03T07:24:18.774 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.775 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.775 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.776 DEBUG:boto:Params: {} 2022-09-03T07:24:18.776 DEBUG:boto:Token: None 2022-09-03T07:24:18.777 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115 2022-09-03T07:24:18.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8tuUEN6sTMau5E+al9ICTDE/zGo= 2022-09-03T07:24:18.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8tuUEN6sTMau5E+al9ICTDE/zGo=', 'Content-Length': '0'} 2022-09-03T07:24:18.780 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000077e8104469698489-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.781 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.781 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.782 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.783 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:18.783 DEBUG:boto:Data: 2022-09-03T07:24:18.784 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.784 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.785 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.785 DEBUG:boto:Params: {} 2022-09-03T07:24:18.786 DEBUG:boto:Token: None 2022-09-03T07:24:18.786 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115 2022-09-03T07:24:18.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8tuUEN6sTMau5E+al9ICTDE/zGo= 2022-09-03T07:24:18.788 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8tuUEN6sTMau5E+al9ICTDE/zGo=', 'Content-Length': '0'} 2022-09-03T07:24:18.790 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d60941dc9b73efd-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.791 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:24:18.791 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.792 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.792 DEBUG:boto:Method: GET 2022-09-03T07:24:18.793 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:18.794 DEBUG:boto:Data: 2022-09-03T07:24:18.795 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.796 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.796 DEBUG:boto:Params: {} 2022-09-03T07:24:18.797 DEBUG:boto:Token: None 2022-09-03T07:24:18.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115 2022-09-03T07:24:18.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CGdbk2dXLOVn4h4yxRhTeMMnaKU= 2022-09-03T07:24:18.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CGdbk2dXLOVn4h4yxRhTeMMnaKU=', 'Content-Length': '0'} 2022-09-03T07:24:18.801 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000994924d4bcffcbb-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.802 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:24:18.802 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:24:18.803 DEBUG:boto:Method: GET 2022-09-03T07:24:18.803 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:24:18.804 DEBUG:boto:Data: 2022-09-03T07:24:18.804 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.805 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.805 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.806 DEBUG:boto:Params: {} 2022-09-03T07:24:18.806 DEBUG:boto:Token: None 2022-09-03T07:24:18.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj115 2022-09-03T07:24:18.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CGdbk2dXLOVn4h4yxRhTeMMnaKU= 2022-09-03T07:24:18.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CGdbk2dXLOVn4h4yxRhTeMMnaKU=', 'Content-Length': '0'} 2022-09-03T07:24:18.811 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007506def9d818228c-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.812 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.813 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.813 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:18.814 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:18.814 DEBUG:boto:Method: GET 2022-09-03T07:24:18.815 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:18.815 DEBUG:boto:Data: 2022-09-03T07:24:18.816 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.817 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.818 DEBUG:boto:Params: {} 2022-09-03T07:24:18.818 DEBUG:boto:Token: None 2022-09-03T07:24:18.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.822 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b826aec506a01e06-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.823 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.824 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.825 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.825 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:18.826 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:24:18.826 DEBUG:boto:Method: GET 2022-09-03T07:24:18.827 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:24:18.827 DEBUG:boto:Data: 2022-09-03T07:24:18.828 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.828 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.829 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.829 DEBUG:boto:Params: {} 2022-09-03T07:24:18.830 DEBUG:boto:Token: None 2022-09-03T07:24:18.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.834 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c3574846213c9f9-0063130122-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.835 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.835 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:24:18.836 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:18.837 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:18.837 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.838 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.838 DEBUG:boto:Method: GET 2022-09-03T07:24:18.839 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.839 DEBUG:boto:Data: 2022-09-03T07:24:18.840 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.840 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.841 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.841 DEBUG:boto:Params: {} 2022-09-03T07:24:18.842 DEBUG:boto:Token: None 2022-09-03T07:24:18.843 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.843 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hvlRLSxgi0Ph6YROTpBXIAk1FRo= 2022-09-03T07:24:18.844 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hvlRLSxgi0Ph6YROTpBXIAk1FRo=', 'Content-Length': '0'} 2022-09-03T07:24:18.846 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2286c5caf256c2b-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.886 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:18.887 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:18.888 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.889 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.889 DEBUG:boto:Method: GET 2022-09-03T07:24:18.890 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.891 DEBUG:boto:Data: 2022-09-03T07:24:18.891 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.892 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.893 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.894 DEBUG:boto:Params: {} 2022-09-03T07:24:18.895 DEBUG:boto:Token: None 2022-09-03T07:24:18.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hvlRLSxgi0Ph6YROTpBXIAk1FRo= 2022-09-03T07:24:18.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hvlRLSxgi0Ph6YROTpBXIAk1FRo=', 'Content-Length': '0'} 2022-09-03T07:24:18.899 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ccc0b8b4ce3d65d5-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.900 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:18.901 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:18.901 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.902 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.902 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.903 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.903 DEBUG:boto:Data: 2022-09-03T07:24:18.904 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.904 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.905 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.905 DEBUG:boto:Params: {} 2022-09-03T07:24:18.906 DEBUG:boto:Token: None 2022-09-03T07:24:18.907 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BWjIbv8okx0EFjd/l9aLfmulxE= 2022-09-03T07:24:18.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BWjIbv8okx0EFjd/l9aLfmulxE=', 'Content-Length': '0'} 2022-09-03T07:24:18.910 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d267a4d85b53d864-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.910 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:18.911 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:18.912 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.914 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.915 DEBUG:boto:Method: HEAD 2022-09-03T07:24:18.915 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.916 DEBUG:boto:Data: 2022-09-03T07:24:18.916 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.917 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.917 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.918 DEBUG:boto:Params: {} 2022-09-03T07:24:18.919 DEBUG:boto:Token: None 2022-09-03T07:24:18.920 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.920 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7BWjIbv8okx0EFjd/l9aLfmulxE= 2022-09-03T07:24:18.921 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7BWjIbv8okx0EFjd/l9aLfmulxE=', 'Content-Length': '0'} 2022-09-03T07:24:18.923 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000236063748cc34d0-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.924 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:24:18.924 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:18.925 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:18.925 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.926 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.926 DEBUG:boto:Method: GET 2022-09-03T07:24:18.927 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.927 DEBUG:boto:Data: 2022-09-03T07:24:18.928 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.928 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.929 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.929 DEBUG:boto:Params: {} 2022-09-03T07:24:18.930 DEBUG:boto:Token: None 2022-09-03T07:24:18.931 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.931 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hvlRLSxgi0Ph6YROTpBXIAk1FRo= 2022-09-03T07:24:18.932 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hvlRLSxgi0Ph6YROTpBXIAk1FRo=', 'Content-Length': '0'} 2022-09-03T07:24:18.934 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089dcdf46e01c387a-0063130122-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.935 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:18.935 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:18.936 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.936 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.937 DEBUG:boto:Method: GET 2022-09-03T07:24:18.937 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.938 DEBUG:boto:Data: 2022-09-03T07:24:18.939 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.940 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:18.940 DEBUG:boto:Port: 8000 2022-09-03T07:24:18.941 DEBUG:boto:Params: {} 2022-09-03T07:24:18.942 DEBUG:boto:Token: None 2022-09-03T07:24:18.942 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:24:18.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hvlRLSxgi0Ph6YROTpBXIAk1FRo= 2022-09-03T07:24:18.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hvlRLSxgi0Ph6YROTpBXIAk1FRo=', 'Content-Length': '0'} 2022-09-03T07:24:18.946 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc7e39cf34150514-0063130122-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.986 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.987 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:18.987 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:18.988 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:18.988 DEBUG:boto:Method: GET 2022-09-03T07:24:18.989 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:18.989 DEBUG:boto:Data: 2022-09-03T07:24:18.990 DEBUG:boto:Headers: {} 2022-09-03T07:24:18.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:18.991 DEBUG:boto:Port: 8001 2022-09-03T07:24:18.992 DEBUG:boto:Params: {} 2022-09-03T07:24:18.992 DEBUG:boto:Token: None 2022-09-03T07:24:18.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:18 GMT /avysmw-75/ 2022-09-03T07:24:18.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg= 2022-09-03T07:24:18.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KLqOlkZrIQ1S6AnNPZuSd+UoZVg=', 'Content-Length': '0'} 2022-09-03T07:24:18.996 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000918931c74623dafe-0063130122-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:18.997 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:18.998 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:18.999 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.000 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:19.000 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:19.001 DEBUG:boto:Method: GET 2022-09-03T07:24:19.001 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:19.002 DEBUG:boto:Data: 2022-09-03T07:24:19.003 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.003 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.004 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.005 DEBUG:boto:Params: {} 2022-09-03T07:24:19.005 DEBUG:boto:Token: None 2022-09-03T07:24:19.006 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002b5785725099293-0063130123-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.010 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.011 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:19.012 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:19.012 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.013 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:19.013 DEBUG:boto:Data: 2022-09-03T07:24:19.014 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.015 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.015 DEBUG:boto:Params: {} 2022-09-03T07:24:19.016 DEBUG:boto:Token: None 2022-09-03T07:24:19.017 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj17 2022-09-03T07:24:19.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S6FeHhh0ZT8Up07ByndtpG+5mMU= 2022-09-03T07:24:19.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S6FeHhh0ZT8Up07ByndtpG+5mMU=', 'Content-Length': '0'} 2022-09-03T07:24:19.020 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055d4f10b46e7ee76-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.020 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:19.021 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:19.022 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.022 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:19.023 DEBUG:boto:Data: 2022-09-03T07:24:19.023 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.024 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.024 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.025 DEBUG:boto:Params: {} 2022-09-03T07:24:19.025 DEBUG:boto:Token: None 2022-09-03T07:24:19.026 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj17 2022-09-03T07:24:19.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S6FeHhh0ZT8Up07ByndtpG+5mMU= 2022-09-03T07:24:19.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S6FeHhh0ZT8Up07ByndtpG+5mMU=', 'Content-Length': '0'} 2022-09-03T07:24:19.029 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080884339e1cc2a80-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.029 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:24:19.030 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:19.031 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:19.031 DEBUG:boto:Method: GET 2022-09-03T07:24:19.032 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:19.032 DEBUG:boto:Data: 2022-09-03T07:24:19.033 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.033 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.034 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.034 DEBUG:boto:Params: {} 2022-09-03T07:24:19.035 DEBUG:boto:Token: None 2022-09-03T07:24:19.035 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj17 2022-09-03T07:24:19.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hqL4GDk598w96nB994y0YxPXXKI= 2022-09-03T07:24:19.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hqL4GDk598w96nB994y0YxPXXKI=', 'Content-Length': '0'} 2022-09-03T07:24:19.038 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004dc87da1143f09eb-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.039 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:24:19.040 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:24:19.040 DEBUG:boto:Method: GET 2022-09-03T07:24:19.041 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:24:19.041 DEBUG:boto:Data: 2022-09-03T07:24:19.042 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.043 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.043 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.044 DEBUG:boto:Params: {} 2022-09-03T07:24:19.044 DEBUG:boto:Token: None 2022-09-03T07:24:19.045 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj17 2022-09-03T07:24:19.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hqL4GDk598w96nB994y0YxPXXKI= 2022-09-03T07:24:19.046 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hqL4GDk598w96nB994y0YxPXXKI=', 'Content-Length': '0'} 2022-09-03T07:24:19.048 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000783dedae7288c385-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.049 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.049 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.050 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:19.050 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:19.051 DEBUG:boto:Method: GET 2022-09-03T07:24:19.051 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:19.052 DEBUG:boto:Data: 2022-09-03T07:24:19.052 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.053 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.053 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.054 DEBUG:boto:Params: {} 2022-09-03T07:24:19.054 DEBUG:boto:Token: None 2022-09-03T07:24:19.055 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.059 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cda4fdc2df8f86cb-0063130123-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.060 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.061 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.061 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.062 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:19.063 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:24:19.063 DEBUG:boto:Method: GET 2022-09-03T07:24:19.064 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:24:19.064 DEBUG:boto:Data: 2022-09-03T07:24:19.065 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.065 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.066 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.066 DEBUG:boto:Params: {} 2022-09-03T07:24:19.067 DEBUG:boto:Token: None 2022-09-03T07:24:19.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.071 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d926e0d9aa4ffc3f-0063130123-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.072 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.073 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:24:19.073 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.074 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.075 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.075 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.076 DEBUG:boto:Method: GET 2022-09-03T07:24:19.076 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.077 DEBUG:boto:Data: 2022-09-03T07:24:19.077 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.078 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.078 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.079 DEBUG:boto:Params: {} 2022-09-03T07:24:19.080 DEBUG:boto:Token: None 2022-09-03T07:24:19.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XmTgDNQfUrjlNJ7cwVMiXPjht8g= 2022-09-03T07:24:19.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XmTgDNQfUrjlNJ7cwVMiXPjht8g=', 'Content-Length': '0'} 2022-09-03T07:24:19.083 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d3e62f5e0acdcc1-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.126 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.127 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.127 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.128 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.128 DEBUG:boto:Method: GET 2022-09-03T07:24:19.129 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.130 DEBUG:boto:Data: 2022-09-03T07:24:19.130 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.131 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.131 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.132 DEBUG:boto:Params: {} 2022-09-03T07:24:19.132 DEBUG:boto:Token: None 2022-09-03T07:24:19.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XmTgDNQfUrjlNJ7cwVMiXPjht8g= 2022-09-03T07:24:19.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XmTgDNQfUrjlNJ7cwVMiXPjht8g=', 'Content-Length': '0'} 2022-09-03T07:24:19.136 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000093cf9d75f79c9eb-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.137 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.138 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.138 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.139 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.139 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.140 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.140 DEBUG:boto:Data: 2022-09-03T07:24:19.141 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.141 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.142 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.142 DEBUG:boto:Params: {} 2022-09-03T07:24:19.143 DEBUG:boto:Token: None 2022-09-03T07:24:19.143 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vaJsWpefC9+Jv0nXYYEZtoYlI5U= 2022-09-03T07:24:19.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vaJsWpefC9+Jv0nXYYEZtoYlI5U=', 'Content-Length': '0'} 2022-09-03T07:24:19.147 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5f7d7e40d216f75-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.147 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.148 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.148 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.149 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.149 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.150 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.150 DEBUG:boto:Data: 2022-09-03T07:24:19.151 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.151 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.152 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.152 DEBUG:boto:Params: {} 2022-09-03T07:24:19.153 DEBUG:boto:Token: None 2022-09-03T07:24:19.154 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vaJsWpefC9+Jv0nXYYEZtoYlI5U= 2022-09-03T07:24:19.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vaJsWpefC9+Jv0nXYYEZtoYlI5U=', 'Content-Length': '0'} 2022-09-03T07:24:19.157 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd0d6285ff4ce26f-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.157 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:24:19.158 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.159 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.159 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.160 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.160 DEBUG:boto:Method: GET 2022-09-03T07:24:19.161 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.161 DEBUG:boto:Data: 2022-09-03T07:24:19.162 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.163 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.163 DEBUG:boto:Params: {} 2022-09-03T07:24:19.164 DEBUG:boto:Token: None 2022-09-03T07:24:19.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XmTgDNQfUrjlNJ7cwVMiXPjht8g= 2022-09-03T07:24:19.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XmTgDNQfUrjlNJ7cwVMiXPjht8g=', 'Content-Length': '0'} 2022-09-03T07:24:19.168 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000003f57587c018956-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.169 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.169 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.170 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.171 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.171 DEBUG:boto:Method: GET 2022-09-03T07:24:19.172 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.172 DEBUG:boto:Data: 2022-09-03T07:24:19.173 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.173 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.174 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.174 DEBUG:boto:Params: {} 2022-09-03T07:24:19.175 DEBUG:boto:Token: None 2022-09-03T07:24:19.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:24:19.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XmTgDNQfUrjlNJ7cwVMiXPjht8g= 2022-09-03T07:24:19.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XmTgDNQfUrjlNJ7cwVMiXPjht8g=', 'Content-Length': '0'} 2022-09-03T07:24:19.179 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000051774dab993c903b-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.179 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.180 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.181 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:19.181 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:19.182 DEBUG:boto:Method: GET 2022-09-03T07:24:19.182 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:19.183 DEBUG:boto:Data: 2022-09-03T07:24:19.183 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.184 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.185 DEBUG:boto:Params: {} 2022-09-03T07:24:19.186 DEBUG:boto:Token: None 2022-09-03T07:24:19.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.191 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb1435b8eb57ff8f-0063130123-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.191 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.192 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.193 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.193 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:19.194 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:19.194 DEBUG:boto:Method: GET 2022-09-03T07:24:19.195 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:19.195 DEBUG:boto:Data: 2022-09-03T07:24:19.196 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.196 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.197 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.198 DEBUG:boto:Params: {} 2022-09-03T07:24:19.198 DEBUG:boto:Token: None 2022-09-03T07:24:19.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.203 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005cff711f18bb3798-0063130123-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.246 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.247 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.248 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.248 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.249 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:19.249 DEBUG:boto:Data: 2022-09-03T07:24:19.250 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.251 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.251 DEBUG:boto:Params: {} 2022-09-03T07:24:19.252 DEBUG:boto:Token: None 2022-09-03T07:24:19.252 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19 2022-09-03T07:24:19.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bmi5ADmz5bM3MyRRi6inBYijTR4= 2022-09-03T07:24:19.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bmi5ADmz5bM3MyRRi6inBYijTR4=', 'Content-Length': '0'} 2022-09-03T07:24:19.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2c86038972b6dfe-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.256 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.257 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.258 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.258 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:19.259 DEBUG:boto:Data: 2022-09-03T07:24:19.259 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.260 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.260 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.261 DEBUG:boto:Params: {} 2022-09-03T07:24:19.261 DEBUG:boto:Token: None 2022-09-03T07:24:19.262 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19 2022-09-03T07:24:19.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bmi5ADmz5bM3MyRRi6inBYijTR4= 2022-09-03T07:24:19.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bmi5ADmz5bM3MyRRi6inBYijTR4=', 'Content-Length': '0'} 2022-09-03T07:24:19.265 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000247648331125ada-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.266 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:24:19.267 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.267 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.268 DEBUG:boto:Method: GET 2022-09-03T07:24:19.268 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:19.269 DEBUG:boto:Data: 2022-09-03T07:24:19.270 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.270 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.271 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.272 DEBUG:boto:Params: {} 2022-09-03T07:24:19.272 DEBUG:boto:Token: None 2022-09-03T07:24:19.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19 2022-09-03T07:24:19.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8xQA9/oThwuQg13MKyK/sIU2qDk= 2022-09-03T07:24:19.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8xQA9/oThwuQg13MKyK/sIU2qDk=', 'Content-Length': '0'} 2022-09-03T07:24:19.277 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035ebc14cba1101dd-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.278 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:24:19.279 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:24:19.280 DEBUG:boto:Method: GET 2022-09-03T07:24:19.281 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:24:19.281 DEBUG:boto:Data: 2022-09-03T07:24:19.282 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.283 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.284 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.286 DEBUG:boto:Params: {} 2022-09-03T07:24:19.287 DEBUG:boto:Token: None 2022-09-03T07:24:19.288 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj19 2022-09-03T07:24:19.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8xQA9/oThwuQg13MKyK/sIU2qDk= 2022-09-03T07:24:19.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8xQA9/oThwuQg13MKyK/sIU2qDk=', 'Content-Length': '0'} 2022-09-03T07:24:19.293 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000862ae16eb9c61e6-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.295 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.295 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.296 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:19.296 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:19.297 DEBUG:boto:Method: GET 2022-09-03T07:24:19.298 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:19.298 DEBUG:boto:Data: 2022-09-03T07:24:19.299 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.300 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.300 DEBUG:boto:Params: {} 2022-09-03T07:24:19.301 DEBUG:boto:Token: None 2022-09-03T07:24:19.302 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.307 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000058ed9adb4aefc00-0063130123-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.307 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.309 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.310 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.311 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:19.312 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:24:19.313 DEBUG:boto:Method: GET 2022-09-03T07:24:19.314 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:24:19.315 DEBUG:boto:Data: 2022-09-03T07:24:19.316 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.317 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.318 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.319 DEBUG:boto:Params: {} 2022-09-03T07:24:19.320 DEBUG:boto:Token: None 2022-09-03T07:24:19.321 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.327 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da0d67a863eea05c-0063130123-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.328 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.329 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:24:19.330 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.330 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.331 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.331 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.332 DEBUG:boto:Method: GET 2022-09-03T07:24:19.333 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.334 DEBUG:boto:Data: 2022-09-03T07:24:19.334 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.336 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.336 DEBUG:boto:Params: {} 2022-09-03T07:24:19.337 DEBUG:boto:Token: None 2022-09-03T07:24:19.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/SSh0NCtjjkMP2s5ALPJlamVBHg= 2022-09-03T07:24:19.339 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/SSh0NCtjjkMP2s5ALPJlamVBHg=', 'Content-Length': '0'} 2022-09-03T07:24:19.341 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049ef6987e2366bbf-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.386 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.387 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.387 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.388 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.388 DEBUG:boto:Method: GET 2022-09-03T07:24:19.389 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.390 DEBUG:boto:Data: 2022-09-03T07:24:19.390 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.391 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.392 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.392 DEBUG:boto:Params: {} 2022-09-03T07:24:19.393 DEBUG:boto:Token: None 2022-09-03T07:24:19.394 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/SSh0NCtjjkMP2s5ALPJlamVBHg= 2022-09-03T07:24:19.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/SSh0NCtjjkMP2s5ALPJlamVBHg=', 'Content-Length': '0'} 2022-09-03T07:24:19.398 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070b4d3751c7bbaab-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.399 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.399 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.401 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.402 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.403 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.403 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.405 DEBUG:boto:Data: 2022-09-03T07:24:19.406 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.408 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.409 DEBUG:boto:Params: {} 2022-09-03T07:24:19.410 DEBUG:boto:Token: None 2022-09-03T07:24:19.411 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2lJx3El7FY0/qS6J+kdHLr11U/w= 2022-09-03T07:24:19.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2lJx3El7FY0/qS6J+kdHLr11U/w=', 'Content-Length': '0'} 2022-09-03T07:24:19.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010f8408570c74a25-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.416 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.417 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.418 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.419 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.420 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.421 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.421 DEBUG:boto:Data: 2022-09-03T07:24:19.422 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.423 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.424 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.425 DEBUG:boto:Params: {} 2022-09-03T07:24:19.426 DEBUG:boto:Token: None 2022-09-03T07:24:19.427 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2lJx3El7FY0/qS6J+kdHLr11U/w= 2022-09-03T07:24:19.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2lJx3El7FY0/qS6J+kdHLr11U/w=', 'Content-Length': '0'} 2022-09-03T07:24:19.431 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ecee1f1656c1452b-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.432 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:24:19.433 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.433 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.434 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.435 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.435 DEBUG:boto:Method: GET 2022-09-03T07:24:19.436 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.436 DEBUG:boto:Data: 2022-09-03T07:24:19.437 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.437 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.438 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.439 DEBUG:boto:Params: {} 2022-09-03T07:24:19.439 DEBUG:boto:Token: None 2022-09-03T07:24:19.440 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/SSh0NCtjjkMP2s5ALPJlamVBHg= 2022-09-03T07:24:19.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/SSh0NCtjjkMP2s5ALPJlamVBHg=', 'Content-Length': '0'} 2022-09-03T07:24:19.443 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a177b702ef5b370-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.444 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.445 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.445 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.446 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.446 DEBUG:boto:Method: GET 2022-09-03T07:24:19.447 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.447 DEBUG:boto:Data: 2022-09-03T07:24:19.448 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.448 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.449 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.450 DEBUG:boto:Params: {} 2022-09-03T07:24:19.450 DEBUG:boto:Token: None 2022-09-03T07:24:19.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:24:19.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/SSh0NCtjjkMP2s5ALPJlamVBHg= 2022-09-03T07:24:19.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/SSh0NCtjjkMP2s5ALPJlamVBHg=', 'Content-Length': '0'} 2022-09-03T07:24:19.454 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c6bb57564fdb9d31-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.455 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.456 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.456 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:19.457 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:19.457 DEBUG:boto:Method: GET 2022-09-03T07:24:19.458 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:19.459 DEBUG:boto:Data: 2022-09-03T07:24:19.459 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.460 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.461 DEBUG:boto:Params: {} 2022-09-03T07:24:19.462 DEBUG:boto:Token: None 2022-09-03T07:24:19.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.466 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000070882d96b6b2ecac-0063130123-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.467 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.468 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.469 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.469 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:19.470 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:19.470 DEBUG:boto:Method: GET 2022-09-03T07:24:19.471 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:19.472 DEBUG:boto:Data: 2022-09-03T07:24:19.472 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.473 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.474 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.475 DEBUG:boto:Params: {} 2022-09-03T07:24:19.475 DEBUG:boto:Token: None 2022-09-03T07:24:19.476 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.480 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f97892134f9e062a-0063130123-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.480 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.481 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.482 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.483 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.483 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:19.484 DEBUG:boto:Data: 2022-09-03T07:24:19.484 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.485 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.486 DEBUG:boto:Params: {} 2022-09-03T07:24:19.487 DEBUG:boto:Token: None 2022-09-03T07:24:19.487 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22 2022-09-03T07:24:19.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2wb+wqnLASDZpLPirAQEPF0zC1Q= 2022-09-03T07:24:19.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2wb+wqnLASDZpLPirAQEPF0zC1Q=', 'Content-Length': '0'} 2022-09-03T07:24:19.491 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7b64e4364c1f3a4-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.492 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.492 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.493 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.493 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:19.494 DEBUG:boto:Data: 2022-09-03T07:24:19.495 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.495 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.496 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.496 DEBUG:boto:Params: {} 2022-09-03T07:24:19.497 DEBUG:boto:Token: None 2022-09-03T07:24:19.497 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22 2022-09-03T07:24:19.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2wb+wqnLASDZpLPirAQEPF0zC1Q= 2022-09-03T07:24:19.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2wb+wqnLASDZpLPirAQEPF0zC1Q=', 'Content-Length': '0'} 2022-09-03T07:24:19.500 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e53d1dc189e85bb5-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.501 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:24:19.502 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.502 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.503 DEBUG:boto:Method: GET 2022-09-03T07:24:19.504 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:19.504 DEBUG:boto:Data: 2022-09-03T07:24:19.505 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.506 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.506 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.507 DEBUG:boto:Params: {} 2022-09-03T07:24:19.507 DEBUG:boto:Token: None 2022-09-03T07:24:19.508 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22 2022-09-03T07:24:19.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:99khC7Ro408205Uh4Hk9k047mh8= 2022-09-03T07:24:19.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:99khC7Ro408205Uh4Hk9k047mh8=', 'Content-Length': '0'} 2022-09-03T07:24:19.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000173da36b427bdb-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.554 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:24:19.555 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:24:19.555 DEBUG:boto:Method: GET 2022-09-03T07:24:19.556 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:24:19.557 DEBUG:boto:Data: 2022-09-03T07:24:19.557 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.558 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.558 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.559 DEBUG:boto:Params: {} 2022-09-03T07:24:19.560 DEBUG:boto:Token: None 2022-09-03T07:24:19.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj22 2022-09-03T07:24:19.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:99khC7Ro408205Uh4Hk9k047mh8= 2022-09-03T07:24:19.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:99khC7Ro408205Uh4Hk9k047mh8=', 'Content-Length': '0'} 2022-09-03T07:24:19.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1f230ba3ef3f711-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.565 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.566 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.566 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:19.567 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:19.568 DEBUG:boto:Method: GET 2022-09-03T07:24:19.568 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:19.569 DEBUG:boto:Data: 2022-09-03T07:24:19.569 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.570 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.571 DEBUG:boto:Params: {} 2022-09-03T07:24:19.571 DEBUG:boto:Token: None 2022-09-03T07:24:19.572 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.576 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001fa17592c3d69796-0063130123-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.576 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.577 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.578 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.578 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:19.579 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:24:19.579 DEBUG:boto:Method: GET 2022-09-03T07:24:19.580 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:24:19.581 DEBUG:boto:Data: 2022-09-03T07:24:19.581 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.582 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.583 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.583 DEBUG:boto:Params: {} 2022-09-03T07:24:19.584 DEBUG:boto:Token: None 2022-09-03T07:24:19.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.588 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005d2d06adf9705e77-0063130123-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.589 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.590 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:24:19.590 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.591 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.591 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.592 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.592 DEBUG:boto:Method: GET 2022-09-03T07:24:19.593 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.593 DEBUG:boto:Data: 2022-09-03T07:24:19.594 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.595 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.596 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.597 DEBUG:boto:Params: {} 2022-09-03T07:24:19.598 DEBUG:boto:Token: None 2022-09-03T07:24:19.598 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zkoVmKZ3RJallz4LsQOgfzI+qhc= 2022-09-03T07:24:19.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zkoVmKZ3RJallz4LsQOgfzI+qhc=', 'Content-Length': '0'} 2022-09-03T07:24:19.602 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cf64cb84875aab9-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.603 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.603 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.605 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.606 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.607 DEBUG:boto:Method: GET 2022-09-03T07:24:19.607 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.608 DEBUG:boto:Data: 2022-09-03T07:24:19.608 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.609 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.610 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.611 DEBUG:boto:Params: {} 2022-09-03T07:24:19.612 DEBUG:boto:Token: None 2022-09-03T07:24:19.613 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zkoVmKZ3RJallz4LsQOgfzI+qhc= 2022-09-03T07:24:19.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zkoVmKZ3RJallz4LsQOgfzI+qhc=', 'Content-Length': '0'} 2022-09-03T07:24:19.616 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000021d1796c77b00333-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.617 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.618 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.619 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.619 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.620 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.620 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.621 DEBUG:boto:Data: 2022-09-03T07:24:19.622 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.623 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.623 DEBUG:boto:Params: {} 2022-09-03T07:24:19.624 DEBUG:boto:Token: None 2022-09-03T07:24:19.625 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UqKOfNUslKJVSKyWS2Cs7Q+QU/k= 2022-09-03T07:24:19.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UqKOfNUslKJVSKyWS2Cs7Q+QU/k=', 'Content-Length': '0'} 2022-09-03T07:24:19.628 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a6d127553f98129-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.628 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.629 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.630 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.630 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.631 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.631 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.632 DEBUG:boto:Data: 2022-09-03T07:24:19.632 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.633 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.633 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.634 DEBUG:boto:Params: {} 2022-09-03T07:24:19.634 DEBUG:boto:Token: None 2022-09-03T07:24:19.635 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.636 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UqKOfNUslKJVSKyWS2Cs7Q+QU/k= 2022-09-03T07:24:19.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UqKOfNUslKJVSKyWS2Cs7Q+QU/k=', 'Content-Length': '0'} 2022-09-03T07:24:19.638 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1984c5ca51c1b2e-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.639 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:24:19.639 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.640 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.640 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.641 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.641 DEBUG:boto:Method: GET 2022-09-03T07:24:19.642 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.643 DEBUG:boto:Data: 2022-09-03T07:24:19.644 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.645 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.645 DEBUG:boto:Params: {} 2022-09-03T07:24:19.646 DEBUG:boto:Token: None 2022-09-03T07:24:19.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.647 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zkoVmKZ3RJallz4LsQOgfzI+qhc= 2022-09-03T07:24:19.648 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zkoVmKZ3RJallz4LsQOgfzI+qhc=', 'Content-Length': '0'} 2022-09-03T07:24:19.650 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000beca8316c7a17a8a-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.651 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.651 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.652 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.653 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.653 DEBUG:boto:Method: GET 2022-09-03T07:24:19.654 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.654 DEBUG:boto:Data: 2022-09-03T07:24:19.655 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.655 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.656 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.657 DEBUG:boto:Params: {} 2022-09-03T07:24:19.657 DEBUG:boto:Token: None 2022-09-03T07:24:19.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:24:19.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zkoVmKZ3RJallz4LsQOgfzI+qhc= 2022-09-03T07:24:19.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zkoVmKZ3RJallz4LsQOgfzI+qhc=', 'Content-Length': '0'} 2022-09-03T07:24:19.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1fb229e955274c7-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.706 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.707 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.708 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:19.708 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:19.709 DEBUG:boto:Method: GET 2022-09-03T07:24:19.709 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:19.710 DEBUG:boto:Data: 2022-09-03T07:24:19.711 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.711 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.712 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.712 DEBUG:boto:Params: {} 2022-09-03T07:24:19.713 DEBUG:boto:Token: None 2022-09-03T07:24:19.714 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.718 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d4893a25a889d39-0063130123-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.719 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.720 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.721 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.721 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:19.722 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:19.723 DEBUG:boto:Method: GET 2022-09-03T07:24:19.724 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:19.724 DEBUG:boto:Data: 2022-09-03T07:24:19.725 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.725 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.726 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.727 DEBUG:boto:Params: {} 2022-09-03T07:24:19.727 DEBUG:boto:Token: None 2022-09-03T07:24:19.728 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.732 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c74c1fca90492cf3-0063130123-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.733 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.735 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.736 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.736 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.737 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:19.737 DEBUG:boto:Data: 2022-09-03T07:24:19.738 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.739 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.739 DEBUG:boto:Params: {} 2022-09-03T07:24:19.740 DEBUG:boto:Token: None 2022-09-03T07:24:19.740 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23 2022-09-03T07:24:19.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UzHhNeITVdU6IHJUuUgGrvu4wjo= 2022-09-03T07:24:19.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UzHhNeITVdU6IHJUuUgGrvu4wjo=', 'Content-Length': '0'} 2022-09-03T07:24:19.744 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e0505558f2e8ad4d-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.745 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.745 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.746 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.747 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:19.747 DEBUG:boto:Data: 2022-09-03T07:24:19.748 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.748 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.749 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.749 DEBUG:boto:Params: {} 2022-09-03T07:24:19.750 DEBUG:boto:Token: None 2022-09-03T07:24:19.750 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23 2022-09-03T07:24:19.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UzHhNeITVdU6IHJUuUgGrvu4wjo= 2022-09-03T07:24:19.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UzHhNeITVdU6IHJUuUgGrvu4wjo=', 'Content-Length': '0'} 2022-09-03T07:24:19.753 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000025bc7e1e5d0678ac-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.754 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:24:19.755 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.755 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.756 DEBUG:boto:Method: GET 2022-09-03T07:24:19.757 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:19.758 DEBUG:boto:Data: 2022-09-03T07:24:19.759 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.760 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.760 DEBUG:boto:Params: {} 2022-09-03T07:24:19.761 DEBUG:boto:Token: None 2022-09-03T07:24:19.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23 2022-09-03T07:24:19.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zovwGpXADK4ym+48LSGm8MWtY6o= 2022-09-03T07:24:19.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zovwGpXADK4ym+48LSGm8MWtY6o=', 'Content-Length': '0'} 2022-09-03T07:24:19.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e7ce9937ba2c287-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.806 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:24:19.807 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:24:19.807 DEBUG:boto:Method: GET 2022-09-03T07:24:19.808 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:24:19.809 DEBUG:boto:Data: 2022-09-03T07:24:19.809 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.810 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.810 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.811 DEBUG:boto:Params: {} 2022-09-03T07:24:19.811 DEBUG:boto:Token: None 2022-09-03T07:24:19.812 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj23 2022-09-03T07:24:19.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zovwGpXADK4ym+48LSGm8MWtY6o= 2022-09-03T07:24:19.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zovwGpXADK4ym+48LSGm8MWtY6o=', 'Content-Length': '0'} 2022-09-03T07:24:19.816 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000909c358b3c8ffb0d-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.817 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.817 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.818 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:19.818 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:19.819 DEBUG:boto:Method: GET 2022-09-03T07:24:19.820 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:19.820 DEBUG:boto:Data: 2022-09-03T07:24:19.821 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.821 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.822 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.823 DEBUG:boto:Params: {} 2022-09-03T07:24:19.823 DEBUG:boto:Token: None 2022-09-03T07:24:19.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.827 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a8a88c0b007beab-0063130123-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.828 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.829 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.829 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.830 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:19.831 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:24:19.831 DEBUG:boto:Method: GET 2022-09-03T07:24:19.832 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:24:19.832 DEBUG:boto:Data: 2022-09-03T07:24:19.833 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.833 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.834 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.834 DEBUG:boto:Params: {} 2022-09-03T07:24:19.835 DEBUG:boto:Token: None 2022-09-03T07:24:19.836 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.839 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e303f20123fe3248-0063130123-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.840 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.841 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:24:19.841 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:19.842 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:19.842 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.843 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.844 DEBUG:boto:Method: GET 2022-09-03T07:24:19.844 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.845 DEBUG:boto:Data: 2022-09-03T07:24:19.845 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.846 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.847 DEBUG:boto:Params: {} 2022-09-03T07:24:19.847 DEBUG:boto:Token: None 2022-09-03T07:24:19.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lnQmPCaFuj3DzPtkLTfYCujkQXQ= 2022-09-03T07:24:19.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lnQmPCaFuj3DzPtkLTfYCujkQXQ=', 'Content-Length': '0'} 2022-09-03T07:24:19.851 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6121abc64edfe87-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.852 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:19.852 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:19.853 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.854 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.854 DEBUG:boto:Method: GET 2022-09-03T07:24:19.855 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.855 DEBUG:boto:Data: 2022-09-03T07:24:19.856 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.856 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.857 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.858 DEBUG:boto:Params: {} 2022-09-03T07:24:19.858 DEBUG:boto:Token: None 2022-09-03T07:24:19.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lnQmPCaFuj3DzPtkLTfYCujkQXQ= 2022-09-03T07:24:19.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lnQmPCaFuj3DzPtkLTfYCujkQXQ=', 'Content-Length': '0'} 2022-09-03T07:24:19.864 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000197c2fc5ea587d06-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.865 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:19.865 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:19.866 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.866 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.867 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.867 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.868 DEBUG:boto:Data: 2022-09-03T07:24:19.868 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.869 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.869 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.870 DEBUG:boto:Params: {} 2022-09-03T07:24:19.870 DEBUG:boto:Token: None 2022-09-03T07:24:19.871 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.871 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n9uQp+b0HyFlOy4r3Lv4qJX7RHs= 2022-09-03T07:24:19.872 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n9uQp+b0HyFlOy4r3Lv4qJX7RHs=', 'Content-Length': '0'} 2022-09-03T07:24:19.874 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048b5d5ec1108eb65-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.875 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:19.875 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:19.876 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.876 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.877 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.878 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.878 DEBUG:boto:Data: 2022-09-03T07:24:19.879 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.880 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.880 DEBUG:boto:Params: {} 2022-09-03T07:24:19.881 DEBUG:boto:Token: None 2022-09-03T07:24:19.881 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.882 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n9uQp+b0HyFlOy4r3Lv4qJX7RHs= 2022-09-03T07:24:19.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n9uQp+b0HyFlOy4r3Lv4qJX7RHs=', 'Content-Length': '0'} 2022-09-03T07:24:19.884 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e85d4514e567b9c-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.885 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:24:19.885 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:19.886 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:19.887 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.887 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.888 DEBUG:boto:Method: GET 2022-09-03T07:24:19.888 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.889 DEBUG:boto:Data: 2022-09-03T07:24:19.889 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.890 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.890 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.891 DEBUG:boto:Params: {} 2022-09-03T07:24:19.891 DEBUG:boto:Token: None 2022-09-03T07:24:19.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lnQmPCaFuj3DzPtkLTfYCujkQXQ= 2022-09-03T07:24:19.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lnQmPCaFuj3DzPtkLTfYCujkQXQ=', 'Content-Length': '0'} 2022-09-03T07:24:19.895 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce89d917bdf32ba2-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.896 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:19.896 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:19.897 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.897 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.898 DEBUG:boto:Method: GET 2022-09-03T07:24:19.898 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.899 DEBUG:boto:Data: 2022-09-03T07:24:19.900 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.900 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.901 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.901 DEBUG:boto:Params: {} 2022-09-03T07:24:19.902 DEBUG:boto:Token: None 2022-09-03T07:24:19.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:24:19.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lnQmPCaFuj3DzPtkLTfYCujkQXQ= 2022-09-03T07:24:19.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lnQmPCaFuj3DzPtkLTfYCujkQXQ=', 'Content-Length': '0'} 2022-09-03T07:24:19.905 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f1e13c262ce9f4a-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.950 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.951 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.951 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:19.952 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:19.952 DEBUG:boto:Method: GET 2022-09-03T07:24:19.953 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:19.953 DEBUG:boto:Data: 2022-09-03T07:24:19.954 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.955 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.956 DEBUG:boto:Params: {} 2022-09-03T07:24:19.956 DEBUG:boto:Token: None 2022-09-03T07:24:19.957 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.961 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011f1d051ed8081c2-0063130123-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.962 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.963 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:19.963 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:19.964 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:19.965 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:19.966 DEBUG:boto:Method: GET 2022-09-03T07:24:19.966 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:19.967 DEBUG:boto:Data: 2022-09-03T07:24:19.967 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.968 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.968 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.969 DEBUG:boto:Params: {} 2022-09-03T07:24:19.970 DEBUG:boto:Token: None 2022-09-03T07:24:19.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/ 2022-09-03T07:24:19.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY= 2022-09-03T07:24:19.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7/NkQu50/UvIcU9YJsfUzBtzCLY=', 'Content-Length': '0'} 2022-09-03T07:24:19.976 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ab323c063153ac6-0063130123-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.976 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:19.977 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:19.978 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:19.978 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.979 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:19.980 DEBUG:boto:Data: 2022-09-03T07:24:19.981 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.982 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:19.982 DEBUG:boto:Port: 8001 2022-09-03T07:24:19.983 DEBUG:boto:Params: {} 2022-09-03T07:24:19.983 DEBUG:boto:Token: None 2022-09-03T07:24:19.984 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj33 2022-09-03T07:24:19.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVdtVCbuwVzkH8NY/P3BdelYsXQ= 2022-09-03T07:24:19.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVdtVCbuwVzkH8NY/P3BdelYsXQ=', 'Content-Length': '0'} 2022-09-03T07:24:19.989 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c5d1dfd19452b96-0063130123-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:19.990 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:19.991 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:19.992 DEBUG:boto:Method: HEAD 2022-09-03T07:24:19.992 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:19.993 DEBUG:boto:Data: 2022-09-03T07:24:19.993 DEBUG:boto:Headers: {} 2022-09-03T07:24:19.994 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:19.994 DEBUG:boto:Port: 8000 2022-09-03T07:24:19.995 DEBUG:boto:Params: {} 2022-09-03T07:24:19.995 DEBUG:boto:Token: None 2022-09-03T07:24:19.996 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:19 GMT /avysmw-75/obj33 2022-09-03T07:24:19.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dVdtVCbuwVzkH8NY/P3BdelYsXQ= 2022-09-03T07:24:19.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dVdtVCbuwVzkH8NY/P3BdelYsXQ=', 'Content-Length': '0'} 2022-09-03T07:24:19.999 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048a542fb910d604f-0063130123-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.000 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:24:20.001 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:20.002 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:20.002 DEBUG:boto:Method: GET 2022-09-03T07:24:20.003 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:20.003 DEBUG:boto:Data: 2022-09-03T07:24:20.004 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.004 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.005 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.005 DEBUG:boto:Params: {} 2022-09-03T07:24:20.006 DEBUG:boto:Token: None 2022-09-03T07:24:20.007 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj33 2022-09-03T07:24:20.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yPh9GOMjQ18qKapQ3brwbAGtMnk= 2022-09-03T07:24:20.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yPh9GOMjQ18qKapQ3brwbAGtMnk=', 'Content-Length': '0'} 2022-09-03T07:24:20.010 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001562a14b270c5843-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.011 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:24:20.012 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:24:20.012 DEBUG:boto:Method: GET 2022-09-03T07:24:20.013 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:24:20.013 DEBUG:boto:Data: 2022-09-03T07:24:20.014 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.014 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.015 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.016 DEBUG:boto:Params: {} 2022-09-03T07:24:20.016 DEBUG:boto:Token: None 2022-09-03T07:24:20.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj33 2022-09-03T07:24:20.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yPh9GOMjQ18qKapQ3brwbAGtMnk= 2022-09-03T07:24:20.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yPh9GOMjQ18qKapQ3brwbAGtMnk=', 'Content-Length': '0'} 2022-09-03T07:24:20.020 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052a178e6838c417b-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.021 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.022 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.022 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:20.023 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:20.023 DEBUG:boto:Method: GET 2022-09-03T07:24:20.024 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:20.024 DEBUG:boto:Data: 2022-09-03T07:24:20.025 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.026 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.027 DEBUG:boto:Params: {} 2022-09-03T07:24:20.027 DEBUG:boto:Token: None 2022-09-03T07:24:20.028 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.035 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ae3caa2168a7a2d-0063130124-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.078 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.079 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.080 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.081 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:20.081 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:24:20.082 DEBUG:boto:Method: GET 2022-09-03T07:24:20.083 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:24:20.084 DEBUG:boto:Data: 2022-09-03T07:24:20.085 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.086 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.087 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.087 DEBUG:boto:Params: {} 2022-09-03T07:24:20.088 DEBUG:boto:Token: None 2022-09-03T07:24:20.089 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.093 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001586a3c71eba8440-0063130124-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.094 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.095 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:24:20.096 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.097 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.098 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.099 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.100 DEBUG:boto:Method: GET 2022-09-03T07:24:20.101 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.102 DEBUG:boto:Data: 2022-09-03T07:24:20.103 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.104 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.105 DEBUG:boto:Params: {} 2022-09-03T07:24:20.106 DEBUG:boto:Token: None 2022-09-03T07:24:20.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:17SQnLz9RqfX3K+N6LAzFThs0iM= 2022-09-03T07:24:20.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:17SQnLz9RqfX3K+N6LAzFThs0iM=', 'Content-Length': '0'} 2022-09-03T07:24:20.113 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000848c51be1e662e6c-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.114 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.115 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.116 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.117 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.118 DEBUG:boto:Method: GET 2022-09-03T07:24:20.119 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.120 DEBUG:boto:Data: 2022-09-03T07:24:20.121 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.122 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.123 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.124 DEBUG:boto:Params: {} 2022-09-03T07:24:20.125 DEBUG:boto:Token: None 2022-09-03T07:24:20.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:17SQnLz9RqfX3K+N6LAzFThs0iM= 2022-09-03T07:24:20.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:17SQnLz9RqfX3K+N6LAzFThs0iM=', 'Content-Length': '0'} 2022-09-03T07:24:20.131 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010b25f261a4045c2-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.132 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.133 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.134 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.134 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.135 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.135 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.136 DEBUG:boto:Data: 2022-09-03T07:24:20.136 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.137 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.138 DEBUG:boto:Params: {} 2022-09-03T07:24:20.138 DEBUG:boto:Token: None 2022-09-03T07:24:20.139 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y6ROuV603w2OWRpXLIz5Soxcimc= 2022-09-03T07:24:20.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y6ROuV603w2OWRpXLIz5Soxcimc=', 'Content-Length': '0'} 2022-09-03T07:24:20.142 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000019ea01d2bc93dc39-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.144 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.144 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.145 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.146 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.147 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.148 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.149 DEBUG:boto:Data: 2022-09-03T07:24:20.149 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.150 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.151 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.152 DEBUG:boto:Params: {} 2022-09-03T07:24:20.152 DEBUG:boto:Token: None 2022-09-03T07:24:20.153 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y6ROuV603w2OWRpXLIz5Soxcimc= 2022-09-03T07:24:20.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y6ROuV603w2OWRpXLIz5Soxcimc=', 'Content-Length': '0'} 2022-09-03T07:24:20.158 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e78ca41de6b0074-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.159 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:24:20.160 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.160 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.161 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.161 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.162 DEBUG:boto:Method: GET 2022-09-03T07:24:20.163 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.163 DEBUG:boto:Data: 2022-09-03T07:24:20.164 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.164 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.165 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.165 DEBUG:boto:Params: {} 2022-09-03T07:24:20.166 DEBUG:boto:Token: None 2022-09-03T07:24:20.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:17SQnLz9RqfX3K+N6LAzFThs0iM= 2022-09-03T07:24:20.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:17SQnLz9RqfX3K+N6LAzFThs0iM=', 'Content-Length': '0'} 2022-09-03T07:24:20.170 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008002e3cb7b690886-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.171 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.171 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.172 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.172 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.173 DEBUG:boto:Method: GET 2022-09-03T07:24:20.173 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.174 DEBUG:boto:Data: 2022-09-03T07:24:20.175 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.175 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.176 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.176 DEBUG:boto:Params: {} 2022-09-03T07:24:20.177 DEBUG:boto:Token: None 2022-09-03T07:24:20.178 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:24:20.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:17SQnLz9RqfX3K+N6LAzFThs0iM= 2022-09-03T07:24:20.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:17SQnLz9RqfX3K+N6LAzFThs0iM=', 'Content-Length': '0'} 2022-09-03T07:24:20.182 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000427ed19a1debaf51-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.226 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.227 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.228 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:20.228 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:20.229 DEBUG:boto:Method: GET 2022-09-03T07:24:20.229 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:20.230 DEBUG:boto:Data: 2022-09-03T07:24:20.230 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.232 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.232 DEBUG:boto:Params: {} 2022-09-03T07:24:20.233 DEBUG:boto:Token: None 2022-09-03T07:24:20.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.237 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000046ce58170eba8adf-0063130124-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.238 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.239 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.240 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.240 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:20.241 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:20.242 DEBUG:boto:Method: GET 2022-09-03T07:24:20.242 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:20.243 DEBUG:boto:Data: 2022-09-03T07:24:20.243 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.244 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.244 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.245 DEBUG:boto:Params: {} 2022-09-03T07:24:20.246 DEBUG:boto:Token: None 2022-09-03T07:24:20.246 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.250 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000001c22b29de36f9f2-0063130124-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.251 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.251 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.252 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.252 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.253 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:20.253 DEBUG:boto:Data: 2022-09-03T07:24:20.254 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.254 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.255 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.256 DEBUG:boto:Params: {} 2022-09-03T07:24:20.256 DEBUG:boto:Token: None 2022-09-03T07:24:20.257 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34 2022-09-03T07:24:20.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S8LECaYrBJdBaHf9kRXANB8Dx3o= 2022-09-03T07:24:20.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S8LECaYrBJdBaHf9kRXANB8Dx3o=', 'Content-Length': '0'} 2022-09-03T07:24:20.262 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d2aab1ddbe5f6cc0-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.263 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.263 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.264 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.265 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:20.265 DEBUG:boto:Data: 2022-09-03T07:24:20.266 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.266 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.267 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.267 DEBUG:boto:Params: {} 2022-09-03T07:24:20.268 DEBUG:boto:Token: None 2022-09-03T07:24:20.269 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34 2022-09-03T07:24:20.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S8LECaYrBJdBaHf9kRXANB8Dx3o= 2022-09-03T07:24:20.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S8LECaYrBJdBaHf9kRXANB8Dx3o=', 'Content-Length': '0'} 2022-09-03T07:24:20.272 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002dfb08a93fffae4f-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.273 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:24:20.273 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.274 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.275 DEBUG:boto:Method: GET 2022-09-03T07:24:20.275 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:20.276 DEBUG:boto:Data: 2022-09-03T07:24:20.276 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.277 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.278 DEBUG:boto:Params: {} 2022-09-03T07:24:20.279 DEBUG:boto:Token: None 2022-09-03T07:24:20.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34 2022-09-03T07:24:20.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:evF27LlaSAKGIvdr/oJVgVva6a8= 2022-09-03T07:24:20.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:evF27LlaSAKGIvdr/oJVgVva6a8=', 'Content-Length': '0'} 2022-09-03T07:24:20.282 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092bcbce77f3b7379-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.283 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:24:20.284 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:24:20.284 DEBUG:boto:Method: GET 2022-09-03T07:24:20.285 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:24:20.285 DEBUG:boto:Data: 2022-09-03T07:24:20.286 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.286 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.287 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.287 DEBUG:boto:Params: {} 2022-09-03T07:24:20.288 DEBUG:boto:Token: None 2022-09-03T07:24:20.289 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj34 2022-09-03T07:24:20.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:evF27LlaSAKGIvdr/oJVgVva6a8= 2022-09-03T07:24:20.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:evF27LlaSAKGIvdr/oJVgVva6a8=', 'Content-Length': '0'} 2022-09-03T07:24:20.292 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000045a438e72a9b8ce-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.293 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.293 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.294 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:20.294 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:20.295 DEBUG:boto:Method: GET 2022-09-03T07:24:20.295 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:20.296 DEBUG:boto:Data: 2022-09-03T07:24:20.297 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.297 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.298 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.298 DEBUG:boto:Params: {} 2022-09-03T07:24:20.299 DEBUG:boto:Token: None 2022-09-03T07:24:20.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.303 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000162a02eee7a57e78-0063130124-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.346 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.348 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.349 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.350 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:20.351 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:24:20.352 DEBUG:boto:Method: GET 2022-09-03T07:24:20.353 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:24:20.354 DEBUG:boto:Data: 2022-09-03T07:24:20.355 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.356 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.357 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.358 DEBUG:boto:Params: {} 2022-09-03T07:24:20.359 DEBUG:boto:Token: None 2022-09-03T07:24:20.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.366 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b26d86859f81f74c-0063130124-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.367 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.368 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:24:20.369 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.370 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.371 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.372 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.373 DEBUG:boto:Method: GET 2022-09-03T07:24:20.374 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.375 DEBUG:boto:Data: 2022-09-03T07:24:20.376 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.378 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.379 DEBUG:boto:Params: {} 2022-09-03T07:24:20.380 DEBUG:boto:Token: None 2022-09-03T07:24:20.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9DzFZ5Tu/Vw4pc04FBEoXsxAUBM= 2022-09-03T07:24:20.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9DzFZ5Tu/Vw4pc04FBEoXsxAUBM=', 'Content-Length': '0'} 2022-09-03T07:24:20.387 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fa166e5fc513ba4-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.388 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.389 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.390 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.391 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.392 DEBUG:boto:Method: GET 2022-09-03T07:24:20.393 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.394 DEBUG:boto:Data: 2022-09-03T07:24:20.395 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.395 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.396 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.397 DEBUG:boto:Params: {} 2022-09-03T07:24:20.398 DEBUG:boto:Token: None 2022-09-03T07:24:20.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9DzFZ5Tu/Vw4pc04FBEoXsxAUBM= 2022-09-03T07:24:20.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9DzFZ5Tu/Vw4pc04FBEoXsxAUBM=', 'Content-Length': '0'} 2022-09-03T07:24:20.401 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d0457e4843b90094-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.402 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.403 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.403 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.404 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.404 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.405 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.405 DEBUG:boto:Data: 2022-09-03T07:24:20.405 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.406 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.407 DEBUG:boto:Params: {} 2022-09-03T07:24:20.408 DEBUG:boto:Token: None 2022-09-03T07:24:20.408 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ckwPxc1C3C5UyHjSM+L42ya+PmE= 2022-09-03T07:24:20.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ckwPxc1C3C5UyHjSM+L42ya+PmE=', 'Content-Length': '0'} 2022-09-03T07:24:20.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e037dc96e60a7f20-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.412 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.413 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.413 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.414 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.414 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.415 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.415 DEBUG:boto:Data: 2022-09-03T07:24:20.416 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.416 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.417 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.417 DEBUG:boto:Params: {} 2022-09-03T07:24:20.418 DEBUG:boto:Token: None 2022-09-03T07:24:20.418 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ckwPxc1C3C5UyHjSM+L42ya+PmE= 2022-09-03T07:24:20.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ckwPxc1C3C5UyHjSM+L42ya+PmE=', 'Content-Length': '0'} 2022-09-03T07:24:20.421 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a28c97f0ee66b0f0-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.422 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:24:20.423 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.423 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.424 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.424 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.425 DEBUG:boto:Method: GET 2022-09-03T07:24:20.425 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.426 DEBUG:boto:Data: 2022-09-03T07:24:20.427 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.428 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.428 DEBUG:boto:Params: {} 2022-09-03T07:24:20.429 DEBUG:boto:Token: None 2022-09-03T07:24:20.429 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9DzFZ5Tu/Vw4pc04FBEoXsxAUBM= 2022-09-03T07:24:20.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9DzFZ5Tu/Vw4pc04FBEoXsxAUBM=', 'Content-Length': '0'} 2022-09-03T07:24:20.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c3d8bcb8854e01f-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.433 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.434 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.434 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.435 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.435 DEBUG:boto:Method: GET 2022-09-03T07:24:20.436 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.436 DEBUG:boto:Data: 2022-09-03T07:24:20.437 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.437 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.438 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.439 DEBUG:boto:Params: {} 2022-09-03T07:24:20.439 DEBUG:boto:Token: None 2022-09-03T07:24:20.440 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:24:20.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9DzFZ5Tu/Vw4pc04FBEoXsxAUBM= 2022-09-03T07:24:20.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9DzFZ5Tu/Vw4pc04FBEoXsxAUBM=', 'Content-Length': '0'} 2022-09-03T07:24:20.443 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007176ad3806f376d0-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.444 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.445 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.445 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:20.446 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:20.446 DEBUG:boto:Method: GET 2022-09-03T07:24:20.447 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:20.447 DEBUG:boto:Data: 2022-09-03T07:24:20.448 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.449 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.449 DEBUG:boto:Params: {} 2022-09-03T07:24:20.450 DEBUG:boto:Token: None 2022-09-03T07:24:20.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.454 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005c88edc0513fa0a8-0063130124-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.455 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.456 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.456 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.457 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:20.457 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:20.458 DEBUG:boto:Method: GET 2022-09-03T07:24:20.458 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:20.459 DEBUG:boto:Data: 2022-09-03T07:24:20.459 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.460 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.460 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.461 DEBUG:boto:Params: {} 2022-09-03T07:24:20.462 DEBUG:boto:Token: None 2022-09-03T07:24:20.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.466 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aeda86d55e2cfac2-0063130124-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.467 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.468 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.468 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.469 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.469 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:20.470 DEBUG:boto:Data: 2022-09-03T07:24:20.471 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.471 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.472 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.472 DEBUG:boto:Params: {} 2022-09-03T07:24:20.473 DEBUG:boto:Token: None 2022-09-03T07:24:20.474 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38 2022-09-03T07:24:20.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0cunYKa4xx1Ls5uxdtFV+NnG1+w= 2022-09-03T07:24:20.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0cunYKa4xx1Ls5uxdtFV+NnG1+w=', 'Content-Length': '0'} 2022-09-03T07:24:20.486 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ddfc2c1a358f8f7-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.486 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.487 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.487 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.488 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:20.488 DEBUG:boto:Data: 2022-09-03T07:24:20.489 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.489 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.490 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.490 DEBUG:boto:Params: {} 2022-09-03T07:24:20.491 DEBUG:boto:Token: None 2022-09-03T07:24:20.492 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38 2022-09-03T07:24:20.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0cunYKa4xx1Ls5uxdtFV+NnG1+w= 2022-09-03T07:24:20.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0cunYKa4xx1Ls5uxdtFV+NnG1+w=', 'Content-Length': '0'} 2022-09-03T07:24:20.494 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003bacdc0b34923e55-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.495 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:24:20.496 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.496 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.497 DEBUG:boto:Method: GET 2022-09-03T07:24:20.497 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:20.498 DEBUG:boto:Data: 2022-09-03T07:24:20.498 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.499 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.500 DEBUG:boto:Params: {} 2022-09-03T07:24:20.500 DEBUG:boto:Token: None 2022-09-03T07:24:20.501 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38 2022-09-03T07:24:20.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FlS1s3PpoTO0ahlVjjcOB7bytw= 2022-09-03T07:24:20.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FlS1s3PpoTO0ahlVjjcOB7bytw=', 'Content-Length': '0'} 2022-09-03T07:24:20.504 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c81ed15c1e0df65a-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.546 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:24:20.547 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:24:20.548 DEBUG:boto:Method: GET 2022-09-03T07:24:20.548 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:24:20.549 DEBUG:boto:Data: 2022-09-03T07:24:20.549 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.550 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.550 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.551 DEBUG:boto:Params: {} 2022-09-03T07:24:20.551 DEBUG:boto:Token: None 2022-09-03T07:24:20.552 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj38 2022-09-03T07:24:20.553 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FlS1s3PpoTO0ahlVjjcOB7bytw= 2022-09-03T07:24:20.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FlS1s3PpoTO0ahlVjjcOB7bytw=', 'Content-Length': '0'} 2022-09-03T07:24:20.555 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc2787023d866e79-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.556 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.557 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.557 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:20.558 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:20.558 DEBUG:boto:Method: GET 2022-09-03T07:24:20.559 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:20.560 DEBUG:boto:Data: 2022-09-03T07:24:20.560 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.561 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.561 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.562 DEBUG:boto:Params: {} 2022-09-03T07:24:20.562 DEBUG:boto:Token: None 2022-09-03T07:24:20.563 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.563 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.567 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0eaa0db1425428d-0063130124-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.568 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.569 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.569 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.570 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:20.571 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:24:20.571 DEBUG:boto:Method: GET 2022-09-03T07:24:20.572 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:24:20.572 DEBUG:boto:Data: 2022-09-03T07:24:20.573 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.573 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.574 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.575 DEBUG:boto:Params: {} 2022-09-03T07:24:20.575 DEBUG:boto:Token: None 2022-09-03T07:24:20.576 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.579 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007325e839aecc4b79-0063130124-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.581 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.582 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:24:20.583 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.586 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.587 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.588 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.589 DEBUG:boto:Method: GET 2022-09-03T07:24:20.590 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.591 DEBUG:boto:Data: 2022-09-03T07:24:20.592 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.593 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.594 DEBUG:boto:Params: {} 2022-09-03T07:24:20.596 DEBUG:boto:Token: None 2022-09-03T07:24:20.597 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.598 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wEKGB0F3TqtHbggUjlRczMEnnA4= 2022-09-03T07:24:20.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wEKGB0F3TqtHbggUjlRczMEnnA4=', 'Content-Length': '0'} 2022-09-03T07:24:20.602 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005cae2b726a958c42-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.603 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.604 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.605 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.606 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.607 DEBUG:boto:Method: GET 2022-09-03T07:24:20.608 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.609 DEBUG:boto:Data: 2022-09-03T07:24:20.610 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.611 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.612 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.613 DEBUG:boto:Params: {} 2022-09-03T07:24:20.614 DEBUG:boto:Token: None 2022-09-03T07:24:20.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wEKGB0F3TqtHbggUjlRczMEnnA4= 2022-09-03T07:24:20.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wEKGB0F3TqtHbggUjlRczMEnnA4=', 'Content-Length': '0'} 2022-09-03T07:24:20.620 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065356c5117e07278-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.621 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.622 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.623 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.624 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.625 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.626 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.627 DEBUG:boto:Data: 2022-09-03T07:24:20.628 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.629 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.630 DEBUG:boto:Params: {} 2022-09-03T07:24:20.631 DEBUG:boto:Token: None 2022-09-03T07:24:20.632 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F4EV6KBg6Acu4L1eaQhQs8FrXr8= 2022-09-03T07:24:20.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F4EV6KBg6Acu4L1eaQhQs8FrXr8=', 'Content-Length': '0'} 2022-09-03T07:24:20.636 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c77d0d6aa80d269-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.637 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.637 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.638 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.638 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.639 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.639 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.640 DEBUG:boto:Data: 2022-09-03T07:24:20.640 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.641 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.641 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.642 DEBUG:boto:Params: {} 2022-09-03T07:24:20.643 DEBUG:boto:Token: None 2022-09-03T07:24:20.643 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F4EV6KBg6Acu4L1eaQhQs8FrXr8= 2022-09-03T07:24:20.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F4EV6KBg6Acu4L1eaQhQs8FrXr8=', 'Content-Length': '0'} 2022-09-03T07:24:20.647 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dd7af16fef198e0b-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.647 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:24:20.648 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.649 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.649 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.650 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.651 DEBUG:boto:Method: GET 2022-09-03T07:24:20.652 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.653 DEBUG:boto:Data: 2022-09-03T07:24:20.654 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.656 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.657 DEBUG:boto:Params: {} 2022-09-03T07:24:20.658 DEBUG:boto:Token: None 2022-09-03T07:24:20.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wEKGB0F3TqtHbggUjlRczMEnnA4= 2022-09-03T07:24:20.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wEKGB0F3TqtHbggUjlRczMEnnA4=', 'Content-Length': '0'} 2022-09-03T07:24:20.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ba7cfb39dcbe685-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.664 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.665 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.666 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.666 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.667 DEBUG:boto:Method: GET 2022-09-03T07:24:20.668 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.668 DEBUG:boto:Data: 2022-09-03T07:24:20.669 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.669 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.670 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.671 DEBUG:boto:Params: {} 2022-09-03T07:24:20.671 DEBUG:boto:Token: None 2022-09-03T07:24:20.672 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:24:20.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wEKGB0F3TqtHbggUjlRczMEnnA4= 2022-09-03T07:24:20.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wEKGB0F3TqtHbggUjlRczMEnnA4=', 'Content-Length': '0'} 2022-09-03T07:24:20.676 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000043e34031e98b2f09-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.676 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.677 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.678 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:20.678 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:20.679 DEBUG:boto:Method: GET 2022-09-03T07:24:20.679 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:20.680 DEBUG:boto:Data: 2022-09-03T07:24:20.680 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.681 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.682 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.682 DEBUG:boto:Params: {} 2022-09-03T07:24:20.683 DEBUG:boto:Token: None 2022-09-03T07:24:20.684 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.685 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.688 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005a6910f7e60f3305-0063130124-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.730 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.731 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.732 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.732 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:20.733 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:20.733 DEBUG:boto:Method: GET 2022-09-03T07:24:20.734 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:20.734 DEBUG:boto:Data: 2022-09-03T07:24:20.735 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.736 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.736 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.737 DEBUG:boto:Params: {} 2022-09-03T07:24:20.737 DEBUG:boto:Token: None 2022-09-03T07:24:20.738 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.743 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e3bd30792481a9b6-0063130124-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.744 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.745 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.745 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.746 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.746 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:20.747 DEBUG:boto:Data: 2022-09-03T07:24:20.747 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.748 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.749 DEBUG:boto:Params: {} 2022-09-03T07:24:20.749 DEBUG:boto:Token: None 2022-09-03T07:24:20.750 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44 2022-09-03T07:24:20.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xlk+E8DDOcb/jvg8J+cnmmJwp6Y= 2022-09-03T07:24:20.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xlk+E8DDOcb/jvg8J+cnmmJwp6Y=', 'Content-Length': '0'} 2022-09-03T07:24:20.753 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000017c2fcc5a5b6bd42-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.754 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.755 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.755 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.756 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:20.757 DEBUG:boto:Data: 2022-09-03T07:24:20.757 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.758 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.759 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.759 DEBUG:boto:Params: {} 2022-09-03T07:24:20.760 DEBUG:boto:Token: None 2022-09-03T07:24:20.760 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44 2022-09-03T07:24:20.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xlk+E8DDOcb/jvg8J+cnmmJwp6Y= 2022-09-03T07:24:20.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xlk+E8DDOcb/jvg8J+cnmmJwp6Y=', 'Content-Length': '0'} 2022-09-03T07:24:20.764 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eeddc5528818087f-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.764 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:24:20.765 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.765 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.766 DEBUG:boto:Method: GET 2022-09-03T07:24:20.767 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:20.767 DEBUG:boto:Data: 2022-09-03T07:24:20.768 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.769 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.769 DEBUG:boto:Params: {} 2022-09-03T07:24:20.770 DEBUG:boto:Token: None 2022-09-03T07:24:20.771 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44 2022-09-03T07:24:20.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lv5c0+x0EbdEXtfdfiSCCP6uaEQ= 2022-09-03T07:24:20.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lv5c0+x0EbdEXtfdfiSCCP6uaEQ=', 'Content-Length': '0'} 2022-09-03T07:24:20.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b6a0f3a127468b7-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.775 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:24:20.775 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:24:20.776 DEBUG:boto:Method: GET 2022-09-03T07:24:20.777 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:24:20.777 DEBUG:boto:Data: 2022-09-03T07:24:20.778 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.778 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.779 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.779 DEBUG:boto:Params: {} 2022-09-03T07:24:20.780 DEBUG:boto:Token: None 2022-09-03T07:24:20.780 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj44 2022-09-03T07:24:20.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lv5c0+x0EbdEXtfdfiSCCP6uaEQ= 2022-09-03T07:24:20.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lv5c0+x0EbdEXtfdfiSCCP6uaEQ=', 'Content-Length': '0'} 2022-09-03T07:24:20.784 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056694fe55e54e812-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.785 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.785 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.786 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:20.786 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:20.787 DEBUG:boto:Method: GET 2022-09-03T07:24:20.788 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:20.788 DEBUG:boto:Data: 2022-09-03T07:24:20.789 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.789 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.790 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.790 DEBUG:boto:Params: {} 2022-09-03T07:24:20.791 DEBUG:boto:Token: None 2022-09-03T07:24:20.792 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.796 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1d394b5cc453444-0063130124-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.797 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.798 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.798 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.799 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:20.799 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:24:20.800 DEBUG:boto:Method: GET 2022-09-03T07:24:20.800 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:24:20.801 DEBUG:boto:Data: 2022-09-03T07:24:20.801 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.802 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.802 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.803 DEBUG:boto:Params: {} 2022-09-03T07:24:20.803 DEBUG:boto:Token: None 2022-09-03T07:24:20.804 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.805 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.809 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000099184c07aff07ae4-0063130124-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.810 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.811 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:24:20.811 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.812 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.812 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.813 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.813 DEBUG:boto:Method: GET 2022-09-03T07:24:20.814 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.814 DEBUG:boto:Data: 2022-09-03T07:24:20.815 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.816 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.816 DEBUG:boto:Params: {} 2022-09-03T07:24:20.817 DEBUG:boto:Token: None 2022-09-03T07:24:20.818 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZnnsDAkjHnRMMk4dioULsZfFdkc= 2022-09-03T07:24:20.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZnnsDAkjHnRMMk4dioULsZfFdkc=', 'Content-Length': '0'} 2022-09-03T07:24:20.821 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c525be814d85279-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.862 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.863 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.863 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.864 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.865 DEBUG:boto:Method: GET 2022-09-03T07:24:20.865 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.866 DEBUG:boto:Data: 2022-09-03T07:24:20.866 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.867 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.867 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.868 DEBUG:boto:Params: {} 2022-09-03T07:24:20.869 DEBUG:boto:Token: None 2022-09-03T07:24:20.869 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZnnsDAkjHnRMMk4dioULsZfFdkc= 2022-09-03T07:24:20.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZnnsDAkjHnRMMk4dioULsZfFdkc=', 'Content-Length': '0'} 2022-09-03T07:24:20.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b473829ec53de53f-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.874 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.875 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.876 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.876 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.877 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.878 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.879 DEBUG:boto:Data: 2022-09-03T07:24:20.879 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.880 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.881 DEBUG:boto:Params: {} 2022-09-03T07:24:20.881 DEBUG:boto:Token: None 2022-09-03T07:24:20.882 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xrsXjuADv3rLdylQQlxCz3xMRcY= 2022-09-03T07:24:20.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xrsXjuADv3rLdylQQlxCz3xMRcY=', 'Content-Length': '0'} 2022-09-03T07:24:20.886 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ef958c96c841e59-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.887 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.888 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.888 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.889 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.890 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.890 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.891 DEBUG:boto:Data: 2022-09-03T07:24:20.891 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.892 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.892 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.893 DEBUG:boto:Params: {} 2022-09-03T07:24:20.893 DEBUG:boto:Token: None 2022-09-03T07:24:20.894 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xrsXjuADv3rLdylQQlxCz3xMRcY= 2022-09-03T07:24:20.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xrsXjuADv3rLdylQQlxCz3xMRcY=', 'Content-Length': '0'} 2022-09-03T07:24:20.898 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6d26a4fc53d17f7-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.898 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:24:20.899 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.900 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.900 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.901 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.902 DEBUG:boto:Method: GET 2022-09-03T07:24:20.903 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.903 DEBUG:boto:Data: 2022-09-03T07:24:20.904 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.905 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.906 DEBUG:boto:Params: {} 2022-09-03T07:24:20.907 DEBUG:boto:Token: None 2022-09-03T07:24:20.908 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZnnsDAkjHnRMMk4dioULsZfFdkc= 2022-09-03T07:24:20.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZnnsDAkjHnRMMk4dioULsZfFdkc=', 'Content-Length': '0'} 2022-09-03T07:24:20.912 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006cb366bbb96f5c70-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.914 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.915 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.915 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.916 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.917 DEBUG:boto:Method: GET 2022-09-03T07:24:20.918 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.919 DEBUG:boto:Data: 2022-09-03T07:24:20.919 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.920 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.921 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.921 DEBUG:boto:Params: {} 2022-09-03T07:24:20.922 DEBUG:boto:Token: None 2022-09-03T07:24:20.923 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:24:20.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZnnsDAkjHnRMMk4dioULsZfFdkc= 2022-09-03T07:24:20.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZnnsDAkjHnRMMk4dioULsZfFdkc=', 'Content-Length': '0'} 2022-09-03T07:24:20.928 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d813630b614cf1f3-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.929 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.930 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.931 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:20.931 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:20.932 DEBUG:boto:Method: GET 2022-09-03T07:24:20.932 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:20.933 DEBUG:boto:Data: 2022-09-03T07:24:20.934 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.935 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.935 DEBUG:boto:Params: {} 2022-09-03T07:24:20.936 DEBUG:boto:Token: None 2022-09-03T07:24:20.936 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.942 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c84a5588ef453289-0063130124-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.943 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.944 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:20.944 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:20.945 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:20.946 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:20.946 DEBUG:boto:Method: GET 2022-09-03T07:24:20.947 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:20.948 DEBUG:boto:Data: 2022-09-03T07:24:20.948 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.949 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.950 DEBUG:boto:Params: {} 2022-09-03T07:24:20.951 DEBUG:boto:Token: None 2022-09-03T07:24:20.952 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/ 2022-09-03T07:24:20.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw= 2022-09-03T07:24:20.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aJIjpfq2Gf8OvIRdHi9uJvZmJJw=', 'Content-Length': '0'} 2022-09-03T07:24:20.957 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000667ab812ebac9fda-0063130124-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.959 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:20.960 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.960 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.961 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.961 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:20.962 DEBUG:boto:Data: 2022-09-03T07:24:20.962 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.963 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.964 DEBUG:boto:Params: {} 2022-09-03T07:24:20.965 DEBUG:boto:Token: None 2022-09-03T07:24:20.965 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj46 2022-09-03T07:24:20.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRMyHqUcG+v3SDRNGx4bsowb84U= 2022-09-03T07:24:20.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRMyHqUcG+v3SDRNGx4bsowb84U=', 'Content-Length': '0'} 2022-09-03T07:24:20.969 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c2f827db583fa0c-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.970 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.970 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.971 DEBUG:boto:Method: HEAD 2022-09-03T07:24:20.971 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:20.972 DEBUG:boto:Data: 2022-09-03T07:24:20.972 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.973 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:20.973 DEBUG:boto:Port: 8000 2022-09-03T07:24:20.974 DEBUG:boto:Params: {} 2022-09-03T07:24:20.974 DEBUG:boto:Token: None 2022-09-03T07:24:20.975 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj46 2022-09-03T07:24:20.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LRMyHqUcG+v3SDRNGx4bsowb84U= 2022-09-03T07:24:20.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LRMyHqUcG+v3SDRNGx4bsowb84U=', 'Content-Length': '0'} 2022-09-03T07:24:20.978 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e8cee76bac3ea882-0063130124-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.979 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:24:20.980 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.980 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.981 DEBUG:boto:Method: GET 2022-09-03T07:24:20.981 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:20.982 DEBUG:boto:Data: 2022-09-03T07:24:20.982 DEBUG:boto:Headers: {} 2022-09-03T07:24:20.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:20.983 DEBUG:boto:Port: 8001 2022-09-03T07:24:20.984 DEBUG:boto:Params: {} 2022-09-03T07:24:20.985 DEBUG:boto:Token: None 2022-09-03T07:24:20.985 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:20 GMT /avysmw-75/obj46 2022-09-03T07:24:20.986 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VYBu7ja8SHfsZLLJ24TXcVp2sjQ= 2022-09-03T07:24:20.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VYBu7ja8SHfsZLLJ24TXcVp2sjQ=', 'Content-Length': '0'} 2022-09-03T07:24:20.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e6f86ac5e289ad3-0063130124-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:20.996 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:24:20.997 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:24:20.997 DEBUG:boto:Method: GET 2022-09-03T07:24:20.998 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:24:20.999 DEBUG:boto:Data: 2022-09-03T07:24:21.000 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.000 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.001 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.001 DEBUG:boto:Params: {} 2022-09-03T07:24:21.002 DEBUG:boto:Token: None 2022-09-03T07:24:21.003 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj46 2022-09-03T07:24:21.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VHz7BBaEmNX/hIyLdJkxrshxp4U= 2022-09-03T07:24:21.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VHz7BBaEmNX/hIyLdJkxrshxp4U=', 'Content-Length': '0'} 2022-09-03T07:24:21.007 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020351193f7563377-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.054 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.055 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.055 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:21.056 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:21.056 DEBUG:boto:Method: GET 2022-09-03T07:24:21.057 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:21.057 DEBUG:boto:Data: 2022-09-03T07:24:21.058 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.058 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.059 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.060 DEBUG:boto:Params: {} 2022-09-03T07:24:21.061 DEBUG:boto:Token: None 2022-09-03T07:24:21.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.067 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a5edf76f3c6a0a79-0063130125-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.068 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.070 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.071 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.072 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:21.073 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:24:21.074 DEBUG:boto:Method: GET 2022-09-03T07:24:21.075 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:24:21.076 DEBUG:boto:Data: 2022-09-03T07:24:21.077 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.078 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.078 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.079 DEBUG:boto:Params: {} 2022-09-03T07:24:21.080 DEBUG:boto:Token: None 2022-09-03T07:24:21.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000000efae60eb840c00-0063130125-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.085 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.086 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:24:21.086 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.087 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.088 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.088 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.089 DEBUG:boto:Method: GET 2022-09-03T07:24:21.090 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.091 DEBUG:boto:Data: 2022-09-03T07:24:21.091 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.093 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.093 DEBUG:boto:Params: {} 2022-09-03T07:24:21.094 DEBUG:boto:Token: None 2022-09-03T07:24:21.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ft2gmAMZND8yiijgTpVLGspqg= 2022-09-03T07:24:21.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ft2gmAMZND8yiijgTpVLGspqg=', 'Content-Length': '0'} 2022-09-03T07:24:21.098 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d152b73a55af4d9c-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.142 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.143 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.144 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.144 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.145 DEBUG:boto:Method: GET 2022-09-03T07:24:21.146 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.146 DEBUG:boto:Data: 2022-09-03T07:24:21.147 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.148 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.148 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.149 DEBUG:boto:Params: {} 2022-09-03T07:24:21.149 DEBUG:boto:Token: None 2022-09-03T07:24:21.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.151 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ft2gmAMZND8yiijgTpVLGspqg= 2022-09-03T07:24:21.152 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ft2gmAMZND8yiijgTpVLGspqg=', 'Content-Length': '0'} 2022-09-03T07:24:21.155 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000021cb0bfe2bd51761-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.156 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.157 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.158 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.159 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.160 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.160 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.161 DEBUG:boto:Data: 2022-09-03T07:24:21.161 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.163 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.163 DEBUG:boto:Params: {} 2022-09-03T07:24:21.164 DEBUG:boto:Token: None 2022-09-03T07:24:21.165 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:unGwOfaVv91tU8LhW1mME56v0Aw= 2022-09-03T07:24:21.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:unGwOfaVv91tU8LhW1mME56v0Aw=', 'Content-Length': '0'} 2022-09-03T07:24:21.169 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016e421bddb36540d-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.170 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.171 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.172 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.173 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.174 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.175 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.175 DEBUG:boto:Data: 2022-09-03T07:24:21.176 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.177 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.178 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.179 DEBUG:boto:Params: {} 2022-09-03T07:24:21.180 DEBUG:boto:Token: None 2022-09-03T07:24:21.181 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:unGwOfaVv91tU8LhW1mME56v0Aw= 2022-09-03T07:24:21.183 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:unGwOfaVv91tU8LhW1mME56v0Aw=', 'Content-Length': '0'} 2022-09-03T07:24:21.186 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057b67035b629722d-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.187 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:24:21.188 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.189 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.189 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.190 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.191 DEBUG:boto:Method: GET 2022-09-03T07:24:21.192 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.193 DEBUG:boto:Data: 2022-09-03T07:24:21.194 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.195 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.196 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.197 DEBUG:boto:Params: {} 2022-09-03T07:24:21.198 DEBUG:boto:Token: None 2022-09-03T07:24:21.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ft2gmAMZND8yiijgTpVLGspqg= 2022-09-03T07:24:21.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ft2gmAMZND8yiijgTpVLGspqg=', 'Content-Length': '0'} 2022-09-03T07:24:21.203 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000adefd5478179fcc5-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.205 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.205 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.206 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.207 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.210 DEBUG:boto:Method: GET 2022-09-03T07:24:21.211 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.213 DEBUG:boto:Data: 2022-09-03T07:24:21.213 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.214 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.214 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.215 DEBUG:boto:Params: {} 2022-09-03T07:24:21.215 DEBUG:boto:Token: None 2022-09-03T07:24:21.216 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:24:21.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o9ft2gmAMZND8yiijgTpVLGspqg= 2022-09-03T07:24:21.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o9ft2gmAMZND8yiijgTpVLGspqg=', 'Content-Length': '0'} 2022-09-03T07:24:21.219 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ce3e51118189b41-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.220 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.221 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.221 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:21.222 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:21.222 DEBUG:boto:Method: GET 2022-09-03T07:24:21.223 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:21.224 DEBUG:boto:Data: 2022-09-03T07:24:21.224 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.225 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.225 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.226 DEBUG:boto:Params: {} 2022-09-03T07:24:21.226 DEBUG:boto:Token: None 2022-09-03T07:24:21.227 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.227 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.231 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000048f9bf21e500f747-0063130125-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.232 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.232 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.233 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.234 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:21.234 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:21.235 DEBUG:boto:Method: GET 2022-09-03T07:24:21.236 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:21.237 DEBUG:boto:Data: 2022-09-03T07:24:21.237 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.238 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.239 DEBUG:boto:Params: {} 2022-09-03T07:24:21.239 DEBUG:boto:Token: None 2022-09-03T07:24:21.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.244 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000baffb48fe752aea0-0063130125-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.245 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.245 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.246 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.247 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.248 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:21.249 DEBUG:boto:Data: 2022-09-03T07:24:21.249 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.250 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.251 DEBUG:boto:Params: {} 2022-09-03T07:24:21.251 DEBUG:boto:Token: None 2022-09-03T07:24:21.252 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51 2022-09-03T07:24:21.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XJpe25tQNAcmzebs89nffZdrupU= 2022-09-03T07:24:21.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XJpe25tQNAcmzebs89nffZdrupU=', 'Content-Length': '0'} 2022-09-03T07:24:21.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d798e1300d701582-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.256 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.257 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.257 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.258 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:21.258 DEBUG:boto:Data: 2022-09-03T07:24:21.259 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.259 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.260 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.260 DEBUG:boto:Params: {} 2022-09-03T07:24:21.261 DEBUG:boto:Token: None 2022-09-03T07:24:21.262 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51 2022-09-03T07:24:21.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XJpe25tQNAcmzebs89nffZdrupU= 2022-09-03T07:24:21.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XJpe25tQNAcmzebs89nffZdrupU=', 'Content-Length': '0'} 2022-09-03T07:24:21.265 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087ad97660ca94c32-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.266 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:24:21.266 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.267 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.267 DEBUG:boto:Method: GET 2022-09-03T07:24:21.268 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:21.268 DEBUG:boto:Data: 2022-09-03T07:24:21.269 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.270 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.270 DEBUG:boto:Params: {} 2022-09-03T07:24:21.271 DEBUG:boto:Token: None 2022-09-03T07:24:21.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51 2022-09-03T07:24:21.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PHDAKCLf+IjFS1gg3RRClGMqqCY= 2022-09-03T07:24:21.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PHDAKCLf+IjFS1gg3RRClGMqqCY=', 'Content-Length': '0'} 2022-09-03T07:24:21.275 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007a350b19f805491c-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.318 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:24:21.319 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:24:21.320 DEBUG:boto:Method: GET 2022-09-03T07:24:21.320 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:24:21.321 DEBUG:boto:Data: 2022-09-03T07:24:21.321 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.322 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.322 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.323 DEBUG:boto:Params: {} 2022-09-03T07:24:21.324 DEBUG:boto:Token: None 2022-09-03T07:24:21.324 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj51 2022-09-03T07:24:21.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PHDAKCLf+IjFS1gg3RRClGMqqCY= 2022-09-03T07:24:21.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PHDAKCLf+IjFS1gg3RRClGMqqCY=', 'Content-Length': '0'} 2022-09-03T07:24:21.328 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000219ca994e210a832-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.328 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.329 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.329 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:21.330 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:21.331 DEBUG:boto:Method: GET 2022-09-03T07:24:21.331 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:21.332 DEBUG:boto:Data: 2022-09-03T07:24:21.332 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.333 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.333 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.334 DEBUG:boto:Params: {} 2022-09-03T07:24:21.335 DEBUG:boto:Token: None 2022-09-03T07:24:21.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.339 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000055f9bf6d27277b4a-0063130125-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.340 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.341 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.342 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.342 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:21.343 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:24:21.343 DEBUG:boto:Method: GET 2022-09-03T07:24:21.344 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:24:21.344 DEBUG:boto:Data: 2022-09-03T07:24:21.345 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.346 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.346 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.347 DEBUG:boto:Params: {} 2022-09-03T07:24:21.347 DEBUG:boto:Token: None 2022-09-03T07:24:21.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.352 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a32af56ca6042b6b-0063130125-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.353 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.353 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:24:21.354 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.355 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.355 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.356 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.356 DEBUG:boto:Method: GET 2022-09-03T07:24:21.357 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.357 DEBUG:boto:Data: 2022-09-03T07:24:21.358 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.359 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.359 DEBUG:boto:Params: {} 2022-09-03T07:24:21.360 DEBUG:boto:Token: None 2022-09-03T07:24:21.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s03rIfYhIC9goSs9ky1gqMBQM0U= 2022-09-03T07:24:21.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s03rIfYhIC9goSs9ky1gqMBQM0U=', 'Content-Length': '0'} 2022-09-03T07:24:21.365 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000127e829b9567a811-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.365 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.366 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.367 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.367 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.368 DEBUG:boto:Method: GET 2022-09-03T07:24:21.368 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.369 DEBUG:boto:Data: 2022-09-03T07:24:21.369 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.370 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.370 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.371 DEBUG:boto:Params: {} 2022-09-03T07:24:21.372 DEBUG:boto:Token: None 2022-09-03T07:24:21.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s03rIfYhIC9goSs9ky1gqMBQM0U= 2022-09-03T07:24:21.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s03rIfYhIC9goSs9ky1gqMBQM0U=', 'Content-Length': '0'} 2022-09-03T07:24:21.375 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f093d90932d02081-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.376 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.377 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.377 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.378 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.379 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.379 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.380 DEBUG:boto:Data: 2022-09-03T07:24:21.380 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.381 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.381 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.382 DEBUG:boto:Params: {} 2022-09-03T07:24:21.382 DEBUG:boto:Token: None 2022-09-03T07:24:21.383 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oc306haWKNySyqJ2+bcpvxKpdnM= 2022-09-03T07:24:21.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oc306haWKNySyqJ2+bcpvxKpdnM=', 'Content-Length': '0'} 2022-09-03T07:24:21.386 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e725ea4737f4295-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.387 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.388 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.388 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.389 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.389 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.390 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.390 DEBUG:boto:Data: 2022-09-03T07:24:21.391 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.391 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.392 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.393 DEBUG:boto:Params: {} 2022-09-03T07:24:21.393 DEBUG:boto:Token: None 2022-09-03T07:24:21.394 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Oc306haWKNySyqJ2+bcpvxKpdnM= 2022-09-03T07:24:21.395 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Oc306haWKNySyqJ2+bcpvxKpdnM=', 'Content-Length': '0'} 2022-09-03T07:24:21.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ba248c86828f7a2-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.398 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:24:21.398 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.399 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.399 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.400 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.400 DEBUG:boto:Method: GET 2022-09-03T07:24:21.401 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.402 DEBUG:boto:Data: 2022-09-03T07:24:21.402 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.403 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.403 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.404 DEBUG:boto:Params: {} 2022-09-03T07:24:21.405 DEBUG:boto:Token: None 2022-09-03T07:24:21.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s03rIfYhIC9goSs9ky1gqMBQM0U= 2022-09-03T07:24:21.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s03rIfYhIC9goSs9ky1gqMBQM0U=', 'Content-Length': '0'} 2022-09-03T07:24:21.409 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b400de887a45915-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.450 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.451 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.451 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.452 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.452 DEBUG:boto:Method: GET 2022-09-03T07:24:21.453 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.454 DEBUG:boto:Data: 2022-09-03T07:24:21.454 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.455 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.455 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.456 DEBUG:boto:Params: {} 2022-09-03T07:24:21.457 DEBUG:boto:Token: None 2022-09-03T07:24:21.457 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:24:21.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s03rIfYhIC9goSs9ky1gqMBQM0U= 2022-09-03T07:24:21.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s03rIfYhIC9goSs9ky1gqMBQM0U=', 'Content-Length': '0'} 2022-09-03T07:24:21.461 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6ce56104db1c8e9-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.462 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.462 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.463 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:21.463 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:21.464 DEBUG:boto:Method: GET 2022-09-03T07:24:21.464 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:21.465 DEBUG:boto:Data: 2022-09-03T07:24:21.465 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.466 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.467 DEBUG:boto:Params: {} 2022-09-03T07:24:21.468 DEBUG:boto:Token: None 2022-09-03T07:24:21.468 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.472 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002dce3454c2631034-0063130125-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.473 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.474 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.475 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.475 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:21.476 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:21.476 DEBUG:boto:Method: GET 2022-09-03T07:24:21.477 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:21.477 DEBUG:boto:Data: 2022-09-03T07:24:21.478 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.478 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.479 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.479 DEBUG:boto:Params: {} 2022-09-03T07:24:21.480 DEBUG:boto:Token: None 2022-09-03T07:24:21.481 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.484 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000074da650608d505d6-0063130125-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.485 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.486 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.486 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.487 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.487 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:21.488 DEBUG:boto:Data: 2022-09-03T07:24:21.488 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.489 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.490 DEBUG:boto:Params: {} 2022-09-03T07:24:21.490 DEBUG:boto:Token: None 2022-09-03T07:24:21.491 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55 2022-09-03T07:24:21.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V9ns7PHMWFmDiAyIA7QHWN2XMa8= 2022-09-03T07:24:21.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V9ns7PHMWFmDiAyIA7QHWN2XMa8=', 'Content-Length': '0'} 2022-09-03T07:24:21.494 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000690a96bdc8343f43-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.495 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.496 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.496 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.497 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:21.497 DEBUG:boto:Data: 2022-09-03T07:24:21.498 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.498 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.499 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.499 DEBUG:boto:Params: {} 2022-09-03T07:24:21.500 DEBUG:boto:Token: None 2022-09-03T07:24:21.501 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55 2022-09-03T07:24:21.501 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V9ns7PHMWFmDiAyIA7QHWN2XMa8= 2022-09-03T07:24:21.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V9ns7PHMWFmDiAyIA7QHWN2XMa8=', 'Content-Length': '0'} 2022-09-03T07:24:21.504 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e373e2258c3c4ac8-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.504 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:24:21.505 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.505 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.506 DEBUG:boto:Method: GET 2022-09-03T07:24:21.506 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:21.507 DEBUG:boto:Data: 2022-09-03T07:24:21.508 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.509 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.509 DEBUG:boto:Params: {} 2022-09-03T07:24:21.510 DEBUG:boto:Token: None 2022-09-03T07:24:21.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55 2022-09-03T07:24:21.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zF1PoCLUYJcijQCYVBL2xt3iHMk= 2022-09-03T07:24:21.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zF1PoCLUYJcijQCYVBL2xt3iHMk=', 'Content-Length': '0'} 2022-09-03T07:24:21.514 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1f0f552bd3326b4-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.515 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:24:21.516 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:24:21.517 DEBUG:boto:Method: GET 2022-09-03T07:24:21.517 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:24:21.518 DEBUG:boto:Data: 2022-09-03T07:24:21.519 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.519 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.520 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.521 DEBUG:boto:Params: {} 2022-09-03T07:24:21.522 DEBUG:boto:Token: None 2022-09-03T07:24:21.522 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj55 2022-09-03T07:24:21.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zF1PoCLUYJcijQCYVBL2xt3iHMk= 2022-09-03T07:24:21.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zF1PoCLUYJcijQCYVBL2xt3iHMk=', 'Content-Length': '0'} 2022-09-03T07:24:21.526 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d765da8d6255031-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.527 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.528 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.528 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:21.529 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:21.529 DEBUG:boto:Method: GET 2022-09-03T07:24:21.530 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:21.530 DEBUG:boto:Data: 2022-09-03T07:24:21.531 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.532 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.532 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.533 DEBUG:boto:Params: {} 2022-09-03T07:24:21.534 DEBUG:boto:Token: None 2022-09-03T07:24:21.534 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.538 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f148b79d74fe75a8-0063130125-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.582 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.583 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.584 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.584 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:21.585 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:24:21.585 DEBUG:boto:Method: GET 2022-09-03T07:24:21.586 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:24:21.587 DEBUG:boto:Data: 2022-09-03T07:24:21.587 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.588 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.589 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.589 DEBUG:boto:Params: {} 2022-09-03T07:24:21.590 DEBUG:boto:Token: None 2022-09-03T07:24:21.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000061360539be4b9b13-0063130125-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.595 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.597 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:24:21.598 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.599 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.599 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.600 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.600 DEBUG:boto:Method: GET 2022-09-03T07:24:21.601 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.602 DEBUG:boto:Data: 2022-09-03T07:24:21.602 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.604 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.604 DEBUG:boto:Params: {} 2022-09-03T07:24:21.605 DEBUG:boto:Token: None 2022-09-03T07:24:21.606 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jX4cBplj5SWuef8mWhNpkHoP8/k= 2022-09-03T07:24:21.607 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jX4cBplj5SWuef8mWhNpkHoP8/k=', 'Content-Length': '0'} 2022-09-03T07:24:21.610 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9dd827893cf32c3-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.610 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.611 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.612 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.613 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.613 DEBUG:boto:Method: GET 2022-09-03T07:24:21.614 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.615 DEBUG:boto:Data: 2022-09-03T07:24:21.615 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.616 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.617 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.618 DEBUG:boto:Params: {} 2022-09-03T07:24:21.620 DEBUG:boto:Token: None 2022-09-03T07:24:21.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jX4cBplj5SWuef8mWhNpkHoP8/k= 2022-09-03T07:24:21.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jX4cBplj5SWuef8mWhNpkHoP8/k=', 'Content-Length': '0'} 2022-09-03T07:24:21.626 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b054ab484c225e84-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.670 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.672 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.673 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.674 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.675 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.676 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.678 DEBUG:boto:Data: 2022-09-03T07:24:21.679 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.680 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.680 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.681 DEBUG:boto:Params: {} 2022-09-03T07:24:21.682 DEBUG:boto:Token: None 2022-09-03T07:24:21.683 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yhQPhJ7kG2zzbzJfCMu9795URT0= 2022-09-03T07:24:21.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yhQPhJ7kG2zzbzJfCMu9795URT0=', 'Content-Length': '0'} 2022-09-03T07:24:21.688 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9e705e094ccf3fb-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.689 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.690 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.691 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.691 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.692 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.692 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.693 DEBUG:boto:Data: 2022-09-03T07:24:21.695 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.696 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.697 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.697 DEBUG:boto:Params: {} 2022-09-03T07:24:21.698 DEBUG:boto:Token: None 2022-09-03T07:24:21.699 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yhQPhJ7kG2zzbzJfCMu9795URT0= 2022-09-03T07:24:21.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yhQPhJ7kG2zzbzJfCMu9795URT0=', 'Content-Length': '0'} 2022-09-03T07:24:21.703 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a455dd86f130d3c7-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.704 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:24:21.704 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.705 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.705 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.706 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.707 DEBUG:boto:Method: GET 2022-09-03T07:24:21.707 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.708 DEBUG:boto:Data: 2022-09-03T07:24:21.709 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.710 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.711 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.712 DEBUG:boto:Params: {} 2022-09-03T07:24:21.713 DEBUG:boto:Token: None 2022-09-03T07:24:21.714 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jX4cBplj5SWuef8mWhNpkHoP8/k= 2022-09-03T07:24:21.716 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jX4cBplj5SWuef8mWhNpkHoP8/k=', 'Content-Length': '0'} 2022-09-03T07:24:21.718 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfe33bcbdf4c12b2-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.720 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.721 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.722 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.723 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.724 DEBUG:boto:Method: GET 2022-09-03T07:24:21.724 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.725 DEBUG:boto:Data: 2022-09-03T07:24:21.726 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.727 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.728 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.729 DEBUG:boto:Params: {} 2022-09-03T07:24:21.730 DEBUG:boto:Token: None 2022-09-03T07:24:21.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:24:21.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jX4cBplj5SWuef8mWhNpkHoP8/k= 2022-09-03T07:24:21.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jX4cBplj5SWuef8mWhNpkHoP8/k=', 'Content-Length': '0'} 2022-09-03T07:24:21.735 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f42a507c3b886ae-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.736 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.737 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.737 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:21.738 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:21.738 DEBUG:boto:Method: GET 2022-09-03T07:24:21.739 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:21.740 DEBUG:boto:Data: 2022-09-03T07:24:21.740 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.741 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.741 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.742 DEBUG:boto:Params: {} 2022-09-03T07:24:21.742 DEBUG:boto:Token: None 2022-09-03T07:24:21.743 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.743 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.744 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.747 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000353bd7bd02b2cd88-0063130125-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.747 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.748 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.749 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.749 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:21.750 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:21.750 DEBUG:boto:Method: GET 2022-09-03T07:24:21.751 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:21.751 DEBUG:boto:Data: 2022-09-03T07:24:21.752 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.752 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.753 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.753 DEBUG:boto:Params: {} 2022-09-03T07:24:21.754 DEBUG:boto:Token: None 2022-09-03T07:24:21.755 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.755 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.760 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000053434912ffd9347e-0063130125-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.760 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.761 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.762 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.762 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.763 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:21.763 DEBUG:boto:Data: 2022-09-03T07:24:21.764 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.764 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.765 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.765 DEBUG:boto:Params: {} 2022-09-03T07:24:21.766 DEBUG:boto:Token: None 2022-09-03T07:24:21.766 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57 2022-09-03T07:24:21.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3gIYBSTc8ZDKGMfAoZ8ya3yD+RE= 2022-09-03T07:24:21.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3gIYBSTc8ZDKGMfAoZ8ya3yD+RE=', 'Content-Length': '0'} 2022-09-03T07:24:21.770 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bb8e79be1f7e7a24-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.771 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.771 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.772 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.772 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:21.773 DEBUG:boto:Data: 2022-09-03T07:24:21.773 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.774 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.774 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.775 DEBUG:boto:Params: {} 2022-09-03T07:24:21.776 DEBUG:boto:Token: None 2022-09-03T07:24:21.776 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57 2022-09-03T07:24:21.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3gIYBSTc8ZDKGMfAoZ8ya3yD+RE= 2022-09-03T07:24:21.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3gIYBSTc8ZDKGMfAoZ8ya3yD+RE=', 'Content-Length': '0'} 2022-09-03T07:24:21.779 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb1e3bd00f5c8dd7-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.780 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:24:21.780 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.781 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.781 DEBUG:boto:Method: GET 2022-09-03T07:24:21.782 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:21.782 DEBUG:boto:Data: 2022-09-03T07:24:21.783 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.784 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.784 DEBUG:boto:Params: {} 2022-09-03T07:24:21.785 DEBUG:boto:Token: None 2022-09-03T07:24:21.786 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57 2022-09-03T07:24:21.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ximJ0RJSWul4e0ectkUfAOLzb8= 2022-09-03T07:24:21.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ximJ0RJSWul4e0ectkUfAOLzb8=', 'Content-Length': '0'} 2022-09-03T07:24:21.789 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007826284ac1a7ca2a-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.830 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:24:21.831 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:24:21.831 DEBUG:boto:Method: GET 2022-09-03T07:24:21.832 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:24:21.832 DEBUG:boto:Data: 2022-09-03T07:24:21.833 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.833 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.834 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.835 DEBUG:boto:Params: {} 2022-09-03T07:24:21.835 DEBUG:boto:Token: None 2022-09-03T07:24:21.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj57 2022-09-03T07:24:21.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ximJ0RJSWul4e0ectkUfAOLzb8= 2022-09-03T07:24:21.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ximJ0RJSWul4e0ectkUfAOLzb8=', 'Content-Length': '0'} 2022-09-03T07:24:21.841 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf6eefcdbacadf29-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.843 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.843 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.844 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:21.844 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:21.845 DEBUG:boto:Method: GET 2022-09-03T07:24:21.845 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:21.846 DEBUG:boto:Data: 2022-09-03T07:24:21.846 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.847 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.848 DEBUG:boto:Params: {} 2022-09-03T07:24:21.849 DEBUG:boto:Token: None 2022-09-03T07:24:21.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.853 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ea14cdaecf47822e-0063130125-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.854 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.855 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.855 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.856 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:21.856 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:24:21.857 DEBUG:boto:Method: GET 2022-09-03T07:24:21.858 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:24:21.858 DEBUG:boto:Data: 2022-09-03T07:24:21.859 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.859 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.860 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.860 DEBUG:boto:Params: {} 2022-09-03T07:24:21.861 DEBUG:boto:Token: None 2022-09-03T07:24:21.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.862 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.865 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003a0bbb7548520f4d-0063130125-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.867 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.867 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:24:21.868 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:21.868 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:21.869 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.869 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.870 DEBUG:boto:Method: GET 2022-09-03T07:24:21.871 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.871 DEBUG:boto:Data: 2022-09-03T07:24:21.872 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.873 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.873 DEBUG:boto:Params: {} 2022-09-03T07:24:21.874 DEBUG:boto:Token: None 2022-09-03T07:24:21.874 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.875 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:++p+7lvBipZbIUMdPImt+EV5nxk= 2022-09-03T07:24:21.876 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:++p+7lvBipZbIUMdPImt+EV5nxk=', 'Content-Length': '0'} 2022-09-03T07:24:21.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe16e047276541ef-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.879 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:21.879 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:21.880 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.880 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.881 DEBUG:boto:Method: GET 2022-09-03T07:24:21.881 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.882 DEBUG:boto:Data: 2022-09-03T07:24:21.882 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.883 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.883 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.884 DEBUG:boto:Params: {} 2022-09-03T07:24:21.885 DEBUG:boto:Token: None 2022-09-03T07:24:21.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:++p+7lvBipZbIUMdPImt+EV5nxk= 2022-09-03T07:24:21.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:++p+7lvBipZbIUMdPImt+EV5nxk=', 'Content-Length': '0'} 2022-09-03T07:24:21.888 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d98196c22f9e8b51-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.889 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:21.890 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:21.890 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.891 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.891 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.892 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.892 DEBUG:boto:Data: 2022-09-03T07:24:21.893 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.894 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.894 DEBUG:boto:Params: {} 2022-09-03T07:24:21.895 DEBUG:boto:Token: None 2022-09-03T07:24:21.895 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QbeQNer/UqI+LHVU/c4U/N8BcSY= 2022-09-03T07:24:21.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QbeQNer/UqI+LHVU/c4U/N8BcSY=', 'Content-Length': '0'} 2022-09-03T07:24:21.899 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000088871c211a755c1f-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.899 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:21.900 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:21.900 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.901 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.901 DEBUG:boto:Method: HEAD 2022-09-03T07:24:21.902 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.902 DEBUG:boto:Data: 2022-09-03T07:24:21.903 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.903 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.904 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.904 DEBUG:boto:Params: {} 2022-09-03T07:24:21.905 DEBUG:boto:Token: None 2022-09-03T07:24:21.906 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QbeQNer/UqI+LHVU/c4U/N8BcSY= 2022-09-03T07:24:21.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QbeQNer/UqI+LHVU/c4U/N8BcSY=', 'Content-Length': '0'} 2022-09-03T07:24:21.908 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae04e3f2ce11fcee-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.909 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:24:21.910 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:21.910 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:21.911 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.911 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.912 DEBUG:boto:Method: GET 2022-09-03T07:24:21.912 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.913 DEBUG:boto:Data: 2022-09-03T07:24:21.914 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.914 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.915 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.915 DEBUG:boto:Params: {} 2022-09-03T07:24:21.916 DEBUG:boto:Token: None 2022-09-03T07:24:21.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.917 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:++p+7lvBipZbIUMdPImt+EV5nxk= 2022-09-03T07:24:21.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:++p+7lvBipZbIUMdPImt+EV5nxk=', 'Content-Length': '0'} 2022-09-03T07:24:21.920 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004bc724de4f6f967e-0063130125-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.962 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:21.963 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:21.963 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.964 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.965 DEBUG:boto:Method: GET 2022-09-03T07:24:21.965 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.966 DEBUG:boto:Data: 2022-09-03T07:24:21.967 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.967 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:21.968 DEBUG:boto:Port: 8000 2022-09-03T07:24:21.968 DEBUG:boto:Params: {} 2022-09-03T07:24:21.969 DEBUG:boto:Token: None 2022-09-03T07:24:21.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:24:21.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:++p+7lvBipZbIUMdPImt+EV5nxk= 2022-09-03T07:24:21.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:++p+7lvBipZbIUMdPImt+EV5nxk=', 'Content-Length': '0'} 2022-09-03T07:24:21.975 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000073cda7bbb64a345-0063130125-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.976 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.977 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.981 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:21.981 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:21.982 DEBUG:boto:Method: GET 2022-09-03T07:24:21.983 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:21.984 DEBUG:boto:Data: 2022-09-03T07:24:21.984 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:21.985 DEBUG:boto:Port: 8001 2022-09-03T07:24:21.986 DEBUG:boto:Params: {} 2022-09-03T07:24:21.986 DEBUG:boto:Token: None 2022-09-03T07:24:21.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:21 GMT /avysmw-75/ 2022-09-03T07:24:21.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw= 2022-09-03T07:24:21.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X3897kJuaHVvRpKghz3q2J9j+zw=', 'Content-Length': '0'} 2022-09-03T07:24:21.991 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a13f214aea4b719e-0063130125-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:21.992 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:21.993 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:21.994 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:21.994 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:21.995 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:21.996 DEBUG:boto:Method: GET 2022-09-03T07:24:21.997 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:21.998 DEBUG:boto:Data: 2022-09-03T07:24:21.998 DEBUG:boto:Headers: {} 2022-09-03T07:24:21.999 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.000 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.001 DEBUG:boto:Params: {} 2022-09-03T07:24:22.002 DEBUG:boto:Token: None 2022-09-03T07:24:22.002 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.006 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005b5aa96eccea2b63-0063130126-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.007 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.008 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:22.009 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:22.009 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.010 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:22.010 DEBUG:boto:Data: 2022-09-03T07:24:22.011 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.012 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.012 DEBUG:boto:Params: {} 2022-09-03T07:24:22.013 DEBUG:boto:Token: None 2022-09-03T07:24:22.014 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj68 2022-09-03T07:24:22.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EQMKP9tiIIJXBfIsP7h+R/VdBCg= 2022-09-03T07:24:22.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EQMKP9tiIIJXBfIsP7h+R/VdBCg=', 'Content-Length': '0'} 2022-09-03T07:24:22.017 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000433acc8505a87900-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.018 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:22.018 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:22.019 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.019 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:22.020 DEBUG:boto:Data: 2022-09-03T07:24:22.020 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.021 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.021 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.022 DEBUG:boto:Params: {} 2022-09-03T07:24:22.022 DEBUG:boto:Token: None 2022-09-03T07:24:22.023 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj68 2022-09-03T07:24:22.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EQMKP9tiIIJXBfIsP7h+R/VdBCg= 2022-09-03T07:24:22.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EQMKP9tiIIJXBfIsP7h+R/VdBCg=', 'Content-Length': '0'} 2022-09-03T07:24:22.026 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004c54064d0f85117-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.027 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:24:22.027 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:22.028 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:22.028 DEBUG:boto:Method: GET 2022-09-03T07:24:22.029 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:22.029 DEBUG:boto:Data: 2022-09-03T07:24:22.030 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.031 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.032 DEBUG:boto:Params: {} 2022-09-03T07:24:22.032 DEBUG:boto:Token: None 2022-09-03T07:24:22.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj68 2022-09-03T07:24:22.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YxTNirvTPuf6N4p7o8gCSLHLUd4= 2022-09-03T07:24:22.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YxTNirvTPuf6N4p7o8gCSLHLUd4=', 'Content-Length': '0'} 2022-09-03T07:24:22.036 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f724b3f5b8b6b6c0-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.037 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:24:22.037 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:24:22.038 DEBUG:boto:Method: GET 2022-09-03T07:24:22.039 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:24:22.039 DEBUG:boto:Data: 2022-09-03T07:24:22.040 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.040 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.041 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.041 DEBUG:boto:Params: {} 2022-09-03T07:24:22.042 DEBUG:boto:Token: None 2022-09-03T07:24:22.042 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj68 2022-09-03T07:24:22.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YxTNirvTPuf6N4p7o8gCSLHLUd4= 2022-09-03T07:24:22.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YxTNirvTPuf6N4p7o8gCSLHLUd4=', 'Content-Length': '0'} 2022-09-03T07:24:22.045 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000543c3e3bad43ccb2-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.086 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.087 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.087 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:22.088 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:22.088 DEBUG:boto:Method: GET 2022-09-03T07:24:22.089 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:22.089 DEBUG:boto:Data: 2022-09-03T07:24:22.090 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.090 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.091 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.092 DEBUG:boto:Params: {} 2022-09-03T07:24:22.092 DEBUG:boto:Token: None 2022-09-03T07:24:22.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.097 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005bc532e6f00e5080-0063130126-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.097 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.099 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.099 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.100 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:22.101 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:24:22.102 DEBUG:boto:Method: GET 2022-09-03T07:24:22.103 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:24:22.104 DEBUG:boto:Data: 2022-09-03T07:24:22.105 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.106 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.107 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.108 DEBUG:boto:Params: {} 2022-09-03T07:24:22.109 DEBUG:boto:Token: None 2022-09-03T07:24:22.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.116 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dcb9bf395a8f2604-0063130126-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.117 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.118 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:24:22.119 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.120 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.121 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.122 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.123 DEBUG:boto:Method: GET 2022-09-03T07:24:22.124 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.125 DEBUG:boto:Data: 2022-09-03T07:24:22.126 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.127 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.128 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.129 DEBUG:boto:Params: {} 2022-09-03T07:24:22.130 DEBUG:boto:Token: None 2022-09-03T07:24:22.132 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIhaN0Ghr3KYmpCnNk3rEw3HqSo= 2022-09-03T07:24:22.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIhaN0Ghr3KYmpCnNk3rEw3HqSo=', 'Content-Length': '0'} 2022-09-03T07:24:22.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1576473dbacd691-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.139 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.140 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.141 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.142 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.143 DEBUG:boto:Method: GET 2022-09-03T07:24:22.144 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.145 DEBUG:boto:Data: 2022-09-03T07:24:22.146 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.147 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.148 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.149 DEBUG:boto:Params: {} 2022-09-03T07:24:22.150 DEBUG:boto:Token: None 2022-09-03T07:24:22.151 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.152 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIhaN0Ghr3KYmpCnNk3rEw3HqSo= 2022-09-03T07:24:22.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIhaN0Ghr3KYmpCnNk3rEw3HqSo=', 'Content-Length': '0'} 2022-09-03T07:24:22.156 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1c349f5f14f58b3-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.156 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.157 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.157 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.158 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.159 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.160 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.160 DEBUG:boto:Data: 2022-09-03T07:24:22.161 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.162 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.163 DEBUG:boto:Params: {} 2022-09-03T07:24:22.163 DEBUG:boto:Token: None 2022-09-03T07:24:22.164 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c717H2xWnXqJsScA+CYAItyA60E= 2022-09-03T07:24:22.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c717H2xWnXqJsScA+CYAItyA60E=', 'Content-Length': '0'} 2022-09-03T07:24:22.167 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a3f172aff18620f-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.168 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.169 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.169 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.170 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.170 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.171 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.171 DEBUG:boto:Data: 2022-09-03T07:24:22.172 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.172 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.173 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.173 DEBUG:boto:Params: {} 2022-09-03T07:24:22.174 DEBUG:boto:Token: None 2022-09-03T07:24:22.175 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c717H2xWnXqJsScA+CYAItyA60E= 2022-09-03T07:24:22.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c717H2xWnXqJsScA+CYAItyA60E=', 'Content-Length': '0'} 2022-09-03T07:24:22.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eddd69588eec8334-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.179 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:24:22.179 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.180 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.180 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.181 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.181 DEBUG:boto:Method: GET 2022-09-03T07:24:22.182 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.183 DEBUG:boto:Data: 2022-09-03T07:24:22.183 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.184 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.185 DEBUG:boto:Params: {} 2022-09-03T07:24:22.185 DEBUG:boto:Token: None 2022-09-03T07:24:22.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIhaN0Ghr3KYmpCnNk3rEw3HqSo= 2022-09-03T07:24:22.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIhaN0Ghr3KYmpCnNk3rEw3HqSo=', 'Content-Length': '0'} 2022-09-03T07:24:22.189 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001acd38a0434cdd09-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.190 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.191 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.191 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.192 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.193 DEBUG:boto:Method: GET 2022-09-03T07:24:22.193 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.194 DEBUG:boto:Data: 2022-09-03T07:24:22.194 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.195 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.195 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.196 DEBUG:boto:Params: {} 2022-09-03T07:24:22.197 DEBUG:boto:Token: None 2022-09-03T07:24:22.197 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:24:22.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIhaN0Ghr3KYmpCnNk3rEw3HqSo= 2022-09-03T07:24:22.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIhaN0Ghr3KYmpCnNk3rEw3HqSo=', 'Content-Length': '0'} 2022-09-03T07:24:22.201 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef344e20e91d4009-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.202 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.202 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.203 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:22.204 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:22.204 DEBUG:boto:Method: GET 2022-09-03T07:24:22.205 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:22.205 DEBUG:boto:Data: 2022-09-03T07:24:22.206 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.206 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.207 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.207 DEBUG:boto:Params: {} 2022-09-03T07:24:22.208 DEBUG:boto:Token: None 2022-09-03T07:24:22.209 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.213 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003c5016e06adc3f8e-0063130126-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.213 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.215 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.215 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.216 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:22.216 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:22.217 DEBUG:boto:Method: GET 2022-09-03T07:24:22.217 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:22.218 DEBUG:boto:Data: 2022-09-03T07:24:22.218 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.219 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.219 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.220 DEBUG:boto:Params: {} 2022-09-03T07:24:22.221 DEBUG:boto:Token: None 2022-09-03T07:24:22.221 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.225 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000077e5bd063824ba9b-0063130126-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.226 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.227 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.227 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.228 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.229 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:22.229 DEBUG:boto:Data: 2022-09-03T07:24:22.230 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.230 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.231 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.231 DEBUG:boto:Params: {} 2022-09-03T07:24:22.232 DEBUG:boto:Token: None 2022-09-03T07:24:22.233 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69 2022-09-03T07:24:22.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YIWFQVJywSWDCBWDI4xSK2CCdN4= 2022-09-03T07:24:22.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YIWFQVJywSWDCBWDI4xSK2CCdN4=', 'Content-Length': '0'} 2022-09-03T07:24:22.236 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba7554dbf059e80a-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.237 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.237 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.238 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.239 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:22.239 DEBUG:boto:Data: 2022-09-03T07:24:22.240 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.240 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.241 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.242 DEBUG:boto:Params: {} 2022-09-03T07:24:22.242 DEBUG:boto:Token: None 2022-09-03T07:24:22.243 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69 2022-09-03T07:24:22.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YIWFQVJywSWDCBWDI4xSK2CCdN4= 2022-09-03T07:24:22.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YIWFQVJywSWDCBWDI4xSK2CCdN4=', 'Content-Length': '0'} 2022-09-03T07:24:22.246 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c2e26aac3210cfc-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.247 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:24:22.247 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.248 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.249 DEBUG:boto:Method: GET 2022-09-03T07:24:22.250 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:22.251 DEBUG:boto:Data: 2022-09-03T07:24:22.252 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.254 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.254 DEBUG:boto:Params: {} 2022-09-03T07:24:22.255 DEBUG:boto:Token: None 2022-09-03T07:24:22.256 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69 2022-09-03T07:24:22.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Amwol9cskRk4Bk9WT5O3ZymhcOk= 2022-09-03T07:24:22.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Amwol9cskRk4Bk9WT5O3ZymhcOk=', 'Content-Length': '0'} 2022-09-03T07:24:22.260 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aef3c16b4e72ac38-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.261 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:24:22.262 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:24:22.262 DEBUG:boto:Method: GET 2022-09-03T07:24:22.263 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:24:22.264 DEBUG:boto:Data: 2022-09-03T07:24:22.265 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.266 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.267 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.268 DEBUG:boto:Params: {} 2022-09-03T07:24:22.269 DEBUG:boto:Token: None 2022-09-03T07:24:22.270 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj69 2022-09-03T07:24:22.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Amwol9cskRk4Bk9WT5O3ZymhcOk= 2022-09-03T07:24:22.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Amwol9cskRk4Bk9WT5O3ZymhcOk=', 'Content-Length': '0'} 2022-09-03T07:24:22.275 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004681a6523997589e-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.275 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.276 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.277 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:22.277 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:22.278 DEBUG:boto:Method: GET 2022-09-03T07:24:22.278 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:22.279 DEBUG:boto:Data: 2022-09-03T07:24:22.279 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.280 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.281 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.281 DEBUG:boto:Params: {} 2022-09-03T07:24:22.282 DEBUG:boto:Token: None 2022-09-03T07:24:22.283 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.284 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.287 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a2ca37bd08fc87cd-0063130126-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.288 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.288 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.289 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.289 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:22.290 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:24:22.291 DEBUG:boto:Method: GET 2022-09-03T07:24:22.291 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:24:22.292 DEBUG:boto:Data: 2022-09-03T07:24:22.292 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.293 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.294 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.294 DEBUG:boto:Params: {} 2022-09-03T07:24:22.295 DEBUG:boto:Token: None 2022-09-03T07:24:22.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000528cc91ee79e5b3b-0063130126-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.342 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.343 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:24:22.344 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.344 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.345 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.345 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.346 DEBUG:boto:Method: GET 2022-09-03T07:24:22.346 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.347 DEBUG:boto:Data: 2022-09-03T07:24:22.347 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.348 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.348 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.349 DEBUG:boto:Params: {} 2022-09-03T07:24:22.349 DEBUG:boto:Token: None 2022-09-03T07:24:22.350 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9MoS4tda30PNoVL7f3O0QS6+Lgw= 2022-09-03T07:24:22.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9MoS4tda30PNoVL7f3O0QS6+Lgw=', 'Content-Length': '0'} 2022-09-03T07:24:22.353 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d84322a7de276ae1-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.354 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.355 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.355 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.356 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.356 DEBUG:boto:Method: GET 2022-09-03T07:24:22.357 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.357 DEBUG:boto:Data: 2022-09-03T07:24:22.358 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.358 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.359 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.359 DEBUG:boto:Params: {} 2022-09-03T07:24:22.360 DEBUG:boto:Token: None 2022-09-03T07:24:22.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9MoS4tda30PNoVL7f3O0QS6+Lgw= 2022-09-03T07:24:22.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9MoS4tda30PNoVL7f3O0QS6+Lgw=', 'Content-Length': '0'} 2022-09-03T07:24:22.366 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fe4ced64dfdfd0e-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.366 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.367 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.367 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.368 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.368 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.369 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.369 DEBUG:boto:Data: 2022-09-03T07:24:22.370 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.371 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.371 DEBUG:boto:Params: {} 2022-09-03T07:24:22.372 DEBUG:boto:Token: None 2022-09-03T07:24:22.373 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Viv6WAPeivdBQZi2w+HrwJaf84M= 2022-09-03T07:24:22.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Viv6WAPeivdBQZi2w+HrwJaf84M=', 'Content-Length': '0'} 2022-09-03T07:24:22.376 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1b66f9eb9de6b08-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.377 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.377 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.378 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.378 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.379 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.379 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.380 DEBUG:boto:Data: 2022-09-03T07:24:22.380 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.381 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.381 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.382 DEBUG:boto:Params: {} 2022-09-03T07:24:22.382 DEBUG:boto:Token: None 2022-09-03T07:24:22.383 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Viv6WAPeivdBQZi2w+HrwJaf84M= 2022-09-03T07:24:22.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Viv6WAPeivdBQZi2w+HrwJaf84M=', 'Content-Length': '0'} 2022-09-03T07:24:22.386 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018b550676c8e8f69-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.388 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:24:22.388 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.389 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.389 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.390 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.390 DEBUG:boto:Method: GET 2022-09-03T07:24:22.391 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.392 DEBUG:boto:Data: 2022-09-03T07:24:22.392 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.393 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.394 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.395 DEBUG:boto:Params: {} 2022-09-03T07:24:22.395 DEBUG:boto:Token: None 2022-09-03T07:24:22.396 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9MoS4tda30PNoVL7f3O0QS6+Lgw= 2022-09-03T07:24:22.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9MoS4tda30PNoVL7f3O0QS6+Lgw=', 'Content-Length': '0'} 2022-09-03T07:24:22.399 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1ec3ccd316a3421-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.400 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.401 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.401 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.402 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.403 DEBUG:boto:Method: GET 2022-09-03T07:24:22.403 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.404 DEBUG:boto:Data: 2022-09-03T07:24:22.404 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.405 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.405 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.406 DEBUG:boto:Params: {} 2022-09-03T07:24:22.407 DEBUG:boto:Token: None 2022-09-03T07:24:22.407 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:24:22.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9MoS4tda30PNoVL7f3O0QS6+Lgw= 2022-09-03T07:24:22.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9MoS4tda30PNoVL7f3O0QS6+Lgw=', 'Content-Length': '0'} 2022-09-03T07:24:22.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3be94b39b322fa8-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.412 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.412 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.413 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:22.413 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:22.414 DEBUG:boto:Method: GET 2022-09-03T07:24:22.415 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:22.415 DEBUG:boto:Data: 2022-09-03T07:24:22.416 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.417 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.417 DEBUG:boto:Params: {} 2022-09-03T07:24:22.418 DEBUG:boto:Token: None 2022-09-03T07:24:22.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.422 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c05856ee4e084094-0063130126-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.423 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.424 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.425 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.425 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:22.426 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:22.426 DEBUG:boto:Method: GET 2022-09-03T07:24:22.427 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:22.427 DEBUG:boto:Data: 2022-09-03T07:24:22.428 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.428 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.429 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.429 DEBUG:boto:Params: {} 2022-09-03T07:24:22.430 DEBUG:boto:Token: None 2022-09-03T07:24:22.431 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.435 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb017a79301d152a-0063130126-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.482 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.483 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.484 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.485 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.486 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:22.487 DEBUG:boto:Data: 2022-09-03T07:24:22.487 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.489 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.490 DEBUG:boto:Params: {} 2022-09-03T07:24:22.490 DEBUG:boto:Token: None 2022-09-03T07:24:22.491 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76 2022-09-03T07:24:22.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bNT6nXZ/fkh+g5p+DkdmAJ5NvtA= 2022-09-03T07:24:22.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bNT6nXZ/fkh+g5p+DkdmAJ5NvtA=', 'Content-Length': '0'} 2022-09-03T07:24:22.504 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089a7429d617232c4-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.506 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.507 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.508 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.509 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:22.510 DEBUG:boto:Data: 2022-09-03T07:24:22.511 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.512 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.513 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.514 DEBUG:boto:Params: {} 2022-09-03T07:24:22.515 DEBUG:boto:Token: None 2022-09-03T07:24:22.516 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76 2022-09-03T07:24:22.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bNT6nXZ/fkh+g5p+DkdmAJ5NvtA= 2022-09-03T07:24:22.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bNT6nXZ/fkh+g5p+DkdmAJ5NvtA=', 'Content-Length': '0'} 2022-09-03T07:24:22.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e75499af2935159-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:24:22.521 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.523 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.523 DEBUG:boto:Method: GET 2022-09-03T07:24:22.524 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:22.524 DEBUG:boto:Data: 2022-09-03T07:24:22.525 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.526 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.527 DEBUG:boto:Params: {} 2022-09-03T07:24:22.527 DEBUG:boto:Token: None 2022-09-03T07:24:22.528 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76 2022-09-03T07:24:22.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PpoHplwfK+3093WKmilgHmpzPDo= 2022-09-03T07:24:22.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PpoHplwfK+3093WKmilgHmpzPDo=', 'Content-Length': '0'} 2022-09-03T07:24:22.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c874af8e2793c726-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.574 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:24:22.575 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:24:22.576 DEBUG:boto:Method: GET 2022-09-03T07:24:22.576 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:24:22.577 DEBUG:boto:Data: 2022-09-03T07:24:22.577 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.578 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.579 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.579 DEBUG:boto:Params: {} 2022-09-03T07:24:22.580 DEBUG:boto:Token: None 2022-09-03T07:24:22.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj76 2022-09-03T07:24:22.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PpoHplwfK+3093WKmilgHmpzPDo= 2022-09-03T07:24:22.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PpoHplwfK+3093WKmilgHmpzPDo=', 'Content-Length': '0'} 2022-09-03T07:24:22.584 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1e1c39f926081f7-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.584 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.585 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.585 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:22.586 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:22.587 DEBUG:boto:Method: GET 2022-09-03T07:24:22.587 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:22.588 DEBUG:boto:Data: 2022-09-03T07:24:22.588 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.589 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.589 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.590 DEBUG:boto:Params: {} 2022-09-03T07:24:22.590 DEBUG:boto:Token: None 2022-09-03T07:24:22.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007e29d436dfb1468d-0063130126-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.596 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.596 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.597 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.598 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:22.598 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:24:22.599 DEBUG:boto:Method: GET 2022-09-03T07:24:22.600 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:24:22.600 DEBUG:boto:Data: 2022-09-03T07:24:22.601 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.601 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.602 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.602 DEBUG:boto:Params: {} 2022-09-03T07:24:22.603 DEBUG:boto:Token: None 2022-09-03T07:24:22.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.610 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008359f0567dfe0ec7-0063130126-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.611 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.612 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:24:22.612 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.613 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.613 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.614 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.614 DEBUG:boto:Method: GET 2022-09-03T07:24:22.615 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.616 DEBUG:boto:Data: 2022-09-03T07:24:22.616 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.617 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.617 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.618 DEBUG:boto:Params: {} 2022-09-03T07:24:22.619 DEBUG:boto:Token: None 2022-09-03T07:24:22.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LxcUsCNxtK9el+PjcbruBbLp3JU= 2022-09-03T07:24:22.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LxcUsCNxtK9el+PjcbruBbLp3JU=', 'Content-Length': '0'} 2022-09-03T07:24:22.625 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fdf450d1edfea954-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.626 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.627 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.627 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.628 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.628 DEBUG:boto:Method: GET 2022-09-03T07:24:22.629 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.629 DEBUG:boto:Data: 2022-09-03T07:24:22.630 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.631 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.631 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.632 DEBUG:boto:Params: {} 2022-09-03T07:24:22.632 DEBUG:boto:Token: None 2022-09-03T07:24:22.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LxcUsCNxtK9el+PjcbruBbLp3JU= 2022-09-03T07:24:22.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LxcUsCNxtK9el+PjcbruBbLp3JU=', 'Content-Length': '0'} 2022-09-03T07:24:22.636 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c48a16b4798c47c3-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.637 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.638 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.638 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.639 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.639 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.640 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.640 DEBUG:boto:Data: 2022-09-03T07:24:22.641 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.642 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.642 DEBUG:boto:Params: {} 2022-09-03T07:24:22.643 DEBUG:boto:Token: None 2022-09-03T07:24:22.643 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b/P/XpQyJTYqhXuDQ1MN1lGpZqo= 2022-09-03T07:24:22.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b/P/XpQyJTYqhXuDQ1MN1lGpZqo=', 'Content-Length': '0'} 2022-09-03T07:24:22.647 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d56ef9537bad380d-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.647 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.648 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.648 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.649 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.649 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.650 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.651 DEBUG:boto:Data: 2022-09-03T07:24:22.651 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.652 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.652 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.653 DEBUG:boto:Params: {} 2022-09-03T07:24:22.653 DEBUG:boto:Token: None 2022-09-03T07:24:22.654 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.654 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b/P/XpQyJTYqhXuDQ1MN1lGpZqo= 2022-09-03T07:24:22.655 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b/P/XpQyJTYqhXuDQ1MN1lGpZqo=', 'Content-Length': '0'} 2022-09-03T07:24:22.657 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000546afa68a51138e4-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.658 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:24:22.658 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.659 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.659 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.660 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.660 DEBUG:boto:Method: GET 2022-09-03T07:24:22.661 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.661 DEBUG:boto:Data: 2022-09-03T07:24:22.662 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.663 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.664 DEBUG:boto:Params: {} 2022-09-03T07:24:22.664 DEBUG:boto:Token: None 2022-09-03T07:24:22.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LxcUsCNxtK9el+PjcbruBbLp3JU= 2022-09-03T07:24:22.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LxcUsCNxtK9el+PjcbruBbLp3JU=', 'Content-Length': '0'} 2022-09-03T07:24:22.669 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a824261919f487e2-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.710 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.711 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.712 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.714 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.715 DEBUG:boto:Method: GET 2022-09-03T07:24:22.716 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.717 DEBUG:boto:Data: 2022-09-03T07:24:22.718 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.721 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.721 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.722 DEBUG:boto:Params: {} 2022-09-03T07:24:22.722 DEBUG:boto:Token: None 2022-09-03T07:24:22.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:24:22.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LxcUsCNxtK9el+PjcbruBbLp3JU= 2022-09-03T07:24:22.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LxcUsCNxtK9el+PjcbruBbLp3JU=', 'Content-Length': '0'} 2022-09-03T07:24:22.726 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec96772ae27d383e-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.727 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.728 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.729 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:22.730 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:22.731 DEBUG:boto:Method: GET 2022-09-03T07:24:22.732 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:22.733 DEBUG:boto:Data: 2022-09-03T07:24:22.734 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.735 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.735 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.736 DEBUG:boto:Params: {} 2022-09-03T07:24:22.737 DEBUG:boto:Token: None 2022-09-03T07:24:22.738 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.743 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000041f6f5c371271d9f-0063130126-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.744 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.745 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.745 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.746 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:22.747 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:22.748 DEBUG:boto:Method: GET 2022-09-03T07:24:22.749 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:22.750 DEBUG:boto:Data: 2022-09-03T07:24:22.751 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.752 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.753 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.754 DEBUG:boto:Params: {} 2022-09-03T07:24:22.755 DEBUG:boto:Token: None 2022-09-03T07:24:22.755 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.759 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093841c0cbe8d259d-0063130126-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.802 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.804 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.805 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.806 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.806 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:22.807 DEBUG:boto:Data: 2022-09-03T07:24:22.807 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.808 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.809 DEBUG:boto:Params: {} 2022-09-03T07:24:22.809 DEBUG:boto:Token: None 2022-09-03T07:24:22.810 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85 2022-09-03T07:24:22.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N05ATbLXnERyifYITKqPsMufG6A= 2022-09-03T07:24:22.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N05ATbLXnERyifYITKqPsMufG6A=', 'Content-Length': '0'} 2022-09-03T07:24:22.813 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3dbeb758711f624-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.814 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.814 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.815 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.815 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:22.816 DEBUG:boto:Data: 2022-09-03T07:24:22.816 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.817 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.817 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.818 DEBUG:boto:Params: {} 2022-09-03T07:24:22.819 DEBUG:boto:Token: None 2022-09-03T07:24:22.819 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85 2022-09-03T07:24:22.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N05ATbLXnERyifYITKqPsMufG6A= 2022-09-03T07:24:22.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N05ATbLXnERyifYITKqPsMufG6A=', 'Content-Length': '0'} 2022-09-03T07:24:22.822 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006bf80cb6a436c73e-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.823 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:24:22.824 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.824 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.825 DEBUG:boto:Method: GET 2022-09-03T07:24:22.826 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:22.826 DEBUG:boto:Data: 2022-09-03T07:24:22.827 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.827 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.828 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.828 DEBUG:boto:Params: {} 2022-09-03T07:24:22.829 DEBUG:boto:Token: None 2022-09-03T07:24:22.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85 2022-09-03T07:24:22.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mtrAZ47fvMakWXbNg0Pk2vBd9M8= 2022-09-03T07:24:22.831 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mtrAZ47fvMakWXbNg0Pk2vBd9M8=', 'Content-Length': '0'} 2022-09-03T07:24:22.834 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e024bbd016bd722c-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.836 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:24:22.837 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:24:22.837 DEBUG:boto:Method: GET 2022-09-03T07:24:22.838 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:24:22.838 DEBUG:boto:Data: 2022-09-03T07:24:22.839 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.839 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.840 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.841 DEBUG:boto:Params: {} 2022-09-03T07:24:22.841 DEBUG:boto:Token: None 2022-09-03T07:24:22.842 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj85 2022-09-03T07:24:22.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mtrAZ47fvMakWXbNg0Pk2vBd9M8= 2022-09-03T07:24:22.843 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mtrAZ47fvMakWXbNg0Pk2vBd9M8=', 'Content-Length': '0'} 2022-09-03T07:24:22.845 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ff1e2f8a31179ce-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.846 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.847 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.847 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:22.848 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:22.848 DEBUG:boto:Method: GET 2022-09-03T07:24:22.849 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:22.850 DEBUG:boto:Data: 2022-09-03T07:24:22.850 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.851 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.852 DEBUG:boto:Params: {} 2022-09-03T07:24:22.853 DEBUG:boto:Token: None 2022-09-03T07:24:22.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.859 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002ed5548db3ef8cbc-0063130126-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.859 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.860 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:22.861 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:22.861 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:22.862 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:24:22.863 DEBUG:boto:Method: GET 2022-09-03T07:24:22.863 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:24:22.864 DEBUG:boto:Data: 2022-09-03T07:24:22.864 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.865 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.866 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.866 DEBUG:boto:Params: {} 2022-09-03T07:24:22.867 DEBUG:boto:Token: None 2022-09-03T07:24:22.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/ 2022-09-03T07:24:22.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc= 2022-09-03T07:24:22.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pZ7yLCMq9SDmwWQfG/9qWFwUDGc=', 'Content-Length': '0'} 2022-09-03T07:24:22.871 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001aa29c9be9257610-0063130126-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.871 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:22.872 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:24:22.873 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:22.874 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:22.874 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.875 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.875 DEBUG:boto:Method: GET 2022-09-03T07:24:22.876 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.876 DEBUG:boto:Data: 2022-09-03T07:24:22.877 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.877 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.878 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.878 DEBUG:boto:Params: {} 2022-09-03T07:24:22.879 DEBUG:boto:Token: None 2022-09-03T07:24:22.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V6kIr3mpYsVpVihMBQxOaHcqT8Q= 2022-09-03T07:24:22.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V6kIr3mpYsVpVihMBQxOaHcqT8Q=', 'Content-Length': '0'} 2022-09-03T07:24:22.883 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009eb749f9abd8be9c-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.930 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:22.931 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:22.931 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.932 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.932 DEBUG:boto:Method: GET 2022-09-03T07:24:22.933 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.933 DEBUG:boto:Data: 2022-09-03T07:24:22.934 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.935 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.935 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.936 DEBUG:boto:Params: {} 2022-09-03T07:24:22.936 DEBUG:boto:Token: None 2022-09-03T07:24:22.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V6kIr3mpYsVpVihMBQxOaHcqT8Q= 2022-09-03T07:24:22.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V6kIr3mpYsVpVihMBQxOaHcqT8Q=', 'Content-Length': '0'} 2022-09-03T07:24:22.940 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075a38bcbb1d18de1-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.941 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:22.942 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:22.942 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.943 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.943 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.944 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.944 DEBUG:boto:Data: 2022-09-03T07:24:22.945 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.945 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.946 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.947 DEBUG:boto:Params: {} 2022-09-03T07:24:22.948 DEBUG:boto:Token: None 2022-09-03T07:24:22.948 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WbNq9JtgNGWzjrdSSNAzjlfJ7yM= 2022-09-03T07:24:22.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WbNq9JtgNGWzjrdSSNAzjlfJ7yM=', 'Content-Length': '0'} 2022-09-03T07:24:22.952 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000894c5ad5428f98f1-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.952 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:22.953 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:22.953 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.954 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.955 DEBUG:boto:Method: HEAD 2022-09-03T07:24:22.955 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.956 DEBUG:boto:Data: 2022-09-03T07:24:22.956 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.957 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.957 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.958 DEBUG:boto:Params: {} 2022-09-03T07:24:22.959 DEBUG:boto:Token: None 2022-09-03T07:24:22.959 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WbNq9JtgNGWzjrdSSNAzjlfJ7yM= 2022-09-03T07:24:22.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WbNq9JtgNGWzjrdSSNAzjlfJ7yM=', 'Content-Length': '0'} 2022-09-03T07:24:22.962 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000150f42d7e59e31b0-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.963 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:24:22.964 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:22.964 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:22.965 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.965 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.966 DEBUG:boto:Method: GET 2022-09-03T07:24:22.966 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.967 DEBUG:boto:Data: 2022-09-03T07:24:22.967 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.968 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:22.968 DEBUG:boto:Port: 8001 2022-09-03T07:24:22.969 DEBUG:boto:Params: {} 2022-09-03T07:24:22.970 DEBUG:boto:Token: None 2022-09-03T07:24:22.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V6kIr3mpYsVpVihMBQxOaHcqT8Q= 2022-09-03T07:24:22.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V6kIr3mpYsVpVihMBQxOaHcqT8Q=', 'Content-Length': '0'} 2022-09-03T07:24:22.974 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d977fb2bc2398e46-0063130126-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:22.974 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:22.975 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:22.976 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.976 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.977 DEBUG:boto:Method: GET 2022-09-03T07:24:22.977 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.978 DEBUG:boto:Data: 2022-09-03T07:24:22.979 DEBUG:boto:Headers: {} 2022-09-03T07:24:22.979 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:22.980 DEBUG:boto:Port: 8000 2022-09-03T07:24:22.981 DEBUG:boto:Params: {} 2022-09-03T07:24:22.981 DEBUG:boto:Token: None 2022-09-03T07:24:22.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:22 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:24:22.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V6kIr3mpYsVpVihMBQxOaHcqT8Q= 2022-09-03T07:24:22.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V6kIr3mpYsVpVihMBQxOaHcqT8Q=', 'Content-Length': '0'} 2022-09-03T07:24:22.985 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006cdf9e8ce6c7b3ad-0063130126-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.026 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.027 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.027 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:23.028 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:23.028 DEBUG:boto:Method: GET 2022-09-03T07:24:23.029 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:23.030 DEBUG:boto:Data: 2022-09-03T07:24:23.031 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.032 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.032 DEBUG:boto:Params: {} 2022-09-03T07:24:23.033 DEBUG:boto:Token: None 2022-09-03T07:24:23.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc= 2022-09-03T07:24:23.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc=', 'Content-Length': '0'} 2022-09-03T07:24:23.038 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f5863d562a34cc7c-0063130127-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.039 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:23.039 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.040 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.041 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:23.041 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:23.042 DEBUG:boto:Method: GET 2022-09-03T07:24:23.042 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:23.043 DEBUG:boto:Data: 2022-09-03T07:24:23.044 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.044 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.045 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.045 DEBUG:boto:Params: {} 2022-09-03T07:24:23.046 DEBUG:boto:Token: None 2022-09-03T07:24:23.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc= 2022-09-03T07:24:23.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc=', 'Content-Length': '0'} 2022-09-03T07:24:23.050 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079e435300330b525-0063130127-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.051 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:23.052 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:23.052 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:23.053 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.053 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:23.054 DEBUG:boto:Data: 2022-09-03T07:24:23.054 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.055 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.056 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.056 DEBUG:boto:Params: {} 2022-09-03T07:24:23.057 DEBUG:boto:Token: None 2022-09-03T07:24:23.057 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj92 2022-09-03T07:24:23.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ni8pxa+LZPKMym+dz2Nhp4tvW8w= 2022-09-03T07:24:23.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ni8pxa+LZPKMym+dz2Nhp4tvW8w=', 'Content-Length': '0'} 2022-09-03T07:24:23.060 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068badb72e450fbe9-0063130127-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.061 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:23.062 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:23.062 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.063 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:23.063 DEBUG:boto:Data: 2022-09-03T07:24:23.064 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.064 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.065 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.065 DEBUG:boto:Params: {} 2022-09-03T07:24:23.066 DEBUG:boto:Token: None 2022-09-03T07:24:23.066 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj92 2022-09-03T07:24:23.067 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ni8pxa+LZPKMym+dz2Nhp4tvW8w= 2022-09-03T07:24:23.068 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ni8pxa+LZPKMym+dz2Nhp4tvW8w=', 'Content-Length': '0'} 2022-09-03T07:24:23.069 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083beeb9e47c44859-0063130127-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.070 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:24:23.071 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:23.071 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:23.072 DEBUG:boto:Method: GET 2022-09-03T07:24:23.072 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:23.073 DEBUG:boto:Data: 2022-09-03T07:24:23.074 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.074 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.075 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.075 DEBUG:boto:Params: {} 2022-09-03T07:24:23.076 DEBUG:boto:Token: None 2022-09-03T07:24:23.077 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj92 2022-09-03T07:24:23.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jr805bjzyhl1IaBv7IsfVza66b8= 2022-09-03T07:24:23.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jr805bjzyhl1IaBv7IsfVza66b8=', 'Content-Length': '0'} 2022-09-03T07:24:23.080 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000814b1e37502f310f-0063130127-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.081 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:24:23.081 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:24:23.082 DEBUG:boto:Method: GET 2022-09-03T07:24:23.083 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:24:23.083 DEBUG:boto:Data: 2022-09-03T07:24:23.084 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.084 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.085 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.085 DEBUG:boto:Params: {} 2022-09-03T07:24:23.086 DEBUG:boto:Token: None 2022-09-03T07:24:23.086 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj92 2022-09-03T07:24:23.087 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jr805bjzyhl1IaBv7IsfVza66b8= 2022-09-03T07:24:23.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jr805bjzyhl1IaBv7IsfVza66b8=', 'Content-Length': '0'} 2022-09-03T07:24:23.090 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016cf9d62558d3f05-0063130127-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.091 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.091 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.092 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:23.092 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:23.093 DEBUG:boto:Method: GET 2022-09-03T07:24:23.093 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:23.094 DEBUG:boto:Data: 2022-09-03T07:24:23.094 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.095 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.096 DEBUG:boto:Params: {} 2022-09-03T07:24:23.097 DEBUG:boto:Token: None 2022-09-03T07:24:23.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc= 2022-09-03T07:24:23.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc=', 'Content-Length': '0'} 2022-09-03T07:24:23.102 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079382c2348d4b158-0063130127-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.103 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:23.104 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.104 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.105 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:23.105 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:24:23.106 DEBUG:boto:Method: GET 2022-09-03T07:24:23.106 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:24:23.107 DEBUG:boto:Data: 2022-09-03T07:24:23.107 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.108 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.109 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.109 DEBUG:boto:Params: {} 2022-09-03T07:24:23.110 DEBUG:boto:Token: None 2022-09-03T07:24:23.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc= 2022-09-03T07:24:23.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc=', 'Content-Length': '0'} 2022-09-03T07:24:23.114 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000015d4abfd237c862c-0063130127-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.154 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:23.155 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:24:23.156 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.156 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.157 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.157 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.158 DEBUG:boto:Method: GET 2022-09-03T07:24:23.158 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.159 DEBUG:boto:Data: 2022-09-03T07:24:23.159 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.160 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.160 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.161 DEBUG:boto:Params: {} 2022-09-03T07:24:23.162 DEBUG:boto:Token: None 2022-09-03T07:24:23.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oeCj/EEQg23AFU5RwmlK0kPCzrA= 2022-09-03T07:24:23.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oeCj/EEQg23AFU5RwmlK0kPCzrA=', 'Content-Length': '0'} 2022-09-03T07:24:23.166 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac352bbcfcc31976-0063130127-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.166 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.167 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.168 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.168 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.169 DEBUG:boto:Method: GET 2022-09-03T07:24:23.169 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.170 DEBUG:boto:Data: 2022-09-03T07:24:23.170 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.171 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.171 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.172 DEBUG:boto:Params: {} 2022-09-03T07:24:23.172 DEBUG:boto:Token: None 2022-09-03T07:24:23.173 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oeCj/EEQg23AFU5RwmlK0kPCzrA= 2022-09-03T07:24:23.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oeCj/EEQg23AFU5RwmlK0kPCzrA=', 'Content-Length': '0'} 2022-09-03T07:24:23.176 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063a76995f96df5d2-0063130127-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.177 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.178 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.178 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.179 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.180 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.180 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.181 DEBUG:boto:Data: 2022-09-03T07:24:23.181 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.182 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.183 DEBUG:boto:Params: {} 2022-09-03T07:24:23.184 DEBUG:boto:Token: None 2022-09-03T07:24:23.184 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B07OSNDrV1UmD3KcZmsQVybwsSE= 2022-09-03T07:24:23.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B07OSNDrV1UmD3KcZmsQVybwsSE=', 'Content-Length': '0'} 2022-09-03T07:24:23.188 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000496d98ec4ef04a21-0063130127-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.188 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.189 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.189 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.190 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.190 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.191 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.192 DEBUG:boto:Data: 2022-09-03T07:24:23.192 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.193 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.193 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.194 DEBUG:boto:Params: {} 2022-09-03T07:24:23.194 DEBUG:boto:Token: None 2022-09-03T07:24:23.195 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B07OSNDrV1UmD3KcZmsQVybwsSE= 2022-09-03T07:24:23.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B07OSNDrV1UmD3KcZmsQVybwsSE=', 'Content-Length': '0'} 2022-09-03T07:24:23.198 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000413013ac71f7fd2d-0063130127-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.199 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:24:23.199 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.200 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.201 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.201 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.202 DEBUG:boto:Method: GET 2022-09-03T07:24:23.202 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.203 DEBUG:boto:Data: 2022-09-03T07:24:23.203 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.204 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.204 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.205 DEBUG:boto:Params: {} 2022-09-03T07:24:23.205 DEBUG:boto:Token: None 2022-09-03T07:24:23.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oeCj/EEQg23AFU5RwmlK0kPCzrA= 2022-09-03T07:24:23.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oeCj/EEQg23AFU5RwmlK0kPCzrA=', 'Content-Length': '0'} 2022-09-03T07:24:23.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b66b75aa38490b17-0063130127-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.210 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.211 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.211 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.212 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.212 DEBUG:boto:Method: GET 2022-09-03T07:24:23.213 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.214 DEBUG:boto:Data: 2022-09-03T07:24:23.214 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.215 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.215 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.216 DEBUG:boto:Params: {} 2022-09-03T07:24:23.216 DEBUG:boto:Token: None 2022-09-03T07:24:23.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:24:23.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oeCj/EEQg23AFU5RwmlK0kPCzrA= 2022-09-03T07:24:23.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oeCj/EEQg23AFU5RwmlK0kPCzrA=', 'Content-Length': '0'} 2022-09-03T07:24:23.220 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075c53474f4dac9dd-0063130127-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.221 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.221 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.222 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:23.223 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:23.223 DEBUG:boto:Method: GET 2022-09-03T07:24:23.224 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:23.224 DEBUG:boto:Data: 2022-09-03T07:24:23.225 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.225 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.226 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.226 DEBUG:boto:Params: {} 2022-09-03T07:24:23.227 DEBUG:boto:Token: None 2022-09-03T07:24:23.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc= 2022-09-03T07:24:23.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc=', 'Content-Length': '0'} 2022-09-03T07:24:23.232 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000327c68464152eef6-0063130127-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.232 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:23.233 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.234 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.234 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:23.235 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:23.235 DEBUG:boto:Method: GET 2022-09-03T07:24:23.236 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:23.236 DEBUG:boto:Data: 2022-09-03T07:24:23.237 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.237 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.238 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.238 DEBUG:boto:Params: {} 2022-09-03T07:24:23.239 DEBUG:boto:Token: None 2022-09-03T07:24:23.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc= 2022-09-03T07:24:23.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc=', 'Content-Length': '0'} 2022-09-03T07:24:23.245 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006946bc7ba0e7626d-0063130127-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.246 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:23.247 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.247 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.248 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.248 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:23.249 DEBUG:boto:Data: 2022-09-03T07:24:23.249 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.250 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.251 DEBUG:boto:Params: {} 2022-09-03T07:24:23.252 DEBUG:boto:Token: None 2022-09-03T07:24:23.252 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95 2022-09-03T07:24:23.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dRMsnA554moKhhdoz9BVwXBhSVc= 2022-09-03T07:24:23.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dRMsnA554moKhhdoz9BVwXBhSVc=', 'Content-Length': '0'} 2022-09-03T07:24:23.255 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3b11c7c92c9bd96-0063130127-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.256 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.256 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.257 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.257 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:23.258 DEBUG:boto:Data: 2022-09-03T07:24:23.258 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.259 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.259 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.260 DEBUG:boto:Params: {} 2022-09-03T07:24:23.261 DEBUG:boto:Token: None 2022-09-03T07:24:23.261 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95 2022-09-03T07:24:23.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dRMsnA554moKhhdoz9BVwXBhSVc= 2022-09-03T07:24:23.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dRMsnA554moKhhdoz9BVwXBhSVc=', 'Content-Length': '0'} 2022-09-03T07:24:23.264 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1010ec634e16093-0063130127-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.265 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:24:23.266 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.266 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.267 DEBUG:boto:Method: GET 2022-09-03T07:24:23.267 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:23.268 DEBUG:boto:Data: 2022-09-03T07:24:23.268 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.270 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.270 DEBUG:boto:Params: {} 2022-09-03T07:24:23.271 DEBUG:boto:Token: None 2022-09-03T07:24:23.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95 2022-09-03T07:24:23.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/I4NSp9uN1wEld/zvbWGBSqcgn4= 2022-09-03T07:24:23.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/I4NSp9uN1wEld/zvbWGBSqcgn4=', 'Content-Length': '0'} 2022-09-03T07:24:23.275 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a87748e03cc86abd-0063130127-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.275 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:24:23.276 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:24:23.276 DEBUG:boto:Method: GET 2022-09-03T07:24:23.277 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:24:23.278 DEBUG:boto:Data: 2022-09-03T07:24:23.278 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.279 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.279 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.280 DEBUG:boto:Params: {} 2022-09-03T07:24:23.280 DEBUG:boto:Token: None 2022-09-03T07:24:23.281 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj95 2022-09-03T07:24:23.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/I4NSp9uN1wEld/zvbWGBSqcgn4= 2022-09-03T07:24:23.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/I4NSp9uN1wEld/zvbWGBSqcgn4=', 'Content-Length': '0'} 2022-09-03T07:24:23.284 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c083ac598744f454-0063130127-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.326 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.327 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.329 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:23.329 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:23.330 DEBUG:boto:Method: GET 2022-09-03T07:24:23.331 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:23.331 DEBUG:boto:Data: 2022-09-03T07:24:23.332 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:24:23.333 DEBUG:boto:Port: 8001 2022-09-03T07:24:23.334 DEBUG:boto:Params: {} 2022-09-03T07:24:23.335 DEBUG:boto:Token: None 2022-09-03T07:24:23.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc= 2022-09-03T07:24:23.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc=', 'Content-Length': '0'} 2022-09-03T07:24:23.340 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eca5c4673b6f3fa5-0063130127-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.341 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:23.342 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.342 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.343 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:23.344 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:24:23.344 DEBUG:boto:Method: GET 2022-09-03T07:24:23.345 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:24:23.345 DEBUG:boto:Data: 2022-09-03T07:24:23.346 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.346 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:24:23.347 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.348 DEBUG:boto:Params: {} 2022-09-03T07:24:23.348 DEBUG:boto:Token: None 2022-09-03T07:24:23.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc= 2022-09-03T07:24:23.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/hf72QVhFWGsbi3y5d97SnltiCc=', 'Content-Length': '0'} 2022-09-03T07:24:23.353 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000046ee7f6e8c27b0cb-0063130127-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.353 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:24:23.354 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:24:23.355 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '29', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:24:23.355 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '29', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:24:23.356 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --num-shards 29 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:24:23.420 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:24:23.420 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:24:23.495 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:24:23.496 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-75 2022-09-03T07:24:23.507 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 20 2022-09-03T07:24:23.578 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.579 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.580 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.581 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:23.581 DEBUG:boto:Data: 2022-09-03T07:24:23.582 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.584 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.584 DEBUG:boto:Params: {} 2022-09-03T07:24:23.585 DEBUG:boto:Token: None 2022-09-03T07:24:23.585 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y= 2022-09-03T07:24:23.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y=', 'Content-Length': '0'} 2022-09-03T07:24:23.591 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '20'), ('X-RGW-Bytes-Used', '60'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001c9db9d07badc6c5-0063130127-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.592 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:24:23.593 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:24:23.593 DEBUG:boto:Method: PUT 2022-09-03T07:24:23.594 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:24:23.594 DEBUG:boto:Data: 2022-09-03T07:24:23.595 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:24:23.595 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.596 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.597 DEBUG:boto:Params: {} 2022-09-03T07:24:23.597 DEBUG:boto:Token: None 2022-09-03T07:24:23.598 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj29 2022-09-03T07:24:23.598 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wELTk+cCAREye8d+1EEUkitm2x8= 2022-09-03T07:24:23.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wELTk+cCAREye8d+1EEUkitm2x8='} 2022-09-03T07:24:23.642 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000a21b9c887d17cc6d-0063130127-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.643 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.644 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.644 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.645 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:23.645 DEBUG:boto:Data: 2022-09-03T07:24:23.646 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.647 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.647 DEBUG:boto:Params: {} 2022-09-03T07:24:23.648 DEBUG:boto:Token: None 2022-09-03T07:24:23.649 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y= 2022-09-03T07:24:23.651 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y=', 'Content-Length': '0'} 2022-09-03T07:24:23.654 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '21'), ('X-RGW-Bytes-Used', '63'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000080bedcb4e21e7946-0063130127-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.656 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:24:23.657 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:24:23.658 DEBUG:boto:Method: PUT 2022-09-03T07:24:23.658 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:24:23.659 DEBUG:boto:Data: 2022-09-03T07:24:23.660 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:24:23.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.661 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.662 DEBUG:boto:Params: {} 2022-09-03T07:24:23.662 DEBUG:boto:Token: None 2022-09-03T07:24:23.663 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj58 2022-09-03T07:24:23.664 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4OeN42LEBF0uKr8Vkx60V8d8TVc= 2022-09-03T07:24:23.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4OeN42LEBF0uKr8Vkx60V8d8TVc='} 2022-09-03T07:24:23.670 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000efd48ed1188a038d-0063130127-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.671 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.672 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.672 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.673 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:23.673 DEBUG:boto:Data: 2022-09-03T07:24:23.674 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.674 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.675 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.675 DEBUG:boto:Params: {} 2022-09-03T07:24:23.676 DEBUG:boto:Token: None 2022-09-03T07:24:23.676 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.677 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y= 2022-09-03T07:24:23.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y=', 'Content-Length': '0'} 2022-09-03T07:24:23.681 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '22'), ('X-RGW-Bytes-Used', '66'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000010a7208655f48d1c-0063130127-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.682 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:24:23.683 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:24:23.684 DEBUG:boto:Method: PUT 2022-09-03T07:24:23.685 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:24:23.686 DEBUG:boto:Data: 2022-09-03T07:24:23.686 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:24:23.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.687 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.688 DEBUG:boto:Params: {} 2022-09-03T07:24:23.688 DEBUG:boto:Token: None 2022-09-03T07:24:23.689 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj87 2022-09-03T07:24:23.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R+e1w3tF/9DwnbY1vWRCidW7Wu8= 2022-09-03T07:24:23.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R+e1w3tF/9DwnbY1vWRCidW7Wu8='} 2022-09-03T07:24:23.734 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000d8d9bacb9989903e-0063130127-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.735 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.736 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.736 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.737 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:23.737 DEBUG:boto:Data: 2022-09-03T07:24:23.738 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.739 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.739 DEBUG:boto:Params: {} 2022-09-03T07:24:23.740 DEBUG:boto:Token: None 2022-09-03T07:24:23.740 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y= 2022-09-03T07:24:23.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y=', 'Content-Length': '0'} 2022-09-03T07:24:23.745 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '23'), ('X-RGW-Bytes-Used', '69'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b52ec928d078e7e3-0063130127-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.746 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:24:23.746 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:24:23.747 DEBUG:boto:Method: PUT 2022-09-03T07:24:23.747 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:24:23.748 DEBUG:boto:Data: 2022-09-03T07:24:23.748 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:24:23.749 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.750 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.750 DEBUG:boto:Params: {} 2022-09-03T07:24:23.751 DEBUG:boto:Token: None 2022-09-03T07:24:23.751 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj116 2022-09-03T07:24:23.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P4MsABJoy5wQM7PPYiZlt/e7nzk= 2022-09-03T07:24:23.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P4MsABJoy5wQM7PPYiZlt/e7nzk='} 2022-09-03T07:24:23.764 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000215077b14e82d1a1-0063130127-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.765 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:24:23.765 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:24:23.766 DEBUG:boto:Method: HEAD 2022-09-03T07:24:23.767 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:24:23.767 DEBUG:boto:Data: 2022-09-03T07:24:23.768 DEBUG:boto:Headers: {} 2022-09-03T07:24:23.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.769 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.769 DEBUG:boto:Params: {} 2022-09-03T07:24:23.770 DEBUG:boto:Token: None 2022-09-03T07:24:23.770 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/ 2022-09-03T07:24:23.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y= 2022-09-03T07:24:23.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VxlOw5GiztnneW00QeygkWqVt2Y=', 'Content-Length': '0'} 2022-09-03T07:24:23.775 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '24'), ('X-RGW-Bytes-Used', '72'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008d18badbf0c6f3dd-0063130127-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.776 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:24:23.777 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:24:23.778 DEBUG:boto:Method: PUT 2022-09-03T07:24:23.779 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:24:23.779 DEBUG:boto:Data: 2022-09-03T07:24:23.780 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:24:23.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:24:23.781 DEBUG:boto:Port: 8000 2022-09-03T07:24:23.781 DEBUG:boto:Params: {} 2022-09-03T07:24:23.782 DEBUG:boto:Token: None 2022-09-03T07:24:23.783 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:24:23 GMT /avysmw-75/obj145 2022-09-03T07:24:23.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RNMekrSdFXQJU/jxPi5Lqkv9oKY= 2022-09-03T07:24:23.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:24:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RNMekrSdFXQJU/jxPi5Lqkv9oKY='} 2022-09-03T07:24:23.826 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000003be88a5950075d4-0063130127-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:24:23.827 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-75 2022-09-03T07:24:23.828 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:23.828 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:24:23.829 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:24:23.894 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:24:23.895 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:24:23.929 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:23.927+0000 7fd010b09600 1 waiting to reach latest gen 4, on 3.. 2022-09-03T07:24:28.930 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:28.928+0000 7fd010b09600 1 waiting to reach latest gen 4, on 3.. 2022-09-03T07:24:33.930 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:33.929+0000 7fd010b09600 1 waiting to reach latest gen 4, on 3.. 2022-09-03T07:24:38.932 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:38.930+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:24:38.933 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:24:38.934 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:24:43.933 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:43.931+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:24:43.934 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:24:43.934 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:24:48.935 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:48.933+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:24:48.936 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:24:48.936 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:24:53.938 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:53.935+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:24:53.938 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:24:53.938 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:24:58.939 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:24:58.937+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:24:58.940 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:24:58.940 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:03.942 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:03.940+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:25:03.942 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:25:03.943 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:08.944 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:08.942+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:25:08.944 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:25:08.945 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:13.946 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:13.944+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:25:13.947 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:25:13.947 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:18.951 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:18.946+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:25:18.951 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:25:18.952 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:23.951 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:23.949+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:25:23.952 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:25:23.952 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:28.953 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:28.951+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:25:28.954 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:25:28.954 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:33.956 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:33.953+0000 7fd010b09600 1 waiting for incremental sync to catch up: 2022-09-03T07:25:33.957 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:25:33.958 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:38.957 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:38.955+0000 7fd010b09600 1 bucket sync caught up with source: 2022-09-03T07:25:38.958 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:38.958 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:38.959 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:38.955+0000 7fd010b09600 0 bucket checkpoint complete 2022-09-03T07:25:38.979 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-75 2022-09-03T07:25:38.980 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:38.981 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:38.982 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:25:39.048 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:25:39.049 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:25:39.108 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:25:39.098+0000 7faec3b79600 1 bucket sync caught up with source: 2022-09-03T07:25:39.109 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:39.109 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:39.110 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:25:39.098+0000 7faec3b79600 0 bucket checkpoint complete 2022-09-03T07:25:39.128 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-75 2022-09-03T07:25:39.128 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:39.129 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:39.130 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:25:39.186 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:25:39.187 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:25:39.252 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:39.250+0000 7fe4e0676600 1 waiting for incremental sync to catch up: 2022-09-03T07:25:39.252 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:25:39.253 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:44.254 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:44.252+0000 7fe4e0676600 1 waiting for incremental sync to catch up: 2022-09-03T07:25:44.255 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:25:44.255 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:49.257 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:49.254+0000 7fe4e0676600 1 bucket sync caught up with source: 2022-09-03T07:25:49.257 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:49.257 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [00000000002.48.5, 00000000001.59.5, , , , , 00000000001.65.5, , , , , , , , , , , 00000000002.52.5, 00000000001.78.5, , , , , , , , , , ] 2022-09-03T07:25:49.258 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:49.254+0000 7fe4e0676600 0 bucket checkpoint complete 2022-09-03T07:25:49.291 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-75 2022-09-03T07:25:49.292 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:49.292 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:49.293 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:25:49.360 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:25:49.361 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:25:49.407 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:25:49.404+0000 7f2b78842600 1 bucket sync caught up with source: 2022-09-03T07:25:49.407 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.118.5, 00000000002.339.5, 00000000001.376.5, 00000000003.345.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000002.376.5, 00000000005.303.5, 00000000004.221.5] 2022-09-03T07:25:49.408 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.118.5, 00000000002.339.5, 00000000001.376.5, 00000000003.345.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000002.376.5, 00000000005.303.5, 00000000004.221.5] 2022-09-03T07:25:49.408 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:25:49.404+0000 7f2b78842600 0 bucket checkpoint complete 2022-09-03T07:25:49.423 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-75 2022-09-03T07:25:49.424 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:49.425 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:49.426 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:25:49.487 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:25:49.487 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:25:49.552 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:25:49.549+0000 7fe18990e600 1 bucket sync caught up with source: 2022-09-03T07:25:49.553 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.118.5, 00000000002.339.5, 00000000001.376.5, 00000000003.345.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000002.376.5, 00000000005.303.5, 00000000004.221.5] 2022-09-03T07:25:49.553 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.118.5, 00000000002.339.5, 00000000001.376.5, 00000000003.345.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000002.376.5, 00000000005.303.5, 00000000004.221.5] 2022-09-03T07:25:49.553 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:25:49.549+0000 7fe18990e600 0 bucket checkpoint complete 2022-09-03T07:25:49.576 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-75 2022-09-03T07:25:49.577 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-75 2022-09-03T07:25:49.577 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:49.578 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:49.579 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:25:49.640 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:25:49.641 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:25:49.691 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:25:49.689+0000 7fed15c1f600 1 bucket sync caught up with source: 2022-09-03T07:25:49.691 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000002.116.5, 00000000002.195.5, 00000000001.185.5, 00000000003.201.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000002.287.5, 00000000005.313.5, 00000000004.140.5] 2022-09-03T07:25:49.691 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000002.116.5, 00000000002.195.5, 00000000001.185.5, 00000000003.201.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000002.287.5, 00000000005.313.5, 00000000004.140.5] 2022-09-03T07:25:49.692 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:25:49.689+0000 7fed15c1f600 0 bucket checkpoint complete 2022-09-03T07:25:49.716 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-75 2022-09-03T07:25:49.717 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:49.718 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:25:49.719 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:25:49.786 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:25:49.787 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:25:49.824 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:49.822+0000 7f123b161600 1 bucket sync caught up with source: 2022-09-03T07:25:49.824 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000002.116.5, 00000000002.195.5, 00000000001.185.5, 00000000003.201.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000002.287.5, 00000000005.313.5, 00000000004.140.5] 2022-09-03T07:25:49.825 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000002.116.5, 00000000002.195.5, 00000000001.185.5, 00000000003.201.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000002.287.5, 00000000005.313.5, 00000000004.140.5] 2022-09-03T07:25:49.825 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:25:49.822+0000 7f123b161600 0 bucket checkpoint complete 2022-09-03T07:25:49.839 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-75 2022-09-03T07:25:49.840 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:25:49.841 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:49.841 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:49.842 DEBUG:boto:Method: HEAD 2022-09-03T07:25:49.843 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:25:49.843 DEBUG:boto:Data: 2022-09-03T07:25:49.844 DEBUG:boto:Headers: {} 2022-09-03T07:25:49.844 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:49.845 DEBUG:boto:Port: 8000 2022-09-03T07:25:49.846 DEBUG:boto:Params: {} 2022-09-03T07:25:49.847 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:25:49.849 DEBUG:boto:Token: None 2022-09-03T07:25:49.851 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:49 GMT /avysmw-75/ 2022-09-03T07:25:49.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SwsipCHGkE3z+hh2WeVkcts1x9M= 2022-09-03T07:25:49.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SwsipCHGkE3z+hh2WeVkcts1x9M=', 'Content-Length': '0'} 2022-09-03T07:25:49.858 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '25'), ('X-RGW-Bytes-Used', '75'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b7c6ff847449737f-006313017d-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:25:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:49.859 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:49.860 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:49.861 DEBUG:boto:Method: HEAD 2022-09-03T07:25:49.862 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:25:49.863 DEBUG:boto:Data: 2022-09-03T07:25:49.863 DEBUG:boto:Headers: {} 2022-09-03T07:25:49.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:49.865 DEBUG:boto:Port: 8000 2022-09-03T07:25:49.866 DEBUG:boto:Params: {} 2022-09-03T07:25:49.867 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:25:49.868 DEBUG:boto:Token: None 2022-09-03T07:25:49.868 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:49 GMT /avysmw-75/ 2022-09-03T07:25:49.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SwsipCHGkE3z+hh2WeVkcts1x9M= 2022-09-03T07:25:49.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SwsipCHGkE3z+hh2WeVkcts1x9M=', 'Content-Length': '0'} 2022-09-03T07:25:49.874 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '25'), ('X-RGW-Bytes-Used', '75'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ebbdd37cc81b9b6e-006313017d-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:25:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:49.874 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:25:49.875 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:49.876 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:49.876 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.877 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.877 DEBUG:boto:Method: GET 2022-09-03T07:25:49.878 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.878 DEBUG:boto:Data: 2022-09-03T07:25:49.879 DEBUG:boto:Headers: {} 2022-09-03T07:25:49.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:49.880 DEBUG:boto:Port: 8000 2022-09-03T07:25:49.880 DEBUG:boto:Params: {} 2022-09-03T07:25:49.881 DEBUG:boto:Token: None 2022-09-03T07:25:49.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:49 GMT /avysmw-75/?versions 2022-09-03T07:25:49.882 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:phX33yU6cvCnpDMMDMjU6zrZtE0= 2022-09-03T07:25:49.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:phX33yU6cvCnpDMMDMjU6zrZtE0=', 'Content-Length': '0'} 2022-09-03T07:25:49.885 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c83783a9c86f295a-006313017d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:49.886 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:49.888 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:25:49.889 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:25:49.890 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:25:49.890 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:25:49.890 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:25:49.891 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:25:49.891 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:25:49.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:25:49.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:25:49.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:25:49.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:25:49.894 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:25:49.894 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:25:49.895 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:25:49.895 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:25:49.896 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:25:49.896 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:25:49.896 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:25:49.897 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:25:49.897 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:25:49.898 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:25:49.898 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:25:49.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:25:49.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:25:49.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:25:49.901 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:25:49.901 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:49.902 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:49.902 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.903 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.903 DEBUG:boto:Method: GET 2022-09-03T07:25:49.904 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.904 DEBUG:boto:Data: 2022-09-03T07:25:49.905 DEBUG:boto:Headers: {} 2022-09-03T07:25:49.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:49.906 DEBUG:boto:Port: 8000 2022-09-03T07:25:49.906 DEBUG:boto:Params: {} 2022-09-03T07:25:49.907 DEBUG:boto:Token: None 2022-09-03T07:25:49.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:49 GMT /avysmw-75/?versions 2022-09-03T07:25:49.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:phX33yU6cvCnpDMMDMjU6zrZtE0= 2022-09-03T07:25:49.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:phX33yU6cvCnpDMMDMjU6zrZtE0=', 'Content-Length': '0'} 2022-09-03T07:25:49.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007907caa2f8f3c081-006313017d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:49.914 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:49.916 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:25:49.917 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:25:49.917 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:25:49.918 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:25:49.918 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:25:49.918 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:25:49.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:25:49.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:25:49.920 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:25:49.920 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:25:49.921 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:25:49.921 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:25:49.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:25:49.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:25:49.923 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:25:49.923 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:25:49.923 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:25:49.924 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:25:49.924 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:25:49.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:25:49.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:25:49.926 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:25:49.926 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:25:49.927 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:25:49.927 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:25:49.928 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:49.928 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:49.929 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.929 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.930 DEBUG:boto:Method: GET 2022-09-03T07:25:49.930 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.931 DEBUG:boto:Data: 2022-09-03T07:25:49.931 DEBUG:boto:Headers: {} 2022-09-03T07:25:49.932 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:49.932 DEBUG:boto:Port: 8000 2022-09-03T07:25:49.933 DEBUG:boto:Params: {} 2022-09-03T07:25:49.934 DEBUG:boto:Token: None 2022-09-03T07:25:49.934 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:49 GMT /avysmw-75/?versions 2022-09-03T07:25:49.935 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:phX33yU6cvCnpDMMDMjU6zrZtE0= 2022-09-03T07:25:49.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:phX33yU6cvCnpDMMDMjU6zrZtE0=', 'Content-Length': '0'} 2022-09-03T07:25:49.945 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004271ff460a95e6fc-006313017d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:49.946 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:49.948 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:49.949 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:49.949 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.950 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.950 DEBUG:boto:Method: GET 2022-09-03T07:25:49.951 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:25:49.952 DEBUG:boto:Data: 2022-09-03T07:25:49.952 DEBUG:boto:Headers: {} 2022-09-03T07:25:49.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:49.953 DEBUG:boto:Port: 8000 2022-09-03T07:25:49.954 DEBUG:boto:Params: {} 2022-09-03T07:25:49.954 DEBUG:boto:Token: None 2022-09-03T07:25:49.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:49 GMT /avysmw-75/?versions 2022-09-03T07:25:49.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:phX33yU6cvCnpDMMDMjU6zrZtE0= 2022-09-03T07:25:49.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:phX33yU6cvCnpDMMDMjU6zrZtE0=', 'Content-Length': '0'} 2022-09-03T07:25:49.960 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000918c9e3d07a15578-006313017d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:49.961 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:49.963 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:25:49.963 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:49.964 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:49.965 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.965 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.966 DEBUG:boto:Method: GET 2022-09-03T07:25:49.967 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.967 DEBUG:boto:Data: 2022-09-03T07:25:49.968 DEBUG:boto:Headers: {} 2022-09-03T07:25:49.969 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:49.969 DEBUG:boto:Port: 8000 2022-09-03T07:25:49.970 DEBUG:boto:Params: {} 2022-09-03T07:25:49.971 DEBUG:boto:Token: None 2022-09-03T07:25:49.972 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:49 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s928LNdf0G5du+Oxsl2C98ynwEc= 2022-09-03T07:25:49.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s928LNdf0G5du+Oxsl2C98ynwEc=', 'Content-Length': '0'} 2022-09-03T07:25:49.975 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e6e5dc0a5e1f230-006313017d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:49.976 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:49.977 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:49.977 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.978 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.979 DEBUG:boto:Method: GET 2022-09-03T07:25:49.979 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.980 DEBUG:boto:Data: 2022-09-03T07:25:49.981 DEBUG:boto:Headers: {} 2022-09-03T07:25:49.981 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:49.982 DEBUG:boto:Port: 8000 2022-09-03T07:25:49.982 DEBUG:boto:Params: {} 2022-09-03T07:25:49.983 DEBUG:boto:Token: None 2022-09-03T07:25:49.984 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:49 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s928LNdf0G5du+Oxsl2C98ynwEc= 2022-09-03T07:25:49.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s928LNdf0G5du+Oxsl2C98ynwEc=', 'Content-Length': '0'} 2022-09-03T07:25:49.988 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c7300b0529945d2-006313017d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:49.989 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:49.989 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:49.990 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.990 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.991 DEBUG:boto:Method: HEAD 2022-09-03T07:25:49.992 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.992 DEBUG:boto:Data: 2022-09-03T07:25:49.993 DEBUG:boto:Headers: {} 2022-09-03T07:25:49.993 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:49.994 DEBUG:boto:Port: 8000 2022-09-03T07:25:49.994 DEBUG:boto:Params: {} 2022-09-03T07:25:49.995 DEBUG:boto:Token: None 2022-09-03T07:25:49.996 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:49 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:49.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HTWyoLKvh8fSocZbFc2QzWfQ+9U= 2022-09-03T07:25:49.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HTWyoLKvh8fSocZbFc2QzWfQ+9U=', 'Content-Length': '0'} 2022-09-03T07:25:49.999 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a16b693c1fe8d21b-006313017d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.000 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:50.001 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:50.001 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.002 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.002 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.003 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.004 DEBUG:boto:Data: 2022-09-03T07:25:50.004 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.005 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.005 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.006 DEBUG:boto:Params: {} 2022-09-03T07:25:50.006 DEBUG:boto:Token: None 2022-09-03T07:25:50.007 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QEgDQ4qzwsPqLlnJkadrr8o1YTE= 2022-09-03T07:25:50.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QEgDQ4qzwsPqLlnJkadrr8o1YTE=', 'Content-Length': '0'} 2022-09-03T07:25:50.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc44570a644e56d6-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.013 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:25:50.013 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:50.014 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:50.015 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.015 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.016 DEBUG:boto:Method: GET 2022-09-03T07:25:50.016 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.017 DEBUG:boto:Data: 2022-09-03T07:25:50.018 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.018 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.019 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.020 DEBUG:boto:Params: {} 2022-09-03T07:25:50.021 DEBUG:boto:Token: None 2022-09-03T07:25:50.022 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gn7gv5twQfeVD3dK7JB/lCugrkA= 2022-09-03T07:25:50.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gn7gv5twQfeVD3dK7JB/lCugrkA=', 'Content-Length': '0'} 2022-09-03T07:25:50.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000776fe9584ac37fd6-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.028 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:50.030 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:50.031 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.032 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.033 DEBUG:boto:Method: GET 2022-09-03T07:25:50.034 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.035 DEBUG:boto:Data: 2022-09-03T07:25:50.036 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.037 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.038 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.039 DEBUG:boto:Params: {} 2022-09-03T07:25:50.040 DEBUG:boto:Token: None 2022-09-03T07:25:50.041 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:50.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gn7gv5twQfeVD3dK7JB/lCugrkA= 2022-09-03T07:25:50.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gn7gv5twQfeVD3dK7JB/lCugrkA=', 'Content-Length': '0'} 2022-09-03T07:25:50.044 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003be07bc0ae384981-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.045 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.046 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.047 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:50.048 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:50.048 DEBUG:boto:Method: GET 2022-09-03T07:25:50.049 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:25:50.049 DEBUG:boto:Data: 2022-09-03T07:25:50.050 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.051 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.051 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.052 DEBUG:boto:Params: {} 2022-09-03T07:25:50.053 DEBUG:boto:Token: None 2022-09-03T07:25:50.054 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.058 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd3691f59e7ae8fa-006313017e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.058 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.059 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.060 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.061 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:50.062 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:50.063 DEBUG:boto:Method: GET 2022-09-03T07:25:50.063 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:25:50.064 DEBUG:boto:Data: 2022-09-03T07:25:50.065 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.066 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.067 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.068 DEBUG:boto:Params: {} 2022-09-03T07:25:50.069 DEBUG:boto:Token: None 2022-09-03T07:25:50.070 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.075 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b4addc5b322d811-006313017e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.076 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.078 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:50.078 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:50.079 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.080 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:25:50.080 DEBUG:boto:Data: 2022-09-03T07:25:50.081 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.082 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.082 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.083 DEBUG:boto:Params: {} 2022-09-03T07:25:50.083 DEBUG:boto:Token: None 2022-09-03T07:25:50.084 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj11 2022-09-03T07:25:50.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wc62KILT6QINeKTmtv1GOHIZQRA= 2022-09-03T07:25:50.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wc62KILT6QINeKTmtv1GOHIZQRA=', 'Content-Length': '0'} 2022-09-03T07:25:50.087 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083f8d3a37a3c2aac-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.088 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:50.089 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:50.090 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.091 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:25:50.092 DEBUG:boto:Data: 2022-09-03T07:25:50.093 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.095 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.095 DEBUG:boto:Params: {} 2022-09-03T07:25:50.096 DEBUG:boto:Token: None 2022-09-03T07:25:50.096 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj11 2022-09-03T07:25:50.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wc62KILT6QINeKTmtv1GOHIZQRA= 2022-09-03T07:25:50.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wc62KILT6QINeKTmtv1GOHIZQRA=', 'Content-Length': '0'} 2022-09-03T07:25:50.100 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c5fd2087c2ed7c6-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.100 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:25:50.101 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:50.102 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:50.103 DEBUG:boto:Method: GET 2022-09-03T07:25:50.103 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:25:50.104 DEBUG:boto:Data: 2022-09-03T07:25:50.104 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.105 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.105 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.106 DEBUG:boto:Params: {} 2022-09-03T07:25:50.107 DEBUG:boto:Token: None 2022-09-03T07:25:50.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj11 2022-09-03T07:25:50.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pluDrjNp8xZKJMQv9lQuaOQfAOE= 2022-09-03T07:25:50.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pluDrjNp8xZKJMQv9lQuaOQfAOE=', 'Content-Length': '0'} 2022-09-03T07:25:50.112 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000516f9b0d269bd2e2-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.113 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:50.113 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:50.114 DEBUG:boto:Method: GET 2022-09-03T07:25:50.114 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:25:50.115 DEBUG:boto:Data: 2022-09-03T07:25:50.115 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.116 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.117 DEBUG:boto:Params: {} 2022-09-03T07:25:50.117 DEBUG:boto:Token: None 2022-09-03T07:25:50.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj11 2022-09-03T07:25:50.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pluDrjNp8xZKJMQv9lQuaOQfAOE= 2022-09-03T07:25:50.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pluDrjNp8xZKJMQv9lQuaOQfAOE=', 'Content-Length': '0'} 2022-09-03T07:25:50.122 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091420eab4a9c609d-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.123 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.124 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.125 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:50.125 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:50.126 DEBUG:boto:Method: GET 2022-09-03T07:25:50.127 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:25:50.128 DEBUG:boto:Data: 2022-09-03T07:25:50.129 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.130 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.131 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.132 DEBUG:boto:Params: {} 2022-09-03T07:25:50.132 DEBUG:boto:Token: None 2022-09-03T07:25:50.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.137 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000021440017e9b4b2b5-006313017e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.137 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.138 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.139 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.140 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:50.141 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:50.142 DEBUG:boto:Method: GET 2022-09-03T07:25:50.143 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:25:50.144 DEBUG:boto:Data: 2022-09-03T07:25:50.144 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.145 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.146 DEBUG:boto:Params: {} 2022-09-03T07:25:50.147 DEBUG:boto:Token: None 2022-09-03T07:25:50.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.152 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000029f06a0125bdd59a-006313017e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.153 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.154 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:25:50.155 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.155 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.156 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.156 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.157 DEBUG:boto:Method: GET 2022-09-03T07:25:50.157 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.158 DEBUG:boto:Data: 2022-09-03T07:25:50.159 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.160 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.161 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.161 DEBUG:boto:Params: {} 2022-09-03T07:25:50.162 DEBUG:boto:Token: None 2022-09-03T07:25:50.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KfBuDZVxC+AFKHBwEkXPITozU7s= 2022-09-03T07:25:50.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KfBuDZVxC+AFKHBwEkXPITozU7s=', 'Content-Length': '0'} 2022-09-03T07:25:50.166 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f7fb8457637368c-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.167 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.168 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.169 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.169 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.170 DEBUG:boto:Method: GET 2022-09-03T07:25:50.171 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.171 DEBUG:boto:Data: 2022-09-03T07:25:50.172 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.173 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.173 DEBUG:boto:Params: {} 2022-09-03T07:25:50.174 DEBUG:boto:Token: None 2022-09-03T07:25:50.174 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KfBuDZVxC+AFKHBwEkXPITozU7s= 2022-09-03T07:25:50.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KfBuDZVxC+AFKHBwEkXPITozU7s=', 'Content-Length': '0'} 2022-09-03T07:25:50.177 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d925b42c101c07f1-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.178 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.179 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.179 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.180 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.180 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.181 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.181 DEBUG:boto:Data: 2022-09-03T07:25:50.182 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.182 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.183 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.183 DEBUG:boto:Params: {} 2022-09-03T07:25:50.184 DEBUG:boto:Token: None 2022-09-03T07:25:50.185 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fCHkTrvA6AWsdLBNgv9whaYYxo0= 2022-09-03T07:25:50.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fCHkTrvA6AWsdLBNgv9whaYYxo0=', 'Content-Length': '0'} 2022-09-03T07:25:50.189 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000faf35e8a13bdc56f-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.190 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.190 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.191 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.192 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.192 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.193 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.193 DEBUG:boto:Data: 2022-09-03T07:25:50.194 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.195 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.195 DEBUG:boto:Params: {} 2022-09-03T07:25:50.196 DEBUG:boto:Token: None 2022-09-03T07:25:50.196 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fCHkTrvA6AWsdLBNgv9whaYYxo0= 2022-09-03T07:25:50.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fCHkTrvA6AWsdLBNgv9whaYYxo0=', 'Content-Length': '0'} 2022-09-03T07:25:50.199 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009cbb3fc21fc36208-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.200 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:25:50.201 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.201 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.202 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.202 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.203 DEBUG:boto:Method: GET 2022-09-03T07:25:50.203 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.204 DEBUG:boto:Data: 2022-09-03T07:25:50.204 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.205 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.206 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.206 DEBUG:boto:Params: {} 2022-09-03T07:25:50.207 DEBUG:boto:Token: None 2022-09-03T07:25:50.207 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KfBuDZVxC+AFKHBwEkXPITozU7s= 2022-09-03T07:25:50.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KfBuDZVxC+AFKHBwEkXPITozU7s=', 'Content-Length': '0'} 2022-09-03T07:25:50.211 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034101371c699ada7-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.211 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.212 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.213 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.213 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.214 DEBUG:boto:Method: GET 2022-09-03T07:25:50.214 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.215 DEBUG:boto:Data: 2022-09-03T07:25:50.215 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.216 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.216 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.217 DEBUG:boto:Params: {} 2022-09-03T07:25:50.217 DEBUG:boto:Token: None 2022-09-03T07:25:50.218 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:50.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KfBuDZVxC+AFKHBwEkXPITozU7s= 2022-09-03T07:25:50.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KfBuDZVxC+AFKHBwEkXPITozU7s=', 'Content-Length': '0'} 2022-09-03T07:25:50.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000536f8dc233cca97-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.222 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.223 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.224 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:50.225 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:50.226 DEBUG:boto:Method: GET 2022-09-03T07:25:50.227 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:25:50.228 DEBUG:boto:Data: 2022-09-03T07:25:50.229 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.230 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.231 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.231 DEBUG:boto:Params: {} 2022-09-03T07:25:50.232 DEBUG:boto:Token: None 2022-09-03T07:25:50.232 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.236 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e6530acc928b8300-006313017e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.278 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.279 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.280 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.280 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:50.281 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:50.281 DEBUG:boto:Method: GET 2022-09-03T07:25:50.282 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:25:50.283 DEBUG:boto:Data: 2022-09-03T07:25:50.283 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.284 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.285 DEBUG:boto:Params: {} 2022-09-03T07:25:50.285 DEBUG:boto:Token: None 2022-09-03T07:25:50.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.291 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007fe1a4b0ee81f994-006313017e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.292 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.292 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.293 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.293 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.294 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:25:50.295 DEBUG:boto:Data: 2022-09-03T07:25:50.295 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.296 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.297 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.297 DEBUG:boto:Params: {} 2022-09-03T07:25:50.298 DEBUG:boto:Token: None 2022-09-03T07:25:50.299 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115 2022-09-03T07:25:50.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2xWQfEyQ5mUaVciFvuV/TgehlYw= 2022-09-03T07:25:50.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2xWQfEyQ5mUaVciFvuV/TgehlYw=', 'Content-Length': '0'} 2022-09-03T07:25:50.302 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f75d07c19be0124c-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.303 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.303 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.304 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.304 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:25:50.305 DEBUG:boto:Data: 2022-09-03T07:25:50.305 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.306 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.307 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.307 DEBUG:boto:Params: {} 2022-09-03T07:25:50.308 DEBUG:boto:Token: None 2022-09-03T07:25:50.308 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115 2022-09-03T07:25:50.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2xWQfEyQ5mUaVciFvuV/TgehlYw= 2022-09-03T07:25:50.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2xWQfEyQ5mUaVciFvuV/TgehlYw=', 'Content-Length': '0'} 2022-09-03T07:25:50.311 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003bc5236ea81ecbb1-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.312 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:25:50.313 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.314 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.314 DEBUG:boto:Method: GET 2022-09-03T07:25:50.315 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:25:50.315 DEBUG:boto:Data: 2022-09-03T07:25:50.316 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.316 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.317 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.318 DEBUG:boto:Params: {} 2022-09-03T07:25:50.319 DEBUG:boto:Token: None 2022-09-03T07:25:50.319 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115 2022-09-03T07:25:50.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AY/I3ldE0jbDb4nqHHFgRMNQWFE= 2022-09-03T07:25:50.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AY/I3ldE0jbDb4nqHHFgRMNQWFE=', 'Content-Length': '0'} 2022-09-03T07:25:50.322 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063f883df91532be4-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.324 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:50.324 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:50.325 DEBUG:boto:Method: GET 2022-09-03T07:25:50.325 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:25:50.326 DEBUG:boto:Data: 2022-09-03T07:25:50.326 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.327 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.328 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.328 DEBUG:boto:Params: {} 2022-09-03T07:25:50.329 DEBUG:boto:Token: None 2022-09-03T07:25:50.329 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj115 2022-09-03T07:25:50.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AY/I3ldE0jbDb4nqHHFgRMNQWFE= 2022-09-03T07:25:50.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AY/I3ldE0jbDb4nqHHFgRMNQWFE=', 'Content-Length': '0'} 2022-09-03T07:25:50.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e219ceff10660cc4-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.375 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.376 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.376 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:50.377 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:50.377 DEBUG:boto:Method: GET 2022-09-03T07:25:50.378 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:25:50.378 DEBUG:boto:Data: 2022-09-03T07:25:50.379 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.379 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.380 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.380 DEBUG:boto:Params: {} 2022-09-03T07:25:50.381 DEBUG:boto:Token: None 2022-09-03T07:25:50.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.385 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000420376d05ace126e-006313017e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.386 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.387 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.388 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.389 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:50.390 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:50.391 DEBUG:boto:Method: GET 2022-09-03T07:25:50.392 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:25:50.393 DEBUG:boto:Data: 2022-09-03T07:25:50.394 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.396 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.396 DEBUG:boto:Params: {} 2022-09-03T07:25:50.397 DEBUG:boto:Token: None 2022-09-03T07:25:50.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.403 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006bf5d0717eba0c4a-006313017e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.404 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.405 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:25:50.406 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.407 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.408 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.409 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.410 DEBUG:boto:Method: GET 2022-09-03T07:25:50.411 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.412 DEBUG:boto:Data: 2022-09-03T07:25:50.414 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.415 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.416 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.417 DEBUG:boto:Params: {} 2022-09-03T07:25:50.418 DEBUG:boto:Token: None 2022-09-03T07:25:50.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iguL9uKpyUTsYEH6fWa1ytpgEBc= 2022-09-03T07:25:50.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iguL9uKpyUTsYEH6fWa1ytpgEBc=', 'Content-Length': '0'} 2022-09-03T07:25:50.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044ac7ea61eacdb05-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.423 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.424 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.424 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.425 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.425 DEBUG:boto:Method: GET 2022-09-03T07:25:50.426 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.427 DEBUG:boto:Data: 2022-09-03T07:25:50.427 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.428 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.429 DEBUG:boto:Params: {} 2022-09-03T07:25:50.429 DEBUG:boto:Token: None 2022-09-03T07:25:50.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iguL9uKpyUTsYEH6fWa1ytpgEBc= 2022-09-03T07:25:50.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iguL9uKpyUTsYEH6fWa1ytpgEBc=', 'Content-Length': '0'} 2022-09-03T07:25:50.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e3bae986799344d-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.434 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.435 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.435 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.436 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.436 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.437 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.437 DEBUG:boto:Data: 2022-09-03T07:25:50.438 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.438 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.439 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.439 DEBUG:boto:Params: {} 2022-09-03T07:25:50.440 DEBUG:boto:Token: None 2022-09-03T07:25:50.440 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JdC186DcNXmelfxK2M7dFAiJIms= 2022-09-03T07:25:50.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JdC186DcNXmelfxK2M7dFAiJIms=', 'Content-Length': '0'} 2022-09-03T07:25:50.443 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018acbaad8575199c-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.444 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.445 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.445 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.446 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.446 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.447 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.447 DEBUG:boto:Data: 2022-09-03T07:25:50.448 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.449 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.449 DEBUG:boto:Params: {} 2022-09-03T07:25:50.450 DEBUG:boto:Token: None 2022-09-03T07:25:50.450 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JdC186DcNXmelfxK2M7dFAiJIms= 2022-09-03T07:25:50.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JdC186DcNXmelfxK2M7dFAiJIms=', 'Content-Length': '0'} 2022-09-03T07:25:50.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e6dad5fa36c48732-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.454 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:25:50.455 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.455 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.456 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.456 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.457 DEBUG:boto:Method: GET 2022-09-03T07:25:50.457 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.458 DEBUG:boto:Data: 2022-09-03T07:25:50.458 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.459 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.460 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.460 DEBUG:boto:Params: {} 2022-09-03T07:25:50.461 DEBUG:boto:Token: None 2022-09-03T07:25:50.461 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iguL9uKpyUTsYEH6fWa1ytpgEBc= 2022-09-03T07:25:50.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iguL9uKpyUTsYEH6fWa1ytpgEBc=', 'Content-Length': '0'} 2022-09-03T07:25:50.464 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3742613af11b0fc-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.506 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.507 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.507 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.508 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.508 DEBUG:boto:Method: GET 2022-09-03T07:25:50.509 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.510 DEBUG:boto:Data: 2022-09-03T07:25:50.510 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.511 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.511 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.512 DEBUG:boto:Params: {} 2022-09-03T07:25:50.512 DEBUG:boto:Token: None 2022-09-03T07:25:50.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:50.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iguL9uKpyUTsYEH6fWa1ytpgEBc= 2022-09-03T07:25:50.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iguL9uKpyUTsYEH6fWa1ytpgEBc=', 'Content-Length': '0'} 2022-09-03T07:25:50.516 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6a130aa4682194e-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.517 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.518 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.518 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:50.519 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:50.520 DEBUG:boto:Method: GET 2022-09-03T07:25:50.520 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:25:50.521 DEBUG:boto:Data: 2022-09-03T07:25:50.521 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.522 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.523 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.523 DEBUG:boto:Params: {} 2022-09-03T07:25:50.524 DEBUG:boto:Token: None 2022-09-03T07:25:50.524 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.525 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.528 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006bed0182fa4a2fd0-006313017e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.529 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.530 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.530 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.531 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:50.532 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:50.532 DEBUG:boto:Method: GET 2022-09-03T07:25:50.533 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:25:50.533 DEBUG:boto:Data: 2022-09-03T07:25:50.534 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.535 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.535 DEBUG:boto:Params: {} 2022-09-03T07:25:50.536 DEBUG:boto:Token: None 2022-09-03T07:25:50.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.541 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf5ce7c77b797b5b-006313017e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.542 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.543 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.543 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.544 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.544 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:25:50.544 DEBUG:boto:Data: 2022-09-03T07:25:50.545 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.545 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.546 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.546 DEBUG:boto:Params: {} 2022-09-03T07:25:50.547 DEBUG:boto:Token: None 2022-09-03T07:25:50.548 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116 2022-09-03T07:25:50.548 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gRxb8llGEY2Mogg7FepdsQKHAgQ= 2022-09-03T07:25:50.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gRxb8llGEY2Mogg7FepdsQKHAgQ=', 'Content-Length': '0'} 2022-09-03T07:25:50.551 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b084ae81eba9b9d4-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.552 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.552 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.553 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.553 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:25:50.554 DEBUG:boto:Data: 2022-09-03T07:25:50.554 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.555 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.556 DEBUG:boto:Params: {} 2022-09-03T07:25:50.556 DEBUG:boto:Token: None 2022-09-03T07:25:50.557 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116 2022-09-03T07:25:50.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gRxb8llGEY2Mogg7FepdsQKHAgQ= 2022-09-03T07:25:50.558 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gRxb8llGEY2Mogg7FepdsQKHAgQ=', 'Content-Length': '0'} 2022-09-03T07:25:50.560 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008af89f0de5447e23-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.561 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:25:50.562 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.562 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.563 DEBUG:boto:Method: GET 2022-09-03T07:25:50.563 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:25:50.564 DEBUG:boto:Data: 2022-09-03T07:25:50.565 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.565 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.566 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.566 DEBUG:boto:Params: {} 2022-09-03T07:25:50.567 DEBUG:boto:Token: None 2022-09-03T07:25:50.568 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116 2022-09-03T07:25:50.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UG/f/Nr9BFMzhQSv7/t11cIhsyM= 2022-09-03T07:25:50.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UG/f/Nr9BFMzhQSv7/t11cIhsyM=', 'Content-Length': '0'} 2022-09-03T07:25:50.571 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e08a97e9d3e62c3-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.571 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:50.572 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:50.573 DEBUG:boto:Method: GET 2022-09-03T07:25:50.573 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:25:50.574 DEBUG:boto:Data: 2022-09-03T07:25:50.574 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.575 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.575 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.576 DEBUG:boto:Params: {} 2022-09-03T07:25:50.577 DEBUG:boto:Token: None 2022-09-03T07:25:50.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj116 2022-09-03T07:25:50.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UG/f/Nr9BFMzhQSv7/t11cIhsyM= 2022-09-03T07:25:50.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UG/f/Nr9BFMzhQSv7/t11cIhsyM=', 'Content-Length': '0'} 2022-09-03T07:25:50.581 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070f5bf746803b149-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.582 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.582 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.583 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:50.583 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:50.584 DEBUG:boto:Method: GET 2022-09-03T07:25:50.584 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:25:50.585 DEBUG:boto:Data: 2022-09-03T07:25:50.585 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.586 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.587 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.587 DEBUG:boto:Params: {} 2022-09-03T07:25:50.588 DEBUG:boto:Token: None 2022-09-03T07:25:50.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.592 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000099d4973340d578b8-006313017e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.634 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.635 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.635 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.636 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:50.636 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:50.637 DEBUG:boto:Method: GET 2022-09-03T07:25:50.638 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:25:50.638 DEBUG:boto:Data: 2022-09-03T07:25:50.639 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.639 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.640 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.640 DEBUG:boto:Params: {} 2022-09-03T07:25:50.641 DEBUG:boto:Token: None 2022-09-03T07:25:50.642 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.643 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.646 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000035dfb4dd8c385b15-006313017e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.647 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.648 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:25:50.648 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.649 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.650 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.650 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.651 DEBUG:boto:Method: GET 2022-09-03T07:25:50.651 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.652 DEBUG:boto:Data: 2022-09-03T07:25:50.652 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.653 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.653 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.654 DEBUG:boto:Params: {} 2022-09-03T07:25:50.655 DEBUG:boto:Token: None 2022-09-03T07:25:50.655 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:60We9je91CLsHhgU8rxIm3MC95c= 2022-09-03T07:25:50.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:60We9je91CLsHhgU8rxIm3MC95c=', 'Content-Length': '0'} 2022-09-03T07:25:50.659 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037db4aefdcbd35f7-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.660 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.660 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.661 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.661 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.662 DEBUG:boto:Method: GET 2022-09-03T07:25:50.663 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.663 DEBUG:boto:Data: 2022-09-03T07:25:50.664 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.664 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.665 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.665 DEBUG:boto:Params: {} 2022-09-03T07:25:50.666 DEBUG:boto:Token: None 2022-09-03T07:25:50.667 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:60We9je91CLsHhgU8rxIm3MC95c= 2022-09-03T07:25:50.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:60We9je91CLsHhgU8rxIm3MC95c=', 'Content-Length': '0'} 2022-09-03T07:25:50.670 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d978b6eba28313c7-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.714 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.715 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.716 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.716 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.717 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.718 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.718 DEBUG:boto:Data: 2022-09-03T07:25:50.719 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.719 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.720 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.720 DEBUG:boto:Params: {} 2022-09-03T07:25:50.721 DEBUG:boto:Token: None 2022-09-03T07:25:50.722 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.722 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9bsIRxviWd3+VZRZQT/75y2JaFs= 2022-09-03T07:25:50.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9bsIRxviWd3+VZRZQT/75y2JaFs=', 'Content-Length': '0'} 2022-09-03T07:25:50.726 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000172daed893adad81-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.727 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.727 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.728 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.728 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.729 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.729 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.730 DEBUG:boto:Data: 2022-09-03T07:25:50.730 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.731 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.732 DEBUG:boto:Params: {} 2022-09-03T07:25:50.732 DEBUG:boto:Token: None 2022-09-03T07:25:50.733 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9bsIRxviWd3+VZRZQT/75y2JaFs= 2022-09-03T07:25:50.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9bsIRxviWd3+VZRZQT/75y2JaFs=', 'Content-Length': '0'} 2022-09-03T07:25:50.736 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091b2ebe02c6e2fbd-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.737 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:25:50.738 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.739 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.740 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.740 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.741 DEBUG:boto:Method: GET 2022-09-03T07:25:50.741 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.742 DEBUG:boto:Data: 2022-09-03T07:25:50.743 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.743 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.744 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.744 DEBUG:boto:Params: {} 2022-09-03T07:25:50.745 DEBUG:boto:Token: None 2022-09-03T07:25:50.746 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:60We9je91CLsHhgU8rxIm3MC95c= 2022-09-03T07:25:50.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:60We9je91CLsHhgU8rxIm3MC95c=', 'Content-Length': '0'} 2022-09-03T07:25:50.749 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009789cc46d15fc881-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.751 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.751 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.753 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.753 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.754 DEBUG:boto:Method: GET 2022-09-03T07:25:50.755 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.755 DEBUG:boto:Data: 2022-09-03T07:25:50.756 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.758 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.758 DEBUG:boto:Params: {} 2022-09-03T07:25:50.759 DEBUG:boto:Token: None 2022-09-03T07:25:50.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:50.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:60We9je91CLsHhgU8rxIm3MC95c= 2022-09-03T07:25:50.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:60We9je91CLsHhgU8rxIm3MC95c=', 'Content-Length': '0'} 2022-09-03T07:25:50.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090913dd59ca2800b-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.764 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.766 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.767 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:50.767 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:50.768 DEBUG:boto:Method: GET 2022-09-03T07:25:50.769 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:25:50.769 DEBUG:boto:Data: 2022-09-03T07:25:50.770 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.770 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.771 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.771 DEBUG:boto:Params: {} 2022-09-03T07:25:50.772 DEBUG:boto:Token: None 2022-09-03T07:25:50.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.786 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da622534596c0f2b-006313017e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.787 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.788 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.789 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.789 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:50.790 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:50.791 DEBUG:boto:Method: GET 2022-09-03T07:25:50.791 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:25:50.792 DEBUG:boto:Data: 2022-09-03T07:25:50.792 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.793 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.794 DEBUG:boto:Params: {} 2022-09-03T07:25:50.795 DEBUG:boto:Token: None 2022-09-03T07:25:50.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.800 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b1a4588af2d37c95-006313017e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.801 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.802 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.802 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.803 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.803 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:25:50.804 DEBUG:boto:Data: 2022-09-03T07:25:50.804 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.805 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.806 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.806 DEBUG:boto:Params: {} 2022-09-03T07:25:50.807 DEBUG:boto:Token: None 2022-09-03T07:25:50.809 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145 2022-09-03T07:25:50.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hEyLDTD6mIkW5udkKl9rD9se3p8= 2022-09-03T07:25:50.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hEyLDTD6mIkW5udkKl9rD9se3p8=', 'Content-Length': '0'} 2022-09-03T07:25:50.813 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b30fbbd2cfb7d4a-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.813 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.814 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.815 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.815 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:25:50.816 DEBUG:boto:Data: 2022-09-03T07:25:50.816 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.817 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.818 DEBUG:boto:Params: {} 2022-09-03T07:25:50.818 DEBUG:boto:Token: None 2022-09-03T07:25:50.819 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145 2022-09-03T07:25:50.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hEyLDTD6mIkW5udkKl9rD9se3p8= 2022-09-03T07:25:50.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hEyLDTD6mIkW5udkKl9rD9se3p8=', 'Content-Length': '0'} 2022-09-03T07:25:50.822 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3ae9c154a5ae436-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.822 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:25:50.823 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.824 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.824 DEBUG:boto:Method: GET 2022-09-03T07:25:50.825 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:25:50.825 DEBUG:boto:Data: 2022-09-03T07:25:50.826 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.826 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.827 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.827 DEBUG:boto:Params: {} 2022-09-03T07:25:50.828 DEBUG:boto:Token: None 2022-09-03T07:25:50.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145 2022-09-03T07:25:50.830 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lgbJM0RaMtq/X8x11CZVpdTzEhU= 2022-09-03T07:25:50.831 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lgbJM0RaMtq/X8x11CZVpdTzEhU=', 'Content-Length': '0'} 2022-09-03T07:25:50.833 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ebb674b88098a35-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.879 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:50.880 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:50.881 DEBUG:boto:Method: GET 2022-09-03T07:25:50.882 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:25:50.882 DEBUG:boto:Data: 2022-09-03T07:25:50.883 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.884 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.885 DEBUG:boto:Params: {} 2022-09-03T07:25:50.885 DEBUG:boto:Token: None 2022-09-03T07:25:50.886 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj145 2022-09-03T07:25:50.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lgbJM0RaMtq/X8x11CZVpdTzEhU= 2022-09-03T07:25:50.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lgbJM0RaMtq/X8x11CZVpdTzEhU=', 'Content-Length': '0'} 2022-09-03T07:25:50.889 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab54fe1384982911-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.890 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.891 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.891 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:50.892 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:50.893 DEBUG:boto:Method: GET 2022-09-03T07:25:50.893 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:25:50.894 DEBUG:boto:Data: 2022-09-03T07:25:50.895 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.895 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.896 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.896 DEBUG:boto:Params: {} 2022-09-03T07:25:50.897 DEBUG:boto:Token: None 2022-09-03T07:25:50.897 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.901 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b203b36bd9e4e501-006313017e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.902 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.903 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:50.903 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:50.904 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:50.904 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:50.905 DEBUG:boto:Method: GET 2022-09-03T07:25:50.905 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:25:50.906 DEBUG:boto:Data: 2022-09-03T07:25:50.906 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.908 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.908 DEBUG:boto:Params: {} 2022-09-03T07:25:50.909 DEBUG:boto:Token: None 2022-09-03T07:25:50.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/ 2022-09-03T07:25:50.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk= 2022-09-03T07:25:50.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:36dFzoaHWcfn0zhi3Tdw3DIzIEk=', 'Content-Length': '0'} 2022-09-03T07:25:50.915 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076c4d6140e73dfd9-006313017e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.916 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:50.916 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:25:50.917 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:50.918 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:50.918 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.919 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.919 DEBUG:boto:Method: GET 2022-09-03T07:25:50.920 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.921 DEBUG:boto:Data: 2022-09-03T07:25:50.921 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.922 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.922 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.923 DEBUG:boto:Params: {} 2022-09-03T07:25:50.924 DEBUG:boto:Token: None 2022-09-03T07:25:50.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cFcSb62VR4ioB0TqRn0I3Ha+ILE= 2022-09-03T07:25:50.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cFcSb62VR4ioB0TqRn0I3Ha+ILE=', 'Content-Length': '0'} 2022-09-03T07:25:50.928 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000fc1676b05d3f7bd-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.929 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:50.930 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:50.930 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.931 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.931 DEBUG:boto:Method: GET 2022-09-03T07:25:50.932 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.933 DEBUG:boto:Data: 2022-09-03T07:25:50.933 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:50.934 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.935 DEBUG:boto:Params: {} 2022-09-03T07:25:50.935 DEBUG:boto:Token: None 2022-09-03T07:25:50.936 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cFcSb62VR4ioB0TqRn0I3Ha+ILE= 2022-09-03T07:25:50.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cFcSb62VR4ioB0TqRn0I3Ha+ILE=', 'Content-Length': '0'} 2022-09-03T07:25:50.939 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000593c62edb215c4ed-006313017e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.982 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:50.984 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:50.985 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.985 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.987 DEBUG:boto:Method: HEAD 2022-09-03T07:25:50.988 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.989 DEBUG:boto:Data: 2022-09-03T07:25:50.989 DEBUG:boto:Headers: {} 2022-09-03T07:25:50.990 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:50.991 DEBUG:boto:Port: 8000 2022-09-03T07:25:50.992 DEBUG:boto:Params: {} 2022-09-03T07:25:50.993 DEBUG:boto:Token: None 2022-09-03T07:25:50.994 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:50 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:50.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b75+6oarU7CUl/jlGdxU01MyyyM= 2022-09-03T07:25:50.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b75+6oarU7CUl/jlGdxU01MyyyM=', 'Content-Length': '0'} 2022-09-03T07:25:50.997 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d899c11d4a4759b1-006313017e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:50.998 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:50.999 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:51.000 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.001 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.002 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.003 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.004 DEBUG:boto:Data: 2022-09-03T07:25:51.004 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.005 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.006 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.007 DEBUG:boto:Params: {} 2022-09-03T07:25:51.007 DEBUG:boto:Token: None 2022-09-03T07:25:51.008 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NphA0/WY7tyEYbR8qCu1wniRQJc= 2022-09-03T07:25:51.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NphA0/WY7tyEYbR8qCu1wniRQJc=', 'Content-Length': '0'} 2022-09-03T07:25:51.012 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff53e2193af63fe8-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.013 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:25:51.014 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:51.014 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:51.015 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.015 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.016 DEBUG:boto:Method: GET 2022-09-03T07:25:51.016 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.017 DEBUG:boto:Data: 2022-09-03T07:25:51.017 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.018 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.019 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.019 DEBUG:boto:Params: {} 2022-09-03T07:25:51.020 DEBUG:boto:Token: None 2022-09-03T07:25:51.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZgOvltm8Z+sDvb0mhLpcHlPAcdw= 2022-09-03T07:25:51.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZgOvltm8Z+sDvb0mhLpcHlPAcdw=', 'Content-Length': '0'} 2022-09-03T07:25:51.024 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc925a87a6a5360d-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.025 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:51.025 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:51.026 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.027 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.027 DEBUG:boto:Method: GET 2022-09-03T07:25:51.028 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.028 DEBUG:boto:Data: 2022-09-03T07:25:51.029 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.030 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.031 DEBUG:boto:Params: {} 2022-09-03T07:25:51.031 DEBUG:boto:Token: None 2022-09-03T07:25:51.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:51.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZgOvltm8Z+sDvb0mhLpcHlPAcdw= 2022-09-03T07:25:51.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZgOvltm8Z+sDvb0mhLpcHlPAcdw=', 'Content-Length': '0'} 2022-09-03T07:25:51.036 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d08be4e3de93b90-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.037 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.037 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.038 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:51.039 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:51.039 DEBUG:boto:Method: GET 2022-09-03T07:25:51.040 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:25:51.041 DEBUG:boto:Data: 2022-09-03T07:25:51.041 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.042 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.042 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.043 DEBUG:boto:Params: {} 2022-09-03T07:25:51.044 DEBUG:boto:Token: None 2022-09-03T07:25:51.044 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.046 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.050 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e6676b1385d91d4-006313017f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.052 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.053 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.054 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.055 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:51.056 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:51.057 DEBUG:boto:Method: GET 2022-09-03T07:25:51.058 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:25:51.059 DEBUG:boto:Data: 2022-09-03T07:25:51.060 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.060 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.062 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.062 DEBUG:boto:Params: {} 2022-09-03T07:25:51.063 DEBUG:boto:Token: None 2022-09-03T07:25:51.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b66ba60f511ab749-006313017f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.069 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.070 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:51.071 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:51.071 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.072 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:25:51.072 DEBUG:boto:Data: 2022-09-03T07:25:51.073 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.073 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.074 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.074 DEBUG:boto:Params: {} 2022-09-03T07:25:51.075 DEBUG:boto:Token: None 2022-09-03T07:25:51.075 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj17 2022-09-03T07:25:51.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UoTDQqe7fO7Vo55fqqOJFcIBYqg= 2022-09-03T07:25:51.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UoTDQqe7fO7Vo55fqqOJFcIBYqg=', 'Content-Length': '0'} 2022-09-03T07:25:51.079 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a17379a942a558a5-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.079 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:51.080 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:51.081 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.081 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:25:51.082 DEBUG:boto:Data: 2022-09-03T07:25:51.083 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.084 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.084 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.085 DEBUG:boto:Params: {} 2022-09-03T07:25:51.085 DEBUG:boto:Token: None 2022-09-03T07:25:51.086 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj17 2022-09-03T07:25:51.087 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UoTDQqe7fO7Vo55fqqOJFcIBYqg= 2022-09-03T07:25:51.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UoTDQqe7fO7Vo55fqqOJFcIBYqg=', 'Content-Length': '0'} 2022-09-03T07:25:51.089 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d4f9ad3953e5188-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.090 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:25:51.091 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:51.091 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:51.092 DEBUG:boto:Method: GET 2022-09-03T07:25:51.092 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:25:51.093 DEBUG:boto:Data: 2022-09-03T07:25:51.093 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.094 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.095 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.095 DEBUG:boto:Params: {} 2022-09-03T07:25:51.096 DEBUG:boto:Token: None 2022-09-03T07:25:51.096 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj17 2022-09-03T07:25:51.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QxXmCYb3i2gTNfI8MvHXJDuN1Nw= 2022-09-03T07:25:51.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QxXmCYb3i2gTNfI8MvHXJDuN1Nw=', 'Content-Length': '0'} 2022-09-03T07:25:51.100 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000275970e7a4a1c513-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.143 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:51.143 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:51.144 DEBUG:boto:Method: GET 2022-09-03T07:25:51.144 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:25:51.145 DEBUG:boto:Data: 2022-09-03T07:25:51.145 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.146 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.147 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.147 DEBUG:boto:Params: {} 2022-09-03T07:25:51.148 DEBUG:boto:Token: None 2022-09-03T07:25:51.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj17 2022-09-03T07:25:51.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QxXmCYb3i2gTNfI8MvHXJDuN1Nw= 2022-09-03T07:25:51.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QxXmCYb3i2gTNfI8MvHXJDuN1Nw=', 'Content-Length': '0'} 2022-09-03T07:25:51.152 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a213f77b116c2dfe-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.154 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.155 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.155 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:51.156 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:51.158 DEBUG:boto:Method: GET 2022-09-03T07:25:51.159 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:25:51.160 DEBUG:boto:Data: 2022-09-03T07:25:51.160 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.161 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.162 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.163 DEBUG:boto:Params: {} 2022-09-03T07:25:51.164 DEBUG:boto:Token: None 2022-09-03T07:25:51.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.169 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000017991c8a2be009e-006313017f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.170 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.171 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.173 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.173 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:51.174 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:51.175 DEBUG:boto:Method: GET 2022-09-03T07:25:51.175 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:25:51.176 DEBUG:boto:Data: 2022-09-03T07:25:51.176 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.177 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.177 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.178 DEBUG:boto:Params: {} 2022-09-03T07:25:51.178 DEBUG:boto:Token: None 2022-09-03T07:25:51.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.185 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004eb665d296d57dae-006313017f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.185 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.187 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:25:51.188 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.188 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.189 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.189 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.190 DEBUG:boto:Method: GET 2022-09-03T07:25:51.190 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.191 DEBUG:boto:Data: 2022-09-03T07:25:51.191 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.192 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.193 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.193 DEBUG:boto:Params: {} 2022-09-03T07:25:51.194 DEBUG:boto:Token: None 2022-09-03T07:25:51.194 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SySBzrJgJR66GYMwmlR87sFGqak= 2022-09-03T07:25:51.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SySBzrJgJR66GYMwmlR87sFGqak=', 'Content-Length': '0'} 2022-09-03T07:25:51.198 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b605ec8e25c5d2f1-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.199 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.199 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.200 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.201 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.201 DEBUG:boto:Method: GET 2022-09-03T07:25:51.202 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.202 DEBUG:boto:Data: 2022-09-03T07:25:51.203 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.204 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.205 DEBUG:boto:Params: {} 2022-09-03T07:25:51.205 DEBUG:boto:Token: None 2022-09-03T07:25:51.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SySBzrJgJR66GYMwmlR87sFGqak= 2022-09-03T07:25:51.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SySBzrJgJR66GYMwmlR87sFGqak=', 'Content-Length': '0'} 2022-09-03T07:25:51.211 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa340545f1c02b03-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.212 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.212 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.213 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.213 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.214 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.214 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.215 DEBUG:boto:Data: 2022-09-03T07:25:51.215 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.216 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.216 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.217 DEBUG:boto:Params: {} 2022-09-03T07:25:51.218 DEBUG:boto:Token: None 2022-09-03T07:25:51.218 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OSioTmC89YFANhi+YQZYLIX6bWU= 2022-09-03T07:25:51.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OSioTmC89YFANhi+YQZYLIX6bWU=', 'Content-Length': '0'} 2022-09-03T07:25:51.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c72d580dc72db931-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.222 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.223 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.223 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.224 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.224 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.225 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.225 DEBUG:boto:Data: 2022-09-03T07:25:51.226 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.227 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.228 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.228 DEBUG:boto:Params: {} 2022-09-03T07:25:51.229 DEBUG:boto:Token: None 2022-09-03T07:25:51.229 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OSioTmC89YFANhi+YQZYLIX6bWU= 2022-09-03T07:25:51.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OSioTmC89YFANhi+YQZYLIX6bWU=', 'Content-Length': '0'} 2022-09-03T07:25:51.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d4754c0f6fda278-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.234 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:25:51.234 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.235 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.235 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.236 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.236 DEBUG:boto:Method: GET 2022-09-03T07:25:51.237 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.238 DEBUG:boto:Data: 2022-09-03T07:25:51.238 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.239 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.240 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.240 DEBUG:boto:Params: {} 2022-09-03T07:25:51.241 DEBUG:boto:Token: None 2022-09-03T07:25:51.241 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SySBzrJgJR66GYMwmlR87sFGqak= 2022-09-03T07:25:51.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SySBzrJgJR66GYMwmlR87sFGqak=', 'Content-Length': '0'} 2022-09-03T07:25:51.245 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb7e4eeff4eb0298-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.245 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.246 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.247 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.247 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.248 DEBUG:boto:Method: GET 2022-09-03T07:25:51.248 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.249 DEBUG:boto:Data: 2022-09-03T07:25:51.249 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.251 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.251 DEBUG:boto:Params: {} 2022-09-03T07:25:51.252 DEBUG:boto:Token: None 2022-09-03T07:25:51.252 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:51.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SySBzrJgJR66GYMwmlR87sFGqak= 2022-09-03T07:25:51.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SySBzrJgJR66GYMwmlR87sFGqak=', 'Content-Length': '0'} 2022-09-03T07:25:51.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f231664949a2f84f-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.302 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.303 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.304 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:51.304 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:51.305 DEBUG:boto:Method: GET 2022-09-03T07:25:51.306 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:25:51.306 DEBUG:boto:Data: 2022-09-03T07:25:51.307 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.307 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.308 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.308 DEBUG:boto:Params: {} 2022-09-03T07:25:51.309 DEBUG:boto:Token: None 2022-09-03T07:25:51.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.314 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e46c04954d954e81-006313017f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.315 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.317 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.317 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.318 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:51.319 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:51.319 DEBUG:boto:Method: GET 2022-09-03T07:25:51.320 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:25:51.320 DEBUG:boto:Data: 2022-09-03T07:25:51.321 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.322 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.322 DEBUG:boto:Params: {} 2022-09-03T07:25:51.323 DEBUG:boto:Token: None 2022-09-03T07:25:51.324 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.328 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e36ea79c5dc50283-006313017f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.329 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.330 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.330 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.331 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.332 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:25:51.332 DEBUG:boto:Data: 2022-09-03T07:25:51.333 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.333 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.334 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.334 DEBUG:boto:Params: {} 2022-09-03T07:25:51.335 DEBUG:boto:Token: None 2022-09-03T07:25:51.335 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19 2022-09-03T07:25:51.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:05MgfulCt7w09cWz94944kCrCHo= 2022-09-03T07:25:51.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:05MgfulCt7w09cWz94944kCrCHo=', 'Content-Length': '0'} 2022-09-03T07:25:51.338 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008651989394abea74-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.339 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.340 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.340 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.341 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:25:51.341 DEBUG:boto:Data: 2022-09-03T07:25:51.342 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.342 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.343 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.343 DEBUG:boto:Params: {} 2022-09-03T07:25:51.344 DEBUG:boto:Token: None 2022-09-03T07:25:51.344 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19 2022-09-03T07:25:51.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:05MgfulCt7w09cWz94944kCrCHo= 2022-09-03T07:25:51.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:05MgfulCt7w09cWz94944kCrCHo=', 'Content-Length': '0'} 2022-09-03T07:25:51.347 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee03e5d54bc0767e-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.348 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:25:51.349 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.349 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.350 DEBUG:boto:Method: GET 2022-09-03T07:25:51.350 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:25:51.351 DEBUG:boto:Data: 2022-09-03T07:25:51.351 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.352 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.353 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.353 DEBUG:boto:Params: {} 2022-09-03T07:25:51.354 DEBUG:boto:Token: None 2022-09-03T07:25:51.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19 2022-09-03T07:25:51.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tW2v9+a5r8g+dozthomz4hR1Uac= 2022-09-03T07:25:51.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tW2v9+a5r8g+dozthomz4hR1Uac=', 'Content-Length': '0'} 2022-09-03T07:25:51.358 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b2b075eff3774a1-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.398 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:51.399 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:51.399 DEBUG:boto:Method: GET 2022-09-03T07:25:51.400 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:25:51.400 DEBUG:boto:Data: 2022-09-03T07:25:51.401 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.402 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.402 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.403 DEBUG:boto:Params: {} 2022-09-03T07:25:51.403 DEBUG:boto:Token: None 2022-09-03T07:25:51.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj19 2022-09-03T07:25:51.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tW2v9+a5r8g+dozthomz4hR1Uac= 2022-09-03T07:25:51.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tW2v9+a5r8g+dozthomz4hR1Uac=', 'Content-Length': '0'} 2022-09-03T07:25:51.408 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f82cf1d9bcc447b7-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.409 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.410 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.410 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:51.411 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:51.412 DEBUG:boto:Method: GET 2022-09-03T07:25:51.412 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:25:51.413 DEBUG:boto:Data: 2022-09-03T07:25:51.413 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.414 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.414 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.415 DEBUG:boto:Params: {} 2022-09-03T07:25:51.416 DEBUG:boto:Token: None 2022-09-03T07:25:51.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.420 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fdd32997ed56a46e-006313017f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.421 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.421 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.422 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.422 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:51.423 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:51.424 DEBUG:boto:Method: GET 2022-09-03T07:25:51.424 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:25:51.425 DEBUG:boto:Data: 2022-09-03T07:25:51.425 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.426 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.426 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.427 DEBUG:boto:Params: {} 2022-09-03T07:25:51.428 DEBUG:boto:Token: None 2022-09-03T07:25:51.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.429 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.433 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000290458519473e96d-006313017f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.434 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.435 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:25:51.436 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.436 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.437 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.437 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.438 DEBUG:boto:Method: GET 2022-09-03T07:25:51.438 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.439 DEBUG:boto:Data: 2022-09-03T07:25:51.440 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.440 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.441 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.441 DEBUG:boto:Params: {} 2022-09-03T07:25:51.442 DEBUG:boto:Token: None 2022-09-03T07:25:51.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nOhRi3W6RbXtppvM0Ak+qq+IYRs= 2022-09-03T07:25:51.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nOhRi3W6RbXtppvM0Ak+qq+IYRs=', 'Content-Length': '0'} 2022-09-03T07:25:51.446 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000640c1f049f61f3aa-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.447 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.447 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.448 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.448 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.449 DEBUG:boto:Method: GET 2022-09-03T07:25:51.449 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.450 DEBUG:boto:Data: 2022-09-03T07:25:51.450 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.452 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.452 DEBUG:boto:Params: {} 2022-09-03T07:25:51.453 DEBUG:boto:Token: None 2022-09-03T07:25:51.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nOhRi3W6RbXtppvM0Ak+qq+IYRs= 2022-09-03T07:25:51.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nOhRi3W6RbXtppvM0Ak+qq+IYRs=', 'Content-Length': '0'} 2022-09-03T07:25:51.457 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d53029b5455a95c-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.457 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.458 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.459 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.459 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.460 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.460 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.461 DEBUG:boto:Data: 2022-09-03T07:25:51.461 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.462 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.463 DEBUG:boto:Params: {} 2022-09-03T07:25:51.463 DEBUG:boto:Token: None 2022-09-03T07:25:51.464 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TiSfQxW4bUnbsiCFfK68HHH+oYE= 2022-09-03T07:25:51.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TiSfQxW4bUnbsiCFfK68HHH+oYE=', 'Content-Length': '0'} 2022-09-03T07:25:51.470 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c3777e0aa61390b-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.471 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.472 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.472 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.473 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.473 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.474 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.474 DEBUG:boto:Data: 2022-09-03T07:25:51.475 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.476 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.476 DEBUG:boto:Params: {} 2022-09-03T07:25:51.477 DEBUG:boto:Token: None 2022-09-03T07:25:51.478 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TiSfQxW4bUnbsiCFfK68HHH+oYE= 2022-09-03T07:25:51.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TiSfQxW4bUnbsiCFfK68HHH+oYE=', 'Content-Length': '0'} 2022-09-03T07:25:51.481 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c343e9b9323e9341-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.482 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:25:51.482 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.483 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.483 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.484 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.484 DEBUG:boto:Method: GET 2022-09-03T07:25:51.485 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.485 DEBUG:boto:Data: 2022-09-03T07:25:51.486 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.486 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.487 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.488 DEBUG:boto:Params: {} 2022-09-03T07:25:51.488 DEBUG:boto:Token: None 2022-09-03T07:25:51.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nOhRi3W6RbXtppvM0Ak+qq+IYRs= 2022-09-03T07:25:51.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nOhRi3W6RbXtppvM0Ak+qq+IYRs=', 'Content-Length': '0'} 2022-09-03T07:25:51.492 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010f2306a54c8d151-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.538 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.539 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.539 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.540 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.541 DEBUG:boto:Method: GET 2022-09-03T07:25:51.542 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.543 DEBUG:boto:Data: 2022-09-03T07:25:51.543 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.544 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.544 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.545 DEBUG:boto:Params: {} 2022-09-03T07:25:51.546 DEBUG:boto:Token: None 2022-09-03T07:25:51.546 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:51.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nOhRi3W6RbXtppvM0Ak+qq+IYRs= 2022-09-03T07:25:51.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nOhRi3W6RbXtppvM0Ak+qq+IYRs=', 'Content-Length': '0'} 2022-09-03T07:25:51.550 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029dce7de24824e68-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.551 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.551 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.552 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:51.553 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:51.553 DEBUG:boto:Method: GET 2022-09-03T07:25:51.554 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:25:51.554 DEBUG:boto:Data: 2022-09-03T07:25:51.555 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.555 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.556 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.556 DEBUG:boto:Params: {} 2022-09-03T07:25:51.557 DEBUG:boto:Token: None 2022-09-03T07:25:51.558 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.561 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023c987c0f8b9a8cb-006313017f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.562 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.563 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.564 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.564 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:51.565 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:51.565 DEBUG:boto:Method: GET 2022-09-03T07:25:51.566 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:25:51.566 DEBUG:boto:Data: 2022-09-03T07:25:51.567 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.568 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.568 DEBUG:boto:Params: {} 2022-09-03T07:25:51.569 DEBUG:boto:Token: None 2022-09-03T07:25:51.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.574 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006023e9102aa6397b-006313017f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.575 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.576 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.577 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.577 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.578 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:25:51.578 DEBUG:boto:Data: 2022-09-03T07:25:51.579 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.581 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.581 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.582 DEBUG:boto:Params: {} 2022-09-03T07:25:51.583 DEBUG:boto:Token: None 2022-09-03T07:25:51.584 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22 2022-09-03T07:25:51.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pnk1L5NuMZ8BeqdrrBICtm88cA0= 2022-09-03T07:25:51.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pnk1L5NuMZ8BeqdrrBICtm88cA0=', 'Content-Length': '0'} 2022-09-03T07:25:51.588 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9f9966ada20f778-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.590 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.591 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.592 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.593 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:25:51.594 DEBUG:boto:Data: 2022-09-03T07:25:51.595 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.596 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.597 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.598 DEBUG:boto:Params: {} 2022-09-03T07:25:51.599 DEBUG:boto:Token: None 2022-09-03T07:25:51.601 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22 2022-09-03T07:25:51.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pnk1L5NuMZ8BeqdrrBICtm88cA0= 2022-09-03T07:25:51.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pnk1L5NuMZ8BeqdrrBICtm88cA0=', 'Content-Length': '0'} 2022-09-03T07:25:51.605 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bfeae9df2f97cb27-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.607 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:25:51.608 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.609 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.610 DEBUG:boto:Method: GET 2022-09-03T07:25:51.611 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:25:51.612 DEBUG:boto:Data: 2022-09-03T07:25:51.613 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.614 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.615 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.616 DEBUG:boto:Params: {} 2022-09-03T07:25:51.617 DEBUG:boto:Token: None 2022-09-03T07:25:51.618 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22 2022-09-03T07:25:51.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YSCylWTuIYvCFpaXz+0YU9LbbD0= 2022-09-03T07:25:51.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YSCylWTuIYvCFpaXz+0YU9LbbD0=', 'Content-Length': '0'} 2022-09-03T07:25:51.623 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000069f8b1f0e4f3779b-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.624 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:51.624 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:51.625 DEBUG:boto:Method: GET 2022-09-03T07:25:51.625 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:25:51.626 DEBUG:boto:Data: 2022-09-03T07:25:51.627 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.630 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.630 DEBUG:boto:Params: {} 2022-09-03T07:25:51.631 DEBUG:boto:Token: None 2022-09-03T07:25:51.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj22 2022-09-03T07:25:51.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YSCylWTuIYvCFpaXz+0YU9LbbD0= 2022-09-03T07:25:51.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YSCylWTuIYvCFpaXz+0YU9LbbD0=', 'Content-Length': '0'} 2022-09-03T07:25:51.639 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000508c4dd214624deb-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.688 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.691 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.692 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:51.693 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:51.694 DEBUG:boto:Method: GET 2022-09-03T07:25:51.695 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:25:51.696 DEBUG:boto:Data: 2022-09-03T07:25:51.699 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.700 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.701 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.702 DEBUG:boto:Params: {} 2022-09-03T07:25:51.703 DEBUG:boto:Token: None 2022-09-03T07:25:51.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.705 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.709 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a8d4cdf4b2ea608a-006313017f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.711 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.712 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.713 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.713 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:51.714 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:51.715 DEBUG:boto:Method: GET 2022-09-03T07:25:51.715 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:25:51.716 DEBUG:boto:Data: 2022-09-03T07:25:51.716 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.718 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.718 DEBUG:boto:Params: {} 2022-09-03T07:25:51.719 DEBUG:boto:Token: None 2022-09-03T07:25:51.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.725 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000afc954045633584d-006313017f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.726 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.727 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:25:51.727 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.728 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.728 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.729 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.730 DEBUG:boto:Method: GET 2022-09-03T07:25:51.731 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.732 DEBUG:boto:Data: 2022-09-03T07:25:51.732 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.733 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.733 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.734 DEBUG:boto:Params: {} 2022-09-03T07:25:51.735 DEBUG:boto:Token: None 2022-09-03T07:25:51.736 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C46ZEpEShbgsyg3yyyWmroBUadI= 2022-09-03T07:25:51.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C46ZEpEShbgsyg3yyyWmroBUadI=', 'Content-Length': '0'} 2022-09-03T07:25:51.742 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e90b3c799d3568cc-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.743 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.744 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.744 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.745 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.745 DEBUG:boto:Method: GET 2022-09-03T07:25:51.746 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.746 DEBUG:boto:Data: 2022-09-03T07:25:51.747 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.748 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.749 DEBUG:boto:Params: {} 2022-09-03T07:25:51.750 DEBUG:boto:Token: None 2022-09-03T07:25:51.751 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C46ZEpEShbgsyg3yyyWmroBUadI= 2022-09-03T07:25:51.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C46ZEpEShbgsyg3yyyWmroBUadI=', 'Content-Length': '0'} 2022-09-03T07:25:51.755 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000315ad70e404f4b9d-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.756 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.757 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.758 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.759 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.760 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.761 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.762 DEBUG:boto:Data: 2022-09-03T07:25:51.763 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.764 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.765 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.766 DEBUG:boto:Params: {} 2022-09-03T07:25:51.767 DEBUG:boto:Token: None 2022-09-03T07:25:51.768 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DheF03B93OsSMp0ZxK4VtBVimeQ= 2022-09-03T07:25:51.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DheF03B93OsSMp0ZxK4VtBVimeQ=', 'Content-Length': '0'} 2022-09-03T07:25:51.772 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c13e47a7baaec39f-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.774 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.775 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.775 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.776 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.776 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.777 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.777 DEBUG:boto:Data: 2022-09-03T07:25:51.778 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.779 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.779 DEBUG:boto:Params: {} 2022-09-03T07:25:51.780 DEBUG:boto:Token: None 2022-09-03T07:25:51.780 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DheF03B93OsSMp0ZxK4VtBVimeQ= 2022-09-03T07:25:51.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DheF03B93OsSMp0ZxK4VtBVimeQ=', 'Content-Length': '0'} 2022-09-03T07:25:51.783 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d72d741c75b6c1f-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.784 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:25:51.785 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.785 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.786 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.786 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.787 DEBUG:boto:Method: GET 2022-09-03T07:25:51.787 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.788 DEBUG:boto:Data: 2022-09-03T07:25:51.788 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.789 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.790 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.790 DEBUG:boto:Params: {} 2022-09-03T07:25:51.791 DEBUG:boto:Token: None 2022-09-03T07:25:51.791 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C46ZEpEShbgsyg3yyyWmroBUadI= 2022-09-03T07:25:51.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C46ZEpEShbgsyg3yyyWmroBUadI=', 'Content-Length': '0'} 2022-09-03T07:25:51.795 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7737f9ba1b23ecf-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.795 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.796 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.796 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.797 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.798 DEBUG:boto:Method: GET 2022-09-03T07:25:51.798 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.799 DEBUG:boto:Data: 2022-09-03T07:25:51.800 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.800 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.801 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.801 DEBUG:boto:Params: {} 2022-09-03T07:25:51.802 DEBUG:boto:Token: None 2022-09-03T07:25:51.803 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:51.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C46ZEpEShbgsyg3yyyWmroBUadI= 2022-09-03T07:25:51.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C46ZEpEShbgsyg3yyyWmroBUadI=', 'Content-Length': '0'} 2022-09-03T07:25:51.806 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f54591afffacfa98-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.807 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.807 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.808 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:51.808 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:51.809 DEBUG:boto:Method: GET 2022-09-03T07:25:51.810 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:25:51.810 DEBUG:boto:Data: 2022-09-03T07:25:51.811 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.811 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.812 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.812 DEBUG:boto:Params: {} 2022-09-03T07:25:51.813 DEBUG:boto:Token: None 2022-09-03T07:25:51.814 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.818 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aeda06fe9c9d019b-006313017f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.819 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.820 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.821 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.822 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:51.823 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:51.824 DEBUG:boto:Method: GET 2022-09-03T07:25:51.824 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:25:51.825 DEBUG:boto:Data: 2022-09-03T07:25:51.825 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.826 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.827 DEBUG:boto:Params: {} 2022-09-03T07:25:51.828 DEBUG:boto:Token: None 2022-09-03T07:25:51.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.836 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001d9a7090163edb49-006313017f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.878 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.882 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.883 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.884 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.885 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:25:51.885 DEBUG:boto:Data: 2022-09-03T07:25:51.886 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.887 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.887 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.888 DEBUG:boto:Params: {} 2022-09-03T07:25:51.888 DEBUG:boto:Token: None 2022-09-03T07:25:51.889 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23 2022-09-03T07:25:51.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j9s6k+LMRI7qCXr+UuR/oW/RKtg= 2022-09-03T07:25:51.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j9s6k+LMRI7qCXr+UuR/oW/RKtg=', 'Content-Length': '0'} 2022-09-03T07:25:51.894 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000501ae62d6b912a6a-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.895 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.895 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.896 DEBUG:boto:Method: HEAD 2022-09-03T07:25:51.896 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:25:51.897 DEBUG:boto:Data: 2022-09-03T07:25:51.897 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.898 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.899 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.899 DEBUG:boto:Params: {} 2022-09-03T07:25:51.900 DEBUG:boto:Token: None 2022-09-03T07:25:51.901 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23 2022-09-03T07:25:51.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j9s6k+LMRI7qCXr+UuR/oW/RKtg= 2022-09-03T07:25:51.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j9s6k+LMRI7qCXr+UuR/oW/RKtg=', 'Content-Length': '0'} 2022-09-03T07:25:51.904 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078f0a364992d6cd8-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.904 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:25:51.906 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.906 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.907 DEBUG:boto:Method: GET 2022-09-03T07:25:51.907 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:25:51.908 DEBUG:boto:Data: 2022-09-03T07:25:51.908 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.909 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.910 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.910 DEBUG:boto:Params: {} 2022-09-03T07:25:51.911 DEBUG:boto:Token: None 2022-09-03T07:25:51.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23 2022-09-03T07:25:51.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yRumiGaxQtOGlWMYBgMaiGrkLs8= 2022-09-03T07:25:51.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yRumiGaxQtOGlWMYBgMaiGrkLs8=', 'Content-Length': '0'} 2022-09-03T07:25:51.916 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000246145484a961e8-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.917 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:51.917 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:51.918 DEBUG:boto:Method: GET 2022-09-03T07:25:51.919 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:25:51.920 DEBUG:boto:Data: 2022-09-03T07:25:51.921 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.922 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.923 DEBUG:boto:Params: {} 2022-09-03T07:25:51.924 DEBUG:boto:Token: None 2022-09-03T07:25:51.926 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj23 2022-09-03T07:25:51.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yRumiGaxQtOGlWMYBgMaiGrkLs8= 2022-09-03T07:25:51.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yRumiGaxQtOGlWMYBgMaiGrkLs8=', 'Content-Length': '0'} 2022-09-03T07:25:51.930 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024f867f4cd00d364-006313017f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.931 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.931 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.932 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:51.933 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:51.934 DEBUG:boto:Method: GET 2022-09-03T07:25:51.935 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:25:51.936 DEBUG:boto:Data: 2022-09-03T07:25:51.937 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.938 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.939 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.940 DEBUG:boto:Params: {} 2022-09-03T07:25:51.941 DEBUG:boto:Token: None 2022-09-03T07:25:51.942 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.947 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006779b0e1aeb23810-006313017f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.948 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.949 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:51.949 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:51.950 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:51.950 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:51.951 DEBUG:boto:Method: GET 2022-09-03T07:25:51.952 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:25:51.953 DEBUG:boto:Data: 2022-09-03T07:25:51.954 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.955 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:51.956 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.957 DEBUG:boto:Params: {} 2022-09-03T07:25:51.958 DEBUG:boto:Token: None 2022-09-03T07:25:51.959 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/ 2022-09-03T07:25:51.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY= 2022-09-03T07:25:51.961 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PrSRDDGjMgJOt+u01Bi1JUTI3KY=', 'Content-Length': '0'} 2022-09-03T07:25:51.965 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f843f1ae886c96f-006313017f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:51.966 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:51.967 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:25:51.968 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:51.969 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:51.970 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:51.971 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:51.972 DEBUG:boto:Method: GET 2022-09-03T07:25:51.972 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:51.973 DEBUG:boto:Data: 2022-09-03T07:25:51.974 DEBUG:boto:Headers: {} 2022-09-03T07:25:51.975 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:51.976 DEBUG:boto:Port: 8000 2022-09-03T07:25:51.977 DEBUG:boto:Params: {} 2022-09-03T07:25:51.978 DEBUG:boto:Token: None 2022-09-03T07:25:51.979 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:51 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:51.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EYvjg2HKgbpfcprIWvWgQvJntQw= 2022-09-03T07:25:51.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EYvjg2HKgbpfcprIWvWgQvJntQw=', 'Content-Length': '0'} 2022-09-03T07:25:51.985 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002206b36143bb233c-006313017f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.026 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:52.027 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:52.027 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.028 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.028 DEBUG:boto:Method: GET 2022-09-03T07:25:52.029 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.030 DEBUG:boto:Data: 2022-09-03T07:25:52.030 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.031 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.032 DEBUG:boto:Params: {} 2022-09-03T07:25:52.033 DEBUG:boto:Token: None 2022-09-03T07:25:52.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZLAYIUOnfNQQhN4fTJ1RfC5JCig= 2022-09-03T07:25:52.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZLAYIUOnfNQQhN4fTJ1RfC5JCig=', 'Content-Length': '0'} 2022-09-03T07:25:52.039 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065406011b5c669a8-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.040 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:52.041 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:52.041 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.042 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.042 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.043 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.043 DEBUG:boto:Data: 2022-09-03T07:25:52.044 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.044 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.045 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.045 DEBUG:boto:Params: {} 2022-09-03T07:25:52.046 DEBUG:boto:Token: None 2022-09-03T07:25:52.047 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BhDnqHW+5u7elWQrFHMXKp+68NU= 2022-09-03T07:25:52.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BhDnqHW+5u7elWQrFHMXKp+68NU=', 'Content-Length': '0'} 2022-09-03T07:25:52.050 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007247531d8827dc7d-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.051 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:52.051 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:52.052 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.052 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.053 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.053 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.054 DEBUG:boto:Data: 2022-09-03T07:25:52.054 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.055 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.055 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.056 DEBUG:boto:Params: {} 2022-09-03T07:25:52.057 DEBUG:boto:Token: None 2022-09-03T07:25:52.057 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BhDnqHW+5u7elWQrFHMXKp+68NU= 2022-09-03T07:25:52.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BhDnqHW+5u7elWQrFHMXKp+68NU=', 'Content-Length': '0'} 2022-09-03T07:25:52.060 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e818f414dc20cba1-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.061 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:25:52.062 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:52.063 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:52.063 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.064 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.065 DEBUG:boto:Method: GET 2022-09-03T07:25:52.065 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.068 DEBUG:boto:Data: 2022-09-03T07:25:52.069 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.070 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.070 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.071 DEBUG:boto:Params: {} 2022-09-03T07:25:52.072 DEBUG:boto:Token: None 2022-09-03T07:25:52.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZLAYIUOnfNQQhN4fTJ1RfC5JCig= 2022-09-03T07:25:52.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZLAYIUOnfNQQhN4fTJ1RfC5JCig=', 'Content-Length': '0'} 2022-09-03T07:25:52.076 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e8c8c72407f9195-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.077 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:52.079 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:52.079 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.080 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.081 DEBUG:boto:Method: GET 2022-09-03T07:25:52.081 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.082 DEBUG:boto:Data: 2022-09-03T07:25:52.083 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.084 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.085 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.085 DEBUG:boto:Params: {} 2022-09-03T07:25:52.086 DEBUG:boto:Token: None 2022-09-03T07:25:52.087 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:52.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZLAYIUOnfNQQhN4fTJ1RfC5JCig= 2022-09-03T07:25:52.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZLAYIUOnfNQQhN4fTJ1RfC5JCig=', 'Content-Length': '0'} 2022-09-03T07:25:52.090 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa22c0891695954f-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.134 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.135 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.135 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:52.136 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:52.136 DEBUG:boto:Method: GET 2022-09-03T07:25:52.137 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:25:52.138 DEBUG:boto:Data: 2022-09-03T07:25:52.138 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.139 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.139 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.140 DEBUG:boto:Params: {} 2022-09-03T07:25:52.141 DEBUG:boto:Token: None 2022-09-03T07:25:52.141 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.145 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f11f609ec187146d-0063130180-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.147 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.148 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.150 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.154 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:52.154 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:52.155 DEBUG:boto:Method: GET 2022-09-03T07:25:52.156 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:25:52.156 DEBUG:boto:Data: 2022-09-03T07:25:52.157 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.159 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.160 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.161 DEBUG:boto:Params: {} 2022-09-03T07:25:52.162 DEBUG:boto:Token: None 2022-09-03T07:25:52.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.170 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d159e90e0642ef46-0063130180-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.173 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.174 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:52.175 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:52.176 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.178 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:25:52.180 DEBUG:boto:Data: 2022-09-03T07:25:52.181 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.181 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.182 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.186 DEBUG:boto:Params: {} 2022-09-03T07:25:52.187 DEBUG:boto:Token: None 2022-09-03T07:25:52.188 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj29 2022-09-03T07:25:52.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VsJaxwi0Qe3DXAtE7U8uCHLAGmw= 2022-09-03T07:25:52.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VsJaxwi0Qe3DXAtE7U8uCHLAGmw=', 'Content-Length': '0'} 2022-09-03T07:25:52.191 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb3beb0ac89c6604-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.192 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:52.193 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:52.193 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.194 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:25:52.194 DEBUG:boto:Data: 2022-09-03T07:25:52.195 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.195 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.196 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.196 DEBUG:boto:Params: {} 2022-09-03T07:25:52.197 DEBUG:boto:Token: None 2022-09-03T07:25:52.198 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj29 2022-09-03T07:25:52.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VsJaxwi0Qe3DXAtE7U8uCHLAGmw= 2022-09-03T07:25:52.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VsJaxwi0Qe3DXAtE7U8uCHLAGmw=', 'Content-Length': '0'} 2022-09-03T07:25:52.202 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fb16a27b29cf2c9-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.202 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:25:52.203 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:52.204 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:52.204 DEBUG:boto:Method: GET 2022-09-03T07:25:52.205 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:25:52.206 DEBUG:boto:Data: 2022-09-03T07:25:52.206 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.207 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.208 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.209 DEBUG:boto:Params: {} 2022-09-03T07:25:52.210 DEBUG:boto:Token: None 2022-09-03T07:25:52.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj29 2022-09-03T07:25:52.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UAfG+djO6L07bPvrT7wPtNL+do= 2022-09-03T07:25:52.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UAfG+djO6L07bPvrT7wPtNL+do=', 'Content-Length': '0'} 2022-09-03T07:25:52.214 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f683eeeac57426c-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.215 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:52.216 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:52.217 DEBUG:boto:Method: GET 2022-09-03T07:25:52.218 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:25:52.218 DEBUG:boto:Data: 2022-09-03T07:25:52.219 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.220 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.221 DEBUG:boto:Params: {} 2022-09-03T07:25:52.222 DEBUG:boto:Token: None 2022-09-03T07:25:52.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj29 2022-09-03T07:25:52.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8UAfG+djO6L07bPvrT7wPtNL+do= 2022-09-03T07:25:52.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8UAfG+djO6L07bPvrT7wPtNL+do=', 'Content-Length': '0'} 2022-09-03T07:25:52.229 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3592412153cfc11-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.230 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.231 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.231 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:52.232 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:52.232 DEBUG:boto:Method: GET 2022-09-03T07:25:52.233 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:25:52.234 DEBUG:boto:Data: 2022-09-03T07:25:52.234 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.235 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.235 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.236 DEBUG:boto:Params: {} 2022-09-03T07:25:52.237 DEBUG:boto:Token: None 2022-09-03T07:25:52.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.238 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.242 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa1c17070be9f8b3-0063130180-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.282 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.284 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.285 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.286 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:52.287 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:52.288 DEBUG:boto:Method: GET 2022-09-03T07:25:52.289 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:25:52.290 DEBUG:boto:Data: 2022-09-03T07:25:52.291 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.293 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.294 DEBUG:boto:Params: {} 2022-09-03T07:25:52.295 DEBUG:boto:Token: None 2022-09-03T07:25:52.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000078dadd696e92c134-0063130180-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.303 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.304 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:25:52.305 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.307 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.308 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.309 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.311 DEBUG:boto:Method: GET 2022-09-03T07:25:52.311 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.312 DEBUG:boto:Data: 2022-09-03T07:25:52.312 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.313 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.314 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.315 DEBUG:boto:Params: {} 2022-09-03T07:25:52.316 DEBUG:boto:Token: None 2022-09-03T07:25:52.318 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mpcW2Ewr+Hi7yqauq4LrPMHwQ+8= 2022-09-03T07:25:52.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mpcW2Ewr+Hi7yqauq4LrPMHwQ+8=', 'Content-Length': '0'} 2022-09-03T07:25:52.322 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c74c38bf9d3967e-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.324 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.325 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.326 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.327 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.328 DEBUG:boto:Method: GET 2022-09-03T07:25:52.329 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.330 DEBUG:boto:Data: 2022-09-03T07:25:52.331 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.333 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.334 DEBUG:boto:Params: {} 2022-09-03T07:25:52.335 DEBUG:boto:Token: None 2022-09-03T07:25:52.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mpcW2Ewr+Hi7yqauq4LrPMHwQ+8= 2022-09-03T07:25:52.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mpcW2Ewr+Hi7yqauq4LrPMHwQ+8=', 'Content-Length': '0'} 2022-09-03T07:25:52.340 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5da4b044580cb1e-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.342 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.342 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.343 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.344 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.345 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.346 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.347 DEBUG:boto:Data: 2022-09-03T07:25:52.347 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.348 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.349 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.350 DEBUG:boto:Params: {} 2022-09-03T07:25:52.352 DEBUG:boto:Token: None 2022-09-03T07:25:52.353 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G0QAwA61tm0C/02aNXyOeMsXyUI= 2022-09-03T07:25:52.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G0QAwA61tm0C/02aNXyOeMsXyUI=', 'Content-Length': '0'} 2022-09-03T07:25:52.358 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e89c1d0f2002d411-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.360 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.361 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.362 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.363 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.364 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.366 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.367 DEBUG:boto:Data: 2022-09-03T07:25:52.368 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.370 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.371 DEBUG:boto:Params: {} 2022-09-03T07:25:52.372 DEBUG:boto:Token: None 2022-09-03T07:25:52.373 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G0QAwA61tm0C/02aNXyOeMsXyUI= 2022-09-03T07:25:52.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G0QAwA61tm0C/02aNXyOeMsXyUI=', 'Content-Length': '0'} 2022-09-03T07:25:52.376 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e4967b8594b8c1b2-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.377 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:25:52.378 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.379 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.379 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.380 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.381 DEBUG:boto:Method: GET 2022-09-03T07:25:52.381 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.382 DEBUG:boto:Data: 2022-09-03T07:25:52.383 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.384 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.385 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.387 DEBUG:boto:Params: {} 2022-09-03T07:25:52.389 DEBUG:boto:Token: None 2022-09-03T07:25:52.390 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mpcW2Ewr+Hi7yqauq4LrPMHwQ+8= 2022-09-03T07:25:52.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mpcW2Ewr+Hi7yqauq4LrPMHwQ+8=', 'Content-Length': '0'} 2022-09-03T07:25:52.396 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7fefc3deacc11f7-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.396 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.397 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.398 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.398 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.399 DEBUG:boto:Method: GET 2022-09-03T07:25:52.399 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.400 DEBUG:boto:Data: 2022-09-03T07:25:52.400 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.401 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.402 DEBUG:boto:Params: {} 2022-09-03T07:25:52.402 DEBUG:boto:Token: None 2022-09-03T07:25:52.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:52.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mpcW2Ewr+Hi7yqauq4LrPMHwQ+8= 2022-09-03T07:25:52.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mpcW2Ewr+Hi7yqauq4LrPMHwQ+8=', 'Content-Length': '0'} 2022-09-03T07:25:52.406 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000415d98db4a4789f2-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.407 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.408 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.408 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:52.409 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:52.409 DEBUG:boto:Method: GET 2022-09-03T07:25:52.410 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:25:52.410 DEBUG:boto:Data: 2022-09-03T07:25:52.411 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.411 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.412 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.413 DEBUG:boto:Params: {} 2022-09-03T07:25:52.413 DEBUG:boto:Token: None 2022-09-03T07:25:52.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.418 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce265fd43a249339-0063130180-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.419 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.421 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.422 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.423 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:52.425 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:52.426 DEBUG:boto:Method: GET 2022-09-03T07:25:52.427 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:25:52.428 DEBUG:boto:Data: 2022-09-03T07:25:52.429 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.430 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.432 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.433 DEBUG:boto:Params: {} 2022-09-03T07:25:52.435 DEBUG:boto:Token: None 2022-09-03T07:25:52.436 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.443 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003bb63d71041f5635-0063130180-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.486 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.487 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.488 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.489 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.490 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:25:52.491 DEBUG:boto:Data: 2022-09-03T07:25:52.491 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.492 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.493 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.494 DEBUG:boto:Params: {} 2022-09-03T07:25:52.495 DEBUG:boto:Token: None 2022-09-03T07:25:52.498 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33 2022-09-03T07:25:52.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9CsEx5p2fsKJBuEh+R3KEupaBVE= 2022-09-03T07:25:52.500 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9CsEx5p2fsKJBuEh+R3KEupaBVE=', 'Content-Length': '0'} 2022-09-03T07:25:52.504 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000df7df90e10e22e4-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.504 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.505 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.506 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.506 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:25:52.507 DEBUG:boto:Data: 2022-09-03T07:25:52.507 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.508 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.509 DEBUG:boto:Params: {} 2022-09-03T07:25:52.509 DEBUG:boto:Token: None 2022-09-03T07:25:52.510 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33 2022-09-03T07:25:52.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9CsEx5p2fsKJBuEh+R3KEupaBVE= 2022-09-03T07:25:52.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9CsEx5p2fsKJBuEh+R3KEupaBVE=', 'Content-Length': '0'} 2022-09-03T07:25:52.514 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be53844b40d9b0bd-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.514 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:25:52.515 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.516 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.516 DEBUG:boto:Method: GET 2022-09-03T07:25:52.517 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:25:52.517 DEBUG:boto:Data: 2022-09-03T07:25:52.518 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.518 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.519 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.519 DEBUG:boto:Params: {} 2022-09-03T07:25:52.520 DEBUG:boto:Token: None 2022-09-03T07:25:52.521 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33 2022-09-03T07:25:52.521 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UkbKr00nTye3FJhbxdpM+9zhLns= 2022-09-03T07:25:52.522 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UkbKr00nTye3FJhbxdpM+9zhLns=', 'Content-Length': '0'} 2022-09-03T07:25:52.525 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b736ea060fb6fe2-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.566 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:52.568 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:52.569 DEBUG:boto:Method: GET 2022-09-03T07:25:52.570 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:25:52.571 DEBUG:boto:Data: 2022-09-03T07:25:52.572 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.573 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.575 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.576 DEBUG:boto:Params: {} 2022-09-03T07:25:52.577 DEBUG:boto:Token: None 2022-09-03T07:25:52.578 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj33 2022-09-03T07:25:52.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UkbKr00nTye3FJhbxdpM+9zhLns= 2022-09-03T07:25:52.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UkbKr00nTye3FJhbxdpM+9zhLns=', 'Content-Length': '0'} 2022-09-03T07:25:52.582 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a66088e8fdf98633-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.583 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.584 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.584 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:52.585 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:52.585 DEBUG:boto:Method: GET 2022-09-03T07:25:52.586 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:25:52.587 DEBUG:boto:Data: 2022-09-03T07:25:52.587 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.588 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.588 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.589 DEBUG:boto:Params: {} 2022-09-03T07:25:52.590 DEBUG:boto:Token: None 2022-09-03T07:25:52.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f6934baa264fedd8-0063130180-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.596 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.597 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.599 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.600 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:52.600 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:52.601 DEBUG:boto:Method: GET 2022-09-03T07:25:52.602 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:25:52.603 DEBUG:boto:Data: 2022-09-03T07:25:52.604 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.605 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.606 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.607 DEBUG:boto:Params: {} 2022-09-03T07:25:52.608 DEBUG:boto:Token: None 2022-09-03T07:25:52.609 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.615 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000117cc01de45f3642-0063130180-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.616 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.617 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:25:52.618 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.618 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.619 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.619 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.620 DEBUG:boto:Method: GET 2022-09-03T07:25:52.621 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.622 DEBUG:boto:Data: 2022-09-03T07:25:52.623 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.624 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.625 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.626 DEBUG:boto:Params: {} 2022-09-03T07:25:52.627 DEBUG:boto:Token: None 2022-09-03T07:25:52.628 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hQ7sBYQBJ9Y4tGwfP9JCvDpr3fc= 2022-09-03T07:25:52.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hQ7sBYQBJ9Y4tGwfP9JCvDpr3fc=', 'Content-Length': '0'} 2022-09-03T07:25:52.634 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e75a50ac976756ce-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.635 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.636 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.636 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.637 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.637 DEBUG:boto:Method: GET 2022-09-03T07:25:52.638 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.640 DEBUG:boto:Data: 2022-09-03T07:25:52.640 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.642 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.643 DEBUG:boto:Params: {} 2022-09-03T07:25:52.644 DEBUG:boto:Token: None 2022-09-03T07:25:52.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.647 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hQ7sBYQBJ9Y4tGwfP9JCvDpr3fc= 2022-09-03T07:25:52.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hQ7sBYQBJ9Y4tGwfP9JCvDpr3fc=', 'Content-Length': '0'} 2022-09-03T07:25:52.649 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bc87e47409385ba-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.690 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.691 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.692 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.692 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.693 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.693 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.694 DEBUG:boto:Data: 2022-09-03T07:25:52.694 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.695 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.695 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.696 DEBUG:boto:Params: {} 2022-09-03T07:25:52.696 DEBUG:boto:Token: None 2022-09-03T07:25:52.697 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WSrXsK0VvVkAqfl9Hgb/2Qt1hEk= 2022-09-03T07:25:52.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WSrXsK0VvVkAqfl9Hgb/2Qt1hEk=', 'Content-Length': '0'} 2022-09-03T07:25:52.701 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ae64ae6fee6289a-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.701 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.702 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.703 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.711 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.711 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.712 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.712 DEBUG:boto:Data: 2022-09-03T07:25:52.713 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.714 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.715 DEBUG:boto:Params: {} 2022-09-03T07:25:52.715 DEBUG:boto:Token: None 2022-09-03T07:25:52.716 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WSrXsK0VvVkAqfl9Hgb/2Qt1hEk= 2022-09-03T07:25:52.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WSrXsK0VvVkAqfl9Hgb/2Qt1hEk=', 'Content-Length': '0'} 2022-09-03T07:25:52.719 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049e77579ce067405-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.720 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:25:52.720 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.721 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.722 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.723 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.723 DEBUG:boto:Method: GET 2022-09-03T07:25:52.724 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.724 DEBUG:boto:Data: 2022-09-03T07:25:52.725 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.725 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.726 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.726 DEBUG:boto:Params: {} 2022-09-03T07:25:52.727 DEBUG:boto:Token: None 2022-09-03T07:25:52.728 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hQ7sBYQBJ9Y4tGwfP9JCvDpr3fc= 2022-09-03T07:25:52.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hQ7sBYQBJ9Y4tGwfP9JCvDpr3fc=', 'Content-Length': '0'} 2022-09-03T07:25:52.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb51f9cb98896ce5-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.733 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.734 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.734 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.735 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.735 DEBUG:boto:Method: GET 2022-09-03T07:25:52.736 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.737 DEBUG:boto:Data: 2022-09-03T07:25:52.737 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.738 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.739 DEBUG:boto:Params: {} 2022-09-03T07:25:52.739 DEBUG:boto:Token: None 2022-09-03T07:25:52.740 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:52.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hQ7sBYQBJ9Y4tGwfP9JCvDpr3fc= 2022-09-03T07:25:52.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hQ7sBYQBJ9Y4tGwfP9JCvDpr3fc=', 'Content-Length': '0'} 2022-09-03T07:25:52.744 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003442507a8a5aabcf-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.745 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.745 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.746 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:52.747 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:52.747 DEBUG:boto:Method: GET 2022-09-03T07:25:52.748 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:25:52.748 DEBUG:boto:Data: 2022-09-03T07:25:52.749 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.749 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.750 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.751 DEBUG:boto:Params: {} 2022-09-03T07:25:52.751 DEBUG:boto:Token: None 2022-09-03T07:25:52.752 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.756 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000035d627c426430f1-0063130180-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.757 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.758 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.758 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.759 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:52.759 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:52.760 DEBUG:boto:Method: GET 2022-09-03T07:25:52.761 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:25:52.761 DEBUG:boto:Data: 2022-09-03T07:25:52.762 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.763 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.764 DEBUG:boto:Params: {} 2022-09-03T07:25:52.764 DEBUG:boto:Token: None 2022-09-03T07:25:52.765 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.770 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c4838c001eaf5bf-0063130180-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.771 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.771 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.772 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.773 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.773 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:25:52.774 DEBUG:boto:Data: 2022-09-03T07:25:52.775 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.775 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.776 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.776 DEBUG:boto:Params: {} 2022-09-03T07:25:52.777 DEBUG:boto:Token: None 2022-09-03T07:25:52.777 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34 2022-09-03T07:25:52.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9CvuAKbpw+Ggsm0X66dH07e0Ohk= 2022-09-03T07:25:52.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9CvuAKbpw+Ggsm0X66dH07e0Ohk=', 'Content-Length': '0'} 2022-09-03T07:25:52.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054cf59d89eeb1fab-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.782 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.782 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.783 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.783 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:25:52.784 DEBUG:boto:Data: 2022-09-03T07:25:52.785 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.785 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.786 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.786 DEBUG:boto:Params: {} 2022-09-03T07:25:52.787 DEBUG:boto:Token: None 2022-09-03T07:25:52.788 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34 2022-09-03T07:25:52.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9CvuAKbpw+Ggsm0X66dH07e0Ohk= 2022-09-03T07:25:52.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9CvuAKbpw+Ggsm0X66dH07e0Ohk=', 'Content-Length': '0'} 2022-09-03T07:25:52.791 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d2270b393a0b132c-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.792 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:25:52.792 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.793 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.794 DEBUG:boto:Method: GET 2022-09-03T07:25:52.794 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:25:52.795 DEBUG:boto:Data: 2022-09-03T07:25:52.795 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.796 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.796 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.797 DEBUG:boto:Params: {} 2022-09-03T07:25:52.798 DEBUG:boto:Token: None 2022-09-03T07:25:52.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34 2022-09-03T07:25:52.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F68uUFLZIEHUBei/GsEqOaxdC3s= 2022-09-03T07:25:52.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F68uUFLZIEHUBei/GsEqOaxdC3s=', 'Content-Length': '0'} 2022-09-03T07:25:52.802 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9adf90a32bd282f-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.802 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:52.803 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:52.804 DEBUG:boto:Method: GET 2022-09-03T07:25:52.804 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:25:52.805 DEBUG:boto:Data: 2022-09-03T07:25:52.805 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.806 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.806 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.807 DEBUG:boto:Params: {} 2022-09-03T07:25:52.807 DEBUG:boto:Token: None 2022-09-03T07:25:52.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj34 2022-09-03T07:25:52.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F68uUFLZIEHUBei/GsEqOaxdC3s= 2022-09-03T07:25:52.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F68uUFLZIEHUBei/GsEqOaxdC3s=', 'Content-Length': '0'} 2022-09-03T07:25:52.811 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003dc82fc8cc526dc5-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.858 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.860 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.861 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:52.861 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:52.862 DEBUG:boto:Method: GET 2022-09-03T07:25:52.863 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:25:52.863 DEBUG:boto:Data: 2022-09-03T07:25:52.865 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.867 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.867 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.868 DEBUG:boto:Params: {} 2022-09-03T07:25:52.869 DEBUG:boto:Token: None 2022-09-03T07:25:52.870 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.875 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000036dfd1cc65104e7-0063130180-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.876 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.877 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:52.877 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:52.878 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:52.878 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:52.879 DEBUG:boto:Method: GET 2022-09-03T07:25:52.879 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:25:52.880 DEBUG:boto:Data: 2022-09-03T07:25:52.880 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.882 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.883 DEBUG:boto:Params: {} 2022-09-03T07:25:52.884 DEBUG:boto:Token: None 2022-09-03T07:25:52.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/ 2022-09-03T07:25:52.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I= 2022-09-03T07:25:52.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ySeBEGiHkLYbMYhBSjDX8U/V0I=', 'Content-Length': '0'} 2022-09-03T07:25:52.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001feac600dd3021ad-0063130180-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.890 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:52.891 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:25:52.891 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:52.893 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:52.893 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.894 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.895 DEBUG:boto:Method: GET 2022-09-03T07:25:52.896 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.897 DEBUG:boto:Data: 2022-09-03T07:25:52.898 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.899 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.900 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.900 DEBUG:boto:Params: {} 2022-09-03T07:25:52.901 DEBUG:boto:Token: None 2022-09-03T07:25:52.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwBE1BYkGp0I3d4a0KNgbpNhduY= 2022-09-03T07:25:52.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwBE1BYkGp0I3d4a0KNgbpNhduY=', 'Content-Length': '0'} 2022-09-03T07:25:52.906 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db0d4d8ca08b02f0-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.950 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:52.951 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:52.952 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.953 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.954 DEBUG:boto:Method: GET 2022-09-03T07:25:52.956 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.957 DEBUG:boto:Data: 2022-09-03T07:25:52.958 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:52.960 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.961 DEBUG:boto:Params: {} 2022-09-03T07:25:52.962 DEBUG:boto:Token: None 2022-09-03T07:25:52.963 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VwBE1BYkGp0I3d4a0KNgbpNhduY= 2022-09-03T07:25:52.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VwBE1BYkGp0I3d4a0KNgbpNhduY=', 'Content-Length': '0'} 2022-09-03T07:25:52.969 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c67a4577f0f090c-0063130180-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:52.970 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:52.971 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:52.972 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.973 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.975 DEBUG:boto:Method: HEAD 2022-09-03T07:25:52.976 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.977 DEBUG:boto:Data: 2022-09-03T07:25:52.978 DEBUG:boto:Headers: {} 2022-09-03T07:25:52.979 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:52.980 DEBUG:boto:Port: 8000 2022-09-03T07:25:52.981 DEBUG:boto:Params: {} 2022-09-03T07:25:52.982 DEBUG:boto:Token: None 2022-09-03T07:25:52.983 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:52 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:25:52.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9hTirTOsIW21S4aZKp/Dta09JD0= 2022-09-03T07:25:52.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9hTirTOsIW21S4aZKp/Dta09JD0=', 'Content-Length': '0'} 2022-09-03T07:25:52.998 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000edf0a8966dd98534-0063130180-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.000 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:53.001 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:53.002 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.003 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.005 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.006 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.007 DEBUG:boto:Data: 2022-09-03T07:25:53.008 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.010 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.011 DEBUG:boto:Params: {} 2022-09-03T07:25:53.012 DEBUG:boto:Token: None 2022-09-03T07:25:53.015 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e18PfZubC+Ng9IXtJgmS9ankt+A= 2022-09-03T07:25:53.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e18PfZubC+Ng9IXtJgmS9ankt+A=', 'Content-Length': '0'} 2022-09-03T07:25:53.020 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3993c8ab44a1b20-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.021 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:25:53.022 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:53.023 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:53.023 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.024 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.024 DEBUG:boto:Method: GET 2022-09-03T07:25:53.025 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.027 DEBUG:boto:Data: 2022-09-03T07:25:53.028 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.029 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.030 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.031 DEBUG:boto:Params: {} 2022-09-03T07:25:53.032 DEBUG:boto:Token: None 2022-09-03T07:25:53.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:12nMclF7hKycQg8e2X6OKsWV0no= 2022-09-03T07:25:53.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:12nMclF7hKycQg8e2X6OKsWV0no=', 'Content-Length': '0'} 2022-09-03T07:25:53.038 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f174fe0729d2ccd4-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.040 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:53.041 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:53.042 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.043 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.044 DEBUG:boto:Method: GET 2022-09-03T07:25:53.045 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.046 DEBUG:boto:Data: 2022-09-03T07:25:53.047 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.048 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.048 DEBUG:boto:Params: {} 2022-09-03T07:25:53.049 DEBUG:boto:Token: None 2022-09-03T07:25:53.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:25:53.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:12nMclF7hKycQg8e2X6OKsWV0no= 2022-09-03T07:25:53.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:12nMclF7hKycQg8e2X6OKsWV0no=', 'Content-Length': '0'} 2022-09-03T07:25:53.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dcdfc95ddcbec238-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.054 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.054 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.055 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:25:53.055 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:25:53.056 DEBUG:boto:Method: GET 2022-09-03T07:25:53.057 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:25:53.057 DEBUG:boto:Data: 2022-09-03T07:25:53.058 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.058 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.059 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.059 DEBUG:boto:Params: {} 2022-09-03T07:25:53.060 DEBUG:boto:Token: None 2022-09-03T07:25:53.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.064 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c6016efdf83d021-0063130181-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.066 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.067 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.068 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.069 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:25:53.071 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:25:53.072 DEBUG:boto:Method: GET 2022-09-03T07:25:53.073 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:25:53.074 DEBUG:boto:Data: 2022-09-03T07:25:53.075 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.078 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.079 DEBUG:boto:Params: {} 2022-09-03T07:25:53.080 DEBUG:boto:Token: None 2022-09-03T07:25:53.081 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.088 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000052a73ed97982d2bf-0063130181-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.089 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.090 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:53.091 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:53.092 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.093 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:25:53.094 DEBUG:boto:Data: 2022-09-03T07:25:53.095 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.096 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.096 DEBUG:boto:Params: {} 2022-09-03T07:25:53.097 DEBUG:boto:Token: None 2022-09-03T07:25:53.097 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj38 2022-09-03T07:25:53.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I07fEq9+1G6UU1AyWe+bJ2W91oY= 2022-09-03T07:25:53.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I07fEq9+1G6UU1AyWe+bJ2W91oY=', 'Content-Length': '0'} 2022-09-03T07:25:53.101 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd9ca31076bb63f3-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.102 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:53.103 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:53.104 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.105 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:25:53.106 DEBUG:boto:Data: 2022-09-03T07:25:53.107 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.108 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.109 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.110 DEBUG:boto:Params: {} 2022-09-03T07:25:53.111 DEBUG:boto:Token: None 2022-09-03T07:25:53.112 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj38 2022-09-03T07:25:53.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I07fEq9+1G6UU1AyWe+bJ2W91oY= 2022-09-03T07:25:53.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I07fEq9+1G6UU1AyWe+bJ2W91oY=', 'Content-Length': '0'} 2022-09-03T07:25:53.116 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000612ddc1e8eea7f5-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.117 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:25:53.117 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:53.118 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:53.119 DEBUG:boto:Method: GET 2022-09-03T07:25:53.119 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:25:53.120 DEBUG:boto:Data: 2022-09-03T07:25:53.120 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.121 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.121 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.122 DEBUG:boto:Params: {} 2022-09-03T07:25:53.122 DEBUG:boto:Token: None 2022-09-03T07:25:53.123 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj38 2022-09-03T07:25:53.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D7QXWKyFHAZ99zF8SYoI29KNnHU= 2022-09-03T07:25:53.124 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D7QXWKyFHAZ99zF8SYoI29KNnHU=', 'Content-Length': '0'} 2022-09-03T07:25:53.126 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca361f8b25cc9f15-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.127 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:53.128 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:53.128 DEBUG:boto:Method: GET 2022-09-03T07:25:53.129 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:25:53.129 DEBUG:boto:Data: 2022-09-03T07:25:53.130 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.130 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.131 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.132 DEBUG:boto:Params: {} 2022-09-03T07:25:53.132 DEBUG:boto:Token: None 2022-09-03T07:25:53.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj38 2022-09-03T07:25:53.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D7QXWKyFHAZ99zF8SYoI29KNnHU= 2022-09-03T07:25:53.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D7QXWKyFHAZ99zF8SYoI29KNnHU=', 'Content-Length': '0'} 2022-09-03T07:25:53.136 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a044d6cb8de8b70-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.137 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.138 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.138 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:25:53.139 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:25:53.139 DEBUG:boto:Method: GET 2022-09-03T07:25:53.140 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:25:53.140 DEBUG:boto:Data: 2022-09-03T07:25:53.141 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.141 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.142 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.142 DEBUG:boto:Params: {} 2022-09-03T07:25:53.143 DEBUG:boto:Token: None 2022-09-03T07:25:53.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.147 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009623abf727517ff1-0063130181-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.190 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.192 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.192 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.193 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:25:53.194 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:25:53.195 DEBUG:boto:Method: GET 2022-09-03T07:25:53.196 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:25:53.196 DEBUG:boto:Data: 2022-09-03T07:25:53.197 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.198 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.198 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.199 DEBUG:boto:Params: {} 2022-09-03T07:25:53.199 DEBUG:boto:Token: None 2022-09-03T07:25:53.200 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.206 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005db43539cd55777a-0063130181-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.207 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.207 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:25:53.208 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.208 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.209 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.210 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.210 DEBUG:boto:Method: GET 2022-09-03T07:25:53.211 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.211 DEBUG:boto:Data: 2022-09-03T07:25:53.212 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.212 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.213 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.213 DEBUG:boto:Params: {} 2022-09-03T07:25:53.214 DEBUG:boto:Token: None 2022-09-03T07:25:53.215 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gCZ/JpXQI+ETEhxhP1OSWBb6ca4= 2022-09-03T07:25:53.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gCZ/JpXQI+ETEhxhP1OSWBb6ca4=', 'Content-Length': '0'} 2022-09-03T07:25:53.219 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049e064642e3d8347-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.220 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.221 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.221 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.222 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.222 DEBUG:boto:Method: GET 2022-09-03T07:25:53.223 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.224 DEBUG:boto:Data: 2022-09-03T07:25:53.224 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.225 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.226 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.228 DEBUG:boto:Params: {} 2022-09-03T07:25:53.229 DEBUG:boto:Token: None 2022-09-03T07:25:53.230 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gCZ/JpXQI+ETEhxhP1OSWBb6ca4= 2022-09-03T07:25:53.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gCZ/JpXQI+ETEhxhP1OSWBb6ca4=', 'Content-Length': '0'} 2022-09-03T07:25:53.234 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000795b880c6b157dd6-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.279 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.280 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.280 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.281 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.281 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.282 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.282 DEBUG:boto:Data: 2022-09-03T07:25:53.283 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.283 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.284 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.285 DEBUG:boto:Params: {} 2022-09-03T07:25:53.287 DEBUG:boto:Token: None 2022-09-03T07:25:53.292 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xSfshLILx536FKbO5CZKUVQsxSY= 2022-09-03T07:25:53.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xSfshLILx536FKbO5CZKUVQsxSY=', 'Content-Length': '0'} 2022-09-03T07:25:53.385 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae9dcb9838ce623c-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.386 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.386 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.387 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.387 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.388 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.388 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.389 DEBUG:boto:Data: 2022-09-03T07:25:53.389 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.390 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.391 DEBUG:boto:Params: {} 2022-09-03T07:25:53.392 DEBUG:boto:Token: None 2022-09-03T07:25:53.392 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xSfshLILx536FKbO5CZKUVQsxSY= 2022-09-03T07:25:53.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xSfshLILx536FKbO5CZKUVQsxSY=', 'Content-Length': '0'} 2022-09-03T07:25:53.395 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f09aa904b135f1bb-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.396 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:25:53.397 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.397 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.398 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.398 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.399 DEBUG:boto:Method: GET 2022-09-03T07:25:53.399 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.400 DEBUG:boto:Data: 2022-09-03T07:25:53.400 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.401 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.401 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.402 DEBUG:boto:Params: {} 2022-09-03T07:25:53.403 DEBUG:boto:Token: None 2022-09-03T07:25:53.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gCZ/JpXQI+ETEhxhP1OSWBb6ca4= 2022-09-03T07:25:53.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gCZ/JpXQI+ETEhxhP1OSWBb6ca4=', 'Content-Length': '0'} 2022-09-03T07:25:53.410 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2dbe4f0d9ba9761-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.412 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.413 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.414 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.414 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.415 DEBUG:boto:Method: GET 2022-09-03T07:25:53.415 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.416 DEBUG:boto:Data: 2022-09-03T07:25:53.416 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.417 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.417 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.418 DEBUG:boto:Params: {} 2022-09-03T07:25:53.418 DEBUG:boto:Token: None 2022-09-03T07:25:53.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:25:53.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gCZ/JpXQI+ETEhxhP1OSWBb6ca4= 2022-09-03T07:25:53.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gCZ/JpXQI+ETEhxhP1OSWBb6ca4=', 'Content-Length': '0'} 2022-09-03T07:25:53.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d814a0e1844d53d0-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.423 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.424 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.424 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:25:53.425 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:25:53.425 DEBUG:boto:Method: GET 2022-09-03T07:25:53.426 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:25:53.426 DEBUG:boto:Data: 2022-09-03T07:25:53.427 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.427 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.428 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.429 DEBUG:boto:Params: {} 2022-09-03T07:25:53.429 DEBUG:boto:Token: None 2022-09-03T07:25:53.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.434 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008630414b7b5ed782-0063130181-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.435 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.436 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.436 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.437 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:25:53.437 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:25:53.438 DEBUG:boto:Method: GET 2022-09-03T07:25:53.438 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:25:53.439 DEBUG:boto:Data: 2022-09-03T07:25:53.439 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.440 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.440 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.441 DEBUG:boto:Params: {} 2022-09-03T07:25:53.441 DEBUG:boto:Token: None 2022-09-03T07:25:53.442 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.447 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037426f1cf810b15c-0063130181-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.448 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.449 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.450 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.450 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.451 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:25:53.451 DEBUG:boto:Data: 2022-09-03T07:25:53.452 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.452 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.453 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.453 DEBUG:boto:Params: {} 2022-09-03T07:25:53.454 DEBUG:boto:Token: None 2022-09-03T07:25:53.455 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44 2022-09-03T07:25:53.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9db1NQQ7ZRYeWL+4U4iPnqUPkDM= 2022-09-03T07:25:53.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9db1NQQ7ZRYeWL+4U4iPnqUPkDM=', 'Content-Length': '0'} 2022-09-03T07:25:53.458 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb54b99d817d13f6-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.458 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.459 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.460 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.460 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:25:53.461 DEBUG:boto:Data: 2022-09-03T07:25:53.461 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.462 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.463 DEBUG:boto:Params: {} 2022-09-03T07:25:53.463 DEBUG:boto:Token: None 2022-09-03T07:25:53.464 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44 2022-09-03T07:25:53.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9db1NQQ7ZRYeWL+4U4iPnqUPkDM= 2022-09-03T07:25:53.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9db1NQQ7ZRYeWL+4U4iPnqUPkDM=', 'Content-Length': '0'} 2022-09-03T07:25:53.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ac5ddbd7dab74fb-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.468 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:25:53.468 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.469 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.469 DEBUG:boto:Method: GET 2022-09-03T07:25:53.470 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:25:53.470 DEBUG:boto:Data: 2022-09-03T07:25:53.471 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.471 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.472 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.472 DEBUG:boto:Params: {} 2022-09-03T07:25:53.473 DEBUG:boto:Token: None 2022-09-03T07:25:53.474 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44 2022-09-03T07:25:53.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dr1AqhivYBE0MPWH+i32LeYeZ8w= 2022-09-03T07:25:53.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dr1AqhivYBE0MPWH+i32LeYeZ8w=', 'Content-Length': '0'} 2022-09-03T07:25:53.477 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091a0f99278407cc4-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.478 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:25:53.478 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:25:53.479 DEBUG:boto:Method: GET 2022-09-03T07:25:53.479 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:25:53.480 DEBUG:boto:Data: 2022-09-03T07:25:53.480 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.481 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.481 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.482 DEBUG:boto:Params: {} 2022-09-03T07:25:53.482 DEBUG:boto:Token: None 2022-09-03T07:25:53.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj44 2022-09-03T07:25:53.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dr1AqhivYBE0MPWH+i32LeYeZ8w= 2022-09-03T07:25:53.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dr1AqhivYBE0MPWH+i32LeYeZ8w=', 'Content-Length': '0'} 2022-09-03T07:25:53.486 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066a36cfecc056e96-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.487 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.488 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.488 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:25:53.489 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:25:53.489 DEBUG:boto:Method: GET 2022-09-03T07:25:53.490 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:25:53.491 DEBUG:boto:Data: 2022-09-03T07:25:53.491 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.492 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.492 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.493 DEBUG:boto:Params: {} 2022-09-03T07:25:53.494 DEBUG:boto:Token: None 2022-09-03T07:25:53.495 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.499 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000458fda9638713f4e-0063130181-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.499 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.500 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.501 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.501 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:25:53.502 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:25:53.502 DEBUG:boto:Method: GET 2022-09-03T07:25:53.503 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:25:53.503 DEBUG:boto:Data: 2022-09-03T07:25:53.504 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.505 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.506 DEBUG:boto:Params: {} 2022-09-03T07:25:53.506 DEBUG:boto:Token: None 2022-09-03T07:25:53.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.511 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c921be206811448-0063130181-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.554 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.555 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:25:53.556 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.557 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.557 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.558 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.558 DEBUG:boto:Method: GET 2022-09-03T07:25:53.559 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.559 DEBUG:boto:Data: 2022-09-03T07:25:53.560 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.560 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.561 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.562 DEBUG:boto:Params: {} 2022-09-03T07:25:53.562 DEBUG:boto:Token: None 2022-09-03T07:25:53.563 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GgaReaDJvF5Rncu4svBVg8XZ2Rc= 2022-09-03T07:25:53.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GgaReaDJvF5Rncu4svBVg8XZ2Rc=', 'Content-Length': '0'} 2022-09-03T07:25:53.566 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000073da96c9d85ab378-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.567 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.568 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.569 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.569 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.570 DEBUG:boto:Method: GET 2022-09-03T07:25:53.570 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.571 DEBUG:boto:Data: 2022-09-03T07:25:53.571 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.572 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.572 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.573 DEBUG:boto:Params: {} 2022-09-03T07:25:53.574 DEBUG:boto:Token: None 2022-09-03T07:25:53.574 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.575 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GgaReaDJvF5Rncu4svBVg8XZ2Rc= 2022-09-03T07:25:53.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GgaReaDJvF5Rncu4svBVg8XZ2Rc=', 'Content-Length': '0'} 2022-09-03T07:25:53.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013711563fca2b07f-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.579 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.579 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.580 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.580 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.581 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.581 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.582 DEBUG:boto:Data: 2022-09-03T07:25:53.582 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.583 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.583 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.584 DEBUG:boto:Params: {} 2022-09-03T07:25:53.584 DEBUG:boto:Token: None 2022-09-03T07:25:53.585 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1mfSugCqLYHDfza6VA0lSfS+hBQ= 2022-09-03T07:25:53.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1mfSugCqLYHDfza6VA0lSfS+hBQ=', 'Content-Length': '0'} 2022-09-03T07:25:53.588 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000833a958ee97978d6-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.589 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.590 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.590 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.591 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.591 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.592 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.592 DEBUG:boto:Data: 2022-09-03T07:25:53.593 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.594 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.594 DEBUG:boto:Params: {} 2022-09-03T07:25:53.595 DEBUG:boto:Token: None 2022-09-03T07:25:53.596 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1mfSugCqLYHDfza6VA0lSfS+hBQ= 2022-09-03T07:25:53.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1mfSugCqLYHDfza6VA0lSfS+hBQ=', 'Content-Length': '0'} 2022-09-03T07:25:53.599 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000085af5a205fd0c1ea-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.599 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:25:53.600 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.601 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.601 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.602 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.602 DEBUG:boto:Method: GET 2022-09-03T07:25:53.603 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.603 DEBUG:boto:Data: 2022-09-03T07:25:53.604 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.605 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.605 DEBUG:boto:Params: {} 2022-09-03T07:25:53.606 DEBUG:boto:Token: None 2022-09-03T07:25:53.607 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GgaReaDJvF5Rncu4svBVg8XZ2Rc= 2022-09-03T07:25:53.610 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GgaReaDJvF5Rncu4svBVg8XZ2Rc=', 'Content-Length': '0'} 2022-09-03T07:25:53.613 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff5c640e17ec2fc3-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.655 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.656 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.656 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.657 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.657 DEBUG:boto:Method: GET 2022-09-03T07:25:53.658 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.658 DEBUG:boto:Data: 2022-09-03T07:25:53.659 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.660 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.661 DEBUG:boto:Params: {} 2022-09-03T07:25:53.662 DEBUG:boto:Token: None 2022-09-03T07:25:53.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:25:53.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GgaReaDJvF5Rncu4svBVg8XZ2Rc= 2022-09-03T07:25:53.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GgaReaDJvF5Rncu4svBVg8XZ2Rc=', 'Content-Length': '0'} 2022-09-03T07:25:53.666 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9c1b821733bbd9f-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.667 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.668 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.668 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:25:53.669 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:25:53.670 DEBUG:boto:Method: GET 2022-09-03T07:25:53.670 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:25:53.671 DEBUG:boto:Data: 2022-09-03T07:25:53.672 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.672 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.673 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.673 DEBUG:boto:Params: {} 2022-09-03T07:25:53.674 DEBUG:boto:Token: None 2022-09-03T07:25:53.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.679 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e0711b9341d1e9d8-0063130181-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.680 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.681 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.682 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.682 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:25:53.683 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:25:53.683 DEBUG:boto:Method: GET 2022-09-03T07:25:53.684 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:25:53.685 DEBUG:boto:Data: 2022-09-03T07:25:53.685 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.686 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.687 DEBUG:boto:Params: {} 2022-09-03T07:25:53.687 DEBUG:boto:Token: None 2022-09-03T07:25:53.688 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.695 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000006183210249cf086-0063130181-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.738 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.739 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.740 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.740 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.741 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:25:53.741 DEBUG:boto:Data: 2022-09-03T07:25:53.742 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.742 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.743 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.743 DEBUG:boto:Params: {} 2022-09-03T07:25:53.744 DEBUG:boto:Token: None 2022-09-03T07:25:53.746 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46 2022-09-03T07:25:53.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pXkDdlHDlurvsOL9ZyL/7ltgWyk= 2022-09-03T07:25:53.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pXkDdlHDlurvsOL9ZyL/7ltgWyk=', 'Content-Length': '0'} 2022-09-03T07:25:53.749 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9a6c242ffa04294-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.750 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.750 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.751 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.752 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:25:53.752 DEBUG:boto:Data: 2022-09-03T07:25:53.753 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.753 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.754 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.754 DEBUG:boto:Params: {} 2022-09-03T07:25:53.755 DEBUG:boto:Token: None 2022-09-03T07:25:53.755 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46 2022-09-03T07:25:53.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pXkDdlHDlurvsOL9ZyL/7ltgWyk= 2022-09-03T07:25:53.757 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pXkDdlHDlurvsOL9ZyL/7ltgWyk=', 'Content-Length': '0'} 2022-09-03T07:25:53.759 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ef028c496495bcd-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.760 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:25:53.760 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.761 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.761 DEBUG:boto:Method: GET 2022-09-03T07:25:53.762 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:25:53.762 DEBUG:boto:Data: 2022-09-03T07:25:53.763 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.763 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.764 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.764 DEBUG:boto:Params: {} 2022-09-03T07:25:53.765 DEBUG:boto:Token: None 2022-09-03T07:25:53.766 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46 2022-09-03T07:25:53.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RnBfI/N0M+idV+iOr91sgnCFrdc= 2022-09-03T07:25:53.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RnBfI/N0M+idV+iOr91sgnCFrdc=', 'Content-Length': '0'} 2022-09-03T07:25:53.770 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f79024ae5273b603-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.771 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:25:53.772 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:25:53.772 DEBUG:boto:Method: GET 2022-09-03T07:25:53.773 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:25:53.773 DEBUG:boto:Data: 2022-09-03T07:25:53.774 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.775 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.775 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.776 DEBUG:boto:Params: {} 2022-09-03T07:25:53.777 DEBUG:boto:Token: None 2022-09-03T07:25:53.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj46 2022-09-03T07:25:53.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RnBfI/N0M+idV+iOr91sgnCFrdc= 2022-09-03T07:25:53.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RnBfI/N0M+idV+iOr91sgnCFrdc=', 'Content-Length': '0'} 2022-09-03T07:25:53.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5af3ce990d6d5ba-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.781 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.783 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.784 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:25:53.784 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:25:53.785 DEBUG:boto:Method: GET 2022-09-03T07:25:53.786 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:25:53.786 DEBUG:boto:Data: 2022-09-03T07:25:53.787 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.787 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.788 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.788 DEBUG:boto:Params: {} 2022-09-03T07:25:53.789 DEBUG:boto:Token: None 2022-09-03T07:25:53.790 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.794 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037ac944867c4b2dc-0063130181-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.794 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.795 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.796 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.796 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:25:53.797 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:25:53.797 DEBUG:boto:Method: GET 2022-09-03T07:25:53.798 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:25:53.798 DEBUG:boto:Data: 2022-09-03T07:25:53.799 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.800 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.800 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.801 DEBUG:boto:Params: {} 2022-09-03T07:25:53.801 DEBUG:boto:Token: None 2022-09-03T07:25:53.802 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.807 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009545c9b2dc654b66-0063130181-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.807 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.808 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:25:53.809 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:53.809 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:53.810 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.810 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.811 DEBUG:boto:Method: GET 2022-09-03T07:25:53.811 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.812 DEBUG:boto:Data: 2022-09-03T07:25:53.813 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.813 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.814 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.814 DEBUG:boto:Params: {} 2022-09-03T07:25:53.815 DEBUG:boto:Token: None 2022-09-03T07:25:53.816 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OcYff9y4txNDGGgJg2rvD4I2ERg= 2022-09-03T07:25:53.817 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OcYff9y4txNDGGgJg2rvD4I2ERg=', 'Content-Length': '0'} 2022-09-03T07:25:53.819 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3d3ee55ae3c0efb-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.820 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:53.821 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:53.822 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.823 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.823 DEBUG:boto:Method: GET 2022-09-03T07:25:53.824 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.825 DEBUG:boto:Data: 2022-09-03T07:25:53.826 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.827 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.828 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.829 DEBUG:boto:Params: {} 2022-09-03T07:25:53.830 DEBUG:boto:Token: None 2022-09-03T07:25:53.832 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OcYff9y4txNDGGgJg2rvD4I2ERg= 2022-09-03T07:25:53.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OcYff9y4txNDGGgJg2rvD4I2ERg=', 'Content-Length': '0'} 2022-09-03T07:25:53.836 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0099cd84e0de86f-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.878 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:53.879 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:53.879 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.880 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.881 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.881 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.882 DEBUG:boto:Data: 2022-09-03T07:25:53.883 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.884 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.885 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.886 DEBUG:boto:Params: {} 2022-09-03T07:25:53.887 DEBUG:boto:Token: None 2022-09-03T07:25:53.888 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vi6CuePXYeokD1G4Y9aAFJfdf+Q= 2022-09-03T07:25:53.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vi6CuePXYeokD1G4Y9aAFJfdf+Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.893 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000014d03ae01251fbbf-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.894 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:53.895 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:53.896 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.897 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.898 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.899 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.900 DEBUG:boto:Data: 2022-09-03T07:25:53.901 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.902 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.903 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.904 DEBUG:boto:Params: {} 2022-09-03T07:25:53.906 DEBUG:boto:Token: None 2022-09-03T07:25:53.907 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vi6CuePXYeokD1G4Y9aAFJfdf+Q= 2022-09-03T07:25:53.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vi6CuePXYeokD1G4Y9aAFJfdf+Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.914 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005183d46105ca613f-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.915 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:25:53.915 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:53.916 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:53.916 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.917 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.917 DEBUG:boto:Method: GET 2022-09-03T07:25:53.918 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.919 DEBUG:boto:Data: 2022-09-03T07:25:53.919 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.920 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.920 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.921 DEBUG:boto:Params: {} 2022-09-03T07:25:53.922 DEBUG:boto:Token: None 2022-09-03T07:25:53.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OcYff9y4txNDGGgJg2rvD4I2ERg= 2022-09-03T07:25:53.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OcYff9y4txNDGGgJg2rvD4I2ERg=', 'Content-Length': '0'} 2022-09-03T07:25:53.926 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005b1705891c7e5a47-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.926 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:53.927 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:53.928 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.928 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.929 DEBUG:boto:Method: GET 2022-09-03T07:25:53.929 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.930 DEBUG:boto:Data: 2022-09-03T07:25:53.930 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.931 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.932 DEBUG:boto:Params: {} 2022-09-03T07:25:53.935 DEBUG:boto:Token: None 2022-09-03T07:25:53.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:25:53.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OcYff9y4txNDGGgJg2rvD4I2ERg= 2022-09-03T07:25:53.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OcYff9y4txNDGGgJg2rvD4I2ERg=', 'Content-Length': '0'} 2022-09-03T07:25:53.940 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d41f5421591fa53-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.941 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.941 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.942 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:25:53.942 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:25:53.943 DEBUG:boto:Method: GET 2022-09-03T07:25:53.943 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:25:53.944 DEBUG:boto:Data: 2022-09-03T07:25:53.945 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.945 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.946 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.946 DEBUG:boto:Params: {} 2022-09-03T07:25:53.947 DEBUG:boto:Token: None 2022-09-03T07:25:53.947 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.948 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.951 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dd1b2d0f51ed6844-0063130181-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.952 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.952 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:53.953 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:53.954 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:25:53.954 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:25:53.955 DEBUG:boto:Method: GET 2022-09-03T07:25:53.955 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:25:53.956 DEBUG:boto:Data: 2022-09-03T07:25:53.956 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.957 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.957 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.958 DEBUG:boto:Params: {} 2022-09-03T07:25:53.958 DEBUG:boto:Token: None 2022-09-03T07:25:53.959 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/ 2022-09-03T07:25:53.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q= 2022-09-03T07:25:53.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRmeTLoHZrOjPQZXb28NW7CqK1Q=', 'Content-Length': '0'} 2022-09-03T07:25:53.964 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003aabd05661473bfb-0063130181-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.965 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:53.965 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:53.966 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:53.966 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.967 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:25:53.967 DEBUG:boto:Data: 2022-09-03T07:25:53.968 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.969 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.969 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.970 DEBUG:boto:Params: {} 2022-09-03T07:25:53.971 DEBUG:boto:Token: None 2022-09-03T07:25:53.972 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj51 2022-09-03T07:25:53.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPu3rJy342PWxkf/cUWZRc4A62U= 2022-09-03T07:25:53.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPu3rJy342PWxkf/cUWZRc4A62U=', 'Content-Length': '0'} 2022-09-03T07:25:53.976 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ffb3e956fa74c3a-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.977 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:53.978 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:53.978 DEBUG:boto:Method: HEAD 2022-09-03T07:25:53.979 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:25:53.979 DEBUG:boto:Data: 2022-09-03T07:25:53.980 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:53.981 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.981 DEBUG:boto:Params: {} 2022-09-03T07:25:53.982 DEBUG:boto:Token: None 2022-09-03T07:25:53.983 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj51 2022-09-03T07:25:53.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wPu3rJy342PWxkf/cUWZRc4A62U= 2022-09-03T07:25:53.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wPu3rJy342PWxkf/cUWZRc4A62U=', 'Content-Length': '0'} 2022-09-03T07:25:53.986 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006723ee873373ba0f-0063130181-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:53.986 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:25:53.987 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:53.988 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:53.988 DEBUG:boto:Method: GET 2022-09-03T07:25:53.989 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:25:53.989 DEBUG:boto:Data: 2022-09-03T07:25:53.990 DEBUG:boto:Headers: {} 2022-09-03T07:25:53.990 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:53.991 DEBUG:boto:Port: 8000 2022-09-03T07:25:53.991 DEBUG:boto:Params: {} 2022-09-03T07:25:53.992 DEBUG:boto:Token: None 2022-09-03T07:25:53.992 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:53 GMT /avysmw-75/obj51 2022-09-03T07:25:53.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NgUDIEJyTRL5jiZGllhODs4Pzyg= 2022-09-03T07:25:53.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NgUDIEJyTRL5jiZGllhODs4Pzyg=', 'Content-Length': '0'} 2022-09-03T07:25:53.997 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ccd0f699229af5cd-0063130181-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.038 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:25:54.039 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:25:54.040 DEBUG:boto:Method: GET 2022-09-03T07:25:54.040 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:25:54.041 DEBUG:boto:Data: 2022-09-03T07:25:54.041 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.042 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.042 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.043 DEBUG:boto:Params: {} 2022-09-03T07:25:54.043 DEBUG:boto:Token: None 2022-09-03T07:25:54.044 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj51 2022-09-03T07:25:54.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AND6YNkgk56xce3nwoU8UcQh5RA= 2022-09-03T07:25:54.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AND6YNkgk56xce3nwoU8UcQh5RA=', 'Content-Length': '0'} 2022-09-03T07:25:54.048 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000166043a13a6f89c9-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.049 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.050 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.050 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:25:54.051 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:25:54.051 DEBUG:boto:Method: GET 2022-09-03T07:25:54.052 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:25:54.053 DEBUG:boto:Data: 2022-09-03T07:25:54.053 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.054 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.054 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.055 DEBUG:boto:Params: {} 2022-09-03T07:25:54.056 DEBUG:boto:Token: None 2022-09-03T07:25:54.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.059 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.061 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d3f42f3338c027a-0063130182-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.062 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.063 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.064 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.064 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:25:54.065 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:25:54.065 DEBUG:boto:Method: GET 2022-09-03T07:25:54.066 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:25:54.066 DEBUG:boto:Data: 2022-09-03T07:25:54.067 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.068 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.068 DEBUG:boto:Params: {} 2022-09-03T07:25:54.069 DEBUG:boto:Token: None 2022-09-03T07:25:54.070 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.075 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c2b02e416e9bf9f9-0063130182-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.075 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.076 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:25:54.077 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.077 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.078 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.078 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.079 DEBUG:boto:Method: GET 2022-09-03T07:25:54.080 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.080 DEBUG:boto:Data: 2022-09-03T07:25:54.081 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.081 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.082 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.082 DEBUG:boto:Params: {} 2022-09-03T07:25:54.083 DEBUG:boto:Token: None 2022-09-03T07:25:54.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NcxcEkpoFo62p44HwZeYFE6qAyA= 2022-09-03T07:25:54.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NcxcEkpoFo62p44HwZeYFE6qAyA=', 'Content-Length': '0'} 2022-09-03T07:25:54.087 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa2be19dc526e46b-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.088 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.088 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.089 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.089 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.090 DEBUG:boto:Method: GET 2022-09-03T07:25:54.090 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.091 DEBUG:boto:Data: 2022-09-03T07:25:54.091 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.092 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.093 DEBUG:boto:Params: {} 2022-09-03T07:25:54.094 DEBUG:boto:Token: None 2022-09-03T07:25:54.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NcxcEkpoFo62p44HwZeYFE6qAyA= 2022-09-03T07:25:54.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NcxcEkpoFo62p44HwZeYFE6qAyA=', 'Content-Length': '0'} 2022-09-03T07:25:54.098 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c6aa894b31906cf2-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.138 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.139 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.139 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.140 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.140 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.141 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.141 DEBUG:boto:Data: 2022-09-03T07:25:54.142 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.142 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.143 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.144 DEBUG:boto:Params: {} 2022-09-03T07:25:54.144 DEBUG:boto:Token: None 2022-09-03T07:25:54.145 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.145 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nbL41QdRVk1WPPCVWsNR0zURV9g= 2022-09-03T07:25:54.146 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nbL41QdRVk1WPPCVWsNR0zURV9g=', 'Content-Length': '0'} 2022-09-03T07:25:54.148 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000072899d02872167f3-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.149 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.149 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.150 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.150 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.151 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.151 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.152 DEBUG:boto:Data: 2022-09-03T07:25:54.152 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.153 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.153 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.154 DEBUG:boto:Params: {} 2022-09-03T07:25:54.154 DEBUG:boto:Token: None 2022-09-03T07:25:54.155 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nbL41QdRVk1WPPCVWsNR0zURV9g= 2022-09-03T07:25:54.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nbL41QdRVk1WPPCVWsNR0zURV9g=', 'Content-Length': '0'} 2022-09-03T07:25:54.158 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005dc20a6d656eb49-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.159 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:25:54.160 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.160 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.161 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.161 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.162 DEBUG:boto:Method: GET 2022-09-03T07:25:54.162 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.163 DEBUG:boto:Data: 2022-09-03T07:25:54.163 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.164 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.165 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.166 DEBUG:boto:Params: {} 2022-09-03T07:25:54.166 DEBUG:boto:Token: None 2022-09-03T07:25:54.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.168 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NcxcEkpoFo62p44HwZeYFE6qAyA= 2022-09-03T07:25:54.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NcxcEkpoFo62p44HwZeYFE6qAyA=', 'Content-Length': '0'} 2022-09-03T07:25:54.170 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000abd01a67a7ad55d7-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.171 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.172 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.172 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.173 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.173 DEBUG:boto:Method: GET 2022-09-03T07:25:54.174 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.174 DEBUG:boto:Data: 2022-09-03T07:25:54.175 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.176 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.176 DEBUG:boto:Params: {} 2022-09-03T07:25:54.177 DEBUG:boto:Token: None 2022-09-03T07:25:54.178 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:25:54.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NcxcEkpoFo62p44HwZeYFE6qAyA= 2022-09-03T07:25:54.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NcxcEkpoFo62p44HwZeYFE6qAyA=', 'Content-Length': '0'} 2022-09-03T07:25:54.181 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be491226e56c04a5-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.182 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.183 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.183 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:25:54.184 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:25:54.184 DEBUG:boto:Method: GET 2022-09-03T07:25:54.185 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:25:54.185 DEBUG:boto:Data: 2022-09-03T07:25:54.186 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.187 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.187 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.188 DEBUG:boto:Params: {} 2022-09-03T07:25:54.189 DEBUG:boto:Token: None 2022-09-03T07:25:54.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.193 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000034507a62cde4c293-0063130182-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.194 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.195 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.195 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.196 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:25:54.197 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:25:54.197 DEBUG:boto:Method: GET 2022-09-03T07:25:54.198 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:25:54.198 DEBUG:boto:Data: 2022-09-03T07:25:54.199 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.199 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.200 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.200 DEBUG:boto:Params: {} 2022-09-03T07:25:54.202 DEBUG:boto:Token: None 2022-09-03T07:25:54.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dbbc52d662d72762-0063130182-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.210 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.211 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.212 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.212 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.213 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:25:54.213 DEBUG:boto:Data: 2022-09-03T07:25:54.214 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.214 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.215 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.216 DEBUG:boto:Params: {} 2022-09-03T07:25:54.216 DEBUG:boto:Token: None 2022-09-03T07:25:54.217 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55 2022-09-03T07:25:54.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cVH5VdpLbmKxIpx18UQhRxlB3jc= 2022-09-03T07:25:54.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cVH5VdpLbmKxIpx18UQhRxlB3jc=', 'Content-Length': '0'} 2022-09-03T07:25:54.220 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b57bb9acc5ce980-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.221 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.221 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.222 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.222 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:25:54.223 DEBUG:boto:Data: 2022-09-03T07:25:54.223 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.224 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.225 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.226 DEBUG:boto:Params: {} 2022-09-03T07:25:54.226 DEBUG:boto:Token: None 2022-09-03T07:25:54.227 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55 2022-09-03T07:25:54.227 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cVH5VdpLbmKxIpx18UQhRxlB3jc= 2022-09-03T07:25:54.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cVH5VdpLbmKxIpx18UQhRxlB3jc=', 'Content-Length': '0'} 2022-09-03T07:25:54.230 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3a19be29f7ebc6a-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.231 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:25:54.231 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.232 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.232 DEBUG:boto:Method: GET 2022-09-03T07:25:54.233 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:25:54.233 DEBUG:boto:Data: 2022-09-03T07:25:54.234 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.235 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.235 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.236 DEBUG:boto:Params: {} 2022-09-03T07:25:54.236 DEBUG:boto:Token: None 2022-09-03T07:25:54.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55 2022-09-03T07:25:54.238 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7D8oP0vJNaz2N+rJw1NUPV7v+FY= 2022-09-03T07:25:54.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7D8oP0vJNaz2N+rJw1NUPV7v+FY=', 'Content-Length': '0'} 2022-09-03T07:25:54.242 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1485d32e69f1ed7-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.244 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:25:54.245 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:25:54.246 DEBUG:boto:Method: GET 2022-09-03T07:25:54.246 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:25:54.247 DEBUG:boto:Data: 2022-09-03T07:25:54.248 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.249 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.249 DEBUG:boto:Params: {} 2022-09-03T07:25:54.250 DEBUG:boto:Token: None 2022-09-03T07:25:54.250 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj55 2022-09-03T07:25:54.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7D8oP0vJNaz2N+rJw1NUPV7v+FY= 2022-09-03T07:25:54.252 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7D8oP0vJNaz2N+rJw1NUPV7v+FY=', 'Content-Length': '0'} 2022-09-03T07:25:54.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a00131d4dcdad63c-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.297 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.298 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.299 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:25:54.299 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:25:54.300 DEBUG:boto:Method: GET 2022-09-03T07:25:54.300 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:25:54.301 DEBUG:boto:Data: 2022-09-03T07:25:54.301 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.302 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.302 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.303 DEBUG:boto:Params: {} 2022-09-03T07:25:54.303 DEBUG:boto:Token: None 2022-09-03T07:25:54.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.310 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000025a5881e16815c95-0063130182-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.312 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.313 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.314 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.315 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:25:54.316 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:25:54.317 DEBUG:boto:Method: GET 2022-09-03T07:25:54.318 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:25:54.319 DEBUG:boto:Data: 2022-09-03T07:25:54.320 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.321 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.322 DEBUG:boto:Params: {} 2022-09-03T07:25:54.323 DEBUG:boto:Token: None 2022-09-03T07:25:54.323 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.328 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f17d0399c6fea25b-0063130182-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.330 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.331 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:25:54.332 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.333 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.334 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.335 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.336 DEBUG:boto:Method: GET 2022-09-03T07:25:54.337 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.338 DEBUG:boto:Data: 2022-09-03T07:25:54.338 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.339 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.340 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.341 DEBUG:boto:Params: {} 2022-09-03T07:25:54.342 DEBUG:boto:Token: None 2022-09-03T07:25:54.343 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dAdR8QSGYd9xdy3DScVKckld3gs= 2022-09-03T07:25:54.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dAdR8QSGYd9xdy3DScVKckld3gs=', 'Content-Length': '0'} 2022-09-03T07:25:54.347 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6e346452aaaa960-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.390 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.392 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.393 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.394 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.394 DEBUG:boto:Method: GET 2022-09-03T07:25:54.395 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.395 DEBUG:boto:Data: 2022-09-03T07:25:54.396 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.397 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.397 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.398 DEBUG:boto:Params: {} 2022-09-03T07:25:54.398 DEBUG:boto:Token: None 2022-09-03T07:25:54.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dAdR8QSGYd9xdy3DScVKckld3gs= 2022-09-03T07:25:54.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dAdR8QSGYd9xdy3DScVKckld3gs=', 'Content-Length': '0'} 2022-09-03T07:25:54.403 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ca7404bde40df9a-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.404 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.404 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.405 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.405 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.406 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.406 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.407 DEBUG:boto:Data: 2022-09-03T07:25:54.407 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.408 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.408 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.409 DEBUG:boto:Params: {} 2022-09-03T07:25:54.409 DEBUG:boto:Token: None 2022-09-03T07:25:54.410 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sWvnt3H4vuONlgI+AabgKfWnixQ= 2022-09-03T07:25:54.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sWvnt3H4vuONlgI+AabgKfWnixQ=', 'Content-Length': '0'} 2022-09-03T07:25:54.413 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dcbe09318da6b15e-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.414 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.415 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.415 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.416 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.416 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.417 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.417 DEBUG:boto:Data: 2022-09-03T07:25:54.418 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.419 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.420 DEBUG:boto:Params: {} 2022-09-03T07:25:54.422 DEBUG:boto:Token: None 2022-09-03T07:25:54.423 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sWvnt3H4vuONlgI+AabgKfWnixQ= 2022-09-03T07:25:54.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sWvnt3H4vuONlgI+AabgKfWnixQ=', 'Content-Length': '0'} 2022-09-03T07:25:54.428 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009872610318b6fbc7-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.430 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:25:54.431 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.431 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.432 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.432 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.433 DEBUG:boto:Method: GET 2022-09-03T07:25:54.433 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.434 DEBUG:boto:Data: 2022-09-03T07:25:54.435 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.435 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.436 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.436 DEBUG:boto:Params: {} 2022-09-03T07:25:54.437 DEBUG:boto:Token: None 2022-09-03T07:25:54.438 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dAdR8QSGYd9xdy3DScVKckld3gs= 2022-09-03T07:25:54.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dAdR8QSGYd9xdy3DScVKckld3gs=', 'Content-Length': '0'} 2022-09-03T07:25:54.441 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e4ee743032672b1-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.442 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.443 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.443 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.444 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.445 DEBUG:boto:Method: GET 2022-09-03T07:25:54.446 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.446 DEBUG:boto:Data: 2022-09-03T07:25:54.447 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.447 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.448 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.449 DEBUG:boto:Params: {} 2022-09-03T07:25:54.449 DEBUG:boto:Token: None 2022-09-03T07:25:54.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:25:54.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dAdR8QSGYd9xdy3DScVKckld3gs= 2022-09-03T07:25:54.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dAdR8QSGYd9xdy3DScVKckld3gs=', 'Content-Length': '0'} 2022-09-03T07:25:54.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000757a87f3a8fea7a1-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.454 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.455 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.455 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:25:54.456 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:25:54.457 DEBUG:boto:Method: GET 2022-09-03T07:25:54.457 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:25:54.458 DEBUG:boto:Data: 2022-09-03T07:25:54.458 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.459 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.460 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.460 DEBUG:boto:Params: {} 2022-09-03T07:25:54.461 DEBUG:boto:Token: None 2022-09-03T07:25:54.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.465 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003fbb0836214507ec-0063130182-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.466 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.467 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.468 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.468 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:25:54.469 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:25:54.469 DEBUG:boto:Method: GET 2022-09-03T07:25:54.470 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:25:54.470 DEBUG:boto:Data: 2022-09-03T07:25:54.471 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.472 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.473 DEBUG:boto:Params: {} 2022-09-03T07:25:54.473 DEBUG:boto:Token: None 2022-09-03T07:25:54.474 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.475 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.479 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2621e68c5ef4984-0063130182-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.480 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.480 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.481 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.482 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.482 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:25:54.483 DEBUG:boto:Data: 2022-09-03T07:25:54.483 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.484 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.484 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.485 DEBUG:boto:Params: {} 2022-09-03T07:25:54.485 DEBUG:boto:Token: None 2022-09-03T07:25:54.486 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57 2022-09-03T07:25:54.487 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gXekX+5F5wks1XmK2gNUQNA7hxU= 2022-09-03T07:25:54.487 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gXekX+5F5wks1XmK2gNUQNA7hxU=', 'Content-Length': '0'} 2022-09-03T07:25:54.489 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da2ed3dd87f706d0-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.490 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.491 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.491 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.492 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:25:54.492 DEBUG:boto:Data: 2022-09-03T07:25:54.493 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.494 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.494 DEBUG:boto:Params: {} 2022-09-03T07:25:54.495 DEBUG:boto:Token: None 2022-09-03T07:25:54.495 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57 2022-09-03T07:25:54.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gXekX+5F5wks1XmK2gNUQNA7hxU= 2022-09-03T07:25:54.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gXekX+5F5wks1XmK2gNUQNA7hxU=', 'Content-Length': '0'} 2022-09-03T07:25:54.498 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d4564a6c890a7327-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.499 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:25:54.500 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.500 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.501 DEBUG:boto:Method: GET 2022-09-03T07:25:54.501 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:25:54.502 DEBUG:boto:Data: 2022-09-03T07:25:54.502 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.503 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.503 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.504 DEBUG:boto:Params: {} 2022-09-03T07:25:54.505 DEBUG:boto:Token: None 2022-09-03T07:25:54.505 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57 2022-09-03T07:25:54.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IaGCYKfzqc1YKz5VLG8skp5urso= 2022-09-03T07:25:54.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IaGCYKfzqc1YKz5VLG8skp5urso=', 'Content-Length': '0'} 2022-09-03T07:25:54.508 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000050ba7909c1d8744b-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.509 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:25:54.510 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:25:54.510 DEBUG:boto:Method: GET 2022-09-03T07:25:54.511 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:25:54.511 DEBUG:boto:Data: 2022-09-03T07:25:54.512 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.513 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.514 DEBUG:boto:Params: {} 2022-09-03T07:25:54.514 DEBUG:boto:Token: None 2022-09-03T07:25:54.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj57 2022-09-03T07:25:54.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IaGCYKfzqc1YKz5VLG8skp5urso= 2022-09-03T07:25:54.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IaGCYKfzqc1YKz5VLG8skp5urso=', 'Content-Length': '0'} 2022-09-03T07:25:54.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031460a3d157ce278-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.562 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.563 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.565 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:25:54.566 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:25:54.568 DEBUG:boto:Method: GET 2022-09-03T07:25:54.569 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:25:54.570 DEBUG:boto:Data: 2022-09-03T07:25:54.571 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.572 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.573 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.573 DEBUG:boto:Params: {} 2022-09-03T07:25:54.574 DEBUG:boto:Token: None 2022-09-03T07:25:54.575 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.579 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005dea48f3890f5976-0063130182-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.580 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.581 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.581 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.582 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:25:54.582 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:25:54.583 DEBUG:boto:Method: GET 2022-09-03T07:25:54.584 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:25:54.585 DEBUG:boto:Data: 2022-09-03T07:25:54.586 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.588 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.589 DEBUG:boto:Params: {} 2022-09-03T07:25:54.590 DEBUG:boto:Token: None 2022-09-03T07:25:54.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.596 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000461022eb8289e909-0063130182-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.597 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.598 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:25:54.599 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.599 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.600 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.601 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.601 DEBUG:boto:Method: GET 2022-09-03T07:25:54.602 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.602 DEBUG:boto:Data: 2022-09-03T07:25:54.603 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.604 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.605 DEBUG:boto:Params: {} 2022-09-03T07:25:54.607 DEBUG:boto:Token: None 2022-09-03T07:25:54.608 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4fI1JNgZIiVKg8KID9lNyolFPHs= 2022-09-03T07:25:54.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4fI1JNgZIiVKg8KID9lNyolFPHs=', 'Content-Length': '0'} 2022-09-03T07:25:54.612 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6344945ec9ec851-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.613 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.614 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.614 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.615 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.616 DEBUG:boto:Method: GET 2022-09-03T07:25:54.617 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.618 DEBUG:boto:Data: 2022-09-03T07:25:54.619 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.620 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.621 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.622 DEBUG:boto:Params: {} 2022-09-03T07:25:54.623 DEBUG:boto:Token: None 2022-09-03T07:25:54.624 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4fI1JNgZIiVKg8KID9lNyolFPHs= 2022-09-03T07:25:54.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4fI1JNgZIiVKg8KID9lNyolFPHs=', 'Content-Length': '0'} 2022-09-03T07:25:54.629 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af0e1504ced66332-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.630 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.631 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.631 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.632 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.632 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.633 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.633 DEBUG:boto:Data: 2022-09-03T07:25:54.634 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.634 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.635 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.635 DEBUG:boto:Params: {} 2022-09-03T07:25:54.636 DEBUG:boto:Token: None 2022-09-03T07:25:54.637 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IPitUNTbGiIcnRxp3ATTDSoNGQ8= 2022-09-03T07:25:54.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IPitUNTbGiIcnRxp3ATTDSoNGQ8=', 'Content-Length': '0'} 2022-09-03T07:25:54.640 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a64c801137cacc9-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.640 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.641 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.641 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.642 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.643 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.643 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.644 DEBUG:boto:Data: 2022-09-03T07:25:54.644 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.645 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.645 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.646 DEBUG:boto:Params: {} 2022-09-03T07:25:54.647 DEBUG:boto:Token: None 2022-09-03T07:25:54.647 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IPitUNTbGiIcnRxp3ATTDSoNGQ8= 2022-09-03T07:25:54.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IPitUNTbGiIcnRxp3ATTDSoNGQ8=', 'Content-Length': '0'} 2022-09-03T07:25:54.650 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000172d177fc60fa46c-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.651 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:25:54.652 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.652 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.653 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.654 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.655 DEBUG:boto:Method: GET 2022-09-03T07:25:54.655 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.656 DEBUG:boto:Data: 2022-09-03T07:25:54.656 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.657 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.658 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.658 DEBUG:boto:Params: {} 2022-09-03T07:25:54.659 DEBUG:boto:Token: None 2022-09-03T07:25:54.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4fI1JNgZIiVKg8KID9lNyolFPHs= 2022-09-03T07:25:54.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4fI1JNgZIiVKg8KID9lNyolFPHs=', 'Content-Length': '0'} 2022-09-03T07:25:54.664 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e7908f3869170f7e-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.665 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.666 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.667 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.667 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.668 DEBUG:boto:Method: GET 2022-09-03T07:25:54.668 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.669 DEBUG:boto:Data: 2022-09-03T07:25:54.670 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.671 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.672 DEBUG:boto:Params: {} 2022-09-03T07:25:54.673 DEBUG:boto:Token: None 2022-09-03T07:25:54.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:25:54.674 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4fI1JNgZIiVKg8KID9lNyolFPHs= 2022-09-03T07:25:54.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4fI1JNgZIiVKg8KID9lNyolFPHs=', 'Content-Length': '0'} 2022-09-03T07:25:54.678 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de5f5ed5951c6072-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.722 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.724 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.725 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:25:54.726 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:25:54.727 DEBUG:boto:Method: GET 2022-09-03T07:25:54.728 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:25:54.730 DEBUG:boto:Data: 2022-09-03T07:25:54.731 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.732 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.733 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.734 DEBUG:boto:Params: {} 2022-09-03T07:25:54.736 DEBUG:boto:Token: None 2022-09-03T07:25:54.737 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.738 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.741 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044553781dc0a9c9e-0063130182-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.742 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.743 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.744 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.744 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:25:54.745 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:25:54.746 DEBUG:boto:Method: GET 2022-09-03T07:25:54.747 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:25:54.748 DEBUG:boto:Data: 2022-09-03T07:25:54.749 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.752 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.753 DEBUG:boto:Params: {} 2022-09-03T07:25:54.754 DEBUG:boto:Token: None 2022-09-03T07:25:54.755 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.756 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.760 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002d096c69d7344622-0063130182-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.762 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.763 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.764 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.765 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.766 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:25:54.767 DEBUG:boto:Data: 2022-09-03T07:25:54.768 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.769 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.770 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.771 DEBUG:boto:Params: {} 2022-09-03T07:25:54.772 DEBUG:boto:Token: None 2022-09-03T07:25:54.772 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58 2022-09-03T07:25:54.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:id6VmIGEoysD0/sHyB0HkR1W9gc= 2022-09-03T07:25:54.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:id6VmIGEoysD0/sHyB0HkR1W9gc=', 'Content-Length': '0'} 2022-09-03T07:25:54.776 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2711b6475b91cc6-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.777 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.778 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.778 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.779 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:25:54.780 DEBUG:boto:Data: 2022-09-03T07:25:54.780 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.781 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.782 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.783 DEBUG:boto:Params: {} 2022-09-03T07:25:54.784 DEBUG:boto:Token: None 2022-09-03T07:25:54.785 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58 2022-09-03T07:25:54.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:id6VmIGEoysD0/sHyB0HkR1W9gc= 2022-09-03T07:25:54.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:id6VmIGEoysD0/sHyB0HkR1W9gc=', 'Content-Length': '0'} 2022-09-03T07:25:54.789 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e69f1f0e16e6e2bd-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.790 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:25:54.790 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.791 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.792 DEBUG:boto:Method: GET 2022-09-03T07:25:54.793 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:25:54.794 DEBUG:boto:Data: 2022-09-03T07:25:54.795 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.796 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.797 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.798 DEBUG:boto:Params: {} 2022-09-03T07:25:54.799 DEBUG:boto:Token: None 2022-09-03T07:25:54.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58 2022-09-03T07:25:54.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MyTKuzFCmT23OvPjsGnsorbe8dI= 2022-09-03T07:25:54.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MyTKuzFCmT23OvPjsGnsorbe8dI=', 'Content-Length': '0'} 2022-09-03T07:25:54.803 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f553ed82dce5c77-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.804 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:25:54.804 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:25:54.805 DEBUG:boto:Method: GET 2022-09-03T07:25:54.806 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:25:54.806 DEBUG:boto:Data: 2022-09-03T07:25:54.807 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.808 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.809 DEBUG:boto:Params: {} 2022-09-03T07:25:54.809 DEBUG:boto:Token: None 2022-09-03T07:25:54.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj58 2022-09-03T07:25:54.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MyTKuzFCmT23OvPjsGnsorbe8dI= 2022-09-03T07:25:54.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MyTKuzFCmT23OvPjsGnsorbe8dI=', 'Content-Length': '0'} 2022-09-03T07:25:54.815 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000360be8f537a7b233-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.816 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.816 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.817 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:25:54.818 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:25:54.818 DEBUG:boto:Method: GET 2022-09-03T07:25:54.819 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:25:54.820 DEBUG:boto:Data: 2022-09-03T07:25:54.820 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.821 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.822 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.823 DEBUG:boto:Params: {} 2022-09-03T07:25:54.824 DEBUG:boto:Token: None 2022-09-03T07:25:54.825 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.832 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e427dfaadafea2f2-0063130182-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.834 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.836 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:54.837 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:54.839 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:25:54.840 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:25:54.841 DEBUG:boto:Method: GET 2022-09-03T07:25:54.842 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:25:54.843 DEBUG:boto:Data: 2022-09-03T07:25:54.845 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.847 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.849 DEBUG:boto:Params: {} 2022-09-03T07:25:54.850 DEBUG:boto:Token: None 2022-09-03T07:25:54.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/ 2022-09-03T07:25:54.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E= 2022-09-03T07:25:54.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CCP3OAHcV6qI/FbGlvUQQ5nfs8E=', 'Content-Length': '0'} 2022-09-03T07:25:54.858 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ec72f6782d5e6b5b-0063130182-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.859 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:54.860 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:25:54.861 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:54.862 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:54.863 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.863 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.864 DEBUG:boto:Method: GET 2022-09-03T07:25:54.866 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.867 DEBUG:boto:Data: 2022-09-03T07:25:54.868 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.871 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.872 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.873 DEBUG:boto:Params: {} 2022-09-03T07:25:54.875 DEBUG:boto:Token: None 2022-09-03T07:25:54.876 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dUtQEStOusYta2tWaCOKCfAQJIc= 2022-09-03T07:25:54.877 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dUtQEStOusYta2tWaCOKCfAQJIc=', 'Content-Length': '0'} 2022-09-03T07:25:54.880 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c240d025514cb35-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.880 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:54.881 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:54.882 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.883 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.884 DEBUG:boto:Method: GET 2022-09-03T07:25:54.884 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.885 DEBUG:boto:Data: 2022-09-03T07:25:54.886 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.887 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.888 DEBUG:boto:Params: {} 2022-09-03T07:25:54.888 DEBUG:boto:Token: None 2022-09-03T07:25:54.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dUtQEStOusYta2tWaCOKCfAQJIc= 2022-09-03T07:25:54.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dUtQEStOusYta2tWaCOKCfAQJIc=', 'Content-Length': '0'} 2022-09-03T07:25:54.894 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5f596d9253263bd-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.942 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:54.943 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:54.944 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.945 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.946 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.947 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.954 DEBUG:boto:Data: 2022-09-03T07:25:54.958 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.959 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:54.960 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.961 DEBUG:boto:Params: {} 2022-09-03T07:25:54.962 DEBUG:boto:Token: None 2022-09-03T07:25:54.963 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ybH/vzFAg55qTwUtflaAtVDd2XA= 2022-09-03T07:25:54.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ybH/vzFAg55qTwUtflaAtVDd2XA=', 'Content-Length': '0'} 2022-09-03T07:25:54.969 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce5ac48198ce7616-0063130182-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.971 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:54.973 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:54.974 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.975 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.976 DEBUG:boto:Method: HEAD 2022-09-03T07:25:54.979 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.981 DEBUG:boto:Data: 2022-09-03T07:25:54.982 DEBUG:boto:Headers: {} 2022-09-03T07:25:54.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:54.984 DEBUG:boto:Port: 8000 2022-09-03T07:25:54.985 DEBUG:boto:Params: {} 2022-09-03T07:25:54.987 DEBUG:boto:Token: None 2022-09-03T07:25:54.987 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:54 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ybH/vzFAg55qTwUtflaAtVDd2XA= 2022-09-03T07:25:54.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ybH/vzFAg55qTwUtflaAtVDd2XA=', 'Content-Length': '0'} 2022-09-03T07:25:54.995 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013c6dd29faa8c213-0063130182-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:54.996 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:25:54.996 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:54.997 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:54.998 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.998 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:54.999 DEBUG:boto:Method: GET 2022-09-03T07:25:54.999 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:25:55.000 DEBUG:boto:Data: 2022-09-03T07:25:55.000 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.001 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.002 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.002 DEBUG:boto:Params: {} 2022-09-03T07:25:55.003 DEBUG:boto:Token: None 2022-09-03T07:25:55.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:25:55.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ceAzlhNrty2fk05CVYddo6ah6Ng= 2022-09-03T07:25:55.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ceAzlhNrty2fk05CVYddo6ah6Ng=', 'Content-Length': '0'} 2022-09-03T07:25:55.007 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7364b5fbe46b2de-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.008 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:55.009 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:55.009 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:55.010 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:25:55.011 DEBUG:boto:Method: GET 2022-09-03T07:25:55.012 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:25:55.012 DEBUG:boto:Data: 2022-09-03T07:25:55.013 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.014 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.015 DEBUG:boto:Params: {} 2022-09-03T07:25:55.016 DEBUG:boto:Token: None 2022-09-03T07:25:55.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:25:55.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ceAzlhNrty2fk05CVYddo6ah6Ng= 2022-09-03T07:25:55.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ceAzlhNrty2fk05CVYddo6ah6Ng=', 'Content-Length': '0'} 2022-09-03T07:25:55.021 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c445527eb543ede0-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.022 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.023 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.023 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:25:55.024 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:25:55.024 DEBUG:boto:Method: GET 2022-09-03T07:25:55.025 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:25:55.026 DEBUG:boto:Data: 2022-09-03T07:25:55.026 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.030 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.031 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.032 DEBUG:boto:Params: {} 2022-09-03T07:25:55.033 DEBUG:boto:Token: None 2022-09-03T07:25:55.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.039 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000397ca1dd64dbe0ca-0063130183-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.040 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.042 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.043 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.044 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:25:55.045 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:25:55.046 DEBUG:boto:Method: GET 2022-09-03T07:25:55.047 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:25:55.048 DEBUG:boto:Data: 2022-09-03T07:25:55.049 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.051 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.052 DEBUG:boto:Params: {} 2022-09-03T07:25:55.054 DEBUG:boto:Token: None 2022-09-03T07:25:55.055 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.060 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a07f5f2e1288bf1d-0063130183-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.061 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.062 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:55.063 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:55.064 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.064 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:25:55.065 DEBUG:boto:Data: 2022-09-03T07:25:55.065 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.066 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.066 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.067 DEBUG:boto:Params: {} 2022-09-03T07:25:55.067 DEBUG:boto:Token: None 2022-09-03T07:25:55.068 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj68 2022-09-03T07:25:55.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qzV/wSXj9TRve7Szn02KmyymGj4= 2022-09-03T07:25:55.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qzV/wSXj9TRve7Szn02KmyymGj4=', 'Content-Length': '0'} 2022-09-03T07:25:55.071 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007bdbd17bc9bec922-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.073 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:55.073 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:55.074 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.075 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:25:55.077 DEBUG:boto:Data: 2022-09-03T07:25:55.077 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.078 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.079 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.080 DEBUG:boto:Params: {} 2022-09-03T07:25:55.081 DEBUG:boto:Token: None 2022-09-03T07:25:55.082 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj68 2022-09-03T07:25:55.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qzV/wSXj9TRve7Szn02KmyymGj4= 2022-09-03T07:25:55.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qzV/wSXj9TRve7Szn02KmyymGj4=', 'Content-Length': '0'} 2022-09-03T07:25:55.085 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074759e49da015f17-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.086 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:25:55.087 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:55.087 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:55.088 DEBUG:boto:Method: GET 2022-09-03T07:25:55.088 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:25:55.090 DEBUG:boto:Data: 2022-09-03T07:25:55.091 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.092 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.093 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.094 DEBUG:boto:Params: {} 2022-09-03T07:25:55.095 DEBUG:boto:Token: None 2022-09-03T07:25:55.096 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj68 2022-09-03T07:25:55.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7WqDPJ/PJlaF4V0dXFGpEgAPow= 2022-09-03T07:25:55.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7WqDPJ/PJlaF4V0dXFGpEgAPow=', 'Content-Length': '0'} 2022-09-03T07:25:55.101 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049d44d417ae82bc2-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.142 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:25:55.143 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:25:55.144 DEBUG:boto:Method: GET 2022-09-03T07:25:55.145 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:25:55.146 DEBUG:boto:Data: 2022-09-03T07:25:55.147 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.148 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.149 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.150 DEBUG:boto:Params: {} 2022-09-03T07:25:55.151 DEBUG:boto:Token: None 2022-09-03T07:25:55.152 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj68 2022-09-03T07:25:55.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k7WqDPJ/PJlaF4V0dXFGpEgAPow= 2022-09-03T07:25:55.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k7WqDPJ/PJlaF4V0dXFGpEgAPow=', 'Content-Length': '0'} 2022-09-03T07:25:55.157 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a64c9bb545eb66d-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.158 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.159 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.160 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:25:55.161 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:25:55.161 DEBUG:boto:Method: GET 2022-09-03T07:25:55.162 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:25:55.163 DEBUG:boto:Data: 2022-09-03T07:25:55.164 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.164 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.166 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.167 DEBUG:boto:Params: {} 2022-09-03T07:25:55.168 DEBUG:boto:Token: None 2022-09-03T07:25:55.169 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.176 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a299ccedb4557920-0063130183-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.177 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.178 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.179 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.179 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:25:55.180 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:25:55.181 DEBUG:boto:Method: GET 2022-09-03T07:25:55.181 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:25:55.182 DEBUG:boto:Data: 2022-09-03T07:25:55.182 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.183 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.183 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.184 DEBUG:boto:Params: {} 2022-09-03T07:25:55.185 DEBUG:boto:Token: None 2022-09-03T07:25:55.185 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.186 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.190 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004d1dec101a73c252-0063130183-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.191 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.191 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:25:55.192 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.192 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.193 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.194 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.194 DEBUG:boto:Method: GET 2022-09-03T07:25:55.195 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.196 DEBUG:boto:Data: 2022-09-03T07:25:55.197 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.197 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.198 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.199 DEBUG:boto:Params: {} 2022-09-03T07:25:55.201 DEBUG:boto:Token: None 2022-09-03T07:25:55.202 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.203 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mhrOIP0XI/OfGYNBVqrdO1lc2QI= 2022-09-03T07:25:55.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mhrOIP0XI/OfGYNBVqrdO1lc2QI=', 'Content-Length': '0'} 2022-09-03T07:25:55.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8f50f505daf69ad-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.208 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.208 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.209 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.210 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.210 DEBUG:boto:Method: GET 2022-09-03T07:25:55.211 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.211 DEBUG:boto:Data: 2022-09-03T07:25:55.212 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.213 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.213 DEBUG:boto:Params: {} 2022-09-03T07:25:55.214 DEBUG:boto:Token: None 2022-09-03T07:25:55.215 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.215 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mhrOIP0XI/OfGYNBVqrdO1lc2QI= 2022-09-03T07:25:55.216 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mhrOIP0XI/OfGYNBVqrdO1lc2QI=', 'Content-Length': '0'} 2022-09-03T07:25:55.218 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e43cf29729e8c60-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.219 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.220 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.220 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.221 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.221 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.222 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.223 DEBUG:boto:Data: 2022-09-03T07:25:55.223 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.224 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.224 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.225 DEBUG:boto:Params: {} 2022-09-03T07:25:55.226 DEBUG:boto:Token: None 2022-09-03T07:25:55.226 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.227 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7OT1PoL8phHlHHG5U9BSzFYjjAk= 2022-09-03T07:25:55.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7OT1PoL8phHlHHG5U9BSzFYjjAk=', 'Content-Length': '0'} 2022-09-03T07:25:55.229 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3e2b101001d72fe-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.230 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.231 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.232 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.233 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.234 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.235 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.236 DEBUG:boto:Data: 2022-09-03T07:25:55.237 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.239 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.239 DEBUG:boto:Params: {} 2022-09-03T07:25:55.240 DEBUG:boto:Token: None 2022-09-03T07:25:55.241 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7OT1PoL8phHlHHG5U9BSzFYjjAk= 2022-09-03T07:25:55.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7OT1PoL8phHlHHG5U9BSzFYjjAk=', 'Content-Length': '0'} 2022-09-03T07:25:55.245 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3622a5b7610cc52-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.246 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:25:55.246 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.247 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.248 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.248 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.249 DEBUG:boto:Method: GET 2022-09-03T07:25:55.250 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.250 DEBUG:boto:Data: 2022-09-03T07:25:55.251 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.251 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.252 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.252 DEBUG:boto:Params: {} 2022-09-03T07:25:55.253 DEBUG:boto:Token: None 2022-09-03T07:25:55.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mhrOIP0XI/OfGYNBVqrdO1lc2QI= 2022-09-03T07:25:55.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mhrOIP0XI/OfGYNBVqrdO1lc2QI=', 'Content-Length': '0'} 2022-09-03T07:25:55.257 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000006cd943b2a4752ae-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.298 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.299 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.299 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.300 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.300 DEBUG:boto:Method: GET 2022-09-03T07:25:55.301 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.302 DEBUG:boto:Data: 2022-09-03T07:25:55.303 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.303 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.304 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.304 DEBUG:boto:Params: {} 2022-09-03T07:25:55.305 DEBUG:boto:Token: None 2022-09-03T07:25:55.307 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:25:55.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mhrOIP0XI/OfGYNBVqrdO1lc2QI= 2022-09-03T07:25:55.308 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mhrOIP0XI/OfGYNBVqrdO1lc2QI=', 'Content-Length': '0'} 2022-09-03T07:25:55.310 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be2c9715ca36be9e-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.311 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.313 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.314 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:25:55.314 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:25:55.315 DEBUG:boto:Method: GET 2022-09-03T07:25:55.315 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:25:55.316 DEBUG:boto:Data: 2022-09-03T07:25:55.316 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.317 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.318 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.318 DEBUG:boto:Params: {} 2022-09-03T07:25:55.319 DEBUG:boto:Token: None 2022-09-03T07:25:55.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.323 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000074d3df50b086c07e-0063130183-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.324 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.325 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.326 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.326 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:25:55.327 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:25:55.327 DEBUG:boto:Method: GET 2022-09-03T07:25:55.328 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:25:55.328 DEBUG:boto:Data: 2022-09-03T07:25:55.329 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.330 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.330 DEBUG:boto:Params: {} 2022-09-03T07:25:55.331 DEBUG:boto:Token: None 2022-09-03T07:25:55.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.340 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f9d7cb52a5701167-0063130183-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.341 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.342 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.343 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.344 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.345 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:25:55.346 DEBUG:boto:Data: 2022-09-03T07:25:55.347 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.347 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.348 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.348 DEBUG:boto:Params: {} 2022-09-03T07:25:55.349 DEBUG:boto:Token: None 2022-09-03T07:25:55.349 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69 2022-09-03T07:25:55.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut4qF9MSsHAoVRa74/Jft68AwwU= 2022-09-03T07:25:55.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut4qF9MSsHAoVRa74/Jft68AwwU=', 'Content-Length': '0'} 2022-09-03T07:25:55.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a53bae654d80cede-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.355 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.356 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.356 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.357 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:25:55.357 DEBUG:boto:Data: 2022-09-03T07:25:55.358 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.359 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.360 DEBUG:boto:Params: {} 2022-09-03T07:25:55.360 DEBUG:boto:Token: None 2022-09-03T07:25:55.361 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69 2022-09-03T07:25:55.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut4qF9MSsHAoVRa74/Jft68AwwU= 2022-09-03T07:25:55.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut4qF9MSsHAoVRa74/Jft68AwwU=', 'Content-Length': '0'} 2022-09-03T07:25:55.364 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a20514d36f295c3-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.366 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:25:55.367 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.368 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.369 DEBUG:boto:Method: GET 2022-09-03T07:25:55.370 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:25:55.371 DEBUG:boto:Data: 2022-09-03T07:25:55.371 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.372 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.372 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.373 DEBUG:boto:Params: {} 2022-09-03T07:25:55.373 DEBUG:boto:Token: None 2022-09-03T07:25:55.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69 2022-09-03T07:25:55.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mvM36UoWlI+uasERJRVdyBSj2fo= 2022-09-03T07:25:55.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mvM36UoWlI+uasERJRVdyBSj2fo=', 'Content-Length': '0'} 2022-09-03T07:25:55.377 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d5de87b8b1f14ce-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.378 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:25:55.379 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:25:55.380 DEBUG:boto:Method: GET 2022-09-03T07:25:55.381 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:25:55.382 DEBUG:boto:Data: 2022-09-03T07:25:55.383 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.385 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.386 DEBUG:boto:Params: {} 2022-09-03T07:25:55.387 DEBUG:boto:Token: None 2022-09-03T07:25:55.387 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj69 2022-09-03T07:25:55.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mvM36UoWlI+uasERJRVdyBSj2fo= 2022-09-03T07:25:55.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mvM36UoWlI+uasERJRVdyBSj2fo=', 'Content-Length': '0'} 2022-09-03T07:25:55.391 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057761e3bb99ed1a7-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.392 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.393 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.393 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:25:55.394 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:25:55.394 DEBUG:boto:Method: GET 2022-09-03T07:25:55.395 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:25:55.396 DEBUG:boto:Data: 2022-09-03T07:25:55.396 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.397 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.397 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.398 DEBUG:boto:Params: {} 2022-09-03T07:25:55.399 DEBUG:boto:Token: None 2022-09-03T07:25:55.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.403 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006dadfa1fbb4a2459-0063130183-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.404 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.405 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.405 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.406 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:25:55.406 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:25:55.407 DEBUG:boto:Method: GET 2022-09-03T07:25:55.407 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:25:55.408 DEBUG:boto:Data: 2022-09-03T07:25:55.408 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.409 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.409 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.410 DEBUG:boto:Params: {} 2022-09-03T07:25:55.410 DEBUG:boto:Token: None 2022-09-03T07:25:55.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.416 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b749b94f352608ad-0063130183-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.458 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.459 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:25:55.460 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.461 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.461 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.462 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.462 DEBUG:boto:Method: GET 2022-09-03T07:25:55.463 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.464 DEBUG:boto:Data: 2022-09-03T07:25:55.465 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.466 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.467 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.468 DEBUG:boto:Params: {} 2022-09-03T07:25:55.469 DEBUG:boto:Token: None 2022-09-03T07:25:55.470 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wkCEVLkLWoNhtuoEBP5W5kTAcDU= 2022-09-03T07:25:55.473 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wkCEVLkLWoNhtuoEBP5W5kTAcDU=', 'Content-Length': '0'} 2022-09-03T07:25:55.476 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c1e6f09939f49eb-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.476 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.477 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.478 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.478 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.479 DEBUG:boto:Method: GET 2022-09-03T07:25:55.480 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.482 DEBUG:boto:Data: 2022-09-03T07:25:55.482 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.483 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.484 DEBUG:boto:Params: {} 2022-09-03T07:25:55.485 DEBUG:boto:Token: None 2022-09-03T07:25:55.486 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.486 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wkCEVLkLWoNhtuoEBP5W5kTAcDU= 2022-09-03T07:25:55.487 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wkCEVLkLWoNhtuoEBP5W5kTAcDU=', 'Content-Length': '0'} 2022-09-03T07:25:55.490 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf7574291267f423-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.491 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.493 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.493 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.494 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.495 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.496 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.496 DEBUG:boto:Data: 2022-09-03T07:25:55.497 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.497 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.498 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.498 DEBUG:boto:Params: {} 2022-09-03T07:25:55.499 DEBUG:boto:Token: None 2022-09-03T07:25:55.500 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vqSwUoFUt2qD6WDQh6++OH2Q/8s= 2022-09-03T07:25:55.501 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vqSwUoFUt2qD6WDQh6++OH2Q/8s=', 'Content-Length': '0'} 2022-09-03T07:25:55.504 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000077d991a27a6c03a6-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.505 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.506 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.507 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.507 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.508 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.508 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.509 DEBUG:boto:Data: 2022-09-03T07:25:55.509 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.510 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.510 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.511 DEBUG:boto:Params: {} 2022-09-03T07:25:55.512 DEBUG:boto:Token: None 2022-09-03T07:25:55.512 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vqSwUoFUt2qD6WDQh6++OH2Q/8s= 2022-09-03T07:25:55.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vqSwUoFUt2qD6WDQh6++OH2Q/8s=', 'Content-Length': '0'} 2022-09-03T07:25:55.516 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049ff57030d57025d-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.517 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:25:55.518 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.518 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.519 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.519 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.520 DEBUG:boto:Method: GET 2022-09-03T07:25:55.521 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.521 DEBUG:boto:Data: 2022-09-03T07:25:55.522 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.522 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.523 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.523 DEBUG:boto:Params: {} 2022-09-03T07:25:55.525 DEBUG:boto:Token: None 2022-09-03T07:25:55.526 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.526 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wkCEVLkLWoNhtuoEBP5W5kTAcDU= 2022-09-03T07:25:55.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wkCEVLkLWoNhtuoEBP5W5kTAcDU=', 'Content-Length': '0'} 2022-09-03T07:25:55.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c4f7c10fb7740a9-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.574 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.575 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.575 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.576 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.577 DEBUG:boto:Method: GET 2022-09-03T07:25:55.578 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.579 DEBUG:boto:Data: 2022-09-03T07:25:55.580 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.581 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.582 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.583 DEBUG:boto:Params: {} 2022-09-03T07:25:55.583 DEBUG:boto:Token: None 2022-09-03T07:25:55.584 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:25:55.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wkCEVLkLWoNhtuoEBP5W5kTAcDU= 2022-09-03T07:25:55.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wkCEVLkLWoNhtuoEBP5W5kTAcDU=', 'Content-Length': '0'} 2022-09-03T07:25:55.588 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b13bbdfecda595c-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.589 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.590 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.590 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:25:55.591 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:25:55.592 DEBUG:boto:Method: GET 2022-09-03T07:25:55.593 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:25:55.594 DEBUG:boto:Data: 2022-09-03T07:25:55.595 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.596 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.597 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.598 DEBUG:boto:Params: {} 2022-09-03T07:25:55.599 DEBUG:boto:Token: None 2022-09-03T07:25:55.601 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.609 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000660b71c75a89696f-0063130183-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.610 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.612 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.613 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.614 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:25:55.615 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:25:55.616 DEBUG:boto:Method: GET 2022-09-03T07:25:55.617 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:25:55.617 DEBUG:boto:Data: 2022-09-03T07:25:55.618 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.619 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.620 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.621 DEBUG:boto:Params: {} 2022-09-03T07:25:55.621 DEBUG:boto:Token: None 2022-09-03T07:25:55.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.623 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.627 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f15146825896f2e3-0063130183-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.628 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.629 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.629 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.630 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.630 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:25:55.631 DEBUG:boto:Data: 2022-09-03T07:25:55.632 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.632 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.633 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.633 DEBUG:boto:Params: {} 2022-09-03T07:25:55.634 DEBUG:boto:Token: None 2022-09-03T07:25:55.634 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76 2022-09-03T07:25:55.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PubKW4VHkUb3G3V0To9RUoq+l8s= 2022-09-03T07:25:55.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PubKW4VHkUb3G3V0To9RUoq+l8s=', 'Content-Length': '0'} 2022-09-03T07:25:55.638 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d951717bfea0b13e-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.639 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.640 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.640 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.641 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:25:55.642 DEBUG:boto:Data: 2022-09-03T07:25:55.643 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.645 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.646 DEBUG:boto:Params: {} 2022-09-03T07:25:55.647 DEBUG:boto:Token: None 2022-09-03T07:25:55.648 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76 2022-09-03T07:25:55.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PubKW4VHkUb3G3V0To9RUoq+l8s= 2022-09-03T07:25:55.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PubKW4VHkUb3G3V0To9RUoq+l8s=', 'Content-Length': '0'} 2022-09-03T07:25:55.653 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000261e3e25fe8c1361-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.654 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:25:55.655 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.656 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.657 DEBUG:boto:Method: GET 2022-09-03T07:25:55.658 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:25:55.659 DEBUG:boto:Data: 2022-09-03T07:25:55.660 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.661 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.662 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.663 DEBUG:boto:Params: {} 2022-09-03T07:25:55.664 DEBUG:boto:Token: None 2022-09-03T07:25:55.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76 2022-09-03T07:25:55.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:13JFmVM3Gd8YNYnzbV4hgnHMB6M= 2022-09-03T07:25:55.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:13JFmVM3Gd8YNYnzbV4hgnHMB6M=', 'Content-Length': '0'} 2022-09-03T07:25:55.670 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b048b6c1aa8c9a71-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.671 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:25:55.672 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:25:55.673 DEBUG:boto:Method: GET 2022-09-03T07:25:55.674 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:25:55.675 DEBUG:boto:Data: 2022-09-03T07:25:55.675 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.676 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.677 DEBUG:boto:Params: {} 2022-09-03T07:25:55.678 DEBUG:boto:Token: None 2022-09-03T07:25:55.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj76 2022-09-03T07:25:55.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:13JFmVM3Gd8YNYnzbV4hgnHMB6M= 2022-09-03T07:25:55.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:13JFmVM3Gd8YNYnzbV4hgnHMB6M=', 'Content-Length': '0'} 2022-09-03T07:25:55.682 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf71734464ca30dc-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.683 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.683 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.684 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:25:55.684 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:25:55.685 DEBUG:boto:Method: GET 2022-09-03T07:25:55.685 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:25:55.686 DEBUG:boto:Data: 2022-09-03T07:25:55.687 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.688 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.689 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.690 DEBUG:boto:Params: {} 2022-09-03T07:25:55.691 DEBUG:boto:Token: None 2022-09-03T07:25:55.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.693 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.696 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000078071cbdd5c54c48-0063130183-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.697 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.698 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.699 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.699 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:25:55.700 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:25:55.700 DEBUG:boto:Method: GET 2022-09-03T07:25:55.701 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:25:55.702 DEBUG:boto:Data: 2022-09-03T07:25:55.702 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.703 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.704 DEBUG:boto:Params: {} 2022-09-03T07:25:55.704 DEBUG:boto:Token: None 2022-09-03T07:25:55.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.710 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1d50149e73f0795-0063130183-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.754 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.755 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:25:55.756 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.756 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.757 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.757 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.758 DEBUG:boto:Method: GET 2022-09-03T07:25:55.758 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.759 DEBUG:boto:Data: 2022-09-03T07:25:55.759 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.760 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.760 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.761 DEBUG:boto:Params: {} 2022-09-03T07:25:55.762 DEBUG:boto:Token: None 2022-09-03T07:25:55.762 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.763 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WYTTfAggdFpsbPhNheC8SoNe2sk= 2022-09-03T07:25:55.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WYTTfAggdFpsbPhNheC8SoNe2sk=', 'Content-Length': '0'} 2022-09-03T07:25:55.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff61a26cc85a9299-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.767 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.767 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.768 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.768 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.769 DEBUG:boto:Method: GET 2022-09-03T07:25:55.769 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.770 DEBUG:boto:Data: 2022-09-03T07:25:55.774 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.775 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.778 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.779 DEBUG:boto:Params: {} 2022-09-03T07:25:55.780 DEBUG:boto:Token: None 2022-09-03T07:25:55.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WYTTfAggdFpsbPhNheC8SoNe2sk= 2022-09-03T07:25:55.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WYTTfAggdFpsbPhNheC8SoNe2sk=', 'Content-Length': '0'} 2022-09-03T07:25:55.788 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034cedae79dd076e1-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.789 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.790 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.791 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.792 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.793 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.794 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.795 DEBUG:boto:Data: 2022-09-03T07:25:55.796 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.797 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.798 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.799 DEBUG:boto:Params: {} 2022-09-03T07:25:55.801 DEBUG:boto:Token: None 2022-09-03T07:25:55.802 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PhwFHmreAg9V17dGl7+4hCEMn7M= 2022-09-03T07:25:55.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PhwFHmreAg9V17dGl7+4hCEMn7M=', 'Content-Length': '0'} 2022-09-03T07:25:55.807 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2acbab089790cfb-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.808 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.808 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.809 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.809 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.810 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.810 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.811 DEBUG:boto:Data: 2022-09-03T07:25:55.811 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.812 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.812 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.813 DEBUG:boto:Params: {} 2022-09-03T07:25:55.813 DEBUG:boto:Token: None 2022-09-03T07:25:55.814 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PhwFHmreAg9V17dGl7+4hCEMn7M= 2022-09-03T07:25:55.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PhwFHmreAg9V17dGl7+4hCEMn7M=', 'Content-Length': '0'} 2022-09-03T07:25:55.820 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb9b87335128439b-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.821 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:25:55.821 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.822 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.823 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.823 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.824 DEBUG:boto:Method: GET 2022-09-03T07:25:55.825 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.825 DEBUG:boto:Data: 2022-09-03T07:25:55.826 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.826 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.827 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.827 DEBUG:boto:Params: {} 2022-09-03T07:25:55.828 DEBUG:boto:Token: None 2022-09-03T07:25:55.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WYTTfAggdFpsbPhNheC8SoNe2sk= 2022-09-03T07:25:55.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WYTTfAggdFpsbPhNheC8SoNe2sk=', 'Content-Length': '0'} 2022-09-03T07:25:55.832 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a33bfff1e35344ee-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.874 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.875 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.876 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.877 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.878 DEBUG:boto:Method: GET 2022-09-03T07:25:55.879 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.880 DEBUG:boto:Data: 2022-09-03T07:25:55.881 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.883 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.884 DEBUG:boto:Params: {} 2022-09-03T07:25:55.885 DEBUG:boto:Token: None 2022-09-03T07:25:55.886 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:25:55.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WYTTfAggdFpsbPhNheC8SoNe2sk= 2022-09-03T07:25:55.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WYTTfAggdFpsbPhNheC8SoNe2sk=', 'Content-Length': '0'} 2022-09-03T07:25:55.891 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024193ea15a5925dc-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.892 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.893 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.893 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:25:55.894 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:25:55.895 DEBUG:boto:Method: GET 2022-09-03T07:25:55.895 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:25:55.896 DEBUG:boto:Data: 2022-09-03T07:25:55.896 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.897 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.898 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.898 DEBUG:boto:Params: {} 2022-09-03T07:25:55.899 DEBUG:boto:Token: None 2022-09-03T07:25:55.899 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.905 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067e9046dc6a102fa-0063130183-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.906 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.908 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:55.909 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:55.910 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:25:55.911 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:25:55.912 DEBUG:boto:Method: GET 2022-09-03T07:25:55.913 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:25:55.914 DEBUG:boto:Data: 2022-09-03T07:25:55.915 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.916 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.916 DEBUG:boto:Params: {} 2022-09-03T07:25:55.917 DEBUG:boto:Token: None 2022-09-03T07:25:55.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/ 2022-09-03T07:25:55.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ= 2022-09-03T07:25:55.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fO4ygtOSu2yNST3bXTUHpMHcTRQ=', 'Content-Length': '0'} 2022-09-03T07:25:55.927 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000472593bc4b7737ea-0063130183-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.928 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:55.930 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.931 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.932 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.932 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:25:55.933 DEBUG:boto:Data: 2022-09-03T07:25:55.934 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.934 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.935 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.935 DEBUG:boto:Params: {} 2022-09-03T07:25:55.936 DEBUG:boto:Token: None 2022-09-03T07:25:55.936 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85 2022-09-03T07:25:55.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kslri0VwesuvSdEoKYQO/h1PIrE= 2022-09-03T07:25:55.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kslri0VwesuvSdEoKYQO/h1PIrE=', 'Content-Length': '0'} 2022-09-03T07:25:55.940 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d22bc837a81fae9c-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.941 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.942 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.942 DEBUG:boto:Method: HEAD 2022-09-03T07:25:55.944 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:25:55.945 DEBUG:boto:Data: 2022-09-03T07:25:55.945 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.946 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.946 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.948 DEBUG:boto:Params: {} 2022-09-03T07:25:55.949 DEBUG:boto:Token: None 2022-09-03T07:25:55.950 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85 2022-09-03T07:25:55.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kslri0VwesuvSdEoKYQO/h1PIrE= 2022-09-03T07:25:55.952 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kslri0VwesuvSdEoKYQO/h1PIrE=', 'Content-Length': '0'} 2022-09-03T07:25:55.955 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b16a1f20169d226b-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.956 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:25:55.957 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.958 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.959 DEBUG:boto:Method: GET 2022-09-03T07:25:55.960 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:25:55.961 DEBUG:boto:Data: 2022-09-03T07:25:55.961 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.962 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:55.962 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.963 DEBUG:boto:Params: {} 2022-09-03T07:25:55.964 DEBUG:boto:Token: None 2022-09-03T07:25:55.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85 2022-09-03T07:25:55.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K2P42TbAMhiznLOdwnNYEAgMQtA= 2022-09-03T07:25:55.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K2P42TbAMhiznLOdwnNYEAgMQtA=', 'Content-Length': '0'} 2022-09-03T07:25:55.969 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a0f0f9b7d57d44c-0063130183-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:55.971 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:25:55.972 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:25:55.973 DEBUG:boto:Method: GET 2022-09-03T07:25:55.974 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:25:55.975 DEBUG:boto:Data: 2022-09-03T07:25:55.976 DEBUG:boto:Headers: {} 2022-09-03T07:25:55.977 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:55.977 DEBUG:boto:Port: 8000 2022-09-03T07:25:55.978 DEBUG:boto:Params: {} 2022-09-03T07:25:55.979 DEBUG:boto:Token: None 2022-09-03T07:25:55.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:55 GMT /avysmw-75/obj85 2022-09-03T07:25:55.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K2P42TbAMhiznLOdwnNYEAgMQtA= 2022-09-03T07:25:55.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K2P42TbAMhiznLOdwnNYEAgMQtA=', 'Content-Length': '0'} 2022-09-03T07:25:55.984 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f5ebe906ce758af-0063130183-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.026 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.027 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.027 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:25:56.028 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:25:56.028 DEBUG:boto:Method: GET 2022-09-03T07:25:56.029 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:25:56.029 DEBUG:boto:Data: 2022-09-03T07:25:56.030 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.031 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.031 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.032 DEBUG:boto:Params: {} 2022-09-03T07:25:56.032 DEBUG:boto:Token: None 2022-09-03T07:25:56.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.037 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c6f0bd213261eb16-0063130184-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.038 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.039 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.040 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.040 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:25:56.041 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:25:56.041 DEBUG:boto:Method: GET 2022-09-03T07:25:56.042 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:25:56.042 DEBUG:boto:Data: 2022-09-03T07:25:56.043 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.045 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.046 DEBUG:boto:Params: {} 2022-09-03T07:25:56.046 DEBUG:boto:Token: None 2022-09-03T07:25:56.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.054 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bac9c741bf4d2bdd-0063130184-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.056 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.057 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:25:56.058 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.058 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.059 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.059 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.060 DEBUG:boto:Method: GET 2022-09-03T07:25:56.060 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.061 DEBUG:boto:Data: 2022-09-03T07:25:56.062 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.062 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.063 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.063 DEBUG:boto:Params: {} 2022-09-03T07:25:56.064 DEBUG:boto:Token: None 2022-09-03T07:25:56.065 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eRPn/Nbzozy48w8ElfA/6Rjo/Vo= 2022-09-03T07:25:56.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eRPn/Nbzozy48w8ElfA/6Rjo/Vo=', 'Content-Length': '0'} 2022-09-03T07:25:56.068 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000559156951b52a46f-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.069 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.069 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.070 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.071 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.071 DEBUG:boto:Method: GET 2022-09-03T07:25:56.072 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.073 DEBUG:boto:Data: 2022-09-03T07:25:56.073 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.074 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.074 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.075 DEBUG:boto:Params: {} 2022-09-03T07:25:56.075 DEBUG:boto:Token: None 2022-09-03T07:25:56.076 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eRPn/Nbzozy48w8ElfA/6Rjo/Vo= 2022-09-03T07:25:56.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eRPn/Nbzozy48w8ElfA/6Rjo/Vo=', 'Content-Length': '0'} 2022-09-03T07:25:56.080 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009eb3a0e4cb441f86-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.081 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.082 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.082 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.083 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.083 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.084 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.084 DEBUG:boto:Data: 2022-09-03T07:25:56.085 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.086 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.086 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.087 DEBUG:boto:Params: {} 2022-09-03T07:25:56.087 DEBUG:boto:Token: None 2022-09-03T07:25:56.088 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l69mnG4Je/gRQ8Yy8pQSkMVGvsI= 2022-09-03T07:25:56.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l69mnG4Je/gRQ8Yy8pQSkMVGvsI=', 'Content-Length': '0'} 2022-09-03T07:25:56.091 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020045b2ffbec4e5d-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.092 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.093 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.093 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.094 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.094 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.095 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.096 DEBUG:boto:Data: 2022-09-03T07:25:56.096 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.097 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.097 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.098 DEBUG:boto:Params: {} 2022-09-03T07:25:56.098 DEBUG:boto:Token: None 2022-09-03T07:25:56.099 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l69mnG4Je/gRQ8Yy8pQSkMVGvsI= 2022-09-03T07:25:56.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l69mnG4Je/gRQ8Yy8pQSkMVGvsI=', 'Content-Length': '0'} 2022-09-03T07:25:56.102 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006386d3fc4374e92c-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.103 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:25:56.104 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.104 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.105 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.105 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.106 DEBUG:boto:Method: GET 2022-09-03T07:25:56.106 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.107 DEBUG:boto:Data: 2022-09-03T07:25:56.108 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.108 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.109 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.109 DEBUG:boto:Params: {} 2022-09-03T07:25:56.110 DEBUG:boto:Token: None 2022-09-03T07:25:56.111 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eRPn/Nbzozy48w8ElfA/6Rjo/Vo= 2022-09-03T07:25:56.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eRPn/Nbzozy48w8ElfA/6Rjo/Vo=', 'Content-Length': '0'} 2022-09-03T07:25:56.114 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000994c26bac7d74bd7-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.115 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.115 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.116 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.116 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.117 DEBUG:boto:Method: GET 2022-09-03T07:25:56.118 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.118 DEBUG:boto:Data: 2022-09-03T07:25:56.119 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.119 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.120 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.120 DEBUG:boto:Params: {} 2022-09-03T07:25:56.121 DEBUG:boto:Token: None 2022-09-03T07:25:56.122 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:25:56.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eRPn/Nbzozy48w8ElfA/6Rjo/Vo= 2022-09-03T07:25:56.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eRPn/Nbzozy48w8ElfA/6Rjo/Vo=', 'Content-Length': '0'} 2022-09-03T07:25:56.125 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae93ae2b88d35b55-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.170 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.171 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.172 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:25:56.173 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:25:56.174 DEBUG:boto:Method: GET 2022-09-03T07:25:56.175 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:25:56.176 DEBUG:boto:Data: 2022-09-03T07:25:56.177 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.178 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.179 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.179 DEBUG:boto:Params: {} 2022-09-03T07:25:56.180 DEBUG:boto:Token: None 2022-09-03T07:25:56.181 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.181 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.184 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f38a6fd9bd44aef6-0063130184-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.186 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.187 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.187 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.188 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:25:56.188 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:25:56.189 DEBUG:boto:Method: GET 2022-09-03T07:25:56.189 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:25:56.190 DEBUG:boto:Data: 2022-09-03T07:25:56.190 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.191 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.191 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.192 DEBUG:boto:Params: {} 2022-09-03T07:25:56.193 DEBUG:boto:Token: None 2022-09-03T07:25:56.193 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.194 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.194 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.198 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000ee405c07cdb7c11-0063130184-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.199 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.200 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.201 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.201 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.202 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:25:56.202 DEBUG:boto:Data: 2022-09-03T07:25:56.203 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.203 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.204 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.204 DEBUG:boto:Params: {} 2022-09-03T07:25:56.205 DEBUG:boto:Token: None 2022-09-03T07:25:56.206 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87 2022-09-03T07:25:56.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yrB94kQccBqOPcpuQS3Yjny7mFI= 2022-09-03T07:25:56.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yrB94kQccBqOPcpuQS3Yjny7mFI=', 'Content-Length': '0'} 2022-09-03T07:25:56.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008063fc90c7274281-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.210 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.210 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.211 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.211 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:25:56.212 DEBUG:boto:Data: 2022-09-03T07:25:56.212 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.213 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.213 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.214 DEBUG:boto:Params: {} 2022-09-03T07:25:56.214 DEBUG:boto:Token: None 2022-09-03T07:25:56.215 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87 2022-09-03T07:25:56.215 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yrB94kQccBqOPcpuQS3Yjny7mFI= 2022-09-03T07:25:56.216 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yrB94kQccBqOPcpuQS3Yjny7mFI=', 'Content-Length': '0'} 2022-09-03T07:25:56.218 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001fe9f175f02f907e-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.219 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:25:56.220 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.220 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.221 DEBUG:boto:Method: GET 2022-09-03T07:25:56.221 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:25:56.222 DEBUG:boto:Data: 2022-09-03T07:25:56.222 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.223 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.224 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.224 DEBUG:boto:Params: {} 2022-09-03T07:25:56.225 DEBUG:boto:Token: None 2022-09-03T07:25:56.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87 2022-09-03T07:25:56.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ktv4WBrjFHWslPCDSqrv2Jx3goE= 2022-09-03T07:25:56.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ktv4WBrjFHWslPCDSqrv2Jx3goE=', 'Content-Length': '0'} 2022-09-03T07:25:56.229 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004409b8f2a474d9fb-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.229 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:25:56.230 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:25:56.231 DEBUG:boto:Method: GET 2022-09-03T07:25:56.231 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:25:56.232 DEBUG:boto:Data: 2022-09-03T07:25:56.232 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.233 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.234 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.234 DEBUG:boto:Params: {} 2022-09-03T07:25:56.235 DEBUG:boto:Token: None 2022-09-03T07:25:56.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj87 2022-09-03T07:25:56.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ktv4WBrjFHWslPCDSqrv2Jx3goE= 2022-09-03T07:25:56.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ktv4WBrjFHWslPCDSqrv2Jx3goE=', 'Content-Length': '0'} 2022-09-03T07:25:56.239 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099e50266ac6fd02c-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.282 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.283 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.283 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:25:56.284 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:25:56.285 DEBUG:boto:Method: GET 2022-09-03T07:25:56.285 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:25:56.286 DEBUG:boto:Data: 2022-09-03T07:25:56.286 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.287 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.288 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.288 DEBUG:boto:Params: {} 2022-09-03T07:25:56.289 DEBUG:boto:Token: None 2022-09-03T07:25:56.289 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.294 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c936435cee34aa1-0063130184-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.295 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.296 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.296 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.297 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:25:56.297 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:25:56.298 DEBUG:boto:Method: GET 2022-09-03T07:25:56.298 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:25:56.299 DEBUG:boto:Data: 2022-09-03T07:25:56.299 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.300 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.300 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.301 DEBUG:boto:Params: {} 2022-09-03T07:25:56.302 DEBUG:boto:Token: None 2022-09-03T07:25:56.302 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.308 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d9d055cc92231d9-0063130184-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.309 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.310 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:25:56.310 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.311 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.311 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.312 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.312 DEBUG:boto:Method: GET 2022-09-03T07:25:56.313 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.314 DEBUG:boto:Data: 2022-09-03T07:25:56.314 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.315 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.315 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.316 DEBUG:boto:Params: {} 2022-09-03T07:25:56.316 DEBUG:boto:Token: None 2022-09-03T07:25:56.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y4m5NxzV442G+pHNAdZ6TObZSH8= 2022-09-03T07:25:56.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y4m5NxzV442G+pHNAdZ6TObZSH8=', 'Content-Length': '0'} 2022-09-03T07:25:56.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003488347fe23d662e-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.321 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.321 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.322 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.323 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.323 DEBUG:boto:Method: GET 2022-09-03T07:25:56.324 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.324 DEBUG:boto:Data: 2022-09-03T07:25:56.325 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.325 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.326 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.326 DEBUG:boto:Params: {} 2022-09-03T07:25:56.327 DEBUG:boto:Token: None 2022-09-03T07:25:56.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y4m5NxzV442G+pHNAdZ6TObZSH8= 2022-09-03T07:25:56.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y4m5NxzV442G+pHNAdZ6TObZSH8=', 'Content-Length': '0'} 2022-09-03T07:25:56.331 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1496a3cf09901bc-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.332 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.333 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.333 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.334 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.334 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.335 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.335 DEBUG:boto:Data: 2022-09-03T07:25:56.336 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.336 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.337 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.337 DEBUG:boto:Params: {} 2022-09-03T07:25:56.338 DEBUG:boto:Token: None 2022-09-03T07:25:56.339 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8ZTy7n/IZXv6FcQOMoOlg8bpTsQ= 2022-09-03T07:25:56.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8ZTy7n/IZXv6FcQOMoOlg8bpTsQ=', 'Content-Length': '0'} 2022-09-03T07:25:56.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db34b52b55f51269-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.343 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.344 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.345 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.346 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.347 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.348 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.349 DEBUG:boto:Data: 2022-09-03T07:25:56.350 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.352 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.352 DEBUG:boto:Params: {} 2022-09-03T07:25:56.353 DEBUG:boto:Token: None 2022-09-03T07:25:56.354 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8ZTy7n/IZXv6FcQOMoOlg8bpTsQ= 2022-09-03T07:25:56.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8ZTy7n/IZXv6FcQOMoOlg8bpTsQ=', 'Content-Length': '0'} 2022-09-03T07:25:56.358 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f79c7ccea741761-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.359 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:25:56.360 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.360 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.361 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.361 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.362 DEBUG:boto:Method: GET 2022-09-03T07:25:56.363 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.363 DEBUG:boto:Data: 2022-09-03T07:25:56.364 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.364 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.365 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.365 DEBUG:boto:Params: {} 2022-09-03T07:25:56.366 DEBUG:boto:Token: None 2022-09-03T07:25:56.367 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y4m5NxzV442G+pHNAdZ6TObZSH8= 2022-09-03T07:25:56.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y4m5NxzV442G+pHNAdZ6TObZSH8=', 'Content-Length': '0'} 2022-09-03T07:25:56.370 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5e4bf99422e08b3-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.414 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.415 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.416 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.417 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.418 DEBUG:boto:Method: GET 2022-09-03T07:25:56.419 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.419 DEBUG:boto:Data: 2022-09-03T07:25:56.420 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.422 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.423 DEBUG:boto:Params: {} 2022-09-03T07:25:56.424 DEBUG:boto:Token: None 2022-09-03T07:25:56.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:25:56.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y4m5NxzV442G+pHNAdZ6TObZSH8= 2022-09-03T07:25:56.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y4m5NxzV442G+pHNAdZ6TObZSH8=', 'Content-Length': '0'} 2022-09-03T07:25:56.428 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a82950cdc050930-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.429 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.430 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.430 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:25:56.431 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:25:56.431 DEBUG:boto:Method: GET 2022-09-03T07:25:56.432 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:25:56.433 DEBUG:boto:Data: 2022-09-03T07:25:56.433 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.435 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.436 DEBUG:boto:Params: {} 2022-09-03T07:25:56.437 DEBUG:boto:Token: None 2022-09-03T07:25:56.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.441 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f36d9afdf4029340-0063130184-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.443 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.444 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.444 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.445 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:25:56.445 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:25:56.446 DEBUG:boto:Method: GET 2022-09-03T07:25:56.447 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:25:56.447 DEBUG:boto:Data: 2022-09-03T07:25:56.448 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.449 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.449 DEBUG:boto:Params: {} 2022-09-03T07:25:56.450 DEBUG:boto:Token: None 2022-09-03T07:25:56.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.456 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c32271fd884d046c-0063130184-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.456 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.457 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.458 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.458 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.459 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:25:56.460 DEBUG:boto:Data: 2022-09-03T07:25:56.461 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.461 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.462 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.462 DEBUG:boto:Params: {} 2022-09-03T07:25:56.463 DEBUG:boto:Token: None 2022-09-03T07:25:56.464 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92 2022-09-03T07:25:56.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HWh2f9NrbJuwfAjhBrMQgvgyV+0= 2022-09-03T07:25:56.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HWh2f9NrbJuwfAjhBrMQgvgyV+0=', 'Content-Length': '0'} 2022-09-03T07:25:56.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0ae8c336435a3ce-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.468 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.468 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.469 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.469 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:25:56.470 DEBUG:boto:Data: 2022-09-03T07:25:56.471 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.471 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.472 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.472 DEBUG:boto:Params: {} 2022-09-03T07:25:56.473 DEBUG:boto:Token: None 2022-09-03T07:25:56.473 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92 2022-09-03T07:25:56.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HWh2f9NrbJuwfAjhBrMQgvgyV+0= 2022-09-03T07:25:56.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HWh2f9NrbJuwfAjhBrMQgvgyV+0=', 'Content-Length': '0'} 2022-09-03T07:25:56.477 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000567ff148f975796c-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.478 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:25:56.478 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.479 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.479 DEBUG:boto:Method: GET 2022-09-03T07:25:56.480 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:25:56.480 DEBUG:boto:Data: 2022-09-03T07:25:56.481 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.481 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.482 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.483 DEBUG:boto:Params: {} 2022-09-03T07:25:56.483 DEBUG:boto:Token: None 2022-09-03T07:25:56.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92 2022-09-03T07:25:56.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w1lYeRaMtMKWeD9wbFmYTSrxmXc= 2022-09-03T07:25:56.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w1lYeRaMtMKWeD9wbFmYTSrxmXc=', 'Content-Length': '0'} 2022-09-03T07:25:56.487 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000955bc6c82342328e-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.489 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:25:56.490 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:25:56.491 DEBUG:boto:Method: GET 2022-09-03T07:25:56.491 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:25:56.492 DEBUG:boto:Data: 2022-09-03T07:25:56.493 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.494 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.494 DEBUG:boto:Params: {} 2022-09-03T07:25:56.495 DEBUG:boto:Token: None 2022-09-03T07:25:56.495 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj92 2022-09-03T07:25:56.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w1lYeRaMtMKWeD9wbFmYTSrxmXc= 2022-09-03T07:25:56.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w1lYeRaMtMKWeD9wbFmYTSrxmXc=', 'Content-Length': '0'} 2022-09-03T07:25:56.500 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf930b639f58489a-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.501 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.502 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.502 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:25:56.503 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:25:56.503 DEBUG:boto:Method: GET 2022-09-03T07:25:56.504 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:25:56.504 DEBUG:boto:Data: 2022-09-03T07:25:56.506 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.506 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.507 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.508 DEBUG:boto:Params: {} 2022-09-03T07:25:56.508 DEBUG:boto:Token: None 2022-09-03T07:25:56.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.514 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c8dd2f753f400fb-0063130184-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.515 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.517 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.517 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.518 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:25:56.519 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:25:56.520 DEBUG:boto:Method: GET 2022-09-03T07:25:56.520 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:25:56.521 DEBUG:boto:Data: 2022-09-03T07:25:56.522 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.522 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.523 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.523 DEBUG:boto:Params: {} 2022-09-03T07:25:56.524 DEBUG:boto:Token: None 2022-09-03T07:25:56.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.526 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.531 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc99e2e5439adc14-0063130184-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.531 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.532 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:25:56.533 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.533 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.534 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.534 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.535 DEBUG:boto:Method: GET 2022-09-03T07:25:56.535 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.536 DEBUG:boto:Data: 2022-09-03T07:25:56.537 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.537 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.538 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.538 DEBUG:boto:Params: {} 2022-09-03T07:25:56.539 DEBUG:boto:Token: None 2022-09-03T07:25:56.539 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yqPJDloGqPMRJo6ZELsIXZ97Ji4= 2022-09-03T07:25:56.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yqPJDloGqPMRJo6ZELsIXZ97Ji4=', 'Content-Length': '0'} 2022-09-03T07:25:56.543 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002dca794ad89922f2-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.543 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.544 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.545 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.546 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.546 DEBUG:boto:Method: GET 2022-09-03T07:25:56.547 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.547 DEBUG:boto:Data: 2022-09-03T07:25:56.548 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.549 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.549 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.550 DEBUG:boto:Params: {} 2022-09-03T07:25:56.550 DEBUG:boto:Token: None 2022-09-03T07:25:56.551 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yqPJDloGqPMRJo6ZELsIXZ97Ji4= 2022-09-03T07:25:56.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yqPJDloGqPMRJo6ZELsIXZ97Ji4=', 'Content-Length': '0'} 2022-09-03T07:25:56.554 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071d18bb8a11389d2-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.555 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.556 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.556 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.557 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.557 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.558 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.558 DEBUG:boto:Data: 2022-09-03T07:25:56.559 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.559 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.560 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.560 DEBUG:boto:Params: {} 2022-09-03T07:25:56.561 DEBUG:boto:Token: None 2022-09-03T07:25:56.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UafkspMa6twadSJJW5nZulQJwOo= 2022-09-03T07:25:56.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UafkspMa6twadSJJW5nZulQJwOo=', 'Content-Length': '0'} 2022-09-03T07:25:56.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000051c5ddfce3cbfc9b-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.565 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.565 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.566 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.566 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.567 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.567 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.568 DEBUG:boto:Data: 2022-09-03T07:25:56.568 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.569 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.569 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.570 DEBUG:boto:Params: {} 2022-09-03T07:25:56.570 DEBUG:boto:Token: None 2022-09-03T07:25:56.571 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UafkspMa6twadSJJW5nZulQJwOo= 2022-09-03T07:25:56.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UafkspMa6twadSJJW5nZulQJwOo=', 'Content-Length': '0'} 2022-09-03T07:25:56.574 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d56d52a822e947f8-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.575 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:25:56.576 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.576 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.577 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.577 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.578 DEBUG:boto:Method: GET 2022-09-03T07:25:56.578 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.579 DEBUG:boto:Data: 2022-09-03T07:25:56.579 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.580 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.580 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.581 DEBUG:boto:Params: {} 2022-09-03T07:25:56.581 DEBUG:boto:Token: None 2022-09-03T07:25:56.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yqPJDloGqPMRJo6ZELsIXZ97Ji4= 2022-09-03T07:25:56.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yqPJDloGqPMRJo6ZELsIXZ97Ji4=', 'Content-Length': '0'} 2022-09-03T07:25:56.586 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008ed2d4cb2bd7c1b8-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.630 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.631 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.632 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.633 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.634 DEBUG:boto:Method: GET 2022-09-03T07:25:56.635 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.636 DEBUG:boto:Data: 2022-09-03T07:25:56.637 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.638 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.639 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.640 DEBUG:boto:Params: {} 2022-09-03T07:25:56.641 DEBUG:boto:Token: None 2022-09-03T07:25:56.642 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:25:56.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yqPJDloGqPMRJo6ZELsIXZ97Ji4= 2022-09-03T07:25:56.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yqPJDloGqPMRJo6ZELsIXZ97Ji4=', 'Content-Length': '0'} 2022-09-03T07:25:56.646 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000884631d3cfcbf8f5-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.647 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.648 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.648 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:25:56.649 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:25:56.649 DEBUG:boto:Method: GET 2022-09-03T07:25:56.650 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:25:56.651 DEBUG:boto:Data: 2022-09-03T07:25:56.651 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.652 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.652 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.653 DEBUG:boto:Params: {} 2022-09-03T07:25:56.654 DEBUG:boto:Token: None 2022-09-03T07:25:56.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.659 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d19374f4097c11fd-0063130184-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.659 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.660 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.661 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.663 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:25:56.664 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:25:56.665 DEBUG:boto:Method: GET 2022-09-03T07:25:56.666 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:25:56.667 DEBUG:boto:Data: 2022-09-03T07:25:56.668 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.669 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.670 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.671 DEBUG:boto:Params: {} 2022-09-03T07:25:56.672 DEBUG:boto:Token: None 2022-09-03T07:25:56.673 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.674 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.679 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000038988fd46db40e46-0063130184-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.680 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.681 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.681 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.682 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.682 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:25:56.683 DEBUG:boto:Data: 2022-09-03T07:25:56.683 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.684 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.684 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.685 DEBUG:boto:Params: {} 2022-09-03T07:25:56.685 DEBUG:boto:Token: None 2022-09-03T07:25:56.686 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95 2022-09-03T07:25:56.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3r4LuzKatUh/s4ZA00zcc+ktC0s= 2022-09-03T07:25:56.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3r4LuzKatUh/s4ZA00zcc+ktC0s=', 'Content-Length': '0'} 2022-09-03T07:25:56.694 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096f1639927c0bd3e-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.695 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.695 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.696 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.697 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:25:56.697 DEBUG:boto:Data: 2022-09-03T07:25:56.698 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.699 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.699 DEBUG:boto:Params: {} 2022-09-03T07:25:56.700 DEBUG:boto:Token: None 2022-09-03T07:25:56.700 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95 2022-09-03T07:25:56.701 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3r4LuzKatUh/s4ZA00zcc+ktC0s= 2022-09-03T07:25:56.702 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3r4LuzKatUh/s4ZA00zcc+ktC0s=', 'Content-Length': '0'} 2022-09-03T07:25:56.704 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007adb37ca30ec864c-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.705 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:25:56.705 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.706 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.706 DEBUG:boto:Method: GET 2022-09-03T07:25:56.707 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:25:56.707 DEBUG:boto:Data: 2022-09-03T07:25:56.708 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.708 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.709 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.709 DEBUG:boto:Params: {} 2022-09-03T07:25:56.710 DEBUG:boto:Token: None 2022-09-03T07:25:56.711 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95 2022-09-03T07:25:56.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vT92C+o9W9wi9lJB7lo653CqhZw= 2022-09-03T07:25:56.712 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vT92C+o9W9wi9lJB7lo653CqhZw=', 'Content-Length': '0'} 2022-09-03T07:25:56.714 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007bb4ddd84fcfb137-0063130184-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.758 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:25:56.759 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:25:56.759 DEBUG:boto:Method: GET 2022-09-03T07:25:56.760 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:25:56.761 DEBUG:boto:Data: 2022-09-03T07:25:56.762 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.764 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.765 DEBUG:boto:Params: {} 2022-09-03T07:25:56.766 DEBUG:boto:Token: None 2022-09-03T07:25:56.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj95 2022-09-03T07:25:56.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vT92C+o9W9wi9lJB7lo653CqhZw= 2022-09-03T07:25:56.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vT92C+o9W9wi9lJB7lo653CqhZw=', 'Content-Length': '0'} 2022-09-03T07:25:56.772 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012496a8adabe4196-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.773 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.774 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.775 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:25:56.776 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:25:56.777 DEBUG:boto:Method: GET 2022-09-03T07:25:56.778 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:25:56.778 DEBUG:boto:Data: 2022-09-03T07:25:56.779 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.780 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:25:56.781 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.782 DEBUG:boto:Params: {} 2022-09-03T07:25:56.783 DEBUG:boto:Token: None 2022-09-03T07:25:56.783 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.790 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4567b634226095f-0063130184-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.791 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.792 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.793 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.794 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:25:56.794 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:25:56.795 DEBUG:boto:Method: GET 2022-09-03T07:25:56.795 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:25:56.796 DEBUG:boto:Data: 2022-09-03T07:25:56.797 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.797 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.798 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.799 DEBUG:boto:Params: {} 2022-09-03T07:25:56.799 DEBUG:boto:Token: None 2022-09-03T07:25:56.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw= 2022-09-03T07:25:56.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OzfS5mll8rsjG2fSuJpTZgZVbiw=', 'Content-Length': '0'} 2022-09-03T07:25:56.806 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009464cf777b5082cc-0063130184-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.807 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.807 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:25:56.808 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:25:56.808 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.809 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.810 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.811 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:25:56.811 DEBUG:boto:Data: 2022-09-03T07:25:56.812 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:56.813 DEBUG:boto:Port: 8001 2022-09-03T07:25:56.814 DEBUG:boto:Params: {} 2022-09-03T07:25:56.815 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:25:56.816 DEBUG:boto:Token: None 2022-09-03T07:25:56.817 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ZD4t4hZS8LYB5Ypg69j4h3jqKc= 2022-09-03T07:25:56.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ZD4t4hZS8LYB5Ypg69j4h3jqKc=', 'Content-Length': '0'} 2022-09-03T07:25:56.824 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '25'), ('X-RGW-Bytes-Used', '75'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000fc155c4c4c6ba852-0063130184-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.825 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.827 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.828 DEBUG:boto:Method: HEAD 2022-09-03T07:25:56.828 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:25:56.829 DEBUG:boto:Data: 2022-09-03T07:25:56.829 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.831 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.831 DEBUG:boto:Params: {} 2022-09-03T07:25:56.832 DEBUG:boto:Token: None 2022-09-03T07:25:56.832 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/ 2022-09-03T07:25:56.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ZD4t4hZS8LYB5Ypg69j4h3jqKc= 2022-09-03T07:25:56.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ZD4t4hZS8LYB5Ypg69j4h3jqKc=', 'Content-Length': '0'} 2022-09-03T07:25:56.857 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '25'), ('X-RGW-Bytes-Used', '75'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ec1db41173bec5e3-0063130184-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.859 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:25:56.859 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.860 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.860 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.861 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.861 DEBUG:boto:Method: GET 2022-09-03T07:25:56.862 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.863 DEBUG:boto:Data: 2022-09-03T07:25:56.863 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:56.864 DEBUG:boto:Port: 8001 2022-09-03T07:25:56.865 DEBUG:boto:Params: {} 2022-09-03T07:25:56.865 DEBUG:boto:Token: None 2022-09-03T07:25:56.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/?versions 2022-09-03T07:25:56.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1HX4FYj3+/8pKN9gpghwl10Ct4= 2022-09-03T07:25:56.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1HX4FYj3+/8pKN9gpghwl10Ct4=', 'Content-Length': '0'} 2022-09-03T07:25:56.870 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c18d96cb671aceb-0063130184-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.872 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.875 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:25:56.876 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:25:56.876 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:25:56.877 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:25:56.877 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:25:56.878 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:25:56.878 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:25:56.879 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:25:56.879 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:25:56.880 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:25:56.880 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:25:56.881 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:25:56.881 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:25:56.882 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:25:56.882 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:25:56.883 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:25:56.884 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:25:56.884 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:25:56.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:25:56.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:25:56.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:25:56.886 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:25:56.886 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:25:56.887 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:25:56.887 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:25:56.888 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:25:56.888 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.889 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.889 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.890 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.891 DEBUG:boto:Method: GET 2022-09-03T07:25:56.891 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.892 DEBUG:boto:Data: 2022-09-03T07:25:56.892 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.894 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.894 DEBUG:boto:Params: {} 2022-09-03T07:25:56.895 DEBUG:boto:Token: None 2022-09-03T07:25:56.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/?versions 2022-09-03T07:25:56.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1HX4FYj3+/8pKN9gpghwl10Ct4= 2022-09-03T07:25:56.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1HX4FYj3+/8pKN9gpghwl10Ct4=', 'Content-Length': '0'} 2022-09-03T07:25:56.901 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001b506f468e319b8b-0063130184-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.903 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:25:56.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:25:56.908 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:25:56.909 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:25:56.910 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:25:56.911 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:25:56.912 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:25:56.913 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:25:56.913 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:25:56.914 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:25:56.915 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:25:56.916 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:25:56.917 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:25:56.917 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:25:56.918 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:25:56.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:25:56.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:25:56.920 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:25:56.920 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:25:56.921 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:25:56.921 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:25:56.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:25:56.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:25:56.923 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:25:56.924 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:25:56.925 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.926 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.927 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.928 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.928 DEBUG:boto:Method: GET 2022-09-03T07:25:56.929 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.930 DEBUG:boto:Data: 2022-09-03T07:25:56.931 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:56.932 DEBUG:boto:Port: 8001 2022-09-03T07:25:56.932 DEBUG:boto:Params: {} 2022-09-03T07:25:56.933 DEBUG:boto:Token: None 2022-09-03T07:25:56.933 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/?versions 2022-09-03T07:25:56.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1HX4FYj3+/8pKN9gpghwl10Ct4= 2022-09-03T07:25:56.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1HX4FYj3+/8pKN9gpghwl10Ct4=', 'Content-Length': '0'} 2022-09-03T07:25:56.938 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011b71e345363ec7f-0063130184-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.938 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.940 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:56.941 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:56.941 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.942 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.942 DEBUG:boto:Method: GET 2022-09-03T07:25:56.943 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:25:56.943 DEBUG:boto:Data: 2022-09-03T07:25:56.944 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.945 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.946 DEBUG:boto:Params: {} 2022-09-03T07:25:56.946 DEBUG:boto:Token: None 2022-09-03T07:25:56.947 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/?versions 2022-09-03T07:25:56.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K1HX4FYj3+/8pKN9gpghwl10Ct4= 2022-09-03T07:25:56.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K1HX4FYj3+/8pKN9gpghwl10Ct4=', 'Content-Length': '0'} 2022-09-03T07:25:56.952 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002eafc2d76f834bfa-0063130184-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.953 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:56.955 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:25:56.956 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:56.956 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:56.957 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:56.957 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:56.958 DEBUG:boto:Method: GET 2022-09-03T07:25:56.958 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:56.959 DEBUG:boto:Data: 2022-09-03T07:25:56.959 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:56.961 DEBUG:boto:Port: 8001 2022-09-03T07:25:56.961 DEBUG:boto:Params: {} 2022-09-03T07:25:56.962 DEBUG:boto:Token: None 2022-09-03T07:25:56.962 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:56.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WLHACkfGeVr5fotDL5eW506lr0g= 2022-09-03T07:25:56.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WLHACkfGeVr5fotDL5eW506lr0g=', 'Content-Length': '0'} 2022-09-03T07:25:56.966 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009270653e584ea151-0063130184-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:56.967 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:56.967 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:56.968 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:56.968 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:56.969 DEBUG:boto:Method: GET 2022-09-03T07:25:56.969 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:56.970 DEBUG:boto:Data: 2022-09-03T07:25:56.971 DEBUG:boto:Headers: {} 2022-09-03T07:25:56.971 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:56.972 DEBUG:boto:Port: 8000 2022-09-03T07:25:56.972 DEBUG:boto:Params: {} 2022-09-03T07:25:56.973 DEBUG:boto:Token: None 2022-09-03T07:25:56.973 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:56 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:56.974 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WLHACkfGeVr5fotDL5eW506lr0g= 2022-09-03T07:25:56.975 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WLHACkfGeVr5fotDL5eW506lr0g=', 'Content-Length': '0'} 2022-09-03T07:25:56.977 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054934a6c51aa6636-0063130184-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.018 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:57.020 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:57.021 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.022 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.023 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.024 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.025 DEBUG:boto:Data: 2022-09-03T07:25:57.027 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.028 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.029 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.030 DEBUG:boto:Params: {} 2022-09-03T07:25:57.031 DEBUG:boto:Token: None 2022-09-03T07:25:57.032 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XxfuGGxz4mTgBuRuFHNJyJscvSw= 2022-09-03T07:25:57.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XxfuGGxz4mTgBuRuFHNJyJscvSw=', 'Content-Length': '0'} 2022-09-03T07:25:57.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000927a71207f4ab6af-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.039 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:57.040 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:57.041 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.042 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.042 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.043 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.044 DEBUG:boto:Data: 2022-09-03T07:25:57.044 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.045 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.046 DEBUG:boto:Params: {} 2022-09-03T07:25:57.046 DEBUG:boto:Token: None 2022-09-03T07:25:57.047 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XxfuGGxz4mTgBuRuFHNJyJscvSw= 2022-09-03T07:25:57.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XxfuGGxz4mTgBuRuFHNJyJscvSw=', 'Content-Length': '0'} 2022-09-03T07:25:57.051 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002ba8d9e40650e1e-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.053 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:25:57.054 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:57.055 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:57.055 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.056 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.058 DEBUG:boto:Method: GET 2022-09-03T07:25:57.059 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.060 DEBUG:boto:Data: 2022-09-03T07:25:57.061 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.063 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.064 DEBUG:boto:Params: {} 2022-09-03T07:25:57.065 DEBUG:boto:Token: None 2022-09-03T07:25:57.066 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.067 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6IJtN1PSXg+rk3FCTQoa0K+lb/U= 2022-09-03T07:25:57.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6IJtN1PSXg+rk3FCTQoa0K+lb/U=', 'Content-Length': '0'} 2022-09-03T07:25:57.072 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007e484d13805a128-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.073 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:57.074 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:57.075 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.076 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.077 DEBUG:boto:Method: GET 2022-09-03T07:25:57.078 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.079 DEBUG:boto:Data: 2022-09-03T07:25:57.079 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.080 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.081 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.082 DEBUG:boto:Params: {} 2022-09-03T07:25:57.083 DEBUG:boto:Token: None 2022-09-03T07:25:57.084 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:25:57.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6IJtN1PSXg+rk3FCTQoa0K+lb/U= 2022-09-03T07:25:57.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6IJtN1PSXg+rk3FCTQoa0K+lb/U=', 'Content-Length': '0'} 2022-09-03T07:25:57.088 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d98a6492b37bbb1f-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.089 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.090 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.091 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:57.092 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:57.093 DEBUG:boto:Method: GET 2022-09-03T07:25:57.094 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:25:57.094 DEBUG:boto:Data: 2022-09-03T07:25:57.096 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.097 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.097 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.098 DEBUG:boto:Params: {} 2022-09-03T07:25:57.099 DEBUG:boto:Token: None 2022-09-03T07:25:57.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.106 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f66c846b4325187a-0063130185-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.107 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.108 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.109 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.110 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:57.111 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:57.112 DEBUG:boto:Method: GET 2022-09-03T07:25:57.113 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:25:57.113 DEBUG:boto:Data: 2022-09-03T07:25:57.115 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.116 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.117 DEBUG:boto:Params: {} 2022-09-03T07:25:57.118 DEBUG:boto:Token: None 2022-09-03T07:25:57.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.125 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000040976d79a8cc1a78-0063130185-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.126 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.127 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:57.128 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:57.129 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.130 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:25:57.131 DEBUG:boto:Data: 2022-09-03T07:25:57.132 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.133 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.133 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.136 DEBUG:boto:Params: {} 2022-09-03T07:25:57.137 DEBUG:boto:Token: None 2022-09-03T07:25:57.138 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj11 2022-09-03T07:25:57.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0EpY+6qkXTsXX5R0qud4F5kOP6o= 2022-09-03T07:25:57.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0EpY+6qkXTsXX5R0qud4F5kOP6o=', 'Content-Length': '0'} 2022-09-03T07:25:57.143 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1af1a9a573d318c-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.145 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:57.146 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:57.147 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.148 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:25:57.149 DEBUG:boto:Data: 2022-09-03T07:25:57.150 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.151 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.152 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.153 DEBUG:boto:Params: {} 2022-09-03T07:25:57.154 DEBUG:boto:Token: None 2022-09-03T07:25:57.155 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj11 2022-09-03T07:25:57.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0EpY+6qkXTsXX5R0qud4F5kOP6o= 2022-09-03T07:25:57.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0EpY+6qkXTsXX5R0qud4F5kOP6o=', 'Content-Length': '0'} 2022-09-03T07:25:57.160 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c14770396d2d834-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.162 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:25:57.163 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:57.164 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:57.165 DEBUG:boto:Method: GET 2022-09-03T07:25:57.166 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:25:57.167 DEBUG:boto:Data: 2022-09-03T07:25:57.168 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.169 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.170 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.171 DEBUG:boto:Params: {} 2022-09-03T07:25:57.171 DEBUG:boto:Token: None 2022-09-03T07:25:57.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj11 2022-09-03T07:25:57.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ftueUKAQKkhKEK8JtW+1sJEjabw= 2022-09-03T07:25:57.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ftueUKAQKkhKEK8JtW+1sJEjabw=', 'Content-Length': '0'} 2022-09-03T07:25:57.177 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e6cb56ffc44fa339-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.218 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:25:57.219 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:25:57.219 DEBUG:boto:Method: GET 2022-09-03T07:25:57.220 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:25:57.220 DEBUG:boto:Data: 2022-09-03T07:25:57.221 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.222 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.223 DEBUG:boto:Params: {} 2022-09-03T07:25:57.223 DEBUG:boto:Token: None 2022-09-03T07:25:57.224 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj11 2022-09-03T07:25:57.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ftueUKAQKkhKEK8JtW+1sJEjabw= 2022-09-03T07:25:57.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ftueUKAQKkhKEK8JtW+1sJEjabw=', 'Content-Length': '0'} 2022-09-03T07:25:57.228 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab886a4b63d61976-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.229 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.229 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.230 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:57.230 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:57.231 DEBUG:boto:Method: GET 2022-09-03T07:25:57.231 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:25:57.232 DEBUG:boto:Data: 2022-09-03T07:25:57.233 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.233 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.234 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.235 DEBUG:boto:Params: {} 2022-09-03T07:25:57.235 DEBUG:boto:Token: None 2022-09-03T07:25:57.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.241 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000081e4efedb2f5b0a-0063130185-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.242 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.244 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.245 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.246 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:57.247 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:25:57.248 DEBUG:boto:Method: GET 2022-09-03T07:25:57.248 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:25:57.249 DEBUG:boto:Data: 2022-09-03T07:25:57.250 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.253 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.253 DEBUG:boto:Params: {} 2022-09-03T07:25:57.254 DEBUG:boto:Token: None 2022-09-03T07:25:57.255 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.261 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004def4954f9b3081b-0063130185-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.262 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.263 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:25:57.263 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.264 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.265 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.266 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.266 DEBUG:boto:Method: GET 2022-09-03T07:25:57.267 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.267 DEBUG:boto:Data: 2022-09-03T07:25:57.268 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.270 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.270 DEBUG:boto:Params: {} 2022-09-03T07:25:57.271 DEBUG:boto:Token: None 2022-09-03T07:25:57.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeTvnSaG8BS6Sw2b77a7/T4VnRI= 2022-09-03T07:25:57.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeTvnSaG8BS6Sw2b77a7/T4VnRI=', 'Content-Length': '0'} 2022-09-03T07:25:57.286 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cf313bea15f8617-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.287 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.288 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.289 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.290 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.291 DEBUG:boto:Method: GET 2022-09-03T07:25:57.292 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.293 DEBUG:boto:Data: 2022-09-03T07:25:57.294 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.295 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.296 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.297 DEBUG:boto:Params: {} 2022-09-03T07:25:57.297 DEBUG:boto:Token: None 2022-09-03T07:25:57.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeTvnSaG8BS6Sw2b77a7/T4VnRI= 2022-09-03T07:25:57.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeTvnSaG8BS6Sw2b77a7/T4VnRI=', 'Content-Length': '0'} 2022-09-03T07:25:57.304 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be0e4ab5405f491a-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.306 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.307 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.307 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.308 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.308 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.309 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.310 DEBUG:boto:Data: 2022-09-03T07:25:57.311 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.313 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.314 DEBUG:boto:Params: {} 2022-09-03T07:25:57.315 DEBUG:boto:Token: None 2022-09-03T07:25:57.316 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nQ4TKKO8wpwHDD/tZ+auttXx94Y= 2022-09-03T07:25:57.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nQ4TKKO8wpwHDD/tZ+auttXx94Y=', 'Content-Length': '0'} 2022-09-03T07:25:57.322 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ede91eca10280a8f-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.324 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.325 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.326 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.327 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.328 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.330 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.331 DEBUG:boto:Data: 2022-09-03T07:25:57.332 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.333 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.334 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.335 DEBUG:boto:Params: {} 2022-09-03T07:25:57.336 DEBUG:boto:Token: None 2022-09-03T07:25:57.338 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nQ4TKKO8wpwHDD/tZ+auttXx94Y= 2022-09-03T07:25:57.339 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nQ4TKKO8wpwHDD/tZ+auttXx94Y=', 'Content-Length': '0'} 2022-09-03T07:25:57.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002000608571809990-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.344 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:25:57.345 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.346 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.348 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.349 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.351 DEBUG:boto:Method: GET 2022-09-03T07:25:57.352 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.353 DEBUG:boto:Data: 2022-09-03T07:25:57.354 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.357 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.358 DEBUG:boto:Params: {} 2022-09-03T07:25:57.359 DEBUG:boto:Token: None 2022-09-03T07:25:57.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeTvnSaG8BS6Sw2b77a7/T4VnRI= 2022-09-03T07:25:57.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeTvnSaG8BS6Sw2b77a7/T4VnRI=', 'Content-Length': '0'} 2022-09-03T07:25:57.365 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca771c2579d8b381-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.367 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.368 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.369 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.369 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.370 DEBUG:boto:Method: GET 2022-09-03T07:25:57.371 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.371 DEBUG:boto:Data: 2022-09-03T07:25:57.372 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.373 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.374 DEBUG:boto:Params: {} 2022-09-03T07:25:57.375 DEBUG:boto:Token: None 2022-09-03T07:25:57.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:25:57.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eeTvnSaG8BS6Sw2b77a7/T4VnRI= 2022-09-03T07:25:57.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eeTvnSaG8BS6Sw2b77a7/T4VnRI=', 'Content-Length': '0'} 2022-09-03T07:25:57.385 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aea6099172e6f8e4-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.427 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.428 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.428 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:57.429 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:57.430 DEBUG:boto:Method: GET 2022-09-03T07:25:57.431 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:25:57.432 DEBUG:boto:Data: 2022-09-03T07:25:57.433 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.435 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.435 DEBUG:boto:Params: {} 2022-09-03T07:25:57.436 DEBUG:boto:Token: None 2022-09-03T07:25:57.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.442 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085154f37dfb4a097-0063130185-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.443 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.444 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.445 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.445 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:57.446 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:57.447 DEBUG:boto:Method: GET 2022-09-03T07:25:57.447 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:25:57.448 DEBUG:boto:Data: 2022-09-03T07:25:57.449 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.457 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.458 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.459 DEBUG:boto:Params: {} 2022-09-03T07:25:57.459 DEBUG:boto:Token: None 2022-09-03T07:25:57.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.469 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031aa13ace33bed3f-0063130185-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.471 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.472 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.473 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.473 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.474 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:25:57.475 DEBUG:boto:Data: 2022-09-03T07:25:57.475 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.477 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.477 DEBUG:boto:Params: {} 2022-09-03T07:25:57.478 DEBUG:boto:Token: None 2022-09-03T07:25:57.479 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115 2022-09-03T07:25:57.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hm59wJbwkt9jXS2V2oRbu2OhkYw= 2022-09-03T07:25:57.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hm59wJbwkt9jXS2V2oRbu2OhkYw=', 'Content-Length': '0'} 2022-09-03T07:25:57.484 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001388bac21d1797c3-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.485 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.486 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.487 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.488 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:25:57.489 DEBUG:boto:Data: 2022-09-03T07:25:57.490 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.493 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.494 DEBUG:boto:Params: {} 2022-09-03T07:25:57.495 DEBUG:boto:Token: None 2022-09-03T07:25:57.496 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115 2022-09-03T07:25:57.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hm59wJbwkt9jXS2V2oRbu2OhkYw= 2022-09-03T07:25:57.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hm59wJbwkt9jXS2V2oRbu2OhkYw=', 'Content-Length': '0'} 2022-09-03T07:25:57.501 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa8bea55e55e7afc-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.502 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:25:57.503 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.503 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.504 DEBUG:boto:Method: GET 2022-09-03T07:25:57.505 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:25:57.505 DEBUG:boto:Data: 2022-09-03T07:25:57.506 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.507 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.507 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.508 DEBUG:boto:Params: {} 2022-09-03T07:25:57.509 DEBUG:boto:Token: None 2022-09-03T07:25:57.510 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115 2022-09-03T07:25:57.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0l22jPxqd76Uj8ublAoDENdkzc= 2022-09-03T07:25:57.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0l22jPxqd76Uj8ublAoDENdkzc=', 'Content-Length': '0'} 2022-09-03T07:25:57.514 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002586888630c0b285-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.516 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:25:57.516 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:25:57.517 DEBUG:boto:Method: GET 2022-09-03T07:25:57.518 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:25:57.519 DEBUG:boto:Data: 2022-09-03T07:25:57.519 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.521 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.521 DEBUG:boto:Params: {} 2022-09-03T07:25:57.522 DEBUG:boto:Token: None 2022-09-03T07:25:57.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj115 2022-09-03T07:25:57.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v0l22jPxqd76Uj8ublAoDENdkzc= 2022-09-03T07:25:57.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v0l22jPxqd76Uj8ublAoDENdkzc=', 'Content-Length': '0'} 2022-09-03T07:25:57.526 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f60f05dd505bd90e-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.528 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.528 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.529 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:57.530 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:57.531 DEBUG:boto:Method: GET 2022-09-03T07:25:57.531 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:25:57.532 DEBUG:boto:Data: 2022-09-03T07:25:57.532 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.533 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.533 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.534 DEBUG:boto:Params: {} 2022-09-03T07:25:57.535 DEBUG:boto:Token: None 2022-09-03T07:25:57.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.540 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000133142a44157c663-0063130185-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.586 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.587 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.588 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.588 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:57.589 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:25:57.590 DEBUG:boto:Method: GET 2022-09-03T07:25:57.590 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:25:57.591 DEBUG:boto:Data: 2022-09-03T07:25:57.591 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.592 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.593 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.593 DEBUG:boto:Params: {} 2022-09-03T07:25:57.594 DEBUG:boto:Token: None 2022-09-03T07:25:57.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.595 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.600 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000074ba5d8d1bbff1bb-0063130185-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.601 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.602 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:25:57.603 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.603 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.604 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.604 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.605 DEBUG:boto:Method: GET 2022-09-03T07:25:57.606 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.606 DEBUG:boto:Data: 2022-09-03T07:25:57.607 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.608 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.609 DEBUG:boto:Params: {} 2022-09-03T07:25:57.609 DEBUG:boto:Token: None 2022-09-03T07:25:57.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hx7k+dWa7qIlz9BnXXI2ZfWyqRA= 2022-09-03T07:25:57.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hx7k+dWa7qIlz9BnXXI2ZfWyqRA=', 'Content-Length': '0'} 2022-09-03T07:25:57.614 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b442f0355c47960e-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.615 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.615 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.616 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.617 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.617 DEBUG:boto:Method: GET 2022-09-03T07:25:57.618 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.618 DEBUG:boto:Data: 2022-09-03T07:25:57.619 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.619 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.620 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.621 DEBUG:boto:Params: {} 2022-09-03T07:25:57.623 DEBUG:boto:Token: None 2022-09-03T07:25:57.624 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hx7k+dWa7qIlz9BnXXI2ZfWyqRA= 2022-09-03T07:25:57.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hx7k+dWa7qIlz9BnXXI2ZfWyqRA=', 'Content-Length': '0'} 2022-09-03T07:25:57.628 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d41421ad9b2530e-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.674 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.676 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.677 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.678 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.680 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.681 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.684 DEBUG:boto:Data: 2022-09-03T07:25:57.685 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.687 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.687 DEBUG:boto:Params: {} 2022-09-03T07:25:57.688 DEBUG:boto:Token: None 2022-09-03T07:25:57.689 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4p532kDK50Mw4cmaafCksRXXnnY= 2022-09-03T07:25:57.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4p532kDK50Mw4cmaafCksRXXnnY=', 'Content-Length': '0'} 2022-09-03T07:25:57.694 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000148aa3f6e1fb211e-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.695 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.696 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.697 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.698 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.699 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.700 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.701 DEBUG:boto:Data: 2022-09-03T07:25:57.701 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.702 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.703 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.703 DEBUG:boto:Params: {} 2022-09-03T07:25:57.704 DEBUG:boto:Token: None 2022-09-03T07:25:57.705 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4p532kDK50Mw4cmaafCksRXXnnY= 2022-09-03T07:25:57.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4p532kDK50Mw4cmaafCksRXXnnY=', 'Content-Length': '0'} 2022-09-03T07:25:57.709 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a221f634bed0bc2b-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.710 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:25:57.721 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.722 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.722 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.723 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.724 DEBUG:boto:Method: GET 2022-09-03T07:25:57.725 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.725 DEBUG:boto:Data: 2022-09-03T07:25:57.726 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.727 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.728 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.729 DEBUG:boto:Params: {} 2022-09-03T07:25:57.730 DEBUG:boto:Token: None 2022-09-03T07:25:57.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hx7k+dWa7qIlz9BnXXI2ZfWyqRA= 2022-09-03T07:25:57.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hx7k+dWa7qIlz9BnXXI2ZfWyqRA=', 'Content-Length': '0'} 2022-09-03T07:25:57.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000310e4761fca87df1-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.738 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.739 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.740 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.740 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.741 DEBUG:boto:Method: GET 2022-09-03T07:25:57.741 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.742 DEBUG:boto:Data: 2022-09-03T07:25:57.743 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.743 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.744 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.744 DEBUG:boto:Params: {} 2022-09-03T07:25:57.745 DEBUG:boto:Token: None 2022-09-03T07:25:57.746 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:25:57.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hx7k+dWa7qIlz9BnXXI2ZfWyqRA= 2022-09-03T07:25:57.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hx7k+dWa7qIlz9BnXXI2ZfWyqRA=', 'Content-Length': '0'} 2022-09-03T07:25:57.750 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e54dbcd71027522-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.751 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.752 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.753 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:57.754 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:57.755 DEBUG:boto:Method: GET 2022-09-03T07:25:57.755 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:25:57.756 DEBUG:boto:Data: 2022-09-03T07:25:57.756 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.757 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.758 DEBUG:boto:Params: {} 2022-09-03T07:25:57.759 DEBUG:boto:Token: None 2022-09-03T07:25:57.759 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.763 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000707a7011aabf152c-0063130185-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.764 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.765 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.766 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.766 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:57.767 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:57.768 DEBUG:boto:Method: GET 2022-09-03T07:25:57.769 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:25:57.770 DEBUG:boto:Data: 2022-09-03T07:25:57.771 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.772 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.773 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.774 DEBUG:boto:Params: {} 2022-09-03T07:25:57.775 DEBUG:boto:Token: None 2022-09-03T07:25:57.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.783 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058fce46f2200789b-0063130185-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.783 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.784 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.785 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.785 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.786 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:25:57.787 DEBUG:boto:Data: 2022-09-03T07:25:57.788 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.788 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.789 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.790 DEBUG:boto:Params: {} 2022-09-03T07:25:57.791 DEBUG:boto:Token: None 2022-09-03T07:25:57.792 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116 2022-09-03T07:25:57.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6lZbTH42M/OU/cZZpJGQLKP9gvA= 2022-09-03T07:25:57.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6lZbTH42M/OU/cZZpJGQLKP9gvA=', 'Content-Length': '0'} 2022-09-03T07:25:57.795 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c3de46ad5c7268c-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.796 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.797 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.797 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.798 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:25:57.799 DEBUG:boto:Data: 2022-09-03T07:25:57.800 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.801 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.802 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.803 DEBUG:boto:Params: {} 2022-09-03T07:25:57.803 DEBUG:boto:Token: None 2022-09-03T07:25:57.804 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116 2022-09-03T07:25:57.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6lZbTH42M/OU/cZZpJGQLKP9gvA= 2022-09-03T07:25:57.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6lZbTH42M/OU/cZZpJGQLKP9gvA=', 'Content-Length': '0'} 2022-09-03T07:25:57.809 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e76dcf2a16c41fa9-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.810 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:25:57.811 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.811 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.812 DEBUG:boto:Method: GET 2022-09-03T07:25:57.813 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:25:57.813 DEBUG:boto:Data: 2022-09-03T07:25:57.814 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.814 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.815 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.815 DEBUG:boto:Params: {} 2022-09-03T07:25:57.816 DEBUG:boto:Token: None 2022-09-03T07:25:57.817 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116 2022-09-03T07:25:57.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F0QG71PLF3rFcZizY4CcgcL5GYo= 2022-09-03T07:25:57.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F0QG71PLF3rFcZizY4CcgcL5GYo=', 'Content-Length': '0'} 2022-09-03T07:25:57.820 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2d325c5bd6001da-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.821 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:25:57.822 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:25:57.823 DEBUG:boto:Method: GET 2022-09-03T07:25:57.823 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:25:57.824 DEBUG:boto:Data: 2022-09-03T07:25:57.824 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.825 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.825 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.826 DEBUG:boto:Params: {} 2022-09-03T07:25:57.827 DEBUG:boto:Token: None 2022-09-03T07:25:57.827 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj116 2022-09-03T07:25:57.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F0QG71PLF3rFcZizY4CcgcL5GYo= 2022-09-03T07:25:57.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F0QG71PLF3rFcZizY4CcgcL5GYo=', 'Content-Length': '0'} 2022-09-03T07:25:57.831 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009286304b0582b884-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.832 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.832 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.833 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:57.833 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:57.834 DEBUG:boto:Method: GET 2022-09-03T07:25:57.834 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:25:57.835 DEBUG:boto:Data: 2022-09-03T07:25:57.835 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.836 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.836 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.837 DEBUG:boto:Params: {} 2022-09-03T07:25:57.838 DEBUG:boto:Token: None 2022-09-03T07:25:57.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.843 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eceb43005f4e64cc-0063130185-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.845 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.846 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:57.847 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:57.848 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:57.849 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:25:57.850 DEBUG:boto:Method: GET 2022-09-03T07:25:57.850 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:25:57.851 DEBUG:boto:Data: 2022-09-03T07:25:57.851 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.852 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.852 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.853 DEBUG:boto:Params: {} 2022-09-03T07:25:57.854 DEBUG:boto:Token: None 2022-09-03T07:25:57.854 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/ 2022-09-03T07:25:57.855 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU= 2022-09-03T07:25:57.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LYwm0UuexiX2nOUChZmXUp3PJOU=', 'Content-Length': '0'} 2022-09-03T07:25:57.859 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000078ffa018c8d2d695-0063130185-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.906 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:57.908 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:25:57.909 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:57.911 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:57.912 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.914 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.915 DEBUG:boto:Method: GET 2022-09-03T07:25:57.915 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.916 DEBUG:boto:Data: 2022-09-03T07:25:57.916 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.917 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.917 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.918 DEBUG:boto:Params: {} 2022-09-03T07:25:57.919 DEBUG:boto:Token: None 2022-09-03T07:25:57.919 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.920 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z088HbQxMuHnn1HszWuVRO34ZT0= 2022-09-03T07:25:57.921 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z088HbQxMuHnn1HszWuVRO34ZT0=', 'Content-Length': '0'} 2022-09-03T07:25:57.924 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045ee36459ca28eea-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.925 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:57.926 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:57.926 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.928 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.929 DEBUG:boto:Method: GET 2022-09-03T07:25:57.930 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.931 DEBUG:boto:Data: 2022-09-03T07:25:57.932 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.934 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.935 DEBUG:boto:Params: {} 2022-09-03T07:25:57.936 DEBUG:boto:Token: None 2022-09-03T07:25:57.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z088HbQxMuHnn1HszWuVRO34ZT0= 2022-09-03T07:25:57.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z088HbQxMuHnn1HszWuVRO34ZT0=', 'Content-Length': '0'} 2022-09-03T07:25:57.943 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064d700d4ab94aa9a-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.944 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:57.945 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:57.946 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.947 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.948 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.949 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.950 DEBUG:boto:Data: 2022-09-03T07:25:57.951 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.954 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.955 DEBUG:boto:Params: {} 2022-09-03T07:25:57.955 DEBUG:boto:Token: None 2022-09-03T07:25:57.956 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pohXVm4TGKtSfmiqu7erkxpjZKs= 2022-09-03T07:25:57.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pohXVm4TGKtSfmiqu7erkxpjZKs=', 'Content-Length': '0'} 2022-09-03T07:25:57.961 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007fbadb018a18eb6-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.963 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:57.964 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:57.964 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.965 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.965 DEBUG:boto:Method: HEAD 2022-09-03T07:25:57.966 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.966 DEBUG:boto:Data: 2022-09-03T07:25:57.967 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:57.968 DEBUG:boto:Port: 8000 2022-09-03T07:25:57.968 DEBUG:boto:Params: {} 2022-09-03T07:25:57.969 DEBUG:boto:Token: None 2022-09-03T07:25:57.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pohXVm4TGKtSfmiqu7erkxpjZKs= 2022-09-03T07:25:57.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pohXVm4TGKtSfmiqu7erkxpjZKs=', 'Content-Length': '0'} 2022-09-03T07:25:57.973 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029262b69ed407825-0063130185-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:57.974 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:25:57.976 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:57.976 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:57.977 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.977 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.978 DEBUG:boto:Method: GET 2022-09-03T07:25:57.978 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.979 DEBUG:boto:Data: 2022-09-03T07:25:57.979 DEBUG:boto:Headers: {} 2022-09-03T07:25:57.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:57.980 DEBUG:boto:Port: 8001 2022-09-03T07:25:57.981 DEBUG:boto:Params: {} 2022-09-03T07:25:57.982 DEBUG:boto:Token: None 2022-09-03T07:25:57.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:57 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:57.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z088HbQxMuHnn1HszWuVRO34ZT0= 2022-09-03T07:25:57.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z088HbQxMuHnn1HszWuVRO34ZT0=', 'Content-Length': '0'} 2022-09-03T07:25:57.986 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006c525a0973e29a99-0063130185-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.032 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:58.034 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:58.035 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:58.036 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:25:58.037 DEBUG:boto:Method: GET 2022-09-03T07:25:58.038 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:25:58.039 DEBUG:boto:Data: 2022-09-03T07:25:58.040 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.042 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.044 DEBUG:boto:Params: {} 2022-09-03T07:25:58.045 DEBUG:boto:Token: None 2022-09-03T07:25:58.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:25:58.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5DSk7V5bfLrIGBOAq386xnOszr4= 2022-09-03T07:25:58.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5DSk7V5bfLrIGBOAq386xnOszr4=', 'Content-Length': '0'} 2022-09-03T07:25:58.050 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000549d1208d5a32efd-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.051 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.052 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.052 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:58.053 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:58.053 DEBUG:boto:Method: GET 2022-09-03T07:25:58.054 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:25:58.055 DEBUG:boto:Data: 2022-09-03T07:25:58.055 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.056 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.057 DEBUG:boto:Params: {} 2022-09-03T07:25:58.058 DEBUG:boto:Token: None 2022-09-03T07:25:58.058 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.059 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.063 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a7c1fb42c1cf1ae-0063130186-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.067 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.068 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.069 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.069 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:58.070 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:58.070 DEBUG:boto:Method: GET 2022-09-03T07:25:58.071 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:25:58.071 DEBUG:boto:Data: 2022-09-03T07:25:58.072 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.072 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.073 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.073 DEBUG:boto:Params: {} 2022-09-03T07:25:58.074 DEBUG:boto:Token: None 2022-09-03T07:25:58.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.084 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf3803d0902830cf-0063130186-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.085 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.087 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:58.088 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:58.089 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.090 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:25:58.091 DEBUG:boto:Data: 2022-09-03T07:25:58.092 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.095 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.096 DEBUG:boto:Params: {} 2022-09-03T07:25:58.097 DEBUG:boto:Token: None 2022-09-03T07:25:58.098 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj145 2022-09-03T07:25:58.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YXqu53S3zfKw2idRs7Aio8KP+0A= 2022-09-03T07:25:58.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YXqu53S3zfKw2idRs7Aio8KP+0A=', 'Content-Length': '0'} 2022-09-03T07:25:58.103 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d158d04bf6468131-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.104 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:58.104 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:58.105 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.106 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:25:58.106 DEBUG:boto:Data: 2022-09-03T07:25:58.107 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.108 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.109 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.109 DEBUG:boto:Params: {} 2022-09-03T07:25:58.110 DEBUG:boto:Token: None 2022-09-03T07:25:58.110 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj145 2022-09-03T07:25:58.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YXqu53S3zfKw2idRs7Aio8KP+0A= 2022-09-03T07:25:58.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YXqu53S3zfKw2idRs7Aio8KP+0A=', 'Content-Length': '0'} 2022-09-03T07:25:58.115 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c26d3be1193ac9f5-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.119 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:25:58.120 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:58.121 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:58.122 DEBUG:boto:Method: GET 2022-09-03T07:25:58.123 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:25:58.124 DEBUG:boto:Data: 2022-09-03T07:25:58.125 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.127 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.129 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.130 DEBUG:boto:Params: {} 2022-09-03T07:25:58.131 DEBUG:boto:Token: None 2022-09-03T07:25:58.132 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj145 2022-09-03T07:25:58.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sVgnHPp4V4QU9tLdg0MgNzF3uBs= 2022-09-03T07:25:58.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sVgnHPp4V4QU9tLdg0MgNzF3uBs=', 'Content-Length': '0'} 2022-09-03T07:25:58.137 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000774440356697b28b-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.182 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:25:58.183 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:25:58.184 DEBUG:boto:Method: GET 2022-09-03T07:25:58.185 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:25:58.186 DEBUG:boto:Data: 2022-09-03T07:25:58.187 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.188 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.189 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.190 DEBUG:boto:Params: {} 2022-09-03T07:25:58.191 DEBUG:boto:Token: None 2022-09-03T07:25:58.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj145 2022-09-03T07:25:58.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sVgnHPp4V4QU9tLdg0MgNzF3uBs= 2022-09-03T07:25:58.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sVgnHPp4V4QU9tLdg0MgNzF3uBs=', 'Content-Length': '0'} 2022-09-03T07:25:58.196 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af5af4aabb80b9d7-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.197 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.197 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.198 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:58.199 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:58.199 DEBUG:boto:Method: GET 2022-09-03T07:25:58.200 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:25:58.200 DEBUG:boto:Data: 2022-09-03T07:25:58.201 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.202 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.203 DEBUG:boto:Params: {} 2022-09-03T07:25:58.203 DEBUG:boto:Token: None 2022-09-03T07:25:58.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.208 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ef69f1326b1057cf-0063130186-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.209 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.210 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.210 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.211 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:58.211 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:25:58.212 DEBUG:boto:Method: GET 2022-09-03T07:25:58.212 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:25:58.213 DEBUG:boto:Data: 2022-09-03T07:25:58.214 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.214 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.215 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.215 DEBUG:boto:Params: {} 2022-09-03T07:25:58.216 DEBUG:boto:Token: None 2022-09-03T07:25:58.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.222 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e2965f6f72511da3-0063130186-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.223 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.224 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:25:58.224 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.225 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.225 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.226 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.226 DEBUG:boto:Method: GET 2022-09-03T07:25:58.227 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.227 DEBUG:boto:Data: 2022-09-03T07:25:58.228 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.228 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.229 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.230 DEBUG:boto:Params: {} 2022-09-03T07:25:58.230 DEBUG:boto:Token: None 2022-09-03T07:25:58.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rzhu+uFkwa7pyyjcmqJ66Yp4F7Y= 2022-09-03T07:25:58.232 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rzhu+uFkwa7pyyjcmqJ66Yp4F7Y=', 'Content-Length': '0'} 2022-09-03T07:25:58.234 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000011f7e5c40c3c0abc-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.235 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.236 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.236 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.237 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.237 DEBUG:boto:Method: GET 2022-09-03T07:25:58.238 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.238 DEBUG:boto:Data: 2022-09-03T07:25:58.239 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.240 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.241 DEBUG:boto:Params: {} 2022-09-03T07:25:58.241 DEBUG:boto:Token: None 2022-09-03T07:25:58.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rzhu+uFkwa7pyyjcmqJ66Yp4F7Y= 2022-09-03T07:25:58.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rzhu+uFkwa7pyyjcmqJ66Yp4F7Y=', 'Content-Length': '0'} 2022-09-03T07:25:58.246 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a90733dc0ba14c29-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.247 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.247 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.248 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.248 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.249 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.249 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.250 DEBUG:boto:Data: 2022-09-03T07:25:58.250 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.251 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.252 DEBUG:boto:Params: {} 2022-09-03T07:25:58.253 DEBUG:boto:Token: None 2022-09-03T07:25:58.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieex1bJuXN34tBbGw/0ZukF+UU0= 2022-09-03T07:25:58.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieex1bJuXN34tBbGw/0ZukF+UU0=', 'Content-Length': '0'} 2022-09-03T07:25:58.257 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1aaf31b55d41be1-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.258 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.259 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.259 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.260 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.260 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.261 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.261 DEBUG:boto:Data: 2022-09-03T07:25:58.262 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.263 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.264 DEBUG:boto:Params: {} 2022-09-03T07:25:58.264 DEBUG:boto:Token: None 2022-09-03T07:25:58.265 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieex1bJuXN34tBbGw/0ZukF+UU0= 2022-09-03T07:25:58.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieex1bJuXN34tBbGw/0ZukF+UU0=', 'Content-Length': '0'} 2022-09-03T07:25:58.269 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf72e393a1ad5659-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.269 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:25:58.270 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.271 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.272 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.273 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.273 DEBUG:boto:Method: GET 2022-09-03T07:25:58.274 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.275 DEBUG:boto:Data: 2022-09-03T07:25:58.275 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.276 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.277 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.278 DEBUG:boto:Params: {} 2022-09-03T07:25:58.279 DEBUG:boto:Token: None 2022-09-03T07:25:58.280 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rzhu+uFkwa7pyyjcmqJ66Yp4F7Y= 2022-09-03T07:25:58.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rzhu+uFkwa7pyyjcmqJ66Yp4F7Y=', 'Content-Length': '0'} 2022-09-03T07:25:58.284 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3d878a3c25297aa-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.285 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.286 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.286 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.287 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.288 DEBUG:boto:Method: GET 2022-09-03T07:25:58.288 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.289 DEBUG:boto:Data: 2022-09-03T07:25:58.289 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.290 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.290 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.291 DEBUG:boto:Params: {} 2022-09-03T07:25:58.291 DEBUG:boto:Token: None 2022-09-03T07:25:58.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:25:58.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rzhu+uFkwa7pyyjcmqJ66Yp4F7Y= 2022-09-03T07:25:58.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rzhu+uFkwa7pyyjcmqJ66Yp4F7Y=', 'Content-Length': '0'} 2022-09-03T07:25:58.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fb9575e034a059a-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.296 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.297 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.297 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:58.298 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:58.298 DEBUG:boto:Method: GET 2022-09-03T07:25:58.299 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:25:58.299 DEBUG:boto:Data: 2022-09-03T07:25:58.300 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.300 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.301 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.301 DEBUG:boto:Params: {} 2022-09-03T07:25:58.302 DEBUG:boto:Token: None 2022-09-03T07:25:58.302 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.306 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd309f00116f0d7d-0063130186-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.307 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.308 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.308 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.309 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:58.309 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:58.310 DEBUG:boto:Method: GET 2022-09-03T07:25:58.310 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:25:58.311 DEBUG:boto:Data: 2022-09-03T07:25:58.311 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.312 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.313 DEBUG:boto:Params: {} 2022-09-03T07:25:58.314 DEBUG:boto:Token: None 2022-09-03T07:25:58.314 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.319 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002933e202f5a78b6d-0063130186-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.320 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.321 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.322 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.323 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.324 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:25:58.325 DEBUG:boto:Data: 2022-09-03T07:25:58.325 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.326 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.327 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.328 DEBUG:boto:Params: {} 2022-09-03T07:25:58.329 DEBUG:boto:Token: None 2022-09-03T07:25:58.330 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17 2022-09-03T07:25:58.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z/8Jrx1oHudyQzfx2AuNRvvFq1U= 2022-09-03T07:25:58.332 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z/8Jrx1oHudyQzfx2AuNRvvFq1U=', 'Content-Length': '0'} 2022-09-03T07:25:58.334 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d77d68bcf6dc6b28-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.336 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.336 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.337 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.338 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:25:58.339 DEBUG:boto:Data: 2022-09-03T07:25:58.340 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.341 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.342 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.343 DEBUG:boto:Params: {} 2022-09-03T07:25:58.344 DEBUG:boto:Token: None 2022-09-03T07:25:58.345 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17 2022-09-03T07:25:58.346 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z/8Jrx1oHudyQzfx2AuNRvvFq1U= 2022-09-03T07:25:58.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z/8Jrx1oHudyQzfx2AuNRvvFq1U=', 'Content-Length': '0'} 2022-09-03T07:25:58.349 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f02a70cca4010918-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.350 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:25:58.350 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.351 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.352 DEBUG:boto:Method: GET 2022-09-03T07:25:58.353 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:25:58.354 DEBUG:boto:Data: 2022-09-03T07:25:58.355 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.356 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.357 DEBUG:boto:Params: {} 2022-09-03T07:25:58.358 DEBUG:boto:Token: None 2022-09-03T07:25:58.359 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17 2022-09-03T07:25:58.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9IVIKhuwCszZ0FPNAAVhePick6o= 2022-09-03T07:25:58.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9IVIKhuwCszZ0FPNAAVhePick6o=', 'Content-Length': '0'} 2022-09-03T07:25:58.365 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000efbabf935c07a32c-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.410 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:25:58.411 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:25:58.412 DEBUG:boto:Method: GET 2022-09-03T07:25:58.412 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:25:58.413 DEBUG:boto:Data: 2022-09-03T07:25:58.413 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.414 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.415 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.415 DEBUG:boto:Params: {} 2022-09-03T07:25:58.416 DEBUG:boto:Token: None 2022-09-03T07:25:58.417 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj17 2022-09-03T07:25:58.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9IVIKhuwCszZ0FPNAAVhePick6o= 2022-09-03T07:25:58.423 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9IVIKhuwCszZ0FPNAAVhePick6o=', 'Content-Length': '0'} 2022-09-03T07:25:58.425 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a08b07f78d49fb5-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.427 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.428 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.429 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:58.430 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:58.430 DEBUG:boto:Method: GET 2022-09-03T07:25:58.431 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:25:58.432 DEBUG:boto:Data: 2022-09-03T07:25:58.433 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.436 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.437 DEBUG:boto:Params: {} 2022-09-03T07:25:58.438 DEBUG:boto:Token: None 2022-09-03T07:25:58.439 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.444 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b399478559eac280-0063130186-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.445 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.446 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.447 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.447 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:58.448 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:25:58.448 DEBUG:boto:Method: GET 2022-09-03T07:25:58.449 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:25:58.449 DEBUG:boto:Data: 2022-09-03T07:25:58.450 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.451 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.451 DEBUG:boto:Params: {} 2022-09-03T07:25:58.452 DEBUG:boto:Token: None 2022-09-03T07:25:58.454 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.462 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000062ead5c7c3488fdf-0063130186-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.463 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.464 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:25:58.464 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.465 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.465 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.471 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.472 DEBUG:boto:Method: GET 2022-09-03T07:25:58.473 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.474 DEBUG:boto:Data: 2022-09-03T07:25:58.475 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.477 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.478 DEBUG:boto:Params: {} 2022-09-03T07:25:58.479 DEBUG:boto:Token: None 2022-09-03T07:25:58.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YopDy0UiUtnl78hIlT2QmRoI2F8= 2022-09-03T07:25:58.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YopDy0UiUtnl78hIlT2QmRoI2F8=', 'Content-Length': '0'} 2022-09-03T07:25:58.485 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba4c701c7779ca05-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.486 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.486 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.487 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.488 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.488 DEBUG:boto:Method: GET 2022-09-03T07:25:58.489 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.489 DEBUG:boto:Data: 2022-09-03T07:25:58.490 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.491 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.492 DEBUG:boto:Params: {} 2022-09-03T07:25:58.493 DEBUG:boto:Token: None 2022-09-03T07:25:58.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YopDy0UiUtnl78hIlT2QmRoI2F8= 2022-09-03T07:25:58.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YopDy0UiUtnl78hIlT2QmRoI2F8=', 'Content-Length': '0'} 2022-09-03T07:25:58.497 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078263f19cd5a4f00-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.498 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.499 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.499 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.500 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.500 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.501 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.501 DEBUG:boto:Data: 2022-09-03T07:25:58.502 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.503 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.504 DEBUG:boto:Params: {} 2022-09-03T07:25:58.504 DEBUG:boto:Token: None 2022-09-03T07:25:58.505 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/+Cw0vy2qYLJ8ocdXTArMIBt9T0= 2022-09-03T07:25:58.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/+Cw0vy2qYLJ8ocdXTArMIBt9T0=', 'Content-Length': '0'} 2022-09-03T07:25:58.508 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020bf81d2f6cae33b-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.509 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.510 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.510 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.511 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.511 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.512 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.512 DEBUG:boto:Data: 2022-09-03T07:25:58.513 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.514 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.514 DEBUG:boto:Params: {} 2022-09-03T07:25:58.515 DEBUG:boto:Token: None 2022-09-03T07:25:58.515 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/+Cw0vy2qYLJ8ocdXTArMIBt9T0= 2022-09-03T07:25:58.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/+Cw0vy2qYLJ8ocdXTArMIBt9T0=', 'Content-Length': '0'} 2022-09-03T07:25:58.519 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003bc35874bcbc316b-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.519 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:25:58.520 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.521 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.521 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.522 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.522 DEBUG:boto:Method: GET 2022-09-03T07:25:58.523 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.523 DEBUG:boto:Data: 2022-09-03T07:25:58.524 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.525 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.526 DEBUG:boto:Params: {} 2022-09-03T07:25:58.526 DEBUG:boto:Token: None 2022-09-03T07:25:58.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YopDy0UiUtnl78hIlT2QmRoI2F8= 2022-09-03T07:25:58.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YopDy0UiUtnl78hIlT2QmRoI2F8=', 'Content-Length': '0'} 2022-09-03T07:25:58.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000006745a3867cd461a-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.531 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.531 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.532 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.532 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.533 DEBUG:boto:Method: GET 2022-09-03T07:25:58.534 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.534 DEBUG:boto:Data: 2022-09-03T07:25:58.535 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.536 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.536 DEBUG:boto:Params: {} 2022-09-03T07:25:58.537 DEBUG:boto:Token: None 2022-09-03T07:25:58.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:25:58.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YopDy0UiUtnl78hIlT2QmRoI2F8= 2022-09-03T07:25:58.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YopDy0UiUtnl78hIlT2QmRoI2F8=', 'Content-Length': '0'} 2022-09-03T07:25:58.541 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013f75dcbda80d719-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.543 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.544 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.544 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:58.545 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:58.546 DEBUG:boto:Method: GET 2022-09-03T07:25:58.547 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:25:58.548 DEBUG:boto:Data: 2022-09-03T07:25:58.549 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.550 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.552 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.552 DEBUG:boto:Params: {} 2022-09-03T07:25:58.554 DEBUG:boto:Token: None 2022-09-03T07:25:58.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.557 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.560 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000050d1f578cd4f3979-0063130186-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.561 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.563 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.564 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.564 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:58.565 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:58.566 DEBUG:boto:Method: GET 2022-09-03T07:25:58.567 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:25:58.569 DEBUG:boto:Data: 2022-09-03T07:25:58.569 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.571 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.572 DEBUG:boto:Params: {} 2022-09-03T07:25:58.573 DEBUG:boto:Token: None 2022-09-03T07:25:58.575 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.581 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000083d85204ce825594-0063130186-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.583 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.584 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.585 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.586 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.587 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:25:58.588 DEBUG:boto:Data: 2022-09-03T07:25:58.589 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.589 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.590 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.591 DEBUG:boto:Params: {} 2022-09-03T07:25:58.592 DEBUG:boto:Token: None 2022-09-03T07:25:58.593 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19 2022-09-03T07:25:58.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kJSByXCT75yr6I4UqQu0UzupnRs= 2022-09-03T07:25:58.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kJSByXCT75yr6I4UqQu0UzupnRs=', 'Content-Length': '0'} 2022-09-03T07:25:58.598 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc7ffb00fa72dfd4-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.599 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.599 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.600 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.600 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:25:58.601 DEBUG:boto:Data: 2022-09-03T07:25:58.601 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.602 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.602 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.603 DEBUG:boto:Params: {} 2022-09-03T07:25:58.604 DEBUG:boto:Token: None 2022-09-03T07:25:58.604 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19 2022-09-03T07:25:58.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kJSByXCT75yr6I4UqQu0UzupnRs= 2022-09-03T07:25:58.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kJSByXCT75yr6I4UqQu0UzupnRs=', 'Content-Length': '0'} 2022-09-03T07:25:58.608 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d51b6b4121d2aede-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.608 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:25:58.609 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.609 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.610 DEBUG:boto:Method: GET 2022-09-03T07:25:58.611 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:25:58.611 DEBUG:boto:Data: 2022-09-03T07:25:58.612 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.613 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.614 DEBUG:boto:Params: {} 2022-09-03T07:25:58.614 DEBUG:boto:Token: None 2022-09-03T07:25:58.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19 2022-09-03T07:25:58.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NQdh9QFRuB7CwEa5HMb5ckn+gAg= 2022-09-03T07:25:58.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NQdh9QFRuB7CwEa5HMb5ckn+gAg=', 'Content-Length': '0'} 2022-09-03T07:25:58.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af60f1a44dafc822-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.619 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:25:58.620 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:25:58.620 DEBUG:boto:Method: GET 2022-09-03T07:25:58.621 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:25:58.621 DEBUG:boto:Data: 2022-09-03T07:25:58.622 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.623 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.623 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.624 DEBUG:boto:Params: {} 2022-09-03T07:25:58.624 DEBUG:boto:Token: None 2022-09-03T07:25:58.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj19 2022-09-03T07:25:58.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NQdh9QFRuB7CwEa5HMb5ckn+gAg= 2022-09-03T07:25:58.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NQdh9QFRuB7CwEa5HMb5ckn+gAg=', 'Content-Length': '0'} 2022-09-03T07:25:58.628 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f1b5e61df37cca9-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.629 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.630 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.630 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:58.631 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:58.631 DEBUG:boto:Method: GET 2022-09-03T07:25:58.632 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:25:58.632 DEBUG:boto:Data: 2022-09-03T07:25:58.633 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.634 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.634 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.635 DEBUG:boto:Params: {} 2022-09-03T07:25:58.635 DEBUG:boto:Token: None 2022-09-03T07:25:58.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.637 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.640 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a4d2c1ef7e4f75d-0063130186-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.682 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.683 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.684 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.684 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:58.685 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:25:58.685 DEBUG:boto:Method: GET 2022-09-03T07:25:58.686 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:25:58.686 DEBUG:boto:Data: 2022-09-03T07:25:58.687 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.688 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.688 DEBUG:boto:Params: {} 2022-09-03T07:25:58.689 DEBUG:boto:Token: None 2022-09-03T07:25:58.690 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.695 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008734f5a1af02fb69-0063130186-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.695 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.696 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:25:58.697 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.697 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.698 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.698 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.699 DEBUG:boto:Method: GET 2022-09-03T07:25:58.700 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.700 DEBUG:boto:Data: 2022-09-03T07:25:58.701 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.702 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.703 DEBUG:boto:Params: {} 2022-09-03T07:25:58.703 DEBUG:boto:Token: None 2022-09-03T07:25:58.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YUqoqtTA89n23Rsz7SR/HdzLVls= 2022-09-03T07:25:58.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YUqoqtTA89n23Rsz7SR/HdzLVls=', 'Content-Length': '0'} 2022-09-03T07:25:58.707 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000cfeeebc35db8552-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.708 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.708 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.709 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.709 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.711 DEBUG:boto:Method: GET 2022-09-03T07:25:58.711 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.712 DEBUG:boto:Data: 2022-09-03T07:25:58.713 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.714 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.715 DEBUG:boto:Params: {} 2022-09-03T07:25:58.716 DEBUG:boto:Token: None 2022-09-03T07:25:58.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YUqoqtTA89n23Rsz7SR/HdzLVls= 2022-09-03T07:25:58.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YUqoqtTA89n23Rsz7SR/HdzLVls=', 'Content-Length': '0'} 2022-09-03T07:25:58.721 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034683c2cfced250f-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.763 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.764 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.765 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.765 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.766 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.766 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.767 DEBUG:boto:Data: 2022-09-03T07:25:58.767 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.768 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.769 DEBUG:boto:Params: {} 2022-09-03T07:25:58.770 DEBUG:boto:Token: None 2022-09-03T07:25:58.771 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QtQHQmBK380vuDxr8ZzVZ7sgqYs= 2022-09-03T07:25:58.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QtQHQmBK380vuDxr8ZzVZ7sgqYs=', 'Content-Length': '0'} 2022-09-03T07:25:58.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036ff621c15076bcd-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.775 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.775 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.776 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.776 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.777 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.778 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.778 DEBUG:boto:Data: 2022-09-03T07:25:58.779 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.780 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.780 DEBUG:boto:Params: {} 2022-09-03T07:25:58.781 DEBUG:boto:Token: None 2022-09-03T07:25:58.782 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QtQHQmBK380vuDxr8ZzVZ7sgqYs= 2022-09-03T07:25:58.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QtQHQmBK380vuDxr8ZzVZ7sgqYs=', 'Content-Length': '0'} 2022-09-03T07:25:58.787 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035facfa472c1edb7-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.788 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:25:58.790 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.790 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.791 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.792 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.792 DEBUG:boto:Method: GET 2022-09-03T07:25:58.793 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.793 DEBUG:boto:Data: 2022-09-03T07:25:58.794 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.796 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.797 DEBUG:boto:Params: {} 2022-09-03T07:25:58.798 DEBUG:boto:Token: None 2022-09-03T07:25:58.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YUqoqtTA89n23Rsz7SR/HdzLVls= 2022-09-03T07:25:58.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YUqoqtTA89n23Rsz7SR/HdzLVls=', 'Content-Length': '0'} 2022-09-03T07:25:58.804 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006942f29a2f80a2ba-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.805 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.806 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.806 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.807 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.807 DEBUG:boto:Method: GET 2022-09-03T07:25:58.808 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.808 DEBUG:boto:Data: 2022-09-03T07:25:58.809 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.810 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.810 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.811 DEBUG:boto:Params: {} 2022-09-03T07:25:58.812 DEBUG:boto:Token: None 2022-09-03T07:25:58.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:25:58.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YUqoqtTA89n23Rsz7SR/HdzLVls= 2022-09-03T07:25:58.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YUqoqtTA89n23Rsz7SR/HdzLVls=', 'Content-Length': '0'} 2022-09-03T07:25:58.817 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d1c7b017d2fbfa1-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.819 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.819 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.820 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:58.821 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:58.822 DEBUG:boto:Method: GET 2022-09-03T07:25:58.822 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:25:58.823 DEBUG:boto:Data: 2022-09-03T07:25:58.824 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.824 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.825 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.825 DEBUG:boto:Params: {} 2022-09-03T07:25:58.826 DEBUG:boto:Token: None 2022-09-03T07:25:58.827 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.831 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088b4c42c84a5d221-0063130186-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.832 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.833 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.834 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.835 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:58.835 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:58.836 DEBUG:boto:Method: GET 2022-09-03T07:25:58.836 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:25:58.837 DEBUG:boto:Data: 2022-09-03T07:25:58.837 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.838 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.838 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.839 DEBUG:boto:Params: {} 2022-09-03T07:25:58.840 DEBUG:boto:Token: None 2022-09-03T07:25:58.840 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.845 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fcd8a7917463ba0c-0063130186-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.846 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.847 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.847 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.848 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.848 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:25:58.849 DEBUG:boto:Data: 2022-09-03T07:25:58.849 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.850 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.851 DEBUG:boto:Params: {} 2022-09-03T07:25:58.852 DEBUG:boto:Token: None 2022-09-03T07:25:58.852 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22 2022-09-03T07:25:58.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8zvs0MIFn8eijd8RtwOMLSVkxCc= 2022-09-03T07:25:58.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8zvs0MIFn8eijd8RtwOMLSVkxCc=', 'Content-Length': '0'} 2022-09-03T07:25:58.856 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc380ba096298dcb-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.856 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.857 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.857 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.858 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:25:58.859 DEBUG:boto:Data: 2022-09-03T07:25:58.859 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.860 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.860 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.861 DEBUG:boto:Params: {} 2022-09-03T07:25:58.861 DEBUG:boto:Token: None 2022-09-03T07:25:58.862 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22 2022-09-03T07:25:58.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8zvs0MIFn8eijd8RtwOMLSVkxCc= 2022-09-03T07:25:58.863 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8zvs0MIFn8eijd8RtwOMLSVkxCc=', 'Content-Length': '0'} 2022-09-03T07:25:58.866 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005b06d8c450e89a44-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.867 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:25:58.867 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.868 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.868 DEBUG:boto:Method: GET 2022-09-03T07:25:58.869 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:25:58.869 DEBUG:boto:Data: 2022-09-03T07:25:58.870 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.871 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.871 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.872 DEBUG:boto:Params: {} 2022-09-03T07:25:58.872 DEBUG:boto:Token: None 2022-09-03T07:25:58.873 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22 2022-09-03T07:25:58.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3/g/IxhKdhvJtbq7hK8eqBbBRVU= 2022-09-03T07:25:58.874 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3/g/IxhKdhvJtbq7hK8eqBbBRVU=', 'Content-Length': '0'} 2022-09-03T07:25:58.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000787d1c94d9cc45e7-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.878 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:25:58.878 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:25:58.879 DEBUG:boto:Method: GET 2022-09-03T07:25:58.879 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:25:58.880 DEBUG:boto:Data: 2022-09-03T07:25:58.881 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.882 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.882 DEBUG:boto:Params: {} 2022-09-03T07:25:58.883 DEBUG:boto:Token: None 2022-09-03T07:25:58.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj22 2022-09-03T07:25:58.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3/g/IxhKdhvJtbq7hK8eqBbBRVU= 2022-09-03T07:25:58.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3/g/IxhKdhvJtbq7hK8eqBbBRVU=', 'Content-Length': '0'} 2022-09-03T07:25:58.887 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000315b96d6a2aa136d-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.888 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.888 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.889 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:58.890 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:58.890 DEBUG:boto:Method: GET 2022-09-03T07:25:58.891 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:25:58.892 DEBUG:boto:Data: 2022-09-03T07:25:58.892 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.893 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.894 DEBUG:boto:Params: {} 2022-09-03T07:25:58.894 DEBUG:boto:Token: None 2022-09-03T07:25:58.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.899 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d89373200ea5376-0063130186-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.900 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.900 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:58.901 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:58.902 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:58.902 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:25:58.903 DEBUG:boto:Method: GET 2022-09-03T07:25:58.903 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:25:58.904 DEBUG:boto:Data: 2022-09-03T07:25:58.904 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.905 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.906 DEBUG:boto:Params: {} 2022-09-03T07:25:58.907 DEBUG:boto:Token: None 2022-09-03T07:25:58.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/ 2022-09-03T07:25:58.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk= 2022-09-03T07:25:58.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+KMI3+nd3k6uj/OjBjjqvPairk=', 'Content-Length': '0'} 2022-09-03T07:25:58.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008ab4f08a2f667fb5-0063130186-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.958 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:58.960 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:25:58.961 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:58.962 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:58.963 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.963 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.964 DEBUG:boto:Method: GET 2022-09-03T07:25:58.965 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.966 DEBUG:boto:Data: 2022-09-03T07:25:58.966 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.968 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:58.969 DEBUG:boto:Port: 8001 2022-09-03T07:25:58.970 DEBUG:boto:Params: {} 2022-09-03T07:25:58.971 DEBUG:boto:Token: None 2022-09-03T07:25:58.971 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:txBPuV53T02SayZEsrKDaU8dEHs= 2022-09-03T07:25:58.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:txBPuV53T02SayZEsrKDaU8dEHs=', 'Content-Length': '0'} 2022-09-03T07:25:58.977 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000458fa4ab9ccaaa05-0063130186-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.978 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:58.979 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:58.979 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.980 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.980 DEBUG:boto:Method: GET 2022-09-03T07:25:58.981 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.982 DEBUG:boto:Data: 2022-09-03T07:25:58.982 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:58.984 DEBUG:boto:Port: 8000 2022-09-03T07:25:58.985 DEBUG:boto:Params: {} 2022-09-03T07:25:58.987 DEBUG:boto:Token: None 2022-09-03T07:25:58.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:58 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:txBPuV53T02SayZEsrKDaU8dEHs= 2022-09-03T07:25:58.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:txBPuV53T02SayZEsrKDaU8dEHs=', 'Content-Length': '0'} 2022-09-03T07:25:58.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056370df4180f438f-0063130186-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:58.994 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:58.995 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:58.995 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.996 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.996 DEBUG:boto:Method: HEAD 2022-09-03T07:25:58.997 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:58.998 DEBUG:boto:Data: 2022-09-03T07:25:58.999 DEBUG:boto:Headers: {} 2022-09-03T07:25:58.999 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.000 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.001 DEBUG:boto:Params: {} 2022-09-03T07:25:59.002 DEBUG:boto:Token: None 2022-09-03T07:25:59.003 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xDDOi4P0+/zqaPNqWSeya/pCz4c= 2022-09-03T07:25:59.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xDDOi4P0+/zqaPNqWSeya/pCz4c=', 'Content-Length': '0'} 2022-09-03T07:25:59.006 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a953ce10d4b18f4-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.007 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:59.008 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:59.009 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.009 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.010 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.011 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.012 DEBUG:boto:Data: 2022-09-03T07:25:59.013 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.015 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.015 DEBUG:boto:Params: {} 2022-09-03T07:25:59.016 DEBUG:boto:Token: None 2022-09-03T07:25:59.017 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xDDOi4P0+/zqaPNqWSeya/pCz4c= 2022-09-03T07:25:59.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xDDOi4P0+/zqaPNqWSeya/pCz4c=', 'Content-Length': '0'} 2022-09-03T07:25:59.020 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa135dcc929b7e41-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.021 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:25:59.022 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:59.024 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:59.025 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.025 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.027 DEBUG:boto:Method: GET 2022-09-03T07:25:59.028 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.029 DEBUG:boto:Data: 2022-09-03T07:25:59.030 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.032 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.034 DEBUG:boto:Params: {} 2022-09-03T07:25:59.035 DEBUG:boto:Token: None 2022-09-03T07:25:59.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x14eN1QR3inhjGhfKfX8JkqxMmk= 2022-09-03T07:25:59.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x14eN1QR3inhjGhfKfX8JkqxMmk=', 'Content-Length': '0'} 2022-09-03T07:25:59.041 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e18254d2e11e1d5-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.042 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:59.043 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:59.044 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.045 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.046 DEBUG:boto:Method: GET 2022-09-03T07:25:59.047 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.048 DEBUG:boto:Data: 2022-09-03T07:25:59.049 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.051 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.052 DEBUG:boto:Params: {} 2022-09-03T07:25:59.053 DEBUG:boto:Token: None 2022-09-03T07:25:59.054 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:25:59.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x14eN1QR3inhjGhfKfX8JkqxMmk= 2022-09-03T07:25:59.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x14eN1QR3inhjGhfKfX8JkqxMmk=', 'Content-Length': '0'} 2022-09-03T07:25:59.059 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ecb740b43333820-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.061 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.062 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.063 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:59.063 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:59.064 DEBUG:boto:Method: GET 2022-09-03T07:25:59.065 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:25:59.066 DEBUG:boto:Data: 2022-09-03T07:25:59.067 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.068 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.069 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.070 DEBUG:boto:Params: {} 2022-09-03T07:25:59.071 DEBUG:boto:Token: None 2022-09-03T07:25:59.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.074 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.078 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a59ee8b63121c91-0063130187-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.079 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.081 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.082 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.083 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:59.084 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:59.085 DEBUG:boto:Method: GET 2022-09-03T07:25:59.086 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:25:59.087 DEBUG:boto:Data: 2022-09-03T07:25:59.088 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.089 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.090 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.091 DEBUG:boto:Params: {} 2022-09-03T07:25:59.092 DEBUG:boto:Token: None 2022-09-03T07:25:59.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.100 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000404e3c461f7686ca-0063130187-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.101 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.102 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:59.102 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:59.103 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.103 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:25:59.104 DEBUG:boto:Data: 2022-09-03T07:25:59.104 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.105 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.106 DEBUG:boto:Params: {} 2022-09-03T07:25:59.106 DEBUG:boto:Token: None 2022-09-03T07:25:59.107 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj23 2022-09-03T07:25:59.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kcy0zi40rMDp0TkMVDgfkgZFxCw= 2022-09-03T07:25:59.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kcy0zi40rMDp0TkMVDgfkgZFxCw=', 'Content-Length': '0'} 2022-09-03T07:25:59.110 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000746d0e8f75fb63a5-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.111 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:59.111 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:59.112 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.112 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:25:59.113 DEBUG:boto:Data: 2022-09-03T07:25:59.113 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.114 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.115 DEBUG:boto:Params: {} 2022-09-03T07:25:59.116 DEBUG:boto:Token: None 2022-09-03T07:25:59.116 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj23 2022-09-03T07:25:59.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kcy0zi40rMDp0TkMVDgfkgZFxCw= 2022-09-03T07:25:59.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kcy0zi40rMDp0TkMVDgfkgZFxCw=', 'Content-Length': '0'} 2022-09-03T07:25:59.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc16375abcc36604-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.120 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:25:59.120 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:59.121 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:59.122 DEBUG:boto:Method: GET 2022-09-03T07:25:59.122 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:25:59.123 DEBUG:boto:Data: 2022-09-03T07:25:59.123 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.124 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.124 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.125 DEBUG:boto:Params: {} 2022-09-03T07:25:59.126 DEBUG:boto:Token: None 2022-09-03T07:25:59.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj23 2022-09-03T07:25:59.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Bmal4hhZee4G+O9hKPlxyb1fOc= 2022-09-03T07:25:59.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Bmal4hhZee4G+O9hKPlxyb1fOc=', 'Content-Length': '0'} 2022-09-03T07:25:59.129 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8fea7ae296d3cd4-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.174 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:25:59.175 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:25:59.176 DEBUG:boto:Method: GET 2022-09-03T07:25:59.176 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:25:59.177 DEBUG:boto:Data: 2022-09-03T07:25:59.177 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.178 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.179 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.179 DEBUG:boto:Params: {} 2022-09-03T07:25:59.180 DEBUG:boto:Token: None 2022-09-03T07:25:59.180 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj23 2022-09-03T07:25:59.181 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Bmal4hhZee4G+O9hKPlxyb1fOc= 2022-09-03T07:25:59.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Bmal4hhZee4G+O9hKPlxyb1fOc=', 'Content-Length': '0'} 2022-09-03T07:25:59.184 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002fb14b4882c690ac-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.185 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.186 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.186 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:59.187 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:59.187 DEBUG:boto:Method: GET 2022-09-03T07:25:59.188 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:25:59.188 DEBUG:boto:Data: 2022-09-03T07:25:59.189 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.189 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.190 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.190 DEBUG:boto:Params: {} 2022-09-03T07:25:59.191 DEBUG:boto:Token: None 2022-09-03T07:25:59.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.195 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db7f010b7042a175-0063130187-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.196 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.197 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.198 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.198 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:59.199 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:25:59.199 DEBUG:boto:Method: GET 2022-09-03T07:25:59.200 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:25:59.200 DEBUG:boto:Data: 2022-09-03T07:25:59.201 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.202 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.202 DEBUG:boto:Params: {} 2022-09-03T07:25:59.203 DEBUG:boto:Token: None 2022-09-03T07:25:59.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eea43fb30816c9a9-0063130187-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.209 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.210 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:25:59.211 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.211 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.212 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.213 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.213 DEBUG:boto:Method: GET 2022-09-03T07:25:59.214 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.214 DEBUG:boto:Data: 2022-09-03T07:25:59.215 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.216 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.216 DEBUG:boto:Params: {} 2022-09-03T07:25:59.217 DEBUG:boto:Token: None 2022-09-03T07:25:59.218 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4FCybQmQDFoIYXlvvPblyeud45c= 2022-09-03T07:25:59.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4FCybQmQDFoIYXlvvPblyeud45c=', 'Content-Length': '0'} 2022-09-03T07:25:59.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e2a69708c38c543-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.222 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.223 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.224 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.224 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.225 DEBUG:boto:Method: GET 2022-09-03T07:25:59.225 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.226 DEBUG:boto:Data: 2022-09-03T07:25:59.226 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.227 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.228 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.228 DEBUG:boto:Params: {} 2022-09-03T07:25:59.229 DEBUG:boto:Token: None 2022-09-03T07:25:59.229 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4FCybQmQDFoIYXlvvPblyeud45c= 2022-09-03T07:25:59.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4FCybQmQDFoIYXlvvPblyeud45c=', 'Content-Length': '0'} 2022-09-03T07:25:59.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004369522de1805564-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.234 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.234 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.235 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.236 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.237 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.237 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.238 DEBUG:boto:Data: 2022-09-03T07:25:59.239 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.240 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.240 DEBUG:boto:Params: {} 2022-09-03T07:25:59.241 DEBUG:boto:Token: None 2022-09-03T07:25:59.241 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MkmiEOHugVKTV7JX1oNKamqF0ns= 2022-09-03T07:25:59.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MkmiEOHugVKTV7JX1oNKamqF0ns=', 'Content-Length': '0'} 2022-09-03T07:25:59.245 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000400a5e5cd149e485-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.246 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.247 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.247 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.248 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.249 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.250 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.251 DEBUG:boto:Data: 2022-09-03T07:25:59.252 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.252 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.253 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.254 DEBUG:boto:Params: {} 2022-09-03T07:25:59.255 DEBUG:boto:Token: None 2022-09-03T07:25:59.256 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MkmiEOHugVKTV7JX1oNKamqF0ns= 2022-09-03T07:25:59.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MkmiEOHugVKTV7JX1oNKamqF0ns=', 'Content-Length': '0'} 2022-09-03T07:25:59.261 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c674ddd52b59bc8-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.261 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:25:59.262 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.263 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.263 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.264 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.266 DEBUG:boto:Method: GET 2022-09-03T07:25:59.267 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.268 DEBUG:boto:Data: 2022-09-03T07:25:59.269 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.272 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.272 DEBUG:boto:Params: {} 2022-09-03T07:25:59.273 DEBUG:boto:Token: None 2022-09-03T07:25:59.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4FCybQmQDFoIYXlvvPblyeud45c= 2022-09-03T07:25:59.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4FCybQmQDFoIYXlvvPblyeud45c=', 'Content-Length': '0'} 2022-09-03T07:25:59.277 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb3dd78f453291d3-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.278 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.279 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.279 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.280 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.280 DEBUG:boto:Method: GET 2022-09-03T07:25:59.281 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.281 DEBUG:boto:Data: 2022-09-03T07:25:59.282 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.283 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.284 DEBUG:boto:Params: {} 2022-09-03T07:25:59.284 DEBUG:boto:Token: None 2022-09-03T07:25:59.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:25:59.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4FCybQmQDFoIYXlvvPblyeud45c= 2022-09-03T07:25:59.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4FCybQmQDFoIYXlvvPblyeud45c=', 'Content-Length': '0'} 2022-09-03T07:25:59.288 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c15a7339ad1f11f0-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.289 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.300 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.301 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:59.301 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:59.302 DEBUG:boto:Method: GET 2022-09-03T07:25:59.302 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:25:59.303 DEBUG:boto:Data: 2022-09-03T07:25:59.303 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.304 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.304 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.305 DEBUG:boto:Params: {} 2022-09-03T07:25:59.306 DEBUG:boto:Token: None 2022-09-03T07:25:59.306 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.307 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.310 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad9c7e40c3da3f69-0063130187-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.311 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.311 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.312 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.312 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:59.313 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:59.313 DEBUG:boto:Method: GET 2022-09-03T07:25:59.314 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:25:59.315 DEBUG:boto:Data: 2022-09-03T07:25:59.315 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.316 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.316 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.317 DEBUG:boto:Params: {} 2022-09-03T07:25:59.317 DEBUG:boto:Token: None 2022-09-03T07:25:59.318 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.323 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000779e547298fcc389-0063130187-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.324 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.324 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.325 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.326 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.326 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:25:59.327 DEBUG:boto:Data: 2022-09-03T07:25:59.327 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.328 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.328 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.329 DEBUG:boto:Params: {} 2022-09-03T07:25:59.329 DEBUG:boto:Token: None 2022-09-03T07:25:59.330 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29 2022-09-03T07:25:59.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vaDzJkDuQrTzqAPlfIdmU2CS0A8= 2022-09-03T07:25:59.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vaDzJkDuQrTzqAPlfIdmU2CS0A8=', 'Content-Length': '0'} 2022-09-03T07:25:59.333 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f53c27822b9f00e8-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.335 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.336 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.337 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.338 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:25:59.339 DEBUG:boto:Data: 2022-09-03T07:25:59.339 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.341 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.341 DEBUG:boto:Params: {} 2022-09-03T07:25:59.342 DEBUG:boto:Token: None 2022-09-03T07:25:59.343 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29 2022-09-03T07:25:59.343 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vaDzJkDuQrTzqAPlfIdmU2CS0A8= 2022-09-03T07:25:59.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vaDzJkDuQrTzqAPlfIdmU2CS0A8=', 'Content-Length': '0'} 2022-09-03T07:25:59.346 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e755ca01c54b1da-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.347 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:25:59.347 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.348 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.348 DEBUG:boto:Method: GET 2022-09-03T07:25:59.349 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:25:59.350 DEBUG:boto:Data: 2022-09-03T07:25:59.350 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.352 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.352 DEBUG:boto:Params: {} 2022-09-03T07:25:59.353 DEBUG:boto:Token: None 2022-09-03T07:25:59.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29 2022-09-03T07:25:59.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8F9LtAOH6bOJqZleONJi7z5Pos= 2022-09-03T07:25:59.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8F9LtAOH6bOJqZleONJi7z5Pos=', 'Content-Length': '0'} 2022-09-03T07:25:59.359 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004927962be74b3e5a-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.402 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:25:59.403 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:25:59.404 DEBUG:boto:Method: GET 2022-09-03T07:25:59.405 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:25:59.405 DEBUG:boto:Data: 2022-09-03T07:25:59.406 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.407 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.408 DEBUG:boto:Params: {} 2022-09-03T07:25:59.409 DEBUG:boto:Token: None 2022-09-03T07:25:59.410 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj29 2022-09-03T07:25:59.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8F9LtAOH6bOJqZleONJi7z5Pos= 2022-09-03T07:25:59.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8F9LtAOH6bOJqZleONJi7z5Pos=', 'Content-Length': '0'} 2022-09-03T07:25:59.413 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015b6e06edbb60502-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.414 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.414 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.415 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:59.415 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:59.416 DEBUG:boto:Method: GET 2022-09-03T07:25:59.416 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:25:59.417 DEBUG:boto:Data: 2022-09-03T07:25:59.418 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.418 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.419 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.419 DEBUG:boto:Params: {} 2022-09-03T07:25:59.420 DEBUG:boto:Token: None 2022-09-03T07:25:59.421 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.424 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090c386c2e67d1d66-0063130187-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.425 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.426 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.427 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.427 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:59.428 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:25:59.428 DEBUG:boto:Method: GET 2022-09-03T07:25:59.429 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:25:59.430 DEBUG:boto:Data: 2022-09-03T07:25:59.430 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.431 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.432 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.432 DEBUG:boto:Params: {} 2022-09-03T07:25:59.433 DEBUG:boto:Token: None 2022-09-03T07:25:59.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.439 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087d9f0a84b27df73-0063130187-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.439 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.440 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:25:59.441 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.442 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.442 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.443 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.443 DEBUG:boto:Method: GET 2022-09-03T07:25:59.444 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.444 DEBUG:boto:Data: 2022-09-03T07:25:59.445 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.446 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.447 DEBUG:boto:Params: {} 2022-09-03T07:25:59.447 DEBUG:boto:Token: None 2022-09-03T07:25:59.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iHkCfwQoF0ahliySctlOh0Xf9jw= 2022-09-03T07:25:59.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iHkCfwQoF0ahliySctlOh0Xf9jw=', 'Content-Length': '0'} 2022-09-03T07:25:59.452 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c6b216f4113cdad4-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.454 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.455 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.455 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.456 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.457 DEBUG:boto:Method: GET 2022-09-03T07:25:59.458 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.459 DEBUG:boto:Data: 2022-09-03T07:25:59.460 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.462 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.462 DEBUG:boto:Params: {} 2022-09-03T07:25:59.463 DEBUG:boto:Token: None 2022-09-03T07:25:59.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iHkCfwQoF0ahliySctlOh0Xf9jw= 2022-09-03T07:25:59.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iHkCfwQoF0ahliySctlOh0Xf9jw=', 'Content-Length': '0'} 2022-09-03T07:25:59.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aac429643de4df41-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.468 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.469 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.470 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.470 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.471 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.471 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.472 DEBUG:boto:Data: 2022-09-03T07:25:59.472 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.473 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.474 DEBUG:boto:Params: {} 2022-09-03T07:25:59.474 DEBUG:boto:Token: None 2022-09-03T07:25:59.476 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aE3KxJy8s2YMTgHkJuwy5UO0g3A= 2022-09-03T07:25:59.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aE3KxJy8s2YMTgHkJuwy5UO0g3A=', 'Content-Length': '0'} 2022-09-03T07:25:59.481 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008da606cce77b3b18-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.482 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.483 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.484 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.485 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.486 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.487 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.487 DEBUG:boto:Data: 2022-09-03T07:25:59.488 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.490 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.490 DEBUG:boto:Params: {} 2022-09-03T07:25:59.491 DEBUG:boto:Token: None 2022-09-03T07:25:59.492 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aE3KxJy8s2YMTgHkJuwy5UO0g3A= 2022-09-03T07:25:59.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aE3KxJy8s2YMTgHkJuwy5UO0g3A=', 'Content-Length': '0'} 2022-09-03T07:25:59.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a26a0e0b37da63f6-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.496 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:25:59.497 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.497 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.498 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.498 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.499 DEBUG:boto:Method: GET 2022-09-03T07:25:59.499 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.500 DEBUG:boto:Data: 2022-09-03T07:25:59.501 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.501 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.502 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.502 DEBUG:boto:Params: {} 2022-09-03T07:25:59.503 DEBUG:boto:Token: None 2022-09-03T07:25:59.504 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iHkCfwQoF0ahliySctlOh0Xf9jw= 2022-09-03T07:25:59.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iHkCfwQoF0ahliySctlOh0Xf9jw=', 'Content-Length': '0'} 2022-09-03T07:25:59.507 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad63a8e79e400875-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.508 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.509 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.509 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.510 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.510 DEBUG:boto:Method: GET 2022-09-03T07:25:59.511 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.511 DEBUG:boto:Data: 2022-09-03T07:25:59.512 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.513 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.514 DEBUG:boto:Params: {} 2022-09-03T07:25:59.515 DEBUG:boto:Token: None 2022-09-03T07:25:59.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:25:59.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iHkCfwQoF0ahliySctlOh0Xf9jw= 2022-09-03T07:25:59.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iHkCfwQoF0ahliySctlOh0Xf9jw=', 'Content-Length': '0'} 2022-09-03T07:25:59.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f5c0c182acd70fe-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.525 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.525 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.526 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:59.526 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:59.527 DEBUG:boto:Method: GET 2022-09-03T07:25:59.528 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:25:59.528 DEBUG:boto:Data: 2022-09-03T07:25:59.529 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.530 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.530 DEBUG:boto:Params: {} 2022-09-03T07:25:59.531 DEBUG:boto:Token: None 2022-09-03T07:25:59.531 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.532 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.535 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3c7221f9e096552-0063130187-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.536 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.537 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.537 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.538 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:59.539 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:59.539 DEBUG:boto:Method: GET 2022-09-03T07:25:59.540 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:25:59.541 DEBUG:boto:Data: 2022-09-03T07:25:59.542 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.543 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.544 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.545 DEBUG:boto:Params: {} 2022-09-03T07:25:59.546 DEBUG:boto:Token: None 2022-09-03T07:25:59.547 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.548 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.553 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000053699b83a71c9aaa-0063130187-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.594 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.595 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.596 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.596 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.597 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:25:59.598 DEBUG:boto:Data: 2022-09-03T07:25:59.599 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.600 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.601 DEBUG:boto:Params: {} 2022-09-03T07:25:59.603 DEBUG:boto:Token: None 2022-09-03T07:25:59.604 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33 2022-09-03T07:25:59.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sn7dKgaSgBT5011VJshAQeRocMA= 2022-09-03T07:25:59.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sn7dKgaSgBT5011VJshAQeRocMA=', 'Content-Length': '0'} 2022-09-03T07:25:59.608 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000312d47b3e2c88bc5-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.609 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.610 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.611 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.612 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:25:59.613 DEBUG:boto:Data: 2022-09-03T07:25:59.614 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.616 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.617 DEBUG:boto:Params: {} 2022-09-03T07:25:59.618 DEBUG:boto:Token: None 2022-09-03T07:25:59.619 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33 2022-09-03T07:25:59.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sn7dKgaSgBT5011VJshAQeRocMA= 2022-09-03T07:25:59.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sn7dKgaSgBT5011VJshAQeRocMA=', 'Content-Length': '0'} 2022-09-03T07:25:59.623 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fcc7e5f95c97315-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.624 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:25:59.624 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.625 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.625 DEBUG:boto:Method: GET 2022-09-03T07:25:59.626 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:25:59.627 DEBUG:boto:Data: 2022-09-03T07:25:59.627 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.628 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.629 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.629 DEBUG:boto:Params: {} 2022-09-03T07:25:59.630 DEBUG:boto:Token: None 2022-09-03T07:25:59.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33 2022-09-03T07:25:59.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4fCiyRr5YEdWswE88r0yv0N8Mto= 2022-09-03T07:25:59.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4fCiyRr5YEdWswE88r0yv0N8Mto=', 'Content-Length': '0'} 2022-09-03T07:25:59.634 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080c2429f2040ee93-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.635 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:25:59.636 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:25:59.637 DEBUG:boto:Method: GET 2022-09-03T07:25:59.638 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:25:59.639 DEBUG:boto:Data: 2022-09-03T07:25:59.640 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.642 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.643 DEBUG:boto:Params: {} 2022-09-03T07:25:59.643 DEBUG:boto:Token: None 2022-09-03T07:25:59.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj33 2022-09-03T07:25:59.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4fCiyRr5YEdWswE88r0yv0N8Mto= 2022-09-03T07:25:59.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4fCiyRr5YEdWswE88r0yv0N8Mto=', 'Content-Length': '0'} 2022-09-03T07:25:59.650 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000274c32cfda078a15-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.651 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.652 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.653 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:59.654 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:59.655 DEBUG:boto:Method: GET 2022-09-03T07:25:59.656 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:25:59.656 DEBUG:boto:Data: 2022-09-03T07:25:59.657 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.658 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.658 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.659 DEBUG:boto:Params: {} 2022-09-03T07:25:59.659 DEBUG:boto:Token: None 2022-09-03T07:25:59.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.666 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075dc643758ff1fae-0063130187-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.667 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.668 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.668 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.669 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:59.669 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:25:59.670 DEBUG:boto:Method: GET 2022-09-03T07:25:59.670 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:25:59.671 DEBUG:boto:Data: 2022-09-03T07:25:59.672 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.673 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.673 DEBUG:boto:Params: {} 2022-09-03T07:25:59.674 DEBUG:boto:Token: None 2022-09-03T07:25:59.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.679 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000073d60ad803703ba-0063130187-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.680 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.681 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:25:59.681 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.682 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.683 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.683 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.684 DEBUG:boto:Method: GET 2022-09-03T07:25:59.684 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.685 DEBUG:boto:Data: 2022-09-03T07:25:59.685 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.686 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.687 DEBUG:boto:Params: {} 2022-09-03T07:25:59.687 DEBUG:boto:Token: None 2022-09-03T07:25:59.688 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rpfWlecPqSNOrG9jwADIS7RB7T4= 2022-09-03T07:25:59.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rpfWlecPqSNOrG9jwADIS7RB7T4=', 'Content-Length': '0'} 2022-09-03T07:25:59.691 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5578e4a9922fe6a-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.692 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.693 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.693 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.694 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.694 DEBUG:boto:Method: GET 2022-09-03T07:25:59.695 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.695 DEBUG:boto:Data: 2022-09-03T07:25:59.696 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.696 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.697 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.698 DEBUG:boto:Params: {} 2022-09-03T07:25:59.698 DEBUG:boto:Token: None 2022-09-03T07:25:59.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.699 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rpfWlecPqSNOrG9jwADIS7RB7T4= 2022-09-03T07:25:59.700 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rpfWlecPqSNOrG9jwADIS7RB7T4=', 'Content-Length': '0'} 2022-09-03T07:25:59.702 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c23f50fb0cc1314a-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.703 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.703 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.704 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.704 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.705 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.705 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.706 DEBUG:boto:Data: 2022-09-03T07:25:59.706 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.707 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.708 DEBUG:boto:Params: {} 2022-09-03T07:25:59.708 DEBUG:boto:Token: None 2022-09-03T07:25:59.709 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.710 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IH8Z7yJQCvRxddg5GMRCTkxk0B4= 2022-09-03T07:25:59.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IH8Z7yJQCvRxddg5GMRCTkxk0B4=', 'Content-Length': '0'} 2022-09-03T07:25:59.712 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000458d88e007f57eb0-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.713 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.713 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.714 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.714 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.715 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.715 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.716 DEBUG:boto:Data: 2022-09-03T07:25:59.716 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.718 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.718 DEBUG:boto:Params: {} 2022-09-03T07:25:59.719 DEBUG:boto:Token: None 2022-09-03T07:25:59.720 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IH8Z7yJQCvRxddg5GMRCTkxk0B4= 2022-09-03T07:25:59.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IH8Z7yJQCvRxddg5GMRCTkxk0B4=', 'Content-Length': '0'} 2022-09-03T07:25:59.723 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f55c13770abaf85-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.724 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:25:59.725 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.725 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.726 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.726 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.727 DEBUG:boto:Method: GET 2022-09-03T07:25:59.728 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.728 DEBUG:boto:Data: 2022-09-03T07:25:59.729 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.730 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.731 DEBUG:boto:Params: {} 2022-09-03T07:25:59.731 DEBUG:boto:Token: None 2022-09-03T07:25:59.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rpfWlecPqSNOrG9jwADIS7RB7T4= 2022-09-03T07:25:59.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rpfWlecPqSNOrG9jwADIS7RB7T4=', 'Content-Length': '0'} 2022-09-03T07:25:59.735 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b8e27ed35887e41-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.736 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.737 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.738 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.738 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.739 DEBUG:boto:Method: GET 2022-09-03T07:25:59.740 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.740 DEBUG:boto:Data: 2022-09-03T07:25:59.741 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.741 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.742 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.743 DEBUG:boto:Params: {} 2022-09-03T07:25:59.743 DEBUG:boto:Token: None 2022-09-03T07:25:59.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:25:59.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rpfWlecPqSNOrG9jwADIS7RB7T4= 2022-09-03T07:25:59.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rpfWlecPqSNOrG9jwADIS7RB7T4=', 'Content-Length': '0'} 2022-09-03T07:25:59.748 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012add03cd7cdbd71-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.790 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.791 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.792 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:59.792 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:59.793 DEBUG:boto:Method: GET 2022-09-03T07:25:59.794 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:25:59.794 DEBUG:boto:Data: 2022-09-03T07:25:59.795 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.795 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.796 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.796 DEBUG:boto:Params: {} 2022-09-03T07:25:59.797 DEBUG:boto:Token: None 2022-09-03T07:25:59.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.802 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037b18a89733724ab-0063130187-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.803 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.806 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.807 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.808 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:59.809 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:59.810 DEBUG:boto:Method: GET 2022-09-03T07:25:59.811 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:25:59.811 DEBUG:boto:Data: 2022-09-03T07:25:59.812 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.814 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.815 DEBUG:boto:Params: {} 2022-09-03T07:25:59.817 DEBUG:boto:Token: None 2022-09-03T07:25:59.818 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.824 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000153f1a43eeaebb19-0063130187-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.825 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.827 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.828 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.829 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.830 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:25:59.831 DEBUG:boto:Data: 2022-09-03T07:25:59.832 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.833 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.833 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.834 DEBUG:boto:Params: {} 2022-09-03T07:25:59.835 DEBUG:boto:Token: None 2022-09-03T07:25:59.836 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34 2022-09-03T07:25:59.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gAg3rebKtTfqcNR36C6CQZ5HsE0= 2022-09-03T07:25:59.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gAg3rebKtTfqcNR36C6CQZ5HsE0=', 'Content-Length': '0'} 2022-09-03T07:25:59.840 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074c0297b14c7ae8d-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.841 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.842 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.842 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.843 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:25:59.844 DEBUG:boto:Data: 2022-09-03T07:25:59.845 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.847 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.848 DEBUG:boto:Params: {} 2022-09-03T07:25:59.849 DEBUG:boto:Token: None 2022-09-03T07:25:59.850 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34 2022-09-03T07:25:59.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gAg3rebKtTfqcNR36C6CQZ5HsE0= 2022-09-03T07:25:59.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gAg3rebKtTfqcNR36C6CQZ5HsE0=', 'Content-Length': '0'} 2022-09-03T07:25:59.853 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c1a78a7eb659982-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.855 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:25:59.856 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.857 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.859 DEBUG:boto:Method: GET 2022-09-03T07:25:59.860 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:25:59.861 DEBUG:boto:Data: 2022-09-03T07:25:59.863 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.865 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.867 DEBUG:boto:Params: {} 2022-09-03T07:25:59.868 DEBUG:boto:Token: None 2022-09-03T07:25:59.869 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34 2022-09-03T07:25:59.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:d7JSg0rb+dDzDZ7Uobcb3nvQrkc= 2022-09-03T07:25:59.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:d7JSg0rb+dDzDZ7Uobcb3nvQrkc=', 'Content-Length': '0'} 2022-09-03T07:25:59.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000640edd1c87732e9d-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.874 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:25:59.875 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:25:59.876 DEBUG:boto:Method: GET 2022-09-03T07:25:59.876 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:25:59.877 DEBUG:boto:Data: 2022-09-03T07:25:59.877 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.879 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.879 DEBUG:boto:Params: {} 2022-09-03T07:25:59.880 DEBUG:boto:Token: None 2022-09-03T07:25:59.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj34 2022-09-03T07:25:59.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:d7JSg0rb+dDzDZ7Uobcb3nvQrkc= 2022-09-03T07:25:59.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:d7JSg0rb+dDzDZ7Uobcb3nvQrkc=', 'Content-Length': '0'} 2022-09-03T07:25:59.884 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078942d25adf142e9-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.885 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.886 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.887 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:59.888 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:59.889 DEBUG:boto:Method: GET 2022-09-03T07:25:59.890 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:25:59.891 DEBUG:boto:Data: 2022-09-03T07:25:59.892 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.893 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.894 DEBUG:boto:Params: {} 2022-09-03T07:25:59.894 DEBUG:boto:Token: None 2022-09-03T07:25:59.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.899 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff553882ea9136e8-0063130187-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.900 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.901 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:25:59.902 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:25:59.903 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:59.903 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:25:59.904 DEBUG:boto:Method: GET 2022-09-03T07:25:59.905 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:25:59.905 DEBUG:boto:Data: 2022-09-03T07:25:59.906 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.907 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.908 DEBUG:boto:Params: {} 2022-09-03T07:25:59.909 DEBUG:boto:Token: None 2022-09-03T07:25:59.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/ 2022-09-03T07:25:59.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg= 2022-09-03T07:25:59.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44K+TTgOVLlqXJjbr8yqgeD2SKg=', 'Content-Length': '0'} 2022-09-03T07:25:59.915 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006394274118eb4a2c-0063130187-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.915 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:25:59.917 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:25:59.917 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:59.918 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:59.919 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.919 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.920 DEBUG:boto:Method: GET 2022-09-03T07:25:59.920 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.921 DEBUG:boto:Data: 2022-09-03T07:25:59.922 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.923 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.924 DEBUG:boto:Params: {} 2022-09-03T07:25:59.924 DEBUG:boto:Token: None 2022-09-03T07:25:59.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HDedJQ1Br1I3FZhjfXebaQKfYSQ= 2022-09-03T07:25:59.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HDedJQ1Br1I3FZhjfXebaQKfYSQ=', 'Content-Length': '0'} 2022-09-03T07:25:59.929 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f6f6151f4ff692f-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.970 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:59.971 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:59.972 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.972 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.973 DEBUG:boto:Method: GET 2022-09-03T07:25:59.974 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.975 DEBUG:boto:Data: 2022-09-03T07:25:59.976 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.976 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:25:59.977 DEBUG:boto:Port: 8000 2022-09-03T07:25:59.978 DEBUG:boto:Params: {} 2022-09-03T07:25:59.979 DEBUG:boto:Token: None 2022-09-03T07:25:59.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HDedJQ1Br1I3FZhjfXebaQKfYSQ= 2022-09-03T07:25:59.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HDedJQ1Br1I3FZhjfXebaQKfYSQ=', 'Content-Length': '0'} 2022-09-03T07:25:59.985 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000860b4acc068c5801-0063130187-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.986 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:59.986 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:25:59.987 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.988 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.988 DEBUG:boto:Method: HEAD 2022-09-03T07:25:59.989 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.990 DEBUG:boto:Data: 2022-09-03T07:25:59.991 DEBUG:boto:Headers: {} 2022-09-03T07:25:59.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:25:59.992 DEBUG:boto:Port: 8001 2022-09-03T07:25:59.992 DEBUG:boto:Params: {} 2022-09-03T07:25:59.993 DEBUG:boto:Token: None 2022-09-03T07:25:59.994 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:25:59 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:25:59.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dzK/0FI/M2sh3AMWNG8FN+wVVF8= 2022-09-03T07:25:59.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:25:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dzK/0FI/M2sh3AMWNG8FN+wVVF8=', 'Content-Length': '0'} 2022-09-03T07:25:59.997 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f1c36f4ee49b091-0063130187-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:25:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:25:59.998 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:25:59.999 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:00.000 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.001 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.001 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.002 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.003 DEBUG:boto:Data: 2022-09-03T07:26:00.004 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.005 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.006 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.007 DEBUG:boto:Params: {} 2022-09-03T07:26:00.009 DEBUG:boto:Token: None 2022-09-03T07:26:00.010 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:STHih2yFizQsMFaXmVlYaZaljnY= 2022-09-03T07:26:00.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:STHih2yFizQsMFaXmVlYaZaljnY=', 'Content-Length': '0'} 2022-09-03T07:26:00.014 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000023c7cee996a6c62-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.015 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:26:00.015 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:00.016 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:00.016 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.017 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.017 DEBUG:boto:Method: GET 2022-09-03T07:26:00.018 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.019 DEBUG:boto:Data: 2022-09-03T07:26:00.019 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.021 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.022 DEBUG:boto:Params: {} 2022-09-03T07:26:00.023 DEBUG:boto:Token: None 2022-09-03T07:26:00.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6VED4Vr8QK0+mGfBO54EhJ9YpT8= 2022-09-03T07:26:00.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6VED4Vr8QK0+mGfBO54EhJ9YpT8=', 'Content-Length': '0'} 2022-09-03T07:26:00.028 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f02797a3ae234f6-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.029 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:00.030 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:00.031 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.031 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.032 DEBUG:boto:Method: GET 2022-09-03T07:26:00.033 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.034 DEBUG:boto:Data: 2022-09-03T07:26:00.035 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.037 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.037 DEBUG:boto:Params: {} 2022-09-03T07:26:00.038 DEBUG:boto:Token: None 2022-09-03T07:26:00.039 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:26:00.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6VED4Vr8QK0+mGfBO54EhJ9YpT8= 2022-09-03T07:26:00.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6VED4Vr8QK0+mGfBO54EhJ9YpT8=', 'Content-Length': '0'} 2022-09-03T07:26:00.042 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c97cdf8524801e6-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.043 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.043 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.044 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:00.044 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:00.045 DEBUG:boto:Method: GET 2022-09-03T07:26:00.045 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:26:00.046 DEBUG:boto:Data: 2022-09-03T07:26:00.046 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.047 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.048 DEBUG:boto:Params: {} 2022-09-03T07:26:00.049 DEBUG:boto:Token: None 2022-09-03T07:26:00.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.050 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.053 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003cadc223f4934bf1-0063130188-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.054 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.055 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.055 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.056 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:00.056 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:00.057 DEBUG:boto:Method: GET 2022-09-03T07:26:00.057 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:26:00.058 DEBUG:boto:Data: 2022-09-03T07:26:00.058 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.059 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.060 DEBUG:boto:Params: {} 2022-09-03T07:26:00.061 DEBUG:boto:Token: None 2022-09-03T07:26:00.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.066 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc8b8f3b2aa64c69-0063130188-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.067 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.068 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:00.068 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:00.069 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.069 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:26:00.070 DEBUG:boto:Data: 2022-09-03T07:26:00.070 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.071 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.071 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.072 DEBUG:boto:Params: {} 2022-09-03T07:26:00.072 DEBUG:boto:Token: None 2022-09-03T07:26:00.073 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj38 2022-09-03T07:26:00.074 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jVtsnz7Y/qBchDHIb/idUPEnYaU= 2022-09-03T07:26:00.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jVtsnz7Y/qBchDHIb/idUPEnYaU=', 'Content-Length': '0'} 2022-09-03T07:26:00.076 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5121bc7fb761652-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.077 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:00.078 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:00.078 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.079 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:26:00.079 DEBUG:boto:Data: 2022-09-03T07:26:00.080 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.080 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.081 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.081 DEBUG:boto:Params: {} 2022-09-03T07:26:00.082 DEBUG:boto:Token: None 2022-09-03T07:26:00.082 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj38 2022-09-03T07:26:00.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jVtsnz7Y/qBchDHIb/idUPEnYaU= 2022-09-03T07:26:00.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jVtsnz7Y/qBchDHIb/idUPEnYaU=', 'Content-Length': '0'} 2022-09-03T07:26:00.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048876e9444bd03c5-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.087 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:26:00.087 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:00.088 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:00.088 DEBUG:boto:Method: GET 2022-09-03T07:26:00.089 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:26:00.089 DEBUG:boto:Data: 2022-09-03T07:26:00.090 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.090 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.091 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.092 DEBUG:boto:Params: {} 2022-09-03T07:26:00.092 DEBUG:boto:Token: None 2022-09-03T07:26:00.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj38 2022-09-03T07:26:00.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c9TOYZ1i0A5B4FeHF4xBxXOlDvw= 2022-09-03T07:26:00.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c9TOYZ1i0A5B4FeHF4xBxXOlDvw=', 'Content-Length': '0'} 2022-09-03T07:26:00.096 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1ff8bcd986b4542-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.097 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:00.098 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:00.098 DEBUG:boto:Method: GET 2022-09-03T07:26:00.099 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:26:00.099 DEBUG:boto:Data: 2022-09-03T07:26:00.100 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.101 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.101 DEBUG:boto:Params: {} 2022-09-03T07:26:00.102 DEBUG:boto:Token: None 2022-09-03T07:26:00.103 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj38 2022-09-03T07:26:00.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c9TOYZ1i0A5B4FeHF4xBxXOlDvw= 2022-09-03T07:26:00.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c9TOYZ1i0A5B4FeHF4xBxXOlDvw=', 'Content-Length': '0'} 2022-09-03T07:26:00.106 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df32691fd06138ae-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.154 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.155 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.156 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:00.156 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:00.157 DEBUG:boto:Method: GET 2022-09-03T07:26:00.158 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:26:00.158 DEBUG:boto:Data: 2022-09-03T07:26:00.159 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.160 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.160 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.161 DEBUG:boto:Params: {} 2022-09-03T07:26:00.161 DEBUG:boto:Token: None 2022-09-03T07:26:00.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.168 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a210a7d95ecf5e1-0063130188-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.168 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.169 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.170 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.171 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:00.171 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:00.172 DEBUG:boto:Method: GET 2022-09-03T07:26:00.173 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:26:00.174 DEBUG:boto:Data: 2022-09-03T07:26:00.175 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.177 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.177 DEBUG:boto:Params: {} 2022-09-03T07:26:00.178 DEBUG:boto:Token: None 2022-09-03T07:26:00.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.185 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb9def4e0d1ee42e-0063130188-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.186 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.188 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:26:00.188 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.189 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.190 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.190 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.191 DEBUG:boto:Method: GET 2022-09-03T07:26:00.191 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.192 DEBUG:boto:Data: 2022-09-03T07:26:00.193 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.193 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.194 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.195 DEBUG:boto:Params: {} 2022-09-03T07:26:00.195 DEBUG:boto:Token: None 2022-09-03T07:26:00.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gLAvbSGpKSS1BPCkk1pAWYQgyrk= 2022-09-03T07:26:00.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gLAvbSGpKSS1BPCkk1pAWYQgyrk=', 'Content-Length': '0'} 2022-09-03T07:26:00.200 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c10b25c0818f09e-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.200 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.201 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.201 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.202 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.203 DEBUG:boto:Method: GET 2022-09-03T07:26:00.204 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.204 DEBUG:boto:Data: 2022-09-03T07:26:00.205 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.205 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.206 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.207 DEBUG:boto:Params: {} 2022-09-03T07:26:00.207 DEBUG:boto:Token: None 2022-09-03T07:26:00.208 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gLAvbSGpKSS1BPCkk1pAWYQgyrk= 2022-09-03T07:26:00.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gLAvbSGpKSS1BPCkk1pAWYQgyrk=', 'Content-Length': '0'} 2022-09-03T07:26:00.212 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df2708bbfac42fba-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.213 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.214 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.215 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.216 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.217 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.218 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.218 DEBUG:boto:Data: 2022-09-03T07:26:00.219 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.221 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.222 DEBUG:boto:Params: {} 2022-09-03T07:26:00.223 DEBUG:boto:Token: None 2022-09-03T07:26:00.224 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XgilI6xC+IeQC54iXlODZBvZRyk= 2022-09-03T07:26:00.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XgilI6xC+IeQC54iXlODZBvZRyk=', 'Content-Length': '0'} 2022-09-03T07:26:00.228 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8de4d05d1af729d-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.229 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.230 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.230 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.231 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.231 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.232 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.232 DEBUG:boto:Data: 2022-09-03T07:26:00.233 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.235 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.235 DEBUG:boto:Params: {} 2022-09-03T07:26:00.236 DEBUG:boto:Token: None 2022-09-03T07:26:00.236 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XgilI6xC+IeQC54iXlODZBvZRyk= 2022-09-03T07:26:00.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XgilI6xC+IeQC54iXlODZBvZRyk=', 'Content-Length': '0'} 2022-09-03T07:26:00.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c9043d5546ba43a-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.240 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:26:00.241 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.242 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.242 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.243 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.243 DEBUG:boto:Method: GET 2022-09-03T07:26:00.244 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.244 DEBUG:boto:Data: 2022-09-03T07:26:00.245 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.245 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.246 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.246 DEBUG:boto:Params: {} 2022-09-03T07:26:00.247 DEBUG:boto:Token: None 2022-09-03T07:26:00.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gLAvbSGpKSS1BPCkk1pAWYQgyrk= 2022-09-03T07:26:00.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gLAvbSGpKSS1BPCkk1pAWYQgyrk=', 'Content-Length': '0'} 2022-09-03T07:26:00.251 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060031017d2205804-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.251 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.253 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.254 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.254 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.255 DEBUG:boto:Method: GET 2022-09-03T07:26:00.256 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.256 DEBUG:boto:Data: 2022-09-03T07:26:00.257 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.257 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.258 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.259 DEBUG:boto:Params: {} 2022-09-03T07:26:00.259 DEBUG:boto:Token: None 2022-09-03T07:26:00.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:00.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gLAvbSGpKSS1BPCkk1pAWYQgyrk= 2022-09-03T07:26:00.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gLAvbSGpKSS1BPCkk1pAWYQgyrk=', 'Content-Length': '0'} 2022-09-03T07:26:00.263 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fec52455069818ae-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.264 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.265 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.265 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:00.266 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:00.266 DEBUG:boto:Method: GET 2022-09-03T07:26:00.267 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:26:00.268 DEBUG:boto:Data: 2022-09-03T07:26:00.268 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.269 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.269 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.270 DEBUG:boto:Params: {} 2022-09-03T07:26:00.271 DEBUG:boto:Token: None 2022-09-03T07:26:00.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.275 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b323df3a239bf9b4-0063130188-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.318 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.319 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.320 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.320 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:00.321 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:00.321 DEBUG:boto:Method: GET 2022-09-03T07:26:00.322 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:26:00.322 DEBUG:boto:Data: 2022-09-03T07:26:00.323 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.323 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.324 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.324 DEBUG:boto:Params: {} 2022-09-03T07:26:00.325 DEBUG:boto:Token: None 2022-09-03T07:26:00.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.331 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067fcdbe973a1c0e7-0063130188-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.332 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.333 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.333 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.334 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.334 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:26:00.335 DEBUG:boto:Data: 2022-09-03T07:26:00.335 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.336 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.337 DEBUG:boto:Params: {} 2022-09-03T07:26:00.338 DEBUG:boto:Token: None 2022-09-03T07:26:00.338 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44 2022-09-03T07:26:00.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8V0/7sNIQQtpZS7C3F4ESEovfcI= 2022-09-03T07:26:00.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8V0/7sNIQQtpZS7C3F4ESEovfcI=', 'Content-Length': '0'} 2022-09-03T07:26:00.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e118637c8bf93083-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.344 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.345 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.346 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.347 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:26:00.347 DEBUG:boto:Data: 2022-09-03T07:26:00.348 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.349 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.350 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.351 DEBUG:boto:Params: {} 2022-09-03T07:26:00.352 DEBUG:boto:Token: None 2022-09-03T07:26:00.353 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44 2022-09-03T07:26:00.353 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8V0/7sNIQQtpZS7C3F4ESEovfcI= 2022-09-03T07:26:00.354 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8V0/7sNIQQtpZS7C3F4ESEovfcI=', 'Content-Length': '0'} 2022-09-03T07:26:00.356 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010175367166c22ed-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.357 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:26:00.357 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.358 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.358 DEBUG:boto:Method: GET 2022-09-03T07:26:00.359 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:26:00.359 DEBUG:boto:Data: 2022-09-03T07:26:00.360 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.361 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.361 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.362 DEBUG:boto:Params: {} 2022-09-03T07:26:00.362 DEBUG:boto:Token: None 2022-09-03T07:26:00.363 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44 2022-09-03T07:26:00.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ftRM7K/IvC6qFES+6/+Fhgbu0+4= 2022-09-03T07:26:00.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ftRM7K/IvC6qFES+6/+Fhgbu0+4=', 'Content-Length': '0'} 2022-09-03T07:26:00.366 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009f82c87e086c330-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.367 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:00.368 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:00.369 DEBUG:boto:Method: GET 2022-09-03T07:26:00.370 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:26:00.371 DEBUG:boto:Data: 2022-09-03T07:26:00.372 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.373 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.374 DEBUG:boto:Params: {} 2022-09-03T07:26:00.374 DEBUG:boto:Token: None 2022-09-03T07:26:00.375 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj44 2022-09-03T07:26:00.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ftRM7K/IvC6qFES+6/+Fhgbu0+4= 2022-09-03T07:26:00.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ftRM7K/IvC6qFES+6/+Fhgbu0+4=', 'Content-Length': '0'} 2022-09-03T07:26:00.380 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d0035bbbd9084ce-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.381 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.382 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.382 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:00.383 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:00.383 DEBUG:boto:Method: GET 2022-09-03T07:26:00.384 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:26:00.385 DEBUG:boto:Data: 2022-09-03T07:26:00.385 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.386 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.387 DEBUG:boto:Params: {} 2022-09-03T07:26:00.388 DEBUG:boto:Token: None 2022-09-03T07:26:00.389 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.393 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000134af36e6e3c3ee1-0063130188-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.394 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.394 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.395 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.396 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:00.396 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:00.397 DEBUG:boto:Method: GET 2022-09-03T07:26:00.397 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:26:00.398 DEBUG:boto:Data: 2022-09-03T07:26:00.398 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.399 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.399 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.400 DEBUG:boto:Params: {} 2022-09-03T07:26:00.400 DEBUG:boto:Token: None 2022-09-03T07:26:00.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.406 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002bf9954eed8d463d-0063130188-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.407 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.408 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:26:00.408 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.409 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.409 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.410 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.411 DEBUG:boto:Method: GET 2022-09-03T07:26:00.411 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.412 DEBUG:boto:Data: 2022-09-03T07:26:00.412 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.413 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.413 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.414 DEBUG:boto:Params: {} 2022-09-03T07:26:00.414 DEBUG:boto:Token: None 2022-09-03T07:26:00.415 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXmfi8BnOMApuBQhb1CV3vtgPdY= 2022-09-03T07:26:00.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXmfi8BnOMApuBQhb1CV3vtgPdY=', 'Content-Length': '0'} 2022-09-03T07:26:00.418 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001445b54d07ed25b3-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.420 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.420 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.421 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.422 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.422 DEBUG:boto:Method: GET 2022-09-03T07:26:00.423 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.423 DEBUG:boto:Data: 2022-09-03T07:26:00.424 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.424 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.425 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.426 DEBUG:boto:Params: {} 2022-09-03T07:26:00.427 DEBUG:boto:Token: None 2022-09-03T07:26:00.427 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXmfi8BnOMApuBQhb1CV3vtgPdY= 2022-09-03T07:26:00.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXmfi8BnOMApuBQhb1CV3vtgPdY=', 'Content-Length': '0'} 2022-09-03T07:26:00.431 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074b31e6c645a10cf-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.475 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.476 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.477 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.478 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.479 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.480 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.481 DEBUG:boto:Data: 2022-09-03T07:26:00.482 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.484 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.485 DEBUG:boto:Params: {} 2022-09-03T07:26:00.487 DEBUG:boto:Token: None 2022-09-03T07:26:00.488 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D4YdLOq95GQ82KauRxEZXvDNl9Y= 2022-09-03T07:26:00.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D4YdLOq95GQ82KauRxEZXvDNl9Y=', 'Content-Length': '0'} 2022-09-03T07:26:00.493 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7bf60c50bd9919c-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.495 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.496 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.497 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.498 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.499 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.500 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.501 DEBUG:boto:Data: 2022-09-03T07:26:00.502 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.504 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.505 DEBUG:boto:Params: {} 2022-09-03T07:26:00.506 DEBUG:boto:Token: None 2022-09-03T07:26:00.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D4YdLOq95GQ82KauRxEZXvDNl9Y= 2022-09-03T07:26:00.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D4YdLOq95GQ82KauRxEZXvDNl9Y=', 'Content-Length': '0'} 2022-09-03T07:26:00.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000995662a80776e563-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.513 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:26:00.514 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.515 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.516 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.517 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.518 DEBUG:boto:Method: GET 2022-09-03T07:26:00.519 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.519 DEBUG:boto:Data: 2022-09-03T07:26:00.520 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.521 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.522 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.523 DEBUG:boto:Params: {} 2022-09-03T07:26:00.524 DEBUG:boto:Token: None 2022-09-03T07:26:00.524 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.525 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXmfi8BnOMApuBQhb1CV3vtgPdY= 2022-09-03T07:26:00.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXmfi8BnOMApuBQhb1CV3vtgPdY=', 'Content-Length': '0'} 2022-09-03T07:26:00.528 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c06cc60e48cdbfe9-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.529 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.529 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.530 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.530 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.531 DEBUG:boto:Method: GET 2022-09-03T07:26:00.531 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.532 DEBUG:boto:Data: 2022-09-03T07:26:00.532 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.533 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.534 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.534 DEBUG:boto:Params: {} 2022-09-03T07:26:00.535 DEBUG:boto:Token: None 2022-09-03T07:26:00.535 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:00.536 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wXmfi8BnOMApuBQhb1CV3vtgPdY= 2022-09-03T07:26:00.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wXmfi8BnOMApuBQhb1CV3vtgPdY=', 'Content-Length': '0'} 2022-09-03T07:26:00.538 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e87ea5a2037d7b1-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.539 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.540 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.540 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:00.541 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:00.541 DEBUG:boto:Method: GET 2022-09-03T07:26:00.542 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:26:00.542 DEBUG:boto:Data: 2022-09-03T07:26:00.543 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.543 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.544 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.544 DEBUG:boto:Params: {} 2022-09-03T07:26:00.545 DEBUG:boto:Token: None 2022-09-03T07:26:00.546 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.546 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.547 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.550 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001fc99e99dc7108c5-0063130188-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.550 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.551 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.552 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.552 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:00.553 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:00.554 DEBUG:boto:Method: GET 2022-09-03T07:26:00.554 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:26:00.555 DEBUG:boto:Data: 2022-09-03T07:26:00.555 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.556 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.557 DEBUG:boto:Params: {} 2022-09-03T07:26:00.557 DEBUG:boto:Token: None 2022-09-03T07:26:00.558 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.559 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.563 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003dbdce62797ad299-0063130188-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.564 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.565 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.565 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.566 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.566 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:26:00.567 DEBUG:boto:Data: 2022-09-03T07:26:00.567 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.568 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.569 DEBUG:boto:Params: {} 2022-09-03T07:26:00.570 DEBUG:boto:Token: None 2022-09-03T07:26:00.570 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46 2022-09-03T07:26:00.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dQxYQv9146pklnXLGWQPG10b2H8= 2022-09-03T07:26:00.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dQxYQv9146pklnXLGWQPG10b2H8=', 'Content-Length': '0'} 2022-09-03T07:26:00.573 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a903562790132de-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.574 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.574 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.575 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.576 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:26:00.576 DEBUG:boto:Data: 2022-09-03T07:26:00.576 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.577 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.578 DEBUG:boto:Params: {} 2022-09-03T07:26:00.579 DEBUG:boto:Token: None 2022-09-03T07:26:00.579 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46 2022-09-03T07:26:00.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dQxYQv9146pklnXLGWQPG10b2H8= 2022-09-03T07:26:00.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dQxYQv9146pklnXLGWQPG10b2H8=', 'Content-Length': '0'} 2022-09-03T07:26:00.582 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aafe181173c6a624-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.583 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:26:00.583 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.584 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.585 DEBUG:boto:Method: GET 2022-09-03T07:26:00.585 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:26:00.586 DEBUG:boto:Data: 2022-09-03T07:26:00.586 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.587 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.588 DEBUG:boto:Params: {} 2022-09-03T07:26:00.588 DEBUG:boto:Token: None 2022-09-03T07:26:00.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46 2022-09-03T07:26:00.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4LPgJEPbWS1BIG+ekYB2gpPlTOY= 2022-09-03T07:26:00.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4LPgJEPbWS1BIG+ekYB2gpPlTOY=', 'Content-Length': '0'} 2022-09-03T07:26:00.592 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a7b24a32b5aabc3-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.593 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:00.594 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:00.595 DEBUG:boto:Method: GET 2022-09-03T07:26:00.595 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:26:00.596 DEBUG:boto:Data: 2022-09-03T07:26:00.596 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.597 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.598 DEBUG:boto:Params: {} 2022-09-03T07:26:00.598 DEBUG:boto:Token: None 2022-09-03T07:26:00.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj46 2022-09-03T07:26:00.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4LPgJEPbWS1BIG+ekYB2gpPlTOY= 2022-09-03T07:26:00.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4LPgJEPbWS1BIG+ekYB2gpPlTOY=', 'Content-Length': '0'} 2022-09-03T07:26:00.602 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009cb005665da76f64-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.646 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.647 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.647 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:00.648 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:00.648 DEBUG:boto:Method: GET 2022-09-03T07:26:00.649 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:26:00.649 DEBUG:boto:Data: 2022-09-03T07:26:00.650 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.650 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.651 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.651 DEBUG:boto:Params: {} 2022-09-03T07:26:00.652 DEBUG:boto:Token: None 2022-09-03T07:26:00.653 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.656 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000863fe22cff36bd41-0063130188-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.657 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.658 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.658 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.659 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:00.659 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:00.660 DEBUG:boto:Method: GET 2022-09-03T07:26:00.660 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:26:00.661 DEBUG:boto:Data: 2022-09-03T07:26:00.661 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.662 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.663 DEBUG:boto:Params: {} 2022-09-03T07:26:00.663 DEBUG:boto:Token: None 2022-09-03T07:26:00.664 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.669 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f92ab5a661508758-0063130188-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.669 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.670 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:26:00.671 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.671 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.672 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.672 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.673 DEBUG:boto:Method: GET 2022-09-03T07:26:00.673 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.674 DEBUG:boto:Data: 2022-09-03T07:26:00.674 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.675 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.676 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.676 DEBUG:boto:Params: {} 2022-09-03T07:26:00.677 DEBUG:boto:Token: None 2022-09-03T07:26:00.677 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:STpgGqz8TnvNR+nv1XJQcprXZjQ= 2022-09-03T07:26:00.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:STpgGqz8TnvNR+nv1XJQcprXZjQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.680 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004388255d307aafbf-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.681 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.681 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.682 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.683 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.683 DEBUG:boto:Method: GET 2022-09-03T07:26:00.684 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.684 DEBUG:boto:Data: 2022-09-03T07:26:00.685 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.685 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.686 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.686 DEBUG:boto:Params: {} 2022-09-03T07:26:00.687 DEBUG:boto:Token: None 2022-09-03T07:26:00.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:STpgGqz8TnvNR+nv1XJQcprXZjQ= 2022-09-03T07:26:00.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:STpgGqz8TnvNR+nv1XJQcprXZjQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.690 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000059b77643d08e5e47-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.691 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.692 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.692 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.693 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.693 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.694 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.694 DEBUG:boto:Data: 2022-09-03T07:26:00.695 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.696 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.696 DEBUG:boto:Params: {} 2022-09-03T07:26:00.697 DEBUG:boto:Token: None 2022-09-03T07:26:00.697 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xaMGKk7JdxgOmrz3zAKfx5+Pq+0= 2022-09-03T07:26:00.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xaMGKk7JdxgOmrz3zAKfx5+Pq+0=', 'Content-Length': '0'} 2022-09-03T07:26:00.700 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000517483da2b280529-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.701 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.702 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.702 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.703 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.703 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.704 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.704 DEBUG:boto:Data: 2022-09-03T07:26:00.705 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.705 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.706 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.706 DEBUG:boto:Params: {} 2022-09-03T07:26:00.707 DEBUG:boto:Token: None 2022-09-03T07:26:00.707 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xaMGKk7JdxgOmrz3zAKfx5+Pq+0= 2022-09-03T07:26:00.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xaMGKk7JdxgOmrz3zAKfx5+Pq+0=', 'Content-Length': '0'} 2022-09-03T07:26:00.710 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d30778bc52f4c4f0-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.711 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:26:00.712 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.712 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.713 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.713 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.714 DEBUG:boto:Method: GET 2022-09-03T07:26:00.714 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.715 DEBUG:boto:Data: 2022-09-03T07:26:00.715 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.716 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.718 DEBUG:boto:Params: {} 2022-09-03T07:26:00.719 DEBUG:boto:Token: None 2022-09-03T07:26:00.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:STpgGqz8TnvNR+nv1XJQcprXZjQ= 2022-09-03T07:26:00.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:STpgGqz8TnvNR+nv1XJQcprXZjQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b672e80889f6344a-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.723 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.724 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.724 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.725 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.725 DEBUG:boto:Method: GET 2022-09-03T07:26:00.726 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.726 DEBUG:boto:Data: 2022-09-03T07:26:00.727 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.727 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.728 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.728 DEBUG:boto:Params: {} 2022-09-03T07:26:00.729 DEBUG:boto:Token: None 2022-09-03T07:26:00.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:00.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:STpgGqz8TnvNR+nv1XJQcprXZjQ= 2022-09-03T07:26:00.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:STpgGqz8TnvNR+nv1XJQcprXZjQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.733 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f2568c7e56ed22c7-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.774 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.775 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.775 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:00.776 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:00.776 DEBUG:boto:Method: GET 2022-09-03T07:26:00.777 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:26:00.778 DEBUG:boto:Data: 2022-09-03T07:26:00.778 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.779 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.780 DEBUG:boto:Params: {} 2022-09-03T07:26:00.780 DEBUG:boto:Token: None 2022-09-03T07:26:00.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.784 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000382890cc7dd33d83-0063130188-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.785 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.786 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.787 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.787 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:00.788 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:00.788 DEBUG:boto:Method: GET 2022-09-03T07:26:00.789 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:26:00.789 DEBUG:boto:Data: 2022-09-03T07:26:00.790 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.790 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.791 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.791 DEBUG:boto:Params: {} 2022-09-03T07:26:00.792 DEBUG:boto:Token: None 2022-09-03T07:26:00.792 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.797 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a5e0afb26cddd86f-0063130188-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.798 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.799 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.799 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.800 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.800 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:26:00.801 DEBUG:boto:Data: 2022-09-03T07:26:00.801 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.802 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.802 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.803 DEBUG:boto:Params: {} 2022-09-03T07:26:00.803 DEBUG:boto:Token: None 2022-09-03T07:26:00.804 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51 2022-09-03T07:26:00.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gJ/Dl2bsOLp5Tk78MeJYfm2dfWQ= 2022-09-03T07:26:00.805 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gJ/Dl2bsOLp5Tk78MeJYfm2dfWQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.807 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b89f1392c502ac0e-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.808 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.809 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.809 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.810 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:26:00.810 DEBUG:boto:Data: 2022-09-03T07:26:00.811 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.811 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.812 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.812 DEBUG:boto:Params: {} 2022-09-03T07:26:00.813 DEBUG:boto:Token: None 2022-09-03T07:26:00.813 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51 2022-09-03T07:26:00.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gJ/Dl2bsOLp5Tk78MeJYfm2dfWQ= 2022-09-03T07:26:00.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gJ/Dl2bsOLp5Tk78MeJYfm2dfWQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.816 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d16b997c98b2fe5-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.817 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:26:00.818 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.818 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.819 DEBUG:boto:Method: GET 2022-09-03T07:26:00.819 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:26:00.820 DEBUG:boto:Data: 2022-09-03T07:26:00.820 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.821 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.822 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.822 DEBUG:boto:Params: {} 2022-09-03T07:26:00.823 DEBUG:boto:Token: None 2022-09-03T07:26:00.823 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51 2022-09-03T07:26:00.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0avKVXbVdFalgndGMSDmp7XpXeA= 2022-09-03T07:26:00.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0avKVXbVdFalgndGMSDmp7XpXeA=', 'Content-Length': '0'} 2022-09-03T07:26:00.827 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008807c09949ce4e4c-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.827 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:00.828 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:00.829 DEBUG:boto:Method: GET 2022-09-03T07:26:00.829 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:26:00.830 DEBUG:boto:Data: 2022-09-03T07:26:00.830 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.831 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.832 DEBUG:boto:Params: {} 2022-09-03T07:26:00.832 DEBUG:boto:Token: None 2022-09-03T07:26:00.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj51 2022-09-03T07:26:00.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0avKVXbVdFalgndGMSDmp7XpXeA= 2022-09-03T07:26:00.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0avKVXbVdFalgndGMSDmp7XpXeA=', 'Content-Length': '0'} 2022-09-03T07:26:00.837 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000522c492fea4212c2-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.838 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.838 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.839 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:00.840 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:00.840 DEBUG:boto:Method: GET 2022-09-03T07:26:00.841 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:26:00.841 DEBUG:boto:Data: 2022-09-03T07:26:00.842 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.843 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.843 DEBUG:boto:Params: {} 2022-09-03T07:26:00.844 DEBUG:boto:Token: None 2022-09-03T07:26:00.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.848 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c02fcc1003420c2-0063130188-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.894 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.895 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.896 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.896 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:00.897 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:00.898 DEBUG:boto:Method: GET 2022-09-03T07:26:00.900 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:26:00.900 DEBUG:boto:Data: 2022-09-03T07:26:00.901 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.901 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.902 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.903 DEBUG:boto:Params: {} 2022-09-03T07:26:00.904 DEBUG:boto:Token: None 2022-09-03T07:26:00.905 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/ 2022-09-03T07:26:00.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ= 2022-09-03T07:26:00.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ut0/j/dEJ7Tm7zs73mFpCh5ucrQ=', 'Content-Length': '0'} 2022-09-03T07:26:00.909 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c3252ffef05273b8-0063130188-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.911 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:00.911 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:26:00.912 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:00.913 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:00.914 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.915 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.915 DEBUG:boto:Method: GET 2022-09-03T07:26:00.916 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.917 DEBUG:boto:Data: 2022-09-03T07:26:00.917 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.918 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.918 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.919 DEBUG:boto:Params: {} 2022-09-03T07:26:00.919 DEBUG:boto:Token: None 2022-09-03T07:26:00.920 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.921 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEjrh2G6HSW13QzPISKWw83KGL8= 2022-09-03T07:26:00.921 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEjrh2G6HSW13QzPISKWw83KGL8=', 'Content-Length': '0'} 2022-09-03T07:26:00.923 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ffbd62fbd66ecbe-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.924 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:00.925 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:00.925 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.926 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.927 DEBUG:boto:Method: GET 2022-09-03T07:26:00.927 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.928 DEBUG:boto:Data: 2022-09-03T07:26:00.928 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.931 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.931 DEBUG:boto:Params: {} 2022-09-03T07:26:00.932 DEBUG:boto:Token: None 2022-09-03T07:26:00.932 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.933 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEjrh2G6HSW13QzPISKWw83KGL8= 2022-09-03T07:26:00.934 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEjrh2G6HSW13QzPISKWw83KGL8=', 'Content-Length': '0'} 2022-09-03T07:26:00.936 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e32be69b21ba944-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.938 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:00.939 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:00.940 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.941 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.942 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.943 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.944 DEBUG:boto:Data: 2022-09-03T07:26:00.945 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.946 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.947 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.948 DEBUG:boto:Params: {} 2022-09-03T07:26:00.949 DEBUG:boto:Token: None 2022-09-03T07:26:00.951 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WysQVcitB7epTpmsnKguhkagg+o= 2022-09-03T07:26:00.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WysQVcitB7epTpmsnKguhkagg+o=', 'Content-Length': '0'} 2022-09-03T07:26:00.956 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7ee6c2e2c49dacc-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.956 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:00.957 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:00.958 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.958 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.959 DEBUG:boto:Method: HEAD 2022-09-03T07:26:00.959 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.960 DEBUG:boto:Data: 2022-09-03T07:26:00.960 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.961 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.961 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.962 DEBUG:boto:Params: {} 2022-09-03T07:26:00.962 DEBUG:boto:Token: None 2022-09-03T07:26:00.963 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.964 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WysQVcitB7epTpmsnKguhkagg+o= 2022-09-03T07:26:00.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WysQVcitB7epTpmsnKguhkagg+o=', 'Content-Length': '0'} 2022-09-03T07:26:00.966 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5a4daaadd908488-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.967 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:26:00.968 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:00.968 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:00.969 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.970 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.970 DEBUG:boto:Method: GET 2022-09-03T07:26:00.971 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.972 DEBUG:boto:Data: 2022-09-03T07:26:00.973 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:00.975 DEBUG:boto:Port: 8001 2022-09-03T07:26:00.976 DEBUG:boto:Params: {} 2022-09-03T07:26:00.977 DEBUG:boto:Token: None 2022-09-03T07:26:00.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEjrh2G6HSW13QzPISKWw83KGL8= 2022-09-03T07:26:00.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEjrh2G6HSW13QzPISKWw83KGL8=', 'Content-Length': '0'} 2022-09-03T07:26:00.982 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e44982041acb5d74-0063130188-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.983 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:00.984 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:00.985 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.986 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.987 DEBUG:boto:Method: GET 2022-09-03T07:26:00.988 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.988 DEBUG:boto:Data: 2022-09-03T07:26:00.989 DEBUG:boto:Headers: {} 2022-09-03T07:26:00.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:00.990 DEBUG:boto:Port: 8000 2022-09-03T07:26:00.990 DEBUG:boto:Params: {} 2022-09-03T07:26:00.991 DEBUG:boto:Token: None 2022-09-03T07:26:00.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:00 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:00.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEjrh2G6HSW13QzPISKWw83KGL8= 2022-09-03T07:26:00.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEjrh2G6HSW13QzPISKWw83KGL8=', 'Content-Length': '0'} 2022-09-03T07:26:00.995 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2fb3d664d0df7e8-0063130188-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:00.996 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:00.997 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:00.998 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:00.999 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:01.000 DEBUG:boto:Method: GET 2022-09-03T07:26:01.001 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:26:01.002 DEBUG:boto:Data: 2022-09-03T07:26:01.003 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.004 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.005 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.005 DEBUG:boto:Params: {} 2022-09-03T07:26:01.007 DEBUG:boto:Token: None 2022-09-03T07:26:01.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.011 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d6867100ceb399d3-0063130189-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.012 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.014 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.015 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.016 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:01.017 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:01.017 DEBUG:boto:Method: GET 2022-09-03T07:26:01.018 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:26:01.018 DEBUG:boto:Data: 2022-09-03T07:26:01.019 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.021 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.022 DEBUG:boto:Params: {} 2022-09-03T07:26:01.023 DEBUG:boto:Token: None 2022-09-03T07:26:01.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.031 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000045769147c0a2387-0063130189-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.078 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.080 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:01.081 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:01.082 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.083 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:26:01.083 DEBUG:boto:Data: 2022-09-03T07:26:01.084 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.086 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.087 DEBUG:boto:Params: {} 2022-09-03T07:26:01.088 DEBUG:boto:Token: None 2022-09-03T07:26:01.089 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj55 2022-09-03T07:26:01.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IkEH44GRAQg4P6z1AHiQ6byqt+I= 2022-09-03T07:26:01.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IkEH44GRAQg4P6z1AHiQ6byqt+I=', 'Content-Length': '0'} 2022-09-03T07:26:01.093 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093272f0c96696198-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.094 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:01.095 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:01.096 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.096 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:26:01.097 DEBUG:boto:Data: 2022-09-03T07:26:01.097 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.102 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.102 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.103 DEBUG:boto:Params: {} 2022-09-03T07:26:01.103 DEBUG:boto:Token: None 2022-09-03T07:26:01.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj55 2022-09-03T07:26:01.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IkEH44GRAQg4P6z1AHiQ6byqt+I= 2022-09-03T07:26:01.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IkEH44GRAQg4P6z1AHiQ6byqt+I=', 'Content-Length': '0'} 2022-09-03T07:26:01.107 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d2feb147527de33-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.108 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:26:01.108 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:01.109 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:01.109 DEBUG:boto:Method: GET 2022-09-03T07:26:01.110 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:26:01.110 DEBUG:boto:Data: 2022-09-03T07:26:01.111 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.112 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.112 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.113 DEBUG:boto:Params: {} 2022-09-03T07:26:01.114 DEBUG:boto:Token: None 2022-09-03T07:26:01.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj55 2022-09-03T07:26:01.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t8lLLOTgUX4Jv0aFDpdTtDVE+yM= 2022-09-03T07:26:01.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t8lLLOTgUX4Jv0aFDpdTtDVE+yM=', 'Content-Length': '0'} 2022-09-03T07:26:01.118 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000845977848a07ad19-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.119 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:01.119 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:01.120 DEBUG:boto:Method: GET 2022-09-03T07:26:01.120 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:26:01.121 DEBUG:boto:Data: 2022-09-03T07:26:01.121 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.122 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.123 DEBUG:boto:Params: {} 2022-09-03T07:26:01.124 DEBUG:boto:Token: None 2022-09-03T07:26:01.124 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj55 2022-09-03T07:26:01.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t8lLLOTgUX4Jv0aFDpdTtDVE+yM= 2022-09-03T07:26:01.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t8lLLOTgUX4Jv0aFDpdTtDVE+yM=', 'Content-Length': '0'} 2022-09-03T07:26:01.128 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ffe90af839fe2df7-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.129 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.130 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.131 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:01.131 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:01.132 DEBUG:boto:Method: GET 2022-09-03T07:26:01.134 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:26:01.135 DEBUG:boto:Data: 2022-09-03T07:26:01.135 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.136 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.136 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.137 DEBUG:boto:Params: {} 2022-09-03T07:26:01.138 DEBUG:boto:Token: None 2022-09-03T07:26:01.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.143 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ecf49970a93cb135-0063130189-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.190 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.191 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.192 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.192 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:01.193 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:01.193 DEBUG:boto:Method: GET 2022-09-03T07:26:01.194 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:26:01.194 DEBUG:boto:Data: 2022-09-03T07:26:01.195 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.195 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.196 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.197 DEBUG:boto:Params: {} 2022-09-03T07:26:01.198 DEBUG:boto:Token: None 2022-09-03T07:26:01.198 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.204 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c9cdf52639feae78-0063130189-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.205 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.206 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:26:01.207 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.208 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.208 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.209 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.209 DEBUG:boto:Method: GET 2022-09-03T07:26:01.210 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.211 DEBUG:boto:Data: 2022-09-03T07:26:01.212 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.213 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.214 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.214 DEBUG:boto:Params: {} 2022-09-03T07:26:01.215 DEBUG:boto:Token: None 2022-09-03T07:26:01.216 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6p4vydzYxKUgWIt6R765EQfMrw= 2022-09-03T07:26:01.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6p4vydzYxKUgWIt6R765EQfMrw=', 'Content-Length': '0'} 2022-09-03T07:26:01.220 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002e241151cb4d3ad-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.221 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.221 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.222 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.222 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.223 DEBUG:boto:Method: GET 2022-09-03T07:26:01.223 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.224 DEBUG:boto:Data: 2022-09-03T07:26:01.224 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.225 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.225 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.226 DEBUG:boto:Params: {} 2022-09-03T07:26:01.226 DEBUG:boto:Token: None 2022-09-03T07:26:01.227 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6p4vydzYxKUgWIt6R765EQfMrw= 2022-09-03T07:26:01.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6p4vydzYxKUgWIt6R765EQfMrw=', 'Content-Length': '0'} 2022-09-03T07:26:01.230 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b567276cb900f3c5-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.231 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.232 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.232 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.233 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.233 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.234 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.234 DEBUG:boto:Data: 2022-09-03T07:26:01.235 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.236 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.236 DEBUG:boto:Params: {} 2022-09-03T07:26:01.237 DEBUG:boto:Token: None 2022-09-03T07:26:01.237 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.238 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:baKzihgebQaPhnxMJyV0g0MBPt0= 2022-09-03T07:26:01.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:baKzihgebQaPhnxMJyV0g0MBPt0=', 'Content-Length': '0'} 2022-09-03T07:26:01.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ecae72cdaf493e5-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.241 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.242 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.242 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.243 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.243 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.244 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.244 DEBUG:boto:Data: 2022-09-03T07:26:01.245 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.245 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.246 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.246 DEBUG:boto:Params: {} 2022-09-03T07:26:01.247 DEBUG:boto:Token: None 2022-09-03T07:26:01.247 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:baKzihgebQaPhnxMJyV0g0MBPt0= 2022-09-03T07:26:01.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:baKzihgebQaPhnxMJyV0g0MBPt0=', 'Content-Length': '0'} 2022-09-03T07:26:01.252 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000120e265ef5ba08a1-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.253 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:26:01.254 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.255 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.256 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.257 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.258 DEBUG:boto:Method: GET 2022-09-03T07:26:01.259 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.260 DEBUG:boto:Data: 2022-09-03T07:26:01.261 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.263 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.264 DEBUG:boto:Params: {} 2022-09-03T07:26:01.265 DEBUG:boto:Token: None 2022-09-03T07:26:01.266 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6p4vydzYxKUgWIt6R765EQfMrw= 2022-09-03T07:26:01.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6p4vydzYxKUgWIt6R765EQfMrw=', 'Content-Length': '0'} 2022-09-03T07:26:01.271 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057f72116635f74d4-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.272 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.273 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.274 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.275 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.276 DEBUG:boto:Method: GET 2022-09-03T07:26:01.277 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.278 DEBUG:boto:Data: 2022-09-03T07:26:01.279 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.280 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.280 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.281 DEBUG:boto:Params: {} 2022-09-03T07:26:01.282 DEBUG:boto:Token: None 2022-09-03T07:26:01.283 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:01.284 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6p4vydzYxKUgWIt6R765EQfMrw= 2022-09-03T07:26:01.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6p4vydzYxKUgWIt6R765EQfMrw=', 'Content-Length': '0'} 2022-09-03T07:26:01.288 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bcef9d663ad1c21e-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.330 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.331 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.332 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:01.332 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:01.333 DEBUG:boto:Method: GET 2022-09-03T07:26:01.334 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:26:01.334 DEBUG:boto:Data: 2022-09-03T07:26:01.335 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.336 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.336 DEBUG:boto:Params: {} 2022-09-03T07:26:01.337 DEBUG:boto:Token: None 2022-09-03T07:26:01.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.346 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e0007eb600ff7a83-0063130189-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.347 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.348 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.349 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.350 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:01.351 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:01.352 DEBUG:boto:Method: GET 2022-09-03T07:26:01.352 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:26:01.353 DEBUG:boto:Data: 2022-09-03T07:26:01.357 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.360 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.361 DEBUG:boto:Params: {} 2022-09-03T07:26:01.362 DEBUG:boto:Token: None 2022-09-03T07:26:01.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.373 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c14c63d3f72d6c65-0063130189-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.375 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.377 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.378 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.379 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.380 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:26:01.381 DEBUG:boto:Data: 2022-09-03T07:26:01.382 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.384 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.385 DEBUG:boto:Params: {} 2022-09-03T07:26:01.387 DEBUG:boto:Token: None 2022-09-03T07:26:01.388 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57 2022-09-03T07:26:01.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n/KNo7ue5njUVZc78K4MYhNeRIc= 2022-09-03T07:26:01.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n/KNo7ue5njUVZc78K4MYhNeRIc=', 'Content-Length': '0'} 2022-09-03T07:26:01.393 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008daf60d533415971-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.395 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.396 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.397 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.399 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:26:01.399 DEBUG:boto:Data: 2022-09-03T07:26:01.400 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.402 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.403 DEBUG:boto:Params: {} 2022-09-03T07:26:01.405 DEBUG:boto:Token: None 2022-09-03T07:26:01.406 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57 2022-09-03T07:26:01.407 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n/KNo7ue5njUVZc78K4MYhNeRIc= 2022-09-03T07:26:01.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n/KNo7ue5njUVZc78K4MYhNeRIc=', 'Content-Length': '0'} 2022-09-03T07:26:01.419 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ebda75bcf77cbf5-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.421 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:26:01.422 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.423 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.424 DEBUG:boto:Method: GET 2022-09-03T07:26:01.426 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:26:01.427 DEBUG:boto:Data: 2022-09-03T07:26:01.428 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.430 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.431 DEBUG:boto:Params: {} 2022-09-03T07:26:01.431 DEBUG:boto:Token: None 2022-09-03T07:26:01.432 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57 2022-09-03T07:26:01.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HEvW0S3janPocGg127CHgm6lIlw= 2022-09-03T07:26:01.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HEvW0S3janPocGg127CHgm6lIlw=', 'Content-Length': '0'} 2022-09-03T07:26:01.436 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c64cb98c9538a98-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.437 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:01.438 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:01.438 DEBUG:boto:Method: GET 2022-09-03T07:26:01.439 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:26:01.440 DEBUG:boto:Data: 2022-09-03T07:26:01.440 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.441 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.442 DEBUG:boto:Params: {} 2022-09-03T07:26:01.442 DEBUG:boto:Token: None 2022-09-03T07:26:01.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj57 2022-09-03T07:26:01.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HEvW0S3janPocGg127CHgm6lIlw= 2022-09-03T07:26:01.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HEvW0S3janPocGg127CHgm6lIlw=', 'Content-Length': '0'} 2022-09-03T07:26:01.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3fd13cee84d1a96-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.454 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.455 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.455 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:01.456 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:01.456 DEBUG:boto:Method: GET 2022-09-03T07:26:01.457 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:26:01.458 DEBUG:boto:Data: 2022-09-03T07:26:01.460 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.463 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.467 DEBUG:boto:Params: {} 2022-09-03T07:26:01.468 DEBUG:boto:Token: None 2022-09-03T07:26:01.469 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.475 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006266bebc054f0385-0063130189-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.476 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.477 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.477 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.478 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:01.478 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:01.479 DEBUG:boto:Method: GET 2022-09-03T07:26:01.479 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:26:01.480 DEBUG:boto:Data: 2022-09-03T07:26:01.481 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.481 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.482 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.483 DEBUG:boto:Params: {} 2022-09-03T07:26:01.483 DEBUG:boto:Token: None 2022-09-03T07:26:01.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.489 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004076e187f7c6d062-0063130189-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.530 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.531 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:26:01.532 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.532 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.533 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.533 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.534 DEBUG:boto:Method: GET 2022-09-03T07:26:01.535 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.535 DEBUG:boto:Data: 2022-09-03T07:26:01.536 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.537 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.538 DEBUG:boto:Params: {} 2022-09-03T07:26:01.539 DEBUG:boto:Token: None 2022-09-03T07:26:01.539 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C5V8jmLNtAiVd1aqPlDMiST7dt8= 2022-09-03T07:26:01.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C5V8jmLNtAiVd1aqPlDMiST7dt8=', 'Content-Length': '0'} 2022-09-03T07:26:01.543 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009300c38b624b4b27-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.544 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.546 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.546 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.547 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.548 DEBUG:boto:Method: GET 2022-09-03T07:26:01.548 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.549 DEBUG:boto:Data: 2022-09-03T07:26:01.549 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.550 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.550 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.551 DEBUG:boto:Params: {} 2022-09-03T07:26:01.552 DEBUG:boto:Token: None 2022-09-03T07:26:01.552 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.553 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C5V8jmLNtAiVd1aqPlDMiST7dt8= 2022-09-03T07:26:01.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C5V8jmLNtAiVd1aqPlDMiST7dt8=', 'Content-Length': '0'} 2022-09-03T07:26:01.559 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a6fd3305d653a49-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.561 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.563 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.563 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.564 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.564 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.565 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.566 DEBUG:boto:Data: 2022-09-03T07:26:01.566 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.568 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.569 DEBUG:boto:Params: {} 2022-09-03T07:26:01.569 DEBUG:boto:Token: None 2022-09-03T07:26:01.570 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOxxVSS2ehPQjAaLVj9Xe/Slsp0= 2022-09-03T07:26:01.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOxxVSS2ehPQjAaLVj9Xe/Slsp0=', 'Content-Length': '0'} 2022-09-03T07:26:01.573 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008bba70ba4b961ea1-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.574 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.575 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.576 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.577 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.578 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.579 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.580 DEBUG:boto:Data: 2022-09-03T07:26:01.581 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.583 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.584 DEBUG:boto:Params: {} 2022-09-03T07:26:01.586 DEBUG:boto:Token: None 2022-09-03T07:26:01.587 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AOxxVSS2ehPQjAaLVj9Xe/Slsp0= 2022-09-03T07:26:01.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AOxxVSS2ehPQjAaLVj9Xe/Slsp0=', 'Content-Length': '0'} 2022-09-03T07:26:01.592 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064d3899864903d99-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.593 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:26:01.594 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.595 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.596 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.596 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.597 DEBUG:boto:Method: GET 2022-09-03T07:26:01.598 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.598 DEBUG:boto:Data: 2022-09-03T07:26:01.599 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.600 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.601 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.602 DEBUG:boto:Params: {} 2022-09-03T07:26:01.602 DEBUG:boto:Token: None 2022-09-03T07:26:01.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C5V8jmLNtAiVd1aqPlDMiST7dt8= 2022-09-03T07:26:01.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C5V8jmLNtAiVd1aqPlDMiST7dt8=', 'Content-Length': '0'} 2022-09-03T07:26:01.608 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a266a8a179fa16f4-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.609 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.609 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.610 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.610 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.611 DEBUG:boto:Method: GET 2022-09-03T07:26:01.612 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.612 DEBUG:boto:Data: 2022-09-03T07:26:01.613 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.614 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.615 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.615 DEBUG:boto:Params: {} 2022-09-03T07:26:01.616 DEBUG:boto:Token: None 2022-09-03T07:26:01.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:01.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C5V8jmLNtAiVd1aqPlDMiST7dt8= 2022-09-03T07:26:01.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C5V8jmLNtAiVd1aqPlDMiST7dt8=', 'Content-Length': '0'} 2022-09-03T07:26:01.621 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000033f30819e1665e19-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.622 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.623 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.624 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:01.625 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:01.626 DEBUG:boto:Method: GET 2022-09-03T07:26:01.627 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:26:01.628 DEBUG:boto:Data: 2022-09-03T07:26:01.629 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.632 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.632 DEBUG:boto:Params: {} 2022-09-03T07:26:01.633 DEBUG:boto:Token: None 2022-09-03T07:26:01.634 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.637 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad4939568f849570-0063130189-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.640 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.641 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.643 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.644 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:01.645 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:01.646 DEBUG:boto:Method: GET 2022-09-03T07:26:01.647 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:26:01.647 DEBUG:boto:Data: 2022-09-03T07:26:01.649 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.650 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.651 DEBUG:boto:Params: {} 2022-09-03T07:26:01.651 DEBUG:boto:Token: None 2022-09-03T07:26:01.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.658 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b0f9e61008f97358-0063130189-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.659 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.661 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.661 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.662 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.663 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:26:01.663 DEBUG:boto:Data: 2022-09-03T07:26:01.664 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.665 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.666 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.667 DEBUG:boto:Params: {} 2022-09-03T07:26:01.668 DEBUG:boto:Token: None 2022-09-03T07:26:01.669 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58 2022-09-03T07:26:01.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jqlr0OuZldmFCphVxs9Dokdp8PY= 2022-09-03T07:26:01.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jqlr0OuZldmFCphVxs9Dokdp8PY=', 'Content-Length': '0'} 2022-09-03T07:26:01.675 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080bd69bc23de13f0-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.676 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.677 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.677 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.678 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:26:01.679 DEBUG:boto:Data: 2022-09-03T07:26:01.679 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.680 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.680 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.681 DEBUG:boto:Params: {} 2022-09-03T07:26:01.681 DEBUG:boto:Token: None 2022-09-03T07:26:01.682 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58 2022-09-03T07:26:01.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jqlr0OuZldmFCphVxs9Dokdp8PY= 2022-09-03T07:26:01.683 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jqlr0OuZldmFCphVxs9Dokdp8PY=', 'Content-Length': '0'} 2022-09-03T07:26:01.687 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042bdef69fc6bbdb2-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.689 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:26:01.689 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.690 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.691 DEBUG:boto:Method: GET 2022-09-03T07:26:01.693 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:26:01.694 DEBUG:boto:Data: 2022-09-03T07:26:01.695 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.696 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.697 DEBUG:boto:Params: {} 2022-09-03T07:26:01.697 DEBUG:boto:Token: None 2022-09-03T07:26:01.698 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58 2022-09-03T07:26:01.699 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wYd/alnWKAp62W6N+bWIPIhqndc= 2022-09-03T07:26:01.700 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wYd/alnWKAp62W6N+bWIPIhqndc=', 'Content-Length': '0'} 2022-09-03T07:26:01.702 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010e77eddd8d69929-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.703 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:01.704 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:01.704 DEBUG:boto:Method: GET 2022-09-03T07:26:01.705 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:26:01.705 DEBUG:boto:Data: 2022-09-03T07:26:01.706 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.706 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.707 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.708 DEBUG:boto:Params: {} 2022-09-03T07:26:01.710 DEBUG:boto:Token: None 2022-09-03T07:26:01.711 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj58 2022-09-03T07:26:01.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wYd/alnWKAp62W6N+bWIPIhqndc= 2022-09-03T07:26:01.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wYd/alnWKAp62W6N+bWIPIhqndc=', 'Content-Length': '0'} 2022-09-03T07:26:01.716 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c88e9a0ef211938b-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.758 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.759 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.760 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:01.760 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:01.761 DEBUG:boto:Method: GET 2022-09-03T07:26:01.761 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:26:01.762 DEBUG:boto:Data: 2022-09-03T07:26:01.763 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.764 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.764 DEBUG:boto:Params: {} 2022-09-03T07:26:01.765 DEBUG:boto:Token: None 2022-09-03T07:26:01.765 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.769 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009f2be0e776c7e83a-0063130189-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.771 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.774 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.775 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.776 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:01.777 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:01.779 DEBUG:boto:Method: GET 2022-09-03T07:26:01.779 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:26:01.780 DEBUG:boto:Data: 2022-09-03T07:26:01.782 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.784 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.785 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.786 DEBUG:boto:Params: {} 2022-09-03T07:26:01.788 DEBUG:boto:Token: None 2022-09-03T07:26:01.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.795 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b812165bf6ae5497-0063130189-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.797 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.799 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:26:01.800 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.800 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.801 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.801 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.802 DEBUG:boto:Method: GET 2022-09-03T07:26:01.803 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.803 DEBUG:boto:Data: 2022-09-03T07:26:01.804 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.805 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.806 DEBUG:boto:Params: {} 2022-09-03T07:26:01.806 DEBUG:boto:Token: None 2022-09-03T07:26:01.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zqbBKOkS8fmISuiLrAAztLJndZU= 2022-09-03T07:26:01.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zqbBKOkS8fmISuiLrAAztLJndZU=', 'Content-Length': '0'} 2022-09-03T07:26:01.811 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001736b9cb36e530c9-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.813 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.814 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.815 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.816 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.817 DEBUG:boto:Method: GET 2022-09-03T07:26:01.818 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.819 DEBUG:boto:Data: 2022-09-03T07:26:01.820 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.821 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.822 DEBUG:boto:Params: {} 2022-09-03T07:26:01.823 DEBUG:boto:Token: None 2022-09-03T07:26:01.825 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zqbBKOkS8fmISuiLrAAztLJndZU= 2022-09-03T07:26:01.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zqbBKOkS8fmISuiLrAAztLJndZU=', 'Content-Length': '0'} 2022-09-03T07:26:01.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007387ee947462b32c-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.830 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.831 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.831 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.832 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.833 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.834 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.835 DEBUG:boto:Data: 2022-09-03T07:26:01.836 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.836 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.837 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.838 DEBUG:boto:Params: {} 2022-09-03T07:26:01.839 DEBUG:boto:Token: None 2022-09-03T07:26:01.841 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.842 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kB3kS+4zekmhtuwHz9AaDnvbTlI= 2022-09-03T07:26:01.843 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kB3kS+4zekmhtuwHz9AaDnvbTlI=', 'Content-Length': '0'} 2022-09-03T07:26:01.845 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb7e379448f0e287-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.847 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.848 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.848 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.849 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.850 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.851 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.851 DEBUG:boto:Data: 2022-09-03T07:26:01.852 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.853 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.853 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.854 DEBUG:boto:Params: {} 2022-09-03T07:26:01.855 DEBUG:boto:Token: None 2022-09-03T07:26:01.856 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kB3kS+4zekmhtuwHz9AaDnvbTlI= 2022-09-03T07:26:01.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kB3kS+4zekmhtuwHz9AaDnvbTlI=', 'Content-Length': '0'} 2022-09-03T07:26:01.860 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b78bf28bab99aae-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.861 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:26:01.861 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.862 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.864 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.865 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.866 DEBUG:boto:Method: GET 2022-09-03T07:26:01.867 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.868 DEBUG:boto:Data: 2022-09-03T07:26:01.869 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.870 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.871 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.872 DEBUG:boto:Params: {} 2022-09-03T07:26:01.873 DEBUG:boto:Token: None 2022-09-03T07:26:01.874 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zqbBKOkS8fmISuiLrAAztLJndZU= 2022-09-03T07:26:01.876 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zqbBKOkS8fmISuiLrAAztLJndZU=', 'Content-Length': '0'} 2022-09-03T07:26:01.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a461bf0d891b972-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.879 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.880 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.881 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.881 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.882 DEBUG:boto:Method: GET 2022-09-03T07:26:01.883 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.883 DEBUG:boto:Data: 2022-09-03T07:26:01.884 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.885 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.886 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.887 DEBUG:boto:Params: {} 2022-09-03T07:26:01.888 DEBUG:boto:Token: None 2022-09-03T07:26:01.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:01.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zqbBKOkS8fmISuiLrAAztLJndZU= 2022-09-03T07:26:01.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zqbBKOkS8fmISuiLrAAztLJndZU=', 'Content-Length': '0'} 2022-09-03T07:26:01.894 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bcdac35689348968-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.895 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.896 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.897 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:01.897 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:01.898 DEBUG:boto:Method: GET 2022-09-03T07:26:01.899 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:26:01.900 DEBUG:boto:Data: 2022-09-03T07:26:01.900 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.901 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.902 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.903 DEBUG:boto:Params: {} 2022-09-03T07:26:01.903 DEBUG:boto:Token: None 2022-09-03T07:26:01.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.909 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006ae62aacae605141-0063130189-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.950 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.951 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:01.952 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:01.952 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:01.953 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:01.954 DEBUG:boto:Method: GET 2022-09-03T07:26:01.955 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:26:01.955 DEBUG:boto:Data: 2022-09-03T07:26:01.956 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.956 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.957 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.958 DEBUG:boto:Params: {} 2022-09-03T07:26:01.958 DEBUG:boto:Token: None 2022-09-03T07:26:01.959 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/ 2022-09-03T07:26:01.959 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag= 2022-09-03T07:26:01.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9QvlwCIAKG+UU7hkEs56tp75Xag=', 'Content-Length': '0'} 2022-09-03T07:26:01.964 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b7006d801e342b3-0063130189-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.965 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:01.966 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.966 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.967 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.967 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:26:01.968 DEBUG:boto:Data: 2022-09-03T07:26:01.968 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.969 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.969 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.970 DEBUG:boto:Params: {} 2022-09-03T07:26:01.970 DEBUG:boto:Token: None 2022-09-03T07:26:01.971 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj68 2022-09-03T07:26:01.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aIbPqYpND7YcMa8DUeRyu1XdtKA= 2022-09-03T07:26:01.972 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aIbPqYpND7YcMa8DUeRyu1XdtKA=', 'Content-Length': '0'} 2022-09-03T07:26:01.974 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5b8cd46987a380f-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.975 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.975 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.976 DEBUG:boto:Method: HEAD 2022-09-03T07:26:01.976 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:26:01.977 DEBUG:boto:Data: 2022-09-03T07:26:01.977 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:01.978 DEBUG:boto:Port: 8000 2022-09-03T07:26:01.979 DEBUG:boto:Params: {} 2022-09-03T07:26:01.979 DEBUG:boto:Token: None 2022-09-03T07:26:01.980 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj68 2022-09-03T07:26:01.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aIbPqYpND7YcMa8DUeRyu1XdtKA= 2022-09-03T07:26:01.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aIbPqYpND7YcMa8DUeRyu1XdtKA=', 'Content-Length': '0'} 2022-09-03T07:26:01.984 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e056ac4f333df415-0063130189-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.985 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:26:01.986 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.986 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.987 DEBUG:boto:Method: GET 2022-09-03T07:26:01.987 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:26:01.988 DEBUG:boto:Data: 2022-09-03T07:26:01.989 DEBUG:boto:Headers: {} 2022-09-03T07:26:01.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:01.990 DEBUG:boto:Port: 8001 2022-09-03T07:26:01.991 DEBUG:boto:Params: {} 2022-09-03T07:26:01.992 DEBUG:boto:Token: None 2022-09-03T07:26:01.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:01 GMT /avysmw-75/obj68 2022-09-03T07:26:01.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AQCrSz/QmqEwoavEo8oLi3E9Pn0= 2022-09-03T07:26:01.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AQCrSz/QmqEwoavEo8oLi3E9Pn0=', 'Content-Length': '0'} 2022-09-03T07:26:01.997 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000beeb2e6d08c0452d-0063130189-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:01.997 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:01.998 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:01.999 DEBUG:boto:Method: GET 2022-09-03T07:26:01.999 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:26:02.000 DEBUG:boto:Data: 2022-09-03T07:26:02.000 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.001 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.001 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.002 DEBUG:boto:Params: {} 2022-09-03T07:26:02.003 DEBUG:boto:Token: None 2022-09-03T07:26:02.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj68 2022-09-03T07:26:02.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mduxgZFvwGG9SGaMWaNgDptd+uU= 2022-09-03T07:26:02.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mduxgZFvwGG9SGaMWaNgDptd+uU=', 'Content-Length': '0'} 2022-09-03T07:26:02.008 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001523eae27d94244e-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.054 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.055 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.056 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:02.056 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:02.057 DEBUG:boto:Method: GET 2022-09-03T07:26:02.057 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:26:02.058 DEBUG:boto:Data: 2022-09-03T07:26:02.058 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.059 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.060 DEBUG:boto:Params: {} 2022-09-03T07:26:02.062 DEBUG:boto:Token: None 2022-09-03T07:26:02.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.067 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f778d5dbde613592-006313018a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.068 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.069 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.070 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.070 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:02.071 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:02.071 DEBUG:boto:Method: GET 2022-09-03T07:26:02.072 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:26:02.073 DEBUG:boto:Data: 2022-09-03T07:26:02.073 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.076 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.077 DEBUG:boto:Params: {} 2022-09-03T07:26:02.079 DEBUG:boto:Token: None 2022-09-03T07:26:02.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.086 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c4f4c95631d5476-006313018a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.087 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.090 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:26:02.091 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.092 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.093 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.094 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.095 DEBUG:boto:Method: GET 2022-09-03T07:26:02.096 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.098 DEBUG:boto:Data: 2022-09-03T07:26:02.099 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.101 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.102 DEBUG:boto:Params: {} 2022-09-03T07:26:02.104 DEBUG:boto:Token: None 2022-09-03T07:26:02.105 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.106 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q0I506ql+oRdmEFkb4mT7jdHWHw= 2022-09-03T07:26:02.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q0I506ql+oRdmEFkb4mT7jdHWHw=', 'Content-Length': '0'} 2022-09-03T07:26:02.110 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc8afca39f77c1fe-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.111 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.112 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.113 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.113 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.114 DEBUG:boto:Method: GET 2022-09-03T07:26:02.115 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.115 DEBUG:boto:Data: 2022-09-03T07:26:02.116 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.117 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.118 DEBUG:boto:Params: {} 2022-09-03T07:26:02.119 DEBUG:boto:Token: None 2022-09-03T07:26:02.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q0I506ql+oRdmEFkb4mT7jdHWHw= 2022-09-03T07:26:02.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q0I506ql+oRdmEFkb4mT7jdHWHw=', 'Content-Length': '0'} 2022-09-03T07:26:02.123 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9d722aecef25f66-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.124 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.125 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.125 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.126 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.126 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.127 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.127 DEBUG:boto:Data: 2022-09-03T07:26:02.128 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.129 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.129 DEBUG:boto:Params: {} 2022-09-03T07:26:02.130 DEBUG:boto:Token: None 2022-09-03T07:26:02.130 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J5esp2TYOKLzNAC6vBpdQZskSPQ= 2022-09-03T07:26:02.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J5esp2TYOKLzNAC6vBpdQZskSPQ=', 'Content-Length': '0'} 2022-09-03T07:26:02.134 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092cd156c614118fc-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.134 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.135 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.136 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.136 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.137 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.137 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.138 DEBUG:boto:Data: 2022-09-03T07:26:02.138 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.139 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.139 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.140 DEBUG:boto:Params: {} 2022-09-03T07:26:02.140 DEBUG:boto:Token: None 2022-09-03T07:26:02.141 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J5esp2TYOKLzNAC6vBpdQZskSPQ= 2022-09-03T07:26:02.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J5esp2TYOKLzNAC6vBpdQZskSPQ=', 'Content-Length': '0'} 2022-09-03T07:26:02.144 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f29d2b9c8ee7628-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.145 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:26:02.145 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.146 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.147 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.147 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.148 DEBUG:boto:Method: GET 2022-09-03T07:26:02.148 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.149 DEBUG:boto:Data: 2022-09-03T07:26:02.150 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.151 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.151 DEBUG:boto:Params: {} 2022-09-03T07:26:02.152 DEBUG:boto:Token: None 2022-09-03T07:26:02.153 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q0I506ql+oRdmEFkb4mT7jdHWHw= 2022-09-03T07:26:02.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q0I506ql+oRdmEFkb4mT7jdHWHw=', 'Content-Length': '0'} 2022-09-03T07:26:02.156 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca039270b8fb45b8-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.198 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.199 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.200 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.200 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.201 DEBUG:boto:Method: GET 2022-09-03T07:26:02.202 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.202 DEBUG:boto:Data: 2022-09-03T07:26:02.203 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.204 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.205 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.206 DEBUG:boto:Params: {} 2022-09-03T07:26:02.206 DEBUG:boto:Token: None 2022-09-03T07:26:02.207 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:02.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q0I506ql+oRdmEFkb4mT7jdHWHw= 2022-09-03T07:26:02.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q0I506ql+oRdmEFkb4mT7jdHWHw=', 'Content-Length': '0'} 2022-09-03T07:26:02.211 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f37159c75fc18e0-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.212 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.213 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.213 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:02.214 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:02.214 DEBUG:boto:Method: GET 2022-09-03T07:26:02.215 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:26:02.216 DEBUG:boto:Data: 2022-09-03T07:26:02.216 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.217 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.218 DEBUG:boto:Params: {} 2022-09-03T07:26:02.219 DEBUG:boto:Token: None 2022-09-03T07:26:02.219 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.224 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000347fd9e399751f68-006313018a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.225 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.226 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.226 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.227 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:02.227 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:02.228 DEBUG:boto:Method: GET 2022-09-03T07:26:02.229 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:26:02.229 DEBUG:boto:Data: 2022-09-03T07:26:02.230 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.232 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.232 DEBUG:boto:Params: {} 2022-09-03T07:26:02.233 DEBUG:boto:Token: None 2022-09-03T07:26:02.234 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.239 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007ca968cd399b3c04-006313018a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.284 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.286 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.286 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.287 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.288 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:26:02.289 DEBUG:boto:Data: 2022-09-03T07:26:02.289 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.290 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.290 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.291 DEBUG:boto:Params: {} 2022-09-03T07:26:02.292 DEBUG:boto:Token: None 2022-09-03T07:26:02.293 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69 2022-09-03T07:26:02.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5AFb0iK1ILHLdWXjJOJEHSCx2vI= 2022-09-03T07:26:02.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5AFb0iK1ILHLdWXjJOJEHSCx2vI=', 'Content-Length': '0'} 2022-09-03T07:26:02.296 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6229b4cdb7d19dd-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.297 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.298 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.299 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.299 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:26:02.300 DEBUG:boto:Data: 2022-09-03T07:26:02.300 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.301 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.301 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.302 DEBUG:boto:Params: {} 2022-09-03T07:26:02.303 DEBUG:boto:Token: None 2022-09-03T07:26:02.303 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69 2022-09-03T07:26:02.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5AFb0iK1ILHLdWXjJOJEHSCx2vI= 2022-09-03T07:26:02.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5AFb0iK1ILHLdWXjJOJEHSCx2vI=', 'Content-Length': '0'} 2022-09-03T07:26:02.308 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000202ba38fa1505a46-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.309 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:26:02.309 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.310 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.311 DEBUG:boto:Method: GET 2022-09-03T07:26:02.311 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:26:02.312 DEBUG:boto:Data: 2022-09-03T07:26:02.313 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.318 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.319 DEBUG:boto:Params: {} 2022-09-03T07:26:02.319 DEBUG:boto:Token: None 2022-09-03T07:26:02.320 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69 2022-09-03T07:26:02.321 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r0EplZb3AvD4aOeV5owlVLNriF8= 2022-09-03T07:26:02.322 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r0EplZb3AvD4aOeV5owlVLNriF8=', 'Content-Length': '0'} 2022-09-03T07:26:02.324 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ecce071ae134a3d7-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.325 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:02.326 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:02.326 DEBUG:boto:Method: GET 2022-09-03T07:26:02.327 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:26:02.327 DEBUG:boto:Data: 2022-09-03T07:26:02.328 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.328 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.329 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.329 DEBUG:boto:Params: {} 2022-09-03T07:26:02.335 DEBUG:boto:Token: None 2022-09-03T07:26:02.337 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj69 2022-09-03T07:26:02.338 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r0EplZb3AvD4aOeV5owlVLNriF8= 2022-09-03T07:26:02.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r0EplZb3AvD4aOeV5owlVLNriF8=', 'Content-Length': '0'} 2022-09-03T07:26:02.340 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8a0bb685e601466-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.341 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.342 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.343 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:02.343 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:02.344 DEBUG:boto:Method: GET 2022-09-03T07:26:02.344 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:26:02.345 DEBUG:boto:Data: 2022-09-03T07:26:02.345 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.346 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.346 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.347 DEBUG:boto:Params: {} 2022-09-03T07:26:02.348 DEBUG:boto:Token: None 2022-09-03T07:26:02.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.352 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a19c5a31b96dcb8-006313018a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.353 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.354 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.354 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.355 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:02.355 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:02.356 DEBUG:boto:Method: GET 2022-09-03T07:26:02.356 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:26:02.357 DEBUG:boto:Data: 2022-09-03T07:26:02.357 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.359 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.359 DEBUG:boto:Params: {} 2022-09-03T07:26:02.360 DEBUG:boto:Token: None 2022-09-03T07:26:02.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.366 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000e5fbca1253339ee-006313018a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.367 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.367 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:26:02.368 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.369 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.369 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.370 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.371 DEBUG:boto:Method: GET 2022-09-03T07:26:02.371 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.372 DEBUG:boto:Data: 2022-09-03T07:26:02.372 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.374 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.374 DEBUG:boto:Params: {} 2022-09-03T07:26:02.375 DEBUG:boto:Token: None 2022-09-03T07:26:02.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x0F8adskDIkuZLNcfCi2dDXdoCg= 2022-09-03T07:26:02.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x0F8adskDIkuZLNcfCi2dDXdoCg=', 'Content-Length': '0'} 2022-09-03T07:26:02.379 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000028396280fdff28d8-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.380 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.380 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.381 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.381 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.382 DEBUG:boto:Method: GET 2022-09-03T07:26:02.383 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.383 DEBUG:boto:Data: 2022-09-03T07:26:02.384 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.385 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.385 DEBUG:boto:Params: {} 2022-09-03T07:26:02.386 DEBUG:boto:Token: None 2022-09-03T07:26:02.387 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x0F8adskDIkuZLNcfCi2dDXdoCg= 2022-09-03T07:26:02.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x0F8adskDIkuZLNcfCi2dDXdoCg=', 'Content-Length': '0'} 2022-09-03T07:26:02.390 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dcdcfa47ebf6a3d9-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.391 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.391 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.392 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.392 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.393 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.393 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.394 DEBUG:boto:Data: 2022-09-03T07:26:02.395 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.396 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.396 DEBUG:boto:Params: {} 2022-09-03T07:26:02.397 DEBUG:boto:Token: None 2022-09-03T07:26:02.397 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OowmIwoRgaBpsnIxg/Gx6u4sajQ= 2022-09-03T07:26:02.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OowmIwoRgaBpsnIxg/Gx6u4sajQ=', 'Content-Length': '0'} 2022-09-03T07:26:02.402 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003035833b51a01672-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.403 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.403 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.404 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.404 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.405 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.405 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.406 DEBUG:boto:Data: 2022-09-03T07:26:02.407 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.408 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.408 DEBUG:boto:Params: {} 2022-09-03T07:26:02.409 DEBUG:boto:Token: None 2022-09-03T07:26:02.410 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OowmIwoRgaBpsnIxg/Gx6u4sajQ= 2022-09-03T07:26:02.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OowmIwoRgaBpsnIxg/Gx6u4sajQ=', 'Content-Length': '0'} 2022-09-03T07:26:02.414 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007452d4a502c3a641-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.415 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:26:02.416 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.417 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.417 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.418 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.418 DEBUG:boto:Method: GET 2022-09-03T07:26:02.419 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.420 DEBUG:boto:Data: 2022-09-03T07:26:02.420 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.421 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.422 DEBUG:boto:Params: {} 2022-09-03T07:26:02.422 DEBUG:boto:Token: None 2022-09-03T07:26:02.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x0F8adskDIkuZLNcfCi2dDXdoCg= 2022-09-03T07:26:02.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x0F8adskDIkuZLNcfCi2dDXdoCg=', 'Content-Length': '0'} 2022-09-03T07:26:02.426 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052dd35a3a665875a-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.427 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.428 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.428 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.429 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.429 DEBUG:boto:Method: GET 2022-09-03T07:26:02.430 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.431 DEBUG:boto:Data: 2022-09-03T07:26:02.431 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.432 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.433 DEBUG:boto:Params: {} 2022-09-03T07:26:02.433 DEBUG:boto:Token: None 2022-09-03T07:26:02.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:02.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x0F8adskDIkuZLNcfCi2dDXdoCg= 2022-09-03T07:26:02.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x0F8adskDIkuZLNcfCi2dDXdoCg=', 'Content-Length': '0'} 2022-09-03T07:26:02.437 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034d02be3caa31560-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.482 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.483 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.484 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:02.484 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:02.485 DEBUG:boto:Method: GET 2022-09-03T07:26:02.486 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:26:02.486 DEBUG:boto:Data: 2022-09-03T07:26:02.487 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.488 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.488 DEBUG:boto:Params: {} 2022-09-03T07:26:02.489 DEBUG:boto:Token: None 2022-09-03T07:26:02.490 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.495 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d33ff4eccad57caf-006313018a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.498 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.499 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.500 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.500 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:02.501 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:02.501 DEBUG:boto:Method: GET 2022-09-03T07:26:02.502 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:26:02.503 DEBUG:boto:Data: 2022-09-03T07:26:02.504 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.506 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.507 DEBUG:boto:Params: {} 2022-09-03T07:26:02.508 DEBUG:boto:Token: None 2022-09-03T07:26:02.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.515 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a1d9ca453b787a26-006313018a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.516 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.517 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.517 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.518 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.519 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:26:02.519 DEBUG:boto:Data: 2022-09-03T07:26:02.520 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.521 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.522 DEBUG:boto:Params: {} 2022-09-03T07:26:02.522 DEBUG:boto:Token: None 2022-09-03T07:26:02.523 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76 2022-09-03T07:26:02.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/pxPTh8kHjRBoFExSa4Kj4auy0U= 2022-09-03T07:26:02.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/pxPTh8kHjRBoFExSa4Kj4auy0U=', 'Content-Length': '0'} 2022-09-03T07:26:02.526 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0a5f177fa8af712-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.527 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.528 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.528 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.529 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:26:02.530 DEBUG:boto:Data: 2022-09-03T07:26:02.530 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.531 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.532 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.532 DEBUG:boto:Params: {} 2022-09-03T07:26:02.533 DEBUG:boto:Token: None 2022-09-03T07:26:02.533 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76 2022-09-03T07:26:02.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/pxPTh8kHjRBoFExSa4Kj4auy0U= 2022-09-03T07:26:02.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/pxPTh8kHjRBoFExSa4Kj4auy0U=', 'Content-Length': '0'} 2022-09-03T07:26:02.537 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000242ba79b4b636c33-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.538 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:26:02.539 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.539 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.540 DEBUG:boto:Method: GET 2022-09-03T07:26:02.541 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:26:02.541 DEBUG:boto:Data: 2022-09-03T07:26:02.542 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.542 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.543 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.544 DEBUG:boto:Params: {} 2022-09-03T07:26:02.546 DEBUG:boto:Token: None 2022-09-03T07:26:02.546 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76 2022-09-03T07:26:02.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cfbE3ChK6pNBuy+ot/X2+qlcyxc= 2022-09-03T07:26:02.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cfbE3ChK6pNBuy+ot/X2+qlcyxc=', 'Content-Length': '0'} 2022-09-03T07:26:02.550 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000542bdaf1c5b87ce3-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.594 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:02.595 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:02.595 DEBUG:boto:Method: GET 2022-09-03T07:26:02.596 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:26:02.597 DEBUG:boto:Data: 2022-09-03T07:26:02.597 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.598 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.598 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.599 DEBUG:boto:Params: {} 2022-09-03T07:26:02.599 DEBUG:boto:Token: None 2022-09-03T07:26:02.600 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj76 2022-09-03T07:26:02.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cfbE3ChK6pNBuy+ot/X2+qlcyxc= 2022-09-03T07:26:02.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cfbE3ChK6pNBuy+ot/X2+qlcyxc=', 'Content-Length': '0'} 2022-09-03T07:26:02.603 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d17f55d2b7e3235-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.604 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.605 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.605 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:02.606 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:02.607 DEBUG:boto:Method: GET 2022-09-03T07:26:02.607 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:26:02.608 DEBUG:boto:Data: 2022-09-03T07:26:02.608 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.609 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.609 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.610 DEBUG:boto:Params: {} 2022-09-03T07:26:02.610 DEBUG:boto:Token: None 2022-09-03T07:26:02.611 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.616 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000046252ba4cc151998-006313018a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.618 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.620 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.621 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.622 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:02.623 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:02.624 DEBUG:boto:Method: GET 2022-09-03T07:26:02.625 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:26:02.626 DEBUG:boto:Data: 2022-09-03T07:26:02.627 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.628 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.629 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.630 DEBUG:boto:Params: {} 2022-09-03T07:26:02.631 DEBUG:boto:Token: None 2022-09-03T07:26:02.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.639 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cae741a7fd75532d-006313018a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.639 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.641 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:26:02.642 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.643 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.643 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.644 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.645 DEBUG:boto:Method: GET 2022-09-03T07:26:02.645 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.646 DEBUG:boto:Data: 2022-09-03T07:26:02.646 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.648 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.648 DEBUG:boto:Params: {} 2022-09-03T07:26:02.649 DEBUG:boto:Token: None 2022-09-03T07:26:02.650 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7EEv1cq4Z/uyF/TVJw9uHdrS35M= 2022-09-03T07:26:02.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7EEv1cq4Z/uyF/TVJw9uHdrS35M=', 'Content-Length': '0'} 2022-09-03T07:26:02.655 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc04b5bcc2601446-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.656 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.657 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.658 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.659 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.660 DEBUG:boto:Method: GET 2022-09-03T07:26:02.660 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.661 DEBUG:boto:Data: 2022-09-03T07:26:02.661 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.663 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.663 DEBUG:boto:Params: {} 2022-09-03T07:26:02.664 DEBUG:boto:Token: None 2022-09-03T07:26:02.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7EEv1cq4Z/uyF/TVJw9uHdrS35M= 2022-09-03T07:26:02.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7EEv1cq4Z/uyF/TVJw9uHdrS35M=', 'Content-Length': '0'} 2022-09-03T07:26:02.668 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ebb5b80b9fb8d9bc-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.710 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.711 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.712 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.712 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.713 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.715 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.715 DEBUG:boto:Data: 2022-09-03T07:26:02.716 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.717 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.718 DEBUG:boto:Params: {} 2022-09-03T07:26:02.719 DEBUG:boto:Token: None 2022-09-03T07:26:02.729 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t6sCoGi88CnzjC4dnqnBLnqiXDc= 2022-09-03T07:26:02.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t6sCoGi88CnzjC4dnqnBLnqiXDc=', 'Content-Length': '0'} 2022-09-03T07:26:02.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d7b0aa43c040369-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.733 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.733 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.734 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.734 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.735 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.736 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.736 DEBUG:boto:Data: 2022-09-03T07:26:02.737 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.737 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.738 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.738 DEBUG:boto:Params: {} 2022-09-03T07:26:02.739 DEBUG:boto:Token: None 2022-09-03T07:26:02.739 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t6sCoGi88CnzjC4dnqnBLnqiXDc= 2022-09-03T07:26:02.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t6sCoGi88CnzjC4dnqnBLnqiXDc=', 'Content-Length': '0'} 2022-09-03T07:26:02.743 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000cc0c338d39ee835-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.743 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:26:02.744 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.744 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.745 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.746 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.746 DEBUG:boto:Method: GET 2022-09-03T07:26:02.747 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.747 DEBUG:boto:Data: 2022-09-03T07:26:02.748 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.749 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.749 DEBUG:boto:Params: {} 2022-09-03T07:26:02.750 DEBUG:boto:Token: None 2022-09-03T07:26:02.750 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7EEv1cq4Z/uyF/TVJw9uHdrS35M= 2022-09-03T07:26:02.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7EEv1cq4Z/uyF/TVJw9uHdrS35M=', 'Content-Length': '0'} 2022-09-03T07:26:02.754 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd9dc0a609327843-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.755 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.755 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.756 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.757 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.757 DEBUG:boto:Method: GET 2022-09-03T07:26:02.758 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.758 DEBUG:boto:Data: 2022-09-03T07:26:02.759 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.760 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.760 DEBUG:boto:Params: {} 2022-09-03T07:26:02.761 DEBUG:boto:Token: None 2022-09-03T07:26:02.762 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:02.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7EEv1cq4Z/uyF/TVJw9uHdrS35M= 2022-09-03T07:26:02.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7EEv1cq4Z/uyF/TVJw9uHdrS35M=', 'Content-Length': '0'} 2022-09-03T07:26:02.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003c9d546607abdc4-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.766 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.767 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.767 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:02.768 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:02.768 DEBUG:boto:Method: GET 2022-09-03T07:26:02.769 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:26:02.769 DEBUG:boto:Data: 2022-09-03T07:26:02.770 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.771 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.771 DEBUG:boto:Params: {} 2022-09-03T07:26:02.772 DEBUG:boto:Token: None 2022-09-03T07:26:02.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.774 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.777 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007718c50519fc44d9-006313018a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.818 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.819 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.820 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.820 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:02.821 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:02.821 DEBUG:boto:Method: GET 2022-09-03T07:26:02.822 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:26:02.823 DEBUG:boto:Data: 2022-09-03T07:26:02.823 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.824 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.825 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.825 DEBUG:boto:Params: {} 2022-09-03T07:26:02.826 DEBUG:boto:Token: None 2022-09-03T07:26:02.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.831 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002ee09ef67e46ca22-006313018a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.832 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.833 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.833 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.834 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.835 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:26:02.835 DEBUG:boto:Data: 2022-09-03T07:26:02.836 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.836 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.837 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.837 DEBUG:boto:Params: {} 2022-09-03T07:26:02.838 DEBUG:boto:Token: None 2022-09-03T07:26:02.839 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85 2022-09-03T07:26:02.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7JzafS131Eu8cWKbhKUjuxeRDeU= 2022-09-03T07:26:02.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7JzafS131Eu8cWKbhKUjuxeRDeU=', 'Content-Length': '0'} 2022-09-03T07:26:02.842 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086136e4a33101475-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.843 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.843 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.844 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.844 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:26:02.845 DEBUG:boto:Data: 2022-09-03T07:26:02.845 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.846 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.847 DEBUG:boto:Params: {} 2022-09-03T07:26:02.848 DEBUG:boto:Token: None 2022-09-03T07:26:02.848 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85 2022-09-03T07:26:02.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7JzafS131Eu8cWKbhKUjuxeRDeU= 2022-09-03T07:26:02.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7JzafS131Eu8cWKbhKUjuxeRDeU=', 'Content-Length': '0'} 2022-09-03T07:26:02.852 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e805491145ee920a-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.852 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:26:02.853 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.854 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.854 DEBUG:boto:Method: GET 2022-09-03T07:26:02.855 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:26:02.855 DEBUG:boto:Data: 2022-09-03T07:26:02.856 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.856 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.857 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.857 DEBUG:boto:Params: {} 2022-09-03T07:26:02.858 DEBUG:boto:Token: None 2022-09-03T07:26:02.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85 2022-09-03T07:26:02.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wZSxoPTKQXOrdYBCIOvdG4ueJJM= 2022-09-03T07:26:02.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wZSxoPTKQXOrdYBCIOvdG4ueJJM=', 'Content-Length': '0'} 2022-09-03T07:26:02.862 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000870dcb3f990094ed-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.863 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:02.863 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:02.864 DEBUG:boto:Method: GET 2022-09-03T07:26:02.865 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:26:02.865 DEBUG:boto:Data: 2022-09-03T07:26:02.866 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.867 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.867 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.868 DEBUG:boto:Params: {} 2022-09-03T07:26:02.869 DEBUG:boto:Token: None 2022-09-03T07:26:02.870 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj85 2022-09-03T07:26:02.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wZSxoPTKQXOrdYBCIOvdG4ueJJM= 2022-09-03T07:26:02.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wZSxoPTKQXOrdYBCIOvdG4ueJJM=', 'Content-Length': '0'} 2022-09-03T07:26:02.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008de2bef38b4429f7-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.918 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.919 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.920 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:02.920 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:02.921 DEBUG:boto:Method: GET 2022-09-03T07:26:02.921 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:26:02.922 DEBUG:boto:Data: 2022-09-03T07:26:02.922 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.923 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.924 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.924 DEBUG:boto:Params: {} 2022-09-03T07:26:02.925 DEBUG:boto:Token: None 2022-09-03T07:26:02.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.930 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4d3dc660f089dbe-006313018a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.931 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.932 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:02.933 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:02.934 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:02.935 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:02.936 DEBUG:boto:Method: GET 2022-09-03T07:26:02.937 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:26:02.938 DEBUG:boto:Data: 2022-09-03T07:26:02.939 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.940 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.941 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.942 DEBUG:boto:Params: {} 2022-09-03T07:26:02.943 DEBUG:boto:Token: None 2022-09-03T07:26:02.944 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/ 2022-09-03T07:26:02.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk= 2022-09-03T07:26:02.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6dEQwIUpFCAfrqa2RcGtdXxPIk=', 'Content-Length': '0'} 2022-09-03T07:26:02.950 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad7e515e4a299ff7-006313018a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.951 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:02.952 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:26:02.952 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:02.953 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:02.954 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.954 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.955 DEBUG:boto:Method: GET 2022-09-03T07:26:02.955 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.956 DEBUG:boto:Data: 2022-09-03T07:26:02.957 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.958 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.958 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.959 DEBUG:boto:Params: {} 2022-09-03T07:26:02.960 DEBUG:boto:Token: None 2022-09-03T07:26:02.961 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Lgb68Nypg/vlgWXzQ8VQXMmVCc= 2022-09-03T07:26:02.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Lgb68Nypg/vlgWXzQ8VQXMmVCc=', 'Content-Length': '0'} 2022-09-03T07:26:02.965 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000280a3c8acff10be6-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.966 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:02.967 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:02.968 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.969 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.970 DEBUG:boto:Method: GET 2022-09-03T07:26:02.971 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.972 DEBUG:boto:Data: 2022-09-03T07:26:02.973 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:02.975 DEBUG:boto:Port: 8000 2022-09-03T07:26:02.976 DEBUG:boto:Params: {} 2022-09-03T07:26:02.977 DEBUG:boto:Token: None 2022-09-03T07:26:02.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3Lgb68Nypg/vlgWXzQ8VQXMmVCc= 2022-09-03T07:26:02.979 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3Lgb68Nypg/vlgWXzQ8VQXMmVCc=', 'Content-Length': '0'} 2022-09-03T07:26:02.981 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045d6e02724d20e81-006313018a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:02.983 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:02.983 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:02.984 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.985 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.986 DEBUG:boto:Method: HEAD 2022-09-03T07:26:02.987 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.988 DEBUG:boto:Data: 2022-09-03T07:26:02.989 DEBUG:boto:Headers: {} 2022-09-03T07:26:02.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:02.992 DEBUG:boto:Port: 8001 2022-09-03T07:26:02.993 DEBUG:boto:Params: {} 2022-09-03T07:26:02.995 DEBUG:boto:Token: None 2022-09-03T07:26:02.996 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:02 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:02.997 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OgonjhRYBMZBjRAk7+0ep1CXaA= 2022-09-03T07:26:02.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OgonjhRYBMZBjRAk7+0ep1CXaA=', 'Content-Length': '0'} 2022-09-03T07:26:03.000 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f748482277d25141-006313018a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.001 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:03.003 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:03.004 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.004 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.005 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.006 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.007 DEBUG:boto:Data: 2022-09-03T07:26:03.008 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.010 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.011 DEBUG:boto:Params: {} 2022-09-03T07:26:03.012 DEBUG:boto:Token: None 2022-09-03T07:26:03.013 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:olk925vGcs74eQRzfAf0/I5cMD8= 2022-09-03T07:26:03.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:olk925vGcs74eQRzfAf0/I5cMD8=', 'Content-Length': '0'} 2022-09-03T07:26:03.018 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000248309ea9bf48608-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.019 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:26:03.020 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:03.021 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:03.022 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.022 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.023 DEBUG:boto:Method: GET 2022-09-03T07:26:03.024 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.025 DEBUG:boto:Data: 2022-09-03T07:26:03.026 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.027 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.028 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.029 DEBUG:boto:Params: {} 2022-09-03T07:26:03.030 DEBUG:boto:Token: None 2022-09-03T07:26:03.031 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZWR8YDxqD9Y0jjY+cksuDX5fmPg= 2022-09-03T07:26:03.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZWR8YDxqD9Y0jjY+cksuDX5fmPg=', 'Content-Length': '0'} 2022-09-03T07:26:03.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bffb1168f87e9576-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.039 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:03.040 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:03.041 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.041 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.043 DEBUG:boto:Method: GET 2022-09-03T07:26:03.044 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.045 DEBUG:boto:Data: 2022-09-03T07:26:03.046 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.047 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.048 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.049 DEBUG:boto:Params: {} 2022-09-03T07:26:03.050 DEBUG:boto:Token: None 2022-09-03T07:26:03.051 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:03.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZWR8YDxqD9Y0jjY+cksuDX5fmPg= 2022-09-03T07:26:03.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZWR8YDxqD9Y0jjY+cksuDX5fmPg=', 'Content-Length': '0'} 2022-09-03T07:26:03.062 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3ce53ba2e7fbb19-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.064 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.065 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.066 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:03.066 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:03.067 DEBUG:boto:Method: GET 2022-09-03T07:26:03.068 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:26:03.069 DEBUG:boto:Data: 2022-09-03T07:26:03.069 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.070 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.071 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.072 DEBUG:boto:Params: {} 2022-09-03T07:26:03.073 DEBUG:boto:Token: None 2022-09-03T07:26:03.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.080 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc9d2bb8b2d1c142-006313018b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.082 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.089 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.090 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.090 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:03.091 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:03.091 DEBUG:boto:Method: GET 2022-09-03T07:26:03.092 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:26:03.092 DEBUG:boto:Data: 2022-09-03T07:26:03.093 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.094 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.094 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.095 DEBUG:boto:Params: {} 2022-09-03T07:26:03.095 DEBUG:boto:Token: None 2022-09-03T07:26:03.096 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.101 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da44e00b56921633-006313018b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.142 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.144 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:03.145 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:03.145 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.147 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:26:03.147 DEBUG:boto:Data: 2022-09-03T07:26:03.148 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.149 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.150 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.151 DEBUG:boto:Params: {} 2022-09-03T07:26:03.152 DEBUG:boto:Token: None 2022-09-03T07:26:03.153 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj87 2022-09-03T07:26:03.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UR89mszzOIM6JsLvjjAoUiIFCWo= 2022-09-03T07:26:03.155 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UR89mszzOIM6JsLvjjAoUiIFCWo=', 'Content-Length': '0'} 2022-09-03T07:26:03.157 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c27ab27700f1ba2-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.159 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:03.159 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:03.160 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.161 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:26:03.162 DEBUG:boto:Data: 2022-09-03T07:26:03.163 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.164 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.164 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.171 DEBUG:boto:Params: {} 2022-09-03T07:26:03.172 DEBUG:boto:Token: None 2022-09-03T07:26:03.173 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj87 2022-09-03T07:26:03.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UR89mszzOIM6JsLvjjAoUiIFCWo= 2022-09-03T07:26:03.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UR89mszzOIM6JsLvjjAoUiIFCWo=', 'Content-Length': '0'} 2022-09-03T07:26:03.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e97d52b51f9048f4-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.180 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:26:03.181 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:03.181 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:03.182 DEBUG:boto:Method: GET 2022-09-03T07:26:03.182 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:26:03.183 DEBUG:boto:Data: 2022-09-03T07:26:03.183 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.185 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.185 DEBUG:boto:Params: {} 2022-09-03T07:26:03.186 DEBUG:boto:Token: None 2022-09-03T07:26:03.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj87 2022-09-03T07:26:03.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Y35602wDD6nPjjNpu40KL8MzXQ= 2022-09-03T07:26:03.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Y35602wDD6nPjjNpu40KL8MzXQ=', 'Content-Length': '0'} 2022-09-03T07:26:03.190 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000758f123bb1b1ba2c-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.191 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:03.191 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:03.192 DEBUG:boto:Method: GET 2022-09-03T07:26:03.192 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:26:03.193 DEBUG:boto:Data: 2022-09-03T07:26:03.193 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.195 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.195 DEBUG:boto:Params: {} 2022-09-03T07:26:03.196 DEBUG:boto:Token: None 2022-09-03T07:26:03.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj87 2022-09-03T07:26:03.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Y35602wDD6nPjjNpu40KL8MzXQ= 2022-09-03T07:26:03.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Y35602wDD6nPjjNpu40KL8MzXQ=', 'Content-Length': '0'} 2022-09-03T07:26:03.201 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f514f7fd92fa354d-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.202 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.203 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.204 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:03.205 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:03.206 DEBUG:boto:Method: GET 2022-09-03T07:26:03.207 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:26:03.208 DEBUG:boto:Data: 2022-09-03T07:26:03.209 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.212 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.213 DEBUG:boto:Params: {} 2022-09-03T07:26:03.214 DEBUG:boto:Token: None 2022-09-03T07:26:03.215 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.221 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000035b296b068bc9ae5-006313018b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.221 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.222 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.223 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.224 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:03.224 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:03.225 DEBUG:boto:Method: GET 2022-09-03T07:26:03.226 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:26:03.227 DEBUG:boto:Data: 2022-09-03T07:26:03.228 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.229 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.229 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.231 DEBUG:boto:Params: {} 2022-09-03T07:26:03.232 DEBUG:boto:Token: None 2022-09-03T07:26:03.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.239 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000047b41da2e53c4a43-006313018b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.240 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.241 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:26:03.241 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.242 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.243 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.244 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.245 DEBUG:boto:Method: GET 2022-09-03T07:26:03.246 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.246 DEBUG:boto:Data: 2022-09-03T07:26:03.247 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.250 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.250 DEBUG:boto:Params: {} 2022-09-03T07:26:03.251 DEBUG:boto:Token: None 2022-09-03T07:26:03.251 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.252 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QLVSa62I3IzcKoqOZ6UHQ/8+55o= 2022-09-03T07:26:03.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QLVSa62I3IzcKoqOZ6UHQ/8+55o=', 'Content-Length': '0'} 2022-09-03T07:26:03.266 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f30c4e388d995145-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.267 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.268 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.269 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.270 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.271 DEBUG:boto:Method: GET 2022-09-03T07:26:03.272 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.273 DEBUG:boto:Data: 2022-09-03T07:26:03.274 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.275 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.276 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.277 DEBUG:boto:Params: {} 2022-09-03T07:26:03.278 DEBUG:boto:Token: None 2022-09-03T07:26:03.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QLVSa62I3IzcKoqOZ6UHQ/8+55o= 2022-09-03T07:26:03.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QLVSa62I3IzcKoqOZ6UHQ/8+55o=', 'Content-Length': '0'} 2022-09-03T07:26:03.284 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010e4b9286cdd922f-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.331 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.333 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.334 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.335 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.335 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.336 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.337 DEBUG:boto:Data: 2022-09-03T07:26:03.338 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.340 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.341 DEBUG:boto:Params: {} 2022-09-03T07:26:03.342 DEBUG:boto:Token: None 2022-09-03T07:26:03.343 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.343 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MIc+Jp1QVjAwWosYbz5yK+h1SdU= 2022-09-03T07:26:03.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MIc+Jp1QVjAwWosYbz5yK+h1SdU=', 'Content-Length': '0'} 2022-09-03T07:26:03.347 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006350256a3851bcc0-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.348 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.349 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.350 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.353 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.354 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.355 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.356 DEBUG:boto:Data: 2022-09-03T07:26:03.357 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.359 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.359 DEBUG:boto:Params: {} 2022-09-03T07:26:03.360 DEBUG:boto:Token: None 2022-09-03T07:26:03.361 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MIc+Jp1QVjAwWosYbz5yK+h1SdU= 2022-09-03T07:26:03.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MIc+Jp1QVjAwWosYbz5yK+h1SdU=', 'Content-Length': '0'} 2022-09-03T07:26:03.364 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000f01868a1e4ca51-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.366 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:26:03.367 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.368 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.368 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.369 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.370 DEBUG:boto:Method: GET 2022-09-03T07:26:03.371 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.372 DEBUG:boto:Data: 2022-09-03T07:26:03.373 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.374 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.375 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.375 DEBUG:boto:Params: {} 2022-09-03T07:26:03.376 DEBUG:boto:Token: None 2022-09-03T07:26:03.377 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QLVSa62I3IzcKoqOZ6UHQ/8+55o= 2022-09-03T07:26:03.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QLVSa62I3IzcKoqOZ6UHQ/8+55o=', 'Content-Length': '0'} 2022-09-03T07:26:03.383 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbd060c3ebc41916-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.385 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.386 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.386 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.387 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.387 DEBUG:boto:Method: GET 2022-09-03T07:26:03.388 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.389 DEBUG:boto:Data: 2022-09-03T07:26:03.389 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.390 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.391 DEBUG:boto:Params: {} 2022-09-03T07:26:03.391 DEBUG:boto:Token: None 2022-09-03T07:26:03.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:03.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QLVSa62I3IzcKoqOZ6UHQ/8+55o= 2022-09-03T07:26:03.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QLVSa62I3IzcKoqOZ6UHQ/8+55o=', 'Content-Length': '0'} 2022-09-03T07:26:03.395 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f066b6c18d39a128-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.396 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.397 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.397 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:03.398 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:03.398 DEBUG:boto:Method: GET 2022-09-03T07:26:03.399 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:26:03.399 DEBUG:boto:Data: 2022-09-03T07:26:03.400 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.401 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.401 DEBUG:boto:Params: {} 2022-09-03T07:26:03.402 DEBUG:boto:Token: None 2022-09-03T07:26:03.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.410 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000026a187d2d332a544-006313018b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.450 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.452 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.452 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.453 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:03.455 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:03.456 DEBUG:boto:Method: GET 2022-09-03T07:26:03.458 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:26:03.458 DEBUG:boto:Data: 2022-09-03T07:26:03.460 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.461 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.462 DEBUG:boto:Params: {} 2022-09-03T07:26:03.463 DEBUG:boto:Token: None 2022-09-03T07:26:03.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.472 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000015e5da697335771b-006313018b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.473 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.475 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.476 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.477 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.478 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:26:03.479 DEBUG:boto:Data: 2022-09-03T07:26:03.480 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.481 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.482 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.482 DEBUG:boto:Params: {} 2022-09-03T07:26:03.483 DEBUG:boto:Token: None 2022-09-03T07:26:03.484 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92 2022-09-03T07:26:03.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wkt67CLy0n8WrRTQIlOrIZfxWy0= 2022-09-03T07:26:03.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wkt67CLy0n8WrRTQIlOrIZfxWy0=', 'Content-Length': '0'} 2022-09-03T07:26:03.487 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b235ef5e1d170f28-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.488 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.488 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.489 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.489 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:26:03.490 DEBUG:boto:Data: 2022-09-03T07:26:03.490 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.491 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.492 DEBUG:boto:Params: {} 2022-09-03T07:26:03.493 DEBUG:boto:Token: None 2022-09-03T07:26:03.493 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92 2022-09-03T07:26:03.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wkt67CLy0n8WrRTQIlOrIZfxWy0= 2022-09-03T07:26:03.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wkt67CLy0n8WrRTQIlOrIZfxWy0=', 'Content-Length': '0'} 2022-09-03T07:26:03.497 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000835cffe70fd7800f-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.497 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:26:03.498 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.499 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.499 DEBUG:boto:Method: GET 2022-09-03T07:26:03.500 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:26:03.500 DEBUG:boto:Data: 2022-09-03T07:26:03.501 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.502 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.503 DEBUG:boto:Params: {} 2022-09-03T07:26:03.503 DEBUG:boto:Token: None 2022-09-03T07:26:03.504 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92 2022-09-03T07:26:03.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yxzT6Ye1Pe+Xk8vwVoff+Z9x8do= 2022-09-03T07:26:03.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yxzT6Ye1Pe+Xk8vwVoff+Z9x8do=', 'Content-Length': '0'} 2022-09-03T07:26:03.508 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020c9c5843a664915-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.508 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:03.509 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:03.509 DEBUG:boto:Method: GET 2022-09-03T07:26:03.510 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:26:03.511 DEBUG:boto:Data: 2022-09-03T07:26:03.511 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.512 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.513 DEBUG:boto:Params: {} 2022-09-03T07:26:03.513 DEBUG:boto:Token: None 2022-09-03T07:26:03.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj92 2022-09-03T07:26:03.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yxzT6Ye1Pe+Xk8vwVoff+Z9x8do= 2022-09-03T07:26:03.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yxzT6Ye1Pe+Xk8vwVoff+Z9x8do=', 'Content-Length': '0'} 2022-09-03T07:26:03.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002305a08a80c7eec1-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.519 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.520 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.521 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:03.521 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:03.522 DEBUG:boto:Method: GET 2022-09-03T07:26:03.522 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:26:03.523 DEBUG:boto:Data: 2022-09-03T07:26:03.523 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.525 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.525 DEBUG:boto:Params: {} 2022-09-03T07:26:03.526 DEBUG:boto:Token: None 2022-09-03T07:26:03.526 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.530 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b6631e44f126d737-006313018b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.531 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.532 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.532 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.533 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:03.533 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:03.534 DEBUG:boto:Method: GET 2022-09-03T07:26:03.535 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:26:03.535 DEBUG:boto:Data: 2022-09-03T07:26:03.536 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.536 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.537 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.537 DEBUG:boto:Params: {} 2022-09-03T07:26:03.538 DEBUG:boto:Token: None 2022-09-03T07:26:03.539 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.543 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce3cd03a3c2542cc-006313018b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.544 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.545 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:26:03.546 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.546 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.547 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.547 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.548 DEBUG:boto:Method: GET 2022-09-03T07:26:03.549 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.550 DEBUG:boto:Data: 2022-09-03T07:26:03.551 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.552 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.552 DEBUG:boto:Params: {} 2022-09-03T07:26:03.553 DEBUG:boto:Token: None 2022-09-03T07:26:03.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dEkNLwjE6e4FiTFf7EUtBVmOGaU= 2022-09-03T07:26:03.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dEkNLwjE6e4FiTFf7EUtBVmOGaU=', 'Content-Length': '0'} 2022-09-03T07:26:03.557 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000458309e625f2dc6-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.601 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.602 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.603 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.605 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.607 DEBUG:boto:Method: GET 2022-09-03T07:26:03.607 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.608 DEBUG:boto:Data: 2022-09-03T07:26:03.608 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.609 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.610 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.611 DEBUG:boto:Params: {} 2022-09-03T07:26:03.611 DEBUG:boto:Token: None 2022-09-03T07:26:03.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dEkNLwjE6e4FiTFf7EUtBVmOGaU= 2022-09-03T07:26:03.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dEkNLwjE6e4FiTFf7EUtBVmOGaU=', 'Content-Length': '0'} 2022-09-03T07:26:03.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008bbb744950b352f8-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.619 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.619 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.620 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.620 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.621 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.621 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.622 DEBUG:boto:Data: 2022-09-03T07:26:03.622 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.623 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.623 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.624 DEBUG:boto:Params: {} 2022-09-03T07:26:03.624 DEBUG:boto:Token: None 2022-09-03T07:26:03.625 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nQ7dX2tnjVlGkShv0yBPCB0oDI0= 2022-09-03T07:26:03.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nQ7dX2tnjVlGkShv0yBPCB0oDI0=', 'Content-Length': '0'} 2022-09-03T07:26:03.628 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086a689502469f6c8-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.629 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.629 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.630 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.630 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.631 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.631 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.632 DEBUG:boto:Data: 2022-09-03T07:26:03.632 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.633 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.633 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.634 DEBUG:boto:Params: {} 2022-09-03T07:26:03.634 DEBUG:boto:Token: None 2022-09-03T07:26:03.635 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nQ7dX2tnjVlGkShv0yBPCB0oDI0= 2022-09-03T07:26:03.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nQ7dX2tnjVlGkShv0yBPCB0oDI0=', 'Content-Length': '0'} 2022-09-03T07:26:03.638 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a1cd5b2adb2764e-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.639 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:26:03.639 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.640 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.641 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.641 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.642 DEBUG:boto:Method: GET 2022-09-03T07:26:03.642 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.643 DEBUG:boto:Data: 2022-09-03T07:26:03.643 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.644 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.645 DEBUG:boto:Params: {} 2022-09-03T07:26:03.646 DEBUG:boto:Token: None 2022-09-03T07:26:03.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.647 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dEkNLwjE6e4FiTFf7EUtBVmOGaU= 2022-09-03T07:26:03.648 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dEkNLwjE6e4FiTFf7EUtBVmOGaU=', 'Content-Length': '0'} 2022-09-03T07:26:03.650 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001027f75b1f216386-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.651 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.652 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.652 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.653 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.653 DEBUG:boto:Method: GET 2022-09-03T07:26:03.654 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.654 DEBUG:boto:Data: 2022-09-03T07:26:03.655 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.656 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.657 DEBUG:boto:Params: {} 2022-09-03T07:26:03.657 DEBUG:boto:Token: None 2022-09-03T07:26:03.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:03.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dEkNLwjE6e4FiTFf7EUtBVmOGaU= 2022-09-03T07:26:03.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dEkNLwjE6e4FiTFf7EUtBVmOGaU=', 'Content-Length': '0'} 2022-09-03T07:26:03.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e96971586d96664-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.707 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.708 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.709 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:03.710 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:03.710 DEBUG:boto:Method: GET 2022-09-03T07:26:03.711 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:26:03.713 DEBUG:boto:Data: 2022-09-03T07:26:03.714 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.714 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.715 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.716 DEBUG:boto:Params: {} 2022-09-03T07:26:03.716 DEBUG:boto:Token: None 2022-09-03T07:26:03.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.721 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063ad80f899281fd1-006313018b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.721 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.722 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.723 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.724 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:03.725 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:03.726 DEBUG:boto:Method: GET 2022-09-03T07:26:03.726 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:26:03.727 DEBUG:boto:Data: 2022-09-03T07:26:03.728 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.729 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.730 DEBUG:boto:Params: {} 2022-09-03T07:26:03.730 DEBUG:boto:Token: None 2022-09-03T07:26:03.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.737 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c6f33160f4944599-006313018b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.738 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.739 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.740 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.740 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.741 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:26:03.741 DEBUG:boto:Data: 2022-09-03T07:26:03.742 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.743 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.744 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.745 DEBUG:boto:Params: {} 2022-09-03T07:26:03.746 DEBUG:boto:Token: None 2022-09-03T07:26:03.747 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95 2022-09-03T07:26:03.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ooBaz/bLMDMchaYarfaA1d3I1aU= 2022-09-03T07:26:03.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ooBaz/bLMDMchaYarfaA1d3I1aU=', 'Content-Length': '0'} 2022-09-03T07:26:03.750 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000798a4518e3558923-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.751 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.752 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.752 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.753 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:26:03.754 DEBUG:boto:Data: 2022-09-03T07:26:03.755 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.756 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.756 DEBUG:boto:Params: {} 2022-09-03T07:26:03.757 DEBUG:boto:Token: None 2022-09-03T07:26:03.758 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95 2022-09-03T07:26:03.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ooBaz/bLMDMchaYarfaA1d3I1aU= 2022-09-03T07:26:03.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ooBaz/bLMDMchaYarfaA1d3I1aU=', 'Content-Length': '0'} 2022-09-03T07:26:03.761 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ba561407e926bbd-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.761 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:26:03.762 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.763 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.763 DEBUG:boto:Method: GET 2022-09-03T07:26:03.764 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:26:03.764 DEBUG:boto:Data: 2022-09-03T07:26:03.765 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.766 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.767 DEBUG:boto:Params: {} 2022-09-03T07:26:03.767 DEBUG:boto:Token: None 2022-09-03T07:26:03.768 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95 2022-09-03T07:26:03.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k2fbLNw/Nq94iYTAMj/Of0xzkhQ= 2022-09-03T07:26:03.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k2fbLNw/Nq94iYTAMj/Of0xzkhQ=', 'Content-Length': '0'} 2022-09-03T07:26:03.771 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005b02eff8cfa72ced-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.772 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:03.773 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:03.773 DEBUG:boto:Method: GET 2022-09-03T07:26:03.774 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:26:03.774 DEBUG:boto:Data: 2022-09-03T07:26:03.775 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.775 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.776 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.777 DEBUG:boto:Params: {} 2022-09-03T07:26:03.777 DEBUG:boto:Token: None 2022-09-03T07:26:03.778 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj95 2022-09-03T07:26:03.779 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k2fbLNw/Nq94iYTAMj/Of0xzkhQ= 2022-09-03T07:26:03.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k2fbLNw/Nq94iYTAMj/Of0xzkhQ=', 'Content-Length': '0'} 2022-09-03T07:26:03.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf27dc4b4743b54d-006313018b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.782 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.783 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.783 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:03.784 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:03.784 DEBUG:boto:Method: GET 2022-09-03T07:26:03.785 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:26:03.785 DEBUG:boto:Data: 2022-09-03T07:26:03.786 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.787 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.788 DEBUG:boto:Params: {} 2022-09-03T07:26:03.788 DEBUG:boto:Token: None 2022-09-03T07:26:03.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.793 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f01ff136c1aea757-006313018b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.794 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.795 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.796 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.797 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:03.798 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:03.799 DEBUG:boto:Method: GET 2022-09-03T07:26:03.800 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:26:03.800 DEBUG:boto:Data: 2022-09-03T07:26:03.801 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.801 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:03.802 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.802 DEBUG:boto:Params: {} 2022-09-03T07:26:03.803 DEBUG:boto:Token: None 2022-09-03T07:26:03.804 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.804 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8= 2022-09-03T07:26:03.805 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4wRcgmZ35ZeLl7JvexF0ifBEbg8=', 'Content-Length': '0'} 2022-09-03T07:26:03.810 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000146cbc118baba828-006313018b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.811 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.812 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:26:03.813 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:26:03.813 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.814 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.815 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.816 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:26:03.817 DEBUG:boto:Data: 2022-09-03T07:26:03.818 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.820 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.821 DEBUG:boto:Params: {} 2022-09-03T07:26:03.822 DEBUG:boto:Token: None 2022-09-03T07:26:03.823 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NpC4guTABbsNbA5wQEHz0swy1nE= 2022-09-03T07:26:03.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NpC4guTABbsNbA5wQEHz0swy1nE=', 'Content-Length': '0'} 2022-09-03T07:26:03.828 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '25'), ('X-RGW-Bytes-Used', '75'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000005496e8d92cc7b716-006313018b-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.829 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.829 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.830 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.830 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:26:03.831 DEBUG:boto:Data: 2022-09-03T07:26:03.832 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:03.833 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.833 DEBUG:boto:Params: {} 2022-09-03T07:26:03.834 DEBUG:boto:Token: None 2022-09-03T07:26:03.835 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/ 2022-09-03T07:26:03.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NpC4guTABbsNbA5wQEHz0swy1nE= 2022-09-03T07:26:03.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NpC4guTABbsNbA5wQEHz0swy1nE=', 'Content-Length': '0'} 2022-09-03T07:26:03.840 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '25'), ('X-RGW-Bytes-Used', '75'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009b9c4b80f5d34172-006313018b-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.841 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:26:03.842 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.843 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.844 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.845 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.846 DEBUG:boto:Method: GET 2022-09-03T07:26:03.847 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.848 DEBUG:boto:Data: 2022-09-03T07:26:03.848 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.849 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.850 DEBUG:boto:Params: {} 2022-09-03T07:26:03.850 DEBUG:boto:Token: None 2022-09-03T07:26:03.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/?versions 2022-09-03T07:26:03.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Fz4aDKk4kP/LWlQDKBnPig6YA0= 2022-09-03T07:26:03.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Fz4aDKk4kP/LWlQDKBnPig6YA0=', 'Content-Length': '0'} 2022-09-03T07:26:03.855 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ba7ad2bb4159a92b-006313018b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.856 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.858 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:26:03.858 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:26:03.859 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:26:03.859 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:26:03.860 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:26:03.860 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:26:03.861 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:26:03.861 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:26:03.862 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:26:03.862 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:26:03.862 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:26:03.863 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:26:03.863 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:26:03.864 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:26:03.864 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:26:03.865 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:26:03.865 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:26:03.866 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:26:03.866 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:26:03.867 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:26:03.867 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:26:03.867 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:26:03.868 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:26:03.868 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:26:03.869 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:26:03.869 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:26:03.870 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.871 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.871 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.872 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.872 DEBUG:boto:Method: GET 2022-09-03T07:26:03.873 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.873 DEBUG:boto:Data: 2022-09-03T07:26:03.874 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.874 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:03.875 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.875 DEBUG:boto:Params: {} 2022-09-03T07:26:03.876 DEBUG:boto:Token: None 2022-09-03T07:26:03.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/?versions 2022-09-03T07:26:03.877 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Fz4aDKk4kP/LWlQDKBnPig6YA0= 2022-09-03T07:26:03.878 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Fz4aDKk4kP/LWlQDKBnPig6YA0=', 'Content-Length': '0'} 2022-09-03T07:26:03.880 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d90351506d9521db-006313018b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.881 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.883 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:26:03.884 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:26:03.884 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:26:03.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:26:03.885 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:26:03.886 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:26:03.886 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:26:03.887 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:26:03.887 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:26:03.887 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:26:03.888 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:26:03.888 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:26:03.889 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:26:03.889 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:26:03.890 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:26:03.890 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:26:03.891 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:26:03.891 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:26:03.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:26:03.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:26:03.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:26:03.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:26:03.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:26:03.894 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:26:03.894 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:26:03.895 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.896 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.896 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.897 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.897 DEBUG:boto:Method: GET 2022-09-03T07:26:03.898 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.898 DEBUG:boto:Data: 2022-09-03T07:26:03.899 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.899 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.900 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.900 DEBUG:boto:Params: {} 2022-09-03T07:26:03.901 DEBUG:boto:Token: None 2022-09-03T07:26:03.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/?versions 2022-09-03T07:26:03.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Fz4aDKk4kP/LWlQDKBnPig6YA0= 2022-09-03T07:26:03.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Fz4aDKk4kP/LWlQDKBnPig6YA0=', 'Content-Length': '0'} 2022-09-03T07:26:03.905 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000051bd4b79fe5899b4-006313018b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.906 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.908 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.909 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.909 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.910 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.910 DEBUG:boto:Method: GET 2022-09-03T07:26:03.911 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:26:03.911 DEBUG:boto:Data: 2022-09-03T07:26:03.912 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.912 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:03.913 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.913 DEBUG:boto:Params: {} 2022-09-03T07:26:03.914 DEBUG:boto:Token: None 2022-09-03T07:26:03.914 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/?versions 2022-09-03T07:26:03.915 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Fz4aDKk4kP/LWlQDKBnPig6YA0= 2022-09-03T07:26:03.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Fz4aDKk4kP/LWlQDKBnPig6YA0=', 'Content-Length': '0'} 2022-09-03T07:26:03.918 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ee18e3cc0fc70354-006313018b-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.919 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:03.921 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:26:03.922 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:03.922 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:03.923 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.923 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.924 DEBUG:boto:Method: GET 2022-09-03T07:26:03.924 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.925 DEBUG:boto:Data: 2022-09-03T07:26:03.926 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.926 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.927 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.927 DEBUG:boto:Params: {} 2022-09-03T07:26:03.928 DEBUG:boto:Token: None 2022-09-03T07:26:03.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NUsz+DPxMRiEk9drAoIhsfBtcFg= 2022-09-03T07:26:03.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NUsz+DPxMRiEk9drAoIhsfBtcFg=', 'Content-Length': '0'} 2022-09-03T07:26:03.932 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a0dc6fbf4cfcb66-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.933 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:03.934 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:03.934 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.935 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.935 DEBUG:boto:Method: GET 2022-09-03T07:26:03.936 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.936 DEBUG:boto:Data: 2022-09-03T07:26:03.937 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.938 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:03.938 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.939 DEBUG:boto:Params: {} 2022-09-03T07:26:03.939 DEBUG:boto:Token: None 2022-09-03T07:26:03.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NUsz+DPxMRiEk9drAoIhsfBtcFg= 2022-09-03T07:26:03.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NUsz+DPxMRiEk9drAoIhsfBtcFg=', 'Content-Length': '0'} 2022-09-03T07:26:03.943 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f2f783cce4160dd-006313018b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.944 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:03.944 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:03.945 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.945 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.946 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.946 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.947 DEBUG:boto:Data: 2022-09-03T07:26:03.947 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.948 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.949 DEBUG:boto:Params: {} 2022-09-03T07:26:03.949 DEBUG:boto:Token: None 2022-09-03T07:26:03.951 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IG2MBP93V2Mc586N2pdjYJfKkr0= 2022-09-03T07:26:03.952 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IG2MBP93V2Mc586N2pdjYJfKkr0=', 'Content-Length': '0'} 2022-09-03T07:26:03.954 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bf37916cb583461-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.955 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:03.955 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:03.956 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.956 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.957 DEBUG:boto:Method: HEAD 2022-09-03T07:26:03.957 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.958 DEBUG:boto:Data: 2022-09-03T07:26:03.958 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.959 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:03.959 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.960 DEBUG:boto:Params: {} 2022-09-03T07:26:03.960 DEBUG:boto:Token: None 2022-09-03T07:26:03.961 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IG2MBP93V2Mc586N2pdjYJfKkr0= 2022-09-03T07:26:03.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IG2MBP93V2Mc586N2pdjYJfKkr0=', 'Content-Length': '0'} 2022-09-03T07:26:03.965 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000919d2fff8701ff42-006313018b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.966 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:26:03.967 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:03.968 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:03.968 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.969 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.969 DEBUG:boto:Method: GET 2022-09-03T07:26:03.970 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.971 DEBUG:boto:Data: 2022-09-03T07:26:03.971 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.972 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:03.972 DEBUG:boto:Port: 8001 2022-09-03T07:26:03.973 DEBUG:boto:Params: {} 2022-09-03T07:26:03.974 DEBUG:boto:Token: None 2022-09-03T07:26:03.974 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NUsz+DPxMRiEk9drAoIhsfBtcFg= 2022-09-03T07:26:03.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NUsz+DPxMRiEk9drAoIhsfBtcFg=', 'Content-Length': '0'} 2022-09-03T07:26:03.978 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cce2740b1aeaf85-006313018b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.980 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:03.981 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:03.984 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.985 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.986 DEBUG:boto:Method: GET 2022-09-03T07:26:03.987 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.987 DEBUG:boto:Data: 2022-09-03T07:26:03.988 DEBUG:boto:Headers: {} 2022-09-03T07:26:03.989 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:03.990 DEBUG:boto:Port: 8000 2022-09-03T07:26:03.991 DEBUG:boto:Params: {} 2022-09-03T07:26:03.992 DEBUG:boto:Token: None 2022-09-03T07:26:03.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:26:03.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NUsz+DPxMRiEk9drAoIhsfBtcFg= 2022-09-03T07:26:03.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NUsz+DPxMRiEk9drAoIhsfBtcFg=', 'Content-Length': '0'} 2022-09-03T07:26:03.997 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008336d3ae7c2dce13-006313018b-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:03.998 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:03.999 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:03.999 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:26:04.000 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:26:04.000 DEBUG:boto:Method: GET 2022-09-03T07:26:04.001 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:26:04.001 DEBUG:boto:Data: 2022-09-03T07:26:04.002 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.002 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.003 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.004 DEBUG:boto:Params: {} 2022-09-03T07:26:04.004 DEBUG:boto:Token: None 2022-09-03T07:26:04.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.015 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000829944d9ae8b84df-006313018c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.054 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.056 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.057 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.058 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:26:04.059 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:26:04.064 DEBUG:boto:Method: GET 2022-09-03T07:26:04.065 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:26:04.066 DEBUG:boto:Data: 2022-09-03T07:26:04.067 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.068 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.069 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.070 DEBUG:boto:Params: {} 2022-09-03T07:26:04.072 DEBUG:boto:Token: None 2022-09-03T07:26:04.073 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.074 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.079 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b375881f84a43cdf-006313018c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.080 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.081 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:04.082 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:04.082 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.083 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:26:04.084 DEBUG:boto:Data: 2022-09-03T07:26:04.084 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.085 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.086 DEBUG:boto:Params: {} 2022-09-03T07:26:04.086 DEBUG:boto:Token: None 2022-09-03T07:26:04.087 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj11 2022-09-03T07:26:04.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8P2oMpbwEPwAUdJD8NEOVWR2eIs= 2022-09-03T07:26:04.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8P2oMpbwEPwAUdJD8NEOVWR2eIs=', 'Content-Length': '0'} 2022-09-03T07:26:04.090 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e2298e72a3eb7eb7-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.092 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:04.092 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:04.093 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.093 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:26:04.094 DEBUG:boto:Data: 2022-09-03T07:26:04.094 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.096 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.096 DEBUG:boto:Params: {} 2022-09-03T07:26:04.097 DEBUG:boto:Token: None 2022-09-03T07:26:04.098 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj11 2022-09-03T07:26:04.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8P2oMpbwEPwAUdJD8NEOVWR2eIs= 2022-09-03T07:26:04.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8P2oMpbwEPwAUdJD8NEOVWR2eIs=', 'Content-Length': '0'} 2022-09-03T07:26:04.101 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de9b5dbdd9e4c7bf-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.102 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:26:04.103 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:04.103 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:04.104 DEBUG:boto:Method: GET 2022-09-03T07:26:04.104 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:26:04.105 DEBUG:boto:Data: 2022-09-03T07:26:04.106 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.107 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.107 DEBUG:boto:Params: {} 2022-09-03T07:26:04.108 DEBUG:boto:Token: None 2022-09-03T07:26:04.109 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj11 2022-09-03T07:26:04.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uzZ+3s7VNpwq4KxRUOdd1JBkEO0= 2022-09-03T07:26:04.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uzZ+3s7VNpwq4KxRUOdd1JBkEO0=', 'Content-Length': '0'} 2022-09-03T07:26:04.112 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be112753b8182a6e-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.113 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:26:04.114 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:26:04.114 DEBUG:boto:Method: GET 2022-09-03T07:26:04.115 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:26:04.115 DEBUG:boto:Data: 2022-09-03T07:26:04.116 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.117 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.117 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.118 DEBUG:boto:Params: {} 2022-09-03T07:26:04.118 DEBUG:boto:Token: None 2022-09-03T07:26:04.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj11 2022-09-03T07:26:04.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uzZ+3s7VNpwq4KxRUOdd1JBkEO0= 2022-09-03T07:26:04.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uzZ+3s7VNpwq4KxRUOdd1JBkEO0=', 'Content-Length': '0'} 2022-09-03T07:26:04.123 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027ad1fe60f5e967a-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.170 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.171 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.172 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:26:04.172 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:26:04.173 DEBUG:boto:Method: GET 2022-09-03T07:26:04.173 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:26:04.174 DEBUG:boto:Data: 2022-09-03T07:26:04.174 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.175 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.176 DEBUG:boto:Params: {} 2022-09-03T07:26:04.177 DEBUG:boto:Token: None 2022-09-03T07:26:04.177 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.182 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004d0c6fcaa2ce45a9-006313018c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.183 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.184 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.185 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.185 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:26:04.186 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:26:04.187 DEBUG:boto:Method: GET 2022-09-03T07:26:04.187 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:26:04.188 DEBUG:boto:Data: 2022-09-03T07:26:04.188 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.189 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.189 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.190 DEBUG:boto:Params: {} 2022-09-03T07:26:04.191 DEBUG:boto:Token: None 2022-09-03T07:26:04.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.196 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008c8b7d1c0159d9cc-006313018c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.196 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.197 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:26:04.198 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.198 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.199 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.200 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.200 DEBUG:boto:Method: GET 2022-09-03T07:26:04.201 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.202 DEBUG:boto:Data: 2022-09-03T07:26:04.202 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.203 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.204 DEBUG:boto:Params: {} 2022-09-03T07:26:04.205 DEBUG:boto:Token: None 2022-09-03T07:26:04.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TWafFP9zsEStNj3y/0YzA7t5BKM= 2022-09-03T07:26:04.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TWafFP9zsEStNj3y/0YzA7t5BKM=', 'Content-Length': '0'} 2022-09-03T07:26:04.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018517702377308d1-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.210 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.211 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.212 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.212 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.213 DEBUG:boto:Method: GET 2022-09-03T07:26:04.214 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.214 DEBUG:boto:Data: 2022-09-03T07:26:04.215 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.215 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.216 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.217 DEBUG:boto:Params: {} 2022-09-03T07:26:04.217 DEBUG:boto:Token: None 2022-09-03T07:26:04.218 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TWafFP9zsEStNj3y/0YzA7t5BKM= 2022-09-03T07:26:04.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TWafFP9zsEStNj3y/0YzA7t5BKM=', 'Content-Length': '0'} 2022-09-03T07:26:04.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053959514b4883367-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.222 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.223 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.224 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.224 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.225 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.225 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.226 DEBUG:boto:Data: 2022-09-03T07:26:04.226 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.227 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.227 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.228 DEBUG:boto:Params: {} 2022-09-03T07:26:04.228 DEBUG:boto:Token: None 2022-09-03T07:26:04.229 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+NAWPlK7pSz1eAfVQxFdQ3K8Lw= 2022-09-03T07:26:04.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+NAWPlK7pSz1eAfVQxFdQ3K8Lw=', 'Content-Length': '0'} 2022-09-03T07:26:04.232 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c3f0da46e368444-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.233 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.234 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.234 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.235 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.235 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.236 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.237 DEBUG:boto:Data: 2022-09-03T07:26:04.237 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.238 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.239 DEBUG:boto:Params: {} 2022-09-03T07:26:04.239 DEBUG:boto:Token: None 2022-09-03T07:26:04.240 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f+NAWPlK7pSz1eAfVQxFdQ3K8Lw= 2022-09-03T07:26:04.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f+NAWPlK7pSz1eAfVQxFdQ3K8Lw=', 'Content-Length': '0'} 2022-09-03T07:26:04.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091116ae158bcaa07-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.244 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:26:04.244 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.245 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.246 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.246 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.247 DEBUG:boto:Method: GET 2022-09-03T07:26:04.247 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.248 DEBUG:boto:Data: 2022-09-03T07:26:04.249 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.249 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.250 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.251 DEBUG:boto:Params: {} 2022-09-03T07:26:04.251 DEBUG:boto:Token: None 2022-09-03T07:26:04.252 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TWafFP9zsEStNj3y/0YzA7t5BKM= 2022-09-03T07:26:04.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TWafFP9zsEStNj3y/0YzA7t5BKM=', 'Content-Length': '0'} 2022-09-03T07:26:04.255 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004519c826aae0dc1a-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.256 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.257 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.257 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.258 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.259 DEBUG:boto:Method: GET 2022-09-03T07:26:04.259 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.260 DEBUG:boto:Data: 2022-09-03T07:26:04.260 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.261 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.262 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.263 DEBUG:boto:Params: {} 2022-09-03T07:26:04.264 DEBUG:boto:Token: None 2022-09-03T07:26:04.264 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:26:04.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TWafFP9zsEStNj3y/0YzA7t5BKM= 2022-09-03T07:26:04.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TWafFP9zsEStNj3y/0YzA7t5BKM=', 'Content-Length': '0'} 2022-09-03T07:26:04.268 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000598eed000698e475-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.268 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.269 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.270 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:26:04.270 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:26:04.271 DEBUG:boto:Method: GET 2022-09-03T07:26:04.271 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:26:04.272 DEBUG:boto:Data: 2022-09-03T07:26:04.272 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.273 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.274 DEBUG:boto:Params: {} 2022-09-03T07:26:04.275 DEBUG:boto:Token: None 2022-09-03T07:26:04.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.279 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c677a30e1708a81c-006313018c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.280 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.281 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.281 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.282 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:26:04.282 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:26:04.283 DEBUG:boto:Method: GET 2022-09-03T07:26:04.283 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:26:04.284 DEBUG:boto:Data: 2022-09-03T07:26:04.285 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.285 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.286 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.286 DEBUG:boto:Params: {} 2022-09-03T07:26:04.287 DEBUG:boto:Token: None 2022-09-03T07:26:04.288 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.291 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8080b4010186e94-006313018c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.292 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.293 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.293 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.294 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.294 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:26:04.295 DEBUG:boto:Data: 2022-09-03T07:26:04.295 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.296 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.296 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.297 DEBUG:boto:Params: {} 2022-09-03T07:26:04.298 DEBUG:boto:Token: None 2022-09-03T07:26:04.298 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115 2022-09-03T07:26:04.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5El0aUoMBAYfMM6NJZwmG1+Chcg= 2022-09-03T07:26:04.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5El0aUoMBAYfMM6NJZwmG1+Chcg=', 'Content-Length': '0'} 2022-09-03T07:26:04.301 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000061287494495b1aae-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.302 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.303 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.303 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.304 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:26:04.304 DEBUG:boto:Data: 2022-09-03T07:26:04.305 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.305 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.306 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.307 DEBUG:boto:Params: {} 2022-09-03T07:26:04.307 DEBUG:boto:Token: None 2022-09-03T07:26:04.308 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115 2022-09-03T07:26:04.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5El0aUoMBAYfMM6NJZwmG1+Chcg= 2022-09-03T07:26:04.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5El0aUoMBAYfMM6NJZwmG1+Chcg=', 'Content-Length': '0'} 2022-09-03T07:26:04.311 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f11d3e71a981f88-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.311 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:26:04.312 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.313 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.313 DEBUG:boto:Method: GET 2022-09-03T07:26:04.314 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:26:04.315 DEBUG:boto:Data: 2022-09-03T07:26:04.315 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.316 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.316 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.317 DEBUG:boto:Params: {} 2022-09-03T07:26:04.318 DEBUG:boto:Token: None 2022-09-03T07:26:04.318 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115 2022-09-03T07:26:04.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kmkFMmyse4vyD3Lvto64Qi5voxo= 2022-09-03T07:26:04.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kmkFMmyse4vyD3Lvto64Qi5voxo=', 'Content-Length': '0'} 2022-09-03T07:26:04.322 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044915711b1926c7c-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.370 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:26:04.371 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:26:04.372 DEBUG:boto:Method: GET 2022-09-03T07:26:04.373 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:26:04.373 DEBUG:boto:Data: 2022-09-03T07:26:04.374 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.374 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.375 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.375 DEBUG:boto:Params: {} 2022-09-03T07:26:04.376 DEBUG:boto:Token: None 2022-09-03T07:26:04.377 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj115 2022-09-03T07:26:04.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kmkFMmyse4vyD3Lvto64Qi5voxo= 2022-09-03T07:26:04.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kmkFMmyse4vyD3Lvto64Qi5voxo=', 'Content-Length': '0'} 2022-09-03T07:26:04.380 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c639a5f14c97a67a-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.381 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.382 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.383 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:26:04.383 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:26:04.384 DEBUG:boto:Method: GET 2022-09-03T07:26:04.384 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:26:04.385 DEBUG:boto:Data: 2022-09-03T07:26:04.385 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.387 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.387 DEBUG:boto:Params: {} 2022-09-03T07:26:04.388 DEBUG:boto:Token: None 2022-09-03T07:26:04.389 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.393 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b243a7ce26949cf-006313018c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.393 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.394 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.395 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.395 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:26:04.396 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:26:04.396 DEBUG:boto:Method: GET 2022-09-03T07:26:04.397 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:26:04.398 DEBUG:boto:Data: 2022-09-03T07:26:04.398 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.399 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.399 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.400 DEBUG:boto:Params: {} 2022-09-03T07:26:04.401 DEBUG:boto:Token: None 2022-09-03T07:26:04.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.405 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000390a1388a4e97e68-006313018c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.406 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.407 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:26:04.408 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.409 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.409 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.410 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.410 DEBUG:boto:Method: GET 2022-09-03T07:26:04.411 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.411 DEBUG:boto:Data: 2022-09-03T07:26:04.412 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.413 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.413 DEBUG:boto:Params: {} 2022-09-03T07:26:04.414 DEBUG:boto:Token: None 2022-09-03T07:26:04.415 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7o+tzRGIZm9bzc/vFWIgJ9Av8UI= 2022-09-03T07:26:04.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7o+tzRGIZm9bzc/vFWIgJ9Av8UI=', 'Content-Length': '0'} 2022-09-03T07:26:04.418 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1318fcdb4ed108e-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.419 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.419 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.420 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.420 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.421 DEBUG:boto:Method: GET 2022-09-03T07:26:04.421 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.422 DEBUG:boto:Data: 2022-09-03T07:26:04.422 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.423 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.423 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.424 DEBUG:boto:Params: {} 2022-09-03T07:26:04.425 DEBUG:boto:Token: None 2022-09-03T07:26:04.425 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7o+tzRGIZm9bzc/vFWIgJ9Av8UI= 2022-09-03T07:26:04.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7o+tzRGIZm9bzc/vFWIgJ9Av8UI=', 'Content-Length': '0'} 2022-09-03T07:26:04.429 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf0c259970c33c06-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.429 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.430 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.431 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.431 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.432 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.432 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.433 DEBUG:boto:Data: 2022-09-03T07:26:04.433 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.435 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.435 DEBUG:boto:Params: {} 2022-09-03T07:26:04.436 DEBUG:boto:Token: None 2022-09-03T07:26:04.437 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UtxbraSSFF1FvV0EEtSW3csKwP0= 2022-09-03T07:26:04.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UtxbraSSFF1FvV0EEtSW3csKwP0=', 'Content-Length': '0'} 2022-09-03T07:26:04.440 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f6a3090cb8d94f8-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.441 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.441 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.442 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.442 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.443 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.443 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.444 DEBUG:boto:Data: 2022-09-03T07:26:04.444 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.445 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.445 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.446 DEBUG:boto:Params: {} 2022-09-03T07:26:04.447 DEBUG:boto:Token: None 2022-09-03T07:26:04.448 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UtxbraSSFF1FvV0EEtSW3csKwP0= 2022-09-03T07:26:04.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UtxbraSSFF1FvV0EEtSW3csKwP0=', 'Content-Length': '0'} 2022-09-03T07:26:04.452 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055bfc1d7e9a0eefd-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.453 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:26:04.454 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.454 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.455 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.455 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.456 DEBUG:boto:Method: GET 2022-09-03T07:26:04.456 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.457 DEBUG:boto:Data: 2022-09-03T07:26:04.457 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.459 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.459 DEBUG:boto:Params: {} 2022-09-03T07:26:04.460 DEBUG:boto:Token: None 2022-09-03T07:26:04.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7o+tzRGIZm9bzc/vFWIgJ9Av8UI= 2022-09-03T07:26:04.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7o+tzRGIZm9bzc/vFWIgJ9Av8UI=', 'Content-Length': '0'} 2022-09-03T07:26:04.465 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de4d3e4649358864-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.467 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.467 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.468 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.469 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.469 DEBUG:boto:Method: GET 2022-09-03T07:26:04.470 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.471 DEBUG:boto:Data: 2022-09-03T07:26:04.472 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.472 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.473 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.473 DEBUG:boto:Params: {} 2022-09-03T07:26:04.474 DEBUG:boto:Token: None 2022-09-03T07:26:04.474 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:26:04.475 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7o+tzRGIZm9bzc/vFWIgJ9Av8UI= 2022-09-03T07:26:04.476 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7o+tzRGIZm9bzc/vFWIgJ9Av8UI=', 'Content-Length': '0'} 2022-09-03T07:26:04.478 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8d22c48596c5f2a-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.479 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.479 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.480 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:26:04.480 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:26:04.481 DEBUG:boto:Method: GET 2022-09-03T07:26:04.481 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:26:04.482 DEBUG:boto:Data: 2022-09-03T07:26:04.482 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.483 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.483 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.484 DEBUG:boto:Params: {} 2022-09-03T07:26:04.484 DEBUG:boto:Token: None 2022-09-03T07:26:04.485 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.486 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.489 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000217de4f19a8c67f5-006313018c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.489 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.490 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.491 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.492 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:26:04.492 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:26:04.493 DEBUG:boto:Method: GET 2022-09-03T07:26:04.494 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:26:04.494 DEBUG:boto:Data: 2022-09-03T07:26:04.495 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.495 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.496 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.496 DEBUG:boto:Params: {} 2022-09-03T07:26:04.497 DEBUG:boto:Token: None 2022-09-03T07:26:04.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.501 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a9872da62d71df10-006313018c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.502 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.503 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.503 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.504 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.504 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:26:04.505 DEBUG:boto:Data: 2022-09-03T07:26:04.505 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.506 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.506 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.507 DEBUG:boto:Params: {} 2022-09-03T07:26:04.507 DEBUG:boto:Token: None 2022-09-03T07:26:04.508 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116 2022-09-03T07:26:04.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9F9cbsdlLqVwuy4t2pE/Zj4O4W4= 2022-09-03T07:26:04.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9F9cbsdlLqVwuy4t2pE/Zj4O4W4=', 'Content-Length': '0'} 2022-09-03T07:26:04.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000caa3edde761efe29-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.512 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.512 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.513 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.513 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:26:04.514 DEBUG:boto:Data: 2022-09-03T07:26:04.514 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.515 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.515 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.516 DEBUG:boto:Params: {} 2022-09-03T07:26:04.516 DEBUG:boto:Token: None 2022-09-03T07:26:04.517 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116 2022-09-03T07:26:04.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9F9cbsdlLqVwuy4t2pE/Zj4O4W4= 2022-09-03T07:26:04.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9F9cbsdlLqVwuy4t2pE/Zj4O4W4=', 'Content-Length': '0'} 2022-09-03T07:26:04.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009df46ba1bbbeb8a-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:26:04.522 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.522 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.523 DEBUG:boto:Method: GET 2022-09-03T07:26:04.523 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:26:04.524 DEBUG:boto:Data: 2022-09-03T07:26:04.524 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.525 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.526 DEBUG:boto:Params: {} 2022-09-03T07:26:04.527 DEBUG:boto:Token: None 2022-09-03T07:26:04.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116 2022-09-03T07:26:04.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eIfDfqKQlf0nDhqllJc1r1WYoLc= 2022-09-03T07:26:04.529 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eIfDfqKQlf0nDhqllJc1r1WYoLc=', 'Content-Length': '0'} 2022-09-03T07:26:04.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009dfe8a286c4916cf-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.531 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:26:04.532 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:26:04.532 DEBUG:boto:Method: GET 2022-09-03T07:26:04.533 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:26:04.533 DEBUG:boto:Data: 2022-09-03T07:26:04.534 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.535 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.535 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.536 DEBUG:boto:Params: {} 2022-09-03T07:26:04.536 DEBUG:boto:Token: None 2022-09-03T07:26:04.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj116 2022-09-03T07:26:04.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eIfDfqKQlf0nDhqllJc1r1WYoLc= 2022-09-03T07:26:04.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eIfDfqKQlf0nDhqllJc1r1WYoLc=', 'Content-Length': '0'} 2022-09-03T07:26:04.540 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000043b34ddc52ad61a1-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.541 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.541 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.542 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:26:04.542 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:26:04.543 DEBUG:boto:Method: GET 2022-09-03T07:26:04.543 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:26:04.544 DEBUG:boto:Data: 2022-09-03T07:26:04.544 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.545 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.546 DEBUG:boto:Params: {} 2022-09-03T07:26:04.547 DEBUG:boto:Token: None 2022-09-03T07:26:04.547 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.548 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.551 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000979bc63789781998-006313018c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.598 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.599 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.600 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.600 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:26:04.601 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:26:04.601 DEBUG:boto:Method: GET 2022-09-03T07:26:04.602 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:26:04.602 DEBUG:boto:Data: 2022-09-03T07:26:04.603 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.604 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.605 DEBUG:boto:Params: {} 2022-09-03T07:26:04.605 DEBUG:boto:Token: None 2022-09-03T07:26:04.606 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.607 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.609 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9ff7d4640f8b9fa-006313018c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.610 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.611 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:26:04.612 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.612 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.613 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.613 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.614 DEBUG:boto:Method: GET 2022-09-03T07:26:04.614 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.615 DEBUG:boto:Data: 2022-09-03T07:26:04.615 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.617 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.617 DEBUG:boto:Params: {} 2022-09-03T07:26:04.618 DEBUG:boto:Token: None 2022-09-03T07:26:04.618 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i4UwAC17pfSOnZ/Buh/1yPMfNr4= 2022-09-03T07:26:04.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i4UwAC17pfSOnZ/Buh/1yPMfNr4=', 'Content-Length': '0'} 2022-09-03T07:26:04.622 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f23773efe45f1df6-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.623 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.623 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.624 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.624 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.625 DEBUG:boto:Method: GET 2022-09-03T07:26:04.625 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.626 DEBUG:boto:Data: 2022-09-03T07:26:04.627 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.627 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.628 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.628 DEBUG:boto:Params: {} 2022-09-03T07:26:04.629 DEBUG:boto:Token: None 2022-09-03T07:26:04.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i4UwAC17pfSOnZ/Buh/1yPMfNr4= 2022-09-03T07:26:04.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i4UwAC17pfSOnZ/Buh/1yPMfNr4=', 'Content-Length': '0'} 2022-09-03T07:26:04.633 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001fc415a305c14dff-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.634 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.635 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.635 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.636 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.636 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.637 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.637 DEBUG:boto:Data: 2022-09-03T07:26:04.638 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.638 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.639 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.639 DEBUG:boto:Params: {} 2022-09-03T07:26:04.640 DEBUG:boto:Token: None 2022-09-03T07:26:04.641 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GPfTsRWir5OZG3LRtvL2OkdLoRU= 2022-09-03T07:26:04.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GPfTsRWir5OZG3LRtvL2OkdLoRU=', 'Content-Length': '0'} 2022-09-03T07:26:04.649 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053244ded1c602962-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.650 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.650 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.651 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.651 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.652 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.652 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.653 DEBUG:boto:Data: 2022-09-03T07:26:04.653 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.654 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.654 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.655 DEBUG:boto:Params: {} 2022-09-03T07:26:04.656 DEBUG:boto:Token: None 2022-09-03T07:26:04.656 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GPfTsRWir5OZG3LRtvL2OkdLoRU= 2022-09-03T07:26:04.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GPfTsRWir5OZG3LRtvL2OkdLoRU=', 'Content-Length': '0'} 2022-09-03T07:26:04.660 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f2d78c3a7f3516a-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.660 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:26:04.661 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.661 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.662 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.663 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.663 DEBUG:boto:Method: GET 2022-09-03T07:26:04.664 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.664 DEBUG:boto:Data: 2022-09-03T07:26:04.665 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.666 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.667 DEBUG:boto:Params: {} 2022-09-03T07:26:04.667 DEBUG:boto:Token: None 2022-09-03T07:26:04.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i4UwAC17pfSOnZ/Buh/1yPMfNr4= 2022-09-03T07:26:04.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i4UwAC17pfSOnZ/Buh/1yPMfNr4=', 'Content-Length': '0'} 2022-09-03T07:26:04.671 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060ec825f0b15f81f-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.672 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.673 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.673 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.674 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.675 DEBUG:boto:Method: GET 2022-09-03T07:26:04.675 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.676 DEBUG:boto:Data: 2022-09-03T07:26:04.676 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.677 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.677 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.678 DEBUG:boto:Params: {} 2022-09-03T07:26:04.679 DEBUG:boto:Token: None 2022-09-03T07:26:04.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:26:04.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i4UwAC17pfSOnZ/Buh/1yPMfNr4= 2022-09-03T07:26:04.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i4UwAC17pfSOnZ/Buh/1yPMfNr4=', 'Content-Length': '0'} 2022-09-03T07:26:04.683 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000666daa1ab9c06c77-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.684 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.685 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.685 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:26:04.686 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:26:04.686 DEBUG:boto:Method: GET 2022-09-03T07:26:04.687 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:26:04.687 DEBUG:boto:Data: 2022-09-03T07:26:04.688 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.689 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.690 DEBUG:boto:Params: {} 2022-09-03T07:26:04.690 DEBUG:boto:Token: None 2022-09-03T07:26:04.691 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.695 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000052bb98dc697460b2-006313018c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.696 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.697 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.697 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.698 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:26:04.698 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:26:04.699 DEBUG:boto:Method: GET 2022-09-03T07:26:04.700 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:26:04.700 DEBUG:boto:Data: 2022-09-03T07:26:04.701 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.701 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.702 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.703 DEBUG:boto:Params: {} 2022-09-03T07:26:04.703 DEBUG:boto:Token: None 2022-09-03T07:26:04.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.705 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.708 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079b06e68f5816a6a-006313018c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.709 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.711 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.712 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.712 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.714 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:26:04.714 DEBUG:boto:Data: 2022-09-03T07:26:04.715 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.715 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.716 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.717 DEBUG:boto:Params: {} 2022-09-03T07:26:04.717 DEBUG:boto:Token: None 2022-09-03T07:26:04.718 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145 2022-09-03T07:26:04.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XJAUJkBYM+KT0W8Q9sQdx8gEQTg= 2022-09-03T07:26:04.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XJAUJkBYM+KT0W8Q9sQdx8gEQTg=', 'Content-Length': '0'} 2022-09-03T07:26:04.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a25eb68b66cc98ea-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.723 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.723 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.724 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.724 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:26:04.725 DEBUG:boto:Data: 2022-09-03T07:26:04.726 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.727 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.727 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.728 DEBUG:boto:Params: {} 2022-09-03T07:26:04.728 DEBUG:boto:Token: None 2022-09-03T07:26:04.729 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145 2022-09-03T07:26:04.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XJAUJkBYM+KT0W8Q9sQdx8gEQTg= 2022-09-03T07:26:04.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XJAUJkBYM+KT0W8Q9sQdx8gEQTg=', 'Content-Length': '0'} 2022-09-03T07:26:04.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8630c4b5beea896-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.733 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:26:04.734 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.735 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.736 DEBUG:boto:Method: GET 2022-09-03T07:26:04.736 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:26:04.737 DEBUG:boto:Data: 2022-09-03T07:26:04.738 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.740 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.741 DEBUG:boto:Params: {} 2022-09-03T07:26:04.742 DEBUG:boto:Token: None 2022-09-03T07:26:04.743 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145 2022-09-03T07:26:04.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06://dBWIG8iI01Xsy7MlrtiKMKd5w= 2022-09-03T07:26:04.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06://dBWIG8iI01Xsy7MlrtiKMKd5w=', 'Content-Length': '0'} 2022-09-03T07:26:04.747 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000420fc0692804a3b5-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.794 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:26:04.795 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:26:04.796 DEBUG:boto:Method: GET 2022-09-03T07:26:04.797 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:26:04.798 DEBUG:boto:Data: 2022-09-03T07:26:04.799 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.799 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.800 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.801 DEBUG:boto:Params: {} 2022-09-03T07:26:04.802 DEBUG:boto:Token: None 2022-09-03T07:26:04.803 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj145 2022-09-03T07:26:04.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06://dBWIG8iI01Xsy7MlrtiKMKd5w= 2022-09-03T07:26:04.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06://dBWIG8iI01Xsy7MlrtiKMKd5w=', 'Content-Length': '0'} 2022-09-03T07:26:04.807 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000446a101f8cea1460-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.808 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.809 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.809 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:26:04.810 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:26:04.811 DEBUG:boto:Method: GET 2022-09-03T07:26:04.812 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:26:04.812 DEBUG:boto:Data: 2022-09-03T07:26:04.813 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.816 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.817 DEBUG:boto:Params: {} 2022-09-03T07:26:04.818 DEBUG:boto:Token: None 2022-09-03T07:26:04.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.825 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000f3503ba063b9303-006313018c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.827 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.829 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.829 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.830 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:26:04.830 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:26:04.831 DEBUG:boto:Method: GET 2022-09-03T07:26:04.832 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:26:04.832 DEBUG:boto:Data: 2022-09-03T07:26:04.833 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.835 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.835 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.836 DEBUG:boto:Params: {} 2022-09-03T07:26:04.837 DEBUG:boto:Token: None 2022-09-03T07:26:04.837 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.841 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000069cd8831c21c6793-006313018c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.841 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.842 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:26:04.843 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:04.843 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:04.844 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.844 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.845 DEBUG:boto:Method: GET 2022-09-03T07:26:04.846 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.846 DEBUG:boto:Data: 2022-09-03T07:26:04.847 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.848 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.848 DEBUG:boto:Params: {} 2022-09-03T07:26:04.849 DEBUG:boto:Token: None 2022-09-03T07:26:04.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:peOOlPV+w3OHm5avfy7sJSwRywg= 2022-09-03T07:26:04.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:peOOlPV+w3OHm5avfy7sJSwRywg=', 'Content-Length': '0'} 2022-09-03T07:26:04.855 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000059bdb75e05ad7dd-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.856 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:04.857 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:04.858 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.858 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.859 DEBUG:boto:Method: GET 2022-09-03T07:26:04.861 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.861 DEBUG:boto:Data: 2022-09-03T07:26:04.862 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.864 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.864 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.866 DEBUG:boto:Params: {} 2022-09-03T07:26:04.867 DEBUG:boto:Token: None 2022-09-03T07:26:04.868 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:peOOlPV+w3OHm5avfy7sJSwRywg= 2022-09-03T07:26:04.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:peOOlPV+w3OHm5avfy7sJSwRywg=', 'Content-Length': '0'} 2022-09-03T07:26:04.871 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000825a4d9fe3d55fda-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.873 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:04.873 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:04.874 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.874 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.875 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.875 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.876 DEBUG:boto:Data: 2022-09-03T07:26:04.876 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.877 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.877 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.878 DEBUG:boto:Params: {} 2022-09-03T07:26:04.878 DEBUG:boto:Token: None 2022-09-03T07:26:04.879 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Rt0EEHwdeTcODZm39ELHJJcbJE= 2022-09-03T07:26:04.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Rt0EEHwdeTcODZm39ELHJJcbJE=', 'Content-Length': '0'} 2022-09-03T07:26:04.882 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa2cbc249537cbcb-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.883 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:04.883 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:04.884 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.884 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.885 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.885 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.886 DEBUG:boto:Data: 2022-09-03T07:26:04.886 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.887 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.887 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.888 DEBUG:boto:Params: {} 2022-09-03T07:26:04.888 DEBUG:boto:Token: None 2022-09-03T07:26:04.889 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Rt0EEHwdeTcODZm39ELHJJcbJE= 2022-09-03T07:26:04.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Rt0EEHwdeTcODZm39ELHJJcbJE=', 'Content-Length': '0'} 2022-09-03T07:26:04.892 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000527205a72ddb726f-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.893 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:26:04.894 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:04.894 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:04.895 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.895 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.896 DEBUG:boto:Method: GET 2022-09-03T07:26:04.897 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.897 DEBUG:boto:Data: 2022-09-03T07:26:04.898 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.898 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.899 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.900 DEBUG:boto:Params: {} 2022-09-03T07:26:04.900 DEBUG:boto:Token: None 2022-09-03T07:26:04.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:peOOlPV+w3OHm5avfy7sJSwRywg= 2022-09-03T07:26:04.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:peOOlPV+w3OHm5avfy7sJSwRywg=', 'Content-Length': '0'} 2022-09-03T07:26:04.904 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c48ce631ea8938e7-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.905 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:04.906 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:04.906 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.907 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.907 DEBUG:boto:Method: GET 2022-09-03T07:26:04.908 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.908 DEBUG:boto:Data: 2022-09-03T07:26:04.909 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.910 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.910 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.911 DEBUG:boto:Params: {} 2022-09-03T07:26:04.911 DEBUG:boto:Token: None 2022-09-03T07:26:04.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:26:04.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:peOOlPV+w3OHm5avfy7sJSwRywg= 2022-09-03T07:26:04.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:peOOlPV+w3OHm5avfy7sJSwRywg=', 'Content-Length': '0'} 2022-09-03T07:26:04.916 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001aa3b66479a88386-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.917 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.917 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.918 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:26:04.918 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:26:04.919 DEBUG:boto:Method: GET 2022-09-03T07:26:04.920 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:26:04.920 DEBUG:boto:Data: 2022-09-03T07:26:04.921 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.922 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.922 DEBUG:boto:Params: {} 2022-09-03T07:26:04.923 DEBUG:boto:Token: None 2022-09-03T07:26:04.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.928 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000062ce4b1c2012b7af-006313018c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.928 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.929 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:04.930 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:04.930 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:26:04.931 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:26:04.932 DEBUG:boto:Method: GET 2022-09-03T07:26:04.932 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:26:04.933 DEBUG:boto:Data: 2022-09-03T07:26:04.933 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.934 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.934 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.935 DEBUG:boto:Params: {} 2022-09-03T07:26:04.935 DEBUG:boto:Token: None 2022-09-03T07:26:04.936 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/ 2022-09-03T07:26:04.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg= 2022-09-03T07:26:04.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lHMbw5B9fzIpiWkCZrWINQyxtCg=', 'Content-Length': '0'} 2022-09-03T07:26:04.940 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d4b7c1590511d968-006313018c-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.940 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:04.941 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:04.942 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:04.942 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.943 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:26:04.943 DEBUG:boto:Data: 2022-09-03T07:26:04.944 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.944 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.945 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.945 DEBUG:boto:Params: {} 2022-09-03T07:26:04.946 DEBUG:boto:Token: None 2022-09-03T07:26:04.947 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj17 2022-09-03T07:26:04.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WhqCoztgfde2GjNjSrZ5Fg1U5o4= 2022-09-03T07:26:04.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WhqCoztgfde2GjNjSrZ5Fg1U5o4=', 'Content-Length': '0'} 2022-09-03T07:26:04.950 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002df1ef489561f6e1-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.950 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:04.951 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:04.953 DEBUG:boto:Method: HEAD 2022-09-03T07:26:04.954 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:26:04.954 DEBUG:boto:Data: 2022-09-03T07:26:04.955 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.955 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:04.956 DEBUG:boto:Port: 8000 2022-09-03T07:26:04.956 DEBUG:boto:Params: {} 2022-09-03T07:26:04.957 DEBUG:boto:Token: None 2022-09-03T07:26:04.957 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj17 2022-09-03T07:26:04.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WhqCoztgfde2GjNjSrZ5Fg1U5o4= 2022-09-03T07:26:04.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WhqCoztgfde2GjNjSrZ5Fg1U5o4=', 'Content-Length': '0'} 2022-09-03T07:26:04.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000069429a2e82281127-006313018c-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:04.961 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:26:04.962 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:04.962 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:04.963 DEBUG:boto:Method: GET 2022-09-03T07:26:04.964 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:26:04.964 DEBUG:boto:Data: 2022-09-03T07:26:04.965 DEBUG:boto:Headers: {} 2022-09-03T07:26:04.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:04.966 DEBUG:boto:Port: 8001 2022-09-03T07:26:04.966 DEBUG:boto:Params: {} 2022-09-03T07:26:04.967 DEBUG:boto:Token: None 2022-09-03T07:26:04.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:04 GMT /avysmw-75/obj17 2022-09-03T07:26:04.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kDWHKwYbQcXGBEIGylwLX43OiTY= 2022-09-03T07:26:04.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kDWHKwYbQcXGBEIGylwLX43OiTY=', 'Content-Length': '0'} 2022-09-03T07:26:04.971 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c30463861e35af42-006313018c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.015 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:26:05.016 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:26:05.017 DEBUG:boto:Method: GET 2022-09-03T07:26:05.017 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:26:05.018 DEBUG:boto:Data: 2022-09-03T07:26:05.019 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.019 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.020 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.020 DEBUG:boto:Params: {} 2022-09-03T07:26:05.021 DEBUG:boto:Token: None 2022-09-03T07:26:05.022 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj17 2022-09-03T07:26:05.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rjfBVjVmWhpujWG9Hf2Wx15EV8Y= 2022-09-03T07:26:05.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rjfBVjVmWhpujWG9Hf2Wx15EV8Y=', 'Content-Length': '0'} 2022-09-03T07:26:05.025 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bccd44611d567789-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.026 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.027 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.028 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:26:05.028 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:26:05.029 DEBUG:boto:Method: GET 2022-09-03T07:26:05.030 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:26:05.031 DEBUG:boto:Data: 2022-09-03T07:26:05.031 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.032 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.033 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.034 DEBUG:boto:Params: {} 2022-09-03T07:26:05.035 DEBUG:boto:Token: None 2022-09-03T07:26:05.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.040 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000089edfd78278205b5-006313018d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.041 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.043 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.044 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.045 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:26:05.046 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:26:05.047 DEBUG:boto:Method: GET 2022-09-03T07:26:05.048 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:26:05.048 DEBUG:boto:Data: 2022-09-03T07:26:05.049 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.049 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.050 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.050 DEBUG:boto:Params: {} 2022-09-03T07:26:05.051 DEBUG:boto:Token: None 2022-09-03T07:26:05.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.056 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022337a5a3a7e1264-006313018d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.057 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.058 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:26:05.058 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.059 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.060 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.060 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.061 DEBUG:boto:Method: GET 2022-09-03T07:26:05.061 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.062 DEBUG:boto:Data: 2022-09-03T07:26:05.062 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.063 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.063 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.064 DEBUG:boto:Params: {} 2022-09-03T07:26:05.065 DEBUG:boto:Token: None 2022-09-03T07:26:05.067 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.068 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:II4q3z8U6QWXRO4hQnmp3mDEnFs= 2022-09-03T07:26:05.068 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:II4q3z8U6QWXRO4hQnmp3mDEnFs=', 'Content-Length': '0'} 2022-09-03T07:26:05.070 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c68cfe8b8b0652f9-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.071 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.072 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.072 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.073 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.074 DEBUG:boto:Method: GET 2022-09-03T07:26:05.074 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.075 DEBUG:boto:Data: 2022-09-03T07:26:05.076 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.076 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.077 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.077 DEBUG:boto:Params: {} 2022-09-03T07:26:05.078 DEBUG:boto:Token: None 2022-09-03T07:26:05.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:II4q3z8U6QWXRO4hQnmp3mDEnFs= 2022-09-03T07:26:05.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:II4q3z8U6QWXRO4hQnmp3mDEnFs=', 'Content-Length': '0'} 2022-09-03T07:26:05.084 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f68ae8ea5981164-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.085 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.086 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.087 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.087 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.088 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.089 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.089 DEBUG:boto:Data: 2022-09-03T07:26:05.090 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.090 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.091 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.091 DEBUG:boto:Params: {} 2022-09-03T07:26:05.092 DEBUG:boto:Token: None 2022-09-03T07:26:05.093 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DfUbUxepAUR9KcgoKBpyQyy9sI= 2022-09-03T07:26:05.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DfUbUxepAUR9KcgoKBpyQyy9sI=', 'Content-Length': '0'} 2022-09-03T07:26:05.097 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ecfaeceb67c8c3ca-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.098 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.098 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.099 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.099 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.100 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.100 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.101 DEBUG:boto:Data: 2022-09-03T07:26:05.101 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.102 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.102 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.103 DEBUG:boto:Params: {} 2022-09-03T07:26:05.103 DEBUG:boto:Token: None 2022-09-03T07:26:05.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0DfUbUxepAUR9KcgoKBpyQyy9sI= 2022-09-03T07:26:05.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0DfUbUxepAUR9KcgoKBpyQyy9sI=', 'Content-Length': '0'} 2022-09-03T07:26:05.107 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008364c35a8b1b209-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.108 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:26:05.108 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.109 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.109 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.110 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.111 DEBUG:boto:Method: GET 2022-09-03T07:26:05.111 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.112 DEBUG:boto:Data: 2022-09-03T07:26:05.113 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.114 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.115 DEBUG:boto:Params: {} 2022-09-03T07:26:05.116 DEBUG:boto:Token: None 2022-09-03T07:26:05.116 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:II4q3z8U6QWXRO4hQnmp3mDEnFs= 2022-09-03T07:26:05.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:II4q3z8U6QWXRO4hQnmp3mDEnFs=', 'Content-Length': '0'} 2022-09-03T07:26:05.120 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093f4d7ca0f3521b2-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.123 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.124 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.125 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.126 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.126 DEBUG:boto:Method: GET 2022-09-03T07:26:05.127 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.127 DEBUG:boto:Data: 2022-09-03T07:26:05.128 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.128 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.129 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.129 DEBUG:boto:Params: {} 2022-09-03T07:26:05.130 DEBUG:boto:Token: None 2022-09-03T07:26:05.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:26:05.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:II4q3z8U6QWXRO4hQnmp3mDEnFs= 2022-09-03T07:26:05.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:II4q3z8U6QWXRO4hQnmp3mDEnFs=', 'Content-Length': '0'} 2022-09-03T07:26:05.136 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090f33ef1df88682d-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.181 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.182 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.183 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:26:05.183 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:26:05.184 DEBUG:boto:Method: GET 2022-09-03T07:26:05.186 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:26:05.187 DEBUG:boto:Data: 2022-09-03T07:26:05.187 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.188 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.189 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.189 DEBUG:boto:Params: {} 2022-09-03T07:26:05.190 DEBUG:boto:Token: None 2022-09-03T07:26:05.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.191 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.194 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a40194bfb795b345-006313018d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.196 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.197 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.198 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.198 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:26:05.199 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:26:05.200 DEBUG:boto:Method: GET 2022-09-03T07:26:05.200 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:26:05.201 DEBUG:boto:Data: 2022-09-03T07:26:05.201 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.202 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.203 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.204 DEBUG:boto:Params: {} 2022-09-03T07:26:05.205 DEBUG:boto:Token: None 2022-09-03T07:26:05.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.211 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090ad3404f3b1f863-006313018d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.212 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.213 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.214 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.215 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.216 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:26:05.217 DEBUG:boto:Data: 2022-09-03T07:26:05.217 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.219 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.219 DEBUG:boto:Params: {} 2022-09-03T07:26:05.221 DEBUG:boto:Token: None 2022-09-03T07:26:05.222 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19 2022-09-03T07:26:05.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kwGbGyhAk5WaExo/lOf5O3beZ7s= 2022-09-03T07:26:05.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kwGbGyhAk5WaExo/lOf5O3beZ7s=', 'Content-Length': '0'} 2022-09-03T07:26:05.227 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007bb75652942e7c6-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.229 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.231 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.231 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.232 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:26:05.232 DEBUG:boto:Data: 2022-09-03T07:26:05.233 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.234 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.235 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.236 DEBUG:boto:Params: {} 2022-09-03T07:26:05.237 DEBUG:boto:Token: None 2022-09-03T07:26:05.238 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19 2022-09-03T07:26:05.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kwGbGyhAk5WaExo/lOf5O3beZ7s= 2022-09-03T07:26:05.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kwGbGyhAk5WaExo/lOf5O3beZ7s=', 'Content-Length': '0'} 2022-09-03T07:26:05.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd3b795dc24f5087-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.244 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:26:05.245 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.246 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.247 DEBUG:boto:Method: GET 2022-09-03T07:26:05.248 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:26:05.249 DEBUG:boto:Data: 2022-09-03T07:26:05.250 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.252 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.253 DEBUG:boto:Params: {} 2022-09-03T07:26:05.254 DEBUG:boto:Token: None 2022-09-03T07:26:05.255 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19 2022-09-03T07:26:05.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BXsDdQ1oQBKoPK9LDxSJSWivE78= 2022-09-03T07:26:05.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BXsDdQ1oQBKoPK9LDxSJSWivE78=', 'Content-Length': '0'} 2022-09-03T07:26:05.260 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091b7027c452ece7f-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.307 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:26:05.308 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:26:05.309 DEBUG:boto:Method: GET 2022-09-03T07:26:05.309 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:26:05.310 DEBUG:boto:Data: 2022-09-03T07:26:05.310 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.311 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.312 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.312 DEBUG:boto:Params: {} 2022-09-03T07:26:05.313 DEBUG:boto:Token: None 2022-09-03T07:26:05.313 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj19 2022-09-03T07:26:05.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BXsDdQ1oQBKoPK9LDxSJSWivE78= 2022-09-03T07:26:05.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BXsDdQ1oQBKoPK9LDxSJSWivE78=', 'Content-Length': '0'} 2022-09-03T07:26:05.318 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000feca04b306d08a9-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.320 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.320 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.321 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:26:05.321 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:26:05.322 DEBUG:boto:Method: GET 2022-09-03T07:26:05.322 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:26:05.323 DEBUG:boto:Data: 2022-09-03T07:26:05.323 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.324 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.324 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.325 DEBUG:boto:Params: {} 2022-09-03T07:26:05.326 DEBUG:boto:Token: None 2022-09-03T07:26:05.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.330 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb7169c0b7ea28d7-006313018d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.331 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.332 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.332 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.333 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:26:05.333 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:26:05.334 DEBUG:boto:Method: GET 2022-09-03T07:26:05.334 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:26:05.335 DEBUG:boto:Data: 2022-09-03T07:26:05.335 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.336 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.336 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.337 DEBUG:boto:Params: {} 2022-09-03T07:26:05.338 DEBUG:boto:Token: None 2022-09-03T07:26:05.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.352 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e8d38fb50fb4ea46-006313018d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.398 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.399 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:26:05.400 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.400 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.401 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.402 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.402 DEBUG:boto:Method: GET 2022-09-03T07:26:05.403 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.403 DEBUG:boto:Data: 2022-09-03T07:26:05.404 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.404 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.405 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.406 DEBUG:boto:Params: {} 2022-09-03T07:26:05.406 DEBUG:boto:Token: None 2022-09-03T07:26:05.407 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wa2TwU60WNUwHtotW86hk4IlA+o= 2022-09-03T07:26:05.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wa2TwU60WNUwHtotW86hk4IlA+o=', 'Content-Length': '0'} 2022-09-03T07:26:05.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009480fe35ef61817a-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.412 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.412 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.413 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.413 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.414 DEBUG:boto:Method: GET 2022-09-03T07:26:05.414 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.415 DEBUG:boto:Data: 2022-09-03T07:26:05.416 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.416 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.417 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.417 DEBUG:boto:Params: {} 2022-09-03T07:26:05.418 DEBUG:boto:Token: None 2022-09-03T07:26:05.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wa2TwU60WNUwHtotW86hk4IlA+o= 2022-09-03T07:26:05.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wa2TwU60WNUwHtotW86hk4IlA+o=', 'Content-Length': '0'} 2022-09-03T07:26:05.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dd1f48306cff22d2-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.423 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.423 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.424 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.424 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.425 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.425 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.426 DEBUG:boto:Data: 2022-09-03T07:26:05.427 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.428 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.428 DEBUG:boto:Params: {} 2022-09-03T07:26:05.429 DEBUG:boto:Token: None 2022-09-03T07:26:05.429 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7K2BDBUGYt9/H8csuyGs/Ppi2+8= 2022-09-03T07:26:05.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7K2BDBUGYt9/H8csuyGs/Ppi2+8=', 'Content-Length': '0'} 2022-09-03T07:26:05.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce9c77bd4a83f9b2-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.433 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.434 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.434 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.435 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.436 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.436 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.437 DEBUG:boto:Data: 2022-09-03T07:26:05.437 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.438 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.438 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.439 DEBUG:boto:Params: {} 2022-09-03T07:26:05.439 DEBUG:boto:Token: None 2022-09-03T07:26:05.440 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7K2BDBUGYt9/H8csuyGs/Ppi2+8= 2022-09-03T07:26:05.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7K2BDBUGYt9/H8csuyGs/Ppi2+8=', 'Content-Length': '0'} 2022-09-03T07:26:05.443 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000cde9a7f2167f48f-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.444 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:26:05.445 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.445 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.446 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.446 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.447 DEBUG:boto:Method: GET 2022-09-03T07:26:05.447 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.448 DEBUG:boto:Data: 2022-09-03T07:26:05.448 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.449 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.449 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.450 DEBUG:boto:Params: {} 2022-09-03T07:26:05.451 DEBUG:boto:Token: None 2022-09-03T07:26:05.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wa2TwU60WNUwHtotW86hk4IlA+o= 2022-09-03T07:26:05.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wa2TwU60WNUwHtotW86hk4IlA+o=', 'Content-Length': '0'} 2022-09-03T07:26:05.455 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b3aac568526c424-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.456 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.457 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.457 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.458 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.459 DEBUG:boto:Method: GET 2022-09-03T07:26:05.460 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.460 DEBUG:boto:Data: 2022-09-03T07:26:05.461 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.461 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.462 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.462 DEBUG:boto:Params: {} 2022-09-03T07:26:05.463 DEBUG:boto:Token: None 2022-09-03T07:26:05.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:26:05.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wa2TwU60WNUwHtotW86hk4IlA+o= 2022-09-03T07:26:05.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wa2TwU60WNUwHtotW86hk4IlA+o=', 'Content-Length': '0'} 2022-09-03T07:26:05.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af76031b9bb6b99d-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.468 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.468 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.469 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:26:05.470 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:26:05.470 DEBUG:boto:Method: GET 2022-09-03T07:26:05.471 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:26:05.471 DEBUG:boto:Data: 2022-09-03T07:26:05.472 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.473 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.473 DEBUG:boto:Params: {} 2022-09-03T07:26:05.474 DEBUG:boto:Token: None 2022-09-03T07:26:05.475 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.475 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.476 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.478 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e4249cff2eef9adc-006313018d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.480 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.481 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.481 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.482 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:26:05.482 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:26:05.483 DEBUG:boto:Method: GET 2022-09-03T07:26:05.483 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:26:05.484 DEBUG:boto:Data: 2022-09-03T07:26:05.484 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.485 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.486 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.486 DEBUG:boto:Params: {} 2022-09-03T07:26:05.487 DEBUG:boto:Token: None 2022-09-03T07:26:05.487 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.489 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.492 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007acff38de9bfd743-006313018d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.493 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.495 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.496 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.497 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.498 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:26:05.499 DEBUG:boto:Data: 2022-09-03T07:26:05.500 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.501 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.502 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.503 DEBUG:boto:Params: {} 2022-09-03T07:26:05.504 DEBUG:boto:Token: None 2022-09-03T07:26:05.505 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22 2022-09-03T07:26:05.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aRFxx4dseKgs/ZbnmDX72ISMf+s= 2022-09-03T07:26:05.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aRFxx4dseKgs/ZbnmDX72ISMf+s=', 'Content-Length': '0'} 2022-09-03T07:26:05.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010c5247b3aa7cf67-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.511 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.513 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.514 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.515 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:26:05.515 DEBUG:boto:Data: 2022-09-03T07:26:05.516 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.516 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.517 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.517 DEBUG:boto:Params: {} 2022-09-03T07:26:05.518 DEBUG:boto:Token: None 2022-09-03T07:26:05.518 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22 2022-09-03T07:26:05.519 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aRFxx4dseKgs/ZbnmDX72ISMf+s= 2022-09-03T07:26:05.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aRFxx4dseKgs/ZbnmDX72ISMf+s=', 'Content-Length': '0'} 2022-09-03T07:26:05.522 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b92cf6c409486bc9-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.523 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:26:05.526 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.527 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.528 DEBUG:boto:Method: GET 2022-09-03T07:26:05.529 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:26:05.530 DEBUG:boto:Data: 2022-09-03T07:26:05.531 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.532 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.534 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.535 DEBUG:boto:Params: {} 2022-09-03T07:26:05.536 DEBUG:boto:Token: None 2022-09-03T07:26:05.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22 2022-09-03T07:26:05.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N8Fm8v8dMVB1NKqLl2nctBJZEOI= 2022-09-03T07:26:05.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N8Fm8v8dMVB1NKqLl2nctBJZEOI=', 'Content-Length': '0'} 2022-09-03T07:26:05.543 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086d5e479edc0cf1d-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.587 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:26:05.588 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:26:05.588 DEBUG:boto:Method: GET 2022-09-03T07:26:05.589 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:26:05.589 DEBUG:boto:Data: 2022-09-03T07:26:05.590 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.590 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.591 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.591 DEBUG:boto:Params: {} 2022-09-03T07:26:05.592 DEBUG:boto:Token: None 2022-09-03T07:26:05.593 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj22 2022-09-03T07:26:05.593 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N8Fm8v8dMVB1NKqLl2nctBJZEOI= 2022-09-03T07:26:05.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N8Fm8v8dMVB1NKqLl2nctBJZEOI=', 'Content-Length': '0'} 2022-09-03T07:26:05.596 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099b1ff4a03191295-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.600 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.601 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.601 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:26:05.602 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:26:05.602 DEBUG:boto:Method: GET 2022-09-03T07:26:05.603 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:26:05.603 DEBUG:boto:Data: 2022-09-03T07:26:05.604 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.605 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.606 DEBUG:boto:Params: {} 2022-09-03T07:26:05.607 DEBUG:boto:Token: None 2022-09-03T07:26:05.608 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.614 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004fc3aaa5aee35cc0-006313018d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.615 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.616 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.616 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.617 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:26:05.617 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:26:05.618 DEBUG:boto:Method: GET 2022-09-03T07:26:05.619 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:26:05.619 DEBUG:boto:Data: 2022-09-03T07:26:05.620 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.620 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.621 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.622 DEBUG:boto:Params: {} 2022-09-03T07:26:05.623 DEBUG:boto:Token: None 2022-09-03T07:26:05.623 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.627 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002602624ab4fd3351-006313018d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.680 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.681 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:26:05.682 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.683 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.683 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.684 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.685 DEBUG:boto:Method: GET 2022-09-03T07:26:05.685 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.686 DEBUG:boto:Data: 2022-09-03T07:26:05.686 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.688 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.688 DEBUG:boto:Params: {} 2022-09-03T07:26:05.689 DEBUG:boto:Token: None 2022-09-03T07:26:05.690 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uwKh4ExEm+OaTOtHBo346hjsLlw= 2022-09-03T07:26:05.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uwKh4ExEm+OaTOtHBo346hjsLlw=', 'Content-Length': '0'} 2022-09-03T07:26:05.693 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d1d8347f4b5fd317-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.694 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.694 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.695 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.696 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.696 DEBUG:boto:Method: GET 2022-09-03T07:26:05.697 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.697 DEBUG:boto:Data: 2022-09-03T07:26:05.698 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.699 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.699 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.700 DEBUG:boto:Params: {} 2022-09-03T07:26:05.701 DEBUG:boto:Token: None 2022-09-03T07:26:05.701 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uwKh4ExEm+OaTOtHBo346hjsLlw= 2022-09-03T07:26:05.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uwKh4ExEm+OaTOtHBo346hjsLlw=', 'Content-Length': '0'} 2022-09-03T07:26:05.710 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c44174ec39df18b-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.711 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.712 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.712 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.713 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.714 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.714 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.715 DEBUG:boto:Data: 2022-09-03T07:26:05.716 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.717 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.718 DEBUG:boto:Params: {} 2022-09-03T07:26:05.718 DEBUG:boto:Token: None 2022-09-03T07:26:05.719 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JX4dB7/5D6cUAQTBwfFP1LoCzOY= 2022-09-03T07:26:05.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JX4dB7/5D6cUAQTBwfFP1LoCzOY=', 'Content-Length': '0'} 2022-09-03T07:26:05.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e318f7496bc5912d-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.724 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.724 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.725 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.726 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.726 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.727 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.727 DEBUG:boto:Data: 2022-09-03T07:26:05.728 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.728 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.729 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.729 DEBUG:boto:Params: {} 2022-09-03T07:26:05.730 DEBUG:boto:Token: None 2022-09-03T07:26:05.731 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JX4dB7/5D6cUAQTBwfFP1LoCzOY= 2022-09-03T07:26:05.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JX4dB7/5D6cUAQTBwfFP1LoCzOY=', 'Content-Length': '0'} 2022-09-03T07:26:05.734 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091388d736abd5e63-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.735 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:26:05.737 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.738 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.739 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.740 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.741 DEBUG:boto:Method: GET 2022-09-03T07:26:05.742 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.743 DEBUG:boto:Data: 2022-09-03T07:26:05.744 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.746 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.747 DEBUG:boto:Params: {} 2022-09-03T07:26:05.748 DEBUG:boto:Token: None 2022-09-03T07:26:05.749 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uwKh4ExEm+OaTOtHBo346hjsLlw= 2022-09-03T07:26:05.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uwKh4ExEm+OaTOtHBo346hjsLlw=', 'Content-Length': '0'} 2022-09-03T07:26:05.754 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004181e31266bf22fd-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.756 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.756 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.757 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.758 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.759 DEBUG:boto:Method: GET 2022-09-03T07:26:05.760 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.761 DEBUG:boto:Data: 2022-09-03T07:26:05.762 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.764 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.765 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.766 DEBUG:boto:Params: {} 2022-09-03T07:26:05.768 DEBUG:boto:Token: None 2022-09-03T07:26:05.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:26:05.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uwKh4ExEm+OaTOtHBo346hjsLlw= 2022-09-03T07:26:05.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uwKh4ExEm+OaTOtHBo346hjsLlw=', 'Content-Length': '0'} 2022-09-03T07:26:05.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006aab8de79f363113-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.776 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.777 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.778 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:26:05.779 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:26:05.780 DEBUG:boto:Method: GET 2022-09-03T07:26:05.781 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:26:05.782 DEBUG:boto:Data: 2022-09-03T07:26:05.783 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.784 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.785 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.786 DEBUG:boto:Params: {} 2022-09-03T07:26:05.787 DEBUG:boto:Token: None 2022-09-03T07:26:05.788 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.794 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028882e9ae73776ad-006313018d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.834 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.835 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.837 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.838 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:26:05.838 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:26:05.841 DEBUG:boto:Method: GET 2022-09-03T07:26:05.841 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:26:05.843 DEBUG:boto:Data: 2022-09-03T07:26:05.844 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.845 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.846 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.847 DEBUG:boto:Params: {} 2022-09-03T07:26:05.848 DEBUG:boto:Token: None 2022-09-03T07:26:05.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.850 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.853 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005969648bc6ec16c7-006313018d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.853 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.854 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.855 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.855 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.856 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:26:05.857 DEBUG:boto:Data: 2022-09-03T07:26:05.857 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.858 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.858 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.859 DEBUG:boto:Params: {} 2022-09-03T07:26:05.859 DEBUG:boto:Token: None 2022-09-03T07:26:05.860 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23 2022-09-03T07:26:05.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F5cMTTHh/CiVWaOvZk2LLZLSzMo= 2022-09-03T07:26:05.862 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F5cMTTHh/CiVWaOvZk2LLZLSzMo=', 'Content-Length': '0'} 2022-09-03T07:26:05.864 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005f30b0016e275c0-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.865 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.865 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.867 DEBUG:boto:Method: HEAD 2022-09-03T07:26:05.867 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:26:05.868 DEBUG:boto:Data: 2022-09-03T07:26:05.868 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.869 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.869 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.869 DEBUG:boto:Params: {} 2022-09-03T07:26:05.870 DEBUG:boto:Token: None 2022-09-03T07:26:05.871 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23 2022-09-03T07:26:05.871 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F5cMTTHh/CiVWaOvZk2LLZLSzMo= 2022-09-03T07:26:05.872 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F5cMTTHh/CiVWaOvZk2LLZLSzMo=', 'Content-Length': '0'} 2022-09-03T07:26:05.874 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f14d74886309529b-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.874 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:26:05.875 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.875 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.876 DEBUG:boto:Method: GET 2022-09-03T07:26:05.877 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:26:05.877 DEBUG:boto:Data: 2022-09-03T07:26:05.878 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.879 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.879 DEBUG:boto:Params: {} 2022-09-03T07:26:05.880 DEBUG:boto:Token: None 2022-09-03T07:26:05.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23 2022-09-03T07:26:05.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:erZ5F2rrzqjJSRXXO3BF27H//bI= 2022-09-03T07:26:05.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:erZ5F2rrzqjJSRXXO3BF27H//bI=', 'Content-Length': '0'} 2022-09-03T07:26:05.884 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0269a6aac2fe428-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.885 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:26:05.885 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:26:05.886 DEBUG:boto:Method: GET 2022-09-03T07:26:05.886 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:26:05.887 DEBUG:boto:Data: 2022-09-03T07:26:05.887 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.888 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.888 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.889 DEBUG:boto:Params: {} 2022-09-03T07:26:05.889 DEBUG:boto:Token: None 2022-09-03T07:26:05.890 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj23 2022-09-03T07:26:05.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:erZ5F2rrzqjJSRXXO3BF27H//bI= 2022-09-03T07:26:05.891 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:erZ5F2rrzqjJSRXXO3BF27H//bI=', 'Content-Length': '0'} 2022-09-03T07:26:05.893 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093cdfeec2300d75e-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.894 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.894 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.895 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:26:05.895 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:26:05.896 DEBUG:boto:Method: GET 2022-09-03T07:26:05.897 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:26:05.897 DEBUG:boto:Data: 2022-09-03T07:26:05.898 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.898 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.899 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.899 DEBUG:boto:Params: {} 2022-09-03T07:26:05.900 DEBUG:boto:Token: None 2022-09-03T07:26:05.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.904 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000064a704668f8cffb-006313018d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.906 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.907 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:05.908 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:05.909 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:26:05.910 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:26:05.911 DEBUG:boto:Method: GET 2022-09-03T07:26:05.912 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:26:05.913 DEBUG:boto:Data: 2022-09-03T07:26:05.913 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.914 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.915 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.916 DEBUG:boto:Params: {} 2022-09-03T07:26:05.917 DEBUG:boto:Token: None 2022-09-03T07:26:05.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/ 2022-09-03T07:26:05.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY= 2022-09-03T07:26:05.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wiS1OLLvoltkSrOgS2dnAH+asEY=', 'Content-Length': '0'} 2022-09-03T07:26:05.923 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f6b31448f3405d4-006313018d-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.925 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:05.926 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:26:05.927 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:05.928 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:05.929 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:05.929 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:05.930 DEBUG:boto:Method: GET 2022-09-03T07:26:05.931 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:26:05.932 DEBUG:boto:Data: 2022-09-03T07:26:05.933 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:05.935 DEBUG:boto:Port: 8001 2022-09-03T07:26:05.935 DEBUG:boto:Params: {} 2022-09-03T07:26:05.937 DEBUG:boto:Token: None 2022-09-03T07:26:05.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:26:05.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EN9tcqZd4eQ6EE1d/KjANZwqot4= 2022-09-03T07:26:05.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EN9tcqZd4eQ6EE1d/KjANZwqot4=', 'Content-Length': '0'} 2022-09-03T07:26:05.943 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f8924dcb381d706e-006313018d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:05.944 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:05.945 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:05.946 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:05.946 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:05.947 DEBUG:boto:Method: GET 2022-09-03T07:26:05.948 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:26:05.948 DEBUG:boto:Data: 2022-09-03T07:26:05.949 DEBUG:boto:Headers: {} 2022-09-03T07:26:05.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:05.950 DEBUG:boto:Port: 8000 2022-09-03T07:26:05.951 DEBUG:boto:Params: {} 2022-09-03T07:26:05.951 DEBUG:boto:Token: None 2022-09-03T07:26:05.952 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:05 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:26:05.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EN9tcqZd4eQ6EE1d/KjANZwqot4= 2022-09-03T07:26:05.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EN9tcqZd4eQ6EE1d/KjANZwqot4=', 'Content-Length': '0'} 2022-09-03T07:26:05.956 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038b1f1dff2364833-006313018d-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.002 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:06.003 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:06.004 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.005 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.006 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.008 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.008 DEBUG:boto:Data: 2022-09-03T07:26:06.009 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.010 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.011 DEBUG:boto:Params: {} 2022-09-03T07:26:06.011 DEBUG:boto:Token: None 2022-09-03T07:26:06.012 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IiM7TlKbmCkrV6LyGoL505L88ss= 2022-09-03T07:26:06.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IiM7TlKbmCkrV6LyGoL505L88ss=', 'Content-Length': '0'} 2022-09-03T07:26:06.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054114e6a28224577-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.016 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:06.017 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:06.017 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.018 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.018 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.019 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.019 DEBUG:boto:Data: 2022-09-03T07:26:06.021 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.022 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.023 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.024 DEBUG:boto:Params: {} 2022-09-03T07:26:06.025 DEBUG:boto:Token: None 2022-09-03T07:26:06.026 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IiM7TlKbmCkrV6LyGoL505L88ss= 2022-09-03T07:26:06.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IiM7TlKbmCkrV6LyGoL505L88ss=', 'Content-Length': '0'} 2022-09-03T07:26:06.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080818782edcb4293-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.032 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:26:06.033 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:06.033 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:06.034 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.034 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.035 DEBUG:boto:Method: GET 2022-09-03T07:26:06.036 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.036 DEBUG:boto:Data: 2022-09-03T07:26:06.037 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.037 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.038 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.038 DEBUG:boto:Params: {} 2022-09-03T07:26:06.039 DEBUG:boto:Token: None 2022-09-03T07:26:06.040 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qLcONZjjWPI0xxi6lreuX2n/E8k= 2022-09-03T07:26:06.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qLcONZjjWPI0xxi6lreuX2n/E8k=', 'Content-Length': '0'} 2022-09-03T07:26:06.043 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed7c3f4fba53c136-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.044 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:06.044 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:06.045 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.045 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.046 DEBUG:boto:Method: GET 2022-09-03T07:26:06.046 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.047 DEBUG:boto:Data: 2022-09-03T07:26:06.048 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.048 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.049 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.049 DEBUG:boto:Params: {} 2022-09-03T07:26:06.050 DEBUG:boto:Token: None 2022-09-03T07:26:06.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:26:06.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qLcONZjjWPI0xxi6lreuX2n/E8k= 2022-09-03T07:26:06.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qLcONZjjWPI0xxi6lreuX2n/E8k=', 'Content-Length': '0'} 2022-09-03T07:26:06.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007b198005a1e3da3f-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.054 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.055 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.055 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:26:06.056 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:26:06.057 DEBUG:boto:Method: GET 2022-09-03T07:26:06.057 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:26:06.058 DEBUG:boto:Data: 2022-09-03T07:26:06.058 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.059 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.060 DEBUG:boto:Params: {} 2022-09-03T07:26:06.060 DEBUG:boto:Token: None 2022-09-03T07:26:06.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.065 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac95a739f7d7c2fb-006313018e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.065 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.066 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.067 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.067 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:26:06.068 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:26:06.068 DEBUG:boto:Method: GET 2022-09-03T07:26:06.069 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:26:06.069 DEBUG:boto:Data: 2022-09-03T07:26:06.070 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.070 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.071 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.071 DEBUG:boto:Params: {} 2022-09-03T07:26:06.072 DEBUG:boto:Token: None 2022-09-03T07:26:06.073 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.076 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022e5b314a109e884-006313018e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.077 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.078 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:06.078 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:06.079 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.079 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:26:06.080 DEBUG:boto:Data: 2022-09-03T07:26:06.080 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.081 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.081 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.082 DEBUG:boto:Params: {} 2022-09-03T07:26:06.083 DEBUG:boto:Token: None 2022-09-03T07:26:06.083 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj29 2022-09-03T07:26:06.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gAhE4vcCcbOKuPlcnKLegFosyWY= 2022-09-03T07:26:06.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gAhE4vcCcbOKuPlcnKLegFosyWY=', 'Content-Length': '0'} 2022-09-03T07:26:06.087 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d0e3ef3b0551becf-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.088 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:06.089 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:06.090 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.091 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:26:06.092 DEBUG:boto:Data: 2022-09-03T07:26:06.094 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.098 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.099 DEBUG:boto:Params: {} 2022-09-03T07:26:06.100 DEBUG:boto:Token: None 2022-09-03T07:26:06.101 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj29 2022-09-03T07:26:06.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gAhE4vcCcbOKuPlcnKLegFosyWY= 2022-09-03T07:26:06.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gAhE4vcCcbOKuPlcnKLegFosyWY=', 'Content-Length': '0'} 2022-09-03T07:26:06.107 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055b7cf5e5640ab56-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.109 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:26:06.110 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:06.111 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:06.112 DEBUG:boto:Method: GET 2022-09-03T07:26:06.113 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:26:06.114 DEBUG:boto:Data: 2022-09-03T07:26:06.115 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.117 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.117 DEBUG:boto:Params: {} 2022-09-03T07:26:06.118 DEBUG:boto:Token: None 2022-09-03T07:26:06.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj29 2022-09-03T07:26:06.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2OW3wrh/i/3npsC5U1Ni2o1SCWU= 2022-09-03T07:26:06.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2OW3wrh/i/3npsC5U1Ni2o1SCWU=', 'Content-Length': '0'} 2022-09-03T07:26:06.122 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092edffc9c2b2b0a5-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.166 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:26:06.167 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:26:06.167 DEBUG:boto:Method: GET 2022-09-03T07:26:06.168 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:26:06.169 DEBUG:boto:Data: 2022-09-03T07:26:06.170 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.171 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.172 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.173 DEBUG:boto:Params: {} 2022-09-03T07:26:06.174 DEBUG:boto:Token: None 2022-09-03T07:26:06.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj29 2022-09-03T07:26:06.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2OW3wrh/i/3npsC5U1Ni2o1SCWU= 2022-09-03T07:26:06.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2OW3wrh/i/3npsC5U1Ni2o1SCWU=', 'Content-Length': '0'} 2022-09-03T07:26:06.181 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c934379fe193dd0-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.183 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.184 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.185 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:26:06.186 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:26:06.187 DEBUG:boto:Method: GET 2022-09-03T07:26:06.188 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:26:06.189 DEBUG:boto:Data: 2022-09-03T07:26:06.190 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.191 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.192 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.193 DEBUG:boto:Params: {} 2022-09-03T07:26:06.195 DEBUG:boto:Token: None 2022-09-03T07:26:06.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.200 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000461f28373697136b-006313018e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.201 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.202 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.202 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.203 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:26:06.204 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:26:06.204 DEBUG:boto:Method: GET 2022-09-03T07:26:06.205 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:26:06.205 DEBUG:boto:Data: 2022-09-03T07:26:06.206 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.207 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.207 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.208 DEBUG:boto:Params: {} 2022-09-03T07:26:06.208 DEBUG:boto:Token: None 2022-09-03T07:26:06.209 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.215 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ebf51b4da1d65111-006313018e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.217 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.219 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:26:06.219 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.222 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.222 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.223 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.224 DEBUG:boto:Method: GET 2022-09-03T07:26:06.225 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.225 DEBUG:boto:Data: 2022-09-03T07:26:06.226 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.226 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.227 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.227 DEBUG:boto:Params: {} 2022-09-03T07:26:06.228 DEBUG:boto:Token: None 2022-09-03T07:26:06.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PmKcDrEWkAWFIii+JyJjU4HCfwc= 2022-09-03T07:26:06.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PmKcDrEWkAWFIii+JyJjU4HCfwc=', 'Content-Length': '0'} 2022-09-03T07:26:06.232 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003fb3250715bc4f4e-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.233 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.234 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.234 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.235 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.235 DEBUG:boto:Method: GET 2022-09-03T07:26:06.236 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.236 DEBUG:boto:Data: 2022-09-03T07:26:06.237 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.237 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.238 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.239 DEBUG:boto:Params: {} 2022-09-03T07:26:06.239 DEBUG:boto:Token: None 2022-09-03T07:26:06.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PmKcDrEWkAWFIii+JyJjU4HCfwc= 2022-09-03T07:26:06.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PmKcDrEWkAWFIii+JyJjU4HCfwc=', 'Content-Length': '0'} 2022-09-03T07:26:06.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ed8ded864b9955f-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.244 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.245 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.246 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.246 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.247 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.247 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.248 DEBUG:boto:Data: 2022-09-03T07:26:06.248 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.249 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.249 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.250 DEBUG:boto:Params: {} 2022-09-03T07:26:06.250 DEBUG:boto:Token: None 2022-09-03T07:26:06.251 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.252 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:STP8xUCTjU7hI7XRB6euuvF/Nrk= 2022-09-03T07:26:06.252 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:STP8xUCTjU7hI7XRB6euuvF/Nrk=', 'Content-Length': '0'} 2022-09-03T07:26:06.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8571641d992ff5a-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.255 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.260 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.261 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.261 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.262 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.263 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.264 DEBUG:boto:Data: 2022-09-03T07:26:06.265 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.266 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.267 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.268 DEBUG:boto:Params: {} 2022-09-03T07:26:06.269 DEBUG:boto:Token: None 2022-09-03T07:26:06.270 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:STP8xUCTjU7hI7XRB6euuvF/Nrk= 2022-09-03T07:26:06.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:STP8xUCTjU7hI7XRB6euuvF/Nrk=', 'Content-Length': '0'} 2022-09-03T07:26:06.274 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c074590b2b461d9-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.275 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:26:06.276 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.277 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.278 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.278 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.279 DEBUG:boto:Method: GET 2022-09-03T07:26:06.280 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.280 DEBUG:boto:Data: 2022-09-03T07:26:06.281 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.281 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.282 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.282 DEBUG:boto:Params: {} 2022-09-03T07:26:06.283 DEBUG:boto:Token: None 2022-09-03T07:26:06.284 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.285 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PmKcDrEWkAWFIii+JyJjU4HCfwc= 2022-09-03T07:26:06.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PmKcDrEWkAWFIii+JyJjU4HCfwc=', 'Content-Length': '0'} 2022-09-03T07:26:06.287 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009bfe789ecf4aa683-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.288 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.289 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.289 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.290 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.291 DEBUG:boto:Method: GET 2022-09-03T07:26:06.292 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.292 DEBUG:boto:Data: 2022-09-03T07:26:06.293 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.293 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.294 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.295 DEBUG:boto:Params: {} 2022-09-03T07:26:06.295 DEBUG:boto:Token: None 2022-09-03T07:26:06.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:26:06.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PmKcDrEWkAWFIii+JyJjU4HCfwc= 2022-09-03T07:26:06.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PmKcDrEWkAWFIii+JyJjU4HCfwc=', 'Content-Length': '0'} 2022-09-03T07:26:06.299 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000092f44c3d7da3614-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.342 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.344 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.344 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:26:06.345 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:26:06.345 DEBUG:boto:Method: GET 2022-09-03T07:26:06.346 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:26:06.346 DEBUG:boto:Data: 2022-09-03T07:26:06.347 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.347 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.348 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.348 DEBUG:boto:Params: {} 2022-09-03T07:26:06.349 DEBUG:boto:Token: None 2022-09-03T07:26:06.350 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.355 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009961228f8ac5d4d9-006313018e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.357 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.358 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.359 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.360 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:26:06.361 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:26:06.362 DEBUG:boto:Method: GET 2022-09-03T07:26:06.362 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:26:06.363 DEBUG:boto:Data: 2022-09-03T07:26:06.364 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.364 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.365 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.366 DEBUG:boto:Params: {} 2022-09-03T07:26:06.367 DEBUG:boto:Token: None 2022-09-03T07:26:06.368 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.373 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ae7ea2abbddb883f-006313018e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.374 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.375 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.376 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.377 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.377 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:26:06.378 DEBUG:boto:Data: 2022-09-03T07:26:06.378 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.379 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.380 DEBUG:boto:Params: {} 2022-09-03T07:26:06.380 DEBUG:boto:Token: None 2022-09-03T07:26:06.381 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33 2022-09-03T07:26:06.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kbXDazerrvsh4MyXGWySoPlXcu4= 2022-09-03T07:26:06.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kbXDazerrvsh4MyXGWySoPlXcu4=', 'Content-Length': '0'} 2022-09-03T07:26:06.386 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087a54a684add10bd-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.387 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.387 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.388 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.388 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:26:06.389 DEBUG:boto:Data: 2022-09-03T07:26:06.389 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.390 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.390 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.391 DEBUG:boto:Params: {} 2022-09-03T07:26:06.391 DEBUG:boto:Token: None 2022-09-03T07:26:06.392 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33 2022-09-03T07:26:06.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kbXDazerrvsh4MyXGWySoPlXcu4= 2022-09-03T07:26:06.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kbXDazerrvsh4MyXGWySoPlXcu4=', 'Content-Length': '0'} 2022-09-03T07:26:06.396 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020d92f58a2029e12-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.396 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:26:06.397 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.397 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.398 DEBUG:boto:Method: GET 2022-09-03T07:26:06.399 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:26:06.399 DEBUG:boto:Data: 2022-09-03T07:26:06.400 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.401 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.401 DEBUG:boto:Params: {} 2022-09-03T07:26:06.402 DEBUG:boto:Token: None 2022-09-03T07:26:06.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33 2022-09-03T07:26:06.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onxleqVF0UlCXeVKM10cBsKZIwI= 2022-09-03T07:26:06.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onxleqVF0UlCXeVKM10cBsKZIwI=', 'Content-Length': '0'} 2022-09-03T07:26:06.405 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc311745851c18d0-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.406 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:26:06.407 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:26:06.407 DEBUG:boto:Method: GET 2022-09-03T07:26:06.408 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:26:06.409 DEBUG:boto:Data: 2022-09-03T07:26:06.409 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.410 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.410 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.411 DEBUG:boto:Params: {} 2022-09-03T07:26:06.411 DEBUG:boto:Token: None 2022-09-03T07:26:06.412 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj33 2022-09-03T07:26:06.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:onxleqVF0UlCXeVKM10cBsKZIwI= 2022-09-03T07:26:06.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:onxleqVF0UlCXeVKM10cBsKZIwI=', 'Content-Length': '0'} 2022-09-03T07:26:06.417 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d90b935329bfbfc2-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.418 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.419 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.420 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:26:06.420 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:26:06.421 DEBUG:boto:Method: GET 2022-09-03T07:26:06.421 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:26:06.422 DEBUG:boto:Data: 2022-09-03T07:26:06.422 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.423 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.424 DEBUG:boto:Params: {} 2022-09-03T07:26:06.425 DEBUG:boto:Token: None 2022-09-03T07:26:06.425 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.430 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b74b0078dceed72d-006313018e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.430 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.431 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.432 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.432 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:26:06.433 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:26:06.433 DEBUG:boto:Method: GET 2022-09-03T07:26:06.434 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:26:06.434 DEBUG:boto:Data: 2022-09-03T07:26:06.435 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.435 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.436 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.436 DEBUG:boto:Params: {} 2022-09-03T07:26:06.437 DEBUG:boto:Token: None 2022-09-03T07:26:06.438 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.441 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf27bba5368511ab-006313018e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.442 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.443 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:26:06.444 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.444 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.445 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.445 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.446 DEBUG:boto:Method: GET 2022-09-03T07:26:06.446 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.447 DEBUG:boto:Data: 2022-09-03T07:26:06.447 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.448 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.449 DEBUG:boto:Params: {} 2022-09-03T07:26:06.449 DEBUG:boto:Token: None 2022-09-03T07:26:06.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+cJB/QWC5RThzxBzA8LVp48V3vQ= 2022-09-03T07:26:06.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+cJB/QWC5RThzxBzA8LVp48V3vQ=', 'Content-Length': '0'} 2022-09-03T07:26:06.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063fd8a57a5167299-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.454 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.455 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.455 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.456 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.456 DEBUG:boto:Method: GET 2022-09-03T07:26:06.457 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.457 DEBUG:boto:Data: 2022-09-03T07:26:06.458 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.459 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.459 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.460 DEBUG:boto:Params: {} 2022-09-03T07:26:06.460 DEBUG:boto:Token: None 2022-09-03T07:26:06.461 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+cJB/QWC5RThzxBzA8LVp48V3vQ= 2022-09-03T07:26:06.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+cJB/QWC5RThzxBzA8LVp48V3vQ=', 'Content-Length': '0'} 2022-09-03T07:26:06.464 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000058c6a2c1ceb5b3bc-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.465 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.465 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.466 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.466 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.467 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.467 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.468 DEBUG:boto:Data: 2022-09-03T07:26:06.468 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.469 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.470 DEBUG:boto:Params: {} 2022-09-03T07:26:06.471 DEBUG:boto:Token: None 2022-09-03T07:26:06.471 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JPiQQb7LslGZLQX98be5NTCUsI4= 2022-09-03T07:26:06.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JPiQQb7LslGZLQX98be5NTCUsI4=', 'Content-Length': '0'} 2022-09-03T07:26:06.474 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000481cb8f60af4a9ac-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.475 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.476 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.476 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.477 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.477 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.478 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.478 DEBUG:boto:Data: 2022-09-03T07:26:06.479 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.480 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.480 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.481 DEBUG:boto:Params: {} 2022-09-03T07:26:06.481 DEBUG:boto:Token: None 2022-09-03T07:26:06.482 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JPiQQb7LslGZLQX98be5NTCUsI4= 2022-09-03T07:26:06.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JPiQQb7LslGZLQX98be5NTCUsI4=', 'Content-Length': '0'} 2022-09-03T07:26:06.486 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d21a39d0273f818-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.486 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:26:06.487 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.487 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.488 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.488 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.489 DEBUG:boto:Method: GET 2022-09-03T07:26:06.490 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.490 DEBUG:boto:Data: 2022-09-03T07:26:06.491 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.492 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.492 DEBUG:boto:Params: {} 2022-09-03T07:26:06.493 DEBUG:boto:Token: None 2022-09-03T07:26:06.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+cJB/QWC5RThzxBzA8LVp48V3vQ= 2022-09-03T07:26:06.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+cJB/QWC5RThzxBzA8LVp48V3vQ=', 'Content-Length': '0'} 2022-09-03T07:26:06.497 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022c332716f3d0e9d-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.538 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.539 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.539 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.540 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.540 DEBUG:boto:Method: GET 2022-09-03T07:26:06.541 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.541 DEBUG:boto:Data: 2022-09-03T07:26:06.542 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.542 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.543 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.544 DEBUG:boto:Params: {} 2022-09-03T07:26:06.545 DEBUG:boto:Token: None 2022-09-03T07:26:06.546 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:26:06.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+cJB/QWC5RThzxBzA8LVp48V3vQ= 2022-09-03T07:26:06.548 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+cJB/QWC5RThzxBzA8LVp48V3vQ=', 'Content-Length': '0'} 2022-09-03T07:26:06.550 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad4a70762f13a30c-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.552 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.553 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.554 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:26:06.555 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:26:06.556 DEBUG:boto:Method: GET 2022-09-03T07:26:06.557 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:26:06.558 DEBUG:boto:Data: 2022-09-03T07:26:06.558 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.560 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.561 DEBUG:boto:Params: {} 2022-09-03T07:26:06.562 DEBUG:boto:Token: None 2022-09-03T07:26:06.563 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.565 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.569 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002f780f8a28a8cfe0-006313018e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.570 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.572 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.573 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.574 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:26:06.575 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:26:06.575 DEBUG:boto:Method: GET 2022-09-03T07:26:06.577 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:26:06.577 DEBUG:boto:Data: 2022-09-03T07:26:06.578 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.579 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.579 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.580 DEBUG:boto:Params: {} 2022-09-03T07:26:06.580 DEBUG:boto:Token: None 2022-09-03T07:26:06.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.586 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b89447d93bc5f72-006313018e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.587 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.588 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.588 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.589 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.589 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:26:06.590 DEBUG:boto:Data: 2022-09-03T07:26:06.590 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.592 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.592 DEBUG:boto:Params: {} 2022-09-03T07:26:06.593 DEBUG:boto:Token: None 2022-09-03T07:26:06.593 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34 2022-09-03T07:26:06.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i6o0AUiv0w4vdYSbqTlQ7tzhsgg= 2022-09-03T07:26:06.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i6o0AUiv0w4vdYSbqTlQ7tzhsgg=', 'Content-Length': '0'} 2022-09-03T07:26:06.596 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003aebdbb7b9a1020-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.597 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.597 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.598 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.598 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:26:06.599 DEBUG:boto:Data: 2022-09-03T07:26:06.599 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.600 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.600 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.601 DEBUG:boto:Params: {} 2022-09-03T07:26:06.602 DEBUG:boto:Token: None 2022-09-03T07:26:06.602 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34 2022-09-03T07:26:06.603 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:i6o0AUiv0w4vdYSbqTlQ7tzhsgg= 2022-09-03T07:26:06.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:i6o0AUiv0w4vdYSbqTlQ7tzhsgg=', 'Content-Length': '0'} 2022-09-03T07:26:06.606 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026d76220f179ebad-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.606 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:26:06.607 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.608 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.608 DEBUG:boto:Method: GET 2022-09-03T07:26:06.609 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:26:06.609 DEBUG:boto:Data: 2022-09-03T07:26:06.610 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.610 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.611 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.611 DEBUG:boto:Params: {} 2022-09-03T07:26:06.612 DEBUG:boto:Token: None 2022-09-03T07:26:06.613 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34 2022-09-03T07:26:06.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ROttSakMnpEXTPT94IDPxVJf1LM= 2022-09-03T07:26:06.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ROttSakMnpEXTPT94IDPxVJf1LM=', 'Content-Length': '0'} 2022-09-03T07:26:06.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045a5c1bb740e40bc-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.620 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:26:06.621 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:26:06.622 DEBUG:boto:Method: GET 2022-09-03T07:26:06.623 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:26:06.624 DEBUG:boto:Data: 2022-09-03T07:26:06.625 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.626 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.627 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.628 DEBUG:boto:Params: {} 2022-09-03T07:26:06.629 DEBUG:boto:Token: None 2022-09-03T07:26:06.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj34 2022-09-03T07:26:06.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ROttSakMnpEXTPT94IDPxVJf1LM= 2022-09-03T07:26:06.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ROttSakMnpEXTPT94IDPxVJf1LM=', 'Content-Length': '0'} 2022-09-03T07:26:06.635 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004bf15da276b2588-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.636 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.637 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.638 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:26:06.639 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:26:06.640 DEBUG:boto:Method: GET 2022-09-03T07:26:06.641 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:26:06.642 DEBUG:boto:Data: 2022-09-03T07:26:06.643 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.645 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.645 DEBUG:boto:Params: {} 2022-09-03T07:26:06.647 DEBUG:boto:Token: None 2022-09-03T07:26:06.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.651 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db94f9b134d97388-006313018e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.653 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.654 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.655 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.655 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:26:06.656 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:26:06.657 DEBUG:boto:Method: GET 2022-09-03T07:26:06.657 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:26:06.658 DEBUG:boto:Data: 2022-09-03T07:26:06.658 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.659 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.659 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.660 DEBUG:boto:Params: {} 2022-09-03T07:26:06.660 DEBUG:boto:Token: None 2022-09-03T07:26:06.661 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.662 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.665 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008cdd67f404344490-006313018e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.707 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.708 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:26:06.708 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.709 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.709 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.710 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.711 DEBUG:boto:Method: GET 2022-09-03T07:26:06.712 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.712 DEBUG:boto:Data: 2022-09-03T07:26:06.713 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.714 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.714 DEBUG:boto:Params: {} 2022-09-03T07:26:06.715 DEBUG:boto:Token: None 2022-09-03T07:26:06.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xkaniUD1Mz5PMwuDh5gVMJps+QI= 2022-09-03T07:26:06.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xkaniUD1Mz5PMwuDh5gVMJps+QI=', 'Content-Length': '0'} 2022-09-03T07:26:06.719 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ba19c912fd71b25-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.720 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.720 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.721 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.722 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.723 DEBUG:boto:Method: GET 2022-09-03T07:26:06.723 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.724 DEBUG:boto:Data: 2022-09-03T07:26:06.724 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.725 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.725 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.726 DEBUG:boto:Params: {} 2022-09-03T07:26:06.727 DEBUG:boto:Token: None 2022-09-03T07:26:06.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xkaniUD1Mz5PMwuDh5gVMJps+QI= 2022-09-03T07:26:06.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xkaniUD1Mz5PMwuDh5gVMJps+QI=', 'Content-Length': '0'} 2022-09-03T07:26:06.731 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d24faedf118811ac-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.732 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.732 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.733 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.733 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.734 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.734 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.735 DEBUG:boto:Data: 2022-09-03T07:26:06.735 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.736 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.736 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.737 DEBUG:boto:Params: {} 2022-09-03T07:26:06.738 DEBUG:boto:Token: None 2022-09-03T07:26:06.738 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V575C+Akr7URTafI1aGcPU4Tbxw= 2022-09-03T07:26:06.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V575C+Akr7URTafI1aGcPU4Tbxw=', 'Content-Length': '0'} 2022-09-03T07:26:06.741 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f76336a147bc4cd-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.742 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.743 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.743 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.744 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.744 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.745 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.745 DEBUG:boto:Data: 2022-09-03T07:26:06.746 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.746 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.747 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.747 DEBUG:boto:Params: {} 2022-09-03T07:26:06.748 DEBUG:boto:Token: None 2022-09-03T07:26:06.749 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V575C+Akr7URTafI1aGcPU4Tbxw= 2022-09-03T07:26:06.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V575C+Akr7URTafI1aGcPU4Tbxw=', 'Content-Length': '0'} 2022-09-03T07:26:06.752 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000288f2ad7fc4f9756-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.752 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:26:06.753 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.754 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.754 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.755 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.756 DEBUG:boto:Method: GET 2022-09-03T07:26:06.757 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.758 DEBUG:boto:Data: 2022-09-03T07:26:06.759 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.761 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.762 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.763 DEBUG:boto:Params: {} 2022-09-03T07:26:06.764 DEBUG:boto:Token: None 2022-09-03T07:26:06.765 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xkaniUD1Mz5PMwuDh5gVMJps+QI= 2022-09-03T07:26:06.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xkaniUD1Mz5PMwuDh5gVMJps+QI=', 'Content-Length': '0'} 2022-09-03T07:26:06.769 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da8cfaa8d1086084-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.771 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.772 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.773 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.774 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.774 DEBUG:boto:Method: GET 2022-09-03T07:26:06.775 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.775 DEBUG:boto:Data: 2022-09-03T07:26:06.776 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.777 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.778 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.778 DEBUG:boto:Params: {} 2022-09-03T07:26:06.779 DEBUG:boto:Token: None 2022-09-03T07:26:06.779 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:26:06.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xkaniUD1Mz5PMwuDh5gVMJps+QI= 2022-09-03T07:26:06.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xkaniUD1Mz5PMwuDh5gVMJps+QI=', 'Content-Length': '0'} 2022-09-03T07:26:06.783 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031c394eff7c093b7-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.784 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.785 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.786 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:06.787 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:06.787 DEBUG:boto:Method: GET 2022-09-03T07:26:06.788 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:26:06.789 DEBUG:boto:Data: 2022-09-03T07:26:06.790 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.791 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.791 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.792 DEBUG:boto:Params: {} 2022-09-03T07:26:06.793 DEBUG:boto:Token: None 2022-09-03T07:26:06.794 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.800 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f923918937dc544-006313018e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.801 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.802 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.803 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.804 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:06.805 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:06.806 DEBUG:boto:Method: GET 2022-09-03T07:26:06.807 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:26:06.807 DEBUG:boto:Data: 2022-09-03T07:26:06.808 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.809 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.809 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.810 DEBUG:boto:Params: {} 2022-09-03T07:26:06.810 DEBUG:boto:Token: None 2022-09-03T07:26:06.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.816 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000209b1faace0c0b86-006313018e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.817 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.818 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.819 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.819 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.820 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:26:06.820 DEBUG:boto:Data: 2022-09-03T07:26:06.821 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.822 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.823 DEBUG:boto:Params: {} 2022-09-03T07:26:06.823 DEBUG:boto:Token: None 2022-09-03T07:26:06.824 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38 2022-09-03T07:26:06.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E77XSsxGpavt6si+MEtbPUWNEh0= 2022-09-03T07:26:06.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E77XSsxGpavt6si+MEtbPUWNEh0=', 'Content-Length': '0'} 2022-09-03T07:26:06.827 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000825cc6942871740a-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.828 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.829 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.829 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.830 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:26:06.831 DEBUG:boto:Data: 2022-09-03T07:26:06.831 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.833 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.833 DEBUG:boto:Params: {} 2022-09-03T07:26:06.834 DEBUG:boto:Token: None 2022-09-03T07:26:06.835 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38 2022-09-03T07:26:06.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E77XSsxGpavt6si+MEtbPUWNEh0= 2022-09-03T07:26:06.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E77XSsxGpavt6si+MEtbPUWNEh0=', 'Content-Length': '0'} 2022-09-03T07:26:06.840 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b84f247dc68fd0aa-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.841 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:26:06.842 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.843 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.844 DEBUG:boto:Method: GET 2022-09-03T07:26:06.844 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:26:06.845 DEBUG:boto:Data: 2022-09-03T07:26:06.846 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.848 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.849 DEBUG:boto:Params: {} 2022-09-03T07:26:06.850 DEBUG:boto:Token: None 2022-09-03T07:26:06.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38 2022-09-03T07:26:06.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y02u7kGvrYE/DUKvi1YZ7rDwQU0= 2022-09-03T07:26:06.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y02u7kGvrYE/DUKvi1YZ7rDwQU0=', 'Content-Length': '0'} 2022-09-03T07:26:06.856 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000241ceed23e93270a-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.857 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:26:06.858 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:26:06.859 DEBUG:boto:Method: GET 2022-09-03T07:26:06.859 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:26:06.860 DEBUG:boto:Data: 2022-09-03T07:26:06.860 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.861 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.862 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.863 DEBUG:boto:Params: {} 2022-09-03T07:26:06.864 DEBUG:boto:Token: None 2022-09-03T07:26:06.864 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj38 2022-09-03T07:26:06.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y02u7kGvrYE/DUKvi1YZ7rDwQU0= 2022-09-03T07:26:06.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y02u7kGvrYE/DUKvi1YZ7rDwQU0=', 'Content-Length': '0'} 2022-09-03T07:26:06.867 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c6be1532b7ea481-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.910 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.911 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.912 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:06.912 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:06.913 DEBUG:boto:Method: GET 2022-09-03T07:26:06.914 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:26:06.914 DEBUG:boto:Data: 2022-09-03T07:26:06.915 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.916 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.917 DEBUG:boto:Params: {} 2022-09-03T07:26:06.917 DEBUG:boto:Token: None 2022-09-03T07:26:06.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.924 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c83b7f655366e011-006313018e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.925 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.926 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:06.927 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:06.928 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:06.929 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:26:06.930 DEBUG:boto:Method: GET 2022-09-03T07:26:06.931 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:26:06.932 DEBUG:boto:Data: 2022-09-03T07:26:06.934 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.935 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.936 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.937 DEBUG:boto:Params: {} 2022-09-03T07:26:06.939 DEBUG:boto:Token: None 2022-09-03T07:26:06.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/ 2022-09-03T07:26:06.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk= 2022-09-03T07:26:06.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QInKVpaI6DFTuasb0YfRwJQ0PGk=', 'Content-Length': '0'} 2022-09-03T07:26:06.946 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e000b1541af4f62f-006313018e-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.946 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:06.947 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:26:06.948 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:06.948 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:06.949 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.949 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.950 DEBUG:boto:Method: GET 2022-09-03T07:26:06.951 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.951 DEBUG:boto:Data: 2022-09-03T07:26:06.952 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.955 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.956 DEBUG:boto:Params: {} 2022-09-03T07:26:06.957 DEBUG:boto:Token: None 2022-09-03T07:26:06.958 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.959 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7OGtu5TGiB9XouyO8xmpslVIlYY= 2022-09-03T07:26:06.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7OGtu5TGiB9XouyO8xmpslVIlYY=', 'Content-Length': '0'} 2022-09-03T07:26:06.963 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d5ab3b3be2d23f00-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.965 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:06.965 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:06.967 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.967 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.969 DEBUG:boto:Method: GET 2022-09-03T07:26:06.970 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.971 DEBUG:boto:Data: 2022-09-03T07:26:06.972 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.972 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:06.973 DEBUG:boto:Port: 8000 2022-09-03T07:26:06.974 DEBUG:boto:Params: {} 2022-09-03T07:26:06.975 DEBUG:boto:Token: None 2022-09-03T07:26:06.975 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7OGtu5TGiB9XouyO8xmpslVIlYY= 2022-09-03T07:26:06.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7OGtu5TGiB9XouyO8xmpslVIlYY=', 'Content-Length': '0'} 2022-09-03T07:26:06.979 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000130e28c940f73bf1-006313018e-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.980 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:06.981 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:06.982 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.982 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.983 DEBUG:boto:Method: HEAD 2022-09-03T07:26:06.983 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.984 DEBUG:boto:Data: 2022-09-03T07:26:06.985 DEBUG:boto:Headers: {} 2022-09-03T07:26:06.986 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:06.987 DEBUG:boto:Port: 8001 2022-09-03T07:26:06.987 DEBUG:boto:Params: {} 2022-09-03T07:26:06.988 DEBUG:boto:Token: None 2022-09-03T07:26:06.989 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:06 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3mGLJIqiqDxRwDOZAGwrSBxOoJg= 2022-09-03T07:26:06.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3mGLJIqiqDxRwDOZAGwrSBxOoJg=', 'Content-Length': '0'} 2022-09-03T07:26:06.994 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfebdcd24c30f21c-006313018e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:06.995 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:06.997 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:06.998 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:06.999 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.000 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.000 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.001 DEBUG:boto:Data: 2022-09-03T07:26:07.003 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.004 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.006 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.007 DEBUG:boto:Params: {} 2022-09-03T07:26:07.008 DEBUG:boto:Token: None 2022-09-03T07:26:07.009 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L3H/BXq9aNkiMf2u7+T3rNdxQ1o= 2022-09-03T07:26:07.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L3H/BXq9aNkiMf2u7+T3rNdxQ1o=', 'Content-Length': '0'} 2022-09-03T07:26:07.015 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b20963a0c4b4f359-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.018 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:26:07.019 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:07.022 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:07.023 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.024 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.025 DEBUG:boto:Method: GET 2022-09-03T07:26:07.026 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.027 DEBUG:boto:Data: 2022-09-03T07:26:07.028 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.030 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.035 DEBUG:boto:Params: {} 2022-09-03T07:26:07.036 DEBUG:boto:Token: None 2022-09-03T07:26:07.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1n5xmOYUtkrEOsYkUzPwiGL6sJ4= 2022-09-03T07:26:07.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1n5xmOYUtkrEOsYkUzPwiGL6sJ4=', 'Content-Length': '0'} 2022-09-03T07:26:07.043 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a11766438a1acc70-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.045 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:07.046 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:07.047 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.048 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.049 DEBUG:boto:Method: GET 2022-09-03T07:26:07.050 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.054 DEBUG:boto:Data: 2022-09-03T07:26:07.055 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.056 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.057 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.058 DEBUG:boto:Params: {} 2022-09-03T07:26:07.060 DEBUG:boto:Token: None 2022-09-03T07:26:07.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:26:07.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1n5xmOYUtkrEOsYkUzPwiGL6sJ4= 2022-09-03T07:26:07.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1n5xmOYUtkrEOsYkUzPwiGL6sJ4=', 'Content-Length': '0'} 2022-09-03T07:26:07.067 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5714b37ec413415-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.069 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.070 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.071 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:07.072 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:07.073 DEBUG:boto:Method: GET 2022-09-03T07:26:07.074 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:26:07.075 DEBUG:boto:Data: 2022-09-03T07:26:07.076 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.078 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.079 DEBUG:boto:Params: {} 2022-09-03T07:26:07.080 DEBUG:boto:Token: None 2022-09-03T07:26:07.081 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.087 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dd5be90404d1dab3-006313018f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.088 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.088 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.089 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.090 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:07.090 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:07.091 DEBUG:boto:Method: GET 2022-09-03T07:26:07.092 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:26:07.093 DEBUG:boto:Data: 2022-09-03T07:26:07.094 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.097 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.099 DEBUG:boto:Params: {} 2022-09-03T07:26:07.100 DEBUG:boto:Token: None 2022-09-03T07:26:07.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.108 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d66a48875dd9ba17-006313018f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.109 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.111 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:07.112 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:07.113 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.114 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:26:07.115 DEBUG:boto:Data: 2022-09-03T07:26:07.116 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.118 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.119 DEBUG:boto:Params: {} 2022-09-03T07:26:07.120 DEBUG:boto:Token: None 2022-09-03T07:26:07.121 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj44 2022-09-03T07:26:07.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B7suAOl0ELA0jQ0GpPRgWY2mKCo= 2022-09-03T07:26:07.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B7suAOl0ELA0jQ0GpPRgWY2mKCo=', 'Content-Length': '0'} 2022-09-03T07:26:07.125 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071d791e637fa9b7f-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.126 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:07.127 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:07.127 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.128 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:26:07.128 DEBUG:boto:Data: 2022-09-03T07:26:07.129 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.129 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.130 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.130 DEBUG:boto:Params: {} 2022-09-03T07:26:07.131 DEBUG:boto:Token: None 2022-09-03T07:26:07.132 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj44 2022-09-03T07:26:07.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B7suAOl0ELA0jQ0GpPRgWY2mKCo= 2022-09-03T07:26:07.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B7suAOl0ELA0jQ0GpPRgWY2mKCo=', 'Content-Length': '0'} 2022-09-03T07:26:07.137 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a10d5dd2e4e288bb-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.139 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:26:07.140 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:07.140 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:07.141 DEBUG:boto:Method: GET 2022-09-03T07:26:07.141 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:26:07.142 DEBUG:boto:Data: 2022-09-03T07:26:07.143 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.144 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.144 DEBUG:boto:Params: {} 2022-09-03T07:26:07.145 DEBUG:boto:Token: None 2022-09-03T07:26:07.146 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj44 2022-09-03T07:26:07.147 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I9rHwJYHcr64hF53GN6IRSXHFq0= 2022-09-03T07:26:07.148 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I9rHwJYHcr64hF53GN6IRSXHFq0=', 'Content-Length': '0'} 2022-09-03T07:26:07.150 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002cd7ac69d392c5ef-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.194 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:26:07.196 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:26:07.197 DEBUG:boto:Method: GET 2022-09-03T07:26:07.197 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:26:07.198 DEBUG:boto:Data: 2022-09-03T07:26:07.199 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.200 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.201 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.202 DEBUG:boto:Params: {} 2022-09-03T07:26:07.203 DEBUG:boto:Token: None 2022-09-03T07:26:07.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj44 2022-09-03T07:26:07.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I9rHwJYHcr64hF53GN6IRSXHFq0= 2022-09-03T07:26:07.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I9rHwJYHcr64hF53GN6IRSXHFq0=', 'Content-Length': '0'} 2022-09-03T07:26:07.210 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009311378e526a5d16-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.211 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.212 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.213 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:07.214 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:07.215 DEBUG:boto:Method: GET 2022-09-03T07:26:07.216 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:26:07.216 DEBUG:boto:Data: 2022-09-03T07:26:07.217 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.219 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.220 DEBUG:boto:Params: {} 2022-09-03T07:26:07.221 DEBUG:boto:Token: None 2022-09-03T07:26:07.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.226 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006231ae914eb255e0-006313018f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.227 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.227 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.228 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.229 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:07.229 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:26:07.230 DEBUG:boto:Method: GET 2022-09-03T07:26:07.230 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:26:07.231 DEBUG:boto:Data: 2022-09-03T07:26:07.232 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.233 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.233 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.234 DEBUG:boto:Params: {} 2022-09-03T07:26:07.235 DEBUG:boto:Token: None 2022-09-03T07:26:07.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.243 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009866118e487f668e-006313018f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.245 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.246 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:26:07.247 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.247 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.248 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.249 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.249 DEBUG:boto:Method: GET 2022-09-03T07:26:07.250 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.250 DEBUG:boto:Data: 2022-09-03T07:26:07.251 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.252 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.252 DEBUG:boto:Params: {} 2022-09-03T07:26:07.253 DEBUG:boto:Token: None 2022-09-03T07:26:07.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p1e9eTnwuHTNGCc7Jsb6t5I+hNM= 2022-09-03T07:26:07.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p1e9eTnwuHTNGCc7Jsb6t5I+hNM=', 'Content-Length': '0'} 2022-09-03T07:26:07.257 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbdf3c768a7bf23e-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.258 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.258 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.259 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.259 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.260 DEBUG:boto:Method: GET 2022-09-03T07:26:07.260 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.261 DEBUG:boto:Data: 2022-09-03T07:26:07.261 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.262 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.263 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.263 DEBUG:boto:Params: {} 2022-09-03T07:26:07.264 DEBUG:boto:Token: None 2022-09-03T07:26:07.264 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p1e9eTnwuHTNGCc7Jsb6t5I+hNM= 2022-09-03T07:26:07.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p1e9eTnwuHTNGCc7Jsb6t5I+hNM=', 'Content-Length': '0'} 2022-09-03T07:26:07.268 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b698b751edf6bc30-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.269 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.270 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.271 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.272 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.273 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.274 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.275 DEBUG:boto:Data: 2022-09-03T07:26:07.276 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.278 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.279 DEBUG:boto:Params: {} 2022-09-03T07:26:07.280 DEBUG:boto:Token: None 2022-09-03T07:26:07.280 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:08UdvhB8lgs3/vTqsZBDI0ixfxA= 2022-09-03T07:26:07.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:08UdvhB8lgs3/vTqsZBDI0ixfxA=', 'Content-Length': '0'} 2022-09-03T07:26:07.286 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4510b7a784fb6a5-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.288 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.289 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.290 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.291 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.292 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.293 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.294 DEBUG:boto:Data: 2022-09-03T07:26:07.294 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.295 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.295 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.296 DEBUG:boto:Params: {} 2022-09-03T07:26:07.296 DEBUG:boto:Token: None 2022-09-03T07:26:07.297 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:08UdvhB8lgs3/vTqsZBDI0ixfxA= 2022-09-03T07:26:07.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:08UdvhB8lgs3/vTqsZBDI0ixfxA=', 'Content-Length': '0'} 2022-09-03T07:26:07.300 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd4cc80b2292d740-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.302 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:26:07.303 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.304 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.306 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.307 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.308 DEBUG:boto:Method: GET 2022-09-03T07:26:07.309 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.310 DEBUG:boto:Data: 2022-09-03T07:26:07.311 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.314 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.315 DEBUG:boto:Params: {} 2022-09-03T07:26:07.316 DEBUG:boto:Token: None 2022-09-03T07:26:07.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p1e9eTnwuHTNGCc7Jsb6t5I+hNM= 2022-09-03T07:26:07.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p1e9eTnwuHTNGCc7Jsb6t5I+hNM=', 'Content-Length': '0'} 2022-09-03T07:26:07.321 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089e69110d8e3fae8-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.366 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.367 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.368 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.369 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.370 DEBUG:boto:Method: GET 2022-09-03T07:26:07.372 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.373 DEBUG:boto:Data: 2022-09-03T07:26:07.377 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.378 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.379 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.380 DEBUG:boto:Params: {} 2022-09-03T07:26:07.385 DEBUG:boto:Token: None 2022-09-03T07:26:07.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:26:07.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p1e9eTnwuHTNGCc7Jsb6t5I+hNM= 2022-09-03T07:26:07.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p1e9eTnwuHTNGCc7Jsb6t5I+hNM=', 'Content-Length': '0'} 2022-09-03T07:26:07.391 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d24bf557a89d7d2-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.392 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.393 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.394 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:07.395 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:07.396 DEBUG:boto:Method: GET 2022-09-03T07:26:07.397 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:26:07.398 DEBUG:boto:Data: 2022-09-03T07:26:07.399 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.401 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.402 DEBUG:boto:Params: {} 2022-09-03T07:26:07.403 DEBUG:boto:Token: None 2022-09-03T07:26:07.405 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.406 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.407 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.410 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000050fd3888fd21cc45-006313018f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.411 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.414 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.415 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.417 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:07.418 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:07.419 DEBUG:boto:Method: GET 2022-09-03T07:26:07.420 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:26:07.421 DEBUG:boto:Data: 2022-09-03T07:26:07.422 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.423 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.424 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.425 DEBUG:boto:Params: {} 2022-09-03T07:26:07.427 DEBUG:boto:Token: None 2022-09-03T07:26:07.427 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.430 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.433 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c143fd92286c58ee-006313018f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.436 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.437 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.437 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.438 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.438 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:26:07.439 DEBUG:boto:Data: 2022-09-03T07:26:07.439 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.440 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.440 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.441 DEBUG:boto:Params: {} 2022-09-03T07:26:07.442 DEBUG:boto:Token: None 2022-09-03T07:26:07.442 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46 2022-09-03T07:26:07.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lKWi3ziz+N+XChGAoZE7dqB11kM= 2022-09-03T07:26:07.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lKWi3ziz+N+XChGAoZE7dqB11kM=', 'Content-Length': '0'} 2022-09-03T07:26:07.445 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e4a69384d5a72e78-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.446 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.447 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.447 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.448 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:26:07.448 DEBUG:boto:Data: 2022-09-03T07:26:07.449 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.449 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.450 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.450 DEBUG:boto:Params: {} 2022-09-03T07:26:07.451 DEBUG:boto:Token: None 2022-09-03T07:26:07.452 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46 2022-09-03T07:26:07.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lKWi3ziz+N+XChGAoZE7dqB11kM= 2022-09-03T07:26:07.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lKWi3ziz+N+XChGAoZE7dqB11kM=', 'Content-Length': '0'} 2022-09-03T07:26:07.455 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b850d85ea63d563-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.456 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:26:07.456 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.457 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.457 DEBUG:boto:Method: GET 2022-09-03T07:26:07.458 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:26:07.459 DEBUG:boto:Data: 2022-09-03T07:26:07.459 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.460 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.461 DEBUG:boto:Params: {} 2022-09-03T07:26:07.462 DEBUG:boto:Token: None 2022-09-03T07:26:07.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46 2022-09-03T07:26:07.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2qQc7UgrDqYLBcio/X6sSSrYqYc= 2022-09-03T07:26:07.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2qQc7UgrDqYLBcio/X6sSSrYqYc=', 'Content-Length': '0'} 2022-09-03T07:26:07.466 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b45ad6969e37765-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.467 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:26:07.468 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:26:07.469 DEBUG:boto:Method: GET 2022-09-03T07:26:07.470 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:26:07.471 DEBUG:boto:Data: 2022-09-03T07:26:07.472 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.473 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.474 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.475 DEBUG:boto:Params: {} 2022-09-03T07:26:07.476 DEBUG:boto:Token: None 2022-09-03T07:26:07.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj46 2022-09-03T07:26:07.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2qQc7UgrDqYLBcio/X6sSSrYqYc= 2022-09-03T07:26:07.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2qQc7UgrDqYLBcio/X6sSSrYqYc=', 'Content-Length': '0'} 2022-09-03T07:26:07.482 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003fbe0ddc2e089beb-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.484 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.485 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.486 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:07.487 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:07.488 DEBUG:boto:Method: GET 2022-09-03T07:26:07.489 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:26:07.490 DEBUG:boto:Data: 2022-09-03T07:26:07.491 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.492 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.493 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.494 DEBUG:boto:Params: {} 2022-09-03T07:26:07.495 DEBUG:boto:Token: None 2022-09-03T07:26:07.496 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.501 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f784cd3d016d5aa8-006313018f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.503 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.504 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.504 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.505 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:07.505 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:26:07.506 DEBUG:boto:Method: GET 2022-09-03T07:26:07.506 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:26:07.507 DEBUG:boto:Data: 2022-09-03T07:26:07.507 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.508 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.508 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.509 DEBUG:boto:Params: {} 2022-09-03T07:26:07.510 DEBUG:boto:Token: None 2022-09-03T07:26:07.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.515 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f3337d2225ee5d6c-006313018f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.568 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.570 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:26:07.571 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.572 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.573 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.574 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.575 DEBUG:boto:Method: GET 2022-09-03T07:26:07.576 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.576 DEBUG:boto:Data: 2022-09-03T07:26:07.577 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.578 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.579 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.580 DEBUG:boto:Params: {} 2022-09-03T07:26:07.581 DEBUG:boto:Token: None 2022-09-03T07:26:07.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dU3NYeRM5n7FaHTTB5w6k/kZxlc= 2022-09-03T07:26:07.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dU3NYeRM5n7FaHTTB5w6k/kZxlc=', 'Content-Length': '0'} 2022-09-03T07:26:07.589 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d381d38e92d5f91-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.590 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.591 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.592 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.592 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.593 DEBUG:boto:Method: GET 2022-09-03T07:26:07.593 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.594 DEBUG:boto:Data: 2022-09-03T07:26:07.594 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.595 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.595 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.596 DEBUG:boto:Params: {} 2022-09-03T07:26:07.597 DEBUG:boto:Token: None 2022-09-03T07:26:07.597 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.598 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dU3NYeRM5n7FaHTTB5w6k/kZxlc= 2022-09-03T07:26:07.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dU3NYeRM5n7FaHTTB5w6k/kZxlc=', 'Content-Length': '0'} 2022-09-03T07:26:07.610 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005506206136589e7e-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.611 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.612 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.612 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.613 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.614 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.614 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.615 DEBUG:boto:Data: 2022-09-03T07:26:07.616 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.617 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.618 DEBUG:boto:Params: {} 2022-09-03T07:26:07.618 DEBUG:boto:Token: None 2022-09-03T07:26:07.619 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PHwXEz6EGuOa+X1ObvUCuqQUkjQ= 2022-09-03T07:26:07.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PHwXEz6EGuOa+X1ObvUCuqQUkjQ=', 'Content-Length': '0'} 2022-09-03T07:26:07.623 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007030e6d97447cfe-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.624 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.625 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.626 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.626 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.627 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.627 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.628 DEBUG:boto:Data: 2022-09-03T07:26:07.628 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.629 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.629 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.630 DEBUG:boto:Params: {} 2022-09-03T07:26:07.631 DEBUG:boto:Token: None 2022-09-03T07:26:07.632 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PHwXEz6EGuOa+X1ObvUCuqQUkjQ= 2022-09-03T07:26:07.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PHwXEz6EGuOa+X1ObvUCuqQUkjQ=', 'Content-Length': '0'} 2022-09-03T07:26:07.636 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000874f444aacde335e-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.637 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:26:07.638 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.638 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.639 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.639 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.641 DEBUG:boto:Method: GET 2022-09-03T07:26:07.641 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.642 DEBUG:boto:Data: 2022-09-03T07:26:07.642 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.644 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.644 DEBUG:boto:Params: {} 2022-09-03T07:26:07.645 DEBUG:boto:Token: None 2022-09-03T07:26:07.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dU3NYeRM5n7FaHTTB5w6k/kZxlc= 2022-09-03T07:26:07.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dU3NYeRM5n7FaHTTB5w6k/kZxlc=', 'Content-Length': '0'} 2022-09-03T07:26:07.649 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac0069deee4e7514-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.650 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.650 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.651 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.652 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.653 DEBUG:boto:Method: GET 2022-09-03T07:26:07.653 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.654 DEBUG:boto:Data: 2022-09-03T07:26:07.655 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.655 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.656 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.656 DEBUG:boto:Params: {} 2022-09-03T07:26:07.657 DEBUG:boto:Token: None 2022-09-03T07:26:07.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:26:07.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dU3NYeRM5n7FaHTTB5w6k/kZxlc= 2022-09-03T07:26:07.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dU3NYeRM5n7FaHTTB5w6k/kZxlc=', 'Content-Length': '0'} 2022-09-03T07:26:07.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d1cdf27c08a0f389-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.662 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.663 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.664 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:07.665 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:07.665 DEBUG:boto:Method: GET 2022-09-03T07:26:07.666 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:26:07.666 DEBUG:boto:Data: 2022-09-03T07:26:07.667 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.668 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.668 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.669 DEBUG:boto:Params: {} 2022-09-03T07:26:07.669 DEBUG:boto:Token: None 2022-09-03T07:26:07.670 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.672 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.675 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e3b5f5edef7f197-006313018f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.676 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.677 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.677 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.678 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:07.678 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:07.679 DEBUG:boto:Method: GET 2022-09-03T07:26:07.679 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:26:07.680 DEBUG:boto:Data: 2022-09-03T07:26:07.681 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.681 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.682 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.683 DEBUG:boto:Params: {} 2022-09-03T07:26:07.683 DEBUG:boto:Token: None 2022-09-03T07:26:07.684 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.686 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.688 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000232bad1084301fa4-006313018f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.690 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.691 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.692 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.693 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.694 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:26:07.695 DEBUG:boto:Data: 2022-09-03T07:26:07.697 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.698 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.699 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.700 DEBUG:boto:Params: {} 2022-09-03T07:26:07.701 DEBUG:boto:Token: None 2022-09-03T07:26:07.702 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51 2022-09-03T07:26:07.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yzoKnXWVk6QVM4ZS4xN28Tihab4= 2022-09-03T07:26:07.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yzoKnXWVk6QVM4ZS4xN28Tihab4=', 'Content-Length': '0'} 2022-09-03T07:26:07.707 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039b0bdd01e394e3e-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.708 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.709 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.710 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.711 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:26:07.712 DEBUG:boto:Data: 2022-09-03T07:26:07.712 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.713 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.715 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.716 DEBUG:boto:Params: {} 2022-09-03T07:26:07.716 DEBUG:boto:Token: None 2022-09-03T07:26:07.717 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51 2022-09-03T07:26:07.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yzoKnXWVk6QVM4ZS4xN28Tihab4= 2022-09-03T07:26:07.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yzoKnXWVk6QVM4ZS4xN28Tihab4=', 'Content-Length': '0'} 2022-09-03T07:26:07.721 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000949a11fa4bbec5e1-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.722 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:26:07.722 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.723 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.723 DEBUG:boto:Method: GET 2022-09-03T07:26:07.724 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:26:07.724 DEBUG:boto:Data: 2022-09-03T07:26:07.725 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.726 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.726 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.727 DEBUG:boto:Params: {} 2022-09-03T07:26:07.739 DEBUG:boto:Token: None 2022-09-03T07:26:07.740 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51 2022-09-03T07:26:07.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZMw+dNFpCxZwLgJRiENOJ5mqxW8= 2022-09-03T07:26:07.742 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZMw+dNFpCxZwLgJRiENOJ5mqxW8=', 'Content-Length': '0'} 2022-09-03T07:26:07.744 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d7c41fa3bafb8cf-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.787 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:26:07.787 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:26:07.788 DEBUG:boto:Method: GET 2022-09-03T07:26:07.789 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:26:07.789 DEBUG:boto:Data: 2022-09-03T07:26:07.790 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.791 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.791 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.792 DEBUG:boto:Params: {} 2022-09-03T07:26:07.793 DEBUG:boto:Token: None 2022-09-03T07:26:07.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj51 2022-09-03T07:26:07.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZMw+dNFpCxZwLgJRiENOJ5mqxW8= 2022-09-03T07:26:07.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZMw+dNFpCxZwLgJRiENOJ5mqxW8=', 'Content-Length': '0'} 2022-09-03T07:26:07.798 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005608be38d7df8652-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.799 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.800 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.801 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:07.801 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:07.802 DEBUG:boto:Method: GET 2022-09-03T07:26:07.802 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:26:07.803 DEBUG:boto:Data: 2022-09-03T07:26:07.803 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.805 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.805 DEBUG:boto:Params: {} 2022-09-03T07:26:07.806 DEBUG:boto:Token: None 2022-09-03T07:26:07.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.811 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a45a24a9a2e5bd0-006313018f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.812 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.812 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.813 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.814 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:07.815 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:26:07.816 DEBUG:boto:Method: GET 2022-09-03T07:26:07.816 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:26:07.817 DEBUG:boto:Data: 2022-09-03T07:26:07.817 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.818 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.818 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.819 DEBUG:boto:Params: {} 2022-09-03T07:26:07.820 DEBUG:boto:Token: None 2022-09-03T07:26:07.820 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.824 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007412b665a0c06924-006313018f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.826 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.827 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:26:07.828 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:07.831 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:07.832 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.832 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.833 DEBUG:boto:Method: GET 2022-09-03T07:26:07.833 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.834 DEBUG:boto:Data: 2022-09-03T07:26:07.835 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.836 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.836 DEBUG:boto:Params: {} 2022-09-03T07:26:07.837 DEBUG:boto:Token: None 2022-09-03T07:26:07.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/iiNOc+GAjirlvjd0wcyYWG180Q= 2022-09-03T07:26:07.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/iiNOc+GAjirlvjd0wcyYWG180Q=', 'Content-Length': '0'} 2022-09-03T07:26:07.842 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005bad02b988691ac0-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.843 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:07.843 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:07.844 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.845 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.846 DEBUG:boto:Method: GET 2022-09-03T07:26:07.847 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.848 DEBUG:boto:Data: 2022-09-03T07:26:07.848 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.849 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.850 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.851 DEBUG:boto:Params: {} 2022-09-03T07:26:07.852 DEBUG:boto:Token: None 2022-09-03T07:26:07.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/iiNOc+GAjirlvjd0wcyYWG180Q= 2022-09-03T07:26:07.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/iiNOc+GAjirlvjd0wcyYWG180Q=', 'Content-Length': '0'} 2022-09-03T07:26:07.857 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe32233aa28936ee-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.898 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:07.899 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:07.900 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.900 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.901 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.902 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.903 DEBUG:boto:Data: 2022-09-03T07:26:07.904 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.906 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.907 DEBUG:boto:Params: {} 2022-09-03T07:26:07.908 DEBUG:boto:Token: None 2022-09-03T07:26:07.909 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4MtKRQgJN0JRGfIBA2TLP/ndAS8= 2022-09-03T07:26:07.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4MtKRQgJN0JRGfIBA2TLP/ndAS8=', 'Content-Length': '0'} 2022-09-03T07:26:07.913 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083854e6c3bac41aa-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.913 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:07.914 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:07.915 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.915 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.916 DEBUG:boto:Method: HEAD 2022-09-03T07:26:07.916 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.917 DEBUG:boto:Data: 2022-09-03T07:26:07.917 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.918 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.918 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.919 DEBUG:boto:Params: {} 2022-09-03T07:26:07.920 DEBUG:boto:Token: None 2022-09-03T07:26:07.921 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4MtKRQgJN0JRGfIBA2TLP/ndAS8= 2022-09-03T07:26:07.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4MtKRQgJN0JRGfIBA2TLP/ndAS8=', 'Content-Length': '0'} 2022-09-03T07:26:07.926 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006610003c30c07646-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.927 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:26:07.927 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:07.928 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:07.928 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.929 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.929 DEBUG:boto:Method: GET 2022-09-03T07:26:07.930 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.935 DEBUG:boto:Data: 2022-09-03T07:26:07.935 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.937 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.937 DEBUG:boto:Params: {} 2022-09-03T07:26:07.938 DEBUG:boto:Token: None 2022-09-03T07:26:07.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/iiNOc+GAjirlvjd0wcyYWG180Q= 2022-09-03T07:26:07.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/iiNOc+GAjirlvjd0wcyYWG180Q=', 'Content-Length': '0'} 2022-09-03T07:26:07.942 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026936b3e167e64c9-006313018f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.944 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:07.945 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:07.946 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.947 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.948 DEBUG:boto:Method: GET 2022-09-03T07:26:07.949 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.950 DEBUG:boto:Data: 2022-09-03T07:26:07.951 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.952 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.953 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.954 DEBUG:boto:Params: {} 2022-09-03T07:26:07.955 DEBUG:boto:Token: None 2022-09-03T07:26:07.957 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:26:07.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/iiNOc+GAjirlvjd0wcyYWG180Q= 2022-09-03T07:26:07.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/iiNOc+GAjirlvjd0wcyYWG180Q=', 'Content-Length': '0'} 2022-09-03T07:26:07.961 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b57b67b61bc0e3f3-006313018f-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.963 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.964 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.965 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:07.966 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:07.971 DEBUG:boto:Method: GET 2022-09-03T07:26:07.977 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:26:07.978 DEBUG:boto:Data: 2022-09-03T07:26:07.979 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:07.980 DEBUG:boto:Port: 8001 2022-09-03T07:26:07.981 DEBUG:boto:Params: {} 2022-09-03T07:26:07.981 DEBUG:boto:Token: None 2022-09-03T07:26:07.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:07.986 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ee76130459e09389-006313018f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:07.987 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:07.988 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:07.989 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:07.989 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:07.990 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:07.991 DEBUG:boto:Method: GET 2022-09-03T07:26:07.992 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:26:07.993 DEBUG:boto:Data: 2022-09-03T07:26:07.994 DEBUG:boto:Headers: {} 2022-09-03T07:26:07.995 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:07.995 DEBUG:boto:Port: 8000 2022-09-03T07:26:07.996 DEBUG:boto:Params: {} 2022-09-03T07:26:07.997 DEBUG:boto:Token: None 2022-09-03T07:26:07.998 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:07 GMT /avysmw-75/ 2022-09-03T07:26:07.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y= 2022-09-03T07:26:07.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu1ZJrwmf8AjWF7v+uZeZqiie+Y=', 'Content-Length': '0'} 2022-09-03T07:26:08.001 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000913c3b8e4de58335-006313018f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.002 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.003 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:08.003 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:08.004 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.004 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:26:08.005 DEBUG:boto:Data: 2022-09-03T07:26:08.005 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.006 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.006 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.007 DEBUG:boto:Params: {} 2022-09-03T07:26:08.007 DEBUG:boto:Token: None 2022-09-03T07:26:08.008 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj55 2022-09-03T07:26:08.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2iwoIKPlCT6pLpvLzw8k2ltVtdQ= 2022-09-03T07:26:08.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2iwoIKPlCT6pLpvLzw8k2ltVtdQ=', 'Content-Length': '0'} 2022-09-03T07:26:08.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000404271d913783c3c-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.012 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:08.013 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:08.013 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.014 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:26:08.014 DEBUG:boto:Data: 2022-09-03T07:26:08.015 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.015 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.016 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.016 DEBUG:boto:Params: {} 2022-09-03T07:26:08.017 DEBUG:boto:Token: None 2022-09-03T07:26:08.017 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj55 2022-09-03T07:26:08.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2iwoIKPlCT6pLpvLzw8k2ltVtdQ= 2022-09-03T07:26:08.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2iwoIKPlCT6pLpvLzw8k2ltVtdQ=', 'Content-Length': '0'} 2022-09-03T07:26:08.020 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ffabf334652c0310-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.021 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:26:08.022 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:08.022 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:08.023 DEBUG:boto:Method: GET 2022-09-03T07:26:08.024 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:26:08.024 DEBUG:boto:Data: 2022-09-03T07:26:08.025 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.025 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.026 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.026 DEBUG:boto:Params: {} 2022-09-03T07:26:08.027 DEBUG:boto:Token: None 2022-09-03T07:26:08.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj55 2022-09-03T07:26:08.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XXlplN2GpxwTIHul6hBtnAB/MOs= 2022-09-03T07:26:08.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XXlplN2GpxwTIHul6hBtnAB/MOs=', 'Content-Length': '0'} 2022-09-03T07:26:08.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078a59f7220797298-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.074 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:26:08.075 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:26:08.075 DEBUG:boto:Method: GET 2022-09-03T07:26:08.076 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:26:08.077 DEBUG:boto:Data: 2022-09-03T07:26:08.077 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.080 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.081 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.081 DEBUG:boto:Params: {} 2022-09-03T07:26:08.082 DEBUG:boto:Token: None 2022-09-03T07:26:08.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj55 2022-09-03T07:26:08.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XXlplN2GpxwTIHul6hBtnAB/MOs= 2022-09-03T07:26:08.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XXlplN2GpxwTIHul6hBtnAB/MOs=', 'Content-Length': '0'} 2022-09-03T07:26:08.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000084d19e7c28b3979f-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.088 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.089 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.090 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:08.091 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:08.092 DEBUG:boto:Method: GET 2022-09-03T07:26:08.093 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:26:08.095 DEBUG:boto:Data: 2022-09-03T07:26:08.096 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.097 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.099 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.101 DEBUG:boto:Params: {} 2022-09-03T07:26:08.102 DEBUG:boto:Token: None 2022-09-03T07:26:08.104 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.114 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007e583b37abd09007-0063130190-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.115 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.117 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.121 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.122 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:08.123 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:26:08.124 DEBUG:boto:Method: GET 2022-09-03T07:26:08.125 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:26:08.126 DEBUG:boto:Data: 2022-09-03T07:26:08.127 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.128 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.129 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.129 DEBUG:boto:Params: {} 2022-09-03T07:26:08.131 DEBUG:boto:Token: None 2022-09-03T07:26:08.132 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.133 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.136 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c389b409fcf8263-0063130190-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.136 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.137 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:26:08.138 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.139 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.140 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.141 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.141 DEBUG:boto:Method: GET 2022-09-03T07:26:08.142 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.143 DEBUG:boto:Data: 2022-09-03T07:26:08.144 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.149 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.150 DEBUG:boto:Params: {} 2022-09-03T07:26:08.151 DEBUG:boto:Token: None 2022-09-03T07:26:08.152 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Sz8KoIik73l7Bx75xzeHSRuwS5Q= 2022-09-03T07:26:08.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Sz8KoIik73l7Bx75xzeHSRuwS5Q=', 'Content-Length': '0'} 2022-09-03T07:26:08.157 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000025213bb7bd2ad2ec-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.159 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.160 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.161 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.163 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.164 DEBUG:boto:Method: GET 2022-09-03T07:26:08.165 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.166 DEBUG:boto:Data: 2022-09-03T07:26:08.167 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.168 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.169 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.170 DEBUG:boto:Params: {} 2022-09-03T07:26:08.171 DEBUG:boto:Token: None 2022-09-03T07:26:08.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Sz8KoIik73l7Bx75xzeHSRuwS5Q= 2022-09-03T07:26:08.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Sz8KoIik73l7Bx75xzeHSRuwS5Q=', 'Content-Length': '0'} 2022-09-03T07:26:08.186 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000021fcd902362366bc-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.190 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.191 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.192 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.192 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.193 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.194 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.195 DEBUG:boto:Data: 2022-09-03T07:26:08.197 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.198 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.198 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.199 DEBUG:boto:Params: {} 2022-09-03T07:26:08.200 DEBUG:boto:Token: None 2022-09-03T07:26:08.201 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.201 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqGuA47Cadfb9oIcWQewwbRVO8U= 2022-09-03T07:26:08.202 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqGuA47Cadfb9oIcWQewwbRVO8U=', 'Content-Length': '0'} 2022-09-03T07:26:08.205 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079e7211a82c1e671-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.206 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.207 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.208 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.209 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.209 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.210 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.211 DEBUG:boto:Data: 2022-09-03T07:26:08.211 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.212 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.212 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.213 DEBUG:boto:Params: {} 2022-09-03T07:26:08.214 DEBUG:boto:Token: None 2022-09-03T07:26:08.215 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pqGuA47Cadfb9oIcWQewwbRVO8U= 2022-09-03T07:26:08.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pqGuA47Cadfb9oIcWQewwbRVO8U=', 'Content-Length': '0'} 2022-09-03T07:26:08.220 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8581af69f479ecd-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.222 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:26:08.222 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.223 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.223 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.224 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.224 DEBUG:boto:Method: GET 2022-09-03T07:26:08.225 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.226 DEBUG:boto:Data: 2022-09-03T07:26:08.226 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.227 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.228 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.228 DEBUG:boto:Params: {} 2022-09-03T07:26:08.229 DEBUG:boto:Token: None 2022-09-03T07:26:08.230 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Sz8KoIik73l7Bx75xzeHSRuwS5Q= 2022-09-03T07:26:08.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Sz8KoIik73l7Bx75xzeHSRuwS5Q=', 'Content-Length': '0'} 2022-09-03T07:26:08.234 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a21d1e33c302dd8-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.235 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.236 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.236 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.237 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.237 DEBUG:boto:Method: GET 2022-09-03T07:26:08.238 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.238 DEBUG:boto:Data: 2022-09-03T07:26:08.239 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.239 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.240 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.241 DEBUG:boto:Params: {} 2022-09-03T07:26:08.241 DEBUG:boto:Token: None 2022-09-03T07:26:08.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:26:08.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Sz8KoIik73l7Bx75xzeHSRuwS5Q= 2022-09-03T07:26:08.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Sz8KoIik73l7Bx75xzeHSRuwS5Q=', 'Content-Length': '0'} 2022-09-03T07:26:08.246 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ffc9cc90340d135-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.286 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.287 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.288 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:08.290 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:08.291 DEBUG:boto:Method: GET 2022-09-03T07:26:08.292 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:26:08.293 DEBUG:boto:Data: 2022-09-03T07:26:08.295 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.296 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.296 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.297 DEBUG:boto:Params: {} 2022-09-03T07:26:08.298 DEBUG:boto:Token: None 2022-09-03T07:26:08.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.303 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001863247dc8556116-0063130190-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.304 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.305 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.306 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.306 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:08.308 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:08.309 DEBUG:boto:Method: GET 2022-09-03T07:26:08.309 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:26:08.310 DEBUG:boto:Data: 2022-09-03T07:26:08.311 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.312 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.312 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.313 DEBUG:boto:Params: {} 2022-09-03T07:26:08.313 DEBUG:boto:Token: None 2022-09-03T07:26:08.314 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.315 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.325 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a147469417d6876-0063130190-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.327 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.328 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.329 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.329 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.330 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:26:08.330 DEBUG:boto:Data: 2022-09-03T07:26:08.331 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.333 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.333 DEBUG:boto:Params: {} 2022-09-03T07:26:08.334 DEBUG:boto:Token: None 2022-09-03T07:26:08.334 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57 2022-09-03T07:26:08.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gDWLw2yfGeO5A0OgmDuZv0RPtC8= 2022-09-03T07:26:08.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gDWLw2yfGeO5A0OgmDuZv0RPtC8=', 'Content-Length': '0'} 2022-09-03T07:26:08.338 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000510a17c017137738-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.339 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.339 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.340 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.340 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:26:08.341 DEBUG:boto:Data: 2022-09-03T07:26:08.341 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.342 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.342 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.343 DEBUG:boto:Params: {} 2022-09-03T07:26:08.344 DEBUG:boto:Token: None 2022-09-03T07:26:08.344 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57 2022-09-03T07:26:08.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gDWLw2yfGeO5A0OgmDuZv0RPtC8= 2022-09-03T07:26:08.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gDWLw2yfGeO5A0OgmDuZv0RPtC8=', 'Content-Length': '0'} 2022-09-03T07:26:08.348 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e13e1b55e418ca7-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.349 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:26:08.349 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.350 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.350 DEBUG:boto:Method: GET 2022-09-03T07:26:08.351 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:26:08.351 DEBUG:boto:Data: 2022-09-03T07:26:08.352 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.353 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.353 DEBUG:boto:Params: {} 2022-09-03T07:26:08.354 DEBUG:boto:Token: None 2022-09-03T07:26:08.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57 2022-09-03T07:26:08.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DZne8gfVF/cgpwd/WZAEKMNuF68= 2022-09-03T07:26:08.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DZne8gfVF/cgpwd/WZAEKMNuF68=', 'Content-Length': '0'} 2022-09-03T07:26:08.358 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dfd9dac7aff10bea-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.359 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:26:08.360 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:26:08.361 DEBUG:boto:Method: GET 2022-09-03T07:26:08.361 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:26:08.362 DEBUG:boto:Data: 2022-09-03T07:26:08.362 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.363 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.365 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.366 DEBUG:boto:Params: {} 2022-09-03T07:26:08.366 DEBUG:boto:Token: None 2022-09-03T07:26:08.368 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj57 2022-09-03T07:26:08.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DZne8gfVF/cgpwd/WZAEKMNuF68= 2022-09-03T07:26:08.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DZne8gfVF/cgpwd/WZAEKMNuF68=', 'Content-Length': '0'} 2022-09-03T07:26:08.373 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037cb024d82dd0f6d-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.375 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.376 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.377 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:08.378 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:08.379 DEBUG:boto:Method: GET 2022-09-03T07:26:08.379 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:26:08.380 DEBUG:boto:Data: 2022-09-03T07:26:08.381 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.382 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.382 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.383 DEBUG:boto:Params: {} 2022-09-03T07:26:08.384 DEBUG:boto:Token: None 2022-09-03T07:26:08.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.392 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005a09f3f29eb74032-0063130190-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.434 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.436 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.437 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.439 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:08.439 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:26:08.440 DEBUG:boto:Method: GET 2022-09-03T07:26:08.440 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:26:08.441 DEBUG:boto:Data: 2022-09-03T07:26:08.442 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.443 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.444 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.445 DEBUG:boto:Params: {} 2022-09-03T07:26:08.446 DEBUG:boto:Token: None 2022-09-03T07:26:08.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.453 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eda1e3df886b02fd-0063130190-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.454 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.455 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:26:08.456 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.457 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.457 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.458 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.458 DEBUG:boto:Method: GET 2022-09-03T07:26:08.459 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.460 DEBUG:boto:Data: 2022-09-03T07:26:08.460 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.461 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.462 DEBUG:boto:Params: {} 2022-09-03T07:26:08.463 DEBUG:boto:Token: None 2022-09-03T07:26:08.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFdhZ143RayXNN44JMx8t0D6D7s= 2022-09-03T07:26:08.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFdhZ143RayXNN44JMx8t0D6D7s=', 'Content-Length': '0'} 2022-09-03T07:26:08.469 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9ac588fdd261544-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.469 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.470 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.471 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.471 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.472 DEBUG:boto:Method: GET 2022-09-03T07:26:08.472 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.473 DEBUG:boto:Data: 2022-09-03T07:26:08.473 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.474 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.475 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.475 DEBUG:boto:Params: {} 2022-09-03T07:26:08.476 DEBUG:boto:Token: None 2022-09-03T07:26:08.476 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFdhZ143RayXNN44JMx8t0D6D7s= 2022-09-03T07:26:08.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFdhZ143RayXNN44JMx8t0D6D7s=', 'Content-Length': '0'} 2022-09-03T07:26:08.480 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbfd2d73e5755424-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.481 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.482 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.483 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.483 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.484 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.484 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.485 DEBUG:boto:Data: 2022-09-03T07:26:08.486 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.488 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.489 DEBUG:boto:Params: {} 2022-09-03T07:26:08.489 DEBUG:boto:Token: None 2022-09-03T07:26:08.490 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:evRE9gez8TMsX8qgWZ0c56NV9Mk= 2022-09-03T07:26:08.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:evRE9gez8TMsX8qgWZ0c56NV9Mk=', 'Content-Length': '0'} 2022-09-03T07:26:08.493 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000006dbb52e693973a0-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.494 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.495 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.495 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.496 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.496 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.497 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.497 DEBUG:boto:Data: 2022-09-03T07:26:08.498 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.498 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.499 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.499 DEBUG:boto:Params: {} 2022-09-03T07:26:08.500 DEBUG:boto:Token: None 2022-09-03T07:26:08.501 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:evRE9gez8TMsX8qgWZ0c56NV9Mk= 2022-09-03T07:26:08.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:evRE9gez8TMsX8qgWZ0c56NV9Mk=', 'Content-Length': '0'} 2022-09-03T07:26:08.509 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008dd15a611fc6200c-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.511 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:26:08.512 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.513 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.514 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.515 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.516 DEBUG:boto:Method: GET 2022-09-03T07:26:08.517 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.518 DEBUG:boto:Data: 2022-09-03T07:26:08.519 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.521 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.522 DEBUG:boto:Params: {} 2022-09-03T07:26:08.523 DEBUG:boto:Token: None 2022-09-03T07:26:08.524 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.525 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFdhZ143RayXNN44JMx8t0D6D7s= 2022-09-03T07:26:08.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFdhZ143RayXNN44JMx8t0D6D7s=', 'Content-Length': '0'} 2022-09-03T07:26:08.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de273ffe4d1b2f6d-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.534 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.535 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.535 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.536 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.536 DEBUG:boto:Method: GET 2022-09-03T07:26:08.537 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.537 DEBUG:boto:Data: 2022-09-03T07:26:08.539 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.539 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.540 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.541 DEBUG:boto:Params: {} 2022-09-03T07:26:08.541 DEBUG:boto:Token: None 2022-09-03T07:26:08.542 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:26:08.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFdhZ143RayXNN44JMx8t0D6D7s= 2022-09-03T07:26:08.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFdhZ143RayXNN44JMx8t0D6D7s=', 'Content-Length': '0'} 2022-09-03T07:26:08.550 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8d297f150a39d24-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.590 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.592 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.592 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:08.593 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:08.594 DEBUG:boto:Method: GET 2022-09-03T07:26:08.594 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:26:08.595 DEBUG:boto:Data: 2022-09-03T07:26:08.595 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.596 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.597 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.597 DEBUG:boto:Params: {} 2022-09-03T07:26:08.598 DEBUG:boto:Token: None 2022-09-03T07:26:08.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.605 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eac57b6deb42babf-0063130190-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.605 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.613 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.614 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.615 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:08.615 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:08.616 DEBUG:boto:Method: GET 2022-09-03T07:26:08.616 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:26:08.617 DEBUG:boto:Data: 2022-09-03T07:26:08.617 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.618 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.619 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.619 DEBUG:boto:Params: {} 2022-09-03T07:26:08.620 DEBUG:boto:Token: None 2022-09-03T07:26:08.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.625 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007cde19c1bf638067-0063130190-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.626 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.627 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.627 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.628 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.628 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:26:08.629 DEBUG:boto:Data: 2022-09-03T07:26:08.630 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.631 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.632 DEBUG:boto:Params: {} 2022-09-03T07:26:08.633 DEBUG:boto:Token: None 2022-09-03T07:26:08.633 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58 2022-09-03T07:26:08.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5sOglNLqyeI+wALeeQiIW8PBvFA= 2022-09-03T07:26:08.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5sOglNLqyeI+wALeeQiIW8PBvFA=', 'Content-Length': '0'} 2022-09-03T07:26:08.638 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d02139772a05e75f-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.639 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.641 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.642 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.644 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:26:08.645 DEBUG:boto:Data: 2022-09-03T07:26:08.646 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.647 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.648 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.649 DEBUG:boto:Params: {} 2022-09-03T07:26:08.650 DEBUG:boto:Token: None 2022-09-03T07:26:08.651 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58 2022-09-03T07:26:08.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5sOglNLqyeI+wALeeQiIW8PBvFA= 2022-09-03T07:26:08.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5sOglNLqyeI+wALeeQiIW8PBvFA=', 'Content-Length': '0'} 2022-09-03T07:26:08.655 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000028ed794c59b7a13e-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.656 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:26:08.656 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.657 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.658 DEBUG:boto:Method: GET 2022-09-03T07:26:08.666 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:26:08.666 DEBUG:boto:Data: 2022-09-03T07:26:08.667 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.668 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.668 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.669 DEBUG:boto:Params: {} 2022-09-03T07:26:08.669 DEBUG:boto:Token: None 2022-09-03T07:26:08.670 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58 2022-09-03T07:26:08.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DLeF4Ei6pnZcjP2C5lLsXMfGVFI= 2022-09-03T07:26:08.672 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DLeF4Ei6pnZcjP2C5lLsXMfGVFI=', 'Content-Length': '0'} 2022-09-03T07:26:08.674 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000867c17145fb03d44-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.675 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:26:08.675 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:26:08.676 DEBUG:boto:Method: GET 2022-09-03T07:26:08.677 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:26:08.677 DEBUG:boto:Data: 2022-09-03T07:26:08.678 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.678 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.679 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.680 DEBUG:boto:Params: {} 2022-09-03T07:26:08.680 DEBUG:boto:Token: None 2022-09-03T07:26:08.681 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj58 2022-09-03T07:26:08.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DLeF4Ei6pnZcjP2C5lLsXMfGVFI= 2022-09-03T07:26:08.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DLeF4Ei6pnZcjP2C5lLsXMfGVFI=', 'Content-Length': '0'} 2022-09-03T07:26:08.685 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046d0a410bc452d80-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.691 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.692 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.692 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:08.694 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:08.694 DEBUG:boto:Method: GET 2022-09-03T07:26:08.695 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:26:08.695 DEBUG:boto:Data: 2022-09-03T07:26:08.696 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.696 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.697 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.699 DEBUG:boto:Params: {} 2022-09-03T07:26:08.700 DEBUG:boto:Token: None 2022-09-03T07:26:08.700 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.701 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.702 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.706 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cacadcaa6d926949-0063130190-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.707 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.708 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.710 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.710 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:08.711 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:26:08.712 DEBUG:boto:Method: GET 2022-09-03T07:26:08.713 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:26:08.714 DEBUG:boto:Data: 2022-09-03T07:26:08.714 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.715 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.715 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.716 DEBUG:boto:Params: {} 2022-09-03T07:26:08.728 DEBUG:boto:Token: None 2022-09-03T07:26:08.729 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.736 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004ce4c6884bba50c7-0063130190-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.737 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.739 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:26:08.739 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.740 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.741 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.741 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.742 DEBUG:boto:Method: GET 2022-09-03T07:26:08.742 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.743 DEBUG:boto:Data: 2022-09-03T07:26:08.744 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.746 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.747 DEBUG:boto:Params: {} 2022-09-03T07:26:08.747 DEBUG:boto:Token: None 2022-09-03T07:26:08.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kf7F14wiEk1DPaE8A0avKyBp7EE= 2022-09-03T07:26:08.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kf7F14wiEk1DPaE8A0avKyBp7EE=', 'Content-Length': '0'} 2022-09-03T07:26:08.761 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000187af2087d64aab7-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.763 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.764 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.765 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.766 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.767 DEBUG:boto:Method: GET 2022-09-03T07:26:08.768 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.769 DEBUG:boto:Data: 2022-09-03T07:26:08.770 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.771 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.772 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.773 DEBUG:boto:Params: {} 2022-09-03T07:26:08.774 DEBUG:boto:Token: None 2022-09-03T07:26:08.775 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kf7F14wiEk1DPaE8A0avKyBp7EE= 2022-09-03T07:26:08.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kf7F14wiEk1DPaE8A0avKyBp7EE=', 'Content-Length': '0'} 2022-09-03T07:26:08.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000754468afe75a1367-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.783 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.784 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.785 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.786 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.787 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.788 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.788 DEBUG:boto:Data: 2022-09-03T07:26:08.789 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.790 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.790 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.791 DEBUG:boto:Params: {} 2022-09-03T07:26:08.791 DEBUG:boto:Token: None 2022-09-03T07:26:08.792 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BJ3+ca1cOOm28dhmuE0Kfg8phmw= 2022-09-03T07:26:08.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BJ3+ca1cOOm28dhmuE0Kfg8phmw=', 'Content-Length': '0'} 2022-09-03T07:26:08.795 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000041f163b995d2593-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.796 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.797 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.797 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.798 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.798 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.799 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.799 DEBUG:boto:Data: 2022-09-03T07:26:08.800 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.800 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.801 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.801 DEBUG:boto:Params: {} 2022-09-03T07:26:08.802 DEBUG:boto:Token: None 2022-09-03T07:26:08.803 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BJ3+ca1cOOm28dhmuE0Kfg8phmw= 2022-09-03T07:26:08.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BJ3+ca1cOOm28dhmuE0Kfg8phmw=', 'Content-Length': '0'} 2022-09-03T07:26:08.808 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b3da28e0a0056b2-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.809 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:26:08.810 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.810 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.811 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.811 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.812 DEBUG:boto:Method: GET 2022-09-03T07:26:08.813 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.814 DEBUG:boto:Data: 2022-09-03T07:26:08.814 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.816 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.817 DEBUG:boto:Params: {} 2022-09-03T07:26:08.818 DEBUG:boto:Token: None 2022-09-03T07:26:08.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kf7F14wiEk1DPaE8A0avKyBp7EE= 2022-09-03T07:26:08.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kf7F14wiEk1DPaE8A0avKyBp7EE=', 'Content-Length': '0'} 2022-09-03T07:26:08.825 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000082825178695e4618-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.827 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.828 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.828 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.829 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.830 DEBUG:boto:Method: GET 2022-09-03T07:26:08.830 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.831 DEBUG:boto:Data: 2022-09-03T07:26:08.831 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.833 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.833 DEBUG:boto:Params: {} 2022-09-03T07:26:08.834 DEBUG:boto:Token: None 2022-09-03T07:26:08.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:26:08.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kf7F14wiEk1DPaE8A0avKyBp7EE= 2022-09-03T07:26:08.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kf7F14wiEk1DPaE8A0avKyBp7EE=', 'Content-Length': '0'} 2022-09-03T07:26:08.838 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000030bbc068a9bbccfd-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.886 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.887 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.888 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:08.889 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:08.890 DEBUG:boto:Method: GET 2022-09-03T07:26:08.891 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:26:08.891 DEBUG:boto:Data: 2022-09-03T07:26:08.892 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.893 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.894 DEBUG:boto:Params: {} 2022-09-03T07:26:08.895 DEBUG:boto:Token: None 2022-09-03T07:26:08.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.898 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.906 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075a89cc24af698a7-0063130190-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.907 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.910 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.911 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.912 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:08.913 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:08.914 DEBUG:boto:Method: GET 2022-09-03T07:26:08.915 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:26:08.917 DEBUG:boto:Data: 2022-09-03T07:26:08.918 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.919 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.921 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.922 DEBUG:boto:Params: {} 2022-09-03T07:26:08.923 DEBUG:boto:Token: None 2022-09-03T07:26:08.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.930 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da0b2405baaaebe1-0063130190-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.931 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:08.933 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.934 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.935 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.936 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:26:08.936 DEBUG:boto:Data: 2022-09-03T07:26:08.938 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.938 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.939 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.939 DEBUG:boto:Params: {} 2022-09-03T07:26:08.940 DEBUG:boto:Token: None 2022-09-03T07:26:08.940 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68 2022-09-03T07:26:08.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nlUEHINN1Wbqo7Tne0/Z1VE6Srw= 2022-09-03T07:26:08.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nlUEHINN1Wbqo7Tne0/Z1VE6Srw=', 'Content-Length': '0'} 2022-09-03T07:26:08.944 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008406d150a71705bc-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.945 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.946 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.947 DEBUG:boto:Method: HEAD 2022-09-03T07:26:08.947 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:26:08.948 DEBUG:boto:Data: 2022-09-03T07:26:08.949 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.949 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.950 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.951 DEBUG:boto:Params: {} 2022-09-03T07:26:08.952 DEBUG:boto:Token: None 2022-09-03T07:26:08.953 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68 2022-09-03T07:26:08.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nlUEHINN1Wbqo7Tne0/Z1VE6Srw= 2022-09-03T07:26:08.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nlUEHINN1Wbqo7Tne0/Z1VE6Srw=', 'Content-Length': '0'} 2022-09-03T07:26:08.958 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b41e45f18e93fb7-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.959 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:26:08.960 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.961 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.962 DEBUG:boto:Method: GET 2022-09-03T07:26:08.963 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:26:08.964 DEBUG:boto:Data: 2022-09-03T07:26:08.965 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.966 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.967 DEBUG:boto:Params: {} 2022-09-03T07:26:08.967 DEBUG:boto:Token: None 2022-09-03T07:26:08.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68 2022-09-03T07:26:08.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FhWVNEtCKrUh6Zg1yvb2ATYz8Oc= 2022-09-03T07:26:08.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FhWVNEtCKrUh6Zg1yvb2ATYz8Oc=', 'Content-Length': '0'} 2022-09-03T07:26:08.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e3a23f3bd360a40-0063130190-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.973 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:26:08.973 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:26:08.974 DEBUG:boto:Method: GET 2022-09-03T07:26:08.975 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:26:08.976 DEBUG:boto:Data: 2022-09-03T07:26:08.977 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.977 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:08.978 DEBUG:boto:Port: 8000 2022-09-03T07:26:08.978 DEBUG:boto:Params: {} 2022-09-03T07:26:08.979 DEBUG:boto:Token: None 2022-09-03T07:26:08.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/obj68 2022-09-03T07:26:08.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FhWVNEtCKrUh6Zg1yvb2ATYz8Oc= 2022-09-03T07:26:08.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FhWVNEtCKrUh6Zg1yvb2ATYz8Oc=', 'Content-Length': '0'} 2022-09-03T07:26:08.985 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa23be0bf897cf2d-0063130190-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:08.986 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:08.986 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:08.987 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:08.988 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:08.988 DEBUG:boto:Method: GET 2022-09-03T07:26:08.989 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:26:08.989 DEBUG:boto:Data: 2022-09-03T07:26:08.990 DEBUG:boto:Headers: {} 2022-09-03T07:26:08.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:08.991 DEBUG:boto:Port: 8001 2022-09-03T07:26:08.992 DEBUG:boto:Params: {} 2022-09-03T07:26:08.993 DEBUG:boto:Token: None 2022-09-03T07:26:08.995 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:08 GMT /avysmw-75/ 2022-09-03T07:26:08.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM= 2022-09-03T07:26:08.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdy87E4eQ8rgo31NlOhdIjc0tNM=', 'Content-Length': '0'} 2022-09-03T07:26:08.999 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c42db590193aa04-0063130190-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.000 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.001 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.001 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.002 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:09.003 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:26:09.004 DEBUG:boto:Method: GET 2022-09-03T07:26:09.005 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:26:09.005 DEBUG:boto:Data: 2022-09-03T07:26:09.006 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.007 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.008 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.008 DEBUG:boto:Params: {} 2022-09-03T07:26:09.009 DEBUG:boto:Token: None 2022-09-03T07:26:09.010 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.016 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c938eaeecaaf5354-0063130191-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.018 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.019 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:26:09.020 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.020 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.021 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.021 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.022 DEBUG:boto:Method: GET 2022-09-03T07:26:09.023 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.024 DEBUG:boto:Data: 2022-09-03T07:26:09.024 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.025 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.025 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.026 DEBUG:boto:Params: {} 2022-09-03T07:26:09.027 DEBUG:boto:Token: None 2022-09-03T07:26:09.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZhCuwdJUF2+n5wkU9QuRAPMb5lI= 2022-09-03T07:26:09.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZhCuwdJUF2+n5wkU9QuRAPMb5lI=', 'Content-Length': '0'} 2022-09-03T07:26:09.032 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086deeddd902b7069-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.074 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.076 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.077 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.078 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.079 DEBUG:boto:Method: GET 2022-09-03T07:26:09.080 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.081 DEBUG:boto:Data: 2022-09-03T07:26:09.081 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.082 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.084 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.085 DEBUG:boto:Params: {} 2022-09-03T07:26:09.085 DEBUG:boto:Token: None 2022-09-03T07:26:09.086 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.087 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZhCuwdJUF2+n5wkU9QuRAPMb5lI= 2022-09-03T07:26:09.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZhCuwdJUF2+n5wkU9QuRAPMb5lI=', 'Content-Length': '0'} 2022-09-03T07:26:09.090 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd7847867161e3a0-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.091 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.091 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.092 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.092 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.093 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.094 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.094 DEBUG:boto:Data: 2022-09-03T07:26:09.096 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.097 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.099 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.101 DEBUG:boto:Params: {} 2022-09-03T07:26:09.102 DEBUG:boto:Token: None 2022-09-03T07:26:09.103 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQhP/lC/PZdyN3Pinw9u8yALvQA= 2022-09-03T07:26:09.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQhP/lC/PZdyN3Pinw9u8yALvQA=', 'Content-Length': '0'} 2022-09-03T07:26:09.108 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b54b755604e6ef5-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.109 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.110 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.110 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.111 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.111 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.112 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.112 DEBUG:boto:Data: 2022-09-03T07:26:09.113 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.113 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.114 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.114 DEBUG:boto:Params: {} 2022-09-03T07:26:09.115 DEBUG:boto:Token: None 2022-09-03T07:26:09.116 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQhP/lC/PZdyN3Pinw9u8yALvQA= 2022-09-03T07:26:09.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQhP/lC/PZdyN3Pinw9u8yALvQA=', 'Content-Length': '0'} 2022-09-03T07:26:09.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba0f761631e8720b-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.120 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:26:09.121 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.121 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.122 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.122 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.123 DEBUG:boto:Method: GET 2022-09-03T07:26:09.123 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.124 DEBUG:boto:Data: 2022-09-03T07:26:09.125 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.125 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.126 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.126 DEBUG:boto:Params: {} 2022-09-03T07:26:09.127 DEBUG:boto:Token: None 2022-09-03T07:26:09.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZhCuwdJUF2+n5wkU9QuRAPMb5lI= 2022-09-03T07:26:09.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZhCuwdJUF2+n5wkU9QuRAPMb5lI=', 'Content-Length': '0'} 2022-09-03T07:26:09.131 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b3f86620222b494-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.132 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.132 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.133 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.133 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.134 DEBUG:boto:Method: GET 2022-09-03T07:26:09.134 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.135 DEBUG:boto:Data: 2022-09-03T07:26:09.135 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.136 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.137 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.137 DEBUG:boto:Params: {} 2022-09-03T07:26:09.138 DEBUG:boto:Token: None 2022-09-03T07:26:09.138 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:26:09.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZhCuwdJUF2+n5wkU9QuRAPMb5lI= 2022-09-03T07:26:09.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZhCuwdJUF2+n5wkU9QuRAPMb5lI=', 'Content-Length': '0'} 2022-09-03T07:26:09.142 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec08df7044eaaa9e-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.142 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.143 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.144 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:09.144 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:09.145 DEBUG:boto:Method: GET 2022-09-03T07:26:09.145 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:26:09.146 DEBUG:boto:Data: 2022-09-03T07:26:09.146 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.147 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.148 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.148 DEBUG:boto:Params: {} 2022-09-03T07:26:09.149 DEBUG:boto:Token: None 2022-09-03T07:26:09.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.154 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a4238abaa2463ef6-0063130191-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.155 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.156 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.156 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.157 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:09.157 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:09.158 DEBUG:boto:Method: GET 2022-09-03T07:26:09.159 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:26:09.159 DEBUG:boto:Data: 2022-09-03T07:26:09.160 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.161 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.161 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.162 DEBUG:boto:Params: {} 2022-09-03T07:26:09.163 DEBUG:boto:Token: None 2022-09-03T07:26:09.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.167 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000528d7e19c46f1cca-0063130191-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.168 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.169 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.169 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.170 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.170 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:26:09.171 DEBUG:boto:Data: 2022-09-03T07:26:09.171 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.173 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.173 DEBUG:boto:Params: {} 2022-09-03T07:26:09.174 DEBUG:boto:Token: None 2022-09-03T07:26:09.174 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69 2022-09-03T07:26:09.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ue9i1LKcMkZinDG00XnjWESBSbM= 2022-09-03T07:26:09.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ue9i1LKcMkZinDG00XnjWESBSbM=', 'Content-Length': '0'} 2022-09-03T07:26:09.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fccd7128b44e864c-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.178 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.179 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.179 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.180 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:26:09.180 DEBUG:boto:Data: 2022-09-03T07:26:09.181 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.182 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.182 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.183 DEBUG:boto:Params: {} 2022-09-03T07:26:09.183 DEBUG:boto:Token: None 2022-09-03T07:26:09.184 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69 2022-09-03T07:26:09.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ue9i1LKcMkZinDG00XnjWESBSbM= 2022-09-03T07:26:09.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ue9i1LKcMkZinDG00XnjWESBSbM=', 'Content-Length': '0'} 2022-09-03T07:26:09.187 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c34c29096d06d069-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.188 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:26:09.189 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.189 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.190 DEBUG:boto:Method: GET 2022-09-03T07:26:09.191 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:26:09.191 DEBUG:boto:Data: 2022-09-03T07:26:09.192 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.193 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.194 DEBUG:boto:Params: {} 2022-09-03T07:26:09.194 DEBUG:boto:Token: None 2022-09-03T07:26:09.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69 2022-09-03T07:26:09.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yTri1IaATeNY6xrqRuxGk1bM5qM= 2022-09-03T07:26:09.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yTri1IaATeNY6xrqRuxGk1bM5qM=', 'Content-Length': '0'} 2022-09-03T07:26:09.200 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f105cd87a3f5a3f-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.246 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:26:09.247 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:26:09.248 DEBUG:boto:Method: GET 2022-09-03T07:26:09.248 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:26:09.249 DEBUG:boto:Data: 2022-09-03T07:26:09.250 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.250 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.251 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.251 DEBUG:boto:Params: {} 2022-09-03T07:26:09.252 DEBUG:boto:Token: None 2022-09-03T07:26:09.253 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj69 2022-09-03T07:26:09.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yTri1IaATeNY6xrqRuxGk1bM5qM= 2022-09-03T07:26:09.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yTri1IaATeNY6xrqRuxGk1bM5qM=', 'Content-Length': '0'} 2022-09-03T07:26:09.257 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa35839a15d2944b-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.258 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.259 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.259 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:09.260 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:09.260 DEBUG:boto:Method: GET 2022-09-03T07:26:09.261 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:26:09.262 DEBUG:boto:Data: 2022-09-03T07:26:09.262 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.263 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.264 DEBUG:boto:Params: {} 2022-09-03T07:26:09.265 DEBUG:boto:Token: None 2022-09-03T07:26:09.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.270 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006903d2dcd0037d32-0063130191-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.270 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.271 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.272 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.272 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:09.273 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:26:09.274 DEBUG:boto:Method: GET 2022-09-03T07:26:09.274 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:26:09.275 DEBUG:boto:Data: 2022-09-03T07:26:09.276 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.277 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.277 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.278 DEBUG:boto:Params: {} 2022-09-03T07:26:09.279 DEBUG:boto:Token: None 2022-09-03T07:26:09.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.284 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bfed18bd4e34f155-0063130191-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.285 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.287 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:26:09.287 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.288 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.289 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.290 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.291 DEBUG:boto:Method: GET 2022-09-03T07:26:09.292 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.292 DEBUG:boto:Data: 2022-09-03T07:26:09.293 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.294 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.294 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.295 DEBUG:boto:Params: {} 2022-09-03T07:26:09.296 DEBUG:boto:Token: None 2022-09-03T07:26:09.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:knhH1ArHG2pspd83lq7xMo5zlWM= 2022-09-03T07:26:09.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:knhH1ArHG2pspd83lq7xMo5zlWM=', 'Content-Length': '0'} 2022-09-03T07:26:09.300 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b30da0d5bb44df1-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.300 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.301 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.302 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.302 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.303 DEBUG:boto:Method: GET 2022-09-03T07:26:09.303 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.304 DEBUG:boto:Data: 2022-09-03T07:26:09.304 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.305 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.305 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.306 DEBUG:boto:Params: {} 2022-09-03T07:26:09.307 DEBUG:boto:Token: None 2022-09-03T07:26:09.307 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.308 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:knhH1ArHG2pspd83lq7xMo5zlWM= 2022-09-03T07:26:09.308 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:knhH1ArHG2pspd83lq7xMo5zlWM=', 'Content-Length': '0'} 2022-09-03T07:26:09.311 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009aa63090bee6a716-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.312 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.312 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.313 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.313 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.314 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.314 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.315 DEBUG:boto:Data: 2022-09-03T07:26:09.315 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.316 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.316 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.317 DEBUG:boto:Params: {} 2022-09-03T07:26:09.318 DEBUG:boto:Token: None 2022-09-03T07:26:09.318 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PxhGNNOe7xLHyO9oR7UErcUJJVQ= 2022-09-03T07:26:09.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PxhGNNOe7xLHyO9oR7UErcUJJVQ=', 'Content-Length': '0'} 2022-09-03T07:26:09.321 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7d26fbc52048bbb-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.322 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.323 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.323 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.324 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.324 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.325 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.325 DEBUG:boto:Data: 2022-09-03T07:26:09.326 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.327 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.327 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.328 DEBUG:boto:Params: {} 2022-09-03T07:26:09.328 DEBUG:boto:Token: None 2022-09-03T07:26:09.329 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PxhGNNOe7xLHyO9oR7UErcUJJVQ= 2022-09-03T07:26:09.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PxhGNNOe7xLHyO9oR7UErcUJJVQ=', 'Content-Length': '0'} 2022-09-03T07:26:09.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009808b7896872d1bc-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.333 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:26:09.333 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.334 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.334 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.335 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.336 DEBUG:boto:Method: GET 2022-09-03T07:26:09.336 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.337 DEBUG:boto:Data: 2022-09-03T07:26:09.337 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.338 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.339 DEBUG:boto:Params: {} 2022-09-03T07:26:09.340 DEBUG:boto:Token: None 2022-09-03T07:26:09.340 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:knhH1ArHG2pspd83lq7xMo5zlWM= 2022-09-03T07:26:09.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:knhH1ArHG2pspd83lq7xMo5zlWM=', 'Content-Length': '0'} 2022-09-03T07:26:09.344 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f4814758d4fc4da-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.345 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.345 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.346 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.346 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.347 DEBUG:boto:Method: GET 2022-09-03T07:26:09.347 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.348 DEBUG:boto:Data: 2022-09-03T07:26:09.348 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.349 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.350 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.350 DEBUG:boto:Params: {} 2022-09-03T07:26:09.351 DEBUG:boto:Token: None 2022-09-03T07:26:09.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:26:09.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:knhH1ArHG2pspd83lq7xMo5zlWM= 2022-09-03T07:26:09.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:knhH1ArHG2pspd83lq7xMo5zlWM=', 'Content-Length': '0'} 2022-09-03T07:26:09.355 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c35c8b8ed42d9b91-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.356 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.356 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.357 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:09.357 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:09.358 DEBUG:boto:Method: GET 2022-09-03T07:26:09.358 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:26:09.359 DEBUG:boto:Data: 2022-09-03T07:26:09.359 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.360 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.361 DEBUG:boto:Params: {} 2022-09-03T07:26:09.362 DEBUG:boto:Token: None 2022-09-03T07:26:09.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.366 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000033ff098a099b1885-0063130191-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.367 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.368 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.369 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.369 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:09.370 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:09.382 DEBUG:boto:Method: GET 2022-09-03T07:26:09.383 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:26:09.384 DEBUG:boto:Data: 2022-09-03T07:26:09.385 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.386 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.387 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.388 DEBUG:boto:Params: {} 2022-09-03T07:26:09.389 DEBUG:boto:Token: None 2022-09-03T07:26:09.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.395 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ebdf436dee0e8be0-0063130191-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.397 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.398 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.399 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.400 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.401 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:26:09.402 DEBUG:boto:Data: 2022-09-03T07:26:09.403 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.404 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.405 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.406 DEBUG:boto:Params: {} 2022-09-03T07:26:09.407 DEBUG:boto:Token: None 2022-09-03T07:26:09.408 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76 2022-09-03T07:26:09.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Imtgimw/m2YAmHmmtpDrmoGdwwo= 2022-09-03T07:26:09.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Imtgimw/m2YAmHmmtpDrmoGdwwo=', 'Content-Length': '0'} 2022-09-03T07:26:09.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ac2ef989cf60155-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.412 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.413 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.414 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.414 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:26:09.415 DEBUG:boto:Data: 2022-09-03T07:26:09.415 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.416 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.416 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.417 DEBUG:boto:Params: {} 2022-09-03T07:26:09.418 DEBUG:boto:Token: None 2022-09-03T07:26:09.418 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76 2022-09-03T07:26:09.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Imtgimw/m2YAmHmmtpDrmoGdwwo= 2022-09-03T07:26:09.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Imtgimw/m2YAmHmmtpDrmoGdwwo=', 'Content-Length': '0'} 2022-09-03T07:26:09.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f79acb89c8ea4401-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.423 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:26:09.423 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.424 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.425 DEBUG:boto:Method: GET 2022-09-03T07:26:09.425 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:26:09.426 DEBUG:boto:Data: 2022-09-03T07:26:09.426 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.427 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.428 DEBUG:boto:Params: {} 2022-09-03T07:26:09.429 DEBUG:boto:Token: None 2022-09-03T07:26:09.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76 2022-09-03T07:26:09.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+pHJUy/8vzywjJ6fyPCRGoJADxs= 2022-09-03T07:26:09.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+pHJUy/8vzywjJ6fyPCRGoJADxs=', 'Content-Length': '0'} 2022-09-03T07:26:09.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000059532f2763318f9-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.435 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:26:09.436 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:26:09.437 DEBUG:boto:Method: GET 2022-09-03T07:26:09.438 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:26:09.438 DEBUG:boto:Data: 2022-09-03T07:26:09.439 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.439 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.440 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.440 DEBUG:boto:Params: {} 2022-09-03T07:26:09.441 DEBUG:boto:Token: None 2022-09-03T07:26:09.442 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj76 2022-09-03T07:26:09.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+pHJUy/8vzywjJ6fyPCRGoJADxs= 2022-09-03T07:26:09.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+pHJUy/8vzywjJ6fyPCRGoJADxs=', 'Content-Length': '0'} 2022-09-03T07:26:09.445 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1cf00ef33544bd7-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.446 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.447 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.447 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:09.448 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:09.448 DEBUG:boto:Method: GET 2022-09-03T07:26:09.449 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:26:09.449 DEBUG:boto:Data: 2022-09-03T07:26:09.450 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.451 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.451 DEBUG:boto:Params: {} 2022-09-03T07:26:09.452 DEBUG:boto:Token: None 2022-09-03T07:26:09.452 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.457 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075fd35dcadc36a27-0063130191-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.458 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.459 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.460 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.460 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:09.461 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:26:09.462 DEBUG:boto:Method: GET 2022-09-03T07:26:09.462 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:26:09.463 DEBUG:boto:Data: 2022-09-03T07:26:09.463 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.464 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.464 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.465 DEBUG:boto:Params: {} 2022-09-03T07:26:09.466 DEBUG:boto:Token: None 2022-09-03T07:26:09.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.471 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1f9f285395de35f-0063130191-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.472 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.473 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:26:09.473 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.474 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.474 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.475 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.475 DEBUG:boto:Method: GET 2022-09-03T07:26:09.476 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.476 DEBUG:boto:Data: 2022-09-03T07:26:09.477 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.477 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.478 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.478 DEBUG:boto:Params: {} 2022-09-03T07:26:09.479 DEBUG:boto:Token: None 2022-09-03T07:26:09.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ysb4+JtbKN1ADKIeoAyF4Xn6Oi8= 2022-09-03T07:26:09.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ysb4+JtbKN1ADKIeoAyF4Xn6Oi8=', 'Content-Length': '0'} 2022-09-03T07:26:09.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002663a7ff97c6f907-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.484 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.485 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.485 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.486 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.486 DEBUG:boto:Method: GET 2022-09-03T07:26:09.487 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.488 DEBUG:boto:Data: 2022-09-03T07:26:09.488 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.489 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.489 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.490 DEBUG:boto:Params: {} 2022-09-03T07:26:09.490 DEBUG:boto:Token: None 2022-09-03T07:26:09.491 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ysb4+JtbKN1ADKIeoAyF4Xn6Oi8= 2022-09-03T07:26:09.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ysb4+JtbKN1ADKIeoAyF4Xn6Oi8=', 'Content-Length': '0'} 2022-09-03T07:26:09.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca6ee82cc4e1427e-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.542 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.543 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.544 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.544 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.545 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.545 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.546 DEBUG:boto:Data: 2022-09-03T07:26:09.547 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.548 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.549 DEBUG:boto:Params: {} 2022-09-03T07:26:09.550 DEBUG:boto:Token: None 2022-09-03T07:26:09.551 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:64P2rbtJZsRbhAf771bAnIayv6Q= 2022-09-03T07:26:09.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:64P2rbtJZsRbhAf771bAnIayv6Q=', 'Content-Length': '0'} 2022-09-03T07:26:09.555 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008388a5eff26c19d6-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.556 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.557 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.558 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.558 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.559 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.560 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.560 DEBUG:boto:Data: 2022-09-03T07:26:09.561 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.562 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.563 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.564 DEBUG:boto:Params: {} 2022-09-03T07:26:09.565 DEBUG:boto:Token: None 2022-09-03T07:26:09.566 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.566 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:64P2rbtJZsRbhAf771bAnIayv6Q= 2022-09-03T07:26:09.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:64P2rbtJZsRbhAf771bAnIayv6Q=', 'Content-Length': '0'} 2022-09-03T07:26:09.569 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005755325dd7680959-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.570 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:26:09.571 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.572 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.573 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.573 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.574 DEBUG:boto:Method: GET 2022-09-03T07:26:09.575 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.576 DEBUG:boto:Data: 2022-09-03T07:26:09.577 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.578 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.579 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.579 DEBUG:boto:Params: {} 2022-09-03T07:26:09.581 DEBUG:boto:Token: None 2022-09-03T07:26:09.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ysb4+JtbKN1ADKIeoAyF4Xn6Oi8= 2022-09-03T07:26:09.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ysb4+JtbKN1ADKIeoAyF4Xn6Oi8=', 'Content-Length': '0'} 2022-09-03T07:26:09.587 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da925e8714b1d8b5-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.630 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.631 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.632 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.632 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.633 DEBUG:boto:Method: GET 2022-09-03T07:26:09.634 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.634 DEBUG:boto:Data: 2022-09-03T07:26:09.635 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.636 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.636 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.637 DEBUG:boto:Params: {} 2022-09-03T07:26:09.638 DEBUG:boto:Token: None 2022-09-03T07:26:09.639 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:26:09.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ysb4+JtbKN1ADKIeoAyF4Xn6Oi8= 2022-09-03T07:26:09.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ysb4+JtbKN1ADKIeoAyF4Xn6Oi8=', 'Content-Length': '0'} 2022-09-03T07:26:09.643 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a44f6ab4a291eb67-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.644 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.645 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.645 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:09.646 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:09.647 DEBUG:boto:Method: GET 2022-09-03T07:26:09.647 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:26:09.648 DEBUG:boto:Data: 2022-09-03T07:26:09.649 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.650 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.650 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.651 DEBUG:boto:Params: {} 2022-09-03T07:26:09.652 DEBUG:boto:Token: None 2022-09-03T07:26:09.653 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.654 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.657 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1c92fd257a4547e-0063130191-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.661 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.663 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.663 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.664 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:09.665 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:09.666 DEBUG:boto:Method: GET 2022-09-03T07:26:09.667 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:26:09.668 DEBUG:boto:Data: 2022-09-03T07:26:09.669 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.669 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.670 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.670 DEBUG:boto:Params: {} 2022-09-03T07:26:09.671 DEBUG:boto:Token: None 2022-09-03T07:26:09.672 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.676 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ef6fd3216f6cbfb-0063130191-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.679 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.680 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.680 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.681 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.681 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:26:09.682 DEBUG:boto:Data: 2022-09-03T07:26:09.683 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.684 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.685 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.687 DEBUG:boto:Params: {} 2022-09-03T07:26:09.688 DEBUG:boto:Token: None 2022-09-03T07:26:09.690 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85 2022-09-03T07:26:09.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n3+LoqxqBNDuOwcuBIy6fCxPZ3E= 2022-09-03T07:26:09.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n3+LoqxqBNDuOwcuBIy6fCxPZ3E=', 'Content-Length': '0'} 2022-09-03T07:26:09.695 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f4f6c58d16700f29-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.696 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.697 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.698 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.699 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:26:09.700 DEBUG:boto:Data: 2022-09-03T07:26:09.701 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.702 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.702 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.703 DEBUG:boto:Params: {} 2022-09-03T07:26:09.704 DEBUG:boto:Token: None 2022-09-03T07:26:09.705 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85 2022-09-03T07:26:09.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n3+LoqxqBNDuOwcuBIy6fCxPZ3E= 2022-09-03T07:26:09.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n3+LoqxqBNDuOwcuBIy6fCxPZ3E=', 'Content-Length': '0'} 2022-09-03T07:26:09.709 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000400f2b1f47749c43-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.710 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:26:09.710 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.711 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.712 DEBUG:boto:Method: GET 2022-09-03T07:26:09.712 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:26:09.713 DEBUG:boto:Data: 2022-09-03T07:26:09.714 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.715 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.716 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.717 DEBUG:boto:Params: {} 2022-09-03T07:26:09.718 DEBUG:boto:Token: None 2022-09-03T07:26:09.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85 2022-09-03T07:26:09.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:caXXvgPMpzT66dB8JJmsrix/zWQ= 2022-09-03T07:26:09.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:caXXvgPMpzT66dB8JJmsrix/zWQ=', 'Content-Length': '0'} 2022-09-03T07:26:09.723 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e0f160488a263ef-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.767 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:26:09.768 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:26:09.768 DEBUG:boto:Method: GET 2022-09-03T07:26:09.769 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:26:09.769 DEBUG:boto:Data: 2022-09-03T07:26:09.770 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.770 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.771 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.771 DEBUG:boto:Params: {} 2022-09-03T07:26:09.772 DEBUG:boto:Token: None 2022-09-03T07:26:09.773 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj85 2022-09-03T07:26:09.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:caXXvgPMpzT66dB8JJmsrix/zWQ= 2022-09-03T07:26:09.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:caXXvgPMpzT66dB8JJmsrix/zWQ=', 'Content-Length': '0'} 2022-09-03T07:26:09.776 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000afce18e24851da7e-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.777 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.778 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.778 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:09.779 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:09.779 DEBUG:boto:Method: GET 2022-09-03T07:26:09.780 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:26:09.780 DEBUG:boto:Data: 2022-09-03T07:26:09.781 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.781 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.782 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.782 DEBUG:boto:Params: {} 2022-09-03T07:26:09.783 DEBUG:boto:Token: None 2022-09-03T07:26:09.784 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.788 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006213c2df76701e4d-0063130191-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.789 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.790 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.790 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.791 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:09.792 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:26:09.793 DEBUG:boto:Method: GET 2022-09-03T07:26:09.794 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:26:09.795 DEBUG:boto:Data: 2022-09-03T07:26:09.797 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.798 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.798 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.799 DEBUG:boto:Params: {} 2022-09-03T07:26:09.800 DEBUG:boto:Token: None 2022-09-03T07:26:09.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.804 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000338dd6b4fbdf677f-0063130191-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.806 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.807 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:26:09.808 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:09.809 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:09.809 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.810 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.811 DEBUG:boto:Method: GET 2022-09-03T07:26:09.811 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.812 DEBUG:boto:Data: 2022-09-03T07:26:09.813 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.814 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.815 DEBUG:boto:Params: {} 2022-09-03T07:26:09.815 DEBUG:boto:Token: None 2022-09-03T07:26:09.816 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:soxCTX9uA0ApQweBy0i9NNyrc24= 2022-09-03T07:26:09.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:soxCTX9uA0ApQweBy0i9NNyrc24=', 'Content-Length': '0'} 2022-09-03T07:26:09.824 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023363933ecf28acc-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.826 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:09.827 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:09.828 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.829 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.830 DEBUG:boto:Method: GET 2022-09-03T07:26:09.831 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.833 DEBUG:boto:Data: 2022-09-03T07:26:09.834 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.835 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.837 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.838 DEBUG:boto:Params: {} 2022-09-03T07:26:09.839 DEBUG:boto:Token: None 2022-09-03T07:26:09.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:soxCTX9uA0ApQweBy0i9NNyrc24= 2022-09-03T07:26:09.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:soxCTX9uA0ApQweBy0i9NNyrc24=', 'Content-Length': '0'} 2022-09-03T07:26:09.844 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc8ff5044c014178-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.845 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:09.845 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:09.846 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.846 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.847 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.847 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.848 DEBUG:boto:Data: 2022-09-03T07:26:09.848 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.849 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.850 DEBUG:boto:Params: {} 2022-09-03T07:26:09.851 DEBUG:boto:Token: None 2022-09-03T07:26:09.851 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vmds/pBjxhdG+BOnzHj3pEz5dEs= 2022-09-03T07:26:09.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vmds/pBjxhdG+BOnzHj3pEz5dEs=', 'Content-Length': '0'} 2022-09-03T07:26:09.857 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006bb7b1e323224b59-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.858 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:09.860 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:09.861 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.864 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.865 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.866 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.866 DEBUG:boto:Data: 2022-09-03T07:26:09.867 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.868 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.869 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.870 DEBUG:boto:Params: {} 2022-09-03T07:26:09.871 DEBUG:boto:Token: None 2022-09-03T07:26:09.872 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vmds/pBjxhdG+BOnzHj3pEz5dEs= 2022-09-03T07:26:09.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vmds/pBjxhdG+BOnzHj3pEz5dEs=', 'Content-Length': '0'} 2022-09-03T07:26:09.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000491575c69b79f070-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.879 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:26:09.880 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:09.880 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:09.881 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.882 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.883 DEBUG:boto:Method: GET 2022-09-03T07:26:09.884 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.884 DEBUG:boto:Data: 2022-09-03T07:26:09.885 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.885 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.886 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.886 DEBUG:boto:Params: {} 2022-09-03T07:26:09.887 DEBUG:boto:Token: None 2022-09-03T07:26:09.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.888 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:soxCTX9uA0ApQweBy0i9NNyrc24= 2022-09-03T07:26:09.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:soxCTX9uA0ApQweBy0i9NNyrc24=', 'Content-Length': '0'} 2022-09-03T07:26:09.892 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d751a1d70391214-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.893 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:09.895 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:09.896 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.897 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.898 DEBUG:boto:Method: GET 2022-09-03T07:26:09.899 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.901 DEBUG:boto:Data: 2022-09-03T07:26:09.902 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.903 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.904 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.905 DEBUG:boto:Params: {} 2022-09-03T07:26:09.907 DEBUG:boto:Token: None 2022-09-03T07:26:09.908 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:26:09.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:soxCTX9uA0ApQweBy0i9NNyrc24= 2022-09-03T07:26:09.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:soxCTX9uA0ApQweBy0i9NNyrc24=', 'Content-Length': '0'} 2022-09-03T07:26:09.915 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000017f2f12a3da2d000-0063130191-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.958 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.959 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.959 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:09.960 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:09.960 DEBUG:boto:Method: GET 2022-09-03T07:26:09.961 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:26:09.961 DEBUG:boto:Data: 2022-09-03T07:26:09.962 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.963 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.964 DEBUG:boto:Params: {} 2022-09-03T07:26:09.964 DEBUG:boto:Token: None 2022-09-03T07:26:09.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.969 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7c03c0efc4fd301-0063130191-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.970 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.971 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:09.971 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:09.972 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:09.972 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:09.973 DEBUG:boto:Method: GET 2022-09-03T07:26:09.973 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:26:09.975 DEBUG:boto:Data: 2022-09-03T07:26:09.975 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.976 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:09.976 DEBUG:boto:Port: 8000 2022-09-03T07:26:09.977 DEBUG:boto:Params: {} 2022-09-03T07:26:09.978 DEBUG:boto:Token: None 2022-09-03T07:26:09.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/ 2022-09-03T07:26:09.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI= 2022-09-03T07:26:09.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lsSfdEcJCjnJw1Oi2lOoWaYJaBI=', 'Content-Length': '0'} 2022-09-03T07:26:09.982 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da6113fe4d4e738e-0063130191-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.983 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:09.984 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:09.985 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:09.986 DEBUG:boto:Method: HEAD 2022-09-03T07:26:09.987 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:26:09.988 DEBUG:boto:Data: 2022-09-03T07:26:09.989 DEBUG:boto:Headers: {} 2022-09-03T07:26:09.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:09.991 DEBUG:boto:Port: 8001 2022-09-03T07:26:09.992 DEBUG:boto:Params: {} 2022-09-03T07:26:09.993 DEBUG:boto:Token: None 2022-09-03T07:26:09.994 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:09 GMT /avysmw-75/obj87 2022-09-03T07:26:09.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Mov+t/BsHYLmwOuxzVTissfZEE= 2022-09-03T07:26:09.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Mov+t/BsHYLmwOuxzVTissfZEE=', 'Content-Length': '0'} 2022-09-03T07:26:09.998 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3fb5015a0a3f31c-0063130191-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:09.999 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:10.000 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:10.000 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.001 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:26:10.001 DEBUG:boto:Data: 2022-09-03T07:26:10.002 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.003 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.003 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.004 DEBUG:boto:Params: {} 2022-09-03T07:26:10.004 DEBUG:boto:Token: None 2022-09-03T07:26:10.005 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj87 2022-09-03T07:26:10.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aqAXYNqLHsrO1GN0NWsSwRN1OhI= 2022-09-03T07:26:10.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aqAXYNqLHsrO1GN0NWsSwRN1OhI=', 'Content-Length': '0'} 2022-09-03T07:26:10.010 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002fd3ff434ee8085e-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.011 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:26:10.011 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:10.012 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:10.013 DEBUG:boto:Method: GET 2022-09-03T07:26:10.013 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:26:10.014 DEBUG:boto:Data: 2022-09-03T07:26:10.014 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.016 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.016 DEBUG:boto:Params: {} 2022-09-03T07:26:10.017 DEBUG:boto:Token: None 2022-09-03T07:26:10.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj87 2022-09-03T07:26:10.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:07J/vIyV0FsVpYxc28pGs9JdTgA= 2022-09-03T07:26:10.019 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:07J/vIyV0FsVpYxc28pGs9JdTgA=', 'Content-Length': '0'} 2022-09-03T07:26:10.021 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d166b3dc15dd489-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.021 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:26:10.022 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:26:10.023 DEBUG:boto:Method: GET 2022-09-03T07:26:10.023 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:26:10.024 DEBUG:boto:Data: 2022-09-03T07:26:10.024 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.025 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.025 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.026 DEBUG:boto:Params: {} 2022-09-03T07:26:10.027 DEBUG:boto:Token: None 2022-09-03T07:26:10.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj87 2022-09-03T07:26:10.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:07J/vIyV0FsVpYxc28pGs9JdTgA= 2022-09-03T07:26:10.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:07J/vIyV0FsVpYxc28pGs9JdTgA=', 'Content-Length': '0'} 2022-09-03T07:26:10.030 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000103c7ef7a675ac58-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.032 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.032 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.033 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:10.033 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:10.034 DEBUG:boto:Method: GET 2022-09-03T07:26:10.034 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:26:10.035 DEBUG:boto:Data: 2022-09-03T07:26:10.036 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.037 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.037 DEBUG:boto:Params: {} 2022-09-03T07:26:10.038 DEBUG:boto:Token: None 2022-09-03T07:26:10.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.043 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000727df4888c8c46a3-0063130192-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.043 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.044 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.045 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.045 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:10.046 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:26:10.046 DEBUG:boto:Method: GET 2022-09-03T07:26:10.047 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:26:10.047 DEBUG:boto:Data: 2022-09-03T07:26:10.048 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.049 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.049 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.050 DEBUG:boto:Params: {} 2022-09-03T07:26:10.050 DEBUG:boto:Token: None 2022-09-03T07:26:10.051 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.054 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002ee3141640ebc3bf-0063130192-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.104 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.105 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:26:10.106 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.107 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.107 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.108 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.108 DEBUG:boto:Method: GET 2022-09-03T07:26:10.109 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.109 DEBUG:boto:Data: 2022-09-03T07:26:10.110 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.111 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.111 DEBUG:boto:Params: {} 2022-09-03T07:26:10.112 DEBUG:boto:Token: None 2022-09-03T07:26:10.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wsSfhejKHdkRsQ2w218ajjwCUPQ= 2022-09-03T07:26:10.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wsSfhejKHdkRsQ2w218ajjwCUPQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.126 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee65c254fbdf723d-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.127 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.128 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.129 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.130 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.131 DEBUG:boto:Method: GET 2022-09-03T07:26:10.132 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.134 DEBUG:boto:Data: 2022-09-03T07:26:10.135 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.136 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.137 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.138 DEBUG:boto:Params: {} 2022-09-03T07:26:10.139 DEBUG:boto:Token: None 2022-09-03T07:26:10.140 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wsSfhejKHdkRsQ2w218ajjwCUPQ= 2022-09-03T07:26:10.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wsSfhejKHdkRsQ2w218ajjwCUPQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.145 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080f07c23d50c5c8f-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.146 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.147 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.148 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.149 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.150 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.151 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.152 DEBUG:boto:Data: 2022-09-03T07:26:10.154 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.156 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.157 DEBUG:boto:Params: {} 2022-09-03T07:26:10.158 DEBUG:boto:Token: None 2022-09-03T07:26:10.159 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngR0jTj64z88ZrpGzTj7xVLo0fM= 2022-09-03T07:26:10.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngR0jTj64z88ZrpGzTj7xVLo0fM=', 'Content-Length': '0'} 2022-09-03T07:26:10.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000341e4b0e912ad672-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.168 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.169 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.170 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.172 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.173 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.173 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.174 DEBUG:boto:Data: 2022-09-03T07:26:10.174 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.175 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.176 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.177 DEBUG:boto:Params: {} 2022-09-03T07:26:10.177 DEBUG:boto:Token: None 2022-09-03T07:26:10.178 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ngR0jTj64z88ZrpGzTj7xVLo0fM= 2022-09-03T07:26:10.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ngR0jTj64z88ZrpGzTj7xVLo0fM=', 'Content-Length': '0'} 2022-09-03T07:26:10.184 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3b8d153668cb01d-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.185 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:26:10.186 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.187 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.188 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.189 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.190 DEBUG:boto:Method: GET 2022-09-03T07:26:10.192 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.193 DEBUG:boto:Data: 2022-09-03T07:26:10.194 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.195 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.196 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.196 DEBUG:boto:Params: {} 2022-09-03T07:26:10.198 DEBUG:boto:Token: None 2022-09-03T07:26:10.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wsSfhejKHdkRsQ2w218ajjwCUPQ= 2022-09-03T07:26:10.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wsSfhejKHdkRsQ2w218ajjwCUPQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a9eeed0862823e9-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.211 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.212 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.213 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.214 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.214 DEBUG:boto:Method: GET 2022-09-03T07:26:10.215 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.216 DEBUG:boto:Data: 2022-09-03T07:26:10.217 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.218 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.219 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.220 DEBUG:boto:Params: {} 2022-09-03T07:26:10.221 DEBUG:boto:Token: None 2022-09-03T07:26:10.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:26:10.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wsSfhejKHdkRsQ2w218ajjwCUPQ= 2022-09-03T07:26:10.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wsSfhejKHdkRsQ2w218ajjwCUPQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.227 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e472002a5f12a93-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.229 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.230 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.231 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:10.232 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:10.233 DEBUG:boto:Method: GET 2022-09-03T07:26:10.234 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:26:10.235 DEBUG:boto:Data: 2022-09-03T07:26:10.236 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.238 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.238 DEBUG:boto:Params: {} 2022-09-03T07:26:10.239 DEBUG:boto:Token: None 2022-09-03T07:26:10.239 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.247 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000765bcf6531aca57d-0063130192-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.291 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.293 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.294 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.295 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:10.296 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:10.297 DEBUG:boto:Method: GET 2022-09-03T07:26:10.298 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:26:10.299 DEBUG:boto:Data: 2022-09-03T07:26:10.300 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.301 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.302 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.303 DEBUG:boto:Params: {} 2022-09-03T07:26:10.304 DEBUG:boto:Token: None 2022-09-03T07:26:10.305 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.306 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.307 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.316 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1aaa540ade7902a-0063130192-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.317 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.318 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.319 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.319 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.320 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:26:10.320 DEBUG:boto:Data: 2022-09-03T07:26:10.321 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.322 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.323 DEBUG:boto:Params: {} 2022-09-03T07:26:10.324 DEBUG:boto:Token: None 2022-09-03T07:26:10.325 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92 2022-09-03T07:26:10.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V87Rk0pN6h8fxEv+wxBoKExEtss= 2022-09-03T07:26:10.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V87Rk0pN6h8fxEv+wxBoKExEtss=', 'Content-Length': '0'} 2022-09-03T07:26:10.333 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d05bcf98b5c871ca-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.335 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.336 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.336 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.337 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:26:10.338 DEBUG:boto:Data: 2022-09-03T07:26:10.338 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.339 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.340 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.340 DEBUG:boto:Params: {} 2022-09-03T07:26:10.343 DEBUG:boto:Token: None 2022-09-03T07:26:10.344 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92 2022-09-03T07:26:10.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V87Rk0pN6h8fxEv+wxBoKExEtss= 2022-09-03T07:26:10.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V87Rk0pN6h8fxEv+wxBoKExEtss=', 'Content-Length': '0'} 2022-09-03T07:26:10.349 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029c821fb7d3e335a-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.350 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:26:10.352 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.354 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.354 DEBUG:boto:Method: GET 2022-09-03T07:26:10.355 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:26:10.356 DEBUG:boto:Data: 2022-09-03T07:26:10.357 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.359 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.360 DEBUG:boto:Params: {} 2022-09-03T07:26:10.361 DEBUG:boto:Token: None 2022-09-03T07:26:10.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92 2022-09-03T07:26:10.363 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qCUyxiiXTeQVV6fWc0Tboc4mDaM= 2022-09-03T07:26:10.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qCUyxiiXTeQVV6fWc0Tboc4mDaM=', 'Content-Length': '0'} 2022-09-03T07:26:10.368 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ce51e71f8cf71d4-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.369 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:26:10.370 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:26:10.371 DEBUG:boto:Method: GET 2022-09-03T07:26:10.372 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:26:10.372 DEBUG:boto:Data: 2022-09-03T07:26:10.373 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.373 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.374 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.374 DEBUG:boto:Params: {} 2022-09-03T07:26:10.375 DEBUG:boto:Token: None 2022-09-03T07:26:10.377 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj92 2022-09-03T07:26:10.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qCUyxiiXTeQVV6fWc0Tboc4mDaM= 2022-09-03T07:26:10.379 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qCUyxiiXTeQVV6fWc0Tboc4mDaM=', 'Content-Length': '0'} 2022-09-03T07:26:10.380 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008085dc1226512293-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.381 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.382 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.382 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:10.383 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:10.384 DEBUG:boto:Method: GET 2022-09-03T07:26:10.384 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:26:10.385 DEBUG:boto:Data: 2022-09-03T07:26:10.385 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.386 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.387 DEBUG:boto:Params: {} 2022-09-03T07:26:10.387 DEBUG:boto:Token: None 2022-09-03T07:26:10.388 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.389 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.392 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f824365342483f4d-0063130192-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.393 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.394 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.394 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.395 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:10.395 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:26:10.396 DEBUG:boto:Method: GET 2022-09-03T07:26:10.396 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:26:10.397 DEBUG:boto:Data: 2022-09-03T07:26:10.397 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.398 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.398 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.399 DEBUG:boto:Params: {} 2022-09-03T07:26:10.401 DEBUG:boto:Token: None 2022-09-03T07:26:10.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.407 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000036fbe0c2c8d58a4-0063130192-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.409 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.410 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:26:10.411 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.411 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.412 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.412 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.413 DEBUG:boto:Method: GET 2022-09-03T07:26:10.413 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.414 DEBUG:boto:Data: 2022-09-03T07:26:10.414 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.415 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.415 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.416 DEBUG:boto:Params: {} 2022-09-03T07:26:10.417 DEBUG:boto:Token: None 2022-09-03T07:26:10.417 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MPj+bCyojV3DGPkx7F0cYp6yekU= 2022-09-03T07:26:10.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MPj+bCyojV3DGPkx7F0cYp6yekU=', 'Content-Length': '0'} 2022-09-03T07:26:10.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b712b42b7dadcb3-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.423 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.423 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.424 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.424 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.425 DEBUG:boto:Method: GET 2022-09-03T07:26:10.425 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.426 DEBUG:boto:Data: 2022-09-03T07:26:10.427 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.427 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.428 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.428 DEBUG:boto:Params: {} 2022-09-03T07:26:10.429 DEBUG:boto:Token: None 2022-09-03T07:26:10.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MPj+bCyojV3DGPkx7F0cYp6yekU= 2022-09-03T07:26:10.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MPj+bCyojV3DGPkx7F0cYp6yekU=', 'Content-Length': '0'} 2022-09-03T07:26:10.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb6061d5dbf08d6e-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.434 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.435 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.435 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.436 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.436 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.437 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.437 DEBUG:boto:Data: 2022-09-03T07:26:10.438 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.439 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.440 DEBUG:boto:Params: {} 2022-09-03T07:26:10.440 DEBUG:boto:Token: None 2022-09-03T07:26:10.441 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v2OziV/krLogJjgiLzdP8MR5oSQ= 2022-09-03T07:26:10.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v2OziV/krLogJjgiLzdP8MR5oSQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0b6c9641de82bf3-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.444 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.445 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.446 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.446 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.447 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.447 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.448 DEBUG:boto:Data: 2022-09-03T07:26:10.448 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.449 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.449 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.450 DEBUG:boto:Params: {} 2022-09-03T07:26:10.450 DEBUG:boto:Token: None 2022-09-03T07:26:10.451 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v2OziV/krLogJjgiLzdP8MR5oSQ= 2022-09-03T07:26:10.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v2OziV/krLogJjgiLzdP8MR5oSQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.454 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1976c6452408c7f-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.455 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:26:10.456 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.456 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.457 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.457 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.458 DEBUG:boto:Method: GET 2022-09-03T07:26:10.459 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.459 DEBUG:boto:Data: 2022-09-03T07:26:10.460 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.461 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.461 DEBUG:boto:Params: {} 2022-09-03T07:26:10.462 DEBUG:boto:Token: None 2022-09-03T07:26:10.462 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MPj+bCyojV3DGPkx7F0cYp6yekU= 2022-09-03T07:26:10.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MPj+bCyojV3DGPkx7F0cYp6yekU=', 'Content-Length': '0'} 2022-09-03T07:26:10.465 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e49799832764257a-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.466 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.467 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.467 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.468 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.468 DEBUG:boto:Method: GET 2022-09-03T07:26:10.469 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.469 DEBUG:boto:Data: 2022-09-03T07:26:10.470 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.471 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.471 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.472 DEBUG:boto:Params: {} 2022-09-03T07:26:10.472 DEBUG:boto:Token: None 2022-09-03T07:26:10.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:26:10.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MPj+bCyojV3DGPkx7F0cYp6yekU= 2022-09-03T07:26:10.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MPj+bCyojV3DGPkx7F0cYp6yekU=', 'Content-Length': '0'} 2022-09-03T07:26:10.476 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5dc8453e9790a9b-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.519 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.520 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.521 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:10.522 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:10.522 DEBUG:boto:Method: GET 2022-09-03T07:26:10.523 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:26:10.523 DEBUG:boto:Data: 2022-09-03T07:26:10.524 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.525 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.527 DEBUG:boto:Params: {} 2022-09-03T07:26:10.528 DEBUG:boto:Token: None 2022-09-03T07:26:10.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.534 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000015ac090fcc5e2681-0063130192-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.536 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.537 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.539 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.540 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:10.541 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:10.542 DEBUG:boto:Method: GET 2022-09-03T07:26:10.543 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:26:10.544 DEBUG:boto:Data: 2022-09-03T07:26:10.545 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.546 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.547 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.548 DEBUG:boto:Params: {} 2022-09-03T07:26:10.549 DEBUG:boto:Token: None 2022-09-03T07:26:10.550 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.555 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000033ef6d300815d871-0063130192-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.556 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.556 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.557 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.558 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.558 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:26:10.559 DEBUG:boto:Data: 2022-09-03T07:26:10.560 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.560 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.561 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.561 DEBUG:boto:Params: {} 2022-09-03T07:26:10.562 DEBUG:boto:Token: None 2022-09-03T07:26:10.563 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95 2022-09-03T07:26:10.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGVbUK+MlIDTc2oweJsTzYAYo7o= 2022-09-03T07:26:10.565 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGVbUK+MlIDTc2oweJsTzYAYo7o=', 'Content-Length': '0'} 2022-09-03T07:26:10.568 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004fa6e1176b6d6f97-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.569 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.570 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.571 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.572 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:26:10.573 DEBUG:boto:Data: 2022-09-03T07:26:10.574 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.575 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.576 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.576 DEBUG:boto:Params: {} 2022-09-03T07:26:10.580 DEBUG:boto:Token: None 2022-09-03T07:26:10.581 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95 2022-09-03T07:26:10.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pGVbUK+MlIDTc2oweJsTzYAYo7o= 2022-09-03T07:26:10.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pGVbUK+MlIDTc2oweJsTzYAYo7o=', 'Content-Length': '0'} 2022-09-03T07:26:10.586 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004663e4e0f7009ad6-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.588 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:26:10.589 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.590 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.590 DEBUG:boto:Method: GET 2022-09-03T07:26:10.591 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:26:10.592 DEBUG:boto:Data: 2022-09-03T07:26:10.593 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.594 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.595 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.596 DEBUG:boto:Params: {} 2022-09-03T07:26:10.597 DEBUG:boto:Token: None 2022-09-03T07:26:10.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95 2022-09-03T07:26:10.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VOOFRdGo5u+6LbndAYyJCuJhG98= 2022-09-03T07:26:10.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VOOFRdGo5u+6LbndAYyJCuJhG98=', 'Content-Length': '0'} 2022-09-03T07:26:10.608 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001704e85daa539e6f-0063130192-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.610 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:26:10.610 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:26:10.611 DEBUG:boto:Method: GET 2022-09-03T07:26:10.611 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:26:10.612 DEBUG:boto:Data: 2022-09-03T07:26:10.612 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.613 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.614 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.614 DEBUG:boto:Params: {} 2022-09-03T07:26:10.615 DEBUG:boto:Token: None 2022-09-03T07:26:10.616 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj95 2022-09-03T07:26:10.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VOOFRdGo5u+6LbndAYyJCuJhG98= 2022-09-03T07:26:10.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VOOFRdGo5u+6LbndAYyJCuJhG98=', 'Content-Length': '0'} 2022-09-03T07:26:10.620 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ceb279b893e3fd56-0063130192-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.666 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.667 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.667 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:10.668 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:10.669 DEBUG:boto:Method: GET 2022-09-03T07:26:10.669 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:26:10.670 DEBUG:boto:Data: 2022-09-03T07:26:10.671 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.671 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:26:10.672 DEBUG:boto:Port: 8001 2022-09-03T07:26:10.673 DEBUG:boto:Params: {} 2022-09-03T07:26:10.676 DEBUG:boto:Token: None 2022-09-03T07:26:10.677 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.681 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ee139f195c19d036-0063130192-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.682 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.683 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.684 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.684 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:10.685 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:26:10.686 DEBUG:boto:Method: GET 2022-09-03T07:26:10.687 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:26:10.687 DEBUG:boto:Data: 2022-09-03T07:26:10.688 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.689 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:26:10.689 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.690 DEBUG:boto:Params: {} 2022-09-03T07:26:10.695 DEBUG:boto:Token: None 2022-09-03T07:26:10.696 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ= 2022-09-03T07:26:10.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K9EpKB+P5j2pZJ3eu3DkxC+90mQ=', 'Content-Length': '0'} 2022-09-03T07:26:10.705 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005d10eb840f6a47dc-0063130192-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.707 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:26:10.708 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:26:10.709 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '31', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:26:10.710 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '31', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:26:10.711 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --num-shards 31 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:26:10.775 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:26:10.776 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:26:10.840 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:26:10.841 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-75 2022-09-03T07:26:10.856 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 25 2022-09-03T07:26:10.942 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.944 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.945 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.945 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:26:10.946 DEBUG:boto:Data: 2022-09-03T07:26:10.946 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:10.948 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.950 DEBUG:boto:Params: {} 2022-09-03T07:26:10.951 DEBUG:boto:Token: None 2022-09-03T07:26:10.952 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gA2YlqcGfoto8FZtt7n4tuyi19s= 2022-09-03T07:26:10.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gA2YlqcGfoto8FZtt7n4tuyi19s=', 'Content-Length': '0'} 2022-09-03T07:26:10.960 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '25'), ('X-RGW-Bytes-Used', '75'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f1208f205caaca5d-0063130192-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.962 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:26:10.963 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:26:10.964 DEBUG:boto:Method: PUT 2022-09-03T07:26:10.965 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:26:10.967 DEBUG:boto:Data: 2022-09-03T07:26:10.968 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:26:10.969 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:10.970 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.972 DEBUG:boto:Params: {} 2022-09-03T07:26:10.973 DEBUG:boto:Token: None 2022-09-03T07:26:10.974 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/obj31 2022-09-03T07:26:10.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MyXsgugVCJ6KvWK2l4lqNdLvaUo= 2022-09-03T07:26:10.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MyXsgugVCJ6KvWK2l4lqNdLvaUo='} 2022-09-03T07:26:10.987 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000033dd090219815912-0063130192-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:10.988 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:10.989 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:10.989 DEBUG:boto:Method: HEAD 2022-09-03T07:26:10.990 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:26:10.990 DEBUG:boto:Data: 2022-09-03T07:26:10.991 DEBUG:boto:Headers: {} 2022-09-03T07:26:10.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:10.992 DEBUG:boto:Port: 8000 2022-09-03T07:26:10.992 DEBUG:boto:Params: {} 2022-09-03T07:26:10.993 DEBUG:boto:Token: None 2022-09-03T07:26:10.994 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:10 GMT /avysmw-75/ 2022-09-03T07:26:10.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gA2YlqcGfoto8FZtt7n4tuyi19s= 2022-09-03T07:26:10.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gA2YlqcGfoto8FZtt7n4tuyi19s=', 'Content-Length': '0'} 2022-09-03T07:26:10.999 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '26'), ('X-RGW-Bytes-Used', '78'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c4aa52c9019c2351-0063130192-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:11.000 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:26:11.001 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:26:11.001 DEBUG:boto:Method: PUT 2022-09-03T07:26:11.002 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:26:11.002 DEBUG:boto:Data: 2022-09-03T07:26:11.003 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:26:11.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:11.004 DEBUG:boto:Port: 8000 2022-09-03T07:26:11.005 DEBUG:boto:Params: {} 2022-09-03T07:26:11.005 DEBUG:boto:Token: None 2022-09-03T07:26:11.006 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:26:11 GMT /avysmw-75/obj62 2022-09-03T07:26:11.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EVaAkMcP/5TiJ0fIIueuqXkixUk= 2022-09-03T07:26:11.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:26:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EVaAkMcP/5TiJ0fIIueuqXkixUk='} 2022-09-03T07:26:11.054 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000fa3d2a6ca267dd53-0063130193-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:11.056 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:11.056 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:11.070 DEBUG:boto:Method: HEAD 2022-09-03T07:26:11.071 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:26:11.074 DEBUG:boto:Data: 2022-09-03T07:26:11.075 DEBUG:boto:Headers: {} 2022-09-03T07:26:11.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:11.077 DEBUG:boto:Port: 8000 2022-09-03T07:26:11.077 DEBUG:boto:Params: {} 2022-09-03T07:26:11.078 DEBUG:boto:Token: None 2022-09-03T07:26:11.079 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:11 GMT /avysmw-75/ 2022-09-03T07:26:11.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6PxuBdx4O7cZtB2nrc/yM7KxnAw= 2022-09-03T07:26:11.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6PxuBdx4O7cZtB2nrc/yM7KxnAw=', 'Content-Length': '0'} 2022-09-03T07:26:11.085 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '27'), ('X-RGW-Bytes-Used', '81'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002a55c0d59e77acef-0063130193-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:11.087 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:26:11.088 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:26:11.089 DEBUG:boto:Method: PUT 2022-09-03T07:26:11.090 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:26:11.091 DEBUG:boto:Data: 2022-09-03T07:26:11.091 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:26:11.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:11.095 DEBUG:boto:Port: 8000 2022-09-03T07:26:11.096 DEBUG:boto:Params: {} 2022-09-03T07:26:11.097 DEBUG:boto:Token: None 2022-09-03T07:26:11.098 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:26:11 GMT /avysmw-75/obj93 2022-09-03T07:26:11.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TAhP3c5m1FM526ZnnxCLGlsBJPI= 2022-09-03T07:26:11.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:26:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TAhP3c5m1FM526ZnnxCLGlsBJPI='} 2022-09-03T07:26:11.112 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000000612a00649459b9c-0063130193-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:11.113 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:11.114 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:11.117 DEBUG:boto:Method: HEAD 2022-09-03T07:26:11.119 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:26:11.120 DEBUG:boto:Data: 2022-09-03T07:26:11.121 DEBUG:boto:Headers: {} 2022-09-03T07:26:11.133 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:11.134 DEBUG:boto:Port: 8000 2022-09-03T07:26:11.135 DEBUG:boto:Params: {} 2022-09-03T07:26:11.136 DEBUG:boto:Token: None 2022-09-03T07:26:11.137 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:11 GMT /avysmw-75/ 2022-09-03T07:26:11.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6PxuBdx4O7cZtB2nrc/yM7KxnAw= 2022-09-03T07:26:11.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6PxuBdx4O7cZtB2nrc/yM7KxnAw=', 'Content-Length': '0'} 2022-09-03T07:26:11.144 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '28'), ('X-RGW-Bytes-Used', '84'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cda9bc344e62a81f-0063130193-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:11.146 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:26:11.147 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:26:11.148 DEBUG:boto:Method: PUT 2022-09-03T07:26:11.149 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:26:11.150 DEBUG:boto:Data: 2022-09-03T07:26:11.151 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:26:11.152 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:11.153 DEBUG:boto:Port: 8000 2022-09-03T07:26:11.156 DEBUG:boto:Params: {} 2022-09-03T07:26:11.157 DEBUG:boto:Token: None 2022-09-03T07:26:11.158 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:26:11 GMT /avysmw-75/obj124 2022-09-03T07:26:11.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jNIFpzV15vrN6S7igqiy40X7rnE= 2022-09-03T07:26:11.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:26:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jNIFpzV15vrN6S7igqiy40X7rnE='} 2022-09-03T07:26:11.213 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000000e9be9e501c21bd4-0063130193-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:11.217 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:26:11.219 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:26:11.219 DEBUG:boto:Method: HEAD 2022-09-03T07:26:11.220 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:26:11.220 DEBUG:boto:Data: 2022-09-03T07:26:11.221 DEBUG:boto:Headers: {} 2022-09-03T07:26:11.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:11.223 DEBUG:boto:Port: 8000 2022-09-03T07:26:11.223 DEBUG:boto:Params: {} 2022-09-03T07:26:11.224 DEBUG:boto:Token: None 2022-09-03T07:26:11.225 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:26:11 GMT /avysmw-75/ 2022-09-03T07:26:11.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6PxuBdx4O7cZtB2nrc/yM7KxnAw= 2022-09-03T07:26:11.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:26:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6PxuBdx4O7cZtB2nrc/yM7KxnAw=', 'Content-Length': '0'} 2022-09-03T07:26:11.230 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '29'), ('X-RGW-Bytes-Used', '87'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000010d3ae8b2f81e816-0063130193-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:11.230 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:26:11.231 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:26:11.232 DEBUG:boto:Method: PUT 2022-09-03T07:26:11.232 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:26:11.233 DEBUG:boto:Data: 2022-09-03T07:26:11.233 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:26:11.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:26:11.236 DEBUG:boto:Port: 8000 2022-09-03T07:26:11.236 DEBUG:boto:Params: {} 2022-09-03T07:26:11.237 DEBUG:boto:Token: None 2022-09-03T07:26:11.237 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:26:11 GMT /avysmw-75/obj155 2022-09-03T07:26:11.238 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8CWaHng2Fu6FIP5Yavn47uSmFy8= 2022-09-03T07:26:11.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:26:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8CWaHng2Fu6FIP5Yavn47uSmFy8='} 2022-09-03T07:26:11.246 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000ebf4ef648b5164b7-0063130193-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:26:11.247 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-75 2022-09-03T07:26:11.248 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:26:11.248 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:26:11.249 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:26:11.310 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:26:11.311 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:26:11.374 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:11.372+0000 7f15ee783600 1 waiting to reach latest gen 5, on 4.. 2022-09-03T07:26:16.377 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:16.375+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:26:16.377 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:26:16.378 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:21.380 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:21.377+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:26:21.381 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:26:21.381 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:26.381 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:26.379+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:26:26.381 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:26:26.381 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:31.383 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:31.381+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:26:31.384 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:26:31.384 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:36.386 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:36.383+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:26:36.387 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , , , 00000000001.101.5, , , , , , , , , , , , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:36.387 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:41.388 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:41.385+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:26:41.388 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , , , 00000000001.101.5, , , , , , , , , , , , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:41.389 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:46.390 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:46.387+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:26:46.391 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , , , 00000000001.101.5, , , , , , , , , , , , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:46.391 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:51.391 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:51.389+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:26:51.392 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , , , 00000000001.101.5, , , , , , , , , , , , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:51.392 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:56.395 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:26:56.391+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:26:56.396 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , , , 00000000001.101.5, , , , , , , , , , , , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:26:56.396 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:01.395 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:01.393+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:01.396 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , , , 00000000001.101.5, , , , , , , , , , , , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:01.396 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:06.397 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:06.395+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:06.398 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , , , 00000000001.101.5, , , , , , , , , , , , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:06.398 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:11.399 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:11.397+0000 7f15ee783600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:11.400 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , , , 00000000001.101.5, , , , , , , , , , , , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:11.401 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:16.402 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:16.399+0000 7f15ee783600 1 bucket sync caught up with source: 2022-09-03T07:27:16.403 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:16.403 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:16.403 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:16.399+0000 7f15ee783600 0 bucket checkpoint complete 2022-09-03T07:27:16.430 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-75 2022-09-03T07:27:16.431 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:16.432 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:16.433 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:27:16.500 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:27:16.501 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:27:16.564 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:16.559+0000 7f6319911600 1 bucket sync caught up with source: 2022-09-03T07:27:16.564 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:16.565 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:16.565 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:16.559+0000 7f6319911600 0 bucket checkpoint complete 2022-09-03T07:27:16.589 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-75 2022-09-03T07:27:16.590 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:16.591 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:16.592 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:27:16.656 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:27:16.657 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:27:16.699 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:16.697+0000 7f203393b600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:16.699 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , 00000000001.75.5, , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:27:16.700 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:21.701 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:21.699+0000 7f203393b600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:21.702 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , 00000000001.75.5, , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:27:21.702 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:26.704 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:26.702+0000 7f203393b600 1 bucket sync caught up with source: 2022-09-03T07:27:26.704 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:26.705 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , 00000000002.83.5, , 00000000001.75.5, , 00000000001.101.5, , , , , , , , , , 00000000001.65.5, , , , , , , , , , , , , , 00000000001.74.5] 2022-09-03T07:27:26.706 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:26.702+0000 7f203393b600 0 bucket checkpoint complete 2022-09-03T07:27:26.725 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-75 2022-09-03T07:27:26.726 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:26.727 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:26.727 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:27:26.795 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:27:26.796 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:27:26.838 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:26.837+0000 7f5d85057600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:26.839 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000004.353.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000002.376.5, 00000000006.310.5, 00000000004.221.5] 2022-09-03T07:27:26.839 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000004.353.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000003.386.5, 00000000006.310.5, 00000000004.221.5] 2022-09-03T07:27:31.840 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:31.838+0000 7f5d85057600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:31.841 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000004.353.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000002.376.5, 00000000006.310.5, 00000000004.221.5] 2022-09-03T07:27:31.841 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000004.353.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000003.386.5, 00000000006.310.5, 00000000004.221.5] 2022-09-03T07:27:36.840 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:36.838+0000 7f5d85057600 1 bucket sync caught up with source: 2022-09-03T07:27:36.841 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000004.353.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000003.386.5, 00000000006.310.5, 00000000004.221.5] 2022-09-03T07:27:36.841 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000004.353.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000003.386.5, 00000000006.310.5, 00000000004.221.5] 2022-09-03T07:27:36.841 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:36.838+0000 7f5d85057600 0 bucket checkpoint complete 2022-09-03T07:27:36.861 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-75 2022-09-03T07:27:36.861 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:36.862 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:36.863 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:27:36.925 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:27:36.926 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:27:36.968 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:36.965+0000 7fa2b121d600 1 bucket sync caught up with source: 2022-09-03T07:27:36.968 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000004.353.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000003.386.5, 00000000006.310.5, 00000000004.221.5] 2022-09-03T07:27:36.969 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000004.353.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000003.386.5, 00000000006.310.5, 00000000004.221.5] 2022-09-03T07:27:36.969 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:36.965+0000 7fa2b121d600 0 bucket checkpoint complete 2022-09-03T07:27:36.983 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-75 2022-09-03T07:27:36.983 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-75 2022-09-03T07:27:36.984 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:36.984 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:36.985 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:27:37.046 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:27:37.046 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:27:37.088 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:37.083+0000 7fb58b5f2600 1 bucket sync caught up with source: 2022-09-03T07:27:37.089 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000003.296.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:37.090 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000003.296.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:37.090 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:27:37.083+0000 7fb58b5f2600 0 bucket checkpoint complete 2022-09-03T07:27:37.102 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-75 2022-09-03T07:27:37.103 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:37.104 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:27:37.104 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:27:37.174 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:27:37.175 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:27:37.226 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:37.221+0000 7f3ce96a9600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:37.227 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000002.287.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:37.228 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000003.296.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:42.224 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:42.222+0000 7f3ce96a9600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:42.225 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000002.287.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:42.225 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000003.296.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:47.228 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:47.225+0000 7f3ce96a9600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:47.229 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000002.287.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:47.232 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000003.296.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:52.231 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:52.227+0000 7f3ce96a9600 1 waiting for incremental sync to catch up: 2022-09-03T07:27:52.232 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000002.287.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:52.232 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000003.296.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:57.231 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:57.229+0000 7f3ce96a9600 1 bucket sync caught up with source: 2022-09-03T07:27:57.231 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000003.296.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:57.232 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000004.207.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000003.296.5, 00000000006.319.5, 00000000004.140.5] 2022-09-03T07:27:57.232 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:27:57.229+0000 7f3ce96a9600 0 bucket checkpoint complete 2022-09-03T07:27:57.268 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-75 2022-09-03T07:27:57.271 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:27:57.272 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.273 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.273 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.281 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:27:57.282 DEBUG:boto:Data: 2022-09-03T07:27:57.282 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.283 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.284 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.284 DEBUG:boto:Params: {} 2022-09-03T07:27:57.285 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:27:57.286 DEBUG:boto:Token: None 2022-09-03T07:27:57.287 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/ 2022-09-03T07:27:57.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JJBLWQ9TYmmuUzxntCmJyPrfUlE= 2022-09-03T07:27:57.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JJBLWQ9TYmmuUzxntCmJyPrfUlE=', 'Content-Length': '0'} 2022-09-03T07:27:57.303 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '30'), ('X-RGW-Bytes-Used', '90'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000008d96ac9507e03d3-00631301fd-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.307 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.310 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.311 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.311 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:27:57.312 DEBUG:boto:Data: 2022-09-03T07:27:57.312 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.313 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.315 DEBUG:boto:Params: {} 2022-09-03T07:27:57.315 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:27:57.316 DEBUG:boto:Token: None 2022-09-03T07:27:57.316 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/ 2022-09-03T07:27:57.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JJBLWQ9TYmmuUzxntCmJyPrfUlE= 2022-09-03T07:27:57.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JJBLWQ9TYmmuUzxntCmJyPrfUlE=', 'Content-Length': '0'} 2022-09-03T07:27:57.323 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '30'), ('X-RGW-Bytes-Used', '90'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ba3a0c2288fff5ae-00631301fd-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.324 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:27:57.325 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.326 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.326 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.328 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.328 DEBUG:boto:Method: GET 2022-09-03T07:27:57.329 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.329 DEBUG:boto:Data: 2022-09-03T07:27:57.330 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.330 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.331 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.332 DEBUG:boto:Params: {} 2022-09-03T07:27:57.332 DEBUG:boto:Token: None 2022-09-03T07:27:57.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/?versions 2022-09-03T07:27:57.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qqtIUnYQdbJuDoILFeYdge0+0mk= 2022-09-03T07:27:57.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qqtIUnYQdbJuDoILFeYdge0+0mk=', 'Content-Length': '0'} 2022-09-03T07:27:57.338 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d724fb759d613da2-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.340 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.348 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:27:57.354 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:27:57.355 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:27:57.356 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:27:57.357 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:27:57.357 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:27:57.359 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:27:57.359 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:27:57.360 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:27:57.361 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:27:57.362 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:27:57.363 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:27:57.364 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:27:57.365 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:27:57.366 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:27:57.367 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:27:57.368 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:27:57.368 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:27:57.370 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:27:57.371 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:27:57.372 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:27:57.373 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:27:57.374 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:27:57.375 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:27:57.376 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:27:57.377 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:27:57.377 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:27:57.378 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:27:57.379 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:27:57.380 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:27:57.381 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:27:57.382 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.383 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.384 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.385 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.386 DEBUG:boto:Method: GET 2022-09-03T07:27:57.387 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.388 DEBUG:boto:Data: 2022-09-03T07:27:57.389 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.390 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.392 DEBUG:boto:Params: {} 2022-09-03T07:27:57.393 DEBUG:boto:Token: None 2022-09-03T07:27:57.394 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/?versions 2022-09-03T07:27:57.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qqtIUnYQdbJuDoILFeYdge0+0mk= 2022-09-03T07:27:57.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qqtIUnYQdbJuDoILFeYdge0+0mk=', 'Content-Length': '0'} 2022-09-03T07:27:57.407 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000df28a65f4ee66b7f-00631301fd-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.408 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.410 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:27:57.411 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:27:57.411 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:27:57.412 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:27:57.413 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:27:57.413 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:27:57.414 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:27:57.414 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:27:57.415 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:27:57.416 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:27:57.416 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:27:57.417 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:27:57.417 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:27:57.418 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:27:57.418 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:27:57.419 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:27:57.419 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:27:57.420 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:27:57.421 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:27:57.421 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:27:57.422 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:27:57.422 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:27:57.422 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:27:57.423 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:27:57.423 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:27:57.424 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:27:57.424 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:27:57.425 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:27:57.426 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:27:57.426 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:27:57.427 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.428 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.428 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.428 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.429 DEBUG:boto:Method: GET 2022-09-03T07:27:57.430 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.430 DEBUG:boto:Data: 2022-09-03T07:27:57.431 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.431 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.432 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.432 DEBUG:boto:Params: {} 2022-09-03T07:27:57.433 DEBUG:boto:Token: None 2022-09-03T07:27:57.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/?versions 2022-09-03T07:27:57.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qqtIUnYQdbJuDoILFeYdge0+0mk= 2022-09-03T07:27:57.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qqtIUnYQdbJuDoILFeYdge0+0mk=', 'Content-Length': '0'} 2022-09-03T07:27:57.439 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b963cdf3a96102e-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.440 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.444 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.445 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.447 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.448 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.449 DEBUG:boto:Method: GET 2022-09-03T07:27:57.450 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:27:57.451 DEBUG:boto:Data: 2022-09-03T07:27:57.452 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.452 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.453 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.454 DEBUG:boto:Params: {} 2022-09-03T07:27:57.455 DEBUG:boto:Token: None 2022-09-03T07:27:57.458 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/?versions 2022-09-03T07:27:57.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qqtIUnYQdbJuDoILFeYdge0+0mk= 2022-09-03T07:27:57.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qqtIUnYQdbJuDoILFeYdge0+0mk=', 'Content-Length': '0'} 2022-09-03T07:27:57.466 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093d8e4e9b9d84cd6-00631301fd-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.467 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.470 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:27:57.470 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.471 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.472 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.472 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.473 DEBUG:boto:Method: GET 2022-09-03T07:27:57.474 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.474 DEBUG:boto:Data: 2022-09-03T07:27:57.475 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.475 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.476 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.477 DEBUG:boto:Params: {} 2022-09-03T07:27:57.478 DEBUG:boto:Token: None 2022-09-03T07:27:57.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fsLtR20X5XXWLay4PME2Mhj1xbQ= 2022-09-03T07:27:57.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fsLtR20X5XXWLay4PME2Mhj1xbQ=', 'Content-Length': '0'} 2022-09-03T07:27:57.482 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038d1f4b45ce9d741-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.484 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.485 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.486 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.488 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.489 DEBUG:boto:Method: GET 2022-09-03T07:27:57.490 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.491 DEBUG:boto:Data: 2022-09-03T07:27:57.492 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.504 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.504 DEBUG:boto:Params: {} 2022-09-03T07:27:57.506 DEBUG:boto:Token: None 2022-09-03T07:27:57.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fsLtR20X5XXWLay4PME2Mhj1xbQ= 2022-09-03T07:27:57.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fsLtR20X5XXWLay4PME2Mhj1xbQ=', 'Content-Length': '0'} 2022-09-03T07:27:57.513 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005bc6c32ebd56dd52-00631301fd-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.514 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.515 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.517 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.518 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.519 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.519 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.520 DEBUG:boto:Data: 2022-09-03T07:27:57.520 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.523 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.524 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.524 DEBUG:boto:Params: {} 2022-09-03T07:27:57.525 DEBUG:boto:Token: None 2022-09-03T07:27:57.525 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.526 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CBD3oZmWzVZSPM+Z+7C4iUB5TH0= 2022-09-03T07:27:57.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CBD3oZmWzVZSPM+Z+7C4iUB5TH0=', 'Content-Length': '0'} 2022-09-03T07:27:57.531 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044fd6a651eb3bd8e-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.533 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.533 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.534 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.535 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.536 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.536 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.537 DEBUG:boto:Data: 2022-09-03T07:27:57.538 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.540 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.542 DEBUG:boto:Params: {} 2022-09-03T07:27:57.544 DEBUG:boto:Token: None 2022-09-03T07:27:57.546 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.547 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CBD3oZmWzVZSPM+Z+7C4iUB5TH0= 2022-09-03T07:27:57.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CBD3oZmWzVZSPM+Z+7C4iUB5TH0=', 'Content-Length': '0'} 2022-09-03T07:27:57.559 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2b6b073c51d7419-00631301fd-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.567 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:27:57.568 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.569 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.570 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.571 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.572 DEBUG:boto:Method: GET 2022-09-03T07:27:57.573 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.574 DEBUG:boto:Data: 2022-09-03T07:27:57.574 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.575 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.576 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.576 DEBUG:boto:Params: {} 2022-09-03T07:27:57.577 DEBUG:boto:Token: None 2022-09-03T07:27:57.578 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fsLtR20X5XXWLay4PME2Mhj1xbQ= 2022-09-03T07:27:57.581 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fsLtR20X5XXWLay4PME2Mhj1xbQ=', 'Content-Length': '0'} 2022-09-03T07:27:57.584 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000342d6a75529b9f48-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.591 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.591 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.592 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.592 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.593 DEBUG:boto:Method: GET 2022-09-03T07:27:57.596 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.597 DEBUG:boto:Data: 2022-09-03T07:27:57.597 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.598 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.598 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.599 DEBUG:boto:Params: {} 2022-09-03T07:27:57.600 DEBUG:boto:Token: None 2022-09-03T07:27:57.600 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:27:57.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fsLtR20X5XXWLay4PME2Mhj1xbQ= 2022-09-03T07:27:57.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fsLtR20X5XXWLay4PME2Mhj1xbQ=', 'Content-Length': '0'} 2022-09-03T07:27:57.605 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064fc0131a5765ca2-00631301fd-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.606 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.606 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.607 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:27:57.607 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:27:57.608 DEBUG:boto:Method: GET 2022-09-03T07:27:57.609 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:27:57.610 DEBUG:boto:Data: 2022-09-03T07:27:57.610 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.615 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.616 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.617 DEBUG:boto:Params: {} 2022-09-03T07:27:57.619 DEBUG:boto:Token: None 2022-09-03T07:27:57.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/ 2022-09-03T07:27:57.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI= 2022-09-03T07:27:57.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI=', 'Content-Length': '0'} 2022-09-03T07:27:57.627 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7a057a46135f0b7-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.628 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.629 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.629 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.630 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:27:57.631 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:27:57.631 DEBUG:boto:Method: GET 2022-09-03T07:27:57.632 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:27:57.632 DEBUG:boto:Data: 2022-09-03T07:27:57.635 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.636 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.636 DEBUG:boto:Params: {} 2022-09-03T07:27:57.637 DEBUG:boto:Token: None 2022-09-03T07:27:57.637 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/ 2022-09-03T07:27:57.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI= 2022-09-03T07:27:57.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI=', 'Content-Length': '0'} 2022-09-03T07:27:57.643 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000805a11cc3126d82a-00631301fd-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.644 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.645 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.646 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.646 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.647 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:27:57.647 DEBUG:boto:Data: 2022-09-03T07:27:57.648 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.648 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.649 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.649 DEBUG:boto:Params: {} 2022-09-03T07:27:57.650 DEBUG:boto:Token: None 2022-09-03T07:27:57.651 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11 2022-09-03T07:27:57.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bs/glr5cEd8iB3tsQj2styaB2gU= 2022-09-03T07:27:57.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bs/glr5cEd8iB3tsQj2styaB2gU=', 'Content-Length': '0'} 2022-09-03T07:27:57.656 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1680c1c06499dd9-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.657 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.659 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.660 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.661 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:27:57.666 DEBUG:boto:Data: 2022-09-03T07:27:57.667 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.668 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.669 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.670 DEBUG:boto:Params: {} 2022-09-03T07:27:57.671 DEBUG:boto:Token: None 2022-09-03T07:27:57.671 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11 2022-09-03T07:27:57.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bs/glr5cEd8iB3tsQj2styaB2gU= 2022-09-03T07:27:57.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bs/glr5cEd8iB3tsQj2styaB2gU=', 'Content-Length': '0'} 2022-09-03T07:27:57.689 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006724228d4e21d5cb-00631301fd-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.690 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:27:57.691 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.692 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.693 DEBUG:boto:Method: GET 2022-09-03T07:27:57.695 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:27:57.696 DEBUG:boto:Data: 2022-09-03T07:27:57.697 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.698 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.699 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.700 DEBUG:boto:Params: {} 2022-09-03T07:27:57.701 DEBUG:boto:Token: None 2022-09-03T07:27:57.702 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11 2022-09-03T07:27:57.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7fcXAmaSng+zghzxHgrucDJ0PfI= 2022-09-03T07:27:57.704 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7fcXAmaSng+zghzxHgrucDJ0PfI=', 'Content-Length': '0'} 2022-09-03T07:27:57.707 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060d4d03ce414d28d-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.709 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:27:57.710 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:27:57.711 DEBUG:boto:Method: GET 2022-09-03T07:27:57.712 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:27:57.713 DEBUG:boto:Data: 2022-09-03T07:27:57.713 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.717 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.718 DEBUG:boto:Params: {} 2022-09-03T07:27:57.719 DEBUG:boto:Token: None 2022-09-03T07:27:57.721 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj11 2022-09-03T07:27:57.722 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7fcXAmaSng+zghzxHgrucDJ0PfI= 2022-09-03T07:27:57.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7fcXAmaSng+zghzxHgrucDJ0PfI=', 'Content-Length': '0'} 2022-09-03T07:27:57.727 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a4747ef126a74b0-00631301fd-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.728 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.729 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.730 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:27:57.731 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:27:57.732 DEBUG:boto:Method: GET 2022-09-03T07:27:57.733 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:27:57.734 DEBUG:boto:Data: 2022-09-03T07:27:57.735 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.736 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.737 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.737 DEBUG:boto:Params: {} 2022-09-03T07:27:57.738 DEBUG:boto:Token: None 2022-09-03T07:27:57.739 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/ 2022-09-03T07:27:57.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI= 2022-09-03T07:27:57.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI=', 'Content-Length': '0'} 2022-09-03T07:27:57.744 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cae06450f835d300-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.746 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.748 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.749 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.750 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:27:57.751 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:27:57.752 DEBUG:boto:Method: GET 2022-09-03T07:27:57.752 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:27:57.753 DEBUG:boto:Data: 2022-09-03T07:27:57.755 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.761 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.762 DEBUG:boto:Params: {} 2022-09-03T07:27:57.763 DEBUG:boto:Token: None 2022-09-03T07:27:57.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/ 2022-09-03T07:27:57.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI= 2022-09-03T07:27:57.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI=', 'Content-Length': '0'} 2022-09-03T07:27:57.778 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b4451c84f995138b-00631301fd-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.780 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.782 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:27:57.783 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:57.784 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:57.785 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.786 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.787 DEBUG:boto:Method: GET 2022-09-03T07:27:57.788 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.789 DEBUG:boto:Data: 2022-09-03T07:27:57.790 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.790 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.791 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.791 DEBUG:boto:Params: {} 2022-09-03T07:27:57.792 DEBUG:boto:Token: None 2022-09-03T07:27:57.793 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iQEJOPLdgEo7wMdyasoBWb3iE+Y= 2022-09-03T07:27:57.794 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iQEJOPLdgEo7wMdyasoBWb3iE+Y=', 'Content-Length': '0'} 2022-09-03T07:27:57.796 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4e9bfd548be28ea-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.797 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:57.798 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:57.798 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.799 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.800 DEBUG:boto:Method: GET 2022-09-03T07:27:57.801 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.801 DEBUG:boto:Data: 2022-09-03T07:27:57.802 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.802 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.803 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.803 DEBUG:boto:Params: {} 2022-09-03T07:27:57.804 DEBUG:boto:Token: None 2022-09-03T07:27:57.805 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iQEJOPLdgEo7wMdyasoBWb3iE+Y= 2022-09-03T07:27:57.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iQEJOPLdgEo7wMdyasoBWb3iE+Y=', 'Content-Length': '0'} 2022-09-03T07:27:57.809 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cae74f3084661622-00631301fd-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.811 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:57.812 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:57.813 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.814 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.815 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.815 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.816 DEBUG:boto:Data: 2022-09-03T07:27:57.818 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.819 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.820 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.822 DEBUG:boto:Params: {} 2022-09-03T07:27:57.823 DEBUG:boto:Token: None 2022-09-03T07:27:57.824 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gNo69wJxaLqAwLqBNf+sjm2KJPI= 2022-09-03T07:27:57.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gNo69wJxaLqAwLqBNf+sjm2KJPI=', 'Content-Length': '0'} 2022-09-03T07:27:57.829 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d9082d446b8a9268-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.830 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:57.831 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:57.831 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.832 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.832 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.833 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.833 DEBUG:boto:Data: 2022-09-03T07:27:57.834 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.835 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.836 DEBUG:boto:Params: {} 2022-09-03T07:27:57.836 DEBUG:boto:Token: None 2022-09-03T07:27:57.837 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gNo69wJxaLqAwLqBNf+sjm2KJPI= 2022-09-03T07:27:57.838 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gNo69wJxaLqAwLqBNf+sjm2KJPI=', 'Content-Length': '0'} 2022-09-03T07:27:57.841 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a03f7119c9396f6e-00631301fd-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.842 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:27:57.843 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:57.844 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:57.844 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.845 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.845 DEBUG:boto:Method: GET 2022-09-03T07:27:57.846 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.846 DEBUG:boto:Data: 2022-09-03T07:27:57.847 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.847 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.848 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.849 DEBUG:boto:Params: {} 2022-09-03T07:27:57.850 DEBUG:boto:Token: None 2022-09-03T07:27:57.858 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iQEJOPLdgEo7wMdyasoBWb3iE+Y= 2022-09-03T07:27:57.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iQEJOPLdgEo7wMdyasoBWb3iE+Y=', 'Content-Length': '0'} 2022-09-03T07:27:57.865 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000059cf495708741add-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.868 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:57.869 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:57.870 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.871 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.872 DEBUG:boto:Method: GET 2022-09-03T07:27:57.873 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.874 DEBUG:boto:Data: 2022-09-03T07:27:57.880 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.882 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.883 DEBUG:boto:Params: {} 2022-09-03T07:27:57.885 DEBUG:boto:Token: None 2022-09-03T07:27:57.886 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:27:57.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iQEJOPLdgEo7wMdyasoBWb3iE+Y= 2022-09-03T07:27:57.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iQEJOPLdgEo7wMdyasoBWb3iE+Y=', 'Content-Length': '0'} 2022-09-03T07:27:57.891 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b76eb286ab769ae-00631301fd-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.893 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.894 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.895 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:27:57.896 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:27:57.897 DEBUG:boto:Method: GET 2022-09-03T07:27:57.898 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:27:57.899 DEBUG:boto:Data: 2022-09-03T07:27:57.900 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.902 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.903 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.904 DEBUG:boto:Params: {} 2022-09-03T07:27:57.905 DEBUG:boto:Token: None 2022-09-03T07:27:57.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/ 2022-09-03T07:27:57.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI= 2022-09-03T07:27:57.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI=', 'Content-Length': '0'} 2022-09-03T07:27:57.927 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3d8f9dd4497fc51-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.928 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.930 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:57.932 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:57.934 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:27:57.935 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:27:57.936 DEBUG:boto:Method: GET 2022-09-03T07:27:57.937 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:27:57.938 DEBUG:boto:Data: 2022-09-03T07:27:57.939 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.940 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.941 DEBUG:boto:Params: {} 2022-09-03T07:27:57.943 DEBUG:boto:Token: None 2022-09-03T07:27:57.944 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/ 2022-09-03T07:27:57.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI= 2022-09-03T07:27:57.945 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XEVcbk8y232uTdGQYweGOVbtOHI=', 'Content-Length': '0'} 2022-09-03T07:27:57.950 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fad513420a762a80-00631301fd-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.951 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:57.952 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:57.952 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:57.954 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.955 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:27:57.956 DEBUG:boto:Data: 2022-09-03T07:27:57.957 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.960 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:57.961 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.962 DEBUG:boto:Params: {} 2022-09-03T07:27:57.963 DEBUG:boto:Token: None 2022-09-03T07:27:57.964 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj115 2022-09-03T07:27:57.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYfNEbR6IgMvO+qQ6mI1G2vX52A= 2022-09-03T07:27:57.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYfNEbR6IgMvO+qQ6mI1G2vX52A=', 'Content-Length': '0'} 2022-09-03T07:27:57.968 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3771d2c5460003b-00631301fd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.971 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:57.972 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:57.972 DEBUG:boto:Method: HEAD 2022-09-03T07:27:57.973 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:27:57.974 DEBUG:boto:Data: 2022-09-03T07:27:57.975 DEBUG:boto:Headers: {} 2022-09-03T07:27:57.976 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:57.977 DEBUG:boto:Port: 8000 2022-09-03T07:27:57.978 DEBUG:boto:Params: {} 2022-09-03T07:27:57.979 DEBUG:boto:Token: None 2022-09-03T07:27:57.980 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:57 GMT /avysmw-75/obj115 2022-09-03T07:27:57.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DYfNEbR6IgMvO+qQ6mI1G2vX52A= 2022-09-03T07:27:57.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DYfNEbR6IgMvO+qQ6mI1G2vX52A=', 'Content-Length': '0'} 2022-09-03T07:27:57.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b87ad7cc2ff28f6-00631301fd-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:57.994 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:27:57.996 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:57.997 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:57.997 DEBUG:boto:Method: GET 2022-09-03T07:27:57.999 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:27:58.006 DEBUG:boto:Data: 2022-09-03T07:27:58.009 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.010 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.011 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.012 DEBUG:boto:Params: {} 2022-09-03T07:27:58.012 DEBUG:boto:Token: None 2022-09-03T07:27:58.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj115 2022-09-03T07:27:58.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0HJe+eMtUKLb0pYE9Gt1NEXPe7s= 2022-09-03T07:27:58.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0HJe+eMtUKLb0pYE9Gt1NEXPe7s=', 'Content-Length': '0'} 2022-09-03T07:27:58.021 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000dba81eae8651ea9-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.023 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:27:58.025 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:27:58.026 DEBUG:boto:Method: GET 2022-09-03T07:27:58.027 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:27:58.028 DEBUG:boto:Data: 2022-09-03T07:27:58.029 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.030 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.034 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.035 DEBUG:boto:Params: {} 2022-09-03T07:27:58.036 DEBUG:boto:Token: None 2022-09-03T07:27:58.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj115 2022-09-03T07:27:58.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0HJe+eMtUKLb0pYE9Gt1NEXPe7s= 2022-09-03T07:27:58.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0HJe+eMtUKLb0pYE9Gt1NEXPe7s=', 'Content-Length': '0'} 2022-09-03T07:27:58.043 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000859a9e253da0fa47-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.044 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.045 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.046 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:27:58.047 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:27:58.048 DEBUG:boto:Method: GET 2022-09-03T07:27:58.049 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:27:58.050 DEBUG:boto:Data: 2022-09-03T07:27:58.051 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.052 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.053 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.055 DEBUG:boto:Params: {} 2022-09-03T07:27:58.057 DEBUG:boto:Token: None 2022-09-03T07:27:58.058 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.063 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000097c8d05fa2b211d-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.106 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.108 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.109 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.109 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:27:58.110 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:27:58.110 DEBUG:boto:Method: GET 2022-09-03T07:27:58.111 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:27:58.113 DEBUG:boto:Data: 2022-09-03T07:27:58.114 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.115 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.115 DEBUG:boto:Params: {} 2022-09-03T07:27:58.116 DEBUG:boto:Token: None 2022-09-03T07:27:58.116 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.122 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063f74828ef6943fc-00631301fe-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.124 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.125 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:27:58.126 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.127 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.127 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.128 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.132 DEBUG:boto:Method: GET 2022-09-03T07:27:58.133 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.134 DEBUG:boto:Data: 2022-09-03T07:27:58.134 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.136 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.136 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.137 DEBUG:boto:Params: {} 2022-09-03T07:27:58.137 DEBUG:boto:Token: None 2022-09-03T07:27:58.138 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2nvvUT8fZPSj0ty4dMouWnABtp8= 2022-09-03T07:27:58.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2nvvUT8fZPSj0ty4dMouWnABtp8=', 'Content-Length': '0'} 2022-09-03T07:27:58.142 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000323eff8f42fd04d1-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.143 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.144 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.144 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.145 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.145 DEBUG:boto:Method: GET 2022-09-03T07:27:58.146 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.147 DEBUG:boto:Data: 2022-09-03T07:27:58.147 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.148 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.148 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.150 DEBUG:boto:Params: {} 2022-09-03T07:27:58.151 DEBUG:boto:Token: None 2022-09-03T07:27:58.151 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.152 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2nvvUT8fZPSj0ty4dMouWnABtp8= 2022-09-03T07:27:58.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2nvvUT8fZPSj0ty4dMouWnABtp8=', 'Content-Length': '0'} 2022-09-03T07:27:58.156 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027736ead7901726a-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.199 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.200 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.201 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.202 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.203 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.203 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.204 DEBUG:boto:Data: 2022-09-03T07:27:58.205 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.206 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.207 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.208 DEBUG:boto:Params: {} 2022-09-03T07:27:58.210 DEBUG:boto:Token: None 2022-09-03T07:27:58.211 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vc6UpcDkbwrwRnODjCoCtsJR6SY= 2022-09-03T07:27:58.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vc6UpcDkbwrwRnODjCoCtsJR6SY=', 'Content-Length': '0'} 2022-09-03T07:27:58.217 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009ecce3fa3fe58457-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.218 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.220 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.220 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.221 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.222 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.222 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.223 DEBUG:boto:Data: 2022-09-03T07:27:58.223 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.224 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.224 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.225 DEBUG:boto:Params: {} 2022-09-03T07:27:58.225 DEBUG:boto:Token: None 2022-09-03T07:27:58.226 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.227 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vc6UpcDkbwrwRnODjCoCtsJR6SY= 2022-09-03T07:27:58.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vc6UpcDkbwrwRnODjCoCtsJR6SY=', 'Content-Length': '0'} 2022-09-03T07:27:58.230 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ac177de5c453790-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.233 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:27:58.233 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.234 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.235 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.236 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.236 DEBUG:boto:Method: GET 2022-09-03T07:27:58.237 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.238 DEBUG:boto:Data: 2022-09-03T07:27:58.239 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.239 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.240 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.240 DEBUG:boto:Params: {} 2022-09-03T07:27:58.241 DEBUG:boto:Token: None 2022-09-03T07:27:58.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2nvvUT8fZPSj0ty4dMouWnABtp8= 2022-09-03T07:27:58.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2nvvUT8fZPSj0ty4dMouWnABtp8=', 'Content-Length': '0'} 2022-09-03T07:27:58.247 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006fda12846c84e2c6-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.249 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.250 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.251 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.252 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.253 DEBUG:boto:Method: GET 2022-09-03T07:27:58.254 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.255 DEBUG:boto:Data: 2022-09-03T07:27:58.256 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.256 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.257 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.258 DEBUG:boto:Params: {} 2022-09-03T07:27:58.259 DEBUG:boto:Token: None 2022-09-03T07:27:58.260 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:27:58.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2nvvUT8fZPSj0ty4dMouWnABtp8= 2022-09-03T07:27:58.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2nvvUT8fZPSj0ty4dMouWnABtp8=', 'Content-Length': '0'} 2022-09-03T07:27:58.267 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0adb91417f011e8-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.269 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.270 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.271 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:27:58.272 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:27:58.273 DEBUG:boto:Method: GET 2022-09-03T07:27:58.274 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:27:58.275 DEBUG:boto:Data: 2022-09-03T07:27:58.276 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.277 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.279 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.280 DEBUG:boto:Params: {} 2022-09-03T07:27:58.280 DEBUG:boto:Token: None 2022-09-03T07:27:58.281 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.285 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008bf55fac84e54877-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.286 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.287 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.288 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.289 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:27:58.289 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:27:58.290 DEBUG:boto:Method: GET 2022-09-03T07:27:58.291 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:27:58.292 DEBUG:boto:Data: 2022-09-03T07:27:58.293 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.293 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.294 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.294 DEBUG:boto:Params: {} 2022-09-03T07:27:58.295 DEBUG:boto:Token: None 2022-09-03T07:27:58.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.297 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000782f2252dc322883-00631301fe-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.302 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.303 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.304 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.304 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.305 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:27:58.306 DEBUG:boto:Data: 2022-09-03T07:27:58.306 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.307 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.307 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.308 DEBUG:boto:Params: {} 2022-09-03T07:27:58.309 DEBUG:boto:Token: None 2022-09-03T07:27:58.309 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116 2022-09-03T07:27:58.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FU3R3XqateyuenrW7Vm5pQkXFQk= 2022-09-03T07:27:58.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FU3R3XqateyuenrW7Vm5pQkXFQk=', 'Content-Length': '0'} 2022-09-03T07:27:58.313 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000014e938ac636afe76-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.314 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.314 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.315 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.316 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:27:58.317 DEBUG:boto:Data: 2022-09-03T07:27:58.318 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.319 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.320 DEBUG:boto:Params: {} 2022-09-03T07:27:58.321 DEBUG:boto:Token: None 2022-09-03T07:27:58.321 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116 2022-09-03T07:27:58.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FU3R3XqateyuenrW7Vm5pQkXFQk= 2022-09-03T07:27:58.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FU3R3XqateyuenrW7Vm5pQkXFQk=', 'Content-Length': '0'} 2022-09-03T07:27:58.326 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ec7307c7365735c-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.327 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:27:58.327 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.328 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.329 DEBUG:boto:Method: GET 2022-09-03T07:27:58.329 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:27:58.330 DEBUG:boto:Data: 2022-09-03T07:27:58.332 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.333 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.334 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.335 DEBUG:boto:Params: {} 2022-09-03T07:27:58.336 DEBUG:boto:Token: None 2022-09-03T07:27:58.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116 2022-09-03T07:27:58.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:meZTJlYdM48c4r6xSkbltgS66fs= 2022-09-03T07:27:58.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:meZTJlYdM48c4r6xSkbltgS66fs=', 'Content-Length': '0'} 2022-09-03T07:27:58.343 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ae3b376aaa7d092-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.344 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:27:58.345 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:27:58.346 DEBUG:boto:Method: GET 2022-09-03T07:27:58.346 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:27:58.348 DEBUG:boto:Data: 2022-09-03T07:27:58.349 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.350 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.350 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.351 DEBUG:boto:Params: {} 2022-09-03T07:27:58.352 DEBUG:boto:Token: None 2022-09-03T07:27:58.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj116 2022-09-03T07:27:58.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:meZTJlYdM48c4r6xSkbltgS66fs= 2022-09-03T07:27:58.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:meZTJlYdM48c4r6xSkbltgS66fs=', 'Content-Length': '0'} 2022-09-03T07:27:58.359 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d914c82c62941f9-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.360 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.361 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.363 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:27:58.364 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:27:58.365 DEBUG:boto:Method: GET 2022-09-03T07:27:58.366 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:27:58.367 DEBUG:boto:Data: 2022-09-03T07:27:58.368 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.370 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.371 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.372 DEBUG:boto:Params: {} 2022-09-03T07:27:58.375 DEBUG:boto:Token: None 2022-09-03T07:27:58.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.387 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b26768f7fc9e03df-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.388 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.389 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.391 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.392 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:27:58.393 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:27:58.393 DEBUG:boto:Method: GET 2022-09-03T07:27:58.394 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:27:58.395 DEBUG:boto:Data: 2022-09-03T07:27:58.395 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.396 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.397 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.398 DEBUG:boto:Params: {} 2022-09-03T07:27:58.398 DEBUG:boto:Token: None 2022-09-03T07:27:58.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.407 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7178176f40e1a64-00631301fe-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.450 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.452 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:27:58.454 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.455 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.456 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.458 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.459 DEBUG:boto:Method: GET 2022-09-03T07:27:58.460 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.461 DEBUG:boto:Data: 2022-09-03T07:27:58.463 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.464 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.465 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.466 DEBUG:boto:Params: {} 2022-09-03T07:27:58.467 DEBUG:boto:Token: None 2022-09-03T07:27:58.468 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aWCF9aZCATUkPhz4GgdLwW1oTtQ= 2022-09-03T07:27:58.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aWCF9aZCATUkPhz4GgdLwW1oTtQ=', 'Content-Length': '0'} 2022-09-03T07:27:58.476 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe6b2a7261b6b3b3-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.477 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.479 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.480 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.481 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.482 DEBUG:boto:Method: GET 2022-09-03T07:27:58.483 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.483 DEBUG:boto:Data: 2022-09-03T07:27:58.484 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.485 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.485 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.486 DEBUG:boto:Params: {} 2022-09-03T07:27:58.487 DEBUG:boto:Token: None 2022-09-03T07:27:58.488 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aWCF9aZCATUkPhz4GgdLwW1oTtQ= 2022-09-03T07:27:58.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aWCF9aZCATUkPhz4GgdLwW1oTtQ=', 'Content-Length': '0'} 2022-09-03T07:27:58.493 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4bbf7fa8e3e3832-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.494 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.495 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.496 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.496 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.497 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.497 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.498 DEBUG:boto:Data: 2022-09-03T07:27:58.499 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.500 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.500 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.501 DEBUG:boto:Params: {} 2022-09-03T07:27:58.506 DEBUG:boto:Token: None 2022-09-03T07:27:58.507 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:buiNyajpT9YfTAW1jbIZHLx5IdM= 2022-09-03T07:27:58.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:buiNyajpT9YfTAW1jbIZHLx5IdM=', 'Content-Length': '0'} 2022-09-03T07:27:58.513 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a12fe2ae18e27e2-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.515 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.516 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.517 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.519 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.523 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.524 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.526 DEBUG:boto:Data: 2022-09-03T07:27:58.527 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.528 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.529 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.530 DEBUG:boto:Params: {} 2022-09-03T07:27:58.531 DEBUG:boto:Token: None 2022-09-03T07:27:58.533 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:buiNyajpT9YfTAW1jbIZHLx5IdM= 2022-09-03T07:27:58.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:buiNyajpT9YfTAW1jbIZHLx5IdM=', 'Content-Length': '0'} 2022-09-03T07:27:58.542 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d981d23a4e29ae2-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.549 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:27:58.550 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.551 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.552 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.552 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.553 DEBUG:boto:Method: GET 2022-09-03T07:27:58.554 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.555 DEBUG:boto:Data: 2022-09-03T07:27:58.556 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.556 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.557 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.558 DEBUG:boto:Params: {} 2022-09-03T07:27:58.559 DEBUG:boto:Token: None 2022-09-03T07:27:58.560 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aWCF9aZCATUkPhz4GgdLwW1oTtQ= 2022-09-03T07:27:58.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aWCF9aZCATUkPhz4GgdLwW1oTtQ=', 'Content-Length': '0'} 2022-09-03T07:27:58.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000290888373a2c3248-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.565 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.566 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.567 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.567 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.568 DEBUG:boto:Method: GET 2022-09-03T07:27:58.569 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.569 DEBUG:boto:Data: 2022-09-03T07:27:58.570 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.571 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.571 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.572 DEBUG:boto:Params: {} 2022-09-03T07:27:58.573 DEBUG:boto:Token: None 2022-09-03T07:27:58.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:27:58.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aWCF9aZCATUkPhz4GgdLwW1oTtQ= 2022-09-03T07:27:58.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aWCF9aZCATUkPhz4GgdLwW1oTtQ=', 'Content-Length': '0'} 2022-09-03T07:27:58.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b99f9ae8d46acdf1-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.579 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.581 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.582 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:27:58.583 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:27:58.583 DEBUG:boto:Method: GET 2022-09-03T07:27:58.584 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:27:58.585 DEBUG:boto:Data: 2022-09-03T07:27:58.586 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.586 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.587 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.588 DEBUG:boto:Params: {} 2022-09-03T07:27:58.589 DEBUG:boto:Token: None 2022-09-03T07:27:58.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.594 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d543f305eb03bcfa-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.595 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.596 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.597 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.597 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:27:58.598 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:27:58.599 DEBUG:boto:Method: GET 2022-09-03T07:27:58.600 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:27:58.601 DEBUG:boto:Data: 2022-09-03T07:27:58.601 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.602 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.603 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.604 DEBUG:boto:Params: {} 2022-09-03T07:27:58.604 DEBUG:boto:Token: None 2022-09-03T07:27:58.605 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.607 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.618 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003abd4b23d82be3f4-00631301fe-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.619 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.620 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.621 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.622 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.622 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:27:58.623 DEBUG:boto:Data: 2022-09-03T07:27:58.624 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.624 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.626 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.627 DEBUG:boto:Params: {} 2022-09-03T07:27:58.627 DEBUG:boto:Token: None 2022-09-03T07:27:58.628 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124 2022-09-03T07:27:58.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UUXuGEL9Qb3XWD3Onan8jPNwdsw= 2022-09-03T07:27:58.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UUXuGEL9Qb3XWD3Onan8jPNwdsw=', 'Content-Length': '0'} 2022-09-03T07:27:58.631 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d2ec060ebf56d62d-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.632 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.632 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.633 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.633 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:27:58.634 DEBUG:boto:Data: 2022-09-03T07:27:58.634 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.635 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.636 DEBUG:boto:Params: {} 2022-09-03T07:27:58.637 DEBUG:boto:Token: None 2022-09-03T07:27:58.637 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124 2022-09-03T07:27:58.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UUXuGEL9Qb3XWD3Onan8jPNwdsw= 2022-09-03T07:27:58.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UUXuGEL9Qb3XWD3Onan8jPNwdsw=', 'Content-Length': '0'} 2022-09-03T07:27:58.642 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe18921405de4c8d-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.644 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:27:58.645 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.647 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.649 DEBUG:boto:Method: GET 2022-09-03T07:27:58.650 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:27:58.652 DEBUG:boto:Data: 2022-09-03T07:27:58.652 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.653 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.654 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.655 DEBUG:boto:Params: {} 2022-09-03T07:27:58.657 DEBUG:boto:Token: None 2022-09-03T07:27:58.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124 2022-09-03T07:27:58.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kl9ORONvGDOPHTcEYoCFnprARmY= 2022-09-03T07:27:58.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kl9ORONvGDOPHTcEYoCFnprARmY=', 'Content-Length': '0'} 2022-09-03T07:27:58.663 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000242ac384d669fa31-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.710 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:27:58.711 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:27:58.712 DEBUG:boto:Method: GET 2022-09-03T07:27:58.713 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:27:58.719 DEBUG:boto:Data: 2022-09-03T07:27:58.720 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.720 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.721 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.721 DEBUG:boto:Params: {} 2022-09-03T07:27:58.722 DEBUG:boto:Token: None 2022-09-03T07:27:58.723 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj124 2022-09-03T07:27:58.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kl9ORONvGDOPHTcEYoCFnprARmY= 2022-09-03T07:27:58.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kl9ORONvGDOPHTcEYoCFnprARmY=', 'Content-Length': '0'} 2022-09-03T07:27:58.727 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b26d1119635b73d-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.729 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.729 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.730 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:27:58.731 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:27:58.734 DEBUG:boto:Method: GET 2022-09-03T07:27:58.735 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:27:58.736 DEBUG:boto:Data: 2022-09-03T07:27:58.736 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.737 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.738 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.738 DEBUG:boto:Params: {} 2022-09-03T07:27:58.740 DEBUG:boto:Token: None 2022-09-03T07:27:58.740 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.744 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000389592dc65640693-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.745 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.746 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.747 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.748 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:27:58.748 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:27:58.749 DEBUG:boto:Method: GET 2022-09-03T07:27:58.750 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:27:58.751 DEBUG:boto:Data: 2022-09-03T07:27:58.751 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.752 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.752 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.753 DEBUG:boto:Params: {} 2022-09-03T07:27:58.753 DEBUG:boto:Token: None 2022-09-03T07:27:58.754 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.755 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.761 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bf0637446616d86e-00631301fe-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.804 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.805 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:27:58.807 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:58.808 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:58.809 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.810 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.811 DEBUG:boto:Method: GET 2022-09-03T07:27:58.812 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.813 DEBUG:boto:Data: 2022-09-03T07:27:58.815 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.816 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.817 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.818 DEBUG:boto:Params: {} 2022-09-03T07:27:58.819 DEBUG:boto:Token: None 2022-09-03T07:27:58.820 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kPJwqfOGMqOfSVRQB1+xkM4SR8k= 2022-09-03T07:27:58.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kPJwqfOGMqOfSVRQB1+xkM4SR8k=', 'Content-Length': '0'} 2022-09-03T07:27:58.825 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062e0605de62d6130-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.826 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:58.827 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:58.828 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.829 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.829 DEBUG:boto:Method: GET 2022-09-03T07:27:58.830 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.831 DEBUG:boto:Data: 2022-09-03T07:27:58.833 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.834 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.835 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.836 DEBUG:boto:Params: {} 2022-09-03T07:27:58.838 DEBUG:boto:Token: None 2022-09-03T07:27:58.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kPJwqfOGMqOfSVRQB1+xkM4SR8k= 2022-09-03T07:27:58.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kPJwqfOGMqOfSVRQB1+xkM4SR8k=', 'Content-Length': '0'} 2022-09-03T07:27:58.844 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9a69520687a8f1b-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.846 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:58.847 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:58.848 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.849 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.850 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.851 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.852 DEBUG:boto:Data: 2022-09-03T07:27:58.853 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.854 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.855 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.856 DEBUG:boto:Params: {} 2022-09-03T07:27:58.857 DEBUG:boto:Token: None 2022-09-03T07:27:58.858 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zaZ3FgZDPHf1gtE8hTwOHTCoksM= 2022-09-03T07:27:58.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zaZ3FgZDPHf1gtE8hTwOHTCoksM=', 'Content-Length': '0'} 2022-09-03T07:27:58.863 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039de6c7f8bd15a02-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.864 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:58.865 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:58.865 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.866 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.866 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.867 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.867 DEBUG:boto:Data: 2022-09-03T07:27:58.868 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.868 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.869 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.870 DEBUG:boto:Params: {} 2022-09-03T07:27:58.870 DEBUG:boto:Token: None 2022-09-03T07:27:58.871 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zaZ3FgZDPHf1gtE8hTwOHTCoksM= 2022-09-03T07:27:58.872 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zaZ3FgZDPHf1gtE8hTwOHTCoksM=', 'Content-Length': '0'} 2022-09-03T07:27:58.876 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006bfa5314cee1386c-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.878 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:27:58.881 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:58.883 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:58.883 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.885 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.886 DEBUG:boto:Method: GET 2022-09-03T07:27:58.887 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.887 DEBUG:boto:Data: 2022-09-03T07:27:58.888 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.889 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.892 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.893 DEBUG:boto:Params: {} 2022-09-03T07:27:58.894 DEBUG:boto:Token: None 2022-09-03T07:27:58.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kPJwqfOGMqOfSVRQB1+xkM4SR8k= 2022-09-03T07:27:58.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kPJwqfOGMqOfSVRQB1+xkM4SR8k=', 'Content-Length': '0'} 2022-09-03T07:27:58.900 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000bb811701421a80b-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.902 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:58.903 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:58.904 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.905 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.906 DEBUG:boto:Method: GET 2022-09-03T07:27:58.907 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.908 DEBUG:boto:Data: 2022-09-03T07:27:58.909 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.910 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.911 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.913 DEBUG:boto:Params: {} 2022-09-03T07:27:58.914 DEBUG:boto:Token: None 2022-09-03T07:27:58.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:27:58.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kPJwqfOGMqOfSVRQB1+xkM4SR8k= 2022-09-03T07:27:58.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kPJwqfOGMqOfSVRQB1+xkM4SR8k=', 'Content-Length': '0'} 2022-09-03T07:27:58.921 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000830b4636a6328136-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.922 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.924 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.925 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:27:58.925 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:27:58.926 DEBUG:boto:Method: GET 2022-09-03T07:27:58.926 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:27:58.927 DEBUG:boto:Data: 2022-09-03T07:27:58.927 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.928 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.929 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.929 DEBUG:boto:Params: {} 2022-09-03T07:27:58.930 DEBUG:boto:Token: None 2022-09-03T07:27:58.930 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.931 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.932 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.934 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dd0ce2097d67a679-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.935 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.935 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:58.936 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:58.937 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:27:58.937 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:27:58.938 DEBUG:boto:Method: GET 2022-09-03T07:27:58.938 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:27:58.939 DEBUG:boto:Data: 2022-09-03T07:27:58.939 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.940 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.940 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.941 DEBUG:boto:Params: {} 2022-09-03T07:27:58.941 DEBUG:boto:Token: None 2022-09-03T07:27:58.942 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/ 2022-09-03T07:27:58.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8= 2022-09-03T07:27:58.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hu7JNK6jC5y/YBKR0ZFwyZB9FL8=', 'Content-Length': '0'} 2022-09-03T07:27:58.948 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000081bd633a6a87ad62-00631301fe-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.950 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:58.951 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:58.952 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:58.953 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.954 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:27:58.956 DEBUG:boto:Data: 2022-09-03T07:27:58.956 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.957 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.957 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.958 DEBUG:boto:Params: {} 2022-09-03T07:27:58.958 DEBUG:boto:Token: None 2022-09-03T07:27:58.959 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj145 2022-09-03T07:27:58.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66djAMhSoCe2ximUFXOarAdZb+E= 2022-09-03T07:27:58.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66djAMhSoCe2ximUFXOarAdZb+E=', 'Content-Length': '0'} 2022-09-03T07:27:58.967 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b13bf056e752c122-00631301fe-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.969 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:58.970 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:58.971 DEBUG:boto:Method: HEAD 2022-09-03T07:27:58.972 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:27:58.973 DEBUG:boto:Data: 2022-09-03T07:27:58.974 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.975 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:58.976 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.977 DEBUG:boto:Params: {} 2022-09-03T07:27:58.978 DEBUG:boto:Token: None 2022-09-03T07:27:58.979 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:58 GMT /avysmw-75/obj145 2022-09-03T07:27:58.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:66djAMhSoCe2ximUFXOarAdZb+E= 2022-09-03T07:27:58.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:66djAMhSoCe2ximUFXOarAdZb+E=', 'Content-Length': '0'} 2022-09-03T07:27:58.986 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f1cf880318bb356-00631301fe-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:58.987 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:27:58.988 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:58.989 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:58.994 DEBUG:boto:Method: GET 2022-09-03T07:27:58.996 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:27:58.997 DEBUG:boto:Data: 2022-09-03T07:27:58.997 DEBUG:boto:Headers: {} 2022-09-03T07:27:58.998 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:58.998 DEBUG:boto:Port: 8000 2022-09-03T07:27:58.999 DEBUG:boto:Params: {} 2022-09-03T07:27:59.000 DEBUG:boto:Token: None 2022-09-03T07:27:59.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj145 2022-09-03T07:27:59.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SntwzHaF714zJfl4RdiNyXNdLh8= 2022-09-03T07:27:59.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SntwzHaF714zJfl4RdiNyXNdLh8=', 'Content-Length': '0'} 2022-09-03T07:27:59.004 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002108d0ae8f56431f-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.050 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:27:59.051 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:27:59.052 DEBUG:boto:Method: GET 2022-09-03T07:27:59.053 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:27:59.053 DEBUG:boto:Data: 2022-09-03T07:27:59.054 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.055 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.056 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.056 DEBUG:boto:Params: {} 2022-09-03T07:27:59.059 DEBUG:boto:Token: None 2022-09-03T07:27:59.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj145 2022-09-03T07:27:59.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SntwzHaF714zJfl4RdiNyXNdLh8= 2022-09-03T07:27:59.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SntwzHaF714zJfl4RdiNyXNdLh8=', 'Content-Length': '0'} 2022-09-03T07:27:59.066 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090619810ad08c5e8-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.067 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.068 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.068 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:27:59.069 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:27:59.069 DEBUG:boto:Method: GET 2022-09-03T07:27:59.070 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:27:59.070 DEBUG:boto:Data: 2022-09-03T07:27:59.071 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.071 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.072 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.073 DEBUG:boto:Params: {} 2022-09-03T07:27:59.073 DEBUG:boto:Token: None 2022-09-03T07:27:59.074 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.078 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1de7f6c45892518-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.079 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.080 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.080 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.081 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:27:59.082 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:27:59.082 DEBUG:boto:Method: GET 2022-09-03T07:27:59.083 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:27:59.083 DEBUG:boto:Data: 2022-09-03T07:27:59.084 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.084 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.085 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.086 DEBUG:boto:Params: {} 2022-09-03T07:27:59.086 DEBUG:boto:Token: None 2022-09-03T07:27:59.088 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.093 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002ecbb367c3b085b6-00631301ff-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.134 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.135 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:27:59.136 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.137 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.138 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.139 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.140 DEBUG:boto:Method: GET 2022-09-03T07:27:59.141 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.143 DEBUG:boto:Data: 2022-09-03T07:27:59.145 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.146 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.147 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.147 DEBUG:boto:Params: {} 2022-09-03T07:27:59.149 DEBUG:boto:Token: None 2022-09-03T07:27:59.150 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.151 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TvxjifPh33BnY50YFWtZAADmNf8= 2022-09-03T07:27:59.152 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TvxjifPh33BnY50YFWtZAADmNf8=', 'Content-Length': '0'} 2022-09-03T07:27:59.155 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b473b2455dd30de-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.156 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.157 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.157 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.158 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.159 DEBUG:boto:Method: GET 2022-09-03T07:27:59.159 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.161 DEBUG:boto:Data: 2022-09-03T07:27:59.161 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.163 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.163 DEBUG:boto:Params: {} 2022-09-03T07:27:59.164 DEBUG:boto:Token: None 2022-09-03T07:27:59.164 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TvxjifPh33BnY50YFWtZAADmNf8= 2022-09-03T07:27:59.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TvxjifPh33BnY50YFWtZAADmNf8=', 'Content-Length': '0'} 2022-09-03T07:27:59.169 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e7a4f320de1ba8a2-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.170 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.170 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.171 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.172 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.173 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.174 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.175 DEBUG:boto:Data: 2022-09-03T07:27:59.175 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.176 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.176 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.177 DEBUG:boto:Params: {} 2022-09-03T07:27:59.178 DEBUG:boto:Token: None 2022-09-03T07:27:59.178 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EClX3BnJyhJSnAxPamM6q+YjeXY= 2022-09-03T07:27:59.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EClX3BnJyhJSnAxPamM6q+YjeXY=', 'Content-Length': '0'} 2022-09-03T07:27:59.182 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc4f23346de7df9d-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.183 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.184 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.185 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.186 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.187 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.188 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.189 DEBUG:boto:Data: 2022-09-03T07:27:59.190 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.191 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.192 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.193 DEBUG:boto:Params: {} 2022-09-03T07:27:59.194 DEBUG:boto:Token: None 2022-09-03T07:27:59.195 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EClX3BnJyhJSnAxPamM6q+YjeXY= 2022-09-03T07:27:59.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EClX3BnJyhJSnAxPamM6q+YjeXY=', 'Content-Length': '0'} 2022-09-03T07:27:59.201 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000773354953920b63a-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.202 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:27:59.203 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.203 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.204 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.204 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.205 DEBUG:boto:Method: GET 2022-09-03T07:27:59.206 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.206 DEBUG:boto:Data: 2022-09-03T07:27:59.207 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.207 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.208 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.208 DEBUG:boto:Params: {} 2022-09-03T07:27:59.209 DEBUG:boto:Token: None 2022-09-03T07:27:59.210 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TvxjifPh33BnY50YFWtZAADmNf8= 2022-09-03T07:27:59.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TvxjifPh33BnY50YFWtZAADmNf8=', 'Content-Length': '0'} 2022-09-03T07:27:59.213 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f233812c5e1b925-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.214 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.215 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.215 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.216 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.216 DEBUG:boto:Method: GET 2022-09-03T07:27:59.217 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.217 DEBUG:boto:Data: 2022-09-03T07:27:59.218 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.219 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.219 DEBUG:boto:Params: {} 2022-09-03T07:27:59.220 DEBUG:boto:Token: None 2022-09-03T07:27:59.221 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:27:59.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TvxjifPh33BnY50YFWtZAADmNf8= 2022-09-03T07:27:59.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TvxjifPh33BnY50YFWtZAADmNf8=', 'Content-Length': '0'} 2022-09-03T07:27:59.224 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000289c0bcc9725bb90-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.225 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.226 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.227 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:27:59.227 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:27:59.228 DEBUG:boto:Method: GET 2022-09-03T07:27:59.229 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:27:59.229 DEBUG:boto:Data: 2022-09-03T07:27:59.230 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.230 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.231 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.231 DEBUG:boto:Params: {} 2022-09-03T07:27:59.232 DEBUG:boto:Token: None 2022-09-03T07:27:59.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.237 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bd192318063c3c17-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.238 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.239 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.240 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.240 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:27:59.241 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:27:59.241 DEBUG:boto:Method: GET 2022-09-03T07:27:59.242 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:27:59.242 DEBUG:boto:Data: 2022-09-03T07:27:59.243 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.244 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.245 DEBUG:boto:Params: {} 2022-09-03T07:27:59.245 DEBUG:boto:Token: None 2022-09-03T07:27:59.246 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.253 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f525f0c3472f709b-00631301ff-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.254 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.256 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.257 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.259 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.260 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:27:59.261 DEBUG:boto:Data: 2022-09-03T07:27:59.262 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.263 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.264 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.264 DEBUG:boto:Params: {} 2022-09-03T07:27:59.266 DEBUG:boto:Token: None 2022-09-03T07:27:59.267 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155 2022-09-03T07:27:59.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nv0eFBEV6RW2xAh9jJdxl3JuY8I= 2022-09-03T07:27:59.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nv0eFBEV6RW2xAh9jJdxl3JuY8I=', 'Content-Length': '0'} 2022-09-03T07:27:59.274 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007339b2ec553e7c4c-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.275 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.276 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.277 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.278 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:27:59.280 DEBUG:boto:Data: 2022-09-03T07:27:59.281 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.283 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.284 DEBUG:boto:Params: {} 2022-09-03T07:27:59.285 DEBUG:boto:Token: None 2022-09-03T07:27:59.286 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155 2022-09-03T07:27:59.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nv0eFBEV6RW2xAh9jJdxl3JuY8I= 2022-09-03T07:27:59.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nv0eFBEV6RW2xAh9jJdxl3JuY8I=', 'Content-Length': '0'} 2022-09-03T07:27:59.293 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003feb5d8aeae7f00a-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.294 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:27:59.295 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.296 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.297 DEBUG:boto:Method: GET 2022-09-03T07:27:59.297 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:27:59.298 DEBUG:boto:Data: 2022-09-03T07:27:59.299 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.301 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.301 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.302 DEBUG:boto:Params: {} 2022-09-03T07:27:59.303 DEBUG:boto:Token: None 2022-09-03T07:27:59.306 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155 2022-09-03T07:27:59.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8a+pHKsuJn9I6HSCS7SSQrVh/WU= 2022-09-03T07:27:59.309 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8a+pHKsuJn9I6HSCS7SSQrVh/WU=', 'Content-Length': '0'} 2022-09-03T07:27:59.312 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dafe9f0698fdfe25-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.314 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:27:59.315 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:27:59.316 DEBUG:boto:Method: GET 2022-09-03T07:27:59.317 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:27:59.318 DEBUG:boto:Data: 2022-09-03T07:27:59.319 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.321 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.322 DEBUG:boto:Params: {} 2022-09-03T07:27:59.324 DEBUG:boto:Token: None 2022-09-03T07:27:59.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj155 2022-09-03T07:27:59.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8a+pHKsuJn9I6HSCS7SSQrVh/WU= 2022-09-03T07:27:59.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8a+pHKsuJn9I6HSCS7SSQrVh/WU=', 'Content-Length': '0'} 2022-09-03T07:27:59.341 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001201c6259ce8cb75-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.380 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.381 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.382 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:27:59.391 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:27:59.391 DEBUG:boto:Method: GET 2022-09-03T07:27:59.392 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:27:59.393 DEBUG:boto:Data: 2022-09-03T07:27:59.394 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.395 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.396 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.397 DEBUG:boto:Params: {} 2022-09-03T07:27:59.398 DEBUG:boto:Token: None 2022-09-03T07:27:59.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.405 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000010e8e418100a89ee-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.407 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.408 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.409 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.411 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:27:59.412 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:27:59.414 DEBUG:boto:Method: GET 2022-09-03T07:27:59.415 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:27:59.416 DEBUG:boto:Data: 2022-09-03T07:27:59.417 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.418 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.420 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.421 DEBUG:boto:Params: {} 2022-09-03T07:27:59.423 DEBUG:boto:Token: None 2022-09-03T07:27:59.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.430 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001d78b5a2c9d749ff-00631301ff-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.431 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.432 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:27:59.433 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.433 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.434 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.435 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.435 DEBUG:boto:Method: GET 2022-09-03T07:27:59.436 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.436 DEBUG:boto:Data: 2022-09-03T07:27:59.437 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.437 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.439 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.439 DEBUG:boto:Params: {} 2022-09-03T07:27:59.440 DEBUG:boto:Token: None 2022-09-03T07:27:59.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9UrCY3WjbFgs3HJcFGJjN4LTiF8= 2022-09-03T07:27:59.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9UrCY3WjbFgs3HJcFGJjN4LTiF8=', 'Content-Length': '0'} 2022-09-03T07:27:59.444 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fca6e4d86f797df-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.446 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.447 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.448 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.449 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.450 DEBUG:boto:Method: GET 2022-09-03T07:27:59.451 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.451 DEBUG:boto:Data: 2022-09-03T07:27:59.452 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.454 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.455 DEBUG:boto:Params: {} 2022-09-03T07:27:59.456 DEBUG:boto:Token: None 2022-09-03T07:27:59.458 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.460 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9UrCY3WjbFgs3HJcFGJjN4LTiF8= 2022-09-03T07:27:59.461 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9UrCY3WjbFgs3HJcFGJjN4LTiF8=', 'Content-Length': '0'} 2022-09-03T07:27:59.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aca335d4ebbc4cd7-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.469 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.470 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.471 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.472 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.473 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.474 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.475 DEBUG:boto:Data: 2022-09-03T07:27:59.476 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.477 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.478 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.479 DEBUG:boto:Params: {} 2022-09-03T07:27:59.480 DEBUG:boto:Token: None 2022-09-03T07:27:59.481 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J47TGcYOwTab6Ayf6mmK80iAYKE= 2022-09-03T07:27:59.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J47TGcYOwTab6Ayf6mmK80iAYKE=', 'Content-Length': '0'} 2022-09-03T07:27:59.487 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d55faf46245ce796-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.488 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.489 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.490 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.492 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.493 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.496 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.498 DEBUG:boto:Data: 2022-09-03T07:27:59.499 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.501 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.510 DEBUG:boto:Params: {} 2022-09-03T07:27:59.514 DEBUG:boto:Token: None 2022-09-03T07:27:59.516 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J47TGcYOwTab6Ayf6mmK80iAYKE= 2022-09-03T07:27:59.520 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J47TGcYOwTab6Ayf6mmK80iAYKE=', 'Content-Length': '0'} 2022-09-03T07:27:59.522 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000409f6ed543ce5173-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.524 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:27:59.526 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.527 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.529 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.530 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.531 DEBUG:boto:Method: GET 2022-09-03T07:27:59.532 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.533 DEBUG:boto:Data: 2022-09-03T07:27:59.534 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.535 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.545 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.547 DEBUG:boto:Params: {} 2022-09-03T07:27:59.548 DEBUG:boto:Token: None 2022-09-03T07:27:59.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9UrCY3WjbFgs3HJcFGJjN4LTiF8= 2022-09-03T07:27:59.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9UrCY3WjbFgs3HJcFGJjN4LTiF8=', 'Content-Length': '0'} 2022-09-03T07:27:59.554 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca4512a7d0867ae1-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.556 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.557 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.558 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.559 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.560 DEBUG:boto:Method: GET 2022-09-03T07:27:59.561 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.562 DEBUG:boto:Data: 2022-09-03T07:27:59.563 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.565 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.567 DEBUG:boto:Params: {} 2022-09-03T07:27:59.568 DEBUG:boto:Token: None 2022-09-03T07:27:59.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:27:59.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9UrCY3WjbFgs3HJcFGJjN4LTiF8= 2022-09-03T07:27:59.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9UrCY3WjbFgs3HJcFGJjN4LTiF8=', 'Content-Length': '0'} 2022-09-03T07:27:59.576 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a7381b4e0c3cc29-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.577 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.578 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.579 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:27:59.580 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:27:59.582 DEBUG:boto:Method: GET 2022-09-03T07:27:59.583 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:27:59.584 DEBUG:boto:Data: 2022-09-03T07:27:59.585 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.586 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.588 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.589 DEBUG:boto:Params: {} 2022-09-03T07:27:59.590 DEBUG:boto:Token: None 2022-09-03T07:27:59.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.595 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.600 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005087b50738de4238-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.601 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.602 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.603 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.604 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:27:59.605 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:27:59.606 DEBUG:boto:Method: GET 2022-09-03T07:27:59.607 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:27:59.607 DEBUG:boto:Data: 2022-09-03T07:27:59.608 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.609 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.611 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.612 DEBUG:boto:Params: {} 2022-09-03T07:27:59.613 DEBUG:boto:Token: None 2022-09-03T07:27:59.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.622 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003926439b99a8db5d-00631301ff-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.624 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.626 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.627 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.628 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.628 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:27:59.629 DEBUG:boto:Data: 2022-09-03T07:27:59.630 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.631 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.632 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.633 DEBUG:boto:Params: {} 2022-09-03T07:27:59.634 DEBUG:boto:Token: None 2022-09-03T07:27:59.635 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17 2022-09-03T07:27:59.636 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1fqY5+YpdqocL/lgvbvmY6vBkGc= 2022-09-03T07:27:59.637 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1fqY5+YpdqocL/lgvbvmY6vBkGc=', 'Content-Length': '0'} 2022-09-03T07:27:59.639 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9495c02794c9b0e-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.640 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.641 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.642 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.643 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:27:59.644 DEBUG:boto:Data: 2022-09-03T07:27:59.645 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.647 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.648 DEBUG:boto:Params: {} 2022-09-03T07:27:59.648 DEBUG:boto:Token: None 2022-09-03T07:27:59.649 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17 2022-09-03T07:27:59.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1fqY5+YpdqocL/lgvbvmY6vBkGc= 2022-09-03T07:27:59.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1fqY5+YpdqocL/lgvbvmY6vBkGc=', 'Content-Length': '0'} 2022-09-03T07:27:59.654 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000dd54976a43f2f97-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.655 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:27:59.656 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.657 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.658 DEBUG:boto:Method: GET 2022-09-03T07:27:59.659 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:27:59.660 DEBUG:boto:Data: 2022-09-03T07:27:59.661 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.662 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.663 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.663 DEBUG:boto:Params: {} 2022-09-03T07:27:59.664 DEBUG:boto:Token: None 2022-09-03T07:27:59.665 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17 2022-09-03T07:27:59.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q0f7Mx/sXRoXIiZiFYUbC1wjEco= 2022-09-03T07:27:59.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q0f7Mx/sXRoXIiZiFYUbC1wjEco=', 'Content-Length': '0'} 2022-09-03T07:27:59.669 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097fed01ed42a6d69-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.670 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:27:59.670 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:27:59.671 DEBUG:boto:Method: GET 2022-09-03T07:27:59.672 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:27:59.673 DEBUG:boto:Data: 2022-09-03T07:27:59.674 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.675 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.676 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.676 DEBUG:boto:Params: {} 2022-09-03T07:27:59.677 DEBUG:boto:Token: None 2022-09-03T07:27:59.678 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj17 2022-09-03T07:27:59.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q0f7Mx/sXRoXIiZiFYUbC1wjEco= 2022-09-03T07:27:59.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q0f7Mx/sXRoXIiZiFYUbC1wjEco=', 'Content-Length': '0'} 2022-09-03T07:27:59.685 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000772e9a884c8e4773-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.687 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.688 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.689 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:27:59.690 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:27:59.691 DEBUG:boto:Method: GET 2022-09-03T07:27:59.692 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:27:59.693 DEBUG:boto:Data: 2022-09-03T07:27:59.694 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.695 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.697 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.698 DEBUG:boto:Params: {} 2022-09-03T07:27:59.699 DEBUG:boto:Token: None 2022-09-03T07:27:59.701 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.707 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090c343c6c422cc8e-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.708 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.709 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.710 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.710 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:27:59.711 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:27:59.711 DEBUG:boto:Method: GET 2022-09-03T07:27:59.712 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:27:59.712 DEBUG:boto:Data: 2022-09-03T07:27:59.713 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.713 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.714 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.715 DEBUG:boto:Params: {} 2022-09-03T07:27:59.716 DEBUG:boto:Token: None 2022-09-03T07:27:59.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.722 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000080fed51c30b93df3-00631301ff-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.766 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.768 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:27:59.769 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:27:59.770 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:27:59.771 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.772 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.774 DEBUG:boto:Method: GET 2022-09-03T07:27:59.775 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.777 DEBUG:boto:Data: 2022-09-03T07:27:59.778 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.779 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.779 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.780 DEBUG:boto:Params: {} 2022-09-03T07:27:59.781 DEBUG:boto:Token: None 2022-09-03T07:27:59.781 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jauz0hgcoXIH59qZvwBiQWW/rNs= 2022-09-03T07:27:59.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jauz0hgcoXIH59qZvwBiQWW/rNs=', 'Content-Length': '0'} 2022-09-03T07:27:59.787 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003936cc0e944325ac-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.788 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:27:59.789 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:27:59.790 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.790 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.791 DEBUG:boto:Method: GET 2022-09-03T07:27:59.792 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.793 DEBUG:boto:Data: 2022-09-03T07:27:59.794 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.796 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.798 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.799 DEBUG:boto:Params: {} 2022-09-03T07:27:59.800 DEBUG:boto:Token: None 2022-09-03T07:27:59.801 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jauz0hgcoXIH59qZvwBiQWW/rNs= 2022-09-03T07:27:59.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jauz0hgcoXIH59qZvwBiQWW/rNs=', 'Content-Length': '0'} 2022-09-03T07:27:59.807 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2fbaa4ce4e713fa-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.808 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:27:59.808 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:27:59.809 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.810 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.810 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.811 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.812 DEBUG:boto:Data: 2022-09-03T07:27:59.813 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.813 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.814 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.815 DEBUG:boto:Params: {} 2022-09-03T07:27:59.819 DEBUG:boto:Token: None 2022-09-03T07:27:59.820 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RkNU/JQ+cplFjH12Yrc76uZmHeg= 2022-09-03T07:27:59.821 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RkNU/JQ+cplFjH12Yrc76uZmHeg=', 'Content-Length': '0'} 2022-09-03T07:27:59.825 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6c7717964bbc8e3-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.826 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:27:59.826 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:27:59.827 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.827 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.828 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.828 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.829 DEBUG:boto:Data: 2022-09-03T07:27:59.829 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.830 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.831 DEBUG:boto:Params: {} 2022-09-03T07:27:59.832 DEBUG:boto:Token: None 2022-09-03T07:27:59.832 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RkNU/JQ+cplFjH12Yrc76uZmHeg= 2022-09-03T07:27:59.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RkNU/JQ+cplFjH12Yrc76uZmHeg=', 'Content-Length': '0'} 2022-09-03T07:27:59.836 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d6aac3b76f991b4-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.837 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:27:59.838 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:27:59.838 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:27:59.839 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.839 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.840 DEBUG:boto:Method: GET 2022-09-03T07:27:59.840 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.841 DEBUG:boto:Data: 2022-09-03T07:27:59.841 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.842 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.843 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.843 DEBUG:boto:Params: {} 2022-09-03T07:27:59.844 DEBUG:boto:Token: None 2022-09-03T07:27:59.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jauz0hgcoXIH59qZvwBiQWW/rNs= 2022-09-03T07:27:59.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jauz0hgcoXIH59qZvwBiQWW/rNs=', 'Content-Length': '0'} 2022-09-03T07:27:59.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e448e234c8d5fcf8-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.894 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:27:59.895 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:27:59.896 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.896 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.897 DEBUG:boto:Method: GET 2022-09-03T07:27:59.897 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.898 DEBUG:boto:Data: 2022-09-03T07:27:59.898 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.900 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.900 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.901 DEBUG:boto:Params: {} 2022-09-03T07:27:59.902 DEBUG:boto:Token: None 2022-09-03T07:27:59.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:27:59.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jauz0hgcoXIH59qZvwBiQWW/rNs= 2022-09-03T07:27:59.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jauz0hgcoXIH59qZvwBiQWW/rNs=', 'Content-Length': '0'} 2022-09-03T07:27:59.908 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b8089ec59bc93c8-00631301ff-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.909 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.909 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.910 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:27:59.911 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:27:59.912 DEBUG:boto:Method: GET 2022-09-03T07:27:59.912 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:27:59.913 DEBUG:boto:Data: 2022-09-03T07:27:59.913 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.914 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:27:59.915 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.915 DEBUG:boto:Params: {} 2022-09-03T07:27:59.916 DEBUG:boto:Token: None 2022-09-03T07:27:59.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.917 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.927 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007dbbba5d68a380bd-00631301ff-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.928 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.929 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:27:59.930 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:27:59.931 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:27:59.931 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:27:59.932 DEBUG:boto:Method: GET 2022-09-03T07:27:59.932 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:27:59.933 DEBUG:boto:Data: 2022-09-03T07:27:59.933 DEBUG:boto:Headers: {} 2022-09-03T07:27:59.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:27:59.934 DEBUG:boto:Port: 8000 2022-09-03T07:27:59.936 DEBUG:boto:Params: {} 2022-09-03T07:27:59.937 DEBUG:boto:Token: None 2022-09-03T07:27:59.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:27:59 GMT /avysmw-75/ 2022-09-03T07:27:59.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY= 2022-09-03T07:27:59.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:27:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QGiIVpT0aRse2xjZHLWqlgvgYWY=', 'Content-Length': '0'} 2022-09-03T07:27:59.945 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cabea0efa778742e-00631301ff-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:27:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:27:59.986 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:27:59.988 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:27:59.989 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:27:59.994 DEBUG:boto:Method: HEAD 2022-09-03T07:27:59.996 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:27:59.998 DEBUG:boto:Data: 2022-09-03T07:28:00.000 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.001 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.002 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.003 DEBUG:boto:Params: {} 2022-09-03T07:28:00.004 DEBUG:boto:Token: None 2022-09-03T07:28:00.005 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj19 2022-09-03T07:28:00.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hrusfD/JIegrRMNEFfvhMLqRcb0= 2022-09-03T07:28:00.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hrusfD/JIegrRMNEFfvhMLqRcb0=', 'Content-Length': '0'} 2022-09-03T07:28:00.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b51b520b54e460c8-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.023 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:00.026 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:00.038 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.039 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:00.040 DEBUG:boto:Data: 2022-09-03T07:28:00.041 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.043 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.050 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.051 DEBUG:boto:Params: {} 2022-09-03T07:28:00.052 DEBUG:boto:Token: None 2022-09-03T07:28:00.053 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj19 2022-09-03T07:28:00.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hrusfD/JIegrRMNEFfvhMLqRcb0= 2022-09-03T07:28:00.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hrusfD/JIegrRMNEFfvhMLqRcb0=', 'Content-Length': '0'} 2022-09-03T07:28:00.063 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1a5b9b3a7f25263-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.064 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:28:00.065 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:00.067 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:00.068 DEBUG:boto:Method: GET 2022-09-03T07:28:00.069 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:00.070 DEBUG:boto:Data: 2022-09-03T07:28:00.071 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.072 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.073 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.074 DEBUG:boto:Params: {} 2022-09-03T07:28:00.075 DEBUG:boto:Token: None 2022-09-03T07:28:00.076 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj19 2022-09-03T07:28:00.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JaqNQsCfSVL28dj7wzBu6zneaGM= 2022-09-03T07:28:00.079 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JaqNQsCfSVL28dj7wzBu6zneaGM=', 'Content-Length': '0'} 2022-09-03T07:28:00.082 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f3fc0f9882ecc81-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.083 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:00.084 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:00.085 DEBUG:boto:Method: GET 2022-09-03T07:28:00.086 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:00.087 DEBUG:boto:Data: 2022-09-03T07:28:00.088 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.089 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.090 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.091 DEBUG:boto:Params: {} 2022-09-03T07:28:00.092 DEBUG:boto:Token: None 2022-09-03T07:28:00.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj19 2022-09-03T07:28:00.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JaqNQsCfSVL28dj7wzBu6zneaGM= 2022-09-03T07:28:00.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JaqNQsCfSVL28dj7wzBu6zneaGM=', 'Content-Length': '0'} 2022-09-03T07:28:00.097 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b0d7e441985a5d4-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.099 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.100 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.102 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:00.103 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:00.103 DEBUG:boto:Method: GET 2022-09-03T07:28:00.104 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:00.104 DEBUG:boto:Data: 2022-09-03T07:28:00.105 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.106 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.106 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.107 DEBUG:boto:Params: {} 2022-09-03T07:28:00.107 DEBUG:boto:Token: None 2022-09-03T07:28:00.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.118 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e34a7351cc8bc555-0063130200-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.119 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.121 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.121 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.122 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:00.122 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:00.123 DEBUG:boto:Method: GET 2022-09-03T07:28:00.124 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:00.124 DEBUG:boto:Data: 2022-09-03T07:28:00.125 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.125 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.126 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.126 DEBUG:boto:Params: {} 2022-09-03T07:28:00.127 DEBUG:boto:Token: None 2022-09-03T07:28:00.128 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cc593c4abba8ef21-0063130200-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.137 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.139 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:28:00.140 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.142 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.142 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.144 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.145 DEBUG:boto:Method: GET 2022-09-03T07:28:00.146 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.147 DEBUG:boto:Data: 2022-09-03T07:28:00.148 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.149 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.150 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.151 DEBUG:boto:Params: {} 2022-09-03T07:28:00.153 DEBUG:boto:Token: None 2022-09-03T07:28:00.154 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:an3/2dsr273M1ZJb3jeXO9XJteY= 2022-09-03T07:28:00.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:an3/2dsr273M1ZJb3jeXO9XJteY=', 'Content-Length': '0'} 2022-09-03T07:28:00.159 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007277167206ea909b-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.161 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.162 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.163 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.164 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.165 DEBUG:boto:Method: GET 2022-09-03T07:28:00.166 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.168 DEBUG:boto:Data: 2022-09-03T07:28:00.169 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.170 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.171 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.172 DEBUG:boto:Params: {} 2022-09-03T07:28:00.173 DEBUG:boto:Token: None 2022-09-03T07:28:00.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:an3/2dsr273M1ZJb3jeXO9XJteY= 2022-09-03T07:28:00.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:an3/2dsr273M1ZJb3jeXO9XJteY=', 'Content-Length': '0'} 2022-09-03T07:28:00.182 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dcbd76a0c27c2073-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.184 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.185 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.186 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.187 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.188 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.189 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.190 DEBUG:boto:Data: 2022-09-03T07:28:00.191 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.192 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.193 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.194 DEBUG:boto:Params: {} 2022-09-03T07:28:00.195 DEBUG:boto:Token: None 2022-09-03T07:28:00.196 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T3/JMvuJVcTChWkPY97ECa+AAMA= 2022-09-03T07:28:00.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T3/JMvuJVcTChWkPY97ECa+AAMA=', 'Content-Length': '0'} 2022-09-03T07:28:00.202 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006496d5f94319f465-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.203 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.204 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.205 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.206 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.207 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.209 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.209 DEBUG:boto:Data: 2022-09-03T07:28:00.210 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.211 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.212 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.213 DEBUG:boto:Params: {} 2022-09-03T07:28:00.213 DEBUG:boto:Token: None 2022-09-03T07:28:00.214 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.215 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T3/JMvuJVcTChWkPY97ECa+AAMA= 2022-09-03T07:28:00.216 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T3/JMvuJVcTChWkPY97ECa+AAMA=', 'Content-Length': '0'} 2022-09-03T07:28:00.218 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000061b771751e1798f-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.219 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:28:00.220 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.220 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.221 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.222 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.223 DEBUG:boto:Method: GET 2022-09-03T07:28:00.224 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.225 DEBUG:boto:Data: 2022-09-03T07:28:00.226 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.227 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.228 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.229 DEBUG:boto:Params: {} 2022-09-03T07:28:00.230 DEBUG:boto:Token: None 2022-09-03T07:28:00.231 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:an3/2dsr273M1ZJb3jeXO9XJteY= 2022-09-03T07:28:00.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:an3/2dsr273M1ZJb3jeXO9XJteY=', 'Content-Length': '0'} 2022-09-03T07:28:00.237 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000628ba3b2f123566b-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.278 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.279 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.280 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.281 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.282 DEBUG:boto:Method: GET 2022-09-03T07:28:00.283 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.284 DEBUG:boto:Data: 2022-09-03T07:28:00.285 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.286 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.286 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.287 DEBUG:boto:Params: {} 2022-09-03T07:28:00.288 DEBUG:boto:Token: None 2022-09-03T07:28:00.289 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:00.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:an3/2dsr273M1ZJb3jeXO9XJteY= 2022-09-03T07:28:00.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:an3/2dsr273M1ZJb3jeXO9XJteY=', 'Content-Length': '0'} 2022-09-03T07:28:00.293 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002225c16e30e5da1e-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.293 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.294 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.295 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:00.295 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:00.296 DEBUG:boto:Method: GET 2022-09-03T07:28:00.297 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:00.297 DEBUG:boto:Data: 2022-09-03T07:28:00.298 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.298 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.299 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.300 DEBUG:boto:Params: {} 2022-09-03T07:28:00.301 DEBUG:boto:Token: None 2022-09-03T07:28:00.302 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.306 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f7e004a2c9249082-0063130200-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.307 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.308 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.309 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.310 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:00.310 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:00.311 DEBUG:boto:Method: GET 2022-09-03T07:28:00.311 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:00.312 DEBUG:boto:Data: 2022-09-03T07:28:00.312 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.317 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.318 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.319 DEBUG:boto:Params: {} 2022-09-03T07:28:00.320 DEBUG:boto:Token: None 2022-09-03T07:28:00.321 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.329 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000556b86aeb701d9cd-0063130200-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.330 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.332 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.333 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.334 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.335 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:00.336 DEBUG:boto:Data: 2022-09-03T07:28:00.336 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.337 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.338 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.339 DEBUG:boto:Params: {} 2022-09-03T07:28:00.340 DEBUG:boto:Token: None 2022-09-03T07:28:00.341 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22 2022-09-03T07:28:00.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QAbba/eoHBNfWC8I2UXN75voDq0= 2022-09-03T07:28:00.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QAbba/eoHBNfWC8I2UXN75voDq0=', 'Content-Length': '0'} 2022-09-03T07:28:00.346 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b5e648376b918b4-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.347 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.348 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.350 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.350 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:00.351 DEBUG:boto:Data: 2022-09-03T07:28:00.352 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.353 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.353 DEBUG:boto:Params: {} 2022-09-03T07:28:00.354 DEBUG:boto:Token: None 2022-09-03T07:28:00.354 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22 2022-09-03T07:28:00.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QAbba/eoHBNfWC8I2UXN75voDq0= 2022-09-03T07:28:00.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QAbba/eoHBNfWC8I2UXN75voDq0=', 'Content-Length': '0'} 2022-09-03T07:28:00.359 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000727ff5d593cb0de7-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.360 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:28:00.361 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.362 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.363 DEBUG:boto:Method: GET 2022-09-03T07:28:00.364 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:00.364 DEBUG:boto:Data: 2022-09-03T07:28:00.365 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.366 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.366 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.367 DEBUG:boto:Params: {} 2022-09-03T07:28:00.367 DEBUG:boto:Token: None 2022-09-03T07:28:00.368 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22 2022-09-03T07:28:00.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MWDmoQ4GiyxVFVwZHLhKkoRM9Us= 2022-09-03T07:28:00.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MWDmoQ4GiyxVFVwZHLhKkoRM9Us=', 'Content-Length': '0'} 2022-09-03T07:28:00.372 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e31c4c50213eba37-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.373 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:00.374 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:00.375 DEBUG:boto:Method: GET 2022-09-03T07:28:00.376 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:00.376 DEBUG:boto:Data: 2022-09-03T07:28:00.377 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.378 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.379 DEBUG:boto:Params: {} 2022-09-03T07:28:00.379 DEBUG:boto:Token: None 2022-09-03T07:28:00.380 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj22 2022-09-03T07:28:00.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MWDmoQ4GiyxVFVwZHLhKkoRM9Us= 2022-09-03T07:28:00.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MWDmoQ4GiyxVFVwZHLhKkoRM9Us=', 'Content-Length': '0'} 2022-09-03T07:28:00.384 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068aa98848d5d9767-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.426 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.428 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.429 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:00.429 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:00.430 DEBUG:boto:Method: GET 2022-09-03T07:28:00.431 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:00.432 DEBUG:boto:Data: 2022-09-03T07:28:00.433 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.437 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.438 DEBUG:boto:Params: {} 2022-09-03T07:28:00.439 DEBUG:boto:Token: None 2022-09-03T07:28:00.440 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.455 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b9c9e74e85bcd3f3-0063130200-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.457 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.458 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.459 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.460 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:00.461 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:00.462 DEBUG:boto:Method: GET 2022-09-03T07:28:00.463 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:00.464 DEBUG:boto:Data: 2022-09-03T07:28:00.465 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.465 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.466 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.467 DEBUG:boto:Params: {} 2022-09-03T07:28:00.468 DEBUG:boto:Token: None 2022-09-03T07:28:00.469 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.475 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f7dfe07d36a4abc-0063130200-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.476 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.477 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:28:00.477 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.478 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.479 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.480 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.480 DEBUG:boto:Method: GET 2022-09-03T07:28:00.481 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.483 DEBUG:boto:Data: 2022-09-03T07:28:00.484 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.484 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.485 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.485 DEBUG:boto:Params: {} 2022-09-03T07:28:00.486 DEBUG:boto:Token: None 2022-09-03T07:28:00.487 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.487 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oC0yzW1SnkLKjh/jM1Bg5EIMj5g= 2022-09-03T07:28:00.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oC0yzW1SnkLKjh/jM1Bg5EIMj5g=', 'Content-Length': '0'} 2022-09-03T07:28:00.490 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000315e911329935e6d-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.530 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.531 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.532 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.532 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.533 DEBUG:boto:Method: GET 2022-09-03T07:28:00.534 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.535 DEBUG:boto:Data: 2022-09-03T07:28:00.538 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.539 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.540 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.541 DEBUG:boto:Params: {} 2022-09-03T07:28:00.542 DEBUG:boto:Token: None 2022-09-03T07:28:00.543 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oC0yzW1SnkLKjh/jM1Bg5EIMj5g= 2022-09-03T07:28:00.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oC0yzW1SnkLKjh/jM1Bg5EIMj5g=', 'Content-Length': '0'} 2022-09-03T07:28:00.557 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a7844a751e5a1dd6-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.559 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.559 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.560 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.561 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.562 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.562 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.564 DEBUG:boto:Data: 2022-09-03T07:28:00.564 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.565 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.567 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.568 DEBUG:boto:Params: {} 2022-09-03T07:28:00.569 DEBUG:boto:Token: None 2022-09-03T07:28:00.570 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g4bHe8MyrvOEGO9eicSyO7tapVM= 2022-09-03T07:28:00.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g4bHe8MyrvOEGO9eicSyO7tapVM=', 'Content-Length': '0'} 2022-09-03T07:28:00.575 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9ce7e126037bb38-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.576 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.577 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.578 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.579 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.580 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.581 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.582 DEBUG:boto:Data: 2022-09-03T07:28:00.582 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.584 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.585 DEBUG:boto:Params: {} 2022-09-03T07:28:00.586 DEBUG:boto:Token: None 2022-09-03T07:28:00.587 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g4bHe8MyrvOEGO9eicSyO7tapVM= 2022-09-03T07:28:00.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g4bHe8MyrvOEGO9eicSyO7tapVM=', 'Content-Length': '0'} 2022-09-03T07:28:00.598 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c85bd323679990c4-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.599 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:28:00.599 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.600 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.601 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.601 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.602 DEBUG:boto:Method: GET 2022-09-03T07:28:00.603 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.603 DEBUG:boto:Data: 2022-09-03T07:28:00.604 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.604 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.605 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.605 DEBUG:boto:Params: {} 2022-09-03T07:28:00.607 DEBUG:boto:Token: None 2022-09-03T07:28:00.607 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oC0yzW1SnkLKjh/jM1Bg5EIMj5g= 2022-09-03T07:28:00.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oC0yzW1SnkLKjh/jM1Bg5EIMj5g=', 'Content-Length': '0'} 2022-09-03T07:28:00.611 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e88f61a7186e6ff4-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.612 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.613 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.615 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.615 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.616 DEBUG:boto:Method: GET 2022-09-03T07:28:00.617 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.618 DEBUG:boto:Data: 2022-09-03T07:28:00.619 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.621 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.622 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.623 DEBUG:boto:Params: {} 2022-09-03T07:28:00.624 DEBUG:boto:Token: None 2022-09-03T07:28:00.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:00.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oC0yzW1SnkLKjh/jM1Bg5EIMj5g= 2022-09-03T07:28:00.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oC0yzW1SnkLKjh/jM1Bg5EIMj5g=', 'Content-Length': '0'} 2022-09-03T07:28:00.631 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a783400415b5462b-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.633 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.634 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.635 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:00.636 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:00.638 DEBUG:boto:Method: GET 2022-09-03T07:28:00.639 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:00.639 DEBUG:boto:Data: 2022-09-03T07:28:00.640 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.641 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.641 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.642 DEBUG:boto:Params: {} 2022-09-03T07:28:00.642 DEBUG:boto:Token: None 2022-09-03T07:28:00.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.647 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002bd7af188af2d6d9-0063130200-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.648 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.648 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.649 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.649 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:00.650 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:00.651 DEBUG:boto:Method: GET 2022-09-03T07:28:00.651 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:00.652 DEBUG:boto:Data: 2022-09-03T07:28:00.653 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.653 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.654 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.655 DEBUG:boto:Params: {} 2022-09-03T07:28:00.655 DEBUG:boto:Token: None 2022-09-03T07:28:00.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.662 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000914cfa9be480a5ad-0063130200-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.663 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.665 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.665 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.667 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.667 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:00.668 DEBUG:boto:Data: 2022-09-03T07:28:00.669 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.670 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.671 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.672 DEBUG:boto:Params: {} 2022-09-03T07:28:00.673 DEBUG:boto:Token: None 2022-09-03T07:28:00.674 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23 2022-09-03T07:28:00.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RleBNgbVtMbgCW8ut2b0ZdJm/so= 2022-09-03T07:28:00.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RleBNgbVtMbgCW8ut2b0ZdJm/so=', 'Content-Length': '0'} 2022-09-03T07:28:00.680 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d3d84566e82f155-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.681 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.682 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.683 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.684 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:00.685 DEBUG:boto:Data: 2022-09-03T07:28:00.686 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.688 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.689 DEBUG:boto:Params: {} 2022-09-03T07:28:00.690 DEBUG:boto:Token: None 2022-09-03T07:28:00.691 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23 2022-09-03T07:28:00.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RleBNgbVtMbgCW8ut2b0ZdJm/so= 2022-09-03T07:28:00.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RleBNgbVtMbgCW8ut2b0ZdJm/so=', 'Content-Length': '0'} 2022-09-03T07:28:00.696 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c7c55645ba37b6b-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.697 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:28:00.698 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.700 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.701 DEBUG:boto:Method: GET 2022-09-03T07:28:00.702 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:00.703 DEBUG:boto:Data: 2022-09-03T07:28:00.705 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.706 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.707 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.708 DEBUG:boto:Params: {} 2022-09-03T07:28:00.709 DEBUG:boto:Token: None 2022-09-03T07:28:00.710 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23 2022-09-03T07:28:00.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:afx6JzCTFQjDf13mAoqjoY02TcU= 2022-09-03T07:28:00.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:afx6JzCTFQjDf13mAoqjoY02TcU=', 'Content-Length': '0'} 2022-09-03T07:28:00.715 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074f8a4438befc63c-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.717 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:00.718 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:00.719 DEBUG:boto:Method: GET 2022-09-03T07:28:00.720 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:00.721 DEBUG:boto:Data: 2022-09-03T07:28:00.722 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.723 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.724 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.725 DEBUG:boto:Params: {} 2022-09-03T07:28:00.726 DEBUG:boto:Token: None 2022-09-03T07:28:00.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj23 2022-09-03T07:28:00.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:afx6JzCTFQjDf13mAoqjoY02TcU= 2022-09-03T07:28:00.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:afx6JzCTFQjDf13mAoqjoY02TcU=', 'Content-Length': '0'} 2022-09-03T07:28:00.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074a7e055a054591d-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.734 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.735 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.736 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:00.737 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:00.738 DEBUG:boto:Method: GET 2022-09-03T07:28:00.739 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:00.740 DEBUG:boto:Data: 2022-09-03T07:28:00.741 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.742 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.744 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.745 DEBUG:boto:Params: {} 2022-09-03T07:28:00.746 DEBUG:boto:Token: None 2022-09-03T07:28:00.746 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.750 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000193f068e8c932c3c-0063130200-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.751 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.751 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.752 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.753 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:00.753 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:00.754 DEBUG:boto:Method: GET 2022-09-03T07:28:00.754 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:00.755 DEBUG:boto:Data: 2022-09-03T07:28:00.756 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.757 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.758 DEBUG:boto:Params: {} 2022-09-03T07:28:00.758 DEBUG:boto:Token: None 2022-09-03T07:28:00.759 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.766 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091903689df9ac2be-0063130200-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.767 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.768 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:28:00.769 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:00.769 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:00.770 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.771 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.771 DEBUG:boto:Method: GET 2022-09-03T07:28:00.772 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.773 DEBUG:boto:Data: 2022-09-03T07:28:00.773 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.774 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.775 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.776 DEBUG:boto:Params: {} 2022-09-03T07:28:00.778 DEBUG:boto:Token: None 2022-09-03T07:28:00.779 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:la0dr7jb0MeY4T2+1Vp6kn63ii0= 2022-09-03T07:28:00.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:la0dr7jb0MeY4T2+1Vp6kn63ii0=', 'Content-Length': '0'} 2022-09-03T07:28:00.784 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002efd8e67d52b158f-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.835 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:00.836 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:00.837 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.840 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.840 DEBUG:boto:Method: GET 2022-09-03T07:28:00.841 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.841 DEBUG:boto:Data: 2022-09-03T07:28:00.842 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.843 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.843 DEBUG:boto:Params: {} 2022-09-03T07:28:00.844 DEBUG:boto:Token: None 2022-09-03T07:28:00.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:la0dr7jb0MeY4T2+1Vp6kn63ii0= 2022-09-03T07:28:00.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:la0dr7jb0MeY4T2+1Vp6kn63ii0=', 'Content-Length': '0'} 2022-09-03T07:28:00.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a17483454aecd9f-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.849 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:00.849 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:00.850 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.851 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.851 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.852 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.853 DEBUG:boto:Data: 2022-09-03T07:28:00.853 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.854 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.855 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.855 DEBUG:boto:Params: {} 2022-09-03T07:28:00.856 DEBUG:boto:Token: None 2022-09-03T07:28:00.856 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8iKv8ZlN1+sMEfqftLczW9NnFwQ= 2022-09-03T07:28:00.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8iKv8ZlN1+sMEfqftLczW9NnFwQ=', 'Content-Length': '0'} 2022-09-03T07:28:00.860 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d5a3d42a97064b8-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.861 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:00.862 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:00.863 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.863 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.864 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.864 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.865 DEBUG:boto:Data: 2022-09-03T07:28:00.866 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.866 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.867 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.868 DEBUG:boto:Params: {} 2022-09-03T07:28:00.868 DEBUG:boto:Token: None 2022-09-03T07:28:00.869 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8iKv8ZlN1+sMEfqftLczW9NnFwQ= 2022-09-03T07:28:00.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8iKv8ZlN1+sMEfqftLczW9NnFwQ=', 'Content-Length': '0'} 2022-09-03T07:28:00.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048d4bcf5ea31d269-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.873 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:28:00.874 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:00.874 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:00.875 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.876 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.876 DEBUG:boto:Method: GET 2022-09-03T07:28:00.877 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.877 DEBUG:boto:Data: 2022-09-03T07:28:00.878 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.880 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.880 DEBUG:boto:Params: {} 2022-09-03T07:28:00.881 DEBUG:boto:Token: None 2022-09-03T07:28:00.881 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.882 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:la0dr7jb0MeY4T2+1Vp6kn63ii0= 2022-09-03T07:28:00.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:la0dr7jb0MeY4T2+1Vp6kn63ii0=', 'Content-Length': '0'} 2022-09-03T07:28:00.885 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be66b21b96d6a096-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.886 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:00.887 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:00.887 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.888 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.889 DEBUG:boto:Method: GET 2022-09-03T07:28:00.890 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.891 DEBUG:boto:Data: 2022-09-03T07:28:00.892 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.894 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.895 DEBUG:boto:Params: {} 2022-09-03T07:28:00.896 DEBUG:boto:Token: None 2022-09-03T07:28:00.897 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:00.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:la0dr7jb0MeY4T2+1Vp6kn63ii0= 2022-09-03T07:28:00.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:la0dr7jb0MeY4T2+1Vp6kn63ii0=', 'Content-Length': '0'} 2022-09-03T07:28:00.902 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000848b1a317e9f3a4e-0063130200-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.903 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.904 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.905 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:00.906 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:00.907 DEBUG:boto:Method: GET 2022-09-03T07:28:00.908 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:00.908 DEBUG:boto:Data: 2022-09-03T07:28:00.909 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.909 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.910 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.910 DEBUG:boto:Params: {} 2022-09-03T07:28:00.911 DEBUG:boto:Token: None 2022-09-03T07:28:00.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.915 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005e64c00857654360-0063130200-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.916 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.918 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:00.919 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:00.920 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:00.921 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:00.921 DEBUG:boto:Method: GET 2022-09-03T07:28:00.924 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:00.925 DEBUG:boto:Data: 2022-09-03T07:28:00.926 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:00.928 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.929 DEBUG:boto:Params: {} 2022-09-03T07:28:00.929 DEBUG:boto:Token: None 2022-09-03T07:28:00.930 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/ 2022-09-03T07:28:00.931 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8= 2022-09-03T07:28:00.932 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PB2jfGgyVWCoHGh1xG7RRPs5u/8=', 'Content-Length': '0'} 2022-09-03T07:28:00.938 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002cbc19792f68f928-0063130200-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:00.982 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:00.983 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:00.984 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:00.985 DEBUG:boto:Method: HEAD 2022-09-03T07:28:00.985 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:00.986 DEBUG:boto:Data: 2022-09-03T07:28:00.987 DEBUG:boto:Headers: {} 2022-09-03T07:28:00.987 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:00.988 DEBUG:boto:Port: 8000 2022-09-03T07:28:00.988 DEBUG:boto:Params: {} 2022-09-03T07:28:00.989 DEBUG:boto:Token: None 2022-09-03T07:28:00.990 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:00 GMT /avysmw-75/obj29 2022-09-03T07:28:00.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zax4fFfDoGg+rhEjaFUqOff2luI= 2022-09-03T07:28:00.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zax4fFfDoGg+rhEjaFUqOff2luI=', 'Content-Length': '0'} 2022-09-03T07:28:00.994 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a10eec7131733af-0063130200-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.001 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:01.002 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:01.003 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.004 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:01.005 DEBUG:boto:Data: 2022-09-03T07:28:01.006 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.007 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.007 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.009 DEBUG:boto:Params: {} 2022-09-03T07:28:01.011 DEBUG:boto:Token: None 2022-09-03T07:28:01.012 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj29 2022-09-03T07:28:01.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jPuMz4jN2AbaeYCb8r7CokCsZdY= 2022-09-03T07:28:01.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jPuMz4jN2AbaeYCb8r7CokCsZdY=', 'Content-Length': '0'} 2022-09-03T07:28:01.017 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f929011b816f3628-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.018 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:28:01.019 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:01.019 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:01.020 DEBUG:boto:Method: GET 2022-09-03T07:28:01.021 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:01.021 DEBUG:boto:Data: 2022-09-03T07:28:01.022 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.023 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.023 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.024 DEBUG:boto:Params: {} 2022-09-03T07:28:01.024 DEBUG:boto:Token: None 2022-09-03T07:28:01.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj29 2022-09-03T07:28:01.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j6pfkl61KEty4d9ZytYcbm9ZysI= 2022-09-03T07:28:01.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j6pfkl61KEty4d9ZytYcbm9ZysI=', 'Content-Length': '0'} 2022-09-03T07:28:01.028 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000684492223a9b5705-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.029 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:01.030 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:01.031 DEBUG:boto:Method: GET 2022-09-03T07:28:01.031 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:01.032 DEBUG:boto:Data: 2022-09-03T07:28:01.032 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.033 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.033 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.034 DEBUG:boto:Params: {} 2022-09-03T07:28:01.035 DEBUG:boto:Token: None 2022-09-03T07:28:01.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj29 2022-09-03T07:28:01.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j6pfkl61KEty4d9ZytYcbm9ZysI= 2022-09-03T07:28:01.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j6pfkl61KEty4d9ZytYcbm9ZysI=', 'Content-Length': '0'} 2022-09-03T07:28:01.040 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001c548acf7378ad3-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.040 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.042 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.042 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:01.043 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:01.043 DEBUG:boto:Method: GET 2022-09-03T07:28:01.044 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:01.045 DEBUG:boto:Data: 2022-09-03T07:28:01.045 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.046 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.046 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.047 DEBUG:boto:Params: {} 2022-09-03T07:28:01.048 DEBUG:boto:Token: None 2022-09-03T07:28:01.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.050 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.053 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023920daf0539295e-0063130201-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.054 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.056 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.057 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.058 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:01.059 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:01.059 DEBUG:boto:Method: GET 2022-09-03T07:28:01.060 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:01.060 DEBUG:boto:Data: 2022-09-03T07:28:01.061 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.062 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.063 DEBUG:boto:Params: {} 2022-09-03T07:28:01.063 DEBUG:boto:Token: None 2022-09-03T07:28:01.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.070 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce1676fd916581a8-0063130201-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.071 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.072 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:28:01.074 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.075 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.075 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.076 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.076 DEBUG:boto:Method: GET 2022-09-03T07:28:01.077 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.078 DEBUG:boto:Data: 2022-09-03T07:28:01.079 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.080 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.080 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.081 DEBUG:boto:Params: {} 2022-09-03T07:28:01.082 DEBUG:boto:Token: None 2022-09-03T07:28:01.082 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T15k0GoxPdiJ58rrD1fPSneAYTA= 2022-09-03T07:28:01.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T15k0GoxPdiJ58rrD1fPSneAYTA=', 'Content-Length': '0'} 2022-09-03T07:28:01.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b04bc689bca49d8-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.087 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.088 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.088 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.089 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.090 DEBUG:boto:Method: GET 2022-09-03T07:28:01.091 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.091 DEBUG:boto:Data: 2022-09-03T07:28:01.092 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.093 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.094 DEBUG:boto:Params: {} 2022-09-03T07:28:01.094 DEBUG:boto:Token: None 2022-09-03T07:28:01.095 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T15k0GoxPdiJ58rrD1fPSneAYTA= 2022-09-03T07:28:01.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T15k0GoxPdiJ58rrD1fPSneAYTA=', 'Content-Length': '0'} 2022-09-03T07:28:01.101 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d761c104480a8637-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.106 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.107 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.108 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.109 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.110 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.111 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.112 DEBUG:boto:Data: 2022-09-03T07:28:01.113 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.116 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.116 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.117 DEBUG:boto:Params: {} 2022-09-03T07:28:01.118 DEBUG:boto:Token: None 2022-09-03T07:28:01.119 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yEiKSRmX1WY4BYN3U1bDiuyRxYQ= 2022-09-03T07:28:01.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yEiKSRmX1WY4BYN3U1bDiuyRxYQ=', 'Content-Length': '0'} 2022-09-03T07:28:01.124 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000072f1fc588e15f93a-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.125 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.125 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.126 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.127 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.127 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.128 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.130 DEBUG:boto:Data: 2022-09-03T07:28:01.130 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.131 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.132 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.132 DEBUG:boto:Params: {} 2022-09-03T07:28:01.133 DEBUG:boto:Token: None 2022-09-03T07:28:01.134 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yEiKSRmX1WY4BYN3U1bDiuyRxYQ= 2022-09-03T07:28:01.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yEiKSRmX1WY4BYN3U1bDiuyRxYQ=', 'Content-Length': '0'} 2022-09-03T07:28:01.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000052526b1ff8a8c61-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.138 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:28:01.139 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.140 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.140 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.141 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.142 DEBUG:boto:Method: GET 2022-09-03T07:28:01.142 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.144 DEBUG:boto:Data: 2022-09-03T07:28:01.145 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.145 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.146 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.147 DEBUG:boto:Params: {} 2022-09-03T07:28:01.147 DEBUG:boto:Token: None 2022-09-03T07:28:01.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T15k0GoxPdiJ58rrD1fPSneAYTA= 2022-09-03T07:28:01.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T15k0GoxPdiJ58rrD1fPSneAYTA=', 'Content-Length': '0'} 2022-09-03T07:28:01.153 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b571653f79bdb29-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.194 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.196 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.196 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.197 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.198 DEBUG:boto:Method: GET 2022-09-03T07:28:01.199 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.199 DEBUG:boto:Data: 2022-09-03T07:28:01.200 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.201 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.202 DEBUG:boto:Params: {} 2022-09-03T07:28:01.202 DEBUG:boto:Token: None 2022-09-03T07:28:01.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:01.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T15k0GoxPdiJ58rrD1fPSneAYTA= 2022-09-03T07:28:01.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T15k0GoxPdiJ58rrD1fPSneAYTA=', 'Content-Length': '0'} 2022-09-03T07:28:01.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000051811b57a05107fd-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.208 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.209 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.209 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:01.210 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:01.211 DEBUG:boto:Method: GET 2022-09-03T07:28:01.211 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:01.212 DEBUG:boto:Data: 2022-09-03T07:28:01.212 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.213 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.214 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.214 DEBUG:boto:Params: {} 2022-09-03T07:28:01.216 DEBUG:boto:Token: None 2022-09-03T07:28:01.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.222 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005f2424beff41c465-0063130201-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.223 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.224 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.225 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.225 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:01.226 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:01.227 DEBUG:boto:Method: GET 2022-09-03T07:28:01.228 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:01.228 DEBUG:boto:Data: 2022-09-03T07:28:01.229 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.230 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.230 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.231 DEBUG:boto:Params: {} 2022-09-03T07:28:01.232 DEBUG:boto:Token: None 2022-09-03T07:28:01.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.239 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000bc2396249820a09-0063130201-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.240 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.242 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.242 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.243 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.244 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:01.245 DEBUG:boto:Data: 2022-09-03T07:28:01.246 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.248 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.250 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.250 DEBUG:boto:Params: {} 2022-09-03T07:28:01.251 DEBUG:boto:Token: None 2022-09-03T07:28:01.252 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31 2022-09-03T07:28:01.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mMXsl67EpDI1RnPtI6hIUn7LkOg= 2022-09-03T07:28:01.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mMXsl67EpDI1RnPtI6hIUn7LkOg=', 'Content-Length': '0'} 2022-09-03T07:28:01.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cbcf1f96e70cc831-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.257 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.257 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.258 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.258 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:01.259 DEBUG:boto:Data: 2022-09-03T07:28:01.260 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.261 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.261 DEBUG:boto:Params: {} 2022-09-03T07:28:01.262 DEBUG:boto:Token: None 2022-09-03T07:28:01.263 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31 2022-09-03T07:28:01.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mMXsl67EpDI1RnPtI6hIUn7LkOg= 2022-09-03T07:28:01.265 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mMXsl67EpDI1RnPtI6hIUn7LkOg=', 'Content-Length': '0'} 2022-09-03T07:28:01.268 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005bbeb91e1ddd8305-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.269 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:28:01.269 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.270 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.271 DEBUG:boto:Method: GET 2022-09-03T07:28:01.271 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:01.272 DEBUG:boto:Data: 2022-09-03T07:28:01.273 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.274 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.275 DEBUG:boto:Params: {} 2022-09-03T07:28:01.276 DEBUG:boto:Token: None 2022-09-03T07:28:01.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31 2022-09-03T07:28:01.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O0WzD7WjQOT0c5GwEHAi67l67dk= 2022-09-03T07:28:01.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O0WzD7WjQOT0c5GwEHAi67l67dk=', 'Content-Length': '0'} 2022-09-03T07:28:01.282 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c98efdb1ef2acab0-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.283 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:01.283 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:01.284 DEBUG:boto:Method: GET 2022-09-03T07:28:01.285 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:01.286 DEBUG:boto:Data: 2022-09-03T07:28:01.287 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.287 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.288 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.289 DEBUG:boto:Params: {} 2022-09-03T07:28:01.289 DEBUG:boto:Token: None 2022-09-03T07:28:01.290 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj31 2022-09-03T07:28:01.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O0WzD7WjQOT0c5GwEHAi67l67dk= 2022-09-03T07:28:01.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O0WzD7WjQOT0c5GwEHAi67l67dk=', 'Content-Length': '0'} 2022-09-03T07:28:01.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbbe5c262e322afd-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.296 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.297 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.298 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:01.298 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:01.299 DEBUG:boto:Method: GET 2022-09-03T07:28:01.300 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:01.301 DEBUG:boto:Data: 2022-09-03T07:28:01.302 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.303 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.304 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.304 DEBUG:boto:Params: {} 2022-09-03T07:28:01.305 DEBUG:boto:Token: None 2022-09-03T07:28:01.306 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.307 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.310 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000054183788b36f15f-0063130201-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.354 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.356 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.357 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.357 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:01.358 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:01.359 DEBUG:boto:Method: GET 2022-09-03T07:28:01.360 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:01.360 DEBUG:boto:Data: 2022-09-03T07:28:01.361 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.363 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.364 DEBUG:boto:Params: {} 2022-09-03T07:28:01.365 DEBUG:boto:Token: None 2022-09-03T07:28:01.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.373 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005f12757bed75d1f5-0063130201-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.374 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.376 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:28:01.377 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.378 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.379 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.379 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.380 DEBUG:boto:Method: GET 2022-09-03T07:28:01.380 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.381 DEBUG:boto:Data: 2022-09-03T07:28:01.382 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.382 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.383 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.383 DEBUG:boto:Params: {} 2022-09-03T07:28:01.384 DEBUG:boto:Token: None 2022-09-03T07:28:01.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W9dkK73YISkNiN81X5Fv36M/rbk= 2022-09-03T07:28:01.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W9dkK73YISkNiN81X5Fv36M/rbk=', 'Content-Length': '0'} 2022-09-03T07:28:01.388 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d1dbe48a2f8bb9e-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.390 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.390 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.391 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.392 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.393 DEBUG:boto:Method: GET 2022-09-03T07:28:01.393 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.394 DEBUG:boto:Data: 2022-09-03T07:28:01.395 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.396 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.396 DEBUG:boto:Params: {} 2022-09-03T07:28:01.397 DEBUG:boto:Token: None 2022-09-03T07:28:01.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W9dkK73YISkNiN81X5Fv36M/rbk= 2022-09-03T07:28:01.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W9dkK73YISkNiN81X5Fv36M/rbk=', 'Content-Length': '0'} 2022-09-03T07:28:01.401 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054fa3490193a3f62-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.442 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.445 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.446 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.447 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.448 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.449 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.451 DEBUG:boto:Data: 2022-09-03T07:28:01.452 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.460 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.462 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.463 DEBUG:boto:Params: {} 2022-09-03T07:28:01.464 DEBUG:boto:Token: None 2022-09-03T07:28:01.465 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EzVfisAh1gcXV8dft35h7bgoR6w= 2022-09-03T07:28:01.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EzVfisAh1gcXV8dft35h7bgoR6w=', 'Content-Length': '0'} 2022-09-03T07:28:01.470 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b59f4b9d03cfe01-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.471 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.472 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.472 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.473 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.474 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.475 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.477 DEBUG:boto:Data: 2022-09-03T07:28:01.478 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.480 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.482 DEBUG:boto:Params: {} 2022-09-03T07:28:01.483 DEBUG:boto:Token: None 2022-09-03T07:28:01.484 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EzVfisAh1gcXV8dft35h7bgoR6w= 2022-09-03T07:28:01.489 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EzVfisAh1gcXV8dft35h7bgoR6w=', 'Content-Length': '0'} 2022-09-03T07:28:01.491 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d37b8d76db1e32d0-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.493 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:28:01.495 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.496 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.497 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.500 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.501 DEBUG:boto:Method: GET 2022-09-03T07:28:01.502 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.503 DEBUG:boto:Data: 2022-09-03T07:28:01.504 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.505 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.506 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.508 DEBUG:boto:Params: {} 2022-09-03T07:28:01.511 DEBUG:boto:Token: None 2022-09-03T07:28:01.512 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W9dkK73YISkNiN81X5Fv36M/rbk= 2022-09-03T07:28:01.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W9dkK73YISkNiN81X5Fv36M/rbk=', 'Content-Length': '0'} 2022-09-03T07:28:01.516 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe3f3873d82d8475-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.517 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.517 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.518 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.519 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.519 DEBUG:boto:Method: GET 2022-09-03T07:28:01.520 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.521 DEBUG:boto:Data: 2022-09-03T07:28:01.522 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.522 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.523 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.524 DEBUG:boto:Params: {} 2022-09-03T07:28:01.524 DEBUG:boto:Token: None 2022-09-03T07:28:01.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:01.526 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W9dkK73YISkNiN81X5Fv36M/rbk= 2022-09-03T07:28:01.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W9dkK73YISkNiN81X5Fv36M/rbk=', 'Content-Length': '0'} 2022-09-03T07:28:01.536 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089e9e4c00dacb881-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.538 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.539 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.540 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:01.541 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:01.542 DEBUG:boto:Method: GET 2022-09-03T07:28:01.543 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:01.544 DEBUG:boto:Data: 2022-09-03T07:28:01.547 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.548 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.548 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.549 DEBUG:boto:Params: {} 2022-09-03T07:28:01.551 DEBUG:boto:Token: None 2022-09-03T07:28:01.552 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.556 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf3414b70bab06ec-0063130201-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.557 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.558 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.558 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.559 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:01.560 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:01.560 DEBUG:boto:Method: GET 2022-09-03T07:28:01.561 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:01.562 DEBUG:boto:Data: 2022-09-03T07:28:01.563 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.565 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.566 DEBUG:boto:Params: {} 2022-09-03T07:28:01.567 DEBUG:boto:Token: None 2022-09-03T07:28:01.568 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.570 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.574 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004abfdc2a49207568-0063130201-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.575 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.576 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.577 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.578 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.579 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:01.580 DEBUG:boto:Data: 2022-09-03T07:28:01.581 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.582 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.583 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.584 DEBUG:boto:Params: {} 2022-09-03T07:28:01.585 DEBUG:boto:Token: None 2022-09-03T07:28:01.585 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33 2022-09-03T07:28:01.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y0BiME2FlB+ZJ24qzcKaTDF6TU8= 2022-09-03T07:28:01.588 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y0BiME2FlB+ZJ24qzcKaTDF6TU8=', 'Content-Length': '0'} 2022-09-03T07:28:01.590 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e356b79f170b2c49-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.591 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.592 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.593 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.594 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:01.595 DEBUG:boto:Data: 2022-09-03T07:28:01.596 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.598 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.599 DEBUG:boto:Params: {} 2022-09-03T07:28:01.600 DEBUG:boto:Token: None 2022-09-03T07:28:01.601 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33 2022-09-03T07:28:01.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Y0BiME2FlB+ZJ24qzcKaTDF6TU8= 2022-09-03T07:28:01.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Y0BiME2FlB+ZJ24qzcKaTDF6TU8=', 'Content-Length': '0'} 2022-09-03T07:28:01.606 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c40a05fd4bf98bb4-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.607 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:28:01.608 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.609 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.610 DEBUG:boto:Method: GET 2022-09-03T07:28:01.611 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:01.613 DEBUG:boto:Data: 2022-09-03T07:28:01.614 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.615 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.616 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.617 DEBUG:boto:Params: {} 2022-09-03T07:28:01.618 DEBUG:boto:Token: None 2022-09-03T07:28:01.619 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33 2022-09-03T07:28:01.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5txk7Rit0DmM3Xo8h6OLov0Nn4c= 2022-09-03T07:28:01.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5txk7Rit0DmM3Xo8h6OLov0Nn4c=', 'Content-Length': '0'} 2022-09-03T07:28:01.623 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7b8ecf5bcb92ac0-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.624 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:01.624 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:01.625 DEBUG:boto:Method: GET 2022-09-03T07:28:01.627 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:01.628 DEBUG:boto:Data: 2022-09-03T07:28:01.628 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.629 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.630 DEBUG:boto:Params: {} 2022-09-03T07:28:01.631 DEBUG:boto:Token: None 2022-09-03T07:28:01.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj33 2022-09-03T07:28:01.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5txk7Rit0DmM3Xo8h6OLov0Nn4c= 2022-09-03T07:28:01.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5txk7Rit0DmM3Xo8h6OLov0Nn4c=', 'Content-Length': '0'} 2022-09-03T07:28:01.635 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091455aee7b96b4fe-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.636 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.636 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.637 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:01.637 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:01.638 DEBUG:boto:Method: GET 2022-09-03T07:28:01.638 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:01.639 DEBUG:boto:Data: 2022-09-03T07:28:01.641 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.642 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.643 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.644 DEBUG:boto:Params: {} 2022-09-03T07:28:01.645 DEBUG:boto:Token: None 2022-09-03T07:28:01.646 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.651 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088e42acdd3dddfdb-0063130201-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.652 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.654 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.655 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.656 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:01.657 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:01.658 DEBUG:boto:Method: GET 2022-09-03T07:28:01.659 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:01.660 DEBUG:boto:Data: 2022-09-03T07:28:01.661 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.663 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.664 DEBUG:boto:Params: {} 2022-09-03T07:28:01.665 DEBUG:boto:Token: None 2022-09-03T07:28:01.666 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.671 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028f079b663cc3f8b-0063130201-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.718 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.720 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:28:01.721 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.722 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.723 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.723 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.724 DEBUG:boto:Method: GET 2022-09-03T07:28:01.724 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.725 DEBUG:boto:Data: 2022-09-03T07:28:01.725 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.726 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.727 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.727 DEBUG:boto:Params: {} 2022-09-03T07:28:01.728 DEBUG:boto:Token: None 2022-09-03T07:28:01.728 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z/kd95TlbtfWHeOCacWK18F5OgI= 2022-09-03T07:28:01.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z/kd95TlbtfWHeOCacWK18F5OgI=', 'Content-Length': '0'} 2022-09-03T07:28:01.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f3e481ba6e8f6db-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.733 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.734 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.735 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.736 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.737 DEBUG:boto:Method: GET 2022-09-03T07:28:01.737 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.738 DEBUG:boto:Data: 2022-09-03T07:28:01.739 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.740 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.741 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.741 DEBUG:boto:Params: {} 2022-09-03T07:28:01.742 DEBUG:boto:Token: None 2022-09-03T07:28:01.743 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z/kd95TlbtfWHeOCacWK18F5OgI= 2022-09-03T07:28:01.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z/kd95TlbtfWHeOCacWK18F5OgI=', 'Content-Length': '0'} 2022-09-03T07:28:01.747 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020e2fc0528940d79-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.749 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.750 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.751 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.752 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.753 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.754 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.755 DEBUG:boto:Data: 2022-09-03T07:28:01.756 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.757 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.758 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.759 DEBUG:boto:Params: {} 2022-09-03T07:28:01.760 DEBUG:boto:Token: None 2022-09-03T07:28:01.764 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MzbQ7oUR56ayXRsYEs/6jKRvYik= 2022-09-03T07:28:01.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MzbQ7oUR56ayXRsYEs/6jKRvYik=', 'Content-Length': '0'} 2022-09-03T07:28:01.769 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b077e2e85be9f17-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.770 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.774 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.775 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.776 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.777 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.778 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.779 DEBUG:boto:Data: 2022-09-03T07:28:01.779 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.781 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.783 DEBUG:boto:Params: {} 2022-09-03T07:28:01.785 DEBUG:boto:Token: None 2022-09-03T07:28:01.786 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MzbQ7oUR56ayXRsYEs/6jKRvYik= 2022-09-03T07:28:01.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MzbQ7oUR56ayXRsYEs/6jKRvYik=', 'Content-Length': '0'} 2022-09-03T07:28:01.792 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd88ce834b3cd2b5-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.792 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:28:01.793 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.795 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.795 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.796 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.796 DEBUG:boto:Method: GET 2022-09-03T07:28:01.797 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.798 DEBUG:boto:Data: 2022-09-03T07:28:01.798 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.800 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.801 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.801 DEBUG:boto:Params: {} 2022-09-03T07:28:01.802 DEBUG:boto:Token: None 2022-09-03T07:28:01.805 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z/kd95TlbtfWHeOCacWK18F5OgI= 2022-09-03T07:28:01.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z/kd95TlbtfWHeOCacWK18F5OgI=', 'Content-Length': '0'} 2022-09-03T07:28:01.810 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000365852e567f421e1-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.811 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.812 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.812 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.813 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.813 DEBUG:boto:Method: GET 2022-09-03T07:28:01.820 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.821 DEBUG:boto:Data: 2022-09-03T07:28:01.822 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.823 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.823 DEBUG:boto:Params: {} 2022-09-03T07:28:01.824 DEBUG:boto:Token: None 2022-09-03T07:28:01.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:01.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:z/kd95TlbtfWHeOCacWK18F5OgI= 2022-09-03T07:28:01.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:z/kd95TlbtfWHeOCacWK18F5OgI=', 'Content-Length': '0'} 2022-09-03T07:28:01.831 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013ce8c0beb5ef56f-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.832 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.832 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.833 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:01.833 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:01.834 DEBUG:boto:Method: GET 2022-09-03T07:28:01.834 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:01.835 DEBUG:boto:Data: 2022-09-03T07:28:01.836 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.837 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.837 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.839 DEBUG:boto:Params: {} 2022-09-03T07:28:01.840 DEBUG:boto:Token: None 2022-09-03T07:28:01.840 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.844 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e874e90b0515a4a-0063130201-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.848 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.850 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.850 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.851 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:01.851 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:01.852 DEBUG:boto:Method: GET 2022-09-03T07:28:01.852 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:01.853 DEBUG:boto:Data: 2022-09-03T07:28:01.853 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.854 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.855 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.855 DEBUG:boto:Params: {} 2022-09-03T07:28:01.856 DEBUG:boto:Token: None 2022-09-03T07:28:01.856 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.863 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000400e9fee00124d19-0063130201-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.864 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.866 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.867 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.868 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.870 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:01.871 DEBUG:boto:Data: 2022-09-03T07:28:01.872 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.873 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.874 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.875 DEBUG:boto:Params: {} 2022-09-03T07:28:01.876 DEBUG:boto:Token: None 2022-09-03T07:28:01.877 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34 2022-09-03T07:28:01.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CWrvGgiq4UnZXEG8R+fRH1jTMF4= 2022-09-03T07:28:01.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CWrvGgiq4UnZXEG8R+fRH1jTMF4=', 'Content-Length': '0'} 2022-09-03T07:28:01.886 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa34321e88bfeb18-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.888 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.889 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.890 DEBUG:boto:Method: HEAD 2022-09-03T07:28:01.891 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:01.892 DEBUG:boto:Data: 2022-09-03T07:28:01.893 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.894 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.894 DEBUG:boto:Params: {} 2022-09-03T07:28:01.895 DEBUG:boto:Token: None 2022-09-03T07:28:01.896 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34 2022-09-03T07:28:01.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CWrvGgiq4UnZXEG8R+fRH1jTMF4= 2022-09-03T07:28:01.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CWrvGgiq4UnZXEG8R+fRH1jTMF4=', 'Content-Length': '0'} 2022-09-03T07:28:01.899 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005aee647b4655a211-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.900 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:28:01.900 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.901 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.901 DEBUG:boto:Method: GET 2022-09-03T07:28:01.902 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:01.902 DEBUG:boto:Data: 2022-09-03T07:28:01.903 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.904 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.904 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.905 DEBUG:boto:Params: {} 2022-09-03T07:28:01.905 DEBUG:boto:Token: None 2022-09-03T07:28:01.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34 2022-09-03T07:28:01.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4jYWP/AjKa75CxDJ7hy0zKJ9M8= 2022-09-03T07:28:01.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4jYWP/AjKa75CxDJ7hy0zKJ9M8=', 'Content-Length': '0'} 2022-09-03T07:28:01.913 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d1283c0c0c6eccb-0063130201-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.915 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:01.916 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:01.917 DEBUG:boto:Method: GET 2022-09-03T07:28:01.918 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:01.919 DEBUG:boto:Data: 2022-09-03T07:28:01.920 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:01.923 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.924 DEBUG:boto:Params: {} 2022-09-03T07:28:01.925 DEBUG:boto:Token: None 2022-09-03T07:28:01.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/obj34 2022-09-03T07:28:01.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4jYWP/AjKa75CxDJ7hy0zKJ9M8= 2022-09-03T07:28:01.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4jYWP/AjKa75CxDJ7hy0zKJ9M8=', 'Content-Length': '0'} 2022-09-03T07:28:01.931 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005db73ef65c31bd7a-0063130201-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.978 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.979 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.979 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:01.980 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:01.980 DEBUG:boto:Method: GET 2022-09-03T07:28:01.981 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:01.981 DEBUG:boto:Data: 2022-09-03T07:28:01.982 DEBUG:boto:Headers: {} 2022-09-03T07:28:01.983 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:01.983 DEBUG:boto:Port: 8000 2022-09-03T07:28:01.984 DEBUG:boto:Params: {} 2022-09-03T07:28:01.985 DEBUG:boto:Token: None 2022-09-03T07:28:01.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:01 GMT /avysmw-75/ 2022-09-03T07:28:01.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY= 2022-09-03T07:28:01.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WRtl7n/ygBeyyR9enMD6SPmPSYY=', 'Content-Length': '0'} 2022-09-03T07:28:01.993 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aadd15c4860e94c0-0063130201-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:01.994 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:01.996 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:01.997 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:01.998 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:01.999 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:02.000 DEBUG:boto:Method: GET 2022-09-03T07:28:02.001 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:02.001 DEBUG:boto:Data: 2022-09-03T07:28:02.003 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.004 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.004 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.005 DEBUG:boto:Params: {} 2022-09-03T07:28:02.006 DEBUG:boto:Token: None 2022-09-03T07:28:02.007 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.011 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.016 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bff07859891baa16-0063130202-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.017 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.018 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:28:02.019 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.019 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.020 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.021 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.021 DEBUG:boto:Method: GET 2022-09-03T07:28:02.022 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.023 DEBUG:boto:Data: 2022-09-03T07:28:02.025 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.030 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.031 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.032 DEBUG:boto:Params: {} 2022-09-03T07:28:02.033 DEBUG:boto:Token: None 2022-09-03T07:28:02.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7pekPPBFwtutfutyw19bAIPdRuc= 2022-09-03T07:28:02.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7pekPPBFwtutfutyw19bAIPdRuc=', 'Content-Length': '0'} 2022-09-03T07:28:02.039 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005c7aaed582ef715-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.041 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.042 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.043 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.044 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.044 DEBUG:boto:Method: GET 2022-09-03T07:28:02.046 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.047 DEBUG:boto:Data: 2022-09-03T07:28:02.048 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.049 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.050 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.051 DEBUG:boto:Params: {} 2022-09-03T07:28:02.052 DEBUG:boto:Token: None 2022-09-03T07:28:02.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7pekPPBFwtutfutyw19bAIPdRuc= 2022-09-03T07:28:02.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7pekPPBFwtutfutyw19bAIPdRuc=', 'Content-Length': '0'} 2022-09-03T07:28:02.057 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000302c5ddf20dcf42b-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.059 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.060 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.061 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.062 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.063 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.065 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.066 DEBUG:boto:Data: 2022-09-03T07:28:02.067 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.068 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.068 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.069 DEBUG:boto:Params: {} 2022-09-03T07:28:02.070 DEBUG:boto:Token: None 2022-09-03T07:28:02.071 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5pKRyIjPEzeRcIsGtCCNyYKqL6g= 2022-09-03T07:28:02.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5pKRyIjPEzeRcIsGtCCNyYKqL6g=', 'Content-Length': '0'} 2022-09-03T07:28:02.074 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000191043db116d09a9-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.076 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.076 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.077 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.077 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.078 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.079 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.080 DEBUG:boto:Data: 2022-09-03T07:28:02.081 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.083 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.084 DEBUG:boto:Params: {} 2022-09-03T07:28:02.085 DEBUG:boto:Token: None 2022-09-03T07:28:02.086 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.087 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5pKRyIjPEzeRcIsGtCCNyYKqL6g= 2022-09-03T07:28:02.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5pKRyIjPEzeRcIsGtCCNyYKqL6g=', 'Content-Length': '0'} 2022-09-03T07:28:02.091 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c5e586bba859470-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.092 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:28:02.092 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.093 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.094 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.094 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.095 DEBUG:boto:Method: GET 2022-09-03T07:28:02.095 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.096 DEBUG:boto:Data: 2022-09-03T07:28:02.097 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.097 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.098 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.098 DEBUG:boto:Params: {} 2022-09-03T07:28:02.099 DEBUG:boto:Token: None 2022-09-03T07:28:02.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7pekPPBFwtutfutyw19bAIPdRuc= 2022-09-03T07:28:02.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7pekPPBFwtutfutyw19bAIPdRuc=', 'Content-Length': '0'} 2022-09-03T07:28:02.104 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000390585f7cf275f79-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.152 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.152 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.153 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.154 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.154 DEBUG:boto:Method: GET 2022-09-03T07:28:02.155 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.155 DEBUG:boto:Data: 2022-09-03T07:28:02.156 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.156 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.157 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.157 DEBUG:boto:Params: {} 2022-09-03T07:28:02.158 DEBUG:boto:Token: None 2022-09-03T07:28:02.159 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:02.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7pekPPBFwtutfutyw19bAIPdRuc= 2022-09-03T07:28:02.160 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7pekPPBFwtutfutyw19bAIPdRuc=', 'Content-Length': '0'} 2022-09-03T07:28:02.162 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009dcfec5a840f0f13-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.163 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.164 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.164 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:02.165 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:02.165 DEBUG:boto:Method: GET 2022-09-03T07:28:02.166 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:02.167 DEBUG:boto:Data: 2022-09-03T07:28:02.168 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.169 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.169 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.170 DEBUG:boto:Params: {} 2022-09-03T07:28:02.170 DEBUG:boto:Token: None 2022-09-03T07:28:02.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.175 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff126a8a27db834f-0063130202-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.176 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.177 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.179 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.179 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:02.180 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:02.181 DEBUG:boto:Method: GET 2022-09-03T07:28:02.182 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:02.183 DEBUG:boto:Data: 2022-09-03T07:28:02.185 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.186 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.187 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.188 DEBUG:boto:Params: {} 2022-09-03T07:28:02.189 DEBUG:boto:Token: None 2022-09-03T07:28:02.190 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.195 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000919a43ee075ccdb0-0063130202-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.196 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.197 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.198 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.198 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.199 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:02.199 DEBUG:boto:Data: 2022-09-03T07:28:02.200 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.201 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.201 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.202 DEBUG:boto:Params: {} 2022-09-03T07:28:02.203 DEBUG:boto:Token: None 2022-09-03T07:28:02.203 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38 2022-09-03T07:28:02.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pdsxjLsGUfwMhp9B6IWRxY6+w0Y= 2022-09-03T07:28:02.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pdsxjLsGUfwMhp9B6IWRxY6+w0Y=', 'Content-Length': '0'} 2022-09-03T07:28:02.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006c37ae50464d0a5e-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.208 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.208 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.209 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.209 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:02.210 DEBUG:boto:Data: 2022-09-03T07:28:02.210 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.211 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.212 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.213 DEBUG:boto:Params: {} 2022-09-03T07:28:02.214 DEBUG:boto:Token: None 2022-09-03T07:28:02.215 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38 2022-09-03T07:28:02.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pdsxjLsGUfwMhp9B6IWRxY6+w0Y= 2022-09-03T07:28:02.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pdsxjLsGUfwMhp9B6IWRxY6+w0Y=', 'Content-Length': '0'} 2022-09-03T07:28:02.220 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ed099c366fd4755-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.221 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:28:02.223 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.224 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.225 DEBUG:boto:Method: GET 2022-09-03T07:28:02.226 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:02.227 DEBUG:boto:Data: 2022-09-03T07:28:02.228 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.229 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.230 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.231 DEBUG:boto:Params: {} 2022-09-03T07:28:02.234 DEBUG:boto:Token: None 2022-09-03T07:28:02.235 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38 2022-09-03T07:28:02.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3mQjXo/YV1Rru/egaooSQoqICJw= 2022-09-03T07:28:02.236 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3mQjXo/YV1Rru/egaooSQoqICJw=', 'Content-Length': '0'} 2022-09-03T07:28:02.238 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000058d29c3072cf5939-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.283 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:02.284 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:02.285 DEBUG:boto:Method: GET 2022-09-03T07:28:02.286 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:02.286 DEBUG:boto:Data: 2022-09-03T07:28:02.287 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.289 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.290 DEBUG:boto:Params: {} 2022-09-03T07:28:02.290 DEBUG:boto:Token: None 2022-09-03T07:28:02.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj38 2022-09-03T07:28:02.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3mQjXo/YV1Rru/egaooSQoqICJw= 2022-09-03T07:28:02.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3mQjXo/YV1Rru/egaooSQoqICJw=', 'Content-Length': '0'} 2022-09-03T07:28:02.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063be7af6cdfd5d4e-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.296 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.297 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.298 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:02.301 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:02.302 DEBUG:boto:Method: GET 2022-09-03T07:28:02.303 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:02.305 DEBUG:boto:Data: 2022-09-03T07:28:02.306 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.307 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.308 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.309 DEBUG:boto:Params: {} 2022-09-03T07:28:02.310 DEBUG:boto:Token: None 2022-09-03T07:28:02.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.314 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.315 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.320 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000cdddb6e3f6b06a4-0063130202-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.323 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.324 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.325 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.326 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:02.327 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:02.328 DEBUG:boto:Method: GET 2022-09-03T07:28:02.329 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:02.330 DEBUG:boto:Data: 2022-09-03T07:28:02.331 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.333 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.334 DEBUG:boto:Params: {} 2022-09-03T07:28:02.335 DEBUG:boto:Token: None 2022-09-03T07:28:02.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.342 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000be5603dc08d9c3aa-0063130202-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.343 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.344 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:28:02.345 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.346 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.347 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.348 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.349 DEBUG:boto:Method: GET 2022-09-03T07:28:02.349 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.350 DEBUG:boto:Data: 2022-09-03T07:28:02.350 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.351 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.352 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.353 DEBUG:boto:Params: {} 2022-09-03T07:28:02.353 DEBUG:boto:Token: None 2022-09-03T07:28:02.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBa/HTtx6puVfmm7FVUemaQHuvg= 2022-09-03T07:28:02.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBa/HTtx6puVfmm7FVUemaQHuvg=', 'Content-Length': '0'} 2022-09-03T07:28:02.363 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000831aaef78d418c10-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.366 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.368 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.368 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.369 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.370 DEBUG:boto:Method: GET 2022-09-03T07:28:02.375 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.377 DEBUG:boto:Data: 2022-09-03T07:28:02.379 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.380 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.380 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.381 DEBUG:boto:Params: {} 2022-09-03T07:28:02.381 DEBUG:boto:Token: None 2022-09-03T07:28:02.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBa/HTtx6puVfmm7FVUemaQHuvg= 2022-09-03T07:28:02.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBa/HTtx6puVfmm7FVUemaQHuvg=', 'Content-Length': '0'} 2022-09-03T07:28:02.386 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f35b35cf04ea4697-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.387 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.388 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.389 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.389 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.390 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.390 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.391 DEBUG:boto:Data: 2022-09-03T07:28:02.391 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.392 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.392 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.394 DEBUG:boto:Params: {} 2022-09-03T07:28:02.395 DEBUG:boto:Token: None 2022-09-03T07:28:02.398 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n5Xpzk5q0pIEUe2KpVLuzZHT4wI= 2022-09-03T07:28:02.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n5Xpzk5q0pIEUe2KpVLuzZHT4wI=', 'Content-Length': '0'} 2022-09-03T07:28:02.402 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dfc45d4d2a8516eb-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.403 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.403 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.404 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.404 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.405 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.406 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.406 DEBUG:boto:Data: 2022-09-03T07:28:02.407 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.408 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.408 DEBUG:boto:Params: {} 2022-09-03T07:28:02.409 DEBUG:boto:Token: None 2022-09-03T07:28:02.409 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n5Xpzk5q0pIEUe2KpVLuzZHT4wI= 2022-09-03T07:28:02.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n5Xpzk5q0pIEUe2KpVLuzZHT4wI=', 'Content-Length': '0'} 2022-09-03T07:28:02.419 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008be6bfc184d5cd7a-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.421 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:28:02.422 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.422 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.423 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.423 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.424 DEBUG:boto:Method: GET 2022-09-03T07:28:02.425 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.425 DEBUG:boto:Data: 2022-09-03T07:28:02.426 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.426 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.427 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.427 DEBUG:boto:Params: {} 2022-09-03T07:28:02.429 DEBUG:boto:Token: None 2022-09-03T07:28:02.430 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBa/HTtx6puVfmm7FVUemaQHuvg= 2022-09-03T07:28:02.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBa/HTtx6puVfmm7FVUemaQHuvg=', 'Content-Length': '0'} 2022-09-03T07:28:02.434 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e58a427f1a01f34b-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.479 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.483 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.484 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.485 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.486 DEBUG:boto:Method: GET 2022-09-03T07:28:02.487 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.488 DEBUG:boto:Data: 2022-09-03T07:28:02.489 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.491 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.492 DEBUG:boto:Params: {} 2022-09-03T07:28:02.493 DEBUG:boto:Token: None 2022-09-03T07:28:02.495 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:02.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sBa/HTtx6puVfmm7FVUemaQHuvg= 2022-09-03T07:28:02.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sBa/HTtx6puVfmm7FVUemaQHuvg=', 'Content-Length': '0'} 2022-09-03T07:28:02.501 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000006b05065a81b7659-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.502 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.503 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.504 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:02.505 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:02.506 DEBUG:boto:Method: GET 2022-09-03T07:28:02.506 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:02.507 DEBUG:boto:Data: 2022-09-03T07:28:02.507 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.508 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.509 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.510 DEBUG:boto:Params: {} 2022-09-03T07:28:02.515 DEBUG:boto:Token: None 2022-09-03T07:28:02.516 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.520 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c0246f222f69db67-0063130202-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.523 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.524 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.524 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.525 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:02.525 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:02.526 DEBUG:boto:Method: GET 2022-09-03T07:28:02.526 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:02.527 DEBUG:boto:Data: 2022-09-03T07:28:02.527 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.528 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.528 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.529 DEBUG:boto:Params: {} 2022-09-03T07:28:02.530 DEBUG:boto:Token: None 2022-09-03T07:28:02.530 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.532 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.536 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e641bee533b41b83-0063130202-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.538 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.539 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.540 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.541 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.542 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:02.543 DEBUG:boto:Data: 2022-09-03T07:28:02.544 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.545 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.546 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.547 DEBUG:boto:Params: {} 2022-09-03T07:28:02.548 DEBUG:boto:Token: None 2022-09-03T07:28:02.549 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44 2022-09-03T07:28:02.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOVNYgX6x6U2EtVQob2YFSzdDiE= 2022-09-03T07:28:02.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOVNYgX6x6U2EtVQob2YFSzdDiE=', 'Content-Length': '0'} 2022-09-03T07:28:02.554 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a7976f6befcd3250-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.555 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.556 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.557 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.557 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:02.558 DEBUG:boto:Data: 2022-09-03T07:28:02.558 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.559 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.560 DEBUG:boto:Params: {} 2022-09-03T07:28:02.560 DEBUG:boto:Token: None 2022-09-03T07:28:02.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44 2022-09-03T07:28:02.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOVNYgX6x6U2EtVQob2YFSzdDiE= 2022-09-03T07:28:02.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOVNYgX6x6U2EtVQob2YFSzdDiE=', 'Content-Length': '0'} 2022-09-03T07:28:02.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006bf5b48570683612-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.565 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:28:02.566 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.566 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.567 DEBUG:boto:Method: GET 2022-09-03T07:28:02.567 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:02.568 DEBUG:boto:Data: 2022-09-03T07:28:02.568 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.569 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.569 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.570 DEBUG:boto:Params: {} 2022-09-03T07:28:02.570 DEBUG:boto:Token: None 2022-09-03T07:28:02.571 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44 2022-09-03T07:28:02.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:os/7bJ/dvPxp5mvBfIRf+3qM8ps= 2022-09-03T07:28:02.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:os/7bJ/dvPxp5mvBfIRf+3qM8ps=', 'Content-Length': '0'} 2022-09-03T07:28:02.574 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000157110000ddef7a4-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.575 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:02.576 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:02.576 DEBUG:boto:Method: GET 2022-09-03T07:28:02.577 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:02.577 DEBUG:boto:Data: 2022-09-03T07:28:02.578 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.578 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.579 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.579 DEBUG:boto:Params: {} 2022-09-03T07:28:02.580 DEBUG:boto:Token: None 2022-09-03T07:28:02.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj44 2022-09-03T07:28:02.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:os/7bJ/dvPxp5mvBfIRf+3qM8ps= 2022-09-03T07:28:02.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:os/7bJ/dvPxp5mvBfIRf+3qM8ps=', 'Content-Length': '0'} 2022-09-03T07:28:02.585 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000094c45df17cd82072-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.586 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.586 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.587 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:02.587 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:02.588 DEBUG:boto:Method: GET 2022-09-03T07:28:02.588 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:02.589 DEBUG:boto:Data: 2022-09-03T07:28:02.589 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.590 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.590 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.592 DEBUG:boto:Params: {} 2022-09-03T07:28:02.593 DEBUG:boto:Token: None 2022-09-03T07:28:02.594 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.600 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004bdbf47f7bb33fc3-0063130202-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.601 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.603 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.604 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.605 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:02.606 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:02.607 DEBUG:boto:Method: GET 2022-09-03T07:28:02.608 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:02.609 DEBUG:boto:Data: 2022-09-03T07:28:02.610 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.612 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.614 DEBUG:boto:Params: {} 2022-09-03T07:28:02.617 DEBUG:boto:Token: None 2022-09-03T07:28:02.618 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.625 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000809a9d734b45abb2-0063130202-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.626 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.628 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:28:02.629 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.629 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.630 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.630 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.631 DEBUG:boto:Method: GET 2022-09-03T07:28:02.632 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.632 DEBUG:boto:Data: 2022-09-03T07:28:02.633 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.633 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.634 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.634 DEBUG:boto:Params: {} 2022-09-03T07:28:02.635 DEBUG:boto:Token: None 2022-09-03T07:28:02.636 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.636 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ETgqJI/drq9KUzhMppPdXKMZCvM= 2022-09-03T07:28:02.637 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ETgqJI/drq9KUzhMppPdXKMZCvM=', 'Content-Length': '0'} 2022-09-03T07:28:02.639 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2cae310e0d9c870-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.640 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.641 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.641 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.642 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.642 DEBUG:boto:Method: GET 2022-09-03T07:28:02.643 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.644 DEBUG:boto:Data: 2022-09-03T07:28:02.644 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.645 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.645 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.646 DEBUG:boto:Params: {} 2022-09-03T07:28:02.647 DEBUG:boto:Token: None 2022-09-03T07:28:02.647 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ETgqJI/drq9KUzhMppPdXKMZCvM= 2022-09-03T07:28:02.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ETgqJI/drq9KUzhMppPdXKMZCvM=', 'Content-Length': '0'} 2022-09-03T07:28:02.651 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000fe3bc42a12ac4ab-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.698 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.699 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.700 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.701 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.702 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.702 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.703 DEBUG:boto:Data: 2022-09-03T07:28:02.703 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.704 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.707 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.708 DEBUG:boto:Params: {} 2022-09-03T07:28:02.710 DEBUG:boto:Token: None 2022-09-03T07:28:02.711 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N6UlZakL14BHoUqUZMlpGVBAjC0= 2022-09-03T07:28:02.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N6UlZakL14BHoUqUZMlpGVBAjC0=', 'Content-Length': '0'} 2022-09-03T07:28:02.716 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aae2938556feb190-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.717 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.718 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.719 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.720 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.721 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.722 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.724 DEBUG:boto:Data: 2022-09-03T07:28:02.725 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.726 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.728 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.728 DEBUG:boto:Params: {} 2022-09-03T07:28:02.729 DEBUG:boto:Token: None 2022-09-03T07:28:02.730 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N6UlZakL14BHoUqUZMlpGVBAjC0= 2022-09-03T07:28:02.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N6UlZakL14BHoUqUZMlpGVBAjC0=', 'Content-Length': '0'} 2022-09-03T07:28:02.736 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003b6c7739ae60758-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.736 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:28:02.737 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.738 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.738 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.739 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.739 DEBUG:boto:Method: GET 2022-09-03T07:28:02.740 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.740 DEBUG:boto:Data: 2022-09-03T07:28:02.741 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.741 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.742 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.742 DEBUG:boto:Params: {} 2022-09-03T07:28:02.743 DEBUG:boto:Token: None 2022-09-03T07:28:02.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ETgqJI/drq9KUzhMppPdXKMZCvM= 2022-09-03T07:28:02.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ETgqJI/drq9KUzhMppPdXKMZCvM=', 'Content-Length': '0'} 2022-09-03T07:28:02.748 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d968b5516d37467-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.749 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.750 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.751 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.751 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.753 DEBUG:boto:Method: GET 2022-09-03T07:28:02.753 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.754 DEBUG:boto:Data: 2022-09-03T07:28:02.754 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.755 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.756 DEBUG:boto:Params: {} 2022-09-03T07:28:02.757 DEBUG:boto:Token: None 2022-09-03T07:28:02.758 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:02.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ETgqJI/drq9KUzhMppPdXKMZCvM= 2022-09-03T07:28:02.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ETgqJI/drq9KUzhMppPdXKMZCvM=', 'Content-Length': '0'} 2022-09-03T07:28:02.762 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5372254f88528d2-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.763 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.764 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.764 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:02.765 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:02.765 DEBUG:boto:Method: GET 2022-09-03T07:28:02.766 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:02.767 DEBUG:boto:Data: 2022-09-03T07:28:02.768 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.769 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.769 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.770 DEBUG:boto:Params: {} 2022-09-03T07:28:02.771 DEBUG:boto:Token: None 2022-09-03T07:28:02.771 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.775 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000669b7a9cb8b1383a-0063130202-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.776 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.777 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.777 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.778 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:02.779 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:02.780 DEBUG:boto:Method: GET 2022-09-03T07:28:02.780 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:02.781 DEBUG:boto:Data: 2022-09-03T07:28:02.782 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.783 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.784 DEBUG:boto:Params: {} 2022-09-03T07:28:02.785 DEBUG:boto:Token: None 2022-09-03T07:28:02.786 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.794 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b4751d0817c0775-0063130202-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.840 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.841 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.842 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.843 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.843 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:02.844 DEBUG:boto:Data: 2022-09-03T07:28:02.844 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.845 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.845 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.847 DEBUG:boto:Params: {} 2022-09-03T07:28:02.847 DEBUG:boto:Token: None 2022-09-03T07:28:02.848 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46 2022-09-03T07:28:02.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ggWD2HAPaY2GsOs0/bv62Uv6yao= 2022-09-03T07:28:02.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ggWD2HAPaY2GsOs0/bv62Uv6yao=', 'Content-Length': '0'} 2022-09-03T07:28:02.852 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000028bd04d0801ac2da-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.853 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.853 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.855 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.856 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:02.856 DEBUG:boto:Data: 2022-09-03T07:28:02.857 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.858 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.858 DEBUG:boto:Params: {} 2022-09-03T07:28:02.862 DEBUG:boto:Token: None 2022-09-03T07:28:02.863 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46 2022-09-03T07:28:02.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ggWD2HAPaY2GsOs0/bv62Uv6yao= 2022-09-03T07:28:02.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ggWD2HAPaY2GsOs0/bv62Uv6yao=', 'Content-Length': '0'} 2022-09-03T07:28:02.867 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000517fc71f71d607cc-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.867 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:28:02.868 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.868 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.869 DEBUG:boto:Method: GET 2022-09-03T07:28:02.870 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:02.870 DEBUG:boto:Data: 2022-09-03T07:28:02.871 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.872 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.873 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.873 DEBUG:boto:Params: {} 2022-09-03T07:28:02.874 DEBUG:boto:Token: None 2022-09-03T07:28:02.875 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46 2022-09-03T07:28:02.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1RWzn7Xt4G8WgKBBq3PjLFTUNNE= 2022-09-03T07:28:02.876 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1RWzn7Xt4G8WgKBBq3PjLFTUNNE=', 'Content-Length': '0'} 2022-09-03T07:28:02.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099cbaf12af79bdca-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.879 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:02.880 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:02.881 DEBUG:boto:Method: GET 2022-09-03T07:28:02.881 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:02.882 DEBUG:boto:Data: 2022-09-03T07:28:02.883 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.885 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.886 DEBUG:boto:Params: {} 2022-09-03T07:28:02.887 DEBUG:boto:Token: None 2022-09-03T07:28:02.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj46 2022-09-03T07:28:02.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1RWzn7Xt4G8WgKBBq3PjLFTUNNE= 2022-09-03T07:28:02.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1RWzn7Xt4G8WgKBBq3PjLFTUNNE=', 'Content-Length': '0'} 2022-09-03T07:28:02.895 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c22e21dc3cacb542-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.896 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.897 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.897 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:02.898 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:02.898 DEBUG:boto:Method: GET 2022-09-03T07:28:02.899 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:02.899 DEBUG:boto:Data: 2022-09-03T07:28:02.900 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.901 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.901 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.902 DEBUG:boto:Params: {} 2022-09-03T07:28:02.903 DEBUG:boto:Token: None 2022-09-03T07:28:02.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.909 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000983129c8a69a56c1-0063130202-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.910 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.911 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:02.912 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:02.912 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:02.913 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:02.913 DEBUG:boto:Method: GET 2022-09-03T07:28:02.914 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:02.915 DEBUG:boto:Data: 2022-09-03T07:28:02.915 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.916 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.916 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.917 DEBUG:boto:Params: {} 2022-09-03T07:28:02.918 DEBUG:boto:Token: None 2022-09-03T07:28:02.919 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/ 2022-09-03T07:28:02.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE= 2022-09-03T07:28:02.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vWB/T9vJ34bYVOWOM0v/w3gHqpE=', 'Content-Length': '0'} 2022-09-03T07:28:02.930 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ee095e4d9a0eb1c-0063130202-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.966 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:02.967 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:28:02.968 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:02.968 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:02.969 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.969 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.970 DEBUG:boto:Method: GET 2022-09-03T07:28:02.971 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.971 DEBUG:boto:Data: 2022-09-03T07:28:02.972 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.972 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:02.973 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.973 DEBUG:boto:Params: {} 2022-09-03T07:28:02.974 DEBUG:boto:Token: None 2022-09-03T07:28:02.975 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NGUvnkb5aqPouAuVtiOcFblEMd4= 2022-09-03T07:28:02.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NGUvnkb5aqPouAuVtiOcFblEMd4=', 'Content-Length': '0'} 2022-09-03T07:28:02.978 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8bff59b622f0044-0063130202-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.980 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:02.980 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:02.981 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.982 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.982 DEBUG:boto:Method: GET 2022-09-03T07:28:02.983 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.983 DEBUG:boto:Data: 2022-09-03T07:28:02.985 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.986 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:02.986 DEBUG:boto:Port: 8000 2022-09-03T07:28:02.987 DEBUG:boto:Params: {} 2022-09-03T07:28:02.987 DEBUG:boto:Token: None 2022-09-03T07:28:02.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:02 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NGUvnkb5aqPouAuVtiOcFblEMd4= 2022-09-03T07:28:02.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NGUvnkb5aqPouAuVtiOcFblEMd4=', 'Content-Length': '0'} 2022-09-03T07:28:02.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003dc2117ba90c447d-0063130202-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:02.994 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:02.995 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:02.996 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.996 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.997 DEBUG:boto:Method: HEAD 2022-09-03T07:28:02.998 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:02.998 DEBUG:boto:Data: 2022-09-03T07:28:02.999 DEBUG:boto:Headers: {} 2022-09-03T07:28:02.999 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.000 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.000 DEBUG:boto:Params: {} 2022-09-03T07:28:03.001 DEBUG:boto:Token: None 2022-09-03T07:28:03.001 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.002 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bSY9VqQjKGH8aVwICKQRtMlH7k8= 2022-09-03T07:28:03.003 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bSY9VqQjKGH8aVwICKQRtMlH7k8=', 'Content-Length': '0'} 2022-09-03T07:28:03.004 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1beb0ca75997d8b-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.005 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:03.006 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:03.006 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.007 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.007 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.008 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.008 DEBUG:boto:Data: 2022-09-03T07:28:03.009 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.010 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.010 DEBUG:boto:Params: {} 2022-09-03T07:28:03.011 DEBUG:boto:Token: None 2022-09-03T07:28:03.012 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bSY9VqQjKGH8aVwICKQRtMlH7k8= 2022-09-03T07:28:03.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bSY9VqQjKGH8aVwICKQRtMlH7k8=', 'Content-Length': '0'} 2022-09-03T07:28:03.015 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba8bf6f08bd53f8e-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.016 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:28:03.017 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:03.018 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:03.018 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.019 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.019 DEBUG:boto:Method: GET 2022-09-03T07:28:03.020 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.020 DEBUG:boto:Data: 2022-09-03T07:28:03.021 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.021 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.022 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.022 DEBUG:boto:Params: {} 2022-09-03T07:28:03.023 DEBUG:boto:Token: None 2022-09-03T07:28:03.024 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6t/7wMWrEyBFC0bjuX+zlji4csc= 2022-09-03T07:28:03.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6t/7wMWrEyBFC0bjuX+zlji4csc=', 'Content-Length': '0'} 2022-09-03T07:28:03.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f584ee7cfe7ab51b-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.070 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:03.071 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:03.071 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.072 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.072 DEBUG:boto:Method: GET 2022-09-03T07:28:03.073 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.073 DEBUG:boto:Data: 2022-09-03T07:28:03.075 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.076 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.076 DEBUG:boto:Params: {} 2022-09-03T07:28:03.077 DEBUG:boto:Token: None 2022-09-03T07:28:03.077 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:03.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6t/7wMWrEyBFC0bjuX+zlji4csc= 2022-09-03T07:28:03.079 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6t/7wMWrEyBFC0bjuX+zlji4csc=', 'Content-Length': '0'} 2022-09-03T07:28:03.081 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000088b8161fb88adb62-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.084 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.085 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.085 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:03.086 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:03.087 DEBUG:boto:Method: GET 2022-09-03T07:28:03.087 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:03.088 DEBUG:boto:Data: 2022-09-03T07:28:03.088 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.089 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.089 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.090 DEBUG:boto:Params: {} 2022-09-03T07:28:03.090 DEBUG:boto:Token: None 2022-09-03T07:28:03.091 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.093 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.095 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000135fe6c7acd121b0-0063130203-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.096 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.097 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.098 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.099 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:03.099 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:03.100 DEBUG:boto:Method: GET 2022-09-03T07:28:03.101 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:03.102 DEBUG:boto:Data: 2022-09-03T07:28:03.103 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.104 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.105 DEBUG:boto:Params: {} 2022-09-03T07:28:03.106 DEBUG:boto:Token: None 2022-09-03T07:28:03.106 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.107 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.113 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c213463bf69c66a-0063130203-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.114 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.115 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:03.116 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:03.116 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.117 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:03.118 DEBUG:boto:Data: 2022-09-03T07:28:03.119 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.119 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.120 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.121 DEBUG:boto:Params: {} 2022-09-03T07:28:03.122 DEBUG:boto:Token: None 2022-09-03T07:28:03.123 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj51 2022-09-03T07:28:03.124 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4tXMlIid25m/b0PXIc8q98XCMsA= 2022-09-03T07:28:03.125 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4tXMlIid25m/b0PXIc8q98XCMsA=', 'Content-Length': '0'} 2022-09-03T07:28:03.128 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e3efef29aad82ec-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.129 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:03.130 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:03.132 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.133 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:03.134 DEBUG:boto:Data: 2022-09-03T07:28:03.134 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.135 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.136 DEBUG:boto:Params: {} 2022-09-03T07:28:03.137 DEBUG:boto:Token: None 2022-09-03T07:28:03.138 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj51 2022-09-03T07:28:03.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4tXMlIid25m/b0PXIc8q98XCMsA= 2022-09-03T07:28:03.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4tXMlIid25m/b0PXIc8q98XCMsA=', 'Content-Length': '0'} 2022-09-03T07:28:03.142 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000410d13887ec7c847-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.143 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:28:03.144 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:03.145 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:03.145 DEBUG:boto:Method: GET 2022-09-03T07:28:03.147 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:03.148 DEBUG:boto:Data: 2022-09-03T07:28:03.149 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.150 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.150 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.151 DEBUG:boto:Params: {} 2022-09-03T07:28:03.152 DEBUG:boto:Token: None 2022-09-03T07:28:03.152 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj51 2022-09-03T07:28:03.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Qb+98tAjX4pYs8URrTtz/XIgGs= 2022-09-03T07:28:03.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Qb+98tAjX4pYs8URrTtz/XIgGs=', 'Content-Length': '0'} 2022-09-03T07:28:03.156 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008307eb636dc0058f-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.157 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:03.157 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:03.158 DEBUG:boto:Method: GET 2022-09-03T07:28:03.159 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:03.160 DEBUG:boto:Data: 2022-09-03T07:28:03.160 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.161 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.161 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.162 DEBUG:boto:Params: {} 2022-09-03T07:28:03.163 DEBUG:boto:Token: None 2022-09-03T07:28:03.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj51 2022-09-03T07:28:03.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2Qb+98tAjX4pYs8URrTtz/XIgGs= 2022-09-03T07:28:03.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2Qb+98tAjX4pYs8URrTtz/XIgGs=', 'Content-Length': '0'} 2022-09-03T07:28:03.168 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000344b2ee616bbf96b-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.210 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.211 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.212 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:03.213 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:03.214 DEBUG:boto:Method: GET 2022-09-03T07:28:03.215 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:03.215 DEBUG:boto:Data: 2022-09-03T07:28:03.216 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.216 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.217 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.218 DEBUG:boto:Params: {} 2022-09-03T07:28:03.218 DEBUG:boto:Token: None 2022-09-03T07:28:03.219 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.224 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f3dec95dff751b74-0063130203-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.225 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.227 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.228 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.229 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:03.229 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:03.230 DEBUG:boto:Method: GET 2022-09-03T07:28:03.234 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:03.235 DEBUG:boto:Data: 2022-09-03T07:28:03.236 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.238 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.239 DEBUG:boto:Params: {} 2022-09-03T07:28:03.240 DEBUG:boto:Token: None 2022-09-03T07:28:03.241 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.244 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.250 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008ac00b3ee146d713-0063130203-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.252 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.253 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:28:03.255 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.257 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.262 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.263 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.264 DEBUG:boto:Method: GET 2022-09-03T07:28:03.265 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.266 DEBUG:boto:Data: 2022-09-03T07:28:03.267 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.268 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.269 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.270 DEBUG:boto:Params: {} 2022-09-03T07:28:03.271 DEBUG:boto:Token: None 2022-09-03T07:28:03.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wz2/549tlxlxLs/YX79ywwQWTOw= 2022-09-03T07:28:03.273 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wz2/549tlxlxLs/YX79ywwQWTOw=', 'Content-Length': '0'} 2022-09-03T07:28:03.278 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f1e1ce8a7805a64-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.280 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.281 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.281 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.282 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.283 DEBUG:boto:Method: GET 2022-09-03T07:28:03.284 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.285 DEBUG:boto:Data: 2022-09-03T07:28:03.285 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.286 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.287 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.288 DEBUG:boto:Params: {} 2022-09-03T07:28:03.289 DEBUG:boto:Token: None 2022-09-03T07:28:03.290 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wz2/549tlxlxLs/YX79ywwQWTOw= 2022-09-03T07:28:03.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wz2/549tlxlxLs/YX79ywwQWTOw=', 'Content-Length': '0'} 2022-09-03T07:28:03.294 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a37a6bade67f9d86-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.295 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.295 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.296 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.296 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.297 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.297 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.298 DEBUG:boto:Data: 2022-09-03T07:28:03.299 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.299 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.300 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.301 DEBUG:boto:Params: {} 2022-09-03T07:28:03.302 DEBUG:boto:Token: None 2022-09-03T07:28:03.303 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tuQC+A23mqyotvkng9F7W/1St18= 2022-09-03T07:28:03.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tuQC+A23mqyotvkng9F7W/1St18=', 'Content-Length': '0'} 2022-09-03T07:28:03.308 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f51aadc580ca833-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.309 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.309 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.310 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.310 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.311 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.311 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.312 DEBUG:boto:Data: 2022-09-03T07:28:03.312 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.313 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.314 DEBUG:boto:Params: {} 2022-09-03T07:28:03.314 DEBUG:boto:Token: None 2022-09-03T07:28:03.315 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tuQC+A23mqyotvkng9F7W/1St18= 2022-09-03T07:28:03.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tuQC+A23mqyotvkng9F7W/1St18=', 'Content-Length': '0'} 2022-09-03T07:28:03.319 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba6e3303c7f89793-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.320 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:28:03.320 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.321 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.321 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.322 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.322 DEBUG:boto:Method: GET 2022-09-03T07:28:03.323 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.323 DEBUG:boto:Data: 2022-09-03T07:28:03.324 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.324 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.325 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.325 DEBUG:boto:Params: {} 2022-09-03T07:28:03.326 DEBUG:boto:Token: None 2022-09-03T07:28:03.327 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wz2/549tlxlxLs/YX79ywwQWTOw= 2022-09-03T07:28:03.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wz2/549tlxlxLs/YX79ywwQWTOw=', 'Content-Length': '0'} 2022-09-03T07:28:03.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc8c4d56ec254273-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.332 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.333 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.334 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.334 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.335 DEBUG:boto:Method: GET 2022-09-03T07:28:03.335 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.336 DEBUG:boto:Data: 2022-09-03T07:28:03.336 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.337 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.338 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.338 DEBUG:boto:Params: {} 2022-09-03T07:28:03.339 DEBUG:boto:Token: None 2022-09-03T07:28:03.340 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:03.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wz2/549tlxlxLs/YX79ywwQWTOw= 2022-09-03T07:28:03.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wz2/549tlxlxLs/YX79ywwQWTOw=', 'Content-Length': '0'} 2022-09-03T07:28:03.344 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037fc3fa5c37d14b5-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.390 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.391 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.392 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:03.393 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:03.394 DEBUG:boto:Method: GET 2022-09-03T07:28:03.395 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:03.396 DEBUG:boto:Data: 2022-09-03T07:28:03.396 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.397 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.399 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.400 DEBUG:boto:Params: {} 2022-09-03T07:28:03.401 DEBUG:boto:Token: None 2022-09-03T07:28:03.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.408 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000035839da9d1ba4b54-0063130203-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.409 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.411 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.412 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.412 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:03.413 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:03.413 DEBUG:boto:Method: GET 2022-09-03T07:28:03.414 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:03.415 DEBUG:boto:Data: 2022-09-03T07:28:03.415 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.416 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.417 DEBUG:boto:Params: {} 2022-09-03T07:28:03.417 DEBUG:boto:Token: None 2022-09-03T07:28:03.418 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.433 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a5bbb4a0f5b350ba-0063130203-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.434 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.435 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.436 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.437 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.438 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:03.439 DEBUG:boto:Data: 2022-09-03T07:28:03.440 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.441 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.442 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.443 DEBUG:boto:Params: {} 2022-09-03T07:28:03.444 DEBUG:boto:Token: None 2022-09-03T07:28:03.445 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55 2022-09-03T07:28:03.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yeOHaMBGxaZ9P2fnqTSpkp4gIOg= 2022-09-03T07:28:03.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yeOHaMBGxaZ9P2fnqTSpkp4gIOg=', 'Content-Length': '0'} 2022-09-03T07:28:03.450 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000202ae0353d20929c-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.452 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.453 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.454 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.455 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:03.456 DEBUG:boto:Data: 2022-09-03T07:28:03.457 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.458 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.459 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.460 DEBUG:boto:Params: {} 2022-09-03T07:28:03.462 DEBUG:boto:Token: None 2022-09-03T07:28:03.463 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55 2022-09-03T07:28:03.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yeOHaMBGxaZ9P2fnqTSpkp4gIOg= 2022-09-03T07:28:03.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yeOHaMBGxaZ9P2fnqTSpkp4gIOg=', 'Content-Length': '0'} 2022-09-03T07:28:03.468 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008777c8c87460f9f4-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.469 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:28:03.470 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.471 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.472 DEBUG:boto:Method: GET 2022-09-03T07:28:03.473 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:03.474 DEBUG:boto:Data: 2022-09-03T07:28:03.475 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.476 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.477 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.478 DEBUG:boto:Params: {} 2022-09-03T07:28:03.478 DEBUG:boto:Token: None 2022-09-03T07:28:03.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55 2022-09-03T07:28:03.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZkuEiWzkt3lnfo3G41JPIsCKQeU= 2022-09-03T07:28:03.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZkuEiWzkt3lnfo3G41JPIsCKQeU=', 'Content-Length': '0'} 2022-09-03T07:28:03.482 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000058d7b2b98ecff7d8-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.530 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:03.531 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:03.532 DEBUG:boto:Method: GET 2022-09-03T07:28:03.532 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:03.533 DEBUG:boto:Data: 2022-09-03T07:28:03.534 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.535 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.535 DEBUG:boto:Params: {} 2022-09-03T07:28:03.536 DEBUG:boto:Token: None 2022-09-03T07:28:03.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj55 2022-09-03T07:28:03.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZkuEiWzkt3lnfo3G41JPIsCKQeU= 2022-09-03T07:28:03.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZkuEiWzkt3lnfo3G41JPIsCKQeU=', 'Content-Length': '0'} 2022-09-03T07:28:03.542 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001aa24c32bfe87cb6-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.549 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.552 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.553 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:03.554 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:03.555 DEBUG:boto:Method: GET 2022-09-03T07:28:03.557 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:03.557 DEBUG:boto:Data: 2022-09-03T07:28:03.558 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.559 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.560 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.561 DEBUG:boto:Params: {} 2022-09-03T07:28:03.562 DEBUG:boto:Token: None 2022-09-03T07:28:03.563 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.563 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.572 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c306b370cdcd385-0063130203-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.573 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.575 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.576 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.577 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:03.578 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:03.580 DEBUG:boto:Method: GET 2022-09-03T07:28:03.580 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:03.581 DEBUG:boto:Data: 2022-09-03T07:28:03.582 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.584 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.585 DEBUG:boto:Params: {} 2022-09-03T07:28:03.586 DEBUG:boto:Token: None 2022-09-03T07:28:03.587 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b887fc926d90c451-0063130203-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.596 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.597 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:28:03.598 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.599 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.600 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.601 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.602 DEBUG:boto:Method: GET 2022-09-03T07:28:03.603 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.605 DEBUG:boto:Data: 2022-09-03T07:28:03.606 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.606 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.607 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.608 DEBUG:boto:Params: {} 2022-09-03T07:28:03.609 DEBUG:boto:Token: None 2022-09-03T07:28:03.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HHEjReqejMj3r8WoAuaSeIWrMjk= 2022-09-03T07:28:03.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HHEjReqejMj3r8WoAuaSeIWrMjk=', 'Content-Length': '0'} 2022-09-03T07:28:03.614 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002c97d05e9383898-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.616 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.617 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.618 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.619 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.619 DEBUG:boto:Method: GET 2022-09-03T07:28:03.620 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.621 DEBUG:boto:Data: 2022-09-03T07:28:03.622 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.623 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.624 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.625 DEBUG:boto:Params: {} 2022-09-03T07:28:03.626 DEBUG:boto:Token: None 2022-09-03T07:28:03.627 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HHEjReqejMj3r8WoAuaSeIWrMjk= 2022-09-03T07:28:03.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HHEjReqejMj3r8WoAuaSeIWrMjk=', 'Content-Length': '0'} 2022-09-03T07:28:03.631 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8cfa95a2fb30e6b-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.674 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.676 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.677 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.678 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.679 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.680 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.681 DEBUG:boto:Data: 2022-09-03T07:28:03.681 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.682 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.684 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.684 DEBUG:boto:Params: {} 2022-09-03T07:28:03.685 DEBUG:boto:Token: None 2022-09-03T07:28:03.685 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t1rrNHHZW1cf9aabSk099EwKdEU= 2022-09-03T07:28:03.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t1rrNHHZW1cf9aabSk099EwKdEU=', 'Content-Length': '0'} 2022-09-03T07:28:03.697 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9d42319254bb235-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.698 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.699 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.700 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.700 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.701 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.702 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.703 DEBUG:boto:Data: 2022-09-03T07:28:03.704 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.705 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.706 DEBUG:boto:Params: {} 2022-09-03T07:28:03.707 DEBUG:boto:Token: None 2022-09-03T07:28:03.708 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t1rrNHHZW1cf9aabSk099EwKdEU= 2022-09-03T07:28:03.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t1rrNHHZW1cf9aabSk099EwKdEU=', 'Content-Length': '0'} 2022-09-03T07:28:03.713 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b04cc4773ebc716c-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.714 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:28:03.715 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.716 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.717 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.718 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.719 DEBUG:boto:Method: GET 2022-09-03T07:28:03.720 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.720 DEBUG:boto:Data: 2022-09-03T07:28:03.721 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.721 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.722 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.723 DEBUG:boto:Params: {} 2022-09-03T07:28:03.723 DEBUG:boto:Token: None 2022-09-03T07:28:03.724 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HHEjReqejMj3r8WoAuaSeIWrMjk= 2022-09-03T07:28:03.725 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HHEjReqejMj3r8WoAuaSeIWrMjk=', 'Content-Length': '0'} 2022-09-03T07:28:03.728 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092f1278460f09c11-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.729 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.729 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.730 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.730 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.731 DEBUG:boto:Method: GET 2022-09-03T07:28:03.731 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.732 DEBUG:boto:Data: 2022-09-03T07:28:03.733 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.734 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.734 DEBUG:boto:Params: {} 2022-09-03T07:28:03.735 DEBUG:boto:Token: None 2022-09-03T07:28:03.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:03.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HHEjReqejMj3r8WoAuaSeIWrMjk= 2022-09-03T07:28:03.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HHEjReqejMj3r8WoAuaSeIWrMjk=', 'Content-Length': '0'} 2022-09-03T07:28:03.740 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f344847fe19f98aa-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.742 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.743 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.744 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:03.747 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:03.747 DEBUG:boto:Method: GET 2022-09-03T07:28:03.748 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:03.749 DEBUG:boto:Data: 2022-09-03T07:28:03.749 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.754 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.755 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.756 DEBUG:boto:Params: {} 2022-09-03T07:28:03.757 DEBUG:boto:Token: None 2022-09-03T07:28:03.758 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.764 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac7f410b7b146df3-0063130203-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.765 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.766 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.767 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.768 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:03.769 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:03.770 DEBUG:boto:Method: GET 2022-09-03T07:28:03.771 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:03.771 DEBUG:boto:Data: 2022-09-03T07:28:03.772 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.772 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.773 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.773 DEBUG:boto:Params: {} 2022-09-03T07:28:03.775 DEBUG:boto:Token: None 2022-09-03T07:28:03.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.783 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f59aea5544b54b47-0063130203-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.784 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.785 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.786 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.786 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.787 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:03.788 DEBUG:boto:Data: 2022-09-03T07:28:03.788 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.789 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.789 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.790 DEBUG:boto:Params: {} 2022-09-03T07:28:03.790 DEBUG:boto:Token: None 2022-09-03T07:28:03.791 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57 2022-09-03T07:28:03.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ooi0Fz5ZiSzMfCdF3Fu5lUytdLo= 2022-09-03T07:28:03.792 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ooi0Fz5ZiSzMfCdF3Fu5lUytdLo=', 'Content-Length': '0'} 2022-09-03T07:28:03.794 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dd253da8fb54c12d-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.795 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.796 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.796 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.797 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:03.797 DEBUG:boto:Data: 2022-09-03T07:28:03.798 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.799 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.799 DEBUG:boto:Params: {} 2022-09-03T07:28:03.800 DEBUG:boto:Token: None 2022-09-03T07:28:03.800 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57 2022-09-03T07:28:03.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ooi0Fz5ZiSzMfCdF3Fu5lUytdLo= 2022-09-03T07:28:03.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ooi0Fz5ZiSzMfCdF3Fu5lUytdLo=', 'Content-Length': '0'} 2022-09-03T07:28:03.804 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000094c9ab5eb4ff8a9f-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.805 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:28:03.805 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.806 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.806 DEBUG:boto:Method: GET 2022-09-03T07:28:03.807 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:03.807 DEBUG:boto:Data: 2022-09-03T07:28:03.808 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.809 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.809 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.810 DEBUG:boto:Params: {} 2022-09-03T07:28:03.810 DEBUG:boto:Token: None 2022-09-03T07:28:03.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57 2022-09-03T07:28:03.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3WNi6rkaKDGN8hWF1FLzcZMuHxs= 2022-09-03T07:28:03.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3WNi6rkaKDGN8hWF1FLzcZMuHxs=', 'Content-Length': '0'} 2022-09-03T07:28:03.814 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a505cf91ace9734-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.862 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:03.863 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:03.864 DEBUG:boto:Method: GET 2022-09-03T07:28:03.865 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:03.866 DEBUG:boto:Data: 2022-09-03T07:28:03.867 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.868 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.869 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.869 DEBUG:boto:Params: {} 2022-09-03T07:28:03.870 DEBUG:boto:Token: None 2022-09-03T07:28:03.871 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj57 2022-09-03T07:28:03.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3WNi6rkaKDGN8hWF1FLzcZMuHxs= 2022-09-03T07:28:03.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3WNi6rkaKDGN8hWF1FLzcZMuHxs=', 'Content-Length': '0'} 2022-09-03T07:28:03.875 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000043db623fe0b2310d-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.876 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.877 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.878 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:03.878 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:03.879 DEBUG:boto:Method: GET 2022-09-03T07:28:03.879 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:03.880 DEBUG:boto:Data: 2022-09-03T07:28:03.880 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.881 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.882 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.882 DEBUG:boto:Params: {} 2022-09-03T07:28:03.883 DEBUG:boto:Token: None 2022-09-03T07:28:03.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000021bafa45ee1e8e7b-0063130203-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.890 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.893 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:03.894 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:03.895 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:03.896 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:03.897 DEBUG:boto:Method: GET 2022-09-03T07:28:03.898 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:03.899 DEBUG:boto:Data: 2022-09-03T07:28:03.899 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.901 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.902 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.903 DEBUG:boto:Params: {} 2022-09-03T07:28:03.904 DEBUG:boto:Token: None 2022-09-03T07:28:03.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/ 2022-09-03T07:28:03.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE= 2022-09-03T07:28:03.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QaQG+3RKmK4Oa8IPu0T45mR5/TE=', 'Content-Length': '0'} 2022-09-03T07:28:03.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037df7a940253e763-0063130203-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.958 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:03.960 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:28:03.961 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:03.962 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:03.963 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.964 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.966 DEBUG:boto:Method: GET 2022-09-03T07:28:03.967 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.968 DEBUG:boto:Data: 2022-09-03T07:28:03.969 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.969 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.970 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.970 DEBUG:boto:Params: {} 2022-09-03T07:28:03.971 DEBUG:boto:Token: None 2022-09-03T07:28:03.972 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OwYaLBdjVSski2C69U1YBTPGBxU= 2022-09-03T07:28:03.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OwYaLBdjVSski2C69U1YBTPGBxU=', 'Content-Length': '0'} 2022-09-03T07:28:03.975 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec812c19f6ebd23e-0063130203-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.976 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:03.977 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:03.977 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.978 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.979 DEBUG:boto:Method: GET 2022-09-03T07:28:03.979 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.980 DEBUG:boto:Data: 2022-09-03T07:28:03.980 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.981 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:03.982 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.983 DEBUG:boto:Params: {} 2022-09-03T07:28:03.983 DEBUG:boto:Token: None 2022-09-03T07:28:03.984 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.985 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OwYaLBdjVSski2C69U1YBTPGBxU= 2022-09-03T07:28:03.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OwYaLBdjVSski2C69U1YBTPGBxU=', 'Content-Length': '0'} 2022-09-03T07:28:03.989 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000040a9347cc6c62a68-0063130203-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:03.990 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:03.991 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:03.992 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.993 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.994 DEBUG:boto:Method: HEAD 2022-09-03T07:28:03.995 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.995 DEBUG:boto:Data: 2022-09-03T07:28:03.996 DEBUG:boto:Headers: {} 2022-09-03T07:28:03.996 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:03.997 DEBUG:boto:Port: 8000 2022-09-03T07:28:03.997 DEBUG:boto:Params: {} 2022-09-03T07:28:03.998 DEBUG:boto:Token: None 2022-09-03T07:28:03.999 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:03 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:03.999 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6jIKCbOStTRdVKWhrL1/OuPkVdM= 2022-09-03T07:28:04.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6jIKCbOStTRdVKWhrL1/OuPkVdM=', 'Content-Length': '0'} 2022-09-03T07:28:04.003 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c7716fb2b31c33c-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.004 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:04.005 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:04.005 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.006 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.007 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.007 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.008 DEBUG:boto:Data: 2022-09-03T07:28:04.008 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.009 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.010 DEBUG:boto:Params: {} 2022-09-03T07:28:04.010 DEBUG:boto:Token: None 2022-09-03T07:28:04.011 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TcDtOd3OXw/d12/wrAe+g3YHsUs= 2022-09-03T07:28:04.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TcDtOd3OXw/d12/wrAe+g3YHsUs=', 'Content-Length': '0'} 2022-09-03T07:28:04.015 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001feef6f1037158b7-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.015 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:28:04.016 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:04.017 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:04.017 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.018 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.018 DEBUG:boto:Method: GET 2022-09-03T07:28:04.019 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.020 DEBUG:boto:Data: 2022-09-03T07:28:04.020 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.021 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.021 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.022 DEBUG:boto:Params: {} 2022-09-03T07:28:04.023 DEBUG:boto:Token: None 2022-09-03T07:28:04.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tK9HjnP2hl/sbaNUGy38djtlbbY= 2022-09-03T07:28:04.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tK9HjnP2hl/sbaNUGy38djtlbbY=', 'Content-Length': '0'} 2022-09-03T07:28:04.028 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3adb6bfb4bd86ef-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.032 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:04.033 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:04.034 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.034 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.035 DEBUG:boto:Method: GET 2022-09-03T07:28:04.036 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.037 DEBUG:boto:Data: 2022-09-03T07:28:04.038 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.040 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.041 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.042 DEBUG:boto:Params: {} 2022-09-03T07:28:04.043 DEBUG:boto:Token: None 2022-09-03T07:28:04.044 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:04.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tK9HjnP2hl/sbaNUGy38djtlbbY= 2022-09-03T07:28:04.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tK9HjnP2hl/sbaNUGy38djtlbbY=', 'Content-Length': '0'} 2022-09-03T07:28:04.050 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f62b1466b3e8696a-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.051 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.052 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.053 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:04.053 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:04.054 DEBUG:boto:Method: GET 2022-09-03T07:28:04.054 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:04.055 DEBUG:boto:Data: 2022-09-03T07:28:04.055 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.056 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.057 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.058 DEBUG:boto:Params: {} 2022-09-03T07:28:04.058 DEBUG:boto:Token: None 2022-09-03T07:28:04.059 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.064 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001432544617ac8499-0063130204-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.106 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.108 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.109 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.110 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:04.111 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:04.112 DEBUG:boto:Method: GET 2022-09-03T07:28:04.114 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:04.116 DEBUG:boto:Data: 2022-09-03T07:28:04.117 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.118 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.119 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.120 DEBUG:boto:Params: {} 2022-09-03T07:28:04.121 DEBUG:boto:Token: None 2022-09-03T07:28:04.122 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.136 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a7c42f5fe3102fd-0063130204-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.142 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.144 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:04.145 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:04.146 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.147 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:04.148 DEBUG:boto:Data: 2022-09-03T07:28:04.149 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.150 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.151 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.152 DEBUG:boto:Params: {} 2022-09-03T07:28:04.153 DEBUG:boto:Token: None 2022-09-03T07:28:04.154 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj58 2022-09-03T07:28:04.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0B7CIciyJDfW2+bJjWQj9zsajto= 2022-09-03T07:28:04.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0B7CIciyJDfW2+bJjWQj9zsajto=', 'Content-Length': '0'} 2022-09-03T07:28:04.162 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005166f0fd35147a39-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.163 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:04.164 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:04.165 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.166 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:04.168 DEBUG:boto:Data: 2022-09-03T07:28:04.169 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.173 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.174 DEBUG:boto:Params: {} 2022-09-03T07:28:04.181 DEBUG:boto:Token: None 2022-09-03T07:28:04.182 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj58 2022-09-03T07:28:04.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0B7CIciyJDfW2+bJjWQj9zsajto= 2022-09-03T07:28:04.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0B7CIciyJDfW2+bJjWQj9zsajto=', 'Content-Length': '0'} 2022-09-03T07:28:04.196 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039411be05e16ae6b-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.198 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:28:04.204 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:04.205 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:04.206 DEBUG:boto:Method: GET 2022-09-03T07:28:04.206 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:04.207 DEBUG:boto:Data: 2022-09-03T07:28:04.208 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.209 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.209 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.210 DEBUG:boto:Params: {} 2022-09-03T07:28:04.211 DEBUG:boto:Token: None 2022-09-03T07:28:04.212 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj58 2022-09-03T07:28:04.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/17aB/q74xiSOgVaINVLxi/vVdA= 2022-09-03T07:28:04.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/17aB/q74xiSOgVaINVLxi/vVdA=', 'Content-Length': '0'} 2022-09-03T07:28:04.218 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf6f70eadd01b67b-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.220 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:04.221 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:04.222 DEBUG:boto:Method: GET 2022-09-03T07:28:04.223 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:04.224 DEBUG:boto:Data: 2022-09-03T07:28:04.226 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.227 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.228 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.230 DEBUG:boto:Params: {} 2022-09-03T07:28:04.231 DEBUG:boto:Token: None 2022-09-03T07:28:04.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj58 2022-09-03T07:28:04.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/17aB/q74xiSOgVaINVLxi/vVdA= 2022-09-03T07:28:04.236 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/17aB/q74xiSOgVaINVLxi/vVdA=', 'Content-Length': '0'} 2022-09-03T07:28:04.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000692799ca2926867e-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.255 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.256 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.261 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:04.263 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:04.264 DEBUG:boto:Method: GET 2022-09-03T07:28:04.265 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:04.266 DEBUG:boto:Data: 2022-09-03T07:28:04.267 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.268 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.269 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.270 DEBUG:boto:Params: {} 2022-09-03T07:28:04.271 DEBUG:boto:Token: None 2022-09-03T07:28:04.272 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.273 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.278 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac0f55801531f4ad-0063130204-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.280 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.282 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.283 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.285 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:04.286 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:04.288 DEBUG:boto:Method: GET 2022-09-03T07:28:04.289 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:04.290 DEBUG:boto:Data: 2022-09-03T07:28:04.291 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.293 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.294 DEBUG:boto:Params: {} 2022-09-03T07:28:04.295 DEBUG:boto:Token: None 2022-09-03T07:28:04.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.304 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a5f3b743af3d7d1-0063130204-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.306 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.308 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:28:04.309 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.310 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.311 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.312 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.314 DEBUG:boto:Method: GET 2022-09-03T07:28:04.315 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.317 DEBUG:boto:Data: 2022-09-03T07:28:04.318 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.320 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.321 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.323 DEBUG:boto:Params: {} 2022-09-03T07:28:04.324 DEBUG:boto:Token: None 2022-09-03T07:28:04.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2WRORa/if3dGM2PywA95TkYo2u0= 2022-09-03T07:28:04.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2WRORa/if3dGM2PywA95TkYo2u0=', 'Content-Length': '0'} 2022-09-03T07:28:04.333 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab2995cb41aeea6f-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.335 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.336 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.337 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.338 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.339 DEBUG:boto:Method: GET 2022-09-03T07:28:04.340 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.341 DEBUG:boto:Data: 2022-09-03T07:28:04.343 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.345 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.346 DEBUG:boto:Params: {} 2022-09-03T07:28:04.347 DEBUG:boto:Token: None 2022-09-03T07:28:04.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2WRORa/if3dGM2PywA95TkYo2u0= 2022-09-03T07:28:04.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2WRORa/if3dGM2PywA95TkYo2u0=', 'Content-Length': '0'} 2022-09-03T07:28:04.355 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3406e3649098ae7-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.356 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.357 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.359 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.360 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.361 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.362 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.363 DEBUG:boto:Data: 2022-09-03T07:28:04.364 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.365 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.375 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.375 DEBUG:boto:Params: {} 2022-09-03T07:28:04.376 DEBUG:boto:Token: None 2022-09-03T07:28:04.377 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KzvOXoQQJ3VNBybEu4kO6Vfqx+w= 2022-09-03T07:28:04.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KzvOXoQQJ3VNBybEu4kO6Vfqx+w=', 'Content-Length': '0'} 2022-09-03T07:28:04.381 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1393b4a83e60c9a-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.381 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.382 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.383 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.383 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.384 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.384 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.385 DEBUG:boto:Data: 2022-09-03T07:28:04.385 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.386 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.387 DEBUG:boto:Params: {} 2022-09-03T07:28:04.388 DEBUG:boto:Token: None 2022-09-03T07:28:04.389 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.389 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KzvOXoQQJ3VNBybEu4kO6Vfqx+w= 2022-09-03T07:28:04.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KzvOXoQQJ3VNBybEu4kO6Vfqx+w=', 'Content-Length': '0'} 2022-09-03T07:28:04.395 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000542741d16a0897f9-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.397 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:28:04.397 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.401 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.402 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.403 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.404 DEBUG:boto:Method: GET 2022-09-03T07:28:04.405 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.406 DEBUG:boto:Data: 2022-09-03T07:28:04.407 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.408 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.409 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.411 DEBUG:boto:Params: {} 2022-09-03T07:28:04.412 DEBUG:boto:Token: None 2022-09-03T07:28:04.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2WRORa/if3dGM2PywA95TkYo2u0= 2022-09-03T07:28:04.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2WRORa/if3dGM2PywA95TkYo2u0=', 'Content-Length': '0'} 2022-09-03T07:28:04.430 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6826893bd39634d-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.432 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.432 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.433 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.434 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.435 DEBUG:boto:Method: GET 2022-09-03T07:28:04.436 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.436 DEBUG:boto:Data: 2022-09-03T07:28:04.437 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.438 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.439 DEBUG:boto:Params: {} 2022-09-03T07:28:04.440 DEBUG:boto:Token: None 2022-09-03T07:28:04.440 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:04.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2WRORa/if3dGM2PywA95TkYo2u0= 2022-09-03T07:28:04.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2WRORa/if3dGM2PywA95TkYo2u0=', 'Content-Length': '0'} 2022-09-03T07:28:04.445 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000417b03b3e0753625-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.446 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.447 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.448 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:04.449 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:04.450 DEBUG:boto:Method: GET 2022-09-03T07:28:04.451 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:04.453 DEBUG:boto:Data: 2022-09-03T07:28:04.455 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.455 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.456 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.457 DEBUG:boto:Params: {} 2022-09-03T07:28:04.459 DEBUG:boto:Token: None 2022-09-03T07:28:04.459 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.460 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.464 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce643ee5f52d44ad-0063130204-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.464 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.468 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.468 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.469 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:04.470 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:04.470 DEBUG:boto:Method: GET 2022-09-03T07:28:04.471 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:04.471 DEBUG:boto:Data: 2022-09-03T07:28:04.472 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.473 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.474 DEBUG:boto:Params: {} 2022-09-03T07:28:04.482 DEBUG:boto:Token: None 2022-09-03T07:28:04.483 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.484 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.489 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d6d5455c3c27cf17-0063130204-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.490 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.491 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.491 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.492 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.492 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:04.493 DEBUG:boto:Data: 2022-09-03T07:28:04.493 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.494 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.494 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.495 DEBUG:boto:Params: {} 2022-09-03T07:28:04.496 DEBUG:boto:Token: None 2022-09-03T07:28:04.503 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62 2022-09-03T07:28:04.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:23T0V2K5WNK3e1GkQCSI9AEPbEo= 2022-09-03T07:28:04.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:23T0V2K5WNK3e1GkQCSI9AEPbEo=', 'Content-Length': '0'} 2022-09-03T07:28:04.514 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cffcfe2f1536ec6a-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.516 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.517 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.519 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.520 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:04.521 DEBUG:boto:Data: 2022-09-03T07:28:04.522 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.524 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.525 DEBUG:boto:Params: {} 2022-09-03T07:28:04.527 DEBUG:boto:Token: None 2022-09-03T07:28:04.528 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62 2022-09-03T07:28:04.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:23T0V2K5WNK3e1GkQCSI9AEPbEo= 2022-09-03T07:28:04.532 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:23T0V2K5WNK3e1GkQCSI9AEPbEo=', 'Content-Length': '0'} 2022-09-03T07:28:04.535 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4dd8fffc8beb8c6-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.543 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:28:04.543 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.544 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.544 DEBUG:boto:Method: GET 2022-09-03T07:28:04.545 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:04.545 DEBUG:boto:Data: 2022-09-03T07:28:04.546 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.546 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.547 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.547 DEBUG:boto:Params: {} 2022-09-03T07:28:04.548 DEBUG:boto:Token: None 2022-09-03T07:28:04.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62 2022-09-03T07:28:04.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p+dRJ5kNbK9uHwme9chp5p49snM= 2022-09-03T07:28:04.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p+dRJ5kNbK9uHwme9chp5p49snM=', 'Content-Length': '0'} 2022-09-03T07:28:04.554 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024cc47c5abc778a1-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.555 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:04.555 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:04.556 DEBUG:boto:Method: GET 2022-09-03T07:28:04.556 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:04.562 DEBUG:boto:Data: 2022-09-03T07:28:04.563 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.563 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.564 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.565 DEBUG:boto:Params: {} 2022-09-03T07:28:04.565 DEBUG:boto:Token: None 2022-09-03T07:28:04.566 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj62 2022-09-03T07:28:04.567 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:p+dRJ5kNbK9uHwme9chp5p49snM= 2022-09-03T07:28:04.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:p+dRJ5kNbK9uHwme9chp5p49snM=', 'Content-Length': '0'} 2022-09-03T07:28:04.570 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063459e7752386177-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.571 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.571 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.572 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:04.572 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:04.573 DEBUG:boto:Method: GET 2022-09-03T07:28:04.573 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:04.574 DEBUG:boto:Data: 2022-09-03T07:28:04.574 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.575 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.575 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.576 DEBUG:boto:Params: {} 2022-09-03T07:28:04.576 DEBUG:boto:Token: None 2022-09-03T07:28:04.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.581 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000033cc4ead85523f14-0063130204-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.582 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.583 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.583 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.584 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:04.584 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:04.585 DEBUG:boto:Method: GET 2022-09-03T07:28:04.585 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:04.586 DEBUG:boto:Data: 2022-09-03T07:28:04.586 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.587 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.588 DEBUG:boto:Params: {} 2022-09-03T07:28:04.589 DEBUG:boto:Token: None 2022-09-03T07:28:04.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.594 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000089242d7227b8d12a-0063130204-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.595 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.596 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:28:04.597 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.597 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.602 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.603 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.604 DEBUG:boto:Method: GET 2022-09-03T07:28:04.605 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.606 DEBUG:boto:Data: 2022-09-03T07:28:04.608 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.609 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.610 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.611 DEBUG:boto:Params: {} 2022-09-03T07:28:04.612 DEBUG:boto:Token: None 2022-09-03T07:28:04.613 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NUPFzLqBav6us72f9AxTJclD6Ig= 2022-09-03T07:28:04.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NUPFzLqBav6us72f9AxTJclD6Ig=', 'Content-Length': '0'} 2022-09-03T07:28:04.617 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000047e1a4353e4269f-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.618 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.618 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.619 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.619 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.620 DEBUG:boto:Method: GET 2022-09-03T07:28:04.621 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.621 DEBUG:boto:Data: 2022-09-03T07:28:04.622 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.623 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.623 DEBUG:boto:Params: {} 2022-09-03T07:28:04.624 DEBUG:boto:Token: None 2022-09-03T07:28:04.624 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NUPFzLqBav6us72f9AxTJclD6Ig= 2022-09-03T07:28:04.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NUPFzLqBav6us72f9AxTJclD6Ig=', 'Content-Length': '0'} 2022-09-03T07:28:04.628 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a03475d5277e645-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.629 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.629 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.630 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.630 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.631 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.631 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.632 DEBUG:boto:Data: 2022-09-03T07:28:04.632 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.633 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.633 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.634 DEBUG:boto:Params: {} 2022-09-03T07:28:04.634 DEBUG:boto:Token: None 2022-09-03T07:28:04.635 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ggKxxPXwKg/YJ+kCrU9dyuDxmWQ= 2022-09-03T07:28:04.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ggKxxPXwKg/YJ+kCrU9dyuDxmWQ=', 'Content-Length': '0'} 2022-09-03T07:28:04.639 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c1a6f3ec691d419-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.640 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.640 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.641 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.641 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.642 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.642 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.643 DEBUG:boto:Data: 2022-09-03T07:28:04.643 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.644 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.645 DEBUG:boto:Params: {} 2022-09-03T07:28:04.645 DEBUG:boto:Token: None 2022-09-03T07:28:04.646 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.647 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ggKxxPXwKg/YJ+kCrU9dyuDxmWQ= 2022-09-03T07:28:04.648 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ggKxxPXwKg/YJ+kCrU9dyuDxmWQ=', 'Content-Length': '0'} 2022-09-03T07:28:04.650 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ce91bb8e49376d8-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.651 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:28:04.651 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.652 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.653 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.653 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.654 DEBUG:boto:Method: GET 2022-09-03T07:28:04.654 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.655 DEBUG:boto:Data: 2022-09-03T07:28:04.656 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.656 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.657 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.657 DEBUG:boto:Params: {} 2022-09-03T07:28:04.658 DEBUG:boto:Token: None 2022-09-03T07:28:04.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NUPFzLqBav6us72f9AxTJclD6Ig= 2022-09-03T07:28:04.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NUPFzLqBav6us72f9AxTJclD6Ig=', 'Content-Length': '0'} 2022-09-03T07:28:04.666 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097302ba7da45fd2f-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.667 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.667 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.668 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.668 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.669 DEBUG:boto:Method: GET 2022-09-03T07:28:04.670 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.670 DEBUG:boto:Data: 2022-09-03T07:28:04.671 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.675 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.675 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.676 DEBUG:boto:Params: {} 2022-09-03T07:28:04.677 DEBUG:boto:Token: None 2022-09-03T07:28:04.677 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:04.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NUPFzLqBav6us72f9AxTJclD6Ig= 2022-09-03T07:28:04.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NUPFzLqBav6us72f9AxTJclD6Ig=', 'Content-Length': '0'} 2022-09-03T07:28:04.683 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005eac275e75ea2485-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.685 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.685 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.686 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:04.687 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:04.687 DEBUG:boto:Method: GET 2022-09-03T07:28:04.688 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:04.688 DEBUG:boto:Data: 2022-09-03T07:28:04.689 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.689 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.690 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.690 DEBUG:boto:Params: {} 2022-09-03T07:28:04.691 DEBUG:boto:Token: None 2022-09-03T07:28:04.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.692 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.693 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.696 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc39fb1e5446a4b3-0063130204-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.696 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.697 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.698 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.699 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:04.700 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:04.700 DEBUG:boto:Method: GET 2022-09-03T07:28:04.701 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:04.702 DEBUG:boto:Data: 2022-09-03T07:28:04.702 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.703 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.713 DEBUG:boto:Params: {} 2022-09-03T07:28:04.715 DEBUG:boto:Token: None 2022-09-03T07:28:04.716 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.724 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027b1bd89ed5f13e7-0063130204-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.725 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.725 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.726 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.727 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.727 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:04.728 DEBUG:boto:Data: 2022-09-03T07:28:04.728 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.729 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.729 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.729 DEBUG:boto:Params: {} 2022-09-03T07:28:04.730 DEBUG:boto:Token: None 2022-09-03T07:28:04.731 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68 2022-09-03T07:28:04.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UaaaGOW2+IAYqijvOeqWUXFZM8E= 2022-09-03T07:28:04.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UaaaGOW2+IAYqijvOeqWUXFZM8E=', 'Content-Length': '0'} 2022-09-03T07:28:04.734 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003bec09f69b550729-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.736 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.737 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.739 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.740 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:04.741 DEBUG:boto:Data: 2022-09-03T07:28:04.742 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.743 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.743 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.744 DEBUG:boto:Params: {} 2022-09-03T07:28:04.745 DEBUG:boto:Token: None 2022-09-03T07:28:04.745 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68 2022-09-03T07:28:04.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UaaaGOW2+IAYqijvOeqWUXFZM8E= 2022-09-03T07:28:04.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UaaaGOW2+IAYqijvOeqWUXFZM8E=', 'Content-Length': '0'} 2022-09-03T07:28:04.750 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc80f83c7c518588-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.751 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:28:04.752 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.753 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.754 DEBUG:boto:Method: GET 2022-09-03T07:28:04.755 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:04.756 DEBUG:boto:Data: 2022-09-03T07:28:04.757 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.758 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.759 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.760 DEBUG:boto:Params: {} 2022-09-03T07:28:04.761 DEBUG:boto:Token: None 2022-09-03T07:28:04.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68 2022-09-03T07:28:04.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8ccY65BXiXPKw+zQrG7R8VxUYPk= 2022-09-03T07:28:04.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8ccY65BXiXPKw+zQrG7R8VxUYPk=', 'Content-Length': '0'} 2022-09-03T07:28:04.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf595581a697611b-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.767 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:04.767 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:04.768 DEBUG:boto:Method: GET 2022-09-03T07:28:04.769 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:04.770 DEBUG:boto:Data: 2022-09-03T07:28:04.772 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.773 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.774 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.775 DEBUG:boto:Params: {} 2022-09-03T07:28:04.776 DEBUG:boto:Token: None 2022-09-03T07:28:04.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj68 2022-09-03T07:28:04.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8ccY65BXiXPKw+zQrG7R8VxUYPk= 2022-09-03T07:28:04.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8ccY65BXiXPKw+zQrG7R8VxUYPk=', 'Content-Length': '0'} 2022-09-03T07:28:04.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000386390c5bfdde85f-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.822 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.824 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.825 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:04.826 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:04.829 DEBUG:boto:Method: GET 2022-09-03T07:28:04.830 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:04.832 DEBUG:boto:Data: 2022-09-03T07:28:04.833 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.841 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.843 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.846 DEBUG:boto:Params: {} 2022-09-03T07:28:04.852 DEBUG:boto:Token: None 2022-09-03T07:28:04.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.859 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb3ee02e6ce7f9e8-0063130204-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.859 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.860 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:04.861 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:04.862 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:04.862 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:04.863 DEBUG:boto:Method: GET 2022-09-03T07:28:04.863 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:04.864 DEBUG:boto:Data: 2022-09-03T07:28:04.865 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.866 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.866 DEBUG:boto:Params: {} 2022-09-03T07:28:04.867 DEBUG:boto:Token: None 2022-09-03T07:28:04.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/ 2022-09-03T07:28:04.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8= 2022-09-03T07:28:04.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Z4BZE0spv5VEMSIEnyRUPEoubn8=', 'Content-Length': '0'} 2022-09-03T07:28:04.873 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f3471923b47e20a-0063130204-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.874 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:04.875 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:28:04.876 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:04.876 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:04.877 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.878 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.878 DEBUG:boto:Method: GET 2022-09-03T07:28:04.879 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.879 DEBUG:boto:Data: 2022-09-03T07:28:04.880 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.880 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.881 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.882 DEBUG:boto:Params: {} 2022-09-03T07:28:04.884 DEBUG:boto:Token: None 2022-09-03T07:28:04.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WdTmyt6ZfqnJhHtlprzN+k+2KAI= 2022-09-03T07:28:04.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WdTmyt6ZfqnJhHtlprzN+k+2KAI=', 'Content-Length': '0'} 2022-09-03T07:28:04.891 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c751771fcd84ea8a-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.894 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:04.902 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:04.909 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.910 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.915 DEBUG:boto:Method: GET 2022-09-03T07:28:04.916 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.917 DEBUG:boto:Data: 2022-09-03T07:28:04.917 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.928 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.930 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.931 DEBUG:boto:Params: {} 2022-09-03T07:28:04.936 DEBUG:boto:Token: None 2022-09-03T07:28:04.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WdTmyt6ZfqnJhHtlprzN+k+2KAI= 2022-09-03T07:28:04.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WdTmyt6ZfqnJhHtlprzN+k+2KAI=', 'Content-Length': '0'} 2022-09-03T07:28:04.943 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9c15d4744a3aaf5-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.944 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:04.945 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:04.945 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.946 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.947 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.947 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.947 DEBUG:boto:Data: 2022-09-03T07:28:04.948 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.948 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.949 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.950 DEBUG:boto:Params: {} 2022-09-03T07:28:04.951 DEBUG:boto:Token: None 2022-09-03T07:28:04.951 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ejzw6L+6qp4mQa1SczIJgQmju2g= 2022-09-03T07:28:04.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ejzw6L+6qp4mQa1SczIJgQmju2g=', 'Content-Length': '0'} 2022-09-03T07:28:04.955 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076da2684fcc6f399-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.955 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:04.963 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:04.963 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.964 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.964 DEBUG:boto:Method: HEAD 2022-09-03T07:28:04.965 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.965 DEBUG:boto:Data: 2022-09-03T07:28:04.966 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:04.967 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.967 DEBUG:boto:Params: {} 2022-09-03T07:28:04.968 DEBUG:boto:Token: None 2022-09-03T07:28:04.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ejzw6L+6qp4mQa1SczIJgQmju2g= 2022-09-03T07:28:04.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ejzw6L+6qp4mQa1SczIJgQmju2g=', 'Content-Length': '0'} 2022-09-03T07:28:04.974 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d0a9dcaf3d12f2d-0063130204-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:04.976 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:28:04.977 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:04.978 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:04.980 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.980 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.981 DEBUG:boto:Method: GET 2022-09-03T07:28:04.982 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.983 DEBUG:boto:Data: 2022-09-03T07:28:04.984 DEBUG:boto:Headers: {} 2022-09-03T07:28:04.986 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:04.987 DEBUG:boto:Port: 8000 2022-09-03T07:28:04.987 DEBUG:boto:Params: {} 2022-09-03T07:28:04.989 DEBUG:boto:Token: None 2022-09-03T07:28:04.990 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:04 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:04.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WdTmyt6ZfqnJhHtlprzN+k+2KAI= 2022-09-03T07:28:04.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WdTmyt6ZfqnJhHtlprzN+k+2KAI=', 'Content-Length': '0'} 2022-09-03T07:28:04.994 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029d1be786b196686-0063130204-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.042 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:05.043 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:05.044 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:05.045 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:05.048 DEBUG:boto:Method: GET 2022-09-03T07:28:05.048 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:05.049 DEBUG:boto:Data: 2022-09-03T07:28:05.050 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.052 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.053 DEBUG:boto:Params: {} 2022-09-03T07:28:05.053 DEBUG:boto:Token: None 2022-09-03T07:28:05.054 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:05.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+uSd4BAl+M3o0PZUX73lZ1N1T8o= 2022-09-03T07:28:05.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+uSd4BAl+M3o0PZUX73lZ1N1T8o=', 'Content-Length': '0'} 2022-09-03T07:28:05.058 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002be8dc1559ef4f0d-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.059 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.059 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.060 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:05.060 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:05.061 DEBUG:boto:Method: GET 2022-09-03T07:28:05.061 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:05.062 DEBUG:boto:Data: 2022-09-03T07:28:05.062 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.063 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.063 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.064 DEBUG:boto:Params: {} 2022-09-03T07:28:05.065 DEBUG:boto:Token: None 2022-09-03T07:28:05.065 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.066 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006dedf4fa0b267744-0063130205-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.070 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.071 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.071 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.072 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:05.072 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:05.073 DEBUG:boto:Method: GET 2022-09-03T07:28:05.074 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:05.074 DEBUG:boto:Data: 2022-09-03T07:28:05.075 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.076 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.076 DEBUG:boto:Params: {} 2022-09-03T07:28:05.077 DEBUG:boto:Token: None 2022-09-03T07:28:05.077 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.079 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.083 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed9d28e91b60d437-0063130205-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.084 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.085 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:05.085 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:05.086 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.086 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:05.087 DEBUG:boto:Data: 2022-09-03T07:28:05.088 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.088 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.089 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.089 DEBUG:boto:Params: {} 2022-09-03T07:28:05.090 DEBUG:boto:Token: None 2022-09-03T07:28:05.090 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj69 2022-09-03T07:28:05.091 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IdslTiAsl7Y6zv/tfa9jgMEFtUM= 2022-09-03T07:28:05.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IdslTiAsl7Y6zv/tfa9jgMEFtUM=', 'Content-Length': '0'} 2022-09-03T07:28:05.093 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018b805ad5019c7a8-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.094 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:05.095 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:05.095 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.096 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:05.096 DEBUG:boto:Data: 2022-09-03T07:28:05.097 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.097 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.098 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.098 DEBUG:boto:Params: {} 2022-09-03T07:28:05.099 DEBUG:boto:Token: None 2022-09-03T07:28:05.099 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj69 2022-09-03T07:28:05.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IdslTiAsl7Y6zv/tfa9jgMEFtUM= 2022-09-03T07:28:05.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IdslTiAsl7Y6zv/tfa9jgMEFtUM=', 'Content-Length': '0'} 2022-09-03T07:28:05.102 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001dc120e4eda9115d-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.104 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:28:05.104 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:05.105 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:05.105 DEBUG:boto:Method: GET 2022-09-03T07:28:05.106 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:05.106 DEBUG:boto:Data: 2022-09-03T07:28:05.107 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.107 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.108 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.108 DEBUG:boto:Params: {} 2022-09-03T07:28:05.109 DEBUG:boto:Token: None 2022-09-03T07:28:05.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj69 2022-09-03T07:28:05.110 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eMAwaYvOeAW9GM+xFADf69AupsY= 2022-09-03T07:28:05.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eMAwaYvOeAW9GM+xFADf69AupsY=', 'Content-Length': '0'} 2022-09-03T07:28:05.113 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009addb0f0b5a2e30f-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.114 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:05.115 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:05.115 DEBUG:boto:Method: GET 2022-09-03T07:28:05.116 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:05.116 DEBUG:boto:Data: 2022-09-03T07:28:05.117 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.118 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.118 DEBUG:boto:Params: {} 2022-09-03T07:28:05.119 DEBUG:boto:Token: None 2022-09-03T07:28:05.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj69 2022-09-03T07:28:05.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eMAwaYvOeAW9GM+xFADf69AupsY= 2022-09-03T07:28:05.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eMAwaYvOeAW9GM+xFADf69AupsY=', 'Content-Length': '0'} 2022-09-03T07:28:05.123 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000306c0cb249bbb810-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.166 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.167 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.168 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:05.169 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:05.170 DEBUG:boto:Method: GET 2022-09-03T07:28:05.171 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:05.171 DEBUG:boto:Data: 2022-09-03T07:28:05.172 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.173 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.173 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.174 DEBUG:boto:Params: {} 2022-09-03T07:28:05.175 DEBUG:boto:Token: None 2022-09-03T07:28:05.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.181 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5c82dbbbb47d81e-0063130205-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.183 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.184 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.186 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.187 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:05.188 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:05.189 DEBUG:boto:Method: GET 2022-09-03T07:28:05.192 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:05.193 DEBUG:boto:Data: 2022-09-03T07:28:05.194 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.195 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.196 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.197 DEBUG:boto:Params: {} 2022-09-03T07:28:05.198 DEBUG:boto:Token: None 2022-09-03T07:28:05.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.203 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.207 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000539732bf88ae0631-0063130205-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.207 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.209 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:28:05.210 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.211 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.211 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.212 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.213 DEBUG:boto:Method: GET 2022-09-03T07:28:05.214 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.215 DEBUG:boto:Data: 2022-09-03T07:28:05.216 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.217 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.218 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.219 DEBUG:boto:Params: {} 2022-09-03T07:28:05.221 DEBUG:boto:Token: None 2022-09-03T07:28:05.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1MHqjUr83YLbNi9JNwe/8JObh/0= 2022-09-03T07:28:05.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1MHqjUr83YLbNi9JNwe/8JObh/0=', 'Content-Length': '0'} 2022-09-03T07:28:05.228 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ef631188e4a6e0c-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.229 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.230 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.230 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.231 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.231 DEBUG:boto:Method: GET 2022-09-03T07:28:05.232 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.232 DEBUG:boto:Data: 2022-09-03T07:28:05.233 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.233 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.234 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.235 DEBUG:boto:Params: {} 2022-09-03T07:28:05.235 DEBUG:boto:Token: None 2022-09-03T07:28:05.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1MHqjUr83YLbNi9JNwe/8JObh/0= 2022-09-03T07:28:05.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1MHqjUr83YLbNi9JNwe/8JObh/0=', 'Content-Length': '0'} 2022-09-03T07:28:05.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007871d89712bfa46f-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.241 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.241 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.242 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.242 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.243 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.243 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.244 DEBUG:boto:Data: 2022-09-03T07:28:05.244 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.245 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.245 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.246 DEBUG:boto:Params: {} 2022-09-03T07:28:05.247 DEBUG:boto:Token: None 2022-09-03T07:28:05.247 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WAJGBR+F/6FZdu/a7pj6vCyrEBA= 2022-09-03T07:28:05.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WAJGBR+F/6FZdu/a7pj6vCyrEBA=', 'Content-Length': '0'} 2022-09-03T07:28:05.250 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d418a67fec599b8-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.252 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.257 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.258 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.259 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.260 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.262 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.263 DEBUG:boto:Data: 2022-09-03T07:28:05.264 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.266 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.266 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.267 DEBUG:boto:Params: {} 2022-09-03T07:28:05.267 DEBUG:boto:Token: None 2022-09-03T07:28:05.268 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WAJGBR+F/6FZdu/a7pj6vCyrEBA= 2022-09-03T07:28:05.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WAJGBR+F/6FZdu/a7pj6vCyrEBA=', 'Content-Length': '0'} 2022-09-03T07:28:05.272 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000915aa73067c48628-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.272 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:28:05.273 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.274 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.274 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.275 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.275 DEBUG:boto:Method: GET 2022-09-03T07:28:05.276 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.277 DEBUG:boto:Data: 2022-09-03T07:28:05.277 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.278 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.278 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.279 DEBUG:boto:Params: {} 2022-09-03T07:28:05.280 DEBUG:boto:Token: None 2022-09-03T07:28:05.280 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1MHqjUr83YLbNi9JNwe/8JObh/0= 2022-09-03T07:28:05.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1MHqjUr83YLbNi9JNwe/8JObh/0=', 'Content-Length': '0'} 2022-09-03T07:28:05.283 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000073c3640372aa727f-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.284 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.285 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.285 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.286 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.286 DEBUG:boto:Method: GET 2022-09-03T07:28:05.287 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.287 DEBUG:boto:Data: 2022-09-03T07:28:05.288 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.289 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.290 DEBUG:boto:Params: {} 2022-09-03T07:28:05.290 DEBUG:boto:Token: None 2022-09-03T07:28:05.291 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:05.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1MHqjUr83YLbNi9JNwe/8JObh/0= 2022-09-03T07:28:05.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1MHqjUr83YLbNi9JNwe/8JObh/0=', 'Content-Length': '0'} 2022-09-03T07:28:05.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022d52658243cc476-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.296 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.296 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.297 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:05.297 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:05.298 DEBUG:boto:Method: GET 2022-09-03T07:28:05.298 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:05.299 DEBUG:boto:Data: 2022-09-03T07:28:05.300 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.301 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.301 DEBUG:boto:Params: {} 2022-09-03T07:28:05.302 DEBUG:boto:Token: None 2022-09-03T07:28:05.303 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.306 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000635a3049dec93fe5-0063130205-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.308 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.309 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.310 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.310 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:05.311 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:05.311 DEBUG:boto:Method: GET 2022-09-03T07:28:05.312 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:05.312 DEBUG:boto:Data: 2022-09-03T07:28:05.313 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.314 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.315 DEBUG:boto:Params: {} 2022-09-03T07:28:05.315 DEBUG:boto:Token: None 2022-09-03T07:28:05.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.323 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000d9d5360f1250ca7-0063130205-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.366 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.367 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.368 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.368 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.369 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:05.370 DEBUG:boto:Data: 2022-09-03T07:28:05.370 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.371 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.372 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.372 DEBUG:boto:Params: {} 2022-09-03T07:28:05.373 DEBUG:boto:Token: None 2022-09-03T07:28:05.373 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76 2022-09-03T07:28:05.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xOequW7MDfdTocIVlkvQM0mvlJM= 2022-09-03T07:28:05.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xOequW7MDfdTocIVlkvQM0mvlJM=', 'Content-Length': '0'} 2022-09-03T07:28:05.378 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6bb5d12f2cd0d3d-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.379 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.380 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.381 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.383 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:05.384 DEBUG:boto:Data: 2022-09-03T07:28:05.385 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.387 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.387 DEBUG:boto:Params: {} 2022-09-03T07:28:05.388 DEBUG:boto:Token: None 2022-09-03T07:28:05.389 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76 2022-09-03T07:28:05.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xOequW7MDfdTocIVlkvQM0mvlJM= 2022-09-03T07:28:05.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xOequW7MDfdTocIVlkvQM0mvlJM=', 'Content-Length': '0'} 2022-09-03T07:28:05.394 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000085180bd1c7037d0-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.395 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:28:05.395 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.396 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.397 DEBUG:boto:Method: GET 2022-09-03T07:28:05.397 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:05.398 DEBUG:boto:Data: 2022-09-03T07:28:05.399 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.399 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.400 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.400 DEBUG:boto:Params: {} 2022-09-03T07:28:05.401 DEBUG:boto:Token: None 2022-09-03T07:28:05.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76 2022-09-03T07:28:05.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9rhEBDeUJpenjF4UjqZ0YCT0GNY= 2022-09-03T07:28:05.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9rhEBDeUJpenjF4UjqZ0YCT0GNY=', 'Content-Length': '0'} 2022-09-03T07:28:05.405 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3425a694575eaa7-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.451 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:05.452 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:05.452 DEBUG:boto:Method: GET 2022-09-03T07:28:05.453 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:05.453 DEBUG:boto:Data: 2022-09-03T07:28:05.454 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.454 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.455 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.455 DEBUG:boto:Params: {} 2022-09-03T07:28:05.456 DEBUG:boto:Token: None 2022-09-03T07:28:05.457 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj76 2022-09-03T07:28:05.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9rhEBDeUJpenjF4UjqZ0YCT0GNY= 2022-09-03T07:28:05.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9rhEBDeUJpenjF4UjqZ0YCT0GNY=', 'Content-Length': '0'} 2022-09-03T07:28:05.460 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006320432129aef875-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.461 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.461 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.462 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:05.462 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:05.463 DEBUG:boto:Method: GET 2022-09-03T07:28:05.463 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:05.464 DEBUG:boto:Data: 2022-09-03T07:28:05.464 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.465 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.466 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.466 DEBUG:boto:Params: {} 2022-09-03T07:28:05.467 DEBUG:boto:Token: None 2022-09-03T07:28:05.468 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.477 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000e7126fd93a94e3d-0063130205-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.479 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.481 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.482 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.483 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:05.484 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:05.485 DEBUG:boto:Method: GET 2022-09-03T07:28:05.486 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:05.487 DEBUG:boto:Data: 2022-09-03T07:28:05.487 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.489 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.491 DEBUG:boto:Params: {} 2022-09-03T07:28:05.493 DEBUG:boto:Token: None 2022-09-03T07:28:05.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.499 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004cec15e98b76277e-0063130205-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.501 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.502 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:28:05.503 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.504 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.505 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.506 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.507 DEBUG:boto:Method: GET 2022-09-03T07:28:05.508 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.509 DEBUG:boto:Data: 2022-09-03T07:28:05.510 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.511 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.512 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.513 DEBUG:boto:Params: {} 2022-09-03T07:28:05.515 DEBUG:boto:Token: None 2022-09-03T07:28:05.516 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HadpdR3d2OZkooRl2rKzmzCXEiQ= 2022-09-03T07:28:05.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HadpdR3d2OZkooRl2rKzmzCXEiQ=', 'Content-Length': '0'} 2022-09-03T07:28:05.522 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bb06f4ae9f43cca-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.524 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.525 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.525 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.527 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.528 DEBUG:boto:Method: GET 2022-09-03T07:28:05.529 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.530 DEBUG:boto:Data: 2022-09-03T07:28:05.531 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.532 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.533 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.534 DEBUG:boto:Params: {} 2022-09-03T07:28:05.535 DEBUG:boto:Token: None 2022-09-03T07:28:05.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HadpdR3d2OZkooRl2rKzmzCXEiQ= 2022-09-03T07:28:05.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HadpdR3d2OZkooRl2rKzmzCXEiQ=', 'Content-Length': '0'} 2022-09-03T07:28:05.540 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000dd67cf841f35787-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.541 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.543 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.544 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.545 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.545 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.547 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.548 DEBUG:boto:Data: 2022-09-03T07:28:05.548 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.549 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.550 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.551 DEBUG:boto:Params: {} 2022-09-03T07:28:05.552 DEBUG:boto:Token: None 2022-09-03T07:28:05.553 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44T++CH5v84NbQYuthTT1e8XQQA= 2022-09-03T07:28:05.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44T++CH5v84NbQYuthTT1e8XQQA=', 'Content-Length': '0'} 2022-09-03T07:28:05.558 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009192ba3329b44c9a-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.561 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.562 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.563 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.565 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.565 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.566 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.568 DEBUG:boto:Data: 2022-09-03T07:28:05.569 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.571 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.573 DEBUG:boto:Params: {} 2022-09-03T07:28:05.574 DEBUG:boto:Token: None 2022-09-03T07:28:05.575 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.575 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:44T++CH5v84NbQYuthTT1e8XQQA= 2022-09-03T07:28:05.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:44T++CH5v84NbQYuthTT1e8XQQA=', 'Content-Length': '0'} 2022-09-03T07:28:05.579 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027289acc98f41013-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.581 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:28:05.582 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.582 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.583 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.583 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.584 DEBUG:boto:Method: GET 2022-09-03T07:28:05.584 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.589 DEBUG:boto:Data: 2022-09-03T07:28:05.590 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.591 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.592 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.592 DEBUG:boto:Params: {} 2022-09-03T07:28:05.594 DEBUG:boto:Token: None 2022-09-03T07:28:05.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HadpdR3d2OZkooRl2rKzmzCXEiQ= 2022-09-03T07:28:05.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HadpdR3d2OZkooRl2rKzmzCXEiQ=', 'Content-Length': '0'} 2022-09-03T07:28:05.599 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c168b8507eab816-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.600 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.601 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.602 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.603 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.604 DEBUG:boto:Method: GET 2022-09-03T07:28:05.606 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.607 DEBUG:boto:Data: 2022-09-03T07:28:05.607 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.608 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.609 DEBUG:boto:Params: {} 2022-09-03T07:28:05.610 DEBUG:boto:Token: None 2022-09-03T07:28:05.612 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:05.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HadpdR3d2OZkooRl2rKzmzCXEiQ= 2022-09-03T07:28:05.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HadpdR3d2OZkooRl2rKzmzCXEiQ=', 'Content-Length': '0'} 2022-09-03T07:28:05.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a47ce31d745474f0-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.619 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.621 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.622 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:05.623 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:05.624 DEBUG:boto:Method: GET 2022-09-03T07:28:05.625 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:05.626 DEBUG:boto:Data: 2022-09-03T07:28:05.627 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.628 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.629 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.630 DEBUG:boto:Params: {} 2022-09-03T07:28:05.631 DEBUG:boto:Token: None 2022-09-03T07:28:05.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.639 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7791a3e4172ff13-0063130205-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.640 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.641 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.642 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.643 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:05.644 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:05.645 DEBUG:boto:Method: GET 2022-09-03T07:28:05.646 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:05.647 DEBUG:boto:Data: 2022-09-03T07:28:05.648 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.649 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.650 DEBUG:boto:Params: {} 2022-09-03T07:28:05.650 DEBUG:boto:Token: None 2022-09-03T07:28:05.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.659 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000034e63813993ee9b1-0063130205-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.661 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.662 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.664 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.665 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.666 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:05.667 DEBUG:boto:Data: 2022-09-03T07:28:05.667 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.668 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.669 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.670 DEBUG:boto:Params: {} 2022-09-03T07:28:05.671 DEBUG:boto:Token: None 2022-09-03T07:28:05.672 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85 2022-09-03T07:28:05.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:63MuZiUCch6JZ3lJJb4WB+jU6CU= 2022-09-03T07:28:05.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:63MuZiUCch6JZ3lJJb4WB+jU6CU=', 'Content-Length': '0'} 2022-09-03T07:28:05.677 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2d1cbbfebea980d-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.679 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.680 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.681 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.681 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:05.682 DEBUG:boto:Data: 2022-09-03T07:28:05.683 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.684 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.685 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.686 DEBUG:boto:Params: {} 2022-09-03T07:28:05.686 DEBUG:boto:Token: None 2022-09-03T07:28:05.687 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85 2022-09-03T07:28:05.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:63MuZiUCch6JZ3lJJb4WB+jU6CU= 2022-09-03T07:28:05.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:63MuZiUCch6JZ3lJJb4WB+jU6CU=', 'Content-Length': '0'} 2022-09-03T07:28:05.693 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c75d507a612b0e3f-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.694 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:28:05.694 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.695 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.696 DEBUG:boto:Method: GET 2022-09-03T07:28:05.696 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:05.697 DEBUG:boto:Data: 2022-09-03T07:28:05.697 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.698 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.699 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.700 DEBUG:boto:Params: {} 2022-09-03T07:28:05.701 DEBUG:boto:Token: None 2022-09-03T07:28:05.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85 2022-09-03T07:28:05.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ax+6S3MFbjWuXN6Ru6x5OqmXPco= 2022-09-03T07:28:05.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ax+6S3MFbjWuXN6Ru6x5OqmXPco=', 'Content-Length': '0'} 2022-09-03T07:28:05.707 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b72a8bbcb7d0e4b4-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.750 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:05.751 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:05.752 DEBUG:boto:Method: GET 2022-09-03T07:28:05.754 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:05.756 DEBUG:boto:Data: 2022-09-03T07:28:05.757 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.760 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.760 DEBUG:boto:Params: {} 2022-09-03T07:28:05.761 DEBUG:boto:Token: None 2022-09-03T07:28:05.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj85 2022-09-03T07:28:05.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ax+6S3MFbjWuXN6Ru6x5OqmXPco= 2022-09-03T07:28:05.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ax+6S3MFbjWuXN6Ru6x5OqmXPco=', 'Content-Length': '0'} 2022-09-03T07:28:05.769 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0f67f294a56c7cc-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.771 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.771 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.772 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:05.773 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:05.774 DEBUG:boto:Method: GET 2022-09-03T07:28:05.775 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:05.776 DEBUG:boto:Data: 2022-09-03T07:28:05.777 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.778 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.779 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.780 DEBUG:boto:Params: {} 2022-09-03T07:28:05.782 DEBUG:boto:Token: None 2022-09-03T07:28:05.783 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.789 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2163d501bd1111b-0063130205-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.790 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.792 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.793 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.795 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:05.796 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:05.797 DEBUG:boto:Method: GET 2022-09-03T07:28:05.798 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:05.799 DEBUG:boto:Data: 2022-09-03T07:28:05.800 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.801 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.802 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.803 DEBUG:boto:Params: {} 2022-09-03T07:28:05.804 DEBUG:boto:Token: None 2022-09-03T07:28:05.805 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.812 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000173c88fd3199ab43-0063130205-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.813 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.815 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:28:05.816 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:05.817 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:05.819 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.819 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.820 DEBUG:boto:Method: GET 2022-09-03T07:28:05.821 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.823 DEBUG:boto:Data: 2022-09-03T07:28:05.824 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.825 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.826 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.827 DEBUG:boto:Params: {} 2022-09-03T07:28:05.829 DEBUG:boto:Token: None 2022-09-03T07:28:05.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n699CIiQXJHAi2I3QPPqw56D0xs= 2022-09-03T07:28:05.831 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n699CIiQXJHAi2I3QPPqw56D0xs=', 'Content-Length': '0'} 2022-09-03T07:28:05.834 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083fd00621e569541-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.835 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:05.836 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:05.837 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.838 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.839 DEBUG:boto:Method: GET 2022-09-03T07:28:05.840 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.840 DEBUG:boto:Data: 2022-09-03T07:28:05.841 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.842 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.843 DEBUG:boto:Params: {} 2022-09-03T07:28:05.844 DEBUG:boto:Token: None 2022-09-03T07:28:05.844 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n699CIiQXJHAi2I3QPPqw56D0xs= 2022-09-03T07:28:05.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n699CIiQXJHAi2I3QPPqw56D0xs=', 'Content-Length': '0'} 2022-09-03T07:28:05.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029d95bb7d2c417c3-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.855 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:05.856 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:05.857 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.857 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.858 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.858 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.859 DEBUG:boto:Data: 2022-09-03T07:28:05.859 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.860 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.860 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.861 DEBUG:boto:Params: {} 2022-09-03T07:28:05.861 DEBUG:boto:Token: None 2022-09-03T07:28:05.862 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7hnyN3GXhmvo/4nMMGSNEH1fLnM= 2022-09-03T07:28:05.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7hnyN3GXhmvo/4nMMGSNEH1fLnM=', 'Content-Length': '0'} 2022-09-03T07:28:05.865 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b823ece79cc2dd5-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.866 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:05.867 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:05.867 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.868 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.868 DEBUG:boto:Method: HEAD 2022-09-03T07:28:05.869 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.869 DEBUG:boto:Data: 2022-09-03T07:28:05.870 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.871 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.871 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.872 DEBUG:boto:Params: {} 2022-09-03T07:28:05.873 DEBUG:boto:Token: None 2022-09-03T07:28:05.873 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7hnyN3GXhmvo/4nMMGSNEH1fLnM= 2022-09-03T07:28:05.874 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7hnyN3GXhmvo/4nMMGSNEH1fLnM=', 'Content-Length': '0'} 2022-09-03T07:28:05.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3ffdea456e6fcd0-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.878 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:28:05.879 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:05.880 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:05.881 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.882 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.883 DEBUG:boto:Method: GET 2022-09-03T07:28:05.883 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.884 DEBUG:boto:Data: 2022-09-03T07:28:05.884 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.885 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.886 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.886 DEBUG:boto:Params: {} 2022-09-03T07:28:05.887 DEBUG:boto:Token: None 2022-09-03T07:28:05.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n699CIiQXJHAi2I3QPPqw56D0xs= 2022-09-03T07:28:05.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n699CIiQXJHAi2I3QPPqw56D0xs=', 'Content-Length': '0'} 2022-09-03T07:28:05.896 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de301fa7da1f92f5-0063130205-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.897 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:05.898 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:05.898 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.899 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.899 DEBUG:boto:Method: GET 2022-09-03T07:28:05.900 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.900 DEBUG:boto:Data: 2022-09-03T07:28:05.901 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.902 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.902 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.903 DEBUG:boto:Params: {} 2022-09-03T07:28:05.904 DEBUG:boto:Token: None 2022-09-03T07:28:05.904 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:05.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n699CIiQXJHAi2I3QPPqw56D0xs= 2022-09-03T07:28:05.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n699CIiQXJHAi2I3QPPqw56D0xs=', 'Content-Length': '0'} 2022-09-03T07:28:05.910 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000464c10963e7adc0f-0063130205-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.913 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.913 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.914 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:05.915 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:05.915 DEBUG:boto:Method: GET 2022-09-03T07:28:05.916 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:05.917 DEBUG:boto:Data: 2022-09-03T07:28:05.918 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.918 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:05.919 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.920 DEBUG:boto:Params: {} 2022-09-03T07:28:05.921 DEBUG:boto:Token: None 2022-09-03T07:28:05.921 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:05 GMT /avysmw-75/ 2022-09-03T07:28:05.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I= 2022-09-03T07:28:05.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDgX5nVzsM87LO985BQeDKNNg/I=', 'Content-Length': '0'} 2022-09-03T07:28:05.939 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000077e6663b0eeb8148-0063130205-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:05.986 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:05.988 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:05.989 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:05.990 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:05.991 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:05.992 DEBUG:boto:Method: GET 2022-09-03T07:28:05.993 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:05.994 DEBUG:boto:Data: 2022-09-03T07:28:05.995 DEBUG:boto:Headers: {} 2022-09-03T07:28:05.996 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:05.997 DEBUG:boto:Port: 8000 2022-09-03T07:28:05.998 DEBUG:boto:Params: {} 2022-09-03T07:28:06.000 DEBUG:boto:Token: None 2022-09-03T07:28:06.001 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.002 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.003 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.008 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000031da5df62734c18a-0063130206-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.008 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.009 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:06.010 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:06.010 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.011 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:06.012 DEBUG:boto:Data: 2022-09-03T07:28:06.012 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.013 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.014 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.014 DEBUG:boto:Params: {} 2022-09-03T07:28:06.015 DEBUG:boto:Token: None 2022-09-03T07:28:06.016 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj87 2022-09-03T07:28:06.016 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x3r8XfP+p4ISDmCJltjRdgmNqD8= 2022-09-03T07:28:06.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x3r8XfP+p4ISDmCJltjRdgmNqD8=', 'Content-Length': '0'} 2022-09-03T07:28:06.019 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000efd5046754d98e7-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.020 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:06.021 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:06.021 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.022 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:06.023 DEBUG:boto:Data: 2022-09-03T07:28:06.023 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.024 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.024 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.025 DEBUG:boto:Params: {} 2022-09-03T07:28:06.025 DEBUG:boto:Token: None 2022-09-03T07:28:06.026 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj87 2022-09-03T07:28:06.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x3r8XfP+p4ISDmCJltjRdgmNqD8= 2022-09-03T07:28:06.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x3r8XfP+p4ISDmCJltjRdgmNqD8=', 'Content-Length': '0'} 2022-09-03T07:28:06.029 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005db3dfdda18eb6e2-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.031 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:28:06.032 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:06.033 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:06.033 DEBUG:boto:Method: GET 2022-09-03T07:28:06.034 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:06.035 DEBUG:boto:Data: 2022-09-03T07:28:06.036 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.037 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.039 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.039 DEBUG:boto:Params: {} 2022-09-03T07:28:06.041 DEBUG:boto:Token: None 2022-09-03T07:28:06.042 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj87 2022-09-03T07:28:06.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kJuTHfTIqDgzwvswn28w0/azHao= 2022-09-03T07:28:06.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kJuTHfTIqDgzwvswn28w0/azHao=', 'Content-Length': '0'} 2022-09-03T07:28:06.047 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c68d53d936c5a3e0-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.049 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:06.050 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:06.051 DEBUG:boto:Method: GET 2022-09-03T07:28:06.052 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:06.052 DEBUG:boto:Data: 2022-09-03T07:28:06.053 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.055 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.056 DEBUG:boto:Params: {} 2022-09-03T07:28:06.056 DEBUG:boto:Token: None 2022-09-03T07:28:06.058 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj87 2022-09-03T07:28:06.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kJuTHfTIqDgzwvswn28w0/azHao= 2022-09-03T07:28:06.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kJuTHfTIqDgzwvswn28w0/azHao=', 'Content-Length': '0'} 2022-09-03T07:28:06.063 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000bf0383460336a92-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.110 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.111 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.112 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:06.112 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:06.113 DEBUG:boto:Method: GET 2022-09-03T07:28:06.113 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:06.114 DEBUG:boto:Data: 2022-09-03T07:28:06.115 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.115 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.116 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.117 DEBUG:boto:Params: {} 2022-09-03T07:28:06.117 DEBUG:boto:Token: None 2022-09-03T07:28:06.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.125 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004cf03aa12b72ab0d-0063130206-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.126 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.127 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.128 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.128 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:06.129 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:06.129 DEBUG:boto:Method: GET 2022-09-03T07:28:06.130 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:06.130 DEBUG:boto:Data: 2022-09-03T07:28:06.131 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.132 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.133 DEBUG:boto:Params: {} 2022-09-03T07:28:06.133 DEBUG:boto:Token: None 2022-09-03T07:28:06.134 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.142 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ced84cdb79ddee9b-0063130206-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.143 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.145 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:28:06.146 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.147 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.147 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.148 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.149 DEBUG:boto:Method: GET 2022-09-03T07:28:06.150 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.151 DEBUG:boto:Data: 2022-09-03T07:28:06.152 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.152 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.153 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.154 DEBUG:boto:Params: {} 2022-09-03T07:28:06.155 DEBUG:boto:Token: None 2022-09-03T07:28:06.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zxYN7phiQsU7BlTolHzHfFxDLOE= 2022-09-03T07:28:06.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zxYN7phiQsU7BlTolHzHfFxDLOE=', 'Content-Length': '0'} 2022-09-03T07:28:06.160 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000acfce6c7eb35cc71-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.162 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.163 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.164 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.166 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.167 DEBUG:boto:Method: GET 2022-09-03T07:28:06.168 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.169 DEBUG:boto:Data: 2022-09-03T07:28:06.170 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.172 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.174 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.175 DEBUG:boto:Params: {} 2022-09-03T07:28:06.176 DEBUG:boto:Token: None 2022-09-03T07:28:06.177 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zxYN7phiQsU7BlTolHzHfFxDLOE= 2022-09-03T07:28:06.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zxYN7phiQsU7BlTolHzHfFxDLOE=', 'Content-Length': '0'} 2022-09-03T07:28:06.188 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000073574c1f0334f1f-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.189 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.190 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.190 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.191 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.192 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.192 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.193 DEBUG:boto:Data: 2022-09-03T07:28:06.193 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.194 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.195 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.195 DEBUG:boto:Params: {} 2022-09-03T07:28:06.196 DEBUG:boto:Token: None 2022-09-03T07:28:06.197 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EjzWnCJexjduh714zEfryCSAErQ= 2022-09-03T07:28:06.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EjzWnCJexjduh714zEfryCSAErQ=', 'Content-Length': '0'} 2022-09-03T07:28:06.210 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbe8835bcc71e11f-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.211 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.211 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.212 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.212 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.213 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.214 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.214 DEBUG:boto:Data: 2022-09-03T07:28:06.215 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.216 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.216 DEBUG:boto:Params: {} 2022-09-03T07:28:06.217 DEBUG:boto:Token: None 2022-09-03T07:28:06.217 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EjzWnCJexjduh714zEfryCSAErQ= 2022-09-03T07:28:06.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EjzWnCJexjduh714zEfryCSAErQ=', 'Content-Length': '0'} 2022-09-03T07:28:06.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026ebbab560af4289-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.222 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:28:06.222 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.223 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.223 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.224 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.224 DEBUG:boto:Method: GET 2022-09-03T07:28:06.225 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.226 DEBUG:boto:Data: 2022-09-03T07:28:06.227 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.227 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.228 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.229 DEBUG:boto:Params: {} 2022-09-03T07:28:06.229 DEBUG:boto:Token: None 2022-09-03T07:28:06.230 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zxYN7phiQsU7BlTolHzHfFxDLOE= 2022-09-03T07:28:06.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zxYN7phiQsU7BlTolHzHfFxDLOE=', 'Content-Length': '0'} 2022-09-03T07:28:06.234 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fa62f1fb0296e8d-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.235 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.235 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.236 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.236 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.237 DEBUG:boto:Method: GET 2022-09-03T07:28:06.237 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.238 DEBUG:boto:Data: 2022-09-03T07:28:06.239 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.240 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.240 DEBUG:boto:Params: {} 2022-09-03T07:28:06.241 DEBUG:boto:Token: None 2022-09-03T07:28:06.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:06.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zxYN7phiQsU7BlTolHzHfFxDLOE= 2022-09-03T07:28:06.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zxYN7phiQsU7BlTolHzHfFxDLOE=', 'Content-Length': '0'} 2022-09-03T07:28:06.245 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f80a79f90f757d89-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.246 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.247 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.247 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:06.248 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:06.248 DEBUG:boto:Method: GET 2022-09-03T07:28:06.249 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:06.250 DEBUG:boto:Data: 2022-09-03T07:28:06.251 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.251 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.252 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.252 DEBUG:boto:Params: {} 2022-09-03T07:28:06.253 DEBUG:boto:Token: None 2022-09-03T07:28:06.253 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.263 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023006c46d5880713-0063130206-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.264 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.267 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.267 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.268 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:06.269 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:06.271 DEBUG:boto:Method: GET 2022-09-03T07:28:06.272 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:06.273 DEBUG:boto:Data: 2022-09-03T07:28:06.274 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.275 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.276 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.277 DEBUG:boto:Params: {} 2022-09-03T07:28:06.278 DEBUG:boto:Token: None 2022-09-03T07:28:06.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.286 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c170369356605665-0063130206-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.287 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.288 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.289 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.289 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.290 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:06.290 DEBUG:boto:Data: 2022-09-03T07:28:06.291 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.292 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.292 DEBUG:boto:Params: {} 2022-09-03T07:28:06.293 DEBUG:boto:Token: None 2022-09-03T07:28:06.293 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92 2022-09-03T07:28:06.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zIgHX+haRqNcM7PsKWRrAcE6kl0= 2022-09-03T07:28:06.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zIgHX+haRqNcM7PsKWRrAcE6kl0=', 'Content-Length': '0'} 2022-09-03T07:28:06.297 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f322bd338edfd6ea-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.298 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.299 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.299 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.300 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:06.300 DEBUG:boto:Data: 2022-09-03T07:28:06.301 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.301 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.302 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.303 DEBUG:boto:Params: {} 2022-09-03T07:28:06.303 DEBUG:boto:Token: None 2022-09-03T07:28:06.304 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92 2022-09-03T07:28:06.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zIgHX+haRqNcM7PsKWRrAcE6kl0= 2022-09-03T07:28:06.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zIgHX+haRqNcM7PsKWRrAcE6kl0=', 'Content-Length': '0'} 2022-09-03T07:28:06.308 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010c08e9860a1946a-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.310 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:28:06.311 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.312 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.313 DEBUG:boto:Method: GET 2022-09-03T07:28:06.314 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:06.319 DEBUG:boto:Data: 2022-09-03T07:28:06.320 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.321 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.322 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.323 DEBUG:boto:Params: {} 2022-09-03T07:28:06.325 DEBUG:boto:Token: None 2022-09-03T07:28:06.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92 2022-09-03T07:28:06.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3/+c9oalPBj1oyuu9Fbwr8jecdQ= 2022-09-03T07:28:06.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3/+c9oalPBj1oyuu9Fbwr8jecdQ=', 'Content-Length': '0'} 2022-09-03T07:28:06.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fcb8de96a75b7203-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.374 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:06.375 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:06.377 DEBUG:boto:Method: GET 2022-09-03T07:28:06.378 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:06.380 DEBUG:boto:Data: 2022-09-03T07:28:06.381 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.382 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.383 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.384 DEBUG:boto:Params: {} 2022-09-03T07:28:06.385 DEBUG:boto:Token: None 2022-09-03T07:28:06.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj92 2022-09-03T07:28:06.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3/+c9oalPBj1oyuu9Fbwr8jecdQ= 2022-09-03T07:28:06.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3/+c9oalPBj1oyuu9Fbwr8jecdQ=', 'Content-Length': '0'} 2022-09-03T07:28:06.393 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f86ec4d27ec58e4-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.395 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.396 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.397 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:06.398 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:06.399 DEBUG:boto:Method: GET 2022-09-03T07:28:06.402 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:06.403 DEBUG:boto:Data: 2022-09-03T07:28:06.403 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.404 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.405 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.406 DEBUG:boto:Params: {} 2022-09-03T07:28:06.407 DEBUG:boto:Token: None 2022-09-03T07:28:06.408 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.409 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.410 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.413 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002cf1083054dde0ea-0063130206-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.414 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.416 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.417 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.418 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:06.418 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:06.419 DEBUG:boto:Method: GET 2022-09-03T07:28:06.420 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:06.421 DEBUG:boto:Data: 2022-09-03T07:28:06.422 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.424 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.425 DEBUG:boto:Params: {} 2022-09-03T07:28:06.427 DEBUG:boto:Token: None 2022-09-03T07:28:06.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.429 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.437 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002eb6f1841e790f5e-0063130206-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.438 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.439 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:28:06.440 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.441 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.441 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.442 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.443 DEBUG:boto:Method: GET 2022-09-03T07:28:06.443 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.444 DEBUG:boto:Data: 2022-09-03T07:28:06.444 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.445 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.446 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.447 DEBUG:boto:Params: {} 2022-09-03T07:28:06.447 DEBUG:boto:Token: None 2022-09-03T07:28:06.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GkZ3YEKhaBQbyN3PYumqHPHV654= 2022-09-03T07:28:06.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GkZ3YEKhaBQbyN3PYumqHPHV654=', 'Content-Length': '0'} 2022-09-03T07:28:06.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000069799c7ab83b9f57-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.455 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.456 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.457 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.458 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.459 DEBUG:boto:Method: GET 2022-09-03T07:28:06.459 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.460 DEBUG:boto:Data: 2022-09-03T07:28:06.461 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.463 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.464 DEBUG:boto:Params: {} 2022-09-03T07:28:06.465 DEBUG:boto:Token: None 2022-09-03T07:28:06.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GkZ3YEKhaBQbyN3PYumqHPHV654= 2022-09-03T07:28:06.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GkZ3YEKhaBQbyN3PYumqHPHV654=', 'Content-Length': '0'} 2022-09-03T07:28:06.471 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008585ec6a20b4316-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.518 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.519 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.520 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.522 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.523 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.524 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.525 DEBUG:boto:Data: 2022-09-03T07:28:06.526 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.527 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.527 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.528 DEBUG:boto:Params: {} 2022-09-03T07:28:06.528 DEBUG:boto:Token: None 2022-09-03T07:28:06.529 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U+rrQSneXjsMNRbPRBHmm7FYl+Y= 2022-09-03T07:28:06.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U+rrQSneXjsMNRbPRBHmm7FYl+Y=', 'Content-Length': '0'} 2022-09-03T07:28:06.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a22c2b57faf542f9-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.534 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.534 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.535 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.536 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.536 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.537 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.538 DEBUG:boto:Data: 2022-09-03T07:28:06.539 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.541 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.542 DEBUG:boto:Params: {} 2022-09-03T07:28:06.543 DEBUG:boto:Token: None 2022-09-03T07:28:06.544 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U+rrQSneXjsMNRbPRBHmm7FYl+Y= 2022-09-03T07:28:06.546 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U+rrQSneXjsMNRbPRBHmm7FYl+Y=', 'Content-Length': '0'} 2022-09-03T07:28:06.550 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009c3e377c80632b4b-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.551 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:28:06.552 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.553 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.554 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.556 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.557 DEBUG:boto:Method: GET 2022-09-03T07:28:06.558 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.559 DEBUG:boto:Data: 2022-09-03T07:28:06.560 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.561 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.562 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.562 DEBUG:boto:Params: {} 2022-09-03T07:28:06.563 DEBUG:boto:Token: None 2022-09-03T07:28:06.564 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.565 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GkZ3YEKhaBQbyN3PYumqHPHV654= 2022-09-03T07:28:06.566 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GkZ3YEKhaBQbyN3PYumqHPHV654=', 'Content-Length': '0'} 2022-09-03T07:28:06.569 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007635ee54448de36c-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.570 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.571 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.572 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.574 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.575 DEBUG:boto:Method: GET 2022-09-03T07:28:06.576 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.577 DEBUG:boto:Data: 2022-09-03T07:28:06.577 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.578 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.578 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.579 DEBUG:boto:Params: {} 2022-09-03T07:28:06.580 DEBUG:boto:Token: None 2022-09-03T07:28:06.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:06.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GkZ3YEKhaBQbyN3PYumqHPHV654= 2022-09-03T07:28:06.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GkZ3YEKhaBQbyN3PYumqHPHV654=', 'Content-Length': '0'} 2022-09-03T07:28:06.586 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000622999f6a9d151af-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.587 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.588 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.589 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:06.590 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:06.591 DEBUG:boto:Method: GET 2022-09-03T07:28:06.592 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:06.593 DEBUG:boto:Data: 2022-09-03T07:28:06.594 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.596 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.597 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.597 DEBUG:boto:Params: {} 2022-09-03T07:28:06.598 DEBUG:boto:Token: None 2022-09-03T07:28:06.599 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.607 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000078ab2bda78eeca9-0063130206-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.608 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.610 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.611 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.611 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:06.612 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:06.612 DEBUG:boto:Method: GET 2022-09-03T07:28:06.613 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:06.613 DEBUG:boto:Data: 2022-09-03T07:28:06.614 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.616 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.617 DEBUG:boto:Params: {} 2022-09-03T07:28:06.618 DEBUG:boto:Token: None 2022-09-03T07:28:06.619 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.626 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7a529d7da2b0e1f-0063130206-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.628 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.630 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.631 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.631 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.632 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:06.633 DEBUG:boto:Data: 2022-09-03T07:28:06.634 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.635 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.636 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.637 DEBUG:boto:Params: {} 2022-09-03T07:28:06.638 DEBUG:boto:Token: None 2022-09-03T07:28:06.639 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93 2022-09-03T07:28:06.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ptdo4XjODXS4lAYrmXNg8n2Tp2I= 2022-09-03T07:28:06.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ptdo4XjODXS4lAYrmXNg8n2Tp2I=', 'Content-Length': '0'} 2022-09-03T07:28:06.642 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e89369caa9d985c-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.643 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.643 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.644 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.645 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:06.645 DEBUG:boto:Data: 2022-09-03T07:28:06.646 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.647 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.647 DEBUG:boto:Params: {} 2022-09-03T07:28:06.648 DEBUG:boto:Token: None 2022-09-03T07:28:06.649 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93 2022-09-03T07:28:06.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ptdo4XjODXS4lAYrmXNg8n2Tp2I= 2022-09-03T07:28:06.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ptdo4XjODXS4lAYrmXNg8n2Tp2I=', 'Content-Length': '0'} 2022-09-03T07:28:06.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063fe6ed09a426894-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.653 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:28:06.654 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.654 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.655 DEBUG:boto:Method: GET 2022-09-03T07:28:06.655 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:06.656 DEBUG:boto:Data: 2022-09-03T07:28:06.656 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.657 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.658 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.658 DEBUG:boto:Params: {} 2022-09-03T07:28:06.659 DEBUG:boto:Token: None 2022-09-03T07:28:06.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93 2022-09-03T07:28:06.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AbQ2L0zQluz6lbGOwDIPV04bfqk= 2022-09-03T07:28:06.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AbQ2L0zQluz6lbGOwDIPV04bfqk=', 'Content-Length': '0'} 2022-09-03T07:28:06.664 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac8e4d65b2c00e06-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.712 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:06.713 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:06.713 DEBUG:boto:Method: GET 2022-09-03T07:28:06.714 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:06.715 DEBUG:boto:Data: 2022-09-03T07:28:06.715 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.716 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.717 DEBUG:boto:Params: {} 2022-09-03T07:28:06.717 DEBUG:boto:Token: None 2022-09-03T07:28:06.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj93 2022-09-03T07:28:06.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AbQ2L0zQluz6lbGOwDIPV04bfqk= 2022-09-03T07:28:06.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AbQ2L0zQluz6lbGOwDIPV04bfqk=', 'Content-Length': '0'} 2022-09-03T07:28:06.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000693c406e40649dfb-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.723 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.726 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.727 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:06.727 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:06.728 DEBUG:boto:Method: GET 2022-09-03T07:28:06.729 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:06.730 DEBUG:boto:Data: 2022-09-03T07:28:06.731 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.732 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.733 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.733 DEBUG:boto:Params: {} 2022-09-03T07:28:06.734 DEBUG:boto:Token: None 2022-09-03T07:28:06.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.739 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6af3efcdbe622a6-0063130206-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.740 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.740 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.741 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.742 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:06.742 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:06.743 DEBUG:boto:Method: GET 2022-09-03T07:28:06.743 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:06.744 DEBUG:boto:Data: 2022-09-03T07:28:06.744 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.745 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.746 DEBUG:boto:Params: {} 2022-09-03T07:28:06.746 DEBUG:boto:Token: None 2022-09-03T07:28:06.747 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.753 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a8d4b29da4e849d-0063130206-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.754 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.754 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:28:06.755 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:06.756 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:06.756 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.757 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.757 DEBUG:boto:Method: GET 2022-09-03T07:28:06.758 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.759 DEBUG:boto:Data: 2022-09-03T07:28:06.759 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.760 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.760 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.761 DEBUG:boto:Params: {} 2022-09-03T07:28:06.761 DEBUG:boto:Token: None 2022-09-03T07:28:06.762 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.763 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9nv3HozGK2Mmc5J4zg/HuhV6Om0= 2022-09-03T07:28:06.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9nv3HozGK2Mmc5J4zg/HuhV6Om0=', 'Content-Length': '0'} 2022-09-03T07:28:06.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e98425bc1f85de01-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.767 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:06.767 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:06.768 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.768 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.769 DEBUG:boto:Method: GET 2022-09-03T07:28:06.769 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.770 DEBUG:boto:Data: 2022-09-03T07:28:06.771 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.772 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.774 DEBUG:boto:Params: {} 2022-09-03T07:28:06.775 DEBUG:boto:Token: None 2022-09-03T07:28:06.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9nv3HozGK2Mmc5J4zg/HuhV6Om0= 2022-09-03T07:28:06.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9nv3HozGK2Mmc5J4zg/HuhV6Om0=', 'Content-Length': '0'} 2022-09-03T07:28:06.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b4b951039208fc13-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.782 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:06.783 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:06.784 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.785 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.786 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.787 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.788 DEBUG:boto:Data: 2022-09-03T07:28:06.789 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.790 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.791 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.792 DEBUG:boto:Params: {} 2022-09-03T07:28:06.793 DEBUG:boto:Token: None 2022-09-03T07:28:06.794 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8AH0DAxqAV+YjR1g7DZJY0QF8GY= 2022-09-03T07:28:06.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8AH0DAxqAV+YjR1g7DZJY0QF8GY=', 'Content-Length': '0'} 2022-09-03T07:28:06.799 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c15b8d49bdd7f90-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.800 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:06.801 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:06.802 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.803 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.804 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.805 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.806 DEBUG:boto:Data: 2022-09-03T07:28:06.807 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.809 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.809 DEBUG:boto:Params: {} 2022-09-03T07:28:06.811 DEBUG:boto:Token: None 2022-09-03T07:28:06.812 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8AH0DAxqAV+YjR1g7DZJY0QF8GY= 2022-09-03T07:28:06.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8AH0DAxqAV+YjR1g7DZJY0QF8GY=', 'Content-Length': '0'} 2022-09-03T07:28:06.816 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e2198b106d968c7-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.817 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:28:06.818 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:06.819 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:06.820 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.821 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.822 DEBUG:boto:Method: GET 2022-09-03T07:28:06.822 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.823 DEBUG:boto:Data: 2022-09-03T07:28:06.823 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.824 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.825 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.825 DEBUG:boto:Params: {} 2022-09-03T07:28:06.826 DEBUG:boto:Token: None 2022-09-03T07:28:06.827 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9nv3HozGK2Mmc5J4zg/HuhV6Om0= 2022-09-03T07:28:06.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9nv3HozGK2Mmc5J4zg/HuhV6Om0=', 'Content-Length': '0'} 2022-09-03T07:28:06.833 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b998f2f6eaa065f-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.874 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:06.875 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:06.876 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.876 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.877 DEBUG:boto:Method: GET 2022-09-03T07:28:06.878 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.878 DEBUG:boto:Data: 2022-09-03T07:28:06.879 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.880 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.881 DEBUG:boto:Params: {} 2022-09-03T07:28:06.881 DEBUG:boto:Token: None 2022-09-03T07:28:06.882 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:06.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9nv3HozGK2Mmc5J4zg/HuhV6Om0= 2022-09-03T07:28:06.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9nv3HozGK2Mmc5J4zg/HuhV6Om0=', 'Content-Length': '0'} 2022-09-03T07:28:06.887 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c45faeadba21dc3-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.888 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.889 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.890 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:06.890 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:06.891 DEBUG:boto:Method: GET 2022-09-03T07:28:06.891 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:06.892 DEBUG:boto:Data: 2022-09-03T07:28:06.893 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.893 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.894 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.894 DEBUG:boto:Params: {} 2022-09-03T07:28:06.895 DEBUG:boto:Token: None 2022-09-03T07:28:06.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.900 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e44c59470fc4dfd5-0063130206-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.901 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.901 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:06.902 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:06.903 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:06.903 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:06.904 DEBUG:boto:Method: GET 2022-09-03T07:28:06.904 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:06.905 DEBUG:boto:Data: 2022-09-03T07:28:06.905 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.907 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.907 DEBUG:boto:Params: {} 2022-09-03T07:28:06.908 DEBUG:boto:Token: None 2022-09-03T07:28:06.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/ 2022-09-03T07:28:06.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA= 2022-09-03T07:28:06.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4nbSvG+/uUC4w4+dMUJKPZV8PrA=', 'Content-Length': '0'} 2022-09-03T07:28:06.914 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fdad081c5f2ec414-0063130206-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.962 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:06.964 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:06.965 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:06.966 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.968 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:06.969 DEBUG:boto:Data: 2022-09-03T07:28:06.969 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.970 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.970 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.971 DEBUG:boto:Params: {} 2022-09-03T07:28:06.971 DEBUG:boto:Token: None 2022-09-03T07:28:06.972 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj95 2022-09-03T07:28:06.973 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YX2lQmdc1ahZr3H7otHexIM2NQo= 2022-09-03T07:28:06.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YX2lQmdc1ahZr3H7otHexIM2NQo=', 'Content-Length': '0'} 2022-09-03T07:28:06.975 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e10bfbd02f55f319-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.977 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:06.978 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:06.979 DEBUG:boto:Method: HEAD 2022-09-03T07:28:06.980 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:06.981 DEBUG:boto:Data: 2022-09-03T07:28:06.982 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:06.983 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.984 DEBUG:boto:Params: {} 2022-09-03T07:28:06.984 DEBUG:boto:Token: None 2022-09-03T07:28:06.985 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj95 2022-09-03T07:28:06.986 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YX2lQmdc1ahZr3H7otHexIM2NQo= 2022-09-03T07:28:06.986 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YX2lQmdc1ahZr3H7otHexIM2NQo=', 'Content-Length': '0'} 2022-09-03T07:28:06.989 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000622bad29ba3df76f-0063130206-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:06.990 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:28:06.991 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:06.991 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:06.992 DEBUG:boto:Method: GET 2022-09-03T07:28:06.993 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:06.993 DEBUG:boto:Data: 2022-09-03T07:28:06.994 DEBUG:boto:Headers: {} 2022-09-03T07:28:06.995 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:06.995 DEBUG:boto:Port: 8000 2022-09-03T07:28:06.996 DEBUG:boto:Params: {} 2022-09-03T07:28:06.997 DEBUG:boto:Token: None 2022-09-03T07:28:06.997 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:06 GMT /avysmw-75/obj95 2022-09-03T07:28:06.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HFYNeAmqIsaPCACoNBTu1sHEy1Q= 2022-09-03T07:28:06.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HFYNeAmqIsaPCACoNBTu1sHEy1Q=', 'Content-Length': '0'} 2022-09-03T07:28:07.001 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075eec65246834c8f-0063130206-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.003 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:07.004 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:07.006 DEBUG:boto:Method: GET 2022-09-03T07:28:07.007 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:07.008 DEBUG:boto:Data: 2022-09-03T07:28:07.010 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.012 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.014 DEBUG:boto:Params: {} 2022-09-03T07:28:07.015 DEBUG:boto:Token: None 2022-09-03T07:28:07.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj95 2022-09-03T07:28:07.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vKG/xEcpwBqggSc/SDnJuRY2XC4= 2022-09-03T07:28:07.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vKG/xEcpwBqggSc/SDnJuRY2XC4=', 'Content-Length': '0'} 2022-09-03T07:28:07.025 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000088e6b90923636f91-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.026 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.027 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.028 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:07.029 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:07.030 DEBUG:boto:Method: GET 2022-09-03T07:28:07.032 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:07.033 DEBUG:boto:Data: 2022-09-03T07:28:07.040 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.041 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:07.042 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.043 DEBUG:boto:Params: {} 2022-09-03T07:28:07.044 DEBUG:boto:Token: None 2022-09-03T07:28:07.045 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.050 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1f0a781d54b5969-0063130207-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.052 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.053 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.054 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.055 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:07.056 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:07.057 DEBUG:boto:Method: GET 2022-09-03T07:28:07.058 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:07.059 DEBUG:boto:Data: 2022-09-03T07:28:07.059 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.060 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.061 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.061 DEBUG:boto:Params: {} 2022-09-03T07:28:07.062 DEBUG:boto:Token: None 2022-09-03T07:28:07.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.070 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000008bfe12237abc473-0063130207-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.071 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.073 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:28:07.074 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:28:07.074 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.075 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.076 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.077 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:28:07.078 DEBUG:boto:Data: 2022-09-03T07:28:07.079 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.080 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.081 DEBUG:boto:Params: {} 2022-09-03T07:28:07.082 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:28:07.083 DEBUG:boto:Token: None 2022-09-03T07:28:07.083 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YCIJh63FTzdTbIlkLUULzdd079k= 2022-09-03T07:28:07.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YCIJh63FTzdTbIlkLUULzdd079k=', 'Content-Length': '0'} 2022-09-03T07:28:07.089 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '30'), ('X-RGW-Bytes-Used', '90'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002d41a7292d84fe91-0063130207-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.089 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.090 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.091 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.091 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:28:07.092 DEBUG:boto:Data: 2022-09-03T07:28:07.092 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.094 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.094 DEBUG:boto:Params: {} 2022-09-03T07:28:07.095 DEBUG:boto:Token: None 2022-09-03T07:28:07.096 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.096 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YCIJh63FTzdTbIlkLUULzdd079k= 2022-09-03T07:28:07.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YCIJh63FTzdTbIlkLUULzdd079k=', 'Content-Length': '0'} 2022-09-03T07:28:07.102 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '30'), ('X-RGW-Bytes-Used', '90'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d6c473f39c228bab-0063130207-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.103 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:28:07.103 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.104 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.104 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.105 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.106 DEBUG:boto:Method: GET 2022-09-03T07:28:07.106 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.107 DEBUG:boto:Data: 2022-09-03T07:28:07.107 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.109 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.111 DEBUG:boto:Params: {} 2022-09-03T07:28:07.111 DEBUG:boto:Token: None 2022-09-03T07:28:07.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/?versions 2022-09-03T07:28:07.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wynG97v4vE5z4XBjeDy+m/dFIQI= 2022-09-03T07:28:07.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wynG97v4vE5z4XBjeDy+m/dFIQI=', 'Content-Length': '0'} 2022-09-03T07:28:07.119 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076b0b00db92bd009-0063130207-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.121 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.125 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:28:07.126 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:28:07.127 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:28:07.128 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:28:07.129 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:28:07.130 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:28:07.131 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:28:07.131 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:28:07.132 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:28:07.133 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:28:07.133 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:28:07.134 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:28:07.134 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:28:07.135 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:28:07.135 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:28:07.136 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:28:07.136 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:28:07.137 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:28:07.137 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:28:07.138 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:28:07.138 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:28:07.139 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:28:07.140 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:28:07.140 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:28:07.141 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:28:07.141 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:28:07.142 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:28:07.143 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:28:07.144 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:28:07.144 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:28:07.145 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:28:07.150 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.151 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.151 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.152 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.153 DEBUG:boto:Method: GET 2022-09-03T07:28:07.154 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.155 DEBUG:boto:Data: 2022-09-03T07:28:07.156 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.157 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.158 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.159 DEBUG:boto:Params: {} 2022-09-03T07:28:07.160 DEBUG:boto:Token: None 2022-09-03T07:28:07.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/?versions 2022-09-03T07:28:07.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wynG97v4vE5z4XBjeDy+m/dFIQI= 2022-09-03T07:28:07.163 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wynG97v4vE5z4XBjeDy+m/dFIQI=', 'Content-Length': '0'} 2022-09-03T07:28:07.173 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a05c88a9b4c8c87-0063130207-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.174 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.178 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:28:07.179 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:28:07.180 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:28:07.181 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:28:07.182 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:28:07.183 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:28:07.184 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:28:07.185 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:28:07.185 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:28:07.186 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:28:07.187 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:28:07.188 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:28:07.189 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:28:07.190 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:28:07.191 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:28:07.191 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:28:07.192 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:28:07.193 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:28:07.194 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:28:07.195 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:28:07.196 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:28:07.196 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:28:07.197 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:28:07.198 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:28:07.199 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:28:07.200 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:28:07.201 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:28:07.201 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:28:07.202 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:28:07.203 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:28:07.204 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.205 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.206 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.207 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.208 DEBUG:boto:Method: GET 2022-09-03T07:28:07.209 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.210 DEBUG:boto:Data: 2022-09-03T07:28:07.211 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.213 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.214 DEBUG:boto:Params: {} 2022-09-03T07:28:07.216 DEBUG:boto:Token: None 2022-09-03T07:28:07.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/?versions 2022-09-03T07:28:07.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wynG97v4vE5z4XBjeDy+m/dFIQI= 2022-09-03T07:28:07.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wynG97v4vE5z4XBjeDy+m/dFIQI=', 'Content-Length': '0'} 2022-09-03T07:28:07.224 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001e7fde9e3fe1d640-0063130207-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.227 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.232 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.234 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.235 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.236 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.237 DEBUG:boto:Method: GET 2022-09-03T07:28:07.238 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:28:07.239 DEBUG:boto:Data: 2022-09-03T07:28:07.239 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.240 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.241 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.242 DEBUG:boto:Params: {} 2022-09-03T07:28:07.243 DEBUG:boto:Token: None 2022-09-03T07:28:07.244 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/?versions 2022-09-03T07:28:07.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wynG97v4vE5z4XBjeDy+m/dFIQI= 2022-09-03T07:28:07.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wynG97v4vE5z4XBjeDy+m/dFIQI=', 'Content-Length': '0'} 2022-09-03T07:28:07.253 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000003d59ac069ef4114-0063130207-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.254 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.259 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:28:07.260 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.262 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.263 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.264 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.265 DEBUG:boto:Method: GET 2022-09-03T07:28:07.266 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.268 DEBUG:boto:Data: 2022-09-03T07:28:07.269 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.272 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.273 DEBUG:boto:Params: {} 2022-09-03T07:28:07.275 DEBUG:boto:Token: None 2022-09-03T07:28:07.276 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wuDk+BLu+rTtc33P+klaQUHJjDY= 2022-09-03T07:28:07.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wuDk+BLu+rTtc33P+klaQUHJjDY=', 'Content-Length': '0'} 2022-09-03T07:28:07.281 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089588c8583834783-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.282 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.283 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.284 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.284 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.285 DEBUG:boto:Method: GET 2022-09-03T07:28:07.286 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.287 DEBUG:boto:Data: 2022-09-03T07:28:07.288 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.290 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.291 DEBUG:boto:Params: {} 2022-09-03T07:28:07.292 DEBUG:boto:Token: None 2022-09-03T07:28:07.294 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wuDk+BLu+rTtc33P+klaQUHJjDY= 2022-09-03T07:28:07.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wuDk+BLu+rTtc33P+klaQUHJjDY=', 'Content-Length': '0'} 2022-09-03T07:28:07.299 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f38ddba9fe937eb-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.301 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.302 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.303 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.305 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.305 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.306 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.306 DEBUG:boto:Data: 2022-09-03T07:28:07.307 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.308 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.309 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.309 DEBUG:boto:Params: {} 2022-09-03T07:28:07.311 DEBUG:boto:Token: None 2022-09-03T07:28:07.312 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.313 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0udhTcLwq/tqhp6MjkmtEa/s6gg= 2022-09-03T07:28:07.316 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0udhTcLwq/tqhp6MjkmtEa/s6gg=', 'Content-Length': '0'} 2022-09-03T07:28:07.318 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000033199551dcced76a-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.320 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.322 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.322 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.323 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.324 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.324 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.326 DEBUG:boto:Data: 2022-09-03T07:28:07.327 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.328 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.329 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.331 DEBUG:boto:Params: {} 2022-09-03T07:28:07.332 DEBUG:boto:Token: None 2022-09-03T07:28:07.334 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0udhTcLwq/tqhp6MjkmtEa/s6gg= 2022-09-03T07:28:07.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0udhTcLwq/tqhp6MjkmtEa/s6gg=', 'Content-Length': '0'} 2022-09-03T07:28:07.338 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003eaaa2d0eea4c6fd-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.339 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:28:07.340 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.340 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.341 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.342 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.342 DEBUG:boto:Method: GET 2022-09-03T07:28:07.343 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.344 DEBUG:boto:Data: 2022-09-03T07:28:07.344 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.345 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.346 DEBUG:boto:Params: {} 2022-09-03T07:28:07.347 DEBUG:boto:Token: None 2022-09-03T07:28:07.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wuDk+BLu+rTtc33P+klaQUHJjDY= 2022-09-03T07:28:07.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wuDk+BLu+rTtc33P+klaQUHJjDY=', 'Content-Length': '0'} 2022-09-03T07:28:07.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000048010add58de0ea-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.356 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.357 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.358 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.360 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.362 DEBUG:boto:Method: GET 2022-09-03T07:28:07.363 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.364 DEBUG:boto:Data: 2022-09-03T07:28:07.365 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.366 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.367 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.368 DEBUG:boto:Params: {} 2022-09-03T07:28:07.369 DEBUG:boto:Token: None 2022-09-03T07:28:07.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:07.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wuDk+BLu+rTtc33P+klaQUHJjDY= 2022-09-03T07:28:07.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wuDk+BLu+rTtc33P+klaQUHJjDY=', 'Content-Length': '0'} 2022-09-03T07:28:07.379 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa0d57cbfbcb8096-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.418 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.420 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.421 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:07.423 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:07.424 DEBUG:boto:Method: GET 2022-09-03T07:28:07.425 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:28:07.427 DEBUG:boto:Data: 2022-09-03T07:28:07.428 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.430 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.431 DEBUG:boto:Params: {} 2022-09-03T07:28:07.432 DEBUG:boto:Token: None 2022-09-03T07:28:07.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.437 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.440 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000abec45e51a1320bb-0063130207-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.441 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.442 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.443 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.443 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:07.444 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:07.444 DEBUG:boto:Method: GET 2022-09-03T07:28:07.445 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:28:07.445 DEBUG:boto:Data: 2022-09-03T07:28:07.446 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.447 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.447 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.448 DEBUG:boto:Params: {} 2022-09-03T07:28:07.448 DEBUG:boto:Token: None 2022-09-03T07:28:07.449 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.450 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.456 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001bea1f5e5fbacb19-0063130207-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.457 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.459 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.459 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.460 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.461 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:28:07.462 DEBUG:boto:Data: 2022-09-03T07:28:07.463 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.464 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.465 DEBUG:boto:Params: {} 2022-09-03T07:28:07.467 DEBUG:boto:Token: None 2022-09-03T07:28:07.468 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11 2022-09-03T07:28:07.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dmg/SMUaUq0AnPUgsc0pdKFvZSM= 2022-09-03T07:28:07.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dmg/SMUaUq0AnPUgsc0pdKFvZSM=', 'Content-Length': '0'} 2022-09-03T07:28:07.472 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd5599aa88637a2e-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.473 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.474 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.475 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.476 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:28:07.477 DEBUG:boto:Data: 2022-09-03T07:28:07.478 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.479 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.480 DEBUG:boto:Params: {} 2022-09-03T07:28:07.481 DEBUG:boto:Token: None 2022-09-03T07:28:07.482 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11 2022-09-03T07:28:07.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Dmg/SMUaUq0AnPUgsc0pdKFvZSM= 2022-09-03T07:28:07.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Dmg/SMUaUq0AnPUgsc0pdKFvZSM=', 'Content-Length': '0'} 2022-09-03T07:28:07.486 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b27fcda0ff2d97d2-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.487 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:28:07.488 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.488 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.489 DEBUG:boto:Method: GET 2022-09-03T07:28:07.490 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:28:07.491 DEBUG:boto:Data: 2022-09-03T07:28:07.492 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.494 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.495 DEBUG:boto:Params: {} 2022-09-03T07:28:07.496 DEBUG:boto:Token: None 2022-09-03T07:28:07.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11 2022-09-03T07:28:07.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AAAJG+EsryXZDOMf9850mk7Xl8c= 2022-09-03T07:28:07.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AAAJG+EsryXZDOMf9850mk7Xl8c=', 'Content-Length': '0'} 2022-09-03T07:28:07.502 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ccb1a1f01153235-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.503 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:07.504 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:07.505 DEBUG:boto:Method: GET 2022-09-03T07:28:07.506 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:28:07.507 DEBUG:boto:Data: 2022-09-03T07:28:07.508 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.509 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.509 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.510 DEBUG:boto:Params: {} 2022-09-03T07:28:07.511 DEBUG:boto:Token: None 2022-09-03T07:28:07.512 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj11 2022-09-03T07:28:07.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AAAJG+EsryXZDOMf9850mk7Xl8c= 2022-09-03T07:28:07.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AAAJG+EsryXZDOMf9850mk7Xl8c=', 'Content-Length': '0'} 2022-09-03T07:28:07.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000efc184f1d4fc0c04-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.520 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.521 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.522 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:07.523 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:07.524 DEBUG:boto:Method: GET 2022-09-03T07:28:07.525 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:28:07.526 DEBUG:boto:Data: 2022-09-03T07:28:07.527 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.528 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.529 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.530 DEBUG:boto:Params: {} 2022-09-03T07:28:07.531 DEBUG:boto:Token: None 2022-09-03T07:28:07.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.532 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.536 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7e792e1691d47a2-0063130207-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.537 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.538 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.538 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.539 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:07.539 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:07.541 DEBUG:boto:Method: GET 2022-09-03T07:28:07.541 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:28:07.542 DEBUG:boto:Data: 2022-09-03T07:28:07.543 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.546 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.547 DEBUG:boto:Params: {} 2022-09-03T07:28:07.549 DEBUG:boto:Token: None 2022-09-03T07:28:07.550 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.558 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098d0809f5e1934e5-0063130207-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.606 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.607 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:28:07.608 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.608 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.609 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.609 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.610 DEBUG:boto:Method: GET 2022-09-03T07:28:07.611 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.611 DEBUG:boto:Data: 2022-09-03T07:28:07.612 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.614 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.614 DEBUG:boto:Params: {} 2022-09-03T07:28:07.615 DEBUG:boto:Token: None 2022-09-03T07:28:07.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6PghccFSPjFURWjmj/5aEhl/cLg= 2022-09-03T07:28:07.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6PghccFSPjFURWjmj/5aEhl/cLg=', 'Content-Length': '0'} 2022-09-03T07:28:07.619 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035a11695ef2003aa-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.621 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.622 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.623 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.625 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.626 DEBUG:boto:Method: GET 2022-09-03T07:28:07.627 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.628 DEBUG:boto:Data: 2022-09-03T07:28:07.629 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.631 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.632 DEBUG:boto:Params: {} 2022-09-03T07:28:07.632 DEBUG:boto:Token: None 2022-09-03T07:28:07.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6PghccFSPjFURWjmj/5aEhl/cLg= 2022-09-03T07:28:07.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6PghccFSPjFURWjmj/5aEhl/cLg=', 'Content-Length': '0'} 2022-09-03T07:28:07.637 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc8197704df72691-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.637 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.638 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.638 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.639 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.640 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.640 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.641 DEBUG:boto:Data: 2022-09-03T07:28:07.642 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.643 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.643 DEBUG:boto:Params: {} 2022-09-03T07:28:07.644 DEBUG:boto:Token: None 2022-09-03T07:28:07.645 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fSShxl5aEdReUO3rURVM6UH+hIw= 2022-09-03T07:28:07.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fSShxl5aEdReUO3rURVM6UH+hIw=', 'Content-Length': '0'} 2022-09-03T07:28:07.648 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034004039a8f0b5c1-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.649 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.650 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.650 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.652 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.655 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.656 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.656 DEBUG:boto:Data: 2022-09-03T07:28:07.657 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.658 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.658 DEBUG:boto:Params: {} 2022-09-03T07:28:07.659 DEBUG:boto:Token: None 2022-09-03T07:28:07.660 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fSShxl5aEdReUO3rURVM6UH+hIw= 2022-09-03T07:28:07.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fSShxl5aEdReUO3rURVM6UH+hIw=', 'Content-Length': '0'} 2022-09-03T07:28:07.664 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000734e529768e924cf-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.665 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:28:07.665 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.666 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.666 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.667 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.668 DEBUG:boto:Method: GET 2022-09-03T07:28:07.668 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.669 DEBUG:boto:Data: 2022-09-03T07:28:07.669 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.670 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.671 DEBUG:boto:Params: {} 2022-09-03T07:28:07.672 DEBUG:boto:Token: None 2022-09-03T07:28:07.673 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6PghccFSPjFURWjmj/5aEhl/cLg= 2022-09-03T07:28:07.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6PghccFSPjFURWjmj/5aEhl/cLg=', 'Content-Length': '0'} 2022-09-03T07:28:07.677 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0bcb0f4251bcb99-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.678 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.679 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.679 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.680 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.681 DEBUG:boto:Method: GET 2022-09-03T07:28:07.681 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.682 DEBUG:boto:Data: 2022-09-03T07:28:07.683 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.684 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.684 DEBUG:boto:Params: {} 2022-09-03T07:28:07.685 DEBUG:boto:Token: None 2022-09-03T07:28:07.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:07.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6PghccFSPjFURWjmj/5aEhl/cLg= 2022-09-03T07:28:07.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6PghccFSPjFURWjmj/5aEhl/cLg=', 'Content-Length': '0'} 2022-09-03T07:28:07.690 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000145b37608fb8510-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.692 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.695 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.696 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:07.697 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:07.698 DEBUG:boto:Method: GET 2022-09-03T07:28:07.699 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:28:07.700 DEBUG:boto:Data: 2022-09-03T07:28:07.700 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.703 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.705 DEBUG:boto:Params: {} 2022-09-03T07:28:07.706 DEBUG:boto:Token: None 2022-09-03T07:28:07.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.713 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000074f15e1b50ef06a5-0063130207-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.754 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.755 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.756 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.756 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:07.757 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:07.757 DEBUG:boto:Method: GET 2022-09-03T07:28:07.758 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:28:07.758 DEBUG:boto:Data: 2022-09-03T07:28:07.759 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.760 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.760 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.761 DEBUG:boto:Params: {} 2022-09-03T07:28:07.761 DEBUG:boto:Token: None 2022-09-03T07:28:07.762 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.763 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.768 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d06797b24095975b-0063130207-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.769 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.770 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.771 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.771 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.772 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:28:07.772 DEBUG:boto:Data: 2022-09-03T07:28:07.773 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.773 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.774 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.774 DEBUG:boto:Params: {} 2022-09-03T07:28:07.775 DEBUG:boto:Token: None 2022-09-03T07:28:07.775 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115 2022-09-03T07:28:07.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5SpSL1DvDDyuDL2QJvEB+ZkP0h0= 2022-09-03T07:28:07.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5SpSL1DvDDyuDL2QJvEB+ZkP0h0=', 'Content-Length': '0'} 2022-09-03T07:28:07.779 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008580a1b869f23baa-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.780 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.780 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.781 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.781 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:28:07.782 DEBUG:boto:Data: 2022-09-03T07:28:07.782 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.783 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.784 DEBUG:boto:Params: {} 2022-09-03T07:28:07.784 DEBUG:boto:Token: None 2022-09-03T07:28:07.785 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115 2022-09-03T07:28:07.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5SpSL1DvDDyuDL2QJvEB+ZkP0h0= 2022-09-03T07:28:07.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5SpSL1DvDDyuDL2QJvEB+ZkP0h0=', 'Content-Length': '0'} 2022-09-03T07:28:07.788 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000033d8d986f3ffae8-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.789 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:28:07.790 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.790 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.791 DEBUG:boto:Method: GET 2022-09-03T07:28:07.791 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:28:07.792 DEBUG:boto:Data: 2022-09-03T07:28:07.793 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.794 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.794 DEBUG:boto:Params: {} 2022-09-03T07:28:07.795 DEBUG:boto:Token: None 2022-09-03T07:28:07.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115 2022-09-03T07:28:07.797 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TWJ7eWz9v5m4FpMkX/f4c+zNL0g= 2022-09-03T07:28:07.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TWJ7eWz9v5m4FpMkX/f4c+zNL0g=', 'Content-Length': '0'} 2022-09-03T07:28:07.800 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d79ab2befc32d5da-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.801 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:07.801 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:07.802 DEBUG:boto:Method: GET 2022-09-03T07:28:07.802 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:28:07.803 DEBUG:boto:Data: 2022-09-03T07:28:07.804 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.805 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.805 DEBUG:boto:Params: {} 2022-09-03T07:28:07.806 DEBUG:boto:Token: None 2022-09-03T07:28:07.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj115 2022-09-03T07:28:07.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TWJ7eWz9v5m4FpMkX/f4c+zNL0g= 2022-09-03T07:28:07.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TWJ7eWz9v5m4FpMkX/f4c+zNL0g=', 'Content-Length': '0'} 2022-09-03T07:28:07.810 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097b3099977acba04-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.811 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.812 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.812 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:07.813 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:07.813 DEBUG:boto:Method: GET 2022-09-03T07:28:07.814 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:28:07.814 DEBUG:boto:Data: 2022-09-03T07:28:07.815 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.816 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.817 DEBUG:boto:Params: {} 2022-09-03T07:28:07.818 DEBUG:boto:Token: None 2022-09-03T07:28:07.818 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.823 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000050a5d71c2d04db69-0063130207-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.824 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.825 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:07.825 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:07.826 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:07.826 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:07.827 DEBUG:boto:Method: GET 2022-09-03T07:28:07.828 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:28:07.828 DEBUG:boto:Data: 2022-09-03T07:28:07.829 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.831 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.832 DEBUG:boto:Params: {} 2022-09-03T07:28:07.834 DEBUG:boto:Token: None 2022-09-03T07:28:07.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/ 2022-09-03T07:28:07.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4= 2022-09-03T07:28:07.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yjqZYU5bQ9lam6/tgLMQMXjLzF4=', 'Content-Length': '0'} 2022-09-03T07:28:07.842 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1f73ff277355ded-0063130207-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.887 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:07.888 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:28:07.888 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:07.889 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:07.890 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.891 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.891 DEBUG:boto:Method: GET 2022-09-03T07:28:07.892 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.892 DEBUG:boto:Data: 2022-09-03T07:28:07.893 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.897 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.898 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.899 DEBUG:boto:Params: {} 2022-09-03T07:28:07.900 DEBUG:boto:Token: None 2022-09-03T07:28:07.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O18ZyQk8liYKD3yTAmwItFrjCyY= 2022-09-03T07:28:07.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O18ZyQk8liYKD3yTAmwItFrjCyY=', 'Content-Length': '0'} 2022-09-03T07:28:07.908 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4ff7d0faedc6f89-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.908 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:07.910 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:07.910 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.911 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.911 DEBUG:boto:Method: GET 2022-09-03T07:28:07.912 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.912 DEBUG:boto:Data: 2022-09-03T07:28:07.913 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.913 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.914 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.914 DEBUG:boto:Params: {} 2022-09-03T07:28:07.915 DEBUG:boto:Token: None 2022-09-03T07:28:07.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O18ZyQk8liYKD3yTAmwItFrjCyY= 2022-09-03T07:28:07.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O18ZyQk8liYKD3yTAmwItFrjCyY=', 'Content-Length': '0'} 2022-09-03T07:28:07.922 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054c74a81fcbd6d22-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.924 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:07.925 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:07.926 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.927 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.928 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.928 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.936 DEBUG:boto:Data: 2022-09-03T07:28:07.937 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.938 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.939 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.940 DEBUG:boto:Params: {} 2022-09-03T07:28:07.941 DEBUG:boto:Token: None 2022-09-03T07:28:07.942 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XxmA+/aL/uffYk7UNW6ETULPMok= 2022-09-03T07:28:07.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XxmA+/aL/uffYk7UNW6ETULPMok=', 'Content-Length': '0'} 2022-09-03T07:28:07.950 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065cbc6f5b3b8d41f-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.952 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:07.952 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:07.953 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.955 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.956 DEBUG:boto:Method: HEAD 2022-09-03T07:28:07.957 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.957 DEBUG:boto:Data: 2022-09-03T07:28:07.958 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.959 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:07.959 DEBUG:boto:Port: 8000 2022-09-03T07:28:07.960 DEBUG:boto:Params: {} 2022-09-03T07:28:07.962 DEBUG:boto:Token: None 2022-09-03T07:28:07.963 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.964 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XxmA+/aL/uffYk7UNW6ETULPMok= 2022-09-03T07:28:07.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XxmA+/aL/uffYk7UNW6ETULPMok=', 'Content-Length': '0'} 2022-09-03T07:28:07.967 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8688d002e627bee-0063130207-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:07.968 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:28:07.969 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:07.971 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:07.971 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.972 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.972 DEBUG:boto:Method: GET 2022-09-03T07:28:07.973 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.973 DEBUG:boto:Data: 2022-09-03T07:28:07.974 DEBUG:boto:Headers: {} 2022-09-03T07:28:07.975 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:07.976 DEBUG:boto:Port: 8001 2022-09-03T07:28:07.978 DEBUG:boto:Params: {} 2022-09-03T07:28:07.979 DEBUG:boto:Token: None 2022-09-03T07:28:07.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:07 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:07.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:O18ZyQk8liYKD3yTAmwItFrjCyY= 2022-09-03T07:28:07.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:O18ZyQk8liYKD3yTAmwItFrjCyY=', 'Content-Length': '0'} 2022-09-03T07:28:07.988 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dce39f4cd453e652-0063130207-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.026 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:08.027 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:08.028 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:08.028 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:08.029 DEBUG:boto:Method: GET 2022-09-03T07:28:08.029 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:08.030 DEBUG:boto:Data: 2022-09-03T07:28:08.030 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.032 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.033 DEBUG:boto:Params: {} 2022-09-03T07:28:08.033 DEBUG:boto:Token: None 2022-09-03T07:28:08.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:08.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jvhXayCm3nTQgkqjKPQIvgJ5z2E= 2022-09-03T07:28:08.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jvhXayCm3nTQgkqjKPQIvgJ5z2E=', 'Content-Length': '0'} 2022-09-03T07:28:08.044 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018f04592290e6f14-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.045 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.046 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.047 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:08.047 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:08.048 DEBUG:boto:Method: GET 2022-09-03T07:28:08.049 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:28:08.049 DEBUG:boto:Data: 2022-09-03T07:28:08.050 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.051 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.051 DEBUG:boto:Params: {} 2022-09-03T07:28:08.052 DEBUG:boto:Token: None 2022-09-03T07:28:08.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.057 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000510f292930733a97-0063130208-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.057 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.058 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.059 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.059 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:08.060 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:08.060 DEBUG:boto:Method: GET 2022-09-03T07:28:08.061 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:28:08.061 DEBUG:boto:Data: 2022-09-03T07:28:08.062 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.063 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.063 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.064 DEBUG:boto:Params: {} 2022-09-03T07:28:08.064 DEBUG:boto:Token: None 2022-09-03T07:28:08.065 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.066 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.070 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000958f261cfbe44055-0063130208-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.071 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.072 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:08.073 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:08.074 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.074 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:28:08.075 DEBUG:boto:Data: 2022-09-03T07:28:08.076 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.078 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.080 DEBUG:boto:Params: {} 2022-09-03T07:28:08.080 DEBUG:boto:Token: None 2022-09-03T07:28:08.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj116 2022-09-03T07:28:08.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xY/hJ/UjIYInB3UlYY156qMVcLU= 2022-09-03T07:28:08.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xY/hJ/UjIYInB3UlYY156qMVcLU=', 'Content-Length': '0'} 2022-09-03T07:28:08.087 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf32bfbd3ccc3696-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.090 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:08.091 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:08.092 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.093 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:28:08.095 DEBUG:boto:Data: 2022-09-03T07:28:08.096 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.097 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.099 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.099 DEBUG:boto:Params: {} 2022-09-03T07:28:08.101 DEBUG:boto:Token: None 2022-09-03T07:28:08.102 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj116 2022-09-03T07:28:08.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xY/hJ/UjIYInB3UlYY156qMVcLU= 2022-09-03T07:28:08.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xY/hJ/UjIYInB3UlYY156qMVcLU=', 'Content-Length': '0'} 2022-09-03T07:28:08.107 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f188dd4edae564c-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.107 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:28:08.108 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:08.109 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:08.109 DEBUG:boto:Method: GET 2022-09-03T07:28:08.110 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:28:08.110 DEBUG:boto:Data: 2022-09-03T07:28:08.111 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.112 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.112 DEBUG:boto:Params: {} 2022-09-03T07:28:08.113 DEBUG:boto:Token: None 2022-09-03T07:28:08.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj116 2022-09-03T07:28:08.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yJt04tO7gJI0INfWAIolFBO9DSE= 2022-09-03T07:28:08.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yJt04tO7gJI0INfWAIolFBO9DSE=', 'Content-Length': '0'} 2022-09-03T07:28:08.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042c7db8d93202a7f-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.118 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:08.119 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:08.119 DEBUG:boto:Method: GET 2022-09-03T07:28:08.120 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:28:08.120 DEBUG:boto:Data: 2022-09-03T07:28:08.121 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.123 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.123 DEBUG:boto:Params: {} 2022-09-03T07:28:08.124 DEBUG:boto:Token: None 2022-09-03T07:28:08.124 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj116 2022-09-03T07:28:08.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yJt04tO7gJI0INfWAIolFBO9DSE= 2022-09-03T07:28:08.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yJt04tO7gJI0INfWAIolFBO9DSE=', 'Content-Length': '0'} 2022-09-03T07:28:08.128 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a65d1f3dbdea381c-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.170 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.171 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.172 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:08.172 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:08.173 DEBUG:boto:Method: GET 2022-09-03T07:28:08.173 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:28:08.174 DEBUG:boto:Data: 2022-09-03T07:28:08.174 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.175 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.176 DEBUG:boto:Params: {} 2022-09-03T07:28:08.177 DEBUG:boto:Token: None 2022-09-03T07:28:08.177 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.183 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023b1a4e2b8ba38e1-0063130208-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.184 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.187 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.188 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.189 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:08.190 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:08.190 DEBUG:boto:Method: GET 2022-09-03T07:28:08.191 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:28:08.192 DEBUG:boto:Data: 2022-09-03T07:28:08.194 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.194 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.195 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.196 DEBUG:boto:Params: {} 2022-09-03T07:28:08.197 DEBUG:boto:Token: None 2022-09-03T07:28:08.198 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.204 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000029641afd30bff1e2-0063130208-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.205 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.206 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:28:08.207 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.207 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.208 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.209 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.209 DEBUG:boto:Method: GET 2022-09-03T07:28:08.210 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.211 DEBUG:boto:Data: 2022-09-03T07:28:08.211 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.212 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.213 DEBUG:boto:Params: {} 2022-09-03T07:28:08.214 DEBUG:boto:Token: None 2022-09-03T07:28:08.215 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.216 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UQuZQWr9qpjNaug1QfC3mfRtiw4= 2022-09-03T07:28:08.216 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UQuZQWr9qpjNaug1QfC3mfRtiw4=', 'Content-Length': '0'} 2022-09-03T07:28:08.219 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b24ce02660ef42b5-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.220 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.220 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.221 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.222 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.222 DEBUG:boto:Method: GET 2022-09-03T07:28:08.223 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.224 DEBUG:boto:Data: 2022-09-03T07:28:08.224 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.225 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.226 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.226 DEBUG:boto:Params: {} 2022-09-03T07:28:08.227 DEBUG:boto:Token: None 2022-09-03T07:28:08.227 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UQuZQWr9qpjNaug1QfC3mfRtiw4= 2022-09-03T07:28:08.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UQuZQWr9qpjNaug1QfC3mfRtiw4=', 'Content-Length': '0'} 2022-09-03T07:28:08.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075a34dfb921dfe1b-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.235 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.236 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.237 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.238 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.239 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.240 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.241 DEBUG:boto:Data: 2022-09-03T07:28:08.242 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.243 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.244 DEBUG:boto:Params: {} 2022-09-03T07:28:08.245 DEBUG:boto:Token: None 2022-09-03T07:28:08.245 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8kr5UIElyky7i9NK0o4d9jxHqEs= 2022-09-03T07:28:08.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8kr5UIElyky7i9NK0o4d9jxHqEs=', 'Content-Length': '0'} 2022-09-03T07:28:08.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000afc67be360c7fbc0-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.256 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.256 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.257 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.257 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.258 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.258 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.259 DEBUG:boto:Data: 2022-09-03T07:28:08.259 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.260 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.261 DEBUG:boto:Params: {} 2022-09-03T07:28:08.261 DEBUG:boto:Token: None 2022-09-03T07:28:08.262 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8kr5UIElyky7i9NK0o4d9jxHqEs= 2022-09-03T07:28:08.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8kr5UIElyky7i9NK0o4d9jxHqEs=', 'Content-Length': '0'} 2022-09-03T07:28:08.265 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab31b02269814e43-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.266 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:28:08.266 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.267 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.268 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.268 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.269 DEBUG:boto:Method: GET 2022-09-03T07:28:08.269 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.270 DEBUG:boto:Data: 2022-09-03T07:28:08.270 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.271 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.272 DEBUG:boto:Params: {} 2022-09-03T07:28:08.272 DEBUG:boto:Token: None 2022-09-03T07:28:08.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UQuZQWr9qpjNaug1QfC3mfRtiw4= 2022-09-03T07:28:08.274 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UQuZQWr9qpjNaug1QfC3mfRtiw4=', 'Content-Length': '0'} 2022-09-03T07:28:08.276 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d9520b6bcd93009f-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.318 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.319 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.320 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.321 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.322 DEBUG:boto:Method: GET 2022-09-03T07:28:08.323 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.325 DEBUG:boto:Data: 2022-09-03T07:28:08.326 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.327 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.328 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.329 DEBUG:boto:Params: {} 2022-09-03T07:28:08.330 DEBUG:boto:Token: None 2022-09-03T07:28:08.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:08.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UQuZQWr9qpjNaug1QfC3mfRtiw4= 2022-09-03T07:28:08.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UQuZQWr9qpjNaug1QfC3mfRtiw4=', 'Content-Length': '0'} 2022-09-03T07:28:08.346 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001b2bd30747a8343-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.348 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.349 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.350 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:08.354 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:08.355 DEBUG:boto:Method: GET 2022-09-03T07:28:08.356 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:28:08.356 DEBUG:boto:Data: 2022-09-03T07:28:08.357 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.359 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.360 DEBUG:boto:Params: {} 2022-09-03T07:28:08.361 DEBUG:boto:Token: None 2022-09-03T07:28:08.362 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.368 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a8776d39cf7428e-0063130208-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.370 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.371 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.372 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.374 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:08.375 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:08.376 DEBUG:boto:Method: GET 2022-09-03T07:28:08.377 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:28:08.378 DEBUG:boto:Data: 2022-09-03T07:28:08.379 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.380 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.381 DEBUG:boto:Params: {} 2022-09-03T07:28:08.381 DEBUG:boto:Token: None 2022-09-03T07:28:08.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.389 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007002d9ad56386fca-0063130208-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.390 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.391 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.392 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.393 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.394 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:28:08.395 DEBUG:boto:Data: 2022-09-03T07:28:08.396 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.397 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.398 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.399 DEBUG:boto:Params: {} 2022-09-03T07:28:08.400 DEBUG:boto:Token: None 2022-09-03T07:28:08.401 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124 2022-09-03T07:28:08.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1+FVa1SHQp7S/NnjZqMUFVjfr6E= 2022-09-03T07:28:08.403 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1+FVa1SHQp7S/NnjZqMUFVjfr6E=', 'Content-Length': '0'} 2022-09-03T07:28:08.406 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed6143146374a70d-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.407 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.408 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.409 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.410 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:28:08.411 DEBUG:boto:Data: 2022-09-03T07:28:08.412 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.413 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.414 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.415 DEBUG:boto:Params: {} 2022-09-03T07:28:08.416 DEBUG:boto:Token: None 2022-09-03T07:28:08.417 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124 2022-09-03T07:28:08.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1+FVa1SHQp7S/NnjZqMUFVjfr6E= 2022-09-03T07:28:08.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1+FVa1SHQp7S/NnjZqMUFVjfr6E=', 'Content-Length': '0'} 2022-09-03T07:28:08.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b83dadc851c49fa1-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.424 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:28:08.425 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.427 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.428 DEBUG:boto:Method: GET 2022-09-03T07:28:08.429 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:28:08.430 DEBUG:boto:Data: 2022-09-03T07:28:08.431 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.433 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.434 DEBUG:boto:Params: {} 2022-09-03T07:28:08.436 DEBUG:boto:Token: None 2022-09-03T07:28:08.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124 2022-09-03T07:28:08.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EFroSXIjtzBiaC5aFPhseEbdXWs= 2022-09-03T07:28:08.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EFroSXIjtzBiaC5aFPhseEbdXWs=', 'Content-Length': '0'} 2022-09-03T07:28:08.443 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec3aac73cf38f9fa-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.444 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:08.445 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:08.446 DEBUG:boto:Method: GET 2022-09-03T07:28:08.447 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:28:08.448 DEBUG:boto:Data: 2022-09-03T07:28:08.449 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.453 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.454 DEBUG:boto:Params: {} 2022-09-03T07:28:08.455 DEBUG:boto:Token: None 2022-09-03T07:28:08.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj124 2022-09-03T07:28:08.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EFroSXIjtzBiaC5aFPhseEbdXWs= 2022-09-03T07:28:08.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EFroSXIjtzBiaC5aFPhseEbdXWs=', 'Content-Length': '0'} 2022-09-03T07:28:08.462 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c2efb397024d2be-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.502 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.503 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.505 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:08.516 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:08.517 DEBUG:boto:Method: GET 2022-09-03T07:28:08.518 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:28:08.519 DEBUG:boto:Data: 2022-09-03T07:28:08.520 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.521 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.522 DEBUG:boto:Params: {} 2022-09-03T07:28:08.523 DEBUG:boto:Token: None 2022-09-03T07:28:08.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.533 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d193950399ed44f8-0063130208-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.534 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.536 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.537 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.538 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:08.538 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:08.539 DEBUG:boto:Method: GET 2022-09-03T07:28:08.540 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:28:08.543 DEBUG:boto:Data: 2022-09-03T07:28:08.544 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.546 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.546 DEBUG:boto:Params: {} 2022-09-03T07:28:08.547 DEBUG:boto:Token: None 2022-09-03T07:28:08.548 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.556 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000df7e392131bc1deb-0063130208-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.557 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.558 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:28:08.559 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.560 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.560 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.561 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.561 DEBUG:boto:Method: GET 2022-09-03T07:28:08.562 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.563 DEBUG:boto:Data: 2022-09-03T07:28:08.564 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.565 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.565 DEBUG:boto:Params: {} 2022-09-03T07:28:08.567 DEBUG:boto:Token: None 2022-09-03T07:28:08.568 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kGhG7c+sp+4DSIxcSbPjtwUVGmo= 2022-09-03T07:28:08.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kGhG7c+sp+4DSIxcSbPjtwUVGmo=', 'Content-Length': '0'} 2022-09-03T07:28:08.579 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fca2032aee77c0b3-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.581 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.582 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.584 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.585 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.586 DEBUG:boto:Method: GET 2022-09-03T07:28:08.588 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.589 DEBUG:boto:Data: 2022-09-03T07:28:08.590 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.592 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.592 DEBUG:boto:Params: {} 2022-09-03T07:28:08.593 DEBUG:boto:Token: None 2022-09-03T07:28:08.596 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kGhG7c+sp+4DSIxcSbPjtwUVGmo= 2022-09-03T07:28:08.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kGhG7c+sp+4DSIxcSbPjtwUVGmo=', 'Content-Length': '0'} 2022-09-03T07:28:08.604 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e86cec98ecaee6b-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.606 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.607 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.608 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.608 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.609 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.610 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.611 DEBUG:boto:Data: 2022-09-03T07:28:08.611 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.613 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.614 DEBUG:boto:Params: {} 2022-09-03T07:28:08.615 DEBUG:boto:Token: None 2022-09-03T07:28:08.616 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mmTYC0iswfxnXGpnaHV4nWAZlPk= 2022-09-03T07:28:08.618 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mmTYC0iswfxnXGpnaHV4nWAZlPk=', 'Content-Length': '0'} 2022-09-03T07:28:08.620 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060a9433e8e5816f5-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.621 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.622 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.622 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.623 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.623 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.624 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.624 DEBUG:boto:Data: 2022-09-03T07:28:08.625 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.626 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.626 DEBUG:boto:Params: {} 2022-09-03T07:28:08.627 DEBUG:boto:Token: None 2022-09-03T07:28:08.628 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mmTYC0iswfxnXGpnaHV4nWAZlPk= 2022-09-03T07:28:08.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mmTYC0iswfxnXGpnaHV4nWAZlPk=', 'Content-Length': '0'} 2022-09-03T07:28:08.635 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b149b50c76f67e27-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.643 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:28:08.644 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.645 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.646 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.647 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.648 DEBUG:boto:Method: GET 2022-09-03T07:28:08.649 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.650 DEBUG:boto:Data: 2022-09-03T07:28:08.651 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.652 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.653 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.654 DEBUG:boto:Params: {} 2022-09-03T07:28:08.655 DEBUG:boto:Token: None 2022-09-03T07:28:08.655 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kGhG7c+sp+4DSIxcSbPjtwUVGmo= 2022-09-03T07:28:08.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kGhG7c+sp+4DSIxcSbPjtwUVGmo=', 'Content-Length': '0'} 2022-09-03T07:28:08.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000094181277206a9ed8-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.662 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.663 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.665 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.666 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.667 DEBUG:boto:Method: GET 2022-09-03T07:28:08.668 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.669 DEBUG:boto:Data: 2022-09-03T07:28:08.670 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.671 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.672 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.672 DEBUG:boto:Params: {} 2022-09-03T07:28:08.674 DEBUG:boto:Token: None 2022-09-03T07:28:08.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:08.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kGhG7c+sp+4DSIxcSbPjtwUVGmo= 2022-09-03T07:28:08.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kGhG7c+sp+4DSIxcSbPjtwUVGmo=', 'Content-Length': '0'} 2022-09-03T07:28:08.679 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b65be95d3eff15c5-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.680 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.680 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.681 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:08.682 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:08.682 DEBUG:boto:Method: GET 2022-09-03T07:28:08.683 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:28:08.683 DEBUG:boto:Data: 2022-09-03T07:28:08.684 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.684 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.685 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.685 DEBUG:boto:Params: {} 2022-09-03T07:28:08.686 DEBUG:boto:Token: None 2022-09-03T07:28:08.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.690 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bde8a3a9f1021d20-0063130208-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.691 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.692 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.693 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.693 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:08.694 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:08.694 DEBUG:boto:Method: GET 2022-09-03T07:28:08.695 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:28:08.695 DEBUG:boto:Data: 2022-09-03T07:28:08.696 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.696 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.697 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.698 DEBUG:boto:Params: {} 2022-09-03T07:28:08.698 DEBUG:boto:Token: None 2022-09-03T07:28:08.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.705 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000690eaadddfaeb22d-0063130208-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.707 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.707 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.708 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.708 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.709 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:28:08.709 DEBUG:boto:Data: 2022-09-03T07:28:08.710 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.710 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.711 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.711 DEBUG:boto:Params: {} 2022-09-03T07:28:08.713 DEBUG:boto:Token: None 2022-09-03T07:28:08.713 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145 2022-09-03T07:28:08.714 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l4jUSE+WpJSwmUFQwRFpZ8V2jdI= 2022-09-03T07:28:08.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l4jUSE+WpJSwmUFQwRFpZ8V2jdI=', 'Content-Length': '0'} 2022-09-03T07:28:08.716 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c02c27b4a5182d48-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.717 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.718 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.718 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.719 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:28:08.720 DEBUG:boto:Data: 2022-09-03T07:28:08.720 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.721 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.721 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.722 DEBUG:boto:Params: {} 2022-09-03T07:28:08.722 DEBUG:boto:Token: None 2022-09-03T07:28:08.723 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145 2022-09-03T07:28:08.724 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l4jUSE+WpJSwmUFQwRFpZ8V2jdI= 2022-09-03T07:28:08.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l4jUSE+WpJSwmUFQwRFpZ8V2jdI=', 'Content-Length': '0'} 2022-09-03T07:28:08.728 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a27d35dd8ac49e63-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.730 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:28:08.730 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.731 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.732 DEBUG:boto:Method: GET 2022-09-03T07:28:08.733 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:28:08.734 DEBUG:boto:Data: 2022-09-03T07:28:08.734 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.735 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.735 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.736 DEBUG:boto:Params: {} 2022-09-03T07:28:08.737 DEBUG:boto:Token: None 2022-09-03T07:28:08.737 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145 2022-09-03T07:28:08.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PcmApJ7L7oRnimKhCW5Yhuss9Yw= 2022-09-03T07:28:08.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PcmApJ7L7oRnimKhCW5Yhuss9Yw=', 'Content-Length': '0'} 2022-09-03T07:28:08.742 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dae18c51b003f6b5-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.743 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:08.744 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:08.744 DEBUG:boto:Method: GET 2022-09-03T07:28:08.745 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:28:08.745 DEBUG:boto:Data: 2022-09-03T07:28:08.746 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.747 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.748 DEBUG:boto:Params: {} 2022-09-03T07:28:08.748 DEBUG:boto:Token: None 2022-09-03T07:28:08.749 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj145 2022-09-03T07:28:08.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PcmApJ7L7oRnimKhCW5Yhuss9Yw= 2022-09-03T07:28:08.750 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PcmApJ7L7oRnimKhCW5Yhuss9Yw=', 'Content-Length': '0'} 2022-09-03T07:28:08.752 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000061d8676305773839-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.801 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.802 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.802 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:08.803 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:08.804 DEBUG:boto:Method: GET 2022-09-03T07:28:08.805 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:28:08.806 DEBUG:boto:Data: 2022-09-03T07:28:08.807 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.807 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.808 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.809 DEBUG:boto:Params: {} 2022-09-03T07:28:08.809 DEBUG:boto:Token: None 2022-09-03T07:28:08.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.815 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c4d4e5d1d7392be-0063130208-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.815 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.816 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.817 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.818 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:08.818 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:08.819 DEBUG:boto:Method: GET 2022-09-03T07:28:08.820 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:28:08.821 DEBUG:boto:Data: 2022-09-03T07:28:08.822 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.823 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.824 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.825 DEBUG:boto:Params: {} 2022-09-03T07:28:08.826 DEBUG:boto:Token: None 2022-09-03T07:28:08.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.840 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002f5ab3dc1a424576-0063130208-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.841 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.842 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:28:08.842 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:08.843 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:08.844 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.844 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.845 DEBUG:boto:Method: GET 2022-09-03T07:28:08.846 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.847 DEBUG:boto:Data: 2022-09-03T07:28:08.847 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.848 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.849 DEBUG:boto:Params: {} 2022-09-03T07:28:08.850 DEBUG:boto:Token: None 2022-09-03T07:28:08.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zg6RtbU5yDHNuFXsdUFsk0VN1Zo= 2022-09-03T07:28:08.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zg6RtbU5yDHNuFXsdUFsk0VN1Zo=', 'Content-Length': '0'} 2022-09-03T07:28:08.855 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d1a15f1e2c5df83-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.902 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:08.903 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:08.904 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.904 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.905 DEBUG:boto:Method: GET 2022-09-03T07:28:08.906 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.906 DEBUG:boto:Data: 2022-09-03T07:28:08.907 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.908 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.908 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.909 DEBUG:boto:Params: {} 2022-09-03T07:28:08.911 DEBUG:boto:Token: None 2022-09-03T07:28:08.912 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zg6RtbU5yDHNuFXsdUFsk0VN1Zo= 2022-09-03T07:28:08.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zg6RtbU5yDHNuFXsdUFsk0VN1Zo=', 'Content-Length': '0'} 2022-09-03T07:28:08.917 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060140e47891c61f6-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.918 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:08.919 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:08.920 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.921 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.923 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.924 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.924 DEBUG:boto:Data: 2022-09-03T07:28:08.925 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.926 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.926 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.927 DEBUG:boto:Params: {} 2022-09-03T07:28:08.928 DEBUG:boto:Token: None 2022-09-03T07:28:08.929 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KgTSYH16qg/n1enx2Zs8yFpcRRw= 2022-09-03T07:28:08.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KgTSYH16qg/n1enx2Zs8yFpcRRw=', 'Content-Length': '0'} 2022-09-03T07:28:08.934 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d8a75614416e093-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.935 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:08.935 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:08.936 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.936 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.937 DEBUG:boto:Method: HEAD 2022-09-03T07:28:08.937 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.938 DEBUG:boto:Data: 2022-09-03T07:28:08.938 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.939 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.939 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.940 DEBUG:boto:Params: {} 2022-09-03T07:28:08.940 DEBUG:boto:Token: None 2022-09-03T07:28:08.941 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KgTSYH16qg/n1enx2Zs8yFpcRRw= 2022-09-03T07:28:08.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KgTSYH16qg/n1enx2Zs8yFpcRRw=', 'Content-Length': '0'} 2022-09-03T07:28:08.944 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9b7186ec07056b6-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.945 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:28:08.946 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:08.946 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:08.947 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.947 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.948 DEBUG:boto:Method: GET 2022-09-03T07:28:08.948 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.949 DEBUG:boto:Data: 2022-09-03T07:28:08.949 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.950 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.950 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.951 DEBUG:boto:Params: {} 2022-09-03T07:28:08.952 DEBUG:boto:Token: None 2022-09-03T07:28:08.952 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zg6RtbU5yDHNuFXsdUFsk0VN1Zo= 2022-09-03T07:28:08.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zg6RtbU5yDHNuFXsdUFsk0VN1Zo=', 'Content-Length': '0'} 2022-09-03T07:28:08.956 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000453a19b0bdf20c1f-0063130208-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.957 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:08.958 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:08.958 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.959 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.959 DEBUG:boto:Method: GET 2022-09-03T07:28:08.960 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.960 DEBUG:boto:Data: 2022-09-03T07:28:08.961 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.962 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.963 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.963 DEBUG:boto:Params: {} 2022-09-03T07:28:08.964 DEBUG:boto:Token: None 2022-09-03T07:28:08.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:08.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zg6RtbU5yDHNuFXsdUFsk0VN1Zo= 2022-09-03T07:28:08.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zg6RtbU5yDHNuFXsdUFsk0VN1Zo=', 'Content-Length': '0'} 2022-09-03T07:28:08.970 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c439b6ea416d4db-0063130208-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.972 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.972 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.973 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:08.974 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:08.975 DEBUG:boto:Method: GET 2022-09-03T07:28:08.976 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:28:08.977 DEBUG:boto:Data: 2022-09-03T07:28:08.977 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:08.978 DEBUG:boto:Port: 8001 2022-09-03T07:28:08.979 DEBUG:boto:Params: {} 2022-09-03T07:28:08.980 DEBUG:boto:Token: None 2022-09-03T07:28:08.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:08.985 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2b798291bf1bb5b-0063130208-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:08.986 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:08.987 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:08.987 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:08.988 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:08.989 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:08.989 DEBUG:boto:Method: GET 2022-09-03T07:28:08.990 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:28:08.991 DEBUG:boto:Data: 2022-09-03T07:28:08.991 DEBUG:boto:Headers: {} 2022-09-03T07:28:08.992 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:08.993 DEBUG:boto:Port: 8000 2022-09-03T07:28:08.993 DEBUG:boto:Params: {} 2022-09-03T07:28:08.994 DEBUG:boto:Token: None 2022-09-03T07:28:08.994 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:08 GMT /avysmw-75/ 2022-09-03T07:28:08.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU= 2022-09-03T07:28:08.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8/qnns7GX/f7btUB+uqC7arAtPU=', 'Content-Length': '0'} 2022-09-03T07:28:09.000 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003119f9edfc67be75-0063130208-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.001 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.003 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:09.004 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:09.005 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.006 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:28:09.006 DEBUG:boto:Data: 2022-09-03T07:28:09.007 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.009 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.009 DEBUG:boto:Params: {} 2022-09-03T07:28:09.010 DEBUG:boto:Token: None 2022-09-03T07:28:09.011 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj155 2022-09-03T07:28:09.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DdDMBbU42bSPBYPoqe0tTAAZZ6g= 2022-09-03T07:28:09.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DdDMBbU42bSPBYPoqe0tTAAZZ6g=', 'Content-Length': '0'} 2022-09-03T07:28:09.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c790b65abd2dc57-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.021 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:09.022 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:09.023 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.023 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:28:09.024 DEBUG:boto:Data: 2022-09-03T07:28:09.025 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.027 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.028 DEBUG:boto:Params: {} 2022-09-03T07:28:09.029 DEBUG:boto:Token: None 2022-09-03T07:28:09.030 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj155 2022-09-03T07:28:09.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DdDMBbU42bSPBYPoqe0tTAAZZ6g= 2022-09-03T07:28:09.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DdDMBbU42bSPBYPoqe0tTAAZZ6g=', 'Content-Length': '0'} 2022-09-03T07:28:09.035 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a786e2f7f3253cb7-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.036 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:28:09.037 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:09.038 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:09.039 DEBUG:boto:Method: GET 2022-09-03T07:28:09.039 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:28:09.040 DEBUG:boto:Data: 2022-09-03T07:28:09.040 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.041 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.042 DEBUG:boto:Params: {} 2022-09-03T07:28:09.043 DEBUG:boto:Token: None 2022-09-03T07:28:09.043 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj155 2022-09-03T07:28:09.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tj1Ao1v/BamQAFt+9AKSFK4qNUc= 2022-09-03T07:28:09.046 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tj1Ao1v/BamQAFt+9AKSFK4qNUc=', 'Content-Length': '0'} 2022-09-03T07:28:09.049 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037b7b8641e77355a-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.050 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:09.051 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:09.052 DEBUG:boto:Method: GET 2022-09-03T07:28:09.052 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:28:09.053 DEBUG:boto:Data: 2022-09-03T07:28:09.054 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.055 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.055 DEBUG:boto:Params: {} 2022-09-03T07:28:09.056 DEBUG:boto:Token: None 2022-09-03T07:28:09.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj155 2022-09-03T07:28:09.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tj1Ao1v/BamQAFt+9AKSFK4qNUc= 2022-09-03T07:28:09.059 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tj1Ao1v/BamQAFt+9AKSFK4qNUc=', 'Content-Length': '0'} 2022-09-03T07:28:09.079 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000bcd23666269fce2-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.126 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.127 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.129 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:09.130 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:09.131 DEBUG:boto:Method: GET 2022-09-03T07:28:09.133 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:28:09.133 DEBUG:boto:Data: 2022-09-03T07:28:09.134 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.134 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.135 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.135 DEBUG:boto:Params: {} 2022-09-03T07:28:09.136 DEBUG:boto:Token: None 2022-09-03T07:28:09.137 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.141 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fa435dfe509639e8-0063130209-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.141 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.142 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.143 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.143 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:09.144 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:09.144 DEBUG:boto:Method: GET 2022-09-03T07:28:09.145 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:28:09.145 DEBUG:boto:Data: 2022-09-03T07:28:09.146 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.146 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.147 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.148 DEBUG:boto:Params: {} 2022-09-03T07:28:09.148 DEBUG:boto:Token: None 2022-09-03T07:28:09.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.154 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f0ca369f946b31ba-0063130209-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.155 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.157 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:28:09.157 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.158 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.159 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.159 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.160 DEBUG:boto:Method: GET 2022-09-03T07:28:09.161 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.161 DEBUG:boto:Data: 2022-09-03T07:28:09.162 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.163 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.163 DEBUG:boto:Params: {} 2022-09-03T07:28:09.164 DEBUG:boto:Token: None 2022-09-03T07:28:09.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:huVpv3wlqOUObKw44NeBVqY2Szc= 2022-09-03T07:28:09.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:huVpv3wlqOUObKw44NeBVqY2Szc=', 'Content-Length': '0'} 2022-09-03T07:28:09.168 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000040f84540d3354f61-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.169 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.170 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.170 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.171 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.171 DEBUG:boto:Method: GET 2022-09-03T07:28:09.172 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.172 DEBUG:boto:Data: 2022-09-03T07:28:09.173 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.173 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.174 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.174 DEBUG:boto:Params: {} 2022-09-03T07:28:09.175 DEBUG:boto:Token: None 2022-09-03T07:28:09.176 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.176 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:huVpv3wlqOUObKw44NeBVqY2Szc= 2022-09-03T07:28:09.177 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:huVpv3wlqOUObKw44NeBVqY2Szc=', 'Content-Length': '0'} 2022-09-03T07:28:09.179 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000598fbe03f1ca5b9e-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.180 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.181 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.181 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.182 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.182 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.183 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.183 DEBUG:boto:Data: 2022-09-03T07:28:09.184 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.185 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.185 DEBUG:boto:Params: {} 2022-09-03T07:28:09.195 DEBUG:boto:Token: None 2022-09-03T07:28:09.197 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cw131GCESI7OLM6RniO3lYwtSKI= 2022-09-03T07:28:09.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cw131GCESI7OLM6RniO3lYwtSKI=', 'Content-Length': '0'} 2022-09-03T07:28:09.201 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001912d062e01340fc-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.202 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.202 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.203 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.203 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.204 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.204 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.205 DEBUG:boto:Data: 2022-09-03T07:28:09.205 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.207 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.208 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.208 DEBUG:boto:Params: {} 2022-09-03T07:28:09.209 DEBUG:boto:Token: None 2022-09-03T07:28:09.210 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cw131GCESI7OLM6RniO3lYwtSKI= 2022-09-03T07:28:09.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cw131GCESI7OLM6RniO3lYwtSKI=', 'Content-Length': '0'} 2022-09-03T07:28:09.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e8d355af7215b11-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:28:09.217 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.217 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.218 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.218 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.219 DEBUG:boto:Method: GET 2022-09-03T07:28:09.219 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.220 DEBUG:boto:Data: 2022-09-03T07:28:09.220 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.221 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.222 DEBUG:boto:Params: {} 2022-09-03T07:28:09.223 DEBUG:boto:Token: None 2022-09-03T07:28:09.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:huVpv3wlqOUObKw44NeBVqY2Szc= 2022-09-03T07:28:09.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:huVpv3wlqOUObKw44NeBVqY2Szc=', 'Content-Length': '0'} 2022-09-03T07:28:09.226 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dfed4631a58556a2-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.227 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.228 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.228 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.229 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.229 DEBUG:boto:Method: GET 2022-09-03T07:28:09.230 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.230 DEBUG:boto:Data: 2022-09-03T07:28:09.231 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.232 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.232 DEBUG:boto:Params: {} 2022-09-03T07:28:09.233 DEBUG:boto:Token: None 2022-09-03T07:28:09.233 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:09.234 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:huVpv3wlqOUObKw44NeBVqY2Szc= 2022-09-03T07:28:09.235 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:huVpv3wlqOUObKw44NeBVqY2Szc=', 'Content-Length': '0'} 2022-09-03T07:28:09.237 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbe178042f9ad34d-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.282 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.284 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.288 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:09.289 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:09.290 DEBUG:boto:Method: GET 2022-09-03T07:28:09.291 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:28:09.292 DEBUG:boto:Data: 2022-09-03T07:28:09.293 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.294 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.294 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.295 DEBUG:boto:Params: {} 2022-09-03T07:28:09.296 DEBUG:boto:Token: None 2022-09-03T07:28:09.296 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.302 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009e2b7761f6aa161c-0063130209-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.302 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.303 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.304 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.304 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:09.305 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:09.306 DEBUG:boto:Method: GET 2022-09-03T07:28:09.307 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:28:09.308 DEBUG:boto:Data: 2022-09-03T07:28:09.308 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.309 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.310 DEBUG:boto:Params: {} 2022-09-03T07:28:09.311 DEBUG:boto:Token: None 2022-09-03T07:28:09.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.317 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000137f93bfeb8023e-0063130209-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.318 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.319 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.319 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.320 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.320 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:28:09.321 DEBUG:boto:Data: 2022-09-03T07:28:09.323 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.323 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.324 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.324 DEBUG:boto:Params: {} 2022-09-03T07:28:09.325 DEBUG:boto:Token: None 2022-09-03T07:28:09.326 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17 2022-09-03T07:28:09.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T764hVoZp6ZO0JJAq7ah1uclWfM= 2022-09-03T07:28:09.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T764hVoZp6ZO0JJAq7ah1uclWfM=', 'Content-Length': '0'} 2022-09-03T07:28:09.329 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066ba65ae93eb618e-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.330 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.331 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.331 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.332 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:28:09.332 DEBUG:boto:Data: 2022-09-03T07:28:09.333 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.335 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.335 DEBUG:boto:Params: {} 2022-09-03T07:28:09.336 DEBUG:boto:Token: None 2022-09-03T07:28:09.337 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17 2022-09-03T07:28:09.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T764hVoZp6ZO0JJAq7ah1uclWfM= 2022-09-03T07:28:09.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T764hVoZp6ZO0JJAq7ah1uclWfM=', 'Content-Length': '0'} 2022-09-03T07:28:09.341 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c01c2755f04774c1-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.342 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:28:09.344 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.345 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.346 DEBUG:boto:Method: GET 2022-09-03T07:28:09.347 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:28:09.348 DEBUG:boto:Data: 2022-09-03T07:28:09.348 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.349 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.350 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.351 DEBUG:boto:Params: {} 2022-09-03T07:28:09.352 DEBUG:boto:Token: None 2022-09-03T07:28:09.352 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17 2022-09-03T07:28:09.353 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1sWInoSmjkLzRWfmhoZYXHQJ0q8= 2022-09-03T07:28:09.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1sWInoSmjkLzRWfmhoZYXHQJ0q8=', 'Content-Length': '0'} 2022-09-03T07:28:09.357 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a9f340ca9a80fb7-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.359 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:09.360 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:09.361 DEBUG:boto:Method: GET 2022-09-03T07:28:09.362 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:28:09.362 DEBUG:boto:Data: 2022-09-03T07:28:09.363 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.364 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.364 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.365 DEBUG:boto:Params: {} 2022-09-03T07:28:09.365 DEBUG:boto:Token: None 2022-09-03T07:28:09.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj17 2022-09-03T07:28:09.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1sWInoSmjkLzRWfmhoZYXHQJ0q8= 2022-09-03T07:28:09.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1sWInoSmjkLzRWfmhoZYXHQJ0q8=', 'Content-Length': '0'} 2022-09-03T07:28:09.370 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbe1bdaad35328ee-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.371 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.371 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.372 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:09.373 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:09.373 DEBUG:boto:Method: GET 2022-09-03T07:28:09.374 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:28:09.374 DEBUG:boto:Data: 2022-09-03T07:28:09.375 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.376 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.376 DEBUG:boto:Params: {} 2022-09-03T07:28:09.377 DEBUG:boto:Token: None 2022-09-03T07:28:09.378 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.383 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a86ac0474f6606f-0063130209-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.384 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.385 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.385 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.386 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:09.386 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:09.387 DEBUG:boto:Method: GET 2022-09-03T07:28:09.388 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:28:09.388 DEBUG:boto:Data: 2022-09-03T07:28:09.389 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.389 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.390 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.391 DEBUG:boto:Params: {} 2022-09-03T07:28:09.391 DEBUG:boto:Token: None 2022-09-03T07:28:09.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.398 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e6baec0dd8d6e334-0063130209-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.399 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.400 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:28:09.401 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.402 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.403 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.404 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.405 DEBUG:boto:Method: GET 2022-09-03T07:28:09.405 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.406 DEBUG:boto:Data: 2022-09-03T07:28:09.407 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.408 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.409 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.410 DEBUG:boto:Params: {} 2022-09-03T07:28:09.412 DEBUG:boto:Token: None 2022-09-03T07:28:09.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0TWjXR8GObfANqccVZKb53pUhE= 2022-09-03T07:28:09.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0TWjXR8GObfANqccVZKb53pUhE=', 'Content-Length': '0'} 2022-09-03T07:28:09.419 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc85cd56f6c3aeb6-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.420 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.421 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.423 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.424 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.425 DEBUG:boto:Method: GET 2022-09-03T07:28:09.426 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.427 DEBUG:boto:Data: 2022-09-03T07:28:09.428 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.431 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.432 DEBUG:boto:Params: {} 2022-09-03T07:28:09.433 DEBUG:boto:Token: None 2022-09-03T07:28:09.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0TWjXR8GObfANqccVZKb53pUhE= 2022-09-03T07:28:09.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0TWjXR8GObfANqccVZKb53pUhE=', 'Content-Length': '0'} 2022-09-03T07:28:09.437 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b10dfaadd5544319-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.438 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.439 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.439 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.440 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.441 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.442 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.442 DEBUG:boto:Data: 2022-09-03T07:28:09.443 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.444 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.445 DEBUG:boto:Params: {} 2022-09-03T07:28:09.447 DEBUG:boto:Token: None 2022-09-03T07:28:09.448 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yuja/jvnOTu+5jxxo+cEnPD0iWs= 2022-09-03T07:28:09.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yuja/jvnOTu+5jxxo+cEnPD0iWs=', 'Content-Length': '0'} 2022-09-03T07:28:09.453 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cda6ff0be6cc40e7-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.454 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.455 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.456 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.457 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.458 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.459 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.460 DEBUG:boto:Data: 2022-09-03T07:28:09.461 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.462 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.463 DEBUG:boto:Params: {} 2022-09-03T07:28:09.464 DEBUG:boto:Token: None 2022-09-03T07:28:09.465 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yuja/jvnOTu+5jxxo+cEnPD0iWs= 2022-09-03T07:28:09.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yuja/jvnOTu+5jxxo+cEnPD0iWs=', 'Content-Length': '0'} 2022-09-03T07:28:09.470 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074b12388e065208a-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.471 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:28:09.471 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.472 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.473 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.474 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.475 DEBUG:boto:Method: GET 2022-09-03T07:28:09.483 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.484 DEBUG:boto:Data: 2022-09-03T07:28:09.485 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.486 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.487 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.487 DEBUG:boto:Params: {} 2022-09-03T07:28:09.488 DEBUG:boto:Token: None 2022-09-03T07:28:09.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0TWjXR8GObfANqccVZKb53pUhE= 2022-09-03T07:28:09.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0TWjXR8GObfANqccVZKb53pUhE=', 'Content-Length': '0'} 2022-09-03T07:28:09.492 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092fdc1fbd7a06929-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.534 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.535 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.535 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.536 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.536 DEBUG:boto:Method: GET 2022-09-03T07:28:09.538 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.538 DEBUG:boto:Data: 2022-09-03T07:28:09.539 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.539 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.540 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.541 DEBUG:boto:Params: {} 2022-09-03T07:28:09.541 DEBUG:boto:Token: None 2022-09-03T07:28:09.542 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:09.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B0TWjXR8GObfANqccVZKb53pUhE= 2022-09-03T07:28:09.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B0TWjXR8GObfANqccVZKb53pUhE=', 'Content-Length': '0'} 2022-09-03T07:28:09.545 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035574d5617d016de-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.546 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.548 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.548 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:09.549 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:09.549 DEBUG:boto:Method: GET 2022-09-03T07:28:09.550 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:09.551 DEBUG:boto:Data: 2022-09-03T07:28:09.551 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.552 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.552 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.553 DEBUG:boto:Params: {} 2022-09-03T07:28:09.554 DEBUG:boto:Token: None 2022-09-03T07:28:09.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.559 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087a02dd3abc2d04b-0063130209-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.560 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.561 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.562 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.562 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:09.563 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:09.564 DEBUG:boto:Method: GET 2022-09-03T07:28:09.564 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:09.565 DEBUG:boto:Data: 2022-09-03T07:28:09.566 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.566 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.567 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.567 DEBUG:boto:Params: {} 2022-09-03T07:28:09.568 DEBUG:boto:Token: None 2022-09-03T07:28:09.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.575 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bed695e778bf9c5b-0063130209-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.576 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.577 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.578 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.578 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.579 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:09.579 DEBUG:boto:Data: 2022-09-03T07:28:09.580 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.581 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.581 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.582 DEBUG:boto:Params: {} 2022-09-03T07:28:09.583 DEBUG:boto:Token: None 2022-09-03T07:28:09.583 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19 2022-09-03T07:28:09.584 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ULVre3Jd3mfzDum6mNm6+uVGsJI= 2022-09-03T07:28:09.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ULVre3Jd3mfzDum6mNm6+uVGsJI=', 'Content-Length': '0'} 2022-09-03T07:28:09.587 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e14b861545753a4c-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.589 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.589 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.590 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.591 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:09.592 DEBUG:boto:Data: 2022-09-03T07:28:09.593 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.594 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.595 DEBUG:boto:Params: {} 2022-09-03T07:28:09.595 DEBUG:boto:Token: None 2022-09-03T07:28:09.596 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19 2022-09-03T07:28:09.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ULVre3Jd3mfzDum6mNm6+uVGsJI= 2022-09-03T07:28:09.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ULVre3Jd3mfzDum6mNm6+uVGsJI=', 'Content-Length': '0'} 2022-09-03T07:28:09.599 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b44b6724b3a14b0-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.600 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:28:09.600 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.601 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.602 DEBUG:boto:Method: GET 2022-09-03T07:28:09.603 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:09.603 DEBUG:boto:Data: 2022-09-03T07:28:09.604 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.605 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.605 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.606 DEBUG:boto:Params: {} 2022-09-03T07:28:09.607 DEBUG:boto:Token: None 2022-09-03T07:28:09.607 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19 2022-09-03T07:28:09.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j0J74qcz4YzYamDkkBnHpdzcGuE= 2022-09-03T07:28:09.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j0J74qcz4YzYamDkkBnHpdzcGuE=', 'Content-Length': '0'} 2022-09-03T07:28:09.611 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a7ef8c3c200a1e5-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.612 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:09.612 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:09.613 DEBUG:boto:Method: GET 2022-09-03T07:28:09.614 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:09.615 DEBUG:boto:Data: 2022-09-03T07:28:09.615 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.617 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.617 DEBUG:boto:Params: {} 2022-09-03T07:28:09.618 DEBUG:boto:Token: None 2022-09-03T07:28:09.619 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj19 2022-09-03T07:28:09.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j0J74qcz4YzYamDkkBnHpdzcGuE= 2022-09-03T07:28:09.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j0J74qcz4YzYamDkkBnHpdzcGuE=', 'Content-Length': '0'} 2022-09-03T07:28:09.622 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000225bebbb7833bd3d-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.668 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.670 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.671 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:09.671 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:09.672 DEBUG:boto:Method: GET 2022-09-03T07:28:09.673 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:09.674 DEBUG:boto:Data: 2022-09-03T07:28:09.675 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.676 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.677 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.678 DEBUG:boto:Params: {} 2022-09-03T07:28:09.679 DEBUG:boto:Token: None 2022-09-03T07:28:09.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.684 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dcba0bcf01d5c9bb-0063130209-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.686 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.687 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.687 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.688 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:09.689 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:09.690 DEBUG:boto:Method: GET 2022-09-03T07:28:09.690 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:09.691 DEBUG:boto:Data: 2022-09-03T07:28:09.692 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.693 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.694 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.695 DEBUG:boto:Params: {} 2022-09-03T07:28:09.696 DEBUG:boto:Token: None 2022-09-03T07:28:09.698 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.703 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006fd93a717db4b447-0063130209-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.704 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.705 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:28:09.706 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.706 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.707 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.707 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.708 DEBUG:boto:Method: GET 2022-09-03T07:28:09.709 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.710 DEBUG:boto:Data: 2022-09-03T07:28:09.710 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.711 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.712 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.713 DEBUG:boto:Params: {} 2022-09-03T07:28:09.714 DEBUG:boto:Token: None 2022-09-03T07:28:09.715 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JkbSY2U5G7w8IFPTbsCe5ePvhu4= 2022-09-03T07:28:09.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JkbSY2U5G7w8IFPTbsCe5ePvhu4=', 'Content-Length': '0'} 2022-09-03T07:28:09.720 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055075b7a57f17eb9-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.722 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.724 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.724 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.725 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.726 DEBUG:boto:Method: GET 2022-09-03T07:28:09.727 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.728 DEBUG:boto:Data: 2022-09-03T07:28:09.728 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.729 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.730 DEBUG:boto:Params: {} 2022-09-03T07:28:09.732 DEBUG:boto:Token: None 2022-09-03T07:28:09.733 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JkbSY2U5G7w8IFPTbsCe5ePvhu4= 2022-09-03T07:28:09.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JkbSY2U5G7w8IFPTbsCe5ePvhu4=', 'Content-Length': '0'} 2022-09-03T07:28:09.740 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9d9a123f661c269-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.742 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.742 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.743 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.743 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.744 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.745 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.746 DEBUG:boto:Data: 2022-09-03T07:28:09.747 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.749 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.749 DEBUG:boto:Params: {} 2022-09-03T07:28:09.751 DEBUG:boto:Token: None 2022-09-03T07:28:09.752 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kj2TeQ6SMIXMNt0JaqlduRqkS7o= 2022-09-03T07:28:09.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kj2TeQ6SMIXMNt0JaqlduRqkS7o=', 'Content-Length': '0'} 2022-09-03T07:28:09.757 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d233744f9767b594-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.758 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.759 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.759 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.760 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.760 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.761 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.761 DEBUG:boto:Data: 2022-09-03T07:28:09.762 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.763 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.764 DEBUG:boto:Params: {} 2022-09-03T07:28:09.764 DEBUG:boto:Token: None 2022-09-03T07:28:09.765 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kj2TeQ6SMIXMNt0JaqlduRqkS7o= 2022-09-03T07:28:09.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kj2TeQ6SMIXMNt0JaqlduRqkS7o=', 'Content-Length': '0'} 2022-09-03T07:28:09.769 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000030065b09a26ce1cd-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.769 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:28:09.770 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.771 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.771 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.772 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.772 DEBUG:boto:Method: GET 2022-09-03T07:28:09.773 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.774 DEBUG:boto:Data: 2022-09-03T07:28:09.774 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.775 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.776 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.776 DEBUG:boto:Params: {} 2022-09-03T07:28:09.777 DEBUG:boto:Token: None 2022-09-03T07:28:09.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JkbSY2U5G7w8IFPTbsCe5ePvhu4= 2022-09-03T07:28:09.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JkbSY2U5G7w8IFPTbsCe5ePvhu4=', 'Content-Length': '0'} 2022-09-03T07:28:09.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000819f8fba9eae4be9-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.782 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.783 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.783 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.784 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.784 DEBUG:boto:Method: GET 2022-09-03T07:28:09.785 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.786 DEBUG:boto:Data: 2022-09-03T07:28:09.786 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.787 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.788 DEBUG:boto:Params: {} 2022-09-03T07:28:09.789 DEBUG:boto:Token: None 2022-09-03T07:28:09.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:09.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JkbSY2U5G7w8IFPTbsCe5ePvhu4= 2022-09-03T07:28:09.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JkbSY2U5G7w8IFPTbsCe5ePvhu4=', 'Content-Length': '0'} 2022-09-03T07:28:09.793 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000537fd8aa8ed53d07-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.794 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.795 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.795 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:09.796 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:09.797 DEBUG:boto:Method: GET 2022-09-03T07:28:09.797 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:09.798 DEBUG:boto:Data: 2022-09-03T07:28:09.798 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.799 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.800 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.800 DEBUG:boto:Params: {} 2022-09-03T07:28:09.801 DEBUG:boto:Token: None 2022-09-03T07:28:09.802 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.806 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a14afd297f3c1d21-0063130209-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.807 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.808 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.809 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.809 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:09.810 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:09.811 DEBUG:boto:Method: GET 2022-09-03T07:28:09.812 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:09.812 DEBUG:boto:Data: 2022-09-03T07:28:09.813 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.814 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.815 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.816 DEBUG:boto:Params: {} 2022-09-03T07:28:09.817 DEBUG:boto:Token: None 2022-09-03T07:28:09.818 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.824 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000047c855504394fd2-0063130209-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.825 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.826 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.827 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.827 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.828 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:09.828 DEBUG:boto:Data: 2022-09-03T07:28:09.829 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.830 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.831 DEBUG:boto:Params: {} 2022-09-03T07:28:09.832 DEBUG:boto:Token: None 2022-09-03T07:28:09.833 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22 2022-09-03T07:28:09.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kwAkcq8R36TPFc+U0v9DldElqPI= 2022-09-03T07:28:09.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kwAkcq8R36TPFc+U0v9DldElqPI=', 'Content-Length': '0'} 2022-09-03T07:28:09.837 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e361c3ddc180b1e5-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.838 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.839 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.839 DEBUG:boto:Method: HEAD 2022-09-03T07:28:09.840 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:09.841 DEBUG:boto:Data: 2022-09-03T07:28:09.842 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.843 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.844 DEBUG:boto:Params: {} 2022-09-03T07:28:09.845 DEBUG:boto:Token: None 2022-09-03T07:28:09.846 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22 2022-09-03T07:28:09.847 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kwAkcq8R36TPFc+U0v9DldElqPI= 2022-09-03T07:28:09.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kwAkcq8R36TPFc+U0v9DldElqPI=', 'Content-Length': '0'} 2022-09-03T07:28:09.849 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092e0122b9e13e545-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.850 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:28:09.851 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.852 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.853 DEBUG:boto:Method: GET 2022-09-03T07:28:09.853 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:09.854 DEBUG:boto:Data: 2022-09-03T07:28:09.855 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.856 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.857 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.858 DEBUG:boto:Params: {} 2022-09-03T07:28:09.858 DEBUG:boto:Token: None 2022-09-03T07:28:09.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22 2022-09-03T07:28:09.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GtcOblDwx0jQyf6NlQ2u2a/2AYw= 2022-09-03T07:28:09.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GtcOblDwx0jQyf6NlQ2u2a/2AYw=', 'Content-Length': '0'} 2022-09-03T07:28:09.863 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd210a9233f92719-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.906 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:09.907 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:09.907 DEBUG:boto:Method: GET 2022-09-03T07:28:09.908 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:09.909 DEBUG:boto:Data: 2022-09-03T07:28:09.910 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.910 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.911 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.912 DEBUG:boto:Params: {} 2022-09-03T07:28:09.915 DEBUG:boto:Token: None 2022-09-03T07:28:09.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj22 2022-09-03T07:28:09.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GtcOblDwx0jQyf6NlQ2u2a/2AYw= 2022-09-03T07:28:09.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GtcOblDwx0jQyf6NlQ2u2a/2AYw=', 'Content-Length': '0'} 2022-09-03T07:28:09.921 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000805fc1741bae96c0-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.922 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.923 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.923 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:09.924 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:09.926 DEBUG:boto:Method: GET 2022-09-03T07:28:09.929 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:09.931 DEBUG:boto:Data: 2022-09-03T07:28:09.932 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.936 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.937 DEBUG:boto:Params: {} 2022-09-03T07:28:09.939 DEBUG:boto:Token: None 2022-09-03T07:28:09.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.947 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b8c876f1feefbb4-0063130209-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.948 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.950 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:09.951 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:09.952 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:09.953 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:09.954 DEBUG:boto:Method: GET 2022-09-03T07:28:09.955 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:09.956 DEBUG:boto:Data: 2022-09-03T07:28:09.957 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.958 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.962 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.966 DEBUG:boto:Params: {} 2022-09-03T07:28:09.967 DEBUG:boto:Token: None 2022-09-03T07:28:09.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/ 2022-09-03T07:28:09.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI= 2022-09-03T07:28:09.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/5I0yRG5G3HksSfRE9oPeqaH4MI=', 'Content-Length': '0'} 2022-09-03T07:28:09.976 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f7fba351f353fc34-0063130209-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.977 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:09.979 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:28:09.979 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:09.980 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:09.981 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:09.981 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:09.982 DEBUG:boto:Method: GET 2022-09-03T07:28:09.983 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:09.983 DEBUG:boto:Data: 2022-09-03T07:28:09.984 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:09.985 DEBUG:boto:Port: 8001 2022-09-03T07:28:09.985 DEBUG:boto:Params: {} 2022-09-03T07:28:09.986 DEBUG:boto:Token: None 2022-09-03T07:28:09.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:09.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E8u9J2BT6F00IlVTut/2kDNIoQU= 2022-09-03T07:28:09.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E8u9J2BT6F00IlVTut/2kDNIoQU=', 'Content-Length': '0'} 2022-09-03T07:28:09.990 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d038334bf2821e09-0063130209-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:09.991 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:09.991 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:09.992 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:09.993 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:09.993 DEBUG:boto:Method: GET 2022-09-03T07:28:09.994 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:09.994 DEBUG:boto:Data: 2022-09-03T07:28:09.995 DEBUG:boto:Headers: {} 2022-09-03T07:28:09.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:09.996 DEBUG:boto:Port: 8000 2022-09-03T07:28:09.996 DEBUG:boto:Params: {} 2022-09-03T07:28:09.997 DEBUG:boto:Token: None 2022-09-03T07:28:09.998 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:09 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:09.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E8u9J2BT6F00IlVTut/2kDNIoQU= 2022-09-03T07:28:09.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E8u9J2BT6F00IlVTut/2kDNIoQU=', 'Content-Length': '0'} 2022-09-03T07:28:10.001 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097ac9514a82f3e31-0063130209-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.002 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:10.002 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:10.003 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.003 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.004 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.004 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.005 DEBUG:boto:Data: 2022-09-03T07:28:10.005 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.006 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.006 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.007 DEBUG:boto:Params: {} 2022-09-03T07:28:10.007 DEBUG:boto:Token: None 2022-09-03T07:28:10.008 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Q5c17UW6x1kZ+8TNkvs3m+DzV8= 2022-09-03T07:28:10.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Q5c17UW6x1kZ+8TNkvs3m+DzV8=', 'Content-Length': '0'} 2022-09-03T07:28:10.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000723038a0aeec3d43-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.012 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:10.012 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:10.013 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.014 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.015 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.016 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.017 DEBUG:boto:Data: 2022-09-03T07:28:10.018 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.019 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.020 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.021 DEBUG:boto:Params: {} 2022-09-03T07:28:10.022 DEBUG:boto:Token: None 2022-09-03T07:28:10.023 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6Q5c17UW6x1kZ+8TNkvs3m+DzV8= 2022-09-03T07:28:10.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6Q5c17UW6x1kZ+8TNkvs3m+DzV8=', 'Content-Length': '0'} 2022-09-03T07:28:10.028 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9c73ddd8842f5dd-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.029 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:28:10.030 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:10.030 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:10.031 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.031 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.032 DEBUG:boto:Method: GET 2022-09-03T07:28:10.033 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.033 DEBUG:boto:Data: 2022-09-03T07:28:10.034 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.034 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.035 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.035 DEBUG:boto:Params: {} 2022-09-03T07:28:10.036 DEBUG:boto:Token: None 2022-09-03T07:28:10.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gy0sZQqAREUDQgVzXOL27T6MZTU= 2022-09-03T07:28:10.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gy0sZQqAREUDQgVzXOL27T6MZTU=', 'Content-Length': '0'} 2022-09-03T07:28:10.040 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df6a6dcb8af696a0-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.041 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:10.041 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:10.042 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.042 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.043 DEBUG:boto:Method: GET 2022-09-03T07:28:10.044 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.044 DEBUG:boto:Data: 2022-09-03T07:28:10.045 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.046 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.046 DEBUG:boto:Params: {} 2022-09-03T07:28:10.047 DEBUG:boto:Token: None 2022-09-03T07:28:10.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:10.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gy0sZQqAREUDQgVzXOL27T6MZTU= 2022-09-03T07:28:10.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gy0sZQqAREUDQgVzXOL27T6MZTU=', 'Content-Length': '0'} 2022-09-03T07:28:10.051 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9e14b83e4c82231-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.052 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.052 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.053 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:10.053 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:10.054 DEBUG:boto:Method: GET 2022-09-03T07:28:10.055 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:10.055 DEBUG:boto:Data: 2022-09-03T07:28:10.056 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.057 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.057 DEBUG:boto:Params: {} 2022-09-03T07:28:10.058 DEBUG:boto:Token: None 2022-09-03T07:28:10.059 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.062 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000290f4ef7b3d62774-006313020a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.063 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.064 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.065 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.065 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:10.066 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:10.066 DEBUG:boto:Method: GET 2022-09-03T07:28:10.067 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:10.067 DEBUG:boto:Data: 2022-09-03T07:28:10.068 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.069 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.069 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.070 DEBUG:boto:Params: {} 2022-09-03T07:28:10.070 DEBUG:boto:Token: None 2022-09-03T07:28:10.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.076 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cc68b38c2a27eca0-006313020a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.077 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.078 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:10.078 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:10.079 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.079 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:10.080 DEBUG:boto:Data: 2022-09-03T07:28:10.080 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.081 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.081 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.082 DEBUG:boto:Params: {} 2022-09-03T07:28:10.082 DEBUG:boto:Token: None 2022-09-03T07:28:10.083 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj23 2022-09-03T07:28:10.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GsE3z0dnZG9f4Hrp7CB6nlnyEWA= 2022-09-03T07:28:10.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GsE3z0dnZG9f4Hrp7CB6nlnyEWA=', 'Content-Length': '0'} 2022-09-03T07:28:10.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e054190961228fd6-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.087 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:10.087 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:10.088 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.089 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:10.089 DEBUG:boto:Data: 2022-09-03T07:28:10.091 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.093 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.094 DEBUG:boto:Params: {} 2022-09-03T07:28:10.095 DEBUG:boto:Token: None 2022-09-03T07:28:10.096 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj23 2022-09-03T07:28:10.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GsE3z0dnZG9f4Hrp7CB6nlnyEWA= 2022-09-03T07:28:10.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GsE3z0dnZG9f4Hrp7CB6nlnyEWA=', 'Content-Length': '0'} 2022-09-03T07:28:10.100 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007040f391af6b97f1-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.102 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:28:10.102 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:10.103 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:10.104 DEBUG:boto:Method: GET 2022-09-03T07:28:10.105 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:10.105 DEBUG:boto:Data: 2022-09-03T07:28:10.106 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.107 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.107 DEBUG:boto:Params: {} 2022-09-03T07:28:10.108 DEBUG:boto:Token: None 2022-09-03T07:28:10.108 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj23 2022-09-03T07:28:10.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFhkncDotdwZlXkzEJ0vdGW5+KY= 2022-09-03T07:28:10.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFhkncDotdwZlXkzEJ0vdGW5+KY=', 'Content-Length': '0'} 2022-09-03T07:28:10.112 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000efbdd287ef9e3b5c-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.158 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:10.159 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:10.160 DEBUG:boto:Method: GET 2022-09-03T07:28:10.160 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:10.161 DEBUG:boto:Data: 2022-09-03T07:28:10.161 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.162 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.163 DEBUG:boto:Params: {} 2022-09-03T07:28:10.164 DEBUG:boto:Token: None 2022-09-03T07:28:10.164 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj23 2022-09-03T07:28:10.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFhkncDotdwZlXkzEJ0vdGW5+KY= 2022-09-03T07:28:10.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFhkncDotdwZlXkzEJ0vdGW5+KY=', 'Content-Length': '0'} 2022-09-03T07:28:10.167 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053c4393ca003dd81-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.168 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.169 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.169 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:10.170 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:10.170 DEBUG:boto:Method: GET 2022-09-03T07:28:10.171 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:10.171 DEBUG:boto:Data: 2022-09-03T07:28:10.172 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.173 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.173 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.174 DEBUG:boto:Params: {} 2022-09-03T07:28:10.174 DEBUG:boto:Token: None 2022-09-03T07:28:10.175 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.175 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.176 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.180 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c23995499c0d1b75-006313020a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.180 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.181 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.182 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.182 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:10.183 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:10.183 DEBUG:boto:Method: GET 2022-09-03T07:28:10.184 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:10.184 DEBUG:boto:Data: 2022-09-03T07:28:10.185 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.186 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.186 DEBUG:boto:Params: {} 2022-09-03T07:28:10.187 DEBUG:boto:Token: None 2022-09-03T07:28:10.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.193 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dc589af088e94d86-006313020a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.193 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.194 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:28:10.196 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.197 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.197 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.198 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.199 DEBUG:boto:Method: GET 2022-09-03T07:28:10.199 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.200 DEBUG:boto:Data: 2022-09-03T07:28:10.200 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.201 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.202 DEBUG:boto:Params: {} 2022-09-03T07:28:10.203 DEBUG:boto:Token: None 2022-09-03T07:28:10.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WiwCe22FXs/q+8WONSxbJ7ygA3c= 2022-09-03T07:28:10.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WiwCe22FXs/q+8WONSxbJ7ygA3c=', 'Content-Length': '0'} 2022-09-03T07:28:10.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5d3b86686931c89-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.208 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.209 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.210 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.211 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.212 DEBUG:boto:Method: GET 2022-09-03T07:28:10.212 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.213 DEBUG:boto:Data: 2022-09-03T07:28:10.214 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.214 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.215 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.216 DEBUG:boto:Params: {} 2022-09-03T07:28:10.218 DEBUG:boto:Token: None 2022-09-03T07:28:10.219 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WiwCe22FXs/q+8WONSxbJ7ygA3c= 2022-09-03T07:28:10.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WiwCe22FXs/q+8WONSxbJ7ygA3c=', 'Content-Length': '0'} 2022-09-03T07:28:10.224 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003bc2147096907c42-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.225 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.227 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.228 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.229 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.230 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.230 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.231 DEBUG:boto:Data: 2022-09-03T07:28:10.232 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.233 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.236 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.237 DEBUG:boto:Params: {} 2022-09-03T07:28:10.238 DEBUG:boto:Token: None 2022-09-03T07:28:10.239 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ltOex86oYD+xDCYi5SSt5xnX0uY= 2022-09-03T07:28:10.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ltOex86oYD+xDCYi5SSt5xnX0uY=', 'Content-Length': '0'} 2022-09-03T07:28:10.244 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb3c04b9b5c80d8b-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.244 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.245 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.246 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.246 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.247 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.247 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.249 DEBUG:boto:Data: 2022-09-03T07:28:10.249 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.251 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.251 DEBUG:boto:Params: {} 2022-09-03T07:28:10.252 DEBUG:boto:Token: None 2022-09-03T07:28:10.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ltOex86oYD+xDCYi5SSt5xnX0uY= 2022-09-03T07:28:10.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ltOex86oYD+xDCYi5SSt5xnX0uY=', 'Content-Length': '0'} 2022-09-03T07:28:10.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f630451b5d54db1c-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.258 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:28:10.259 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.260 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.261 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.262 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.263 DEBUG:boto:Method: GET 2022-09-03T07:28:10.264 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.265 DEBUG:boto:Data: 2022-09-03T07:28:10.266 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.267 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.267 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.268 DEBUG:boto:Params: {} 2022-09-03T07:28:10.268 DEBUG:boto:Token: None 2022-09-03T07:28:10.269 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.270 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WiwCe22FXs/q+8WONSxbJ7ygA3c= 2022-09-03T07:28:10.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WiwCe22FXs/q+8WONSxbJ7ygA3c=', 'Content-Length': '0'} 2022-09-03T07:28:10.272 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092b4a05556447b3d-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.314 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.315 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.316 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.316 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.317 DEBUG:boto:Method: GET 2022-09-03T07:28:10.317 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.318 DEBUG:boto:Data: 2022-09-03T07:28:10.319 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.319 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.320 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.321 DEBUG:boto:Params: {} 2022-09-03T07:28:10.321 DEBUG:boto:Token: None 2022-09-03T07:28:10.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:10.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WiwCe22FXs/q+8WONSxbJ7ygA3c= 2022-09-03T07:28:10.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WiwCe22FXs/q+8WONSxbJ7ygA3c=', 'Content-Length': '0'} 2022-09-03T07:28:10.326 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b99361858f844ca1-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.327 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.327 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.328 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:10.329 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:10.330 DEBUG:boto:Method: GET 2022-09-03T07:28:10.331 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:10.332 DEBUG:boto:Data: 2022-09-03T07:28:10.334 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.335 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.336 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.337 DEBUG:boto:Params: {} 2022-09-03T07:28:10.338 DEBUG:boto:Token: None 2022-09-03T07:28:10.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.345 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000007df071b671b6a2c-006313020a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.347 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.348 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.349 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.350 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:10.351 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:10.352 DEBUG:boto:Method: GET 2022-09-03T07:28:10.353 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:10.354 DEBUG:boto:Data: 2022-09-03T07:28:10.355 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.356 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.357 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.358 DEBUG:boto:Params: {} 2022-09-03T07:28:10.359 DEBUG:boto:Token: None 2022-09-03T07:28:10.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.367 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000072d16f2b5c6f2317-006313020a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.368 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.370 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.371 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.372 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.373 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:10.374 DEBUG:boto:Data: 2022-09-03T07:28:10.375 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.376 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.377 DEBUG:boto:Params: {} 2022-09-03T07:28:10.378 DEBUG:boto:Token: None 2022-09-03T07:28:10.378 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29 2022-09-03T07:28:10.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GghydYzMh87DtsbAYnBnij+aqiY= 2022-09-03T07:28:10.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GghydYzMh87DtsbAYnBnij+aqiY=', 'Content-Length': '0'} 2022-09-03T07:28:10.388 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c38783d15da36676-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.389 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.389 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.390 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.390 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:10.391 DEBUG:boto:Data: 2022-09-03T07:28:10.391 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.393 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.393 DEBUG:boto:Params: {} 2022-09-03T07:28:10.394 DEBUG:boto:Token: None 2022-09-03T07:28:10.395 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29 2022-09-03T07:28:10.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GghydYzMh87DtsbAYnBnij+aqiY= 2022-09-03T07:28:10.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GghydYzMh87DtsbAYnBnij+aqiY=', 'Content-Length': '0'} 2022-09-03T07:28:10.399 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a607f3300fd410a-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.399 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:28:10.400 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.401 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.401 DEBUG:boto:Method: GET 2022-09-03T07:28:10.403 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:10.404 DEBUG:boto:Data: 2022-09-03T07:28:10.405 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.405 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.406 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.410 DEBUG:boto:Params: {} 2022-09-03T07:28:10.411 DEBUG:boto:Token: None 2022-09-03T07:28:10.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29 2022-09-03T07:28:10.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4C4QzWNV/8R+/87Glz84/RXHVRU= 2022-09-03T07:28:10.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4C4QzWNV/8R+/87Glz84/RXHVRU=', 'Content-Length': '0'} 2022-09-03T07:28:10.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b4b6989dacf73b8-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.459 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:10.460 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:10.460 DEBUG:boto:Method: GET 2022-09-03T07:28:10.461 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:10.461 DEBUG:boto:Data: 2022-09-03T07:28:10.462 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.463 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.464 DEBUG:boto:Params: {} 2022-09-03T07:28:10.464 DEBUG:boto:Token: None 2022-09-03T07:28:10.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj29 2022-09-03T07:28:10.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4C4QzWNV/8R+/87Glz84/RXHVRU= 2022-09-03T07:28:10.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4C4QzWNV/8R+/87Glz84/RXHVRU=', 'Content-Length': '0'} 2022-09-03T07:28:10.468 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001bf8fdfafff139e5-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.469 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.470 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.470 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:10.471 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:10.471 DEBUG:boto:Method: GET 2022-09-03T07:28:10.472 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:10.472 DEBUG:boto:Data: 2022-09-03T07:28:10.473 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.474 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.475 DEBUG:boto:Params: {} 2022-09-03T07:28:10.476 DEBUG:boto:Token: None 2022-09-03T07:28:10.476 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.481 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000853a7137ead674bd-006313020a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.482 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.483 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.484 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.485 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:10.486 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:10.487 DEBUG:boto:Method: GET 2022-09-03T07:28:10.488 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:10.489 DEBUG:boto:Data: 2022-09-03T07:28:10.490 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.492 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.493 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.494 DEBUG:boto:Params: {} 2022-09-03T07:28:10.495 DEBUG:boto:Token: None 2022-09-03T07:28:10.496 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.502 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fce79ef721506f98-006313020a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.503 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.505 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:28:10.506 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.507 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.508 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.509 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.510 DEBUG:boto:Method: GET 2022-09-03T07:28:10.511 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.511 DEBUG:boto:Data: 2022-09-03T07:28:10.512 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.513 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.513 DEBUG:boto:Params: {} 2022-09-03T07:28:10.514 DEBUG:boto:Token: None 2022-09-03T07:28:10.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.515 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6okvFrNR/QNP9yHFtEcsyKSQHw= 2022-09-03T07:28:10.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6okvFrNR/QNP9yHFtEcsyKSQHw=', 'Content-Length': '0'} 2022-09-03T07:28:10.522 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000716978c7c14cfe63-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.524 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.525 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.526 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.532 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.533 DEBUG:boto:Method: GET 2022-09-03T07:28:10.534 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.534 DEBUG:boto:Data: 2022-09-03T07:28:10.535 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.536 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.537 DEBUG:boto:Params: {} 2022-09-03T07:28:10.537 DEBUG:boto:Token: None 2022-09-03T07:28:10.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6okvFrNR/QNP9yHFtEcsyKSQHw= 2022-09-03T07:28:10.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6okvFrNR/QNP9yHFtEcsyKSQHw=', 'Content-Length': '0'} 2022-09-03T07:28:10.541 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000146d0b49344c3659-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.582 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.583 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.584 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.584 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.585 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.586 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.587 DEBUG:boto:Data: 2022-09-03T07:28:10.587 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.588 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.588 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.589 DEBUG:boto:Params: {} 2022-09-03T07:28:10.589 DEBUG:boto:Token: None 2022-09-03T07:28:10.590 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gS/SMuoCXaKxwb8mUxVw5FeSGuQ= 2022-09-03T07:28:10.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gS/SMuoCXaKxwb8mUxVw5FeSGuQ=', 'Content-Length': '0'} 2022-09-03T07:28:10.594 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8f2de46ddf5f422-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.594 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.595 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.596 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.597 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.597 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.598 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.598 DEBUG:boto:Data: 2022-09-03T07:28:10.599 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.600 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.600 DEBUG:boto:Params: {} 2022-09-03T07:28:10.601 DEBUG:boto:Token: None 2022-09-03T07:28:10.602 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gS/SMuoCXaKxwb8mUxVw5FeSGuQ= 2022-09-03T07:28:10.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gS/SMuoCXaKxwb8mUxVw5FeSGuQ=', 'Content-Length': '0'} 2022-09-03T07:28:10.611 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006515261c8a9f49aa-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.612 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:28:10.613 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.614 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.615 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.616 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.617 DEBUG:boto:Method: GET 2022-09-03T07:28:10.618 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.619 DEBUG:boto:Data: 2022-09-03T07:28:10.620 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.621 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.622 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.623 DEBUG:boto:Params: {} 2022-09-03T07:28:10.624 DEBUG:boto:Token: None 2022-09-03T07:28:10.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6okvFrNR/QNP9yHFtEcsyKSQHw= 2022-09-03T07:28:10.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6okvFrNR/QNP9yHFtEcsyKSQHw=', 'Content-Length': '0'} 2022-09-03T07:28:10.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005de0ff2fc5a0bac4-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.633 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.634 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.635 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.636 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.637 DEBUG:boto:Method: GET 2022-09-03T07:28:10.637 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.638 DEBUG:boto:Data: 2022-09-03T07:28:10.639 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.641 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.642 DEBUG:boto:Params: {} 2022-09-03T07:28:10.643 DEBUG:boto:Token: None 2022-09-03T07:28:10.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:10.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r6okvFrNR/QNP9yHFtEcsyKSQHw= 2022-09-03T07:28:10.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r6okvFrNR/QNP9yHFtEcsyKSQHw=', 'Content-Length': '0'} 2022-09-03T07:28:10.648 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024314c0f302c51c1-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.649 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.650 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.650 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:10.651 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:10.652 DEBUG:boto:Method: GET 2022-09-03T07:28:10.652 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:10.653 DEBUG:boto:Data: 2022-09-03T07:28:10.653 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.654 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.654 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.655 DEBUG:boto:Params: {} 2022-09-03T07:28:10.656 DEBUG:boto:Token: None 2022-09-03T07:28:10.657 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.661 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009dd73c7e442ea583-006313020a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.662 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.663 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.664 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.664 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:10.665 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:10.665 DEBUG:boto:Method: GET 2022-09-03T07:28:10.666 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:10.667 DEBUG:boto:Data: 2022-09-03T07:28:10.667 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.668 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.668 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.669 DEBUG:boto:Params: {} 2022-09-03T07:28:10.669 DEBUG:boto:Token: None 2022-09-03T07:28:10.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.677 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed9ff5a78af57ecb-006313020a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.679 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.680 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.681 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.683 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.684 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:10.684 DEBUG:boto:Data: 2022-09-03T07:28:10.685 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.685 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.686 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.687 DEBUG:boto:Params: {} 2022-09-03T07:28:10.687 DEBUG:boto:Token: None 2022-09-03T07:28:10.688 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31 2022-09-03T07:28:10.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vYvtr6W3PjwN8Ny0NVwBTq1KoYw= 2022-09-03T07:28:10.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vYvtr6W3PjwN8Ny0NVwBTq1KoYw=', 'Content-Length': '0'} 2022-09-03T07:28:10.692 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b59bc2b333aae9dc-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.693 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.693 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.694 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.696 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:10.697 DEBUG:boto:Data: 2022-09-03T07:28:10.698 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.700 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.701 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.702 DEBUG:boto:Params: {} 2022-09-03T07:28:10.703 DEBUG:boto:Token: None 2022-09-03T07:28:10.704 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31 2022-09-03T07:28:10.705 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vYvtr6W3PjwN8Ny0NVwBTq1KoYw= 2022-09-03T07:28:10.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vYvtr6W3PjwN8Ny0NVwBTq1KoYw=', 'Content-Length': '0'} 2022-09-03T07:28:10.710 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e1521c30264f90f-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.711 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:28:10.712 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.713 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.714 DEBUG:boto:Method: GET 2022-09-03T07:28:10.715 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:10.716 DEBUG:boto:Data: 2022-09-03T07:28:10.716 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.718 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.719 DEBUG:boto:Params: {} 2022-09-03T07:28:10.721 DEBUG:boto:Token: None 2022-09-03T07:28:10.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31 2022-09-03T07:28:10.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nl4YA/HQd7QUc0JKxBc0M22/vV0= 2022-09-03T07:28:10.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nl4YA/HQd7QUc0JKxBc0M22/vV0=', 'Content-Length': '0'} 2022-09-03T07:28:10.727 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000403e844a0fc563e2-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.728 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:10.729 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:10.730 DEBUG:boto:Method: GET 2022-09-03T07:28:10.731 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:10.732 DEBUG:boto:Data: 2022-09-03T07:28:10.732 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.733 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.734 DEBUG:boto:Params: {} 2022-09-03T07:28:10.735 DEBUG:boto:Token: None 2022-09-03T07:28:10.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj31 2022-09-03T07:28:10.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Nl4YA/HQd7QUc0JKxBc0M22/vV0= 2022-09-03T07:28:10.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Nl4YA/HQd7QUc0JKxBc0M22/vV0=', 'Content-Length': '0'} 2022-09-03T07:28:10.739 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000039cee54d9526747-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.740 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.741 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.742 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:10.743 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:10.744 DEBUG:boto:Method: GET 2022-09-03T07:28:10.745 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:10.746 DEBUG:boto:Data: 2022-09-03T07:28:10.747 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.749 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.750 DEBUG:boto:Params: {} 2022-09-03T07:28:10.751 DEBUG:boto:Token: None 2022-09-03T07:28:10.752 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.757 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000efd3754f882b5f9d-006313020a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.758 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.760 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.761 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.762 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:10.762 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:10.763 DEBUG:boto:Method: GET 2022-09-03T07:28:10.763 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:10.764 DEBUG:boto:Data: 2022-09-03T07:28:10.764 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.766 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.766 DEBUG:boto:Params: {} 2022-09-03T07:28:10.767 DEBUG:boto:Token: None 2022-09-03T07:28:10.768 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.773 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed61d8cb303c514e-006313020a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.814 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.815 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:28:10.816 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:10.816 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:10.817 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.817 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.818 DEBUG:boto:Method: GET 2022-09-03T07:28:10.818 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.819 DEBUG:boto:Data: 2022-09-03T07:28:10.819 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.820 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.821 DEBUG:boto:Params: {} 2022-09-03T07:28:10.822 DEBUG:boto:Token: None 2022-09-03T07:28:10.823 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ayUnhJTaEvpQIWI/v8BN08ZOn2Y= 2022-09-03T07:28:10.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ayUnhJTaEvpQIWI/v8BN08ZOn2Y=', 'Content-Length': '0'} 2022-09-03T07:28:10.826 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015ac967594e1f658-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.827 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:10.828 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:10.829 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.829 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.830 DEBUG:boto:Method: GET 2022-09-03T07:28:10.830 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.831 DEBUG:boto:Data: 2022-09-03T07:28:10.832 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.833 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.833 DEBUG:boto:Params: {} 2022-09-03T07:28:10.834 DEBUG:boto:Token: None 2022-09-03T07:28:10.834 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.835 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ayUnhJTaEvpQIWI/v8BN08ZOn2Y= 2022-09-03T07:28:10.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ayUnhJTaEvpQIWI/v8BN08ZOn2Y=', 'Content-Length': '0'} 2022-09-03T07:28:10.838 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dea5ff124d85a7b7-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.839 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:10.840 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:10.840 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.841 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.841 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.842 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.842 DEBUG:boto:Data: 2022-09-03T07:28:10.843 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.843 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.844 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.844 DEBUG:boto:Params: {} 2022-09-03T07:28:10.845 DEBUG:boto:Token: None 2022-09-03T07:28:10.845 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GR4rg4QFq26WzKfuFBbq9EBoX5U= 2022-09-03T07:28:10.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GR4rg4QFq26WzKfuFBbq9EBoX5U=', 'Content-Length': '0'} 2022-09-03T07:28:10.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026948afc2a0fcbb6-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.849 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:10.850 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:10.850 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.851 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.851 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.852 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.852 DEBUG:boto:Data: 2022-09-03T07:28:10.853 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.853 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.854 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.854 DEBUG:boto:Params: {} 2022-09-03T07:28:10.855 DEBUG:boto:Token: None 2022-09-03T07:28:10.856 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GR4rg4QFq26WzKfuFBbq9EBoX5U= 2022-09-03T07:28:10.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GR4rg4QFq26WzKfuFBbq9EBoX5U=', 'Content-Length': '0'} 2022-09-03T07:28:10.859 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003548a58813b52b43-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.860 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:28:10.861 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:10.861 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:10.862 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.862 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.863 DEBUG:boto:Method: GET 2022-09-03T07:28:10.863 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.864 DEBUG:boto:Data: 2022-09-03T07:28:10.864 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.865 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.866 DEBUG:boto:Params: {} 2022-09-03T07:28:10.867 DEBUG:boto:Token: None 2022-09-03T07:28:10.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ayUnhJTaEvpQIWI/v8BN08ZOn2Y= 2022-09-03T07:28:10.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ayUnhJTaEvpQIWI/v8BN08ZOn2Y=', 'Content-Length': '0'} 2022-09-03T07:28:10.871 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a4c6a1ab463e006-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.914 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:10.915 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:10.917 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.918 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.919 DEBUG:boto:Method: GET 2022-09-03T07:28:10.920 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.921 DEBUG:boto:Data: 2022-09-03T07:28:10.922 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.923 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.924 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.924 DEBUG:boto:Params: {} 2022-09-03T07:28:10.926 DEBUG:boto:Token: None 2022-09-03T07:28:10.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:10.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ayUnhJTaEvpQIWI/v8BN08ZOn2Y= 2022-09-03T07:28:10.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ayUnhJTaEvpQIWI/v8BN08ZOn2Y=', 'Content-Length': '0'} 2022-09-03T07:28:10.931 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a400fb2b68d250d7-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.932 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.933 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.934 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:10.935 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:10.936 DEBUG:boto:Method: GET 2022-09-03T07:28:10.937 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:10.938 DEBUG:boto:Data: 2022-09-03T07:28:10.939 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.940 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.940 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.941 DEBUG:boto:Params: {} 2022-09-03T07:28:10.942 DEBUG:boto:Token: None 2022-09-03T07:28:10.943 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.947 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000040c7f1c80a8684db-006313020a-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.949 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.950 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:10.951 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:10.952 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:10.953 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:10.955 DEBUG:boto:Method: GET 2022-09-03T07:28:10.956 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:10.957 DEBUG:boto:Data: 2022-09-03T07:28:10.958 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.958 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.959 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.960 DEBUG:boto:Params: {} 2022-09-03T07:28:10.960 DEBUG:boto:Token: None 2022-09-03T07:28:10.961 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/ 2022-09-03T07:28:10.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig= 2022-09-03T07:28:10.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Nm7ZPZq5x6bSCGutOl6x5i3tig=', 'Content-Length': '0'} 2022-09-03T07:28:10.966 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000034819f69820bca26-006313020a-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.967 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:10.968 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:10.969 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:10.970 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.971 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:10.972 DEBUG:boto:Data: 2022-09-03T07:28:10.973 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:10.975 DEBUG:boto:Port: 8001 2022-09-03T07:28:10.976 DEBUG:boto:Params: {} 2022-09-03T07:28:10.977 DEBUG:boto:Token: None 2022-09-03T07:28:10.978 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj33 2022-09-03T07:28:10.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:svqZ/4hvavBZ8Giq0uVjvnyZ32Y= 2022-09-03T07:28:10.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:svqZ/4hvavBZ8Giq0uVjvnyZ32Y=', 'Content-Length': '0'} 2022-09-03T07:28:10.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e20cabe9d3a5e7ff-006313020a-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:10.984 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:10.985 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:10.986 DEBUG:boto:Method: HEAD 2022-09-03T07:28:10.987 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:10.989 DEBUG:boto:Data: 2022-09-03T07:28:10.990 DEBUG:boto:Headers: {} 2022-09-03T07:28:10.991 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:10.992 DEBUG:boto:Port: 8000 2022-09-03T07:28:10.993 DEBUG:boto:Params: {} 2022-09-03T07:28:10.994 DEBUG:boto:Token: None 2022-09-03T07:28:10.995 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:10 GMT /avysmw-75/obj33 2022-09-03T07:28:10.997 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:svqZ/4hvavBZ8Giq0uVjvnyZ32Y= 2022-09-03T07:28:10.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:svqZ/4hvavBZ8Giq0uVjvnyZ32Y=', 'Content-Length': '0'} 2022-09-03T07:28:11.000 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d35d5897dd67c73-006313020a-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.002 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:28:11.003 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:11.004 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:11.005 DEBUG:boto:Method: GET 2022-09-03T07:28:11.006 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:11.007 DEBUG:boto:Data: 2022-09-03T07:28:11.008 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.010 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.011 DEBUG:boto:Params: {} 2022-09-03T07:28:11.012 DEBUG:boto:Token: None 2022-09-03T07:28:11.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj33 2022-09-03T07:28:11.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zkI3lEGaVvUvF8aV5W/tXxTByOo= 2022-09-03T07:28:11.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zkI3lEGaVvUvF8aV5W/tXxTByOo=', 'Content-Length': '0'} 2022-09-03T07:28:11.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e1bc7b4cb3ab009-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.018 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:11.019 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:11.020 DEBUG:boto:Method: GET 2022-09-03T07:28:11.021 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:11.022 DEBUG:boto:Data: 2022-09-03T07:28:11.023 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.024 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.025 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.026 DEBUG:boto:Params: {} 2022-09-03T07:28:11.027 DEBUG:boto:Token: None 2022-09-03T07:28:11.028 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj33 2022-09-03T07:28:11.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zkI3lEGaVvUvF8aV5W/tXxTByOo= 2022-09-03T07:28:11.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zkI3lEGaVvUvF8aV5W/tXxTByOo=', 'Content-Length': '0'} 2022-09-03T07:28:11.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f8a40d33d230794-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.074 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.075 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.076 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:11.077 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:11.078 DEBUG:boto:Method: GET 2022-09-03T07:28:11.079 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:11.080 DEBUG:boto:Data: 2022-09-03T07:28:11.081 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.082 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.082 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.083 DEBUG:boto:Params: {} 2022-09-03T07:28:11.084 DEBUG:boto:Token: None 2022-09-03T07:28:11.084 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.085 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.085 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.088 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c19925dcd4a81e4e-006313020b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.089 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.090 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.091 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.092 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:11.092 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:11.093 DEBUG:boto:Method: GET 2022-09-03T07:28:11.093 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:11.094 DEBUG:boto:Data: 2022-09-03T07:28:11.094 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.096 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.096 DEBUG:boto:Params: {} 2022-09-03T07:28:11.097 DEBUG:boto:Token: None 2022-09-03T07:28:11.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.103 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e09dc036a6f92bc9-006313020b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.104 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.105 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:28:11.106 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.107 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.108 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.108 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.109 DEBUG:boto:Method: GET 2022-09-03T07:28:11.111 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.111 DEBUG:boto:Data: 2022-09-03T07:28:11.112 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.113 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.114 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.115 DEBUG:boto:Params: {} 2022-09-03T07:28:11.116 DEBUG:boto:Token: None 2022-09-03T07:28:11.116 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+2fzrmzlYoa0P5Ni8sLleK1PZdY= 2022-09-03T07:28:11.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+2fzrmzlYoa0P5Ni8sLleK1PZdY=', 'Content-Length': '0'} 2022-09-03T07:28:11.122 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6b2658c598bacd3-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.123 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.124 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.124 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.125 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.126 DEBUG:boto:Method: GET 2022-09-03T07:28:11.127 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.128 DEBUG:boto:Data: 2022-09-03T07:28:11.129 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.129 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.131 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.132 DEBUG:boto:Params: {} 2022-09-03T07:28:11.133 DEBUG:boto:Token: None 2022-09-03T07:28:11.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+2fzrmzlYoa0P5Ni8sLleK1PZdY= 2022-09-03T07:28:11.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+2fzrmzlYoa0P5Ni8sLleK1PZdY=', 'Content-Length': '0'} 2022-09-03T07:28:11.139 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087e6ae4953e0130a-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.141 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.142 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.143 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.144 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.145 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.146 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.147 DEBUG:boto:Data: 2022-09-03T07:28:11.148 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.148 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.150 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.151 DEBUG:boto:Params: {} 2022-09-03T07:28:11.152 DEBUG:boto:Token: None 2022-09-03T07:28:11.153 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/i7yAEx4HTcqdn7h1lJeGHq7BM= 2022-09-03T07:28:11.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/i7yAEx4HTcqdn7h1lJeGHq7BM=', 'Content-Length': '0'} 2022-09-03T07:28:11.158 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a2f18eeab61ebc8-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.160 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.161 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.161 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.162 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.163 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.164 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.165 DEBUG:boto:Data: 2022-09-03T07:28:11.166 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.168 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.169 DEBUG:boto:Params: {} 2022-09-03T07:28:11.170 DEBUG:boto:Token: None 2022-09-03T07:28:11.171 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o/i7yAEx4HTcqdn7h1lJeGHq7BM= 2022-09-03T07:28:11.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o/i7yAEx4HTcqdn7h1lJeGHq7BM=', 'Content-Length': '0'} 2022-09-03T07:28:11.176 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c56b73250cfe92af-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.177 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:28:11.178 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.179 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.180 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.181 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.182 DEBUG:boto:Method: GET 2022-09-03T07:28:11.183 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.184 DEBUG:boto:Data: 2022-09-03T07:28:11.185 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.186 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.187 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.188 DEBUG:boto:Params: {} 2022-09-03T07:28:11.190 DEBUG:boto:Token: None 2022-09-03T07:28:11.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.192 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+2fzrmzlYoa0P5Ni8sLleK1PZdY= 2022-09-03T07:28:11.193 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+2fzrmzlYoa0P5Ni8sLleK1PZdY=', 'Content-Length': '0'} 2022-09-03T07:28:11.195 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da40062a59acf824-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.238 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.239 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.240 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.241 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.241 DEBUG:boto:Method: GET 2022-09-03T07:28:11.242 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.243 DEBUG:boto:Data: 2022-09-03T07:28:11.243 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.244 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.244 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.245 DEBUG:boto:Params: {} 2022-09-03T07:28:11.246 DEBUG:boto:Token: None 2022-09-03T07:28:11.246 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:11.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+2fzrmzlYoa0P5Ni8sLleK1PZdY= 2022-09-03T07:28:11.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+2fzrmzlYoa0P5Ni8sLleK1PZdY=', 'Content-Length': '0'} 2022-09-03T07:28:11.250 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003337257c06218b9-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.251 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.251 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.252 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:11.253 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:11.254 DEBUG:boto:Method: GET 2022-09-03T07:28:11.254 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:11.255 DEBUG:boto:Data: 2022-09-03T07:28:11.255 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.256 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.256 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.257 DEBUG:boto:Params: {} 2022-09-03T07:28:11.258 DEBUG:boto:Token: None 2022-09-03T07:28:11.258 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.259 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.260 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.262 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f31ea2b97423c1a2-006313020b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.263 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.264 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.265 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.266 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:11.266 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:11.267 DEBUG:boto:Method: GET 2022-09-03T07:28:11.267 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:11.268 DEBUG:boto:Data: 2022-09-03T07:28:11.268 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.270 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.271 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.272 DEBUG:boto:Params: {} 2022-09-03T07:28:11.273 DEBUG:boto:Token: None 2022-09-03T07:28:11.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.281 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009dd62d2a39083b3a-006313020b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.283 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.284 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.285 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.286 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.287 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:11.288 DEBUG:boto:Data: 2022-09-03T07:28:11.288 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.290 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.291 DEBUG:boto:Params: {} 2022-09-03T07:28:11.292 DEBUG:boto:Token: None 2022-09-03T07:28:11.292 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34 2022-09-03T07:28:11.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wZt4HbTuFt9buevxW3jd33X+qm8= 2022-09-03T07:28:11.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wZt4HbTuFt9buevxW3jd33X+qm8=', 'Content-Length': '0'} 2022-09-03T07:28:11.296 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a88791b996c4c62d-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.297 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.298 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.299 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.300 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:11.301 DEBUG:boto:Data: 2022-09-03T07:28:11.302 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.303 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.304 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.305 DEBUG:boto:Params: {} 2022-09-03T07:28:11.307 DEBUG:boto:Token: None 2022-09-03T07:28:11.308 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34 2022-09-03T07:28:11.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wZt4HbTuFt9buevxW3jd33X+qm8= 2022-09-03T07:28:11.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wZt4HbTuFt9buevxW3jd33X+qm8=', 'Content-Length': '0'} 2022-09-03T07:28:11.312 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000619059d711b4a977-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.314 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:28:11.314 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.315 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.316 DEBUG:boto:Method: GET 2022-09-03T07:28:11.317 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:11.318 DEBUG:boto:Data: 2022-09-03T07:28:11.320 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.321 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.322 DEBUG:boto:Params: {} 2022-09-03T07:28:11.324 DEBUG:boto:Token: None 2022-09-03T07:28:11.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34 2022-09-03T07:28:11.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SD+z424riMYKNTpfT6X41MYrgak= 2022-09-03T07:28:11.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SD+z424riMYKNTpfT6X41MYrgak=', 'Content-Length': '0'} 2022-09-03T07:28:11.328 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f8773be956239efe-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.329 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:11.330 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:11.331 DEBUG:boto:Method: GET 2022-09-03T07:28:11.332 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:11.332 DEBUG:boto:Data: 2022-09-03T07:28:11.333 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.334 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.335 DEBUG:boto:Params: {} 2022-09-03T07:28:11.336 DEBUG:boto:Token: None 2022-09-03T07:28:11.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj34 2022-09-03T07:28:11.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SD+z424riMYKNTpfT6X41MYrgak= 2022-09-03T07:28:11.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SD+z424riMYKNTpfT6X41MYrgak=', 'Content-Length': '0'} 2022-09-03T07:28:11.340 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016eaaa3e34f38ab9-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.341 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.341 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.342 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:11.342 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:11.343 DEBUG:boto:Method: GET 2022-09-03T07:28:11.344 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:11.344 DEBUG:boto:Data: 2022-09-03T07:28:11.345 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.346 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.346 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.347 DEBUG:boto:Params: {} 2022-09-03T07:28:11.347 DEBUG:boto:Token: None 2022-09-03T07:28:11.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.352 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eae04664a8dd899b-006313020b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.353 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.354 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.355 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.355 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:11.356 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:11.356 DEBUG:boto:Method: GET 2022-09-03T07:28:11.357 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:11.357 DEBUG:boto:Data: 2022-09-03T07:28:11.358 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.358 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.359 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.359 DEBUG:boto:Params: {} 2022-09-03T07:28:11.360 DEBUG:boto:Token: None 2022-09-03T07:28:11.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.366 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d8ab798e60eb9f4-006313020b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.367 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.368 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:28:11.368 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.369 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.369 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.370 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.370 DEBUG:boto:Method: GET 2022-09-03T07:28:11.371 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.371 DEBUG:boto:Data: 2022-09-03T07:28:11.372 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.372 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.373 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.374 DEBUG:boto:Params: {} 2022-09-03T07:28:11.374 DEBUG:boto:Token: None 2022-09-03T07:28:11.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sVnbkBVaY/qrrals47IY2poY9nE= 2022-09-03T07:28:11.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sVnbkBVaY/qrrals47IY2poY9nE=', 'Content-Length': '0'} 2022-09-03T07:28:11.379 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000edf89c2c81cbb098-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.380 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.380 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.381 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.381 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.382 DEBUG:boto:Method: GET 2022-09-03T07:28:11.383 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.383 DEBUG:boto:Data: 2022-09-03T07:28:11.384 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.384 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.385 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.386 DEBUG:boto:Params: {} 2022-09-03T07:28:11.386 DEBUG:boto:Token: None 2022-09-03T07:28:11.387 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.388 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sVnbkBVaY/qrrals47IY2poY9nE= 2022-09-03T07:28:11.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sVnbkBVaY/qrrals47IY2poY9nE=', 'Content-Length': '0'} 2022-09-03T07:28:11.391 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078f769ec92ba682f-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.438 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.439 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.440 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.441 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.441 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.442 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.442 DEBUG:boto:Data: 2022-09-03T07:28:11.443 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.444 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.445 DEBUG:boto:Params: {} 2022-09-03T07:28:11.445 DEBUG:boto:Token: None 2022-09-03T07:28:11.446 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0AmftsAGEP+SHcw78t4qCoCAySE= 2022-09-03T07:28:11.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0AmftsAGEP+SHcw78t4qCoCAySE=', 'Content-Length': '0'} 2022-09-03T07:28:11.449 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac31fa76599fc792-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.450 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.451 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.451 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.452 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.452 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.453 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.454 DEBUG:boto:Data: 2022-09-03T07:28:11.454 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.455 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.455 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.456 DEBUG:boto:Params: {} 2022-09-03T07:28:11.456 DEBUG:boto:Token: None 2022-09-03T07:28:11.457 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0AmftsAGEP+SHcw78t4qCoCAySE= 2022-09-03T07:28:11.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0AmftsAGEP+SHcw78t4qCoCAySE=', 'Content-Length': '0'} 2022-09-03T07:28:11.461 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3cca39590e08583-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.461 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:28:11.462 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.462 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.463 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.464 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.464 DEBUG:boto:Method: GET 2022-09-03T07:28:11.465 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.465 DEBUG:boto:Data: 2022-09-03T07:28:11.466 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.467 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.467 DEBUG:boto:Params: {} 2022-09-03T07:28:11.468 DEBUG:boto:Token: None 2022-09-03T07:28:11.469 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sVnbkBVaY/qrrals47IY2poY9nE= 2022-09-03T07:28:11.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sVnbkBVaY/qrrals47IY2poY9nE=', 'Content-Length': '0'} 2022-09-03T07:28:11.473 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fe1ff6fdd0b8f52-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.474 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.475 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.475 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.476 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.477 DEBUG:boto:Method: GET 2022-09-03T07:28:11.477 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.478 DEBUG:boto:Data: 2022-09-03T07:28:11.479 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.480 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.480 DEBUG:boto:Params: {} 2022-09-03T07:28:11.481 DEBUG:boto:Token: None 2022-09-03T07:28:11.481 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:11.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sVnbkBVaY/qrrals47IY2poY9nE= 2022-09-03T07:28:11.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sVnbkBVaY/qrrals47IY2poY9nE=', 'Content-Length': '0'} 2022-09-03T07:28:11.485 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044e1d122f74caa5e-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.486 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.487 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.487 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:11.488 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:11.488 DEBUG:boto:Method: GET 2022-09-03T07:28:11.489 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:11.489 DEBUG:boto:Data: 2022-09-03T07:28:11.490 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.492 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.492 DEBUG:boto:Params: {} 2022-09-03T07:28:11.493 DEBUG:boto:Token: None 2022-09-03T07:28:11.494 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.498 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d954ce7b3830519d-006313020b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.499 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.500 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.500 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.501 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:11.501 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:11.502 DEBUG:boto:Method: GET 2022-09-03T07:28:11.504 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:11.505 DEBUG:boto:Data: 2022-09-03T07:28:11.506 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.507 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.508 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.509 DEBUG:boto:Params: {} 2022-09-03T07:28:11.510 DEBUG:boto:Token: None 2022-09-03T07:28:11.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.512 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.513 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.517 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006e8240e618de2e14-006313020b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.518 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.519 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.519 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.520 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.520 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:11.521 DEBUG:boto:Data: 2022-09-03T07:28:11.522 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.525 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.525 DEBUG:boto:Params: {} 2022-09-03T07:28:11.526 DEBUG:boto:Token: None 2022-09-03T07:28:11.527 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38 2022-09-03T07:28:11.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:quJRaaF8fihT9az2+1h5q1WTuao= 2022-09-03T07:28:11.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:quJRaaF8fihT9az2+1h5q1WTuao=', 'Content-Length': '0'} 2022-09-03T07:28:11.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000043593a332094376b-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.531 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.531 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.532 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.532 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:11.533 DEBUG:boto:Data: 2022-09-03T07:28:11.534 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.535 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.537 DEBUG:boto:Params: {} 2022-09-03T07:28:11.538 DEBUG:boto:Token: None 2022-09-03T07:28:11.538 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38 2022-09-03T07:28:11.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:quJRaaF8fihT9az2+1h5q1WTuao= 2022-09-03T07:28:11.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:quJRaaF8fihT9az2+1h5q1WTuao=', 'Content-Length': '0'} 2022-09-03T07:28:11.542 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027735e03f075e4eb-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.542 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:28:11.543 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.544 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.544 DEBUG:boto:Method: GET 2022-09-03T07:28:11.545 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:11.545 DEBUG:boto:Data: 2022-09-03T07:28:11.546 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.546 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.547 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.547 DEBUG:boto:Params: {} 2022-09-03T07:28:11.548 DEBUG:boto:Token: None 2022-09-03T07:28:11.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38 2022-09-03T07:28:11.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8xJCvf97FMZ1G2xi+qBewiSQELE= 2022-09-03T07:28:11.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8xJCvf97FMZ1G2xi+qBewiSQELE=', 'Content-Length': '0'} 2022-09-03T07:28:11.554 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016e7fd1855930bba-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.598 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:11.599 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:11.600 DEBUG:boto:Method: GET 2022-09-03T07:28:11.601 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:11.601 DEBUG:boto:Data: 2022-09-03T07:28:11.602 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.604 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.605 DEBUG:boto:Params: {} 2022-09-03T07:28:11.606 DEBUG:boto:Token: None 2022-09-03T07:28:11.607 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj38 2022-09-03T07:28:11.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8xJCvf97FMZ1G2xi+qBewiSQELE= 2022-09-03T07:28:11.610 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8xJCvf97FMZ1G2xi+qBewiSQELE=', 'Content-Length': '0'} 2022-09-03T07:28:11.614 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b12b47c05c555ea-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.616 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.617 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.618 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:11.619 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:11.620 DEBUG:boto:Method: GET 2022-09-03T07:28:11.621 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:11.622 DEBUG:boto:Data: 2022-09-03T07:28:11.623 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.624 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.625 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.626 DEBUG:boto:Params: {} 2022-09-03T07:28:11.627 DEBUG:boto:Token: None 2022-09-03T07:28:11.628 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.635 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009773a203f357480c-006313020b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.636 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.638 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.639 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.640 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:11.641 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:11.642 DEBUG:boto:Method: GET 2022-09-03T07:28:11.643 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:11.644 DEBUG:boto:Data: 2022-09-03T07:28:11.645 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.647 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.648 DEBUG:boto:Params: {} 2022-09-03T07:28:11.649 DEBUG:boto:Token: None 2022-09-03T07:28:11.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.657 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.661 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000541bbfed833d6005-006313020b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.663 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.664 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:28:11.665 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.666 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.666 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.667 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.668 DEBUG:boto:Method: GET 2022-09-03T07:28:11.669 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.670 DEBUG:boto:Data: 2022-09-03T07:28:11.671 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.673 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.673 DEBUG:boto:Params: {} 2022-09-03T07:28:11.675 DEBUG:boto:Token: None 2022-09-03T07:28:11.676 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9KaNvv8TOVw26hSh9myYU30Lbas= 2022-09-03T07:28:11.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9KaNvv8TOVw26hSh9myYU30Lbas=', 'Content-Length': '0'} 2022-09-03T07:28:11.679 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008754ae57f3b3c3cb-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.680 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.681 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.681 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.682 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.682 DEBUG:boto:Method: GET 2022-09-03T07:28:11.683 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.683 DEBUG:boto:Data: 2022-09-03T07:28:11.684 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.685 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.685 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.686 DEBUG:boto:Params: {} 2022-09-03T07:28:11.686 DEBUG:boto:Token: None 2022-09-03T07:28:11.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9KaNvv8TOVw26hSh9myYU30Lbas= 2022-09-03T07:28:11.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9KaNvv8TOVw26hSh9myYU30Lbas=', 'Content-Length': '0'} 2022-09-03T07:28:11.691 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d53e4c44850c8485-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.692 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.692 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.693 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.693 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.694 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.695 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.696 DEBUG:boto:Data: 2022-09-03T07:28:11.696 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.697 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.697 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.698 DEBUG:boto:Params: {} 2022-09-03T07:28:11.699 DEBUG:boto:Token: None 2022-09-03T07:28:11.699 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TrdycS/yXpsgAtXO3NJ3m9m1AwQ= 2022-09-03T07:28:11.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TrdycS/yXpsgAtXO3NJ3m9m1AwQ=', 'Content-Length': '0'} 2022-09-03T07:28:11.703 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c284662dfa926b4f-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.704 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.704 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.705 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.705 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.706 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.706 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.707 DEBUG:boto:Data: 2022-09-03T07:28:11.708 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.708 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.709 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.709 DEBUG:boto:Params: {} 2022-09-03T07:28:11.710 DEBUG:boto:Token: None 2022-09-03T07:28:11.711 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TrdycS/yXpsgAtXO3NJ3m9m1AwQ= 2022-09-03T07:28:11.712 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TrdycS/yXpsgAtXO3NJ3m9m1AwQ=', 'Content-Length': '0'} 2022-09-03T07:28:11.714 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e095e909a198f1bf-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.715 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:28:11.715 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.716 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.717 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.717 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.718 DEBUG:boto:Method: GET 2022-09-03T07:28:11.718 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.719 DEBUG:boto:Data: 2022-09-03T07:28:11.719 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.720 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.721 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.721 DEBUG:boto:Params: {} 2022-09-03T07:28:11.722 DEBUG:boto:Token: None 2022-09-03T07:28:11.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9KaNvv8TOVw26hSh9myYU30Lbas= 2022-09-03T07:28:11.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9KaNvv8TOVw26hSh9myYU30Lbas=', 'Content-Length': '0'} 2022-09-03T07:28:11.726 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c701d37375be9681-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.727 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.727 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.728 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.728 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.730 DEBUG:boto:Method: GET 2022-09-03T07:28:11.730 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.731 DEBUG:boto:Data: 2022-09-03T07:28:11.731 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.732 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.733 DEBUG:boto:Params: {} 2022-09-03T07:28:11.734 DEBUG:boto:Token: None 2022-09-03T07:28:11.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:11.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9KaNvv8TOVw26hSh9myYU30Lbas= 2022-09-03T07:28:11.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9KaNvv8TOVw26hSh9myYU30Lbas=', 'Content-Length': '0'} 2022-09-03T07:28:11.738 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000640889a048e65719-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.784 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.786 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.787 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:11.788 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:11.789 DEBUG:boto:Method: GET 2022-09-03T07:28:11.790 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:11.791 DEBUG:boto:Data: 2022-09-03T07:28:11.792 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.794 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.795 DEBUG:boto:Params: {} 2022-09-03T07:28:11.796 DEBUG:boto:Token: None 2022-09-03T07:28:11.797 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.801 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005921907570a1447c-006313020b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.803 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.805 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.807 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.808 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:11.809 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:11.810 DEBUG:boto:Method: GET 2022-09-03T07:28:11.812 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:11.813 DEBUG:boto:Data: 2022-09-03T07:28:11.815 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.818 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.819 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.820 DEBUG:boto:Params: {} 2022-09-03T07:28:11.821 DEBUG:boto:Token: None 2022-09-03T07:28:11.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.822 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.829 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b788eaf21df783f-006313020b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.829 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.830 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.831 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.832 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.832 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:11.833 DEBUG:boto:Data: 2022-09-03T07:28:11.833 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.834 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.834 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.835 DEBUG:boto:Params: {} 2022-09-03T07:28:11.836 DEBUG:boto:Token: None 2022-09-03T07:28:11.836 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44 2022-09-03T07:28:11.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aSherHSr+f0+VHTFGRbhiwNL/pI= 2022-09-03T07:28:11.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aSherHSr+f0+VHTFGRbhiwNL/pI=', 'Content-Length': '0'} 2022-09-03T07:28:11.844 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ccc26189a81cf0c3-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.846 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.847 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.849 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.850 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:11.850 DEBUG:boto:Data: 2022-09-03T07:28:11.851 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.852 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.852 DEBUG:boto:Params: {} 2022-09-03T07:28:11.853 DEBUG:boto:Token: None 2022-09-03T07:28:11.853 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44 2022-09-03T07:28:11.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aSherHSr+f0+VHTFGRbhiwNL/pI= 2022-09-03T07:28:11.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aSherHSr+f0+VHTFGRbhiwNL/pI=', 'Content-Length': '0'} 2022-09-03T07:28:11.860 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046cc9ab5aa50e82e-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.862 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:28:11.863 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.864 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.864 DEBUG:boto:Method: GET 2022-09-03T07:28:11.865 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:11.865 DEBUG:boto:Data: 2022-09-03T07:28:11.866 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.867 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.868 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.870 DEBUG:boto:Params: {} 2022-09-03T07:28:11.871 DEBUG:boto:Token: None 2022-09-03T07:28:11.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44 2022-09-03T07:28:11.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l84dT0MooeXEsVgo9ZhfD+iU7MI= 2022-09-03T07:28:11.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l84dT0MooeXEsVgo9ZhfD+iU7MI=', 'Content-Length': '0'} 2022-09-03T07:28:11.875 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000073099abaaaf5e7a6-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.918 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:11.920 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:11.922 DEBUG:boto:Method: GET 2022-09-03T07:28:11.923 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:11.924 DEBUG:boto:Data: 2022-09-03T07:28:11.925 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.926 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.927 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.928 DEBUG:boto:Params: {} 2022-09-03T07:28:11.928 DEBUG:boto:Token: None 2022-09-03T07:28:11.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj44 2022-09-03T07:28:11.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l84dT0MooeXEsVgo9ZhfD+iU7MI= 2022-09-03T07:28:11.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l84dT0MooeXEsVgo9ZhfD+iU7MI=', 'Content-Length': '0'} 2022-09-03T07:28:11.933 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098c80e4389231244-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.934 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.935 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.935 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:11.936 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:11.936 DEBUG:boto:Method: GET 2022-09-03T07:28:11.937 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:11.937 DEBUG:boto:Data: 2022-09-03T07:28:11.938 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.938 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.939 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.939 DEBUG:boto:Params: {} 2022-09-03T07:28:11.940 DEBUG:boto:Token: None 2022-09-03T07:28:11.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.946 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000806aa903b354781-006313020b-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.947 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.948 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:11.949 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:11.950 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:11.951 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:11.951 DEBUG:boto:Method: GET 2022-09-03T07:28:11.952 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:11.952 DEBUG:boto:Data: 2022-09-03T07:28:11.953 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.954 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.955 DEBUG:boto:Params: {} 2022-09-03T07:28:11.955 DEBUG:boto:Token: None 2022-09-03T07:28:11.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/ 2022-09-03T07:28:11.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU= 2022-09-03T07:28:11.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ozedtAbheUG7jimRkccfdn4VMzU=', 'Content-Length': '0'} 2022-09-03T07:28:11.962 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c83e5c5acb4c5f8b-006313020b-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.963 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:11.968 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:28:11.970 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:11.971 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:11.971 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.972 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.973 DEBUG:boto:Method: GET 2022-09-03T07:28:11.973 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.974 DEBUG:boto:Data: 2022-09-03T07:28:11.974 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.975 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.975 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.976 DEBUG:boto:Params: {} 2022-09-03T07:28:11.976 DEBUG:boto:Token: None 2022-09-03T07:28:11.977 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qr7HtnB8+X0O3xt7mTwVpY7YFTM= 2022-09-03T07:28:11.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qr7HtnB8+X0O3xt7mTwVpY7YFTM=', 'Content-Length': '0'} 2022-09-03T07:28:11.980 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081ff3c75afb9eb59-006313020b-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.981 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:11.982 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:11.982 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.983 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.983 DEBUG:boto:Method: GET 2022-09-03T07:28:11.984 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.984 DEBUG:boto:Data: 2022-09-03T07:28:11.985 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:11.986 DEBUG:boto:Port: 8000 2022-09-03T07:28:11.986 DEBUG:boto:Params: {} 2022-09-03T07:28:11.987 DEBUG:boto:Token: None 2022-09-03T07:28:11.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qr7HtnB8+X0O3xt7mTwVpY7YFTM= 2022-09-03T07:28:11.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qr7HtnB8+X0O3xt7mTwVpY7YFTM=', 'Content-Length': '0'} 2022-09-03T07:28:11.991 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024a7309e26359090-006313020b-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:11.992 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:11.993 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:11.993 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.994 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.994 DEBUG:boto:Method: HEAD 2022-09-03T07:28:11.995 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.995 DEBUG:boto:Data: 2022-09-03T07:28:11.996 DEBUG:boto:Headers: {} 2022-09-03T07:28:11.996 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:11.997 DEBUG:boto:Port: 8001 2022-09-03T07:28:11.997 DEBUG:boto:Params: {} 2022-09-03T07:28:11.998 DEBUG:boto:Token: None 2022-09-03T07:28:11.998 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:11 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:11.999 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3EXVsGrlosoUE+nxq14zt1lDKdk= 2022-09-03T07:28:12.000 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3EXVsGrlosoUE+nxq14zt1lDKdk=', 'Content-Length': '0'} 2022-09-03T07:28:12.002 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004771ba39592c4f5a-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.002 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:12.003 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:12.003 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.004 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.004 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.005 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.005 DEBUG:boto:Data: 2022-09-03T07:28:12.006 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.006 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.007 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.007 DEBUG:boto:Params: {} 2022-09-03T07:28:12.008 DEBUG:boto:Token: None 2022-09-03T07:28:12.008 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4uF28qRuO3G/e702K2pcUc80rUQ= 2022-09-03T07:28:12.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4uF28qRuO3G/e702K2pcUc80rUQ=', 'Content-Length': '0'} 2022-09-03T07:28:12.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e22d38ac85310ce-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.012 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:28:12.013 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:12.013 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:12.014 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.014 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.015 DEBUG:boto:Method: GET 2022-09-03T07:28:12.016 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.016 DEBUG:boto:Data: 2022-09-03T07:28:12.017 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.018 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.019 DEBUG:boto:Params: {} 2022-09-03T07:28:12.019 DEBUG:boto:Token: None 2022-09-03T07:28:12.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WA6CDHSDdi6q7H57QF/jgpHyDRc= 2022-09-03T07:28:12.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WA6CDHSDdi6q7H57QF/jgpHyDRc=', 'Content-Length': '0'} 2022-09-03T07:28:12.023 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000629c0d615bf62e6d-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.024 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:12.024 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:12.025 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.025 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.026 DEBUG:boto:Method: GET 2022-09-03T07:28:12.026 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.027 DEBUG:boto:Data: 2022-09-03T07:28:12.027 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.028 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.028 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.029 DEBUG:boto:Params: {} 2022-09-03T07:28:12.030 DEBUG:boto:Token: None 2022-09-03T07:28:12.030 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:12.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WA6CDHSDdi6q7H57QF/jgpHyDRc= 2022-09-03T07:28:12.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WA6CDHSDdi6q7H57QF/jgpHyDRc=', 'Content-Length': '0'} 2022-09-03T07:28:12.033 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034c631e12cd14777-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.034 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.035 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.035 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:12.036 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:12.036 DEBUG:boto:Method: GET 2022-09-03T07:28:12.037 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:12.037 DEBUG:boto:Data: 2022-09-03T07:28:12.038 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.039 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.039 DEBUG:boto:Params: {} 2022-09-03T07:28:12.040 DEBUG:boto:Token: None 2022-09-03T07:28:12.041 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.045 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f2f98d53ef78a1a5-006313020c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.045 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.046 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.047 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.048 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:12.048 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:12.049 DEBUG:boto:Method: GET 2022-09-03T07:28:12.049 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:12.050 DEBUG:boto:Data: 2022-09-03T07:28:12.050 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.051 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.052 DEBUG:boto:Params: {} 2022-09-03T07:28:12.052 DEBUG:boto:Token: None 2022-09-03T07:28:12.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.058 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fedfcd4032bd2db7-006313020c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.059 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.060 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:12.060 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:12.061 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.062 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:12.062 DEBUG:boto:Data: 2022-09-03T07:28:12.063 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.064 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.064 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.065 DEBUG:boto:Params: {} 2022-09-03T07:28:12.066 DEBUG:boto:Token: None 2022-09-03T07:28:12.066 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj46 2022-09-03T07:28:12.067 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VObS2Me5zN9MBoPJI+SsHUN6lJQ= 2022-09-03T07:28:12.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VObS2Me5zN9MBoPJI+SsHUN6lJQ=', 'Content-Length': '0'} 2022-09-03T07:28:12.069 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000135fb334925e0032-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.070 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:12.071 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:12.071 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.072 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:12.072 DEBUG:boto:Data: 2022-09-03T07:28:12.076 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.078 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.078 DEBUG:boto:Params: {} 2022-09-03T07:28:12.080 DEBUG:boto:Token: None 2022-09-03T07:28:12.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj46 2022-09-03T07:28:12.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VObS2Me5zN9MBoPJI+SsHUN6lJQ= 2022-09-03T07:28:12.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VObS2Me5zN9MBoPJI+SsHUN6lJQ=', 'Content-Length': '0'} 2022-09-03T07:28:12.087 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f19ba0a07a3259db-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.088 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:28:12.089 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:12.089 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:12.090 DEBUG:boto:Method: GET 2022-09-03T07:28:12.090 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:12.091 DEBUG:boto:Data: 2022-09-03T07:28:12.091 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.092 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.093 DEBUG:boto:Params: {} 2022-09-03T07:28:12.094 DEBUG:boto:Token: None 2022-09-03T07:28:12.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj46 2022-09-03T07:28:12.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZKISljwPSVM+yiKKmziUn9D82k= 2022-09-03T07:28:12.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZKISljwPSVM+yiKKmziUn9D82k=', 'Content-Length': '0'} 2022-09-03T07:28:12.098 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003274ed6ceb4c1f8f-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.099 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:12.100 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:12.101 DEBUG:boto:Method: GET 2022-09-03T07:28:12.102 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:12.102 DEBUG:boto:Data: 2022-09-03T07:28:12.103 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.104 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.104 DEBUG:boto:Params: {} 2022-09-03T07:28:12.105 DEBUG:boto:Token: None 2022-09-03T07:28:12.106 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj46 2022-09-03T07:28:12.107 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6ZKISljwPSVM+yiKKmziUn9D82k= 2022-09-03T07:28:12.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6ZKISljwPSVM+yiKKmziUn9D82k=', 'Content-Length': '0'} 2022-09-03T07:28:12.110 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003decc18c0d7b9aee-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.150 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.151 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.152 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:12.153 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:12.153 DEBUG:boto:Method: GET 2022-09-03T07:28:12.154 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:12.154 DEBUG:boto:Data: 2022-09-03T07:28:12.155 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.155 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.156 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.156 DEBUG:boto:Params: {} 2022-09-03T07:28:12.157 DEBUG:boto:Token: None 2022-09-03T07:28:12.158 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.161 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000658bb59c5853037e-006313020c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.162 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.163 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.164 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.164 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:12.165 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:12.165 DEBUG:boto:Method: GET 2022-09-03T07:28:12.166 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:12.166 DEBUG:boto:Data: 2022-09-03T07:28:12.167 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.168 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.168 DEBUG:boto:Params: {} 2022-09-03T07:28:12.169 DEBUG:boto:Token: None 2022-09-03T07:28:12.169 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.176 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009fc55c42ad4563cb-006313020c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.177 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.178 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:28:12.179 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.180 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.180 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.181 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.181 DEBUG:boto:Method: GET 2022-09-03T07:28:12.182 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.183 DEBUG:boto:Data: 2022-09-03T07:28:12.184 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.185 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.186 DEBUG:boto:Params: {} 2022-09-03T07:28:12.187 DEBUG:boto:Token: None 2022-09-03T07:28:12.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FumKbwq+kD3zHpcqKqe3ZBS6ShI= 2022-09-03T07:28:12.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FumKbwq+kD3zHpcqKqe3ZBS6ShI=', 'Content-Length': '0'} 2022-09-03T07:28:12.196 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097eb506a542e25a8-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.197 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.197 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.198 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.199 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.199 DEBUG:boto:Method: GET 2022-09-03T07:28:12.200 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.201 DEBUG:boto:Data: 2022-09-03T07:28:12.201 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.203 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.204 DEBUG:boto:Params: {} 2022-09-03T07:28:12.205 DEBUG:boto:Token: None 2022-09-03T07:28:12.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FumKbwq+kD3zHpcqKqe3ZBS6ShI= 2022-09-03T07:28:12.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FumKbwq+kD3zHpcqKqe3ZBS6ShI=', 'Content-Length': '0'} 2022-09-03T07:28:12.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1a12f5c33d77e2a-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.210 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.211 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.211 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.212 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.213 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.214 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.214 DEBUG:boto:Data: 2022-09-03T07:28:12.215 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.216 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.216 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.217 DEBUG:boto:Params: {} 2022-09-03T07:28:12.219 DEBUG:boto:Token: None 2022-09-03T07:28:12.220 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uDaxwW+ortDi9HdiWG0r3Ut/0u0= 2022-09-03T07:28:12.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uDaxwW+ortDi9HdiWG0r3Ut/0u0=', 'Content-Length': '0'} 2022-09-03T07:28:12.223 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007243ef5bcb21020b-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.224 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.225 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.225 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.226 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.226 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.227 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.228 DEBUG:boto:Data: 2022-09-03T07:28:12.228 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.229 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.229 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.230 DEBUG:boto:Params: {} 2022-09-03T07:28:12.231 DEBUG:boto:Token: None 2022-09-03T07:28:12.231 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uDaxwW+ortDi9HdiWG0r3Ut/0u0= 2022-09-03T07:28:12.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uDaxwW+ortDi9HdiWG0r3Ut/0u0=', 'Content-Length': '0'} 2022-09-03T07:28:12.235 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e5650ce904dba04-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.236 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:28:12.236 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.237 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.238 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.238 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.239 DEBUG:boto:Method: GET 2022-09-03T07:28:12.239 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.240 DEBUG:boto:Data: 2022-09-03T07:28:12.241 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.241 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.242 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.242 DEBUG:boto:Params: {} 2022-09-03T07:28:12.243 DEBUG:boto:Token: None 2022-09-03T07:28:12.244 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FumKbwq+kD3zHpcqKqe3ZBS6ShI= 2022-09-03T07:28:12.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FumKbwq+kD3zHpcqKqe3ZBS6ShI=', 'Content-Length': '0'} 2022-09-03T07:28:12.249 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090980e269b342a7c-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.250 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.250 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.252 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.253 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.254 DEBUG:boto:Method: GET 2022-09-03T07:28:12.255 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.258 DEBUG:boto:Data: 2022-09-03T07:28:12.259 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.261 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.262 DEBUG:boto:Params: {} 2022-09-03T07:28:12.263 DEBUG:boto:Token: None 2022-09-03T07:28:12.264 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:12.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FumKbwq+kD3zHpcqKqe3ZBS6ShI= 2022-09-03T07:28:12.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FumKbwq+kD3zHpcqKqe3ZBS6ShI=', 'Content-Length': '0'} 2022-09-03T07:28:12.268 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000daddfbd35c0b76a0-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.269 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.270 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.270 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:12.271 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:12.272 DEBUG:boto:Method: GET 2022-09-03T07:28:12.272 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:12.273 DEBUG:boto:Data: 2022-09-03T07:28:12.274 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.275 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.276 DEBUG:boto:Params: {} 2022-09-03T07:28:12.276 DEBUG:boto:Token: None 2022-09-03T07:28:12.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.282 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b7aecf618cc69a8-006313020c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.283 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.284 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.285 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.285 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:12.286 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:12.286 DEBUG:boto:Method: GET 2022-09-03T07:28:12.287 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:12.288 DEBUG:boto:Data: 2022-09-03T07:28:12.289 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.290 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.291 DEBUG:boto:Params: {} 2022-09-03T07:28:12.292 DEBUG:boto:Token: None 2022-09-03T07:28:12.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.295 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.300 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000045fd082cb65535b-006313020c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.301 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.303 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.303 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.304 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.305 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:12.306 DEBUG:boto:Data: 2022-09-03T07:28:12.307 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.307 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.308 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.310 DEBUG:boto:Params: {} 2022-09-03T07:28:12.311 DEBUG:boto:Token: None 2022-09-03T07:28:12.312 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51 2022-09-03T07:28:12.313 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EfQb8a6ljz/NktFOEcIwwz7ax+M= 2022-09-03T07:28:12.314 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EfQb8a6ljz/NktFOEcIwwz7ax+M=', 'Content-Length': '0'} 2022-09-03T07:28:12.317 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc353aacc46c9d50-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.318 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.319 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.319 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.320 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:12.321 DEBUG:boto:Data: 2022-09-03T07:28:12.321 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.322 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.322 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.323 DEBUG:boto:Params: {} 2022-09-03T07:28:12.324 DEBUG:boto:Token: None 2022-09-03T07:28:12.324 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51 2022-09-03T07:28:12.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EfQb8a6ljz/NktFOEcIwwz7ax+M= 2022-09-03T07:28:12.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EfQb8a6ljz/NktFOEcIwwz7ax+M=', 'Content-Length': '0'} 2022-09-03T07:28:12.328 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f59c441bf61455b-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.329 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:28:12.330 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.331 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.332 DEBUG:boto:Method: GET 2022-09-03T07:28:12.332 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:12.333 DEBUG:boto:Data: 2022-09-03T07:28:12.333 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.335 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.335 DEBUG:boto:Params: {} 2022-09-03T07:28:12.337 DEBUG:boto:Token: None 2022-09-03T07:28:12.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51 2022-09-03T07:28:12.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WG4qryGtAu4MwmtdiV9jnoEoqQ8= 2022-09-03T07:28:12.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WG4qryGtAu4MwmtdiV9jnoEoqQ8=', 'Content-Length': '0'} 2022-09-03T07:28:12.345 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b201104effa9e79c-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.346 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:12.347 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:12.348 DEBUG:boto:Method: GET 2022-09-03T07:28:12.349 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:12.349 DEBUG:boto:Data: 2022-09-03T07:28:12.350 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.351 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.352 DEBUG:boto:Params: {} 2022-09-03T07:28:12.352 DEBUG:boto:Token: None 2022-09-03T07:28:12.353 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj51 2022-09-03T07:28:12.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WG4qryGtAu4MwmtdiV9jnoEoqQ8= 2022-09-03T07:28:12.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WG4qryGtAu4MwmtdiV9jnoEoqQ8=', 'Content-Length': '0'} 2022-09-03T07:28:12.357 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000349fa66be850187e-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.398 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.401 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.402 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:12.403 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:12.404 DEBUG:boto:Method: GET 2022-09-03T07:28:12.405 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:12.406 DEBUG:boto:Data: 2022-09-03T07:28:12.407 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.408 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.409 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.410 DEBUG:boto:Params: {} 2022-09-03T07:28:12.412 DEBUG:boto:Token: None 2022-09-03T07:28:12.413 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.414 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.420 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c8e73990654ef3d5-006313020c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.422 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.423 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.424 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.425 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:12.426 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:12.427 DEBUG:boto:Method: GET 2022-09-03T07:28:12.429 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:12.429 DEBUG:boto:Data: 2022-09-03T07:28:12.430 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.431 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.431 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.432 DEBUG:boto:Params: {} 2022-09-03T07:28:12.432 DEBUG:boto:Token: None 2022-09-03T07:28:12.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.440 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090eba8027b1a386a-006313020c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.440 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.442 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:28:12.443 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.443 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.444 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.445 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.446 DEBUG:boto:Method: GET 2022-09-03T07:28:12.447 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.447 DEBUG:boto:Data: 2022-09-03T07:28:12.449 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.451 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.451 DEBUG:boto:Params: {} 2022-09-03T07:28:12.453 DEBUG:boto:Token: None 2022-09-03T07:28:12.454 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UKVfbhCXMeh16u9N4pgkaZLm3Dk= 2022-09-03T07:28:12.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UKVfbhCXMeh16u9N4pgkaZLm3Dk=', 'Content-Length': '0'} 2022-09-03T07:28:12.458 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008363ffb6a6c4a091-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.459 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.460 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.461 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.462 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.463 DEBUG:boto:Method: GET 2022-09-03T07:28:12.464 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.465 DEBUG:boto:Data: 2022-09-03T07:28:12.466 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.468 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.469 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.470 DEBUG:boto:Params: {} 2022-09-03T07:28:12.471 DEBUG:boto:Token: None 2022-09-03T07:28:12.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UKVfbhCXMeh16u9N4pgkaZLm3Dk= 2022-09-03T07:28:12.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UKVfbhCXMeh16u9N4pgkaZLm3Dk=', 'Content-Length': '0'} 2022-09-03T07:28:12.477 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000725020988bd93314-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.482 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.484 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.485 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.486 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.487 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.488 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.488 DEBUG:boto:Data: 2022-09-03T07:28:12.489 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.491 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.492 DEBUG:boto:Params: {} 2022-09-03T07:28:12.493 DEBUG:boto:Token: None 2022-09-03T07:28:12.494 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sovkqz7o4cp8Ws01KUAMopmHZ68= 2022-09-03T07:28:12.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sovkqz7o4cp8Ws01KUAMopmHZ68=', 'Content-Length': '0'} 2022-09-03T07:28:12.497 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bea190a7c7c4ce0-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.498 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.499 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.499 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.500 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.500 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.501 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.501 DEBUG:boto:Data: 2022-09-03T07:28:12.502 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.503 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.503 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.504 DEBUG:boto:Params: {} 2022-09-03T07:28:12.505 DEBUG:boto:Token: None 2022-09-03T07:28:12.506 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sovkqz7o4cp8Ws01KUAMopmHZ68= 2022-09-03T07:28:12.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sovkqz7o4cp8Ws01KUAMopmHZ68=', 'Content-Length': '0'} 2022-09-03T07:28:12.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000006c23a39d7f089b3-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.512 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:28:12.513 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.514 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.515 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.516 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.517 DEBUG:boto:Method: GET 2022-09-03T07:28:12.518 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.519 DEBUG:boto:Data: 2022-09-03T07:28:12.520 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.521 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.521 DEBUG:boto:Params: {} 2022-09-03T07:28:12.522 DEBUG:boto:Token: None 2022-09-03T07:28:12.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UKVfbhCXMeh16u9N4pgkaZLm3Dk= 2022-09-03T07:28:12.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UKVfbhCXMeh16u9N4pgkaZLm3Dk=', 'Content-Length': '0'} 2022-09-03T07:28:12.528 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000033bd69ae61d5904f-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.530 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.531 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.532 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.533 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.534 DEBUG:boto:Method: GET 2022-09-03T07:28:12.535 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.536 DEBUG:boto:Data: 2022-09-03T07:28:12.537 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.548 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.550 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.551 DEBUG:boto:Params: {} 2022-09-03T07:28:12.552 DEBUG:boto:Token: None 2022-09-03T07:28:12.553 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:12.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UKVfbhCXMeh16u9N4pgkaZLm3Dk= 2022-09-03T07:28:12.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UKVfbhCXMeh16u9N4pgkaZLm3Dk=', 'Content-Length': '0'} 2022-09-03T07:28:12.558 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ccf01d589c7027c8-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.598 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.599 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.600 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:12.600 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:12.601 DEBUG:boto:Method: GET 2022-09-03T07:28:12.601 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:12.602 DEBUG:boto:Data: 2022-09-03T07:28:12.603 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.604 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.604 DEBUG:boto:Params: {} 2022-09-03T07:28:12.605 DEBUG:boto:Token: None 2022-09-03T07:28:12.606 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.612 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003d8f08a4fcc5b122-006313020c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.613 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.614 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.615 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.616 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:12.616 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:12.617 DEBUG:boto:Method: GET 2022-09-03T07:28:12.618 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:12.620 DEBUG:boto:Data: 2022-09-03T07:28:12.620 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.621 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.621 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.622 DEBUG:boto:Params: {} 2022-09-03T07:28:12.622 DEBUG:boto:Token: None 2022-09-03T07:28:12.623 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.629 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000049713fb9ae59aa0c-006313020c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.630 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.632 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.633 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.633 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.634 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:12.634 DEBUG:boto:Data: 2022-09-03T07:28:12.635 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.637 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.637 DEBUG:boto:Params: {} 2022-09-03T07:28:12.638 DEBUG:boto:Token: None 2022-09-03T07:28:12.639 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55 2022-09-03T07:28:12.640 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8WltekqEs/JiB9eaDabQyo/b4HQ= 2022-09-03T07:28:12.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8WltekqEs/JiB9eaDabQyo/b4HQ=', 'Content-Length': '0'} 2022-09-03T07:28:12.643 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc935284a5d35c30-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.644 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.644 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.645 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.645 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:12.647 DEBUG:boto:Data: 2022-09-03T07:28:12.648 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.648 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.649 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.650 DEBUG:boto:Params: {} 2022-09-03T07:28:12.650 DEBUG:boto:Token: None 2022-09-03T07:28:12.651 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55 2022-09-03T07:28:12.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8WltekqEs/JiB9eaDabQyo/b4HQ= 2022-09-03T07:28:12.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8WltekqEs/JiB9eaDabQyo/b4HQ=', 'Content-Length': '0'} 2022-09-03T07:28:12.657 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e815b1f4b0594e5e-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.659 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:28:12.660 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.661 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.662 DEBUG:boto:Method: GET 2022-09-03T07:28:12.663 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:12.664 DEBUG:boto:Data: 2022-09-03T07:28:12.665 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.668 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.669 DEBUG:boto:Params: {} 2022-09-03T07:28:12.670 DEBUG:boto:Token: None 2022-09-03T07:28:12.671 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55 2022-09-03T07:28:12.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u3r1IjSfUE8h3ShsLjMYebucZog= 2022-09-03T07:28:12.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u3r1IjSfUE8h3ShsLjMYebucZog=', 'Content-Length': '0'} 2022-09-03T07:28:12.676 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7ba7ea18ab808d4-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.719 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:12.719 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:12.720 DEBUG:boto:Method: GET 2022-09-03T07:28:12.720 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:12.721 DEBUG:boto:Data: 2022-09-03T07:28:12.721 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.722 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.723 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.723 DEBUG:boto:Params: {} 2022-09-03T07:28:12.724 DEBUG:boto:Token: None 2022-09-03T07:28:12.725 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj55 2022-09-03T07:28:12.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u3r1IjSfUE8h3ShsLjMYebucZog= 2022-09-03T07:28:12.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u3r1IjSfUE8h3ShsLjMYebucZog=', 'Content-Length': '0'} 2022-09-03T07:28:12.728 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e9dc09cf32a5506-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.729 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.730 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.730 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:12.731 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:12.732 DEBUG:boto:Method: GET 2022-09-03T07:28:12.732 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:12.733 DEBUG:boto:Data: 2022-09-03T07:28:12.733 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.734 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.735 DEBUG:boto:Params: {} 2022-09-03T07:28:12.735 DEBUG:boto:Token: None 2022-09-03T07:28:12.736 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.740 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006e14963b5ad598db-006313020c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.742 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.743 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.744 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.745 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:12.746 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:12.746 DEBUG:boto:Method: GET 2022-09-03T07:28:12.747 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:12.748 DEBUG:boto:Data: 2022-09-03T07:28:12.749 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.752 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.753 DEBUG:boto:Params: {} 2022-09-03T07:28:12.753 DEBUG:boto:Token: None 2022-09-03T07:28:12.754 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.755 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.756 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.761 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0f7dd885e05539c-006313020c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.762 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.764 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:28:12.765 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:12.766 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:12.767 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.768 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.769 DEBUG:boto:Method: GET 2022-09-03T07:28:12.770 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.770 DEBUG:boto:Data: 2022-09-03T07:28:12.771 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.772 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.772 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.773 DEBUG:boto:Params: {} 2022-09-03T07:28:12.773 DEBUG:boto:Token: None 2022-09-03T07:28:12.774 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OBuUWfb9xK39fET8NetRhzZ0nlE= 2022-09-03T07:28:12.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OBuUWfb9xK39fET8NetRhzZ0nlE=', 'Content-Length': '0'} 2022-09-03T07:28:12.779 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d14254896e723361-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.781 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:12.781 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:12.782 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.785 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.786 DEBUG:boto:Method: GET 2022-09-03T07:28:12.787 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.788 DEBUG:boto:Data: 2022-09-03T07:28:12.789 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.790 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.791 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.792 DEBUG:boto:Params: {} 2022-09-03T07:28:12.793 DEBUG:boto:Token: None 2022-09-03T07:28:12.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OBuUWfb9xK39fET8NetRhzZ0nlE= 2022-09-03T07:28:12.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OBuUWfb9xK39fET8NetRhzZ0nlE=', 'Content-Length': '0'} 2022-09-03T07:28:12.798 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004e2ca28f701b183-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.846 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:12.847 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:12.847 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.848 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.849 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.850 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.850 DEBUG:boto:Data: 2022-09-03T07:28:12.851 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.852 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.852 DEBUG:boto:Params: {} 2022-09-03T07:28:12.853 DEBUG:boto:Token: None 2022-09-03T07:28:12.854 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BTZ6myCZ/Om6TMKvlMz4rRelfQ8= 2022-09-03T07:28:12.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BTZ6myCZ/Om6TMKvlMz4rRelfQ8=', 'Content-Length': '0'} 2022-09-03T07:28:12.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b66e1a6413579538-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.859 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:12.860 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:12.861 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.861 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.862 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.863 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.863 DEBUG:boto:Data: 2022-09-03T07:28:12.864 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.866 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.867 DEBUG:boto:Params: {} 2022-09-03T07:28:12.868 DEBUG:boto:Token: None 2022-09-03T07:28:12.869 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BTZ6myCZ/Om6TMKvlMz4rRelfQ8= 2022-09-03T07:28:12.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BTZ6myCZ/Om6TMKvlMz4rRelfQ8=', 'Content-Length': '0'} 2022-09-03T07:28:12.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f11080632090d7d-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.875 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:28:12.875 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:12.876 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:12.877 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.877 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.878 DEBUG:boto:Method: GET 2022-09-03T07:28:12.879 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.879 DEBUG:boto:Data: 2022-09-03T07:28:12.880 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.883 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.884 DEBUG:boto:Params: {} 2022-09-03T07:28:12.886 DEBUG:boto:Token: None 2022-09-03T07:28:12.887 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OBuUWfb9xK39fET8NetRhzZ0nlE= 2022-09-03T07:28:12.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OBuUWfb9xK39fET8NetRhzZ0nlE=', 'Content-Length': '0'} 2022-09-03T07:28:12.890 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004198235bd01354d3-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.891 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:12.891 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:12.892 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.892 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.893 DEBUG:boto:Method: GET 2022-09-03T07:28:12.893 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.894 DEBUG:boto:Data: 2022-09-03T07:28:12.895 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.895 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.896 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.896 DEBUG:boto:Params: {} 2022-09-03T07:28:12.897 DEBUG:boto:Token: None 2022-09-03T07:28:12.897 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:12.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OBuUWfb9xK39fET8NetRhzZ0nlE= 2022-09-03T07:28:12.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OBuUWfb9xK39fET8NetRhzZ0nlE=', 'Content-Length': '0'} 2022-09-03T07:28:12.901 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045d0f2361a58f3d1-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.901 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.902 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.903 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:12.903 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:12.904 DEBUG:boto:Method: GET 2022-09-03T07:28:12.904 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:12.905 DEBUG:boto:Data: 2022-09-03T07:28:12.905 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.906 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.907 DEBUG:boto:Params: {} 2022-09-03T07:28:12.910 DEBUG:boto:Token: None 2022-09-03T07:28:12.911 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.912 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.913 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.917 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000348274556f3e1a5f-006313020c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.918 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.919 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:12.920 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:12.921 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:12.922 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:12.923 DEBUG:boto:Method: GET 2022-09-03T07:28:12.924 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:12.925 DEBUG:boto:Data: 2022-09-03T07:28:12.926 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.928 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.930 DEBUG:boto:Params: {} 2022-09-03T07:28:12.931 DEBUG:boto:Token: None 2022-09-03T07:28:12.933 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/ 2022-09-03T07:28:12.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0= 2022-09-03T07:28:12.944 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jFRxYmQ9BH5qDlOXFBhPQAzliU0=', 'Content-Length': '0'} 2022-09-03T07:28:12.949 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002f25ae4de917838-006313020c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.950 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:12.951 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:12.951 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:12.952 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.953 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:12.954 DEBUG:boto:Data: 2022-09-03T07:28:12.955 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.956 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.957 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.958 DEBUG:boto:Params: {} 2022-09-03T07:28:12.959 DEBUG:boto:Token: None 2022-09-03T07:28:12.961 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj57 2022-09-03T07:28:12.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iZV2BHdNZASLcdJlfSFBmkZsUKg= 2022-09-03T07:28:12.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iZV2BHdNZASLcdJlfSFBmkZsUKg=', 'Content-Length': '0'} 2022-09-03T07:28:12.974 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ea924edcaca963fb-006313020c-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.976 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:12.977 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:12.978 DEBUG:boto:Method: HEAD 2022-09-03T07:28:12.979 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:12.980 DEBUG:boto:Data: 2022-09-03T07:28:12.981 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.982 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:12.983 DEBUG:boto:Port: 8000 2022-09-03T07:28:12.984 DEBUG:boto:Params: {} 2022-09-03T07:28:12.984 DEBUG:boto:Token: None 2022-09-03T07:28:12.985 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj57 2022-09-03T07:28:12.986 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iZV2BHdNZASLcdJlfSFBmkZsUKg= 2022-09-03T07:28:12.986 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iZV2BHdNZASLcdJlfSFBmkZsUKg=', 'Content-Length': '0'} 2022-09-03T07:28:12.990 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032f31f88a36aae83-006313020c-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:12.991 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:28:12.991 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:12.992 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:12.993 DEBUG:boto:Method: GET 2022-09-03T07:28:12.993 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:12.994 DEBUG:boto:Data: 2022-09-03T07:28:12.994 DEBUG:boto:Headers: {} 2022-09-03T07:28:12.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:12.995 DEBUG:boto:Port: 8001 2022-09-03T07:28:12.996 DEBUG:boto:Params: {} 2022-09-03T07:28:12.996 DEBUG:boto:Token: None 2022-09-03T07:28:12.997 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:12 GMT /avysmw-75/obj57 2022-09-03T07:28:12.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w+7K5eaXakwlVB+u1nMBYfSIBT0= 2022-09-03T07:28:12.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w+7K5eaXakwlVB+u1nMBYfSIBT0=', 'Content-Length': '0'} 2022-09-03T07:28:13.001 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff05bb28a671ec93-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.046 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:13.047 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:13.047 DEBUG:boto:Method: GET 2022-09-03T07:28:13.048 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:13.048 DEBUG:boto:Data: 2022-09-03T07:28:13.049 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.050 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.051 DEBUG:boto:Params: {} 2022-09-03T07:28:13.051 DEBUG:boto:Token: None 2022-09-03T07:28:13.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj57 2022-09-03T07:28:13.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:slxGrNRsmnQeB0xrcsqvYsDdtIs= 2022-09-03T07:28:13.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:slxGrNRsmnQeB0xrcsqvYsDdtIs=', 'Content-Length': '0'} 2022-09-03T07:28:13.057 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b5f3a4c445c6dc0-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.058 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.059 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.059 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:13.060 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:13.060 DEBUG:boto:Method: GET 2022-09-03T07:28:13.061 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:13.061 DEBUG:boto:Data: 2022-09-03T07:28:13.062 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.063 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.064 DEBUG:boto:Params: {} 2022-09-03T07:28:13.065 DEBUG:boto:Token: None 2022-09-03T07:28:13.065 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.066 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.067 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.073 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003635b45026cbca06-006313020d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.074 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.079 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.080 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.081 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:13.082 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:13.083 DEBUG:boto:Method: GET 2022-09-03T07:28:13.084 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:13.086 DEBUG:boto:Data: 2022-09-03T07:28:13.087 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.087 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.089 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.093 DEBUG:boto:Params: {} 2022-09-03T07:28:13.093 DEBUG:boto:Token: None 2022-09-03T07:28:13.094 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.100 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000060f7d2a958710b8e-006313020d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.101 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.101 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:28:13.102 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.103 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.103 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.104 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.104 DEBUG:boto:Method: GET 2022-09-03T07:28:13.105 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.105 DEBUG:boto:Data: 2022-09-03T07:28:13.106 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.107 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.108 DEBUG:boto:Params: {} 2022-09-03T07:28:13.108 DEBUG:boto:Token: None 2022-09-03T07:28:13.109 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.110 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UcAItoWmmhJDEbPRSBA36xVi8ng= 2022-09-03T07:28:13.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UcAItoWmmhJDEbPRSBA36xVi8ng=', 'Content-Length': '0'} 2022-09-03T07:28:13.113 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022c800d9e6403257-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.116 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.117 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.118 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.120 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.121 DEBUG:boto:Method: GET 2022-09-03T07:28:13.122 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.124 DEBUG:boto:Data: 2022-09-03T07:28:13.125 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.126 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.127 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.128 DEBUG:boto:Params: {} 2022-09-03T07:28:13.129 DEBUG:boto:Token: None 2022-09-03T07:28:13.130 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UcAItoWmmhJDEbPRSBA36xVi8ng= 2022-09-03T07:28:13.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UcAItoWmmhJDEbPRSBA36xVi8ng=', 'Content-Length': '0'} 2022-09-03T07:28:13.135 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b144e68affe68833-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.136 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.137 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.137 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.139 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.140 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.141 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.142 DEBUG:boto:Data: 2022-09-03T07:28:13.143 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.144 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.145 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.146 DEBUG:boto:Params: {} 2022-09-03T07:28:13.148 DEBUG:boto:Token: None 2022-09-03T07:28:13.149 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m/AuX17CMFv7hsvLa7d9D0XN5k= 2022-09-03T07:28:13.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m/AuX17CMFv7hsvLa7d9D0XN5k=', 'Content-Length': '0'} 2022-09-03T07:28:13.153 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1940dfbb7f7143f-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.154 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.155 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.156 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.157 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.158 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.159 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.161 DEBUG:boto:Data: 2022-09-03T07:28:13.162 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.164 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.164 DEBUG:boto:Params: {} 2022-09-03T07:28:13.165 DEBUG:boto:Token: None 2022-09-03T07:28:13.165 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6m/AuX17CMFv7hsvLa7d9D0XN5k= 2022-09-03T07:28:13.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6m/AuX17CMFv7hsvLa7d9D0XN5k=', 'Content-Length': '0'} 2022-09-03T07:28:13.174 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d98504bcf1e3853d-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.175 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:28:13.176 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.176 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.177 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.177 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.178 DEBUG:boto:Method: GET 2022-09-03T07:28:13.178 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.179 DEBUG:boto:Data: 2022-09-03T07:28:13.179 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.180 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.180 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.181 DEBUG:boto:Params: {} 2022-09-03T07:28:13.183 DEBUG:boto:Token: None 2022-09-03T07:28:13.185 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.186 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UcAItoWmmhJDEbPRSBA36xVi8ng= 2022-09-03T07:28:13.187 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UcAItoWmmhJDEbPRSBA36xVi8ng=', 'Content-Length': '0'} 2022-09-03T07:28:13.190 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000873cf383483d881-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.191 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.192 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.194 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.195 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.196 DEBUG:boto:Method: GET 2022-09-03T07:28:13.197 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.198 DEBUG:boto:Data: 2022-09-03T07:28:13.200 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.212 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.213 DEBUG:boto:Params: {} 2022-09-03T07:28:13.221 DEBUG:boto:Token: None 2022-09-03T07:28:13.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:13.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UcAItoWmmhJDEbPRSBA36xVi8ng= 2022-09-03T07:28:13.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UcAItoWmmhJDEbPRSBA36xVi8ng=', 'Content-Length': '0'} 2022-09-03T07:28:13.229 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f34b9263ffcfcc2-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.231 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.231 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.232 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:13.232 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:13.233 DEBUG:boto:Method: GET 2022-09-03T07:28:13.234 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:13.235 DEBUG:boto:Data: 2022-09-03T07:28:13.235 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.236 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.237 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.237 DEBUG:boto:Params: {} 2022-09-03T07:28:13.239 DEBUG:boto:Token: None 2022-09-03T07:28:13.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.248 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000255a664f8e7ceb17-006313020d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.250 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.251 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.252 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.253 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:13.257 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:13.258 DEBUG:boto:Method: GET 2022-09-03T07:28:13.259 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:13.259 DEBUG:boto:Data: 2022-09-03T07:28:13.260 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.261 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.261 DEBUG:boto:Params: {} 2022-09-03T07:28:13.263 DEBUG:boto:Token: None 2022-09-03T07:28:13.264 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.276 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d27f13c4dfdcee59-006313020d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.278 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.279 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.280 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.281 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.281 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:13.282 DEBUG:boto:Data: 2022-09-03T07:28:13.283 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.283 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.284 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.285 DEBUG:boto:Params: {} 2022-09-03T07:28:13.286 DEBUG:boto:Token: None 2022-09-03T07:28:13.287 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58 2022-09-03T07:28:13.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qhSnDFyv7HQc4hWSKVZdRTZ3beY= 2022-09-03T07:28:13.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qhSnDFyv7HQc4hWSKVZdRTZ3beY=', 'Content-Length': '0'} 2022-09-03T07:28:13.292 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b74caf2fb9d2fd15-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.294 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.295 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.296 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.296 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:13.297 DEBUG:boto:Data: 2022-09-03T07:28:13.299 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.300 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.300 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.301 DEBUG:boto:Params: {} 2022-09-03T07:28:13.301 DEBUG:boto:Token: None 2022-09-03T07:28:13.302 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58 2022-09-03T07:28:13.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qhSnDFyv7HQc4hWSKVZdRTZ3beY= 2022-09-03T07:28:13.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qhSnDFyv7HQc4hWSKVZdRTZ3beY=', 'Content-Length': '0'} 2022-09-03T07:28:13.306 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e20769fea165990c-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.310 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:28:13.310 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.311 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.312 DEBUG:boto:Method: GET 2022-09-03T07:28:13.312 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:13.313 DEBUG:boto:Data: 2022-09-03T07:28:13.313 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.314 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.315 DEBUG:boto:Params: {} 2022-09-03T07:28:13.315 DEBUG:boto:Token: None 2022-09-03T07:28:13.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58 2022-09-03T07:28:13.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+TU9sHtBhyXPxAOKi/37byZcac= 2022-09-03T07:28:13.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+TU9sHtBhyXPxAOKi/37byZcac=', 'Content-Length': '0'} 2022-09-03T07:28:13.319 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000febc8a109d2a7358-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.320 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:13.321 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:13.322 DEBUG:boto:Method: GET 2022-09-03T07:28:13.323 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:13.324 DEBUG:boto:Data: 2022-09-03T07:28:13.324 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.325 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.326 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.327 DEBUG:boto:Params: {} 2022-09-03T07:28:13.327 DEBUG:boto:Token: None 2022-09-03T07:28:13.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj58 2022-09-03T07:28:13.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l+TU9sHtBhyXPxAOKi/37byZcac= 2022-09-03T07:28:13.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l+TU9sHtBhyXPxAOKi/37byZcac=', 'Content-Length': '0'} 2022-09-03T07:28:13.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a36d862c496a929-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.333 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.334 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.335 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:13.335 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:13.336 DEBUG:boto:Method: GET 2022-09-03T07:28:13.337 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:13.338 DEBUG:boto:Data: 2022-09-03T07:28:13.339 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.341 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.342 DEBUG:boto:Params: {} 2022-09-03T07:28:13.343 DEBUG:boto:Token: None 2022-09-03T07:28:13.344 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.348 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6f24ffb40fb47fc-006313020d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.348 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.349 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.350 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.350 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:13.351 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:13.352 DEBUG:boto:Method: GET 2022-09-03T07:28:13.352 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:13.353 DEBUG:boto:Data: 2022-09-03T07:28:13.353 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.354 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.354 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.355 DEBUG:boto:Params: {} 2022-09-03T07:28:13.355 DEBUG:boto:Token: None 2022-09-03T07:28:13.356 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.357 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.361 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b478b5ac6fde8fab-006313020d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.365 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.365 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:28:13.366 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.367 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.367 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.368 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.368 DEBUG:boto:Method: GET 2022-09-03T07:28:13.370 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.371 DEBUG:boto:Data: 2022-09-03T07:28:13.371 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.372 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.372 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.373 DEBUG:boto:Params: {} 2022-09-03T07:28:13.374 DEBUG:boto:Token: None 2022-09-03T07:28:13.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jKEys3OMsX9hXE5H97bNGA23JoY= 2022-09-03T07:28:13.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jKEys3OMsX9hXE5H97bNGA23JoY=', 'Content-Length': '0'} 2022-09-03T07:28:13.378 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc3bee4ad446b512-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.379 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.381 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.381 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.383 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.384 DEBUG:boto:Method: GET 2022-09-03T07:28:13.385 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.386 DEBUG:boto:Data: 2022-09-03T07:28:13.387 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.387 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.388 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.389 DEBUG:boto:Params: {} 2022-09-03T07:28:13.390 DEBUG:boto:Token: None 2022-09-03T07:28:13.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jKEys3OMsX9hXE5H97bNGA23JoY= 2022-09-03T07:28:13.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jKEys3OMsX9hXE5H97bNGA23JoY=', 'Content-Length': '0'} 2022-09-03T07:28:13.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b076079368016d54-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.398 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.399 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.400 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.401 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.403 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.403 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.404 DEBUG:boto:Data: 2022-09-03T07:28:13.405 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.406 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.408 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.410 DEBUG:boto:Params: {} 2022-09-03T07:28:13.410 DEBUG:boto:Token: None 2022-09-03T07:28:13.411 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bz6NoYuuu3foydMq8YSbM2hivCk= 2022-09-03T07:28:13.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bz6NoYuuu3foydMq8YSbM2hivCk=', 'Content-Length': '0'} 2022-09-03T07:28:13.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c611e711ea6e5cdc-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.415 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.416 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.417 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.418 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.419 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.419 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.420 DEBUG:boto:Data: 2022-09-03T07:28:13.421 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.422 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.422 DEBUG:boto:Params: {} 2022-09-03T07:28:13.423 DEBUG:boto:Token: None 2022-09-03T07:28:13.423 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bz6NoYuuu3foydMq8YSbM2hivCk= 2022-09-03T07:28:13.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bz6NoYuuu3foydMq8YSbM2hivCk=', 'Content-Length': '0'} 2022-09-03T07:28:13.427 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e38bf6e465f4f47-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.428 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:28:13.429 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.430 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.430 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.431 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.431 DEBUG:boto:Method: GET 2022-09-03T07:28:13.432 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.433 DEBUG:boto:Data: 2022-09-03T07:28:13.433 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.434 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.434 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.435 DEBUG:boto:Params: {} 2022-09-03T07:28:13.436 DEBUG:boto:Token: None 2022-09-03T07:28:13.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jKEys3OMsX9hXE5H97bNGA23JoY= 2022-09-03T07:28:13.439 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jKEys3OMsX9hXE5H97bNGA23JoY=', 'Content-Length': '0'} 2022-09-03T07:28:13.441 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f305fa744efc527-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.489 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.490 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.491 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.492 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.493 DEBUG:boto:Method: GET 2022-09-03T07:28:13.494 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.496 DEBUG:boto:Data: 2022-09-03T07:28:13.497 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.500 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.501 DEBUG:boto:Params: {} 2022-09-03T07:28:13.502 DEBUG:boto:Token: None 2022-09-03T07:28:13.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:13.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jKEys3OMsX9hXE5H97bNGA23JoY= 2022-09-03T07:28:13.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jKEys3OMsX9hXE5H97bNGA23JoY=', 'Content-Length': '0'} 2022-09-03T07:28:13.508 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd6f0731b2c9cc04-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.509 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.509 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.510 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:13.511 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:13.512 DEBUG:boto:Method: GET 2022-09-03T07:28:13.513 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:13.514 DEBUG:boto:Data: 2022-09-03T07:28:13.516 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.517 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.518 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.519 DEBUG:boto:Params: {} 2022-09-03T07:28:13.520 DEBUG:boto:Token: None 2022-09-03T07:28:13.521 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.526 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000026e0329cf6b5a38e-006313020d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.527 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.528 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.528 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.529 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:13.530 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:13.530 DEBUG:boto:Method: GET 2022-09-03T07:28:13.531 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:13.531 DEBUG:boto:Data: 2022-09-03T07:28:13.533 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.535 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.535 DEBUG:boto:Params: {} 2022-09-03T07:28:13.536 DEBUG:boto:Token: None 2022-09-03T07:28:13.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.542 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020bcf26da6acc107-006313020d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.545 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.546 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.547 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.548 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.549 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:13.549 DEBUG:boto:Data: 2022-09-03T07:28:13.550 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.552 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.553 DEBUG:boto:Params: {} 2022-09-03T07:28:13.554 DEBUG:boto:Token: None 2022-09-03T07:28:13.555 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62 2022-09-03T07:28:13.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:40eIv30Hzh8QBwf+rsQrAJw9gHM= 2022-09-03T07:28:13.557 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:40eIv30Hzh8QBwf+rsQrAJw9gHM=', 'Content-Length': '0'} 2022-09-03T07:28:13.559 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000994880bce8d85667-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.560 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.561 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.562 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.563 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:13.564 DEBUG:boto:Data: 2022-09-03T07:28:13.565 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.566 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.567 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.568 DEBUG:boto:Params: {} 2022-09-03T07:28:13.568 DEBUG:boto:Token: None 2022-09-03T07:28:13.569 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62 2022-09-03T07:28:13.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:40eIv30Hzh8QBwf+rsQrAJw9gHM= 2022-09-03T07:28:13.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:40eIv30Hzh8QBwf+rsQrAJw9gHM=', 'Content-Length': '0'} 2022-09-03T07:28:13.573 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ba3d62073f03d8c-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.574 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:28:13.575 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.575 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.576 DEBUG:boto:Method: GET 2022-09-03T07:28:13.576 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:13.577 DEBUG:boto:Data: 2022-09-03T07:28:13.577 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.578 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.578 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.579 DEBUG:boto:Params: {} 2022-09-03T07:28:13.580 DEBUG:boto:Token: None 2022-09-03T07:28:13.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62 2022-09-03T07:28:13.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UrrDFCN2tX2FnbJztZ/Gno1NWwo= 2022-09-03T07:28:13.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UrrDFCN2tX2FnbJztZ/Gno1NWwo=', 'Content-Length': '0'} 2022-09-03T07:28:13.584 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001af33a7c413cfac8-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.584 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:13.585 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:13.585 DEBUG:boto:Method: GET 2022-09-03T07:28:13.586 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:13.587 DEBUG:boto:Data: 2022-09-03T07:28:13.587 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.588 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.588 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.589 DEBUG:boto:Params: {} 2022-09-03T07:28:13.590 DEBUG:boto:Token: None 2022-09-03T07:28:13.592 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj62 2022-09-03T07:28:13.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UrrDFCN2tX2FnbJztZ/Gno1NWwo= 2022-09-03T07:28:13.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UrrDFCN2tX2FnbJztZ/Gno1NWwo=', 'Content-Length': '0'} 2022-09-03T07:28:13.598 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a5c600ce3dbce0a-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.599 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.600 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.600 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:13.601 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:13.602 DEBUG:boto:Method: GET 2022-09-03T07:28:13.603 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:13.604 DEBUG:boto:Data: 2022-09-03T07:28:13.605 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.606 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.607 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.608 DEBUG:boto:Params: {} 2022-09-03T07:28:13.609 DEBUG:boto:Token: None 2022-09-03T07:28:13.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.615 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f7021a437a0ff183-006313020d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.616 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.617 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.618 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.619 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:13.619 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:13.620 DEBUG:boto:Method: GET 2022-09-03T07:28:13.620 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:13.621 DEBUG:boto:Data: 2022-09-03T07:28:13.621 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.623 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.624 DEBUG:boto:Params: {} 2022-09-03T07:28:13.625 DEBUG:boto:Token: None 2022-09-03T07:28:13.625 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.631 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ca39488344ad5ec8-006313020d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.674 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.676 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:28:13.677 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.678 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.678 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.679 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.680 DEBUG:boto:Method: GET 2022-09-03T07:28:13.681 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.683 DEBUG:boto:Data: 2022-09-03T07:28:13.685 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.686 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.687 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.688 DEBUG:boto:Params: {} 2022-09-03T07:28:13.689 DEBUG:boto:Token: None 2022-09-03T07:28:13.690 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:av6bB8djXjqzgxXxgEDsEgwKQOk= 2022-09-03T07:28:13.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:av6bB8djXjqzgxXxgEDsEgwKQOk=', 'Content-Length': '0'} 2022-09-03T07:28:13.695 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000974376653f18908a-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.696 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.697 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.697 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.698 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.699 DEBUG:boto:Method: GET 2022-09-03T07:28:13.699 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.700 DEBUG:boto:Data: 2022-09-03T07:28:13.700 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.701 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.702 DEBUG:boto:Params: {} 2022-09-03T07:28:13.703 DEBUG:boto:Token: None 2022-09-03T07:28:13.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.705 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:av6bB8djXjqzgxXxgEDsEgwKQOk= 2022-09-03T07:28:13.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:av6bB8djXjqzgxXxgEDsEgwKQOk=', 'Content-Length': '0'} 2022-09-03T07:28:13.709 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6754b3da36af035-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.710 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.711 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.712 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.713 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.713 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.714 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.715 DEBUG:boto:Data: 2022-09-03T07:28:13.716 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.718 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.719 DEBUG:boto:Params: {} 2022-09-03T07:28:13.720 DEBUG:boto:Token: None 2022-09-03T07:28:13.721 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.722 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uF5YFqEVqEfZrKD/CVTwpmaee68= 2022-09-03T07:28:13.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uF5YFqEVqEfZrKD/CVTwpmaee68=', 'Content-Length': '0'} 2022-09-03T07:28:13.726 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aac5f030a722fbe0-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.727 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.727 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.728 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.728 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.729 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.729 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.730 DEBUG:boto:Data: 2022-09-03T07:28:13.730 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.731 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.732 DEBUG:boto:Params: {} 2022-09-03T07:28:13.732 DEBUG:boto:Token: None 2022-09-03T07:28:13.733 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uF5YFqEVqEfZrKD/CVTwpmaee68= 2022-09-03T07:28:13.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uF5YFqEVqEfZrKD/CVTwpmaee68=', 'Content-Length': '0'} 2022-09-03T07:28:13.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f04aae35acc31ba-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.738 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:28:13.739 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.740 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.741 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.741 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.742 DEBUG:boto:Method: GET 2022-09-03T07:28:13.743 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.743 DEBUG:boto:Data: 2022-09-03T07:28:13.744 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.746 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.746 DEBUG:boto:Params: {} 2022-09-03T07:28:13.747 DEBUG:boto:Token: None 2022-09-03T07:28:13.747 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:av6bB8djXjqzgxXxgEDsEgwKQOk= 2022-09-03T07:28:13.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:av6bB8djXjqzgxXxgEDsEgwKQOk=', 'Content-Length': '0'} 2022-09-03T07:28:13.751 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008aa8c298f05ee4ff-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.752 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.753 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.753 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.754 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.754 DEBUG:boto:Method: GET 2022-09-03T07:28:13.755 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.755 DEBUG:boto:Data: 2022-09-03T07:28:13.756 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.756 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.757 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.758 DEBUG:boto:Params: {} 2022-09-03T07:28:13.758 DEBUG:boto:Token: None 2022-09-03T07:28:13.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:13.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:av6bB8djXjqzgxXxgEDsEgwKQOk= 2022-09-03T07:28:13.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:av6bB8djXjqzgxXxgEDsEgwKQOk=', 'Content-Length': '0'} 2022-09-03T07:28:13.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000803fce05b8e0ad2e-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.764 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.765 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.765 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:13.766 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:13.766 DEBUG:boto:Method: GET 2022-09-03T07:28:13.767 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:13.767 DEBUG:boto:Data: 2022-09-03T07:28:13.769 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.769 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.770 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.770 DEBUG:boto:Params: {} 2022-09-03T07:28:13.771 DEBUG:boto:Token: None 2022-09-03T07:28:13.772 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.775 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb226d4fd5e6e9fa-006313020d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.776 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.777 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.778 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.779 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:13.780 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:13.781 DEBUG:boto:Method: GET 2022-09-03T07:28:13.782 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:13.783 DEBUG:boto:Data: 2022-09-03T07:28:13.784 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.784 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.785 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.786 DEBUG:boto:Params: {} 2022-09-03T07:28:13.786 DEBUG:boto:Token: None 2022-09-03T07:28:13.787 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.794 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ab94c1a8b81bf164-006313020d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.795 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.796 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.797 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.798 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.799 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:13.800 DEBUG:boto:Data: 2022-09-03T07:28:13.801 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.802 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.803 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.804 DEBUG:boto:Params: {} 2022-09-03T07:28:13.804 DEBUG:boto:Token: None 2022-09-03T07:28:13.805 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68 2022-09-03T07:28:13.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BKOVi1EvbJfK4+b9QeMQJXY4s50= 2022-09-03T07:28:13.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BKOVi1EvbJfK4+b9QeMQJXY4s50=', 'Content-Length': '0'} 2022-09-03T07:28:13.808 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009805a16dd33e5b46-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.809 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.810 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.810 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.811 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:13.811 DEBUG:boto:Data: 2022-09-03T07:28:13.812 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.812 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.813 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.813 DEBUG:boto:Params: {} 2022-09-03T07:28:13.814 DEBUG:boto:Token: None 2022-09-03T07:28:13.814 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68 2022-09-03T07:28:13.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BKOVi1EvbJfK4+b9QeMQJXY4s50= 2022-09-03T07:28:13.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BKOVi1EvbJfK4+b9QeMQJXY4s50=', 'Content-Length': '0'} 2022-09-03T07:28:13.817 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e22a6e2b96a7d532-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.818 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:28:13.819 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.819 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.820 DEBUG:boto:Method: GET 2022-09-03T07:28:13.820 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:13.821 DEBUG:boto:Data: 2022-09-03T07:28:13.821 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.823 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.823 DEBUG:boto:Params: {} 2022-09-03T07:28:13.824 DEBUG:boto:Token: None 2022-09-03T07:28:13.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68 2022-09-03T07:28:13.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OLUw3h2QV2xZ4m1no03MTIf66Yc= 2022-09-03T07:28:13.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OLUw3h2QV2xZ4m1no03MTIf66Yc=', 'Content-Length': '0'} 2022-09-03T07:28:13.828 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f865c74fe26d5e0d-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.874 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:13.875 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:13.876 DEBUG:boto:Method: GET 2022-09-03T07:28:13.877 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:13.878 DEBUG:boto:Data: 2022-09-03T07:28:13.879 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.881 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.882 DEBUG:boto:Params: {} 2022-09-03T07:28:13.883 DEBUG:boto:Token: None 2022-09-03T07:28:13.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj68 2022-09-03T07:28:13.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OLUw3h2QV2xZ4m1no03MTIf66Yc= 2022-09-03T07:28:13.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OLUw3h2QV2xZ4m1no03MTIf66Yc=', 'Content-Length': '0'} 2022-09-03T07:28:13.887 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee9bb76927e1863c-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.888 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.889 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.889 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:13.890 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:13.890 DEBUG:boto:Method: GET 2022-09-03T07:28:13.891 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:13.891 DEBUG:boto:Data: 2022-09-03T07:28:13.892 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.894 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.895 DEBUG:boto:Params: {} 2022-09-03T07:28:13.896 DEBUG:boto:Token: None 2022-09-03T07:28:13.897 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.902 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eeaf50bab5008c5f-006313020d-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.903 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.904 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:13.905 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:13.905 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:13.906 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:13.907 DEBUG:boto:Method: GET 2022-09-03T07:28:13.908 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:13.909 DEBUG:boto:Data: 2022-09-03T07:28:13.910 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.911 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.912 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.913 DEBUG:boto:Params: {} 2022-09-03T07:28:13.914 DEBUG:boto:Token: None 2022-09-03T07:28:13.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/ 2022-09-03T07:28:13.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg= 2022-09-03T07:28:13.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fvH+g4gDiAa7Asyc6U+EoePx6bg=', 'Content-Length': '0'} 2022-09-03T07:28:13.922 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000645c3acab4c04d38-006313020d-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.923 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:13.924 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:28:13.925 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:13.926 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:13.927 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.928 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.929 DEBUG:boto:Method: GET 2022-09-03T07:28:13.930 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.931 DEBUG:boto:Data: 2022-09-03T07:28:13.932 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.932 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.933 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.936 DEBUG:boto:Params: {} 2022-09-03T07:28:13.936 DEBUG:boto:Token: None 2022-09-03T07:28:13.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8x/D9XF975ah+Zvw2KYjp+yxiJQ= 2022-09-03T07:28:13.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8x/D9XF975ah+Zvw2KYjp+yxiJQ=', 'Content-Length': '0'} 2022-09-03T07:28:13.942 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000085efe20778038198-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.945 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:13.946 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:13.947 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.948 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.950 DEBUG:boto:Method: GET 2022-09-03T07:28:13.951 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.952 DEBUG:boto:Data: 2022-09-03T07:28:13.953 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.954 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.955 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.956 DEBUG:boto:Params: {} 2022-09-03T07:28:13.957 DEBUG:boto:Token: None 2022-09-03T07:28:13.959 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8x/D9XF975ah+Zvw2KYjp+yxiJQ= 2022-09-03T07:28:13.961 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8x/D9XF975ah+Zvw2KYjp+yxiJQ=', 'Content-Length': '0'} 2022-09-03T07:28:13.965 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054532ad96ef90c76-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.966 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:13.969 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:13.969 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.970 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.970 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.971 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.972 DEBUG:boto:Data: 2022-09-03T07:28:13.972 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.973 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:13.973 DEBUG:boto:Port: 8001 2022-09-03T07:28:13.974 DEBUG:boto:Params: {} 2022-09-03T07:28:13.974 DEBUG:boto:Token: None 2022-09-03T07:28:13.975 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AlTZhD/uOBoYKN1Vnh/ToCEnoHQ= 2022-09-03T07:28:13.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AlTZhD/uOBoYKN1Vnh/ToCEnoHQ=', 'Content-Length': '0'} 2022-09-03T07:28:13.980 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f31345be20462e63-006313020d-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:13.980 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:13.981 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:13.983 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.984 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.986 DEBUG:boto:Method: HEAD 2022-09-03T07:28:13.987 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.988 DEBUG:boto:Data: 2022-09-03T07:28:13.989 DEBUG:boto:Headers: {} 2022-09-03T07:28:13.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:13.991 DEBUG:boto:Port: 8000 2022-09-03T07:28:13.992 DEBUG:boto:Params: {} 2022-09-03T07:28:13.996 DEBUG:boto:Token: None 2022-09-03T07:28:13.996 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:13 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:13.997 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AlTZhD/uOBoYKN1Vnh/ToCEnoHQ= 2022-09-03T07:28:13.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AlTZhD/uOBoYKN1Vnh/ToCEnoHQ=', 'Content-Length': '0'} 2022-09-03T07:28:14.000 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d9dfa7106d272c3d-006313020d-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.000 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:28:14.001 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:14.001 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:14.002 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:14.003 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:14.003 DEBUG:boto:Method: GET 2022-09-03T07:28:14.004 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:14.005 DEBUG:boto:Data: 2022-09-03T07:28:14.005 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.006 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.007 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.007 DEBUG:boto:Params: {} 2022-09-03T07:28:14.008 DEBUG:boto:Token: None 2022-09-03T07:28:14.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:14.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/wSQP+wSKTyDxttEdidkb0YJLnE= 2022-09-03T07:28:14.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/wSQP+wSKTyDxttEdidkb0YJLnE=', 'Content-Length': '0'} 2022-09-03T07:28:14.012 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055690c03f184e35c-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.054 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:14.055 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:14.055 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:14.056 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:14.057 DEBUG:boto:Method: GET 2022-09-03T07:28:14.058 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:14.058 DEBUG:boto:Data: 2022-09-03T07:28:14.059 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.060 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.060 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.061 DEBUG:boto:Params: {} 2022-09-03T07:28:14.061 DEBUG:boto:Token: None 2022-09-03T07:28:14.062 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:14.063 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/wSQP+wSKTyDxttEdidkb0YJLnE= 2022-09-03T07:28:14.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/wSQP+wSKTyDxttEdidkb0YJLnE=', 'Content-Length': '0'} 2022-09-03T07:28:14.066 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be99ea9adb8df280-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.067 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.068 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.068 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:14.069 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:14.069 DEBUG:boto:Method: GET 2022-09-03T07:28:14.070 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:14.070 DEBUG:boto:Data: 2022-09-03T07:28:14.071 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.071 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.072 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.073 DEBUG:boto:Params: {} 2022-09-03T07:28:14.073 DEBUG:boto:Token: None 2022-09-03T07:28:14.074 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.079 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e909074bfb5bc123-006313020e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.080 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.081 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.081 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.082 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:14.083 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:14.083 DEBUG:boto:Method: GET 2022-09-03T07:28:14.084 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:14.085 DEBUG:boto:Data: 2022-09-03T07:28:14.085 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.086 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.087 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.088 DEBUG:boto:Params: {} 2022-09-03T07:28:14.089 DEBUG:boto:Token: None 2022-09-03T07:28:14.089 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.095 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006b754c4795f259a2-006313020e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.096 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.097 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:14.098 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:14.098 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.099 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:14.099 DEBUG:boto:Data: 2022-09-03T07:28:14.100 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.101 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.101 DEBUG:boto:Params: {} 2022-09-03T07:28:14.102 DEBUG:boto:Token: None 2022-09-03T07:28:14.103 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj69 2022-09-03T07:28:14.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GfbCrF6sKb3WZgBKvK/JJbj4f7k= 2022-09-03T07:28:14.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GfbCrF6sKb3WZgBKvK/JJbj4f7k=', 'Content-Length': '0'} 2022-09-03T07:28:14.106 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c838001b8841784f-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.107 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:14.108 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:14.108 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.109 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:14.110 DEBUG:boto:Data: 2022-09-03T07:28:14.111 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.112 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.112 DEBUG:boto:Params: {} 2022-09-03T07:28:14.113 DEBUG:boto:Token: None 2022-09-03T07:28:14.114 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj69 2022-09-03T07:28:14.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GfbCrF6sKb3WZgBKvK/JJbj4f7k= 2022-09-03T07:28:14.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GfbCrF6sKb3WZgBKvK/JJbj4f7k=', 'Content-Length': '0'} 2022-09-03T07:28:14.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c583aa15f81bc772-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.120 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:28:14.120 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:14.121 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:14.122 DEBUG:boto:Method: GET 2022-09-03T07:28:14.122 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:14.123 DEBUG:boto:Data: 2022-09-03T07:28:14.123 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.124 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.125 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.125 DEBUG:boto:Params: {} 2022-09-03T07:28:14.126 DEBUG:boto:Token: None 2022-09-03T07:28:14.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj69 2022-09-03T07:28:14.129 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wMhPTM6+N3AEMhtNg53c6N9HlzI= 2022-09-03T07:28:14.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wMhPTM6+N3AEMhtNg53c6N9HlzI=', 'Content-Length': '0'} 2022-09-03T07:28:14.132 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000011ed91233ff172cd-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.178 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:14.179 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:14.179 DEBUG:boto:Method: GET 2022-09-03T07:28:14.180 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:14.181 DEBUG:boto:Data: 2022-09-03T07:28:14.182 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.183 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.184 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.185 DEBUG:boto:Params: {} 2022-09-03T07:28:14.186 DEBUG:boto:Token: None 2022-09-03T07:28:14.187 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj69 2022-09-03T07:28:14.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wMhPTM6+N3AEMhtNg53c6N9HlzI= 2022-09-03T07:28:14.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wMhPTM6+N3AEMhtNg53c6N9HlzI=', 'Content-Length': '0'} 2022-09-03T07:28:14.192 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005072caaffee7fffd-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.193 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.194 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.194 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:14.195 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:14.195 DEBUG:boto:Method: GET 2022-09-03T07:28:14.196 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:14.196 DEBUG:boto:Data: 2022-09-03T07:28:14.197 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.198 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.198 DEBUG:boto:Params: {} 2022-09-03T07:28:14.199 DEBUG:boto:Token: None 2022-09-03T07:28:14.200 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.203 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000418afcd81190e843-006313020e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.204 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.205 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.205 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.206 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:14.207 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:14.207 DEBUG:boto:Method: GET 2022-09-03T07:28:14.208 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:14.208 DEBUG:boto:Data: 2022-09-03T07:28:14.209 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.210 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.211 DEBUG:boto:Params: {} 2022-09-03T07:28:14.212 DEBUG:boto:Token: None 2022-09-03T07:28:14.212 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.214 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.218 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000883fe53014c4760-006313020e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.219 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.220 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:28:14.221 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.221 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.222 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.222 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.223 DEBUG:boto:Method: GET 2022-09-03T07:28:14.223 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.224 DEBUG:boto:Data: 2022-09-03T07:28:14.225 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.225 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.226 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.226 DEBUG:boto:Params: {} 2022-09-03T07:28:14.227 DEBUG:boto:Token: None 2022-09-03T07:28:14.227 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+843HPOWRUYkU0Itc4R7WBdmU84= 2022-09-03T07:28:14.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+843HPOWRUYkU0Itc4R7WBdmU84=', 'Content-Length': '0'} 2022-09-03T07:28:14.232 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aca6b45c367df4e0-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.233 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.234 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.234 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.235 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.235 DEBUG:boto:Method: GET 2022-09-03T07:28:14.236 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.236 DEBUG:boto:Data: 2022-09-03T07:28:14.237 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.238 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.238 DEBUG:boto:Params: {} 2022-09-03T07:28:14.239 DEBUG:boto:Token: None 2022-09-03T07:28:14.240 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+843HPOWRUYkU0Itc4R7WBdmU84= 2022-09-03T07:28:14.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+843HPOWRUYkU0Itc4R7WBdmU84=', 'Content-Length': '0'} 2022-09-03T07:28:14.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f43c9d309df90e2d-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.244 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.244 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.245 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.245 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.246 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.246 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.247 DEBUG:boto:Data: 2022-09-03T07:28:14.247 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.248 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.249 DEBUG:boto:Params: {} 2022-09-03T07:28:14.250 DEBUG:boto:Token: None 2022-09-03T07:28:14.250 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r9oWd3/1y6sX63s+o4MRKo9oJ98= 2022-09-03T07:28:14.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r9oWd3/1y6sX63s+o4MRKo9oJ98=', 'Content-Length': '0'} 2022-09-03T07:28:14.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b0eb98f3b3723eb-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.255 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.256 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.257 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.258 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.259 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.260 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.261 DEBUG:boto:Data: 2022-09-03T07:28:14.262 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.263 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.264 DEBUG:boto:Params: {} 2022-09-03T07:28:14.265 DEBUG:boto:Token: None 2022-09-03T07:28:14.266 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r9oWd3/1y6sX63s+o4MRKo9oJ98= 2022-09-03T07:28:14.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r9oWd3/1y6sX63s+o4MRKo9oJ98=', 'Content-Length': '0'} 2022-09-03T07:28:14.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074757216c3567f59-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.271 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:28:14.272 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.273 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.273 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.274 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.275 DEBUG:boto:Method: GET 2022-09-03T07:28:14.276 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.277 DEBUG:boto:Data: 2022-09-03T07:28:14.278 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.279 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.279 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.280 DEBUG:boto:Params: {} 2022-09-03T07:28:14.281 DEBUG:boto:Token: None 2022-09-03T07:28:14.281 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+843HPOWRUYkU0Itc4R7WBdmU84= 2022-09-03T07:28:14.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+843HPOWRUYkU0Itc4R7WBdmU84=', 'Content-Length': '0'} 2022-09-03T07:28:14.284 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000121a9bdad9727286-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.285 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.286 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.286 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.287 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.287 DEBUG:boto:Method: GET 2022-09-03T07:28:14.288 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.288 DEBUG:boto:Data: 2022-09-03T07:28:14.289 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.289 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.290 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.291 DEBUG:boto:Params: {} 2022-09-03T07:28:14.291 DEBUG:boto:Token: None 2022-09-03T07:28:14.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:14.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+843HPOWRUYkU0Itc4R7WBdmU84= 2022-09-03T07:28:14.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+843HPOWRUYkU0Itc4R7WBdmU84=', 'Content-Length': '0'} 2022-09-03T07:28:14.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed7fe43418c258c7-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.296 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.297 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.297 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:14.298 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:14.298 DEBUG:boto:Method: GET 2022-09-03T07:28:14.299 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:14.300 DEBUG:boto:Data: 2022-09-03T07:28:14.300 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.301 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.301 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.302 DEBUG:boto:Params: {} 2022-09-03T07:28:14.302 DEBUG:boto:Token: None 2022-09-03T07:28:14.303 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.308 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008276be44f9b2a04f-006313020e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.308 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.309 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.310 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.310 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:14.311 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:14.311 DEBUG:boto:Method: GET 2022-09-03T07:28:14.312 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:14.312 DEBUG:boto:Data: 2022-09-03T07:28:14.313 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.314 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.314 DEBUG:boto:Params: {} 2022-09-03T07:28:14.315 DEBUG:boto:Token: None 2022-09-03T07:28:14.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.322 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000021ee1ddda81da0a0-006313020e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.324 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.325 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.326 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.327 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.330 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:14.331 DEBUG:boto:Data: 2022-09-03T07:28:14.331 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.333 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.333 DEBUG:boto:Params: {} 2022-09-03T07:28:14.334 DEBUG:boto:Token: None 2022-09-03T07:28:14.335 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76 2022-09-03T07:28:14.335 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HjY5pwZI85m3Udb1nOIWc7l9nxI= 2022-09-03T07:28:14.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HjY5pwZI85m3Udb1nOIWc7l9nxI=', 'Content-Length': '0'} 2022-09-03T07:28:14.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4dbcdd3e1e0140c-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.343 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.344 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.345 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.346 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:14.347 DEBUG:boto:Data: 2022-09-03T07:28:14.348 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.348 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.349 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.349 DEBUG:boto:Params: {} 2022-09-03T07:28:14.350 DEBUG:boto:Token: None 2022-09-03T07:28:14.350 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76 2022-09-03T07:28:14.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HjY5pwZI85m3Udb1nOIWc7l9nxI= 2022-09-03T07:28:14.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HjY5pwZI85m3Udb1nOIWc7l9nxI=', 'Content-Length': '0'} 2022-09-03T07:28:14.362 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000657a509dec9dd801-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.364 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:28:14.365 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.366 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.367 DEBUG:boto:Method: GET 2022-09-03T07:28:14.368 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:14.369 DEBUG:boto:Data: 2022-09-03T07:28:14.370 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.371 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.372 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.373 DEBUG:boto:Params: {} 2022-09-03T07:28:14.374 DEBUG:boto:Token: None 2022-09-03T07:28:14.375 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76 2022-09-03T07:28:14.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+cVvjaVkRfKivT3oKqLP2MNSs1U= 2022-09-03T07:28:14.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+cVvjaVkRfKivT3oKqLP2MNSs1U=', 'Content-Length': '0'} 2022-09-03T07:28:14.380 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000133e7d540a328bab-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.425 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:14.426 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:14.428 DEBUG:boto:Method: GET 2022-09-03T07:28:14.429 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:14.429 DEBUG:boto:Data: 2022-09-03T07:28:14.430 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.431 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.431 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.432 DEBUG:boto:Params: {} 2022-09-03T07:28:14.433 DEBUG:boto:Token: None 2022-09-03T07:28:14.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj76 2022-09-03T07:28:14.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+cVvjaVkRfKivT3oKqLP2MNSs1U= 2022-09-03T07:28:14.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+cVvjaVkRfKivT3oKqLP2MNSs1U=', 'Content-Length': '0'} 2022-09-03T07:28:14.437 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065d154968ccd27af-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.438 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.440 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.441 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:14.442 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:14.444 DEBUG:boto:Method: GET 2022-09-03T07:28:14.445 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:14.447 DEBUG:boto:Data: 2022-09-03T07:28:14.447 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.449 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.451 DEBUG:boto:Params: {} 2022-09-03T07:28:14.452 DEBUG:boto:Token: None 2022-09-03T07:28:14.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.458 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000097c63b7a4fbf4822-006313020e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.460 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.462 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.463 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.463 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:14.464 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:14.465 DEBUG:boto:Method: GET 2022-09-03T07:28:14.466 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:14.466 DEBUG:boto:Data: 2022-09-03T07:28:14.467 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.468 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.469 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.470 DEBUG:boto:Params: {} 2022-09-03T07:28:14.471 DEBUG:boto:Token: None 2022-09-03T07:28:14.472 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.479 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008bfd45e891fe89d7-006313020e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.480 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.481 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:28:14.482 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.483 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.484 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.486 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.487 DEBUG:boto:Method: GET 2022-09-03T07:28:14.487 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.488 DEBUG:boto:Data: 2022-09-03T07:28:14.489 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.491 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.492 DEBUG:boto:Params: {} 2022-09-03T07:28:14.493 DEBUG:boto:Token: None 2022-09-03T07:28:14.494 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.495 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pi/76EIdPWMQ2YVJdFa8k+owCBs= 2022-09-03T07:28:14.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pi/76EIdPWMQ2YVJdFa8k+owCBs=', 'Content-Length': '0'} 2022-09-03T07:28:14.499 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007a3b6d6cbe2e2e21-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.500 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.501 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.501 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.503 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.504 DEBUG:boto:Method: GET 2022-09-03T07:28:14.505 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.506 DEBUG:boto:Data: 2022-09-03T07:28:14.507 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.509 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.509 DEBUG:boto:Params: {} 2022-09-03T07:28:14.510 DEBUG:boto:Token: None 2022-09-03T07:28:14.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.511 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pi/76EIdPWMQ2YVJdFa8k+owCBs= 2022-09-03T07:28:14.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pi/76EIdPWMQ2YVJdFa8k+owCBs=', 'Content-Length': '0'} 2022-09-03T07:28:14.514 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031e4a2426b1fa532-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.515 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.515 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.516 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.516 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.517 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.518 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.518 DEBUG:boto:Data: 2022-09-03T07:28:14.519 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.519 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.520 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.520 DEBUG:boto:Params: {} 2022-09-03T07:28:14.521 DEBUG:boto:Token: None 2022-09-03T07:28:14.522 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6WVglc7rmKugFheaaHj9DYt9WYI= 2022-09-03T07:28:14.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6WVglc7rmKugFheaaHj9DYt9WYI=', 'Content-Length': '0'} 2022-09-03T07:28:14.525 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006938bbe5f28eb6a8-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.526 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.526 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.527 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.527 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.529 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.530 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.530 DEBUG:boto:Data: 2022-09-03T07:28:14.531 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.532 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.532 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.533 DEBUG:boto:Params: {} 2022-09-03T07:28:14.533 DEBUG:boto:Token: None 2022-09-03T07:28:14.534 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6WVglc7rmKugFheaaHj9DYt9WYI= 2022-09-03T07:28:14.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6WVglc7rmKugFheaaHj9DYt9WYI=', 'Content-Length': '0'} 2022-09-03T07:28:14.538 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be0fc33c0178f37c-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.539 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:28:14.540 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.541 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.542 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.543 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.544 DEBUG:boto:Method: GET 2022-09-03T07:28:14.545 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.546 DEBUG:boto:Data: 2022-09-03T07:28:14.546 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.547 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.548 DEBUG:boto:Params: {} 2022-09-03T07:28:14.548 DEBUG:boto:Token: None 2022-09-03T07:28:14.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pi/76EIdPWMQ2YVJdFa8k+owCBs= 2022-09-03T07:28:14.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pi/76EIdPWMQ2YVJdFa8k+owCBs=', 'Content-Length': '0'} 2022-09-03T07:28:14.552 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000017b24943950711f7-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.553 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.554 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.563 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.564 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.565 DEBUG:boto:Method: GET 2022-09-03T07:28:14.565 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.566 DEBUG:boto:Data: 2022-09-03T07:28:14.567 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.568 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.568 DEBUG:boto:Params: {} 2022-09-03T07:28:14.569 DEBUG:boto:Token: None 2022-09-03T07:28:14.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:14.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pi/76EIdPWMQ2YVJdFa8k+owCBs= 2022-09-03T07:28:14.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pi/76EIdPWMQ2YVJdFa8k+owCBs=', 'Content-Length': '0'} 2022-09-03T07:28:14.574 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000efb997f8446ddf5d-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.618 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.619 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.619 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:14.620 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:14.622 DEBUG:boto:Method: GET 2022-09-03T07:28:14.622 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:14.623 DEBUG:boto:Data: 2022-09-03T07:28:14.624 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.624 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.625 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.625 DEBUG:boto:Params: {} 2022-09-03T07:28:14.626 DEBUG:boto:Token: None 2022-09-03T07:28:14.626 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.631 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005df6da907b34863f-006313020e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.634 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.635 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.637 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.637 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:14.641 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:14.642 DEBUG:boto:Method: GET 2022-09-03T07:28:14.643 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:14.644 DEBUG:boto:Data: 2022-09-03T07:28:14.645 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.646 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.647 DEBUG:boto:Params: {} 2022-09-03T07:28:14.648 DEBUG:boto:Token: None 2022-09-03T07:28:14.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.651 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.657 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000529fb2ef9abba313-006313020e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.658 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.659 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.659 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.660 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.661 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:14.661 DEBUG:boto:Data: 2022-09-03T07:28:14.662 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.662 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.663 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.663 DEBUG:boto:Params: {} 2022-09-03T07:28:14.664 DEBUG:boto:Token: None 2022-09-03T07:28:14.664 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85 2022-09-03T07:28:14.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9yxsADILSuecpzJ25dUGo04MB1M= 2022-09-03T07:28:14.666 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9yxsADILSuecpzJ25dUGo04MB1M=', 'Content-Length': '0'} 2022-09-03T07:28:14.668 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dbc50f04dda9585c-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.669 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.670 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.670 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.671 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:14.672 DEBUG:boto:Data: 2022-09-03T07:28:14.672 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.673 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.674 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.674 DEBUG:boto:Params: {} 2022-09-03T07:28:14.675 DEBUG:boto:Token: None 2022-09-03T07:28:14.676 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85 2022-09-03T07:28:14.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9yxsADILSuecpzJ25dUGo04MB1M= 2022-09-03T07:28:14.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9yxsADILSuecpzJ25dUGo04MB1M=', 'Content-Length': '0'} 2022-09-03T07:28:14.679 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2c3628c27eae505-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.679 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:28:14.680 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.681 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.681 DEBUG:boto:Method: GET 2022-09-03T07:28:14.682 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:14.683 DEBUG:boto:Data: 2022-09-03T07:28:14.684 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.684 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.685 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.686 DEBUG:boto:Params: {} 2022-09-03T07:28:14.686 DEBUG:boto:Token: None 2022-09-03T07:28:14.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85 2022-09-03T07:28:14.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kacQ9+VgZ5oQv1dbRCY+ffOwDWs= 2022-09-03T07:28:14.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kacQ9+VgZ5oQv1dbRCY+ffOwDWs=', 'Content-Length': '0'} 2022-09-03T07:28:14.692 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090ed931376f1de56-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.693 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:14.694 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:14.694 DEBUG:boto:Method: GET 2022-09-03T07:28:14.695 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:14.696 DEBUG:boto:Data: 2022-09-03T07:28:14.696 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.697 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.697 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.698 DEBUG:boto:Params: {} 2022-09-03T07:28:14.699 DEBUG:boto:Token: None 2022-09-03T07:28:14.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj85 2022-09-03T07:28:14.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kacQ9+VgZ5oQv1dbRCY+ffOwDWs= 2022-09-03T07:28:14.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kacQ9+VgZ5oQv1dbRCY+ffOwDWs=', 'Content-Length': '0'} 2022-09-03T07:28:14.703 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000084c5cc9d98e411c2-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.746 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.747 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.748 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:14.748 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:14.749 DEBUG:boto:Method: GET 2022-09-03T07:28:14.749 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:14.750 DEBUG:boto:Data: 2022-09-03T07:28:14.751 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.751 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.752 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.752 DEBUG:boto:Params: {} 2022-09-03T07:28:14.753 DEBUG:boto:Token: None 2022-09-03T07:28:14.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.757 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008147ed2f2d2becbc-006313020e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.758 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.759 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.759 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.760 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:14.760 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:14.761 DEBUG:boto:Method: GET 2022-09-03T07:28:14.761 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:14.762 DEBUG:boto:Data: 2022-09-03T07:28:14.763 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.763 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.764 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.764 DEBUG:boto:Params: {} 2022-09-03T07:28:14.765 DEBUG:boto:Token: None 2022-09-03T07:28:14.766 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.771 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e72e4e91059aa82e-006313020e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.772 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.773 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:28:14.773 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.774 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.775 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.775 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.776 DEBUG:boto:Method: GET 2022-09-03T07:28:14.777 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.781 DEBUG:boto:Data: 2022-09-03T07:28:14.781 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.782 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.783 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.783 DEBUG:boto:Params: {} 2022-09-03T07:28:14.784 DEBUG:boto:Token: None 2022-09-03T07:28:14.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.785 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mqQtM0NdfXvRthG+S5iBRBSCAY= 2022-09-03T07:28:14.786 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mqQtM0NdfXvRthG+S5iBRBSCAY=', 'Content-Length': '0'} 2022-09-03T07:28:14.788 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000059085d1b84115e5e-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.790 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.791 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.792 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.793 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.794 DEBUG:boto:Method: GET 2022-09-03T07:28:14.795 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.796 DEBUG:boto:Data: 2022-09-03T07:28:14.797 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.799 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.800 DEBUG:boto:Params: {} 2022-09-03T07:28:14.801 DEBUG:boto:Token: None 2022-09-03T07:28:14.802 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.804 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mqQtM0NdfXvRthG+S5iBRBSCAY= 2022-09-03T07:28:14.805 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mqQtM0NdfXvRthG+S5iBRBSCAY=', 'Content-Length': '0'} 2022-09-03T07:28:14.808 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001af676ef6a1b4d09-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.809 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.809 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.810 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.811 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.811 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.812 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.812 DEBUG:boto:Data: 2022-09-03T07:28:14.813 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.814 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.814 DEBUG:boto:Params: {} 2022-09-03T07:28:14.815 DEBUG:boto:Token: None 2022-09-03T07:28:14.816 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RUQkNspw3Hd1c4N+yaO3EBXa9+4= 2022-09-03T07:28:14.817 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RUQkNspw3Hd1c4N+yaO3EBXa9+4=', 'Content-Length': '0'} 2022-09-03T07:28:14.819 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7d10aa49ae31161-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.820 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.820 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.821 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.821 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.822 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.822 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.823 DEBUG:boto:Data: 2022-09-03T07:28:14.824 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.824 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.825 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.825 DEBUG:boto:Params: {} 2022-09-03T07:28:14.826 DEBUG:boto:Token: None 2022-09-03T07:28:14.827 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RUQkNspw3Hd1c4N+yaO3EBXa9+4= 2022-09-03T07:28:14.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RUQkNspw3Hd1c4N+yaO3EBXa9+4=', 'Content-Length': '0'} 2022-09-03T07:28:14.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000011c18c7e2cd76a48-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.831 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:28:14.832 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.832 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.833 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.833 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.834 DEBUG:boto:Method: GET 2022-09-03T07:28:14.835 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.835 DEBUG:boto:Data: 2022-09-03T07:28:14.836 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.836 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.837 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.838 DEBUG:boto:Params: {} 2022-09-03T07:28:14.838 DEBUG:boto:Token: None 2022-09-03T07:28:14.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mqQtM0NdfXvRthG+S5iBRBSCAY= 2022-09-03T07:28:14.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mqQtM0NdfXvRthG+S5iBRBSCAY=', 'Content-Length': '0'} 2022-09-03T07:28:14.842 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c808043aec7ec7b9-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.844 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.844 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.845 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.846 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.846 DEBUG:boto:Method: GET 2022-09-03T07:28:14.847 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.848 DEBUG:boto:Data: 2022-09-03T07:28:14.848 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.850 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.850 DEBUG:boto:Params: {} 2022-09-03T07:28:14.851 DEBUG:boto:Token: None 2022-09-03T07:28:14.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:14.852 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/mqQtM0NdfXvRthG+S5iBRBSCAY= 2022-09-03T07:28:14.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/mqQtM0NdfXvRthG+S5iBRBSCAY=', 'Content-Length': '0'} 2022-09-03T07:28:14.855 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d16f4b837dd385d3-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.898 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.899 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.900 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:14.900 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:14.901 DEBUG:boto:Method: GET 2022-09-03T07:28:14.901 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:14.902 DEBUG:boto:Data: 2022-09-03T07:28:14.903 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.904 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.905 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.905 DEBUG:boto:Params: {} 2022-09-03T07:28:14.906 DEBUG:boto:Token: None 2022-09-03T07:28:14.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000003cacee0e07cf3ee-006313020e-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.914 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.915 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.916 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.916 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:14.917 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:14.918 DEBUG:boto:Method: GET 2022-09-03T07:28:14.919 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:14.919 DEBUG:boto:Data: 2022-09-03T07:28:14.920 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.922 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.922 DEBUG:boto:Params: {} 2022-09-03T07:28:14.923 DEBUG:boto:Token: None 2022-09-03T07:28:14.924 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/ 2022-09-03T07:28:14.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk= 2022-09-03T07:28:14.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sc6ypS60+QiTcNljR+q8xbDytk=', 'Content-Length': '0'} 2022-09-03T07:28:14.929 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000377ce3e272b7832f-006313020e-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.930 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:14.931 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.932 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.933 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.933 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:14.934 DEBUG:boto:Data: 2022-09-03T07:28:14.934 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.935 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.936 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.937 DEBUG:boto:Params: {} 2022-09-03T07:28:14.937 DEBUG:boto:Token: None 2022-09-03T07:28:14.938 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87 2022-09-03T07:28:14.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yynENe5eHEy8MB0ZBbcr99oxYPQ= 2022-09-03T07:28:14.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yynENe5eHEy8MB0ZBbcr99oxYPQ=', 'Content-Length': '0'} 2022-09-03T07:28:14.942 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bdea5999d19a8482-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.943 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.944 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.945 DEBUG:boto:Method: HEAD 2022-09-03T07:28:14.947 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:14.948 DEBUG:boto:Data: 2022-09-03T07:28:14.949 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.952 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.952 DEBUG:boto:Params: {} 2022-09-03T07:28:14.953 DEBUG:boto:Token: None 2022-09-03T07:28:14.954 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87 2022-09-03T07:28:14.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yynENe5eHEy8MB0ZBbcr99oxYPQ= 2022-09-03T07:28:14.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yynENe5eHEy8MB0ZBbcr99oxYPQ=', 'Content-Length': '0'} 2022-09-03T07:28:14.958 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1597ca90eac4d8c-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.959 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:28:14.961 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.962 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.962 DEBUG:boto:Method: GET 2022-09-03T07:28:14.963 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:14.964 DEBUG:boto:Data: 2022-09-03T07:28:14.964 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.966 DEBUG:boto:Port: 8001 2022-09-03T07:28:14.968 DEBUG:boto:Params: {} 2022-09-03T07:28:14.969 DEBUG:boto:Token: None 2022-09-03T07:28:14.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87 2022-09-03T07:28:14.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9R0NVr10QZMs7jdpwPzhV7E3bMU= 2022-09-03T07:28:14.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9R0NVr10QZMs7jdpwPzhV7E3bMU=', 'Content-Length': '0'} 2022-09-03T07:28:14.975 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000021b04c0a73dfbebd-006313020e-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.976 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:14.977 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:14.978 DEBUG:boto:Method: GET 2022-09-03T07:28:14.979 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:14.979 DEBUG:boto:Data: 2022-09-03T07:28:14.980 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.981 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:14.983 DEBUG:boto:Port: 8000 2022-09-03T07:28:14.983 DEBUG:boto:Params: {} 2022-09-03T07:28:14.984 DEBUG:boto:Token: None 2022-09-03T07:28:14.985 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:14 GMT /avysmw-75/obj87 2022-09-03T07:28:14.986 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9R0NVr10QZMs7jdpwPzhV7E3bMU= 2022-09-03T07:28:14.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9R0NVr10QZMs7jdpwPzhV7E3bMU=', 'Content-Length': '0'} 2022-09-03T07:28:14.990 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f91fe5eed5e595c2-006313020e-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:14.991 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:14.993 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:14.993 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:14.994 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:14.995 DEBUG:boto:Method: GET 2022-09-03T07:28:14.996 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:14.996 DEBUG:boto:Data: 2022-09-03T07:28:14.997 DEBUG:boto:Headers: {} 2022-09-03T07:28:14.999 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:14.999 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.000 DEBUG:boto:Params: {} 2022-09-03T07:28:15.001 DEBUG:boto:Token: None 2022-09-03T07:28:15.003 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.007 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a18cc829bd337d2-006313020f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.010 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.011 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.012 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.014 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:15.016 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:15.017 DEBUG:boto:Method: GET 2022-09-03T07:28:15.018 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:15.020 DEBUG:boto:Data: 2022-09-03T07:28:15.021 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.022 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.022 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.023 DEBUG:boto:Params: {} 2022-09-03T07:28:15.024 DEBUG:boto:Token: None 2022-09-03T07:28:15.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.030 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000099a396641f414fb0-006313020f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.031 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.032 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:28:15.033 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.036 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.036 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.037 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.038 DEBUG:boto:Method: GET 2022-09-03T07:28:15.039 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.039 DEBUG:boto:Data: 2022-09-03T07:28:15.040 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.042 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.043 DEBUG:boto:Params: {} 2022-09-03T07:28:15.044 DEBUG:boto:Token: None 2022-09-03T07:28:15.045 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ZjbxVFonXc2uyJYhlnfCX10HfU= 2022-09-03T07:28:15.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ZjbxVFonXc2uyJYhlnfCX10HfU=', 'Content-Length': '0'} 2022-09-03T07:28:15.049 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c724a6d951fda91-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.050 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.051 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.052 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.052 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.053 DEBUG:boto:Method: GET 2022-09-03T07:28:15.054 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.055 DEBUG:boto:Data: 2022-09-03T07:28:15.056 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.058 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.059 DEBUG:boto:Params: {} 2022-09-03T07:28:15.060 DEBUG:boto:Token: None 2022-09-03T07:28:15.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ZjbxVFonXc2uyJYhlnfCX10HfU= 2022-09-03T07:28:15.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ZjbxVFonXc2uyJYhlnfCX10HfU=', 'Content-Length': '0'} 2022-09-03T07:28:15.064 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000379ade7052989a5a-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.106 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.107 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.108 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.108 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.109 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.110 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.110 DEBUG:boto:Data: 2022-09-03T07:28:15.111 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.112 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.112 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.113 DEBUG:boto:Params: {} 2022-09-03T07:28:15.114 DEBUG:boto:Token: None 2022-09-03T07:28:15.114 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.116 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QfsihB+6sM9K/YqvHJ/d2Ha+M/Y= 2022-09-03T07:28:15.117 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QfsihB+6sM9K/YqvHJ/d2Ha+M/Y=', 'Content-Length': '0'} 2022-09-03T07:28:15.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b9de63748755db5-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.120 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.120 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.121 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.122 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.122 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.123 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.123 DEBUG:boto:Data: 2022-09-03T07:28:15.124 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.127 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.128 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.128 DEBUG:boto:Params: {} 2022-09-03T07:28:15.129 DEBUG:boto:Token: None 2022-09-03T07:28:15.131 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.131 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QfsihB+6sM9K/YqvHJ/d2Ha+M/Y= 2022-09-03T07:28:15.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QfsihB+6sM9K/YqvHJ/d2Ha+M/Y=', 'Content-Length': '0'} 2022-09-03T07:28:15.134 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000073f4604d91ed3e15-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.135 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:28:15.135 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.136 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.136 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.137 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.138 DEBUG:boto:Method: GET 2022-09-03T07:28:15.138 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.139 DEBUG:boto:Data: 2022-09-03T07:28:15.139 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.140 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.141 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.141 DEBUG:boto:Params: {} 2022-09-03T07:28:15.142 DEBUG:boto:Token: None 2022-09-03T07:28:15.143 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ZjbxVFonXc2uyJYhlnfCX10HfU= 2022-09-03T07:28:15.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ZjbxVFonXc2uyJYhlnfCX10HfU=', 'Content-Length': '0'} 2022-09-03T07:28:15.146 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000663327ae3819335e-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.191 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.192 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.196 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.197 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.198 DEBUG:boto:Method: GET 2022-09-03T07:28:15.199 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.200 DEBUG:boto:Data: 2022-09-03T07:28:15.201 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.202 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.203 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.204 DEBUG:boto:Params: {} 2022-09-03T07:28:15.204 DEBUG:boto:Token: None 2022-09-03T07:28:15.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:15.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ZjbxVFonXc2uyJYhlnfCX10HfU= 2022-09-03T07:28:15.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ZjbxVFonXc2uyJYhlnfCX10HfU=', 'Content-Length': '0'} 2022-09-03T07:28:15.211 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012ca768f0931c742-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.212 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.213 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.214 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:15.215 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:15.216 DEBUG:boto:Method: GET 2022-09-03T07:28:15.217 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:15.218 DEBUG:boto:Data: 2022-09-03T07:28:15.219 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.220 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.221 DEBUG:boto:Params: {} 2022-09-03T07:28:15.222 DEBUG:boto:Token: None 2022-09-03T07:28:15.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.227 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076887577178f9b39-006313020f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.228 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.229 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.230 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.230 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:15.231 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:15.232 DEBUG:boto:Method: GET 2022-09-03T07:28:15.232 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:15.233 DEBUG:boto:Data: 2022-09-03T07:28:15.234 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.236 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.237 DEBUG:boto:Params: {} 2022-09-03T07:28:15.238 DEBUG:boto:Token: None 2022-09-03T07:28:15.239 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.246 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b44fa5f660f5e2e5-006313020f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.246 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.247 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.248 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.248 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.249 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:15.249 DEBUG:boto:Data: 2022-09-03T07:28:15.250 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.251 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.251 DEBUG:boto:Params: {} 2022-09-03T07:28:15.252 DEBUG:boto:Token: None 2022-09-03T07:28:15.253 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92 2022-09-03T07:28:15.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdUK80LwzDeoz3uOGoVKkpW6ykA= 2022-09-03T07:28:15.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdUK80LwzDeoz3uOGoVKkpW6ykA=', 'Content-Length': '0'} 2022-09-03T07:28:15.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f4bdadebcee77bf-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.258 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.259 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.260 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.260 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:15.261 DEBUG:boto:Data: 2022-09-03T07:28:15.262 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.264 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.265 DEBUG:boto:Params: {} 2022-09-03T07:28:15.266 DEBUG:boto:Token: None 2022-09-03T07:28:15.266 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92 2022-09-03T07:28:15.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rdUK80LwzDeoz3uOGoVKkpW6ykA= 2022-09-03T07:28:15.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rdUK80LwzDeoz3uOGoVKkpW6ykA=', 'Content-Length': '0'} 2022-09-03T07:28:15.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c381ab1ede0f6b57-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.271 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:28:15.271 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.272 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.272 DEBUG:boto:Method: GET 2022-09-03T07:28:15.273 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:15.274 DEBUG:boto:Data: 2022-09-03T07:28:15.275 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.276 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.277 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.278 DEBUG:boto:Params: {} 2022-09-03T07:28:15.279 DEBUG:boto:Token: None 2022-09-03T07:28:15.280 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92 2022-09-03T07:28:15.281 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JBFnNWUvDsMQxyM6JPNpVjEtjwY= 2022-09-03T07:28:15.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JBFnNWUvDsMQxyM6JPNpVjEtjwY=', 'Content-Length': '0'} 2022-09-03T07:28:15.285 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d0ecb1c603671c06-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.286 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:15.286 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:15.287 DEBUG:boto:Method: GET 2022-09-03T07:28:15.288 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:15.289 DEBUG:boto:Data: 2022-09-03T07:28:15.290 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.291 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.292 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.293 DEBUG:boto:Params: {} 2022-09-03T07:28:15.294 DEBUG:boto:Token: None 2022-09-03T07:28:15.295 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj92 2022-09-03T07:28:15.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JBFnNWUvDsMQxyM6JPNpVjEtjwY= 2022-09-03T07:28:15.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JBFnNWUvDsMQxyM6JPNpVjEtjwY=', 'Content-Length': '0'} 2022-09-03T07:28:15.299 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000298ec30511109c5e-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.300 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.300 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.301 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:15.301 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:15.302 DEBUG:boto:Method: GET 2022-09-03T07:28:15.303 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:15.303 DEBUG:boto:Data: 2022-09-03T07:28:15.304 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.306 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.307 DEBUG:boto:Params: {} 2022-09-03T07:28:15.308 DEBUG:boto:Token: None 2022-09-03T07:28:15.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.314 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dd42ac8e939681c8-006313020f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.315 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.316 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.316 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.317 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:15.317 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:15.318 DEBUG:boto:Method: GET 2022-09-03T07:28:15.318 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:15.319 DEBUG:boto:Data: 2022-09-03T07:28:15.320 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.322 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.323 DEBUG:boto:Params: {} 2022-09-03T07:28:15.324 DEBUG:boto:Token: None 2022-09-03T07:28:15.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.331 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001bf338d20a624105-006313020f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.334 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.337 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:28:15.337 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.338 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.339 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.339 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.340 DEBUG:boto:Method: GET 2022-09-03T07:28:15.340 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.341 DEBUG:boto:Data: 2022-09-03T07:28:15.342 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.343 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.344 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.345 DEBUG:boto:Params: {} 2022-09-03T07:28:15.346 DEBUG:boto:Token: None 2022-09-03T07:28:15.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1eZSBFzrDOBT7baL+hPaBd43L1M= 2022-09-03T07:28:15.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1eZSBFzrDOBT7baL+hPaBd43L1M=', 'Content-Length': '0'} 2022-09-03T07:28:15.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000039589a0cea71ece0-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.394 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.395 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.396 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.396 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.397 DEBUG:boto:Method: GET 2022-09-03T07:28:15.397 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.398 DEBUG:boto:Data: 2022-09-03T07:28:15.398 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.399 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.400 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.401 DEBUG:boto:Params: {} 2022-09-03T07:28:15.402 DEBUG:boto:Token: None 2022-09-03T07:28:15.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1eZSBFzrDOBT7baL+hPaBd43L1M= 2022-09-03T07:28:15.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1eZSBFzrDOBT7baL+hPaBd43L1M=', 'Content-Length': '0'} 2022-09-03T07:28:15.408 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099b8ff319a12c3ad-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.409 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.409 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.410 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.411 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.411 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.412 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.413 DEBUG:boto:Data: 2022-09-03T07:28:15.414 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.414 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.415 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.416 DEBUG:boto:Params: {} 2022-09-03T07:28:15.416 DEBUG:boto:Token: None 2022-09-03T07:28:15.417 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hkZexz7Mk2Z0bT4N9TM8lejMxY0= 2022-09-03T07:28:15.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hkZexz7Mk2Z0bT4N9TM8lejMxY0=', 'Content-Length': '0'} 2022-09-03T07:28:15.422 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f42a0862168a2b0-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.423 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.424 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.425 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.426 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.426 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.427 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.427 DEBUG:boto:Data: 2022-09-03T07:28:15.428 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.429 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.430 DEBUG:boto:Params: {} 2022-09-03T07:28:15.430 DEBUG:boto:Token: None 2022-09-03T07:28:15.431 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hkZexz7Mk2Z0bT4N9TM8lejMxY0= 2022-09-03T07:28:15.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hkZexz7Mk2Z0bT4N9TM8lejMxY0=', 'Content-Length': '0'} 2022-09-03T07:28:15.434 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010e502db12a93748-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.434 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:28:15.435 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.436 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.436 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.437 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.437 DEBUG:boto:Method: GET 2022-09-03T07:28:15.438 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.438 DEBUG:boto:Data: 2022-09-03T07:28:15.439 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.440 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.441 DEBUG:boto:Params: {} 2022-09-03T07:28:15.441 DEBUG:boto:Token: None 2022-09-03T07:28:15.442 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1eZSBFzrDOBT7baL+hPaBd43L1M= 2022-09-03T07:28:15.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1eZSBFzrDOBT7baL+hPaBd43L1M=', 'Content-Length': '0'} 2022-09-03T07:28:15.445 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3d6deac70116905-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.446 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.447 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.447 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.448 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.448 DEBUG:boto:Method: GET 2022-09-03T07:28:15.449 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.450 DEBUG:boto:Data: 2022-09-03T07:28:15.450 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.451 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.451 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.452 DEBUG:boto:Params: {} 2022-09-03T07:28:15.452 DEBUG:boto:Token: None 2022-09-03T07:28:15.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:15.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1eZSBFzrDOBT7baL+hPaBd43L1M= 2022-09-03T07:28:15.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1eZSBFzrDOBT7baL+hPaBd43L1M=', 'Content-Length': '0'} 2022-09-03T07:28:15.457 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005573dba9fce6db63-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.458 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.458 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.459 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:15.460 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:15.460 DEBUG:boto:Method: GET 2022-09-03T07:28:15.461 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:15.461 DEBUG:boto:Data: 2022-09-03T07:28:15.462 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.462 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.463 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.463 DEBUG:boto:Params: {} 2022-09-03T07:28:15.464 DEBUG:boto:Token: None 2022-09-03T07:28:15.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.466 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.469 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000631aeb56ec36f866-006313020f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.470 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.471 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.471 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.472 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:15.472 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:15.473 DEBUG:boto:Method: GET 2022-09-03T07:28:15.473 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:15.474 DEBUG:boto:Data: 2022-09-03T07:28:15.474 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.475 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.476 DEBUG:boto:Params: {} 2022-09-03T07:28:15.477 DEBUG:boto:Token: None 2022-09-03T07:28:15.477 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.478 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.482 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dfd7e7bfb003b398-006313020f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.483 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.483 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.484 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.485 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.485 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:15.486 DEBUG:boto:Data: 2022-09-03T07:28:15.486 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.487 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.488 DEBUG:boto:Params: {} 2022-09-03T07:28:15.488 DEBUG:boto:Token: None 2022-09-03T07:28:15.489 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93 2022-09-03T07:28:15.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pWHS3S1qn4EThDIIDBRsZIadT8U= 2022-09-03T07:28:15.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pWHS3S1qn4EThDIIDBRsZIadT8U=', 'Content-Length': '0'} 2022-09-03T07:28:15.492 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010b31c9efb6bf5bd-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.493 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.494 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.494 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.495 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:15.495 DEBUG:boto:Data: 2022-09-03T07:28:15.496 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.497 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.497 DEBUG:boto:Params: {} 2022-09-03T07:28:15.498 DEBUG:boto:Token: None 2022-09-03T07:28:15.499 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93 2022-09-03T07:28:15.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pWHS3S1qn4EThDIIDBRsZIadT8U= 2022-09-03T07:28:15.500 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pWHS3S1qn4EThDIIDBRsZIadT8U=', 'Content-Length': '0'} 2022-09-03T07:28:15.502 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c34d3943f8a3b279-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.503 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:28:15.503 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.504 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.504 DEBUG:boto:Method: GET 2022-09-03T07:28:15.505 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:15.505 DEBUG:boto:Data: 2022-09-03T07:28:15.506 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.506 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.507 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.508 DEBUG:boto:Params: {} 2022-09-03T07:28:15.508 DEBUG:boto:Token: None 2022-09-03T07:28:15.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93 2022-09-03T07:28:15.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFyKOSQ4Gbk2WQTGal/t7z6ISo= 2022-09-03T07:28:15.510 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFyKOSQ4Gbk2WQTGal/t7z6ISo=', 'Content-Length': '0'} 2022-09-03T07:28:15.515 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8408c31aa689494-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.516 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:15.517 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:15.518 DEBUG:boto:Method: GET 2022-09-03T07:28:15.519 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:15.520 DEBUG:boto:Data: 2022-09-03T07:28:15.521 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.522 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.524 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.524 DEBUG:boto:Params: {} 2022-09-03T07:28:15.525 DEBUG:boto:Token: None 2022-09-03T07:28:15.526 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj93 2022-09-03T07:28:15.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:biFyKOSQ4Gbk2WQTGal/t7z6ISo= 2022-09-03T07:28:15.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:biFyKOSQ4Gbk2WQTGal/t7z6ISo=', 'Content-Length': '0'} 2022-09-03T07:28:15.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5b76b44c2eccd10-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.570 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.572 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.573 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:15.574 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:15.574 DEBUG:boto:Method: GET 2022-09-03T07:28:15.575 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:15.575 DEBUG:boto:Data: 2022-09-03T07:28:15.576 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.577 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.578 DEBUG:boto:Params: {} 2022-09-03T07:28:15.579 DEBUG:boto:Token: None 2022-09-03T07:28:15.580 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.585 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008ff3e6e6ebab52f9-006313020f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.585 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.586 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.587 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.587 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:15.588 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:15.588 DEBUG:boto:Method: GET 2022-09-03T07:28:15.589 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:15.590 DEBUG:boto:Data: 2022-09-03T07:28:15.590 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.591 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.592 DEBUG:boto:Params: {} 2022-09-03T07:28:15.592 DEBUG:boto:Token: None 2022-09-03T07:28:15.593 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.594 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.598 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000482ce193d46a480-006313020f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.600 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.602 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:28:15.602 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.603 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.603 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.604 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.604 DEBUG:boto:Method: GET 2022-09-03T07:28:15.605 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.605 DEBUG:boto:Data: 2022-09-03T07:28:15.606 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.607 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.608 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.608 DEBUG:boto:Params: {} 2022-09-03T07:28:15.609 DEBUG:boto:Token: None 2022-09-03T07:28:15.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.610 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fTHY4ps8W9ZUmPmSeZ0WJP6uAXY= 2022-09-03T07:28:15.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fTHY4ps8W9ZUmPmSeZ0WJP6uAXY=', 'Content-Length': '0'} 2022-09-03T07:28:15.613 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066cd5a194c7887ea-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.614 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.615 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.615 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.616 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.616 DEBUG:boto:Method: GET 2022-09-03T07:28:15.618 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.619 DEBUG:boto:Data: 2022-09-03T07:28:15.620 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.621 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.622 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.624 DEBUG:boto:Params: {} 2022-09-03T07:28:15.625 DEBUG:boto:Token: None 2022-09-03T07:28:15.626 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fTHY4ps8W9ZUmPmSeZ0WJP6uAXY= 2022-09-03T07:28:15.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fTHY4ps8W9ZUmPmSeZ0WJP6uAXY=', 'Content-Length': '0'} 2022-09-03T07:28:15.631 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000abae47becf19f5da-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.633 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.634 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.635 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.635 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.636 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.637 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.639 DEBUG:boto:Data: 2022-09-03T07:28:15.640 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.641 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.641 DEBUG:boto:Params: {} 2022-09-03T07:28:15.642 DEBUG:boto:Token: None 2022-09-03T07:28:15.643 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ibw+KxnP/9qAjIBQyQUJRi7shT4= 2022-09-03T07:28:15.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ibw+KxnP/9qAjIBQyQUJRi7shT4=', 'Content-Length': '0'} 2022-09-03T07:28:15.646 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec36ec5a62d66617-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.647 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.648 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.649 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.650 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.651 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.652 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.653 DEBUG:boto:Data: 2022-09-03T07:28:15.653 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.654 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.655 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.656 DEBUG:boto:Params: {} 2022-09-03T07:28:15.657 DEBUG:boto:Token: None 2022-09-03T07:28:15.657 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ibw+KxnP/9qAjIBQyQUJRi7shT4= 2022-09-03T07:28:15.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ibw+KxnP/9qAjIBQyQUJRi7shT4=', 'Content-Length': '0'} 2022-09-03T07:28:15.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff18e9f47cef95f9-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.663 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:28:15.663 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.664 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.664 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.665 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.665 DEBUG:boto:Method: GET 2022-09-03T07:28:15.666 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.667 DEBUG:boto:Data: 2022-09-03T07:28:15.668 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.669 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.669 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.670 DEBUG:boto:Params: {} 2022-09-03T07:28:15.671 DEBUG:boto:Token: None 2022-09-03T07:28:15.672 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fTHY4ps8W9ZUmPmSeZ0WJP6uAXY= 2022-09-03T07:28:15.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fTHY4ps8W9ZUmPmSeZ0WJP6uAXY=', 'Content-Length': '0'} 2022-09-03T07:28:15.675 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9c948cf3474063d-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.676 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.677 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.677 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.678 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.679 DEBUG:boto:Method: GET 2022-09-03T07:28:15.680 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.681 DEBUG:boto:Data: 2022-09-03T07:28:15.682 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.684 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.685 DEBUG:boto:Params: {} 2022-09-03T07:28:15.686 DEBUG:boto:Token: None 2022-09-03T07:28:15.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:15.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fTHY4ps8W9ZUmPmSeZ0WJP6uAXY= 2022-09-03T07:28:15.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fTHY4ps8W9ZUmPmSeZ0WJP6uAXY=', 'Content-Length': '0'} 2022-09-03T07:28:15.692 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000028e452d028610d5d-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.694 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.695 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.696 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:15.697 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:15.698 DEBUG:boto:Method: GET 2022-09-03T07:28:15.699 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:15.700 DEBUG:boto:Data: 2022-09-03T07:28:15.700 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.701 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.702 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.702 DEBUG:boto:Params: {} 2022-09-03T07:28:15.703 DEBUG:boto:Token: None 2022-09-03T07:28:15.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.708 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1aa1050c185a260-006313020f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.710 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.711 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.712 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.713 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:15.714 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:15.715 DEBUG:boto:Method: GET 2022-09-03T07:28:15.716 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:15.717 DEBUG:boto:Data: 2022-09-03T07:28:15.718 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.719 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.720 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.720 DEBUG:boto:Params: {} 2022-09-03T07:28:15.721 DEBUG:boto:Token: None 2022-09-03T07:28:15.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.728 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb76e0a9e557b2cd-006313020f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.729 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.730 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.731 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.732 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.732 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:15.733 DEBUG:boto:Data: 2022-09-03T07:28:15.734 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.735 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.735 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.736 DEBUG:boto:Params: {} 2022-09-03T07:28:15.736 DEBUG:boto:Token: None 2022-09-03T07:28:15.737 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95 2022-09-03T07:28:15.738 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DI74BUk49IGO3lsQDPKl+aNlGLQ= 2022-09-03T07:28:15.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DI74BUk49IGO3lsQDPKl+aNlGLQ=', 'Content-Length': '0'} 2022-09-03T07:28:15.741 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5e865f6615d1279-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.743 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.743 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.744 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.745 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:15.746 DEBUG:boto:Data: 2022-09-03T07:28:15.747 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.749 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.750 DEBUG:boto:Params: {} 2022-09-03T07:28:15.751 DEBUG:boto:Token: None 2022-09-03T07:28:15.752 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95 2022-09-03T07:28:15.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DI74BUk49IGO3lsQDPKl+aNlGLQ= 2022-09-03T07:28:15.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DI74BUk49IGO3lsQDPKl+aNlGLQ=', 'Content-Length': '0'} 2022-09-03T07:28:15.757 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035a74c2f1e58ccea-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.758 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:28:15.759 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.760 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.761 DEBUG:boto:Method: GET 2022-09-03T07:28:15.762 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:15.763 DEBUG:boto:Data: 2022-09-03T07:28:15.764 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.766 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.767 DEBUG:boto:Params: {} 2022-09-03T07:28:15.768 DEBUG:boto:Token: None 2022-09-03T07:28:15.770 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95 2022-09-03T07:28:15.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nhP8HuRdk75JyZvPi6th/kGdV+E= 2022-09-03T07:28:15.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nhP8HuRdk75JyZvPi6th/kGdV+E=', 'Content-Length': '0'} 2022-09-03T07:28:15.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000027b706f62379147-006313020f-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.775 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:15.775 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:15.776 DEBUG:boto:Method: GET 2022-09-03T07:28:15.776 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:15.777 DEBUG:boto:Data: 2022-09-03T07:28:15.777 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.779 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.779 DEBUG:boto:Params: {} 2022-09-03T07:28:15.780 DEBUG:boto:Token: None 2022-09-03T07:28:15.780 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/obj95 2022-09-03T07:28:15.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nhP8HuRdk75JyZvPi6th/kGdV+E= 2022-09-03T07:28:15.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nhP8HuRdk75JyZvPi6th/kGdV+E=', 'Content-Length': '0'} 2022-09-03T07:28:15.783 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c350e08ed84d2d85-006313020f-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.830 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.831 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.832 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:15.832 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:15.833 DEBUG:boto:Method: GET 2022-09-03T07:28:15.833 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:15.834 DEBUG:boto:Data: 2022-09-03T07:28:15.835 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.836 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.836 DEBUG:boto:Params: {} 2022-09-03T07:28:15.837 DEBUG:boto:Token: None 2022-09-03T07:28:15.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.843 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2f291999aa30bbc-006313020f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.844 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.844 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.845 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.846 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:15.846 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:15.847 DEBUG:boto:Method: GET 2022-09-03T07:28:15.847 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:15.848 DEBUG:boto:Data: 2022-09-03T07:28:15.849 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:15.851 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.852 DEBUG:boto:Params: {} 2022-09-03T07:28:15.853 DEBUG:boto:Token: None 2022-09-03T07:28:15.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg= 2022-09-03T07:28:15.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iOTADEfwRl/oAyA95n/PbMGexgg=', 'Content-Length': '0'} 2022-09-03T07:28:15.860 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb53ed4af8b3bc33-006313020f-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.861 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.862 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:28:15.862 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:28:15.863 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.863 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.864 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.865 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:28:15.865 DEBUG:boto:Data: 2022-09-03T07:28:15.866 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.866 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.867 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.867 DEBUG:boto:Params: {} 2022-09-03T07:28:15.868 DEBUG:boto:Token: None 2022-09-03T07:28:15.868 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yRZdfGw1Bd1f4ivT90EbKwJXlE0= 2022-09-03T07:28:15.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yRZdfGw1Bd1f4ivT90EbKwJXlE0=', 'Content-Length': '0'} 2022-09-03T07:28:15.873 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '30'), ('X-RGW-Bytes-Used', '90'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000067dd20b02da735a7-006313020f-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.875 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.875 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.876 DEBUG:boto:Method: HEAD 2022-09-03T07:28:15.876 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:28:15.877 DEBUG:boto:Data: 2022-09-03T07:28:15.877 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.878 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:15.878 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.879 DEBUG:boto:Params: {} 2022-09-03T07:28:15.880 DEBUG:boto:Token: None 2022-09-03T07:28:15.880 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/ 2022-09-03T07:28:15.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yRZdfGw1Bd1f4ivT90EbKwJXlE0= 2022-09-03T07:28:15.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yRZdfGw1Bd1f4ivT90EbKwJXlE0=', 'Content-Length': '0'} 2022-09-03T07:28:15.884 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '30'), ('X-RGW-Bytes-Used', '90'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cd3360f99b7d1797-006313020f-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.885 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:28:15.885 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.886 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.886 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.887 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.888 DEBUG:boto:Method: GET 2022-09-03T07:28:15.889 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.890 DEBUG:boto:Data: 2022-09-03T07:28:15.891 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.895 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.896 DEBUG:boto:Params: {} 2022-09-03T07:28:15.897 DEBUG:boto:Token: None 2022-09-03T07:28:15.899 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/?versions 2022-09-03T07:28:15.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yqFLmiy0EZnN0rNmI0x7BmyltLk= 2022-09-03T07:28:15.901 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yqFLmiy0EZnN0rNmI0x7BmyltLk=', 'Content-Length': '0'} 2022-09-03T07:28:15.903 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aef158841211e2ca-006313020f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.904 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.906 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:28:15.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:28:15.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:28:15.908 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:28:15.908 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:28:15.909 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:28:15.909 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:28:15.910 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:28:15.910 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:28:15.911 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:28:15.912 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:28:15.913 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:28:15.913 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:28:15.914 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:28:15.914 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:28:15.915 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:28:15.915 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:28:15.916 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:28:15.916 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:28:15.917 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:28:15.917 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:28:15.918 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:28:15.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:28:15.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:28:15.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:28:15.920 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:28:15.920 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:28:15.921 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:28:15.921 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:28:15.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:28:15.923 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:28:15.924 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.924 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.925 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.925 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.926 DEBUG:boto:Method: GET 2022-09-03T07:28:15.926 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.927 DEBUG:boto:Data: 2022-09-03T07:28:15.927 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.928 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:15.928 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.929 DEBUG:boto:Params: {} 2022-09-03T07:28:15.929 DEBUG:boto:Token: None 2022-09-03T07:28:15.930 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/?versions 2022-09-03T07:28:15.931 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yqFLmiy0EZnN0rNmI0x7BmyltLk= 2022-09-03T07:28:15.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yqFLmiy0EZnN0rNmI0x7BmyltLk=', 'Content-Length': '0'} 2022-09-03T07:28:15.935 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000455e9d9477fdd82d-006313020f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.935 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.938 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:28:15.939 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:28:15.939 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:28:15.940 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:28:15.940 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:28:15.941 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:28:15.941 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:28:15.942 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:28:15.942 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:28:15.943 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:28:15.944 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:28:15.944 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:28:15.944 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:28:15.945 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:28:15.946 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:28:15.947 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:28:15.948 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:28:15.949 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:28:15.949 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:28:15.950 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:28:15.950 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:28:15.951 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:28:15.951 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:28:15.952 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:28:15.952 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:28:15.953 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:28:15.953 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:28:15.954 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:28:15.955 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:28:15.955 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:28:15.956 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.957 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.958 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.958 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.959 DEBUG:boto:Method: GET 2022-09-03T07:28:15.959 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.960 DEBUG:boto:Data: 2022-09-03T07:28:15.961 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.961 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.962 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.962 DEBUG:boto:Params: {} 2022-09-03T07:28:15.963 DEBUG:boto:Token: None 2022-09-03T07:28:15.964 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/?versions 2022-09-03T07:28:15.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yqFLmiy0EZnN0rNmI0x7BmyltLk= 2022-09-03T07:28:15.965 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yqFLmiy0EZnN0rNmI0x7BmyltLk=', 'Content-Length': '0'} 2022-09-03T07:28:15.968 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007fd53b6211085d57-006313020f-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.969 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.971 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:15.972 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:15.973 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.973 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.974 DEBUG:boto:Method: GET 2022-09-03T07:28:15.974 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:28:15.975 DEBUG:boto:Data: 2022-09-03T07:28:15.976 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.976 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:15.977 DEBUG:boto:Port: 8000 2022-09-03T07:28:15.977 DEBUG:boto:Params: {} 2022-09-03T07:28:15.978 DEBUG:boto:Token: None 2022-09-03T07:28:15.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:15 GMT /avysmw-75/?versions 2022-09-03T07:28:15.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yqFLmiy0EZnN0rNmI0x7BmyltLk= 2022-09-03T07:28:15.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yqFLmiy0EZnN0rNmI0x7BmyltLk=', 'Content-Length': '0'} 2022-09-03T07:28:15.986 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8bf6ac79afc62fb-006313020f-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:15.987 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:15.989 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:28:15.989 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:15.990 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:15.991 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:15.992 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:15.993 DEBUG:boto:Method: GET 2022-09-03T07:28:15.994 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:15.995 DEBUG:boto:Data: 2022-09-03T07:28:15.995 DEBUG:boto:Headers: {} 2022-09-03T07:28:15.996 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:15.997 DEBUG:boto:Port: 8001 2022-09-03T07:28:15.998 DEBUG:boto:Params: {} 2022-09-03T07:28:15.999 DEBUG:boto:Token: None 2022-09-03T07:28:16.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nZ3aUF/Udy3aNiolrfvV2bx2BEY= 2022-09-03T07:28:16.003 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nZ3aUF/Udy3aNiolrfvV2bx2BEY=', 'Content-Length': '0'} 2022-09-03T07:28:16.005 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e52a5508e7c3ab27-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.006 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:16.006 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:16.007 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.007 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.008 DEBUG:boto:Method: GET 2022-09-03T07:28:16.008 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.009 DEBUG:boto:Data: 2022-09-03T07:28:16.010 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.011 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.011 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.012 DEBUG:boto:Params: {} 2022-09-03T07:28:16.012 DEBUG:boto:Token: None 2022-09-03T07:28:16.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nZ3aUF/Udy3aNiolrfvV2bx2BEY= 2022-09-03T07:28:16.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nZ3aUF/Udy3aNiolrfvV2bx2BEY=', 'Content-Length': '0'} 2022-09-03T07:28:16.019 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c34f816824501a6-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.020 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:16.020 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:16.021 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.021 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.022 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.023 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.023 DEBUG:boto:Data: 2022-09-03T07:28:16.024 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.024 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.025 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.025 DEBUG:boto:Params: {} 2022-09-03T07:28:16.026 DEBUG:boto:Token: None 2022-09-03T07:28:16.027 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zrNbaZb2GDGFwacozWu3oSJRr8c= 2022-09-03T07:28:16.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zrNbaZb2GDGFwacozWu3oSJRr8c=', 'Content-Length': '0'} 2022-09-03T07:28:16.030 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098d8291f4b8969ed-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.031 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:16.031 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:16.032 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.032 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.033 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.033 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.034 DEBUG:boto:Data: 2022-09-03T07:28:16.035 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.035 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.036 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.036 DEBUG:boto:Params: {} 2022-09-03T07:28:16.037 DEBUG:boto:Token: None 2022-09-03T07:28:16.038 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zrNbaZb2GDGFwacozWu3oSJRr8c= 2022-09-03T07:28:16.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zrNbaZb2GDGFwacozWu3oSJRr8c=', 'Content-Length': '0'} 2022-09-03T07:28:16.041 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000747e888bec74dc34-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.042 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:28:16.043 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:16.043 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:16.044 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.044 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.045 DEBUG:boto:Method: GET 2022-09-03T07:28:16.046 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.046 DEBUG:boto:Data: 2022-09-03T07:28:16.047 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.048 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.048 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.049 DEBUG:boto:Params: {} 2022-09-03T07:28:16.049 DEBUG:boto:Token: None 2022-09-03T07:28:16.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nZ3aUF/Udy3aNiolrfvV2bx2BEY= 2022-09-03T07:28:16.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nZ3aUF/Udy3aNiolrfvV2bx2BEY=', 'Content-Length': '0'} 2022-09-03T07:28:16.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf0003ba37ec5484-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.054 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:16.055 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:16.055 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.056 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.056 DEBUG:boto:Method: GET 2022-09-03T07:28:16.057 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.058 DEBUG:boto:Data: 2022-09-03T07:28:16.058 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.059 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.059 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.060 DEBUG:boto:Params: {} 2022-09-03T07:28:16.060 DEBUG:boto:Token: None 2022-09-03T07:28:16.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:28:16.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nZ3aUF/Udy3aNiolrfvV2bx2BEY= 2022-09-03T07:28:16.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nZ3aUF/Udy3aNiolrfvV2bx2BEY=', 'Content-Length': '0'} 2022-09-03T07:28:16.064 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e4152a4e42c7e0a2-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.106 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.107 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.107 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:16.108 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:16.108 DEBUG:boto:Method: GET 2022-09-03T07:28:16.109 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:28:16.109 DEBUG:boto:Data: 2022-09-03T07:28:16.110 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.111 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.112 DEBUG:boto:Params: {} 2022-09-03T07:28:16.112 DEBUG:boto:Token: None 2022-09-03T07:28:16.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.113 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.117 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000081ee774715fa2057-0063130210-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.119 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.120 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.121 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.122 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:16.123 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:16.123 DEBUG:boto:Method: GET 2022-09-03T07:28:16.124 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:28:16.125 DEBUG:boto:Data: 2022-09-03T07:28:16.126 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.126 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.127 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.128 DEBUG:boto:Params: {} 2022-09-03T07:28:16.128 DEBUG:boto:Token: None 2022-09-03T07:28:16.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.133 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008c5b2b9439a9b04a-0063130210-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.133 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.134 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:16.135 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:16.136 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.137 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:28:16.137 DEBUG:boto:Data: 2022-09-03T07:28:16.138 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.138 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.139 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.140 DEBUG:boto:Params: {} 2022-09-03T07:28:16.141 DEBUG:boto:Token: None 2022-09-03T07:28:16.141 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11 2022-09-03T07:28:16.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dNRUVKTYJ3Na2SaTo+xlsseG2k= 2022-09-03T07:28:16.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dNRUVKTYJ3Na2SaTo+xlsseG2k=', 'Content-Length': '0'} 2022-09-03T07:28:16.146 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004273aeab67d3492f-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.147 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:16.148 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:16.149 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.150 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:28:16.151 DEBUG:boto:Data: 2022-09-03T07:28:16.152 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.153 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.154 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.155 DEBUG:boto:Params: {} 2022-09-03T07:28:16.157 DEBUG:boto:Token: None 2022-09-03T07:28:16.158 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11 2022-09-03T07:28:16.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8dNRUVKTYJ3Na2SaTo+xlsseG2k= 2022-09-03T07:28:16.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8dNRUVKTYJ3Na2SaTo+xlsseG2k=', 'Content-Length': '0'} 2022-09-03T07:28:16.163 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f9679c561d0081c-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.164 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:28:16.165 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:16.166 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:16.167 DEBUG:boto:Method: GET 2022-09-03T07:28:16.167 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:28:16.168 DEBUG:boto:Data: 2022-09-03T07:28:16.168 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.169 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.170 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.171 DEBUG:boto:Params: {} 2022-09-03T07:28:16.172 DEBUG:boto:Token: None 2022-09-03T07:28:16.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11 2022-09-03T07:28:16.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yq10C3rge980Y2WVACg6StE7IkU= 2022-09-03T07:28:16.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yq10C3rge980Y2WVACg6StE7IkU=', 'Content-Length': '0'} 2022-09-03T07:28:16.178 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052bd9331ed207deb-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.179 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:28:16.180 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:28:16.182 DEBUG:boto:Method: GET 2022-09-03T07:28:16.183 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:28:16.183 DEBUG:boto:Data: 2022-09-03T07:28:16.184 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.185 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.185 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.186 DEBUG:boto:Params: {} 2022-09-03T07:28:16.186 DEBUG:boto:Token: None 2022-09-03T07:28:16.187 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj11 2022-09-03T07:28:16.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yq10C3rge980Y2WVACg6StE7IkU= 2022-09-03T07:28:16.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yq10C3rge980Y2WVACg6StE7IkU=', 'Content-Length': '0'} 2022-09-03T07:28:16.192 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007b232bd19798e741-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.194 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.195 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.195 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:16.196 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:16.196 DEBUG:boto:Method: GET 2022-09-03T07:28:16.197 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:28:16.198 DEBUG:boto:Data: 2022-09-03T07:28:16.199 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.200 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.201 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.202 DEBUG:boto:Params: {} 2022-09-03T07:28:16.203 DEBUG:boto:Token: None 2022-09-03T07:28:16.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000068fb6d6d81ae5e14-0063130210-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.210 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.212 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.213 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.214 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:16.215 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:28:16.215 DEBUG:boto:Method: GET 2022-09-03T07:28:16.216 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:28:16.217 DEBUG:boto:Data: 2022-09-03T07:28:16.217 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.219 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.220 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.221 DEBUG:boto:Params: {} 2022-09-03T07:28:16.222 DEBUG:boto:Token: None 2022-09-03T07:28:16.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.228 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c4c9ad0b95dc422a-0063130210-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.270 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.271 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:28:16.273 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.273 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.275 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.275 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.276 DEBUG:boto:Method: GET 2022-09-03T07:28:16.276 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.277 DEBUG:boto:Data: 2022-09-03T07:28:16.278 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.279 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.280 DEBUG:boto:Params: {} 2022-09-03T07:28:16.281 DEBUG:boto:Token: None 2022-09-03T07:28:16.281 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7ltRvcv9YSr65IvlS1ti7/iBwE0= 2022-09-03T07:28:16.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7ltRvcv9YSr65IvlS1ti7/iBwE0=', 'Content-Length': '0'} 2022-09-03T07:28:16.285 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000025c8d235af9ece04-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.287 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.288 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.288 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.289 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.290 DEBUG:boto:Method: GET 2022-09-03T07:28:16.291 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.292 DEBUG:boto:Data: 2022-09-03T07:28:16.293 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.294 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.295 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.296 DEBUG:boto:Params: {} 2022-09-03T07:28:16.296 DEBUG:boto:Token: None 2022-09-03T07:28:16.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7ltRvcv9YSr65IvlS1ti7/iBwE0= 2022-09-03T07:28:16.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7ltRvcv9YSr65IvlS1ti7/iBwE0=', 'Content-Length': '0'} 2022-09-03T07:28:16.301 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c529114001b61739-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.302 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.303 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.303 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.304 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.304 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.305 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.305 DEBUG:boto:Data: 2022-09-03T07:28:16.306 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.306 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.307 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.307 DEBUG:boto:Params: {} 2022-09-03T07:28:16.308 DEBUG:boto:Token: None 2022-09-03T07:28:16.308 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Q3Kp+2fR5Y9yuMKZaE1ikeel+U= 2022-09-03T07:28:16.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Q3Kp+2fR5Y9yuMKZaE1ikeel+U=', 'Content-Length': '0'} 2022-09-03T07:28:16.312 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007720a14f21fbd045-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.313 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.313 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.314 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.314 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.316 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.317 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.318 DEBUG:boto:Data: 2022-09-03T07:28:16.319 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.319 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.320 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.320 DEBUG:boto:Params: {} 2022-09-03T07:28:16.321 DEBUG:boto:Token: None 2022-09-03T07:28:16.322 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Q3Kp+2fR5Y9yuMKZaE1ikeel+U= 2022-09-03T07:28:16.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Q3Kp+2fR5Y9yuMKZaE1ikeel+U=', 'Content-Length': '0'} 2022-09-03T07:28:16.325 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007745538ef3ce16eb-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.325 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:28:16.326 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.327 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.327 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.328 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.329 DEBUG:boto:Method: GET 2022-09-03T07:28:16.330 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.331 DEBUG:boto:Data: 2022-09-03T07:28:16.332 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.333 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.334 DEBUG:boto:Params: {} 2022-09-03T07:28:16.335 DEBUG:boto:Token: None 2022-09-03T07:28:16.336 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7ltRvcv9YSr65IvlS1ti7/iBwE0= 2022-09-03T07:28:16.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7ltRvcv9YSr65IvlS1ti7/iBwE0=', 'Content-Length': '0'} 2022-09-03T07:28:16.341 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a001ce972d3fadf6-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.343 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.344 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.345 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.346 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.347 DEBUG:boto:Method: GET 2022-09-03T07:28:16.348 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.349 DEBUG:boto:Data: 2022-09-03T07:28:16.350 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.351 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.352 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.352 DEBUG:boto:Params: {} 2022-09-03T07:28:16.353 DEBUG:boto:Token: None 2022-09-03T07:28:16.354 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:28:16.355 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7ltRvcv9YSr65IvlS1ti7/iBwE0= 2022-09-03T07:28:16.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7ltRvcv9YSr65IvlS1ti7/iBwE0=', 'Content-Length': '0'} 2022-09-03T07:28:16.358 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3ac6c1d4b2d7094-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.359 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.359 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.360 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:16.360 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:16.361 DEBUG:boto:Method: GET 2022-09-03T07:28:16.361 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:28:16.362 DEBUG:boto:Data: 2022-09-03T07:28:16.363 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.364 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.364 DEBUG:boto:Params: {} 2022-09-03T07:28:16.365 DEBUG:boto:Token: None 2022-09-03T07:28:16.365 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.370 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001eb30f8d80b8488a-0063130210-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.370 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.371 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.372 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.373 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:16.373 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:16.374 DEBUG:boto:Method: GET 2022-09-03T07:28:16.374 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:28:16.375 DEBUG:boto:Data: 2022-09-03T07:28:16.376 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.376 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.377 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.377 DEBUG:boto:Params: {} 2022-09-03T07:28:16.378 DEBUG:boto:Token: None 2022-09-03T07:28:16.379 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.382 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bbda1109e3c3d8c4-0063130210-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.383 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.384 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.384 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.385 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.385 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:28:16.386 DEBUG:boto:Data: 2022-09-03T07:28:16.386 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.387 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.387 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.388 DEBUG:boto:Params: {} 2022-09-03T07:28:16.388 DEBUG:boto:Token: None 2022-09-03T07:28:16.389 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115 2022-09-03T07:28:16.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kKRr2JkfbAlJ7LmqTaDq96fX1qU= 2022-09-03T07:28:16.390 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kKRr2JkfbAlJ7LmqTaDq96fX1qU=', 'Content-Length': '0'} 2022-09-03T07:28:16.392 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087429db5aff7cd3f-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.393 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.393 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.394 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.395 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:28:16.395 DEBUG:boto:Data: 2022-09-03T07:28:16.396 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.396 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.399 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.400 DEBUG:boto:Params: {} 2022-09-03T07:28:16.400 DEBUG:boto:Token: None 2022-09-03T07:28:16.401 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115 2022-09-03T07:28:16.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kKRr2JkfbAlJ7LmqTaDq96fX1qU= 2022-09-03T07:28:16.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kKRr2JkfbAlJ7LmqTaDq96fX1qU=', 'Content-Length': '0'} 2022-09-03T07:28:16.404 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000947a91eba41c146a-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.405 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:28:16.406 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.406 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.407 DEBUG:boto:Method: GET 2022-09-03T07:28:16.408 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:28:16.408 DEBUG:boto:Data: 2022-09-03T07:28:16.409 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.409 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.410 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.410 DEBUG:boto:Params: {} 2022-09-03T07:28:16.411 DEBUG:boto:Token: None 2022-09-03T07:28:16.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115 2022-09-03T07:28:16.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2B/eNGYAcHZ8zCZlhjwkI1+bQQ= 2022-09-03T07:28:16.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2B/eNGYAcHZ8zCZlhjwkI1+bQQ=', 'Content-Length': '0'} 2022-09-03T07:28:16.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a82ec36c8b8da27-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.458 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:28:16.459 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:28:16.459 DEBUG:boto:Method: GET 2022-09-03T07:28:16.460 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:28:16.461 DEBUG:boto:Data: 2022-09-03T07:28:16.461 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.462 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.463 DEBUG:boto:Params: {} 2022-09-03T07:28:16.464 DEBUG:boto:Token: None 2022-09-03T07:28:16.465 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj115 2022-09-03T07:28:16.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G2B/eNGYAcHZ8zCZlhjwkI1+bQQ= 2022-09-03T07:28:16.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G2B/eNGYAcHZ8zCZlhjwkI1+bQQ=', 'Content-Length': '0'} 2022-09-03T07:28:16.468 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001dd911ca34da244d-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.469 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.470 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.470 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:16.471 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:16.471 DEBUG:boto:Method: GET 2022-09-03T07:28:16.472 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:28:16.473 DEBUG:boto:Data: 2022-09-03T07:28:16.474 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.475 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.476 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.477 DEBUG:boto:Params: {} 2022-09-03T07:28:16.478 DEBUG:boto:Token: None 2022-09-03T07:28:16.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.483 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c75bd3b19de94f4-0063130210-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.484 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.486 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.487 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.487 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:16.488 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:28:16.489 DEBUG:boto:Method: GET 2022-09-03T07:28:16.489 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:28:16.490 DEBUG:boto:Data: 2022-09-03T07:28:16.490 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.491 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.492 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.492 DEBUG:boto:Params: {} 2022-09-03T07:28:16.493 DEBUG:boto:Token: None 2022-09-03T07:28:16.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.497 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000071e4a6a7ed61e1a1-0063130210-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.498 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.499 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:28:16.499 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.500 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.500 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.501 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.502 DEBUG:boto:Method: GET 2022-09-03T07:28:16.502 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.503 DEBUG:boto:Data: 2022-09-03T07:28:16.503 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.504 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.504 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.505 DEBUG:boto:Params: {} 2022-09-03T07:28:16.506 DEBUG:boto:Token: None 2022-09-03T07:28:16.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNDMS3scgo0MIYYEOIzYUbykaEM= 2022-09-03T07:28:16.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNDMS3scgo0MIYYEOIzYUbykaEM=', 'Content-Length': '0'} 2022-09-03T07:28:16.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ea416bf7312c5cd1-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.513 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.514 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.515 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.516 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.517 DEBUG:boto:Method: GET 2022-09-03T07:28:16.518 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.519 DEBUG:boto:Data: 2022-09-03T07:28:16.520 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.521 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.522 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.524 DEBUG:boto:Params: {} 2022-09-03T07:28:16.524 DEBUG:boto:Token: None 2022-09-03T07:28:16.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.526 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNDMS3scgo0MIYYEOIzYUbykaEM= 2022-09-03T07:28:16.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNDMS3scgo0MIYYEOIzYUbykaEM=', 'Content-Length': '0'} 2022-09-03T07:28:16.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad2b8d7ad7ff955f-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.531 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.532 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.532 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.533 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.533 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.535 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.535 DEBUG:boto:Data: 2022-09-03T07:28:16.536 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.537 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.539 DEBUG:boto:Params: {} 2022-09-03T07:28:16.539 DEBUG:boto:Token: None 2022-09-03T07:28:16.540 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VSoBrjeJm+INTPwJNB80bBWCs9g= 2022-09-03T07:28:16.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VSoBrjeJm+INTPwJNB80bBWCs9g=', 'Content-Length': '0'} 2022-09-03T07:28:16.545 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076bfc8ee7743e27d-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.547 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.548 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.549 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.550 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.551 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.551 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.552 DEBUG:boto:Data: 2022-09-03T07:28:16.552 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.553 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.553 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.554 DEBUG:boto:Params: {} 2022-09-03T07:28:16.554 DEBUG:boto:Token: None 2022-09-03T07:28:16.556 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VSoBrjeJm+INTPwJNB80bBWCs9g= 2022-09-03T07:28:16.557 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VSoBrjeJm+INTPwJNB80bBWCs9g=', 'Content-Length': '0'} 2022-09-03T07:28:16.559 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a7e5d46b2435ce2-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.560 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:28:16.560 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.561 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.561 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.562 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.563 DEBUG:boto:Method: GET 2022-09-03T07:28:16.564 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.565 DEBUG:boto:Data: 2022-09-03T07:28:16.566 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.568 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.569 DEBUG:boto:Params: {} 2022-09-03T07:28:16.570 DEBUG:boto:Token: None 2022-09-03T07:28:16.571 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNDMS3scgo0MIYYEOIzYUbykaEM= 2022-09-03T07:28:16.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNDMS3scgo0MIYYEOIzYUbykaEM=', 'Content-Length': '0'} 2022-09-03T07:28:16.585 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e22a59a70c35550-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.626 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.627 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.628 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.629 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.630 DEBUG:boto:Method: GET 2022-09-03T07:28:16.631 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.631 DEBUG:boto:Data: 2022-09-03T07:28:16.632 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.632 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.633 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.635 DEBUG:boto:Params: {} 2022-09-03T07:28:16.637 DEBUG:boto:Token: None 2022-09-03T07:28:16.638 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:28:16.639 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zNDMS3scgo0MIYYEOIzYUbykaEM= 2022-09-03T07:28:16.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zNDMS3scgo0MIYYEOIzYUbykaEM=', 'Content-Length': '0'} 2022-09-03T07:28:16.643 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a032e45a59a9529-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.644 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.645 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.646 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:16.647 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:16.648 DEBUG:boto:Method: GET 2022-09-03T07:28:16.648 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:28:16.649 DEBUG:boto:Data: 2022-09-03T07:28:16.649 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.651 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.651 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.652 DEBUG:boto:Params: {} 2022-09-03T07:28:16.653 DEBUG:boto:Token: None 2022-09-03T07:28:16.653 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.654 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.655 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.657 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000378955e054a570a8-0063130210-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.658 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.660 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.661 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.662 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:16.663 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:16.664 DEBUG:boto:Method: GET 2022-09-03T07:28:16.665 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:28:16.665 DEBUG:boto:Data: 2022-09-03T07:28:16.666 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.666 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.667 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.667 DEBUG:boto:Params: {} 2022-09-03T07:28:16.668 DEBUG:boto:Token: None 2022-09-03T07:28:16.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.672 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000552437b724fc1dd3-0063130210-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.673 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.674 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.674 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.675 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.675 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:28:16.676 DEBUG:boto:Data: 2022-09-03T07:28:16.676 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.677 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.678 DEBUG:boto:Params: {} 2022-09-03T07:28:16.678 DEBUG:boto:Token: None 2022-09-03T07:28:16.679 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116 2022-09-03T07:28:16.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DL3i96i8+wdwg0GlcprP2J5qeRI= 2022-09-03T07:28:16.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DL3i96i8+wdwg0GlcprP2J5qeRI=', 'Content-Length': '0'} 2022-09-03T07:28:16.682 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a006f2fed05d2a66-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.683 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.683 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.684 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.684 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:28:16.685 DEBUG:boto:Data: 2022-09-03T07:28:16.685 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.686 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.686 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.687 DEBUG:boto:Params: {} 2022-09-03T07:28:16.687 DEBUG:boto:Token: None 2022-09-03T07:28:16.688 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116 2022-09-03T07:28:16.688 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DL3i96i8+wdwg0GlcprP2J5qeRI= 2022-09-03T07:28:16.689 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DL3i96i8+wdwg0GlcprP2J5qeRI=', 'Content-Length': '0'} 2022-09-03T07:28:16.691 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046399b20470985e0-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.692 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:28:16.692 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.693 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.693 DEBUG:boto:Method: GET 2022-09-03T07:28:16.694 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:28:16.694 DEBUG:boto:Data: 2022-09-03T07:28:16.695 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.696 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.696 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.697 DEBUG:boto:Params: {} 2022-09-03T07:28:16.697 DEBUG:boto:Token: None 2022-09-03T07:28:16.698 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116 2022-09-03T07:28:16.699 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n7CywOkie4BMA+WHfrsjZAwonrM= 2022-09-03T07:28:16.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n7CywOkie4BMA+WHfrsjZAwonrM=', 'Content-Length': '0'} 2022-09-03T07:28:16.701 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000948885fc4c095463-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.702 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:28:16.702 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:28:16.703 DEBUG:boto:Method: GET 2022-09-03T07:28:16.704 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:28:16.704 DEBUG:boto:Data: 2022-09-03T07:28:16.705 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.705 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.706 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.706 DEBUG:boto:Params: {} 2022-09-03T07:28:16.707 DEBUG:boto:Token: None 2022-09-03T07:28:16.707 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj116 2022-09-03T07:28:16.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n7CywOkie4BMA+WHfrsjZAwonrM= 2022-09-03T07:28:16.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n7CywOkie4BMA+WHfrsjZAwonrM=', 'Content-Length': '0'} 2022-09-03T07:28:16.711 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074bd3ec795d5fc27-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.711 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.712 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.712 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:16.713 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:16.714 DEBUG:boto:Method: GET 2022-09-03T07:28:16.714 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:28:16.715 DEBUG:boto:Data: 2022-09-03T07:28:16.715 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.716 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.717 DEBUG:boto:Params: {} 2022-09-03T07:28:16.717 DEBUG:boto:Token: None 2022-09-03T07:28:16.718 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.722 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000225a88e342386126-0063130210-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.722 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.723 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.724 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.724 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:16.725 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:28:16.725 DEBUG:boto:Method: GET 2022-09-03T07:28:16.726 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:28:16.727 DEBUG:boto:Data: 2022-09-03T07:28:16.727 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.728 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.728 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.729 DEBUG:boto:Params: {} 2022-09-03T07:28:16.729 DEBUG:boto:Token: None 2022-09-03T07:28:16.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.734 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000869baa35911ae994-0063130210-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.735 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.736 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:28:16.737 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.738 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.739 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.740 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.741 DEBUG:boto:Method: GET 2022-09-03T07:28:16.742 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.743 DEBUG:boto:Data: 2022-09-03T07:28:16.744 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.745 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.746 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.747 DEBUG:boto:Params: {} 2022-09-03T07:28:16.748 DEBUG:boto:Token: None 2022-09-03T07:28:16.749 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ml1dADoOVm0pFfBJh31d4rYmdZQ= 2022-09-03T07:28:16.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ml1dADoOVm0pFfBJh31d4rYmdZQ=', 'Content-Length': '0'} 2022-09-03T07:28:16.754 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3dc0bed97b8c0d4-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.798 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.799 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.800 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.801 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.802 DEBUG:boto:Method: GET 2022-09-03T07:28:16.803 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.803 DEBUG:boto:Data: 2022-09-03T07:28:16.804 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.805 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.806 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.807 DEBUG:boto:Params: {} 2022-09-03T07:28:16.808 DEBUG:boto:Token: None 2022-09-03T07:28:16.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ml1dADoOVm0pFfBJh31d4rYmdZQ= 2022-09-03T07:28:16.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ml1dADoOVm0pFfBJh31d4rYmdZQ=', 'Content-Length': '0'} 2022-09-03T07:28:16.812 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000bde28419cd17541-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.812 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.813 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.814 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.814 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.815 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.815 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.816 DEBUG:boto:Data: 2022-09-03T07:28:16.816 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.817 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.817 DEBUG:boto:Params: {} 2022-09-03T07:28:16.818 DEBUG:boto:Token: None 2022-09-03T07:28:16.819 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7oV2MlB0fmcuSbTQi7AKS/eF+Xc= 2022-09-03T07:28:16.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7oV2MlB0fmcuSbTQi7AKS/eF+Xc=', 'Content-Length': '0'} 2022-09-03T07:28:16.822 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fc450f4a42f353e-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.823 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.823 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.824 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.824 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.825 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.825 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.826 DEBUG:boto:Data: 2022-09-03T07:28:16.826 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.827 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.827 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.827 DEBUG:boto:Params: {} 2022-09-03T07:28:16.828 DEBUG:boto:Token: None 2022-09-03T07:28:16.829 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7oV2MlB0fmcuSbTQi7AKS/eF+Xc= 2022-09-03T07:28:16.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7oV2MlB0fmcuSbTQi7AKS/eF+Xc=', 'Content-Length': '0'} 2022-09-03T07:28:16.832 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb023d47b405626b-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.833 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:28:16.834 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.835 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.836 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.837 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.839 DEBUG:boto:Method: GET 2022-09-03T07:28:16.840 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.840 DEBUG:boto:Data: 2022-09-03T07:28:16.841 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.842 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.843 DEBUG:boto:Params: {} 2022-09-03T07:28:16.843 DEBUG:boto:Token: None 2022-09-03T07:28:16.844 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ml1dADoOVm0pFfBJh31d4rYmdZQ= 2022-09-03T07:28:16.845 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ml1dADoOVm0pFfBJh31d4rYmdZQ=', 'Content-Length': '0'} 2022-09-03T07:28:16.849 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9951012ec5430a2-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.849 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.850 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.851 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.851 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.852 DEBUG:boto:Method: GET 2022-09-03T07:28:16.852 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.853 DEBUG:boto:Data: 2022-09-03T07:28:16.853 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.854 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.855 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.855 DEBUG:boto:Params: {} 2022-09-03T07:28:16.856 DEBUG:boto:Token: None 2022-09-03T07:28:16.856 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:28:16.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ml1dADoOVm0pFfBJh31d4rYmdZQ= 2022-09-03T07:28:16.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ml1dADoOVm0pFfBJh31d4rYmdZQ=', 'Content-Length': '0'} 2022-09-03T07:28:16.860 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000840d98f8213772ae-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.861 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.861 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.862 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:16.862 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:16.863 DEBUG:boto:Method: GET 2022-09-03T07:28:16.863 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:28:16.864 DEBUG:boto:Data: 2022-09-03T07:28:16.864 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.865 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.866 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.866 DEBUG:boto:Params: {} 2022-09-03T07:28:16.867 DEBUG:boto:Token: None 2022-09-03T07:28:16.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.871 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c12335c758b5b71e-0063130210-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.872 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.873 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.874 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.874 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:16.875 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:16.876 DEBUG:boto:Method: GET 2022-09-03T07:28:16.877 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:28:16.878 DEBUG:boto:Data: 2022-09-03T07:28:16.879 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.880 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.882 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.882 DEBUG:boto:Params: {} 2022-09-03T07:28:16.883 DEBUG:boto:Token: None 2022-09-03T07:28:16.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.888 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000008f810eb885db16c-0063130210-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.888 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.889 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.890 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.890 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.891 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:28:16.891 DEBUG:boto:Data: 2022-09-03T07:28:16.892 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.893 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.893 DEBUG:boto:Params: {} 2022-09-03T07:28:16.894 DEBUG:boto:Token: None 2022-09-03T07:28:16.895 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124 2022-09-03T07:28:16.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4y/ffipqtTU4wuVHfSgmtdS1tQ= 2022-09-03T07:28:16.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4y/ffipqtTU4wuVHfSgmtdS1tQ=', 'Content-Length': '0'} 2022-09-03T07:28:16.898 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066f6193db20f7621-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.899 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.900 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.901 DEBUG:boto:Method: HEAD 2022-09-03T07:28:16.901 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:28:16.902 DEBUG:boto:Data: 2022-09-03T07:28:16.903 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.903 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.904 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.905 DEBUG:boto:Params: {} 2022-09-03T07:28:16.906 DEBUG:boto:Token: None 2022-09-03T07:28:16.908 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124 2022-09-03T07:28:16.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H4y/ffipqtTU4wuVHfSgmtdS1tQ= 2022-09-03T07:28:16.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H4y/ffipqtTU4wuVHfSgmtdS1tQ=', 'Content-Length': '0'} 2022-09-03T07:28:16.912 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000eeaf7c68cb7072-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.912 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:28:16.913 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.914 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.914 DEBUG:boto:Method: GET 2022-09-03T07:28:16.915 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:28:16.915 DEBUG:boto:Data: 2022-09-03T07:28:16.916 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.916 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.917 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.918 DEBUG:boto:Params: {} 2022-09-03T07:28:16.918 DEBUG:boto:Token: None 2022-09-03T07:28:16.919 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124 2022-09-03T07:28:16.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ii5vyCF3FreFgj27Tpw+1GzWBW8= 2022-09-03T07:28:16.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ii5vyCF3FreFgj27Tpw+1GzWBW8=', 'Content-Length': '0'} 2022-09-03T07:28:16.922 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d9ddf0178c0ed05-0063130210-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.966 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:28:16.967 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:28:16.968 DEBUG:boto:Method: GET 2022-09-03T07:28:16.969 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:28:16.969 DEBUG:boto:Data: 2022-09-03T07:28:16.970 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.970 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:16.972 DEBUG:boto:Port: 8000 2022-09-03T07:28:16.973 DEBUG:boto:Params: {} 2022-09-03T07:28:16.974 DEBUG:boto:Token: None 2022-09-03T07:28:16.974 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/obj124 2022-09-03T07:28:16.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ii5vyCF3FreFgj27Tpw+1GzWBW8= 2022-09-03T07:28:16.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ii5vyCF3FreFgj27Tpw+1GzWBW8=', 'Content-Length': '0'} 2022-09-03T07:28:16.978 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086a83a71028b0d54-0063130210-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.979 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.980 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.980 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:16.981 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:16.982 DEBUG:boto:Method: GET 2022-09-03T07:28:16.982 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:28:16.983 DEBUG:boto:Data: 2022-09-03T07:28:16.984 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:16.986 DEBUG:boto:Port: 8001 2022-09-03T07:28:16.987 DEBUG:boto:Params: {} 2022-09-03T07:28:16.988 DEBUG:boto:Token: None 2022-09-03T07:28:16.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:16 GMT /avysmw-75/ 2022-09-03T07:28:16.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA= 2022-09-03T07:28:16.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4k/TqqC1vZpW8J9dkpdnU+1r0RA=', 'Content-Length': '0'} 2022-09-03T07:28:16.992 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000014bdea752de03d64-0063130210-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:16.993 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:16.994 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:16.995 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:16.995 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:16.996 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:28:16.997 DEBUG:boto:Method: GET 2022-09-03T07:28:16.997 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:28:16.998 DEBUG:boto:Data: 2022-09-03T07:28:16.998 DEBUG:boto:Headers: {} 2022-09-03T07:28:16.999 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.000 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.000 DEBUG:boto:Params: {} 2022-09-03T07:28:17.001 DEBUG:boto:Token: None 2022-09-03T07:28:17.002 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.007 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb913c22e9c50ee2-0063130211-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.050 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.051 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:28:17.052 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.052 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.053 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.053 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.054 DEBUG:boto:Method: GET 2022-09-03T07:28:17.054 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.055 DEBUG:boto:Data: 2022-09-03T07:28:17.056 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.057 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.057 DEBUG:boto:Params: {} 2022-09-03T07:28:17.058 DEBUG:boto:Token: None 2022-09-03T07:28:17.059 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tLHH+IkY4ylY9r18QtBtZIIvlQk= 2022-09-03T07:28:17.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tLHH+IkY4ylY9r18QtBtZIIvlQk=', 'Content-Length': '0'} 2022-09-03T07:28:17.062 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000637630953b07f855-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.063 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.063 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.064 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.064 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.065 DEBUG:boto:Method: GET 2022-09-03T07:28:17.066 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.066 DEBUG:boto:Data: 2022-09-03T07:28:17.067 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.067 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.068 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.068 DEBUG:boto:Params: {} 2022-09-03T07:28:17.069 DEBUG:boto:Token: None 2022-09-03T07:28:17.070 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.070 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tLHH+IkY4ylY9r18QtBtZIIvlQk= 2022-09-03T07:28:17.071 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tLHH+IkY4ylY9r18QtBtZIIvlQk=', 'Content-Length': '0'} 2022-09-03T07:28:17.073 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000871872368f853d5e-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.074 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.075 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.075 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.076 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.076 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.077 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.077 DEBUG:boto:Data: 2022-09-03T07:28:17.078 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.078 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.079 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.079 DEBUG:boto:Params: {} 2022-09-03T07:28:17.080 DEBUG:boto:Token: None 2022-09-03T07:28:17.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xSj2fw2IGVzc70s/ujCEwfLMtQo= 2022-09-03T07:28:17.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xSj2fw2IGVzc70s/ujCEwfLMtQo=', 'Content-Length': '0'} 2022-09-03T07:28:17.084 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d409957eb2489db2-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.085 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.085 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.086 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.086 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.087 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.087 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.088 DEBUG:boto:Data: 2022-09-03T07:28:17.088 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.089 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.089 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.090 DEBUG:boto:Params: {} 2022-09-03T07:28:17.090 DEBUG:boto:Token: None 2022-09-03T07:28:17.091 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xSj2fw2IGVzc70s/ujCEwfLMtQo= 2022-09-03T07:28:17.092 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xSj2fw2IGVzc70s/ujCEwfLMtQo=', 'Content-Length': '0'} 2022-09-03T07:28:17.094 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae14eb5f8680a7db-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.095 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:28:17.095 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.096 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.097 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.097 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.098 DEBUG:boto:Method: GET 2022-09-03T07:28:17.098 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.099 DEBUG:boto:Data: 2022-09-03T07:28:17.099 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.100 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.100 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.101 DEBUG:boto:Params: {} 2022-09-03T07:28:17.102 DEBUG:boto:Token: None 2022-09-03T07:28:17.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tLHH+IkY4ylY9r18QtBtZIIvlQk= 2022-09-03T07:28:17.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tLHH+IkY4ylY9r18QtBtZIIvlQk=', 'Content-Length': '0'} 2022-09-03T07:28:17.106 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000585451d52e41496a-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.107 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.107 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.108 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.108 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.109 DEBUG:boto:Method: GET 2022-09-03T07:28:17.109 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.110 DEBUG:boto:Data: 2022-09-03T07:28:17.111 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.111 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.112 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.112 DEBUG:boto:Params: {} 2022-09-03T07:28:17.113 DEBUG:boto:Token: None 2022-09-03T07:28:17.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:28:17.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tLHH+IkY4ylY9r18QtBtZIIvlQk= 2022-09-03T07:28:17.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tLHH+IkY4ylY9r18QtBtZIIvlQk=', 'Content-Length': '0'} 2022-09-03T07:28:17.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e64c9e6bc75adba-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.118 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.118 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.119 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:17.120 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:17.120 DEBUG:boto:Method: GET 2022-09-03T07:28:17.121 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:28:17.121 DEBUG:boto:Data: 2022-09-03T07:28:17.122 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.123 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.123 DEBUG:boto:Params: {} 2022-09-03T07:28:17.124 DEBUG:boto:Token: None 2022-09-03T07:28:17.124 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.128 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004651d54b33ce5767-0063130211-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.170 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.171 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.172 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.172 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:17.173 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:17.173 DEBUG:boto:Method: GET 2022-09-03T07:28:17.174 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:28:17.175 DEBUG:boto:Data: 2022-09-03T07:28:17.176 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.177 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.178 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.178 DEBUG:boto:Params: {} 2022-09-03T07:28:17.179 DEBUG:boto:Token: None 2022-09-03T07:28:17.180 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.181 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.184 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001bc6d370c5ac4fce-0063130211-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.185 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.186 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.187 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.188 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.189 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:28:17.190 DEBUG:boto:Data: 2022-09-03T07:28:17.191 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.193 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.194 DEBUG:boto:Params: {} 2022-09-03T07:28:17.195 DEBUG:boto:Token: None 2022-09-03T07:28:17.196 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145 2022-09-03T07:28:17.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JdUtrBthaNAOrWTeyYi9ROT5HJ0= 2022-09-03T07:28:17.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JdUtrBthaNAOrWTeyYi9ROT5HJ0=', 'Content-Length': '0'} 2022-09-03T07:28:17.201 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a024a6130bbce33-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.203 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.204 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.205 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.205 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:28:17.206 DEBUG:boto:Data: 2022-09-03T07:28:17.207 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.208 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.209 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.210 DEBUG:boto:Params: {} 2022-09-03T07:28:17.211 DEBUG:boto:Token: None 2022-09-03T07:28:17.212 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145 2022-09-03T07:28:17.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JdUtrBthaNAOrWTeyYi9ROT5HJ0= 2022-09-03T07:28:17.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JdUtrBthaNAOrWTeyYi9ROT5HJ0=', 'Content-Length': '0'} 2022-09-03T07:28:17.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002993667cf20e4513-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.218 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:28:17.219 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.220 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.221 DEBUG:boto:Method: GET 2022-09-03T07:28:17.221 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:28:17.222 DEBUG:boto:Data: 2022-09-03T07:28:17.223 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.224 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.225 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.226 DEBUG:boto:Params: {} 2022-09-03T07:28:17.227 DEBUG:boto:Token: None 2022-09-03T07:28:17.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145 2022-09-03T07:28:17.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:67P0u8MHXZy1fMgIXW5uE2+rUX8= 2022-09-03T07:28:17.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:67P0u8MHXZy1fMgIXW5uE2+rUX8=', 'Content-Length': '0'} 2022-09-03T07:28:17.235 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1e58b58e2ab9664-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.236 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:28:17.237 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:28:17.237 DEBUG:boto:Method: GET 2022-09-03T07:28:17.239 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:28:17.239 DEBUG:boto:Data: 2022-09-03T07:28:17.240 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.240 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.241 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.241 DEBUG:boto:Params: {} 2022-09-03T07:28:17.243 DEBUG:boto:Token: None 2022-09-03T07:28:17.244 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj145 2022-09-03T07:28:17.244 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:67P0u8MHXZy1fMgIXW5uE2+rUX8= 2022-09-03T07:28:17.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:67P0u8MHXZy1fMgIXW5uE2+rUX8=', 'Content-Length': '0'} 2022-09-03T07:28:17.251 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a324685049e9cba7-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.252 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.252 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.254 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:17.255 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:17.255 DEBUG:boto:Method: GET 2022-09-03T07:28:17.256 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:28:17.256 DEBUG:boto:Data: 2022-09-03T07:28:17.257 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.258 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.258 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.259 DEBUG:boto:Params: {} 2022-09-03T07:28:17.260 DEBUG:boto:Token: None 2022-09-03T07:28:17.261 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.267 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007ba6aaeb07a57001-0063130211-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.268 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.269 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.270 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.271 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:17.272 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:28:17.273 DEBUG:boto:Method: GET 2022-09-03T07:28:17.275 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:28:17.275 DEBUG:boto:Data: 2022-09-03T07:28:17.276 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.277 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.277 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.278 DEBUG:boto:Params: {} 2022-09-03T07:28:17.278 DEBUG:boto:Token: None 2022-09-03T07:28:17.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.283 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d5099db7807fe88d-0063130211-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.331 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.333 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:28:17.334 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.335 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.336 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.337 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.338 DEBUG:boto:Method: GET 2022-09-03T07:28:17.340 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.343 DEBUG:boto:Data: 2022-09-03T07:28:17.344 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.345 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.347 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.348 DEBUG:boto:Params: {} 2022-09-03T07:28:17.349 DEBUG:boto:Token: None 2022-09-03T07:28:17.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SPXVJoEXJw3//3gugXBVyLfQGjE= 2022-09-03T07:28:17.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SPXVJoEXJw3//3gugXBVyLfQGjE=', 'Content-Length': '0'} 2022-09-03T07:28:17.355 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083f1218f07e23337-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.357 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.358 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.359 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.360 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.361 DEBUG:boto:Method: GET 2022-09-03T07:28:17.362 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.363 DEBUG:boto:Data: 2022-09-03T07:28:17.364 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.365 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.366 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.367 DEBUG:boto:Params: {} 2022-09-03T07:28:17.368 DEBUG:boto:Token: None 2022-09-03T07:28:17.369 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SPXVJoEXJw3//3gugXBVyLfQGjE= 2022-09-03T07:28:17.371 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SPXVJoEXJw3//3gugXBVyLfQGjE=', 'Content-Length': '0'} 2022-09-03T07:28:17.376 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cde719c7c701eddb-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.381 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.382 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.383 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.384 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.385 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.393 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.395 DEBUG:boto:Data: 2022-09-03T07:28:17.396 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.397 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.397 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.398 DEBUG:boto:Params: {} 2022-09-03T07:28:17.400 DEBUG:boto:Token: None 2022-09-03T07:28:17.401 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hXpOaN0oaNcai2k9nhOWary6ynI= 2022-09-03T07:28:17.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hXpOaN0oaNcai2k9nhOWary6ynI=', 'Content-Length': '0'} 2022-09-03T07:28:17.409 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031991d5a3048402f-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.410 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.411 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.411 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.412 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.412 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.413 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.413 DEBUG:boto:Data: 2022-09-03T07:28:17.414 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.414 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.415 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.415 DEBUG:boto:Params: {} 2022-09-03T07:28:17.416 DEBUG:boto:Token: None 2022-09-03T07:28:17.417 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hXpOaN0oaNcai2k9nhOWary6ynI= 2022-09-03T07:28:17.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hXpOaN0oaNcai2k9nhOWary6ynI=', 'Content-Length': '0'} 2022-09-03T07:28:17.421 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ffdc69bebfd726cb-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.421 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:28:17.422 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.423 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.423 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.424 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.425 DEBUG:boto:Method: GET 2022-09-03T07:28:17.426 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.426 DEBUG:boto:Data: 2022-09-03T07:28:17.427 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.428 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.429 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.430 DEBUG:boto:Params: {} 2022-09-03T07:28:17.431 DEBUG:boto:Token: None 2022-09-03T07:28:17.431 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.432 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SPXVJoEXJw3//3gugXBVyLfQGjE= 2022-09-03T07:28:17.433 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SPXVJoEXJw3//3gugXBVyLfQGjE=', 'Content-Length': '0'} 2022-09-03T07:28:17.435 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044d339a8fb91730b-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.436 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.437 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.438 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.439 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.440 DEBUG:boto:Method: GET 2022-09-03T07:28:17.440 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.441 DEBUG:boto:Data: 2022-09-03T07:28:17.442 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.443 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.443 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.444 DEBUG:boto:Params: {} 2022-09-03T07:28:17.445 DEBUG:boto:Token: None 2022-09-03T07:28:17.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:28:17.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SPXVJoEXJw3//3gugXBVyLfQGjE= 2022-09-03T07:28:17.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SPXVJoEXJw3//3gugXBVyLfQGjE=', 'Content-Length': '0'} 2022-09-03T07:28:17.450 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb39fcda4b2aba7d-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.452 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.453 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.454 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:17.455 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:17.456 DEBUG:boto:Method: GET 2022-09-03T07:28:17.457 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:28:17.458 DEBUG:boto:Data: 2022-09-03T07:28:17.459 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.461 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.461 DEBUG:boto:Params: {} 2022-09-03T07:28:17.462 DEBUG:boto:Token: None 2022-09-03T07:28:17.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.463 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.467 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a8a45511570b0415-0063130211-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.468 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.469 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.470 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.471 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:17.472 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:17.473 DEBUG:boto:Method: GET 2022-09-03T07:28:17.474 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:28:17.474 DEBUG:boto:Data: 2022-09-03T07:28:17.475 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.475 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.476 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.477 DEBUG:boto:Params: {} 2022-09-03T07:28:17.478 DEBUG:boto:Token: None 2022-09-03T07:28:17.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.485 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000026e1c9fb47a07ebb-0063130211-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.486 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.488 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.489 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.490 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.491 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:28:17.492 DEBUG:boto:Data: 2022-09-03T07:28:17.493 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.495 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.495 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.496 DEBUG:boto:Params: {} 2022-09-03T07:28:17.497 DEBUG:boto:Token: None 2022-09-03T07:28:17.497 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155 2022-09-03T07:28:17.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3LwLjS4CG1qvKqm03cnmU/ibE7o= 2022-09-03T07:28:17.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3LwLjS4CG1qvKqm03cnmU/ibE7o=', 'Content-Length': '0'} 2022-09-03T07:28:17.502 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d51e71862efbce6-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.503 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.504 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.504 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.505 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:28:17.506 DEBUG:boto:Data: 2022-09-03T07:28:17.507 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.508 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.509 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.510 DEBUG:boto:Params: {} 2022-09-03T07:28:17.511 DEBUG:boto:Token: None 2022-09-03T07:28:17.512 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155 2022-09-03T07:28:17.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3LwLjS4CG1qvKqm03cnmU/ibE7o= 2022-09-03T07:28:17.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3LwLjS4CG1qvKqm03cnmU/ibE7o=', 'Content-Length': '0'} 2022-09-03T07:28:17.517 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee694404483fbd7c-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.518 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:28:17.519 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.519 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.520 DEBUG:boto:Method: GET 2022-09-03T07:28:17.521 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:28:17.521 DEBUG:boto:Data: 2022-09-03T07:28:17.522 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.523 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.523 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.524 DEBUG:boto:Params: {} 2022-09-03T07:28:17.525 DEBUG:boto:Token: None 2022-09-03T07:28:17.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155 2022-09-03T07:28:17.526 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1dTe+ZSQ+b0Q43S0nSBnUQvLe8Y= 2022-09-03T07:28:17.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1dTe+ZSQ+b0Q43S0nSBnUQvLe8Y=', 'Content-Length': '0'} 2022-09-03T07:28:17.529 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098d4cddf12df07c4-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.530 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:28:17.533 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:28:17.533 DEBUG:boto:Method: GET 2022-09-03T07:28:17.534 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:28:17.535 DEBUG:boto:Data: 2022-09-03T07:28:17.535 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.536 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.536 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.537 DEBUG:boto:Params: {} 2022-09-03T07:28:17.538 DEBUG:boto:Token: None 2022-09-03T07:28:17.539 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj155 2022-09-03T07:28:17.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1dTe+ZSQ+b0Q43S0nSBnUQvLe8Y= 2022-09-03T07:28:17.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1dTe+ZSQ+b0Q43S0nSBnUQvLe8Y=', 'Content-Length': '0'} 2022-09-03T07:28:17.543 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000996aa71b6c59c95b-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.544 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.545 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.546 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:17.546 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:17.547 DEBUG:boto:Method: GET 2022-09-03T07:28:17.548 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:28:17.548 DEBUG:boto:Data: 2022-09-03T07:28:17.549 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.550 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.551 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.552 DEBUG:boto:Params: {} 2022-09-03T07:28:17.553 DEBUG:boto:Token: None 2022-09-03T07:28:17.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.559 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000006972a9fc2823903-0063130211-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.560 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.561 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.562 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.562 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:17.563 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:28:17.564 DEBUG:boto:Method: GET 2022-09-03T07:28:17.564 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:28:17.565 DEBUG:boto:Data: 2022-09-03T07:28:17.566 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.566 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.567 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.568 DEBUG:boto:Params: {} 2022-09-03T07:28:17.569 DEBUG:boto:Token: None 2022-09-03T07:28:17.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.576 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000962cb3cb48ef80e8-0063130211-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.577 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.578 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:28:17.579 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.580 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.581 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.583 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.584 DEBUG:boto:Method: GET 2022-09-03T07:28:17.585 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.585 DEBUG:boto:Data: 2022-09-03T07:28:17.586 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.588 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.589 DEBUG:boto:Params: {} 2022-09-03T07:28:17.590 DEBUG:boto:Token: None 2022-09-03T07:28:17.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZjnuTyr1WrojX//o6jYiNf0hPQ= 2022-09-03T07:28:17.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZjnuTyr1WrojX//o6jYiNf0hPQ=', 'Content-Length': '0'} 2022-09-03T07:28:17.596 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b58aed7891b90f6-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.640 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.641 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.642 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.643 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.644 DEBUG:boto:Method: GET 2022-09-03T07:28:17.644 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.645 DEBUG:boto:Data: 2022-09-03T07:28:17.646 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.647 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.649 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.650 DEBUG:boto:Params: {} 2022-09-03T07:28:17.651 DEBUG:boto:Token: None 2022-09-03T07:28:17.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZjnuTyr1WrojX//o6jYiNf0hPQ= 2022-09-03T07:28:17.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZjnuTyr1WrojX//o6jYiNf0hPQ=', 'Content-Length': '0'} 2022-09-03T07:28:17.656 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035367310be9fafcb-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.657 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.658 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.659 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.660 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.661 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.662 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.663 DEBUG:boto:Data: 2022-09-03T07:28:17.663 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.664 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.664 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.665 DEBUG:boto:Params: {} 2022-09-03T07:28:17.666 DEBUG:boto:Token: None 2022-09-03T07:28:17.666 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IJzB9/TC9q2POfnJyqL7wQT6hlE= 2022-09-03T07:28:17.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IJzB9/TC9q2POfnJyqL7wQT6hlE=', 'Content-Length': '0'} 2022-09-03T07:28:17.671 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000094b638b43bb325c-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.672 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.673 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.674 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.675 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.676 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.677 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.678 DEBUG:boto:Data: 2022-09-03T07:28:17.679 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.680 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.681 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.682 DEBUG:boto:Params: {} 2022-09-03T07:28:17.683 DEBUG:boto:Token: None 2022-09-03T07:28:17.684 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IJzB9/TC9q2POfnJyqL7wQT6hlE= 2022-09-03T07:28:17.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IJzB9/TC9q2POfnJyqL7wQT6hlE=', 'Content-Length': '0'} 2022-09-03T07:28:17.690 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008ed1e2db2fc40257-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.691 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:28:17.692 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.693 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.694 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.695 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.695 DEBUG:boto:Method: GET 2022-09-03T07:28:17.696 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.697 DEBUG:boto:Data: 2022-09-03T07:28:17.698 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.700 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.701 DEBUG:boto:Params: {} 2022-09-03T07:28:17.702 DEBUG:boto:Token: None 2022-09-03T07:28:17.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZjnuTyr1WrojX//o6jYiNf0hPQ= 2022-09-03T07:28:17.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZjnuTyr1WrojX//o6jYiNf0hPQ=', 'Content-Length': '0'} 2022-09-03T07:28:17.709 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f42aaccc6acfa08-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.711 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.712 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.713 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.713 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.714 DEBUG:boto:Method: GET 2022-09-03T07:28:17.715 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.715 DEBUG:boto:Data: 2022-09-03T07:28:17.716 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.716 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.717 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.717 DEBUG:boto:Params: {} 2022-09-03T07:28:17.718 DEBUG:boto:Token: None 2022-09-03T07:28:17.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:28:17.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZjnuTyr1WrojX//o6jYiNf0hPQ= 2022-09-03T07:28:17.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZjnuTyr1WrojX//o6jYiNf0hPQ=', 'Content-Length': '0'} 2022-09-03T07:28:17.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cfd35d543bde5bf1-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.723 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.724 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.724 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:17.725 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:17.725 DEBUG:boto:Method: GET 2022-09-03T07:28:17.726 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:28:17.727 DEBUG:boto:Data: 2022-09-03T07:28:17.727 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.728 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.729 DEBUG:boto:Params: {} 2022-09-03T07:28:17.730 DEBUG:boto:Token: None 2022-09-03T07:28:17.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.737 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009fca6789437fb9b1-0063130211-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.739 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.740 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.741 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.742 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:17.743 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:17.744 DEBUG:boto:Method: GET 2022-09-03T07:28:17.745 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:28:17.745 DEBUG:boto:Data: 2022-09-03T07:28:17.746 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.747 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.748 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.749 DEBUG:boto:Params: {} 2022-09-03T07:28:17.751 DEBUG:boto:Token: None 2022-09-03T07:28:17.751 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.753 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.756 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dce27467bb0a511f-0063130211-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.758 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.759 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.760 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.761 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.762 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:28:17.763 DEBUG:boto:Data: 2022-09-03T07:28:17.764 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.765 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.766 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.767 DEBUG:boto:Params: {} 2022-09-03T07:28:17.768 DEBUG:boto:Token: None 2022-09-03T07:28:17.769 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17 2022-09-03T07:28:17.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uk8jDTrdWog04u/iWFz45LgZ/vg= 2022-09-03T07:28:17.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uk8jDTrdWog04u/iWFz45LgZ/vg=', 'Content-Length': '0'} 2022-09-03T07:28:17.773 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002da10b8d99777180-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.774 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.775 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.775 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.776 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:28:17.777 DEBUG:boto:Data: 2022-09-03T07:28:17.777 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.778 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.778 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.779 DEBUG:boto:Params: {} 2022-09-03T07:28:17.780 DEBUG:boto:Token: None 2022-09-03T07:28:17.781 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17 2022-09-03T07:28:17.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uk8jDTrdWog04u/iWFz45LgZ/vg= 2022-09-03T07:28:17.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uk8jDTrdWog04u/iWFz45LgZ/vg=', 'Content-Length': '0'} 2022-09-03T07:28:17.786 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013d2a4830512ecd9-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.787 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:28:17.788 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.789 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.790 DEBUG:boto:Method: GET 2022-09-03T07:28:17.791 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:28:17.792 DEBUG:boto:Data: 2022-09-03T07:28:17.793 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.793 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.794 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.794 DEBUG:boto:Params: {} 2022-09-03T07:28:17.795 DEBUG:boto:Token: None 2022-09-03T07:28:17.796 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17 2022-09-03T07:28:17.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:baKN+Sm78S1SnS2m5HxGIiiLJKw= 2022-09-03T07:28:17.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:baKN+Sm78S1SnS2m5HxGIiiLJKw=', 'Content-Length': '0'} 2022-09-03T07:28:17.801 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d4dedc6f133befec-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.802 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:28:17.802 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:28:17.803 DEBUG:boto:Method: GET 2022-09-03T07:28:17.804 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:28:17.805 DEBUG:boto:Data: 2022-09-03T07:28:17.805 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.806 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.807 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.807 DEBUG:boto:Params: {} 2022-09-03T07:28:17.808 DEBUG:boto:Token: None 2022-09-03T07:28:17.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj17 2022-09-03T07:28:17.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:baKN+Sm78S1SnS2m5HxGIiiLJKw= 2022-09-03T07:28:17.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:baKN+Sm78S1SnS2m5HxGIiiLJKw=', 'Content-Length': '0'} 2022-09-03T07:28:17.812 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000446b28561c9ef7a2-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.813 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.813 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.814 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:17.815 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:17.816 DEBUG:boto:Method: GET 2022-09-03T07:28:17.817 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:28:17.818 DEBUG:boto:Data: 2022-09-03T07:28:17.819 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.819 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.820 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.821 DEBUG:boto:Params: {} 2022-09-03T07:28:17.822 DEBUG:boto:Token: None 2022-09-03T07:28:17.823 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.829 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f41ca9d03b32944d-0063130211-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.830 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.832 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.832 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.833 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:17.834 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:28:17.835 DEBUG:boto:Method: GET 2022-09-03T07:28:17.835 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:28:17.836 DEBUG:boto:Data: 2022-09-03T07:28:17.836 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.837 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.838 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.838 DEBUG:boto:Params: {} 2022-09-03T07:28:17.839 DEBUG:boto:Token: None 2022-09-03T07:28:17.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.843 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079dd35366c2e0a7f-0063130211-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.844 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.845 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:28:17.845 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:17.846 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:17.846 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.847 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.847 DEBUG:boto:Method: GET 2022-09-03T07:28:17.848 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.849 DEBUG:boto:Data: 2022-09-03T07:28:17.849 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.850 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.851 DEBUG:boto:Params: {} 2022-09-03T07:28:17.852 DEBUG:boto:Token: None 2022-09-03T07:28:17.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TOsvTRzFNlOO9d0t7GiZWInpiJo= 2022-09-03T07:28:17.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TOsvTRzFNlOO9d0t7GiZWInpiJo=', 'Content-Length': '0'} 2022-09-03T07:28:17.856 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5498ed32b5c9cd8-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.857 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:17.858 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:17.858 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.859 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.859 DEBUG:boto:Method: GET 2022-09-03T07:28:17.860 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.860 DEBUG:boto:Data: 2022-09-03T07:28:17.861 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.861 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.862 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.862 DEBUG:boto:Params: {} 2022-09-03T07:28:17.863 DEBUG:boto:Token: None 2022-09-03T07:28:17.864 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TOsvTRzFNlOO9d0t7GiZWInpiJo= 2022-09-03T07:28:17.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TOsvTRzFNlOO9d0t7GiZWInpiJo=', 'Content-Length': '0'} 2022-09-03T07:28:17.867 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000846348d112f852c0-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.910 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:17.911 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:17.911 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.912 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.912 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.913 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.913 DEBUG:boto:Data: 2022-09-03T07:28:17.914 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.915 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.916 DEBUG:boto:Params: {} 2022-09-03T07:28:17.916 DEBUG:boto:Token: None 2022-09-03T07:28:17.917 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kl6sNyatHFQmd0R2HWkpZ9qJncs= 2022-09-03T07:28:17.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kl6sNyatHFQmd0R2HWkpZ9qJncs=', 'Content-Length': '0'} 2022-09-03T07:28:17.921 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e02ca2a9a44df0d-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.922 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:17.922 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:17.923 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.923 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.924 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.924 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.925 DEBUG:boto:Data: 2022-09-03T07:28:17.925 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.926 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.927 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.928 DEBUG:boto:Params: {} 2022-09-03T07:28:17.928 DEBUG:boto:Token: None 2022-09-03T07:28:17.929 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kl6sNyatHFQmd0R2HWkpZ9qJncs= 2022-09-03T07:28:17.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kl6sNyatHFQmd0R2HWkpZ9qJncs=', 'Content-Length': '0'} 2022-09-03T07:28:17.932 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026284ccc26f34167-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.933 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:28:17.934 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:17.934 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:17.935 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.935 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.936 DEBUG:boto:Method: GET 2022-09-03T07:28:17.936 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.937 DEBUG:boto:Data: 2022-09-03T07:28:17.937 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.938 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.939 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.939 DEBUG:boto:Params: {} 2022-09-03T07:28:17.940 DEBUG:boto:Token: None 2022-09-03T07:28:17.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TOsvTRzFNlOO9d0t7GiZWInpiJo= 2022-09-03T07:28:17.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TOsvTRzFNlOO9d0t7GiZWInpiJo=', 'Content-Length': '0'} 2022-09-03T07:28:17.944 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b0ea908f2bd2710-0063130211-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.950 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:17.950 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:17.951 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.951 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.952 DEBUG:boto:Method: GET 2022-09-03T07:28:17.952 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.953 DEBUG:boto:Data: 2022-09-03T07:28:17.953 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.954 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.954 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.955 DEBUG:boto:Params: {} 2022-09-03T07:28:17.955 DEBUG:boto:Token: None 2022-09-03T07:28:17.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:28:17.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TOsvTRzFNlOO9d0t7GiZWInpiJo= 2022-09-03T07:28:17.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TOsvTRzFNlOO9d0t7GiZWInpiJo=', 'Content-Length': '0'} 2022-09-03T07:28:17.961 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000197ec8d6bf6c6ef3-0063130211-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.963 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.964 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.965 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:17.965 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:17.966 DEBUG:boto:Method: GET 2022-09-03T07:28:17.967 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:17.968 DEBUG:boto:Data: 2022-09-03T07:28:17.968 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.969 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.969 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.970 DEBUG:boto:Params: {} 2022-09-03T07:28:17.971 DEBUG:boto:Token: None 2022-09-03T07:28:17.972 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.973 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.976 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000255c3194fa063db0-0063130211-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.977 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.978 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:17.979 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:17.979 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:17.980 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:17.981 DEBUG:boto:Method: GET 2022-09-03T07:28:17.982 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:17.983 DEBUG:boto:Data: 2022-09-03T07:28:17.983 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.984 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:17.984 DEBUG:boto:Port: 8000 2022-09-03T07:28:17.985 DEBUG:boto:Params: {} 2022-09-03T07:28:17.986 DEBUG:boto:Token: None 2022-09-03T07:28:17.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:17 GMT /avysmw-75/ 2022-09-03T07:28:17.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI= 2022-09-03T07:28:17.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:17 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bBVsdiSdzsXgjFjlkfr89be5mnI=', 'Content-Length': '0'} 2022-09-03T07:28:17.991 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009ee70687a721e9d9-0063130211-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:17 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:17.991 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:17.992 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:17.993 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:17.993 DEBUG:boto:Method: HEAD 2022-09-03T07:28:17.994 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:17.994 DEBUG:boto:Data: 2022-09-03T07:28:17.995 DEBUG:boto:Headers: {} 2022-09-03T07:28:17.996 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:17.997 DEBUG:boto:Port: 8001 2022-09-03T07:28:17.998 DEBUG:boto:Params: {} 2022-09-03T07:28:17.999 DEBUG:boto:Token: None 2022-09-03T07:28:18.000 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj19 2022-09-03T07:28:18.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b1JzrBOmbl8cynuDD9xEI71bk/U= 2022-09-03T07:28:18.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b1JzrBOmbl8cynuDD9xEI71bk/U=', 'Content-Length': '0'} 2022-09-03T07:28:18.006 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d8c49ffd46bc3f5-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.007 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:18.008 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:18.008 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.009 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:18.009 DEBUG:boto:Data: 2022-09-03T07:28:18.010 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.010 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.011 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.011 DEBUG:boto:Params: {} 2022-09-03T07:28:18.012 DEBUG:boto:Token: None 2022-09-03T07:28:18.012 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj19 2022-09-03T07:28:18.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b1JzrBOmbl8cynuDD9xEI71bk/U= 2022-09-03T07:28:18.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b1JzrBOmbl8cynuDD9xEI71bk/U=', 'Content-Length': '0'} 2022-09-03T07:28:18.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a84be9017e2fd8c0-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.017 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:28:18.017 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:18.018 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:18.018 DEBUG:boto:Method: GET 2022-09-03T07:28:18.019 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:18.020 DEBUG:boto:Data: 2022-09-03T07:28:18.020 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.021 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.022 DEBUG:boto:Params: {} 2022-09-03T07:28:18.022 DEBUG:boto:Token: None 2022-09-03T07:28:18.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj19 2022-09-03T07:28:18.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aGwn0sAd6JvM2AU/6JdrlUAhDqs= 2022-09-03T07:28:18.024 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aGwn0sAd6JvM2AU/6JdrlUAhDqs=', 'Content-Length': '0'} 2022-09-03T07:28:18.026 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ee6ee3bdaab5bc1-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.070 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:28:18.071 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:28:18.071 DEBUG:boto:Method: GET 2022-09-03T07:28:18.072 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:28:18.073 DEBUG:boto:Data: 2022-09-03T07:28:18.073 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.074 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.074 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.075 DEBUG:boto:Params: {} 2022-09-03T07:28:18.075 DEBUG:boto:Token: None 2022-09-03T07:28:18.076 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj19 2022-09-03T07:28:18.077 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aGwn0sAd6JvM2AU/6JdrlUAhDqs= 2022-09-03T07:28:18.078 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aGwn0sAd6JvM2AU/6JdrlUAhDqs=', 'Content-Length': '0'} 2022-09-03T07:28:18.080 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d1f66c67caba866-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.081 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.081 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.082 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:18.083 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:18.083 DEBUG:boto:Method: GET 2022-09-03T07:28:18.084 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:18.084 DEBUG:boto:Data: 2022-09-03T07:28:18.085 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.086 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.087 DEBUG:boto:Params: {} 2022-09-03T07:28:18.087 DEBUG:boto:Token: None 2022-09-03T07:28:18.088 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.089 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.092 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036b01f926f11d33d-0063130212-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.093 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.094 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.094 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.095 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:18.095 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:28:18.096 DEBUG:boto:Method: GET 2022-09-03T07:28:18.096 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:28:18.097 DEBUG:boto:Data: 2022-09-03T07:28:18.098 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.098 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.099 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.100 DEBUG:boto:Params: {} 2022-09-03T07:28:18.100 DEBUG:boto:Token: None 2022-09-03T07:28:18.102 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.107 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d1ad68ce58df453-0063130212-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.108 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.109 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:28:18.110 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.111 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.112 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.113 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.114 DEBUG:boto:Method: GET 2022-09-03T07:28:18.115 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.116 DEBUG:boto:Data: 2022-09-03T07:28:18.116 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.117 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.118 DEBUG:boto:Params: {} 2022-09-03T07:28:18.119 DEBUG:boto:Token: None 2022-09-03T07:28:18.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mSFyzwz+19q9idl0uW2rkRWTx2g= 2022-09-03T07:28:18.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mSFyzwz+19q9idl0uW2rkRWTx2g=', 'Content-Length': '0'} 2022-09-03T07:28:18.123 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089eff079276bc701-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.124 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.125 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.125 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.126 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.126 DEBUG:boto:Method: GET 2022-09-03T07:28:18.127 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.127 DEBUG:boto:Data: 2022-09-03T07:28:18.128 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.128 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.129 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.130 DEBUG:boto:Params: {} 2022-09-03T07:28:18.131 DEBUG:boto:Token: None 2022-09-03T07:28:18.132 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mSFyzwz+19q9idl0uW2rkRWTx2g= 2022-09-03T07:28:18.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mSFyzwz+19q9idl0uW2rkRWTx2g=', 'Content-Length': '0'} 2022-09-03T07:28:18.139 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056eae24fd1b5bae0-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.185 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.189 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.190 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.191 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.192 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.193 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.194 DEBUG:boto:Data: 2022-09-03T07:28:18.195 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.196 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.197 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.199 DEBUG:boto:Params: {} 2022-09-03T07:28:18.200 DEBUG:boto:Token: None 2022-09-03T07:28:18.201 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.202 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ADSGXC4tL6VWS7u/Khq1iZaCF5w= 2022-09-03T07:28:18.203 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ADSGXC4tL6VWS7u/Khq1iZaCF5w=', 'Content-Length': '0'} 2022-09-03T07:28:18.206 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000484896e58a345f94-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.207 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.208 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.209 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.210 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.211 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.212 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.213 DEBUG:boto:Data: 2022-09-03T07:28:18.214 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.215 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.216 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.217 DEBUG:boto:Params: {} 2022-09-03T07:28:18.218 DEBUG:boto:Token: None 2022-09-03T07:28:18.219 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ADSGXC4tL6VWS7u/Khq1iZaCF5w= 2022-09-03T07:28:18.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ADSGXC4tL6VWS7u/Khq1iZaCF5w=', 'Content-Length': '0'} 2022-09-03T07:28:18.226 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a0afe47af4c56ceb-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.227 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:28:18.228 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.229 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.231 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.232 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.234 DEBUG:boto:Method: GET 2022-09-03T07:28:18.235 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.236 DEBUG:boto:Data: 2022-09-03T07:28:18.237 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.241 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.242 DEBUG:boto:Params: {} 2022-09-03T07:28:18.244 DEBUG:boto:Token: None 2022-09-03T07:28:18.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mSFyzwz+19q9idl0uW2rkRWTx2g= 2022-09-03T07:28:18.246 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mSFyzwz+19q9idl0uW2rkRWTx2g=', 'Content-Length': '0'} 2022-09-03T07:28:18.248 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e9e9fb34593c7ea-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.250 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.250 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.251 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.251 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.252 DEBUG:boto:Method: GET 2022-09-03T07:28:18.253 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.253 DEBUG:boto:Data: 2022-09-03T07:28:18.254 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.255 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.255 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.256 DEBUG:boto:Params: {} 2022-09-03T07:28:18.257 DEBUG:boto:Token: None 2022-09-03T07:28:18.257 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:28:18.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mSFyzwz+19q9idl0uW2rkRWTx2g= 2022-09-03T07:28:18.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mSFyzwz+19q9idl0uW2rkRWTx2g=', 'Content-Length': '0'} 2022-09-03T07:28:18.261 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005cbe9f86dabd97c0-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.263 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.264 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.265 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:18.266 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:18.267 DEBUG:boto:Method: GET 2022-09-03T07:28:18.268 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:18.269 DEBUG:boto:Data: 2022-09-03T07:28:18.270 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.272 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.272 DEBUG:boto:Params: {} 2022-09-03T07:28:18.274 DEBUG:boto:Token: None 2022-09-03T07:28:18.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.280 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eee30bf9bcc5e9ca-0063130212-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.282 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.283 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.284 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.285 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:18.286 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:18.287 DEBUG:boto:Method: GET 2022-09-03T07:28:18.288 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:18.289 DEBUG:boto:Data: 2022-09-03T07:28:18.290 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.292 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.293 DEBUG:boto:Params: {} 2022-09-03T07:28:18.294 DEBUG:boto:Token: None 2022-09-03T07:28:18.294 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.298 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a9106e0d4713315c-0063130212-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.300 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.302 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.303 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.304 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.304 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:18.305 DEBUG:boto:Data: 2022-09-03T07:28:18.306 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.307 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.308 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.309 DEBUG:boto:Params: {} 2022-09-03T07:28:18.311 DEBUG:boto:Token: None 2022-09-03T07:28:18.312 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22 2022-09-03T07:28:18.313 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:57LtoLvTL4LDb53wNDMW6UXiD/c= 2022-09-03T07:28:18.314 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:57LtoLvTL4LDb53wNDMW6UXiD/c=', 'Content-Length': '0'} 2022-09-03T07:28:18.317 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000527f1447e01e1007-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.319 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.320 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.321 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.322 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:18.323 DEBUG:boto:Data: 2022-09-03T07:28:18.323 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.324 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.325 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.325 DEBUG:boto:Params: {} 2022-09-03T07:28:18.326 DEBUG:boto:Token: None 2022-09-03T07:28:18.326 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22 2022-09-03T07:28:18.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:57LtoLvTL4LDb53wNDMW6UXiD/c= 2022-09-03T07:28:18.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:57LtoLvTL4LDb53wNDMW6UXiD/c=', 'Content-Length': '0'} 2022-09-03T07:28:18.330 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aeb906aeafe9fa59-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.330 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:28:18.331 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.331 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.332 DEBUG:boto:Method: GET 2022-09-03T07:28:18.333 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:18.333 DEBUG:boto:Data: 2022-09-03T07:28:18.334 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.335 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.335 DEBUG:boto:Params: {} 2022-09-03T07:28:18.336 DEBUG:boto:Token: None 2022-09-03T07:28:18.337 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22 2022-09-03T07:28:18.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZS44L6NcrgBRDT7ZBZkH91R5pWA= 2022-09-03T07:28:18.338 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZS44L6NcrgBRDT7ZBZkH91R5pWA=', 'Content-Length': '0'} 2022-09-03T07:28:18.340 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e3bcdce179aec53-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.341 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:28:18.342 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:28:18.342 DEBUG:boto:Method: GET 2022-09-03T07:28:18.343 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:28:18.343 DEBUG:boto:Data: 2022-09-03T07:28:18.344 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.344 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.345 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.346 DEBUG:boto:Params: {} 2022-09-03T07:28:18.346 DEBUG:boto:Token: None 2022-09-03T07:28:18.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj22 2022-09-03T07:28:18.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZS44L6NcrgBRDT7ZBZkH91R5pWA= 2022-09-03T07:28:18.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZS44L6NcrgBRDT7ZBZkH91R5pWA=', 'Content-Length': '0'} 2022-09-03T07:28:18.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7db3c69d54d8503-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.358 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.359 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.360 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:18.360 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:18.361 DEBUG:boto:Method: GET 2022-09-03T07:28:18.361 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:18.362 DEBUG:boto:Data: 2022-09-03T07:28:18.362 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.364 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.364 DEBUG:boto:Params: {} 2022-09-03T07:28:18.365 DEBUG:boto:Token: None 2022-09-03T07:28:18.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.372 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011632ba846022bb1-0063130212-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.373 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.375 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.375 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.376 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:18.377 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:28:18.378 DEBUG:boto:Method: GET 2022-09-03T07:28:18.379 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:28:18.380 DEBUG:boto:Data: 2022-09-03T07:28:18.381 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.382 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.382 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.384 DEBUG:boto:Params: {} 2022-09-03T07:28:18.385 DEBUG:boto:Token: None 2022-09-03T07:28:18.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.391 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004affec5323ea1c84-0063130212-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.434 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.435 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:28:18.436 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.436 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.437 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.437 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.438 DEBUG:boto:Method: GET 2022-09-03T07:28:18.439 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.440 DEBUG:boto:Data: 2022-09-03T07:28:18.441 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.442 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.442 DEBUG:boto:Params: {} 2022-09-03T07:28:18.443 DEBUG:boto:Token: None 2022-09-03T07:28:18.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MCT1EXMaUOlahJUDuxnmuFxlKdA= 2022-09-03T07:28:18.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MCT1EXMaUOlahJUDuxnmuFxlKdA=', 'Content-Length': '0'} 2022-09-03T07:28:18.447 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000069ba7e1e7e73c502-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.448 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.448 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.449 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.449 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.450 DEBUG:boto:Method: GET 2022-09-03T07:28:18.451 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.451 DEBUG:boto:Data: 2022-09-03T07:28:18.452 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.452 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.454 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.454 DEBUG:boto:Params: {} 2022-09-03T07:28:18.455 DEBUG:boto:Token: None 2022-09-03T07:28:18.455 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.456 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MCT1EXMaUOlahJUDuxnmuFxlKdA= 2022-09-03T07:28:18.457 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MCT1EXMaUOlahJUDuxnmuFxlKdA=', 'Content-Length': '0'} 2022-09-03T07:28:18.459 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007bb750f3116bc808-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.462 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.462 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.463 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.463 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.464 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.464 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.465 DEBUG:boto:Data: 2022-09-03T07:28:18.466 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.467 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.467 DEBUG:boto:Params: {} 2022-09-03T07:28:18.468 DEBUG:boto:Token: None 2022-09-03T07:28:18.468 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SznnF9y/nA/39iZXZCcoTPViDjg= 2022-09-03T07:28:18.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SznnF9y/nA/39iZXZCcoTPViDjg=', 'Content-Length': '0'} 2022-09-03T07:28:18.472 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa0347fff2cea295-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.472 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.473 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.473 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.474 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.474 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.475 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.475 DEBUG:boto:Data: 2022-09-03T07:28:18.476 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.476 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.477 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.477 DEBUG:boto:Params: {} 2022-09-03T07:28:18.478 DEBUG:boto:Token: None 2022-09-03T07:28:18.479 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SznnF9y/nA/39iZXZCcoTPViDjg= 2022-09-03T07:28:18.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SznnF9y/nA/39iZXZCcoTPViDjg=', 'Content-Length': '0'} 2022-09-03T07:28:18.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa7195bb0477cc14-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.483 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:28:18.484 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.484 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.485 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.486 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.487 DEBUG:boto:Method: GET 2022-09-03T07:28:18.487 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.488 DEBUG:boto:Data: 2022-09-03T07:28:18.488 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.489 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.490 DEBUG:boto:Params: {} 2022-09-03T07:28:18.491 DEBUG:boto:Token: None 2022-09-03T07:28:18.491 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MCT1EXMaUOlahJUDuxnmuFxlKdA= 2022-09-03T07:28:18.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MCT1EXMaUOlahJUDuxnmuFxlKdA=', 'Content-Length': '0'} 2022-09-03T07:28:18.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d88d200ad03452cb-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.496 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.497 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.498 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.499 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.500 DEBUG:boto:Method: GET 2022-09-03T07:28:18.500 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.501 DEBUG:boto:Data: 2022-09-03T07:28:18.501 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.502 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.504 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.504 DEBUG:boto:Params: {} 2022-09-03T07:28:18.506 DEBUG:boto:Token: None 2022-09-03T07:28:18.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:28:18.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MCT1EXMaUOlahJUDuxnmuFxlKdA= 2022-09-03T07:28:18.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MCT1EXMaUOlahJUDuxnmuFxlKdA=', 'Content-Length': '0'} 2022-09-03T07:28:18.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd74adad789dd2c4-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.513 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.514 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.514 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:18.515 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:18.515 DEBUG:boto:Method: GET 2022-09-03T07:28:18.516 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:18.516 DEBUG:boto:Data: 2022-09-03T07:28:18.517 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.517 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.518 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.519 DEBUG:boto:Params: {} 2022-09-03T07:28:18.521 DEBUG:boto:Token: None 2022-09-03T07:28:18.522 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.523 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.524 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.527 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000063c42aa23888b2f-0063130212-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.574 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.575 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.576 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.577 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:18.577 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:18.578 DEBUG:boto:Method: GET 2022-09-03T07:28:18.579 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:18.579 DEBUG:boto:Data: 2022-09-03T07:28:18.580 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.581 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.582 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.583 DEBUG:boto:Params: {} 2022-09-03T07:28:18.585 DEBUG:boto:Token: None 2022-09-03T07:28:18.586 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.587 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.588 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.598 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000af0ca3d0117d8ade-0063130212-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.599 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.601 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.602 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.603 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.604 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:18.605 DEBUG:boto:Data: 2022-09-03T07:28:18.606 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.607 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.608 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.609 DEBUG:boto:Params: {} 2022-09-03T07:28:18.609 DEBUG:boto:Token: None 2022-09-03T07:28:18.610 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23 2022-09-03T07:28:18.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LaZhu9480lMmGaftzqQfA16loJk= 2022-09-03T07:28:18.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LaZhu9480lMmGaftzqQfA16loJk=', 'Content-Length': '0'} 2022-09-03T07:28:18.613 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000050d731577c75634f-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.614 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.615 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.615 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.616 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:18.616 DEBUG:boto:Data: 2022-09-03T07:28:18.617 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.618 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.618 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.619 DEBUG:boto:Params: {} 2022-09-03T07:28:18.620 DEBUG:boto:Token: None 2022-09-03T07:28:18.620 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23 2022-09-03T07:28:18.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LaZhu9480lMmGaftzqQfA16loJk= 2022-09-03T07:28:18.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LaZhu9480lMmGaftzqQfA16loJk=', 'Content-Length': '0'} 2022-09-03T07:28:18.624 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026b12a5657e0e9d7-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.624 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:28:18.625 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.626 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.626 DEBUG:boto:Method: GET 2022-09-03T07:28:18.627 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:18.628 DEBUG:boto:Data: 2022-09-03T07:28:18.629 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.631 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.631 DEBUG:boto:Params: {} 2022-09-03T07:28:18.632 DEBUG:boto:Token: None 2022-09-03T07:28:18.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23 2022-09-03T07:28:18.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZl+gcJMA0rgQ0TU/q62PTqPO20= 2022-09-03T07:28:18.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZl+gcJMA0rgQ0TU/q62PTqPO20=', 'Content-Length': '0'} 2022-09-03T07:28:18.637 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000067a86bbfa5d3d66c-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.638 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:28:18.639 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:28:18.640 DEBUG:boto:Method: GET 2022-09-03T07:28:18.641 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:28:18.641 DEBUG:boto:Data: 2022-09-03T07:28:18.642 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.643 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.644 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.645 DEBUG:boto:Params: {} 2022-09-03T07:28:18.647 DEBUG:boto:Token: None 2022-09-03T07:28:18.647 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj23 2022-09-03T07:28:18.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HZl+gcJMA0rgQ0TU/q62PTqPO20= 2022-09-03T07:28:18.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HZl+gcJMA0rgQ0TU/q62PTqPO20=', 'Content-Length': '0'} 2022-09-03T07:28:18.656 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000332b9efb9cf75efb-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.694 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.695 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.696 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:18.696 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:18.697 DEBUG:boto:Method: GET 2022-09-03T07:28:18.697 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:18.698 DEBUG:boto:Data: 2022-09-03T07:28:18.699 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.700 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.700 DEBUG:boto:Params: {} 2022-09-03T07:28:18.701 DEBUG:boto:Token: None 2022-09-03T07:28:18.701 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.707 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000be5e658ed54458cb-0063130212-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.708 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.709 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.709 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.710 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:18.710 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:28:18.711 DEBUG:boto:Method: GET 2022-09-03T07:28:18.711 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:28:18.712 DEBUG:boto:Data: 2022-09-03T07:28:18.713 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.713 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.714 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.715 DEBUG:boto:Params: {} 2022-09-03T07:28:18.715 DEBUG:boto:Token: None 2022-09-03T07:28:18.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.721 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000008199b2271c9c96e-0063130212-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.722 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.723 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:28:18.724 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.724 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.725 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.726 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.727 DEBUG:boto:Method: GET 2022-09-03T07:28:18.728 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.729 DEBUG:boto:Data: 2022-09-03T07:28:18.730 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.732 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.732 DEBUG:boto:Params: {} 2022-09-03T07:28:18.733 DEBUG:boto:Token: None 2022-09-03T07:28:18.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sAUmdLyClxBxFLmOp3jyDWYWivg= 2022-09-03T07:28:18.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sAUmdLyClxBxFLmOp3jyDWYWivg=', 'Content-Length': '0'} 2022-09-03T07:28:18.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c26649de89618db5-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.739 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.740 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.741 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.742 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.743 DEBUG:boto:Method: GET 2022-09-03T07:28:18.743 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.744 DEBUG:boto:Data: 2022-09-03T07:28:18.744 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.745 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.745 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.746 DEBUG:boto:Params: {} 2022-09-03T07:28:18.747 DEBUG:boto:Token: None 2022-09-03T07:28:18.747 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.748 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sAUmdLyClxBxFLmOp3jyDWYWivg= 2022-09-03T07:28:18.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sAUmdLyClxBxFLmOp3jyDWYWivg=', 'Content-Length': '0'} 2022-09-03T07:28:18.752 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002baad06f8809fed1-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.753 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.754 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.754 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.755 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.755 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.756 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.756 DEBUG:boto:Data: 2022-09-03T07:28:18.757 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.758 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.758 DEBUG:boto:Params: {} 2022-09-03T07:28:18.759 DEBUG:boto:Token: None 2022-09-03T07:28:18.760 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jg/uL5YlTIwVg7FPLIx5vgNZi8o= 2022-09-03T07:28:18.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jg/uL5YlTIwVg7FPLIx5vgNZi8o=', 'Content-Length': '0'} 2022-09-03T07:28:18.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064448af98eef87b6-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.764 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.764 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.765 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.765 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.766 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.766 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.767 DEBUG:boto:Data: 2022-09-03T07:28:18.767 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.768 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.769 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.769 DEBUG:boto:Params: {} 2022-09-03T07:28:18.770 DEBUG:boto:Token: None 2022-09-03T07:28:18.770 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jg/uL5YlTIwVg7FPLIx5vgNZi8o= 2022-09-03T07:28:18.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jg/uL5YlTIwVg7FPLIx5vgNZi8o=', 'Content-Length': '0'} 2022-09-03T07:28:18.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000dcef984e7837f42-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.774 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:28:18.775 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.775 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.776 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.776 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.777 DEBUG:boto:Method: GET 2022-09-03T07:28:18.778 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.778 DEBUG:boto:Data: 2022-09-03T07:28:18.779 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.780 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.780 DEBUG:boto:Params: {} 2022-09-03T07:28:18.781 DEBUG:boto:Token: None 2022-09-03T07:28:18.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.782 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sAUmdLyClxBxFLmOp3jyDWYWivg= 2022-09-03T07:28:18.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sAUmdLyClxBxFLmOp3jyDWYWivg=', 'Content-Length': '0'} 2022-09-03T07:28:18.785 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005bacde5675bfed22-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.826 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.827 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.827 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.828 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.828 DEBUG:boto:Method: GET 2022-09-03T07:28:18.829 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.829 DEBUG:boto:Data: 2022-09-03T07:28:18.830 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.831 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.831 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.832 DEBUG:boto:Params: {} 2022-09-03T07:28:18.833 DEBUG:boto:Token: None 2022-09-03T07:28:18.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:28:18.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sAUmdLyClxBxFLmOp3jyDWYWivg= 2022-09-03T07:28:18.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sAUmdLyClxBxFLmOp3jyDWYWivg=', 'Content-Length': '0'} 2022-09-03T07:28:18.837 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c92309a892156302-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.838 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.838 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.839 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:18.839 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:18.840 DEBUG:boto:Method: GET 2022-09-03T07:28:18.840 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:18.841 DEBUG:boto:Data: 2022-09-03T07:28:18.841 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.842 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.843 DEBUG:boto:Params: {} 2022-09-03T07:28:18.844 DEBUG:boto:Token: None 2022-09-03T07:28:18.844 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.850 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000065b45884af6f145d-0063130212-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.851 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.851 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.852 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.853 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:18.853 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:18.854 DEBUG:boto:Method: GET 2022-09-03T07:28:18.854 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:18.855 DEBUG:boto:Data: 2022-09-03T07:28:18.856 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.857 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.858 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.859 DEBUG:boto:Params: {} 2022-09-03T07:28:18.860 DEBUG:boto:Token: None 2022-09-03T07:28:18.860 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.861 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.865 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c410e192505243cd-0063130212-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.867 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.868 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.868 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.869 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.870 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:18.871 DEBUG:boto:Data: 2022-09-03T07:28:18.872 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.872 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.873 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.873 DEBUG:boto:Params: {} 2022-09-03T07:28:18.874 DEBUG:boto:Token: None 2022-09-03T07:28:18.875 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29 2022-09-03T07:28:18.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sUTfUT4jWGB0zbGdHJKblNibMY= 2022-09-03T07:28:18.877 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sUTfUT4jWGB0zbGdHJKblNibMY=', 'Content-Length': '0'} 2022-09-03T07:28:18.883 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079e913ab8272375d-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.884 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.885 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.885 DEBUG:boto:Method: HEAD 2022-09-03T07:28:18.886 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:18.886 DEBUG:boto:Data: 2022-09-03T07:28:18.887 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.887 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.888 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.888 DEBUG:boto:Params: {} 2022-09-03T07:28:18.889 DEBUG:boto:Token: None 2022-09-03T07:28:18.889 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29 2022-09-03T07:28:18.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3sUTfUT4jWGB0zbGdHJKblNibMY= 2022-09-03T07:28:18.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3sUTfUT4jWGB0zbGdHJKblNibMY=', 'Content-Length': '0'} 2022-09-03T07:28:18.892 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b0aa336229a29c5-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.894 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:28:18.895 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.896 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.897 DEBUG:boto:Method: GET 2022-09-03T07:28:18.897 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:18.898 DEBUG:boto:Data: 2022-09-03T07:28:18.898 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.899 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.899 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.900 DEBUG:boto:Params: {} 2022-09-03T07:28:18.901 DEBUG:boto:Token: None 2022-09-03T07:28:18.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29 2022-09-03T07:28:18.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P7OlhrVJhkoJkEdvOSUlvIwHL90= 2022-09-03T07:28:18.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P7OlhrVJhkoJkEdvOSUlvIwHL90=', 'Content-Length': '0'} 2022-09-03T07:28:18.905 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000425476ee6547756d-0063130212-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.952 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:28:18.953 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:28:18.954 DEBUG:boto:Method: GET 2022-09-03T07:28:18.955 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:28:18.956 DEBUG:boto:Data: 2022-09-03T07:28:18.958 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.961 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.962 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.963 DEBUG:boto:Params: {} 2022-09-03T07:28:18.964 DEBUG:boto:Token: None 2022-09-03T07:28:18.966 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/obj29 2022-09-03T07:28:18.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P7OlhrVJhkoJkEdvOSUlvIwHL90= 2022-09-03T07:28:18.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P7OlhrVJhkoJkEdvOSUlvIwHL90=', 'Content-Length': '0'} 2022-09-03T07:28:18.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f7e5939c22fe150-0063130212-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.974 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.975 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.976 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:18.977 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:18.977 DEBUG:boto:Method: GET 2022-09-03T07:28:18.978 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:18.978 DEBUG:boto:Data: 2022-09-03T07:28:18.980 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:18.982 DEBUG:boto:Port: 8001 2022-09-03T07:28:18.982 DEBUG:boto:Params: {} 2022-09-03T07:28:18.983 DEBUG:boto:Token: None 2022-09-03T07:28:18.984 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.984 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.985 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:18.988 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da8ff75259302606-0063130212-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:18.989 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:18.990 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:18.990 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:18.991 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:18.991 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:28:18.992 DEBUG:boto:Method: GET 2022-09-03T07:28:18.993 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:28:18.993 DEBUG:boto:Data: 2022-09-03T07:28:18.994 DEBUG:boto:Headers: {} 2022-09-03T07:28:18.994 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:18.995 DEBUG:boto:Port: 8000 2022-09-03T07:28:18.995 DEBUG:boto:Params: {} 2022-09-03T07:28:18.996 DEBUG:boto:Token: None 2022-09-03T07:28:18.996 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:18 GMT /avysmw-75/ 2022-09-03T07:28:18.997 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg= 2022-09-03T07:28:18.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:18 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nLaZypUs5nAe/gDnp3OOJY2qdfg=', 'Content-Length': '0'} 2022-09-03T07:28:19.000 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000032ae4042d14193a6-0063130212-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:18 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.001 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.002 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:28:19.002 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.003 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.004 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.004 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.005 DEBUG:boto:Method: GET 2022-09-03T07:28:19.006 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.007 DEBUG:boto:Data: 2022-09-03T07:28:19.008 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.008 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.009 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.010 DEBUG:boto:Params: {} 2022-09-03T07:28:19.012 DEBUG:boto:Token: None 2022-09-03T07:28:19.013 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.013 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ueWRD0g+b7OHI4jyf+T9o/s6Nw= 2022-09-03T07:28:19.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ueWRD0g+b7OHI4jyf+T9o/s6Nw=', 'Content-Length': '0'} 2022-09-03T07:28:19.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5dad22dead1029c-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.017 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.018 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.018 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.019 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.020 DEBUG:boto:Method: GET 2022-09-03T07:28:19.021 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.021 DEBUG:boto:Data: 2022-09-03T07:28:19.022 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.023 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.024 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.025 DEBUG:boto:Params: {} 2022-09-03T07:28:19.026 DEBUG:boto:Token: None 2022-09-03T07:28:19.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ueWRD0g+b7OHI4jyf+T9o/s6Nw= 2022-09-03T07:28:19.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ueWRD0g+b7OHI4jyf+T9o/s6Nw=', 'Content-Length': '0'} 2022-09-03T07:28:19.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba1091cc436896a8-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.032 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.033 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.034 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.035 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.035 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.036 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.037 DEBUG:boto:Data: 2022-09-03T07:28:19.038 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.039 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.040 DEBUG:boto:Params: {} 2022-09-03T07:28:19.041 DEBUG:boto:Token: None 2022-09-03T07:28:19.042 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UQXU/Fi2j4km+LAPdynDnigdOqk= 2022-09-03T07:28:19.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UQXU/Fi2j4km+LAPdynDnigdOqk=', 'Content-Length': '0'} 2022-09-03T07:28:19.047 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036d010ac50add2e2-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.048 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.049 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.050 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.051 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.052 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.052 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.053 DEBUG:boto:Data: 2022-09-03T07:28:19.054 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.055 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.055 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.056 DEBUG:boto:Params: {} 2022-09-03T07:28:19.056 DEBUG:boto:Token: None 2022-09-03T07:28:19.057 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UQXU/Fi2j4km+LAPdynDnigdOqk= 2022-09-03T07:28:19.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UQXU/Fi2j4km+LAPdynDnigdOqk=', 'Content-Length': '0'} 2022-09-03T07:28:19.061 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe072034138e0361-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.062 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:28:19.063 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.063 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.064 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.065 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.066 DEBUG:boto:Method: GET 2022-09-03T07:28:19.067 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.068 DEBUG:boto:Data: 2022-09-03T07:28:19.068 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.069 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.070 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.071 DEBUG:boto:Params: {} 2022-09-03T07:28:19.072 DEBUG:boto:Token: None 2022-09-03T07:28:19.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ueWRD0g+b7OHI4jyf+T9o/s6Nw= 2022-09-03T07:28:19.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ueWRD0g+b7OHI4jyf+T9o/s6Nw=', 'Content-Length': '0'} 2022-09-03T07:28:19.076 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c793a2da3d05fe37-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.077 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.078 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.079 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.080 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.081 DEBUG:boto:Method: GET 2022-09-03T07:28:19.082 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.083 DEBUG:boto:Data: 2022-09-03T07:28:19.083 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.084 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.085 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.086 DEBUG:boto:Params: {} 2022-09-03T07:28:19.086 DEBUG:boto:Token: None 2022-09-03T07:28:19.087 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:28:19.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5ueWRD0g+b7OHI4jyf+T9o/s6Nw= 2022-09-03T07:28:19.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5ueWRD0g+b7OHI4jyf+T9o/s6Nw=', 'Content-Length': '0'} 2022-09-03T07:28:19.091 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb08a4899cb54cfe-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.092 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.093 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.093 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:19.094 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:19.094 DEBUG:boto:Method: GET 2022-09-03T07:28:19.095 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:19.095 DEBUG:boto:Data: 2022-09-03T07:28:19.096 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.096 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.097 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.098 DEBUG:boto:Params: {} 2022-09-03T07:28:19.098 DEBUG:boto:Token: None 2022-09-03T07:28:19.099 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.104 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000587ea2a561d715b5-0063130213-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.107 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.108 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.109 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.110 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:19.111 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:19.112 DEBUG:boto:Method: GET 2022-09-03T07:28:19.113 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:19.114 DEBUG:boto:Data: 2022-09-03T07:28:19.115 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.116 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.117 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.119 DEBUG:boto:Params: {} 2022-09-03T07:28:19.120 DEBUG:boto:Token: None 2022-09-03T07:28:19.120 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.125 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000000a521ecb019a01-0063130213-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.170 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.172 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.172 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.173 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.174 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:19.174 DEBUG:boto:Data: 2022-09-03T07:28:19.175 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.175 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.176 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.177 DEBUG:boto:Params: {} 2022-09-03T07:28:19.177 DEBUG:boto:Token: None 2022-09-03T07:28:19.178 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31 2022-09-03T07:28:19.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lgdfz/Msdmu+x8H5TAIUcPoonIg= 2022-09-03T07:28:19.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lgdfz/Msdmu+x8H5TAIUcPoonIg=', 'Content-Length': '0'} 2022-09-03T07:28:19.185 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b1c0cbe90a384cc-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.187 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.187 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.188 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.189 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:19.193 DEBUG:boto:Data: 2022-09-03T07:28:19.193 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.194 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.194 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.195 DEBUG:boto:Params: {} 2022-09-03T07:28:19.196 DEBUG:boto:Token: None 2022-09-03T07:28:19.197 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31 2022-09-03T07:28:19.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lgdfz/Msdmu+x8H5TAIUcPoonIg= 2022-09-03T07:28:19.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lgdfz/Msdmu+x8H5TAIUcPoonIg=', 'Content-Length': '0'} 2022-09-03T07:28:19.202 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000afab8c7f9080eb9-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.203 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:28:19.204 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.204 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.205 DEBUG:boto:Method: GET 2022-09-03T07:28:19.205 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:19.206 DEBUG:boto:Data: 2022-09-03T07:28:19.207 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.207 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.208 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.208 DEBUG:boto:Params: {} 2022-09-03T07:28:19.209 DEBUG:boto:Token: None 2022-09-03T07:28:19.209 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31 2022-09-03T07:28:19.210 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JfTNRE+Z1y+uK37K1N9jbsnXNI8= 2022-09-03T07:28:19.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JfTNRE+Z1y+uK37K1N9jbsnXNI8=', 'Content-Length': '0'} 2022-09-03T07:28:19.213 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000855049269ea940b-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.214 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:28:19.214 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:28:19.215 DEBUG:boto:Method: GET 2022-09-03T07:28:19.215 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:28:19.216 DEBUG:boto:Data: 2022-09-03T07:28:19.217 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.218 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.219 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.220 DEBUG:boto:Params: {} 2022-09-03T07:28:19.221 DEBUG:boto:Token: None 2022-09-03T07:28:19.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj31 2022-09-03T07:28:19.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JfTNRE+Z1y+uK37K1N9jbsnXNI8= 2022-09-03T07:28:19.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JfTNRE+Z1y+uK37K1N9jbsnXNI8=', 'Content-Length': '0'} 2022-09-03T07:28:19.227 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f4647c9e10f7ce5-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.229 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.230 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.230 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:19.231 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:19.232 DEBUG:boto:Method: GET 2022-09-03T07:28:19.232 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:19.233 DEBUG:boto:Data: 2022-09-03T07:28:19.234 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.235 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.236 DEBUG:boto:Params: {} 2022-09-03T07:28:19.237 DEBUG:boto:Token: None 2022-09-03T07:28:19.238 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.250 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003a02afea24cc9816-0063130213-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.252 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.254 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.255 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.256 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:19.257 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:28:19.258 DEBUG:boto:Method: GET 2022-09-03T07:28:19.259 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:28:19.260 DEBUG:boto:Data: 2022-09-03T07:28:19.261 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.262 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.264 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.265 DEBUG:boto:Params: {} 2022-09-03T07:28:19.267 DEBUG:boto:Token: None 2022-09-03T07:28:19.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.273 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002ecfef81efae056-0063130213-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.275 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.276 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:28:19.277 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.278 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.278 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.279 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.279 DEBUG:boto:Method: GET 2022-09-03T07:28:19.280 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.280 DEBUG:boto:Data: 2022-09-03T07:28:19.281 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.283 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.283 DEBUG:boto:Params: {} 2022-09-03T07:28:19.284 DEBUG:boto:Token: None 2022-09-03T07:28:19.284 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JReB0ZNiWk5pb+05FiEyNf8dB8M= 2022-09-03T07:28:19.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JReB0ZNiWk5pb+05FiEyNf8dB8M=', 'Content-Length': '0'} 2022-09-03T07:28:19.289 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000132cc66c747b362e-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.334 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.335 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.336 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.337 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.339 DEBUG:boto:Method: GET 2022-09-03T07:28:19.340 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.341 DEBUG:boto:Data: 2022-09-03T07:28:19.342 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.343 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.345 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.346 DEBUG:boto:Params: {} 2022-09-03T07:28:19.347 DEBUG:boto:Token: None 2022-09-03T07:28:19.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JReB0ZNiWk5pb+05FiEyNf8dB8M= 2022-09-03T07:28:19.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JReB0ZNiWk5pb+05FiEyNf8dB8M=', 'Content-Length': '0'} 2022-09-03T07:28:19.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005743f60d808f6882-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.353 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.353 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.354 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.354 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.355 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.355 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.356 DEBUG:boto:Data: 2022-09-03T07:28:19.357 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.357 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.358 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.358 DEBUG:boto:Params: {} 2022-09-03T07:28:19.360 DEBUG:boto:Token: None 2022-09-03T07:28:19.361 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ugX2FfmdIVx+SMiB21UpNQf5Vpo= 2022-09-03T07:28:19.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ugX2FfmdIVx+SMiB21UpNQf5Vpo=', 'Content-Length': '0'} 2022-09-03T07:28:19.366 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c7b04290eb07d4d-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.367 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.369 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.370 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.370 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.371 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.372 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.372 DEBUG:boto:Data: 2022-09-03T07:28:19.373 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.374 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.375 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.376 DEBUG:boto:Params: {} 2022-09-03T07:28:19.376 DEBUG:boto:Token: None 2022-09-03T07:28:19.377 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.378 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ugX2FfmdIVx+SMiB21UpNQf5Vpo= 2022-09-03T07:28:19.379 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ugX2FfmdIVx+SMiB21UpNQf5Vpo=', 'Content-Length': '0'} 2022-09-03T07:28:19.381 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fa076f78144d532-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.382 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:28:19.382 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.383 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.384 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.385 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.386 DEBUG:boto:Method: GET 2022-09-03T07:28:19.387 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.388 DEBUG:boto:Data: 2022-09-03T07:28:19.389 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.391 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.392 DEBUG:boto:Params: {} 2022-09-03T07:28:19.393 DEBUG:boto:Token: None 2022-09-03T07:28:19.394 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JReB0ZNiWk5pb+05FiEyNf8dB8M= 2022-09-03T07:28:19.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JReB0ZNiWk5pb+05FiEyNf8dB8M=', 'Content-Length': '0'} 2022-09-03T07:28:19.398 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056a12725019e9f26-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.399 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.400 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.401 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.402 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.403 DEBUG:boto:Method: GET 2022-09-03T07:28:19.403 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.404 DEBUG:boto:Data: 2022-09-03T07:28:19.404 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.405 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.405 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.406 DEBUG:boto:Params: {} 2022-09-03T07:28:19.406 DEBUG:boto:Token: None 2022-09-03T07:28:19.407 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:28:19.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JReB0ZNiWk5pb+05FiEyNf8dB8M= 2022-09-03T07:28:19.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JReB0ZNiWk5pb+05FiEyNf8dB8M=', 'Content-Length': '0'} 2022-09-03T07:28:19.413 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c5e479a31e689f9-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.415 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.416 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.417 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:19.418 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:19.419 DEBUG:boto:Method: GET 2022-09-03T07:28:19.420 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:19.420 DEBUG:boto:Data: 2022-09-03T07:28:19.421 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.422 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.423 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.424 DEBUG:boto:Params: {} 2022-09-03T07:28:19.425 DEBUG:boto:Token: None 2022-09-03T07:28:19.426 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.427 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.433 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000024e0408ae9da30a7-0063130213-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.478 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.480 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.481 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.486 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:19.487 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:19.489 DEBUG:boto:Method: GET 2022-09-03T07:28:19.490 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:19.491 DEBUG:boto:Data: 2022-09-03T07:28:19.492 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.493 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.494 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.495 DEBUG:boto:Params: {} 2022-09-03T07:28:19.497 DEBUG:boto:Token: None 2022-09-03T07:28:19.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.502 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000349b628d38599ea9-0063130213-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.503 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.504 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.505 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.506 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.507 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:19.508 DEBUG:boto:Data: 2022-09-03T07:28:19.510 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.512 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.512 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.513 DEBUG:boto:Params: {} 2022-09-03T07:28:19.514 DEBUG:boto:Token: None 2022-09-03T07:28:19.515 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33 2022-09-03T07:28:19.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tx5BsNVt6I03LMV11qC6VH7lwCQ= 2022-09-03T07:28:19.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tx5BsNVt6I03LMV11qC6VH7lwCQ=', 'Content-Length': '0'} 2022-09-03T07:28:19.521 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dceaa0216bc6e146-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.522 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.524 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.525 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.525 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:19.526 DEBUG:boto:Data: 2022-09-03T07:28:19.527 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.528 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.529 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.531 DEBUG:boto:Params: {} 2022-09-03T07:28:19.532 DEBUG:boto:Token: None 2022-09-03T07:28:19.533 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33 2022-09-03T07:28:19.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Tx5BsNVt6I03LMV11qC6VH7lwCQ= 2022-09-03T07:28:19.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Tx5BsNVt6I03LMV11qC6VH7lwCQ=', 'Content-Length': '0'} 2022-09-03T07:28:19.539 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e86905b6ec656f9-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.541 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:28:19.543 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.544 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.545 DEBUG:boto:Method: GET 2022-09-03T07:28:19.546 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:19.547 DEBUG:boto:Data: 2022-09-03T07:28:19.548 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.549 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.550 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.552 DEBUG:boto:Params: {} 2022-09-03T07:28:19.553 DEBUG:boto:Token: None 2022-09-03T07:28:19.554 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33 2022-09-03T07:28:19.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BbIqx2rZ+ObZC3DgvbL+d97Nriw= 2022-09-03T07:28:19.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BbIqx2rZ+ObZC3DgvbL+d97Nriw=', 'Content-Length': '0'} 2022-09-03T07:28:19.559 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000041bf76dc7b2c7b46-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.563 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:28:19.564 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:28:19.565 DEBUG:boto:Method: GET 2022-09-03T07:28:19.566 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:28:19.567 DEBUG:boto:Data: 2022-09-03T07:28:19.569 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.569 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.570 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.571 DEBUG:boto:Params: {} 2022-09-03T07:28:19.572 DEBUG:boto:Token: None 2022-09-03T07:28:19.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj33 2022-09-03T07:28:19.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BbIqx2rZ+ObZC3DgvbL+d97Nriw= 2022-09-03T07:28:19.576 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BbIqx2rZ+ObZC3DgvbL+d97Nriw=', 'Content-Length': '0'} 2022-09-03T07:28:19.579 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000547763478251a667-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.622 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.623 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.624 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:19.625 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:19.626 DEBUG:boto:Method: GET 2022-09-03T07:28:19.627 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:19.628 DEBUG:boto:Data: 2022-09-03T07:28:19.629 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.630 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.631 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.632 DEBUG:boto:Params: {} 2022-09-03T07:28:19.633 DEBUG:boto:Token: None 2022-09-03T07:28:19.634 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.638 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.641 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d66b510865803262-0063130213-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.643 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.644 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.645 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.646 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:19.647 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:28:19.648 DEBUG:boto:Method: GET 2022-09-03T07:28:19.649 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:28:19.650 DEBUG:boto:Data: 2022-09-03T07:28:19.651 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.652 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.653 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.654 DEBUG:boto:Params: {} 2022-09-03T07:28:19.655 DEBUG:boto:Token: None 2022-09-03T07:28:19.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.661 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f6ce5e1c71d39908-0063130213-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.662 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.663 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:28:19.664 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.665 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.666 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.666 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.667 DEBUG:boto:Method: GET 2022-09-03T07:28:19.668 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.669 DEBUG:boto:Data: 2022-09-03T07:28:19.669 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.670 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.671 DEBUG:boto:Params: {} 2022-09-03T07:28:19.671 DEBUG:boto:Token: None 2022-09-03T07:28:19.672 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j+wBwqgxVAF2TtCGIB7edju4EPw= 2022-09-03T07:28:19.673 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j+wBwqgxVAF2TtCGIB7edju4EPw=', 'Content-Length': '0'} 2022-09-03T07:28:19.676 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003547f14ab745daa1-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.677 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.677 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.678 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.679 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.679 DEBUG:boto:Method: GET 2022-09-03T07:28:19.680 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.680 DEBUG:boto:Data: 2022-09-03T07:28:19.681 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.681 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.682 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.682 DEBUG:boto:Params: {} 2022-09-03T07:28:19.683 DEBUG:boto:Token: None 2022-09-03T07:28:19.684 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.685 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j+wBwqgxVAF2TtCGIB7edju4EPw= 2022-09-03T07:28:19.685 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j+wBwqgxVAF2TtCGIB7edju4EPw=', 'Content-Length': '0'} 2022-09-03T07:28:19.687 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd2dc7871c58d2e8-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.688 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.689 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.689 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.690 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.691 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.692 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.693 DEBUG:boto:Data: 2022-09-03T07:28:19.693 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.694 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.694 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.695 DEBUG:boto:Params: {} 2022-09-03T07:28:19.696 DEBUG:boto:Token: None 2022-09-03T07:28:19.697 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.697 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r0/p/rCXDvBSNIzUJTuzxUiLirU= 2022-09-03T07:28:19.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r0/p/rCXDvBSNIzUJTuzxUiLirU=', 'Content-Length': '0'} 2022-09-03T07:28:19.700 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000173d01d9ef957080-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.701 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.702 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.702 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.703 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.703 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.704 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.704 DEBUG:boto:Data: 2022-09-03T07:28:19.705 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.705 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.706 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.706 DEBUG:boto:Params: {} 2022-09-03T07:28:19.707 DEBUG:boto:Token: None 2022-09-03T07:28:19.708 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r0/p/rCXDvBSNIzUJTuzxUiLirU= 2022-09-03T07:28:19.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r0/p/rCXDvBSNIzUJTuzxUiLirU=', 'Content-Length': '0'} 2022-09-03T07:28:19.712 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045785c212bf0d264-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.714 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:28:19.726 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.727 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.728 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.729 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.730 DEBUG:boto:Method: GET 2022-09-03T07:28:19.732 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.733 DEBUG:boto:Data: 2022-09-03T07:28:19.733 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.735 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.736 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.736 DEBUG:boto:Params: {} 2022-09-03T07:28:19.738 DEBUG:boto:Token: None 2022-09-03T07:28:19.739 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j+wBwqgxVAF2TtCGIB7edju4EPw= 2022-09-03T07:28:19.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j+wBwqgxVAF2TtCGIB7edju4EPw=', 'Content-Length': '0'} 2022-09-03T07:28:19.744 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001736faccd83bbfb8-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.745 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.747 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.747 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.748 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.748 DEBUG:boto:Method: GET 2022-09-03T07:28:19.749 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.749 DEBUG:boto:Data: 2022-09-03T07:28:19.750 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.750 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.751 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.751 DEBUG:boto:Params: {} 2022-09-03T07:28:19.752 DEBUG:boto:Token: None 2022-09-03T07:28:19.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:28:19.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j+wBwqgxVAF2TtCGIB7edju4EPw= 2022-09-03T07:28:19.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j+wBwqgxVAF2TtCGIB7edju4EPw=', 'Content-Length': '0'} 2022-09-03T07:28:19.756 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049497998c8778159-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.757 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.758 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.758 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:19.759 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:19.759 DEBUG:boto:Method: GET 2022-09-03T07:28:19.760 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:19.760 DEBUG:boto:Data: 2022-09-03T07:28:19.761 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.761 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.762 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.763 DEBUG:boto:Params: {} 2022-09-03T07:28:19.763 DEBUG:boto:Token: None 2022-09-03T07:28:19.764 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.768 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000048b6b8338514ea91-0063130213-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.768 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.769 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.770 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.770 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:19.771 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:19.771 DEBUG:boto:Method: GET 2022-09-03T07:28:19.772 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:19.772 DEBUG:boto:Data: 2022-09-03T07:28:19.773 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.774 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.774 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.775 DEBUG:boto:Params: {} 2022-09-03T07:28:19.775 DEBUG:boto:Token: None 2022-09-03T07:28:19.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.781 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ea14b3aec1f1fe2e-0063130213-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.782 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.783 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.783 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.784 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.785 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:19.786 DEBUG:boto:Data: 2022-09-03T07:28:19.786 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.787 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.788 DEBUG:boto:Params: {} 2022-09-03T07:28:19.788 DEBUG:boto:Token: None 2022-09-03T07:28:19.789 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34 2022-09-03T07:28:19.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mdx3oCJGvEn/fajBPJcX5A6zW1I= 2022-09-03T07:28:19.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mdx3oCJGvEn/fajBPJcX5A6zW1I=', 'Content-Length': '0'} 2022-09-03T07:28:19.792 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f21a9f1f9b2ba01-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.793 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.794 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.794 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.795 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:19.795 DEBUG:boto:Data: 2022-09-03T07:28:19.796 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.796 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.797 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.797 DEBUG:boto:Params: {} 2022-09-03T07:28:19.798 DEBUG:boto:Token: None 2022-09-03T07:28:19.798 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34 2022-09-03T07:28:19.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mdx3oCJGvEn/fajBPJcX5A6zW1I= 2022-09-03T07:28:19.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mdx3oCJGvEn/fajBPJcX5A6zW1I=', 'Content-Length': '0'} 2022-09-03T07:28:19.802 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006800906fda1481a6-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.802 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:28:19.803 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.804 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.805 DEBUG:boto:Method: GET 2022-09-03T07:28:19.806 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:19.806 DEBUG:boto:Data: 2022-09-03T07:28:19.807 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.808 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.809 DEBUG:boto:Params: {} 2022-09-03T07:28:19.809 DEBUG:boto:Token: None 2022-09-03T07:28:19.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34 2022-09-03T07:28:19.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZEVIOLqoo42SUnzARJi/C2br8k= 2022-09-03T07:28:19.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZEVIOLqoo42SUnzARJi/C2br8k=', 'Content-Length': '0'} 2022-09-03T07:28:19.814 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ed588668e8631fe-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.815 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:28:19.815 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:28:19.816 DEBUG:boto:Method: GET 2022-09-03T07:28:19.816 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:28:19.817 DEBUG:boto:Data: 2022-09-03T07:28:19.817 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.818 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.818 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.819 DEBUG:boto:Params: {} 2022-09-03T07:28:19.820 DEBUG:boto:Token: None 2022-09-03T07:28:19.820 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj34 2022-09-03T07:28:19.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZEVIOLqoo42SUnzARJi/C2br8k= 2022-09-03T07:28:19.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZEVIOLqoo42SUnzARJi/C2br8k=', 'Content-Length': '0'} 2022-09-03T07:28:19.824 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000abe665b996aa11a2-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.825 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.826 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.827 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:19.828 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:19.829 DEBUG:boto:Method: GET 2022-09-03T07:28:19.830 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:19.831 DEBUG:boto:Data: 2022-09-03T07:28:19.832 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.832 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.833 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.834 DEBUG:boto:Params: {} 2022-09-03T07:28:19.834 DEBUG:boto:Token: None 2022-09-03T07:28:19.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.836 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.839 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a057ba073d98c4b-0063130213-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.840 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.841 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.841 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.842 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:19.842 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:28:19.843 DEBUG:boto:Method: GET 2022-09-03T07:28:19.843 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:28:19.844 DEBUG:boto:Data: 2022-09-03T07:28:19.845 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.845 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.846 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.847 DEBUG:boto:Params: {} 2022-09-03T07:28:19.847 DEBUG:boto:Token: None 2022-09-03T07:28:19.848 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.849 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.854 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000041f903cbfeedc73c-0063130213-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.855 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:19.856 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:28:19.857 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:19.858 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:19.858 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.859 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.860 DEBUG:boto:Method: GET 2022-09-03T07:28:19.861 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.861 DEBUG:boto:Data: 2022-09-03T07:28:19.862 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.863 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.864 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.865 DEBUG:boto:Params: {} 2022-09-03T07:28:19.865 DEBUG:boto:Token: None 2022-09-03T07:28:19.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q5hQbV/L562IV9/4G0mO0tnY4IY= 2022-09-03T07:28:19.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q5hQbV/L562IV9/4G0mO0tnY4IY=', 'Content-Length': '0'} 2022-09-03T07:28:19.870 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005aaa0b82390c7ed1-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.870 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:19.871 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:19.872 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.873 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.873 DEBUG:boto:Method: GET 2022-09-03T07:28:19.874 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.874 DEBUG:boto:Data: 2022-09-03T07:28:19.875 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.876 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.877 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.878 DEBUG:boto:Params: {} 2022-09-03T07:28:19.879 DEBUG:boto:Token: None 2022-09-03T07:28:19.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q5hQbV/L562IV9/4G0mO0tnY4IY= 2022-09-03T07:28:19.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q5hQbV/L562IV9/4G0mO0tnY4IY=', 'Content-Length': '0'} 2022-09-03T07:28:19.884 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b651f0b0a1de784-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.885 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:19.886 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:19.886 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.887 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.888 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.888 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.889 DEBUG:boto:Data: 2022-09-03T07:28:19.890 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.890 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.891 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.891 DEBUG:boto:Params: {} 2022-09-03T07:28:19.892 DEBUG:boto:Token: None 2022-09-03T07:28:19.892 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1wfiN4EE94mKQ30NFB7A/5ohrPA= 2022-09-03T07:28:19.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1wfiN4EE94mKQ30NFB7A/5ohrPA=', 'Content-Length': '0'} 2022-09-03T07:28:19.896 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000621efdf0b8e925e8-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.897 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:19.898 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:19.899 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.899 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.900 DEBUG:boto:Method: HEAD 2022-09-03T07:28:19.901 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.901 DEBUG:boto:Data: 2022-09-03T07:28:19.902 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.902 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.903 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.903 DEBUG:boto:Params: {} 2022-09-03T07:28:19.904 DEBUG:boto:Token: None 2022-09-03T07:28:19.905 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1wfiN4EE94mKQ30NFB7A/5ohrPA= 2022-09-03T07:28:19.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1wfiN4EE94mKQ30NFB7A/5ohrPA=', 'Content-Length': '0'} 2022-09-03T07:28:19.910 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000910771743b3787d5-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.911 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:28:19.912 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:19.913 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:19.914 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.914 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.915 DEBUG:boto:Method: GET 2022-09-03T07:28:19.916 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.917 DEBUG:boto:Data: 2022-09-03T07:28:19.918 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.919 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.920 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.921 DEBUG:boto:Params: {} 2022-09-03T07:28:19.921 DEBUG:boto:Token: None 2022-09-03T07:28:19.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q5hQbV/L562IV9/4G0mO0tnY4IY= 2022-09-03T07:28:19.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q5hQbV/L562IV9/4G0mO0tnY4IY=', 'Content-Length': '0'} 2022-09-03T07:28:19.926 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eaeb919173994ba9-0063130213-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.928 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:19.929 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:19.929 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.930 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.931 DEBUG:boto:Method: GET 2022-09-03T07:28:19.932 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.932 DEBUG:boto:Data: 2022-09-03T07:28:19.933 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.934 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:19.934 DEBUG:boto:Port: 8000 2022-09-03T07:28:19.935 DEBUG:boto:Params: {} 2022-09-03T07:28:19.936 DEBUG:boto:Token: None 2022-09-03T07:28:19.936 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:28:19.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q5hQbV/L562IV9/4G0mO0tnY4IY= 2022-09-03T07:28:19.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q5hQbV/L562IV9/4G0mO0tnY4IY=', 'Content-Length': '0'} 2022-09-03T07:28:19.941 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000040b934fbfba74c2e-0063130213-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:19.942 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:19.943 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:19.944 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:19.944 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:19.945 DEBUG:boto:Method: GET 2022-09-03T07:28:19.946 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:19.947 DEBUG:boto:Data: 2022-09-03T07:28:19.948 DEBUG:boto:Headers: {} 2022-09-03T07:28:19.949 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:19.950 DEBUG:boto:Port: 8001 2022-09-03T07:28:19.951 DEBUG:boto:Params: {} 2022-09-03T07:28:19.952 DEBUG:boto:Token: None 2022-09-03T07:28:19.954 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:19 GMT /avysmw-75/ 2022-09-03T07:28:19.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU= 2022-09-03T07:28:19.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:19 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lfV7rvyvUSBotmL30Xn/7Wg4XCU=', 'Content-Length': '0'} 2022-09-03T07:28:19.960 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000062414380cc69f1ab-0063130213-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:19 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.002 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.004 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.005 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.006 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:20.007 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:20.008 DEBUG:boto:Method: GET 2022-09-03T07:28:20.009 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:20.010 DEBUG:boto:Data: 2022-09-03T07:28:20.011 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.012 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.013 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.014 DEBUG:boto:Params: {} 2022-09-03T07:28:20.016 DEBUG:boto:Token: None 2022-09-03T07:28:20.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.022 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000666295ac0b17f8fb-0063130214-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.023 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.024 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:20.025 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:20.025 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.026 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:20.026 DEBUG:boto:Data: 2022-09-03T07:28:20.027 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.028 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.028 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.029 DEBUG:boto:Params: {} 2022-09-03T07:28:20.029 DEBUG:boto:Token: None 2022-09-03T07:28:20.030 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj38 2022-09-03T07:28:20.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s2miAcEQHu/i9oG8HV6S3LHCXSQ= 2022-09-03T07:28:20.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s2miAcEQHu/i9oG8HV6S3LHCXSQ=', 'Content-Length': '0'} 2022-09-03T07:28:20.034 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a6c201f37b226d4-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.035 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:20.036 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:20.037 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.037 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:20.038 DEBUG:boto:Data: 2022-09-03T07:28:20.039 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.039 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.040 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.041 DEBUG:boto:Params: {} 2022-09-03T07:28:20.041 DEBUG:boto:Token: None 2022-09-03T07:28:20.043 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj38 2022-09-03T07:28:20.044 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s2miAcEQHu/i9oG8HV6S3LHCXSQ= 2022-09-03T07:28:20.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s2miAcEQHu/i9oG8HV6S3LHCXSQ=', 'Content-Length': '0'} 2022-09-03T07:28:20.048 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003577e3fece541eda-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.049 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:28:20.051 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:20.052 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:20.053 DEBUG:boto:Method: GET 2022-09-03T07:28:20.054 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:20.055 DEBUG:boto:Data: 2022-09-03T07:28:20.056 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.056 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.057 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.058 DEBUG:boto:Params: {} 2022-09-03T07:28:20.059 DEBUG:boto:Token: None 2022-09-03T07:28:20.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj38 2022-09-03T07:28:20.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ke/6HRi7uW0AqYBSqwlGwx+rggM= 2022-09-03T07:28:20.063 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ke/6HRi7uW0AqYBSqwlGwx+rggM=', 'Content-Length': '0'} 2022-09-03T07:28:20.068 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a1075155cabef593-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.069 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:28:20.070 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:28:20.071 DEBUG:boto:Method: GET 2022-09-03T07:28:20.071 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:28:20.072 DEBUG:boto:Data: 2022-09-03T07:28:20.074 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.075 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.076 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.077 DEBUG:boto:Params: {} 2022-09-03T07:28:20.078 DEBUG:boto:Token: None 2022-09-03T07:28:20.079 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj38 2022-09-03T07:28:20.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ke/6HRi7uW0AqYBSqwlGwx+rggM= 2022-09-03T07:28:20.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ke/6HRi7uW0AqYBSqwlGwx+rggM=', 'Content-Length': '0'} 2022-09-03T07:28:20.083 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005510f8eb4649d46d-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.084 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.085 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.086 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:20.087 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:20.087 DEBUG:boto:Method: GET 2022-09-03T07:28:20.088 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:20.089 DEBUG:boto:Data: 2022-09-03T07:28:20.090 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.092 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.093 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.094 DEBUG:boto:Params: {} 2022-09-03T07:28:20.096 DEBUG:boto:Token: None 2022-09-03T07:28:20.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.102 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb4bd53f720ef6c1-0063130214-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.103 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.104 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.104 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.106 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:20.107 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:28:20.108 DEBUG:boto:Method: GET 2022-09-03T07:28:20.109 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:28:20.110 DEBUG:boto:Data: 2022-09-03T07:28:20.112 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.113 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.115 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.116 DEBUG:boto:Params: {} 2022-09-03T07:28:20.117 DEBUG:boto:Token: None 2022-09-03T07:28:20.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.128 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002bb95f7f7f748980-0063130214-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.170 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.171 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:28:20.172 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.172 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.173 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.173 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.174 DEBUG:boto:Method: GET 2022-09-03T07:28:20.174 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.175 DEBUG:boto:Data: 2022-09-03T07:28:20.176 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.177 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.177 DEBUG:boto:Params: {} 2022-09-03T07:28:20.178 DEBUG:boto:Token: None 2022-09-03T07:28:20.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j8hB+5n4IT7xodooCjLJMGEYXos= 2022-09-03T07:28:20.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j8hB+5n4IT7xodooCjLJMGEYXos=', 'Content-Length': '0'} 2022-09-03T07:28:20.182 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dbffab2bec8270de-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.183 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.183 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.184 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.184 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.185 DEBUG:boto:Method: GET 2022-09-03T07:28:20.185 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.186 DEBUG:boto:Data: 2022-09-03T07:28:20.187 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.187 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.188 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.188 DEBUG:boto:Params: {} 2022-09-03T07:28:20.189 DEBUG:boto:Token: None 2022-09-03T07:28:20.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.190 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j8hB+5n4IT7xodooCjLJMGEYXos= 2022-09-03T07:28:20.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j8hB+5n4IT7xodooCjLJMGEYXos=', 'Content-Length': '0'} 2022-09-03T07:28:20.196 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001fdb515c70083c90-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.198 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.199 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.200 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.200 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.202 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.203 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.204 DEBUG:boto:Data: 2022-09-03T07:28:20.204 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.205 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.206 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.207 DEBUG:boto:Params: {} 2022-09-03T07:28:20.209 DEBUG:boto:Token: None 2022-09-03T07:28:20.210 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bCi92mGAYe5x84R4GeqsTZxE2WU= 2022-09-03T07:28:20.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bCi92mGAYe5x84R4GeqsTZxE2WU=', 'Content-Length': '0'} 2022-09-03T07:28:20.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6e3d22a5268f683-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.217 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.218 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.219 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.220 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.221 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.222 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.223 DEBUG:boto:Data: 2022-09-03T07:28:20.224 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.225 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.226 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.227 DEBUG:boto:Params: {} 2022-09-03T07:28:20.228 DEBUG:boto:Token: None 2022-09-03T07:28:20.229 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bCi92mGAYe5x84R4GeqsTZxE2WU= 2022-09-03T07:28:20.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bCi92mGAYe5x84R4GeqsTZxE2WU=', 'Content-Length': '0'} 2022-09-03T07:28:20.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fcb16a90d4d4d63f-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.234 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:28:20.235 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.236 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.237 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.237 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.238 DEBUG:boto:Method: GET 2022-09-03T07:28:20.239 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.239 DEBUG:boto:Data: 2022-09-03T07:28:20.240 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.242 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.243 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.244 DEBUG:boto:Params: {} 2022-09-03T07:28:20.245 DEBUG:boto:Token: None 2022-09-03T07:28:20.246 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.247 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j8hB+5n4IT7xodooCjLJMGEYXos= 2022-09-03T07:28:20.248 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j8hB+5n4IT7xodooCjLJMGEYXos=', 'Content-Length': '0'} 2022-09-03T07:28:20.251 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f9c1382e11b051f-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.294 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.295 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.296 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.298 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.299 DEBUG:boto:Method: GET 2022-09-03T07:28:20.300 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.301 DEBUG:boto:Data: 2022-09-03T07:28:20.302 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.303 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.304 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.305 DEBUG:boto:Params: {} 2022-09-03T07:28:20.306 DEBUG:boto:Token: None 2022-09-03T07:28:20.308 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:28:20.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j8hB+5n4IT7xodooCjLJMGEYXos= 2022-09-03T07:28:20.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j8hB+5n4IT7xodooCjLJMGEYXos=', 'Content-Length': '0'} 2022-09-03T07:28:20.313 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038c902b2467cdca4-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.314 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.315 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.316 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:20.317 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:20.318 DEBUG:boto:Method: GET 2022-09-03T07:28:20.319 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:20.320 DEBUG:boto:Data: 2022-09-03T07:28:20.321 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.322 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.322 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.323 DEBUG:boto:Params: {} 2022-09-03T07:28:20.324 DEBUG:boto:Token: None 2022-09-03T07:28:20.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.330 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f0815625b4859ec-0063130214-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.331 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.333 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.334 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.335 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:20.336 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:20.337 DEBUG:boto:Method: GET 2022-09-03T07:28:20.338 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:20.339 DEBUG:boto:Data: 2022-09-03T07:28:20.339 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.340 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.341 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.343 DEBUG:boto:Params: {} 2022-09-03T07:28:20.344 DEBUG:boto:Token: None 2022-09-03T07:28:20.345 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.351 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c5d23031bd8b152-0063130214-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.353 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.354 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.355 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.356 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.357 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:20.358 DEBUG:boto:Data: 2022-09-03T07:28:20.359 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.361 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.362 DEBUG:boto:Params: {} 2022-09-03T07:28:20.363 DEBUG:boto:Token: None 2022-09-03T07:28:20.363 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44 2022-09-03T07:28:20.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VKNCFEODwbf/o7m1zwLEutNj7NM= 2022-09-03T07:28:20.364 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VKNCFEODwbf/o7m1zwLEutNj7NM=', 'Content-Length': '0'} 2022-09-03T07:28:20.367 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005cc51eec2fcf5b9e-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.368 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.369 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.370 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.370 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:20.371 DEBUG:boto:Data: 2022-09-03T07:28:20.372 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.372 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.373 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.373 DEBUG:boto:Params: {} 2022-09-03T07:28:20.374 DEBUG:boto:Token: None 2022-09-03T07:28:20.375 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44 2022-09-03T07:28:20.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VKNCFEODwbf/o7m1zwLEutNj7NM= 2022-09-03T07:28:20.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VKNCFEODwbf/o7m1zwLEutNj7NM=', 'Content-Length': '0'} 2022-09-03T07:28:20.381 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000209d734aff83596e-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.382 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:28:20.383 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.384 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.386 DEBUG:boto:Method: GET 2022-09-03T07:28:20.387 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:20.388 DEBUG:boto:Data: 2022-09-03T07:28:20.389 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.391 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.392 DEBUG:boto:Params: {} 2022-09-03T07:28:20.394 DEBUG:boto:Token: None 2022-09-03T07:28:20.394 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44 2022-09-03T07:28:20.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zela9bsu7sipxTwkOeC5WP8O1hw= 2022-09-03T07:28:20.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zela9bsu7sipxTwkOeC5WP8O1hw=', 'Content-Length': '0'} 2022-09-03T07:28:20.400 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089052bc2c0e585cf-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.444 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:28:20.446 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:28:20.447 DEBUG:boto:Method: GET 2022-09-03T07:28:20.448 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:28:20.448 DEBUG:boto:Data: 2022-09-03T07:28:20.449 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.449 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.450 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.450 DEBUG:boto:Params: {} 2022-09-03T07:28:20.451 DEBUG:boto:Token: None 2022-09-03T07:28:20.452 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj44 2022-09-03T07:28:20.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zela9bsu7sipxTwkOeC5WP8O1hw= 2022-09-03T07:28:20.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zela9bsu7sipxTwkOeC5WP8O1hw=', 'Content-Length': '0'} 2022-09-03T07:28:20.456 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da61d769b186f322-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.458 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.459 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.460 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:20.461 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:20.463 DEBUG:boto:Method: GET 2022-09-03T07:28:20.464 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:20.465 DEBUG:boto:Data: 2022-09-03T07:28:20.466 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.467 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.468 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.469 DEBUG:boto:Params: {} 2022-09-03T07:28:20.471 DEBUG:boto:Token: None 2022-09-03T07:28:20.472 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.473 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.474 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.478 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d08925c8f5fe2d7e-0063130214-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.479 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.480 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.481 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.481 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:20.483 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:28:20.484 DEBUG:boto:Method: GET 2022-09-03T07:28:20.485 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:28:20.486 DEBUG:boto:Data: 2022-09-03T07:28:20.487 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.488 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.496 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.497 DEBUG:boto:Params: {} 2022-09-03T07:28:20.498 DEBUG:boto:Token: None 2022-09-03T07:28:20.499 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.508 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000082a7b3ea158d1133-0063130214-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.518 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.519 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:28:20.521 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.523 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.524 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.524 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.525 DEBUG:boto:Method: GET 2022-09-03T07:28:20.526 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.527 DEBUG:boto:Data: 2022-09-03T07:28:20.528 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.530 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.532 DEBUG:boto:Params: {} 2022-09-03T07:28:20.533 DEBUG:boto:Token: None 2022-09-03T07:28:20.534 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.535 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jhDb3bVaPANYISxOXYRvTEyAWAQ= 2022-09-03T07:28:20.536 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jhDb3bVaPANYISxOXYRvTEyAWAQ=', 'Content-Length': '0'} 2022-09-03T07:28:20.539 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007004cda1d69f11f9-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.541 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.542 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.543 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.544 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.545 DEBUG:boto:Method: GET 2022-09-03T07:28:20.546 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.547 DEBUG:boto:Data: 2022-09-03T07:28:20.548 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.549 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.550 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.551 DEBUG:boto:Params: {} 2022-09-03T07:28:20.552 DEBUG:boto:Token: None 2022-09-03T07:28:20.553 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jhDb3bVaPANYISxOXYRvTEyAWAQ= 2022-09-03T07:28:20.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jhDb3bVaPANYISxOXYRvTEyAWAQ=', 'Content-Length': '0'} 2022-09-03T07:28:20.562 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5919e9e004681b2-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.563 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.564 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.565 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.566 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.567 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.568 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.568 DEBUG:boto:Data: 2022-09-03T07:28:20.569 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.571 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.572 DEBUG:boto:Params: {} 2022-09-03T07:28:20.573 DEBUG:boto:Token: None 2022-09-03T07:28:20.574 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.575 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bG+zGKLObiNZ8jvoB/jAwFQ9wjk= 2022-09-03T07:28:20.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bG+zGKLObiNZ8jvoB/jAwFQ9wjk=', 'Content-Length': '0'} 2022-09-03T07:28:20.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000061183d08783e6825-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.579 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.581 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.582 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.582 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.583 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.584 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.585 DEBUG:boto:Data: 2022-09-03T07:28:20.585 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.586 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.587 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.587 DEBUG:boto:Params: {} 2022-09-03T07:28:20.588 DEBUG:boto:Token: None 2022-09-03T07:28:20.588 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bG+zGKLObiNZ8jvoB/jAwFQ9wjk= 2022-09-03T07:28:20.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bG+zGKLObiNZ8jvoB/jAwFQ9wjk=', 'Content-Length': '0'} 2022-09-03T07:28:20.598 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a42f322d9c6944c-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.599 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:28:20.600 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.601 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.602 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.603 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.604 DEBUG:boto:Method: GET 2022-09-03T07:28:20.604 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.605 DEBUG:boto:Data: 2022-09-03T07:28:20.609 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.610 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.611 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.611 DEBUG:boto:Params: {} 2022-09-03T07:28:20.612 DEBUG:boto:Token: None 2022-09-03T07:28:20.612 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.613 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jhDb3bVaPANYISxOXYRvTEyAWAQ= 2022-09-03T07:28:20.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jhDb3bVaPANYISxOXYRvTEyAWAQ=', 'Content-Length': '0'} 2022-09-03T07:28:20.616 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009140cad4a487bf75-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.617 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.617 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.618 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.618 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.619 DEBUG:boto:Method: GET 2022-09-03T07:28:20.620 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.620 DEBUG:boto:Data: 2022-09-03T07:28:20.621 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.621 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.622 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.622 DEBUG:boto:Params: {} 2022-09-03T07:28:20.623 DEBUG:boto:Token: None 2022-09-03T07:28:20.623 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:28:20.624 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jhDb3bVaPANYISxOXYRvTEyAWAQ= 2022-09-03T07:28:20.625 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jhDb3bVaPANYISxOXYRvTEyAWAQ=', 'Content-Length': '0'} 2022-09-03T07:28:20.629 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c36205ed23615668-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.630 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.631 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.631 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:20.632 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:20.632 DEBUG:boto:Method: GET 2022-09-03T07:28:20.633 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:20.634 DEBUG:boto:Data: 2022-09-03T07:28:20.635 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.637 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.648 DEBUG:boto:Params: {} 2022-09-03T07:28:20.649 DEBUG:boto:Token: None 2022-09-03T07:28:20.650 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.655 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e2fce32b855ca80a-0063130214-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.656 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.658 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.659 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.660 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:20.661 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:20.661 DEBUG:boto:Method: GET 2022-09-03T07:28:20.662 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:20.663 DEBUG:boto:Data: 2022-09-03T07:28:20.664 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.665 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.666 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.667 DEBUG:boto:Params: {} 2022-09-03T07:28:20.668 DEBUG:boto:Token: None 2022-09-03T07:28:20.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.676 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ab5c3130b5d4c67a-0063130214-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.720 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.721 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.722 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.722 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.723 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:20.723 DEBUG:boto:Data: 2022-09-03T07:28:20.724 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.724 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.725 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.725 DEBUG:boto:Params: {} 2022-09-03T07:28:20.726 DEBUG:boto:Token: None 2022-09-03T07:28:20.727 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46 2022-09-03T07:28:20.727 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NxgVM6ZlD3LD9ia/xb1pGrfbObM= 2022-09-03T07:28:20.728 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NxgVM6ZlD3LD9ia/xb1pGrfbObM=', 'Content-Length': '0'} 2022-09-03T07:28:20.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e34f552e4d88e497-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.733 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.733 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.734 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.734 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:20.735 DEBUG:boto:Data: 2022-09-03T07:28:20.735 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.736 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.737 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.737 DEBUG:boto:Params: {} 2022-09-03T07:28:20.738 DEBUG:boto:Token: None 2022-09-03T07:28:20.738 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46 2022-09-03T07:28:20.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NxgVM6ZlD3LD9ia/xb1pGrfbObM= 2022-09-03T07:28:20.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NxgVM6ZlD3LD9ia/xb1pGrfbObM=', 'Content-Length': '0'} 2022-09-03T07:28:20.742 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c192cffd14e1ed2d-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.743 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:28:20.743 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.744 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.745 DEBUG:boto:Method: GET 2022-09-03T07:28:20.745 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:20.746 DEBUG:boto:Data: 2022-09-03T07:28:20.746 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.748 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.748 DEBUG:boto:Params: {} 2022-09-03T07:28:20.749 DEBUG:boto:Token: None 2022-09-03T07:28:20.749 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46 2022-09-03T07:28:20.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:67dMxfBPF6WyKiyi1/ME0C+xkDU= 2022-09-03T07:28:20.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:67dMxfBPF6WyKiyi1/ME0C+xkDU=', 'Content-Length': '0'} 2022-09-03T07:28:20.753 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022feb2b77313f516-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.794 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:28:20.796 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:28:20.797 DEBUG:boto:Method: GET 2022-09-03T07:28:20.798 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:28:20.799 DEBUG:boto:Data: 2022-09-03T07:28:20.799 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.801 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.801 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.802 DEBUG:boto:Params: {} 2022-09-03T07:28:20.805 DEBUG:boto:Token: None 2022-09-03T07:28:20.805 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj46 2022-09-03T07:28:20.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:67dMxfBPF6WyKiyi1/ME0C+xkDU= 2022-09-03T07:28:20.807 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:67dMxfBPF6WyKiyi1/ME0C+xkDU=', 'Content-Length': '0'} 2022-09-03T07:28:20.809 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c67570c776ceaac-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.810 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.811 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.812 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:20.812 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:20.813 DEBUG:boto:Method: GET 2022-09-03T07:28:20.813 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:20.815 DEBUG:boto:Data: 2022-09-03T07:28:20.816 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.819 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.819 DEBUG:boto:Params: {} 2022-09-03T07:28:20.820 DEBUG:boto:Token: None 2022-09-03T07:28:20.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.827 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e605010a8cfe6a9c-0063130214-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.828 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.829 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.829 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.830 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:20.830 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:28:20.831 DEBUG:boto:Method: GET 2022-09-03T07:28:20.832 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:28:20.833 DEBUG:boto:Data: 2022-09-03T07:28:20.835 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.836 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.836 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.837 DEBUG:boto:Params: {} 2022-09-03T07:28:20.838 DEBUG:boto:Token: None 2022-09-03T07:28:20.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.844 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d1c33a8d09687b68-0063130214-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.845 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.846 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:28:20.847 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:20.847 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:20.848 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.849 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.850 DEBUG:boto:Method: GET 2022-09-03T07:28:20.850 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.851 DEBUG:boto:Data: 2022-09-03T07:28:20.852 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.853 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.855 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.856 DEBUG:boto:Params: {} 2022-09-03T07:28:20.857 DEBUG:boto:Token: None 2022-09-03T07:28:20.858 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KXZrMGkh6QzyDQk353VSifwMXbA= 2022-09-03T07:28:20.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KXZrMGkh6QzyDQk353VSifwMXbA=', 'Content-Length': '0'} 2022-09-03T07:28:20.863 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c43e02627544656-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.864 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:20.865 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:20.865 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.866 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.866 DEBUG:boto:Method: GET 2022-09-03T07:28:20.867 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.867 DEBUG:boto:Data: 2022-09-03T07:28:20.869 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.869 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.870 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.871 DEBUG:boto:Params: {} 2022-09-03T07:28:20.871 DEBUG:boto:Token: None 2022-09-03T07:28:20.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.873 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KXZrMGkh6QzyDQk353VSifwMXbA= 2022-09-03T07:28:20.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KXZrMGkh6QzyDQk353VSifwMXbA=', 'Content-Length': '0'} 2022-09-03T07:28:20.876 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000175f8064a04ee42c-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.879 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:20.880 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:20.880 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.881 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.882 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.882 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.883 DEBUG:boto:Data: 2022-09-03T07:28:20.883 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.885 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.885 DEBUG:boto:Params: {} 2022-09-03T07:28:20.886 DEBUG:boto:Token: None 2022-09-03T07:28:20.886 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aVuRmzv2JzqSYJrQf6RjEh0oLTg= 2022-09-03T07:28:20.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aVuRmzv2JzqSYJrQf6RjEh0oLTg=', 'Content-Length': '0'} 2022-09-03T07:28:20.890 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c6de7bbe55a55b65-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.891 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:20.891 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:20.892 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.892 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.893 DEBUG:boto:Method: HEAD 2022-09-03T07:28:20.893 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.894 DEBUG:boto:Data: 2022-09-03T07:28:20.894 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.895 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.895 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.896 DEBUG:boto:Params: {} 2022-09-03T07:28:20.896 DEBUG:boto:Token: None 2022-09-03T07:28:20.897 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aVuRmzv2JzqSYJrQf6RjEh0oLTg= 2022-09-03T07:28:20.898 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aVuRmzv2JzqSYJrQf6RjEh0oLTg=', 'Content-Length': '0'} 2022-09-03T07:28:20.900 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a7133b4322ebcdc-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.903 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:28:20.903 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:20.904 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:20.904 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.905 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.905 DEBUG:boto:Method: GET 2022-09-03T07:28:20.906 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.907 DEBUG:boto:Data: 2022-09-03T07:28:20.908 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.910 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.911 DEBUG:boto:Params: {} 2022-09-03T07:28:20.912 DEBUG:boto:Token: None 2022-09-03T07:28:20.913 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KXZrMGkh6QzyDQk353VSifwMXbA= 2022-09-03T07:28:20.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KXZrMGkh6QzyDQk353VSifwMXbA=', 'Content-Length': '0'} 2022-09-03T07:28:20.919 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f439a7181930a9d-0063130214-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.966 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:20.967 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:20.967 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.968 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.969 DEBUG:boto:Method: GET 2022-09-03T07:28:20.969 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.970 DEBUG:boto:Data: 2022-09-03T07:28:20.971 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.972 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:20.973 DEBUG:boto:Port: 8000 2022-09-03T07:28:20.974 DEBUG:boto:Params: {} 2022-09-03T07:28:20.975 DEBUG:boto:Token: None 2022-09-03T07:28:20.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:28:20.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KXZrMGkh6QzyDQk353VSifwMXbA= 2022-09-03T07:28:20.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KXZrMGkh6QzyDQk353VSifwMXbA=', 'Content-Length': '0'} 2022-09-03T07:28:20.979 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e819be59c3dc9294-0063130214-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.981 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.981 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.982 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:20.983 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:20.983 DEBUG:boto:Method: GET 2022-09-03T07:28:20.984 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:20.985 DEBUG:boto:Data: 2022-09-03T07:28:20.985 DEBUG:boto:Headers: {} 2022-09-03T07:28:20.986 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:20.986 DEBUG:boto:Port: 8001 2022-09-03T07:28:20.987 DEBUG:boto:Params: {} 2022-09-03T07:28:20.988 DEBUG:boto:Token: None 2022-09-03T07:28:20.989 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:20 GMT /avysmw-75/ 2022-09-03T07:28:20.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0= 2022-09-03T07:28:20.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:20 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PJBs3lv4gX9RepLk14AZFQS20q0=', 'Content-Length': '0'} 2022-09-03T07:28:20.993 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000255feddb64727bef-0063130214-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:20 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:20.994 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:20.995 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:20.996 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:20.996 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:20.997 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:20.998 DEBUG:boto:Method: GET 2022-09-03T07:28:20.999 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:21.000 DEBUG:boto:Data: 2022-09-03T07:28:21.001 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.002 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.003 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.003 DEBUG:boto:Params: {} 2022-09-03T07:28:21.004 DEBUG:boto:Token: None 2022-09-03T07:28:21.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.006 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.011 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1ffc2a5548cbeb4-0063130215-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.011 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.012 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:21.013 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:21.013 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.014 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:21.014 DEBUG:boto:Data: 2022-09-03T07:28:21.015 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.016 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.017 DEBUG:boto:Params: {} 2022-09-03T07:28:21.018 DEBUG:boto:Token: None 2022-09-03T07:28:21.018 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj51 2022-09-03T07:28:21.019 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1ErUEWI8RmoFcjG0lvUu0JvNBuQ= 2022-09-03T07:28:21.019 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1ErUEWI8RmoFcjG0lvUu0JvNBuQ=', 'Content-Length': '0'} 2022-09-03T07:28:21.021 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006169c743498c8e87-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.022 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:21.023 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:21.023 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.024 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:21.024 DEBUG:boto:Data: 2022-09-03T07:28:21.025 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.025 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.026 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.026 DEBUG:boto:Params: {} 2022-09-03T07:28:21.027 DEBUG:boto:Token: None 2022-09-03T07:28:21.027 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj51 2022-09-03T07:28:21.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1ErUEWI8RmoFcjG0lvUu0JvNBuQ= 2022-09-03T07:28:21.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1ErUEWI8RmoFcjG0lvUu0JvNBuQ=', 'Content-Length': '0'} 2022-09-03T07:28:21.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001de067a577e806a2-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.032 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:28:21.032 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:21.033 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:21.033 DEBUG:boto:Method: GET 2022-09-03T07:28:21.034 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:21.034 DEBUG:boto:Data: 2022-09-03T07:28:21.035 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.036 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.036 DEBUG:boto:Params: {} 2022-09-03T07:28:21.037 DEBUG:boto:Token: None 2022-09-03T07:28:21.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj51 2022-09-03T07:28:21.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XNIRj0avEf+TRWPuW/A6fN+e+xI= 2022-09-03T07:28:21.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XNIRj0avEf+TRWPuW/A6fN+e+xI=', 'Content-Length': '0'} 2022-09-03T07:28:21.041 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d16df5f4cea8df21-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.042 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:28:21.042 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:28:21.043 DEBUG:boto:Method: GET 2022-09-03T07:28:21.043 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:28:21.044 DEBUG:boto:Data: 2022-09-03T07:28:21.045 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.045 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.046 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.047 DEBUG:boto:Params: {} 2022-09-03T07:28:21.047 DEBUG:boto:Token: None 2022-09-03T07:28:21.048 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj51 2022-09-03T07:28:21.049 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XNIRj0avEf+TRWPuW/A6fN+e+xI= 2022-09-03T07:28:21.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XNIRj0avEf+TRWPuW/A6fN+e+xI=', 'Content-Length': '0'} 2022-09-03T07:28:21.054 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d2a69c05a60352a6-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.098 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.099 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.100 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:21.101 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:21.102 DEBUG:boto:Method: GET 2022-09-03T07:28:21.103 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:21.104 DEBUG:boto:Data: 2022-09-03T07:28:21.105 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.105 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.106 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.107 DEBUG:boto:Params: {} 2022-09-03T07:28:21.108 DEBUG:boto:Token: None 2022-09-03T07:28:21.109 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.110 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.114 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000086c1ab0d8e85b02b-0063130215-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.116 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.117 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.118 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.119 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:21.119 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:28:21.120 DEBUG:boto:Method: GET 2022-09-03T07:28:21.121 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:28:21.122 DEBUG:boto:Data: 2022-09-03T07:28:21.124 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.124 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.125 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.126 DEBUG:boto:Params: {} 2022-09-03T07:28:21.126 DEBUG:boto:Token: None 2022-09-03T07:28:21.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.129 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.134 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2184778e79a57d8-0063130215-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.136 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.137 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:28:21.138 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.138 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.139 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.140 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.141 DEBUG:boto:Method: GET 2022-09-03T07:28:21.141 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.142 DEBUG:boto:Data: 2022-09-03T07:28:21.143 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.144 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.145 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.145 DEBUG:boto:Params: {} 2022-09-03T07:28:21.147 DEBUG:boto:Token: None 2022-09-03T07:28:21.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iWbYZiHXb7n/Gz8BJl+lu+86IZw= 2022-09-03T07:28:21.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iWbYZiHXb7n/Gz8BJl+lu+86IZw=', 'Content-Length': '0'} 2022-09-03T07:28:21.151 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b4e4a3e1c9f6cc11-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.152 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.153 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.153 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.154 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.154 DEBUG:boto:Method: GET 2022-09-03T07:28:21.155 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.156 DEBUG:boto:Data: 2022-09-03T07:28:21.156 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.157 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.157 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.159 DEBUG:boto:Params: {} 2022-09-03T07:28:21.159 DEBUG:boto:Token: None 2022-09-03T07:28:21.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iWbYZiHXb7n/Gz8BJl+lu+86IZw= 2022-09-03T07:28:21.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iWbYZiHXb7n/Gz8BJl+lu+86IZw=', 'Content-Length': '0'} 2022-09-03T07:28:21.165 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007600e1df0fd828be-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.166 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.167 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.168 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.169 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.170 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.171 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.171 DEBUG:boto:Data: 2022-09-03T07:28:21.172 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.173 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.174 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.175 DEBUG:boto:Params: {} 2022-09-03T07:28:21.176 DEBUG:boto:Token: None 2022-09-03T07:28:21.177 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b6WdaRhxk8cTcYPZsdUMExH0pio= 2022-09-03T07:28:21.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b6WdaRhxk8cTcYPZsdUMExH0pio=', 'Content-Length': '0'} 2022-09-03T07:28:21.181 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b1f1f00fbdca5857-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.182 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.182 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.183 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.183 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.184 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.184 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.185 DEBUG:boto:Data: 2022-09-03T07:28:21.185 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.186 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.187 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.187 DEBUG:boto:Params: {} 2022-09-03T07:28:21.188 DEBUG:boto:Token: None 2022-09-03T07:28:21.189 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:b6WdaRhxk8cTcYPZsdUMExH0pio= 2022-09-03T07:28:21.191 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:b6WdaRhxk8cTcYPZsdUMExH0pio=', 'Content-Length': '0'} 2022-09-03T07:28:21.193 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2faec54437aa4c4-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.194 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:28:21.194 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.195 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.197 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.199 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.201 DEBUG:boto:Method: GET 2022-09-03T07:28:21.202 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.203 DEBUG:boto:Data: 2022-09-03T07:28:21.204 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.205 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.206 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.210 DEBUG:boto:Params: {} 2022-09-03T07:28:21.211 DEBUG:boto:Token: None 2022-09-03T07:28:21.212 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iWbYZiHXb7n/Gz8BJl+lu+86IZw= 2022-09-03T07:28:21.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iWbYZiHXb7n/Gz8BJl+lu+86IZw=', 'Content-Length': '0'} 2022-09-03T07:28:21.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c64b23f8c523eb37-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.262 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.263 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.264 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.264 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.265 DEBUG:boto:Method: GET 2022-09-03T07:28:21.265 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.266 DEBUG:boto:Data: 2022-09-03T07:28:21.266 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.267 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.268 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.268 DEBUG:boto:Params: {} 2022-09-03T07:28:21.269 DEBUG:boto:Token: None 2022-09-03T07:28:21.270 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:28:21.270 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iWbYZiHXb7n/Gz8BJl+lu+86IZw= 2022-09-03T07:28:21.271 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iWbYZiHXb7n/Gz8BJl+lu+86IZw=', 'Content-Length': '0'} 2022-09-03T07:28:21.273 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009faed94b21bee1a9-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.274 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.274 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.275 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:21.275 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:21.276 DEBUG:boto:Method: GET 2022-09-03T07:28:21.277 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:21.277 DEBUG:boto:Data: 2022-09-03T07:28:21.278 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.279 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.280 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.281 DEBUG:boto:Params: {} 2022-09-03T07:28:21.282 DEBUG:boto:Token: None 2022-09-03T07:28:21.283 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.284 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.288 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a29e1e13eb3753c-0063130215-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.289 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.290 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.291 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.291 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:21.292 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:21.293 DEBUG:boto:Method: GET 2022-09-03T07:28:21.293 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:21.294 DEBUG:boto:Data: 2022-09-03T07:28:21.294 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.295 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.296 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.297 DEBUG:boto:Params: {} 2022-09-03T07:28:21.298 DEBUG:boto:Token: None 2022-09-03T07:28:21.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.304 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000224648435f4bd2d1-0063130215-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.305 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.306 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.308 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.308 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.309 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:21.310 DEBUG:boto:Data: 2022-09-03T07:28:21.311 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.312 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.313 DEBUG:boto:Params: {} 2022-09-03T07:28:21.314 DEBUG:boto:Token: None 2022-09-03T07:28:21.315 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55 2022-09-03T07:28:21.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k5DHgcFGNs0xBQ24tWDLpegeFNE= 2022-09-03T07:28:21.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k5DHgcFGNs0xBQ24tWDLpegeFNE=', 'Content-Length': '0'} 2022-09-03T07:28:21.319 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac0a0803dd031ad0-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.320 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.320 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.321 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.321 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:21.322 DEBUG:boto:Data: 2022-09-03T07:28:21.322 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.323 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.323 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.324 DEBUG:boto:Params: {} 2022-09-03T07:28:21.325 DEBUG:boto:Token: None 2022-09-03T07:28:21.326 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55 2022-09-03T07:28:21.327 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k5DHgcFGNs0xBQ24tWDLpegeFNE= 2022-09-03T07:28:21.328 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k5DHgcFGNs0xBQ24tWDLpegeFNE=', 'Content-Length': '0'} 2022-09-03T07:28:21.331 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ffe93a5dd24cd1ce-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.333 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:28:21.333 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.334 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.335 DEBUG:boto:Method: GET 2022-09-03T07:28:21.336 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:21.337 DEBUG:boto:Data: 2022-09-03T07:28:21.338 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.339 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.340 DEBUG:boto:Params: {} 2022-09-03T07:28:21.341 DEBUG:boto:Token: None 2022-09-03T07:28:21.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55 2022-09-03T07:28:21.342 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xy0FSidbum+E2z+IteTZVX9iMAc= 2022-09-03T07:28:21.343 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xy0FSidbum+E2z+IteTZVX9iMAc=', 'Content-Length': '0'} 2022-09-03T07:28:21.345 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000792308d1f1a5c733-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.346 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:28:21.346 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:28:21.347 DEBUG:boto:Method: GET 2022-09-03T07:28:21.347 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:28:21.348 DEBUG:boto:Data: 2022-09-03T07:28:21.348 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.349 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.349 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.350 DEBUG:boto:Params: {} 2022-09-03T07:28:21.350 DEBUG:boto:Token: None 2022-09-03T07:28:21.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj55 2022-09-03T07:28:21.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xy0FSidbum+E2z+IteTZVX9iMAc= 2022-09-03T07:28:21.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xy0FSidbum+E2z+IteTZVX9iMAc=', 'Content-Length': '0'} 2022-09-03T07:28:21.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009376a9905916654-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.400 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.404 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.405 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:21.406 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:21.408 DEBUG:boto:Method: GET 2022-09-03T07:28:21.409 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:21.410 DEBUG:boto:Data: 2022-09-03T07:28:21.411 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.411 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.412 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.413 DEBUG:boto:Params: {} 2022-09-03T07:28:21.414 DEBUG:boto:Token: None 2022-09-03T07:28:21.415 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.425 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079565c79444aac53-0063130215-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.425 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.426 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.427 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.427 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:21.428 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:28:21.429 DEBUG:boto:Method: GET 2022-09-03T07:28:21.429 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:28:21.430 DEBUG:boto:Data: 2022-09-03T07:28:21.430 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.431 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.431 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.432 DEBUG:boto:Params: {} 2022-09-03T07:28:21.433 DEBUG:boto:Token: None 2022-09-03T07:28:21.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.440 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002887d43938dd3058-0063130215-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.440 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.441 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:28:21.442 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.442 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.443 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.443 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.444 DEBUG:boto:Method: GET 2022-09-03T07:28:21.444 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.445 DEBUG:boto:Data: 2022-09-03T07:28:21.445 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.446 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.446 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.447 DEBUG:boto:Params: {} 2022-09-03T07:28:21.448 DEBUG:boto:Token: None 2022-09-03T07:28:21.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BRiizvNOpFU4H7MrCF/WvBbmgDs= 2022-09-03T07:28:21.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BRiizvNOpFU4H7MrCF/WvBbmgDs=', 'Content-Length': '0'} 2022-09-03T07:28:21.452 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031260e392e47fd2f-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.453 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.454 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.454 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.455 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.455 DEBUG:boto:Method: GET 2022-09-03T07:28:21.456 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.456 DEBUG:boto:Data: 2022-09-03T07:28:21.457 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.458 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.458 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.459 DEBUG:boto:Params: {} 2022-09-03T07:28:21.459 DEBUG:boto:Token: None 2022-09-03T07:28:21.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.460 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BRiizvNOpFU4H7MrCF/WvBbmgDs= 2022-09-03T07:28:21.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BRiizvNOpFU4H7MrCF/WvBbmgDs=', 'Content-Length': '0'} 2022-09-03T07:28:21.465 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3b4835c3169c84b-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.467 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.468 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.469 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.469 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.470 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.470 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.471 DEBUG:boto:Data: 2022-09-03T07:28:21.472 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.473 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.474 DEBUG:boto:Params: {} 2022-09-03T07:28:21.475 DEBUG:boto:Token: None 2022-09-03T07:28:21.477 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.478 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JyJLe2tiV/++TEjzlivSOy5D/Ck= 2022-09-03T07:28:21.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JyJLe2tiV/++TEjzlivSOy5D/Ck=', 'Content-Length': '0'} 2022-09-03T07:28:21.481 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd40d78d630f8d4f-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.483 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.484 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.485 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.485 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.486 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.487 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.487 DEBUG:boto:Data: 2022-09-03T07:28:21.488 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.488 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.489 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.490 DEBUG:boto:Params: {} 2022-09-03T07:28:21.490 DEBUG:boto:Token: None 2022-09-03T07:28:21.491 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JyJLe2tiV/++TEjzlivSOy5D/Ck= 2022-09-03T07:28:21.492 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JyJLe2tiV/++TEjzlivSOy5D/Ck=', 'Content-Length': '0'} 2022-09-03T07:28:21.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000026cb5f5d4897d99-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.496 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:28:21.496 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.497 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.497 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.498 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.498 DEBUG:boto:Method: GET 2022-09-03T07:28:21.499 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.499 DEBUG:boto:Data: 2022-09-03T07:28:21.500 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.501 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.502 DEBUG:boto:Params: {} 2022-09-03T07:28:21.503 DEBUG:boto:Token: None 2022-09-03T07:28:21.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.504 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BRiizvNOpFU4H7MrCF/WvBbmgDs= 2022-09-03T07:28:21.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BRiizvNOpFU4H7MrCF/WvBbmgDs=', 'Content-Length': '0'} 2022-09-03T07:28:21.507 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065b68d46d1119a73-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.510 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.511 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.512 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.513 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.514 DEBUG:boto:Method: GET 2022-09-03T07:28:21.516 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.517 DEBUG:boto:Data: 2022-09-03T07:28:21.518 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.519 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.520 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.521 DEBUG:boto:Params: {} 2022-09-03T07:28:21.522 DEBUG:boto:Token: None 2022-09-03T07:28:21.524 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:28:21.525 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BRiizvNOpFU4H7MrCF/WvBbmgDs= 2022-09-03T07:28:21.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BRiizvNOpFU4H7MrCF/WvBbmgDs=', 'Content-Length': '0'} 2022-09-03T07:28:21.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b374e954b34ac49-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.570 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.571 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.572 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:21.572 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:21.573 DEBUG:boto:Method: GET 2022-09-03T07:28:21.573 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:21.574 DEBUG:boto:Data: 2022-09-03T07:28:21.575 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.575 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.576 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.576 DEBUG:boto:Params: {} 2022-09-03T07:28:21.577 DEBUG:boto:Token: None 2022-09-03T07:28:21.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.582 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008efd88cc924c8ed3-0063130215-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.582 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.583 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.584 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.584 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:21.585 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:21.585 DEBUG:boto:Method: GET 2022-09-03T07:28:21.586 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:21.587 DEBUG:boto:Data: 2022-09-03T07:28:21.587 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.588 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.588 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.589 DEBUG:boto:Params: {} 2022-09-03T07:28:21.589 DEBUG:boto:Token: None 2022-09-03T07:28:21.590 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.594 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd35868fb557b6f5-0063130215-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.594 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.595 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.596 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.596 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.597 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:21.597 DEBUG:boto:Data: 2022-09-03T07:28:21.598 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.599 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.600 DEBUG:boto:Params: {} 2022-09-03T07:28:21.600 DEBUG:boto:Token: None 2022-09-03T07:28:21.601 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57 2022-09-03T07:28:21.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:d5DKIGN0fV/vOq0GQSEK/AN1rjQ= 2022-09-03T07:28:21.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:d5DKIGN0fV/vOq0GQSEK/AN1rjQ=', 'Content-Length': '0'} 2022-09-03T07:28:21.604 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5b1032bf9ce2c75-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.605 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.606 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.606 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.607 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:21.607 DEBUG:boto:Data: 2022-09-03T07:28:21.608 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.608 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.609 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.609 DEBUG:boto:Params: {} 2022-09-03T07:28:21.610 DEBUG:boto:Token: None 2022-09-03T07:28:21.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57 2022-09-03T07:28:21.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:d5DKIGN0fV/vOq0GQSEK/AN1rjQ= 2022-09-03T07:28:21.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:d5DKIGN0fV/vOq0GQSEK/AN1rjQ=', 'Content-Length': '0'} 2022-09-03T07:28:21.614 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8ecc0753aeedf67-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.615 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:28:21.615 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.616 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.616 DEBUG:boto:Method: GET 2022-09-03T07:28:21.617 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:21.617 DEBUG:boto:Data: 2022-09-03T07:28:21.618 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.619 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.619 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.620 DEBUG:boto:Params: {} 2022-09-03T07:28:21.620 DEBUG:boto:Token: None 2022-09-03T07:28:21.621 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57 2022-09-03T07:28:21.622 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VBboYpMNBeghtfRatQ6CGtcK36M= 2022-09-03T07:28:21.622 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VBboYpMNBeghtfRatQ6CGtcK36M=', 'Content-Length': '0'} 2022-09-03T07:28:21.624 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec32189d69177a70-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.625 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:28:21.626 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:28:21.627 DEBUG:boto:Method: GET 2022-09-03T07:28:21.627 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:28:21.628 DEBUG:boto:Data: 2022-09-03T07:28:21.628 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.629 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.630 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.630 DEBUG:boto:Params: {} 2022-09-03T07:28:21.631 DEBUG:boto:Token: None 2022-09-03T07:28:21.631 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj57 2022-09-03T07:28:21.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VBboYpMNBeghtfRatQ6CGtcK36M= 2022-09-03T07:28:21.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VBboYpMNBeghtfRatQ6CGtcK36M=', 'Content-Length': '0'} 2022-09-03T07:28:21.635 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000041ccbdf84a661bc6-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.636 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.636 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.637 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:21.637 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:21.638 DEBUG:boto:Method: GET 2022-09-03T07:28:21.638 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:21.639 DEBUG:boto:Data: 2022-09-03T07:28:21.639 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.641 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.641 DEBUG:boto:Params: {} 2022-09-03T07:28:21.642 DEBUG:boto:Token: None 2022-09-03T07:28:21.642 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.646 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000040dc066f56e5f235-0063130215-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.647 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.648 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.649 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.649 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:21.650 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:28:21.650 DEBUG:boto:Method: GET 2022-09-03T07:28:21.651 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:28:21.652 DEBUG:boto:Data: 2022-09-03T07:28:21.652 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.653 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.653 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.654 DEBUG:boto:Params: {} 2022-09-03T07:28:21.654 DEBUG:boto:Token: None 2022-09-03T07:28:21.655 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.659 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bfb77d13500df3fa-0063130215-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.660 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.661 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:28:21.661 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.662 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.663 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.663 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.664 DEBUG:boto:Method: GET 2022-09-03T07:28:21.664 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.665 DEBUG:boto:Data: 2022-09-03T07:28:21.665 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.666 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.667 DEBUG:boto:Params: {} 2022-09-03T07:28:21.667 DEBUG:boto:Token: None 2022-09-03T07:28:21.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:adlRMXZioBEZrtHVr2xP/N1+XeI= 2022-09-03T07:28:21.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:adlRMXZioBEZrtHVr2xP/N1+XeI=', 'Content-Length': '0'} 2022-09-03T07:28:21.672 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d51d47ab9647768-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.673 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.674 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.674 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.675 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.675 DEBUG:boto:Method: GET 2022-09-03T07:28:21.676 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.676 DEBUG:boto:Data: 2022-09-03T07:28:21.677 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.678 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.678 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.679 DEBUG:boto:Params: {} 2022-09-03T07:28:21.679 DEBUG:boto:Token: None 2022-09-03T07:28:21.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:adlRMXZioBEZrtHVr2xP/N1+XeI= 2022-09-03T07:28:21.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:adlRMXZioBEZrtHVr2xP/N1+XeI=', 'Content-Length': '0'} 2022-09-03T07:28:21.683 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f55bf28ef9b9ea4-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.726 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.728 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.728 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.729 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.729 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.730 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.730 DEBUG:boto:Data: 2022-09-03T07:28:21.731 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.732 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.732 DEBUG:boto:Params: {} 2022-09-03T07:28:21.733 DEBUG:boto:Token: None 2022-09-03T07:28:21.734 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qu6I9SXoX3oIdxs890y4LedMhZE= 2022-09-03T07:28:21.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qu6I9SXoX3oIdxs890y4LedMhZE=', 'Content-Length': '0'} 2022-09-03T07:28:21.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2fb30eb66a7bd6a-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.738 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.738 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.739 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.739 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.740 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.741 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.741 DEBUG:boto:Data: 2022-09-03T07:28:21.742 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.742 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.743 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.743 DEBUG:boto:Params: {} 2022-09-03T07:28:21.744 DEBUG:boto:Token: None 2022-09-03T07:28:21.745 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qu6I9SXoX3oIdxs890y4LedMhZE= 2022-09-03T07:28:21.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qu6I9SXoX3oIdxs890y4LedMhZE=', 'Content-Length': '0'} 2022-09-03T07:28:21.748 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004960e9434d62083a-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.749 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:28:21.749 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.750 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.751 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.751 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.752 DEBUG:boto:Method: GET 2022-09-03T07:28:21.753 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.753 DEBUG:boto:Data: 2022-09-03T07:28:21.754 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.754 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.755 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.755 DEBUG:boto:Params: {} 2022-09-03T07:28:21.756 DEBUG:boto:Token: None 2022-09-03T07:28:21.756 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.757 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:adlRMXZioBEZrtHVr2xP/N1+XeI= 2022-09-03T07:28:21.758 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:adlRMXZioBEZrtHVr2xP/N1+XeI=', 'Content-Length': '0'} 2022-09-03T07:28:21.761 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001de937337ca9afb7-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.762 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.763 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.764 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.764 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.765 DEBUG:boto:Method: GET 2022-09-03T07:28:21.766 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.767 DEBUG:boto:Data: 2022-09-03T07:28:21.768 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.768 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.769 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.770 DEBUG:boto:Params: {} 2022-09-03T07:28:21.770 DEBUG:boto:Token: None 2022-09-03T07:28:21.772 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:28:21.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:adlRMXZioBEZrtHVr2xP/N1+XeI= 2022-09-03T07:28:21.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:adlRMXZioBEZrtHVr2xP/N1+XeI=', 'Content-Length': '0'} 2022-09-03T07:28:21.777 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080760cf60ac3f06b-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.778 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.779 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.780 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:21.781 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:21.782 DEBUG:boto:Method: GET 2022-09-03T07:28:21.783 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:21.783 DEBUG:boto:Data: 2022-09-03T07:28:21.784 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.784 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.785 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.785 DEBUG:boto:Params: {} 2022-09-03T07:28:21.786 DEBUG:boto:Token: None 2022-09-03T07:28:21.787 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.793 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a15e48b3e4779118-0063130215-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.793 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.795 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.796 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.796 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:21.797 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:21.798 DEBUG:boto:Method: GET 2022-09-03T07:28:21.798 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:21.799 DEBUG:boto:Data: 2022-09-03T07:28:21.800 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.800 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.801 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.801 DEBUG:boto:Params: {} 2022-09-03T07:28:21.802 DEBUG:boto:Token: None 2022-09-03T07:28:21.802 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.806 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000abcd378facc33b64-0063130215-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.807 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.808 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.808 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.809 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.809 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:21.810 DEBUG:boto:Data: 2022-09-03T07:28:21.810 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.811 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.812 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.812 DEBUG:boto:Params: {} 2022-09-03T07:28:21.813 DEBUG:boto:Token: None 2022-09-03T07:28:21.813 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58 2022-09-03T07:28:21.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7pwPwR/Mkc9qW5QBPxSaHVDmg+A= 2022-09-03T07:28:21.815 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7pwPwR/Mkc9qW5QBPxSaHVDmg+A=', 'Content-Length': '0'} 2022-09-03T07:28:21.817 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004faec959799058b3-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.818 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.819 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.819 DEBUG:boto:Method: HEAD 2022-09-03T07:28:21.820 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:21.820 DEBUG:boto:Data: 2022-09-03T07:28:21.821 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.822 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.822 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.823 DEBUG:boto:Params: {} 2022-09-03T07:28:21.824 DEBUG:boto:Token: None 2022-09-03T07:28:21.825 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58 2022-09-03T07:28:21.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7pwPwR/Mkc9qW5QBPxSaHVDmg+A= 2022-09-03T07:28:21.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7pwPwR/Mkc9qW5QBPxSaHVDmg+A=', 'Content-Length': '0'} 2022-09-03T07:28:21.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f22d08130c3a00e-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.831 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:28:21.831 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.832 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.833 DEBUG:boto:Method: GET 2022-09-03T07:28:21.834 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:21.835 DEBUG:boto:Data: 2022-09-03T07:28:21.836 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.837 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.838 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.839 DEBUG:boto:Params: {} 2022-09-03T07:28:21.840 DEBUG:boto:Token: None 2022-09-03T07:28:21.841 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58 2022-09-03T07:28:21.843 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MWkbSjZya0BginUuUdd+n4Whvrs= 2022-09-03T07:28:21.844 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MWkbSjZya0BginUuUdd+n4Whvrs=', 'Content-Length': '0'} 2022-09-03T07:28:21.846 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8cf91dfc3a8e792-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.847 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:28:21.848 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:28:21.848 DEBUG:boto:Method: GET 2022-09-03T07:28:21.849 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:28:21.850 DEBUG:boto:Data: 2022-09-03T07:28:21.850 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.851 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.851 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.852 DEBUG:boto:Params: {} 2022-09-03T07:28:21.853 DEBUG:boto:Token: None 2022-09-03T07:28:21.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj58 2022-09-03T07:28:21.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MWkbSjZya0BginUuUdd+n4Whvrs= 2022-09-03T07:28:21.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MWkbSjZya0BginUuUdd+n4Whvrs=', 'Content-Length': '0'} 2022-09-03T07:28:21.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006eb684cce6b90410-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.859 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.860 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.860 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:21.861 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:21.861 DEBUG:boto:Method: GET 2022-09-03T07:28:21.862 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:21.863 DEBUG:boto:Data: 2022-09-03T07:28:21.863 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.864 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.865 DEBUG:boto:Params: {} 2022-09-03T07:28:21.865 DEBUG:boto:Token: None 2022-09-03T07:28:21.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.871 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aad9367209a8cf63-0063130215-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.918 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.920 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:21.921 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:21.922 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:21.922 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:28:21.923 DEBUG:boto:Method: GET 2022-09-03T07:28:21.924 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:28:21.924 DEBUG:boto:Data: 2022-09-03T07:28:21.925 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.926 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.927 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.928 DEBUG:boto:Params: {} 2022-09-03T07:28:21.929 DEBUG:boto:Token: None 2022-09-03T07:28:21.930 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/ 2022-09-03T07:28:21.931 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4= 2022-09-03T07:28:21.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UBiqZxlU84UGi39R8Yq6qgoK4M4=', 'Content-Length': '0'} 2022-09-03T07:28:21.934 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fd26deab91323786-0063130215-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.935 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:21.936 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:28:21.936 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:21.937 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:21.937 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:21.938 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:21.938 DEBUG:boto:Method: GET 2022-09-03T07:28:21.939 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:21.940 DEBUG:boto:Data: 2022-09-03T07:28:21.941 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.941 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:21.942 DEBUG:boto:Port: 8001 2022-09-03T07:28:21.942 DEBUG:boto:Params: {} 2022-09-03T07:28:21.944 DEBUG:boto:Token: None 2022-09-03T07:28:21.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:21.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qIFFDSVtiHj6FegwSRqAHVLSb4g= 2022-09-03T07:28:21.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qIFFDSVtiHj6FegwSRqAHVLSb4g=', 'Content-Length': '0'} 2022-09-03T07:28:21.950 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f31d5cdacd04b3d0-0063130215-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:21.952 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:21.953 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:21.954 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:21.955 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:21.956 DEBUG:boto:Method: GET 2022-09-03T07:28:21.957 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:21.958 DEBUG:boto:Data: 2022-09-03T07:28:21.959 DEBUG:boto:Headers: {} 2022-09-03T07:28:21.960 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:21.961 DEBUG:boto:Port: 8000 2022-09-03T07:28:21.962 DEBUG:boto:Params: {} 2022-09-03T07:28:21.963 DEBUG:boto:Token: None 2022-09-03T07:28:21.964 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:21 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:21.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qIFFDSVtiHj6FegwSRqAHVLSb4g= 2022-09-03T07:28:21.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:21 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qIFFDSVtiHj6FegwSRqAHVLSb4g=', 'Content-Length': '0'} 2022-09-03T07:28:21.968 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa5aff0a6ed3eb78-0063130215-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:21 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.010 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:22.011 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:22.011 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.012 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.012 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.013 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.013 DEBUG:boto:Data: 2022-09-03T07:28:22.014 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.015 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.016 DEBUG:boto:Params: {} 2022-09-03T07:28:22.016 DEBUG:boto:Token: None 2022-09-03T07:28:22.017 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g5ggL8Ipl5KOWRRSBa9qGbNzgcI= 2022-09-03T07:28:22.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g5ggL8Ipl5KOWRRSBa9qGbNzgcI=', 'Content-Length': '0'} 2022-09-03T07:28:22.021 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe0d849d5ac8c4c1-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.021 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:22.022 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:22.023 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.023 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.024 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.024 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.025 DEBUG:boto:Data: 2022-09-03T07:28:22.026 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.026 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.027 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.028 DEBUG:boto:Params: {} 2022-09-03T07:28:22.029 DEBUG:boto:Token: None 2022-09-03T07:28:22.029 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.030 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g5ggL8Ipl5KOWRRSBa9qGbNzgcI= 2022-09-03T07:28:22.031 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g5ggL8Ipl5KOWRRSBa9qGbNzgcI=', 'Content-Length': '0'} 2022-09-03T07:28:22.033 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb30f35b08b1acaf-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.033 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:28:22.034 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:22.035 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:22.036 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.037 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.037 DEBUG:boto:Method: GET 2022-09-03T07:28:22.038 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.039 DEBUG:boto:Data: 2022-09-03T07:28:22.040 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.042 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.044 DEBUG:boto:Params: {} 2022-09-03T07:28:22.045 DEBUG:boto:Token: None 2022-09-03T07:28:22.045 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:guDBKlDRh7+CAf03+GCA35xRJTs= 2022-09-03T07:28:22.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:guDBKlDRh7+CAf03+GCA35xRJTs=', 'Content-Length': '0'} 2022-09-03T07:28:22.050 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076d32de8e98b427a-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.051 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:22.052 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:22.053 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.054 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.055 DEBUG:boto:Method: GET 2022-09-03T07:28:22.055 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.056 DEBUG:boto:Data: 2022-09-03T07:28:22.056 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.057 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.058 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.059 DEBUG:boto:Params: {} 2022-09-03T07:28:22.059 DEBUG:boto:Token: None 2022-09-03T07:28:22.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:28:22.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:guDBKlDRh7+CAf03+GCA35xRJTs= 2022-09-03T07:28:22.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:guDBKlDRh7+CAf03+GCA35xRJTs=', 'Content-Length': '0'} 2022-09-03T07:28:22.064 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f8242b453c9ce341-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.070 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.071 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.071 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:22.072 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:22.073 DEBUG:boto:Method: GET 2022-09-03T07:28:22.074 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:22.075 DEBUG:boto:Data: 2022-09-03T07:28:22.076 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.078 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.079 DEBUG:boto:Params: {} 2022-09-03T07:28:22.080 DEBUG:boto:Token: None 2022-09-03T07:28:22.080 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.085 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dfca8e25d74dff06-0063130216-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.088 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.090 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.091 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.092 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:22.093 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:22.093 DEBUG:boto:Method: GET 2022-09-03T07:28:22.094 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:22.095 DEBUG:boto:Data: 2022-09-03T07:28:22.096 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.096 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.097 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.097 DEBUG:boto:Params: {} 2022-09-03T07:28:22.098 DEBUG:boto:Token: None 2022-09-03T07:28:22.099 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.106 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003d1fa76c5dd3acdb-0063130216-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.107 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.108 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:22.108 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:22.109 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.109 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:22.110 DEBUG:boto:Data: 2022-09-03T07:28:22.111 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.112 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.112 DEBUG:boto:Params: {} 2022-09-03T07:28:22.113 DEBUG:boto:Token: None 2022-09-03T07:28:22.114 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj62 2022-09-03T07:28:22.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e6XHkl7jA8Qptl0k7vOO1R87+1Q= 2022-09-03T07:28:22.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e6XHkl7jA8Qptl0k7vOO1R87+1Q=', 'Content-Length': '0'} 2022-09-03T07:28:22.118 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e9129185dfe2e0e-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.119 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:22.119 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:22.120 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.121 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:22.122 DEBUG:boto:Data: 2022-09-03T07:28:22.123 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.124 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.125 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.125 DEBUG:boto:Params: {} 2022-09-03T07:28:22.127 DEBUG:boto:Token: None 2022-09-03T07:28:22.128 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj62 2022-09-03T07:28:22.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e6XHkl7jA8Qptl0k7vOO1R87+1Q= 2022-09-03T07:28:22.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e6XHkl7jA8Qptl0k7vOO1R87+1Q=', 'Content-Length': '0'} 2022-09-03T07:28:22.132 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000559b38112c9087f7-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.133 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:28:22.133 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:22.134 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:22.134 DEBUG:boto:Method: GET 2022-09-03T07:28:22.135 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:22.136 DEBUG:boto:Data: 2022-09-03T07:28:22.136 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.137 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.138 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.139 DEBUG:boto:Params: {} 2022-09-03T07:28:22.140 DEBUG:boto:Token: None 2022-09-03T07:28:22.141 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj62 2022-09-03T07:28:22.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BQUUIEHV+dJm4iR8vcMr0uAVZV8= 2022-09-03T07:28:22.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BQUUIEHV+dJm4iR8vcMr0uAVZV8=', 'Content-Length': '0'} 2022-09-03T07:28:22.147 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c1dc040c1faea6ce-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.148 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:28:22.148 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:28:22.149 DEBUG:boto:Method: GET 2022-09-03T07:28:22.149 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:28:22.150 DEBUG:boto:Data: 2022-09-03T07:28:22.150 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.151 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.151 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.152 DEBUG:boto:Params: {} 2022-09-03T07:28:22.152 DEBUG:boto:Token: None 2022-09-03T07:28:22.153 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj62 2022-09-03T07:28:22.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BQUUIEHV+dJm4iR8vcMr0uAVZV8= 2022-09-03T07:28:22.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BQUUIEHV+dJm4iR8vcMr0uAVZV8=', 'Content-Length': '0'} 2022-09-03T07:28:22.156 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d4492312ca206e4-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.157 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.158 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.158 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:22.159 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:22.159 DEBUG:boto:Method: GET 2022-09-03T07:28:22.160 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:22.160 DEBUG:boto:Data: 2022-09-03T07:28:22.161 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.161 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.162 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.163 DEBUG:boto:Params: {} 2022-09-03T07:28:22.164 DEBUG:boto:Token: None 2022-09-03T07:28:22.164 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.165 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.168 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000049bc8d45ccccb8d1-0063130216-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.210 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.211 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.212 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.212 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:22.213 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:28:22.214 DEBUG:boto:Method: GET 2022-09-03T07:28:22.215 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:28:22.216 DEBUG:boto:Data: 2022-09-03T07:28:22.217 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.218 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.219 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.220 DEBUG:boto:Params: {} 2022-09-03T07:28:22.221 DEBUG:boto:Token: None 2022-09-03T07:28:22.222 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.222 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.225 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004266619015862e54-0063130216-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.226 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.227 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:28:22.227 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.228 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.228 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.229 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.230 DEBUG:boto:Method: GET 2022-09-03T07:28:22.231 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.232 DEBUG:boto:Data: 2022-09-03T07:28:22.233 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.235 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.235 DEBUG:boto:Params: {} 2022-09-03T07:28:22.236 DEBUG:boto:Token: None 2022-09-03T07:28:22.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.238 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YBod8+n1uIjogdMhsn1FBTF5QY= 2022-09-03T07:28:22.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YBod8+n1uIjogdMhsn1FBTF5QY=', 'Content-Length': '0'} 2022-09-03T07:28:22.242 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000061947d87b12a4c00-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.244 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.245 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.245 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.246 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.247 DEBUG:boto:Method: GET 2022-09-03T07:28:22.248 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.248 DEBUG:boto:Data: 2022-09-03T07:28:22.249 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.250 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.251 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.251 DEBUG:boto:Params: {} 2022-09-03T07:28:22.252 DEBUG:boto:Token: None 2022-09-03T07:28:22.253 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YBod8+n1uIjogdMhsn1FBTF5QY= 2022-09-03T07:28:22.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YBod8+n1uIjogdMhsn1FBTF5QY=', 'Content-Length': '0'} 2022-09-03T07:28:22.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001c903e8a91b39421-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.257 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.258 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.259 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.259 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.260 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.260 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.261 DEBUG:boto:Data: 2022-09-03T07:28:22.261 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.262 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.263 DEBUG:boto:Params: {} 2022-09-03T07:28:22.263 DEBUG:boto:Token: None 2022-09-03T07:28:22.264 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pmi98jLVywVKUSyuM8hrU3rU5Mk= 2022-09-03T07:28:22.265 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pmi98jLVywVKUSyuM8hrU3rU5Mk=', 'Content-Length': '0'} 2022-09-03T07:28:22.267 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008d0bc38dca79c85-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.268 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.269 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.269 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.271 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.271 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.272 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.272 DEBUG:boto:Data: 2022-09-03T07:28:22.273 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.274 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.274 DEBUG:boto:Params: {} 2022-09-03T07:28:22.275 DEBUG:boto:Token: None 2022-09-03T07:28:22.275 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Pmi98jLVywVKUSyuM8hrU3rU5Mk= 2022-09-03T07:28:22.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Pmi98jLVywVKUSyuM8hrU3rU5Mk=', 'Content-Length': '0'} 2022-09-03T07:28:22.286 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000112bd33963c1ce62-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.287 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:28:22.288 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.289 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.290 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.291 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.292 DEBUG:boto:Method: GET 2022-09-03T07:28:22.293 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.294 DEBUG:boto:Data: 2022-09-03T07:28:22.294 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.295 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.295 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.296 DEBUG:boto:Params: {} 2022-09-03T07:28:22.296 DEBUG:boto:Token: None 2022-09-03T07:28:22.297 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YBod8+n1uIjogdMhsn1FBTF5QY= 2022-09-03T07:28:22.298 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YBod8+n1uIjogdMhsn1FBTF5QY=', 'Content-Length': '0'} 2022-09-03T07:28:22.301 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3335023cedaff88-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.301 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.302 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.303 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.303 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.304 DEBUG:boto:Method: GET 2022-09-03T07:28:22.304 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.305 DEBUG:boto:Data: 2022-09-03T07:28:22.305 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.306 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.306 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.307 DEBUG:boto:Params: {} 2022-09-03T07:28:22.308 DEBUG:boto:Token: None 2022-09-03T07:28:22.308 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:28:22.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/YBod8+n1uIjogdMhsn1FBTF5QY= 2022-09-03T07:28:22.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/YBod8+n1uIjogdMhsn1FBTF5QY=', 'Content-Length': '0'} 2022-09-03T07:28:22.312 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098f43d9db224121b-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.358 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.359 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.360 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:22.360 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:22.361 DEBUG:boto:Method: GET 2022-09-03T07:28:22.361 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:22.362 DEBUG:boto:Data: 2022-09-03T07:28:22.362 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.364 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.365 DEBUG:boto:Params: {} 2022-09-03T07:28:22.365 DEBUG:boto:Token: None 2022-09-03T07:28:22.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.371 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aea3de515624b862-0063130216-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.372 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.373 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.374 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.374 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:22.375 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:22.376 DEBUG:boto:Method: GET 2022-09-03T07:28:22.376 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:22.377 DEBUG:boto:Data: 2022-09-03T07:28:22.377 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.378 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.378 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.379 DEBUG:boto:Params: {} 2022-09-03T07:28:22.380 DEBUG:boto:Token: None 2022-09-03T07:28:22.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.387 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c94627d114aad45b-0063130216-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.387 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.388 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.389 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.389 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.390 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:22.390 DEBUG:boto:Data: 2022-09-03T07:28:22.391 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.391 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.392 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.392 DEBUG:boto:Params: {} 2022-09-03T07:28:22.393 DEBUG:boto:Token: None 2022-09-03T07:28:22.394 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68 2022-09-03T07:28:22.394 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q8cQy4o0ynkv9QI9PFvSmR+QZfg= 2022-09-03T07:28:22.395 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q8cQy4o0ynkv9QI9PFvSmR+QZfg=', 'Content-Length': '0'} 2022-09-03T07:28:22.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ced588570e99f10-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.398 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.398 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.399 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.400 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:22.400 DEBUG:boto:Data: 2022-09-03T07:28:22.401 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.401 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.402 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.402 DEBUG:boto:Params: {} 2022-09-03T07:28:22.403 DEBUG:boto:Token: None 2022-09-03T07:28:22.404 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68 2022-09-03T07:28:22.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q8cQy4o0ynkv9QI9PFvSmR+QZfg= 2022-09-03T07:28:22.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q8cQy4o0ynkv9QI9PFvSmR+QZfg=', 'Content-Length': '0'} 2022-09-03T07:28:22.407 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc0007c7fc00f98d-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.408 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:28:22.409 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.409 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.410 DEBUG:boto:Method: GET 2022-09-03T07:28:22.410 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:22.411 DEBUG:boto:Data: 2022-09-03T07:28:22.412 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.413 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.413 DEBUG:boto:Params: {} 2022-09-03T07:28:22.414 DEBUG:boto:Token: None 2022-09-03T07:28:22.414 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68 2022-09-03T07:28:22.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WFvyJkyrpEQDspIazIHrqyH+Y4g= 2022-09-03T07:28:22.416 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WFvyJkyrpEQDspIazIHrqyH+Y4g=', 'Content-Length': '0'} 2022-09-03T07:28:22.419 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b8b8d29bf945e94-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.462 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:28:22.463 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:28:22.463 DEBUG:boto:Method: GET 2022-09-03T07:28:22.465 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:28:22.465 DEBUG:boto:Data: 2022-09-03T07:28:22.466 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.467 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.468 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.468 DEBUG:boto:Params: {} 2022-09-03T07:28:22.469 DEBUG:boto:Token: None 2022-09-03T07:28:22.469 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj68 2022-09-03T07:28:22.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WFvyJkyrpEQDspIazIHrqyH+Y4g= 2022-09-03T07:28:22.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WFvyJkyrpEQDspIazIHrqyH+Y4g=', 'Content-Length': '0'} 2022-09-03T07:28:22.473 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f4ad2cc057f93e41-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.474 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.475 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.475 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:22.476 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:22.476 DEBUG:boto:Method: GET 2022-09-03T07:28:22.477 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:22.477 DEBUG:boto:Data: 2022-09-03T07:28:22.478 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.479 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.479 DEBUG:boto:Params: {} 2022-09-03T07:28:22.480 DEBUG:boto:Token: None 2022-09-03T07:28:22.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.486 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006e73b6a4a3eab296-0063130216-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.487 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.488 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.489 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.490 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:22.490 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:28:22.491 DEBUG:boto:Method: GET 2022-09-03T07:28:22.492 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:28:22.492 DEBUG:boto:Data: 2022-09-03T07:28:22.493 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.493 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.494 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.495 DEBUG:boto:Params: {} 2022-09-03T07:28:22.496 DEBUG:boto:Token: None 2022-09-03T07:28:22.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.502 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3aecb79c2a8b2b4-0063130216-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.503 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.504 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:28:22.505 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.506 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.506 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.507 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.508 DEBUG:boto:Method: GET 2022-09-03T07:28:22.508 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.509 DEBUG:boto:Data: 2022-09-03T07:28:22.510 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.510 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.511 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.511 DEBUG:boto:Params: {} 2022-09-03T07:28:22.512 DEBUG:boto:Token: None 2022-09-03T07:28:22.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s4ydoJhqJtT8iCfEBpeWjpDHXZU= 2022-09-03T07:28:22.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s4ydoJhqJtT8iCfEBpeWjpDHXZU=', 'Content-Length': '0'} 2022-09-03T07:28:22.517 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5c13766c9362e56-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.518 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.519 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.519 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.520 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.521 DEBUG:boto:Method: GET 2022-09-03T07:28:22.521 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.522 DEBUG:boto:Data: 2022-09-03T07:28:22.522 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.523 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.523 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.524 DEBUG:boto:Params: {} 2022-09-03T07:28:22.525 DEBUG:boto:Token: None 2022-09-03T07:28:22.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s4ydoJhqJtT8iCfEBpeWjpDHXZU= 2022-09-03T07:28:22.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s4ydoJhqJtT8iCfEBpeWjpDHXZU=', 'Content-Length': '0'} 2022-09-03T07:28:22.529 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000006bc5e95d6da2045-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.571 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.572 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.572 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.573 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.573 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.574 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.574 DEBUG:boto:Data: 2022-09-03T07:28:22.575 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.577 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.577 DEBUG:boto:Params: {} 2022-09-03T07:28:22.578 DEBUG:boto:Token: None 2022-09-03T07:28:22.579 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.581 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FAY1cIR863HikNQVSUhDeeAu+s= 2022-09-03T07:28:22.582 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FAY1cIR863HikNQVSUhDeeAu+s=', 'Content-Length': '0'} 2022-09-03T07:28:22.584 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e4be93dba4d9da7-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.585 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.585 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.586 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.587 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.587 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.588 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.588 DEBUG:boto:Data: 2022-09-03T07:28:22.589 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.589 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.590 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.590 DEBUG:boto:Params: {} 2022-09-03T07:28:22.591 DEBUG:boto:Token: None 2022-09-03T07:28:22.592 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FAY1cIR863HikNQVSUhDeeAu+s= 2022-09-03T07:28:22.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FAY1cIR863HikNQVSUhDeeAu+s=', 'Content-Length': '0'} 2022-09-03T07:28:22.595 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9673cc1ebf1a171-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.596 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:28:22.596 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.597 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.598 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.598 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.599 DEBUG:boto:Method: GET 2022-09-03T07:28:22.599 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.600 DEBUG:boto:Data: 2022-09-03T07:28:22.600 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.601 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.602 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.602 DEBUG:boto:Params: {} 2022-09-03T07:28:22.603 DEBUG:boto:Token: None 2022-09-03T07:28:22.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s4ydoJhqJtT8iCfEBpeWjpDHXZU= 2022-09-03T07:28:22.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s4ydoJhqJtT8iCfEBpeWjpDHXZU=', 'Content-Length': '0'} 2022-09-03T07:28:22.608 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000acdfe52f0b020b19-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.609 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.609 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.610 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.611 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.611 DEBUG:boto:Method: GET 2022-09-03T07:28:22.612 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.613 DEBUG:boto:Data: 2022-09-03T07:28:22.613 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.614 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.615 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.615 DEBUG:boto:Params: {} 2022-09-03T07:28:22.616 DEBUG:boto:Token: None 2022-09-03T07:28:22.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:28:22.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s4ydoJhqJtT8iCfEBpeWjpDHXZU= 2022-09-03T07:28:22.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s4ydoJhqJtT8iCfEBpeWjpDHXZU=', 'Content-Length': '0'} 2022-09-03T07:28:22.621 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002678ee2c00d6077a-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.622 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.623 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.623 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:22.624 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:22.624 DEBUG:boto:Method: GET 2022-09-03T07:28:22.625 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:22.626 DEBUG:boto:Data: 2022-09-03T07:28:22.626 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.627 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.628 DEBUG:boto:Params: {} 2022-09-03T07:28:22.629 DEBUG:boto:Token: None 2022-09-03T07:28:22.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.634 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090c18bf7e294bdb6-0063130216-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.634 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.636 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.636 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.637 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:22.637 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:22.638 DEBUG:boto:Method: GET 2022-09-03T07:28:22.639 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:22.639 DEBUG:boto:Data: 2022-09-03T07:28:22.640 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.641 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.641 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.642 DEBUG:boto:Params: {} 2022-09-03T07:28:22.642 DEBUG:boto:Token: None 2022-09-03T07:28:22.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.645 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.648 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022821dc68ed9d592-0063130216-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.649 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.650 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.651 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.651 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.652 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:22.653 DEBUG:boto:Data: 2022-09-03T07:28:22.654 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.655 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.656 DEBUG:boto:Params: {} 2022-09-03T07:28:22.657 DEBUG:boto:Token: None 2022-09-03T07:28:22.657 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69 2022-09-03T07:28:22.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UXsHTfaMnrGSHhN6iHB0iCH+8sg= 2022-09-03T07:28:22.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UXsHTfaMnrGSHhN6iHB0iCH+8sg=', 'Content-Length': '0'} 2022-09-03T07:28:22.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000061b556e009e16937-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.662 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.663 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.664 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.664 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:22.665 DEBUG:boto:Data: 2022-09-03T07:28:22.666 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.667 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.667 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.668 DEBUG:boto:Params: {} 2022-09-03T07:28:22.670 DEBUG:boto:Token: None 2022-09-03T07:28:22.671 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69 2022-09-03T07:28:22.672 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UXsHTfaMnrGSHhN6iHB0iCH+8sg= 2022-09-03T07:28:22.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UXsHTfaMnrGSHhN6iHB0iCH+8sg=', 'Content-Length': '0'} 2022-09-03T07:28:22.676 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6b42a0b019a90c6-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.677 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:28:22.678 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.679 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.679 DEBUG:boto:Method: GET 2022-09-03T07:28:22.680 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:22.680 DEBUG:boto:Data: 2022-09-03T07:28:22.681 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.682 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.691 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.693 DEBUG:boto:Params: {} 2022-09-03T07:28:22.694 DEBUG:boto:Token: None 2022-09-03T07:28:22.695 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69 2022-09-03T07:28:22.696 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lAT65WFLJ2bU1JTg932x5WIBRkw= 2022-09-03T07:28:22.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lAT65WFLJ2bU1JTg932x5WIBRkw=', 'Content-Length': '0'} 2022-09-03T07:28:22.702 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2ce2c32b03962b1-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.705 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:28:22.706 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:28:22.707 DEBUG:boto:Method: GET 2022-09-03T07:28:22.708 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:28:22.710 DEBUG:boto:Data: 2022-09-03T07:28:22.711 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.712 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.714 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.715 DEBUG:boto:Params: {} 2022-09-03T07:28:22.716 DEBUG:boto:Token: None 2022-09-03T07:28:22.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj69 2022-09-03T07:28:22.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lAT65WFLJ2bU1JTg932x5WIBRkw= 2022-09-03T07:28:22.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lAT65WFLJ2bU1JTg932x5WIBRkw=', 'Content-Length': '0'} 2022-09-03T07:28:22.720 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ead4b287b8dab4ad-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.721 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.722 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.723 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:22.723 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:22.724 DEBUG:boto:Method: GET 2022-09-03T07:28:22.725 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:22.726 DEBUG:boto:Data: 2022-09-03T07:28:22.726 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.727 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.728 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.729 DEBUG:boto:Params: {} 2022-09-03T07:28:22.730 DEBUG:boto:Token: None 2022-09-03T07:28:22.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.735 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000065d2045ee6f34f7f-0063130216-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.736 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.737 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.738 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.738 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:22.739 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:28:22.740 DEBUG:boto:Method: GET 2022-09-03T07:28:22.741 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:28:22.742 DEBUG:boto:Data: 2022-09-03T07:28:22.743 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.744 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.744 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.746 DEBUG:boto:Params: {} 2022-09-03T07:28:22.747 DEBUG:boto:Token: None 2022-09-03T07:28:22.748 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.757 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075fe41b5b720e1e2-0063130216-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.758 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.760 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:28:22.760 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.761 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.761 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.762 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.762 DEBUG:boto:Method: GET 2022-09-03T07:28:22.763 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.763 DEBUG:boto:Data: 2022-09-03T07:28:22.764 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.764 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.765 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.766 DEBUG:boto:Params: {} 2022-09-03T07:28:22.766 DEBUG:boto:Token: None 2022-09-03T07:28:22.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gHUraGuyohQY2nj3XX0h+BNa+Jw= 2022-09-03T07:28:22.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gHUraGuyohQY2nj3XX0h+BNa+Jw=', 'Content-Length': '0'} 2022-09-03T07:28:22.772 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc2f9ad60201b633-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.814 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.815 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.816 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.817 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.818 DEBUG:boto:Method: GET 2022-09-03T07:28:22.819 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.820 DEBUG:boto:Data: 2022-09-03T07:28:22.821 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.821 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.822 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.823 DEBUG:boto:Params: {} 2022-09-03T07:28:22.824 DEBUG:boto:Token: None 2022-09-03T07:28:22.825 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gHUraGuyohQY2nj3XX0h+BNa+Jw= 2022-09-03T07:28:22.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gHUraGuyohQY2nj3XX0h+BNa+Jw=', 'Content-Length': '0'} 2022-09-03T07:28:22.828 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ec493585c3931e1-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.829 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.830 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.831 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.831 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.832 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.832 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.833 DEBUG:boto:Data: 2022-09-03T07:28:22.834 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.836 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.836 DEBUG:boto:Params: {} 2022-09-03T07:28:22.838 DEBUG:boto:Token: None 2022-09-03T07:28:22.839 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zRCvXAXVyoJRGdfq7wPq2GTORTA= 2022-09-03T07:28:22.841 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zRCvXAXVyoJRGdfq7wPq2GTORTA=', 'Content-Length': '0'} 2022-09-03T07:28:22.843 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000902b296fc2b713-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.845 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.846 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.846 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.847 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.848 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.848 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.849 DEBUG:boto:Data: 2022-09-03T07:28:22.849 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.850 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.850 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.851 DEBUG:boto:Params: {} 2022-09-03T07:28:22.852 DEBUG:boto:Token: None 2022-09-03T07:28:22.852 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zRCvXAXVyoJRGdfq7wPq2GTORTA= 2022-09-03T07:28:22.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zRCvXAXVyoJRGdfq7wPq2GTORTA=', 'Content-Length': '0'} 2022-09-03T07:28:22.856 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098e8815bad69e0fb-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.857 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:28:22.858 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.858 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.859 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.859 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.860 DEBUG:boto:Method: GET 2022-09-03T07:28:22.861 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.862 DEBUG:boto:Data: 2022-09-03T07:28:22.863 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.865 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.866 DEBUG:boto:Params: {} 2022-09-03T07:28:22.867 DEBUG:boto:Token: None 2022-09-03T07:28:22.869 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gHUraGuyohQY2nj3XX0h+BNa+Jw= 2022-09-03T07:28:22.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gHUraGuyohQY2nj3XX0h+BNa+Jw=', 'Content-Length': '0'} 2022-09-03T07:28:22.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db6e7764750a621e-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.874 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.874 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.875 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.875 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.876 DEBUG:boto:Method: GET 2022-09-03T07:28:22.877 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.877 DEBUG:boto:Data: 2022-09-03T07:28:22.878 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.878 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.879 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.879 DEBUG:boto:Params: {} 2022-09-03T07:28:22.880 DEBUG:boto:Token: None 2022-09-03T07:28:22.882 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:28:22.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gHUraGuyohQY2nj3XX0h+BNa+Jw= 2022-09-03T07:28:22.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gHUraGuyohQY2nj3XX0h+BNa+Jw=', 'Content-Length': '0'} 2022-09-03T07:28:22.886 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b82b65a37190d5dc-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.887 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.887 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.888 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:22.888 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:22.889 DEBUG:boto:Method: GET 2022-09-03T07:28:22.889 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:22.890 DEBUG:boto:Data: 2022-09-03T07:28:22.891 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.891 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.892 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.892 DEBUG:boto:Params: {} 2022-09-03T07:28:22.893 DEBUG:boto:Token: None 2022-09-03T07:28:22.894 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.899 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045c97b4f46586003-0063130216-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.900 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.900 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.901 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.902 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:22.902 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:22.903 DEBUG:boto:Method: GET 2022-09-03T07:28:22.903 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:22.904 DEBUG:boto:Data: 2022-09-03T07:28:22.904 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.905 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.906 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.907 DEBUG:boto:Params: {} 2022-09-03T07:28:22.907 DEBUG:boto:Token: None 2022-09-03T07:28:22.908 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.912 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000efab3561e3d29df-0063130216-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.912 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:22.913 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.914 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.914 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.915 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:22.915 DEBUG:boto:Data: 2022-09-03T07:28:22.916 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.916 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.917 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.917 DEBUG:boto:Params: {} 2022-09-03T07:28:22.918 DEBUG:boto:Token: None 2022-09-03T07:28:22.919 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76 2022-09-03T07:28:22.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2QuKBZdEaeQaFjdUaNxzIlQGQD4= 2022-09-03T07:28:22.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2QuKBZdEaeQaFjdUaNxzIlQGQD4=', 'Content-Length': '0'} 2022-09-03T07:28:22.922 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000242a17e87e678b19-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.923 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.923 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.924 DEBUG:boto:Method: HEAD 2022-09-03T07:28:22.924 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:22.925 DEBUG:boto:Data: 2022-09-03T07:28:22.925 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.926 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.927 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.927 DEBUG:boto:Params: {} 2022-09-03T07:28:22.928 DEBUG:boto:Token: None 2022-09-03T07:28:22.928 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76 2022-09-03T07:28:22.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2QuKBZdEaeQaFjdUaNxzIlQGQD4= 2022-09-03T07:28:22.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2QuKBZdEaeQaFjdUaNxzIlQGQD4=', 'Content-Length': '0'} 2022-09-03T07:28:22.932 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024bee0324be70d70-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.932 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:28:22.933 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.934 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.934 DEBUG:boto:Method: GET 2022-09-03T07:28:22.935 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:22.935 DEBUG:boto:Data: 2022-09-03T07:28:22.936 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.937 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.937 DEBUG:boto:Params: {} 2022-09-03T07:28:22.938 DEBUG:boto:Token: None 2022-09-03T07:28:22.939 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76 2022-09-03T07:28:22.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UFVZJ1raaZjzz80FPKIMktm74Dw= 2022-09-03T07:28:22.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UFVZJ1raaZjzz80FPKIMktm74Dw=', 'Content-Length': '0'} 2022-09-03T07:28:22.942 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae6f19df465d9938-0063130216-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.943 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:28:22.943 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:28:22.944 DEBUG:boto:Method: GET 2022-09-03T07:28:22.945 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:28:22.945 DEBUG:boto:Data: 2022-09-03T07:28:22.946 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.946 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:22.947 DEBUG:boto:Port: 8000 2022-09-03T07:28:22.947 DEBUG:boto:Params: {} 2022-09-03T07:28:22.948 DEBUG:boto:Token: None 2022-09-03T07:28:22.955 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/obj76 2022-09-03T07:28:22.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UFVZJ1raaZjzz80FPKIMktm74Dw= 2022-09-03T07:28:22.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UFVZJ1raaZjzz80FPKIMktm74Dw=', 'Content-Length': '0'} 2022-09-03T07:28:22.958 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f8784f0e633f16b1-0063130216-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:22.960 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:22.961 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:22.962 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:22.963 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:22.964 DEBUG:boto:Method: GET 2022-09-03T07:28:22.965 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:22.966 DEBUG:boto:Data: 2022-09-03T07:28:22.967 DEBUG:boto:Headers: {} 2022-09-03T07:28:22.968 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:22.970 DEBUG:boto:Port: 8001 2022-09-03T07:28:22.971 DEBUG:boto:Params: {} 2022-09-03T07:28:22.972 DEBUG:boto:Token: None 2022-09-03T07:28:22.972 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:22 GMT /avysmw-75/ 2022-09-03T07:28:22.974 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo= 2022-09-03T07:28:22.975 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:22 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DRzQ31IarAcwLiOwwJI8TmLS+Uo=', 'Content-Length': '0'} 2022-09-03T07:28:22.978 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000008cc2b8d854e4a13-0063130216-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:22 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.023 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.024 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.025 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.025 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:23.026 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:28:23.026 DEBUG:boto:Method: GET 2022-09-03T07:28:23.027 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:28:23.027 DEBUG:boto:Data: 2022-09-03T07:28:23.028 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.031 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.032 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.033 DEBUG:boto:Params: {} 2022-09-03T07:28:23.034 DEBUG:boto:Token: None 2022-09-03T07:28:23.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.041 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000049edbe7c0733d645-0063130217-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.042 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.044 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:28:23.045 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.046 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.048 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.048 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.049 DEBUG:boto:Method: GET 2022-09-03T07:28:23.050 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.052 DEBUG:boto:Data: 2022-09-03T07:28:23.053 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.055 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.056 DEBUG:boto:Params: {} 2022-09-03T07:28:23.057 DEBUG:boto:Token: None 2022-09-03T07:28:23.058 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U4du3lINXafJxIjB6SAunvV4MFc= 2022-09-03T07:28:23.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U4du3lINXafJxIjB6SAunvV4MFc=', 'Content-Length': '0'} 2022-09-03T07:28:23.064 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089eea96dffb29cd2-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.065 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.066 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.067 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.068 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.069 DEBUG:boto:Method: GET 2022-09-03T07:28:23.070 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.071 DEBUG:boto:Data: 2022-09-03T07:28:23.072 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.074 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.074 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.075 DEBUG:boto:Params: {} 2022-09-03T07:28:23.076 DEBUG:boto:Token: None 2022-09-03T07:28:23.078 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.079 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U4du3lINXafJxIjB6SAunvV4MFc= 2022-09-03T07:28:23.079 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U4du3lINXafJxIjB6SAunvV4MFc=', 'Content-Length': '0'} 2022-09-03T07:28:23.082 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2a9426f61ef593e-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.083 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.084 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.084 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.085 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.085 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.086 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.086 DEBUG:boto:Data: 2022-09-03T07:28:23.087 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.088 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.089 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.090 DEBUG:boto:Params: {} 2022-09-03T07:28:23.091 DEBUG:boto:Token: None 2022-09-03T07:28:23.092 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LL4mfW1zN31a++nuJfv2dyQoZ6Q= 2022-09-03T07:28:23.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LL4mfW1zN31a++nuJfv2dyQoZ6Q=', 'Content-Length': '0'} 2022-09-03T07:28:23.096 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e0ac0b04fb6b70b-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.098 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.098 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.099 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.100 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.101 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.102 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.102 DEBUG:boto:Data: 2022-09-03T07:28:23.103 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.104 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.105 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.106 DEBUG:boto:Params: {} 2022-09-03T07:28:23.107 DEBUG:boto:Token: None 2022-09-03T07:28:23.108 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.109 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LL4mfW1zN31a++nuJfv2dyQoZ6Q= 2022-09-03T07:28:23.110 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LL4mfW1zN31a++nuJfv2dyQoZ6Q=', 'Content-Length': '0'} 2022-09-03T07:28:23.112 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004184e80a73297ab0-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.113 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:28:23.114 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.115 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.116 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.116 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.117 DEBUG:boto:Method: GET 2022-09-03T07:28:23.118 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.119 DEBUG:boto:Data: 2022-09-03T07:28:23.120 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.121 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.122 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.123 DEBUG:boto:Params: {} 2022-09-03T07:28:23.124 DEBUG:boto:Token: None 2022-09-03T07:28:23.125 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U4du3lINXafJxIjB6SAunvV4MFc= 2022-09-03T07:28:23.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U4du3lINXafJxIjB6SAunvV4MFc=', 'Content-Length': '0'} 2022-09-03T07:28:23.129 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000041cc29497e25d5d5-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.131 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.131 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.132 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.133 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.134 DEBUG:boto:Method: GET 2022-09-03T07:28:23.134 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.135 DEBUG:boto:Data: 2022-09-03T07:28:23.136 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.136 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.137 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.137 DEBUG:boto:Params: {} 2022-09-03T07:28:23.138 DEBUG:boto:Token: None 2022-09-03T07:28:23.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:28:23.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U4du3lINXafJxIjB6SAunvV4MFc= 2022-09-03T07:28:23.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U4du3lINXafJxIjB6SAunvV4MFc=', 'Content-Length': '0'} 2022-09-03T07:28:23.142 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cafe8e1ef05252ab-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.143 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.144 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.144 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:23.145 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:23.145 DEBUG:boto:Method: GET 2022-09-03T07:28:23.146 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:23.147 DEBUG:boto:Data: 2022-09-03T07:28:23.148 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.149 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.150 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.151 DEBUG:boto:Params: {} 2022-09-03T07:28:23.152 DEBUG:boto:Token: None 2022-09-03T07:28:23.153 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.157 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006421769a115ec577-0063130217-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.158 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.159 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.159 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.160 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:23.160 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:23.161 DEBUG:boto:Method: GET 2022-09-03T07:28:23.161 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:23.162 DEBUG:boto:Data: 2022-09-03T07:28:23.162 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.163 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.164 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.164 DEBUG:boto:Params: {} 2022-09-03T07:28:23.165 DEBUG:boto:Token: None 2022-09-03T07:28:23.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.169 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bcb94b995f4235a4-0063130217-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.210 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.212 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.212 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.213 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.214 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:23.215 DEBUG:boto:Data: 2022-09-03T07:28:23.216 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.216 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.217 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.217 DEBUG:boto:Params: {} 2022-09-03T07:28:23.218 DEBUG:boto:Token: None 2022-09-03T07:28:23.218 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85 2022-09-03T07:28:23.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W9M33JJ41vh9NNvYr4Qe0huShAs= 2022-09-03T07:28:23.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W9M33JJ41vh9NNvYr4Qe0huShAs=', 'Content-Length': '0'} 2022-09-03T07:28:23.222 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000971d30a99e59340a-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.224 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.224 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.225 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.226 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:23.227 DEBUG:boto:Data: 2022-09-03T07:28:23.228 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.229 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.229 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.230 DEBUG:boto:Params: {} 2022-09-03T07:28:23.231 DEBUG:boto:Token: None 2022-09-03T07:28:23.231 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85 2022-09-03T07:28:23.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W9M33JJ41vh9NNvYr4Qe0huShAs= 2022-09-03T07:28:23.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W9M33JJ41vh9NNvYr4Qe0huShAs=', 'Content-Length': '0'} 2022-09-03T07:28:23.236 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002486896ce736e79f-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.237 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:28:23.238 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.239 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.240 DEBUG:boto:Method: GET 2022-09-03T07:28:23.240 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:23.241 DEBUG:boto:Data: 2022-09-03T07:28:23.242 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.243 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.244 DEBUG:boto:Params: {} 2022-09-03T07:28:23.245 DEBUG:boto:Token: None 2022-09-03T07:28:23.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85 2022-09-03T07:28:23.246 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S6utGAKD+K+cvT54cnrMliUjoLs= 2022-09-03T07:28:23.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S6utGAKD+K+cvT54cnrMliUjoLs=', 'Content-Length': '0'} 2022-09-03T07:28:23.250 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e508ba11abc7257-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.290 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:28:23.291 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:28:23.292 DEBUG:boto:Method: GET 2022-09-03T07:28:23.292 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:28:23.293 DEBUG:boto:Data: 2022-09-03T07:28:23.295 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.295 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.296 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.296 DEBUG:boto:Params: {} 2022-09-03T07:28:23.297 DEBUG:boto:Token: None 2022-09-03T07:28:23.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj85 2022-09-03T07:28:23.298 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S6utGAKD+K+cvT54cnrMliUjoLs= 2022-09-03T07:28:23.299 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S6utGAKD+K+cvT54cnrMliUjoLs=', 'Content-Length': '0'} 2022-09-03T07:28:23.302 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e091d41509b1c58e-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.303 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.303 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.304 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:23.304 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:23.305 DEBUG:boto:Method: GET 2022-09-03T07:28:23.306 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:23.306 DEBUG:boto:Data: 2022-09-03T07:28:23.307 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.307 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.308 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.308 DEBUG:boto:Params: {} 2022-09-03T07:28:23.309 DEBUG:boto:Token: None 2022-09-03T07:28:23.310 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.311 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.316 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f3b70e191f3c005-0063130217-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.325 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.327 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.328 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.328 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:23.329 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:28:23.329 DEBUG:boto:Method: GET 2022-09-03T07:28:23.330 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:28:23.330 DEBUG:boto:Data: 2022-09-03T07:28:23.331 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.331 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.332 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.333 DEBUG:boto:Params: {} 2022-09-03T07:28:23.333 DEBUG:boto:Token: None 2022-09-03T07:28:23.334 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.338 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f801e6d8a9b19641-0063130217-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.339 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.340 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:28:23.341 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.342 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.342 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.343 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.344 DEBUG:boto:Method: GET 2022-09-03T07:28:23.344 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.345 DEBUG:boto:Data: 2022-09-03T07:28:23.345 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.346 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.346 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.347 DEBUG:boto:Params: {} 2022-09-03T07:28:23.348 DEBUG:boto:Token: None 2022-09-03T07:28:23.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eiQGHtwY5BTPxtMoFG5aCh7l/es= 2022-09-03T07:28:23.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eiQGHtwY5BTPxtMoFG5aCh7l/es=', 'Content-Length': '0'} 2022-09-03T07:28:23.356 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9e174c699f06261-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.358 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.359 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.359 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.360 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.360 DEBUG:boto:Method: GET 2022-09-03T07:28:23.361 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.362 DEBUG:boto:Data: 2022-09-03T07:28:23.362 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.363 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.363 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.364 DEBUG:boto:Params: {} 2022-09-03T07:28:23.365 DEBUG:boto:Token: None 2022-09-03T07:28:23.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eiQGHtwY5BTPxtMoFG5aCh7l/es= 2022-09-03T07:28:23.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eiQGHtwY5BTPxtMoFG5aCh7l/es=', 'Content-Length': '0'} 2022-09-03T07:28:23.370 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e83c73ec90cf4836-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.410 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.411 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.411 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.412 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.412 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.413 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.413 DEBUG:boto:Data: 2022-09-03T07:28:23.414 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.414 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.415 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.415 DEBUG:boto:Params: {} 2022-09-03T07:28:23.416 DEBUG:boto:Token: None 2022-09-03T07:28:23.416 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rY4gxaOJYGQXqyZ41y58uA4+ZdM= 2022-09-03T07:28:23.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rY4gxaOJYGQXqyZ41y58uA4+ZdM=', 'Content-Length': '0'} 2022-09-03T07:28:23.420 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000271259a92f301139-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.420 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.421 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.421 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.422 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.422 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.423 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.424 DEBUG:boto:Data: 2022-09-03T07:28:23.424 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.425 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.425 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.426 DEBUG:boto:Params: {} 2022-09-03T07:28:23.426 DEBUG:boto:Token: None 2022-09-03T07:28:23.427 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.427 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rY4gxaOJYGQXqyZ41y58uA4+ZdM= 2022-09-03T07:28:23.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rY4gxaOJYGQXqyZ41y58uA4+ZdM=', 'Content-Length': '0'} 2022-09-03T07:28:23.430 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad1688ab907602ce-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.431 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:28:23.432 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.432 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.433 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.434 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.434 DEBUG:boto:Method: GET 2022-09-03T07:28:23.435 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.436 DEBUG:boto:Data: 2022-09-03T07:28:23.436 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.437 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.437 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.438 DEBUG:boto:Params: {} 2022-09-03T07:28:23.438 DEBUG:boto:Token: None 2022-09-03T07:28:23.439 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eiQGHtwY5BTPxtMoFG5aCh7l/es= 2022-09-03T07:28:23.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eiQGHtwY5BTPxtMoFG5aCh7l/es=', 'Content-Length': '0'} 2022-09-03T07:28:23.443 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b608d143a1a4cfeb-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.444 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.444 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.445 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.445 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.446 DEBUG:boto:Method: GET 2022-09-03T07:28:23.447 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.447 DEBUG:boto:Data: 2022-09-03T07:28:23.448 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.449 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.449 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.450 DEBUG:boto:Params: {} 2022-09-03T07:28:23.450 DEBUG:boto:Token: None 2022-09-03T07:28:23.451 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:28:23.452 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eiQGHtwY5BTPxtMoFG5aCh7l/es= 2022-09-03T07:28:23.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eiQGHtwY5BTPxtMoFG5aCh7l/es=', 'Content-Length': '0'} 2022-09-03T07:28:23.455 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000073f7414b0be3ffdf-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.456 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.457 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.458 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:23.459 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:23.460 DEBUG:boto:Method: GET 2022-09-03T07:28:23.461 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:23.461 DEBUG:boto:Data: 2022-09-03T07:28:23.462 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.464 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.465 DEBUG:boto:Params: {} 2022-09-03T07:28:23.466 DEBUG:boto:Token: None 2022-09-03T07:28:23.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.472 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a4a08185b8b2cc9-0063130217-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.474 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.475 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.476 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.476 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:23.477 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:23.477 DEBUG:boto:Method: GET 2022-09-03T07:28:23.478 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:23.478 DEBUG:boto:Data: 2022-09-03T07:28:23.479 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.479 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.480 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.480 DEBUG:boto:Params: {} 2022-09-03T07:28:23.481 DEBUG:boto:Token: None 2022-09-03T07:28:23.482 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.488 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b0e783d51b64dd37-0063130217-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.530 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.531 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.532 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.533 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.534 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:23.534 DEBUG:boto:Data: 2022-09-03T07:28:23.535 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.536 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.536 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.537 DEBUG:boto:Params: {} 2022-09-03T07:28:23.537 DEBUG:boto:Token: None 2022-09-03T07:28:23.538 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87 2022-09-03T07:28:23.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EFI3WrLaPKSNvatFUt09iGyXgg4= 2022-09-03T07:28:23.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EFI3WrLaPKSNvatFUt09iGyXgg4=', 'Content-Length': '0'} 2022-09-03T07:28:23.542 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007876254ac73dec7c-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.543 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.544 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.545 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.546 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:23.547 DEBUG:boto:Data: 2022-09-03T07:28:23.548 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.550 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.551 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.552 DEBUG:boto:Params: {} 2022-09-03T07:28:23.553 DEBUG:boto:Token: None 2022-09-03T07:28:23.554 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87 2022-09-03T07:28:23.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EFI3WrLaPKSNvatFUt09iGyXgg4= 2022-09-03T07:28:23.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EFI3WrLaPKSNvatFUt09iGyXgg4=', 'Content-Length': '0'} 2022-09-03T07:28:23.559 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d10bac156695cac-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.560 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:28:23.560 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.561 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.561 DEBUG:boto:Method: GET 2022-09-03T07:28:23.562 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:23.563 DEBUG:boto:Data: 2022-09-03T07:28:23.563 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.564 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.565 DEBUG:boto:Params: {} 2022-09-03T07:28:23.565 DEBUG:boto:Token: None 2022-09-03T07:28:23.566 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87 2022-09-03T07:28:23.567 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3GqlbJ84xf2DnCTAcGiknxEu3sA= 2022-09-03T07:28:23.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3GqlbJ84xf2DnCTAcGiknxEu3sA=', 'Content-Length': '0'} 2022-09-03T07:28:23.570 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000016f421298f8a2bb0-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.571 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:28:23.571 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:28:23.572 DEBUG:boto:Method: GET 2022-09-03T07:28:23.572 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:28:23.573 DEBUG:boto:Data: 2022-09-03T07:28:23.574 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.574 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.575 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.575 DEBUG:boto:Params: {} 2022-09-03T07:28:23.576 DEBUG:boto:Token: None 2022-09-03T07:28:23.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj87 2022-09-03T07:28:23.577 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3GqlbJ84xf2DnCTAcGiknxEu3sA= 2022-09-03T07:28:23.578 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3GqlbJ84xf2DnCTAcGiknxEu3sA=', 'Content-Length': '0'} 2022-09-03T07:28:23.580 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e03c09d78a67d6ef-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.581 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.581 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.582 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:23.582 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:23.583 DEBUG:boto:Method: GET 2022-09-03T07:28:23.584 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:23.584 DEBUG:boto:Data: 2022-09-03T07:28:23.585 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.585 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.586 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.586 DEBUG:boto:Params: {} 2022-09-03T07:28:23.587 DEBUG:boto:Token: None 2022-09-03T07:28:23.587 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.592 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2db5f34f52468f8-0063130217-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.634 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.636 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.636 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.637 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:23.637 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:28:23.638 DEBUG:boto:Method: GET 2022-09-03T07:28:23.639 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:28:23.639 DEBUG:boto:Data: 2022-09-03T07:28:23.640 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.640 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.641 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.641 DEBUG:boto:Params: {} 2022-09-03T07:28:23.642 DEBUG:boto:Token: None 2022-09-03T07:28:23.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.647 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000929b5ae6e06eddb2-0063130217-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.648 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.649 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:28:23.649 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.650 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.650 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.651 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.652 DEBUG:boto:Method: GET 2022-09-03T07:28:23.652 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.653 DEBUG:boto:Data: 2022-09-03T07:28:23.653 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.654 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.655 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.655 DEBUG:boto:Params: {} 2022-09-03T07:28:23.656 DEBUG:boto:Token: None 2022-09-03T07:28:23.657 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qsXd8PaK2UVU6CJmTGh2BLouOtM= 2022-09-03T07:28:23.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qsXd8PaK2UVU6CJmTGh2BLouOtM=', 'Content-Length': '0'} 2022-09-03T07:28:23.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8cd79c7d0b9faaf-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.662 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.662 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.663 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.663 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.664 DEBUG:boto:Method: GET 2022-09-03T07:28:23.664 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.665 DEBUG:boto:Data: 2022-09-03T07:28:23.665 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.666 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.667 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.667 DEBUG:boto:Params: {} 2022-09-03T07:28:23.668 DEBUG:boto:Token: None 2022-09-03T07:28:23.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qsXd8PaK2UVU6CJmTGh2BLouOtM= 2022-09-03T07:28:23.670 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qsXd8PaK2UVU6CJmTGh2BLouOtM=', 'Content-Length': '0'} 2022-09-03T07:28:23.672 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4be0f8de0e89d92-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.718 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.719 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.720 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.721 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.722 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.723 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.724 DEBUG:boto:Data: 2022-09-03T07:28:23.724 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.725 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.726 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.727 DEBUG:boto:Params: {} 2022-09-03T07:28:23.728 DEBUG:boto:Token: None 2022-09-03T07:28:23.729 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u0oiEE9pUmkXrAkPumQR/WLVvSI= 2022-09-03T07:28:23.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u0oiEE9pUmkXrAkPumQR/WLVvSI=', 'Content-Length': '0'} 2022-09-03T07:28:23.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056edaea8c116579d-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.733 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.733 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.734 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.735 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.735 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.736 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.736 DEBUG:boto:Data: 2022-09-03T07:28:23.737 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.737 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.738 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.738 DEBUG:boto:Params: {} 2022-09-03T07:28:23.739 DEBUG:boto:Token: None 2022-09-03T07:28:23.739 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u0oiEE9pUmkXrAkPumQR/WLVvSI= 2022-09-03T07:28:23.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u0oiEE9pUmkXrAkPumQR/WLVvSI=', 'Content-Length': '0'} 2022-09-03T07:28:23.743 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e245b32b12d8440-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.744 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:28:23.746 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.746 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.747 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.747 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.748 DEBUG:boto:Method: GET 2022-09-03T07:28:23.748 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.749 DEBUG:boto:Data: 2022-09-03T07:28:23.750 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.750 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.751 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.751 DEBUG:boto:Params: {} 2022-09-03T07:28:23.752 DEBUG:boto:Token: None 2022-09-03T07:28:23.752 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qsXd8PaK2UVU6CJmTGh2BLouOtM= 2022-09-03T07:28:23.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qsXd8PaK2UVU6CJmTGh2BLouOtM=', 'Content-Length': '0'} 2022-09-03T07:28:23.756 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f4a5279cbb069776-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.758 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.759 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.760 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.760 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.761 DEBUG:boto:Method: GET 2022-09-03T07:28:23.762 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.762 DEBUG:boto:Data: 2022-09-03T07:28:23.763 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.764 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.764 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.765 DEBUG:boto:Params: {} 2022-09-03T07:28:23.767 DEBUG:boto:Token: None 2022-09-03T07:28:23.768 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:28:23.769 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qsXd8PaK2UVU6CJmTGh2BLouOtM= 2022-09-03T07:28:23.770 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qsXd8PaK2UVU6CJmTGh2BLouOtM=', 'Content-Length': '0'} 2022-09-03T07:28:23.773 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5808474baf1f6c7-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.774 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.774 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.775 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:23.775 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:23.777 DEBUG:boto:Method: GET 2022-09-03T07:28:23.778 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:23.779 DEBUG:boto:Data: 2022-09-03T07:28:23.780 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.781 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.781 DEBUG:boto:Params: {} 2022-09-03T07:28:23.782 DEBUG:boto:Token: None 2022-09-03T07:28:23.783 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.786 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006ce1e901938ad6dc-0063130217-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.787 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.788 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.789 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.789 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:23.790 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:23.790 DEBUG:boto:Method: GET 2022-09-03T07:28:23.791 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:23.791 DEBUG:boto:Data: 2022-09-03T07:28:23.792 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.792 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.793 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.793 DEBUG:boto:Params: {} 2022-09-03T07:28:23.794 DEBUG:boto:Token: None 2022-09-03T07:28:23.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.798 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000073d95cf03765ae18-0063130217-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.799 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.800 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.800 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.801 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.801 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:23.802 DEBUG:boto:Data: 2022-09-03T07:28:23.802 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.803 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.803 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.804 DEBUG:boto:Params: {} 2022-09-03T07:28:23.804 DEBUG:boto:Token: None 2022-09-03T07:28:23.805 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92 2022-09-03T07:28:23.805 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xPX3ywV+QtGXcTEhuIBcyKvpHBo= 2022-09-03T07:28:23.806 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xPX3ywV+QtGXcTEhuIBcyKvpHBo=', 'Content-Length': '0'} 2022-09-03T07:28:23.808 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c00b8604e8ec6046-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.809 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.810 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.810 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.811 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:23.811 DEBUG:boto:Data: 2022-09-03T07:28:23.812 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.812 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.813 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.813 DEBUG:boto:Params: {} 2022-09-03T07:28:23.814 DEBUG:boto:Token: None 2022-09-03T07:28:23.814 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92 2022-09-03T07:28:23.815 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xPX3ywV+QtGXcTEhuIBcyKvpHBo= 2022-09-03T07:28:23.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xPX3ywV+QtGXcTEhuIBcyKvpHBo=', 'Content-Length': '0'} 2022-09-03T07:28:23.817 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd89c1f7c816c94b-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.818 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:28:23.819 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.819 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.820 DEBUG:boto:Method: GET 2022-09-03T07:28:23.820 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:23.821 DEBUG:boto:Data: 2022-09-03T07:28:23.821 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.823 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.823 DEBUG:boto:Params: {} 2022-09-03T07:28:23.824 DEBUG:boto:Token: None 2022-09-03T07:28:23.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92 2022-09-03T07:28:23.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CclgsYg7uGgumiL27FQCjirq2AI= 2022-09-03T07:28:23.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CclgsYg7uGgumiL27FQCjirq2AI=', 'Content-Length': '0'} 2022-09-03T07:28:23.828 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e8687978a470f6f-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.829 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:28:23.829 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:28:23.830 DEBUG:boto:Method: GET 2022-09-03T07:28:23.830 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:28:23.831 DEBUG:boto:Data: 2022-09-03T07:28:23.831 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.832 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.833 DEBUG:boto:Params: {} 2022-09-03T07:28:23.837 DEBUG:boto:Token: None 2022-09-03T07:28:23.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj92 2022-09-03T07:28:23.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CclgsYg7uGgumiL27FQCjirq2AI= 2022-09-03T07:28:23.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CclgsYg7uGgumiL27FQCjirq2AI=', 'Content-Length': '0'} 2022-09-03T07:28:23.842 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000cb8792d93559e59-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.843 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.843 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.844 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:23.844 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:23.845 DEBUG:boto:Method: GET 2022-09-03T07:28:23.845 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:23.846 DEBUG:boto:Data: 2022-09-03T07:28:23.847 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.849 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.849 DEBUG:boto:Params: {} 2022-09-03T07:28:23.850 DEBUG:boto:Token: None 2022-09-03T07:28:23.851 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.855 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045c8827746b9e992-0063130217-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.855 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.856 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:23.857 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:23.857 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:23.858 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:28:23.858 DEBUG:boto:Method: GET 2022-09-03T07:28:23.859 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:28:23.859 DEBUG:boto:Data: 2022-09-03T07:28:23.860 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.860 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.861 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.861 DEBUG:boto:Params: {} 2022-09-03T07:28:23.862 DEBUG:boto:Token: None 2022-09-03T07:28:23.863 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/ 2022-09-03T07:28:23.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8= 2022-09-03T07:28:23.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzq5iu6o0S/htnM/GCcYC4AsoB8=', 'Content-Length': '0'} 2022-09-03T07:28:23.868 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b012634c4e442cc-0063130217-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.910 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:23.912 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:28:23.913 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:23.914 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:23.915 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.915 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.916 DEBUG:boto:Method: GET 2022-09-03T07:28:23.917 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.918 DEBUG:boto:Data: 2022-09-03T07:28:23.918 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.919 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.919 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.920 DEBUG:boto:Params: {} 2022-09-03T07:28:23.920 DEBUG:boto:Token: None 2022-09-03T07:28:23.921 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kDFVf44kVuMXkMPwDhYM1O0owlE= 2022-09-03T07:28:23.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kDFVf44kVuMXkMPwDhYM1O0owlE=', 'Content-Length': '0'} 2022-09-03T07:28:23.927 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044ebe7cda3d07731-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.929 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:23.929 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:23.930 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.931 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.931 DEBUG:boto:Method: GET 2022-09-03T07:28:23.932 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.934 DEBUG:boto:Data: 2022-09-03T07:28:23.935 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.935 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.936 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.937 DEBUG:boto:Params: {} 2022-09-03T07:28:23.939 DEBUG:boto:Token: None 2022-09-03T07:28:23.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kDFVf44kVuMXkMPwDhYM1O0owlE= 2022-09-03T07:28:23.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kDFVf44kVuMXkMPwDhYM1O0owlE=', 'Content-Length': '0'} 2022-09-03T07:28:23.944 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003731b8c065eaa50e-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.945 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:23.946 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:23.946 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.947 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.947 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.948 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.949 DEBUG:boto:Data: 2022-09-03T07:28:23.949 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.950 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.950 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.951 DEBUG:boto:Params: {} 2022-09-03T07:28:23.951 DEBUG:boto:Token: None 2022-09-03T07:28:23.952 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s05uqvMFism+OurpXz1sB4xkUOQ= 2022-09-03T07:28:23.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s05uqvMFism+OurpXz1sB4xkUOQ=', 'Content-Length': '0'} 2022-09-03T07:28:23.957 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089f276ba72da1a73-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.959 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:23.960 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:23.961 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.962 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.962 DEBUG:boto:Method: HEAD 2022-09-03T07:28:23.963 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.964 DEBUG:boto:Data: 2022-09-03T07:28:23.965 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:23.966 DEBUG:boto:Port: 8000 2022-09-03T07:28:23.966 DEBUG:boto:Params: {} 2022-09-03T07:28:23.967 DEBUG:boto:Token: None 2022-09-03T07:28:23.967 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:s05uqvMFism+OurpXz1sB4xkUOQ= 2022-09-03T07:28:23.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:s05uqvMFism+OurpXz1sB4xkUOQ=', 'Content-Length': '0'} 2022-09-03T07:28:23.970 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c558d2c57f8b5881-0063130217-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:23.971 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:28:23.972 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:23.972 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:23.973 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.973 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.974 DEBUG:boto:Method: GET 2022-09-03T07:28:23.974 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.975 DEBUG:boto:Data: 2022-09-03T07:28:23.976 DEBUG:boto:Headers: {} 2022-09-03T07:28:23.976 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:23.977 DEBUG:boto:Port: 8001 2022-09-03T07:28:23.977 DEBUG:boto:Params: {} 2022-09-03T07:28:23.978 DEBUG:boto:Token: None 2022-09-03T07:28:23.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:23 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:23.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kDFVf44kVuMXkMPwDhYM1O0owlE= 2022-09-03T07:28:23.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:23 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kDFVf44kVuMXkMPwDhYM1O0owlE=', 'Content-Length': '0'} 2022-09-03T07:28:23.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091d180aadb69138b-0063130217-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:23 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.026 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:24.027 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:24.028 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:24.028 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:28:24.029 DEBUG:boto:Method: GET 2022-09-03T07:28:24.029 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:28:24.030 DEBUG:boto:Data: 2022-09-03T07:28:24.030 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.031 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.031 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.032 DEBUG:boto:Params: {} 2022-09-03T07:28:24.033 DEBUG:boto:Token: None 2022-09-03T07:28:24.035 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:28:24.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QXDJ8Gyz5KYvm0i3TCh9wzs4shA= 2022-09-03T07:28:24.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QXDJ8Gyz5KYvm0i3TCh9wzs4shA=', 'Content-Length': '0'} 2022-09-03T07:28:24.039 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008773d269ec29671f-0063130218-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.040 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.040 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.041 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:24.042 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:24.042 DEBUG:boto:Method: GET 2022-09-03T07:28:24.043 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:24.043 DEBUG:boto:Data: 2022-09-03T07:28:24.044 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.045 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.046 DEBUG:boto:Params: {} 2022-09-03T07:28:24.046 DEBUG:boto:Token: None 2022-09-03T07:28:24.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78= 2022-09-03T07:28:24.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78=', 'Content-Length': '0'} 2022-09-03T07:28:24.052 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004e959b054593b5a5-0063130218-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.053 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:24.054 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.054 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.055 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:24.055 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:24.056 DEBUG:boto:Method: GET 2022-09-03T07:28:24.056 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:24.057 DEBUG:boto:Data: 2022-09-03T07:28:24.057 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.058 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.059 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.059 DEBUG:boto:Params: {} 2022-09-03T07:28:24.060 DEBUG:boto:Token: None 2022-09-03T07:28:24.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78= 2022-09-03T07:28:24.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78=', 'Content-Length': '0'} 2022-09-03T07:28:24.064 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000edc1732d37df1804-0063130218-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.065 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:24.066 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:24.067 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:24.067 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.068 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:24.068 DEBUG:boto:Data: 2022-09-03T07:28:24.069 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.070 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.070 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.071 DEBUG:boto:Params: {} 2022-09-03T07:28:24.071 DEBUG:boto:Token: None 2022-09-03T07:28:24.072 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj93 2022-09-03T07:28:24.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R9d+/uzemWnK2g5Mvk982Y3H0GI= 2022-09-03T07:28:24.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R9d+/uzemWnK2g5Mvk982Y3H0GI=', 'Content-Length': '0'} 2022-09-03T07:28:24.075 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f2912fc3e96ce59a-0063130218-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.076 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:24.077 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:24.077 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.078 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:24.078 DEBUG:boto:Data: 2022-09-03T07:28:24.079 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.079 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.080 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.080 DEBUG:boto:Params: {} 2022-09-03T07:28:24.081 DEBUG:boto:Token: None 2022-09-03T07:28:24.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj93 2022-09-03T07:28:24.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R9d+/uzemWnK2g5Mvk982Y3H0GI= 2022-09-03T07:28:24.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R9d+/uzemWnK2g5Mvk982Y3H0GI=', 'Content-Length': '0'} 2022-09-03T07:28:24.085 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec848f5333acabbb-0063130218-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.085 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:28:24.086 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:24.087 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:24.087 DEBUG:boto:Method: GET 2022-09-03T07:28:24.088 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:24.088 DEBUG:boto:Data: 2022-09-03T07:28:24.089 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.089 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.090 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.090 DEBUG:boto:Params: {} 2022-09-03T07:28:24.091 DEBUG:boto:Token: None 2022-09-03T07:28:24.092 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj93 2022-09-03T07:28:24.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t0Ei6bDbbj25P8iqv0T95qC12uM= 2022-09-03T07:28:24.093 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t0Ei6bDbbj25P8iqv0T95qC12uM=', 'Content-Length': '0'} 2022-09-03T07:28:24.095 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065320fb8de85a4df-0063130218-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.096 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:28:24.097 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:28:24.097 DEBUG:boto:Method: GET 2022-09-03T07:28:24.098 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:28:24.099 DEBUG:boto:Data: 2022-09-03T07:28:24.100 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.100 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.101 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.101 DEBUG:boto:Params: {} 2022-09-03T07:28:24.102 DEBUG:boto:Token: None 2022-09-03T07:28:24.103 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj93 2022-09-03T07:28:24.103 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t0Ei6bDbbj25P8iqv0T95qC12uM= 2022-09-03T07:28:24.104 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t0Ei6bDbbj25P8iqv0T95qC12uM=', 'Content-Length': '0'} 2022-09-03T07:28:24.106 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080df641610cc8017-0063130218-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.107 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.107 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.108 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:24.108 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:24.109 DEBUG:boto:Method: GET 2022-09-03T07:28:24.109 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:24.110 DEBUG:boto:Data: 2022-09-03T07:28:24.110 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.111 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.112 DEBUG:boto:Params: {} 2022-09-03T07:28:24.113 DEBUG:boto:Token: None 2022-09-03T07:28:24.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78= 2022-09-03T07:28:24.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78=', 'Content-Length': '0'} 2022-09-03T07:28:24.117 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c294f0bcf641a5b6-0063130218-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.118 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:24.119 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.120 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.120 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:24.121 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:28:24.121 DEBUG:boto:Method: GET 2022-09-03T07:28:24.122 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:28:24.123 DEBUG:boto:Data: 2022-09-03T07:28:24.123 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.124 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.124 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.125 DEBUG:boto:Params: {} 2022-09-03T07:28:24.125 DEBUG:boto:Token: None 2022-09-03T07:28:24.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78= 2022-09-03T07:28:24.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78=', 'Content-Length': '0'} 2022-09-03T07:28:24.130 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000080d493cc05166ca8-0063130218-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.174 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:24.176 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:28:24.177 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.178 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.179 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.180 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.181 DEBUG:boto:Method: GET 2022-09-03T07:28:24.182 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.183 DEBUG:boto:Data: 2022-09-03T07:28:24.184 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.186 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.186 DEBUG:boto:Params: {} 2022-09-03T07:28:24.187 DEBUG:boto:Token: None 2022-09-03T07:28:24.187 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.188 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L69iNOc821sJfPTKBCcYpAKhz7I= 2022-09-03T07:28:24.189 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L69iNOc821sJfPTKBCcYpAKhz7I=', 'Content-Length': '0'} 2022-09-03T07:28:24.191 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d2b37f82c2264d7b-0063130218-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.193 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.193 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.194 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.195 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.196 DEBUG:boto:Method: GET 2022-09-03T07:28:24.197 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.197 DEBUG:boto:Data: 2022-09-03T07:28:24.198 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.198 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.199 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.200 DEBUG:boto:Params: {} 2022-09-03T07:28:24.201 DEBUG:boto:Token: None 2022-09-03T07:28:24.201 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.202 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L69iNOc821sJfPTKBCcYpAKhz7I= 2022-09-03T07:28:24.203 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L69iNOc821sJfPTKBCcYpAKhz7I=', 'Content-Length': '0'} 2022-09-03T07:28:24.205 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006fd5920d9d7adb5b-0063130218-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.207 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.207 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.208 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.208 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.209 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.209 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.210 DEBUG:boto:Data: 2022-09-03T07:28:24.211 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.212 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.213 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.214 DEBUG:boto:Params: {} 2022-09-03T07:28:24.215 DEBUG:boto:Token: None 2022-09-03T07:28:24.216 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ALIpsQ/1FSRpO4zQfaOIyJVZm+s= 2022-09-03T07:28:24.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ALIpsQ/1FSRpO4zQfaOIyJVZm+s=', 'Content-Length': '0'} 2022-09-03T07:28:24.221 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c0d98e6d12d28ab-0063130218-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.223 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.224 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.225 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.226 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.227 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.227 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.228 DEBUG:boto:Data: 2022-09-03T07:28:24.229 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.231 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.232 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.233 DEBUG:boto:Params: {} 2022-09-03T07:28:24.234 DEBUG:boto:Token: None 2022-09-03T07:28:24.235 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ALIpsQ/1FSRpO4zQfaOIyJVZm+s= 2022-09-03T07:28:24.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ALIpsQ/1FSRpO4zQfaOIyJVZm+s=', 'Content-Length': '0'} 2022-09-03T07:28:24.238 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e01c4f7544f5f89-0063130218-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.239 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:28:24.240 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.240 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.241 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.241 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.242 DEBUG:boto:Method: GET 2022-09-03T07:28:24.243 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.243 DEBUG:boto:Data: 2022-09-03T07:28:24.245 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.245 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.246 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.247 DEBUG:boto:Params: {} 2022-09-03T07:28:24.248 DEBUG:boto:Token: None 2022-09-03T07:28:24.249 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.250 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L69iNOc821sJfPTKBCcYpAKhz7I= 2022-09-03T07:28:24.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L69iNOc821sJfPTKBCcYpAKhz7I=', 'Content-Length': '0'} 2022-09-03T07:28:24.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d3f0b6c1713eda4-0063130218-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.255 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.256 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.257 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.257 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.258 DEBUG:boto:Method: GET 2022-09-03T07:28:24.258 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.259 DEBUG:boto:Data: 2022-09-03T07:28:24.259 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.260 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.260 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.261 DEBUG:boto:Params: {} 2022-09-03T07:28:24.262 DEBUG:boto:Token: None 2022-09-03T07:28:24.262 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:28:24.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:L69iNOc821sJfPTKBCcYpAKhz7I= 2022-09-03T07:28:24.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:L69iNOc821sJfPTKBCcYpAKhz7I=', 'Content-Length': '0'} 2022-09-03T07:28:24.265 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d476c0322ec7a33-0063130218-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.266 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.267 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.267 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:24.268 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:24.269 DEBUG:boto:Method: GET 2022-09-03T07:28:24.269 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:24.270 DEBUG:boto:Data: 2022-09-03T07:28:24.270 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.272 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.273 DEBUG:boto:Params: {} 2022-09-03T07:28:24.273 DEBUG:boto:Token: None 2022-09-03T07:28:24.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78= 2022-09-03T07:28:24.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78=', 'Content-Length': '0'} 2022-09-03T07:28:24.280 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091c79b8348499f96-0063130218-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.322 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:24.323 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.324 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.324 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:24.325 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:24.325 DEBUG:boto:Method: GET 2022-09-03T07:28:24.326 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:24.327 DEBUG:boto:Data: 2022-09-03T07:28:24.327 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.328 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.328 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.329 DEBUG:boto:Params: {} 2022-09-03T07:28:24.329 DEBUG:boto:Token: None 2022-09-03T07:28:24.330 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.331 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78= 2022-09-03T07:28:24.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78=', 'Content-Length': '0'} 2022-09-03T07:28:24.334 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000ff3b8bc53fc2e00-0063130218-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.334 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:24.335 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.336 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.336 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.337 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:24.337 DEBUG:boto:Data: 2022-09-03T07:28:24.338 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.339 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.339 DEBUG:boto:Params: {} 2022-09-03T07:28:24.340 DEBUG:boto:Token: None 2022-09-03T07:28:24.341 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95 2022-09-03T07:28:24.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CGdjTnfKZn7NqhSm5Jeo5CDnMt8= 2022-09-03T07:28:24.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CGdjTnfKZn7NqhSm5Jeo5CDnMt8=', 'Content-Length': '0'} 2022-09-03T07:28:24.344 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c731701c60ac18e6-0063130218-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.345 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.346 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.346 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.347 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:24.347 DEBUG:boto:Data: 2022-09-03T07:28:24.348 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.348 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.349 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.349 DEBUG:boto:Params: {} 2022-09-03T07:28:24.350 DEBUG:boto:Token: None 2022-09-03T07:28:24.351 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95 2022-09-03T07:28:24.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CGdjTnfKZn7NqhSm5Jeo5CDnMt8= 2022-09-03T07:28:24.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CGdjTnfKZn7NqhSm5Jeo5CDnMt8=', 'Content-Length': '0'} 2022-09-03T07:28:24.356 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004713b61321dfa9fe-0063130218-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.357 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:28:24.358 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.358 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.359 DEBUG:boto:Method: GET 2022-09-03T07:28:24.359 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:24.360 DEBUG:boto:Data: 2022-09-03T07:28:24.361 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.361 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.362 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.362 DEBUG:boto:Params: {} 2022-09-03T07:28:24.363 DEBUG:boto:Token: None 2022-09-03T07:28:24.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95 2022-09-03T07:28:24.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zssYNxOG/3bWklCZs/a5yEgNDzo= 2022-09-03T07:28:24.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zssYNxOG/3bWklCZs/a5yEgNDzo=', 'Content-Length': '0'} 2022-09-03T07:28:24.372 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a55b3abd7b84690-0063130218-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.374 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:28:24.374 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:28:24.375 DEBUG:boto:Method: GET 2022-09-03T07:28:24.376 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:28:24.376 DEBUG:boto:Data: 2022-09-03T07:28:24.377 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.377 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.378 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.379 DEBUG:boto:Params: {} 2022-09-03T07:28:24.379 DEBUG:boto:Token: None 2022-09-03T07:28:24.380 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj95 2022-09-03T07:28:24.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zssYNxOG/3bWklCZs/a5yEgNDzo= 2022-09-03T07:28:24.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zssYNxOG/3bWklCZs/a5yEgNDzo=', 'Content-Length': '0'} 2022-09-03T07:28:24.385 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f87d749986f211e4-0063130218-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.430 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.432 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.435 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:24.437 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:24.438 DEBUG:boto:Method: GET 2022-09-03T07:28:24.439 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:24.440 DEBUG:boto:Data: 2022-09-03T07:28:24.441 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:28:24.443 DEBUG:boto:Port: 8001 2022-09-03T07:28:24.444 DEBUG:boto:Params: {} 2022-09-03T07:28:24.446 DEBUG:boto:Token: None 2022-09-03T07:28:24.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78= 2022-09-03T07:28:24.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78=', 'Content-Length': '0'} 2022-09-03T07:28:24.454 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000352a61b70e419dba-0063130218-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.455 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:24.456 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.457 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.458 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:24.458 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:28:24.459 DEBUG:boto:Method: GET 2022-09-03T07:28:24.459 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:28:24.460 DEBUG:boto:Data: 2022-09-03T07:28:24.460 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.461 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:28:24.461 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.463 DEBUG:boto:Params: {} 2022-09-03T07:28:24.463 DEBUG:boto:Token: None 2022-09-03T07:28:24.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78= 2022-09-03T07:28:24.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rzWA6Bi2f5QH/kyUxlHkQOEet78=', 'Content-Length': '0'} 2022-09-03T07:28:24.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b147a8d0ea1c8272-0063130218-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.469 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:28:24.470 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:28:24.471 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '37', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:28:24.471 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--num-shards', '37', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:28:24.473 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --num-shards 37 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:28:24.537 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:28:24.538 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:28:24.609 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:28:24.609 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-75 2022-09-03T07:28:24.623 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 30 2022-09-03T07:28:24.707 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.708 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.708 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.709 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:28:24.710 DEBUG:boto:Data: 2022-09-03T07:28:24.711 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.713 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.715 DEBUG:boto:Params: {} 2022-09-03T07:28:24.716 DEBUG:boto:Token: None 2022-09-03T07:28:24.717 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.717 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q= 2022-09-03T07:28:24.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q=', 'Content-Length': '0'} 2022-09-03T07:28:24.722 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '30'), ('X-RGW-Bytes-Used', '90'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000163ffe9d2dca8a80-0063130218-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.723 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:28:24.724 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:28:24.725 DEBUG:boto:Method: PUT 2022-09-03T07:28:24.725 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:28:24.726 DEBUG:boto:Data: 2022-09-03T07:28:24.726 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:28:24.727 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.727 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.728 DEBUG:boto:Params: {} 2022-09-03T07:28:24.728 DEBUG:boto:Token: None 2022-09-03T07:28:24.729 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj37 2022-09-03T07:28:24.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cIAsP3EsTG1kUy83g0l75lPaX18= 2022-09-03T07:28:24.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cIAsP3EsTG1kUy83g0l75lPaX18='} 2022-09-03T07:28:24.738 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000001501396a32bc6f7b-0063130218-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.739 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.740 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.741 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.742 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:28:24.743 DEBUG:boto:Data: 2022-09-03T07:28:24.744 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.744 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.745 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.745 DEBUG:boto:Params: {} 2022-09-03T07:28:24.746 DEBUG:boto:Token: None 2022-09-03T07:28:24.746 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q= 2022-09-03T07:28:24.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q=', 'Content-Length': '0'} 2022-09-03T07:28:24.752 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '31'), ('X-RGW-Bytes-Used', '93'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000600f9975c00b273d-0063130218-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.753 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:28:24.753 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:28:24.754 DEBUG:boto:Method: PUT 2022-09-03T07:28:24.754 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:28:24.755 DEBUG:boto:Data: 2022-09-03T07:28:24.756 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:28:24.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.758 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.759 DEBUG:boto:Params: {} 2022-09-03T07:28:24.760 DEBUG:boto:Token: None 2022-09-03T07:28:24.761 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj74 2022-09-03T07:28:24.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U6aMUJo5eFAQU6woXSev//TIQmk= 2022-09-03T07:28:24.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U6aMUJo5eFAQU6woXSev//TIQmk='} 2022-09-03T07:28:24.806 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000399e58db2f3ec059-0063130218-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.809 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.810 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.810 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.811 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:28:24.812 DEBUG:boto:Data: 2022-09-03T07:28:24.812 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.813 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.814 DEBUG:boto:Params: {} 2022-09-03T07:28:24.815 DEBUG:boto:Token: None 2022-09-03T07:28:24.816 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q= 2022-09-03T07:28:24.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q=', 'Content-Length': '0'} 2022-09-03T07:28:24.823 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '32'), ('X-RGW-Bytes-Used', '96'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000044a2bc3c9d5eabfc-0063130218-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.824 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:28:24.824 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:28:24.825 DEBUG:boto:Method: PUT 2022-09-03T07:28:24.826 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:28:24.827 DEBUG:boto:Data: 2022-09-03T07:28:24.828 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:28:24.828 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.829 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.829 DEBUG:boto:Params: {} 2022-09-03T07:28:24.830 DEBUG:boto:Token: None 2022-09-03T07:28:24.831 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj111 2022-09-03T07:28:24.831 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UFzfwLA71D/aq65yP8kMX7CuzpU= 2022-09-03T07:28:24.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UFzfwLA71D/aq65yP8kMX7CuzpU='} 2022-09-03T07:28:24.842 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000002eeebad2ba9c6bc8-0063130218-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.843 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.843 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.844 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.844 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:28:24.845 DEBUG:boto:Data: 2022-09-03T07:28:24.846 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.847 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.848 DEBUG:boto:Params: {} 2022-09-03T07:28:24.849 DEBUG:boto:Token: None 2022-09-03T07:28:24.850 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q= 2022-09-03T07:28:24.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q=', 'Content-Length': '0'} 2022-09-03T07:28:24.856 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '33'), ('X-RGW-Bytes-Used', '99'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004e1470c0879c6fd4-0063130218-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.857 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:28:24.857 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:28:24.858 DEBUG:boto:Method: PUT 2022-09-03T07:28:24.859 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:28:24.860 DEBUG:boto:Data: 2022-09-03T07:28:24.861 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:28:24.862 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.863 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.863 DEBUG:boto:Params: {} 2022-09-03T07:28:24.864 DEBUG:boto:Token: None 2022-09-03T07:28:24.865 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj148 2022-09-03T07:28:24.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TamI/UOPToUrYze6xHNSUNviJq0= 2022-09-03T07:28:24.866 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TamI/UOPToUrYze6xHNSUNviJq0='} 2022-09-03T07:28:24.914 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000008d4d73cefaa8c351-0063130218-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.916 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:28:24.917 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:28:24.917 DEBUG:boto:Method: HEAD 2022-09-03T07:28:24.918 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:28:24.918 DEBUG:boto:Data: 2022-09-03T07:28:24.919 DEBUG:boto:Headers: {} 2022-09-03T07:28:24.919 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.920 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.920 DEBUG:boto:Params: {} 2022-09-03T07:28:24.921 DEBUG:boto:Token: None 2022-09-03T07:28:24.921 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/ 2022-09-03T07:28:24.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q= 2022-09-03T07:28:24.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:txClDzYJqf6I7B3ABvyOmbMzf7Q=', 'Content-Length': '0'} 2022-09-03T07:28:24.926 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '34'), ('X-RGW-Bytes-Used', '102'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cb16da4566500965-0063130218-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.929 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:28:24.930 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:28:24.931 DEBUG:boto:Method: PUT 2022-09-03T07:28:24.931 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:28:24.932 DEBUG:boto:Data: 2022-09-03T07:28:24.933 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:28:24.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:28:24.935 DEBUG:boto:Port: 8000 2022-09-03T07:28:24.936 DEBUG:boto:Params: {} 2022-09-03T07:28:24.937 DEBUG:boto:Token: None 2022-09-03T07:28:24.938 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:28:24 GMT /avysmw-75/obj185 2022-09-03T07:28:24.938 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tpALix2dhropKh5Mm0C7DXsCJEQ= 2022-09-03T07:28:24.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:28:24 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tpALix2dhropKh5Mm0C7DXsCJEQ='} 2022-09-03T07:28:24.950 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000090ab26f16669363e-0063130218-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:28:24.951 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-75 2022-09-03T07:28:24.952 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:28:24.952 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:28:24.953 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:28:25.021 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:28:25.022 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:28:25.084 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:28:25.083+0000 7fa93137e600 1 waiting to reach latest gen 6, on 5.. 2022-09-03T07:28:30.085 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:28:30.083+0000 7fa93137e600 1 waiting to reach latest gen 6, on 5.. 2022-09-03T07:28:35.085 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:28:35.083+0000 7fa93137e600 1 waiting to reach latest gen 6, on 5.. 2022-09-03T07:28:40.089 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:28:40.086+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:28:40.090 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:28:40.090 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:28:45.090 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:28:45.089+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:28:45.091 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:28:45.091 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:28:50.093 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:28:50.091+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:28:50.094 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:28:50.094 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:28:55.095 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:28:55.093+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:28:55.096 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:28:55.096 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:00.097 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:00.095+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:00.097 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:00.097 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:05.099 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:05.097+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:05.100 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:05.100 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:10.101 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:10.100+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:10.102 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:10.102 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:15.106 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:15.103+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:15.106 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:15.107 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:20.107 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:20.105+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:20.107 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:20.107 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:25.109 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:25.107+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:25.110 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:25.110 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:30.112 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:30.110+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:30.113 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:30.113 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:35.114 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:35.112+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:35.114 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:35.115 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:40.117 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:40.116+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:40.118 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:40.119 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:45.120 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:45.118+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:45.120 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:45.120 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:50.124 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:50.121+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:50.124 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:50.125 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:29:55.125 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:29:55.123+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:29:55.126 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 2022-09-03T07:29:55.126 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:00.131 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:00.127+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:30:00.131 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , , , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:00.132 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:05.131 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:05.129+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:30:05.131 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , , , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:05.131 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:10.133 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:10.131+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:30:10.133 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , , , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:10.134 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:15.136 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:15.134+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:30:15.136 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , , , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:15.136 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:20.139 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:20.137+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:30:20.139 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , , , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:20.140 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:25.141 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:25.139+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:30:25.141 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , , , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:25.142 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:30.144 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:30.141+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:30:30.145 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , , , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:30.145 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:35.146 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:35.144+0000 7fa93137e600 1 waiting for incremental sync to catch up: 2022-09-03T07:30:35.147 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , , , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:35.147 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:40.149 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:40.146+0000 7fa93137e600 1 bucket sync caught up with source: 2022-09-03T07:30:40.150 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:40.150 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:40.150 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:40.146+0000 7fa93137e600 0 bucket checkpoint complete 2022-09-03T07:30:40.169 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-75 2022-09-03T07:30:40.172 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:40.173 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:40.175 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:30:40.245 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:30:40.245 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:30:40.294 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:30:40.292+0000 7f135bfec600 1 bucket sync caught up with source: 2022-09-03T07:30:40.295 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:40.296 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:40.296 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:30:40.292+0000 7f135bfec600 0 bucket checkpoint complete 2022-09-03T07:30:40.310 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-75 2022-09-03T07:30:40.311 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:40.311 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:40.312 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:30:40.478 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:30:40.479 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:30:40.556 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:40.555+0000 7feb91531600 1 bucket sync caught up with source: 2022-09-03T07:30:40.557 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:40.557 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , 00000000002.78.5, , , , , , 00000000002.79.5, , , , , 00000000002.76.5, , , , , , , , , , , , , , , 00000000001.55.5, , , , 00000000001.50.5] 2022-09-03T07:30:40.557 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:40.555+0000 7feb91531600 0 bucket checkpoint complete 2022-09-03T07:30:40.578 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-75 2022-09-03T07:30:40.579 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:40.579 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:40.580 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:30:40.641 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:30:40.642 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:30:40.697 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:30:40.695+0000 7f75a83b5600 1 bucket sync caught up with source: 2022-09-03T07:30:40.697 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000005.359.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000104.589.5, 00000000007.314.5, 00000000006.232.5] 2022-09-03T07:30:40.698 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000005.359.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000104.589.5, 00000000007.314.5, 00000000006.232.5] 2022-09-03T07:30:40.698 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:30:40.695+0000 7f75a83b5600 0 bucket checkpoint complete 2022-09-03T07:30:40.720 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-75 2022-09-03T07:30:40.721 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:40.722 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:40.723 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:30:40.781 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:30:40.782 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:30:41.840 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:30:41.837+0000 7f2241306600 1 bucket sync caught up with source: 2022-09-03T07:30:41.841 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000005.359.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000104.589.5, 00000000007.314.5, 00000000006.232.5] 2022-09-03T07:30:41.842 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.127.5, 00000000002.339.5, 00000000002.385.5, 00000000005.359.5, 00000000004.264.5, 00000000001.350.5, 00000000001.384.5, 00000000104.589.5, 00000000007.314.5, 00000000006.232.5] 2022-09-03T07:30:41.842 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:30:41.837+0000 7f2241306600 0 bucket checkpoint complete 2022-09-03T07:30:41.857 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-75 2022-09-03T07:30:41.857 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-75 2022-09-03T07:30:41.858 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:41.859 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:41.859 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:30:41.921 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:30:41.923 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:30:41.965 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:30:41.963+0000 7f9f324c7600 1 bucket sync caught up with source: 2022-09-03T07:30:41.965 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000005.213.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000004.300.5, 00000000007.323.5, 00000000006.150.5] 2022-09-03T07:30:41.966 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000005.213.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000004.300.5, 00000000007.323.5, 00000000006.150.5] 2022-09-03T07:30:41.966 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:30:41.963+0000 7f9f324c7600 0 bucket checkpoint complete 2022-09-03T07:30:41.983 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-75 2022-09-03T07:30:41.984 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:41.985 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-75', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:30:41.985 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-75 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:30:42.051 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:30:42.052 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:30:42.098 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:42.097+0000 7f5d62177600 1 bucket sync caught up with source: 2022-09-03T07:30:42.099 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000005.213.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000004.300.5, 00000000007.323.5, 00000000006.150.5] 2022-09-03T07:30:42.099 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, 00000000003.125.5, 00000000002.195.5, 00000000002.194.5, 00000000005.213.5, 00000000004.151.5, 00000000001.366.5, 00000000001.221.5, 00000000004.300.5, 00000000007.323.5, 00000000006.150.5] 2022-09-03T07:30:42.100 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:30:42.097+0000 7f5d62177600 0 bucket checkpoint complete 2022-09-03T07:30:42.117 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-75 2022-09-03T07:30:42.119 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:30:42.120 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.121 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.122 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.123 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:30:42.124 DEBUG:boto:Data: 2022-09-03T07:30:42.125 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.126 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.127 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.128 DEBUG:boto:Params: {} 2022-09-03T07:30:42.129 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:30:42.130 DEBUG:boto:Token: None 2022-09-03T07:30:42.131 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OMTqpOnLYvWNmkMTCFOyxYXjmZg= 2022-09-03T07:30:42.133 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OMTqpOnLYvWNmkMTCFOyxYXjmZg=', 'Content-Length': '0'} 2022-09-03T07:30:42.140 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '35'), ('X-RGW-Bytes-Used', '105'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000007551f42559a183ec-00631302a2-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.142 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.143 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.144 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.147 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:30:42.148 DEBUG:boto:Data: 2022-09-03T07:30:42.150 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.151 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.152 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.153 DEBUG:boto:Params: {} 2022-09-03T07:30:42.154 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:30:42.155 DEBUG:boto:Token: None 2022-09-03T07:30:42.155 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OMTqpOnLYvWNmkMTCFOyxYXjmZg= 2022-09-03T07:30:42.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OMTqpOnLYvWNmkMTCFOyxYXjmZg=', 'Content-Length': '0'} 2022-09-03T07:30:42.161 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '35'), ('X-RGW-Bytes-Used', '105'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a0ddce1a009793b6-00631302a2-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.162 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:30:42.163 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.164 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.164 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.165 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.165 DEBUG:boto:Method: GET 2022-09-03T07:30:42.166 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.167 DEBUG:boto:Data: 2022-09-03T07:30:42.167 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.168 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.168 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.169 DEBUG:boto:Params: {} 2022-09-03T07:30:42.169 DEBUG:boto:Token: None 2022-09-03T07:30:42.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/?versions 2022-09-03T07:30:42.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CrL4TB1z8O8QpLYVduEhfu0yvw= 2022-09-03T07:30:42.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CrL4TB1z8O8QpLYVduEhfu0yvw=', 'Content-Length': '0'} 2022-09-03T07:30:42.174 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf01c4f03e4d2232-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.175 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.183 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:30:42.183 DEBUG:tasks.rgw_multi.zone_rados:o=obj111 2022-09-03T07:30:42.184 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:30:42.184 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:30:42.185 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:30:42.185 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:30:42.186 DEBUG:tasks.rgw_multi.zone_rados:o=obj148 2022-09-03T07:30:42.186 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:30:42.187 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:30:42.187 DEBUG:tasks.rgw_multi.zone_rados:o=obj185 2022-09-03T07:30:42.188 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:30:42.188 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:30:42.189 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:30:42.189 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:30:42.190 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:30:42.190 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:30:42.191 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:30:42.191 DEBUG:tasks.rgw_multi.zone_rados:o=obj37 2022-09-03T07:30:42.192 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:30:42.192 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:30:42.192 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:30:42.193 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:30:42.193 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:30:42.194 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:30:42.194 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:30:42.195 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:30:42.195 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:30:42.196 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:30:42.196 DEBUG:tasks.rgw_multi.zone_rados:o=obj74 2022-09-03T07:30:42.197 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:30:42.197 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:30:42.198 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:30:42.198 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:30:42.198 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:30:42.199 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:30:42.200 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:30:42.200 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.201 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.201 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.202 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.203 DEBUG:boto:Method: GET 2022-09-03T07:30:42.203 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.204 DEBUG:boto:Data: 2022-09-03T07:30:42.204 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.205 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.205 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.206 DEBUG:boto:Params: {} 2022-09-03T07:30:42.207 DEBUG:boto:Token: None 2022-09-03T07:30:42.207 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/?versions 2022-09-03T07:30:42.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CrL4TB1z8O8QpLYVduEhfu0yvw= 2022-09-03T07:30:42.211 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CrL4TB1z8O8QpLYVduEhfu0yvw=', 'Content-Length': '0'} 2022-09-03T07:30:42.222 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000018fa8cf2ee8d7283-00631302a2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.224 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.228 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:30:42.229 DEBUG:tasks.rgw_multi.zone_rados:o=obj111 2022-09-03T07:30:42.231 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:30:42.232 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:30:42.233 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:30:42.234 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:30:42.235 DEBUG:tasks.rgw_multi.zone_rados:o=obj148 2022-09-03T07:30:42.236 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:30:42.237 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:30:42.238 DEBUG:tasks.rgw_multi.zone_rados:o=obj185 2022-09-03T07:30:42.238 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:30:42.239 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:30:42.240 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:30:42.243 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:30:42.244 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:30:42.245 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:30:42.246 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:30:42.246 DEBUG:tasks.rgw_multi.zone_rados:o=obj37 2022-09-03T07:30:42.247 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:30:42.248 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:30:42.248 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:30:42.252 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:30:42.253 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:30:42.253 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:30:42.254 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:30:42.254 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:30:42.255 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:30:42.255 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:30:42.256 DEBUG:tasks.rgw_multi.zone_rados:o=obj74 2022-09-03T07:30:42.256 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:30:42.257 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:30:42.257 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:30:42.258 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:30:42.258 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:30:42.259 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:30:42.259 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.260 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.261 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.261 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.262 DEBUG:boto:Method: GET 2022-09-03T07:30:42.263 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.264 DEBUG:boto:Data: 2022-09-03T07:30:42.264 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.265 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.266 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.267 DEBUG:boto:Params: {} 2022-09-03T07:30:42.267 DEBUG:boto:Token: None 2022-09-03T07:30:42.268 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/?versions 2022-09-03T07:30:42.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CrL4TB1z8O8QpLYVduEhfu0yvw= 2022-09-03T07:30:42.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CrL4TB1z8O8QpLYVduEhfu0yvw=', 'Content-Length': '0'} 2022-09-03T07:30:42.272 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ceef2350303589df-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.273 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.277 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.278 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.278 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.279 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.279 DEBUG:boto:Method: GET 2022-09-03T07:30:42.280 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:30:42.281 DEBUG:boto:Data: 2022-09-03T07:30:42.281 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.282 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.283 DEBUG:boto:Params: {} 2022-09-03T07:30:42.284 DEBUG:boto:Token: None 2022-09-03T07:30:42.284 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/?versions 2022-09-03T07:30:42.285 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6CrL4TB1z8O8QpLYVduEhfu0yvw= 2022-09-03T07:30:42.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6CrL4TB1z8O8QpLYVduEhfu0yvw=', 'Content-Length': '0'} 2022-09-03T07:30:42.290 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000422e2c9c422e0392-00631302a2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.291 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.294 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:30:42.296 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.297 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.297 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.298 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.299 DEBUG:boto:Method: GET 2022-09-03T07:30:42.299 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.300 DEBUG:boto:Data: 2022-09-03T07:30:42.300 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.301 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.302 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.302 DEBUG:boto:Params: {} 2022-09-03T07:30:42.303 DEBUG:boto:Token: None 2022-09-03T07:30:42.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xl+82w+1UlDtnb8Qx6IDXJBo4sk= 2022-09-03T07:30:42.305 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xl+82w+1UlDtnb8Qx6IDXJBo4sk=', 'Content-Length': '0'} 2022-09-03T07:30:42.308 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a8ad442b6c0013e-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.309 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.310 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.310 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.311 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.312 DEBUG:boto:Method: GET 2022-09-03T07:30:42.312 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.313 DEBUG:boto:Data: 2022-09-03T07:30:42.314 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.315 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.316 DEBUG:boto:Params: {} 2022-09-03T07:30:42.316 DEBUG:boto:Token: None 2022-09-03T07:30:42.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xl+82w+1UlDtnb8Qx6IDXJBo4sk= 2022-09-03T07:30:42.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xl+82w+1UlDtnb8Qx6IDXJBo4sk=', 'Content-Length': '0'} 2022-09-03T07:30:42.321 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eafe40248854fb3a-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.322 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.323 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.323 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.324 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.324 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.325 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.326 DEBUG:boto:Data: 2022-09-03T07:30:42.326 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.327 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.327 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.328 DEBUG:boto:Params: {} 2022-09-03T07:30:42.328 DEBUG:boto:Token: None 2022-09-03T07:30:42.329 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JD8i2HDVmFzoEuFdxYaIyoLY3Tk= 2022-09-03T07:30:42.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JD8i2HDVmFzoEuFdxYaIyoLY3Tk=', 'Content-Length': '0'} 2022-09-03T07:30:42.332 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db3eab0971a6be9f-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.333 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.335 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.336 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.337 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.338 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.339 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.340 DEBUG:boto:Data: 2022-09-03T07:30:42.340 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.341 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.343 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.343 DEBUG:boto:Params: {} 2022-09-03T07:30:42.345 DEBUG:boto:Token: None 2022-09-03T07:30:42.346 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JD8i2HDVmFzoEuFdxYaIyoLY3Tk= 2022-09-03T07:30:42.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JD8i2HDVmFzoEuFdxYaIyoLY3Tk=', 'Content-Length': '0'} 2022-09-03T07:30:42.351 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e324f559925687d-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.352 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:30:42.352 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.353 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.354 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.355 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.355 DEBUG:boto:Method: GET 2022-09-03T07:30:42.356 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.357 DEBUG:boto:Data: 2022-09-03T07:30:42.357 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.358 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.359 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.359 DEBUG:boto:Params: {} 2022-09-03T07:30:42.360 DEBUG:boto:Token: None 2022-09-03T07:30:42.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xl+82w+1UlDtnb8Qx6IDXJBo4sk= 2022-09-03T07:30:42.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xl+82w+1UlDtnb8Qx6IDXJBo4sk=', 'Content-Length': '0'} 2022-09-03T07:30:42.364 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a365b355462d0a5c-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.365 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.366 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.366 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.367 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.368 DEBUG:boto:Method: GET 2022-09-03T07:30:42.368 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.369 DEBUG:boto:Data: 2022-09-03T07:30:42.369 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.370 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.371 DEBUG:boto:Params: {} 2022-09-03T07:30:42.372 DEBUG:boto:Token: None 2022-09-03T07:30:42.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:42.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xl+82w+1UlDtnb8Qx6IDXJBo4sk= 2022-09-03T07:30:42.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xl+82w+1UlDtnb8Qx6IDXJBo4sk=', 'Content-Length': '0'} 2022-09-03T07:30:42.376 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001df63b5ca98e2044-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.377 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.377 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.378 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:42.378 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:42.379 DEBUG:boto:Method: GET 2022-09-03T07:30:42.379 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:30:42.380 DEBUG:boto:Data: 2022-09-03T07:30:42.381 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.381 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.382 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.382 DEBUG:boto:Params: {} 2022-09-03T07:30:42.383 DEBUG:boto:Token: None 2022-09-03T07:30:42.384 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.388 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bc8195eb4d2fa71f-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.388 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.390 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.390 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.391 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:42.391 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:42.392 DEBUG:boto:Method: GET 2022-09-03T07:30:42.392 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:30:42.393 DEBUG:boto:Data: 2022-09-03T07:30:42.394 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.394 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.395 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.395 DEBUG:boto:Params: {} 2022-09-03T07:30:42.396 DEBUG:boto:Token: None 2022-09-03T07:30:42.396 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.403 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf85d36bf7e2c692-00631302a2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.404 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.405 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.405 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.406 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.406 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:30:42.407 DEBUG:boto:Data: 2022-09-03T07:30:42.407 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.408 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.409 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.409 DEBUG:boto:Params: {} 2022-09-03T07:30:42.410 DEBUG:boto:Token: None 2022-09-03T07:30:42.411 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11 2022-09-03T07:30:42.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6qar+Koz5uO9PqCE1Pt3KK97264= 2022-09-03T07:30:42.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6qar+Koz5uO9PqCE1Pt3KK97264=', 'Content-Length': '0'} 2022-09-03T07:30:42.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005b426a025d73506d-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.416 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.416 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.417 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.418 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:30:42.418 DEBUG:boto:Data: 2022-09-03T07:30:42.419 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.420 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.420 DEBUG:boto:Params: {} 2022-09-03T07:30:42.421 DEBUG:boto:Token: None 2022-09-03T07:30:42.422 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11 2022-09-03T07:30:42.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6qar+Koz5uO9PqCE1Pt3KK97264= 2022-09-03T07:30:42.424 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6qar+Koz5uO9PqCE1Pt3KK97264=', 'Content-Length': '0'} 2022-09-03T07:30:42.428 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078bea3fc75076095-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.429 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:30:42.430 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.430 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.431 DEBUG:boto:Method: GET 2022-09-03T07:30:42.432 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:30:42.432 DEBUG:boto:Data: 2022-09-03T07:30:42.433 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.434 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.435 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.436 DEBUG:boto:Params: {} 2022-09-03T07:30:42.436 DEBUG:boto:Token: None 2022-09-03T07:30:42.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11 2022-09-03T07:30:42.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vY4tpIfW+3UF82/JThy9IGGdd5s= 2022-09-03T07:30:42.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vY4tpIfW+3UF82/JThy9IGGdd5s=', 'Content-Length': '0'} 2022-09-03T07:30:42.441 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a50f51a68d4bbfc-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.442 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:42.443 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:42.443 DEBUG:boto:Method: GET 2022-09-03T07:30:42.444 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:30:42.445 DEBUG:boto:Data: 2022-09-03T07:30:42.446 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.446 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.447 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.447 DEBUG:boto:Params: {} 2022-09-03T07:30:42.448 DEBUG:boto:Token: None 2022-09-03T07:30:42.449 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj11 2022-09-03T07:30:42.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vY4tpIfW+3UF82/JThy9IGGdd5s= 2022-09-03T07:30:42.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vY4tpIfW+3UF82/JThy9IGGdd5s=', 'Content-Length': '0'} 2022-09-03T07:30:42.452 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fef24ff5b113bd3-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.499 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.500 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.501 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:42.501 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:42.502 DEBUG:boto:Method: GET 2022-09-03T07:30:42.503 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:30:42.503 DEBUG:boto:Data: 2022-09-03T07:30:42.504 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.504 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.505 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.506 DEBUG:boto:Params: {} 2022-09-03T07:30:42.506 DEBUG:boto:Token: None 2022-09-03T07:30:42.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.511 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000022559c983be1b273-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.511 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.512 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.513 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.513 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:42.514 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:42.515 DEBUG:boto:Method: GET 2022-09-03T07:30:42.517 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:30:42.518 DEBUG:boto:Data: 2022-09-03T07:30:42.519 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.521 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.521 DEBUG:boto:Params: {} 2022-09-03T07:30:42.523 DEBUG:boto:Token: None 2022-09-03T07:30:42.523 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.524 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.531 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000ce5adb242469a54-00631302a2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.532 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.533 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj111 2022-09-03T07:30:42.534 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.536 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.537 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.538 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.539 DEBUG:boto:Method: GET 2022-09-03T07:30:42.541 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.542 DEBUG:boto:Data: 2022-09-03T07:30:42.543 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.544 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.545 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.546 DEBUG:boto:Params: {} 2022-09-03T07:30:42.547 DEBUG:boto:Token: None 2022-09-03T07:30:42.548 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KbWWbD1lDV51VAJ8FP/+HAdiPTA= 2022-09-03T07:30:42.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KbWWbD1lDV51VAJ8FP/+HAdiPTA=', 'Content-Length': '0'} 2022-09-03T07:30:42.554 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002c8e3027a3bed2ea-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.555 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.556 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.557 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.558 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.559 DEBUG:boto:Method: GET 2022-09-03T07:30:42.560 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.561 DEBUG:boto:Data: 2022-09-03T07:30:42.562 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.563 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.564 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.565 DEBUG:boto:Params: {} 2022-09-03T07:30:42.566 DEBUG:boto:Token: None 2022-09-03T07:30:42.567 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.568 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KbWWbD1lDV51VAJ8FP/+HAdiPTA= 2022-09-03T07:30:42.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KbWWbD1lDV51VAJ8FP/+HAdiPTA=', 'Content-Length': '0'} 2022-09-03T07:30:42.572 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097ee30fdcdee8ace-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.573 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.574 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.575 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.576 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.577 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.578 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.579 DEBUG:boto:Data: 2022-09-03T07:30:42.580 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.581 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.582 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.583 DEBUG:boto:Params: {} 2022-09-03T07:30:42.584 DEBUG:boto:Token: None 2022-09-03T07:30:42.584 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Nax+FwcWwKo3NoTT8v+hKk0XEY= 2022-09-03T07:30:42.586 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Nax+FwcWwKo3NoTT8v+hKk0XEY=', 'Content-Length': '0'} 2022-09-03T07:30:42.588 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000059509bd8110df6a5-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.588 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.589 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.589 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.590 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.591 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.591 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.592 DEBUG:boto:Data: 2022-09-03T07:30:42.592 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.593 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.594 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.595 DEBUG:boto:Params: {} 2022-09-03T07:30:42.596 DEBUG:boto:Token: None 2022-09-03T07:30:42.596 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5Nax+FwcWwKo3NoTT8v+hKk0XEY= 2022-09-03T07:30:42.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5Nax+FwcWwKo3NoTT8v+hKk0XEY=', 'Content-Length': '0'} 2022-09-03T07:30:42.601 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a90bb815dca26e4-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.602 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj111 2022-09-03T07:30:42.602 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.603 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.603 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.604 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.604 DEBUG:boto:Method: GET 2022-09-03T07:30:42.605 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.606 DEBUG:boto:Data: 2022-09-03T07:30:42.606 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.607 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.607 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.608 DEBUG:boto:Params: {} 2022-09-03T07:30:42.608 DEBUG:boto:Token: None 2022-09-03T07:30:42.609 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.610 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KbWWbD1lDV51VAJ8FP/+HAdiPTA= 2022-09-03T07:30:42.610 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KbWWbD1lDV51VAJ8FP/+HAdiPTA=', 'Content-Length': '0'} 2022-09-03T07:30:42.612 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008dee3fedf197a1ec-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.654 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.655 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.656 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.657 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.658 DEBUG:boto:Method: GET 2022-09-03T07:30:42.658 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.659 DEBUG:boto:Data: 2022-09-03T07:30:42.659 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.660 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.661 DEBUG:boto:Params: {} 2022-09-03T07:30:42.662 DEBUG:boto:Token: None 2022-09-03T07:30:42.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:42.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KbWWbD1lDV51VAJ8FP/+HAdiPTA= 2022-09-03T07:30:42.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KbWWbD1lDV51VAJ8FP/+HAdiPTA=', 'Content-Length': '0'} 2022-09-03T07:30:42.668 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002314644a9337c450-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.669 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.669 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.670 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:42.670 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:42.671 DEBUG:boto:Method: GET 2022-09-03T07:30:42.671 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:30:42.672 DEBUG:boto:Data: 2022-09-03T07:30:42.673 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.673 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.674 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.674 DEBUG:boto:Params: {} 2022-09-03T07:30:42.675 DEBUG:boto:Token: None 2022-09-03T07:30:42.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.679 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000639205ad011ed962-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.680 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.681 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.681 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.682 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:42.683 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:42.683 DEBUG:boto:Method: GET 2022-09-03T07:30:42.684 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:30:42.684 DEBUG:boto:Data: 2022-09-03T07:30:42.685 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.685 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.686 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.687 DEBUG:boto:Params: {} 2022-09-03T07:30:42.687 DEBUG:boto:Token: None 2022-09-03T07:30:42.688 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.689 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.690 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.694 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000166da5eb17998f95-00631302a2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.695 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.695 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.696 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.697 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.697 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:30:42.698 DEBUG:boto:Data: 2022-09-03T07:30:42.698 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.699 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.700 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.700 DEBUG:boto:Params: {} 2022-09-03T07:30:42.701 DEBUG:boto:Token: None 2022-09-03T07:30:42.702 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111 2022-09-03T07:30:42.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HdfMfkIzNj4VoVNcWRs5n2N1V+8= 2022-09-03T07:30:42.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HdfMfkIzNj4VoVNcWRs5n2N1V+8=', 'Content-Length': '0'} 2022-09-03T07:30:42.705 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000acefe51bbf980b06-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.706 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.706 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.707 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.707 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:30:42.708 DEBUG:boto:Data: 2022-09-03T07:30:42.708 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.709 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.710 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.710 DEBUG:boto:Params: {} 2022-09-03T07:30:42.711 DEBUG:boto:Token: None 2022-09-03T07:30:42.712 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111 2022-09-03T07:30:42.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HdfMfkIzNj4VoVNcWRs5n2N1V+8= 2022-09-03T07:30:42.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HdfMfkIzNj4VoVNcWRs5n2N1V+8=', 'Content-Length': '0'} 2022-09-03T07:30:42.715 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004739da87318d8fad-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.716 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj111 2022-09-03T07:30:42.716 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.717 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.717 DEBUG:boto:Method: GET 2022-09-03T07:30:42.718 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:30:42.718 DEBUG:boto:Data: 2022-09-03T07:30:42.719 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.719 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.721 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.721 DEBUG:boto:Params: {} 2022-09-03T07:30:42.722 DEBUG:boto:Token: None 2022-09-03T07:30:42.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111 2022-09-03T07:30:42.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bptV2wctTdT+hbL2858smNTwevI= 2022-09-03T07:30:42.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bptV2wctTdT+hbL2858smNTwevI=', 'Content-Length': '0'} 2022-09-03T07:30:42.725 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004b6b765dcaa3a52-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.726 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:42.727 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:42.727 DEBUG:boto:Method: GET 2022-09-03T07:30:42.728 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:30:42.728 DEBUG:boto:Data: 2022-09-03T07:30:42.729 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.730 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.731 DEBUG:boto:Params: {} 2022-09-03T07:30:42.731 DEBUG:boto:Token: None 2022-09-03T07:30:42.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj111 2022-09-03T07:30:42.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bptV2wctTdT+hbL2858smNTwevI= 2022-09-03T07:30:42.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bptV2wctTdT+hbL2858smNTwevI=', 'Content-Length': '0'} 2022-09-03T07:30:42.735 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005c5f62af8317100-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.778 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.779 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.779 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:42.780 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:42.780 DEBUG:boto:Method: GET 2022-09-03T07:30:42.781 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:30:42.782 DEBUG:boto:Data: 2022-09-03T07:30:42.783 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.784 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.785 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.785 DEBUG:boto:Params: {} 2022-09-03T07:30:42.786 DEBUG:boto:Token: None 2022-09-03T07:30:42.787 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.791 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a9ca0935771fc550-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.792 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.793 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.793 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.794 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:42.794 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:42.795 DEBUG:boto:Method: GET 2022-09-03T07:30:42.795 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:30:42.796 DEBUG:boto:Data: 2022-09-03T07:30:42.796 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.797 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.797 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.798 DEBUG:boto:Params: {} 2022-09-03T07:30:42.798 DEBUG:boto:Token: None 2022-09-03T07:30:42.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.804 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000092a66c510d8b2461-00631302a2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.805 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.806 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:30:42.806 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.807 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.807 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.808 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.808 DEBUG:boto:Method: GET 2022-09-03T07:30:42.809 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.809 DEBUG:boto:Data: 2022-09-03T07:30:42.810 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.810 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.811 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.811 DEBUG:boto:Params: {} 2022-09-03T07:30:42.812 DEBUG:boto:Token: None 2022-09-03T07:30:42.813 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.814 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sn/z8mAkzabWGkcNaC2ZYw2RtDk= 2022-09-03T07:30:42.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sn/z8mAkzabWGkcNaC2ZYw2RtDk=', 'Content-Length': '0'} 2022-09-03T07:30:42.816 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d99a3ec5d4b8e6ed-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.858 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.859 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.860 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.861 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.863 DEBUG:boto:Method: GET 2022-09-03T07:30:42.864 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.865 DEBUG:boto:Data: 2022-09-03T07:30:42.865 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.866 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.866 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.868 DEBUG:boto:Params: {} 2022-09-03T07:30:42.869 DEBUG:boto:Token: None 2022-09-03T07:30:42.869 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sn/z8mAkzabWGkcNaC2ZYw2RtDk= 2022-09-03T07:30:42.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sn/z8mAkzabWGkcNaC2ZYw2RtDk=', 'Content-Length': '0'} 2022-09-03T07:30:42.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b6f80f49d3a1a9e-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.874 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.874 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.875 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.876 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.876 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.877 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.878 DEBUG:boto:Data: 2022-09-03T07:30:42.878 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.879 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.880 DEBUG:boto:Params: {} 2022-09-03T07:30:42.880 DEBUG:boto:Token: None 2022-09-03T07:30:42.881 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzk2EM9D9nwDMPqWU7HXi8fBn74= 2022-09-03T07:30:42.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzk2EM9D9nwDMPqWU7HXi8fBn74=', 'Content-Length': '0'} 2022-09-03T07:30:42.884 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a275ad0aad93185d-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.886 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.887 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.887 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.888 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.889 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.889 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.890 DEBUG:boto:Data: 2022-09-03T07:30:42.890 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.891 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.891 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.892 DEBUG:boto:Params: {} 2022-09-03T07:30:42.892 DEBUG:boto:Token: None 2022-09-03T07:30:42.893 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zzk2EM9D9nwDMPqWU7HXi8fBn74= 2022-09-03T07:30:42.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zzk2EM9D9nwDMPqWU7HXi8fBn74=', 'Content-Length': '0'} 2022-09-03T07:30:42.896 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3831d301bd958e8-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.897 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:30:42.897 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.898 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.898 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.899 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.899 DEBUG:boto:Method: GET 2022-09-03T07:30:42.900 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.901 DEBUG:boto:Data: 2022-09-03T07:30:42.902 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.902 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.903 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.903 DEBUG:boto:Params: {} 2022-09-03T07:30:42.904 DEBUG:boto:Token: None 2022-09-03T07:30:42.905 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sn/z8mAkzabWGkcNaC2ZYw2RtDk= 2022-09-03T07:30:42.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sn/z8mAkzabWGkcNaC2ZYw2RtDk=', 'Content-Length': '0'} 2022-09-03T07:30:42.908 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ab171d5648c7b6d-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.910 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.911 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.912 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.913 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.914 DEBUG:boto:Method: GET 2022-09-03T07:30:42.915 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.916 DEBUG:boto:Data: 2022-09-03T07:30:42.917 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.918 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.919 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.920 DEBUG:boto:Params: {} 2022-09-03T07:30:42.920 DEBUG:boto:Token: None 2022-09-03T07:30:42.921 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:42.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sn/z8mAkzabWGkcNaC2ZYw2RtDk= 2022-09-03T07:30:42.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sn/z8mAkzabWGkcNaC2ZYw2RtDk=', 'Content-Length': '0'} 2022-09-03T07:30:42.925 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000017167fbcafdefcf7-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.926 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.927 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.927 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:42.928 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:42.928 DEBUG:boto:Method: GET 2022-09-03T07:30:42.929 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:30:42.929 DEBUG:boto:Data: 2022-09-03T07:30:42.930 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.931 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.931 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.932 DEBUG:boto:Params: {} 2022-09-03T07:30:42.933 DEBUG:boto:Token: None 2022-09-03T07:30:42.934 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.937 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b37fb62b3d25b062-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.938 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.939 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.939 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.940 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:42.940 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:42.941 DEBUG:boto:Method: GET 2022-09-03T07:30:42.941 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:30:42.942 DEBUG:boto:Data: 2022-09-03T07:30:42.943 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.943 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.944 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.944 DEBUG:boto:Params: {} 2022-09-03T07:30:42.945 DEBUG:boto:Token: None 2022-09-03T07:30:42.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/ 2022-09-03T07:30:42.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8= 2022-09-03T07:30:42.947 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Ca8sPgB6CYpmlyAA/kjqXruWO8=', 'Content-Length': '0'} 2022-09-03T07:30:42.951 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000ab4ec7cad505e1e-00631302a2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.952 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:42.954 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.954 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.955 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.955 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:30:42.956 DEBUG:boto:Data: 2022-09-03T07:30:42.956 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.957 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.957 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.958 DEBUG:boto:Params: {} 2022-09-03T07:30:42.958 DEBUG:boto:Token: None 2022-09-03T07:30:42.959 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115 2022-09-03T07:30:42.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kn8zr/oMZ1xArN4czj1NlZNmf1Q= 2022-09-03T07:30:42.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kn8zr/oMZ1xArN4czj1NlZNmf1Q=', 'Content-Length': '0'} 2022-09-03T07:30:42.962 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5bd076e237426fd-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.963 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.964 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.964 DEBUG:boto:Method: HEAD 2022-09-03T07:30:42.965 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:30:42.965 DEBUG:boto:Data: 2022-09-03T07:30:42.966 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.967 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.967 DEBUG:boto:Params: {} 2022-09-03T07:30:42.968 DEBUG:boto:Token: None 2022-09-03T07:30:42.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115 2022-09-03T07:30:42.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kn8zr/oMZ1xArN4czj1NlZNmf1Q= 2022-09-03T07:30:42.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kn8zr/oMZ1xArN4czj1NlZNmf1Q=', 'Content-Length': '0'} 2022-09-03T07:30:42.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002990138f3da88243-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.973 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:30:42.974 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.975 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.975 DEBUG:boto:Method: GET 2022-09-03T07:30:42.976 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:30:42.976 DEBUG:boto:Data: 2022-09-03T07:30:42.977 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.977 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:42.978 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.979 DEBUG:boto:Params: {} 2022-09-03T07:30:42.979 DEBUG:boto:Token: None 2022-09-03T07:30:42.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115 2022-09-03T07:30:42.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g8q9NTNqJhijhIYN85By73H3SAA= 2022-09-03T07:30:42.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g8q9NTNqJhijhIYN85By73H3SAA=', 'Content-Length': '0'} 2022-09-03T07:30:42.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032e6af0942412cc1-00631302a2-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.985 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:42.985 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:42.986 DEBUG:boto:Method: GET 2022-09-03T07:30:42.987 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:30:42.987 DEBUG:boto:Data: 2022-09-03T07:30:42.988 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:42.989 DEBUG:boto:Port: 8000 2022-09-03T07:30:42.989 DEBUG:boto:Params: {} 2022-09-03T07:30:42.990 DEBUG:boto:Token: None 2022-09-03T07:30:42.990 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:42 GMT /avysmw-75/obj115 2022-09-03T07:30:42.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g8q9NTNqJhijhIYN85By73H3SAA= 2022-09-03T07:30:42.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:42 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g8q9NTNqJhijhIYN85By73H3SAA=', 'Content-Length': '0'} 2022-09-03T07:30:42.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006c8b11a18a3a74c3-00631302a2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:42 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:42.995 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:42.995 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:42.996 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:42.997 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:42.997 DEBUG:boto:Method: GET 2022-09-03T07:30:42.998 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:30:42.998 DEBUG:boto:Data: 2022-09-03T07:30:42.999 DEBUG:boto:Headers: {} 2022-09-03T07:30:42.999 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.000 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.001 DEBUG:boto:Params: {} 2022-09-03T07:30:43.001 DEBUG:boto:Token: None 2022-09-03T07:30:43.002 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.007 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e3b7923ec2e9fd47-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.008 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.009 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.010 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.011 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:43.011 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:43.012 DEBUG:boto:Method: GET 2022-09-03T07:30:43.012 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:30:43.013 DEBUG:boto:Data: 2022-09-03T07:30:43.014 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.016 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.017 DEBUG:boto:Params: {} 2022-09-03T07:30:43.018 DEBUG:boto:Token: None 2022-09-03T07:30:43.018 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.019 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.019 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.024 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008335ab18e41350b5-00631302a3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.025 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.026 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:30:43.026 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.027 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.027 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.028 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.028 DEBUG:boto:Method: GET 2022-09-03T07:30:43.029 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.029 DEBUG:boto:Data: 2022-09-03T07:30:43.030 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.030 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.031 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.031 DEBUG:boto:Params: {} 2022-09-03T07:30:43.032 DEBUG:boto:Token: None 2022-09-03T07:30:43.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VtfMdxv7J0GV/0oMGMRbUD8XjYs= 2022-09-03T07:30:43.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VtfMdxv7J0GV/0oMGMRbUD8XjYs=', 'Content-Length': '0'} 2022-09-03T07:30:43.037 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000adc414509115d032-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.078 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.079 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.079 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.080 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.081 DEBUG:boto:Method: GET 2022-09-03T07:30:43.082 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.082 DEBUG:boto:Data: 2022-09-03T07:30:43.083 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.084 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.085 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.085 DEBUG:boto:Params: {} 2022-09-03T07:30:43.086 DEBUG:boto:Token: None 2022-09-03T07:30:43.087 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.088 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VtfMdxv7J0GV/0oMGMRbUD8XjYs= 2022-09-03T07:30:43.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VtfMdxv7J0GV/0oMGMRbUD8XjYs=', 'Content-Length': '0'} 2022-09-03T07:30:43.090 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db31d2065ae37ab4-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.091 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.092 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.092 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.093 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.094 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.094 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.095 DEBUG:boto:Data: 2022-09-03T07:30:43.096 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.097 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.098 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.098 DEBUG:boto:Params: {} 2022-09-03T07:30:43.099 DEBUG:boto:Token: None 2022-09-03T07:30:43.100 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VfwDMrqUyK/ZFb8bXf2Dw9Z/j9k= 2022-09-03T07:30:43.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VfwDMrqUyK/ZFb8bXf2Dw9Z/j9k=', 'Content-Length': '0'} 2022-09-03T07:30:43.103 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000054ba8043ed8e43c-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.104 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.105 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.105 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.106 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.106 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.107 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.108 DEBUG:boto:Data: 2022-09-03T07:30:43.109 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.110 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.111 DEBUG:boto:Params: {} 2022-09-03T07:30:43.112 DEBUG:boto:Token: None 2022-09-03T07:30:43.114 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VfwDMrqUyK/ZFb8bXf2Dw9Z/j9k= 2022-09-03T07:30:43.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VfwDMrqUyK/ZFb8bXf2Dw9Z/j9k=', 'Content-Length': '0'} 2022-09-03T07:30:43.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f35acb9b4e733934-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.119 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:30:43.119 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.120 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.121 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.122 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.123 DEBUG:boto:Method: GET 2022-09-03T07:30:43.124 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.125 DEBUG:boto:Data: 2022-09-03T07:30:43.126 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.127 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.128 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.129 DEBUG:boto:Params: {} 2022-09-03T07:30:43.130 DEBUG:boto:Token: None 2022-09-03T07:30:43.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VtfMdxv7J0GV/0oMGMRbUD8XjYs= 2022-09-03T07:30:43.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VtfMdxv7J0GV/0oMGMRbUD8XjYs=', 'Content-Length': '0'} 2022-09-03T07:30:43.137 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007559a14cc974c7df-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.139 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.140 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.141 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.142 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.143 DEBUG:boto:Method: GET 2022-09-03T07:30:43.144 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.145 DEBUG:boto:Data: 2022-09-03T07:30:43.146 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.147 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.148 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.149 DEBUG:boto:Params: {} 2022-09-03T07:30:43.150 DEBUG:boto:Token: None 2022-09-03T07:30:43.151 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:43.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VtfMdxv7J0GV/0oMGMRbUD8XjYs= 2022-09-03T07:30:43.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VtfMdxv7J0GV/0oMGMRbUD8XjYs=', 'Content-Length': '0'} 2022-09-03T07:30:43.166 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007b1a5c3212cfce8-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.167 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.169 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.169 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:43.170 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:43.171 DEBUG:boto:Method: GET 2022-09-03T07:30:43.172 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:30:43.173 DEBUG:boto:Data: 2022-09-03T07:30:43.174 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.175 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.175 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.176 DEBUG:boto:Params: {} 2022-09-03T07:30:43.177 DEBUG:boto:Token: None 2022-09-03T07:30:43.178 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.185 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b4af76800467c2d-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.186 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.187 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.187 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.188 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:43.189 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:43.189 DEBUG:boto:Method: GET 2022-09-03T07:30:43.190 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:30:43.191 DEBUG:boto:Data: 2022-09-03T07:30:43.192 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.193 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.193 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.194 DEBUG:boto:Params: {} 2022-09-03T07:30:43.194 DEBUG:boto:Token: None 2022-09-03T07:30:43.195 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.201 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008495b0bdecb862d0-00631302a3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.201 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.202 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.203 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.204 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.204 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:30:43.205 DEBUG:boto:Data: 2022-09-03T07:30:43.205 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.206 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.206 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.207 DEBUG:boto:Params: {} 2022-09-03T07:30:43.207 DEBUG:boto:Token: None 2022-09-03T07:30:43.208 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116 2022-09-03T07:30:43.208 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eWFbnP55kNtlOs8mB57jtrIdr0I= 2022-09-03T07:30:43.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eWFbnP55kNtlOs8mB57jtrIdr0I=', 'Content-Length': '0'} 2022-09-03T07:30:43.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b273fbe4ef6fc36e-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.216 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.216 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.217 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.218 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:30:43.220 DEBUG:boto:Data: 2022-09-03T07:30:43.220 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.222 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.223 DEBUG:boto:Params: {} 2022-09-03T07:30:43.224 DEBUG:boto:Token: None 2022-09-03T07:30:43.225 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116 2022-09-03T07:30:43.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eWFbnP55kNtlOs8mB57jtrIdr0I= 2022-09-03T07:30:43.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eWFbnP55kNtlOs8mB57jtrIdr0I=', 'Content-Length': '0'} 2022-09-03T07:30:43.229 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000dbfc2ef9a2f377b-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.230 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:30:43.231 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.231 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.232 DEBUG:boto:Method: GET 2022-09-03T07:30:43.232 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:30:43.233 DEBUG:boto:Data: 2022-09-03T07:30:43.233 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.234 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.236 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.237 DEBUG:boto:Params: {} 2022-09-03T07:30:43.238 DEBUG:boto:Token: None 2022-09-03T07:30:43.239 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116 2022-09-03T07:30:43.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KqR1UmMw4F34SzN5WnvGd2ptgTA= 2022-09-03T07:30:43.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KqR1UmMw4F34SzN5WnvGd2ptgTA=', 'Content-Length': '0'} 2022-09-03T07:30:43.242 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7897062e23fef11-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.243 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:43.243 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:43.244 DEBUG:boto:Method: GET 2022-09-03T07:30:43.245 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:30:43.245 DEBUG:boto:Data: 2022-09-03T07:30:43.246 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.246 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.247 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.248 DEBUG:boto:Params: {} 2022-09-03T07:30:43.249 DEBUG:boto:Token: None 2022-09-03T07:30:43.250 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj116 2022-09-03T07:30:43.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KqR1UmMw4F34SzN5WnvGd2ptgTA= 2022-09-03T07:30:43.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KqR1UmMw4F34SzN5WnvGd2ptgTA=', 'Content-Length': '0'} 2022-09-03T07:30:43.253 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e09213492a1ba696-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.254 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.255 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.255 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:43.256 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:43.256 DEBUG:boto:Method: GET 2022-09-03T07:30:43.257 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:30:43.257 DEBUG:boto:Data: 2022-09-03T07:30:43.258 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.259 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.259 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.260 DEBUG:boto:Params: {} 2022-09-03T07:30:43.261 DEBUG:boto:Token: None 2022-09-03T07:30:43.262 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.266 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e24f235ce9fc4881-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.315 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.316 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.317 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.318 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:43.319 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:43.319 DEBUG:boto:Method: GET 2022-09-03T07:30:43.320 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:30:43.322 DEBUG:boto:Data: 2022-09-03T07:30:43.322 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.323 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.324 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.324 DEBUG:boto:Params: {} 2022-09-03T07:30:43.325 DEBUG:boto:Token: None 2022-09-03T07:30:43.326 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.326 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.327 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.332 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036ecc5eb89b97435-00631302a3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.333 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.335 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:30:43.336 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.337 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.338 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.339 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.339 DEBUG:boto:Method: GET 2022-09-03T07:30:43.340 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.340 DEBUG:boto:Data: 2022-09-03T07:30:43.341 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.342 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.343 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.344 DEBUG:boto:Params: {} 2022-09-03T07:30:43.345 DEBUG:boto:Token: None 2022-09-03T07:30:43.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1tCc4X8JWq7/eV0QKhBiASJwRJc= 2022-09-03T07:30:43.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1tCc4X8JWq7/eV0QKhBiASJwRJc=', 'Content-Length': '0'} 2022-09-03T07:30:43.353 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000077b4ef05309eab1f-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.358 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.359 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.359 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.360 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.361 DEBUG:boto:Method: GET 2022-09-03T07:30:43.362 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.365 DEBUG:boto:Data: 2022-09-03T07:30:43.366 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.368 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.369 DEBUG:boto:Params: {} 2022-09-03T07:30:43.370 DEBUG:boto:Token: None 2022-09-03T07:30:43.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1tCc4X8JWq7/eV0QKhBiASJwRJc= 2022-09-03T07:30:43.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1tCc4X8JWq7/eV0QKhBiASJwRJc=', 'Content-Length': '0'} 2022-09-03T07:30:43.377 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052cf7a4a9b8ed492-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.379 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.380 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.381 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.381 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.382 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.383 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.384 DEBUG:boto:Data: 2022-09-03T07:30:43.385 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.387 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.388 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.389 DEBUG:boto:Params: {} 2022-09-03T07:30:43.390 DEBUG:boto:Token: None 2022-09-03T07:30:43.391 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g4IYMefnh9vCOhD10FYUNfKegDA= 2022-09-03T07:30:43.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g4IYMefnh9vCOhD10FYUNfKegDA=', 'Content-Length': '0'} 2022-09-03T07:30:43.399 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008df89ffb43be0cb8-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.400 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.401 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.402 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.403 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.404 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.406 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.407 DEBUG:boto:Data: 2022-09-03T07:30:43.408 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.409 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.410 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.411 DEBUG:boto:Params: {} 2022-09-03T07:30:43.412 DEBUG:boto:Token: None 2022-09-03T07:30:43.413 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.414 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g4IYMefnh9vCOhD10FYUNfKegDA= 2022-09-03T07:30:43.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g4IYMefnh9vCOhD10FYUNfKegDA=', 'Content-Length': '0'} 2022-09-03T07:30:43.418 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ffa08247071ccf8-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.420 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:30:43.421 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.422 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.423 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.424 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.425 DEBUG:boto:Method: GET 2022-09-03T07:30:43.427 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.428 DEBUG:boto:Data: 2022-09-03T07:30:43.429 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.430 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.431 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.432 DEBUG:boto:Params: {} 2022-09-03T07:30:43.433 DEBUG:boto:Token: None 2022-09-03T07:30:43.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1tCc4X8JWq7/eV0QKhBiASJwRJc= 2022-09-03T07:30:43.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1tCc4X8JWq7/eV0QKhBiASJwRJc=', 'Content-Length': '0'} 2022-09-03T07:30:43.438 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009728f75fa8700858-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.482 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.483 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.483 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.484 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.484 DEBUG:boto:Method: GET 2022-09-03T07:30:43.485 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.485 DEBUG:boto:Data: 2022-09-03T07:30:43.486 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.487 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.488 DEBUG:boto:Params: {} 2022-09-03T07:30:43.488 DEBUG:boto:Token: None 2022-09-03T07:30:43.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:43.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1tCc4X8JWq7/eV0QKhBiASJwRJc= 2022-09-03T07:30:43.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1tCc4X8JWq7/eV0QKhBiASJwRJc=', 'Content-Length': '0'} 2022-09-03T07:30:43.493 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd6b170f8dc5b0b4-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.495 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.495 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.496 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:43.497 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:43.498 DEBUG:boto:Method: GET 2022-09-03T07:30:43.499 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:30:43.500 DEBUG:boto:Data: 2022-09-03T07:30:43.501 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.502 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.503 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.504 DEBUG:boto:Params: {} 2022-09-03T07:30:43.505 DEBUG:boto:Token: None 2022-09-03T07:30:43.505 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.514 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000001af3abb27b4ee6e-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.515 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.516 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.518 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.519 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:43.520 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:43.520 DEBUG:boto:Method: GET 2022-09-03T07:30:43.521 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:30:43.521 DEBUG:boto:Data: 2022-09-03T07:30:43.522 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.523 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.524 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.525 DEBUG:boto:Params: {} 2022-09-03T07:30:43.526 DEBUG:boto:Token: None 2022-09-03T07:30:43.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.533 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000806c6e419bc5b2b4-00631302a3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.534 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.534 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.535 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.536 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.536 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:30:43.537 DEBUG:boto:Data: 2022-09-03T07:30:43.537 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.538 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.541 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.542 DEBUG:boto:Params: {} 2022-09-03T07:30:43.543 DEBUG:boto:Token: None 2022-09-03T07:30:43.544 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124 2022-09-03T07:30:43.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X5g9X67+WNfCEJFLAJcEAXKjyOY= 2022-09-03T07:30:43.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X5g9X67+WNfCEJFLAJcEAXKjyOY=', 'Content-Length': '0'} 2022-09-03T07:30:43.548 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be01fc31f0a34c8a-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.549 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.550 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.552 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.553 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:30:43.554 DEBUG:boto:Data: 2022-09-03T07:30:43.556 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.557 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.558 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.559 DEBUG:boto:Params: {} 2022-09-03T07:30:43.560 DEBUG:boto:Token: None 2022-09-03T07:30:43.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124 2022-09-03T07:30:43.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X5g9X67+WNfCEJFLAJcEAXKjyOY= 2022-09-03T07:30:43.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X5g9X67+WNfCEJFLAJcEAXKjyOY=', 'Content-Length': '0'} 2022-09-03T07:30:43.567 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000061e544cc9e7cb89-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.569 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:30:43.569 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.570 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.571 DEBUG:boto:Method: GET 2022-09-03T07:30:43.571 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:30:43.572 DEBUG:boto:Data: 2022-09-03T07:30:43.572 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.573 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.573 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.574 DEBUG:boto:Params: {} 2022-09-03T07:30:43.574 DEBUG:boto:Token: None 2022-09-03T07:30:43.575 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124 2022-09-03T07:30:43.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AIvxEcdSpSlIhzskdJ2C0yafFc8= 2022-09-03T07:30:43.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AIvxEcdSpSlIhzskdJ2C0yafFc8=', 'Content-Length': '0'} 2022-09-03T07:30:43.579 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007a46143db49f5996-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.580 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:43.581 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:43.581 DEBUG:boto:Method: GET 2022-09-03T07:30:43.582 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:30:43.582 DEBUG:boto:Data: 2022-09-03T07:30:43.583 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.584 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.584 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.585 DEBUG:boto:Params: {} 2022-09-03T07:30:43.587 DEBUG:boto:Token: None 2022-09-03T07:30:43.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj124 2022-09-03T07:30:43.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AIvxEcdSpSlIhzskdJ2C0yafFc8= 2022-09-03T07:30:43.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AIvxEcdSpSlIhzskdJ2C0yafFc8=', 'Content-Length': '0'} 2022-09-03T07:30:43.592 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0666c54670d500e-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.595 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.595 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.596 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:43.597 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:43.598 DEBUG:boto:Method: GET 2022-09-03T07:30:43.599 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:30:43.600 DEBUG:boto:Data: 2022-09-03T07:30:43.600 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.601 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.601 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.602 DEBUG:boto:Params: {} 2022-09-03T07:30:43.603 DEBUG:boto:Token: None 2022-09-03T07:30:43.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.605 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.607 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb9c8caa373f6af3-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.608 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.609 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.609 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.610 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:43.610 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:43.611 DEBUG:boto:Method: GET 2022-09-03T07:30:43.611 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:30:43.612 DEBUG:boto:Data: 2022-09-03T07:30:43.613 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.614 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.614 DEBUG:boto:Params: {} 2022-09-03T07:30:43.615 DEBUG:boto:Token: None 2022-09-03T07:30:43.616 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.617 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.622 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000880c1176006ec533-00631302a3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.662 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.664 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:30:43.665 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.665 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.666 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.667 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.668 DEBUG:boto:Method: GET 2022-09-03T07:30:43.668 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.669 DEBUG:boto:Data: 2022-09-03T07:30:43.670 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.671 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.672 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.673 DEBUG:boto:Params: {} 2022-09-03T07:30:43.674 DEBUG:boto:Token: None 2022-09-03T07:30:43.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.675 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PZFbFD4Ig76WY7Wdijr/m3iH/Rs= 2022-09-03T07:30:43.676 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PZFbFD4Ig76WY7Wdijr/m3iH/Rs=', 'Content-Length': '0'} 2022-09-03T07:30:43.680 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e821d2b1689b7c0d-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.681 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.682 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.683 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.684 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.684 DEBUG:boto:Method: GET 2022-09-03T07:30:43.685 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.686 DEBUG:boto:Data: 2022-09-03T07:30:43.687 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.688 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.689 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.690 DEBUG:boto:Params: {} 2022-09-03T07:30:43.691 DEBUG:boto:Token: None 2022-09-03T07:30:43.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PZFbFD4Ig76WY7Wdijr/m3iH/Rs= 2022-09-03T07:30:43.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PZFbFD4Ig76WY7Wdijr/m3iH/Rs=', 'Content-Length': '0'} 2022-09-03T07:30:43.698 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c21dca57db2a9b64-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.699 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.700 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.701 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.702 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.702 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.703 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.704 DEBUG:boto:Data: 2022-09-03T07:30:43.704 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.705 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.706 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.706 DEBUG:boto:Params: {} 2022-09-03T07:30:43.707 DEBUG:boto:Token: None 2022-09-03T07:30:43.707 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.708 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/RbmYkhyKSGvcidyCc7jWmyjEfo= 2022-09-03T07:30:43.709 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/RbmYkhyKSGvcidyCc7jWmyjEfo=', 'Content-Length': '0'} 2022-09-03T07:30:43.710 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000496f56f1b3ae083a-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.711 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.712 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.713 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.713 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.714 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.714 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.715 DEBUG:boto:Data: 2022-09-03T07:30:43.715 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.716 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.716 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.717 DEBUG:boto:Params: {} 2022-09-03T07:30:43.717 DEBUG:boto:Token: None 2022-09-03T07:30:43.718 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/RbmYkhyKSGvcidyCc7jWmyjEfo= 2022-09-03T07:30:43.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/RbmYkhyKSGvcidyCc7jWmyjEfo=', 'Content-Length': '0'} 2022-09-03T07:30:43.721 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006dff650a7c894d9b-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.723 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:30:43.723 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.724 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.724 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.725 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.726 DEBUG:boto:Method: GET 2022-09-03T07:30:43.726 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.727 DEBUG:boto:Data: 2022-09-03T07:30:43.727 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.728 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.729 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.730 DEBUG:boto:Params: {} 2022-09-03T07:30:43.731 DEBUG:boto:Token: None 2022-09-03T07:30:43.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PZFbFD4Ig76WY7Wdijr/m3iH/Rs= 2022-09-03T07:30:43.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PZFbFD4Ig76WY7Wdijr/m3iH/Rs=', 'Content-Length': '0'} 2022-09-03T07:30:43.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048ddb52903cbaa30-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.778 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.779 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.780 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.780 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.781 DEBUG:boto:Method: GET 2022-09-03T07:30:43.781 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.782 DEBUG:boto:Data: 2022-09-03T07:30:43.783 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.783 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.784 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.784 DEBUG:boto:Params: {} 2022-09-03T07:30:43.785 DEBUG:boto:Token: None 2022-09-03T07:30:43.785 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:43.786 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PZFbFD4Ig76WY7Wdijr/m3iH/Rs= 2022-09-03T07:30:43.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PZFbFD4Ig76WY7Wdijr/m3iH/Rs=', 'Content-Length': '0'} 2022-09-03T07:30:43.790 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c39246c2629d487-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.791 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.791 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.792 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:43.792 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:43.793 DEBUG:boto:Method: GET 2022-09-03T07:30:43.794 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:30:43.794 DEBUG:boto:Data: 2022-09-03T07:30:43.795 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.796 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.796 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.797 DEBUG:boto:Params: {} 2022-09-03T07:30:43.797 DEBUG:boto:Token: None 2022-09-03T07:30:43.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.798 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.802 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005f86e76c256cd372-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.802 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.803 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.804 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.804 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:43.805 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:43.805 DEBUG:boto:Method: GET 2022-09-03T07:30:43.806 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:30:43.807 DEBUG:boto:Data: 2022-09-03T07:30:43.807 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.808 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.809 DEBUG:boto:Params: {} 2022-09-03T07:30:43.810 DEBUG:boto:Token: None 2022-09-03T07:30:43.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.817 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000097eb5c40da405529-00631302a3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.818 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.819 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.819 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.820 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.821 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:30:43.821 DEBUG:boto:Data: 2022-09-03T07:30:43.822 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.822 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.823 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.824 DEBUG:boto:Params: {} 2022-09-03T07:30:43.824 DEBUG:boto:Token: None 2022-09-03T07:30:43.825 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145 2022-09-03T07:30:43.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BcN/WdW+KmCKFBfZsX5Kv7c9Awk= 2022-09-03T07:30:43.826 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BcN/WdW+KmCKFBfZsX5Kv7c9Awk=', 'Content-Length': '0'} 2022-09-03T07:30:43.829 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a4ca0e4d07d146a-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.831 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.832 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.832 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.833 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:30:43.834 DEBUG:boto:Data: 2022-09-03T07:30:43.835 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.836 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.836 DEBUG:boto:Params: {} 2022-09-03T07:30:43.837 DEBUG:boto:Token: None 2022-09-03T07:30:43.837 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145 2022-09-03T07:30:43.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BcN/WdW+KmCKFBfZsX5Kv7c9Awk= 2022-09-03T07:30:43.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BcN/WdW+KmCKFBfZsX5Kv7c9Awk=', 'Content-Length': '0'} 2022-09-03T07:30:43.841 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bbdcb4183df76678-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.842 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:30:43.843 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.843 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.844 DEBUG:boto:Method: GET 2022-09-03T07:30:43.845 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:30:43.845 DEBUG:boto:Data: 2022-09-03T07:30:43.846 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.846 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.847 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.848 DEBUG:boto:Params: {} 2022-09-03T07:30:43.849 DEBUG:boto:Token: None 2022-09-03T07:30:43.850 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145 2022-09-03T07:30:43.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ucbsHrw1muT9vxLgajJX6SbineY= 2022-09-03T07:30:43.853 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ucbsHrw1muT9vxLgajJX6SbineY=', 'Content-Length': '0'} 2022-09-03T07:30:43.856 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003e5b3270b12d2aec-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.857 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:43.858 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:43.858 DEBUG:boto:Method: GET 2022-09-03T07:30:43.859 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:30:43.859 DEBUG:boto:Data: 2022-09-03T07:30:43.860 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.861 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.861 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.862 DEBUG:boto:Params: {} 2022-09-03T07:30:43.862 DEBUG:boto:Token: None 2022-09-03T07:30:43.863 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj145 2022-09-03T07:30:43.864 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ucbsHrw1muT9vxLgajJX6SbineY= 2022-09-03T07:30:43.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ucbsHrw1muT9vxLgajJX6SbineY=', 'Content-Length': '0'} 2022-09-03T07:30:43.867 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf4fcd3823180d42-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.869 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.875 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.876 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:43.876 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:43.877 DEBUG:boto:Method: GET 2022-09-03T07:30:43.878 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:30:43.878 DEBUG:boto:Data: 2022-09-03T07:30:43.879 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.880 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.881 DEBUG:boto:Params: {} 2022-09-03T07:30:43.881 DEBUG:boto:Token: None 2022-09-03T07:30:43.882 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.886 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e972305fe9c6dd57-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.887 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.888 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:43.888 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:43.889 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:43.889 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:43.890 DEBUG:boto:Method: GET 2022-09-03T07:30:43.891 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:30:43.891 DEBUG:boto:Data: 2022-09-03T07:30:43.892 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.893 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.893 DEBUG:boto:Params: {} 2022-09-03T07:30:43.894 DEBUG:boto:Token: None 2022-09-03T07:30:43.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/ 2022-09-03T07:30:43.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws= 2022-09-03T07:30:43.896 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lg4sYZZVi+PuPQ5ZIKlA+CT4rws=', 'Content-Length': '0'} 2022-09-03T07:30:43.908 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed62659af873756f-00631302a3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.946 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:43.947 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj148 2022-09-03T07:30:43.948 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:43.948 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:43.949 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.949 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.950 DEBUG:boto:Method: GET 2022-09-03T07:30:43.951 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.952 DEBUG:boto:Data: 2022-09-03T07:30:43.952 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.953 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.953 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.954 DEBUG:boto:Params: {} 2022-09-03T07:30:43.955 DEBUG:boto:Token: None 2022-09-03T07:30:43.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RezRPeLoM+LxoZbmzAETZ2fNKJY= 2022-09-03T07:30:43.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RezRPeLoM+LxoZbmzAETZ2fNKJY=', 'Content-Length': '0'} 2022-09-03T07:30:43.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007fa29c649b0aaf5c-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.961 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:43.961 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:43.962 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.963 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.963 DEBUG:boto:Method: GET 2022-09-03T07:30:43.964 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.964 DEBUG:boto:Data: 2022-09-03T07:30:43.965 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.966 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.966 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.967 DEBUG:boto:Params: {} 2022-09-03T07:30:43.967 DEBUG:boto:Token: None 2022-09-03T07:30:43.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RezRPeLoM+LxoZbmzAETZ2fNKJY= 2022-09-03T07:30:43.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RezRPeLoM+LxoZbmzAETZ2fNKJY=', 'Content-Length': '0'} 2022-09-03T07:30:43.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000514c0c846a64c5c9-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.973 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:43.973 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:43.974 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.974 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.975 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.975 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.976 DEBUG:boto:Data: 2022-09-03T07:30:43.976 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.977 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:43.978 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.978 DEBUG:boto:Params: {} 2022-09-03T07:30:43.979 DEBUG:boto:Token: None 2022-09-03T07:30:43.980 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VBH88NCTBO16g0/6e1Y8w6sHIeY= 2022-09-03T07:30:43.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VBH88NCTBO16g0/6e1Y8w6sHIeY=', 'Content-Length': '0'} 2022-09-03T07:30:43.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000500655dfdfc7cb6e-00631302a3-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.984 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:43.985 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:43.985 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.986 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.986 DEBUG:boto:Method: HEAD 2022-09-03T07:30:43.987 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.988 DEBUG:boto:Data: 2022-09-03T07:30:43.988 DEBUG:boto:Headers: {} 2022-09-03T07:30:43.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:43.989 DEBUG:boto:Port: 8000 2022-09-03T07:30:43.990 DEBUG:boto:Params: {} 2022-09-03T07:30:43.990 DEBUG:boto:Token: None 2022-09-03T07:30:43.991 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:43 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VBH88NCTBO16g0/6e1Y8w6sHIeY= 2022-09-03T07:30:43.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:43 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VBH88NCTBO16g0/6e1Y8w6sHIeY=', 'Content-Length': '0'} 2022-09-03T07:30:43.994 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000996650ce32ef7015-00631302a3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:43 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:43.995 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj148 2022-09-03T07:30:43.995 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:43.996 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:43.996 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.997 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.998 DEBUG:boto:Method: GET 2022-09-03T07:30:43.998 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:43.999 DEBUG:boto:Data: 2022-09-03T07:30:43.999 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.000 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.001 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.001 DEBUG:boto:Params: {} 2022-09-03T07:30:44.002 DEBUG:boto:Token: None 2022-09-03T07:30:44.003 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:44.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PxIhFYM5FZRGajS1zZkXaZpdhas= 2022-09-03T07:30:44.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PxIhFYM5FZRGajS1zZkXaZpdhas=', 'Content-Length': '0'} 2022-09-03T07:30:44.006 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002769de12913e5492-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.008 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:44.009 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:44.010 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:44.010 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:44.011 DEBUG:boto:Method: GET 2022-09-03T07:30:44.011 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:44.012 DEBUG:boto:Data: 2022-09-03T07:30:44.013 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.014 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.015 DEBUG:boto:Params: {} 2022-09-03T07:30:44.015 DEBUG:boto:Token: None 2022-09-03T07:30:44.016 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:44.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PxIhFYM5FZRGajS1zZkXaZpdhas= 2022-09-03T07:30:44.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PxIhFYM5FZRGajS1zZkXaZpdhas=', 'Content-Length': '0'} 2022-09-03T07:30:44.019 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6cdd2210a0858d3-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.020 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.021 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.021 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:44.022 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:44.023 DEBUG:boto:Method: GET 2022-09-03T07:30:44.023 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:30:44.024 DEBUG:boto:Data: 2022-09-03T07:30:44.024 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.025 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.026 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.026 DEBUG:boto:Params: {} 2022-09-03T07:30:44.027 DEBUG:boto:Token: None 2022-09-03T07:30:44.028 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.029 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.031 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e3c0a697eaf36a0a-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.032 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.033 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.033 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.034 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:44.035 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:44.035 DEBUG:boto:Method: GET 2022-09-03T07:30:44.036 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:30:44.037 DEBUG:boto:Data: 2022-09-03T07:30:44.037 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.038 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.039 DEBUG:boto:Params: {} 2022-09-03T07:30:44.040 DEBUG:boto:Token: None 2022-09-03T07:30:44.040 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.052 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e0015781f02e47d6-00631302a4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.054 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.056 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:44.057 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:44.058 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.059 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:30:44.060 DEBUG:boto:Data: 2022-09-03T07:30:44.061 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.062 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.063 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.064 DEBUG:boto:Params: {} 2022-09-03T07:30:44.065 DEBUG:boto:Token: None 2022-09-03T07:30:44.066 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj148 2022-09-03T07:30:44.067 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K7reigT3XJOK3gwG6c6WpYVaPlc= 2022-09-03T07:30:44.069 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K7reigT3XJOK3gwG6c6WpYVaPlc=', 'Content-Length': '0'} 2022-09-03T07:30:44.071 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b650e4d41883910-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.073 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:44.074 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:44.075 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.076 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:30:44.077 DEBUG:boto:Data: 2022-09-03T07:30:44.077 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.079 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.080 DEBUG:boto:Params: {} 2022-09-03T07:30:44.080 DEBUG:boto:Token: None 2022-09-03T07:30:44.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj148 2022-09-03T07:30:44.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K7reigT3XJOK3gwG6c6WpYVaPlc= 2022-09-03T07:30:44.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K7reigT3XJOK3gwG6c6WpYVaPlc=', 'Content-Length': '0'} 2022-09-03T07:30:44.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3463a87624358cf-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.087 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj148 2022-09-03T07:30:44.089 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:44.090 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:44.091 DEBUG:boto:Method: GET 2022-09-03T07:30:44.092 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:30:44.093 DEBUG:boto:Data: 2022-09-03T07:30:44.094 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.096 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.096 DEBUG:boto:Params: {} 2022-09-03T07:30:44.097 DEBUG:boto:Token: None 2022-09-03T07:30:44.098 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj148 2022-09-03T07:30:44.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D/KA2tUJuVj33otipCfk2T/e0Cw= 2022-09-03T07:30:44.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D/KA2tUJuVj33otipCfk2T/e0Cw=', 'Content-Length': '0'} 2022-09-03T07:30:44.103 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d2915adbbf864012-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.104 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:44.105 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:44.106 DEBUG:boto:Method: GET 2022-09-03T07:30:44.106 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:30:44.107 DEBUG:boto:Data: 2022-09-03T07:30:44.108 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.110 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.111 DEBUG:boto:Params: {} 2022-09-03T07:30:44.112 DEBUG:boto:Token: None 2022-09-03T07:30:44.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj148 2022-09-03T07:30:44.145 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D/KA2tUJuVj33otipCfk2T/e0Cw= 2022-09-03T07:30:44.146 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D/KA2tUJuVj33otipCfk2T/e0Cw=', 'Content-Length': '0'} 2022-09-03T07:30:44.153 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad34bd8a8e3f6863-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.155 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.156 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.157 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:44.161 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:44.162 DEBUG:boto:Method: GET 2022-09-03T07:30:44.162 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:30:44.163 DEBUG:boto:Data: 2022-09-03T07:30:44.164 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.164 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.165 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.165 DEBUG:boto:Params: {} 2022-09-03T07:30:44.166 DEBUG:boto:Token: None 2022-09-03T07:30:44.167 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.171 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011d208a6a124078e-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.172 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.172 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.173 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.173 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:44.174 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:44.175 DEBUG:boto:Method: GET 2022-09-03T07:30:44.176 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:30:44.176 DEBUG:boto:Data: 2022-09-03T07:30:44.177 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.177 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.178 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.179 DEBUG:boto:Params: {} 2022-09-03T07:30:44.179 DEBUG:boto:Token: None 2022-09-03T07:30:44.180 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.186 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005624f0d84dedc70a-00631302a4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.190 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.191 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:30:44.192 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.193 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.194 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.199 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.199 DEBUG:boto:Method: GET 2022-09-03T07:30:44.200 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.201 DEBUG:boto:Data: 2022-09-03T07:30:44.201 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.202 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.202 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.203 DEBUG:boto:Params: {} 2022-09-03T07:30:44.203 DEBUG:boto:Token: None 2022-09-03T07:30:44.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zLgWceqk13qRYZeMrm2aupzsTds= 2022-09-03T07:30:44.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zLgWceqk13qRYZeMrm2aupzsTds=', 'Content-Length': '0'} 2022-09-03T07:30:44.208 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c565145348a1b3f4-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.254 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.255 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.256 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.257 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.258 DEBUG:boto:Method: GET 2022-09-03T07:30:44.259 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.261 DEBUG:boto:Data: 2022-09-03T07:30:44.262 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.264 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.265 DEBUG:boto:Params: {} 2022-09-03T07:30:44.266 DEBUG:boto:Token: None 2022-09-03T07:30:44.266 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zLgWceqk13qRYZeMrm2aupzsTds= 2022-09-03T07:30:44.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zLgWceqk13qRYZeMrm2aupzsTds=', 'Content-Length': '0'} 2022-09-03T07:30:44.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000feb7885b8bd85601-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.271 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.272 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.272 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.273 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.273 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.274 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.275 DEBUG:boto:Data: 2022-09-03T07:30:44.275 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.276 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.276 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.277 DEBUG:boto:Params: {} 2022-09-03T07:30:44.278 DEBUG:boto:Token: None 2022-09-03T07:30:44.279 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cbYyMVd1+BpPSX859+0NqS6QEhg= 2022-09-03T07:30:44.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cbYyMVd1+BpPSX859+0NqS6QEhg=', 'Content-Length': '0'} 2022-09-03T07:30:44.282 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000539a40b419b42c61-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.284 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.284 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.285 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.285 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.286 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.286 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.287 DEBUG:boto:Data: 2022-09-03T07:30:44.287 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.288 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.289 DEBUG:boto:Params: {} 2022-09-03T07:30:44.289 DEBUG:boto:Token: None 2022-09-03T07:30:44.290 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cbYyMVd1+BpPSX859+0NqS6QEhg= 2022-09-03T07:30:44.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cbYyMVd1+BpPSX859+0NqS6QEhg=', 'Content-Length': '0'} 2022-09-03T07:30:44.294 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000025455c5ec451b857-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.295 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:30:44.296 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.296 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.297 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.297 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.298 DEBUG:boto:Method: GET 2022-09-03T07:30:44.298 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.299 DEBUG:boto:Data: 2022-09-03T07:30:44.299 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.301 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.301 DEBUG:boto:Params: {} 2022-09-03T07:30:44.302 DEBUG:boto:Token: None 2022-09-03T07:30:44.302 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.303 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zLgWceqk13qRYZeMrm2aupzsTds= 2022-09-03T07:30:44.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zLgWceqk13qRYZeMrm2aupzsTds=', 'Content-Length': '0'} 2022-09-03T07:30:44.306 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4c7bf1f188d764b-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.307 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.308 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.308 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.309 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.310 DEBUG:boto:Method: GET 2022-09-03T07:30:44.311 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.312 DEBUG:boto:Data: 2022-09-03T07:30:44.313 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.314 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.315 DEBUG:boto:Params: {} 2022-09-03T07:30:44.316 DEBUG:boto:Token: None 2022-09-03T07:30:44.316 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:44.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zLgWceqk13qRYZeMrm2aupzsTds= 2022-09-03T07:30:44.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zLgWceqk13qRYZeMrm2aupzsTds=', 'Content-Length': '0'} 2022-09-03T07:30:44.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036db69a1a27ae8bf-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.366 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.367 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.368 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:44.369 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:44.370 DEBUG:boto:Method: GET 2022-09-03T07:30:44.371 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:30:44.372 DEBUG:boto:Data: 2022-09-03T07:30:44.373 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.374 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.375 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.376 DEBUG:boto:Params: {} 2022-09-03T07:30:44.377 DEBUG:boto:Token: None 2022-09-03T07:30:44.378 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.383 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bfeea4591918206f-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.385 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.386 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.387 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.388 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:44.389 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:44.390 DEBUG:boto:Method: GET 2022-09-03T07:30:44.391 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:30:44.392 DEBUG:boto:Data: 2022-09-03T07:30:44.393 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.394 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.395 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.396 DEBUG:boto:Params: {} 2022-09-03T07:30:44.397 DEBUG:boto:Token: None 2022-09-03T07:30:44.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.405 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e5027b7f77edae65-00631302a4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.406 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.407 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.408 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.408 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.409 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:30:44.409 DEBUG:boto:Data: 2022-09-03T07:30:44.410 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.411 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.411 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.412 DEBUG:boto:Params: {} 2022-09-03T07:30:44.412 DEBUG:boto:Token: None 2022-09-03T07:30:44.413 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155 2022-09-03T07:30:44.414 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J2TEovkCfyCsN1IGJBIJUUf9fZY= 2022-09-03T07:30:44.414 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J2TEovkCfyCsN1IGJBIJUUf9fZY=', 'Content-Length': '0'} 2022-09-03T07:30:44.416 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000184426b4e38229dc-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.417 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.418 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.419 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.420 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:30:44.421 DEBUG:boto:Data: 2022-09-03T07:30:44.422 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.424 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.425 DEBUG:boto:Params: {} 2022-09-03T07:30:44.426 DEBUG:boto:Token: None 2022-09-03T07:30:44.427 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155 2022-09-03T07:30:44.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:J2TEovkCfyCsN1IGJBIJUUf9fZY= 2022-09-03T07:30:44.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:J2TEovkCfyCsN1IGJBIJUUf9fZY=', 'Content-Length': '0'} 2022-09-03T07:30:44.432 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c65480aa9247ccb5-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.433 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:30:44.434 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.435 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.436 DEBUG:boto:Method: GET 2022-09-03T07:30:44.437 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:30:44.437 DEBUG:boto:Data: 2022-09-03T07:30:44.438 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.439 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.440 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.441 DEBUG:boto:Params: {} 2022-09-03T07:30:44.442 DEBUG:boto:Token: None 2022-09-03T07:30:44.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155 2022-09-03T07:30:44.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D8jWtFPJXYwnSc1PhSKg+aeB2n8= 2022-09-03T07:30:44.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D8jWtFPJXYwnSc1PhSKg+aeB2n8=', 'Content-Length': '0'} 2022-09-03T07:30:44.446 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081e12ea0c5f5eca8-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.447 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:44.448 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:44.448 DEBUG:boto:Method: GET 2022-09-03T07:30:44.449 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:30:44.449 DEBUG:boto:Data: 2022-09-03T07:30:44.450 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.451 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.451 DEBUG:boto:Params: {} 2022-09-03T07:30:44.452 DEBUG:boto:Token: None 2022-09-03T07:30:44.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj155 2022-09-03T07:30:44.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D8jWtFPJXYwnSc1PhSKg+aeB2n8= 2022-09-03T07:30:44.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D8jWtFPJXYwnSc1PhSKg+aeB2n8=', 'Content-Length': '0'} 2022-09-03T07:30:44.456 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000946fbfc417711e74-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.457 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.457 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.458 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:44.459 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:44.459 DEBUG:boto:Method: GET 2022-09-03T07:30:44.460 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:30:44.460 DEBUG:boto:Data: 2022-09-03T07:30:44.461 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.462 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.463 DEBUG:boto:Params: {} 2022-09-03T07:30:44.463 DEBUG:boto:Token: None 2022-09-03T07:30:44.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.468 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ec340170e3c90215-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.510 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.511 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.512 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.512 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:44.513 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:44.513 DEBUG:boto:Method: GET 2022-09-03T07:30:44.514 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:30:44.514 DEBUG:boto:Data: 2022-09-03T07:30:44.515 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.516 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.516 DEBUG:boto:Params: {} 2022-09-03T07:30:44.517 DEBUG:boto:Token: None 2022-09-03T07:30:44.518 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.523 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e342a218ca404df-00631302a4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.524 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.525 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:30:44.525 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.526 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.526 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.527 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.527 DEBUG:boto:Method: GET 2022-09-03T07:30:44.528 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.529 DEBUG:boto:Data: 2022-09-03T07:30:44.529 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.530 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.530 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.531 DEBUG:boto:Params: {} 2022-09-03T07:30:44.531 DEBUG:boto:Token: None 2022-09-03T07:30:44.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:De+ZjrrdauLb+AtK4yOq0Ystu4k= 2022-09-03T07:30:44.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:De+ZjrrdauLb+AtK4yOq0Ystu4k=', 'Content-Length': '0'} 2022-09-03T07:30:44.535 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000051feb5e00ab38d73-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.536 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.537 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.537 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.538 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.538 DEBUG:boto:Method: GET 2022-09-03T07:30:44.539 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.539 DEBUG:boto:Data: 2022-09-03T07:30:44.540 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.541 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.542 DEBUG:boto:Params: {} 2022-09-03T07:30:44.542 DEBUG:boto:Token: None 2022-09-03T07:30:44.543 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:De+ZjrrdauLb+AtK4yOq0Ystu4k= 2022-09-03T07:30:44.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:De+ZjrrdauLb+AtK4yOq0Ystu4k=', 'Content-Length': '0'} 2022-09-03T07:30:44.546 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081f797b6c4368f1b-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.590 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.591 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.591 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.592 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.593 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.594 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.595 DEBUG:boto:Data: 2022-09-03T07:30:44.596 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.597 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.598 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.599 DEBUG:boto:Params: {} 2022-09-03T07:30:44.600 DEBUG:boto:Token: None 2022-09-03T07:30:44.601 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8f3Q82kRmc0gZUIZOfNy177OskE= 2022-09-03T07:30:44.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8f3Q82kRmc0gZUIZOfNy177OskE=', 'Content-Length': '0'} 2022-09-03T07:30:44.607 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000064e865bf9740179a-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.609 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.610 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.611 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.611 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.612 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.613 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.614 DEBUG:boto:Data: 2022-09-03T07:30:44.615 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.617 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.618 DEBUG:boto:Params: {} 2022-09-03T07:30:44.619 DEBUG:boto:Token: None 2022-09-03T07:30:44.619 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8f3Q82kRmc0gZUIZOfNy177OskE= 2022-09-03T07:30:44.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8f3Q82kRmc0gZUIZOfNy177OskE=', 'Content-Length': '0'} 2022-09-03T07:30:44.623 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000777e2490172e046f-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.624 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:30:44.624 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.625 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.625 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.626 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.627 DEBUG:boto:Method: GET 2022-09-03T07:30:44.627 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.628 DEBUG:boto:Data: 2022-09-03T07:30:44.629 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.630 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.630 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.631 DEBUG:boto:Params: {} 2022-09-03T07:30:44.632 DEBUG:boto:Token: None 2022-09-03T07:30:44.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:De+ZjrrdauLb+AtK4yOq0Ystu4k= 2022-09-03T07:30:44.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:De+ZjrrdauLb+AtK4yOq0Ystu4k=', 'Content-Length': '0'} 2022-09-03T07:30:44.636 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2760afa7dc02027-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.637 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.637 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.638 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.639 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.640 DEBUG:boto:Method: GET 2022-09-03T07:30:44.641 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.642 DEBUG:boto:Data: 2022-09-03T07:30:44.643 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.645 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.646 DEBUG:boto:Params: {} 2022-09-03T07:30:44.647 DEBUG:boto:Token: None 2022-09-03T07:30:44.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:44.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:De+ZjrrdauLb+AtK4yOq0Ystu4k= 2022-09-03T07:30:44.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:De+ZjrrdauLb+AtK4yOq0Ystu4k=', 'Content-Length': '0'} 2022-09-03T07:30:44.651 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007253d87df406cbef-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.652 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.653 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.654 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:44.654 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:44.655 DEBUG:boto:Method: GET 2022-09-03T07:30:44.655 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:30:44.656 DEBUG:boto:Data: 2022-09-03T07:30:44.657 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.658 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.658 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.659 DEBUG:boto:Params: {} 2022-09-03T07:30:44.660 DEBUG:boto:Token: None 2022-09-03T07:30:44.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.665 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006ebb32494cd9df92-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.666 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.667 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.668 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.669 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:44.669 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:44.670 DEBUG:boto:Method: GET 2022-09-03T07:30:44.671 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:30:44.672 DEBUG:boto:Data: 2022-09-03T07:30:44.672 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.673 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.673 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.674 DEBUG:boto:Params: {} 2022-09-03T07:30:44.675 DEBUG:boto:Token: None 2022-09-03T07:30:44.676 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.686 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000dc30aa248d24897-00631302a4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.688 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.689 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.690 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.691 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.692 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:30:44.693 DEBUG:boto:Data: 2022-09-03T07:30:44.694 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.695 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.695 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.696 DEBUG:boto:Params: {} 2022-09-03T07:30:44.697 DEBUG:boto:Token: None 2022-09-03T07:30:44.697 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17 2022-09-03T07:30:44.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5q6lXA3G7XlRlE6OG0WlIiUG7k= 2022-09-03T07:30:44.698 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5q6lXA3G7XlRlE6OG0WlIiUG7k=', 'Content-Length': '0'} 2022-09-03T07:30:44.700 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0ef12f7fbd42cf5-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.701 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.702 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.702 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.703 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:30:44.703 DEBUG:boto:Data: 2022-09-03T07:30:44.704 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.705 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.705 DEBUG:boto:Params: {} 2022-09-03T07:30:44.706 DEBUG:boto:Token: None 2022-09-03T07:30:44.706 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17 2022-09-03T07:30:44.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5q6lXA3G7XlRlE6OG0WlIiUG7k= 2022-09-03T07:30:44.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5q6lXA3G7XlRlE6OG0WlIiUG7k=', 'Content-Length': '0'} 2022-09-03T07:30:44.712 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000816fd6a2252b4123-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.713 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:30:44.713 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.714 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.715 DEBUG:boto:Method: GET 2022-09-03T07:30:44.715 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:30:44.716 DEBUG:boto:Data: 2022-09-03T07:30:44.717 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.717 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.718 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.719 DEBUG:boto:Params: {} 2022-09-03T07:30:44.719 DEBUG:boto:Token: None 2022-09-03T07:30:44.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17 2022-09-03T07:30:44.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uqcG/hGO9GLeF8+wAudH++FP32k= 2022-09-03T07:30:44.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uqcG/hGO9GLeF8+wAudH++FP32k=', 'Content-Length': '0'} 2022-09-03T07:30:44.724 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071db49d4a02cd1f4-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.770 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:44.772 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:44.773 DEBUG:boto:Method: GET 2022-09-03T07:30:44.774 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:30:44.775 DEBUG:boto:Data: 2022-09-03T07:30:44.776 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.777 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.778 DEBUG:boto:Params: {} 2022-09-03T07:30:44.779 DEBUG:boto:Token: None 2022-09-03T07:30:44.780 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj17 2022-09-03T07:30:44.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uqcG/hGO9GLeF8+wAudH++FP32k= 2022-09-03T07:30:44.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uqcG/hGO9GLeF8+wAudH++FP32k=', 'Content-Length': '0'} 2022-09-03T07:30:44.787 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098266f55261756d3-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.788 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.788 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.789 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:44.791 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:44.792 DEBUG:boto:Method: GET 2022-09-03T07:30:44.793 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:30:44.793 DEBUG:boto:Data: 2022-09-03T07:30:44.795 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.795 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.796 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.797 DEBUG:boto:Params: {} 2022-09-03T07:30:44.798 DEBUG:boto:Token: None 2022-09-03T07:30:44.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.804 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000995176e5816637a5-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.805 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.806 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.807 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.809 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:44.810 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:44.811 DEBUG:boto:Method: GET 2022-09-03T07:30:44.811 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:30:44.812 DEBUG:boto:Data: 2022-09-03T07:30:44.813 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.814 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.815 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.816 DEBUG:boto:Params: {} 2022-09-03T07:30:44.817 DEBUG:boto:Token: None 2022-09-03T07:30:44.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.826 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e20450bd8a94093f-00631302a4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.827 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.828 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj185 2022-09-03T07:30:44.830 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:44.831 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:44.831 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.832 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.833 DEBUG:boto:Method: GET 2022-09-03T07:30:44.834 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.835 DEBUG:boto:Data: 2022-09-03T07:30:44.836 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.837 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.838 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.838 DEBUG:boto:Params: {} 2022-09-03T07:30:44.839 DEBUG:boto:Token: None 2022-09-03T07:30:44.840 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hosU1yvIPxygXcBgfwhd1hSzf3A= 2022-09-03T07:30:44.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hosU1yvIPxygXcBgfwhd1hSzf3A=', 'Content-Length': '0'} 2022-09-03T07:30:44.844 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000562d0d75f6eaa519-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.845 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:44.846 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:44.847 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.848 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.848 DEBUG:boto:Method: GET 2022-09-03T07:30:44.849 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.850 DEBUG:boto:Data: 2022-09-03T07:30:44.851 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.851 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.852 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.853 DEBUG:boto:Params: {} 2022-09-03T07:30:44.854 DEBUG:boto:Token: None 2022-09-03T07:30:44.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hosU1yvIPxygXcBgfwhd1hSzf3A= 2022-09-03T07:30:44.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hosU1yvIPxygXcBgfwhd1hSzf3A=', 'Content-Length': '0'} 2022-09-03T07:30:44.862 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006661414a5c876ea5-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.864 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:44.865 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:44.866 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.868 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.869 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.870 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.872 DEBUG:boto:Data: 2022-09-03T07:30:44.873 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.874 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.875 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.876 DEBUG:boto:Params: {} 2022-09-03T07:30:44.877 DEBUG:boto:Token: None 2022-09-03T07:30:44.879 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EZEHtlFefPxlsCiV9gy/1G0uvrw= 2022-09-03T07:30:44.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EZEHtlFefPxlsCiV9gy/1G0uvrw=', 'Content-Length': '0'} 2022-09-03T07:30:44.884 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2afa385a775ed46-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.885 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:44.885 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:44.886 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.886 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.887 DEBUG:boto:Method: HEAD 2022-09-03T07:30:44.887 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.888 DEBUG:boto:Data: 2022-09-03T07:30:44.888 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.889 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.889 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.890 DEBUG:boto:Params: {} 2022-09-03T07:30:44.890 DEBUG:boto:Token: None 2022-09-03T07:30:44.891 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EZEHtlFefPxlsCiV9gy/1G0uvrw= 2022-09-03T07:30:44.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EZEHtlFefPxlsCiV9gy/1G0uvrw=', 'Content-Length': '0'} 2022-09-03T07:30:44.894 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef5b5a13ef840d6f-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.895 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj185 2022-09-03T07:30:44.895 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:44.896 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:44.896 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.897 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.898 DEBUG:boto:Method: GET 2022-09-03T07:30:44.898 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.899 DEBUG:boto:Data: 2022-09-03T07:30:44.899 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.900 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.900 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.901 DEBUG:boto:Params: {} 2022-09-03T07:30:44.902 DEBUG:boto:Token: None 2022-09-03T07:30:44.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.903 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hosU1yvIPxygXcBgfwhd1hSzf3A= 2022-09-03T07:30:44.904 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hosU1yvIPxygXcBgfwhd1hSzf3A=', 'Content-Length': '0'} 2022-09-03T07:30:44.908 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000020dd35dab293be-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.909 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:44.911 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:44.912 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.913 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.913 DEBUG:boto:Method: GET 2022-09-03T07:30:44.914 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.915 DEBUG:boto:Data: 2022-09-03T07:30:44.916 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.917 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.918 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.919 DEBUG:boto:Params: {} 2022-09-03T07:30:44.920 DEBUG:boto:Token: None 2022-09-03T07:30:44.921 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:44.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hosU1yvIPxygXcBgfwhd1hSzf3A= 2022-09-03T07:30:44.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hosU1yvIPxygXcBgfwhd1hSzf3A=', 'Content-Length': '0'} 2022-09-03T07:30:44.926 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e490422be3a92d6-00631302a4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.927 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.928 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.928 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:44.929 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:44.929 DEBUG:boto:Method: GET 2022-09-03T07:30:44.930 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:30:44.931 DEBUG:boto:Data: 2022-09-03T07:30:44.931 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.932 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:44.933 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.933 DEBUG:boto:Params: {} 2022-09-03T07:30:44.934 DEBUG:boto:Token: None 2022-09-03T07:30:44.935 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.940 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000486141fd30e2daf8-00631302a4-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:44.941 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:44.942 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:44.943 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:44.943 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:44.944 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:44.944 DEBUG:boto:Method: GET 2022-09-03T07:30:44.945 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:30:44.946 DEBUG:boto:Data: 2022-09-03T07:30:44.947 DEBUG:boto:Headers: {} 2022-09-03T07:30:44.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:44.949 DEBUG:boto:Port: 8000 2022-09-03T07:30:44.950 DEBUG:boto:Params: {} 2022-09-03T07:30:44.951 DEBUG:boto:Token: None 2022-09-03T07:30:44.952 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:44 GMT /avysmw-75/ 2022-09-03T07:30:44.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg= 2022-09-03T07:30:44.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:44 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4P6zaBXGmaOi/7Le7b7s6sMU7kg=', 'Content-Length': '0'} 2022-09-03T07:30:44.959 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e9d839ae3e08784b-00631302a4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:44 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.002 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.003 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:45.004 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:45.005 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.005 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:30:45.006 DEBUG:boto:Data: 2022-09-03T07:30:45.006 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.007 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.008 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.008 DEBUG:boto:Params: {} 2022-09-03T07:30:45.009 DEBUG:boto:Token: None 2022-09-03T07:30:45.009 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj185 2022-09-03T07:30:45.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QtVqR1LcJlGwEup86dIDPPbJR3Q= 2022-09-03T07:30:45.011 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QtVqR1LcJlGwEup86dIDPPbJR3Q=', 'Content-Length': '0'} 2022-09-03T07:30:45.013 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e49316a95731f2e-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.013 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:45.014 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:45.014 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.015 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:30:45.015 DEBUG:boto:Data: 2022-09-03T07:30:45.016 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.018 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.018 DEBUG:boto:Params: {} 2022-09-03T07:30:45.020 DEBUG:boto:Token: None 2022-09-03T07:30:45.020 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj185 2022-09-03T07:30:45.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QtVqR1LcJlGwEup86dIDPPbJR3Q= 2022-09-03T07:30:45.022 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QtVqR1LcJlGwEup86dIDPPbJR3Q=', 'Content-Length': '0'} 2022-09-03T07:30:45.025 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063104fa67b7ae77c-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.027 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj185 2022-09-03T07:30:45.027 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:45.028 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:45.029 DEBUG:boto:Method: GET 2022-09-03T07:30:45.030 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:30:45.030 DEBUG:boto:Data: 2022-09-03T07:30:45.031 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.032 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.032 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.033 DEBUG:boto:Params: {} 2022-09-03T07:30:45.033 DEBUG:boto:Token: None 2022-09-03T07:30:45.034 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj185 2022-09-03T07:30:45.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WUhE93lW3SFBX5ETSU3NCbaaiBw= 2022-09-03T07:30:45.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WUhE93lW3SFBX5ETSU3NCbaaiBw=', 'Content-Length': '0'} 2022-09-03T07:30:45.038 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d6749c79a9e74b0-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.039 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:45.039 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:45.040 DEBUG:boto:Method: GET 2022-09-03T07:30:45.042 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:30:45.043 DEBUG:boto:Data: 2022-09-03T07:30:45.044 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.046 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.047 DEBUG:boto:Params: {} 2022-09-03T07:30:45.048 DEBUG:boto:Token: None 2022-09-03T07:30:45.049 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj185 2022-09-03T07:30:45.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WUhE93lW3SFBX5ETSU3NCbaaiBw= 2022-09-03T07:30:45.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WUhE93lW3SFBX5ETSU3NCbaaiBw=', 'Content-Length': '0'} 2022-09-03T07:30:45.055 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000063358b2b13b2e96f-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.057 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.058 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.059 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:45.060 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:45.061 DEBUG:boto:Method: GET 2022-09-03T07:30:45.061 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:30:45.062 DEBUG:boto:Data: 2022-09-03T07:30:45.063 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.064 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.065 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.066 DEBUG:boto:Params: {} 2022-09-03T07:30:45.067 DEBUG:boto:Token: None 2022-09-03T07:30:45.068 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.073 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f7ff76d369c071e-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.074 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.076 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.077 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.077 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:45.078 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:45.079 DEBUG:boto:Method: GET 2022-09-03T07:30:45.080 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:30:45.081 DEBUG:boto:Data: 2022-09-03T07:30:45.082 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.083 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.083 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.084 DEBUG:boto:Params: {} 2022-09-03T07:30:45.084 DEBUG:boto:Token: None 2022-09-03T07:30:45.085 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.086 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.091 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce29b730f0d06b1f-00631302a5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.092 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.093 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:30:45.094 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.094 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.095 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.095 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.096 DEBUG:boto:Method: GET 2022-09-03T07:30:45.096 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.097 DEBUG:boto:Data: 2022-09-03T07:30:45.098 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.098 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.099 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.099 DEBUG:boto:Params: {} 2022-09-03T07:30:45.100 DEBUG:boto:Token: None 2022-09-03T07:30:45.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/pYPke5UUIPTCIj2n1XNWVN8ddg= 2022-09-03T07:30:45.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/pYPke5UUIPTCIj2n1XNWVN8ddg=', 'Content-Length': '0'} 2022-09-03T07:30:45.104 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a13c491ff4daf42d-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.104 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.105 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.106 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.106 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.107 DEBUG:boto:Method: GET 2022-09-03T07:30:45.107 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.108 DEBUG:boto:Data: 2022-09-03T07:30:45.108 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.110 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.110 DEBUG:boto:Params: {} 2022-09-03T07:30:45.111 DEBUG:boto:Token: None 2022-09-03T07:30:45.111 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/pYPke5UUIPTCIj2n1XNWVN8ddg= 2022-09-03T07:30:45.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/pYPke5UUIPTCIj2n1XNWVN8ddg=', 'Content-Length': '0'} 2022-09-03T07:30:45.116 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d89547857ec36bbc-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.162 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.163 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.163 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.164 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.164 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.165 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.165 DEBUG:boto:Data: 2022-09-03T07:30:45.166 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.166 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.167 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.167 DEBUG:boto:Params: {} 2022-09-03T07:30:45.168 DEBUG:boto:Token: None 2022-09-03T07:30:45.169 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.169 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wIzt/EoKz2G9WCoQJbUXVxoZioQ= 2022-09-03T07:30:45.170 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wIzt/EoKz2G9WCoQJbUXVxoZioQ=', 'Content-Length': '0'} 2022-09-03T07:30:45.173 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f59027d940139f64-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.174 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.175 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.175 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.176 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.176 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.177 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.177 DEBUG:boto:Data: 2022-09-03T07:30:45.178 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.178 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.179 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.180 DEBUG:boto:Params: {} 2022-09-03T07:30:45.180 DEBUG:boto:Token: None 2022-09-03T07:30:45.181 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wIzt/EoKz2G9WCoQJbUXVxoZioQ= 2022-09-03T07:30:45.183 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wIzt/EoKz2G9WCoQJbUXVxoZioQ=', 'Content-Length': '0'} 2022-09-03T07:30:45.185 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000091e8a9f83c3dac8a-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.186 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:30:45.187 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.187 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.188 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.189 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.189 DEBUG:boto:Method: GET 2022-09-03T07:30:45.190 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.191 DEBUG:boto:Data: 2022-09-03T07:30:45.192 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.193 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.194 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.195 DEBUG:boto:Params: {} 2022-09-03T07:30:45.196 DEBUG:boto:Token: None 2022-09-03T07:30:45.197 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/pYPke5UUIPTCIj2n1XNWVN8ddg= 2022-09-03T07:30:45.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/pYPke5UUIPTCIj2n1XNWVN8ddg=', 'Content-Length': '0'} 2022-09-03T07:30:45.203 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007a968568f5c91156-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.204 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.204 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.205 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.205 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.206 DEBUG:boto:Method: GET 2022-09-03T07:30:45.207 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.208 DEBUG:boto:Data: 2022-09-03T07:30:45.209 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.210 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.211 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.211 DEBUG:boto:Params: {} 2022-09-03T07:30:45.212 DEBUG:boto:Token: None 2022-09-03T07:30:45.213 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:45.213 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/pYPke5UUIPTCIj2n1XNWVN8ddg= 2022-09-03T07:30:45.214 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/pYPke5UUIPTCIj2n1XNWVN8ddg=', 'Content-Length': '0'} 2022-09-03T07:30:45.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002eb5d6e48d89070-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.217 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.218 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.219 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:45.219 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:45.220 DEBUG:boto:Method: GET 2022-09-03T07:30:45.220 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:30:45.221 DEBUG:boto:Data: 2022-09-03T07:30:45.221 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.222 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.222 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.223 DEBUG:boto:Params: {} 2022-09-03T07:30:45.223 DEBUG:boto:Token: None 2022-09-03T07:30:45.224 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.227 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003e0379b8b44b9d62-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.228 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.229 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.230 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.230 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:45.231 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:45.231 DEBUG:boto:Method: GET 2022-09-03T07:30:45.232 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:30:45.232 DEBUG:boto:Data: 2022-09-03T07:30:45.233 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.233 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.234 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.234 DEBUG:boto:Params: {} 2022-09-03T07:30:45.235 DEBUG:boto:Token: None 2022-09-03T07:30:45.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.244 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d0a66c487987ccf7-00631302a5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.245 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.245 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.246 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.247 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.247 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:30:45.248 DEBUG:boto:Data: 2022-09-03T07:30:45.248 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.249 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.249 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.250 DEBUG:boto:Params: {} 2022-09-03T07:30:45.251 DEBUG:boto:Token: None 2022-09-03T07:30:45.252 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19 2022-09-03T07:30:45.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kg5qI9FHvKW3mKzYtEqKP56oZhs= 2022-09-03T07:30:45.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kg5qI9FHvKW3mKzYtEqKP56oZhs=', 'Content-Length': '0'} 2022-09-03T07:30:45.257 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004991f22d731e9bb6-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.259 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.259 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.260 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.261 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:30:45.262 DEBUG:boto:Data: 2022-09-03T07:30:45.263 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.265 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.266 DEBUG:boto:Params: {} 2022-09-03T07:30:45.267 DEBUG:boto:Token: None 2022-09-03T07:30:45.268 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19 2022-09-03T07:30:45.269 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kg5qI9FHvKW3mKzYtEqKP56oZhs= 2022-09-03T07:30:45.271 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kg5qI9FHvKW3mKzYtEqKP56oZhs=', 'Content-Length': '0'} 2022-09-03T07:30:45.273 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b27d4f7ab7e7aed4-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.275 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:30:45.275 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.276 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.277 DEBUG:boto:Method: GET 2022-09-03T07:30:45.278 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:30:45.278 DEBUG:boto:Data: 2022-09-03T07:30:45.279 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.279 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.280 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.281 DEBUG:boto:Params: {} 2022-09-03T07:30:45.281 DEBUG:boto:Token: None 2022-09-03T07:30:45.283 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19 2022-09-03T07:30:45.284 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PDDxiaBsU1c58EgJf4xIXiUPsKA= 2022-09-03T07:30:45.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PDDxiaBsU1c58EgJf4xIXiUPsKA=', 'Content-Length': '0'} 2022-09-03T07:30:45.287 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d13292e459ccca9f-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.289 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:45.290 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:45.291 DEBUG:boto:Method: GET 2022-09-03T07:30:45.292 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:30:45.293 DEBUG:boto:Data: 2022-09-03T07:30:45.294 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.295 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.296 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.297 DEBUG:boto:Params: {} 2022-09-03T07:30:45.298 DEBUG:boto:Token: None 2022-09-03T07:30:45.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj19 2022-09-03T07:30:45.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PDDxiaBsU1c58EgJf4xIXiUPsKA= 2022-09-03T07:30:45.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PDDxiaBsU1c58EgJf4xIXiUPsKA=', 'Content-Length': '0'} 2022-09-03T07:30:45.302 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076ff3127fbed6fdd-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.303 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.304 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.305 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:45.305 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:45.306 DEBUG:boto:Method: GET 2022-09-03T07:30:45.306 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:30:45.307 DEBUG:boto:Data: 2022-09-03T07:30:45.308 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.308 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.309 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.309 DEBUG:boto:Params: {} 2022-09-03T07:30:45.310 DEBUG:boto:Token: None 2022-09-03T07:30:45.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.314 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036db8b42b3ba45ef-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.315 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.316 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.316 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.317 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:45.317 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:45.318 DEBUG:boto:Method: GET 2022-09-03T07:30:45.318 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:30:45.319 DEBUG:boto:Data: 2022-09-03T07:30:45.319 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.320 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.320 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.321 DEBUG:boto:Params: {} 2022-09-03T07:30:45.322 DEBUG:boto:Token: None 2022-09-03T07:30:45.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.328 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb3888602c23ecb0-00631302a5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.374 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.376 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:30:45.376 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.377 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.378 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.379 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.379 DEBUG:boto:Method: GET 2022-09-03T07:30:45.380 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.380 DEBUG:boto:Data: 2022-09-03T07:30:45.381 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.382 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.382 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.383 DEBUG:boto:Params: {} 2022-09-03T07:30:45.384 DEBUG:boto:Token: None 2022-09-03T07:30:45.384 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w2gQlLsgX+wnyB0Jzj9X3/Y8zfE= 2022-09-03T07:30:45.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w2gQlLsgX+wnyB0Jzj9X3/Y8zfE=', 'Content-Length': '0'} 2022-09-03T07:30:45.388 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000f7af16863162a5-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.390 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.391 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.392 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.392 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.393 DEBUG:boto:Method: GET 2022-09-03T07:30:45.394 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.394 DEBUG:boto:Data: 2022-09-03T07:30:45.395 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.396 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.397 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.397 DEBUG:boto:Params: {} 2022-09-03T07:30:45.398 DEBUG:boto:Token: None 2022-09-03T07:30:45.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w2gQlLsgX+wnyB0Jzj9X3/Y8zfE= 2022-09-03T07:30:45.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w2gQlLsgX+wnyB0Jzj9X3/Y8zfE=', 'Content-Length': '0'} 2022-09-03T07:30:45.403 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031db271d0e5ddbd1-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.404 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.404 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.405 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.405 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.406 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.407 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.407 DEBUG:boto:Data: 2022-09-03T07:30:45.408 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.408 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.409 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.409 DEBUG:boto:Params: {} 2022-09-03T07:30:45.411 DEBUG:boto:Token: None 2022-09-03T07:30:45.411 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.412 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KjP3Q1ZhrUeka1KNT/Q8xb5V+m0= 2022-09-03T07:30:45.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KjP3Q1ZhrUeka1KNT/Q8xb5V+m0=', 'Content-Length': '0'} 2022-09-03T07:30:45.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008cb8a89eabac3533-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.416 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.417 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.417 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.418 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.418 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.419 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.419 DEBUG:boto:Data: 2022-09-03T07:30:45.420 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.420 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.421 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.421 DEBUG:boto:Params: {} 2022-09-03T07:30:45.422 DEBUG:boto:Token: None 2022-09-03T07:30:45.423 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.423 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KjP3Q1ZhrUeka1KNT/Q8xb5V+m0= 2022-09-03T07:30:45.424 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KjP3Q1ZhrUeka1KNT/Q8xb5V+m0=', 'Content-Length': '0'} 2022-09-03T07:30:45.427 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d005ab3a8845d446-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.427 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:30:45.428 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.429 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.429 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.430 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.430 DEBUG:boto:Method: GET 2022-09-03T07:30:45.431 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.431 DEBUG:boto:Data: 2022-09-03T07:30:45.432 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.432 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.433 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.433 DEBUG:boto:Params: {} 2022-09-03T07:30:45.434 DEBUG:boto:Token: None 2022-09-03T07:30:45.435 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.436 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w2gQlLsgX+wnyB0Jzj9X3/Y8zfE= 2022-09-03T07:30:45.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w2gQlLsgX+wnyB0Jzj9X3/Y8zfE=', 'Content-Length': '0'} 2022-09-03T07:30:45.438 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d0d6971aad5da3dc-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.439 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.440 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.441 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.441 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.442 DEBUG:boto:Method: GET 2022-09-03T07:30:45.442 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.443 DEBUG:boto:Data: 2022-09-03T07:30:45.444 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.445 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.447 DEBUG:boto:Params: {} 2022-09-03T07:30:45.447 DEBUG:boto:Token: None 2022-09-03T07:30:45.448 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:45.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w2gQlLsgX+wnyB0Jzj9X3/Y8zfE= 2022-09-03T07:30:45.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w2gQlLsgX+wnyB0Jzj9X3/Y8zfE=', 'Content-Length': '0'} 2022-09-03T07:30:45.454 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef115ad07c80b920-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.502 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.503 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.504 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:45.505 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:45.506 DEBUG:boto:Method: GET 2022-09-03T07:30:45.507 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:30:45.508 DEBUG:boto:Data: 2022-09-03T07:30:45.509 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.510 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.511 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.512 DEBUG:boto:Params: {} 2022-09-03T07:30:45.513 DEBUG:boto:Token: None 2022-09-03T07:30:45.514 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.518 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c3aa6335c0086727-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.518 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.519 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.520 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.520 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:45.521 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:45.521 DEBUG:boto:Method: GET 2022-09-03T07:30:45.522 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:30:45.523 DEBUG:boto:Data: 2022-09-03T07:30:45.523 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.524 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.525 DEBUG:boto:Params: {} 2022-09-03T07:30:45.526 DEBUG:boto:Token: None 2022-09-03T07:30:45.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.533 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c8768c089bf03211-00631302a5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.533 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.534 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.535 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.535 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.536 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:30:45.536 DEBUG:boto:Data: 2022-09-03T07:30:45.537 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.537 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.538 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.538 DEBUG:boto:Params: {} 2022-09-03T07:30:45.539 DEBUG:boto:Token: None 2022-09-03T07:30:45.540 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22 2022-09-03T07:30:45.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVEiy1fepvbz8WxRaj40Ks/tF6A= 2022-09-03T07:30:45.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVEiy1fepvbz8WxRaj40Ks/tF6A=', 'Content-Length': '0'} 2022-09-03T07:30:45.543 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e71510ca13b1f78e-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.544 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.544 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.545 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.546 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:30:45.546 DEBUG:boto:Data: 2022-09-03T07:30:45.547 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.548 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.548 DEBUG:boto:Params: {} 2022-09-03T07:30:45.549 DEBUG:boto:Token: None 2022-09-03T07:30:45.550 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22 2022-09-03T07:30:45.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oVEiy1fepvbz8WxRaj40Ks/tF6A= 2022-09-03T07:30:45.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oVEiy1fepvbz8WxRaj40Ks/tF6A=', 'Content-Length': '0'} 2022-09-03T07:30:45.553 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f3b1af945e5e77e-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.553 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:30:45.554 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.555 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.555 DEBUG:boto:Method: GET 2022-09-03T07:30:45.556 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:30:45.556 DEBUG:boto:Data: 2022-09-03T07:30:45.557 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.557 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.558 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.558 DEBUG:boto:Params: {} 2022-09-03T07:30:45.559 DEBUG:boto:Token: None 2022-09-03T07:30:45.560 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22 2022-09-03T07:30:45.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TluaMUxVJKeMH/4TrtXcugN0X3A= 2022-09-03T07:30:45.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TluaMUxVJKeMH/4TrtXcugN0X3A=', 'Content-Length': '0'} 2022-09-03T07:30:45.563 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000041099b24372b8db6-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.563 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:45.564 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:45.565 DEBUG:boto:Method: GET 2022-09-03T07:30:45.565 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:30:45.566 DEBUG:boto:Data: 2022-09-03T07:30:45.566 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.567 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.568 DEBUG:boto:Params: {} 2022-09-03T07:30:45.568 DEBUG:boto:Token: None 2022-09-03T07:30:45.569 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj22 2022-09-03T07:30:45.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TluaMUxVJKeMH/4TrtXcugN0X3A= 2022-09-03T07:30:45.570 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TluaMUxVJKeMH/4TrtXcugN0X3A=', 'Content-Length': '0'} 2022-09-03T07:30:45.572 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cab8cbf6a9cd792e-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.573 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.574 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.575 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:45.576 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:45.577 DEBUG:boto:Method: GET 2022-09-03T07:30:45.577 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:30:45.578 DEBUG:boto:Data: 2022-09-03T07:30:45.578 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.579 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.579 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.580 DEBUG:boto:Params: {} 2022-09-03T07:30:45.581 DEBUG:boto:Token: None 2022-09-03T07:30:45.581 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.585 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a25eb23364924f7a-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.626 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.628 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.629 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.630 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:45.630 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:45.631 DEBUG:boto:Method: GET 2022-09-03T07:30:45.632 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:30:45.633 DEBUG:boto:Data: 2022-09-03T07:30:45.634 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.637 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.638 DEBUG:boto:Params: {} 2022-09-03T07:30:45.639 DEBUG:boto:Token: None 2022-09-03T07:30:45.640 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.648 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000662af94785fd5635-00631302a5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.650 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.650 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:30:45.651 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.652 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.652 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.653 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.653 DEBUG:boto:Method: GET 2022-09-03T07:30:45.654 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.654 DEBUG:boto:Data: 2022-09-03T07:30:45.655 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.655 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.656 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.656 DEBUG:boto:Params: {} 2022-09-03T07:30:45.657 DEBUG:boto:Token: None 2022-09-03T07:30:45.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0TJ6jTFWuLEkQMveEvoNtpw4e0A= 2022-09-03T07:30:45.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0TJ6jTFWuLEkQMveEvoNtpw4e0A=', 'Content-Length': '0'} 2022-09-03T07:30:45.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001abe0e96fc67d9ba-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.663 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.664 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.665 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.666 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.667 DEBUG:boto:Method: GET 2022-09-03T07:30:45.668 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.669 DEBUG:boto:Data: 2022-09-03T07:30:45.670 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.671 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.672 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.673 DEBUG:boto:Params: {} 2022-09-03T07:30:45.674 DEBUG:boto:Token: None 2022-09-03T07:30:45.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0TJ6jTFWuLEkQMveEvoNtpw4e0A= 2022-09-03T07:30:45.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0TJ6jTFWuLEkQMveEvoNtpw4e0A=', 'Content-Length': '0'} 2022-09-03T07:30:45.680 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5242bcd3ee24d6e-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.722 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.723 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.724 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.725 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.726 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.727 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.727 DEBUG:boto:Data: 2022-09-03T07:30:45.728 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.729 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.730 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.732 DEBUG:boto:Params: {} 2022-09-03T07:30:45.733 DEBUG:boto:Token: None 2022-09-03T07:30:45.734 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XmkmovVMpPAXshMcdmr19gvaOgo= 2022-09-03T07:30:45.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XmkmovVMpPAXshMcdmr19gvaOgo=', 'Content-Length': '0'} 2022-09-03T07:30:45.740 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe4a97042cdacbc6-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.743 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.744 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.744 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.745 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.745 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.746 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.747 DEBUG:boto:Data: 2022-09-03T07:30:45.747 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.748 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.749 DEBUG:boto:Params: {} 2022-09-03T07:30:45.751 DEBUG:boto:Token: None 2022-09-03T07:30:45.752 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XmkmovVMpPAXshMcdmr19gvaOgo= 2022-09-03T07:30:45.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XmkmovVMpPAXshMcdmr19gvaOgo=', 'Content-Length': '0'} 2022-09-03T07:30:45.757 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c59fb03757587018-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.759 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:30:45.760 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.761 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.762 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.763 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.763 DEBUG:boto:Method: GET 2022-09-03T07:30:45.764 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.765 DEBUG:boto:Data: 2022-09-03T07:30:45.766 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.766 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.767 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.768 DEBUG:boto:Params: {} 2022-09-03T07:30:45.769 DEBUG:boto:Token: None 2022-09-03T07:30:45.770 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0TJ6jTFWuLEkQMveEvoNtpw4e0A= 2022-09-03T07:30:45.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0TJ6jTFWuLEkQMveEvoNtpw4e0A=', 'Content-Length': '0'} 2022-09-03T07:30:45.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a310fcfad460045c-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.775 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.775 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.776 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.777 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.777 DEBUG:boto:Method: GET 2022-09-03T07:30:45.778 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.778 DEBUG:boto:Data: 2022-09-03T07:30:45.779 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.780 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.781 DEBUG:boto:Params: {} 2022-09-03T07:30:45.782 DEBUG:boto:Token: None 2022-09-03T07:30:45.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:45.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0TJ6jTFWuLEkQMveEvoNtpw4e0A= 2022-09-03T07:30:45.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0TJ6jTFWuLEkQMveEvoNtpw4e0A=', 'Content-Length': '0'} 2022-09-03T07:30:45.786 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a7d9a296e332fe9-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.787 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.788 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.789 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:45.789 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:45.790 DEBUG:boto:Method: GET 2022-09-03T07:30:45.790 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:30:45.791 DEBUG:boto:Data: 2022-09-03T07:30:45.792 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.792 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.793 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.794 DEBUG:boto:Params: {} 2022-09-03T07:30:45.794 DEBUG:boto:Token: None 2022-09-03T07:30:45.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.801 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d24d2680451d2ab-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.802 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.803 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.803 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.804 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:45.805 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:45.805 DEBUG:boto:Method: GET 2022-09-03T07:30:45.806 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:30:45.807 DEBUG:boto:Data: 2022-09-03T07:30:45.807 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.809 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.809 DEBUG:boto:Params: {} 2022-09-03T07:30:45.810 DEBUG:boto:Token: None 2022-09-03T07:30:45.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.818 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005a714f2ca0464d60-00631302a5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.820 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.821 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.823 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.825 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.826 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:30:45.828 DEBUG:boto:Data: 2022-09-03T07:30:45.828 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.831 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.832 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.833 DEBUG:boto:Params: {} 2022-09-03T07:30:45.835 DEBUG:boto:Token: None 2022-09-03T07:30:45.836 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23 2022-09-03T07:30:45.838 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UTLMbQ4zaLSKgl2sA7nBsSpN8S8= 2022-09-03T07:30:45.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UTLMbQ4zaLSKgl2sA7nBsSpN8S8=', 'Content-Length': '0'} 2022-09-03T07:30:45.842 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000643dcdb81a244e40-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.843 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.844 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.844 DEBUG:boto:Method: HEAD 2022-09-03T07:30:45.845 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:30:45.845 DEBUG:boto:Data: 2022-09-03T07:30:45.846 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.846 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.847 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.847 DEBUG:boto:Params: {} 2022-09-03T07:30:45.848 DEBUG:boto:Token: None 2022-09-03T07:30:45.849 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23 2022-09-03T07:30:45.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UTLMbQ4zaLSKgl2sA7nBsSpN8S8= 2022-09-03T07:30:45.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UTLMbQ4zaLSKgl2sA7nBsSpN8S8=', 'Content-Length': '0'} 2022-09-03T07:30:45.854 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000336c3d26409518ac-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.855 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:30:45.856 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.857 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.858 DEBUG:boto:Method: GET 2022-09-03T07:30:45.859 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:30:45.860 DEBUG:boto:Data: 2022-09-03T07:30:45.861 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.862 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.863 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.865 DEBUG:boto:Params: {} 2022-09-03T07:30:45.866 DEBUG:boto:Token: None 2022-09-03T07:30:45.867 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23 2022-09-03T07:30:45.868 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ibNVB6NlBKx1nIkgaRxQ5a+m//Q= 2022-09-03T07:30:45.869 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ibNVB6NlBKx1nIkgaRxQ5a+m//Q=', 'Content-Length': '0'} 2022-09-03T07:30:45.872 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ec01d7a9a9e7d7b6-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.874 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:45.876 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:45.877 DEBUG:boto:Method: GET 2022-09-03T07:30:45.878 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:30:45.879 DEBUG:boto:Data: 2022-09-03T07:30:45.879 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.881 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.881 DEBUG:boto:Params: {} 2022-09-03T07:30:45.882 DEBUG:boto:Token: None 2022-09-03T07:30:45.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/obj23 2022-09-03T07:30:45.883 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ibNVB6NlBKx1nIkgaRxQ5a+m//Q= 2022-09-03T07:30:45.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ibNVB6NlBKx1nIkgaRxQ5a+m//Q=', 'Content-Length': '0'} 2022-09-03T07:30:45.886 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d393d6c64fa71264-00631302a5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.949 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.950 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.951 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:45.952 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:45.953 DEBUG:boto:Method: GET 2022-09-03T07:30:45.954 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:30:45.955 DEBUG:boto:Data: 2022-09-03T07:30:45.957 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.958 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:45.960 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.961 DEBUG:boto:Params: {} 2022-09-03T07:30:45.962 DEBUG:boto:Token: None 2022-09-03T07:30:45.963 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.965 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.970 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ec1a4ee135fbb45a-00631302a5-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.971 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.972 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:45.973 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:45.975 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:45.976 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:45.977 DEBUG:boto:Method: GET 2022-09-03T07:30:45.978 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:30:45.979 DEBUG:boto:Data: 2022-09-03T07:30:45.980 DEBUG:boto:Headers: {} 2022-09-03T07:30:45.981 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:45.983 DEBUG:boto:Port: 8000 2022-09-03T07:30:45.984 DEBUG:boto:Params: {} 2022-09-03T07:30:45.985 DEBUG:boto:Token: None 2022-09-03T07:30:45.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:45 GMT /avysmw-75/ 2022-09-03T07:30:45.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60= 2022-09-03T07:30:45.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:45 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zucmfw4PYnqV81KDuUFLr0Oxk60=', 'Content-Length': '0'} 2022-09-03T07:30:45.994 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004cd637ed548c0890-00631302a5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:45 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:45.995 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:45.996 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:30:45.997 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:45.998 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:45.999 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:45.999 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.000 DEBUG:boto:Method: GET 2022-09-03T07:30:46.000 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.001 DEBUG:boto:Data: 2022-09-03T07:30:46.002 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.003 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.004 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.005 DEBUG:boto:Params: {} 2022-09-03T07:30:46.006 DEBUG:boto:Token: None 2022-09-03T07:30:46.007 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.008 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:igEOzNKhzReZ6HMlIVo/EPwFU+E= 2022-09-03T07:30:46.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:igEOzNKhzReZ6HMlIVo/EPwFU+E=', 'Content-Length': '0'} 2022-09-03T07:30:46.015 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b5a64db49beeea9-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.059 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:46.060 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:46.061 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.062 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.063 DEBUG:boto:Method: GET 2022-09-03T07:30:46.064 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.065 DEBUG:boto:Data: 2022-09-03T07:30:46.066 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.068 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.069 DEBUG:boto:Params: {} 2022-09-03T07:30:46.070 DEBUG:boto:Token: None 2022-09-03T07:30:46.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:igEOzNKhzReZ6HMlIVo/EPwFU+E= 2022-09-03T07:30:46.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:igEOzNKhzReZ6HMlIVo/EPwFU+E=', 'Content-Length': '0'} 2022-09-03T07:30:46.081 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009efc0c30ef1b07c8-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.083 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:46.084 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:46.084 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.085 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.086 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.087 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.088 DEBUG:boto:Data: 2022-09-03T07:30:46.089 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.090 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.091 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.092 DEBUG:boto:Params: {} 2022-09-03T07:30:46.093 DEBUG:boto:Token: None 2022-09-03T07:30:46.094 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0sh+94zzU5CVEmtfWFkHb90YJ/4= 2022-09-03T07:30:46.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0sh+94zzU5CVEmtfWFkHb90YJ/4=', 'Content-Length': '0'} 2022-09-03T07:30:46.099 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000279d960d9756f455-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.100 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:46.101 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:46.102 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.103 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.104 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.105 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.106 DEBUG:boto:Data: 2022-09-03T07:30:46.107 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.107 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.108 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.108 DEBUG:boto:Params: {} 2022-09-03T07:30:46.109 DEBUG:boto:Token: None 2022-09-03T07:30:46.110 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0sh+94zzU5CVEmtfWFkHb90YJ/4= 2022-09-03T07:30:46.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0sh+94zzU5CVEmtfWFkHb90YJ/4=', 'Content-Length': '0'} 2022-09-03T07:30:46.114 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000225ecce37240092a-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.115 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:30:46.116 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:46.117 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:46.118 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.118 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.119 DEBUG:boto:Method: GET 2022-09-03T07:30:46.120 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.120 DEBUG:boto:Data: 2022-09-03T07:30:46.121 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.121 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.122 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.123 DEBUG:boto:Params: {} 2022-09-03T07:30:46.124 DEBUG:boto:Token: None 2022-09-03T07:30:46.124 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:igEOzNKhzReZ6HMlIVo/EPwFU+E= 2022-09-03T07:30:46.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:igEOzNKhzReZ6HMlIVo/EPwFU+E=', 'Content-Length': '0'} 2022-09-03T07:30:46.128 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004259b862174abcfb-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.129 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:46.129 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:46.130 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.130 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.131 DEBUG:boto:Method: GET 2022-09-03T07:30:46.132 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.132 DEBUG:boto:Data: 2022-09-03T07:30:46.133 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.133 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.134 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.134 DEBUG:boto:Params: {} 2022-09-03T07:30:46.135 DEBUG:boto:Token: None 2022-09-03T07:30:46.135 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:46.136 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:igEOzNKhzReZ6HMlIVo/EPwFU+E= 2022-09-03T07:30:46.137 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:igEOzNKhzReZ6HMlIVo/EPwFU+E=', 'Content-Length': '0'} 2022-09-03T07:30:46.139 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a79e585f7b01233a-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.140 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.141 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.141 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:46.142 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:46.142 DEBUG:boto:Method: GET 2022-09-03T07:30:46.143 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:30:46.144 DEBUG:boto:Data: 2022-09-03T07:30:46.145 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.145 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.146 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.146 DEBUG:boto:Params: {} 2022-09-03T07:30:46.147 DEBUG:boto:Token: None 2022-09-03T07:30:46.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.152 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f63dc36248d9b0b0-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.153 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.154 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.155 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.156 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:46.157 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:46.159 DEBUG:boto:Method: GET 2022-09-03T07:30:46.159 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:30:46.161 DEBUG:boto:Data: 2022-09-03T07:30:46.162 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.164 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.164 DEBUG:boto:Params: {} 2022-09-03T07:30:46.165 DEBUG:boto:Token: None 2022-09-03T07:30:46.165 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.172 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c9e85309bfa93cd6-00631302a6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.214 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.215 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:46.216 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:46.217 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.218 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:30:46.219 DEBUG:boto:Data: 2022-09-03T07:30:46.220 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.220 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.221 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.221 DEBUG:boto:Params: {} 2022-09-03T07:30:46.222 DEBUG:boto:Token: None 2022-09-03T07:30:46.222 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29 2022-09-03T07:30:46.223 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ADedh2VvfW5PcpdakcQIbm2u7ms= 2022-09-03T07:30:46.224 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ADedh2VvfW5PcpdakcQIbm2u7ms=', 'Content-Length': '0'} 2022-09-03T07:30:46.226 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d4fd8967ce9d5b06-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.226 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:46.227 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:46.227 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.228 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:30:46.228 DEBUG:boto:Data: 2022-09-03T07:30:46.229 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.229 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.230 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.230 DEBUG:boto:Params: {} 2022-09-03T07:30:46.231 DEBUG:boto:Token: None 2022-09-03T07:30:46.232 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29 2022-09-03T07:30:46.232 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ADedh2VvfW5PcpdakcQIbm2u7ms= 2022-09-03T07:30:46.233 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ADedh2VvfW5PcpdakcQIbm2u7ms=', 'Content-Length': '0'} 2022-09-03T07:30:46.235 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cce285d123f8040f-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.236 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:30:46.236 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:46.237 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:46.237 DEBUG:boto:Method: GET 2022-09-03T07:30:46.238 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:30:46.238 DEBUG:boto:Data: 2022-09-03T07:30:46.239 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.239 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.240 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.241 DEBUG:boto:Params: {} 2022-09-03T07:30:46.241 DEBUG:boto:Token: None 2022-09-03T07:30:46.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29 2022-09-03T07:30:46.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lr9bFInYM9C35VxFs45Mj/jAEIQ= 2022-09-03T07:30:46.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lr9bFInYM9C35VxFs45Mj/jAEIQ=', 'Content-Length': '0'} 2022-09-03T07:30:46.245 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a699fcb7f4d7ed44-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.246 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:46.246 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:46.247 DEBUG:boto:Method: GET 2022-09-03T07:30:46.247 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:30:46.248 DEBUG:boto:Data: 2022-09-03T07:30:46.248 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.249 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.249 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.250 DEBUG:boto:Params: {} 2022-09-03T07:30:46.251 DEBUG:boto:Token: None 2022-09-03T07:30:46.251 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj29 2022-09-03T07:30:46.252 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lr9bFInYM9C35VxFs45Mj/jAEIQ= 2022-09-03T07:30:46.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lr9bFInYM9C35VxFs45Mj/jAEIQ=', 'Content-Length': '0'} 2022-09-03T07:30:46.255 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002873a4145e649837-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.256 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.257 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.257 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:46.258 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:46.258 DEBUG:boto:Method: GET 2022-09-03T07:30:46.259 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:30:46.259 DEBUG:boto:Data: 2022-09-03T07:30:46.260 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.260 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.261 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.261 DEBUG:boto:Params: {} 2022-09-03T07:30:46.262 DEBUG:boto:Token: None 2022-09-03T07:30:46.263 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.266 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb770e9d1ce8a61a-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.267 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.268 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.269 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.269 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:46.270 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:46.270 DEBUG:boto:Method: GET 2022-09-03T07:30:46.271 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:30:46.271 DEBUG:boto:Data: 2022-09-03T07:30:46.272 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.272 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.273 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.273 DEBUG:boto:Params: {} 2022-09-03T07:30:46.274 DEBUG:boto:Token: None 2022-09-03T07:30:46.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.284 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003dac732e92cb1b25-00631302a6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.285 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.286 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:30:46.286 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.287 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.287 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.288 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.288 DEBUG:boto:Method: GET 2022-09-03T07:30:46.289 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.290 DEBUG:boto:Data: 2022-09-03T07:30:46.290 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.291 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.292 DEBUG:boto:Params: {} 2022-09-03T07:30:46.292 DEBUG:boto:Token: None 2022-09-03T07:30:46.293 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.294 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SKzgxCumu0mtG3krSnS6D9N0RM8= 2022-09-03T07:30:46.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SKzgxCumu0mtG3krSnS6D9N0RM8=', 'Content-Length': '0'} 2022-09-03T07:30:46.296 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ec4239002e5bb4d-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.338 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.339 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.339 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.340 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.341 DEBUG:boto:Method: GET 2022-09-03T07:30:46.341 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.342 DEBUG:boto:Data: 2022-09-03T07:30:46.343 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.345 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.346 DEBUG:boto:Params: {} 2022-09-03T07:30:46.348 DEBUG:boto:Token: None 2022-09-03T07:30:46.349 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SKzgxCumu0mtG3krSnS6D9N0RM8= 2022-09-03T07:30:46.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SKzgxCumu0mtG3krSnS6D9N0RM8=', 'Content-Length': '0'} 2022-09-03T07:30:46.353 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009882e6bd91ec1341-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.355 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.356 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.356 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.357 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.358 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.359 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.360 DEBUG:boto:Data: 2022-09-03T07:30:46.360 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.361 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.361 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.362 DEBUG:boto:Params: {} 2022-09-03T07:30:46.362 DEBUG:boto:Token: None 2022-09-03T07:30:46.363 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zokvviz+5ddGo118KpXGzHjJDWo= 2022-09-03T07:30:46.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zokvviz+5ddGo118KpXGzHjJDWo=', 'Content-Length': '0'} 2022-09-03T07:30:46.367 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe1da1dd9e693af7-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.367 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.368 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.369 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.369 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.370 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.370 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.371 DEBUG:boto:Data: 2022-09-03T07:30:46.371 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.372 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.373 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.373 DEBUG:boto:Params: {} 2022-09-03T07:30:46.374 DEBUG:boto:Token: None 2022-09-03T07:30:46.375 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.375 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zokvviz+5ddGo118KpXGzHjJDWo= 2022-09-03T07:30:46.376 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zokvviz+5ddGo118KpXGzHjJDWo=', 'Content-Length': '0'} 2022-09-03T07:30:46.378 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3e1eb3f7ab33abd-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.379 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:30:46.380 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.380 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.381 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.381 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.382 DEBUG:boto:Method: GET 2022-09-03T07:30:46.382 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.383 DEBUG:boto:Data: 2022-09-03T07:30:46.384 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.384 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.385 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.385 DEBUG:boto:Params: {} 2022-09-03T07:30:46.386 DEBUG:boto:Token: None 2022-09-03T07:30:46.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SKzgxCumu0mtG3krSnS6D9N0RM8= 2022-09-03T07:30:46.388 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SKzgxCumu0mtG3krSnS6D9N0RM8=', 'Content-Length': '0'} 2022-09-03T07:30:46.389 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f0187769ec31ccb-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.390 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.391 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.391 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.392 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.392 DEBUG:boto:Method: GET 2022-09-03T07:30:46.393 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.393 DEBUG:boto:Data: 2022-09-03T07:30:46.394 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.394 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.395 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.395 DEBUG:boto:Params: {} 2022-09-03T07:30:46.396 DEBUG:boto:Token: None 2022-09-03T07:30:46.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:46.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SKzgxCumu0mtG3krSnS6D9N0RM8= 2022-09-03T07:30:46.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SKzgxCumu0mtG3krSnS6D9N0RM8=', 'Content-Length': '0'} 2022-09-03T07:30:46.400 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d824aaef3e2ec91-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.442 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.443 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.444 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:46.445 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:46.446 DEBUG:boto:Method: GET 2022-09-03T07:30:46.447 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:30:46.448 DEBUG:boto:Data: 2022-09-03T07:30:46.449 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.449 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.450 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.450 DEBUG:boto:Params: {} 2022-09-03T07:30:46.451 DEBUG:boto:Token: None 2022-09-03T07:30:46.452 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.453 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.457 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000002d5c71736ac9634-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.458 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.460 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.461 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.461 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:46.462 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:46.462 DEBUG:boto:Method: GET 2022-09-03T07:30:46.463 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:30:46.464 DEBUG:boto:Data: 2022-09-03T07:30:46.466 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.467 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.467 DEBUG:boto:Params: {} 2022-09-03T07:30:46.468 DEBUG:boto:Token: None 2022-09-03T07:30:46.469 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.471 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.476 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003939d5f22333ad98-00631302a6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.477 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.479 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.479 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.480 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.480 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:30:46.481 DEBUG:boto:Data: 2022-09-03T07:30:46.482 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.482 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.483 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.483 DEBUG:boto:Params: {} 2022-09-03T07:30:46.484 DEBUG:boto:Token: None 2022-09-03T07:30:46.484 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31 2022-09-03T07:30:46.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K5BubzalnTLSmNtTVg8Z6BvLsV4= 2022-09-03T07:30:46.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K5BubzalnTLSmNtTVg8Z6BvLsV4=', 'Content-Length': '0'} 2022-09-03T07:30:46.487 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c626683eee77c82-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.488 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.489 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.489 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.490 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:30:46.490 DEBUG:boto:Data: 2022-09-03T07:30:46.491 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.492 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.492 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.493 DEBUG:boto:Params: {} 2022-09-03T07:30:46.493 DEBUG:boto:Token: None 2022-09-03T07:30:46.494 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31 2022-09-03T07:30:46.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:K5BubzalnTLSmNtTVg8Z6BvLsV4= 2022-09-03T07:30:46.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:K5BubzalnTLSmNtTVg8Z6BvLsV4=', 'Content-Length': '0'} 2022-09-03T07:30:46.497 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045c2e7a7e6e2fb11-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.498 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:30:46.499 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.500 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.500 DEBUG:boto:Method: GET 2022-09-03T07:30:46.501 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:30:46.501 DEBUG:boto:Data: 2022-09-03T07:30:46.502 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.503 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.503 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.504 DEBUG:boto:Params: {} 2022-09-03T07:30:46.504 DEBUG:boto:Token: None 2022-09-03T07:30:46.505 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31 2022-09-03T07:30:46.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vLngik9wRNg1Uwfak5rwGXVTkx4= 2022-09-03T07:30:46.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vLngik9wRNg1Uwfak5rwGXVTkx4=', 'Content-Length': '0'} 2022-09-03T07:30:46.509 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000590f0bac7154074f-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.510 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:46.511 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:46.511 DEBUG:boto:Method: GET 2022-09-03T07:30:46.512 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:30:46.512 DEBUG:boto:Data: 2022-09-03T07:30:46.513 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.514 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.514 DEBUG:boto:Params: {} 2022-09-03T07:30:46.515 DEBUG:boto:Token: None 2022-09-03T07:30:46.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj31 2022-09-03T07:30:46.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vLngik9wRNg1Uwfak5rwGXVTkx4= 2022-09-03T07:30:46.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vLngik9wRNg1Uwfak5rwGXVTkx4=', 'Content-Length': '0'} 2022-09-03T07:30:46.519 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe36bee17f846893-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.521 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.521 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.522 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:46.523 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:46.523 DEBUG:boto:Method: GET 2022-09-03T07:30:46.524 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:30:46.524 DEBUG:boto:Data: 2022-09-03T07:30:46.525 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.525 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.526 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.526 DEBUG:boto:Params: {} 2022-09-03T07:30:46.527 DEBUG:boto:Token: None 2022-09-03T07:30:46.528 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.529 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.532 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000000485bf71bfa3c56-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.532 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.533 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.534 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.535 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:46.535 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:46.536 DEBUG:boto:Method: GET 2022-09-03T07:30:46.536 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:30:46.537 DEBUG:boto:Data: 2022-09-03T07:30:46.538 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.539 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.539 DEBUG:boto:Params: {} 2022-09-03T07:30:46.540 DEBUG:boto:Token: None 2022-09-03T07:30:46.541 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.542 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.546 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a2c2e360c6c5049-00631302a6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.547 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.548 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:30:46.548 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.549 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.549 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.550 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.550 DEBUG:boto:Method: GET 2022-09-03T07:30:46.551 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.552 DEBUG:boto:Data: 2022-09-03T07:30:46.552 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.553 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.553 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.554 DEBUG:boto:Params: {} 2022-09-03T07:30:46.554 DEBUG:boto:Token: None 2022-09-03T07:30:46.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sHsD667nejsLkRxhAyY1JD5bLCY= 2022-09-03T07:30:46.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sHsD667nejsLkRxhAyY1JD5bLCY=', 'Content-Length': '0'} 2022-09-03T07:30:46.559 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f3d8134c8e5985de-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.560 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.560 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.561 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.561 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.562 DEBUG:boto:Method: GET 2022-09-03T07:30:46.563 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.563 DEBUG:boto:Data: 2022-09-03T07:30:46.564 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.565 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.565 DEBUG:boto:Params: {} 2022-09-03T07:30:46.566 DEBUG:boto:Token: None 2022-09-03T07:30:46.567 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.567 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sHsD667nejsLkRxhAyY1JD5bLCY= 2022-09-03T07:30:46.568 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sHsD667nejsLkRxhAyY1JD5bLCY=', 'Content-Length': '0'} 2022-09-03T07:30:46.570 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fdaaa9e7284a69c0-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.618 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.619 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.619 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.620 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.621 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.621 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.622 DEBUG:boto:Data: 2022-09-03T07:30:46.622 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.623 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.623 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.624 DEBUG:boto:Params: {} 2022-09-03T07:30:46.625 DEBUG:boto:Token: None 2022-09-03T07:30:46.625 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.626 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EB7N2a+sRNkfYYLEE3x5qynuqYs= 2022-09-03T07:30:46.627 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EB7N2a+sRNkfYYLEE3x5qynuqYs=', 'Content-Length': '0'} 2022-09-03T07:30:46.629 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6ee29f7405490df-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.630 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.631 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.631 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.632 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.633 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.634 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.634 DEBUG:boto:Data: 2022-09-03T07:30:46.635 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.636 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.637 DEBUG:boto:Params: {} 2022-09-03T07:30:46.638 DEBUG:boto:Token: None 2022-09-03T07:30:46.639 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.639 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EB7N2a+sRNkfYYLEE3x5qynuqYs= 2022-09-03T07:30:46.640 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EB7N2a+sRNkfYYLEE3x5qynuqYs=', 'Content-Length': '0'} 2022-09-03T07:30:46.642 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001093e1b2773de7c0-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.643 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:30:46.643 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.644 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.644 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.645 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.645 DEBUG:boto:Method: GET 2022-09-03T07:30:46.646 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.646 DEBUG:boto:Data: 2022-09-03T07:30:46.648 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.648 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.649 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.649 DEBUG:boto:Params: {} 2022-09-03T07:30:46.650 DEBUG:boto:Token: None 2022-09-03T07:30:46.650 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.651 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sHsD667nejsLkRxhAyY1JD5bLCY= 2022-09-03T07:30:46.652 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sHsD667nejsLkRxhAyY1JD5bLCY=', 'Content-Length': '0'} 2022-09-03T07:30:46.654 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b844286924647532-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.655 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.655 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.656 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.656 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.657 DEBUG:boto:Method: GET 2022-09-03T07:30:46.659 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.659 DEBUG:boto:Data: 2022-09-03T07:30:46.660 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.660 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.661 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.661 DEBUG:boto:Params: {} 2022-09-03T07:30:46.662 DEBUG:boto:Token: None 2022-09-03T07:30:46.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:46.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sHsD667nejsLkRxhAyY1JD5bLCY= 2022-09-03T07:30:46.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sHsD667nejsLkRxhAyY1JD5bLCY=', 'Content-Length': '0'} 2022-09-03T07:30:46.666 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049f819e7d6fa45e2-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.667 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.668 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.668 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:46.669 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:46.669 DEBUG:boto:Method: GET 2022-09-03T07:30:46.670 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:30:46.670 DEBUG:boto:Data: 2022-09-03T07:30:46.671 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.671 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.672 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.673 DEBUG:boto:Params: {} 2022-09-03T07:30:46.673 DEBUG:boto:Token: None 2022-09-03T07:30:46.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.674 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.677 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b7313dbba6d5d571-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.678 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.679 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.680 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.680 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:46.681 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:46.681 DEBUG:boto:Method: GET 2022-09-03T07:30:46.682 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:30:46.682 DEBUG:boto:Data: 2022-09-03T07:30:46.683 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.683 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.684 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.684 DEBUG:boto:Params: {} 2022-09-03T07:30:46.685 DEBUG:boto:Token: None 2022-09-03T07:30:46.686 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.691 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000fc750f88e0ce32a-00631302a6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.734 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.735 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.736 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.736 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.737 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:30:46.737 DEBUG:boto:Data: 2022-09-03T07:30:46.738 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.739 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.739 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.740 DEBUG:boto:Params: {} 2022-09-03T07:30:46.741 DEBUG:boto:Token: None 2022-09-03T07:30:46.741 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33 2022-09-03T07:30:46.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+Q6Vg7y4ZlTN4HUaqvqG9FCds0I= 2022-09-03T07:30:46.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+Q6Vg7y4ZlTN4HUaqvqG9FCds0I=', 'Content-Length': '0'} 2022-09-03T07:30:46.744 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c98a35e87b8cedec-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.745 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.746 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.746 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.747 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:30:46.747 DEBUG:boto:Data: 2022-09-03T07:30:46.748 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.748 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.749 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.749 DEBUG:boto:Params: {} 2022-09-03T07:30:46.750 DEBUG:boto:Token: None 2022-09-03T07:30:46.750 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33 2022-09-03T07:30:46.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+Q6Vg7y4ZlTN4HUaqvqG9FCds0I= 2022-09-03T07:30:46.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+Q6Vg7y4ZlTN4HUaqvqG9FCds0I=', 'Content-Length': '0'} 2022-09-03T07:30:46.755 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026d6eb12346ad456-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.756 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:30:46.757 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.758 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.759 DEBUG:boto:Method: GET 2022-09-03T07:30:46.760 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:30:46.761 DEBUG:boto:Data: 2022-09-03T07:30:46.762 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.763 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.764 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.765 DEBUG:boto:Params: {} 2022-09-03T07:30:46.765 DEBUG:boto:Token: None 2022-09-03T07:30:46.766 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33 2022-09-03T07:30:46.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yOJi9/IB0/NRp8ArB+9pe/kI6FA= 2022-09-03T07:30:46.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yOJi9/IB0/NRp8ArB+9pe/kI6FA=', 'Content-Length': '0'} 2022-09-03T07:30:46.770 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0cbf33b986f17f2-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.771 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:46.771 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:46.772 DEBUG:boto:Method: GET 2022-09-03T07:30:46.773 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:30:46.773 DEBUG:boto:Data: 2022-09-03T07:30:46.774 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.775 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.775 DEBUG:boto:Params: {} 2022-09-03T07:30:46.776 DEBUG:boto:Token: None 2022-09-03T07:30:46.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj33 2022-09-03T07:30:46.777 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yOJi9/IB0/NRp8ArB+9pe/kI6FA= 2022-09-03T07:30:46.778 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yOJi9/IB0/NRp8ArB+9pe/kI6FA=', 'Content-Length': '0'} 2022-09-03T07:30:46.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005669c382199f68cf-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.782 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.783 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.784 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:46.784 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:46.785 DEBUG:boto:Method: GET 2022-09-03T07:30:46.785 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:30:46.788 DEBUG:boto:Data: 2022-09-03T07:30:46.789 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.790 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.791 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.792 DEBUG:boto:Params: {} 2022-09-03T07:30:46.793 DEBUG:boto:Token: None 2022-09-03T07:30:46.793 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.795 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.800 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ccb89cb8a593194f-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.846 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.848 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:46.849 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:46.850 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:46.851 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:46.852 DEBUG:boto:Method: GET 2022-09-03T07:30:46.852 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:30:46.853 DEBUG:boto:Data: 2022-09-03T07:30:46.853 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.854 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.855 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.855 DEBUG:boto:Params: {} 2022-09-03T07:30:46.856 DEBUG:boto:Token: None 2022-09-03T07:30:46.856 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/ 2022-09-03T07:30:46.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw= 2022-09-03T07:30:46.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EEx+lM2MBahmbtntSVctqe1bMpw=', 'Content-Length': '0'} 2022-09-03T07:30:46.862 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000674b7ca17c9a5d61-00631302a6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.863 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:46.864 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:30:46.865 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:46.866 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:46.866 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.867 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.867 DEBUG:boto:Method: GET 2022-09-03T07:30:46.868 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.869 DEBUG:boto:Data: 2022-09-03T07:30:46.870 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.871 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.871 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.872 DEBUG:boto:Params: {} 2022-09-03T07:30:46.873 DEBUG:boto:Token: None 2022-09-03T07:30:46.874 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.875 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9tJrPjxT2eGEULWw4I9CNQISH9k= 2022-09-03T07:30:46.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9tJrPjxT2eGEULWw4I9CNQISH9k=', 'Content-Length': '0'} 2022-09-03T07:30:46.883 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000199061d7c5ce2c4f-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.884 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:46.885 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:46.886 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.888 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.890 DEBUG:boto:Method: GET 2022-09-03T07:30:46.891 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.891 DEBUG:boto:Data: 2022-09-03T07:30:46.892 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.893 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.896 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.897 DEBUG:boto:Params: {} 2022-09-03T07:30:46.898 DEBUG:boto:Token: None 2022-09-03T07:30:46.900 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9tJrPjxT2eGEULWw4I9CNQISH9k= 2022-09-03T07:30:46.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9tJrPjxT2eGEULWw4I9CNQISH9k=', 'Content-Length': '0'} 2022-09-03T07:30:46.905 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000017dbbda1578ecf4e-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.906 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:46.907 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:46.908 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.909 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.910 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.911 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.912 DEBUG:boto:Data: 2022-09-03T07:30:46.912 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.913 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.914 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.914 DEBUG:boto:Params: {} 2022-09-03T07:30:46.915 DEBUG:boto:Token: None 2022-09-03T07:30:46.915 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:am9BIu5P9pl9mKH4HyQP3lsv0W4= 2022-09-03T07:30:46.917 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:am9BIu5P9pl9mKH4HyQP3lsv0W4=', 'Content-Length': '0'} 2022-09-03T07:30:46.921 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054bc4036d3481269-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.922 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:46.923 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:46.925 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.925 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.926 DEBUG:boto:Method: HEAD 2022-09-03T07:30:46.927 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.928 DEBUG:boto:Data: 2022-09-03T07:30:46.929 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.931 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.933 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.934 DEBUG:boto:Params: {} 2022-09-03T07:30:46.934 DEBUG:boto:Token: None 2022-09-03T07:30:46.936 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:am9BIu5P9pl9mKH4HyQP3lsv0W4= 2022-09-03T07:30:46.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:am9BIu5P9pl9mKH4HyQP3lsv0W4=', 'Content-Length': '0'} 2022-09-03T07:30:46.939 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057e29f495f06087d-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.942 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:30:46.943 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:46.944 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:46.944 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.945 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.946 DEBUG:boto:Method: GET 2022-09-03T07:30:46.947 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.948 DEBUG:boto:Data: 2022-09-03T07:30:46.949 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.950 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:46.951 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.952 DEBUG:boto:Params: {} 2022-09-03T07:30:46.953 DEBUG:boto:Token: None 2022-09-03T07:30:46.954 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9tJrPjxT2eGEULWw4I9CNQISH9k= 2022-09-03T07:30:46.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9tJrPjxT2eGEULWw4I9CNQISH9k=', 'Content-Length': '0'} 2022-09-03T07:30:46.960 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df0eecc89d7ec264-00631302a6-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:46.961 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:46.961 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:46.962 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.963 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.963 DEBUG:boto:Method: GET 2022-09-03T07:30:46.964 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.964 DEBUG:boto:Data: 2022-09-03T07:30:46.965 DEBUG:boto:Headers: {} 2022-09-03T07:30:46.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:46.966 DEBUG:boto:Port: 8000 2022-09-03T07:30:46.966 DEBUG:boto:Params: {} 2022-09-03T07:30:46.967 DEBUG:boto:Token: None 2022-09-03T07:30:46.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:46 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:46.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9tJrPjxT2eGEULWw4I9CNQISH9k= 2022-09-03T07:30:46.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:46 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9tJrPjxT2eGEULWw4I9CNQISH9k=', 'Content-Length': '0'} 2022-09-03T07:30:46.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f7365e0447f0ef5-00631302a6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:46 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.014 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.015 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.016 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:47.016 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:47.017 DEBUG:boto:Method: GET 2022-09-03T07:30:47.018 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:30:47.018 DEBUG:boto:Data: 2022-09-03T07:30:47.019 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.020 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.020 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.021 DEBUG:boto:Params: {} 2022-09-03T07:30:47.022 DEBUG:boto:Token: None 2022-09-03T07:30:47.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.029 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000483d7872d668b94a-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.035 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.037 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.038 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.039 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:47.040 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:47.041 DEBUG:boto:Method: GET 2022-09-03T07:30:47.042 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:30:47.043 DEBUG:boto:Data: 2022-09-03T07:30:47.044 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.045 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.046 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.047 DEBUG:boto:Params: {} 2022-09-03T07:30:47.049 DEBUG:boto:Token: None 2022-09-03T07:30:47.050 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.051 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.052 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.058 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079b8036dd8f40ce4-00631302a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.059 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.061 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:47.062 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:47.062 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.063 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:30:47.063 DEBUG:boto:Data: 2022-09-03T07:30:47.064 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.064 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.065 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.065 DEBUG:boto:Params: {} 2022-09-03T07:30:47.066 DEBUG:boto:Token: None 2022-09-03T07:30:47.067 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj34 2022-09-03T07:30:47.067 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w2JOfKOIo7y1XgNySAy+PImN3OU= 2022-09-03T07:30:47.068 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w2JOfKOIo7y1XgNySAy+PImN3OU=', 'Content-Length': '0'} 2022-09-03T07:30:47.070 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000df58147aa1f4522-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.071 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:47.072 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:47.072 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.073 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:30:47.074 DEBUG:boto:Data: 2022-09-03T07:30:47.074 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.076 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.077 DEBUG:boto:Params: {} 2022-09-03T07:30:47.078 DEBUG:boto:Token: None 2022-09-03T07:30:47.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj34 2022-09-03T07:30:47.081 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w2JOfKOIo7y1XgNySAy+PImN3OU= 2022-09-03T07:30:47.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w2JOfKOIo7y1XgNySAy+PImN3OU=', 'Content-Length': '0'} 2022-09-03T07:30:47.089 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2660c90e9f0eb28-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.090 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:30:47.090 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:47.091 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:47.091 DEBUG:boto:Method: GET 2022-09-03T07:30:47.092 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:30:47.092 DEBUG:boto:Data: 2022-09-03T07:30:47.093 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.093 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.095 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.096 DEBUG:boto:Params: {} 2022-09-03T07:30:47.097 DEBUG:boto:Token: None 2022-09-03T07:30:47.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj34 2022-09-03T07:30:47.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1CKKCywieY/1z8wNZ6oqE3wiPuY= 2022-09-03T07:30:47.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1CKKCywieY/1z8wNZ6oqE3wiPuY=', 'Content-Length': '0'} 2022-09-03T07:30:47.103 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000236c25a51486f853-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.146 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:47.147 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:47.148 DEBUG:boto:Method: GET 2022-09-03T07:30:47.149 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:30:47.150 DEBUG:boto:Data: 2022-09-03T07:30:47.151 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.152 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.153 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.153 DEBUG:boto:Params: {} 2022-09-03T07:30:47.154 DEBUG:boto:Token: None 2022-09-03T07:30:47.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj34 2022-09-03T07:30:47.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1CKKCywieY/1z8wNZ6oqE3wiPuY= 2022-09-03T07:30:47.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1CKKCywieY/1z8wNZ6oqE3wiPuY=', 'Content-Length': '0'} 2022-09-03T07:30:47.158 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b0033ebc8a2deb1-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.160 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.161 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.161 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:47.163 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:47.164 DEBUG:boto:Method: GET 2022-09-03T07:30:47.165 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:30:47.166 DEBUG:boto:Data: 2022-09-03T07:30:47.167 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.168 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.169 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.169 DEBUG:boto:Params: {} 2022-09-03T07:30:47.170 DEBUG:boto:Token: None 2022-09-03T07:30:47.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.177 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000978d1d4784c87429-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.178 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.179 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.180 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.180 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:47.181 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:47.181 DEBUG:boto:Method: GET 2022-09-03T07:30:47.182 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:30:47.183 DEBUG:boto:Data: 2022-09-03T07:30:47.183 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.184 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.185 DEBUG:boto:Params: {} 2022-09-03T07:30:47.185 DEBUG:boto:Token: None 2022-09-03T07:30:47.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.193 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027cba18d348885df-00631302a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.234 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.236 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2022-09-03T07:30:47.237 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.238 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.239 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.239 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.240 DEBUG:boto:Method: GET 2022-09-03T07:30:47.241 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.242 DEBUG:boto:Data: 2022-09-03T07:30:47.243 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.244 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.245 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.246 DEBUG:boto:Params: {} 2022-09-03T07:30:47.247 DEBUG:boto:Token: None 2022-09-03T07:30:47.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8wR5/ZrysiAeg7BqsqkQprw2468= 2022-09-03T07:30:47.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8wR5/ZrysiAeg7BqsqkQprw2468=', 'Content-Length': '0'} 2022-09-03T07:30:47.252 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef8d561811ecf2ca-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.255 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.256 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.256 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.257 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.259 DEBUG:boto:Method: GET 2022-09-03T07:30:47.260 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.261 DEBUG:boto:Data: 2022-09-03T07:30:47.262 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.263 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.266 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.268 DEBUG:boto:Params: {} 2022-09-03T07:30:47.269 DEBUG:boto:Token: None 2022-09-03T07:30:47.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.272 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8wR5/ZrysiAeg7BqsqkQprw2468= 2022-09-03T07:30:47.274 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8wR5/ZrysiAeg7BqsqkQprw2468=', 'Content-Length': '0'} 2022-09-03T07:30:47.277 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f65de0aac11e4146-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.280 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.281 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.282 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.282 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.283 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.284 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.285 DEBUG:boto:Data: 2022-09-03T07:30:47.286 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.287 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.288 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.289 DEBUG:boto:Params: {} 2022-09-03T07:30:47.290 DEBUG:boto:Token: None 2022-09-03T07:30:47.291 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.292 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yR5Du/2XU0vtJUs5qtrmpRDn6w4= 2022-09-03T07:30:47.293 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yR5Du/2XU0vtJUs5qtrmpRDn6w4=', 'Content-Length': '0'} 2022-09-03T07:30:47.295 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002987a6546b53ffd2-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.297 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.298 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.298 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.299 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.300 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.301 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.301 DEBUG:boto:Data: 2022-09-03T07:30:47.302 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.303 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.304 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.305 DEBUG:boto:Params: {} 2022-09-03T07:30:47.305 DEBUG:boto:Token: None 2022-09-03T07:30:47.306 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.307 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yR5Du/2XU0vtJUs5qtrmpRDn6w4= 2022-09-03T07:30:47.308 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yR5Du/2XU0vtJUs5qtrmpRDn6w4=', 'Content-Length': '0'} 2022-09-03T07:30:47.318 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9c9d5c9dcc1ec55-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.320 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2022-09-03T07:30:47.321 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.322 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.323 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.324 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.325 DEBUG:boto:Method: GET 2022-09-03T07:30:47.326 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.327 DEBUG:boto:Data: 2022-09-03T07:30:47.328 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.328 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.329 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.330 DEBUG:boto:Params: {} 2022-09-03T07:30:47.331 DEBUG:boto:Token: None 2022-09-03T07:30:47.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8wR5/ZrysiAeg7BqsqkQprw2468= 2022-09-03T07:30:47.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8wR5/ZrysiAeg7BqsqkQprw2468=', 'Content-Length': '0'} 2022-09-03T07:30:47.335 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d68877a7d8b38b9-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.336 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.337 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.337 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.338 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.339 DEBUG:boto:Method: GET 2022-09-03T07:30:47.339 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.340 DEBUG:boto:Data: 2022-09-03T07:30:47.340 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.341 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.342 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.342 DEBUG:boto:Params: {} 2022-09-03T07:30:47.343 DEBUG:boto:Token: None 2022-09-03T07:30:47.344 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:47.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8wR5/ZrysiAeg7BqsqkQprw2468= 2022-09-03T07:30:47.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8wR5/ZrysiAeg7BqsqkQprw2468=', 'Content-Length': '0'} 2022-09-03T07:30:47.350 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000433ec6848fbd405-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.351 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.352 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.352 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:47.353 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:47.354 DEBUG:boto:Method: GET 2022-09-03T07:30:47.355 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:30:47.355 DEBUG:boto:Data: 2022-09-03T07:30:47.356 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.357 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.358 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.359 DEBUG:boto:Params: {} 2022-09-03T07:30:47.360 DEBUG:boto:Token: None 2022-09-03T07:30:47.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.361 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.362 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.365 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2326cca1042387c-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.367 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.368 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.369 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.369 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:47.370 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:47.371 DEBUG:boto:Method: GET 2022-09-03T07:30:47.371 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:30:47.372 DEBUG:boto:Data: 2022-09-03T07:30:47.373 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.374 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.374 DEBUG:boto:Params: {} 2022-09-03T07:30:47.375 DEBUG:boto:Token: None 2022-09-03T07:30:47.376 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.386 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e2f9bce7e7decc5e-00631302a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.387 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.388 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.389 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.389 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.390 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:30:47.391 DEBUG:boto:Data: 2022-09-03T07:30:47.391 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.392 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.393 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.393 DEBUG:boto:Params: {} 2022-09-03T07:30:47.394 DEBUG:boto:Token: None 2022-09-03T07:30:47.395 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37 2022-09-03T07:30:47.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ljHa52oq0aJpcXZ3UbCeJ/S/YGw= 2022-09-03T07:30:47.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ljHa52oq0aJpcXZ3UbCeJ/S/YGw=', 'Content-Length': '0'} 2022-09-03T07:30:47.398 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003fdb5551321022db-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.399 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.400 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.400 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.401 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:30:47.402 DEBUG:boto:Data: 2022-09-03T07:30:47.404 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.404 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.405 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.406 DEBUG:boto:Params: {} 2022-09-03T07:30:47.407 DEBUG:boto:Token: None 2022-09-03T07:30:47.407 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37 2022-09-03T07:30:47.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ljHa52oq0aJpcXZ3UbCeJ/S/YGw= 2022-09-03T07:30:47.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ljHa52oq0aJpcXZ3UbCeJ/S/YGw=', 'Content-Length': '0'} 2022-09-03T07:30:47.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000014f1d4614d959fd1-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.411 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2022-09-03T07:30:47.412 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.413 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.414 DEBUG:boto:Method: GET 2022-09-03T07:30:47.415 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:30:47.416 DEBUG:boto:Data: 2022-09-03T07:30:47.417 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.419 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.420 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.421 DEBUG:boto:Params: {} 2022-09-03T07:30:47.423 DEBUG:boto:Token: None 2022-09-03T07:30:47.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37 2022-09-03T07:30:47.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vhA64kzH96hQO6K7+9W0PzZCbkI= 2022-09-03T07:30:47.427 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vhA64kzH96hQO6K7+9W0PzZCbkI=', 'Content-Length': '0'} 2022-09-03T07:30:47.430 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000061dfd7acd61349ff-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.432 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:47.433 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:47.434 DEBUG:boto:Method: GET 2022-09-03T07:30:47.436 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:30:47.437 DEBUG:boto:Data: 2022-09-03T07:30:47.438 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.439 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.439 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.440 DEBUG:boto:Params: {} 2022-09-03T07:30:47.441 DEBUG:boto:Token: None 2022-09-03T07:30:47.442 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj37 2022-09-03T07:30:47.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vhA64kzH96hQO6K7+9W0PzZCbkI= 2022-09-03T07:30:47.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vhA64kzH96hQO6K7+9W0PzZCbkI=', 'Content-Length': '0'} 2022-09-03T07:30:47.446 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4ca841f0429684e-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.491 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.491 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.492 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:47.493 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:47.494 DEBUG:boto:Method: GET 2022-09-03T07:30:47.495 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:30:47.496 DEBUG:boto:Data: 2022-09-03T07:30:47.497 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.498 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.499 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.500 DEBUG:boto:Params: {} 2022-09-03T07:30:47.501 DEBUG:boto:Token: None 2022-09-03T07:30:47.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.507 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d89e872d3d02ca56-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.508 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.510 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.510 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.511 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:47.512 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:47.513 DEBUG:boto:Method: GET 2022-09-03T07:30:47.513 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:30:47.514 DEBUG:boto:Data: 2022-09-03T07:30:47.515 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.516 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.516 DEBUG:boto:Params: {} 2022-09-03T07:30:47.517 DEBUG:boto:Token: None 2022-09-03T07:30:47.517 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.519 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.524 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001c854291ef96531b-00631302a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.525 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.527 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:30:47.528 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.529 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.530 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.530 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.531 DEBUG:boto:Method: GET 2022-09-03T07:30:47.532 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.532 DEBUG:boto:Data: 2022-09-03T07:30:47.533 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.533 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.534 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.535 DEBUG:boto:Params: {} 2022-09-03T07:30:47.536 DEBUG:boto:Token: None 2022-09-03T07:30:47.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bQ3CL9l4cYmXJuRb9DimcLM7UKI= 2022-09-03T07:30:47.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bQ3CL9l4cYmXJuRb9DimcLM7UKI=', 'Content-Length': '0'} 2022-09-03T07:30:47.540 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbb3bc36bb486137-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.582 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.583 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.583 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.584 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.584 DEBUG:boto:Method: GET 2022-09-03T07:30:47.585 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.585 DEBUG:boto:Data: 2022-09-03T07:30:47.586 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.587 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.588 DEBUG:boto:Params: {} 2022-09-03T07:30:47.588 DEBUG:boto:Token: None 2022-09-03T07:30:47.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bQ3CL9l4cYmXJuRb9DimcLM7UKI= 2022-09-03T07:30:47.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bQ3CL9l4cYmXJuRb9DimcLM7UKI=', 'Content-Length': '0'} 2022-09-03T07:30:47.593 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076b98e024550163f-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.593 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.594 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.595 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.595 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.596 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.596 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.597 DEBUG:boto:Data: 2022-09-03T07:30:47.597 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.598 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.598 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.599 DEBUG:boto:Params: {} 2022-09-03T07:30:47.599 DEBUG:boto:Token: None 2022-09-03T07:30:47.600 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ym+0p3sePlPKpggEZZQsZSylMvI= 2022-09-03T07:30:47.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ym+0p3sePlPKpggEZZQsZSylMvI=', 'Content-Length': '0'} 2022-09-03T07:30:47.603 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078ba1d08a2a9c3ae-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.604 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.605 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.605 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.606 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.606 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.607 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.607 DEBUG:boto:Data: 2022-09-03T07:30:47.608 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.608 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.609 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.609 DEBUG:boto:Params: {} 2022-09-03T07:30:47.610 DEBUG:boto:Token: None 2022-09-03T07:30:47.610 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ym+0p3sePlPKpggEZZQsZSylMvI= 2022-09-03T07:30:47.612 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ym+0p3sePlPKpggEZZQsZSylMvI=', 'Content-Length': '0'} 2022-09-03T07:30:47.614 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000efd69d89a6aaae64-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.615 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:30:47.615 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.616 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.617 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.618 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.619 DEBUG:boto:Method: GET 2022-09-03T07:30:47.620 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.621 DEBUG:boto:Data: 2022-09-03T07:30:47.622 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.623 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.624 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.624 DEBUG:boto:Params: {} 2022-09-03T07:30:47.625 DEBUG:boto:Token: None 2022-09-03T07:30:47.626 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bQ3CL9l4cYmXJuRb9DimcLM7UKI= 2022-09-03T07:30:47.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bQ3CL9l4cYmXJuRb9DimcLM7UKI=', 'Content-Length': '0'} 2022-09-03T07:30:47.630 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a579a92d96bfe693-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.631 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.632 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.632 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.633 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.633 DEBUG:boto:Method: GET 2022-09-03T07:30:47.634 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.635 DEBUG:boto:Data: 2022-09-03T07:30:47.636 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.637 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.638 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.639 DEBUG:boto:Params: {} 2022-09-03T07:30:47.640 DEBUG:boto:Token: None 2022-09-03T07:30:47.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:47.642 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bQ3CL9l4cYmXJuRb9DimcLM7UKI= 2022-09-03T07:30:47.643 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bQ3CL9l4cYmXJuRb9DimcLM7UKI=', 'Content-Length': '0'} 2022-09-03T07:30:47.645 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000300150358c6d4480-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.647 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.647 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.648 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:47.648 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:47.649 DEBUG:boto:Method: GET 2022-09-03T07:30:47.649 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:30:47.650 DEBUG:boto:Data: 2022-09-03T07:30:47.651 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.652 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.652 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.653 DEBUG:boto:Params: {} 2022-09-03T07:30:47.654 DEBUG:boto:Token: None 2022-09-03T07:30:47.654 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.655 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.658 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076dc7d33afcf8f32-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.659 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.660 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.660 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.661 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:47.662 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:47.662 DEBUG:boto:Method: GET 2022-09-03T07:30:47.663 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:30:47.663 DEBUG:boto:Data: 2022-09-03T07:30:47.664 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.665 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.666 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.667 DEBUG:boto:Params: {} 2022-09-03T07:30:47.667 DEBUG:boto:Token: None 2022-09-03T07:30:47.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.674 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f25bd9250eae7e6d-00631302a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.675 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.676 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.677 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.677 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.678 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:30:47.678 DEBUG:boto:Data: 2022-09-03T07:30:47.679 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.680 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.681 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.681 DEBUG:boto:Params: {} 2022-09-03T07:30:47.683 DEBUG:boto:Token: None 2022-09-03T07:30:47.683 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38 2022-09-03T07:30:47.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vOIWEV4TPzYh8TwBrPWmvNYgXMw= 2022-09-03T07:30:47.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vOIWEV4TPzYh8TwBrPWmvNYgXMw=', 'Content-Length': '0'} 2022-09-03T07:30:47.691 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8c061d5b9af427f-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.693 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.694 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.695 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.696 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:30:47.697 DEBUG:boto:Data: 2022-09-03T07:30:47.698 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.699 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.700 DEBUG:boto:Params: {} 2022-09-03T07:30:47.701 DEBUG:boto:Token: None 2022-09-03T07:30:47.702 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38 2022-09-03T07:30:47.703 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vOIWEV4TPzYh8TwBrPWmvNYgXMw= 2022-09-03T07:30:47.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vOIWEV4TPzYh8TwBrPWmvNYgXMw=', 'Content-Length': '0'} 2022-09-03T07:30:47.707 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000009b143f88309b20-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.708 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:30:47.709 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.710 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.711 DEBUG:boto:Method: GET 2022-09-03T07:30:47.711 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:30:47.712 DEBUG:boto:Data: 2022-09-03T07:30:47.712 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.713 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.714 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.714 DEBUG:boto:Params: {} 2022-09-03T07:30:47.715 DEBUG:boto:Token: None 2022-09-03T07:30:47.715 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38 2022-09-03T07:30:47.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4SVeBY79u/Gj7ZHtZlHosRevImM= 2022-09-03T07:30:47.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4SVeBY79u/Gj7ZHtZlHosRevImM=', 'Content-Length': '0'} 2022-09-03T07:30:47.720 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000972f140f4ae22199-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.762 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:47.763 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:47.764 DEBUG:boto:Method: GET 2022-09-03T07:30:47.765 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:30:47.765 DEBUG:boto:Data: 2022-09-03T07:30:47.766 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.766 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.767 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.767 DEBUG:boto:Params: {} 2022-09-03T07:30:47.768 DEBUG:boto:Token: None 2022-09-03T07:30:47.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj38 2022-09-03T07:30:47.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4SVeBY79u/Gj7ZHtZlHosRevImM= 2022-09-03T07:30:47.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4SVeBY79u/Gj7ZHtZlHosRevImM=', 'Content-Length': '0'} 2022-09-03T07:30:47.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d5a4065130a8f3b0-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.776 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.777 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.778 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:47.779 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:47.779 DEBUG:boto:Method: GET 2022-09-03T07:30:47.780 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:30:47.781 DEBUG:boto:Data: 2022-09-03T07:30:47.782 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.783 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.784 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.785 DEBUG:boto:Params: {} 2022-09-03T07:30:47.786 DEBUG:boto:Token: None 2022-09-03T07:30:47.788 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.794 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000021e8aa32dc59b02c-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.798 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.799 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.800 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.801 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:47.802 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:47.803 DEBUG:boto:Method: GET 2022-09-03T07:30:47.804 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:30:47.805 DEBUG:boto:Data: 2022-09-03T07:30:47.806 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.809 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.809 DEBUG:boto:Params: {} 2022-09-03T07:30:47.810 DEBUG:boto:Token: None 2022-09-03T07:30:47.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.817 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f76a514fec5b410b-00631302a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.859 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.861 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:30:47.862 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:47.863 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:47.864 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.864 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.865 DEBUG:boto:Method: GET 2022-09-03T07:30:47.866 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.866 DEBUG:boto:Data: 2022-09-03T07:30:47.867 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.870 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.871 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.872 DEBUG:boto:Params: {} 2022-09-03T07:30:47.872 DEBUG:boto:Token: None 2022-09-03T07:30:47.873 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WyRd4EQl8IMNOpTsGD7Vz8jJGwk= 2022-09-03T07:30:47.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WyRd4EQl8IMNOpTsGD7Vz8jJGwk=', 'Content-Length': '0'} 2022-09-03T07:30:47.877 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6273ec4bc7ac292-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.878 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:47.879 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:47.879 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.880 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.880 DEBUG:boto:Method: GET 2022-09-03T07:30:47.881 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.882 DEBUG:boto:Data: 2022-09-03T07:30:47.882 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.883 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.884 DEBUG:boto:Params: {} 2022-09-03T07:30:47.884 DEBUG:boto:Token: None 2022-09-03T07:30:47.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WyRd4EQl8IMNOpTsGD7Vz8jJGwk= 2022-09-03T07:30:47.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WyRd4EQl8IMNOpTsGD7Vz8jJGwk=', 'Content-Length': '0'} 2022-09-03T07:30:47.889 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a31ac15056aa859d-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.891 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:47.892 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:47.893 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.894 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.894 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.895 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.897 DEBUG:boto:Data: 2022-09-03T07:30:47.898 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.900 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.901 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.901 DEBUG:boto:Params: {} 2022-09-03T07:30:47.903 DEBUG:boto:Token: None 2022-09-03T07:30:47.904 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8qX2mi9hIUeCFk1ctqfn+2Oe4zI= 2022-09-03T07:30:47.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8qX2mi9hIUeCFk1ctqfn+2Oe4zI=', 'Content-Length': '0'} 2022-09-03T07:30:47.910 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000085588e8b5c8ca97f-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.912 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:47.912 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:47.913 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.914 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.915 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.916 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.917 DEBUG:boto:Data: 2022-09-03T07:30:47.918 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.919 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.920 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.921 DEBUG:boto:Params: {} 2022-09-03T07:30:47.922 DEBUG:boto:Token: None 2022-09-03T07:30:47.923 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8qX2mi9hIUeCFk1ctqfn+2Oe4zI= 2022-09-03T07:30:47.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8qX2mi9hIUeCFk1ctqfn+2Oe4zI=', 'Content-Length': '0'} 2022-09-03T07:30:47.928 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009743fafa0316e0f7-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.930 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:30:47.936 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:47.937 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:47.938 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.938 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.939 DEBUG:boto:Method: GET 2022-09-03T07:30:47.939 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.940 DEBUG:boto:Data: 2022-09-03T07:30:47.940 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.941 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.941 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.942 DEBUG:boto:Params: {} 2022-09-03T07:30:47.943 DEBUG:boto:Token: None 2022-09-03T07:30:47.943 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.944 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WyRd4EQl8IMNOpTsGD7Vz8jJGwk= 2022-09-03T07:30:47.945 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WyRd4EQl8IMNOpTsGD7Vz8jJGwk=', 'Content-Length': '0'} 2022-09-03T07:30:47.948 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb3467cf4e48b23a-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.949 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:47.949 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:47.950 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.950 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.951 DEBUG:boto:Method: GET 2022-09-03T07:30:47.952 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.952 DEBUG:boto:Data: 2022-09-03T07:30:47.953 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.953 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.954 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.954 DEBUG:boto:Params: {} 2022-09-03T07:30:47.955 DEBUG:boto:Token: None 2022-09-03T07:30:47.956 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:47.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WyRd4EQl8IMNOpTsGD7Vz8jJGwk= 2022-09-03T07:30:47.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WyRd4EQl8IMNOpTsGD7Vz8jJGwk=', 'Content-Length': '0'} 2022-09-03T07:30:47.959 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098ff02bcb81fea1c-00631302a7-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.960 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.961 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.961 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:47.962 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:47.963 DEBUG:boto:Method: GET 2022-09-03T07:30:47.963 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:30:47.964 DEBUG:boto:Data: 2022-09-03T07:30:47.964 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.965 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.966 DEBUG:boto:Params: {} 2022-09-03T07:30:47.967 DEBUG:boto:Token: None 2022-09-03T07:30:47.967 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.972 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000216cc43f51dd9cca-00631302a7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.972 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.973 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:47.974 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:47.975 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:47.975 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:47.976 DEBUG:boto:Method: GET 2022-09-03T07:30:47.976 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:30:47.977 DEBUG:boto:Data: 2022-09-03T07:30:47.977 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:47.978 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.979 DEBUG:boto:Params: {} 2022-09-03T07:30:47.980 DEBUG:boto:Token: None 2022-09-03T07:30:47.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/ 2022-09-03T07:30:47.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw= 2022-09-03T07:30:47.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bylKqX9YyTyy8Shpt/AieL8MHBw=', 'Content-Length': '0'} 2022-09-03T07:30:47.988 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011cb1bc293eae19f-00631302a7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:47 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:47.990 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:47.992 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:47.992 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:47.993 DEBUG:boto:Method: HEAD 2022-09-03T07:30:47.993 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:30:47.994 DEBUG:boto:Data: 2022-09-03T07:30:47.994 DEBUG:boto:Headers: {} 2022-09-03T07:30:47.995 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:47.996 DEBUG:boto:Port: 8000 2022-09-03T07:30:47.997 DEBUG:boto:Params: {} 2022-09-03T07:30:47.997 DEBUG:boto:Token: None 2022-09-03T07:30:47.998 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:47 GMT /avysmw-75/obj44 2022-09-03T07:30:47.999 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UiZ41wcxO4k9ZfKNxBQAdh2XBaE= 2022-09-03T07:30:48.000 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:47 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UiZ41wcxO4k9ZfKNxBQAdh2XBaE=', 'Content-Length': '0'} 2022-09-03T07:30:48.002 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010c88bebf9928915-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.003 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:48.004 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:48.004 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.005 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:30:48.005 DEBUG:boto:Data: 2022-09-03T07:30:48.006 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.006 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.007 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.007 DEBUG:boto:Params: {} 2022-09-03T07:30:48.008 DEBUG:boto:Token: None 2022-09-03T07:30:48.008 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj44 2022-09-03T07:30:48.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:R9JUExJedVZs7o85ZvX5Z2dOgds= 2022-09-03T07:30:48.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:R9JUExJedVZs7o85ZvX5Z2dOgds=', 'Content-Length': '0'} 2022-09-03T07:30:48.012 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000269a716bb0af7677-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.012 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:30:48.013 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:48.014 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:48.014 DEBUG:boto:Method: GET 2022-09-03T07:30:48.015 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:30:48.015 DEBUG:boto:Data: 2022-09-03T07:30:48.016 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.016 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.017 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.017 DEBUG:boto:Params: {} 2022-09-03T07:30:48.018 DEBUG:boto:Token: None 2022-09-03T07:30:48.019 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj44 2022-09-03T07:30:48.019 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ukemkrE9xTOZp22VR36SF/nKQtw= 2022-09-03T07:30:48.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ukemkrE9xTOZp22VR36SF/nKQtw=', 'Content-Length': '0'} 2022-09-03T07:30:48.022 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027e98c610c58e346-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.066 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:48.067 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:48.067 DEBUG:boto:Method: GET 2022-09-03T07:30:48.068 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:30:48.069 DEBUG:boto:Data: 2022-09-03T07:30:48.069 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.070 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.070 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.071 DEBUG:boto:Params: {} 2022-09-03T07:30:48.072 DEBUG:boto:Token: None 2022-09-03T07:30:48.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj44 2022-09-03T07:30:48.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ukemkrE9xTOZp22VR36SF/nKQtw= 2022-09-03T07:30:48.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ukemkrE9xTOZp22VR36SF/nKQtw=', 'Content-Length': '0'} 2022-09-03T07:30:48.076 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ae933693cb846a9-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.077 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.077 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.078 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:48.078 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:48.079 DEBUG:boto:Method: GET 2022-09-03T07:30:48.080 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:30:48.080 DEBUG:boto:Data: 2022-09-03T07:30:48.081 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.081 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.082 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.082 DEBUG:boto:Params: {} 2022-09-03T07:30:48.083 DEBUG:boto:Token: None 2022-09-03T07:30:48.083 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.084 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.084 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.087 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ac59457d80600cc0-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.088 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.089 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.089 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.090 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:48.090 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:48.091 DEBUG:boto:Method: GET 2022-09-03T07:30:48.091 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:30:48.092 DEBUG:boto:Data: 2022-09-03T07:30:48.093 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.093 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.094 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.094 DEBUG:boto:Params: {} 2022-09-03T07:30:48.095 DEBUG:boto:Token: None 2022-09-03T07:30:48.096 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.097 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.102 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037e65ca58a0034f9-00631302a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.102 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.104 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:30:48.105 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.105 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.106 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.107 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.108 DEBUG:boto:Method: GET 2022-09-03T07:30:48.109 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.110 DEBUG:boto:Data: 2022-09-03T07:30:48.111 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.111 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.112 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.113 DEBUG:boto:Params: {} 2022-09-03T07:30:48.113 DEBUG:boto:Token: None 2022-09-03T07:30:48.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YRGzbKwXj8FFL2n/Q79kl15VLQE= 2022-09-03T07:30:48.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YRGzbKwXj8FFL2n/Q79kl15VLQE=', 'Content-Length': '0'} 2022-09-03T07:30:48.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000061cd20e9bd24ac8e-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.119 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.120 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.121 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.121 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.122 DEBUG:boto:Method: GET 2022-09-03T07:30:48.123 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.124 DEBUG:boto:Data: 2022-09-03T07:30:48.125 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.126 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.126 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.127 DEBUG:boto:Params: {} 2022-09-03T07:30:48.128 DEBUG:boto:Token: None 2022-09-03T07:30:48.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.129 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YRGzbKwXj8FFL2n/Q79kl15VLQE= 2022-09-03T07:30:48.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YRGzbKwXj8FFL2n/Q79kl15VLQE=', 'Content-Length': '0'} 2022-09-03T07:30:48.132 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a3d432ecf168317-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.133 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.134 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.135 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.136 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.136 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.137 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.137 DEBUG:boto:Data: 2022-09-03T07:30:48.138 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.139 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.139 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.140 DEBUG:boto:Params: {} 2022-09-03T07:30:48.140 DEBUG:boto:Token: None 2022-09-03T07:30:48.141 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gAPCzO9TpuwP2ZvGxR8Q8HYJtBo= 2022-09-03T07:30:48.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gAPCzO9TpuwP2ZvGxR8Q8HYJtBo=', 'Content-Length': '0'} 2022-09-03T07:30:48.144 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036d8a99a396bd1b7-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.145 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.146 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.146 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.147 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.147 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.148 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.149 DEBUG:boto:Data: 2022-09-03T07:30:48.149 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.150 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.150 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.151 DEBUG:boto:Params: {} 2022-09-03T07:30:48.151 DEBUG:boto:Token: None 2022-09-03T07:30:48.152 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.152 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gAPCzO9TpuwP2ZvGxR8Q8HYJtBo= 2022-09-03T07:30:48.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gAPCzO9TpuwP2ZvGxR8Q8HYJtBo=', 'Content-Length': '0'} 2022-09-03T07:30:48.155 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf8d80b767c880ed-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.156 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:30:48.156 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.157 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.157 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.158 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.159 DEBUG:boto:Method: GET 2022-09-03T07:30:48.159 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.160 DEBUG:boto:Data: 2022-09-03T07:30:48.160 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.170 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.172 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.174 DEBUG:boto:Params: {} 2022-09-03T07:30:48.175 DEBUG:boto:Token: None 2022-09-03T07:30:48.176 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YRGzbKwXj8FFL2n/Q79kl15VLQE= 2022-09-03T07:30:48.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YRGzbKwXj8FFL2n/Q79kl15VLQE=', 'Content-Length': '0'} 2022-09-03T07:30:48.181 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001a2dcaacfff4c1c-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.183 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.183 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.184 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.185 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.185 DEBUG:boto:Method: GET 2022-09-03T07:30:48.186 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.187 DEBUG:boto:Data: 2022-09-03T07:30:48.188 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.189 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.190 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.191 DEBUG:boto:Params: {} 2022-09-03T07:30:48.192 DEBUG:boto:Token: None 2022-09-03T07:30:48.193 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:48.194 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YRGzbKwXj8FFL2n/Q79kl15VLQE= 2022-09-03T07:30:48.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YRGzbKwXj8FFL2n/Q79kl15VLQE=', 'Content-Length': '0'} 2022-09-03T07:30:48.197 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d11f48497522382-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.198 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.199 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.200 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:48.200 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:48.201 DEBUG:boto:Method: GET 2022-09-03T07:30:48.201 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:30:48.202 DEBUG:boto:Data: 2022-09-03T07:30:48.202 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.203 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.203 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.204 DEBUG:boto:Params: {} 2022-09-03T07:30:48.205 DEBUG:boto:Token: None 2022-09-03T07:30:48.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.209 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000024910506477e2e89-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.210 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.210 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.211 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.212 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:48.212 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:48.213 DEBUG:boto:Method: GET 2022-09-03T07:30:48.213 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:30:48.214 DEBUG:boto:Data: 2022-09-03T07:30:48.214 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.215 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.216 DEBUG:boto:Params: {} 2022-09-03T07:30:48.216 DEBUG:boto:Token: None 2022-09-03T07:30:48.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.223 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eb7884ccbccecc47-00631302a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.224 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.225 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.225 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.226 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.226 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:30:48.227 DEBUG:boto:Data: 2022-09-03T07:30:48.227 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.228 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.228 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.229 DEBUG:boto:Params: {} 2022-09-03T07:30:48.229 DEBUG:boto:Token: None 2022-09-03T07:30:48.230 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46 2022-09-03T07:30:48.231 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c8tr/s8yWAzPj/sm3y25cLIGZoo= 2022-09-03T07:30:48.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c8tr/s8yWAzPj/sm3y25cLIGZoo=', 'Content-Length': '0'} 2022-09-03T07:30:48.234 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1d00dd8dd3dd5e6-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.235 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.235 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.236 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.236 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:30:48.237 DEBUG:boto:Data: 2022-09-03T07:30:48.237 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.238 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.239 DEBUG:boto:Params: {} 2022-09-03T07:30:48.239 DEBUG:boto:Token: None 2022-09-03T07:30:48.240 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46 2022-09-03T07:30:48.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:c8tr/s8yWAzPj/sm3y25cLIGZoo= 2022-09-03T07:30:48.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:c8tr/s8yWAzPj/sm3y25cLIGZoo=', 'Content-Length': '0'} 2022-09-03T07:30:48.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ae7a1898e9ecdf6-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.244 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:30:48.244 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.245 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.246 DEBUG:boto:Method: GET 2022-09-03T07:30:48.246 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:30:48.247 DEBUG:boto:Data: 2022-09-03T07:30:48.247 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.248 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.248 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.249 DEBUG:boto:Params: {} 2022-09-03T07:30:48.249 DEBUG:boto:Token: None 2022-09-03T07:30:48.250 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46 2022-09-03T07:30:48.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pcoWhBvaa2bgr3RDBqTCR/8C3TQ= 2022-09-03T07:30:48.252 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pcoWhBvaa2bgr3RDBqTCR/8C3TQ=', 'Content-Length': '0'} 2022-09-03T07:30:48.253 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c27cc5be1144bae7-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.254 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:48.255 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:48.256 DEBUG:boto:Method: GET 2022-09-03T07:30:48.256 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:30:48.257 DEBUG:boto:Data: 2022-09-03T07:30:48.257 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.258 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.259 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.259 DEBUG:boto:Params: {} 2022-09-03T07:30:48.260 DEBUG:boto:Token: None 2022-09-03T07:30:48.261 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj46 2022-09-03T07:30:48.261 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pcoWhBvaa2bgr3RDBqTCR/8C3TQ= 2022-09-03T07:30:48.262 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pcoWhBvaa2bgr3RDBqTCR/8C3TQ=', 'Content-Length': '0'} 2022-09-03T07:30:48.265 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000067f8c8a8dc99ed18-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.307 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.308 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.310 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:48.311 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:48.312 DEBUG:boto:Method: GET 2022-09-03T07:30:48.313 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:30:48.314 DEBUG:boto:Data: 2022-09-03T07:30:48.315 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.316 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.317 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.318 DEBUG:boto:Params: {} 2022-09-03T07:30:48.319 DEBUG:boto:Token: None 2022-09-03T07:30:48.321 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.326 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000015b1a2642b72b674-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.327 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.328 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.329 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.329 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:48.330 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:48.331 DEBUG:boto:Method: GET 2022-09-03T07:30:48.331 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:30:48.332 DEBUG:boto:Data: 2022-09-03T07:30:48.332 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.333 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.333 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.334 DEBUG:boto:Params: {} 2022-09-03T07:30:48.334 DEBUG:boto:Token: None 2022-09-03T07:30:48.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.342 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000095800e8a3e77786d-00631302a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.343 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.344 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:30:48.344 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.345 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.345 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.346 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.347 DEBUG:boto:Method: GET 2022-09-03T07:30:48.347 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.348 DEBUG:boto:Data: 2022-09-03T07:30:48.348 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.349 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.350 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.350 DEBUG:boto:Params: {} 2022-09-03T07:30:48.351 DEBUG:boto:Token: None 2022-09-03T07:30:48.352 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AbZTTAY7EAqUWUZ8nAvkJGlLv+A= 2022-09-03T07:30:48.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AbZTTAY7EAqUWUZ8nAvkJGlLv+A=', 'Content-Length': '0'} 2022-09-03T07:30:48.360 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5ca90f994c15276-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.399 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.400 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.400 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.401 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.402 DEBUG:boto:Method: GET 2022-09-03T07:30:48.404 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.405 DEBUG:boto:Data: 2022-09-03T07:30:48.406 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.409 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.410 DEBUG:boto:Params: {} 2022-09-03T07:30:48.411 DEBUG:boto:Token: None 2022-09-03T07:30:48.417 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AbZTTAY7EAqUWUZ8nAvkJGlLv+A= 2022-09-03T07:30:48.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AbZTTAY7EAqUWUZ8nAvkJGlLv+A=', 'Content-Length': '0'} 2022-09-03T07:30:48.423 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a971fbb84ed8237-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.425 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.426 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.427 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.427 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.428 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.429 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.430 DEBUG:boto:Data: 2022-09-03T07:30:48.431 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.432 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.433 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.434 DEBUG:boto:Params: {} 2022-09-03T07:30:48.435 DEBUG:boto:Token: None 2022-09-03T07:30:48.436 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.436 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zbi4fvqqlbDDApJkWJc+4ULvcDw= 2022-09-03T07:30:48.437 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zbi4fvqqlbDDApJkWJc+4ULvcDw=', 'Content-Length': '0'} 2022-09-03T07:30:48.439 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b83bbdbd535f9b71-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.440 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.441 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.441 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.442 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.442 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.443 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.443 DEBUG:boto:Data: 2022-09-03T07:30:48.444 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.445 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.445 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.446 DEBUG:boto:Params: {} 2022-09-03T07:30:48.446 DEBUG:boto:Token: None 2022-09-03T07:30:48.447 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zbi4fvqqlbDDApJkWJc+4ULvcDw= 2022-09-03T07:30:48.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zbi4fvqqlbDDApJkWJc+4ULvcDw=', 'Content-Length': '0'} 2022-09-03T07:30:48.451 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cedf91d83457088-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.452 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:30:48.453 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.453 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.454 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.455 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.455 DEBUG:boto:Method: GET 2022-09-03T07:30:48.456 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.456 DEBUG:boto:Data: 2022-09-03T07:30:48.457 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.457 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.458 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.458 DEBUG:boto:Params: {} 2022-09-03T07:30:48.459 DEBUG:boto:Token: None 2022-09-03T07:30:48.459 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AbZTTAY7EAqUWUZ8nAvkJGlLv+A= 2022-09-03T07:30:48.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AbZTTAY7EAqUWUZ8nAvkJGlLv+A=', 'Content-Length': '0'} 2022-09-03T07:30:48.464 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000450e663254766de1-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.464 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.465 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.466 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.466 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.467 DEBUG:boto:Method: GET 2022-09-03T07:30:48.467 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.468 DEBUG:boto:Data: 2022-09-03T07:30:48.468 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.469 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.470 DEBUG:boto:Params: {} 2022-09-03T07:30:48.470 DEBUG:boto:Token: None 2022-09-03T07:30:48.471 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:48.472 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AbZTTAY7EAqUWUZ8nAvkJGlLv+A= 2022-09-03T07:30:48.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AbZTTAY7EAqUWUZ8nAvkJGlLv+A=', 'Content-Length': '0'} 2022-09-03T07:30:48.475 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000881b88f76f0e9532-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.476 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.477 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.477 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:48.478 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:48.479 DEBUG:boto:Method: GET 2022-09-03T07:30:48.480 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:30:48.480 DEBUG:boto:Data: 2022-09-03T07:30:48.481 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.481 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.482 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.483 DEBUG:boto:Params: {} 2022-09-03T07:30:48.483 DEBUG:boto:Token: None 2022-09-03T07:30:48.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.488 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c0b775e5165f995c-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.488 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.489 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.490 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.491 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:48.491 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:48.492 DEBUG:boto:Method: GET 2022-09-03T07:30:48.492 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:30:48.493 DEBUG:boto:Data: 2022-09-03T07:30:48.493 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.494 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.494 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.495 DEBUG:boto:Params: {} 2022-09-03T07:30:48.496 DEBUG:boto:Token: None 2022-09-03T07:30:48.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.504 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004a91ecf0ba6ee21a-00631302a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.550 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.552 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.553 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.554 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.555 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:30:48.556 DEBUG:boto:Data: 2022-09-03T07:30:48.557 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.558 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.559 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.560 DEBUG:boto:Params: {} 2022-09-03T07:30:48.561 DEBUG:boto:Token: None 2022-09-03T07:30:48.562 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51 2022-09-03T07:30:48.563 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X9BNGd1Oc9Ny9qfpTl+zYNRY55o= 2022-09-03T07:30:48.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X9BNGd1Oc9Ny9qfpTl+zYNRY55o=', 'Content-Length': '0'} 2022-09-03T07:30:48.566 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b26724fdd9c16f9b-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.567 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.567 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.568 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.568 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:30:48.569 DEBUG:boto:Data: 2022-09-03T07:30:48.569 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.570 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.571 DEBUG:boto:Params: {} 2022-09-03T07:30:48.572 DEBUG:boto:Token: None 2022-09-03T07:30:48.572 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51 2022-09-03T07:30:48.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:X9BNGd1Oc9Ny9qfpTl+zYNRY55o= 2022-09-03T07:30:48.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:X9BNGd1Oc9Ny9qfpTl+zYNRY55o=', 'Content-Length': '0'} 2022-09-03T07:30:48.576 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009cb3b4b6416e0c88-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.577 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:30:48.586 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.587 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.587 DEBUG:boto:Method: GET 2022-09-03T07:30:48.588 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:30:48.589 DEBUG:boto:Data: 2022-09-03T07:30:48.590 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.590 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.591 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.591 DEBUG:boto:Params: {} 2022-09-03T07:30:48.592 DEBUG:boto:Token: None 2022-09-03T07:30:48.593 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51 2022-09-03T07:30:48.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mviIkUPFNtSe5asBLBgfSOVlYIE= 2022-09-03T07:30:48.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mviIkUPFNtSe5asBLBgfSOVlYIE=', 'Content-Length': '0'} 2022-09-03T07:30:48.598 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd72c09e7bbf98d4-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.599 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:48.600 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:48.600 DEBUG:boto:Method: GET 2022-09-03T07:30:48.601 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:30:48.602 DEBUG:boto:Data: 2022-09-03T07:30:48.602 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.606 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.607 DEBUG:boto:Params: {} 2022-09-03T07:30:48.608 DEBUG:boto:Token: None 2022-09-03T07:30:48.609 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj51 2022-09-03T07:30:48.609 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mviIkUPFNtSe5asBLBgfSOVlYIE= 2022-09-03T07:30:48.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mviIkUPFNtSe5asBLBgfSOVlYIE=', 'Content-Length': '0'} 2022-09-03T07:30:48.614 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d49a4946681bac71-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.615 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.615 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.616 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:48.616 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:48.617 DEBUG:boto:Method: GET 2022-09-03T07:30:48.617 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:30:48.618 DEBUG:boto:Data: 2022-09-03T07:30:48.619 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.619 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.620 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.620 DEBUG:boto:Params: {} 2022-09-03T07:30:48.621 DEBUG:boto:Token: None 2022-09-03T07:30:48.622 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.623 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.624 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.626 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000000d6c1c2233a8a96-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.627 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.629 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.630 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.631 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:48.632 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:48.633 DEBUG:boto:Method: GET 2022-09-03T07:30:48.633 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:30:48.634 DEBUG:boto:Data: 2022-09-03T07:30:48.635 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.636 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.637 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.638 DEBUG:boto:Params: {} 2022-09-03T07:30:48.639 DEBUG:boto:Token: None 2022-09-03T07:30:48.640 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.642 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.647 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000019e9d064233a06c8-00631302a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.648 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.648 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:30:48.649 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.650 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.650 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.651 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.652 DEBUG:boto:Method: GET 2022-09-03T07:30:48.652 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.653 DEBUG:boto:Data: 2022-09-03T07:30:48.653 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.654 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.655 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.656 DEBUG:boto:Params: {} 2022-09-03T07:30:48.657 DEBUG:boto:Token: None 2022-09-03T07:30:48.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.658 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j4Iw5rNuefoDy/BEU6mESug1kBM= 2022-09-03T07:30:48.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j4Iw5rNuefoDy/BEU6mESug1kBM=', 'Content-Length': '0'} 2022-09-03T07:30:48.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2195766130fcd11-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.663 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.664 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.664 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.665 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.668 DEBUG:boto:Method: GET 2022-09-03T07:30:48.669 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.670 DEBUG:boto:Data: 2022-09-03T07:30:48.671 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.671 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.672 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.672 DEBUG:boto:Params: {} 2022-09-03T07:30:48.673 DEBUG:boto:Token: None 2022-09-03T07:30:48.674 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.674 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j4Iw5rNuefoDy/BEU6mESug1kBM= 2022-09-03T07:30:48.675 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j4Iw5rNuefoDy/BEU6mESug1kBM=', 'Content-Length': '0'} 2022-09-03T07:30:48.677 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071ea0814afca9083-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.718 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.719 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.720 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.721 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.721 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.722 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.722 DEBUG:boto:Data: 2022-09-03T07:30:48.723 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.724 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.725 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.725 DEBUG:boto:Params: {} 2022-09-03T07:30:48.727 DEBUG:boto:Token: None 2022-09-03T07:30:48.728 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fdfyo1z43EvYRUZa3KvZLkQ222Y= 2022-09-03T07:30:48.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fdfyo1z43EvYRUZa3KvZLkQ222Y=', 'Content-Length': '0'} 2022-09-03T07:30:48.732 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6b1c69248042d07-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.732 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.733 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.734 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.734 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.735 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.736 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.736 DEBUG:boto:Data: 2022-09-03T07:30:48.737 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.737 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.738 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.738 DEBUG:boto:Params: {} 2022-09-03T07:30:48.739 DEBUG:boto:Token: None 2022-09-03T07:30:48.739 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fdfyo1z43EvYRUZa3KvZLkQ222Y= 2022-09-03T07:30:48.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fdfyo1z43EvYRUZa3KvZLkQ222Y=', 'Content-Length': '0'} 2022-09-03T07:30:48.743 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053b7c307bbecd4c8-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.743 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:30:48.744 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.744 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.745 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.746 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.747 DEBUG:boto:Method: GET 2022-09-03T07:30:48.747 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.748 DEBUG:boto:Data: 2022-09-03T07:30:48.749 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.750 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.751 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.752 DEBUG:boto:Params: {} 2022-09-03T07:30:48.753 DEBUG:boto:Token: None 2022-09-03T07:30:48.754 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.754 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j4Iw5rNuefoDy/BEU6mESug1kBM= 2022-09-03T07:30:48.755 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j4Iw5rNuefoDy/BEU6mESug1kBM=', 'Content-Length': '0'} 2022-09-03T07:30:48.758 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081228379fd43bf44-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.759 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.760 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.761 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.761 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.762 DEBUG:boto:Method: GET 2022-09-03T07:30:48.762 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.763 DEBUG:boto:Data: 2022-09-03T07:30:48.764 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.764 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.765 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.766 DEBUG:boto:Params: {} 2022-09-03T07:30:48.767 DEBUG:boto:Token: None 2022-09-03T07:30:48.769 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:48.770 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j4Iw5rNuefoDy/BEU6mESug1kBM= 2022-09-03T07:30:48.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j4Iw5rNuefoDy/BEU6mESug1kBM=', 'Content-Length': '0'} 2022-09-03T07:30:48.775 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d449d17923b1dcc-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.777 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.779 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.780 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:48.781 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:48.781 DEBUG:boto:Method: GET 2022-09-03T07:30:48.782 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:30:48.783 DEBUG:boto:Data: 2022-09-03T07:30:48.783 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.784 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.784 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.785 DEBUG:boto:Params: {} 2022-09-03T07:30:48.785 DEBUG:boto:Token: None 2022-09-03T07:30:48.786 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.790 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b4aa39f354df2164-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.791 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.791 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.792 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.793 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:48.793 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:48.794 DEBUG:boto:Method: GET 2022-09-03T07:30:48.794 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:30:48.795 DEBUG:boto:Data: 2022-09-03T07:30:48.795 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.796 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.796 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.797 DEBUG:boto:Params: {} 2022-09-03T07:30:48.797 DEBUG:boto:Token: None 2022-09-03T07:30:48.798 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.799 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.803 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001ed379eda0866a11-00631302a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.846 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.848 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.849 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.849 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.850 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:30:48.851 DEBUG:boto:Data: 2022-09-03T07:30:48.852 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.854 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.855 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.856 DEBUG:boto:Params: {} 2022-09-03T07:30:48.857 DEBUG:boto:Token: None 2022-09-03T07:30:48.858 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55 2022-09-03T07:30:48.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZHW4lpUE1SMYrV6doJEa4BBOhw= 2022-09-03T07:30:48.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZHW4lpUE1SMYrV6doJEa4BBOhw=', 'Content-Length': '0'} 2022-09-03T07:30:48.864 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5553d2268eec89a-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.865 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.866 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.868 DEBUG:boto:Method: HEAD 2022-09-03T07:30:48.868 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:30:48.869 DEBUG:boto:Data: 2022-09-03T07:30:48.870 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.871 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.872 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.872 DEBUG:boto:Params: {} 2022-09-03T07:30:48.874 DEBUG:boto:Token: None 2022-09-03T07:30:48.875 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55 2022-09-03T07:30:48.876 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tZHW4lpUE1SMYrV6doJEa4BBOhw= 2022-09-03T07:30:48.877 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tZHW4lpUE1SMYrV6doJEa4BBOhw=', 'Content-Length': '0'} 2022-09-03T07:30:48.882 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a7aeeb82fb2dea6-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.884 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:30:48.886 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.886 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.887 DEBUG:boto:Method: GET 2022-09-03T07:30:48.888 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:30:48.888 DEBUG:boto:Data: 2022-09-03T07:30:48.889 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.890 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.891 DEBUG:boto:Params: {} 2022-09-03T07:30:48.891 DEBUG:boto:Token: None 2022-09-03T07:30:48.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55 2022-09-03T07:30:48.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XwhSp+/jCWKHsgDmaEt2lW4xXg0= 2022-09-03T07:30:48.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XwhSp+/jCWKHsgDmaEt2lW4xXg0=', 'Content-Length': '0'} 2022-09-03T07:30:48.897 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057fc44c5861e526f-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.938 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:48.944 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:48.945 DEBUG:boto:Method: GET 2022-09-03T07:30:48.946 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:30:48.947 DEBUG:boto:Data: 2022-09-03T07:30:48.948 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.949 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.950 DEBUG:boto:Params: {} 2022-09-03T07:30:48.951 DEBUG:boto:Token: None 2022-09-03T07:30:48.951 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/obj55 2022-09-03T07:30:48.952 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XwhSp+/jCWKHsgDmaEt2lW4xXg0= 2022-09-03T07:30:48.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XwhSp+/jCWKHsgDmaEt2lW4xXg0=', 'Content-Length': '0'} 2022-09-03T07:30:48.955 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000013aabd6d0e49de54-00631302a8-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.957 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.958 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.959 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:48.960 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:48.961 DEBUG:boto:Method: GET 2022-09-03T07:30:48.962 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:30:48.963 DEBUG:boto:Data: 2022-09-03T07:30:48.964 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:48.966 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.967 DEBUG:boto:Params: {} 2022-09-03T07:30:48.968 DEBUG:boto:Token: None 2022-09-03T07:30:48.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.974 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c0acce801fa4110-00631302a8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:48.975 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:48.976 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:48.977 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:48.978 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:48.979 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:48.980 DEBUG:boto:Method: GET 2022-09-03T07:30:48.983 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:30:48.984 DEBUG:boto:Data: 2022-09-03T07:30:48.985 DEBUG:boto:Headers: {} 2022-09-03T07:30:48.986 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:48.987 DEBUG:boto:Port: 8000 2022-09-03T07:30:48.989 DEBUG:boto:Params: {} 2022-09-03T07:30:48.990 DEBUG:boto:Token: None 2022-09-03T07:30:48.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:48 GMT /avysmw-75/ 2022-09-03T07:30:48.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI= 2022-09-03T07:30:48.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:48 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FCs/0WIetanKQZVfKxROb7vQrhI=', 'Content-Length': '0'} 2022-09-03T07:30:48.998 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027a82716829c7bfa-00631302a8-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:48 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.000 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.001 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:30:49.002 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.003 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.004 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.004 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.005 DEBUG:boto:Method: GET 2022-09-03T07:30:49.005 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.006 DEBUG:boto:Data: 2022-09-03T07:30:49.007 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.007 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.008 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.008 DEBUG:boto:Params: {} 2022-09-03T07:30:49.009 DEBUG:boto:Token: None 2022-09-03T07:30:49.009 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UvFpZZjDNxepZbWtmeTpSoVYNgk= 2022-09-03T07:30:49.011 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UvFpZZjDNxepZbWtmeTpSoVYNgk=', 'Content-Length': '0'} 2022-09-03T07:30:49.013 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026c6a4bfdbba48ec-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.015 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.016 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.017 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.018 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.019 DEBUG:boto:Method: GET 2022-09-03T07:30:49.019 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.020 DEBUG:boto:Data: 2022-09-03T07:30:49.021 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.022 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.023 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.024 DEBUG:boto:Params: {} 2022-09-03T07:30:49.025 DEBUG:boto:Token: None 2022-09-03T07:30:49.026 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UvFpZZjDNxepZbWtmeTpSoVYNgk= 2022-09-03T07:30:49.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UvFpZZjDNxepZbWtmeTpSoVYNgk=', 'Content-Length': '0'} 2022-09-03T07:30:49.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e03ade52f0e2c48-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.032 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.032 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.033 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.033 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.034 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.034 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.035 DEBUG:boto:Data: 2022-09-03T07:30:49.036 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.036 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.037 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.037 DEBUG:boto:Params: {} 2022-09-03T07:30:49.038 DEBUG:boto:Token: None 2022-09-03T07:30:49.039 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmLLpBJdAbMWfVOH6Ptqy7+OCqE= 2022-09-03T07:30:49.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmLLpBJdAbMWfVOH6Ptqy7+OCqE=', 'Content-Length': '0'} 2022-09-03T07:30:49.042 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000922e9fd29d454bd5-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.044 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.045 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.046 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.046 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.047 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.048 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.049 DEBUG:boto:Data: 2022-09-03T07:30:49.050 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.051 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.052 DEBUG:boto:Params: {} 2022-09-03T07:30:49.053 DEBUG:boto:Token: None 2022-09-03T07:30:49.054 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.055 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:AmLLpBJdAbMWfVOH6Ptqy7+OCqE= 2022-09-03T07:30:49.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:AmLLpBJdAbMWfVOH6Ptqy7+OCqE=', 'Content-Length': '0'} 2022-09-03T07:30:49.058 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8570fcd9644284c-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.059 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:30:49.060 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.061 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.062 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.063 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.064 DEBUG:boto:Method: GET 2022-09-03T07:30:49.065 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.066 DEBUG:boto:Data: 2022-09-03T07:30:49.067 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.068 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.069 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.069 DEBUG:boto:Params: {} 2022-09-03T07:30:49.070 DEBUG:boto:Token: None 2022-09-03T07:30:49.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UvFpZZjDNxepZbWtmeTpSoVYNgk= 2022-09-03T07:30:49.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UvFpZZjDNxepZbWtmeTpSoVYNgk=', 'Content-Length': '0'} 2022-09-03T07:30:49.077 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b747baef2f6760eb-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.118 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.119 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.120 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.121 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.121 DEBUG:boto:Method: GET 2022-09-03T07:30:49.122 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.123 DEBUG:boto:Data: 2022-09-03T07:30:49.124 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.125 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.126 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.127 DEBUG:boto:Params: {} 2022-09-03T07:30:49.128 DEBUG:boto:Token: None 2022-09-03T07:30:49.129 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:49.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UvFpZZjDNxepZbWtmeTpSoVYNgk= 2022-09-03T07:30:49.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UvFpZZjDNxepZbWtmeTpSoVYNgk=', 'Content-Length': '0'} 2022-09-03T07:30:49.134 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005157dba378cb81cb-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.135 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.136 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.137 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:49.138 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:49.139 DEBUG:boto:Method: GET 2022-09-03T07:30:49.140 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:30:49.141 DEBUG:boto:Data: 2022-09-03T07:30:49.142 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.143 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.144 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.145 DEBUG:boto:Params: {} 2022-09-03T07:30:49.146 DEBUG:boto:Token: None 2022-09-03T07:30:49.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.153 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001f0881b1d2d3f4ef-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.154 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.155 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.156 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.157 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:49.158 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:49.158 DEBUG:boto:Method: GET 2022-09-03T07:30:49.159 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:30:49.159 DEBUG:boto:Data: 2022-09-03T07:30:49.160 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.160 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.161 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.161 DEBUG:boto:Params: {} 2022-09-03T07:30:49.162 DEBUG:boto:Token: None 2022-09-03T07:30:49.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.169 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000970ba1051cc480eb-00631302a9-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.170 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.171 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.172 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.173 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.174 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:30:49.174 DEBUG:boto:Data: 2022-09-03T07:30:49.175 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.176 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.177 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.178 DEBUG:boto:Params: {} 2022-09-03T07:30:49.179 DEBUG:boto:Token: None 2022-09-03T07:30:49.180 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57 2022-09-03T07:30:49.181 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NwvXhhzD/KQfMHV9d0IImQcLOSs= 2022-09-03T07:30:49.182 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NwvXhhzD/KQfMHV9d0IImQcLOSs=', 'Content-Length': '0'} 2022-09-03T07:30:49.185 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076bee1b100ac9235-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.186 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.187 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.188 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.189 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:30:49.189 DEBUG:boto:Data: 2022-09-03T07:30:49.190 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.191 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.191 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.192 DEBUG:boto:Params: {} 2022-09-03T07:30:49.193 DEBUG:boto:Token: None 2022-09-03T07:30:49.193 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57 2022-09-03T07:30:49.194 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NwvXhhzD/KQfMHV9d0IImQcLOSs= 2022-09-03T07:30:49.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NwvXhhzD/KQfMHV9d0IImQcLOSs=', 'Content-Length': '0'} 2022-09-03T07:30:49.197 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f17449dfcd6ec48e-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.197 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:30:49.198 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.199 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.199 DEBUG:boto:Method: GET 2022-09-03T07:30:49.200 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:30:49.200 DEBUG:boto:Data: 2022-09-03T07:30:49.201 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.201 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.202 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.202 DEBUG:boto:Params: {} 2022-09-03T07:30:49.203 DEBUG:boto:Token: None 2022-09-03T07:30:49.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57 2022-09-03T07:30:49.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bop/ToVqHLxzdBdgUGqNjWeZ39M= 2022-09-03T07:30:49.205 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bop/ToVqHLxzdBdgUGqNjWeZ39M=', 'Content-Length': '0'} 2022-09-03T07:30:49.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb79fe73fe7cad34-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.208 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:49.208 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:49.209 DEBUG:boto:Method: GET 2022-09-03T07:30:49.209 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:30:49.210 DEBUG:boto:Data: 2022-09-03T07:30:49.210 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.211 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.211 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.212 DEBUG:boto:Params: {} 2022-09-03T07:30:49.213 DEBUG:boto:Token: None 2022-09-03T07:30:49.214 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj57 2022-09-03T07:30:49.215 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bop/ToVqHLxzdBdgUGqNjWeZ39M= 2022-09-03T07:30:49.216 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bop/ToVqHLxzdBdgUGqNjWeZ39M=', 'Content-Length': '0'} 2022-09-03T07:30:49.219 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ffcc838b9877e59-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.220 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.221 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.221 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:49.222 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:49.223 DEBUG:boto:Method: GET 2022-09-03T07:30:49.223 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:30:49.224 DEBUG:boto:Data: 2022-09-03T07:30:49.224 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.225 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.225 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.226 DEBUG:boto:Params: {} 2022-09-03T07:30:49.226 DEBUG:boto:Token: None 2022-09-03T07:30:49.227 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.231 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007d2b141139ba8a31-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.231 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.232 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.233 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.233 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:49.234 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:49.234 DEBUG:boto:Method: GET 2022-09-03T07:30:49.235 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:30:49.235 DEBUG:boto:Data: 2022-09-03T07:30:49.236 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.236 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.237 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.237 DEBUG:boto:Params: {} 2022-09-03T07:30:49.238 DEBUG:boto:Token: None 2022-09-03T07:30:49.239 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.245 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003fe846c9d55e993d-00631302a9-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.246 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.247 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:30:49.247 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.248 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.248 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.249 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.249 DEBUG:boto:Method: GET 2022-09-03T07:30:49.250 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.250 DEBUG:boto:Data: 2022-09-03T07:30:49.251 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.252 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.252 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.253 DEBUG:boto:Params: {} 2022-09-03T07:30:49.253 DEBUG:boto:Token: None 2022-09-03T07:30:49.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.254 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E1J23LNyZ/ZlNUTp9LsVjN8t6C0= 2022-09-03T07:30:49.255 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E1J23LNyZ/ZlNUTp9LsVjN8t6C0=', 'Content-Length': '0'} 2022-09-03T07:30:49.257 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001cf3bd25183f3cb0-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.302 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.304 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.304 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.305 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.306 DEBUG:boto:Method: GET 2022-09-03T07:30:49.307 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.307 DEBUG:boto:Data: 2022-09-03T07:30:49.308 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.308 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.309 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.309 DEBUG:boto:Params: {} 2022-09-03T07:30:49.310 DEBUG:boto:Token: None 2022-09-03T07:30:49.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E1J23LNyZ/ZlNUTp9LsVjN8t6C0= 2022-09-03T07:30:49.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E1J23LNyZ/ZlNUTp9LsVjN8t6C0=', 'Content-Length': '0'} 2022-09-03T07:30:49.314 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000800e76096a97087e-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.315 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.316 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.316 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.317 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.317 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.318 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.318 DEBUG:boto:Data: 2022-09-03T07:30:49.319 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.319 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.320 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.320 DEBUG:boto:Params: {} 2022-09-03T07:30:49.321 DEBUG:boto:Token: None 2022-09-03T07:30:49.322 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5o96qANiskm6FyAxdKWXjuj8/3I= 2022-09-03T07:30:49.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5o96qANiskm6FyAxdKWXjuj8/3I=', 'Content-Length': '0'} 2022-09-03T07:30:49.326 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb2c749aba888785-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.327 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.328 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.329 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.330 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.331 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.332 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.333 DEBUG:boto:Data: 2022-09-03T07:30:49.333 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.335 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.336 DEBUG:boto:Params: {} 2022-09-03T07:30:49.337 DEBUG:boto:Token: None 2022-09-03T07:30:49.338 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5o96qANiskm6FyAxdKWXjuj8/3I= 2022-09-03T07:30:49.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5o96qANiskm6FyAxdKWXjuj8/3I=', 'Content-Length': '0'} 2022-09-03T07:30:49.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a0b02dc09a7c0bb-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.344 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:30:49.345 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.346 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.347 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.348 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.349 DEBUG:boto:Method: GET 2022-09-03T07:30:49.349 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.351 DEBUG:boto:Data: 2022-09-03T07:30:49.352 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.353 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.354 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.355 DEBUG:boto:Params: {} 2022-09-03T07:30:49.355 DEBUG:boto:Token: None 2022-09-03T07:30:49.357 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E1J23LNyZ/ZlNUTp9LsVjN8t6C0= 2022-09-03T07:30:49.359 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E1J23LNyZ/ZlNUTp9LsVjN8t6C0=', 'Content-Length': '0'} 2022-09-03T07:30:49.361 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062a444ef92c42927-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.363 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.364 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.364 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.365 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.365 DEBUG:boto:Method: GET 2022-09-03T07:30:49.366 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.367 DEBUG:boto:Data: 2022-09-03T07:30:49.367 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.368 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.368 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.369 DEBUG:boto:Params: {} 2022-09-03T07:30:49.369 DEBUG:boto:Token: None 2022-09-03T07:30:49.370 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:49.371 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:E1J23LNyZ/ZlNUTp9LsVjN8t6C0= 2022-09-03T07:30:49.371 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:E1J23LNyZ/ZlNUTp9LsVjN8t6C0=', 'Content-Length': '0'} 2022-09-03T07:30:49.374 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a04116eca18e576e-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.414 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.415 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.415 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:49.416 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:49.417 DEBUG:boto:Method: GET 2022-09-03T07:30:49.417 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:30:49.418 DEBUG:boto:Data: 2022-09-03T07:30:49.419 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.419 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.420 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.421 DEBUG:boto:Params: {} 2022-09-03T07:30:49.421 DEBUG:boto:Token: None 2022-09-03T07:30:49.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.428 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f68c0998ed67e041-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.430 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.431 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.432 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.435 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:49.435 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:49.436 DEBUG:boto:Method: GET 2022-09-03T07:30:49.437 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:30:49.438 DEBUG:boto:Data: 2022-09-03T07:30:49.439 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.440 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.441 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.442 DEBUG:boto:Params: {} 2022-09-03T07:30:49.443 DEBUG:boto:Token: None 2022-09-03T07:30:49.444 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.450 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008f2763f10989950b-00631302a9-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.451 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.452 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.452 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.453 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.453 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:30:49.454 DEBUG:boto:Data: 2022-09-03T07:30:49.454 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.455 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.456 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.456 DEBUG:boto:Params: {} 2022-09-03T07:30:49.457 DEBUG:boto:Token: None 2022-09-03T07:30:49.457 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58 2022-09-03T07:30:49.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fZ4/x8+XW+wUICxadBC9seVHUVo= 2022-09-03T07:30:49.458 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fZ4/x8+XW+wUICxadBC9seVHUVo=', 'Content-Length': '0'} 2022-09-03T07:30:49.460 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c466dd32e8f0220a-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.461 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.461 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.462 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.463 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:30:49.463 DEBUG:boto:Data: 2022-09-03T07:30:49.464 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.464 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.465 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.465 DEBUG:boto:Params: {} 2022-09-03T07:30:49.466 DEBUG:boto:Token: None 2022-09-03T07:30:49.466 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58 2022-09-03T07:30:49.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fZ4/x8+XW+wUICxadBC9seVHUVo= 2022-09-03T07:30:49.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fZ4/x8+XW+wUICxadBC9seVHUVo=', 'Content-Length': '0'} 2022-09-03T07:30:49.470 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007dd58625235fc0a8-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.471 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:30:49.472 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.473 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.474 DEBUG:boto:Method: GET 2022-09-03T07:30:49.475 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:30:49.476 DEBUG:boto:Data: 2022-09-03T07:30:49.477 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.478 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.479 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.480 DEBUG:boto:Params: {} 2022-09-03T07:30:49.481 DEBUG:boto:Token: None 2022-09-03T07:30:49.482 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58 2022-09-03T07:30:49.483 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xtBIbQ721NUfVI1+zHPTiQ9B8Lg= 2022-09-03T07:30:49.484 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xtBIbQ721NUfVI1+zHPTiQ9B8Lg=', 'Content-Length': '0'} 2022-09-03T07:30:49.487 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c352d05dd6a7ae9-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.488 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:49.489 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:49.489 DEBUG:boto:Method: GET 2022-09-03T07:30:49.490 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:30:49.491 DEBUG:boto:Data: 2022-09-03T07:30:49.491 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.493 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.494 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.495 DEBUG:boto:Params: {} 2022-09-03T07:30:49.496 DEBUG:boto:Token: None 2022-09-03T07:30:49.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj58 2022-09-03T07:30:49.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xtBIbQ721NUfVI1+zHPTiQ9B8Lg= 2022-09-03T07:30:49.499 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xtBIbQ721NUfVI1+zHPTiQ9B8Lg=', 'Content-Length': '0'} 2022-09-03T07:30:49.504 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e9936b69f4b9210-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.546 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.547 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.547 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:49.548 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:49.548 DEBUG:boto:Method: GET 2022-09-03T07:30:49.549 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:30:49.549 DEBUG:boto:Data: 2022-09-03T07:30:49.550 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.551 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.551 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.552 DEBUG:boto:Params: {} 2022-09-03T07:30:49.552 DEBUG:boto:Token: None 2022-09-03T07:30:49.553 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.559 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000422377604457ec2f-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.560 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.561 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.562 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.562 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:49.563 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:49.564 DEBUG:boto:Method: GET 2022-09-03T07:30:49.565 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:30:49.566 DEBUG:boto:Data: 2022-09-03T07:30:49.567 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.568 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.569 DEBUG:boto:Params: {} 2022-09-03T07:30:49.569 DEBUG:boto:Token: None 2022-09-03T07:30:49.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.577 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000640ccc94f86510f0-00631302a9-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.578 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.580 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:30:49.581 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.581 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.585 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.586 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.587 DEBUG:boto:Method: GET 2022-09-03T07:30:49.587 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.588 DEBUG:boto:Data: 2022-09-03T07:30:49.588 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.589 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.589 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.590 DEBUG:boto:Params: {} 2022-09-03T07:30:49.591 DEBUG:boto:Token: None 2022-09-03T07:30:49.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mL7fTcInt6iQAaEZQaorerGsF+c= 2022-09-03T07:30:49.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mL7fTcInt6iQAaEZQaorerGsF+c=', 'Content-Length': '0'} 2022-09-03T07:30:49.595 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045c30538760cb9fc-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.596 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.597 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.597 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.598 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.598 DEBUG:boto:Method: GET 2022-09-03T07:30:49.599 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.600 DEBUG:boto:Data: 2022-09-03T07:30:49.600 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.601 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.601 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.602 DEBUG:boto:Params: {} 2022-09-03T07:30:49.602 DEBUG:boto:Token: None 2022-09-03T07:30:49.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mL7fTcInt6iQAaEZQaorerGsF+c= 2022-09-03T07:30:49.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mL7fTcInt6iQAaEZQaorerGsF+c=', 'Content-Length': '0'} 2022-09-03T07:30:49.607 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc0d3af0a0ff2abf-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.607 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.608 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.608 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.609 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.609 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.610 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.611 DEBUG:boto:Data: 2022-09-03T07:30:49.611 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.612 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.612 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.613 DEBUG:boto:Params: {} 2022-09-03T07:30:49.613 DEBUG:boto:Token: None 2022-09-03T07:30:49.614 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gPj9s0bpuUflGFyXEoiaCWf48rA= 2022-09-03T07:30:49.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gPj9s0bpuUflGFyXEoiaCWf48rA=', 'Content-Length': '0'} 2022-09-03T07:30:49.618 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff273e5556419d12-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.619 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.620 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.621 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.622 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.623 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.624 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.625 DEBUG:boto:Data: 2022-09-03T07:30:49.626 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.627 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.628 DEBUG:boto:Params: {} 2022-09-03T07:30:49.629 DEBUG:boto:Token: None 2022-09-03T07:30:49.630 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gPj9s0bpuUflGFyXEoiaCWf48rA= 2022-09-03T07:30:49.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gPj9s0bpuUflGFyXEoiaCWf48rA=', 'Content-Length': '0'} 2022-09-03T07:30:49.635 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009bcc503a961fcd68-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.636 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:30:49.636 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.637 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.637 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.638 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.638 DEBUG:boto:Method: GET 2022-09-03T07:30:49.639 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.639 DEBUG:boto:Data: 2022-09-03T07:30:49.640 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.640 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.641 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.641 DEBUG:boto:Params: {} 2022-09-03T07:30:49.642 DEBUG:boto:Token: None 2022-09-03T07:30:49.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mL7fTcInt6iQAaEZQaorerGsF+c= 2022-09-03T07:30:49.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mL7fTcInt6iQAaEZQaorerGsF+c=', 'Content-Length': '0'} 2022-09-03T07:30:49.646 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006137d84c2dd515ee-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.690 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.691 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.692 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.693 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.693 DEBUG:boto:Method: GET 2022-09-03T07:30:49.695 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.695 DEBUG:boto:Data: 2022-09-03T07:30:49.700 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.702 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.704 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.705 DEBUG:boto:Params: {} 2022-09-03T07:30:49.706 DEBUG:boto:Token: None 2022-09-03T07:30:49.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:30:49.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mL7fTcInt6iQAaEZQaorerGsF+c= 2022-09-03T07:30:49.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mL7fTcInt6iQAaEZQaorerGsF+c=', 'Content-Length': '0'} 2022-09-03T07:30:49.711 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe55eeb47e0134b1-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.712 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.713 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.714 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:30:49.715 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:30:49.716 DEBUG:boto:Method: GET 2022-09-03T07:30:49.716 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:30:49.717 DEBUG:boto:Data: 2022-09-03T07:30:49.717 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.718 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.718 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.719 DEBUG:boto:Params: {} 2022-09-03T07:30:49.720 DEBUG:boto:Token: None 2022-09-03T07:30:49.720 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.724 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c2a9cb7728360d8f-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.725 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.726 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.727 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.727 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:30:49.728 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:30:49.728 DEBUG:boto:Method: GET 2022-09-03T07:30:49.729 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:30:49.730 DEBUG:boto:Data: 2022-09-03T07:30:49.730 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.731 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.731 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.732 DEBUG:boto:Params: {} 2022-09-03T07:30:49.732 DEBUG:boto:Token: None 2022-09-03T07:30:49.733 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.734 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.739 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000376fdb0b436ceffc-00631302a9-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.740 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.742 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.743 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.744 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.745 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:30:49.745 DEBUG:boto:Data: 2022-09-03T07:30:49.746 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.747 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.748 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.748 DEBUG:boto:Params: {} 2022-09-03T07:30:49.749 DEBUG:boto:Token: None 2022-09-03T07:30:49.750 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62 2022-09-03T07:30:49.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WsNJ7faRffTwP5TTGgCwR5LCtT4= 2022-09-03T07:30:49.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WsNJ7faRffTwP5TTGgCwR5LCtT4=', 'Content-Length': '0'} 2022-09-03T07:30:49.753 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f6be2d5ea44ce64-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.754 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.755 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.755 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.756 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:30:49.756 DEBUG:boto:Data: 2022-09-03T07:30:49.757 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.758 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.759 DEBUG:boto:Params: {} 2022-09-03T07:30:49.760 DEBUG:boto:Token: None 2022-09-03T07:30:49.761 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62 2022-09-03T07:30:49.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WsNJ7faRffTwP5TTGgCwR5LCtT4= 2022-09-03T07:30:49.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WsNJ7faRffTwP5TTGgCwR5LCtT4=', 'Content-Length': '0'} 2022-09-03T07:30:49.767 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b18013bfd94698f-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.768 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:30:49.769 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.770 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.771 DEBUG:boto:Method: GET 2022-09-03T07:30:49.772 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:30:49.773 DEBUG:boto:Data: 2022-09-03T07:30:49.773 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.775 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.776 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.777 DEBUG:boto:Params: {} 2022-09-03T07:30:49.778 DEBUG:boto:Token: None 2022-09-03T07:30:49.779 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62 2022-09-03T07:30:49.780 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frOGNgf45bh8toBZbuWY4IjNPsA= 2022-09-03T07:30:49.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frOGNgf45bh8toBZbuWY4IjNPsA=', 'Content-Length': '0'} 2022-09-03T07:30:49.784 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000afb07f7a1a440f4-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.785 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:30:49.786 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:30:49.787 DEBUG:boto:Method: GET 2022-09-03T07:30:49.787 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:30:49.788 DEBUG:boto:Data: 2022-09-03T07:30:49.788 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.789 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.789 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.790 DEBUG:boto:Params: {} 2022-09-03T07:30:49.791 DEBUG:boto:Token: None 2022-09-03T07:30:49.791 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj62 2022-09-03T07:30:49.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frOGNgf45bh8toBZbuWY4IjNPsA= 2022-09-03T07:30:49.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frOGNgf45bh8toBZbuWY4IjNPsA=', 'Content-Length': '0'} 2022-09-03T07:30:49.795 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075db9388edf981bc-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.838 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.839 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.840 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:30:49.841 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:30:49.842 DEBUG:boto:Method: GET 2022-09-03T07:30:49.842 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:30:49.843 DEBUG:boto:Data: 2022-09-03T07:30:49.844 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.845 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.846 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.847 DEBUG:boto:Params: {} 2022-09-03T07:30:49.848 DEBUG:boto:Token: None 2022-09-03T07:30:49.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.850 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.851 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.855 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b75a34afa90c1423-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.856 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.857 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:49.858 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:49.859 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:30:49.861 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:30:49.863 DEBUG:boto:Method: GET 2022-09-03T07:30:49.863 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:30:49.864 DEBUG:boto:Data: 2022-09-03T07:30:49.865 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.866 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.867 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.869 DEBUG:boto:Params: {} 2022-09-03T07:30:49.870 DEBUG:boto:Token: None 2022-09-03T07:30:49.871 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/ 2022-09-03T07:30:49.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4= 2022-09-03T07:30:49.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uX0p7sjNFLMQY9xTHS+Q0rpHbM4=', 'Content-Length': '0'} 2022-09-03T07:30:49.879 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e5751655be49f2b6-00631302a9-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.880 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:49.881 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:30:49.882 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:49.883 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:49.884 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.885 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.886 DEBUG:boto:Method: GET 2022-09-03T07:30:49.887 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.890 DEBUG:boto:Data: 2022-09-03T07:30:49.890 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.891 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.893 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.894 DEBUG:boto:Params: {} 2022-09-03T07:30:49.895 DEBUG:boto:Token: None 2022-09-03T07:30:49.896 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.897 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fb7oYzyU8prNhepHOTuGMyrHdqE= 2022-09-03T07:30:49.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fb7oYzyU8prNhepHOTuGMyrHdqE=', 'Content-Length': '0'} 2022-09-03T07:30:49.904 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000131f8f752d88ac15-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.907 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:49.908 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:49.909 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.910 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.911 DEBUG:boto:Method: GET 2022-09-03T07:30:49.912 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.913 DEBUG:boto:Data: 2022-09-03T07:30:49.914 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.916 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.916 DEBUG:boto:Params: {} 2022-09-03T07:30:49.917 DEBUG:boto:Token: None 2022-09-03T07:30:49.918 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.920 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fb7oYzyU8prNhepHOTuGMyrHdqE= 2022-09-03T07:30:49.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fb7oYzyU8prNhepHOTuGMyrHdqE=', 'Content-Length': '0'} 2022-09-03T07:30:49.923 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006db86b91965c4558-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.924 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:49.925 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:49.926 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.927 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.928 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.929 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.930 DEBUG:boto:Data: 2022-09-03T07:30:49.931 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.932 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.933 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.934 DEBUG:boto:Params: {} 2022-09-03T07:30:49.935 DEBUG:boto:Token: None 2022-09-03T07:30:49.936 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zb9kDd8gVnYzb0Awm6ocWIwbrzo= 2022-09-03T07:30:49.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zb9kDd8gVnYzb0Awm6ocWIwbrzo=', 'Content-Length': '0'} 2022-09-03T07:30:49.941 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e218b063fd691e0-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.942 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:49.942 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:49.943 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.944 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.944 DEBUG:boto:Method: HEAD 2022-09-03T07:30:49.945 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.945 DEBUG:boto:Data: 2022-09-03T07:30:49.946 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.946 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.947 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.947 DEBUG:boto:Params: {} 2022-09-03T07:30:49.948 DEBUG:boto:Token: None 2022-09-03T07:30:49.948 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zb9kDd8gVnYzb0Awm6ocWIwbrzo= 2022-09-03T07:30:49.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zb9kDd8gVnYzb0Awm6ocWIwbrzo=', 'Content-Length': '0'} 2022-09-03T07:30:49.958 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b35486851747b694-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.960 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:30:49.961 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:49.961 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:49.962 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.962 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.963 DEBUG:boto:Method: GET 2022-09-03T07:30:49.963 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.964 DEBUG:boto:Data: 2022-09-03T07:30:49.965 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:49.966 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.966 DEBUG:boto:Params: {} 2022-09-03T07:30:49.967 DEBUG:boto:Token: None 2022-09-03T07:30:49.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.968 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fb7oYzyU8prNhepHOTuGMyrHdqE= 2022-09-03T07:30:49.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fb7oYzyU8prNhepHOTuGMyrHdqE=', 'Content-Length': '0'} 2022-09-03T07:30:49.971 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad8627d74f2d89f9-00631302a9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:49.972 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:49.973 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:49.975 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.976 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.977 DEBUG:boto:Method: GET 2022-09-03T07:30:49.980 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.981 DEBUG:boto:Data: 2022-09-03T07:30:49.982 DEBUG:boto:Headers: {} 2022-09-03T07:30:49.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:49.984 DEBUG:boto:Port: 8000 2022-09-03T07:30:49.985 DEBUG:boto:Params: {} 2022-09-03T07:30:49.987 DEBUG:boto:Token: None 2022-09-03T07:30:49.988 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:49 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:30:49.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fb7oYzyU8prNhepHOTuGMyrHdqE= 2022-09-03T07:30:49.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:49 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fb7oYzyU8prNhepHOTuGMyrHdqE=', 'Content-Length': '0'} 2022-09-03T07:30:49.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed893f4b6513b41c-00631302a9-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:49 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.035 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.036 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.037 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:30:50.038 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:30:50.039 DEBUG:boto:Method: GET 2022-09-03T07:30:50.040 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:30:50.041 DEBUG:boto:Data: 2022-09-03T07:30:50.042 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.043 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.044 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.046 DEBUG:boto:Params: {} 2022-09-03T07:30:50.047 DEBUG:boto:Token: None 2022-09-03T07:30:50.048 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.050 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.051 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.058 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000067a674afa3d53bca-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.059 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.060 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.060 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.062 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:30:50.063 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:30:50.064 DEBUG:boto:Method: GET 2022-09-03T07:30:50.065 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:30:50.067 DEBUG:boto:Data: 2022-09-03T07:30:50.068 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.069 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.070 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.071 DEBUG:boto:Params: {} 2022-09-03T07:30:50.072 DEBUG:boto:Token: None 2022-09-03T07:30:50.073 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.074 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.075 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.081 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006c0d0cc6e9513d99-00631302aa-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.082 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.082 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:50.083 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:50.084 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.085 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:30:50.086 DEBUG:boto:Data: 2022-09-03T07:30:50.087 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.089 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.090 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.091 DEBUG:boto:Params: {} 2022-09-03T07:30:50.092 DEBUG:boto:Token: None 2022-09-03T07:30:50.094 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj68 2022-09-03T07:30:50.095 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azj+w+CUeEnWrjGe+56yiPNwwgI= 2022-09-03T07:30:50.096 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azj+w+CUeEnWrjGe+56yiPNwwgI=', 'Content-Length': '0'} 2022-09-03T07:30:50.100 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dfce3b0739d55cb9-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.102 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:50.103 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:50.104 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.105 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:30:50.106 DEBUG:boto:Data: 2022-09-03T07:30:50.107 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.108 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.109 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.110 DEBUG:boto:Params: {} 2022-09-03T07:30:50.112 DEBUG:boto:Token: None 2022-09-03T07:30:50.113 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj68 2022-09-03T07:30:50.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:azj+w+CUeEnWrjGe+56yiPNwwgI= 2022-09-03T07:30:50.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:azj+w+CUeEnWrjGe+56yiPNwwgI=', 'Content-Length': '0'} 2022-09-03T07:30:50.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004709baa20352284a-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.120 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:30:50.121 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:50.123 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:50.124 DEBUG:boto:Method: GET 2022-09-03T07:30:50.126 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:30:50.127 DEBUG:boto:Data: 2022-09-03T07:30:50.128 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.130 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.131 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.132 DEBUG:boto:Params: {} 2022-09-03T07:30:50.134 DEBUG:boto:Token: None 2022-09-03T07:30:50.135 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj68 2022-09-03T07:30:50.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k5j4MnNtypXvuLP7ccQqQCf622U= 2022-09-03T07:30:50.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k5j4MnNtypXvuLP7ccQqQCf622U=', 'Content-Length': '0'} 2022-09-03T07:30:50.140 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002e774444766ac48-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.141 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:30:50.142 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:30:50.143 DEBUG:boto:Method: GET 2022-09-03T07:30:50.143 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:30:50.144 DEBUG:boto:Data: 2022-09-03T07:30:50.145 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.146 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.146 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.147 DEBUG:boto:Params: {} 2022-09-03T07:30:50.148 DEBUG:boto:Token: None 2022-09-03T07:30:50.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj68 2022-09-03T07:30:50.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k5j4MnNtypXvuLP7ccQqQCf622U= 2022-09-03T07:30:50.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k5j4MnNtypXvuLP7ccQqQCf622U=', 'Content-Length': '0'} 2022-09-03T07:30:50.152 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd52841473737acf-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.154 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.155 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.156 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:30:50.157 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:30:50.158 DEBUG:boto:Method: GET 2022-09-03T07:30:50.158 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:30:50.159 DEBUG:boto:Data: 2022-09-03T07:30:50.160 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.160 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.161 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.161 DEBUG:boto:Params: {} 2022-09-03T07:30:50.162 DEBUG:boto:Token: None 2022-09-03T07:30:50.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.164 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.168 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000085ab0503a7029229-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.169 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.171 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.172 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.174 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:30:50.175 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:30:50.176 DEBUG:boto:Method: GET 2022-09-03T07:30:50.177 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:30:50.178 DEBUG:boto:Data: 2022-09-03T07:30:50.179 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.179 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.180 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.181 DEBUG:boto:Params: {} 2022-09-03T07:30:50.181 DEBUG:boto:Token: None 2022-09-03T07:30:50.182 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.183 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.183 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.188 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a35e0502d5c06efb-00631302aa-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.189 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.190 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:30:50.191 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.192 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.193 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.193 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.194 DEBUG:boto:Method: GET 2022-09-03T07:30:50.195 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.196 DEBUG:boto:Data: 2022-09-03T07:30:50.197 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.197 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.199 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.199 DEBUG:boto:Params: {} 2022-09-03T07:30:50.200 DEBUG:boto:Token: None 2022-09-03T07:30:50.201 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.202 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KqzXS86fpAyzgF1jahrCiJ1Moj8= 2022-09-03T07:30:50.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KqzXS86fpAyzgF1jahrCiJ1Moj8=', 'Content-Length': '0'} 2022-09-03T07:30:50.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ec5a041c623e35b-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.208 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.210 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.210 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.211 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.212 DEBUG:boto:Method: GET 2022-09-03T07:30:50.212 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.213 DEBUG:boto:Data: 2022-09-03T07:30:50.214 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.215 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.216 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.217 DEBUG:boto:Params: {} 2022-09-03T07:30:50.218 DEBUG:boto:Token: None 2022-09-03T07:30:50.218 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KqzXS86fpAyzgF1jahrCiJ1Moj8= 2022-09-03T07:30:50.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KqzXS86fpAyzgF1jahrCiJ1Moj8=', 'Content-Length': '0'} 2022-09-03T07:30:50.222 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a82377b8c08fb82-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.270 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.342 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.343 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.344 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.345 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.345 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.346 DEBUG:boto:Data: 2022-09-03T07:30:50.347 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.347 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.348 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.349 DEBUG:boto:Params: {} 2022-09-03T07:30:50.349 DEBUG:boto:Token: None 2022-09-03T07:30:50.350 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XxJUA1yH+bZRU+4gTPJCJvskl5g= 2022-09-03T07:30:50.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XxJUA1yH+bZRU+4gTPJCJvskl5g=', 'Content-Length': '0'} 2022-09-03T07:30:50.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000330981bf1e24f5db-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.355 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.356 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.357 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.358 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.359 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.361 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.362 DEBUG:boto:Data: 2022-09-03T07:30:50.363 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.363 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.364 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.364 DEBUG:boto:Params: {} 2022-09-03T07:30:50.365 DEBUG:boto:Token: None 2022-09-03T07:30:50.366 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XxJUA1yH+bZRU+4gTPJCJvskl5g= 2022-09-03T07:30:50.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XxJUA1yH+bZRU+4gTPJCJvskl5g=', 'Content-Length': '0'} 2022-09-03T07:30:50.369 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054213dccd43d2126-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.370 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:30:50.371 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.372 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.372 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.373 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.374 DEBUG:boto:Method: GET 2022-09-03T07:30:50.375 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.376 DEBUG:boto:Data: 2022-09-03T07:30:50.377 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.378 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.379 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.380 DEBUG:boto:Params: {} 2022-09-03T07:30:50.381 DEBUG:boto:Token: None 2022-09-03T07:30:50.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KqzXS86fpAyzgF1jahrCiJ1Moj8= 2022-09-03T07:30:50.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KqzXS86fpAyzgF1jahrCiJ1Moj8=', 'Content-Length': '0'} 2022-09-03T07:30:50.387 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b827c95497893a02-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.388 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.389 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.390 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.391 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.392 DEBUG:boto:Method: GET 2022-09-03T07:30:50.392 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.393 DEBUG:boto:Data: 2022-09-03T07:30:50.394 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.394 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.395 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.396 DEBUG:boto:Params: {} 2022-09-03T07:30:50.396 DEBUG:boto:Token: None 2022-09-03T07:30:50.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:30:50.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KqzXS86fpAyzgF1jahrCiJ1Moj8= 2022-09-03T07:30:50.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KqzXS86fpAyzgF1jahrCiJ1Moj8=', 'Content-Length': '0'} 2022-09-03T07:30:50.400 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002de3a9144354ce25-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.401 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.402 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.402 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:30:50.403 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:30:50.404 DEBUG:boto:Method: GET 2022-09-03T07:30:50.405 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:30:50.406 DEBUG:boto:Data: 2022-09-03T07:30:50.407 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.408 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.409 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.409 DEBUG:boto:Params: {} 2022-09-03T07:30:50.410 DEBUG:boto:Token: None 2022-09-03T07:30:50.411 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.416 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008b99a80f7b6cddb7-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.463 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.465 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.466 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.466 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:30:50.467 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:30:50.467 DEBUG:boto:Method: GET 2022-09-03T07:30:50.468 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:30:50.468 DEBUG:boto:Data: 2022-09-03T07:30:50.469 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.469 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.470 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.471 DEBUG:boto:Params: {} 2022-09-03T07:30:50.472 DEBUG:boto:Token: None 2022-09-03T07:30:50.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.474 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.480 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008ba6baa2afb57542-00631302aa-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.481 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.482 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.483 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.483 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.484 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:30:50.484 DEBUG:boto:Data: 2022-09-03T07:30:50.485 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.485 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.486 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.487 DEBUG:boto:Params: {} 2022-09-03T07:30:50.487 DEBUG:boto:Token: None 2022-09-03T07:30:50.488 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69 2022-09-03T07:30:50.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v9p8Ikw+u+Lya+aTYuSFivIMKss= 2022-09-03T07:30:50.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v9p8Ikw+u+Lya+aTYuSFivIMKss=', 'Content-Length': '0'} 2022-09-03T07:30:50.492 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2be675079eac2e4-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.492 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.493 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.494 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.494 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:30:50.495 DEBUG:boto:Data: 2022-09-03T07:30:50.496 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.497 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.497 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.498 DEBUG:boto:Params: {} 2022-09-03T07:30:50.498 DEBUG:boto:Token: None 2022-09-03T07:30:50.499 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69 2022-09-03T07:30:50.500 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:v9p8Ikw+u+Lya+aTYuSFivIMKss= 2022-09-03T07:30:50.501 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:v9p8Ikw+u+Lya+aTYuSFivIMKss=', 'Content-Length': '0'} 2022-09-03T07:30:50.504 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f7823acf2fe35b6-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.506 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:30:50.507 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.508 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.509 DEBUG:boto:Method: GET 2022-09-03T07:30:50.511 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:30:50.512 DEBUG:boto:Data: 2022-09-03T07:30:50.512 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.513 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.514 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.514 DEBUG:boto:Params: {} 2022-09-03T07:30:50.515 DEBUG:boto:Token: None 2022-09-03T07:30:50.516 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69 2022-09-03T07:30:50.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G/CWCrx3SkGlagAwNqgzRVkKHr4= 2022-09-03T07:30:50.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G/CWCrx3SkGlagAwNqgzRVkKHr4=', 'Content-Length': '0'} 2022-09-03T07:30:50.519 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000daa1308d2e73e226-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.520 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:30:50.521 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:30:50.521 DEBUG:boto:Method: GET 2022-09-03T07:30:50.522 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:30:50.522 DEBUG:boto:Data: 2022-09-03T07:30:50.523 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.524 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.525 DEBUG:boto:Params: {} 2022-09-03T07:30:50.525 DEBUG:boto:Token: None 2022-09-03T07:30:50.526 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj69 2022-09-03T07:30:50.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G/CWCrx3SkGlagAwNqgzRVkKHr4= 2022-09-03T07:30:50.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G/CWCrx3SkGlagAwNqgzRVkKHr4=', 'Content-Length': '0'} 2022-09-03T07:30:50.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d90bf57f5c18a47a-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.533 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.534 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.534 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:30:50.535 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:30:50.535 DEBUG:boto:Method: GET 2022-09-03T07:30:50.536 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:30:50.536 DEBUG:boto:Data: 2022-09-03T07:30:50.537 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.538 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.538 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.539 DEBUG:boto:Params: {} 2022-09-03T07:30:50.539 DEBUG:boto:Token: None 2022-09-03T07:30:50.540 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.545 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b8d54bb193668096-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.546 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.547 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.548 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.548 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:30:50.549 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:30:50.549 DEBUG:boto:Method: GET 2022-09-03T07:30:50.550 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:30:50.550 DEBUG:boto:Data: 2022-09-03T07:30:50.551 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.552 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.553 DEBUG:boto:Params: {} 2022-09-03T07:30:50.554 DEBUG:boto:Token: None 2022-09-03T07:30:50.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.562 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad33a18a7c995062-00631302aa-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.602 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.603 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj74 2022-09-03T07:30:50.604 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.606 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.607 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.608 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.609 DEBUG:boto:Method: GET 2022-09-03T07:30:50.610 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.611 DEBUG:boto:Data: 2022-09-03T07:30:50.612 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.612 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.614 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.615 DEBUG:boto:Params: {} 2022-09-03T07:30:50.616 DEBUG:boto:Token: None 2022-09-03T07:30:50.617 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieQck77ZtdlJe992kMHssUn6FRg= 2022-09-03T07:30:50.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieQck77ZtdlJe992kMHssUn6FRg=', 'Content-Length': '0'} 2022-09-03T07:30:50.622 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a480eee38791b85-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.623 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.624 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.625 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.625 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.626 DEBUG:boto:Method: GET 2022-09-03T07:30:50.627 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.627 DEBUG:boto:Data: 2022-09-03T07:30:50.628 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.629 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.630 DEBUG:boto:Params: {} 2022-09-03T07:30:50.630 DEBUG:boto:Token: None 2022-09-03T07:30:50.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieQck77ZtdlJe992kMHssUn6FRg= 2022-09-03T07:30:50.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieQck77ZtdlJe992kMHssUn6FRg=', 'Content-Length': '0'} 2022-09-03T07:30:50.636 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000084f55ffc7f0e4869-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.637 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.637 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.639 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.639 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.640 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.640 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.641 DEBUG:boto:Data: 2022-09-03T07:30:50.641 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.642 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.643 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.644 DEBUG:boto:Params: {} 2022-09-03T07:30:50.645 DEBUG:boto:Token: None 2022-09-03T07:30:50.645 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DmKeEWjJPKkOKQt88oU7T5wRrRI= 2022-09-03T07:30:50.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DmKeEWjJPKkOKQt88oU7T5wRrRI=', 'Content-Length': '0'} 2022-09-03T07:30:50.650 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d92941cae2b1e5f-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.652 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.653 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.653 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.654 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.655 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.656 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.657 DEBUG:boto:Data: 2022-09-03T07:30:50.658 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.659 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.659 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.660 DEBUG:boto:Params: {} 2022-09-03T07:30:50.661 DEBUG:boto:Token: None 2022-09-03T07:30:50.661 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DmKeEWjJPKkOKQt88oU7T5wRrRI= 2022-09-03T07:30:50.663 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DmKeEWjJPKkOKQt88oU7T5wRrRI=', 'Content-Length': '0'} 2022-09-03T07:30:50.666 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3742de2b313e66a-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.667 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj74 2022-09-03T07:30:50.668 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.669 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.670 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.670 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.671 DEBUG:boto:Method: GET 2022-09-03T07:30:50.672 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.672 DEBUG:boto:Data: 2022-09-03T07:30:50.673 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.674 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.674 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.675 DEBUG:boto:Params: {} 2022-09-03T07:30:50.676 DEBUG:boto:Token: None 2022-09-03T07:30:50.677 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieQck77ZtdlJe992kMHssUn6FRg= 2022-09-03T07:30:50.678 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieQck77ZtdlJe992kMHssUn6FRg=', 'Content-Length': '0'} 2022-09-03T07:30:50.681 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d3353dd1cc09bdd6-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.682 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.682 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.683 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.684 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.685 DEBUG:boto:Method: GET 2022-09-03T07:30:50.686 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.687 DEBUG:boto:Data: 2022-09-03T07:30:50.688 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.689 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.692 DEBUG:boto:Params: {} 2022-09-03T07:30:50.692 DEBUG:boto:Token: None 2022-09-03T07:30:50.693 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:30:50.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieQck77ZtdlJe992kMHssUn6FRg= 2022-09-03T07:30:50.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieQck77ZtdlJe992kMHssUn6FRg=', 'Content-Length': '0'} 2022-09-03T07:30:50.696 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005945d91f0c4d0220-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.698 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.699 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.701 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:30:50.702 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:30:50.703 DEBUG:boto:Method: GET 2022-09-03T07:30:50.704 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:30:50.704 DEBUG:boto:Data: 2022-09-03T07:30:50.705 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.706 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.707 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.708 DEBUG:boto:Params: {} 2022-09-03T07:30:50.709 DEBUG:boto:Token: None 2022-09-03T07:30:50.710 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.711 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.712 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.715 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3b7a704ad90fe74-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.717 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.718 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.719 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.720 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:30:50.721 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:30:50.722 DEBUG:boto:Method: GET 2022-09-03T07:30:50.723 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:30:50.725 DEBUG:boto:Data: 2022-09-03T07:30:50.726 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.727 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.728 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.729 DEBUG:boto:Params: {} 2022-09-03T07:30:50.730 DEBUG:boto:Token: None 2022-09-03T07:30:50.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.738 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004d30fbf9e11b1a0a-00631302aa-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.740 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.741 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.742 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.743 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.744 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:30:50.745 DEBUG:boto:Data: 2022-09-03T07:30:50.745 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.746 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.747 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.748 DEBUG:boto:Params: {} 2022-09-03T07:30:50.749 DEBUG:boto:Token: None 2022-09-03T07:30:50.750 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74 2022-09-03T07:30:50.751 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RqzcccsJM0wwtZ5Du2VFe4/83ek= 2022-09-03T07:30:50.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RqzcccsJM0wwtZ5Du2VFe4/83ek=', 'Content-Length': '0'} 2022-09-03T07:30:50.754 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002cea73b274fa7a77-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.756 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.757 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.757 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.758 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:30:50.759 DEBUG:boto:Data: 2022-09-03T07:30:50.760 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.761 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.762 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.762 DEBUG:boto:Params: {} 2022-09-03T07:30:50.763 DEBUG:boto:Token: None 2022-09-03T07:30:50.764 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74 2022-09-03T07:30:50.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RqzcccsJM0wwtZ5Du2VFe4/83ek= 2022-09-03T07:30:50.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RqzcccsJM0wwtZ5Du2VFe4/83ek=', 'Content-Length': '0'} 2022-09-03T07:30:50.767 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b28590ef73c93809-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.768 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj74 2022-09-03T07:30:50.768 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.769 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.769 DEBUG:boto:Method: GET 2022-09-03T07:30:50.770 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:30:50.770 DEBUG:boto:Data: 2022-09-03T07:30:50.771 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.772 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.772 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.773 DEBUG:boto:Params: {} 2022-09-03T07:30:50.773 DEBUG:boto:Token: None 2022-09-03T07:30:50.774 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74 2022-09-03T07:30:50.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XAtg/IcUTVh1ZIuMJ8qRbLcoueA= 2022-09-03T07:30:50.775 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XAtg/IcUTVh1ZIuMJ8qRbLcoueA=', 'Content-Length': '0'} 2022-09-03T07:30:50.777 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023d31aa6e1a98fab-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.818 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:30:50.819 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:30:50.819 DEBUG:boto:Method: GET 2022-09-03T07:30:50.820 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:30:50.821 DEBUG:boto:Data: 2022-09-03T07:30:50.821 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.822 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.823 DEBUG:boto:Params: {} 2022-09-03T07:30:50.823 DEBUG:boto:Token: None 2022-09-03T07:30:50.824 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj74 2022-09-03T07:30:50.825 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XAtg/IcUTVh1ZIuMJ8qRbLcoueA= 2022-09-03T07:30:50.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XAtg/IcUTVh1ZIuMJ8qRbLcoueA=', 'Content-Length': '0'} 2022-09-03T07:30:50.827 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f038919862fba0b-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.828 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.829 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.830 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:30:50.830 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:30:50.831 DEBUG:boto:Method: GET 2022-09-03T07:30:50.831 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:30:50.832 DEBUG:boto:Data: 2022-09-03T07:30:50.832 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.833 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.833 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.834 DEBUG:boto:Params: {} 2022-09-03T07:30:50.835 DEBUG:boto:Token: None 2022-09-03T07:30:50.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.840 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003aa04e50fa6ad5d8-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.841 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.842 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.843 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.843 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:30:50.844 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:30:50.844 DEBUG:boto:Method: GET 2022-09-03T07:30:50.845 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:30:50.845 DEBUG:boto:Data: 2022-09-03T07:30:50.846 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.848 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.848 DEBUG:boto:Params: {} 2022-09-03T07:30:50.849 DEBUG:boto:Token: None 2022-09-03T07:30:50.849 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.859 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006363449706ccc6d6-00631302aa-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.860 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.861 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:30:50.862 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:50.863 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:50.864 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.865 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.866 DEBUG:boto:Method: GET 2022-09-03T07:30:50.867 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.867 DEBUG:boto:Data: 2022-09-03T07:30:50.868 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.869 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.870 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.871 DEBUG:boto:Params: {} 2022-09-03T07:30:50.872 DEBUG:boto:Token: None 2022-09-03T07:30:50.873 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsRHfP3fLA2Bsq10xkZ+fbWgOhI= 2022-09-03T07:30:50.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsRHfP3fLA2Bsq10xkZ+fbWgOhI=', 'Content-Length': '0'} 2022-09-03T07:30:50.878 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000029436b98ed8d12d9-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.881 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:50.882 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:50.883 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.883 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.884 DEBUG:boto:Method: GET 2022-09-03T07:30:50.884 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.885 DEBUG:boto:Data: 2022-09-03T07:30:50.885 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.887 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.887 DEBUG:boto:Params: {} 2022-09-03T07:30:50.889 DEBUG:boto:Token: None 2022-09-03T07:30:50.889 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsRHfP3fLA2Bsq10xkZ+fbWgOhI= 2022-09-03T07:30:50.892 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsRHfP3fLA2Bsq10xkZ+fbWgOhI=', 'Content-Length': '0'} 2022-09-03T07:30:50.895 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf470be6a5a3772a-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.942 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:50.943 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:50.943 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.944 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.944 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.945 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.945 DEBUG:boto:Data: 2022-09-03T07:30:50.946 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.946 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.947 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.947 DEBUG:boto:Params: {} 2022-09-03T07:30:50.948 DEBUG:boto:Token: None 2022-09-03T07:30:50.949 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/aoEdpXg7/7yTz+ffhclXEHmUFg= 2022-09-03T07:30:50.950 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/aoEdpXg7/7yTz+ffhclXEHmUFg=', 'Content-Length': '0'} 2022-09-03T07:30:50.952 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000920d0f9466c1dfc7-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.953 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:50.953 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:50.954 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.954 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.955 DEBUG:boto:Method: HEAD 2022-09-03T07:30:50.955 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.956 DEBUG:boto:Data: 2022-09-03T07:30:50.956 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.957 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.957 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.958 DEBUG:boto:Params: {} 2022-09-03T07:30:50.958 DEBUG:boto:Token: None 2022-09-03T07:30:50.959 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.959 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/aoEdpXg7/7yTz+ffhclXEHmUFg= 2022-09-03T07:30:50.960 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/aoEdpXg7/7yTz+ffhclXEHmUFg=', 'Content-Length': '0'} 2022-09-03T07:30:50.962 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f776f4f167beb59-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.963 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:30:50.963 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:50.964 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:50.965 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.965 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.966 DEBUG:boto:Method: GET 2022-09-03T07:30:50.966 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.967 DEBUG:boto:Data: 2022-09-03T07:30:50.967 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.968 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.968 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.969 DEBUG:boto:Params: {} 2022-09-03T07:30:50.970 DEBUG:boto:Token: None 2022-09-03T07:30:50.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsRHfP3fLA2Bsq10xkZ+fbWgOhI= 2022-09-03T07:30:50.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsRHfP3fLA2Bsq10xkZ+fbWgOhI=', 'Content-Length': '0'} 2022-09-03T07:30:50.974 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000150b3d2f558f5577-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.975 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:50.975 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:50.976 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.976 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.977 DEBUG:boto:Method: GET 2022-09-03T07:30:50.977 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.978 DEBUG:boto:Data: 2022-09-03T07:30:50.978 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:50.979 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.980 DEBUG:boto:Params: {} 2022-09-03T07:30:50.981 DEBUG:boto:Token: None 2022-09-03T07:30:50.981 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:30:50.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UsRHfP3fLA2Bsq10xkZ+fbWgOhI= 2022-09-03T07:30:50.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UsRHfP3fLA2Bsq10xkZ+fbWgOhI=', 'Content-Length': '0'} 2022-09-03T07:30:50.985 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000059f39f1d2ad979e4-00631302aa-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.986 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.986 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.987 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:30:50.987 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:30:50.988 DEBUG:boto:Method: GET 2022-09-03T07:30:50.989 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:30:50.989 DEBUG:boto:Data: 2022-09-03T07:30:50.990 DEBUG:boto:Headers: {} 2022-09-03T07:30:50.990 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:50.991 DEBUG:boto:Port: 8000 2022-09-03T07:30:50.991 DEBUG:boto:Params: {} 2022-09-03T07:30:50.992 DEBUG:boto:Token: None 2022-09-03T07:30:50.992 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:50 GMT /avysmw-75/ 2022-09-03T07:30:50.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo= 2022-09-03T07:30:50.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:50 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:q5CvN97uasvG2lOheezr2hel3Uo=', 'Content-Length': '0'} 2022-09-03T07:30:50.996 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000defd95aa38e6f801-00631302aa-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:50 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:50.997 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:50.997 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:50.998 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:50.999 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:30:50.999 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:30:51.000 DEBUG:boto:Method: GET 2022-09-03T07:30:51.000 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:30:51.001 DEBUG:boto:Data: 2022-09-03T07:30:51.001 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.002 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.002 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.003 DEBUG:boto:Params: {} 2022-09-03T07:30:51.003 DEBUG:boto:Token: None 2022-09-03T07:30:51.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.005 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ce3f48ca808e0e14-00631302ab-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.012 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.013 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:51.014 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:51.015 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.016 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:30:51.017 DEBUG:boto:Data: 2022-09-03T07:30:51.018 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.019 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.020 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.020 DEBUG:boto:Params: {} 2022-09-03T07:30:51.021 DEBUG:boto:Token: None 2022-09-03T07:30:51.022 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj76 2022-09-03T07:30:51.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TAKW0wbLO8O0oj8Uw0W1U42YXNU= 2022-09-03T07:30:51.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TAKW0wbLO8O0oj8Uw0W1U42YXNU=', 'Content-Length': '0'} 2022-09-03T07:30:51.026 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006772a4821153c5ee-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.027 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:51.030 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:51.031 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.031 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:30:51.032 DEBUG:boto:Data: 2022-09-03T07:30:51.032 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.033 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.034 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.034 DEBUG:boto:Params: {} 2022-09-03T07:30:51.035 DEBUG:boto:Token: None 2022-09-03T07:30:51.035 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj76 2022-09-03T07:30:51.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TAKW0wbLO8O0oj8Uw0W1U42YXNU= 2022-09-03T07:30:51.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TAKW0wbLO8O0oj8Uw0W1U42YXNU=', 'Content-Length': '0'} 2022-09-03T07:30:51.040 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3488891d0dca33b-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.041 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:30:51.041 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:51.042 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:51.042 DEBUG:boto:Method: GET 2022-09-03T07:30:51.043 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:30:51.043 DEBUG:boto:Data: 2022-09-03T07:30:51.044 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.044 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.045 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.046 DEBUG:boto:Params: {} 2022-09-03T07:30:51.046 DEBUG:boto:Token: None 2022-09-03T07:30:51.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj76 2022-09-03T07:30:51.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U5fTRCudWjxsHQnIh9NZgJiMo7k= 2022-09-03T07:30:51.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U5fTRCudWjxsHQnIh9NZgJiMo7k=', 'Content-Length': '0'} 2022-09-03T07:30:51.050 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000057c3479a4c19ee5-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.051 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:30:51.052 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:30:51.052 DEBUG:boto:Method: GET 2022-09-03T07:30:51.053 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:30:51.053 DEBUG:boto:Data: 2022-09-03T07:30:51.054 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.055 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.055 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.056 DEBUG:boto:Params: {} 2022-09-03T07:30:51.056 DEBUG:boto:Token: None 2022-09-03T07:30:51.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj76 2022-09-03T07:30:51.057 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:U5fTRCudWjxsHQnIh9NZgJiMo7k= 2022-09-03T07:30:51.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:U5fTRCudWjxsHQnIh9NZgJiMo7k=', 'Content-Length': '0'} 2022-09-03T07:30:51.061 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008a425947eae24f0-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.106 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.107 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.107 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:30:51.108 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:30:51.109 DEBUG:boto:Method: GET 2022-09-03T07:30:51.109 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:30:51.110 DEBUG:boto:Data: 2022-09-03T07:30:51.110 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.111 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.111 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.112 DEBUG:boto:Params: {} 2022-09-03T07:30:51.112 DEBUG:boto:Token: None 2022-09-03T07:30:51.113 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.114 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.117 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000044bc8aad5977ea1b-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.118 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.119 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.119 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.120 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:30:51.120 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:30:51.121 DEBUG:boto:Method: GET 2022-09-03T07:30:51.121 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:30:51.122 DEBUG:boto:Data: 2022-09-03T07:30:51.122 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.123 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.124 DEBUG:boto:Params: {} 2022-09-03T07:30:51.125 DEBUG:boto:Token: None 2022-09-03T07:30:51.125 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.132 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a0c08007f92b8abe-00631302ab-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.133 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.135 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:30:51.135 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.136 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.137 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.137 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.138 DEBUG:boto:Method: GET 2022-09-03T07:30:51.139 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.140 DEBUG:boto:Data: 2022-09-03T07:30:51.141 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.142 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.143 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.144 DEBUG:boto:Params: {} 2022-09-03T07:30:51.145 DEBUG:boto:Token: None 2022-09-03T07:30:51.146 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.147 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iP9Sn+386TlSwGk8+NCCMMHNOT0= 2022-09-03T07:30:51.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iP9Sn+386TlSwGk8+NCCMMHNOT0=', 'Content-Length': '0'} 2022-09-03T07:30:51.151 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed7c5ca5495efc6d-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.152 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.153 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.154 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.155 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.156 DEBUG:boto:Method: GET 2022-09-03T07:30:51.157 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.158 DEBUG:boto:Data: 2022-09-03T07:30:51.158 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.159 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.159 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.160 DEBUG:boto:Params: {} 2022-09-03T07:30:51.160 DEBUG:boto:Token: None 2022-09-03T07:30:51.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.162 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iP9Sn+386TlSwGk8+NCCMMHNOT0= 2022-09-03T07:30:51.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iP9Sn+386TlSwGk8+NCCMMHNOT0=', 'Content-Length': '0'} 2022-09-03T07:30:51.165 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de6b758cc83a9664-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.165 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.166 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.167 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.167 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.168 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.169 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.169 DEBUG:boto:Data: 2022-09-03T07:30:51.170 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.170 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.171 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.171 DEBUG:boto:Params: {} 2022-09-03T07:30:51.172 DEBUG:boto:Token: None 2022-09-03T07:30:51.173 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2FeRR7mvw/pLRoBVvEfEWa08G+E= 2022-09-03T07:30:51.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2FeRR7mvw/pLRoBVvEfEWa08G+E=', 'Content-Length': '0'} 2022-09-03T07:30:51.180 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cffa90495657e60-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.182 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.183 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.184 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.185 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.186 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.186 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.187 DEBUG:boto:Data: 2022-09-03T07:30:51.188 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.189 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.191 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.192 DEBUG:boto:Params: {} 2022-09-03T07:30:51.193 DEBUG:boto:Token: None 2022-09-03T07:30:51.194 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.195 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2FeRR7mvw/pLRoBVvEfEWa08G+E= 2022-09-03T07:30:51.196 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2FeRR7mvw/pLRoBVvEfEWa08G+E=', 'Content-Length': '0'} 2022-09-03T07:30:51.199 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df67f0654412e37c-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.200 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:30:51.201 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.202 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.203 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.204 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.205 DEBUG:boto:Method: GET 2022-09-03T07:30:51.206 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.207 DEBUG:boto:Data: 2022-09-03T07:30:51.208 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.209 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.210 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.211 DEBUG:boto:Params: {} 2022-09-03T07:30:51.211 DEBUG:boto:Token: None 2022-09-03T07:30:51.213 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iP9Sn+386TlSwGk8+NCCMMHNOT0= 2022-09-03T07:30:51.215 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iP9Sn+386TlSwGk8+NCCMMHNOT0=', 'Content-Length': '0'} 2022-09-03T07:30:51.217 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d59e6cdcd183fa7d-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.219 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.220 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.221 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.221 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.222 DEBUG:boto:Method: GET 2022-09-03T07:30:51.222 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.223 DEBUG:boto:Data: 2022-09-03T07:30:51.224 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.224 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.225 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.226 DEBUG:boto:Params: {} 2022-09-03T07:30:51.227 DEBUG:boto:Token: None 2022-09-03T07:30:51.228 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:30:51.229 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iP9Sn+386TlSwGk8+NCCMMHNOT0= 2022-09-03T07:30:51.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iP9Sn+386TlSwGk8+NCCMMHNOT0=', 'Content-Length': '0'} 2022-09-03T07:30:51.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a04367fad1b3ef7c-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.274 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.275 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.276 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:30:51.277 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:30:51.278 DEBUG:boto:Method: GET 2022-09-03T07:30:51.279 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:30:51.280 DEBUG:boto:Data: 2022-09-03T07:30:51.281 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.283 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.284 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.285 DEBUG:boto:Params: {} 2022-09-03T07:30:51.286 DEBUG:boto:Token: None 2022-09-03T07:30:51.287 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.292 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000054bd5710e24a3b6e-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.292 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.294 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.295 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.295 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:30:51.296 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:30:51.297 DEBUG:boto:Method: GET 2022-09-03T07:30:51.298 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:30:51.298 DEBUG:boto:Data: 2022-09-03T07:30:51.299 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.299 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.300 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.301 DEBUG:boto:Params: {} 2022-09-03T07:30:51.301 DEBUG:boto:Token: None 2022-09-03T07:30:51.302 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.308 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000091f63b20b1711a27-00631302ab-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.310 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.312 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.312 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.313 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.315 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:30:51.316 DEBUG:boto:Data: 2022-09-03T07:30:51.317 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.318 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.319 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.320 DEBUG:boto:Params: {} 2022-09-03T07:30:51.321 DEBUG:boto:Token: None 2022-09-03T07:30:51.322 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85 2022-09-03T07:30:51.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kNP3H1KaHt4ZLpPEB+XgqYTXTH8= 2022-09-03T07:30:51.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kNP3H1KaHt4ZLpPEB+XgqYTXTH8=', 'Content-Length': '0'} 2022-09-03T07:30:51.326 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cae6a4812cdcf0e6-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.327 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.328 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.329 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.330 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:30:51.331 DEBUG:boto:Data: 2022-09-03T07:30:51.332 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.333 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.334 DEBUG:boto:Params: {} 2022-09-03T07:30:51.335 DEBUG:boto:Token: None 2022-09-03T07:30:51.336 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85 2022-09-03T07:30:51.337 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kNP3H1KaHt4ZLpPEB+XgqYTXTH8= 2022-09-03T07:30:51.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kNP3H1KaHt4ZLpPEB+XgqYTXTH8=', 'Content-Length': '0'} 2022-09-03T07:30:51.340 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f5a4259e80b6c96a-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.341 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:30:51.342 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.343 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.344 DEBUG:boto:Method: GET 2022-09-03T07:30:51.345 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:30:51.346 DEBUG:boto:Data: 2022-09-03T07:30:51.347 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.348 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.349 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.349 DEBUG:boto:Params: {} 2022-09-03T07:30:51.350 DEBUG:boto:Token: None 2022-09-03T07:30:51.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85 2022-09-03T07:30:51.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/rH1FwCaNfpSGSfe2MFOHDPDxQ= 2022-09-03T07:30:51.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/rH1FwCaNfpSGSfe2MFOHDPDxQ=', 'Content-Length': '0'} 2022-09-03T07:30:51.354 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000080a484fddf9e7945-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.355 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:30:51.356 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:30:51.356 DEBUG:boto:Method: GET 2022-09-03T07:30:51.357 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:30:51.357 DEBUG:boto:Data: 2022-09-03T07:30:51.358 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.359 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.360 DEBUG:boto:Params: {} 2022-09-03T07:30:51.360 DEBUG:boto:Token: None 2022-09-03T07:30:51.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj85 2022-09-03T07:30:51.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:e/rH1FwCaNfpSGSfe2MFOHDPDxQ= 2022-09-03T07:30:51.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:e/rH1FwCaNfpSGSfe2MFOHDPDxQ=', 'Content-Length': '0'} 2022-09-03T07:30:51.365 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c52003d7dd3bbd7a-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.366 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.367 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.367 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:30:51.368 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:30:51.368 DEBUG:boto:Method: GET 2022-09-03T07:30:51.369 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:30:51.370 DEBUG:boto:Data: 2022-09-03T07:30:51.371 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.371 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.372 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.372 DEBUG:boto:Params: {} 2022-09-03T07:30:51.373 DEBUG:boto:Token: None 2022-09-03T07:30:51.374 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.374 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.377 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000961338b7b796f2ab-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.418 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.420 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.422 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.424 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:30:51.425 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:30:51.426 DEBUG:boto:Method: GET 2022-09-03T07:30:51.427 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:30:51.428 DEBUG:boto:Data: 2022-09-03T07:30:51.429 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.430 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.431 DEBUG:boto:Params: {} 2022-09-03T07:30:51.433 DEBUG:boto:Token: None 2022-09-03T07:30:51.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.441 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001e6436f11115d722-00631302ab-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.443 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.444 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:30:51.445 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.446 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.447 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.448 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.449 DEBUG:boto:Method: GET 2022-09-03T07:30:51.450 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.451 DEBUG:boto:Data: 2022-09-03T07:30:51.452 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.453 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.454 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.455 DEBUG:boto:Params: {} 2022-09-03T07:30:51.455 DEBUG:boto:Token: None 2022-09-03T07:30:51.456 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bhPMPqYFJvxGf+LbjyR8UmQPZkE= 2022-09-03T07:30:51.457 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bhPMPqYFJvxGf+LbjyR8UmQPZkE=', 'Content-Length': '0'} 2022-09-03T07:30:51.460 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf72edd826f15ba6-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.460 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.461 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.462 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.462 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.463 DEBUG:boto:Method: GET 2022-09-03T07:30:51.463 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.464 DEBUG:boto:Data: 2022-09-03T07:30:51.464 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.465 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.466 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.466 DEBUG:boto:Params: {} 2022-09-03T07:30:51.467 DEBUG:boto:Token: None 2022-09-03T07:30:51.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bhPMPqYFJvxGf+LbjyR8UmQPZkE= 2022-09-03T07:30:51.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bhPMPqYFJvxGf+LbjyR8UmQPZkE=', 'Content-Length': '0'} 2022-09-03T07:30:51.471 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000399dfb29ff31d689-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.514 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.515 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.516 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.517 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.517 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.520 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.520 DEBUG:boto:Data: 2022-09-03T07:30:51.521 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.521 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.522 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.523 DEBUG:boto:Params: {} 2022-09-03T07:30:51.523 DEBUG:boto:Token: None 2022-09-03T07:30:51.524 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.525 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m5mFPjYvAVYYFSRAQ6uE1pb8tOA= 2022-09-03T07:30:51.525 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m5mFPjYvAVYYFSRAQ6uE1pb8tOA=', 'Content-Length': '0'} 2022-09-03T07:30:51.528 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000969099140423e1c3-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.529 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.530 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.531 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.531 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.532 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.533 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.534 DEBUG:boto:Data: 2022-09-03T07:30:51.534 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.536 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.537 DEBUG:boto:Params: {} 2022-09-03T07:30:51.538 DEBUG:boto:Token: None 2022-09-03T07:30:51.541 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:m5mFPjYvAVYYFSRAQ6uE1pb8tOA= 2022-09-03T07:30:51.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:m5mFPjYvAVYYFSRAQ6uE1pb8tOA=', 'Content-Length': '0'} 2022-09-03T07:30:51.545 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000025ce4db77f68079e-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.547 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:30:51.548 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.549 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.551 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.552 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.554 DEBUG:boto:Method: GET 2022-09-03T07:30:51.555 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.556 DEBUG:boto:Data: 2022-09-03T07:30:51.557 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.558 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.559 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.559 DEBUG:boto:Params: {} 2022-09-03T07:30:51.560 DEBUG:boto:Token: None 2022-09-03T07:30:51.561 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.563 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bhPMPqYFJvxGf+LbjyR8UmQPZkE= 2022-09-03T07:30:51.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bhPMPqYFJvxGf+LbjyR8UmQPZkE=', 'Content-Length': '0'} 2022-09-03T07:30:51.568 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e4c67e483403159c-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.568 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.569 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.570 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.571 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.572 DEBUG:boto:Method: GET 2022-09-03T07:30:51.573 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.575 DEBUG:boto:Data: 2022-09-03T07:30:51.576 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.578 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.579 DEBUG:boto:Params: {} 2022-09-03T07:30:51.580 DEBUG:boto:Token: None 2022-09-03T07:30:51.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:30:51.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bhPMPqYFJvxGf+LbjyR8UmQPZkE= 2022-09-03T07:30:51.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bhPMPqYFJvxGf+LbjyR8UmQPZkE=', 'Content-Length': '0'} 2022-09-03T07:30:51.586 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af4457f322c58635-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.587 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.588 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.588 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:30:51.589 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:30:51.589 DEBUG:boto:Method: GET 2022-09-03T07:30:51.590 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:30:51.590 DEBUG:boto:Data: 2022-09-03T07:30:51.591 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.591 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.592 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.593 DEBUG:boto:Params: {} 2022-09-03T07:30:51.593 DEBUG:boto:Token: None 2022-09-03T07:30:51.594 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.594 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.595 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.598 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004baef295e6b63596-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.598 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.599 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.600 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.600 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:30:51.601 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:30:51.601 DEBUG:boto:Method: GET 2022-09-03T07:30:51.602 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:30:51.603 DEBUG:boto:Data: 2022-09-03T07:30:51.603 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.604 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.604 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.605 DEBUG:boto:Params: {} 2022-09-03T07:30:51.605 DEBUG:boto:Token: None 2022-09-03T07:30:51.606 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.607 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.613 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075fabe6a69ed4b8b-00631302ab-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.613 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.615 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.615 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.616 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.616 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:30:51.617 DEBUG:boto:Data: 2022-09-03T07:30:51.618 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.619 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.620 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.622 DEBUG:boto:Params: {} 2022-09-03T07:30:51.623 DEBUG:boto:Token: None 2022-09-03T07:30:51.624 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87 2022-09-03T07:30:51.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JRjJ1h9mEf9gmqK+TZJxSpLY/BY= 2022-09-03T07:30:51.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JRjJ1h9mEf9gmqK+TZJxSpLY/BY=', 'Content-Length': '0'} 2022-09-03T07:30:51.629 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000598cbea8e840fa4d-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.630 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.630 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.631 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.632 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:30:51.632 DEBUG:boto:Data: 2022-09-03T07:30:51.633 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.633 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.634 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.634 DEBUG:boto:Params: {} 2022-09-03T07:30:51.635 DEBUG:boto:Token: None 2022-09-03T07:30:51.636 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87 2022-09-03T07:30:51.637 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JRjJ1h9mEf9gmqK+TZJxSpLY/BY= 2022-09-03T07:30:51.637 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JRjJ1h9mEf9gmqK+TZJxSpLY/BY=', 'Content-Length': '0'} 2022-09-03T07:30:51.639 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008efde83bc33df1e9-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.641 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:30:51.642 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.643 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.644 DEBUG:boto:Method: GET 2022-09-03T07:30:51.645 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:30:51.645 DEBUG:boto:Data: 2022-09-03T07:30:51.646 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.646 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.647 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.647 DEBUG:boto:Params: {} 2022-09-03T07:30:51.648 DEBUG:boto:Token: None 2022-09-03T07:30:51.649 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87 2022-09-03T07:30:51.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kt77lXYburmRLhDO5lVghs3cZz8= 2022-09-03T07:30:51.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kt77lXYburmRLhDO5lVghs3cZz8=', 'Content-Length': '0'} 2022-09-03T07:30:51.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dee64836c5887a06-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.653 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:30:51.653 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:30:51.654 DEBUG:boto:Method: GET 2022-09-03T07:30:51.654 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:30:51.655 DEBUG:boto:Data: 2022-09-03T07:30:51.655 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.656 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.657 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.657 DEBUG:boto:Params: {} 2022-09-03T07:30:51.658 DEBUG:boto:Token: None 2022-09-03T07:30:51.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj87 2022-09-03T07:30:51.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kt77lXYburmRLhDO5lVghs3cZz8= 2022-09-03T07:30:51.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kt77lXYburmRLhDO5lVghs3cZz8=', 'Content-Length': '0'} 2022-09-03T07:30:51.663 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d119a3cc903da0a-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.710 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.711 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.712 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:30:51.713 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:30:51.714 DEBUG:boto:Method: GET 2022-09-03T07:30:51.715 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:30:51.716 DEBUG:boto:Data: 2022-09-03T07:30:51.716 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.717 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.718 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.719 DEBUG:boto:Params: {} 2022-09-03T07:30:51.720 DEBUG:boto:Token: None 2022-09-03T07:30:51.721 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.722 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.723 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.726 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000019b8be17f8ad96f8-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.727 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.728 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.728 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.729 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:30:51.729 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:30:51.730 DEBUG:boto:Method: GET 2022-09-03T07:30:51.731 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:30:51.731 DEBUG:boto:Data: 2022-09-03T07:30:51.732 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.733 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.733 DEBUG:boto:Params: {} 2022-09-03T07:30:51.734 DEBUG:boto:Token: None 2022-09-03T07:30:51.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.747 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007aa33c637b18254c-00631302ab-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.749 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.750 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:30:51.751 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.752 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.753 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.754 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.755 DEBUG:boto:Method: GET 2022-09-03T07:30:51.756 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.756 DEBUG:boto:Data: 2022-09-03T07:30:51.758 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.759 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.760 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.761 DEBUG:boto:Params: {} 2022-09-03T07:30:51.762 DEBUG:boto:Token: None 2022-09-03T07:30:51.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qHTsJLU4Gg04xaitvWrI9O8GIXI= 2022-09-03T07:30:51.765 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qHTsJLU4Gg04xaitvWrI9O8GIXI=', 'Content-Length': '0'} 2022-09-03T07:30:51.768 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070294bac0daa9f02-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.769 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.769 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.770 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.770 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.771 DEBUG:boto:Method: GET 2022-09-03T07:30:51.772 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.772 DEBUG:boto:Data: 2022-09-03T07:30:51.773 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.774 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.775 DEBUG:boto:Params: {} 2022-09-03T07:30:51.775 DEBUG:boto:Token: None 2022-09-03T07:30:51.776 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qHTsJLU4Gg04xaitvWrI9O8GIXI= 2022-09-03T07:30:51.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qHTsJLU4Gg04xaitvWrI9O8GIXI=', 'Content-Length': '0'} 2022-09-03T07:30:51.780 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e05381ba9389a7a5-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.780 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.781 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.782 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.782 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.783 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.784 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.784 DEBUG:boto:Data: 2022-09-03T07:30:51.785 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.785 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.786 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.786 DEBUG:boto:Params: {} 2022-09-03T07:30:51.787 DEBUG:boto:Token: None 2022-09-03T07:30:51.787 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8re/uyN0mi8k5WHC+gkkdLfshi0= 2022-09-03T07:30:51.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8re/uyN0mi8k5WHC+gkkdLfshi0=', 'Content-Length': '0'} 2022-09-03T07:30:51.791 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071a43f16e5c4be75-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.792 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.793 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.794 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.795 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.796 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.797 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.799 DEBUG:boto:Data: 2022-09-03T07:30:51.800 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.801 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.802 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.803 DEBUG:boto:Params: {} 2022-09-03T07:30:51.804 DEBUG:boto:Token: None 2022-09-03T07:30:51.805 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8re/uyN0mi8k5WHC+gkkdLfshi0= 2022-09-03T07:30:51.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8re/uyN0mi8k5WHC+gkkdLfshi0=', 'Content-Length': '0'} 2022-09-03T07:30:51.812 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078beb4a47d922925-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.813 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:30:51.815 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.817 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.818 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.819 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.820 DEBUG:boto:Method: GET 2022-09-03T07:30:51.821 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.821 DEBUG:boto:Data: 2022-09-03T07:30:51.822 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.823 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.824 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.825 DEBUG:boto:Params: {} 2022-09-03T07:30:51.825 DEBUG:boto:Token: None 2022-09-03T07:30:51.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.827 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qHTsJLU4Gg04xaitvWrI9O8GIXI= 2022-09-03T07:30:51.828 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qHTsJLU4Gg04xaitvWrI9O8GIXI=', 'Content-Length': '0'} 2022-09-03T07:30:51.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3223e26cdb6b135-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.832 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.833 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.834 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.836 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.837 DEBUG:boto:Method: GET 2022-09-03T07:30:51.837 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.838 DEBUG:boto:Data: 2022-09-03T07:30:51.839 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.840 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.841 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.842 DEBUG:boto:Params: {} 2022-09-03T07:30:51.843 DEBUG:boto:Token: None 2022-09-03T07:30:51.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:30:51.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qHTsJLU4Gg04xaitvWrI9O8GIXI= 2022-09-03T07:30:51.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qHTsJLU4Gg04xaitvWrI9O8GIXI=', 'Content-Length': '0'} 2022-09-03T07:30:51.850 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f777bfadcfbfb8a-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.851 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.852 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.853 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:30:51.853 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:30:51.854 DEBUG:boto:Method: GET 2022-09-03T07:30:51.854 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:30:51.855 DEBUG:boto:Data: 2022-09-03T07:30:51.855 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.856 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.856 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.857 DEBUG:boto:Params: {} 2022-09-03T07:30:51.858 DEBUG:boto:Token: None 2022-09-03T07:30:51.858 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.862 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090df52aea7df9215-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.864 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.865 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.866 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.866 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:30:51.867 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:30:51.867 DEBUG:boto:Method: GET 2022-09-03T07:30:51.868 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:30:51.868 DEBUG:boto:Data: 2022-09-03T07:30:51.869 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.869 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.870 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.871 DEBUG:boto:Params: {} 2022-09-03T07:30:51.871 DEBUG:boto:Token: None 2022-09-03T07:30:51.872 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/ 2022-09-03T07:30:51.874 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E= 2022-09-03T07:30:51.875 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rNy96/uwMHewrlcodg64td7Vk6E=', 'Content-Length': '0'} 2022-09-03T07:30:51.880 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a57af39dc05ae36-00631302ab-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.881 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:51.883 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.884 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.885 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.885 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:30:51.886 DEBUG:boto:Data: 2022-09-03T07:30:51.886 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.887 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.887 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.888 DEBUG:boto:Params: {} 2022-09-03T07:30:51.888 DEBUG:boto:Token: None 2022-09-03T07:30:51.889 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92 2022-09-03T07:30:51.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:etGn0DZdYX7/x657Zcnz3MRkC1E= 2022-09-03T07:30:51.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:etGn0DZdYX7/x657Zcnz3MRkC1E=', 'Content-Length': '0'} 2022-09-03T07:30:51.892 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000182052bf83b0444-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.894 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.895 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.896 DEBUG:boto:Method: HEAD 2022-09-03T07:30:51.897 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:30:51.898 DEBUG:boto:Data: 2022-09-03T07:30:51.899 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.900 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.901 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.902 DEBUG:boto:Params: {} 2022-09-03T07:30:51.903 DEBUG:boto:Token: None 2022-09-03T07:30:51.903 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92 2022-09-03T07:30:51.904 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:etGn0DZdYX7/x657Zcnz3MRkC1E= 2022-09-03T07:30:51.905 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:etGn0DZdYX7/x657Zcnz3MRkC1E=', 'Content-Length': '0'} 2022-09-03T07:30:51.907 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004733328cc7abd1bd-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.908 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:30:51.909 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.909 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.910 DEBUG:boto:Method: GET 2022-09-03T07:30:51.910 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:30:51.911 DEBUG:boto:Data: 2022-09-03T07:30:51.911 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.912 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:51.913 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.913 DEBUG:boto:Params: {} 2022-09-03T07:30:51.914 DEBUG:boto:Token: None 2022-09-03T07:30:51.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92 2022-09-03T07:30:51.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VepFVO40bmsQGi1IPqbIJS72QE4= 2022-09-03T07:30:51.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VepFVO40bmsQGi1IPqbIJS72QE4=', 'Content-Length': '0'} 2022-09-03T07:30:51.918 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2d4077d912216fc-00631302ab-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.972 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:30:51.973 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:30:51.974 DEBUG:boto:Method: GET 2022-09-03T07:30:51.975 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:30:51.975 DEBUG:boto:Data: 2022-09-03T07:30:51.976 DEBUG:boto:Headers: {} 2022-09-03T07:30:51.977 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:51.978 DEBUG:boto:Port: 8000 2022-09-03T07:30:51.979 DEBUG:boto:Params: {} 2022-09-03T07:30:51.990 DEBUG:boto:Token: None 2022-09-03T07:30:51.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:51 GMT /avysmw-75/obj92 2022-09-03T07:30:51.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VepFVO40bmsQGi1IPqbIJS72QE4= 2022-09-03T07:30:51.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:51 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VepFVO40bmsQGi1IPqbIJS72QE4=', 'Content-Length': '0'} 2022-09-03T07:30:51.997 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054f5325887a534d2-00631302ab-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:51 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:51.998 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:51.999 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:51.999 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:30:52.000 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:30:52.001 DEBUG:boto:Method: GET 2022-09-03T07:30:52.001 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:30:52.002 DEBUG:boto:Data: 2022-09-03T07:30:52.003 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.003 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.004 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.004 DEBUG:boto:Params: {} 2022-09-03T07:30:52.005 DEBUG:boto:Token: None 2022-09-03T07:30:52.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.009 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b2b868e606fec8fe-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.010 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.011 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.012 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.012 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:30:52.013 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:30:52.013 DEBUG:boto:Method: GET 2022-09-03T07:30:52.014 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:30:52.015 DEBUG:boto:Data: 2022-09-03T07:30:52.015 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.016 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.017 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.017 DEBUG:boto:Params: {} 2022-09-03T07:30:52.018 DEBUG:boto:Token: None 2022-09-03T07:30:52.018 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.019 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.020 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.025 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000607e57808b4db7dd-00631302ac-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.026 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.027 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:30:52.027 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.028 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.029 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.029 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.030 DEBUG:boto:Method: GET 2022-09-03T07:30:52.031 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.031 DEBUG:boto:Data: 2022-09-03T07:30:52.032 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.033 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.034 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.040 DEBUG:boto:Params: {} 2022-09-03T07:30:52.041 DEBUG:boto:Token: None 2022-09-03T07:30:52.042 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6WfdPJJxRFTlaxO01oUBxk0uZds= 2022-09-03T07:30:52.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6WfdPJJxRFTlaxO01oUBxk0uZds=', 'Content-Length': '0'} 2022-09-03T07:30:52.048 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003d59aff520f83fb1-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.052 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.053 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.054 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.055 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.056 DEBUG:boto:Method: GET 2022-09-03T07:30:52.057 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.058 DEBUG:boto:Data: 2022-09-03T07:30:52.059 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.062 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.063 DEBUG:boto:Params: {} 2022-09-03T07:30:52.064 DEBUG:boto:Token: None 2022-09-03T07:30:52.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6WfdPJJxRFTlaxO01oUBxk0uZds= 2022-09-03T07:30:52.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6WfdPJJxRFTlaxO01oUBxk0uZds=', 'Content-Length': '0'} 2022-09-03T07:30:52.069 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6ef90203803b7da-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.070 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.071 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.072 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.073 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.074 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.074 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.075 DEBUG:boto:Data: 2022-09-03T07:30:52.075 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.076 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.076 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.077 DEBUG:boto:Params: {} 2022-09-03T07:30:52.077 DEBUG:boto:Token: None 2022-09-03T07:30:52.078 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.079 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vNIC7Ve2+rk626FRQZ9pb78jFc8= 2022-09-03T07:30:52.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vNIC7Ve2+rk626FRQZ9pb78jFc8=', 'Content-Length': '0'} 2022-09-03T07:30:52.082 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008b3483e79caff1bf-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.082 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.083 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.083 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.084 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.085 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.085 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.086 DEBUG:boto:Data: 2022-09-03T07:30:52.087 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.087 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.088 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.088 DEBUG:boto:Params: {} 2022-09-03T07:30:52.089 DEBUG:boto:Token: None 2022-09-03T07:30:52.089 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vNIC7Ve2+rk626FRQZ9pb78jFc8= 2022-09-03T07:30:52.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vNIC7Ve2+rk626FRQZ9pb78jFc8=', 'Content-Length': '0'} 2022-09-03T07:30:52.093 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ceec030315c6cc9a-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.093 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:30:52.094 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.094 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.095 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.095 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.096 DEBUG:boto:Method: GET 2022-09-03T07:30:52.096 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.097 DEBUG:boto:Data: 2022-09-03T07:30:52.098 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.098 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.099 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.099 DEBUG:boto:Params: {} 2022-09-03T07:30:52.100 DEBUG:boto:Token: None 2022-09-03T07:30:52.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6WfdPJJxRFTlaxO01oUBxk0uZds= 2022-09-03T07:30:52.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6WfdPJJxRFTlaxO01oUBxk0uZds=', 'Content-Length': '0'} 2022-09-03T07:30:52.104 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046e29363eeec1636-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.104 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.105 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.106 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.106 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.107 DEBUG:boto:Method: GET 2022-09-03T07:30:52.107 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.108 DEBUG:boto:Data: 2022-09-03T07:30:52.108 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.109 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.109 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.110 DEBUG:boto:Params: {} 2022-09-03T07:30:52.111 DEBUG:boto:Token: None 2022-09-03T07:30:52.111 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:30:52.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6WfdPJJxRFTlaxO01oUBxk0uZds= 2022-09-03T07:30:52.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6WfdPJJxRFTlaxO01oUBxk0uZds=', 'Content-Length': '0'} 2022-09-03T07:30:52.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df1c632284f30676-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.119 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.119 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.120 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:30:52.121 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:30:52.121 DEBUG:boto:Method: GET 2022-09-03T07:30:52.122 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:30:52.122 DEBUG:boto:Data: 2022-09-03T07:30:52.123 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.123 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.124 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.124 DEBUG:boto:Params: {} 2022-09-03T07:30:52.125 DEBUG:boto:Token: None 2022-09-03T07:30:52.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.129 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa6d8c2fd269903b-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.130 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.131 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.132 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.133 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:30:52.133 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:30:52.134 DEBUG:boto:Method: GET 2022-09-03T07:30:52.134 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:30:52.135 DEBUG:boto:Data: 2022-09-03T07:30:52.135 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.136 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.136 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.137 DEBUG:boto:Params: {} 2022-09-03T07:30:52.138 DEBUG:boto:Token: None 2022-09-03T07:30:52.138 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.144 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003030d3506a3ffb7f-00631302ac-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.145 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.146 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.147 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.148 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.148 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:30:52.149 DEBUG:boto:Data: 2022-09-03T07:30:52.149 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.150 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.150 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.151 DEBUG:boto:Params: {} 2022-09-03T07:30:52.151 DEBUG:boto:Token: None 2022-09-03T07:30:52.152 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93 2022-09-03T07:30:52.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yMM8nSOLfo8NA/qY7HxUDggxljE= 2022-09-03T07:30:52.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yMM8nSOLfo8NA/qY7HxUDggxljE=', 'Content-Length': '0'} 2022-09-03T07:30:52.165 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035d153d4ac68f512-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.166 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.168 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.168 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.169 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:30:52.169 DEBUG:boto:Data: 2022-09-03T07:30:52.170 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.171 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.171 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.172 DEBUG:boto:Params: {} 2022-09-03T07:30:52.172 DEBUG:boto:Token: None 2022-09-03T07:30:52.173 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93 2022-09-03T07:30:52.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yMM8nSOLfo8NA/qY7HxUDggxljE= 2022-09-03T07:30:52.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yMM8nSOLfo8NA/qY7HxUDggxljE=', 'Content-Length': '0'} 2022-09-03T07:30:52.177 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089e0941a46b6767b-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.177 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:30:52.178 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.178 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.179 DEBUG:boto:Method: GET 2022-09-03T07:30:52.180 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:30:52.180 DEBUG:boto:Data: 2022-09-03T07:30:52.181 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.181 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.182 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.182 DEBUG:boto:Params: {} 2022-09-03T07:30:52.183 DEBUG:boto:Token: None 2022-09-03T07:30:52.185 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93 2022-09-03T07:30:52.186 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYqDEi5D5KAqXtrYearnB+4KLww= 2022-09-03T07:30:52.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYqDEi5D5KAqXtrYearnB+4KLww=', 'Content-Length': '0'} 2022-09-03T07:30:52.192 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000240f32fab37b9431-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.194 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:30:52.195 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:30:52.196 DEBUG:boto:Method: GET 2022-09-03T07:30:52.198 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:30:52.199 DEBUG:boto:Data: 2022-09-03T07:30:52.201 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.202 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.203 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.204 DEBUG:boto:Params: {} 2022-09-03T07:30:52.205 DEBUG:boto:Token: None 2022-09-03T07:30:52.206 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj93 2022-09-03T07:30:52.207 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cYqDEi5D5KAqXtrYearnB+4KLww= 2022-09-03T07:30:52.208 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cYqDEi5D5KAqXtrYearnB+4KLww=', 'Content-Length': '0'} 2022-09-03T07:30:52.212 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8353ee7be985553-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.214 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.215 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.216 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:30:52.217 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:30:52.218 DEBUG:boto:Method: GET 2022-09-03T07:30:52.219 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:30:52.220 DEBUG:boto:Data: 2022-09-03T07:30:52.220 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.221 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.222 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.223 DEBUG:boto:Params: {} 2022-09-03T07:30:52.224 DEBUG:boto:Token: None 2022-09-03T07:30:52.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.232 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001619f9462e117a33-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.233 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.234 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.235 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.236 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:30:52.237 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:30:52.238 DEBUG:boto:Method: GET 2022-09-03T07:30:52.239 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:30:52.240 DEBUG:boto:Data: 2022-09-03T07:30:52.241 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.242 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.242 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.243 DEBUG:boto:Params: {} 2022-09-03T07:30:52.243 DEBUG:boto:Token: None 2022-09-03T07:30:52.244 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.250 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007be5ef5279f02cad-00631302ac-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.252 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.253 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:30:52.254 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.255 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.256 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.257 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.259 DEBUG:boto:Method: GET 2022-09-03T07:30:52.260 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.261 DEBUG:boto:Data: 2022-09-03T07:30:52.262 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.263 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.264 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.265 DEBUG:boto:Params: {} 2022-09-03T07:30:52.266 DEBUG:boto:Token: None 2022-09-03T07:30:52.267 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rsK42DKj7hqjLBKY9AiG3HHSQUg= 2022-09-03T07:30:52.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rsK42DKj7hqjLBKY9AiG3HHSQUg=', 'Content-Length': '0'} 2022-09-03T07:30:52.272 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000335f6045a7d61c94-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.273 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.274 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.275 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.276 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.277 DEBUG:boto:Method: GET 2022-09-03T07:30:52.278 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.280 DEBUG:boto:Data: 2022-09-03T07:30:52.281 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.283 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.284 DEBUG:boto:Params: {} 2022-09-03T07:30:52.285 DEBUG:boto:Token: None 2022-09-03T07:30:52.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rsK42DKj7hqjLBKY9AiG3HHSQUg= 2022-09-03T07:30:52.288 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rsK42DKj7hqjLBKY9AiG3HHSQUg=', 'Content-Length': '0'} 2022-09-03T07:30:52.291 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b683d8d8ba168221-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.292 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.293 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.294 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.295 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.296 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.297 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.298 DEBUG:boto:Data: 2022-09-03T07:30:52.299 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.301 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.302 DEBUG:boto:Params: {} 2022-09-03T07:30:52.303 DEBUG:boto:Token: None 2022-09-03T07:30:52.304 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HWsyGjoUK4CSPvUScKqXyxWI5ow= 2022-09-03T07:30:52.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HWsyGjoUK4CSPvUScKqXyxWI5ow=', 'Content-Length': '0'} 2022-09-03T07:30:52.308 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d649c7b632898a8c-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.309 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.310 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.311 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.312 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.313 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.314 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.315 DEBUG:boto:Data: 2022-09-03T07:30:52.316 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.317 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.318 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.318 DEBUG:boto:Params: {} 2022-09-03T07:30:52.319 DEBUG:boto:Token: None 2022-09-03T07:30:52.320 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HWsyGjoUK4CSPvUScKqXyxWI5ow= 2022-09-03T07:30:52.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HWsyGjoUK4CSPvUScKqXyxWI5ow=', 'Content-Length': '0'} 2022-09-03T07:30:52.324 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002dd80f00502249f7-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.325 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:30:52.325 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.326 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.327 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.328 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.329 DEBUG:boto:Method: GET 2022-09-03T07:30:52.330 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.331 DEBUG:boto:Data: 2022-09-03T07:30:52.332 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.332 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.333 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.333 DEBUG:boto:Params: {} 2022-09-03T07:30:52.335 DEBUG:boto:Token: None 2022-09-03T07:30:52.335 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.336 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rsK42DKj7hqjLBKY9AiG3HHSQUg= 2022-09-03T07:30:52.337 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rsK42DKj7hqjLBKY9AiG3HHSQUg=', 'Content-Length': '0'} 2022-09-03T07:30:52.339 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005b10983250af3a1a-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.340 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.341 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.341 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.342 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.343 DEBUG:boto:Method: GET 2022-09-03T07:30:52.344 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.345 DEBUG:boto:Data: 2022-09-03T07:30:52.345 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.346 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.346 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.347 DEBUG:boto:Params: {} 2022-09-03T07:30:52.347 DEBUG:boto:Token: None 2022-09-03T07:30:52.348 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:30:52.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rsK42DKj7hqjLBKY9AiG3HHSQUg= 2022-09-03T07:30:52.349 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rsK42DKj7hqjLBKY9AiG3HHSQUg=', 'Content-Length': '0'} 2022-09-03T07:30:52.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d387a830104a9756-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.394 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.395 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.395 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:30:52.396 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:30:52.396 DEBUG:boto:Method: GET 2022-09-03T07:30:52.397 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:30:52.397 DEBUG:boto:Data: 2022-09-03T07:30:52.399 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.400 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.401 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.402 DEBUG:boto:Params: {} 2022-09-03T07:30:52.403 DEBUG:boto:Token: None 2022-09-03T07:30:52.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.409 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004bf00a36489dea18-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.410 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.411 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.412 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.413 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:30:52.414 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:30:52.415 DEBUG:boto:Method: GET 2022-09-03T07:30:52.416 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:30:52.417 DEBUG:boto:Data: 2022-09-03T07:30:52.418 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.420 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.421 DEBUG:boto:Params: {} 2022-09-03T07:30:52.422 DEBUG:boto:Token: None 2022-09-03T07:30:52.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.425 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.431 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000127f6d5b8dcbc392-00631302ac-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.432 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.434 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.434 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.435 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.436 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:30:52.436 DEBUG:boto:Data: 2022-09-03T07:30:52.437 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.438 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.441 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.442 DEBUG:boto:Params: {} 2022-09-03T07:30:52.443 DEBUG:boto:Token: None 2022-09-03T07:30:52.444 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95 2022-09-03T07:30:52.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oCsRgBrSpv1yQFmV/ZP7Qrr6xYo= 2022-09-03T07:30:52.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oCsRgBrSpv1yQFmV/ZP7Qrr6xYo=', 'Content-Length': '0'} 2022-09-03T07:30:52.448 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002af11547c7946c82-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.449 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.449 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.451 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.453 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:30:52.455 DEBUG:boto:Data: 2022-09-03T07:30:52.456 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.457 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.457 DEBUG:boto:Params: {} 2022-09-03T07:30:52.458 DEBUG:boto:Token: None 2022-09-03T07:30:52.459 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95 2022-09-03T07:30:52.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oCsRgBrSpv1yQFmV/ZP7Qrr6xYo= 2022-09-03T07:30:52.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oCsRgBrSpv1yQFmV/ZP7Qrr6xYo=', 'Content-Length': '0'} 2022-09-03T07:30:52.469 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f4200de378633ace-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.471 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:30:52.471 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.472 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.473 DEBUG:boto:Method: GET 2022-09-03T07:30:52.474 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:30:52.474 DEBUG:boto:Data: 2022-09-03T07:30:52.475 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.476 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.477 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.477 DEBUG:boto:Params: {} 2022-09-03T07:30:52.478 DEBUG:boto:Token: None 2022-09-03T07:30:52.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95 2022-09-03T07:30:52.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yc4B8ZMQGc77xBkp3bYvclv7UjY= 2022-09-03T07:30:52.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yc4B8ZMQGc77xBkp3bYvclv7UjY=', 'Content-Length': '0'} 2022-09-03T07:30:52.482 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001df3af75db3a7a7b-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.483 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:30:52.483 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:30:52.484 DEBUG:boto:Method: GET 2022-09-03T07:30:52.485 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:30:52.485 DEBUG:boto:Data: 2022-09-03T07:30:52.486 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.487 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.488 DEBUG:boto:Params: {} 2022-09-03T07:30:52.488 DEBUG:boto:Token: None 2022-09-03T07:30:52.489 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj95 2022-09-03T07:30:52.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yc4B8ZMQGc77xBkp3bYvclv7UjY= 2022-09-03T07:30:52.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yc4B8ZMQGc77xBkp3bYvclv7UjY=', 'Content-Length': '0'} 2022-09-03T07:30:52.492 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062332c7f129186ae-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.534 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.535 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.535 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:30:52.536 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:30:52.536 DEBUG:boto:Method: GET 2022-09-03T07:30:52.537 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:30:52.538 DEBUG:boto:Data: 2022-09-03T07:30:52.539 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.540 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:30:52.541 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.542 DEBUG:boto:Params: {} 2022-09-03T07:30:52.543 DEBUG:boto:Token: None 2022-09-03T07:30:52.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.547 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.554 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006f99725ec7e3672d-00631302ac-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.556 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.557 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.559 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.560 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:30:52.561 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:30:52.562 DEBUG:boto:Method: GET 2022-09-03T07:30:52.563 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:30:52.564 DEBUG:boto:Data: 2022-09-03T07:30:52.565 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.566 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.567 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.567 DEBUG:boto:Params: {} 2022-09-03T07:30:52.569 DEBUG:boto:Token: None 2022-09-03T07:30:52.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.578 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000095f0ce141112331f-00631302ac-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.579 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.581 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone2, test-zone1} 2022-09-03T07:30:52.582 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:30:52.583 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.584 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.585 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.585 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:30:52.586 DEBUG:boto:Data: 2022-09-03T07:30:52.586 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:52.589 DEBUG:boto:Port: 8001 2022-09-03T07:30:52.589 DEBUG:boto:Params: {} 2022-09-03T07:30:52.590 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:30:52.591 DEBUG:boto:Token: None 2022-09-03T07:30:52.592 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+cnebgCjgDDAhZQzudn+EG0GIk= 2022-09-03T07:30:52.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+cnebgCjgDDAhZQzudn+EG0GIk=', 'Content-Length': '0'} 2022-09-03T07:30:52.598 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '35'), ('X-RGW-Bytes-Used', '105'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c590bd6eacc9a894-00631302ac-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.599 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.600 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.600 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.601 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:30:52.601 DEBUG:boto:Data: 2022-09-03T07:30:52.602 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.602 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.603 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.603 DEBUG:boto:Params: {} 2022-09-03T07:30:52.604 DEBUG:boto:Token: None 2022-09-03T07:30:52.605 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r+cnebgCjgDDAhZQzudn+EG0GIk= 2022-09-03T07:30:52.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r+cnebgCjgDDAhZQzudn+EG0GIk=', 'Content-Length': '0'} 2022-09-03T07:30:52.611 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '35'), ('X-RGW-Bytes-Used', '105'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000717ac475a006a71b-00631302ac-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.612 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:30:52.613 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.613 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.614 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.615 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.615 DEBUG:boto:Method: GET 2022-09-03T07:30:52.616 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.616 DEBUG:boto:Data: 2022-09-03T07:30:52.617 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.617 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:52.618 DEBUG:boto:Port: 8001 2022-09-03T07:30:52.618 DEBUG:boto:Params: {} 2022-09-03T07:30:52.619 DEBUG:boto:Token: None 2022-09-03T07:30:52.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/?versions 2022-09-03T07:30:52.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8wMILqG1PSRU5rVyCsVB6i/kQY= 2022-09-03T07:30:52.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8wMILqG1PSRU5rVyCsVB6i/kQY=', 'Content-Length': '0'} 2022-09-03T07:30:52.624 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000078d231e17c4a93db-00631302ac-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.625 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.629 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:30:52.630 DEBUG:tasks.rgw_multi.zone_rados:o=obj111 2022-09-03T07:30:52.631 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:30:52.632 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:30:52.632 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:30:52.633 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:30:52.634 DEBUG:tasks.rgw_multi.zone_rados:o=obj148 2022-09-03T07:30:52.635 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:30:52.635 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:30:52.635 DEBUG:tasks.rgw_multi.zone_rados:o=obj185 2022-09-03T07:30:52.636 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:30:52.636 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:30:52.637 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:30:52.638 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:30:52.639 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:30:52.640 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:30:52.642 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:30:52.642 DEBUG:tasks.rgw_multi.zone_rados:o=obj37 2022-09-03T07:30:52.643 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:30:52.644 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:30:52.645 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:30:52.646 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:30:52.646 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:30:52.647 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:30:52.648 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:30:52.649 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:30:52.650 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:30:52.650 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:30:52.651 DEBUG:tasks.rgw_multi.zone_rados:o=obj74 2022-09-03T07:30:52.652 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:30:52.653 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:30:52.654 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:30:52.655 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:30:52.656 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:30:52.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:30:52.658 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:30:52.659 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.660 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.661 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.662 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.663 DEBUG:boto:Method: GET 2022-09-03T07:30:52.663 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.664 DEBUG:boto:Data: 2022-09-03T07:30:52.665 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.667 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.668 DEBUG:boto:Params: {} 2022-09-03T07:30:52.669 DEBUG:boto:Token: None 2022-09-03T07:30:52.670 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/?versions 2022-09-03T07:30:52.671 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8wMILqG1PSRU5rVyCsVB6i/kQY= 2022-09-03T07:30:52.672 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8wMILqG1PSRU5rVyCsVB6i/kQY=', 'Content-Length': '0'} 2022-09-03T07:30:52.678 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ccac79076bcd3519-00631302ac-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.679 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.684 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:30:52.684 DEBUG:tasks.rgw_multi.zone_rados:o=obj111 2022-09-03T07:30:52.685 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:30:52.685 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:30:52.686 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:30:52.687 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:30:52.692 DEBUG:tasks.rgw_multi.zone_rados:o=obj148 2022-09-03T07:30:52.693 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:30:52.693 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:30:52.694 DEBUG:tasks.rgw_multi.zone_rados:o=obj185 2022-09-03T07:30:52.697 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:30:52.698 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:30:52.699 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:30:52.700 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:30:52.701 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:30:52.702 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:30:52.703 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:30:52.703 DEBUG:tasks.rgw_multi.zone_rados:o=obj37 2022-09-03T07:30:52.704 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:30:52.705 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:30:52.705 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:30:52.706 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:30:52.706 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:30:52.707 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:30:52.707 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:30:52.708 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:30:52.708 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:30:52.709 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:30:52.709 DEBUG:tasks.rgw_multi.zone_rados:o=obj74 2022-09-03T07:30:52.710 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:30:52.711 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:30:52.711 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:30:52.711 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:30:52.712 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:30:52.712 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:30:52.713 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.714 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.715 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.716 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.717 DEBUG:boto:Method: GET 2022-09-03T07:30:52.718 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.719 DEBUG:boto:Data: 2022-09-03T07:30:52.720 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.721 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:52.723 DEBUG:boto:Port: 8001 2022-09-03T07:30:52.724 DEBUG:boto:Params: {} 2022-09-03T07:30:52.725 DEBUG:boto:Token: None 2022-09-03T07:30:52.725 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/?versions 2022-09-03T07:30:52.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8wMILqG1PSRU5rVyCsVB6i/kQY= 2022-09-03T07:30:52.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8wMILqG1PSRU5rVyCsVB6i/kQY=', 'Content-Length': '0'} 2022-09-03T07:30:52.731 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000068d4046a4b8d3868-00631302ac-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.732 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.736 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.736 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.737 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.738 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.739 DEBUG:boto:Method: GET 2022-09-03T07:30:52.739 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:30:52.740 DEBUG:boto:Data: 2022-09-03T07:30:52.740 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.741 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.741 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.742 DEBUG:boto:Params: {} 2022-09-03T07:30:52.743 DEBUG:boto:Token: None 2022-09-03T07:30:52.743 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/?versions 2022-09-03T07:30:52.744 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q8wMILqG1PSRU5rVyCsVB6i/kQY= 2022-09-03T07:30:52.745 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q8wMILqG1PSRU5rVyCsVB6i/kQY=', 'Content-Length': '0'} 2022-09-03T07:30:52.750 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000010003bb8bea0fee4-00631302ac-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.751 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.753 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:30:52.754 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.755 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.755 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.756 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.756 DEBUG:boto:Method: GET 2022-09-03T07:30:52.757 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.757 DEBUG:boto:Data: 2022-09-03T07:30:52.758 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.760 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:52.760 DEBUG:boto:Port: 8001 2022-09-03T07:30:52.761 DEBUG:boto:Params: {} 2022-09-03T07:30:52.762 DEBUG:boto:Token: None 2022-09-03T07:30:52.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.763 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XFzfTsX7ICvxbys7Vq5EMCxPLXA= 2022-09-03T07:30:52.764 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XFzfTsX7ICvxbys7Vq5EMCxPLXA=', 'Content-Length': '0'} 2022-09-03T07:30:52.766 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6e1b2c53bbb344f-00631302ac-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.767 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.768 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.768 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.769 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.769 DEBUG:boto:Method: GET 2022-09-03T07:30:52.770 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.771 DEBUG:boto:Data: 2022-09-03T07:30:52.771 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.772 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.772 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.773 DEBUG:boto:Params: {} 2022-09-03T07:30:52.774 DEBUG:boto:Token: None 2022-09-03T07:30:52.774 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XFzfTsX7ICvxbys7Vq5EMCxPLXA= 2022-09-03T07:30:52.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XFzfTsX7ICvxbys7Vq5EMCxPLXA=', 'Content-Length': '0'} 2022-09-03T07:30:52.779 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b544bfb693ff678-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.818 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.819 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.820 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.821 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.822 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.823 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.824 DEBUG:boto:Data: 2022-09-03T07:30:52.825 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:52.827 DEBUG:boto:Port: 8001 2022-09-03T07:30:52.828 DEBUG:boto:Params: {} 2022-09-03T07:30:52.829 DEBUG:boto:Token: None 2022-09-03T07:30:52.831 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.832 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pANgX+KYfW9UhjA01Sk5M2jNHGs= 2022-09-03T07:30:52.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pANgX+KYfW9UhjA01Sk5M2jNHGs=', 'Content-Length': '0'} 2022-09-03T07:30:52.837 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c6b679e8d13644c-00631302ac-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.839 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.840 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.841 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.841 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.842 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.843 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.843 DEBUG:boto:Data: 2022-09-03T07:30:52.844 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.845 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.845 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.846 DEBUG:boto:Params: {} 2022-09-03T07:30:52.847 DEBUG:boto:Token: None 2022-09-03T07:30:52.847 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.848 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pANgX+KYfW9UhjA01Sk5M2jNHGs= 2022-09-03T07:30:52.848 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pANgX+KYfW9UhjA01Sk5M2jNHGs=', 'Content-Length': '0'} 2022-09-03T07:30:52.851 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008812bc9a6befeb47-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.852 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:30:52.852 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.853 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.853 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.854 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.855 DEBUG:boto:Method: GET 2022-09-03T07:30:52.855 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.856 DEBUG:boto:Data: 2022-09-03T07:30:52.856 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.857 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:52.857 DEBUG:boto:Port: 8001 2022-09-03T07:30:52.858 DEBUG:boto:Params: {} 2022-09-03T07:30:52.859 DEBUG:boto:Token: None 2022-09-03T07:30:52.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XFzfTsX7ICvxbys7Vq5EMCxPLXA= 2022-09-03T07:30:52.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XFzfTsX7ICvxbys7Vq5EMCxPLXA=', 'Content-Length': '0'} 2022-09-03T07:30:52.862 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005377e5f57b5675ad-00631302ac-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.863 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.864 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.864 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.865 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.866 DEBUG:boto:Method: GET 2022-09-03T07:30:52.866 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.867 DEBUG:boto:Data: 2022-09-03T07:30:52.867 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.868 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.868 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.869 DEBUG:boto:Params: {} 2022-09-03T07:30:52.869 DEBUG:boto:Token: None 2022-09-03T07:30:52.870 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:30:52.871 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XFzfTsX7ICvxbys7Vq5EMCxPLXA= 2022-09-03T07:30:52.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XFzfTsX7ICvxbys7Vq5EMCxPLXA=', 'Content-Length': '0'} 2022-09-03T07:30:52.874 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a18ea0d0af87e0ac-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.875 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.876 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.877 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:52.878 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:52.878 DEBUG:boto:Method: GET 2022-09-03T07:30:52.879 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:30:52.880 DEBUG:boto:Data: 2022-09-03T07:30:52.881 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:52.882 DEBUG:boto:Port: 8001 2022-09-03T07:30:52.883 DEBUG:boto:Params: {} 2022-09-03T07:30:52.884 DEBUG:boto:Token: None 2022-09-03T07:30:52.884 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f710170aa67519bd-00631302ac-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.890 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.891 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:52.892 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:52.893 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:52.894 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:52.895 DEBUG:boto:Method: GET 2022-09-03T07:30:52.896 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:30:52.897 DEBUG:boto:Data: 2022-09-03T07:30:52.897 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.898 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.899 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.899 DEBUG:boto:Params: {} 2022-09-03T07:30:52.900 DEBUG:boto:Token: None 2022-09-03T07:30:52.900 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/ 2022-09-03T07:30:52.901 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0= 2022-09-03T07:30:52.902 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/y3Fk8xTzi9lo9QHBEXmRNnq/M0=', 'Content-Length': '0'} 2022-09-03T07:30:52.908 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000144b2d9c56129b34-00631302ac-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.909 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:52.911 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.912 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.913 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.913 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:30:52.915 DEBUG:boto:Data: 2022-09-03T07:30:52.915 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.916 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:52.917 DEBUG:boto:Port: 8001 2022-09-03T07:30:52.918 DEBUG:boto:Params: {} 2022-09-03T07:30:52.919 DEBUG:boto:Token: None 2022-09-03T07:30:52.920 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11 2022-09-03T07:30:52.921 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bJJqok/UtqjXCykIA9ECSWgj8pI= 2022-09-03T07:30:52.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bJJqok/UtqjXCykIA9ECSWgj8pI=', 'Content-Length': '0'} 2022-09-03T07:30:52.925 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000239e176331ab1372-00631302ac-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.926 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.927 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.927 DEBUG:boto:Method: HEAD 2022-09-03T07:30:52.928 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:30:52.928 DEBUG:boto:Data: 2022-09-03T07:30:52.929 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.930 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.931 DEBUG:boto:Params: {} 2022-09-03T07:30:52.932 DEBUG:boto:Token: None 2022-09-03T07:30:52.933 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11 2022-09-03T07:30:52.933 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bJJqok/UtqjXCykIA9ECSWgj8pI= 2022-09-03T07:30:52.934 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bJJqok/UtqjXCykIA9ECSWgj8pI=', 'Content-Length': '0'} 2022-09-03T07:30:52.937 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bda65afb72a7ceeb-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.937 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:30:52.938 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.939 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.939 DEBUG:boto:Method: GET 2022-09-03T07:30:52.940 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:30:52.941 DEBUG:boto:Data: 2022-09-03T07:30:52.942 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.942 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:52.943 DEBUG:boto:Port: 8001 2022-09-03T07:30:52.944 DEBUG:boto:Params: {} 2022-09-03T07:30:52.945 DEBUG:boto:Token: None 2022-09-03T07:30:52.946 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11 2022-09-03T07:30:52.947 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bv+ABTj+Ko3qaVLvkULEj0djUI8= 2022-09-03T07:30:52.948 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bv+ABTj+Ko3qaVLvkULEj0djUI8=', 'Content-Length': '0'} 2022-09-03T07:30:52.950 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a3dc59891fa80ec-00631302ac-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:52.951 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:30:52.952 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:30:52.952 DEBUG:boto:Method: GET 2022-09-03T07:30:52.953 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:30:52.954 DEBUG:boto:Data: 2022-09-03T07:30:52.955 DEBUG:boto:Headers: {} 2022-09-03T07:30:52.956 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:52.957 DEBUG:boto:Port: 8000 2022-09-03T07:30:52.958 DEBUG:boto:Params: {} 2022-09-03T07:30:52.959 DEBUG:boto:Token: None 2022-09-03T07:30:52.960 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:52 GMT /avysmw-75/obj11 2022-09-03T07:30:52.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Bv+ABTj+Ko3qaVLvkULEj0djUI8= 2022-09-03T07:30:52.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:52 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Bv+ABTj+Ko3qaVLvkULEj0djUI8=', 'Content-Length': '0'} 2022-09-03T07:30:52.964 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f0ea6cdf99a103d-00631302ac-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:52 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.006 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.007 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.008 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:53.009 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:53.010 DEBUG:boto:Method: GET 2022-09-03T07:30:53.011 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:30:53.012 DEBUG:boto:Data: 2022-09-03T07:30:53.012 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.014 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.015 DEBUG:boto:Params: {} 2022-09-03T07:30:53.016 DEBUG:boto:Token: None 2022-09-03T07:30:53.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.019 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.022 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000056599f40d267aaf5-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.023 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.024 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.025 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.026 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:53.027 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:30:53.027 DEBUG:boto:Method: GET 2022-09-03T07:30:53.028 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:30:53.029 DEBUG:boto:Data: 2022-09-03T07:30:53.030 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.031 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.032 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.033 DEBUG:boto:Params: {} 2022-09-03T07:30:53.035 DEBUG:boto:Token: None 2022-09-03T07:30:53.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.045 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087599315762adb17-00631302ad-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.046 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.047 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj111 2022-09-03T07:30:53.048 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.049 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.050 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.050 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.051 DEBUG:boto:Method: GET 2022-09-03T07:30:53.052 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.052 DEBUG:boto:Data: 2022-09-03T07:30:53.053 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.054 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.055 DEBUG:boto:Params: {} 2022-09-03T07:30:53.056 DEBUG:boto:Token: None 2022-09-03T07:30:53.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sJNvBZ5u7QFilvh6FGAMdOPOwXA= 2022-09-03T07:30:53.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sJNvBZ5u7QFilvh6FGAMdOPOwXA=', 'Content-Length': '0'} 2022-09-03T07:30:53.063 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000068ff7987150ae9f7-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.106 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.108 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.109 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.110 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.111 DEBUG:boto:Method: GET 2022-09-03T07:30:53.113 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.114 DEBUG:boto:Data: 2022-09-03T07:30:53.115 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.117 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.118 DEBUG:boto:Params: {} 2022-09-03T07:30:53.119 DEBUG:boto:Token: None 2022-09-03T07:30:53.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.120 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sJNvBZ5u7QFilvh6FGAMdOPOwXA= 2022-09-03T07:30:53.121 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sJNvBZ5u7QFilvh6FGAMdOPOwXA=', 'Content-Length': '0'} 2022-09-03T07:30:53.124 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090ccb6019c51c100-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.125 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.126 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.126 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.127 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.128 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.129 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.130 DEBUG:boto:Data: 2022-09-03T07:30:53.131 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.131 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.132 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.133 DEBUG:boto:Params: {} 2022-09-03T07:30:53.134 DEBUG:boto:Token: None 2022-09-03T07:30:53.135 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Td2LsNAkyNk7m4odOQj1ssM4Q8o= 2022-09-03T07:30:53.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Td2LsNAkyNk7m4odOQj1ssM4Q8o=', 'Content-Length': '0'} 2022-09-03T07:30:53.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a6b23b77e3f60f37-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.140 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.141 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.142 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.143 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.144 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.145 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.145 DEBUG:boto:Data: 2022-09-03T07:30:53.147 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.148 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.149 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.151 DEBUG:boto:Params: {} 2022-09-03T07:30:53.152 DEBUG:boto:Token: None 2022-09-03T07:30:53.153 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.155 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Td2LsNAkyNk7m4odOQj1ssM4Q8o= 2022-09-03T07:30:53.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Td2LsNAkyNk7m4odOQj1ssM4Q8o=', 'Content-Length': '0'} 2022-09-03T07:30:53.159 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002c2807053f2b4da-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.161 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj111 2022-09-03T07:30:53.162 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.163 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.164 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.165 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.166 DEBUG:boto:Method: GET 2022-09-03T07:30:53.166 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.167 DEBUG:boto:Data: 2022-09-03T07:30:53.168 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.169 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.170 DEBUG:boto:Params: {} 2022-09-03T07:30:53.182 DEBUG:boto:Token: None 2022-09-03T07:30:53.183 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sJNvBZ5u7QFilvh6FGAMdOPOwXA= 2022-09-03T07:30:53.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sJNvBZ5u7QFilvh6FGAMdOPOwXA=', 'Content-Length': '0'} 2022-09-03T07:30:53.189 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d91e47d5c186b50e-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.191 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.192 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.194 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.195 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.196 DEBUG:boto:Method: GET 2022-09-03T07:30:53.199 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.199 DEBUG:boto:Data: 2022-09-03T07:30:53.200 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.201 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.202 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.202 DEBUG:boto:Params: {} 2022-09-03T07:30:53.203 DEBUG:boto:Token: None 2022-09-03T07:30:53.204 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:30:53.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sJNvBZ5u7QFilvh6FGAMdOPOwXA= 2022-09-03T07:30:53.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sJNvBZ5u7QFilvh6FGAMdOPOwXA=', 'Content-Length': '0'} 2022-09-03T07:30:53.208 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005de0f458c011939e-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.209 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.211 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.212 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:53.212 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:53.213 DEBUG:boto:Method: GET 2022-09-03T07:30:53.214 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:30:53.215 DEBUG:boto:Data: 2022-09-03T07:30:53.215 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.216 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.217 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.218 DEBUG:boto:Params: {} 2022-09-03T07:30:53.218 DEBUG:boto:Token: None 2022-09-03T07:30:53.219 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.224 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000075d1c0e2e457abb6-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.225 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.226 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.227 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.228 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:53.229 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:53.230 DEBUG:boto:Method: GET 2022-09-03T07:30:53.230 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:30:53.231 DEBUG:boto:Data: 2022-09-03T07:30:53.232 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.232 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.233 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.234 DEBUG:boto:Params: {} 2022-09-03T07:30:53.235 DEBUG:boto:Token: None 2022-09-03T07:30:53.236 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.242 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d0d2bd4233c6e1c3-00631302ad-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.244 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.245 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.246 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.247 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.248 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:30:53.249 DEBUG:boto:Data: 2022-09-03T07:30:53.250 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.252 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.253 DEBUG:boto:Params: {} 2022-09-03T07:30:53.254 DEBUG:boto:Token: None 2022-09-03T07:30:53.255 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111 2022-09-03T07:30:53.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MnpNADo+o4BwG1tXPFLMQ+dAnJE= 2022-09-03T07:30:53.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MnpNADo+o4BwG1tXPFLMQ+dAnJE=', 'Content-Length': '0'} 2022-09-03T07:30:53.259 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055fff002d3ce48ad-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.261 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.262 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.263 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.264 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:30:53.265 DEBUG:boto:Data: 2022-09-03T07:30:53.265 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.266 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.267 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.268 DEBUG:boto:Params: {} 2022-09-03T07:30:53.269 DEBUG:boto:Token: None 2022-09-03T07:30:53.270 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111 2022-09-03T07:30:53.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MnpNADo+o4BwG1tXPFLMQ+dAnJE= 2022-09-03T07:30:53.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MnpNADo+o4BwG1tXPFLMQ+dAnJE=', 'Content-Length': '0'} 2022-09-03T07:30:53.275 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf577ea015198d5a-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.276 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj111 2022-09-03T07:30:53.277 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.278 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.279 DEBUG:boto:Method: GET 2022-09-03T07:30:53.281 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:30:53.282 DEBUG:boto:Data: 2022-09-03T07:30:53.283 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.284 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.285 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.286 DEBUG:boto:Params: {} 2022-09-03T07:30:53.287 DEBUG:boto:Token: None 2022-09-03T07:30:53.288 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111 2022-09-03T07:30:53.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2grT2UrCYvs5MypvPaLeBSkrzrA= 2022-09-03T07:30:53.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2grT2UrCYvs5MypvPaLeBSkrzrA=', 'Content-Length': '0'} 2022-09-03T07:30:53.291 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000826653888e555041-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.292 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:30:53.293 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:30:53.294 DEBUG:boto:Method: GET 2022-09-03T07:30:53.295 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:30:53.295 DEBUG:boto:Data: 2022-09-03T07:30:53.296 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.297 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.298 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.298 DEBUG:boto:Params: {} 2022-09-03T07:30:53.299 DEBUG:boto:Token: None 2022-09-03T07:30:53.300 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj111 2022-09-03T07:30:53.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2grT2UrCYvs5MypvPaLeBSkrzrA= 2022-09-03T07:30:53.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2grT2UrCYvs5MypvPaLeBSkrzrA=', 'Content-Length': '0'} 2022-09-03T07:30:53.304 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087bce6e7e6c5cdd6-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.350 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.351 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.352 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:53.352 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:53.353 DEBUG:boto:Method: GET 2022-09-03T07:30:53.353 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:30:53.354 DEBUG:boto:Data: 2022-09-03T07:30:53.354 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.355 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.355 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.356 DEBUG:boto:Params: {} 2022-09-03T07:30:53.356 DEBUG:boto:Token: None 2022-09-03T07:30:53.357 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.359 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.362 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000059ee0b4cce990e19-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.363 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.365 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.366 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.367 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:53.368 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:30:53.370 DEBUG:boto:Method: GET 2022-09-03T07:30:53.371 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:30:53.372 DEBUG:boto:Data: 2022-09-03T07:30:53.373 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.374 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.375 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.376 DEBUG:boto:Params: {} 2022-09-03T07:30:53.377 DEBUG:boto:Token: None 2022-09-03T07:30:53.378 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.386 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000053d53e8b5aa6d369-00631302ad-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.387 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.388 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:30:53.389 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.389 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.390 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.390 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.391 DEBUG:boto:Method: GET 2022-09-03T07:30:53.392 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.394 DEBUG:boto:Data: 2022-09-03T07:30:53.395 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.396 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.396 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.397 DEBUG:boto:Params: {} 2022-09-03T07:30:53.398 DEBUG:boto:Token: None 2022-09-03T07:30:53.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BqmQWqWDavYA3LZh5YzlAfkUCUQ= 2022-09-03T07:30:53.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BqmQWqWDavYA3LZh5YzlAfkUCUQ=', 'Content-Length': '0'} 2022-09-03T07:30:53.402 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3f6f09ce284aa2b-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.443 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.444 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.445 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.446 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.450 DEBUG:boto:Method: GET 2022-09-03T07:30:53.450 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.451 DEBUG:boto:Data: 2022-09-03T07:30:53.452 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.452 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.453 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.453 DEBUG:boto:Params: {} 2022-09-03T07:30:53.454 DEBUG:boto:Token: None 2022-09-03T07:30:53.455 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.456 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BqmQWqWDavYA3LZh5YzlAfkUCUQ= 2022-09-03T07:30:53.457 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BqmQWqWDavYA3LZh5YzlAfkUCUQ=', 'Content-Length': '0'} 2022-09-03T07:30:53.460 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006987922a60fdb446-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.463 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.464 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.465 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.467 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.468 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.469 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.471 DEBUG:boto:Data: 2022-09-03T07:30:53.471 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.472 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.474 DEBUG:boto:Params: {} 2022-09-03T07:30:53.475 DEBUG:boto:Token: None 2022-09-03T07:30:53.476 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fXE5CGwxP43hXym0VIgxHmSGRH8= 2022-09-03T07:30:53.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fXE5CGwxP43hXym0VIgxHmSGRH8=', 'Content-Length': '0'} 2022-09-03T07:30:53.481 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de2b97c387465d63-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.483 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.483 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.484 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.485 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.485 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.486 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.487 DEBUG:boto:Data: 2022-09-03T07:30:53.488 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.489 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.490 DEBUG:boto:Params: {} 2022-09-03T07:30:53.491 DEBUG:boto:Token: None 2022-09-03T07:30:53.492 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fXE5CGwxP43hXym0VIgxHmSGRH8= 2022-09-03T07:30:53.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fXE5CGwxP43hXym0VIgxHmSGRH8=', 'Content-Length': '0'} 2022-09-03T07:30:53.500 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000334c0104b87e925c-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.501 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:30:53.501 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.502 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.503 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.510 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.511 DEBUG:boto:Method: GET 2022-09-03T07:30:53.512 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.513 DEBUG:boto:Data: 2022-09-03T07:30:53.514 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.516 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.517 DEBUG:boto:Params: {} 2022-09-03T07:30:53.518 DEBUG:boto:Token: None 2022-09-03T07:30:53.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.520 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BqmQWqWDavYA3LZh5YzlAfkUCUQ= 2022-09-03T07:30:53.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BqmQWqWDavYA3LZh5YzlAfkUCUQ=', 'Content-Length': '0'} 2022-09-03T07:30:53.527 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020bf0fc6bc4dc7c3-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.528 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.529 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.529 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.530 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.531 DEBUG:boto:Method: GET 2022-09-03T07:30:53.533 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.534 DEBUG:boto:Data: 2022-09-03T07:30:53.536 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.538 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.539 DEBUG:boto:Params: {} 2022-09-03T07:30:53.540 DEBUG:boto:Token: None 2022-09-03T07:30:53.541 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:30:53.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BqmQWqWDavYA3LZh5YzlAfkUCUQ= 2022-09-03T07:30:53.544 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BqmQWqWDavYA3LZh5YzlAfkUCUQ=', 'Content-Length': '0'} 2022-09-03T07:30:53.547 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b1c8ade71505d46-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.548 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.549 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.550 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:53.551 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:53.553 DEBUG:boto:Method: GET 2022-09-03T07:30:53.554 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:30:53.555 DEBUG:boto:Data: 2022-09-03T07:30:53.556 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.557 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.558 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.559 DEBUG:boto:Params: {} 2022-09-03T07:30:53.560 DEBUG:boto:Token: None 2022-09-03T07:30:53.560 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.564 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005c6bbf835a270be8-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.567 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.570 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.572 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.573 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:53.576 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:53.578 DEBUG:boto:Method: GET 2022-09-03T07:30:53.579 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:30:53.580 DEBUG:boto:Data: 2022-09-03T07:30:53.581 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.584 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.585 DEBUG:boto:Params: {} 2022-09-03T07:30:53.592 DEBUG:boto:Token: None 2022-09-03T07:30:53.593 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.597 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.602 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006756ff2ae3ba75ac-00631302ad-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.642 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.644 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.644 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.645 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.646 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:30:53.647 DEBUG:boto:Data: 2022-09-03T07:30:53.648 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.650 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.651 DEBUG:boto:Params: {} 2022-09-03T07:30:53.652 DEBUG:boto:Token: None 2022-09-03T07:30:53.653 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115 2022-09-03T07:30:53.655 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DG7GI3apQjEW12SJn9WkuwSH674= 2022-09-03T07:30:53.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DG7GI3apQjEW12SJn9WkuwSH674=', 'Content-Length': '0'} 2022-09-03T07:30:53.658 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000abe255628120d378-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.660 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.661 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.662 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.664 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:30:53.665 DEBUG:boto:Data: 2022-09-03T07:30:53.666 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.668 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.672 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.673 DEBUG:boto:Params: {} 2022-09-03T07:30:53.675 DEBUG:boto:Token: None 2022-09-03T07:30:53.677 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115 2022-09-03T07:30:53.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DG7GI3apQjEW12SJn9WkuwSH674= 2022-09-03T07:30:53.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DG7GI3apQjEW12SJn9WkuwSH674=', 'Content-Length': '0'} 2022-09-03T07:30:53.682 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a72f76c79da74d4-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.683 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:30:53.684 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.686 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.687 DEBUG:boto:Method: GET 2022-09-03T07:30:53.688 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:30:53.689 DEBUG:boto:Data: 2022-09-03T07:30:53.690 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.692 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.693 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.694 DEBUG:boto:Params: {} 2022-09-03T07:30:53.696 DEBUG:boto:Token: None 2022-09-03T07:30:53.697 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115 2022-09-03T07:30:53.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UthfZmo/eDIpO2TJ8OdTj8u8Ldg= 2022-09-03T07:30:53.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UthfZmo/eDIpO2TJ8OdTj8u8Ldg=', 'Content-Length': '0'} 2022-09-03T07:30:53.702 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001055e2a82bcfcef7-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.746 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:30:53.747 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:30:53.749 DEBUG:boto:Method: GET 2022-09-03T07:30:53.750 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:30:53.751 DEBUG:boto:Data: 2022-09-03T07:30:53.751 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.752 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.753 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.754 DEBUG:boto:Params: {} 2022-09-03T07:30:53.756 DEBUG:boto:Token: None 2022-09-03T07:30:53.757 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj115 2022-09-03T07:30:53.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UthfZmo/eDIpO2TJ8OdTj8u8Ldg= 2022-09-03T07:30:53.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UthfZmo/eDIpO2TJ8OdTj8u8Ldg=', 'Content-Length': '0'} 2022-09-03T07:30:53.762 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1e6dd2b8fecfb56-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.764 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.765 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.766 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:53.767 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:53.768 DEBUG:boto:Method: GET 2022-09-03T07:30:53.770 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:30:53.770 DEBUG:boto:Data: 2022-09-03T07:30:53.771 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.772 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.772 DEBUG:boto:Params: {} 2022-09-03T07:30:53.773 DEBUG:boto:Token: None 2022-09-03T07:30:53.774 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.775 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.780 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000071dc89aa4433c7d2-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.781 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.783 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.784 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.785 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:53.785 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:30:53.786 DEBUG:boto:Method: GET 2022-09-03T07:30:53.787 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:30:53.788 DEBUG:boto:Data: 2022-09-03T07:30:53.788 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.789 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.790 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.790 DEBUG:boto:Params: {} 2022-09-03T07:30:53.791 DEBUG:boto:Token: None 2022-09-03T07:30:53.792 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.793 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.798 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000032f00e5310b95489-00631302ad-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.799 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.800 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:30:53.800 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:53.801 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:53.802 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.802 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.803 DEBUG:boto:Method: GET 2022-09-03T07:30:53.804 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.804 DEBUG:boto:Data: 2022-09-03T07:30:53.805 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.805 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.806 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.807 DEBUG:boto:Params: {} 2022-09-03T07:30:53.807 DEBUG:boto:Token: None 2022-09-03T07:30:53.808 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPX1Mqrlnz7ojaqSgfG2RIQ0I98= 2022-09-03T07:30:53.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPX1Mqrlnz7ojaqSgfG2RIQ0I98=', 'Content-Length': '0'} 2022-09-03T07:30:53.812 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc6727b9bf287835-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.813 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:53.813 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:53.814 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.814 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.815 DEBUG:boto:Method: GET 2022-09-03T07:30:53.815 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.816 DEBUG:boto:Data: 2022-09-03T07:30:53.817 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.818 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.819 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.820 DEBUG:boto:Params: {} 2022-09-03T07:30:53.821 DEBUG:boto:Token: None 2022-09-03T07:30:53.823 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.824 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPX1Mqrlnz7ojaqSgfG2RIQ0I98= 2022-09-03T07:30:53.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPX1Mqrlnz7ojaqSgfG2RIQ0I98=', 'Content-Length': '0'} 2022-09-03T07:30:53.828 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000000c2e01070fd370d-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.829 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:53.829 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:53.830 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.831 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.831 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.832 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.833 DEBUG:boto:Data: 2022-09-03T07:30:53.834 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.835 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.837 DEBUG:boto:Params: {} 2022-09-03T07:30:53.838 DEBUG:boto:Token: None 2022-09-03T07:30:53.839 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ii7N2vE3S4RYXe9CVltkMcZMNI= 2022-09-03T07:30:53.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ii7N2vE3S4RYXe9CVltkMcZMNI=', 'Content-Length': '0'} 2022-09-03T07:30:53.845 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf2016604abd3e57-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.845 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:53.846 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:53.846 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.847 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.847 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.848 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.849 DEBUG:boto:Data: 2022-09-03T07:30:53.849 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.850 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.850 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.851 DEBUG:boto:Params: {} 2022-09-03T07:30:53.851 DEBUG:boto:Token: None 2022-09-03T07:30:53.854 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.855 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ii7N2vE3S4RYXe9CVltkMcZMNI= 2022-09-03T07:30:53.856 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ii7N2vE3S4RYXe9CVltkMcZMNI=', 'Content-Length': '0'} 2022-09-03T07:30:53.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048b1d319026e2163-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.859 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:30:53.859 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:53.860 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:53.860 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.861 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.861 DEBUG:boto:Method: GET 2022-09-03T07:30:53.862 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.862 DEBUG:boto:Data: 2022-09-03T07:30:53.863 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.865 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.866 DEBUG:boto:Params: {} 2022-09-03T07:30:53.867 DEBUG:boto:Token: None 2022-09-03T07:30:53.868 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPX1Mqrlnz7ojaqSgfG2RIQ0I98= 2022-09-03T07:30:53.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPX1Mqrlnz7ojaqSgfG2RIQ0I98=', 'Content-Length': '0'} 2022-09-03T07:30:53.874 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef445a2123e4b819-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.876 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:53.877 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:53.878 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.879 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.879 DEBUG:boto:Method: GET 2022-09-03T07:30:53.880 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.881 DEBUG:boto:Data: 2022-09-03T07:30:53.881 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.883 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.884 DEBUG:boto:Params: {} 2022-09-03T07:30:53.885 DEBUG:boto:Token: None 2022-09-03T07:30:53.886 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:30:53.887 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPX1Mqrlnz7ojaqSgfG2RIQ0I98= 2022-09-03T07:30:53.888 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPX1Mqrlnz7ojaqSgfG2RIQ0I98=', 'Content-Length': '0'} 2022-09-03T07:30:53.891 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1bd78eeb5aaba20-00631302ad-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.893 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.894 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.895 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:53.896 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:53.897 DEBUG:boto:Method: GET 2022-09-03T07:30:53.898 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:30:53.898 DEBUG:boto:Data: 2022-09-03T07:30:53.899 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.899 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.900 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.900 DEBUG:boto:Params: {} 2022-09-03T07:30:53.901 DEBUG:boto:Token: None 2022-09-03T07:30:53.902 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.906 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003ce3f3fe3fc084d2-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.907 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.908 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:53.909 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:53.909 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:53.910 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:53.911 DEBUG:boto:Method: GET 2022-09-03T07:30:53.911 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:30:53.912 DEBUG:boto:Data: 2022-09-03T07:30:53.912 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.913 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.913 DEBUG:boto:Port: 8000 2022-09-03T07:30:53.914 DEBUG:boto:Params: {} 2022-09-03T07:30:53.915 DEBUG:boto:Token: None 2022-09-03T07:30:53.915 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/ 2022-09-03T07:30:53.916 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0= 2022-09-03T07:30:53.916 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RM4vT4JRxLONGDCt+FN8oc+xeC0=', 'Content-Length': '0'} 2022-09-03T07:30:53.922 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a945d12b283a323f-00631302ad-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.966 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:53.967 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:53.968 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:53.969 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.969 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:30:53.970 DEBUG:boto:Data: 2022-09-03T07:30:53.970 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.971 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:53.971 DEBUG:boto:Port: 8001 2022-09-03T07:30:53.972 DEBUG:boto:Params: {} 2022-09-03T07:30:53.973 DEBUG:boto:Token: None 2022-09-03T07:30:53.974 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:53 GMT /avysmw-75/obj116 2022-09-03T07:30:53.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LmDCQM3fYoKZW+hUu3D8d6TNrd0= 2022-09-03T07:30:53.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:53 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LmDCQM3fYoKZW+hUu3D8d6TNrd0=', 'Content-Length': '0'} 2022-09-03T07:30:53.978 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa02ab0e0250579f-00631302ad-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:53 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:53.991 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:53.992 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:53.993 DEBUG:boto:Method: HEAD 2022-09-03T07:30:53.994 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:30:53.995 DEBUG:boto:Data: 2022-09-03T07:30:53.996 DEBUG:boto:Headers: {} 2022-09-03T07:30:53.997 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:53.998 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.003 DEBUG:boto:Params: {} 2022-09-03T07:30:54.005 DEBUG:boto:Token: None 2022-09-03T07:30:54.006 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj116 2022-09-03T07:30:54.007 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:awDrfh688ofO8YbN66xt90UaOZg= 2022-09-03T07:30:54.008 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:awDrfh688ofO8YbN66xt90UaOZg=', 'Content-Length': '0'} 2022-09-03T07:30:54.011 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc4bebdb67a6e1fe-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.014 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:30:54.015 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:54.015 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:54.016 DEBUG:boto:Method: GET 2022-09-03T07:30:54.016 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:30:54.018 DEBUG:boto:Data: 2022-09-03T07:30:54.019 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.020 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.020 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.021 DEBUG:boto:Params: {} 2022-09-03T07:30:54.021 DEBUG:boto:Token: None 2022-09-03T07:30:54.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj116 2022-09-03T07:30:54.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kotdOxX6TVSdLmvTe9OGGio37K4= 2022-09-03T07:30:54.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kotdOxX6TVSdLmvTe9OGGio37K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.031 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054f816f58377a137-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.032 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:30:54.034 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:30:54.035 DEBUG:boto:Method: GET 2022-09-03T07:30:54.036 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:30:54.037 DEBUG:boto:Data: 2022-09-03T07:30:54.039 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.041 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.042 DEBUG:boto:Params: {} 2022-09-03T07:30:54.044 DEBUG:boto:Token: None 2022-09-03T07:30:54.045 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj116 2022-09-03T07:30:54.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kotdOxX6TVSdLmvTe9OGGio37K4= 2022-09-03T07:30:54.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kotdOxX6TVSdLmvTe9OGGio37K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.050 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090a96d92040dc221-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.052 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.052 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.054 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:54.057 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:54.057 DEBUG:boto:Method: GET 2022-09-03T07:30:54.058 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:30:54.059 DEBUG:boto:Data: 2022-09-03T07:30:54.059 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.060 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.061 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.062 DEBUG:boto:Params: {} 2022-09-03T07:30:54.062 DEBUG:boto:Token: None 2022-09-03T07:30:54.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.069 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000131c997aa8ce6229-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.110 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.112 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.112 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.113 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:54.114 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:30:54.115 DEBUG:boto:Method: GET 2022-09-03T07:30:54.116 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:30:54.116 DEBUG:boto:Data: 2022-09-03T07:30:54.117 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.117 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.118 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.119 DEBUG:boto:Params: {} 2022-09-03T07:30:54.120 DEBUG:boto:Token: None 2022-09-03T07:30:54.121 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.127 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009bdb3dd33e05850c-00631302ae-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.128 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.129 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:30:54.129 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.130 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.130 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.131 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.131 DEBUG:boto:Method: GET 2022-09-03T07:30:54.132 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.132 DEBUG:boto:Data: 2022-09-03T07:30:54.133 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.133 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.134 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.134 DEBUG:boto:Params: {} 2022-09-03T07:30:54.136 DEBUG:boto:Token: None 2022-09-03T07:30:54.137 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.137 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mzPyFUKUjZzZFQCzEP1GgZd/Few= 2022-09-03T07:30:54.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mzPyFUKUjZzZFQCzEP1GgZd/Few=', 'Content-Length': '0'} 2022-09-03T07:30:54.140 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb23cff18bc6112a-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.141 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.141 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.142 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.143 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.143 DEBUG:boto:Method: GET 2022-09-03T07:30:54.144 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.144 DEBUG:boto:Data: 2022-09-03T07:30:54.145 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.146 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.146 DEBUG:boto:Params: {} 2022-09-03T07:30:54.147 DEBUG:boto:Token: None 2022-09-03T07:30:54.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mzPyFUKUjZzZFQCzEP1GgZd/Few= 2022-09-03T07:30:54.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mzPyFUKUjZzZFQCzEP1GgZd/Few=', 'Content-Length': '0'} 2022-09-03T07:30:54.152 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006820e1226cb76e86-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.153 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.153 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.154 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.155 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.155 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.157 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.157 DEBUG:boto:Data: 2022-09-03T07:30:54.158 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.159 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.159 DEBUG:boto:Params: {} 2022-09-03T07:30:54.160 DEBUG:boto:Token: None 2022-09-03T07:30:54.161 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MwYw/lriMAcupabImOdw14/WiNA= 2022-09-03T07:30:54.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MwYw/lriMAcupabImOdw14/WiNA=', 'Content-Length': '0'} 2022-09-03T07:30:54.164 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022501d7f6a412a33-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.165 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.165 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.166 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.166 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.167 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.168 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.168 DEBUG:boto:Data: 2022-09-03T07:30:54.169 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.169 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.170 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.170 DEBUG:boto:Params: {} 2022-09-03T07:30:54.171 DEBUG:boto:Token: None 2022-09-03T07:30:54.172 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MwYw/lriMAcupabImOdw14/WiNA= 2022-09-03T07:30:54.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MwYw/lriMAcupabImOdw14/WiNA=', 'Content-Length': '0'} 2022-09-03T07:30:54.176 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee9311f238b5cccd-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.177 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:30:54.178 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.179 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.180 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.181 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.182 DEBUG:boto:Method: GET 2022-09-03T07:30:54.183 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.184 DEBUG:boto:Data: 2022-09-03T07:30:54.185 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.186 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.187 DEBUG:boto:Params: {} 2022-09-03T07:30:54.188 DEBUG:boto:Token: None 2022-09-03T07:30:54.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.191 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mzPyFUKUjZzZFQCzEP1GgZd/Few= 2022-09-03T07:30:54.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mzPyFUKUjZzZFQCzEP1GgZd/Few=', 'Content-Length': '0'} 2022-09-03T07:30:54.195 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a30038febb3d6973-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.196 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.198 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.198 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.199 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.199 DEBUG:boto:Method: GET 2022-09-03T07:30:54.200 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.201 DEBUG:boto:Data: 2022-09-03T07:30:54.201 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.202 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.202 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.203 DEBUG:boto:Params: {} 2022-09-03T07:30:54.204 DEBUG:boto:Token: None 2022-09-03T07:30:54.205 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:30:54.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mzPyFUKUjZzZFQCzEP1GgZd/Few= 2022-09-03T07:30:54.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mzPyFUKUjZzZFQCzEP1GgZd/Few=', 'Content-Length': '0'} 2022-09-03T07:30:54.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096c57f1ea5260228-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.210 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.211 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.211 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:54.212 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:54.212 DEBUG:boto:Method: GET 2022-09-03T07:30:54.213 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:30:54.213 DEBUG:boto:Data: 2022-09-03T07:30:54.214 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.214 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.215 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.217 DEBUG:boto:Params: {} 2022-09-03T07:30:54.218 DEBUG:boto:Token: None 2022-09-03T07:30:54.218 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.223 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b40a4d6050958542-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.224 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.225 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.226 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.227 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:54.228 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:54.229 DEBUG:boto:Method: GET 2022-09-03T07:30:54.230 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:30:54.230 DEBUG:boto:Data: 2022-09-03T07:30:54.231 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.232 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.232 DEBUG:boto:Params: {} 2022-09-03T07:30:54.233 DEBUG:boto:Token: None 2022-09-03T07:30:54.234 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.235 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.236 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.241 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000019f080c9116c715b-00631302ae-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.243 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.244 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.245 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.246 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.247 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:30:54.247 DEBUG:boto:Data: 2022-09-03T07:30:54.248 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.249 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.250 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.251 DEBUG:boto:Params: {} 2022-09-03T07:30:54.251 DEBUG:boto:Token: None 2022-09-03T07:30:54.252 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124 2022-09-03T07:30:54.252 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fTnb9HhoVkZizYI+77nSegYLpQs= 2022-09-03T07:30:54.253 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fTnb9HhoVkZizYI+77nSegYLpQs=', 'Content-Length': '0'} 2022-09-03T07:30:54.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000879060839121871b-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.256 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.257 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.258 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.258 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:30:54.259 DEBUG:boto:Data: 2022-09-03T07:30:54.259 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.260 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.261 DEBUG:boto:Params: {} 2022-09-03T07:30:54.261 DEBUG:boto:Token: None 2022-09-03T07:30:54.262 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124 2022-09-03T07:30:54.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fTnb9HhoVkZizYI+77nSegYLpQs= 2022-09-03T07:30:54.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fTnb9HhoVkZizYI+77nSegYLpQs=', 'Content-Length': '0'} 2022-09-03T07:30:54.266 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ebced53d80ebe04a-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.267 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:30:54.267 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.268 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.268 DEBUG:boto:Method: GET 2022-09-03T07:30:54.269 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:30:54.270 DEBUG:boto:Data: 2022-09-03T07:30:54.270 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.271 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.272 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.272 DEBUG:boto:Params: {} 2022-09-03T07:30:54.273 DEBUG:boto:Token: None 2022-09-03T07:30:54.273 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124 2022-09-03T07:30:54.274 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xm9NE4rTTj6J8do6eIEu+2Is8Mc= 2022-09-03T07:30:54.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xm9NE4rTTj6J8do6eIEu+2Is8Mc=', 'Content-Length': '0'} 2022-09-03T07:30:54.277 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007803eba5b2b9d355-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.318 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:30:54.319 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:30:54.319 DEBUG:boto:Method: GET 2022-09-03T07:30:54.320 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:30:54.321 DEBUG:boto:Data: 2022-09-03T07:30:54.321 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.322 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.322 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.323 DEBUG:boto:Params: {} 2022-09-03T07:30:54.323 DEBUG:boto:Token: None 2022-09-03T07:30:54.324 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj124 2022-09-03T07:30:54.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xm9NE4rTTj6J8do6eIEu+2Is8Mc= 2022-09-03T07:30:54.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xm9NE4rTTj6J8do6eIEu+2Is8Mc=', 'Content-Length': '0'} 2022-09-03T07:30:54.338 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005df0523ca18e6396-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.339 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.340 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.340 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:54.341 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:54.342 DEBUG:boto:Method: GET 2022-09-03T07:30:54.342 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:30:54.343 DEBUG:boto:Data: 2022-09-03T07:30:54.343 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.344 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.344 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.345 DEBUG:boto:Params: {} 2022-09-03T07:30:54.345 DEBUG:boto:Token: None 2022-09-03T07:30:54.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.350 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cbd70793750c0cb6-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.351 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.352 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.353 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.353 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:54.354 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:30:54.354 DEBUG:boto:Method: GET 2022-09-03T07:30:54.355 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:30:54.356 DEBUG:boto:Data: 2022-09-03T07:30:54.356 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.357 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.357 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.358 DEBUG:boto:Params: {} 2022-09-03T07:30:54.359 DEBUG:boto:Token: None 2022-09-03T07:30:54.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.366 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004c8ab99a1d14c99f-00631302ae-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.368 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.369 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:30:54.370 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.371 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.372 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.373 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.374 DEBUG:boto:Method: GET 2022-09-03T07:30:54.375 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.376 DEBUG:boto:Data: 2022-09-03T07:30:54.376 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.377 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.378 DEBUG:boto:Params: {} 2022-09-03T07:30:54.379 DEBUG:boto:Token: None 2022-09-03T07:30:54.380 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7HSUIGSbDvsRcdp4rBk4l7Nlowk= 2022-09-03T07:30:54.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7HSUIGSbDvsRcdp4rBk4l7Nlowk=', 'Content-Length': '0'} 2022-09-03T07:30:54.384 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002994c7b0a731b877-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.385 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.386 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.387 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.388 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.389 DEBUG:boto:Method: GET 2022-09-03T07:30:54.390 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.391 DEBUG:boto:Data: 2022-09-03T07:30:54.391 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.393 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.394 DEBUG:boto:Params: {} 2022-09-03T07:30:54.395 DEBUG:boto:Token: None 2022-09-03T07:30:54.397 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7HSUIGSbDvsRcdp4rBk4l7Nlowk= 2022-09-03T07:30:54.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7HSUIGSbDvsRcdp4rBk4l7Nlowk=', 'Content-Length': '0'} 2022-09-03T07:30:54.402 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b606aa38870ae5bb-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.404 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.406 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.407 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.408 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.409 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.410 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.411 DEBUG:boto:Data: 2022-09-03T07:30:54.413 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.414 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.415 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.416 DEBUG:boto:Params: {} 2022-09-03T07:30:54.417 DEBUG:boto:Token: None 2022-09-03T07:30:54.419 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M2RUvBx494+Nvy+TgWSsqcYUTpE= 2022-09-03T07:30:54.421 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M2RUvBx494+Nvy+TgWSsqcYUTpE=', 'Content-Length': '0'} 2022-09-03T07:30:54.424 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079e4b242c35595a4-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.425 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.426 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.427 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.428 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.429 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.430 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.431 DEBUG:boto:Data: 2022-09-03T07:30:54.432 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.433 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.434 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.435 DEBUG:boto:Params: {} 2022-09-03T07:30:54.435 DEBUG:boto:Token: None 2022-09-03T07:30:54.436 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M2RUvBx494+Nvy+TgWSsqcYUTpE= 2022-09-03T07:30:54.437 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M2RUvBx494+Nvy+TgWSsqcYUTpE=', 'Content-Length': '0'} 2022-09-03T07:30:54.440 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e58cfafb7f379a7-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.440 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:30:54.441 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.442 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.442 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.443 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.444 DEBUG:boto:Method: GET 2022-09-03T07:30:54.446 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.446 DEBUG:boto:Data: 2022-09-03T07:30:54.447 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.447 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.448 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.448 DEBUG:boto:Params: {} 2022-09-03T07:30:54.449 DEBUG:boto:Token: None 2022-09-03T07:30:54.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.450 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7HSUIGSbDvsRcdp4rBk4l7Nlowk= 2022-09-03T07:30:54.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7HSUIGSbDvsRcdp4rBk4l7Nlowk=', 'Content-Length': '0'} 2022-09-03T07:30:54.455 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e2c1e394c3889b8-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.456 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.457 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.457 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.458 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.458 DEBUG:boto:Method: GET 2022-09-03T07:30:54.459 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.459 DEBUG:boto:Data: 2022-09-03T07:30:54.460 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.461 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.461 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.462 DEBUG:boto:Params: {} 2022-09-03T07:30:54.463 DEBUG:boto:Token: None 2022-09-03T07:30:54.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:30:54.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7HSUIGSbDvsRcdp4rBk4l7Nlowk= 2022-09-03T07:30:54.465 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7HSUIGSbDvsRcdp4rBk4l7Nlowk=', 'Content-Length': '0'} 2022-09-03T07:30:54.467 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000303105d304275cb8-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.514 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.515 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.516 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:54.517 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:54.518 DEBUG:boto:Method: GET 2022-09-03T07:30:54.519 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:30:54.520 DEBUG:boto:Data: 2022-09-03T07:30:54.520 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.521 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.522 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.523 DEBUG:boto:Params: {} 2022-09-03T07:30:54.524 DEBUG:boto:Token: None 2022-09-03T07:30:54.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.526 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.527 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.532 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045d8c83b1f21604b-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.533 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.535 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.535 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.536 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:54.537 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:54.537 DEBUG:boto:Method: GET 2022-09-03T07:30:54.538 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:30:54.539 DEBUG:boto:Data: 2022-09-03T07:30:54.539 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.540 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.540 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.541 DEBUG:boto:Params: {} 2022-09-03T07:30:54.541 DEBUG:boto:Token: None 2022-09-03T07:30:54.542 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.543 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.548 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d3f3fc029e7f8a1e-00631302ae-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.549 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.550 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.551 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.551 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.552 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:30:54.557 DEBUG:boto:Data: 2022-09-03T07:30:54.558 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.558 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.559 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.560 DEBUG:boto:Params: {} 2022-09-03T07:30:54.560 DEBUG:boto:Token: None 2022-09-03T07:30:54.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145 2022-09-03T07:30:54.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fr7IXFUEh+MzFTw2I0Sxegb0OYQ= 2022-09-03T07:30:54.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fr7IXFUEh+MzFTw2I0Sxegb0OYQ=', 'Content-Length': '0'} 2022-09-03T07:30:54.565 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ea7d8fdc08ecec41-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.567 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.568 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.570 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.570 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:30:54.571 DEBUG:boto:Data: 2022-09-03T07:30:54.572 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.572 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.573 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.573 DEBUG:boto:Params: {} 2022-09-03T07:30:54.574 DEBUG:boto:Token: None 2022-09-03T07:30:54.575 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145 2022-09-03T07:30:54.576 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fr7IXFUEh+MzFTw2I0Sxegb0OYQ= 2022-09-03T07:30:54.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fr7IXFUEh+MzFTw2I0Sxegb0OYQ=', 'Content-Length': '0'} 2022-09-03T07:30:54.583 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000059a3498bf7d74902-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.584 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:30:54.585 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.586 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.587 DEBUG:boto:Method: GET 2022-09-03T07:30:54.588 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:30:54.589 DEBUG:boto:Data: 2022-09-03T07:30:54.590 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.592 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.593 DEBUG:boto:Params: {} 2022-09-03T07:30:54.594 DEBUG:boto:Token: None 2022-09-03T07:30:54.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145 2022-09-03T07:30:54.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EYyRbXgCNUARba0Y0gkvG7J4s3c= 2022-09-03T07:30:54.596 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EYyRbXgCNUARba0Y0gkvG7J4s3c=', 'Content-Length': '0'} 2022-09-03T07:30:54.599 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000077b8c8fc2a451189-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.642 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:30:54.643 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:30:54.645 DEBUG:boto:Method: GET 2022-09-03T07:30:54.646 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:30:54.647 DEBUG:boto:Data: 2022-09-03T07:30:54.648 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.650 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.651 DEBUG:boto:Params: {} 2022-09-03T07:30:54.652 DEBUG:boto:Token: None 2022-09-03T07:30:54.655 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj145 2022-09-03T07:30:54.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EYyRbXgCNUARba0Y0gkvG7J4s3c= 2022-09-03T07:30:54.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EYyRbXgCNUARba0Y0gkvG7J4s3c=', 'Content-Length': '0'} 2022-09-03T07:30:54.663 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a558da68437510da-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.664 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.665 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.667 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:54.668 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:54.669 DEBUG:boto:Method: GET 2022-09-03T07:30:54.669 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:30:54.670 DEBUG:boto:Data: 2022-09-03T07:30:54.671 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.672 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.674 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.675 DEBUG:boto:Params: {} 2022-09-03T07:30:54.676 DEBUG:boto:Token: None 2022-09-03T07:30:54.679 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.685 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036f3575fe80e1d69-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.687 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.688 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.689 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.690 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:54.690 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:30:54.691 DEBUG:boto:Method: GET 2022-09-03T07:30:54.692 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:30:54.693 DEBUG:boto:Data: 2022-09-03T07:30:54.694 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.696 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.697 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.698 DEBUG:boto:Params: {} 2022-09-03T07:30:54.699 DEBUG:boto:Token: None 2022-09-03T07:30:54.699 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.700 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.701 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.707 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c26c8fdfc1044d05-00631302ae-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.750 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.752 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj148 2022-09-03T07:30:54.753 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.754 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.754 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.755 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.756 DEBUG:boto:Method: GET 2022-09-03T07:30:54.757 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.757 DEBUG:boto:Data: 2022-09-03T07:30:54.758 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.759 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.760 DEBUG:boto:Params: {} 2022-09-03T07:30:54.760 DEBUG:boto:Token: None 2022-09-03T07:30:54.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIzdRQnnJ2xLpCPezsNDlxQQ0CU= 2022-09-03T07:30:54.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIzdRQnnJ2xLpCPezsNDlxQQ0CU=', 'Content-Length': '0'} 2022-09-03T07:30:54.765 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000213b6aa782c1304d-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.767 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.768 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.769 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.770 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.771 DEBUG:boto:Method: GET 2022-09-03T07:30:54.772 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.773 DEBUG:boto:Data: 2022-09-03T07:30:54.774 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.774 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.775 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.776 DEBUG:boto:Params: {} 2022-09-03T07:30:54.776 DEBUG:boto:Token: None 2022-09-03T07:30:54.777 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIzdRQnnJ2xLpCPezsNDlxQQ0CU= 2022-09-03T07:30:54.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIzdRQnnJ2xLpCPezsNDlxQQ0CU=', 'Content-Length': '0'} 2022-09-03T07:30:54.786 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000381c60c3bdb90182-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.789 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.790 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.791 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.792 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.794 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.795 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.795 DEBUG:boto:Data: 2022-09-03T07:30:54.796 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.797 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.797 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.798 DEBUG:boto:Params: {} 2022-09-03T07:30:54.798 DEBUG:boto:Token: None 2022-09-03T07:30:54.799 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fOVCZu4LF57yR4yYKaxnoi3lfuo= 2022-09-03T07:30:54.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fOVCZu4LF57yR4yYKaxnoi3lfuo=', 'Content-Length': '0'} 2022-09-03T07:30:54.803 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a5cfdbfdcbab070b-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.804 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.805 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.805 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.806 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.806 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.807 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.807 DEBUG:boto:Data: 2022-09-03T07:30:54.808 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.809 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.810 DEBUG:boto:Params: {} 2022-09-03T07:30:54.810 DEBUG:boto:Token: None 2022-09-03T07:30:54.811 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fOVCZu4LF57yR4yYKaxnoi3lfuo= 2022-09-03T07:30:54.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fOVCZu4LF57yR4yYKaxnoi3lfuo=', 'Content-Length': '0'} 2022-09-03T07:30:54.815 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a663afc31bd01b0-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.816 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj148 2022-09-03T07:30:54.817 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.818 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.819 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.819 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.821 DEBUG:boto:Method: GET 2022-09-03T07:30:54.821 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.822 DEBUG:boto:Data: 2022-09-03T07:30:54.822 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.823 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.824 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.824 DEBUG:boto:Params: {} 2022-09-03T07:30:54.825 DEBUG:boto:Token: None 2022-09-03T07:30:54.826 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIzdRQnnJ2xLpCPezsNDlxQQ0CU= 2022-09-03T07:30:54.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIzdRQnnJ2xLpCPezsNDlxQQ0CU=', 'Content-Length': '0'} 2022-09-03T07:30:54.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d5fbeff9c6a3b83f-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.831 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.832 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.832 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.833 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.833 DEBUG:boto:Method: GET 2022-09-03T07:30:54.834 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.835 DEBUG:boto:Data: 2022-09-03T07:30:54.835 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.836 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.836 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.837 DEBUG:boto:Params: {} 2022-09-03T07:30:54.838 DEBUG:boto:Token: None 2022-09-03T07:30:54.839 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:30:54.840 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yIzdRQnnJ2xLpCPezsNDlxQQ0CU= 2022-09-03T07:30:54.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yIzdRQnnJ2xLpCPezsNDlxQQ0CU=', 'Content-Length': '0'} 2022-09-03T07:30:54.844 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7ec1fe3fe95adc0-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.886 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.887 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.887 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:54.888 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:54.889 DEBUG:boto:Method: GET 2022-09-03T07:30:54.889 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:30:54.890 DEBUG:boto:Data: 2022-09-03T07:30:54.890 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.891 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.891 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.892 DEBUG:boto:Params: {} 2022-09-03T07:30:54.892 DEBUG:boto:Token: None 2022-09-03T07:30:54.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.894 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.897 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000777c42ea202ad871-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.898 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.899 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:54.899 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:54.900 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:54.901 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:54.901 DEBUG:boto:Method: GET 2022-09-03T07:30:54.902 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:30:54.902 DEBUG:boto:Data: 2022-09-03T07:30:54.903 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.903 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.904 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.904 DEBUG:boto:Params: {} 2022-09-03T07:30:54.905 DEBUG:boto:Token: None 2022-09-03T07:30:54.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/ 2022-09-03T07:30:54.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4= 2022-09-03T07:30:54.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cdMxeuKJKrnQIbU7n2UhLwbi3K4=', 'Content-Length': '0'} 2022-09-03T07:30:54.917 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c5a41d7d1c8ad859-00631302ae-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.918 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:54.919 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.920 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.921 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.922 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:30:54.923 DEBUG:boto:Data: 2022-09-03T07:30:54.924 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.925 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.926 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.927 DEBUG:boto:Params: {} 2022-09-03T07:30:54.927 DEBUG:boto:Token: None 2022-09-03T07:30:54.928 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148 2022-09-03T07:30:54.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xNeQ4gmHEOMU9NLKsAR9bjOmmdo= 2022-09-03T07:30:54.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xNeQ4gmHEOMU9NLKsAR9bjOmmdo=', 'Content-Length': '0'} 2022-09-03T07:30:54.932 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004a19eb7caf001a17-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.933 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.934 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.935 DEBUG:boto:Method: HEAD 2022-09-03T07:30:54.935 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:30:54.936 DEBUG:boto:Data: 2022-09-03T07:30:54.936 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.937 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.937 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.938 DEBUG:boto:Params: {} 2022-09-03T07:30:54.939 DEBUG:boto:Token: None 2022-09-03T07:30:54.939 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148 2022-09-03T07:30:54.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xNeQ4gmHEOMU9NLKsAR9bjOmmdo= 2022-09-03T07:30:54.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xNeQ4gmHEOMU9NLKsAR9bjOmmdo=', 'Content-Length': '0'} 2022-09-03T07:30:54.943 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045905123f53e2e5b-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.944 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj148 2022-09-03T07:30:54.944 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.945 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.945 DEBUG:boto:Method: GET 2022-09-03T07:30:54.946 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:30:54.946 DEBUG:boto:Data: 2022-09-03T07:30:54.947 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:54.948 DEBUG:boto:Port: 8001 2022-09-03T07:30:54.948 DEBUG:boto:Params: {} 2022-09-03T07:30:54.949 DEBUG:boto:Token: None 2022-09-03T07:30:54.950 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148 2022-09-03T07:30:54.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:giUinb5VKReR4C7TsOXv/SoJPQk= 2022-09-03T07:30:54.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:giUinb5VKReR4C7TsOXv/SoJPQk=', 'Content-Length': '0'} 2022-09-03T07:30:54.953 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000caba370d640332d6-00631302ae-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:54.954 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:30:54.955 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:30:54.955 DEBUG:boto:Method: GET 2022-09-03T07:30:54.956 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:30:54.957 DEBUG:boto:Data: 2022-09-03T07:30:54.957 DEBUG:boto:Headers: {} 2022-09-03T07:30:54.958 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:54.958 DEBUG:boto:Port: 8000 2022-09-03T07:30:54.959 DEBUG:boto:Params: {} 2022-09-03T07:30:54.959 DEBUG:boto:Token: None 2022-09-03T07:30:54.960 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:54 GMT /avysmw-75/obj148 2022-09-03T07:30:54.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:giUinb5VKReR4C7TsOXv/SoJPQk= 2022-09-03T07:30:54.961 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:54 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:giUinb5VKReR4C7TsOXv/SoJPQk=', 'Content-Length': '0'} 2022-09-03T07:30:54.964 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b649cacc741dcb2-00631302ae-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:54 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.010 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.011 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.011 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:55.012 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:55.013 DEBUG:boto:Method: GET 2022-09-03T07:30:55.013 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:30:55.014 DEBUG:boto:Data: 2022-09-03T07:30:55.014 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.015 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.015 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.016 DEBUG:boto:Params: {} 2022-09-03T07:30:55.016 DEBUG:boto:Token: None 2022-09-03T07:30:55.017 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.018 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.021 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f9048132496e01a9-00631302af-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.022 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.024 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.024 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.025 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:55.025 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:30:55.026 DEBUG:boto:Method: GET 2022-09-03T07:30:55.027 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:30:55.027 DEBUG:boto:Data: 2022-09-03T07:30:55.028 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.029 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.030 DEBUG:boto:Params: {} 2022-09-03T07:30:55.031 DEBUG:boto:Token: None 2022-09-03T07:30:55.031 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.038 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000366ff5e68732c343-00631302af-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.040 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.040 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:30:55.041 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.042 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.043 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.043 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.044 DEBUG:boto:Method: GET 2022-09-03T07:30:55.045 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.046 DEBUG:boto:Data: 2022-09-03T07:30:55.047 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.048 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.049 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.050 DEBUG:boto:Params: {} 2022-09-03T07:30:55.051 DEBUG:boto:Token: None 2022-09-03T07:30:55.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEnObn0kNyRtQyISgfdedrsWmUE= 2022-09-03T07:30:55.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEnObn0kNyRtQyISgfdedrsWmUE=', 'Content-Length': '0'} 2022-09-03T07:30:55.058 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000620c8833c1536633-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.059 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.060 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.061 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.062 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.063 DEBUG:boto:Method: GET 2022-09-03T07:30:55.065 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.065 DEBUG:boto:Data: 2022-09-03T07:30:55.066 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.067 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.068 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.069 DEBUG:boto:Params: {} 2022-09-03T07:30:55.071 DEBUG:boto:Token: None 2022-09-03T07:30:55.072 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.073 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEnObn0kNyRtQyISgfdedrsWmUE= 2022-09-03T07:30:55.074 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEnObn0kNyRtQyISgfdedrsWmUE=', 'Content-Length': '0'} 2022-09-03T07:30:55.078 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc129dc3041fc52e-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.079 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.079 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.080 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.080 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.081 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.082 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.082 DEBUG:boto:Data: 2022-09-03T07:30:55.083 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.083 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.084 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.084 DEBUG:boto:Params: {} 2022-09-03T07:30:55.085 DEBUG:boto:Token: None 2022-09-03T07:30:55.086 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JiMRw2dQIgm9nXAocqxo45MwCYI= 2022-09-03T07:30:55.088 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JiMRw2dQIgm9nXAocqxo45MwCYI=', 'Content-Length': '0'} 2022-09-03T07:30:55.091 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009629953d2f6601d9-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.093 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.093 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.094 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.094 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.095 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.095 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.096 DEBUG:boto:Data: 2022-09-03T07:30:55.098 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.099 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.101 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.101 DEBUG:boto:Params: {} 2022-09-03T07:30:55.103 DEBUG:boto:Token: None 2022-09-03T07:30:55.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.105 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JiMRw2dQIgm9nXAocqxo45MwCYI= 2022-09-03T07:30:55.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JiMRw2dQIgm9nXAocqxo45MwCYI=', 'Content-Length': '0'} 2022-09-03T07:30:55.108 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000070d0a6486168e66b-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.109 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:30:55.110 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.110 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.111 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.111 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.112 DEBUG:boto:Method: GET 2022-09-03T07:30:55.113 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.113 DEBUG:boto:Data: 2022-09-03T07:30:55.114 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.115 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.115 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.116 DEBUG:boto:Params: {} 2022-09-03T07:30:55.116 DEBUG:boto:Token: None 2022-09-03T07:30:55.117 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.118 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEnObn0kNyRtQyISgfdedrsWmUE= 2022-09-03T07:30:55.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEnObn0kNyRtQyISgfdedrsWmUE=', 'Content-Length': '0'} 2022-09-03T07:30:55.121 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032d765316b0c4c41-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.122 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.122 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.123 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.123 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.124 DEBUG:boto:Method: GET 2022-09-03T07:30:55.124 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.125 DEBUG:boto:Data: 2022-09-03T07:30:55.125 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.126 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.127 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.127 DEBUG:boto:Params: {} 2022-09-03T07:30:55.128 DEBUG:boto:Token: None 2022-09-03T07:30:55.128 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:30:55.129 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GEnObn0kNyRtQyISgfdedrsWmUE= 2022-09-03T07:30:55.130 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GEnObn0kNyRtQyISgfdedrsWmUE=', 'Content-Length': '0'} 2022-09-03T07:30:55.132 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2c29184bcb7e3ff-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.178 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.179 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.180 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:55.181 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:55.182 DEBUG:boto:Method: GET 2022-09-03T07:30:55.183 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:30:55.184 DEBUG:boto:Data: 2022-09-03T07:30:55.184 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.185 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.186 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.189 DEBUG:boto:Params: {} 2022-09-03T07:30:55.192 DEBUG:boto:Token: None 2022-09-03T07:30:55.193 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.194 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.198 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c4b7ebe27ae2713e-00631302af-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.199 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.200 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.201 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.202 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:55.203 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:55.204 DEBUG:boto:Method: GET 2022-09-03T07:30:55.204 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:30:55.205 DEBUG:boto:Data: 2022-09-03T07:30:55.205 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.206 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.206 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.207 DEBUG:boto:Params: {} 2022-09-03T07:30:55.208 DEBUG:boto:Token: None 2022-09-03T07:30:55.208 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.209 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.214 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008a9b14226c23d330-00631302af-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.216 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.217 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.217 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.218 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.218 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:30:55.219 DEBUG:boto:Data: 2022-09-03T07:30:55.220 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.220 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.221 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.221 DEBUG:boto:Params: {} 2022-09-03T07:30:55.222 DEBUG:boto:Token: None 2022-09-03T07:30:55.223 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155 2022-09-03T07:30:55.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8BMW6crVcbGgeNdbP45OSh8q2Q4= 2022-09-03T07:30:55.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8BMW6crVcbGgeNdbP45OSh8q2Q4=', 'Content-Length': '0'} 2022-09-03T07:30:55.229 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000602febb9d37bbb06-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.231 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.231 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.232 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.232 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:30:55.233 DEBUG:boto:Data: 2022-09-03T07:30:55.233 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.234 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.235 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.236 DEBUG:boto:Params: {} 2022-09-03T07:30:55.236 DEBUG:boto:Token: None 2022-09-03T07:30:55.237 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155 2022-09-03T07:30:55.237 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8BMW6crVcbGgeNdbP45OSh8q2Q4= 2022-09-03T07:30:55.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8BMW6crVcbGgeNdbP45OSh8q2Q4=', 'Content-Length': '0'} 2022-09-03T07:30:55.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007644437b1e45341b-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.241 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:30:55.241 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.242 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.243 DEBUG:boto:Method: GET 2022-09-03T07:30:55.243 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:30:55.244 DEBUG:boto:Data: 2022-09-03T07:30:55.244 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.245 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.245 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.246 DEBUG:boto:Params: {} 2022-09-03T07:30:55.247 DEBUG:boto:Token: None 2022-09-03T07:30:55.248 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155 2022-09-03T07:30:55.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hn34xJKNFqs7dCh3aSbp71Xhl/A= 2022-09-03T07:30:55.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hn34xJKNFqs7dCh3aSbp71Xhl/A=', 'Content-Length': '0'} 2022-09-03T07:30:55.254 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004e1365047e4dd77b-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.255 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:30:55.256 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:30:55.257 DEBUG:boto:Method: GET 2022-09-03T07:30:55.258 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:30:55.259 DEBUG:boto:Data: 2022-09-03T07:30:55.260 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.261 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.262 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.263 DEBUG:boto:Params: {} 2022-09-03T07:30:55.264 DEBUG:boto:Token: None 2022-09-03T07:30:55.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj155 2022-09-03T07:30:55.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Hn34xJKNFqs7dCh3aSbp71Xhl/A= 2022-09-03T07:30:55.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Hn34xJKNFqs7dCh3aSbp71Xhl/A=', 'Content-Length': '0'} 2022-09-03T07:30:55.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff9a5237dab24481-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.272 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.273 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.274 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:55.275 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:55.276 DEBUG:boto:Method: GET 2022-09-03T07:30:55.276 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:30:55.277 DEBUG:boto:Data: 2022-09-03T07:30:55.278 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.279 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.280 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.280 DEBUG:boto:Params: {} 2022-09-03T07:30:55.281 DEBUG:boto:Token: None 2022-09-03T07:30:55.283 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.284 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.285 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.289 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001882e5e4117a7ab2-00631302af-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.290 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.291 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.292 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.293 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:55.294 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:30:55.294 DEBUG:boto:Method: GET 2022-09-03T07:30:55.295 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:30:55.296 DEBUG:boto:Data: 2022-09-03T07:30:55.296 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.297 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.298 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.298 DEBUG:boto:Params: {} 2022-09-03T07:30:55.299 DEBUG:boto:Token: None 2022-09-03T07:30:55.300 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.307 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000032e435697d4d247e-00631302af-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.354 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.355 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:30:55.356 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.356 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.357 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.358 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.358 DEBUG:boto:Method: GET 2022-09-03T07:30:55.359 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.359 DEBUG:boto:Data: 2022-09-03T07:30:55.360 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.360 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.361 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.361 DEBUG:boto:Params: {} 2022-09-03T07:30:55.362 DEBUG:boto:Token: None 2022-09-03T07:30:55.363 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.364 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:flveii3hNtUIApbyz0wiivWbkHI= 2022-09-03T07:30:55.365 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:flveii3hNtUIApbyz0wiivWbkHI=', 'Content-Length': '0'} 2022-09-03T07:30:55.367 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044e2b44eebcd9771-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.368 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.372 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.373 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.374 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.374 DEBUG:boto:Method: GET 2022-09-03T07:30:55.376 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.376 DEBUG:boto:Data: 2022-09-03T07:30:55.377 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.379 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.382 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.383 DEBUG:boto:Params: {} 2022-09-03T07:30:55.384 DEBUG:boto:Token: None 2022-09-03T07:30:55.384 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.385 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:flveii3hNtUIApbyz0wiivWbkHI= 2022-09-03T07:30:55.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:flveii3hNtUIApbyz0wiivWbkHI=', 'Content-Length': '0'} 2022-09-03T07:30:55.388 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a83f5ba88e3a71f9-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.389 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.390 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.391 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.391 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.392 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.393 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.393 DEBUG:boto:Data: 2022-09-03T07:30:55.394 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.394 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.395 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.395 DEBUG:boto:Params: {} 2022-09-03T07:30:55.396 DEBUG:boto:Token: None 2022-09-03T07:30:55.396 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.397 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P/2UGvjZWrx7rz6QBo1R3VSKq9o= 2022-09-03T07:30:55.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P/2UGvjZWrx7rz6QBo1R3VSKq9o=', 'Content-Length': '0'} 2022-09-03T07:30:55.400 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d9805ac00d8deb16-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.400 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.401 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.401 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.402 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.403 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.403 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.404 DEBUG:boto:Data: 2022-09-03T07:30:55.404 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.405 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.405 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.406 DEBUG:boto:Params: {} 2022-09-03T07:30:55.406 DEBUG:boto:Token: None 2022-09-03T07:30:55.407 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P/2UGvjZWrx7rz6QBo1R3VSKq9o= 2022-09-03T07:30:55.409 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P/2UGvjZWrx7rz6QBo1R3VSKq9o=', 'Content-Length': '0'} 2022-09-03T07:30:55.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000391ecdbc462fabe0-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.412 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:30:55.412 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.413 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.413 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.414 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.414 DEBUG:boto:Method: GET 2022-09-03T07:30:55.415 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.415 DEBUG:boto:Data: 2022-09-03T07:30:55.416 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.417 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.417 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.418 DEBUG:boto:Params: {} 2022-09-03T07:30:55.419 DEBUG:boto:Token: None 2022-09-03T07:30:55.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:flveii3hNtUIApbyz0wiivWbkHI= 2022-09-03T07:30:55.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:flveii3hNtUIApbyz0wiivWbkHI=', 'Content-Length': '0'} 2022-09-03T07:30:55.423 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f9d14f2c5212bca-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.466 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.467 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.468 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.468 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.469 DEBUG:boto:Method: GET 2022-09-03T07:30:55.470 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.471 DEBUG:boto:Data: 2022-09-03T07:30:55.471 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.473 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.474 DEBUG:boto:Params: {} 2022-09-03T07:30:55.475 DEBUG:boto:Token: None 2022-09-03T07:30:55.476 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:30:55.477 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:flveii3hNtUIApbyz0wiivWbkHI= 2022-09-03T07:30:55.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:flveii3hNtUIApbyz0wiivWbkHI=', 'Content-Length': '0'} 2022-09-03T07:30:55.479 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000fa7398f98cb63af-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.481 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.481 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.482 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:55.483 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:55.484 DEBUG:boto:Method: GET 2022-09-03T07:30:55.484 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:30:55.485 DEBUG:boto:Data: 2022-09-03T07:30:55.486 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.488 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.488 DEBUG:boto:Params: {} 2022-09-03T07:30:55.491 DEBUG:boto:Token: None 2022-09-03T07:30:55.492 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.498 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000efb758af9d653940-00631302af-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.499 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.501 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.502 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.503 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:55.503 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:55.504 DEBUG:boto:Method: GET 2022-09-03T07:30:55.505 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:30:55.506 DEBUG:boto:Data: 2022-09-03T07:30:55.507 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.509 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.510 DEBUG:boto:Params: {} 2022-09-03T07:30:55.511 DEBUG:boto:Token: None 2022-09-03T07:30:55.512 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.520 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000088f059f693857cc2-00631302af-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.522 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.523 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.524 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.525 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.526 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:30:55.527 DEBUG:boto:Data: 2022-09-03T07:30:55.528 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.530 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.531 DEBUG:boto:Params: {} 2022-09-03T07:30:55.532 DEBUG:boto:Token: None 2022-09-03T07:30:55.533 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17 2022-09-03T07:30:55.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mbUEEOUcPhxJaRqumo5LYidzOXE= 2022-09-03T07:30:55.535 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mbUEEOUcPhxJaRqumo5LYidzOXE=', 'Content-Length': '0'} 2022-09-03T07:30:55.537 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b9c0a5694664297-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.538 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.539 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.539 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.540 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:30:55.540 DEBUG:boto:Data: 2022-09-03T07:30:55.541 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.542 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.542 DEBUG:boto:Params: {} 2022-09-03T07:30:55.543 DEBUG:boto:Token: None 2022-09-03T07:30:55.543 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17 2022-09-03T07:30:55.544 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:mbUEEOUcPhxJaRqumo5LYidzOXE= 2022-09-03T07:30:55.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:mbUEEOUcPhxJaRqumo5LYidzOXE=', 'Content-Length': '0'} 2022-09-03T07:30:55.547 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000173533696eb03273-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.548 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:30:55.548 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.549 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.549 DEBUG:boto:Method: GET 2022-09-03T07:30:55.550 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:30:55.550 DEBUG:boto:Data: 2022-09-03T07:30:55.551 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.551 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.552 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.552 DEBUG:boto:Params: {} 2022-09-03T07:30:55.553 DEBUG:boto:Token: None 2022-09-03T07:30:55.553 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17 2022-09-03T07:30:55.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Zn/Rxl711TVQrZB8SEGFnMQEEI= 2022-09-03T07:30:55.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Zn/Rxl711TVQrZB8SEGFnMQEEI=', 'Content-Length': '0'} 2022-09-03T07:30:55.563 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087dbe6b8cca371ba-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.564 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:30:55.565 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:30:55.565 DEBUG:boto:Method: GET 2022-09-03T07:30:55.566 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:30:55.567 DEBUG:boto:Data: 2022-09-03T07:30:55.567 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.568 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.569 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.570 DEBUG:boto:Params: {} 2022-09-03T07:30:55.570 DEBUG:boto:Token: None 2022-09-03T07:30:55.571 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj17 2022-09-03T07:30:55.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7Zn/Rxl711TVQrZB8SEGFnMQEEI= 2022-09-03T07:30:55.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7Zn/Rxl711TVQrZB8SEGFnMQEEI=', 'Content-Length': '0'} 2022-09-03T07:30:55.575 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000094987ef57d0eb76d-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.618 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.619 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.621 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:55.622 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:55.623 DEBUG:boto:Method: GET 2022-09-03T07:30:55.624 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:30:55.625 DEBUG:boto:Data: 2022-09-03T07:30:55.626 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.627 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.628 DEBUG:boto:Params: {} 2022-09-03T07:30:55.628 DEBUG:boto:Token: None 2022-09-03T07:30:55.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.634 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000565e17ff9d7c0d8a-00631302af-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.636 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.636 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.637 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.638 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:55.638 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:30:55.639 DEBUG:boto:Method: GET 2022-09-03T07:30:55.640 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:30:55.642 DEBUG:boto:Data: 2022-09-03T07:30:55.643 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.645 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.646 DEBUG:boto:Params: {} 2022-09-03T07:30:55.647 DEBUG:boto:Token: None 2022-09-03T07:30:55.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.651 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.657 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000095a2752962ec7f86-00631302af-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.657 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.659 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj185 2022-09-03T07:30:55.659 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.660 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.661 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.661 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.662 DEBUG:boto:Method: GET 2022-09-03T07:30:55.663 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.663 DEBUG:boto:Data: 2022-09-03T07:30:55.664 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.664 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.665 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.665 DEBUG:boto:Params: {} 2022-09-03T07:30:55.667 DEBUG:boto:Token: None 2022-09-03T07:30:55.668 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.668 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pzeSNkrNXsTtoaDgVWgWxIW9hf0= 2022-09-03T07:30:55.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pzeSNkrNXsTtoaDgVWgWxIW9hf0=', 'Content-Length': '0'} 2022-09-03T07:30:55.672 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b26dcd6dc3df7002-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.718 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.719 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.719 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.720 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.720 DEBUG:boto:Method: GET 2022-09-03T07:30:55.721 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.722 DEBUG:boto:Data: 2022-09-03T07:30:55.722 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.723 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.723 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.724 DEBUG:boto:Params: {} 2022-09-03T07:30:55.725 DEBUG:boto:Token: None 2022-09-03T07:30:55.726 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pzeSNkrNXsTtoaDgVWgWxIW9hf0= 2022-09-03T07:30:55.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pzeSNkrNXsTtoaDgVWgWxIW9hf0=', 'Content-Length': '0'} 2022-09-03T07:30:55.729 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000badea30271c83317-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.730 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.731 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.731 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.732 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.732 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.733 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.733 DEBUG:boto:Data: 2022-09-03T07:30:55.734 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.735 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.737 DEBUG:boto:Params: {} 2022-09-03T07:30:55.738 DEBUG:boto:Token: None 2022-09-03T07:30:55.738 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KGoxCvGjLqRnRAeapK0gOFBh0rk= 2022-09-03T07:30:55.739 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KGoxCvGjLqRnRAeapK0gOFBh0rk=', 'Content-Length': '0'} 2022-09-03T07:30:55.741 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000072b55890f4bf8ad1-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.742 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.743 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.743 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.744 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.744 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.745 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.745 DEBUG:boto:Data: 2022-09-03T07:30:55.746 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.747 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.748 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.748 DEBUG:boto:Params: {} 2022-09-03T07:30:55.749 DEBUG:boto:Token: None 2022-09-03T07:30:55.749 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.752 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KGoxCvGjLqRnRAeapK0gOFBh0rk= 2022-09-03T07:30:55.752 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KGoxCvGjLqRnRAeapK0gOFBh0rk=', 'Content-Length': '0'} 2022-09-03T07:30:55.756 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018804fa6bdd9d60e-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.757 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj185 2022-09-03T07:30:55.758 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.759 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.760 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.761 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.761 DEBUG:boto:Method: GET 2022-09-03T07:30:55.762 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.763 DEBUG:boto:Data: 2022-09-03T07:30:55.764 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.764 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.765 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.766 DEBUG:boto:Params: {} 2022-09-03T07:30:55.767 DEBUG:boto:Token: None 2022-09-03T07:30:55.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pzeSNkrNXsTtoaDgVWgWxIW9hf0= 2022-09-03T07:30:55.769 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pzeSNkrNXsTtoaDgVWgWxIW9hf0=', 'Content-Length': '0'} 2022-09-03T07:30:55.771 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2add2c3f91becb6-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.772 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.773 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.773 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.774 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.775 DEBUG:boto:Method: GET 2022-09-03T07:30:55.775 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.776 DEBUG:boto:Data: 2022-09-03T07:30:55.776 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.778 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.779 DEBUG:boto:Params: {} 2022-09-03T07:30:55.779 DEBUG:boto:Token: None 2022-09-03T07:30:55.780 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:30:55.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pzeSNkrNXsTtoaDgVWgWxIW9hf0= 2022-09-03T07:30:55.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pzeSNkrNXsTtoaDgVWgWxIW9hf0=', 'Content-Length': '0'} 2022-09-03T07:30:55.784 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7057e0677172e6c-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.785 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.786 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.786 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:55.787 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:55.788 DEBUG:boto:Method: GET 2022-09-03T07:30:55.788 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:30:55.789 DEBUG:boto:Data: 2022-09-03T07:30:55.790 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.790 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.791 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.792 DEBUG:boto:Params: {} 2022-09-03T07:30:55.793 DEBUG:boto:Token: None 2022-09-03T07:30:55.793 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.797 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008e5fabee9ca51c05-00631302af-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.798 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.800 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.801 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.802 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:55.803 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:55.803 DEBUG:boto:Method: GET 2022-09-03T07:30:55.804 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:30:55.805 DEBUG:boto:Data: 2022-09-03T07:30:55.805 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.806 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.806 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.807 DEBUG:boto:Params: {} 2022-09-03T07:30:55.808 DEBUG:boto:Token: None 2022-09-03T07:30:55.809 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.809 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.810 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.816 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b664a0c8b7601d61-00631302af-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.817 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.819 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.819 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.820 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.821 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:30:55.822 DEBUG:boto:Data: 2022-09-03T07:30:55.823 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.824 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.825 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.826 DEBUG:boto:Params: {} 2022-09-03T07:30:55.827 DEBUG:boto:Token: None 2022-09-03T07:30:55.828 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185 2022-09-03T07:30:55.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tUe8sCn8238utVXWtMPQBmKmRRs= 2022-09-03T07:30:55.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tUe8sCn8238utVXWtMPQBmKmRRs=', 'Content-Length': '0'} 2022-09-03T07:30:55.833 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f4fabb9a1e7dd5ef-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.834 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.835 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.836 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.836 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:30:55.837 DEBUG:boto:Data: 2022-09-03T07:30:55.837 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.838 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.838 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.839 DEBUG:boto:Params: {} 2022-09-03T07:30:55.840 DEBUG:boto:Token: None 2022-09-03T07:30:55.840 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185 2022-09-03T07:30:55.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tUe8sCn8238utVXWtMPQBmKmRRs= 2022-09-03T07:30:55.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tUe8sCn8238utVXWtMPQBmKmRRs=', 'Content-Length': '0'} 2022-09-03T07:30:55.844 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ebb6835a07fcc9ee-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.845 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj185 2022-09-03T07:30:55.845 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.846 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.847 DEBUG:boto:Method: GET 2022-09-03T07:30:55.847 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:30:55.848 DEBUG:boto:Data: 2022-09-03T07:30:55.849 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.850 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.851 DEBUG:boto:Params: {} 2022-09-03T07:30:55.852 DEBUG:boto:Token: None 2022-09-03T07:30:55.852 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185 2022-09-03T07:30:55.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YIoS9Bj6TH9SBIiilydgtCom86o= 2022-09-03T07:30:55.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YIoS9Bj6TH9SBIiilydgtCom86o=', 'Content-Length': '0'} 2022-09-03T07:30:55.857 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c732791ae8932f9e-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.858 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:30:55.858 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:30:55.859 DEBUG:boto:Method: GET 2022-09-03T07:30:55.860 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:30:55.860 DEBUG:boto:Data: 2022-09-03T07:30:55.861 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.862 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.862 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.863 DEBUG:boto:Params: {} 2022-09-03T07:30:55.864 DEBUG:boto:Token: None 2022-09-03T07:30:55.865 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj185 2022-09-03T07:30:55.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YIoS9Bj6TH9SBIiilydgtCom86o= 2022-09-03T07:30:55.866 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YIoS9Bj6TH9SBIiilydgtCom86o=', 'Content-Length': '0'} 2022-09-03T07:30:55.868 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d2fd7b15b344fd9f-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.869 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.870 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.871 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:55.872 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:55.872 DEBUG:boto:Method: GET 2022-09-03T07:30:55.873 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:30:55.873 DEBUG:boto:Data: 2022-09-03T07:30:55.874 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.875 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.876 DEBUG:boto:Params: {} 2022-09-03T07:30:55.877 DEBUG:boto:Token: None 2022-09-03T07:30:55.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.882 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f9520fbb0fc90eed-00631302af-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.882 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.883 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:55.884 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:55.885 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:55.886 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:30:55.887 DEBUG:boto:Method: GET 2022-09-03T07:30:55.888 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:30:55.888 DEBUG:boto:Data: 2022-09-03T07:30:55.889 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.890 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.891 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.891 DEBUG:boto:Params: {} 2022-09-03T07:30:55.892 DEBUG:boto:Token: None 2022-09-03T07:30:55.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/ 2022-09-03T07:30:55.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM= 2022-09-03T07:30:55.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4NqXRz+Tu5xFXXpmhCxG+IfEncM=', 'Content-Length': '0'} 2022-09-03T07:30:55.898 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002f8282eb04b8ccf5-00631302af-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.899 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:55.900 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:30:55.901 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:55.902 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:55.903 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.903 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.904 DEBUG:boto:Method: GET 2022-09-03T07:30:55.904 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.905 DEBUG:boto:Data: 2022-09-03T07:30:55.906 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.907 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.908 DEBUG:boto:Params: {} 2022-09-03T07:30:55.909 DEBUG:boto:Token: None 2022-09-03T07:30:55.909 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IUwA16Gja/fHtifxHMzuaSPrmjM= 2022-09-03T07:30:55.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IUwA16Gja/fHtifxHMzuaSPrmjM=', 'Content-Length': '0'} 2022-09-03T07:30:55.914 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da40a6daac15bcaa-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.915 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:55.916 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:55.916 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.917 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.917 DEBUG:boto:Method: GET 2022-09-03T07:30:55.918 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.920 DEBUG:boto:Data: 2022-09-03T07:30:55.920 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:55.922 DEBUG:boto:Port: 8000 2022-09-03T07:30:55.923 DEBUG:boto:Params: {} 2022-09-03T07:30:55.924 DEBUG:boto:Token: None 2022-09-03T07:30:55.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IUwA16Gja/fHtifxHMzuaSPrmjM= 2022-09-03T07:30:55.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IUwA16Gja/fHtifxHMzuaSPrmjM=', 'Content-Length': '0'} 2022-09-03T07:30:55.929 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006040ca617a4bce02-00631302af-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:55.975 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:55.977 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:55.978 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.980 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.981 DEBUG:boto:Method: HEAD 2022-09-03T07:30:55.982 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.983 DEBUG:boto:Data: 2022-09-03T07:30:55.984 DEBUG:boto:Headers: {} 2022-09-03T07:30:55.986 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:55.987 DEBUG:boto:Port: 8001 2022-09-03T07:30:55.988 DEBUG:boto:Params: {} 2022-09-03T07:30:55.990 DEBUG:boto:Token: None 2022-09-03T07:30:55.991 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:55 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:55.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BztXQ6n4bA1vox+vHTqj3hHeLgI= 2022-09-03T07:30:55.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BztXQ6n4bA1vox+vHTqj3hHeLgI=', 'Content-Length': '0'} 2022-09-03T07:30:55.996 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000633f1688f8c64ca5-00631302af-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.006 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:56.007 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:56.007 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.008 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.009 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.009 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.010 DEBUG:boto:Data: 2022-09-03T07:30:56.011 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.012 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.012 DEBUG:boto:Params: {} 2022-09-03T07:30:56.013 DEBUG:boto:Token: None 2022-09-03T07:30:56.014 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.016 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u6V18pGnD7oTrorrSvfan7xAB+M= 2022-09-03T07:30:56.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u6V18pGnD7oTrorrSvfan7xAB+M=', 'Content-Length': '0'} 2022-09-03T07:30:56.020 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bdb59eac46b4f325-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.022 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:30:56.023 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:56.024 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:56.025 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.026 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.027 DEBUG:boto:Method: GET 2022-09-03T07:30:56.027 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.028 DEBUG:boto:Data: 2022-09-03T07:30:56.028 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.029 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.029 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.030 DEBUG:boto:Params: {} 2022-09-03T07:30:56.031 DEBUG:boto:Token: None 2022-09-03T07:30:56.031 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.032 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eqYW7LZUecIB/ppDXgZXgcjln/s= 2022-09-03T07:30:56.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eqYW7LZUecIB/ppDXgZXgcjln/s=', 'Content-Length': '0'} 2022-09-03T07:30:56.034 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006273271fe2501ad4-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.036 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:56.037 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:56.039 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.041 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.042 DEBUG:boto:Method: GET 2022-09-03T07:30:56.042 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.043 DEBUG:boto:Data: 2022-09-03T07:30:56.043 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.044 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.045 DEBUG:boto:Params: {} 2022-09-03T07:30:56.046 DEBUG:boto:Token: None 2022-09-03T07:30:56.046 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:30:56.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eqYW7LZUecIB/ppDXgZXgcjln/s= 2022-09-03T07:30:56.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eqYW7LZUecIB/ppDXgZXgcjln/s=', 'Content-Length': '0'} 2022-09-03T07:30:56.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006643e21fcc572cbe-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.054 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.055 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.056 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:56.057 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:56.057 DEBUG:boto:Method: GET 2022-09-03T07:30:56.058 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:30:56.058 DEBUG:boto:Data: 2022-09-03T07:30:56.059 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.060 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.062 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.062 DEBUG:boto:Params: {} 2022-09-03T07:30:56.063 DEBUG:boto:Token: None 2022-09-03T07:30:56.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.068 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b1870fef6e34ed04-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.110 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.111 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.112 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.112 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:56.113 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:56.113 DEBUG:boto:Method: GET 2022-09-03T07:30:56.114 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:30:56.114 DEBUG:boto:Data: 2022-09-03T07:30:56.115 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.116 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.117 DEBUG:boto:Params: {} 2022-09-03T07:30:56.117 DEBUG:boto:Token: None 2022-09-03T07:30:56.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.126 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ad8e929b203e7104-00631302b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.127 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.128 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:56.129 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:56.129 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.130 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:30:56.130 DEBUG:boto:Data: 2022-09-03T07:30:56.131 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.131 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.132 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.132 DEBUG:boto:Params: {} 2022-09-03T07:30:56.133 DEBUG:boto:Token: None 2022-09-03T07:30:56.134 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj19 2022-09-03T07:30:56.135 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/uwcypHK0zkeGyNg4SJ0Om9m1Jo= 2022-09-03T07:30:56.136 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/uwcypHK0zkeGyNg4SJ0Om9m1Jo=', 'Content-Length': '0'} 2022-09-03T07:30:56.138 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0da406c4e11ee0d-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.139 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:56.139 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:56.140 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.140 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:30:56.141 DEBUG:boto:Data: 2022-09-03T07:30:56.141 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.142 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.142 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.143 DEBUG:boto:Params: {} 2022-09-03T07:30:56.143 DEBUG:boto:Token: None 2022-09-03T07:30:56.144 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj19 2022-09-03T07:30:56.144 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/uwcypHK0zkeGyNg4SJ0Om9m1Jo= 2022-09-03T07:30:56.145 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/uwcypHK0zkeGyNg4SJ0Om9m1Jo=', 'Content-Length': '0'} 2022-09-03T07:30:56.147 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002182a0f1881ebffa-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.148 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:30:56.149 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:56.150 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:56.150 DEBUG:boto:Method: GET 2022-09-03T07:30:56.151 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:30:56.151 DEBUG:boto:Data: 2022-09-03T07:30:56.152 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.153 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.153 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.154 DEBUG:boto:Params: {} 2022-09-03T07:30:56.154 DEBUG:boto:Token: None 2022-09-03T07:30:56.155 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj19 2022-09-03T07:30:56.156 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GBqdRGaeU6rbXpHoPtccl3KVxiI= 2022-09-03T07:30:56.156 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GBqdRGaeU6rbXpHoPtccl3KVxiI=', 'Content-Length': '0'} 2022-09-03T07:30:56.163 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb9baec04570874e-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.164 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:30:56.165 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:30:56.165 DEBUG:boto:Method: GET 2022-09-03T07:30:56.167 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:30:56.167 DEBUG:boto:Data: 2022-09-03T07:30:56.168 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.169 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.169 DEBUG:boto:Params: {} 2022-09-03T07:30:56.170 DEBUG:boto:Token: None 2022-09-03T07:30:56.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj19 2022-09-03T07:30:56.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GBqdRGaeU6rbXpHoPtccl3KVxiI= 2022-09-03T07:30:56.173 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GBqdRGaeU6rbXpHoPtccl3KVxiI=', 'Content-Length': '0'} 2022-09-03T07:30:56.175 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005964c129e5f9b66e-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.176 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.176 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.177 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:56.177 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:56.179 DEBUG:boto:Method: GET 2022-09-03T07:30:56.181 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:30:56.182 DEBUG:boto:Data: 2022-09-03T07:30:56.183 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.184 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.186 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.187 DEBUG:boto:Params: {} 2022-09-03T07:30:56.188 DEBUG:boto:Token: None 2022-09-03T07:30:56.189 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.191 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.196 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037a71abc98fcec5e-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.197 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.204 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.205 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.206 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:56.206 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:30:56.207 DEBUG:boto:Method: GET 2022-09-03T07:30:56.207 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:30:56.208 DEBUG:boto:Data: 2022-09-03T07:30:56.208 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.209 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.209 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.210 DEBUG:boto:Params: {} 2022-09-03T07:30:56.210 DEBUG:boto:Token: None 2022-09-03T07:30:56.211 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.219 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.220 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.227 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005968ce1ff9c7578e-00631302b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.228 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.229 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:30:56.230 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.231 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.231 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.232 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.232 DEBUG:boto:Method: GET 2022-09-03T07:30:56.233 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.234 DEBUG:boto:Data: 2022-09-03T07:30:56.236 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.238 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.239 DEBUG:boto:Params: {} 2022-09-03T07:30:56.241 DEBUG:boto:Token: None 2022-09-03T07:30:56.242 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.243 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4D+C255qkWfqHsp2ejMh1DxluYI= 2022-09-03T07:30:56.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4D+C255qkWfqHsp2ejMh1DxluYI=', 'Content-Length': '0'} 2022-09-03T07:30:56.248 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d570b04e5cce8d9d-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.250 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.251 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.252 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.253 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.254 DEBUG:boto:Method: GET 2022-09-03T07:30:56.256 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.257 DEBUG:boto:Data: 2022-09-03T07:30:56.258 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.261 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.263 DEBUG:boto:Params: {} 2022-09-03T07:30:56.264 DEBUG:boto:Token: None 2022-09-03T07:30:56.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4D+C255qkWfqHsp2ejMh1DxluYI= 2022-09-03T07:30:56.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4D+C255qkWfqHsp2ejMh1DxluYI=', 'Content-Length': '0'} 2022-09-03T07:30:56.272 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037c24ed88c845092-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.273 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.274 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.275 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.275 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.276 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.277 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.278 DEBUG:boto:Data: 2022-09-03T07:30:56.279 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.279 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.280 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.280 DEBUG:boto:Params: {} 2022-09-03T07:30:56.281 DEBUG:boto:Token: None 2022-09-03T07:30:56.282 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W8waIqwevS8idSmUkRgadz+5UTE= 2022-09-03T07:30:56.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W8waIqwevS8idSmUkRgadz+5UTE=', 'Content-Length': '0'} 2022-09-03T07:30:56.288 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3ff5ee59563943f-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.291 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.292 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.293 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.294 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.296 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.299 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.299 DEBUG:boto:Data: 2022-09-03T07:30:56.300 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.300 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.301 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.301 DEBUG:boto:Params: {} 2022-09-03T07:30:56.303 DEBUG:boto:Token: None 2022-09-03T07:30:56.303 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.304 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W8waIqwevS8idSmUkRgadz+5UTE= 2022-09-03T07:30:56.304 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W8waIqwevS8idSmUkRgadz+5UTE=', 'Content-Length': '0'} 2022-09-03T07:30:56.308 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b5b1a1b66b2f257-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.310 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:30:56.311 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.312 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.312 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.313 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.313 DEBUG:boto:Method: GET 2022-09-03T07:30:56.314 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.314 DEBUG:boto:Data: 2022-09-03T07:30:56.315 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.315 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.316 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.317 DEBUG:boto:Params: {} 2022-09-03T07:30:56.319 DEBUG:boto:Token: None 2022-09-03T07:30:56.319 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.320 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4D+C255qkWfqHsp2ejMh1DxluYI= 2022-09-03T07:30:56.321 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4D+C255qkWfqHsp2ejMh1DxluYI=', 'Content-Length': '0'} 2022-09-03T07:30:56.325 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4a13ed219c454c2-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.326 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.326 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.327 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.327 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.328 DEBUG:boto:Method: GET 2022-09-03T07:30:56.328 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.329 DEBUG:boto:Data: 2022-09-03T07:30:56.329 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.330 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.331 DEBUG:boto:Params: {} 2022-09-03T07:30:56.332 DEBUG:boto:Token: None 2022-09-03T07:30:56.332 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:30:56.333 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4D+C255qkWfqHsp2ejMh1DxluYI= 2022-09-03T07:30:56.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4D+C255qkWfqHsp2ejMh1DxluYI=', 'Content-Length': '0'} 2022-09-03T07:30:56.336 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c97b73c8e9f2b242-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.337 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.337 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.338 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:56.338 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:56.339 DEBUG:boto:Method: GET 2022-09-03T07:30:56.339 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:30:56.340 DEBUG:boto:Data: 2022-09-03T07:30:56.341 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.341 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.342 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.342 DEBUG:boto:Params: {} 2022-09-03T07:30:56.343 DEBUG:boto:Token: None 2022-09-03T07:30:56.343 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.347 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d8aa97c4ecfd1cad-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.348 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.349 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.349 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.350 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:56.350 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:56.351 DEBUG:boto:Method: GET 2022-09-03T07:30:56.351 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:30:56.352 DEBUG:boto:Data: 2022-09-03T07:30:56.352 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.353 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.353 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.354 DEBUG:boto:Params: {} 2022-09-03T07:30:56.354 DEBUG:boto:Token: None 2022-09-03T07:30:56.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.361 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000080c5174213e316f-00631302b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.361 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.362 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.363 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.363 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.364 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:30:56.364 DEBUG:boto:Data: 2022-09-03T07:30:56.365 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.365 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.366 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.366 DEBUG:boto:Params: {} 2022-09-03T07:30:56.367 DEBUG:boto:Token: None 2022-09-03T07:30:56.367 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22 2022-09-03T07:30:56.368 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DW1CXOHVkF2UL9XzWRERYDnQIVM= 2022-09-03T07:30:56.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DW1CXOHVkF2UL9XzWRERYDnQIVM=', 'Content-Length': '0'} 2022-09-03T07:30:56.371 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a219d5dc13ea190e-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.372 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.373 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.374 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.377 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:30:56.377 DEBUG:boto:Data: 2022-09-03T07:30:56.378 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.378 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.379 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.379 DEBUG:boto:Params: {} 2022-09-03T07:30:56.380 DEBUG:boto:Token: None 2022-09-03T07:30:56.381 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22 2022-09-03T07:30:56.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DW1CXOHVkF2UL9XzWRERYDnQIVM= 2022-09-03T07:30:56.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DW1CXOHVkF2UL9XzWRERYDnQIVM=', 'Content-Length': '0'} 2022-09-03T07:30:56.384 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006c9c7109f312828a-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.384 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:30:56.385 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.386 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.386 DEBUG:boto:Method: GET 2022-09-03T07:30:56.387 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:30:56.387 DEBUG:boto:Data: 2022-09-03T07:30:56.388 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.389 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.389 DEBUG:boto:Params: {} 2022-09-03T07:30:56.390 DEBUG:boto:Token: None 2022-09-03T07:30:56.391 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22 2022-09-03T07:30:56.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OcaxXgLXtZlTssPwXDP2r+XbC98= 2022-09-03T07:30:56.392 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OcaxXgLXtZlTssPwXDP2r+XbC98=', 'Content-Length': '0'} 2022-09-03T07:30:56.394 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000234dd00e05f5f58e-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.438 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:30:56.439 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:30:56.439 DEBUG:boto:Method: GET 2022-09-03T07:30:56.440 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:30:56.440 DEBUG:boto:Data: 2022-09-03T07:30:56.441 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.442 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.443 DEBUG:boto:Params: {} 2022-09-03T07:30:56.443 DEBUG:boto:Token: None 2022-09-03T07:30:56.444 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj22 2022-09-03T07:30:56.445 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OcaxXgLXtZlTssPwXDP2r+XbC98= 2022-09-03T07:30:56.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OcaxXgLXtZlTssPwXDP2r+XbC98=', 'Content-Length': '0'} 2022-09-03T07:30:56.448 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000372f82f543c0fc5-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.449 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.449 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.450 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:56.450 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:56.451 DEBUG:boto:Method: GET 2022-09-03T07:30:56.451 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:30:56.452 DEBUG:boto:Data: 2022-09-03T07:30:56.452 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.453 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.454 DEBUG:boto:Params: {} 2022-09-03T07:30:56.454 DEBUG:boto:Token: None 2022-09-03T07:30:56.455 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.456 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.459 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000303d9686f5d54ccd-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.460 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.460 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.461 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.462 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:56.462 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:30:56.463 DEBUG:boto:Method: GET 2022-09-03T07:30:56.463 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:30:56.464 DEBUG:boto:Data: 2022-09-03T07:30:56.464 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.465 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.465 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.466 DEBUG:boto:Params: {} 2022-09-03T07:30:56.466 DEBUG:boto:Token: None 2022-09-03T07:30:56.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.472 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006b438dc59314c1db-00631302b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.473 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.474 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:30:56.475 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.475 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.476 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.476 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.477 DEBUG:boto:Method: GET 2022-09-03T07:30:56.478 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.478 DEBUG:boto:Data: 2022-09-03T07:30:56.479 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.479 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.480 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.480 DEBUG:boto:Params: {} 2022-09-03T07:30:56.481 DEBUG:boto:Token: None 2022-09-03T07:30:56.482 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.482 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Z4XOS9xOsbT0fc8V2maLk8ufD0= 2022-09-03T07:30:56.483 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Z4XOS9xOsbT0fc8V2maLk8ufD0=', 'Content-Length': '0'} 2022-09-03T07:30:56.485 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da98e75cee9ae898-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.486 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.487 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.487 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.488 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.488 DEBUG:boto:Method: GET 2022-09-03T07:30:56.489 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.489 DEBUG:boto:Data: 2022-09-03T07:30:56.490 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.491 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.491 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.492 DEBUG:boto:Params: {} 2022-09-03T07:30:56.493 DEBUG:boto:Token: None 2022-09-03T07:30:56.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.494 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Z4XOS9xOsbT0fc8V2maLk8ufD0= 2022-09-03T07:30:56.495 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Z4XOS9xOsbT0fc8V2maLk8ufD0=', 'Content-Length': '0'} 2022-09-03T07:30:56.497 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000246c1dccda80eb44-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.498 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.498 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.499 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.499 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.500 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.500 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.501 DEBUG:boto:Data: 2022-09-03T07:30:56.501 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.502 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.503 DEBUG:boto:Params: {} 2022-09-03T07:30:56.504 DEBUG:boto:Token: None 2022-09-03T07:30:56.504 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FeQYPZ62qUZDHL6vFTsYkRQOiao= 2022-09-03T07:30:56.506 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FeQYPZ62qUZDHL6vFTsYkRQOiao=', 'Content-Length': '0'} 2022-09-03T07:30:56.508 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015f688ba813b4d96-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.509 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.510 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.510 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.511 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.511 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.512 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.513 DEBUG:boto:Data: 2022-09-03T07:30:56.513 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.514 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.514 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.515 DEBUG:boto:Params: {} 2022-09-03T07:30:56.515 DEBUG:boto:Token: None 2022-09-03T07:30:56.516 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.517 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FeQYPZ62qUZDHL6vFTsYkRQOiao= 2022-09-03T07:30:56.517 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FeQYPZ62qUZDHL6vFTsYkRQOiao=', 'Content-Length': '0'} 2022-09-03T07:30:56.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008289b069b1154c6f-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:30:56.521 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.522 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.522 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.523 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.524 DEBUG:boto:Method: GET 2022-09-03T07:30:56.524 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.525 DEBUG:boto:Data: 2022-09-03T07:30:56.526 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.526 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.527 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.528 DEBUG:boto:Params: {} 2022-09-03T07:30:56.528 DEBUG:boto:Token: None 2022-09-03T07:30:56.529 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.530 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Z4XOS9xOsbT0fc8V2maLk8ufD0= 2022-09-03T07:30:56.531 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Z4XOS9xOsbT0fc8V2maLk8ufD0=', 'Content-Length': '0'} 2022-09-03T07:30:56.533 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000488db63cfd3935a6-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.534 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.534 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.535 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.535 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.536 DEBUG:boto:Method: GET 2022-09-03T07:30:56.536 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.537 DEBUG:boto:Data: 2022-09-03T07:30:56.538 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.539 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.540 DEBUG:boto:Params: {} 2022-09-03T07:30:56.540 DEBUG:boto:Token: None 2022-09-03T07:30:56.541 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:30:56.542 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9Z4XOS9xOsbT0fc8V2maLk8ufD0= 2022-09-03T07:30:56.543 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9Z4XOS9xOsbT0fc8V2maLk8ufD0=', 'Content-Length': '0'} 2022-09-03T07:30:56.545 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005144ad8922a89e8e-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.546 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.547 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.547 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:56.548 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:56.548 DEBUG:boto:Method: GET 2022-09-03T07:30:56.549 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:30:56.549 DEBUG:boto:Data: 2022-09-03T07:30:56.550 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.550 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.551 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.552 DEBUG:boto:Params: {} 2022-09-03T07:30:56.553 DEBUG:boto:Token: None 2022-09-03T07:30:56.553 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.557 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007fa14f1bf01dfd40-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.558 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.559 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.560 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.560 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:56.561 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:56.561 DEBUG:boto:Method: GET 2022-09-03T07:30:56.562 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:30:56.563 DEBUG:boto:Data: 2022-09-03T07:30:56.563 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.564 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.564 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.565 DEBUG:boto:Params: {} 2022-09-03T07:30:56.565 DEBUG:boto:Token: None 2022-09-03T07:30:56.566 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.567 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.572 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b8fa4bc18d28252-00631302b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.572 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.573 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.574 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.574 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.575 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:30:56.576 DEBUG:boto:Data: 2022-09-03T07:30:56.576 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.577 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.578 DEBUG:boto:Params: {} 2022-09-03T07:30:56.578 DEBUG:boto:Token: None 2022-09-03T07:30:56.579 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23 2022-09-03T07:30:56.579 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XdFEsk7ncR6wrEUcXLGrrAnpOIU= 2022-09-03T07:30:56.580 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XdFEsk7ncR6wrEUcXLGrrAnpOIU=', 'Content-Length': '0'} 2022-09-03T07:30:56.582 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e01e73505df5186-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.583 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.584 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.585 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.586 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:30:56.586 DEBUG:boto:Data: 2022-09-03T07:30:56.587 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.588 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.588 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.589 DEBUG:boto:Params: {} 2022-09-03T07:30:56.589 DEBUG:boto:Token: None 2022-09-03T07:30:56.590 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23 2022-09-03T07:30:56.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XdFEsk7ncR6wrEUcXLGrrAnpOIU= 2022-09-03T07:30:56.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XdFEsk7ncR6wrEUcXLGrrAnpOIU=', 'Content-Length': '0'} 2022-09-03T07:30:56.593 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001df3685d36c463e6-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.594 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:30:56.595 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.595 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.596 DEBUG:boto:Method: GET 2022-09-03T07:30:56.597 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:30:56.597 DEBUG:boto:Data: 2022-09-03T07:30:56.598 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.598 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.599 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.600 DEBUG:boto:Params: {} 2022-09-03T07:30:56.600 DEBUG:boto:Token: None 2022-09-03T07:30:56.601 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23 2022-09-03T07:30:56.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j4t3yRfnIQlc7nzFxitL+Z3BQfc= 2022-09-03T07:30:56.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j4t3yRfnIQlc7nzFxitL+Z3BQfc=', 'Content-Length': '0'} 2022-09-03T07:30:56.607 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001965d7fe74f88a1-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.650 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:30:56.651 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:30:56.652 DEBUG:boto:Method: GET 2022-09-03T07:30:56.653 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:30:56.654 DEBUG:boto:Data: 2022-09-03T07:30:56.654 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.656 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.657 DEBUG:boto:Params: {} 2022-09-03T07:30:56.659 DEBUG:boto:Token: None 2022-09-03T07:30:56.661 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj23 2022-09-03T07:30:56.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:j4t3yRfnIQlc7nzFxitL+Z3BQfc= 2022-09-03T07:30:56.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:j4t3yRfnIQlc7nzFxitL+Z3BQfc=', 'Content-Length': '0'} 2022-09-03T07:30:56.667 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3e188d3c8398384-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.670 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.671 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.671 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:56.672 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:56.672 DEBUG:boto:Method: GET 2022-09-03T07:30:56.673 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:30:56.674 DEBUG:boto:Data: 2022-09-03T07:30:56.675 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.679 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.680 DEBUG:boto:Params: {} 2022-09-03T07:30:56.681 DEBUG:boto:Token: None 2022-09-03T07:30:56.682 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.688 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005cd933e1fc570b29-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.689 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.691 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.692 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.693 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:56.694 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:30:56.695 DEBUG:boto:Method: GET 2022-09-03T07:30:56.696 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:30:56.697 DEBUG:boto:Data: 2022-09-03T07:30:56.698 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.699 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.700 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.701 DEBUG:boto:Params: {} 2022-09-03T07:30:56.702 DEBUG:boto:Token: None 2022-09-03T07:30:56.703 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.704 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.705 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.710 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bf0d2f8c79277545-00631302b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.711 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.711 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:30:56.712 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.713 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.713 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.714 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.715 DEBUG:boto:Method: GET 2022-09-03T07:30:56.715 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.716 DEBUG:boto:Data: 2022-09-03T07:30:56.716 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.717 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.718 DEBUG:boto:Params: {} 2022-09-03T07:30:56.719 DEBUG:boto:Token: None 2022-09-03T07:30:56.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lU0xTpJF4tzXbuTrlbPEElk9UDA= 2022-09-03T07:30:56.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lU0xTpJF4tzXbuTrlbPEElk9UDA=', 'Content-Length': '0'} 2022-09-03T07:30:56.723 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000750a3b3b914651fe-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.724 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.725 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.725 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.726 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.727 DEBUG:boto:Method: GET 2022-09-03T07:30:56.727 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.728 DEBUG:boto:Data: 2022-09-03T07:30:56.728 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.729 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.730 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.730 DEBUG:boto:Params: {} 2022-09-03T07:30:56.731 DEBUG:boto:Token: None 2022-09-03T07:30:56.732 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lU0xTpJF4tzXbuTrlbPEElk9UDA= 2022-09-03T07:30:56.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lU0xTpJF4tzXbuTrlbPEElk9UDA=', 'Content-Length': '0'} 2022-09-03T07:30:56.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026edaab35f0d9bf7-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.782 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.783 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.783 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.784 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.784 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.785 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.785 DEBUG:boto:Data: 2022-09-03T07:30:56.786 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.787 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.788 DEBUG:boto:Params: {} 2022-09-03T07:30:56.788 DEBUG:boto:Token: None 2022-09-03T07:30:56.789 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pRrNkNw16VhilFWkovoISPps0fE= 2022-09-03T07:30:56.790 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pRrNkNw16VhilFWkovoISPps0fE=', 'Content-Length': '0'} 2022-09-03T07:30:56.792 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1bd923962a37b38-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.793 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.793 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.794 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.795 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.796 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.796 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.797 DEBUG:boto:Data: 2022-09-03T07:30:56.797 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.798 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.798 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.799 DEBUG:boto:Params: {} 2022-09-03T07:30:56.799 DEBUG:boto:Token: None 2022-09-03T07:30:56.800 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pRrNkNw16VhilFWkovoISPps0fE= 2022-09-03T07:30:56.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pRrNkNw16VhilFWkovoISPps0fE=', 'Content-Length': '0'} 2022-09-03T07:30:56.803 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024aac6f1fd2c00dc-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.804 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:30:56.805 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.805 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.806 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.806 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.807 DEBUG:boto:Method: GET 2022-09-03T07:30:56.807 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.808 DEBUG:boto:Data: 2022-09-03T07:30:56.809 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.810 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.810 DEBUG:boto:Params: {} 2022-09-03T07:30:56.811 DEBUG:boto:Token: None 2022-09-03T07:30:56.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lU0xTpJF4tzXbuTrlbPEElk9UDA= 2022-09-03T07:30:56.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lU0xTpJF4tzXbuTrlbPEElk9UDA=', 'Content-Length': '0'} 2022-09-03T07:30:56.815 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ba6b15dcaf2bd2f-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.815 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.816 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.817 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.817 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.818 DEBUG:boto:Method: GET 2022-09-03T07:30:56.818 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.819 DEBUG:boto:Data: 2022-09-03T07:30:56.819 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.820 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.821 DEBUG:boto:Params: {} 2022-09-03T07:30:56.821 DEBUG:boto:Token: None 2022-09-03T07:30:56.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:30:56.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lU0xTpJF4tzXbuTrlbPEElk9UDA= 2022-09-03T07:30:56.823 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lU0xTpJF4tzXbuTrlbPEElk9UDA=', 'Content-Length': '0'} 2022-09-03T07:30:56.825 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a60fab6c0558133-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.826 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.827 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.827 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:56.828 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:56.828 DEBUG:boto:Method: GET 2022-09-03T07:30:56.829 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:30:56.829 DEBUG:boto:Data: 2022-09-03T07:30:56.830 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.831 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.831 DEBUG:boto:Params: {} 2022-09-03T07:30:56.832 DEBUG:boto:Token: None 2022-09-03T07:30:56.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.836 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c2fe705e110fedac-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.878 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.879 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.880 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.880 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:56.881 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:56.881 DEBUG:boto:Method: GET 2022-09-03T07:30:56.882 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:30:56.882 DEBUG:boto:Data: 2022-09-03T07:30:56.883 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.883 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.884 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.884 DEBUG:boto:Params: {} 2022-09-03T07:30:56.885 DEBUG:boto:Token: None 2022-09-03T07:30:56.886 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.891 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a45cc7e2fd1e29b-00631302b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.892 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.893 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.893 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.894 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.894 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:30:56.895 DEBUG:boto:Data: 2022-09-03T07:30:56.895 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.896 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.896 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.897 DEBUG:boto:Params: {} 2022-09-03T07:30:56.897 DEBUG:boto:Token: None 2022-09-03T07:30:56.898 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29 2022-09-03T07:30:56.899 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:d8ZOnM3PJ5Y8OHwvM2BMo7q5GQw= 2022-09-03T07:30:56.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:d8ZOnM3PJ5Y8OHwvM2BMo7q5GQw=', 'Content-Length': '0'} 2022-09-03T07:30:56.901 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e39cbae47ef4af28-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.902 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.902 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.903 DEBUG:boto:Method: HEAD 2022-09-03T07:30:56.903 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:30:56.904 DEBUG:boto:Data: 2022-09-03T07:30:56.904 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.905 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.906 DEBUG:boto:Params: {} 2022-09-03T07:30:56.906 DEBUG:boto:Token: None 2022-09-03T07:30:56.907 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29 2022-09-03T07:30:56.907 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:d8ZOnM3PJ5Y8OHwvM2BMo7q5GQw= 2022-09-03T07:30:56.908 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:d8ZOnM3PJ5Y8OHwvM2BMo7q5GQw=', 'Content-Length': '0'} 2022-09-03T07:30:56.910 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000da92d555d1a01d8c-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.911 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:30:56.912 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.912 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.913 DEBUG:boto:Method: GET 2022-09-03T07:30:56.913 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:30:56.914 DEBUG:boto:Data: 2022-09-03T07:30:56.914 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.915 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.916 DEBUG:boto:Params: {} 2022-09-03T07:30:56.917 DEBUG:boto:Token: None 2022-09-03T07:30:56.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29 2022-09-03T07:30:56.918 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eBoo7WwT7fZea58iFCbkKPWiyiw= 2022-09-03T07:30:56.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eBoo7WwT7fZea58iFCbkKPWiyiw=', 'Content-Length': '0'} 2022-09-03T07:30:56.920 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060bed18a1c63db5e-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.921 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:30:56.922 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:30:56.922 DEBUG:boto:Method: GET 2022-09-03T07:30:56.923 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:30:56.923 DEBUG:boto:Data: 2022-09-03T07:30:56.924 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.925 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.925 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.926 DEBUG:boto:Params: {} 2022-09-03T07:30:56.926 DEBUG:boto:Token: None 2022-09-03T07:30:56.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/obj29 2022-09-03T07:30:56.927 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eBoo7WwT7fZea58iFCbkKPWiyiw= 2022-09-03T07:30:56.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eBoo7WwT7fZea58iFCbkKPWiyiw=', 'Content-Length': '0'} 2022-09-03T07:30:56.930 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f504eaa85783dbab-00631302b0-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.974 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.975 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.975 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:56.976 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:56.976 DEBUG:boto:Method: GET 2022-09-03T07:30:56.977 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:30:56.978 DEBUG:boto:Data: 2022-09-03T07:30:56.978 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.979 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:56.979 DEBUG:boto:Port: 8001 2022-09-03T07:30:56.980 DEBUG:boto:Params: {} 2022-09-03T07:30:56.980 DEBUG:boto:Token: None 2022-09-03T07:30:56.981 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:56.985 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f24e9351401d14bd-00631302b0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:56.986 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:56.987 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:56.988 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:56.988 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:56.989 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:30:56.989 DEBUG:boto:Method: GET 2022-09-03T07:30:56.990 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:30:56.991 DEBUG:boto:Data: 2022-09-03T07:30:56.991 DEBUG:boto:Headers: {} 2022-09-03T07:30:56.992 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:56.993 DEBUG:boto:Port: 8000 2022-09-03T07:30:56.993 DEBUG:boto:Params: {} 2022-09-03T07:30:56.994 DEBUG:boto:Token: None 2022-09-03T07:30:56.995 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:56 GMT /avysmw-75/ 2022-09-03T07:30:56.995 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos= 2022-09-03T07:30:56.996 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8OWCKTeFpNQ0RdaGpMDg4MpyEos=', 'Content-Length': '0'} 2022-09-03T07:30:57.000 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a058eecbdbe9dafc-00631302b0-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.001 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.002 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:30:57.002 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.003 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.005 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.008 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.009 DEBUG:boto:Method: GET 2022-09-03T07:30:57.010 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.010 DEBUG:boto:Data: 2022-09-03T07:30:57.011 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.012 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.012 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.013 DEBUG:boto:Params: {} 2022-09-03T07:30:57.014 DEBUG:boto:Token: None 2022-09-03T07:30:57.014 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QYEeXqxaI5vY6J79cleNuHuqEbo= 2022-09-03T07:30:57.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QYEeXqxaI5vY6J79cleNuHuqEbo=', 'Content-Length': '0'} 2022-09-03T07:30:57.018 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e4541bde0a966483-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.019 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.019 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.020 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.021 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.021 DEBUG:boto:Method: GET 2022-09-03T07:30:57.022 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.022 DEBUG:boto:Data: 2022-09-03T07:30:57.023 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.023 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.024 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.024 DEBUG:boto:Params: {} 2022-09-03T07:30:57.025 DEBUG:boto:Token: None 2022-09-03T07:30:57.026 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QYEeXqxaI5vY6J79cleNuHuqEbo= 2022-09-03T07:30:57.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QYEeXqxaI5vY6J79cleNuHuqEbo=', 'Content-Length': '0'} 2022-09-03T07:30:57.030 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005257afb22c423a82-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.031 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.032 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.032 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.033 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.034 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.034 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.035 DEBUG:boto:Data: 2022-09-03T07:30:57.035 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.036 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.036 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.037 DEBUG:boto:Params: {} 2022-09-03T07:30:57.038 DEBUG:boto:Token: None 2022-09-03T07:30:57.038 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3ClTyoa/lFaWNKXX04ejpbG02kY= 2022-09-03T07:30:57.040 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3ClTyoa/lFaWNKXX04ejpbG02kY=', 'Content-Length': '0'} 2022-09-03T07:30:57.042 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0dd0fc6defc7931-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.043 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.043 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.044 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.045 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.047 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.047 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.048 DEBUG:boto:Data: 2022-09-03T07:30:57.048 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.049 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.049 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.050 DEBUG:boto:Params: {} 2022-09-03T07:30:57.051 DEBUG:boto:Token: None 2022-09-03T07:30:57.052 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.052 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3ClTyoa/lFaWNKXX04ejpbG02kY= 2022-09-03T07:30:57.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3ClTyoa/lFaWNKXX04ejpbG02kY=', 'Content-Length': '0'} 2022-09-03T07:30:57.055 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e1a92a74cbb820a-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.055 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:30:57.056 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.057 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.057 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.058 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.058 DEBUG:boto:Method: GET 2022-09-03T07:30:57.059 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.059 DEBUG:boto:Data: 2022-09-03T07:30:57.060 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.061 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.061 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.062 DEBUG:boto:Params: {} 2022-09-03T07:30:57.063 DEBUG:boto:Token: None 2022-09-03T07:30:57.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QYEeXqxaI5vY6J79cleNuHuqEbo= 2022-09-03T07:30:57.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QYEeXqxaI5vY6J79cleNuHuqEbo=', 'Content-Length': '0'} 2022-09-03T07:30:57.067 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037f730db2b752ea7-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.115 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.115 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.116 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.116 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.117 DEBUG:boto:Method: GET 2022-09-03T07:30:57.118 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.118 DEBUG:boto:Data: 2022-09-03T07:30:57.119 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.120 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.121 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.123 DEBUG:boto:Params: {} 2022-09-03T07:30:57.124 DEBUG:boto:Token: None 2022-09-03T07:30:57.125 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:30:57.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QYEeXqxaI5vY6J79cleNuHuqEbo= 2022-09-03T07:30:57.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QYEeXqxaI5vY6J79cleNuHuqEbo=', 'Content-Length': '0'} 2022-09-03T07:30:57.130 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000127ab21ca09c6657-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.131 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.132 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.132 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:57.133 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:57.134 DEBUG:boto:Method: GET 2022-09-03T07:30:57.134 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:30:57.135 DEBUG:boto:Data: 2022-09-03T07:30:57.135 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.136 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.136 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.137 DEBUG:boto:Params: {} 2022-09-03T07:30:57.137 DEBUG:boto:Token: None 2022-09-03T07:30:57.138 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.142 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fda84eb7f2f2bf82-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.143 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.144 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.144 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.145 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:57.145 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:57.146 DEBUG:boto:Method: GET 2022-09-03T07:30:57.146 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:30:57.147 DEBUG:boto:Data: 2022-09-03T07:30:57.148 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.149 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.149 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.150 DEBUG:boto:Params: {} 2022-09-03T07:30:57.151 DEBUG:boto:Token: None 2022-09-03T07:30:57.151 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.152 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.153 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.158 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000099a4a0bb27d8d1d7-00631302b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.198 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.199 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.200 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.200 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.201 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:30:57.202 DEBUG:boto:Data: 2022-09-03T07:30:57.202 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.203 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.203 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.204 DEBUG:boto:Params: {} 2022-09-03T07:30:57.204 DEBUG:boto:Token: None 2022-09-03T07:30:57.205 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31 2022-09-03T07:30:57.206 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dYgV5LQsXKF5u2Gs63VhQQ3C3yc= 2022-09-03T07:30:57.207 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dYgV5LQsXKF5u2Gs63VhQQ3C3yc=', 'Content-Length': '0'} 2022-09-03T07:30:57.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006675ec72f616e395-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.210 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.211 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.211 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.212 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:30:57.213 DEBUG:boto:Data: 2022-09-03T07:30:57.213 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.214 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.214 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.215 DEBUG:boto:Params: {} 2022-09-03T07:30:57.216 DEBUG:boto:Token: None 2022-09-03T07:30:57.216 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31 2022-09-03T07:30:57.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dYgV5LQsXKF5u2Gs63VhQQ3C3yc= 2022-09-03T07:30:57.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dYgV5LQsXKF5u2Gs63VhQQ3C3yc=', 'Content-Length': '0'} 2022-09-03T07:30:57.220 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b75187ce3612b66b-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.221 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:30:57.221 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.222 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.223 DEBUG:boto:Method: GET 2022-09-03T07:30:57.223 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:30:57.224 DEBUG:boto:Data: 2022-09-03T07:30:57.224 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.225 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.226 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.227 DEBUG:boto:Params: {} 2022-09-03T07:30:57.228 DEBUG:boto:Token: None 2022-09-03T07:30:57.229 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31 2022-09-03T07:30:57.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:My97StUVxaAbE3qkrMaYpoUJFUg= 2022-09-03T07:30:57.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:My97StUVxaAbE3qkrMaYpoUJFUg=', 'Content-Length': '0'} 2022-09-03T07:30:57.234 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000081f4ac44c4e32a24-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.235 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:30:57.236 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:30:57.237 DEBUG:boto:Method: GET 2022-09-03T07:30:57.238 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:30:57.240 DEBUG:boto:Data: 2022-09-03T07:30:57.241 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.242 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.243 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.244 DEBUG:boto:Params: {} 2022-09-03T07:30:57.245 DEBUG:boto:Token: None 2022-09-03T07:30:57.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj31 2022-09-03T07:30:57.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:My97StUVxaAbE3qkrMaYpoUJFUg= 2022-09-03T07:30:57.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:My97StUVxaAbE3qkrMaYpoUJFUg=', 'Content-Length': '0'} 2022-09-03T07:30:57.252 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000082a905f942ec768a-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.253 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.253 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.254 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:57.254 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:57.255 DEBUG:boto:Method: GET 2022-09-03T07:30:57.255 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:30:57.256 DEBUG:boto:Data: 2022-09-03T07:30:57.256 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.257 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.258 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.258 DEBUG:boto:Params: {} 2022-09-03T07:30:57.259 DEBUG:boto:Token: None 2022-09-03T07:30:57.259 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.260 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.261 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.265 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e5131fe480dce2a8-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.266 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.267 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.268 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.269 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:57.270 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:30:57.271 DEBUG:boto:Method: GET 2022-09-03T07:30:57.271 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:30:57.272 DEBUG:boto:Data: 2022-09-03T07:30:57.272 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.273 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.274 DEBUG:boto:Params: {} 2022-09-03T07:30:57.275 DEBUG:boto:Token: None 2022-09-03T07:30:57.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.281 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eefec41ff404d888-00631302b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.282 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.282 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:30:57.283 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.284 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.284 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.285 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.285 DEBUG:boto:Method: GET 2022-09-03T07:30:57.286 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.286 DEBUG:boto:Data: 2022-09-03T07:30:57.287 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.287 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.288 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.288 DEBUG:boto:Params: {} 2022-09-03T07:30:57.289 DEBUG:boto:Token: None 2022-09-03T07:30:57.289 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.290 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D2ixlOuyzKqgW8hrxQMXU7/PWWk= 2022-09-03T07:30:57.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D2ixlOuyzKqgW8hrxQMXU7/PWWk=', 'Content-Length': '0'} 2022-09-03T07:30:57.293 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e276b51234fc47a1-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.293 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.294 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.295 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.295 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.296 DEBUG:boto:Method: GET 2022-09-03T07:30:57.296 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.297 DEBUG:boto:Data: 2022-09-03T07:30:57.297 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.298 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.299 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.300 DEBUG:boto:Params: {} 2022-09-03T07:30:57.300 DEBUG:boto:Token: None 2022-09-03T07:30:57.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.301 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D2ixlOuyzKqgW8hrxQMXU7/PWWk= 2022-09-03T07:30:57.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D2ixlOuyzKqgW8hrxQMXU7/PWWk=', 'Content-Length': '0'} 2022-09-03T07:30:57.304 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000079ea6b05b288c708-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.306 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.307 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.307 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.308 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.308 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.309 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.311 DEBUG:boto:Data: 2022-09-03T07:30:57.312 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.313 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.314 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.314 DEBUG:boto:Params: {} 2022-09-03T07:30:57.315 DEBUG:boto:Token: None 2022-09-03T07:30:57.317 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Te5BodhuFAobC/j3dIo5hAkDy+I= 2022-09-03T07:30:57.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Te5BodhuFAobC/j3dIo5hAkDy+I=', 'Content-Length': '0'} 2022-09-03T07:30:57.321 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000886cea2b2b5208e8-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.323 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.324 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.325 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.326 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.326 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.327 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.328 DEBUG:boto:Data: 2022-09-03T07:30:57.329 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.331 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.332 DEBUG:boto:Params: {} 2022-09-03T07:30:57.332 DEBUG:boto:Token: None 2022-09-03T07:30:57.333 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Te5BodhuFAobC/j3dIo5hAkDy+I= 2022-09-03T07:30:57.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Te5BodhuFAobC/j3dIo5hAkDy+I=', 'Content-Length': '0'} 2022-09-03T07:30:57.336 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ee4463f5def8040-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.337 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:30:57.338 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.338 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.339 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.339 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.340 DEBUG:boto:Method: GET 2022-09-03T07:30:57.340 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.341 DEBUG:boto:Data: 2022-09-03T07:30:57.341 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.342 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.343 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.343 DEBUG:boto:Params: {} 2022-09-03T07:30:57.344 DEBUG:boto:Token: None 2022-09-03T07:30:57.344 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.345 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D2ixlOuyzKqgW8hrxQMXU7/PWWk= 2022-09-03T07:30:57.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D2ixlOuyzKqgW8hrxQMXU7/PWWk=', 'Content-Length': '0'} 2022-09-03T07:30:57.348 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff78c63ba65dd987-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.349 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.349 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.350 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.351 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.351 DEBUG:boto:Method: GET 2022-09-03T07:30:57.352 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.352 DEBUG:boto:Data: 2022-09-03T07:30:57.353 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.354 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.354 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.355 DEBUG:boto:Params: {} 2022-09-03T07:30:57.355 DEBUG:boto:Token: None 2022-09-03T07:30:57.356 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:30:57.357 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:D2ixlOuyzKqgW8hrxQMXU7/PWWk= 2022-09-03T07:30:57.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:D2ixlOuyzKqgW8hrxQMXU7/PWWk=', 'Content-Length': '0'} 2022-09-03T07:30:57.359 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000051c4d061d987bccc-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.360 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.361 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.361 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:57.362 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:57.362 DEBUG:boto:Method: GET 2022-09-03T07:30:57.363 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:30:57.363 DEBUG:boto:Data: 2022-09-03T07:30:57.364 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.365 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.365 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.366 DEBUG:boto:Params: {} 2022-09-03T07:30:57.366 DEBUG:boto:Token: None 2022-09-03T07:30:57.367 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.367 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.368 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.371 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3a44f1de1fdac11-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.372 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.373 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.373 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.374 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:57.374 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:57.375 DEBUG:boto:Method: GET 2022-09-03T07:30:57.375 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:30:57.376 DEBUG:boto:Data: 2022-09-03T07:30:57.376 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.377 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.378 DEBUG:boto:Params: {} 2022-09-03T07:30:57.378 DEBUG:boto:Token: None 2022-09-03T07:30:57.379 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.385 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a437f74a9bbb50ab-00631302b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.386 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.387 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.388 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.388 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.389 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:30:57.389 DEBUG:boto:Data: 2022-09-03T07:30:57.390 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.390 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.391 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.391 DEBUG:boto:Params: {} 2022-09-03T07:30:57.392 DEBUG:boto:Token: None 2022-09-03T07:30:57.393 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33 2022-09-03T07:30:57.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eIaztlo9e3ShC+oEWVuWaCKL9bQ= 2022-09-03T07:30:57.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eIaztlo9e3ShC+oEWVuWaCKL9bQ=', 'Content-Length': '0'} 2022-09-03T07:30:57.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d545f55272c58c99-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.398 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.398 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.399 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.399 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:30:57.400 DEBUG:boto:Data: 2022-09-03T07:30:57.400 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.402 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.402 DEBUG:boto:Params: {} 2022-09-03T07:30:57.403 DEBUG:boto:Token: None 2022-09-03T07:30:57.404 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33 2022-09-03T07:30:57.404 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eIaztlo9e3ShC+oEWVuWaCKL9bQ= 2022-09-03T07:30:57.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eIaztlo9e3ShC+oEWVuWaCKL9bQ=', 'Content-Length': '0'} 2022-09-03T07:30:57.407 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066fcb405952c8bf8-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.408 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:30:57.409 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.410 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.412 DEBUG:boto:Method: GET 2022-09-03T07:30:57.413 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:30:57.414 DEBUG:boto:Data: 2022-09-03T07:30:57.415 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.417 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.418 DEBUG:boto:Params: {} 2022-09-03T07:30:57.419 DEBUG:boto:Token: None 2022-09-03T07:30:57.420 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33 2022-09-03T07:30:57.421 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G4HATl4zIbl0hPJHULgFMhhCQ34= 2022-09-03T07:30:57.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G4HATl4zIbl0hPJHULgFMhhCQ34=', 'Content-Length': '0'} 2022-09-03T07:30:57.425 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086aed507658ec05c-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.426 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:30:57.427 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:30:57.428 DEBUG:boto:Method: GET 2022-09-03T07:30:57.429 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:30:57.430 DEBUG:boto:Data: 2022-09-03T07:30:57.431 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.433 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.434 DEBUG:boto:Params: {} 2022-09-03T07:30:57.435 DEBUG:boto:Token: None 2022-09-03T07:30:57.436 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj33 2022-09-03T07:30:57.437 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G4HATl4zIbl0hPJHULgFMhhCQ34= 2022-09-03T07:30:57.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G4HATl4zIbl0hPJHULgFMhhCQ34=', 'Content-Length': '0'} 2022-09-03T07:30:57.440 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089a8b57d1fa5c5e0-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.482 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.483 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.483 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:57.484 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:57.484 DEBUG:boto:Method: GET 2022-09-03T07:30:57.485 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:30:57.486 DEBUG:boto:Data: 2022-09-03T07:30:57.486 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.487 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.488 DEBUG:boto:Params: {} 2022-09-03T07:30:57.489 DEBUG:boto:Token: None 2022-09-03T07:30:57.490 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.490 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.493 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f26eba64bd7a96a-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.494 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.495 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.495 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.496 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:57.496 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:30:57.497 DEBUG:boto:Method: GET 2022-09-03T07:30:57.498 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:30:57.498 DEBUG:boto:Data: 2022-09-03T07:30:57.499 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.499 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.500 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.500 DEBUG:boto:Params: {} 2022-09-03T07:30:57.501 DEBUG:boto:Token: None 2022-09-03T07:30:57.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.502 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.503 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.507 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000040bfec40ed609527-00631302b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.508 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.509 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:30:57.510 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.510 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.511 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.512 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.513 DEBUG:boto:Method: GET 2022-09-03T07:30:57.513 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.514 DEBUG:boto:Data: 2022-09-03T07:30:57.515 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.516 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.517 DEBUG:boto:Params: {} 2022-09-03T07:30:57.519 DEBUG:boto:Token: None 2022-09-03T07:30:57.520 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.521 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OTn2ojwzwn5dV7V2hfOtWnWNOFI= 2022-09-03T07:30:57.522 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OTn2ojwzwn5dV7V2hfOtWnWNOFI=', 'Content-Length': '0'} 2022-09-03T07:30:57.525 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003531d79fe1c3853-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.526 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.527 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.529 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.530 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.531 DEBUG:boto:Method: GET 2022-09-03T07:30:57.532 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.533 DEBUG:boto:Data: 2022-09-03T07:30:57.534 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.536 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.536 DEBUG:boto:Params: {} 2022-09-03T07:30:57.538 DEBUG:boto:Token: None 2022-09-03T07:30:57.539 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OTn2ojwzwn5dV7V2hfOtWnWNOFI= 2022-09-03T07:30:57.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OTn2ojwzwn5dV7V2hfOtWnWNOFI=', 'Content-Length': '0'} 2022-09-03T07:30:57.544 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000087dd36e9f90df48b-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.545 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.545 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.546 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.546 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.547 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.547 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.548 DEBUG:boto:Data: 2022-09-03T07:30:57.548 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.549 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.549 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.550 DEBUG:boto:Params: {} 2022-09-03T07:30:57.551 DEBUG:boto:Token: None 2022-09-03T07:30:57.551 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XDq1wziOWybXVshAd/zisQ0yC2Q= 2022-09-03T07:30:57.552 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XDq1wziOWybXVshAd/zisQ0yC2Q=', 'Content-Length': '0'} 2022-09-03T07:30:57.555 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aba3c4feeb282bf5-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.556 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.556 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.557 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.557 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.558 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.558 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.559 DEBUG:boto:Data: 2022-09-03T07:30:57.559 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.560 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.560 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.561 DEBUG:boto:Params: {} 2022-09-03T07:30:57.562 DEBUG:boto:Token: None 2022-09-03T07:30:57.562 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.563 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XDq1wziOWybXVshAd/zisQ0yC2Q= 2022-09-03T07:30:57.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XDq1wziOWybXVshAd/zisQ0yC2Q=', 'Content-Length': '0'} 2022-09-03T07:30:57.566 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009cbea992dfe66ec-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.567 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:30:57.568 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.569 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.570 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.571 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.572 DEBUG:boto:Method: GET 2022-09-03T07:30:57.573 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.574 DEBUG:boto:Data: 2022-09-03T07:30:57.574 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.575 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.576 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.577 DEBUG:boto:Params: {} 2022-09-03T07:30:57.577 DEBUG:boto:Token: None 2022-09-03T07:30:57.578 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OTn2ojwzwn5dV7V2hfOtWnWNOFI= 2022-09-03T07:30:57.581 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OTn2ojwzwn5dV7V2hfOtWnWNOFI=', 'Content-Length': '0'} 2022-09-03T07:30:57.584 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ab58c7b375dd5155-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.630 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.631 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.632 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.632 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.633 DEBUG:boto:Method: GET 2022-09-03T07:30:57.633 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.634 DEBUG:boto:Data: 2022-09-03T07:30:57.635 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.635 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.636 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.636 DEBUG:boto:Params: {} 2022-09-03T07:30:57.637 DEBUG:boto:Token: None 2022-09-03T07:30:57.637 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:30:57.638 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OTn2ojwzwn5dV7V2hfOtWnWNOFI= 2022-09-03T07:30:57.639 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OTn2ojwzwn5dV7V2hfOtWnWNOFI=', 'Content-Length': '0'} 2022-09-03T07:30:57.641 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7a22b399102bb43-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.642 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.642 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.643 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:57.643 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:57.644 DEBUG:boto:Method: GET 2022-09-03T07:30:57.644 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:30:57.645 DEBUG:boto:Data: 2022-09-03T07:30:57.645 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.646 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.647 DEBUG:boto:Params: {} 2022-09-03T07:30:57.648 DEBUG:boto:Token: None 2022-09-03T07:30:57.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.652 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d19c828cae0f099b-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.653 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.654 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.654 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.655 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:57.655 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:57.656 DEBUG:boto:Method: GET 2022-09-03T07:30:57.657 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:30:57.657 DEBUG:boto:Data: 2022-09-03T07:30:57.658 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.658 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.659 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.659 DEBUG:boto:Params: {} 2022-09-03T07:30:57.660 DEBUG:boto:Token: None 2022-09-03T07:30:57.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.666 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fefdedd1a61d0c65-00631302b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.667 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.667 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.668 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.668 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.669 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:30:57.669 DEBUG:boto:Data: 2022-09-03T07:30:57.670 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.670 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.671 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.671 DEBUG:boto:Params: {} 2022-09-03T07:30:57.672 DEBUG:boto:Token: None 2022-09-03T07:30:57.673 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34 2022-09-03T07:30:57.673 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zXnCiXXZ9juVFejMQu7OYfL4clI= 2022-09-03T07:30:57.674 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zXnCiXXZ9juVFejMQu7OYfL4clI=', 'Content-Length': '0'} 2022-09-03T07:30:57.676 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000acc77c5e0650cb96-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.677 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.677 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.678 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.679 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:30:57.679 DEBUG:boto:Data: 2022-09-03T07:30:57.679 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.680 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.680 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.681 DEBUG:boto:Params: {} 2022-09-03T07:30:57.682 DEBUG:boto:Token: None 2022-09-03T07:30:57.683 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34 2022-09-03T07:30:57.684 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zXnCiXXZ9juVFejMQu7OYfL4clI= 2022-09-03T07:30:57.685 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zXnCiXXZ9juVFejMQu7OYfL4clI=', 'Content-Length': '0'} 2022-09-03T07:30:57.686 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006c8778d231fe1a44-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.687 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:30:57.688 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.688 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.689 DEBUG:boto:Method: GET 2022-09-03T07:30:57.689 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:30:57.690 DEBUG:boto:Data: 2022-09-03T07:30:57.691 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.692 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.692 DEBUG:boto:Params: {} 2022-09-03T07:30:57.693 DEBUG:boto:Token: None 2022-09-03T07:30:57.693 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34 2022-09-03T07:30:57.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RDfSeiVAtwiE0Z07XutvMkRJHVM= 2022-09-03T07:30:57.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RDfSeiVAtwiE0Z07XutvMkRJHVM=', 'Content-Length': '0'} 2022-09-03T07:30:57.698 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad2dec3d0f2c3f75-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.699 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:30:57.700 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:30:57.701 DEBUG:boto:Method: GET 2022-09-03T07:30:57.702 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:30:57.703 DEBUG:boto:Data: 2022-09-03T07:30:57.704 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.705 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.705 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.707 DEBUG:boto:Params: {} 2022-09-03T07:30:57.708 DEBUG:boto:Token: None 2022-09-03T07:30:57.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj34 2022-09-03T07:30:57.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RDfSeiVAtwiE0Z07XutvMkRJHVM= 2022-09-03T07:30:57.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RDfSeiVAtwiE0Z07XutvMkRJHVM=', 'Content-Length': '0'} 2022-09-03T07:30:57.712 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d23810fb74ff52b-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.754 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.755 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.755 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:57.756 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:57.756 DEBUG:boto:Method: GET 2022-09-03T07:30:57.757 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:30:57.757 DEBUG:boto:Data: 2022-09-03T07:30:57.758 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.759 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.759 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.760 DEBUG:boto:Params: {} 2022-09-03T07:30:57.761 DEBUG:boto:Token: None 2022-09-03T07:30:57.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.765 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007fdfa84869c589dd-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.766 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.766 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.767 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.768 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:57.768 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:30:57.769 DEBUG:boto:Method: GET 2022-09-03T07:30:57.769 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:30:57.770 DEBUG:boto:Data: 2022-09-03T07:30:57.771 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.771 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.772 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.773 DEBUG:boto:Params: {} 2022-09-03T07:30:57.774 DEBUG:boto:Token: None 2022-09-03T07:30:57.775 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.777 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.782 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e23391dac68eb2d1-00631302b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.782 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.783 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2022-09-03T07:30:57.784 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.784 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.785 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.785 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.786 DEBUG:boto:Method: GET 2022-09-03T07:30:57.787 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.787 DEBUG:boto:Data: 2022-09-03T07:30:57.788 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.788 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.789 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.789 DEBUG:boto:Params: {} 2022-09-03T07:30:57.790 DEBUG:boto:Token: None 2022-09-03T07:30:57.791 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.792 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ftqrs1/Oqm08TT4VmqrzOGJk+B0= 2022-09-03T07:30:57.792 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ftqrs1/Oqm08TT4VmqrzOGJk+B0=', 'Content-Length': '0'} 2022-09-03T07:30:57.795 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000336f8ac77242ec39-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.796 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.796 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.797 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.797 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.798 DEBUG:boto:Method: GET 2022-09-03T07:30:57.799 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.800 DEBUG:boto:Data: 2022-09-03T07:30:57.800 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.801 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.802 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.802 DEBUG:boto:Params: {} 2022-09-03T07:30:57.803 DEBUG:boto:Token: None 2022-09-03T07:30:57.803 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.804 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ftqrs1/Oqm08TT4VmqrzOGJk+B0= 2022-09-03T07:30:57.805 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ftqrs1/Oqm08TT4VmqrzOGJk+B0=', 'Content-Length': '0'} 2022-09-03T07:30:57.808 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004347d9f6b74f705c-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.809 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.810 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.811 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.812 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.813 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.814 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.814 DEBUG:boto:Data: 2022-09-03T07:30:57.815 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.815 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.816 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.816 DEBUG:boto:Params: {} 2022-09-03T07:30:57.817 DEBUG:boto:Token: None 2022-09-03T07:30:57.817 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.818 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B2y0RScgGtl00bxUvUWswdPB//k= 2022-09-03T07:30:57.819 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B2y0RScgGtl00bxUvUWswdPB//k=', 'Content-Length': '0'} 2022-09-03T07:30:57.820 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000241674616c0bef59-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.821 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.822 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.822 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.823 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.823 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.824 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.825 DEBUG:boto:Data: 2022-09-03T07:30:57.825 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.826 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.827 DEBUG:boto:Params: {} 2022-09-03T07:30:57.827 DEBUG:boto:Token: None 2022-09-03T07:30:57.828 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:B2y0RScgGtl00bxUvUWswdPB//k= 2022-09-03T07:30:57.829 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:B2y0RScgGtl00bxUvUWswdPB//k=', 'Content-Length': '0'} 2022-09-03T07:30:57.831 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000005172c64546c135-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.832 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2022-09-03T07:30:57.833 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.834 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.835 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.836 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.837 DEBUG:boto:Method: GET 2022-09-03T07:30:57.837 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.838 DEBUG:boto:Data: 2022-09-03T07:30:57.839 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.840 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.841 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.842 DEBUG:boto:Params: {} 2022-09-03T07:30:57.842 DEBUG:boto:Token: None 2022-09-03T07:30:57.843 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.844 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ftqrs1/Oqm08TT4VmqrzOGJk+B0= 2022-09-03T07:30:57.845 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ftqrs1/Oqm08TT4VmqrzOGJk+B0=', 'Content-Length': '0'} 2022-09-03T07:30:57.847 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099aa1400f368b473-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.848 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.849 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.849 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.850 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.850 DEBUG:boto:Method: GET 2022-09-03T07:30:57.851 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.851 DEBUG:boto:Data: 2022-09-03T07:30:57.852 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.853 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.853 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.854 DEBUG:boto:Params: {} 2022-09-03T07:30:57.855 DEBUG:boto:Token: None 2022-09-03T07:30:57.855 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:30:57.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ftqrs1/Oqm08TT4VmqrzOGJk+B0= 2022-09-03T07:30:57.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ftqrs1/Oqm08TT4VmqrzOGJk+B0=', 'Content-Length': '0'} 2022-09-03T07:30:57.859 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b02ba01f3f98a67-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.859 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.860 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.860 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:57.861 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:57.862 DEBUG:boto:Method: GET 2022-09-03T07:30:57.862 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:30:57.863 DEBUG:boto:Data: 2022-09-03T07:30:57.863 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.864 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.865 DEBUG:boto:Params: {} 2022-09-03T07:30:57.865 DEBUG:boto:Token: None 2022-09-03T07:30:57.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.870 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036413e1891a33787-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.871 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.873 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.874 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.875 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:57.876 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:57.877 DEBUG:boto:Method: GET 2022-09-03T07:30:57.878 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:30:57.879 DEBUG:boto:Data: 2022-09-03T07:30:57.880 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.881 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.882 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.883 DEBUG:boto:Params: {} 2022-09-03T07:30:57.884 DEBUG:boto:Token: None 2022-09-03T07:30:57.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/ 2022-09-03T07:30:57.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU= 2022-09-03T07:30:57.887 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:77lW+W+P8GuEIhgBNIR39IQPJbU=', 'Content-Length': '0'} 2022-09-03T07:30:57.892 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bd21fbd90fb5e890-00631302b1-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.893 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:57.894 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.895 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.896 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.897 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:30:57.898 DEBUG:boto:Data: 2022-09-03T07:30:57.899 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.900 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.901 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.902 DEBUG:boto:Params: {} 2022-09-03T07:30:57.903 DEBUG:boto:Token: None 2022-09-03T07:30:57.904 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37 2022-09-03T07:30:57.905 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09sB5JtEWZdFdT/rZ9StHYkXq90= 2022-09-03T07:30:57.906 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09sB5JtEWZdFdT/rZ9StHYkXq90=', 'Content-Length': '0'} 2022-09-03T07:30:57.908 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd719249e0b2f6e6-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.910 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.910 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.911 DEBUG:boto:Method: HEAD 2022-09-03T07:30:57.912 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:30:57.913 DEBUG:boto:Data: 2022-09-03T07:30:57.914 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.915 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.916 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.917 DEBUG:boto:Params: {} 2022-09-03T07:30:57.917 DEBUG:boto:Token: None 2022-09-03T07:30:57.918 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37 2022-09-03T07:30:57.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:09sB5JtEWZdFdT/rZ9StHYkXq90= 2022-09-03T07:30:57.919 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:09sB5JtEWZdFdT/rZ9StHYkXq90=', 'Content-Length': '0'} 2022-09-03T07:30:57.921 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022dca997fe9274fe-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.922 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2022-09-03T07:30:57.922 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.923 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.924 DEBUG:boto:Method: GET 2022-09-03T07:30:57.924 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:30:57.925 DEBUG:boto:Data: 2022-09-03T07:30:57.925 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.926 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:57.927 DEBUG:boto:Port: 8001 2022-09-03T07:30:57.927 DEBUG:boto:Params: {} 2022-09-03T07:30:57.928 DEBUG:boto:Token: None 2022-09-03T07:30:57.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37 2022-09-03T07:30:57.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RjMEP+DvnHr7w5pTuRYn4mliFbw= 2022-09-03T07:30:57.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RjMEP+DvnHr7w5pTuRYn4mliFbw=', 'Content-Length': '0'} 2022-09-03T07:30:57.933 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000939cb411db73a628-00631302b1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.978 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:30:57.980 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:30:57.981 DEBUG:boto:Method: GET 2022-09-03T07:30:57.982 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:30:57.983 DEBUG:boto:Data: 2022-09-03T07:30:57.984 DEBUG:boto:Headers: {} 2022-09-03T07:30:57.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:57.986 DEBUG:boto:Port: 8000 2022-09-03T07:30:57.987 DEBUG:boto:Params: {} 2022-09-03T07:30:57.988 DEBUG:boto:Token: None 2022-09-03T07:30:57.989 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:57 GMT /avysmw-75/obj37 2022-09-03T07:30:57.990 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RjMEP+DvnHr7w5pTuRYn4mliFbw= 2022-09-03T07:30:57.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RjMEP+DvnHr7w5pTuRYn4mliFbw=', 'Content-Length': '0'} 2022-09-03T07:30:57.994 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004dbd4c86d6e28a5c-00631302b1-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:57.996 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:57.997 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:57.998 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:57.998 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:57.999 DEBUG:boto:Method: GET 2022-09-03T07:30:58.000 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:30:58.001 DEBUG:boto:Data: 2022-09-03T07:30:58.002 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.005 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.006 DEBUG:boto:Params: {} 2022-09-03T07:30:58.008 DEBUG:boto:Token: None 2022-09-03T07:30:58.009 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.011 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.024 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dff4dc55898b2912-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.026 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.027 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.028 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.028 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:58.029 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:30:58.030 DEBUG:boto:Method: GET 2022-09-03T07:30:58.031 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:30:58.032 DEBUG:boto:Data: 2022-09-03T07:30:58.032 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.033 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.034 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.035 DEBUG:boto:Params: {} 2022-09-03T07:30:58.036 DEBUG:boto:Token: None 2022-09-03T07:30:58.037 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.038 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.045 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008f625abc46df0e9b-00631302b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.046 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.047 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:30:58.048 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.048 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.049 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.049 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.050 DEBUG:boto:Method: GET 2022-09-03T07:30:58.051 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.051 DEBUG:boto:Data: 2022-09-03T07:30:58.052 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.052 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.053 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.054 DEBUG:boto:Params: {} 2022-09-03T07:30:58.054 DEBUG:boto:Token: None 2022-09-03T07:30:58.055 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OssSL97NNBGnGoeII5ARR8v8pk4= 2022-09-03T07:30:58.056 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OssSL97NNBGnGoeII5ARR8v8pk4=', 'Content-Length': '0'} 2022-09-03T07:30:58.058 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009ca451cddd5e7bc3-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.059 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.060 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.060 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.061 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.062 DEBUG:boto:Method: GET 2022-09-03T07:30:58.062 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.063 DEBUG:boto:Data: 2022-09-03T07:30:58.063 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.064 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.064 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.065 DEBUG:boto:Params: {} 2022-09-03T07:30:58.066 DEBUG:boto:Token: None 2022-09-03T07:30:58.066 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.067 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OssSL97NNBGnGoeII5ARR8v8pk4= 2022-09-03T07:30:58.068 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OssSL97NNBGnGoeII5ARR8v8pk4=', 'Content-Length': '0'} 2022-09-03T07:30:58.070 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fb64474f7b7e1f7a-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.071 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.072 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.072 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.073 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.073 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.074 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.074 DEBUG:boto:Data: 2022-09-03T07:30:58.075 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.075 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.076 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.076 DEBUG:boto:Params: {} 2022-09-03T07:30:58.077 DEBUG:boto:Token: None 2022-09-03T07:30:58.078 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.078 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:laVUeUZDY/X2ky82SNfA3T9SNd8= 2022-09-03T07:30:58.079 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:laVUeUZDY/X2ky82SNfA3T9SNd8=', 'Content-Length': '0'} 2022-09-03T07:30:58.082 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000223b4325387dfe1a-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.083 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.084 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.085 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.086 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.088 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.091 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.092 DEBUG:boto:Data: 2022-09-03T07:30:58.093 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.094 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.095 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.096 DEBUG:boto:Params: {} 2022-09-03T07:30:58.097 DEBUG:boto:Token: None 2022-09-03T07:30:58.098 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.099 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:laVUeUZDY/X2ky82SNfA3T9SNd8= 2022-09-03T07:30:58.100 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:laVUeUZDY/X2ky82SNfA3T9SNd8=', 'Content-Length': '0'} 2022-09-03T07:30:58.103 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fab356343d1c895-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.104 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:30:58.105 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.106 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.107 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.109 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.110 DEBUG:boto:Method: GET 2022-09-03T07:30:58.111 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.112 DEBUG:boto:Data: 2022-09-03T07:30:58.113 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.115 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.116 DEBUG:boto:Params: {} 2022-09-03T07:30:58.117 DEBUG:boto:Token: None 2022-09-03T07:30:58.118 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OssSL97NNBGnGoeII5ARR8v8pk4= 2022-09-03T07:30:58.119 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OssSL97NNBGnGoeII5ARR8v8pk4=', 'Content-Length': '0'} 2022-09-03T07:30:58.121 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b8dbff60036ca35-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.122 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.123 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.123 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.125 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.125 DEBUG:boto:Method: GET 2022-09-03T07:30:58.126 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.127 DEBUG:boto:Data: 2022-09-03T07:30:58.127 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.129 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.129 DEBUG:boto:Params: {} 2022-09-03T07:30:58.130 DEBUG:boto:Token: None 2022-09-03T07:30:58.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:30:58.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:OssSL97NNBGnGoeII5ARR8v8pk4= 2022-09-03T07:30:58.133 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:OssSL97NNBGnGoeII5ARR8v8pk4=', 'Content-Length': '0'} 2022-09-03T07:30:58.135 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c3020b39ea5903a2-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.178 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.190 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.191 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:58.192 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:58.193 DEBUG:boto:Method: GET 2022-09-03T07:30:58.194 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:30:58.195 DEBUG:boto:Data: 2022-09-03T07:30:58.195 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.196 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.197 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.197 DEBUG:boto:Params: {} 2022-09-03T07:30:58.198 DEBUG:boto:Token: None 2022-09-03T07:30:58.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.203 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b620c6c76b34542c-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.204 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.205 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.210 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.211 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:58.212 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:58.213 DEBUG:boto:Method: GET 2022-09-03T07:30:58.214 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:30:58.215 DEBUG:boto:Data: 2022-09-03T07:30:58.216 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.217 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.218 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.220 DEBUG:boto:Params: {} 2022-09-03T07:30:58.221 DEBUG:boto:Token: None 2022-09-03T07:30:58.223 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.231 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000090b5fc7e74d2180b-00631302b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.232 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.234 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.235 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.235 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.236 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:30:58.237 DEBUG:boto:Data: 2022-09-03T07:30:58.238 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.239 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.240 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.241 DEBUG:boto:Params: {} 2022-09-03T07:30:58.243 DEBUG:boto:Token: None 2022-09-03T07:30:58.244 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38 2022-09-03T07:30:58.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUSSmdTmYB41JQvlWrdPIUp9qxY= 2022-09-03T07:30:58.247 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUSSmdTmYB41JQvlWrdPIUp9qxY=', 'Content-Length': '0'} 2022-09-03T07:30:58.250 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f8f77afe9b7d9a93-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.251 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.252 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.253 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.253 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:30:58.254 DEBUG:boto:Data: 2022-09-03T07:30:58.254 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.255 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.256 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.256 DEBUG:boto:Params: {} 2022-09-03T07:30:58.257 DEBUG:boto:Token: None 2022-09-03T07:30:58.258 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38 2022-09-03T07:30:58.258 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vUSSmdTmYB41JQvlWrdPIUp9qxY= 2022-09-03T07:30:58.259 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vUSSmdTmYB41JQvlWrdPIUp9qxY=', 'Content-Length': '0'} 2022-09-03T07:30:58.261 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f5086f7825974e0-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.262 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:30:58.263 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.264 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.264 DEBUG:boto:Method: GET 2022-09-03T07:30:58.265 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:30:58.266 DEBUG:boto:Data: 2022-09-03T07:30:58.266 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.267 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.268 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.269 DEBUG:boto:Params: {} 2022-09-03T07:30:58.269 DEBUG:boto:Token: None 2022-09-03T07:30:58.270 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38 2022-09-03T07:30:58.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDowoxsk2CseN2epjs0ghmETGBE= 2022-09-03T07:30:58.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDowoxsk2CseN2epjs0ghmETGBE=', 'Content-Length': '0'} 2022-09-03T07:30:58.274 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003aafcf076dc36e29-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.275 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:30:58.275 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:30:58.276 DEBUG:boto:Method: GET 2022-09-03T07:30:58.277 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:30:58.277 DEBUG:boto:Data: 2022-09-03T07:30:58.278 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.279 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.280 DEBUG:boto:Params: {} 2022-09-03T07:30:58.281 DEBUG:boto:Token: None 2022-09-03T07:30:58.282 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj38 2022-09-03T07:30:58.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDowoxsk2CseN2epjs0ghmETGBE= 2022-09-03T07:30:58.284 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDowoxsk2CseN2epjs0ghmETGBE=', 'Content-Length': '0'} 2022-09-03T07:30:58.286 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056e6e357c5151f2c-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.288 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.289 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.290 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:58.291 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:58.292 DEBUG:boto:Method: GET 2022-09-03T07:30:58.293 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:30:58.302 DEBUG:boto:Data: 2022-09-03T07:30:58.303 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.304 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.306 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.306 DEBUG:boto:Params: {} 2022-09-03T07:30:58.308 DEBUG:boto:Token: None 2022-09-03T07:30:58.308 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.314 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e8b504a365b6c298-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.316 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.317 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.317 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.319 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:58.320 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:30:58.321 DEBUG:boto:Method: GET 2022-09-03T07:30:58.323 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:30:58.325 DEBUG:boto:Data: 2022-09-03T07:30:58.327 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.331 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.336 DEBUG:boto:Params: {} 2022-09-03T07:30:58.337 DEBUG:boto:Token: None 2022-09-03T07:30:58.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.338 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.339 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.344 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000084bfdb758cc4e42c-00631302b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.346 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.347 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:30:58.348 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.349 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.350 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.350 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.352 DEBUG:boto:Method: GET 2022-09-03T07:30:58.353 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.355 DEBUG:boto:Data: 2022-09-03T07:30:58.356 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.357 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.359 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.360 DEBUG:boto:Params: {} 2022-09-03T07:30:58.364 DEBUG:boto:Token: None 2022-09-03T07:30:58.366 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.368 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bU35pSdRbcqLCjQZe+zqtCTpi9o= 2022-09-03T07:30:58.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bU35pSdRbcqLCjQZe+zqtCTpi9o=', 'Content-Length': '0'} 2022-09-03T07:30:58.373 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007c03f391decbb13-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.379 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.380 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.382 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.383 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.384 DEBUG:boto:Method: GET 2022-09-03T07:30:58.385 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.388 DEBUG:boto:Data: 2022-09-03T07:30:58.389 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.391 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.392 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.393 DEBUG:boto:Params: {} 2022-09-03T07:30:58.394 DEBUG:boto:Token: None 2022-09-03T07:30:58.395 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.396 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bU35pSdRbcqLCjQZe+zqtCTpi9o= 2022-09-03T07:30:58.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bU35pSdRbcqLCjQZe+zqtCTpi9o=', 'Content-Length': '0'} 2022-09-03T07:30:58.400 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000362f22a4485aebed-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.401 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.401 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.402 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.403 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.404 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.404 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.405 DEBUG:boto:Data: 2022-09-03T07:30:58.406 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.408 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.409 DEBUG:boto:Params: {} 2022-09-03T07:30:58.409 DEBUG:boto:Token: None 2022-09-03T07:30:58.410 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.411 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3S6KxLCMOxW+OqULkWPTGeZ1Blg= 2022-09-03T07:30:58.412 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3S6KxLCMOxW+OqULkWPTGeZ1Blg=', 'Content-Length': '0'} 2022-09-03T07:30:58.413 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac426f1f4a13b9d7-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.414 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.415 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.416 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.416 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.417 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.418 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.418 DEBUG:boto:Data: 2022-09-03T07:30:58.419 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.420 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.420 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.421 DEBUG:boto:Params: {} 2022-09-03T07:30:58.422 DEBUG:boto:Token: None 2022-09-03T07:30:58.423 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.423 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3S6KxLCMOxW+OqULkWPTGeZ1Blg= 2022-09-03T07:30:58.424 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3S6KxLCMOxW+OqULkWPTGeZ1Blg=', 'Content-Length': '0'} 2022-09-03T07:30:58.426 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6d267a983e07c36-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.427 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:30:58.428 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.428 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.429 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.430 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.430 DEBUG:boto:Method: GET 2022-09-03T07:30:58.431 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.432 DEBUG:boto:Data: 2022-09-03T07:30:58.432 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.433 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.434 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.435 DEBUG:boto:Params: {} 2022-09-03T07:30:58.436 DEBUG:boto:Token: None 2022-09-03T07:30:58.437 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bU35pSdRbcqLCjQZe+zqtCTpi9o= 2022-09-03T07:30:58.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bU35pSdRbcqLCjQZe+zqtCTpi9o=', 'Content-Length': '0'} 2022-09-03T07:30:58.445 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008c59caaa84ad36e-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.490 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.491 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.493 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.494 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.495 DEBUG:boto:Method: GET 2022-09-03T07:30:58.496 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.497 DEBUG:boto:Data: 2022-09-03T07:30:58.499 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.501 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.502 DEBUG:boto:Params: {} 2022-09-03T07:30:58.502 DEBUG:boto:Token: None 2022-09-03T07:30:58.503 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:30:58.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bU35pSdRbcqLCjQZe+zqtCTpi9o= 2022-09-03T07:30:58.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bU35pSdRbcqLCjQZe+zqtCTpi9o=', 'Content-Length': '0'} 2022-09-03T07:30:58.509 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009497d74aa711befe-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.511 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.512 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.513 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:58.514 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:58.516 DEBUG:boto:Method: GET 2022-09-03T07:30:58.517 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:30:58.518 DEBUG:boto:Data: 2022-09-03T07:30:58.519 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.520 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.522 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.523 DEBUG:boto:Params: {} 2022-09-03T07:30:58.524 DEBUG:boto:Token: None 2022-09-03T07:30:58.525 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.525 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.526 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.529 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a262f3e193699770-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.531 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.532 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.533 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.535 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:58.536 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:58.537 DEBUG:boto:Method: GET 2022-09-03T07:30:58.538 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:30:58.539 DEBUG:boto:Data: 2022-09-03T07:30:58.540 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.542 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.543 DEBUG:boto:Params: {} 2022-09-03T07:30:58.543 DEBUG:boto:Token: None 2022-09-03T07:30:58.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.546 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.551 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed1662177cae269e-00631302b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.552 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.552 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.553 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.554 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.554 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:30:58.555 DEBUG:boto:Data: 2022-09-03T07:30:58.555 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.556 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.556 DEBUG:boto:Params: {} 2022-09-03T07:30:58.557 DEBUG:boto:Token: None 2022-09-03T07:30:58.558 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44 2022-09-03T07:30:58.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2dTRQMqbkDRGlHceCdTxItr6UmU= 2022-09-03T07:30:58.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2dTRQMqbkDRGlHceCdTxItr6UmU=', 'Content-Length': '0'} 2022-09-03T07:30:58.561 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007dbfbada760b9be3-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.562 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.563 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.563 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.564 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:30:58.564 DEBUG:boto:Data: 2022-09-03T07:30:58.565 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.566 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.566 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.567 DEBUG:boto:Params: {} 2022-09-03T07:30:58.567 DEBUG:boto:Token: None 2022-09-03T07:30:58.568 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44 2022-09-03T07:30:58.569 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2dTRQMqbkDRGlHceCdTxItr6UmU= 2022-09-03T07:30:58.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2dTRQMqbkDRGlHceCdTxItr6UmU=', 'Content-Length': '0'} 2022-09-03T07:30:58.572 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005184d492092d826a-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.573 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:30:58.573 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.574 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.575 DEBUG:boto:Method: GET 2022-09-03T07:30:58.575 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:30:58.576 DEBUG:boto:Data: 2022-09-03T07:30:58.576 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.577 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.578 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.578 DEBUG:boto:Params: {} 2022-09-03T07:30:58.579 DEBUG:boto:Token: None 2022-09-03T07:30:58.579 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44 2022-09-03T07:30:58.580 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+PhuEiAcLu6MSE9EFfjaxGZl4uI= 2022-09-03T07:30:58.581 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+PhuEiAcLu6MSE9EFfjaxGZl4uI=', 'Content-Length': '0'} 2022-09-03T07:30:58.583 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c30abf916125eb76-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.584 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:30:58.584 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:30:58.585 DEBUG:boto:Method: GET 2022-09-03T07:30:58.585 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:30:58.586 DEBUG:boto:Data: 2022-09-03T07:30:58.587 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.587 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.588 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.588 DEBUG:boto:Params: {} 2022-09-03T07:30:58.589 DEBUG:boto:Token: None 2022-09-03T07:30:58.590 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj44 2022-09-03T07:30:58.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+PhuEiAcLu6MSE9EFfjaxGZl4uI= 2022-09-03T07:30:58.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+PhuEiAcLu6MSE9EFfjaxGZl4uI=', 'Content-Length': '0'} 2022-09-03T07:30:58.593 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002075117d49cbb05-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.595 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.596 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.597 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:58.597 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:58.598 DEBUG:boto:Method: GET 2022-09-03T07:30:58.599 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:30:58.600 DEBUG:boto:Data: 2022-09-03T07:30:58.600 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.601 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.601 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.602 DEBUG:boto:Params: {} 2022-09-03T07:30:58.603 DEBUG:boto:Token: None 2022-09-03T07:30:58.603 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.604 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.604 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.607 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a95e30a2a81ff163-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.608 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.609 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.609 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.610 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:58.611 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:30:58.611 DEBUG:boto:Method: GET 2022-09-03T07:30:58.612 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:30:58.612 DEBUG:boto:Data: 2022-09-03T07:30:58.613 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.613 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.614 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.614 DEBUG:boto:Params: {} 2022-09-03T07:30:58.615 DEBUG:boto:Token: None 2022-09-03T07:30:58.616 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.622 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bef402de100380e0-00631302b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.622 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.623 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:30:58.624 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.624 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.625 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.625 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.626 DEBUG:boto:Method: GET 2022-09-03T07:30:58.627 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.627 DEBUG:boto:Data: 2022-09-03T07:30:58.628 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.629 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.629 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.630 DEBUG:boto:Params: {} 2022-09-03T07:30:58.631 DEBUG:boto:Token: None 2022-09-03T07:30:58.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.632 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7EtwUboTocp7MBpdFQMUuk7G6F0= 2022-09-03T07:30:58.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7EtwUboTocp7MBpdFQMUuk7G6F0=', 'Content-Length': '0'} 2022-09-03T07:30:58.635 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d862bd29935a58f0-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.679 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.680 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.681 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.682 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.682 DEBUG:boto:Method: GET 2022-09-03T07:30:58.683 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.683 DEBUG:boto:Data: 2022-09-03T07:30:58.684 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.684 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.685 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.685 DEBUG:boto:Params: {} 2022-09-03T07:30:58.686 DEBUG:boto:Token: None 2022-09-03T07:30:58.687 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.687 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7EtwUboTocp7MBpdFQMUuk7G6F0= 2022-09-03T07:30:58.688 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7EtwUboTocp7MBpdFQMUuk7G6F0=', 'Content-Length': '0'} 2022-09-03T07:30:58.690 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd40df3c06ff9e5e-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.691 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.692 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.692 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.693 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.693 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.694 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.694 DEBUG:boto:Data: 2022-09-03T07:30:58.695 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.695 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.696 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.696 DEBUG:boto:Params: {} 2022-09-03T07:30:58.697 DEBUG:boto:Token: None 2022-09-03T07:30:58.698 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iprQyykihytK3+utmjBoWo+wgpg= 2022-09-03T07:30:58.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iprQyykihytK3+utmjBoWo+wgpg=', 'Content-Length': '0'} 2022-09-03T07:30:58.701 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe9d7f17b193dc3e-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.702 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.703 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.704 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.704 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.705 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.705 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.706 DEBUG:boto:Data: 2022-09-03T07:30:58.706 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.707 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.708 DEBUG:boto:Params: {} 2022-09-03T07:30:58.708 DEBUG:boto:Token: None 2022-09-03T07:30:58.709 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iprQyykihytK3+utmjBoWo+wgpg= 2022-09-03T07:30:58.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iprQyykihytK3+utmjBoWo+wgpg=', 'Content-Length': '0'} 2022-09-03T07:30:58.712 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009419dd458bb6ea0b-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.713 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:30:58.713 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.714 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.715 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.716 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.717 DEBUG:boto:Method: GET 2022-09-03T07:30:58.717 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.718 DEBUG:boto:Data: 2022-09-03T07:30:58.719 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.720 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.721 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.722 DEBUG:boto:Params: {} 2022-09-03T07:30:58.723 DEBUG:boto:Token: None 2022-09-03T07:30:58.724 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7EtwUboTocp7MBpdFQMUuk7G6F0= 2022-09-03T07:30:58.727 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7EtwUboTocp7MBpdFQMUuk7G6F0=', 'Content-Length': '0'} 2022-09-03T07:30:58.729 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0c64aae918cab0c-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.730 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.731 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.731 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.732 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.732 DEBUG:boto:Method: GET 2022-09-03T07:30:58.733 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.733 DEBUG:boto:Data: 2022-09-03T07:30:58.734 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.735 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.736 DEBUG:boto:Params: {} 2022-09-03T07:30:58.737 DEBUG:boto:Token: None 2022-09-03T07:30:58.738 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:30:58.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7EtwUboTocp7MBpdFQMUuk7G6F0= 2022-09-03T07:30:58.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7EtwUboTocp7MBpdFQMUuk7G6F0=', 'Content-Length': '0'} 2022-09-03T07:30:58.743 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023323f8d630c2d27-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.793 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.795 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.796 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:58.797 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:58.798 DEBUG:boto:Method: GET 2022-09-03T07:30:58.799 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:30:58.799 DEBUG:boto:Data: 2022-09-03T07:30:58.800 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.801 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.802 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.803 DEBUG:boto:Params: {} 2022-09-03T07:30:58.804 DEBUG:boto:Token: None 2022-09-03T07:30:58.805 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.806 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.811 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000871e51e692aef644-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.812 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.814 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.815 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.816 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:58.817 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:58.818 DEBUG:boto:Method: GET 2022-09-03T07:30:58.819 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:30:58.820 DEBUG:boto:Data: 2022-09-03T07:30:58.821 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.822 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.823 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.824 DEBUG:boto:Params: {} 2022-09-03T07:30:58.825 DEBUG:boto:Token: None 2022-09-03T07:30:58.827 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.828 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.839 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000040efa72b67d29ca1-00631302b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.841 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.842 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.843 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.844 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.845 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:30:58.846 DEBUG:boto:Data: 2022-09-03T07:30:58.847 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.848 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.849 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.849 DEBUG:boto:Params: {} 2022-09-03T07:30:58.851 DEBUG:boto:Token: None 2022-09-03T07:30:58.851 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46 2022-09-03T07:30:58.853 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r40J8vmvQy8RwmSdLYJ0m7+EPOo= 2022-09-03T07:30:58.854 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r40J8vmvQy8RwmSdLYJ0m7+EPOo=', 'Content-Length': '0'} 2022-09-03T07:30:58.861 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000095a0387bbd3a2cdd-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.863 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.865 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.866 DEBUG:boto:Method: HEAD 2022-09-03T07:30:58.866 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:30:58.867 DEBUG:boto:Data: 2022-09-03T07:30:58.868 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.868 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.869 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.870 DEBUG:boto:Params: {} 2022-09-03T07:30:58.870 DEBUG:boto:Token: None 2022-09-03T07:30:58.871 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46 2022-09-03T07:30:58.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r40J8vmvQy8RwmSdLYJ0m7+EPOo= 2022-09-03T07:30:58.872 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r40J8vmvQy8RwmSdLYJ0m7+EPOo=', 'Content-Length': '0'} 2022-09-03T07:30:58.875 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba5332f66f23ab10-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.877 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:30:58.878 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.879 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.880 DEBUG:boto:Method: GET 2022-09-03T07:30:58.881 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:30:58.882 DEBUG:boto:Data: 2022-09-03T07:30:58.884 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.884 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.886 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.888 DEBUG:boto:Params: {} 2022-09-03T07:30:58.889 DEBUG:boto:Token: None 2022-09-03T07:30:58.890 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46 2022-09-03T07:30:58.891 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GgUDYHEmODEajPn5rulQXjQV4As= 2022-09-03T07:30:58.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GgUDYHEmODEajPn5rulQXjQV4As=', 'Content-Length': '0'} 2022-09-03T07:30:58.895 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ef6472cb99e647b-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.897 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:30:58.898 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:30:58.899 DEBUG:boto:Method: GET 2022-09-03T07:30:58.899 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:30:58.900 DEBUG:boto:Data: 2022-09-03T07:30:58.901 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.902 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.903 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.904 DEBUG:boto:Params: {} 2022-09-03T07:30:58.905 DEBUG:boto:Token: None 2022-09-03T07:30:58.907 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/obj46 2022-09-03T07:30:58.908 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GgUDYHEmODEajPn5rulQXjQV4As= 2022-09-03T07:30:58.909 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GgUDYHEmODEajPn5rulQXjQV4As=', 'Content-Length': '0'} 2022-09-03T07:30:58.917 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000019933a813b7f89d1-00631302b2-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.919 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.920 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.920 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:58.921 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:58.922 DEBUG:boto:Method: GET 2022-09-03T07:30:58.922 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:30:58.923 DEBUG:boto:Data: 2022-09-03T07:30:58.923 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:58.925 DEBUG:boto:Port: 8001 2022-09-03T07:30:58.928 DEBUG:boto:Params: {} 2022-09-03T07:30:58.929 DEBUG:boto:Token: None 2022-09-03T07:30:58.929 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.930 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.931 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.933 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008fdaee2f1d2d45af-00631302b2-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.934 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.935 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:58.935 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:58.936 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:58.936 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:30:58.937 DEBUG:boto:Method: GET 2022-09-03T07:30:58.938 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:30:58.939 DEBUG:boto:Data: 2022-09-03T07:30:58.939 DEBUG:boto:Headers: {} 2022-09-03T07:30:58.940 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:58.940 DEBUG:boto:Port: 8000 2022-09-03T07:30:58.941 DEBUG:boto:Params: {} 2022-09-03T07:30:58.941 DEBUG:boto:Token: None 2022-09-03T07:30:58.942 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:58 GMT /avysmw-75/ 2022-09-03T07:30:58.943 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg= 2022-09-03T07:30:58.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eC4J513pxytkSb3ZU506N+aTpKg=', 'Content-Length': '0'} 2022-09-03T07:30:58.952 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bc49efa7c3050ce3-00631302b2-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:58.990 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:58.992 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:30:58.993 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:58.997 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:58.997 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:58.998 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:58.998 DEBUG:boto:Method: GET 2022-09-03T07:30:58.999 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:58.999 DEBUG:boto:Data: 2022-09-03T07:30:59.000 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.001 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.001 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.002 DEBUG:boto:Params: {} 2022-09-03T07:30:59.002 DEBUG:boto:Token: None 2022-09-03T07:30:59.003 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xf9GaEMHzztS9GBIrssduiRxJVw= 2022-09-03T07:30:59.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xf9GaEMHzztS9GBIrssduiRxJVw=', 'Content-Length': '0'} 2022-09-03T07:30:59.006 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000400abb4d00f3d38b-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.007 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:59.008 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:59.008 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.009 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.009 DEBUG:boto:Method: GET 2022-09-03T07:30:59.010 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.010 DEBUG:boto:Data: 2022-09-03T07:30:59.011 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.012 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.012 DEBUG:boto:Params: {} 2022-09-03T07:30:59.013 DEBUG:boto:Token: None 2022-09-03T07:30:59.014 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xf9GaEMHzztS9GBIrssduiRxJVw= 2022-09-03T07:30:59.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xf9GaEMHzztS9GBIrssduiRxJVw=', 'Content-Length': '0'} 2022-09-03T07:30:59.017 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a151dcd505ad873b-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.018 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:59.021 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:59.021 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.022 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.022 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.023 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.023 DEBUG:boto:Data: 2022-09-03T07:30:59.024 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.025 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.025 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.026 DEBUG:boto:Params: {} 2022-09-03T07:30:59.026 DEBUG:boto:Token: None 2022-09-03T07:30:59.027 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9oiW2D1z7Qp0YJ+xsUG9+vKkr3E= 2022-09-03T07:30:59.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9oiW2D1z7Qp0YJ+xsUG9+vKkr3E=', 'Content-Length': '0'} 2022-09-03T07:30:59.030 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d45c48689bcf83c-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.031 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:59.031 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:59.032 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.032 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.033 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.034 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.035 DEBUG:boto:Data: 2022-09-03T07:30:59.036 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.037 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.038 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.038 DEBUG:boto:Params: {} 2022-09-03T07:30:59.039 DEBUG:boto:Token: None 2022-09-03T07:30:59.040 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9oiW2D1z7Qp0YJ+xsUG9+vKkr3E= 2022-09-03T07:30:59.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9oiW2D1z7Qp0YJ+xsUG9+vKkr3E=', 'Content-Length': '0'} 2022-09-03T07:30:59.044 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ba8a24461c815c69-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.045 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:30:59.046 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:59.047 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:59.047 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.048 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.048 DEBUG:boto:Method: GET 2022-09-03T07:30:59.049 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.049 DEBUG:boto:Data: 2022-09-03T07:30:59.050 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.051 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.051 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.052 DEBUG:boto:Params: {} 2022-09-03T07:30:59.052 DEBUG:boto:Token: None 2022-09-03T07:30:59.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xf9GaEMHzztS9GBIrssduiRxJVw= 2022-09-03T07:30:59.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xf9GaEMHzztS9GBIrssduiRxJVw=', 'Content-Length': '0'} 2022-09-03T07:30:59.057 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002a0376faeae383f2-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.098 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:59.099 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:59.100 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.101 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.102 DEBUG:boto:Method: GET 2022-09-03T07:30:59.103 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.104 DEBUG:boto:Data: 2022-09-03T07:30:59.105 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.107 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.108 DEBUG:boto:Params: {} 2022-09-03T07:30:59.109 DEBUG:boto:Token: None 2022-09-03T07:30:59.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:30:59.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xf9GaEMHzztS9GBIrssduiRxJVw= 2022-09-03T07:30:59.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xf9GaEMHzztS9GBIrssduiRxJVw=', 'Content-Length': '0'} 2022-09-03T07:30:59.115 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001b2330bada1e736-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.116 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.117 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.118 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:59.119 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:59.119 DEBUG:boto:Method: GET 2022-09-03T07:30:59.120 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:30:59.121 DEBUG:boto:Data: 2022-09-03T07:30:59.122 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.122 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.123 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.124 DEBUG:boto:Params: {} 2022-09-03T07:30:59.124 DEBUG:boto:Token: None 2022-09-03T07:30:59.125 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.129 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000647abeb4db132cfb-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.130 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.131 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.132 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.133 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:59.134 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:59.135 DEBUG:boto:Method: GET 2022-09-03T07:30:59.136 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:30:59.137 DEBUG:boto:Data: 2022-09-03T07:30:59.137 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.138 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.139 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.140 DEBUG:boto:Params: {} 2022-09-03T07:30:59.140 DEBUG:boto:Token: None 2022-09-03T07:30:59.141 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.142 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.147 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000097b6b5b6d9cf930f-00631302b3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.148 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.150 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:59.151 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:59.151 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.152 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:30:59.152 DEBUG:boto:Data: 2022-09-03T07:30:59.153 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.153 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.154 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.155 DEBUG:boto:Params: {} 2022-09-03T07:30:59.155 DEBUG:boto:Token: None 2022-09-03T07:30:59.156 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51 2022-09-03T07:30:59.157 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDLok4Fqod2xGGWOo/NkQJ3+gh4= 2022-09-03T07:30:59.157 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDLok4Fqod2xGGWOo/NkQJ3+gh4=', 'Content-Length': '0'} 2022-09-03T07:30:59.159 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020884bcaa62f2cb3-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.160 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:59.161 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:59.161 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.162 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:30:59.162 DEBUG:boto:Data: 2022-09-03T07:30:59.163 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.163 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.164 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.164 DEBUG:boto:Params: {} 2022-09-03T07:30:59.165 DEBUG:boto:Token: None 2022-09-03T07:30:59.165 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51 2022-09-03T07:30:59.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UDLok4Fqod2xGGWOo/NkQJ3+gh4= 2022-09-03T07:30:59.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UDLok4Fqod2xGGWOo/NkQJ3+gh4=', 'Content-Length': '0'} 2022-09-03T07:30:59.169 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075090627f2c3ffda-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.170 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:30:59.170 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:59.171 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:59.171 DEBUG:boto:Method: GET 2022-09-03T07:30:59.172 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:30:59.173 DEBUG:boto:Data: 2022-09-03T07:30:59.173 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.174 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.177 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.177 DEBUG:boto:Params: {} 2022-09-03T07:30:59.178 DEBUG:boto:Token: None 2022-09-03T07:30:59.179 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51 2022-09-03T07:30:59.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/4wev2a+jJYivsyex2LNOGNqDqQ= 2022-09-03T07:30:59.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/4wev2a+jJYivsyex2LNOGNqDqQ=', 'Content-Length': '0'} 2022-09-03T07:30:59.183 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089652ce57d3ac350-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.184 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:30:59.185 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:30:59.186 DEBUG:boto:Method: GET 2022-09-03T07:30:59.187 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:30:59.187 DEBUG:boto:Data: 2022-09-03T07:30:59.188 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.188 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.189 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.190 DEBUG:boto:Params: {} 2022-09-03T07:30:59.191 DEBUG:boto:Token: None 2022-09-03T07:30:59.192 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj51 2022-09-03T07:30:59.193 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/4wev2a+jJYivsyex2LNOGNqDqQ= 2022-09-03T07:30:59.195 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/4wev2a+jJYivsyex2LNOGNqDqQ=', 'Content-Length': '0'} 2022-09-03T07:30:59.198 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf6ef63a3e164006-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.238 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.240 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.241 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:59.243 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:59.244 DEBUG:boto:Method: GET 2022-09-03T07:30:59.244 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:30:59.246 DEBUG:boto:Data: 2022-09-03T07:30:59.247 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.249 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.249 DEBUG:boto:Params: {} 2022-09-03T07:30:59.251 DEBUG:boto:Token: None 2022-09-03T07:30:59.252 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.257 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e11096fc8c12937b-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.258 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.259 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.260 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.261 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:59.261 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:30:59.262 DEBUG:boto:Method: GET 2022-09-03T07:30:59.262 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:30:59.263 DEBUG:boto:Data: 2022-09-03T07:30:59.264 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.264 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.265 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.265 DEBUG:boto:Params: {} 2022-09-03T07:30:59.266 DEBUG:boto:Token: None 2022-09-03T07:30:59.267 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.269 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.274 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a7b8fb282400caca-00631302b3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.275 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.278 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:30:59.279 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.279 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.280 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.280 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.281 DEBUG:boto:Method: GET 2022-09-03T07:30:59.281 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.282 DEBUG:boto:Data: 2022-09-03T07:30:59.282 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.283 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.283 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.284 DEBUG:boto:Params: {} 2022-09-03T07:30:59.285 DEBUG:boto:Token: None 2022-09-03T07:30:59.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpfxHF/1wyp2C3vFtnmevk16Oig= 2022-09-03T07:30:59.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpfxHF/1wyp2C3vFtnmevk16Oig=', 'Content-Length': '0'} 2022-09-03T07:30:59.289 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004426da5b14f8fbce-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.290 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.292 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.292 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.293 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.293 DEBUG:boto:Method: GET 2022-09-03T07:30:59.294 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.295 DEBUG:boto:Data: 2022-09-03T07:30:59.296 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.297 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.298 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.299 DEBUG:boto:Params: {} 2022-09-03T07:30:59.300 DEBUG:boto:Token: None 2022-09-03T07:30:59.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpfxHF/1wyp2C3vFtnmevk16Oig= 2022-09-03T07:30:59.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpfxHF/1wyp2C3vFtnmevk16Oig=', 'Content-Length': '0'} 2022-09-03T07:30:59.305 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7fa634f0f358fdc-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.306 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.307 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.307 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.308 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.308 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.309 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.309 DEBUG:boto:Data: 2022-09-03T07:30:59.310 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.311 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.312 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.313 DEBUG:boto:Params: {} 2022-09-03T07:30:59.314 DEBUG:boto:Token: None 2022-09-03T07:30:59.315 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YeWt8B7MHJ9AM+inF5jHVC7rjg= 2022-09-03T07:30:59.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YeWt8B7MHJ9AM+inF5jHVC7rjg=', 'Content-Length': '0'} 2022-09-03T07:30:59.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010e93351b8abdd0d-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.321 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.322 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.323 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.324 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.325 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.326 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.327 DEBUG:boto:Data: 2022-09-03T07:30:59.328 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.330 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.331 DEBUG:boto:Params: {} 2022-09-03T07:30:59.332 DEBUG:boto:Token: None 2022-09-03T07:30:59.333 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YeWt8B7MHJ9AM+inF5jHVC7rjg= 2022-09-03T07:30:59.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YeWt8B7MHJ9AM+inF5jHVC7rjg=', 'Content-Length': '0'} 2022-09-03T07:30:59.337 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cdfd4656a637fc48-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.337 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:30:59.338 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.339 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.339 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.340 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.340 DEBUG:boto:Method: GET 2022-09-03T07:30:59.341 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.341 DEBUG:boto:Data: 2022-09-03T07:30:59.342 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.343 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.343 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.344 DEBUG:boto:Params: {} 2022-09-03T07:30:59.344 DEBUG:boto:Token: None 2022-09-03T07:30:59.345 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.346 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpfxHF/1wyp2C3vFtnmevk16Oig= 2022-09-03T07:30:59.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpfxHF/1wyp2C3vFtnmevk16Oig=', 'Content-Length': '0'} 2022-09-03T07:30:59.349 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa46f89fa9c1911b-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.350 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.351 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.352 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.353 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.354 DEBUG:boto:Method: GET 2022-09-03T07:30:59.355 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.356 DEBUG:boto:Data: 2022-09-03T07:30:59.356 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.357 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.357 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.358 DEBUG:boto:Params: {} 2022-09-03T07:30:59.359 DEBUG:boto:Token: None 2022-09-03T07:30:59.359 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:30:59.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QpfxHF/1wyp2C3vFtnmevk16Oig= 2022-09-03T07:30:59.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QpfxHF/1wyp2C3vFtnmevk16Oig=', 'Content-Length': '0'} 2022-09-03T07:30:59.363 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006cf84128091c0594-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.364 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.364 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.365 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:59.365 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:59.366 DEBUG:boto:Method: GET 2022-09-03T07:30:59.367 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:30:59.367 DEBUG:boto:Data: 2022-09-03T07:30:59.368 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.369 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.369 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.370 DEBUG:boto:Params: {} 2022-09-03T07:30:59.370 DEBUG:boto:Token: None 2022-09-03T07:30:59.371 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.372 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.375 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000686c21a30479d578-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.375 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.376 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.377 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.378 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:59.378 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:59.379 DEBUG:boto:Method: GET 2022-09-03T07:30:59.379 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:30:59.380 DEBUG:boto:Data: 2022-09-03T07:30:59.380 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.381 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.381 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.382 DEBUG:boto:Params: {} 2022-09-03T07:30:59.383 DEBUG:boto:Token: None 2022-09-03T07:30:59.383 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.388 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000abb4a82a1448d774-00631302b3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.435 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.436 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.436 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.437 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.438 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:30:59.439 DEBUG:boto:Data: 2022-09-03T07:30:59.440 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.441 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.442 DEBUG:boto:Params: {} 2022-09-03T07:30:59.442 DEBUG:boto:Token: None 2022-09-03T07:30:59.443 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55 2022-09-03T07:30:59.443 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xTnEnj2Zx14SWlj2ubZ3dbvZf7w= 2022-09-03T07:30:59.444 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xTnEnj2Zx14SWlj2ubZ3dbvZf7w=', 'Content-Length': '0'} 2022-09-03T07:30:59.447 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac6d7c1fde2cff4e-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.449 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.450 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.451 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.451 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:30:59.452 DEBUG:boto:Data: 2022-09-03T07:30:59.452 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.453 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.454 DEBUG:boto:Params: {} 2022-09-03T07:30:59.455 DEBUG:boto:Token: None 2022-09-03T07:30:59.455 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55 2022-09-03T07:30:59.456 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xTnEnj2Zx14SWlj2ubZ3dbvZf7w= 2022-09-03T07:30:59.457 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xTnEnj2Zx14SWlj2ubZ3dbvZf7w=', 'Content-Length': '0'} 2022-09-03T07:30:59.459 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe28f4c4ed1b66c8-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.460 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:30:59.460 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.461 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.462 DEBUG:boto:Method: GET 2022-09-03T07:30:59.462 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:30:59.463 DEBUG:boto:Data: 2022-09-03T07:30:59.463 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.464 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.465 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.466 DEBUG:boto:Params: {} 2022-09-03T07:30:59.467 DEBUG:boto:Token: None 2022-09-03T07:30:59.468 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55 2022-09-03T07:30:59.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ArZfvDmLakTMfsWdZkWkiQEss9E= 2022-09-03T07:30:59.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ArZfvDmLakTMfsWdZkWkiQEss9E=', 'Content-Length': '0'} 2022-09-03T07:30:59.471 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027adefb679f25a40-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.472 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:30:59.475 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:30:59.476 DEBUG:boto:Method: GET 2022-09-03T07:30:59.477 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:30:59.478 DEBUG:boto:Data: 2022-09-03T07:30:59.479 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.480 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.480 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.481 DEBUG:boto:Params: {} 2022-09-03T07:30:59.482 DEBUG:boto:Token: None 2022-09-03T07:30:59.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj55 2022-09-03T07:30:59.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ArZfvDmLakTMfsWdZkWkiQEss9E= 2022-09-03T07:30:59.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ArZfvDmLakTMfsWdZkWkiQEss9E=', 'Content-Length': '0'} 2022-09-03T07:30:59.488 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099e24d5f4904a57e-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.489 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.490 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.491 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:59.492 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:59.493 DEBUG:boto:Method: GET 2022-09-03T07:30:59.494 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:30:59.495 DEBUG:boto:Data: 2022-09-03T07:30:59.496 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.496 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.497 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.498 DEBUG:boto:Params: {} 2022-09-03T07:30:59.499 DEBUG:boto:Token: None 2022-09-03T07:30:59.500 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.501 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.502 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.505 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001e733afec47feb98-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.507 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.508 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.509 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.510 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:59.511 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:30:59.512 DEBUG:boto:Method: GET 2022-09-03T07:30:59.513 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:30:59.513 DEBUG:boto:Data: 2022-09-03T07:30:59.514 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.515 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.516 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.517 DEBUG:boto:Params: {} 2022-09-03T07:30:59.518 DEBUG:boto:Token: None 2022-09-03T07:30:59.519 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.521 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.522 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.527 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000087753dcb2685142a-00631302b3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.528 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.529 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:30:59.529 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.530 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.531 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.531 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.532 DEBUG:boto:Method: GET 2022-09-03T07:30:59.532 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.533 DEBUG:boto:Data: 2022-09-03T07:30:59.533 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.534 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.534 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.535 DEBUG:boto:Params: {} 2022-09-03T07:30:59.536 DEBUG:boto:Token: None 2022-09-03T07:30:59.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vgc/qpn/C5brHhsCBMt/tH515Hc= 2022-09-03T07:30:59.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vgc/qpn/C5brHhsCBMt/tH515Hc=', 'Content-Length': '0'} 2022-09-03T07:30:59.540 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086b84a30bf3e993f-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.541 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.541 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.542 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.542 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.543 DEBUG:boto:Method: GET 2022-09-03T07:30:59.543 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.544 DEBUG:boto:Data: 2022-09-03T07:30:59.545 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.546 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.547 DEBUG:boto:Params: {} 2022-09-03T07:30:59.548 DEBUG:boto:Token: None 2022-09-03T07:30:59.548 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vgc/qpn/C5brHhsCBMt/tH515Hc= 2022-09-03T07:30:59.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vgc/qpn/C5brHhsCBMt/tH515Hc=', 'Content-Length': '0'} 2022-09-03T07:30:59.552 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000220d924ec9cb1c6a-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.553 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.554 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.555 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.556 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.556 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.557 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.558 DEBUG:boto:Data: 2022-09-03T07:30:59.559 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.560 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.560 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.561 DEBUG:boto:Params: {} 2022-09-03T07:30:59.562 DEBUG:boto:Token: None 2022-09-03T07:30:59.562 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.563 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ydoa2vG0tIUFeAISXP33P2WKVw= 2022-09-03T07:30:59.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ydoa2vG0tIUFeAISXP33P2WKVw=', 'Content-Length': '0'} 2022-09-03T07:30:59.565 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000088d8d62304442707-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.566 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.567 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.568 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.568 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.569 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.569 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.570 DEBUG:boto:Data: 2022-09-03T07:30:59.571 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.571 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.572 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.572 DEBUG:boto:Params: {} 2022-09-03T07:30:59.573 DEBUG:boto:Token: None 2022-09-03T07:30:59.573 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ydoa2vG0tIUFeAISXP33P2WKVw= 2022-09-03T07:30:59.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ydoa2vG0tIUFeAISXP33P2WKVw=', 'Content-Length': '0'} 2022-09-03T07:30:59.577 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a80730453fc9fc7c-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.578 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:30:59.578 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.579 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.579 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.580 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.581 DEBUG:boto:Method: GET 2022-09-03T07:30:59.581 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.582 DEBUG:boto:Data: 2022-09-03T07:30:59.582 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.583 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.584 DEBUG:boto:Params: {} 2022-09-03T07:30:59.585 DEBUG:boto:Token: None 2022-09-03T07:30:59.585 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.586 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vgc/qpn/C5brHhsCBMt/tH515Hc= 2022-09-03T07:30:59.587 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vgc/qpn/C5brHhsCBMt/tH515Hc=', 'Content-Length': '0'} 2022-09-03T07:30:59.589 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057bfbb6efa575bb0-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.590 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.590 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.591 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.591 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.592 DEBUG:boto:Method: GET 2022-09-03T07:30:59.592 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.593 DEBUG:boto:Data: 2022-09-03T07:30:59.593 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.594 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.595 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.595 DEBUG:boto:Params: {} 2022-09-03T07:30:59.596 DEBUG:boto:Token: None 2022-09-03T07:30:59.596 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:30:59.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Vgc/qpn/C5brHhsCBMt/tH515Hc= 2022-09-03T07:30:59.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Vgc/qpn/C5brHhsCBMt/tH515Hc=', 'Content-Length': '0'} 2022-09-03T07:30:59.600 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000efcec0889e4d4912-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.601 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.601 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.602 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:59.602 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:59.603 DEBUG:boto:Method: GET 2022-09-03T07:30:59.604 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:30:59.604 DEBUG:boto:Data: 2022-09-03T07:30:59.605 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.605 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.606 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.606 DEBUG:boto:Params: {} 2022-09-03T07:30:59.607 DEBUG:boto:Token: None 2022-09-03T07:30:59.608 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.608 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.609 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.611 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b8ebcbec1d90c99-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.612 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.613 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.613 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.614 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:59.615 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:59.615 DEBUG:boto:Method: GET 2022-09-03T07:30:59.616 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:30:59.616 DEBUG:boto:Data: 2022-09-03T07:30:59.617 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.618 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.618 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.619 DEBUG:boto:Params: {} 2022-09-03T07:30:59.619 DEBUG:boto:Token: None 2022-09-03T07:30:59.620 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.621 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.626 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a06023e48e2aff3e-00631302b3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.627 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.628 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.628 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.629 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.629 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:30:59.630 DEBUG:boto:Data: 2022-09-03T07:30:59.630 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.631 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.632 DEBUG:boto:Params: {} 2022-09-03T07:30:59.632 DEBUG:boto:Token: None 2022-09-03T07:30:59.633 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57 2022-09-03T07:30:59.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2qarMYPtrb9nWMTplc9SUXTKw1w= 2022-09-03T07:30:59.634 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2qarMYPtrb9nWMTplc9SUXTKw1w=', 'Content-Length': '0'} 2022-09-03T07:30:59.636 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000018d6d5cc9d820f63-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.637 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.638 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.639 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.640 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:30:59.641 DEBUG:boto:Data: 2022-09-03T07:30:59.642 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.643 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.644 DEBUG:boto:Params: {} 2022-09-03T07:30:59.645 DEBUG:boto:Token: None 2022-09-03T07:30:59.645 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57 2022-09-03T07:30:59.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2qarMYPtrb9nWMTplc9SUXTKw1w= 2022-09-03T07:30:59.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2qarMYPtrb9nWMTplc9SUXTKw1w=', 'Content-Length': '0'} 2022-09-03T07:30:59.648 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d55aab9b122a2a2-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.649 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:30:59.649 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.650 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.650 DEBUG:boto:Method: GET 2022-09-03T07:30:59.651 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:30:59.652 DEBUG:boto:Data: 2022-09-03T07:30:59.652 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.653 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.653 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.654 DEBUG:boto:Params: {} 2022-09-03T07:30:59.654 DEBUG:boto:Token: None 2022-09-03T07:30:59.655 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57 2022-09-03T07:30:59.656 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZUDfs+qv5eJwScdHH4WKNbz+5U4= 2022-09-03T07:30:59.656 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZUDfs+qv5eJwScdHH4WKNbz+5U4=', 'Content-Length': '0'} 2022-09-03T07:30:59.658 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035ef7c70a3e844a3-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.706 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:30:59.707 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:30:59.708 DEBUG:boto:Method: GET 2022-09-03T07:30:59.708 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:30:59.709 DEBUG:boto:Data: 2022-09-03T07:30:59.709 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.710 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.710 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.711 DEBUG:boto:Params: {} 2022-09-03T07:30:59.711 DEBUG:boto:Token: None 2022-09-03T07:30:59.712 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj57 2022-09-03T07:30:59.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZUDfs+qv5eJwScdHH4WKNbz+5U4= 2022-09-03T07:30:59.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZUDfs+qv5eJwScdHH4WKNbz+5U4=', 'Content-Length': '0'} 2022-09-03T07:30:59.715 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000899a4eb23ce22e23-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.716 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.717 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.717 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:59.718 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:59.718 DEBUG:boto:Method: GET 2022-09-03T07:30:59.719 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:30:59.719 DEBUG:boto:Data: 2022-09-03T07:30:59.720 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.720 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.721 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.721 DEBUG:boto:Params: {} 2022-09-03T07:30:59.722 DEBUG:boto:Token: None 2022-09-03T07:30:59.722 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.723 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.724 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.726 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9b2cd2d6cb10771-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.727 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.729 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.730 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.731 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:59.732 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:30:59.733 DEBUG:boto:Method: GET 2022-09-03T07:30:59.734 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:30:59.735 DEBUG:boto:Data: 2022-09-03T07:30:59.736 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.736 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.737 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.738 DEBUG:boto:Params: {} 2022-09-03T07:30:59.739 DEBUG:boto:Token: None 2022-09-03T07:30:59.740 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.748 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cc4f81c43e6630b6-00631302b3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.750 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.751 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:30:59.752 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.752 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.753 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.754 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.755 DEBUG:boto:Method: GET 2022-09-03T07:30:59.755 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.756 DEBUG:boto:Data: 2022-09-03T07:30:59.757 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.759 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.760 DEBUG:boto:Params: {} 2022-09-03T07:30:59.766 DEBUG:boto:Token: None 2022-09-03T07:30:59.767 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.768 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vpgYRQiAUsDjWUnb23pWrUmOXx0= 2022-09-03T07:30:59.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vpgYRQiAUsDjWUnb23pWrUmOXx0=', 'Content-Length': '0'} 2022-09-03T07:30:59.773 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053c8863bd4a8dde1-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.774 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.775 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.776 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.779 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.780 DEBUG:boto:Method: GET 2022-09-03T07:30:59.781 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.782 DEBUG:boto:Data: 2022-09-03T07:30:59.783 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.784 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.784 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.785 DEBUG:boto:Params: {} 2022-09-03T07:30:59.788 DEBUG:boto:Token: None 2022-09-03T07:30:59.789 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vpgYRQiAUsDjWUnb23pWrUmOXx0= 2022-09-03T07:30:59.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vpgYRQiAUsDjWUnb23pWrUmOXx0=', 'Content-Length': '0'} 2022-09-03T07:30:59.794 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e72ee753d3f7fc2-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.795 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.796 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.796 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.797 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.798 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.799 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.799 DEBUG:boto:Data: 2022-09-03T07:30:59.800 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.800 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.801 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.801 DEBUG:boto:Params: {} 2022-09-03T07:30:59.802 DEBUG:boto:Token: None 2022-09-03T07:30:59.802 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PqCgpzbmFPsmB4qnH14VwFEU9WU= 2022-09-03T07:30:59.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PqCgpzbmFPsmB4qnH14VwFEU9WU=', 'Content-Length': '0'} 2022-09-03T07:30:59.806 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d82f7b40c9cf5207-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.806 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.807 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.807 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.808 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.809 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.809 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.810 DEBUG:boto:Data: 2022-09-03T07:30:59.810 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.811 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.812 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.813 DEBUG:boto:Params: {} 2022-09-03T07:30:59.815 DEBUG:boto:Token: None 2022-09-03T07:30:59.815 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PqCgpzbmFPsmB4qnH14VwFEU9WU= 2022-09-03T07:30:59.817 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PqCgpzbmFPsmB4qnH14VwFEU9WU=', 'Content-Length': '0'} 2022-09-03T07:30:59.819 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000966b5c316b3860a4-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.820 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:30:59.821 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.822 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.824 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.825 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.826 DEBUG:boto:Method: GET 2022-09-03T07:30:59.827 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.828 DEBUG:boto:Data: 2022-09-03T07:30:59.829 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.830 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.831 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.832 DEBUG:boto:Params: {} 2022-09-03T07:30:59.834 DEBUG:boto:Token: None 2022-09-03T07:30:59.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vpgYRQiAUsDjWUnb23pWrUmOXx0= 2022-09-03T07:30:59.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vpgYRQiAUsDjWUnb23pWrUmOXx0=', 'Content-Length': '0'} 2022-09-03T07:30:59.840 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e5f692a826561a94-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.841 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.842 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.843 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.844 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.845 DEBUG:boto:Method: GET 2022-09-03T07:30:59.847 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.848 DEBUG:boto:Data: 2022-09-03T07:30:59.848 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.849 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.850 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.851 DEBUG:boto:Params: {} 2022-09-03T07:30:59.852 DEBUG:boto:Token: None 2022-09-03T07:30:59.853 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:30:59.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vpgYRQiAUsDjWUnb23pWrUmOXx0= 2022-09-03T07:30:59.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vpgYRQiAUsDjWUnb23pWrUmOXx0=', 'Content-Length': '0'} 2022-09-03T07:30:59.858 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000781877abbc515f26-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.859 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.859 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.860 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:59.860 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:59.861 DEBUG:boto:Method: GET 2022-09-03T07:30:59.862 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:30:59.862 DEBUG:boto:Data: 2022-09-03T07:30:59.863 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.863 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.864 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.864 DEBUG:boto:Params: {} 2022-09-03T07:30:59.865 DEBUG:boto:Token: None 2022-09-03T07:30:59.866 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.868 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.871 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cf471aa7cd3511aa-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.872 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.873 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.873 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.874 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:59.874 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:59.875 DEBUG:boto:Method: GET 2022-09-03T07:30:59.876 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:30:59.876 DEBUG:boto:Data: 2022-09-03T07:30:59.877 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.878 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.879 DEBUG:boto:Params: {} 2022-09-03T07:30:59.880 DEBUG:boto:Token: None 2022-09-03T07:30:59.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.886 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001dd062fe40e5394c-00631302b3-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.887 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.888 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.889 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.890 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.890 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:30:59.891 DEBUG:boto:Data: 2022-09-03T07:30:59.892 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.893 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.894 DEBUG:boto:Params: {} 2022-09-03T07:30:59.895 DEBUG:boto:Token: None 2022-09-03T07:30:59.897 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58 2022-09-03T07:30:59.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZ+KlSBPESi5AwAohyN7pyTI33I= 2022-09-03T07:30:59.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZ+KlSBPESi5AwAohyN7pyTI33I=', 'Content-Length': '0'} 2022-09-03T07:30:59.901 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000981a201855fc3824-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.903 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.904 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.905 DEBUG:boto:Method: HEAD 2022-09-03T07:30:59.906 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:30:59.906 DEBUG:boto:Data: 2022-09-03T07:30:59.907 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.908 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.908 DEBUG:boto:Params: {} 2022-09-03T07:30:59.909 DEBUG:boto:Token: None 2022-09-03T07:30:59.910 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58 2022-09-03T07:30:59.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BZ+KlSBPESi5AwAohyN7pyTI33I= 2022-09-03T07:30:59.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BZ+KlSBPESi5AwAohyN7pyTI33I=', 'Content-Length': '0'} 2022-09-03T07:30:59.913 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000095f00f1d2f22591-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.914 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:30:59.915 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.915 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.916 DEBUG:boto:Method: GET 2022-09-03T07:30:59.916 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:30:59.917 DEBUG:boto:Data: 2022-09-03T07:30:59.918 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.918 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.919 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.919 DEBUG:boto:Params: {} 2022-09-03T07:30:59.920 DEBUG:boto:Token: None 2022-09-03T07:30:59.921 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58 2022-09-03T07:30:59.922 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6xp4nO7hHOJVQVTodiPVr6WryiQ= 2022-09-03T07:30:59.922 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6xp4nO7hHOJVQVTodiPVr6WryiQ=', 'Content-Length': '0'} 2022-09-03T07:30:59.925 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f69b66104e29c15c-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.927 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:30:59.928 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:30:59.929 DEBUG:boto:Method: GET 2022-09-03T07:30:59.930 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:30:59.931 DEBUG:boto:Data: 2022-09-03T07:30:59.932 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:30:59.933 DEBUG:boto:Port: 8000 2022-09-03T07:30:59.934 DEBUG:boto:Params: {} 2022-09-03T07:30:59.935 DEBUG:boto:Token: None 2022-09-03T07:30:59.936 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/obj58 2022-09-03T07:30:59.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6xp4nO7hHOJVQVTodiPVr6WryiQ= 2022-09-03T07:30:59.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6xp4nO7hHOJVQVTodiPVr6WryiQ=', 'Content-Length': '0'} 2022-09-03T07:30:59.939 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b44c8d654a89198-00631302b3-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.982 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.983 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.984 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:59.984 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:59.985 DEBUG:boto:Method: GET 2022-09-03T07:30:59.985 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:30:59.986 DEBUG:boto:Data: 2022-09-03T07:30:59.987 DEBUG:boto:Headers: {} 2022-09-03T07:30:59.987 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:30:59.988 DEBUG:boto:Port: 8001 2022-09-03T07:30:59.988 DEBUG:boto:Params: {} 2022-09-03T07:30:59.989 DEBUG:boto:Token: None 2022-09-03T07:30:59.990 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:30:59 GMT /avysmw-75/ 2022-09-03T07:30:59.990 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo= 2022-09-03T07:30:59.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:30:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+x0ba/re+o7yXlbX9jMBqL07dmo=', 'Content-Length': '0'} 2022-09-03T07:30:59.994 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000043ae708d5405fbc6-00631302b3-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:30:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:30:59.995 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:30:59.996 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:30:59.997 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:30:59.998 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:59.998 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:30:59.999 DEBUG:boto:Method: GET 2022-09-03T07:30:59.999 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:31:00.000 DEBUG:boto:Data: 2022-09-03T07:31:00.000 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.001 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.002 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.002 DEBUG:boto:Params: {} 2022-09-03T07:31:00.003 DEBUG:boto:Token: None 2022-09-03T07:31:00.003 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000ef3d9d41af4c444-00631302b4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.011 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.013 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:31:00.014 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.015 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.016 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.017 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.018 DEBUG:boto:Method: GET 2022-09-03T07:31:00.019 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.019 DEBUG:boto:Data: 2022-09-03T07:31:00.020 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.021 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.022 DEBUG:boto:Params: {} 2022-09-03T07:31:00.022 DEBUG:boto:Token: None 2022-09-03T07:31:00.023 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xof/EplS2eZEAIVlZlp2m/KPrPA= 2022-09-03T07:31:00.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xof/EplS2eZEAIVlZlp2m/KPrPA=', 'Content-Length': '0'} 2022-09-03T07:31:00.033 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e9c0cc8b3a8fb56-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.074 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.076 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.077 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.078 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.079 DEBUG:boto:Method: GET 2022-09-03T07:31:00.080 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.081 DEBUG:boto:Data: 2022-09-03T07:31:00.082 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.083 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.085 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.087 DEBUG:boto:Params: {} 2022-09-03T07:31:00.088 DEBUG:boto:Token: None 2022-09-03T07:31:00.090 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.091 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xof/EplS2eZEAIVlZlp2m/KPrPA= 2022-09-03T07:31:00.092 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xof/EplS2eZEAIVlZlp2m/KPrPA=', 'Content-Length': '0'} 2022-09-03T07:31:00.095 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f168858dc4920ece-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.096 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.097 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.098 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.099 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.100 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.101 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.103 DEBUG:boto:Data: 2022-09-03T07:31:00.105 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.106 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.107 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.108 DEBUG:boto:Params: {} 2022-09-03T07:31:00.109 DEBUG:boto:Token: None 2022-09-03T07:31:00.110 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.110 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FHPDfrPwk85EQPxMGVXKklHLzA= 2022-09-03T07:31:00.112 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FHPDfrPwk85EQPxMGVXKklHLzA=', 'Content-Length': '0'} 2022-09-03T07:31:00.114 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093ac881e79895755-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.115 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.115 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.116 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.116 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.117 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.117 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.118 DEBUG:boto:Data: 2022-09-03T07:31:00.118 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.119 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.119 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.120 DEBUG:boto:Params: {} 2022-09-03T07:31:00.120 DEBUG:boto:Token: None 2022-09-03T07:31:00.121 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8FHPDfrPwk85EQPxMGVXKklHLzA= 2022-09-03T07:31:00.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8FHPDfrPwk85EQPxMGVXKklHLzA=', 'Content-Length': '0'} 2022-09-03T07:31:00.125 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000751ae8819ec33a4b-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.125 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:31:00.126 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.127 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.127 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.128 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.128 DEBUG:boto:Method: GET 2022-09-03T07:31:00.129 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.130 DEBUG:boto:Data: 2022-09-03T07:31:00.130 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.131 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.131 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.132 DEBUG:boto:Params: {} 2022-09-03T07:31:00.133 DEBUG:boto:Token: None 2022-09-03T07:31:00.133 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.134 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xof/EplS2eZEAIVlZlp2m/KPrPA= 2022-09-03T07:31:00.135 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xof/EplS2eZEAIVlZlp2m/KPrPA=', 'Content-Length': '0'} 2022-09-03T07:31:00.139 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3667da84df0516a-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.140 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.140 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.141 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.141 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.142 DEBUG:boto:Method: GET 2022-09-03T07:31:00.142 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.143 DEBUG:boto:Data: 2022-09-03T07:31:00.143 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.144 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.144 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.145 DEBUG:boto:Params: {} 2022-09-03T07:31:00.146 DEBUG:boto:Token: None 2022-09-03T07:31:00.146 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:00.147 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xof/EplS2eZEAIVlZlp2m/KPrPA= 2022-09-03T07:31:00.148 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xof/EplS2eZEAIVlZlp2m/KPrPA=', 'Content-Length': '0'} 2022-09-03T07:31:00.150 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f391fa6c1b5b7331-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.152 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.153 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.156 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:00.156 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:00.157 DEBUG:boto:Method: GET 2022-09-03T07:31:00.158 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:31:00.159 DEBUG:boto:Data: 2022-09-03T07:31:00.160 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.161 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.162 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.163 DEBUG:boto:Params: {} 2022-09-03T07:31:00.163 DEBUG:boto:Token: None 2022-09-03T07:31:00.164 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.166 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.169 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007bbd7653b852f773-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.170 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.172 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.173 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.174 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:00.175 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:00.176 DEBUG:boto:Method: GET 2022-09-03T07:31:00.177 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:31:00.178 DEBUG:boto:Data: 2022-09-03T07:31:00.179 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.179 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.180 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.181 DEBUG:boto:Params: {} 2022-09-03T07:31:00.182 DEBUG:boto:Token: None 2022-09-03T07:31:00.183 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.190 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cad0f17a0bfa9bca-00631302b4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.238 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.240 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.241 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.242 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.242 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:31:00.243 DEBUG:boto:Data: 2022-09-03T07:31:00.244 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.245 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.246 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.248 DEBUG:boto:Params: {} 2022-09-03T07:31:00.249 DEBUG:boto:Token: None 2022-09-03T07:31:00.250 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62 2022-09-03T07:31:00.251 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HqZnK34gpBg2fG1X3ZOTZN2sjPI= 2022-09-03T07:31:00.252 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HqZnK34gpBg2fG1X3ZOTZN2sjPI=', 'Content-Length': '0'} 2022-09-03T07:31:00.255 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009df7840f5222888e-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.256 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.257 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.258 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.259 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:31:00.260 DEBUG:boto:Data: 2022-09-03T07:31:00.261 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.262 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.263 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.264 DEBUG:boto:Params: {} 2022-09-03T07:31:00.265 DEBUG:boto:Token: None 2022-09-03T07:31:00.266 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62 2022-09-03T07:31:00.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HqZnK34gpBg2fG1X3ZOTZN2sjPI= 2022-09-03T07:31:00.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HqZnK34gpBg2fG1X3ZOTZN2sjPI=', 'Content-Length': '0'} 2022-09-03T07:31:00.270 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008435cef940799221-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.272 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:31:00.272 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.273 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.275 DEBUG:boto:Method: GET 2022-09-03T07:31:00.276 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:31:00.276 DEBUG:boto:Data: 2022-09-03T07:31:00.277 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.278 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.279 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.280 DEBUG:boto:Params: {} 2022-09-03T07:31:00.281 DEBUG:boto:Token: None 2022-09-03T07:31:00.282 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62 2022-09-03T07:31:00.283 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G879gWk4EGj/mp5/FNs5f65XglE= 2022-09-03T07:31:00.284 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G879gWk4EGj/mp5/FNs5f65XglE=', 'Content-Length': '0'} 2022-09-03T07:31:00.287 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5372fc53f68f8b9-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.330 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:00.331 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:00.331 DEBUG:boto:Method: GET 2022-09-03T07:31:00.332 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:31:00.333 DEBUG:boto:Data: 2022-09-03T07:31:00.334 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.334 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.336 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.336 DEBUG:boto:Params: {} 2022-09-03T07:31:00.337 DEBUG:boto:Token: None 2022-09-03T07:31:00.338 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj62 2022-09-03T07:31:00.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G879gWk4EGj/mp5/FNs5f65XglE= 2022-09-03T07:31:00.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G879gWk4EGj/mp5/FNs5f65XglE=', 'Content-Length': '0'} 2022-09-03T07:31:00.343 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000031a561847ab5b405-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.345 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.346 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.347 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:00.348 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:00.348 DEBUG:boto:Method: GET 2022-09-03T07:31:00.349 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:31:00.350 DEBUG:boto:Data: 2022-09-03T07:31:00.351 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.352 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.352 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.353 DEBUG:boto:Params: {} 2022-09-03T07:31:00.354 DEBUG:boto:Token: None 2022-09-03T07:31:00.355 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.357 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.361 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000070222f26e8045189-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.362 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.363 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.364 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.365 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:00.366 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:00.367 DEBUG:boto:Method: GET 2022-09-03T07:31:00.368 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:31:00.369 DEBUG:boto:Data: 2022-09-03T07:31:00.369 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.371 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.372 DEBUG:boto:Params: {} 2022-09-03T07:31:00.373 DEBUG:boto:Token: None 2022-09-03T07:31:00.373 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.374 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.375 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.379 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c7dd7f3e1e26df6b-00631302b4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.380 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.381 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:31:00.382 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.382 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.383 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.383 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.384 DEBUG:boto:Method: GET 2022-09-03T07:31:00.385 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.386 DEBUG:boto:Data: 2022-09-03T07:31:00.387 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.388 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.389 DEBUG:boto:Params: {} 2022-09-03T07:31:00.389 DEBUG:boto:Token: None 2022-09-03T07:31:00.390 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.391 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:08+zgJKuTxny0B17JadTkzefFww= 2022-09-03T07:31:00.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:08+zgJKuTxny0B17JadTkzefFww=', 'Content-Length': '0'} 2022-09-03T07:31:00.393 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000daa3102a246bdc8a-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.394 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.395 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.395 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.396 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.396 DEBUG:boto:Method: GET 2022-09-03T07:31:00.397 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.397 DEBUG:boto:Data: 2022-09-03T07:31:00.398 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.399 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.399 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.400 DEBUG:boto:Params: {} 2022-09-03T07:31:00.401 DEBUG:boto:Token: None 2022-09-03T07:31:00.401 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.402 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:08+zgJKuTxny0B17JadTkzefFww= 2022-09-03T07:31:00.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:08+zgJKuTxny0B17JadTkzefFww=', 'Content-Length': '0'} 2022-09-03T07:31:00.405 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000496a118f7fa6d3e1-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.406 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.407 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.408 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.409 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.410 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.411 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.412 DEBUG:boto:Data: 2022-09-03T07:31:00.413 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.414 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.415 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.416 DEBUG:boto:Params: {} 2022-09-03T07:31:00.418 DEBUG:boto:Token: None 2022-09-03T07:31:00.419 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rd/Vob5o+dkCqivJt72B52tgJN8= 2022-09-03T07:31:00.421 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rd/Vob5o+dkCqivJt72B52tgJN8=', 'Content-Length': '0'} 2022-09-03T07:31:00.424 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee944965586feeb2-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.425 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.426 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.427 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.428 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.428 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.429 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.429 DEBUG:boto:Data: 2022-09-03T07:31:00.430 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.430 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.431 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.431 DEBUG:boto:Params: {} 2022-09-03T07:31:00.432 DEBUG:boto:Token: None 2022-09-03T07:31:00.432 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rd/Vob5o+dkCqivJt72B52tgJN8= 2022-09-03T07:31:00.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rd/Vob5o+dkCqivJt72B52tgJN8=', 'Content-Length': '0'} 2022-09-03T07:31:00.436 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4464ba3b509e193-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.437 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:31:00.438 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.438 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.439 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.440 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.440 DEBUG:boto:Method: GET 2022-09-03T07:31:00.441 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.441 DEBUG:boto:Data: 2022-09-03T07:31:00.442 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.442 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.443 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.443 DEBUG:boto:Params: {} 2022-09-03T07:31:00.445 DEBUG:boto:Token: None 2022-09-03T07:31:00.445 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:08+zgJKuTxny0B17JadTkzefFww= 2022-09-03T07:31:00.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:08+zgJKuTxny0B17JadTkzefFww=', 'Content-Length': '0'} 2022-09-03T07:31:00.450 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000732b8b0e92c05c11-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.451 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.452 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.453 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.454 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.455 DEBUG:boto:Method: GET 2022-09-03T07:31:00.457 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.458 DEBUG:boto:Data: 2022-09-03T07:31:00.459 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.461 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.462 DEBUG:boto:Params: {} 2022-09-03T07:31:00.462 DEBUG:boto:Token: None 2022-09-03T07:31:00.463 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:00.464 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:08+zgJKuTxny0B17JadTkzefFww= 2022-09-03T07:31:00.464 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:08+zgJKuTxny0B17JadTkzefFww=', 'Content-Length': '0'} 2022-09-03T07:31:00.466 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002f14cb50c00daa5b-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.468 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.468 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.469 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:00.470 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:00.470 DEBUG:boto:Method: GET 2022-09-03T07:31:00.471 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:31:00.471 DEBUG:boto:Data: 2022-09-03T07:31:00.472 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.472 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.473 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.473 DEBUG:boto:Params: {} 2022-09-03T07:31:00.474 DEBUG:boto:Token: None 2022-09-03T07:31:00.475 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.475 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.476 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.479 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000debff31f1ce8da94-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.480 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.482 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.483 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.484 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:00.484 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:00.485 DEBUG:boto:Method: GET 2022-09-03T07:31:00.486 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:31:00.487 DEBUG:boto:Data: 2022-09-03T07:31:00.488 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.488 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.489 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.490 DEBUG:boto:Params: {} 2022-09-03T07:31:00.491 DEBUG:boto:Token: None 2022-09-03T07:31:00.492 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.499 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009c604358764c2d06-00631302b4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.542 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.543 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.544 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.544 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.545 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:31:00.545 DEBUG:boto:Data: 2022-09-03T07:31:00.546 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.548 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.549 DEBUG:boto:Params: {} 2022-09-03T07:31:00.550 DEBUG:boto:Token: None 2022-09-03T07:31:00.551 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68 2022-09-03T07:31:00.552 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bB1YHBf7W+paz+XXhQbG4bBfVmo= 2022-09-03T07:31:00.553 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bB1YHBf7W+paz+XXhQbG4bBfVmo=', 'Content-Length': '0'} 2022-09-03T07:31:00.555 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d08648f8fe2be95b-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.556 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.557 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.557 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.558 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:31:00.558 DEBUG:boto:Data: 2022-09-03T07:31:00.559 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.559 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.560 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.560 DEBUG:boto:Params: {} 2022-09-03T07:31:00.561 DEBUG:boto:Token: None 2022-09-03T07:31:00.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68 2022-09-03T07:31:00.562 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bB1YHBf7W+paz+XXhQbG4bBfVmo= 2022-09-03T07:31:00.563 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bB1YHBf7W+paz+XXhQbG4bBfVmo=', 'Content-Length': '0'} 2022-09-03T07:31:00.565 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065ca5128f4052181-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.566 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:31:00.566 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.567 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.567 DEBUG:boto:Method: GET 2022-09-03T07:31:00.568 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:31:00.569 DEBUG:boto:Data: 2022-09-03T07:31:00.571 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.572 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.573 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.574 DEBUG:boto:Params: {} 2022-09-03T07:31:00.576 DEBUG:boto:Token: None 2022-09-03T07:31:00.576 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68 2022-09-03T07:31:00.577 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRJF2GWb0triSxx47e8j1OmAufA= 2022-09-03T07:31:00.577 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRJF2GWb0triSxx47e8j1OmAufA=', 'Content-Length': '0'} 2022-09-03T07:31:00.580 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000069a55c8c282a1913-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.581 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:00.582 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:00.583 DEBUG:boto:Method: GET 2022-09-03T07:31:00.584 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:31:00.584 DEBUG:boto:Data: 2022-09-03T07:31:00.585 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.586 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.587 DEBUG:boto:Params: {} 2022-09-03T07:31:00.587 DEBUG:boto:Token: None 2022-09-03T07:31:00.588 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj68 2022-09-03T07:31:00.588 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sRJF2GWb0triSxx47e8j1OmAufA= 2022-09-03T07:31:00.589 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sRJF2GWb0triSxx47e8j1OmAufA=', 'Content-Length': '0'} 2022-09-03T07:31:00.592 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6b673612dcb074a-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.593 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.593 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.594 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:00.594 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:00.595 DEBUG:boto:Method: GET 2022-09-03T07:31:00.595 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:31:00.596 DEBUG:boto:Data: 2022-09-03T07:31:00.597 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.597 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.598 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.599 DEBUG:boto:Params: {} 2022-09-03T07:31:00.599 DEBUG:boto:Token: None 2022-09-03T07:31:00.600 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.604 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000583b0628b53823f3-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.604 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.605 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.606 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.607 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:00.607 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:00.608 DEBUG:boto:Method: GET 2022-09-03T07:31:00.609 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:31:00.610 DEBUG:boto:Data: 2022-09-03T07:31:00.611 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.612 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.612 DEBUG:boto:Params: {} 2022-09-03T07:31:00.613 DEBUG:boto:Token: None 2022-09-03T07:31:00.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.620 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000014b34a293ac70361-00631302b4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.621 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.622 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:31:00.622 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.623 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.623 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.624 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.624 DEBUG:boto:Method: GET 2022-09-03T07:31:00.625 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.625 DEBUG:boto:Data: 2022-09-03T07:31:00.626 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.627 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.627 DEBUG:boto:Params: {} 2022-09-03T07:31:00.628 DEBUG:boto:Token: None 2022-09-03T07:31:00.629 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kz6vOnSovp4TiPbUtDo12SKvPRU= 2022-09-03T07:31:00.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kz6vOnSovp4TiPbUtDo12SKvPRU=', 'Content-Length': '0'} 2022-09-03T07:31:00.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ad34272bc12153ec-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.633 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.634 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.634 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.635 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.635 DEBUG:boto:Method: GET 2022-09-03T07:31:00.636 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.637 DEBUG:boto:Data: 2022-09-03T07:31:00.637 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.638 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.638 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.639 DEBUG:boto:Params: {} 2022-09-03T07:31:00.639 DEBUG:boto:Token: None 2022-09-03T07:31:00.640 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.641 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kz6vOnSovp4TiPbUtDo12SKvPRU= 2022-09-03T07:31:00.641 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kz6vOnSovp4TiPbUtDo12SKvPRU=', 'Content-Length': '0'} 2022-09-03T07:31:00.644 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b1b4fa7afa9108d-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.645 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.645 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.646 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.646 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.647 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.648 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.648 DEBUG:boto:Data: 2022-09-03T07:31:00.649 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.650 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.650 DEBUG:boto:Params: {} 2022-09-03T07:31:00.651 DEBUG:boto:Token: None 2022-09-03T07:31:00.651 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TB20GcC2HkM+ZqHLB4XBcifz3bo= 2022-09-03T07:31:00.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TB20GcC2HkM+ZqHLB4XBcifz3bo=', 'Content-Length': '0'} 2022-09-03T07:31:00.654 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000205c1b09fc7651bb-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.655 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.658 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.658 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.659 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.659 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.660 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.661 DEBUG:boto:Data: 2022-09-03T07:31:00.662 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.663 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.664 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.665 DEBUG:boto:Params: {} 2022-09-03T07:31:00.666 DEBUG:boto:Token: None 2022-09-03T07:31:00.667 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.668 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TB20GcC2HkM+ZqHLB4XBcifz3bo= 2022-09-03T07:31:00.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TB20GcC2HkM+ZqHLB4XBcifz3bo=', 'Content-Length': '0'} 2022-09-03T07:31:00.671 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000078b91db89fba068d-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.672 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:31:00.673 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.673 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.674 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.675 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.675 DEBUG:boto:Method: GET 2022-09-03T07:31:00.676 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.677 DEBUG:boto:Data: 2022-09-03T07:31:00.677 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.679 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.679 DEBUG:boto:Params: {} 2022-09-03T07:31:00.680 DEBUG:boto:Token: None 2022-09-03T07:31:00.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kz6vOnSovp4TiPbUtDo12SKvPRU= 2022-09-03T07:31:00.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kz6vOnSovp4TiPbUtDo12SKvPRU=', 'Content-Length': '0'} 2022-09-03T07:31:00.684 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000060fa743a31e0f27b-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.685 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.686 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.686 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.687 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.687 DEBUG:boto:Method: GET 2022-09-03T07:31:00.688 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.689 DEBUG:boto:Data: 2022-09-03T07:31:00.689 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.690 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.691 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.691 DEBUG:boto:Params: {} 2022-09-03T07:31:00.692 DEBUG:boto:Token: None 2022-09-03T07:31:00.693 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:00.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kz6vOnSovp4TiPbUtDo12SKvPRU= 2022-09-03T07:31:00.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kz6vOnSovp4TiPbUtDo12SKvPRU=', 'Content-Length': '0'} 2022-09-03T07:31:00.696 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f704c335d748df1f-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.697 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.698 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.698 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:00.699 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:00.700 DEBUG:boto:Method: GET 2022-09-03T07:31:00.700 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:31:00.701 DEBUG:boto:Data: 2022-09-03T07:31:00.701 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.702 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.702 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.703 DEBUG:boto:Params: {} 2022-09-03T07:31:00.704 DEBUG:boto:Token: None 2022-09-03T07:31:00.704 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.705 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.706 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.708 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a98d9da87ab7e3e-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.709 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.710 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.711 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.711 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:00.712 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:00.712 DEBUG:boto:Method: GET 2022-09-03T07:31:00.713 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:31:00.714 DEBUG:boto:Data: 2022-09-03T07:31:00.714 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.715 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.715 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.716 DEBUG:boto:Params: {} 2022-09-03T07:31:00.716 DEBUG:boto:Token: None 2022-09-03T07:31:00.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.718 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.723 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002970ff48cd4c9242-00631302b4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.723 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.724 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.725 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.725 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.726 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:31:00.726 DEBUG:boto:Data: 2022-09-03T07:31:00.727 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.727 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.728 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.729 DEBUG:boto:Params: {} 2022-09-03T07:31:00.729 DEBUG:boto:Token: None 2022-09-03T07:31:00.730 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69 2022-09-03T07:31:00.730 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MA9S7xIPvok1mzeIlhiILjsdpk0= 2022-09-03T07:31:00.731 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MA9S7xIPvok1mzeIlhiILjsdpk0=', 'Content-Length': '0'} 2022-09-03T07:31:00.733 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065f9fe0be74502aa-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.734 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.735 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.735 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.736 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:31:00.736 DEBUG:boto:Data: 2022-09-03T07:31:00.737 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.738 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.738 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.739 DEBUG:boto:Params: {} 2022-09-03T07:31:00.739 DEBUG:boto:Token: None 2022-09-03T07:31:00.740 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69 2022-09-03T07:31:00.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MA9S7xIPvok1mzeIlhiILjsdpk0= 2022-09-03T07:31:00.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MA9S7xIPvok1mzeIlhiILjsdpk0=', 'Content-Length': '0'} 2022-09-03T07:31:00.744 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d28d00fe0447abe1-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.745 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:31:00.745 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.746 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.747 DEBUG:boto:Method: GET 2022-09-03T07:31:00.747 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:31:00.748 DEBUG:boto:Data: 2022-09-03T07:31:00.749 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.750 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.750 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.751 DEBUG:boto:Params: {} 2022-09-03T07:31:00.752 DEBUG:boto:Token: None 2022-09-03T07:31:00.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69 2022-09-03T07:31:00.753 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:leWexDtbjtuU3nn/uSkY0Fu6HOQ= 2022-09-03T07:31:00.754 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:leWexDtbjtuU3nn/uSkY0Fu6HOQ=', 'Content-Length': '0'} 2022-09-03T07:31:00.756 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006048c63512781553-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.757 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:00.758 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:00.759 DEBUG:boto:Method: GET 2022-09-03T07:31:00.759 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:31:00.760 DEBUG:boto:Data: 2022-09-03T07:31:00.761 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.762 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.763 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.764 DEBUG:boto:Params: {} 2022-09-03T07:31:00.765 DEBUG:boto:Token: None 2022-09-03T07:31:00.765 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj69 2022-09-03T07:31:00.766 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:leWexDtbjtuU3nn/uSkY0Fu6HOQ= 2022-09-03T07:31:00.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:leWexDtbjtuU3nn/uSkY0Fu6HOQ=', 'Content-Length': '0'} 2022-09-03T07:31:00.769 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e1daf22e535f6d8-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.810 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.811 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.811 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:00.812 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:00.813 DEBUG:boto:Method: GET 2022-09-03T07:31:00.814 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:31:00.814 DEBUG:boto:Data: 2022-09-03T07:31:00.815 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.816 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.817 DEBUG:boto:Params: {} 2022-09-03T07:31:00.818 DEBUG:boto:Token: None 2022-09-03T07:31:00.819 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.823 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000045f1b9b3e9e8185f-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.824 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.825 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.826 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.827 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:00.827 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:00.828 DEBUG:boto:Method: GET 2022-09-03T07:31:00.829 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:31:00.829 DEBUG:boto:Data: 2022-09-03T07:31:00.830 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.831 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.831 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.832 DEBUG:boto:Params: {} 2022-09-03T07:31:00.833 DEBUG:boto:Token: None 2022-09-03T07:31:00.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.836 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.837 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.848 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011867471de745294-00631302b4-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.849 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.852 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj74 2022-09-03T07:31:00.853 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:00.853 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:00.854 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.855 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.856 DEBUG:boto:Method: GET 2022-09-03T07:31:00.856 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.857 DEBUG:boto:Data: 2022-09-03T07:31:00.858 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.858 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.859 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.860 DEBUG:boto:Params: {} 2022-09-03T07:31:00.861 DEBUG:boto:Token: None 2022-09-03T07:31:00.861 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WEgs/ZdKUr/98MIaZPyTWMF2mHw= 2022-09-03T07:31:00.864 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WEgs/ZdKUr/98MIaZPyTWMF2mHw=', 'Content-Length': '0'} 2022-09-03T07:31:00.867 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f591b0000930ae06-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.868 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:00.870 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:00.871 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.872 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.873 DEBUG:boto:Method: GET 2022-09-03T07:31:00.876 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.878 DEBUG:boto:Data: 2022-09-03T07:31:00.879 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.882 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.883 DEBUG:boto:Params: {} 2022-09-03T07:31:00.885 DEBUG:boto:Token: None 2022-09-03T07:31:00.887 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WEgs/ZdKUr/98MIaZPyTWMF2mHw= 2022-09-03T07:31:00.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WEgs/ZdKUr/98MIaZPyTWMF2mHw=', 'Content-Length': '0'} 2022-09-03T07:31:00.893 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc1c8811f9e45ff0-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.896 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:00.897 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:00.898 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.899 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.899 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.900 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.901 DEBUG:boto:Data: 2022-09-03T07:31:00.902 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.903 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.904 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.904 DEBUG:boto:Params: {} 2022-09-03T07:31:00.905 DEBUG:boto:Token: None 2022-09-03T07:31:00.908 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:elm0CoQxovEKM1WT09ead8cOuro= 2022-09-03T07:31:00.911 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:elm0CoQxovEKM1WT09ead8cOuro=', 'Content-Length': '0'} 2022-09-03T07:31:00.914 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dfc09e9e3edc6b51-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.916 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:00.917 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:00.919 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.920 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.922 DEBUG:boto:Method: HEAD 2022-09-03T07:31:00.923 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.924 DEBUG:boto:Data: 2022-09-03T07:31:00.926 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.927 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.928 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.929 DEBUG:boto:Params: {} 2022-09-03T07:31:00.930 DEBUG:boto:Token: None 2022-09-03T07:31:00.932 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.933 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:elm0CoQxovEKM1WT09ead8cOuro= 2022-09-03T07:31:00.934 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:elm0CoQxovEKM1WT09ead8cOuro=', 'Content-Length': '0'} 2022-09-03T07:31:00.936 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000167b2ad4068535fd-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.937 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj74 2022-09-03T07:31:00.939 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:00.940 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:00.941 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.942 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.944 DEBUG:boto:Method: GET 2022-09-03T07:31:00.945 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.946 DEBUG:boto:Data: 2022-09-03T07:31:00.947 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.949 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.950 DEBUG:boto:Params: {} 2022-09-03T07:31:00.951 DEBUG:boto:Token: None 2022-09-03T07:31:00.951 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.954 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WEgs/ZdKUr/98MIaZPyTWMF2mHw= 2022-09-03T07:31:00.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WEgs/ZdKUr/98MIaZPyTWMF2mHw=', 'Content-Length': '0'} 2022-09-03T07:31:00.957 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5f4dae784cf18f5-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.958 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:00.959 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:00.960 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.960 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.961 DEBUG:boto:Method: GET 2022-09-03T07:31:00.962 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.963 DEBUG:boto:Data: 2022-09-03T07:31:00.964 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.965 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.966 DEBUG:boto:Params: {} 2022-09-03T07:31:00.967 DEBUG:boto:Token: None 2022-09-03T07:31:00.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:00.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WEgs/ZdKUr/98MIaZPyTWMF2mHw= 2022-09-03T07:31:00.969 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WEgs/ZdKUr/98MIaZPyTWMF2mHw=', 'Content-Length': '0'} 2022-09-03T07:31:00.972 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9356a7f3ef28076-00631302b4-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.975 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.977 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.978 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:00.979 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:00.980 DEBUG:boto:Method: GET 2022-09-03T07:31:00.981 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:31:00.981 DEBUG:boto:Data: 2022-09-03T07:31:00.982 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.983 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:00.984 DEBUG:boto:Port: 8001 2022-09-03T07:31:00.984 DEBUG:boto:Params: {} 2022-09-03T07:31:00.985 DEBUG:boto:Token: None 2022-09-03T07:31:00.986 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:00 GMT /avysmw-75/ 2022-09-03T07:31:00.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE= 2022-09-03T07:31:00.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pesHHfGJ5XZ/+5qL5CdNq/uF4QE=', 'Content-Length': '0'} 2022-09-03T07:31:00.990 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004ffcb2fbeea4097c-00631302b4-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:00.991 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:00.992 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:00.992 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:00.993 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:00.994 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:00.994 DEBUG:boto:Method: GET 2022-09-03T07:31:00.995 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:31:00.996 DEBUG:boto:Data: 2022-09-03T07:31:00.996 DEBUG:boto:Headers: {} 2022-09-03T07:31:00.997 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:00.998 DEBUG:boto:Port: 8000 2022-09-03T07:31:00.999 DEBUG:boto:Params: {} 2022-09-03T07:31:00.999 DEBUG:boto:Token: None 2022-09-03T07:31:01.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.007 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000587aa72b5700f66d-00631302b5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.009 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.010 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:01.011 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:01.011 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.012 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:31:01.012 DEBUG:boto:Data: 2022-09-03T07:31:01.013 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.014 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.014 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.015 DEBUG:boto:Params: {} 2022-09-03T07:31:01.016 DEBUG:boto:Token: None 2022-09-03T07:31:01.016 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj74 2022-09-03T07:31:01.017 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dmW/g4XJyir/AInuJXY8CpmBd60= 2022-09-03T07:31:01.018 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dmW/g4XJyir/AInuJXY8CpmBd60=', 'Content-Length': '0'} 2022-09-03T07:31:01.020 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4c1426b44dd4e3c-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.021 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:01.021 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:01.022 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.024 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:31:01.025 DEBUG:boto:Data: 2022-09-03T07:31:01.025 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.026 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.033 DEBUG:boto:Params: {} 2022-09-03T07:31:01.034 DEBUG:boto:Token: None 2022-09-03T07:31:01.035 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj74 2022-09-03T07:31:01.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dmW/g4XJyir/AInuJXY8CpmBd60= 2022-09-03T07:31:01.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dmW/g4XJyir/AInuJXY8CpmBd60=', 'Content-Length': '0'} 2022-09-03T07:31:01.043 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fd1c791ebc39b5b9-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.045 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj74 2022-09-03T07:31:01.046 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:01.046 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:01.047 DEBUG:boto:Method: GET 2022-09-03T07:31:01.048 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:31:01.049 DEBUG:boto:Data: 2022-09-03T07:31:01.049 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.050 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.051 DEBUG:boto:Params: {} 2022-09-03T07:31:01.052 DEBUG:boto:Token: None 2022-09-03T07:31:01.052 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj74 2022-09-03T07:31:01.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gVgfcK7eH5mOV0n4Sq4a/r1N9kc= 2022-09-03T07:31:01.054 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gVgfcK7eH5mOV0n4Sq4a/r1N9kc=', 'Content-Length': '0'} 2022-09-03T07:31:01.056 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000eaa267a4ee01f9a-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.098 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:01.099 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:01.100 DEBUG:boto:Method: GET 2022-09-03T07:31:01.100 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:31:01.101 DEBUG:boto:Data: 2022-09-03T07:31:01.101 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.102 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.103 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.105 DEBUG:boto:Params: {} 2022-09-03T07:31:01.106 DEBUG:boto:Token: None 2022-09-03T07:31:01.107 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj74 2022-09-03T07:31:01.108 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gVgfcK7eH5mOV0n4Sq4a/r1N9kc= 2022-09-03T07:31:01.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gVgfcK7eH5mOV0n4Sq4a/r1N9kc=', 'Content-Length': '0'} 2022-09-03T07:31:01.111 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000e8ac6db37dfdcde-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.112 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.112 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.113 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:01.114 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:01.114 DEBUG:boto:Method: GET 2022-09-03T07:31:01.115 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:31:01.115 DEBUG:boto:Data: 2022-09-03T07:31:01.116 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.116 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.117 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.118 DEBUG:boto:Params: {} 2022-09-03T07:31:01.118 DEBUG:boto:Token: None 2022-09-03T07:31:01.119 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.119 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.120 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.122 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a2fc7b13cff94281-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.123 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.127 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.128 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.129 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:01.134 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:01.135 DEBUG:boto:Method: GET 2022-09-03T07:31:01.136 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:31:01.137 DEBUG:boto:Data: 2022-09-03T07:31:01.138 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.138 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.139 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.140 DEBUG:boto:Params: {} 2022-09-03T07:31:01.140 DEBUG:boto:Token: None 2022-09-03T07:31:01.141 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.143 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.152 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff2fd5b9be35cda8-00631302b5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.194 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.195 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:31:01.196 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.197 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.198 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.199 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.200 DEBUG:boto:Method: GET 2022-09-03T07:31:01.201 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.203 DEBUG:boto:Data: 2022-09-03T07:31:01.204 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.205 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.206 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.207 DEBUG:boto:Params: {} 2022-09-03T07:31:01.209 DEBUG:boto:Token: None 2022-09-03T07:31:01.210 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kn47096/2FpaFl/T4k/omUXGQ00= 2022-09-03T07:31:01.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kn47096/2FpaFl/T4k/omUXGQ00=', 'Content-Length': '0'} 2022-09-03T07:31:01.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f6c7c5df10041eba-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.217 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.218 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.220 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.221 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.222 DEBUG:boto:Method: GET 2022-09-03T07:31:01.223 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.224 DEBUG:boto:Data: 2022-09-03T07:31:01.225 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.226 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.227 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.227 DEBUG:boto:Params: {} 2022-09-03T07:31:01.228 DEBUG:boto:Token: None 2022-09-03T07:31:01.229 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kn47096/2FpaFl/T4k/omUXGQ00= 2022-09-03T07:31:01.230 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kn47096/2FpaFl/T4k/omUXGQ00=', 'Content-Length': '0'} 2022-09-03T07:31:01.233 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e2e98d1d1abc5c45-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.233 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.234 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.235 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.235 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.236 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.236 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.237 DEBUG:boto:Data: 2022-09-03T07:31:01.237 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.238 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.239 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.240 DEBUG:boto:Params: {} 2022-09-03T07:31:01.240 DEBUG:boto:Token: None 2022-09-03T07:31:01.241 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.242 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nDZFOfFTFs8LaFvHTPOYUuAyHmU= 2022-09-03T07:31:01.243 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nDZFOfFTFs8LaFvHTPOYUuAyHmU=', 'Content-Length': '0'} 2022-09-03T07:31:01.246 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c8acb8f264b6009-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.246 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.247 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.247 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.248 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.249 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.249 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.250 DEBUG:boto:Data: 2022-09-03T07:31:01.250 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.252 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.252 DEBUG:boto:Params: {} 2022-09-03T07:31:01.254 DEBUG:boto:Token: None 2022-09-03T07:31:01.255 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nDZFOfFTFs8LaFvHTPOYUuAyHmU= 2022-09-03T07:31:01.257 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nDZFOfFTFs8LaFvHTPOYUuAyHmU=', 'Content-Length': '0'} 2022-09-03T07:31:01.260 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003daa3d84f0b199fe-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.262 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:31:01.263 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.264 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.265 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.266 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.267 DEBUG:boto:Method: GET 2022-09-03T07:31:01.268 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.270 DEBUG:boto:Data: 2022-09-03T07:31:01.271 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.272 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.273 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.273 DEBUG:boto:Params: {} 2022-09-03T07:31:01.274 DEBUG:boto:Token: None 2022-09-03T07:31:01.275 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kn47096/2FpaFl/T4k/omUXGQ00= 2022-09-03T07:31:01.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kn47096/2FpaFl/T4k/omUXGQ00=', 'Content-Length': '0'} 2022-09-03T07:31:01.279 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a25efee70be89783-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.281 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.281 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.282 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.283 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.285 DEBUG:boto:Method: GET 2022-09-03T07:31:01.285 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.286 DEBUG:boto:Data: 2022-09-03T07:31:01.287 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.287 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.288 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.289 DEBUG:boto:Params: {} 2022-09-03T07:31:01.291 DEBUG:boto:Token: None 2022-09-03T07:31:01.292 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:01.293 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kn47096/2FpaFl/T4k/omUXGQ00= 2022-09-03T07:31:01.294 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kn47096/2FpaFl/T4k/omUXGQ00=', 'Content-Length': '0'} 2022-09-03T07:31:01.297 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000058812faa30bd682a-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.298 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.299 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.300 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:01.301 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:01.302 DEBUG:boto:Method: GET 2022-09-03T07:31:01.303 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:31:01.304 DEBUG:boto:Data: 2022-09-03T07:31:01.305 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.307 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.308 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.309 DEBUG:boto:Params: {} 2022-09-03T07:31:01.310 DEBUG:boto:Token: None 2022-09-03T07:31:01.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.316 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000048d54a4eb2aef77e-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.316 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.317 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.318 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.319 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:01.319 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:01.320 DEBUG:boto:Method: GET 2022-09-03T07:31:01.320 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:31:01.321 DEBUG:boto:Data: 2022-09-03T07:31:01.322 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.322 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.323 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.323 DEBUG:boto:Params: {} 2022-09-03T07:31:01.324 DEBUG:boto:Token: None 2022-09-03T07:31:01.325 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.325 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.326 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.330 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000066a1a518e236f418-00631302b5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.331 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.332 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.333 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.333 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.334 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:31:01.335 DEBUG:boto:Data: 2022-09-03T07:31:01.335 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.336 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.336 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.337 DEBUG:boto:Params: {} 2022-09-03T07:31:01.338 DEBUG:boto:Token: None 2022-09-03T07:31:01.339 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76 2022-09-03T07:31:01.339 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:keaXQKk390/tdBU78MsJvR6qj6s= 2022-09-03T07:31:01.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:keaXQKk390/tdBU78MsJvR6qj6s=', 'Content-Length': '0'} 2022-09-03T07:31:01.342 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a0115e73a85d1943-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.343 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.344 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.345 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.345 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:31:01.346 DEBUG:boto:Data: 2022-09-03T07:31:01.346 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.347 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.348 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.348 DEBUG:boto:Params: {} 2022-09-03T07:31:01.349 DEBUG:boto:Token: None 2022-09-03T07:31:01.349 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76 2022-09-03T07:31:01.350 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:keaXQKk390/tdBU78MsJvR6qj6s= 2022-09-03T07:31:01.350 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:keaXQKk390/tdBU78MsJvR6qj6s=', 'Content-Length': '0'} 2022-09-03T07:31:01.352 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000025eeb7911ef0ed1e-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.353 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:31:01.354 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.354 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.355 DEBUG:boto:Method: GET 2022-09-03T07:31:01.355 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:31:01.356 DEBUG:boto:Data: 2022-09-03T07:31:01.357 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.357 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.358 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.358 DEBUG:boto:Params: {} 2022-09-03T07:31:01.359 DEBUG:boto:Token: None 2022-09-03T07:31:01.360 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76 2022-09-03T07:31:01.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yhMEpDbDuzXnNo4OXew7XESEh2c= 2022-09-03T07:31:01.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yhMEpDbDuzXnNo4OXew7XESEh2c=', 'Content-Length': '0'} 2022-09-03T07:31:01.363 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bb9d3039ab206c66-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.406 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:01.407 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:01.408 DEBUG:boto:Method: GET 2022-09-03T07:31:01.408 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:31:01.409 DEBUG:boto:Data: 2022-09-03T07:31:01.409 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.410 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.410 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.411 DEBUG:boto:Params: {} 2022-09-03T07:31:01.412 DEBUG:boto:Token: None 2022-09-03T07:31:01.412 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj76 2022-09-03T07:31:01.413 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yhMEpDbDuzXnNo4OXew7XESEh2c= 2022-09-03T07:31:01.413 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yhMEpDbDuzXnNo4OXew7XESEh2c=', 'Content-Length': '0'} 2022-09-03T07:31:01.415 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002fe7b0bbe869fec7-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.416 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.417 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.417 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:01.418 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:01.418 DEBUG:boto:Method: GET 2022-09-03T07:31:01.419 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:31:01.419 DEBUG:boto:Data: 2022-09-03T07:31:01.420 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.421 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.421 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.422 DEBUG:boto:Params: {} 2022-09-03T07:31:01.423 DEBUG:boto:Token: None 2022-09-03T07:31:01.424 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.428 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002945d2e029184261-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.428 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.429 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.430 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.431 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:01.431 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:01.432 DEBUG:boto:Method: GET 2022-09-03T07:31:01.432 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:31:01.433 DEBUG:boto:Data: 2022-09-03T07:31:01.433 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.435 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.438 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.439 DEBUG:boto:Params: {} 2022-09-03T07:31:01.440 DEBUG:boto:Token: None 2022-09-03T07:31:01.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.448 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e866c65a45392488-00631302b5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.450 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.451 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:31:01.452 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.453 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.454 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.455 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.456 DEBUG:boto:Method: GET 2022-09-03T07:31:01.457 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.458 DEBUG:boto:Data: 2022-09-03T07:31:01.459 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.460 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.461 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.462 DEBUG:boto:Params: {} 2022-09-03T07:31:01.463 DEBUG:boto:Token: None 2022-09-03T07:31:01.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:01jFzkMZNErlh5oIMlZ3FP3T5GQ= 2022-09-03T07:31:01.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:01jFzkMZNErlh5oIMlZ3FP3T5GQ=', 'Content-Length': '0'} 2022-09-03T07:31:01.469 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000078ff4d8d9e7d97c-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.471 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.472 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.473 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.474 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.474 DEBUG:boto:Method: GET 2022-09-03T07:31:01.475 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.475 DEBUG:boto:Data: 2022-09-03T07:31:01.476 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.477 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.477 DEBUG:boto:Params: {} 2022-09-03T07:31:01.478 DEBUG:boto:Token: None 2022-09-03T07:31:01.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:01jFzkMZNErlh5oIMlZ3FP3T5GQ= 2022-09-03T07:31:01.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:01jFzkMZNErlh5oIMlZ3FP3T5GQ=', 'Content-Length': '0'} 2022-09-03T07:31:01.482 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008c7de7e50a11513-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.526 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.527 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.527 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.528 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.529 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.529 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.530 DEBUG:boto:Data: 2022-09-03T07:31:01.530 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.531 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.531 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.532 DEBUG:boto:Params: {} 2022-09-03T07:31:01.532 DEBUG:boto:Token: None 2022-09-03T07:31:01.533 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.534 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lTu88KOgrglLe2yOye7zh8eEjVk= 2022-09-03T07:31:01.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lTu88KOgrglLe2yOye7zh8eEjVk=', 'Content-Length': '0'} 2022-09-03T07:31:01.536 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b768aa2d13590153-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.538 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.540 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.540 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.541 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.541 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.542 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.543 DEBUG:boto:Data: 2022-09-03T07:31:01.544 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.546 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.547 DEBUG:boto:Params: {} 2022-09-03T07:31:01.548 DEBUG:boto:Token: None 2022-09-03T07:31:01.549 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.551 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lTu88KOgrglLe2yOye7zh8eEjVk= 2022-09-03T07:31:01.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lTu88KOgrglLe2yOye7zh8eEjVk=', 'Content-Length': '0'} 2022-09-03T07:31:01.555 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e9b2a490f7dd86c3-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.557 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:31:01.558 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.559 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.562 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.563 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.563 DEBUG:boto:Method: GET 2022-09-03T07:31:01.564 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.564 DEBUG:boto:Data: 2022-09-03T07:31:01.566 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.566 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.568 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.570 DEBUG:boto:Params: {} 2022-09-03T07:31:01.571 DEBUG:boto:Token: None 2022-09-03T07:31:01.572 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:01jFzkMZNErlh5oIMlZ3FP3T5GQ= 2022-09-03T07:31:01.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:01jFzkMZNErlh5oIMlZ3FP3T5GQ=', 'Content-Length': '0'} 2022-09-03T07:31:01.576 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000029519ed0b5e09ce-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.577 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.577 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.578 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.579 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.579 DEBUG:boto:Method: GET 2022-09-03T07:31:01.580 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.580 DEBUG:boto:Data: 2022-09-03T07:31:01.581 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.582 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.582 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.583 DEBUG:boto:Params: {} 2022-09-03T07:31:01.583 DEBUG:boto:Token: None 2022-09-03T07:31:01.584 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:01.585 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:01jFzkMZNErlh5oIMlZ3FP3T5GQ= 2022-09-03T07:31:01.585 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:01jFzkMZNErlh5oIMlZ3FP3T5GQ=', 'Content-Length': '0'} 2022-09-03T07:31:01.588 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009b9bb74c29d18cc5-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.589 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.590 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.590 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:01.591 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:01.592 DEBUG:boto:Method: GET 2022-09-03T07:31:01.592 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:31:01.593 DEBUG:boto:Data: 2022-09-03T07:31:01.594 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.594 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.595 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.595 DEBUG:boto:Params: {} 2022-09-03T07:31:01.596 DEBUG:boto:Token: None 2022-09-03T07:31:01.598 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.599 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.599 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.602 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009be93da1ebf8224b-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.603 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.605 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.605 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.606 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:01.607 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:01.608 DEBUG:boto:Method: GET 2022-09-03T07:31:01.608 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:31:01.610 DEBUG:boto:Data: 2022-09-03T07:31:01.611 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.611 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.612 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.612 DEBUG:boto:Params: {} 2022-09-03T07:31:01.613 DEBUG:boto:Token: None 2022-09-03T07:31:01.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.626 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000005583784afe920ee-00631302b5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.627 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.627 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.628 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.629 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.629 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:31:01.630 DEBUG:boto:Data: 2022-09-03T07:31:01.630 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.631 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.632 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.632 DEBUG:boto:Params: {} 2022-09-03T07:31:01.633 DEBUG:boto:Token: None 2022-09-03T07:31:01.634 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85 2022-09-03T07:31:01.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:brV+cA+1zwPBM1RCeXpcs6vs5Po= 2022-09-03T07:31:01.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:brV+cA+1zwPBM1RCeXpcs6vs5Po=', 'Content-Length': '0'} 2022-09-03T07:31:01.639 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c8e4eeab3851dcfb-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.639 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.640 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.640 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.641 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:31:01.641 DEBUG:boto:Data: 2022-09-03T07:31:01.642 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.643 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.643 DEBUG:boto:Params: {} 2022-09-03T07:31:01.644 DEBUG:boto:Token: None 2022-09-03T07:31:01.644 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85 2022-09-03T07:31:01.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:brV+cA+1zwPBM1RCeXpcs6vs5Po= 2022-09-03T07:31:01.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:brV+cA+1zwPBM1RCeXpcs6vs5Po=', 'Content-Length': '0'} 2022-09-03T07:31:01.647 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000730db08e434fb2f3-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.648 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:31:01.649 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.650 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.651 DEBUG:boto:Method: GET 2022-09-03T07:31:01.652 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:31:01.653 DEBUG:boto:Data: 2022-09-03T07:31:01.654 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.655 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.656 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.657 DEBUG:boto:Params: {} 2022-09-03T07:31:01.658 DEBUG:boto:Token: None 2022-09-03T07:31:01.659 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85 2022-09-03T07:31:01.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xmEjOwDyasPs2zBSRaIDoct2INk= 2022-09-03T07:31:01.660 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xmEjOwDyasPs2zBSRaIDoct2INk=', 'Content-Length': '0'} 2022-09-03T07:31:01.662 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032e5e44d5a485e68-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.708 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:01.709 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:01.710 DEBUG:boto:Method: GET 2022-09-03T07:31:01.710 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:31:01.711 DEBUG:boto:Data: 2022-09-03T07:31:01.711 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.712 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.712 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.713 DEBUG:boto:Params: {} 2022-09-03T07:31:01.714 DEBUG:boto:Token: None 2022-09-03T07:31:01.714 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj85 2022-09-03T07:31:01.715 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xmEjOwDyasPs2zBSRaIDoct2INk= 2022-09-03T07:31:01.716 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xmEjOwDyasPs2zBSRaIDoct2INk=', 'Content-Length': '0'} 2022-09-03T07:31:01.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac12e322e7b34ecf-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.723 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.724 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.724 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:01.725 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:01.726 DEBUG:boto:Method: GET 2022-09-03T07:31:01.726 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:31:01.727 DEBUG:boto:Data: 2022-09-03T07:31:01.727 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.728 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.728 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.729 DEBUG:boto:Params: {} 2022-09-03T07:31:01.730 DEBUG:boto:Token: None 2022-09-03T07:31:01.731 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.732 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.733 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.735 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000068e1f0e4ef968505-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.736 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.737 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.738 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.738 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:01.739 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:01.739 DEBUG:boto:Method: GET 2022-09-03T07:31:01.740 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:31:01.741 DEBUG:boto:Data: 2022-09-03T07:31:01.741 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.742 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.743 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.743 DEBUG:boto:Params: {} 2022-09-03T07:31:01.744 DEBUG:boto:Token: None 2022-09-03T07:31:01.744 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.745 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.746 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.750 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001a0a557450b7ad89-00631302b5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.794 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.795 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:31:01.796 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.797 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.798 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.799 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.800 DEBUG:boto:Method: GET 2022-09-03T07:31:01.801 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.802 DEBUG:boto:Data: 2022-09-03T07:31:01.803 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.805 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.805 DEBUG:boto:Params: {} 2022-09-03T07:31:01.806 DEBUG:boto:Token: None 2022-09-03T07:31:01.807 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.808 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JR6pzfQhpkXSrZTtCdlMdivL+6I= 2022-09-03T07:31:01.809 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JR6pzfQhpkXSrZTtCdlMdivL+6I=', 'Content-Length': '0'} 2022-09-03T07:31:01.812 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000767be86ea7c3bedd-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.813 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.814 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.815 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.816 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.816 DEBUG:boto:Method: GET 2022-09-03T07:31:01.817 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.817 DEBUG:boto:Data: 2022-09-03T07:31:01.818 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.818 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.819 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.819 DEBUG:boto:Params: {} 2022-09-03T07:31:01.820 DEBUG:boto:Token: None 2022-09-03T07:31:01.821 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.821 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JR6pzfQhpkXSrZTtCdlMdivL+6I= 2022-09-03T07:31:01.822 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JR6pzfQhpkXSrZTtCdlMdivL+6I=', 'Content-Length': '0'} 2022-09-03T07:31:01.824 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037a28e11c3b10391-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.825 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.825 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.826 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.826 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.827 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.827 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.828 DEBUG:boto:Data: 2022-09-03T07:31:01.828 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.829 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.829 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.830 DEBUG:boto:Params: {} 2022-09-03T07:31:01.831 DEBUG:boto:Token: None 2022-09-03T07:31:01.831 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.832 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PFkpDeR8xtTsN/Yd3xyGSAZ90CE= 2022-09-03T07:31:01.832 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PFkpDeR8xtTsN/Yd3xyGSAZ90CE=', 'Content-Length': '0'} 2022-09-03T07:31:01.835 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020cc2a0e7489f6fb-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.836 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.837 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.837 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.838 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.838 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.840 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.841 DEBUG:boto:Data: 2022-09-03T07:31:01.842 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.843 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.843 DEBUG:boto:Params: {} 2022-09-03T07:31:01.844 DEBUG:boto:Token: None 2022-09-03T07:31:01.845 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PFkpDeR8xtTsN/Yd3xyGSAZ90CE= 2022-09-03T07:31:01.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PFkpDeR8xtTsN/Yd3xyGSAZ90CE=', 'Content-Length': '0'} 2022-09-03T07:31:01.848 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be972adf72a593ca-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.848 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:31:01.849 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.849 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.850 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.851 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.851 DEBUG:boto:Method: GET 2022-09-03T07:31:01.852 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.852 DEBUG:boto:Data: 2022-09-03T07:31:01.853 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.853 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.854 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.854 DEBUG:boto:Params: {} 2022-09-03T07:31:01.855 DEBUG:boto:Token: None 2022-09-03T07:31:01.856 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JR6pzfQhpkXSrZTtCdlMdivL+6I= 2022-09-03T07:31:01.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JR6pzfQhpkXSrZTtCdlMdivL+6I=', 'Content-Length': '0'} 2022-09-03T07:31:01.870 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076f75138720f261a-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.872 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.873 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.873 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.874 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.874 DEBUG:boto:Method: GET 2022-09-03T07:31:01.875 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.875 DEBUG:boto:Data: 2022-09-03T07:31:01.876 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.877 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.878 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.878 DEBUG:boto:Params: {} 2022-09-03T07:31:01.879 DEBUG:boto:Token: None 2022-09-03T07:31:01.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:01.880 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JR6pzfQhpkXSrZTtCdlMdivL+6I= 2022-09-03T07:31:01.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JR6pzfQhpkXSrZTtCdlMdivL+6I=', 'Content-Length': '0'} 2022-09-03T07:31:01.883 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008a32c2296c09b5cb-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.884 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.885 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.885 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:01.886 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:01.887 DEBUG:boto:Method: GET 2022-09-03T07:31:01.888 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:31:01.888 DEBUG:boto:Data: 2022-09-03T07:31:01.889 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.889 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.890 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.890 DEBUG:boto:Params: {} 2022-09-03T07:31:01.891 DEBUG:boto:Token: None 2022-09-03T07:31:01.892 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.893 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.905 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000de2a6aa16cd6eb8e-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.938 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.939 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:01.940 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:01.940 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:01.941 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:01.941 DEBUG:boto:Method: GET 2022-09-03T07:31:01.942 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:31:01.942 DEBUG:boto:Data: 2022-09-03T07:31:01.943 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.943 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.944 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.944 DEBUG:boto:Params: {} 2022-09-03T07:31:01.948 DEBUG:boto:Token: None 2022-09-03T07:31:01.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/ 2022-09-03T07:31:01.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA= 2022-09-03T07:31:01.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gUjaZVOVTGA00hTfOeqdkv31dSA=', 'Content-Length': '0'} 2022-09-03T07:31:01.955 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003929460d24c7e655-00631302b5-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.956 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:01.957 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.957 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.958 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.958 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:31:01.959 DEBUG:boto:Data: 2022-09-03T07:31:01.959 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.960 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.961 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.961 DEBUG:boto:Params: {} 2022-09-03T07:31:01.962 DEBUG:boto:Token: None 2022-09-03T07:31:01.962 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj87 2022-09-03T07:31:01.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YO3GLD/2M4m1U2scXuanTgX1nr4= 2022-09-03T07:31:01.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YO3GLD/2M4m1U2scXuanTgX1nr4=', 'Content-Length': '0'} 2022-09-03T07:31:01.966 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d571909ccfd78fcf-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.967 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.968 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.968 DEBUG:boto:Method: HEAD 2022-09-03T07:31:01.969 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:31:01.969 DEBUG:boto:Data: 2022-09-03T07:31:01.970 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.971 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:01.972 DEBUG:boto:Port: 8000 2022-09-03T07:31:01.972 DEBUG:boto:Params: {} 2022-09-03T07:31:01.973 DEBUG:boto:Token: None 2022-09-03T07:31:01.974 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj87 2022-09-03T07:31:01.975 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YO3GLD/2M4m1U2scXuanTgX1nr4= 2022-09-03T07:31:01.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YO3GLD/2M4m1U2scXuanTgX1nr4=', 'Content-Length': '0'} 2022-09-03T07:31:01.979 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d8fe2a31c9133785-00631302b5-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.981 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:31:01.982 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.983 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.984 DEBUG:boto:Method: GET 2022-09-03T07:31:01.985 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:31:01.986 DEBUG:boto:Data: 2022-09-03T07:31:01.987 DEBUG:boto:Headers: {} 2022-09-03T07:31:01.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:01.988 DEBUG:boto:Port: 8001 2022-09-03T07:31:01.989 DEBUG:boto:Params: {} 2022-09-03T07:31:01.991 DEBUG:boto:Token: None 2022-09-03T07:31:01.992 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:01 GMT /avysmw-75/obj87 2022-09-03T07:31:01.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FPpXMAQMvCQ0zHrGN4Unt4JHKL8= 2022-09-03T07:31:01.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:01 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FPpXMAQMvCQ0zHrGN4Unt4JHKL8=', 'Content-Length': '0'} 2022-09-03T07:31:01.997 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c2e3a430d2de164-00631302b5-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:01 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:01.998 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:01.998 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:01.999 DEBUG:boto:Method: GET 2022-09-03T07:31:01.999 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:31:02.000 DEBUG:boto:Data: 2022-09-03T07:31:02.001 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.001 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.002 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.002 DEBUG:boto:Params: {} 2022-09-03T07:31:02.003 DEBUG:boto:Token: None 2022-09-03T07:31:02.004 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj87 2022-09-03T07:31:02.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qa+ikaf0wUDLI2HAYmpcjp/wtPY= 2022-09-03T07:31:02.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qa+ikaf0wUDLI2HAYmpcjp/wtPY=', 'Content-Length': '0'} 2022-09-03T07:31:02.008 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062aaf51400d5c13b-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.008 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.009 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.010 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:02.010 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:02.011 DEBUG:boto:Method: GET 2022-09-03T07:31:02.011 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:31:02.012 DEBUG:boto:Data: 2022-09-03T07:31:02.012 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.013 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.014 DEBUG:boto:Params: {} 2022-09-03T07:31:02.015 DEBUG:boto:Token: None 2022-09-03T07:31:02.015 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.016 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.019 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000028d80d575d484efc-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.020 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.021 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.022 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.023 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:02.023 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:02.024 DEBUG:boto:Method: GET 2022-09-03T07:31:02.024 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:31:02.025 DEBUG:boto:Data: 2022-09-03T07:31:02.025 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.026 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.026 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.027 DEBUG:boto:Params: {} 2022-09-03T07:31:02.028 DEBUG:boto:Token: None 2022-09-03T07:31:02.028 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.029 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.034 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f0b978a5fe5d9b0a-00631302b6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.035 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.036 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:31:02.037 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.038 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.038 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.039 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.039 DEBUG:boto:Method: GET 2022-09-03T07:31:02.040 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.041 DEBUG:boto:Data: 2022-09-03T07:31:02.041 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.042 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.050 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.052 DEBUG:boto:Params: {} 2022-09-03T07:31:02.053 DEBUG:boto:Token: None 2022-09-03T07:31:02.053 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.054 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UXBw7//SYbj5nCVP8EytxUG2ICA= 2022-09-03T07:31:02.055 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UXBw7//SYbj5nCVP8EytxUG2ICA=', 'Content-Length': '0'} 2022-09-03T07:31:02.057 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044660877fbd52771-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.058 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.058 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.059 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.059 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.060 DEBUG:boto:Method: GET 2022-09-03T07:31:02.061 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.061 DEBUG:boto:Data: 2022-09-03T07:31:02.062 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.062 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.063 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.063 DEBUG:boto:Params: {} 2022-09-03T07:31:02.064 DEBUG:boto:Token: None 2022-09-03T07:31:02.064 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UXBw7//SYbj5nCVP8EytxUG2ICA= 2022-09-03T07:31:02.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UXBw7//SYbj5nCVP8EytxUG2ICA=', 'Content-Length': '0'} 2022-09-03T07:31:02.073 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0a9e308204c0e6c-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.114 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.115 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.116 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.117 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.119 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.120 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.121 DEBUG:boto:Data: 2022-09-03T07:31:02.122 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.123 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.123 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.124 DEBUG:boto:Params: {} 2022-09-03T07:31:02.125 DEBUG:boto:Token: None 2022-09-03T07:31:02.126 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.126 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tw4lS1gh5rBXrDcOmx8tLXmyQ48= 2022-09-03T07:31:02.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tw4lS1gh5rBXrDcOmx8tLXmyQ48=', 'Content-Length': '0'} 2022-09-03T07:31:02.129 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024cb69beb9e9b59a-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.131 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.131 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.132 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.134 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.134 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.135 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.136 DEBUG:boto:Data: 2022-09-03T07:31:02.138 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.139 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.140 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.141 DEBUG:boto:Params: {} 2022-09-03T07:31:02.142 DEBUG:boto:Token: None 2022-09-03T07:31:02.143 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.143 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tw4lS1gh5rBXrDcOmx8tLXmyQ48= 2022-09-03T07:31:02.144 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tw4lS1gh5rBXrDcOmx8tLXmyQ48=', 'Content-Length': '0'} 2022-09-03T07:31:02.146 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000672d75fd16864435-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.149 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:31:02.151 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.152 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.152 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.153 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.153 DEBUG:boto:Method: GET 2022-09-03T07:31:02.154 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.155 DEBUG:boto:Data: 2022-09-03T07:31:02.156 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.157 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.157 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.159 DEBUG:boto:Params: {} 2022-09-03T07:31:02.160 DEBUG:boto:Token: None 2022-09-03T07:31:02.162 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UXBw7//SYbj5nCVP8EytxUG2ICA= 2022-09-03T07:31:02.165 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UXBw7//SYbj5nCVP8EytxUG2ICA=', 'Content-Length': '0'} 2022-09-03T07:31:02.167 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bed6f465293566c8-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.169 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.170 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.171 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.172 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.173 DEBUG:boto:Method: GET 2022-09-03T07:31:02.174 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.175 DEBUG:boto:Data: 2022-09-03T07:31:02.177 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.178 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.181 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.181 DEBUG:boto:Params: {} 2022-09-03T07:31:02.182 DEBUG:boto:Token: None 2022-09-03T07:31:02.183 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:02.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UXBw7//SYbj5nCVP8EytxUG2ICA= 2022-09-03T07:31:02.185 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UXBw7//SYbj5nCVP8EytxUG2ICA=', 'Content-Length': '0'} 2022-09-03T07:31:02.191 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d45c6dbcaeec154-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.192 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.192 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.193 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:02.193 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:02.194 DEBUG:boto:Method: GET 2022-09-03T07:31:02.195 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:31:02.195 DEBUG:boto:Data: 2022-09-03T07:31:02.196 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.196 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.197 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.197 DEBUG:boto:Params: {} 2022-09-03T07:31:02.198 DEBUG:boto:Token: None 2022-09-03T07:31:02.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.203 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d706c1351cd5655e-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.204 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.205 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.206 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.207 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:02.208 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:02.209 DEBUG:boto:Method: GET 2022-09-03T07:31:02.210 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:31:02.211 DEBUG:boto:Data: 2022-09-03T07:31:02.212 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.213 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.214 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.215 DEBUG:boto:Params: {} 2022-09-03T07:31:02.216 DEBUG:boto:Token: None 2022-09-03T07:31:02.217 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.218 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.219 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.223 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000004171491fe32ee0c-00631302b6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.270 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.271 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.272 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.272 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.273 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:31:02.273 DEBUG:boto:Data: 2022-09-03T07:31:02.274 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.275 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.276 DEBUG:boto:Params: {} 2022-09-03T07:31:02.277 DEBUG:boto:Token: None 2022-09-03T07:31:02.277 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92 2022-09-03T07:31:02.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:innri8qbPo89A9qrM2oHXCwkm1M= 2022-09-03T07:31:02.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:innri8qbPo89A9qrM2oHXCwkm1M=', 'Content-Length': '0'} 2022-09-03T07:31:02.281 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e25ad72c6a617108-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.282 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.283 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.283 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.284 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:31:02.284 DEBUG:boto:Data: 2022-09-03T07:31:02.285 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.286 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.286 DEBUG:boto:Params: {} 2022-09-03T07:31:02.287 DEBUG:boto:Token: None 2022-09-03T07:31:02.287 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92 2022-09-03T07:31:02.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:innri8qbPo89A9qrM2oHXCwkm1M= 2022-09-03T07:31:02.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:innri8qbPo89A9qrM2oHXCwkm1M=', 'Content-Length': '0'} 2022-09-03T07:31:02.292 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b41097708a5f52f-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.293 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:31:02.293 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.294 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.295 DEBUG:boto:Method: GET 2022-09-03T07:31:02.295 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:31:02.296 DEBUG:boto:Data: 2022-09-03T07:31:02.296 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.297 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.297 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.298 DEBUG:boto:Params: {} 2022-09-03T07:31:02.298 DEBUG:boto:Token: None 2022-09-03T07:31:02.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92 2022-09-03T07:31:02.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3w7b3msthM4vpsQA4e2mNhKxOk= 2022-09-03T07:31:02.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3w7b3msthM4vpsQA4e2mNhKxOk=', 'Content-Length': '0'} 2022-09-03T07:31:02.303 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001fa0b9c5e3866753-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.304 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:02.304 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:02.305 DEBUG:boto:Method: GET 2022-09-03T07:31:02.307 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:31:02.307 DEBUG:boto:Data: 2022-09-03T07:31:02.308 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.308 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.309 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.310 DEBUG:boto:Params: {} 2022-09-03T07:31:02.310 DEBUG:boto:Token: None 2022-09-03T07:31:02.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj92 2022-09-03T07:31:02.311 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:I3w7b3msthM4vpsQA4e2mNhKxOk= 2022-09-03T07:31:02.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:I3w7b3msthM4vpsQA4e2mNhKxOk=', 'Content-Length': '0'} 2022-09-03T07:31:02.316 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aaf74916f5e4a234-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.317 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.317 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.318 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:02.319 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:02.319 DEBUG:boto:Method: GET 2022-09-03T07:31:02.320 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:31:02.320 DEBUG:boto:Data: 2022-09-03T07:31:02.321 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.321 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.322 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.322 DEBUG:boto:Params: {} 2022-09-03T07:31:02.323 DEBUG:boto:Token: None 2022-09-03T07:31:02.323 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.324 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.325 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.327 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c18539769ced96be-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.370 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.372 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.373 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.374 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:02.374 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:02.375 DEBUG:boto:Method: GET 2022-09-03T07:31:02.375 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:31:02.376 DEBUG:boto:Data: 2022-09-03T07:31:02.376 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.377 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.378 DEBUG:boto:Params: {} 2022-09-03T07:31:02.380 DEBUG:boto:Token: None 2022-09-03T07:31:02.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.383 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.388 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002dbdd5e3dd54321f-00631302b6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.388 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.389 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:31:02.390 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.391 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.391 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.392 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.392 DEBUG:boto:Method: GET 2022-09-03T07:31:02.393 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.394 DEBUG:boto:Data: 2022-09-03T07:31:02.395 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.396 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.397 DEBUG:boto:Params: {} 2022-09-03T07:31:02.397 DEBUG:boto:Token: None 2022-09-03T07:31:02.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wgVqEQJSKoe/EJmkFBTyFcEF7Xs= 2022-09-03T07:31:02.400 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wgVqEQJSKoe/EJmkFBTyFcEF7Xs=', 'Content-Length': '0'} 2022-09-03T07:31:02.402 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000076d3de14c3e4b333-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.403 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.403 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.404 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.404 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.405 DEBUG:boto:Method: GET 2022-09-03T07:31:02.406 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.406 DEBUG:boto:Data: 2022-09-03T07:31:02.407 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.407 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.408 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.408 DEBUG:boto:Params: {} 2022-09-03T07:31:02.409 DEBUG:boto:Token: None 2022-09-03T07:31:02.409 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.410 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wgVqEQJSKoe/EJmkFBTyFcEF7Xs= 2022-09-03T07:31:02.411 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wgVqEQJSKoe/EJmkFBTyFcEF7Xs=', 'Content-Length': '0'} 2022-09-03T07:31:02.413 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb59323bac80bb3d-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.414 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.415 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.415 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.416 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.417 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.417 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.418 DEBUG:boto:Data: 2022-09-03T07:31:02.418 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.419 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.420 DEBUG:boto:Params: {} 2022-09-03T07:31:02.420 DEBUG:boto:Token: None 2022-09-03T07:31:02.421 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.422 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5nPXh1Wf2uuS0bbiDT/vgecfXEE= 2022-09-03T07:31:02.422 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5nPXh1Wf2uuS0bbiDT/vgecfXEE=', 'Content-Length': '0'} 2022-09-03T07:31:02.425 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000592f3ed474d04900-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.426 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.426 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.427 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.427 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.428 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.429 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.430 DEBUG:boto:Data: 2022-09-03T07:31:02.431 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.433 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.433 DEBUG:boto:Params: {} 2022-09-03T07:31:02.434 DEBUG:boto:Token: None 2022-09-03T07:31:02.436 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.438 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5nPXh1Wf2uuS0bbiDT/vgecfXEE= 2022-09-03T07:31:02.438 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5nPXh1Wf2uuS0bbiDT/vgecfXEE=', 'Content-Length': '0'} 2022-09-03T07:31:02.440 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001efd9fa4be033e6-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.441 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:31:02.442 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.442 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.443 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.443 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.444 DEBUG:boto:Method: GET 2022-09-03T07:31:02.445 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.445 DEBUG:boto:Data: 2022-09-03T07:31:02.446 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.446 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.447 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.447 DEBUG:boto:Params: {} 2022-09-03T07:31:02.448 DEBUG:boto:Token: None 2022-09-03T07:31:02.449 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.449 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wgVqEQJSKoe/EJmkFBTyFcEF7Xs= 2022-09-03T07:31:02.450 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wgVqEQJSKoe/EJmkFBTyFcEF7Xs=', 'Content-Length': '0'} 2022-09-03T07:31:02.452 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000753ed8b50d1103d3-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.453 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.454 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.455 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.456 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.460 DEBUG:boto:Method: GET 2022-09-03T07:31:02.461 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.462 DEBUG:boto:Data: 2022-09-03T07:31:02.463 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.464 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.465 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.466 DEBUG:boto:Params: {} 2022-09-03T07:31:02.468 DEBUG:boto:Token: None 2022-09-03T07:31:02.469 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:02.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wgVqEQJSKoe/EJmkFBTyFcEF7Xs= 2022-09-03T07:31:02.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wgVqEQJSKoe/EJmkFBTyFcEF7Xs=', 'Content-Length': '0'} 2022-09-03T07:31:02.475 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ecf5c628c375c666-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.476 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.477 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.478 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:02.479 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:02.480 DEBUG:boto:Method: GET 2022-09-03T07:31:02.481 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:31:02.482 DEBUG:boto:Data: 2022-09-03T07:31:02.483 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.484 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.485 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.486 DEBUG:boto:Params: {} 2022-09-03T07:31:02.487 DEBUG:boto:Token: None 2022-09-03T07:31:02.488 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.491 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.494 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc74da3cc57821b9-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.495 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.497 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.498 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.499 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:02.500 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:02.501 DEBUG:boto:Method: GET 2022-09-03T07:31:02.502 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:31:02.503 DEBUG:boto:Data: 2022-09-03T07:31:02.504 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.506 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.507 DEBUG:boto:Params: {} 2022-09-03T07:31:02.508 DEBUG:boto:Token: None 2022-09-03T07:31:02.509 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.511 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.516 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000123125efe89f1baf-00631302b6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.563 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.567 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.570 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.572 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.573 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:31:02.574 DEBUG:boto:Data: 2022-09-03T07:31:02.575 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.576 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.577 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.578 DEBUG:boto:Params: {} 2022-09-03T07:31:02.579 DEBUG:boto:Token: None 2022-09-03T07:31:02.581 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93 2022-09-03T07:31:02.582 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VsltOC0bHkd1Rjkr/5SGNIcOASI= 2022-09-03T07:31:02.583 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VsltOC0bHkd1Rjkr/5SGNIcOASI=', 'Content-Length': '0'} 2022-09-03T07:31:02.586 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000897e6d13f05f1c5b-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.588 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.589 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.590 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.591 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:31:02.592 DEBUG:boto:Data: 2022-09-03T07:31:02.593 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.595 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.596 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.597 DEBUG:boto:Params: {} 2022-09-03T07:31:02.598 DEBUG:boto:Token: None 2022-09-03T07:31:02.599 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93 2022-09-03T07:31:02.600 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:VsltOC0bHkd1Rjkr/5SGNIcOASI= 2022-09-03T07:31:02.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:VsltOC0bHkd1Rjkr/5SGNIcOASI=', 'Content-Length': '0'} 2022-09-03T07:31:02.604 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066d874edfe694888-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.605 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:31:02.606 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.608 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.609 DEBUG:boto:Method: GET 2022-09-03T07:31:02.615 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:31:02.615 DEBUG:boto:Data: 2022-09-03T07:31:02.616 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.616 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.617 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.617 DEBUG:boto:Params: {} 2022-09-03T07:31:02.618 DEBUG:boto:Token: None 2022-09-03T07:31:02.619 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93 2022-09-03T07:31:02.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SzIxzK3iuxFfkDSB5Tdzji6LCHo= 2022-09-03T07:31:02.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SzIxzK3iuxFfkDSB5Tdzji6LCHo=', 'Content-Length': '0'} 2022-09-03T07:31:02.622 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d94fe978b5b28ff-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.623 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:02.624 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:02.624 DEBUG:boto:Method: GET 2022-09-03T07:31:02.625 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:31:02.625 DEBUG:boto:Data: 2022-09-03T07:31:02.626 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.627 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.628 DEBUG:boto:Params: {} 2022-09-03T07:31:02.629 DEBUG:boto:Token: None 2022-09-03T07:31:02.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj93 2022-09-03T07:31:02.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SzIxzK3iuxFfkDSB5Tdzji6LCHo= 2022-09-03T07:31:02.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SzIxzK3iuxFfkDSB5Tdzji6LCHo=', 'Content-Length': '0'} 2022-09-03T07:31:02.634 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fbf9c6012689d588-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.635 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.636 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.636 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:02.637 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:02.637 DEBUG:boto:Method: GET 2022-09-03T07:31:02.638 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:31:02.639 DEBUG:boto:Data: 2022-09-03T07:31:02.639 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.641 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.642 DEBUG:boto:Params: {} 2022-09-03T07:31:02.642 DEBUG:boto:Token: None 2022-09-03T07:31:02.643 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.644 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.647 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000aebad7d41e769fb-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.695 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.697 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.699 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.700 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:02.701 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:02.702 DEBUG:boto:Method: GET 2022-09-03T07:31:02.703 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:31:02.704 DEBUG:boto:Data: 2022-09-03T07:31:02.705 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.707 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.709 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.710 DEBUG:boto:Params: {} 2022-09-03T07:31:02.712 DEBUG:boto:Token: None 2022-09-03T07:31:02.713 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.714 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.715 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.722 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc5dfa7b0a5b2d8d-00631302b6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.723 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.724 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:31:02.725 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.726 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.727 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.728 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.729 DEBUG:boto:Method: GET 2022-09-03T07:31:02.730 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.731 DEBUG:boto:Data: 2022-09-03T07:31:02.732 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.733 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.735 DEBUG:boto:Params: {} 2022-09-03T07:31:02.736 DEBUG:boto:Token: None 2022-09-03T07:31:02.736 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yaj4MqV4OgV0GyI+zH6wnaXVG7s= 2022-09-03T07:31:02.738 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yaj4MqV4OgV0GyI+zH6wnaXVG7s=', 'Content-Length': '0'} 2022-09-03T07:31:02.740 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b477c7ccf5e06e8f-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.741 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.743 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.744 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.745 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.747 DEBUG:boto:Method: GET 2022-09-03T07:31:02.748 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.749 DEBUG:boto:Data: 2022-09-03T07:31:02.749 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.750 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.751 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.752 DEBUG:boto:Params: {} 2022-09-03T07:31:02.753 DEBUG:boto:Token: None 2022-09-03T07:31:02.753 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yaj4MqV4OgV0GyI+zH6wnaXVG7s= 2022-09-03T07:31:02.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yaj4MqV4OgV0GyI+zH6wnaXVG7s=', 'Content-Length': '0'} 2022-09-03T07:31:02.764 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000134f3c0c20a1d4ec-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.766 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.766 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.767 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.767 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.768 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.768 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.769 DEBUG:boto:Data: 2022-09-03T07:31:02.769 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.770 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.770 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.771 DEBUG:boto:Params: {} 2022-09-03T07:31:02.772 DEBUG:boto:Token: None 2022-09-03T07:31:02.772 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BDiHw8DIsEHvrdstdJPwFXOAqGQ= 2022-09-03T07:31:02.773 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BDiHw8DIsEHvrdstdJPwFXOAqGQ=', 'Content-Length': '0'} 2022-09-03T07:31:02.775 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004ecdbe6d0fec29ef-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.776 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.776 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.777 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.777 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.778 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.779 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.779 DEBUG:boto:Data: 2022-09-03T07:31:02.780 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.781 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.781 DEBUG:boto:Params: {} 2022-09-03T07:31:02.783 DEBUG:boto:Token: None 2022-09-03T07:31:02.784 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BDiHw8DIsEHvrdstdJPwFXOAqGQ= 2022-09-03T07:31:02.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BDiHw8DIsEHvrdstdJPwFXOAqGQ=', 'Content-Length': '0'} 2022-09-03T07:31:02.787 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b7305f43b4be1ec0-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.788 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:31:02.789 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.789 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.790 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.790 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.791 DEBUG:boto:Method: GET 2022-09-03T07:31:02.793 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.794 DEBUG:boto:Data: 2022-09-03T07:31:02.795 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.796 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.797 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.798 DEBUG:boto:Params: {} 2022-09-03T07:31:02.798 DEBUG:boto:Token: None 2022-09-03T07:31:02.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.802 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yaj4MqV4OgV0GyI+zH6wnaXVG7s= 2022-09-03T07:31:02.803 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yaj4MqV4OgV0GyI+zH6wnaXVG7s=', 'Content-Length': '0'} 2022-09-03T07:31:02.809 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000915f908b6c998b7b-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.811 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.812 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.813 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.814 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.815 DEBUG:boto:Method: GET 2022-09-03T07:31:02.817 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.819 DEBUG:boto:Data: 2022-09-03T07:31:02.819 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.820 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.821 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.821 DEBUG:boto:Params: {} 2022-09-03T07:31:02.822 DEBUG:boto:Token: None 2022-09-03T07:31:02.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:02.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yaj4MqV4OgV0GyI+zH6wnaXVG7s= 2022-09-03T07:31:02.824 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yaj4MqV4OgV0GyI+zH6wnaXVG7s=', 'Content-Length': '0'} 2022-09-03T07:31:02.832 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0a0d5505ba89d22-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.833 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.833 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.834 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:02.835 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:02.836 DEBUG:boto:Method: GET 2022-09-03T07:31:02.837 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:31:02.840 DEBUG:boto:Data: 2022-09-03T07:31:02.840 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.842 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.843 DEBUG:boto:Params: {} 2022-09-03T07:31:02.844 DEBUG:boto:Token: None 2022-09-03T07:31:02.844 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.845 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.846 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.849 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000454dabf64773e4ca-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.850 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.851 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.852 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.852 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:02.853 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:02.853 DEBUG:boto:Method: GET 2022-09-03T07:31:02.854 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:31:02.854 DEBUG:boto:Data: 2022-09-03T07:31:02.855 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.856 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.856 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.857 DEBUG:boto:Params: {} 2022-09-03T07:31:02.857 DEBUG:boto:Token: None 2022-09-03T07:31:02.858 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.863 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000adf6bb6f8b414b3-00631302b6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.906 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.908 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.908 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.909 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.910 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:31:02.910 DEBUG:boto:Data: 2022-09-03T07:31:02.911 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.911 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.912 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.912 DEBUG:boto:Params: {} 2022-09-03T07:31:02.913 DEBUG:boto:Token: None 2022-09-03T07:31:02.914 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95 2022-09-03T07:31:02.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HtMH8ZaA+YXucDY6NqsfWNHp4rg= 2022-09-03T07:31:02.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HtMH8ZaA+YXucDY6NqsfWNHp4rg=', 'Content-Length': '0'} 2022-09-03T07:31:02.917 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b0de03643bebe983-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.918 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.919 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.919 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.920 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:31:02.920 DEBUG:boto:Data: 2022-09-03T07:31:02.921 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.921 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.922 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.922 DEBUG:boto:Params: {} 2022-09-03T07:31:02.923 DEBUG:boto:Token: None 2022-09-03T07:31:02.924 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95 2022-09-03T07:31:02.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HtMH8ZaA+YXucDY6NqsfWNHp4rg= 2022-09-03T07:31:02.925 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HtMH8ZaA+YXucDY6NqsfWNHp4rg=', 'Content-Length': '0'} 2022-09-03T07:31:02.927 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071f3a3669d0c6c41-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.928 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:31:02.929 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.929 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.930 DEBUG:boto:Method: GET 2022-09-03T07:31:02.930 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:31:02.931 DEBUG:boto:Data: 2022-09-03T07:31:02.932 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.933 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.934 DEBUG:boto:Params: {} 2022-09-03T07:31:02.934 DEBUG:boto:Token: None 2022-09-03T07:31:02.935 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95 2022-09-03T07:31:02.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ElUyn2tTYYnQXBf8jma4I2HF+wI= 2022-09-03T07:31:02.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ElUyn2tTYYnQXBf8jma4I2HF+wI=', 'Content-Length': '0'} 2022-09-03T07:31:02.939 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f215a69a0e86c0f7-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.943 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:02.944 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:02.945 DEBUG:boto:Method: GET 2022-09-03T07:31:02.945 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:31:02.946 DEBUG:boto:Data: 2022-09-03T07:31:02.947 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.949 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.949 DEBUG:boto:Params: {} 2022-09-03T07:31:02.950 DEBUG:boto:Token: None 2022-09-03T07:31:02.951 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/obj95 2022-09-03T07:31:02.953 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ElUyn2tTYYnQXBf8jma4I2HF+wI= 2022-09-03T07:31:02.954 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ElUyn2tTYYnQXBf8jma4I2HF+wI=', 'Content-Length': '0'} 2022-09-03T07:31:02.956 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d48c536f3f5efa6-00631302b6-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.958 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.959 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.960 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:02.961 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:02.962 DEBUG:boto:Method: GET 2022-09-03T07:31:02.963 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:31:02.964 DEBUG:boto:Data: 2022-09-03T07:31:02.964 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.965 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.966 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.966 DEBUG:boto:Params: {} 2022-09-03T07:31:02.967 DEBUG:boto:Token: None 2022-09-03T07:31:02.968 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.972 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006735ca41dc0dad8a-00631302b6-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.973 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.974 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.975 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.975 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:02.976 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:02.976 DEBUG:boto:Method: GET 2022-09-03T07:31:02.977 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:31:02.977 DEBUG:boto:Data: 2022-09-03T07:31:02.978 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.978 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:31:02.979 DEBUG:boto:Port: 8000 2022-09-03T07:31:02.979 DEBUG:boto:Params: {} 2022-09-03T07:31:02.980 DEBUG:boto:Token: None 2022-09-03T07:31:02.981 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.981 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc= 2022-09-03T07:31:02.982 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zu9qjNkRx0DIeAhgKirapGWaYKc=', 'Content-Length': '0'} 2022-09-03T07:31:02.986 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007419d3a146c49084-00631302b6-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:02.987 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:02.988 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone1} 2022-09-03T07:31:02.989 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:31:02.990 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:02.991 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:02.992 DEBUG:boto:Method: HEAD 2022-09-03T07:31:02.992 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:31:02.993 DEBUG:boto:Data: 2022-09-03T07:31:02.994 DEBUG:boto:Headers: {} 2022-09-03T07:31:02.994 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:02.995 DEBUG:boto:Port: 8001 2022-09-03T07:31:02.996 DEBUG:boto:Params: {} 2022-09-03T07:31:02.997 DEBUG:boto:Token: None 2022-09-03T07:31:02.998 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:02 GMT /avysmw-75/ 2022-09-03T07:31:02.999 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LEJCh8nIuhc1ASCuYod6fBOchIo= 2022-09-03T07:31:03.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LEJCh8nIuhc1ASCuYod6fBOchIo=', 'Content-Length': '0'} 2022-09-03T07:31:03.006 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '35'), ('X-RGW-Bytes-Used', '105'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000009b71b2801d37c5a2-00631302b7-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.007 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.008 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.008 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.009 DEBUG:boto:Path: /avysmw-75/ 2022-09-03T07:31:03.009 DEBUG:boto:Data: 2022-09-03T07:31:03.012 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.013 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.022 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.023 DEBUG:boto:Params: {} 2022-09-03T07:31:03.024 DEBUG:boto:Token: None 2022-09-03T07:31:03.024 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.025 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lYMSplkpT28Z0izbSESb82Jx1gM= 2022-09-03T07:31:03.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lYMSplkpT28Z0izbSESb82Jx1gM=', 'Content-Length': '0'} 2022-09-03T07:31:03.028 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '35'), ('X-RGW-Bytes-Used', '105'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000a5d32882fcc7f97-00631302b7-18f5-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.029 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:31:03.029 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.030 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.030 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.031 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.031 DEBUG:boto:Method: GET 2022-09-03T07:31:03.032 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.033 DEBUG:boto:Data: 2022-09-03T07:31:03.034 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.034 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.035 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.035 DEBUG:boto:Params: {} 2022-09-03T07:31:03.036 DEBUG:boto:Token: None 2022-09-03T07:31:03.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/?versions 2022-09-03T07:31:03.037 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f7Re/Znl2ovQbAe/s/Bn8tSD5g8= 2022-09-03T07:31:03.038 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f7Re/Znl2ovQbAe/s/Bn8tSD5g8=', 'Content-Length': '0'} 2022-09-03T07:31:03.041 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000037b7160ccb35c334-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.041 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.044 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:31:03.044 DEBUG:tasks.rgw_multi.zone_rados:o=obj111 2022-09-03T07:31:03.045 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:31:03.045 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:31:03.046 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:31:03.046 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:31:03.047 DEBUG:tasks.rgw_multi.zone_rados:o=obj148 2022-09-03T07:31:03.047 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:31:03.048 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:31:03.048 DEBUG:tasks.rgw_multi.zone_rados:o=obj185 2022-09-03T07:31:03.049 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:31:03.050 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:31:03.050 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:31:03.051 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:31:03.051 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:31:03.052 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:31:03.052 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:31:03.053 DEBUG:tasks.rgw_multi.zone_rados:o=obj37 2022-09-03T07:31:03.053 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:31:03.054 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:31:03.054 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:31:03.055 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:31:03.055 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:31:03.056 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:31:03.056 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:31:03.057 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:31:03.057 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:31:03.058 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:31:03.059 DEBUG:tasks.rgw_multi.zone_rados:o=obj74 2022-09-03T07:31:03.059 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:31:03.060 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:31:03.060 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:31:03.061 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:31:03.061 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:31:03.062 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:31:03.063 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:31:03.063 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.064 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.064 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.065 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.066 DEBUG:boto:Method: GET 2022-09-03T07:31:03.066 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.067 DEBUG:boto:Data: 2022-09-03T07:31:03.068 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.068 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.069 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.070 DEBUG:boto:Params: {} 2022-09-03T07:31:03.070 DEBUG:boto:Token: None 2022-09-03T07:31:03.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/?versions 2022-09-03T07:31:03.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f7Re/Znl2ovQbAe/s/Bn8tSD5g8= 2022-09-03T07:31:03.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f7Re/Znl2ovQbAe/s/Bn8tSD5g8=', 'Content-Length': '0'} 2022-09-03T07:31:03.075 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000033bcf0e6a3e44e7e-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.082 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.084 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:31:03.085 DEBUG:tasks.rgw_multi.zone_rados:o=obj111 2022-09-03T07:31:03.085 DEBUG:tasks.rgw_multi.zone_rados:o=obj115 2022-09-03T07:31:03.086 DEBUG:tasks.rgw_multi.zone_rados:o=obj116 2022-09-03T07:31:03.086 DEBUG:tasks.rgw_multi.zone_rados:o=obj124 2022-09-03T07:31:03.087 DEBUG:tasks.rgw_multi.zone_rados:o=obj145 2022-09-03T07:31:03.087 DEBUG:tasks.rgw_multi.zone_rados:o=obj148 2022-09-03T07:31:03.088 DEBUG:tasks.rgw_multi.zone_rados:o=obj155 2022-09-03T07:31:03.088 DEBUG:tasks.rgw_multi.zone_rados:o=obj17 2022-09-03T07:31:03.089 DEBUG:tasks.rgw_multi.zone_rados:o=obj185 2022-09-03T07:31:03.089 DEBUG:tasks.rgw_multi.zone_rados:o=obj19 2022-09-03T07:31:03.090 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:31:03.090 DEBUG:tasks.rgw_multi.zone_rados:o=obj23 2022-09-03T07:31:03.091 DEBUG:tasks.rgw_multi.zone_rados:o=obj29 2022-09-03T07:31:03.091 DEBUG:tasks.rgw_multi.zone_rados:o=obj31 2022-09-03T07:31:03.092 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:31:03.092 DEBUG:tasks.rgw_multi.zone_rados:o=obj34 2022-09-03T07:31:03.092 DEBUG:tasks.rgw_multi.zone_rados:o=obj37 2022-09-03T07:31:03.093 DEBUG:tasks.rgw_multi.zone_rados:o=obj38 2022-09-03T07:31:03.093 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:31:03.095 DEBUG:tasks.rgw_multi.zone_rados:o=obj46 2022-09-03T07:31:03.096 DEBUG:tasks.rgw_multi.zone_rados:o=obj51 2022-09-03T07:31:03.096 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:31:03.097 DEBUG:tasks.rgw_multi.zone_rados:o=obj57 2022-09-03T07:31:03.097 DEBUG:tasks.rgw_multi.zone_rados:o=obj58 2022-09-03T07:31:03.098 DEBUG:tasks.rgw_multi.zone_rados:o=obj62 2022-09-03T07:31:03.098 DEBUG:tasks.rgw_multi.zone_rados:o=obj68 2022-09-03T07:31:03.099 DEBUG:tasks.rgw_multi.zone_rados:o=obj69 2022-09-03T07:31:03.099 DEBUG:tasks.rgw_multi.zone_rados:o=obj74 2022-09-03T07:31:03.100 DEBUG:tasks.rgw_multi.zone_rados:o=obj76 2022-09-03T07:31:03.100 DEBUG:tasks.rgw_multi.zone_rados:o=obj85 2022-09-03T07:31:03.101 DEBUG:tasks.rgw_multi.zone_rados:o=obj87 2022-09-03T07:31:03.101 DEBUG:tasks.rgw_multi.zone_rados:o=obj92 2022-09-03T07:31:03.102 DEBUG:tasks.rgw_multi.zone_rados:o=obj93 2022-09-03T07:31:03.103 DEBUG:tasks.rgw_multi.zone_rados:o=obj95 2022-09-03T07:31:03.103 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.104 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.105 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.105 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.106 DEBUG:boto:Method: GET 2022-09-03T07:31:03.106 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.107 DEBUG:boto:Data: 2022-09-03T07:31:03.107 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.108 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.108 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.109 DEBUG:boto:Params: {} 2022-09-03T07:31:03.109 DEBUG:boto:Token: None 2022-09-03T07:31:03.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/?versions 2022-09-03T07:31:03.111 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f7Re/Znl2ovQbAe/s/Bn8tSD5g8= 2022-09-03T07:31:03.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f7Re/Znl2ovQbAe/s/Bn8tSD5g8=', 'Content-Length': '0'} 2022-09-03T07:31:03.114 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003b5f60ddf8245a5e-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.115 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.119 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.119 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.120 DEBUG:boto:path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.121 DEBUG:boto:auth_path=/avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.122 DEBUG:boto:Method: GET 2022-09-03T07:31:03.124 DEBUG:boto:Path: /avysmw-75/?versions&max-keys=999 2022-09-03T07:31:03.125 DEBUG:boto:Data: 2022-09-03T07:31:03.126 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.127 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.128 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.129 DEBUG:boto:Params: {} 2022-09-03T07:31:03.130 DEBUG:boto:Token: None 2022-09-03T07:31:03.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/?versions 2022-09-03T07:31:03.133 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f7Re/Znl2ovQbAe/s/Bn8tSD5g8= 2022-09-03T07:31:03.134 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f7Re/Znl2ovQbAe/s/Bn8tSD5g8=', 'Content-Length': '0'} 2022-09-03T07:31:03.139 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d3ba6e7d9a95379-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.140 DEBUG:boto:b'avysmw-75999falseobj11nulltrue2022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj111nulltrue2022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj115nulltrue2022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj116nulltrue2022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj124nulltrue2022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj145nulltrue2022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj148nulltrue2022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj155nulltrue2022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj17nulltrue2022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj185nulltrue2022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj19nulltrue2022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj23nulltrue2022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj29nulltrue2022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj31nulltrue2022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj34nulltrue2022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj37nulltrue2022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj38nulltrue2022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj46nulltrue2022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj51nulltrue2022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj57nulltrue2022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj58nulltrue2022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj62nulltrue2022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj68nulltrue2022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj69nulltrue2022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj74nulltrue2022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj76nulltrue2022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj85nulltrue2022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj87nulltrue2022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj92nulltrue2022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj93nulltrue2022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj95nulltrue2022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.143 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:31:03.143 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.144 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.144 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.145 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.146 DEBUG:boto:Method: GET 2022-09-03T07:31:03.147 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.148 DEBUG:boto:Data: 2022-09-03T07:31:03.148 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.149 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.149 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.150 DEBUG:boto:Params: {} 2022-09-03T07:31:03.151 DEBUG:boto:Token: None 2022-09-03T07:31:03.151 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.153 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPi/k/ptlLJ4de+RZOzg4Zip8C4= 2022-09-03T07:31:03.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPi/k/ptlLJ4de+RZOzg4Zip8C4=', 'Content-Length': '0'} 2022-09-03T07:31:03.156 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000022363c4324845ac1-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.157 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.157 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.158 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.159 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.160 DEBUG:boto:Method: GET 2022-09-03T07:31:03.161 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.162 DEBUG:boto:Data: 2022-09-03T07:31:03.163 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.163 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.164 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.165 DEBUG:boto:Params: {} 2022-09-03T07:31:03.165 DEBUG:boto:Token: None 2022-09-03T07:31:03.166 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPi/k/ptlLJ4de+RZOzg4Zip8C4= 2022-09-03T07:31:03.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPi/k/ptlLJ4de+RZOzg4Zip8C4=', 'Content-Length': '0'} 2022-09-03T07:31:03.169 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000095f69aa60a6e0b59-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.211 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.212 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.213 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.214 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.214 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.215 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.216 DEBUG:boto:Data: 2022-09-03T07:31:03.217 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.218 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.218 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.220 DEBUG:boto:Params: {} 2022-09-03T07:31:03.221 DEBUG:boto:Token: None 2022-09-03T07:31:03.222 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.224 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l3wCTjNZHJuL1SrFtoKSzOHymF4= 2022-09-03T07:31:03.226 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l3wCTjNZHJuL1SrFtoKSzOHymF4=', 'Content-Length': '0'} 2022-09-03T07:31:03.228 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d9ef3dc1ebc5901c-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.230 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.233 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.234 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.235 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.236 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.236 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.237 DEBUG:boto:Data: 2022-09-03T07:31:03.238 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.238 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.239 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.239 DEBUG:boto:Params: {} 2022-09-03T07:31:03.240 DEBUG:boto:Token: None 2022-09-03T07:31:03.240 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.241 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:l3wCTjNZHJuL1SrFtoKSzOHymF4= 2022-09-03T07:31:03.242 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:l3wCTjNZHJuL1SrFtoKSzOHymF4=', 'Content-Length': '0'} 2022-09-03T07:31:03.244 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d6d8a328f626b94-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.245 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:31:03.245 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.246 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.247 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.247 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.248 DEBUG:boto:Method: GET 2022-09-03T07:31:03.248 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.249 DEBUG:boto:Data: 2022-09-03T07:31:03.249 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.251 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.251 DEBUG:boto:Params: {} 2022-09-03T07:31:03.253 DEBUG:boto:Token: None 2022-09-03T07:31:03.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPi/k/ptlLJ4de+RZOzg4Zip8C4= 2022-09-03T07:31:03.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPi/k/ptlLJ4de+RZOzg4Zip8C4=', 'Content-Length': '0'} 2022-09-03T07:31:03.258 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc1cbf75164b469c-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.259 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.259 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.260 DEBUG:boto:path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.260 DEBUG:boto:auth_path=/avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.261 DEBUG:boto:Method: GET 2022-09-03T07:31:03.261 DEBUG:boto:Path: /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.262 DEBUG:boto:Data: 2022-09-03T07:31:03.262 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.263 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.264 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.264 DEBUG:boto:Params: {} 2022-09-03T07:31:03.265 DEBUG:boto:Token: None 2022-09-03T07:31:03.265 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11?versionId=null 2022-09-03T07:31:03.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WPi/k/ptlLJ4de+RZOzg4Zip8C4= 2022-09-03T07:31:03.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WPi/k/ptlLJ4de+RZOzg4Zip8C4=', 'Content-Length': '0'} 2022-09-03T07:31:03.269 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a24bd85cb9a693e6-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.270 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.270 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.271 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:31:03.272 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:31:03.273 DEBUG:boto:Method: GET 2022-09-03T07:31:03.274 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:31:03.275 DEBUG:boto:Data: 2022-09-03T07:31:03.276 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.278 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.279 DEBUG:boto:Params: {} 2022-09-03T07:31:03.280 DEBUG:boto:Token: None 2022-09-03T07:31:03.281 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.284 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.288 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009f953d15ff685997-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.289 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.291 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.292 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.293 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:31:03.293 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:31:03.294 DEBUG:boto:Method: GET 2022-09-03T07:31:03.295 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:31:03.296 DEBUG:boto:Data: 2022-09-03T07:31:03.296 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.297 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.298 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.298 DEBUG:boto:Params: {} 2022-09-03T07:31:03.299 DEBUG:boto:Token: None 2022-09-03T07:31:03.300 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.304 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fb7a3d3b3eaef9c9-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.304 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.305 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.306 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.307 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.307 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:31:03.308 DEBUG:boto:Data: 2022-09-03T07:31:03.309 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.310 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.310 DEBUG:boto:Params: {} 2022-09-03T07:31:03.311 DEBUG:boto:Token: None 2022-09-03T07:31:03.311 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11 2022-09-03T07:31:03.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frgeLqOrWTVlVXRD6nU/I+m2jtw= 2022-09-03T07:31:03.312 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frgeLqOrWTVlVXRD6nU/I+m2jtw=', 'Content-Length': '0'} 2022-09-03T07:31:03.315 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000514e0f40f26005c8-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.315 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.316 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.317 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.317 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:31:03.318 DEBUG:boto:Data: 2022-09-03T07:31:03.318 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.319 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.319 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.320 DEBUG:boto:Params: {} 2022-09-03T07:31:03.321 DEBUG:boto:Token: None 2022-09-03T07:31:03.321 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11 2022-09-03T07:31:03.322 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:frgeLqOrWTVlVXRD6nU/I+m2jtw= 2022-09-03T07:31:03.323 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:frgeLqOrWTVlVXRD6nU/I+m2jtw=', 'Content-Length': '0'} 2022-09-03T07:31:03.325 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000033df470fbfe87321-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.325 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:31:03.326 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.327 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.327 DEBUG:boto:Method: GET 2022-09-03T07:31:03.328 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:31:03.328 DEBUG:boto:Data: 2022-09-03T07:31:03.329 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.330 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.331 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.331 DEBUG:boto:Params: {} 2022-09-03T07:31:03.333 DEBUG:boto:Token: None 2022-09-03T07:31:03.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11 2022-09-03T07:31:03.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CPOD7H8OXUPw/qtinla2DaSWMtI= 2022-09-03T07:31:03.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CPOD7H8OXUPw/qtinla2DaSWMtI=', 'Content-Length': '0'} 2022-09-03T07:31:03.337 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006697aef24660196d-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.378 DEBUG:boto:path=/avysmw-75/obj11 2022-09-03T07:31:03.379 DEBUG:boto:auth_path=/avysmw-75/obj11 2022-09-03T07:31:03.380 DEBUG:boto:Method: GET 2022-09-03T07:31:03.380 DEBUG:boto:Path: /avysmw-75/obj11 2022-09-03T07:31:03.381 DEBUG:boto:Data: 2022-09-03T07:31:03.381 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.382 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.383 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.384 DEBUG:boto:Params: {} 2022-09-03T07:31:03.385 DEBUG:boto:Token: None 2022-09-03T07:31:03.386 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj11 2022-09-03T07:31:03.387 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CPOD7H8OXUPw/qtinla2DaSWMtI= 2022-09-03T07:31:03.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CPOD7H8OXUPw/qtinla2DaSWMtI=', 'Content-Length': '0'} 2022-09-03T07:31:03.389 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bdfa61518a2fe7c3-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.390 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.391 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.391 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:31:03.392 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:31:03.393 DEBUG:boto:Method: GET 2022-09-03T07:31:03.395 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:31:03.399 DEBUG:boto:Data: 2022-09-03T07:31:03.399 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.400 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.401 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.401 DEBUG:boto:Params: {} 2022-09-03T07:31:03.402 DEBUG:boto:Token: None 2022-09-03T07:31:03.403 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.407 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000093d2630e638370dc-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.408 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.409 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.410 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.410 DEBUG:boto:path=/avysmw-75/?prefix=obj11 2022-09-03T07:31:03.411 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj11 2022-09-03T07:31:03.411 DEBUG:boto:Method: GET 2022-09-03T07:31:03.412 DEBUG:boto:Path: /avysmw-75/?prefix=obj11 2022-09-03T07:31:03.412 DEBUG:boto:Data: 2022-09-03T07:31:03.413 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.413 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.414 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.415 DEBUG:boto:Params: {} 2022-09-03T07:31:03.416 DEBUG:boto:Token: None 2022-09-03T07:31:03.417 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.422 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cdc6b0fc28038067-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.423 DEBUG:boto:b'avysmw-75obj111000falseobj112022-09-03T07:11:00.918Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.424 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj111 2022-09-03T07:31:03.425 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.426 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.427 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.428 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.428 DEBUG:boto:Method: GET 2022-09-03T07:31:03.430 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.431 DEBUG:boto:Data: 2022-09-03T07:31:03.431 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.432 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.432 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.433 DEBUG:boto:Params: {} 2022-09-03T07:31:03.433 DEBUG:boto:Token: None 2022-09-03T07:31:03.434 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.435 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lr1BGepCz/tqGGlU+FYmvAwx4Gs= 2022-09-03T07:31:03.436 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lr1BGepCz/tqGGlU+FYmvAwx4Gs=', 'Content-Length': '0'} 2022-09-03T07:31:03.439 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dc9be4070ac2f1cc-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.443 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.444 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.444 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.445 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.445 DEBUG:boto:Method: GET 2022-09-03T07:31:03.446 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.447 DEBUG:boto:Data: 2022-09-03T07:31:03.447 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.448 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.448 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.449 DEBUG:boto:Params: {} 2022-09-03T07:31:03.450 DEBUG:boto:Token: None 2022-09-03T07:31:03.450 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.451 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lr1BGepCz/tqGGlU+FYmvAwx4Gs= 2022-09-03T07:31:03.452 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lr1BGepCz/tqGGlU+FYmvAwx4Gs=', 'Content-Length': '0'} 2022-09-03T07:31:03.454 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e99c2d82f272f2a0-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.499 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.500 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.501 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.502 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.503 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.504 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.505 DEBUG:boto:Data: 2022-09-03T07:31:03.506 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.507 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.508 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.510 DEBUG:boto:Params: {} 2022-09-03T07:31:03.511 DEBUG:boto:Token: None 2022-09-03T07:31:03.512 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yBZx1stzZwN+4WRjDSczdv4sH4A= 2022-09-03T07:31:03.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yBZx1stzZwN+4WRjDSczdv4sH4A=', 'Content-Length': '0'} 2022-09-03T07:31:03.517 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000072e418916e834ca6-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.519 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.520 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.521 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.522 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.524 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.525 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.526 DEBUG:boto:Data: 2022-09-03T07:31:03.527 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.528 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.529 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.529 DEBUG:boto:Params: {} 2022-09-03T07:31:03.531 DEBUG:boto:Token: None 2022-09-03T07:31:03.532 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yBZx1stzZwN+4WRjDSczdv4sH4A= 2022-09-03T07:31:03.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yBZx1stzZwN+4WRjDSczdv4sH4A=', 'Content-Length': '0'} 2022-09-03T07:31:03.537 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008c7e7c34d62161f9-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.538 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj111 2022-09-03T07:31:03.539 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.540 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.541 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.542 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.543 DEBUG:boto:Method: GET 2022-09-03T07:31:03.544 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.545 DEBUG:boto:Data: 2022-09-03T07:31:03.546 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.548 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.549 DEBUG:boto:Params: {} 2022-09-03T07:31:03.551 DEBUG:boto:Token: None 2022-09-03T07:31:03.552 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.553 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lr1BGepCz/tqGGlU+FYmvAwx4Gs= 2022-09-03T07:31:03.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lr1BGepCz/tqGGlU+FYmvAwx4Gs=', 'Content-Length': '0'} 2022-09-03T07:31:03.557 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006bfae1ca98119b09-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.560 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.561 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.562 DEBUG:boto:path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.562 DEBUG:boto:auth_path=/avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.563 DEBUG:boto:Method: GET 2022-09-03T07:31:03.564 DEBUG:boto:Path: /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.565 DEBUG:boto:Data: 2022-09-03T07:31:03.566 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.566 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.567 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.568 DEBUG:boto:Params: {} 2022-09-03T07:31:03.569 DEBUG:boto:Token: None 2022-09-03T07:31:03.570 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111?versionId=null 2022-09-03T07:31:03.571 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lr1BGepCz/tqGGlU+FYmvAwx4Gs= 2022-09-03T07:31:03.572 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lr1BGepCz/tqGGlU+FYmvAwx4Gs=', 'Content-Length': '0'} 2022-09-03T07:31:03.575 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a0e4f4d9ab7cc598-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.577 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.577 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.582 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:31:03.583 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:31:03.584 DEBUG:boto:Method: GET 2022-09-03T07:31:03.585 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:31:03.586 DEBUG:boto:Data: 2022-09-03T07:31:03.588 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.589 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.589 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.590 DEBUG:boto:Params: {} 2022-09-03T07:31:03.590 DEBUG:boto:Token: None 2022-09-03T07:31:03.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.595 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023ba8bc8497467c1-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.597 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.598 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.599 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.600 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:31:03.601 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:31:03.602 DEBUG:boto:Method: GET 2022-09-03T07:31:03.604 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:31:03.605 DEBUG:boto:Data: 2022-09-03T07:31:03.606 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.607 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.610 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.611 DEBUG:boto:Params: {} 2022-09-03T07:31:03.613 DEBUG:boto:Token: None 2022-09-03T07:31:03.615 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.616 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.617 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.621 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000038924b87b6f12aa3-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.622 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.623 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.623 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.624 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.624 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:31:03.625 DEBUG:boto:Data: 2022-09-03T07:31:03.625 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.626 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.627 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.627 DEBUG:boto:Params: {} 2022-09-03T07:31:03.628 DEBUG:boto:Token: None 2022-09-03T07:31:03.628 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111 2022-09-03T07:31:03.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zJCzZFt7VU8pi3wZs9qXqsTnRp8= 2022-09-03T07:31:03.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zJCzZFt7VU8pi3wZs9qXqsTnRp8=', 'Content-Length': '0'} 2022-09-03T07:31:03.634 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c2e30b9b849545d7-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.635 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.636 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.637 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.638 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:31:03.639 DEBUG:boto:Data: 2022-09-03T07:31:03.639 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.640 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.640 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.641 DEBUG:boto:Params: {} 2022-09-03T07:31:03.642 DEBUG:boto:Token: None 2022-09-03T07:31:03.643 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111 2022-09-03T07:31:03.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zJCzZFt7VU8pi3wZs9qXqsTnRp8= 2022-09-03T07:31:03.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zJCzZFt7VU8pi3wZs9qXqsTnRp8=', 'Content-Length': '0'} 2022-09-03T07:31:03.648 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e6c918459da4540-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.649 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj111 2022-09-03T07:31:03.650 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.651 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.652 DEBUG:boto:Method: GET 2022-09-03T07:31:03.654 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:31:03.655 DEBUG:boto:Data: 2022-09-03T07:31:03.656 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.658 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.659 DEBUG:boto:Params: {} 2022-09-03T07:31:03.660 DEBUG:boto:Token: None 2022-09-03T07:31:03.661 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111 2022-09-03T07:31:03.662 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GcTB6g0H+UcbRjNqxQ6tDVIl1dM= 2022-09-03T07:31:03.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GcTB6g0H+UcbRjNqxQ6tDVIl1dM=', 'Content-Length': '0'} 2022-09-03T07:31:03.667 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000819931375a79af5f-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.710 DEBUG:boto:path=/avysmw-75/obj111 2022-09-03T07:31:03.712 DEBUG:boto:auth_path=/avysmw-75/obj111 2022-09-03T07:31:03.713 DEBUG:boto:Method: GET 2022-09-03T07:31:03.714 DEBUG:boto:Path: /avysmw-75/obj111 2022-09-03T07:31:03.715 DEBUG:boto:Data: 2022-09-03T07:31:03.715 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.716 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.717 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.717 DEBUG:boto:Params: {} 2022-09-03T07:31:03.718 DEBUG:boto:Token: None 2022-09-03T07:31:03.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj111 2022-09-03T07:31:03.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GcTB6g0H+UcbRjNqxQ6tDVIl1dM= 2022-09-03T07:31:03.721 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GcTB6g0H+UcbRjNqxQ6tDVIl1dM=', 'Content-Length': '0'} 2022-09-03T07:31:03.724 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f99544d9876aa9c-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.725 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.727 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.727 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:31:03.728 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:31:03.730 DEBUG:boto:Method: GET 2022-09-03T07:31:03.731 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:31:03.732 DEBUG:boto:Data: 2022-09-03T07:31:03.733 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.735 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.736 DEBUG:boto:Params: {} 2022-09-03T07:31:03.737 DEBUG:boto:Token: None 2022-09-03T07:31:03.738 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.745 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b0400eb63a0e2495-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.746 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.748 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.749 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.750 DEBUG:boto:path=/avysmw-75/?prefix=obj111 2022-09-03T07:31:03.751 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj111 2022-09-03T07:31:03.752 DEBUG:boto:Method: GET 2022-09-03T07:31:03.753 DEBUG:boto:Path: /avysmw-75/?prefix=obj111 2022-09-03T07:31:03.754 DEBUG:boto:Data: 2022-09-03T07:31:03.755 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.756 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.757 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.758 DEBUG:boto:Params: {} 2022-09-03T07:31:03.760 DEBUG:boto:Token: None 2022-09-03T07:31:03.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.762 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.763 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.766 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1645affc1966336-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.768 DEBUG:boto:b'avysmw-75obj1111000falseobj1112022-09-03T07:28:24.834Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.769 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:31:03.770 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.771 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.772 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.773 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.774 DEBUG:boto:Method: GET 2022-09-03T07:31:03.776 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.777 DEBUG:boto:Data: 2022-09-03T07:31:03.778 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.779 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.779 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.780 DEBUG:boto:Params: {} 2022-09-03T07:31:03.781 DEBUG:boto:Token: None 2022-09-03T07:31:03.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/COwtOqr7fWyOgV7MkMI4x89Lyw= 2022-09-03T07:31:03.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/COwtOqr7fWyOgV7MkMI4x89Lyw=', 'Content-Length': '0'} 2022-09-03T07:31:03.786 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fee194cd9c5e4520-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.787 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.788 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.789 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.790 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.790 DEBUG:boto:Method: GET 2022-09-03T07:31:03.791 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.792 DEBUG:boto:Data: 2022-09-03T07:31:03.793 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.794 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.794 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.796 DEBUG:boto:Params: {} 2022-09-03T07:31:03.797 DEBUG:boto:Token: None 2022-09-03T07:31:03.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.800 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/COwtOqr7fWyOgV7MkMI4x89Lyw= 2022-09-03T07:31:03.802 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/COwtOqr7fWyOgV7MkMI4x89Lyw=', 'Content-Length': '0'} 2022-09-03T07:31:03.805 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008f60c91f78e7a037-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.806 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.808 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.809 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.809 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.810 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.811 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.811 DEBUG:boto:Data: 2022-09-03T07:31:03.812 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.813 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.813 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.815 DEBUG:boto:Params: {} 2022-09-03T07:31:03.815 DEBUG:boto:Token: None 2022-09-03T07:31:03.816 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.817 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lwKwRmrfHynYPYJw/3uQJDxAvqo= 2022-09-03T07:31:03.818 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lwKwRmrfHynYPYJw/3uQJDxAvqo=', 'Content-Length': '0'} 2022-09-03T07:31:03.820 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b2508d294387ce97-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.822 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.822 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.823 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.824 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.824 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.825 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.826 DEBUG:boto:Data: 2022-09-03T07:31:03.827 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.828 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.829 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.830 DEBUG:boto:Params: {} 2022-09-03T07:31:03.830 DEBUG:boto:Token: None 2022-09-03T07:31:03.831 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.832 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lwKwRmrfHynYPYJw/3uQJDxAvqo= 2022-09-03T07:31:03.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lwKwRmrfHynYPYJw/3uQJDxAvqo=', 'Content-Length': '0'} 2022-09-03T07:31:03.836 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ddc2f5afde1788d5-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.837 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:31:03.837 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.838 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.839 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.839 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.840 DEBUG:boto:Method: GET 2022-09-03T07:31:03.841 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.841 DEBUG:boto:Data: 2022-09-03T07:31:03.842 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.843 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.843 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.844 DEBUG:boto:Params: {} 2022-09-03T07:31:03.845 DEBUG:boto:Token: None 2022-09-03T07:31:03.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/COwtOqr7fWyOgV7MkMI4x89Lyw= 2022-09-03T07:31:03.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/COwtOqr7fWyOgV7MkMI4x89Lyw=', 'Content-Length': '0'} 2022-09-03T07:31:03.849 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071316b4b02322107-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.850 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.851 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.851 DEBUG:boto:path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.852 DEBUG:boto:auth_path=/avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.853 DEBUG:boto:Method: GET 2022-09-03T07:31:03.853 DEBUG:boto:Path: /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.854 DEBUG:boto:Data: 2022-09-03T07:31:03.855 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.855 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.856 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.857 DEBUG:boto:Params: {} 2022-09-03T07:31:03.857 DEBUG:boto:Token: None 2022-09-03T07:31:03.858 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115?versionId=null 2022-09-03T07:31:03.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/COwtOqr7fWyOgV7MkMI4x89Lyw= 2022-09-03T07:31:03.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/COwtOqr7fWyOgV7MkMI4x89Lyw=', 'Content-Length': '0'} 2022-09-03T07:31:03.862 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036e8ef5a11d0f9a2-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.863 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.863 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.864 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:31:03.865 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:31:03.865 DEBUG:boto:Method: GET 2022-09-03T07:31:03.866 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:31:03.867 DEBUG:boto:Data: 2022-09-03T07:31:03.867 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.868 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.869 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.869 DEBUG:boto:Params: {} 2022-09-03T07:31:03.870 DEBUG:boto:Token: None 2022-09-03T07:31:03.871 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.872 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.875 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000014e9a12b1901ee87-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.876 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.877 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.878 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.878 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:31:03.879 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:31:03.880 DEBUG:boto:Method: GET 2022-09-03T07:31:03.880 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:31:03.881 DEBUG:boto:Data: 2022-09-03T07:31:03.882 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.882 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.883 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.884 DEBUG:boto:Params: {} 2022-09-03T07:31:03.884 DEBUG:boto:Token: None 2022-09-03T07:31:03.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/ 2022-09-03T07:31:03.886 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI= 2022-09-03T07:31:03.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NmaqNEuVhyPDFLeguBCMzpWYSMI=', 'Content-Length': '0'} 2022-09-03T07:31:03.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000e996b90ba6459e4-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.930 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:03.932 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.933 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.934 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.935 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:31:03.936 DEBUG:boto:Data: 2022-09-03T07:31:03.937 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.937 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.938 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.939 DEBUG:boto:Params: {} 2022-09-03T07:31:03.940 DEBUG:boto:Token: None 2022-09-03T07:31:03.940 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115 2022-09-03T07:31:03.941 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1H5xPgT6SXjo7ik9O8ABSn1Lk9E= 2022-09-03T07:31:03.942 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1H5xPgT6SXjo7ik9O8ABSn1Lk9E=', 'Content-Length': '0'} 2022-09-03T07:31:03.946 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006134c71b1b260ac1-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.947 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.948 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.949 DEBUG:boto:Method: HEAD 2022-09-03T07:31:03.950 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:31:03.951 DEBUG:boto:Data: 2022-09-03T07:31:03.952 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.953 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.954 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.955 DEBUG:boto:Params: {} 2022-09-03T07:31:03.956 DEBUG:boto:Token: None 2022-09-03T07:31:03.956 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115 2022-09-03T07:31:03.957 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1H5xPgT6SXjo7ik9O8ABSn1Lk9E= 2022-09-03T07:31:03.958 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1H5xPgT6SXjo7ik9O8ABSn1Lk9E=', 'Content-Length': '0'} 2022-09-03T07:31:03.962 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9beab87a2e6a6f8-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.963 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj115 2022-09-03T07:31:03.964 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.965 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.965 DEBUG:boto:Method: GET 2022-09-03T07:31:03.966 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:31:03.966 DEBUG:boto:Data: 2022-09-03T07:31:03.967 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:03.968 DEBUG:boto:Port: 8001 2022-09-03T07:31:03.968 DEBUG:boto:Params: {} 2022-09-03T07:31:03.969 DEBUG:boto:Token: None 2022-09-03T07:31:03.971 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115 2022-09-03T07:31:03.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4xRvAf684Ls3x4KZsfmoGdm6SQo= 2022-09-03T07:31:03.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4xRvAf684Ls3x4KZsfmoGdm6SQo=', 'Content-Length': '0'} 2022-09-03T07:31:03.975 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cd48b2258a301265-00631302b7-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.977 DEBUG:boto:path=/avysmw-75/obj115 2022-09-03T07:31:03.978 DEBUG:boto:auth_path=/avysmw-75/obj115 2022-09-03T07:31:03.979 DEBUG:boto:Method: GET 2022-09-03T07:31:03.980 DEBUG:boto:Path: /avysmw-75/obj115 2022-09-03T07:31:03.981 DEBUG:boto:Data: 2022-09-03T07:31:03.982 DEBUG:boto:Headers: {} 2022-09-03T07:31:03.983 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:03.984 DEBUG:boto:Port: 8000 2022-09-03T07:31:03.985 DEBUG:boto:Params: {} 2022-09-03T07:31:03.986 DEBUG:boto:Token: None 2022-09-03T07:31:03.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:03 GMT /avysmw-75/obj115 2022-09-03T07:31:03.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4xRvAf684Ls3x4KZsfmoGdm6SQo= 2022-09-03T07:31:03.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:03 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4xRvAf684Ls3x4KZsfmoGdm6SQo=', 'Content-Length': '0'} 2022-09-03T07:31:03.992 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cb5a9b8225c7daa7-00631302b7-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:03 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:03.993 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:03.994 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:03.995 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:31:03.996 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:31:03.997 DEBUG:boto:Method: GET 2022-09-03T07:31:03.998 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:31:03.999 DEBUG:boto:Data: 2022-09-03T07:31:04.000 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.001 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.002 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.003 DEBUG:boto:Params: {} 2022-09-03T07:31:04.004 DEBUG:boto:Token: None 2022-09-03T07:31:04.005 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.006 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.007 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.010 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000afecf8299d7fba23-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.012 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.013 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.014 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.015 DEBUG:boto:path=/avysmw-75/?prefix=obj115 2022-09-03T07:31:04.016 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj115 2022-09-03T07:31:04.017 DEBUG:boto:Method: GET 2022-09-03T07:31:04.018 DEBUG:boto:Path: /avysmw-75/?prefix=obj115 2022-09-03T07:31:04.019 DEBUG:boto:Data: 2022-09-03T07:31:04.020 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.021 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.022 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.023 DEBUG:boto:Params: {} 2022-09-03T07:31:04.024 DEBUG:boto:Token: None 2022-09-03T07:31:04.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.027 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.029 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000af8712b05b9e79c1-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.030 DEBUG:boto:b'avysmw-75obj1151000falseobj1152022-09-03T07:21:21.696Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.031 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:31:04.032 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.034 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.035 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.035 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.036 DEBUG:boto:Method: GET 2022-09-03T07:31:04.036 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.037 DEBUG:boto:Data: 2022-09-03T07:31:04.037 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.038 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.039 DEBUG:boto:Params: {} 2022-09-03T07:31:04.039 DEBUG:boto:Token: None 2022-09-03T07:31:04.040 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CJvO4HIC7GG+IkfdERC5j99h1KQ= 2022-09-03T07:31:04.041 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CJvO4HIC7GG+IkfdERC5j99h1KQ=', 'Content-Length': '0'} 2022-09-03T07:31:04.043 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc68d4a51b21fd62-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.093 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.095 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.095 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.096 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.096 DEBUG:boto:Method: GET 2022-09-03T07:31:04.097 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.098 DEBUG:boto:Data: 2022-09-03T07:31:04.098 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.099 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.099 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.100 DEBUG:boto:Params: {} 2022-09-03T07:31:04.100 DEBUG:boto:Token: None 2022-09-03T07:31:04.101 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.102 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CJvO4HIC7GG+IkfdERC5j99h1KQ= 2022-09-03T07:31:04.103 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CJvO4HIC7GG+IkfdERC5j99h1KQ=', 'Content-Length': '0'} 2022-09-03T07:31:04.105 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000576244ce0b9eadc6-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.106 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.107 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.108 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.108 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.108 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.109 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.110 DEBUG:boto:Data: 2022-09-03T07:31:04.111 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.112 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.112 DEBUG:boto:Params: {} 2022-09-03T07:31:04.113 DEBUG:boto:Token: None 2022-09-03T07:31:04.114 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XhmfFWPDO5eOL1xfstBrlGBSQ1A= 2022-09-03T07:31:04.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XhmfFWPDO5eOL1xfstBrlGBSQ1A=', 'Content-Length': '0'} 2022-09-03T07:31:04.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006875a162e3980302-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.120 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.121 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.122 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.123 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.123 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.124 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.124 DEBUG:boto:Data: 2022-09-03T07:31:04.125 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.125 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.126 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.126 DEBUG:boto:Params: {} 2022-09-03T07:31:04.127 DEBUG:boto:Token: None 2022-09-03T07:31:04.128 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XhmfFWPDO5eOL1xfstBrlGBSQ1A= 2022-09-03T07:31:04.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XhmfFWPDO5eOL1xfstBrlGBSQ1A=', 'Content-Length': '0'} 2022-09-03T07:31:04.131 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e4263a810300a872-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.131 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:31:04.132 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.132 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.133 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.133 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.134 DEBUG:boto:Method: GET 2022-09-03T07:31:04.135 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.135 DEBUG:boto:Data: 2022-09-03T07:31:04.136 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.136 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.137 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.138 DEBUG:boto:Params: {} 2022-09-03T07:31:04.138 DEBUG:boto:Token: None 2022-09-03T07:31:04.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CJvO4HIC7GG+IkfdERC5j99h1KQ= 2022-09-03T07:31:04.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CJvO4HIC7GG+IkfdERC5j99h1KQ=', 'Content-Length': '0'} 2022-09-03T07:31:04.142 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000602f69bf10d2f91d-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.143 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.143 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.144 DEBUG:boto:path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.144 DEBUG:boto:auth_path=/avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.145 DEBUG:boto:Method: GET 2022-09-03T07:31:04.146 DEBUG:boto:Path: /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.146 DEBUG:boto:Data: 2022-09-03T07:31:04.147 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.147 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.148 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.148 DEBUG:boto:Params: {} 2022-09-03T07:31:04.149 DEBUG:boto:Token: None 2022-09-03T07:31:04.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116?versionId=null 2022-09-03T07:31:04.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:CJvO4HIC7GG+IkfdERC5j99h1KQ= 2022-09-03T07:31:04.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:CJvO4HIC7GG+IkfdERC5j99h1KQ=', 'Content-Length': '0'} 2022-09-03T07:31:04.153 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cabc1e3eddde5ba7-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.154 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.154 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.155 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:31:04.155 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:31:04.156 DEBUG:boto:Method: GET 2022-09-03T07:31:04.156 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:31:04.157 DEBUG:boto:Data: 2022-09-03T07:31:04.157 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.158 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.158 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.159 DEBUG:boto:Params: {} 2022-09-03T07:31:04.160 DEBUG:boto:Token: None 2022-09-03T07:31:04.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.164 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004feab38d33ba6653-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.206 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.208 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.209 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.210 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:31:04.211 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:31:04.212 DEBUG:boto:Method: GET 2022-09-03T07:31:04.213 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:31:04.214 DEBUG:boto:Data: 2022-09-03T07:31:04.215 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.216 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.217 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.218 DEBUG:boto:Params: {} 2022-09-03T07:31:04.219 DEBUG:boto:Token: None 2022-09-03T07:31:04.220 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.222 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.225 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005bd10c5903c08180-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.226 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.228 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.228 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.229 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.230 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:31:04.231 DEBUG:boto:Data: 2022-09-03T07:31:04.232 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.233 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.234 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.235 DEBUG:boto:Params: {} 2022-09-03T07:31:04.236 DEBUG:boto:Token: None 2022-09-03T07:31:04.238 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116 2022-09-03T07:31:04.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttJCWVAecRrKx0axB022CKOMdO4= 2022-09-03T07:31:04.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttJCWVAecRrKx0axB022CKOMdO4=', 'Content-Length': '0'} 2022-09-03T07:31:04.242 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa49a03786488968-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.243 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.244 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.245 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.245 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:31:04.246 DEBUG:boto:Data: 2022-09-03T07:31:04.246 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.247 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.248 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.248 DEBUG:boto:Params: {} 2022-09-03T07:31:04.249 DEBUG:boto:Token: None 2022-09-03T07:31:04.249 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116 2022-09-03T07:31:04.250 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttJCWVAecRrKx0axB022CKOMdO4= 2022-09-03T07:31:04.251 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttJCWVAecRrKx0axB022CKOMdO4=', 'Content-Length': '0'} 2022-09-03T07:31:04.253 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003fdf7829d116fef4-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.254 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj116 2022-09-03T07:31:04.255 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.256 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.257 DEBUG:boto:Method: GET 2022-09-03T07:31:04.258 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:31:04.259 DEBUG:boto:Data: 2022-09-03T07:31:04.259 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.260 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.260 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.261 DEBUG:boto:Params: {} 2022-09-03T07:31:04.262 DEBUG:boto:Token: None 2022-09-03T07:31:04.263 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116 2022-09-03T07:31:04.264 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zGJoZZFUjuVnXcqp0y9/lmfGyaA= 2022-09-03T07:31:04.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zGJoZZFUjuVnXcqp0y9/lmfGyaA=', 'Content-Length': '0'} 2022-09-03T07:31:04.267 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a04b8e6c7fbb6279-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.268 DEBUG:boto:path=/avysmw-75/obj116 2022-09-03T07:31:04.269 DEBUG:boto:auth_path=/avysmw-75/obj116 2022-09-03T07:31:04.270 DEBUG:boto:Method: GET 2022-09-03T07:31:04.271 DEBUG:boto:Path: /avysmw-75/obj116 2022-09-03T07:31:04.272 DEBUG:boto:Data: 2022-09-03T07:31:04.273 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.274 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.275 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.276 DEBUG:boto:Params: {} 2022-09-03T07:31:04.277 DEBUG:boto:Token: None 2022-09-03T07:31:04.278 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj116 2022-09-03T07:31:04.279 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zGJoZZFUjuVnXcqp0y9/lmfGyaA= 2022-09-03T07:31:04.280 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zGJoZZFUjuVnXcqp0y9/lmfGyaA=', 'Content-Length': '0'} 2022-09-03T07:31:04.282 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003436089ebe0893d0-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.283 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.284 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.284 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:31:04.285 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:31:04.285 DEBUG:boto:Method: GET 2022-09-03T07:31:04.286 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:31:04.287 DEBUG:boto:Data: 2022-09-03T07:31:04.287 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.288 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.288 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.289 DEBUG:boto:Params: {} 2022-09-03T07:31:04.290 DEBUG:boto:Token: None 2022-09-03T07:31:04.290 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.296 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa412ac20fba5de7-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.338 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.339 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.340 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.340 DEBUG:boto:path=/avysmw-75/?prefix=obj116 2022-09-03T07:31:04.341 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj116 2022-09-03T07:31:04.341 DEBUG:boto:Method: GET 2022-09-03T07:31:04.342 DEBUG:boto:Path: /avysmw-75/?prefix=obj116 2022-09-03T07:31:04.343 DEBUG:boto:Data: 2022-09-03T07:31:04.344 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.345 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.345 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.346 DEBUG:boto:Params: {} 2022-09-03T07:31:04.346 DEBUG:boto:Token: None 2022-09-03T07:31:04.347 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.348 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.348 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.351 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000202a25d05d890269-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.352 DEBUG:boto:b'avysmw-75obj1161000falseobj1162022-09-03T07:24:23.754Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.353 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:31:04.354 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.355 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.355 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.356 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.357 DEBUG:boto:Method: GET 2022-09-03T07:31:04.358 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.359 DEBUG:boto:Data: 2022-09-03T07:31:04.360 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.361 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.361 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.362 DEBUG:boto:Params: {} 2022-09-03T07:31:04.363 DEBUG:boto:Token: None 2022-09-03T07:31:04.364 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EBAEuhGbHMB624QclKpDFht6Kk= 2022-09-03T07:31:04.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EBAEuhGbHMB624QclKpDFht6Kk=', 'Content-Length': '0'} 2022-09-03T07:31:04.370 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008580ab32eeceea39-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.371 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.371 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.372 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.372 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.373 DEBUG:boto:Method: GET 2022-09-03T07:31:04.374 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.375 DEBUG:boto:Data: 2022-09-03T07:31:04.375 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.376 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.376 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.377 DEBUG:boto:Params: {} 2022-09-03T07:31:04.377 DEBUG:boto:Token: None 2022-09-03T07:31:04.378 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.379 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EBAEuhGbHMB624QclKpDFht6Kk= 2022-09-03T07:31:04.379 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EBAEuhGbHMB624QclKpDFht6Kk=', 'Content-Length': '0'} 2022-09-03T07:31:04.381 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000092346f3d1816c024-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.422 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.423 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.423 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.424 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.424 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.425 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.425 DEBUG:boto:Data: 2022-09-03T07:31:04.426 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.427 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.428 DEBUG:boto:Params: {} 2022-09-03T07:31:04.429 DEBUG:boto:Token: None 2022-09-03T07:31:04.429 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JtSZq9CWcfQaq4TRE1fze/WgDCg= 2022-09-03T07:31:04.430 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JtSZq9CWcfQaq4TRE1fze/WgDCg=', 'Content-Length': '0'} 2022-09-03T07:31:04.433 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a522e63e97f5406-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.434 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.435 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.435 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.436 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.436 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.437 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.438 DEBUG:boto:Data: 2022-09-03T07:31:04.438 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.439 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.439 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.440 DEBUG:boto:Params: {} 2022-09-03T07:31:04.441 DEBUG:boto:Token: None 2022-09-03T07:31:04.441 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JtSZq9CWcfQaq4TRE1fze/WgDCg= 2022-09-03T07:31:04.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JtSZq9CWcfQaq4TRE1fze/WgDCg=', 'Content-Length': '0'} 2022-09-03T07:31:04.445 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004aac6319cc8c050f-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.446 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:31:04.447 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.448 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.449 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.453 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.454 DEBUG:boto:Method: GET 2022-09-03T07:31:04.455 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.455 DEBUG:boto:Data: 2022-09-03T07:31:04.456 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.457 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.458 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.459 DEBUG:boto:Params: {} 2022-09-03T07:31:04.460 DEBUG:boto:Token: None 2022-09-03T07:31:04.461 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EBAEuhGbHMB624QclKpDFht6Kk= 2022-09-03T07:31:04.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EBAEuhGbHMB624QclKpDFht6Kk=', 'Content-Length': '0'} 2022-09-03T07:31:04.465 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023c4c48e07762308-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.466 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.467 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.468 DEBUG:boto:path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.469 DEBUG:boto:auth_path=/avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.469 DEBUG:boto:Method: GET 2022-09-03T07:31:04.470 DEBUG:boto:Path: /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.471 DEBUG:boto:Data: 2022-09-03T07:31:04.472 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.473 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.474 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.475 DEBUG:boto:Params: {} 2022-09-03T07:31:04.476 DEBUG:boto:Token: None 2022-09-03T07:31:04.478 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124?versionId=null 2022-09-03T07:31:04.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EBAEuhGbHMB624QclKpDFht6Kk= 2022-09-03T07:31:04.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EBAEuhGbHMB624QclKpDFht6Kk=', 'Content-Length': '0'} 2022-09-03T07:31:04.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d17bfdbd00cd226-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.484 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.485 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.485 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:31:04.486 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:31:04.486 DEBUG:boto:Method: GET 2022-09-03T07:31:04.487 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:31:04.487 DEBUG:boto:Data: 2022-09-03T07:31:04.488 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.489 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.490 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.490 DEBUG:boto:Params: {} 2022-09-03T07:31:04.491 DEBUG:boto:Token: None 2022-09-03T07:31:04.492 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.496 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b9b9b85eb90ab428-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.497 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.498 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.499 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.499 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:31:04.500 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:31:04.501 DEBUG:boto:Method: GET 2022-09-03T07:31:04.501 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:31:04.502 DEBUG:boto:Data: 2022-09-03T07:31:04.502 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.503 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.504 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.504 DEBUG:boto:Params: {} 2022-09-03T07:31:04.505 DEBUG:boto:Token: None 2022-09-03T07:31:04.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.506 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.512 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002f5617123cb1950d-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.513 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.514 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.515 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.516 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.516 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:31:04.517 DEBUG:boto:Data: 2022-09-03T07:31:04.517 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.518 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.519 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.519 DEBUG:boto:Params: {} 2022-09-03T07:31:04.520 DEBUG:boto:Token: None 2022-09-03T07:31:04.521 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124 2022-09-03T07:31:04.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TwMoCFu1Qmx0Wvk9ayJGV+LCLps= 2022-09-03T07:31:04.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TwMoCFu1Qmx0Wvk9ayJGV+LCLps=', 'Content-Length': '0'} 2022-09-03T07:31:04.525 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b5fd18059e7689f3-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.526 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.526 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.527 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.527 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:31:04.528 DEBUG:boto:Data: 2022-09-03T07:31:04.528 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.529 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.529 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.530 DEBUG:boto:Params: {} 2022-09-03T07:31:04.531 DEBUG:boto:Token: None 2022-09-03T07:31:04.531 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124 2022-09-03T07:31:04.532 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TwMoCFu1Qmx0Wvk9ayJGV+LCLps= 2022-09-03T07:31:04.533 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TwMoCFu1Qmx0Wvk9ayJGV+LCLps=', 'Content-Length': '0'} 2022-09-03T07:31:04.536 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008778d6137e7c32a3-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.537 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj124 2022-09-03T07:31:04.537 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.538 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.539 DEBUG:boto:Method: GET 2022-09-03T07:31:04.539 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:31:04.540 DEBUG:boto:Data: 2022-09-03T07:31:04.540 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.541 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.542 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.542 DEBUG:boto:Params: {} 2022-09-03T07:31:04.543 DEBUG:boto:Token: None 2022-09-03T07:31:04.544 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124 2022-09-03T07:31:04.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WcWon7crNT4A4YaXSyeFraQ/yv0= 2022-09-03T07:31:04.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WcWon7crNT4A4YaXSyeFraQ/yv0=', 'Content-Length': '0'} 2022-09-03T07:31:04.547 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe16b64711db1def-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.548 DEBUG:boto:path=/avysmw-75/obj124 2022-09-03T07:31:04.549 DEBUG:boto:auth_path=/avysmw-75/obj124 2022-09-03T07:31:04.549 DEBUG:boto:Method: GET 2022-09-03T07:31:04.550 DEBUG:boto:Path: /avysmw-75/obj124 2022-09-03T07:31:04.551 DEBUG:boto:Data: 2022-09-03T07:31:04.551 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.552 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.553 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.553 DEBUG:boto:Params: {} 2022-09-03T07:31:04.554 DEBUG:boto:Token: None 2022-09-03T07:31:04.555 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj124 2022-09-03T07:31:04.555 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:WcWon7crNT4A4YaXSyeFraQ/yv0= 2022-09-03T07:31:04.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:WcWon7crNT4A4YaXSyeFraQ/yv0=', 'Content-Length': '0'} 2022-09-03T07:31:04.558 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee337831316597a1-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.602 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.603 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.604 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:31:04.604 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:31:04.605 DEBUG:boto:Method: GET 2022-09-03T07:31:04.606 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:31:04.606 DEBUG:boto:Data: 2022-09-03T07:31:04.607 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.607 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.608 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.608 DEBUG:boto:Params: {} 2022-09-03T07:31:04.609 DEBUG:boto:Token: None 2022-09-03T07:31:04.610 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.611 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.614 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000034af64ce7b498887-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.615 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.616 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.616 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.618 DEBUG:boto:path=/avysmw-75/?prefix=obj124 2022-09-03T07:31:04.619 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj124 2022-09-03T07:31:04.620 DEBUG:boto:Method: GET 2022-09-03T07:31:04.621 DEBUG:boto:Path: /avysmw-75/?prefix=obj124 2022-09-03T07:31:04.621 DEBUG:boto:Data: 2022-09-03T07:31:04.623 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.624 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.625 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.626 DEBUG:boto:Params: {} 2022-09-03T07:31:04.627 DEBUG:boto:Token: None 2022-09-03T07:31:04.628 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.629 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.633 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ddf481f94733b2c7-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.634 DEBUG:boto:b'avysmw-75obj1241000falseobj1242022-09-03T07:26:11.165Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.635 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:31:04.635 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.636 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.636 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.637 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.638 DEBUG:boto:Method: GET 2022-09-03T07:31:04.638 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.639 DEBUG:boto:Data: 2022-09-03T07:31:04.640 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.641 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.642 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.642 DEBUG:boto:Params: {} 2022-09-03T07:31:04.643 DEBUG:boto:Token: None 2022-09-03T07:31:04.644 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.645 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lZSGWEXKsSGyPd2B1iaE+rv9gBY= 2022-09-03T07:31:04.646 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lZSGWEXKsSGyPd2B1iaE+rv9gBY=', 'Content-Length': '0'} 2022-09-03T07:31:04.648 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000def8ed8ad5864061-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.649 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.650 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.651 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.652 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.653 DEBUG:boto:Method: GET 2022-09-03T07:31:04.654 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.654 DEBUG:boto:Data: 2022-09-03T07:31:04.655 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.656 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.656 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.657 DEBUG:boto:Params: {} 2022-09-03T07:31:04.657 DEBUG:boto:Token: None 2022-09-03T07:31:04.658 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.659 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lZSGWEXKsSGyPd2B1iaE+rv9gBY= 2022-09-03T07:31:04.659 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lZSGWEXKsSGyPd2B1iaE+rv9gBY=', 'Content-Length': '0'} 2022-09-03T07:31:04.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d78ce9436a1c6599-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.662 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.662 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.663 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.663 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.664 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.664 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.665 DEBUG:boto:Data: 2022-09-03T07:31:04.665 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.666 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.666 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.667 DEBUG:boto:Params: {} 2022-09-03T07:31:04.667 DEBUG:boto:Token: None 2022-09-03T07:31:04.668 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.669 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EQhq8JInCSviWm/xNwsm0hazfqc= 2022-09-03T07:31:04.669 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EQhq8JInCSviWm/xNwsm0hazfqc=', 'Content-Length': '0'} 2022-09-03T07:31:04.671 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a842fb29e9e5d1ec-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.672 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.673 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.673 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.674 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.674 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.675 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.675 DEBUG:boto:Data: 2022-09-03T07:31:04.676 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.676 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.677 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.677 DEBUG:boto:Params: {} 2022-09-03T07:31:04.678 DEBUG:boto:Token: None 2022-09-03T07:31:04.678 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.679 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EQhq8JInCSviWm/xNwsm0hazfqc= 2022-09-03T07:31:04.680 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EQhq8JInCSviWm/xNwsm0hazfqc=', 'Content-Length': '0'} 2022-09-03T07:31:04.681 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d371c7f9583be3ff-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.682 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:31:04.683 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.683 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.684 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.684 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.685 DEBUG:boto:Method: GET 2022-09-03T07:31:04.685 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.686 DEBUG:boto:Data: 2022-09-03T07:31:04.686 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.687 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.688 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.688 DEBUG:boto:Params: {} 2022-09-03T07:31:04.689 DEBUG:boto:Token: None 2022-09-03T07:31:04.689 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lZSGWEXKsSGyPd2B1iaE+rv9gBY= 2022-09-03T07:31:04.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lZSGWEXKsSGyPd2B1iaE+rv9gBY=', 'Content-Length': '0'} 2022-09-03T07:31:04.693 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003ac43049616c5152-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.694 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.694 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.695 DEBUG:boto:path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.695 DEBUG:boto:auth_path=/avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.696 DEBUG:boto:Method: GET 2022-09-03T07:31:04.697 DEBUG:boto:Path: /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.697 DEBUG:boto:Data: 2022-09-03T07:31:04.698 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.698 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.699 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.700 DEBUG:boto:Params: {} 2022-09-03T07:31:04.700 DEBUG:boto:Token: None 2022-09-03T07:31:04.701 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145?versionId=null 2022-09-03T07:31:04.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lZSGWEXKsSGyPd2B1iaE+rv9gBY= 2022-09-03T07:31:04.702 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lZSGWEXKsSGyPd2B1iaE+rv9gBY=', 'Content-Length': '0'} 2022-09-03T07:31:04.704 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d109018e086983f3-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.705 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.706 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.706 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:31:04.707 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:31:04.707 DEBUG:boto:Method: GET 2022-09-03T07:31:04.708 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:31:04.708 DEBUG:boto:Data: 2022-09-03T07:31:04.709 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.711 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.711 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.712 DEBUG:boto:Params: {} 2022-09-03T07:31:04.712 DEBUG:boto:Token: None 2022-09-03T07:31:04.713 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.713 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.714 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.717 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f2ae2a4546108da-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.717 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.718 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.719 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.720 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:31:04.720 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:31:04.721 DEBUG:boto:Method: GET 2022-09-03T07:31:04.721 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:31:04.722 DEBUG:boto:Data: 2022-09-03T07:31:04.722 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.723 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.723 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.724 DEBUG:boto:Params: {} 2022-09-03T07:31:04.724 DEBUG:boto:Token: None 2022-09-03T07:31:04.725 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.726 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.731 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009013d93a4de819bd-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.732 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.733 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.733 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.734 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.734 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:31:04.735 DEBUG:boto:Data: 2022-09-03T07:31:04.735 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.736 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.736 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.737 DEBUG:boto:Params: {} 2022-09-03T07:31:04.737 DEBUG:boto:Token: None 2022-09-03T07:31:04.738 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145 2022-09-03T07:31:04.739 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PAhhtYd+nNSaS3aBSTTvd0TDhLc= 2022-09-03T07:31:04.740 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PAhhtYd+nNSaS3aBSTTvd0TDhLc=', 'Content-Length': '0'} 2022-09-03T07:31:04.741 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009332e48e17ad1d93-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.742 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.743 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.743 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.744 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:31:04.745 DEBUG:boto:Data: 2022-09-03T07:31:04.745 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.746 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.746 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.747 DEBUG:boto:Params: {} 2022-09-03T07:31:04.749 DEBUG:boto:Token: None 2022-09-03T07:31:04.750 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145 2022-09-03T07:31:04.750 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PAhhtYd+nNSaS3aBSTTvd0TDhLc= 2022-09-03T07:31:04.751 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PAhhtYd+nNSaS3aBSTTvd0TDhLc=', 'Content-Length': '0'} 2022-09-03T07:31:04.753 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004957c46d1ce463a-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.754 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj145 2022-09-03T07:31:04.754 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.755 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.755 DEBUG:boto:Method: GET 2022-09-03T07:31:04.756 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:31:04.756 DEBUG:boto:Data: 2022-09-03T07:31:04.757 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.758 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.759 DEBUG:boto:Params: {} 2022-09-03T07:31:04.760 DEBUG:boto:Token: None 2022-09-03T07:31:04.761 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145 2022-09-03T07:31:04.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a4aUkTHjwgNhygKdv3ze1UOqkWM= 2022-09-03T07:31:04.762 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a4aUkTHjwgNhygKdv3ze1UOqkWM=', 'Content-Length': '0'} 2022-09-03T07:31:04.764 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056190a5f338d3d04-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.765 DEBUG:boto:path=/avysmw-75/obj145 2022-09-03T07:31:04.766 DEBUG:boto:auth_path=/avysmw-75/obj145 2022-09-03T07:31:04.766 DEBUG:boto:Method: GET 2022-09-03T07:31:04.767 DEBUG:boto:Path: /avysmw-75/obj145 2022-09-03T07:31:04.767 DEBUG:boto:Data: 2022-09-03T07:31:04.768 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.769 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.769 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.770 DEBUG:boto:Params: {} 2022-09-03T07:31:04.771 DEBUG:boto:Token: None 2022-09-03T07:31:04.771 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj145 2022-09-03T07:31:04.772 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:a4aUkTHjwgNhygKdv3ze1UOqkWM= 2022-09-03T07:31:04.772 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:a4aUkTHjwgNhygKdv3ze1UOqkWM=', 'Content-Length': '0'} 2022-09-03T07:31:04.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000116930855c3caa99-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.775 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.776 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.776 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:31:04.777 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:31:04.777 DEBUG:boto:Method: GET 2022-09-03T07:31:04.778 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:31:04.779 DEBUG:boto:Data: 2022-09-03T07:31:04.779 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.780 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.780 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.781 DEBUG:boto:Params: {} 2022-09-03T07:31:04.782 DEBUG:boto:Token: None 2022-09-03T07:31:04.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.783 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.786 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020b7cb050204e34d-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.787 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.788 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.789 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.789 DEBUG:boto:path=/avysmw-75/?prefix=obj145 2022-09-03T07:31:04.790 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj145 2022-09-03T07:31:04.791 DEBUG:boto:Method: GET 2022-09-03T07:31:04.791 DEBUG:boto:Path: /avysmw-75/?prefix=obj145 2022-09-03T07:31:04.792 DEBUG:boto:Data: 2022-09-03T07:31:04.792 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.793 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.793 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.794 DEBUG:boto:Params: {} 2022-09-03T07:31:04.795 DEBUG:boto:Token: None 2022-09-03T07:31:04.795 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.796 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.799 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098369f9873779341-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.846 DEBUG:boto:b'avysmw-75obj1451000falseobj1452022-09-03T07:24:23.785Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:04.847 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj148 2022-09-03T07:31:04.848 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:04.848 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:04.849 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.849 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.850 DEBUG:boto:Method: GET 2022-09-03T07:31:04.855 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.855 DEBUG:boto:Data: 2022-09-03T07:31:04.856 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.856 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.857 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.857 DEBUG:boto:Params: {} 2022-09-03T07:31:04.858 DEBUG:boto:Token: None 2022-09-03T07:31:04.859 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.859 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2WWN9gWtwJjq+EZajAJMVJxLfBs= 2022-09-03T07:31:04.860 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2WWN9gWtwJjq+EZajAJMVJxLfBs=', 'Content-Length': '0'} 2022-09-03T07:31:04.869 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e33d86012b5efa83-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.872 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:04.876 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:04.877 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.878 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.879 DEBUG:boto:Method: GET 2022-09-03T07:31:04.880 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.881 DEBUG:boto:Data: 2022-09-03T07:31:04.883 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.884 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.885 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.886 DEBUG:boto:Params: {} 2022-09-03T07:31:04.887 DEBUG:boto:Token: None 2022-09-03T07:31:04.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2WWN9gWtwJjq+EZajAJMVJxLfBs= 2022-09-03T07:31:04.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2WWN9gWtwJjq+EZajAJMVJxLfBs=', 'Content-Length': '0'} 2022-09-03T07:31:04.893 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee0a20de7dc3221e-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.896 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:04.896 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:04.897 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.898 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.899 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.901 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.902 DEBUG:boto:Data: 2022-09-03T07:31:04.903 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.904 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.905 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.907 DEBUG:boto:Params: {} 2022-09-03T07:31:04.908 DEBUG:boto:Token: None 2022-09-03T07:31:04.910 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ps+gu3tWtTRdyudojWxTkK3e6wY= 2022-09-03T07:31:04.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ps+gu3tWtTRdyudojWxTkK3e6wY=', 'Content-Length': '0'} 2022-09-03T07:31:04.917 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000044a30cefc6c198ba-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.918 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:04.918 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:04.919 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.919 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.920 DEBUG:boto:Method: HEAD 2022-09-03T07:31:04.921 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.921 DEBUG:boto:Data: 2022-09-03T07:31:04.922 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.922 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.923 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.923 DEBUG:boto:Params: {} 2022-09-03T07:31:04.926 DEBUG:boto:Token: None 2022-09-03T07:31:04.927 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ps+gu3tWtTRdyudojWxTkK3e6wY= 2022-09-03T07:31:04.928 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ps+gu3tWtTRdyudojWxTkK3e6wY=', 'Content-Length': '0'} 2022-09-03T07:31:04.930 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057362dd9ef212f11-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.931 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj148 2022-09-03T07:31:04.932 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:04.932 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:04.933 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.933 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.934 DEBUG:boto:Method: GET 2022-09-03T07:31:04.934 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.935 DEBUG:boto:Data: 2022-09-03T07:31:04.936 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.936 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.937 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.937 DEBUG:boto:Params: {} 2022-09-03T07:31:04.938 DEBUG:boto:Token: None 2022-09-03T07:31:04.939 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2WWN9gWtwJjq+EZajAJMVJxLfBs= 2022-09-03T07:31:04.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2WWN9gWtwJjq+EZajAJMVJxLfBs=', 'Content-Length': '0'} 2022-09-03T07:31:04.942 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000448be2fe6598e453-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.943 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:04.944 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:04.945 DEBUG:boto:path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.945 DEBUG:boto:auth_path=/avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.946 DEBUG:boto:Method: GET 2022-09-03T07:31:04.946 DEBUG:boto:Path: /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.947 DEBUG:boto:Data: 2022-09-03T07:31:04.947 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.948 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:04.949 DEBUG:boto:Port: 8000 2022-09-03T07:31:04.949 DEBUG:boto:Params: {} 2022-09-03T07:31:04.950 DEBUG:boto:Token: None 2022-09-03T07:31:04.950 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/obj148?versionId=null 2022-09-03T07:31:04.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2WWN9gWtwJjq+EZajAJMVJxLfBs= 2022-09-03T07:31:04.952 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2WWN9gWtwJjq+EZajAJMVJxLfBs=', 'Content-Length': '0'} 2022-09-03T07:31:04.956 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d416dd1e81c5f05a-00631302b8-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:04.958 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:04.959 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:04.961 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:31:04.962 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:31:04.963 DEBUG:boto:Method: GET 2022-09-03T07:31:04.964 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:31:04.965 DEBUG:boto:Data: 2022-09-03T07:31:04.966 DEBUG:boto:Headers: {} 2022-09-03T07:31:04.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:04.968 DEBUG:boto:Port: 8001 2022-09-03T07:31:04.969 DEBUG:boto:Params: {} 2022-09-03T07:31:04.970 DEBUG:boto:Token: None 2022-09-03T07:31:04.972 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:04 GMT /avysmw-75/ 2022-09-03T07:31:04.973 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM= 2022-09-03T07:31:04.974 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:04 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IcuHcX/hHJHYtflQxwG/MdsFJYM=', 'Content-Length': '0'} 2022-09-03T07:31:04.977 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001f2e4933b2d726aa-00631302b8-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:04 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.022 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.023 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.024 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.024 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:31:05.025 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:31:05.026 DEBUG:boto:Method: GET 2022-09-03T07:31:05.027 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:31:05.028 DEBUG:boto:Data: 2022-09-03T07:31:05.029 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.030 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.031 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.031 DEBUG:boto:Params: {} 2022-09-03T07:31:05.032 DEBUG:boto:Token: None 2022-09-03T07:31:05.033 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.034 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.035 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.038 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008f6bd69623fd2803-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.039 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.040 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:05.041 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:05.042 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.043 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:31:05.043 DEBUG:boto:Data: 2022-09-03T07:31:05.044 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.045 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.045 DEBUG:boto:Params: {} 2022-09-03T07:31:05.046 DEBUG:boto:Token: None 2022-09-03T07:31:05.047 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj148 2022-09-03T07:31:05.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SuIBeNub7kUYJrcan+xHZ/2QMig= 2022-09-03T07:31:05.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SuIBeNub7kUYJrcan+xHZ/2QMig=', 'Content-Length': '0'} 2022-09-03T07:31:05.053 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000514e60969c1648ff-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.054 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:05.054 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:05.055 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.055 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:31:05.056 DEBUG:boto:Data: 2022-09-03T07:31:05.056 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.057 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.057 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.058 DEBUG:boto:Params: {} 2022-09-03T07:31:05.059 DEBUG:boto:Token: None 2022-09-03T07:31:05.059 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj148 2022-09-03T07:31:05.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SuIBeNub7kUYJrcan+xHZ/2QMig= 2022-09-03T07:31:05.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SuIBeNub7kUYJrcan+xHZ/2QMig=', 'Content-Length': '0'} 2022-09-03T07:31:05.063 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ea517b4f6dc330e-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.063 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj148 2022-09-03T07:31:05.064 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:05.065 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:05.065 DEBUG:boto:Method: GET 2022-09-03T07:31:05.066 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:31:05.067 DEBUG:boto:Data: 2022-09-03T07:31:05.067 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.068 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.069 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.069 DEBUG:boto:Params: {} 2022-09-03T07:31:05.070 DEBUG:boto:Token: None 2022-09-03T07:31:05.070 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj148 2022-09-03T07:31:05.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cbXk5adEOo/kG3WSbqwx4+lqAsY= 2022-09-03T07:31:05.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cbXk5adEOo/kG3WSbqwx4+lqAsY=', 'Content-Length': '0'} 2022-09-03T07:31:05.074 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000020a2d43541a91b17-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.075 DEBUG:boto:path=/avysmw-75/obj148 2022-09-03T07:31:05.075 DEBUG:boto:auth_path=/avysmw-75/obj148 2022-09-03T07:31:05.076 DEBUG:boto:Method: GET 2022-09-03T07:31:05.076 DEBUG:boto:Path: /avysmw-75/obj148 2022-09-03T07:31:05.077 DEBUG:boto:Data: 2022-09-03T07:31:05.078 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.078 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.079 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.079 DEBUG:boto:Params: {} 2022-09-03T07:31:05.080 DEBUG:boto:Token: None 2022-09-03T07:31:05.081 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj148 2022-09-03T07:31:05.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cbXk5adEOo/kG3WSbqwx4+lqAsY= 2022-09-03T07:31:05.082 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cbXk5adEOo/kG3WSbqwx4+lqAsY=', 'Content-Length': '0'} 2022-09-03T07:31:05.084 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e19d9e83dfb0d611-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.130 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.131 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.131 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:31:05.132 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:31:05.132 DEBUG:boto:Method: GET 2022-09-03T07:31:05.133 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:31:05.134 DEBUG:boto:Data: 2022-09-03T07:31:05.134 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.135 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.136 DEBUG:boto:Params: {} 2022-09-03T07:31:05.136 DEBUG:boto:Token: None 2022-09-03T07:31:05.137 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.138 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.141 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a1eb90e8595af32-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.142 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.143 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.143 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.144 DEBUG:boto:path=/avysmw-75/?prefix=obj148 2022-09-03T07:31:05.144 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj148 2022-09-03T07:31:05.145 DEBUG:boto:Method: GET 2022-09-03T07:31:05.145 DEBUG:boto:Path: /avysmw-75/?prefix=obj148 2022-09-03T07:31:05.146 DEBUG:boto:Data: 2022-09-03T07:31:05.146 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.147 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.147 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.148 DEBUG:boto:Params: {} 2022-09-03T07:31:05.149 DEBUG:boto:Token: None 2022-09-03T07:31:05.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.154 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b99830e2070492e3-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.154 DEBUG:boto:b'avysmw-75obj1481000falseobj1482022-09-03T07:28:24.868Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.155 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:31:05.156 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.156 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.157 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.157 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.158 DEBUG:boto:Method: GET 2022-09-03T07:31:05.159 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.159 DEBUG:boto:Data: 2022-09-03T07:31:05.160 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.160 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.161 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.161 DEBUG:boto:Params: {} 2022-09-03T07:31:05.162 DEBUG:boto:Token: None 2022-09-03T07:31:05.163 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.163 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZBtLRqU2H+vQ79bS8aKg1ahqfg= 2022-09-03T07:31:05.164 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZBtLRqU2H+vQ79bS8aKg1ahqfg=', 'Content-Length': '0'} 2022-09-03T07:31:05.166 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c91bb7f2d127a25e-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.167 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.167 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.168 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.168 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.169 DEBUG:boto:Method: GET 2022-09-03T07:31:05.170 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.170 DEBUG:boto:Data: 2022-09-03T07:31:05.171 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.171 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.172 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.172 DEBUG:boto:Params: {} 2022-09-03T07:31:05.173 DEBUG:boto:Token: None 2022-09-03T07:31:05.174 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.174 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZBtLRqU2H+vQ79bS8aKg1ahqfg= 2022-09-03T07:31:05.175 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZBtLRqU2H+vQ79bS8aKg1ahqfg=', 'Content-Length': '0'} 2022-09-03T07:31:05.177 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f93ca9941657ee9-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.178 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.178 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.179 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.179 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.180 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.181 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.181 DEBUG:boto:Data: 2022-09-03T07:31:05.182 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.183 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.183 DEBUG:boto:Params: {} 2022-09-03T07:31:05.184 DEBUG:boto:Token: None 2022-09-03T07:31:05.185 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+m2fgwHPiPzKsPG6niJiHvJ6uSM= 2022-09-03T07:31:05.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+m2fgwHPiPzKsPG6niJiHvJ6uSM=', 'Content-Length': '0'} 2022-09-03T07:31:05.188 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9f3f8e478e59b11-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.189 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.190 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.190 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.191 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.191 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.192 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.193 DEBUG:boto:Data: 2022-09-03T07:31:05.193 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.194 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.194 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.195 DEBUG:boto:Params: {} 2022-09-03T07:31:05.196 DEBUG:boto:Token: None 2022-09-03T07:31:05.197 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.198 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+m2fgwHPiPzKsPG6niJiHvJ6uSM= 2022-09-03T07:31:05.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+m2fgwHPiPzKsPG6niJiHvJ6uSM=', 'Content-Length': '0'} 2022-09-03T07:31:05.202 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000007f2e893993afef0-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.203 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:31:05.203 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.204 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.204 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.205 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.206 DEBUG:boto:Method: GET 2022-09-03T07:31:05.206 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.207 DEBUG:boto:Data: 2022-09-03T07:31:05.207 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.208 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.208 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.209 DEBUG:boto:Params: {} 2022-09-03T07:31:05.210 DEBUG:boto:Token: None 2022-09-03T07:31:05.210 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.211 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZBtLRqU2H+vQ79bS8aKg1ahqfg= 2022-09-03T07:31:05.212 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZBtLRqU2H+vQ79bS8aKg1ahqfg=', 'Content-Length': '0'} 2022-09-03T07:31:05.214 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037a2b4b95783e67c-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.254 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.255 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.256 DEBUG:boto:path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.257 DEBUG:boto:auth_path=/avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.257 DEBUG:boto:Method: GET 2022-09-03T07:31:05.258 DEBUG:boto:Path: /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.258 DEBUG:boto:Data: 2022-09-03T07:31:05.259 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.260 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.260 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.261 DEBUG:boto:Params: {} 2022-09-03T07:31:05.262 DEBUG:boto:Token: None 2022-09-03T07:31:05.262 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155?versionId=null 2022-09-03T07:31:05.263 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QZBtLRqU2H+vQ79bS8aKg1ahqfg= 2022-09-03T07:31:05.264 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QZBtLRqU2H+vQ79bS8aKg1ahqfg=', 'Content-Length': '0'} 2022-09-03T07:31:05.266 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000baeb373d65453d32-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.267 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.268 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.268 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:31:05.269 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:31:05.269 DEBUG:boto:Method: GET 2022-09-03T07:31:05.270 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:31:05.271 DEBUG:boto:Data: 2022-09-03T07:31:05.271 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.272 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.272 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.273 DEBUG:boto:Params: {} 2022-09-03T07:31:05.273 DEBUG:boto:Token: None 2022-09-03T07:31:05.274 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.275 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.278 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aa90951cca1b6be1-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.279 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.280 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.281 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.282 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:31:05.282 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:31:05.283 DEBUG:boto:Method: GET 2022-09-03T07:31:05.284 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:31:05.284 DEBUG:boto:Data: 2022-09-03T07:31:05.285 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.285 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.286 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.287 DEBUG:boto:Params: {} 2022-09-03T07:31:05.290 DEBUG:boto:Token: None 2022-09-03T07:31:05.290 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.295 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bdbb0e6090709ad8-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.342 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.343 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.344 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.345 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.345 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:31:05.346 DEBUG:boto:Data: 2022-09-03T07:31:05.347 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.348 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.349 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.351 DEBUG:boto:Params: {} 2022-09-03T07:31:05.352 DEBUG:boto:Token: None 2022-09-03T07:31:05.353 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155 2022-09-03T07:31:05.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rkwv95McMUklLh8Ev85M00lkkKo= 2022-09-03T07:31:05.354 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rkwv95McMUklLh8Ev85M00lkkKo=', 'Content-Length': '0'} 2022-09-03T07:31:05.357 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8130c8cd4f03fb3-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.358 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.359 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.360 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.360 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:31:05.361 DEBUG:boto:Data: 2022-09-03T07:31:05.361 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.362 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.362 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.363 DEBUG:boto:Params: {} 2022-09-03T07:31:05.364 DEBUG:boto:Token: None 2022-09-03T07:31:05.364 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155 2022-09-03T07:31:05.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rkwv95McMUklLh8Ev85M00lkkKo= 2022-09-03T07:31:05.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rkwv95McMUklLh8Ev85M00lkkKo=', 'Content-Length': '0'} 2022-09-03T07:31:05.370 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d788d86f546c6f7-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.371 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj155 2022-09-03T07:31:05.372 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.373 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.373 DEBUG:boto:Method: GET 2022-09-03T07:31:05.374 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:31:05.375 DEBUG:boto:Data: 2022-09-03T07:31:05.376 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.376 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.377 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.378 DEBUG:boto:Params: {} 2022-09-03T07:31:05.379 DEBUG:boto:Token: None 2022-09-03T07:31:05.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155 2022-09-03T07:31:05.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Iqf5w3I9T+dPCWAK8QKRy3/GIg8= 2022-09-03T07:31:05.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Iqf5w3I9T+dPCWAK8QKRy3/GIg8=', 'Content-Length': '0'} 2022-09-03T07:31:05.387 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000023e58470531e407d-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.389 DEBUG:boto:path=/avysmw-75/obj155 2022-09-03T07:31:05.389 DEBUG:boto:auth_path=/avysmw-75/obj155 2022-09-03T07:31:05.390 DEBUG:boto:Method: GET 2022-09-03T07:31:05.391 DEBUG:boto:Path: /avysmw-75/obj155 2022-09-03T07:31:05.392 DEBUG:boto:Data: 2022-09-03T07:31:05.393 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.395 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.396 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.396 DEBUG:boto:Params: {} 2022-09-03T07:31:05.398 DEBUG:boto:Token: None 2022-09-03T07:31:05.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj155 2022-09-03T07:31:05.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Iqf5w3I9T+dPCWAK8QKRy3/GIg8= 2022-09-03T07:31:05.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Iqf5w3I9T+dPCWAK8QKRy3/GIg8=', 'Content-Length': '0'} 2022-09-03T07:31:05.404 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000af8c028663c64fa2-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.405 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.406 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.407 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:31:05.408 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:31:05.409 DEBUG:boto:Method: GET 2022-09-03T07:31:05.410 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:31:05.411 DEBUG:boto:Data: 2022-09-03T07:31:05.412 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.413 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.414 DEBUG:boto:Params: {} 2022-09-03T07:31:05.415 DEBUG:boto:Token: None 2022-09-03T07:31:05.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.420 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ee7c11096c989001-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.421 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.422 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.422 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.423 DEBUG:boto:path=/avysmw-75/?prefix=obj155 2022-09-03T07:31:05.424 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj155 2022-09-03T07:31:05.425 DEBUG:boto:Method: GET 2022-09-03T07:31:05.426 DEBUG:boto:Path: /avysmw-75/?prefix=obj155 2022-09-03T07:31:05.427 DEBUG:boto:Data: 2022-09-03T07:31:05.428 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.429 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.430 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.431 DEBUG:boto:Params: {} 2022-09-03T07:31:05.432 DEBUG:boto:Token: None 2022-09-03T07:31:05.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.438 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000da1cfbb261308950-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.438 DEBUG:boto:b'avysmw-75obj1551000falseobj1552022-09-03T07:26:11.240Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.439 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:31:05.440 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.440 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.441 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.441 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.442 DEBUG:boto:Method: GET 2022-09-03T07:31:05.442 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.443 DEBUG:boto:Data: 2022-09-03T07:31:05.443 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.444 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.445 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.445 DEBUG:boto:Params: {} 2022-09-03T07:31:05.446 DEBUG:boto:Token: None 2022-09-03T07:31:05.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ob9ewaLyQOqQsfROfuhQ6k/+1fs= 2022-09-03T07:31:05.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ob9ewaLyQOqQsfROfuhQ6k/+1fs=', 'Content-Length': '0'} 2022-09-03T07:31:05.450 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004b1df3860a474a6-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.451 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.451 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.452 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.453 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.453 DEBUG:boto:Method: GET 2022-09-03T07:31:05.454 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.454 DEBUG:boto:Data: 2022-09-03T07:31:05.455 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.455 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.456 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.456 DEBUG:boto:Params: {} 2022-09-03T07:31:05.457 DEBUG:boto:Token: None 2022-09-03T07:31:05.457 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ob9ewaLyQOqQsfROfuhQ6k/+1fs= 2022-09-03T07:31:05.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ob9ewaLyQOqQsfROfuhQ6k/+1fs=', 'Content-Length': '0'} 2022-09-03T07:31:05.461 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000036b7de5b67efd45f-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.462 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.463 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.463 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.464 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.464 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.465 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.465 DEBUG:boto:Data: 2022-09-03T07:31:05.466 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.466 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.467 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.468 DEBUG:boto:Params: {} 2022-09-03T07:31:05.468 DEBUG:boto:Token: None 2022-09-03T07:31:05.469 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.470 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q7rCK5RwJvDu26nvCqYQaTDi2Hs= 2022-09-03T07:31:05.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q7rCK5RwJvDu26nvCqYQaTDi2Hs=', 'Content-Length': '0'} 2022-09-03T07:31:05.473 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000134b490e9a25df21-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.474 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.475 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.476 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.477 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.478 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.479 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.480 DEBUG:boto:Data: 2022-09-03T07:31:05.481 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.481 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.483 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.484 DEBUG:boto:Params: {} 2022-09-03T07:31:05.485 DEBUG:boto:Token: None 2022-09-03T07:31:05.486 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.487 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q7rCK5RwJvDu26nvCqYQaTDi2Hs= 2022-09-03T07:31:05.489 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q7rCK5RwJvDu26nvCqYQaTDi2Hs=', 'Content-Length': '0'} 2022-09-03T07:31:05.492 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000afd7559b050fd15c-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.493 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:31:05.494 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.495 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.496 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.497 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.497 DEBUG:boto:Method: GET 2022-09-03T07:31:05.498 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.499 DEBUG:boto:Data: 2022-09-03T07:31:05.499 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.500 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.500 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.501 DEBUG:boto:Params: {} 2022-09-03T07:31:05.502 DEBUG:boto:Token: None 2022-09-03T07:31:05.502 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.503 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ob9ewaLyQOqQsfROfuhQ6k/+1fs= 2022-09-03T07:31:05.504 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ob9ewaLyQOqQsfROfuhQ6k/+1fs=', 'Content-Length': '0'} 2022-09-03T07:31:05.506 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009a28547b32e607de-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.507 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.508 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.508 DEBUG:boto:path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.509 DEBUG:boto:auth_path=/avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.509 DEBUG:boto:Method: GET 2022-09-03T07:31:05.510 DEBUG:boto:Path: /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.511 DEBUG:boto:Data: 2022-09-03T07:31:05.511 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.512 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.512 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.513 DEBUG:boto:Params: {} 2022-09-03T07:31:05.514 DEBUG:boto:Token: None 2022-09-03T07:31:05.515 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17?versionId=null 2022-09-03T07:31:05.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ob9ewaLyQOqQsfROfuhQ6k/+1fs= 2022-09-03T07:31:05.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ob9ewaLyQOqQsfROfuhQ6k/+1fs=', 'Content-Length': '0'} 2022-09-03T07:31:05.519 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003694276416fc4a2f-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.520 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.521 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.522 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:31:05.523 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:31:05.525 DEBUG:boto:Method: GET 2022-09-03T07:31:05.526 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:31:05.527 DEBUG:boto:Data: 2022-09-03T07:31:05.528 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.529 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.532 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.533 DEBUG:boto:Params: {} 2022-09-03T07:31:05.534 DEBUG:boto:Token: None 2022-09-03T07:31:05.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.536 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.537 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.540 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d995cc0b8cba4d7f-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.582 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.584 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.585 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.585 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:31:05.586 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:31:05.587 DEBUG:boto:Method: GET 2022-09-03T07:31:05.587 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:31:05.588 DEBUG:boto:Data: 2022-09-03T07:31:05.589 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.591 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.591 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.592 DEBUG:boto:Params: {} 2022-09-03T07:31:05.594 DEBUG:boto:Token: None 2022-09-03T07:31:05.595 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.596 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.598 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.602 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d60cbd03e46b1b34-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.603 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.604 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.604 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.605 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.606 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:31:05.606 DEBUG:boto:Data: 2022-09-03T07:31:05.607 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.607 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.608 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.608 DEBUG:boto:Params: {} 2022-09-03T07:31:05.609 DEBUG:boto:Token: None 2022-09-03T07:31:05.611 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17 2022-09-03T07:31:05.611 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oa2AD6PN5dAJ9KsBD3RGxW/T5P8= 2022-09-03T07:31:05.615 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oa2AD6PN5dAJ9KsBD3RGxW/T5P8=', 'Content-Length': '0'} 2022-09-03T07:31:05.622 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000027a3bd0d3f12f8b6-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.625 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.626 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.627 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.627 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:31:05.628 DEBUG:boto:Data: 2022-09-03T07:31:05.629 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.630 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.631 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.632 DEBUG:boto:Params: {} 2022-09-03T07:31:05.633 DEBUG:boto:Token: None 2022-09-03T07:31:05.634 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17 2022-09-03T07:31:05.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oa2AD6PN5dAJ9KsBD3RGxW/T5P8= 2022-09-03T07:31:05.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oa2AD6PN5dAJ9KsBD3RGxW/T5P8=', 'Content-Length': '0'} 2022-09-03T07:31:05.638 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005e13d88e598eebc9-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.639 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2022-09-03T07:31:05.639 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.640 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.641 DEBUG:boto:Method: GET 2022-09-03T07:31:05.641 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:31:05.642 DEBUG:boto:Data: 2022-09-03T07:31:05.643 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.644 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.645 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.646 DEBUG:boto:Params: {} 2022-09-03T07:31:05.647 DEBUG:boto:Token: None 2022-09-03T07:31:05.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17 2022-09-03T07:31:05.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YkVCWwIInvjQzyofQ1dXNrVt2Qk= 2022-09-03T07:31:05.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YkVCWwIInvjQzyofQ1dXNrVt2Qk=', 'Content-Length': '0'} 2022-09-03T07:31:05.653 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000041ddbdae822a6a19-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.654 DEBUG:boto:path=/avysmw-75/obj17 2022-09-03T07:31:05.655 DEBUG:boto:auth_path=/avysmw-75/obj17 2022-09-03T07:31:05.656 DEBUG:boto:Method: GET 2022-09-03T07:31:05.656 DEBUG:boto:Path: /avysmw-75/obj17 2022-09-03T07:31:05.657 DEBUG:boto:Data: 2022-09-03T07:31:05.657 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.658 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.659 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.660 DEBUG:boto:Params: {} 2022-09-03T07:31:05.661 DEBUG:boto:Token: None 2022-09-03T07:31:05.662 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj17 2022-09-03T07:31:05.663 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YkVCWwIInvjQzyofQ1dXNrVt2Qk= 2022-09-03T07:31:05.664 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YkVCWwIInvjQzyofQ1dXNrVt2Qk=', 'Content-Length': '0'} 2022-09-03T07:31:05.669 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007e68c579a7584d2c-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.671 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.672 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.673 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:31:05.674 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:31:05.675 DEBUG:boto:Method: GET 2022-09-03T07:31:05.677 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:31:05.678 DEBUG:boto:Data: 2022-09-03T07:31:05.679 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.680 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.682 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.683 DEBUG:boto:Params: {} 2022-09-03T07:31:05.684 DEBUG:boto:Token: None 2022-09-03T07:31:05.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.690 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000483892ffd3948d07-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.692 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.693 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.695 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.695 DEBUG:boto:path=/avysmw-75/?prefix=obj17 2022-09-03T07:31:05.696 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj17 2022-09-03T07:31:05.697 DEBUG:boto:Method: GET 2022-09-03T07:31:05.698 DEBUG:boto:Path: /avysmw-75/?prefix=obj17 2022-09-03T07:31:05.699 DEBUG:boto:Data: 2022-09-03T07:31:05.700 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.701 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.702 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.703 DEBUG:boto:Params: {} 2022-09-03T07:31:05.704 DEBUG:boto:Token: None 2022-09-03T07:31:05.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.710 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000032fbc870945a399c-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.754 DEBUG:boto:b'avysmw-75obj171000falseobj172022-09-03T07:12:49.441Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.755 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj185 2022-09-03T07:31:05.756 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.757 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.757 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.758 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.758 DEBUG:boto:Method: GET 2022-09-03T07:31:05.759 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.760 DEBUG:boto:Data: 2022-09-03T07:31:05.761 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.762 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.763 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.764 DEBUG:boto:Params: {} 2022-09-03T07:31:05.765 DEBUG:boto:Token: None 2022-09-03T07:31:05.766 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lF4htKwqBUriHYerpipHpXR1eW0= 2022-09-03T07:31:05.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lF4htKwqBUriHYerpipHpXR1eW0=', 'Content-Length': '0'} 2022-09-03T07:31:05.770 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001408567f0eac53e8-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.771 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.772 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.772 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.773 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.773 DEBUG:boto:Method: GET 2022-09-03T07:31:05.774 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.774 DEBUG:boto:Data: 2022-09-03T07:31:05.775 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.775 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.776 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.776 DEBUG:boto:Params: {} 2022-09-03T07:31:05.777 DEBUG:boto:Token: None 2022-09-03T07:31:05.778 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.778 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lF4htKwqBUriHYerpipHpXR1eW0= 2022-09-03T07:31:05.779 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lF4htKwqBUriHYerpipHpXR1eW0=', 'Content-Length': '0'} 2022-09-03T07:31:05.781 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ff3e128a8b725521-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.782 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.782 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.783 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.783 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.784 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.784 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.785 DEBUG:boto:Data: 2022-09-03T07:31:05.785 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.786 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.786 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.787 DEBUG:boto:Params: {} 2022-09-03T07:31:05.788 DEBUG:boto:Token: None 2022-09-03T07:31:05.788 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+BKFDAp9zrwSIFX7/sBCTRD3sag= 2022-09-03T07:31:05.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+BKFDAp9zrwSIFX7/sBCTRD3sag=', 'Content-Length': '0'} 2022-09-03T07:31:05.791 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000071aca8e9b65a0cd6-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.792 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.793 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.793 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.794 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.794 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.795 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.795 DEBUG:boto:Data: 2022-09-03T07:31:05.796 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.796 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.797 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.797 DEBUG:boto:Params: {} 2022-09-03T07:31:05.798 DEBUG:boto:Token: None 2022-09-03T07:31:05.799 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+BKFDAp9zrwSIFX7/sBCTRD3sag= 2022-09-03T07:31:05.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+BKFDAp9zrwSIFX7/sBCTRD3sag=', 'Content-Length': '0'} 2022-09-03T07:31:05.802 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c1f8a4e04e4a583-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.803 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj185 2022-09-03T07:31:05.803 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.804 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.804 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.805 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.805 DEBUG:boto:Method: GET 2022-09-03T07:31:05.806 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.806 DEBUG:boto:Data: 2022-09-03T07:31:05.807 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.807 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.808 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.808 DEBUG:boto:Params: {} 2022-09-03T07:31:05.809 DEBUG:boto:Token: None 2022-09-03T07:31:05.810 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.810 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lF4htKwqBUriHYerpipHpXR1eW0= 2022-09-03T07:31:05.811 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lF4htKwqBUriHYerpipHpXR1eW0=', 'Content-Length': '0'} 2022-09-03T07:31:05.813 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c2ae73c58bb8e00-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.858 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.859 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.859 DEBUG:boto:path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.860 DEBUG:boto:auth_path=/avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.860 DEBUG:boto:Method: GET 2022-09-03T07:31:05.861 DEBUG:boto:Path: /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.861 DEBUG:boto:Data: 2022-09-03T07:31:05.862 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.863 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.863 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.864 DEBUG:boto:Params: {} 2022-09-03T07:31:05.865 DEBUG:boto:Token: None 2022-09-03T07:31:05.865 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185?versionId=null 2022-09-03T07:31:05.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lF4htKwqBUriHYerpipHpXR1eW0= 2022-09-03T07:31:05.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lF4htKwqBUriHYerpipHpXR1eW0=', 'Content-Length': '0'} 2022-09-03T07:31:05.870 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e8d8fd19d840f6d8-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.871 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.872 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.872 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:31:05.873 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:31:05.874 DEBUG:boto:Method: GET 2022-09-03T07:31:05.874 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:31:05.875 DEBUG:boto:Data: 2022-09-03T07:31:05.877 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.877 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.879 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.880 DEBUG:boto:Params: {} 2022-09-03T07:31:05.880 DEBUG:boto:Token: None 2022-09-03T07:31:05.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd942c1d3f6afabd-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.890 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.891 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.892 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.892 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:31:05.893 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:31:05.893 DEBUG:boto:Method: GET 2022-09-03T07:31:05.894 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:31:05.894 DEBUG:boto:Data: 2022-09-03T07:31:05.895 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.895 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.896 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.896 DEBUG:boto:Params: {} 2022-09-03T07:31:05.897 DEBUG:boto:Token: None 2022-09-03T07:31:05.898 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.902 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b3b4bb466e15fb66-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.902 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.903 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.904 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.904 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.905 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:31:05.905 DEBUG:boto:Data: 2022-09-03T07:31:05.906 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.906 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.907 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.907 DEBUG:boto:Params: {} 2022-09-03T07:31:05.908 DEBUG:boto:Token: None 2022-09-03T07:31:05.909 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185 2022-09-03T07:31:05.910 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TafgOr84qNaapFoEu+ZVQMWP/0s= 2022-09-03T07:31:05.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TafgOr84qNaapFoEu+ZVQMWP/0s=', 'Content-Length': '0'} 2022-09-03T07:31:05.913 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000044ebd5409a4fc59-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.913 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.914 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.915 DEBUG:boto:Method: HEAD 2022-09-03T07:31:05.915 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:31:05.916 DEBUG:boto:Data: 2022-09-03T07:31:05.916 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.916 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.917 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.917 DEBUG:boto:Params: {} 2022-09-03T07:31:05.918 DEBUG:boto:Token: None 2022-09-03T07:31:05.919 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185 2022-09-03T07:31:05.919 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TafgOr84qNaapFoEu+ZVQMWP/0s= 2022-09-03T07:31:05.920 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TafgOr84qNaapFoEu+ZVQMWP/0s=', 'Content-Length': '0'} 2022-09-03T07:31:05.922 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d9409a165b897d56-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.922 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj185 2022-09-03T07:31:05.923 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.924 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.924 DEBUG:boto:Method: GET 2022-09-03T07:31:05.925 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:31:05.925 DEBUG:boto:Data: 2022-09-03T07:31:05.926 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.926 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.927 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.927 DEBUG:boto:Params: {} 2022-09-03T07:31:05.928 DEBUG:boto:Token: None 2022-09-03T07:31:05.928 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185 2022-09-03T07:31:05.929 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FxdH7ZwRiFqTY5iE+4jIE7hFd98= 2022-09-03T07:31:05.930 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FxdH7ZwRiFqTY5iE+4jIE7hFd98=', 'Content-Length': '0'} 2022-09-03T07:31:05.932 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006214c28bf98f8d51-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.933 DEBUG:boto:path=/avysmw-75/obj185 2022-09-03T07:31:05.933 DEBUG:boto:auth_path=/avysmw-75/obj185 2022-09-03T07:31:05.934 DEBUG:boto:Method: GET 2022-09-03T07:31:05.934 DEBUG:boto:Path: /avysmw-75/obj185 2022-09-03T07:31:05.935 DEBUG:boto:Data: 2022-09-03T07:31:05.935 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.936 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.936 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.937 DEBUG:boto:Params: {} 2022-09-03T07:31:05.938 DEBUG:boto:Token: None 2022-09-03T07:31:05.939 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/obj185 2022-09-03T07:31:05.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FxdH7ZwRiFqTY5iE+4jIE7hFd98= 2022-09-03T07:31:05.940 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FxdH7ZwRiFqTY5iE+4jIE7hFd98=', 'Content-Length': '0'} 2022-09-03T07:31:05.942 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b64d19c3267b038c-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.943 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.944 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.944 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:31:05.945 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:31:05.945 DEBUG:boto:Method: GET 2022-09-03T07:31:05.946 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:31:05.946 DEBUG:boto:Data: 2022-09-03T07:31:05.947 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:05.948 DEBUG:boto:Port: 8001 2022-09-03T07:31:05.948 DEBUG:boto:Params: {} 2022-09-03T07:31:05.949 DEBUG:boto:Token: None 2022-09-03T07:31:05.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.954 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009013da29ce62ed78-00631302b9-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:05.954 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:05.955 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:05.956 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:05.956 DEBUG:boto:path=/avysmw-75/?prefix=obj185 2022-09-03T07:31:05.957 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj185 2022-09-03T07:31:05.957 DEBUG:boto:Method: GET 2022-09-03T07:31:05.958 DEBUG:boto:Path: /avysmw-75/?prefix=obj185 2022-09-03T07:31:05.958 DEBUG:boto:Data: 2022-09-03T07:31:05.959 DEBUG:boto:Headers: {} 2022-09-03T07:31:05.959 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:05.960 DEBUG:boto:Port: 8000 2022-09-03T07:31:05.961 DEBUG:boto:Params: {} 2022-09-03T07:31:05.962 DEBUG:boto:Token: None 2022-09-03T07:31:05.962 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:05 GMT /avysmw-75/ 2022-09-03T07:31:05.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM= 2022-09-03T07:31:05.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:05 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G480icO10ecdX1MBhqk9TptEpNM=', 'Content-Length': '0'} 2022-09-03T07:31:05.966 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e837195dd1ebedda-00631302b9-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:05 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.006 DEBUG:boto:b'avysmw-75obj1851000falseobj1852022-09-03T07:28:24.941Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.007 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:31:06.008 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.009 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.010 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.010 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.011 DEBUG:boto:Method: GET 2022-09-03T07:31:06.011 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.012 DEBUG:boto:Data: 2022-09-03T07:31:06.012 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.013 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.014 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.014 DEBUG:boto:Params: {} 2022-09-03T07:31:06.015 DEBUG:boto:Token: None 2022-09-03T07:31:06.016 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.016 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TkKzm3yQ3wyNF3K/8GD2thMGGf4= 2022-09-03T07:31:06.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TkKzm3yQ3wyNF3K/8GD2thMGGf4=', 'Content-Length': '0'} 2022-09-03T07:31:06.019 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b6e7c616f95fd073-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.020 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.020 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.021 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.021 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.022 DEBUG:boto:Method: GET 2022-09-03T07:31:06.023 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.023 DEBUG:boto:Data: 2022-09-03T07:31:06.024 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.025 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.025 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.026 DEBUG:boto:Params: {} 2022-09-03T07:31:06.026 DEBUG:boto:Token: None 2022-09-03T07:31:06.027 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.028 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TkKzm3yQ3wyNF3K/8GD2thMGGf4= 2022-09-03T07:31:06.030 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TkKzm3yQ3wyNF3K/8GD2thMGGf4=', 'Content-Length': '0'} 2022-09-03T07:31:06.036 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002505cd90abdf068a-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.039 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.040 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.041 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.041 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.042 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.042 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.043 DEBUG:boto:Data: 2022-09-03T07:31:06.043 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.045 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.045 DEBUG:boto:Params: {} 2022-09-03T07:31:06.046 DEBUG:boto:Token: None 2022-09-03T07:31:06.046 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3dTG0iqcKdC8PVyazd1HY+wZwLU= 2022-09-03T07:31:06.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3dTG0iqcKdC8PVyazd1HY+wZwLU=', 'Content-Length': '0'} 2022-09-03T07:31:06.050 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000377595f5b3baf74b-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.051 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.051 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.053 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.054 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.055 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.055 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.056 DEBUG:boto:Data: 2022-09-03T07:31:06.056 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.057 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.057 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.058 DEBUG:boto:Params: {} 2022-09-03T07:31:06.059 DEBUG:boto:Token: None 2022-09-03T07:31:06.060 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.061 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3dTG0iqcKdC8PVyazd1HY+wZwLU= 2022-09-03T07:31:06.062 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3dTG0iqcKdC8PVyazd1HY+wZwLU=', 'Content-Length': '0'} 2022-09-03T07:31:06.064 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b997590a9852e817-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.066 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:31:06.066 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.067 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.067 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.068 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.069 DEBUG:boto:Method: GET 2022-09-03T07:31:06.069 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.070 DEBUG:boto:Data: 2022-09-03T07:31:06.071 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.074 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.075 DEBUG:boto:Params: {} 2022-09-03T07:31:06.080 DEBUG:boto:Token: None 2022-09-03T07:31:06.081 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.083 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TkKzm3yQ3wyNF3K/8GD2thMGGf4= 2022-09-03T07:31:06.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TkKzm3yQ3wyNF3K/8GD2thMGGf4=', 'Content-Length': '0'} 2022-09-03T07:31:06.086 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f78b982ffd56373-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.130 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.131 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.132 DEBUG:boto:path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.132 DEBUG:boto:auth_path=/avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.133 DEBUG:boto:Method: GET 2022-09-03T07:31:06.134 DEBUG:boto:Path: /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.136 DEBUG:boto:Data: 2022-09-03T07:31:06.137 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.138 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.138 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.139 DEBUG:boto:Params: {} 2022-09-03T07:31:06.140 DEBUG:boto:Token: None 2022-09-03T07:31:06.140 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19?versionId=null 2022-09-03T07:31:06.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TkKzm3yQ3wyNF3K/8GD2thMGGf4= 2022-09-03T07:31:06.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TkKzm3yQ3wyNF3K/8GD2thMGGf4=', 'Content-Length': '0'} 2022-09-03T07:31:06.145 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b5d150e6aaeceff-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.146 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.147 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.147 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:31:06.148 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:31:06.148 DEBUG:boto:Method: GET 2022-09-03T07:31:06.149 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:31:06.149 DEBUG:boto:Data: 2022-09-03T07:31:06.150 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.151 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.151 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.152 DEBUG:boto:Params: {} 2022-09-03T07:31:06.152 DEBUG:boto:Token: None 2022-09-03T07:31:06.153 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.154 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.154 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.158 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000094a432f07d8a2f07-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.159 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.160 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.161 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.162 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:31:06.163 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:31:06.164 DEBUG:boto:Method: GET 2022-09-03T07:31:06.165 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:31:06.166 DEBUG:boto:Data: 2022-09-03T07:31:06.168 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.170 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.172 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.173 DEBUG:boto:Params: {} 2022-09-03T07:31:06.174 DEBUG:boto:Token: None 2022-09-03T07:31:06.176 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.177 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.178 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.182 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e99044e156f6cdbf-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.183 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.184 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.185 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.186 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.187 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:31:06.188 DEBUG:boto:Data: 2022-09-03T07:31:06.189 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.190 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.192 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.193 DEBUG:boto:Params: {} 2022-09-03T07:31:06.194 DEBUG:boto:Token: None 2022-09-03T07:31:06.195 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19 2022-09-03T07:31:06.196 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EVejFROgOnHsIqOm1FCv4dfkh18= 2022-09-03T07:31:06.197 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EVejFROgOnHsIqOm1FCv4dfkh18=', 'Content-Length': '0'} 2022-09-03T07:31:06.200 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b5fd4ae8b88faa37-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.202 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.203 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.204 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.205 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:31:06.206 DEBUG:boto:Data: 2022-09-03T07:31:06.207 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.208 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.209 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.210 DEBUG:boto:Params: {} 2022-09-03T07:31:06.211 DEBUG:boto:Token: None 2022-09-03T07:31:06.212 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19 2022-09-03T07:31:06.212 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EVejFROgOnHsIqOm1FCv4dfkh18= 2022-09-03T07:31:06.213 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EVejFROgOnHsIqOm1FCv4dfkh18=', 'Content-Length': '0'} 2022-09-03T07:31:06.215 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f213c43b9724f624-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2022-09-03T07:31:06.217 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.217 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.218 DEBUG:boto:Method: GET 2022-09-03T07:31:06.219 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:31:06.219 DEBUG:boto:Data: 2022-09-03T07:31:06.220 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.221 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.222 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.222 DEBUG:boto:Params: {} 2022-09-03T07:31:06.223 DEBUG:boto:Token: None 2022-09-03T07:31:06.224 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19 2022-09-03T07:31:06.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w0+pQs1BancpK2N0ymmyv7mR9Vk= 2022-09-03T07:31:06.225 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w0+pQs1BancpK2N0ymmyv7mR9Vk=', 'Content-Length': '0'} 2022-09-03T07:31:06.228 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000031a984001990755-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.229 DEBUG:boto:path=/avysmw-75/obj19 2022-09-03T07:31:06.230 DEBUG:boto:auth_path=/avysmw-75/obj19 2022-09-03T07:31:06.231 DEBUG:boto:Method: GET 2022-09-03T07:31:06.232 DEBUG:boto:Path: /avysmw-75/obj19 2022-09-03T07:31:06.233 DEBUG:boto:Data: 2022-09-03T07:31:06.234 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.234 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.235 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.235 DEBUG:boto:Params: {} 2022-09-03T07:31:06.236 DEBUG:boto:Token: None 2022-09-03T07:31:06.237 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj19 2022-09-03T07:31:06.238 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:w0+pQs1BancpK2N0ymmyv7mR9Vk= 2022-09-03T07:31:06.238 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:w0+pQs1BancpK2N0ymmyv7mR9Vk=', 'Content-Length': '0'} 2022-09-03T07:31:06.240 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008ef0f4ee9eff369a-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.282 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.284 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.284 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:31:06.285 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:31:06.286 DEBUG:boto:Method: GET 2022-09-03T07:31:06.287 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:31:06.291 DEBUG:boto:Data: 2022-09-03T07:31:06.291 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.292 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.293 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.294 DEBUG:boto:Params: {} 2022-09-03T07:31:06.294 DEBUG:boto:Token: None 2022-09-03T07:31:06.295 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.296 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.297 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.301 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000110c51493b282d3c-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.302 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.303 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.303 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.304 DEBUG:boto:path=/avysmw-75/?prefix=obj19 2022-09-03T07:31:06.305 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj19 2022-09-03T07:31:06.305 DEBUG:boto:Method: GET 2022-09-03T07:31:06.306 DEBUG:boto:Path: /avysmw-75/?prefix=obj19 2022-09-03T07:31:06.307 DEBUG:boto:Data: 2022-09-03T07:31:06.309 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.310 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.313 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.314 DEBUG:boto:Params: {} 2022-09-03T07:31:06.316 DEBUG:boto:Token: None 2022-09-03T07:31:06.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.323 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002c101658f816b138-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.324 DEBUG:boto:b'avysmw-75obj191000falseobj192022-09-03T07:15:42.415Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.326 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:31:06.328 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.329 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.330 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.331 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.333 DEBUG:boto:Method: GET 2022-09-03T07:31:06.336 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.337 DEBUG:boto:Data: 2022-09-03T07:31:06.337 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.340 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.340 DEBUG:boto:Params: {} 2022-09-03T07:31:06.341 DEBUG:boto:Token: None 2022-09-03T07:31:06.343 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H1ahVVbq1B6L2I1CEHxzRlVMfRs= 2022-09-03T07:31:06.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H1ahVVbq1B6L2I1CEHxzRlVMfRs=', 'Content-Length': '0'} 2022-09-03T07:31:06.347 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eee07b366bf4486a-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.348 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.349 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.349 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.350 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.352 DEBUG:boto:Method: GET 2022-09-03T07:31:06.353 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.354 DEBUG:boto:Data: 2022-09-03T07:31:06.355 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.356 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.357 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.358 DEBUG:boto:Params: {} 2022-09-03T07:31:06.359 DEBUG:boto:Token: None 2022-09-03T07:31:06.359 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.360 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H1ahVVbq1B6L2I1CEHxzRlVMfRs= 2022-09-03T07:31:06.361 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H1ahVVbq1B6L2I1CEHxzRlVMfRs=', 'Content-Length': '0'} 2022-09-03T07:31:06.364 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f33c5c8032cc08a1-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.366 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.367 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.369 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.370 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.372 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.373 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.375 DEBUG:boto:Data: 2022-09-03T07:31:06.376 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.377 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.379 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.380 DEBUG:boto:Params: {} 2022-09-03T07:31:06.382 DEBUG:boto:Token: None 2022-09-03T07:31:06.383 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.384 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LG3E1ygqwb+e54EXMWBXOfZjPSk= 2022-09-03T07:31:06.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LG3E1ygqwb+e54EXMWBXOfZjPSk=', 'Content-Length': '0'} 2022-09-03T07:31:06.387 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c5ed5639a9bdb4a1-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.388 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.388 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.389 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.389 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.390 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.391 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.392 DEBUG:boto:Data: 2022-09-03T07:31:06.392 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.393 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.393 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.394 DEBUG:boto:Params: {} 2022-09-03T07:31:06.395 DEBUG:boto:Token: None 2022-09-03T07:31:06.396 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.396 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LG3E1ygqwb+e54EXMWBXOfZjPSk= 2022-09-03T07:31:06.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LG3E1ygqwb+e54EXMWBXOfZjPSk=', 'Content-Length': '0'} 2022-09-03T07:31:06.400 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000424c11d7971fb964-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.401 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:31:06.402 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.403 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.403 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.404 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.405 DEBUG:boto:Method: GET 2022-09-03T07:31:06.407 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.408 DEBUG:boto:Data: 2022-09-03T07:31:06.409 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.411 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.412 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.413 DEBUG:boto:Params: {} 2022-09-03T07:31:06.414 DEBUG:boto:Token: None 2022-09-03T07:31:06.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.416 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H1ahVVbq1B6L2I1CEHxzRlVMfRs= 2022-09-03T07:31:06.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H1ahVVbq1B6L2I1CEHxzRlVMfRs=', 'Content-Length': '0'} 2022-09-03T07:31:06.421 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a1ef31dc35aab0e-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.463 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.464 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.465 DEBUG:boto:path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.466 DEBUG:boto:auth_path=/avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.467 DEBUG:boto:Method: GET 2022-09-03T07:31:06.468 DEBUG:boto:Path: /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.468 DEBUG:boto:Data: 2022-09-03T07:31:06.469 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.469 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.471 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.472 DEBUG:boto:Params: {} 2022-09-03T07:31:06.473 DEBUG:boto:Token: None 2022-09-03T07:31:06.473 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22?versionId=null 2022-09-03T07:31:06.475 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H1ahVVbq1B6L2I1CEHxzRlVMfRs= 2022-09-03T07:31:06.475 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H1ahVVbq1B6L2I1CEHxzRlVMfRs=', 'Content-Length': '0'} 2022-09-03T07:31:06.477 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000089beb9adea280f21-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.478 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.479 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.479 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:31:06.480 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:31:06.480 DEBUG:boto:Method: GET 2022-09-03T07:31:06.481 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:31:06.481 DEBUG:boto:Data: 2022-09-03T07:31:06.482 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.482 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.483 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.484 DEBUG:boto:Params: {} 2022-09-03T07:31:06.484 DEBUG:boto:Token: None 2022-09-03T07:31:06.485 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.486 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.489 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006e253398e029ee73-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.490 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.491 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.491 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.492 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:31:06.492 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:31:06.493 DEBUG:boto:Method: GET 2022-09-03T07:31:06.493 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:31:06.494 DEBUG:boto:Data: 2022-09-03T07:31:06.494 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.495 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.495 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.496 DEBUG:boto:Params: {} 2022-09-03T07:31:06.497 DEBUG:boto:Token: None 2022-09-03T07:31:06.497 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.498 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.501 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000079d3d1669b6401fb-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.502 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.503 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.503 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.504 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.504 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:31:06.505 DEBUG:boto:Data: 2022-09-03T07:31:06.505 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.506 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.506 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.507 DEBUG:boto:Params: {} 2022-09-03T07:31:06.507 DEBUG:boto:Token: None 2022-09-03T07:31:06.508 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22 2022-09-03T07:31:06.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NeuuIQkqDtrb91tcTQcKPg2S/NE= 2022-09-03T07:31:06.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NeuuIQkqDtrb91tcTQcKPg2S/NE=', 'Content-Length': '0'} 2022-09-03T07:31:06.511 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000042fe03d0d9621714-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.512 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.512 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.513 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.514 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:31:06.514 DEBUG:boto:Data: 2022-09-03T07:31:06.515 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.515 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.516 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.516 DEBUG:boto:Params: {} 2022-09-03T07:31:06.517 DEBUG:boto:Token: None 2022-09-03T07:31:06.517 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22 2022-09-03T07:31:06.518 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NeuuIQkqDtrb91tcTQcKPg2S/NE= 2022-09-03T07:31:06.518 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NeuuIQkqDtrb91tcTQcKPg2S/NE=', 'Content-Length': '0'} 2022-09-03T07:31:06.520 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ca4c634290e318b7-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:31:06.522 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.522 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.523 DEBUG:boto:Method: GET 2022-09-03T07:31:06.523 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:31:06.524 DEBUG:boto:Data: 2022-09-03T07:31:06.524 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.525 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.525 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.526 DEBUG:boto:Params: {} 2022-09-03T07:31:06.527 DEBUG:boto:Token: None 2022-09-03T07:31:06.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22 2022-09-03T07:31:06.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zr1nFw7TMdJCXvNT+jroid+qm0g= 2022-09-03T07:31:06.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zr1nFw7TMdJCXvNT+jroid+qm0g=', 'Content-Length': '0'} 2022-09-03T07:31:06.531 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f795480cf522ae0-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.531 DEBUG:boto:path=/avysmw-75/obj22 2022-09-03T07:31:06.532 DEBUG:boto:auth_path=/avysmw-75/obj22 2022-09-03T07:31:06.532 DEBUG:boto:Method: GET 2022-09-03T07:31:06.533 DEBUG:boto:Path: /avysmw-75/obj22 2022-09-03T07:31:06.533 DEBUG:boto:Data: 2022-09-03T07:31:06.534 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.535 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.535 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.536 DEBUG:boto:Params: {} 2022-09-03T07:31:06.536 DEBUG:boto:Token: None 2022-09-03T07:31:06.537 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj22 2022-09-03T07:31:06.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zr1nFw7TMdJCXvNT+jroid+qm0g= 2022-09-03T07:31:06.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zr1nFw7TMdJCXvNT+jroid+qm0g=', 'Content-Length': '0'} 2022-09-03T07:31:06.540 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000024603c9f42d78b91-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.541 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.541 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.542 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:31:06.542 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:31:06.543 DEBUG:boto:Method: GET 2022-09-03T07:31:06.543 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:31:06.544 DEBUG:boto:Data: 2022-09-03T07:31:06.544 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.546 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.546 DEBUG:boto:Params: {} 2022-09-03T07:31:06.547 DEBUG:boto:Token: None 2022-09-03T07:31:06.547 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.548 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.552 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005ed2b00638fb9d3d-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.553 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.555 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.556 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.557 DEBUG:boto:path=/avysmw-75/?prefix=obj22 2022-09-03T07:31:06.557 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj22 2022-09-03T07:31:06.558 DEBUG:boto:Method: GET 2022-09-03T07:31:06.559 DEBUG:boto:Path: /avysmw-75/?prefix=obj22 2022-09-03T07:31:06.560 DEBUG:boto:Data: 2022-09-03T07:31:06.561 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.562 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.563 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.564 DEBUG:boto:Params: {} 2022-09-03T07:31:06.565 DEBUG:boto:Token: None 2022-09-03T07:31:06.566 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.567 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.569 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.573 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f27092a1af96d1ff-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.574 DEBUG:boto:b'avysmw-75obj221000falseobj222022-09-03T07:11:00.952Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.575 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:31:06.576 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.577 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.578 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.579 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.580 DEBUG:boto:Method: GET 2022-09-03T07:31:06.581 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.581 DEBUG:boto:Data: 2022-09-03T07:31:06.582 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.583 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.584 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.585 DEBUG:boto:Params: {} 2022-09-03T07:31:06.586 DEBUG:boto:Token: None 2022-09-03T07:31:06.587 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.589 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hRRtl1c4bR5g1ZgB526DV2m1uUU= 2022-09-03T07:31:06.590 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hRRtl1c4bR5g1ZgB526DV2m1uUU=', 'Content-Length': '0'} 2022-09-03T07:31:06.593 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003235eb4c1afe9ec4-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.594 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.595 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.596 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.597 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.598 DEBUG:boto:Method: GET 2022-09-03T07:31:06.599 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.600 DEBUG:boto:Data: 2022-09-03T07:31:06.601 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.602 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.603 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.604 DEBUG:boto:Params: {} 2022-09-03T07:31:06.605 DEBUG:boto:Token: None 2022-09-03T07:31:06.606 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.607 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hRRtl1c4bR5g1ZgB526DV2m1uUU= 2022-09-03T07:31:06.608 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hRRtl1c4bR5g1ZgB526DV2m1uUU=', 'Content-Length': '0'} 2022-09-03T07:31:06.611 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096766b75973a9657-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.612 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.613 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.614 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.615 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.616 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.616 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.617 DEBUG:boto:Data: 2022-09-03T07:31:06.617 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.618 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.618 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.619 DEBUG:boto:Params: {} 2022-09-03T07:31:06.619 DEBUG:boto:Token: None 2022-09-03T07:31:06.620 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.620 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:22efUsPmOVY+iCSimFSZnThfM+I= 2022-09-03T07:31:06.621 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:22efUsPmOVY+iCSimFSZnThfM+I=', 'Content-Length': '0'} 2022-09-03T07:31:06.623 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d322cf30d8196039-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.624 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.625 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.625 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.626 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.627 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.628 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.628 DEBUG:boto:Data: 2022-09-03T07:31:06.629 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.630 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.630 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.631 DEBUG:boto:Params: {} 2022-09-03T07:31:06.633 DEBUG:boto:Token: None 2022-09-03T07:31:06.634 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:22efUsPmOVY+iCSimFSZnThfM+I= 2022-09-03T07:31:06.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:22efUsPmOVY+iCSimFSZnThfM+I=', 'Content-Length': '0'} 2022-09-03T07:31:06.638 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000569c1a07ea72fdbf-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.639 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:31:06.640 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.641 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.641 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.642 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.643 DEBUG:boto:Method: GET 2022-09-03T07:31:06.644 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.644 DEBUG:boto:Data: 2022-09-03T07:31:06.645 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.646 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.646 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.647 DEBUG:boto:Params: {} 2022-09-03T07:31:06.647 DEBUG:boto:Token: None 2022-09-03T07:31:06.648 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.649 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hRRtl1c4bR5g1ZgB526DV2m1uUU= 2022-09-03T07:31:06.650 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hRRtl1c4bR5g1ZgB526DV2m1uUU=', 'Content-Length': '0'} 2022-09-03T07:31:06.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005ae7fc88fd4a7e71-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.653 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.654 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.655 DEBUG:boto:path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.655 DEBUG:boto:auth_path=/avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.656 DEBUG:boto:Method: GET 2022-09-03T07:31:06.657 DEBUG:boto:Path: /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.657 DEBUG:boto:Data: 2022-09-03T07:31:06.658 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.658 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.659 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.659 DEBUG:boto:Params: {} 2022-09-03T07:31:06.660 DEBUG:boto:Token: None 2022-09-03T07:31:06.661 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23?versionId=null 2022-09-03T07:31:06.661 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hRRtl1c4bR5g1ZgB526DV2m1uUU= 2022-09-03T07:31:06.662 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hRRtl1c4bR5g1ZgB526DV2m1uUU=', 'Content-Length': '0'} 2022-09-03T07:31:06.666 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c32e0305f6dc4a6d-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.710 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.711 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.711 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:31:06.712 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:31:06.713 DEBUG:boto:Method: GET 2022-09-03T07:31:06.713 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:31:06.714 DEBUG:boto:Data: 2022-09-03T07:31:06.714 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.715 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.716 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.716 DEBUG:boto:Params: {} 2022-09-03T07:31:06.717 DEBUG:boto:Token: None 2022-09-03T07:31:06.717 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.718 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.719 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.722 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f191b368e2f06146-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.723 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.724 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.725 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.726 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:31:06.727 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:31:06.728 DEBUG:boto:Method: GET 2022-09-03T07:31:06.729 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:31:06.730 DEBUG:boto:Data: 2022-09-03T07:31:06.731 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.732 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.732 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.733 DEBUG:boto:Params: {} 2022-09-03T07:31:06.733 DEBUG:boto:Token: None 2022-09-03T07:31:06.734 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.735 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.740 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000998672c2ad995e63-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.741 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.742 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.743 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.744 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.744 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:31:06.745 DEBUG:boto:Data: 2022-09-03T07:31:06.745 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.746 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.746 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.747 DEBUG:boto:Params: {} 2022-09-03T07:31:06.748 DEBUG:boto:Token: None 2022-09-03T07:31:06.748 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23 2022-09-03T07:31:06.749 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0cIEkRKXdMyVX6aNfTr2VvfzmA4= 2022-09-03T07:31:06.749 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0cIEkRKXdMyVX6aNfTr2VvfzmA4=', 'Content-Length': '0'} 2022-09-03T07:31:06.752 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000764fd5778d519ca4-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.753 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.753 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.754 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.755 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:31:06.755 DEBUG:boto:Data: 2022-09-03T07:31:06.756 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.756 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.757 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.757 DEBUG:boto:Params: {} 2022-09-03T07:31:06.758 DEBUG:boto:Token: None 2022-09-03T07:31:06.758 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23 2022-09-03T07:31:06.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0cIEkRKXdMyVX6aNfTr2VvfzmA4= 2022-09-03T07:31:06.760 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0cIEkRKXdMyVX6aNfTr2VvfzmA4=', 'Content-Length': '0'} 2022-09-03T07:31:06.762 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa5bbfe8d1bd874a-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.763 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2022-09-03T07:31:06.763 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.764 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.765 DEBUG:boto:Method: GET 2022-09-03T07:31:06.765 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:31:06.766 DEBUG:boto:Data: 2022-09-03T07:31:06.767 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.768 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.768 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.769 DEBUG:boto:Params: {} 2022-09-03T07:31:06.769 DEBUG:boto:Token: None 2022-09-03T07:31:06.770 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23 2022-09-03T07:31:06.771 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x/3xg2uFgkhRGESPJjuFpMo5i3U= 2022-09-03T07:31:06.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x/3xg2uFgkhRGESPJjuFpMo5i3U=', 'Content-Length': '0'} 2022-09-03T07:31:06.774 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000638248aceedaf5ac-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.776 DEBUG:boto:path=/avysmw-75/obj23 2022-09-03T07:31:06.777 DEBUG:boto:auth_path=/avysmw-75/obj23 2022-09-03T07:31:06.777 DEBUG:boto:Method: GET 2022-09-03T07:31:06.778 DEBUG:boto:Path: /avysmw-75/obj23 2022-09-03T07:31:06.779 DEBUG:boto:Data: 2022-09-03T07:31:06.780 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.781 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.783 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.785 DEBUG:boto:Params: {} 2022-09-03T07:31:06.786 DEBUG:boto:Token: None 2022-09-03T07:31:06.787 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj23 2022-09-03T07:31:06.788 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x/3xg2uFgkhRGESPJjuFpMo5i3U= 2022-09-03T07:31:06.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x/3xg2uFgkhRGESPJjuFpMo5i3U=', 'Content-Length': '0'} 2022-09-03T07:31:06.794 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aac81f4d45e02f9f-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.837 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.838 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.839 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:31:06.839 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:31:06.841 DEBUG:boto:Method: GET 2022-09-03T07:31:06.842 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:31:06.842 DEBUG:boto:Data: 2022-09-03T07:31:06.843 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.844 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.844 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.846 DEBUG:boto:Params: {} 2022-09-03T07:31:06.847 DEBUG:boto:Token: None 2022-09-03T07:31:06.847 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.848 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.849 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.852 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000807a0e52989eaaac-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.853 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.855 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:06.856 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:06.857 DEBUG:boto:path=/avysmw-75/?prefix=obj23 2022-09-03T07:31:06.858 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj23 2022-09-03T07:31:06.859 DEBUG:boto:Method: GET 2022-09-03T07:31:06.860 DEBUG:boto:Path: /avysmw-75/?prefix=obj23 2022-09-03T07:31:06.861 DEBUG:boto:Data: 2022-09-03T07:31:06.862 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.863 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.864 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.864 DEBUG:boto:Params: {} 2022-09-03T07:31:06.865 DEBUG:boto:Token: None 2022-09-03T07:31:06.865 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/ 2022-09-03T07:31:06.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM= 2022-09-03T07:31:06.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:N+JGl2fdBGlA16lyvdFtHQixwfM=', 'Content-Length': '0'} 2022-09-03T07:31:06.870 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005db3e9c826a4942b-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.871 DEBUG:boto:b'avysmw-75obj231000falseobj232022-09-03T07:21:21.541Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:06.873 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:31:06.874 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:06.875 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:06.876 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.877 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.878 DEBUG:boto:Method: GET 2022-09-03T07:31:06.879 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.879 DEBUG:boto:Data: 2022-09-03T07:31:06.880 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.880 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.881 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.882 DEBUG:boto:Params: {} 2022-09-03T07:31:06.882 DEBUG:boto:Token: None 2022-09-03T07:31:06.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uF9bT5DG3o8dcFiZvsvlmcDuYM4= 2022-09-03T07:31:06.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uF9bT5DG3o8dcFiZvsvlmcDuYM4=', 'Content-Length': '0'} 2022-09-03T07:31:06.887 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f534ac30555a652c-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.934 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:06.935 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:06.935 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.936 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.936 DEBUG:boto:Method: GET 2022-09-03T07:31:06.937 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.938 DEBUG:boto:Data: 2022-09-03T07:31:06.938 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.939 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.940 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.940 DEBUG:boto:Params: {} 2022-09-03T07:31:06.941 DEBUG:boto:Token: None 2022-09-03T07:31:06.941 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.942 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uF9bT5DG3o8dcFiZvsvlmcDuYM4= 2022-09-03T07:31:06.943 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uF9bT5DG3o8dcFiZvsvlmcDuYM4=', 'Content-Length': '0'} 2022-09-03T07:31:06.946 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e11d8b30b0c7c690-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.947 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:06.947 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:06.948 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.949 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.950 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.950 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.951 DEBUG:boto:Data: 2022-09-03T07:31:06.951 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.952 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.952 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.953 DEBUG:boto:Params: {} 2022-09-03T07:31:06.953 DEBUG:boto:Token: None 2022-09-03T07:31:06.954 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YLJIL8vDq/gH2R2Iw+n+t3xZnjg= 2022-09-03T07:31:06.955 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YLJIL8vDq/gH2R2Iw+n+t3xZnjg=', 'Content-Length': '0'} 2022-09-03T07:31:06.958 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002fcf930a0a3f533a-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.959 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:06.959 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:06.960 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.960 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.961 DEBUG:boto:Method: HEAD 2022-09-03T07:31:06.961 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.962 DEBUG:boto:Data: 2022-09-03T07:31:06.963 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.963 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.964 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.965 DEBUG:boto:Params: {} 2022-09-03T07:31:06.966 DEBUG:boto:Token: None 2022-09-03T07:31:06.966 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YLJIL8vDq/gH2R2Iw+n+t3xZnjg= 2022-09-03T07:31:06.967 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YLJIL8vDq/gH2R2Iw+n+t3xZnjg=', 'Content-Length': '0'} 2022-09-03T07:31:06.969 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000898da9ddc0e1cb29-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.970 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:31:06.971 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:06.972 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:06.972 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.973 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.973 DEBUG:boto:Method: GET 2022-09-03T07:31:06.974 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.974 DEBUG:boto:Data: 2022-09-03T07:31:06.975 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.976 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:06.976 DEBUG:boto:Port: 8001 2022-09-03T07:31:06.977 DEBUG:boto:Params: {} 2022-09-03T07:31:06.978 DEBUG:boto:Token: None 2022-09-03T07:31:06.978 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uF9bT5DG3o8dcFiZvsvlmcDuYM4= 2022-09-03T07:31:06.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uF9bT5DG3o8dcFiZvsvlmcDuYM4=', 'Content-Length': '0'} 2022-09-03T07:31:06.986 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001019aed8cbe5e7e4-00631302ba-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:06.987 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:06.988 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:06.988 DEBUG:boto:path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.989 DEBUG:boto:auth_path=/avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.989 DEBUG:boto:Method: GET 2022-09-03T07:31:06.991 DEBUG:boto:Path: /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.991 DEBUG:boto:Data: 2022-09-03T07:31:06.992 DEBUG:boto:Headers: {} 2022-09-03T07:31:06.993 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:06.993 DEBUG:boto:Port: 8000 2022-09-03T07:31:06.994 DEBUG:boto:Params: {} 2022-09-03T07:31:06.995 DEBUG:boto:Token: None 2022-09-03T07:31:06.996 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:06 GMT /avysmw-75/obj29?versionId=null 2022-09-03T07:31:06.996 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uF9bT5DG3o8dcFiZvsvlmcDuYM4= 2022-09-03T07:31:06.997 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:06 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uF9bT5DG3o8dcFiZvsvlmcDuYM4=', 'Content-Length': '0'} 2022-09-03T07:31:07.004 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a04dc0eba034f831-00631302ba-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:06 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.005 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.007 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.009 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:31:07.010 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:31:07.011 DEBUG:boto:Method: GET 2022-09-03T07:31:07.012 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:31:07.015 DEBUG:boto:Data: 2022-09-03T07:31:07.016 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.018 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.019 DEBUG:boto:Params: {} 2022-09-03T07:31:07.020 DEBUG:boto:Token: None 2022-09-03T07:31:07.021 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.022 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.023 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.029 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000334951fcd826e209-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.030 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.032 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.033 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.034 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:31:07.035 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:31:07.036 DEBUG:boto:Method: GET 2022-09-03T07:31:07.036 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:31:07.037 DEBUG:boto:Data: 2022-09-03T07:31:07.037 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.038 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.039 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.039 DEBUG:boto:Params: {} 2022-09-03T07:31:07.040 DEBUG:boto:Token: None 2022-09-03T07:31:07.041 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.041 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.044 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000071182f301124f01f-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.045 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.046 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:07.047 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:07.047 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.048 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:31:07.048 DEBUG:boto:Data: 2022-09-03T07:31:07.049 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.050 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.050 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.051 DEBUG:boto:Params: {} 2022-09-03T07:31:07.051 DEBUG:boto:Token: None 2022-09-03T07:31:07.052 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj29 2022-09-03T07:31:07.053 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tCz5QX07we9yYHIk220BW5SlP1k= 2022-09-03T07:31:07.053 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tCz5QX07we9yYHIk220BW5SlP1k=', 'Content-Length': '0'} 2022-09-03T07:31:07.055 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000035deb8d88af8f2a2-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.056 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:07.057 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:07.057 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.058 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:31:07.058 DEBUG:boto:Data: 2022-09-03T07:31:07.059 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.060 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.060 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.061 DEBUG:boto:Params: {} 2022-09-03T07:31:07.062 DEBUG:boto:Token: None 2022-09-03T07:31:07.063 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj29 2022-09-03T07:31:07.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tCz5QX07we9yYHIk220BW5SlP1k= 2022-09-03T07:31:07.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tCz5QX07we9yYHIk220BW5SlP1k=', 'Content-Length': '0'} 2022-09-03T07:31:07.067 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e8386db40076925d-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.068 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2022-09-03T07:31:07.069 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:07.070 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:07.070 DEBUG:boto:Method: GET 2022-09-03T07:31:07.071 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:31:07.072 DEBUG:boto:Data: 2022-09-03T07:31:07.072 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.073 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.073 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.074 DEBUG:boto:Params: {} 2022-09-03T07:31:07.075 DEBUG:boto:Token: None 2022-09-03T07:31:07.075 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj29 2022-09-03T07:31:07.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YaCNEuJ4Pa/M5gRP2+Wikf1bu78= 2022-09-03T07:31:07.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YaCNEuJ4Pa/M5gRP2+Wikf1bu78=', 'Content-Length': '0'} 2022-09-03T07:31:07.080 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006703e90d4764b6aa-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.081 DEBUG:boto:path=/avysmw-75/obj29 2022-09-03T07:31:07.082 DEBUG:boto:auth_path=/avysmw-75/obj29 2022-09-03T07:31:07.083 DEBUG:boto:Method: GET 2022-09-03T07:31:07.084 DEBUG:boto:Path: /avysmw-75/obj29 2022-09-03T07:31:07.084 DEBUG:boto:Data: 2022-09-03T07:31:07.085 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.086 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.086 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.087 DEBUG:boto:Params: {} 2022-09-03T07:31:07.088 DEBUG:boto:Token: None 2022-09-03T07:31:07.088 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj29 2022-09-03T07:31:07.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YaCNEuJ4Pa/M5gRP2+Wikf1bu78= 2022-09-03T07:31:07.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YaCNEuJ4Pa/M5gRP2+Wikf1bu78=', 'Content-Length': '0'} 2022-09-03T07:31:07.093 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007d0418d7bbe77eea-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.094 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.095 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.097 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:31:07.097 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:31:07.098 DEBUG:boto:Method: GET 2022-09-03T07:31:07.098 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:31:07.099 DEBUG:boto:Data: 2022-09-03T07:31:07.100 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.101 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.102 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.103 DEBUG:boto:Params: {} 2022-09-03T07:31:07.105 DEBUG:boto:Token: None 2022-09-03T07:31:07.106 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.107 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.108 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.111 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000538a85f80899fdf9-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.113 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.114 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.115 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.116 DEBUG:boto:path=/avysmw-75/?prefix=obj29 2022-09-03T07:31:07.117 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj29 2022-09-03T07:31:07.118 DEBUG:boto:Method: GET 2022-09-03T07:31:07.119 DEBUG:boto:Path: /avysmw-75/?prefix=obj29 2022-09-03T07:31:07.120 DEBUG:boto:Data: 2022-09-03T07:31:07.121 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.122 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.123 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.124 DEBUG:boto:Params: {} 2022-09-03T07:31:07.125 DEBUG:boto:Token: None 2022-09-03T07:31:07.127 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.128 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.129 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.132 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6b07af1fdf0181f-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.133 DEBUG:boto:b'avysmw-75obj291000falseobj292022-09-03T07:24:23.601Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.133 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:31:07.134 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.135 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.135 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.136 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.136 DEBUG:boto:Method: GET 2022-09-03T07:31:07.137 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.137 DEBUG:boto:Data: 2022-09-03T07:31:07.138 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.138 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.139 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.139 DEBUG:boto:Params: {} 2022-09-03T07:31:07.140 DEBUG:boto:Token: None 2022-09-03T07:31:07.141 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.141 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gTgXm0ppHTM2uYW9ttGjLMpvE6c= 2022-09-03T07:31:07.142 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gTgXm0ppHTM2uYW9ttGjLMpvE6c=', 'Content-Length': '0'} 2022-09-03T07:31:07.144 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000028fa392163915583-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.193 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.194 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.195 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.195 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.196 DEBUG:boto:Method: GET 2022-09-03T07:31:07.196 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.198 DEBUG:boto:Data: 2022-09-03T07:31:07.199 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.200 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.201 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.201 DEBUG:boto:Params: {} 2022-09-03T07:31:07.202 DEBUG:boto:Token: None 2022-09-03T07:31:07.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gTgXm0ppHTM2uYW9ttGjLMpvE6c= 2022-09-03T07:31:07.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gTgXm0ppHTM2uYW9ttGjLMpvE6c=', 'Content-Length': '0'} 2022-09-03T07:31:07.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f570e62eb870647b-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.209 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.209 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.210 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.211 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.212 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.212 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.213 DEBUG:boto:Data: 2022-09-03T07:31:07.213 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.214 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.214 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.215 DEBUG:boto:Params: {} 2022-09-03T07:31:07.216 DEBUG:boto:Token: None 2022-09-03T07:31:07.216 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6IAnwjsYHjG/amBeUaos/Mr4Ik= 2022-09-03T07:31:07.218 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6IAnwjsYHjG/amBeUaos/Mr4Ik=', 'Content-Length': '0'} 2022-09-03T07:31:07.220 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009e6c92375a371896-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.221 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.222 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.222 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.223 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.223 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.224 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.225 DEBUG:boto:Data: 2022-09-03T07:31:07.225 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.226 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.226 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.227 DEBUG:boto:Params: {} 2022-09-03T07:31:07.228 DEBUG:boto:Token: None 2022-09-03T07:31:07.229 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.230 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n6IAnwjsYHjG/amBeUaos/Mr4Ik= 2022-09-03T07:31:07.231 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n6IAnwjsYHjG/amBeUaos/Mr4Ik=', 'Content-Length': '0'} 2022-09-03T07:31:07.234 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e7483045cb9543f5-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.235 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:31:07.235 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.236 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.237 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.238 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.239 DEBUG:boto:Method: GET 2022-09-03T07:31:07.240 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.241 DEBUG:boto:Data: 2022-09-03T07:31:07.242 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.243 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.244 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.245 DEBUG:boto:Params: {} 2022-09-03T07:31:07.246 DEBUG:boto:Token: None 2022-09-03T07:31:07.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.249 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gTgXm0ppHTM2uYW9ttGjLMpvE6c= 2022-09-03T07:31:07.250 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gTgXm0ppHTM2uYW9ttGjLMpvE6c=', 'Content-Length': '0'} 2022-09-03T07:31:07.252 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3f1e348ab3c81cf-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.254 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.255 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.256 DEBUG:boto:path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.257 DEBUG:boto:auth_path=/avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.258 DEBUG:boto:Method: GET 2022-09-03T07:31:07.259 DEBUG:boto:Path: /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.260 DEBUG:boto:Data: 2022-09-03T07:31:07.261 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.262 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.263 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.264 DEBUG:boto:Params: {} 2022-09-03T07:31:07.266 DEBUG:boto:Token: None 2022-09-03T07:31:07.267 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31?versionId=null 2022-09-03T07:31:07.268 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gTgXm0ppHTM2uYW9ttGjLMpvE6c= 2022-09-03T07:31:07.270 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gTgXm0ppHTM2uYW9ttGjLMpvE6c=', 'Content-Length': '0'} 2022-09-03T07:31:07.272 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001437f4187a156ef8-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.274 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.275 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.275 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:31:07.276 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:31:07.276 DEBUG:boto:Method: GET 2022-09-03T07:31:07.277 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:31:07.277 DEBUG:boto:Data: 2022-09-03T07:31:07.278 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.280 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.281 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.282 DEBUG:boto:Params: {} 2022-09-03T07:31:07.283 DEBUG:boto:Token: None 2022-09-03T07:31:07.284 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.285 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.290 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c610f4fa6e465b90-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.291 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.293 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.294 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.294 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:31:07.295 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:31:07.296 DEBUG:boto:Method: GET 2022-09-03T07:31:07.297 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:31:07.298 DEBUG:boto:Data: 2022-09-03T07:31:07.299 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.300 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.302 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.303 DEBUG:boto:Params: {} 2022-09-03T07:31:07.304 DEBUG:boto:Token: None 2022-09-03T07:31:07.305 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.306 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.309 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000222d81a22a7e052e-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.310 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.311 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.311 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.312 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.312 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:31:07.313 DEBUG:boto:Data: 2022-09-03T07:31:07.313 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.314 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.315 DEBUG:boto:Params: {} 2022-09-03T07:31:07.315 DEBUG:boto:Token: None 2022-09-03T07:31:07.316 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31 2022-09-03T07:31:07.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5HlEAiviwGa82R/7V4SwHzH1YI4= 2022-09-03T07:31:07.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5HlEAiviwGa82R/7V4SwHzH1YI4=', 'Content-Length': '0'} 2022-09-03T07:31:07.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce2b1500a436b79b-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.321 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.321 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.322 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.323 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:31:07.323 DEBUG:boto:Data: 2022-09-03T07:31:07.324 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.325 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.326 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.327 DEBUG:boto:Params: {} 2022-09-03T07:31:07.328 DEBUG:boto:Token: None 2022-09-03T07:31:07.328 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31 2022-09-03T07:31:07.329 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5HlEAiviwGa82R/7V4SwHzH1YI4= 2022-09-03T07:31:07.334 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5HlEAiviwGa82R/7V4SwHzH1YI4=', 'Content-Length': '0'} 2022-09-03T07:31:07.336 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000900adedd3331e22c-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.337 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2022-09-03T07:31:07.338 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.339 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.339 DEBUG:boto:Method: GET 2022-09-03T07:31:07.340 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:31:07.340 DEBUG:boto:Data: 2022-09-03T07:31:07.342 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.343 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.344 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.344 DEBUG:boto:Params: {} 2022-09-03T07:31:07.345 DEBUG:boto:Token: None 2022-09-03T07:31:07.346 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31 2022-09-03T07:31:07.347 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MDQZBIDRxWPvihIzo3iRB3Z/jNc= 2022-09-03T07:31:07.347 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MDQZBIDRxWPvihIzo3iRB3Z/jNc=', 'Content-Length': '0'} 2022-09-03T07:31:07.349 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000048787208152c5f3e-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.350 DEBUG:boto:path=/avysmw-75/obj31 2022-09-03T07:31:07.351 DEBUG:boto:auth_path=/avysmw-75/obj31 2022-09-03T07:31:07.352 DEBUG:boto:Method: GET 2022-09-03T07:31:07.353 DEBUG:boto:Path: /avysmw-75/obj31 2022-09-03T07:31:07.354 DEBUG:boto:Data: 2022-09-03T07:31:07.355 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.356 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.357 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.358 DEBUG:boto:Params: {} 2022-09-03T07:31:07.360 DEBUG:boto:Token: None 2022-09-03T07:31:07.361 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj31 2022-09-03T07:31:07.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:MDQZBIDRxWPvihIzo3iRB3Z/jNc= 2022-09-03T07:31:07.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:MDQZBIDRxWPvihIzo3iRB3Z/jNc=', 'Content-Length': '0'} 2022-09-03T07:31:07.366 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:10 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b59fb8353b74bce-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.367 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.368 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.369 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:31:07.370 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:31:07.371 DEBUG:boto:Method: GET 2022-09-03T07:31:07.372 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:31:07.373 DEBUG:boto:Data: 2022-09-03T07:31:07.374 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.375 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.376 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.377 DEBUG:boto:Params: {} 2022-09-03T07:31:07.378 DEBUG:boto:Token: None 2022-09-03T07:31:07.379 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.381 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.385 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000072d5cb5f98e06623-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.386 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.388 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.388 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.389 DEBUG:boto:path=/avysmw-75/?prefix=obj31 2022-09-03T07:31:07.390 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj31 2022-09-03T07:31:07.391 DEBUG:boto:Method: GET 2022-09-03T07:31:07.392 DEBUG:boto:Path: /avysmw-75/?prefix=obj31 2022-09-03T07:31:07.393 DEBUG:boto:Data: 2022-09-03T07:31:07.394 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.395 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.396 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.397 DEBUG:boto:Params: {} 2022-09-03T07:31:07.397 DEBUG:boto:Token: None 2022-09-03T07:31:07.398 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.399 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.399 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.403 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006a53e19328af22cc-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.405 DEBUG:boto:b'avysmw-75obj311000falseobj312022-09-03T07:26:10.978Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.406 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:31:07.407 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.409 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.411 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.412 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.415 DEBUG:boto:Method: GET 2022-09-03T07:31:07.416 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.417 DEBUG:boto:Data: 2022-09-03T07:31:07.418 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.419 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.420 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.421 DEBUG:boto:Params: {} 2022-09-03T07:31:07.422 DEBUG:boto:Token: None 2022-09-03T07:31:07.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H0B0nur5RnV9J8KeFr7nnviXo7g= 2022-09-03T07:31:07.425 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H0B0nur5RnV9J8KeFr7nnviXo7g=', 'Content-Length': '0'} 2022-09-03T07:31:07.429 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d71b1c020d2e8bb5-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.435 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.436 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.437 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.439 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.440 DEBUG:boto:Method: GET 2022-09-03T07:31:07.441 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.442 DEBUG:boto:Data: 2022-09-03T07:31:07.444 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.444 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.445 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.445 DEBUG:boto:Params: {} 2022-09-03T07:31:07.446 DEBUG:boto:Token: None 2022-09-03T07:31:07.447 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H0B0nur5RnV9J8KeFr7nnviXo7g= 2022-09-03T07:31:07.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H0B0nur5RnV9J8KeFr7nnviXo7g=', 'Content-Length': '0'} 2022-09-03T07:31:07.451 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d14afc595ed2258a-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.498 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.499 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.500 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.500 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.501 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.502 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.503 DEBUG:boto:Data: 2022-09-03T07:31:07.504 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.505 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.507 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.508 DEBUG:boto:Params: {} 2022-09-03T07:31:07.509 DEBUG:boto:Token: None 2022-09-03T07:31:07.509 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.510 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:90UpAY1Aw91C6KkvNmZxx7w95d8= 2022-09-03T07:31:07.512 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:90UpAY1Aw91C6KkvNmZxx7w95d8=', 'Content-Length': '0'} 2022-09-03T07:31:07.515 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003160a065b80303c8-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.516 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.517 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.518 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.519 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.519 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.520 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.522 DEBUG:boto:Data: 2022-09-03T07:31:07.523 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.524 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.524 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.525 DEBUG:boto:Params: {} 2022-09-03T07:31:07.526 DEBUG:boto:Token: None 2022-09-03T07:31:07.527 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.528 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:90UpAY1Aw91C6KkvNmZxx7w95d8= 2022-09-03T07:31:07.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:90UpAY1Aw91C6KkvNmZxx7w95d8=', 'Content-Length': '0'} 2022-09-03T07:31:07.530 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000001e6afdd18ad307b-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.531 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:31:07.532 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.533 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.534 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.534 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.535 DEBUG:boto:Method: GET 2022-09-03T07:31:07.536 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.536 DEBUG:boto:Data: 2022-09-03T07:31:07.537 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.538 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.539 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.539 DEBUG:boto:Params: {} 2022-09-03T07:31:07.540 DEBUG:boto:Token: None 2022-09-03T07:31:07.541 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.541 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H0B0nur5RnV9J8KeFr7nnviXo7g= 2022-09-03T07:31:07.542 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H0B0nur5RnV9J8KeFr7nnviXo7g=', 'Content-Length': '0'} 2022-09-03T07:31:07.545 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000095f07af0e66f4201-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.545 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.546 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.547 DEBUG:boto:path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.547 DEBUG:boto:auth_path=/avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.548 DEBUG:boto:Method: GET 2022-09-03T07:31:07.548 DEBUG:boto:Path: /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.549 DEBUG:boto:Data: 2022-09-03T07:31:07.549 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.550 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.551 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.551 DEBUG:boto:Params: {} 2022-09-03T07:31:07.552 DEBUG:boto:Token: None 2022-09-03T07:31:07.552 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33?versionId=null 2022-09-03T07:31:07.553 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H0B0nur5RnV9J8KeFr7nnviXo7g= 2022-09-03T07:31:07.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H0B0nur5RnV9J8KeFr7nnviXo7g=', 'Content-Length': '0'} 2022-09-03T07:31:07.556 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000886ff9503cf4589d-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.557 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.558 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.558 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:31:07.559 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:31:07.559 DEBUG:boto:Method: GET 2022-09-03T07:31:07.560 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:31:07.560 DEBUG:boto:Data: 2022-09-03T07:31:07.561 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.561 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.562 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.563 DEBUG:boto:Params: {} 2022-09-03T07:31:07.563 DEBUG:boto:Token: None 2022-09-03T07:31:07.564 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.564 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.566 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.569 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000038701bffaebcd530-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.570 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.571 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.572 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.572 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:31:07.573 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:31:07.573 DEBUG:boto:Method: GET 2022-09-03T07:31:07.574 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:31:07.576 DEBUG:boto:Data: 2022-09-03T07:31:07.577 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.578 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.579 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.580 DEBUG:boto:Params: {} 2022-09-03T07:31:07.581 DEBUG:boto:Token: None 2022-09-03T07:31:07.582 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.583 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.584 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.589 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000896c86ee426d234c-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.591 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.592 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.592 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.593 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.593 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:31:07.594 DEBUG:boto:Data: 2022-09-03T07:31:07.595 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.596 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.597 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.598 DEBUG:boto:Params: {} 2022-09-03T07:31:07.599 DEBUG:boto:Token: None 2022-09-03T07:31:07.600 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33 2022-09-03T07:31:07.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u0BF/ahHYvoz1N9Q0AOq39jOIL8= 2022-09-03T07:31:07.602 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u0BF/ahHYvoz1N9Q0AOq39jOIL8=', 'Content-Length': '0'} 2022-09-03T07:31:07.604 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007dcd061c47662305-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.606 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.606 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.607 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.607 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:31:07.608 DEBUG:boto:Data: 2022-09-03T07:31:07.608 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.609 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.610 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.610 DEBUG:boto:Params: {} 2022-09-03T07:31:07.611 DEBUG:boto:Token: None 2022-09-03T07:31:07.612 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33 2022-09-03T07:31:07.612 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:u0BF/ahHYvoz1N9Q0AOq39jOIL8= 2022-09-03T07:31:07.613 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:u0BF/ahHYvoz1N9Q0AOq39jOIL8=', 'Content-Length': '0'} 2022-09-03T07:31:07.616 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002b5229969108b8bc-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.616 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:31:07.617 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.618 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.618 DEBUG:boto:Method: GET 2022-09-03T07:31:07.619 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:31:07.619 DEBUG:boto:Data: 2022-09-03T07:31:07.620 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.621 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.621 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.622 DEBUG:boto:Params: {} 2022-09-03T07:31:07.623 DEBUG:boto:Token: None 2022-09-03T07:31:07.624 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33 2022-09-03T07:31:07.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o6f9grfr+s4tAtZuGjz9YvPJRIQ= 2022-09-03T07:31:07.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o6f9grfr+s4tAtZuGjz9YvPJRIQ=', 'Content-Length': '0'} 2022-09-03T07:31:07.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009767beb5da99fee-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.634 DEBUG:boto:path=/avysmw-75/obj33 2022-09-03T07:31:07.634 DEBUG:boto:auth_path=/avysmw-75/obj33 2022-09-03T07:31:07.635 DEBUG:boto:Method: GET 2022-09-03T07:31:07.636 DEBUG:boto:Path: /avysmw-75/obj33 2022-09-03T07:31:07.637 DEBUG:boto:Data: 2022-09-03T07:31:07.638 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.638 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.639 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.640 DEBUG:boto:Params: {} 2022-09-03T07:31:07.641 DEBUG:boto:Token: None 2022-09-03T07:31:07.641 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj33 2022-09-03T07:31:07.643 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o6f9grfr+s4tAtZuGjz9YvPJRIQ= 2022-09-03T07:31:07.644 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o6f9grfr+s4tAtZuGjz9YvPJRIQ=', 'Content-Length': '0'} 2022-09-03T07:31:07.646 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:00 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000062f9fda80911f854-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.686 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.687 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.687 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:31:07.688 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:31:07.688 DEBUG:boto:Method: GET 2022-09-03T07:31:07.689 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:31:07.690 DEBUG:boto:Data: 2022-09-03T07:31:07.691 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.691 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.692 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.692 DEBUG:boto:Params: {} 2022-09-03T07:31:07.693 DEBUG:boto:Token: None 2022-09-03T07:31:07.694 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.694 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.699 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000076ba5fb030d8bc8c-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.703 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.704 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.704 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.705 DEBUG:boto:path=/avysmw-75/?prefix=obj33 2022-09-03T07:31:07.706 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj33 2022-09-03T07:31:07.706 DEBUG:boto:Method: GET 2022-09-03T07:31:07.707 DEBUG:boto:Path: /avysmw-75/?prefix=obj33 2022-09-03T07:31:07.707 DEBUG:boto:Data: 2022-09-03T07:31:07.710 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.711 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.712 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.712 DEBUG:boto:Params: {} 2022-09-03T07:31:07.713 DEBUG:boto:Token: None 2022-09-03T07:31:07.714 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.716 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.717 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.721 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000db8f1e7a33faf09c-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.722 DEBUG:boto:b'avysmw-75obj331000falseobj332022-09-03T07:11:00.977Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.727 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:31:07.729 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.730 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.731 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.731 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.732 DEBUG:boto:Method: GET 2022-09-03T07:31:07.732 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.733 DEBUG:boto:Data: 2022-09-03T07:31:07.733 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.734 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.735 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.735 DEBUG:boto:Params: {} 2022-09-03T07:31:07.736 DEBUG:boto:Token: None 2022-09-03T07:31:07.736 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.737 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JMI1fCY+nQzzY6xTxeDHSngusrQ= 2022-09-03T07:31:07.737 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JMI1fCY+nQzzY6xTxeDHSngusrQ=', 'Content-Length': '0'} 2022-09-03T07:31:07.740 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e2129cf4dfcfdb0-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.786 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.787 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.787 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.788 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.789 DEBUG:boto:Method: GET 2022-09-03T07:31:07.789 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.790 DEBUG:boto:Data: 2022-09-03T07:31:07.790 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.791 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.792 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.792 DEBUG:boto:Params: {} 2022-09-03T07:31:07.793 DEBUG:boto:Token: None 2022-09-03T07:31:07.794 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.794 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JMI1fCY+nQzzY6xTxeDHSngusrQ= 2022-09-03T07:31:07.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JMI1fCY+nQzzY6xTxeDHSngusrQ=', 'Content-Length': '0'} 2022-09-03T07:31:07.797 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce716ed33883663c-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.798 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.799 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.799 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.800 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.801 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.802 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.803 DEBUG:boto:Data: 2022-09-03T07:31:07.803 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.804 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.804 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.805 DEBUG:boto:Params: {} 2022-09-03T07:31:07.805 DEBUG:boto:Token: None 2022-09-03T07:31:07.806 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.807 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UvStc44hn0wmBr5mIB3WLhRyCW0= 2022-09-03T07:31:07.808 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UvStc44hn0wmBr5mIB3WLhRyCW0=', 'Content-Length': '0'} 2022-09-03T07:31:07.809 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a3091e74cb3b1f98-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.810 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.811 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.812 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.814 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.814 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.815 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.815 DEBUG:boto:Data: 2022-09-03T07:31:07.816 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.816 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.817 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.817 DEBUG:boto:Params: {} 2022-09-03T07:31:07.818 DEBUG:boto:Token: None 2022-09-03T07:31:07.819 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.819 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UvStc44hn0wmBr5mIB3WLhRyCW0= 2022-09-03T07:31:07.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UvStc44hn0wmBr5mIB3WLhRyCW0=', 'Content-Length': '0'} 2022-09-03T07:31:07.822 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000053903ef7f7ec77ff-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.824 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:31:07.824 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.825 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.826 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.827 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.827 DEBUG:boto:Method: GET 2022-09-03T07:31:07.828 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.829 DEBUG:boto:Data: 2022-09-03T07:31:07.830 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.833 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.834 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.836 DEBUG:boto:Params: {} 2022-09-03T07:31:07.838 DEBUG:boto:Token: None 2022-09-03T07:31:07.840 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.841 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JMI1fCY+nQzzY6xTxeDHSngusrQ= 2022-09-03T07:31:07.842 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JMI1fCY+nQzzY6xTxeDHSngusrQ=', 'Content-Length': '0'} 2022-09-03T07:31:07.845 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000083925c57ed91ca4a-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.847 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.848 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.849 DEBUG:boto:path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.850 DEBUG:boto:auth_path=/avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.851 DEBUG:boto:Method: GET 2022-09-03T07:31:07.852 DEBUG:boto:Path: /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.853 DEBUG:boto:Data: 2022-09-03T07:31:07.856 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.858 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.859 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.860 DEBUG:boto:Params: {} 2022-09-03T07:31:07.861 DEBUG:boto:Token: None 2022-09-03T07:31:07.862 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34?versionId=null 2022-09-03T07:31:07.863 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JMI1fCY+nQzzY6xTxeDHSngusrQ= 2022-09-03T07:31:07.865 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JMI1fCY+nQzzY6xTxeDHSngusrQ=', 'Content-Length': '0'} 2022-09-03T07:31:07.868 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000871a72a71652ef65-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.869 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.870 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.871 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:31:07.872 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:31:07.873 DEBUG:boto:Method: GET 2022-09-03T07:31:07.874 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:31:07.875 DEBUG:boto:Data: 2022-09-03T07:31:07.878 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.880 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.881 DEBUG:boto:Params: {} 2022-09-03T07:31:07.882 DEBUG:boto:Token: None 2022-09-03T07:31:07.883 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.884 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.889 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a07d2168ec46d383-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.890 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.890 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.891 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.892 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:31:07.892 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:31:07.893 DEBUG:boto:Method: GET 2022-09-03T07:31:07.893 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:31:07.894 DEBUG:boto:Data: 2022-09-03T07:31:07.895 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.895 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.896 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.896 DEBUG:boto:Params: {} 2022-09-03T07:31:07.897 DEBUG:boto:Token: None 2022-09-03T07:31:07.898 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.902 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000825d981213e17c77-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.903 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.904 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.905 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.906 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.907 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:31:07.908 DEBUG:boto:Data: 2022-09-03T07:31:07.909 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.909 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.910 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.911 DEBUG:boto:Params: {} 2022-09-03T07:31:07.913 DEBUG:boto:Token: None 2022-09-03T07:31:07.913 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34 2022-09-03T07:31:07.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P01G5n+hrdwisU5C5KuJZvMDykk= 2022-09-03T07:31:07.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P01G5n+hrdwisU5C5KuJZvMDykk=', 'Content-Length': '0'} 2022-09-03T07:31:07.918 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c412df7855e5544-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.919 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.919 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.920 DEBUG:boto:Method: HEAD 2022-09-03T07:31:07.921 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:31:07.921 DEBUG:boto:Data: 2022-09-03T07:31:07.922 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.923 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.924 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.924 DEBUG:boto:Params: {} 2022-09-03T07:31:07.925 DEBUG:boto:Token: None 2022-09-03T07:31:07.926 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34 2022-09-03T07:31:07.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P01G5n+hrdwisU5C5KuJZvMDykk= 2022-09-03T07:31:07.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P01G5n+hrdwisU5C5KuJZvMDykk=', 'Content-Length': '0'} 2022-09-03T07:31:07.929 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006d1dce9dcb5a0a31-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.930 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2022-09-03T07:31:07.930 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.931 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.932 DEBUG:boto:Method: GET 2022-09-03T07:31:07.932 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:31:07.933 DEBUG:boto:Data: 2022-09-03T07:31:07.933 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.934 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.934 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.935 DEBUG:boto:Params: {} 2022-09-03T07:31:07.936 DEBUG:boto:Token: None 2022-09-03T07:31:07.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34 2022-09-03T07:31:07.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jRBaEBcvkykUaXOX0zMGRG7tz5g= 2022-09-03T07:31:07.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jRBaEBcvkykUaXOX0zMGRG7tz5g=', 'Content-Length': '0'} 2022-09-03T07:31:07.940 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000200b1e5cf3c0c066-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.941 DEBUG:boto:path=/avysmw-75/obj34 2022-09-03T07:31:07.942 DEBUG:boto:auth_path=/avysmw-75/obj34 2022-09-03T07:31:07.942 DEBUG:boto:Method: GET 2022-09-03T07:31:07.943 DEBUG:boto:Path: /avysmw-75/obj34 2022-09-03T07:31:07.944 DEBUG:boto:Data: 2022-09-03T07:31:07.945 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.946 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.947 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.947 DEBUG:boto:Params: {} 2022-09-03T07:31:07.948 DEBUG:boto:Token: None 2022-09-03T07:31:07.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj34 2022-09-03T07:31:07.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jRBaEBcvkykUaXOX0zMGRG7tz5g= 2022-09-03T07:31:07.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jRBaEBcvkykUaXOX0zMGRG7tz5g=', 'Content-Length': '0'} 2022-09-03T07:31:07.954 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000639211cf0757c136-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.955 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.956 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.957 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:31:07.958 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:31:07.959 DEBUG:boto:Method: GET 2022-09-03T07:31:07.960 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:31:07.961 DEBUG:boto:Data: 2022-09-03T07:31:07.962 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.963 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.964 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.964 DEBUG:boto:Params: {} 2022-09-03T07:31:07.966 DEBUG:boto:Token: None 2022-09-03T07:31:07.966 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.967 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.968 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.970 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fbcf2708cadeb807-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.971 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.972 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:07.972 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:07.973 DEBUG:boto:path=/avysmw-75/?prefix=obj34 2022-09-03T07:31:07.973 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj34 2022-09-03T07:31:07.974 DEBUG:boto:Method: GET 2022-09-03T07:31:07.975 DEBUG:boto:Path: /avysmw-75/?prefix=obj34 2022-09-03T07:31:07.975 DEBUG:boto:Data: 2022-09-03T07:31:07.976 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.976 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:07.977 DEBUG:boto:Port: 8000 2022-09-03T07:31:07.977 DEBUG:boto:Params: {} 2022-09-03T07:31:07.978 DEBUG:boto:Token: None 2022-09-03T07:31:07.979 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/ 2022-09-03T07:31:07.979 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4= 2022-09-03T07:31:07.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Fuzm4jLSJ2c3+iRsK2XkkP+iQO4=', 'Content-Length': '0'} 2022-09-03T07:31:07.982 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000602e039459236fad-00631302bb-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.983 DEBUG:boto:b'avysmw-75obj341000falseobj342022-09-03T07:12:49.510Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:07.984 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2022-09-03T07:31:07.985 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:07.985 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:07.986 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:07.986 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:07.987 DEBUG:boto:Method: GET 2022-09-03T07:31:07.988 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:31:07.988 DEBUG:boto:Data: 2022-09-03T07:31:07.989 DEBUG:boto:Headers: {} 2022-09-03T07:31:07.989 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:07.990 DEBUG:boto:Port: 8001 2022-09-03T07:31:07.991 DEBUG:boto:Params: {} 2022-09-03T07:31:07.991 DEBUG:boto:Token: None 2022-09-03T07:31:07.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:07 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:31:07.994 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NBn8qPJ0IN+IFthwIMnY1O4dWvw= 2022-09-03T07:31:07.995 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:07 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NBn8qPJ0IN+IFthwIMnY1O4dWvw=', 'Content-Length': '0'} 2022-09-03T07:31:07.998 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cc44ca923f9f930-00631302bb-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:07 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:07.999 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:08.000 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:08.001 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.002 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.003 DEBUG:boto:Method: GET 2022-09-03T07:31:08.004 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.005 DEBUG:boto:Data: 2022-09-03T07:31:08.006 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.006 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.007 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.007 DEBUG:boto:Params: {} 2022-09-03T07:31:08.008 DEBUG:boto:Token: None 2022-09-03T07:31:08.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/dhxwZV37rXGyiCTGYerTOS3Hwk= 2022-09-03T07:31:08.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/dhxwZV37rXGyiCTGYerTOS3Hwk=', 'Content-Length': '0'} 2022-09-03T07:31:08.012 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008c37ad5905d0cda-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.058 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:08.059 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:08.060 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.060 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.061 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.061 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.062 DEBUG:boto:Data: 2022-09-03T07:31:08.062 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.063 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.063 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.064 DEBUG:boto:Params: {} 2022-09-03T07:31:08.065 DEBUG:boto:Token: None 2022-09-03T07:31:08.065 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.066 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xhmn9VlXE2a90bUq68fCrkusmAk= 2022-09-03T07:31:08.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xhmn9VlXE2a90bUq68fCrkusmAk=', 'Content-Length': '0'} 2022-09-03T07:31:08.069 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007f64f5d7fcfe635a-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.069 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:08.070 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:08.071 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.071 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.071 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.072 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.072 DEBUG:boto:Data: 2022-09-03T07:31:08.073 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.073 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.074 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.074 DEBUG:boto:Params: {} 2022-09-03T07:31:08.075 DEBUG:boto:Token: None 2022-09-03T07:31:08.076 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xhmn9VlXE2a90bUq68fCrkusmAk= 2022-09-03T07:31:08.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xhmn9VlXE2a90bUq68fCrkusmAk=', 'Content-Length': '0'} 2022-09-03T07:31:08.079 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a50287b67e3054b3-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.080 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2022-09-03T07:31:08.081 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:08.082 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:08.082 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.083 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.084 DEBUG:boto:Method: GET 2022-09-03T07:31:08.084 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.085 DEBUG:boto:Data: 2022-09-03T07:31:08.085 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.086 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.087 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.088 DEBUG:boto:Params: {} 2022-09-03T07:31:08.089 DEBUG:boto:Token: None 2022-09-03T07:31:08.089 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/dhxwZV37rXGyiCTGYerTOS3Hwk= 2022-09-03T07:31:08.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/dhxwZV37rXGyiCTGYerTOS3Hwk=', 'Content-Length': '0'} 2022-09-03T07:31:08.106 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005a78f23a0aeb1629-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.107 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:08.108 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:08.109 DEBUG:boto:path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.110 DEBUG:boto:auth_path=/avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.112 DEBUG:boto:Method: GET 2022-09-03T07:31:08.113 DEBUG:boto:Path: /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.114 DEBUG:boto:Data: 2022-09-03T07:31:08.115 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.116 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.117 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.118 DEBUG:boto:Params: {} 2022-09-03T07:31:08.119 DEBUG:boto:Token: None 2022-09-03T07:31:08.121 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj37?versionId=null 2022-09-03T07:31:08.122 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/dhxwZV37rXGyiCTGYerTOS3Hwk= 2022-09-03T07:31:08.123 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/dhxwZV37rXGyiCTGYerTOS3Hwk=', 'Content-Length': '0'} 2022-09-03T07:31:08.126 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a488fbbc286cae4d-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.126 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.127 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.127 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:31:08.128 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:31:08.128 DEBUG:boto:Method: GET 2022-09-03T07:31:08.129 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:31:08.130 DEBUG:boto:Data: 2022-09-03T07:31:08.131 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.132 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.133 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.134 DEBUG:boto:Params: {} 2022-09-03T07:31:08.136 DEBUG:boto:Token: None 2022-09-03T07:31:08.137 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.138 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.139 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.142 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003228841f3b53065a-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.144 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.146 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.146 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.147 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:31:08.148 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:31:08.149 DEBUG:boto:Method: GET 2022-09-03T07:31:08.150 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:31:08.151 DEBUG:boto:Data: 2022-09-03T07:31:08.152 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.153 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.154 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.155 DEBUG:boto:Params: {} 2022-09-03T07:31:08.156 DEBUG:boto:Token: None 2022-09-03T07:31:08.157 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.162 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006885adf41ea9d7fc-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.163 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.164 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:08.164 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:08.165 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.165 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:31:08.166 DEBUG:boto:Data: 2022-09-03T07:31:08.166 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.167 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.168 DEBUG:boto:Params: {} 2022-09-03T07:31:08.169 DEBUG:boto:Token: None 2022-09-03T07:31:08.169 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj37 2022-09-03T07:31:08.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/bBYBooF9aM+TRPXz/CNsSjCxjs= 2022-09-03T07:31:08.170 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/bBYBooF9aM+TRPXz/CNsSjCxjs=', 'Content-Length': '0'} 2022-09-03T07:31:08.173 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa2fb74fbc9cbfe3-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.174 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:08.174 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:08.175 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.175 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:31:08.176 DEBUG:boto:Data: 2022-09-03T07:31:08.176 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.177 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.178 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.178 DEBUG:boto:Params: {} 2022-09-03T07:31:08.179 DEBUG:boto:Token: None 2022-09-03T07:31:08.180 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj37 2022-09-03T07:31:08.180 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/bBYBooF9aM+TRPXz/CNsSjCxjs= 2022-09-03T07:31:08.181 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/bBYBooF9aM+TRPXz/CNsSjCxjs=', 'Content-Length': '0'} 2022-09-03T07:31:08.183 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a16c41d48efd77b1-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.184 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2022-09-03T07:31:08.185 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:08.185 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:08.186 DEBUG:boto:Method: GET 2022-09-03T07:31:08.186 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:31:08.187 DEBUG:boto:Data: 2022-09-03T07:31:08.187 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.188 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.189 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.189 DEBUG:boto:Params: {} 2022-09-03T07:31:08.190 DEBUG:boto:Token: None 2022-09-03T07:31:08.191 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj37 2022-09-03T07:31:08.191 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:08HUpO9FUY8gcYNk7R5MG+NTcyM= 2022-09-03T07:31:08.192 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:08HUpO9FUY8gcYNk7R5MG+NTcyM=', 'Content-Length': '0'} 2022-09-03T07:31:08.194 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000378aa8ea82858f8d-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.238 DEBUG:boto:path=/avysmw-75/obj37 2022-09-03T07:31:08.239 DEBUG:boto:auth_path=/avysmw-75/obj37 2022-09-03T07:31:08.240 DEBUG:boto:Method: GET 2022-09-03T07:31:08.241 DEBUG:boto:Path: /avysmw-75/obj37 2022-09-03T07:31:08.243 DEBUG:boto:Data: 2022-09-03T07:31:08.243 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.244 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.245 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.245 DEBUG:boto:Params: {} 2022-09-03T07:31:08.246 DEBUG:boto:Token: None 2022-09-03T07:31:08.247 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj37 2022-09-03T07:31:08.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:08HUpO9FUY8gcYNk7R5MG+NTcyM= 2022-09-03T07:31:08.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:08HUpO9FUY8gcYNk7R5MG+NTcyM=', 'Content-Length': '0'} 2022-09-03T07:31:08.251 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005278b2109a034575-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.253 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.254 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.255 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:31:08.256 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:31:08.256 DEBUG:boto:Method: GET 2022-09-03T07:31:08.257 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:31:08.257 DEBUG:boto:Data: 2022-09-03T07:31:08.258 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.259 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.259 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.260 DEBUG:boto:Params: {} 2022-09-03T07:31:08.261 DEBUG:boto:Token: None 2022-09-03T07:31:08.262 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.262 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.263 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.266 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b77e9c84d4d9878-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.268 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.269 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.270 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.271 DEBUG:boto:path=/avysmw-75/?prefix=obj37 2022-09-03T07:31:08.272 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj37 2022-09-03T07:31:08.273 DEBUG:boto:Method: GET 2022-09-03T07:31:08.274 DEBUG:boto:Path: /avysmw-75/?prefix=obj37 2022-09-03T07:31:08.275 DEBUG:boto:Data: 2022-09-03T07:31:08.276 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.277 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.278 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.279 DEBUG:boto:Params: {} 2022-09-03T07:31:08.280 DEBUG:boto:Token: None 2022-09-03T07:31:08.281 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.282 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.283 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.287 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000014ef4b5900baa9d9-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.288 DEBUG:boto:b'avysmw-75obj371000falseobj372022-09-03T07:28:24.732Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.289 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:31:08.291 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.291 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.292 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.293 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.295 DEBUG:boto:Method: GET 2022-09-03T07:31:08.295 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.297 DEBUG:boto:Data: 2022-09-03T07:31:08.297 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.298 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.299 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.299 DEBUG:boto:Params: {} 2022-09-03T07:31:08.300 DEBUG:boto:Token: None 2022-09-03T07:31:08.301 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.302 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QEdETLO0bBQG9BVURjJlMug8/8w= 2022-09-03T07:31:08.303 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QEdETLO0bBQG9BVURjJlMug8/8w=', 'Content-Length': '0'} 2022-09-03T07:31:08.306 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e7d099d7c1097eb5-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.307 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.308 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.309 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.310 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.311 DEBUG:boto:Method: GET 2022-09-03T07:31:08.312 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.313 DEBUG:boto:Data: 2022-09-03T07:31:08.314 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.315 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.315 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.316 DEBUG:boto:Params: {} 2022-09-03T07:31:08.317 DEBUG:boto:Token: None 2022-09-03T07:31:08.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.318 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QEdETLO0bBQG9BVURjJlMug8/8w= 2022-09-03T07:31:08.319 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QEdETLO0bBQG9BVURjJlMug8/8w=', 'Content-Length': '0'} 2022-09-03T07:31:08.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000033f8eaf6e6310901-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.362 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.363 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.364 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.365 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.366 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.367 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.368 DEBUG:boto:Data: 2022-09-03T07:31:08.369 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.370 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.371 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.372 DEBUG:boto:Params: {} 2022-09-03T07:31:08.374 DEBUG:boto:Token: None 2022-09-03T07:31:08.375 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.376 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hbXp1aUs+VE+dKXydWXDu+7NDyM= 2022-09-03T07:31:08.377 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hbXp1aUs+VE+dKXydWXDu+7NDyM=', 'Content-Length': '0'} 2022-09-03T07:31:08.384 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002397ad7067991a2d-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.385 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.385 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.386 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.386 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.387 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.387 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.388 DEBUG:boto:Data: 2022-09-03T07:31:08.388 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.389 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.390 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.390 DEBUG:boto:Params: {} 2022-09-03T07:31:08.391 DEBUG:boto:Token: None 2022-09-03T07:31:08.392 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:hbXp1aUs+VE+dKXydWXDu+7NDyM= 2022-09-03T07:31:08.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:hbXp1aUs+VE+dKXydWXDu+7NDyM=', 'Content-Length': '0'} 2022-09-03T07:31:08.395 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000349df26108b06c4e-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.396 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:31:08.397 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.397 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.398 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.398 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.399 DEBUG:boto:Method: GET 2022-09-03T07:31:08.400 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.400 DEBUG:boto:Data: 2022-09-03T07:31:08.401 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.401 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.402 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.403 DEBUG:boto:Params: {} 2022-09-03T07:31:08.403 DEBUG:boto:Token: None 2022-09-03T07:31:08.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QEdETLO0bBQG9BVURjJlMug8/8w= 2022-09-03T07:31:08.405 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QEdETLO0bBQG9BVURjJlMug8/8w=', 'Content-Length': '0'} 2022-09-03T07:31:08.408 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a979a62ebd368ff7-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.409 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.410 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.410 DEBUG:boto:path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.411 DEBUG:boto:auth_path=/avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.411 DEBUG:boto:Method: GET 2022-09-03T07:31:08.412 DEBUG:boto:Path: /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.413 DEBUG:boto:Data: 2022-09-03T07:31:08.413 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.414 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.414 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.415 DEBUG:boto:Params: {} 2022-09-03T07:31:08.416 DEBUG:boto:Token: None 2022-09-03T07:31:08.416 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38?versionId=null 2022-09-03T07:31:08.417 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QEdETLO0bBQG9BVURjJlMug8/8w= 2022-09-03T07:31:08.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QEdETLO0bBQG9BVURjJlMug8/8w=', 'Content-Length': '0'} 2022-09-03T07:31:08.420 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a61224d34a19d8d-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.421 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.421 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.422 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:31:08.422 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:31:08.423 DEBUG:boto:Method: GET 2022-09-03T07:31:08.423 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:31:08.424 DEBUG:boto:Data: 2022-09-03T07:31:08.424 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.425 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.426 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.426 DEBUG:boto:Params: {} 2022-09-03T07:31:08.427 DEBUG:boto:Token: None 2022-09-03T07:31:08.428 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.428 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.429 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.433 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000910718cf8dc1bac8-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.434 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.435 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.435 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.436 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:31:08.437 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:31:08.437 DEBUG:boto:Method: GET 2022-09-03T07:31:08.438 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:31:08.438 DEBUG:boto:Data: 2022-09-03T07:31:08.439 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.439 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.440 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.441 DEBUG:boto:Params: {} 2022-09-03T07:31:08.441 DEBUG:boto:Token: None 2022-09-03T07:31:08.442 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.442 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.443 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.445 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cfd939bc3d594a07-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.447 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.449 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.450 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.451 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.451 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:31:08.452 DEBUG:boto:Data: 2022-09-03T07:31:08.452 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.453 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.453 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.455 DEBUG:boto:Params: {} 2022-09-03T07:31:08.455 DEBUG:boto:Token: None 2022-09-03T07:31:08.456 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38 2022-09-03T07:31:08.457 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:754aJ1/JnDr+USNlbIbrEBjtgtM= 2022-09-03T07:31:08.457 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:754aJ1/JnDr+USNlbIbrEBjtgtM=', 'Content-Length': '0'} 2022-09-03T07:31:08.459 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6d969df5c0965f9-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.460 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.461 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.463 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.463 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:31:08.464 DEBUG:boto:Data: 2022-09-03T07:31:08.464 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.466 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.467 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.468 DEBUG:boto:Params: {} 2022-09-03T07:31:08.469 DEBUG:boto:Token: None 2022-09-03T07:31:08.470 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38 2022-09-03T07:31:08.471 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:754aJ1/JnDr+USNlbIbrEBjtgtM= 2022-09-03T07:31:08.472 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:754aJ1/JnDr+USNlbIbrEBjtgtM=', 'Content-Length': '0'} 2022-09-03T07:31:08.475 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e6c4672535aae1b-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.476 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2022-09-03T07:31:08.477 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.478 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.479 DEBUG:boto:Method: GET 2022-09-03T07:31:08.480 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:31:08.480 DEBUG:boto:Data: 2022-09-03T07:31:08.481 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.482 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.482 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.483 DEBUG:boto:Params: {} 2022-09-03T07:31:08.483 DEBUG:boto:Token: None 2022-09-03T07:31:08.484 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38 2022-09-03T07:31:08.485 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FN/0h0Psq0qsTbmxYKKYr7dJqq8= 2022-09-03T07:31:08.485 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FN/0h0Psq0qsTbmxYKKYr7dJqq8=', 'Content-Length': '0'} 2022-09-03T07:31:08.488 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b6ec18d5b101ba9-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.489 DEBUG:boto:path=/avysmw-75/obj38 2022-09-03T07:31:08.490 DEBUG:boto:auth_path=/avysmw-75/obj38 2022-09-03T07:31:08.491 DEBUG:boto:Method: GET 2022-09-03T07:31:08.491 DEBUG:boto:Path: /avysmw-75/obj38 2022-09-03T07:31:08.492 DEBUG:boto:Data: 2022-09-03T07:31:08.492 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.493 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.493 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.494 DEBUG:boto:Params: {} 2022-09-03T07:31:08.495 DEBUG:boto:Token: None 2022-09-03T07:31:08.495 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj38 2022-09-03T07:31:08.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FN/0h0Psq0qsTbmxYKKYr7dJqq8= 2022-09-03T07:31:08.496 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FN/0h0Psq0qsTbmxYKKYr7dJqq8=', 'Content-Length': '0'} 2022-09-03T07:31:08.498 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000618a2bcfe65d84c5-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.499 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.500 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.500 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:31:08.501 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:31:08.501 DEBUG:boto:Method: GET 2022-09-03T07:31:08.502 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:31:08.503 DEBUG:boto:Data: 2022-09-03T07:31:08.503 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.504 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.504 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.505 DEBUG:boto:Params: {} 2022-09-03T07:31:08.505 DEBUG:boto:Token: None 2022-09-03T07:31:08.506 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.507 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.510 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000df8cb2d31e31616e-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.511 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.512 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.513 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.513 DEBUG:boto:path=/avysmw-75/?prefix=obj38 2022-09-03T07:31:08.514 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj38 2022-09-03T07:31:08.514 DEBUG:boto:Method: GET 2022-09-03T07:31:08.517 DEBUG:boto:Path: /avysmw-75/?prefix=obj38 2022-09-03T07:31:08.517 DEBUG:boto:Data: 2022-09-03T07:31:08.518 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.519 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.520 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.520 DEBUG:boto:Params: {} 2022-09-03T07:31:08.521 DEBUG:boto:Token: None 2022-09-03T07:31:08.521 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.522 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.523 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.525 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c1f03363f9be5e53-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.566 DEBUG:boto:b'avysmw-75obj381000falseobj382022-09-03T07:15:42.462Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.568 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:31:08.569 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.570 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.571 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.571 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.572 DEBUG:boto:Method: GET 2022-09-03T07:31:08.573 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.573 DEBUG:boto:Data: 2022-09-03T07:31:08.574 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.574 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.575 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.576 DEBUG:boto:Params: {} 2022-09-03T07:31:08.576 DEBUG:boto:Token: None 2022-09-03T07:31:08.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7p6M8BtAIi+kbx6LOR3dqFC7GUQ= 2022-09-03T07:31:08.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7p6M8BtAIi+kbx6LOR3dqFC7GUQ=', 'Content-Length': '0'} 2022-09-03T07:31:08.582 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005d1a5d0c673db135-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.583 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.583 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.584 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.584 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.585 DEBUG:boto:Method: GET 2022-09-03T07:31:08.585 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.586 DEBUG:boto:Data: 2022-09-03T07:31:08.587 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.587 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.588 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.588 DEBUG:boto:Params: {} 2022-09-03T07:31:08.589 DEBUG:boto:Token: None 2022-09-03T07:31:08.590 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7p6M8BtAIi+kbx6LOR3dqFC7GUQ= 2022-09-03T07:31:08.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7p6M8BtAIi+kbx6LOR3dqFC7GUQ=', 'Content-Length': '0'} 2022-09-03T07:31:08.593 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000778b0845b36df919-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.594 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.595 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.595 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.596 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.596 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.597 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.597 DEBUG:boto:Data: 2022-09-03T07:31:08.598 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.598 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.599 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.599 DEBUG:boto:Params: {} 2022-09-03T07:31:08.600 DEBUG:boto:Token: None 2022-09-03T07:31:08.600 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.601 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:myCiO5EwiQfKXWzTrb4gbi+Pl+M= 2022-09-03T07:31:08.601 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:myCiO5EwiQfKXWzTrb4gbi+Pl+M=', 'Content-Length': '0'} 2022-09-03T07:31:08.604 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000fe2b1ce98846322-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.604 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.605 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.605 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.606 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.606 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.607 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.608 DEBUG:boto:Data: 2022-09-03T07:31:08.608 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.609 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.610 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.611 DEBUG:boto:Params: {} 2022-09-03T07:31:08.612 DEBUG:boto:Token: None 2022-09-03T07:31:08.613 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.614 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:myCiO5EwiQfKXWzTrb4gbi+Pl+M= 2022-09-03T07:31:08.614 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:myCiO5EwiQfKXWzTrb4gbi+Pl+M=', 'Content-Length': '0'} 2022-09-03T07:31:08.616 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d112248ab7f4d679-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.617 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:31:08.618 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.618 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.619 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.620 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.620 DEBUG:boto:Method: GET 2022-09-03T07:31:08.621 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.622 DEBUG:boto:Data: 2022-09-03T07:31:08.623 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.623 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.624 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.625 DEBUG:boto:Params: {} 2022-09-03T07:31:08.626 DEBUG:boto:Token: None 2022-09-03T07:31:08.626 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7p6M8BtAIi+kbx6LOR3dqFC7GUQ= 2022-09-03T07:31:08.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7p6M8BtAIi+kbx6LOR3dqFC7GUQ=', 'Content-Length': '0'} 2022-09-03T07:31:08.630 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001bd6c331f1f760a8-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.674 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.675 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.676 DEBUG:boto:path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.677 DEBUG:boto:auth_path=/avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.678 DEBUG:boto:Method: GET 2022-09-03T07:31:08.679 DEBUG:boto:Path: /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.680 DEBUG:boto:Data: 2022-09-03T07:31:08.680 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.681 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.682 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.683 DEBUG:boto:Params: {} 2022-09-03T07:31:08.684 DEBUG:boto:Token: None 2022-09-03T07:31:08.685 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44?versionId=null 2022-09-03T07:31:08.686 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7p6M8BtAIi+kbx6LOR3dqFC7GUQ= 2022-09-03T07:31:08.687 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7p6M8BtAIi+kbx6LOR3dqFC7GUQ=', 'Content-Length': '0'} 2022-09-03T07:31:08.690 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002bec5bcaa54bd862-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.692 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.693 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.694 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:31:08.695 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:31:08.697 DEBUG:boto:Method: GET 2022-09-03T07:31:08.698 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:31:08.699 DEBUG:boto:Data: 2022-09-03T07:31:08.702 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.704 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.705 DEBUG:boto:Params: {} 2022-09-03T07:31:08.707 DEBUG:boto:Token: None 2022-09-03T07:31:08.708 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.709 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.710 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.715 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b3578905a26e313c-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.717 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.718 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.719 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.719 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:31:08.720 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:31:08.721 DEBUG:boto:Method: GET 2022-09-03T07:31:08.721 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:31:08.723 DEBUG:boto:Data: 2022-09-03T07:31:08.724 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.724 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.725 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.726 DEBUG:boto:Params: {} 2022-09-03T07:31:08.726 DEBUG:boto:Token: None 2022-09-03T07:31:08.727 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.728 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.729 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.732 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eef86f65b4d4eb18-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.734 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.736 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.737 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.738 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.740 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:31:08.740 DEBUG:boto:Data: 2022-09-03T07:31:08.741 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.742 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.743 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.744 DEBUG:boto:Params: {} 2022-09-03T07:31:08.745 DEBUG:boto:Token: None 2022-09-03T07:31:08.745 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44 2022-09-03T07:31:08.746 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rx/krCbHS2yd4pNqrlU85aZ7Rp4= 2022-09-03T07:31:08.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rx/krCbHS2yd4pNqrlU85aZ7Rp4=', 'Content-Length': '0'} 2022-09-03T07:31:08.749 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000673a3a57a2b155eb-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.750 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.750 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.751 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.751 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:31:08.752 DEBUG:boto:Data: 2022-09-03T07:31:08.753 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.753 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.755 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.756 DEBUG:boto:Params: {} 2022-09-03T07:31:08.756 DEBUG:boto:Token: None 2022-09-03T07:31:08.757 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44 2022-09-03T07:31:08.758 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Rx/krCbHS2yd4pNqrlU85aZ7Rp4= 2022-09-03T07:31:08.759 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Rx/krCbHS2yd4pNqrlU85aZ7Rp4=', 'Content-Length': '0'} 2022-09-03T07:31:08.763 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000067950ae2b5a8bf76-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.766 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:31:08.767 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.767 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.769 DEBUG:boto:Method: GET 2022-09-03T07:31:08.770 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:31:08.772 DEBUG:boto:Data: 2022-09-03T07:31:08.773 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.775 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.776 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.778 DEBUG:boto:Params: {} 2022-09-03T07:31:08.779 DEBUG:boto:Token: None 2022-09-03T07:31:08.780 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44 2022-09-03T07:31:08.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ZFwSJmS9+xwlnUNv/cBcr4r3fk= 2022-09-03T07:31:08.782 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ZFwSJmS9+xwlnUNv/cBcr4r3fk=', 'Content-Length': '0'} 2022-09-03T07:31:08.784 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002fc1067f941fd3a2-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.785 DEBUG:boto:path=/avysmw-75/obj44 2022-09-03T07:31:08.786 DEBUG:boto:auth_path=/avysmw-75/obj44 2022-09-03T07:31:08.787 DEBUG:boto:Method: GET 2022-09-03T07:31:08.787 DEBUG:boto:Path: /avysmw-75/obj44 2022-09-03T07:31:08.788 DEBUG:boto:Data: 2022-09-03T07:31:08.789 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.790 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.791 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.792 DEBUG:boto:Params: {} 2022-09-03T07:31:08.793 DEBUG:boto:Token: None 2022-09-03T07:31:08.794 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj44 2022-09-03T07:31:08.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2ZFwSJmS9+xwlnUNv/cBcr4r3fk= 2022-09-03T07:31:08.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2ZFwSJmS9+xwlnUNv/cBcr4r3fk=', 'Content-Length': '0'} 2022-09-03T07:31:08.800 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000ce2b740bd4f4902-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.802 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.802 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.803 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:31:08.804 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:31:08.805 DEBUG:boto:Method: GET 2022-09-03T07:31:08.806 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:31:08.807 DEBUG:boto:Data: 2022-09-03T07:31:08.807 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.809 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.810 DEBUG:boto:Params: {} 2022-09-03T07:31:08.811 DEBUG:boto:Token: None 2022-09-03T07:31:08.812 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.813 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.814 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.818 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000518b0485b580f953-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.819 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.820 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:08.821 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:08.823 DEBUG:boto:path=/avysmw-75/?prefix=obj44 2022-09-03T07:31:08.824 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj44 2022-09-03T07:31:08.826 DEBUG:boto:Method: GET 2022-09-03T07:31:08.827 DEBUG:boto:Path: /avysmw-75/?prefix=obj44 2022-09-03T07:31:08.829 DEBUG:boto:Data: 2022-09-03T07:31:08.830 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.832 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.833 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.834 DEBUG:boto:Params: {} 2022-09-03T07:31:08.835 DEBUG:boto:Token: None 2022-09-03T07:31:08.835 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/ 2022-09-03T07:31:08.837 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0= 2022-09-03T07:31:08.839 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JQQNnhaiWsvpNjS+B+cYMd6a0H0=', 'Content-Length': '0'} 2022-09-03T07:31:08.842 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006ca85e2690daa253-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.844 DEBUG:boto:b'avysmw-75obj441000falseobj442022-09-03T07:11:01.065Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:08.846 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:31:08.847 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:08.848 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:08.848 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.850 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.850 DEBUG:boto:Method: GET 2022-09-03T07:31:08.851 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.852 DEBUG:boto:Data: 2022-09-03T07:31:08.853 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.854 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.855 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.856 DEBUG:boto:Params: {} 2022-09-03T07:31:08.857 DEBUG:boto:Token: None 2022-09-03T07:31:08.858 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ixnGkpxBEj8RWeW8SgCQ346Pdvw= 2022-09-03T07:31:08.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ixnGkpxBEj8RWeW8SgCQ346Pdvw=', 'Content-Length': '0'} 2022-09-03T07:31:08.863 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004da5ce7a4a97b527-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.911 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:08.913 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:08.915 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.916 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.917 DEBUG:boto:Method: GET 2022-09-03T07:31:08.923 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.924 DEBUG:boto:Data: 2022-09-03T07:31:08.926 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.928 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.929 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.931 DEBUG:boto:Params: {} 2022-09-03T07:31:08.932 DEBUG:boto:Token: None 2022-09-03T07:31:08.933 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.934 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ixnGkpxBEj8RWeW8SgCQ346Pdvw= 2022-09-03T07:31:08.935 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ixnGkpxBEj8RWeW8SgCQ346Pdvw=', 'Content-Length': '0'} 2022-09-03T07:31:08.938 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d6f792dad3b9f14b-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.940 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:08.941 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:08.942 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.943 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.945 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.946 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.948 DEBUG:boto:Data: 2022-09-03T07:31:08.949 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.951 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.952 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.953 DEBUG:boto:Params: {} 2022-09-03T07:31:08.954 DEBUG:boto:Token: None 2022-09-03T07:31:08.955 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.956 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wCTV0kETc+uA21FtYERZpTh8eo4= 2022-09-03T07:31:08.957 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wCTV0kETc+uA21FtYERZpTh8eo4=', 'Content-Length': '0'} 2022-09-03T07:31:08.959 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000040ce02a8a6ea4746-00631302bc-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.960 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:08.961 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:08.962 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.964 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.965 DEBUG:boto:Method: HEAD 2022-09-03T07:31:08.967 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.968 DEBUG:boto:Data: 2022-09-03T07:31:08.970 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.971 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:08.973 DEBUG:boto:Port: 8000 2022-09-03T07:31:08.978 DEBUG:boto:Params: {} 2022-09-03T07:31:08.979 DEBUG:boto:Token: None 2022-09-03T07:31:08.980 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:08 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.982 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wCTV0kETc+uA21FtYERZpTh8eo4= 2022-09-03T07:31:08.984 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:08 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wCTV0kETc+uA21FtYERZpTh8eo4=', 'Content-Length': '0'} 2022-09-03T07:31:08.987 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa05aa6c0bdb42ce-00631302bc-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:08 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:08.988 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:31:08.990 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:08.991 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:08.992 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.993 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.994 DEBUG:boto:Method: GET 2022-09-03T07:31:08.995 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:31:08.996 DEBUG:boto:Data: 2022-09-03T07:31:08.997 DEBUG:boto:Headers: {} 2022-09-03T07:31:08.998 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:08.998 DEBUG:boto:Port: 8001 2022-09-03T07:31:08.999 DEBUG:boto:Params: {} 2022-09-03T07:31:09.000 DEBUG:boto:Token: None 2022-09-03T07:31:09.001 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:31:09.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zV0/HoVxEIwLlwOFOm/XZC1/yHg= 2022-09-03T07:31:09.002 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zV0/HoVxEIwLlwOFOm/XZC1/yHg=', 'Content-Length': '0'} 2022-09-03T07:31:09.005 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e6c0ddd3654cbcb-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.007 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:09.008 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:09.009 DEBUG:boto:path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:09.010 DEBUG:boto:auth_path=/avysmw-75/obj46?versionId=null 2022-09-03T07:31:09.011 DEBUG:boto:Method: GET 2022-09-03T07:31:09.011 DEBUG:boto:Path: /avysmw-75/obj46?versionId=null 2022-09-03T07:31:09.012 DEBUG:boto:Data: 2022-09-03T07:31:09.012 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.013 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.014 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.014 DEBUG:boto:Params: {} 2022-09-03T07:31:09.015 DEBUG:boto:Token: None 2022-09-03T07:31:09.016 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj46?versionId=null 2022-09-03T07:31:09.016 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zV0/HoVxEIwLlwOFOm/XZC1/yHg= 2022-09-03T07:31:09.017 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zV0/HoVxEIwLlwOFOm/XZC1/yHg=', 'Content-Length': '0'} 2022-09-03T07:31:09.019 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ab138e9075a5d70-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.062 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.063 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.064 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:31:09.065 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:31:09.066 DEBUG:boto:Method: GET 2022-09-03T07:31:09.067 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:31:09.067 DEBUG:boto:Data: 2022-09-03T07:31:09.068 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.068 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.069 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.069 DEBUG:boto:Params: {} 2022-09-03T07:31:09.070 DEBUG:boto:Token: None 2022-09-03T07:31:09.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.082 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b59e73f8952876d0-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.084 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.085 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.086 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.087 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:31:09.088 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:31:09.088 DEBUG:boto:Method: GET 2022-09-03T07:31:09.089 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:31:09.091 DEBUG:boto:Data: 2022-09-03T07:31:09.092 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.094 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.095 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.096 DEBUG:boto:Params: {} 2022-09-03T07:31:09.100 DEBUG:boto:Token: None 2022-09-03T07:31:09.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.102 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.105 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000355b821b4008687a-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.105 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.106 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:09.107 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:09.108 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.108 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:31:09.109 DEBUG:boto:Data: 2022-09-03T07:31:09.110 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.111 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.111 DEBUG:boto:Params: {} 2022-09-03T07:31:09.112 DEBUG:boto:Token: None 2022-09-03T07:31:09.113 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj46 2022-09-03T07:31:09.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FAR+6hTjKLWYPpJaFilOtM9XBA= 2022-09-03T07:31:09.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FAR+6hTjKLWYPpJaFilOtM9XBA=', 'Content-Length': '0'} 2022-09-03T07:31:09.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007c56115c7bc52204-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.119 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:09.120 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:09.121 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.122 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:31:09.123 DEBUG:boto:Data: 2022-09-03T07:31:09.124 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.125 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.126 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.127 DEBUG:boto:Params: {} 2022-09-03T07:31:09.128 DEBUG:boto:Token: None 2022-09-03T07:31:09.131 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj46 2022-09-03T07:31:09.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6FAR+6hTjKLWYPpJaFilOtM9XBA= 2022-09-03T07:31:09.133 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6FAR+6hTjKLWYPpJaFilOtM9XBA=', 'Content-Length': '0'} 2022-09-03T07:31:09.137 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008e158f741aa630d0-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.138 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2022-09-03T07:31:09.139 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:09.140 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:09.141 DEBUG:boto:Method: GET 2022-09-03T07:31:09.142 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:31:09.144 DEBUG:boto:Data: 2022-09-03T07:31:09.144 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.145 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.145 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.146 DEBUG:boto:Params: {} 2022-09-03T07:31:09.147 DEBUG:boto:Token: None 2022-09-03T07:31:09.147 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj46 2022-09-03T07:31:09.148 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4txBXU4okti/TWzXw7OLHxx5VdY= 2022-09-03T07:31:09.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4txBXU4okti/TWzXw7OLHxx5VdY=', 'Content-Length': '0'} 2022-09-03T07:31:09.151 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc78351d9df0d53f-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.151 DEBUG:boto:path=/avysmw-75/obj46 2022-09-03T07:31:09.152 DEBUG:boto:auth_path=/avysmw-75/obj46 2022-09-03T07:31:09.153 DEBUG:boto:Method: GET 2022-09-03T07:31:09.153 DEBUG:boto:Path: /avysmw-75/obj46 2022-09-03T07:31:09.154 DEBUG:boto:Data: 2022-09-03T07:31:09.154 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.155 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.155 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.156 DEBUG:boto:Params: {} 2022-09-03T07:31:09.157 DEBUG:boto:Token: None 2022-09-03T07:31:09.157 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj46 2022-09-03T07:31:09.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4txBXU4okti/TWzXw7OLHxx5VdY= 2022-09-03T07:31:09.158 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4txBXU4okti/TWzXw7OLHxx5VdY=', 'Content-Length': '0'} 2022-09-03T07:31:09.160 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000043886509d1225d1c-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.162 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.165 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.165 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:31:09.166 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:31:09.166 DEBUG:boto:Method: GET 2022-09-03T07:31:09.167 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:31:09.167 DEBUG:boto:Data: 2022-09-03T07:31:09.168 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.169 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.169 DEBUG:boto:Params: {} 2022-09-03T07:31:09.170 DEBUG:boto:Token: None 2022-09-03T07:31:09.171 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.180 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cd8040bd8837128e-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.182 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.183 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.184 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.185 DEBUG:boto:path=/avysmw-75/?prefix=obj46 2022-09-03T07:31:09.186 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj46 2022-09-03T07:31:09.187 DEBUG:boto:Method: GET 2022-09-03T07:31:09.188 DEBUG:boto:Path: /avysmw-75/?prefix=obj46 2022-09-03T07:31:09.189 DEBUG:boto:Data: 2022-09-03T07:31:09.190 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.191 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.192 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.193 DEBUG:boto:Params: {} 2022-09-03T07:31:09.195 DEBUG:boto:Token: None 2022-09-03T07:31:09.196 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.202 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000138271d7e3f6c619-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.203 DEBUG:boto:b'avysmw-75obj461000falseobj462022-09-03T07:21:21.569Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.204 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:31:09.206 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.207 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.208 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.209 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.209 DEBUG:boto:Method: GET 2022-09-03T07:31:09.211 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.212 DEBUG:boto:Data: 2022-09-03T07:31:09.213 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.214 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.215 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.216 DEBUG:boto:Params: {} 2022-09-03T07:31:09.217 DEBUG:boto:Token: None 2022-09-03T07:31:09.218 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.220 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bIann/ltNoTNZU+N5amRSzbzdHI= 2022-09-03T07:31:09.221 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bIann/ltNoTNZU+N5amRSzbzdHI=', 'Content-Length': '0'} 2022-09-03T07:31:09.223 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000103ef9a0ebf1ddda-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.266 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.267 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.268 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.269 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.270 DEBUG:boto:Method: GET 2022-09-03T07:31:09.271 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.271 DEBUG:boto:Data: 2022-09-03T07:31:09.272 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.275 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.275 DEBUG:boto:Params: {} 2022-09-03T07:31:09.276 DEBUG:boto:Token: None 2022-09-03T07:31:09.277 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.277 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bIann/ltNoTNZU+N5amRSzbzdHI= 2022-09-03T07:31:09.278 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bIann/ltNoTNZU+N5amRSzbzdHI=', 'Content-Length': '0'} 2022-09-03T07:31:09.281 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000611a49ec056ece10-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.282 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.283 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.283 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.284 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.284 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.285 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.285 DEBUG:boto:Data: 2022-09-03T07:31:09.286 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.286 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.287 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.287 DEBUG:boto:Params: {} 2022-09-03T07:31:09.288 DEBUG:boto:Token: None 2022-09-03T07:31:09.289 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.289 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wqqi7nxMYkYipMt82gY0EfYg1Fc= 2022-09-03T07:31:09.290 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wqqi7nxMYkYipMt82gY0EfYg1Fc=', 'Content-Length': '0'} 2022-09-03T07:31:09.292 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9ab6ef70559d7bf-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.293 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.294 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.294 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.295 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.295 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.296 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.297 DEBUG:boto:Data: 2022-09-03T07:31:09.297 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.298 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.298 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.299 DEBUG:boto:Params: {} 2022-09-03T07:31:09.299 DEBUG:boto:Token: None 2022-09-03T07:31:09.300 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Wqqi7nxMYkYipMt82gY0EfYg1Fc= 2022-09-03T07:31:09.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Wqqi7nxMYkYipMt82gY0EfYg1Fc=', 'Content-Length': '0'} 2022-09-03T07:31:09.303 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007409d69513a80556-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.304 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:31:09.305 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.305 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.306 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.306 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.307 DEBUG:boto:Method: GET 2022-09-03T07:31:09.308 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.308 DEBUG:boto:Data: 2022-09-03T07:31:09.309 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.309 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.310 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.310 DEBUG:boto:Params: {} 2022-09-03T07:31:09.311 DEBUG:boto:Token: None 2022-09-03T07:31:09.311 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.312 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bIann/ltNoTNZU+N5amRSzbzdHI= 2022-09-03T07:31:09.313 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bIann/ltNoTNZU+N5amRSzbzdHI=', 'Content-Length': '0'} 2022-09-03T07:31:09.315 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a41eff014d75eed2-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.316 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.316 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.317 DEBUG:boto:path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.317 DEBUG:boto:auth_path=/avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.318 DEBUG:boto:Method: GET 2022-09-03T07:31:09.318 DEBUG:boto:Path: /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.319 DEBUG:boto:Data: 2022-09-03T07:31:09.319 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.320 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.320 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.321 DEBUG:boto:Params: {} 2022-09-03T07:31:09.322 DEBUG:boto:Token: None 2022-09-03T07:31:09.322 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51?versionId=null 2022-09-03T07:31:09.323 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bIann/ltNoTNZU+N5amRSzbzdHI= 2022-09-03T07:31:09.324 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bIann/ltNoTNZU+N5amRSzbzdHI=', 'Content-Length': '0'} 2022-09-03T07:31:09.326 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f09f16e0f6dc41e-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.327 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.328 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.328 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:31:09.329 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:31:09.329 DEBUG:boto:Method: GET 2022-09-03T07:31:09.330 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:31:09.330 DEBUG:boto:Data: 2022-09-03T07:31:09.331 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.332 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.332 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.333 DEBUG:boto:Params: {} 2022-09-03T07:31:09.333 DEBUG:boto:Token: None 2022-09-03T07:31:09.334 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.335 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.338 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e8f029e23ade8265-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.339 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.340 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.341 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.342 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:31:09.343 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:31:09.344 DEBUG:boto:Method: GET 2022-09-03T07:31:09.345 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:31:09.346 DEBUG:boto:Data: 2022-09-03T07:31:09.347 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.347 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.348 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.349 DEBUG:boto:Params: {} 2022-09-03T07:31:09.350 DEBUG:boto:Token: None 2022-09-03T07:31:09.351 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.356 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000093f33e203d1f076-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.357 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.358 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.359 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.359 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.360 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:31:09.361 DEBUG:boto:Data: 2022-09-03T07:31:09.361 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.362 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.362 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.363 DEBUG:boto:Params: {} 2022-09-03T07:31:09.364 DEBUG:boto:Token: None 2022-09-03T07:31:09.364 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51 2022-09-03T07:31:09.365 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M3egOgG58YMWqzdw4LO627ixgwY= 2022-09-03T07:31:09.366 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M3egOgG58YMWqzdw4LO627ixgwY=', 'Content-Length': '0'} 2022-09-03T07:31:09.368 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004aac6586cdca7f91-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.369 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.369 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.370 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.370 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:31:09.371 DEBUG:boto:Data: 2022-09-03T07:31:09.371 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.372 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.373 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.374 DEBUG:boto:Params: {} 2022-09-03T07:31:09.375 DEBUG:boto:Token: None 2022-09-03T07:31:09.376 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51 2022-09-03T07:31:09.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M3egOgG58YMWqzdw4LO627ixgwY= 2022-09-03T07:31:09.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M3egOgG58YMWqzdw4LO627ixgwY=', 'Content-Length': '0'} 2022-09-03T07:31:09.380 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f661517704d54041-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.381 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj51 2022-09-03T07:31:09.382 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.383 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.384 DEBUG:boto:Method: GET 2022-09-03T07:31:09.385 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:31:09.386 DEBUG:boto:Data: 2022-09-03T07:31:09.387 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.388 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.389 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.390 DEBUG:boto:Params: {} 2022-09-03T07:31:09.391 DEBUG:boto:Token: None 2022-09-03T07:31:09.392 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51 2022-09-03T07:31:09.393 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+HPfMeivnB2F1I5l613RjTsxezo= 2022-09-03T07:31:09.394 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+HPfMeivnB2F1I5l613RjTsxezo=', 'Content-Length': '0'} 2022-09-03T07:31:09.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006196f8a9f6e449d6-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.445 DEBUG:boto:path=/avysmw-75/obj51 2022-09-03T07:31:09.448 DEBUG:boto:auth_path=/avysmw-75/obj51 2022-09-03T07:31:09.449 DEBUG:boto:Method: GET 2022-09-03T07:31:09.450 DEBUG:boto:Path: /avysmw-75/obj51 2022-09-03T07:31:09.451 DEBUG:boto:Data: 2022-09-03T07:31:09.452 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.453 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.456 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.458 DEBUG:boto:Params: {} 2022-09-03T07:31:09.459 DEBUG:boto:Token: None 2022-09-03T07:31:09.460 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj51 2022-09-03T07:31:09.461 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+HPfMeivnB2F1I5l613RjTsxezo= 2022-09-03T07:31:09.462 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+HPfMeivnB2F1I5l613RjTsxezo=', 'Content-Length': '0'} 2022-09-03T07:31:09.471 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090d8f36ce34984a0-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.472 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.473 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.473 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:31:09.474 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:31:09.474 DEBUG:boto:Method: GET 2022-09-03T07:31:09.475 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:31:09.475 DEBUG:boto:Data: 2022-09-03T07:31:09.476 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.477 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.477 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.478 DEBUG:boto:Params: {} 2022-09-03T07:31:09.479 DEBUG:boto:Token: None 2022-09-03T07:31:09.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.483 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000071b5d51dea9fb18d-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.484 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.485 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.486 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.487 DEBUG:boto:path=/avysmw-75/?prefix=obj51 2022-09-03T07:31:09.487 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj51 2022-09-03T07:31:09.488 DEBUG:boto:Method: GET 2022-09-03T07:31:09.488 DEBUG:boto:Path: /avysmw-75/?prefix=obj51 2022-09-03T07:31:09.489 DEBUG:boto:Data: 2022-09-03T07:31:09.490 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.490 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.491 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.491 DEBUG:boto:Params: {} 2022-09-03T07:31:09.492 DEBUG:boto:Token: None 2022-09-03T07:31:09.493 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.494 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.497 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000006832a97d8b61e9e6-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.497 DEBUG:boto:b'avysmw-75obj511000falseobj512022-09-03T07:12:49.547Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.498 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:31:09.499 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.499 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.500 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.500 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.501 DEBUG:boto:Method: GET 2022-09-03T07:31:09.502 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.503 DEBUG:boto:Data: 2022-09-03T07:31:09.503 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.504 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.505 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.505 DEBUG:boto:Params: {} 2022-09-03T07:31:09.506 DEBUG:boto:Token: None 2022-09-03T07:31:09.508 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.509 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KVDVJEuadKgBz4VEqI65rpLy+IU= 2022-09-03T07:31:09.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KVDVJEuadKgBz4VEqI65rpLy+IU=', 'Content-Length': '0'} 2022-09-03T07:31:09.513 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005bda7069d4ac2174-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.516 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.517 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.518 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.519 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.520 DEBUG:boto:Method: GET 2022-09-03T07:31:09.521 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.522 DEBUG:boto:Data: 2022-09-03T07:31:09.522 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.523 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.524 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.524 DEBUG:boto:Params: {} 2022-09-03T07:31:09.525 DEBUG:boto:Token: None 2022-09-03T07:31:09.526 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KVDVJEuadKgBz4VEqI65rpLy+IU= 2022-09-03T07:31:09.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KVDVJEuadKgBz4VEqI65rpLy+IU=', 'Content-Length': '0'} 2022-09-03T07:31:09.531 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000848dbf60d9ea06e6-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.532 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.532 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.533 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.534 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.535 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.535 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.536 DEBUG:boto:Data: 2022-09-03T07:31:09.536 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.537 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.537 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.538 DEBUG:boto:Params: {} 2022-09-03T07:31:09.538 DEBUG:boto:Token: None 2022-09-03T07:31:09.539 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8+lE0lqqusyg5FxpaUMsdm70Lzo= 2022-09-03T07:31:09.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8+lE0lqqusyg5FxpaUMsdm70Lzo=', 'Content-Length': '0'} 2022-09-03T07:31:09.544 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000775b3ed7b18a751c-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.545 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.546 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.546 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.547 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.547 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.548 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.548 DEBUG:boto:Data: 2022-09-03T07:31:09.549 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.549 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.550 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.551 DEBUG:boto:Params: {} 2022-09-03T07:31:09.552 DEBUG:boto:Token: None 2022-09-03T07:31:09.553 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.553 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8+lE0lqqusyg5FxpaUMsdm70Lzo= 2022-09-03T07:31:09.554 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8+lE0lqqusyg5FxpaUMsdm70Lzo=', 'Content-Length': '0'} 2022-09-03T07:31:09.558 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007a13564eaec3254c-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.559 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:31:09.560 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.562 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.563 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.564 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.565 DEBUG:boto:Method: GET 2022-09-03T07:31:09.566 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.567 DEBUG:boto:Data: 2022-09-03T07:31:09.568 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.569 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.570 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.571 DEBUG:boto:Params: {} 2022-09-03T07:31:09.572 DEBUG:boto:Token: None 2022-09-03T07:31:09.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KVDVJEuadKgBz4VEqI65rpLy+IU= 2022-09-03T07:31:09.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KVDVJEuadKgBz4VEqI65rpLy+IU=', 'Content-Length': '0'} 2022-09-03T07:31:09.577 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000090e8b2f8f6db8728-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.620 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.621 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.621 DEBUG:boto:path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.622 DEBUG:boto:auth_path=/avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.623 DEBUG:boto:Method: GET 2022-09-03T07:31:09.623 DEBUG:boto:Path: /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.624 DEBUG:boto:Data: 2022-09-03T07:31:09.624 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.625 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.625 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.626 DEBUG:boto:Params: {} 2022-09-03T07:31:09.627 DEBUG:boto:Token: None 2022-09-03T07:31:09.627 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55?versionId=null 2022-09-03T07:31:09.628 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KVDVJEuadKgBz4VEqI65rpLy+IU= 2022-09-03T07:31:09.629 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KVDVJEuadKgBz4VEqI65rpLy+IU=', 'Content-Length': '0'} 2022-09-03T07:31:09.632 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e37a2301413b5cc1-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.633 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.634 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.635 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:31:09.635 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:31:09.636 DEBUG:boto:Method: GET 2022-09-03T07:31:09.638 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:31:09.639 DEBUG:boto:Data: 2022-09-03T07:31:09.639 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.641 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.643 DEBUG:boto:Params: {} 2022-09-03T07:31:09.644 DEBUG:boto:Token: None 2022-09-03T07:31:09.645 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.647 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.651 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007f85e6606199c471-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.652 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.654 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.655 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.656 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:31:09.657 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:31:09.658 DEBUG:boto:Method: GET 2022-09-03T07:31:09.659 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:31:09.660 DEBUG:boto:Data: 2022-09-03T07:31:09.660 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.661 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.663 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.664 DEBUG:boto:Params: {} 2022-09-03T07:31:09.665 DEBUG:boto:Token: None 2022-09-03T07:31:09.666 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.671 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cb3aeae5653eb7ca-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.672 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.674 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.675 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.676 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.676 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:31:09.677 DEBUG:boto:Data: 2022-09-03T07:31:09.677 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.678 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.679 DEBUG:boto:Params: {} 2022-09-03T07:31:09.679 DEBUG:boto:Token: None 2022-09-03T07:31:09.680 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55 2022-09-03T07:31:09.680 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KMVOeIigO4f43TqEbqVCCIesstQ= 2022-09-03T07:31:09.681 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KMVOeIigO4f43TqEbqVCCIesstQ=', 'Content-Length': '0'} 2022-09-03T07:31:09.683 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e2859255d4855ec-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.684 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.684 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.685 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.687 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:31:09.687 DEBUG:boto:Data: 2022-09-03T07:31:09.688 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.688 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.689 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.689 DEBUG:boto:Params: {} 2022-09-03T07:31:09.690 DEBUG:boto:Token: None 2022-09-03T07:31:09.691 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55 2022-09-03T07:31:09.691 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KMVOeIigO4f43TqEbqVCCIesstQ= 2022-09-03T07:31:09.692 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KMVOeIigO4f43TqEbqVCCIesstQ=', 'Content-Length': '0'} 2022-09-03T07:31:09.695 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006f843659e4097ea5-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.696 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:31:09.697 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.698 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.699 DEBUG:boto:Method: GET 2022-09-03T07:31:09.700 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:31:09.701 DEBUG:boto:Data: 2022-09-03T07:31:09.702 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.703 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.704 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.705 DEBUG:boto:Params: {} 2022-09-03T07:31:09.705 DEBUG:boto:Token: None 2022-09-03T07:31:09.711 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55 2022-09-03T07:31:09.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1RnZ1rcelXluhNq3wzoBv2A7vTk= 2022-09-03T07:31:09.714 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1RnZ1rcelXluhNq3wzoBv2A7vTk=', 'Content-Length': '0'} 2022-09-03T07:31:09.717 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000190ca44d35bb823f-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.719 DEBUG:boto:path=/avysmw-75/obj55 2022-09-03T07:31:09.720 DEBUG:boto:auth_path=/avysmw-75/obj55 2022-09-03T07:31:09.722 DEBUG:boto:Method: GET 2022-09-03T07:31:09.723 DEBUG:boto:Path: /avysmw-75/obj55 2022-09-03T07:31:09.724 DEBUG:boto:Data: 2022-09-03T07:31:09.725 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.726 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.726 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.727 DEBUG:boto:Params: {} 2022-09-03T07:31:09.728 DEBUG:boto:Token: None 2022-09-03T07:31:09.730 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj55 2022-09-03T07:31:09.731 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1RnZ1rcelXluhNq3wzoBv2A7vTk= 2022-09-03T07:31:09.732 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1RnZ1rcelXluhNq3wzoBv2A7vTk=', 'Content-Length': '0'} 2022-09-03T07:31:09.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:11:01 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099408ad9d3b819c1-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.739 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.739 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.740 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:31:09.741 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:31:09.741 DEBUG:boto:Method: GET 2022-09-03T07:31:09.742 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:31:09.742 DEBUG:boto:Data: 2022-09-03T07:31:09.743 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.743 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.744 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.744 DEBUG:boto:Params: {} 2022-09-03T07:31:09.745 DEBUG:boto:Token: None 2022-09-03T07:31:09.746 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.747 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.750 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f8d348230b00947b-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.751 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.752 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.753 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.754 DEBUG:boto:path=/avysmw-75/?prefix=obj55 2022-09-03T07:31:09.755 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj55 2022-09-03T07:31:09.756 DEBUG:boto:Method: GET 2022-09-03T07:31:09.758 DEBUG:boto:Path: /avysmw-75/?prefix=obj55 2022-09-03T07:31:09.759 DEBUG:boto:Data: 2022-09-03T07:31:09.760 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.761 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.762 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.763 DEBUG:boto:Params: {} 2022-09-03T07:31:09.765 DEBUG:boto:Token: None 2022-09-03T07:31:09.766 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.767 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.768 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.772 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ff07fc124e502c94-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.818 DEBUG:boto:b'avysmw-75obj551000falseobj552022-09-03T07:11:01.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.820 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:31:09.820 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:09.821 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:09.822 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.823 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.824 DEBUG:boto:Method: GET 2022-09-03T07:31:09.824 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.825 DEBUG:boto:Data: 2022-09-03T07:31:09.825 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.826 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.827 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.828 DEBUG:boto:Params: {} 2022-09-03T07:31:09.829 DEBUG:boto:Token: None 2022-09-03T07:31:09.830 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.832 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sGNbqwBHbE8Ksk3bfbgOzxICFNU= 2022-09-03T07:31:09.833 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sGNbqwBHbE8Ksk3bfbgOzxICFNU=', 'Content-Length': '0'} 2022-09-03T07:31:09.835 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008688142d85ab7330-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.836 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:09.837 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:09.838 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.838 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.839 DEBUG:boto:Method: GET 2022-09-03T07:31:09.839 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.840 DEBUG:boto:Data: 2022-09-03T07:31:09.840 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.841 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.842 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.842 DEBUG:boto:Params: {} 2022-09-03T07:31:09.843 DEBUG:boto:Token: None 2022-09-03T07:31:09.843 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.844 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sGNbqwBHbE8Ksk3bfbgOzxICFNU= 2022-09-03T07:31:09.845 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sGNbqwBHbE8Ksk3bfbgOzxICFNU=', 'Content-Length': '0'} 2022-09-03T07:31:09.847 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005c7862b8d55299ec-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.848 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:09.849 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:09.849 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.850 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.850 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.851 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.851 DEBUG:boto:Data: 2022-09-03T07:31:09.852 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.852 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.853 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.854 DEBUG:boto:Params: {} 2022-09-03T07:31:09.855 DEBUG:boto:Token: None 2022-09-03T07:31:09.855 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.856 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xQxNLRVF1CO/DAlTe4GcyzLGAD4= 2022-09-03T07:31:09.857 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xQxNLRVF1CO/DAlTe4GcyzLGAD4=', 'Content-Length': '0'} 2022-09-03T07:31:09.859 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e0e4586d5b450f52-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.860 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:09.860 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:09.861 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.861 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.862 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.862 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.863 DEBUG:boto:Data: 2022-09-03T07:31:09.863 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.864 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.865 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.865 DEBUG:boto:Params: {} 2022-09-03T07:31:09.866 DEBUG:boto:Token: None 2022-09-03T07:31:09.866 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.867 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xQxNLRVF1CO/DAlTe4GcyzLGAD4= 2022-09-03T07:31:09.868 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xQxNLRVF1CO/DAlTe4GcyzLGAD4=', 'Content-Length': '0'} 2022-09-03T07:31:09.872 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c19008e4283294f0-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.873 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:31:09.874 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:09.875 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:09.876 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.877 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.878 DEBUG:boto:Method: GET 2022-09-03T07:31:09.879 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.880 DEBUG:boto:Data: 2022-09-03T07:31:09.881 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.882 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.882 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.883 DEBUG:boto:Params: {} 2022-09-03T07:31:09.884 DEBUG:boto:Token: None 2022-09-03T07:31:09.885 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.885 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sGNbqwBHbE8Ksk3bfbgOzxICFNU= 2022-09-03T07:31:09.886 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sGNbqwBHbE8Ksk3bfbgOzxICFNU=', 'Content-Length': '0'} 2022-09-03T07:31:09.889 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed0824e2af3e563d-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.930 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:09.931 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:09.932 DEBUG:boto:path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.932 DEBUG:boto:auth_path=/avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.933 DEBUG:boto:Method: GET 2022-09-03T07:31:09.934 DEBUG:boto:Path: /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.934 DEBUG:boto:Data: 2022-09-03T07:31:09.935 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.935 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.936 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.936 DEBUG:boto:Params: {} 2022-09-03T07:31:09.937 DEBUG:boto:Token: None 2022-09-03T07:31:09.938 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj57?versionId=null 2022-09-03T07:31:09.939 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sGNbqwBHbE8Ksk3bfbgOzxICFNU= 2022-09-03T07:31:09.939 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sGNbqwBHbE8Ksk3bfbgOzxICFNU=', 'Content-Length': '0'} 2022-09-03T07:31:09.941 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c36f723105a96e1e-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.942 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.943 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.943 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:31:09.944 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:31:09.945 DEBUG:boto:Method: GET 2022-09-03T07:31:09.945 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:31:09.946 DEBUG:boto:Data: 2022-09-03T07:31:09.947 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.947 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.948 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.948 DEBUG:boto:Params: {} 2022-09-03T07:31:09.949 DEBUG:boto:Token: None 2022-09-03T07:31:09.949 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.953 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c8cb2f1c03da508b-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.954 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.955 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:09.955 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:09.956 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:31:09.957 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:31:09.957 DEBUG:boto:Method: GET 2022-09-03T07:31:09.958 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:31:09.958 DEBUG:boto:Data: 2022-09-03T07:31:09.959 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.959 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.960 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.960 DEBUG:boto:Params: {} 2022-09-03T07:31:09.961 DEBUG:boto:Token: None 2022-09-03T07:31:09.962 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/ 2022-09-03T07:31:09.962 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is= 2022-09-03T07:31:09.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cPEIzJ59OCqv+BE0RWY1iPzc2Is=', 'Content-Length': '0'} 2022-09-03T07:31:09.968 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002b1941c661f64662-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.970 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:09.971 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:09.972 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:09.972 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.973 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:31:09.973 DEBUG:boto:Data: 2022-09-03T07:31:09.974 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.975 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.975 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.975 DEBUG:boto:Params: {} 2022-09-03T07:31:09.976 DEBUG:boto:Token: None 2022-09-03T07:31:09.977 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj57 2022-09-03T07:31:09.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BrVCpi2CZaY66XARFN1Dlww7Rjo= 2022-09-03T07:31:09.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BrVCpi2CZaY66XARFN1Dlww7Rjo=', 'Content-Length': '0'} 2022-09-03T07:31:09.980 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009bef4dd188011e8c-00631302bd-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.981 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:09.981 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:09.982 DEBUG:boto:Method: HEAD 2022-09-03T07:31:09.982 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:31:09.983 DEBUG:boto:Data: 2022-09-03T07:31:09.983 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.984 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:09.984 DEBUG:boto:Port: 8000 2022-09-03T07:31:09.985 DEBUG:boto:Params: {} 2022-09-03T07:31:09.985 DEBUG:boto:Token: None 2022-09-03T07:31:09.986 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj57 2022-09-03T07:31:09.986 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BrVCpi2CZaY66XARFN1Dlww7Rjo= 2022-09-03T07:31:09.987 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BrVCpi2CZaY66XARFN1Dlww7Rjo=', 'Content-Length': '0'} 2022-09-03T07:31:09.990 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001a019090516e1798-00631302bd-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:09 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:09.991 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj57 2022-09-03T07:31:09.992 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:09.992 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:09.993 DEBUG:boto:Method: GET 2022-09-03T07:31:09.994 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:31:09.994 DEBUG:boto:Data: 2022-09-03T07:31:09.995 DEBUG:boto:Headers: {} 2022-09-03T07:31:09.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:09.996 DEBUG:boto:Port: 8001 2022-09-03T07:31:09.996 DEBUG:boto:Params: {} 2022-09-03T07:31:09.997 DEBUG:boto:Token: None 2022-09-03T07:31:09.998 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:09 GMT /avysmw-75/obj57 2022-09-03T07:31:09.998 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TNOZdqc2YbThrZRcCqFF0JT8kdw= 2022-09-03T07:31:09.999 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:09 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TNOZdqc2YbThrZRcCqFF0JT8kdw=', 'Content-Length': '0'} 2022-09-03T07:31:10.002 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037011be3f798779f-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.003 DEBUG:boto:path=/avysmw-75/obj57 2022-09-03T07:31:10.003 DEBUG:boto:auth_path=/avysmw-75/obj57 2022-09-03T07:31:10.004 DEBUG:boto:Method: GET 2022-09-03T07:31:10.004 DEBUG:boto:Path: /avysmw-75/obj57 2022-09-03T07:31:10.005 DEBUG:boto:Data: 2022-09-03T07:31:10.005 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.006 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.007 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.007 DEBUG:boto:Params: {} 2022-09-03T07:31:10.008 DEBUG:boto:Token: None 2022-09-03T07:31:10.009 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj57 2022-09-03T07:31:10.010 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eq+XU7ZlaT5oB8qdSYCYDVZDPR0= 2022-09-03T07:31:10.010 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eq+XU7ZlaT5oB8qdSYCYDVZDPR0=', 'Content-Length': '0'} 2022-09-03T07:31:10.012 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009d218dd5974d440-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.013 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.014 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.014 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:31:10.015 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:31:10.015 DEBUG:boto:Method: GET 2022-09-03T07:31:10.016 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:31:10.016 DEBUG:boto:Data: 2022-09-03T07:31:10.017 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.018 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.018 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.019 DEBUG:boto:Params: {} 2022-09-03T07:31:10.019 DEBUG:boto:Token: None 2022-09-03T07:31:10.020 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.024 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a8f4fa0432debda-00631302be-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.025 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.026 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.026 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.027 DEBUG:boto:path=/avysmw-75/?prefix=obj57 2022-09-03T07:31:10.027 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj57 2022-09-03T07:31:10.028 DEBUG:boto:Method: GET 2022-09-03T07:31:10.028 DEBUG:boto:Path: /avysmw-75/?prefix=obj57 2022-09-03T07:31:10.029 DEBUG:boto:Data: 2022-09-03T07:31:10.029 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.030 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.030 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.031 DEBUG:boto:Params: {} 2022-09-03T07:31:10.031 DEBUG:boto:Token: None 2022-09-03T07:31:10.032 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.033 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.036 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000df5f7a526d3508f5-00631302be-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.036 DEBUG:boto:b'avysmw-75obj571000falseobj572022-09-03T07:15:42.542Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.037 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:31:10.038 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.038 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.039 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.040 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.040 DEBUG:boto:Method: GET 2022-09-03T07:31:10.041 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.042 DEBUG:boto:Data: 2022-09-03T07:31:10.043 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.044 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.044 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.045 DEBUG:boto:Params: {} 2022-09-03T07:31:10.046 DEBUG:boto:Token: None 2022-09-03T07:31:10.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.048 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dj9Z6la0dj5+29cg2nP1L5bcZQo= 2022-09-03T07:31:10.049 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dj9Z6la0dj5+29cg2nP1L5bcZQo=', 'Content-Length': '0'} 2022-09-03T07:31:10.052 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4f4fd350c6606b6-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.053 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.054 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.055 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.055 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.056 DEBUG:boto:Method: GET 2022-09-03T07:31:10.056 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.057 DEBUG:boto:Data: 2022-09-03T07:31:10.058 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.059 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.060 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.061 DEBUG:boto:Params: {} 2022-09-03T07:31:10.062 DEBUG:boto:Token: None 2022-09-03T07:31:10.063 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.065 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dj9Z6la0dj5+29cg2nP1L5bcZQo= 2022-09-03T07:31:10.066 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dj9Z6la0dj5+29cg2nP1L5bcZQo=', 'Content-Length': '0'} 2022-09-03T07:31:10.069 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000003e0e8c2b61f1559-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.070 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.071 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.072 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.073 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.073 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.074 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.075 DEBUG:boto:Data: 2022-09-03T07:31:10.075 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.076 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.077 DEBUG:boto:Params: {} 2022-09-03T07:31:10.077 DEBUG:boto:Token: None 2022-09-03T07:31:10.078 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.079 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:alp/I07mLW1QkObGzz75sS0n9gQ= 2022-09-03T07:31:10.079 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:alp/I07mLW1QkObGzz75sS0n9gQ=', 'Content-Length': '0'} 2022-09-03T07:31:10.081 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000716a9c84d6eb0a06-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.083 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.084 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.085 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.086 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.087 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.088 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.088 DEBUG:boto:Data: 2022-09-03T07:31:10.089 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.090 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.091 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.091 DEBUG:boto:Params: {} 2022-09-03T07:31:10.092 DEBUG:boto:Token: None 2022-09-03T07:31:10.092 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.093 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:alp/I07mLW1QkObGzz75sS0n9gQ= 2022-09-03T07:31:10.094 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:alp/I07mLW1QkObGzz75sS0n9gQ=', 'Content-Length': '0'} 2022-09-03T07:31:10.096 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009ef3727785f4ed46-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.097 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:31:10.097 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.098 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.098 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.099 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.100 DEBUG:boto:Method: GET 2022-09-03T07:31:10.101 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.102 DEBUG:boto:Data: 2022-09-03T07:31:10.103 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.103 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.104 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.104 DEBUG:boto:Params: {} 2022-09-03T07:31:10.105 DEBUG:boto:Token: None 2022-09-03T07:31:10.106 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.114 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dj9Z6la0dj5+29cg2nP1L5bcZQo= 2022-09-03T07:31:10.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dj9Z6la0dj5+29cg2nP1L5bcZQo=', 'Content-Length': '0'} 2022-09-03T07:31:10.118 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007126e1259a7501a5-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.119 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.119 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.120 DEBUG:boto:path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.120 DEBUG:boto:auth_path=/avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.121 DEBUG:boto:Method: GET 2022-09-03T07:31:10.121 DEBUG:boto:Path: /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.122 DEBUG:boto:Data: 2022-09-03T07:31:10.122 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.123 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.123 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.124 DEBUG:boto:Params: {} 2022-09-03T07:31:10.125 DEBUG:boto:Token: None 2022-09-03T07:31:10.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58?versionId=null 2022-09-03T07:31:10.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:dj9Z6la0dj5+29cg2nP1L5bcZQo= 2022-09-03T07:31:10.128 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:dj9Z6la0dj5+29cg2nP1L5bcZQo=', 'Content-Length': '0'} 2022-09-03T07:31:10.130 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099d416d0fefdf4be-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.131 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.132 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.132 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:31:10.133 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:31:10.133 DEBUG:boto:Method: GET 2022-09-03T07:31:10.134 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:31:10.134 DEBUG:boto:Data: 2022-09-03T07:31:10.135 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.136 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.137 DEBUG:boto:Params: {} 2022-09-03T07:31:10.138 DEBUG:boto:Token: None 2022-09-03T07:31:10.139 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.146 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f15d491481606723-00631302be-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.147 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.148 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.148 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.149 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:31:10.150 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:31:10.150 DEBUG:boto:Method: GET 2022-09-03T07:31:10.151 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:31:10.158 DEBUG:boto:Data: 2022-09-03T07:31:10.159 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.160 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.161 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.164 DEBUG:boto:Params: {} 2022-09-03T07:31:10.165 DEBUG:boto:Token: None 2022-09-03T07:31:10.166 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.167 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.168 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.171 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000062ff96c7680b184f-00631302be-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.214 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.217 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.218 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.219 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.220 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:31:10.221 DEBUG:boto:Data: 2022-09-03T07:31:10.222 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.223 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.224 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.225 DEBUG:boto:Params: {} 2022-09-03T07:31:10.226 DEBUG:boto:Token: None 2022-09-03T07:31:10.226 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58 2022-09-03T07:31:10.227 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uFKzFNdciZIwZDFe0c0vd7VGPls= 2022-09-03T07:31:10.228 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uFKzFNdciZIwZDFe0c0vd7VGPls=', 'Content-Length': '0'} 2022-09-03T07:31:10.230 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc1eb16595aa4e3b-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.232 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.232 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.233 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.234 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:31:10.234 DEBUG:boto:Data: 2022-09-03T07:31:10.235 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.235 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.236 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.236 DEBUG:boto:Params: {} 2022-09-03T07:31:10.237 DEBUG:boto:Token: None 2022-09-03T07:31:10.238 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58 2022-09-03T07:31:10.239 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uFKzFNdciZIwZDFe0c0vd7VGPls= 2022-09-03T07:31:10.239 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uFKzFNdciZIwZDFe0c0vd7VGPls=', 'Content-Length': '0'} 2022-09-03T07:31:10.241 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c02a024b8b54a1ec-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.242 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj58 2022-09-03T07:31:10.243 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.243 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.244 DEBUG:boto:Method: GET 2022-09-03T07:31:10.245 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:31:10.246 DEBUG:boto:Data: 2022-09-03T07:31:10.247 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.248 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.249 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.249 DEBUG:boto:Params: {} 2022-09-03T07:31:10.251 DEBUG:boto:Token: None 2022-09-03T07:31:10.252 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58 2022-09-03T07:31:10.253 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:krlco3krxKpepxAidDF7UXMSLjs= 2022-09-03T07:31:10.254 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:krlco3krxKpepxAidDF7UXMSLjs=', 'Content-Length': '0'} 2022-09-03T07:31:10.256 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ce5b1530422f1407-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.258 DEBUG:boto:path=/avysmw-75/obj58 2022-09-03T07:31:10.258 DEBUG:boto:auth_path=/avysmw-75/obj58 2022-09-03T07:31:10.259 DEBUG:boto:Method: GET 2022-09-03T07:31:10.259 DEBUG:boto:Path: /avysmw-75/obj58 2022-09-03T07:31:10.260 DEBUG:boto:Data: 2022-09-03T07:31:10.260 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.261 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.262 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.262 DEBUG:boto:Params: {} 2022-09-03T07:31:10.263 DEBUG:boto:Token: None 2022-09-03T07:31:10.263 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj58 2022-09-03T07:31:10.264 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:krlco3krxKpepxAidDF7UXMSLjs= 2022-09-03T07:31:10.265 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:krlco3krxKpepxAidDF7UXMSLjs=', 'Content-Length': '0'} 2022-09-03T07:31:10.267 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000085b1a3820c071e41-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.268 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.269 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.269 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:31:10.270 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:31:10.271 DEBUG:boto:Method: GET 2022-09-03T07:31:10.271 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:31:10.272 DEBUG:boto:Data: 2022-09-03T07:31:10.272 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.273 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.274 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.274 DEBUG:boto:Params: {} 2022-09-03T07:31:10.275 DEBUG:boto:Token: None 2022-09-03T07:31:10.276 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.276 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.277 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.281 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000848d50ffa14e7bdc-00631302be-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.281 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.282 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.283 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.284 DEBUG:boto:path=/avysmw-75/?prefix=obj58 2022-09-03T07:31:10.284 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj58 2022-09-03T07:31:10.285 DEBUG:boto:Method: GET 2022-09-03T07:31:10.285 DEBUG:boto:Path: /avysmw-75/?prefix=obj58 2022-09-03T07:31:10.286 DEBUG:boto:Data: 2022-09-03T07:31:10.287 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.287 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.288 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.288 DEBUG:boto:Params: {} 2022-09-03T07:31:10.289 DEBUG:boto:Token: None 2022-09-03T07:31:10.290 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.290 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.291 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.294 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000397dbd27f624e3d4-00631302be-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.295 DEBUG:boto:b'avysmw-75obj581000falseobj582022-09-03T07:24:23.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.297 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:31:10.298 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.299 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.300 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.301 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.302 DEBUG:boto:Method: GET 2022-09-03T07:31:10.303 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.303 DEBUG:boto:Data: 2022-09-03T07:31:10.304 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.305 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.307 DEBUG:boto:Params: {} 2022-09-03T07:31:10.307 DEBUG:boto:Token: None 2022-09-03T07:31:10.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.309 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xCjSdV97x/CtOG++9EJZMPcyklE= 2022-09-03T07:31:10.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xCjSdV97x/CtOG++9EJZMPcyklE=', 'Content-Length': '0'} 2022-09-03T07:31:10.313 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eaebacf381a45bfd-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.315 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.316 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.317 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.318 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.319 DEBUG:boto:Method: GET 2022-09-03T07:31:10.320 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.321 DEBUG:boto:Data: 2022-09-03T07:31:10.322 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.322 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.323 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.324 DEBUG:boto:Params: {} 2022-09-03T07:31:10.325 DEBUG:boto:Token: None 2022-09-03T07:31:10.327 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xCjSdV97x/CtOG++9EJZMPcyklE= 2022-09-03T07:31:10.330 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xCjSdV97x/CtOG++9EJZMPcyklE=', 'Content-Length': '0'} 2022-09-03T07:31:10.333 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000093d20e57b811909b-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.334 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.335 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.336 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.336 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.337 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.338 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.338 DEBUG:boto:Data: 2022-09-03T07:31:10.339 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.339 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.340 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.340 DEBUG:boto:Params: {} 2022-09-03T07:31:10.341 DEBUG:boto:Token: None 2022-09-03T07:31:10.342 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S6WXTjG/rRjyToeKvBoza+d+vLw= 2022-09-03T07:31:10.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S6WXTjG/rRjyToeKvBoza+d+vLw=', 'Content-Length': '0'} 2022-09-03T07:31:10.347 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cac5c5c1d1608ee9-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.348 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.349 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.350 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.350 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.351 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.351 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.352 DEBUG:boto:Data: 2022-09-03T07:31:10.352 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.353 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.353 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.354 DEBUG:boto:Params: {} 2022-09-03T07:31:10.354 DEBUG:boto:Token: None 2022-09-03T07:31:10.355 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.356 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S6WXTjG/rRjyToeKvBoza+d+vLw= 2022-09-03T07:31:10.356 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S6WXTjG/rRjyToeKvBoza+d+vLw=', 'Content-Length': '0'} 2022-09-03T07:31:10.360 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000d0fa964ca35b398-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.361 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:31:10.362 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.362 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.363 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.364 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.364 DEBUG:boto:Method: GET 2022-09-03T07:31:10.365 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.365 DEBUG:boto:Data: 2022-09-03T07:31:10.366 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.366 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.367 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.367 DEBUG:boto:Params: {} 2022-09-03T07:31:10.368 DEBUG:boto:Token: None 2022-09-03T07:31:10.369 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.369 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xCjSdV97x/CtOG++9EJZMPcyklE= 2022-09-03T07:31:10.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xCjSdV97x/CtOG++9EJZMPcyklE=', 'Content-Length': '0'} 2022-09-03T07:31:10.372 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000489a2d97e1e74701-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.373 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.374 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.375 DEBUG:boto:path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.375 DEBUG:boto:auth_path=/avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.376 DEBUG:boto:Method: GET 2022-09-03T07:31:10.377 DEBUG:boto:Path: /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.377 DEBUG:boto:Data: 2022-09-03T07:31:10.378 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.379 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.379 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.380 DEBUG:boto:Params: {} 2022-09-03T07:31:10.380 DEBUG:boto:Token: None 2022-09-03T07:31:10.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62?versionId=null 2022-09-03T07:31:10.382 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xCjSdV97x/CtOG++9EJZMPcyklE= 2022-09-03T07:31:10.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xCjSdV97x/CtOG++9EJZMPcyklE=', 'Content-Length': '0'} 2022-09-03T07:31:10.384 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1d6ed3ee38b9b69-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.385 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.386 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.387 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:10.388 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:10.389 DEBUG:boto:Method: GET 2022-09-03T07:31:10.390 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:31:10.391 DEBUG:boto:Data: 2022-09-03T07:31:10.392 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.393 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.394 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.395 DEBUG:boto:Params: {} 2022-09-03T07:31:10.404 DEBUG:boto:Token: None 2022-09-03T07:31:10.404 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.409 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a63a091f3f7437a7-00631302be-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.410 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.411 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.411 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.412 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:10.412 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:10.413 DEBUG:boto:Method: GET 2022-09-03T07:31:10.414 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:31:10.414 DEBUG:boto:Data: 2022-09-03T07:31:10.415 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.415 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.416 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.417 DEBUG:boto:Params: {} 2022-09-03T07:31:10.417 DEBUG:boto:Token: None 2022-09-03T07:31:10.418 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.419 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.422 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000c8870c271246e3a-00631302be-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.422 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.423 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.424 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.425 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.425 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:31:10.426 DEBUG:boto:Data: 2022-09-03T07:31:10.426 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.427 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.428 DEBUG:boto:Params: {} 2022-09-03T07:31:10.429 DEBUG:boto:Token: None 2022-09-03T07:31:10.429 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62 2022-09-03T07:31:10.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q3nXxk5OJ4fqyy3Uqh4xJZrHDm8= 2022-09-03T07:31:10.430 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q3nXxk5OJ4fqyy3Uqh4xJZrHDm8=', 'Content-Length': '0'} 2022-09-03T07:31:10.432 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008709bc7484426d3b-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.433 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.435 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.435 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.443 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:31:10.444 DEBUG:boto:Data: 2022-09-03T07:31:10.445 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.445 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.446 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.446 DEBUG:boto:Params: {} 2022-09-03T07:31:10.447 DEBUG:boto:Token: None 2022-09-03T07:31:10.447 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62 2022-09-03T07:31:10.448 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q3nXxk5OJ4fqyy3Uqh4xJZrHDm8= 2022-09-03T07:31:10.449 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q3nXxk5OJ4fqyy3Uqh4xJZrHDm8=', 'Content-Length': '0'} 2022-09-03T07:31:10.451 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ca0644b79b157fd-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.452 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj62 2022-09-03T07:31:10.453 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.453 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.454 DEBUG:boto:Method: GET 2022-09-03T07:31:10.455 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:31:10.455 DEBUG:boto:Data: 2022-09-03T07:31:10.456 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.457 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.457 DEBUG:boto:Params: {} 2022-09-03T07:31:10.458 DEBUG:boto:Token: None 2022-09-03T07:31:10.459 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62 2022-09-03T07:31:10.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cfNz60iN/RhXDKTPvDnFfEKH0C8= 2022-09-03T07:31:10.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cfNz60iN/RhXDKTPvDnFfEKH0C8=', 'Content-Length': '0'} 2022-09-03T07:31:10.473 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000075215f1eef541c59-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.474 DEBUG:boto:path=/avysmw-75/obj62 2022-09-03T07:31:10.475 DEBUG:boto:auth_path=/avysmw-75/obj62 2022-09-03T07:31:10.475 DEBUG:boto:Method: GET 2022-09-03T07:31:10.476 DEBUG:boto:Path: /avysmw-75/obj62 2022-09-03T07:31:10.476 DEBUG:boto:Data: 2022-09-03T07:31:10.477 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.477 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.478 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.478 DEBUG:boto:Params: {} 2022-09-03T07:31:10.479 DEBUG:boto:Token: None 2022-09-03T07:31:10.480 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj62 2022-09-03T07:31:10.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:cfNz60iN/RhXDKTPvDnFfEKH0C8= 2022-09-03T07:31:10.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:cfNz60iN/RhXDKTPvDnFfEKH0C8=', 'Content-Length': '0'} 2022-09-03T07:31:10.484 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ec96f7d1a48eb03-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.528 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.529 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.530 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:10.531 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:10.532 DEBUG:boto:Method: GET 2022-09-03T07:31:10.533 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:31:10.533 DEBUG:boto:Data: 2022-09-03T07:31:10.534 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.535 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.535 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.537 DEBUG:boto:Params: {} 2022-09-03T07:31:10.537 DEBUG:boto:Token: None 2022-09-03T07:31:10.538 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.539 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.540 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.543 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004578aed5d65b1811-00631302be-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.543 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.545 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.545 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.546 DEBUG:boto:path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:10.548 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj62 2022-09-03T07:31:10.549 DEBUG:boto:Method: GET 2022-09-03T07:31:10.549 DEBUG:boto:Path: /avysmw-75/?prefix=obj62 2022-09-03T07:31:10.550 DEBUG:boto:Data: 2022-09-03T07:31:10.551 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.551 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.552 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.552 DEBUG:boto:Params: {} 2022-09-03T07:31:10.553 DEBUG:boto:Token: None 2022-09-03T07:31:10.553 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.554 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.555 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.558 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a42d8d3644702933-00631302be-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.558 DEBUG:boto:b'avysmw-75obj621000falseobj622022-09-03T07:26:11.009Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.559 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:31:10.560 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:10.561 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:10.562 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.562 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.563 DEBUG:boto:Method: GET 2022-09-03T07:31:10.563 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.564 DEBUG:boto:Data: 2022-09-03T07:31:10.565 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.565 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.566 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.567 DEBUG:boto:Params: {} 2022-09-03T07:31:10.571 DEBUG:boto:Token: None 2022-09-03T07:31:10.572 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M1FVU1mpcQEWx+VTqnZ+T5j25GM= 2022-09-03T07:31:10.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M1FVU1mpcQEWx+VTqnZ+T5j25GM=', 'Content-Length': '0'} 2022-09-03T07:31:10.576 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004b9f842c7cc04e87-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.623 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:10.624 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:10.624 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.625 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.625 DEBUG:boto:Method: GET 2022-09-03T07:31:10.626 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.627 DEBUG:boto:Data: 2022-09-03T07:31:10.627 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.628 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.628 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.629 DEBUG:boto:Params: {} 2022-09-03T07:31:10.629 DEBUG:boto:Token: None 2022-09-03T07:31:10.630 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.631 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M1FVU1mpcQEWx+VTqnZ+T5j25GM= 2022-09-03T07:31:10.632 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M1FVU1mpcQEWx+VTqnZ+T5j25GM=', 'Content-Length': '0'} 2022-09-03T07:31:10.636 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000afd3bb7c2c276762-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.638 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:10.639 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:10.639 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.640 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.640 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.641 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.642 DEBUG:boto:Data: 2022-09-03T07:31:10.642 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.645 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.646 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.647 DEBUG:boto:Params: {} 2022-09-03T07:31:10.648 DEBUG:boto:Token: None 2022-09-03T07:31:10.649 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jkGH9sjx0UD7dEdOinNbhdTEvcg= 2022-09-03T07:31:10.651 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jkGH9sjx0UD7dEdOinNbhdTEvcg=', 'Content-Length': '0'} 2022-09-03T07:31:10.656 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000dbc389d8161313fe-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.658 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:10.659 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:10.660 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.661 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.664 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.669 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.670 DEBUG:boto:Data: 2022-09-03T07:31:10.671 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.672 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.673 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.674 DEBUG:boto:Params: {} 2022-09-03T07:31:10.675 DEBUG:boto:Token: None 2022-09-03T07:31:10.677 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.678 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jkGH9sjx0UD7dEdOinNbhdTEvcg= 2022-09-03T07:31:10.679 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jkGH9sjx0UD7dEdOinNbhdTEvcg=', 'Content-Length': '0'} 2022-09-03T07:31:10.682 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c88ade6b87f8abc4-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.683 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:31:10.684 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:10.685 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:10.686 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.686 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.687 DEBUG:boto:Method: GET 2022-09-03T07:31:10.687 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.688 DEBUG:boto:Data: 2022-09-03T07:31:10.688 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.689 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.690 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.690 DEBUG:boto:Params: {} 2022-09-03T07:31:10.691 DEBUG:boto:Token: None 2022-09-03T07:31:10.692 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.693 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M1FVU1mpcQEWx+VTqnZ+T5j25GM= 2022-09-03T07:31:10.694 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M1FVU1mpcQEWx+VTqnZ+T5j25GM=', 'Content-Length': '0'} 2022-09-03T07:31:10.697 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d889395f5009e2b4-00631302be-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.697 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:10.698 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:10.699 DEBUG:boto:path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.700 DEBUG:boto:auth_path=/avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.701 DEBUG:boto:Method: GET 2022-09-03T07:31:10.701 DEBUG:boto:Path: /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.702 DEBUG:boto:Data: 2022-09-03T07:31:10.703 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.704 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.704 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.705 DEBUG:boto:Params: {} 2022-09-03T07:31:10.705 DEBUG:boto:Token: None 2022-09-03T07:31:10.706 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/obj68?versionId=null 2022-09-03T07:31:10.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:M1FVU1mpcQEWx+VTqnZ+T5j25GM= 2022-09-03T07:31:10.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:M1FVU1mpcQEWx+VTqnZ+T5j25GM=', 'Content-Length': '0'} 2022-09-03T07:31:10.714 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a69f0bb10cf7ac43-00631302be-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.717 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.718 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.719 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:10.721 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:10.722 DEBUG:boto:Method: GET 2022-09-03T07:31:10.723 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:31:10.970 DEBUG:boto:Data: 2022-09-03T07:31:10.971 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.972 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:10.973 DEBUG:boto:Port: 8001 2022-09-03T07:31:10.974 DEBUG:boto:Params: {} 2022-09-03T07:31:10.975 DEBUG:boto:Token: None 2022-09-03T07:31:10.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.981 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000dbdd1dc5342a1d9a-00631302be-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.982 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.983 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:10.983 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:10.984 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:10.985 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:10.986 DEBUG:boto:Method: GET 2022-09-03T07:31:10.987 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:31:10.987 DEBUG:boto:Data: 2022-09-03T07:31:10.988 DEBUG:boto:Headers: {} 2022-09-03T07:31:10.988 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:10.989 DEBUG:boto:Port: 8000 2022-09-03T07:31:10.989 DEBUG:boto:Params: {} 2022-09-03T07:31:10.990 DEBUG:boto:Token: None 2022-09-03T07:31:10.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:10 GMT /avysmw-75/ 2022-09-03T07:31:10.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE= 2022-09-03T07:31:10.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:10 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GDWJW/zNE/OVKION+yfN8tsyiFE=', 'Content-Length': '0'} 2022-09-03T07:31:10.994 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098af558edb79507e-00631302be-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:10 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:10.995 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:10.996 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:10.997 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:10.997 DEBUG:boto:Method: HEAD 2022-09-03T07:31:10.998 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:31:11.000 DEBUG:boto:Data: 2022-09-03T07:31:11.002 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.003 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.005 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.006 DEBUG:boto:Params: {} 2022-09-03T07:31:11.010 DEBUG:boto:Token: None 2022-09-03T07:31:11.011 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj68 2022-09-03T07:31:11.012 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ubn+wC9KyfwjuvrUTbuPlXxfU5I= 2022-09-03T07:31:11.013 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ubn+wC9KyfwjuvrUTbuPlXxfU5I=', 'Content-Length': '0'} 2022-09-03T07:31:11.015 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c217e952d75200ec-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.017 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:11.018 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:11.019 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.019 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:31:11.020 DEBUG:boto:Data: 2022-09-03T07:31:11.021 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.021 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.022 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.023 DEBUG:boto:Params: {} 2022-09-03T07:31:11.024 DEBUG:boto:Token: None 2022-09-03T07:31:11.026 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj68 2022-09-03T07:31:11.027 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Ubn+wC9KyfwjuvrUTbuPlXxfU5I= 2022-09-03T07:31:11.028 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Ubn+wC9KyfwjuvrUTbuPlXxfU5I=', 'Content-Length': '0'} 2022-09-03T07:31:11.030 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000056e990ac248d771c-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.031 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj68 2022-09-03T07:31:11.032 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:11.033 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:11.034 DEBUG:boto:Method: GET 2022-09-03T07:31:11.035 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:31:11.036 DEBUG:boto:Data: 2022-09-03T07:31:11.037 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.038 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.038 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.039 DEBUG:boto:Params: {} 2022-09-03T07:31:11.041 DEBUG:boto:Token: None 2022-09-03T07:31:11.042 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj68 2022-09-03T07:31:11.043 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q3vOSrhRamDjCsDV0218mJxXGqY= 2022-09-03T07:31:11.043 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q3vOSrhRamDjCsDV0218mJxXGqY=', 'Content-Length': '0'} 2022-09-03T07:31:11.046 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fed48a54ec3d5318-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.047 DEBUG:boto:path=/avysmw-75/obj68 2022-09-03T07:31:11.048 DEBUG:boto:auth_path=/avysmw-75/obj68 2022-09-03T07:31:11.049 DEBUG:boto:Method: GET 2022-09-03T07:31:11.050 DEBUG:boto:Path: /avysmw-75/obj68 2022-09-03T07:31:11.051 DEBUG:boto:Data: 2022-09-03T07:31:11.052 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.053 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.054 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.055 DEBUG:boto:Params: {} 2022-09-03T07:31:11.056 DEBUG:boto:Token: None 2022-09-03T07:31:11.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj68 2022-09-03T07:31:11.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q3vOSrhRamDjCsDV0218mJxXGqY= 2022-09-03T07:31:11.059 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q3vOSrhRamDjCsDV0218mJxXGqY=', 'Content-Length': '0'} 2022-09-03T07:31:11.063 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000054d3709eb7c14891-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.115 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.116 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.116 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:11.117 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:11.117 DEBUG:boto:Method: GET 2022-09-03T07:31:11.118 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:31:11.119 DEBUG:boto:Data: 2022-09-03T07:31:11.119 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.120 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.120 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.121 DEBUG:boto:Params: {} 2022-09-03T07:31:11.121 DEBUG:boto:Token: None 2022-09-03T07:31:11.124 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.125 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.126 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.135 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b0d63a1ebdad6c8a-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.136 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.137 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.138 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.139 DEBUG:boto:path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:11.139 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj68 2022-09-03T07:31:11.140 DEBUG:boto:Method: GET 2022-09-03T07:31:11.141 DEBUG:boto:Path: /avysmw-75/?prefix=obj68 2022-09-03T07:31:11.142 DEBUG:boto:Data: 2022-09-03T07:31:11.143 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.144 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.145 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.146 DEBUG:boto:Params: {} 2022-09-03T07:31:11.148 DEBUG:boto:Token: None 2022-09-03T07:31:11.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.151 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.154 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004f8003037c8d01a7-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.156 DEBUG:boto:b'avysmw-75obj681000falseobj682022-09-03T07:12:49.631Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.157 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:31:11.158 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.159 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.160 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.161 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.162 DEBUG:boto:Method: GET 2022-09-03T07:31:11.163 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.164 DEBUG:boto:Data: 2022-09-03T07:31:11.165 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.166 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.167 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.168 DEBUG:boto:Params: {} 2022-09-03T07:31:11.169 DEBUG:boto:Token: None 2022-09-03T07:31:11.170 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.171 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZQBwQJSd1AL6F51zKcZUj/O1wos= 2022-09-03T07:31:11.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZQBwQJSd1AL6F51zKcZUj/O1wos=', 'Content-Length': '0'} 2022-09-03T07:31:11.175 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a8237ff8c4c1e0fc-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.177 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.177 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.178 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.179 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.180 DEBUG:boto:Method: GET 2022-09-03T07:31:11.181 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.182 DEBUG:boto:Data: 2022-09-03T07:31:11.183 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.184 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.184 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.185 DEBUG:boto:Params: {} 2022-09-03T07:31:11.185 DEBUG:boto:Token: None 2022-09-03T07:31:11.186 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.187 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZQBwQJSd1AL6F51zKcZUj/O1wos= 2022-09-03T07:31:11.188 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZQBwQJSd1AL6F51zKcZUj/O1wos=', 'Content-Length': '0'} 2022-09-03T07:31:11.191 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006367bbfad12f341d-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.192 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.193 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.194 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.195 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.195 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.196 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.197 DEBUG:boto:Data: 2022-09-03T07:31:11.198 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.199 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.200 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.202 DEBUG:boto:Params: {} 2022-09-03T07:31:11.203 DEBUG:boto:Token: None 2022-09-03T07:31:11.204 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.205 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DBGQ8IwBfBAF3BG4tNSKh00/wu8= 2022-09-03T07:31:11.206 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DBGQ8IwBfBAF3BG4tNSKh00/wu8=', 'Content-Length': '0'} 2022-09-03T07:31:11.209 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006a2baa2888b39e5a-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.211 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.212 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.213 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.216 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.217 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.219 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.220 DEBUG:boto:Data: 2022-09-03T07:31:11.221 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.222 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.223 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.224 DEBUG:boto:Params: {} 2022-09-03T07:31:11.224 DEBUG:boto:Token: None 2022-09-03T07:31:11.225 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.226 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DBGQ8IwBfBAF3BG4tNSKh00/wu8= 2022-09-03T07:31:11.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DBGQ8IwBfBAF3BG4tNSKh00/wu8=', 'Content-Length': '0'} 2022-09-03T07:31:11.231 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006cb91a842614edeb-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.232 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:31:11.233 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.233 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.234 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.234 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.235 DEBUG:boto:Method: GET 2022-09-03T07:31:11.236 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.236 DEBUG:boto:Data: 2022-09-03T07:31:11.237 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.237 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.238 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.238 DEBUG:boto:Params: {} 2022-09-03T07:31:11.239 DEBUG:boto:Token: None 2022-09-03T07:31:11.239 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZQBwQJSd1AL6F51zKcZUj/O1wos= 2022-09-03T07:31:11.241 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZQBwQJSd1AL6F51zKcZUj/O1wos=', 'Content-Length': '0'} 2022-09-03T07:31:11.243 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b3167d4bc6a0e0db-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.286 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.287 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.288 DEBUG:boto:path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.289 DEBUG:boto:auth_path=/avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.290 DEBUG:boto:Method: GET 2022-09-03T07:31:11.291 DEBUG:boto:Path: /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.292 DEBUG:boto:Data: 2022-09-03T07:31:11.293 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.294 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.295 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.296 DEBUG:boto:Params: {} 2022-09-03T07:31:11.297 DEBUG:boto:Token: None 2022-09-03T07:31:11.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69?versionId=null 2022-09-03T07:31:11.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZQBwQJSd1AL6F51zKcZUj/O1wos= 2022-09-03T07:31:11.300 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZQBwQJSd1AL6F51zKcZUj/O1wos=', 'Content-Length': '0'} 2022-09-03T07:31:11.303 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e8529a696cff73c-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.304 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.304 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.305 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:11.305 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:11.306 DEBUG:boto:Method: GET 2022-09-03T07:31:11.306 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:31:11.307 DEBUG:boto:Data: 2022-09-03T07:31:11.308 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.308 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.309 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.310 DEBUG:boto:Params: {} 2022-09-03T07:31:11.312 DEBUG:boto:Token: None 2022-09-03T07:31:11.312 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.313 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.314 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.320 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f75c8f68de27ff37-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.322 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.325 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.326 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.327 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:11.328 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:11.328 DEBUG:boto:Method: GET 2022-09-03T07:31:11.329 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:31:11.332 DEBUG:boto:Data: 2022-09-03T07:31:11.333 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.334 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.335 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.336 DEBUG:boto:Params: {} 2022-09-03T07:31:11.337 DEBUG:boto:Token: None 2022-09-03T07:31:11.339 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.341 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.344 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c4afdf20da077b21-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.345 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.347 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.348 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.348 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.349 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:31:11.350 DEBUG:boto:Data: 2022-09-03T07:31:11.350 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.351 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.351 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.352 DEBUG:boto:Params: {} 2022-09-03T07:31:11.353 DEBUG:boto:Token: None 2022-09-03T07:31:11.353 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69 2022-09-03T07:31:11.354 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DkyvRxv1lcdAGuNRVaMPpGiU+wo= 2022-09-03T07:31:11.355 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DkyvRxv1lcdAGuNRVaMPpGiU+wo=', 'Content-Length': '0'} 2022-09-03T07:31:11.359 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006e59dba96162f4ee-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.360 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.361 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.361 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.362 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:31:11.363 DEBUG:boto:Data: 2022-09-03T07:31:11.365 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.366 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.367 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.368 DEBUG:boto:Params: {} 2022-09-03T07:31:11.369 DEBUG:boto:Token: None 2022-09-03T07:31:11.370 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69 2022-09-03T07:31:11.372 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:DkyvRxv1lcdAGuNRVaMPpGiU+wo= 2022-09-03T07:31:11.373 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:DkyvRxv1lcdAGuNRVaMPpGiU+wo=', 'Content-Length': '0'} 2022-09-03T07:31:11.376 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007b5acb801e635b6b-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.377 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj69 2022-09-03T07:31:11.378 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.379 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.381 DEBUG:boto:Method: GET 2022-09-03T07:31:11.382 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:31:11.383 DEBUG:boto:Data: 2022-09-03T07:31:11.384 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.385 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.386 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.388 DEBUG:boto:Params: {} 2022-09-03T07:31:11.389 DEBUG:boto:Token: None 2022-09-03T07:31:11.390 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69 2022-09-03T07:31:11.392 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HkKVIIIYGic0Wyh++wLOF3EHCkA= 2022-09-03T07:31:11.393 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HkKVIIIYGic0Wyh++wLOF3EHCkA=', 'Content-Length': '0'} 2022-09-03T07:31:11.397 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002e05db4ecceb8457-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.440 DEBUG:boto:path=/avysmw-75/obj69 2022-09-03T07:31:11.441 DEBUG:boto:auth_path=/avysmw-75/obj69 2022-09-03T07:31:11.441 DEBUG:boto:Method: GET 2022-09-03T07:31:11.442 DEBUG:boto:Path: /avysmw-75/obj69 2022-09-03T07:31:11.442 DEBUG:boto:Data: 2022-09-03T07:31:11.443 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.443 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.444 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.444 DEBUG:boto:Params: {} 2022-09-03T07:31:11.445 DEBUG:boto:Token: None 2022-09-03T07:31:11.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj69 2022-09-03T07:31:11.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HkKVIIIYGic0Wyh++wLOF3EHCkA= 2022-09-03T07:31:11.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HkKVIIIYGic0Wyh++wLOF3EHCkA=', 'Content-Length': '0'} 2022-09-03T07:31:11.450 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012da1ec990ce57bb-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.452 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.453 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.457 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:11.458 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:11.459 DEBUG:boto:Method: GET 2022-09-03T07:31:11.460 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:31:11.461 DEBUG:boto:Data: 2022-09-03T07:31:11.462 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.464 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.465 DEBUG:boto:Params: {} 2022-09-03T07:31:11.466 DEBUG:boto:Token: None 2022-09-03T07:31:11.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.469 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.470 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.473 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000051fe0e4534bec9a4-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.475 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.477 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.479 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.480 DEBUG:boto:path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:11.481 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj69 2022-09-03T07:31:11.482 DEBUG:boto:Method: GET 2022-09-03T07:31:11.483 DEBUG:boto:Path: /avysmw-75/?prefix=obj69 2022-09-03T07:31:11.484 DEBUG:boto:Data: 2022-09-03T07:31:11.485 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.486 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.487 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.488 DEBUG:boto:Params: {} 2022-09-03T07:31:11.489 DEBUG:boto:Token: None 2022-09-03T07:31:11.490 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.491 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.496 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000074acf4443aff6939-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.498 DEBUG:boto:b'avysmw-75obj691000falseobj692022-09-03T07:21:21.604Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.499 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj74 2022-09-03T07:31:11.500 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.501 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.502 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.503 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.504 DEBUG:boto:Method: GET 2022-09-03T07:31:11.505 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.506 DEBUG:boto:Data: 2022-09-03T07:31:11.507 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.508 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.509 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.510 DEBUG:boto:Params: {} 2022-09-03T07:31:11.512 DEBUG:boto:Token: None 2022-09-03T07:31:11.513 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.514 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+6JIri/RUB8z59PUWWBahPzRRow= 2022-09-03T07:31:11.515 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+6JIri/RUB8z59PUWWBahPzRRow=', 'Content-Length': '0'} 2022-09-03T07:31:11.518 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d5ca551c562da99-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.519 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.520 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.521 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.523 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.523 DEBUG:boto:Method: GET 2022-09-03T07:31:11.525 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.526 DEBUG:boto:Data: 2022-09-03T07:31:11.527 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.528 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.529 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.530 DEBUG:boto:Params: {} 2022-09-03T07:31:11.531 DEBUG:boto:Token: None 2022-09-03T07:31:11.532 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+6JIri/RUB8z59PUWWBahPzRRow= 2022-09-03T07:31:11.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+6JIri/RUB8z59PUWWBahPzRRow=', 'Content-Length': '0'} 2022-09-03T07:31:11.537 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003bc1a1a55bb12aee-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.539 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.540 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.541 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.541 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.542 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.543 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.544 DEBUG:boto:Data: 2022-09-03T07:31:11.544 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.546 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.546 DEBUG:boto:Params: {} 2022-09-03T07:31:11.547 DEBUG:boto:Token: None 2022-09-03T07:31:11.548 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5dda44sZ9lX4jIX9izDydOWvntc= 2022-09-03T07:31:11.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5dda44sZ9lX4jIX9izDydOWvntc=', 'Content-Length': '0'} 2022-09-03T07:31:11.551 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c03968d57140a23a-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.552 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.553 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.553 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.554 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.554 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.555 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.556 DEBUG:boto:Data: 2022-09-03T07:31:11.556 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.557 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.557 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.558 DEBUG:boto:Params: {} 2022-09-03T07:31:11.558 DEBUG:boto:Token: None 2022-09-03T07:31:11.559 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.559 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:5dda44sZ9lX4jIX9izDydOWvntc= 2022-09-03T07:31:11.560 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:5dda44sZ9lX4jIX9izDydOWvntc=', 'Content-Length': '0'} 2022-09-03T07:31:11.563 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aa20c871740ec101-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.564 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj74 2022-09-03T07:31:11.565 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.566 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.566 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.567 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.568 DEBUG:boto:Method: GET 2022-09-03T07:31:11.568 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.569 DEBUG:boto:Data: 2022-09-03T07:31:11.569 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.571 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.572 DEBUG:boto:Params: {} 2022-09-03T07:31:11.572 DEBUG:boto:Token: None 2022-09-03T07:31:11.573 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.574 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+6JIri/RUB8z59PUWWBahPzRRow= 2022-09-03T07:31:11.575 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+6JIri/RUB8z59PUWWBahPzRRow=', 'Content-Length': '0'} 2022-09-03T07:31:11.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000228eed7aa0d6fba7-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.580 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.581 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.583 DEBUG:boto:path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.583 DEBUG:boto:auth_path=/avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.584 DEBUG:boto:Method: GET 2022-09-03T07:31:11.585 DEBUG:boto:Path: /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.586 DEBUG:boto:Data: 2022-09-03T07:31:11.587 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.588 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.588 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.589 DEBUG:boto:Params: {} 2022-09-03T07:31:11.590 DEBUG:boto:Token: None 2022-09-03T07:31:11.591 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74?versionId=null 2022-09-03T07:31:11.592 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+6JIri/RUB8z59PUWWBahPzRRow= 2022-09-03T07:31:11.593 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+6JIri/RUB8z59PUWWBahPzRRow=', 'Content-Length': '0'} 2022-09-03T07:31:11.595 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009790faa5002d49b4-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.596 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.597 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.597 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:11.598 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:11.599 DEBUG:boto:Method: GET 2022-09-03T07:31:11.599 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:31:11.600 DEBUG:boto:Data: 2022-09-03T07:31:11.600 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.601 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.602 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.602 DEBUG:boto:Params: {} 2022-09-03T07:31:11.603 DEBUG:boto:Token: None 2022-09-03T07:31:11.604 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.605 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.607 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.611 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000169fb610c455acc5-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.613 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.615 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.616 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.617 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:11.618 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:11.619 DEBUG:boto:Method: GET 2022-09-03T07:31:11.621 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:31:11.622 DEBUG:boto:Data: 2022-09-03T07:31:11.623 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.624 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.625 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.626 DEBUG:boto:Params: {} 2022-09-03T07:31:11.627 DEBUG:boto:Token: None 2022-09-03T07:31:11.628 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.631 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.634 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000caa684d2ca1d7cb3-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.636 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.637 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.638 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.639 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.639 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:31:11.640 DEBUG:boto:Data: 2022-09-03T07:31:11.641 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.642 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.643 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.644 DEBUG:boto:Params: {} 2022-09-03T07:31:11.645 DEBUG:boto:Token: None 2022-09-03T07:31:11.649 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74 2022-09-03T07:31:11.650 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xInb100Z0fKHwPrtKA6YE4JI4c0= 2022-09-03T07:31:11.651 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xInb100Z0fKHwPrtKA6YE4JI4c0=', 'Content-Length': '0'} 2022-09-03T07:31:11.654 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a356ecb6b24484fc-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.655 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.656 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.658 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.658 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:31:11.659 DEBUG:boto:Data: 2022-09-03T07:31:11.660 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.661 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.662 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.663 DEBUG:boto:Params: {} 2022-09-03T07:31:11.664 DEBUG:boto:Token: None 2022-09-03T07:31:11.665 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74 2022-09-03T07:31:11.666 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xInb100Z0fKHwPrtKA6YE4JI4c0= 2022-09-03T07:31:11.667 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xInb100Z0fKHwPrtKA6YE4JI4c0=', 'Content-Length': '0'} 2022-09-03T07:31:11.670 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a234ded37151158f-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.671 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj74 2022-09-03T07:31:11.672 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.673 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.673 DEBUG:boto:Method: GET 2022-09-03T07:31:11.674 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:31:11.675 DEBUG:boto:Data: 2022-09-03T07:31:11.676 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.677 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.678 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.681 DEBUG:boto:Params: {} 2022-09-03T07:31:11.682 DEBUG:boto:Token: None 2022-09-03T07:31:11.683 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74 2022-09-03T07:31:11.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y0QbCeXsGm3p5s8tCx/sIZOsyUo= 2022-09-03T07:31:11.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y0QbCeXsGm3p5s8tCx/sIZOsyUo=', 'Content-Length': '0'} 2022-09-03T07:31:11.688 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ef61047ed75c1163-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.730 DEBUG:boto:path=/avysmw-75/obj74 2022-09-03T07:31:11.731 DEBUG:boto:auth_path=/avysmw-75/obj74 2022-09-03T07:31:11.732 DEBUG:boto:Method: GET 2022-09-03T07:31:11.733 DEBUG:boto:Path: /avysmw-75/obj74 2022-09-03T07:31:11.734 DEBUG:boto:Data: 2022-09-03T07:31:11.735 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.736 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.737 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.737 DEBUG:boto:Params: {} 2022-09-03T07:31:11.739 DEBUG:boto:Token: None 2022-09-03T07:31:11.740 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj74 2022-09-03T07:31:11.741 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:y0QbCeXsGm3p5s8tCx/sIZOsyUo= 2022-09-03T07:31:11.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:y0QbCeXsGm3p5s8tCx/sIZOsyUo=', 'Content-Length': '0'} 2022-09-03T07:31:11.746 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:28:24 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8163ccb178c3db1-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.748 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.749 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.750 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:11.751 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:11.752 DEBUG:boto:Method: GET 2022-09-03T07:31:11.753 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:31:11.754 DEBUG:boto:Data: 2022-09-03T07:31:11.756 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.757 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.759 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.760 DEBUG:boto:Params: {} 2022-09-03T07:31:11.761 DEBUG:boto:Token: None 2022-09-03T07:31:11.763 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.764 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.767 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.771 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008541a7fe1df3dda6-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.773 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.776 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.777 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.778 DEBUG:boto:path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:11.779 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj74 2022-09-03T07:31:11.781 DEBUG:boto:Method: GET 2022-09-03T07:31:11.782 DEBUG:boto:Path: /avysmw-75/?prefix=obj74 2022-09-03T07:31:11.783 DEBUG:boto:Data: 2022-09-03T07:31:11.784 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.786 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.787 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.788 DEBUG:boto:Params: {} 2022-09-03T07:31:11.790 DEBUG:boto:Token: None 2022-09-03T07:31:11.792 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.793 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.795 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.798 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c8f6865fbe0914ba-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.801 DEBUG:boto:b'avysmw-75obj741000falseobj742022-09-03T07:28:24.765Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.804 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:31:11.805 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:11.807 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:11.808 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.809 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.811 DEBUG:boto:Method: GET 2022-09-03T07:31:11.812 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.813 DEBUG:boto:Data: 2022-09-03T07:31:11.815 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.816 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.817 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.819 DEBUG:boto:Params: {} 2022-09-03T07:31:11.820 DEBUG:boto:Token: None 2022-09-03T07:31:11.822 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.823 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:swN161nWstZjAUfnxygaxY+bV6g= 2022-09-03T07:31:11.825 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:swN161nWstZjAUfnxygaxY+bV6g=', 'Content-Length': '0'} 2022-09-03T07:31:11.828 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b416481df55af895-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.830 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:11.831 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:11.833 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.834 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.834 DEBUG:boto:Method: GET 2022-09-03T07:31:11.836 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.837 DEBUG:boto:Data: 2022-09-03T07:31:11.839 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.840 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.841 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.842 DEBUG:boto:Params: {} 2022-09-03T07:31:11.844 DEBUG:boto:Token: None 2022-09-03T07:31:11.845 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.847 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:swN161nWstZjAUfnxygaxY+bV6g= 2022-09-03T07:31:11.848 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:swN161nWstZjAUfnxygaxY+bV6g=', 'Content-Length': '0'} 2022-09-03T07:31:11.854 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c099fa01f7745efa-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.856 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:11.857 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:11.858 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.859 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.860 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.861 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.862 DEBUG:boto:Data: 2022-09-03T07:31:11.863 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.864 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.865 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.866 DEBUG:boto:Params: {} 2022-09-03T07:31:11.867 DEBUG:boto:Token: None 2022-09-03T07:31:11.868 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.869 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HbnyVYK0w7E5gPFiw02qwbqTCc8= 2022-09-03T07:31:11.870 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HbnyVYK0w7E5gPFiw02qwbqTCc8=', 'Content-Length': '0'} 2022-09-03T07:31:11.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f612c00d39b63af2-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.874 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:11.874 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:11.875 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.875 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.876 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.876 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.877 DEBUG:boto:Data: 2022-09-03T07:31:11.877 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.878 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.878 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.879 DEBUG:boto:Params: {} 2022-09-03T07:31:11.879 DEBUG:boto:Token: None 2022-09-03T07:31:11.880 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HbnyVYK0w7E5gPFiw02qwbqTCc8= 2022-09-03T07:31:11.881 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HbnyVYK0w7E5gPFiw02qwbqTCc8=', 'Content-Length': '0'} 2022-09-03T07:31:11.883 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c2a3c3d8109a349-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.885 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:31:11.885 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:11.886 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:11.886 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.887 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.887 DEBUG:boto:Method: GET 2022-09-03T07:31:11.888 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.889 DEBUG:boto:Data: 2022-09-03T07:31:11.889 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.890 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.892 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.893 DEBUG:boto:Params: {} 2022-09-03T07:31:11.894 DEBUG:boto:Token: None 2022-09-03T07:31:11.895 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.896 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:swN161nWstZjAUfnxygaxY+bV6g= 2022-09-03T07:31:11.897 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:swN161nWstZjAUfnxygaxY+bV6g=', 'Content-Length': '0'} 2022-09-03T07:31:11.900 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000702e465f1b64afc5-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.901 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:11.902 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:11.902 DEBUG:boto:path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.903 DEBUG:boto:auth_path=/avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.903 DEBUG:boto:Method: GET 2022-09-03T07:31:11.904 DEBUG:boto:Path: /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.904 DEBUG:boto:Data: 2022-09-03T07:31:11.905 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.905 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.906 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.906 DEBUG:boto:Params: {} 2022-09-03T07:31:11.907 DEBUG:boto:Token: None 2022-09-03T07:31:11.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj76?versionId=null 2022-09-03T07:31:11.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:swN161nWstZjAUfnxygaxY+bV6g= 2022-09-03T07:31:11.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:swN161nWstZjAUfnxygaxY+bV6g=', 'Content-Length': '0'} 2022-09-03T07:31:11.913 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000098e31a4ce81d699f-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.914 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.915 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.915 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:11.916 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:11.917 DEBUG:boto:Method: GET 2022-09-03T07:31:11.917 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:31:11.918 DEBUG:boto:Data: 2022-09-03T07:31:11.919 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.920 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.921 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.921 DEBUG:boto:Params: {} 2022-09-03T07:31:11.922 DEBUG:boto:Token: None 2022-09-03T07:31:11.923 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.924 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.924 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.927 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a6ea0fee85d25ed7-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.928 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.930 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:11.931 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:11.931 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:11.932 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:11.932 DEBUG:boto:Method: GET 2022-09-03T07:31:11.933 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:31:11.934 DEBUG:boto:Data: 2022-09-03T07:31:11.934 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.935 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.935 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.936 DEBUG:boto:Params: {} 2022-09-03T07:31:11.936 DEBUG:boto:Token: None 2022-09-03T07:31:11.937 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/ 2022-09-03T07:31:11.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY= 2022-09-03T07:31:11.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3YEMQyt8qqXuJOC/RfsQD4C0xmY=', 'Content-Length': '0'} 2022-09-03T07:31:11.940 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e0379ea34644fc3d-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.941 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:11.942 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:11.943 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:11.944 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.944 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:31:11.945 DEBUG:boto:Data: 2022-09-03T07:31:11.945 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.946 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.947 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.947 DEBUG:boto:Params: {} 2022-09-03T07:31:11.948 DEBUG:boto:Token: None 2022-09-03T07:31:11.949 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj76 2022-09-03T07:31:11.950 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t9Isnr1PBXGX5Rt578Ba7viYR8g= 2022-09-03T07:31:11.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t9Isnr1PBXGX5Rt578Ba7viYR8g=', 'Content-Length': '0'} 2022-09-03T07:31:11.956 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f2b291f1887706a-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.958 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:11.959 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:11.961 DEBUG:boto:Method: HEAD 2022-09-03T07:31:11.962 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:31:11.963 DEBUG:boto:Data: 2022-09-03T07:31:11.964 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.965 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:11.966 DEBUG:boto:Port: 8000 2022-09-03T07:31:11.967 DEBUG:boto:Params: {} 2022-09-03T07:31:11.968 DEBUG:boto:Token: None 2022-09-03T07:31:11.969 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj76 2022-09-03T07:31:11.970 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:t9Isnr1PBXGX5Rt578Ba7viYR8g= 2022-09-03T07:31:11.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:t9Isnr1PBXGX5Rt578Ba7viYR8g=', 'Content-Length': '0'} 2022-09-03T07:31:11.975 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b9fcbb5b96f0c6e9-00631302bf-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:11.976 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj76 2022-09-03T07:31:11.977 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:11.978 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:11.979 DEBUG:boto:Method: GET 2022-09-03T07:31:11.980 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:31:11.980 DEBUG:boto:Data: 2022-09-03T07:31:11.981 DEBUG:boto:Headers: {} 2022-09-03T07:31:11.981 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:11.982 DEBUG:boto:Port: 8001 2022-09-03T07:31:11.983 DEBUG:boto:Params: {} 2022-09-03T07:31:11.985 DEBUG:boto:Token: None 2022-09-03T07:31:11.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:11 GMT /avysmw-75/obj76 2022-09-03T07:31:11.989 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iBhXMZDlml4qRwoQ1vmhvm9EYnM= 2022-09-03T07:31:11.990 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:11 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iBhXMZDlml4qRwoQ1vmhvm9EYnM=', 'Content-Length': '0'} 2022-09-03T07:31:11.992 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000075ed35c279175cd-00631302bf-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:11 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.049 DEBUG:boto:path=/avysmw-75/obj76 2022-09-03T07:31:12.051 DEBUG:boto:auth_path=/avysmw-75/obj76 2022-09-03T07:31:12.052 DEBUG:boto:Method: GET 2022-09-03T07:31:12.052 DEBUG:boto:Path: /avysmw-75/obj76 2022-09-03T07:31:12.053 DEBUG:boto:Data: 2022-09-03T07:31:12.054 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.054 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.055 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.055 DEBUG:boto:Params: {} 2022-09-03T07:31:12.057 DEBUG:boto:Token: None 2022-09-03T07:31:12.057 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj76 2022-09-03T07:31:12.058 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GD0C753nHwjBGMN00cTNrRyB/ok= 2022-09-03T07:31:12.059 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GD0C753nHwjBGMN00cTNrRyB/ok=', 'Content-Length': '0'} 2022-09-03T07:31:12.062 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002d9073b496f781b5-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.063 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.064 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.064 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:12.065 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:12.066 DEBUG:boto:Method: GET 2022-09-03T07:31:12.066 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:31:12.067 DEBUG:boto:Data: 2022-09-03T07:31:12.068 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.068 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.069 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.069 DEBUG:boto:Params: {} 2022-09-03T07:31:12.070 DEBUG:boto:Token: None 2022-09-03T07:31:12.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.071 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.075 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ba9e78bf899bb277-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.075 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.076 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.077 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.078 DEBUG:boto:path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:12.079 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj76 2022-09-03T07:31:12.080 DEBUG:boto:Method: GET 2022-09-03T07:31:12.081 DEBUG:boto:Path: /avysmw-75/?prefix=obj76 2022-09-03T07:31:12.082 DEBUG:boto:Data: 2022-09-03T07:31:12.082 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.083 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.083 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.084 DEBUG:boto:Params: {} 2022-09-03T07:31:12.085 DEBUG:boto:Token: None 2022-09-03T07:31:12.085 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.086 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.087 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.090 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000060725df3e7246e0e-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.141 DEBUG:boto:b'avysmw-75obj761000falseobj762022-09-03T07:15:42.592Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.143 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:31:12.144 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.145 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.146 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.147 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.148 DEBUG:boto:Method: GET 2022-09-03T07:31:12.149 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.150 DEBUG:boto:Data: 2022-09-03T07:31:12.151 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.152 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.153 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.154 DEBUG:boto:Params: {} 2022-09-03T07:31:12.156 DEBUG:boto:Token: None 2022-09-03T07:31:12.157 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.158 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sFhk4r7nR6qOUNIdlS9W93emmE4= 2022-09-03T07:31:12.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sFhk4r7nR6qOUNIdlS9W93emmE4=', 'Content-Length': '0'} 2022-09-03T07:31:12.163 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fc111f5fa0050671-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.164 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.165 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.166 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.167 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.168 DEBUG:boto:Method: GET 2022-09-03T07:31:12.169 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.170 DEBUG:boto:Data: 2022-09-03T07:31:12.171 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.173 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.174 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.175 DEBUG:boto:Params: {} 2022-09-03T07:31:12.176 DEBUG:boto:Token: None 2022-09-03T07:31:12.177 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sFhk4r7nR6qOUNIdlS9W93emmE4= 2022-09-03T07:31:12.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sFhk4r7nR6qOUNIdlS9W93emmE4=', 'Content-Length': '0'} 2022-09-03T07:31:12.182 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ded4f87d64d5eb9-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.184 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.185 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.186 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.187 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.188 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.189 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.190 DEBUG:boto:Data: 2022-09-03T07:31:12.191 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.192 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.193 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.194 DEBUG:boto:Params: {} 2022-09-03T07:31:12.195 DEBUG:boto:Token: None 2022-09-03T07:31:12.196 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jf4k5/DGoGPJsPxH0XTnZgr2WhU= 2022-09-03T07:31:12.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jf4k5/DGoGPJsPxH0XTnZgr2WhU=', 'Content-Length': '0'} 2022-09-03T07:31:12.206 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004428c7e9121750dc-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.207 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.207 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.208 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.208 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.209 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.209 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.210 DEBUG:boto:Data: 2022-09-03T07:31:12.210 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.211 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.211 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.212 DEBUG:boto:Params: {} 2022-09-03T07:31:12.212 DEBUG:boto:Token: None 2022-09-03T07:31:12.213 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.214 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jf4k5/DGoGPJsPxH0XTnZgr2WhU= 2022-09-03T07:31:12.214 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jf4k5/DGoGPJsPxH0XTnZgr2WhU=', 'Content-Length': '0'} 2022-09-03T07:31:12.216 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003855736f02c5e82a-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.217 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:31:12.218 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.218 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.219 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.219 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.220 DEBUG:boto:Method: GET 2022-09-03T07:31:12.220 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.221 DEBUG:boto:Data: 2022-09-03T07:31:12.221 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.222 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.222 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.223 DEBUG:boto:Params: {} 2022-09-03T07:31:12.224 DEBUG:boto:Token: None 2022-09-03T07:31:12.225 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.225 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sFhk4r7nR6qOUNIdlS9W93emmE4= 2022-09-03T07:31:12.227 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sFhk4r7nR6qOUNIdlS9W93emmE4=', 'Content-Length': '0'} 2022-09-03T07:31:12.229 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d5ef51d14e0f46fd-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.271 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.273 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.273 DEBUG:boto:path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.275 DEBUG:boto:auth_path=/avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.275 DEBUG:boto:Method: GET 2022-09-03T07:31:12.276 DEBUG:boto:Path: /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.279 DEBUG:boto:Data: 2022-09-03T07:31:12.280 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.281 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.282 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.283 DEBUG:boto:Params: {} 2022-09-03T07:31:12.285 DEBUG:boto:Token: None 2022-09-03T07:31:12.286 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85?versionId=null 2022-09-03T07:31:12.287 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sFhk4r7nR6qOUNIdlS9W93emmE4= 2022-09-03T07:31:12.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sFhk4r7nR6qOUNIdlS9W93emmE4=', 'Content-Length': '0'} 2022-09-03T07:31:12.294 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009ccd8e5b034623af-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.296 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.296 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.297 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:12.297 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:12.298 DEBUG:boto:Method: GET 2022-09-03T07:31:12.299 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:31:12.299 DEBUG:boto:Data: 2022-09-03T07:31:12.300 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.301 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.301 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.302 DEBUG:boto:Params: {} 2022-09-03T07:31:12.303 DEBUG:boto:Token: None 2022-09-03T07:31:12.304 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.309 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a034a3f0bf1e2689-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.310 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.312 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.312 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.313 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:12.313 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:12.314 DEBUG:boto:Method: GET 2022-09-03T07:31:12.315 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:31:12.315 DEBUG:boto:Data: 2022-09-03T07:31:12.316 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.316 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.317 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.317 DEBUG:boto:Params: {} 2022-09-03T07:31:12.318 DEBUG:boto:Token: None 2022-09-03T07:31:12.319 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.319 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.322 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a47319e26f8e055-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.323 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.324 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.325 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.325 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.326 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:31:12.326 DEBUG:boto:Data: 2022-09-03T07:31:12.327 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.327 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.328 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.328 DEBUG:boto:Params: {} 2022-09-03T07:31:12.329 DEBUG:boto:Token: None 2022-09-03T07:31:12.330 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85 2022-09-03T07:31:12.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yBgG+fSvqo4B796k7CX3/7MbtX0= 2022-09-03T07:31:12.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yBgG+fSvqo4B796k7CX3/7MbtX0=', 'Content-Length': '0'} 2022-09-03T07:31:12.333 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009d813a5d283b24a1-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.334 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.334 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.335 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.335 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:31:12.336 DEBUG:boto:Data: 2022-09-03T07:31:12.336 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.337 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.337 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.338 DEBUG:boto:Params: {} 2022-09-03T07:31:12.338 DEBUG:boto:Token: None 2022-09-03T07:31:12.339 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85 2022-09-03T07:31:12.340 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:yBgG+fSvqo4B796k7CX3/7MbtX0= 2022-09-03T07:31:12.340 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:yBgG+fSvqo4B796k7CX3/7MbtX0=', 'Content-Length': '0'} 2022-09-03T07:31:12.343 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d04c3243b3c27828-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.344 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj85 2022-09-03T07:31:12.344 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.345 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.346 DEBUG:boto:Method: GET 2022-09-03T07:31:12.346 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:31:12.347 DEBUG:boto:Data: 2022-09-03T07:31:12.347 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.348 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.349 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.349 DEBUG:boto:Params: {} 2022-09-03T07:31:12.350 DEBUG:boto:Token: None 2022-09-03T07:31:12.350 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85 2022-09-03T07:31:12.351 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yn2Bgbulug/hYVMTCOv6l4DK+E4= 2022-09-03T07:31:12.352 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yn2Bgbulug/hYVMTCOv6l4DK+E4=', 'Content-Length': '0'} 2022-09-03T07:31:12.355 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003684b0a1a8beb00c-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.357 DEBUG:boto:path=/avysmw-75/obj85 2022-09-03T07:31:12.358 DEBUG:boto:auth_path=/avysmw-75/obj85 2022-09-03T07:31:12.359 DEBUG:boto:Method: GET 2022-09-03T07:31:12.360 DEBUG:boto:Path: /avysmw-75/obj85 2022-09-03T07:31:12.363 DEBUG:boto:Data: 2022-09-03T07:31:12.364 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.365 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.366 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.367 DEBUG:boto:Params: {} 2022-09-03T07:31:12.368 DEBUG:boto:Token: None 2022-09-03T07:31:12.369 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj85 2022-09-03T07:31:12.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Yn2Bgbulug/hYVMTCOv6l4DK+E4= 2022-09-03T07:31:12.371 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Yn2Bgbulug/hYVMTCOv6l4DK+E4=', 'Content-Length': '0'} 2022-09-03T07:31:12.373 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:12:49 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004680ee1edb1fbf3a-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.418 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.419 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.420 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:12.420 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:12.421 DEBUG:boto:Method: GET 2022-09-03T07:31:12.421 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:31:12.422 DEBUG:boto:Data: 2022-09-03T07:31:12.422 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.423 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.423 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.424 DEBUG:boto:Params: {} 2022-09-03T07:31:12.425 DEBUG:boto:Token: None 2022-09-03T07:31:12.426 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.426 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.428 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.431 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bb122bc35fa87e16-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.431 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.432 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.433 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.433 DEBUG:boto:path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:12.434 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj85 2022-09-03T07:31:12.435 DEBUG:boto:Method: GET 2022-09-03T07:31:12.435 DEBUG:boto:Path: /avysmw-75/?prefix=obj85 2022-09-03T07:31:12.436 DEBUG:boto:Data: 2022-09-03T07:31:12.436 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.437 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.437 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.438 DEBUG:boto:Params: {} 2022-09-03T07:31:12.439 DEBUG:boto:Token: None 2022-09-03T07:31:12.439 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.440 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.441 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.444 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ebacf965a3a2cbb7-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.445 DEBUG:boto:b'avysmw-75obj851000falseobj852022-09-03T07:12:49.663Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.445 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:31:12.446 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.447 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.447 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.448 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.448 DEBUG:boto:Method: GET 2022-09-03T07:31:12.449 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.449 DEBUG:boto:Data: 2022-09-03T07:31:12.450 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.451 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.451 DEBUG:boto:Params: {} 2022-09-03T07:31:12.452 DEBUG:boto:Token: None 2022-09-03T07:31:12.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S3Th6wa9trTvvvOlnnQVgcEdyak= 2022-09-03T07:31:12.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S3Th6wa9trTvvvOlnnQVgcEdyak=', 'Content-Length': '0'} 2022-09-03T07:31:12.456 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f18edd29ee8ce13-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.457 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.458 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.459 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.459 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.460 DEBUG:boto:Method: GET 2022-09-03T07:31:12.460 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.461 DEBUG:boto:Data: 2022-09-03T07:31:12.462 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.462 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.463 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.463 DEBUG:boto:Params: {} 2022-09-03T07:31:12.464 DEBUG:boto:Token: None 2022-09-03T07:31:12.464 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.465 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S3Th6wa9trTvvvOlnnQVgcEdyak= 2022-09-03T07:31:12.466 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S3Th6wa9trTvvvOlnnQVgcEdyak=', 'Content-Length': '0'} 2022-09-03T07:31:12.468 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fe165ea3e4e91e2b-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.469 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.469 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.470 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.470 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.471 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.471 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.472 DEBUG:boto:Data: 2022-09-03T07:31:12.473 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.473 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.474 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.474 DEBUG:boto:Params: {} 2022-09-03T07:31:12.475 DEBUG:boto:Token: None 2022-09-03T07:31:12.475 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.476 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fjfTLtsmsMaMYMaZCCdE4TtUs78= 2022-09-03T07:31:12.477 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fjfTLtsmsMaMYMaZCCdE4TtUs78=', 'Content-Length': '0'} 2022-09-03T07:31:12.481 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000db1f86d77920b20b-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.482 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.482 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.483 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.483 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.484 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.484 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.485 DEBUG:boto:Data: 2022-09-03T07:31:12.486 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.486 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.487 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.487 DEBUG:boto:Params: {} 2022-09-03T07:31:12.488 DEBUG:boto:Token: None 2022-09-03T07:31:12.488 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.489 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fjfTLtsmsMaMYMaZCCdE4TtUs78= 2022-09-03T07:31:12.490 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fjfTLtsmsMaMYMaZCCdE4TtUs78=', 'Content-Length': '0'} 2022-09-03T07:31:12.492 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fdb5bd3bd6623db8-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.493 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:31:12.494 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.495 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.496 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.497 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.498 DEBUG:boto:Method: GET 2022-09-03T07:31:12.499 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.500 DEBUG:boto:Data: 2022-09-03T07:31:12.501 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.503 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.504 DEBUG:boto:Params: {} 2022-09-03T07:31:12.505 DEBUG:boto:Token: None 2022-09-03T07:31:12.507 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.508 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S3Th6wa9trTvvvOlnnQVgcEdyak= 2022-09-03T07:31:12.509 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S3Th6wa9trTvvvOlnnQVgcEdyak=', 'Content-Length': '0'} 2022-09-03T07:31:12.512 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000424fac8af0540a25-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.513 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.514 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.514 DEBUG:boto:path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.515 DEBUG:boto:auth_path=/avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.516 DEBUG:boto:Method: GET 2022-09-03T07:31:12.516 DEBUG:boto:Path: /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.517 DEBUG:boto:Data: 2022-09-03T07:31:12.517 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.518 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.519 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.519 DEBUG:boto:Params: {} 2022-09-03T07:31:12.520 DEBUG:boto:Token: None 2022-09-03T07:31:12.520 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87?versionId=null 2022-09-03T07:31:12.521 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:S3Th6wa9trTvvvOlnnQVgcEdyak= 2022-09-03T07:31:12.522 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:S3Th6wa9trTvvvOlnnQVgcEdyak=', 'Content-Length': '0'} 2022-09-03T07:31:12.524 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c6e3208837505bce-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.525 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.526 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.527 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:12.528 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:12.529 DEBUG:boto:Method: GET 2022-09-03T07:31:12.529 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:31:12.530 DEBUG:boto:Data: 2022-09-03T07:31:12.531 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.532 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.532 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.533 DEBUG:boto:Params: {} 2022-09-03T07:31:12.535 DEBUG:boto:Token: None 2022-09-03T07:31:12.536 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.542 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002e8f815a64ad66b7-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.544 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.545 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.546 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.547 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:12.549 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:12.550 DEBUG:boto:Method: GET 2022-09-03T07:31:12.551 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:31:12.552 DEBUG:boto:Data: 2022-09-03T07:31:12.553 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.553 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.554 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.555 DEBUG:boto:Params: {} 2022-09-03T07:31:12.556 DEBUG:boto:Token: None 2022-09-03T07:31:12.557 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.558 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.559 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.562 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000aad092f89b8261bb-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.563 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.564 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.564 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.565 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.565 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:31:12.566 DEBUG:boto:Data: 2022-09-03T07:31:12.566 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.567 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.568 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.568 DEBUG:boto:Params: {} 2022-09-03T07:31:12.569 DEBUG:boto:Token: None 2022-09-03T07:31:12.570 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87 2022-09-03T07:31:12.570 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:18L6w3Ceiz+fonLa69viZ+ptluk= 2022-09-03T07:31:12.571 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:18L6w3Ceiz+fonLa69viZ+ptluk=', 'Content-Length': '0'} 2022-09-03T07:31:12.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000024402b6750c2e94-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.579 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.580 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.580 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.581 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:31:12.581 DEBUG:boto:Data: 2022-09-03T07:31:12.586 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.587 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.588 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.589 DEBUG:boto:Params: {} 2022-09-03T07:31:12.590 DEBUG:boto:Token: None 2022-09-03T07:31:12.590 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87 2022-09-03T07:31:12.591 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:18L6w3Ceiz+fonLa69viZ+ptluk= 2022-09-03T07:31:12.592 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:18L6w3Ceiz+fonLa69viZ+ptluk=', 'Content-Length': '0'} 2022-09-03T07:31:12.594 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eb79c2ef5ff6ab67-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.595 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj87 2022-09-03T07:31:12.596 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.596 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.597 DEBUG:boto:Method: GET 2022-09-03T07:31:12.597 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:31:12.598 DEBUG:boto:Data: 2022-09-03T07:31:12.598 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.599 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.600 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.600 DEBUG:boto:Params: {} 2022-09-03T07:31:12.601 DEBUG:boto:Token: None 2022-09-03T07:31:12.601 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87 2022-09-03T07:31:12.602 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rjq2lDRQfaWk6IRG9MLVITtHkGc= 2022-09-03T07:31:12.603 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rjq2lDRQfaWk6IRG9MLVITtHkGc=', 'Content-Length': '0'} 2022-09-03T07:31:12.605 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000a20fb684e2f3019-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.646 DEBUG:boto:path=/avysmw-75/obj87 2022-09-03T07:31:12.648 DEBUG:boto:auth_path=/avysmw-75/obj87 2022-09-03T07:31:12.649 DEBUG:boto:Method: GET 2022-09-03T07:31:12.649 DEBUG:boto:Path: /avysmw-75/obj87 2022-09-03T07:31:12.650 DEBUG:boto:Data: 2022-09-03T07:31:12.651 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.652 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.653 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.654 DEBUG:boto:Params: {} 2022-09-03T07:31:12.655 DEBUG:boto:Token: None 2022-09-03T07:31:12.656 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj87 2022-09-03T07:31:12.657 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:rjq2lDRQfaWk6IRG9MLVITtHkGc= 2022-09-03T07:31:12.658 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:rjq2lDRQfaWk6IRG9MLVITtHkGc=', 'Content-Length': '0'} 2022-09-03T07:31:12.661 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:24:23 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a99ea964388dde14-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.662 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.663 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.665 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:12.666 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:12.667 DEBUG:boto:Method: GET 2022-09-03T07:31:12.668 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:31:12.669 DEBUG:boto:Data: 2022-09-03T07:31:12.670 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.671 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.672 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.673 DEBUG:boto:Params: {} 2022-09-03T07:31:12.674 DEBUG:boto:Token: None 2022-09-03T07:31:12.675 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.676 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.677 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.691 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000011facfec8ae673f4-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.692 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.694 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.695 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.695 DEBUG:boto:path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:12.696 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj87 2022-09-03T07:31:12.697 DEBUG:boto:Method: GET 2022-09-03T07:31:12.697 DEBUG:boto:Path: /avysmw-75/?prefix=obj87 2022-09-03T07:31:12.698 DEBUG:boto:Data: 2022-09-03T07:31:12.698 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.699 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.699 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.700 DEBUG:boto:Params: {} 2022-09-03T07:31:12.701 DEBUG:boto:Token: None 2022-09-03T07:31:12.701 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.702 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.703 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.705 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008003cd04f304dfa6-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.706 DEBUG:boto:b'avysmw-75obj871000falseobj872022-09-03T07:24:23.692Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.707 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:31:12.708 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.709 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.710 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.711 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.715 DEBUG:boto:Method: GET 2022-09-03T07:31:12.716 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.717 DEBUG:boto:Data: 2022-09-03T07:31:12.719 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.719 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.720 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.721 DEBUG:boto:Params: {} 2022-09-03T07:31:12.723 DEBUG:boto:Token: None 2022-09-03T07:31:12.724 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.725 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9pib992TRBpIoyeLZfYR9ZBuLy4= 2022-09-03T07:31:12.726 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9pib992TRBpIoyeLZfYR9ZBuLy4=', 'Content-Length': '0'} 2022-09-03T07:31:12.728 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ca2d029b7961993-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.730 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.730 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.731 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.732 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.733 DEBUG:boto:Method: GET 2022-09-03T07:31:12.734 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.735 DEBUG:boto:Data: 2022-09-03T07:31:12.736 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.737 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.738 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.739 DEBUG:boto:Params: {} 2022-09-03T07:31:12.740 DEBUG:boto:Token: None 2022-09-03T07:31:12.741 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9pib992TRBpIoyeLZfYR9ZBuLy4= 2022-09-03T07:31:12.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9pib992TRBpIoyeLZfYR9ZBuLy4=', 'Content-Length': '0'} 2022-09-03T07:31:12.749 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c009b11d78725ed9-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.751 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.751 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.752 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.752 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.753 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.753 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.754 DEBUG:boto:Data: 2022-09-03T07:31:12.755 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.759 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.759 DEBUG:boto:Params: {} 2022-09-03T07:31:12.760 DEBUG:boto:Token: None 2022-09-03T07:31:12.760 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.761 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0QzE+DWw8nPlYtqZaA/dPCd23L0= 2022-09-03T07:31:12.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0QzE+DWw8nPlYtqZaA/dPCd23L0=', 'Content-Length': '0'} 2022-09-03T07:31:12.764 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000660c259436bf49b2-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.765 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.766 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.766 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.767 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.768 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.769 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.769 DEBUG:boto:Data: 2022-09-03T07:31:12.770 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.770 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.771 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.771 DEBUG:boto:Params: {} 2022-09-03T07:31:12.772 DEBUG:boto:Token: None 2022-09-03T07:31:12.773 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.773 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0QzE+DWw8nPlYtqZaA/dPCd23L0= 2022-09-03T07:31:12.774 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0QzE+DWw8nPlYtqZaA/dPCd23L0=', 'Content-Length': '0'} 2022-09-03T07:31:12.776 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d4b2ff5214682e5d-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.777 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:31:12.779 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.780 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.781 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.782 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.783 DEBUG:boto:Method: GET 2022-09-03T07:31:12.783 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.784 DEBUG:boto:Data: 2022-09-03T07:31:12.784 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.785 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.786 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.786 DEBUG:boto:Params: {} 2022-09-03T07:31:12.787 DEBUG:boto:Token: None 2022-09-03T07:31:12.788 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.789 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9pib992TRBpIoyeLZfYR9ZBuLy4= 2022-09-03T07:31:12.789 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9pib992TRBpIoyeLZfYR9ZBuLy4=', 'Content-Length': '0'} 2022-09-03T07:31:12.791 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000004b3a98fcbb3b873-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.792 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.793 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.794 DEBUG:boto:path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.795 DEBUG:boto:auth_path=/avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.795 DEBUG:boto:Method: GET 2022-09-03T07:31:12.796 DEBUG:boto:Path: /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.797 DEBUG:boto:Data: 2022-09-03T07:31:12.797 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.798 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.798 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.799 DEBUG:boto:Params: {} 2022-09-03T07:31:12.799 DEBUG:boto:Token: None 2022-09-03T07:31:12.800 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92?versionId=null 2022-09-03T07:31:12.801 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9pib992TRBpIoyeLZfYR9ZBuLy4= 2022-09-03T07:31:12.801 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9pib992TRBpIoyeLZfYR9ZBuLy4=', 'Content-Length': '0'} 2022-09-03T07:31:12.803 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f7d35085454d676e-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.804 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.805 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.805 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:12.806 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:12.807 DEBUG:boto:Method: GET 2022-09-03T07:31:12.807 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:31:12.808 DEBUG:boto:Data: 2022-09-03T07:31:12.808 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.809 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.810 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.810 DEBUG:boto:Params: {} 2022-09-03T07:31:12.811 DEBUG:boto:Token: None 2022-09-03T07:31:12.811 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.812 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.813 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.816 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007c6fbbdb256f67d5-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.821 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.823 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.824 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.825 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:12.826 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:12.827 DEBUG:boto:Method: GET 2022-09-03T07:31:12.828 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:31:12.829 DEBUG:boto:Data: 2022-09-03T07:31:12.830 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.831 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.831 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.832 DEBUG:boto:Params: {} 2022-09-03T07:31:12.833 DEBUG:boto:Token: None 2022-09-03T07:31:12.833 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.834 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.835 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.838 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d7e7f9711db65b9d-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.841 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.843 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.844 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.844 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.845 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:31:12.845 DEBUG:boto:Data: 2022-09-03T07:31:12.846 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.848 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.849 DEBUG:boto:Params: {} 2022-09-03T07:31:12.850 DEBUG:boto:Token: None 2022-09-03T07:31:12.852 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92 2022-09-03T07:31:12.854 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ChQu0dPV4BNQozzDrx3QDPlLb4= 2022-09-03T07:31:12.855 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ChQu0dPV4BNQozzDrx3QDPlLb4=', 'Content-Length': '0'} 2022-09-03T07:31:12.857 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005678fb29e780776f-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.858 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.859 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.859 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.860 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:31:12.860 DEBUG:boto:Data: 2022-09-03T07:31:12.861 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.861 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.862 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.863 DEBUG:boto:Params: {} 2022-09-03T07:31:12.864 DEBUG:boto:Token: None 2022-09-03T07:31:12.865 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92 2022-09-03T07:31:12.865 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+ChQu0dPV4BNQozzDrx3QDPlLb4= 2022-09-03T07:31:12.867 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+ChQu0dPV4BNQozzDrx3QDPlLb4=', 'Content-Length': '0'} 2022-09-03T07:31:12.870 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000750e228e11c6675b-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.871 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj92 2022-09-03T07:31:12.871 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.872 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.873 DEBUG:boto:Method: GET 2022-09-03T07:31:12.873 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:31:12.874 DEBUG:boto:Data: 2022-09-03T07:31:12.874 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.875 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.875 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.876 DEBUG:boto:Params: {} 2022-09-03T07:31:12.877 DEBUG:boto:Token: None 2022-09-03T07:31:12.877 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92 2022-09-03T07:31:12.879 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vlPoDl0hOUoTl/+7ceBVwbnoPgo= 2022-09-03T07:31:12.880 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vlPoDl0hOUoTl/+7ceBVwbnoPgo=', 'Content-Length': '0'} 2022-09-03T07:31:12.882 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d4888a074e3fc5b-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.926 DEBUG:boto:path=/avysmw-75/obj92 2022-09-03T07:31:12.927 DEBUG:boto:auth_path=/avysmw-75/obj92 2022-09-03T07:31:12.928 DEBUG:boto:Method: GET 2022-09-03T07:31:12.928 DEBUG:boto:Path: /avysmw-75/obj92 2022-09-03T07:31:12.929 DEBUG:boto:Data: 2022-09-03T07:31:12.929 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.930 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.930 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.931 DEBUG:boto:Params: {} 2022-09-03T07:31:12.932 DEBUG:boto:Token: None 2022-09-03T07:31:12.932 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj92 2022-09-03T07:31:12.933 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vlPoDl0hOUoTl/+7ceBVwbnoPgo= 2022-09-03T07:31:12.934 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vlPoDl0hOUoTl/+7ceBVwbnoPgo=', 'Content-Length': '0'} 2022-09-03T07:31:12.937 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:21:21 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034bceb186aea37f9-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.938 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.938 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.939 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:12.939 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:12.940 DEBUG:boto:Method: GET 2022-09-03T07:31:12.940 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:31:12.941 DEBUG:boto:Data: 2022-09-03T07:31:12.941 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.942 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.943 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.943 DEBUG:boto:Params: {} 2022-09-03T07:31:12.944 DEBUG:boto:Token: None 2022-09-03T07:31:12.945 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.945 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.949 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ea5e5819065e04c3-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.950 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.951 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:12.952 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:12.952 DEBUG:boto:path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:12.953 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj92 2022-09-03T07:31:12.954 DEBUG:boto:Method: GET 2022-09-03T07:31:12.954 DEBUG:boto:Path: /avysmw-75/?prefix=obj92 2022-09-03T07:31:12.955 DEBUG:boto:Data: 2022-09-03T07:31:12.956 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.956 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.957 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.958 DEBUG:boto:Params: {} 2022-09-03T07:31:12.959 DEBUG:boto:Token: None 2022-09-03T07:31:12.960 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/ 2022-09-03T07:31:12.961 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus= 2022-09-03T07:31:12.962 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:aNMdIVYirYmvI6K9o29YFJuuXus=', 'Content-Length': '0'} 2022-09-03T07:31:12.966 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e061d51aec57e173-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.967 DEBUG:boto:b'avysmw-75obj921000falseobj922022-09-03T07:21:21.666Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:12.968 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:31:12.969 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:12.970 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:12.971 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.971 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.972 DEBUG:boto:Method: GET 2022-09-03T07:31:12.972 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.973 DEBUG:boto:Data: 2022-09-03T07:31:12.973 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.974 DEBUG:boto:Port: 8001 2022-09-03T07:31:12.975 DEBUG:boto:Params: {} 2022-09-03T07:31:12.976 DEBUG:boto:Token: None 2022-09-03T07:31:12.976 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IIooSLmRWrKVHq5xiIDawg0VMEo= 2022-09-03T07:31:12.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IIooSLmRWrKVHq5xiIDawg0VMEo=', 'Content-Length': '0'} 2022-09-03T07:31:12.981 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f9fa2341425c87fb-00631302c0-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.982 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:12.982 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:12.983 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.983 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.984 DEBUG:boto:Method: GET 2022-09-03T07:31:12.984 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.985 DEBUG:boto:Data: 2022-09-03T07:31:12.986 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.987 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:12.987 DEBUG:boto:Port: 8000 2022-09-03T07:31:12.988 DEBUG:boto:Params: {} 2022-09-03T07:31:12.989 DEBUG:boto:Token: None 2022-09-03T07:31:12.990 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:12 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IIooSLmRWrKVHq5xiIDawg0VMEo= 2022-09-03T07:31:12.991 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IIooSLmRWrKVHq5xiIDawg0VMEo=', 'Content-Length': '0'} 2022-09-03T07:31:12.993 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d783ba44b468947f-00631302c0-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:12.994 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:12.995 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:12.996 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.996 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.997 DEBUG:boto:Method: HEAD 2022-09-03T07:31:12.997 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:12.998 DEBUG:boto:Data: 2022-09-03T07:31:12.998 DEBUG:boto:Headers: {} 2022-09-03T07:31:12.999 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:12.999 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.000 DEBUG:boto:Params: {} 2022-09-03T07:31:13.001 DEBUG:boto:Token: None 2022-09-03T07:31:13.002 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.002 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vIJLPtFcZhdtyRSqNgHk/L2X3d4= 2022-09-03T07:31:13.003 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vIJLPtFcZhdtyRSqNgHk/L2X3d4=', 'Content-Length': '0'} 2022-09-03T07:31:13.005 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009ab625a1edeb3110-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.006 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:13.007 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:13.007 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.008 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.008 DEBUG:boto:Method: HEAD 2022-09-03T07:31:13.009 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.010 DEBUG:boto:Data: 2022-09-03T07:31:13.010 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.011 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.012 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.012 DEBUG:boto:Params: {} 2022-09-03T07:31:13.013 DEBUG:boto:Token: None 2022-09-03T07:31:13.014 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vIJLPtFcZhdtyRSqNgHk/L2X3d4= 2022-09-03T07:31:13.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vIJLPtFcZhdtyRSqNgHk/L2X3d4=', 'Content-Length': '0'} 2022-09-03T07:31:13.019 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000485b79c5b9c29467-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.020 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:31:13.021 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:13.022 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:13.023 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.024 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.025 DEBUG:boto:Method: GET 2022-09-03T07:31:13.026 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.027 DEBUG:boto:Data: 2022-09-03T07:31:13.028 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.028 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.029 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.029 DEBUG:boto:Params: {} 2022-09-03T07:31:13.030 DEBUG:boto:Token: None 2022-09-03T07:31:13.031 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.031 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nKHseQZNgnksdl4HALZdzotxmIA= 2022-09-03T07:31:13.032 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nKHseQZNgnksdl4HALZdzotxmIA=', 'Content-Length': '0'} 2022-09-03T07:31:13.035 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003a7c62370e37482f-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.036 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:13.037 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:13.037 DEBUG:boto:path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.038 DEBUG:boto:auth_path=/avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.038 DEBUG:boto:Method: GET 2022-09-03T07:31:13.039 DEBUG:boto:Path: /avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.040 DEBUG:boto:Data: 2022-09-03T07:31:13.040 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.041 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.041 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.042 DEBUG:boto:Params: {} 2022-09-03T07:31:13.043 DEBUG:boto:Token: None 2022-09-03T07:31:13.044 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj93?versionId=null 2022-09-03T07:31:13.045 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:nKHseQZNgnksdl4HALZdzotxmIA= 2022-09-03T07:31:13.045 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:nKHseQZNgnksdl4HALZdzotxmIA=', 'Content-Length': '0'} 2022-09-03T07:31:13.047 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000074920bdb6e4a7fc3-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.048 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:13.049 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:13.049 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:13.050 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:13.051 DEBUG:boto:Method: GET 2022-09-03T07:31:13.051 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:31:13.052 DEBUG:boto:Data: 2022-09-03T07:31:13.052 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.053 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.053 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.054 DEBUG:boto:Params: {} 2022-09-03T07:31:13.055 DEBUG:boto:Token: None 2022-09-03T07:31:13.056 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/ 2022-09-03T07:31:13.056 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E= 2022-09-03T07:31:13.058 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E=', 'Content-Length': '0'} 2022-09-03T07:31:13.061 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000074088769e7c5ed7-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.061 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:13.062 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:13.063 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:13.063 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:13.064 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:13.064 DEBUG:boto:Method: GET 2022-09-03T07:31:13.065 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:31:13.065 DEBUG:boto:Data: 2022-09-03T07:31:13.066 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.066 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.067 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.067 DEBUG:boto:Params: {} 2022-09-03T07:31:13.068 DEBUG:boto:Token: None 2022-09-03T07:31:13.069 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/ 2022-09-03T07:31:13.069 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E= 2022-09-03T07:31:13.070 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E=', 'Content-Length': '0'} 2022-09-03T07:31:13.073 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000797f854fe0163dfe-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.074 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:13.074 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:13.075 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:13.075 DEBUG:boto:Method: HEAD 2022-09-03T07:31:13.076 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:31:13.076 DEBUG:boto:Data: 2022-09-03T07:31:13.077 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.077 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.078 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.078 DEBUG:boto:Params: {} 2022-09-03T07:31:13.079 DEBUG:boto:Token: None 2022-09-03T07:31:13.080 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj93 2022-09-03T07:31:13.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uomzwnR01s5C6oaa9HZIGHCBxCY= 2022-09-03T07:31:13.081 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uomzwnR01s5C6oaa9HZIGHCBxCY=', 'Content-Length': '0'} 2022-09-03T07:31:13.083 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000005b8370f6b6d207f-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.084 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:13.084 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:13.085 DEBUG:boto:Method: HEAD 2022-09-03T07:31:13.085 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:31:13.086 DEBUG:boto:Data: 2022-09-03T07:31:13.086 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.087 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.087 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.088 DEBUG:boto:Params: {} 2022-09-03T07:31:13.088 DEBUG:boto:Token: None 2022-09-03T07:31:13.089 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj93 2022-09-03T07:31:13.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uomzwnR01s5C6oaa9HZIGHCBxCY= 2022-09-03T07:31:13.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uomzwnR01s5C6oaa9HZIGHCBxCY=', 'Content-Length': '0'} 2022-09-03T07:31:13.093 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000313435c2f09b0922-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.094 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj93 2022-09-03T07:31:13.094 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:13.095 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:13.096 DEBUG:boto:Method: GET 2022-09-03T07:31:13.096 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:31:13.097 DEBUG:boto:Data: 2022-09-03T07:31:13.097 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.098 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.098 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.099 DEBUG:boto:Params: {} 2022-09-03T07:31:13.100 DEBUG:boto:Token: None 2022-09-03T07:31:13.100 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj93 2022-09-03T07:31:13.101 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pN66bBP4iITEQStLmnxkiKTcL7o= 2022-09-03T07:31:13.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pN66bBP4iITEQStLmnxkiKTcL7o=', 'Content-Length': '0'} 2022-09-03T07:31:13.103 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eed054aca3908675-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.150 DEBUG:boto:path=/avysmw-75/obj93 2022-09-03T07:31:13.151 DEBUG:boto:auth_path=/avysmw-75/obj93 2022-09-03T07:31:13.152 DEBUG:boto:Method: GET 2022-09-03T07:31:13.152 DEBUG:boto:Path: /avysmw-75/obj93 2022-09-03T07:31:13.153 DEBUG:boto:Data: 2022-09-03T07:31:13.154 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.155 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.156 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.157 DEBUG:boto:Params: {} 2022-09-03T07:31:13.158 DEBUG:boto:Token: None 2022-09-03T07:31:13.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj93 2022-09-03T07:31:13.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:pN66bBP4iITEQStLmnxkiKTcL7o= 2022-09-03T07:31:13.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:pN66bBP4iITEQStLmnxkiKTcL7o=', 'Content-Length': '0'} 2022-09-03T07:31:13.165 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:26:11 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000289f833d5e45153e-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.167 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:13.168 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:13.169 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:13.171 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:13.172 DEBUG:boto:Method: GET 2022-09-03T07:31:13.173 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:31:13.174 DEBUG:boto:Data: 2022-09-03T07:31:13.175 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.177 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.178 DEBUG:boto:Params: {} 2022-09-03T07:31:13.179 DEBUG:boto:Token: None 2022-09-03T07:31:13.180 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/ 2022-09-03T07:31:13.182 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E= 2022-09-03T07:31:13.183 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E=', 'Content-Length': '0'} 2022-09-03T07:31:13.188 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cca088ce4c6c57ed-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.189 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:13.190 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:13.190 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:13.191 DEBUG:boto:path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:13.191 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj93 2022-09-03T07:31:13.192 DEBUG:boto:Method: GET 2022-09-03T07:31:13.192 DEBUG:boto:Path: /avysmw-75/?prefix=obj93 2022-09-03T07:31:13.193 DEBUG:boto:Data: 2022-09-03T07:31:13.193 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.194 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.195 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.195 DEBUG:boto:Params: {} 2022-09-03T07:31:13.196 DEBUG:boto:Token: None 2022-09-03T07:31:13.197 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/ 2022-09-03T07:31:13.197 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E= 2022-09-03T07:31:13.198 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E=', 'Content-Length': '0'} 2022-09-03T07:31:13.201 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000026f150e43e26b925-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.202 DEBUG:boto:b'avysmw-75obj931000falseobj932022-09-03T07:26:11.102Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:13.203 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:31:13.204 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.205 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.206 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.217 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.218 DEBUG:boto:Method: GET 2022-09-03T07:31:13.219 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.220 DEBUG:boto:Data: 2022-09-03T07:31:13.221 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.222 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.223 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.224 DEBUG:boto:Params: {} 2022-09-03T07:31:13.225 DEBUG:boto:Token: None 2022-09-03T07:31:13.226 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.228 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cu/Boss5bAPkssoeFJK+6bLlxc4= 2022-09-03T07:31:13.229 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cu/Boss5bAPkssoeFJK+6bLlxc4=', 'Content-Length': '0'} 2022-09-03T07:31:13.232 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000010b9b29adc13449c-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.234 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.235 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.235 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.236 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.237 DEBUG:boto:Method: GET 2022-09-03T07:31:13.238 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.239 DEBUG:boto:Data: 2022-09-03T07:31:13.240 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.241 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.242 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.243 DEBUG:boto:Params: {} 2022-09-03T07:31:13.243 DEBUG:boto:Token: None 2022-09-03T07:31:13.244 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cu/Boss5bAPkssoeFJK+6bLlxc4= 2022-09-03T07:31:13.245 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cu/Boss5bAPkssoeFJK+6bLlxc4=', 'Content-Length': '0'} 2022-09-03T07:31:13.247 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000115b2c06abe260d4-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.248 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.249 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.249 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.250 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.250 DEBUG:boto:Method: HEAD 2022-09-03T07:31:13.251 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.251 DEBUG:boto:Data: 2022-09-03T07:31:13.252 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.252 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.253 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.253 DEBUG:boto:Params: {} 2022-09-03T07:31:13.254 DEBUG:boto:Token: None 2022-09-03T07:31:13.255 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zxetg3vN7+GNUc33QaOC5utIcFw= 2022-09-03T07:31:13.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zxetg3vN7+GNUc33QaOC5utIcFw=', 'Content-Length': '0'} 2022-09-03T07:31:13.258 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000c0ced6699386123-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.259 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.260 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.261 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.261 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.262 DEBUG:boto:Method: HEAD 2022-09-03T07:31:13.262 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.263 DEBUG:boto:Data: 2022-09-03T07:31:13.263 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.264 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.264 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.265 DEBUG:boto:Params: {} 2022-09-03T07:31:13.266 DEBUG:boto:Token: None 2022-09-03T07:31:13.266 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.267 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zxetg3vN7+GNUc33QaOC5utIcFw= 2022-09-03T07:31:13.268 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zxetg3vN7+GNUc33QaOC5utIcFw=', 'Content-Length': '0'} 2022-09-03T07:31:13.271 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a4c96d68b538cfec-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.272 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:31:13.273 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.274 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.276 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.276 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.277 DEBUG:boto:Method: GET 2022-09-03T07:31:13.279 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.280 DEBUG:boto:Data: 2022-09-03T07:31:13.281 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.282 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.283 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.283 DEBUG:boto:Params: {} 2022-09-03T07:31:13.284 DEBUG:boto:Token: None 2022-09-03T07:31:13.285 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cu/Boss5bAPkssoeFJK+6bLlxc4= 2022-09-03T07:31:13.286 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cu/Boss5bAPkssoeFJK+6bLlxc4=', 'Content-Length': '0'} 2022-09-03T07:31:13.289 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002ff7f908c21e4c1a-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.334 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.335 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.335 DEBUG:boto:path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.336 DEBUG:boto:auth_path=/avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.337 DEBUG:boto:Method: GET 2022-09-03T07:31:13.338 DEBUG:boto:Path: /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.339 DEBUG:boto:Data: 2022-09-03T07:31:13.339 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.340 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.340 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.341 DEBUG:boto:Params: {} 2022-09-03T07:31:13.342 DEBUG:boto:Token: None 2022-09-03T07:31:13.343 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95?versionId=null 2022-09-03T07:31:13.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Cu/Boss5bAPkssoeFJK+6bLlxc4= 2022-09-03T07:31:13.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Cu/Boss5bAPkssoeFJK+6bLlxc4=', 'Content-Length': '0'} 2022-09-03T07:31:13.347 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000084e25c933ed04f56-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.348 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:13.349 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:13.350 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:13.351 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:13.352 DEBUG:boto:Method: GET 2022-09-03T07:31:13.353 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:31:13.354 DEBUG:boto:Data: 2022-09-03T07:31:13.355 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.355 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.356 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.356 DEBUG:boto:Params: {} 2022-09-03T07:31:13.357 DEBUG:boto:Token: None 2022-09-03T07:31:13.358 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/ 2022-09-03T07:31:13.358 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E= 2022-09-03T07:31:13.359 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E=', 'Content-Length': '0'} 2022-09-03T07:31:13.362 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007315a7a5590de4a1-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.363 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:13.364 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:13.365 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:13.366 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:13.367 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:13.367 DEBUG:boto:Method: GET 2022-09-03T07:31:13.368 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:31:13.369 DEBUG:boto:Data: 2022-09-03T07:31:13.369 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.370 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.370 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.371 DEBUG:boto:Params: {} 2022-09-03T07:31:13.372 DEBUG:boto:Token: None 2022-09-03T07:31:13.372 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/ 2022-09-03T07:31:13.373 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E= 2022-09-03T07:31:13.374 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E=', 'Content-Length': '0'} 2022-09-03T07:31:13.377 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008436e897d0e71573-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.378 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:13.380 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.380 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.381 DEBUG:boto:Method: HEAD 2022-09-03T07:31:13.381 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:31:13.382 DEBUG:boto:Data: 2022-09-03T07:31:13.382 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.383 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.383 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.384 DEBUG:boto:Params: {} 2022-09-03T07:31:13.384 DEBUG:boto:Token: None 2022-09-03T07:31:13.385 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95 2022-09-03T07:31:13.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/TLZKJ6x7H69a7ivkb50+uEWurg= 2022-09-03T07:31:13.386 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/TLZKJ6x7H69a7ivkb50+uEWurg=', 'Content-Length': '0'} 2022-09-03T07:31:13.388 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e41ee487bfe321cb-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.389 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.390 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.390 DEBUG:boto:Method: HEAD 2022-09-03T07:31:13.391 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:31:13.391 DEBUG:boto:Data: 2022-09-03T07:31:13.392 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.392 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.393 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.393 DEBUG:boto:Params: {} 2022-09-03T07:31:13.394 DEBUG:boto:Token: None 2022-09-03T07:31:13.394 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95 2022-09-03T07:31:13.395 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/TLZKJ6x7H69a7ivkb50+uEWurg= 2022-09-03T07:31:13.396 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/TLZKJ6x7H69a7ivkb50+uEWurg=', 'Content-Length': '0'} 2022-09-03T07:31:13.398 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e22f531919378a5e-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.399 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj95 2022-09-03T07:31:13.400 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.401 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.402 DEBUG:boto:Method: GET 2022-09-03T07:31:13.403 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:31:13.403 DEBUG:boto:Data: 2022-09-03T07:31:13.404 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.405 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.405 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.406 DEBUG:boto:Params: {} 2022-09-03T07:31:13.407 DEBUG:boto:Token: None 2022-09-03T07:31:13.407 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95 2022-09-03T07:31:13.408 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2EeBdGhTAetRlUNf+AQthkq8PcE= 2022-09-03T07:31:13.408 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2EeBdGhTAetRlUNf+AQthkq8PcE=', 'Content-Length': '0'} 2022-09-03T07:31:13.411 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cf65f58ea0244bea-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.411 DEBUG:boto:path=/avysmw-75/obj95 2022-09-03T07:31:13.412 DEBUG:boto:auth_path=/avysmw-75/obj95 2022-09-03T07:31:13.413 DEBUG:boto:Method: GET 2022-09-03T07:31:13.413 DEBUG:boto:Path: /avysmw-75/obj95 2022-09-03T07:31:13.414 DEBUG:boto:Data: 2022-09-03T07:31:13.414 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.415 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.415 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.416 DEBUG:boto:Params: {} 2022-09-03T07:31:13.416 DEBUG:boto:Token: None 2022-09-03T07:31:13.417 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/obj95 2022-09-03T07:31:13.418 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:2EeBdGhTAetRlUNf+AQthkq8PcE= 2022-09-03T07:31:13.418 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:2EeBdGhTAetRlUNf+AQthkq8PcE=', 'Content-Length': '0'} 2022-09-03T07:31:13.421 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:15:42 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009fc3dc7be8739f43-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.422 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:13.423 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:13.424 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:13.425 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:13.426 DEBUG:boto:Method: GET 2022-09-03T07:31:13.427 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:31:13.428 DEBUG:boto:Data: 2022-09-03T07:31:13.429 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:31:13.430 DEBUG:boto:Port: 8001 2022-09-03T07:31:13.431 DEBUG:boto:Params: {} 2022-09-03T07:31:13.432 DEBUG:boto:Token: None 2022-09-03T07:31:13.433 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/ 2022-09-03T07:31:13.434 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E= 2022-09-03T07:31:13.435 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E=', 'Content-Length': '0'} 2022-09-03T07:31:13.438 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000092088cc616846491-00631302c1-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.438 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:13.439 DEBUG:boto:path=/avysmw-75/ 2022-09-03T07:31:13.440 DEBUG:boto:auth_path=/avysmw-75/ 2022-09-03T07:31:13.440 DEBUG:boto:path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:13.441 DEBUG:boto:auth_path=/avysmw-75/?prefix=obj95 2022-09-03T07:31:13.441 DEBUG:boto:Method: GET 2022-09-03T07:31:13.442 DEBUG:boto:Path: /avysmw-75/?prefix=obj95 2022-09-03T07:31:13.443 DEBUG:boto:Data: 2022-09-03T07:31:13.443 DEBUG:boto:Headers: {} 2022-09-03T07:31:13.444 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:13.444 DEBUG:boto:Port: 8000 2022-09-03T07:31:13.445 DEBUG:boto:Params: {} 2022-09-03T07:31:13.445 DEBUG:boto:Token: None 2022-09-03T07:31:13.446 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:31:13 GMT /avysmw-75/ 2022-09-03T07:31:13.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E= 2022-09-03T07:31:13.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:31:13 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JjsZus5bx2jRbzxriHt6bm5Ct7E=', 'Content-Length': '0'} 2022-09-03T07:31:13.452 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000062e8c83fb5c40e8d-00631302c1-18f5-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:31:13 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:13.453 DEBUG:boto:b'avysmw-75obj951000falseobj952022-09-03T07:15:42.668Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:31:13.454 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-75 zones={test-zone3, test-zone2} 2022-09-03T07:31:13.455 DEBUG:tasks.rgw.c2.client.0:waiting for process to exit 2022-09-03T07:31:13.455 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T07:31:13.457 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T07:31:13.454+0000 7fdce6b7c700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm (PID: 148385) UID: 0 2022-09-03T07:31:13.457 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T07:31:13.454+0000 7fdceade7600 -1 shutting down 2022-09-03T07:31:14.635 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:31:14.633+0000 7f8d0d1f4700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5) 2022-09-03T07:31:14.960 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~0s 2022-09-03T07:31:19.554 INFO:tasks.rgw.c2.client.0:Stopped 2022-09-03T07:31:19.555 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'list', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--max-entries', '10000', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:31:19.556 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'list', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--max-entries', '10000', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:31:19.557 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket list --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --max-entries 10000 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:31:19.620 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:19.620 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:19.750 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T07:31:19.750 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.751 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj11", 2022-09-03T07:31:19.751 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.751 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.752 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.752 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 3 2022-09-03T07:31:19.752 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.753 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.753 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.753 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.754 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.754 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:19.754 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:11:00.918709Z", 2022-09-03T07:31:19.754 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.755 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.755 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.755 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.756 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.756 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.756 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.757 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.757 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.757 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_DbF5FpJbQeZisbyQA_4NfHn6RiO1hno", 2022-09-03T07:31:19.758 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.758 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.758 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.759 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.759 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj111", 2022-09-03T07:31:19.760 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.760 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.761 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.761 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 1 2022-09-03T07:31:19.761 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.762 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.762 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.762 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.763 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.763 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 5, 2022-09-03T07:31:19.764 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:28:24.834105Z", 2022-09-03T07:31:19.764 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.764 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.764 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.765 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.765 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.765 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.766 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.766 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.766 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.766 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_NFUkiitIKzwgROapMwpK9-5IJCwyK_K", 2022-09-03T07:31:19.767 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.767 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.767 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.768 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.768 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.768 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj115", 2022-09-03T07:31:19.768 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.769 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.769 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.769 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 13 2022-09-03T07:31:19.769 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.770 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.770 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.770 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.771 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.771 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 5, 2022-09-03T07:31:19.771 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:21:21.696513Z", 2022-09-03T07:31:19.771 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.772 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.772 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.773 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.773 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.774 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.775 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.775 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.775 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.776 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_xG_2REHAM1twQ33gW0AIw7HEYLpPN3x", 2022-09-03T07:31:19.776 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.776 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.776 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.777 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.777 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.777 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj116", 2022-09-03T07:31:19.778 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.778 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.778 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.778 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 210 2022-09-03T07:31:19.779 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.779 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.779 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.780 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.780 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.780 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 3, 2022-09-03T07:31:19.780 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:24:23.754510Z", 2022-09-03T07:31:19.781 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.781 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.781 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.782 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.782 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.783 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.784 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.786 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.787 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.787 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_ZbbIR3NcSxNoD-0iwEPiysOMtOgKv01", 2022-09-03T07:31:19.788 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.788 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.789 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.789 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.789 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.790 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj124", 2022-09-03T07:31:19.791 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.791 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.792 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.792 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 2 2022-09-03T07:31:19.792 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.793 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.793 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.794 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.794 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.795 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:19.795 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:26:11.165862Z", 2022-09-03T07:31:19.796 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.796 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.796 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.796 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.797 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.797 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.797 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.797 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.798 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.798 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3iDQUo81IFbriHCJ0y6K0rz3z0QGKOP", 2022-09-03T07:31:19.798 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.798 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.799 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.799 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.800 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.800 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj145", 2022-09-03T07:31:19.801 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.801 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.801 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.802 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 58 2022-09-03T07:31:19.802 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.803 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.803 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.804 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.804 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.804 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:19.805 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:24:23.785862Z", 2022-09-03T07:31:19.805 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.806 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.809 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.809 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.810 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.811 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.811 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.812 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.812 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.813 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_J8U-ipYB-Er4WqlzAOcdPT-pUEsCVjS", 2022-09-03T07:31:19.813 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.816 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.816 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.816 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.817 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.817 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj148", 2022-09-03T07:31:19.817 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.817 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.818 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.818 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 2 2022-09-03T07:31:19.818 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.819 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.819 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.819 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.819 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.820 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 3, 2022-09-03T07:31:19.820 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:28:24.868437Z", 2022-09-03T07:31:19.820 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.820 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.821 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.821 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.821 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.821 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.822 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.822 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.822 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.823 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_jnkeiu8yfJNEqW-Iiy7X6BenzW1Rm5_", 2022-09-03T07:31:19.823 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.823 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.824 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.824 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.824 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.824 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj155", 2022-09-03T07:31:19.825 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.825 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.825 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.825 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 7 2022-09-03T07:31:19.826 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.826 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.826 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.827 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.827 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.827 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 6, 2022-09-03T07:31:19.827 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:26:11.240592Z", 2022-09-03T07:31:19.828 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.828 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.828 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.828 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.829 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.829 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.829 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.830 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.830 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.830 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_AJBGPV79szkXG7A5q5o0pnR9gZidPYN", 2022-09-03T07:31:19.830 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.831 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.831 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.831 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.831 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.832 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj17", 2022-09-03T07:31:19.832 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.832 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.832 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.833 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 4 2022-09-03T07:31:19.833 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.833 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.834 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.834 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.835 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.835 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:19.836 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:12:49.441197Z", 2022-09-03T07:31:19.836 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.836 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.837 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.837 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.838 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.839 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.840 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.840 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.841 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.841 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_rABtXDa6YR9X5S2paZeM82AkYkHrkRG", 2022-09-03T07:31:19.842 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.842 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.843 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.843 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.844 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.844 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj185", 2022-09-03T07:31:19.845 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.845 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.846 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.846 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 4 2022-09-03T07:31:19.847 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.847 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.848 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.848 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.849 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.849 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 6, 2022-09-03T07:31:19.850 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:28:24.941185Z", 2022-09-03T07:31:19.850 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.851 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.851 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.852 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.852 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.853 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.853 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.854 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.854 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.855 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3VUY7bYkq4pLh0kfLuZlk3psKJrbvwW", 2022-09-03T07:31:19.855 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.856 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.856 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.857 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.857 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.858 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj19", 2022-09-03T07:31:19.858 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.859 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.859 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.860 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 4 2022-09-03T07:31:19.860 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.861 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.861 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.862 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.862 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.863 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:19.863 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:15:42.415526Z", 2022-09-03T07:31:19.864 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.864 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.865 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.865 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.866 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.867 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.867 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.868 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.868 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.869 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_sqXIjjLjzCrsrrRurltBD1XaKp1zH_2", 2022-09-03T07:31:19.869 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.870 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.870 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.871 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.871 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.871 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj22", 2022-09-03T07:31:19.872 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.872 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.872 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.873 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 6 2022-09-03T07:31:19.873 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.874 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.874 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.875 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.875 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.876 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:19.876 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:11:00.952615Z", 2022-09-03T07:31:19.877 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.877 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.878 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.878 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.879 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.879 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.880 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.880 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.881 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.881 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_aU-SefYiUH8DLOYpISBULwSWI6Pckj1", 2022-09-03T07:31:19.882 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.882 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.883 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.883 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.884 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.884 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj23", 2022-09-03T07:31:19.885 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.885 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.886 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.886 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 4 2022-09-03T07:31:19.886 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.887 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.887 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.887 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.888 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.888 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 3, 2022-09-03T07:31:19.889 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:21:21.541369Z", 2022-09-03T07:31:19.889 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.890 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.890 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.891 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.891 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.892 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.892 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.893 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.893 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.894 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_0mOQSCDwtJgIC1kZBXYwYIURIDKoEIg", 2022-09-03T07:31:19.894 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.895 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.895 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.895 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.896 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.896 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj29", 2022-09-03T07:31:19.896 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.896 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.897 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.897 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 16 2022-09-03T07:31:19.897 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.897 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.898 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.898 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.898 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.898 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 6, 2022-09-03T07:31:19.899 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:24:23.601424Z", 2022-09-03T07:31:19.899 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.899 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.899 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.900 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.900 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.900 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.900 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.901 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.901 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.901 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_R5jQWKYS1LTvruTaSJWtkL604X7X6_E", 2022-09-03T07:31:19.901 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.902 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.902 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.903 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.903 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.903 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj31", 2022-09-03T07:31:19.903 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.904 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.904 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.904 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 14 2022-09-03T07:31:19.905 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.905 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.905 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.905 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.906 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.906 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 3, 2022-09-03T07:31:19.906 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:26:10.978539Z", 2022-09-03T07:31:19.906 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.907 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.907 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.908 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.908 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.909 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.909 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.910 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.910 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.910 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_u1gSvK_uyoMv7gr2NQlaQF6O69L_Ea6", 2022-09-03T07:31:19.911 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.911 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.912 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.912 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.913 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.913 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj33", 2022-09-03T07:31:19.914 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.914 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.915 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.915 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 209 2022-09-03T07:31:19.916 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.916 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.917 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.917 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.917 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.919 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 3, 2022-09-03T07:31:19.919 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:11:00.977103Z", 2022-09-03T07:31:19.919 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.920 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.920 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.922 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.922 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.922 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.923 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.923 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.923 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.924 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3XtMTgj8XOIFYNXgpyLGKBV5S6JkKaF", 2022-09-03T07:31:19.924 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.924 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.924 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.925 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.925 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.925 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj34", 2022-09-03T07:31:19.925 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.926 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.926 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.926 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 3 2022-09-03T07:31:19.927 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.927 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.927 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.927 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.928 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.928 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 5, 2022-09-03T07:31:19.928 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:12:49.510495Z", 2022-09-03T07:31:19.928 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.929 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.929 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.929 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.929 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.930 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.930 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.930 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.931 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.931 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8LXTrdPajVcMMU3qg7H_md2D3RPRsne", 2022-09-03T07:31:19.931 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.932 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.932 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.932 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.932 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.932 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj37", 2022-09-03T07:31:19.933 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.933 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.933 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.933 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 2 2022-09-03T07:31:19.934 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.934 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.934 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.935 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.935 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.935 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:19.935 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:28:24.732230Z", 2022-09-03T07:31:19.936 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.936 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.936 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.936 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.937 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.937 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.937 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.937 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.938 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.938 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_3nR045S90jedCxdW8pIA7aUxTmX5nhD", 2022-09-03T07:31:19.938 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.938 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.939 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.939 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.940 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj38", 2022-09-03T07:31:19.940 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.941 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.941 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.942 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 7 2022-09-03T07:31:19.942 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.943 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.943 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.943 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.944 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.944 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:19.945 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:15:42.462137Z", 2022-09-03T07:31:19.945 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.945 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.946 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.946 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.947 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.947 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.948 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.948 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.948 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.949 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_QYxIjGqolQ-jOTPwr1dwSCHNFYGeD33", 2022-09-03T07:31:19.949 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.949 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.950 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.950 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.951 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj44", 2022-09-03T07:31:19.951 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.952 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.952 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.952 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 15 2022-09-03T07:31:19.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.953 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.953 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.954 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.954 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.955 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 3, 2022-09-03T07:31:19.955 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:11:01.065637Z", 2022-09-03T07:31:19.956 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.956 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.957 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.957 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.957 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.958 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.958 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.959 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.959 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.960 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "__MSwfStBND-y2cGkO-tQKl866etj6le", 2022-09-03T07:31:19.960 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.960 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.961 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.962 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.962 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj46", 2022-09-03T07:31:19.962 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.963 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.963 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.963 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 1 2022-09-03T07:31:19.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.964 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.964 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.964 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.965 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.965 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 6, 2022-09-03T07:31:19.965 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:21:21.569739Z", 2022-09-03T07:31:19.965 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.966 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.966 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.966 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.967 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.967 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.967 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.967 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.968 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.968 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_pHQxKQ9dT1kjrWeqOvYzSAZ9vRyX2mX", 2022-09-03T07:31:19.968 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.968 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.969 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.969 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.969 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.969 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj51", 2022-09-03T07:31:19.970 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.970 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.970 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.971 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 3 2022-09-03T07:31:19.971 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.972 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.972 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.972 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.973 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.973 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:19.973 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:12:49.547024Z", 2022-09-03T07:31:19.974 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.974 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.974 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.974 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.975 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.975 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.975 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.975 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.976 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_RaGP923ANNsl1DdTWY9eZ235nsmYW9h", 2022-09-03T07:31:19.976 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.976 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.977 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.977 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.977 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj55", 2022-09-03T07:31:19.978 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.978 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.978 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.978 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 3 2022-09-03T07:31:19.979 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.979 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.980 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.980 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.981 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.981 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 6, 2022-09-03T07:31:19.982 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:11:01.102684Z", 2022-09-03T07:31:19.983 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.983 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.984 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.984 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.985 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.985 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.986 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.986 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.987 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.987 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_0nqKZwjWV7nTe0yjuWSHReDnZ_WRWHt", 2022-09-03T07:31:19.988 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.988 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:19.989 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:19.990 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.990 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:19.991 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj57", 2022-09-03T07:31:19.991 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:19.992 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:19.992 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:19.992 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 14 2022-09-03T07:31:19.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.993 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:19.993 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:19.994 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:19.994 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:19.995 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 5, 2022-09-03T07:31:19.995 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:15:42.542775Z", 2022-09-03T07:31:19.995 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:19.996 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:19.996 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:19.996 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:19.997 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:19.997 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:19.997 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:19.998 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:19.998 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:19.998 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Du8rez6jHErD5ZsO_mKGqp3YjDSGH8-", 2022-09-03T07:31:19.999 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:19.999 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.000 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.000 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.000 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.001 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj58", 2022-09-03T07:31:20.001 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.002 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.002 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.002 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 7 2022-09-03T07:31:20.003 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.003 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.004 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.004 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.005 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.005 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:20.005 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:24:23.666184Z", 2022-09-03T07:31:20.006 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.006 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.006 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.007 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.007 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.007 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.007 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.008 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.008 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.008 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_wsbqxb7RjloveTMbKjeVj5Q3v2PskbT", 2022-09-03T07:31:20.008 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.008 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.009 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.009 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj62", 2022-09-03T07:31:20.010 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.010 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.010 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": -1, 2022-09-03T07:31:20.011 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 0 2022-09-03T07:31:20.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.011 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.011 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.012 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.012 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.012 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 3, 2022-09-03T07:31:20.012 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:26:11.009907Z", 2022-09-03T07:31:20.013 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.013 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.013 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.013 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.014 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.014 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.014 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.014 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.015 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_sNTy1CLlyH0pRFCft3NceOLlE7i6Aap", 2022-09-03T07:31:20.015 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.015 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.016 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.016 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.016 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj68", 2022-09-03T07:31:20.016 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.017 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.017 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.017 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 3 2022-09-03T07:31:20.017 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.018 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.018 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.018 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.018 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.019 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 3, 2022-09-03T07:31:20.019 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:12:49.631167Z", 2022-09-03T07:31:20.019 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.019 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.020 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.020 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.020 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.020 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.021 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.021 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.021 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_KrmCQYVta9pp_we0n3ypQWTq9tFQSz8", 2022-09-03T07:31:20.022 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.022 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.022 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.022 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.023 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj69", 2022-09-03T07:31:20.023 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.023 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.024 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.024 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 16 2022-09-03T07:31:20.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.024 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.025 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.025 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.025 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.025 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:20.026 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:21:21.604820Z", 2022-09-03T07:31:20.026 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.026 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.026 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.027 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.027 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.027 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.027 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.028 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.028 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_qvIivSgCZbyB0aeH0orbF43D7OLS-F0", 2022-09-03T07:31:20.028 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.029 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.029 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.029 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.029 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.030 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj74", 2022-09-03T07:31:20.030 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.030 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.030 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.031 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 7 2022-09-03T07:31:20.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.031 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.031 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.032 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.032 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.032 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:20.032 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:28:24.765927Z", 2022-09-03T07:31:20.033 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.033 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.033 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.033 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.034 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.034 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.034 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.034 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.035 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_9XPVbJLZF8wJTML1taS3AduapEImD76", 2022-09-03T07:31:20.035 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.035 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.036 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.036 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.036 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj76", 2022-09-03T07:31:20.037 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.037 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.037 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.037 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 3 2022-09-03T07:31:20.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.038 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.038 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.038 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.039 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.039 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 3, 2022-09-03T07:31:20.039 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:15:42.592244Z", 2022-09-03T07:31:20.039 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.040 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.040 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.040 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.040 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.041 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.041 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.041 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.042 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_i_SIEf0BRPj5a4CHl2KIRm8PW5Zg-9b", 2022-09-03T07:31:20.042 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.042 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.042 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.043 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj85", 2022-09-03T07:31:20.043 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.044 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.044 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.044 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 2 2022-09-03T07:31:20.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.045 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.045 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.045 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.045 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.046 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:20.046 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:12:49.663790Z", 2022-09-03T07:31:20.046 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.046 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.047 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.047 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.047 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.047 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.048 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.048 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.048 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_6uu5xZD1s-1FSP_zrpWuTiPTaRHFJoR", 2022-09-03T07:31:20.049 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.049 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.049 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.050 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj87", 2022-09-03T07:31:20.050 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.050 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.050 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.051 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 4 2022-09-03T07:31:20.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.051 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.051 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.052 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.053 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.054 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 6, 2022-09-03T07:31:20.054 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:24:23.692367Z", 2022-09-03T07:31:20.054 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.054 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.055 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.055 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.055 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.055 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.056 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.056 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.056 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_fMvj4ZimwgZwpCSGBGS-TpnekvdyNTI", 2022-09-03T07:31:20.057 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.057 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.057 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.058 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj92", 2022-09-03T07:31:20.058 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.058 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.059 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.059 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 5 2022-09-03T07:31:20.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.059 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.060 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.060 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.060 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.060 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:20.061 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:21:21.666800Z", 2022-09-03T07:31:20.061 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.061 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.061 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.062 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.062 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.062 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.062 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.063 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.063 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.063 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_2sGRe6g2TIonIHdfYfg9fSnG3oaCt0O", 2022-09-03T07:31:20.063 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.064 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.064 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.064 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.065 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj93", 2022-09-03T07:31:20.065 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.065 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.065 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.066 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 5 2022-09-03T07:31:20.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.066 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.066 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.067 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.067 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.067 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 16, 2022-09-03T07:31:20.067 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:26:11.102394Z", 2022-09-03T07:31:20.068 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.068 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.068 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.068 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.069 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.069 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.069 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.069 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.070 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_8b-d3899sRNwpsmH6YQxUj7u6uzmdEK", 2022-09-03T07:31:20.070 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.070 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.071 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.071 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:20.071 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "obj95", 2022-09-03T07:31:20.072 INFO:teuthology.orchestra.run.smithi087.stdout: "instance": "", 2022-09-03T07:31:20.072 INFO:teuthology.orchestra.run.smithi087.stdout: "ver": { 2022-09-03T07:31:20.072 INFO:teuthology.orchestra.run.smithi087.stdout: "pool": 9, 2022-09-03T07:31:20.072 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 15 2022-09-03T07:31:20.072 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.073 INFO:teuthology.orchestra.run.smithi087.stdout: "locator": "", 2022-09-03T07:31:20.073 INFO:teuthology.orchestra.run.smithi087.stdout: "exists": true, 2022-09-03T07:31:20.073 INFO:teuthology.orchestra.run.smithi087.stdout: "meta": { 2022-09-03T07:31:20.073 INFO:teuthology.orchestra.run.smithi087.stdout: "category": 1, 2022-09-03T07:31:20.074 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 5, 2022-09-03T07:31:20.074 INFO:teuthology.orchestra.run.smithi087.stdout: "mtime": "2022-09-03T07:15:42.668902Z", 2022-09-03T07:31:20.074 INFO:teuthology.orchestra.run.smithi087.stdout: "etag": "acbd18db4cc2f85cedef654fccc4a4d8", 2022-09-03T07:31:20.074 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_class": "", 2022-09-03T07:31:20.075 INFO:teuthology.orchestra.run.smithi087.stdout: "owner": "rgw-multisite-test-user", 2022-09-03T07:31:20.075 INFO:teuthology.orchestra.run.smithi087.stdout: "owner_display_name": "Multisite Test User", 2022-09-03T07:31:20.075 INFO:teuthology.orchestra.run.smithi087.stdout: "content_type": "application/octet-stream", 2022-09-03T07:31:20.076 INFO:teuthology.orchestra.run.smithi087.stdout: "accounted_size": 3, 2022-09-03T07:31:20.076 INFO:teuthology.orchestra.run.smithi087.stdout: "user_data": "", 2022-09-03T07:31:20.076 INFO:teuthology.orchestra.run.smithi087.stdout: "appendable": false 2022-09-03T07:31:20.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:20.076 INFO:teuthology.orchestra.run.smithi087.stdout: "tag": "_Gw5_yolHmgKuDYY2jj70Otod575nJt-", 2022-09-03T07:31:20.077 INFO:teuthology.orchestra.run.smithi087.stdout: "flags": 0, 2022-09-03T07:31:20.077 INFO:teuthology.orchestra.run.smithi087.stdout: "pending_map": [], 2022-09-03T07:31:20.077 INFO:teuthology.orchestra.run.smithi087.stdout: "versioned_epoch": 0 2022-09-03T07:31:20.077 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:31:20.078 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T07:31:20.079 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj11', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.079 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj11', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.080 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj11 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:20.132 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:20.132 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:20.306 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj111', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.307 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj111', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.308 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj111 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:20.361 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:20.361 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:20.488 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj115', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.489 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj115', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.490 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj115 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:20.535 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T07:31:20.534+0000 7f000b05c700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5) 2022-09-03T07:31:20.551 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:20.552 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:20.655 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 has been restored 2022-09-03T07:31:20.687 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj116', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.688 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj116', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.689 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj116 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:20.750 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:20.751 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:20.890 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj124', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.891 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj124', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:20.892 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj124 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:20.943 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:20.944 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:21.109 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj145', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.110 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj145', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.111 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj145 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:21.166 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:21.166 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:21.313 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj148', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.313 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj148', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.314 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj148 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:21.367 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:21.368 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:21.516 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj155', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.517 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj155', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.518 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj155 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:21.577 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:21.578 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:21.720 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj17', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.720 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj17', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.721 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj17 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:21.783 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:21.784 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:21.908 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj185', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.910 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj185', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:21.911 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj185 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:21.970 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:21.970 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:22.127 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj19', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.128 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj19', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.129 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj19 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:22.187 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:22.188 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:22.320 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj22', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.321 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj22', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.322 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj22 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:22.380 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:22.380 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:22.529 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj23', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.531 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj23', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.531 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj23 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:22.587 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:22.588 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:22.758 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj29', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.759 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj29', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.760 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj29 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:22.815 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:22.816 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:22.947 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj31', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.947 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj31', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:22.948 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj31 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:23.005 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:23.006 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:23.139 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj33', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.140 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj33', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.140 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj33 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:23.199 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:23.200 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:23.326 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj34', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.326 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj34', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.327 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj34 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:23.379 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:23.380 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:23.532 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj37', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.533 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj37', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.534 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj37 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:23.587 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:23.587 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:23.708 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj38', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.709 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj38', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.710 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj38 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:23.772 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:23.773 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:23.902 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj44', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.903 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj44', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:23.904 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj44 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:23.963 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:23.963 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:24.117 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj46', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:24.118 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj46', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:24.119 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj46 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:24.175 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:24.176 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:24.338 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj51', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:24.339 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj51', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:24.340 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj51 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:24.392 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:24.393 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:24.586 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj55', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:24.587 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj55', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:24.588 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj55 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:24.648 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:24.649 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:24.812 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj57', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:24.813 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj57', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:24.815 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj57 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:24.873 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:24.874 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:25.042 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj58', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.043 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj58', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.044 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj58 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:25.101 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:25.101 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:25.250 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj62', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.251 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj62', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.252 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj62 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:25.309 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:25.310 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:25.457 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj68', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.458 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj68', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.458 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj68 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:25.517 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:25.518 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:25.651 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj69', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.652 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj69', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.652 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj69 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:25.711 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:25.712 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:25.867 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj74', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.870 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj74', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:25.872 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj74 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:25.930 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:25.931 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:26.084 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj76', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.085 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj76', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.086 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj76 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:26.149 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:26.149 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:26.296 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj85', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.297 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj85', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.315 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj85 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:26.378 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:26.378 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:26.510 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj87', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.511 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj87', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.514 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj87 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:26.578 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:26.580 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:26.755 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj92', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.756 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj92', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.757 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj92 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:26.815 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:26.815 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:26.975 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj93', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.976 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj93', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:26.977 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj93 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:27.039 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:27.039 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:27.177 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj95', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:27.178 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'object', 'rm', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--object', 'obj95', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:27.178 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 object rm --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --object obj95 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:27.234 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:27.235 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:27.386 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bilog', 'trim', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:27.386 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bilog', 'trim', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:27.387 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bilog trim --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:27.443 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:27.444 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:27.517 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'init', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:27.518 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'init', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:27.521 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync init --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:27.586 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:27.586 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:27.667 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'run', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:27.668 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'run', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-75', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:31:27.669 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync run --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-75 --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:31:27.725 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:27.726 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:27.840 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.837+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: sync status for source bucket: init. lease is: not taken. stop indications is: 0 2022-09-03T07:31:27.844 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.842+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: waiting on bucket lease 2022-09-03T07:31:27.846 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.843+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: status after acquiring the lock is: init 2022-09-03T07:31:27.885 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.881+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: listing bucket for full sync 2022-09-03T07:31:27.932 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj11 2022-09-03T07:31:27.933 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj11]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj11 op=0 op_state=1 2022-09-03T07:31:27.933 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj11]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj11[0] 2022-09-03T07:31:27.933 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj111 2022-09-03T07:31:27.933 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj111]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj111 op=0 op_state=1 2022-09-03T07:31:27.934 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj111]: : injecting data sync error on key=obj111 2022-09-03T07:31:27.934 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj111]: failed, retcode=-5 ((5) Input/output error) 2022-09-03T07:31:27.934 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj111]: ERROR: failed to sync object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj111 2022-09-03T07:31:27.934 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj115 2022-09-03T07:31:27.935 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj115]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj115 op=0 op_state=1 2022-09-03T07:31:27.935 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj115]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj115[0] 2022-09-03T07:31:27.935 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj116 2022-09-03T07:31:27.935 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj116]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj116 op=0 op_state=1 2022-09-03T07:31:27.936 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj116]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj116[0] 2022-09-03T07:31:27.936 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj124 2022-09-03T07:31:27.936 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj124]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj124 op=0 op_state=1 2022-09-03T07:31:27.936 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj124]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj124[0] 2022-09-03T07:31:27.937 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj145 2022-09-03T07:31:27.937 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj145]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj145 op=0 op_state=1 2022-09-03T07:31:27.937 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj145]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj145[0] 2022-09-03T07:31:27.937 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj148 2022-09-03T07:31:27.938 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj148]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj148 op=0 op_state=1 2022-09-03T07:31:27.938 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj148]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj148[0] 2022-09-03T07:31:27.939 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj155 2022-09-03T07:31:27.942 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj155]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj155 op=0 op_state=1 2022-09-03T07:31:27.942 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj155]: : injecting data sync error on key=obj155 2022-09-03T07:31:27.943 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj155]: failed, retcode=-5 ((5) Input/output error) 2022-09-03T07:31:27.943 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj155]: ERROR: failed to sync object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj155 2022-09-03T07:31:27.944 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj17 2022-09-03T07:31:27.944 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj17]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj17 op=0 op_state=1 2022-09-03T07:31:27.945 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj17]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj17[0] 2022-09-03T07:31:27.945 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj185 2022-09-03T07:31:27.946 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj185]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj185 op=0 op_state=1 2022-09-03T07:31:27.946 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj185]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj185[0] 2022-09-03T07:31:27.947 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj19 2022-09-03T07:31:27.948 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj19]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj19 op=0 op_state=1 2022-09-03T07:31:27.948 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj19]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj19[0] 2022-09-03T07:31:27.949 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj22 2022-09-03T07:31:27.949 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj22]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj22 op=0 op_state=1 2022-09-03T07:31:27.950 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj22]: : injecting data sync error on key=obj22 2022-09-03T07:31:27.951 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj22]: failed, retcode=-5 ((5) Input/output error) 2022-09-03T07:31:27.951 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj22]: ERROR: failed to sync object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj22 2022-09-03T07:31:27.952 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj23 2022-09-03T07:31:27.952 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj23]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj23 op=0 op_state=1 2022-09-03T07:31:27.952 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj23]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj23[0] 2022-09-03T07:31:27.953 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj29 2022-09-03T07:31:27.953 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj29]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj29 op=0 op_state=1 2022-09-03T07:31:27.953 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj29]: : injecting data sync error on key=obj29 2022-09-03T07:31:27.954 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj29]: failed, retcode=-5 ((5) Input/output error) 2022-09-03T07:31:27.954 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj29]: ERROR: failed to sync object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj29 2022-09-03T07:31:27.954 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj31 2022-09-03T07:31:27.954 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj31]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj31 op=0 op_state=1 2022-09-03T07:31:27.955 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj31]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj31[0] 2022-09-03T07:31:27.955 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj33 2022-09-03T07:31:27.955 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj33]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj33 op=0 op_state=1 2022-09-03T07:31:27.956 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj33]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj33[0] 2022-09-03T07:31:27.956 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj34 2022-09-03T07:31:27.956 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj34]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj34 op=0 op_state=1 2022-09-03T07:31:27.957 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj34]: : injecting data sync error on key=obj34 2022-09-03T07:31:27.957 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj34]: failed, retcode=-5 ((5) Input/output error) 2022-09-03T07:31:27.958 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj34]: ERROR: failed to sync object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj34 2022-09-03T07:31:27.958 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj37 2022-09-03T07:31:27.959 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj37]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj37 op=0 op_state=1 2022-09-03T07:31:27.960 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj37]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj37[0] 2022-09-03T07:31:27.960 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj38 2022-09-03T07:31:27.961 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj38]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj38 op=0 op_state=1 2022-09-03T07:31:27.961 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj38]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj38[0] 2022-09-03T07:31:27.961 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj44 2022-09-03T07:31:27.962 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj44]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj44 op=0 op_state=1 2022-09-03T07:31:27.962 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj44]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj44[0] 2022-09-03T07:31:27.963 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj46 2022-09-03T07:31:27.963 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj46]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj46 op=0 op_state=1 2022-09-03T07:31:27.964 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:27.929+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj46]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj46[0] 2022-09-03T07:31:28.011 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:28.008+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: ERROR: full sync failed. error: -5 2022-09-03T07:31:28.061 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:28.059+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj37]: success 2022-09-03T07:31:28.062 INFO:teuthology.orchestra.run.smithi087.stderr:*** Caught signal (Segmentation fault) ** 2022-09-03T07:31:28.062 INFO:teuthology.orchestra.run.smithi087.stderr: in thread 7f2cb4395600 thread_name:radosgw-admin 2022-09-03T07:31:28.074 INFO:teuthology.orchestra.run.smithi087.stderr: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:31:28.075 INFO:teuthology.orchestra.run.smithi087.stderr: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f2cbbb5bcf0] 2022-09-03T07:31:28.075 INFO:teuthology.orchestra.run.smithi087.stderr: 2: (tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned int, int)+0x93) [0x7f2cbcba9523] 2022-09-03T07:31:28.075 INFO:teuthology.orchestra.run.smithi087.stderr: 3: (tcmalloc::ThreadCache::Scavenge()+0x4e) [0x7f2cbcba986e] 2022-09-03T07:31:28.076 INFO:teuthology.orchestra.run.smithi087.stderr: 4: (ceph::common::RefCountedObject::put() const+0x1ba) [0x7f2cbaf8748a] 2022-09-03T07:31:28.076 INFO:teuthology.orchestra.run.smithi087.stderr: 5: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x360) [0x55e25ec0e5e0] 2022-09-03T07:31:28.076 INFO:teuthology.orchestra.run.smithi087.stderr: 6: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55e25ec0ebe1] 2022-09-03T07:31:28.076 INFO:teuthology.orchestra.run.smithi087.stderr: 7: (RGWBucketPipeSyncStatusManager::run(DoutPrefixProvider const*)+0xaca) [0x55e25eabecaa] 2022-09-03T07:31:28.077 INFO:teuthology.orchestra.run.smithi087.stderr: 8: main() 2022-09-03T07:31:28.077 INFO:teuthology.orchestra.run.smithi087.stderr: 9: __libc_start_main() 2022-09-03T07:31:28.077 INFO:teuthology.orchestra.run.smithi087.stderr: 10: _start() 2022-09-03T07:31:28.078 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:28.061+0000 7f2cb4395600 -1 *** Caught signal (Segmentation fault) ** 2022-09-03T07:31:28.078 INFO:teuthology.orchestra.run.smithi087.stderr: in thread 7f2cb4395600 thread_name:radosgw-admin 2022-09-03T07:31:28.078 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.078 INFO:teuthology.orchestra.run.smithi087.stderr: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:31:28.079 INFO:teuthology.orchestra.run.smithi087.stderr: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f2cbbb5bcf0] 2022-09-03T07:31:28.079 INFO:teuthology.orchestra.run.smithi087.stderr: 2: (tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned int, int)+0x93) [0x7f2cbcba9523] 2022-09-03T07:31:28.079 INFO:teuthology.orchestra.run.smithi087.stderr: 3: (tcmalloc::ThreadCache::Scavenge()+0x4e) [0x7f2cbcba986e] 2022-09-03T07:31:28.079 INFO:teuthology.orchestra.run.smithi087.stderr: 4: (ceph::common::RefCountedObject::put() const+0x1ba) [0x7f2cbaf8748a] 2022-09-03T07:31:28.080 INFO:teuthology.orchestra.run.smithi087.stderr: 5: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x360) [0x55e25ec0e5e0] 2022-09-03T07:31:28.080 INFO:teuthology.orchestra.run.smithi087.stderr: 6: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55e25ec0ebe1] 2022-09-03T07:31:28.080 INFO:teuthology.orchestra.run.smithi087.stderr: 7: (RGWBucketPipeSyncStatusManager::run(DoutPrefixProvider const*)+0xaca) [0x55e25eabecaa] 2022-09-03T07:31:28.080 INFO:teuthology.orchestra.run.smithi087.stderr: 8: main() 2022-09-03T07:31:28.081 INFO:teuthology.orchestra.run.smithi087.stderr: 9: __libc_start_main() 2022-09-03T07:31:28.081 INFO:teuthology.orchestra.run.smithi087.stderr: 10: _start() 2022-09-03T07:31:28.081 INFO:teuthology.orchestra.run.smithi087.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2022-09-03T07:31:28.081 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.083 INFO:teuthology.orchestra.run.smithi087.stderr:--- begin dump of recent events --- 2022-09-03T07:31:28.084 INFO:teuthology.orchestra.run.smithi087.stderr: -499> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.085 INFO:teuthology.orchestra.run.smithi087.stderr: -498> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.085 INFO:teuthology.orchestra.run.smithi087.stderr: -497> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.085 INFO:teuthology.orchestra.run.smithi087.stderr: -496> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.086 INFO:teuthology.orchestra.run.smithi087.stderr: -495> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.087 INFO:teuthology.orchestra.run.smithi087.stderr: -494> 2022-09-03T07:31:27.723+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.088 INFO:teuthology.orchestra.run.smithi087.stderr: -493> 2022-09-03T07:31:27.723+0000 7f2cb4395600 10 monclient: init 2022-09-03T07:31:28.088 INFO:teuthology.orchestra.run.smithi087.stderr: -492> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding auth protocol: cephx 2022-09-03T07:31:28.089 INFO:teuthology.orchestra.run.smithi087.stderr: -491> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding auth protocol: cephx 2022-09-03T07:31:28.090 INFO:teuthology.orchestra.run.smithi087.stderr: -490> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding auth protocol: cephx 2022-09-03T07:31:28.090 INFO:teuthology.orchestra.run.smithi087.stderr: -489> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: secure 2022-09-03T07:31:28.091 INFO:teuthology.orchestra.run.smithi087.stderr: -488> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: crc 2022-09-03T07:31:28.091 INFO:teuthology.orchestra.run.smithi087.stderr: -487> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: secure 2022-09-03T07:31:28.092 INFO:teuthology.orchestra.run.smithi087.stderr: -486> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: crc 2022-09-03T07:31:28.093 INFO:teuthology.orchestra.run.smithi087.stderr: -485> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: secure 2022-09-03T07:31:28.093 INFO:teuthology.orchestra.run.smithi087.stderr: -484> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: crc 2022-09-03T07:31:28.093 INFO:teuthology.orchestra.run.smithi087.stderr: -483> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: crc 2022-09-03T07:31:28.093 INFO:teuthology.orchestra.run.smithi087.stderr: -482> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: secure 2022-09-03T07:31:28.094 INFO:teuthology.orchestra.run.smithi087.stderr: -481> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: crc 2022-09-03T07:31:28.094 INFO:teuthology.orchestra.run.smithi087.stderr: -480> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: secure 2022-09-03T07:31:28.094 INFO:teuthology.orchestra.run.smithi087.stderr: -479> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: crc 2022-09-03T07:31:28.095 INFO:teuthology.orchestra.run.smithi087.stderr: -478> 2022-09-03T07:31:27.723+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1150) adding con mode: secure 2022-09-03T07:31:28.095 INFO:teuthology.orchestra.run.smithi087.stderr: -477> 2022-09-03T07:31:27.723+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.095 INFO:teuthology.orchestra.run.smithi087.stderr: -476> 2022-09-03T07:31:27.723+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.095 INFO:teuthology.orchestra.run.smithi087.stderr: -475> 2022-09-03T07:31:27.723+0000 7f2cb4395600 10 monclient: _reopen_session rank -1 2022-09-03T07:31:28.096 INFO:teuthology.orchestra.run.smithi087.stderr: -474> 2022-09-03T07:31:27.723+0000 7f2cb4395600 10 monclient: _add_conns ranks=[0,1] 2022-09-03T07:31:28.096 INFO:teuthology.orchestra.run.smithi087.stderr: -473> 2022-09-03T07:31:27.723+0000 7f2cb4395600 10 monclient(hunting): picked mon.noname-a-legacy con 0x55e260530800 addr v1:172.21.15.87:6789/0 2022-09-03T07:31:28.096 INFO:teuthology.orchestra.run.smithi087.stderr: -472> 2022-09-03T07:31:27.723+0000 7f2cb4395600 10 monclient(hunting): picked mon.noname-a con 0x55e260530c00 addr v2:172.21.15.87:3300/0 2022-09-03T07:31:28.096 INFO:teuthology.orchestra.run.smithi087.stderr: -471> 2022-09-03T07:31:27.723+0000 7f2cb4395600 10 monclient(hunting): start opening mon connection 2022-09-03T07:31:28.097 INFO:teuthology.orchestra.run.smithi087.stderr: -470> 2022-09-03T07:31:27.723+0000 7f2cb4395600 10 monclient(hunting): _renew_subs 2022-09-03T07:31:28.097 INFO:teuthology.orchestra.run.smithi087.stderr: -469> 2022-09-03T07:31:27.723+0000 7f2cb4395600 10 monclient(hunting): authenticate will time out at 7208.143924s 2022-09-03T07:31:28.097 INFO:teuthology.orchestra.run.smithi087.stderr: -468> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient(hunting): get_auth_request con 0x55e260530c00 auth_method 0 2022-09-03T07:31:28.097 INFO:teuthology.orchestra.run.smithi087.stderr: -467> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] 2022-09-03T07:31:28.098 INFO:teuthology.orchestra.run.smithi087.stderr: -466> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient(hunting): _init_auth method 2 2022-09-03T07:31:28.098 INFO:teuthology.orchestra.run.smithi087.stderr: -465> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient(hunting): _init_auth creating new auth 2022-09-03T07:31:28.099 INFO:teuthology.orchestra.run.smithi087.stderr: -464> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient(hunting): handle_auth_reply_more payload 9 2022-09-03T07:31:28.099 INFO:teuthology.orchestra.run.smithi087.stderr: -463> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient(hunting): handle_auth_reply_more payload_len 9 2022-09-03T07:31:28.099 INFO:teuthology.orchestra.run.smithi087.stderr: -462> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes 2022-09-03T07:31:28.100 INFO:teuthology.orchestra.run.smithi087.stderr: -461> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient(hunting): handle_auth_done global_id 6811 payload 274 2022-09-03T07:31:28.100 INFO:teuthology.orchestra.run.smithi087.stderr: -460> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient: _finish_hunting 0 2022-09-03T07:31:28.101 INFO:teuthology.orchestra.run.smithi087.stderr: -459> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 1 monclient: found mon.noname-a 2022-09-03T07:31:28.101 INFO:teuthology.orchestra.run.smithi087.stderr: -458> 2022-09-03T07:31:27.724+0000 7f2cb1b40700 10 monclient: _send_mon_message to mon.noname-a at v2:172.21.15.87:3300/0 2022-09-03T07:31:28.102 INFO:teuthology.orchestra.run.smithi087.stderr: -457> 2022-09-03T07:31:27.724+0000 7f2cb133f700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.102 INFO:teuthology.orchestra.run.smithi087.stderr: -456> 2022-09-03T07:31:27.724+0000 7f2cb133f700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) 2022-09-03T07:31:28.103 INFO:teuthology.orchestra.run.smithi087.stderr: -455> 2022-09-03T07:31:27.724+0000 7f2cb133f700 10 monclient: dump: 2022-09-03T07:31:28.103 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.104 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.104 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.105 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.105 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.106 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.106 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.106 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.107 INFO:teuthology.orchestra.run.smithi087.stderr: -454> 2022-09-03T07:31:27.724+0000 7f2cb133f700 10 monclient: _finish_auth 0 2022-09-03T07:31:28.108 INFO:teuthology.orchestra.run.smithi087.stderr: -453> 2022-09-03T07:31:27.724+0000 7f2cb133f700 10 monclient: _check_auth_tickets 2022-09-03T07:31:28.108 INFO:teuthology.orchestra.run.smithi087.stderr: -452> 2022-09-03T07:31:27.724+0000 7f2cb4395600 5 monclient: authenticate success, global_id 6811 2022-09-03T07:31:28.109 INFO:teuthology.orchestra.run.smithi087.stderr: -451> 2022-09-03T07:31:27.724+0000 7f2cb133f700 10 monclient: handle_config config(0 keys) v1 2022-09-03T07:31:28.109 INFO:teuthology.orchestra.run.smithi087.stderr: -450> 2022-09-03T07:31:27.724+0000 7f2cb133f700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.109 INFO:teuthology.orchestra.run.smithi087.stderr: -449> 2022-09-03T07:31:27.724+0000 7f2cb133f700 10 monclient: got monmap 1 from mon.a (according to old e1) 2022-09-03T07:31:28.110 INFO:teuthology.orchestra.run.smithi087.stderr: -448> 2022-09-03T07:31:27.724+0000 7f2cb133f700 10 monclient: dump: 2022-09-03T07:31:28.110 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.111 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.111 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.111 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.111 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.112 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.112 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.112 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.112 INFO:teuthology.orchestra.run.smithi087.stderr: -447> 2022-09-03T07:31:27.724+0000 7f2cb4395600 10 monclient: get_monmap_and_config success 2022-09-03T07:31:28.113 INFO:teuthology.orchestra.run.smithi087.stderr: -446> 2022-09-03T07:31:27.724+0000 7f2cb4395600 4 set_mon_vals no callback set 2022-09-03T07:31:28.113 INFO:teuthology.orchestra.run.smithi087.stderr: -445> 2022-09-03T07:31:27.724+0000 7f2cb4395600 10 monclient: shutdown 2022-09-03T07:31:28.114 INFO:teuthology.orchestra.run.smithi087.stderr: -444> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 asok(0x55e2605c4000) init /var/run/ceph/c2-client.0.156266.asok 2022-09-03T07:31:28.114 INFO:teuthology.orchestra.run.smithi087.stderr: -443> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 asok(0x55e2605c4000) bind_and_listen /var/run/ceph/c2-client.0.156266.asok 2022-09-03T07:31:28.114 INFO:teuthology.orchestra.run.smithi087.stderr: -442> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command 0 hook 0x55e260581378 2022-09-03T07:31:28.115 INFO:teuthology.orchestra.run.smithi087.stderr: -441> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command version hook 0x55e260581378 2022-09-03T07:31:28.115 INFO:teuthology.orchestra.run.smithi087.stderr: -440> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command git_version hook 0x55e260581378 2022-09-03T07:31:28.115 INFO:teuthology.orchestra.run.smithi087.stderr: -439> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command help hook 0x55e260524bf0 2022-09-03T07:31:28.115 INFO:teuthology.orchestra.run.smithi087.stderr: -438> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command get_command_descriptions hook 0x55e260524b50 2022-09-03T07:31:28.116 INFO:teuthology.orchestra.run.smithi087.stderr: -437> 2022-09-03T07:31:27.725+0000 7f2cb4395600 10 monclient: get_monmap_and_config 2022-09-03T07:31:28.116 INFO:teuthology.orchestra.run.smithi087.stderr: -436> 2022-09-03T07:31:27.725+0000 7f2cb4395600 10 monclient: build_initial_monmap 2022-09-03T07:31:28.116 INFO:teuthology.orchestra.run.smithi087.stderr: -435> 2022-09-03T07:31:27.725+0000 7f2cb4395600 1 build_initial for_mkfs: 0 2022-09-03T07:31:28.117 INFO:teuthology.orchestra.run.smithi087.stderr: -434> 2022-09-03T07:31:27.725+0000 7f2cb4395600 10 monclient: monmap: 2022-09-03T07:31:28.117 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 0 2022-09-03T07:31:28.117 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 00000000-0000-0000-0000-000000000000 2022-09-03T07:31:28.117 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 0.000000 2022-09-03T07:31:28.118 INFO:teuthology.orchestra.run.smithi087.stderr:created 0.000000 2022-09-03T07:31:28.118 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 0 (unknown) 2022-09-03T07:31:28.118 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.119 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.noname-a 2022-09-03T07:31:28.119 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.119 INFO:teuthology.orchestra.run.smithi087.stderr: -433> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding auth protocol: cephx 2022-09-03T07:31:28.120 INFO:teuthology.orchestra.run.smithi087.stderr: -432> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding auth protocol: cephx 2022-09-03T07:31:28.120 INFO:teuthology.orchestra.run.smithi087.stderr: -431> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding auth protocol: cephx 2022-09-03T07:31:28.121 INFO:teuthology.orchestra.run.smithi087.stderr: -430> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.121 INFO:teuthology.orchestra.run.smithi087.stderr: -429> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.121 INFO:teuthology.orchestra.run.smithi087.stderr: -428> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.122 INFO:teuthology.orchestra.run.smithi087.stderr: -427> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.123 INFO:teuthology.orchestra.run.smithi087.stderr: -426> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.123 INFO:teuthology.orchestra.run.smithi087.stderr: -425> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.124 INFO:teuthology.orchestra.run.smithi087.stderr: -424> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.124 INFO:teuthology.orchestra.run.smithi087.stderr: -423> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.124 INFO:teuthology.orchestra.run.smithi087.stderr: -422> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.125 INFO:teuthology.orchestra.run.smithi087.stderr: -421> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.125 INFO:teuthology.orchestra.run.smithi087.stderr: -420> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.126 INFO:teuthology.orchestra.run.smithi087.stderr: -419> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.126 INFO:teuthology.orchestra.run.smithi087.stderr: -418> 2022-09-03T07:31:27.725+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.126 INFO:teuthology.orchestra.run.smithi087.stderr: -417> 2022-09-03T07:31:27.725+0000 7f2cb133f700 5 asok(0x55e2605c4000) entry start 2022-09-03T07:31:28.126 INFO:teuthology.orchestra.run.smithi087.stderr: -416> 2022-09-03T07:31:27.725+0000 7f2cb4395600 10 monclient: init 2022-09-03T07:31:28.127 INFO:teuthology.orchestra.run.smithi087.stderr: -415> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding auth protocol: cephx 2022-09-03T07:31:28.127 INFO:teuthology.orchestra.run.smithi087.stderr: -414> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding auth protocol: cephx 2022-09-03T07:31:28.127 INFO:teuthology.orchestra.run.smithi087.stderr: -413> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding auth protocol: cephx 2022-09-03T07:31:28.127 INFO:teuthology.orchestra.run.smithi087.stderr: -412> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: secure 2022-09-03T07:31:28.128 INFO:teuthology.orchestra.run.smithi087.stderr: -411> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: crc 2022-09-03T07:31:28.128 INFO:teuthology.orchestra.run.smithi087.stderr: -410> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: secure 2022-09-03T07:31:28.128 INFO:teuthology.orchestra.run.smithi087.stderr: -409> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: crc 2022-09-03T07:31:28.128 INFO:teuthology.orchestra.run.smithi087.stderr: -408> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: secure 2022-09-03T07:31:28.129 INFO:teuthology.orchestra.run.smithi087.stderr: -407> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: crc 2022-09-03T07:31:28.129 INFO:teuthology.orchestra.run.smithi087.stderr: -406> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: crc 2022-09-03T07:31:28.129 INFO:teuthology.orchestra.run.smithi087.stderr: -405> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: secure 2022-09-03T07:31:28.130 INFO:teuthology.orchestra.run.smithi087.stderr: -404> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: crc 2022-09-03T07:31:28.130 INFO:teuthology.orchestra.run.smithi087.stderr: -403> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: secure 2022-09-03T07:31:28.130 INFO:teuthology.orchestra.run.smithi087.stderr: -402> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: crc 2022-09-03T07:31:28.130 INFO:teuthology.orchestra.run.smithi087.stderr: -401> 2022-09-03T07:31:27.725+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a1a90) adding con mode: secure 2022-09-03T07:31:28.131 INFO:teuthology.orchestra.run.smithi087.stderr: -400> 2022-09-03T07:31:27.725+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.131 INFO:teuthology.orchestra.run.smithi087.stderr: -399> 2022-09-03T07:31:27.725+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.131 INFO:teuthology.orchestra.run.smithi087.stderr: -398> 2022-09-03T07:31:27.726+0000 7f2cb4395600 10 monclient: _reopen_session rank -1 2022-09-03T07:31:28.131 INFO:teuthology.orchestra.run.smithi087.stderr: -397> 2022-09-03T07:31:27.726+0000 7f2cb4395600 10 monclient: _add_conns ranks=[0] 2022-09-03T07:31:28.132 INFO:teuthology.orchestra.run.smithi087.stderr: -396> 2022-09-03T07:31:27.726+0000 7f2cb4395600 10 monclient(hunting): picked mon.noname-a con 0x55e260530800 addr [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] 2022-09-03T07:31:28.132 INFO:teuthology.orchestra.run.smithi087.stderr: -395> 2022-09-03T07:31:27.726+0000 7f2cb4395600 10 monclient(hunting): start opening mon connection 2022-09-03T07:31:28.132 INFO:teuthology.orchestra.run.smithi087.stderr: -394> 2022-09-03T07:31:27.726+0000 7f2cb4395600 10 monclient(hunting): _renew_subs 2022-09-03T07:31:28.133 INFO:teuthology.orchestra.run.smithi087.stderr: -393> 2022-09-03T07:31:27.726+0000 7f2cb4395600 10 monclient(hunting): authenticate will time out at 7208.146041s 2022-09-03T07:31:28.133 INFO:teuthology.orchestra.run.smithi087.stderr: -392> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient(hunting): get_auth_request con 0x55e260530800 auth_method 0 2022-09-03T07:31:28.133 INFO:teuthology.orchestra.run.smithi087.stderr: -391> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] 2022-09-03T07:31:28.133 INFO:teuthology.orchestra.run.smithi087.stderr: -390> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient(hunting): _init_auth method 2 2022-09-03T07:31:28.134 INFO:teuthology.orchestra.run.smithi087.stderr: -389> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient(hunting): _init_auth creating new auth 2022-09-03T07:31:28.134 INFO:teuthology.orchestra.run.smithi087.stderr: -388> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more payload 9 2022-09-03T07:31:28.134 INFO:teuthology.orchestra.run.smithi087.stderr: -387> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more payload_len 9 2022-09-03T07:31:28.134 INFO:teuthology.orchestra.run.smithi087.stderr: -386> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes 2022-09-03T07:31:28.135 INFO:teuthology.orchestra.run.smithi087.stderr: -385> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient(hunting): handle_auth_done global_id 6813 payload 274 2022-09-03T07:31:28.135 INFO:teuthology.orchestra.run.smithi087.stderr: -384> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient: _finish_hunting 0 2022-09-03T07:31:28.135 INFO:teuthology.orchestra.run.smithi087.stderr: -383> 2022-09-03T07:31:27.726+0000 7f2cb2341700 1 monclient: found mon.noname-a 2022-09-03T07:31:28.136 INFO:teuthology.orchestra.run.smithi087.stderr: -382> 2022-09-03T07:31:27.726+0000 7f2cb2341700 10 monclient: _send_mon_message to mon.noname-a at v2:172.21.15.87:3300/0 2022-09-03T07:31:28.136 INFO:teuthology.orchestra.run.smithi087.stderr: -381> 2022-09-03T07:31:27.726+0000 7f2cb033d700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.136 INFO:teuthology.orchestra.run.smithi087.stderr: -380> 2022-09-03T07:31:27.726+0000 7f2cb033d700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) 2022-09-03T07:31:28.137 INFO:teuthology.orchestra.run.smithi087.stderr: -379> 2022-09-03T07:31:27.726+0000 7f2cb033d700 10 monclient: dump: 2022-09-03T07:31:28.137 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.137 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.138 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.138 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.139 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.139 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.140 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.140 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.141 INFO:teuthology.orchestra.run.smithi087.stderr: -378> 2022-09-03T07:31:27.726+0000 7f2cb033d700 10 monclient: _finish_auth 0 2022-09-03T07:31:28.141 INFO:teuthology.orchestra.run.smithi087.stderr: -377> 2022-09-03T07:31:27.726+0000 7f2cb033d700 10 monclient: _check_auth_tickets 2022-09-03T07:31:28.142 INFO:teuthology.orchestra.run.smithi087.stderr: -376> 2022-09-03T07:31:27.726+0000 7f2cb4395600 5 monclient: authenticate success, global_id 6813 2022-09-03T07:31:28.142 INFO:teuthology.orchestra.run.smithi087.stderr: -375> 2022-09-03T07:31:27.726+0000 7f2cb033d700 10 monclient: handle_config config(0 keys) v1 2022-09-03T07:31:28.143 INFO:teuthology.orchestra.run.smithi087.stderr: -374> 2022-09-03T07:31:27.726+0000 7f2cb4395600 10 monclient: get_monmap_and_config success 2022-09-03T07:31:28.143 INFO:teuthology.orchestra.run.smithi087.stderr: -373> 2022-09-03T07:31:27.726+0000 7f2cb4395600 4 set_mon_vals no callback set 2022-09-03T07:31:28.144 INFO:teuthology.orchestra.run.smithi087.stderr: -372> 2022-09-03T07:31:27.726+0000 7f2cb033d700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.144 INFO:teuthology.orchestra.run.smithi087.stderr: -371> 2022-09-03T07:31:27.726+0000 7f2cb033d700 10 monclient: got monmap 1 from mon.a (according to old e1) 2022-09-03T07:31:28.145 INFO:teuthology.orchestra.run.smithi087.stderr: -370> 2022-09-03T07:31:27.726+0000 7f2cb033d700 10 monclient: dump: 2022-09-03T07:31:28.146 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.146 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.147 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.147 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.147 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.147 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.148 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.148 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.148 INFO:teuthology.orchestra.run.smithi087.stderr: -369> 2022-09-03T07:31:27.726+0000 7f2cb4395600 10 monclient: shutdown 2022-09-03T07:31:28.148 INFO:teuthology.orchestra.run.smithi087.stderr: -368> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient: build_initial_monmap 2022-09-03T07:31:28.149 INFO:teuthology.orchestra.run.smithi087.stderr: -367> 2022-09-03T07:31:27.727+0000 7f2cb4395600 1 build_initial for_mkfs: 0 2022-09-03T07:31:28.149 INFO:teuthology.orchestra.run.smithi087.stderr: -366> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient: monmap: 2022-09-03T07:31:28.149 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 0 2022-09-03T07:31:28.149 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 00000000-0000-0000-0000-000000000000 2022-09-03T07:31:28.150 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 0.000000 2022-09-03T07:31:28.150 INFO:teuthology.orchestra.run.smithi087.stderr:created 0.000000 2022-09-03T07:31:28.150 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 0 (unknown) 2022-09-03T07:31:28.151 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.151 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.noname-a 2022-09-03T07:31:28.151 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.151 INFO:teuthology.orchestra.run.smithi087.stderr: -365> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding auth protocol: cephx 2022-09-03T07:31:28.152 INFO:teuthology.orchestra.run.smithi087.stderr: -364> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding auth protocol: cephx 2022-09-03T07:31:28.152 INFO:teuthology.orchestra.run.smithi087.stderr: -363> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding auth protocol: cephx 2022-09-03T07:31:28.152 INFO:teuthology.orchestra.run.smithi087.stderr: -362> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.152 INFO:teuthology.orchestra.run.smithi087.stderr: -361> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.153 INFO:teuthology.orchestra.run.smithi087.stderr: -360> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.153 INFO:teuthology.orchestra.run.smithi087.stderr: -359> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.153 INFO:teuthology.orchestra.run.smithi087.stderr: -358> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.153 INFO:teuthology.orchestra.run.smithi087.stderr: -357> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.154 INFO:teuthology.orchestra.run.smithi087.stderr: -356> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.154 INFO:teuthology.orchestra.run.smithi087.stderr: -355> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.154 INFO:teuthology.orchestra.run.smithi087.stderr: -354> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.154 INFO:teuthology.orchestra.run.smithi087.stderr: -353> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.155 INFO:teuthology.orchestra.run.smithi087.stderr: -352> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: crc 2022-09-03T07:31:28.155 INFO:teuthology.orchestra.run.smithi087.stderr: -351> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e261290a40) adding con mode: secure 2022-09-03T07:31:28.155 INFO:teuthology.orchestra.run.smithi087.stderr: -350> 2022-09-03T07:31:27.727+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.155 INFO:teuthology.orchestra.run.smithi087.stderr: -349> 2022-09-03T07:31:27.727+0000 7f2cb4395600 1 librados: starting msgr at 2022-09-03T07:31:28.156 INFO:teuthology.orchestra.run.smithi087.stderr: -348> 2022-09-03T07:31:27.727+0000 7f2cb4395600 1 librados: starting objecter 2022-09-03T07:31:28.156 INFO:teuthology.orchestra.run.smithi087.stderr: -347> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command objecter_requests hook 0x55e260524b90 2022-09-03T07:31:28.156 INFO:teuthology.orchestra.run.smithi087.stderr: -346> 2022-09-03T07:31:27.727+0000 7f2cb4395600 1 librados: setting wanted keys 2022-09-03T07:31:28.156 INFO:teuthology.orchestra.run.smithi087.stderr: -345> 2022-09-03T07:31:27.727+0000 7f2cb4395600 1 librados: calling monclient init 2022-09-03T07:31:28.157 INFO:teuthology.orchestra.run.smithi087.stderr: -344> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient: init 2022-09-03T07:31:28.157 INFO:teuthology.orchestra.run.smithi087.stderr: -343> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding auth protocol: cephx 2022-09-03T07:31:28.157 INFO:teuthology.orchestra.run.smithi087.stderr: -342> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding auth protocol: cephx 2022-09-03T07:31:28.157 INFO:teuthology.orchestra.run.smithi087.stderr: -341> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding auth protocol: cephx 2022-09-03T07:31:28.158 INFO:teuthology.orchestra.run.smithi087.stderr: -340> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: secure 2022-09-03T07:31:28.158 INFO:teuthology.orchestra.run.smithi087.stderr: -339> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: crc 2022-09-03T07:31:28.158 INFO:teuthology.orchestra.run.smithi087.stderr: -338> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: secure 2022-09-03T07:31:28.158 INFO:teuthology.orchestra.run.smithi087.stderr: -337> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: crc 2022-09-03T07:31:28.159 INFO:teuthology.orchestra.run.smithi087.stderr: -336> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: secure 2022-09-03T07:31:28.159 INFO:teuthology.orchestra.run.smithi087.stderr: -335> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: crc 2022-09-03T07:31:28.159 INFO:teuthology.orchestra.run.smithi087.stderr: -334> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: crc 2022-09-03T07:31:28.159 INFO:teuthology.orchestra.run.smithi087.stderr: -333> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: secure 2022-09-03T07:31:28.160 INFO:teuthology.orchestra.run.smithi087.stderr: -332> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: crc 2022-09-03T07:31:28.160 INFO:teuthology.orchestra.run.smithi087.stderr: -331> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: secure 2022-09-03T07:31:28.160 INFO:teuthology.orchestra.run.smithi087.stderr: -330> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: crc 2022-09-03T07:31:28.160 INFO:teuthology.orchestra.run.smithi087.stderr: -329> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 AuthRegistry(0x55e26058c0d8) adding con mode: secure 2022-09-03T07:31:28.161 INFO:teuthology.orchestra.run.smithi087.stderr: -328> 2022-09-03T07:31:27.727+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.161 INFO:teuthology.orchestra.run.smithi087.stderr: -327> 2022-09-03T07:31:27.727+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.161 INFO:teuthology.orchestra.run.smithi087.stderr: -326> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient: _reopen_session rank -1 2022-09-03T07:31:28.161 INFO:teuthology.orchestra.run.smithi087.stderr: -325> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient: _add_conns ranks=[0] 2022-09-03T07:31:28.162 INFO:teuthology.orchestra.run.smithi087.stderr: -324> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient(hunting): picked mon.noname-a con 0x55e260530800 addr [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] 2022-09-03T07:31:28.162 INFO:teuthology.orchestra.run.smithi087.stderr: -323> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient(hunting): start opening mon connection 2022-09-03T07:31:28.162 INFO:teuthology.orchestra.run.smithi087.stderr: -322> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient(hunting): _renew_subs 2022-09-03T07:31:28.162 INFO:teuthology.orchestra.run.smithi087.stderr: -321> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient(hunting): authenticate will time out at 7208.147393s 2022-09-03T07:31:28.163 INFO:teuthology.orchestra.run.smithi087.stderr: -320> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient(hunting): get_auth_request con 0x55e260530800 auth_method 0 2022-09-03T07:31:28.163 INFO:teuthology.orchestra.run.smithi087.stderr: -319> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] 2022-09-03T07:31:28.163 INFO:teuthology.orchestra.run.smithi087.stderr: -318> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient(hunting): _init_auth method 2 2022-09-03T07:31:28.163 INFO:teuthology.orchestra.run.smithi087.stderr: -317> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient(hunting): _init_auth creating new auth 2022-09-03T07:31:28.164 INFO:teuthology.orchestra.run.smithi087.stderr: -316> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more payload 9 2022-09-03T07:31:28.164 INFO:teuthology.orchestra.run.smithi087.stderr: -315> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more payload_len 9 2022-09-03T07:31:28.164 INFO:teuthology.orchestra.run.smithi087.stderr: -314> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes 2022-09-03T07:31:28.164 INFO:teuthology.orchestra.run.smithi087.stderr: -313> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient(hunting): handle_auth_done global_id 6814 payload 824 2022-09-03T07:31:28.165 INFO:teuthology.orchestra.run.smithi087.stderr: -312> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient: _finish_hunting 0 2022-09-03T07:31:28.165 INFO:teuthology.orchestra.run.smithi087.stderr: -311> 2022-09-03T07:31:27.727+0000 7f2cb2341700 1 monclient: found mon.noname-a 2022-09-03T07:31:28.165 INFO:teuthology.orchestra.run.smithi087.stderr: -310> 2022-09-03T07:31:27.727+0000 7f2cb2341700 10 monclient: _send_mon_message to mon.noname-a at v2:172.21.15.87:3300/0 2022-09-03T07:31:28.165 INFO:teuthology.orchestra.run.smithi087.stderr: -309> 2022-09-03T07:31:27.727+0000 7f2caeb3a700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.166 INFO:teuthology.orchestra.run.smithi087.stderr: -308> 2022-09-03T07:31:27.727+0000 7f2caeb3a700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) 2022-09-03T07:31:28.166 INFO:teuthology.orchestra.run.smithi087.stderr: -307> 2022-09-03T07:31:27.727+0000 7f2caeb3a700 10 monclient: dump: 2022-09-03T07:31:28.166 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.166 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.167 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.167 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.167 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.167 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.168 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.168 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.168 INFO:teuthology.orchestra.run.smithi087.stderr: -306> 2022-09-03T07:31:27.727+0000 7f2caeb3a700 10 monclient: _finish_auth 0 2022-09-03T07:31:28.168 INFO:teuthology.orchestra.run.smithi087.stderr: -305> 2022-09-03T07:31:27.727+0000 7f2caeb3a700 10 monclient: _check_auth_tickets 2022-09-03T07:31:28.169 INFO:teuthology.orchestra.run.smithi087.stderr: -304> 2022-09-03T07:31:27.727+0000 7f2caeb3a700 10 monclient: handle_config config(0 keys) v1 2022-09-03T07:31:28.169 INFO:teuthology.orchestra.run.smithi087.stderr: -303> 2022-09-03T07:31:27.727+0000 7f2caeb3a700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.169 INFO:teuthology.orchestra.run.smithi087.stderr: -302> 2022-09-03T07:31:27.727+0000 7f2caeb3a700 10 monclient: got monmap 1 from mon.a (according to old e1) 2022-09-03T07:31:28.169 INFO:teuthology.orchestra.run.smithi087.stderr: -301> 2022-09-03T07:31:27.727+0000 7f2cb033d700 4 set_mon_vals no callback set 2022-09-03T07:31:28.170 INFO:teuthology.orchestra.run.smithi087.stderr: -300> 2022-09-03T07:31:27.727+0000 7f2caeb3a700 10 monclient: dump: 2022-09-03T07:31:28.170 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.170 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.170 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.171 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.171 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.171 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.171 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.172 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.172 INFO:teuthology.orchestra.run.smithi087.stderr: -299> 2022-09-03T07:31:27.727+0000 7f2cb4395600 5 monclient: authenticate success, global_id 6814 2022-09-03T07:31:28.172 INFO:teuthology.orchestra.run.smithi087.stderr: -298> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient: _renew_subs 2022-09-03T07:31:28.173 INFO:teuthology.orchestra.run.smithi087.stderr: -297> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient: _send_mon_message to mon.a at v2:172.21.15.87:3300/0 2022-09-03T07:31:28.173 INFO:teuthology.orchestra.run.smithi087.stderr: -296> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient: _renew_subs 2022-09-03T07:31:28.173 INFO:teuthology.orchestra.run.smithi087.stderr: -295> 2022-09-03T07:31:27.727+0000 7f2cb4395600 10 monclient: _send_mon_message to mon.a at v2:172.21.15.87:3300/0 2022-09-03T07:31:28.174 INFO:teuthology.orchestra.run.smithi087.stderr: -294> 2022-09-03T07:31:27.727+0000 7f2cb4395600 1 librados: init done 2022-09-03T07:31:28.174 INFO:teuthology.orchestra.run.smithi087.stderr: -293> 2022-09-03T07:31:27.728+0000 7f2caeb3a700 4 mgrc handle_mgr_map Got map version 5 2022-09-03T07:31:28.174 INFO:teuthology.orchestra.run.smithi087.stderr: -292> 2022-09-03T07:31:27.728+0000 7f2caeb3a700 4 mgrc handle_mgr_map Active mgr is now [v2:172.21.15.87:6824/103775,v1:172.21.15.87:6825/103775] 2022-09-03T07:31:28.175 INFO:teuthology.orchestra.run.smithi087.stderr: -291> 2022-09-03T07:31:27.728+0000 7f2caeb3a700 4 mgrc reconnect Starting new session with [v2:172.21.15.87:6824/103775,v1:172.21.15.87:6825/103775] 2022-09-03T07:31:28.175 INFO:teuthology.orchestra.run.smithi087.stderr: -290> 2022-09-03T07:31:27.728+0000 7f2cb1b40700 10 monclient: get_auth_request con 0x55e260531400 auth_method 0 2022-09-03T07:31:28.175 INFO:teuthology.orchestra.run.smithi087.stderr: -289> 2022-09-03T07:31:27.729+0000 7f2cb2b42700 10 monclient: get_auth_request con 0x55e260530c00 auth_method 0 2022-09-03T07:31:28.176 INFO:teuthology.orchestra.run.smithi087.stderr: -288> 2022-09-03T07:31:27.729+0000 7f2cb2341700 10 monclient: get_auth_request con 0x55e260531c00 auth_method 0 2022-09-03T07:31:28.176 INFO:teuthology.orchestra.run.smithi087.stderr: -287> 2022-09-03T07:31:27.730+0000 7f2cb1b40700 10 monclient: get_auth_request con 0x55e260856000 auth_method 0 2022-09-03T07:31:28.176 INFO:teuthology.orchestra.run.smithi087.stderr: -286> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding auth protocol: cephx 2022-09-03T07:31:28.176 INFO:teuthology.orchestra.run.smithi087.stderr: -285> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding auth protocol: cephx 2022-09-03T07:31:28.177 INFO:teuthology.orchestra.run.smithi087.stderr: -284> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding auth protocol: cephx 2022-09-03T07:31:28.177 INFO:teuthology.orchestra.run.smithi087.stderr: -283> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: secure 2022-09-03T07:31:28.177 INFO:teuthology.orchestra.run.smithi087.stderr: -282> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: crc 2022-09-03T07:31:28.177 INFO:teuthology.orchestra.run.smithi087.stderr: -281> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: secure 2022-09-03T07:31:28.178 INFO:teuthology.orchestra.run.smithi087.stderr: -280> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: crc 2022-09-03T07:31:28.178 INFO:teuthology.orchestra.run.smithi087.stderr: -279> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: secure 2022-09-03T07:31:28.178 INFO:teuthology.orchestra.run.smithi087.stderr: -278> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: crc 2022-09-03T07:31:28.178 INFO:teuthology.orchestra.run.smithi087.stderr: -277> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: crc 2022-09-03T07:31:28.179 INFO:teuthology.orchestra.run.smithi087.stderr: -276> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: secure 2022-09-03T07:31:28.179 INFO:teuthology.orchestra.run.smithi087.stderr: -275> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: crc 2022-09-03T07:31:28.179 INFO:teuthology.orchestra.run.smithi087.stderr: -274> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: secure 2022-09-03T07:31:28.179 INFO:teuthology.orchestra.run.smithi087.stderr: -273> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: crc 2022-09-03T07:31:28.180 INFO:teuthology.orchestra.run.smithi087.stderr: -272> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a24a0) adding con mode: secure 2022-09-03T07:31:28.180 INFO:teuthology.orchestra.run.smithi087.stderr: -271> 2022-09-03T07:31:27.763+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.180 INFO:teuthology.orchestra.run.smithi087.stderr: -270> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command cache list hook 0x55e260524b60 2022-09-03T07:31:28.180 INFO:teuthology.orchestra.run.smithi087.stderr: -269> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command cache inspect hook 0x55e260524b60 2022-09-03T07:31:28.181 INFO:teuthology.orchestra.run.smithi087.stderr: -268> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command cache erase hook 0x55e260524b60 2022-09-03T07:31:28.181 INFO:teuthology.orchestra.run.smithi087.stderr: -267> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command cache zap hook 0x55e260524b60 2022-09-03T07:31:28.181 INFO:teuthology.orchestra.run.smithi087.stderr: -266> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient: get_monmap_and_config 2022-09-03T07:31:28.181 INFO:teuthology.orchestra.run.smithi087.stderr: -265> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient: build_initial_monmap 2022-09-03T07:31:28.182 INFO:teuthology.orchestra.run.smithi087.stderr: -264> 2022-09-03T07:31:27.763+0000 7f2cb4395600 1 build_initial for_mkfs: 0 2022-09-03T07:31:28.182 INFO:teuthology.orchestra.run.smithi087.stderr: -263> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient: monmap: 2022-09-03T07:31:28.182 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 0 2022-09-03T07:31:28.182 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 00000000-0000-0000-0000-000000000000 2022-09-03T07:31:28.183 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 0.000000 2022-09-03T07:31:28.183 INFO:teuthology.orchestra.run.smithi087.stderr:created 0.000000 2022-09-03T07:31:28.183 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 0 (unknown) 2022-09-03T07:31:28.183 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.184 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.noname-a 2022-09-03T07:31:28.184 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.184 INFO:teuthology.orchestra.run.smithi087.stderr: -262> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding auth protocol: cephx 2022-09-03T07:31:28.184 INFO:teuthology.orchestra.run.smithi087.stderr: -261> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding auth protocol: cephx 2022-09-03T07:31:28.185 INFO:teuthology.orchestra.run.smithi087.stderr: -260> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding auth protocol: cephx 2022-09-03T07:31:28.185 INFO:teuthology.orchestra.run.smithi087.stderr: -259> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.185 INFO:teuthology.orchestra.run.smithi087.stderr: -258> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.185 INFO:teuthology.orchestra.run.smithi087.stderr: -257> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.186 INFO:teuthology.orchestra.run.smithi087.stderr: -256> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.186 INFO:teuthology.orchestra.run.smithi087.stderr: -255> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.186 INFO:teuthology.orchestra.run.smithi087.stderr: -254> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.186 INFO:teuthology.orchestra.run.smithi087.stderr: -253> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.187 INFO:teuthology.orchestra.run.smithi087.stderr: -252> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.187 INFO:teuthology.orchestra.run.smithi087.stderr: -251> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.187 INFO:teuthology.orchestra.run.smithi087.stderr: -250> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.187 INFO:teuthology.orchestra.run.smithi087.stderr: -249> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.188 INFO:teuthology.orchestra.run.smithi087.stderr: -248> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.188 INFO:teuthology.orchestra.run.smithi087.stderr: -247> 2022-09-03T07:31:27.763+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.188 INFO:teuthology.orchestra.run.smithi087.stderr: -246> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient: init 2022-09-03T07:31:28.188 INFO:teuthology.orchestra.run.smithi087.stderr: -245> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding auth protocol: cephx 2022-09-03T07:31:28.189 INFO:teuthology.orchestra.run.smithi087.stderr: -244> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding auth protocol: cephx 2022-09-03T07:31:28.189 INFO:teuthology.orchestra.run.smithi087.stderr: -243> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding auth protocol: cephx 2022-09-03T07:31:28.189 INFO:teuthology.orchestra.run.smithi087.stderr: -242> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: secure 2022-09-03T07:31:28.189 INFO:teuthology.orchestra.run.smithi087.stderr: -241> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: crc 2022-09-03T07:31:28.190 INFO:teuthology.orchestra.run.smithi087.stderr: -240> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: secure 2022-09-03T07:31:28.190 INFO:teuthology.orchestra.run.smithi087.stderr: -239> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: crc 2022-09-03T07:31:28.190 INFO:teuthology.orchestra.run.smithi087.stderr: -238> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: secure 2022-09-03T07:31:28.191 INFO:teuthology.orchestra.run.smithi087.stderr: -237> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: crc 2022-09-03T07:31:28.191 INFO:teuthology.orchestra.run.smithi087.stderr: -236> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: crc 2022-09-03T07:31:28.191 INFO:teuthology.orchestra.run.smithi087.stderr: -235> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: secure 2022-09-03T07:31:28.191 INFO:teuthology.orchestra.run.smithi087.stderr: -234> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: crc 2022-09-03T07:31:28.192 INFO:teuthology.orchestra.run.smithi087.stderr: -233> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: secure 2022-09-03T07:31:28.192 INFO:teuthology.orchestra.run.smithi087.stderr: -232> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: crc 2022-09-03T07:31:28.192 INFO:teuthology.orchestra.run.smithi087.stderr: -231> 2022-09-03T07:31:27.763+0000 7f2cb4395600 5 AuthRegistry(0x7fff733a20d0) adding con mode: secure 2022-09-03T07:31:28.192 INFO:teuthology.orchestra.run.smithi087.stderr: -230> 2022-09-03T07:31:27.763+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.193 INFO:teuthology.orchestra.run.smithi087.stderr: -229> 2022-09-03T07:31:27.763+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.193 INFO:teuthology.orchestra.run.smithi087.stderr: -228> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient: _reopen_session rank -1 2022-09-03T07:31:28.193 INFO:teuthology.orchestra.run.smithi087.stderr: -227> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient: _add_conns ranks=[0] 2022-09-03T07:31:28.193 INFO:teuthology.orchestra.run.smithi087.stderr: -226> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient(hunting): picked mon.noname-a con 0x55e260856400 addr [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] 2022-09-03T07:31:28.194 INFO:teuthology.orchestra.run.smithi087.stderr: -225> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient(hunting): start opening mon connection 2022-09-03T07:31:28.194 INFO:teuthology.orchestra.run.smithi087.stderr: -224> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient(hunting): _renew_subs 2022-09-03T07:31:28.195 INFO:teuthology.orchestra.run.smithi087.stderr: -223> 2022-09-03T07:31:27.763+0000 7f2cb4395600 10 monclient(hunting): authenticate will time out at 7208.183637s 2022-09-03T07:31:28.195 INFO:teuthology.orchestra.run.smithi087.stderr: -222> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient(hunting): get_auth_request con 0x55e260856400 auth_method 0 2022-09-03T07:31:28.195 INFO:teuthology.orchestra.run.smithi087.stderr: -221> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] 2022-09-03T07:31:28.196 INFO:teuthology.orchestra.run.smithi087.stderr: -220> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient(hunting): _init_auth method 2 2022-09-03T07:31:28.196 INFO:teuthology.orchestra.run.smithi087.stderr: -219> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient(hunting): _init_auth creating new auth 2022-09-03T07:31:28.196 INFO:teuthology.orchestra.run.smithi087.stderr: -218> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more payload 9 2022-09-03T07:31:28.197 INFO:teuthology.orchestra.run.smithi087.stderr: -217> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more payload_len 9 2022-09-03T07:31:28.197 INFO:teuthology.orchestra.run.smithi087.stderr: -216> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes 2022-09-03T07:31:28.197 INFO:teuthology.orchestra.run.smithi087.stderr: -215> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient(hunting): handle_auth_done global_id 6815 payload 274 2022-09-03T07:31:28.197 INFO:teuthology.orchestra.run.smithi087.stderr: -214> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient: _finish_hunting 0 2022-09-03T07:31:28.198 INFO:teuthology.orchestra.run.smithi087.stderr: -213> 2022-09-03T07:31:27.763+0000 7f2cb2341700 1 monclient: found mon.noname-a 2022-09-03T07:31:28.198 INFO:teuthology.orchestra.run.smithi087.stderr: -212> 2022-09-03T07:31:27.763+0000 7f2cb2341700 10 monclient: _send_mon_message to mon.noname-a at v2:172.21.15.87:3300/0 2022-09-03T07:31:28.198 INFO:teuthology.orchestra.run.smithi087.stderr: -211> 2022-09-03T07:31:27.764+0000 7f2c9c315700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.198 INFO:teuthology.orchestra.run.smithi087.stderr: -210> 2022-09-03T07:31:27.764+0000 7f2c9c315700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) 2022-09-03T07:31:28.199 INFO:teuthology.orchestra.run.smithi087.stderr: -209> 2022-09-03T07:31:27.764+0000 7f2c9c315700 10 monclient: dump: 2022-09-03T07:31:28.199 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.199 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.200 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.200 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.200 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.200 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.201 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.201 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.201 INFO:teuthology.orchestra.run.smithi087.stderr: -208> 2022-09-03T07:31:27.764+0000 7f2c9c315700 10 monclient: _finish_auth 0 2022-09-03T07:31:28.201 INFO:teuthology.orchestra.run.smithi087.stderr: -207> 2022-09-03T07:31:27.764+0000 7f2c9c315700 10 monclient: _check_auth_tickets 2022-09-03T07:31:28.202 INFO:teuthology.orchestra.run.smithi087.stderr: -206> 2022-09-03T07:31:27.764+0000 7f2cb4395600 5 monclient: authenticate success, global_id 6815 2022-09-03T07:31:28.202 INFO:teuthology.orchestra.run.smithi087.stderr: -205> 2022-09-03T07:31:27.764+0000 7f2c9c315700 10 monclient: handle_config config(0 keys) v1 2022-09-03T07:31:28.202 INFO:teuthology.orchestra.run.smithi087.stderr: -204> 2022-09-03T07:31:27.764+0000 7f2c9c315700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.202 INFO:teuthology.orchestra.run.smithi087.stderr: -203> 2022-09-03T07:31:27.764+0000 7f2c9c315700 10 monclient: got monmap 1 from mon.a (according to old e1) 2022-09-03T07:31:28.203 INFO:teuthology.orchestra.run.smithi087.stderr: -202> 2022-09-03T07:31:27.764+0000 7f2c9c315700 10 monclient: dump: 2022-09-03T07:31:28.203 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.203 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.203 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.204 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.204 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.204 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.204 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.205 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.205 INFO:teuthology.orchestra.run.smithi087.stderr: -201> 2022-09-03T07:31:27.764+0000 7f2cb4395600 10 monclient: get_monmap_and_config success 2022-09-03T07:31:28.205 INFO:teuthology.orchestra.run.smithi087.stderr: -200> 2022-09-03T07:31:27.764+0000 7f2cb4395600 4 set_mon_vals no callback set 2022-09-03T07:31:28.205 INFO:teuthology.orchestra.run.smithi087.stderr: -199> 2022-09-03T07:31:27.764+0000 7f2cb4395600 10 monclient: shutdown 2022-09-03T07:31:28.206 INFO:teuthology.orchestra.run.smithi087.stderr: -198> 2022-09-03T07:31:27.764+0000 7f2cb4395600 10 monclient: build_initial_monmap 2022-09-03T07:31:28.206 INFO:teuthology.orchestra.run.smithi087.stderr: -197> 2022-09-03T07:31:27.764+0000 7f2cb4395600 1 build_initial for_mkfs: 0 2022-09-03T07:31:28.206 INFO:teuthology.orchestra.run.smithi087.stderr: -196> 2022-09-03T07:31:27.765+0000 7f2cb4395600 10 monclient: monmap: 2022-09-03T07:31:28.206 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 0 2022-09-03T07:31:28.207 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 00000000-0000-0000-0000-000000000000 2022-09-03T07:31:28.207 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 0.000000 2022-09-03T07:31:28.207 INFO:teuthology.orchestra.run.smithi087.stderr:created 0.000000 2022-09-03T07:31:28.207 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 0 (unknown) 2022-09-03T07:31:28.208 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.208 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.noname-a 2022-09-03T07:31:28.209 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.209 INFO:teuthology.orchestra.run.smithi087.stderr: -195> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding auth protocol: cephx 2022-09-03T07:31:28.209 INFO:teuthology.orchestra.run.smithi087.stderr: -194> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding auth protocol: cephx 2022-09-03T07:31:28.209 INFO:teuthology.orchestra.run.smithi087.stderr: -193> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding auth protocol: cephx 2022-09-03T07:31:28.210 INFO:teuthology.orchestra.run.smithi087.stderr: -192> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.210 INFO:teuthology.orchestra.run.smithi087.stderr: -191> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.210 INFO:teuthology.orchestra.run.smithi087.stderr: -190> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.210 INFO:teuthology.orchestra.run.smithi087.stderr: -189> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.211 INFO:teuthology.orchestra.run.smithi087.stderr: -188> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.211 INFO:teuthology.orchestra.run.smithi087.stderr: -187> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.211 INFO:teuthology.orchestra.run.smithi087.stderr: -186> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.211 INFO:teuthology.orchestra.run.smithi087.stderr: -185> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.212 INFO:teuthology.orchestra.run.smithi087.stderr: -184> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.212 INFO:teuthology.orchestra.run.smithi087.stderr: -183> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.212 INFO:teuthology.orchestra.run.smithi087.stderr: -182> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: crc 2022-09-03T07:31:28.212 INFO:teuthology.orchestra.run.smithi087.stderr: -181> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e261290140) adding con mode: secure 2022-09-03T07:31:28.213 INFO:teuthology.orchestra.run.smithi087.stderr: -180> 2022-09-03T07:31:27.765+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.213 INFO:teuthology.orchestra.run.smithi087.stderr: -179> 2022-09-03T07:31:27.765+0000 7f2cb4395600 1 librados: starting msgr at 2022-09-03T07:31:28.213 INFO:teuthology.orchestra.run.smithi087.stderr: -178> 2022-09-03T07:31:27.765+0000 7f2cb4395600 1 librados: starting objecter 2022-09-03T07:31:28.213 INFO:teuthology.orchestra.run.smithi087.stderr: -177> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command objecter_requests cmddesc objecter_requests hook 0x55e260524730 EEXIST 2022-09-03T07:31:28.214 INFO:teuthology.orchestra.run.smithi087.stderr: -176> 2022-09-03T07:31:27.765+0000 7f2cb4395600 1 librados: setting wanted keys 2022-09-03T07:31:28.214 INFO:teuthology.orchestra.run.smithi087.stderr: -175> 2022-09-03T07:31:27.765+0000 7f2cb4395600 1 librados: calling monclient init 2022-09-03T07:31:28.214 INFO:teuthology.orchestra.run.smithi087.stderr: -174> 2022-09-03T07:31:27.765+0000 7f2cb4395600 10 monclient: init 2022-09-03T07:31:28.214 INFO:teuthology.orchestra.run.smithi087.stderr: -173> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding auth protocol: cephx 2022-09-03T07:31:28.215 INFO:teuthology.orchestra.run.smithi087.stderr: -172> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding auth protocol: cephx 2022-09-03T07:31:28.215 INFO:teuthology.orchestra.run.smithi087.stderr: -171> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding auth protocol: cephx 2022-09-03T07:31:28.215 INFO:teuthology.orchestra.run.smithi087.stderr: -170> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: secure 2022-09-03T07:31:28.215 INFO:teuthology.orchestra.run.smithi087.stderr: -169> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: crc 2022-09-03T07:31:28.216 INFO:teuthology.orchestra.run.smithi087.stderr: -168> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: secure 2022-09-03T07:31:28.216 INFO:teuthology.orchestra.run.smithi087.stderr: -167> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: crc 2022-09-03T07:31:28.216 INFO:teuthology.orchestra.run.smithi087.stderr: -166> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: secure 2022-09-03T07:31:28.216 INFO:teuthology.orchestra.run.smithi087.stderr: -165> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: crc 2022-09-03T07:31:28.217 INFO:teuthology.orchestra.run.smithi087.stderr: -164> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: crc 2022-09-03T07:31:28.217 INFO:teuthology.orchestra.run.smithi087.stderr: -163> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: secure 2022-09-03T07:31:28.217 INFO:teuthology.orchestra.run.smithi087.stderr: -162> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: crc 2022-09-03T07:31:28.217 INFO:teuthology.orchestra.run.smithi087.stderr: -161> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: secure 2022-09-03T07:31:28.218 INFO:teuthology.orchestra.run.smithi087.stderr: -160> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: crc 2022-09-03T07:31:28.218 INFO:teuthology.orchestra.run.smithi087.stderr: -159> 2022-09-03T07:31:27.765+0000 7f2cb4395600 5 AuthRegistry(0x55e2606850d8) adding con mode: secure 2022-09-03T07:31:28.218 INFO:teuthology.orchestra.run.smithi087.stderr: -158> 2022-09-03T07:31:27.765+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.218 INFO:teuthology.orchestra.run.smithi087.stderr: -157> 2022-09-03T07:31:27.765+0000 7f2cb4395600 2 auth: KeyRing::load: loaded key file /etc/ceph/c2.client.0.keyring 2022-09-03T07:31:28.219 INFO:teuthology.orchestra.run.smithi087.stderr: -156> 2022-09-03T07:31:27.765+0000 7f2cb4395600 10 monclient: _reopen_session rank -1 2022-09-03T07:31:28.219 INFO:teuthology.orchestra.run.smithi087.stderr: -155> 2022-09-03T07:31:27.765+0000 7f2cb4395600 10 monclient: _add_conns ranks=[0] 2022-09-03T07:31:28.219 INFO:teuthology.orchestra.run.smithi087.stderr: -154> 2022-09-03T07:31:27.765+0000 7f2cb4395600 10 monclient(hunting): picked mon.noname-a con 0x55e260856400 addr [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] 2022-09-03T07:31:28.219 INFO:teuthology.orchestra.run.smithi087.stderr: -153> 2022-09-03T07:31:27.765+0000 7f2cb4395600 10 monclient(hunting): start opening mon connection 2022-09-03T07:31:28.220 INFO:teuthology.orchestra.run.smithi087.stderr: -152> 2022-09-03T07:31:27.765+0000 7f2cb4395600 10 monclient(hunting): _renew_subs 2022-09-03T07:31:28.220 INFO:teuthology.orchestra.run.smithi087.stderr: -151> 2022-09-03T07:31:27.765+0000 7f2cb4395600 10 monclient(hunting): authenticate will time out at 7208.185536s 2022-09-03T07:31:28.220 INFO:teuthology.orchestra.run.smithi087.stderr: -150> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient(hunting): get_auth_request con 0x55e260856400 auth_method 0 2022-09-03T07:31:28.220 INFO:teuthology.orchestra.run.smithi087.stderr: -149> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] 2022-09-03T07:31:28.221 INFO:teuthology.orchestra.run.smithi087.stderr: -148> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient(hunting): _init_auth method 2 2022-09-03T07:31:28.221 INFO:teuthology.orchestra.run.smithi087.stderr: -147> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient(hunting): _init_auth creating new auth 2022-09-03T07:31:28.221 INFO:teuthology.orchestra.run.smithi087.stderr: -146> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more payload 9 2022-09-03T07:31:28.222 INFO:teuthology.orchestra.run.smithi087.stderr: -145> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more payload_len 9 2022-09-03T07:31:28.222 INFO:teuthology.orchestra.run.smithi087.stderr: -144> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes 2022-09-03T07:31:28.222 INFO:teuthology.orchestra.run.smithi087.stderr: -143> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient(hunting): handle_auth_done global_id 6816 payload 824 2022-09-03T07:31:28.222 INFO:teuthology.orchestra.run.smithi087.stderr: -142> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient: _finish_hunting 0 2022-09-03T07:31:28.223 INFO:teuthology.orchestra.run.smithi087.stderr: -141> 2022-09-03T07:31:27.765+0000 7f2cb2341700 1 monclient: found mon.noname-a 2022-09-03T07:31:28.223 INFO:teuthology.orchestra.run.smithi087.stderr: -140> 2022-09-03T07:31:27.765+0000 7f2cb2341700 10 monclient: _send_mon_message to mon.noname-a at v2:172.21.15.87:3300/0 2022-09-03T07:31:28.223 INFO:teuthology.orchestra.run.smithi087.stderr: -139> 2022-09-03T07:31:27.765+0000 7f2c9ab12700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.223 INFO:teuthology.orchestra.run.smithi087.stderr: -138> 2022-09-03T07:31:27.765+0000 7f2c9ab12700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) 2022-09-03T07:31:28.224 INFO:teuthology.orchestra.run.smithi087.stderr: -137> 2022-09-03T07:31:27.765+0000 7f2c9ab12700 10 monclient: dump: 2022-09-03T07:31:28.224 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.224 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.224 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.225 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.225 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.225 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.225 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.226 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.226 INFO:teuthology.orchestra.run.smithi087.stderr: -136> 2022-09-03T07:31:27.766+0000 7f2c9ab12700 10 monclient: _finish_auth 0 2022-09-03T07:31:28.226 INFO:teuthology.orchestra.run.smithi087.stderr: -135> 2022-09-03T07:31:27.766+0000 7f2c9ab12700 10 monclient: _check_auth_tickets 2022-09-03T07:31:28.226 INFO:teuthology.orchestra.run.smithi087.stderr: -134> 2022-09-03T07:31:27.766+0000 7f2cb4395600 5 monclient: authenticate success, global_id 6816 2022-09-03T07:31:28.227 INFO:teuthology.orchestra.run.smithi087.stderr: -133> 2022-09-03T07:31:27.766+0000 7f2c9ab12700 10 monclient: handle_config config(0 keys) v1 2022-09-03T07:31:28.227 INFO:teuthology.orchestra.run.smithi087.stderr: -132> 2022-09-03T07:31:27.766+0000 7f2c9ab12700 10 monclient: handle_monmap mon_map magic: 0 v1 2022-09-03T07:31:28.227 INFO:teuthology.orchestra.run.smithi087.stderr: -131> 2022-09-03T07:31:27.766+0000 7f2c9ab12700 10 monclient: got monmap 1 from mon.a (according to old e1) 2022-09-03T07:31:28.227 INFO:teuthology.orchestra.run.smithi087.stderr: -130> 2022-09-03T07:31:27.766+0000 7f2c9c315700 4 set_mon_vals no callback set 2022-09-03T07:31:28.228 INFO:teuthology.orchestra.run.smithi087.stderr: -129> 2022-09-03T07:31:27.766+0000 7f2c9ab12700 10 monclient: dump: 2022-09-03T07:31:28.228 INFO:teuthology.orchestra.run.smithi087.stderr:epoch 1 2022-09-03T07:31:28.228 INFO:teuthology.orchestra.run.smithi087.stderr:fsid 716c39f3-2c98-4096-aa80-186bfe26000f 2022-09-03T07:31:28.228 INFO:teuthology.orchestra.run.smithi087.stderr:last_changed 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.229 INFO:teuthology.orchestra.run.smithi087.stderr:created 2022-09-03T05:47:47.228157+0000 2022-09-03T07:31:28.229 INFO:teuthology.orchestra.run.smithi087.stderr:min_mon_release 17 (quincy) 2022-09-03T07:31:28.229 INFO:teuthology.orchestra.run.smithi087.stderr:election_strategy: 1 2022-09-03T07:31:28.229 INFO:teuthology.orchestra.run.smithi087.stderr:0: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.a 2022-09-03T07:31:28.230 INFO:teuthology.orchestra.run.smithi087.stderr: 2022-09-03T07:31:28.230 INFO:teuthology.orchestra.run.smithi087.stderr: -128> 2022-09-03T07:31:27.766+0000 7f2cb4395600 10 monclient: _renew_subs 2022-09-03T07:31:28.230 INFO:teuthology.orchestra.run.smithi087.stderr: -127> 2022-09-03T07:31:27.766+0000 7f2cb4395600 10 monclient: _send_mon_message to mon.a at v2:172.21.15.87:3300/0 2022-09-03T07:31:28.231 INFO:teuthology.orchestra.run.smithi087.stderr: -126> 2022-09-03T07:31:27.766+0000 7f2cb4395600 10 monclient: _renew_subs 2022-09-03T07:31:28.231 INFO:teuthology.orchestra.run.smithi087.stderr: -125> 2022-09-03T07:31:27.766+0000 7f2cb4395600 10 monclient: _send_mon_message to mon.a at v2:172.21.15.87:3300/0 2022-09-03T07:31:28.231 INFO:teuthology.orchestra.run.smithi087.stderr: -124> 2022-09-03T07:31:27.766+0000 7f2cb4395600 1 librados: init done 2022-09-03T07:31:28.231 INFO:teuthology.orchestra.run.smithi087.stderr: -123> 2022-09-03T07:31:27.766+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command cr dump hook 0x55e2606a0dd8 2022-09-03T07:31:28.232 INFO:teuthology.orchestra.run.smithi087.stderr: -122> 2022-09-03T07:31:27.766+0000 7f2c9ab12700 4 mgrc handle_mgr_map Got map version 5 2022-09-03T07:31:28.232 INFO:teuthology.orchestra.run.smithi087.stderr: -121> 2022-09-03T07:31:27.766+0000 7f2c9ab12700 4 mgrc handle_mgr_map Active mgr is now [v2:172.21.15.87:6824/103775,v1:172.21.15.87:6825/103775] 2022-09-03T07:31:28.232 INFO:teuthology.orchestra.run.smithi087.stderr: -120> 2022-09-03T07:31:27.766+0000 7f2c9ab12700 4 mgrc reconnect Starting new session with [v2:172.21.15.87:6824/103775,v1:172.21.15.87:6825/103775] 2022-09-03T07:31:28.233 INFO:teuthology.orchestra.run.smithi087.stderr: -119> 2022-09-03T07:31:27.766+0000 7f2cb1b40700 10 monclient: get_auth_request con 0x55e260857800 auth_method 0 2022-09-03T07:31:28.233 INFO:teuthology.orchestra.run.smithi087.stderr: -118> 2022-09-03T07:31:27.767+0000 7f2cb2b42700 10 monclient: get_auth_request con 0x55e260899c00 auth_method 0 2022-09-03T07:31:28.242 INFO:teuthology.orchestra.run.smithi087.stderr: -117> 2022-09-03T07:31:27.769+0000 7f2cb2341700 10 monclient: get_auth_request con 0x55e26095a000 auth_method 0 2022-09-03T07:31:28.242 INFO:teuthology.orchestra.run.smithi087.stderr: -116> 2022-09-03T07:31:27.775+0000 7f2cb1b40700 10 monclient: get_auth_request con 0x55e26095ac00 auth_method 0 2022-09-03T07:31:28.243 INFO:teuthology.orchestra.run.smithi087.stderr: -115> 2022-09-03T07:31:27.834+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command sync trace show hook 0x55e2605596c0 2022-09-03T07:31:28.243 INFO:teuthology.orchestra.run.smithi087.stderr: -114> 2022-09-03T07:31:27.834+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command sync trace history hook 0x55e2605596c0 2022-09-03T07:31:28.244 INFO:teuthology.orchestra.run.smithi087.stderr: -113> 2022-09-03T07:31:27.834+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command sync trace active hook 0x55e2605596c0 2022-09-03T07:31:28.244 INFO:teuthology.orchestra.run.smithi087.stderr: -112> 2022-09-03T07:31:27.834+0000 7f2cb4395600 5 asok(0x55e2605c4000) register_command sync trace active_short hook 0x55e2605596c0 2022-09-03T07:31:28.245 INFO:teuthology.orchestra.run.smithi087.stderr: -111> 2022-09-03T07:31:27.837+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: sync status for source bucket: init. lease is: not taken. stop indications is: 0 2022-09-03T07:31:28.245 INFO:teuthology.orchestra.run.smithi087.stderr: -110> 2022-09-03T07:31:27.842+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: waiting on bucket lease 2022-09-03T07:31:28.246 INFO:teuthology.orchestra.run.smithi087.stderr: -109> 2022-09-03T07:31:27.843+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: status after acquiring the lock is: init 2022-09-03T07:31:28.247 INFO:teuthology.orchestra.run.smithi087.stderr: -108> 2022-09-03T07:31:27.881+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: listing bucket for full sync 2022-09-03T07:31:28.247 INFO:teuthology.orchestra.run.smithi087.stderr: -107> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj11 2022-09-03T07:31:28.248 INFO:teuthology.orchestra.run.smithi087.stderr: -106> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj11]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj11 op=0 op_state=1 2022-09-03T07:31:28.248 INFO:teuthology.orchestra.run.smithi087.stderr: -105> 2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj11]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj11[0] 2022-09-03T07:31:28.249 INFO:teuthology.orchestra.run.smithi087.stderr: -104> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj111 2022-09-03T07:31:28.249 INFO:teuthology.orchestra.run.smithi087.stderr: -103> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj111]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj111 op=0 op_state=1 2022-09-03T07:31:28.249 INFO:teuthology.orchestra.run.smithi087.stderr: -102> 2022-09-03T07:31:27.928+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj111]: : injecting data sync error on key=obj111 2022-09-03T07:31:28.250 INFO:teuthology.orchestra.run.smithi087.stderr: -101> 2022-09-03T07:31:27.928+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj111]: failed, retcode=-5 ((5) Input/output error) 2022-09-03T07:31:28.251 INFO:teuthology.orchestra.run.smithi087.stderr: -100> 2022-09-03T07:31:27.928+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj111]: ERROR: failed to sync object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj111 2022-09-03T07:31:28.251 INFO:teuthology.orchestra.run.smithi087.stderr: -99> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj115 2022-09-03T07:31:28.251 INFO:teuthology.orchestra.run.smithi087.stderr: -98> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj115]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj115 op=0 op_state=1 2022-09-03T07:31:28.252 INFO:teuthology.orchestra.run.smithi087.stderr: -97> 2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj115]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj115[0] 2022-09-03T07:31:28.252 INFO:teuthology.orchestra.run.smithi087.stderr: -96> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj116 2022-09-03T07:31:28.253 INFO:teuthology.orchestra.run.smithi087.stderr: -95> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj116]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj116 op=0 op_state=1 2022-09-03T07:31:28.253 INFO:teuthology.orchestra.run.smithi087.stderr: -94> 2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj116]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj116[0] 2022-09-03T07:31:28.253 INFO:teuthology.orchestra.run.smithi087.stderr: -93> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj124 2022-09-03T07:31:28.254 INFO:teuthology.orchestra.run.smithi087.stderr: -92> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj124]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj124 op=0 op_state=1 2022-09-03T07:31:28.255 INFO:teuthology.orchestra.run.smithi087.stderr: -91> 2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj124]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj124[0] 2022-09-03T07:31:28.256 INFO:teuthology.orchestra.run.smithi087.stderr: -90> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj145 2022-09-03T07:31:28.256 INFO:teuthology.orchestra.run.smithi087.stderr: -89> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj145]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj145 op=0 op_state=1 2022-09-03T07:31:28.256 INFO:teuthology.orchestra.run.smithi087.stderr: -88> 2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj145]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj145[0] 2022-09-03T07:31:28.257 INFO:teuthology.orchestra.run.smithi087.stderr: -87> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj148 2022-09-03T07:31:28.257 INFO:teuthology.orchestra.run.smithi087.stderr: -86> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj148]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj148 op=0 op_state=1 2022-09-03T07:31:28.257 INFO:teuthology.orchestra.run.smithi087.stderr: -85> 2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj148]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj148[0] 2022-09-03T07:31:28.257 INFO:teuthology.orchestra.run.smithi087.stderr: -84> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj155 2022-09-03T07:31:28.258 INFO:teuthology.orchestra.run.smithi087.stderr: -83> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj155]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj155 op=0 op_state=1 2022-09-03T07:31:28.258 INFO:teuthology.orchestra.run.smithi087.stderr: -82> 2022-09-03T07:31:27.928+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj155]: : injecting data sync error on key=obj155 2022-09-03T07:31:28.258 INFO:teuthology.orchestra.run.smithi087.stderr: -81> 2022-09-03T07:31:27.928+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj155]: failed, retcode=-5 ((5) Input/output error) 2022-09-03T07:31:28.259 INFO:teuthology.orchestra.run.smithi087.stderr: -80> 2022-09-03T07:31:27.928+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj155]: ERROR: failed to sync object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj155 2022-09-03T07:31:28.259 INFO:teuthology.orchestra.run.smithi087.stderr: -79> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj17 2022-09-03T07:31:28.260 INFO:teuthology.orchestra.run.smithi087.stderr: -78> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj17]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj17 op=0 op_state=1 2022-09-03T07:31:28.260 INFO:teuthology.orchestra.run.smithi087.stderr: -77> 2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj17]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj17[0] 2022-09-03T07:31:28.261 INFO:teuthology.orchestra.run.smithi087.stderr: -76> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj185 2022-09-03T07:31:28.261 INFO:teuthology.orchestra.run.smithi087.stderr: -75> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj185]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj185 op=0 op_state=1 2022-09-03T07:31:28.261 INFO:teuthology.orchestra.run.smithi087.stderr: -74> 2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj185]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj185[0] 2022-09-03T07:31:28.262 INFO:teuthology.orchestra.run.smithi087.stderr: -73> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj19 2022-09-03T07:31:28.262 INFO:teuthology.orchestra.run.smithi087.stderr: -72> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj19]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj19 op=0 op_state=1 2022-09-03T07:31:28.263 INFO:teuthology.orchestra.run.smithi087.stderr: -71> 2022-09-03T07:31:27.928+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj19]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj19[0] 2022-09-03T07:31:28.263 INFO:teuthology.orchestra.run.smithi087.stderr: -70> 2022-09-03T07:31:27.928+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj22 2022-09-03T07:31:28.264 INFO:teuthology.orchestra.run.smithi087.stderr: -69> 2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj22]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj22 op=0 op_state=1 2022-09-03T07:31:28.264 INFO:teuthology.orchestra.run.smithi087.stderr: -68> 2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj22]: : injecting data sync error on key=obj22 2022-09-03T07:31:28.265 INFO:teuthology.orchestra.run.smithi087.stderr: -67> 2022-09-03T07:31:27.929+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj22]: failed, retcode=-5 ((5) Input/output error) 2022-09-03T07:31:28.265 INFO:teuthology.orchestra.run.smithi087.stderr: -66> 2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj22]: ERROR: failed to sync object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj22 2022-09-03T07:31:28.266 INFO:teuthology.orchestra.run.smithi087.stderr: -65> 2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj23 2022-09-03T07:31:28.266 INFO:teuthology.orchestra.run.smithi087.stderr: -64> 2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj23]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj23 op=0 op_state=1 2022-09-03T07:31:28.267 INFO:teuthology.orchestra.run.smithi087.stderr: -63> 2022-09-03T07:31:27.929+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj23]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj23[0] 2022-09-03T07:31:28.267 INFO:teuthology.orchestra.run.smithi087.stderr: -62> 2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj29 2022-09-03T07:31:28.268 INFO:teuthology.orchestra.run.smithi087.stderr: -61> 2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj29]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj29 op=0 op_state=1 2022-09-03T07:31:28.268 INFO:teuthology.orchestra.run.smithi087.stderr: -60> 2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj29]: : injecting data sync error on key=obj29 2022-09-03T07:31:28.269 INFO:teuthology.orchestra.run.smithi087.stderr: -59> 2022-09-03T07:31:27.929+0000 7f2cb4395600 10 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj29]: failed, retcode=-5 ((5) Input/output error) 2022-09-03T07:31:28.269 INFO:teuthology.orchestra.run.smithi087.stderr: -58> 2022-09-03T07:31:27.929+0000 7f2cb4395600 0 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj29]: ERROR: failed to sync object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj29 2022-09-03T07:31:28.270 INFO:teuthology.orchestra.run.smithi087.stderr: -57> 2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj31 2022-09-03T07:31:28.270 INFO:teuthology.orchestra.run.smithi087.stderr: -56> 2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj31]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj31 op=0 op_state=1 2022-09-03T07:31:28.271 INFO:teuthology.orchestra.run.smithi087.stderr: -55> 2022-09-03T07:31:27.929+0000 7f2cb4395600 5 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj31]: bucket sync: sync obj: fca49d30-abba-4c2b-8462-66395ac063ba/:avysmw-75[fca49d30-abba-4c2b-8462-66395ac063ba.5638.26])/obj31[0] 2022-09-03T07:31:28.272 INFO:teuthology.orchestra.run.smithi087.stderr: -54> 2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]: [full sync] syncing object: avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0/obj33 2022-09-03T07:31:28.272 INFO:teuthology.orchestra.run.smithi087.stderr: -53> 2022-09-03T07:31:27.929+0000 7f2cb4395600 20 RGW-SYNC:bucket[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26<-avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:full_sync[avysmw-75:fca49d30-abba-4c2b-8462-66395ac063ba.5638.26:0]:entry[obj33]: bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b=bucket sync single entry (source_zone=fca49d30-abba-4c2b-8462-66395ac063ba) b= log_entry=obj33 op=0 op_state=1 2022-09-03T07:31:28.283 DEBUG:teuthology.orchestra.run:got remote process result: None 2022-09-03T07:31:28.284 INFO:tasks.rgw.c2.client.0:Restarting daemon 2022-09-03T07:31:28.285 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2022-09-03T07:31:28.291 INFO:tasks.rgw.c2.client.0:Started 2022-09-03T07:31:28.293 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:28.341 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T07:31:28.344 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:31:28.344 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:31:28.345 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T07:31:29.346 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T07:31:29.383 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:31:29.384 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:31:29.384 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T07:31:31.386 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:31.412 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T07:31:31.415 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:31:31.416 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:31:31.416 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T07:31:35.418 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:31:35.444 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:31:35.444 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:31:35.445 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 208k 0 --:--:-- --:--:-- --:--:-- 208k 2022-09-03T07:31:35.448 INFO:teuthology.orchestra.run.smithi087.stdout:anonymous 2022-09-03T07:31:35.530 INFO:tasks.rgw_multisite_tests:Create several generations of objects, trash them, then run bucket sync init ... FAIL 2022-09-03T07:31:35.534 INFO:rgw_multi.tests:create role zone=test-zone1 name=roles-avysmw-1 2022-09-03T07:31:35.535 DEBUG:boto:Method: POST 2022-09-03T07:31:35.536 DEBUG:boto:Path: / 2022-09-03T07:31:35.537 DEBUG:boto:Data: 2022-09-03T07:31:35.538 DEBUG:boto:Headers: {} 2022-09-03T07:31:35.539 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:31:35.540 DEBUG:boto:Port: 8000 2022-09-03T07:31:35.540 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-1', 'AssumeRolePolicyDocument': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}', 'Path': '', 'Action': 'CreateRole', 'Version': '2010-05-08'} 2022-09-03T07:31:35.541 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:31:35.542 DEBUG:boto:Token: None 2022-09-03T07:31:35.543 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8000 x-amz-date:20220903T073135Z host;x-amz-date 426da61825548674725b106694751eca17ca6518dcf0ea99bada5169f6498e46 2022-09-03T07:31:35.543 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073135Z 20220903/front/smithi062/aws4_request a693f0b96dac380e658c8be70a70e62b7f09a6d6c942e35ceb718fdd0e53c01a 2022-09-03T07:31:35.544 DEBUG:boto:Signature: a958a082a3bcac97d16fac6e67128930bba656f005a897449fb6727d5547faae 2022-09-03T07:31:35.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073135Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '325', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=a958a082a3bcac97d16fac6e67128930bba656f005a897449fb6727d5547faae', 'Host': 'smithi062.front.sepia.ceph.com:8000'} 2022-09-03T07:31:35.558 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000080965644999c4cbd-00631302d7-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '731'), ('Date', 'Sat, 03 Sep 2022 07:31:35 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:35.559 DEBUG:boto:b'1c2474d0-84e1-4718-84b0-bd4c4fb59791roles-avysmw-1/arn:aws:iam:::role/roles-avysmw-12022-09-03T07:31:35.550Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}tx0000080965644999c4cbd-00631302d7-1604-test-zone1' 2022-09-03T07:31:35.560 INFO:rgw_multi.tests:create role zone=test-zone2 name=roles-avysmw-2 2022-09-03T07:31:35.561 DEBUG:boto:Method: POST 2022-09-03T07:31:35.561 DEBUG:boto:Path: / 2022-09-03T07:31:35.562 DEBUG:boto:Data: 2022-09-03T07:31:35.562 DEBUG:boto:Headers: {} 2022-09-03T07:31:35.563 DEBUG:boto:Host: smithi087.front.sepia.ceph.com 2022-09-03T07:31:35.563 DEBUG:boto:Port: 8000 2022-09-03T07:31:35.564 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-2', 'AssumeRolePolicyDocument': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}', 'Path': '', 'Action': 'CreateRole', 'Version': '2010-05-08'} 2022-09-03T07:31:35.565 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:31:35.565 DEBUG:boto:Token: None 2022-09-03T07:31:35.566 DEBUG:boto:CanonicalRequest: POST / host:smithi087.front.sepia.ceph.com:8000 x-amz-date:20220903T073135Z host;x-amz-date 3f3de033f95cee527a9b2c1e51c8aa374fbdef2e66d024b66eb10ce32dc17a50 2022-09-03T07:31:35.566 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073135Z 20220903/front/smithi087/aws4_request 1ec60163b8932802686c1d8d597506f55ba055a963922c2cfcd95b6935243802 2022-09-03T07:31:35.567 DEBUG:boto:Signature: 648355fb931891bfebf52d1407f5e7527f10248bff86e209f1853849c46c0d65 2022-09-03T07:31:35.567 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073135Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '325', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi087/aws4_request,SignedHeaders=host;x-amz-date,Signature=648355fb931891bfebf52d1407f5e7527f10248bff86e209f1853849c46c0d65', 'Host': 'smithi087.front.sepia.ceph.com:8000'} 2022-09-03T07:31:57.759 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000032ef7582425a76c4-00631302d7-1aa4-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '731'), ('Date', 'Sat, 03 Sep 2022 07:31:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:57.760 DEBUG:boto:b'f124dc1d-0b22-4775-92f8-577fa33be9e3roles-avysmw-2/arn:aws:iam:::role/roles-avysmw-22022-09-03T07:31:35.582Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}tx0000032ef7582425a76c4-00631302d7-1aa4-test-zone2' 2022-09-03T07:31:57.762 INFO:rgw_multi.tests:create role zone=test-zone3 name=roles-avysmw-3 2022-09-03T07:31:57.762 DEBUG:boto:Method: POST 2022-09-03T07:31:57.763 DEBUG:boto:Path: / 2022-09-03T07:31:57.763 DEBUG:boto:Data: 2022-09-03T07:31:57.764 DEBUG:boto:Headers: {} 2022-09-03T07:31:57.764 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:31:57.765 DEBUG:boto:Port: 8001 2022-09-03T07:31:57.765 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-3', 'AssumeRolePolicyDocument': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}', 'Path': '', 'Action': 'CreateRole', 'Version': '2010-05-08'} 2022-09-03T07:31:57.766 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:31:57.767 DEBUG:boto:Token: None 2022-09-03T07:31:57.767 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8001 x-amz-date:20220903T073157Z host;x-amz-date 2ab2d65e0e32e96bb72c14e52e02fa7b3114699493da5bb2e1f259a6afeb87b6 2022-09-03T07:31:57.768 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073157Z 20220903/front/smithi062/aws4_request 2af0275b98698339312c94f2eb4c1be69b66470a45a7d02a901624da84dc3ee2 2022-09-03T07:31:57.770 DEBUG:boto:Signature: 753861995d189891eb0667b9f6e407b2f5a99d331190304b88c51b08538cc890 2022-09-03T07:31:57.771 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073157Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '325', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=753861995d189891eb0667b9f6e407b2f5a99d331190304b88c51b08538cc890', 'Host': 'smithi062.front.sepia.ceph.com:8001'} 2022-09-03T07:31:57.800 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000035e0549b5e24c202-00631302ed-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '731'), ('Date', 'Sat, 03 Sep 2022 07:31:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:31:57.801 DEBUG:boto:b'20e2ef6d-021c-4e26-8cf8-afbf9a2a4aferoles-avysmw-3/arn:aws:iam:::role/roles-avysmw-32022-09-03T07:31:57.791Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}tx0000035e0549b5e24c202-00631302ed-18f3-test-zone3' 2022-09-03T07:31:57.803 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:31:57.804 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:31:57.804 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:31:57.862 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:57.863 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:57.924 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T07:31:57.947 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:31:57.947 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:31:57.947 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:31:57.948 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:31:57.780536Z" 2022-09-03T07:31:57.948 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:31:57.948 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:31:57.949 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:31:57.949 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:28:24.634541Z" 2022-09-03T07:31:57.949 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:31:57.949 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:31:57.950 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:31:57.950 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:31:57.950 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:31:57.950 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:31:57.951 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:31:57.951 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:31:57.951 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:31:57.951 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:31:57.953 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:31:57.954 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T07:31:57.954 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:31:57.955 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:31:57.956 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:31:58.013 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:31:58.014 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:31:58.067 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:58.065+0000 7f0fc54f5600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:31:58.068 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:31:58.066+0000 7f0fc54f5600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:31:58.087 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:31:58.088 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:31:58.088 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:31:58.088 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:31:58.089 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:31:58.090 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:31:58.090 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:31:58.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:58.091 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:31:58.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:58.093 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:31:58.094 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:31:58.094 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:31:58.094 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:31:58.094 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:31:58.095 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:31:58.095 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:31:58.095 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:31:58.096 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:31:58.096 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:31:58.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:58.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:58.097 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:31:58.097 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:31:58.099 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:31:58.099 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:31:58.100 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:31:58.100 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:31:58.100 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:31:58.100 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:31:58.101 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:31:58.101 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:31:58.101 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:58.101 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:58.102 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:31:58.102 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:31:58.102 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:31:58.103 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:31:58.103 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:31:58.103 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:31:58.103 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:31:58.104 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:31:58.104 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:31:58.104 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:31:58.105 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:58.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:31:58.105 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:31:58.106 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:31:58.107 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:31:58.107 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:31:58.107 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:31:58.109 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:31:58.109 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:31:58.109 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:31:58.109 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:31:58.110 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:31:58.110 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:31:58.110 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:31:58.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:58.111 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:31:58.111 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:31:58.111 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:31:58.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:31:58.112 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:31:58Z" 2022-09-03T07:31:58.112 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:31:58.113 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:31:58Z" } 2022-09-03T07:31:58.114 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:31:58.115 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:31:58.115 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:31:58.116 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:03.117 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:03.118 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:03.119 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:03.173 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:03.173 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:03.223 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:03.221+0000 7f01af495600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:03.227 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:03.226+0000 7f01af495600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:03.249 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:03.250 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:03.250 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:03.250 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:03.251 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:03.251 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:03.251 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:03.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:03.252 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:03.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:03.252 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:03.253 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:03.253 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:03.253 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:03.253 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:03.254 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:03.254 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:03.255 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:03.255 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:03.255 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:03.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:03.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:03.256 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:03.257 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:03.258 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:03.258 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:03.258 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:03.259 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:03.259 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:03.260 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:03.260 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:03.260 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:03.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:03.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:03.261 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:03.261 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:03.262 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:03.262 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:03.263 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:03.263 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:03.264 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:03.264 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:03.269 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:03.269 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:03.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:03.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:03.270 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:03.271 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:03.271 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:03.271 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:03.272 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:03.272 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:03.272 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:03.273 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:03.273 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:03.274 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:03.274 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:03.275 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:03.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:03.275 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:03.276 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:03.276 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:03.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:03.276 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:03Z" 2022-09-03T07:32:03.277 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:03.277 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:03Z" } 2022-09-03T07:32:03.279 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:03.280 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:03.280 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:03.281 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:08.282 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:08.284 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:08.287 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:08.353 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:08.354 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:08.396 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:08.394+0000 7feef4186600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:08.397 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:08.395+0000 7feef4186600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:08.410 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:08.411 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:08.411 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:08.412 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:08.412 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:08.413 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:08.413 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:08.414 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:08.415 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:08.415 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:08.417 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:08.417 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:08.418 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:08.419 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:08.419 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:08.419 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:08.420 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:08.420 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:08.420 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:08.421 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:08.422 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:08.423 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:08.424 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:08.424 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:08.425 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:08.426 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:08.426 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:08.427 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:08.427 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:08.428 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:08.428 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:08.429 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:08.430 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:08.430 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:08.431 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:08.431 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:08.432 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:08.432 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:08.433 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:08.434 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:08.434 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:08.435 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:08.435 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:08.436 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:08.436 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:08.437 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:08.437 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:08.438 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:08.438 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:08.438 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:08.439 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:08.439 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:08.440 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:08.440 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:08.440 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:08.441 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:08.442 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:08.442 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:08.442 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:08.443 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:08.443 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:08.443 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:08.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:08.444 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:08Z" 2022-09-03T07:32:08.444 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:08.446 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:08Z" } 2022-09-03T07:32:08.447 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:08.447 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:08.448 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:08.448 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:13.450 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:13.451 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:13.452 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:13.505 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:13.506 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:13.544 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:13.542+0000 7ff67af8f600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:13.545 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:13.544+0000 7ff67af8f600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:13.583 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:13Z" 2022-09-03T07:32:13.601 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:13.601 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:13Z" } 2022-09-03T07:32:13.602 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:13.602 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:13.603 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:13.603 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:18.604 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:18.605 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:18.606 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:18.661 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:18.661 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:18.726 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:18.718+0000 7f6d7b033600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:18.727 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:18.719+0000 7f6d7b033600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:18.736 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:18.737 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:18.738 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:18.738 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:18.740 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:18.741 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:18.742 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:18.742 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:18.743 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:18.743 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:18.743 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:18.744 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:18.744 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:18.745 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:18.745 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:18.746 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:18.746 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:18.747 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:18.747 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:18.747 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:18.748 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:18.748 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:18.748 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:18.748 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:18.749 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:18.749 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:18.749 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:18.750 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:18.750 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:18.750 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:18.751 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:18.752 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:18.752 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:18.753 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:18.753 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:18.754 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:18.754 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:18.755 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:18.756 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:18.756 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:18.757 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:18.757 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:18.757 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:18.758 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:18.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:18.759 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:18.759 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:18.760 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:18.760 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:18.761 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:18.763 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:18.763 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:18.763 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:18.764 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:18.764 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:18.764 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:18.764 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:18.765 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:18.765 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:18.765 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:18.766 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:18.766 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:18.766 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:18.767 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:18Z" 2022-09-03T07:32:18.767 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:18.768 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:18Z" } 2022-09-03T07:32:18.769 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:18.770 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:18.770 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:18.771 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:23.772 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:23.773 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:23.773 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:23.829 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:23.829 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:23.874 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:23.872+0000 7fd3a5868600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:23.875 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:23.873+0000 7fd3a5868600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:23.888 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:23.889 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:23.889 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:23.890 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:23.891 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:23.891 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:23.892 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:23.893 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:23.894 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:23.895 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:23.896 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:23.896 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:23.897 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:23.897 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:23.898 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:23.898 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:23.899 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:23.899 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:23.900 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:23.900 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:23.901 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:23.901 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:23.902 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:23.902 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:23.903 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:23.903 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:23.904 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:23.904 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:23.904 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:23.905 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:23.905 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:23.906 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:23.906 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:23.907 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:23.907 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:23.908 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:23.908 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:23.909 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:23.909 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:23.910 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:23.910 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:23.911 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:23.911 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:23.912 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:23.912 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:23.913 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:23.913 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:23.914 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:23.914 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:23.915 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:23.915 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:23.916 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:23.916 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:23.922 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:23.922 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:23.923 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:23.923 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:23.924 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:23.924 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:23.925 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:23.925 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:23.925 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:23.926 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:23.927 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:23Z" 2022-09-03T07:32:23.927 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:23.929 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:23Z" } 2022-09-03T07:32:23.931 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:23.932 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:23.934 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:23.934 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:28.936 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:28.937 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:28.939 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:28.992 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:28.992 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:29.039 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:29.038+0000 7ff9816c8600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:29.043 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:29.042+0000 7ff9816c8600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:29.059 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:29.059 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:29.060 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:29.060 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:29.060 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:29.061 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:29.061 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:29.062 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:29.062 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:29.063 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:29.063 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:29.064 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:29.064 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:29.065 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:29.065 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:29.065 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:29.066 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:29.066 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:29.066 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:29.067 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:29.067 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:29.068 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:29.068 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:29.068 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:29.069 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:29.069 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:29.069 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:29.070 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:29.070 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:29.070 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:29.070 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:29.071 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:29.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:29.071 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:29.071 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:29.072 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:29.072 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:29.072 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:29.072 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:29.073 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:29.073 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:29.074 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:29.074 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:29.074 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:29.075 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:29.075 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:29.076 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:29.076 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:29.076 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:29.076 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:29.077 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:29.077 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:29.078 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:29.078 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:29.078 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:29.079 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:29.079 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:29.079 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:29.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:29.080 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:29.080 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:29.080 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:29.081 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:29.081 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:29Z" 2022-09-03T07:32:29.081 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:29.082 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:29Z" } 2022-09-03T07:32:29.083 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:29.084 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:29.084 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:29.085 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:34.086 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:34.088 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:34.089 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:34.145 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:34.146 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:34.182 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:34.179+0000 7f4f78131600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:34.183 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:34.180+0000 7f4f78131600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:34.196 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:34.196 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:34.197 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:34.197 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:34.197 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:34.198 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:34.198 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:34.199 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:34.200 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:34.200 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:34.201 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:34.201 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:34.202 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:34.202 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:34.203 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:34.203 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:34.204 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:34.204 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:34.205 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:34.205 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:34.206 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:34.206 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:34.207 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:34.207 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:34.208 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:34.208 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:34.209 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:34.209 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:34.210 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:34.210 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:34.210 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:34.211 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:34.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:34.212 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:34.212 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:34.213 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:34.213 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:34.214 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:34.214 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:34.215 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:34.215 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:34.216 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:34.216 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:34.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:34.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:34.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:34.218 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:34.218 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:34.219 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:34.219 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:34.219 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:34.220 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:34.220 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:34.220 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:34.220 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:34.221 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:34.221 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:34.221 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:34.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:34.222 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:34.222 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:34.222 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:34.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:34.223 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:34Z" 2022-09-03T07:32:34.223 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:34.224 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:34Z" } 2022-09-03T07:32:34.225 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:34.226 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:34.227 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:34.227 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:39.229 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:39.230 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:39.231 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:39.287 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:39.288 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:39.339 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:39.337+0000 7f3e90011600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:39.340 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:39.338+0000 7f3e90011600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:39.357 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:39.357 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:39.357 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:39.358 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:39.359 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:39.359 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:39.359 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:39.360 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:39.360 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:39.360 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:39.360 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:39.361 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:39.361 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:39.361 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:39.361 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:39.362 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:39.362 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:39.362 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:39.363 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:39.363 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:39.363 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:39.363 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:39.364 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:39.364 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:39.364 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:39.365 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:39.365 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:39.365 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:39.365 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:39.366 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:39.366 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:39.366 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:39.367 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:39.367 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:39.367 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:39.367 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:39.368 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:39.368 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:39.368 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:39.369 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:39.370 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:39.370 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:39.370 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:39.370 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:39.371 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:39.371 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:39.371 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:39.371 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:39.372 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:39.372 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:39.372 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:39.372 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:39.373 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:39.373 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:39.373 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:39.373 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:39.374 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:39.374 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:39.374 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:39.374 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:39.375 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:39.375 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:39.376 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:39.376 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:39Z" 2022-09-03T07:32:39.376 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:39.378 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:39Z" } 2022-09-03T07:32:39.379 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:39.380 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:39.380 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:39.381 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:44.382 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:44.383 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:44.385 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:44.437 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:44.438 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:44.475 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:44.474+0000 7f7d162de600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:44.476 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:44.475+0000 7f7d162de600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:44.494 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:44.495 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:44.495 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:44.495 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:44.496 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:44.496 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:44.496 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:44.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:44.497 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:44.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:44.498 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:44.498 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:44.498 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:44.498 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:44.499 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:44.499 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:44.499 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:44.499 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:44.500 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:44.500 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:44.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:44.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:44.501 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:44.501 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:44.501 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:44.502 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:44.502 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:44.502 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:44.502 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:44.503 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:44.503 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:44.504 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:44.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:44.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:44.505 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:44.505 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:44.506 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:44.506 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:44.506 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:44.507 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:44.507 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:44.507 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:44.508 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:44.508 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:44.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:44.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:44.509 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:44.509 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:44.509 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:44.510 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:44.510 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:44.510 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:44.511 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:44.511 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:44.511 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:44.511 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:44.512 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:44.512 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:44.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:44.513 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:44.513 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:44.513 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:44.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:44.514 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:44Z" 2022-09-03T07:32:44.514 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:44.516 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:44Z" } 2022-09-03T07:32:44.517 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:44.517 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:44.518 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:44.518 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:49.519 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:49.521 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:49.522 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:49.579 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:49.579 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:49.610 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:49.609+0000 7fb60ef61600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:49.611 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:49.610+0000 7fb60ef61600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:49.625 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:49.626 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:49.626 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:49.627 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:49.627 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:49.628 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:49.628 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:49.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:49.629 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:49.629 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:49.630 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:49.630 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:49.631 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:49.631 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:49.631 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:49.632 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:49.632 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:49.632 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:49.633 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:49.633 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:49.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:49.634 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:49.634 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:49.634 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:49.635 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:49.635 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:49.635 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:49.636 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:49.636 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:49.636 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:49.637 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:49.637 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:49.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:49.637 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:49.638 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:49.638 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:49.638 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:49.639 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:49.639 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:49.639 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:49.639 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:49.640 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:49.640 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:49.640 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:49.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:49.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:49.641 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:49.641 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:49.642 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:49.642 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:49.642 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:49.643 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:49.643 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:49.643 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:49.644 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:49.644 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:49.644 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:49.645 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:49.645 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:49.645 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:49.646 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:49.646 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:49.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:49.646 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:49Z" 2022-09-03T07:32:49.647 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:49.648 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:49Z" } 2022-09-03T07:32:49.649 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:49.650 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:49.650 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:49.651 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:54.652 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:54.653 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:54.655 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:54.710 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:54.711 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:54.768 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:54.762+0000 7efe456b1600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:54.769 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:54.763+0000 7efe456b1600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:54.779 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:54.780 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:54.780 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:54.780 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:54.780 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:54.781 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:54.781 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:54.781 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:54.781 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:54.782 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:54.782 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:54.782 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:54.783 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:54.783 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662184350.715807_109.1", 2022-09-03T07:32:54.783 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:54.784 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:54.784 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:54.784 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T05:52:30.715807Z", 2022-09-03T07:32:54.785 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:54.785 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:54.786 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:54.786 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:54.787 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:54.787 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:54.788 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:54.788 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:54.790 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:54.791 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:54.791 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:54.791 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:54.792 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:54.793 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:54.794 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:54.794 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:54.795 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:54.796 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:54.796 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:54.797 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:54.797 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:54.798 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:54.798 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:54.799 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:54.799 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:54.800 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:54.801 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:54.801 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:54.802 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:54.802 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:54.803 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:54.803 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:54.804 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:54.804 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:54.805 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:54.805 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:54.806 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:54.806 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:54.807 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:54.807 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:54.808 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:54.808 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:54.809 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:54.809 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:54.810 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:54.811 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:54Z" 2022-09-03T07:32:54.812 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:54.814 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662184350.715807_109.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T05:52:30.715807Z", "realm_epoch": 2 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:54Z" } 2022-09-03T07:32:54.816 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662184350.715807_109.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T05:52:30.715807Z', 'realm_epoch': 2}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:54.817 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:54.817 DEBUG:rgw_multi.tests:sync_status={0: '', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:54.818 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1662190317.780536_375.1 target= 2022-09-03T07:32:59.820 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:59.821 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:59.822 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:32:59.872 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:32:59.872 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:32:59.931 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:59.930+0000 7f6bebfb4600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:59.932 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:32:59.931+0000 7f6bebfb4600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:32:59.953 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:32:59.954 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:32:59.954 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:32:59.955 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:32:59.955 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:32:59.956 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:32:59.956 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:59.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:59.957 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:32:59.958 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:59.958 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:32:59.959 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:59.959 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:59.960 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:32:59.960 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:59.961 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:32:59.961 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:59.961 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:32:59.961 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:59.962 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:59.962 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:59.962 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:59.962 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:32:59.963 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:59.963 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:59.963 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:32:59.963 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:59.964 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:59.964 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:59.964 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:32:59.964 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:32:59.964 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:59.965 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:59.965 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:59.965 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:32:59.965 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:59.966 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:59.966 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:59.966 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:59.966 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:59.967 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:59.967 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:59.967 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:59.967 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:59.968 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:59.968 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:32:59.968 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:32:59.968 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:32:59.969 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:32:59.969 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:32:59.969 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:32:59.969 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:32:59.970 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:32:59.970 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:32:59.970 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:32:59.970 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:59.971 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:32:59.971 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:32:59.971 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:59.971 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:32:59.972 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:32:59.972 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:32:59.972 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:32:59.973 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:32:59Z" 2022-09-03T07:32:59.973 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:32:59.974 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:32:59Z" } 2022-09-03T07:32:59.975 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:32:59.975 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:59.976 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:32:59.977 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T07:32:59.977 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T07:32:59.978 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:59.978 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:32:59.979 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:33:00.039 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:33:00.040 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:33:00.079 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:33:00.078+0000 7f1a4b6b9600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:00.082 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:33:00.079+0000 7f1a4b6b9600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:00.095 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:33:00.096 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:33:00.096 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:33:00.096 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:33:00.097 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:33:00.097 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:33:00.097 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:33:00.097 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:00.098 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:33:00.098 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:00.098 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:33:00.099 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:33:00.099 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:33:00.099 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:33:00.099 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:33:00.100 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T07:33:00.100 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:33:00.100 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:33:00.100 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:33:00.101 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:00.102 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:00.102 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:00.103 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:33:00.103 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:33:00.103 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:33:00.103 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:33:00.104 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:33:00.104 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:33:00.104 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:33:00.105 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:33:00.105 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:33:00.105 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:00.106 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:00.106 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:00.106 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:33:00.107 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:33:00.107 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:33:00.107 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:33:00.107 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:33:00.108 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:33:00.108 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:33:00.109 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:33:00.109 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:33:00.109 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:00.110 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:00.110 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:00.110 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:33:00.111 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:33:00.111 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:33:00.111 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:33:00.111 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:33:00.112 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:33:00.112 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:33:00.112 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:33:00.112 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:33:00.113 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:00.113 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:00.113 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:33:00.113 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:00.114 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:33:00.114 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T07:33:00.114 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T07:33:00.114 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:00.115 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:33:00Z" 2022-09-03T07:33:00.115 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:33:00.116 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:33:00Z" } 2022-09-03T07:33:00.117 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:33:00.118 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:33:00.118 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:33:00.119 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T07:33:00.119 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T07:33:00.120 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:00.120 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:00.121 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:33:00.170 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:33:00.170 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:33:00.221 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:00.219+0000 7fa9dbc35600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:00.222 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:00.220+0000 7fa9dbc35600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:00.234 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:33:00.234 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:33:00.234 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:33:00.235 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:33:00.236 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:33:00.236 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:33:00.237 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:00.237 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.238 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:33:00.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:00.239 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:33:00.239 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:00.240 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:00.240 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:33:00.241 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:00.241 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:33:00.242 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:00.242 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:33:00.243 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:00.243 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:00.245 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:33:00.245 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:00.246 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:00.246 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190104.634541_499.1", 2022-09-03T07:33:00.247 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:00.247 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:00.248 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:00.248 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.634541Z", 2022-09-03T07:33:00.248 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:00.249 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:00.249 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:33:00.250 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:00.251 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:00.252 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:33:00.252 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:00.252 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:00.253 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:00.253 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:33:00.253 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:33:00.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:00.254 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:33:00.255 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:00.255 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:00.256 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:33:00.256 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:00.257 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:00.257 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:00.257 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:33:00.258 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:33:00.258 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.259 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.259 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:33:00.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.259 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:33:00.260 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:33:00.260 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:33:00.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.261 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:33:00Z" 2022-09-03T07:33:00.262 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:33:00.264 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190104.634541_499.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.634541Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:33:00Z" } 2022-09-03T07:33:00.265 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190104.634541_499.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.634541Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:33:00.266 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:33:00.267 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190104.634541_499.1', 2: '', 3: ''} 2022-09-03T07:33:00.268 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T07:33:00.269 INFO:tasks.rgw_multi.zone_rados:comparing role=roles-avysmw-1 zones={test-zone2, test-zone1} 2022-09-03T07:33:00.270 DEBUG:boto:Method: POST 2022-09-03T07:33:00.271 DEBUG:boto:Path: / 2022-09-03T07:33:00.272 DEBUG:boto:Data: 2022-09-03T07:33:00.272 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.273 DEBUG:boto:Host: smithi087.front.sepia.ceph.com 2022-09-03T07:33:00.273 DEBUG:boto:Port: 8000 2022-09-03T07:33:00.274 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-1', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.276 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:33:00.277 DEBUG:boto:Token: None 2022-09-03T07:33:00.279 DEBUG:boto:CanonicalRequest: POST / host:smithi087.front.sepia.ceph.com:8000 x-amz-date:20220903T073300Z host;x-amz-date 1478980818b36528f8cee3cd0e8760f5eb0d1bc9506feba4af2c00ec17c2b525 2022-09-03T07:33:00.280 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi087/aws4_request 579a43e0c57f0adb6e4ea9b8f2370b9af2a397f4ca8a4ea7857899e00dc00ae1 2022-09-03T07:33:00.281 DEBUG:boto:Signature: 91ba3838c0c0474f6f9aa0fa229153d4a2fbe0d3526dccf5498c62491d65b9f0 2022-09-03T07:33:00.282 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi087/aws4_request,SignedHeaders=host;x-amz-date,Signature=91ba3838c0c0474f6f9aa0fa229153d4a2fbe0d3526dccf5498c62491d65b9f0', 'Host': 'smithi087.front.sepia.ceph.com:8000'} 2022-09-03T07:33:00.287 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000f77584342aa2e4a4-006313032c-1aa4-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.288 DEBUG:boto:b'tx00000f77584342aa2e4a4-006313032c-1aa4-test-zone21c2474d0-84e1-4718-84b0-bd4c4fb59791roles-avysmw-1/arn:aws:iam:::role/roles-avysmw-12022-09-03T07:32:58.887Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.289 DEBUG:boto:Method: POST 2022-09-03T07:33:00.290 DEBUG:boto:Path: / 2022-09-03T07:33:00.291 DEBUG:boto:Data: 2022-09-03T07:33:00.291 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.291 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:33:00.292 DEBUG:boto:Port: 8000 2022-09-03T07:33:00.292 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-1', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.293 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:33:00.294 DEBUG:boto:Token: None 2022-09-03T07:33:00.295 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8000 x-amz-date:20220903T073300Z host;x-amz-date 1478980818b36528f8cee3cd0e8760f5eb0d1bc9506feba4af2c00ec17c2b525 2022-09-03T07:33:00.295 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi062/aws4_request 81fef1b1d08d6f57016fee026db47d927e79402a5742515cdfebf7b9a83f9435 2022-09-03T07:33:00.296 DEBUG:boto:Signature: 557d90476b7b3a0c95b560ca548c31ca6be260b62e93a8a54d0ea62cda593f3e 2022-09-03T07:33:00.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=557d90476b7b3a0c95b560ca548c31ca6be260b62e93a8a54d0ea62cda593f3e', 'Host': 'smithi062.front.sepia.ceph.com:8000'} 2022-09-03T07:33:00.299 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000007938c80c98c9fdb2-006313032c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.300 DEBUG:boto:b'tx000007938c80c98c9fdb2-006313032c-1604-test-zone11c2474d0-84e1-4718-84b0-bd4c4fb59791roles-avysmw-1/arn:aws:iam:::role/roles-avysmw-12022-09-03T07:31:35.550Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.301 DEBUG:tasks.rgw_multi.zone_rados:comparing role name=roles-avysmw-1 2022-09-03T07:33:00.302 INFO:tasks.rgw_multi.zone_rados:success, role identical: role=roles-avysmw-1 zones={test-zone2, test-zone1} 2022-09-03T07:33:00.302 INFO:tasks.rgw_multi.zone_rados:comparing role=roles-avysmw-1 zones={test-zone3, test-zone1} 2022-09-03T07:33:00.303 DEBUG:boto:Method: POST 2022-09-03T07:33:00.303 DEBUG:boto:Path: / 2022-09-03T07:33:00.304 DEBUG:boto:Data: 2022-09-03T07:33:00.304 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:33:00.305 DEBUG:boto:Port: 8001 2022-09-03T07:33:00.306 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-1', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.307 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:33:00.307 DEBUG:boto:Token: None 2022-09-03T07:33:00.308 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8001 x-amz-date:20220903T073300Z host;x-amz-date 1478980818b36528f8cee3cd0e8760f5eb0d1bc9506feba4af2c00ec17c2b525 2022-09-03T07:33:00.308 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi062/aws4_request c81c007ab32c9bdea26b73210a5d792e23e5347b999cd2326414b728fc0cdc6a 2022-09-03T07:33:00.309 DEBUG:boto:Signature: e37a5dd1e4d0be943c43bd1bc69e9c06d51850e89eb0260318daf5e928cc272e 2022-09-03T07:33:00.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=e37a5dd1e4d0be943c43bd1bc69e9c06d51850e89eb0260318daf5e928cc272e', 'Host': 'smithi062.front.sepia.ceph.com:8001'} 2022-09-03T07:33:00.312 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000cfc42c45715fc231-006313032c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.313 DEBUG:boto:b'tx00000cfc42c45715fc231-006313032c-18f3-test-zone31c2474d0-84e1-4718-84b0-bd4c4fb59791roles-avysmw-1/arn:aws:iam:::role/roles-avysmw-12022-09-03T07:31:36.720Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.314 DEBUG:boto:Method: POST 2022-09-03T07:33:00.315 DEBUG:boto:Path: / 2022-09-03T07:33:00.315 DEBUG:boto:Data: 2022-09-03T07:33:00.316 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.316 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:33:00.317 DEBUG:boto:Port: 8000 2022-09-03T07:33:00.317 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-1', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.318 DEBUG:boto:Token: None 2022-09-03T07:33:00.319 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8000 x-amz-date:20220903T073300Z host;x-amz-date 1478980818b36528f8cee3cd0e8760f5eb0d1bc9506feba4af2c00ec17c2b525 2022-09-03T07:33:00.319 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi062/aws4_request 81fef1b1d08d6f57016fee026db47d927e79402a5742515cdfebf7b9a83f9435 2022-09-03T07:33:00.320 DEBUG:boto:Signature: 557d90476b7b3a0c95b560ca548c31ca6be260b62e93a8a54d0ea62cda593f3e 2022-09-03T07:33:00.320 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=557d90476b7b3a0c95b560ca548c31ca6be260b62e93a8a54d0ea62cda593f3e', 'Host': 'smithi062.front.sepia.ceph.com:8000'} 2022-09-03T07:33:00.322 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000003c0b41a51dd96b3a-006313032c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.323 DEBUG:boto:b'tx000003c0b41a51dd96b3a-006313032c-1604-test-zone11c2474d0-84e1-4718-84b0-bd4c4fb59791roles-avysmw-1/arn:aws:iam:::role/roles-avysmw-12022-09-03T07:31:35.550Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.324 DEBUG:tasks.rgw_multi.zone_rados:comparing role name=roles-avysmw-1 2022-09-03T07:33:00.324 INFO:tasks.rgw_multi.zone_rados:success, role identical: role=roles-avysmw-1 zones={test-zone3, test-zone1} 2022-09-03T07:33:00.325 INFO:tasks.rgw_multi.zone_rados:comparing role=roles-avysmw-2 zones={test-zone1, test-zone2} 2022-09-03T07:33:00.326 DEBUG:boto:Method: POST 2022-09-03T07:33:00.327 DEBUG:boto:Path: / 2022-09-03T07:33:00.328 DEBUG:boto:Data: 2022-09-03T07:33:00.328 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.329 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:33:00.330 DEBUG:boto:Port: 8000 2022-09-03T07:33:00.332 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-2', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.333 DEBUG:boto:Token: None 2022-09-03T07:33:00.334 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8000 x-amz-date:20220903T073300Z host;x-amz-date cfd7adaea42c3909d1c15b620cd325cdf201ae96bb222d87edfc3db54144127f 2022-09-03T07:33:00.335 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi062/aws4_request 8da7f7aa15b031c62096738136f035fbfa48dce33e7a627245f70fa19b170571 2022-09-03T07:33:00.336 DEBUG:boto:Signature: a97d4ea74b1da57d8f2f8c27b7e3b2ce3944b7e925cb221fe2ec54fff1a70b30 2022-09-03T07:33:00.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=a97d4ea74b1da57d8f2f8c27b7e3b2ce3944b7e925cb221fe2ec54fff1a70b30', 'Host': 'smithi062.front.sepia.ceph.com:8000'} 2022-09-03T07:33:00.339 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000008ed01299c1ac4ae6-006313032c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.339 DEBUG:boto:b'tx000008ed01299c1ac4ae6-006313032c-1604-test-zone1f124dc1d-0b22-4775-92f8-577fa33be9e3roles-avysmw-2/arn:aws:iam:::role/roles-avysmw-22022-09-03T07:31:35.572Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.340 DEBUG:boto:Method: POST 2022-09-03T07:33:00.341 DEBUG:boto:Path: / 2022-09-03T07:33:00.341 DEBUG:boto:Data: 2022-09-03T07:33:00.342 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.342 DEBUG:boto:Host: smithi087.front.sepia.ceph.com 2022-09-03T07:33:00.343 DEBUG:boto:Port: 8000 2022-09-03T07:33:00.343 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-2', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.344 DEBUG:boto:Token: None 2022-09-03T07:33:00.345 DEBUG:boto:CanonicalRequest: POST / host:smithi087.front.sepia.ceph.com:8000 x-amz-date:20220903T073300Z host;x-amz-date cfd7adaea42c3909d1c15b620cd325cdf201ae96bb222d87edfc3db54144127f 2022-09-03T07:33:00.345 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi087/aws4_request 04057e9caa0453825d08c24b56d2b4a54ca016668b8c249796f7443435b50f2e 2022-09-03T07:33:00.346 DEBUG:boto:Signature: d9ec3d5e80a4351f7e6263b5b6eeeac133cfa25029e79e894c0c1b5224426869 2022-09-03T07:33:00.346 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi087/aws4_request,SignedHeaders=host;x-amz-date,Signature=d9ec3d5e80a4351f7e6263b5b6eeeac133cfa25029e79e894c0c1b5224426869', 'Host': 'smithi087.front.sepia.ceph.com:8000'} 2022-09-03T07:33:00.349 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000eb1b926391e4fe8e-006313032c-1aa4-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.349 DEBUG:boto:b'tx00000eb1b926391e4fe8e-006313032c-1aa4-test-zone2f124dc1d-0b22-4775-92f8-577fa33be9e3roles-avysmw-2/arn:aws:iam:::role/roles-avysmw-22022-09-03T07:31:35.572Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.350 DEBUG:tasks.rgw_multi.zone_rados:comparing role name=roles-avysmw-2 2022-09-03T07:33:00.351 INFO:tasks.rgw_multi.zone_rados:success, role identical: role=roles-avysmw-2 zones={test-zone1, test-zone2} 2022-09-03T07:33:00.352 INFO:tasks.rgw_multi.zone_rados:comparing role=roles-avysmw-2 zones={test-zone3, test-zone2} 2022-09-03T07:33:00.352 DEBUG:boto:Method: POST 2022-09-03T07:33:00.353 DEBUG:boto:Path: / 2022-09-03T07:33:00.353 DEBUG:boto:Data: 2022-09-03T07:33:00.354 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.355 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:33:00.355 DEBUG:boto:Port: 8001 2022-09-03T07:33:00.356 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-2', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.357 DEBUG:boto:Token: None 2022-09-03T07:33:00.357 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8001 x-amz-date:20220903T073300Z host;x-amz-date cfd7adaea42c3909d1c15b620cd325cdf201ae96bb222d87edfc3db54144127f 2022-09-03T07:33:00.358 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi062/aws4_request ff5ef3ce0d31624fca284666141efd5984575bb64bb8c08bde0b3e95fc770173 2022-09-03T07:33:00.360 DEBUG:boto:Signature: 1f1fedd50e3b70200c3d7459a9ef14004894e4c233034ea966f80e2bd25c568e 2022-09-03T07:33:00.360 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=1f1fedd50e3b70200c3d7459a9ef14004894e4c233034ea966f80e2bd25c568e', 'Host': 'smithi062.front.sepia.ceph.com:8001'} 2022-09-03T07:33:00.362 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000030d6f6451e293456-006313032c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.363 DEBUG:boto:b'tx0000030d6f6451e293456-006313032c-18f3-test-zone3f124dc1d-0b22-4775-92f8-577fa33be9e3roles-avysmw-2/arn:aws:iam:::role/roles-avysmw-22022-09-03T07:31:35.572Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.364 DEBUG:boto:Method: POST 2022-09-03T07:33:00.365 DEBUG:boto:Path: / 2022-09-03T07:33:00.365 DEBUG:boto:Data: 2022-09-03T07:33:00.366 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.367 DEBUG:boto:Host: smithi087.front.sepia.ceph.com 2022-09-03T07:33:00.368 DEBUG:boto:Port: 8000 2022-09-03T07:33:00.368 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-2', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.369 DEBUG:boto:Token: None 2022-09-03T07:33:00.370 DEBUG:boto:CanonicalRequest: POST / host:smithi087.front.sepia.ceph.com:8000 x-amz-date:20220903T073300Z host;x-amz-date cfd7adaea42c3909d1c15b620cd325cdf201ae96bb222d87edfc3db54144127f 2022-09-03T07:33:00.371 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi087/aws4_request 04057e9caa0453825d08c24b56d2b4a54ca016668b8c249796f7443435b50f2e 2022-09-03T07:33:00.371 DEBUG:boto:Signature: d9ec3d5e80a4351f7e6263b5b6eeeac133cfa25029e79e894c0c1b5224426869 2022-09-03T07:33:00.372 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi087/aws4_request,SignedHeaders=host;x-amz-date,Signature=d9ec3d5e80a4351f7e6263b5b6eeeac133cfa25029e79e894c0c1b5224426869', 'Host': 'smithi087.front.sepia.ceph.com:8000'} 2022-09-03T07:33:00.373 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000004c506d933754c44-006313032c-1aa4-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.374 DEBUG:boto:b'tx0000004c506d933754c44-006313032c-1aa4-test-zone2f124dc1d-0b22-4775-92f8-577fa33be9e3roles-avysmw-2/arn:aws:iam:::role/roles-avysmw-22022-09-03T07:31:35.572Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.375 DEBUG:tasks.rgw_multi.zone_rados:comparing role name=roles-avysmw-2 2022-09-03T07:33:00.376 INFO:tasks.rgw_multi.zone_rados:success, role identical: role=roles-avysmw-2 zones={test-zone3, test-zone2} 2022-09-03T07:33:00.376 INFO:tasks.rgw_multi.zone_rados:comparing role=roles-avysmw-3 zones={test-zone1, test-zone3} 2022-09-03T07:33:00.378 DEBUG:boto:Method: POST 2022-09-03T07:33:00.379 DEBUG:boto:Path: / 2022-09-03T07:33:00.380 DEBUG:boto:Data: 2022-09-03T07:33:00.381 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.381 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:33:00.382 DEBUG:boto:Port: 8000 2022-09-03T07:33:00.382 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-3', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.383 DEBUG:boto:Token: None 2022-09-03T07:33:00.383 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8000 x-amz-date:20220903T073300Z host;x-amz-date 0056ad11ffc4fc340c1f79fae874523394e1e73f2de9bcfb56b8291c7d00baf4 2022-09-03T07:33:00.384 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi062/aws4_request 89e309697423f5b2bf87793cf58b4d4ef1f7976b479c52f0ee1b0a0ed8605ea2 2022-09-03T07:33:00.384 DEBUG:boto:Signature: 9930366b8fd509d81d487947ad8502c079067602b5094f7616e0cf1f1d57e03d 2022-09-03T07:33:00.385 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=9930366b8fd509d81d487947ad8502c079067602b5094f7616e0cf1f1d57e03d', 'Host': 'smithi062.front.sepia.ceph.com:8000'} 2022-09-03T07:33:00.387 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000000f48ca4caa8ff0f0-006313032c-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.388 DEBUG:boto:b'tx000000f48ca4caa8ff0f0-006313032c-1604-test-zone120e2ef6d-021c-4e26-8cf8-afbf9a2a4aferoles-avysmw-3/arn:aws:iam:::role/roles-avysmw-32022-09-03T07:31:57.776Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.389 DEBUG:boto:Method: POST 2022-09-03T07:33:00.389 DEBUG:boto:Path: / 2022-09-03T07:33:00.390 DEBUG:boto:Data: 2022-09-03T07:33:00.391 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.391 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:33:00.392 DEBUG:boto:Port: 8001 2022-09-03T07:33:00.393 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-3', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.394 DEBUG:boto:Token: None 2022-09-03T07:33:00.395 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8001 x-amz-date:20220903T073300Z host;x-amz-date 0056ad11ffc4fc340c1f79fae874523394e1e73f2de9bcfb56b8291c7d00baf4 2022-09-03T07:33:00.396 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi062/aws4_request cb546a24717f666ca8d7a3d010f34ae28b4437d3140972cce5d332a42511def8 2022-09-03T07:33:00.397 DEBUG:boto:Signature: b3283c5fcbfe16bbf191abdb3107d60bd830c6a03d0c8a0d2a319a5840f771d5 2022-09-03T07:33:00.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=b3283c5fcbfe16bbf191abdb3107d60bd830c6a03d0c8a0d2a319a5840f771d5', 'Host': 'smithi062.front.sepia.ceph.com:8001'} 2022-09-03T07:33:00.401 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx000007145f0320822c0e2-006313032c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.402 DEBUG:boto:b'tx000007145f0320822c0e2-006313032c-18f3-test-zone320e2ef6d-021c-4e26-8cf8-afbf9a2a4aferoles-avysmw-3/arn:aws:iam:::role/roles-avysmw-32022-09-03T07:31:57.776Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.404 DEBUG:tasks.rgw_multi.zone_rados:comparing role name=roles-avysmw-3 2022-09-03T07:33:00.405 INFO:tasks.rgw_multi.zone_rados:success, role identical: role=roles-avysmw-3 zones={test-zone1, test-zone3} 2022-09-03T07:33:00.406 INFO:tasks.rgw_multi.zone_rados:comparing role=roles-avysmw-3 zones={test-zone2, test-zone3} 2022-09-03T07:33:00.407 DEBUG:boto:Method: POST 2022-09-03T07:33:00.408 DEBUG:boto:Path: / 2022-09-03T07:33:00.409 DEBUG:boto:Data: 2022-09-03T07:33:00.410 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.411 DEBUG:boto:Host: smithi087.front.sepia.ceph.com 2022-09-03T07:33:00.411 DEBUG:boto:Port: 8000 2022-09-03T07:33:00.412 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-3', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.413 DEBUG:boto:Token: None 2022-09-03T07:33:00.414 DEBUG:boto:CanonicalRequest: POST / host:smithi087.front.sepia.ceph.com:8000 x-amz-date:20220903T073300Z host;x-amz-date 0056ad11ffc4fc340c1f79fae874523394e1e73f2de9bcfb56b8291c7d00baf4 2022-09-03T07:33:00.415 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi087/aws4_request b7dbcdf49422e4291a2e3c56129175d79700eca15769c00f679b433c24e04678 2022-09-03T07:33:00.416 DEBUG:boto:Signature: a19fd2e93aafadab972372c145ea26eaea597d77bc23efd63bf578f34884ab08 2022-09-03T07:33:00.417 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi087/aws4_request,SignedHeaders=host;x-amz-date,Signature=a19fd2e93aafadab972372c145ea26eaea597d77bc23efd63bf578f34884ab08', 'Host': 'smithi087.front.sepia.ceph.com:8000'} 2022-09-03T07:33:00.420 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000535b37adc5dd7e65-006313032c-1aa4-test-zone2'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.421 DEBUG:boto:b'tx00000535b37adc5dd7e65-006313032c-1aa4-test-zone220e2ef6d-021c-4e26-8cf8-afbf9a2a4aferoles-avysmw-3/arn:aws:iam:::role/roles-avysmw-32022-09-03T07:31:57.776Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.423 DEBUG:boto:Method: POST 2022-09-03T07:33:00.424 DEBUG:boto:Path: / 2022-09-03T07:33:00.425 DEBUG:boto:Data: 2022-09-03T07:33:00.426 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com 2022-09-03T07:33:00.428 DEBUG:boto:Port: 8001 2022-09-03T07:33:00.429 DEBUG:boto:Params: {'RoleName': 'roles-avysmw-3', 'Action': 'GetRole', 'Version': '2010-05-08'} 2022-09-03T07:33:00.430 DEBUG:boto:Token: None 2022-09-03T07:33:00.431 DEBUG:boto:CanonicalRequest: POST / host:smithi062.front.sepia.ceph.com:8001 x-amz-date:20220903T073300Z host;x-amz-date 0056ad11ffc4fc340c1f79fae874523394e1e73f2de9bcfb56b8291c7d00baf4 2022-09-03T07:33:00.432 DEBUG:boto:StringToSign: AWS4-HMAC-SHA256 20220903T073300Z 20220903/front/smithi062/aws4_request cb546a24717f666ca8d7a3d010f34ae28b4437d3140972cce5d332a42511def8 2022-09-03T07:33:00.433 DEBUG:boto:Signature: b3283c5fcbfe16bbf191abdb3107d60bd830c6a03d0c8a0d2a319a5840f771d5 2022-09-03T07:33:00.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'X-Amz-Date': '20220903T073300Z', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length': '57', 'Authorization': 'AWS4-HMAC-SHA256 Credential=1KKLJ23JMA7EOB7UYR06/20220903/front/smithi062/aws4_request,SignedHeaders=host;x-amz-date,Signature=b3283c5fcbfe16bbf191abdb3107d60bd830c6a03d0c8a0d2a319a5840f771d5', 'Host': 'smithi062.front.sepia.ceph.com:8001'} 2022-09-03T07:33:00.436 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000bd82611d7f64fa9a-006313032c-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Content-Length', '719'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.437 DEBUG:boto:b'tx00000bd82611d7f64fa9a-006313032c-18f3-test-zone320e2ef6d-021c-4e26-8cf8-afbf9a2a4aferoles-avysmw-3/arn:aws:iam:::role/roles-avysmw-32022-09-03T07:31:57.776Z3600{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/testuser"]},"Action":["sts:AssumeRole"]}]}' 2022-09-03T07:33:00.439 DEBUG:tasks.rgw_multi.zone_rados:comparing role name=roles-avysmw-3 2022-09-03T07:33:00.440 INFO:tasks.rgw_multi.zone_rados:success, role identical: role=roles-avysmw-3 zones={test-zone2, test-zone3} 2022-09-03T07:33:00.441 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_role_sync ... ok 2022-09-03T07:33:00.443 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:33:00.444 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:33:00.444 DEBUG:boto:Method: PUT 2022-09-03T07:33:00.445 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:33:00.445 DEBUG:boto:Data: 2022-09-03T07:33:00.446 DEBUG:boto:Headers: {} 2022-09-03T07:33:00.446 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:00.447 DEBUG:boto:Port: 8000 2022-09-03T07:33:00.447 DEBUG:boto:Params: {} 2022-09-03T07:33:00.448 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:33:00.449 DEBUG:boto:Token: None 2022-09-03T07:33:00.449 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 07:33:00 GMT /avysmw-76/ 2022-09-03T07:33:00.450 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TUYZ0psjRhNz3uX3Mvr2BIc+oYA= 2022-09-03T07:33:00.451 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:33:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TUYZ0psjRhNz3uX3Mvr2BIc+oYA=', 'Content-Length': '0'} 2022-09-03T07:33:00.472 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000d2775fbcec82cc01-006313032c-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:33:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:00.474 DEBUG:rgw_multi.tests:created bucket=avysmw-76 2022-09-03T07:33:00.474 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:00.475 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:00.477 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:33:00.529 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:33:00.529 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:33:00.562 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T07:33:00.577 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:33:00.578 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:00.578 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:33:00.578 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:31:57.780536Z" 2022-09-03T07:33:00.579 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:00.579 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:00.580 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190380.469800_511.1", 2022-09-03T07:33:00.580 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:00.469800Z" 2022-09-03T07:33:00.580 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:00.580 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:00.581 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:33:00.581 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:33:00.582 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:00.582 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:00.582 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:33:00.582 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:33:00.583 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:00.583 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:33:00.585 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662190317.780536_375.1', 1: '1_1662190380.469800_511.1', 2: '', 3: ''} 2022-09-03T07:33:00.586 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T07:33:00.586 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:00.587 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:00.588 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:33:00.643 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:33:00.643 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:33:00.700 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:00.699+0000 7effe44f2600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:00.701 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:00.700+0000 7effe44f2600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:00.744 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:33:00.745 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:33:00.745 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:33:00.745 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:33:00.746 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:33:00.746 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:33:00.747 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:00.747 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.748 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:33:00.748 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:00.748 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:33:00.749 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:00.749 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:00.749 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:33:00.749 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:00.750 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:33:00.750 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:00.750 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:33:00.751 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:00.751 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.751 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.752 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:00.752 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:33:00.752 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:00.753 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:00.753 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190380.461325_508.1", 2022-09-03T07:33:00.754 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:00.755 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:00.755 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:00.755 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:00.461325Z", 2022-09-03T07:33:00.756 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:00.756 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.756 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.757 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:00.757 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:33:00.757 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:00.757 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:00.758 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:33:00.758 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:00.758 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:00.759 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:00.759 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:33:00.759 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:33:00.759 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.760 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.760 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:00.760 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:33:00.760 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:00.761 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:00.761 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:33:00.761 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:00.761 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:00.762 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:00.762 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:33:00.762 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:33:00.762 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.763 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:00.763 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:33:00.763 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.763 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:33:00.764 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:33:00.764 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:33:00.764 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:00.765 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:33:00Z" 2022-09-03T07:33:00.765 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:33:00.766 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190380.461325_508.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:00.461325Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:33:00Z" } 2022-09-03T07:33:00.767 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190380.461325_508.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:00.461325Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:33:00.767 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190380.469800_511.1', 2: '', 3: ''} 2022-09-03T07:33:00.768 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190380.461325_508.1', 2: '', 3: ''} 2022-09-03T07:33:00.768 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190380.469800_511.1 target=1_1662190380.461325_508.1 2022-09-03T07:33:05.770 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:05.770 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:05.771 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:33:05.837 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:33:05.838 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:33:05.901 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:05.900+0000 7f615724c600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:05.904 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:05.903+0000 7f615724c600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:05.921 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:33:05.921 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:33:05.922 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:33:05.922 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:33:05.922 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:33:05.923 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:33:05.923 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:05.923 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:05.923 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:33:05.924 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:05.924 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:33:05.924 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:05.925 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:05.925 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:33:05.925 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:05.925 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:33:05.926 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:05.926 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:33:05.927 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:05.927 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:05.928 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:05.928 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:05.928 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:33:05.929 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:05.929 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:05.929 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190380.469800_511.1", 2022-09-03T07:33:05.930 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:05.930 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:05.930 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:05.930 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:00.469800Z", 2022-09-03T07:33:05.931 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:05.931 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:05.932 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:05.932 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:05.933 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:33:05.933 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:05.933 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:05.934 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:33:05.934 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:05.934 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:05.935 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:05.935 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:33:05.935 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:33:05.936 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:05.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:05.936 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:05.936 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:33:05.937 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:05.937 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:05.937 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:33:05.938 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:05.938 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:05.938 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:05.939 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:33:05.939 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:33:05.939 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:05.940 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:05.940 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:33:05.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:05.940 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:33:05.941 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:33:05.941 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:33:05.941 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:05.941 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:33:05Z" 2022-09-03T07:33:05.942 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:33:05.943 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190380.469800_511.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:00.469800Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:33:05Z" } 2022-09-03T07:33:05.944 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190380.469800_511.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:00.469800Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:33:05.945 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190380.469800_511.1', 2: '', 3: ''} 2022-09-03T07:33:05.945 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190380.469800_511.1', 2: '', 3: ''} 2022-09-03T07:33:05.946 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T07:33:05.946 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T07:33:05.947 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:05.947 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:05.948 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:33:06.006 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:33:06.006 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:33:06.054 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:33:06.053+0000 7fe51e22f600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:06.055 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:33:06.054+0000 7fe51e22f600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:06.072 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:33:06.072 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:33:06.072 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:33:06.073 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:33:06.073 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:33:06.073 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:33:06.073 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:33:06.074 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:06.074 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:33:06.074 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:06.075 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:33:06.075 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:33:06.075 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:33:06.075 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:33:06.076 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:33:06.077 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T07:33:06.077 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:33:06.077 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:33:06.078 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:33:06.078 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:06.078 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:06.078 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:06.079 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:33:06.079 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:33:06.079 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:33:06.079 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190380.469800_511.1", 2022-09-03T07:33:06.080 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:33:06.080 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:33:06.080 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:33:06.080 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:33:00.469800Z", 2022-09-03T07:33:06.081 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:33:06.081 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:06.081 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:06.081 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:06.082 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:33:06.082 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:33:06.082 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:33:06.082 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:33:06.083 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:33:06.083 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:33:06.083 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:33:06.083 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:33:06.084 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:33:06.084 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:06.084 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:06.084 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:33:06.085 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:33:06.085 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:33:06.085 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:33:06.086 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:33:06.086 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:33:06.086 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:33:06.086 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:33:06.087 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:33:06.087 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:33:06.087 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:06.087 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:33:06.088 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:33:06.088 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:06.088 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:33:06.088 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T07:33:06.089 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T07:33:06.089 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:33:06.089 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:33:06Z" 2022-09-03T07:33:06.089 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:33:06.091 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190380.469800_511.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:00.469800Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:33:06Z" } 2022-09-03T07:33:06.091 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190380.469800_511.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:00.469800Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:33:06.094 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190380.469800_511.1', 2: '', 3: ''} 2022-09-03T07:33:06.095 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190380.469800_511.1', 2: '', 3: ''} 2022-09-03T07:33:06.095 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T07:33:06.096 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T07:33:06.097 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:06.098 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:33:06.099 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:33:06.156 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:33:06.156 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:33:06.201 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:06.200+0000 7f9976917600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:06.204 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:06.202+0000 7f9976917600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:33:06.243 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:33:06.244 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:33:06.244 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:33:06.244 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:33:06.245 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:33:06.245 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:33:06.245 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:06.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:06.246 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:33:06.247 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:06.247 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:33:06.247 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:06.247 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:06.248 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:33:06.248 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:06.248 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:33:06.248 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:06.249 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:33:06.249 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:06.249 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:06.250 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:06.250 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:06.251 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:33:06.251 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:06.252 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:06.252 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190380.469800_511.1", 2022-09-03T07:33:06.253 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:06.253 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:06.254 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:06.254 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:00.469800Z", 2022-09-03T07:33:06.254 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:33:06.255 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:06.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:06.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:06.267 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:33:06.268 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:06.268 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:06.268 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:33:06.268 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:06.269 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:06.269 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:06.269 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:33:06.269 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:33:06.270 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:06.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:06.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:33:06.270 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:33:06.271 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:33:06.271 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:33:06.271 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:33:06.272 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:33:06.272 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:33:06.272 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:33:06.272 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:33:06.273 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:33:06.273 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:06.273 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:33:06.273 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:33:06.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:06.274 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:33:06.274 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:33:06.274 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:33:06.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:33:06.275 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:33:06Z" 2022-09-03T07:33:06.275 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:33:06.276 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190380.469800_511.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:00.469800Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:33:06Z" } 2022-09-03T07:33:06.277 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190380.469800_511.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:00.469800Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:33:06.278 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190380.469800_511.1', 2: '', 3: ''} 2022-09-03T07:33:06.278 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190380.469800_511.1', 2: '', 3: ''} 2022-09-03T07:33:06.279 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T07:33:06.279 DEBUG:tasks.rgw.c2.client.0:waiting for process to exit 2022-09-03T07:33:06.280 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T07:33:06.281 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T07:33:06.279+0000 7f4c403a4700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm (PID: 156420) UID: 0 2022-09-03T07:33:06.281 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T07:33:06.279+0000 7f4c4460f600 -1 shutting down 2022-09-03T07:33:08.893 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~0s 2022-09-03T07:33:12.382 INFO:tasks.rgw.c2.client.0:Stopped 2022-09-03T07:33:12.383 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:33:12.384 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:33:12.384 DEBUG:boto:Method: HEAD 2022-09-03T07:33:12.385 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:33:12.385 DEBUG:boto:Data: 2022-09-03T07:33:12.386 DEBUG:boto:Headers: {} 2022-09-03T07:33:12.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.387 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.388 DEBUG:boto:Params: {} 2022-09-03T07:33:12.389 DEBUG:boto:Token: None 2022-09-03T07:33:12.389 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/ 2022-09-03T07:33:12.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY= 2022-09-03T07:33:12.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY=', 'Content-Length': '0'} 2022-09-03T07:33:12.394 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000c45ffc74e1f1f117-0063130338-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.394 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:33:12.395 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:33:12.396 DEBUG:boto:Method: PUT 2022-09-03T07:33:12.396 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:33:12.397 DEBUG:boto:Data: 2022-09-03T07:33:12.397 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:33:12.398 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.398 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.399 DEBUG:boto:Params: {} 2022-09-03T07:33:12.399 DEBUG:boto:Token: None 2022-09-03T07:33:12.400 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/obj11 2022-09-03T07:33:12.401 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:YK1/Ooa+M8dyb5WKRLms7DNgKpA= 2022-09-03T07:33:12.402 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:YK1/Ooa+M8dyb5WKRLms7DNgKpA='} 2022-09-03T07:33:12.411 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000007f18bffee768c752-0063130338-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.412 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:33:12.413 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:33:12.413 DEBUG:boto:Method: HEAD 2022-09-03T07:33:12.414 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:33:12.415 DEBUG:boto:Data: 2022-09-03T07:33:12.415 DEBUG:boto:Headers: {} 2022-09-03T07:33:12.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.416 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.417 DEBUG:boto:Params: {} 2022-09-03T07:33:12.417 DEBUG:boto:Token: None 2022-09-03T07:33:12.418 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/ 2022-09-03T07:33:12.419 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY= 2022-09-03T07:33:12.420 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY=', 'Content-Length': '0'} 2022-09-03T07:33:12.423 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000865dcccfd6edb7d8-0063130338-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.424 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:33:12.424 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:33:12.425 DEBUG:boto:Method: PUT 2022-09-03T07:33:12.425 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:33:12.426 DEBUG:boto:Data: 2022-09-03T07:33:12.426 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:33:12.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.427 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.428 DEBUG:boto:Params: {} 2022-09-03T07:33:12.429 DEBUG:boto:Token: None 2022-09-03T07:33:12.429 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/obj22 2022-09-03T07:33:12.430 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/ajtnf01jRaWOFth/heg2dGfflQ= 2022-09-03T07:33:12.431 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/ajtnf01jRaWOFth/heg2dGfflQ='} 2022-09-03T07:33:12.437 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000749d5777b93f708c-0063130338-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.439 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:33:12.440 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:33:12.440 DEBUG:boto:Method: HEAD 2022-09-03T07:33:12.441 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:33:12.442 DEBUG:boto:Data: 2022-09-03T07:33:12.442 DEBUG:boto:Headers: {} 2022-09-03T07:33:12.443 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.443 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.444 DEBUG:boto:Params: {} 2022-09-03T07:33:12.445 DEBUG:boto:Token: None 2022-09-03T07:33:12.446 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/ 2022-09-03T07:33:12.446 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY= 2022-09-03T07:33:12.447 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY=', 'Content-Length': '0'} 2022-09-03T07:33:12.450 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000fc969f2e9702e5d0-0063130338-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.451 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:33:12.452 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:33:12.453 DEBUG:boto:Method: PUT 2022-09-03T07:33:12.454 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:33:12.455 DEBUG:boto:Data: 2022-09-03T07:33:12.455 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:33:12.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.456 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.457 DEBUG:boto:Params: {} 2022-09-03T07:33:12.457 DEBUG:boto:Token: None 2022-09-03T07:33:12.458 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/obj33 2022-09-03T07:33:12.459 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:6K8CAy5nLXvr/p3B+BgB5qWE9fQ= 2022-09-03T07:33:12.460 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:6K8CAy5nLXvr/p3B+BgB5qWE9fQ='} 2022-09-03T07:33:12.507 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000324fd3c86473885c-0063130338-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.508 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:33:12.509 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:33:12.510 DEBUG:boto:Method: HEAD 2022-09-03T07:33:12.510 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:33:12.511 DEBUG:boto:Data: 2022-09-03T07:33:12.512 DEBUG:boto:Headers: {} 2022-09-03T07:33:12.513 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.513 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.514 DEBUG:boto:Params: {} 2022-09-03T07:33:12.515 DEBUG:boto:Token: None 2022-09-03T07:33:12.515 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/ 2022-09-03T07:33:12.516 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY= 2022-09-03T07:33:12.516 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY=', 'Content-Length': '0'} 2022-09-03T07:33:12.519 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '9'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000006f5b04dac3cd32c5-0063130338-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.520 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:33:12.521 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:33:12.521 DEBUG:boto:Method: PUT 2022-09-03T07:33:12.524 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:33:12.525 DEBUG:boto:Data: 2022-09-03T07:33:12.526 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:33:12.526 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.527 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.527 DEBUG:boto:Params: {} 2022-09-03T07:33:12.528 DEBUG:boto:Token: None 2022-09-03T07:33:12.528 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/obj44 2022-09-03T07:33:12.529 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:G57jS3QIQEKbD7A5zZSeAK5ngc0= 2022-09-03T07:33:12.530 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:G57jS3QIQEKbD7A5zZSeAK5ngc0='} 2022-09-03T07:33:12.538 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000aff34b705bf121b0-0063130338-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.539 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:33:12.539 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:33:12.540 DEBUG:boto:Method: HEAD 2022-09-03T07:33:12.540 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:33:12.541 DEBUG:boto:Data: 2022-09-03T07:33:12.542 DEBUG:boto:Headers: {} 2022-09-03T07:33:12.544 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.545 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.546 DEBUG:boto:Params: {} 2022-09-03T07:33:12.546 DEBUG:boto:Token: None 2022-09-03T07:33:12.548 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/ 2022-09-03T07:33:12.548 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY= 2022-09-03T07:33:12.549 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lisphUL65nVQsg1PyJujhI+xHgY=', 'Content-Length': '0'} 2022-09-03T07:33:12.553 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003bd3d8f533431821-0063130338-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.553 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:33:12.554 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:33:12.555 DEBUG:boto:Method: PUT 2022-09-03T07:33:12.555 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:33:12.555 DEBUG:boto:Data: 2022-09-03T07:33:12.556 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:33:12.556 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:33:12.557 DEBUG:boto:Port: 8000 2022-09-03T07:33:12.557 DEBUG:boto:Params: {} 2022-09-03T07:33:12.558 DEBUG:boto:Token: None 2022-09-03T07:33:12.559 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:33:12 GMT /avysmw-76/obj55 2022-09-03T07:33:12.559 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:52MUOsDCTxfU39ctAIGoPB4av98= 2022-09-03T07:33:12.560 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:33:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:52MUOsDCTxfU39ctAIGoPB4av98='} 2022-09-03T07:33:12.606 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000005675eb3f557c339-0063130338-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:33:12.608 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'init', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:33:12.610 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'init', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:33:12.611 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync init --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:33:12.668 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:33:12.668 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:33:12.741 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:12.739+0000 7fae7fc9d600 10 RGW-SYNC:data:init_data_sync_status: took lease 2022-09-03T07:33:12.776 INFO:tasks.rgw.c2.client.0:Restarting daemon 2022-09-03T07:33:12.777 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2022-09-03T07:33:12.780 INFO:tasks.rgw.c2.client.0:Started 2022-09-03T07:33:12.781 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:33:12.802 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:33:12.802 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:33:12.804 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T07:33:12.806 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T07:33:13.807 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:33:13.831 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:33:13.832 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:33:13.833 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 208k 0 --:--:-- --:--:-- --:--:-- 208k 2022-09-03T07:33:13.835 INFO:teuthology.orchestra.run.smithi087.stdout:anonymous 2022-09-03T07:33:13.836 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-76 2022-09-03T07:33:13.837 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:33:13.838 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:33:13.839 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-76 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:33:13.935 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:33:13.936 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:33:14.019 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:14.018+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:33:14.599 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 has been restored 2022-09-03T07:33:19.020 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:19.018+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:33:24.022 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:24.019+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:33:29.021 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:29.020+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:33:34.021 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:34.020+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:33:39.022 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:39.020+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:33:44.022 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:44.020+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:33:49.023 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:49.021+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:33:54.023 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:54.021+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:33:59.023 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:33:59.022+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:04.025 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:04.023+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:09.024 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:09.023+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:14.027 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:14.024+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:19.027 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:19.025+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:24.034 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:24.025+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:29.027 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:29.026+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:34.028 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:34.026+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:39.028 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:39.026+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:44.029 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:44.027+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:49.029 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:49.028+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:54.029 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:54.028+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:34:59.030 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:34:59.029+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:35:04.030 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:04.029+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:35:09.031 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:09.029+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:35:14.033 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:14.031+0000 7fdf6ddd6600 1 waiting to reach incremental sync.. 2022-09-03T07:35:19.038 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:19.036+0000 7fdf6ddd6600 1 bucket sync caught up with source: 2022-09-03T07:35:19.038 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.100.5, 00000000002.85.5, 00000000002.84.5] 2022-09-03T07:35:19.039 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.100.5, 00000000002.85.5, 00000000002.84.5] 2022-09-03T07:35:19.039 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:19.036+0000 7fdf6ddd6600 0 bucket checkpoint complete 2022-09-03T07:35:19.070 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone3 bucket=avysmw-76 2022-09-03T07:35:19.072 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:19.073 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:19.075 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-76 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:35:19.135 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:35:19.135 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:35:19.185 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:19.184+0000 7fe408b0e600 1 bucket sync caught up with source: 2022-09-03T07:35:19.185 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.100.5, 00000000002.85.5, 00000000002.84.5] 2022-09-03T07:35:19.185 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.100.5, 00000000002.85.5, 00000000002.84.5] 2022-09-03T07:35:19.186 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:19.184+0000 7fe408b0e600 0 bucket checkpoint complete 2022-09-03T07:35:19.199 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone4 bucket=avysmw-76 2022-09-03T07:35:19.200 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:19.202 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:19.203 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-76 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:35:19.267 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:35:19.267 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:35:19.319 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:19.315+0000 7fab18c52600 1 bucket sync caught up with source: 2022-09-03T07:35:19.319 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.100.5, 00000000002.85.5, 00000000002.84.5] 2022-09-03T07:35:19.320 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.100.5, 00000000002.85.5, 00000000002.84.5] 2022-09-03T07:35:19.321 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:19.315+0000 7fab18c52600 0 bucket checkpoint complete 2022-09-03T07:35:19.331 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone1 bucket=avysmw-76 2022-09-03T07:35:19.332 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:19.333 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:19.334 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-76 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:35:19.394 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:35:19.394 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:35:19.435 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:19.434+0000 7f66411d4600 1 waiting for incremental sync to catch up: 2022-09-03T07:35:19.436 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000001.286.5] 2022-09-03T07:35:19.436 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000002.288.5] 2022-09-03T07:35:24.437 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:24.435+0000 7f66411d4600 1 waiting for incremental sync to catch up: 2022-09-03T07:35:24.438 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000001.286.5] 2022-09-03T07:35:24.441 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000002.288.5] 2022-09-03T07:35:29.438 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:29.437+0000 7f66411d4600 1 waiting for incremental sync to catch up: 2022-09-03T07:35:29.439 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000001.286.5] 2022-09-03T07:35:29.439 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000002.288.5] 2022-09-03T07:35:34.441 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:34.438+0000 7f66411d4600 1 waiting for incremental sync to catch up: 2022-09-03T07:35:34.441 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000001.286.5] 2022-09-03T07:35:34.442 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000002.288.5] 2022-09-03T07:35:39.441 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:39.439+0000 7f66411d4600 1 bucket sync caught up with source: 2022-09-03T07:35:39.442 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000002.288.5] 2022-09-03T07:35:39.442 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000002.288.5] 2022-09-03T07:35:39.442 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:39.439+0000 7f66411d4600 0 bucket checkpoint complete 2022-09-03T07:35:39.461 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone3 bucket=avysmw-76 2022-09-03T07:35:39.463 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:39.465 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:39.466 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-76 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:35:39.532 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:35:39.532 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:35:39.575 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:39.572+0000 7f2fa7450600 1 bucket sync caught up with source: 2022-09-03T07:35:39.575 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000002.288.5] 2022-09-03T07:35:39.575 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.606.5, 00000000002.270.5, 00000000002.288.5] 2022-09-03T07:35:39.575 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:39.572+0000 7f2fa7450600 0 bucket checkpoint complete 2022-09-03T07:35:39.592 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone2 target=test-zone4 bucket=avysmw-76 2022-09-03T07:35:39.593 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone1 bucket=avysmw-76 2022-09-03T07:35:39.594 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:39.594 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:39.595 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket avysmw-76 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2022-09-03T07:35:39.654 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:35:39.655 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:35:40.701 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:40.697+0000 7fcb2080c600 1 bucket sync caught up with source: 2022-09-03T07:35:40.702 INFO:teuthology.orchestra.run.smithi062.stderr: local status: [, , , , , , , , 00000000001.303.5, 00000000002.331.5, 00000000002.160.5] 2022-09-03T07:35:40.702 INFO:teuthology.orchestra.run.smithi062.stderr: remote markers: [, , , , , , , , 00000000001.303.5, 00000000002.331.5, 00000000002.160.5] 2022-09-03T07:35:40.703 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:35:40.697+0000 7fcb2080c600 0 bucket checkpoint complete 2022-09-03T07:35:40.719 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone2 bucket=avysmw-76 2022-09-03T07:35:40.720 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:40.720 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-76', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:35:40.721 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-76 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:35:40.782 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:35:40.782 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:35:40.841 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:40.839+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:35:45.841 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:45.840+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:35:50.842 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:50.840+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:35:55.842 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:35:55.841+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:00.843 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:00.841+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:05.847 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:05.842+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:10.844 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:10.842+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:15.847 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:15.843+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:20.845 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:20.843+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:25.845 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:25.845+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:30.847 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:30.844+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:35.847 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:35.845+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:40.847 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:40.846+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:45.848 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:45.846+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:50.848 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:50.847+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:36:55.849 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:36:55.847+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:00.849 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:00.848+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:05.850 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:05.849+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:10.850 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:10.849+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:15.851 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:15.849+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:20.852 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:20.850+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:25.852 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:25.850+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:30.852 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:30.851+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:35.853 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:35.851+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:40.854 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:40.852+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:45.854 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:45.852+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:50.855 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:50.853+0000 7fbae53e3600 1 waiting to reach incremental sync.. 2022-09-03T07:37:55.856 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:55.855+0000 7fbae53e3600 1 bucket sync caught up with source: 2022-09-03T07:37:55.857 INFO:teuthology.orchestra.run.smithi087.stderr: local status: [, , , , , , , , 00000000001.303.5, 00000000002.331.5, 00000000002.160.5] 2022-09-03T07:37:55.857 INFO:teuthology.orchestra.run.smithi087.stderr: remote markers: [, , , , , , , , 00000000001.303.5, 00000000002.331.5, 00000000002.160.5] 2022-09-03T07:37:55.857 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:37:55.855+0000 7fbae53e3600 0 bucket checkpoint complete 2022-09-03T07:37:55.872 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone3 target=test-zone4 bucket=avysmw-76 2022-09-03T07:37:55.874 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-76 zones={test-zone2, test-zone1} 2022-09-03T07:37:55.875 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:55.876 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:55.876 DEBUG:boto:Method: HEAD 2022-09-03T07:37:55.877 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:37:55.877 DEBUG:boto:Data: 2022-09-03T07:37:55.878 DEBUG:boto:Headers: {} 2022-09-03T07:37:55.879 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:55.879 DEBUG:boto:Port: 8000 2022-09-03T07:37:55.880 DEBUG:boto:Params: {} 2022-09-03T07:37:55.881 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:37:55.882 DEBUG:boto:Token: None 2022-09-03T07:37:55.883 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:55 GMT /avysmw-76/ 2022-09-03T07:37:55.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZYwPxOXvcd/GBxaJozXJWyLjA5Q= 2022-09-03T07:37:55.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZYwPxOXvcd/GBxaJozXJWyLjA5Q=', 'Content-Length': '0'} 2022-09-03T07:37:55.890 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004cc5a3d7d3fa39c8-0063130453-1b17-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:37:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:55.891 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:55.892 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:55.892 DEBUG:boto:Method: HEAD 2022-09-03T07:37:55.893 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:37:55.893 DEBUG:boto:Data: 2022-09-03T07:37:55.894 DEBUG:boto:Headers: {} 2022-09-03T07:37:55.895 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:55.895 DEBUG:boto:Port: 8000 2022-09-03T07:37:55.896 DEBUG:boto:Params: {} 2022-09-03T07:37:55.896 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:37:55.897 DEBUG:boto:Token: None 2022-09-03T07:37:55.898 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:55 GMT /avysmw-76/ 2022-09-03T07:37:55.898 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZYwPxOXvcd/GBxaJozXJWyLjA5Q= 2022-09-03T07:37:55.899 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZYwPxOXvcd/GBxaJozXJWyLjA5Q=', 'Content-Length': '0'} 2022-09-03T07:37:55.904 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000042e79428d7c2d86d-0063130453-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:37:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:55.905 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:37:55.906 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:55.906 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:55.907 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.908 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.908 DEBUG:boto:Method: GET 2022-09-03T07:37:55.909 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.909 DEBUG:boto:Data: 2022-09-03T07:37:55.910 DEBUG:boto:Headers: {} 2022-09-03T07:37:55.910 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:55.911 DEBUG:boto:Port: 8000 2022-09-03T07:37:55.911 DEBUG:boto:Params: {} 2022-09-03T07:37:55.912 DEBUG:boto:Token: None 2022-09-03T07:37:55.913 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:55 GMT /avysmw-76/?versions 2022-09-03T07:37:55.913 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:20wp6S7g/I5wl6ydvh64TzWHGIg= 2022-09-03T07:37:55.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:20wp6S7g/I5wl6ydvh64TzWHGIg=', 'Content-Length': '0'} 2022-09-03T07:37:55.917 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004604e6f23f99e65e-0063130453-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:55.918 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:55.920 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:37:55.921 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:37:55.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:37:55.923 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:37:55.924 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:37:55.924 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:37:55.925 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:55.926 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:55.927 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.927 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.928 DEBUG:boto:Method: GET 2022-09-03T07:37:55.928 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.929 DEBUG:boto:Data: 2022-09-03T07:37:55.929 DEBUG:boto:Headers: {} 2022-09-03T07:37:55.930 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:55.930 DEBUG:boto:Port: 8000 2022-09-03T07:37:55.931 DEBUG:boto:Params: {} 2022-09-03T07:37:55.931 DEBUG:boto:Token: None 2022-09-03T07:37:55.932 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:55 GMT /avysmw-76/?versions 2022-09-03T07:37:55.932 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:20wp6S7g/I5wl6ydvh64TzWHGIg= 2022-09-03T07:37:55.933 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:20wp6S7g/I5wl6ydvh64TzWHGIg=', 'Content-Length': '0'} 2022-09-03T07:37:55.936 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a3b76c5d2f7b9f00-0063130453-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:55.937 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:55.939 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:37:55.940 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:37:55.941 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:37:55.942 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:37:55.942 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:37:55.943 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:55.944 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:55.945 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.946 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.947 DEBUG:boto:Method: GET 2022-09-03T07:37:55.948 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.949 DEBUG:boto:Data: 2022-09-03T07:37:55.950 DEBUG:boto:Headers: {} 2022-09-03T07:37:55.951 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:55.951 DEBUG:boto:Port: 8000 2022-09-03T07:37:55.952 DEBUG:boto:Params: {} 2022-09-03T07:37:55.953 DEBUG:boto:Token: None 2022-09-03T07:37:55.954 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:55 GMT /avysmw-76/?versions 2022-09-03T07:37:55.955 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:20wp6S7g/I5wl6ydvh64TzWHGIg= 2022-09-03T07:37:55.956 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:20wp6S7g/I5wl6ydvh64TzWHGIg=', 'Content-Length': '0'} 2022-09-03T07:37:55.960 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005c917f3cb62c642e-0063130453-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:55.961 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:55.962 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:55.963 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:55.963 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.964 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.964 DEBUG:boto:Method: GET 2022-09-03T07:37:55.965 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:55.966 DEBUG:boto:Data: 2022-09-03T07:37:55.966 DEBUG:boto:Headers: {} 2022-09-03T07:37:55.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:55.967 DEBUG:boto:Port: 8000 2022-09-03T07:37:55.968 DEBUG:boto:Params: {} 2022-09-03T07:37:55.968 DEBUG:boto:Token: None 2022-09-03T07:37:55.969 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:55 GMT /avysmw-76/?versions 2022-09-03T07:37:55.969 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:20wp6S7g/I5wl6ydvh64TzWHGIg= 2022-09-03T07:37:55.970 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:20wp6S7g/I5wl6ydvh64TzWHGIg=', 'Content-Length': '0'} 2022-09-03T07:37:55.973 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008abcee04c86b74a6-0063130453-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:55.974 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:55.975 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:37:55.975 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:55.976 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:55.977 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.977 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.978 DEBUG:boto:Method: GET 2022-09-03T07:37:55.978 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.979 DEBUG:boto:Data: 2022-09-03T07:37:55.979 DEBUG:boto:Headers: {} 2022-09-03T07:37:55.980 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:55.980 DEBUG:boto:Port: 8000 2022-09-03T07:37:55.981 DEBUG:boto:Params: {} 2022-09-03T07:37:55.981 DEBUG:boto:Token: None 2022-09-03T07:37:55.982 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:55 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gWyiUcVWQegMzlfrXNMwfGBGX5c= 2022-09-03T07:37:55.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gWyiUcVWQegMzlfrXNMwfGBGX5c=', 'Content-Length': '0'} 2022-09-03T07:37:55.985 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e01331ec981c2bd7-0063130453-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:55.986 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:55.987 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:55.987 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.988 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.988 DEBUG:boto:Method: GET 2022-09-03T07:37:55.989 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.989 DEBUG:boto:Data: 2022-09-03T07:37:55.990 DEBUG:boto:Headers: {} 2022-09-03T07:37:55.990 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:55.991 DEBUG:boto:Port: 8000 2022-09-03T07:37:55.991 DEBUG:boto:Params: {} 2022-09-03T07:37:55.992 DEBUG:boto:Token: None 2022-09-03T07:37:55.993 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:55 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.993 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gWyiUcVWQegMzlfrXNMwfGBGX5c= 2022-09-03T07:37:55.994 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:55 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gWyiUcVWQegMzlfrXNMwfGBGX5c=', 'Content-Length': '0'} 2022-09-03T07:37:55.996 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000108c109074f70de4-0063130453-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:55 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:55.997 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:55.997 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:55.998 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.998 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:55.999 DEBUG:boto:Method: HEAD 2022-09-03T07:37:55.999 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.000 DEBUG:boto:Data: 2022-09-03T07:37:56.001 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.001 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.002 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.002 DEBUG:boto:Params: {} 2022-09-03T07:37:56.003 DEBUG:boto:Token: None 2022-09-03T07:37:56.003 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qsEfi0U4S7gMK4rO8LYqrukpOVU= 2022-09-03T07:37:56.005 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qsEfi0U4S7gMK4rO8LYqrukpOVU=', 'Content-Length': '0'} 2022-09-03T07:37:56.006 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000033df00c01bd93000-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.007 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:56.008 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:56.008 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.009 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.009 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.010 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.010 DEBUG:boto:Data: 2022-09-03T07:37:56.011 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.011 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.012 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.012 DEBUG:boto:Params: {} 2022-09-03T07:37:56.013 DEBUG:boto:Token: None 2022-09-03T07:37:56.013 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qsEfi0U4S7gMK4rO8LYqrukpOVU= 2022-09-03T07:37:56.015 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qsEfi0U4S7gMK4rO8LYqrukpOVU=', 'Content-Length': '0'} 2022-09-03T07:37:56.017 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cfade1ba02d9c01-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.018 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:37:56.018 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:56.019 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:56.019 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.020 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.020 DEBUG:boto:Method: GET 2022-09-03T07:37:56.021 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.021 DEBUG:boto:Data: 2022-09-03T07:37:56.022 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.022 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.023 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.023 DEBUG:boto:Params: {} 2022-09-03T07:37:56.024 DEBUG:boto:Token: None 2022-09-03T07:37:56.025 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.026 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttKheDNgjQDUj20F0kGsO7Q4kug= 2022-09-03T07:37:56.026 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttKheDNgjQDUj20F0kGsO7Q4kug=', 'Content-Length': '0'} 2022-09-03T07:37:56.028 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f4f732d5ce7c0144-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.029 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:56.030 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:56.030 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.031 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.031 DEBUG:boto:Method: GET 2022-09-03T07:37:56.032 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.032 DEBUG:boto:Data: 2022-09-03T07:37:56.033 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.033 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.034 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.034 DEBUG:boto:Params: {} 2022-09-03T07:37:56.035 DEBUG:boto:Token: None 2022-09-03T07:37:56.036 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:56.036 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ttKheDNgjQDUj20F0kGsO7Q4kug= 2022-09-03T07:37:56.037 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ttKheDNgjQDUj20F0kGsO7Q4kug=', 'Content-Length': '0'} 2022-09-03T07:37:56.039 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a16f366b9301705a-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.040 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.041 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.041 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:56.042 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:56.042 DEBUG:boto:Method: GET 2022-09-03T07:37:56.043 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:56.043 DEBUG:boto:Data: 2022-09-03T07:37:56.044 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.044 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.045 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.045 DEBUG:boto:Params: {} 2022-09-03T07:37:56.046 DEBUG:boto:Token: None 2022-09-03T07:37:56.047 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.047 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.048 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.051 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000af0b28994f03099c-0063130454-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.052 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.053 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.054 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.054 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:56.055 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:56.055 DEBUG:boto:Method: GET 2022-09-03T07:37:56.056 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:56.056 DEBUG:boto:Data: 2022-09-03T07:37:56.057 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.058 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.058 DEBUG:boto:Params: {} 2022-09-03T07:37:56.059 DEBUG:boto:Token: None 2022-09-03T07:37:56.060 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.060 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.061 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.064 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000001370c716d47aa73d-0063130454-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.064 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.065 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:56.066 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:56.066 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.067 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:56.068 DEBUG:boto:Data: 2022-09-03T07:37:56.068 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.069 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.069 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.070 DEBUG:boto:Params: {} 2022-09-03T07:37:56.070 DEBUG:boto:Token: None 2022-09-03T07:37:56.071 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj11 2022-09-03T07:37:56.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/PM1tl+IkH4iyCfM0wIylqF8IkM= 2022-09-03T07:37:56.072 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/PM1tl+IkH4iyCfM0wIylqF8IkM=', 'Content-Length': '0'} 2022-09-03T07:37:56.075 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c9bea228de193a47-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.076 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:56.076 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:56.077 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.077 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:56.078 DEBUG:boto:Data: 2022-09-03T07:37:56.078 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.079 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.080 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.080 DEBUG:boto:Params: {} 2022-09-03T07:37:56.081 DEBUG:boto:Token: None 2022-09-03T07:37:56.081 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj11 2022-09-03T07:37:56.082 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/PM1tl+IkH4iyCfM0wIylqF8IkM= 2022-09-03T07:37:56.083 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/PM1tl+IkH4iyCfM0wIylqF8IkM=', 'Content-Length': '0'} 2022-09-03T07:37:56.085 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000109621471e6d462f-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.086 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:37:56.087 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:56.087 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:56.088 DEBUG:boto:Method: GET 2022-09-03T07:37:56.089 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:56.089 DEBUG:boto:Data: 2022-09-03T07:37:56.090 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.090 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.091 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.091 DEBUG:boto:Params: {} 2022-09-03T07:37:56.092 DEBUG:boto:Token: None 2022-09-03T07:37:56.093 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj11 2022-09-03T07:37:56.094 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vsHQfGGDmUkLk9Pd4iM1TYv29k4= 2022-09-03T07:37:56.095 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vsHQfGGDmUkLk9Pd4iM1TYv29k4=', 'Content-Length': '0'} 2022-09-03T07:37:56.097 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004637ffae74078290-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.138 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:56.139 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:56.139 DEBUG:boto:Method: GET 2022-09-03T07:37:56.140 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:56.141 DEBUG:boto:Data: 2022-09-03T07:37:56.141 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.142 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.142 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.143 DEBUG:boto:Params: {} 2022-09-03T07:37:56.144 DEBUG:boto:Token: None 2022-09-03T07:37:56.145 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj11 2022-09-03T07:37:56.145 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vsHQfGGDmUkLk9Pd4iM1TYv29k4= 2022-09-03T07:37:56.146 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vsHQfGGDmUkLk9Pd4iM1TYv29k4=', 'Content-Length': '0'} 2022-09-03T07:37:56.149 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000081930eb3b50aee2-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.150 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.151 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.151 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:56.152 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:56.152 DEBUG:boto:Method: GET 2022-09-03T07:37:56.153 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:56.153 DEBUG:boto:Data: 2022-09-03T07:37:56.154 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.155 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.155 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.156 DEBUG:boto:Params: {} 2022-09-03T07:37:56.157 DEBUG:boto:Token: None 2022-09-03T07:37:56.158 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.159 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.159 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.163 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008d368e503c9a6f83-0063130454-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.163 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.164 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.165 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.165 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:56.166 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:56.167 DEBUG:boto:Method: GET 2022-09-03T07:37:56.167 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:56.168 DEBUG:boto:Data: 2022-09-03T07:37:56.169 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.170 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.170 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.171 DEBUG:boto:Params: {} 2022-09-03T07:37:56.172 DEBUG:boto:Token: None 2022-09-03T07:37:56.172 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.173 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.174 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.177 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004d6ccfa00fb43a43-0063130454-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.177 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.179 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:37:56.180 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.180 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.181 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.182 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.182 DEBUG:boto:Method: GET 2022-09-03T07:37:56.183 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.184 DEBUG:boto:Data: 2022-09-03T07:37:56.184 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.185 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.186 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.186 DEBUG:boto:Params: {} 2022-09-03T07:37:56.187 DEBUG:boto:Token: None 2022-09-03T07:37:56.188 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9KtJba0DGlM5URv0Vfmx4CgxmVc= 2022-09-03T07:37:56.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9KtJba0DGlM5URv0Vfmx4CgxmVc=', 'Content-Length': '0'} 2022-09-03T07:37:56.193 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000acd330f00719b3d6-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.194 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.195 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.196 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.197 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.197 DEBUG:boto:Method: GET 2022-09-03T07:37:56.198 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.199 DEBUG:boto:Data: 2022-09-03T07:37:56.199 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.200 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.201 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.202 DEBUG:boto:Params: {} 2022-09-03T07:37:56.202 DEBUG:boto:Token: None 2022-09-03T07:37:56.203 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.204 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9KtJba0DGlM5URv0Vfmx4CgxmVc= 2022-09-03T07:37:56.204 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9KtJba0DGlM5URv0Vfmx4CgxmVc=', 'Content-Length': '0'} 2022-09-03T07:37:56.207 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee99e568b8e49095-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.207 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.208 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.209 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.209 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.210 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.211 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.212 DEBUG:boto:Data: 2022-09-03T07:37:56.213 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.213 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.214 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.215 DEBUG:boto:Params: {} 2022-09-03T07:37:56.215 DEBUG:boto:Token: None 2022-09-03T07:37:56.216 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.217 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EdOgD2iMdqeH/hEzY9NTSMdXEUY= 2022-09-03T07:37:56.217 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EdOgD2iMdqeH/hEzY9NTSMdXEUY=', 'Content-Length': '0'} 2022-09-03T07:37:56.219 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057fbb661bd183f2c-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.221 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.222 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.223 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.224 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.225 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.226 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.228 DEBUG:boto:Data: 2022-09-03T07:37:56.230 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.231 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.232 DEBUG:boto:Params: {} 2022-09-03T07:37:56.232 DEBUG:boto:Token: None 2022-09-03T07:37:56.233 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.233 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:EdOgD2iMdqeH/hEzY9NTSMdXEUY= 2022-09-03T07:37:56.234 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:EdOgD2iMdqeH/hEzY9NTSMdXEUY=', 'Content-Length': '0'} 2022-09-03T07:37:56.237 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002dcac68b80377917-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.238 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:37:56.238 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.239 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.239 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.240 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.240 DEBUG:boto:Method: GET 2022-09-03T07:37:56.241 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.241 DEBUG:boto:Data: 2022-09-03T07:37:56.242 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.243 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.243 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.244 DEBUG:boto:Params: {} 2022-09-03T07:37:56.244 DEBUG:boto:Token: None 2022-09-03T07:37:56.245 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.245 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9KtJba0DGlM5URv0Vfmx4CgxmVc= 2022-09-03T07:37:56.246 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9KtJba0DGlM5URv0Vfmx4CgxmVc=', 'Content-Length': '0'} 2022-09-03T07:37:56.248 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000046f05e1c01a4d774-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.249 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.249 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.250 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.250 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.251 DEBUG:boto:Method: GET 2022-09-03T07:37:56.251 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.252 DEBUG:boto:Data: 2022-09-03T07:37:56.252 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.253 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.254 DEBUG:boto:Params: {} 2022-09-03T07:37:56.254 DEBUG:boto:Token: None 2022-09-03T07:37:56.255 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:56.256 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:9KtJba0DGlM5URv0Vfmx4CgxmVc= 2022-09-03T07:37:56.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:9KtJba0DGlM5URv0Vfmx4CgxmVc=', 'Content-Length': '0'} 2022-09-03T07:37:56.258 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000225c36fd1ecc02ac-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.259 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.260 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.260 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:56.261 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:56.261 DEBUG:boto:Method: GET 2022-09-03T07:37:56.262 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:56.262 DEBUG:boto:Data: 2022-09-03T07:37:56.263 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.264 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.264 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.265 DEBUG:boto:Params: {} 2022-09-03T07:37:56.265 DEBUG:boto:Token: None 2022-09-03T07:37:56.266 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.266 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.267 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.271 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ea1bcacf64d490c0-0063130454-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.271 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.272 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.273 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.273 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:56.274 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:56.274 DEBUG:boto:Method: GET 2022-09-03T07:37:56.275 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:56.275 DEBUG:boto:Data: 2022-09-03T07:37:56.276 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.277 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.277 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.278 DEBUG:boto:Params: {} 2022-09-03T07:37:56.279 DEBUG:boto:Token: None 2022-09-03T07:37:56.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.284 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c96ef183ffabbb43-0063130454-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.285 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.286 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.287 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.288 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.289 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:56.290 DEBUG:boto:Data: 2022-09-03T07:37:56.290 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.291 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.292 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.293 DEBUG:boto:Params: {} 2022-09-03T07:37:56.294 DEBUG:boto:Token: None 2022-09-03T07:37:56.294 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22 2022-09-03T07:37:56.295 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jSn3fqUydI1LUE8ITQVgBg2J5Jo= 2022-09-03T07:37:56.296 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jSn3fqUydI1LUE8ITQVgBg2J5Jo=', 'Content-Length': '0'} 2022-09-03T07:37:56.298 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003c6a650868754f39-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.298 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.299 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.300 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.301 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:56.301 DEBUG:boto:Data: 2022-09-03T07:37:56.302 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.302 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.303 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.303 DEBUG:boto:Params: {} 2022-09-03T07:37:56.304 DEBUG:boto:Token: None 2022-09-03T07:37:56.305 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22 2022-09-03T07:37:56.305 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jSn3fqUydI1LUE8ITQVgBg2J5Jo= 2022-09-03T07:37:56.306 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jSn3fqUydI1LUE8ITQVgBg2J5Jo=', 'Content-Length': '0'} 2022-09-03T07:37:56.308 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003f575193fdfd150c-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.309 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:37:56.309 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.310 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.311 DEBUG:boto:Method: GET 2022-09-03T07:37:56.312 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:56.313 DEBUG:boto:Data: 2022-09-03T07:37:56.314 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.314 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.315 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.315 DEBUG:boto:Params: {} 2022-09-03T07:37:56.316 DEBUG:boto:Token: None 2022-09-03T07:37:56.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22 2022-09-03T07:37:56.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W98WlieggsGGlwoTyXks7No+uE8= 2022-09-03T07:37:56.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W98WlieggsGGlwoTyXks7No+uE8=', 'Content-Length': '0'} 2022-09-03T07:37:56.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f17c60b1f372d34a-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.362 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:56.363 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:56.364 DEBUG:boto:Method: GET 2022-09-03T07:37:56.365 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:56.365 DEBUG:boto:Data: 2022-09-03T07:37:56.366 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.367 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.367 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.368 DEBUG:boto:Params: {} 2022-09-03T07:37:56.368 DEBUG:boto:Token: None 2022-09-03T07:37:56.369 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj22 2022-09-03T07:37:56.370 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W98WlieggsGGlwoTyXks7No+uE8= 2022-09-03T07:37:56.370 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W98WlieggsGGlwoTyXks7No+uE8=', 'Content-Length': '0'} 2022-09-03T07:37:56.373 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007ab33a0cae8534dc-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.374 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.374 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.375 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:56.375 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:56.376 DEBUG:boto:Method: GET 2022-09-03T07:37:56.376 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:56.377 DEBUG:boto:Data: 2022-09-03T07:37:56.378 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.378 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.379 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.379 DEBUG:boto:Params: {} 2022-09-03T07:37:56.380 DEBUG:boto:Token: None 2022-09-03T07:37:56.381 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.381 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.382 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.385 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009b3476feb6114d74-0063130454-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.386 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.387 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.388 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.388 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:56.389 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:56.389 DEBUG:boto:Method: GET 2022-09-03T07:37:56.390 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:56.391 DEBUG:boto:Data: 2022-09-03T07:37:56.391 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.392 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.392 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.393 DEBUG:boto:Params: {} 2022-09-03T07:37:56.394 DEBUG:boto:Token: None 2022-09-03T07:37:56.395 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.396 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.397 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.401 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a83242f056cae316-0063130454-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.405 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.406 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:37:56.408 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.409 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.410 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.411 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.412 DEBUG:boto:Method: GET 2022-09-03T07:37:56.413 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.414 DEBUG:boto:Data: 2022-09-03T07:37:56.415 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.416 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.416 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.417 DEBUG:boto:Params: {} 2022-09-03T07:37:56.418 DEBUG:boto:Token: None 2022-09-03T07:37:56.419 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.420 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zx8jGLwtOHII51FYw7XHVgv4PFI= 2022-09-03T07:37:56.421 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zx8jGLwtOHII51FYw7XHVgv4PFI=', 'Content-Length': '0'} 2022-09-03T07:37:56.423 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c610b2be4c4e4092-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.425 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.425 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.426 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.427 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.427 DEBUG:boto:Method: GET 2022-09-03T07:37:56.428 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.428 DEBUG:boto:Data: 2022-09-03T07:37:56.429 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.429 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.430 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.431 DEBUG:boto:Params: {} 2022-09-03T07:37:56.431 DEBUG:boto:Token: None 2022-09-03T07:37:56.432 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.433 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zx8jGLwtOHII51FYw7XHVgv4PFI= 2022-09-03T07:37:56.434 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zx8jGLwtOHII51FYw7XHVgv4PFI=', 'Content-Length': '0'} 2022-09-03T07:37:56.436 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001dddb50095b63593-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.437 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.437 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.438 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.438 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.439 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.439 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.443 DEBUG:boto:Data: 2022-09-03T07:37:56.443 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.444 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.445 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.445 DEBUG:boto:Params: {} 2022-09-03T07:37:56.446 DEBUG:boto:Token: None 2022-09-03T07:37:56.447 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.447 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EcljgbvFGNZ5AbIjKZwps5qPVU= 2022-09-03T07:37:56.448 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EcljgbvFGNZ5AbIjKZwps5qPVU=', 'Content-Length': '0'} 2022-09-03T07:37:56.450 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000019a904bb6847acab-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.451 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.451 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.452 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.452 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.453 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.453 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.454 DEBUG:boto:Data: 2022-09-03T07:37:56.454 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.456 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.457 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.459 DEBUG:boto:Params: {} 2022-09-03T07:37:56.460 DEBUG:boto:Token: None 2022-09-03T07:37:56.461 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.462 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8EcljgbvFGNZ5AbIjKZwps5qPVU= 2022-09-03T07:37:56.463 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8EcljgbvFGNZ5AbIjKZwps5qPVU=', 'Content-Length': '0'} 2022-09-03T07:37:56.466 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000511a84588364e4de-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.467 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:37:56.468 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.469 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.470 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.472 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.473 DEBUG:boto:Method: GET 2022-09-03T07:37:56.474 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.475 DEBUG:boto:Data: 2022-09-03T07:37:56.475 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.476 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.476 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.477 DEBUG:boto:Params: {} 2022-09-03T07:37:56.478 DEBUG:boto:Token: None 2022-09-03T07:37:56.479 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zx8jGLwtOHII51FYw7XHVgv4PFI= 2022-09-03T07:37:56.481 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zx8jGLwtOHII51FYw7XHVgv4PFI=', 'Content-Length': '0'} 2022-09-03T07:37:56.483 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc275aea06e8d3d3-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.485 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.486 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.487 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.488 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.489 DEBUG:boto:Method: GET 2022-09-03T07:37:56.491 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.492 DEBUG:boto:Data: 2022-09-03T07:37:56.493 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.494 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.495 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.496 DEBUG:boto:Params: {} 2022-09-03T07:37:56.497 DEBUG:boto:Token: None 2022-09-03T07:37:56.498 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:56.499 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Zx8jGLwtOHII51FYw7XHVgv4PFI= 2022-09-03T07:37:56.500 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Zx8jGLwtOHII51FYw7XHVgv4PFI=', 'Content-Length': '0'} 2022-09-03T07:37:56.503 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000f225bf2f4fc3e6a-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.504 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.504 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.505 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:56.506 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:56.506 DEBUG:boto:Method: GET 2022-09-03T07:37:56.507 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:56.507 DEBUG:boto:Data: 2022-09-03T07:37:56.508 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.508 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.509 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.509 DEBUG:boto:Params: {} 2022-09-03T07:37:56.510 DEBUG:boto:Token: None 2022-09-03T07:37:56.511 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.513 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.514 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.517 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003fd4c85356056768-0063130454-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.518 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.519 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.519 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.520 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:56.521 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:56.521 DEBUG:boto:Method: GET 2022-09-03T07:37:56.522 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:56.522 DEBUG:boto:Data: 2022-09-03T07:37:56.523 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.524 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.524 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.525 DEBUG:boto:Params: {} 2022-09-03T07:37:56.526 DEBUG:boto:Token: None 2022-09-03T07:37:56.527 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.527 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.528 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.532 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000051b01a67ba6c3473-0063130454-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.532 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.533 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.534 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.534 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.535 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:56.536 DEBUG:boto:Data: 2022-09-03T07:37:56.536 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.537 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.537 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.538 DEBUG:boto:Params: {} 2022-09-03T07:37:56.538 DEBUG:boto:Token: None 2022-09-03T07:37:56.539 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33 2022-09-03T07:37:56.540 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:raZlQsF+zZcxr7O0O6aY0TWeyu4= 2022-09-03T07:37:56.541 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:raZlQsF+zZcxr7O0O6aY0TWeyu4=', 'Content-Length': '0'} 2022-09-03T07:37:56.543 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000959e49003f1ec0a8-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.543 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.544 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.545 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.545 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:56.546 DEBUG:boto:Data: 2022-09-03T07:37:56.546 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.547 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.548 DEBUG:boto:Params: {} 2022-09-03T07:37:56.549 DEBUG:boto:Token: None 2022-09-03T07:37:56.549 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33 2022-09-03T07:37:56.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:raZlQsF+zZcxr7O0O6aY0TWeyu4= 2022-09-03T07:37:56.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:raZlQsF+zZcxr7O0O6aY0TWeyu4=', 'Content-Length': '0'} 2022-09-03T07:37:56.552 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004c3b21a1cb4eb1f8-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.553 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:37:56.554 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.555 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.555 DEBUG:boto:Method: GET 2022-09-03T07:37:56.556 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:56.556 DEBUG:boto:Data: 2022-09-03T07:37:56.557 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.557 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.558 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.559 DEBUG:boto:Params: {} 2022-09-03T07:37:56.559 DEBUG:boto:Token: None 2022-09-03T07:37:56.560 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33 2022-09-03T07:37:56.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1nKUzwVTv7S+QDLi8ukTrsPfPwg= 2022-09-03T07:37:56.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1nKUzwVTv7S+QDLi8ukTrsPfPwg=', 'Content-Length': '0'} 2022-09-03T07:37:56.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000038a73dad74e85a0f-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.564 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:56.565 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:56.566 DEBUG:boto:Method: GET 2022-09-03T07:37:56.567 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:56.567 DEBUG:boto:Data: 2022-09-03T07:37:56.568 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.569 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.570 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.570 DEBUG:boto:Params: {} 2022-09-03T07:37:56.571 DEBUG:boto:Token: None 2022-09-03T07:37:56.572 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj33 2022-09-03T07:37:56.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:1nKUzwVTv7S+QDLi8ukTrsPfPwg= 2022-09-03T07:37:56.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:1nKUzwVTv7S+QDLi8ukTrsPfPwg=', 'Content-Length': '0'} 2022-09-03T07:37:56.578 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000049a331aacb97ece9-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.618 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.619 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.621 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:56.622 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:56.623 DEBUG:boto:Method: GET 2022-09-03T07:37:56.624 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:56.625 DEBUG:boto:Data: 2022-09-03T07:37:56.626 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.627 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.628 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.629 DEBUG:boto:Params: {} 2022-09-03T07:37:56.632 DEBUG:boto:Token: None 2022-09-03T07:37:56.633 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.634 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.635 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.639 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b53434bd1816bcb0-0063130454-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.640 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.643 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.643 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.644 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:56.644 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:56.645 DEBUG:boto:Method: GET 2022-09-03T07:37:56.646 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:56.646 DEBUG:boto:Data: 2022-09-03T07:37:56.647 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.647 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.648 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.649 DEBUG:boto:Params: {} 2022-09-03T07:37:56.650 DEBUG:boto:Token: None 2022-09-03T07:37:56.652 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.653 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.654 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.657 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000cdcde982dc06ec75-0063130454-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.658 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.659 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:37:56.660 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.660 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.661 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.661 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.662 DEBUG:boto:Method: GET 2022-09-03T07:37:56.662 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.663 DEBUG:boto:Data: 2022-09-03T07:37:56.663 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.664 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.664 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.665 DEBUG:boto:Params: {} 2022-09-03T07:37:56.666 DEBUG:boto:Token: None 2022-09-03T07:37:56.666 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.667 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3W2e39Novtxo5xH6rTLNjO3IHeQ= 2022-09-03T07:37:56.668 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3W2e39Novtxo5xH6rTLNjO3IHeQ=', 'Content-Length': '0'} 2022-09-03T07:37:56.670 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000009a1381fc852a321-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.710 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.711 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.712 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.713 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.714 DEBUG:boto:Method: GET 2022-09-03T07:37:56.715 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.715 DEBUG:boto:Data: 2022-09-03T07:37:56.716 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.717 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.717 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.718 DEBUG:boto:Params: {} 2022-09-03T07:37:56.719 DEBUG:boto:Token: None 2022-09-03T07:37:56.719 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.720 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3W2e39Novtxo5xH6rTLNjO3IHeQ= 2022-09-03T07:37:56.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3W2e39Novtxo5xH6rTLNjO3IHeQ=', 'Content-Length': '0'} 2022-09-03T07:37:56.723 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009683cc3e739f6b7f-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.725 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.725 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.726 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.726 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.727 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.727 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.728 DEBUG:boto:Data: 2022-09-03T07:37:56.728 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.729 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.729 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.730 DEBUG:boto:Params: {} 2022-09-03T07:37:56.731 DEBUG:boto:Token: None 2022-09-03T07:37:56.732 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:izq1COGvjcyeSwUhPfA/n5HVLyo= 2022-09-03T07:37:56.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:izq1COGvjcyeSwUhPfA/n5HVLyo=', 'Content-Length': '0'} 2022-09-03T07:37:56.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000045303f703069f3f0-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.830 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.833 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.834 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.835 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.836 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.837 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.838 DEBUG:boto:Data: 2022-09-03T07:37:56.840 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.841 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.843 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.844 DEBUG:boto:Params: {} 2022-09-03T07:37:56.845 DEBUG:boto:Token: None 2022-09-03T07:37:56.846 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.846 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:izq1COGvjcyeSwUhPfA/n5HVLyo= 2022-09-03T07:37:56.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:izq1COGvjcyeSwUhPfA/n5HVLyo=', 'Content-Length': '0'} 2022-09-03T07:37:56.850 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006aeb53e753f72baf-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.851 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:37:56.852 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.853 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.853 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.854 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.855 DEBUG:boto:Method: GET 2022-09-03T07:37:56.856 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.856 DEBUG:boto:Data: 2022-09-03T07:37:56.857 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.857 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.858 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.859 DEBUG:boto:Params: {} 2022-09-03T07:37:56.859 DEBUG:boto:Token: None 2022-09-03T07:37:56.860 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.860 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3W2e39Novtxo5xH6rTLNjO3IHeQ= 2022-09-03T07:37:56.861 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3W2e39Novtxo5xH6rTLNjO3IHeQ=', 'Content-Length': '0'} 2022-09-03T07:37:56.863 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003b7bcc938b795d72-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.864 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.865 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.865 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.866 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.866 DEBUG:boto:Method: GET 2022-09-03T07:37:56.867 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.868 DEBUG:boto:Data: 2022-09-03T07:37:56.868 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.869 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.869 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.870 DEBUG:boto:Params: {} 2022-09-03T07:37:56.870 DEBUG:boto:Token: None 2022-09-03T07:37:56.871 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:56.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:3W2e39Novtxo5xH6rTLNjO3IHeQ= 2022-09-03T07:37:56.873 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:3W2e39Novtxo5xH6rTLNjO3IHeQ=', 'Content-Length': '0'} 2022-09-03T07:37:56.875 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a2bdad271a71b664-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.877 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.878 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.878 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:56.879 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:56.880 DEBUG:boto:Method: GET 2022-09-03T07:37:56.881 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:56.882 DEBUG:boto:Data: 2022-09-03T07:37:56.883 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.883 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.884 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.886 DEBUG:boto:Params: {} 2022-09-03T07:37:56.887 DEBUG:boto:Token: None 2022-09-03T07:37:56.888 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.888 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.889 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.892 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000859ccd205c4cffd7-0063130454-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.892 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.893 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.894 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.894 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:56.895 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:56.896 DEBUG:boto:Method: GET 2022-09-03T07:37:56.897 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:56.897 DEBUG:boto:Data: 2022-09-03T07:37:56.898 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.898 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.899 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.900 DEBUG:boto:Params: {} 2022-09-03T07:37:56.900 DEBUG:boto:Token: None 2022-09-03T07:37:56.901 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.902 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.903 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.906 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000073662ebfa5774077-0063130454-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.907 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.908 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.908 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.909 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.909 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:56.910 DEBUG:boto:Data: 2022-09-03T07:37:56.911 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.911 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.912 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.912 DEBUG:boto:Params: {} 2022-09-03T07:37:56.913 DEBUG:boto:Token: None 2022-09-03T07:37:56.913 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44 2022-09-03T07:37:56.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KGtrnqwbYblYbt9JF34mSXyULiM= 2022-09-03T07:37:56.914 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KGtrnqwbYblYbt9JF34mSXyULiM=', 'Content-Length': '0'} 2022-09-03T07:37:56.918 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e3cea9919a86345d-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.918 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.919 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.920 DEBUG:boto:Method: HEAD 2022-09-03T07:37:56.920 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:56.921 DEBUG:boto:Data: 2022-09-03T07:37:56.921 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.922 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.922 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.923 DEBUG:boto:Params: {} 2022-09-03T07:37:56.923 DEBUG:boto:Token: None 2022-09-03T07:37:56.924 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44 2022-09-03T07:37:56.925 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:KGtrnqwbYblYbt9JF34mSXyULiM= 2022-09-03T07:37:56.926 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:KGtrnqwbYblYbt9JF34mSXyULiM=', 'Content-Length': '0'} 2022-09-03T07:37:56.928 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001e42e287f9417400-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.929 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:37:56.929 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.930 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.931 DEBUG:boto:Method: GET 2022-09-03T07:37:56.931 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:56.932 DEBUG:boto:Data: 2022-09-03T07:37:56.933 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.933 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.934 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.935 DEBUG:boto:Params: {} 2022-09-03T07:37:56.936 DEBUG:boto:Token: None 2022-09-03T07:37:56.936 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44 2022-09-03T07:37:56.937 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n7TAdnAo8K+BpdUvVukl1iDMR84= 2022-09-03T07:37:56.938 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n7TAdnAo8K+BpdUvVukl1iDMR84=', 'Content-Length': '0'} 2022-09-03T07:37:56.941 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000096ab8dcbd728c70f-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.942 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:56.942 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:56.943 DEBUG:boto:Method: GET 2022-09-03T07:37:56.944 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:56.944 DEBUG:boto:Data: 2022-09-03T07:37:56.945 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.946 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.946 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.947 DEBUG:boto:Params: {} 2022-09-03T07:37:56.949 DEBUG:boto:Token: None 2022-09-03T07:37:56.950 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj44 2022-09-03T07:37:56.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n7TAdnAo8K+BpdUvVukl1iDMR84= 2022-09-03T07:37:56.953 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n7TAdnAo8K+BpdUvVukl1iDMR84=', 'Content-Length': '0'} 2022-09-03T07:37:56.956 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e8241add14329b22-0063130454-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.957 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.958 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.959 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:56.960 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:56.960 DEBUG:boto:Method: GET 2022-09-03T07:37:56.961 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:56.962 DEBUG:boto:Data: 2022-09-03T07:37:56.962 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.963 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.963 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.964 DEBUG:boto:Params: {} 2022-09-03T07:37:56.964 DEBUG:boto:Token: None 2022-09-03T07:37:56.965 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.966 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.966 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.969 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004380f2147f27c343-0063130454-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.969 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.970 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:56.971 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:56.971 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:56.972 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:56.972 DEBUG:boto:Method: GET 2022-09-03T07:37:56.973 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:56.973 DEBUG:boto:Data: 2022-09-03T07:37:56.974 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.975 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:56.975 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.976 DEBUG:boto:Params: {} 2022-09-03T07:37:56.976 DEBUG:boto:Token: None 2022-09-03T07:37:56.977 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/ 2022-09-03T07:37:56.978 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4= 2022-09-03T07:37:56.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gOYGRXGKnr61+jVBpcsWrnqiEJ4=', 'Content-Length': '0'} 2022-09-03T07:37:56.982 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d2a3fa774c94f598-0063130454-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.983 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:56.984 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:37:56.985 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:56.985 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:56.986 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:56.986 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:56.987 DEBUG:boto:Method: GET 2022-09-03T07:37:56.988 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:56.988 DEBUG:boto:Data: 2022-09-03T07:37:56.989 DEBUG:boto:Headers: {} 2022-09-03T07:37:56.989 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:56.990 DEBUG:boto:Port: 8000 2022-09-03T07:37:56.990 DEBUG:boto:Params: {} 2022-09-03T07:37:56.991 DEBUG:boto:Token: None 2022-09-03T07:37:56.992 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:56 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:56.992 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vMPcxFrwRPKSQSLpcV6YTq/ok5o= 2022-09-03T07:37:56.993 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:56 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vMPcxFrwRPKSQSLpcV6YTq/ok5o=', 'Content-Length': '0'} 2022-09-03T07:37:56.995 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006979c41d4c8f3bd0-0063130454-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:56 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:56.995 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:56.996 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:56.996 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:56.997 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:56.998 DEBUG:boto:Method: GET 2022-09-03T07:37:56.998 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:56.999 DEBUG:boto:Data: 2022-09-03T07:37:56.999 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.000 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.000 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.001 DEBUG:boto:Params: {} 2022-09-03T07:37:57.001 DEBUG:boto:Token: None 2022-09-03T07:37:57.002 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7yTtluaIERDGPeuHNqIe8/7PdBY= 2022-09-03T07:37:57.003 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7yTtluaIERDGPeuHNqIe8/7PdBY=', 'Content-Length': '0'} 2022-09-03T07:37:57.005 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001db3f6052cea4e35-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.006 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:57.007 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:57.007 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.008 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.008 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.009 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.010 DEBUG:boto:Data: 2022-09-03T07:37:57.010 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.011 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:57.011 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.012 DEBUG:boto:Params: {} 2022-09-03T07:37:57.012 DEBUG:boto:Token: None 2022-09-03T07:37:57.013 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.014 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RNXF6FZMRACbkVSX9EOF7AxOOrY= 2022-09-03T07:37:57.014 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RNXF6FZMRACbkVSX9EOF7AxOOrY=', 'Content-Length': '0'} 2022-09-03T07:37:57.016 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae1e05673174048f-0063130455-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.017 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:57.018 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:57.018 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.019 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.019 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.020 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.020 DEBUG:boto:Data: 2022-09-03T07:37:57.021 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.021 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.021 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.022 DEBUG:boto:Params: {} 2022-09-03T07:37:57.023 DEBUG:boto:Token: None 2022-09-03T07:37:57.023 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.024 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RNXF6FZMRACbkVSX9EOF7AxOOrY= 2022-09-03T07:37:57.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RNXF6FZMRACbkVSX9EOF7AxOOrY=', 'Content-Length': '0'} 2022-09-03T07:37:57.027 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c0f7b6e10a335676-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.027 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:37:57.028 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:57.029 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:57.029 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.030 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.030 DEBUG:boto:Method: GET 2022-09-03T07:37:57.031 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.031 DEBUG:boto:Data: 2022-09-03T07:37:57.032 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.032 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:57.033 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.033 DEBUG:boto:Params: {} 2022-09-03T07:37:57.034 DEBUG:boto:Token: None 2022-09-03T07:37:57.035 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.035 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7yTtluaIERDGPeuHNqIe8/7PdBY= 2022-09-03T07:37:57.036 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7yTtluaIERDGPeuHNqIe8/7PdBY=', 'Content-Length': '0'} 2022-09-03T07:37:57.038 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f1cd887a1243ab6d-0063130455-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.039 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:57.039 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:57.040 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.040 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.041 DEBUG:boto:Method: GET 2022-09-03T07:37:57.041 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.042 DEBUG:boto:Data: 2022-09-03T07:37:57.043 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.043 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.044 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.044 DEBUG:boto:Params: {} 2022-09-03T07:37:57.045 DEBUG:boto:Token: None 2022-09-03T07:37:57.045 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:57.046 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:7yTtluaIERDGPeuHNqIe8/7PdBY= 2022-09-03T07:37:57.047 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:7yTtluaIERDGPeuHNqIe8/7PdBY=', 'Content-Length': '0'} 2022-09-03T07:37:57.049 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000007042739f7461c5ff-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.090 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.091 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.091 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:57.092 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:57.092 DEBUG:boto:Method: GET 2022-09-03T07:37:57.093 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:37:57.094 DEBUG:boto:Data: 2022-09-03T07:37:57.094 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.095 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:57.095 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.096 DEBUG:boto:Params: {} 2022-09-03T07:37:57.097 DEBUG:boto:Token: None 2022-09-03T07:37:57.097 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.098 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.099 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.101 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f2dc48d8ecf7e4f9-0063130455-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.102 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.103 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.104 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.104 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:57.105 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:57.105 DEBUG:boto:Method: GET 2022-09-03T07:37:57.106 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:37:57.106 DEBUG:boto:Data: 2022-09-03T07:37:57.107 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.107 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.108 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.108 DEBUG:boto:Params: {} 2022-09-03T07:37:57.109 DEBUG:boto:Token: None 2022-09-03T07:37:57.110 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.110 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.111 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.114 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000de7b9a4e8fb47c89-0063130455-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.114 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.115 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:57.116 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:57.116 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.117 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:37:57.117 DEBUG:boto:Data: 2022-09-03T07:37:57.118 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.118 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:57.119 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.119 DEBUG:boto:Params: {} 2022-09-03T07:37:57.120 DEBUG:boto:Token: None 2022-09-03T07:37:57.120 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj55 2022-09-03T07:37:57.121 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H1GK8zTflhPUlFMTVHZxLnDnJpA= 2022-09-03T07:37:57.122 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H1GK8zTflhPUlFMTVHZxLnDnJpA=', 'Content-Length': '0'} 2022-09-03T07:37:57.123 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c43d7eecb1bd0fa4-0063130455-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.124 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:57.125 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:57.126 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.126 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:37:57.127 DEBUG:boto:Data: 2022-09-03T07:37:57.127 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.128 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.128 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.129 DEBUG:boto:Params: {} 2022-09-03T07:37:57.129 DEBUG:boto:Token: None 2022-09-03T07:37:57.130 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj55 2022-09-03T07:37:57.130 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:H1GK8zTflhPUlFMTVHZxLnDnJpA= 2022-09-03T07:37:57.131 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:H1GK8zTflhPUlFMTVHZxLnDnJpA=', 'Content-Length': '0'} 2022-09-03T07:37:57.133 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000097dfe0fef6dbee09-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.133 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:37:57.134 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:57.135 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:57.135 DEBUG:boto:Method: GET 2022-09-03T07:37:57.136 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:37:57.136 DEBUG:boto:Data: 2022-09-03T07:37:57.137 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.137 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:57.138 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.138 DEBUG:boto:Params: {} 2022-09-03T07:37:57.139 DEBUG:boto:Token: None 2022-09-03T07:37:57.140 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj55 2022-09-03T07:37:57.140 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BhwN9QNUUTvBPWTt0dI4Dba6Mio= 2022-09-03T07:37:57.141 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BhwN9QNUUTvBPWTt0dI4Dba6Mio=', 'Content-Length': '0'} 2022-09-03T07:37:57.143 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000052e1bb0377f4351f-0063130455-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.143 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:57.144 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:57.145 DEBUG:boto:Method: GET 2022-09-03T07:37:57.145 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:37:57.146 DEBUG:boto:Data: 2022-09-03T07:37:57.146 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.147 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.147 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.148 DEBUG:boto:Params: {} 2022-09-03T07:37:57.148 DEBUG:boto:Token: None 2022-09-03T07:37:57.149 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj55 2022-09-03T07:37:57.150 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BhwN9QNUUTvBPWTt0dI4Dba6Mio= 2022-09-03T07:37:57.150 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BhwN9QNUUTvBPWTt0dI4Dba6Mio=', 'Content-Length': '0'} 2022-09-03T07:37:57.152 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e365de6adc4685ab-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.153 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.154 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.154 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:57.155 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:57.155 DEBUG:boto:Method: GET 2022-09-03T07:37:57.156 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:37:57.156 DEBUG:boto:Data: 2022-09-03T07:37:57.157 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.157 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:57.158 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.159 DEBUG:boto:Params: {} 2022-09-03T07:37:57.159 DEBUG:boto:Token: None 2022-09-03T07:37:57.160 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.160 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.161 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.163 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000030981153dcd53d5-0063130455-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.210 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.211 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.212 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.212 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:57.213 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:57.213 DEBUG:boto:Method: GET 2022-09-03T07:37:57.214 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:37:57.214 DEBUG:boto:Data: 2022-09-03T07:37:57.215 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.216 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.217 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.218 DEBUG:boto:Params: {} 2022-09-03T07:37:57.219 DEBUG:boto:Token: None 2022-09-03T07:37:57.220 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.221 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.223 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.227 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bcb22644b5172db0-0063130455-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.228 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.229 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-76 zones={test-zone2, test-zone1} 2022-09-03T07:37:57.230 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-76 zones={test-zone3, test-zone1} 2022-09-03T07:37:57.231 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.232 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.232 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.233 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:37:57.233 DEBUG:boto:Data: 2022-09-03T07:37:57.234 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.235 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.235 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.236 DEBUG:boto:Params: {} 2022-09-03T07:37:57.237 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8001} 2022-09-03T07:37:57.238 DEBUG:boto:Token: None 2022-09-03T07:37:57.239 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.240 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HhLJadUmKWqxRmJjZcwnxrH9ONY= 2022-09-03T07:37:57.240 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HhLJadUmKWqxRmJjZcwnxrH9ONY=', 'Content-Length': '0'} 2022-09-03T07:37:57.246 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003f5175467c8fbe1f-0063130455-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.247 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.247 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.248 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.248 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:37:57.249 DEBUG:boto:Data: 2022-09-03T07:37:57.249 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.250 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.251 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.252 DEBUG:boto:Params: {} 2022-09-03T07:37:57.253 DEBUG:boto:Token: None 2022-09-03T07:37:57.254 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HhLJadUmKWqxRmJjZcwnxrH9ONY= 2022-09-03T07:37:57.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HhLJadUmKWqxRmJjZcwnxrH9ONY=', 'Content-Length': '0'} 2022-09-03T07:37:57.260 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000bf127d1db16155da-0063130455-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.261 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:37:57.262 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.263 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.264 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.264 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.265 DEBUG:boto:Method: GET 2022-09-03T07:37:57.266 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.267 DEBUG:boto:Data: 2022-09-03T07:37:57.268 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.269 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.269 DEBUG:boto:Params: {} 2022-09-03T07:37:57.270 DEBUG:boto:Token: None 2022-09-03T07:37:57.271 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/?versions 2022-09-03T07:37:57.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f3EOESqw81aYtkPU9Lny9VELKLk= 2022-09-03T07:37:57.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f3EOESqw81aYtkPU9Lny9VELKLk=', 'Content-Length': '0'} 2022-09-03T07:37:57.275 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003f6782354b8dc84c-0063130455-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.276 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.277 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:37:57.277 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:37:57.278 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:37:57.278 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:37:57.279 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:37:57.280 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:37:57.280 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.281 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.281 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.282 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.283 DEBUG:boto:Method: GET 2022-09-03T07:37:57.283 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.284 DEBUG:boto:Data: 2022-09-03T07:37:57.285 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.285 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.286 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.286 DEBUG:boto:Params: {} 2022-09-03T07:37:57.287 DEBUG:boto:Token: None 2022-09-03T07:37:57.287 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/?versions 2022-09-03T07:37:57.288 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f3EOESqw81aYtkPU9Lny9VELKLk= 2022-09-03T07:37:57.289 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f3EOESqw81aYtkPU9Lny9VELKLk=', 'Content-Length': '0'} 2022-09-03T07:37:57.292 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000020d99551564e4167-0063130455-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.293 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.294 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:37:57.295 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:37:57.296 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:37:57.297 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:37:57.298 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:37:57.299 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.300 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.301 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.302 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.303 DEBUG:boto:Method: GET 2022-09-03T07:37:57.303 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.304 DEBUG:boto:Data: 2022-09-03T07:37:57.305 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.305 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.306 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.307 DEBUG:boto:Params: {} 2022-09-03T07:37:57.308 DEBUG:boto:Token: None 2022-09-03T07:37:57.309 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/?versions 2022-09-03T07:37:57.310 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f3EOESqw81aYtkPU9Lny9VELKLk= 2022-09-03T07:37:57.310 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f3EOESqw81aYtkPU9Lny9VELKLk=', 'Content-Length': '0'} 2022-09-03T07:37:57.314 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000036b14ef20aa76359-0063130455-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.315 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.317 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.318 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.319 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.320 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.321 DEBUG:boto:Method: GET 2022-09-03T07:37:57.322 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:57.323 DEBUG:boto:Data: 2022-09-03T07:37:57.324 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.326 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.327 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.328 DEBUG:boto:Params: {} 2022-09-03T07:37:57.329 DEBUG:boto:Token: None 2022-09-03T07:37:57.329 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/?versions 2022-09-03T07:37:57.330 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f3EOESqw81aYtkPU9Lny9VELKLk= 2022-09-03T07:37:57.331 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f3EOESqw81aYtkPU9Lny9VELKLk=', 'Content-Length': '0'} 2022-09-03T07:37:57.334 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000002a80ed70c2c5e2e9-0063130455-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.334 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.335 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:37:57.336 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.336 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.337 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.338 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.338 DEBUG:boto:Method: GET 2022-09-03T07:37:57.339 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.339 DEBUG:boto:Data: 2022-09-03T07:37:57.340 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.341 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.341 DEBUG:boto:Params: {} 2022-09-03T07:37:57.342 DEBUG:boto:Token: None 2022-09-03T07:37:57.343 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.344 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F4eCt233aCYt9OO8AJrd/ac5+Vs= 2022-09-03T07:37:57.345 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F4eCt233aCYt9OO8AJrd/ac5+Vs=', 'Content-Length': '0'} 2022-09-03T07:37:57.348 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bf5ea5c54032cd13-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.390 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.391 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.392 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.392 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.393 DEBUG:boto:Method: GET 2022-09-03T07:37:57.394 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.394 DEBUG:boto:Data: 2022-09-03T07:37:57.395 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.397 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.398 DEBUG:boto:Params: {} 2022-09-03T07:37:57.399 DEBUG:boto:Token: None 2022-09-03T07:37:57.399 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.400 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F4eCt233aCYt9OO8AJrd/ac5+Vs= 2022-09-03T07:37:57.401 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F4eCt233aCYt9OO8AJrd/ac5+Vs=', 'Content-Length': '0'} 2022-09-03T07:37:57.403 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000116704d83f428395-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.405 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.406 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.407 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.407 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.408 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.409 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.410 DEBUG:boto:Data: 2022-09-03T07:37:57.411 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.412 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.412 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.413 DEBUG:boto:Params: {} 2022-09-03T07:37:57.413 DEBUG:boto:Token: None 2022-09-03T07:37:57.414 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.415 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g6/u3Cieb/TUaaL2j23yTLQWkG8= 2022-09-03T07:37:57.415 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g6/u3Cieb/TUaaL2j23yTLQWkG8=', 'Content-Length': '0'} 2022-09-03T07:37:57.418 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000002528a60b8ab88b3c-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.419 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.420 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.421 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.422 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.423 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.424 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.425 DEBUG:boto:Data: 2022-09-03T07:37:57.426 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.427 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.428 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.428 DEBUG:boto:Params: {} 2022-09-03T07:37:57.430 DEBUG:boto:Token: None 2022-09-03T07:37:57.431 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.431 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g6/u3Cieb/TUaaL2j23yTLQWkG8= 2022-09-03T07:37:57.432 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g6/u3Cieb/TUaaL2j23yTLQWkG8=', 'Content-Length': '0'} 2022-09-03T07:37:57.435 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000032d23723c2fc13d5-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.436 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:37:57.436 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.437 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.438 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.438 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.439 DEBUG:boto:Method: GET 2022-09-03T07:37:57.439 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.440 DEBUG:boto:Data: 2022-09-03T07:37:57.441 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.441 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.442 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.442 DEBUG:boto:Params: {} 2022-09-03T07:37:57.443 DEBUG:boto:Token: None 2022-09-03T07:37:57.443 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.444 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F4eCt233aCYt9OO8AJrd/ac5+Vs= 2022-09-03T07:37:57.445 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F4eCt233aCYt9OO8AJrd/ac5+Vs=', 'Content-Length': '0'} 2022-09-03T07:37:57.447 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005b2c082e62eb8d0e-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.448 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.448 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.449 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.449 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.450 DEBUG:boto:Method: GET 2022-09-03T07:37:57.451 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.451 DEBUG:boto:Data: 2022-09-03T07:37:57.452 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.452 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.453 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.453 DEBUG:boto:Params: {} 2022-09-03T07:37:57.454 DEBUG:boto:Token: None 2022-09-03T07:37:57.454 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:57.455 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F4eCt233aCYt9OO8AJrd/ac5+Vs= 2022-09-03T07:37:57.456 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F4eCt233aCYt9OO8AJrd/ac5+Vs=', 'Content-Length': '0'} 2022-09-03T07:37:57.458 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f23b5d34660386a-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.459 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.460 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.460 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:57.461 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:57.461 DEBUG:boto:Method: GET 2022-09-03T07:37:57.462 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:57.462 DEBUG:boto:Data: 2022-09-03T07:37:57.463 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.464 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.464 DEBUG:boto:Params: {} 2022-09-03T07:37:57.465 DEBUG:boto:Token: None 2022-09-03T07:37:57.466 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.467 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.470 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000059564fd1fdcceaef-0063130455-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.471 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.472 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.472 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.473 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:57.473 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:57.474 DEBUG:boto:Method: GET 2022-09-03T07:37:57.474 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:57.475 DEBUG:boto:Data: 2022-09-03T07:37:57.475 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.476 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.477 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.477 DEBUG:boto:Params: {} 2022-09-03T07:37:57.478 DEBUG:boto:Token: None 2022-09-03T07:37:57.478 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.479 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.479 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.482 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1e4a0ac6a52819e-0063130455-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.526 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.527 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.528 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.528 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.529 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:57.529 DEBUG:boto:Data: 2022-09-03T07:37:57.530 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.530 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.531 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.531 DEBUG:boto:Params: {} 2022-09-03T07:37:57.532 DEBUG:boto:Token: None 2022-09-03T07:37:57.533 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11 2022-09-03T07:37:57.533 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JlWwNKGslXMP173KjVWREHTAW5I= 2022-09-03T07:37:57.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JlWwNKGslXMP173KjVWREHTAW5I=', 'Content-Length': '0'} 2022-09-03T07:37:57.536 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000012009e9905567757-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.537 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.538 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.538 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.539 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:57.540 DEBUG:boto:Data: 2022-09-03T07:37:57.541 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.542 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.542 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.543 DEBUG:boto:Params: {} 2022-09-03T07:37:57.544 DEBUG:boto:Token: None 2022-09-03T07:37:57.544 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11 2022-09-03T07:37:57.545 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JlWwNKGslXMP173KjVWREHTAW5I= 2022-09-03T07:37:57.545 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JlWwNKGslXMP173KjVWREHTAW5I=', 'Content-Length': '0'} 2022-09-03T07:37:57.548 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000997f2bd6f2341a23-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.549 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:37:57.550 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.551 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.552 DEBUG:boto:Method: GET 2022-09-03T07:37:57.552 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:57.553 DEBUG:boto:Data: 2022-09-03T07:37:57.554 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.556 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.557 DEBUG:boto:Params: {} 2022-09-03T07:37:57.558 DEBUG:boto:Token: None 2022-09-03T07:37:57.559 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11 2022-09-03T07:37:57.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:65iWEDRGN+rmUjeArKScpbZN8G4= 2022-09-03T07:37:57.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:65iWEDRGN+rmUjeArKScpbZN8G4=', 'Content-Length': '0'} 2022-09-03T07:37:57.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000086ee02cacc3cc3fa-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.565 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:57.567 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:57.568 DEBUG:boto:Method: GET 2022-09-03T07:37:57.569 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:57.570 DEBUG:boto:Data: 2022-09-03T07:37:57.571 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.572 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.573 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.574 DEBUG:boto:Params: {} 2022-09-03T07:37:57.575 DEBUG:boto:Token: None 2022-09-03T07:37:57.577 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj11 2022-09-03T07:37:57.578 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:65iWEDRGN+rmUjeArKScpbZN8G4= 2022-09-03T07:37:57.579 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:65iWEDRGN+rmUjeArKScpbZN8G4=', 'Content-Length': '0'} 2022-09-03T07:37:57.582 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000682bdfb010276bf8-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.583 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.584 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.585 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:57.586 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:57.587 DEBUG:boto:Method: GET 2022-09-03T07:37:57.588 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:57.589 DEBUG:boto:Data: 2022-09-03T07:37:57.590 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.591 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.592 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.593 DEBUG:boto:Params: {} 2022-09-03T07:37:57.595 DEBUG:boto:Token: None 2022-09-03T07:37:57.596 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.597 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.600 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.604 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004ef74aad5eb3b63f-0063130455-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.604 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.605 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.606 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.606 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:57.607 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:57.607 DEBUG:boto:Method: GET 2022-09-03T07:37:57.608 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:57.609 DEBUG:boto:Data: 2022-09-03T07:37:57.609 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.612 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.613 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.613 DEBUG:boto:Params: {} 2022-09-03T07:37:57.614 DEBUG:boto:Token: None 2022-09-03T07:37:57.614 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.615 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.616 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.618 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000048d67311cae088dd-0063130455-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.619 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.620 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:37:57.620 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.621 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.621 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.622 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.622 DEBUG:boto:Method: GET 2022-09-03T07:37:57.623 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.623 DEBUG:boto:Data: 2022-09-03T07:37:57.624 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.625 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.625 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.626 DEBUG:boto:Params: {} 2022-09-03T07:37:57.626 DEBUG:boto:Token: None 2022-09-03T07:37:57.627 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.627 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xd233f9arrnhHfWfWIUWNRj/23k= 2022-09-03T07:37:57.628 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xd233f9arrnhHfWfWIUWNRj/23k=', 'Content-Length': '0'} 2022-09-03T07:37:57.630 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000060d3e0a069354bf-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.674 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.675 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.675 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.676 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.676 DEBUG:boto:Method: GET 2022-09-03T07:37:57.677 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.678 DEBUG:boto:Data: 2022-09-03T07:37:57.678 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.679 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.679 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.680 DEBUG:boto:Params: {} 2022-09-03T07:37:57.681 DEBUG:boto:Token: None 2022-09-03T07:37:57.682 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.683 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xd233f9arrnhHfWfWIUWNRj/23k= 2022-09-03T07:37:57.684 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xd233f9arrnhHfWfWIUWNRj/23k=', 'Content-Length': '0'} 2022-09-03T07:37:57.686 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003bbff88a01ad3679-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.687 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.688 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.688 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.688 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.689 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.690 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.691 DEBUG:boto:Data: 2022-09-03T07:37:57.692 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.693 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.694 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.694 DEBUG:boto:Params: {} 2022-09-03T07:37:57.697 DEBUG:boto:Token: None 2022-09-03T07:37:57.698 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.698 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XK+e0HsGL88u8n0zIOmndt6/Nps= 2022-09-03T07:37:57.699 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XK+e0HsGL88u8n0zIOmndt6/Nps=', 'Content-Length': '0'} 2022-09-03T07:37:57.701 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000015e7e1a1fd7d76f2-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.702 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.703 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.704 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.705 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.706 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.707 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.707 DEBUG:boto:Data: 2022-09-03T07:37:57.708 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.709 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.710 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.710 DEBUG:boto:Params: {} 2022-09-03T07:37:57.711 DEBUG:boto:Token: None 2022-09-03T07:37:57.712 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.712 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XK+e0HsGL88u8n0zIOmndt6/Nps= 2022-09-03T07:37:57.713 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XK+e0HsGL88u8n0zIOmndt6/Nps=', 'Content-Length': '0'} 2022-09-03T07:37:57.715 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001b647c72d6bb2ffd-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.716 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:37:57.717 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.718 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.719 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.719 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.720 DEBUG:boto:Method: GET 2022-09-03T07:37:57.731 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.731 DEBUG:boto:Data: 2022-09-03T07:37:57.732 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.733 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.733 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.734 DEBUG:boto:Params: {} 2022-09-03T07:37:57.734 DEBUG:boto:Token: None 2022-09-03T07:37:57.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.736 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xd233f9arrnhHfWfWIUWNRj/23k= 2022-09-03T07:37:57.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xd233f9arrnhHfWfWIUWNRj/23k=', 'Content-Length': '0'} 2022-09-03T07:37:57.739 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a26f3eefe15f7388-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.740 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.740 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.741 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.741 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.742 DEBUG:boto:Method: GET 2022-09-03T07:37:57.742 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.743 DEBUG:boto:Data: 2022-09-03T07:37:57.743 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.744 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.744 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.745 DEBUG:boto:Params: {} 2022-09-03T07:37:57.746 DEBUG:boto:Token: None 2022-09-03T07:37:57.746 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:57.747 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:xd233f9arrnhHfWfWIUWNRj/23k= 2022-09-03T07:37:57.748 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:xd233f9arrnhHfWfWIUWNRj/23k=', 'Content-Length': '0'} 2022-09-03T07:37:57.750 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006de454d09b68eebb-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.751 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.751 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.752 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:57.752 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:57.753 DEBUG:boto:Method: GET 2022-09-03T07:37:57.753 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:57.754 DEBUG:boto:Data: 2022-09-03T07:37:57.755 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.756 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.757 DEBUG:boto:Params: {} 2022-09-03T07:37:57.757 DEBUG:boto:Token: None 2022-09-03T07:37:57.758 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.759 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.766 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a1c604cc17b47a18-0063130455-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.767 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.769 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.770 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.771 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:57.772 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:57.773 DEBUG:boto:Method: GET 2022-09-03T07:37:57.774 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:57.775 DEBUG:boto:Data: 2022-09-03T07:37:57.776 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.778 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.780 DEBUG:boto:Params: {} 2022-09-03T07:37:57.781 DEBUG:boto:Token: None 2022-09-03T07:37:57.782 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.783 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.784 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.788 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000004d10459c9a548a61-0063130455-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.834 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.836 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.837 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.838 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.839 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:57.840 DEBUG:boto:Data: 2022-09-03T07:37:57.841 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.842 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.843 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.844 DEBUG:boto:Params: {} 2022-09-03T07:37:57.845 DEBUG:boto:Token: None 2022-09-03T07:37:57.846 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22 2022-09-03T07:37:57.847 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eCkgtdT35Zx5PEFt76i2jW3aamA= 2022-09-03T07:37:57.847 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eCkgtdT35Zx5PEFt76i2jW3aamA=', 'Content-Length': '0'} 2022-09-03T07:37:57.849 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000eee1b4150c68943c-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.850 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.851 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.851 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.852 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:57.852 DEBUG:boto:Data: 2022-09-03T07:37:57.853 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.853 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.854 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.854 DEBUG:boto:Params: {} 2022-09-03T07:37:57.855 DEBUG:boto:Token: None 2022-09-03T07:37:57.856 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22 2022-09-03T07:37:57.857 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:eCkgtdT35Zx5PEFt76i2jW3aamA= 2022-09-03T07:37:57.858 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:eCkgtdT35Zx5PEFt76i2jW3aamA=', 'Content-Length': '0'} 2022-09-03T07:37:57.861 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f4eb813f4f3aa837-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.863 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:37:57.863 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.864 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.865 DEBUG:boto:Method: GET 2022-09-03T07:37:57.865 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:57.866 DEBUG:boto:Data: 2022-09-03T07:37:57.866 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.867 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.868 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.868 DEBUG:boto:Params: {} 2022-09-03T07:37:57.869 DEBUG:boto:Token: None 2022-09-03T07:37:57.869 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22 2022-09-03T07:37:57.870 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x6WAiKVGtasMpvGImy2/ZZe+h0s= 2022-09-03T07:37:57.871 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x6WAiKVGtasMpvGImy2/ZZe+h0s=', 'Content-Length': '0'} 2022-09-03T07:37:57.873 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f0cc2a6d14bdd3a3-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.874 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:57.875 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:57.876 DEBUG:boto:Method: GET 2022-09-03T07:37:57.876 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:57.877 DEBUG:boto:Data: 2022-09-03T07:37:57.877 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.878 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.878 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.879 DEBUG:boto:Params: {} 2022-09-03T07:37:57.879 DEBUG:boto:Token: None 2022-09-03T07:37:57.880 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj22 2022-09-03T07:37:57.881 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:x6WAiKVGtasMpvGImy2/ZZe+h0s= 2022-09-03T07:37:57.882 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:x6WAiKVGtasMpvGImy2/ZZe+h0s=', 'Content-Length': '0'} 2022-09-03T07:37:57.886 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a9b4fb79ab0298e2-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.887 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.888 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.888 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:57.889 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:57.890 DEBUG:boto:Method: GET 2022-09-03T07:37:57.890 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:57.891 DEBUG:boto:Data: 2022-09-03T07:37:57.891 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.892 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.892 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.893 DEBUG:boto:Params: {} 2022-09-03T07:37:57.894 DEBUG:boto:Token: None 2022-09-03T07:37:57.894 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.895 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.895 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.899 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000063d32714e80a780f-0063130455-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.900 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.901 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:57.901 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:57.904 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:57.905 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:57.905 DEBUG:boto:Method: GET 2022-09-03T07:37:57.906 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:57.906 DEBUG:boto:Data: 2022-09-03T07:37:57.907 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.907 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.908 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.909 DEBUG:boto:Params: {} 2022-09-03T07:37:57.910 DEBUG:boto:Token: None 2022-09-03T07:37:57.910 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/ 2022-09-03T07:37:57.911 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM= 2022-09-03T07:37:57.912 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uYlnb8Xp6lfxQJEnCPsmMqdgXSM=', 'Content-Length': '0'} 2022-09-03T07:37:57.916 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000008c0797805cbae7d2-0063130455-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.917 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:57.918 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:37:57.919 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:57.919 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:57.920 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.920 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.921 DEBUG:boto:Method: GET 2022-09-03T07:37:57.921 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.922 DEBUG:boto:Data: 2022-09-03T07:37:57.922 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.923 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.923 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.924 DEBUG:boto:Params: {} 2022-09-03T07:37:57.925 DEBUG:boto:Token: None 2022-09-03T07:37:57.925 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gfoVn45j4uY8lL2g4arIQVYwiB4= 2022-09-03T07:37:57.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gfoVn45j4uY8lL2g4arIQVYwiB4=', 'Content-Length': '0'} 2022-09-03T07:37:57.929 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000854863b511f4c593-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.970 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:57.971 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:57.971 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.972 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.972 DEBUG:boto:Method: GET 2022-09-03T07:37:57.973 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.973 DEBUG:boto:Data: 2022-09-03T07:37:57.974 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.975 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.975 DEBUG:boto:Params: {} 2022-09-03T07:37:57.976 DEBUG:boto:Token: None 2022-09-03T07:37:57.977 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gfoVn45j4uY8lL2g4arIQVYwiB4= 2022-09-03T07:37:57.978 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gfoVn45j4uY8lL2g4arIQVYwiB4=', 'Content-Length': '0'} 2022-09-03T07:37:57.980 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000de44947d57d29941-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.980 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:57.981 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:57.981 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.982 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.982 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.983 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.983 DEBUG:boto:Data: 2022-09-03T07:37:57.984 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:57.985 DEBUG:boto:Port: 8001 2022-09-03T07:37:57.985 DEBUG:boto:Params: {} 2022-09-03T07:37:57.986 DEBUG:boto:Token: None 2022-09-03T07:37:57.987 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4radzTnQtDCAYYTYLdhQYYy2NQM= 2022-09-03T07:37:57.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4radzTnQtDCAYYTYLdhQYYy2NQM=', 'Content-Length': '0'} 2022-09-03T07:37:57.990 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000033b3754246e1a50d-0063130455-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:57.991 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:57.991 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:57.992 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.992 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.993 DEBUG:boto:Method: HEAD 2022-09-03T07:37:57.993 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.993 DEBUG:boto:Data: 2022-09-03T07:37:57.994 DEBUG:boto:Headers: {} 2022-09-03T07:37:57.995 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:57.995 DEBUG:boto:Port: 8000 2022-09-03T07:37:57.996 DEBUG:boto:Params: {} 2022-09-03T07:37:57.996 DEBUG:boto:Token: None 2022-09-03T07:37:57.997 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:57 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:57.997 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4radzTnQtDCAYYTYLdhQYYy2NQM= 2022-09-03T07:37:57.998 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:57 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4radzTnQtDCAYYTYLdhQYYy2NQM=', 'Content-Length': '0'} 2022-09-03T07:37:58.000 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005fe6e37ddf28a563-0063130455-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:57 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.001 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:37:58.001 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:58.002 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:58.002 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:58.003 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:58.004 DEBUG:boto:Method: GET 2022-09-03T07:37:58.004 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:58.005 DEBUG:boto:Data: 2022-09-03T07:37:58.005 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.006 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.006 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.007 DEBUG:boto:Params: {} 2022-09-03T07:37:58.007 DEBUG:boto:Token: None 2022-09-03T07:37:58.008 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:58.009 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+dt1NxJYLcay+j0E9z2qK5kSZ5w= 2022-09-03T07:37:58.009 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+dt1NxJYLcay+j0E9z2qK5kSZ5w=', 'Content-Length': '0'} 2022-09-03T07:37:58.012 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d148c6023c5feb62-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.012 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:58.013 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:58.014 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:58.014 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:58.015 DEBUG:boto:Method: GET 2022-09-03T07:37:58.015 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:58.016 DEBUG:boto:Data: 2022-09-03T07:37:58.016 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.017 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.018 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.018 DEBUG:boto:Params: {} 2022-09-03T07:37:58.019 DEBUG:boto:Token: None 2022-09-03T07:37:58.019 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:58.020 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+dt1NxJYLcay+j0E9z2qK5kSZ5w= 2022-09-03T07:37:58.021 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+dt1NxJYLcay+j0E9z2qK5kSZ5w=', 'Content-Length': '0'} 2022-09-03T07:37:58.023 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000931f2b26fa19d0b9-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.025 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.026 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.027 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:58.028 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:58.029 DEBUG:boto:Method: GET 2022-09-03T07:37:58.031 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:58.032 DEBUG:boto:Data: 2022-09-03T07:37:58.034 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.035 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.036 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.037 DEBUG:boto:Params: {} 2022-09-03T07:37:58.038 DEBUG:boto:Token: None 2022-09-03T07:37:58.039 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.040 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.042 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.045 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000eff437efd53bc3a4-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.046 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.048 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.049 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.051 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:58.052 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:58.053 DEBUG:boto:Method: GET 2022-09-03T07:37:58.054 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:58.055 DEBUG:boto:Data: 2022-09-03T07:37:58.056 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.057 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.058 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.059 DEBUG:boto:Params: {} 2022-09-03T07:37:58.060 DEBUG:boto:Token: None 2022-09-03T07:37:58.061 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.062 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.064 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.067 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b85bceb9c555ba0f-0063130456-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.068 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.068 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:58.069 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:58.069 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.070 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:58.071 DEBUG:boto:Data: 2022-09-03T07:37:58.071 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.072 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.073 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.073 DEBUG:boto:Params: {} 2022-09-03T07:37:58.074 DEBUG:boto:Token: None 2022-09-03T07:37:58.075 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj33 2022-09-03T07:37:58.076 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zdH8YDiAC8rAY/Smt5acQzN8NmU= 2022-09-03T07:37:58.077 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zdH8YDiAC8rAY/Smt5acQzN8NmU=', 'Content-Length': '0'} 2022-09-03T07:37:58.079 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e1c60c1b9b754c2b-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.080 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:58.081 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:58.081 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.082 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:58.082 DEBUG:boto:Data: 2022-09-03T07:37:58.083 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.085 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.087 DEBUG:boto:Params: {} 2022-09-03T07:37:58.088 DEBUG:boto:Token: None 2022-09-03T07:37:58.089 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj33 2022-09-03T07:37:58.089 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:zdH8YDiAC8rAY/Smt5acQzN8NmU= 2022-09-03T07:37:58.090 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:zdH8YDiAC8rAY/Smt5acQzN8NmU=', 'Content-Length': '0'} 2022-09-03T07:37:58.092 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000240b8e295516bb1e-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.093 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:37:58.094 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:58.094 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:58.095 DEBUG:boto:Method: GET 2022-09-03T07:37:58.095 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:58.096 DEBUG:boto:Data: 2022-09-03T07:37:58.096 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.097 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.097 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.098 DEBUG:boto:Params: {} 2022-09-03T07:37:58.099 DEBUG:boto:Token: None 2022-09-03T07:37:58.099 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj33 2022-09-03T07:37:58.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QvJcUjqieb8//NG0P+UyZ5fvWz8= 2022-09-03T07:37:58.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QvJcUjqieb8//NG0P+UyZ5fvWz8=', 'Content-Length': '0'} 2022-09-03T07:37:58.104 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d7adab82b6fb0a96-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.105 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:58.105 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:58.107 DEBUG:boto:Method: GET 2022-09-03T07:37:58.107 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:58.108 DEBUG:boto:Data: 2022-09-03T07:37:58.109 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.110 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.110 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.112 DEBUG:boto:Params: {} 2022-09-03T07:37:58.113 DEBUG:boto:Token: None 2022-09-03T07:37:58.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj33 2022-09-03T07:37:58.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QvJcUjqieb8//NG0P+UyZ5fvWz8= 2022-09-03T07:37:58.116 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QvJcUjqieb8//NG0P+UyZ5fvWz8=', 'Content-Length': '0'} 2022-09-03T07:37:58.119 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000055a9e81634e1f878-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.162 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.163 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.163 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:58.164 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:58.164 DEBUG:boto:Method: GET 2022-09-03T07:37:58.165 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:58.165 DEBUG:boto:Data: 2022-09-03T07:37:58.166 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.167 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.168 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.168 DEBUG:boto:Params: {} 2022-09-03T07:37:58.169 DEBUG:boto:Token: None 2022-09-03T07:37:58.169 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.170 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.171 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.175 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000665a1e98b3b692cf-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.176 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.178 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.178 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.179 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:58.179 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:58.180 DEBUG:boto:Method: GET 2022-09-03T07:37:58.180 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:58.181 DEBUG:boto:Data: 2022-09-03T07:37:58.181 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.182 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.183 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.183 DEBUG:boto:Params: {} 2022-09-03T07:37:58.184 DEBUG:boto:Token: None 2022-09-03T07:37:58.185 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.185 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.186 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.190 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000386ce7e40a257114-0063130456-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.191 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.192 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:37:58.192 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.193 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.193 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.194 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.195 DEBUG:boto:Method: GET 2022-09-03T07:37:58.195 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.196 DEBUG:boto:Data: 2022-09-03T07:37:58.196 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.197 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.198 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.199 DEBUG:boto:Params: {} 2022-09-03T07:37:58.199 DEBUG:boto:Token: None 2022-09-03T07:37:58.200 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZR8cVrg9Vsit/4aOAt0PFEDEpXc= 2022-09-03T07:37:58.201 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZR8cVrg9Vsit/4aOAt0PFEDEpXc=', 'Content-Length': '0'} 2022-09-03T07:37:58.203 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b5372036176db8ad-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.246 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.247 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.247 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.248 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.248 DEBUG:boto:Method: GET 2022-09-03T07:37:58.249 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.250 DEBUG:boto:Data: 2022-09-03T07:37:58.250 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.251 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.251 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.252 DEBUG:boto:Params: {} 2022-09-03T07:37:58.253 DEBUG:boto:Token: None 2022-09-03T07:37:58.254 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.255 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZR8cVrg9Vsit/4aOAt0PFEDEpXc= 2022-09-03T07:37:58.256 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZR8cVrg9Vsit/4aOAt0PFEDEpXc=', 'Content-Length': '0'} 2022-09-03T07:37:58.259 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000034c87bbcd9463512-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.260 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.261 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.262 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.263 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.264 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.265 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.266 DEBUG:boto:Data: 2022-09-03T07:37:58.267 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.268 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.268 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.269 DEBUG:boto:Params: {} 2022-09-03T07:37:58.270 DEBUG:boto:Token: None 2022-09-03T07:37:58.271 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.271 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:shm+6JUYdh/zGuc4QUtnm1NTu8Y= 2022-09-03T07:37:58.272 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:shm+6JUYdh/zGuc4QUtnm1NTu8Y=', 'Content-Length': '0'} 2022-09-03T07:37:58.275 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000008754775f578546f-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.276 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.277 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.277 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.278 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.279 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.280 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.281 DEBUG:boto:Data: 2022-09-03T07:37:58.282 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.283 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.284 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.284 DEBUG:boto:Params: {} 2022-09-03T07:37:58.285 DEBUG:boto:Token: None 2022-09-03T07:37:58.285 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.286 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:shm+6JUYdh/zGuc4QUtnm1NTu8Y= 2022-09-03T07:37:58.287 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:shm+6JUYdh/zGuc4QUtnm1NTu8Y=', 'Content-Length': '0'} 2022-09-03T07:37:58.289 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4b2cbfa5a133e83-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.290 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:37:58.291 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.291 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.292 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.293 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.293 DEBUG:boto:Method: GET 2022-09-03T07:37:58.294 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.295 DEBUG:boto:Data: 2022-09-03T07:37:58.295 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.296 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.296 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.297 DEBUG:boto:Params: {} 2022-09-03T07:37:58.298 DEBUG:boto:Token: None 2022-09-03T07:37:58.298 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.299 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZR8cVrg9Vsit/4aOAt0PFEDEpXc= 2022-09-03T07:37:58.302 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZR8cVrg9Vsit/4aOAt0PFEDEpXc=', 'Content-Length': '0'} 2022-09-03T07:37:58.309 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000369e4c504acffc17-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.310 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.310 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.311 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.311 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.312 DEBUG:boto:Method: GET 2022-09-03T07:37:58.312 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.313 DEBUG:boto:Data: 2022-09-03T07:37:58.313 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.314 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.315 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.315 DEBUG:boto:Params: {} 2022-09-03T07:37:58.316 DEBUG:boto:Token: None 2022-09-03T07:37:58.317 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:58.317 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZR8cVrg9Vsit/4aOAt0PFEDEpXc= 2022-09-03T07:37:58.318 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZR8cVrg9Vsit/4aOAt0PFEDEpXc=', 'Content-Length': '0'} 2022-09-03T07:37:58.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000396a4729fb23ce54-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.321 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.322 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.322 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:58.323 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:58.323 DEBUG:boto:Method: GET 2022-09-03T07:37:58.324 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:58.324 DEBUG:boto:Data: 2022-09-03T07:37:58.325 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.325 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.326 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.326 DEBUG:boto:Params: {} 2022-09-03T07:37:58.327 DEBUG:boto:Token: None 2022-09-03T07:37:58.328 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.328 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.329 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.332 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007b0fd32eaba5df0f-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.333 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.334 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.335 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.335 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:58.336 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:58.336 DEBUG:boto:Method: GET 2022-09-03T07:37:58.337 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:58.337 DEBUG:boto:Data: 2022-09-03T07:37:58.338 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.338 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.339 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.339 DEBUG:boto:Params: {} 2022-09-03T07:37:58.340 DEBUG:boto:Token: None 2022-09-03T07:37:58.341 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.341 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.342 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.345 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000016b10137a2ca1977-0063130456-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.345 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.346 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.347 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.347 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.348 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:58.348 DEBUG:boto:Data: 2022-09-03T07:37:58.349 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.349 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.350 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.350 DEBUG:boto:Params: {} 2022-09-03T07:37:58.351 DEBUG:boto:Token: None 2022-09-03T07:37:58.351 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44 2022-09-03T07:37:58.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sbA9NBXEkp5Z3g9yIeLKcRap/wg= 2022-09-03T07:37:58.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sbA9NBXEkp5Z3g9yIeLKcRap/wg=', 'Content-Length': '0'} 2022-09-03T07:37:58.355 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008d0cc49bce4f0551-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.356 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.356 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.357 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.357 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:58.358 DEBUG:boto:Data: 2022-09-03T07:37:58.358 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.359 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.359 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.360 DEBUG:boto:Params: {} 2022-09-03T07:37:58.361 DEBUG:boto:Token: None 2022-09-03T07:37:58.361 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44 2022-09-03T07:37:58.362 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:sbA9NBXEkp5Z3g9yIeLKcRap/wg= 2022-09-03T07:37:58.363 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:sbA9NBXEkp5Z3g9yIeLKcRap/wg=', 'Content-Length': '0'} 2022-09-03T07:37:58.371 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009f6a53927d828ee8-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.373 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:37:58.374 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.375 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.376 DEBUG:boto:Method: GET 2022-09-03T07:37:58.377 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:58.378 DEBUG:boto:Data: 2022-09-03T07:37:58.379 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.380 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.380 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.381 DEBUG:boto:Params: {} 2022-09-03T07:37:58.382 DEBUG:boto:Token: None 2022-09-03T07:37:58.382 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44 2022-09-03T07:37:58.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oE0RkIJAXhYxNHz6Cyp3ztVeSqk= 2022-09-03T07:37:58.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oE0RkIJAXhYxNHz6Cyp3ztVeSqk=', 'Content-Length': '0'} 2022-09-03T07:37:58.386 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a800e99587cdbf3b-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.430 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:58.432 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:58.433 DEBUG:boto:Method: GET 2022-09-03T07:37:58.434 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:58.436 DEBUG:boto:Data: 2022-09-03T07:37:58.436 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.437 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.438 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.439 DEBUG:boto:Params: {} 2022-09-03T07:37:58.440 DEBUG:boto:Token: None 2022-09-03T07:37:58.441 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj44 2022-09-03T07:37:58.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:oE0RkIJAXhYxNHz6Cyp3ztVeSqk= 2022-09-03T07:37:58.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:oE0RkIJAXhYxNHz6Cyp3ztVeSqk=', 'Content-Length': '0'} 2022-09-03T07:37:58.445 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bc92c7c01ea80dc3-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.446 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.446 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.447 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:58.447 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:58.448 DEBUG:boto:Method: GET 2022-09-03T07:37:58.449 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:58.449 DEBUG:boto:Data: 2022-09-03T07:37:58.450 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.451 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.451 DEBUG:boto:Params: {} 2022-09-03T07:37:58.452 DEBUG:boto:Token: None 2022-09-03T07:37:58.453 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.454 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.455 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.458 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000005a8a727e849d3788-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.459 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.459 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.460 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.461 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:58.461 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:58.462 DEBUG:boto:Method: GET 2022-09-03T07:37:58.462 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:58.463 DEBUG:boto:Data: 2022-09-03T07:37:58.464 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.465 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.465 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.466 DEBUG:boto:Params: {} 2022-09-03T07:37:58.467 DEBUG:boto:Token: None 2022-09-03T07:37:58.467 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.468 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.469 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.472 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000777c522127f8a4fd-0063130456-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.472 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.473 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:37:58.474 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.475 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.475 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.476 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.476 DEBUG:boto:Method: GET 2022-09-03T07:37:58.477 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.477 DEBUG:boto:Data: 2022-09-03T07:37:58.478 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.478 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.479 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.480 DEBUG:boto:Params: {} 2022-09-03T07:37:58.480 DEBUG:boto:Token: None 2022-09-03T07:37:58.481 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.481 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jws1C5eBk8LjmegDxA/Wedbae2M= 2022-09-03T07:37:58.482 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jws1C5eBk8LjmegDxA/Wedbae2M=', 'Content-Length': '0'} 2022-09-03T07:37:58.484 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000026d20f83b6a971a0-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.485 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.486 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.486 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.487 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.487 DEBUG:boto:Method: GET 2022-09-03T07:37:58.488 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.489 DEBUG:boto:Data: 2022-09-03T07:37:58.489 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.490 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.491 DEBUG:boto:Params: {} 2022-09-03T07:37:58.491 DEBUG:boto:Token: None 2022-09-03T07:37:58.492 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.492 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jws1C5eBk8LjmegDxA/Wedbae2M= 2022-09-03T07:37:58.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jws1C5eBk8LjmegDxA/Wedbae2M=', 'Content-Length': '0'} 2022-09-03T07:37:58.495 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000085585b3dc23d2c98-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.538 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.540 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.540 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.541 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.542 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.543 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.544 DEBUG:boto:Data: 2022-09-03T07:37:58.545 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.546 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.547 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.548 DEBUG:boto:Params: {} 2022-09-03T07:37:58.549 DEBUG:boto:Token: None 2022-09-03T07:37:58.550 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gCVn1tuGiyIFjASyCbnJ03d92xQ= 2022-09-03T07:37:58.551 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gCVn1tuGiyIFjASyCbnJ03d92xQ=', 'Content-Length': '0'} 2022-09-03T07:37:58.553 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f99ce1f16efbb67b-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.554 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.555 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.555 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.556 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.556 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.557 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.557 DEBUG:boto:Data: 2022-09-03T07:37:58.558 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.558 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.559 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.559 DEBUG:boto:Params: {} 2022-09-03T07:37:58.560 DEBUG:boto:Token: None 2022-09-03T07:37:58.561 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.561 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gCVn1tuGiyIFjASyCbnJ03d92xQ= 2022-09-03T07:37:58.562 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gCVn1tuGiyIFjASyCbnJ03d92xQ=', 'Content-Length': '0'} 2022-09-03T07:37:58.564 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003befbde7817b1c31-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.565 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:37:58.565 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.566 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.567 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.567 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.568 DEBUG:boto:Method: GET 2022-09-03T07:37:58.568 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.569 DEBUG:boto:Data: 2022-09-03T07:37:58.569 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.570 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.571 DEBUG:boto:Params: {} 2022-09-03T07:37:58.571 DEBUG:boto:Token: None 2022-09-03T07:37:58.572 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.573 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jws1C5eBk8LjmegDxA/Wedbae2M= 2022-09-03T07:37:58.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jws1C5eBk8LjmegDxA/Wedbae2M=', 'Content-Length': '0'} 2022-09-03T07:37:58.577 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ffb4d85074fb9131-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.578 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.579 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.580 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.581 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.582 DEBUG:boto:Method: GET 2022-09-03T07:37:58.583 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.584 DEBUG:boto:Data: 2022-09-03T07:37:58.585 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.586 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.586 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.587 DEBUG:boto:Params: {} 2022-09-03T07:37:58.588 DEBUG:boto:Token: None 2022-09-03T07:37:58.589 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:58.590 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Jws1C5eBk8LjmegDxA/Wedbae2M= 2022-09-03T07:37:58.591 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Jws1C5eBk8LjmegDxA/Wedbae2M=', 'Content-Length': '0'} 2022-09-03T07:37:58.595 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000812fda7b54a4689b-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.598 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.599 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.599 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:58.600 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:58.600 DEBUG:boto:Method: GET 2022-09-03T07:37:58.601 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:37:58.602 DEBUG:boto:Data: 2022-09-03T07:37:58.602 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.603 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.603 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.604 DEBUG:boto:Params: {} 2022-09-03T07:37:58.604 DEBUG:boto:Token: None 2022-09-03T07:37:58.605 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.606 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.606 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.609 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c0e6d7d2e32e70b3-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.610 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.611 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.612 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.612 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:58.613 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:58.613 DEBUG:boto:Method: GET 2022-09-03T07:37:58.614 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:37:58.614 DEBUG:boto:Data: 2022-09-03T07:37:58.615 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.616 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.616 DEBUG:boto:Params: {} 2022-09-03T07:37:58.617 DEBUG:boto:Token: None 2022-09-03T07:37:58.618 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.618 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.619 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.622 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000030d987b8ef559be1-0063130456-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.623 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.624 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.624 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.625 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.625 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:37:58.626 DEBUG:boto:Data: 2022-09-03T07:37:58.626 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.627 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.627 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.628 DEBUG:boto:Params: {} 2022-09-03T07:37:58.628 DEBUG:boto:Token: None 2022-09-03T07:37:58.629 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55 2022-09-03T07:37:58.630 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Om7IzxOKE/5Alr/VdI9Vjyjrok= 2022-09-03T07:37:58.630 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Om7IzxOKE/5Alr/VdI9Vjyjrok=', 'Content-Length': '0'} 2022-09-03T07:37:58.633 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008634067aea5d25c1-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.634 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.634 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.635 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.637 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:37:58.638 DEBUG:boto:Data: 2022-09-03T07:37:58.639 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.640 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.641 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.642 DEBUG:boto:Params: {} 2022-09-03T07:37:58.644 DEBUG:boto:Token: None 2022-09-03T07:37:58.645 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55 2022-09-03T07:37:58.646 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:0Om7IzxOKE/5Alr/VdI9Vjyjrok= 2022-09-03T07:37:58.648 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:0Om7IzxOKE/5Alr/VdI9Vjyjrok=', 'Content-Length': '0'} 2022-09-03T07:37:58.652 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d29ca77b0affc313-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.653 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:37:58.653 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.654 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.655 DEBUG:boto:Method: GET 2022-09-03T07:37:58.656 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:37:58.656 DEBUG:boto:Data: 2022-09-03T07:37:58.657 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.657 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.658 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.658 DEBUG:boto:Params: {} 2022-09-03T07:37:58.659 DEBUG:boto:Token: None 2022-09-03T07:37:58.660 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55 2022-09-03T07:37:58.660 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4c1YRPJSE/FauZkMrgpD8Owfw9g= 2022-09-03T07:37:58.661 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4c1YRPJSE/FauZkMrgpD8Owfw9g=', 'Content-Length': '0'} 2022-09-03T07:37:58.663 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000d71c5f4332f68126-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.664 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:58.664 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:58.665 DEBUG:boto:Method: GET 2022-09-03T07:37:58.665 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:37:58.666 DEBUG:boto:Data: 2022-09-03T07:37:58.666 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.667 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.668 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.668 DEBUG:boto:Params: {} 2022-09-03T07:37:58.669 DEBUG:boto:Token: None 2022-09-03T07:37:58.669 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj55 2022-09-03T07:37:58.670 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4c1YRPJSE/FauZkMrgpD8Owfw9g= 2022-09-03T07:37:58.671 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4c1YRPJSE/FauZkMrgpD8Owfw9g=', 'Content-Length': '0'} 2022-09-03T07:37:58.673 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000066a04cc631a134be-0063130456-1604-test-zone1'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.674 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.674 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.675 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:58.675 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:58.676 DEBUG:boto:Method: GET 2022-09-03T07:37:58.676 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:37:58.677 DEBUG:boto:Data: 2022-09-03T07:37:58.677 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.678 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.679 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.679 DEBUG:boto:Params: {} 2022-09-03T07:37:58.680 DEBUG:boto:Token: None 2022-09-03T07:37:58.680 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.681 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.682 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.685 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007a2d9667252635fc-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.686 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.687 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.688 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.689 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:58.689 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:37:58.690 DEBUG:boto:Method: GET 2022-09-03T07:37:58.690 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:37:58.691 DEBUG:boto:Data: 2022-09-03T07:37:58.691 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.692 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:37:58.692 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.693 DEBUG:boto:Params: {} 2022-09-03T07:37:58.693 DEBUG:boto:Token: None 2022-09-03T07:37:58.694 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.695 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.695 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.698 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c4c83e3cd9be4d52-0063130456-1604-test-zone1'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.699 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.700 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-76 zones={test-zone3, test-zone1} 2022-09-03T07:37:58.701 INFO:tasks.rgw_multi.zone_rados:comparing bucket=avysmw-76 zones={test-zone3, test-zone2} 2022-09-03T07:37:58.701 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.702 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.702 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.703 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:37:58.703 DEBUG:boto:Data: 2022-09-03T07:37:58.704 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.704 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.705 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.706 DEBUG:boto:Params: {} 2022-09-03T07:37:58.706 DEBUG:boto:Token: None 2022-09-03T07:37:58.707 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.707 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F5walCUQnY79//e8ef5erpBdGtI= 2022-09-03T07:37:58.708 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F5walCUQnY79//e8ef5erpBdGtI=', 'Content-Length': '0'} 2022-09-03T07:37:58.711 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000002addfdf77d21bf4f-0063130456-18f3-test-zone3'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.712 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.713 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.714 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.715 DEBUG:boto:Path: /avysmw-76/ 2022-09-03T07:37:58.716 DEBUG:boto:Data: 2022-09-03T07:37:58.717 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.717 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:58.718 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.719 DEBUG:boto:Params: {} 2022-09-03T07:37:58.719 DEBUG:boto:Token: None 2022-09-03T07:37:58.720 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.721 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:F5walCUQnY79//e8ef5erpBdGtI= 2022-09-03T07:37:58.722 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:F5walCUQnY79//e8ef5erpBdGtI=', 'Content-Length': '0'} 2022-09-03T07:37:58.726 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003007aaa6ce8b7dd6-0063130456-1b17-test-zone2'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.727 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2022-09-03T07:37:58.728 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.728 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.729 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.729 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.730 DEBUG:boto:Method: GET 2022-09-03T07:37:58.731 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.731 DEBUG:boto:Data: 2022-09-03T07:37:58.732 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.732 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.733 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.733 DEBUG:boto:Params: {} 2022-09-03T07:37:58.734 DEBUG:boto:Token: None 2022-09-03T07:37:58.735 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/?versions 2022-09-03T07:37:58.735 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r/cPHKk4YLeBNeSt5hvE4EtCksA= 2022-09-03T07:37:58.736 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r/cPHKk4YLeBNeSt5hvE4EtCksA=', 'Content-Length': '0'} 2022-09-03T07:37:58.741 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007cc925080019948d-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.742 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.744 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:37:58.745 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:37:58.746 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:37:58.747 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:37:58.748 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:37:58.749 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2022-09-03T07:37:58.750 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.751 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.752 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.753 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.754 DEBUG:boto:Method: GET 2022-09-03T07:37:58.755 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.755 DEBUG:boto:Data: 2022-09-03T07:37:58.756 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.756 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:58.757 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.758 DEBUG:boto:Params: {} 2022-09-03T07:37:58.759 DEBUG:boto:Token: None 2022-09-03T07:37:58.760 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/?versions 2022-09-03T07:37:58.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r/cPHKk4YLeBNeSt5hvE4EtCksA= 2022-09-03T07:37:58.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r/cPHKk4YLeBNeSt5hvE4EtCksA=', 'Content-Length': '0'} 2022-09-03T07:37:58.764 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000003cea9afda9218934-0063130456-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.764 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.765 DEBUG:tasks.rgw_multi.zone_rados:o=obj11 2022-09-03T07:37:58.766 DEBUG:tasks.rgw_multi.zone_rados:o=obj22 2022-09-03T07:37:58.767 DEBUG:tasks.rgw_multi.zone_rados:o=obj33 2022-09-03T07:37:58.767 DEBUG:tasks.rgw_multi.zone_rados:o=obj44 2022-09-03T07:37:58.768 DEBUG:tasks.rgw_multi.zone_rados:o=obj55 2022-09-03T07:37:58.769 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.770 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.771 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.772 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.773 DEBUG:boto:Method: GET 2022-09-03T07:37:58.774 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.775 DEBUG:boto:Data: 2022-09-03T07:37:58.776 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.777 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.778 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.779 DEBUG:boto:Params: {} 2022-09-03T07:37:58.779 DEBUG:boto:Token: None 2022-09-03T07:37:58.780 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/?versions 2022-09-03T07:37:58.781 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r/cPHKk4YLeBNeSt5hvE4EtCksA= 2022-09-03T07:37:58.781 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r/cPHKk4YLeBNeSt5hvE4EtCksA=', 'Content-Length': '0'} 2022-09-03T07:37:58.785 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc9625d158c4f848-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.785 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.787 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.787 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.788 DEBUG:boto:path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.788 DEBUG:boto:auth_path=/avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.789 DEBUG:boto:Method: GET 2022-09-03T07:37:58.789 DEBUG:boto:Path: /avysmw-76/?versions&max-keys=999 2022-09-03T07:37:58.790 DEBUG:boto:Data: 2022-09-03T07:37:58.792 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.793 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:58.793 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.794 DEBUG:boto:Params: {} 2022-09-03T07:37:58.795 DEBUG:boto:Token: None 2022-09-03T07:37:58.796 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/?versions 2022-09-03T07:37:58.796 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r/cPHKk4YLeBNeSt5hvE4EtCksA= 2022-09-03T07:37:58.797 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r/cPHKk4YLeBNeSt5hvE4EtCksA=', 'Content-Length': '0'} 2022-09-03T07:37:58.799 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000a8c9b35c8118e88f-0063130456-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.800 DEBUG:boto:b'avysmw-76999falseobj11nulltrue2022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj22nulltrue2022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj33nulltrue2022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj44nulltrue2022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormalobj55nulltrue2022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.802 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:37:58.802 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.803 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.803 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.804 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.804 DEBUG:boto:Method: GET 2022-09-03T07:37:58.805 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.805 DEBUG:boto:Data: 2022-09-03T07:37:58.806 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.807 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.807 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.808 DEBUG:boto:Params: {} 2022-09-03T07:37:58.808 DEBUG:boto:Token: None 2022-09-03T07:37:58.815 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.816 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HpMIy2Vkgz2FLvubSTvgdG8LUqg= 2022-09-03T07:37:58.816 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HpMIy2Vkgz2FLvubSTvgdG8LUqg=', 'Content-Length': '0'} 2022-09-03T07:37:58.819 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099e0fac79b5f3f32-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.820 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.820 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.821 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.822 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.822 DEBUG:boto:Method: GET 2022-09-03T07:37:58.823 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.823 DEBUG:boto:Data: 2022-09-03T07:37:58.824 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.825 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:58.826 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.827 DEBUG:boto:Params: {} 2022-09-03T07:37:58.828 DEBUG:boto:Token: None 2022-09-03T07:37:58.829 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.829 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HpMIy2Vkgz2FLvubSTvgdG8LUqg= 2022-09-03T07:37:58.830 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HpMIy2Vkgz2FLvubSTvgdG8LUqg=', 'Content-Length': '0'} 2022-09-03T07:37:58.832 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ae47c2ec197fdc2c-0063130456-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.874 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.875 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.875 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.876 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.877 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.877 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.878 DEBUG:boto:Data: 2022-09-03T07:37:58.878 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.879 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.880 DEBUG:boto:Params: {} 2022-09-03T07:37:58.880 DEBUG:boto:Token: None 2022-09-03T07:37:58.881 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.882 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SAEX1D7H+8BJj/ZYXI9XdPMcwt8= 2022-09-03T07:37:58.883 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SAEX1D7H+8BJj/ZYXI9XdPMcwt8=', 'Content-Length': '0'} 2022-09-03T07:37:58.885 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000099f897a1b776244e-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.886 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.886 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.887 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.887 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.888 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.888 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.889 DEBUG:boto:Data: 2022-09-03T07:37:58.889 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:58.890 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.891 DEBUG:boto:Params: {} 2022-09-03T07:37:58.892 DEBUG:boto:Token: None 2022-09-03T07:37:58.892 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:SAEX1D7H+8BJj/ZYXI9XdPMcwt8= 2022-09-03T07:37:58.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:SAEX1D7H+8BJj/ZYXI9XdPMcwt8=', 'Content-Length': '0'} 2022-09-03T07:37:58.896 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000065e4f1a31bd5f2e0-0063130456-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.897 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:37:58.897 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.898 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.899 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.900 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.900 DEBUG:boto:Method: GET 2022-09-03T07:37:58.901 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.901 DEBUG:boto:Data: 2022-09-03T07:37:58.902 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.903 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.904 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.904 DEBUG:boto:Params: {} 2022-09-03T07:37:58.905 DEBUG:boto:Token: None 2022-09-03T07:37:58.906 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.906 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HpMIy2Vkgz2FLvubSTvgdG8LUqg= 2022-09-03T07:37:58.907 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HpMIy2Vkgz2FLvubSTvgdG8LUqg=', 'Content-Length': '0'} 2022-09-03T07:37:58.909 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000449d58e459f9aae1-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.910 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.911 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.911 DEBUG:boto:path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.912 DEBUG:boto:auth_path=/avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.912 DEBUG:boto:Method: GET 2022-09-03T07:37:58.913 DEBUG:boto:Path: /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.913 DEBUG:boto:Data: 2022-09-03T07:37:58.914 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.914 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:58.915 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.915 DEBUG:boto:Params: {} 2022-09-03T07:37:58.916 DEBUG:boto:Token: None 2022-09-03T07:37:58.917 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11?versionId=null 2022-09-03T07:37:58.917 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:HpMIy2Vkgz2FLvubSTvgdG8LUqg= 2022-09-03T07:37:58.918 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:HpMIy2Vkgz2FLvubSTvgdG8LUqg=', 'Content-Length': '0'} 2022-09-03T07:37:58.920 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000cc62794b788c1475-0063130456-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.921 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.921 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.922 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:58.922 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:58.923 DEBUG:boto:Method: GET 2022-09-03T07:37:58.923 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:58.924 DEBUG:boto:Data: 2022-09-03T07:37:58.924 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.925 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.926 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.926 DEBUG:boto:Params: {} 2022-09-03T07:37:58.927 DEBUG:boto:Token: None 2022-09-03T07:37:58.927 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.928 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.929 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.932 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000d9b4a98de1001cd8-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.932 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.933 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.934 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.934 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:58.935 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:58.935 DEBUG:boto:Method: GET 2022-09-03T07:37:58.936 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:58.936 DEBUG:boto:Data: 2022-09-03T07:37:58.937 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.937 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:58.938 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.939 DEBUG:boto:Params: {} 2022-09-03T07:37:58.939 DEBUG:boto:Token: None 2022-09-03T07:37:58.940 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.940 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.941 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.944 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000e1aa0d29bfe7c41a-0063130456-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.944 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.945 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.946 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.946 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.947 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:58.947 DEBUG:boto:Data: 2022-09-03T07:37:58.948 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.948 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.949 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.949 DEBUG:boto:Params: {} 2022-09-03T07:37:58.950 DEBUG:boto:Token: None 2022-09-03T07:37:58.950 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11 2022-09-03T07:37:58.951 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mdy0AjkCEoDx2jX2GZLjYw3oPaY= 2022-09-03T07:37:58.951 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mdy0AjkCEoDx2jX2GZLjYw3oPaY=', 'Content-Length': '0'} 2022-09-03T07:37:58.954 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004cb3eda63e9ff941-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.954 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.955 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.955 DEBUG:boto:Method: HEAD 2022-09-03T07:37:58.956 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:58.956 DEBUG:boto:Data: 2022-09-03T07:37:58.957 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.957 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:58.958 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.958 DEBUG:boto:Params: {} 2022-09-03T07:37:58.959 DEBUG:boto:Token: None 2022-09-03T07:37:58.960 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11 2022-09-03T07:37:58.960 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Mdy0AjkCEoDx2jX2GZLjYw3oPaY= 2022-09-03T07:37:58.961 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Mdy0AjkCEoDx2jX2GZLjYw3oPaY=', 'Content-Length': '0'} 2022-09-03T07:37:58.963 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000009294bcb0fb8f800f-0063130456-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.964 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2022-09-03T07:37:58.964 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.965 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.965 DEBUG:boto:Method: GET 2022-09-03T07:37:58.966 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:58.966 DEBUG:boto:Data: 2022-09-03T07:37:58.967 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.967 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.968 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.969 DEBUG:boto:Params: {} 2022-09-03T07:37:58.969 DEBUG:boto:Token: None 2022-09-03T07:37:58.970 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11 2022-09-03T07:37:58.971 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wOwpuGKOkfbny4+Br4E+lDQQ4ZQ= 2022-09-03T07:37:58.971 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wOwpuGKOkfbny4+Br4E+lDQQ4ZQ=', 'Content-Length': '0'} 2022-09-03T07:37:58.973 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004d834d03185e21f8-0063130456-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.974 DEBUG:boto:path=/avysmw-76/obj11 2022-09-03T07:37:58.975 DEBUG:boto:auth_path=/avysmw-76/obj11 2022-09-03T07:37:58.975 DEBUG:boto:Method: GET 2022-09-03T07:37:58.976 DEBUG:boto:Path: /avysmw-76/obj11 2022-09-03T07:37:58.976 DEBUG:boto:Data: 2022-09-03T07:37:58.977 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.977 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:58.978 DEBUG:boto:Port: 8000 2022-09-03T07:37:58.979 DEBUG:boto:Params: {} 2022-09-03T07:37:58.979 DEBUG:boto:Token: None 2022-09-03T07:37:58.980 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/obj11 2022-09-03T07:37:58.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:wOwpuGKOkfbny4+Br4E+lDQQ4ZQ= 2022-09-03T07:37:58.981 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:wOwpuGKOkfbny4+Br4E+lDQQ4ZQ=', 'Content-Length': '0'} 2022-09-03T07:37:58.983 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000005f9c44891a3f94ea-0063130456-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.984 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.984 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.985 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:58.986 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:58.986 DEBUG:boto:Method: GET 2022-09-03T07:37:58.987 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:58.987 DEBUG:boto:Data: 2022-09-03T07:37:58.988 DEBUG:boto:Headers: {} 2022-09-03T07:37:58.988 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:58.989 DEBUG:boto:Port: 8001 2022-09-03T07:37:58.989 DEBUG:boto:Params: {} 2022-09-03T07:37:58.990 DEBUG:boto:Token: None 2022-09-03T07:37:58.991 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:58 GMT /avysmw-76/ 2022-09-03T07:37:58.991 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM= 2022-09-03T07:37:58.992 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:58 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:IFBpFASUW3noowH1HLDjcOhhnNM=', 'Content-Length': '0'} 2022-09-03T07:37:58.995 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000023cd3e36602a45a0-0063130456-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:58 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:58.995 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:58.996 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:58.997 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:58.997 DEBUG:boto:path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:58.998 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj11 2022-09-03T07:37:58.998 DEBUG:boto:Method: GET 2022-09-03T07:37:58.999 DEBUG:boto:Path: /avysmw-76/?prefix=obj11 2022-09-03T07:37:59.000 DEBUG:boto:Data: 2022-09-03T07:37:59.000 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.001 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.001 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.002 DEBUG:boto:Params: {} 2022-09-03T07:37:59.002 DEBUG:boto:Token: None 2022-09-03T07:37:59.003 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.004 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.004 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.007 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000edd4995089ec48db-0063130457-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.008 DEBUG:boto:b'avysmw-76obj111000falseobj112022-09-03T07:33:12.405Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.009 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:37:59.010 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.011 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.012 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.013 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.014 DEBUG:boto:Method: GET 2022-09-03T07:37:59.015 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.016 DEBUG:boto:Data: 2022-09-03T07:37:59.017 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.018 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.019 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.020 DEBUG:boto:Params: {} 2022-09-03T07:37:59.021 DEBUG:boto:Token: None 2022-09-03T07:37:59.022 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.023 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q7ZzP9kwYp38qhBQ3NWq1xGGImA= 2022-09-03T07:37:59.025 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q7ZzP9kwYp38qhBQ3NWq1xGGImA=', 'Content-Length': '0'} 2022-09-03T07:37:59.028 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000153bd3a9409b42ed-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.029 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.030 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.031 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.032 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.033 DEBUG:boto:Method: GET 2022-09-03T07:37:59.034 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.035 DEBUG:boto:Data: 2022-09-03T07:37:59.035 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.036 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.036 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.037 DEBUG:boto:Params: {} 2022-09-03T07:37:59.038 DEBUG:boto:Token: None 2022-09-03T07:37:59.038 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.039 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q7ZzP9kwYp38qhBQ3NWq1xGGImA= 2022-09-03T07:37:59.039 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q7ZzP9kwYp38qhBQ3NWq1xGGImA=', 'Content-Length': '0'} 2022-09-03T07:37:59.041 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000fa9960c4ecde9b21-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.082 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.083 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.084 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.084 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.085 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.085 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.086 DEBUG:boto:Data: 2022-09-03T07:37:59.086 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.087 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.088 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.089 DEBUG:boto:Params: {} 2022-09-03T07:37:59.090 DEBUG:boto:Token: None 2022-09-03T07:37:59.091 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.092 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BIKWiK5oANYkg30yYh5lV6ia0sg= 2022-09-03T07:37:59.093 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BIKWiK5oANYkg30yYh5lV6ia0sg=', 'Content-Length': '0'} 2022-09-03T07:37:59.095 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001f1000ee2b1ab1f4-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.096 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.097 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.097 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.098 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.099 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.099 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.100 DEBUG:boto:Data: 2022-09-03T07:37:59.100 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.101 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.101 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.102 DEBUG:boto:Params: {} 2022-09-03T07:37:59.103 DEBUG:boto:Token: None 2022-09-03T07:37:59.104 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.104 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:BIKWiK5oANYkg30yYh5lV6ia0sg= 2022-09-03T07:37:59.105 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:BIKWiK5oANYkg30yYh5lV6ia0sg=', 'Content-Length': '0'} 2022-09-03T07:37:59.108 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c23f0bb9693c6635-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.108 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:37:59.109 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.109 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.110 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.111 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.112 DEBUG:boto:Method: GET 2022-09-03T07:37:59.113 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.113 DEBUG:boto:Data: 2022-09-03T07:37:59.114 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.114 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.115 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.115 DEBUG:boto:Params: {} 2022-09-03T07:37:59.116 DEBUG:boto:Token: None 2022-09-03T07:37:59.117 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.117 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q7ZzP9kwYp38qhBQ3NWq1xGGImA= 2022-09-03T07:37:59.118 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q7ZzP9kwYp38qhBQ3NWq1xGGImA=', 'Content-Length': '0'} 2022-09-03T07:37:59.120 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006ecde689647eba25-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.121 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.123 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.124 DEBUG:boto:path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.125 DEBUG:boto:auth_path=/avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.126 DEBUG:boto:Method: GET 2022-09-03T07:37:59.127 DEBUG:boto:Path: /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.127 DEBUG:boto:Data: 2022-09-03T07:37:59.128 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.129 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.129 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.130 DEBUG:boto:Params: {} 2022-09-03T07:37:59.130 DEBUG:boto:Token: None 2022-09-03T07:37:59.131 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22?versionId=null 2022-09-03T07:37:59.132 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Q7ZzP9kwYp38qhBQ3NWq1xGGImA= 2022-09-03T07:37:59.132 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Q7ZzP9kwYp38qhBQ3NWq1xGGImA=', 'Content-Length': '0'} 2022-09-03T07:37:59.135 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ac7b4df0f48998c6-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.136 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.137 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.138 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:59.139 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:59.140 DEBUG:boto:Method: GET 2022-09-03T07:37:59.141 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:59.141 DEBUG:boto:Data: 2022-09-03T07:37:59.142 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.143 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.144 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.145 DEBUG:boto:Params: {} 2022-09-03T07:37:59.147 DEBUG:boto:Token: None 2022-09-03T07:37:59.148 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.149 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.149 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.153 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000058758ddf3995c270-0063130457-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.153 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.154 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.155 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.155 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:59.156 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:59.156 DEBUG:boto:Method: GET 2022-09-03T07:37:59.157 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:59.157 DEBUG:boto:Data: 2022-09-03T07:37:59.158 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.159 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.159 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.160 DEBUG:boto:Params: {} 2022-09-03T07:37:59.160 DEBUG:boto:Token: None 2022-09-03T07:37:59.161 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.161 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.162 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.166 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ed4d15b10a659db5-0063130457-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.167 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.168 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.169 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.170 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.171 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:59.171 DEBUG:boto:Data: 2022-09-03T07:37:59.172 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.173 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.174 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.175 DEBUG:boto:Params: {} 2022-09-03T07:37:59.176 DEBUG:boto:Token: None 2022-09-03T07:37:59.177 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22 2022-09-03T07:37:59.178 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kFCbXGHcLwFPwtoTUmmExWn506I= 2022-09-03T07:37:59.179 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kFCbXGHcLwFPwtoTUmmExWn506I=', 'Content-Length': '0'} 2022-09-03T07:37:59.181 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000e2930b223d37585e-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.183 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.184 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.184 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.185 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:59.185 DEBUG:boto:Data: 2022-09-03T07:37:59.186 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.187 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.187 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.188 DEBUG:boto:Params: {} 2022-09-03T07:37:59.188 DEBUG:boto:Token: None 2022-09-03T07:37:59.189 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22 2022-09-03T07:37:59.189 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kFCbXGHcLwFPwtoTUmmExWn506I= 2022-09-03T07:37:59.190 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kFCbXGHcLwFPwtoTUmmExWn506I=', 'Content-Length': '0'} 2022-09-03T07:37:59.192 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000aeb7a26279301860-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.193 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2022-09-03T07:37:59.193 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.194 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.194 DEBUG:boto:Method: GET 2022-09-03T07:37:59.195 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:59.195 DEBUG:boto:Data: 2022-09-03T07:37:59.196 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.196 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.197 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.198 DEBUG:boto:Params: {} 2022-09-03T07:37:59.198 DEBUG:boto:Token: None 2022-09-03T07:37:59.199 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22 2022-09-03T07:37:59.200 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W6YPVTvlN/0Z0JJYW9ekXDlox+w= 2022-09-03T07:37:59.200 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W6YPVTvlN/0Z0JJYW9ekXDlox+w=', 'Content-Length': '0'} 2022-09-03T07:37:59.203 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000019ad9e2bc42098b5-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.204 DEBUG:boto:path=/avysmw-76/obj22 2022-09-03T07:37:59.204 DEBUG:boto:auth_path=/avysmw-76/obj22 2022-09-03T07:37:59.205 DEBUG:boto:Method: GET 2022-09-03T07:37:59.206 DEBUG:boto:Path: /avysmw-76/obj22 2022-09-03T07:37:59.207 DEBUG:boto:Data: 2022-09-03T07:37:59.208 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.209 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.210 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.211 DEBUG:boto:Params: {} 2022-09-03T07:37:59.212 DEBUG:boto:Token: None 2022-09-03T07:37:59.213 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj22 2022-09-03T07:37:59.215 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:W6YPVTvlN/0Z0JJYW9ekXDlox+w= 2022-09-03T07:37:59.216 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:W6YPVTvlN/0Z0JJYW9ekXDlox+w=', 'Content-Length': '0'} 2022-09-03T07:37:59.219 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ee031c2be4546efe-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.266 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.268 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.269 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:59.269 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:59.270 DEBUG:boto:Method: GET 2022-09-03T07:37:59.271 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:59.272 DEBUG:boto:Data: 2022-09-03T07:37:59.273 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.275 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.276 DEBUG:boto:Params: {} 2022-09-03T07:37:59.277 DEBUG:boto:Token: None 2022-09-03T07:37:59.279 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.280 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.281 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.285 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000bcd3469832354075-0063130457-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.286 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.288 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.289 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.290 DEBUG:boto:path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:59.291 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj22 2022-09-03T07:37:59.292 DEBUG:boto:Method: GET 2022-09-03T07:37:59.292 DEBUG:boto:Path: /avysmw-76/?prefix=obj22 2022-09-03T07:37:59.293 DEBUG:boto:Data: 2022-09-03T07:37:59.294 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.295 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.296 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.297 DEBUG:boto:Params: {} 2022-09-03T07:37:59.298 DEBUG:boto:Token: None 2022-09-03T07:37:59.299 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.300 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.301 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.304 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f1d015cbe1517b24-0063130457-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.305 DEBUG:boto:b'avysmw-76obj221000falseobj222022-09-03T07:33:12.434Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.306 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:37:59.307 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.308 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.308 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.309 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.310 DEBUG:boto:Method: GET 2022-09-03T07:37:59.311 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.311 DEBUG:boto:Data: 2022-09-03T07:37:59.312 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.312 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.313 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.314 DEBUG:boto:Params: {} 2022-09-03T07:37:59.315 DEBUG:boto:Token: None 2022-09-03T07:37:59.315 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.316 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uapRZYKXvrf8OIUnjfeEvqSXh2s= 2022-09-03T07:37:59.317 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uapRZYKXvrf8OIUnjfeEvqSXh2s=', 'Content-Length': '0'} 2022-09-03T07:37:59.320 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001d91a2ce12691b2c-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.322 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.323 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.324 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.325 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.326 DEBUG:boto:Method: GET 2022-09-03T07:37:59.327 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.327 DEBUG:boto:Data: 2022-09-03T07:37:59.328 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.329 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.330 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.331 DEBUG:boto:Params: {} 2022-09-03T07:37:59.332 DEBUG:boto:Token: None 2022-09-03T07:37:59.333 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.334 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uapRZYKXvrf8OIUnjfeEvqSXh2s= 2022-09-03T07:37:59.336 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uapRZYKXvrf8OIUnjfeEvqSXh2s=', 'Content-Length': '0'} 2022-09-03T07:37:59.338 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000f39c024440458151-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.340 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.341 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.342 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.342 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.343 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.344 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.346 DEBUG:boto:Data: 2022-09-03T07:37:59.347 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.347 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.348 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.349 DEBUG:boto:Params: {} 2022-09-03T07:37:59.350 DEBUG:boto:Token: None 2022-09-03T07:37:59.351 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.352 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lXpSYcDQ+0hyxjqJPgYnQNzfS28= 2022-09-03T07:37:59.353 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lXpSYcDQ+0hyxjqJPgYnQNzfS28=', 'Content-Length': '0'} 2022-09-03T07:37:59.356 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001693987c8db2224b-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.358 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.359 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.360 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.361 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.361 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.362 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.363 DEBUG:boto:Data: 2022-09-03T07:37:59.364 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.364 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.365 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.366 DEBUG:boto:Params: {} 2022-09-03T07:37:59.367 DEBUG:boto:Token: None 2022-09-03T07:37:59.368 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.368 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:lXpSYcDQ+0hyxjqJPgYnQNzfS28= 2022-09-03T07:37:59.369 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:lXpSYcDQ+0hyxjqJPgYnQNzfS28=', 'Content-Length': '0'} 2022-09-03T07:37:59.374 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000567483999d06f625-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.375 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:37:59.375 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.376 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.377 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.378 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.378 DEBUG:boto:Method: GET 2022-09-03T07:37:59.379 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.380 DEBUG:boto:Data: 2022-09-03T07:37:59.381 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.382 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.383 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.384 DEBUG:boto:Params: {} 2022-09-03T07:37:59.385 DEBUG:boto:Token: None 2022-09-03T07:37:59.385 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.386 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uapRZYKXvrf8OIUnjfeEvqSXh2s= 2022-09-03T07:37:59.387 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uapRZYKXvrf8OIUnjfeEvqSXh2s=', 'Content-Length': '0'} 2022-09-03T07:37:59.390 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000008832eac26cdbbaf8-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.391 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.392 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.393 DEBUG:boto:path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.393 DEBUG:boto:auth_path=/avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.394 DEBUG:boto:Method: GET 2022-09-03T07:37:59.395 DEBUG:boto:Path: /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.396 DEBUG:boto:Data: 2022-09-03T07:37:59.397 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.398 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.399 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.400 DEBUG:boto:Params: {} 2022-09-03T07:37:59.401 DEBUG:boto:Token: None 2022-09-03T07:37:59.402 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33?versionId=null 2022-09-03T07:37:59.403 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uapRZYKXvrf8OIUnjfeEvqSXh2s= 2022-09-03T07:37:59.404 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uapRZYKXvrf8OIUnjfeEvqSXh2s=', 'Content-Length': '0'} 2022-09-03T07:37:59.407 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b8d080739e03484b-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.409 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.410 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.411 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:59.412 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:59.413 DEBUG:boto:Method: GET 2022-09-03T07:37:59.413 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:59.414 DEBUG:boto:Data: 2022-09-03T07:37:59.415 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.416 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.418 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.419 DEBUG:boto:Params: {} 2022-09-03T07:37:59.421 DEBUG:boto:Token: None 2022-09-03T07:37:59.423 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.424 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.426 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.430 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000098ba1ad7e18ab2a3-0063130457-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.474 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.476 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.477 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.478 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:59.479 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:59.480 DEBUG:boto:Method: GET 2022-09-03T07:37:59.481 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:59.482 DEBUG:boto:Data: 2022-09-03T07:37:59.483 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.484 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.485 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.486 DEBUG:boto:Params: {} 2022-09-03T07:37:59.487 DEBUG:boto:Token: None 2022-09-03T07:37:59.488 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.488 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.489 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.495 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000ef95be5861b416f8-0063130457-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.495 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.497 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.498 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.498 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.499 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:59.500 DEBUG:boto:Data: 2022-09-03T07:37:59.501 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.502 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.503 DEBUG:boto:Params: {} 2022-09-03T07:37:59.504 DEBUG:boto:Token: None 2022-09-03T07:37:59.505 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33 2022-09-03T07:37:59.507 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T587Vc/rDLlMchpdTNyPSeEyP7k= 2022-09-03T07:37:59.508 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T587Vc/rDLlMchpdTNyPSeEyP7k=', 'Content-Length': '0'} 2022-09-03T07:37:59.510 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b74ee583dc43ad5f-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.511 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.512 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.513 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.514 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:59.518 DEBUG:boto:Data: 2022-09-03T07:37:59.519 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.521 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.523 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.525 DEBUG:boto:Params: {} 2022-09-03T07:37:59.527 DEBUG:boto:Token: None 2022-09-03T07:37:59.529 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33 2022-09-03T07:37:59.531 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T587Vc/rDLlMchpdTNyPSeEyP7k= 2022-09-03T07:37:59.534 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T587Vc/rDLlMchpdTNyPSeEyP7k=', 'Content-Length': '0'} 2022-09-03T07:37:59.537 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c43c249980207aa5-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.538 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2022-09-03T07:37:59.539 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.540 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.541 DEBUG:boto:Method: GET 2022-09-03T07:37:59.542 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:59.544 DEBUG:boto:Data: 2022-09-03T07:37:59.544 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.545 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.547 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.548 DEBUG:boto:Params: {} 2022-09-03T07:37:59.548 DEBUG:boto:Token: None 2022-09-03T07:37:59.549 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33 2022-09-03T07:37:59.550 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/rdZh2EM3zzlcCd+39LeTrpyrs0= 2022-09-03T07:37:59.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/rdZh2EM3zzlcCd+39LeTrpyrs0=', 'Content-Length': '0'} 2022-09-03T07:37:59.553 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b06a7a4e7b604856-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.555 DEBUG:boto:path=/avysmw-76/obj33 2022-09-03T07:37:59.555 DEBUG:boto:auth_path=/avysmw-76/obj33 2022-09-03T07:37:59.556 DEBUG:boto:Method: GET 2022-09-03T07:37:59.557 DEBUG:boto:Path: /avysmw-76/obj33 2022-09-03T07:37:59.557 DEBUG:boto:Data: 2022-09-03T07:37:59.558 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.560 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.561 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.561 DEBUG:boto:Params: {} 2022-09-03T07:37:59.562 DEBUG:boto:Token: None 2022-09-03T07:37:59.563 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj33 2022-09-03T07:37:59.563 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/rdZh2EM3zzlcCd+39LeTrpyrs0= 2022-09-03T07:37:59.564 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/rdZh2EM3zzlcCd+39LeTrpyrs0=', 'Content-Length': '0'} 2022-09-03T07:37:59.567 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000006b197859fdf48e1d-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.610 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.611 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.612 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:59.612 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:59.613 DEBUG:boto:Method: GET 2022-09-03T07:37:59.614 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:59.614 DEBUG:boto:Data: 2022-09-03T07:37:59.615 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.615 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.616 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.616 DEBUG:boto:Params: {} 2022-09-03T07:37:59.617 DEBUG:boto:Token: None 2022-09-03T07:37:59.618 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.619 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.620 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.623 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000000a767046a1b0d442-0063130457-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.624 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.625 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.625 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.626 DEBUG:boto:path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:59.626 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj33 2022-09-03T07:37:59.627 DEBUG:boto:Method: GET 2022-09-03T07:37:59.627 DEBUG:boto:Path: /avysmw-76/?prefix=obj33 2022-09-03T07:37:59.628 DEBUG:boto:Data: 2022-09-03T07:37:59.629 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.629 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.630 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.631 DEBUG:boto:Params: {} 2022-09-03T07:37:59.631 DEBUG:boto:Token: None 2022-09-03T07:37:59.632 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.633 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.633 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.636 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx0000027b24df3b76ecee6-0063130457-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.637 DEBUG:boto:b'avysmw-76obj331000falseobj332022-09-03T07:33:12.463Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.638 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:37:59.639 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.639 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.640 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.641 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.641 DEBUG:boto:Method: GET 2022-09-03T07:37:59.642 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.642 DEBUG:boto:Data: 2022-09-03T07:37:59.643 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.643 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.644 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.645 DEBUG:boto:Params: {} 2022-09-03T07:37:59.646 DEBUG:boto:Token: None 2022-09-03T07:37:59.647 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.648 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g4s10rcZy7UoKlKST2R3ZTdocjg= 2022-09-03T07:37:59.649 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g4s10rcZy7UoKlKST2R3ZTdocjg=', 'Content-Length': '0'} 2022-09-03T07:37:59.653 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a75e890cc177bbfa-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.698 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.699 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.699 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.700 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.701 DEBUG:boto:Method: GET 2022-09-03T07:37:59.701 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.702 DEBUG:boto:Data: 2022-09-03T07:37:59.702 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.703 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.704 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.704 DEBUG:boto:Params: {} 2022-09-03T07:37:59.705 DEBUG:boto:Token: None 2022-09-03T07:37:59.705 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.706 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g4s10rcZy7UoKlKST2R3ZTdocjg= 2022-09-03T07:37:59.707 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g4s10rcZy7UoKlKST2R3ZTdocjg=', 'Content-Length': '0'} 2022-09-03T07:37:59.709 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000001ed2e8bbd60fdb9c-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.710 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.711 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.712 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.712 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.713 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.713 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.714 DEBUG:boto:Data: 2022-09-03T07:37:59.715 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.715 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.716 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.716 DEBUG:boto:Params: {} 2022-09-03T07:37:59.717 DEBUG:boto:Token: None 2022-09-03T07:37:59.718 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.719 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PtUEgl77imEBV2sdUmws8gRoqSo= 2022-09-03T07:37:59.720 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PtUEgl77imEBV2sdUmws8gRoqSo=', 'Content-Length': '0'} 2022-09-03T07:37:59.722 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000c4326d170469a762-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.723 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.723 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.724 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.724 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.725 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.726 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.727 DEBUG:boto:Data: 2022-09-03T07:37:59.728 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.728 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.729 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.730 DEBUG:boto:Params: {} 2022-09-03T07:37:59.731 DEBUG:boto:Token: None 2022-09-03T07:37:59.732 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.733 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PtUEgl77imEBV2sdUmws8gRoqSo= 2022-09-03T07:37:59.734 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PtUEgl77imEBV2sdUmws8gRoqSo=', 'Content-Length': '0'} 2022-09-03T07:37:59.737 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000732a6f8f815c6c5a-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.738 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:37:59.747 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.748 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.749 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.750 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.751 DEBUG:boto:Method: GET 2022-09-03T07:37:59.752 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.753 DEBUG:boto:Data: 2022-09-03T07:37:59.754 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.755 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.756 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.757 DEBUG:boto:Params: {} 2022-09-03T07:37:59.758 DEBUG:boto:Token: None 2022-09-03T07:37:59.759 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.760 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g4s10rcZy7UoKlKST2R3ZTdocjg= 2022-09-03T07:37:59.761 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g4s10rcZy7UoKlKST2R3ZTdocjg=', 'Content-Length': '0'} 2022-09-03T07:37:59.764 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b53021cd0138734c-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.765 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.766 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.767 DEBUG:boto:path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.768 DEBUG:boto:auth_path=/avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.769 DEBUG:boto:Method: GET 2022-09-03T07:37:59.770 DEBUG:boto:Path: /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.771 DEBUG:boto:Data: 2022-09-03T07:37:59.772 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.773 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.773 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.774 DEBUG:boto:Params: {} 2022-09-03T07:37:59.774 DEBUG:boto:Token: None 2022-09-03T07:37:59.775 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44?versionId=null 2022-09-03T07:37:59.776 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:g4s10rcZy7UoKlKST2R3ZTdocjg= 2022-09-03T07:37:59.776 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:g4s10rcZy7UoKlKST2R3ZTdocjg=', 'Content-Length': '0'} 2022-09-03T07:37:59.779 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000b4f7518b5e3abba3-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.779 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.780 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.781 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:59.781 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:59.782 DEBUG:boto:Method: GET 2022-09-03T07:37:59.782 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:59.783 DEBUG:boto:Data: 2022-09-03T07:37:59.783 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.784 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.784 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.785 DEBUG:boto:Params: {} 2022-09-03T07:37:59.785 DEBUG:boto:Token: None 2022-09-03T07:37:59.786 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.787 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.787 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.790 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000009a33486d50e561dd-0063130457-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.791 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.792 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.792 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.793 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:59.793 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:59.794 DEBUG:boto:Method: GET 2022-09-03T07:37:59.795 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:59.795 DEBUG:boto:Data: 2022-09-03T07:37:59.796 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.796 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.797 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.797 DEBUG:boto:Params: {} 2022-09-03T07:37:59.798 DEBUG:boto:Token: None 2022-09-03T07:37:59.799 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.799 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.800 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.803 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000227dc958d66523c1-0063130457-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.804 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.805 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.805 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.806 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.806 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:59.807 DEBUG:boto:Data: 2022-09-03T07:37:59.807 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.808 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.809 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.809 DEBUG:boto:Params: {} 2022-09-03T07:37:59.810 DEBUG:boto:Token: None 2022-09-03T07:37:59.810 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44 2022-09-03T07:37:59.811 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzm1n5EvD/2ajH+ijieaSg+kDtI= 2022-09-03T07:37:59.812 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzm1n5EvD/2ajH+ijieaSg+kDtI=', 'Content-Length': '0'} 2022-09-03T07:37:59.814 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000312c564d66fb50a3-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.815 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.816 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.817 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.818 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:59.819 DEBUG:boto:Data: 2022-09-03T07:37:59.820 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.821 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.822 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.823 DEBUG:boto:Params: {} 2022-09-03T07:37:59.824 DEBUG:boto:Token: None 2022-09-03T07:37:59.825 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44 2022-09-03T07:37:59.826 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:fzm1n5EvD/2ajH+ijieaSg+kDtI= 2022-09-03T07:37:59.827 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:fzm1n5EvD/2ajH+ijieaSg+kDtI=', 'Content-Length': '0'} 2022-09-03T07:37:59.830 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000b95dd66099d7f04-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.831 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2022-09-03T07:37:59.831 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.832 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.833 DEBUG:boto:Method: GET 2022-09-03T07:37:59.833 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:59.834 DEBUG:boto:Data: 2022-09-03T07:37:59.835 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.835 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.836 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.836 DEBUG:boto:Params: {} 2022-09-03T07:37:59.837 DEBUG:boto:Token: None 2022-09-03T07:37:59.838 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44 2022-09-03T07:37:59.839 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieAd342weWMjciPdB6oQrjVnocQ= 2022-09-03T07:37:59.840 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieAd342weWMjciPdB6oQrjVnocQ=', 'Content-Length': '0'} 2022-09-03T07:37:59.843 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000057e48447d7c07590-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.886 DEBUG:boto:path=/avysmw-76/obj44 2022-09-03T07:37:59.887 DEBUG:boto:auth_path=/avysmw-76/obj44 2022-09-03T07:37:59.888 DEBUG:boto:Method: GET 2022-09-03T07:37:59.888 DEBUG:boto:Path: /avysmw-76/obj44 2022-09-03T07:37:59.889 DEBUG:boto:Data: 2022-09-03T07:37:59.890 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.890 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.891 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.891 DEBUG:boto:Params: {} 2022-09-03T07:37:59.892 DEBUG:boto:Token: None 2022-09-03T07:37:59.893 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj44 2022-09-03T07:37:59.893 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ieAd342weWMjciPdB6oQrjVnocQ= 2022-09-03T07:37:59.894 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ieAd342weWMjciPdB6oQrjVnocQ=', 'Content-Length': '0'} 2022-09-03T07:37:59.896 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000ed9ef8b2093abf53-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.898 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.899 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.900 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:59.901 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:59.902 DEBUG:boto:Method: GET 2022-09-03T07:37:59.903 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:59.903 DEBUG:boto:Data: 2022-09-03T07:37:59.904 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.905 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.906 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.907 DEBUG:boto:Params: {} 2022-09-03T07:37:59.908 DEBUG:boto:Token: None 2022-09-03T07:37:59.908 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.909 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.910 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.913 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000c7bd6d6f2bc4cc74-0063130457-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.914 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.914 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:37:59.915 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:37:59.916 DEBUG:boto:path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:59.916 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj44 2022-09-03T07:37:59.917 DEBUG:boto:Method: GET 2022-09-03T07:37:59.917 DEBUG:boto:Path: /avysmw-76/?prefix=obj44 2022-09-03T07:37:59.918 DEBUG:boto:Data: 2022-09-03T07:37:59.919 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.920 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.920 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.921 DEBUG:boto:Params: {} 2022-09-03T07:37:59.921 DEBUG:boto:Token: None 2022-09-03T07:37:59.922 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/ 2022-09-03T07:37:59.923 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0= 2022-09-03T07:37:59.923 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZSgk0CWme1vOWXwUbp1Blyjrqw0=', 'Content-Length': '0'} 2022-09-03T07:37:59.926 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007fbd4d9ccb7191a7-0063130457-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.927 DEBUG:boto:b'avysmw-76obj441000falseobj442022-09-03T07:33:12.532Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:37:59.927 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:37:59.928 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:59.929 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:59.930 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.930 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.931 DEBUG:boto:Method: GET 2022-09-03T07:37:59.931 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.932 DEBUG:boto:Data: 2022-09-03T07:37:59.932 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.933 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.933 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.934 DEBUG:boto:Params: {} 2022-09-03T07:37:59.935 DEBUG:boto:Token: None 2022-09-03T07:37:59.935 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.936 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GOI0EVhK9Bz8EQ0BvFrYvoqrLjQ= 2022-09-03T07:37:59.937 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GOI0EVhK9Bz8EQ0BvFrYvoqrLjQ=', 'Content-Length': '0'} 2022-09-03T07:37:59.940 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000002f30228b4599de1-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.941 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:59.942 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:59.942 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.943 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.943 DEBUG:boto:Method: GET 2022-09-03T07:37:59.944 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.944 DEBUG:boto:Data: 2022-09-03T07:37:59.945 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.945 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.946 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.947 DEBUG:boto:Params: {} 2022-09-03T07:37:59.947 DEBUG:boto:Token: None 2022-09-03T07:37:59.948 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.949 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GOI0EVhK9Bz8EQ0BvFrYvoqrLjQ= 2022-09-03T07:37:59.949 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GOI0EVhK9Bz8EQ0BvFrYvoqrLjQ=', 'Content-Length': '0'} 2022-09-03T07:37:59.952 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000795b43c9c813bb1d-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.953 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:59.954 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:59.954 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.955 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.955 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.956 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.956 DEBUG:boto:Data: 2022-09-03T07:37:59.957 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.958 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.959 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.960 DEBUG:boto:Params: {} 2022-09-03T07:37:59.961 DEBUG:boto:Token: None 2022-09-03T07:37:59.962 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PyfPXaZzzFvAyg0j8tg/R2GVd8Q= 2022-09-03T07:37:59.964 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PyfPXaZzzFvAyg0j8tg/R2GVd8Q=', 'Content-Length': '0'} 2022-09-03T07:37:59.967 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000003209b145ec84c6db-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.968 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:59.969 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:59.970 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.970 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.971 DEBUG:boto:Method: HEAD 2022-09-03T07:37:59.971 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.972 DEBUG:boto:Data: 2022-09-03T07:37:59.972 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.973 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.973 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.974 DEBUG:boto:Params: {} 2022-09-03T07:37:59.974 DEBUG:boto:Token: None 2022-09-03T07:37:59.975 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.976 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PyfPXaZzzFvAyg0j8tg/R2GVd8Q= 2022-09-03T07:37:59.976 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PyfPXaZzzFvAyg0j8tg/R2GVd8Q=', 'Content-Length': '0'} 2022-09-03T07:37:59.978 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000a01d137ca96308e9-0063130457-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.979 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:37:59.979 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:59.980 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:59.981 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.981 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.982 DEBUG:boto:Method: GET 2022-09-03T07:37:59.983 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.983 DEBUG:boto:Data: 2022-09-03T07:37:59.984 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.985 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:37:59.985 DEBUG:boto:Port: 8001 2022-09-03T07:37:59.986 DEBUG:boto:Params: {} 2022-09-03T07:37:59.986 DEBUG:boto:Token: None 2022-09-03T07:37:59.987 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:37:59 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.988 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:GOI0EVhK9Bz8EQ0BvFrYvoqrLjQ= 2022-09-03T07:37:59.989 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:37:59 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:GOI0EVhK9Bz8EQ0BvFrYvoqrLjQ=', 'Content-Length': '0'} 2022-09-03T07:37:59.991 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx0000037dd9da88dbd88ce-0063130457-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:37:59 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:37:59.992 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:37:59.993 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:37:59.994 DEBUG:boto:path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.995 DEBUG:boto:auth_path=/avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.995 DEBUG:boto:Method: GET 2022-09-03T07:37:59.996 DEBUG:boto:Path: /avysmw-76/obj55?versionId=null 2022-09-03T07:37:59.997 DEBUG:boto:Data: 2022-09-03T07:37:59.997 DEBUG:boto:Headers: {} 2022-09-03T07:37:59.998 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:37:59.998 DEBUG:boto:Port: 8000 2022-09-03T07:37:59.999 DEBUG:boto:Params: {} 2022-09-03T07:37:59.999 DEBUG:boto:Token: None 2022-09-03T07:38:00.000 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:38:00 GMT /avysmw-76/obj55?versionId=null 2022-09-03T07:38:00.001 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8O8Q8jGlyjW3NkVeimEU/76jJfM= 2022-09-03T07:38:00.001 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8O8Q8jGlyjW3NkVeimEU/76jJfM=', 'Content-Length': '0'} 2022-09-03T07:38:00.003 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-amz-version-id', 'null'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000df076c8544b12459-0063130458-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:38:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:00.004 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:38:00.005 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:38:00.005 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:38:00.006 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:38:00.006 DEBUG:boto:Method: GET 2022-09-03T07:38:00.007 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:38:00.007 DEBUG:boto:Data: 2022-09-03T07:38:00.008 DEBUG:boto:Headers: {} 2022-09-03T07:38:00.009 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:38:00.009 DEBUG:boto:Port: 8001 2022-09-03T07:38:00.010 DEBUG:boto:Params: {} 2022-09-03T07:38:00.010 DEBUG:boto:Token: None 2022-09-03T07:38:00.011 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:38:00 GMT /avysmw-76/ 2022-09-03T07:38:00.011 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uu+PRdSs+tstzkREfGiBlgcSrs4= 2022-09-03T07:38:00.012 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uu+PRdSs+tstzkREfGiBlgcSrs4=', 'Content-Length': '0'} 2022-09-03T07:38:00.015 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000fc7ba0323eb9fb37-0063130458-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:38:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:00.058 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:38:00.060 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:38:00.060 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:38:00.061 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:38:00.062 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:38:00.063 DEBUG:boto:Method: GET 2022-09-03T07:38:00.064 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:38:00.065 DEBUG:boto:Data: 2022-09-03T07:38:00.066 DEBUG:boto:Headers: {} 2022-09-03T07:38:00.067 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:38:00.068 DEBUG:boto:Port: 8000 2022-09-03T07:38:00.069 DEBUG:boto:Params: {} 2022-09-03T07:38:00.070 DEBUG:boto:Token: None 2022-09-03T07:38:00.071 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:38:00 GMT /avysmw-76/ 2022-09-03T07:38:00.072 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uu+PRdSs+tstzkREfGiBlgcSrs4= 2022-09-03T07:38:00.073 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uu+PRdSs+tstzkREfGiBlgcSrs4=', 'Content-Length': '0'} 2022-09-03T07:38:00.077 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx000007da0c55b1578a80e-0063130458-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:38:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:00.078 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:38:00.079 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:38:00.080 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:38:00.081 DEBUG:boto:Method: HEAD 2022-09-03T07:38:00.082 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:38:00.083 DEBUG:boto:Data: 2022-09-03T07:38:00.084 DEBUG:boto:Headers: {} 2022-09-03T07:38:00.085 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:38:00.086 DEBUG:boto:Port: 8001 2022-09-03T07:38:00.088 DEBUG:boto:Params: {} 2022-09-03T07:38:00.089 DEBUG:boto:Token: None 2022-09-03T07:38:00.090 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:00 GMT /avysmw-76/obj55 2022-09-03T07:38:00.090 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Mf4nHppaqc+edJUm5sGp/GvmWs= 2022-09-03T07:38:00.091 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Mf4nHppaqc+edJUm5sGp/GvmWs=', 'Content-Length': '0'} 2022-09-03T07:38:00.093 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000bd7f549f6f1ee42b-0063130458-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:38:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:00.094 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:38:00.095 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:38:00.095 DEBUG:boto:Method: HEAD 2022-09-03T07:38:00.096 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:38:00.096 DEBUG:boto:Data: 2022-09-03T07:38:00.097 DEBUG:boto:Headers: {} 2022-09-03T07:38:00.097 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:38:00.098 DEBUG:boto:Port: 8000 2022-09-03T07:38:00.098 DEBUG:boto:Params: {} 2022-09-03T07:38:00.099 DEBUG:boto:Token: None 2022-09-03T07:38:00.100 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:00 GMT /avysmw-76/obj55 2022-09-03T07:38:00.100 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:/Mf4nHppaqc+edJUm5sGp/GvmWs= 2022-09-03T07:38:00.101 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:/Mf4nHppaqc+edJUm5sGp/GvmWs=', 'Content-Length': '0'} 2022-09-03T07:38:00.103 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx00000be0a0c97935c3c0f-0063130458-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:38:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:00.105 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj55 2022-09-03T07:38:00.106 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:38:00.107 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:38:00.108 DEBUG:boto:Method: GET 2022-09-03T07:38:00.109 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:38:00.109 DEBUG:boto:Data: 2022-09-03T07:38:00.110 DEBUG:boto:Headers: {} 2022-09-03T07:38:00.111 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:38:00.111 DEBUG:boto:Port: 8001 2022-09-03T07:38:00.112 DEBUG:boto:Params: {} 2022-09-03T07:38:00.113 DEBUG:boto:Token: None 2022-09-03T07:38:00.114 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:38:00 GMT /avysmw-76/obj55 2022-09-03T07:38:00.115 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Iecrdaxf9slKS9k2ECUiYXvOsb8= 2022-09-03T07:38:00.115 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Iecrdaxf9slKS9k2ECUiYXvOsb8=', 'Content-Length': '0'} 2022-09-03T07:38:00.117 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000000660d4f664f7b94c-0063130458-18f3-test-zone3'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:38:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:00.119 DEBUG:boto:path=/avysmw-76/obj55 2022-09-03T07:38:00.120 DEBUG:boto:auth_path=/avysmw-76/obj55 2022-09-03T07:38:00.120 DEBUG:boto:Method: GET 2022-09-03T07:38:00.121 DEBUG:boto:Path: /avysmw-76/obj55 2022-09-03T07:38:00.121 DEBUG:boto:Data: 2022-09-03T07:38:00.122 DEBUG:boto:Headers: {} 2022-09-03T07:38:00.122 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:38:00.123 DEBUG:boto:Port: 8000 2022-09-03T07:38:00.123 DEBUG:boto:Params: {} 2022-09-03T07:38:00.124 DEBUG:boto:Token: None 2022-09-03T07:38:00.126 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:38:00 GMT /avysmw-76/obj55 2022-09-03T07:38:00.127 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Iecrdaxf9slKS9k2ECUiYXvOsb8= 2022-09-03T07:38:00.127 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Iecrdaxf9slKS9k2ECUiYXvOsb8=', 'Content-Length': '0'} 2022-09-03T07:38:00.130 DEBUG:boto:Response headers: [('Content-Length', '3'), ('Accept-Ranges', 'bytes'), ('Last-Modified', 'Sat, 03 Sep 2022 07:33:12 GMT'), ('x-rgw-object-type', 'Normal'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('x-amz-request-id', 'tx000004f98418beb8eeeb5-0063130458-1b17-test-zone2'), ('Content-Type', 'application/octet-stream'), ('Date', 'Sat, 03 Sep 2022 07:38:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:00.130 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:38:00.131 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:38:00.132 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:38:00.132 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:38:00.133 DEBUG:boto:Method: GET 2022-09-03T07:38:00.133 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:38:00.134 DEBUG:boto:Data: 2022-09-03T07:38:00.134 DEBUG:boto:Headers: {} 2022-09-03T07:38:00.135 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8001 2022-09-03T07:38:00.135 DEBUG:boto:Port: 8001 2022-09-03T07:38:00.136 DEBUG:boto:Params: {} 2022-09-03T07:38:00.137 DEBUG:boto:Token: None 2022-09-03T07:38:00.138 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:38:00 GMT /avysmw-76/ 2022-09-03T07:38:00.139 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uu+PRdSs+tstzkREfGiBlgcSrs4= 2022-09-03T07:38:00.140 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uu+PRdSs+tstzkREfGiBlgcSrs4=', 'Content-Length': '0'} 2022-09-03T07:38:00.143 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000b45e78e97b313632-0063130458-18f3-test-zone3'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:38:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:00.144 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:38:00.145 DEBUG:boto:path=/avysmw-76/ 2022-09-03T07:38:00.145 DEBUG:boto:auth_path=/avysmw-76/ 2022-09-03T07:38:00.146 DEBUG:boto:path=/avysmw-76/?prefix=obj55 2022-09-03T07:38:00.146 DEBUG:boto:auth_path=/avysmw-76/?prefix=obj55 2022-09-03T07:38:00.147 DEBUG:boto:Method: GET 2022-09-03T07:38:00.147 DEBUG:boto:Path: /avysmw-76/?prefix=obj55 2022-09-03T07:38:00.148 DEBUG:boto:Data: 2022-09-03T07:38:00.148 DEBUG:boto:Headers: {} 2022-09-03T07:38:00.149 DEBUG:boto:Host: smithi087.front.sepia.ceph.com:8000 2022-09-03T07:38:00.149 DEBUG:boto:Port: 8000 2022-09-03T07:38:00.150 DEBUG:boto:Params: {} 2022-09-03T07:38:00.151 DEBUG:boto:Token: None 2022-09-03T07:38:00.151 DEBUG:boto:StringToSign: GET Sat, 03 Sep 2022 07:38:00 GMT /avysmw-76/ 2022-09-03T07:38:00.152 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:uu+PRdSs+tstzkREfGiBlgcSrs4= 2022-09-03T07:38:00.152 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:00 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:uu+PRdSs+tstzkREfGiBlgcSrs4=', 'Content-Length': '0'} 2022-09-03T07:38:00.155 DEBUG:boto:Response headers: [('Transfer-Encoding', 'chunked'), ('x-amz-request-id', 'tx00000f6afef94d6784f3e-0063130458-1b17-test-zone2'), ('Content-Type', 'application/xml'), ('Date', 'Sat, 03 Sep 2022 07:38:00 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:00.156 DEBUG:boto:b'avysmw-76obj551000falseobj552022-09-03T07:33:12.562Z"acbd18db4cc2f85cedef654fccc4a4d8"3STANDARDrgw-multisite-test-userMultisite Test UserNormal' 2022-09-03T07:38:00.157 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=avysmw-76 zones={test-zone3, test-zone2} 2022-09-03T07:38:00.158 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone2 2022-09-03T07:38:00.158 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.159 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.160 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:00.219 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:00.220 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:00.309 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:38:00.309 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.310 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000021391", 2022-09-03T07:38:00.310 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:12.536534Z" 2022-09-03T07:38:00.310 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.310 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.311 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000020399", 2022-09-03T07:38:00.311 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:30.981193Z" 2022-09-03T07:38:00.311 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.311 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.312 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000015639", 2022-09-03T07:38:00.312 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:30.983121Z" 2022-09-03T07:38:00.312 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.312 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.313 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011802", 2022-09-03T07:38:00.313 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:28:24.675559Z" 2022-09-03T07:38:00.313 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:00.314 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:38:00.315 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000021391', 1: '00000000000000000000:00000000000000020399', 2: '00000000000000000000:00000000000000015639', 3: '00000000000000000000:00000000000000011802'} 2022-09-03T07:38:00.315 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2022-09-03T07:38:00.316 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.317 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.317 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:00.376 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:00.376 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:00.473 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:38:00.474 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:38:00.474 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:38:00.475 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:38:00.475 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:38:00.475 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 17546439358371764503 2022-09-03T07:38:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:38:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:38:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:00.478 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:00.478 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000021391", 2022-09-03T07:38:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 67, 2022-09-03T07:38:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:12.536534Z" 2022-09-03T07:38:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:38:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000020399", 2022-09-03T07:38:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 9, 2022-09-03T07:38:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:30.981193Z" 2022-09-03T07:38:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:38:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000015639", 2022-09-03T07:38:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:30.983121Z" 2022-09-03T07:38:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:38:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000011802", 2022-09-03T07:38:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.675559Z" 2022-09-03T07:38:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:38:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:38:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 76, 2022-09-03T07:38:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 76 2022-09-03T07:38:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:38:00Z" 2022-09-03T07:38:00.492 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:38:00.494 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 17546439358371764503 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000021391", "next_step_marker": "", "total_entries": 67, "pos": 0, "timestamp": "2022-09-03T07:33:12.536534Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000020399", "next_step_marker": "", "total_entries": 9, "pos": 0, "timestamp": "2022-09-03T07:33:30.981193Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000015639", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:30.983121Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000011802", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.675559Z" } } ] }, "full_sync": { "total": 76, "complete": 76 }, "current_time": "2022-09-03T07:38:00Z" } 2022-09-03T07:38:00.495 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000021391', 'next_step_marker': '', 'total_entries': 67, 'pos': 0, 'timestamp': '2022-09-03T07:33:12.536534Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000020399', 'next_step_marker': '', 'total_entries': 9, 'pos': 0, 'timestamp': '2022-09-03T07:33:30.981193Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000015639', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:30.983121Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000011802', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.675559Z'}}] 2022-09-03T07:38:00.496 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000021391', 1: '00000000000000000000:00000000000000020399', 2: '00000000000000000000:00000000000000015639', 3: '00000000000000000000:00000000000000011802'} 2022-09-03T07:38:00.496 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000021391', 1: '00000000000000000000:00000000000000020399', 2: '00000000000000000000:00000000000000015639', 3: '00000000000000000000:00000000000000011802'} 2022-09-03T07:38:00.497 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2022-09-03T07:38:00.497 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone3 2022-09-03T07:38:00.498 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.499 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.499 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:00.553 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:00.553 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:00.610 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:38:00.611 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.611 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000021391", 2022-09-03T07:38:00.611 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:12.536534Z" 2022-09-03T07:38:00.612 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.612 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.612 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000020399", 2022-09-03T07:38:00.613 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:30.981193Z" 2022-09-03T07:38:00.613 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.614 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.614 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000015639", 2022-09-03T07:38:00.614 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:30.983121Z" 2022-09-03T07:38:00.615 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.615 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.615 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011802", 2022-09-03T07:38:00.615 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:28:24.675559Z" 2022-09-03T07:38:00.616 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:00.616 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:38:00.617 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000021391', 1: '00000000000000000000:00000000000000020399', 2: '00000000000000000000:00000000000000015639', 3: '00000000000000000000:00000000000000011802'} 2022-09-03T07:38:00.618 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2022-09-03T07:38:00.618 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.619 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.620 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:00.675 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:00.676 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:00.723 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:38:00.723 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:38:00.724 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:38:00.724 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:38:00.725 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:38:00.725 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 11727837677587591728 2022-09-03T07:38:00.726 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.726 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:38:00.727 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.727 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:38:00.727 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:00.728 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:00.728 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000021391", 2022-09-03T07:38:00.729 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:00.729 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:00.729 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:00.729 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:33:12.536534Z" 2022-09-03T07:38:00.730 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:00.730 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.730 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.730 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:38:00.731 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:00.731 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:00.731 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000020399", 2022-09-03T07:38:00.731 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:00.732 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:00.732 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:00.732 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:33:30.981193Z" 2022-09-03T07:38:00.733 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:00.733 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.733 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.734 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:38:00.734 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:00.734 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:00.735 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000015639", 2022-09-03T07:38:00.735 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:00.735 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:00.735 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:00.736 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:33:30.983121Z" 2022-09-03T07:38:00.736 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:00.736 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.736 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.737 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:38:00.737 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:00.737 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:00.737 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011802", 2022-09-03T07:38:00.738 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:00.738 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:00.738 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:00.739 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:28:24.675559Z" 2022-09-03T07:38:00.739 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:00.739 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:00.739 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:38:00.740 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.740 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:38:00.740 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T07:38:00.740 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T07:38:00.741 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.741 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:38:00Z" 2022-09-03T07:38:00.741 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:38:00.742 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 11727837677587591728 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000021391", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:12.536534Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000020399", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:30.981193Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000015639", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:30.983121Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000011802", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.675559Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T07:38:00Z" } 2022-09-03T07:38:00.743 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000021391', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:12.536534Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000020399', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:30.981193Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000015639', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:30.983121Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000011802', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.675559Z'}}] 2022-09-03T07:38:00.744 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000021391', 1: '00000000000000000000:00000000000000020399', 2: '00000000000000000000:00000000000000015639', 3: '00000000000000000000:00000000000000011802'} 2022-09-03T07:38:00.744 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000021391', 1: '00000000000000000000:00000000000000020399', 2: '00000000000000000000:00000000000000015639', 3: '00000000000000000000:00000000000000011802'} 2022-09-03T07:38:00.745 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2022-09-03T07:38:00.745 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone1 target=test-zone4 2022-09-03T07:38:00.746 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.746 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.747 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:00.797 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:00.798 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:00.874 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:38:00.875 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.875 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000021391", 2022-09-03T07:38:00.876 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:12.536534Z" 2022-09-03T07:38:00.877 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.877 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.878 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000020399", 2022-09-03T07:38:00.879 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:30.981193Z" 2022-09-03T07:38:00.880 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.880 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.881 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000015639", 2022-09-03T07:38:00.881 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:30.983121Z" 2022-09-03T07:38:00.882 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:00.882 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:00.883 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011802", 2022-09-03T07:38:00.883 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:28:24.675559Z" 2022-09-03T07:38:00.883 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:00.884 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:38:00.885 DEBUG:rgw_multi.tests:data markers for zone=test-zone1 markers={0: '00000000000000000000:00000000000000021391', 1: '00000000000000000000:00000000000000020399', 2: '00000000000000000000:00000000000000015639', 3: '00000000000000000000:00000000000000011802'} 2022-09-03T07:38:00.886 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2022-09-03T07:38:00.886 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.887 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:00.887 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:00.944 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:00.945 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:01.006 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:38:01.006 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:38:01.007 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:38:01.007 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:38:01.007 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:38:01.008 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 6281331864854677188 2022-09-03T07:38:01.008 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.008 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:38:01.008 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.009 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:38:01.009 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:01.009 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.010 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000021391", 2022-09-03T07:38:01.010 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:01.010 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:01.010 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:01.011 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:12.536534Z" 2022-09-03T07:38:01.011 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:01.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.011 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.012 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:38:01.012 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:01.012 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.013 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000020399", 2022-09-03T07:38:01.013 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:01.013 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:01.013 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:01.014 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:30.981193Z" 2022-09-03T07:38:01.014 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:01.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.015 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:38:01.015 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:01.015 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.015 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000015639", 2022-09-03T07:38:01.016 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:01.016 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:01.016 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:01.017 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:30.983121Z" 2022-09-03T07:38:01.017 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:01.017 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.017 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.018 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:38:01.018 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:01.018 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.018 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000011802", 2022-09-03T07:38:01.019 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:01.019 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:01.019 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:01.020 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:28:24.675559Z" 2022-09-03T07:38:01.020 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:01.020 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:01.021 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:38:01.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.021 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:38:01.021 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T07:38:01.022 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T07:38:01.022 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.022 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:38:00Z" 2022-09-03T07:38:01.023 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:38:01.024 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 6281331864854677188 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000021391", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:12.536534Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000020399", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:30.981193Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000015639", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:30.983121Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000011802", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:28:24.675559Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T07:38:00Z" } 2022-09-03T07:38:01.024 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000021391', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:12.536534Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000020399', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:30.981193Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000015639', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:30.983121Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000011802', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:28:24.675559Z'}}] 2022-09-03T07:38:01.025 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000021391', 1: '00000000000000000000:00000000000000020399', 2: '00000000000000000000:00000000000000015639', 3: '00000000000000000000:00000000000000011802'} 2022-09-03T07:38:01.025 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000021391', 1: '00000000000000000000:00000000000000020399', 2: '00000000000000000000:00000000000000015639', 3: '00000000000000000000:00000000000000011802'} 2022-09-03T07:38:01.026 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2022-09-03T07:38:01.026 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone1 2022-09-03T07:38:01.027 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.028 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.028 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:01.082 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:01.082 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:01.156 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T07:38:01.156 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.157 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000015164", 2022-09-03T07:38:01.157 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T07:35:24.901761Z" 2022-09-03T07:38:01.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.159 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000016153", 2022-09-03T07:38:01.159 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T07:35:24.908306Z" 2022-09-03T07:38:01.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.160 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000008994", 2022-09-03T07:38:01.161 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T07:35:24.911573Z" 2022-09-03T07:38:01.161 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.161 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.162 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006428", 2022-09-03T07:38:01.162 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T07:35:18.347491Z" 2022-09-03T07:38:01.163 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:01.163 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T07:38:01.164 DEBUG:rgw_multi.tests:data markers for zone=test-zone2 markers={0: '00000000000000000000:00000000000000015164', 1: '00000000000000000000:00000000000000016153', 2: '00000000000000000000:00000000000000008994', 3: '00000000000000000000:00000000000000006428'} 2022-09-03T07:38:01.165 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2022-09-03T07:38:01.165 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.166 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.167 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:01.221 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:01.222 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:01.294 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:38:01.295 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:38:01.295 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:38:01.295 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:38:01.296 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:38:01.296 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 8557007329946676474 2022-09-03T07:38:01.296 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.296 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:38:01.297 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.297 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:38:01.297 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.298 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.298 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000015164", 2022-09-03T07:38:01.299 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.300 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.300 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.300 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:35:24.901761Z" 2022-09-03T07:38:01.301 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.301 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.302 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.302 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:38:01.303 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.303 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.304 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000016153", 2022-09-03T07:38:01.305 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.305 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.306 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.306 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:35:24.908306Z" 2022-09-03T07:38:01.306 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.307 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.307 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.308 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:38:01.308 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.309 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.309 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000008994", 2022-09-03T07:38:01.309 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.310 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.310 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.310 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:35:24.911573Z" 2022-09-03T07:38:01.311 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.311 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.311 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.311 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:38:01.312 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.312 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.312 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006428", 2022-09-03T07:38:01.313 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.313 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.313 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.313 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:35:18.347491Z" 2022-09-03T07:38:01.314 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.314 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.314 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:38:01.314 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.315 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:38:01.315 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T07:38:01.315 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T07:38:01.315 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.316 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:38:01Z" 2022-09-03T07:38:01.316 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:38:01.316 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 8557007329946676474 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000015164", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:35:24.901761Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000016153", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:35:24.908306Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000008994", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:35:24.911573Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006428", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:35:18.347491Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T07:38:01Z" } 2022-09-03T07:38:01.317 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000015164', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:35:24.901761Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000016153', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:35:24.908306Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000008994', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:35:24.911573Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006428', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:35:18.347491Z'}}] 2022-09-03T07:38:01.318 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000015164', 1: '00000000000000000000:00000000000000016153', 2: '00000000000000000000:00000000000000008994', 3: '00000000000000000000:00000000000000006428'} 2022-09-03T07:38:01.318 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000015164', 1: '00000000000000000000:00000000000000016153', 2: '00000000000000000000:00000000000000008994', 3: '00000000000000000000:00000000000000006428'} 2022-09-03T07:38:01.319 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2022-09-03T07:38:01.319 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone3 2022-09-03T07:38:01.320 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.320 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.321 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:01.377 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:01.377 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:01.455 INFO:teuthology.orchestra.run.smithi087.stdout:[ 2022-09-03T07:38:01.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.456 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000015164", 2022-09-03T07:38:01.456 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T07:35:24.901761Z" 2022-09-03T07:38:01.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.457 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.457 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000016153", 2022-09-03T07:38:01.458 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T07:35:24.908306Z" 2022-09-03T07:38:01.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.458 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000008994", 2022-09-03T07:38:01.459 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T07:35:24.911573Z" 2022-09-03T07:38:01.459 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:01.459 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:01.460 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006428", 2022-09-03T07:38:01.460 INFO:teuthology.orchestra.run.smithi087.stdout: "last_update": "2022-09-03T07:35:18.347491Z" 2022-09-03T07:38:01.460 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:01.460 INFO:teuthology.orchestra.run.smithi087.stdout:] 2022-09-03T07:38:01.462 DEBUG:rgw_multi.tests:data markers for zone=test-zone2 markers={0: '00000000000000000000:00000000000000015164', 1: '00000000000000000000:00000000000000016153', 2: '00000000000000000000:00000000000000008994', 3: '00000000000000000000:00000000000000006428'} 2022-09-03T07:38:01.462 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2022-09-03T07:38:01.463 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.463 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.464 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:01.520 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:01.521 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:01.582 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:38:01.583 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:38:01.583 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:38:01.583 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:38:01.583 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:38:01.584 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 14782160398262406281 2022-09-03T07:38:01.584 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.585 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:38:01.585 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.585 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:38:01.586 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.586 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.587 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000015164", 2022-09-03T07:38:01.587 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.588 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.588 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.588 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:35:24.901761Z" 2022-09-03T07:38:01.589 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.589 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.589 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.589 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:38:01.590 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.590 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.590 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000016153", 2022-09-03T07:38:01.591 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.591 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.591 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.591 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:35:24.908306Z" 2022-09-03T07:38:01.592 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.592 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.592 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.592 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:38:01.593 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.593 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.593 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000008994", 2022-09-03T07:38:01.593 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.593 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.594 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.594 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:35:24.911573Z" 2022-09-03T07:38:01.594 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.595 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.595 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.595 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:38:01.596 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.596 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.596 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006428", 2022-09-03T07:38:01.597 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.597 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.597 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.598 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:35:18.347491Z" 2022-09-03T07:38:01.598 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.599 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.599 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:38:01.599 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.600 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:38:01.600 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T07:38:01.600 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T07:38:01.600 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.601 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:38:01Z" 2022-09-03T07:38:01.601 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:38:01.602 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 14782160398262406281 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000015164", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:35:24.901761Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000016153", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:35:24.908306Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000008994", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:35:24.911573Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006428", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:35:18.347491Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T07:38:01Z" } 2022-09-03T07:38:01.603 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000015164', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:35:24.901761Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000016153', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:35:24.908306Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000008994', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:35:24.911573Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006428', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:35:18.347491Z'}}] 2022-09-03T07:38:01.603 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000015164', 1: '00000000000000000000:00000000000000016153', 2: '00000000000000000000:00000000000000008994', 3: '00000000000000000000:00000000000000006428'} 2022-09-03T07:38:01.604 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000015164', 1: '00000000000000000000:00000000000000016153', 2: '00000000000000000000:00000000000000008994', 3: '00000000000000000000:00000000000000006428'} 2022-09-03T07:38:01.604 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2022-09-03T07:38:01.605 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone2 target=test-zone4 2022-09-03T07:38:01.605 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone1 2022-09-03T07:38:01.606 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.606 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.608 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:01.659 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:01.659 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:01.728 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:38:01.729 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.729 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011779", 2022-09-03T07:38:01.729 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:21.606679Z" 2022-09-03T07:38:01.730 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.730 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.730 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011923", 2022-09-03T07:38:01.731 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:38.372275Z" 2022-09-03T07:38:01.731 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.731 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.731 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006447", 2022-09-03T07:38:01.732 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:38.373934Z" 2022-09-03T07:38:01.732 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.732 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.732 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005300", 2022-09-03T07:38:01.733 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:27:01.673410Z" 2022-09-03T07:38:01.734 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.734 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:38:01.736 DEBUG:rgw_multi.tests:data markers for zone=test-zone3 markers={0: '00000000000000000000:00000000000000011779', 1: '00000000000000000000:00000000000000011923', 2: '00000000000000000000:00000000000000006447', 3: '00000000000000000000:00000000000000005300'} 2022-09-03T07:38:01.737 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2022-09-03T07:38:01.737 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.738 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.739 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:01.791 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:01.792 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:01.839 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:38:01.839 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:38:01.839 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:38:01.840 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:38:01.840 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:38:01.840 INFO:teuthology.orchestra.run.smithi062.stdout: "instance_id": 13955113675122108513 2022-09-03T07:38:01.841 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.841 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:38:01.842 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.842 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:38:01.842 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.843 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.844 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011779", 2022-09-03T07:38:01.844 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.845 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.845 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.846 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:33:21.606679Z" 2022-09-03T07:38:01.846 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.847 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.847 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.848 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:38:01.848 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.849 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.849 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011923", 2022-09-03T07:38:01.850 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.850 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.851 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.851 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:33:38.372275Z" 2022-09-03T07:38:01.852 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.852 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.853 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.854 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:38:01.855 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.855 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.856 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006447", 2022-09-03T07:38:01.856 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.856 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.857 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.857 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:33:38.373934Z" 2022-09-03T07:38:01.857 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.858 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.858 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.858 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:38:01.858 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:01.859 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "incremental-sync", 2022-09-03T07:38:01.859 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005300", 2022-09-03T07:38:01.859 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:01.860 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:01.860 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:01.860 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:27:01.673410Z" 2022-09-03T07:38:01.860 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.861 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:01.861 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:38:01.861 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.861 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:38:01.862 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 0, 2022-09-03T07:38:01.862 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 0 2022-09-03T07:38:01.862 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.863 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:38:01Z" 2022-09-03T07:38:01.863 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:38:01.864 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 13955113675122108513 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000011779", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:21.606679Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000011923", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:38.372275Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006447", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:38.373934Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005300", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:27:01.673410Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T07:38:01Z" } 2022-09-03T07:38:01.865 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000011779', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:21.606679Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000011923', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:38.372275Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006447', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:38.373934Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005300', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:27:01.673410Z'}}] 2022-09-03T07:38:01.866 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000011779', 1: '00000000000000000000:00000000000000011923', 2: '00000000000000000000:00000000000000006447', 3: '00000000000000000000:00000000000000005300'} 2022-09-03T07:38:01.866 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000011779', 1: '00000000000000000000:00000000000000011923', 2: '00000000000000000000:00000000000000006447', 3: '00000000000000000000:00000000000000005300'} 2022-09-03T07:38:01.867 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2022-09-03T07:38:01.867 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone2 2022-09-03T07:38:01.868 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.869 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:01.869 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:01.920 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:01.920 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:01.994 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:38:01.995 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.995 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011779", 2022-09-03T07:38:01.996 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:21.606679Z" 2022-09-03T07:38:01.996 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.996 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.997 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000011923", 2022-09-03T07:38:01.997 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:38.372275Z" 2022-09-03T07:38:01.998 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:01.998 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:01.998 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000006447", 2022-09-03T07:38:01.999 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:33:38.373934Z" 2022-09-03T07:38:02.000 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:02.000 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:02.000 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "00000000000000000000:00000000000000005300", 2022-09-03T07:38:02.001 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:27:01.673410Z" 2022-09-03T07:38:02.001 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:02.002 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:38:02.004 DEBUG:rgw_multi.tests:data markers for zone=test-zone3 markers={0: '00000000000000000000:00000000000000011779', 1: '00000000000000000000:00000000000000011923', 2: '00000000000000000000:00000000000000006447', 3: '00000000000000000000:00000000000000005300'} 2022-09-03T07:38:02.005 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2022-09-03T07:38:02.005 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:02.006 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:02.007 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:02.067 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:02.067 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:02.149 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:38:02.150 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:38:02.150 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:38:02.151 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:38:02.151 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:38:02.152 INFO:teuthology.orchestra.run.smithi087.stdout: "instance_id": 16273219177139511486 2022-09-03T07:38:02.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.152 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:38:02.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:02.153 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:38:02.153 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:02.153 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:02.153 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000011779", 2022-09-03T07:38:02.154 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:02.154 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:02.155 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:02.155 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:21.606679Z" 2022-09-03T07:38:02.155 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.156 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:02.156 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:38:02.156 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:02.156 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:02.157 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000011923", 2022-09-03T07:38:02.157 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:02.157 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:02.157 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:02.158 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:38.372275Z" 2022-09-03T07:38:02.158 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:02.159 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:38:02.159 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:02.159 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:02.159 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000006447", 2022-09-03T07:38:02.160 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:02.160 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:02.160 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:02.161 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:33:38.373934Z" 2022-09-03T07:38:02.161 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.161 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.161 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:02.162 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:38:02.162 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:02.162 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "incremental-sync", 2022-09-03T07:38:02.162 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "00000000000000000000:00000000000000005300", 2022-09-03T07:38:02.163 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:02.163 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:02.163 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:02.163 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:27:01.673410Z" 2022-09-03T07:38:02.164 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.164 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.164 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:38:02.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.165 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:38:02.165 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 0, 2022-09-03T07:38:02.165 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 0 2022-09-03T07:38:02.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.166 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:38:02Z" 2022-09-03T07:38:02.166 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:38:02.167 DEBUG:rgw_multi.tests:current data sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "instance_id": 16273219177139511486 }, "markers": [ { "key": 0, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000011779", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:21.606679Z" } }, { "key": 1, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000011923", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:38.372275Z" } }, { "key": 2, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000006447", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:33:38.373934Z" } }, { "key": 3, "val": { "status": "incremental-sync", "marker": "00000000000000000000:00000000000000005300", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:27:01.673410Z" } } ] }, "full_sync": { "total": 0, "complete": 0 }, "current_time": "2022-09-03T07:38:02Z" } 2022-09-03T07:38:02.168 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000011779', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:21.606679Z'}}, {'key': 1, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000011923', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:38.372275Z'}}, {'key': 2, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000006447', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:33:38.373934Z'}}, {'key': 3, 'val': {'status': 'incremental-sync', 'marker': '00000000000000000000:00000000000000005300', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:27:01.673410Z'}}] 2022-09-03T07:38:02.168 DEBUG:rgw_multi.tests:log_status={0: '00000000000000000000:00000000000000011779', 1: '00000000000000000000:00000000000000011923', 2: '00000000000000000000:00000000000000006447', 3: '00000000000000000000:00000000000000005300'} 2022-09-03T07:38:02.169 DEBUG:rgw_multi.tests:sync_status={0: '00000000000000000000:00000000000000011779', 1: '00000000000000000000:00000000000000011923', 2: '00000000000000000000:00000000000000006447', 3: '00000000000000000000:00000000000000005300'} 2022-09-03T07:38:02.169 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2022-09-03T07:38:02.170 DEBUG:rgw_multi.tests:data checkpoint: source=test-zone3 target=test-zone4 2022-09-03T07:38:02.171 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_full_sync_after_data_sync_init ... ok 2022-09-03T07:38:02.173 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:02.174 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:02.174 DEBUG:boto:Method: PUT 2022-09-03T07:38:02.175 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:02.175 DEBUG:boto:Data: 2022-09-03T07:38:02.176 DEBUG:boto:Headers: {} 2022-09-03T07:38:02.176 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:02.177 DEBUG:boto:Port: 8000 2022-09-03T07:38:02.177 DEBUG:boto:Params: {} 2022-09-03T07:38:02.178 DEBUG:boto:Token: None 2022-09-03T07:38:02.179 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 07:38:02 GMT /avysmw-77/ 2022-09-03T07:38:02.179 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:iBNT4LQsQfIqRUWToYExAIgT5vA= 2022-09-03T07:38:02.180 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:02 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:iBNT4LQsQfIqRUWToYExAIgT5vA=', 'Content-Length': '0'} 2022-09-03T07:38:02.208 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx00000dd27deae547d1d47-006313045a-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:02 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:02.208 DEBUG:rgw_multi.tests:created bucket=avysmw-77 2022-09-03T07:38:02.209 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:02.210 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:02.210 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:02.261 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:02.262 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:02.293 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T07:38:02.309 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:38:02.309 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:02.309 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:38:02.310 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:31:57.780536Z" 2022-09-03T07:38:02.310 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:02.310 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:02.310 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190682.205062_515.1", 2022-09-03T07:38:02.311 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:38:02.205062Z" 2022-09-03T07:38:02.311 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:02.311 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:02.312 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:38:02.312 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:38:02.312 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:02.312 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:02.313 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:38:02.313 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:38:02.313 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:02.313 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:38:02.315 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662190317.780536_375.1', 1: '1_1662190682.205062_515.1', 2: '', 3: ''} 2022-09-03T07:38:02.315 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T07:38:02.316 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:02.316 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:02.317 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:02.375 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:02.376 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:02.435 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:02.434+0000 7fb2a2f77600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:38:02.436 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:02.435+0000 7fb2a2f77600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:38:02.467 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:38:02.467 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:38:02.467 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:38:02.468 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:38:02.468 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:38:02.468 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:38:02.468 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:38:02.469 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.469 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:38:02.469 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:02.469 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:38:02.470 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:02.470 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:02.470 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:38:02.471 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:02.471 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:38:02.471 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:02.471 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:38:02.472 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:38:02.472 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.472 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:02.473 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:38:02.473 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:02.473 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:02.473 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190682.192834_512.1", 2022-09-03T07:38:02.474 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:02.474 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:02.475 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:02.475 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:02.192834Z", 2022-09-03T07:38:02.475 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:38:02.475 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.476 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:02.476 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:38:02.477 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:02.477 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:02.477 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:38:02.477 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:02.478 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:02.478 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:02.478 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:38:02.478 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:38:02.479 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:02.479 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:38:02.480 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:02.480 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:02.480 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:38:02.480 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:02.481 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:02.481 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:02.481 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:38:02.481 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:38:02.482 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.482 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:02.482 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:38:02.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.483 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:38:02.483 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:38:02.483 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:38:02.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:02.484 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:38:02Z" 2022-09-03T07:38:02.484 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:38:02.486 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190682.192834_512.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:02.192834Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:38:02Z" } 2022-09-03T07:38:02.486 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190682.192834_512.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:02.192834Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:38:02.487 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190682.205062_515.1', 2: '', 3: ''} 2022-09-03T07:38:02.487 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190682.192834_512.1', 2: '', 3: ''} 2022-09-03T07:38:02.488 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190682.205062_515.1 target=1_1662190682.192834_512.1 2022-09-03T07:38:07.490 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:07.491 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:07.491 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:07.542 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:07.542 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:07.573 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:07.571+0000 7f063939c600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:38:07.574 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:07.572+0000 7f063939c600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:38:07.611 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:38:07.612 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:38:07.612 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:38:07.612 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:38:07.613 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:38:07.613 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:38:07.613 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:38:07.614 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.614 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:38:07.614 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:07.614 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:38:07.615 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:07.615 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:07.615 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:38:07.616 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:07.616 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:38:07.616 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:07.616 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:38:07.617 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:38:07.617 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:07.618 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:38:07.618 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:07.618 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:07.619 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190682.205062_515.1", 2022-09-03T07:38:07.619 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:07.619 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:07.619 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:07.620 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:02.205062Z", 2022-09-03T07:38:07.620 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:38:07.620 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.621 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:07.621 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:38:07.621 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:07.622 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:07.622 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:38:07.622 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:07.622 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:07.623 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:07.623 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:38:07.623 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:38:07.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:07.624 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:38:07.624 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:07.625 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:07.625 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:38:07.625 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:07.625 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:07.626 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:07.626 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:38:07.626 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:38:07.627 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.627 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.627 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:38:07.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.628 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:38:07.628 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:38:07.628 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:38:07.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.629 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:38:07Z" 2022-09-03T07:38:07.629 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:38:07.631 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190682.205062_515.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:02.205062Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:38:07Z" } 2022-09-03T07:38:07.631 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190682.205062_515.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:02.205062Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:38:07.632 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190682.205062_515.1', 2: '', 3: ''} 2022-09-03T07:38:07.632 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190682.205062_515.1', 2: '', 3: ''} 2022-09-03T07:38:07.633 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2022-09-03T07:38:07.633 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2022-09-03T07:38:07.634 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:07.635 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:07.635 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:07.695 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:07.695 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:07.745 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:38:07.744+0000 7f29222ee600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:38:07.747 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T07:38:07.745+0000 7f29222ee600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:38:07.765 INFO:teuthology.orchestra.run.smithi062.stdout:{ 2022-09-03T07:38:07.766 INFO:teuthology.orchestra.run.smithi062.stdout: "sync_status": { 2022-09-03T07:38:07.766 INFO:teuthology.orchestra.run.smithi062.stdout: "info": { 2022-09-03T07:38:07.766 INFO:teuthology.orchestra.run.smithi062.stdout: "status": "sync", 2022-09-03T07:38:07.767 INFO:teuthology.orchestra.run.smithi062.stdout: "num_shards": 4, 2022-09-03T07:38:07.767 INFO:teuthology.orchestra.run.smithi062.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:38:07.767 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:38:07.767 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:07.768 INFO:teuthology.orchestra.run.smithi062.stdout: "markers": [ 2022-09-03T07:38:07.768 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:07.768 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 0, 2022-09-03T07:38:07.769 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:07.769 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:38:07.770 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:38:07.770 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:07.770 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 1, 2022-09-03T07:38:07.770 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:07.772 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:38:07.772 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:38:07.772 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:07.773 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:07.773 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:07.773 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 1, 2022-09-03T07:38:07.773 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:07.774 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:38:07.774 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190682.205062_515.1", 2022-09-03T07:38:07.774 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:07.775 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:07.775 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:07.775 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "2022-09-03T07:38:02.205062Z", 2022-09-03T07:38:07.775 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 4 2022-09-03T07:38:07.776 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:07.776 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:07.776 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:07.776 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 2, 2022-09-03T07:38:07.777 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:07.777 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:38:07.777 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:38:07.777 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:07.778 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:07.778 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:07.778 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:38:07.778 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:38:07.779 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:07.779 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:07.780 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:38:07.781 INFO:teuthology.orchestra.run.smithi062.stdout: "key": 3, 2022-09-03T07:38:07.781 INFO:teuthology.orchestra.run.smithi062.stdout: "val": { 2022-09-03T07:38:07.782 INFO:teuthology.orchestra.run.smithi062.stdout: "state": 1, 2022-09-03T07:38:07.782 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:38:07.783 INFO:teuthology.orchestra.run.smithi062.stdout: "next_step_marker": "", 2022-09-03T07:38:07.783 INFO:teuthology.orchestra.run.smithi062.stdout: "total_entries": 0, 2022-09-03T07:38:07.784 INFO:teuthology.orchestra.run.smithi062.stdout: "pos": 0, 2022-09-03T07:38:07.784 INFO:teuthology.orchestra.run.smithi062.stdout: "timestamp": "0.000000", 2022-09-03T07:38:07.785 INFO:teuthology.orchestra.run.smithi062.stdout: "realm_epoch": 2 2022-09-03T07:38:07.785 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:07.785 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:38:07.786 INFO:teuthology.orchestra.run.smithi062.stdout: ] 2022-09-03T07:38:07.786 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:07.786 INFO:teuthology.orchestra.run.smithi062.stdout: "full_sync": { 2022-09-03T07:38:07.787 INFO:teuthology.orchestra.run.smithi062.stdout: "total": 1, 2022-09-03T07:38:07.787 INFO:teuthology.orchestra.run.smithi062.stdout: "complete": 1 2022-09-03T07:38:07.787 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:38:07.787 INFO:teuthology.orchestra.run.smithi062.stdout: "current_time": "2022-09-03T07:38:07Z" 2022-09-03T07:38:07.788 INFO:teuthology.orchestra.run.smithi062.stdout:} 2022-09-03T07:38:07.789 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190682.205062_515.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:02.205062Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:38:07Z" } 2022-09-03T07:38:07.790 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190682.205062_515.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:02.205062Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:38:07.791 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190682.205062_515.1', 2: '', 3: ''} 2022-09-03T07:38:07.791 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190682.205062_515.1', 2: '', 3: ''} 2022-09-03T07:38:07.792 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2022-09-03T07:38:07.792 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2022-09-03T07:38:07.793 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:07.793 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:38:07.794 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:38:07.852 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:07.852 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:07.892 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:07.889+0000 7fe5360f7600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:38:07.900 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:07.890+0000 7fe5360f7600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:38:07.904 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:38:07.905 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:38:07.905 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:38:07.905 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:38:07.906 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:38:07.906 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:38:07.907 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:38:07.907 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.907 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:38:07.907 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:07.908 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:38:07.908 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:07.908 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:07.908 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:38:07.909 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:07.909 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:38:07.909 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:07.909 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:38:07.910 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:38:07.910 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.911 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.911 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:07.912 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:38:07.912 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:07.914 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:07.914 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190682.205062_515.1", 2022-09-03T07:38:07.915 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:07.915 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:07.916 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:07.916 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:02.205062Z", 2022-09-03T07:38:07.917 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:38:07.917 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.917 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.918 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:07.918 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:38:07.919 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:07.919 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:07.920 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:38:07.920 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:07.921 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:07.921 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:07.923 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:38:07.923 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:38:07.924 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.924 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.925 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:38:07.929 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:38:07.930 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:38:07.931 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:38:07.932 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:38:07.932 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:38:07.933 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:38:07.933 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:38:07.934 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:38:07.934 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:38:07.935 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.935 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:38:07.936 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:38:07.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.937 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:38:07.937 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:38:07.938 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:38:07.938 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:38:07.938 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:38:07Z" 2022-09-03T07:38:07.939 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:38:07.942 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190682.205062_515.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:02.205062Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:38:07Z" } 2022-09-03T07:38:07.943 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190682.205062_515.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:02.205062Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:38:07.944 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190682.205062_515.1', 2: '', 3: ''} 2022-09-03T07:38:07.945 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190682.205062_515.1', 2: '', 3: ''} 2022-09-03T07:38:07.946 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2022-09-03T07:38:07.947 DEBUG:tasks.rgw.c2.client.0:waiting for process to exit 2022-09-03T07:38:07.948 INFO:teuthology.orchestra.run:waiting for 300 2022-09-03T07:38:07.950 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T07:38:07.947+0000 7fe159c64700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm (PID: 158971) UID: 0 2022-09-03T07:38:07.952 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T07:38:07.947+0000 7fe15decf600 -1 shutting down 2022-09-03T07:38:11.326 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~0s 2022-09-03T07:38:14.051 INFO:tasks.rgw.c2.client.0:Stopped 2022-09-03T07:38:14.052 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.052 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.053 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.055 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.057 DEBUG:boto:Data: 2022-09-03T07:38:14.058 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.059 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.060 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.060 DEBUG:boto:Params: {} 2022-09-03T07:38:14.062 DEBUG:boto:Token: None 2022-09-03T07:38:14.063 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.064 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.065 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.068 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '0'), ('X-RGW-Bytes-Used', '0'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000032b115eac5684e39-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.070 DEBUG:boto:path=/avysmw-77/obj11 2022-09-03T07:38:14.071 DEBUG:boto:auth_path=/avysmw-77/obj11 2022-09-03T07:38:14.072 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.073 DEBUG:boto:Path: /avysmw-77/obj11 2022-09-03T07:38:14.074 DEBUG:boto:Data: 2022-09-03T07:38:14.075 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.076 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.076 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.077 DEBUG:boto:Params: {} 2022-09-03T07:38:14.078 DEBUG:boto:Token: None 2022-09-03T07:38:14.079 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj11 2022-09-03T07:38:14.080 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:QSJQb02A3zOvnImQvF1WHdz5wA8= 2022-09-03T07:38:14.080 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:QSJQb02A3zOvnImQvF1WHdz5wA8='} 2022-09-03T07:38:14.090 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000791be134464ebb25-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.091 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.091 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.092 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.093 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.094 DEBUG:boto:Data: 2022-09-03T07:38:14.094 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.095 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.095 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.096 DEBUG:boto:Params: {} 2022-09-03T07:38:14.096 DEBUG:boto:Token: None 2022-09-03T07:38:14.097 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.097 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.098 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.101 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '1'), ('X-RGW-Bytes-Used', '3'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004e13034c4c30704b-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.102 DEBUG:boto:path=/avysmw-77/obj22 2022-09-03T07:38:14.102 DEBUG:boto:auth_path=/avysmw-77/obj22 2022-09-03T07:38:14.103 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.104 DEBUG:boto:Path: /avysmw-77/obj22 2022-09-03T07:38:14.105 DEBUG:boto:Data: 2022-09-03T07:38:14.106 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.107 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.108 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.108 DEBUG:boto:Params: {} 2022-09-03T07:38:14.110 DEBUG:boto:Token: None 2022-09-03T07:38:14.111 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj22 2022-09-03T07:38:14.112 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:LSNgE1t6AVVTgLlYAkvi/fuqkDk= 2022-09-03T07:38:14.113 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:LSNgE1t6AVVTgLlYAkvi/fuqkDk='} 2022-09-03T07:38:14.164 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000001bdadb250b67a5ea-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.165 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.166 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.166 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.167 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.167 DEBUG:boto:Data: 2022-09-03T07:38:14.168 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.168 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.169 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.170 DEBUG:boto:Params: {} 2022-09-03T07:38:14.171 DEBUG:boto:Token: None 2022-09-03T07:38:14.171 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.172 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.172 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.176 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '2'), ('X-RGW-Bytes-Used', '6'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000081b9abf882ae2ff0-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.177 DEBUG:boto:path=/avysmw-77/obj33 2022-09-03T07:38:14.178 DEBUG:boto:auth_path=/avysmw-77/obj33 2022-09-03T07:38:14.178 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.179 DEBUG:boto:Path: /avysmw-77/obj33 2022-09-03T07:38:14.180 DEBUG:boto:Data: 2022-09-03T07:38:14.180 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.181 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.181 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.182 DEBUG:boto:Params: {} 2022-09-03T07:38:14.182 DEBUG:boto:Token: None 2022-09-03T07:38:14.183 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj33 2022-09-03T07:38:14.184 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+W/42ZNLk7JRUaIV3AucaMJiSI8= 2022-09-03T07:38:14.184 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+W/42ZNLk7JRUaIV3AucaMJiSI8='} 2022-09-03T07:38:14.192 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000007cca668124f7ae2e-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.193 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.193 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.194 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.194 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.195 DEBUG:boto:Data: 2022-09-03T07:38:14.195 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.196 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.196 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.197 DEBUG:boto:Params: {} 2022-09-03T07:38:14.197 DEBUG:boto:Token: None 2022-09-03T07:38:14.198 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.199 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.199 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.202 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '3'), ('X-RGW-Bytes-Used', '9'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000b52c55818e94c2f5-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.203 DEBUG:boto:path=/avysmw-77/obj44 2022-09-03T07:38:14.204 DEBUG:boto:auth_path=/avysmw-77/obj44 2022-09-03T07:38:14.204 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.205 DEBUG:boto:Path: /avysmw-77/obj44 2022-09-03T07:38:14.205 DEBUG:boto:Data: 2022-09-03T07:38:14.206 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.206 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.207 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.208 DEBUG:boto:Params: {} 2022-09-03T07:38:14.208 DEBUG:boto:Token: None 2022-09-03T07:38:14.209 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj44 2022-09-03T07:38:14.209 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:tOJhwRfJAmE+elBXeOCpz++u9J8= 2022-09-03T07:38:14.210 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:tOJhwRfJAmE+elBXeOCpz++u9J8='} 2022-09-03T07:38:14.254 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000024a2c579e21aea7e-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.255 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.256 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.257 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.257 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.258 DEBUG:boto:Data: 2022-09-03T07:38:14.258 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.259 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.260 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.260 DEBUG:boto:Params: {} 2022-09-03T07:38:14.262 DEBUG:boto:Token: None 2022-09-03T07:38:14.263 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.265 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.266 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.269 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '4'), ('X-RGW-Bytes-Used', '12'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000041e8360160c3697c-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.271 DEBUG:boto:path=/avysmw-77/obj55 2022-09-03T07:38:14.271 DEBUG:boto:auth_path=/avysmw-77/obj55 2022-09-03T07:38:14.272 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.272 DEBUG:boto:Path: /avysmw-77/obj55 2022-09-03T07:38:14.273 DEBUG:boto:Data: 2022-09-03T07:38:14.273 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.274 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.275 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.276 DEBUG:boto:Params: {} 2022-09-03T07:38:14.277 DEBUG:boto:Token: None 2022-09-03T07:38:14.278 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj55 2022-09-03T07:38:14.278 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:vL8t/084TsUpSZ6aB8pGmILnbcM= 2022-09-03T07:38:14.279 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:vL8t/084TsUpSZ6aB8pGmILnbcM='} 2022-09-03T07:38:14.285 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000e0ed5db67b94f175-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.287 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '17', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:14.287 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '17', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:14.288 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-77 --num-shards 17 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:38:14.352 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:14.352 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:14.412 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:38:14.412 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-77 2022-09-03T07:38:14.417 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 5 2022-09-03T07:38:14.482 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.483 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.483 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.485 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.486 DEBUG:boto:Data: 2022-09-03T07:38:14.487 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.487 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.490 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.490 DEBUG:boto:Params: {} 2022-09-03T07:38:14.491 DEBUG:boto:Token: None 2022-09-03T07:38:14.492 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.493 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.493 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.497 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '5'), ('X-RGW-Bytes-Used', '15'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004583e2be152812d2-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.498 DEBUG:boto:path=/avysmw-77/obj17 2022-09-03T07:38:14.499 DEBUG:boto:auth_path=/avysmw-77/obj17 2022-09-03T07:38:14.499 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.500 DEBUG:boto:Path: /avysmw-77/obj17 2022-09-03T07:38:14.501 DEBUG:boto:Data: 2022-09-03T07:38:14.501 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.502 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.502 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.503 DEBUG:boto:Params: {} 2022-09-03T07:38:14.503 DEBUG:boto:Token: None 2022-09-03T07:38:14.504 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj17 2022-09-03T07:38:14.505 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:+g52aJp+JDed2FQCkOX4h3Fnm9s= 2022-09-03T07:38:14.505 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:+g52aJp+JDed2FQCkOX4h3Fnm9s='} 2022-09-03T07:38:14.514 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000c6b3911e532da8f9-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.515 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.516 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.516 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.517 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.517 DEBUG:boto:Data: 2022-09-03T07:38:14.518 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.518 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.519 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.519 DEBUG:boto:Params: {} 2022-09-03T07:38:14.520 DEBUG:boto:Token: None 2022-09-03T07:38:14.520 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.521 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.522 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.526 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '6'), ('X-RGW-Bytes-Used', '18'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000044cd2bf2e0cd18c3-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.527 DEBUG:boto:path=/avysmw-77/obj34 2022-09-03T07:38:14.528 DEBUG:boto:auth_path=/avysmw-77/obj34 2022-09-03T07:38:14.529 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.530 DEBUG:boto:Path: /avysmw-77/obj34 2022-09-03T07:38:14.531 DEBUG:boto:Data: 2022-09-03T07:38:14.532 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.533 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.533 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.534 DEBUG:boto:Params: {} 2022-09-03T07:38:14.535 DEBUG:boto:Token: None 2022-09-03T07:38:14.537 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj34 2022-09-03T07:38:14.538 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gnHH03vQFJ3N84iE+Xkp3udeNfQ= 2022-09-03T07:38:14.539 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gnHH03vQFJ3N84iE+Xkp3udeNfQ='} 2022-09-03T07:38:14.548 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000007ba1f80e1ec2e6c0-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.549 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.550 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.551 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.552 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.554 DEBUG:boto:Data: 2022-09-03T07:38:14.555 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.555 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.556 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.557 DEBUG:boto:Params: {} 2022-09-03T07:38:14.558 DEBUG:boto:Token: None 2022-09-03T07:38:14.559 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.565 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '7'), ('X-RGW-Bytes-Used', '21'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000004e7521349ddb92b-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.566 DEBUG:boto:path=/avysmw-77/obj51 2022-09-03T07:38:14.567 DEBUG:boto:auth_path=/avysmw-77/obj51 2022-09-03T07:38:14.567 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.568 DEBUG:boto:Path: /avysmw-77/obj51 2022-09-03T07:38:14.569 DEBUG:boto:Data: 2022-09-03T07:38:14.569 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.570 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.570 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.571 DEBUG:boto:Params: {} 2022-09-03T07:38:14.571 DEBUG:boto:Token: None 2022-09-03T07:38:14.572 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj51 2022-09-03T07:38:14.572 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:k9WbJXSFGZnQVFzzbLTlmpEiqqI= 2022-09-03T07:38:14.573 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:k9WbJXSFGZnQVFzzbLTlmpEiqqI='} 2022-09-03T07:38:14.618 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000852723718475a76e-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.619 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.620 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.620 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.621 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.621 DEBUG:boto:Data: 2022-09-03T07:38:14.622 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.622 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.623 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.623 DEBUG:boto:Params: {} 2022-09-03T07:38:14.624 DEBUG:boto:Token: None 2022-09-03T07:38:14.625 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.625 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.626 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.629 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '8'), ('X-RGW-Bytes-Used', '24'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000cc9b7804b35e6129-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.629 DEBUG:boto:path=/avysmw-77/obj68 2022-09-03T07:38:14.630 DEBUG:boto:auth_path=/avysmw-77/obj68 2022-09-03T07:38:14.630 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.631 DEBUG:boto:Path: /avysmw-77/obj68 2022-09-03T07:38:14.631 DEBUG:boto:Data: 2022-09-03T07:38:14.632 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.632 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.633 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.633 DEBUG:boto:Params: {} 2022-09-03T07:38:14.634 DEBUG:boto:Token: None 2022-09-03T07:38:14.635 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj68 2022-09-03T07:38:14.635 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ux0ZKUP+OgIINGeps6Tyf8rasME= 2022-09-03T07:38:14.636 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ux0ZKUP+OgIINGeps6Tyf8rasME='} 2022-09-03T07:38:14.645 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000067c82a46ed804038-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.646 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.646 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.647 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.647 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.648 DEBUG:boto:Data: 2022-09-03T07:38:14.648 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.649 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.649 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.650 DEBUG:boto:Params: {} 2022-09-03T07:38:14.650 DEBUG:boto:Token: None 2022-09-03T07:38:14.651 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.652 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.653 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.656 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '9'), ('X-RGW-Bytes-Used', '27'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008c6e1fe8f22bcff0-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.657 DEBUG:boto:path=/avysmw-77/obj85 2022-09-03T07:38:14.658 DEBUG:boto:auth_path=/avysmw-77/obj85 2022-09-03T07:38:14.659 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.659 DEBUG:boto:Path: /avysmw-77/obj85 2022-09-03T07:38:14.660 DEBUG:boto:Data: 2022-09-03T07:38:14.661 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.661 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.662 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.663 DEBUG:boto:Params: {} 2022-09-03T07:38:14.663 DEBUG:boto:Token: None 2022-09-03T07:38:14.664 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj85 2022-09-03T07:38:14.665 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:V9lBgkmhqL+F0cGcis144ZDz/Ww= 2022-09-03T07:38:14.665 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:V9lBgkmhqL+F0cGcis144ZDz/Ww='} 2022-09-03T07:38:14.673 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000004c04aac75cbec8bd-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.674 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '19', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:14.675 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '19', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:14.675 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-77 --num-shards 19 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:38:14.731 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:14.731 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:14.779 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:38:14.780 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-77 2022-09-03T07:38:14.787 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 10 2022-09-03T07:38:14.867 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.869 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.870 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.871 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.872 DEBUG:boto:Data: 2022-09-03T07:38:14.873 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.874 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.874 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.875 DEBUG:boto:Params: {} 2022-09-03T07:38:14.877 DEBUG:boto:Token: None 2022-09-03T07:38:14.877 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.878 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.879 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.882 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '10'), ('X-RGW-Bytes-Used', '30'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004268f23d664c8758-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.883 DEBUG:boto:path=/avysmw-77/obj19 2022-09-03T07:38:14.883 DEBUG:boto:auth_path=/avysmw-77/obj19 2022-09-03T07:38:14.884 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.885 DEBUG:boto:Path: /avysmw-77/obj19 2022-09-03T07:38:14.885 DEBUG:boto:Data: 2022-09-03T07:38:14.886 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.887 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.888 DEBUG:boto:Params: {} 2022-09-03T07:38:14.888 DEBUG:boto:Token: None 2022-09-03T07:38:14.889 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj19 2022-09-03T07:38:14.890 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:C7m3ff6XKGjsab9Zd3SJUMUVPfA= 2022-09-03T07:38:14.891 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:C7m3ff6XKGjsab9Zd3SJUMUVPfA='} 2022-09-03T07:38:14.938 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000075d26c78bc570713-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.939 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.940 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.940 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.941 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.941 DEBUG:boto:Data: 2022-09-03T07:38:14.942 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.943 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.943 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.944 DEBUG:boto:Params: {} 2022-09-03T07:38:14.944 DEBUG:boto:Token: None 2022-09-03T07:38:14.945 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.946 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.946 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.949 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '11'), ('X-RGW-Bytes-Used', '33'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a26ffeaac3b39097-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.951 DEBUG:boto:path=/avysmw-77/obj38 2022-09-03T07:38:14.951 DEBUG:boto:auth_path=/avysmw-77/obj38 2022-09-03T07:38:14.952 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.953 DEBUG:boto:Path: /avysmw-77/obj38 2022-09-03T07:38:14.954 DEBUG:boto:Data: 2022-09-03T07:38:14.955 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.956 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.956 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.957 DEBUG:boto:Params: {} 2022-09-03T07:38:14.957 DEBUG:boto:Token: None 2022-09-03T07:38:14.958 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj38 2022-09-03T07:38:14.958 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:FokFolEel+ObT4p48zKO1Imcn2g= 2022-09-03T07:38:14.959 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:FokFolEel+ObT4p48zKO1Imcn2g='} 2022-09-03T07:38:14.965 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000c91ff2dd7c86eb51-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.966 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:14.967 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:14.967 DEBUG:boto:Method: HEAD 2022-09-03T07:38:14.968 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:14.968 DEBUG:boto:Data: 2022-09-03T07:38:14.969 DEBUG:boto:Headers: {} 2022-09-03T07:38:14.969 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.970 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.970 DEBUG:boto:Params: {} 2022-09-03T07:38:14.971 DEBUG:boto:Token: None 2022-09-03T07:38:14.971 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/ 2022-09-03T07:38:14.972 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU= 2022-09-03T07:38:14.973 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RFwt4KzHg5jo11FuOe678L6lwzU=', 'Content-Length': '0'} 2022-09-03T07:38:14.975 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '12'), ('X-RGW-Bytes-Used', '36'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d0c2d6d30c43789d-0063130466-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:14.976 DEBUG:boto:path=/avysmw-77/obj57 2022-09-03T07:38:14.977 DEBUG:boto:auth_path=/avysmw-77/obj57 2022-09-03T07:38:14.977 DEBUG:boto:Method: PUT 2022-09-03T07:38:14.978 DEBUG:boto:Path: /avysmw-77/obj57 2022-09-03T07:38:14.979 DEBUG:boto:Data: 2022-09-03T07:38:14.979 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:14.980 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:14.980 DEBUG:boto:Port: 8000 2022-09-03T07:38:14.981 DEBUG:boto:Params: {} 2022-09-03T07:38:14.982 DEBUG:boto:Token: None 2022-09-03T07:38:14.982 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:14 GMT /avysmw-77/obj57 2022-09-03T07:38:14.983 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:8ByH9RbYNIM3cFuxY93dURTF08E= 2022-09-03T07:38:14.983 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:8ByH9RbYNIM3cFuxY93dURTF08E='} 2022-09-03T07:38:15.026 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000272c11085f74afc9-0063130466-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.027 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.028 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.028 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.029 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.029 DEBUG:boto:Data: 2022-09-03T07:38:15.030 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.030 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.031 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.031 DEBUG:boto:Params: {} 2022-09-03T07:38:15.032 DEBUG:boto:Token: None 2022-09-03T07:38:15.032 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.033 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.034 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.036 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '13'), ('X-RGW-Bytes-Used', '39'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000b9e1139be81b4af-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.037 DEBUG:boto:path=/avysmw-77/obj76 2022-09-03T07:38:15.038 DEBUG:boto:auth_path=/avysmw-77/obj76 2022-09-03T07:38:15.038 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.039 DEBUG:boto:Path: /avysmw-77/obj76 2022-09-03T07:38:15.039 DEBUG:boto:Data: 2022-09-03T07:38:15.040 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.041 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.041 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.042 DEBUG:boto:Params: {} 2022-09-03T07:38:15.042 DEBUG:boto:Token: None 2022-09-03T07:38:15.043 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj76 2022-09-03T07:38:15.044 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Xx5LVPvxdUgthKZlaycR4sAJa/M= 2022-09-03T07:38:15.044 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Xx5LVPvxdUgthKZlaycR4sAJa/M='} 2022-09-03T07:38:15.050 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000007e1c35e4e3bd8ccb-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.051 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.051 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.052 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.053 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.053 DEBUG:boto:Data: 2022-09-03T07:38:15.054 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.054 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.055 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.056 DEBUG:boto:Params: {} 2022-09-03T07:38:15.057 DEBUG:boto:Token: None 2022-09-03T07:38:15.058 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.059 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.060 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.063 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '14'), ('X-RGW-Bytes-Used', '42'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000f0afc7d58615cd3b-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.064 DEBUG:boto:path=/avysmw-77/obj95 2022-09-03T07:38:15.065 DEBUG:boto:auth_path=/avysmw-77/obj95 2022-09-03T07:38:15.066 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.067 DEBUG:boto:Path: /avysmw-77/obj95 2022-09-03T07:38:15.068 DEBUG:boto:Data: 2022-09-03T07:38:15.069 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.070 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.070 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.071 DEBUG:boto:Params: {} 2022-09-03T07:38:15.073 DEBUG:boto:Token: None 2022-09-03T07:38:15.074 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj95 2022-09-03T07:38:15.075 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:UI4iWtTIt7MOp+n632QMNyv62rU= 2022-09-03T07:38:15.076 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:UI4iWtTIt7MOp+n632QMNyv62rU='} 2022-09-03T07:38:15.122 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx000001de18afc8d52046f-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.123 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '23', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:15.124 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '23', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:15.125 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-77 --num-shards 23 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:38:15.180 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:15.180 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:15.265 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:38:15.266 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-77 2022-09-03T07:38:15.274 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 15 2022-09-03T07:38:15.342 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.343 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.343 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.344 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.345 DEBUG:boto:Data: 2022-09-03T07:38:15.346 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.346 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.347 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.347 DEBUG:boto:Params: {} 2022-09-03T07:38:15.348 DEBUG:boto:Token: None 2022-09-03T07:38:15.349 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.349 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.351 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.355 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '15'), ('X-RGW-Bytes-Used', '45'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a183d80827ab3d2c-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.356 DEBUG:boto:path=/avysmw-77/obj23 2022-09-03T07:38:15.357 DEBUG:boto:auth_path=/avysmw-77/obj23 2022-09-03T07:38:15.357 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.358 DEBUG:boto:Path: /avysmw-77/obj23 2022-09-03T07:38:15.359 DEBUG:boto:Data: 2022-09-03T07:38:15.360 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.361 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.362 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.363 DEBUG:boto:Params: {} 2022-09-03T07:38:15.364 DEBUG:boto:Token: None 2022-09-03T07:38:15.364 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj23 2022-09-03T07:38:15.366 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:n2pEinNdOw4nQ+FrLE3bRuhn2iI= 2022-09-03T07:38:15.367 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:n2pEinNdOw4nQ+FrLE3bRuhn2iI='} 2022-09-03T07:38:15.376 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000b72a7e1130ce2ea1-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.377 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.377 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.378 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.379 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.379 DEBUG:boto:Data: 2022-09-03T07:38:15.379 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.380 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.380 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.381 DEBUG:boto:Params: {} 2022-09-03T07:38:15.382 DEBUG:boto:Token: None 2022-09-03T07:38:15.382 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.383 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.384 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.387 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '16'), ('X-RGW-Bytes-Used', '48'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000fa59a52c3d766b1-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.388 DEBUG:boto:path=/avysmw-77/obj46 2022-09-03T07:38:15.389 DEBUG:boto:auth_path=/avysmw-77/obj46 2022-09-03T07:38:15.390 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.392 DEBUG:boto:Path: /avysmw-77/obj46 2022-09-03T07:38:15.393 DEBUG:boto:Data: 2022-09-03T07:38:15.394 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.395 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.395 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.396 DEBUG:boto:Params: {} 2022-09-03T07:38:15.396 DEBUG:boto:Token: None 2022-09-03T07:38:15.397 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj46 2022-09-03T07:38:15.398 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:kX3vIPg8/nF94+EpycSCAPnNKLs= 2022-09-03T07:38:15.398 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:kX3vIPg8/nF94+EpycSCAPnNKLs='} 2022-09-03T07:38:15.442 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000aff38ae5fc7c9a27-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.443 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.444 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.445 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.445 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.446 DEBUG:boto:Data: 2022-09-03T07:38:15.447 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.448 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.448 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.449 DEBUG:boto:Params: {} 2022-09-03T07:38:15.452 DEBUG:boto:Token: None 2022-09-03T07:38:15.452 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.457 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '17'), ('X-RGW-Bytes-Used', '51'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000ac2acc142acff791-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.458 DEBUG:boto:path=/avysmw-77/obj69 2022-09-03T07:38:15.459 DEBUG:boto:auth_path=/avysmw-77/obj69 2022-09-03T07:38:15.460 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.460 DEBUG:boto:Path: /avysmw-77/obj69 2022-09-03T07:38:15.461 DEBUG:boto:Data: 2022-09-03T07:38:15.462 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.463 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.463 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.464 DEBUG:boto:Params: {} 2022-09-03T07:38:15.465 DEBUG:boto:Token: None 2022-09-03T07:38:15.466 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj69 2022-09-03T07:38:15.467 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:JEw/XU5etmimQG5IRHWFlfz2rA8= 2022-09-03T07:38:15.468 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:JEw/XU5etmimQG5IRHWFlfz2rA8='} 2022-09-03T07:38:15.480 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000cdac46359bbd0f6b-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.481 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.481 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.482 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.483 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.483 DEBUG:boto:Data: 2022-09-03T07:38:15.484 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.484 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.485 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.485 DEBUG:boto:Params: {} 2022-09-03T07:38:15.486 DEBUG:boto:Token: None 2022-09-03T07:38:15.486 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.487 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.488 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.491 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '18'), ('X-RGW-Bytes-Used', '54'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000067c1edb3bab3430e-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.491 DEBUG:boto:path=/avysmw-77/obj92 2022-09-03T07:38:15.492 DEBUG:boto:auth_path=/avysmw-77/obj92 2022-09-03T07:38:15.493 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.493 DEBUG:boto:Path: /avysmw-77/obj92 2022-09-03T07:38:15.494 DEBUG:boto:Data: 2022-09-03T07:38:15.494 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.495 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.495 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.496 DEBUG:boto:Params: {} 2022-09-03T07:38:15.496 DEBUG:boto:Token: None 2022-09-03T07:38:15.497 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj92 2022-09-03T07:38:15.497 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:la3bKYM4egbVG2DrFr8fQV/hbR0= 2022-09-03T07:38:15.498 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:la3bKYM4egbVG2DrFr8fQV/hbR0='} 2022-09-03T07:38:15.542 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000692f2c060b2b7fde-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.543 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.544 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.544 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.545 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.545 DEBUG:boto:Data: 2022-09-03T07:38:15.546 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.547 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.547 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.548 DEBUG:boto:Params: {} 2022-09-03T07:38:15.548 DEBUG:boto:Token: None 2022-09-03T07:38:15.549 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.549 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.550 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.553 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '19'), ('X-RGW-Bytes-Used', '57'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000110fc2f9ff549d6a-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.554 DEBUG:boto:path=/avysmw-77/obj115 2022-09-03T07:38:15.555 DEBUG:boto:auth_path=/avysmw-77/obj115 2022-09-03T07:38:15.555 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.556 DEBUG:boto:Path: /avysmw-77/obj115 2022-09-03T07:38:15.556 DEBUG:boto:Data: 2022-09-03T07:38:15.557 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.557 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.558 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.558 DEBUG:boto:Params: {} 2022-09-03T07:38:15.559 DEBUG:boto:Token: None 2022-09-03T07:38:15.559 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj115 2022-09-03T07:38:15.560 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:bUwyVv7ipXe+akv9w78nslJwBI0= 2022-09-03T07:38:15.561 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:bUwyVv7ipXe+akv9w78nslJwBI0='} 2022-09-03T07:38:15.572 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000069f68db7cbcb9279-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.573 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '29', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:15.573 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '29', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:15.574 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-77 --num-shards 29 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:38:15.630 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:15.630 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:15.686 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:38:15.686 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-77 2022-09-03T07:38:15.697 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 20 2022-09-03T07:38:15.784 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.785 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.785 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.786 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.786 DEBUG:boto:Data: 2022-09-03T07:38:15.787 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.787 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.788 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.789 DEBUG:boto:Params: {} 2022-09-03T07:38:15.789 DEBUG:boto:Token: None 2022-09-03T07:38:15.790 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.790 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.791 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.794 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '20'), ('X-RGW-Bytes-Used', '60'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000a4ba7d5e8c124ef9-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.795 DEBUG:boto:path=/avysmw-77/obj29 2022-09-03T07:38:15.796 DEBUG:boto:auth_path=/avysmw-77/obj29 2022-09-03T07:38:15.796 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.797 DEBUG:boto:Path: /avysmw-77/obj29 2022-09-03T07:38:15.797 DEBUG:boto:Data: 2022-09-03T07:38:15.799 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.800 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.800 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.801 DEBUG:boto:Params: {} 2022-09-03T07:38:15.802 DEBUG:boto:Token: None 2022-09-03T07:38:15.803 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj29 2022-09-03T07:38:15.803 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:o110bd2TlkK1h80QPPBdBMT/u08= 2022-09-03T07:38:15.804 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:o110bd2TlkK1h80QPPBdBMT/u08='} 2022-09-03T07:38:15.813 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f903615a35cf4538-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.813 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.814 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.815 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.816 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.816 DEBUG:boto:Data: 2022-09-03T07:38:15.817 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.817 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.818 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.818 DEBUG:boto:Params: {} 2022-09-03T07:38:15.819 DEBUG:boto:Token: None 2022-09-03T07:38:15.819 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.820 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.820 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.824 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '21'), ('X-RGW-Bytes-Used', '63'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000004f3d185cab21b81c-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.825 DEBUG:boto:path=/avysmw-77/obj58 2022-09-03T07:38:15.826 DEBUG:boto:auth_path=/avysmw-77/obj58 2022-09-03T07:38:15.826 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.827 DEBUG:boto:Path: /avysmw-77/obj58 2022-09-03T07:38:15.827 DEBUG:boto:Data: 2022-09-03T07:38:15.828 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.828 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.829 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.829 DEBUG:boto:Params: {} 2022-09-03T07:38:15.830 DEBUG:boto:Token: None 2022-09-03T07:38:15.832 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj58 2022-09-03T07:38:15.833 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:qLyfIEOIySh6WMV9T8d+LO/4jaY= 2022-09-03T07:38:15.834 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:qLyfIEOIySh6WMV9T8d+LO/4jaY='} 2022-09-03T07:38:15.882 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000435a11820effe142-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.883 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.884 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.884 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.885 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.885 DEBUG:boto:Data: 2022-09-03T07:38:15.886 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.886 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.887 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.887 DEBUG:boto:Params: {} 2022-09-03T07:38:15.888 DEBUG:boto:Token: None 2022-09-03T07:38:15.888 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.889 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.890 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.893 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '22'), ('X-RGW-Bytes-Used', '66'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000d4eadf36bc2115c9-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.894 DEBUG:boto:path=/avysmw-77/obj87 2022-09-03T07:38:15.894 DEBUG:boto:auth_path=/avysmw-77/obj87 2022-09-03T07:38:15.895 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.895 DEBUG:boto:Path: /avysmw-77/obj87 2022-09-03T07:38:15.896 DEBUG:boto:Data: 2022-09-03T07:38:15.896 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.897 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.897 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.898 DEBUG:boto:Params: {} 2022-09-03T07:38:15.898 DEBUG:boto:Token: None 2022-09-03T07:38:15.899 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj87 2022-09-03T07:38:15.899 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:PKEO/uRzJ+fFFA0ZhJY6AM5k+aQ= 2022-09-03T07:38:15.900 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:PKEO/uRzJ+fFFA0ZhJY6AM5k+aQ='} 2022-09-03T07:38:15.907 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000330beb165abf2fa9-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.908 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.909 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.909 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.910 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.911 DEBUG:boto:Data: 2022-09-03T07:38:15.911 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.912 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.912 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.913 DEBUG:boto:Params: {} 2022-09-03T07:38:15.913 DEBUG:boto:Token: None 2022-09-03T07:38:15.914 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.914 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.915 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.919 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '23'), ('X-RGW-Bytes-Used', '69'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000001f3d0c6fc5034a11-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.921 DEBUG:boto:path=/avysmw-77/obj116 2022-09-03T07:38:15.921 DEBUG:boto:auth_path=/avysmw-77/obj116 2022-09-03T07:38:15.922 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.922 DEBUG:boto:Path: /avysmw-77/obj116 2022-09-03T07:38:15.923 DEBUG:boto:Data: 2022-09-03T07:38:15.923 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.924 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.924 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.925 DEBUG:boto:Params: {} 2022-09-03T07:38:15.925 DEBUG:boto:Token: None 2022-09-03T07:38:15.926 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj116 2022-09-03T07:38:15.926 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:TUd7xfHzfeFEUuKJvzvZbYzm2Gc= 2022-09-03T07:38:15.927 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:TUd7xfHzfeFEUuKJvzvZbYzm2Gc='} 2022-09-03T07:38:15.970 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000039467f8121a0ddfb-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.971 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:15.972 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:15.972 DEBUG:boto:Method: HEAD 2022-09-03T07:38:15.972 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:15.973 DEBUG:boto:Data: 2022-09-03T07:38:15.973 DEBUG:boto:Headers: {} 2022-09-03T07:38:15.974 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.974 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.975 DEBUG:boto:Params: {} 2022-09-03T07:38:15.976 DEBUG:boto:Token: None 2022-09-03T07:38:15.976 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/ 2022-09-03T07:38:15.977 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo= 2022-09-03T07:38:15.977 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:P3SaR1E+zrx0SiKnKsPhV7fQWfo=', 'Content-Length': '0'} 2022-09-03T07:38:15.981 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '24'), ('X-RGW-Bytes-Used', '72'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000721283bff2c712c0-0063130467-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.981 DEBUG:boto:path=/avysmw-77/obj145 2022-09-03T07:38:15.982 DEBUG:boto:auth_path=/avysmw-77/obj145 2022-09-03T07:38:15.982 DEBUG:boto:Method: PUT 2022-09-03T07:38:15.983 DEBUG:boto:Path: /avysmw-77/obj145 2022-09-03T07:38:15.983 DEBUG:boto:Data: 2022-09-03T07:38:15.984 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:15.984 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:15.985 DEBUG:boto:Port: 8000 2022-09-03T07:38:15.985 DEBUG:boto:Params: {} 2022-09-03T07:38:15.986 DEBUG:boto:Token: None 2022-09-03T07:38:15.987 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:15 GMT /avysmw-77/obj145 2022-09-03T07:38:15.987 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f8CfbfwZh1fyJDtHsEGOELIgnfg= 2022-09-03T07:38:15.988 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:15 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f8CfbfwZh1fyJDtHsEGOELIgnfg='} 2022-09-03T07:38:15.995 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000aed30b0b87aad456-0063130467-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:15 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:15.996 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '31', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:15.997 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '31', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:15.997 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-77 --num-shards 31 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:38:16.051 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:16.052 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:16.120 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:38:16.120 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-77 2022-09-03T07:38:16.133 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 25 2022-09-03T07:38:16.224 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.225 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.226 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.227 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.229 DEBUG:boto:Data: 2022-09-03T07:38:16.230 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.231 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.232 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.233 DEBUG:boto:Params: {} 2022-09-03T07:38:16.234 DEBUG:boto:Token: None 2022-09-03T07:38:16.235 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.236 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.237 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.246 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '25'), ('X-RGW-Bytes-Used', '75'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000008400de6b1a135edf-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.248 DEBUG:boto:path=/avysmw-77/obj31 2022-09-03T07:38:16.249 DEBUG:boto:auth_path=/avysmw-77/obj31 2022-09-03T07:38:16.250 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.251 DEBUG:boto:Path: /avysmw-77/obj31 2022-09-03T07:38:16.252 DEBUG:boto:Data: 2022-09-03T07:38:16.252 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.253 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.254 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.255 DEBUG:boto:Params: {} 2022-09-03T07:38:16.256 DEBUG:boto:Token: None 2022-09-03T07:38:16.256 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj31 2022-09-03T07:38:16.257 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:jvwHBN13IeQqNziwgWwGkbqXMsw= 2022-09-03T07:38:16.258 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:jvwHBN13IeQqNziwgWwGkbqXMsw='} 2022-09-03T07:38:16.265 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000a83eb7dfba3b3ede-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.266 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.267 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.267 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.268 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.268 DEBUG:boto:Data: 2022-09-03T07:38:16.269 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.270 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.271 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.272 DEBUG:boto:Params: {} 2022-09-03T07:38:16.273 DEBUG:boto:Token: None 2022-09-03T07:38:16.274 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.275 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.276 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.280 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '26'), ('X-RGW-Bytes-Used', '78'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000000aeeacce4b6baa68-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.282 DEBUG:boto:path=/avysmw-77/obj62 2022-09-03T07:38:16.283 DEBUG:boto:auth_path=/avysmw-77/obj62 2022-09-03T07:38:16.284 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.284 DEBUG:boto:Path: /avysmw-77/obj62 2022-09-03T07:38:16.285 DEBUG:boto:Data: 2022-09-03T07:38:16.285 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.286 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.289 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.289 DEBUG:boto:Params: {} 2022-09-03T07:38:16.290 DEBUG:boto:Token: None 2022-09-03T07:38:16.290 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj62 2022-09-03T07:38:16.291 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:4Ip2lbp2uchqPACGojvzwIk2YmA= 2022-09-03T07:38:16.292 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:4Ip2lbp2uchqPACGojvzwIk2YmA='} 2022-09-03T07:38:16.335 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f8b56436b6baee25-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.336 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.336 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.337 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.338 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.338 DEBUG:boto:Data: 2022-09-03T07:38:16.339 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.340 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.340 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.341 DEBUG:boto:Params: {} 2022-09-03T07:38:16.342 DEBUG:boto:Token: None 2022-09-03T07:38:16.342 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.343 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.344 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.348 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '27'), ('X-RGW-Bytes-Used', '81'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e986b65ef0471622-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.350 DEBUG:boto:path=/avysmw-77/obj93 2022-09-03T07:38:16.351 DEBUG:boto:auth_path=/avysmw-77/obj93 2022-09-03T07:38:16.352 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.353 DEBUG:boto:Path: /avysmw-77/obj93 2022-09-03T07:38:16.354 DEBUG:boto:Data: 2022-09-03T07:38:16.355 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.355 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.356 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.357 DEBUG:boto:Params: {} 2022-09-03T07:38:16.358 DEBUG:boto:Token: None 2022-09-03T07:38:16.359 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj93 2022-09-03T07:38:16.359 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:udnH7peFUmslRmSrBhhMa9qufqM= 2022-09-03T07:38:16.360 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:udnH7peFUmslRmSrBhhMa9qufqM='} 2022-09-03T07:38:16.369 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000c6ec03ae461e76d3-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.370 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.371 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.371 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.372 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.372 DEBUG:boto:Data: 2022-09-03T07:38:16.373 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.373 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.374 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.375 DEBUG:boto:Params: {} 2022-09-03T07:38:16.375 DEBUG:boto:Token: None 2022-09-03T07:38:16.376 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.377 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.378 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.381 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '28'), ('X-RGW-Bytes-Used', '84'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000008d3ef6e00c8ca80-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.382 DEBUG:boto:path=/avysmw-77/obj124 2022-09-03T07:38:16.383 DEBUG:boto:auth_path=/avysmw-77/obj124 2022-09-03T07:38:16.384 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.384 DEBUG:boto:Path: /avysmw-77/obj124 2022-09-03T07:38:16.385 DEBUG:boto:Data: 2022-09-03T07:38:16.385 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.386 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.387 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.388 DEBUG:boto:Params: {} 2022-09-03T07:38:16.388 DEBUG:boto:Token: None 2022-09-03T07:38:16.389 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj124 2022-09-03T07:38:16.390 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:f5zBlEMvCXJzouSNmpxBh9c/kmk= 2022-09-03T07:38:16.391 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:f5zBlEMvCXJzouSNmpxBh9c/kmk='} 2022-09-03T07:38:16.434 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f7fe0152aa1dab81-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.435 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.436 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.436 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.437 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.437 DEBUG:boto:Data: 2022-09-03T07:38:16.438 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.438 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.439 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.439 DEBUG:boto:Params: {} 2022-09-03T07:38:16.440 DEBUG:boto:Token: None 2022-09-03T07:38:16.441 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.441 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.442 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.446 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '29'), ('X-RGW-Bytes-Used', '87'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000506af1faf8bebca3-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.447 DEBUG:boto:path=/avysmw-77/obj155 2022-09-03T07:38:16.447 DEBUG:boto:auth_path=/avysmw-77/obj155 2022-09-03T07:38:16.448 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.448 DEBUG:boto:Path: /avysmw-77/obj155 2022-09-03T07:38:16.449 DEBUG:boto:Data: 2022-09-03T07:38:16.449 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.450 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.451 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.451 DEBUG:boto:Params: {} 2022-09-03T07:38:16.452 DEBUG:boto:Token: None 2022-09-03T07:38:16.453 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj155 2022-09-03T07:38:16.453 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:T4JTAuoq8yRgk3rP+OQJ93oQ0cY= 2022-09-03T07:38:16.454 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:T4JTAuoq8yRgk3rP+OQJ93oQ0cY='} 2022-09-03T07:38:16.460 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000069b1ee3a1a583f6c-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.461 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '37', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:16.461 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--bucket', 'avysmw-77', '--num-shards', '37', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:16.462 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --bucket avysmw-77 --num-shards 37 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2022-09-03T07:38:16.517 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:16.518 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:16.584 INFO:teuthology.orchestra.run.smithi062.stdout:tenant: 2022-09-03T07:38:16.585 INFO:teuthology.orchestra.run.smithi062.stdout:bucket name: avysmw-77 2022-09-03T07:38:16.599 INFO:teuthology.orchestra.run.smithi062.stdout:total entries: 30 2022-09-03T07:38:16.722 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.723 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.724 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.724 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.725 DEBUG:boto:Data: 2022-09-03T07:38:16.725 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.726 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.727 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.727 DEBUG:boto:Params: {} 2022-09-03T07:38:16.728 DEBUG:boto:Token: None 2022-09-03T07:38:16.729 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.729 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.730 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.734 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '30'), ('X-RGW-Bytes-Used', '90'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000870617ad25e2b50c-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.735 DEBUG:boto:path=/avysmw-77/obj37 2022-09-03T07:38:16.736 DEBUG:boto:auth_path=/avysmw-77/obj37 2022-09-03T07:38:16.736 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.737 DEBUG:boto:Path: /avysmw-77/obj37 2022-09-03T07:38:16.738 DEBUG:boto:Data: 2022-09-03T07:38:16.738 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.739 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.739 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.740 DEBUG:boto:Params: {} 2022-09-03T07:38:16.741 DEBUG:boto:Token: None 2022-09-03T07:38:16.742 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj37 2022-09-03T07:38:16.742 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ZBIGKHKUtqFdWuDsQ1j1qnOLWug= 2022-09-03T07:38:16.743 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ZBIGKHKUtqFdWuDsQ1j1qnOLWug='} 2022-09-03T07:38:16.753 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000f6e326ba9a7e70c9-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.753 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.754 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.755 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.755 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.756 DEBUG:boto:Data: 2022-09-03T07:38:16.757 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.758 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.759 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.760 DEBUG:boto:Params: {} 2022-09-03T07:38:16.761 DEBUG:boto:Token: None 2022-09-03T07:38:16.761 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.765 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.766 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.770 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '31'), ('X-RGW-Bytes-Used', '93'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx0000093bb9cb2356d062d-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.772 DEBUG:boto:path=/avysmw-77/obj74 2022-09-03T07:38:16.773 DEBUG:boto:auth_path=/avysmw-77/obj74 2022-09-03T07:38:16.774 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.775 DEBUG:boto:Path: /avysmw-77/obj74 2022-09-03T07:38:16.776 DEBUG:boto:Data: 2022-09-03T07:38:16.777 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.778 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.779 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.780 DEBUG:boto:Params: {} 2022-09-03T07:38:16.781 DEBUG:boto:Token: None 2022-09-03T07:38:16.782 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj74 2022-09-03T07:38:16.784 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:r5U6l3YRLxp/FUBi0ZZ+BkxyAVU= 2022-09-03T07:38:16.785 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:r5U6l3YRLxp/FUBi0ZZ+BkxyAVU='} 2022-09-03T07:38:16.842 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000b521bcc29f5adeb4-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.843 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.844 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.844 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.845 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.845 DEBUG:boto:Data: 2022-09-03T07:38:16.846 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.847 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.847 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.848 DEBUG:boto:Params: {} 2022-09-03T07:38:16.849 DEBUG:boto:Token: None 2022-09-03T07:38:16.850 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.851 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.852 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.858 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '32'), ('X-RGW-Bytes-Used', '96'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000e6db6b79836ffc8e-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.859 DEBUG:boto:path=/avysmw-77/obj111 2022-09-03T07:38:16.860 DEBUG:boto:auth_path=/avysmw-77/obj111 2022-09-03T07:38:16.860 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.861 DEBUG:boto:Path: /avysmw-77/obj111 2022-09-03T07:38:16.861 DEBUG:boto:Data: 2022-09-03T07:38:16.862 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.863 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.863 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.864 DEBUG:boto:Params: {} 2022-09-03T07:38:16.864 DEBUG:boto:Token: None 2022-09-03T07:38:16.865 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj111 2022-09-03T07:38:16.866 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:ENOHHJib9WM10hOU7a1xXHyZ14Q= 2022-09-03T07:38:16.866 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:ENOHHJib9WM10hOU7a1xXHyZ14Q='} 2022-09-03T07:38:16.874 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000d2c8e04bb8db22ab-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.875 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.876 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.876 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.877 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.878 DEBUG:boto:Data: 2022-09-03T07:38:16.878 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.879 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.879 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.880 DEBUG:boto:Params: {} 2022-09-03T07:38:16.882 DEBUG:boto:Token: None 2022-09-03T07:38:16.883 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.884 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.885 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.889 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '33'), ('X-RGW-Bytes-Used', '99'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx00000553672b77e726d8b-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.891 DEBUG:boto:path=/avysmw-77/obj148 2022-09-03T07:38:16.892 DEBUG:boto:auth_path=/avysmw-77/obj148 2022-09-03T07:38:16.893 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.894 DEBUG:boto:Path: /avysmw-77/obj148 2022-09-03T07:38:16.895 DEBUG:boto:Data: 2022-09-03T07:38:16.896 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.897 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.897 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.898 DEBUG:boto:Params: {} 2022-09-03T07:38:16.899 DEBUG:boto:Token: None 2022-09-03T07:38:16.899 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj148 2022-09-03T07:38:16.900 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:RHufzkfpGtuzQ33Vwt7MPhXRZqM= 2022-09-03T07:38:16.900 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:RHufzkfpGtuzQ33Vwt7MPhXRZqM='} 2022-09-03T07:38:16.946 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 has been restored 2022-09-03T07:38:16.947 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx00000624424cb468830b7-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.948 DEBUG:boto:path=/avysmw-77/ 2022-09-03T07:38:16.949 DEBUG:boto:auth_path=/avysmw-77/ 2022-09-03T07:38:16.953 DEBUG:boto:Method: HEAD 2022-09-03T07:38:16.954 DEBUG:boto:Path: /avysmw-77/ 2022-09-03T07:38:16.954 DEBUG:boto:Data: 2022-09-03T07:38:16.957 DEBUG:boto:Headers: {} 2022-09-03T07:38:16.958 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.959 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.961 DEBUG:boto:Params: {} 2022-09-03T07:38:16.961 DEBUG:boto:Token: None 2022-09-03T07:38:16.962 DEBUG:boto:StringToSign: HEAD Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/ 2022-09-03T07:38:16.963 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw= 2022-09-03T07:38:16.963 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Gu13LcX+erecmCkwHlelKGEHnOw=', 'Content-Length': '0'} 2022-09-03T07:38:16.968 DEBUG:boto:Response headers: [('X-RGW-Object-Count', '34'), ('X-RGW-Bytes-Used', '102'), ('X-RGW-Quota-User-Size', '-1'), ('X-RGW-Quota-User-Objects', '-1'), ('X-RGW-Quota-Max-Buckets', '1000'), ('X-RGW-Quota-Bucket-Size', '-1'), ('X-RGW-Quota-Bucket-Objects', '-1'), ('x-amz-request-id', 'tx000003ae51a56c2b4e1db-0063130468-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.970 DEBUG:boto:path=/avysmw-77/obj185 2022-09-03T07:38:16.971 DEBUG:boto:auth_path=/avysmw-77/obj185 2022-09-03T07:38:16.972 DEBUG:boto:Method: PUT 2022-09-03T07:38:16.973 DEBUG:boto:Path: /avysmw-77/obj185 2022-09-03T07:38:16.973 DEBUG:boto:Data: 2022-09-03T07:38:16.974 DEBUG:boto:Headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue'} 2022-09-03T07:38:16.975 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:38:16.976 DEBUG:boto:Port: 8000 2022-09-03T07:38:16.977 DEBUG:boto:Params: {} 2022-09-03T07:38:16.978 DEBUG:boto:Token: None 2022-09-03T07:38:16.979 DEBUG:boto:StringToSign: PUT rL0Y20zC+Fzt72VPzMSk2A== application/octet-stream Sat, 03 Sep 2022 07:38:16 GMT /avysmw-77/obj185 2022-09-03T07:38:16.980 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:NGrd/qQIjm7KxpfLTkTy/vun5pg= 2022-09-03T07:38:16.980 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Content-Type': 'application/octet-stream', 'Content-MD5': 'rL0Y20zC+Fzt72VPzMSk2A==', 'Content-Length': '3', 'Expect': '100-Continue', 'Date': 'Sat, 03 Sep 2022 07:38:16 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:NGrd/qQIjm7KxpfLTkTy/vun5pg='} 2022-09-03T07:38:16.986 DEBUG:boto:Response headers: [('Content-Length', '0'), ('ETag', '"acbd18db4cc2f85cedef654fccc4a4d8"'), ('Accept-Ranges', 'bytes'), ('x-amz-request-id', 'tx0000057ed64d0e5377833-0063130468-1604-test-zone1'), ('Date', 'Sat, 03 Sep 2022 07:38:16 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:38:16.987 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'init', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:16.987 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'init', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2022-09-03T07:38:16.988 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync init --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 2022-09-03T07:38:17.074 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:17.074 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:18.147 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:18.146+0000 7f960486e600 10 RGW-SYNC:data:init_data_sync_status: took lease 2022-09-03T07:38:18.194 INFO:tasks.rgw.c2.client.0:Restarting daemon 2022-09-03T07:38:18.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2022-09-03T07:38:18.199 INFO:tasks.rgw.c2.client.0:Started 2022-09-03T07:38:18.200 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:38:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 7 2022-09-03T07:38:18.222 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:38:18.223 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:38:18.223 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi087.front.sepia.ceph.com port 8000: Connection refused 2022-09-03T07:38:19.224 DEBUG:teuthology.orchestra.run.smithi087:> curl http://smithi087.front.sepia.ceph.com:8000 2022-09-03T07:38:19.247 INFO:teuthology.orchestra.run.smithi087.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2022-09-03T07:38:19.248 INFO:teuthology.orchestra.run.smithi087.stderr: Dload Upload Total Spent Left Speed 2022-09-03T07:38:19.248 INFO:teuthology.orchestra.run.smithi087.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 208k 0 --:--:-- --:--:-- --:--:-- 208k 2022-09-03T07:38:19.249 INFO:teuthology.orchestra.run.smithi087.stdout:anonymous 2022-09-03T07:38:19.250 DEBUG:rgw_multi.tests:bucket checkpoint: source=test-zone1 target=test-zone2 bucket=avysmw-77 2022-09-03T07:38:19.251 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-77', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:38:19.251 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'avysmw-77', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2022-09-03T07:38:19.252 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket avysmw-77 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2022-09-03T07:38:19.358 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:38:19.359 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:38:19.424 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:19.422+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:38:21.902 INFO:tasks.rgw.c1.client.1.smithi062.stdout:*** Caught signal (Segmentation fault) ** 2022-09-03T07:38:21.903 INFO:tasks.rgw.c1.client.1.smithi062.stdout: in thread 7f0009859700 thread_name:data-sync 2022-09-03T07:38:21.909 INFO:tasks.rgw.c1.client.1.smithi062.stdout: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:38:21.909 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f0033569cf0] 2022-09-03T07:38:21.910 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2: (RGWBucketSyncSingleEntryCR::operate(DoutPrefixProvider const*)+0x870) [0x55f17d3b6ae0] 2022-09-03T07:38:21.910 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 3: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x18f) [0x55f17d4a0e2f] 2022-09-03T07:38:21.910 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 4: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55f17d4a1601] 2022-09-03T07:38:21.911 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 5: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x55f17d4a2611] 2022-09-03T07:38:21.911 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 6: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x20a) [0x55f17d35aaca] 2022-09-03T07:38:21.911 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 7: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x58) [0x55f17d594528] 2022-09-03T07:38:21.911 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 8: (RGWRadosThread::Worker::entry()+0xb3) [0x55f17d55e8c3] 2022-09-03T07:38:21.912 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 9: /lib64/libpthread.so.0(+0x81ca) [0x7f003355f1ca] 2022-09-03T07:38:21.912 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 10: clone() 2022-09-03T07:38:21.912 INFO:tasks.rgw.c1.client.1.smithi062.stdout:2022-09-03T07:38:21.905+0000 7f0009859700 -1 *** Caught signal (Segmentation fault) ** 2022-09-03T07:38:21.925 INFO:tasks.rgw.c1.client.1.smithi062.stdout: in thread 7f0009859700 thread_name:data-sync 2022-09-03T07:38:21.926 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2022-09-03T07:38:21.926 INFO:tasks.rgw.c1.client.1.smithi062.stdout: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:38:21.927 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f0033569cf0] 2022-09-03T07:38:21.927 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2: (RGWBucketSyncSingleEntryCR::operate(DoutPrefixProvider const*)+0x870) [0x55f17d3b6ae0] 2022-09-03T07:38:21.928 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 3: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x18f) [0x55f17d4a0e2f] 2022-09-03T07:38:21.928 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 4: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55f17d4a1601] 2022-09-03T07:38:21.929 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 5: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x55f17d4a2611] 2022-09-03T07:38:21.929 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 6: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x20a) [0x55f17d35aaca] 2022-09-03T07:38:21.930 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 7: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x58) [0x55f17d594528] 2022-09-03T07:38:21.930 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 8: (RGWRadosThread::Worker::entry()+0xb3) [0x55f17d55e8c3] 2022-09-03T07:38:21.931 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 9: /lib64/libpthread.so.0(+0x81ca) [0x7f003355f1ca] 2022-09-03T07:38:21.931 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 10: clone() 2022-09-03T07:38:21.932 INFO:tasks.rgw.c1.client.1.smithi062.stdout: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2022-09-03T07:38:21.932 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2022-09-03T07:38:21.933 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 0> 2022-09-03T07:38:21.905+0000 7f0009859700 -1 *** Caught signal (Segmentation fault) ** 2022-09-03T07:38:21.934 INFO:tasks.rgw.c1.client.1.smithi062.stdout: in thread 7f0009859700 thread_name:data-sync 2022-09-03T07:38:21.934 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2022-09-03T07:38:21.935 INFO:tasks.rgw.c1.client.1.smithi062.stdout: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:38:21.935 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f0033569cf0] 2022-09-03T07:38:21.936 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2: (RGWBucketSyncSingleEntryCR::operate(DoutPrefixProvider const*)+0x870) [0x55f17d3b6ae0] 2022-09-03T07:38:21.936 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 3: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x18f) [0x55f17d4a0e2f] 2022-09-03T07:38:21.937 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 4: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55f17d4a1601] 2022-09-03T07:38:21.937 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 5: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x55f17d4a2611] 2022-09-03T07:38:21.938 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 6: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x20a) [0x55f17d35aaca] 2022-09-03T07:38:21.938 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 7: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x58) [0x55f17d594528] 2022-09-03T07:38:21.939 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 8: (RGWRadosThread::Worker::entry()+0xb3) [0x55f17d55e8c3] 2022-09-03T07:38:21.939 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 9: /lib64/libpthread.so.0(+0x81ca) [0x7f003355f1ca] 2022-09-03T07:38:21.940 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 10: clone() 2022-09-03T07:38:21.941 INFO:tasks.rgw.c1.client.1.smithi062.stdout: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2022-09-03T07:38:21.941 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2022-09-03T07:38:21.943 INFO:tasks.rgw.c1.client.1.smithi062.stdout: -169> 2022-09-03T07:38:21.905+0000 7f0009859700 -1 *** Caught signal (Segmentation fault) ** 2022-09-03T07:38:21.943 INFO:tasks.rgw.c1.client.1.smithi062.stdout: in thread 7f0009859700 thread_name:data-sync 2022-09-03T07:38:21.944 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2022-09-03T07:38:21.944 INFO:tasks.rgw.c1.client.1.smithi062.stdout: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:38:21.945 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f0033569cf0] 2022-09-03T07:38:21.945 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2: (RGWBucketSyncSingleEntryCR::operate(DoutPrefixProvider const*)+0x870) [0x55f17d3b6ae0] 2022-09-03T07:38:21.946 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 3: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x18f) [0x55f17d4a0e2f] 2022-09-03T07:38:21.946 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 4: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55f17d4a1601] 2022-09-03T07:38:21.947 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 5: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x55f17d4a2611] 2022-09-03T07:38:21.947 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 6: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x20a) [0x55f17d35aaca] 2022-09-03T07:38:21.948 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 7: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x58) [0x55f17d594528] 2022-09-03T07:38:21.948 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 8: (RGWRadosThread::Worker::entry()+0xb3) [0x55f17d55e8c3] 2022-09-03T07:38:21.949 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 9: /lib64/libpthread.so.0(+0x81ca) [0x7f003355f1ca] 2022-09-03T07:38:21.949 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 10: clone() 2022-09-03T07:38:21.949 INFO:tasks.rgw.c1.client.1.smithi062.stdout: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2022-09-03T07:38:21.950 INFO:tasks.rgw.c1.client.1.smithi062.stdout: 2022-09-03T07:38:21.985 INFO:tasks.rgw.c1.client.1.smithi062.stderr:daemon-helper: command crashed with signal 11 2022-09-03T07:38:24.424 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:24.422+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:38:27.542 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~0s 2022-09-03T07:38:29.424 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:29.423+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:38:33.148 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~6s 2022-09-03T07:38:34.426 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:34.423+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:38:38.755 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~11s 2022-09-03T07:38:39.426 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:39.424+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:38:41.326 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:38:41.325+0000 7f8d0e9f7700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16) 2022-09-03T07:38:41.369 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:38:41.368+0000 7f8d0e9f7700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16) 2022-09-03T07:38:44.362 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~17s 2022-09-03T07:38:44.426 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:44.424+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:38:49.426 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:49.424+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:38:49.969 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~22s 2022-09-03T07:38:54.427 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:54.425+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:38:55.576 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~28s 2022-09-03T07:38:59.427 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:38:59.425+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:01.183 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~34s 2022-09-03T07:39:04.428 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:04.426+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:06.789 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~39s 2022-09-03T07:39:09.428 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:09.427+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:12.394 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~45s 2022-09-03T07:39:14.428 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:14.427+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:18.001 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~50s 2022-09-03T07:39:19.429 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:19.427+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:23.448 INFO:tasks.rgw.c2.client.0.smithi087.stdout:*** Caught signal (Segmentation fault) ** 2022-09-03T07:39:23.448 INFO:tasks.rgw.c2.client.0.smithi087.stdout: in thread 7f650f42b700 thread_name:data-sync 2022-09-03T07:39:23.539 INFO:tasks.rgw.c2.client.0.smithi087.stdout: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:39:23.540 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f653913bcf0] 2022-09-03T07:39:23.540 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2: /lib64/libc.so.6(+0xc96e4) [0x7f6537f5b6e4] 2022-09-03T07:39:23.540 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 3: radosgw(+0x92bb63) [0x55c1404bbb63] 2022-09-03T07:39:23.541 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 4: (RGWBucketSyncSingleEntryCR::operate(DoutPrefixProvider const*)+0x87f) [0x55c14052daef] 2022-09-03T07:39:23.541 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 5: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x18f) [0x55c140617e2f] 2022-09-03T07:39:23.541 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 6: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55c140618601] 2022-09-03T07:39:23.541 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 7: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x55c140619611] 2022-09-03T07:39:23.542 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 8: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x20a) [0x55c1404d1aca] 2022-09-03T07:39:23.542 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 9: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x58) [0x55c14070b528] 2022-09-03T07:39:23.542 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 10: (RGWRadosThread::Worker::entry()+0xb3) [0x55c1406d58c3] 2022-09-03T07:39:23.543 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 11: /lib64/libpthread.so.0(+0x81ca) [0x7f65391311ca] 2022-09-03T07:39:23.543 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 12: clone() 2022-09-03T07:39:23.543 INFO:tasks.rgw.c2.client.0.smithi087.stdout:2022-09-03T07:39:23.538+0000 7f650f42b700 -1 *** Caught signal (Segmentation fault) ** 2022-09-03T07:39:23.543 INFO:tasks.rgw.c2.client.0.smithi087.stdout: in thread 7f650f42b700 thread_name:data-sync 2022-09-03T07:39:23.544 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2022-09-03T07:39:23.544 INFO:tasks.rgw.c2.client.0.smithi087.stdout: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:39:23.544 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f653913bcf0] 2022-09-03T07:39:23.544 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2: /lib64/libc.so.6(+0xc96e4) [0x7f6537f5b6e4] 2022-09-03T07:39:23.545 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 3: radosgw(+0x92bb63) [0x55c1404bbb63] 2022-09-03T07:39:23.545 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 4: (RGWBucketSyncSingleEntryCR::operate(DoutPrefixProvider const*)+0x87f) [0x55c14052daef] 2022-09-03T07:39:23.545 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 5: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x18f) [0x55c140617e2f] 2022-09-03T07:39:23.545 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 6: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55c140618601] 2022-09-03T07:39:23.546 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 7: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x55c140619611] 2022-09-03T07:39:23.546 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 8: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x20a) [0x55c1404d1aca] 2022-09-03T07:39:23.546 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 9: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x58) [0x55c14070b528] 2022-09-03T07:39:23.546 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 10: (RGWRadosThread::Worker::entry()+0xb3) [0x55c1406d58c3] 2022-09-03T07:39:23.547 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 11: /lib64/libpthread.so.0(+0x81ca) [0x7f65391311ca] 2022-09-03T07:39:23.547 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 12: clone() 2022-09-03T07:39:23.547 INFO:tasks.rgw.c2.client.0.smithi087.stdout: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2022-09-03T07:39:23.548 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2022-09-03T07:39:23.560 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 0> 2022-09-03T07:39:23.538+0000 7f650f42b700 -1 *** Caught signal (Segmentation fault) ** 2022-09-03T07:39:23.560 INFO:tasks.rgw.c2.client.0.smithi087.stdout: in thread 7f650f42b700 thread_name:data-sync 2022-09-03T07:39:23.561 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2022-09-03T07:39:23.561 INFO:tasks.rgw.c2.client.0.smithi087.stdout: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:39:23.561 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f653913bcf0] 2022-09-03T07:39:23.562 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2: /lib64/libc.so.6(+0xc96e4) [0x7f6537f5b6e4] 2022-09-03T07:39:23.562 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 3: radosgw(+0x92bb63) [0x55c1404bbb63] 2022-09-03T07:39:23.563 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 4: (RGWBucketSyncSingleEntryCR::operate(DoutPrefixProvider const*)+0x87f) [0x55c14052daef] 2022-09-03T07:39:23.563 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 5: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x18f) [0x55c140617e2f] 2022-09-03T07:39:23.564 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 6: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55c140618601] 2022-09-03T07:39:23.564 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 7: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x55c140619611] 2022-09-03T07:39:23.564 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 8: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x20a) [0x55c1404d1aca] 2022-09-03T07:39:23.565 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 9: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x58) [0x55c14070b528] 2022-09-03T07:39:23.565 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 10: (RGWRadosThread::Worker::entry()+0xb3) [0x55c1406d58c3] 2022-09-03T07:39:23.565 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 11: /lib64/libpthread.so.0(+0x81ca) [0x7f65391311ca] 2022-09-03T07:39:23.565 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 12: clone() 2022-09-03T07:39:23.566 INFO:tasks.rgw.c2.client.0.smithi087.stdout: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2022-09-03T07:39:23.566 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2022-09-03T07:39:23.572 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 0> 2022-09-03T07:39:23.538+0000 7f650f42b700 -1 *** Caught signal (Segmentation fault) ** 2022-09-03T07:39:23.572 INFO:tasks.rgw.c2.client.0.smithi087.stdout: in thread 7f650f42b700 thread_name:data-sync 2022-09-03T07:39:23.573 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2022-09-03T07:39:23.573 INFO:tasks.rgw.c2.client.0.smithi087.stdout: ceph version 17.0.0-14689-gafd42238 (afd422385fab8119a539d289dd1aeaad4c056fe4) quincy (dev) 2022-09-03T07:39:23.574 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f653913bcf0] 2022-09-03T07:39:23.574 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2: /lib64/libc.so.6(+0xc96e4) [0x7f6537f5b6e4] 2022-09-03T07:39:23.575 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 3: radosgw(+0x92bb63) [0x55c1404bbb63] 2022-09-03T07:39:23.575 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 4: (RGWBucketSyncSingleEntryCR::operate(DoutPrefixProvider const*)+0x87f) [0x55c14052daef] 2022-09-03T07:39:23.575 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 5: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x18f) [0x55c140617e2f] 2022-09-03T07:39:23.576 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 6: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x521) [0x55c140618601] 2022-09-03T07:39:23.576 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 7: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x55c140619611] 2022-09-03T07:39:23.576 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 8: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x20a) [0x55c1404d1aca] 2022-09-03T07:39:23.577 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 9: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x58) [0x55c14070b528] 2022-09-03T07:39:23.577 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 10: (RGWRadosThread::Worker::entry()+0xb3) [0x55c1406d58c3] 2022-09-03T07:39:23.577 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 11: /lib64/libpthread.so.0(+0x81ca) [0x7f65391311ca] 2022-09-03T07:39:23.577 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 12: clone() 2022-09-03T07:39:23.578 INFO:tasks.rgw.c2.client.0.smithi087.stdout: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2022-09-03T07:39:23.578 INFO:tasks.rgw.c2.client.0.smithi087.stdout: 2022-09-03T07:39:23.607 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~56s 2022-09-03T07:39:23.742 INFO:tasks.rgw.c2.client.0.smithi087.stderr:daemon-helper: command crashed with signal 11 2022-09-03T07:39:24.430 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:24.428+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:25.342 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~0s 2022-09-03T07:39:29.219 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~62s 2022-09-03T07:39:29.430 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:29.429+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:30.948 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~6s 2022-09-03T07:39:34.431 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:34.429+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:34.836 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~67s 2022-09-03T07:39:36.556 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~11s 2022-09-03T07:39:39.431 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:39.430+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:40.453 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~73s 2022-09-03T07:39:42.164 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~17s 2022-09-03T07:39:44.432 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:44.430+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:46.060 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~79s 2022-09-03T07:39:47.773 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~22s 2022-09-03T07:39:49.432 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:49.431+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:51.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~84s 2022-09-03T07:39:53.380 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~28s 2022-09-03T07:39:54.433 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:54.431+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:39:57.274 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~90s 2022-09-03T07:39:58.988 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~34s 2022-09-03T07:39:59.433 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:39:59.432+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:02.888 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~95s 2022-09-03T07:40:04.434 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:04.432+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:04.593 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~39s 2022-09-03T07:40:08.495 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~101s 2022-09-03T07:40:09.434 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:09.433+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:10.201 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~45s 2022-09-03T07:40:14.100 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~107s 2022-09-03T07:40:14.434 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:14.433+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:15.810 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~50s 2022-09-03T07:40:19.435 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:19.433+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:19.706 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~112s 2022-09-03T07:40:21.417 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~56s 2022-09-03T07:40:24.436 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:24.434+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:25.313 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~118s 2022-09-03T07:40:27.025 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~62s 2022-09-03T07:40:29.447 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:29.434+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:30.918 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~123s 2022-09-03T07:40:32.632 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~67s 2022-09-03T07:40:34.437 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:34.435+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:36.525 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~129s 2022-09-03T07:40:38.241 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~73s 2022-09-03T07:40:39.437 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:39.436+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:42.135 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~135s 2022-09-03T07:40:43.846 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~79s 2022-09-03T07:40:44.437 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:44.436+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:47.740 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~140s 2022-09-03T07:40:49.439 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:49.438+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:49.454 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~84s 2022-09-03T07:40:53.348 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~146s 2022-09-03T07:40:54.443 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:54.441+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:40:55.062 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~90s 2022-09-03T07:40:58.955 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~151s 2022-09-03T07:40:59.443 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:40:59.442+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:00.669 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~95s 2022-09-03T07:41:04.444 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:04.442+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:04.563 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~157s 2022-09-03T07:41:06.274 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~101s 2022-09-03T07:41:09.444 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:09.443+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:10.172 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~163s 2022-09-03T07:41:11.880 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~107s 2022-09-03T07:41:14.445 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:14.443+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:15.779 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~168s 2022-09-03T07:41:17.487 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~112s 2022-09-03T07:41:19.445 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:19.444+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:21.386 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~174s 2022-09-03T07:41:23.092 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~118s 2022-09-03T07:41:24.446 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:24.444+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:26.992 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~179s 2022-09-03T07:41:28.700 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~123s 2022-09-03T07:41:29.446 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:29.445+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:32.598 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~185s 2022-09-03T07:41:34.307 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~129s 2022-09-03T07:41:34.447 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:34.445+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:38.207 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~191s 2022-09-03T07:41:39.448 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:39.447+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:39.916 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~135s 2022-09-03T07:41:43.813 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~196s 2022-09-03T07:41:44.449 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:44.447+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:45.523 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~140s 2022-09-03T07:41:49.420 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~202s 2022-09-03T07:41:49.449 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:49.448+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:51.130 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~146s 2022-09-03T07:41:54.450 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:54.448+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:41:55.026 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~207s 2022-09-03T07:41:56.737 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~151s 2022-09-03T07:41:59.450 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:41:59.448+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:00.633 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~213s 2022-09-03T07:42:02.344 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~157s 2022-09-03T07:42:04.450 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:04.449+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:06.239 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~219s 2022-09-03T07:42:07.949 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~163s 2022-09-03T07:42:09.451 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:09.450+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:11.844 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~224s 2022-09-03T07:42:13.555 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~168s 2022-09-03T07:42:14.451 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:14.450+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:17.451 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~230s 2022-09-03T07:42:19.162 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~174s 2022-09-03T07:42:19.452 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:19.450+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:23.057 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~236s 2022-09-03T07:42:24.453 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:24.451+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:24.768 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~179s 2022-09-03T07:42:28.665 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~241s 2022-09-03T07:42:29.453 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:29.451+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:30.375 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~185s 2022-09-03T07:42:34.271 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~247s 2022-09-03T07:42:34.453 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:34.452+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:35.981 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~191s 2022-09-03T07:42:39.454 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:39.452+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:39.877 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~252s 2022-09-03T07:42:41.585 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~196s 2022-09-03T07:42:44.454 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:44.452+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:45.485 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~258s 2022-09-03T07:42:47.194 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~202s 2022-09-03T07:42:49.455 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:49.453+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:51.091 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~264s 2022-09-03T07:42:52.801 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~207s 2022-09-03T07:42:54.455 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:54.453+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:42:56.698 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~269s 2022-09-03T07:42:58.407 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~213s 2022-09-03T07:42:59.456 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:42:59.454+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:43:02.304 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~275s 2022-09-03T07:43:04.014 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~219s 2022-09-03T07:43:04.456 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:04.454+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:43:07.911 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~280s 2022-09-03T07:43:09.456 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:09.455+0000 7fed24d42600 1 waiting to reach incremental sync.. 2022-09-03T07:43:09.621 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~224s 2022-09-03T07:43:13.516 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~286s 2022-09-03T07:43:14.457 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:14.455+0000 7fed24d42600 -1 bucket checkpoint timed out waiting to reach incremental sync 2022-09-03T07:43:14.458 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:14.455+0000 7fed24d42600 0 bucket sync checkpoint failed: (110) Connection timed out 2022-09-03T07:43:14.458 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:14.455+0000 7fed24d42600 -1 bucket sync checkpoint failed: (110) Connection timed out 2022-09-03T07:43:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 110 2022-09-03T07:43:14.482 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_resharded_bucket_full_sync_after_data_sync_init ... FAIL 2022-09-03T07:43:14.486 DEBUG:boto:path=/avysmw-78/ 2022-09-03T07:43:14.486 DEBUG:boto:auth_path=/avysmw-78/ 2022-09-03T07:43:14.488 DEBUG:boto:Method: PUT 2022-09-03T07:43:14.488 DEBUG:boto:Path: /avysmw-78/ 2022-09-03T07:43:14.489 DEBUG:boto:Data: 2022-09-03T07:43:14.489 DEBUG:boto:Headers: {} 2022-09-03T07:43:14.490 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T07:43:14.490 DEBUG:boto:Port: 8000 2022-09-03T07:43:14.491 DEBUG:boto:Params: {} 2022-09-03T07:43:14.492 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T07:43:14.494 DEBUG:boto:Token: None 2022-09-03T07:43:14.495 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 07:43:14 GMT /avysmw-78/ 2022-09-03T07:43:14.496 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:gjBQ/1KN7mamsGDZq+/AKWgv6Xk= 2022-09-03T07:43:14.497 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 07:43:14 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:gjBQ/1KN7mamsGDZq+/AKWgv6Xk=', 'Content-Length': '0'} 2022-09-03T07:43:14.529 DEBUG:boto:Response headers: [('x-amz-request-id', 'tx0000086b1c4e0e72461be-0063130592-1604-test-zone1'), ('Content-Length', '0'), ('Date', 'Sat, 03 Sep 2022 07:43:14 GMT'), ('Connection', 'Keep-Alive')] 2022-09-03T07:43:14.530 DEBUG:rgw_multi.tests:created bucket=avysmw-78 2022-09-03T07:43:14.530 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:14.531 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:14.532 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:14.587 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:14.588 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:14.638 INFO:teuthology.orchestra.run.smithi062.stderr:No --period given, using current period=2962e93e-ca58-4758-b6d9-6fe2e67c2976 2022-09-03T07:43:14.654 INFO:teuthology.orchestra.run.smithi062.stdout:[ 2022-09-03T07:43:14.655 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:43:14.655 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:14.656 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:31:57.780536Z" 2022-09-03T07:43:14.656 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:43:14.656 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:43:14.657 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "1_1662190994.526229_549.1", 2022-09-03T07:43:14.657 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "2022-09-03T07:43:14.526229Z" 2022-09-03T07:43:14.658 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:43:14.658 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:43:14.658 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:43:14.659 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:43:14.659 INFO:teuthology.orchestra.run.smithi062.stdout: }, 2022-09-03T07:43:14.659 INFO:teuthology.orchestra.run.smithi062.stdout: { 2022-09-03T07:43:14.659 INFO:teuthology.orchestra.run.smithi062.stdout: "marker": "", 2022-09-03T07:43:14.660 INFO:teuthology.orchestra.run.smithi062.stdout: "last_update": "0.000000" 2022-09-03T07:43:14.660 INFO:teuthology.orchestra.run.smithi062.stdout: } 2022-09-03T07:43:14.660 INFO:teuthology.orchestra.run.smithi062.stdout:] 2022-09-03T07:43:14.662 DEBUG:rgw_multi.tests:master meta markers={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:14.663 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2022-09-03T07:43:14.663 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:14.664 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:14.665 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:14.916 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:14.916 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:14.958 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:14.957+0000 7f276e7b9600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:14.959 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:14.958+0000 7f276e7b9600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:14.998 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:43:14.999 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:43:14.999 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:43:14.999 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:43:15.000 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:43:15.000 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:43:15.000 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:15.001 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:15.001 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:43:15.001 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:15.001 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:43:15.002 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:15.002 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:15.002 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:15.002 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:15.003 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:43:15.003 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:15.003 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:43:15.003 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:15.004 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:15.004 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:15.004 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:15.005 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:43:15.005 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:15.005 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:15.005 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:43:15.006 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:15.006 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:15.006 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:15.006 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:43:15.007 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:15.007 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:15.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:15.007 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:15.008 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:43:15.008 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:15.008 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:15.009 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:15.009 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:15.009 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:15.009 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:15.010 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:15.010 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:15.010 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:15.010 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:15.011 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:15.011 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:43:15.011 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:15.011 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:15.012 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:15.012 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:15.012 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:15.012 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:15.013 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:15.013 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:15.013 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:15.013 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:15.014 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:43:15.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:15.014 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:43:15.014 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:43:15.015 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:43:15.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:15.015 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:43:14Z" 2022-09-03T07:43:15.016 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:43:15.017 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:43:14Z" } 2022-09-03T07:43:15.018 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:43:15.018 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:15.019 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:43:15.019 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:43:15.227 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~230s 2022-09-03T07:43:19.122 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~292s 2022-09-03T07:43:20.021 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:20.023 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:20.024 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:20.082 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:20.083 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:20.130 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:20.129+0000 7fc0b3a45600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:20.131 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:20.130+0000 7fc0b3a45600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:20.176 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:43:20.176 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:43:20.177 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:43:20.177 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:43:20.177 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:43:20.177 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:43:20.178 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:20.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:20.178 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:43:20.178 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:20.179 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:43:20.179 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:20.179 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:20.179 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:20.180 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:20.180 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:43:20.180 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:20.180 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:43:20.181 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:20.181 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:20.181 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:20.181 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:20.182 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:43:20.182 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:20.182 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:20.182 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:43:20.183 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:20.183 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:20.183 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:20.183 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:43:20.184 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:20.184 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:20.184 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:20.184 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:20.185 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:43:20.185 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:20.185 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:20.185 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:20.185 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:20.186 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:20.186 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:20.186 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:20.186 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:20.187 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:20.187 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:20.187 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:20.187 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:43:20.188 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:20.188 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:20.188 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:20.188 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:20.189 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:20.189 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:20.189 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:20.189 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:20.190 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:20.190 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:20.190 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:43:20.190 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:20.191 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:43:20.191 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:43:20.191 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:43:20.191 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:20.191 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:43:20Z" 2022-09-03T07:43:20.192 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:43:20.193 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:43:20Z" } 2022-09-03T07:43:20.194 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:43:20.194 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:20.195 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:43:20.195 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:43:20.833 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~235s 2022-09-03T07:43:24.727 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~297s 2022-09-03T07:43:25.197 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:25.198 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:25.198 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:25.251 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:25.251 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:25.296 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:25.295+0000 7fb55bcd2600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:25.297 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:25.296+0000 7fb55bcd2600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:25.323 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:43:25.324 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:43:25.324 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:43:25.324 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:43:25.325 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:43:25.325 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:43:25.325 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:25.326 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:25.326 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:43:25.326 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:25.327 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:43:25.327 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:25.327 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:25.327 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:25.328 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:25.328 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:43:25.328 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:25.329 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:43:25.329 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:25.329 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:25.329 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:25.330 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:25.330 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:43:25.330 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:25.330 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:25.331 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:43:25.331 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:25.331 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:25.331 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:25.332 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:43:25.332 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:25.332 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:25.332 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:25.333 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:25.333 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:43:25.333 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:25.334 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:25.334 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:25.334 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:25.335 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:25.335 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:25.335 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:25.335 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:25.336 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:25.336 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:25.336 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:25.336 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:43:25.336 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:25.337 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:25.337 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:25.337 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:25.338 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:25.338 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:25.338 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:25.339 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:25.339 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:25.339 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:25.339 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:43:25.340 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:25.340 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:43:25.340 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:43:25.341 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:43:25.341 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:25.341 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:43:25Z" 2022-09-03T07:43:25.341 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:43:25.343 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:43:25Z" } 2022-09-03T07:43:25.343 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:43:25.344 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:25.345 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:43:25.345 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:43:26.438 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~241s 2022-09-03T07:43:30.333 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~303s 2022-09-03T07:43:30.333 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons 2022-09-03T07:43:30.346 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:30.347 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:30.348 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:30.407 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:30.407 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:30.441 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:30.440+0000 7febd5b86600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:30.442 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:30.441+0000 7febd5b86600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:30.455 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:43:30.456 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:43:30.456 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:43:30.456 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:43:30.456 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:43:30.457 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:43:30.457 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:30.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:30.458 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:43:30.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:30.458 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:43:30.459 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:30.459 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:30.459 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:30.460 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:30.460 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:43:30.460 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:30.460 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:43:30.461 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:30.462 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:30.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:30.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:30.463 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:43:30.463 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:30.464 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:30.464 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:43:30.464 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:30.464 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:30.465 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:30.465 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:43:30.466 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:30.466 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:30.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:30.467 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:30.468 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:43:30.468 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:30.469 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:30.469 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:30.470 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:30.470 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:30.471 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:30.471 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:30.471 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:30.472 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:30.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:30.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:30.473 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:43:30.474 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:30.474 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:30.475 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:30.475 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:30.476 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:30.476 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:30.476 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:30.477 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:30.478 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:30.478 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:30.479 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:43:30.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:30.480 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:43:30.480 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:43:30.481 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:43:30.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:30.482 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:43:30Z" 2022-09-03T07:43:30.482 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:43:30.485 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:43:30Z" } 2022-09-03T07:43:30.487 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:43:30.488 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:30.489 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:43:30.490 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:43:30.938 INFO:tasks.ceph.c1.osd.0:Sent signal 15 2022-09-03T07:43:30.941 INFO:tasks.ceph.c1.osd.1:Sent signal 15 2022-09-03T07:43:30.942 INFO:tasks.ceph.c1.osd.2:Sent signal 15 2022-09-03T07:43:30.942 INFO:tasks.ceph.c1.mon.a:Sent signal 15 2022-09-03T07:43:30.943 INFO:tasks.rgw.c1.client.0:Sent signal 15 2022-09-03T07:43:30.943 INFO:tasks.ceph.c1.mgr.x:Sent signal 15 2022-09-03T07:43:30.944 INFO:tasks.ceph.c1.osd.0.smithi062.stderr:2022-09-03T07:43:30.938+0000 7f453407e700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster c1 -i 0 (PID: 104138) UID: 0 2022-09-03T07:43:30.944 INFO:tasks.ceph.c1.osd.0.smithi062.stderr:2022-09-03T07:43:30.938+0000 7f453407e700 -1 osd.0 63 *** Got signal Terminated *** 2022-09-03T07:43:30.945 INFO:tasks.ceph.c1.osd.0.smithi062.stderr:2022-09-03T07:43:30.938+0000 7f453407e700 -1 osd.0 63 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-03T07:43:30.945 INFO:tasks.ceph.c1.osd.1.smithi062.stderr:2022-09-03T07:43:30.941+0000 7efcbe7be700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster c1 -i 1 (PID: 104145) UID: 0 2022-09-03T07:43:30.946 INFO:tasks.ceph.c1.osd.1.smithi062.stderr:2022-09-03T07:43:30.941+0000 7efcbe7be700 -1 osd.1 63 *** Got signal Terminated *** 2022-09-03T07:43:30.946 INFO:tasks.ceph.c1.osd.1.smithi062.stderr:2022-09-03T07:43:30.941+0000 7efcbe7be700 -1 osd.1 63 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-03T07:43:30.947 INFO:tasks.ceph.c1.osd.2.smithi062.stderr:2022-09-03T07:43:30.942+0000 7f77a4343700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster c1 -i 2 (PID: 104148) UID: 0 2022-09-03T07:43:30.947 INFO:tasks.ceph.c1.osd.2.smithi062.stderr:2022-09-03T07:43:30.942+0000 7f77a4343700 -1 osd.2 63 *** Got signal Terminated *** 2022-09-03T07:43:30.947 INFO:tasks.ceph.c1.osd.2.smithi062.stderr:2022-09-03T07:43:30.942+0000 7f77a4343700 -1 osd.2 63 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-03T07:43:30.948 INFO:tasks.ceph.c1.mon.a.smithi062.stderr:2022-09-03T07:43:30.942+0000 7f1ef839b700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-mon -f --cluster c1 -i a (PID: 103738) UID: 0 2022-09-03T07:43:30.948 INFO:tasks.ceph.c1.mon.a.smithi062.stderr:2022-09-03T07:43:30.942+0000 7f1ef839b700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2022-09-03T07:43:30.949 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:43:30.943+0000 7f8d2d234700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm (PID: 111679) UID: 0 2022-09-03T07:43:30.949 INFO:tasks.rgw.c1.client.0.smithi062.stdout:2022-09-03T07:43:30.943+0000 7f8d3149f600 -1 shutting down 2022-09-03T07:43:30.950 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T07:43:30.943+0000 7ff84bccd700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-mgr -f --cluster c1 -i x (PID: 103742) UID: 0 2022-09-03T07:43:30.950 INFO:tasks.ceph.c1.mgr.x.smithi062.stderr:2022-09-03T07:43:30.943+0000 7ff84bccd700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2022-09-03T07:43:32.045 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~247s 2022-09-03T07:43:35.491 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:35.492 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:35.493 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:35.553 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:35.554 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:35.609 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:35.608+0000 7ff9ee01f600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:35.610 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:35.609+0000 7ff9ee01f600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:35.624 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:43:35.625 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:43:35.625 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:43:35.625 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:43:35.626 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:43:35.627 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:43:35.627 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:35.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:35.628 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:43:35.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:35.629 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:43:35.629 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:35.630 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:35.630 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:35.631 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:35.631 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:43:35.631 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:35.632 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:43:35.632 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:35.633 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:35.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:35.634 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:35.634 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:43:35.635 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:35.635 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:35.635 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:43:35.635 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:35.636 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:35.636 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:35.636 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:43:35.637 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:35.637 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:35.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:35.638 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:35.638 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:43:35.639 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:35.640 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:35.640 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:35.641 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:35.641 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:35.641 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:35.642 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:35.642 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:35.643 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:35.643 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:35.644 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:35.644 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:43:35.644 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:35.645 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:35.645 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:35.645 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:35.645 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:35.646 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:35.646 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:35.647 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:35.647 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:35.647 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:35.648 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:43:35.648 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:35.649 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:43:35.649 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:43:35.650 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:43:35.650 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:35.650 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:43:35Z" 2022-09-03T07:43:35.651 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:43:35.653 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:43:35Z" } 2022-09-03T07:43:35.654 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:43:35.654 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:35.655 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:43:35.655 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:43:37.650 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~252s 2022-09-03T07:43:40.657 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:40.658 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:40.659 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:40.718 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:40.718 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:40.751 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:40.751+0000 7f51f9b26600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:40.752 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:40.752+0000 7f51f9b26600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:40.768 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:43:40.768 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:43:40.768 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:43:40.769 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:43:40.769 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:43:40.769 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:43:40.769 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:40.770 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:40.770 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:43:40.770 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:40.771 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:43:40.771 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:40.771 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:40.772 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:40.772 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:40.772 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:43:40.772 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:40.773 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:43:40.773 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:40.773 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:40.774 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:40.775 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:40.775 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:43:40.775 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:40.776 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:40.776 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:43:40.776 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:40.777 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:40.777 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:40.778 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:43:40.778 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:40.778 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:40.779 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:40.779 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:40.780 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:43:40.780 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:40.780 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:40.781 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:40.781 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:40.781 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:40.781 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:40.782 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:40.782 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:40.783 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:40.783 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:40.784 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:40.784 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:43:40.785 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:40.786 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:40.786 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:40.787 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:40.787 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:40.787 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:40.788 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:40.788 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:40.788 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:40.788 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:40.789 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:43:40.789 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:40.789 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:43:40.790 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:43:40.790 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:43:40.790 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:40.790 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:43:40Z" 2022-09-03T07:43:40.791 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:43:40.792 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:43:40Z" } 2022-09-03T07:43:40.793 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:43:40.793 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:40.794 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:43:40.795 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:43:43.255 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~258s 2022-09-03T07:43:45.797 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:45.798 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:45.799 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:45.857 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:45.857 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:45.873 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T07:43:45.865+0000 7fa97f5a5700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16) 2022-09-03T07:43:45.892 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:45.890+0000 7fa4c40b4600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:45.892 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:45.891+0000 7fa4c40b4600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:45.913 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:43:45.914 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:43:45.914 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:43:45.914 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:43:45.915 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:43:45.915 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:43:45.915 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:45.916 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:45.916 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:43:45.916 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:45.916 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:43:45.917 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:45.917 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:45.918 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:45.918 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:45.919 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:43:45.919 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:45.920 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:43:45.920 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:45.921 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:45.921 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:45.922 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:45.922 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:43:45.923 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:45.924 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:45.924 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:43:45.925 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:45.925 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:45.926 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:45.926 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:43:45.927 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:45.927 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:45.927 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:45.928 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:45.928 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:43:45.929 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:45.929 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:45.930 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:45.930 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:45.930 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:45.930 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:45.931 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:45.931 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:45.931 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:45.932 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:45.932 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:45.932 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:43:45.932 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:45.933 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:45.933 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:45.933 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:45.934 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:45.934 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:45.934 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:45.934 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:45.935 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:45.935 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:45.935 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:43:45.935 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:45.936 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:43:45.936 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:43:45.936 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:43:45.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:45.937 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:43:45Z" 2022-09-03T07:43:45.937 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:43:45.939 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:43:45Z" } 2022-09-03T07:43:45.939 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:43:45.940 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:45.940 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:43:45.941 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:43:46.297 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T07:43:46.296+0000 7fa97f5a5700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16) 2022-09-03T07:43:46.356 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T07:43:46.354+0000 7fa97f5a5700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16) 2022-09-03T07:43:46.597 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T07:43:46.594+0000 7fa97f5a5700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-5)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-11)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16)ERROR: collect() returned error (ret=-16) 2022-09-03T07:43:48.861 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~264s 2022-09-03T07:43:50.942 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:50.943 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:50.944 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:51.004 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:51.004 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:51.059 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:51.057+0000 7fcc84f39600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:51.060 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:51.058+0000 7fcc84f39600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:51.073 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:43:51.074 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:43:51.074 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:43:51.075 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:43:51.075 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:43:51.075 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:43:51.075 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:51.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:51.076 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:43:51.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:51.076 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:43:51.077 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:51.077 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:51.077 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:51.077 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:51.078 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:43:51.078 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:51.078 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:43:51.079 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:51.079 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:51.079 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:51.080 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:51.080 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:43:51.080 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:51.080 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:51.081 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:43:51.081 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:51.081 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:51.081 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:51.082 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:43:51.082 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:51.082 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:51.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:51.083 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:51.083 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:43:51.084 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:51.084 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:51.085 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:51.085 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:51.085 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:51.086 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:51.086 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:51.086 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:51.087 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:51.087 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:51.087 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:51.087 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:43:51.088 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:51.088 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:51.088 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:51.088 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:51.089 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:51.089 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:51.090 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:51.090 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:51.090 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:51.091 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:51.091 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:43:51.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:51.091 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:43:51.092 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:43:51.092 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:43:51.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:51.092 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:43:51Z" 2022-09-03T07:43:51.093 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:43:51.094 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:43:51Z" } 2022-09-03T07:43:51.095 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:43:51.095 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:51.096 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:43:51.096 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:43:54.467 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~269s 2022-09-03T07:43:56.098 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:56.099 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:43:56.100 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:43:56.155 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:43:56.155 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:43:56.189 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:56.186+0000 7fe22334c600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:56.189 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:43:56.187+0000 7fe22334c600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:43:56.202 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:43:56.203 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:43:56.203 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:43:56.203 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:43:56.204 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:43:56.204 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:43:56.204 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:56.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:56.205 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:43:56.205 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:56.205 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:43:56.205 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:56.206 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:56.206 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:43:56.206 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:56.206 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:43:56.207 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:56.207 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:43:56.207 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:56.207 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:56.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:56.208 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:56.208 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:43:56.209 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:56.209 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:56.209 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:43:56.209 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:56.210 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:56.210 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:56.210 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:43:56.211 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:43:56.211 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:56.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:56.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:56.212 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:43:56.212 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:56.212 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:56.212 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:56.213 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:56.213 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:56.213 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:56.213 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:56.214 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:56.214 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:56.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:56.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:43:56.215 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:43:56.215 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:43:56.216 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:43:56.216 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:43:56.216 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:43:56.216 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:43:56.217 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:43:56.217 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:43:56.217 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:43:56.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:56.218 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:43:56.218 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:43:56.218 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:56.218 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:43:56.219 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:43:56.219 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:43:56.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:43:56.220 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:43:56Z" 2022-09-03T07:43:56.220 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:43:56.221 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:43:56Z" } 2022-09-03T07:43:56.222 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:43:56.222 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:43:56.223 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:43:56.223 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:44:00.073 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~275s 2022-09-03T07:44:01.225 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:01.226 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:01.227 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:44:01.280 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:44:01.281 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:44:01.324 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:01.323+0000 7fc198d7e600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:01.326 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:01.324+0000 7fc198d7e600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:01.350 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:44:01.351 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:44:01.351 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:44:01.351 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:44:01.352 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:44:01.353 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:44:01.353 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:01.353 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:01.353 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:44:01.354 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:01.354 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:44:01.355 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:01.355 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:01.355 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:44:01.356 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:01.356 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:44:01.357 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:01.357 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:44:01.357 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:01.358 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:01.358 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:01.359 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:01.359 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:44:01.359 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:01.359 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:01.360 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:44:01.360 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:01.360 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:01.360 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:01.361 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:44:01.361 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:01.361 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:01.361 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:01.361 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:01.362 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:44:01.362 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:01.362 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:01.363 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:01.363 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:01.363 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:01.363 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:01.364 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:01.364 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:01.364 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:01.365 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:01.365 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:01.365 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:44:01.366 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:01.366 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:01.367 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:01.367 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:01.367 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:01.368 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:01.368 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:01.368 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:01.368 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:01.369 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:01.369 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:44:01.369 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:01.370 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:44:01.370 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:44:01.370 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:44:01.371 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:01.371 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:44:01Z" 2022-09-03T07:44:01.371 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:44:01.373 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:44:01Z" } 2022-09-03T07:44:01.373 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:44:01.374 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:44:01.375 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:44:01.376 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:44:05.679 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~280s 2022-09-03T07:44:06.378 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:06.379 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:06.379 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:44:06.436 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:44:06.437 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:44:06.467 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:06.466+0000 7ff8f8c95600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:06.468 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:06.467+0000 7ff8f8c95600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:06.485 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:44:06.485 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:44:06.486 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:44:06.486 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:44:06.486 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:44:06.487 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:44:06.487 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:06.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:06.488 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:44:06.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:06.488 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:44:06.489 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:06.489 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:06.490 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:44:06.490 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:06.490 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:44:06.491 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:06.491 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:44:06.491 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:06.492 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:06.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:06.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:06.493 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:44:06.494 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:06.494 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:06.495 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:44:06.495 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:06.495 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:06.496 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:06.496 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:44:06.496 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:06.497 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:06.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:06.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:06.497 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:44:06.498 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:06.498 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:06.498 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:06.499 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:06.499 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:06.499 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:06.499 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:06.500 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:06.500 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:06.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:06.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:06.501 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:44:06.501 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:06.501 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:06.502 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:06.502 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:06.502 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:06.503 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:06.503 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:06.503 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:06.504 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:06.504 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:06.504 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:44:06.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:06.505 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:44:06.505 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:44:06.505 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:44:06.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:06.506 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:44:06Z" 2022-09-03T07:44:06.506 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:44:06.508 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:44:06Z" } 2022-09-03T07:44:06.509 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:44:06.509 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:44:06.510 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:44:06.510 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:44:11.285 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~286s 2022-09-03T07:44:11.512 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:11.514 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:11.515 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:44:11.570 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:44:11.570 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:44:11.600 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:11.598+0000 7f1385b32600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:11.601 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:11.600+0000 7f1385b32600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:11.614 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:44:11.615 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:44:11.615 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:44:11.616 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:44:11.616 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:44:11.617 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:44:11.617 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:11.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:11.618 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:44:11.618 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:11.618 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:44:11.618 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:11.619 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:11.619 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:44:11.619 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:11.619 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:44:11.620 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:11.620 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:44:11.620 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:11.620 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:11.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:11.621 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:11.621 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:44:11.621 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:11.622 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:11.622 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:44:11.622 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:11.622 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:11.623 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:11.623 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:44:11.623 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:11.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:11.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:11.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:11.624 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:44:11.624 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:11.625 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:11.625 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:11.625 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:11.625 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:11.626 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:11.626 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:11.626 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:11.626 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:11.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:11.627 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:11.627 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:44:11.627 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:11.628 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:11.628 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:11.628 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:11.628 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:11.629 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:11.629 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:11.629 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:11.629 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:11.630 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:11.630 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:44:11.630 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:11.630 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:44:11.631 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:44:11.631 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:44:11.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:11.631 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:44:11Z" 2022-09-03T07:44:11.632 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:44:11.633 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:44:11Z" } 2022-09-03T07:44:11.634 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:44:11.634 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:44:11.635 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:44:11.635 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:44:16.636 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:16.637 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:16.638 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:44:16.694 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:44:16.695 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:44:16.727 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:16.726+0000 7ff4e085e600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:16.728 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:16.727+0000 7ff4e085e600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:16.742 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:44:16.743 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:44:16.743 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:44:16.743 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:44:16.744 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:44:16.744 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:44:16.744 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:16.744 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:16.745 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:44:16.745 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:16.745 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:44:16.745 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:16.746 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:16.746 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:44:16.746 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:16.746 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:44:16.747 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:16.747 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:44:16.747 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:16.747 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:16.748 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:16.748 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:16.748 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:44:16.748 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:16.749 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:16.749 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:44:16.750 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:16.750 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:16.751 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:16.751 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:44:16.751 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:16.751 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:16.752 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:16.752 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:16.752 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:44:16.752 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:16.753 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:16.753 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:16.753 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:16.753 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:16.754 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:16.754 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:16.754 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:16.754 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:16.755 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:16.755 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:16.755 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:44:16.755 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:16.756 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:16.756 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:16.756 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:16.756 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:16.756 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:16.757 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:16.757 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:16.758 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:16.758 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:16.758 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:44:16.758 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:16.759 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:44:16.759 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:44:16.759 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:44:16.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:16.760 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:44:16Z" 2022-09-03T07:44:16.761 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:44:16.762 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:44:16Z" } 2022-09-03T07:44:16.763 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:44:16.763 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:44:16.764 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:44:16.765 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:44:16.889 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~292s 2022-09-03T07:44:21.766 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:21.767 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:21.768 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:44:21.826 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:44:21.827 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:44:21.875 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:21.874+0000 7ff230930600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:21.876 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:21.875+0000 7ff230930600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:21.894 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:44:21.895 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:44:21.895 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:44:21.895 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:44:21.896 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:44:21.896 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:44:21.896 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:21.896 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:21.897 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:44:21.897 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:21.897 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:44:21.897 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:21.898 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:21.898 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:44:21.898 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:21.899 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:44:21.899 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:21.899 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:44:21.899 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:21.900 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:21.900 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:21.900 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:21.900 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:44:21.901 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:21.901 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:21.901 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:44:21.901 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:21.902 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:21.902 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:21.902 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:44:21.902 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:21.903 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:21.903 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:21.903 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:21.903 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:44:21.904 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:21.904 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:21.904 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:21.904 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:21.905 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:21.905 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:21.905 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:21.905 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:21.906 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:21.906 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:21.906 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:21.906 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:44:21.907 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:21.907 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:21.907 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:21.907 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:21.908 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:21.908 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:21.908 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:21.908 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:21.909 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:21.909 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:21.909 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:44:21.909 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:21.910 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:44:21.910 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:44:21.910 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:44:21.910 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:21.911 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:44:21Z" 2022-09-03T07:44:21.911 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:44:21.912 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:44:21Z" } 2022-09-03T07:44:21.913 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:44:21.914 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:44:21.914 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:44:21.915 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:44:22.517 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~297s 2022-09-03T07:44:26.916 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:26.917 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:26.918 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:44:26.981 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:44:26.982 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:44:27.027 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:27.025+0000 7fc28b571600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:27.028 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:44:27.026+0000 7fc28b571600 20 RGW-SYNC:meta: read sync status 2022-09-03T07:44:27.051 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2022-09-03T07:44:27.051 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": { 2022-09-03T07:44:27.052 INFO:teuthology.orchestra.run.smithi087.stdout: "info": { 2022-09-03T07:44:27.052 INFO:teuthology.orchestra.run.smithi087.stdout: "status": "sync", 2022-09-03T07:44:27.052 INFO:teuthology.orchestra.run.smithi087.stdout: "num_shards": 4, 2022-09-03T07:44:27.052 INFO:teuthology.orchestra.run.smithi087.stdout: "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", 2022-09-03T07:44:27.053 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:27.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:27.054 INFO:teuthology.orchestra.run.smithi087.stdout: "markers": [ 2022-09-03T07:44:27.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:27.054 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 0, 2022-09-03T07:44:27.055 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:27.055 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:27.055 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190317.780536_375.1", 2022-09-03T07:44:27.056 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:27.056 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 1, 2022-09-03T07:44:27.056 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:27.056 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:31:57.780536Z", 2022-09-03T07:44:27.057 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:27.057 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:27.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:27.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:27.058 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 1, 2022-09-03T07:44:27.058 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:27.059 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:27.059 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "1_1662190696.611944_539.1", 2022-09-03T07:44:27.059 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:27.060 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:27.060 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:27.060 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "2022-09-03T07:38:16.611944Z", 2022-09-03T07:44:27.060 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 4 2022-09-03T07:44:27.061 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:27.061 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:27.061 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:27.061 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 2, 2022-09-03T07:44:27.062 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:27.062 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:27.062 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:27.063 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:27.063 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:27.063 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:27.064 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:27.064 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:27.064 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:27.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:27.065 INFO:teuthology.orchestra.run.smithi087.stdout: { 2022-09-03T07:44:27.065 INFO:teuthology.orchestra.run.smithi087.stdout: "key": 3, 2022-09-03T07:44:27.065 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2022-09-03T07:44:27.066 INFO:teuthology.orchestra.run.smithi087.stdout: "state": 1, 2022-09-03T07:44:27.066 INFO:teuthology.orchestra.run.smithi087.stdout: "marker": "", 2022-09-03T07:44:27.066 INFO:teuthology.orchestra.run.smithi087.stdout: "next_step_marker": "", 2022-09-03T07:44:27.067 INFO:teuthology.orchestra.run.smithi087.stdout: "total_entries": 0, 2022-09-03T07:44:27.067 INFO:teuthology.orchestra.run.smithi087.stdout: "pos": 0, 2022-09-03T07:44:27.067 INFO:teuthology.orchestra.run.smithi087.stdout: "timestamp": "0.000000", 2022-09-03T07:44:27.067 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2022-09-03T07:44:27.068 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:27.068 INFO:teuthology.orchestra.run.smithi087.stdout: } 2022-09-03T07:44:27.068 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2022-09-03T07:44:27.068 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:27.069 INFO:teuthology.orchestra.run.smithi087.stdout: "full_sync": { 2022-09-03T07:44:27.069 INFO:teuthology.orchestra.run.smithi087.stdout: "total": 1, 2022-09-03T07:44:27.070 INFO:teuthology.orchestra.run.smithi087.stdout: "complete": 1 2022-09-03T07:44:27.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2022-09-03T07:44:27.070 INFO:teuthology.orchestra.run.smithi087.stdout: "current_time": "2022-09-03T07:44:27Z" 2022-09-03T07:44:27.070 INFO:teuthology.orchestra.run.smithi087.stdout:} 2022-09-03T07:44:27.072 DEBUG:rgw_multi.tests:current meta sync status={ "sync_status": { "info": { "status": "sync", "num_shards": 4, "period": "2962e93e-ca58-4758-b6d9-6fe2e67c2976", "realm_epoch": 4 }, "markers": [ { "key": 0, "val": { "state": 1, "marker": "1_1662190317.780536_375.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "2022-09-03T07:31:57.780536Z", "realm_epoch": 4 } }, { "key": 1, "val": { "state": 1, "marker": "1_1662190696.611944_539.1", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "2022-09-03T07:38:16.611944Z", "realm_epoch": 4 } }, { "key": 2, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } }, { "key": 3, "val": { "state": 1, "marker": "", "next_step_marker": "", "total_entries": 0, "pos": 0, "timestamp": "0.000000", "realm_epoch": 2 } } ] }, "full_sync": { "total": 1, "complete": 1 }, "current_time": "2022-09-03T07:44:27Z" } 2022-09-03T07:44:27.072 DEBUG:rgw_multi.tests:sync_markers=[{'key': 0, 'val': {'state': 1, 'marker': '1_1662190317.780536_375.1', 'next_step_marker': '', 'total_entries': 1, 'pos': 0, 'timestamp': '2022-09-03T07:31:57.780536Z', 'realm_epoch': 4}}, {'key': 1, 'val': {'state': 1, 'marker': '1_1662190696.611944_539.1', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '2022-09-03T07:38:16.611944Z', 'realm_epoch': 4}}, {'key': 2, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}, {'key': 3, 'val': {'state': 1, 'marker': '', 'next_step_marker': '', 'total_entries': 0, 'pos': 0, 'timestamp': '0.000000', 'realm_epoch': 2}}] 2022-09-03T07:44:27.073 DEBUG:rgw_multi.tests:log_status={0: '1_1662190317.780536_375.1', 1: '1_1662190994.526229_549.1', 2: '', 3: ''} 2022-09-03T07:44:27.074 DEBUG:rgw_multi.tests:sync_status={0: '1_1662190317.780536_375.1', 1: '1_1662190696.611944_539.1', 2: '', 3: ''} 2022-09-03T07:44:27.074 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=1 master=1_1662190994.526229_549.1 target=1_1662190696.611944_539.1 2022-09-03T07:44:28.124 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~303s 2022-09-03T07:44:28.125 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons 2022-09-03T07:44:28.729 INFO:tasks.ceph.c2.osd.0:Sent signal 15 2022-09-03T07:44:28.730 INFO:tasks.ceph.c2.osd.1:Sent signal 15 2022-09-03T07:44:28.731 INFO:tasks.ceph.c2.osd.2:Sent signal 15 2022-09-03T07:44:28.731 INFO:tasks.ceph.c2.mon.a:Sent signal 15 2022-09-03T07:44:28.732 INFO:tasks.rgw.c2.client.1:Sent signal 15 2022-09-03T07:44:28.732 INFO:tasks.ceph.c2.mgr.x:Sent signal 15 2022-09-03T07:44:28.734 INFO:tasks.ceph.c2.osd.0.smithi087.stderr:2022-09-03T07:44:28.729+0000 7f5b31d9a700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster c2 -i 0 (PID: 104162) UID: 0 2022-09-03T07:44:28.734 INFO:tasks.ceph.c2.osd.0.smithi087.stderr:2022-09-03T07:44:28.729+0000 7f5b31d9a700 -1 osd.0 67 *** Got signal Terminated *** 2022-09-03T07:44:28.735 INFO:tasks.ceph.c2.osd.0.smithi087.stderr:2022-09-03T07:44:28.729+0000 7f5b31d9a700 -1 osd.0 67 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-03T07:44:28.736 INFO:tasks.ceph.c2.osd.1.smithi087.stderr:2022-09-03T07:44:28.730+0000 7f54cba35700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster c2 -i 1 (PID: 104173) UID: 0 2022-09-03T07:44:28.737 INFO:tasks.ceph.c2.osd.1.smithi087.stderr:2022-09-03T07:44:28.730+0000 7f54cba35700 -1 osd.1 67 *** Got signal Terminated *** 2022-09-03T07:44:28.737 INFO:tasks.ceph.c2.osd.1.smithi087.stderr:2022-09-03T07:44:28.730+0000 7f54cba35700 -1 osd.1 67 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-03T07:44:28.738 INFO:tasks.ceph.c2.osd.2.smithi087.stderr:2022-09-03T07:44:28.730+0000 7f64aa120700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster c2 -i 2 (PID: 104177) UID: 0 2022-09-03T07:44:28.739 INFO:tasks.ceph.c2.osd.2.smithi087.stderr:2022-09-03T07:44:28.730+0000 7f64aa120700 -1 osd.2 67 *** Got signal Terminated *** 2022-09-03T07:44:28.739 INFO:tasks.ceph.c2.osd.2.smithi087.stderr:2022-09-03T07:44:28.730+0000 7f64aa120700 -1 osd.2 67 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-09-03T07:44:28.740 INFO:tasks.ceph.c2.mon.a.smithi087.stderr:2022-09-03T07:44:28.731+0000 7f578d257700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-mon -f --cluster c2 -i a (PID: 103766) UID: 0 2022-09-03T07:44:28.741 INFO:tasks.ceph.c2.mon.a.smithi087.stderr:2022-09-03T07:44:28.731+0000 7f578d257700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2022-09-03T07:44:28.742 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T07:44:28.731+0000 7fa99cde0700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm (PID: 112887) UID: 0 2022-09-03T07:44:28.743 INFO:tasks.rgw.c2.client.1.smithi087.stdout:2022-09-03T07:44:28.731+0000 7fa9a104b600 -1 shutting down 2022-09-03T07:44:28.743 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T07:44:28.732+0000 7f075f40c700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-mgr -f --cluster c2 -i x (PID: 103770) UID: 0 2022-09-03T07:44:28.809 INFO:tasks.ceph.c2.mgr.x.smithi087.stderr:2022-09-03T07:44:28.732+0000 7f075f40c700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2022-09-03T07:44:32.075 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:32.076 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T07:44:32.077 DEBUG:teuthology.orchestra.run.smithi087:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T07:44:32.134 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setuser ceph since I am not root 2022-09-03T07:44:32.135 INFO:teuthology.orchestra.run.smithi087.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T07:49:32.145 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:49:32.134+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T07:54:32.139 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:54:32.135+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T07:59:32.147 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T07:59:32.135+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:04:32.145 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T08:04:32.136+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:09:32.142 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T08:09:32.137+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:14:32.143 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T08:14:32.138+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:19:32.142 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T08:19:32.138+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:24:32.145 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T08:24:32.138+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:29:32.143 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T08:29:32.139+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:34:32.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T08:34:32.150 INFO:teuthology.orchestra.run.smithi087.stderr:2022-09-03T08:34:32.140+0000 7f6af48ea600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:34:32.151 INFO:teuthology.orchestra.run.smithi087.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T08:34:32.156 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_incremental_sync_after_data_sync_init ... FAIL 2022-09-03T08:34:32.158 DEBUG:boto:path=/avysmw-79/ 2022-09-03T08:34:32.159 DEBUG:boto:auth_path=/avysmw-79/ 2022-09-03T08:34:32.160 DEBUG:boto:Method: PUT 2022-09-03T08:34:32.160 DEBUG:boto:Path: /avysmw-79/ 2022-09-03T08:34:32.161 DEBUG:boto:Data: 2022-09-03T08:34:32.161 DEBUG:boto:Headers: {} 2022-09-03T08:34:32.162 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T08:34:32.162 DEBUG:boto:Port: 8000 2022-09-03T08:34:32.163 DEBUG:boto:Params: {} 2022-09-03T08:34:32.164 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:34:32.165 DEBUG:boto:Token: None 2022-09-03T08:34:32.166 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:34:32 GMT /avysmw-79/ 2022-09-03T08:34:32.166 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw= 2022-09-03T08:34:32.167 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:34:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw=', 'Content-Length': '0'} 2022-09-03T08:34:32.170 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:34:32.177 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:34:32.478 DEBUG:boto:Token: None 2022-09-03T08:34:32.479 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:34:32 GMT /avysmw-79/ 2022-09-03T08:34:32.480 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw= 2022-09-03T08:34:32.480 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:34:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw=', 'Content-Length': '0'} 2022-09-03T08:34:32.482 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:34:32.483 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:34:33.739 DEBUG:boto:Token: None 2022-09-03T08:34:33.739 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:34:32 GMT /avysmw-79/ 2022-09-03T08:34:33.740 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw= 2022-09-03T08:34:33.741 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:34:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw=', 'Content-Length': '0'} 2022-09-03T08:34:33.742 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:34:33.743 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:34:35.013 DEBUG:boto:Token: None 2022-09-03T08:34:35.014 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:34:32 GMT /avysmw-79/ 2022-09-03T08:34:35.015 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw= 2022-09-03T08:34:35.016 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:34:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw=', 'Content-Length': '0'} 2022-09-03T08:34:35.017 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:34:35.018 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:34:40.536 DEBUG:boto:Token: None 2022-09-03T08:34:40.537 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:34:32 GMT /avysmw-79/ 2022-09-03T08:34:40.537 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw= 2022-09-03T08:34:40.538 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:34:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw=', 'Content-Length': '0'} 2022-09-03T08:34:40.540 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:34:40.540 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:34:53.855 DEBUG:boto:Token: None 2022-09-03T08:34:53.857 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:34:32 GMT /avysmw-79/ 2022-09-03T08:34:53.858 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw= 2022-09-03T08:34:53.859 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:34:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw=', 'Content-Length': '0'} 2022-09-03T08:34:53.861 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:34:53.861 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:35:22.455 DEBUG:boto:Token: None 2022-09-03T08:35:22.457 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:34:32 GMT /avysmw-79/ 2022-09-03T08:35:22.458 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw= 2022-09-03T08:35:22.459 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:34:32 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:XkJeht+Yhyw8VPwNyMQYzCLSfmw=', 'Content-Length': '0'} 2022-09-03T08:35:22.461 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:35:22.461 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:36:11.995 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_resharded_bucket_incremental_sync_latest_after_data_sync_init ... ERROR 2022-09-03T08:36:11.996 DEBUG:boto:path=/avysmw-80/ 2022-09-03T08:36:11.997 DEBUG:boto:auth_path=/avysmw-80/ 2022-09-03T08:36:11.997 DEBUG:boto:Method: PUT 2022-09-03T08:36:11.998 DEBUG:boto:Path: /avysmw-80/ 2022-09-03T08:36:11.998 DEBUG:boto:Data: 2022-09-03T08:36:11.999 DEBUG:boto:Headers: {} 2022-09-03T08:36:11.999 DEBUG:boto:Host: smithi062.front.sepia.ceph.com:8000 2022-09-03T08:36:12.000 DEBUG:boto:Port: 8000 2022-09-03T08:36:12.000 DEBUG:boto:Params: {} 2022-09-03T08:36:12.001 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:36:12.001 DEBUG:boto:Token: None 2022-09-03T08:36:12.002 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:36:12 GMT /avysmw-80/ 2022-09-03T08:36:12.003 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ= 2022-09-03T08:36:12.003 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:36:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ=', 'Content-Length': '0'} 2022-09-03T08:36:12.005 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:36:12.006 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:36:12.404 DEBUG:boto:Token: None 2022-09-03T08:36:12.405 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:36:12 GMT /avysmw-80/ 2022-09-03T08:36:12.405 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ= 2022-09-03T08:36:12.406 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:36:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ=', 'Content-Length': '0'} 2022-09-03T08:36:12.408 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:36:12.408 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:36:14.378 DEBUG:boto:Token: None 2022-09-03T08:36:14.379 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:36:12 GMT /avysmw-80/ 2022-09-03T08:36:14.380 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ= 2022-09-03T08:36:14.380 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:36:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ=', 'Content-Length': '0'} 2022-09-03T08:36:14.382 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:36:14.383 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:36:17.689 DEBUG:boto:Token: None 2022-09-03T08:36:17.690 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:36:12 GMT /avysmw-80/ 2022-09-03T08:36:17.690 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ= 2022-09-03T08:36:17.691 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:36:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ=', 'Content-Length': '0'} 2022-09-03T08:36:17.693 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:36:17.693 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:36:18.555 DEBUG:boto:Token: None 2022-09-03T08:36:18.555 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:36:12 GMT /avysmw-80/ 2022-09-03T08:36:18.556 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ= 2022-09-03T08:36:18.556 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:36:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ=', 'Content-Length': '0'} 2022-09-03T08:36:18.558 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:36:18.559 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:36:22.870 DEBUG:boto:Token: None 2022-09-03T08:36:22.871 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:36:12 GMT /avysmw-80/ 2022-09-03T08:36:22.872 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ= 2022-09-03T08:36:22.872 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:36:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ=', 'Content-Length': '0'} 2022-09-03T08:36:22.874 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:36:22.875 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:36:53.246 DEBUG:boto:Token: None 2022-09-03T08:36:53.247 DEBUG:boto:StringToSign: PUT Sat, 03 Sep 2022 08:36:12 GMT /avysmw-80/ 2022-09-03T08:36:53.248 DEBUG:boto:Signature: AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ= 2022-09-03T08:36:53.249 DEBUG:boto:Final headers: {'User-Agent': 'Boto/2.49.0 Python/3.6.9 Linux/5.4.0-113-generic', 'Date': 'Sat, 03 Sep 2022 08:36:12 GMT', 'Authorization': 'AWS 1KKLJ23JMA7EOB7UYR06:Kjjf97N1BC1yWiyoTqrEP2t3+xQ=', 'Content-Length': '0'} 2022-09-03T08:36:53.253 DEBUG:boto:encountered ConnectionRefusedError exception, reconnecting 2022-09-03T08:36:53.254 DEBUG:boto:establishing HTTP connection: kwargs={'timeout': 70, 'port': 8000} 2022-09-03T08:37:21.980 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_resharded_bucket_incremental_sync_oldest_after_data_sync_init ... ERROR 2022-09-03T08:37:21.981 INFO:tasks.rgw_multisite_tests:log information for manual testing ... ok 2022-09-03T08:37:21.983 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T08:37:21.989 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T08:37:21.990 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T08:37:22.049 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T08:37:22.050 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T08:42:22.053 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T08:42:22.049+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:47:22.059 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T08:47:22.050+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:52:22.055 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T08:52:22.051+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T08:57:22.055 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T08:57:22.051+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:02:22.056 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:02:22.050+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:07:22.056 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:07:22.052+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:12:22.056 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:12:22.052+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:17:22.058 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:17:22.053+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:22:22.058 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:22:22.052+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:27:22.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T09:27:22.078 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:27:22.054+0000 7f897dc68600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:27:22.079 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T09:27:22.083 INFO:tasks.rgw_multisite_tests:test low level implementation of s3 notifications ... FAIL 2022-09-03T09:27:22.086 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T09:27:22.088 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T09:27:22.089 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T09:27:22.146 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T09:27:22.146 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T09:32:22.149 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:32:22.145+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:37:22.152 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:37:22.146+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:42:22.150 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:42:22.146+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:47:22.150 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:47:22.145+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:52:22.151 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:52:22.146+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T09:57:22.164 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T09:57:22.147+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:02:22.152 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:02:22.147+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:07:22.153 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:07:22.148+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:12:22.154 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:12:22.150+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:17:22.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T10:17:22.201 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:17:22.150+0000 7f6959fe5600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:17:22.202 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T10:17:22.205 INFO:tasks.rgw_multisite_tests:test s3 records fetching ... FAIL 2022-09-03T10:17:22.207 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T10:17:22.208 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T10:17:22.209 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T10:17:22.266 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T10:17:22.267 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T10:22:22.271 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:22:22.267+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:27:22.273 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:27:22.266+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:32:22.271 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:32:22.267+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:37:22.273 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:37:22.268+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:42:22.274 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:42:22.269+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:47:22.273 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:47:22.269+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:52:22.273 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:52:22.269+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T10:57:22.276 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T10:57:22.269+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:02:22.276 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:02:22.269+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:07:22.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T11:07:22.281 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:07:22.270+0000 7f00c52b1600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:07:22.282 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T11:07:22.284 INFO:tasks.rgw_multisite_tests:test s3 notification set/get/delete ... FAIL 2022-09-03T11:07:22.286 INFO:tasks.rgw_multisite_tests:test s3 notification filter on master ... ok 2022-09-03T11:07:22.287 INFO:tasks.rgw_multisite_tests:rgw_multi.tests_ps.test_object_timing ... ok 2022-09-03T11:07:22.287 INFO:tasks.rgw_multisite_tests:test that opaque id set in topic, is sent in notification ... ok 2022-09-03T11:07:22.289 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T11:07:22.289 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T11:07:22.291 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T11:07:22.350 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T11:07:22.350 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T11:12:22.354 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:12:22.349+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:17:22.355 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:17:22.350+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:22:22.356 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:22:22.351+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:27:22.356 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:27:22.352+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:32:22.358 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:32:22.352+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:37:22.358 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:37:22.351+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:42:22.358 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:42:22.352+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:47:22.359 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:47:22.353+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:52:22.358 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:52:22.354+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:57:22.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T11:57:22.368 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T11:57:22.354+0000 7f9cf6369600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T11:57:22.369 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T11:57:22.372 INFO:tasks.rgw_multisite_tests:test set/get/delete of topic ... FAIL 2022-09-03T11:57:22.377 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T11:57:22.379 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T11:57:22.381 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T11:57:22.439 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T11:57:22.441 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T12:02:22.443 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:02:22.438+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:07:22.444 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:07:22.439+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:12:22.444 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:12:22.440+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:17:22.445 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:17:22.441+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:22:22.445 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:22:22.442+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:27:22.446 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:27:22.442+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:32:22.447 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:32:22.441+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:37:22.447 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:37:22.442+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:42:22.448 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:42:22.442+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:47:22.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T12:47:22.474 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:47:22.442+0000 7f01b1780600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:47:22.475 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T12:47:22.478 INFO:tasks.rgw_multisite_tests:test set topic with endpoint ... FAIL 2022-09-03T12:47:22.481 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T12:47:22.482 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T12:47:22.484 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T12:47:22.545 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T12:47:22.545 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T12:52:22.549 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:52:22.544+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T12:57:22.549 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T12:57:22.545+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:02:22.550 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:02:22.545+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:07:22.551 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:07:22.546+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:12:22.550 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:12:22.546+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:17:22.551 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:17:22.547+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:22:22.551 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:22:22.546+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:27:22.552 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:27:22.546+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:32:22.552 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:32:22.548+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:37:22.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T13:37:22.558 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:37:22.548+0000 7f6976d79600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:37:22.558 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T13:37:22.561 INFO:tasks.rgw_multisite_tests:test set/get/delete of notification ... FAIL 2022-09-03T13:37:22.563 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T13:37:22.564 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T13:37:22.565 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T13:37:22.621 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T13:37:22.621 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T13:42:22.625 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:42:22.621+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:47:22.625 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:47:22.621+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:52:22.626 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:52:22.622+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T13:57:22.626 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T13:57:22.623+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:02:22.627 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:02:22.623+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:07:22.628 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:07:22.623+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:12:22.630 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:12:22.625+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:17:22.629 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:17:22.625+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:22:22.629 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:22:22.625+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:27:22.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T14:27:22.635 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:27:22.625+0000 7f38a0f27600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:27:22.636 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T14:27:22.638 INFO:tasks.rgw_multisite_tests:test set/get/delete of notification on specific events ... FAIL 2022-09-03T14:27:22.640 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T14:27:22.641 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T14:27:22.642 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T14:27:22.700 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T14:27:22.701 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T14:32:22.704 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:32:22.701+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:37:22.705 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:37:22.701+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:42:22.706 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:42:22.702+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:47:22.706 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:47:22.702+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:52:22.707 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:52:22.703+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T14:57:22.707 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T14:57:22.702+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:02:22.707 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:02:22.702+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:07:22.708 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:07:22.703+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:12:22.709 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:12:22.704+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:17:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T15:17:22.713 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:17:22.704+0000 7fbfce9ca600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:17:22.714 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T15:17:22.716 INFO:tasks.rgw_multisite_tests:test set/get/delete of subscription ... FAIL 2022-09-03T15:17:22.718 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T15:17:22.719 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T15:17:22.720 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T15:17:22.779 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T15:17:22.780 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T15:22:22.783 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:22:22.779+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:27:22.784 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:27:22.780+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:32:22.784 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:32:22.780+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:37:22.785 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:37:22.781+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:42:22.785 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:42:22.782+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:47:22.791 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:47:22.782+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:52:22.796 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:52:22.782+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T15:57:22.799 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T15:57:22.783+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:02:22.802 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:02:22.783+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:07:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T16:07:22.810 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:07:22.784+0000 7f1410504600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:07:22.810 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T16:07:22.813 INFO:tasks.rgw_multisite_tests:test radosgw-admin commands ... FAIL 2022-09-03T16:07:22.815 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T16:07:22.815 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T16:07:22.817 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T16:07:22.876 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T16:07:22.877 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T16:12:22.882 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:12:22.860+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:17:22.884 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:17:22.861+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:22:22.885 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:22:22.862+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:27:22.887 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:27:22.863+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:32:22.889 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:32:22.863+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:37:22.890 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:37:22.864+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:42:22.892 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:42:22.864+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:47:22.893 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:47:22.865+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:52:22.894 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:52:22.865+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:57:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-09-03T16:57:22.892 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T16:57:22.864+0000 7ffb2fe75600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T16:57:22.892 INFO:teuthology.orchestra.run.smithi062.stderr:failed to fetch mon config (--no-mon-config to skip) 2022-09-03T16:57:22.894 INFO:tasks.rgw_multisite_tests:test that events are only sent on incremental sync ... FAIL 2022-09-03T16:57:22.896 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T16:57:22.897 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2022-09-03T16:57:22.898 DEBUG:teuthology.orchestra.run.smithi062:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2022-09-03T16:57:22.953 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setuser ceph since I am not root 2022-09-03T16:57:22.953 INFO:teuthology.orchestra.run.smithi062.stderr:ignoring --setgroup ceph since I am not root 2022-09-03T17:02:22.952 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T17:02:22.935+0000 7ff225702600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T17:07:22.950 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T17:07:22.936+0000 7ff225702600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T17:12:22.947 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T17:12:22.937+0000 7ff225702600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T17:17:22.947 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T17:17:22.937+0000 7ff225702600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T17:22:22.946 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T17:22:22.938+0000 7ff225702600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T17:27:22.945 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T17:27:22.938+0000 7ff225702600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T17:32:22.943 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T17:32:22.938+0000 7ff225702600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T17:37:22.942 INFO:teuthology.orchestra.run.smithi062.stderr:2022-09-03T17:37:22.939+0000 7ff225702600 0 monclient(hunting): authenticate timed out after 300 2022-09-03T17:38:16.915 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2022-09-03T17:38:16.918 DEBUG:teuthology.task.console_log:Killing console logger for smithi062 2022-09-03T17:38:16.919 DEBUG:teuthology.task.console_log:Killing console logger for smithi087 2022-09-03T17:38:16.920 DEBUG:teuthology.exit:Finished running handlers